2024-03-06T07:10:31.479 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-06T07:10:31.479 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi/7583072 --description fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/strays} --name leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi --owner scheduled_leonidus@teuthology --verbose -- /home/teuthworker/archive/leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi/7583072/orig.config.yaml 2024-03-06T07:10:31.525 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-06T07:10:31.609 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi/7583072 branch: wip-lusov-qdb description: fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/strays} email: null first_in_suite: false job_id: '7583072' kernel: client: branch: testing kdb: true sha1: distro ktype: upstream last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi name: leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-lusov-qdb ceph: conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 pid file: '' rados mon op timeout: 900 rados osd op timeout: 900 global: ms die on skipped message: false mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon op complaint time: 120 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - MDS_UP_LESS_THAN_MAX - POOL_APP_NOT_ENABLED - overall HEALTH_ - Replacing daemon - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd kclient: syntax: v2 subvolume_version: 2 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-lusov-qdb sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd owner: scheduled_leonidus@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.y - mds.a - mds.c - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - client.2 - - mon.b - mon.c - mgr.x - mds.b - mds.d - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - client.3 seed: 5072 sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd sleep_before_teardown: 0 subset: 246/10000 suite: fs suite_branch: wip-lusov-qdb suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi086.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPVCtziu+Te91xV2xttKKhwBOQSLhAoybWR/BQVXCRhKuBLqgWwARAjP8hbs3gmer5xVWdViejcSfyxOS0CxelA= smithi194.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBELQGeOZN5IijIK+pDDgpWgRV+ELeGVFlFYEd3OMij6eM91I0YhemVkUUa2z/vNicbLP/3SXmZbxbCY88l761aQ= tasks: - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - ceph: create_rbd_pool: false - kclient: null - cephfs_test_runner: modules: - tasks.cephfs.test_strays teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return yaml.ktype == 'upstream' end - local function is_kdistro() return yaml.ktype == 'distro' end - local function is_fuse() return false end - local function syntax_version() return yaml.overrides.kclient.syntax end teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-06_07:04:29 tube: smithi user: leonidus verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-06T07:10:31.610 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa; will attempt to use it 2024-03-06T07:10:31.610 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks 2024-03-06T07:10:31.610 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-06T07:10:31.612 INFO:teuthology.task.internal:Checking packages... 2024-03-06T07:10:31.636 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd' 2024-03-06T07:10:31.636 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-06T07:10:31.636 INFO:teuthology.packaging:ref: None 2024-03-06T07:10:31.636 INFO:teuthology.packaging:tag: None 2024-03-06T07:10:31.636 INFO:teuthology.packaging:branch: wip-lusov-qdb 2024-03-06T07:10:31.636 INFO:teuthology.packaging:sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-06T07:10:31.637 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-lusov-qdb 2024-03-06T07:10:31.948 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1739.g1bc250c7 2024-03-06T07:10:31.949 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-06T07:10:31.955 INFO:teuthology.task.internal:no buildpackages task found 2024-03-06T07:10:31.956 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-06T07:10:31.990 INFO:teuthology.task.internal:Saving configuration 2024-03-06T07:10:32.007 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-06T07:10:32.015 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-06T07:10:32.037 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi086.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi/7583072', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-06 07:04:45.095172', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPVCtziu+Te91xV2xttKKhwBOQSLhAoybWR/BQVXCRhKuBLqgWwARAjP8hbs3gmer5xVWdViejcSfyxOS0CxelA='} 2024-03-06T07:10:32.062 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi194.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi/7583072', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-06 07:04:45.096515', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBELQGeOZN5IijIK+pDDgpWgRV+ELeGVFlFYEd3OMij6eM91I0YhemVkUUa2z/vNicbLP/3SXmZbxbCY88l761aQ='} 2024-03-06T07:10:32.062 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-06T07:10:32.068 INFO:teuthology.task.internal:roles: ubuntu@smithi086.front.sepia.ceph.com - ['mon.a', 'mgr.y', 'mds.a', 'mds.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'client.2'] 2024-03-06T07:10:32.068 INFO:teuthology.task.internal:roles: ubuntu@smithi194.front.sepia.ceph.com - ['mon.b', 'mon.c', 'mgr.x', 'mds.b', 'mds.d', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'client.3'] 2024-03-06T07:10:32.068 INFO:teuthology.run_tasks:Running task console_log... 2024-03-06T07:10:32.148 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8519c75f70>, signals=[15]) 2024-03-06T07:10:32.149 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-06T07:10:32.154 INFO:teuthology.task.internal:Opening connections... 2024-03-06T07:10:32.154 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:10:32.156 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:10:32.245 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:10:32.246 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:10:32.335 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-06T07:10:32.342 DEBUG:teuthology.orchestra.run.smithi086:> uname -m 2024-03-06T07:10:32.358 INFO:teuthology.orchestra.run.smithi086.stdout:x86_64 2024-03-06T07:10:32.358 DEBUG:teuthology.orchestra.run.smithi086:> cat /etc/os-release 2024-03-06T07:10:32.414 INFO:teuthology.orchestra.run.smithi086.stdout:NAME="CentOS Stream" 2024-03-06T07:10:32.414 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION="9" 2024-03-06T07:10:32.414 INFO:teuthology.orchestra.run.smithi086.stdout:ID="centos" 2024-03-06T07:10:32.414 INFO:teuthology.orchestra.run.smithi086.stdout:ID_LIKE="rhel fedora" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION_ID="9" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:PLATFORM_ID="platform:el9" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:ANSI_COLOR="0;31" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:LOGO="fedora-logo-icon" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:HOME_URL="https://centos.org/" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-06T07:10:32.415 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-06T07:10:32.416 INFO:teuthology.lock.ops:Updating smithi086.front.sepia.ceph.com on lock server 2024-03-06T07:10:32.433 DEBUG:teuthology.orchestra.run.smithi194:> uname -m 2024-03-06T07:10:32.450 INFO:teuthology.orchestra.run.smithi194.stdout:x86_64 2024-03-06T07:10:32.450 DEBUG:teuthology.orchestra.run.smithi194:> cat /etc/os-release 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:NAME="CentOS Stream" 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION="9" 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:ID="centos" 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:ID_LIKE="rhel fedora" 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION_ID="9" 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:PLATFORM_ID="platform:el9" 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-06T07:10:32.507 INFO:teuthology.orchestra.run.smithi194.stdout:ANSI_COLOR="0;31" 2024-03-06T07:10:32.508 INFO:teuthology.orchestra.run.smithi194.stdout:LOGO="fedora-logo-icon" 2024-03-06T07:10:32.508 INFO:teuthology.orchestra.run.smithi194.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-06T07:10:32.508 INFO:teuthology.orchestra.run.smithi194.stdout:HOME_URL="https://centos.org/" 2024-03-06T07:10:32.508 INFO:teuthology.orchestra.run.smithi194.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-06T07:10:32.508 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-06T07:10:32.508 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-06T07:10:32.508 INFO:teuthology.lock.ops:Updating smithi194.front.sepia.ceph.com on lock server 2024-03-06T07:10:32.602 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-06T07:10:32.611 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-06T07:10:32.624 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-06T07:10:32.624 DEBUG:teuthology.orchestra.run.smithi086:> test '!' -e /home/ubuntu/cephtest 2024-03-06T07:10:32.626 DEBUG:teuthology.orchestra.run.smithi194:> test '!' -e /home/ubuntu/cephtest 2024-03-06T07:10:32.641 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-06T07:10:32.669 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-06T07:10:32.669 DEBUG:teuthology.orchestra.run.smithi086:> test -z $(ls -A /var/lib/ceph) 2024-03-06T07:10:32.683 DEBUG:teuthology.orchestra.run.smithi194:> test -z $(ls -A /var/lib/ceph) 2024-03-06T07:10:32.712 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-06T07:10:32.759 INFO:teuthology.run_tasks:Running task kernel... 2024-03-06T07:10:32.774 INFO:teuthology.task.kernel:normalize config orig: {'client': {'branch': 'testing'}, 'kdb': True, 'sha1': 'distro'} 2024-03-06T07:10:32.774 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'branch': 'testing'}, 'client.2': {'branch': 'testing'}, 'client.1': {'branch': 'testing'}, 'client.3': {'branch': 'testing'}} 2024-03-06T07:10:32.774 INFO:teuthology.task.kernel:config {'client.0': {'branch': 'testing'}, 'client.1': {'branch': 'testing'}}, timeout 300 2024-03-06T07:10:32.774 DEBUG:teuthology.orchestra.run.smithi086:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-06T07:10:32.775 DEBUG:teuthology.orchestra.run.smithi194:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-06T07:10:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:10:32.791 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F9%2Fx86_64&ref=testing 2024-03-06T07:10:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:10:32.794 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F9%2Fx86_64&ref=testing 2024-03-06T07:10:33.042 DEBUG:teuthology.task.kernel:sha1 for client.1 is b47552b3c0e3993e47c2c21f2f0d35fc872144cc 2024-03-06T07:10:33.042 INFO:teuthology.task.kernel:Checking kernel version of client.1, want "b47552b3c0e3993e47c2c21f2f0d35fc872144cc"... 2024-03-06T07:10:33.042 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-03-06T07:10:33.059 INFO:teuthology.orchestra.run.smithi194.stdout:5.14.0-361.el9.x86_64 2024-03-06T07:10:33.060 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-361.el9.x86_64 vs b47552b3c0e3993e47c2c21f2f0d35fc872144cc 2024-03-06T07:10:33.060 DEBUG:teuthology.task.kernel:failed to parse current kernel version 2024-03-06T07:10:33.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo yum upgrade -y linux-firmware 2024-03-06T07:10:33.112 DEBUG:teuthology.task.kernel:sha1 for client.0 is b47552b3c0e3993e47c2c21f2f0d35fc872144cc 2024-03-06T07:10:33.112 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "b47552b3c0e3993e47c2c21f2f0d35fc872144cc"... 2024-03-06T07:10:33.112 DEBUG:teuthology.orchestra.run.smithi086:> uname -r 2024-03-06T07:10:33.129 INFO:teuthology.orchestra.run.smithi086.stdout:5.14.0-361.el9.x86_64 2024-03-06T07:10:33.129 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-361.el9.x86_64 vs b47552b3c0e3993e47c2c21f2f0d35fc872144cc 2024-03-06T07:10:33.129 DEBUG:teuthology.task.kernel:failed to parse current kernel version 2024-03-06T07:10:33.130 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum upgrade -y linux-firmware 2024-03-06T07:10:35.253 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS-9 - AppStream 15 MB/s | 6.7 MB 00:00 2024-03-06T07:10:35.268 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS-9 - AppStream 14 MB/s | 6.7 MB 00:00 2024-03-06T07:10:36.920 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS-9 - BaseOS 3.6 MB/s | 2.0 MB 00:00 2024-03-06T07:10:36.927 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS-9 - BaseOS 3.6 MB/s | 2.0 MB 00:00 2024-03-06T07:10:41.042 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - BaseOS 2.0 MB/s | 8.0 MB 00:03 2024-03-06T07:10:45.069 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - BaseOS 1.0 MB/s | 8.0 MB 00:07 2024-03-06T07:10:47.087 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - AppStream 20 MB/s | 19 MB 00:00 2024-03-06T07:10:51.970 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - CRB 4.2 MB/s | 6.1 MB 00:01 2024-03-06T07:10:53.446 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - Extras packages 45 kB/s | 15 kB 00:00 2024-03-06T07:10:55.672 INFO:teuthology.orchestra.run.smithi086.stdout:Extra Packages for Enterprise Linux 9.7 MB/s | 21 MB 00:02 2024-03-06T07:11:00.622 INFO:teuthology.orchestra.run.smithi086.stdout:lab-extras 23 kB/s | 1.7 kB 00:00 2024-03-06T07:11:02.960 INFO:teuthology.orchestra.run.smithi086.stdout:Dependencies resolved. 2024-03-06T07:11:02.974 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout: Package Arch Version Repository Size 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout:Upgrading: 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout: linux-firmware noarch 20240219-143.el9 CentOS-BaseOS 375 M 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction Summary 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout:Upgrade 1 Package 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout:Total download size: 375 M 2024-03-06T07:11:02.975 INFO:teuthology.orchestra.run.smithi086.stdout:Downloading Packages: 2024-03-06T07:11:05.674 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - AppStream 817 kB/s | 19 MB 00:23 2024-03-06T07:11:10.226 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - CRB 5.6 MB/s | 6.1 MB 00:01 2024-03-06T07:11:11.616 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 15 kB 00:00 2024-03-06T07:11:15.451 INFO:teuthology.orchestra.run.smithi194.stdout:Extra Packages for Enterprise Linux 5.5 MB/s | 21 MB 00:03 2024-03-06T07:11:20.450 INFO:teuthology.orchestra.run.smithi194.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-03-06T07:11:22.817 INFO:teuthology.orchestra.run.smithi194.stdout:Dependencies resolved. 2024-03-06T07:11:22.829 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-03-06T07:11:22.829 INFO:teuthology.orchestra.run.smithi194.stdout: Package Arch Version Repository Size 2024-03-06T07:11:22.829 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-03-06T07:11:22.829 INFO:teuthology.orchestra.run.smithi194.stdout:Upgrading: 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout: linux-firmware noarch 20240219-143.el9 CentOS-BaseOS 375 M 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction Summary 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout:Upgrade 1 Package 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout:Total download size: 375 M 2024-03-06T07:11:22.830 INFO:teuthology.orchestra.run.smithi194.stdout:Downloading Packages: 2024-03-06T07:11:25.505 INFO:teuthology.orchestra.run.smithi086.stdout:linux-firmware-20240219-143.el9.noarch.rpm 17 MB/s | 375 MB 00:22 2024-03-06T07:11:25.506 INFO:teuthology.orchestra.run.smithi086.stdout:-------------------------------------------------------------------------------- 2024-03-06T07:11:25.506 INFO:teuthology.orchestra.run.smithi086.stdout:Total 17 MB/s | 375 MB 00:22 2024-03-06T07:11:25.507 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction check 2024-03-06T07:11:25.601 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction check succeeded. 2024-03-06T07:11:25.601 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction test 2024-03-06T07:11:28.060 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction test succeeded. 2024-03-06T07:11:28.060 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction 2024-03-06T07:11:34.253 INFO:teuthology.orchestra.run.smithi086.stdout: Preparing : 1/1 2024-03-06T07:11:34.465 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : linux-firmware-20240219-143.el9.noarch 1/2 2024-03-06T07:11:34.546 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : linux-firmware-20230814-139.el9.noarch 2/2 2024-03-06T07:11:35.504 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: linux-firmware-20230814-139.el9.noarch 2/2 2024-03-06T07:11:35.504 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : linux-firmware-20240219-143.el9.noarch 1/2 2024-03-06T07:11:35.916 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : linux-firmware-20230814-139.el9.noarch 2/2 2024-03-06T07:11:35.916 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:11:35.916 INFO:teuthology.orchestra.run.smithi086.stdout:Upgraded: 2024-03-06T07:11:35.916 INFO:teuthology.orchestra.run.smithi086.stdout: linux-firmware-20240219-143.el9.noarch 2024-03-06T07:11:35.916 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:11:35.917 INFO:teuthology.orchestra.run.smithi086.stdout:Complete! 2024-03-06T07:11:36.175 INFO:teuthology.orchestra.run.smithi194.stdout:linux-firmware-20240219-143.el9.noarch.rpm 28 MB/s | 375 MB 00:13 2024-03-06T07:11:36.176 INFO:teuthology.orchestra.run.smithi194.stdout:-------------------------------------------------------------------------------- 2024-03-06T07:11:36.176 INFO:teuthology.orchestra.run.smithi194.stdout:Total 28 MB/s | 375 MB 00:13 2024-03-06T07:11:36.179 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction check 2024-03-06T07:11:36.210 INFO:teuthology.task.kernel:Downloading kernel b47552b3c0e3993e47c2c21f2f0d35fc872144cc on client.0... 2024-03-06T07:11:36.210 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b47552b3c0e3993e47c2c21f2f0d35fc872144cc 2024-03-06T07:11:36.267 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction check succeeded. 2024-03-06T07:11:36.267 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction test 2024-03-06T07:11:36.574 INFO:teuthology.task.kernel:fetching, builder baseurl is https://3.chacra.ceph.com/r/kernel/testing/b47552b3c0e3993e47c2c21f2f0d35fc872144cc/centos/9/flavors/default/x86_64/ 2024-03-06T07:11:36.575 DEBUG:teuthology.orchestra.run.smithi086:> rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.8.0_rc6_gb47552b3c0e3-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://3.chacra.ceph.com/r/kernel/testing/b47552b3c0e3993e47c2c21f2f0d35fc872144cc/centos/9/flavors/default/x86_64/ --input-file=- 2024-03-06T07:11:36.578 DEBUG:teuthology.task.kernel:Waiting for download/copy to ubuntu@smithi086.front.sepia.ceph.com to complete... 2024-03-06T07:11:38.650 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction test succeeded. 2024-03-06T07:11:38.650 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction 2024-03-06T07:11:44.902 INFO:teuthology.orchestra.run.smithi194.stdout: Preparing : 1/1 2024-03-06T07:11:45.144 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : linux-firmware-20240219-143.el9.noarch 1/2 2024-03-06T07:11:45.224 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : linux-firmware-20230814-139.el9.noarch 2/2 2024-03-06T07:11:46.366 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: linux-firmware-20230814-139.el9.noarch 2/2 2024-03-06T07:11:46.366 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : linux-firmware-20240219-143.el9.noarch 1/2 2024-03-06T07:11:46.711 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : linux-firmware-20230814-139.el9.noarch 2/2 2024-03-06T07:11:46.711 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:11:46.711 INFO:teuthology.orchestra.run.smithi194.stdout:Upgraded: 2024-03-06T07:11:46.711 INFO:teuthology.orchestra.run.smithi194.stdout: linux-firmware-20240219-143.el9.noarch 2024-03-06T07:11:46.711 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:11:46.712 INFO:teuthology.orchestra.run.smithi194.stdout:Complete! 2024-03-06T07:11:46.981 INFO:teuthology.task.kernel:Downloading kernel b47552b3c0e3993e47c2c21f2f0d35fc872144cc on client.1... 2024-03-06T07:11:46.981 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b47552b3c0e3993e47c2c21f2f0d35fc872144cc 2024-03-06T07:11:47.131 INFO:teuthology.task.kernel:fetching, builder baseurl is https://3.chacra.ceph.com/r/kernel/testing/b47552b3c0e3993e47c2c21f2f0d35fc872144cc/centos/9/flavors/default/x86_64/ 2024-03-06T07:11:47.131 DEBUG:teuthology.orchestra.run.smithi194:> rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.8.0_rc6_gb47552b3c0e3-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://3.chacra.ceph.com/r/kernel/testing/b47552b3c0e3993e47c2c21f2f0d35fc872144cc/centos/9/flavors/default/x86_64/ --input-file=- 2024-03-06T07:11:47.134 DEBUG:teuthology.task.kernel:Waiting for download/copy to ubuntu@smithi194.front.sepia.ceph.com to complete... 2024-03-06T07:11:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06 07:11:50 URL:https://3.chacra.ceph.com/r/kernel/testing/b47552b3c0e3993e47c2c21f2f0d35fc872144cc/centos/9/flavors/default/x86_64/kernel-6.8.0_rc6_gb47552b3c0e3-1.x86_64.rpm [336022261/336022261] -> "/tmp/kernel.x86_64.rpm" [1] 2024-03-06T07:11:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:FINISHED --2024-03-06 07:11:50-- 2024-03-06T07:11:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:Total wall clock time: 14s 2024-03-06T07:11:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:Downloaded: 1 files, 320M in 13s (24.1 MB/s) 2024-03-06T07:11:50.246 INFO:teuthology.task.kernel:Installing kernel b47552b3c0e3993e47c2c21f2f0d35fc872144cc on client.0... 2024-03-06T07:11:50.246 DEBUG:teuthology.orchestra.run.smithi086:> sudo rpm -ivh --oldpackage --replacefiles --replacepkgs /tmp/kernel.x86_64.rpm 2024-03-06T07:11:52.172 INFO:teuthology.orchestra.run.smithi086.stdout:Verifying... ######################################## 2024-03-06T07:11:52.224 INFO:teuthology.orchestra.run.smithi086.stdout:Preparing... ######################################## 2024-03-06T07:11:52.303 INFO:teuthology.orchestra.run.smithi086.stdout:Updating / installing... 2024-03-06T07:11:59.825 INFO:teuthology.orchestra.run.smithi086.stdout:kernel-6.8.0_rc6_gb47552b3c0e3-1 ######################################## 2024-03-06T07:12:15.560 INFO:teuthology.orchestra.run.smithi086.stderr:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-06T07:12:37.937 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi086.front.sepia.ceph.com, path=/tmp/kernel.x86_64.rpm, version=None) 2024-03-06T07:12:37.937 DEBUG:teuthology.orchestra.run.smithi086:> rpm -qlp /tmp/kernel.x86_64.rpm 2024-03-06T07:12:38.037 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:12:38.037 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/System.map 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/config 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/blowfish-x86_64.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/camellia-aesni-avx-x86_64.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/camellia-aesni-avx2.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/camellia-x86_64.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/cast5-avx-x86_64.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/cast6-avx-x86_64.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/crc32-pclmul.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/crc32c-intel.ko 2024-03-06T07:12:38.038 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/crct10dif-pclmul.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/des3_ede-x86_64.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/ghash-clmulni-intel.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/serpent-avx-x86_64.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/serpent-avx2.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/serpent-sse2-x86_64.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/sha1-ssse3.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/sha256-ssse3.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/sha512-ssse3.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/twofish-avx-x86_64.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/twofish-x86_64-3way.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/twofish-x86_64.ko 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel/cpu 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel/cpu/mce 2024-03-06T07:12:38.039 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel/cpu/mce/mce-inject.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm/kvm-amd.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm/kvm-intel.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm/kvm.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/ansi_cprng.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/anubis.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/arc4.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_memcpy.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_pq.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_raid6_recov.ko 2024-03-06T07:12:38.040 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_tx.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_xor.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/raid6test.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/authenc.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/authencesn.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/blake2b_generic.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/blowfish_common.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/blowfish_generic.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/camellia_generic.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cast5_generic.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cast6_generic.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cast_common.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/ccm.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cmac.ko 2024-03-06T07:12:38.041 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/crc32_generic.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/crypto_engine.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/crypto_user.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cts.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/des_generic.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/dh_generic.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/echainiv.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/essiv.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/fcrypt.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/khazad.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/lz4.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/lz4hc.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/md4.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/michael_mic.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/pcbc.ko 2024-03-06T07:12:38.042 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/pcrypt.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/rmd160.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/seed.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/serpent_generic.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/tcrypt.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/tea.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/twofish_common.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/twofish_generic.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/vmac.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/wp512.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/xcbc.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/xor.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/xxhash_generic.ko 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi 2024-03-06T07:12:38.043 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/acpi_ipmi.ko 2024-03-06T07:12:38.044 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/acpi_pad.ko 2024-03-06T07:12:38.044 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/custom_method.ko 2024-03-06T07:12:38.044 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/ec_sys.ko 2024-03-06T07:12:38.044 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/platform_profile.ko 2024-03-06T07:12:38.044 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/sbs.ko 2024-03-06T07:12:38.044 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/sbshc.ko 2024-03-06T07:12:38.047 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/video.ko 2024-03-06T07:12:38.047 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata 2024-03-06T07:12:38.047 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/acard-ahci.ko 2024-03-06T07:12:38.047 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/ahci_platform.ko 2024-03-06T07:12:38.047 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/ata_generic.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/libahci_platform.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_acpi.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ali.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_amd.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_artop.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_atiixp.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_atp867x.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_cmd640.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_cmd64x.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_cypress.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_efar.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt366.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt37x.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt3x2n.ko 2024-03-06T07:12:38.048 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt3x3.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_it8213.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_it821x.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_jmicron.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_marvell.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_mpiix.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_netcell.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ninja32.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ns87410.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ns87415.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_oldpiix.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_opti.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_optidma.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_pcmcia.ko 2024-03-06T07:12:38.049 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_pdc2027x.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_pdc202xx_old.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_piccolo.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_rdc.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sch.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_serverworks.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sil680.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sis.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sl82c105.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_triflex.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_via.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pdc_adma.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_inic162x.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_mv.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_nv.ko 2024-03-06T07:12:38.050 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_promise.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_qstor.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sil.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sil24.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sis.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_svw.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sx4.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_uli.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_via.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_vsc.ko 2024-03-06T07:12:38.051 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/auxdisplay 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/auxdisplay/ks0108.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base/regmap 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base/regmap/regmap-i2c.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base/regmap/regmap-mmio.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/bcma 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/bcma/bcma.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/aoe 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/aoe/aoe.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/brd.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/drbd 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/drbd/drbd.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/floppy.ko 2024-03-06T07:12:38.052 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/loop.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/mtip32xx 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/mtip32xx/mtip32xx.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/nbd.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/null_blk 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/null_blk/null_blk.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/pktcdvd.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/rbd.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/virtio_blk.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/xen-blkback 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/xen-blkback/xen-blkback.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/xen-blkfront.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/zram 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/zram/zram.ko 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char 2024-03-06T07:12:38.053 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hangcheck-timer.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/amd-rng.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/intel-rng.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/timeriomem-rng.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/via-rng.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/virtio-rng.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_devintf.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_msghandler.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_poweroff.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_si.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_ssif.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_watchdog.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/lp.ko 2024-03-06T07:12:38.054 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/mwave 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/mwave/mwave.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ppdev.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tlclk.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_atmel.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_infineon.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_nsc.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_tis.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_tis_core.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/uv_mmtimer.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/virtio_console.ko 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq 2024-03-06T07:12:38.055 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/acpi-cpufreq.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/amd_freq_sensitivity.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/p4-clockmod.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/pcc-cpufreq.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/powernow-k8.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/speedstep-lib.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/ccp 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/ccp/ccp-crypto.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/ccp/ccp.ko 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel 2024-03-06T07:12:38.056 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_common 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_common/intel_qat.ko 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_dh895xcc 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_dh895xcc/qat_dh895xcc.ko 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/padlock-aes.ko 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/padlock-sha.ko 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/virtio 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/virtio/virtio_crypto.ko 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dca 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dca/dca.ko 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/devfreq 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/devfreq/governor_simpleondemand.ko 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/dw 2024-03-06T07:12:38.057 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/dw/dw_dmac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/ioat 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/ioat/ioatdma.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/amd64_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/e752x_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/edac_mce_amd.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i3000_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i3200_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i5100_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i5400_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i7300_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i7core_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i82975x_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/ie31200_edac.ko 2024-03-06T07:12:38.058 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/sb_edac.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/x38_edac.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/firmware 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/firmware/edd.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/firmware/iscsi_ibft.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/gpio 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/gpio/gpio-ich.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/gpio/gpio-viperboard.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-appleir.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-aureal.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-axff.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-dr.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-elecom.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-elo.ko 2024-03-06T07:12:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-emsff.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-gaff.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-gt683r.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-gyration.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-holtek-kbd.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-holtek-mouse.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-holtekff.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-hyperv.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-icade.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-keytouch.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-kye.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-lcpower.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-led.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-lenovo.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-logitech-dj.ko 2024-03-06T07:12:38.060 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-logitech-hidpp.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-multitouch.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-ortek.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-penmount.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-petalynx.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-picolcd.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-pl.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-plantronics.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-primax.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-rmi.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-arvo.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-common.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-isku.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-kone.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-koneplus.ko 2024-03-06T07:12:38.061 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-konepure.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-kovaplus.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-lua.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-pyra.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-ryos.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-savu.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-saitek.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-samsung.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sensor-hub.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sjoy.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sony.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-speedlink.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-steelseries.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sunplus.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-thrustmaster.ko 2024-03-06T07:12:38.062 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-tivo.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-tmff.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-topseed.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-twinhan.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-uclogic.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-waltop.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-wiimote.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-xinmo.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-zpff.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-zydacron.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/uhid.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/wacom.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv/hv_balloon.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv/hv_utils.ko 2024-03-06T07:12:38.063 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv/hv_vmbus.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/abituguru.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/abituguru3.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/acpi_power_meter.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ad7414.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ad7418.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adc128d818.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1025.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1026.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1029.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1031.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm9240.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ads7828.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7410.ko 2024-03-06T07:12:38.064 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7411.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7462.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7470.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7475.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7x10.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/amc6821.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/applesmc.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/asb100.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/asc7621.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/asus_atk0110.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/atxp1.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/coretemp.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/dell-smm-hwmon.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/dme1737.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ds1621.ko 2024-03-06T07:12:38.065 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ds620.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/emc1403.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/emc6w201.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/f71805f.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/f71882fg.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/f75375s.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/fam15h_power.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/fschmd.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/g760a.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/g762.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/gl518sm.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/gl520sm.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/hwmon-vid.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/i5500_temp.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/i5k_amb.ko 2024-03-06T07:12:38.066 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ibmaem.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ibmpex.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ina209.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ina2xx.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/it87.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/k10temp.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/k8temp.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lineage-pem.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm63.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm73.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm75.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm77.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm78.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm80.ko 2024-03-06T07:12:38.067 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm83.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm85.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm87.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm90.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm92.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm93.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm95234.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm95241.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm95245.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc2945.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4151.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4215.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4222.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4245.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4260.ko 2024-03-06T07:12:38.068 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4261.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max16065.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max1619.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max1668.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max197.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max6639.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max6650.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max6697.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/mcp3021.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct6683.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct6775-core.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct6775.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct7802.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pc87360.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pc87427.ko 2024-03-06T07:12:38.069 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pcf8591.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/adm1275.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/lm25066.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/ltc2978.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/max16064.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/max34440.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/max8688.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/pmbus.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/pmbus_core.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/tps40422.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/ucd9000.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/ucd9200.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/zl6100.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/powr1220.ko 2024-03-06T07:12:38.070 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sch5627.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sch5636.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sch56xx-common.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sht15.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sht21.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/shtc1.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sis5595.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/smsc47b397.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/smsc47m1.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/smsc47m192.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/thmc50.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp102.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp103.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp401.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp421.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/via-cputemp.ko 2024-03-06T07:12:38.071 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/via686a.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/vt1211.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/vt8231.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83627ehf.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83627hf.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83781d.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83791d.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83792d.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83793.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83795.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83l785ts.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83l786ng.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/algos 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/algos/i2c-algo-bit.ko 2024-03-06T07:12:38.072 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/algos/i2c-algo-pca.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-amd756-s4882.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-amd756.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-amd8111.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-ccgx-ucsi.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-designware-core.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-designware-pci.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-designware-platform.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-diolan-u2c.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-i801.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-isch.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-ismt.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-nforce2-s4985.ko 2024-03-06T07:12:38.073 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-nforce2.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-parport.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-pca-platform.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-piix4.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-scmi.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-simtec.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-sis96x.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-tiny-usb.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-via.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-viapro.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-viperboard.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-dev.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-mux.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-slave-eeprom.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-smbus.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-stub.ko 2024-03-06T07:12:38.074 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_cm.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_core.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_umad.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_uverbs.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/iw_cm.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/rdma_cm.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/rdma_ucm.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/cxgb4 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/cxgb4/iw_cxgb4.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx4 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx4/mlx4_ib.ko 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx5 2024-03-06T07:12:38.075 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mthca 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mthca/ib_mthca.ko 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/ipoib 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/iser 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/iser/ib_iser.ko 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/isert 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/isert/ib_isert.ko 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srp 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srp/ib_srp.ko 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srpt 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srpt/ib_srpt.ko 2024-03-06T07:12:38.076 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport/emu10k1-gp.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport/fm801-gp.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport/gameport.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/joydev.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/keyboard 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/keyboard/gpio_keys.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/appletouch.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/bcm5974.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/cyapatp.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/elan_i2c.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/sermouse.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/synaptics_i2c.ko 2024-03-06T07:12:38.077 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/synaptics_usb.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/vsxxxaa.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/rmi4 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/rmi4/rmi_core.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/altera_ps2.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/arc_ps2.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/hyperv-keyboard.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/serio_raw.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/sparse-keymap.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-blinkm.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-lm3530.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-lp3944.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-ss4200.ko 2024-03-06T07:12:38.078 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-audio.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-backlight.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-camera.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-default-on.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-gpio.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-heartbeat.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-oneshot.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-timer.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-transient.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/bcache 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/bcache/bcache.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-bio-prison.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-cache-smq.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-cache.ko 2024-03-06T07:12:38.079 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-crypt.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-delay.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-flakey.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-log-userspace.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-multipath.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-queue-length.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-raid.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-round-robin.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-service-time.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-switch.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-thin-pool.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-verity.ko 2024-03-06T07:12:38.080 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/persistent-data 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/persistent-data/dm-persistent-data.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid0.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid1.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid10.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid456.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptbase.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptctl.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptfc.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptlan.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptsas.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptscsih.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptspi.ko 2024-03-06T07:12:38.081 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/lpc_ich.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/lpc_sch.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/mfd-core.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/sm501.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/viperboard.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/vx855.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/wl1273-core.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/altera-stapl 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/altera-stapl/altera-stapl.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/apds9802als.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/apds990x.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/bh1770glc.ko 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/cb710 2024-03-06T07:12:38.082 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/cb710/cb710.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom/at24.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom/eeprom_93cx6.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom/max6875.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/enclosure.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/hpilo.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/isl29003.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/isl29020.ko 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/lis3lv02d 2024-03-06T07:12:38.083 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/lis3lv02d/lis3lv02d.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/lis3lv02d/lis3lv02d_i2c.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei/mei-me.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei/mei-txe.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei/mei.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-gru 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-gru/gru.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp/xp.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp/xpc.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp/xpnet.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/tifm_7xx1.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/tifm_core.ko 2024-03-06T07:12:38.084 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/tsl2550.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/vmw_balloon.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/vmw_vmci 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/vmw_vmci/vmw_vmci.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/bonding 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/bonding/bonding.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/b53 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/b53/b53_common.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/bcm-sf2.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/mv88e6060.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/mv88e6xxx 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/mv88e6xxx/mv88e6xxx.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dummy.ko 2024-03-06T07:12:38.085 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/eql.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/3c574_cs.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/3c589_cs.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/3c59x.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/typhoon.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/8390.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/axnet_cs.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/ne2k-pci.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/pcnet_cs.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/adaptec 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/adaptec/starfire.ko 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/agere 2024-03-06T07:12:38.086 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/agere/et131x.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/alteon 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/alteon/acenic.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/altera 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/altera/altera_tse.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd/amd8111e.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd/nmclan_cs.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd/pcnet32.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/alx 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/alx/alx.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1c 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1c/atl1c.ko 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1e 2024-03-06T07:12:38.087 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1e/atl1e.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atlx 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atlx/atl1.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atlx/atl2.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/b44.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/bnx2.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/bnx2x 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/bnx2x/bnx2x.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/cnic.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/genet 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/genet/genet.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/tg3.ko 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/brocade 2024-03-06T07:12:38.088 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/brocade/bna 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/brocade/bna/bna.ko 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb/cxgb.ko 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb3 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb3/cxgb3.ko 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4/cxgb4.ko 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4vf 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf.ko 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/libcxgb 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/libcxgb/libcxgb.ko 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/cisco 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/cisco/enic 2024-03-06T07:12:38.089 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/cisco/enic/enic.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/de2104x.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/dmfe.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/tulip.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/uli526x.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/winbond-840.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/xircom_cb.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dlink 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dlink/dl2k.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dlink/sundance.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dnet.ko 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/emulex 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/emulex/benet 2024-03-06T07:12:38.090 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/emulex/benet/be2net.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/ethoc.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/fealnx.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e100.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000/e1000.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000e 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000e/e1000e.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/fm10k 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/fm10k/fm10k.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/i40e 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/i40e/i40e.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/iavf 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/iavf/iavf.ko 2024-03-06T07:12:38.091 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igb 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igb/igb.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igbvf 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igbvf/igbvf.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbe 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbe/ixgbe.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbevf 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbevf/ixgbevf.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/jme.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell/mvmdio.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell/skge.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell/sky2.ko 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox 2024-03-06T07:12:38.092 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx4 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_core.ko 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_en.ko 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx5 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx5/core 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.ko 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/micrel 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/micrel/ksz884x.ko 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/myricom 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/myricom/myri10ge 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/myricom/myri10ge/myri10ge.ko 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/natsemi 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/natsemi/natsemi.ko 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/natsemi/ns83820.ko 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/neterion 2024-03-06T07:12:38.093 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/neterion/s2io.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/nvidia 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/nvidia/forcedeth.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/packetengines 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/packetengines/hamachi.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/packetengines/yellowfin.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/netxen 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/netxen/netxen_nic.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/qla3xxx.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/qlcnic 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/qlcnic/qlcnic.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rdc 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rdc/r6040.ko 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek 2024-03-06T07:12:38.094 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/8139cp.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/8139too.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/atp.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/r8169.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rocker 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rocker/rocker.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sfc 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sfc/sfc.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/silan 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/silan/sc92031.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sis 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sis/sis190.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sis/sis900.ko 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc 2024-03-06T07:12:38.095 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/epic100.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/smc91c92_cs.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/smsc911x.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/smsc9420.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro/stmmac 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro/stmmac/stmmac.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/cassini.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/niu.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/sungem.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/sunhme.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/tehuti 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/tehuti/tehuti.ko 2024-03-06T07:12:38.096 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/ti 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/ti/tlan.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/via 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/via/via-rhine.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/via/via-velocity.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/wiznet 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/wiznet/w5100.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/wiznet/w5300.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/xircom 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/xircom/xirc2ps_cs.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/geneve.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/hyperv 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/hyperv/hv_netvsc.ko 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ieee802154 2024-03-06T07:12:38.097 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ieee802154/fakelb.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ifb.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ipvlan 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ipvlan/ipvlan.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/macvlan.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/macvtap.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio/mdio-bcm-unimac.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio/mdio-bitbang.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio/mdio-regmap.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mii.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/net_failover.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/netconsole.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/nlmon.ko 2024-03-06T07:12:38.098 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/pcs 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/pcs/pcs-lynx.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/pcs/pcs_xpcs.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/amd.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm-phy-lib.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm-phy-ptp.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm7xxx.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm87xx.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/broadcom.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/cicada.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/davicom.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/dp83640.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/et1011c.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/icplus.ko 2024-03-06T07:12:38.099 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/lxt.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/marvell.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/micrel.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/national.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/phylink.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/qsemi.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/realtek.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/smsc.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/ste10Xp.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/vitesse.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/bsd_comp.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_async.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_deflate.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_generic.ko 2024-03-06T07:12:38.100 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_mppe.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_synctty.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/pppoe.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/pppox.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/pptp.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/slip 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/slip/slhc.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/slip/slip.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/sungem_phy.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/tap.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_activebackup.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_broadcast.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_loadbalance.ko 2024-03-06T07:12:38.101 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_random.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_roundrobin.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/tun.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/veth.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/virtio_net.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vmxnet3 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vmxnet3/vmxnet3.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vxlan 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vxlan/vxlan.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/xen-netback 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/xen-netback/xen-netback.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/xen-netfront.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ntb 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ntb/ntb.ko 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme 2024-03-06T07:12:38.102 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme/host 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme/host/nvme-core.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme/host/nvme.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport_cs.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport_pc.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport_serial.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/hotplug 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/hotplug/acpiphp_ibm.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/pcie 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/pcie/aer_inject.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/xen-pcifront.ko 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia 2024-03-06T07:12:38.103 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia/i82092.ko 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia/pd6729.ko 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia/yenta_socket.ko 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pinctrl 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pinctrl/intel 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pinctrl/intel/pinctrl-cherryview.ko 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/acer-wmi.ko 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/acerhdf.ko 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/apple-gmux.ko 2024-03-06T07:12:38.104 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/asus-laptop.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/asus-nb-wmi.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/asus-wmi.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/classmate-laptop.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/eeepc-laptop.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/eeepc-wmi.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/fujitsu-laptop.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/fujitsu-tablet.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/hdaps.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/intel 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/intel/intel-rst.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/intel_ips.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/msi-wmi.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/mxm-wmi.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/panasonic-laptop.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/samsung-laptop.ko 2024-03-06T07:12:38.105 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/samsung-q10.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/topstar-laptop.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/toshiba_bluetooth.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/toshiba_haps.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/wmi-bmof.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/wmi.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/powercap 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/powercap/intel_rapl_common.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/powercap/intel_rapl_msr.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients/pps-gpio.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients/pps-ldisc.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients/pps_parport.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/pps_core.ko 2024-03-06T07:12:38.106 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ptp 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ptp/ptp.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ptp/ptp_kvm.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-bq32k.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1286.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1307.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1374.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1511.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1553.ko 2024-03-06T07:12:38.107 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1672.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1742.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds2404.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds3232.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-em3027.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-fm3130.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-isl12022.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-isl1208.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-m41t80.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-m48t35.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-m48t59.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-max6900.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-msm6242.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf2127.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf85063.ko 2024-03-06T07:12:38.108 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf8523.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf8563.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf8583.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rp5c01.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rs5c372.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rv3029c2.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rx8025.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rx8581.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-stk17ta8.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-x1205.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/3w-9xxx.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/3w-sas.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/3w-xxxx.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/BusLogic.ko 2024-03-06T07:12:38.109 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/a100u2w.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aacraid 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aacraid/aacraid.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/advansys.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aic7xxx 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aic7xxx/aic79xx.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aic7xxx/aic7xxx.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/am53c974.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/arcmsr 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/arcmsr/arcmsr.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/atp870u.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/be2iscsi 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/be2iscsi/be2iscsi.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bfa 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bfa/bfa.ko 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2fc 2024-03-06T07:12:38.110 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2fc/bnx2fc.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2i 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2i/bnx2i.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ch.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/csiostor 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/csiostor/csiostor.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb3i 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb3i/cxgb3i.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb4i 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb4i/cxgb4i.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/libcxgbi.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/dc395x.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_alua.ko 2024-03-06T07:12:38.111 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_emc.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_hp_sw.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_rdac.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/dmx3191d.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/esas2r 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/esas2r/esas2r.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/esp_scsi.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fcoe 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fcoe/fcoe.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fcoe/libfcoe.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fnic 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fnic/fnic.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/hpsa.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/hptiop.ko 2024-03-06T07:12:38.112 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/hv_storvsc.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/initio.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ipr.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ips.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/isci 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/isci/isci.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/iscsi_boot_sysfs.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/iscsi_tcp.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libfc 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libfc/libfc.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libiscsi.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libiscsi_tcp.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libsas 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libsas/libsas.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/lpfc 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/lpfc/lpfc.ko 2024-03-06T07:12:38.113 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid/megaraid_mbox.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid/megaraid_mm.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid/megaraid_sas.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mpt3sas 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mpt3sas/mpt3sas.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mvsas 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mvsas/mvsas.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mvumi.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/pm8001 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/pm8001/pm80xx.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/pmcraid.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla1280.ko 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla2xxx 2024-03-06T07:12:38.114 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla2xxx/qla2xxx.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla2xxx/tcm_qla2xxx.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla4xxx 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla4xxx/qla4xxx.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/raid_class.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_debug.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_fc.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_iscsi.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_sas.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_spi.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_srp.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ses.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/st.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/stex.ko 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/sym53c8xx_2 2024-03-06T07:12:38.115 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/sym53c8xx_2/sym53c8xx.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/virtio_scsi.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/vmw_pvscsi.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/wd719x.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ssb 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ssb/ssb.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/iscsi 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/iscsi/iscsi_target_mod.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/loopback 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/loopback/tcm_loop.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_file.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_iblock.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_mod.ko 2024-03-06T07:12:38.116 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_pscsi.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_user.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/tcm_fc 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/tcm_fc/tcm_fc.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/acpi_thermal_rel.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3400_thermal.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3401_thermal.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3402_thermal.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3403_thermal.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.ko 2024-03-06T07:12:38.117 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_power_floor.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_wt_hint.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_wt_req.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/intel_soc_dts_iosf.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/intel_soc_dts_thermal.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/x86_pkg_temp_thermal.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/ipwireless 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/ipwireless/ipwireless.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/n_gsm.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/n_hdlc.ko 2024-03-06T07:12:38.118 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/nozomi.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/8250 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/8250/serial_cs.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/arc_uart.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/jsm 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/jsm/jsm.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/core 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/core/ufshcd-core.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/host 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/host/ufshcd-pci.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_aec.ko 2024-03-06T07:12:38.119 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_cif.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_pci_generic.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_sercos3.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/cdc-acm.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/cdc-wdm.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/usblp.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/usbtmc.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/host 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/host/sl811-hcd.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/image 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/image/mdc800.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/image/microtek.ko 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc 2024-03-06T07:12:38.120 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/adutux.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/appledisplay.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/emi26.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/emi62.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/ezusb.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/idmouse.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/iowarrior.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/isight_firmware.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/ldusb.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/legousbtower.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/sisusbvga 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/sisusbvga/sisusbvga.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/trancevibrator.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/usb3503.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/usblcd.ko 2024-03-06T07:12:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/usbsevseg.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/uss720.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/yurex.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/phy 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/phy/phy-generic.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/aircable.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ark3116.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/belkin_sa.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ch341.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/cp210x.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/cyberjack.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/cypress_m8.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/digi_acceleport.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/empeg.ko 2024-03-06T07:12:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ftdi_sio.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/garmin_gps.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/io_edgeport.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/io_ti.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ipaq.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ipw.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ir-usb.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/iuu_phoenix.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/keyspan.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/keyspan_pda.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/kl5kusb105.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/kobil_sct.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/mct_u232.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/mos7720.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/mos7840.ko 2024-03-06T07:12:38.123 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/navman.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/omninet.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/opticon.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/option.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/oti6858.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/pl2303.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/qcaux.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/qcserial.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/quatech2.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/safe_serial.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/sierra.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/spcp8x5.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ssu100.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/symbolserial.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ti_usb_3410_5052.ko 2024-03-06T07:12:38.124 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/usb-serial-simple.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/usb_debug.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/usb_wwan.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/visor.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/whiteheat.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/xsens_mt.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/uas.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-alauda.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-cypress.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-datafab.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-eneub6250.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-freecom.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-isd200.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-jumpshot.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-karma.ko 2024-03-06T07:12:38.125 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-onetouch.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-realtek.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-sddr09.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-sddr55.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-usbat.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/usb-storage.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip/usbip-core.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip/usbip-host.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip/vhci-hcd.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/pci 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/pci/vfio-pci-core.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/pci/vfio-pci.ko 2024-03-06T07:12:38.126 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/vfio.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/vfio_iommu_type1.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost_iotlb.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost_net.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost_scsi.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight/apple_bl.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight/lcd.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight/platform_lcd.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_balloon.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_mmio.ko 2024-03-06T07:12:38.127 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_pci.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_pci_legacy_dev.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_pci_modern_dev.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_ring.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/masters 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/masters/ds2482.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/masters/ds2490.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2406.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2408.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2413.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2423.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2431.ko 2024-03-06T07:12:38.128 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2433.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2780.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2781.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds28e04.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_smem.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_therm.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/wire.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/alim1535_wdt.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/alim7101_wdt.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/f71808e_wdt.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/hpwdt.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/i6300esb.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/iTCO_vendor_support.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/iTCO_wdt.ko 2024-03-06T07:12:38.129 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/ib700wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/ibmasr.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/ie6xx_wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/it8712f_wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/it87_wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/machzwd.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/nv_tco.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/pcwd_pci.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/pcwd_usb.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/sbc_fitpc2_wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/sch311x_wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/softdog.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/sp5100_tco.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/via_wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/w83627hf_wdt.ko 2024-03-06T07:12:38.130 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/w83877f_wdt.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/w83977f_wdt.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/wdt_pci.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/xen_wdt.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-acpi-processor.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-evtchn.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-gntalloc.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-gntdev.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-pciback 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-pciback/xen-pciback.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-privcmd.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xenfs 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xenfs/xenfs.ko 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/9p 2024-03-06T07:12:38.131 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/9p/9p.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/affs 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/affs/affs.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/befs 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/befs/befs.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/binfmt_misc.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/btrfs 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/btrfs/btrfs.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cachefiles 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cachefiles/cachefiles.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ceph 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ceph/ceph.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/coda 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/coda/coda.ko 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cramfs 2024-03-06T07:12:38.132 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cramfs/cramfs.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/dlm 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/dlm/dlm.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ecryptfs 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ecryptfs/ecryptfs.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/f2fs 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/f2fs/f2fs.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat/fat.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat/msdos.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat/vfat.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fuse 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fuse/cuse.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fuse/fuse.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/gfs2 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/gfs2/gfs2.ko 2024-03-06T07:12:38.133 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfs 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfs/hfs.ko 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfsplus 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfsplus/hfsplus.ko 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/isofs 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/isofs/isofs.ko 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/jfs 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/jfs/jfs.ko 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/lockd 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/lockd/lockd.ko 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/minix 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/minix/minix.ko 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/netfs 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/netfs/netfs.ko 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs 2024-03-06T07:12:38.134 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/blocklayout 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/blocklayout/blocklayoutdriver.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/filelayout 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/filelayout/nfs_layout_nfsv41_files.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/flexfilelayout 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/flexfilelayout/nfs_layout_flexfiles.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/nfs.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/nfsv3.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/nfsv4.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs_common 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs_common/grace.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs_common/nfs_acl.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfsd 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfsd/nfsd.ko 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nilfs2 2024-03-06T07:12:38.135 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nilfs2/nilfs2.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-celtic.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-centeuro.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-croatian.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-cyrillic.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-gaelic.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-greek.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-iceland.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-inuit.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-roman.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-romanian.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-turkish.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp1250.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp1251.ko 2024-03-06T07:12:38.136 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp1255.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp737.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp775.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp850.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp852.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp855.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp857.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp860.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp861.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp862.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp863.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp864.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp865.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp866.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp869.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp874.ko 2024-03-06T07:12:38.137 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp932.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp936.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp949.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp950.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_euc-jp.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-1.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-13.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-14.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-15.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-2.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-3.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-4.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-5.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-6.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-7.ko 2024-03-06T07:12:38.138 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-9.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_koi8-r.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_koi8-ru.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_koi8-u.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_ucs2_utils.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_utf8.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/cluster 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/cluster/ocfs2_nodemanager.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlm 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlm/ocfs2_dlm.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlmfs 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlmfs/ocfs2_dlmfs.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2_stack_o2cb.ko 2024-03-06T07:12:38.139 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2_stack_user.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2_stackglue.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/overlayfs 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/overlayfs/overlay.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/pstore 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/pstore/ramoops.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/reiserfs 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/reiserfs/reiserfs.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/romfs 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/romfs/romfs.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/client 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/client/cifs.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/common 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/common/cifs_arc4.ko 2024-03-06T07:12:38.140 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/common/cifs_md4.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/squashfs 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/squashfs/squashfs.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/sysv 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/sysv/sysv.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/udf 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/udf/udf.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ufs 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ufs/ufs.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/xfs 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/xfs/xfs.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/rcu 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/rcu/rcutorture.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/torture.ko 2024-03-06T07:12:38.141 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/trace 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/trace/ring_buffer_benchmark.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/asn1_encoder.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crc-itu-t.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crc8.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crypto 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crypto/libarc4.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crypto/libdes.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/libcrc32c.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lru_cache.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lz4 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lz4/lz4_compress.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lz4/lz4hc_compress.ko 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/math 2024-03-06T07:12:38.142 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/math/cordic.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/raid6 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/raid6/raid6_pq.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/reed_solomon 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/reed_solomon/reed_solomon.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/ts_bm.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/ts_fsm.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/ts_kmp.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/zstd 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/zstd/zstd_compress.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/mm 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/mm/hwpoison-inject.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/6lowpan.ko 2024-03-06T07:12:38.143 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_dest.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_fragment.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_hop.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_ipv6.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_mobility.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_routing.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_udp.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/garp.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/mrp.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/p8022.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/psnap.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/stp.ko 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/8021q 2024-03-06T07:12:38.144 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/8021q/8021q.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet_fd.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet_rdma.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet_virtio.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/batman-adv 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/batman-adv/batman-adv.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/br_netfilter.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/bridge.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_802_3.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_among.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_arp.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_arpreply.ko 2024-03-06T07:12:38.145 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_dnat.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_ip.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_ip6.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_limit.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_log.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_mark.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_mark_m.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_nflog.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_pkttype.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_redirect.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_snat.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_stp.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_vlan.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtable_broute.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtable_filter.ko 2024-03-06T07:12:38.146 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtable_nat.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtables.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/nf_conntrack_bridge.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/nft_meta_bridge.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/nft_reject_bridge.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ceph 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ceph/libceph.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/core 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/core/failover.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/core/pktgen.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp_diag.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp_ipv4.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp_ipv6.ko 2024-03-06T07:12:38.147 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dns_resolver 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dns_resolver/dns_resolver.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/dsa_core.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_brcm.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_dsa.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_none.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_trailer.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/6lowpan 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/6lowpan/ieee802154_6lowpan.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/ieee802154.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/ieee802154_socket.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ah4.ko 2024-03-06T07:12:38.148 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/esp4.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/fou.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/gre.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/inet_diag.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ip_gre.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ip_tunnel.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ip_vti.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ipcomp.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ipip.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/arp_tables.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/arpt_mangle.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/arptable_filter.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_ECN.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_SYNPROXY.ko 2024-03-06T07:12:38.149 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_ah.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_rpfilter.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_mangle.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_nat.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_raw.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_security.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_dup_ipv4.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_nat_h323.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_nat_pptp.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_nat_snmp_basic.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_socket_ipv4.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_tproxy_ipv4.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nft_dup_ipv4.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nft_fib_ipv4.ko 2024-03-06T07:12:38.150 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nft_reject_ipv4.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_bic.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_dctcp.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_diag.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_highspeed.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_htcp.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_hybla.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_illinois.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_lp.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_scalable.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_vegas.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_veno.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_westwood.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_yeah.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tunnel4.ko 2024-03-06T07:12:38.151 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/udp_diag.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/udp_tunnel.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/xfrm4_tunnel.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ah6.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/esp6.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/fou6.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ip6_tunnel.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ip6_udp_tunnel.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ip6_vti.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ipcomp6.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6_tables.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_REJECT.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_SYNPROXY.ko 2024-03-06T07:12:38.152 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_ah.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_eui64.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_frag.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_hbh.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_ipv6header.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_mh.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_rpfilter.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_rt.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_filter.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_mangle.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_nat.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_raw.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_security.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_dup_ipv6.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_reject_ipv6.ko 2024-03-06T07:12:38.153 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_socket_ipv6.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_tproxy_ipv6.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nft_dup_ipv6.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nft_fib_ipv6.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nft_reject_ipv6.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/sit.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/tunnel6.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/xfrm6_tunnel.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/key 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/key/af_key.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_core.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_debugfs.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_eth.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_ip.ko 2024-03-06T07:12:38.154 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_ip6.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_netlink.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_ppp.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/llc 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/llc/llc.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mac802154 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mac802154/mac802154.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mpls 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mpls/mpls_gso.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_bitmap_ip.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_bitmap_ipmac.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_bitmap_port.ko 2024-03-06T07:12:38.155 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ip.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipmark.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipport.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipportip.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipportnet.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_mac.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_net.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netiface.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netnet.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netport.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netportnet.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_list_set.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs.ko 2024-03-06T07:12:38.156 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_dh.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_fo.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_ftp.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_lblc.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_lblcr.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_lc.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_nq.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_pe_sip.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_rr.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_sed.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_sh.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_wlc.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_wrr.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conncount.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_amanda.ko 2024-03-06T07:12:38.157 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_broadcast.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_ftp.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_h323.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_irc.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_netbios_ns.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_netlink.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_pptp.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_sane.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_sip.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_snmp.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_tftp.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_dup_netdev.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_flow_table.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_flow_table_inet.ko 2024-03-06T07:12:38.158 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_log_syslog.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_amanda.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_ftp.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_irc.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_sip.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_tftp.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_synproxy_core.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_tables.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_acct.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_log.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_osf.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_queue.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_chain_nat.ko 2024-03-06T07:12:38.159 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_compat.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_ct.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_dup_netdev.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_fib.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_fwd_netdev.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_hash.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_limit.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_log.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_masq.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_nat.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_numgen.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_queue.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_quota.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_redir.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_reject.ko 2024-03-06T07:12:38.160 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_reject_inet.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_socket.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_tunnel.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_AUDIT.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CHECKSUM.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CLASSIFY.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CONNSECMARK.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CT.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_DSCP.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_HL.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_HMARK.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_IDLETIMER.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_LED.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_LOG.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_MASQUERADE.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_NETMAP.ko 2024-03-06T07:12:38.161 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_NFLOG.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_NFQUEUE.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_RATEEST.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_REDIRECT.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_SECMARK.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TCPMSS.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TCPOPTSTRIP.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TEE.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TPROXY.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TRACE.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_addrtype.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_bpf.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_cgroup.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_cluster.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_comment.ko 2024-03-06T07:12:38.162 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connbytes.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connlabel.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connlimit.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connmark.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_conntrack.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_cpu.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_dccp.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_devgroup.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_dscp.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_ecn.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_esp.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_hashlimit.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_helper.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_hl.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_ipcomp.ko 2024-03-06T07:12:38.163 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_iprange.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_ipvs.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_l2tp.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_length.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_limit.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_mac.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_mark.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_multiport.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_nat.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_nfacct.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_osf.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_owner.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_physdev.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_pkttype.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_policy.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_quota.ko 2024-03-06T07:12:38.164 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_rateest.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_realm.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_recent.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_sctp.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_set.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_socket.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_state.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_statistic.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_string.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_tcpmss.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_time.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_u32.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netlink 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netlink/netlink_diag.ko 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/nsh 2024-03-06T07:12:38.165 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/nsh/nsh.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/openvswitch.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/vport-geneve.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/vport-gre.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/vport-vxlan.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/packet 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/packet/af_packet_diag.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds/rds.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds/rds_rdma.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds/rds_tcp.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_csum.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_gact.ko 2024-03-06T07:12:38.166 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_mirred.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_nat.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_pedit.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_police.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_simple.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_skbedit.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_vlan.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_basic.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_bpf.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_flow.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_fw.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_route.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_u32.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_cmp.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_ipset.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_meta.ko 2024-03-06T07:12:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_nbyte.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_text.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_u32.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_choke.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_codel.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_drr.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_fq.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_gred.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_hfsc.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_hhf.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_htb.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_ingress.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_mqprio.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_mqprio_lib.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_multiq.ko 2024-03-06T07:12:38.168 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_netem.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_pie.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_plug.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_prio.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_qfq.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_red.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_sfb.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_sfq.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_tbf.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_teql.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sctp 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sctp/sctp.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sctp/sctp_diag.ko 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/auth_gss 2024-03-06T07:12:38.169 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/auth_gss/auth_rpcgss.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/auth_gss/rpcsec_gss_krb5.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/sunrpc.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/xprtrdma 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/xprtrdma/rpcrdma.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/tipc 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/tipc/diag.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/tipc/tipc.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/unix 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/unix/unix_diag.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vmw_vsock_virtio_transport_common.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vmw_vsock_vmci_transport.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vsock.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vsock_diag.ko 2024-03-06T07:12:38.170 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vsock_loopback.ko 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/xfrm 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/xfrm/xfrm_ipcomp.ko 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/encrypted-keys 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/encrypted-keys/encrypted-keys.ko 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/trusted-keys 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/trusted-keys/trusted.ko 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/virt 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/virt/lib 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/virt/lib/irqbypass.ko 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.alias 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.alias.bin 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin 2024-03-06T07:12:38.171 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin.alias.bin 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin.bin 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin.modinfo 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.dep 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.dep.bin 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.devname 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.order 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.softdep 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.symbols 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.symbols.bin 2024-03-06T07:12:38.172 INFO:teuthology.orchestra.run.smithi086.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/vmlinuz 2024-03-06T07:12:38.176 DEBUG:teuthology.task.kernel:get_image_version: 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:12:38.177 DEBUG:teuthology.orchestra.run.smithi086:> rpm --scripts -qp /tmp/kernel.x86_64.rpm 2024-03-06T07:12:38.200 INFO:teuthology.orchestra.run.smithi086.stdout:postinstall scriptlet (using /bin/sh): 2024-03-06T07:12:38.200 INFO:teuthology.orchestra.run.smithi086.stdout:if [ -x /usr/bin/kernel-install ]; then 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout: /usr/bin/kernel-install add 6.8.0-rc6-gb47552b3c0e3 /lib/modules/6.8.0-rc6-gb47552b3c0e3/vmlinuz 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:fi 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:for file in vmlinuz System.map config; do 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout: if ! cmp --silent "/lib/modules/6.8.0-rc6-gb47552b3c0e3/${file}" "/boot/${file}-6.8.0-rc6-gb47552b3c0e3"; then 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout: cp "/lib/modules/6.8.0-rc6-gb47552b3c0e3/${file}" "/boot/${file}-6.8.0-rc6-gb47552b3c0e3" 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout: fi 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:done 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:preuninstall scriptlet (using /bin/sh): 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:if [ -x /sbin/new-kernel-pkg ]; then 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:new-kernel-pkg --remove 6.8.0-rc6-gb47552b3c0e3 --rminitrd --initrdfile=/boot/initramfs-6.8.0-rc6-gb47552b3c0e3.img 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:elif [ -x /usr/bin/kernel-install ]; then 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:kernel-install remove 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:fi 2024-03-06T07:12:38.201 INFO:teuthology.orchestra.run.smithi086.stdout:postuninstall scriptlet (using /bin/sh): 2024-03-06T07:12:38.202 INFO:teuthology.orchestra.run.smithi086.stdout:if [ -x /sbin/update-bootloader ]; then 2024-03-06T07:12:38.202 INFO:teuthology.orchestra.run.smithi086.stdout:/sbin/update-bootloader --remove 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:12:38.202 INFO:teuthology.orchestra.run.smithi086.stdout:fi 2024-03-06T07:12:38.203 DEBUG:teuthology.orchestra.run.smithi086:> sudo rpm -qi grub2-tools 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Name : grub2-tools 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Epoch : 1 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Version : 2.06 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Release : 68.el9 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Architecture: x86_64 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Group : Unspecified 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:Size : 8264254 2024-03-06T07:12:38.280 INFO:teuthology.orchestra.run.smithi086.stdout:License : GPLv3+ 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Packager : builder@centos.org 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Vendor : CentOS 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Summary : Support tools for GRUB. 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:Description : 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:hardware devices. 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:12:38.281 INFO:teuthology.orchestra.run.smithi086.stdout:This subpackage provides tools for support of all platforms. 2024-03-06T07:12:38.282 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-06T07:12:38.282 INFO:teuthology.task.kernel:Updating grub on smithi086 to boot 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:12:38.283 DEBUG:teuthology.orchestra.run.smithi086:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-06T07:12:38.894 INFO:teuthology.orchestra.run.smithi086.stderr:Generating grub configuration file ... 2024-03-06T07:12:40.347 INFO:teuthology.orchestra.run.smithi086.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-06T07:12:40.375 INFO:teuthology.orchestra.run.smithi086.stderr:done 2024-03-06T07:12:40.377 DEBUG:teuthology.orchestra.run.smithi086:> mktemp 2024-03-06T07:12:40.393 INFO:teuthology.orchestra.run.smithi086.stdout:/tmp/tmp.ZpxhQiXurV 2024-03-06T07:12:40.394 DEBUG:teuthology.orchestra.run.smithi086:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ZpxhQiXurV 2024-03-06T07:12:40.468 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 0666 /tmp/tmp.ZpxhQiXurV 2024-03-06T07:12:40.612 DEBUG:teuthology.orchestra.remote:smithi086:/tmp/tmp.ZpxhQiXurV is 6KB 2024-03-06T07:12:40.660 DEBUG:teuthology.orchestra.run.smithi086:> rm -fr /tmp/tmp.ZpxhQiXurV 2024-03-06T07:12:40.675 DEBUG:teuthology.orchestra.run.smithi086:> sudo /bin/ls /boot/loader/entries || true 2024-03-06T07:12:40.740 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-06T07:12:40.740 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-06T07:12:40.740 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-06T07:12:40.740 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-6.8.0-rc6-gb47552b3c0e3.conf 2024-03-06T07:12:40.742 DEBUG:teuthology.orchestra.run.smithi086:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:12:40.923 DEBUG:teuthology.orchestra.run.smithi086:> sudo shutdown -r now 2024-03-06T07:13:10.926 INFO:teuthology.task.kernel:Checking client client.0 for new kernel version... 2024-03-06T07:13:10.927 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-03-06T07:13:10.928 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:14:10.931 DEBUG:teuthology.orchestra.remote:timed out 2024-03-06T07:14:19.933 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-03-06T07:14:19.934 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:14:27.438 DEBUG:teuthology.orchestra.run.smithi086:> true 2024-03-06T07:14:28.177 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-03-06T07:14:28.177 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "6.8.0-rc6-gb47552b3c0e3"... 2024-03-06T07:14:28.177 DEBUG:teuthology.orchestra.run.smithi086:> uname -r 2024-03-06T07:14:28.213 INFO:teuthology.orchestra.run.smithi086.stdout:6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:14:28.213 DEBUG:teuthology.task.kernel:current kernel version is 6.8.0-rc6-gb47552b3c0e3 vs 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:14:28.213 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-06T07:14:28.214 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-06T07:14:29.217 DEBUG:teuthology.parallel:result is None 2024-03-06T07:15:10.364 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06 07:15:10 URL:https://3.chacra.ceph.com/r/kernel/testing/b47552b3c0e3993e47c2c21f2f0d35fc872144cc/centos/9/flavors/default/x86_64/kernel-6.8.0_rc6_gb47552b3c0e3-1.x86_64.rpm [336022261/336022261] -> "/tmp/kernel.x86_64.rpm" [1] 2024-03-06T07:15:10.365 INFO:teuthology.orchestra.run.smithi194.stderr:FINISHED --2024-03-06 07:15:10-- 2024-03-06T07:15:10.365 INFO:teuthology.orchestra.run.smithi194.stderr:Total wall clock time: 3m 23s 2024-03-06T07:15:10.365 INFO:teuthology.orchestra.run.smithi194.stderr:Downloaded: 1 files, 320M in 3m 23s (1.58 MB/s) 2024-03-06T07:15:10.368 INFO:teuthology.task.kernel:Installing kernel b47552b3c0e3993e47c2c21f2f0d35fc872144cc on client.1... 2024-03-06T07:15:10.369 DEBUG:teuthology.orchestra.run.smithi194:> sudo rpm -ivh --oldpackage --replacefiles --replacepkgs /tmp/kernel.x86_64.rpm 2024-03-06T07:15:12.295 INFO:teuthology.orchestra.run.smithi194.stdout:Verifying... ######################################## 2024-03-06T07:15:12.415 INFO:teuthology.orchestra.run.smithi194.stdout:Preparing... ######################################## 2024-03-06T07:15:12.494 INFO:teuthology.orchestra.run.smithi194.stdout:Updating / installing... 2024-03-06T07:15:19.999 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-6.8.0_rc6_gb47552b3c0e3-1 ######################################## 2024-03-06T07:15:33.628 INFO:teuthology.orchestra.run.smithi194.stderr:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-06T07:15:56.065 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi194.front.sepia.ceph.com, path=/tmp/kernel.x86_64.rpm, version=None) 2024-03-06T07:15:56.065 DEBUG:teuthology.orchestra.run.smithi194:> rpm -qlp /tmp/kernel.x86_64.rpm 2024-03-06T07:15:56.149 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:15:56.149 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/System.map 2024-03-06T07:15:56.149 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/config 2024-03-06T07:15:56.149 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel 2024-03-06T07:15:56.149 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch 2024-03-06T07:15:56.149 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/blowfish-x86_64.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/camellia-aesni-avx-x86_64.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/camellia-aesni-avx2.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/camellia-x86_64.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/cast5-avx-x86_64.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/cast6-avx-x86_64.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/crc32-pclmul.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/crc32c-intel.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/crct10dif-pclmul.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/des3_ede-x86_64.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/ghash-clmulni-intel.ko 2024-03-06T07:15:56.150 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/serpent-avx-x86_64.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/serpent-avx2.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/serpent-sse2-x86_64.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/sha1-ssse3.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/sha256-ssse3.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/sha512-ssse3.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/twofish-avx-x86_64.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/twofish-x86_64-3way.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/crypto/twofish-x86_64.ko 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel/cpu 2024-03-06T07:15:56.151 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel/cpu/mce 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kernel/cpu/mce/mce-inject.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm/kvm-amd.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm/kvm-intel.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/arch/x86/kvm/kvm.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/ansi_cprng.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/anubis.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/arc4.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_memcpy.ko 2024-03-06T07:15:56.152 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_pq.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_raid6_recov.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_tx.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/async_xor.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/async_tx/raid6test.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/authenc.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/authencesn.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/blake2b_generic.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/blowfish_common.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/blowfish_generic.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/camellia_generic.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cast5_generic.ko 2024-03-06T07:15:56.153 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cast6_generic.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cast_common.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/ccm.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cmac.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/crc32_generic.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/crypto_engine.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/crypto_user.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/cts.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/des_generic.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/dh_generic.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/echainiv.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/essiv.ko 2024-03-06T07:15:56.154 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/fcrypt.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/khazad.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/lz4.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/lz4hc.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/md4.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/michael_mic.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/pcbc.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/pcrypt.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/rmd160.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/seed.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/serpent_generic.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/tcrypt.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/tea.ko 2024-03-06T07:15:56.155 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/twofish_common.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/twofish_generic.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/vmac.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/wp512.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/xcbc.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/xor.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/crypto/xxhash_generic.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/acpi_ipmi.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/acpi_pad.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/custom_method.ko 2024-03-06T07:15:56.156 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/ec_sys.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/platform_profile.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/sbs.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/sbshc.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/acpi/video.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/acard-ahci.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/ahci_platform.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/ata_generic.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/libahci_platform.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_acpi.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ali.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_amd.ko 2024-03-06T07:15:56.157 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_artop.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_atiixp.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_atp867x.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_cmd640.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_cmd64x.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_cypress.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_efar.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt366.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt37x.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt3x2n.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_hpt3x3.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_it8213.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_it821x.ko 2024-03-06T07:15:56.158 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_jmicron.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_marvell.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_mpiix.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_netcell.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ninja32.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ns87410.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_ns87415.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_oldpiix.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_opti.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_optidma.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_pcmcia.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_pdc2027x.ko 2024-03-06T07:15:56.159 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_pdc202xx_old.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_piccolo.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_rdc.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sch.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_serverworks.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sil680.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sis.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_sl82c105.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_triflex.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pata_via.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/pdc_adma.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_inic162x.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_mv.ko 2024-03-06T07:15:56.160 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_nv.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_promise.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_qstor.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sil.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sil24.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sis.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_svw.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_sx4.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_uli.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_via.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ata/sata_vsc.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/auxdisplay 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/auxdisplay/ks0108.ko 2024-03-06T07:15:56.161 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base/regmap 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base/regmap/regmap-i2c.ko 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/base/regmap/regmap-mmio.ko 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/bcma 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/bcma/bcma.ko 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/aoe 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/aoe/aoe.ko 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/brd.ko 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/drbd 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/drbd/drbd.ko 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/floppy.ko 2024-03-06T07:15:56.162 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/loop.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/mtip32xx 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/mtip32xx/mtip32xx.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/nbd.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/null_blk 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/null_blk/null_blk.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/pktcdvd.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/rbd.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/virtio_blk.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/xen-blkback 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/xen-blkback/xen-blkback.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/xen-blkfront.ko 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/zram 2024-03-06T07:15:56.163 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/block/zram/zram.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hangcheck-timer.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/amd-rng.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/intel-rng.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/timeriomem-rng.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/via-rng.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/hw_random/virtio-rng.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_devintf.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_msghandler.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_poweroff.ko 2024-03-06T07:15:56.164 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_si.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_ssif.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ipmi/ipmi_watchdog.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/lp.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/mwave 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/mwave/mwave.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/ppdev.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tlclk.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_atmel.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_infineon.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_nsc.ko 2024-03-06T07:15:56.165 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_tis.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/tpm/tpm_tis_core.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/uv_mmtimer.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/char/virtio_console.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/acpi-cpufreq.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/amd_freq_sensitivity.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/p4-clockmod.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/pcc-cpufreq.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/powernow-k8.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/cpufreq/speedstep-lib.ko 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/ccp 2024-03-06T07:15:56.166 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/ccp/ccp-crypto.ko 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/ccp/ccp.ko 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_common 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_common/intel_qat.ko 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_dh895xcc 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/intel/qat/qat_dh895xcc/qat_dh895xcc.ko 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/padlock-aes.ko 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/padlock-sha.ko 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/virtio 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/crypto/virtio/virtio_crypto.ko 2024-03-06T07:15:56.167 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dca 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dca/dca.ko 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/devfreq 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/devfreq/governor_simpleondemand.ko 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/dw 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/dw/dw_dmac.ko 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/ioat 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/dma/ioat/ioatdma.ko 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/amd64_edac.ko 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/e752x_edac.ko 2024-03-06T07:15:56.168 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/edac_mce_amd.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i3000_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i3200_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i5100_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i5400_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i7300_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i7core_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/i82975x_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/ie31200_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/sb_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/edac/x38_edac.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/firmware 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/firmware/edd.ko 2024-03-06T07:15:56.169 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/firmware/iscsi_ibft.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/gpio 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/gpio/gpio-ich.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/gpio/gpio-viperboard.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-appleir.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-aureal.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-axff.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-dr.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-elecom.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-elo.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-emsff.ko 2024-03-06T07:15:56.170 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-gaff.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-gt683r.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-gyration.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-holtek-kbd.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-holtek-mouse.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-holtekff.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-hyperv.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-icade.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-keytouch.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-kye.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-lcpower.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-led.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-lenovo.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-logitech-dj.ko 2024-03-06T07:15:56.171 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-logitech-hidpp.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-multitouch.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-ortek.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-penmount.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-petalynx.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-picolcd.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-pl.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-plantronics.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-primax.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-rmi.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-arvo.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-common.ko 2024-03-06T07:15:56.172 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-isku.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-kone.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-koneplus.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-konepure.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-kovaplus.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-lua.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-pyra.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-ryos.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat-savu.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-roccat.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-saitek.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-samsung.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sensor-hub.ko 2024-03-06T07:15:56.173 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sjoy.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sony.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-speedlink.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-steelseries.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-sunplus.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-thrustmaster.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-tivo.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-tmff.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-topseed.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-twinhan.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-uclogic.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-waltop.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-wiimote.ko 2024-03-06T07:15:56.174 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-xinmo.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-zpff.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/hid-zydacron.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/uhid.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hid/wacom.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv/hv_balloon.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv/hv_utils.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hv/hv_vmbus.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/abituguru.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/abituguru3.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/acpi_power_meter.ko 2024-03-06T07:15:56.175 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ad7414.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ad7418.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adc128d818.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1025.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1026.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1029.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm1031.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adm9240.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ads7828.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7410.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7411.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7462.ko 2024-03-06T07:15:56.176 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7470.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7475.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/adt7x10.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/amc6821.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/applesmc.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/asb100.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/asc7621.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/asus_atk0110.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/atxp1.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/coretemp.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/dell-smm-hwmon.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/dme1737.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ds1621.ko 2024-03-06T07:15:56.177 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ds620.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/emc1403.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/emc6w201.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/f71805f.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/f71882fg.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/f75375s.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/fam15h_power.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/fschmd.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/g760a.ko 2024-03-06T07:15:56.178 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/g762.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/gl518sm.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/gl520sm.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/hwmon-vid.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/i5500_temp.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/i5k_amb.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ibmaem.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ibmpex.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ina209.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ina2xx.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/it87.ko 2024-03-06T07:15:56.179 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/k10temp.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/k8temp.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lineage-pem.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm63.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm73.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm75.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm77.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm78.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm80.ko 2024-03-06T07:15:56.180 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm83.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm85.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm87.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm90.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm92.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm93.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm95234.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm95241.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/lm95245.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc2945.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4151.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4215.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4222.ko 2024-03-06T07:15:56.181 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4245.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4260.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/ltc4261.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max16065.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max1619.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max1668.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max197.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max6639.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max6650.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/max6697.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/mcp3021.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct6683.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct6775-core.ko 2024-03-06T07:15:56.182 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct6775.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/nct7802.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pc87360.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pc87427.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pcf8591.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/adm1275.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/lm25066.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/ltc2978.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/max16064.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/max34440.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/max8688.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/pmbus.ko 2024-03-06T07:15:56.183 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/pmbus_core.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/tps40422.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/ucd9000.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/ucd9200.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/pmbus/zl6100.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/powr1220.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sch5627.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sch5636.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sch56xx-common.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sht15.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sht21.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/shtc1.ko 2024-03-06T07:15:56.184 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/sis5595.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/smsc47b397.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/smsc47m1.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/smsc47m192.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/thmc50.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp102.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp103.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp401.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/tmp421.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/via-cputemp.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/via686a.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/vt1211.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/vt8231.ko 2024-03-06T07:15:56.185 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83627ehf.ko 2024-03-06T07:15:56.186 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83627hf.ko 2024-03-06T07:15:56.186 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83781d.ko 2024-03-06T07:15:56.186 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83791d.ko 2024-03-06T07:15:56.187 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83792d.ko 2024-03-06T07:15:56.187 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83793.ko 2024-03-06T07:15:56.187 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83795.ko 2024-03-06T07:15:56.187 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83l785ts.ko 2024-03-06T07:15:56.187 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/hwmon/w83l786ng.ko 2024-03-06T07:15:56.187 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/algos 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/algos/i2c-algo-bit.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/algos/i2c-algo-pca.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-amd756-s4882.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-amd756.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-amd8111.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-ccgx-ucsi.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-designware-core.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-designware-pci.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-designware-platform.ko 2024-03-06T07:15:56.188 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-diolan-u2c.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-i801.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-isch.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-ismt.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-nforce2-s4985.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-nforce2.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-parport.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-pca-platform.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-piix4.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-scmi.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-simtec.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-sis96x.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-tiny-usb.ko 2024-03-06T07:15:56.189 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-via.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-viapro.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/busses/i2c-viperboard.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-dev.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-mux.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-slave-eeprom.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-smbus.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/i2c/i2c-stub.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_cm.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_core.ko 2024-03-06T07:15:56.190 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_umad.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/ib_uverbs.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/iw_cm.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/rdma_cm.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/core/rdma_ucm.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/cxgb4 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/cxgb4/iw_cxgb4.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx4 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx4/mlx4_ib.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx5 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mthca 2024-03-06T07:15:56.191 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/hw/mthca/ib_mthca.ko 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/ipoib 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/iser 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/iser/ib_iser.ko 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/isert 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/isert/ib_isert.ko 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srp 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srp/ib_srp.ko 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srpt 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/infiniband/ulp/srpt/ib_srpt.ko 2024-03-06T07:15:56.192 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport/emu10k1-gp.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport/fm801-gp.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/gameport/gameport.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/joydev.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/keyboard 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/keyboard/gpio_keys.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/appletouch.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/bcm5974.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/cyapatp.ko 2024-03-06T07:15:56.193 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/elan_i2c.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/sermouse.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/synaptics_i2c.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/synaptics_usb.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/mouse/vsxxxaa.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/rmi4 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/rmi4/rmi_core.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/altera_ps2.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/arc_ps2.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/hyperv-keyboard.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/serio/serio_raw.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/input/sparse-keymap.ko 2024-03-06T07:15:56.194 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-blinkm.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-lm3530.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-lp3944.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/leds-ss4200.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-audio.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-backlight.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-camera.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-default-on.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-gpio.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-heartbeat.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-oneshot.ko 2024-03-06T07:15:56.195 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-timer.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/leds/trigger/ledtrig-transient.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/bcache 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/bcache/bcache.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-bio-prison.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-cache-smq.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-cache.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-crypt.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-delay.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-flakey.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-log-userspace.ko 2024-03-06T07:15:56.196 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-multipath.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-queue-length.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-raid.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-round-robin.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-service-time.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-switch.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-thin-pool.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/dm-verity.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/persistent-data 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/persistent-data/dm-persistent-data.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid0.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid1.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid10.ko 2024-03-06T07:15:56.197 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/md/raid456.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptbase.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptctl.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptfc.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptlan.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptsas.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptscsih.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/message/fusion/mptspi.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/lpc_ich.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/lpc_sch.ko 2024-03-06T07:15:56.198 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/mfd-core.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/sm501.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/viperboard.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/vx855.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/mfd/wl1273-core.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/altera-stapl 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/altera-stapl/altera-stapl.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/apds9802als.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/apds990x.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/bh1770glc.ko 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/cb710 2024-03-06T07:15:56.199 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/cb710/cb710.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom/at24.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom/eeprom_93cx6.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/eeprom/max6875.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/enclosure.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/hpilo.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/isl29003.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/isl29020.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/lis3lv02d 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/lis3lv02d/lis3lv02d.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/lis3lv02d/lis3lv02d_i2c.ko 2024-03-06T07:15:56.200 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei/mei-me.ko 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei/mei-txe.ko 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/mei/mei.ko 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-gru 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-gru/gru.ko 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp/xp.ko 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp/xpc.ko 2024-03-06T07:15:56.201 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/sgi-xp/xpnet.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/tifm_7xx1.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/tifm_core.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/tsl2550.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/vmw_balloon.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/vmw_vmci 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/misc/vmw_vmci/vmw_vmci.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/bonding 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/bonding/bonding.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/b53 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/b53/b53_common.ko 2024-03-06T07:15:56.202 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/bcm-sf2.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/mv88e6060.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/mv88e6xxx 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dsa/mv88e6xxx/mv88e6xxx.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/dummy.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/eql.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/3c574_cs.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/3c589_cs.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/3c59x.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/3com/typhoon.ko 2024-03-06T07:15:56.203 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/8390.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/axnet_cs.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/ne2k-pci.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/8390/pcnet_cs.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/adaptec 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/adaptec/starfire.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/agere 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/agere/et131x.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/alteon 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/alteon/acenic.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/altera 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/altera/altera_tse.ko 2024-03-06T07:15:56.204 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd/amd8111e.ko 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd/nmclan_cs.ko 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/amd/pcnet32.ko 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/alx 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/alx/alx.ko 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1c 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1c/atl1c.ko 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1e 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atl1e/atl1e.ko 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atlx 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atlx/atl1.ko 2024-03-06T07:15:56.205 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/atheros/atlx/atl2.ko 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/b44.ko 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/bnx2.ko 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/bnx2x 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/bnx2x/bnx2x.ko 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/cnic.ko 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/genet 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/genet/genet.ko 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/broadcom/tg3.ko 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/brocade 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/brocade/bna 2024-03-06T07:15:56.206 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/brocade/bna/bna.ko 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb/cxgb.ko 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb3 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb3/cxgb3.ko 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4/cxgb4.ko 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4vf 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf.ko 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/libcxgb 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/chelsio/libcxgb/libcxgb.ko 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/cisco 2024-03-06T07:15:56.207 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/cisco/enic 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/cisco/enic/enic.ko 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/de2104x.ko 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/dmfe.ko 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/tulip.ko 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/uli526x.ko 2024-03-06T07:15:56.208 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/winbond-840.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dec/tulip/xircom_cb.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dlink 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dlink/dl2k.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dlink/sundance.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/dnet.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/emulex 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/emulex/benet 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/emulex/benet/be2net.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/ethoc.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/fealnx.ko 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel 2024-03-06T07:15:56.209 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e100.ko 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000/e1000.ko 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000e 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/e1000e/e1000e.ko 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/fm10k 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/fm10k/fm10k.ko 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/i40e 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/i40e/i40e.ko 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/iavf 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/iavf/iavf.ko 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igb 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igb/igb.ko 2024-03-06T07:15:56.210 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igbvf 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/igbvf/igbvf.ko 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbe 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbe/ixgbe.ko 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbevf 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/intel/ixgbevf/ixgbevf.ko 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/jme.ko 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell/mvmdio.ko 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell/skge.ko 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/marvell/sky2.ko 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx4 2024-03-06T07:15:56.211 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_core.ko 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_en.ko 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx5 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx5/core 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.ko 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/micrel 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/micrel/ksz884x.ko 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/myricom 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/myricom/myri10ge 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/myricom/myri10ge/myri10ge.ko 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/natsemi 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/natsemi/natsemi.ko 2024-03-06T07:15:56.212 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/natsemi/ns83820.ko 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/neterion 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/neterion/s2io.ko 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/nvidia 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/nvidia/forcedeth.ko 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/packetengines 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/packetengines/hamachi.ko 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/packetengines/yellowfin.ko 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/netxen 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/netxen/netxen_nic.ko 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/qla3xxx.ko 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/qlcnic 2024-03-06T07:15:56.213 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/qlogic/qlcnic/qlcnic.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rdc 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rdc/r6040.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/8139cp.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/8139too.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/atp.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/realtek/r8169.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rocker 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/rocker/rocker.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sfc 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sfc/sfc.ko 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/silan 2024-03-06T07:15:56.214 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/silan/sc92031.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sis 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sis/sis190.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sis/sis900.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/epic100.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/smc91c92_cs.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/smsc911x.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/smsc/smsc9420.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro/stmmac 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.ko 2024-03-06T07:15:56.215 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/stmicro/stmmac/stmmac.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/cassini.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/niu.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/sungem.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/sun/sunhme.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/tehuti 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/tehuti/tehuti.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/ti 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/ti/tlan.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/via 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/via/via-rhine.ko 2024-03-06T07:15:56.216 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/via/via-velocity.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/wiznet 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/wiznet/w5100.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/wiznet/w5300.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/xircom 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ethernet/xircom/xirc2ps_cs.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/geneve.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/hyperv 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/hyperv/hv_netvsc.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ieee802154 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ieee802154/fakelb.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ifb.ko 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ipvlan 2024-03-06T07:15:56.217 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ipvlan/ipvlan.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/macvlan.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/macvtap.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio/mdio-bcm-unimac.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio/mdio-bitbang.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mdio/mdio-regmap.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/mii.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/net_failover.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/netconsole.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/nlmon.ko 2024-03-06T07:15:56.218 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/pcs 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/pcs/pcs-lynx.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/pcs/pcs_xpcs.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/amd.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm-phy-lib.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm-phy-ptp.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm7xxx.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/bcm87xx.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/broadcom.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/cicada.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/davicom.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/dp83640.ko 2024-03-06T07:15:56.219 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/et1011c.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/icplus.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/lxt.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/marvell.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/micrel.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/national.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/phylink.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/qsemi.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/realtek.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/smsc.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/ste10Xp.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/phy/vitesse.ko 2024-03-06T07:15:56.220 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/bsd_comp.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_async.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_deflate.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_generic.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_mppe.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/ppp_synctty.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/pppoe.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/pppox.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/ppp/pptp.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/slip 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/slip/slhc.ko 2024-03-06T07:15:56.221 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/slip/slip.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/sungem_phy.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/tap.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_activebackup.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_broadcast.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_loadbalance.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_random.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/team/team_mode_roundrobin.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/tun.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/veth.ko 2024-03-06T07:15:56.222 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/virtio_net.ko 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vmxnet3 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vmxnet3/vmxnet3.ko 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vxlan 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/vxlan/vxlan.ko 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/xen-netback 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/xen-netback/xen-netback.ko 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/net/xen-netfront.ko 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ntb 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ntb/ntb.ko 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme/host 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme/host/nvme-core.ko 2024-03-06T07:15:56.223 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/nvme/host/nvme.ko 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport.ko 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport_cs.ko 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport_pc.ko 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/parport/parport_serial.ko 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/hotplug 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/hotplug/acpiphp_ibm.ko 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/pcie 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/pcie/aer_inject.ko 2024-03-06T07:15:56.224 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pci/xen-pcifront.ko 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia/i82092.ko 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia/pd6729.ko 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pcmcia/yenta_socket.ko 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pinctrl 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pinctrl/intel 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pinctrl/intel/pinctrl-cherryview.ko 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/acer-wmi.ko 2024-03-06T07:15:56.225 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/acerhdf.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/apple-gmux.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/asus-laptop.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/asus-nb-wmi.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/asus-wmi.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/classmate-laptop.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/eeepc-laptop.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/eeepc-wmi.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/fujitsu-laptop.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/fujitsu-tablet.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/hdaps.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/intel 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/intel/intel-rst.ko 2024-03-06T07:15:56.226 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/intel_ips.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/msi-wmi.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/mxm-wmi.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/panasonic-laptop.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/samsung-laptop.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/samsung-q10.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/topstar-laptop.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/toshiba_bluetooth.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/toshiba_haps.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/wmi-bmof.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/platform/x86/wmi.ko 2024-03-06T07:15:56.227 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/powercap 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/powercap/intel_rapl_common.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/powercap/intel_rapl_msr.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients/pps-gpio.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients/pps-ldisc.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/clients/pps_parport.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/pps/pps_core.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ptp 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ptp/ptp.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ptp/ptp_kvm.ko 2024-03-06T07:15:56.228 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-bq32k.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1286.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1307.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1374.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1511.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1553.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1672.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds1742.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds2404.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-ds3232.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-em3027.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-fm3130.ko 2024-03-06T07:15:56.229 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-isl12022.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-isl1208.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-m41t80.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-m48t35.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-m48t59.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-max6900.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-msm6242.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf2127.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf85063.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf8523.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf8563.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-pcf8583.ko 2024-03-06T07:15:56.230 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rp5c01.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rs5c372.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rv3029c2.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rx8025.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-rx8581.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-stk17ta8.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/rtc/rtc-x1205.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/3w-9xxx.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/3w-sas.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/3w-xxxx.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/BusLogic.ko 2024-03-06T07:15:56.231 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/a100u2w.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aacraid 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aacraid/aacraid.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/advansys.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aic7xxx 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aic7xxx/aic79xx.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/aic7xxx/aic7xxx.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/am53c974.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/arcmsr 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/arcmsr/arcmsr.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/atp870u.ko 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/be2iscsi 2024-03-06T07:15:56.232 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/be2iscsi/be2iscsi.ko 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bfa 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bfa/bfa.ko 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2fc 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2fc/bnx2fc.ko 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2i 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/bnx2i/bnx2i.ko 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ch.ko 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/csiostor 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/csiostor/csiostor.ko 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb3i 2024-03-06T07:15:56.233 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb3i/cxgb3i.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb4i 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/cxgb4i/cxgb4i.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/cxgbi/libcxgbi.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/dc395x.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_alua.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_emc.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_hp_sw.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/device_handler/scsi_dh_rdac.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/dmx3191d.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/esas2r 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/esas2r/esas2r.ko 2024-03-06T07:15:56.234 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/esp_scsi.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fcoe 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fcoe/fcoe.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fcoe/libfcoe.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fnic 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/fnic/fnic.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/hpsa.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/hptiop.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/hv_storvsc.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/initio.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ipr.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ips.ko 2024-03-06T07:15:56.235 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/isci 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/isci/isci.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/iscsi_boot_sysfs.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/iscsi_tcp.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libfc 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libfc/libfc.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libiscsi.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libiscsi_tcp.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libsas 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/libsas/libsas.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/lpfc 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/lpfc/lpfc.ko 2024-03-06T07:15:56.236 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid/megaraid_mbox.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid/megaraid_mm.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/megaraid/megaraid_sas.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mpt3sas 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mpt3sas/mpt3sas.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mvsas 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mvsas/mvsas.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/mvumi.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/pm8001 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/pm8001/pm80xx.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/pmcraid.ko 2024-03-06T07:15:56.237 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla1280.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla2xxx 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla2xxx/qla2xxx.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla2xxx/tcm_qla2xxx.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla4xxx 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/qla4xxx/qla4xxx.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/raid_class.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_debug.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_fc.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_iscsi.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_sas.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_spi.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/scsi_transport_srp.ko 2024-03-06T07:15:56.238 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/ses.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/st.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/stex.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/sym53c8xx_2 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/sym53c8xx_2/sym53c8xx.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/virtio_scsi.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/vmw_pvscsi.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/scsi/wd719x.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ssb 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ssb/ssb.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/iscsi 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/iscsi/iscsi_target_mod.ko 2024-03-06T07:15:56.239 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/loopback 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/loopback/tcm_loop.ko 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_file.ko 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_iblock.ko 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_mod.ko 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_pscsi.ko 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/target_core_user.ko 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/tcm_fc 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/target/tcm_fc/tcm_fc.ko 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel 2024-03-06T07:15:56.240 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/acpi_thermal_rel.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3400_thermal.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3401_thermal.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3402_thermal.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int3403_thermal.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_power_floor.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.ko 2024-03-06T07:15:56.241 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_wt_hint.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_wt_req.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/intel_soc_dts_iosf.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/intel_soc_dts_thermal.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/thermal/intel/x86_pkg_temp_thermal.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/ipwireless 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/ipwireless/ipwireless.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/n_gsm.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/n_hdlc.ko 2024-03-06T07:15:56.242 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/nozomi.ko 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/8250 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/8250/serial_cs.ko 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/arc_uart.ko 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/jsm 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/tty/serial/jsm/jsm.ko 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/core 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/core/ufshcd-core.ko 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/host 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/ufs/host/ufshcd-pci.ko 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio 2024-03-06T07:15:56.243 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_aec.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_cif.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_pci_generic.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/uio/uio_sercos3.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/cdc-acm.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/cdc-wdm.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/usblp.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/class/usbtmc.ko 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/host 2024-03-06T07:15:56.244 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/host/sl811-hcd.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/image 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/image/mdc800.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/image/microtek.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/adutux.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/appledisplay.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/emi26.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/emi62.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/ezusb.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/idmouse.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/iowarrior.ko 2024-03-06T07:15:56.245 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/isight_firmware.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/ldusb.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/legousbtower.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/sisusbvga 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/sisusbvga/sisusbvga.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/trancevibrator.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/usb3503.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/usblcd.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/usbsevseg.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/uss720.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/misc/yurex.ko 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/phy 2024-03-06T07:15:56.246 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/phy/phy-generic.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/aircable.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ark3116.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/belkin_sa.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ch341.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/cp210x.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/cyberjack.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/cypress_m8.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/digi_acceleport.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/empeg.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ftdi_sio.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/garmin_gps.ko 2024-03-06T07:15:56.247 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/io_edgeport.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/io_ti.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ipaq.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ipw.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ir-usb.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/iuu_phoenix.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/keyspan.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/keyspan_pda.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/kl5kusb105.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/kobil_sct.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/mct_u232.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/mos7720.ko 2024-03-06T07:15:56.248 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/mos7840.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/navman.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/omninet.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/opticon.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/option.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/oti6858.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/pl2303.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/qcaux.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/qcserial.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/quatech2.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/safe_serial.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/sierra.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/spcp8x5.ko 2024-03-06T07:15:56.249 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ssu100.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/symbolserial.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/ti_usb_3410_5052.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/usb-serial-simple.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/usb_debug.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/usb_wwan.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/visor.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/whiteheat.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/serial/xsens_mt.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/uas.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-alauda.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-cypress.ko 2024-03-06T07:15:56.250 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-datafab.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-eneub6250.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-freecom.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-isd200.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-jumpshot.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-karma.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-onetouch.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-realtek.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-sddr09.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-sddr55.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/ums-usbat.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/storage/usb-storage.ko 2024-03-06T07:15:56.251 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip/usbip-core.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip/usbip-host.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/usb/usbip/vhci-hcd.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/pci 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/pci/vfio-pci-core.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/pci/vfio-pci.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/vfio.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vfio/vfio_iommu_type1.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost.ko 2024-03-06T07:15:56.252 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost_iotlb.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost_net.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/vhost/vhost_scsi.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight/apple_bl.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight/lcd.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/video/backlight/platform_lcd.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_balloon.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_mmio.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_pci.ko 2024-03-06T07:15:56.253 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_pci_legacy_dev.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_pci_modern_dev.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/virtio/virtio_ring.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/masters 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/masters/ds2482.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/masters/ds2490.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2406.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2408.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2413.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2423.ko 2024-03-06T07:15:56.254 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2431.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2433.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2780.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds2781.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_ds28e04.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_smem.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/slaves/w1_therm.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/w1/wire.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/alim1535_wdt.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/alim7101_wdt.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/f71808e_wdt.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/hpwdt.ko 2024-03-06T07:15:56.255 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/i6300esb.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/iTCO_vendor_support.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/iTCO_wdt.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/ib700wdt.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/ibmasr.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/ie6xx_wdt.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/it8712f_wdt.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/it87_wdt.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/machzwd.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/nv_tco.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/pcwd_pci.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/pcwd_usb.ko 2024-03-06T07:15:56.256 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/sbc_fitpc2_wdt.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/sch311x_wdt.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/softdog.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/sp5100_tco.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/via_wdt.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/w83627hf_wdt.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/w83877f_wdt.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/w83977f_wdt.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/wdt_pci.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/watchdog/xen_wdt.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-acpi-processor.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-evtchn.ko 2024-03-06T07:15:56.257 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-gntalloc.ko 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-gntdev.ko 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-pciback 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-pciback/xen-pciback.ko 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xen-privcmd.ko 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xenfs 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/drivers/xen/xenfs/xenfs.ko 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/9p 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/9p/9p.ko 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/affs 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/affs/affs.ko 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/befs 2024-03-06T07:15:56.258 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/befs/befs.ko 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/binfmt_misc.ko 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/btrfs 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/btrfs/btrfs.ko 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cachefiles 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cachefiles/cachefiles.ko 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ceph 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ceph/ceph.ko 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/coda 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/coda/coda.ko 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cramfs 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/cramfs/cramfs.ko 2024-03-06T07:15:56.259 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/dlm 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/dlm/dlm.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ecryptfs 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ecryptfs/ecryptfs.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/f2fs 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/f2fs/f2fs.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat/fat.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat/msdos.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fat/vfat.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fuse 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fuse/cuse.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/fuse/fuse.ko 2024-03-06T07:15:56.260 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/gfs2 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/gfs2/gfs2.ko 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfs 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfs/hfs.ko 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfsplus 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/hfsplus/hfsplus.ko 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/isofs 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/isofs/isofs.ko 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/jfs 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/jfs/jfs.ko 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/lockd 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/lockd/lockd.ko 2024-03-06T07:15:56.261 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/minix 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/minix/minix.ko 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/netfs 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/netfs/netfs.ko 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/blocklayout 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/blocklayout/blocklayoutdriver.ko 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/filelayout 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/filelayout/nfs_layout_nfsv41_files.ko 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/flexfilelayout 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/flexfilelayout/nfs_layout_flexfiles.ko 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/nfs.ko 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/nfsv3.ko 2024-03-06T07:15:56.262 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs/nfsv4.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs_common 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs_common/grace.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfs_common/nfs_acl.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfsd 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nfsd/nfsd.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nilfs2 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nilfs2/nilfs2.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-celtic.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-centeuro.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-croatian.ko 2024-03-06T07:15:56.263 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-cyrillic.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-gaelic.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-greek.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-iceland.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-inuit.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-roman.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-romanian.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/mac-turkish.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp1250.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp1251.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp1255.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp737.ko 2024-03-06T07:15:56.264 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp775.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp850.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp852.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp855.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp857.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp860.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp861.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp862.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp863.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp864.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp865.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp866.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp869.ko 2024-03-06T07:15:56.265 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp874.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp932.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp936.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp949.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_cp950.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_euc-jp.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-1.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-13.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-14.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-15.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-2.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-3.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-4.ko 2024-03-06T07:15:56.266 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-5.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-6.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-7.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_iso8859-9.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_koi8-r.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_koi8-ru.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_koi8-u.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_ucs2_utils.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/nls/nls_utf8.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/cluster 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/cluster/ocfs2_nodemanager.ko 2024-03-06T07:15:56.267 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlm 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlm/ocfs2_dlm.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlmfs 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/dlmfs/ocfs2_dlmfs.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2_stack_o2cb.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2_stack_user.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ocfs2/ocfs2_stackglue.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/overlayfs 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/overlayfs/overlay.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/pstore 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/pstore/ramoops.ko 2024-03-06T07:15:56.268 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/reiserfs 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/reiserfs/reiserfs.ko 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/romfs 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/romfs/romfs.ko 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/client 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/client/cifs.ko 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/common 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/common/cifs_arc4.ko 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/smb/common/cifs_md4.ko 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/squashfs 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/squashfs/squashfs.ko 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/sysv 2024-03-06T07:15:56.269 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/sysv/sysv.ko 2024-03-06T07:15:56.270 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/udf 2024-03-06T07:15:56.270 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/udf/udf.ko 2024-03-06T07:15:56.270 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ufs 2024-03-06T07:15:56.270 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/ufs/ufs.ko 2024-03-06T07:15:56.270 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/xfs 2024-03-06T07:15:56.270 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/fs/xfs/xfs.ko 2024-03-06T07:15:56.270 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel 2024-03-06T07:15:56.271 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/rcu 2024-03-06T07:15:56.271 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/rcu/rcutorture.ko 2024-03-06T07:15:56.271 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/torture.ko 2024-03-06T07:15:56.271 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/trace 2024-03-06T07:15:56.271 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/kernel/trace/ring_buffer_benchmark.ko 2024-03-06T07:15:56.271 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib 2024-03-06T07:15:56.271 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/asn1_encoder.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crc-itu-t.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crc8.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crypto 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crypto/libarc4.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/crypto/libdes.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/libcrc32c.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lru_cache.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lz4 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lz4/lz4_compress.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/lz4/lz4hc_compress.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/math 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/math/cordic.ko 2024-03-06T07:15:56.272 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/raid6 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/raid6/raid6_pq.ko 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/reed_solomon 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/reed_solomon/reed_solomon.ko 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/ts_bm.ko 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/ts_fsm.ko 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/ts_kmp.ko 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/zstd 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/lib/zstd/zstd_compress.ko 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/mm 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/mm/hwpoison-inject.ko 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan 2024-03-06T07:15:56.273 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/6lowpan.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_dest.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_fragment.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_hop.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_ipv6.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_mobility.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_routing.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/6lowpan/nhc_udp.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/garp.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/mrp.ko 2024-03-06T07:15:56.274 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/p8022.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/psnap.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/802/stp.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/8021q 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/8021q/8021q.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet_fd.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet_rdma.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/9p/9pnet_virtio.ko 2024-03-06T07:15:56.275 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/batman-adv 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/batman-adv/batman-adv.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/br_netfilter.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/bridge.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_802_3.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_among.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_arp.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_arpreply.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_dnat.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_ip.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_ip6.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_limit.ko 2024-03-06T07:15:56.276 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_log.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_mark.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_mark_m.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_nflog.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_pkttype.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_redirect.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_snat.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_stp.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebt_vlan.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtable_broute.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtable_filter.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtable_nat.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/ebtables.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/nf_conntrack_bridge.ko 2024-03-06T07:15:56.277 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/nft_meta_bridge.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/bridge/netfilter/nft_reject_bridge.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ceph 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ceph/libceph.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/core 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/core/failover.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/core/pktgen.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp_diag.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp_ipv4.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dccp/dccp_ipv6.ko 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dns_resolver 2024-03-06T07:15:56.278 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dns_resolver/dns_resolver.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/dsa_core.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_brcm.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_dsa.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_none.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/dsa/tag_trailer.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/6lowpan 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/6lowpan/ieee802154_6lowpan.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/ieee802154.ko 2024-03-06T07:15:56.279 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ieee802154/ieee802154_socket.ko 2024-03-06T07:15:56.280 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4 2024-03-06T07:15:56.280 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ah4.ko 2024-03-06T07:15:56.280 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/esp4.ko 2024-03-06T07:15:56.280 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/fou.ko 2024-03-06T07:15:56.280 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/gre.ko 2024-03-06T07:15:56.280 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/inet_diag.ko 2024-03-06T07:15:56.280 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ip_gre.ko 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ip_tunnel.ko 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ip_vti.ko 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ipcomp.ko 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/ipip.ko 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/arp_tables.ko 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/arpt_mangle.ko 2024-03-06T07:15:56.281 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/arptable_filter.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_ECN.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_SYNPROXY.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_ah.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/ipt_rpfilter.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_mangle.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_nat.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_raw.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/iptable_security.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_dup_ipv4.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_nat_h323.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_nat_pptp.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_nat_snmp_basic.ko 2024-03-06T07:15:56.282 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_socket_ipv4.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nf_tproxy_ipv4.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nft_dup_ipv4.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nft_fib_ipv4.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/netfilter/nft_reject_ipv4.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_bic.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_dctcp.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_diag.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_highspeed.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_htcp.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_hybla.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_illinois.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_lp.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_scalable.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_vegas.ko 2024-03-06T07:15:56.283 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_veno.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_westwood.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tcp_yeah.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/tunnel4.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/udp_diag.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/udp_tunnel.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv4/xfrm4_tunnel.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ah6.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/esp6.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/fou6.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ip6_tunnel.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ip6_udp_tunnel.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ip6_vti.ko 2024-03-06T07:15:56.284 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/ipcomp6.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6_tables.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_REJECT.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_SYNPROXY.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_ah.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_eui64.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_frag.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_hbh.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_ipv6header.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_mh.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_rpfilter.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6t_rt.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_filter.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_mangle.ko 2024-03-06T07:15:56.285 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_nat.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_raw.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/ip6table_security.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_dup_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_reject_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_socket_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nf_tproxy_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nft_dup_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nft_fib_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/netfilter/nft_reject_ipv6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/sit.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/tunnel6.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/ipv6/xfrm6_tunnel.ko 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/key 2024-03-06T07:15:56.286 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/key/af_key.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_core.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_debugfs.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_eth.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_ip.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_ip6.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_netlink.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/l2tp/l2tp_ppp.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/llc 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/llc/llc.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mac802154 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mac802154/mac802154.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mpls 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/mpls/mpls_gso.ko 2024-03-06T07:15:56.287 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_bitmap_ip.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_bitmap_ipmac.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_bitmap_port.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ip.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipmark.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipport.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipportip.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_ipportnet.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_mac.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_net.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netiface.ko 2024-03-06T07:15:56.288 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netnet.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netport.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_hash_netportnet.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipset/ip_set_list_set.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_dh.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_fo.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_ftp.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_lblc.ko 2024-03-06T07:15:56.289 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_lblcr.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_lc.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_nq.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_pe_sip.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_rr.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_sed.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_sh.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_wlc.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/ipvs/ip_vs_wrr.ko 2024-03-06T07:15:56.290 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conncount.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_amanda.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_broadcast.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_ftp.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_h323.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_irc.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_netbios_ns.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_netlink.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_pptp.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_sane.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_sip.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_snmp.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_conntrack_tftp.ko 2024-03-06T07:15:56.291 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_dup_netdev.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_flow_table.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_flow_table_inet.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_log_syslog.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_amanda.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_ftp.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_irc.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_sip.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_nat_tftp.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_synproxy_core.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nf_tables.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_acct.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_log.ko 2024-03-06T07:15:56.292 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_osf.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nfnetlink_queue.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_chain_nat.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_compat.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_ct.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_dup_netdev.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_fib.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_fwd_netdev.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_hash.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_limit.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_log.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_masq.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_nat.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_numgen.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_queue.ko 2024-03-06T07:15:56.293 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_quota.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_redir.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_reject.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_reject_inet.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_socket.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/nft_tunnel.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_AUDIT.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CHECKSUM.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CLASSIFY.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CONNSECMARK.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_CT.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_DSCP.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_HL.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_HMARK.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_IDLETIMER.ko 2024-03-06T07:15:56.294 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_LED.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_LOG.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_MASQUERADE.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_NETMAP.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_NFLOG.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_NFQUEUE.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_RATEEST.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_REDIRECT.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_SECMARK.ko 2024-03-06T07:15:56.295 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TCPMSS.ko 2024-03-06T07:15:56.296 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TCPOPTSTRIP.ko 2024-03-06T07:15:56.296 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TEE.ko 2024-03-06T07:15:56.296 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TPROXY.ko 2024-03-06T07:15:56.296 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_TRACE.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_addrtype.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_bpf.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_cgroup.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_cluster.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_comment.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connbytes.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connlabel.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connlimit.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_connmark.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_conntrack.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_cpu.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_dccp.ko 2024-03-06T07:15:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_devgroup.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_dscp.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_ecn.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_esp.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_hashlimit.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_helper.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_hl.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_ipcomp.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_iprange.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_ipvs.ko 2024-03-06T07:15:56.298 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_l2tp.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_length.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_limit.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_mac.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_mark.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_multiport.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_nat.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_nfacct.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_osf.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_owner.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_physdev.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_pkttype.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_policy.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_quota.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_rateest.ko 2024-03-06T07:15:56.299 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_realm.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_recent.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_sctp.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_set.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_socket.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_state.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_statistic.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_string.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_tcpmss.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_time.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netfilter/xt_u32.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netlink 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/netlink/netlink_diag.ko 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/nsh 2024-03-06T07:15:56.300 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/nsh/nsh.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/openvswitch.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/vport-geneve.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/vport-gre.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/openvswitch/vport-vxlan.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/packet 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/packet/af_packet_diag.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds/rds.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds/rds_rdma.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/rds/rds_tcp.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_csum.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_gact.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_mirred.ko 2024-03-06T07:15:56.301 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_nat.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_pedit.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_police.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_simple.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_skbedit.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/act_vlan.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_basic.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_bpf.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_flow.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_fw.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_route.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/cls_u32.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_cmp.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_ipset.ko 2024-03-06T07:15:56.302 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_meta.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_nbyte.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_text.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/em_u32.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_choke.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_codel.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_drr.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_fq.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_gred.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_hfsc.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_hhf.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_htb.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_ingress.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_mqprio.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_mqprio_lib.ko 2024-03-06T07:15:56.303 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_multiq.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_netem.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_pie.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_plug.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_prio.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_qfq.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_red.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_sfb.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_sfq.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_tbf.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sched/sch_teql.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sctp 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sctp/sctp.ko 2024-03-06T07:15:56.304 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sctp/sctp_diag.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/auth_gss 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/auth_gss/auth_rpcgss.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/auth_gss/rpcsec_gss_krb5.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/sunrpc.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/xprtrdma 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/sunrpc/xprtrdma/rpcrdma.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/tipc 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/tipc/diag.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/tipc/tipc.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/unix 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/unix/unix_diag.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vmw_vsock_virtio_transport_common.ko 2024-03-06T07:15:56.305 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vmw_vsock_vmci_transport.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vsock.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vsock_diag.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/vmw_vsock/vsock_loopback.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/xfrm 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/net/xfrm/xfrm_ipcomp.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/encrypted-keys 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/encrypted-keys/encrypted-keys.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/trusted-keys 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/security/keys/trusted-keys/trusted.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/virt 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/virt/lib 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/kernel/virt/lib/irqbypass.ko 2024-03-06T07:15:56.306 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.alias 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.alias.bin 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin.alias.bin 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin.bin 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.builtin.modinfo 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.dep 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.dep.bin 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.devname 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.order 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.softdep 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.symbols 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/modules.symbols.bin 2024-03-06T07:15:56.307 INFO:teuthology.orchestra.run.smithi194.stdout:/lib/modules/6.8.0-rc6-gb47552b3c0e3/vmlinuz 2024-03-06T07:15:56.311 DEBUG:teuthology.task.kernel:get_image_version: 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:15:56.312 DEBUG:teuthology.orchestra.run.smithi194:> rpm --scripts -qp /tmp/kernel.x86_64.rpm 2024-03-06T07:15:56.335 INFO:teuthology.orchestra.run.smithi194.stdout:postinstall scriptlet (using /bin/sh): 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:if [ -x /usr/bin/kernel-install ]; then 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout: /usr/bin/kernel-install add 6.8.0-rc6-gb47552b3c0e3 /lib/modules/6.8.0-rc6-gb47552b3c0e3/vmlinuz 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:fi 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:for file in vmlinuz System.map config; do 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout: if ! cmp --silent "/lib/modules/6.8.0-rc6-gb47552b3c0e3/${file}" "/boot/${file}-6.8.0-rc6-gb47552b3c0e3"; then 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout: cp "/lib/modules/6.8.0-rc6-gb47552b3c0e3/${file}" "/boot/${file}-6.8.0-rc6-gb47552b3c0e3" 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout: fi 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:done 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:preuninstall scriptlet (using /bin/sh): 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:if [ -x /sbin/new-kernel-pkg ]; then 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:new-kernel-pkg --remove 6.8.0-rc6-gb47552b3c0e3 --rminitrd --initrdfile=/boot/initramfs-6.8.0-rc6-gb47552b3c0e3.img 2024-03-06T07:15:56.336 INFO:teuthology.orchestra.run.smithi194.stdout:elif [ -x /usr/bin/kernel-install ]; then 2024-03-06T07:15:56.337 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-install remove 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:15:56.337 INFO:teuthology.orchestra.run.smithi194.stdout:fi 2024-03-06T07:15:56.337 INFO:teuthology.orchestra.run.smithi194.stdout:postuninstall scriptlet (using /bin/sh): 2024-03-06T07:15:56.337 INFO:teuthology.orchestra.run.smithi194.stdout:if [ -x /sbin/update-bootloader ]; then 2024-03-06T07:15:56.337 INFO:teuthology.orchestra.run.smithi194.stdout:/sbin/update-bootloader --remove 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:15:56.337 INFO:teuthology.orchestra.run.smithi194.stdout:fi 2024-03-06T07:15:56.338 DEBUG:teuthology.orchestra.run.smithi194:> sudo rpm -qi grub2-tools 2024-03-06T07:15:56.413 INFO:teuthology.orchestra.run.smithi194.stdout:Name : grub2-tools 2024-03-06T07:15:56.413 INFO:teuthology.orchestra.run.smithi194.stdout:Epoch : 1 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Version : 2.06 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Release : 68.el9 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Architecture: x86_64 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Group : Unspecified 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Size : 8264254 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:License : GPLv3+ 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-06T07:15:56.414 INFO:teuthology.orchestra.run.smithi194.stdout:Packager : builder@centos.org 2024-03-06T07:15:56.415 INFO:teuthology.orchestra.run.smithi194.stdout:Vendor : CentOS 2024-03-06T07:15:56.415 INFO:teuthology.orchestra.run.smithi194.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-06T07:15:56.415 INFO:teuthology.orchestra.run.smithi194.stdout:Summary : Support tools for GRUB. 2024-03-06T07:15:56.415 INFO:teuthology.orchestra.run.smithi194.stdout:Description : 2024-03-06T07:15:56.415 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:15:56.415 INFO:teuthology.orchestra.run.smithi194.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-06T07:15:56.415 INFO:teuthology.orchestra.run.smithi194.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-06T07:15:56.416 INFO:teuthology.orchestra.run.smithi194.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-06T07:15:56.416 INFO:teuthology.orchestra.run.smithi194.stdout:hardware devices. 2024-03-06T07:15:56.416 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:15:56.416 INFO:teuthology.orchestra.run.smithi194.stdout:This subpackage provides tools for support of all platforms. 2024-03-06T07:15:56.417 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-06T07:15:56.417 INFO:teuthology.task.kernel:Updating grub on smithi194 to boot 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:15:56.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-06T07:15:57.023 INFO:teuthology.orchestra.run.smithi194.stderr:Generating grub configuration file ... 2024-03-06T07:15:58.443 INFO:teuthology.orchestra.run.smithi194.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-06T07:15:58.464 INFO:teuthology.orchestra.run.smithi194.stderr:done 2024-03-06T07:15:58.466 DEBUG:teuthology.orchestra.run.smithi194:> mktemp 2024-03-06T07:15:58.481 INFO:teuthology.orchestra.run.smithi194.stdout:/tmp/tmp.yJWrIH8axe 2024-03-06T07:15:58.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.yJWrIH8axe 2024-03-06T07:15:58.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 0666 /tmp/tmp.yJWrIH8axe 2024-03-06T07:15:58.712 DEBUG:teuthology.orchestra.remote:smithi194:/tmp/tmp.yJWrIH8axe is 6KB 2024-03-06T07:15:58.761 DEBUG:teuthology.orchestra.run.smithi194:> rm -fr /tmp/tmp.yJWrIH8axe 2024-03-06T07:15:58.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo /bin/ls /boot/loader/entries || true 2024-03-06T07:15:58.839 INFO:teuthology.orchestra.run.smithi194.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-06T07:15:58.840 INFO:teuthology.orchestra.run.smithi194.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-06T07:15:58.840 INFO:teuthology.orchestra.run.smithi194.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-06T07:15:58.840 INFO:teuthology.orchestra.run.smithi194.stdout:15949e3c88704300882482eddfd7baae-6.8.0-rc6-gb47552b3c0e3.conf 2024-03-06T07:15:58.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:15:58.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo shutdown -r now 2024-03-06T07:16:28.979 INFO:teuthology.task.kernel:Checking client client.1 for new kernel version... 2024-03-06T07:16:28.981 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-03-06T07:16:28.982 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:17:28.985 DEBUG:teuthology.orchestra.remote:timed out 2024-03-06T07:17:37.986 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-03-06T07:17:37.987 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:17:45.646 DEBUG:teuthology.orchestra.run.smithi194:> true 2024-03-06T07:17:46.186 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-03-06T07:17:46.186 INFO:teuthology.task.kernel:Checking kernel version of client.1, want "6.8.0-rc6-gb47552b3c0e3"... 2024-03-06T07:17:46.186 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-03-06T07:17:46.230 INFO:teuthology.orchestra.run.smithi194.stdout:6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:17:46.230 DEBUG:teuthology.task.kernel:current kernel version is 6.8.0-rc6-gb47552b3c0e3 vs 6.8.0-rc6-gb47552b3c0e3 2024-03-06T07:17:46.230 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-06T07:17:46.230 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-06T07:17:47.233 DEBUG:teuthology.parallel:result is None 2024-03-06T07:17:47.233 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-06T07:17:47.241 INFO:teuthology.task.internal:Creating test directory... 2024-03-06T07:17:47.241 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-06T07:17:47.250 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-06T07:17:47.300 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-06T07:17:47.307 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-06T07:17:47.329 INFO:teuthology.task.internal:Creating archive directory... 2024-03-06T07:17:47.330 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-06T07:17:47.337 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-06T07:17:47.403 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-06T07:17:47.411 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-06T07:17:47.411 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-06T07:17:47.418 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-06T07:17:47.538 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-06T07:17:47.564 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-06T07:17:47.623 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-06T07:17:47.625 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-06T07:17:47.631 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-06T07:17:47.640 INFO:teuthology.task.internal:Configuring sudo... 2024-03-06T07:17:47.640 DEBUG:teuthology.orchestra.run.smithi086:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-06T07:17:47.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-06T07:17:47.734 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-06T07:17:47.743 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-06T07:17:47.744 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-06T07:17:47.749 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-06T07:17:47.796 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-06T07:17:47.869 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-06T07:17:47.980 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-06T07:17:48.065 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-06T07:17:48.161 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:17:48.162 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-06T07:17:48.253 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-06T07:17:48.332 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-06T07:17:48.435 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-06T07:17:48.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-06T07:17:48.583 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:17:48.583 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-06T07:17:48.705 DEBUG:teuthology.orchestra.run.smithi086:> sudo service rsyslog restart 2024-03-06T07:17:48.713 DEBUG:teuthology.orchestra.run.smithi194:> sudo service rsyslog restart 2024-03-06T07:17:48.816 INFO:teuthology.orchestra.run.smithi194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-06T07:17:48.846 INFO:teuthology.orchestra.run.smithi086.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-06T07:17:49.181 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-06T07:17:49.193 INFO:teuthology.task.internal:Starting timer... 2024-03-06T07:17:49.193 INFO:teuthology.run_tasks:Running task pcp... 2024-03-06T07:17:49.267 INFO:teuthology.run_tasks:Running task selinux... 2024-03-06T07:17:49.300 DEBUG:teuthology.orchestra.run.smithi086:> sudo service auditd rotate 2024-03-06T07:17:49.420 INFO:teuthology.orchestra.run.smithi086.stdout:Rotating logs: 2024-03-06T07:17:49.425 DEBUG:teuthology.orchestra.run.smithi194:> sudo service auditd rotate 2024-03-06T07:17:49.542 INFO:teuthology.orchestra.run.smithi194.stdout:Rotating logs: 2024-03-06T07:17:49.547 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-06T07:17:49.547 DEBUG:teuthology.orchestra.run.smithi086:> /usr/sbin/getenforce 2024-03-06T07:17:49.599 INFO:teuthology.orchestra.run.smithi086.stdout:Permissive 2024-03-06T07:17:49.600 DEBUG:teuthology.orchestra.run.smithi194:> /usr/sbin/getenforce 2024-03-06T07:17:49.659 INFO:teuthology.orchestra.run.smithi194.stdout:Permissive 2024-03-06T07:17:49.659 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi086.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi194.front.sepia.ceph.com': 'permissive'} 2024-03-06T07:17:49.659 DEBUG:teuthology.orchestra.run.smithi086:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-03-06T07:17:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:17:49.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-03-06T07:17:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:17:49.815 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-06T07:17:49.815 DEBUG:teuthology.orchestra.run.smithi086:> sudo /usr/sbin/setenforce permissive 2024-03-06T07:17:49.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo /usr/sbin/setenforce permissive 2024-03-06T07:17:49.997 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-06T07:17:50.007 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-06T07:17:50.024 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-06T07:17:50.025 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi086.front.sepia.ceph.com,smithi194.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-06T07:23:44.711 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi086.front.sepia.ceph.com'), Remote(name='ubuntu@smithi194.front.sepia.ceph.com')] 2024-03-06T07:23:44.712 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-03-06T07:23:44.714 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:23:44.859 DEBUG:teuthology.orchestra.run.smithi086:> true 2024-03-06T07:23:45.017 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-03-06T07:23:45.017 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-03-06T07:23:45.018 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-06T07:23:45.169 DEBUG:teuthology.orchestra.run.smithi194:> true 2024-03-06T07:23:45.318 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-03-06T07:23:45.319 INFO:teuthology.run_tasks:Running task clock... 2024-03-06T07:23:45.330 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-06T07:23:45.330 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-06T07:23:45.330 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-06T07:23:45.338 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-06T07:23:45.338 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-06T07:23:45.436 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-06T07:23:45.438 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-06T07:23:45.487 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-06T07:23:45.488 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-06T07:23:45.574 INFO:teuthology.orchestra.run.smithi086.stderr:sudo: ntpd: command not found 2024-03-06T07:23:45.579 INFO:teuthology.orchestra.run.smithi194.stderr:sudo: ntpd: command not found 2024-03-06T07:23:45.615 INFO:teuthology.orchestra.run.smithi086.stdout:506 Cannot talk to daemon 2024-03-06T07:23:45.619 INFO:teuthology.orchestra.run.smithi194.stdout:506 Cannot talk to daemon 2024-03-06T07:23:45.668 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-06T07:23:45.668 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-06T07:23:45.716 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-06T07:23:45.722 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-06T07:23:45.846 INFO:teuthology.orchestra.run.smithi194.stderr:bash: line 1: ntpq: command not found 2024-03-06T07:23:45.848 INFO:teuthology.orchestra.run.smithi086.stderr:bash: line 1: ntpq: command not found 2024-03-06T07:23:45.853 INFO:teuthology.orchestra.run.smithi194.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-06T07:23:45.854 INFO:teuthology.orchestra.run.smithi194.stdout:=============================================================================== 2024-03-06T07:23:45.854 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.854 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.854 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.854 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.856 INFO:teuthology.orchestra.run.smithi086.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-06T07:23:45.856 INFO:teuthology.orchestra.run.smithi086.stdout:=============================================================================== 2024-03-06T07:23:45.856 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.856 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.856 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.857 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-06T07:23:45.858 INFO:teuthology.run_tasks:Running task install... 2024-03-06T07:23:45.868 DEBUG:teuthology.task.install:project ceph 2024-03-06T07:23:45.868 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}} 2024-03-06T07:23:45.868 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'} 2024-03-06T07:23:45.868 INFO:teuthology.task.install:Using flavor: default 2024-03-06T07:23:45.874 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-06T07:23:45.874 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']} 2024-03-06T07:23:45.874 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'tag': None, 'wait_for_package': False} 2024-03-06T07:23:45.875 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-06T07:23:45.877 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'tag': None, 'wait_for_package': False} 2024-03-06T07:23:45.877 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-06T07:23:46.035 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/ 2024-03-06T07:23:46.035 INFO:teuthology.task.install.rpm:Package version is 19.0.0-1739.g1bc250c7 2024-03-06T07:23:46.109 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/ 2024-03-06T07:23:46.109 INFO:teuthology.task.install.rpm:Package version is 19.0.0-1739.g1bc250c7 2024-03-06T07:23:46.175 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-06T07:23:46.175 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:23:46.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-06T07:23:46.277 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-03-06T07:23:46.277 DEBUG:teuthology.orchestra.run.smithi194:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-06T07:23:46.294 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-06T07:23:46.294 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:23:46.294 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-06T07:23:46.394 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-03-06T07:23:46.394 DEBUG:teuthology.orchestra.run.smithi086:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-06T07:23:46.410 DEBUG:teuthology.orchestra.run.smithi194:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-06T07:23:46.533 DEBUG:teuthology.orchestra.run.smithi086:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-06T07:23:46.542 DEBUG:teuthology.orchestra.run.smithi194:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-06T07:23:46.628 INFO:teuthology.orchestra.run.smithi194.stdout:check_obsoletes = 1 2024-03-06T07:23:46.632 DEBUG:teuthology.orchestra.run.smithi194:> sudo yum clean all 2024-03-06T07:23:46.656 DEBUG:teuthology.orchestra.run.smithi086:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-06T07:23:46.745 INFO:teuthology.orchestra.run.smithi086.stdout:check_obsoletes = 1 2024-03-06T07:23:46.750 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum clean all 2024-03-06T07:23:46.994 INFO:teuthology.orchestra.run.smithi194.stdout:74 files removed 2024-03-06T07:23:47.036 DEBUG:teuthology.orchestra.run.smithi194:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-03-06T07:23:47.110 INFO:teuthology.orchestra.run.smithi086.stdout:74 files removed 2024-03-06T07:23:47.149 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-03-06T07:23:47.989 INFO:teuthology.orchestra.run.smithi194.stdout:ceph packages for x86_64 186 kB/s | 83 kB 00:00 2024-03-06T07:23:48.118 INFO:teuthology.orchestra.run.smithi086.stdout:ceph packages for x86_64 173 kB/s | 83 kB 00:00 2024-03-06T07:23:48.630 INFO:teuthology.orchestra.run.smithi194.stdout:ceph noarch packages 18 kB/s | 10 kB 00:00 2024-03-06T07:23:48.652 INFO:teuthology.orchestra.run.smithi086.stdout:ceph noarch packages 23 kB/s | 10 kB 00:00 2024-03-06T07:23:49.147 INFO:teuthology.orchestra.run.smithi194.stdout:ceph source packages 4.3 kB/s | 1.9 kB 00:00 2024-03-06T07:23:49.236 INFO:teuthology.orchestra.run.smithi086.stdout:ceph source packages 3.7 kB/s | 1.9 kB 00:00 2024-03-06T07:23:49.842 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-06T07:23:50.488 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS-9 - AppStream 5.7 MB/s | 6.7 MB 00:01 2024-03-06T07:23:51.568 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS-9 - BaseOS 4.7 MB/s | 2.0 MB 00:00 2024-03-06T07:23:52.098 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS-9 - BaseOS 5.8 MB/s | 2.0 MB 00:00 2024-03-06T07:23:52.722 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.0 MB 00:00 2024-03-06T07:23:55.565 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - AppStream 12 MB/s | 19 MB 00:01 2024-03-06T07:23:56.235 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - BaseOS 2.2 MB/s | 8.0 MB 00:03 2024-03-06T07:23:59.370 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - AppStream 9.8 MB/s | 19 MB 00:01 2024-03-06T07:24:00.191 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - CRB 7.0 MB/s | 6.1 MB 00:00 2024-03-06T07:24:01.808 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - Extras packages 54 kB/s | 15 kB 00:00 2024-03-06T07:24:04.329 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - CRB 5.3 MB/s | 6.1 MB 00:01 2024-03-06T07:24:04.790 INFO:teuthology.orchestra.run.smithi194.stdout:Extra Packages for Enterprise Linux 7.3 MB/s | 21 MB 00:02 2024-03-06T07:24:06.213 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - Extras packages 29 kB/s | 15 kB 00:00 2024-03-06T07:24:08.062 INFO:teuthology.orchestra.run.smithi086.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-03-06T07:24:10.618 INFO:teuthology.orchestra.run.smithi194.stdout:lab-extras 23 kB/s | 1.7 kB 00:00 2024-03-06T07:24:13.338 INFO:teuthology.orchestra.run.smithi194.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-06T07:24:13.339 INFO:teuthology.orchestra.run.smithi194.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-06T07:24:13.359 INFO:teuthology.orchestra.run.smithi194.stdout:Package NetworkManager-1:1.44.0-2.el9.x86_64 is already installed. 2024-03-06T07:24:13.361 INFO:teuthology.orchestra.run.smithi194.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-03-06T07:24:13.362 INFO:teuthology.orchestra.run.smithi194.stdout:Package util-linux-2.37.4-15.el9.x86_64 is already installed. 2024-03-06T07:24:13.363 INFO:teuthology.orchestra.run.smithi194.stdout:Package libacl-devel-2.3.1-3.el9.x86_64 is already installed. 2024-03-06T07:24:13.366 INFO:teuthology.orchestra.run.smithi194.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-03-06T07:24:13.369 INFO:teuthology.orchestra.run.smithi194.stdout:Package xfsdump-3.1.12-3.el9.x86_64 is already installed. 2024-03-06T07:24:13.370 INFO:teuthology.orchestra.run.smithi194.stdout:Package xfsprogs-5.19.0-4.el9.x86_64 is already installed. 2024-03-06T07:24:13.460 INFO:teuthology.orchestra.run.smithi194.stdout:Dependencies resolved. 2024-03-06T07:24:13.496 INFO:teuthology.orchestra.run.smithi194.stdout:============================================================================================ 2024-03-06T07:24:13.496 INFO:teuthology.orchestra.run.smithi194.stdout: Package Arch Version Repository Size 2024-03-06T07:24:13.496 INFO:teuthology.orchestra.run.smithi194.stdout:============================================================================================ 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout:Installing: 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: bison x86_64 3.7.4-5.el9 CentOS-AppStream 920 k 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 6.6 k 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-base x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.3 M 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 888 k 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 145 k 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 1.5 M 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 138 k 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-dashboard noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 1.8 M 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 7.4 M 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-rook noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 49 k 2024-03-06T07:24:13.497 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-radosgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 9.8 M 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-test x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 47 M 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-volume noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 264 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 748 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: cephfs-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 220 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: cephfs-shell x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 27 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: cephfs-top noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 18 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 CentOS-AppStream 20 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: flex x86_64 2.6.4-9.el9 CentOS-AppStream 307 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: libaio-devel x86_64 0.3.111-13.el9 CentOS-AppStream 10 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: libcephfs-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 31 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: libcephfs2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 747 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: librados-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 127 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: libtool x86_64 2.4.6-45.el9 CentOS-AppStream 578 k 2024-03-06T07:24:13.498 INFO:teuthology.orchestra.run.smithi194.stdout: libuuid-devel x86_64 2.37.4-18.el9 CentOS-AppStream 24 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: openssl-devel x86_64 1:3.0.7-27.el9 CentOS-AppStream 3.0 M 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql x86_64 13.11-1.el9 CentOS-AppStream 1.5 M 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql-contrib x86_64 13.11-1.el9 CentOS-AppStream 811 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql-server x86_64 13.11-1.el9 CentOS-AppStream 5.7 M 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cephfs x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 162 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rados x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 321 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 298 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 99 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: rbd-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 85 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: rbd-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: rbd-nbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 171 k 2024-03-06T07:24:13.499 INFO:teuthology.orchestra.run.smithi194.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 CentOS-AppStream 87 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout:Upgrading: 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager x86_64 1:1.46.0-1.el9 CentOS-BaseOS 2.3 M 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager-libnm x86_64 1:1.46.0-1.el9 CentOS-BaseOS 1.8 M 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager-team x86_64 1:1.46.0-1.el9 CentOS-BaseOS 41 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager-tui x86_64 1:1.46.0-1.el9 CentOS-BaseOS 247 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: acl x86_64 2.3.1-4.el9 CentOS-BaseOS 69 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-debuginfod-client x86_64 0.190-2.el9 CentOS-BaseOS 35 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-libelf x86_64 0.190-2.el9 CentOS-BaseOS 191 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-libs x86_64 0.190-2.el9 CentOS-BaseOS 254 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: iproute x86_64 6.7.0-2.el9 CentOS-BaseOS 802 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: iproute-tc x86_64 6.7.0-2.el9 CentOS-BaseOS 433 k 2024-03-06T07:24:13.500 INFO:teuthology.orchestra.run.smithi194.stdout: libacl x86_64 2.3.1-4.el9 CentOS-BaseOS 22 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: libacl-devel x86_64 2.3.1-4.el9 CentOS-AppStream 68 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: libblkid x86_64 2.37.4-18.el9 CentOS-BaseOS 107 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: libfdisk x86_64 2.37.4-18.el9 CentOS-BaseOS 154 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: libmount x86_64 2.37.4-18.el9 CentOS-BaseOS 135 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: librados2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.4 M 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: librbd1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: libsmartcols x86_64 2.37.4-18.el9 CentOS-BaseOS 63 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: libuuid x86_64 2.37.4-18.el9 CentOS-BaseOS 28 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: openssl x86_64 1:3.0.7-27.el9 CentOS-BaseOS 1.2 M 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: openssl-libs x86_64 1:3.0.7-27.el9 CentOS-BaseOS 2.2 M 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: util-linux x86_64 2.37.4-18.el9 CentOS-BaseOS 2.2 M 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: util-linux-core x86_64 2.37.4-18.el9 CentOS-BaseOS 438 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: util-linux-user x86_64 2.37.4-18.el9 CentOS-BaseOS 30 k 2024-03-06T07:24:13.501 INFO:teuthology.orchestra.run.smithi194.stdout: xfsdump x86_64 3.1.12-4.el9 CentOS-BaseOS 321 k 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: xfsprogs x86_64 6.3.0-1.el9 CentOS-BaseOS 1.1 M 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout:Installing dependencies: 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: automake noarch 1.16.2-8.el9 CentOS-AppStream 662 k 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: boost-program-options x86_64 1.75.0-8.el9 CentOS-AppStream 106 k 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 20 M 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-grafana-dashboards noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 31 k 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mds x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 2.3 M 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-modules-core noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 248 k 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mon x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 4.7 M 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-osd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 17 M 2024-03-06T07:24:13.502 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-prometheus-alerts noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 15 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-selinux x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 25 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: jq x86_64 1.6-16.el9 CentOS-BaseOS 186 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: libcephsqlite x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 163 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: libgfortran x86_64 11.4.1-3.el9 CentOS-BaseOS 802 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-06T07:24:13.503 INFO:teuthology.orchestra.run.smithi194.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: libquadmath x86_64 11.4.1-3.el9 CentOS-BaseOS 194 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: libradosstriper1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 501 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: librgw2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.1 M 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: libzstd-devel x86_64 1.5.1-2.el9 CentOS-AppStream 46 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: lua x86_64 5.4.4-4.el9 CentOS-AppStream 187 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-BaseOS 217 k 2024-03-06T07:24:13.504 INFO:teuthology.orchestra.run.smithi194.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql-private-libs x86_64 13.11-1.el9 CentOS-AppStream 134 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: protobuf x86_64 3.14.0-13.el9 CentOS-AppStream 1.0 M 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-asyncssh noarch 2.13.2-4.el9 epel 548 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-attrs noarch 20.3.0-7.el9 CentOS-AppStream 78 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ceph-argparse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 45 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 131 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-06T07:24:13.505 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-google-auth noarch 1:2.28.0-1.el9 epel 203 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-06T07:24:13.506 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jinja2 noarch 2.11.3-5.el9 CentOS-AppStream 227 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-packaging noarch 20.9-5.el9 CentOS-AppStream 69 k 2024-03-06T07:24:13.507 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-protobuf noarch 3.14.0-13.el9 CentOS-AppStream 238 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-requests noarch 2.25.1-8.el9 CentOS-BaseOS 113 k 2024-03-06T07:24:13.508 INFO:teuthology.orchestra.run.smithi194.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-wcwidth noarch 0.2.5-8.el9 CentOS-AppStream 41 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: qatlib x86_64 23.11.0-1.el9 CentOS-AppStream 215 k 2024-03-06T07:24:13.509 INFO:teuthology.orchestra.run.smithi194.stdout: qatzip-libs x86_64 1.1.2-2.el9 CentOS-AppStream 43 k 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: uuid x86_64 1.6.2-55.el9 CentOS-AppStream 56 k 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: zlib-devel x86_64 1.2.11-40.el9 CentOS-AppStream 44 k 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout:Installing weak dependencies: 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: qatlib-service x86_64 23.11.0-1.el9 CentOS-AppStream 34 k 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction Summary 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout:============================================================================================ 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout:Install 150 Packages 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout:Upgrade 26 Packages 2024-03-06T07:24:13.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:24:13.511 INFO:teuthology.orchestra.run.smithi194.stdout:Total download size: 224 M 2024-03-06T07:24:13.511 INFO:teuthology.orchestra.run.smithi194.stdout:Downloading Packages: 2024-03-06T07:24:13.798 INFO:teuthology.orchestra.run.smithi086.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-03-06T07:24:14.435 INFO:teuthology.orchestra.run.smithi194.stdout:(1/176): ceph-19.0.0-1739.g1bc250c7.el9.x86_64. 22 kB/s | 6.6 kB 00:00 2024-03-06T07:24:14.760 INFO:teuthology.orchestra.run.smithi194.stdout:(2/176): ceph-fuse-19.0.0-1739.g1bc250c7.el9.x8 2.7 MB/s | 888 kB 00:00 2024-03-06T07:24:14.869 INFO:teuthology.orchestra.run.smithi194.stdout:(3/176): ceph-immutable-object-cache-19.0.0-173 1.3 MB/s | 145 kB 00:00 2024-03-06T07:24:15.078 INFO:teuthology.orchestra.run.smithi194.stdout:(4/176): ceph-base-19.0.0-1739.g1bc250c7.el9.x8 5.7 MB/s | 5.3 MB 00:00 2024-03-06T07:24:15.194 INFO:teuthology.orchestra.run.smithi194.stdout:(5/176): ceph-mds-19.0.0-1739.g1bc250c7.el9.x86 7.1 MB/s | 2.3 MB 00:00 2024-03-06T07:24:15.270 INFO:teuthology.orchestra.run.smithi194.stdout:(6/176): ceph-mgr-19.0.0-1739.g1bc250c7.el9.x86 7.8 MB/s | 1.5 MB 00:00 2024-03-06T07:24:15.787 INFO:teuthology.orchestra.run.smithi194.stdout:(7/176): ceph-mon-19.0.0-1739.g1bc250c7.el9.x86 7.9 MB/s | 4.7 MB 00:00 2024-03-06T07:24:16.467 INFO:teuthology.orchestra.run.smithi086.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-06T07:24:16.468 INFO:teuthology.orchestra.run.smithi086.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-06T07:24:16.488 INFO:teuthology.orchestra.run.smithi086.stdout:Package NetworkManager-1:1.44.0-2.el9.x86_64 is already installed. 2024-03-06T07:24:16.489 INFO:teuthology.orchestra.run.smithi086.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-03-06T07:24:16.491 INFO:teuthology.orchestra.run.smithi086.stdout:Package util-linux-2.37.4-15.el9.x86_64 is already installed. 2024-03-06T07:24:16.492 INFO:teuthology.orchestra.run.smithi086.stdout:Package libacl-devel-2.3.1-3.el9.x86_64 is already installed. 2024-03-06T07:24:16.494 INFO:teuthology.orchestra.run.smithi086.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-03-06T07:24:16.498 INFO:teuthology.orchestra.run.smithi086.stdout:Package xfsdump-3.1.12-3.el9.x86_64 is already installed. 2024-03-06T07:24:16.499 INFO:teuthology.orchestra.run.smithi086.stdout:Package xfsprogs-5.19.0-4.el9.x86_64 is already installed. 2024-03-06T07:24:16.589 INFO:teuthology.orchestra.run.smithi086.stdout:Dependencies resolved. 2024-03-06T07:24:16.595 INFO:teuthology.orchestra.run.smithi194.stdout:(8/176): ceph-common-19.0.0-1739.g1bc250c7.el9. 8.3 MB/s | 20 MB 00:02 2024-03-06T07:24:16.609 INFO:teuthology.orchestra.run.smithi086.stdout:============================================================================================ 2024-03-06T07:24:16.609 INFO:teuthology.orchestra.run.smithi086.stdout: Package Arch Version Repository Size 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout:============================================================================================ 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout:Installing: 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: bison x86_64 3.7.4-5.el9 CentOS-AppStream 920 k 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 6.6 k 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-base x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.3 M 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 888 k 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 145 k 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 1.5 M 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 138 k 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-dashboard noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 1.8 M 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 7.4 M 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-rook noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 49 k 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-radosgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 9.8 M 2024-03-06T07:24:16.610 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-test x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 47 M 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-volume noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 264 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 748 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: cephfs-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 220 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: cephfs-shell x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 27 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: cephfs-top noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 18 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 CentOS-AppStream 20 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: flex x86_64 2.6.4-9.el9 CentOS-AppStream 307 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: libaio-devel x86_64 0.3.111-13.el9 CentOS-AppStream 10 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: libcephfs-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 31 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: libcephfs2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 747 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: librados-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 127 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: libtool x86_64 2.4.6-45.el9 CentOS-AppStream 578 k 2024-03-06T07:24:16.611 INFO:teuthology.orchestra.run.smithi086.stdout: libuuid-devel x86_64 2.37.4-18.el9 CentOS-AppStream 24 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: openssl-devel x86_64 1:3.0.7-27.el9 CentOS-AppStream 3.0 M 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql x86_64 13.11-1.el9 CentOS-AppStream 1.5 M 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql-contrib x86_64 13.11-1.el9 CentOS-AppStream 811 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql-server x86_64 13.11-1.el9 CentOS-AppStream 5.7 M 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cephfs x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 162 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rados x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 321 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 298 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 99 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: rbd-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 85 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: rbd-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: rbd-nbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 171 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 CentOS-AppStream 87 k 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout:Upgrading: 2024-03-06T07:24:16.612 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager x86_64 1:1.46.0-1.el9 CentOS-BaseOS 2.3 M 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager-libnm x86_64 1:1.46.0-1.el9 CentOS-BaseOS 1.8 M 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager-team x86_64 1:1.46.0-1.el9 CentOS-BaseOS 41 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager-tui x86_64 1:1.46.0-1.el9 CentOS-BaseOS 247 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: acl x86_64 2.3.1-4.el9 CentOS-BaseOS 69 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-debuginfod-client x86_64 0.190-2.el9 CentOS-BaseOS 35 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-libelf x86_64 0.190-2.el9 CentOS-BaseOS 191 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-libs x86_64 0.190-2.el9 CentOS-BaseOS 254 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: iproute x86_64 6.7.0-2.el9 CentOS-BaseOS 802 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: iproute-tc x86_64 6.7.0-2.el9 CentOS-BaseOS 433 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: libacl x86_64 2.3.1-4.el9 CentOS-BaseOS 22 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: libacl-devel x86_64 2.3.1-4.el9 CentOS-AppStream 68 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: libblkid x86_64 2.37.4-18.el9 CentOS-BaseOS 107 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: libfdisk x86_64 2.37.4-18.el9 CentOS-BaseOS 154 k 2024-03-06T07:24:16.613 INFO:teuthology.orchestra.run.smithi086.stdout: libmount x86_64 2.37.4-18.el9 CentOS-BaseOS 135 k 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: librados2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.4 M 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: librbd1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: libsmartcols x86_64 2.37.4-18.el9 CentOS-BaseOS 63 k 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: libuuid x86_64 2.37.4-18.el9 CentOS-BaseOS 28 k 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: openssl x86_64 1:3.0.7-27.el9 CentOS-BaseOS 1.2 M 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: openssl-libs x86_64 1:3.0.7-27.el9 CentOS-BaseOS 2.2 M 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: util-linux x86_64 2.37.4-18.el9 CentOS-BaseOS 2.2 M 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: util-linux-core x86_64 2.37.4-18.el9 CentOS-BaseOS 438 k 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: util-linux-user x86_64 2.37.4-18.el9 CentOS-BaseOS 30 k 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: xfsdump x86_64 3.1.12-4.el9 CentOS-BaseOS 321 k 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: xfsprogs x86_64 6.3.0-1.el9 CentOS-BaseOS 1.1 M 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout:Installing dependencies: 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-03-06T07:24:16.614 INFO:teuthology.orchestra.run.smithi086.stdout: automake noarch 1.16.2-8.el9 CentOS-AppStream 662 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: boost-program-options x86_64 1.75.0-8.el9 CentOS-AppStream 106 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 20 M 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-grafana-dashboards noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 31 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mds x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 2.3 M 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-modules-core noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 248 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mon x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 4.7 M 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-osd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 17 M 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-prometheus-alerts noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 15 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-selinux x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 25 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-06T07:24:16.615 INFO:teuthology.orchestra.run.smithi086.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: jq x86_64 1.6-16.el9 CentOS-BaseOS 186 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libcephsqlite x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 163 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libgfortran x86_64 11.4.1-3.el9 CentOS-BaseOS 802 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libquadmath x86_64 11.4.1-3.el9 CentOS-BaseOS 194 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libradosstriper1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 501 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: librgw2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.1 M 2024-03-06T07:24:16.616 INFO:teuthology.orchestra.run.smithi086.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: libzstd-devel x86_64 1.5.1-2.el9 CentOS-AppStream 46 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: lua x86_64 5.4.4-4.el9 CentOS-AppStream 187 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-BaseOS 217 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql-private-libs x86_64 13.11-1.el9 CentOS-AppStream 134 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: protobuf x86_64 3.14.0-13.el9 CentOS-AppStream 1.0 M 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: python3-asyncssh noarch 2.13.2-4.el9 epel 548 k 2024-03-06T07:24:16.617 INFO:teuthology.orchestra.run.smithi086.stdout: python3-attrs noarch 20.3.0-7.el9 CentOS-AppStream 78 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-ceph-argparse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 45 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 131 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-06T07:24:16.618 INFO:teuthology.orchestra.run.smithi086.stdout: python3-google-auth noarch 1:2.28.0-1.el9 epel 203 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jinja2 noarch 2.11.3-5.el9 CentOS-AppStream 227 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-06T07:24:16.619 INFO:teuthology.orchestra.run.smithi086.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-packaging noarch 20.9-5.el9 CentOS-AppStream 69 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-protobuf noarch 3.14.0-13.el9 CentOS-AppStream 238 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-06T07:24:16.620 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-requests noarch 2.25.1-8.el9 CentOS-BaseOS 113 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-wcwidth noarch 0.2.5-8.el9 CentOS-AppStream 41 k 2024-03-06T07:24:16.621 INFO:teuthology.orchestra.run.smithi086.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: qatlib x86_64 23.11.0-1.el9 CentOS-AppStream 215 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: qatzip-libs x86_64 1.1.2-2.el9 CentOS-AppStream 43 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: uuid x86_64 1.6.2-55.el9 CentOS-AppStream 56 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: zlib-devel x86_64 1.2.11-40.el9 CentOS-AppStream 44 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout:Installing weak dependencies: 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: qatlib-service x86_64 23.11.0-1.el9 CentOS-AppStream 34 k 2024-03-06T07:24:16.622 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:24:16.623 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction Summary 2024-03-06T07:24:16.623 INFO:teuthology.orchestra.run.smithi086.stdout:============================================================================================ 2024-03-06T07:24:16.623 INFO:teuthology.orchestra.run.smithi086.stdout:Install 150 Packages 2024-03-06T07:24:16.623 INFO:teuthology.orchestra.run.smithi086.stdout:Upgrade 26 Packages 2024-03-06T07:24:16.623 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:24:16.623 INFO:teuthology.orchestra.run.smithi086.stdout:Total download size: 224 M 2024-03-06T07:24:16.623 INFO:teuthology.orchestra.run.smithi086.stdout:Downloading Packages: 2024-03-06T07:24:16.695 INFO:teuthology.orchestra.run.smithi194.stdout:(9/176): ceph-selinux-19.0.0-1739.g1bc250c7.el9 254 kB/s | 25 kB 00:00 2024-03-06T07:24:17.221 INFO:teuthology.orchestra.run.smithi194.stdout:(10/176): ceph-osd-19.0.0-1739.g1bc250c7.el9.x8 8.6 MB/s | 17 MB 00:01 2024-03-06T07:24:17.371 INFO:teuthology.orchestra.run.smithi194.stdout:(11/176): ceph-radosgw-19.0.0-1739.g1bc250c7.el 6.2 MB/s | 9.8 MB 00:01 2024-03-06T07:24:17.383 INFO:teuthology.orchestra.run.smithi086.stdout:(1/176): ceph-19.0.0-1739.g1bc250c7.el9.x86_64. 26 kB/s | 6.6 kB 00:00 2024-03-06T07:24:17.405 INFO:teuthology.orchestra.run.smithi194.stdout:(12/176): cephfs-mirror-19.0.0-1739.g1bc250c7.e 1.2 MB/s | 220 kB 00:00 2024-03-06T07:24:17.455 INFO:teuthology.orchestra.run.smithi194.stdout:(13/176): cephfs-shell-19.0.0-1739.g1bc250c7.el 328 kB/s | 27 kB 00:00 2024-03-06T07:24:17.480 INFO:teuthology.orchestra.run.smithi194.stdout:(14/176): libcephfs-devel-19.0.0-1739.g1bc250c7 419 kB/s | 31 kB 00:00 2024-03-06T07:24:17.556 INFO:teuthology.orchestra.run.smithi194.stdout:(15/176): libcephsqlite-19.0.0-1739.g1bc250c7.e 2.1 MB/s | 163 kB 00:00 2024-03-06T07:24:17.597 INFO:teuthology.orchestra.run.smithi194.stdout:(16/176): libcephfs2-19.0.0-1739.g1bc250c7.el9. 5.2 MB/s | 747 kB 00:00 2024-03-06T07:24:17.648 INFO:teuthology.orchestra.run.smithi194.stdout:(17/176): librados-devel-19.0.0-1739.g1bc250c7. 1.4 MB/s | 127 kB 00:00 2024-03-06T07:24:17.698 INFO:teuthology.orchestra.run.smithi194.stdout:(18/176): libradosstriper1-19.0.0-1739.g1bc250c 4.9 MB/s | 501 kB 00:00 2024-03-06T07:24:17.758 INFO:teuthology.orchestra.run.smithi086.stdout:(2/176): ceph-fuse-19.0.0-1739.g1bc250c7.el9.x8 2.3 MB/s | 888 kB 00:00 2024-03-06T07:24:17.790 INFO:teuthology.orchestra.run.smithi194.stdout:(19/176): python3-ceph-argparse-19.0.0-1739.g1b 495 kB/s | 45 kB 00:00 2024-03-06T07:24:17.850 INFO:teuthology.orchestra.run.smithi086.stdout:(3/176): ceph-immutable-object-cache-19.0.0-173 1.6 MB/s | 145 kB 00:00 2024-03-06T07:24:17.898 INFO:teuthology.orchestra.run.smithi194.stdout:(20/176): python3-ceph-common-19.0.0-1739.g1bc2 1.2 MB/s | 131 kB 00:00 2024-03-06T07:24:18.007 INFO:teuthology.orchestra.run.smithi194.stdout:(21/176): python3-cephfs-19.0.0-1739.g1bc250c7. 1.5 MB/s | 162 kB 00:00 2024-03-06T07:24:18.116 INFO:teuthology.orchestra.run.smithi194.stdout:(22/176): python3-rados-19.0.0-1739.g1bc250c7.e 2.9 MB/s | 321 kB 00:00 2024-03-06T07:24:18.232 INFO:teuthology.orchestra.run.smithi194.stdout:(23/176): python3-rbd-19.0.0-1739.g1bc250c7.el9 2.5 MB/s | 298 kB 00:00 2024-03-06T07:24:18.374 INFO:teuthology.orchestra.run.smithi194.stdout:(24/176): librgw2-19.0.0-1739.g1bc250c7.el9.x86 7.1 MB/s | 5.1 MB 00:00 2024-03-06T07:24:18.408 INFO:teuthology.orchestra.run.smithi194.stdout:(25/176): python3-rgw-19.0.0-1739.g1bc250c7.el9 569 kB/s | 99 kB 00:00 2024-03-06T07:24:18.483 INFO:teuthology.orchestra.run.smithi194.stdout:(26/176): rbd-fuse-19.0.0-1739.g1bc250c7.el9.x8 791 kB/s | 85 kB 00:00 2024-03-06T07:24:18.575 INFO:teuthology.orchestra.run.smithi086.stdout:(4/176): ceph-mds-19.0.0-1739.g1bc250c7.el9.x86 3.2 MB/s | 2.3 MB 00:00 2024-03-06T07:24:18.592 INFO:teuthology.orchestra.run.smithi194.stdout:(27/176): rbd-nbd-19.0.0-1739.g1bc250c7.el9.x86 1.6 MB/s | 171 kB 00:00 2024-03-06T07:24:18.701 INFO:teuthology.orchestra.run.smithi194.stdout:(28/176): ceph-grafana-dashboards-19.0.0-1739.g 293 kB/s | 31 kB 00:00 2024-03-06T07:24:18.834 INFO:teuthology.orchestra.run.smithi194.stdout:(29/176): ceph-mgr-cephadm-19.0.0-1739.g1bc250c 1.0 MB/s | 138 kB 00:00 2024-03-06T07:24:18.917 INFO:teuthology.orchestra.run.smithi086.stdout:(5/176): ceph-mgr-19.0.0-1739.g1bc250c7.el9.x86 4.4 MB/s | 1.5 MB 00:00 2024-03-06T07:24:19.018 INFO:teuthology.orchestra.run.smithi194.stdout:(30/176): rbd-mirror-19.0.0-1739.g1bc250c7.el9. 5.1 MB/s | 3.1 MB 00:00 2024-03-06T07:24:19.185 INFO:teuthology.orchestra.run.smithi194.stdout:(31/176): ceph-mgr-dashboard-19.0.0-1739.g1bc25 5.1 MB/s | 1.8 MB 00:00 2024-03-06T07:24:19.335 INFO:teuthology.orchestra.run.smithi194.stdout:(32/176): ceph-mgr-modules-core-19.0.0-1739.g1b 1.6 MB/s | 248 kB 00:00 2024-03-06T07:24:19.460 INFO:teuthology.orchestra.run.smithi194.stdout:(33/176): ceph-mgr-rook-19.0.0-1739.g1bc250c7.e 397 kB/s | 49 kB 00:00 2024-03-06T07:24:19.586 INFO:teuthology.orchestra.run.smithi194.stdout:(34/176): ceph-prometheus-alerts-19.0.0-1739.g1 123 kB/s | 15 kB 00:00 2024-03-06T07:24:19.792 INFO:teuthology.orchestra.run.smithi086.stdout:(6/176): ceph-base-19.0.0-1739.g1bc250c7.el9.x8 2.0 MB/s | 5.3 MB 00:02 2024-03-06T07:24:19.901 INFO:teuthology.orchestra.run.smithi086.stdout:(7/176): ceph-mon-19.0.0-1739.g1bc250c7.el9.x86 4.8 MB/s | 4.7 MB 00:00 2024-03-06T07:24:20.319 INFO:teuthology.orchestra.run.smithi194.stdout:(35/176): ceph-mgr-diskprediction-local-19.0.0- 5.7 MB/s | 7.4 MB 00:01 2024-03-06T07:24:20.353 INFO:teuthology.orchestra.run.smithi194.stdout:(36/176): ceph-volume-19.0.0-1739.g1bc250c7.el9 344 kB/s | 264 kB 00:00 2024-03-06T07:24:20.478 INFO:teuthology.orchestra.run.smithi194.stdout:(37/176): cephfs-top-19.0.0-1739.g1bc250c7.el9. 146 kB/s | 18 kB 00:00 2024-03-06T07:24:20.595 INFO:teuthology.orchestra.run.smithi194.stdout:(38/176): cephadm-19.0.0-1739.g1bc250c7.el9.noa 2.7 MB/s | 748 kB 00:00 2024-03-06T07:24:20.720 INFO:teuthology.orchestra.run.smithi194.stdout:(39/176): automake-1.16.2-8.el9.noarch.rpm 2.7 MB/s | 662 kB 00:00 2024-03-06T07:24:20.763 INFO:teuthology.orchestra.run.smithi194.stdout:(40/176): bison-3.7.4-5.el9.x86_64.rpm 5.5 MB/s | 920 kB 00:00 2024-03-06T07:24:20.788 INFO:teuthology.orchestra.run.smithi194.stdout:(41/176): boost-program-options-1.75.0-8.el9.x8 1.7 MB/s | 106 kB 00:00 2024-03-06T07:24:20.813 INFO:teuthology.orchestra.run.smithi194.stdout:(42/176): elfutils-libelf-devel-0.190-2.el9.x86 420 kB/s | 20 kB 00:00 2024-03-06T07:24:20.847 INFO:teuthology.orchestra.run.smithi194.stdout:(43/176): flex-2.6.4-9.el9.x86_64.rpm 5.2 MB/s | 307 kB 00:00 2024-03-06T07:24:20.872 INFO:teuthology.orchestra.run.smithi194.stdout:(44/176): flexiblas-3.0.4-8.el9.x86_64.rpm 575 kB/s | 32 kB 00:00 2024-03-06T07:24:20.964 INFO:teuthology.orchestra.run.smithi194.stdout:(45/176): flexiblas-netlib-3.0.4-8.el9.x86_64.r 26 MB/s | 3.0 MB 00:00 2024-03-06T07:24:20.989 INFO:teuthology.orchestra.run.smithi194.stdout:(46/176): flexiblas-openblas-openmp-3.0.4-8.el9 167 kB/s | 17 kB 00:00 2024-03-06T07:24:21.056 INFO:teuthology.orchestra.run.smithi194.stdout:(47/176): libpmemobj-1.12.1-1.el9.x86_64.rpm 2.4 MB/s | 159 kB 00:00 2024-03-06T07:24:21.081 INFO:teuthology.orchestra.run.smithi194.stdout:(48/176): libaio-devel-0.3.111-13.el9.x86_64.rp 111 kB/s | 10 kB 00:00 2024-03-06T07:24:21.107 INFO:teuthology.orchestra.run.smithi194.stdout:(49/176): librabbitmq-0.11.0-7.el9.x86_64.rpm 896 kB/s | 44 kB 00:00 2024-03-06T07:24:21.149 INFO:teuthology.orchestra.run.smithi194.stdout:(50/176): librdkafka-1.6.1-102.el9.x86_64.rpm 9.8 MB/s | 661 kB 00:00 2024-03-06T07:24:21.190 INFO:teuthology.orchestra.run.smithi194.stdout:(51/176): libtool-2.4.6-45.el9.x86_64.rpm 7.3 MB/s | 578 kB 00:00 2024-03-06T07:24:21.207 INFO:teuthology.orchestra.run.smithi194.stdout:(52/176): libuuid-devel-2.37.4-18.el9.x86_64.rp 461 kB/s | 24 kB 00:00 2024-03-06T07:24:21.224 INFO:teuthology.orchestra.run.smithi194.stdout:(53/176): libzstd-devel-1.5.1-2.el9.x86_64.rpm 1.4 MB/s | 46 kB 00:00 2024-03-06T07:24:21.258 INFO:teuthology.orchestra.run.smithi194.stdout:(54/176): lttng-ust-2.12.0-6.el9.x86_64.rpm 5.7 MB/s | 282 kB 00:00 2024-03-06T07:24:21.283 INFO:teuthology.orchestra.run.smithi194.stdout:(55/176): lua-5.4.4-4.el9.x86_64.rpm 3.3 MB/s | 187 kB 00:00 2024-03-06T07:24:21.300 INFO:teuthology.orchestra.run.smithi194.stdout:(56/176): openblas-0.3.21-2.el9.x86_64.rpm 861 kB/s | 34 kB 00:00 2024-03-06T07:24:21.434 INFO:teuthology.orchestra.run.smithi194.stdout:(57/176): openblas-openmp-0.3.21-2.el9.x86_64.r 32 MB/s | 4.7 MB 00:00 2024-03-06T07:24:21.544 INFO:teuthology.orchestra.run.smithi194.stdout:(58/176): openssl-devel-3.0.7-27.el9.x86_64.rpm 13 MB/s | 3.0 MB 00:00 2024-03-06T07:24:21.610 INFO:teuthology.orchestra.run.smithi194.stdout:(59/176): postgresql-13.11-1.el9.x86_64.rpm 9.3 MB/s | 1.5 MB 00:00 2024-03-06T07:24:21.652 INFO:teuthology.orchestra.run.smithi194.stdout:(60/176): postgresql-contrib-13.11-1.el9.x86_64 8.4 MB/s | 811 kB 00:00 2024-03-06T07:24:21.677 INFO:teuthology.orchestra.run.smithi194.stdout:(61/176): postgresql-private-libs-13.11-1.el9.x 2.2 MB/s | 134 kB 00:00 2024-03-06T07:24:22.109 INFO:teuthology.orchestra.run.smithi086.stdout:(8/176): ceph-common-19.0.0-1739.g1bc250c7.el9. 4.1 MB/s | 20 MB 00:04 2024-03-06T07:24:22.235 INFO:teuthology.orchestra.run.smithi086.stdout:(9/176): ceph-selinux-19.0.0-1739.g1bc250c7.el9 203 kB/s | 25 kB 00:00 2024-03-06T07:24:22.278 INFO:teuthology.orchestra.run.smithi194.stdout:(62/176): ceph-test-19.0.0-1739.g1bc250c7.el9.x 8.5 MB/s | 47 MB 00:05 2024-03-06T07:24:22.411 INFO:teuthology.orchestra.run.smithi194.stdout:(63/176): postgresql-server-13.11-1.el9.x86_64. 7.5 MB/s | 5.7 MB 00:00 2024-03-06T07:24:22.460 INFO:teuthology.orchestra.run.smithi194.stdout:(64/176): protobuf-3.14.0-13.el9.x86_64.rpm 1.3 MB/s | 1.0 MB 00:00 2024-03-06T07:24:22.478 INFO:teuthology.orchestra.run.smithi194.stdout:(65/176): python3-attrs-20.3.0-7.el9.noarch.rpm 422 kB/s | 78 kB 00:00 2024-03-06T07:24:22.570 INFO:teuthology.orchestra.run.smithi194.stdout:(66/176): python3-jinja2-2.11.3-5.el9.noarch.rp 1.9 MB/s | 227 kB 00:00 2024-03-06T07:24:22.593 INFO:teuthology.orchestra.run.smithi086.stdout:(10/176): ceph-radosgw-19.0.0-1739.g1bc250c7.el 3.6 MB/s | 9.8 MB 00:02 2024-03-06T07:24:22.695 INFO:teuthology.orchestra.run.smithi194.stdout:(67/176): python3-babel-2.9.1-2.el9.noarch.rpm 24 MB/s | 5.8 MB 00:00 2024-03-06T07:24:22.721 INFO:teuthology.orchestra.run.smithi194.stdout:(68/176): python3-jsonpatch-1.21-16.el9.noarch. 111 kB/s | 24 kB 00:00 2024-03-06T07:24:22.746 INFO:teuthology.orchestra.run.smithi194.stdout:(69/176): python3-jsonpointer-2.0-4.el9.noarch. 111 kB/s | 18 kB 00:00 2024-03-06T07:24:22.813 INFO:teuthology.orchestra.run.smithi194.stdout:(70/176): python3-mako-1.1.4-6.el9.noarch.rpm 2.2 MB/s | 144 kB 00:00 2024-03-06T07:24:22.818 INFO:teuthology.orchestra.run.smithi086.stdout:(11/176): ceph-osd-19.0.0-1739.g1bc250c7.el9.x8 5.6 MB/s | 17 MB 00:03 2024-03-06T07:24:22.839 INFO:teuthology.orchestra.run.smithi194.stdout:(71/176): python3-markupsafe-1.1.1-12.el9.x86_6 350 kB/s | 32 kB 00:00 2024-03-06T07:24:22.852 INFO:teuthology.orchestra.run.smithi086.stdout:(12/176): cephfs-mirror-19.0.0-1739.g1bc250c7.e 856 kB/s | 220 kB 00:00 2024-03-06T07:24:22.919 INFO:teuthology.orchestra.run.smithi086.stdout:(13/176): cephfs-shell-19.0.0-1739.g1bc250c7.el 273 kB/s | 27 kB 00:00 2024-03-06T07:24:22.944 INFO:teuthology.orchestra.run.smithi086.stdout:(14/176): libcephfs-devel-19.0.0-1739.g1bc250c7 342 kB/s | 31 kB 00:00 2024-03-06T07:24:22.972 INFO:teuthology.orchestra.run.smithi194.stdout:(72/176): python3-numpy-1.20.1-5.el9.x86_64.rpm 21 MB/s | 4.8 MB 00:00 2024-03-06T07:24:22.997 INFO:teuthology.orchestra.run.smithi194.stdout:(73/176): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.5 MB/s | 217 kB 00:00 2024-03-06T07:24:23.022 INFO:teuthology.orchestra.run.smithi194.stdout:(74/176): python3-oauthlib-3.1.1-5.el9.noarch.r 1.0 MB/s | 174 kB 00:00 2024-03-06T07:24:23.064 INFO:teuthology.orchestra.run.smithi194.stdout:(75/176): python3-packaging-20.9-5.el9.noarch.r 1.6 MB/s | 69 kB 00:00 2024-03-06T07:24:23.089 INFO:teuthology.orchestra.run.smithi194.stdout:(76/176): python3-prettytable-0.7.2-27.el9.noar 627 kB/s | 41 kB 00:00 2024-03-06T07:24:23.114 INFO:teuthology.orchestra.run.smithi194.stdout:(77/176): python3-protobuf-3.14.0-13.el9.noarch 2.6 MB/s | 238 kB 00:00 2024-03-06T07:24:23.148 INFO:teuthology.orchestra.run.smithi194.stdout:(78/176): python3-pyasn1-0.4.8-6.el9.noarch.rpm 4.0 MB/s | 132 kB 00:00 2024-03-06T07:24:23.173 INFO:teuthology.orchestra.run.smithi194.stdout:(79/176): python3-pyasn1-modules-0.4.8-6.el9.no 3.6 MB/s | 211 kB 00:00 2024-03-06T07:24:23.189 INFO:teuthology.orchestra.run.smithi194.stdout:(80/176): python3-pytz-2021.1-5.el9.noarch.rpm 644 kB/s | 47 kB 00:00 2024-03-06T07:24:23.211 INFO:teuthology.orchestra.run.smithi086.stdout:(15/176): libcephfs2-19.0.0-1739.g1bc250c7.el9. 2.5 MB/s | 747 kB 00:00 2024-03-06T07:24:23.223 INFO:teuthology.orchestra.run.smithi194.stdout:(81/176): python3-requests-oauthlib-1.3.0-12.el 1.3 MB/s | 43 kB 00:00 2024-03-06T07:24:23.274 INFO:teuthology.orchestra.run.smithi194.stdout:(82/176): python3-toml-0.10.2-6.el9.noarch.rpm 471 kB/s | 39 kB 00:00 2024-03-06T07:24:23.328 INFO:teuthology.orchestra.run.smithi086.stdout:(16/176): librados-devel-19.0.0-1739.g1bc250c7. 1.1 MB/s | 127 kB 00:00 2024-03-06T07:24:23.553 INFO:teuthology.orchestra.run.smithi086.stdout:(17/176): libradosstriper1-19.0.0-1739.g1bc250c 2.2 MB/s | 501 kB 00:00 2024-03-06T07:24:23.599 INFO:teuthology.orchestra.run.smithi194.stdout:(83/176): python3-scipy-1.6.2-8.el9.x86_64.rpm 38 MB/s | 16 MB 00:00 2024-03-06T07:24:23.616 INFO:teuthology.orchestra.run.smithi194.stdout:(84/176): python3-wcwidth-0.2.5-8.el9.noarch.rp 111 kB/s | 41 kB 00:00 2024-03-06T07:24:23.641 INFO:teuthology.orchestra.run.smithi194.stdout:(85/176): qatlib-23.11.0-1.el9.x86_64.rpm 619 kB/s | 215 kB 00:00 2024-03-06T07:24:23.691 INFO:teuthology.orchestra.run.smithi194.stdout:(86/176): qatlib-service-23.11.0-1.el9.x86_64.r 695 kB/s | 34 kB 00:00 2024-03-06T07:24:23.716 INFO:teuthology.orchestra.run.smithi194.stdout:(87/176): qatzip-libs-1.1.2-2.el9.x86_64.rpm 590 kB/s | 43 kB 00:00 2024-03-06T07:24:23.741 INFO:teuthology.orchestra.run.smithi194.stdout:(88/176): socat-1.7.4.1-5.el9.x86_64.rpm 3.0 MB/s | 300 kB 00:00 2024-03-06T07:24:23.770 INFO:teuthology.orchestra.run.smithi086.stdout:(18/176): libcephsqlite-19.0.0-1739.g1bc250c7.e 198 kB/s | 163 kB 00:00 2024-03-06T07:24:23.783 INFO:teuthology.orchestra.run.smithi194.stdout:(89/176): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 63 kB 00:00 2024-03-06T07:24:23.808 INFO:teuthology.orchestra.run.smithi194.stdout:(90/176): uuid-1.6.2-55.el9.x86_64.rpm 858 kB/s | 56 kB 00:00 2024-03-06T07:24:23.834 INFO:teuthology.orchestra.run.smithi194.stdout:(91/176): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 959 kB/s | 87 kB 00:00 2024-03-06T07:24:23.854 INFO:teuthology.orchestra.run.smithi086.stdout:(19/176): python3-ceph-argparse-19.0.0-1739.g1b 547 kB/s | 45 kB 00:00 2024-03-06T07:24:23.859 INFO:teuthology.orchestra.run.smithi194.stdout:(92/176): zlib-devel-1.2.11-40.el9.x86_64.rpm 600 kB/s | 44 kB 00:00 2024-03-06T07:24:23.884 INFO:teuthology.orchestra.run.smithi194.stdout:(93/176): fuse-2.9.9-15.el9.x86_64.rpm 1.6 MB/s | 79 kB 00:00 2024-03-06T07:24:23.901 INFO:teuthology.orchestra.run.smithi194.stdout:(94/176): jq-1.6-16.el9.x86_64.rpm 2.8 MB/s | 186 kB 00:00 2024-03-06T07:24:23.946 INFO:teuthology.orchestra.run.smithi086.stdout:(20/176): python3-ceph-common-19.0.0-1739.g1bc2 1.4 MB/s | 131 kB 00:00 2024-03-06T07:24:23.951 INFO:teuthology.orchestra.run.smithi194.stdout:(95/176): libgfortran-11.4.1-3.el9.x86_64.rpm 8.8 MB/s | 802 kB 00:00 2024-03-06T07:24:23.985 INFO:teuthology.orchestra.run.smithi194.stdout:(96/176): oniguruma-6.9.6-1.el9.5.x86_64.rpm 2.8 MB/s | 217 kB 00:00 2024-03-06T07:24:24.018 INFO:teuthology.orchestra.run.smithi194.stdout:(97/176): libquadmath-11.4.1-3.el9.x86_64.rpm 1.7 MB/s | 194 kB 00:00 2024-03-06T07:24:24.044 INFO:teuthology.orchestra.run.smithi194.stdout:(98/176): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.6 MB/s | 241 kB 00:00 2024-03-06T07:24:24.046 INFO:teuthology.orchestra.run.smithi086.stdout:(21/176): python3-cephfs-19.0.0-1739.g1bc250c7. 1.6 MB/s | 162 kB 00:00 2024-03-06T07:24:24.094 INFO:teuthology.orchestra.run.smithi194.stdout:(99/176): python3-cryptography-36.0.1-4.el9.x86 11 MB/s | 1.1 MB 00:00 2024-03-06T07:24:24.119 INFO:teuthology.orchestra.run.smithi194.stdout:(100/176): python3-ply-3.11-14.el9.noarch.rpm 1.6 MB/s | 103 kB 00:00 2024-03-06T07:24:24.144 INFO:teuthology.orchestra.run.smithi194.stdout:(101/176): python3-pycparser-2.20-6.el9.noarch. 1.4 MB/s | 124 kB 00:00 2024-03-06T07:24:24.170 INFO:teuthology.orchestra.run.smithi194.stdout:(102/176): python3-requests-2.25.1-8.el9.noarch 1.5 MB/s | 113 kB 00:00 2024-03-06T07:24:24.203 INFO:teuthology.orchestra.run.smithi194.stdout:(103/176): python3-urllib3-1.26.5-5.el9.noarch. 3.2 MB/s | 187 kB 00:00 2024-03-06T07:24:24.213 INFO:teuthology.orchestra.run.smithi086.stdout:(22/176): python3-rados-19.0.0-1739.g1bc250c7.e 1.9 MB/s | 321 kB 00:00 2024-03-06T07:24:24.313 INFO:teuthology.orchestra.run.smithi194.stdout:(104/176): lua-devel-5.4.4-4.el9.x86_64.rpm 159 kB/s | 22 kB 00:00 2024-03-06T07:24:24.396 INFO:teuthology.orchestra.run.smithi086.stdout:(23/176): python3-rbd-19.0.0-1739.g1bc250c7.el9 1.6 MB/s | 298 kB 00:00 2024-03-06T07:24:24.438 INFO:teuthology.orchestra.run.smithi194.stdout:(105/176): python3-devel-3.9.17-2.el9.x86_64.rp 845 kB/s | 245 kB 00:00 2024-03-06T07:24:24.488 INFO:teuthology.orchestra.run.smithi194.stdout:(106/176): protobuf-compiler-3.14.0-13.el9.x86_ 3.0 MB/s | 863 kB 00:00 2024-03-06T07:24:24.545 INFO:teuthology.orchestra.run.smithi086.stdout:(24/176): python3-rgw-19.0.0-1739.g1bc250c7.el9 671 kB/s | 99 kB 00:00 2024-03-06T07:24:24.614 INFO:teuthology.orchestra.run.smithi086.stdout:(25/176): rbd-fuse-19.0.0-1739.g1bc250c7.el9.x8 1.2 MB/s | 85 kB 00:00 2024-03-06T07:24:24.638 INFO:teuthology.orchestra.run.smithi194.stdout:(107/176): abseil-cpp-20211102.0-3.el9.x86_64.r 1.7 MB/s | 548 kB 00:00 2024-03-06T07:24:24.672 INFO:teuthology.orchestra.run.smithi194.stdout:(108/176): grpc-data-1.46.7-10.el9.noarch.rpm 106 kB/s | 19 kB 00:00 2024-03-06T07:24:24.714 INFO:teuthology.orchestra.run.smithi086.stdout:(26/176): librgw2-19.0.0-1739.g1bc250c7.el9.x86 4.4 MB/s | 5.1 MB 00:01 2024-03-06T07:24:24.747 INFO:teuthology.orchestra.run.smithi194.stdout:(109/176): gperftools-libs-2.9.1-2.el9.x86_64.r 1.0 MB/s | 309 kB 00:00 2024-03-06T07:24:24.772 INFO:teuthology.orchestra.run.smithi194.stdout:(110/176): libarrow-doc-9.0.0-10.el9.noarch.rpm 261 kB/s | 26 kB 00:00 2024-03-06T07:24:24.806 INFO:teuthology.orchestra.run.smithi086.stdout:(27/176): rbd-nbd-19.0.0-1739.g1bc250c7.el9.x86 1.8 MB/s | 171 kB 00:00 2024-03-06T07:24:24.814 INFO:teuthology.orchestra.run.smithi194.stdout:(111/176): liboath-2.6.7-2.el9.x86_64.rpm 741 kB/s | 49 kB 00:00 2024-03-06T07:24:24.915 INFO:teuthology.orchestra.run.smithi086.stdout:(28/176): ceph-grafana-dashboards-19.0.0-1739.g 292 kB/s | 31 kB 00:00 2024-03-06T07:24:24.923 INFO:teuthology.orchestra.run.smithi194.stdout:(112/176): libarrow-9.0.0-10.el9.x86_64.rpm 16 MB/s | 4.4 MB 00:00 2024-03-06T07:24:24.948 INFO:teuthology.orchestra.run.smithi194.stdout:(113/176): libunwind-1.6.2-1.el9.x86_64.rpm 385 kB/s | 67 kB 00:00 2024-03-06T07:24:24.973 INFO:teuthology.orchestra.run.smithi194.stdout:(114/176): luarocks-3.9.2-1.el9.noarch.rpm 956 kB/s | 151 kB 00:00 2024-03-06T07:24:25.024 INFO:teuthology.orchestra.run.smithi194.stdout:(115/176): parquet-libs-9.0.0-10.el9.x86_64.rpm 8.2 MB/s | 838 kB 00:00 2024-03-06T07:24:25.039 INFO:teuthology.orchestra.run.smithi086.stdout:(29/176): ceph-mgr-cephadm-19.0.0-1739.g1bc250c 1.1 MB/s | 138 kB 00:00 2024-03-06T07:24:25.049 INFO:teuthology.orchestra.run.smithi194.stdout:(116/176): python3-bcrypt-3.2.2-1.el9.x86_64.rp 583 kB/s | 43 kB 00:00 2024-03-06T07:24:25.082 INFO:teuthology.orchestra.run.smithi194.stdout:(117/176): python3-cachetools-4.2.4-1.el9.noarc 558 kB/s | 32 kB 00:00 2024-03-06T07:24:25.108 INFO:teuthology.orchestra.run.smithi194.stdout:(118/176): python3-certifi-2021.10.8-2.el9.noar 260 kB/s | 15 kB 00:00 2024-03-06T07:24:25.141 INFO:teuthology.orchestra.run.smithi194.stdout:(119/176): python3-asyncssh-2.13.2-4.el9.noarch 2.8 MB/s | 548 kB 00:00 2024-03-06T07:24:25.175 INFO:teuthology.orchestra.run.smithi194.stdout:(120/176): python3-cheroot-8.6.0-4.el9.noarch.r 1.8 MB/s | 172 kB 00:00 2024-03-06T07:24:25.208 INFO:teuthology.orchestra.run.smithi194.stdout:(121/176): python3-cherrypy-18.6.1-2.el9.noarch 3.5 MB/s | 358 kB 00:00 2024-03-06T07:24:25.234 INFO:teuthology.orchestra.run.smithi194.stdout:(122/176): python3-colorama-0.4.4-12.el9.noarch 611 kB/s | 35 kB 00:00 2024-03-06T07:24:25.267 INFO:teuthology.orchestra.run.smithi194.stdout:(123/176): python3-cmd2-2.4.2-2.el9.noarch.rpm 2.4 MB/s | 303 kB 00:00 2024-03-06T07:24:25.301 INFO:teuthology.orchestra.run.smithi194.stdout:(124/176): python3-google-auth-2.28.0-1.el9.noa 2.2 MB/s | 203 kB 00:00 2024-03-06T07:24:25.342 INFO:teuthology.orchestra.run.smithi194.stdout:(125/176): python3-grpcio-tools-1.46.7-10.el9.x 1.9 MB/s | 144 kB 00:00 2024-03-06T07:24:25.368 INFO:teuthology.orchestra.run.smithi194.stdout:(126/176): python3-jaraco-8.2.1-3.el9.noarch.rp 161 kB/s | 11 kB 00:00 2024-03-06T07:24:25.415 INFO:teuthology.orchestra.run.smithi086.stdout:(30/176): ceph-mgr-dashboard-19.0.0-1739.g1bc25 4.8 MB/s | 1.8 MB 00:00 2024-03-06T07:24:25.435 INFO:teuthology.orchestra.run.smithi194.stdout:(127/176): python3-grpcio-1.46.7-10.el9.x86_64. 10 MB/s | 2.0 MB 00:00 2024-03-06T07:24:25.460 INFO:teuthology.orchestra.run.smithi194.stdout:(128/176): python3-jaraco-classes-3.2.1-5.el9.n 152 kB/s | 18 kB 00:00 2024-03-06T07:24:25.485 INFO:teuthology.orchestra.run.smithi194.stdout:(129/176): python3-jaraco-collections-3.0.0-8.e 199 kB/s | 23 kB 00:00 2024-03-06T07:24:25.511 INFO:teuthology.orchestra.run.smithi194.stdout:(130/176): python3-jaraco-functools-3.5.0-2.el9 261 kB/s | 19 kB 00:00 2024-03-06T07:24:25.544 INFO:teuthology.orchestra.run.smithi194.stdout:(131/176): python3-jaraco-text-3.2.0-6.el9.noar 342 kB/s | 20 kB 00:00 2024-03-06T07:24:25.569 INFO:teuthology.orchestra.run.smithi194.stdout:(132/176): python3-logutils-0.3.5-21.el9.noarch 798 kB/s | 46 kB 00:00 2024-03-06T07:24:25.627 INFO:teuthology.orchestra.run.smithi194.stdout:(133/176): python3-more-itertools-8.12.0-2.el9. 968 kB/s | 79 kB 00:00 2024-03-06T07:24:25.669 INFO:teuthology.orchestra.run.smithi194.stdout:(134/176): python3-kubernetes-26.1.0-2.el9.noar 5.6 MB/s | 1.0 MB 00:00 2024-03-06T07:24:25.695 INFO:teuthology.orchestra.run.smithi194.stdout:(135/176): python3-natsort-7.1.1-5.el9.noarch.r 462 kB/s | 58 kB 00:00 2024-03-06T07:24:25.737 INFO:teuthology.orchestra.run.smithi194.stdout:(136/176): python3-pecan-1.4.2-3.el9.noarch.rpm 2.4 MB/s | 272 kB 00:00 2024-03-06T07:24:25.762 INFO:teuthology.orchestra.run.smithi194.stdout:(137/176): python3-portend-3.1.0-2.el9.noarch.r 178 kB/s | 16 kB 00:00 2024-03-06T07:24:25.788 INFO:teuthology.orchestra.run.smithi194.stdout:(138/176): python3-pyOpenSSL-21.0.0-1.el9.noarc 981 kB/s | 90 kB 00:00 2024-03-06T07:24:25.805 INFO:teuthology.orchestra.run.smithi194.stdout:(139/176): python3-pyperclip-1.8.2-2.el9.noarch 426 kB/s | 28 kB 00:00 2024-03-06T07:24:25.821 INFO:teuthology.orchestra.run.smithi194.stdout:(140/176): python3-repoze-lru-0.7-16.el9.noarch 530 kB/s | 31 kB 00:00 2024-03-06T07:24:25.840 INFO:teuthology.orchestra.run.smithi086.stdout:(31/176): rbd-mirror-19.0.0-1739.g1bc250c7.el9. 2.5 MB/s | 3.1 MB 00:01 2024-03-06T07:24:25.855 INFO:teuthology.orchestra.run.smithi194.stdout:(141/176): python3-routes-2.5.1-5.el9.noarch.rp 2.8 MB/s | 188 kB 00:00 2024-03-06T07:24:25.872 INFO:teuthology.orchestra.run.smithi194.stdout:(142/176): python3-rsa-4.9-2.el9.noarch.rpm 889 kB/s | 59 kB 00:00 2024-03-06T07:24:25.889 INFO:teuthology.orchestra.run.smithi194.stdout:(143/176): python3-tempora-5.0.0-2.el9.noarch.r 538 kB/s | 36 kB 00:00 2024-03-06T07:24:25.906 INFO:teuthology.orchestra.run.smithi194.stdout:(144/176): python3-typing-extensions-4.4.0-2.el 1.0 MB/s | 51 kB 00:00 2024-03-06T07:24:25.948 INFO:teuthology.orchestra.run.smithi194.stdout:(145/176): python3-webob-1.8.7-6.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-03-06T07:24:25.973 INFO:teuthology.orchestra.run.smithi194.stdout:(146/176): python3-websocket-client-1.2.3-2.el9 1.1 MB/s | 90 kB 00:00 2024-03-06T07:24:26.007 INFO:teuthology.orchestra.run.smithi194.stdout:(147/176): python3-werkzeug-2.0.3-3.el9.1.noarc 4.2 MB/s | 427 kB 00:00 2024-03-06T07:24:26.011 INFO:teuthology.orchestra.run.smithi086.stdout:(32/176): ceph-mgr-modules-core-19.0.0-1739.g1b 1.4 MB/s | 248 kB 00:00 2024-03-06T07:24:26.032 INFO:teuthology.orchestra.run.smithi194.stdout:(148/176): python3-zc-lockfile-2.0-10.el9.noarc 241 kB/s | 20 kB 00:00 2024-03-06T07:24:26.066 INFO:teuthology.orchestra.run.smithi194.stdout:(149/176): re2-20211101-3.el9.x86_64.rpm 2.0 MB/s | 192 kB 00:00 2024-03-06T07:24:26.091 INFO:teuthology.orchestra.run.smithi086.stdout:(33/176): ceph-mgr-rook-19.0.0-1739.g1bc250c7.e 622 kB/s | 49 kB 00:00 2024-03-06T07:24:26.141 INFO:teuthology.orchestra.run.smithi194.stdout:(150/176): thrift-0.15.0-2.el9.x86_64.rpm 12 MB/s | 1.6 MB 00:00 2024-03-06T07:24:26.175 INFO:teuthology.orchestra.run.smithi086.stdout:(34/176): ceph-prometheus-alerts-19.0.0-1739.g1 186 kB/s | 15 kB 00:00 2024-03-06T07:24:26.183 INFO:teuthology.orchestra.run.smithi194.stdout:(151/176): libacl-devel-2.3.1-4.el9.x86_64.rpm 1.6 MB/s | 68 kB 00:00 2024-03-06T07:24:26.291 INFO:teuthology.orchestra.run.smithi194.stdout:(152/176): NetworkManager-1.46.0-1.el9.x86_64.r 21 MB/s | 2.3 MB 00:00 2024-03-06T07:24:26.342 INFO:teuthology.orchestra.run.smithi086.stdout:(35/176): ceph-volume-19.0.0-1739.g1bc250c7.el9 1.6 MB/s | 264 kB 00:00 2024-03-06T07:24:26.425 INFO:teuthology.orchestra.run.smithi194.stdout:(153/176): NetworkManager-libnm-1.46.0-1.el9.x8 14 MB/s | 1.8 MB 00:00 2024-03-06T07:24:26.459 INFO:teuthology.orchestra.run.smithi194.stdout:(154/176): NetworkManager-team-1.46.0-1.el9.x86 1.2 MB/s | 41 kB 00:00 2024-03-06T07:24:26.500 INFO:teuthology.orchestra.run.smithi194.stdout:(155/176): NetworkManager-tui-1.46.0-1.el9.x86_ 5.9 MB/s | 247 kB 00:00 2024-03-06T07:24:26.543 INFO:teuthology.orchestra.run.smithi194.stdout:(156/176): acl-2.3.1-4.el9.x86_64.rpm 1.6 MB/s | 69 kB 00:00 2024-03-06T07:24:26.568 INFO:teuthology.orchestra.run.smithi194.stdout:(157/176): elfutils-debuginfod-client-0.190-2.e 1.4 MB/s | 35 kB 00:00 2024-03-06T07:24:26.610 INFO:teuthology.orchestra.run.smithi194.stdout:(158/176): elfutils-libelf-0.190-2.el9.x86_64.r 4.5 MB/s | 191 kB 00:00 2024-03-06T07:24:26.677 INFO:teuthology.orchestra.run.smithi194.stdout:(159/176): elfutils-libs-0.190-2.el9.x86_64.rpm 3.8 MB/s | 254 kB 00:00 2024-03-06T07:24:26.736 INFO:teuthology.orchestra.run.smithi194.stdout:(160/176): iproute-6.7.0-2.el9.x86_64.rpm 14 MB/s | 802 kB 00:00 2024-03-06T07:24:26.785 INFO:teuthology.orchestra.run.smithi194.stdout:(161/176): iproute-tc-6.7.0-2.el9.x86_64.rpm 8.6 MB/s | 433 kB 00:00 2024-03-06T07:24:26.817 INFO:teuthology.orchestra.run.smithi086.stdout:(36/176): cephadm-19.0.0-1739.g1bc250c7.el9.noa 1.5 MB/s | 748 kB 00:00 2024-03-06T07:24:26.819 INFO:teuthology.orchestra.run.smithi194.stdout:(162/176): libacl-2.3.1-4.el9.x86_64.rpm 676 kB/s | 22 kB 00:00 2024-03-06T07:24:26.861 INFO:teuthology.orchestra.run.smithi194.stdout:(163/176): libblkid-2.37.4-18.el9.x86_64.rpm 2.6 MB/s | 107 kB 00:00 2024-03-06T07:24:26.903 INFO:teuthology.orchestra.run.smithi194.stdout:(164/176): libfdisk-2.37.4-18.el9.x86_64.rpm 3.7 MB/s | 154 kB 00:00 2024-03-06T07:24:26.909 INFO:teuthology.orchestra.run.smithi086.stdout:(37/176): cephfs-top-19.0.0-1739.g1bc250c7.el9. 201 kB/s | 18 kB 00:00 2024-03-06T07:24:26.945 INFO:teuthology.orchestra.run.smithi194.stdout:(165/176): libmount-2.37.4-18.el9.x86_64.rpm 3.2 MB/s | 135 kB 00:00 2024-03-06T07:24:26.987 INFO:teuthology.orchestra.run.smithi194.stdout:(166/176): libsmartcols-2.37.4-18.el9.x86_64.rp 1.5 MB/s | 63 kB 00:00 2024-03-06T07:24:27.021 INFO:teuthology.orchestra.run.smithi194.stdout:(167/176): libuuid-2.37.4-18.el9.x86_64.rpm 845 kB/s | 28 kB 00:00 2024-03-06T07:24:27.096 INFO:teuthology.orchestra.run.smithi194.stdout:(168/176): openssl-3.0.7-27.el9.x86_64.rpm 16 MB/s | 1.2 MB 00:00 2024-03-06T07:24:27.126 INFO:teuthology.orchestra.run.smithi086.stdout:(38/176): automake-1.16.2-8.el9.noarch.rpm 3.0 MB/s | 662 kB 00:00 2024-03-06T07:24:27.209 INFO:teuthology.orchestra.run.smithi086.stdout:(39/176): bison-3.7.4-5.el9.x86_64.rpm 11 MB/s | 920 kB 00:00 2024-03-06T07:24:27.251 INFO:teuthology.orchestra.run.smithi086.stdout:(40/176): boost-program-options-1.75.0-8.el9.x8 2.5 MB/s | 106 kB 00:00 2024-03-06T07:24:27.254 INFO:teuthology.orchestra.run.smithi194.stdout:(169/176): openssl-libs-3.0.7-27.el9.x86_64.rpm 14 MB/s | 2.2 MB 00:00 2024-03-06T07:24:27.335 INFO:teuthology.orchestra.run.smithi086.stdout:(41/176): elfutils-libelf-devel-0.190-2.el9.x86 247 kB/s | 20 kB 00:00 2024-03-06T07:24:27.348 INFO:teuthology.orchestra.run.smithi194.stdout:(170/176): librbd1-19.0.0-1739.g1bc250c7.el9.x8 2.4 MB/s | 3.1 MB 00:01 2024-03-06T07:24:27.385 INFO:teuthology.orchestra.run.smithi086.stdout:(42/176): flex-2.6.4-9.el9.x86_64.rpm 6.1 MB/s | 307 kB 00:00 2024-03-06T07:24:27.427 INFO:teuthology.orchestra.run.smithi086.stdout:(43/176): flexiblas-3.0.4-8.el9.x86_64.rpm 775 kB/s | 32 kB 00:00 2024-03-06T07:24:27.431 INFO:teuthology.orchestra.run.smithi194.stdout:(171/176): util-linux-2.37.4-18.el9.x86_64.rpm 13 MB/s | 2.2 MB 00:00 2024-03-06T07:24:27.465 INFO:teuthology.orchestra.run.smithi194.stdout:(172/176): util-linux-core-2.37.4-18.el9.x86_64 4.1 MB/s | 438 kB 00:00 2024-03-06T07:24:27.498 INFO:teuthology.orchestra.run.smithi194.stdout:(173/176): util-linux-user-2.37.4-18.el9.x86_64 484 kB/s | 30 kB 00:00 2024-03-06T07:24:27.532 INFO:teuthology.orchestra.run.smithi194.stdout:(174/176): xfsdump-3.1.12-4.el9.x86_64.rpm 4.8 MB/s | 321 kB 00:00 2024-03-06T07:24:27.561 INFO:teuthology.orchestra.run.smithi086.stdout:(44/176): flexiblas-netlib-3.0.4-8.el9.x86_64.r 23 MB/s | 3.0 MB 00:00 2024-03-06T07:24:27.583 INFO:teuthology.orchestra.run.smithi194.stdout:(175/176): xfsprogs-6.3.0-1.el9.x86_64.rpm 13 MB/s | 1.1 MB 00:00 2024-03-06T07:24:27.689 INFO:teuthology.orchestra.run.smithi194.stdout:(176/176): librados2-19.0.0-1739.g1bc250c7.el9. 2.0 MB/s | 3.4 MB 00:01 2024-03-06T07:24:27.694 INFO:teuthology.orchestra.run.smithi086.stdout:(45/176): ceph-mgr-diskprediction-local-19.0.0- 3.3 MB/s | 7.4 MB 00:02 2024-03-06T07:24:27.697 INFO:teuthology.orchestra.run.smithi194.stdout:-------------------------------------------------------------------------------- 2024-03-06T07:24:27.697 INFO:teuthology.orchestra.run.smithi194.stdout:Total 16 MB/s | 224 MB 00:14 2024-03-06T07:24:27.720 INFO:teuthology.orchestra.run.smithi086.stdout:(46/176): flexiblas-openblas-openmp-3.0.4-8.el9 108 kB/s | 17 kB 00:00 2024-03-06T07:24:27.745 INFO:teuthology.orchestra.run.smithi086.stdout:(47/176): libaio-devel-0.3.111-13.el9.x86_64.rp 205 kB/s | 10 kB 00:00 2024-03-06T07:24:27.770 INFO:teuthology.orchestra.run.smithi086.stdout:(48/176): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.1 MB/s | 159 kB 00:00 2024-03-06T07:24:27.795 INFO:teuthology.orchestra.run.smithi086.stdout:(49/176): librabbitmq-0.11.0-7.el9.x86_64.rpm 918 kB/s | 44 kB 00:00 2024-03-06T07:24:27.833 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction check 2024-03-06T07:24:27.837 INFO:teuthology.orchestra.run.smithi086.stdout:(50/176): librdkafka-1.6.1-102.el9.x86_64.rpm 9.9 MB/s | 661 kB 00:00 2024-03-06T07:24:27.879 INFO:teuthology.orchestra.run.smithi086.stdout:(51/176): libtool-2.4.6-45.el9.x86_64.rpm 7.3 MB/s | 578 kB 00:00 2024-03-06T07:24:27.905 INFO:teuthology.orchestra.run.smithi086.stdout:(52/176): libuuid-devel-2.37.4-18.el9.x86_64.rp 394 kB/s | 24 kB 00:00 2024-03-06T07:24:27.930 INFO:teuthology.orchestra.run.smithi086.stdout:(53/176): libzstd-devel-1.5.1-2.el9.x86_64.rpm 942 kB/s | 46 kB 00:00 2024-03-06T07:24:27.989 INFO:teuthology.orchestra.run.smithi086.stdout:(54/176): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 282 kB 00:00 2024-03-06T07:24:28.022 INFO:teuthology.orchestra.run.smithi086.stdout:(55/176): lua-5.4.4-4.el9.x86_64.rpm 2.1 MB/s | 187 kB 00:00 2024-03-06T07:24:28.047 INFO:teuthology.orchestra.run.smithi086.stdout:(56/176): openblas-0.3.21-2.el9.x86_64.rpm 608 kB/s | 34 kB 00:00 2024-03-06T07:24:28.181 INFO:teuthology.orchestra.run.smithi086.stdout:(57/176): openblas-openmp-0.3.21-2.el9.x86_64.r 30 MB/s | 4.7 MB 00:00 2024-03-06T07:24:28.206 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction check succeeded. 2024-03-06T07:24:28.206 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction test 2024-03-06T07:24:28.273 INFO:teuthology.orchestra.run.smithi086.stdout:(58/176): openssl-devel-3.0.7-27.el9.x86_64.rpm 14 MB/s | 3.0 MB 00:00 2024-03-06T07:24:28.332 INFO:teuthology.orchestra.run.smithi086.stdout:(59/176): postgresql-13.11-1.el9.x86_64.rpm 11 MB/s | 1.5 MB 00:00 2024-03-06T07:24:28.382 INFO:teuthology.orchestra.run.smithi086.stdout:(60/176): postgresql-contrib-13.11-1.el9.x86_64 8.3 MB/s | 811 kB 00:00 2024-03-06T07:24:28.408 INFO:teuthology.orchestra.run.smithi086.stdout:(61/176): postgresql-private-libs-13.11-1.el9.x 1.9 MB/s | 134 kB 00:00 2024-03-06T07:24:28.550 INFO:teuthology.orchestra.run.smithi086.stdout:(62/176): postgresql-server-13.11-1.el9.x86_64. 34 MB/s | 5.7 MB 00:00 2024-03-06T07:24:28.600 INFO:teuthology.orchestra.run.smithi086.stdout:(63/176): protobuf-3.14.0-13.el9.x86_64.rpm 5.7 MB/s | 1.0 MB 00:00 2024-03-06T07:24:28.625 INFO:teuthology.orchestra.run.smithi086.stdout:(64/176): python3-attrs-20.3.0-7.el9.noarch.rpm 1.2 MB/s | 78 kB 00:00 2024-03-06T07:24:28.784 INFO:teuthology.orchestra.run.smithi086.stdout:(65/176): python3-babel-2.9.1-2.el9.noarch.rpm 32 MB/s | 5.8 MB 00:00 2024-03-06T07:24:28.809 INFO:teuthology.orchestra.run.smithi086.stdout:(66/176): python3-jinja2-2.11.3-5.el9.noarch.rp 1.3 MB/s | 227 kB 00:00 2024-03-06T07:24:28.834 INFO:teuthology.orchestra.run.smithi086.stdout:(67/176): python3-jsonpatch-1.21-16.el9.noarch. 511 kB/s | 24 kB 00:00 2024-03-06T07:24:28.859 INFO:teuthology.orchestra.run.smithi086.stdout:(68/176): python3-jsonpointer-2.0-4.el9.noarch. 368 kB/s | 18 kB 00:00 2024-03-06T07:24:28.885 INFO:teuthology.orchestra.run.smithi086.stdout:(69/176): python3-mako-1.1.4-6.el9.noarch.rpm 2.9 MB/s | 144 kB 00:00 2024-03-06T07:24:28.910 INFO:teuthology.orchestra.run.smithi086.stdout:(70/176): python3-markupsafe-1.1.1-12.el9.x86_6 654 kB/s | 32 kB 00:00 2024-03-06T07:24:29.044 INFO:teuthology.orchestra.run.smithi086.stdout:(71/176): python3-numpy-1.20.1-5.el9.x86_64.rpm 30 MB/s | 4.8 MB 00:00 2024-03-06T07:24:29.077 INFO:teuthology.orchestra.run.smithi086.stdout:(72/176): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.4 MB/s | 217 kB 00:00 2024-03-06T07:24:29.111 INFO:teuthology.orchestra.run.smithi086.stdout:(73/176): python3-oauthlib-3.1.1-5.el9.noarch.r 2.6 MB/s | 174 kB 00:00 2024-03-06T07:24:29.136 INFO:teuthology.orchestra.run.smithi086.stdout:(74/176): python3-packaging-20.9-5.el9.noarch.r 1.2 MB/s | 69 kB 00:00 2024-03-06T07:24:29.161 INFO:teuthology.orchestra.run.smithi086.stdout:(75/176): python3-prettytable-0.7.2-27.el9.noar 836 kB/s | 41 kB 00:00 2024-03-06T07:24:29.195 INFO:teuthology.orchestra.run.smithi086.stdout:(76/176): python3-protobuf-3.14.0-13.el9.noarch 4.1 MB/s | 238 kB 00:00 2024-03-06T07:24:29.220 INFO:teuthology.orchestra.run.smithi086.stdout:(77/176): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.3 MB/s | 132 kB 00:00 2024-03-06T07:24:29.254 INFO:teuthology.orchestra.run.smithi086.stdout:(78/176): python3-pyasn1-modules-0.4.8-6.el9.no 3.6 MB/s | 211 kB 00:00 2024-03-06T07:24:29.279 INFO:teuthology.orchestra.run.smithi086.stdout:(79/176): python3-pytz-2021.1-5.el9.noarch.rpm 845 kB/s | 47 kB 00:00 2024-03-06T07:24:29.304 INFO:teuthology.orchestra.run.smithi086.stdout:(80/176): python3-requests-oauthlib-1.3.0-12.el 873 kB/s | 43 kB 00:00 2024-03-06T07:24:29.822 INFO:teuthology.orchestra.run.smithi086.stdout:(81/176): ceph-test-19.0.0-1739.g1bc250c7.el9.x 6.2 MB/s | 47 MB 00:07 2024-03-06T07:24:29.938 INFO:teuthology.orchestra.run.smithi086.stdout:(82/176): python3-toml-0.10.2-6.el9.noarch.rpm 62 kB/s | 39 kB 00:00 2024-03-06T07:24:30.131 INFO:teuthology.orchestra.run.smithi086.stdout:(83/176): python3-scipy-1.6.2-8.el9.x86_64.rpm 18 MB/s | 16 MB 00:00 2024-03-06T07:24:30.155 INFO:teuthology.orchestra.run.smithi086.stdout:(84/176): python3-wcwidth-0.2.5-8.el9.noarch.rp 128 kB/s | 41 kB 00:00 2024-03-06T07:24:30.189 INFO:teuthology.orchestra.run.smithi086.stdout:(85/176): qatlib-23.11.0-1.el9.x86_64.rpm 877 kB/s | 215 kB 00:00 2024-03-06T07:24:30.214 INFO:teuthology.orchestra.run.smithi086.stdout:(86/176): qatlib-service-23.11.0-1.el9.x86_64.r 607 kB/s | 34 kB 00:00 2024-03-06T07:24:30.239 INFO:teuthology.orchestra.run.smithi086.stdout:(87/176): qatzip-libs-1.1.2-2.el9.x86_64.rpm 539 kB/s | 43 kB 00:00 2024-03-06T07:24:30.273 INFO:teuthology.orchestra.run.smithi086.stdout:(88/176): socat-1.7.4.1-5.el9.x86_64.rpm 3.6 MB/s | 300 kB 00:00 2024-03-06T07:24:30.298 INFO:teuthology.orchestra.run.smithi086.stdout:(89/176): uuid-1.6.2-55.el9.x86_64.rpm 1.0 MB/s | 56 kB 00:00 2024-03-06T07:24:30.315 INFO:teuthology.orchestra.run.smithi086.stdout:(90/176): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 1.2 MB/s | 87 kB 00:00 2024-03-06T07:24:30.341 INFO:teuthology.orchestra.run.smithi086.stdout:(91/176): xmlstarlet-1.6.1-20.el9.x86_64.rpm 958 kB/s | 63 kB 00:00 2024-03-06T07:24:30.366 INFO:teuthology.orchestra.run.smithi086.stdout:(92/176): zlib-devel-1.2.11-40.el9.x86_64.rpm 893 kB/s | 44 kB 00:00 2024-03-06T07:24:30.425 INFO:teuthology.orchestra.run.smithi086.stdout:(93/176): fuse-2.9.9-15.el9.x86_64.rpm 730 kB/s | 79 kB 00:00 2024-03-06T07:24:30.431 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction test succeeded. 2024-03-06T07:24:30.431 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction 2024-03-06T07:24:30.458 INFO:teuthology.orchestra.run.smithi086.stdout:(94/176): jq-1.6-16.el9.x86_64.rpm 1.6 MB/s | 186 kB 00:00 2024-03-06T07:24:30.508 INFO:teuthology.orchestra.run.smithi086.stdout:(95/176): libgfortran-11.4.1-3.el9.x86_64.rpm 9.7 MB/s | 802 kB 00:00 2024-03-06T07:24:30.533 INFO:teuthology.orchestra.run.smithi086.stdout:(96/176): libquadmath-11.4.1-3.el9.x86_64.rpm 1.8 MB/s | 194 kB 00:00 2024-03-06T07:24:30.567 INFO:teuthology.orchestra.run.smithi086.stdout:(97/176): oniguruma-6.9.6-1.el9.5.x86_64.rpm 2.1 MB/s | 217 kB 00:00 2024-03-06T07:24:30.609 INFO:teuthology.orchestra.run.smithi086.stdout:(98/176): python3-cffi-1.14.5-5.el9.x86_64.rpm 3.2 MB/s | 241 kB 00:00 2024-03-06T07:24:30.659 INFO:teuthology.orchestra.run.smithi086.stdout:(99/176): python3-cryptography-36.0.1-4.el9.x86 9.4 MB/s | 1.1 MB 00:00 2024-03-06T07:24:30.684 INFO:teuthology.orchestra.run.smithi086.stdout:(100/176): python3-ply-3.11-14.el9.noarch.rpm 982 kB/s | 103 kB 00:00 2024-03-06T07:24:30.710 INFO:teuthology.orchestra.run.smithi086.stdout:(101/176): python3-pycparser-2.20-6.el9.noarch. 2.5 MB/s | 124 kB 00:00 2024-03-06T07:24:30.735 INFO:teuthology.orchestra.run.smithi086.stdout:(102/176): python3-requests-2.25.1-8.el9.noarch 1.5 MB/s | 113 kB 00:00 2024-03-06T07:24:30.768 INFO:teuthology.orchestra.run.smithi086.stdout:(103/176): python3-urllib3-1.26.5-5.el9.noarch. 2.3 MB/s | 187 kB 00:00 2024-03-06T07:24:30.861 INFO:teuthology.orchestra.run.smithi086.stdout:(104/176): lua-devel-5.4.4-4.el9.x86_64.rpm 182 kB/s | 22 kB 00:00 2024-03-06T07:24:30.986 INFO:teuthology.orchestra.run.smithi086.stdout:(105/176): python3-devel-3.9.17-2.el9.x86_64.rp 990 kB/s | 245 kB 00:00 2024-03-06T07:24:31.128 INFO:teuthology.orchestra.run.smithi086.stdout:(106/176): protobuf-compiler-3.14.0-13.el9.x86_ 2.4 MB/s | 863 kB 00:00 2024-03-06T07:24:31.395 INFO:teuthology.orchestra.run.smithi086.stdout:(107/176): grpc-data-1.46.7-10.el9.noarch.rpm 73 kB/s | 19 kB 00:00 2024-03-06T07:24:31.495 INFO:teuthology.orchestra.run.smithi086.stdout:(108/176): abseil-cpp-20211102.0-3.el9.x86_64.r 865 kB/s | 548 kB 00:00 2024-03-06T07:24:31.537 INFO:teuthology.orchestra.run.smithi086.stdout:(109/176): gperftools-libs-2.9.1-2.el9.x86_64.r 561 kB/s | 309 kB 00:00 2024-03-06T07:24:31.579 INFO:teuthology.orchestra.run.smithi086.stdout:(110/176): libarrow-doc-9.0.0-10.el9.noarch.rpm 313 kB/s | 26 kB 00:00 2024-03-06T07:24:31.629 INFO:teuthology.orchestra.run.smithi086.stdout:(111/176): liboath-2.6.7-2.el9.x86_64.rpm 531 kB/s | 49 kB 00:00 2024-03-06T07:24:31.671 INFO:teuthology.orchestra.run.smithi086.stdout:(112/176): libunwind-1.6.2-1.el9.x86_64.rpm 738 kB/s | 67 kB 00:00 2024-03-06T07:24:31.746 INFO:teuthology.orchestra.run.smithi086.stdout:(113/176): luarocks-3.9.2-1.el9.noarch.rpm 1.3 MB/s | 151 kB 00:00 2024-03-06T07:24:31.888 INFO:teuthology.orchestra.run.smithi086.stdout:(114/176): parquet-libs-9.0.0-10.el9.x86_64.rpm 3.8 MB/s | 838 kB 00:00 2024-03-06T07:24:31.938 INFO:teuthology.orchestra.run.smithi086.stdout:(115/176): python3-asyncssh-2.13.2-4.el9.noarch 2.8 MB/s | 548 kB 00:00 2024-03-06T07:24:31.980 INFO:teuthology.orchestra.run.smithi086.stdout:(116/176): python3-bcrypt-3.2.2-1.el9.x86_64.rp 477 kB/s | 43 kB 00:00 2024-03-06T07:24:32.031 INFO:teuthology.orchestra.run.smithi086.stdout:(117/176): python3-cachetools-4.2.4-1.el9.noarc 353 kB/s | 32 kB 00:00 2024-03-06T07:24:32.072 INFO:teuthology.orchestra.run.smithi086.stdout:(118/176): python3-certifi-2021.10.8-2.el9.noar 164 kB/s | 15 kB 00:00 2024-03-06T07:24:32.189 INFO:teuthology.orchestra.run.smithi086.stdout:(119/176): libarrow-9.0.0-10.el9.x86_64.rpm 5.6 MB/s | 4.4 MB 00:00 2024-03-06T07:24:32.223 INFO:teuthology.orchestra.run.smithi086.stdout:(120/176): python3-cheroot-8.6.0-4.el9.noarch.r 897 kB/s | 172 kB 00:00 2024-03-06T07:24:32.256 INFO:teuthology.orchestra.run.smithi086.stdout:(121/176): python3-cherrypy-18.6.1-2.el9.noarch 1.9 MB/s | 358 kB 00:00 2024-03-06T07:24:32.299 INFO:teuthology.orchestra.run.smithi086.stdout:(122/176): python3-cmd2-2.4.2-2.el9.noarch.rpm 2.7 MB/s | 303 kB 00:00 2024-03-06T07:24:32.324 INFO:teuthology.orchestra.run.smithi086.stdout:(123/176): python3-colorama-0.4.4-12.el9.noarch 353 kB/s | 35 kB 00:00 2024-03-06T07:24:32.391 INFO:teuthology.orchestra.run.smithi086.stdout:(124/176): python3-google-auth-2.28.0-1.el9.noa 1.5 MB/s | 203 kB 00:00 2024-03-06T07:24:32.508 INFO:teuthology.orchestra.run.smithi086.stdout:(125/176): python3-jaraco-8.2.1-3.el9.noarch.rp 92 kB/s | 11 kB 00:00 2024-03-06T07:24:32.558 INFO:teuthology.orchestra.run.smithi086.stdout:(126/176): python3-grpcio-tools-1.46.7-10.el9.x 619 kB/s | 144 kB 00:00 2024-03-06T07:24:32.608 INFO:teuthology.orchestra.run.smithi086.stdout:(127/176): python3-jaraco-classes-3.2.1-5.el9.n 177 kB/s | 18 kB 00:00 2024-03-06T07:24:32.659 INFO:teuthology.orchestra.run.smithi086.stdout:(128/176): python3-grpcio-1.46.7-10.el9.x86_64. 5.7 MB/s | 2.0 MB 00:00 2024-03-06T07:24:32.684 INFO:teuthology.orchestra.run.smithi086.stdout:(129/176): python3-jaraco-collections-3.0.0-8.e 185 kB/s | 23 kB 00:00 2024-03-06T07:24:32.709 INFO:teuthology.orchestra.run.smithi086.stdout:(130/176): python3-jaraco-functools-3.5.0-2.el9 194 kB/s | 19 kB 00:00 2024-03-06T07:24:32.759 INFO:teuthology.orchestra.run.smithi086.stdout:(131/176): python3-jaraco-text-3.2.0-6.el9.noar 197 kB/s | 20 kB 00:00 2024-03-06T07:24:32.810 INFO:teuthology.orchestra.run.smithi086.stdout:(132/176): python3-logutils-0.3.5-21.el9.noarch 464 kB/s | 46 kB 00:00 2024-03-06T07:24:32.868 INFO:teuthology.orchestra.run.smithi086.stdout:(133/176): python3-kubernetes-26.1.0-2.el9.noar 5.6 MB/s | 1.0 MB 00:00 2024-03-06T07:24:32.894 INFO:teuthology.orchestra.run.smithi086.stdout:(134/176): python3-more-itertools-8.12.0-2.el9. 592 kB/s | 79 kB 00:00 2024-03-06T07:24:32.919 INFO:teuthology.orchestra.run.smithi086.stdout:(135/176): python3-natsort-7.1.1-5.el9.noarch.r 533 kB/s | 58 kB 00:00 2024-03-06T07:24:32.978 INFO:teuthology.orchestra.run.smithi086.stdout:(136/176): python3-pecan-1.4.2-3.el9.noarch.rpm 2.5 MB/s | 272 kB 00:00 2024-03-06T07:24:33.003 INFO:teuthology.orchestra.run.smithi086.stdout:(137/176): python3-portend-3.1.0-2.el9.noarch.r 152 kB/s | 16 kB 00:00 2024-03-06T07:24:33.028 INFO:teuthology.orchestra.run.smithi086.stdout:(138/176): python3-pyOpenSSL-21.0.0-1.el9.noarc 830 kB/s | 90 kB 00:00 2024-03-06T07:24:33.070 INFO:teuthology.orchestra.run.smithi086.stdout:(139/176): python3-pyperclip-1.8.2-2.el9.noarch 309 kB/s | 28 kB 00:00 2024-03-06T07:24:33.104 INFO:teuthology.orchestra.run.smithi086.stdout:(140/176): python3-repoze-lru-0.7-16.el9.noarch 308 kB/s | 31 kB 00:00 2024-03-06T07:24:33.137 INFO:teuthology.orchestra.run.smithi086.stdout:(141/176): python3-routes-2.5.1-5.el9.noarch.rp 1.7 MB/s | 188 kB 00:00 2024-03-06T07:24:33.162 INFO:teuthology.orchestra.run.smithi086.stdout:(142/176): python3-rsa-4.9-2.el9.noarch.rpm 645 kB/s | 59 kB 00:00 2024-03-06T07:24:33.196 INFO:teuthology.orchestra.run.smithi086.stdout:(143/176): python3-tempora-5.0.0-2.el9.noarch.r 390 kB/s | 36 kB 00:00 2024-03-06T07:24:33.229 INFO:teuthology.orchestra.run.smithi086.stdout:(144/176): python3-typing-extensions-4.4.0-2.el 556 kB/s | 51 kB 00:00 2024-03-06T07:24:33.271 INFO:teuthology.orchestra.run.smithi086.stdout:(145/176): python3-webob-1.8.7-6.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-03-06T07:24:33.297 INFO:teuthology.orchestra.run.smithi086.stdout:(146/176): python3-websocket-client-1.2.3-2.el9 896 kB/s | 90 kB 00:00 2024-03-06T07:24:33.347 INFO:teuthology.orchestra.run.smithi086.stdout:(147/176): python3-werkzeug-2.0.3-3.el9.1.noarc 3.6 MB/s | 427 kB 00:00 2024-03-06T07:24:33.372 INFO:teuthology.orchestra.run.smithi086.stdout:(148/176): python3-zc-lockfile-2.0-10.el9.noarc 200 kB/s | 20 kB 00:00 2024-03-06T07:24:33.406 INFO:teuthology.orchestra.run.smithi086.stdout:(149/176): re2-20211101-3.el9.x86_64.rpm 1.7 MB/s | 192 kB 00:00 2024-03-06T07:24:33.523 INFO:teuthology.orchestra.run.smithi086.stdout:(150/176): thrift-0.15.0-2.el9.x86_64.rpm 9.0 MB/s | 1.6 MB 00:00 2024-03-06T07:24:33.556 INFO:teuthology.orchestra.run.smithi086.stdout:(151/176): libacl-devel-2.3.1-4.el9.x86_64.rpm 2.0 MB/s | 68 kB 00:00 2024-03-06T07:24:33.665 INFO:teuthology.orchestra.run.smithi086.stdout:(152/176): NetworkManager-1.46.0-1.el9.x86_64.r 21 MB/s | 2.3 MB 00:00 2024-03-06T07:24:33.757 INFO:teuthology.orchestra.run.smithi086.stdout:(153/176): NetworkManager-libnm-1.46.0-1.el9.x8 20 MB/s | 1.8 MB 00:00 2024-03-06T07:24:33.822 INFO:teuthology.orchestra.run.smithi086.stdout:(154/176): NetworkManager-team-1.46.0-1.el9.x86 646 kB/s | 41 kB 00:00 2024-03-06T07:24:33.866 INFO:teuthology.orchestra.run.smithi086.stdout:(155/176): NetworkManager-tui-1.46.0-1.el9.x86_ 5.6 MB/s | 247 kB 00:00 2024-03-06T07:24:33.908 INFO:teuthology.orchestra.run.smithi086.stdout:(156/176): acl-2.3.1-4.el9.x86_64.rpm 1.6 MB/s | 69 kB 00:00 2024-03-06T07:24:33.942 INFO:teuthology.orchestra.run.smithi086.stdout:(157/176): elfutils-debuginfod-client-0.190-2.e 1.1 MB/s | 35 kB 00:00 2024-03-06T07:24:33.984 INFO:teuthology.orchestra.run.smithi086.stdout:(158/176): elfutils-libelf-0.190-2.el9.x86_64.r 4.6 MB/s | 191 kB 00:00 2024-03-06T07:24:34.026 INFO:teuthology.orchestra.run.smithi086.stdout:(159/176): elfutils-libs-0.190-2.el9.x86_64.rpm 6.1 MB/s | 254 kB 00:00 2024-03-06T07:24:34.084 INFO:teuthology.orchestra.run.smithi086.stdout:(160/176): iproute-6.7.0-2.el9.x86_64.rpm 14 MB/s | 802 kB 00:00 2024-03-06T07:24:34.134 INFO:teuthology.orchestra.run.smithi086.stdout:(161/176): iproute-tc-6.7.0-2.el9.x86_64.rpm 8.6 MB/s | 433 kB 00:00 2024-03-06T07:24:34.168 INFO:teuthology.orchestra.run.smithi086.stdout:(162/176): libacl-2.3.1-4.el9.x86_64.rpm 669 kB/s | 22 kB 00:00 2024-03-06T07:24:34.202 INFO:teuthology.orchestra.run.smithi086.stdout:(163/176): libblkid-2.37.4-18.el9.x86_64.rpm 3.2 MB/s | 107 kB 00:00 2024-03-06T07:24:34.243 INFO:teuthology.orchestra.run.smithi086.stdout:(164/176): libfdisk-2.37.4-18.el9.x86_64.rpm 3.7 MB/s | 154 kB 00:00 2024-03-06T07:24:34.316 INFO:teuthology.orchestra.run.smithi086.stdout:(165/176): libmount-2.37.4-18.el9.x86_64.rpm 1.8 MB/s | 135 kB 00:00 2024-03-06T07:24:34.361 INFO:teuthology.orchestra.run.smithi086.stdout:(166/176): libsmartcols-2.37.4-18.el9.x86_64.rp 1.4 MB/s | 63 kB 00:00 2024-03-06T07:24:34.398 INFO:teuthology.orchestra.run.smithi086.stdout:(167/176): libuuid-2.37.4-18.el9.x86_64.rpm 759 kB/s | 28 kB 00:00 2024-03-06T07:24:34.473 INFO:teuthology.orchestra.run.smithi086.stdout:(168/176): openssl-3.0.7-27.el9.x86_64.rpm 16 MB/s | 1.2 MB 00:00 2024-03-06T07:24:34.575 INFO:teuthology.orchestra.run.smithi086.stdout:(169/176): openssl-libs-3.0.7-27.el9.x86_64.rpm 22 MB/s | 2.2 MB 00:00 2024-03-06T07:24:34.682 INFO:teuthology.orchestra.run.smithi086.stdout:(170/176): util-linux-2.37.4-18.el9.x86_64.rpm 20 MB/s | 2.2 MB 00:00 2024-03-06T07:24:34.738 INFO:teuthology.orchestra.run.smithi086.stdout:(171/176): util-linux-core-2.37.4-18.el9.x86_64 8.7 MB/s | 438 kB 00:00 2024-03-06T07:24:34.767 INFO:teuthology.orchestra.run.smithi086.stdout:(172/176): util-linux-user-2.37.4-18.el9.x86_64 917 kB/s | 30 kB 00:00 2024-03-06T07:24:34.842 INFO:teuthology.orchestra.run.smithi086.stdout:(173/176): librbd1-19.0.0-1739.g1bc250c7.el9.x8 2.2 MB/s | 3.1 MB 00:01 2024-03-06T07:24:34.876 INFO:teuthology.orchestra.run.smithi086.stdout:(174/176): xfsdump-3.1.12-4.el9.x86_64.rpm 2.9 MB/s | 321 kB 00:00 2024-03-06T07:24:34.926 INFO:teuthology.orchestra.run.smithi086.stdout:(175/176): xfsprogs-6.3.0-1.el9.x86_64.rpm 13 MB/s | 1.1 MB 00:00 2024-03-06T07:24:37.159 INFO:teuthology.orchestra.run.smithi086.stdout:(176/176): librados2-19.0.0-1739.g1bc250c7.el9. 917 kB/s | 3.4 MB 00:03 2024-03-06T07:24:37.167 INFO:teuthology.orchestra.run.smithi086.stdout:-------------------------------------------------------------------------------- 2024-03-06T07:24:37.168 INFO:teuthology.orchestra.run.smithi086.stdout:Total 11 MB/s | 224 MB 00:20 2024-03-06T07:24:37.300 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction check 2024-03-06T07:24:37.673 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction check succeeded. 2024-03-06T07:24:37.673 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction test 2024-03-06T07:24:37.887 INFO:teuthology.orchestra.run.smithi194.stdout: Preparing : 1/1 2024-03-06T07:24:37.998 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : openssl-libs-1:3.0.7-27.el9.x86_64 1/202 2024-03-06T07:24:38.037 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : libuuid-2.37.4-18.el9.x86_64 2/202 2024-03-06T07:24:38.068 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-06T07:24:38.157 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-06T07:24:38.220 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : elfutils-libelf-0.190-2.el9.x86_64 4/202 2024-03-06T07:24:38.299 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 5/202 2024-03-06T07:24:38.395 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/202 2024-03-06T07:24:38.619 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : thrift-0.15.0-2.el9.x86_64 7/202 2024-03-06T07:24:38.652 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 8/202 2024-03-06T07:24:38.751 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 9/202 2024-03-06T07:24:39.175 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 10/202 2024-03-06T07:24:39.215 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-06T07:24:39.365 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-06T07:24:39.391 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-06T07:24:39.510 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-06T07:24:39.552 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-06T07:24:39.810 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : postgresql-private-libs-13.11-1.el9.x86_64 14/202 2024-03-06T07:24:39.875 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : postgresql-13.11-1.el9.x86_64 15/202 2024-03-06T07:24:39.900 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction test succeeded. 2024-03-06T07:24:39.901 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction 2024-03-06T07:24:39.903 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-06T07:24:40.082 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-06T07:24:40.121 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-06T07:24:40.194 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-06T07:24:40.239 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : libmount-2.37.4-18.el9.x86_64 18/202 2024-03-06T07:24:40.264 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-06T07:24:40.334 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-06T07:24:40.622 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-06T07:24:40.664 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-06T07:24:40.707 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : libsmartcols-2.37.4-18.el9.x86_64 21/202 2024-03-06T07:24:41.067 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : libacl-2.3.1-4.el9.x86_64 22/202 2024-03-06T07:24:41.263 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-06T07:24:41.351 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-06T07:24:41.547 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-06T07:24:41.610 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-06T07:24:41.927 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-06T07:24:42.087 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : re2-1:20211101-3.el9.x86_64 25/202 2024-03-06T07:24:42.202 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 26/202 2024-03-06T07:24:42.241 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 27/202 2024-03-06T07:24:42.326 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : liboath-2.6.7-2.el9.x86_64 28/202 2024-03-06T07:24:42.386 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 29/202 2024-03-06T07:24:42.430 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 30/202 2024-03-06T07:24:42.495 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 31/202 2024-03-06T07:24:42.549 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 32/202 2024-03-06T07:24:42.648 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 33/202 2024-03-06T07:24:42.688 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 34/202 2024-03-06T07:24:42.750 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 35/202 2024-03-06T07:24:42.793 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 36/202 2024-03-06T07:24:42.833 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : lua-5.4.4-4.el9.x86_64 37/202 2024-03-06T07:24:42.945 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 38/202 2024-03-06T07:24:42.987 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 39/202 2024-03-06T07:24:43.047 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 40/202 2024-03-06T07:24:43.085 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 41/202 2024-03-06T07:24:43.219 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 42/202 2024-03-06T07:24:43.289 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : luarocks-3.9.2-1.el9.noarch 43/202 2024-03-06T07:24:43.388 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 44/202 2024-03-06T07:24:43.567 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 45/202 2024-03-06T07:24:43.633 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 46/202 2024-03-06T07:24:44.338 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-rsa-4.9-2.el9.noarch 47/202 2024-03-06T07:24:44.430 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 48/202 2024-03-06T07:24:44.532 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 49/202 2024-03-06T07:24:44.589 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-06T07:24:44.614 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-06T07:24:44.665 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : qatzip-libs-1.1.2-2.el9.x86_64 51/202 2024-03-06T07:24:44.709 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 52/202 2024-03-06T07:24:44.766 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 53/202 2024-03-06T07:24:44.942 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : elfutils-libs-0.190-2.el9.x86_64 54/202 2024-03-06T07:24:44.993 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : iproute-6.7.0-2.el9.x86_64 55/202 2024-03-06T07:24:45.350 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : libfdisk-2.37.4-18.el9.x86_64 56/202 2024-03-06T07:24:45.550 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : util-linux-2.37.4-18.el9.x86_64 57/202 2024-03-06T07:24:45.654 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : xfsprogs-6.3.0-1.el9.x86_64 58/202 2024-03-06T07:24:45.720 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 59/202 2024-03-06T07:24:45.930 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 60/202 2024-03-06T07:24:45.983 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : openssl-1:3.0.7-27.el9.x86_64 61/202 2024-03-06T07:24:46.057 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 62/202 2024-03-06T07:24:46.153 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 63/202 2024-03-06T07:24:46.200 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 64/202 2024-03-06T07:24:46.238 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 65/202 2024-03-06T07:24:46.307 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 66/202 2024-03-06T07:24:46.345 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 67/202 2024-03-06T07:24:46.418 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 68/202 2024-03-06T07:24:46.485 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 69/202 2024-03-06T07:24:46.671 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 70/202 2024-03-06T07:24:46.726 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 71/202 2024-03-06T07:24:46.764 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 72/202 2024-03-06T07:24:46.808 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 73/202 2024-03-06T07:24:46.935 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 74/202 2024-03-06T07:24:46.972 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-google-auth-1:2.28.0-1.el9.noarch 75/202 2024-03-06T07:24:47.015 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 76/202 2024-03-06T07:24:47.046 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 77/202 2024-03-06T07:24:47.078 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 78/202 2024-03-06T07:24:47.271 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 79/202 2024-03-06T07:24:47.449 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 80/202 2024-03-06T07:24:47.491 INFO:teuthology.orchestra.run.smithi086.stdout: Preparing : 1/1 2024-03-06T07:24:47.595 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 81/202 2024-03-06T07:24:47.599 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : openssl-libs-1:3.0.7-27.el9.x86_64 1/202 2024-03-06T07:24:47.648 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : libuuid-2.37.4-18.el9.x86_64 2/202 2024-03-06T07:24:47.648 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 82/202 2024-03-06T07:24:47.678 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-06T07:24:47.710 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-ply-3.11-14.el9.noarch 83/202 2024-03-06T07:24:47.766 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-06T07:24:47.786 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 84/202 2024-03-06T07:24:47.847 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : elfutils-libelf-0.190-2.el9.x86_64 4/202 2024-03-06T07:24:47.956 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 5/202 2024-03-06T07:24:48.037 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 85/202 2024-03-06T07:24:48.091 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/202 2024-03-06T07:24:48.102 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 86/202 2024-03-06T07:24:48.150 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : thrift-0.15.0-2.el9.x86_64 7/202 2024-03-06T07:24:48.187 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 8/202 2024-03-06T07:24:48.278 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/202 2024-03-06T07:24:48.324 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-asyncssh-2.13.2-4.el9.noarch 88/202 2024-03-06T07:24:48.365 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 89/202 2024-03-06T07:24:48.405 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 90/202 2024-03-06T07:24:48.439 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 9/202 2024-03-06T07:24:48.444 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : jq-1.6-16.el9.x86_64 91/202 2024-03-06T07:24:48.480 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : fuse-2.9.9-15.el9.x86_64 92/202 2024-03-06T07:24:48.514 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : zlib-devel-1.2.11-40.el9.x86_64 93/202 2024-03-06T07:24:48.558 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 94/202 2024-03-06T07:24:48.627 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : uuid-1.6.2-55.el9.x86_64 95/202 2024-03-06T07:24:48.683 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 96/202 2024-03-06T07:24:48.730 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 97/202 2024-03-06T07:24:48.858 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 98/202 2024-03-06T07:24:48.900 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 99/202 2024-03-06T07:24:48.901 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 10/202 2024-03-06T07:24:48.934 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 100/202 2024-03-06T07:24:48.950 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-06T07:24:48.997 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 101/202 2024-03-06T07:24:49.038 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 102/202 2024-03-06T07:24:49.117 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-06T07:24:49.135 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-06T07:24:49.180 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 103/202 2024-03-06T07:24:49.245 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-06T07:24:49.283 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-06T07:24:49.300 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 104/202 2024-03-06T07:24:49.352 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 105/202 2024-03-06T07:24:49.387 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 106/202 2024-03-06T07:24:49.444 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 107/202 2024-03-06T07:24:49.518 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : postgresql-private-libs-13.11-1.el9.x86_64 14/202 2024-03-06T07:24:49.573 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : postgresql-13.11-1.el9.x86_64 15/202 2024-03-06T07:24:49.583 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-packaging-20.9-5.el9.noarch 108/202 2024-03-06T07:24:49.590 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-06T07:24:49.654 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 109/202 2024-03-06T07:24:49.734 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-06T07:24:49.758 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-06T07:24:49.863 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-06T07:24:49.936 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : libmount-2.37.4-18.el9.x86_64 18/202 2024-03-06T07:24:49.936 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-06T07:24:49.987 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-06T07:24:50.240 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-06T07:24:50.318 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-06T07:24:50.351 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : libsmartcols-2.37.4-18.el9.x86_64 21/202 2024-03-06T07:24:50.627 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 110/202 2024-03-06T07:24:50.681 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : libacl-2.3.1-4.el9.x86_64 22/202 2024-03-06T07:24:50.693 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 111/202 2024-03-06T07:24:50.727 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 112/202 2024-03-06T07:24:50.785 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 113/202 2024-03-06T07:24:50.896 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 114/202 2024-03-06T07:24:50.900 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-06T07:24:50.952 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 115/202 2024-03-06T07:24:51.014 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-06T07:24:51.179 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : openblas-0.3.21-2.el9.x86_64 116/202 2024-03-06T07:24:51.234 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-06T07:24:51.234 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 117/202 2024-03-06T07:24:51.287 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-06T07:24:51.342 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 118/202 2024-03-06T07:24:51.600 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-06T07:24:51.778 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : re2-1:20211101-3.el9.x86_64 25/202 2024-03-06T07:24:51.903 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 26/202 2024-03-06T07:24:51.959 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 27/202 2024-03-06T07:24:52.062 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : liboath-2.6.7-2.el9.x86_64 28/202 2024-03-06T07:24:52.140 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 29/202 2024-03-06T07:24:52.142 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 119/202 2024-03-06T07:24:52.172 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 30/202 2024-03-06T07:24:52.210 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 31/202 2024-03-06T07:24:52.262 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 32/202 2024-03-06T07:24:52.279 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 120/202 2024-03-06T07:24:52.364 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 33/202 2024-03-06T07:24:52.430 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 34/202 2024-03-06T07:24:52.504 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 35/202 2024-03-06T07:24:52.555 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 36/202 2024-03-06T07:24:52.594 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : lua-5.4.4-4.el9.x86_64 37/202 2024-03-06T07:24:52.710 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 38/202 2024-03-06T07:24:52.748 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 39/202 2024-03-06T07:24:52.803 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 40/202 2024-03-06T07:24:52.837 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 41/202 2024-03-06T07:24:52.935 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 42/202 2024-03-06T07:24:53.006 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : luarocks-3.9.2-1.el9.noarch 43/202 2024-03-06T07:24:53.106 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 44/202 2024-03-06T07:24:53.281 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 45/202 2024-03-06T07:24:53.350 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 46/202 2024-03-06T07:24:53.815 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 121/202 2024-03-06T07:24:53.896 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 122/202 2024-03-06T07:24:53.931 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 123/202 2024-03-06T07:24:54.063 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-rsa-4.9-2.el9.noarch 47/202 2024-03-06T07:24:54.164 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 48/202 2024-03-06T07:24:54.281 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 124/202 2024-03-06T07:24:54.291 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 49/202 2024-03-06T07:24:54.325 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-06T07:24:54.341 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-06T07:24:54.366 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-06T07:24:54.425 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : qatzip-libs-1.1.2-2.el9.x86_64 51/202 2024-03-06T07:24:54.444 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-06T07:24:54.471 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 52/202 2024-03-06T07:24:54.472 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 126/202 2024-03-06T07:24:54.525 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 53/202 2024-03-06T07:24:54.528 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 127/202 2024-03-06T07:24:54.703 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : elfutils-libs-0.190-2.el9.x86_64 54/202 2024-03-06T07:24:54.763 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : iproute-6.7.0-2.el9.x86_64 55/202 2024-03-06T07:24:55.028 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 128/202 2024-03-06T07:24:55.062 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-06T07:24:55.127 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : libfdisk-2.37.4-18.el9.x86_64 56/202 2024-03-06T07:24:55.135 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-06T07:24:55.163 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 130/202 2024-03-06T07:24:55.311 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : util-linux-2.37.4-18.el9.x86_64 57/202 2024-03-06T07:24:55.422 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : xfsprogs-6.3.0-1.el9.x86_64 58/202 2024-03-06T07:24:55.496 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 59/202 2024-03-06T07:24:55.695 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 60/202 2024-03-06T07:24:55.745 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : openssl-1:3.0.7-27.el9.x86_64 61/202 2024-03-06T07:24:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 62/202 2024-03-06T07:24:55.904 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 63/202 2024-03-06T07:24:55.944 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 64/202 2024-03-06T07:24:55.983 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 65/202 2024-03-06T07:24:56.049 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 66/202 2024-03-06T07:24:56.091 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 67/202 2024-03-06T07:24:56.153 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 68/202 2024-03-06T07:24:56.215 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 69/202 2024-03-06T07:24:56.399 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 70/202 2024-03-06T07:24:56.456 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 71/202 2024-03-06T07:24:56.494 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 72/202 2024-03-06T07:24:56.551 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 73/202 2024-03-06T07:24:56.658 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 74/202 2024-03-06T07:24:56.693 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-google-auth-1:2.28.0-1.el9.noarch 75/202 2024-03-06T07:24:56.735 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 76/202 2024-03-06T07:24:56.766 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 77/202 2024-03-06T07:24:56.799 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 78/202 2024-03-06T07:24:56.998 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 79/202 2024-03-06T07:24:57.173 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 80/202 2024-03-06T07:24:57.307 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 81/202 2024-03-06T07:24:57.384 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 82/202 2024-03-06T07:24:57.454 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-ply-3.11-14.el9.noarch 83/202 2024-03-06T07:24:57.531 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 84/202 2024-03-06T07:24:57.636 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-06T07:24:57.676 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-06T07:24:57.774 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 85/202 2024-03-06T07:24:57.830 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 86/202 2024-03-06T07:24:58.027 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/202 2024-03-06T07:24:58.069 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-asyncssh-2.13.2-4.el9.noarch 88/202 2024-03-06T07:24:58.110 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 89/202 2024-03-06T07:24:58.150 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 90/202 2024-03-06T07:24:58.190 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : jq-1.6-16.el9.x86_64 91/202 2024-03-06T07:24:58.226 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : fuse-2.9.9-15.el9.x86_64 92/202 2024-03-06T07:24:58.268 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : zlib-devel-1.2.11-40.el9.x86_64 93/202 2024-03-06T07:24:58.291 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-06T07:24:58.313 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 94/202 2024-03-06T07:24:58.329 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-06T07:24:58.415 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : uuid-1.6.2-55.el9.x86_64 95/202 2024-03-06T07:24:58.469 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 96/202 2024-03-06T07:24:58.509 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-06T07:24:58.509 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-06T07:24:58.509 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-06T07:24:58.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:24:58.518 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 97/202 2024-03-06T07:24:58.547 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-06T07:24:58.645 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 98/202 2024-03-06T07:24:58.687 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 99/202 2024-03-06T07:24:58.722 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 100/202 2024-03-06T07:24:58.775 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 101/202 2024-03-06T07:24:58.845 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 102/202 2024-03-06T07:24:58.984 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 103/202 2024-03-06T07:24:59.102 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 104/202 2024-03-06T07:24:59.162 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 105/202 2024-03-06T07:24:59.199 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 106/202 2024-03-06T07:24:59.256 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 107/202 2024-03-06T07:24:59.395 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-packaging-20.9-5.el9.noarch 108/202 2024-03-06T07:24:59.464 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 109/202 2024-03-06T07:25:00.417 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 110/202 2024-03-06T07:25:00.472 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 111/202 2024-03-06T07:25:00.505 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 112/202 2024-03-06T07:25:00.561 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 113/202 2024-03-06T07:25:00.671 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 114/202 2024-03-06T07:25:00.714 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 115/202 2024-03-06T07:25:00.935 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : openblas-0.3.21-2.el9.x86_64 116/202 2024-03-06T07:25:00.961 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 117/202 2024-03-06T07:25:01.064 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 118/202 2024-03-06T07:25:01.837 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 119/202 2024-03-06T07:25:01.972 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 120/202 2024-03-06T07:25:03.474 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 121/202 2024-03-06T07:25:03.574 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 122/202 2024-03-06T07:25:03.602 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 123/202 2024-03-06T07:25:03.954 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 124/202 2024-03-06T07:25:03.986 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-06T07:25:04.114 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-06T07:25:04.142 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 126/202 2024-03-06T07:25:04.196 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 127/202 2024-03-06T07:25:04.732 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 128/202 2024-03-06T07:25:04.773 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-06T07:25:04.873 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-06T07:25:04.900 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 130/202 2024-03-06T07:25:07.503 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-06T07:25:07.537 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-06T07:25:08.156 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-06T07:25:08.227 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-06T07:25:08.390 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-06T07:25:08.390 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-06T07:25:08.390 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-06T07:25:08.390 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:08.433 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-06T07:25:11.232 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-06T07:25:11.232 INFO:teuthology.orchestra.run.smithi194.stdout:skipping the directory /sys 2024-03-06T07:25:11.232 INFO:teuthology.orchestra.run.smithi194.stdout:skipping the directory /proc 2024-03-06T07:25:11.233 INFO:teuthology.orchestra.run.smithi194.stdout:skipping the directory /mnt 2024-03-06T07:25:11.233 INFO:teuthology.orchestra.run.smithi194.stdout:skipping the directory /var/tmp 2024-03-06T07:25:11.233 INFO:teuthology.orchestra.run.smithi194.stdout:skipping the directory /home 2024-03-06T07:25:11.233 INFO:teuthology.orchestra.run.smithi194.stdout:skipping the directory /root 2024-03-06T07:25:11.233 INFO:teuthology.orchestra.run.smithi194.stdout:skipping the directory /tmp 2024-03-06T07:25:11.233 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:11.451 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-06T07:25:11.494 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-06T07:25:11.494 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:11.494 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-03-06T07:25:11.494 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:13.095 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-06T07:25:13.198 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-06T07:25:13.198 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:13.199 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-06T07:25:13.199 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-06T07:25:13.199 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-06T07:25:13.199 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:13.445 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-06T07:25:13.537 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-06T07:25:13.537 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:13.537 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-06T07:25:13.537 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-06T07:25:13.537 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-06T07:25:13.537 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:14.012 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-06T07:25:14.077 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-06T07:25:14.078 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:14.078 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-06T07:25:14.078 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-06T07:25:14.078 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-06T07:25:14.078 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:14.197 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : automake-1.16.2-8.el9.noarch 138/202 2024-03-06T07:25:14.769 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-06T07:25:14.806 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-06T07:25:14.866 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 140/202 2024-03-06T07:25:14.967 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 141/202 2024-03-06T07:25:15.002 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-06T07:25:15.327 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-06T07:25:15.361 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-06T07:25:16.477 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-06T07:25:16.503 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-06T07:25:16.644 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-06T07:25:16.815 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 145/202 2024-03-06T07:25:16.847 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-06T07:25:16.913 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-06T07:25:16.913 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:16.913 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-06T07:25:16.913 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-06T07:25:16.913 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-06T07:25:16.913 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:16.972 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-06T07:25:17.009 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-06T07:25:17.111 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 148/202 2024-03-06T07:25:18.056 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libtool-2.4.6-45.el9.x86_64 149/202 2024-03-06T07:25:18.091 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-06T07:25:18.158 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-06T07:25:18.158 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:18.158 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-06T07:25:18.158 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-06T07:25:18.158 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-06T07:25:18.158 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:18.216 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-06T07:25:18.287 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-06T07:25:18.287 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:18.287 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-06T07:25:18.287 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:18.359 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-06T07:25:18.430 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-06T07:25:18.431 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:18.431 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-03-06T07:25:18.431 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-06T07:25:18.431 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-06T07:25:18.431 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:18.776 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-06T07:25:18.846 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-06T07:25:18.846 INFO:teuthology.orchestra.run.smithi194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:18.846 INFO:teuthology.orchestra.run.smithi194.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-06T07:25:18.846 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-06T07:25:18.846 INFO:teuthology.orchestra.run.smithi194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-06T07:25:18.846 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:21.791 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-06T07:25:21.791 INFO:teuthology.orchestra.run.smithi086.stdout:skipping the directory /sys 2024-03-06T07:25:21.791 INFO:teuthology.orchestra.run.smithi086.stdout:skipping the directory /proc 2024-03-06T07:25:21.791 INFO:teuthology.orchestra.run.smithi086.stdout:skipping the directory /mnt 2024-03-06T07:25:21.792 INFO:teuthology.orchestra.run.smithi086.stdout:skipping the directory /var/tmp 2024-03-06T07:25:21.792 INFO:teuthology.orchestra.run.smithi086.stdout:skipping the directory /home 2024-03-06T07:25:21.792 INFO:teuthology.orchestra.run.smithi086.stdout:skipping the directory /root 2024-03-06T07:25:21.792 INFO:teuthology.orchestra.run.smithi086.stdout:skipping the directory /tmp 2024-03-06T07:25:21.792 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:21.996 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-06T07:25:22.042 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-06T07:25:22.043 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:22.043 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-03-06T07:25:22.043 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:23.573 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 154/202 2024-03-06T07:25:23.614 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 155/202 2024-03-06T07:25:23.645 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 156/202 2024-03-06T07:25:23.673 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-06T07:25:23.682 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 157/202 2024-03-06T07:25:23.749 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-06T07:25:23.749 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:23.749 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-06T07:25:23.749 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-06T07:25:23.750 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-06T07:25:23.750 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:23.944 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 158/202 2024-03-06T07:25:24.006 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-06T07:25:24.066 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : postgresql-contrib-13.11-1.el9.x86_64 159/202 2024-03-06T07:25:24.074 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-06T07:25:24.074 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:24.074 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-06T07:25:24.074 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-06T07:25:24.074 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-06T07:25:24.075 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:24.114 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 160/202 2024-03-06T07:25:24.158 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 161/202 2024-03-06T07:25:24.246 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : xfsdump-3.1.12-4.el9.x86_64 162/202 2024-03-06T07:25:24.341 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : util-linux-user-2.37.4-18.el9.x86_64 163/202 2024-03-06T07:25:24.382 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : iproute-tc-6.7.0-2.el9.x86_64 164/202 2024-03-06T07:25:24.427 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : elfutils-debuginfod-client-0.190-2.el9.x86_64 165/202 2024-03-06T07:25:24.455 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 166/202 2024-03-06T07:25:24.507 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : NetworkManager-team-1:1.46.0-1.el9.x86_64 167/202 2024-03-06T07:25:24.557 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-06T07:25:24.623 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-06T07:25:24.624 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:24.624 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-06T07:25:24.624 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-06T07:25:24.624 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-06T07:25:24.624 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:24.645 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : NetworkManager-tui-1:1.46.0-1.el9.x86_64 168/202 2024-03-06T07:25:24.703 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : libacl-devel-2.3.1-4.el9.x86_64 169/202 2024-03-06T07:25:24.742 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : automake-1.16.2-8.el9.noarch 138/202 2024-03-06T07:25:24.749 INFO:teuthology.orchestra.run.smithi194.stdout: Upgrading : acl-2.3.1-4.el9.x86_64 170/202 2024-03-06T07:25:25.373 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-06T07:25:25.409 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-06T07:25:25.460 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 140/202 2024-03-06T07:25:25.562 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 141/202 2024-03-06T07:25:25.588 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-06T07:25:25.593 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-06T07:25:25.647 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-06T07:25:25.701 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-06T07:25:25.910 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-06T07:25:25.948 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-06T07:25:27.020 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 172/202 2024-03-06T07:25:27.061 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-06T07:25:27.101 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : openssl-devel-1:3.0.7-27.el9.x86_64 173/202 2024-03-06T07:25:27.114 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-06T07:25:27.149 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 174/202 2024-03-06T07:25:27.258 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-06T07:25:27.285 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : flex-2.6.4-9.el9.x86_64 175/202 2024-03-06T07:25:27.337 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : bison-3.7.4-5.el9.x86_64 176/202 2024-03-06T07:25:27.370 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : NetworkManager-tui-1:1.44.0-2.el9.x86_64 177/202 2024-03-06T07:25:27.405 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : elfutils-debuginfod-client-0.189-3.el9.x86_64 178/202 2024-03-06T07:25:27.435 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 145/202 2024-03-06T07:25:27.449 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : elfutils-libs-0.189-3.el9.x86_64 179/202 2024-03-06T07:25:27.449 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-06T07:25:27.467 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-06T07:25:27.535 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-06T07:25:27.535 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:27.535 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-06T07:25:27.535 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-06T07:25:27.535 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-06T07:25:27.535 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:27.605 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-06T07:25:27.609 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-06T07:25:27.647 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-06T07:25:27.655 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : libacl-devel-2.3.1-3.el9.x86_64 181/202 2024-03-06T07:25:27.701 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : openssl-1:3.0.7-24.el9.x86_64 182/202 2024-03-06T07:25:27.701 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-06T07:25:27.728 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 148/202 2024-03-06T07:25:27.839 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-06T07:25:27.892 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : xfsdump-3.1.12-3.el9.x86_64 184/202 2024-03-06T07:25:27.948 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : xfsprogs-5.19.0-4.el9.x86_64 185/202 2024-03-06T07:25:27.987 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : iproute-tc-6.2.0-5.el9.x86_64 186/202 2024-03-06T07:25:28.034 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : acl-2.3.1-3.el9.x86_64 187/202 2024-03-06T07:25:28.080 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : iproute-6.2.0-5.el9.x86_64 188/202 2024-03-06T07:25:28.156 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : util-linux-user-2.37.4-15.el9.x86_64 189/202 2024-03-06T07:25:28.235 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : util-linux-2.37.4-15.el9.x86_64 190/202 2024-03-06T07:25:28.306 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : util-linux-core-2.37.4-15.el9.x86_64 191/202 2024-03-06T07:25:28.342 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : libfdisk-2.37.4-15.el9.x86_64 192/202 2024-03-06T07:25:28.376 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : libmount-2.37.4-15.el9.x86_64 193/202 2024-03-06T07:25:28.410 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : libblkid-2.37.4-15.el9.x86_64 194/202 2024-03-06T07:25:28.443 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : NetworkManager-team-1:1.44.0-2.el9.x86_64 195/202 2024-03-06T07:25:28.479 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-06T07:25:28.480 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-06T07:25:28.676 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libtool-2.4.6-45.el9.x86_64 149/202 2024-03-06T07:25:28.711 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-06T07:25:28.788 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-06T07:25:28.788 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:28.788 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-06T07:25:28.788 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-06T07:25:28.788 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-06T07:25:28.788 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:28.809 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-06T07:25:28.809 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-06T07:25:28.845 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-06T07:25:28.915 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-06T07:25:28.916 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:28.916 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-06T07:25:28.916 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:28.948 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-06T07:25:28.996 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-06T07:25:29.000 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : libuuid-2.37.4-15.el9.x86_64 198/202 2024-03-06T07:25:29.038 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : libsmartcols-2.37.4-15.el9.x86_64 199/202 2024-03-06T07:25:29.070 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : elfutils-libelf-0.189-3.el9.x86_64 200/202 2024-03-06T07:25:29.075 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-06T07:25:29.075 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:29.075 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-03-06T07:25:29.075 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-06T07:25:29.075 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-06T07:25:29.075 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:29.107 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : libacl-2.3.1-3.el9.x86_64 201/202 2024-03-06T07:25:29.234 INFO:teuthology.orchestra.run.smithi194.stdout: Cleanup : openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-06T07:25:29.471 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-06T07:25:29.510 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-06T07:25:29.510 INFO:teuthology.orchestra.run.smithi194.stdout:Couldn't write '0' to 'kernel/yama/ptrace_scope', ignoring: No such file or directory 2024-03-06T07:25:29.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:29.541 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-06T07:25:29.541 INFO:teuthology.orchestra.run.smithi086.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-06T07:25:29.541 INFO:teuthology.orchestra.run.smithi086.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-06T07:25:29.541 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-06T07:25:29.541 INFO:teuthology.orchestra.run.smithi086.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-06T07:25:29.541 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 1/202 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2/202 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 3/202 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 4/202 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 5/202 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 6/202 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 7/202 2024-03-06T07:25:34.119 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 8/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 9/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 10/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 14/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 15/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 17/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 18/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 19/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 20/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 21/202 2024-03-06T07:25:34.120 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 22/202 2024-03-06T07:25:34.121 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 23/202 2024-03-06T07:25:34.121 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 24/202 2024-03-06T07:25:34.121 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 25/202 2024-03-06T07:25:34.121 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 26/202 2024-03-06T07:25:34.121 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 27/202 2024-03-06T07:25:34.121 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 28/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 29/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 30/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 31/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 32/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 33/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 34/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 35/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 36/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 37/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 38/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 39/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : automake-1.16.2-8.el9.noarch 40/202 2024-03-06T07:25:34.137 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : bison-3.7.4-5.el9.x86_64 41/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 42/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 43/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : flex-2.6.4-9.el9.x86_64 44/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 48/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 49/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 50/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 51/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 52/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 53/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 54/202 2024-03-06T07:25:34.138 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 55/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : lua-5.4.4-4.el9.x86_64 56/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 57/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 58/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : openssl-devel-1:3.0.7-27.el9.x86_64 59/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : postgresql-13.11-1.el9.x86_64 60/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : postgresql-contrib-13.11-1.el9.x86_64 61/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : postgresql-private-libs-13.11-1.el9.x86_64 62/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : postgresql-server-13.11-1.el9.x86_64 63/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 64/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 65/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 66/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 67/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 68/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 69/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 70/202 2024-03-06T07:25:34.139 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 71/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 72/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 73/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 74/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 75/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 76/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 77/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 78/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 79/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 80/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 81/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 82/202 2024-03-06T07:25:34.140 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 83/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 84/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : qatlib-23.11.0-1.el9.x86_64 85/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : qatlib-service-23.11.0-1.el9.x86_64 86/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : qatzip-libs-1.1.2-2.el9.x86_64 87/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 88/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 89/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 90/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 91/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : zlib-devel-1.2.11-40.el9.x86_64 92/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 93/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : jq-1.6-16.el9.x86_64 94/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 95/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 96/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 97/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 98/202 2024-03-06T07:25:34.141 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 99/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-ply-3.11-14.el9.noarch 100/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 101/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 102/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 103/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 104/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 105/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 106/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 107/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 108/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 109/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 110/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 111/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 112/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 113/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 114/202 2024-03-06T07:25:34.142 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 115/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-asyncssh-2.13.2-4.el9.noarch 116/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 117/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 118/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 119/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 120/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 121/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 122/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 123/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-google-auth-1:2.28.0-1.el9.noarch 124/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 125/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 126/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 127/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 128/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 129/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 130/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 131/202 2024-03-06T07:25:34.143 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 132/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 133/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 134/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 135/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 136/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 137/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 138/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 139/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 140/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 141/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 142/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 143/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 144/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 145/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 146/202 2024-03-06T07:25:34.144 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 147/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 148/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : re2-1:20211101-3.el9.x86_64 149/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 150/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 151/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 152/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 154/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libacl-devel-2.3.1-4.el9.x86_64 155/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libacl-devel-2.3.1-3.el9.x86_64 156/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-1:1.46.0-1.el9.x86_64 157/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-1:1.44.0-2.el9.x86_64 158/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 159/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 160/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-team-1:1.46.0-1.el9.x86_64 161/202 2024-03-06T07:25:34.145 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-team-1:1.44.0-2.el9.x86_64 162/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-tui-1:1.46.0-1.el9.x86_64 163/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : NetworkManager-tui-1:1.44.0-2.el9.x86_64 164/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : acl-2.3.1-4.el9.x86_64 165/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : acl-2.3.1-3.el9.x86_64 166/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 167/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : elfutils-debuginfod-client-0.189-3.el9.x86_64 168/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 169/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : elfutils-libelf-0.189-3.el9.x86_64 170/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 171/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : elfutils-libs-0.189-3.el9.x86_64 172/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : iproute-6.7.0-2.el9.x86_64 173/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : iproute-6.2.0-5.el9.x86_64 174/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : iproute-tc-6.7.0-2.el9.x86_64 175/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : iproute-tc-6.2.0-5.el9.x86_64 176/202 2024-03-06T07:25:34.146 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libacl-2.3.1-4.el9.x86_64 177/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libacl-2.3.1-3.el9.x86_64 178/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libblkid-2.37.4-18.el9.x86_64 179/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libblkid-2.37.4-15.el9.x86_64 180/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libfdisk-2.37.4-18.el9.x86_64 181/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libfdisk-2.37.4-15.el9.x86_64 182/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libmount-2.37.4-18.el9.x86_64 183/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libmount-2.37.4-15.el9.x86_64 184/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libsmartcols-2.37.4-18.el9.x86_64 185/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libsmartcols-2.37.4-15.el9.x86_64 186/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libuuid-2.37.4-18.el9.x86_64 187/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : libuuid-2.37.4-15.el9.x86_64 188/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 189/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : openssl-1:3.0.7-24.el9.x86_64 190/202 2024-03-06T07:25:34.147 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 191/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : openssl-libs-1:3.0.7-24.el9.x86_64 192/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : util-linux-2.37.4-18.el9.x86_64 193/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : util-linux-2.37.4-15.el9.x86_64 194/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : util-linux-core-2.37.4-18.el9.x86_64 195/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : util-linux-core-2.37.4-15.el9.x86_64 196/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : util-linux-user-2.37.4-18.el9.x86_64 197/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : util-linux-user-2.37.4-15.el9.x86_64 198/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : xfsdump-3.1.12-4.el9.x86_64 199/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : xfsdump-3.1.12-3.el9.x86_64 200/202 2024-03-06T07:25:34.148 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 201/202 2024-03-06T07:25:34.268 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 154/202 2024-03-06T07:25:34.309 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 155/202 2024-03-06T07:25:34.340 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 156/202 2024-03-06T07:25:34.370 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 157/202 2024-03-06T07:25:34.634 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 158/202 2024-03-06T07:25:34.760 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : postgresql-contrib-13.11-1.el9.x86_64 159/202 2024-03-06T07:25:34.809 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 160/202 2024-03-06T07:25:34.859 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 161/202 2024-03-06T07:25:34.951 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : xfsdump-3.1.12-4.el9.x86_64 162/202 2024-03-06T07:25:35.064 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : util-linux-user-2.37.4-18.el9.x86_64 163/202 2024-03-06T07:25:35.110 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : iproute-tc-6.7.0-2.el9.x86_64 164/202 2024-03-06T07:25:35.156 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : elfutils-debuginfod-client-0.190-2.el9.x86_64 165/202 2024-03-06T07:25:35.184 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 166/202 2024-03-06T07:25:35.236 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : NetworkManager-team-1:1.46.0-1.el9.x86_64 167/202 2024-03-06T07:25:35.356 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : NetworkManager-tui-1:1.46.0-1.el9.x86_64 168/202 2024-03-06T07:25:35.416 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : libacl-devel-2.3.1-4.el9.x86_64 169/202 2024-03-06T07:25:35.485 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrading : acl-2.3.1-4.el9.x86_64 170/202 2024-03-06T07:25:36.438 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-06T07:25:36.564 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : xfsprogs-5.19.0-4.el9.x86_64 202/202 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout:Upgraded: 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager-team-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: NetworkManager-tui-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: acl-2.3.1-4.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-debuginfod-client-0.190-2.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-libelf-0.190-2.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-libs-0.190-2.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: iproute-6.7.0-2.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: iproute-tc-6.7.0-2.el9.x86_64 2024-03-06T07:25:36.565 INFO:teuthology.orchestra.run.smithi194.stdout: libacl-2.3.1-4.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: libacl-devel-2.3.1-4.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: libblkid-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: libfdisk-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: libmount-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: libsmartcols-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: libuuid-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: openssl-1:3.0.7-27.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: openssl-libs-1:3.0.7-27.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: util-linux-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: util-linux-core-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: util-linux-user-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.566 INFO:teuthology.orchestra.run.smithi194.stdout: xfsdump-3.1.12-4.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: xfsprogs-6.3.0-1.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout:Installed: 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: automake-1.16.2-8.el9.noarch 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: bison-3.7.4-5.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-immutable-object-cache-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.567 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.568 INFO:teuthology.orchestra.run.smithi194.stdout: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: flex-2.6.4-9.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: fuse-2.9.9-15.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: grpc-data-1.46.7-10.el9.noarch 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: jq-1.6-16.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: libarrow-9.0.0-10.el9.x86_64 2024-03-06T07:25:36.569 INFO:teuthology.orchestra.run.smithi194.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libtool-2.4.6-45.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-06T07:25:36.570 INFO:teuthology.orchestra.run.smithi194.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: lua-5.4.4-4.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: luarocks-3.9.2-1.el9.noarch 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: openssl-devel-1:3.0.7-27.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql-13.11-1.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql-contrib-13.11-1.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql-private-libs-13.11-1.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: postgresql-server-13.11-1.el9.x86_64 2024-03-06T07:25:36.571 INFO:teuthology.orchestra.run.smithi194.stdout: protobuf-3.14.0-13.el9.x86_64 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-asyncssh-2.13.2-4.el9.noarch 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-06T07:25:36.572 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-google-auth-1:2.28.0-1.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-06T07:25:36.573 INFO:teuthology.orchestra.run.smithi194.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-packaging-20.9-5.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-ply-3.11-14.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-03-06T07:25:36.574 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-06T07:25:36.575 INFO:teuthology.orchestra.run.smithi194.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: qatlib-23.11.0-1.el9.x86_64 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: qatlib-service-23.11.0-1.el9.x86_64 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: qatzip-libs-1.1.2-2.el9.x86_64 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.576 INFO:teuthology.orchestra.run.smithi194.stdout: rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: re2-1:20211101-3.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: uuid-1.6.2-55.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: zlib-devel-1.2.11-40.el9.x86_64 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:36.577 INFO:teuthology.orchestra.run.smithi194.stdout:Complete! 2024-03-06T07:25:36.765 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-06T07:25:36.854 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-06T07:25:37.190 DEBUG:teuthology.parallel:result is None 2024-03-06T07:25:38.263 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 172/202 2024-03-06T07:25:38.361 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : openssl-devel-1:3.0.7-27.el9.x86_64 173/202 2024-03-06T07:25:38.443 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 174/202 2024-03-06T07:25:38.592 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : flex-2.6.4-9.el9.x86_64 175/202 2024-03-06T07:25:38.630 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : bison-3.7.4-5.el9.x86_64 176/202 2024-03-06T07:25:38.662 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : NetworkManager-tui-1:1.44.0-2.el9.x86_64 177/202 2024-03-06T07:25:38.697 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : elfutils-debuginfod-client-0.189-3.el9.x86_64 178/202 2024-03-06T07:25:38.739 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : elfutils-libs-0.189-3.el9.x86_64 179/202 2024-03-06T07:25:38.739 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-06T07:25:38.905 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-06T07:25:38.955 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : libacl-devel-2.3.1-3.el9.x86_64 181/202 2024-03-06T07:25:39.001 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : openssl-1:3.0.7-24.el9.x86_64 182/202 2024-03-06T07:25:39.001 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-06T07:25:39.131 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-06T07:25:39.183 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : xfsdump-3.1.12-3.el9.x86_64 184/202 2024-03-06T07:25:39.239 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : xfsprogs-5.19.0-4.el9.x86_64 185/202 2024-03-06T07:25:39.279 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : iproute-tc-6.2.0-5.el9.x86_64 186/202 2024-03-06T07:25:39.329 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : acl-2.3.1-3.el9.x86_64 187/202 2024-03-06T07:25:39.397 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : iproute-6.2.0-5.el9.x86_64 188/202 2024-03-06T07:25:39.462 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : util-linux-user-2.37.4-15.el9.x86_64 189/202 2024-03-06T07:25:39.534 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : util-linux-2.37.4-15.el9.x86_64 190/202 2024-03-06T07:25:39.576 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : util-linux-core-2.37.4-15.el9.x86_64 191/202 2024-03-06T07:25:39.609 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : libfdisk-2.37.4-15.el9.x86_64 192/202 2024-03-06T07:25:39.642 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : libmount-2.37.4-15.el9.x86_64 193/202 2024-03-06T07:25:39.676 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : libblkid-2.37.4-15.el9.x86_64 194/202 2024-03-06T07:25:39.709 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : NetworkManager-team-1:1.44.0-2.el9.x86_64 195/202 2024-03-06T07:25:39.745 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-06T07:25:39.746 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-06T07:25:40.062 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-06T07:25:40.062 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-06T07:25:40.201 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-06T07:25:40.252 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : libuuid-2.37.4-15.el9.x86_64 198/202 2024-03-06T07:25:40.281 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : libsmartcols-2.37.4-15.el9.x86_64 199/202 2024-03-06T07:25:40.314 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : elfutils-libelf-0.189-3.el9.x86_64 200/202 2024-03-06T07:25:40.384 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : libacl-2.3.1-3.el9.x86_64 201/202 2024-03-06T07:25:40.510 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-06T07:25:40.769 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-06T07:25:40.769 INFO:teuthology.orchestra.run.smithi086.stdout:Couldn't write '0' to 'kernel/yama/ptrace_scope', ignoring: No such file or directory 2024-03-06T07:25:40.769 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:45.348 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 1/202 2024-03-06T07:25:45.348 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2/202 2024-03-06T07:25:45.348 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 3/202 2024-03-06T07:25:45.348 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 4/202 2024-03-06T07:25:45.348 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 5/202 2024-03-06T07:25:45.348 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 6/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 7/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 8/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 9/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 10/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 14/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 15/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 17/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 18/202 2024-03-06T07:25:45.349 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 19/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 20/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 21/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 22/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 23/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 24/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 25/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 26/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 27/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 28/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 29/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 30/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 31/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 32/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 33/202 2024-03-06T07:25:45.350 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 34/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 35/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 36/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 37/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 38/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 39/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : automake-1.16.2-8.el9.noarch 40/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : bison-3.7.4-5.el9.x86_64 41/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 42/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 43/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : flex-2.6.4-9.el9.x86_64 44/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/202 2024-03-06T07:25:45.351 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 48/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 49/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 50/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 51/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 52/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 53/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 54/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 55/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : lua-5.4.4-4.el9.x86_64 56/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 57/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 58/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : openssl-devel-1:3.0.7-27.el9.x86_64 59/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : postgresql-13.11-1.el9.x86_64 60/202 2024-03-06T07:25:45.352 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : postgresql-contrib-13.11-1.el9.x86_64 61/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : postgresql-private-libs-13.11-1.el9.x86_64 62/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : postgresql-server-13.11-1.el9.x86_64 63/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 64/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 65/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 66/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 67/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 68/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 69/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 70/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 71/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 72/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 73/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 74/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 75/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 76/202 2024-03-06T07:25:45.353 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 77/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 78/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 79/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 80/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 81/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 82/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 83/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 84/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : qatlib-23.11.0-1.el9.x86_64 85/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : qatlib-service-23.11.0-1.el9.x86_64 86/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : qatzip-libs-1.1.2-2.el9.x86_64 87/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 88/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 89/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 90/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 91/202 2024-03-06T07:25:45.354 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : zlib-devel-1.2.11-40.el9.x86_64 92/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 93/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : jq-1.6-16.el9.x86_64 94/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 95/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 96/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 97/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 98/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 99/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-ply-3.11-14.el9.noarch 100/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 101/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 102/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 103/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 104/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 105/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 106/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 107/202 2024-03-06T07:25:45.355 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 108/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 109/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 110/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 111/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 112/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 113/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 114/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 115/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-asyncssh-2.13.2-4.el9.noarch 116/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 117/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 118/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 119/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 120/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 121/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 122/202 2024-03-06T07:25:45.356 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 123/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-google-auth-1:2.28.0-1.el9.noarch 124/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 125/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 126/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 127/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 128/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 129/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 130/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 131/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 132/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 133/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 134/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 135/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 136/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 137/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 138/202 2024-03-06T07:25:45.357 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 139/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 140/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 141/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 142/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 143/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 144/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 145/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 146/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 147/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 148/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : re2-1:20211101-3.el9.x86_64 149/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 150/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 151/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 152/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 154/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libacl-devel-2.3.1-4.el9.x86_64 155/202 2024-03-06T07:25:45.358 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libacl-devel-2.3.1-3.el9.x86_64 156/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-1:1.46.0-1.el9.x86_64 157/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-1:1.44.0-2.el9.x86_64 158/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 159/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 160/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-team-1:1.46.0-1.el9.x86_64 161/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-team-1:1.44.0-2.el9.x86_64 162/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-tui-1:1.46.0-1.el9.x86_64 163/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : NetworkManager-tui-1:1.44.0-2.el9.x86_64 164/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : acl-2.3.1-4.el9.x86_64 165/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : acl-2.3.1-3.el9.x86_64 166/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 167/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : elfutils-debuginfod-client-0.189-3.el9.x86_64 168/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 169/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : elfutils-libelf-0.189-3.el9.x86_64 170/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 171/202 2024-03-06T07:25:45.359 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : elfutils-libs-0.189-3.el9.x86_64 172/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : iproute-6.7.0-2.el9.x86_64 173/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : iproute-6.2.0-5.el9.x86_64 174/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : iproute-tc-6.7.0-2.el9.x86_64 175/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : iproute-tc-6.2.0-5.el9.x86_64 176/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libacl-2.3.1-4.el9.x86_64 177/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libacl-2.3.1-3.el9.x86_64 178/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libblkid-2.37.4-18.el9.x86_64 179/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libblkid-2.37.4-15.el9.x86_64 180/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libfdisk-2.37.4-18.el9.x86_64 181/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libfdisk-2.37.4-15.el9.x86_64 182/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libmount-2.37.4-18.el9.x86_64 183/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libmount-2.37.4-15.el9.x86_64 184/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libsmartcols-2.37.4-18.el9.x86_64 185/202 2024-03-06T07:25:45.360 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libsmartcols-2.37.4-15.el9.x86_64 186/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libuuid-2.37.4-18.el9.x86_64 187/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : libuuid-2.37.4-15.el9.x86_64 188/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 189/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : openssl-1:3.0.7-24.el9.x86_64 190/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 191/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : openssl-libs-1:3.0.7-24.el9.x86_64 192/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : util-linux-2.37.4-18.el9.x86_64 193/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : util-linux-2.37.4-15.el9.x86_64 194/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : util-linux-core-2.37.4-18.el9.x86_64 195/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : util-linux-core-2.37.4-15.el9.x86_64 196/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : util-linux-user-2.37.4-18.el9.x86_64 197/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : util-linux-user-2.37.4-15.el9.x86_64 198/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : xfsdump-3.1.12-4.el9.x86_64 199/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : xfsdump-3.1.12-3.el9.x86_64 200/202 2024-03-06T07:25:45.361 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 201/202 2024-03-06T07:25:47.886 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : xfsprogs-5.19.0-4.el9.x86_64 202/202 2024-03-06T07:25:47.886 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout:Upgraded: 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager-team-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: NetworkManager-tui-1:1.46.0-1.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: acl-2.3.1-4.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-debuginfod-client-0.190-2.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-libelf-0.190-2.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-libs-0.190-2.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: iproute-6.7.0-2.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: iproute-tc-6.7.0-2.el9.x86_64 2024-03-06T07:25:47.887 INFO:teuthology.orchestra.run.smithi086.stdout: libacl-2.3.1-4.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: libacl-devel-2.3.1-4.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: libblkid-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: libfdisk-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: libmount-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: libsmartcols-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: libuuid-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: openssl-1:3.0.7-27.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: openssl-libs-1:3.0.7-27.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: util-linux-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.888 INFO:teuthology.orchestra.run.smithi086.stdout: util-linux-core-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: util-linux-user-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: xfsdump-3.1.12-4.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: xfsprogs-6.3.0-1.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout:Installed: 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: automake-1.16.2-8.el9.noarch 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: bison-3.7.4-5.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.889 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-immutable-object-cache-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.890 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: flex-2.6.4-9.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-06T07:25:47.891 INFO:teuthology.orchestra.run.smithi086.stdout: fuse-2.9.9-15.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: grpc-data-1.46.7-10.el9.noarch 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: jq-1.6-16.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libarrow-9.0.0-10.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-06T07:25:47.892 INFO:teuthology.orchestra.run.smithi086.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: libtool-2.4.6-45.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: lua-5.4.4-4.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: luarocks-3.9.2-1.el9.noarch 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-06T07:25:47.893 INFO:teuthology.orchestra.run.smithi086.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: openssl-devel-1:3.0.7-27.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql-13.11-1.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql-contrib-13.11-1.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql-private-libs-13.11-1.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: postgresql-server-13.11-1.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: protobuf-3.14.0-13.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: python3-asyncssh-2.13.2-4.el9.noarch 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-06T07:25:47.894 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-google-auth-1:2.28.0-1.el9.noarch 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-03-06T07:25:47.895 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-06T07:25:47.896 INFO:teuthology.orchestra.run.smithi086.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-packaging-20.9-5.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-ply-3.11-14.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-06T07:25:47.897 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-03-06T07:25:47.898 INFO:teuthology.orchestra.run.smithi086.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: qatlib-23.11.0-1.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: qatlib-service-23.11.0-1.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: qatzip-libs-1.1.2-2.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: re2-1:20211101-3.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: uuid-1.6.2-55.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-03-06T07:25:47.899 INFO:teuthology.orchestra.run.smithi086.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-06T07:25:47.900 INFO:teuthology.orchestra.run.smithi086.stdout: zlib-devel-1.2.11-40.el9.x86_64 2024-03-06T07:25:47.900 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:47.900 INFO:teuthology.orchestra.run.smithi086.stdout:Complete! 2024-03-06T07:25:48.681 DEBUG:teuthology.parallel:result is None 2024-03-06T07:25:48.682 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-06T07:25:48.901 DEBUG:teuthology.orchestra.run.smithi086:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-06T07:25:48.962 INFO:teuthology.orchestra.run.smithi086.stdout:19.0.0-1739.g1bc250c7.el9 2024-03-06T07:25:48.963 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-1739.g1bc250c7.el9 2024-03-06T07:25:48.963 INFO:teuthology.task.install:The correct ceph version 19.0.0-1739.g1bc250c7 is installed. 2024-03-06T07:25:48.965 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-06T07:25:49.176 DEBUG:teuthology.orchestra.run.smithi194:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-06T07:25:49.245 INFO:teuthology.orchestra.run.smithi194.stdout:19.0.0-1739.g1bc250c7.el9 2024-03-06T07:25:49.245 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-1739.g1bc250c7.el9 2024-03-06T07:25:49.245 INFO:teuthology.task.install:The correct ceph version 19.0.0-1739.g1bc250c7 is installed. 2024-03-06T07:25:49.247 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-06T07:25:49.247 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:49.247 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-06T07:25:49.328 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:49.328 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-06T07:25:49.431 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-06T07:25:49.431 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:49.431 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/usr/bin/daemon-helper 2024-03-06T07:25:49.527 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-06T07:25:49.618 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:49.618 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/usr/bin/daemon-helper 2024-03-06T07:25:49.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-06T07:25:49.799 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-06T07:25:49.799 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:49.799 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-06T07:25:49.882 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-06T07:25:49.996 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:49.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-06T07:25:50.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-06T07:25:50.229 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-06T07:25:50.229 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:50.229 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/usr/bin/stdin-killer 2024-03-06T07:25:50.308 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-06T07:25:50.422 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:50.422 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/usr/bin/stdin-killer 2024-03-06T07:25:50.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-06T07:25:50.612 INFO:teuthology.run_tasks:Running task ceph... 2024-03-06T07:25:50.699 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-03-06T07:25:50.699 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 777 /var/log/ceph 2024-03-06T07:25:50.704 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 777 /var/log/ceph 2024-03-06T07:25:50.796 INFO:tasks.ceph:Disabling ceph logrotate... 2024-03-06T07:25:50.797 DEBUG:teuthology.orchestra.run.smithi086:> sudo rm -f -- /etc/logrotate.d/ceph 2024-03-06T07:25:50.804 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f -- /etc/logrotate.d/ceph 2024-03-06T07:25:50.902 INFO:tasks.ceph:Creating extra log directories... 2024-03-06T07:25:50.902 DEBUG:teuthology.orchestra.run.smithi086:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-03-06T07:25:50.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-03-06T07:25:51.005 INFO:tasks.ceph:Setting up log rotation with {'ceph-mds': '10G', 'ceph-osd': '10G'} 2024-03-06T07:25:51.005 INFO:tasks.ceph:writing logrotate stanza for ceph-mds 2024-03-06T07:25:51.006 INFO:tasks.ceph:writing logrotate stanza for ceph-osd 2024-03-06T07:25:51.006 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:51.006 DEBUG:teuthology.orchestra.run.smithi086:> dd of=/home/ubuntu/cephtest/logrotate.ceph-test.conf 2024-03-06T07:25:51.046 DEBUG:teuthology.orchestra.run.smithi086:> sudo mv /home/ubuntu/cephtest/logrotate.ceph-test.conf /etc/logrotate.d/ceph-test.conf && sudo chmod 0644 /etc/logrotate.d/ceph-test.conf && sudo chown root.root /etc/logrotate.d/ceph-test.conf 2024-03-06T07:25:51.256 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:etc_t:s0 /etc/logrotate.d/ceph-test.conf 2024-03-06T07:25:51.324 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:51.324 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/home/ubuntu/cephtest/logrotate.ceph-test.conf 2024-03-06T07:25:51.382 DEBUG:teuthology.orchestra.run.smithi194:> sudo mv /home/ubuntu/cephtest/logrotate.ceph-test.conf /etc/logrotate.d/ceph-test.conf && sudo chmod 0644 /etc/logrotate.d/ceph-test.conf && sudo chown root.root /etc/logrotate.d/ceph-test.conf 2024-03-06T07:25:51.645 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:etc_t:s0 /etc/logrotate.d/ceph-test.conf 2024-03-06T07:25:51.752 INFO:tasks.ceph:Creating ceph cluster ceph... 2024-03-06T07:25:51.752 INFO:tasks.ceph:config {'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-03-06T07:25:51.752 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi/7583072', 'branch': 'wip-lusov-qdb', 'description': 'fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/strays}', 'email': None, 'first_in_suite': False, 'job_id': '7583072', 'kernel': {'client': {'branch': 'testing'}, 'kdb': True, 'sha1': 'distro'}, 'ktype': 'upstream', 'last_in_suite': False, 'log-rotate': {'ceph-mds': '10G', 'ceph-osd': '10G'}, 'machine_type': 'smithi', 'name': 'leonidus-2024-03-06_07:04:29-fs-wip-lusov-qdb-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'openstack': [{'volumes': {'count': 4, 'size': 20}}, {'machine': {'disk': 200}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-lusov-qdb'}, 'ceph': {'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}}, 'kclient': {'syntax': 'v2'}, 'subvolume_version': 2, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-lusov-qdb', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}}, 'owner': 'scheduled_leonidus@teuthology', 'priority': 70, 'repo': 'https://git.ceph.com/ceph-ci.git', 'roles': [['mon.a', 'mgr.y', 'mds.a', 'mds.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'client.2'], ['mon.b', 'mon.c', 'mgr.x', 'mds.b', 'mds.d', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'client.3']], 'seed': 5072, 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'sleep_before_teardown': 0, 'subset': '246/10000', 'suite': 'fs', 'suite_branch': 'wip-lusov-qdb', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph-ci.git', 'suite_sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'syslog': {'ignorelist': ['WARNING*.*check_session_state', 'WARNING*.*__ceph_remove_cap']}, 'targets': {'smithi086.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPVCtziu+Te91xV2xttKKhwBOQSLhAoybWR/BQVXCRhKuBLqgWwARAjP8hbs3gmer5xVWdViejcSfyxOS0CxelA=', 'smithi194.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBELQGeOZN5IijIK+pDDgpWgRV+ELeGVFlFYEd3OMij6eM91I0YhemVkUUa2z/vNicbLP/3SXmZbxbCY88l761aQ='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'client': {'branch': 'testing'}, 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}}, {'ceph': {'create_rbd_pool': False, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'cluster': 'ceph'}}, {'kclient': None}, {'cephfs_test_runner': {'modules': ['tasks.cephfs.test_strays']}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': ["local function is_kupstream() return yaml.ktype == 'upstream' end", "local function is_kdistro() return yaml.ktype == 'distro' end", 'local function is_fuse() return false end', 'local function syntax_version() return yaml.overrides.kclient.syntax end']}, 'teuthology_branch': 'main', 'teuthology_sha1': 'e691533f9cbb33d85b2187bba20d7102f098636d', 'timestamp': '2024-03-06_07:04:29', 'tube': 'smithi', 'user': 'leonidus', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885'} 2024-03-06T07:25:51.753 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-03-06T07:25:51.757 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-03-06T07:25:51.814 DEBUG:teuthology.orchestra.run.smithi086:> sudo install -d -m0777 -- /var/run/ceph 2024-03-06T07:25:51.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo install -d -m0777 -- /var/run/ceph 2024-03-06T07:25:51.931 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:51.931 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/scratch_devs of=/dev/stdout 2024-03-06T07:25:51.968 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-06T07:25:51.968 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_1 2024-03-06T07:25:52.048 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-06T07:25:52.048 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:52.048 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 631 Links: 1 2024-03-06T07:25:52.049 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:52.049 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:52.049 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-03-06 07:25:41.251653279 +0000 2024-03-06T07:25:52.049 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-03-06 07:23:12.102455146 +0000 2024-03-06T07:25:52.049 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-03-06 07:23:12.102455146 +0000 2024-03-06T07:25:52.049 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-03-06 07:23:12.102455146 +0000 2024-03-06T07:25:52.050 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-06T07:25:52.152 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-03-06T07:25:52.152 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-03-06T07:25:52.152 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000342453 s, 1.5 MB/s 2024-03-06T07:25:52.154 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-06T07:25:52.199 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_2 2024-03-06T07:25:52.276 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 653 Links: 1 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-03-06 07:25:41.252653258 +0000 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-03-06 07:23:12.669443006 +0000 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-03-06 07:23:12.669443006 +0000 2024-03-06T07:25:52.277 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-03-06 07:23:12.669443006 +0000 2024-03-06T07:25:52.277 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-06T07:25:52.379 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-03-06T07:25:52.379 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-03-06T07:25:52.380 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000288691 s, 1.8 MB/s 2024-03-06T07:25:52.382 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-06T07:25:52.423 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_3 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 669 Links: 1 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-03-06 07:25:41.252653258 +0000 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-03-06 07:23:13.238430823 +0000 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-03-06 07:23:13.238430823 +0000 2024-03-06T07:25:52.503 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-03-06 07:23:13.238430823 +0000 2024-03-06T07:25:52.504 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-06T07:25:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-03-06T07:25:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-03-06T07:25:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000297926 s, 1.7 MB/s 2024-03-06T07:25:52.611 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-06T07:25:52.653 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_4 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 689 Links: 1 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-03-06 07:25:41.253653237 +0000 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-03-06 07:23:13.827418212 +0000 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-03-06 07:23:13.827418212 +0000 2024-03-06T07:25:52.738 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-03-06 07:23:13.827418212 +0000 2024-03-06T07:25:52.739 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-06T07:25:52.845 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-03-06T07:25:52.845 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-03-06T07:25:52.845 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000295655 s, 1.7 MB/s 2024-03-06T07:25:52.848 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-06T07:25:52.889 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-06T07:25:52.889 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:52.889 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/scratch_devs of=/dev/stdout 2024-03-06T07:25:52.945 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-06T07:25:52.945 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_1 2024-03-06T07:25:52.995 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 633 Links: 1 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-03-06 07:25:29.935418479 +0000 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-03-06 07:23:12.107485768 +0000 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-03-06 07:23:12.107485768 +0000 2024-03-06T07:25:52.996 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-03-06 07:23:12.107485768 +0000 2024-03-06T07:25:52.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-06T07:25:53.082 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-03-06T07:25:53.083 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-03-06T07:25:53.083 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000343702 s, 1.5 MB/s 2024-03-06T07:25:53.087 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-06T07:25:53.141 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_2 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 653 Links: 1 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-03-06 07:25:29.935418479 +0000 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-03-06 07:23:12.672474312 +0000 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-03-06 07:23:12.672474312 +0000 2024-03-06T07:25:53.187 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-03-06 07:23:12.672474312 +0000 2024-03-06T07:25:53.187 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-06T07:25:53.262 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-03-06T07:25:53.262 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-03-06T07:25:53.262 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000306449 s, 1.7 MB/s 2024-03-06T07:25:53.268 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-06T07:25:53.323 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_3 2024-03-06T07:25:53.372 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 669 Links: 1 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-03-06 07:25:29.936418458 +0000 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-03-06 07:23:13.241462775 +0000 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-03-06 07:23:13.241462775 +0000 2024-03-06T07:25:53.373 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-03-06 07:23:13.241462775 +0000 2024-03-06T07:25:53.373 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-06T07:25:53.453 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-03-06T07:25:53.453 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-03-06T07:25:53.453 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000304898 s, 1.7 MB/s 2024-03-06T07:25:53.456 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-06T07:25:53.508 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_4 2024-03-06T07:25:53.557 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-06T07:25:53.557 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-06T07:25:53.558 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 689 Links: 1 2024-03-06T07:25:53.558 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-06T07:25:53.558 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-03-06T07:25:53.558 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-03-06 07:25:29.937418438 +0000 2024-03-06T07:25:53.558 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-03-06 07:23:13.804451359 +0000 2024-03-06T07:25:53.558 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-03-06 07:23:13.804451359 +0000 2024-03-06T07:25:53.558 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-03-06 07:23:13.804451359 +0000 2024-03-06T07:25:53.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-06T07:25:53.641 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-03-06T07:25:53.641 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-03-06T07:25:53.641 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000309994 s, 1.7 MB/s 2024-03-06T07:25:53.644 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-06T07:25:53.698 INFO:tasks.ceph:osd dev map: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-06T07:25:53.698 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi086.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi194.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2024-03-06T07:25:53.698 INFO:tasks.ceph:Generating config... 2024-03-06T07:25:53.699 INFO:tasks.ceph:[client] client mount timeout = 600 2024-03-06T07:25:53.700 INFO:tasks.ceph:[client] debug client = 20 2024-03-06T07:25:53.700 INFO:tasks.ceph:[client] debug ms = 1 2024-03-06T07:25:53.700 INFO:tasks.ceph:[client] pid file = 2024-03-06T07:25:53.700 INFO:tasks.ceph:[client] rados mon op timeout = 900 2024-03-06T07:25:53.700 INFO:tasks.ceph:[client] rados osd op timeout = 900 2024-03-06T07:25:53.700 INFO:tasks.ceph:[global] ms die on skipped message = False 2024-03-06T07:25:53.700 INFO:tasks.ceph:[mds] debug mds = 20 2024-03-06T07:25:53.700 INFO:tasks.ceph:[mds] debug mds balancer = 20 2024-03-06T07:25:53.700 INFO:tasks.ceph:[mds] debug ms = 1 2024-03-06T07:25:53.700 INFO:tasks.ceph:[mds] mds debug frag = True 2024-03-06T07:25:53.700 INFO:tasks.ceph:[mds] mds debug scatterstat = True 2024-03-06T07:25:53.700 INFO:tasks.ceph:[mds] mds op complaint time = 180 2024-03-06T07:25:53.700 INFO:tasks.ceph:[mds] mds verify scatter = True 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mds] osd op complaint time = 180 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mds] rados mon op timeout = 900 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mds] rados osd op timeout = 900 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mgr] debug ms = 1 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mon] debug mon = 20 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mon] debug ms = 1 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mon] debug paxos = 20 2024-03-06T07:25:53.701 INFO:tasks.ceph:[mon] mon op complaint time = 120 2024-03-06T07:25:53.701 INFO:tasks.ceph:[osd] bdev async discard = True 2024-03-06T07:25:53.701 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-03-06T07:25:53.701 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2024-03-06T07:25:53.701 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-03-06T07:25:53.701 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-03-06T07:25:53.701 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-03-06T07:25:53.701 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] debug ms = 1 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] debug osd = 20 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-03-06T07:25:53.702 INFO:tasks.ceph:[osd] osd op complaint time = 180 2024-03-06T07:25:53.702 INFO:tasks.ceph:Setting up mon.a... 2024-03-06T07:25:53.702 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2024-03-06T07:25:53.850 INFO:teuthology.orchestra.run.smithi086.stdout:creating /etc/ceph/ceph.keyring 2024-03-06T07:25:53.858 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2024-03-06T07:25:53.985 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-03-06T07:25:54.074 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.86'), ('mon.b', '172.21.15.194'), ('mon.c', '[v2:172.21.15.194:3301,v1:172.21.15.194:6790]')] 2024-03-06T07:25:54.075 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]', 'ms die on skipped message': False}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false', 'mon op complaint time': 120}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon.a': {}, 'mon.b': {}, 'mon.c': {}, 'mds.a': {}, 'mds.c': {}, 'mds.b': {}, 'mds.d': {}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}} 2024-03-06T07:25:54.076 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:54.076 DEBUG:teuthology.orchestra.run.smithi086:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-03-06T07:25:54.127 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.86 --add b 172.21.15.194 --addv c '[v2:172.21.15.194:3301,v1:172.21.15.194:6790]' --print /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:25:54.275 INFO:teuthology.orchestra.run.smithi086.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:monmaptool: generated fsid 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:setting min_mon_release = quincy 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:epoch 0 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:fsid 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:last_changed 2024-03-06T07:25:54.275098+0000 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:created 2024-03-06T07:25:54.275098+0000 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:min_mon_release 17 (quincy) 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:election_strategy: 1 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:0: [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon.a 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.b 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:2: [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] mon.c 2024-03-06T07:25:54.276 INFO:teuthology.orchestra.run.smithi086.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2024-03-06T07:25:54.279 DEBUG:teuthology.orchestra.run.smithi086:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-03-06T07:25:54.326 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 013155b5-b812-4a77-8af3-f8c90b517e11... 2024-03-06T07:25:54.326 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:25:54.331 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:25:54.489 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-03-06T07:25:54.489 INFO:teuthology.orchestra.run.smithi086.stdout: chdir = "" 2024-03-06T07:25:54.489 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:25:54.489 INFO:teuthology.orchestra.run.smithi086.stdout: auth supported = cephx 2024-03-06T07:25:54.489 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.489 INFO:teuthology.orchestra.run.smithi086.stdout: filestore xattr use omap = true 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: mon clock drift allowed = 1.000 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: osd crush chooseleaf type = 0 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: auth debug = true 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on old message = true 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on bug = true 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: mon pg warn max object skew = 0 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:25:54.490 INFO:teuthology.orchestra.run.smithi086.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default size = 2 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow primary affinity = true 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow pg remap = true 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on no sortbitwise = false 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on too few osds = false 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: mon_allow_pool_size_one = true 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:25:54.491 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: osd default data pool replay window = 5 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: mon allow pool delete = true 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: mon cluster log file level = debug 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: debug asserts on shutdown = true 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: mon health detail to clog = false 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on skipped message = False 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout:[osd] 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: osd journal size = 100 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.492 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub load threshold = 5.0 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub max interval = 600 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd recover clone overlap = true 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd recovery max chunk = 1048576 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug shutdown = true 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug op order = true 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug verify stray on activate = true 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug trim objects = true 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd open classes on start = true 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug pg log writeout = true 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:25:54.493 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: osd map max advance = 10 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: journal zero on create = true 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: filestore apply finisher threads = 3 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: bdev debug aio = true 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug misdirected ops = true 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: bdev async discard = True 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: bdev enable discard = True 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore allocator = bitmap 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore block size = 96636764160 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore fsck on mount = True 2024-03-06T07:25:54.494 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluefs = 1/20 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluestore = 1/20 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: debug osd = 20 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: debug rocksdb = 4/10 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd full ratio = 0.9 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: osd objectstore = bluestore 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr] 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: debug mgr = 20 2024-03-06T07:25:54.495 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: mgr/telemetry/nag = false 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout:[mon] 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: debug paxos = 20 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: mon data avail warn = 5 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: mon mgr mkfs grace = 240 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:25:54.496 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd reporter subtree level = osd 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd prime pg temp = true 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: # 1m isn't quite enough 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:25:54.497 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: mon op complaint time = 120 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout:[client] 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: rgw cache enabled = true 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable ops log = true 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable usage log = true 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: client mount timeout = 600 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: debug client = 20 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = "" 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:25:54.498 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.a] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.b] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.c] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout:[mds] 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds = 20 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds balancer = 20 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug frag = True 2024-03-06T07:25:54.499 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug scatterstat = True 2024-03-06T07:25:54.500 INFO:teuthology.orchestra.run.smithi086.stdout: mds op complaint time = 180 2024-03-06T07:25:54.500 INFO:teuthology.orchestra.run.smithi086.stdout: mds verify scatter = True 2024-03-06T07:25:54.500 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:25:54.500 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:25:54.500 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:25:54.509 INFO:teuthology.orchestra.run.smithi194.stdout:[global] 2024-03-06T07:25:54.509 INFO:teuthology.orchestra.run.smithi194.stdout: chdir = "" 2024-03-06T07:25:54.509 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:25:54.509 INFO:teuthology.orchestra.run.smithi194.stdout: auth supported = cephx 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: filestore xattr use omap = true 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: mon clock drift allowed = 1.000 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: osd crush chooseleaf type = 0 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: auth debug = true 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on old message = true 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on bug = true 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:25:54.510 INFO:teuthology.orchestra.run.smithi194.stdout: mon pg warn max object skew = 0 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default size = 2 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow primary affinity = true 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow pg remap = true 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on no sortbitwise = false 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on too few osds = false 2024-03-06T07:25:54.511 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: mon_allow_pool_size_one = true 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: osd default data pool replay window = 5 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: mon allow pool delete = true 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: mon cluster log file level = debug 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: debug asserts on shutdown = true 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: mon health detail to clog = false 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:25:54.512 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on skipped message = False 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout:[osd] 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd journal size = 100 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub load threshold = 5.0 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub max interval = 600 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd recover clone overlap = true 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd recovery max chunk = 1048576 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug shutdown = true 2024-03-06T07:25:54.513 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug op order = true 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug verify stray on activate = true 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug trim objects = true 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: osd open classes on start = true 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug pg log writeout = true 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: osd map max advance = 10 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: journal zero on create = true 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: filestore apply finisher threads = 3 2024-03-06T07:25:54.514 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: bdev debug aio = true 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug misdirected ops = true 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: bdev async discard = True 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: bdev enable discard = True 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore allocator = bitmap 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore block size = 96636764160 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore fsck on mount = True 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluefs = 1/20 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluestore = 1/20 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: debug osd = 20 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: debug rocksdb = 4/10 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd full ratio = 0.9 2024-03-06T07:25:54.515 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: osd objectstore = bluestore 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr] 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: debug mgr = 20 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: mgr/telemetry/nag = false 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout:[mon] 2024-03-06T07:25:54.516 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: debug paxos = 20 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: mon data avail warn = 5 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: mon mgr mkfs grace = 240 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd reporter subtree level = osd 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd prime pg temp = true 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:25:54.517 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: # 1m isn't quite enough 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: mon op complaint time = 120 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout:[client] 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: rgw cache enabled = true 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable ops log = true 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable usage log = true 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: client mount timeout = 600 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: debug client = 20 2024-03-06T07:25:54.518 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = "" 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.a] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.b] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.c] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout:[mds] 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds = 20 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds balancer = 20 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug frag = True 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug scatterstat = True 2024-03-06T07:25:54.519 INFO:teuthology.orchestra.run.smithi194.stdout: mds op complaint time = 180 2024-03-06T07:25:54.520 INFO:teuthology.orchestra.run.smithi194.stdout: mds verify scatter = True 2024-03-06T07:25:54.520 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:25:54.520 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:25:54.520 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:25:54.554 INFO:tasks.ceph:Creating admin key on mon.a... 2024-03-06T07:25:54.554 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2024-03-06T07:25:54.696 INFO:tasks.ceph:Copying monmap to all nodes... 2024-03-06T07:25:54.696 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:54.696 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2024-03-06T07:25:54.742 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:54.742 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2024-03-06T07:25:54.836 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:25:54.836 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:54.836 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/ceph/ceph.keyring 2024-03-06T07:25:54.836 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-03-06T07:25:54.978 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:25:54.978 DEBUG:teuthology.orchestra.run.smithi086:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:25:55.077 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:25:55.077 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:55.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.keyring 2024-03-06T07:25:55.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-03-06T07:25:55.228 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:25:55.228 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:25:55.323 INFO:tasks.ceph:Setting up mon nodes... 2024-03-06T07:25:55.323 INFO:tasks.ceph:Setting up mgr nodes... 2024-03-06T07:25:55.323 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2024-03-06T07:25:55.509 INFO:teuthology.orchestra.run.smithi086.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2024-03-06T07:25:55.519 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2024-03-06T07:25:55.711 INFO:teuthology.orchestra.run.smithi194.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2024-03-06T07:25:55.720 INFO:tasks.ceph:Setting up mds nodes... 2024-03-06T07:25:55.721 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/mds/ceph-a && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.a /var/lib/ceph/mds/ceph-a/keyring 2024-03-06T07:25:55.902 INFO:teuthology.orchestra.run.smithi086.stdout:creating /var/lib/ceph/mds/ceph-a/keyring 2024-03-06T07:25:55.911 DEBUG:teuthology.orchestra.run.smithi086:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-a 2024-03-06T07:25:56.001 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/mds/ceph-c && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.c /var/lib/ceph/mds/ceph-c/keyring 2024-03-06T07:25:56.178 INFO:teuthology.orchestra.run.smithi086.stdout:creating /var/lib/ceph/mds/ceph-c/keyring 2024-03-06T07:25:56.187 DEBUG:teuthology.orchestra.run.smithi086:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-c 2024-03-06T07:25:56.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/mds/ceph-b && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.b /var/lib/ceph/mds/ceph-b/keyring 2024-03-06T07:25:56.457 INFO:teuthology.orchestra.run.smithi194.stdout:creating /var/lib/ceph/mds/ceph-b/keyring 2024-03-06T07:25:56.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-b 2024-03-06T07:25:56.566 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/mds/ceph-d && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.d /var/lib/ceph/mds/ceph-d/keyring 2024-03-06T07:25:56.735 INFO:teuthology.orchestra.run.smithi194.stdout:creating /var/lib/ceph/mds/ceph-d/keyring 2024-03-06T07:25:56.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-d 2024-03-06T07:25:56.839 INFO:tasks.ceph_client:Setting up client nodes... 2024-03-06T07:25:56.839 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/ceph.client.0.keyring && sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-06T07:25:56.961 INFO:teuthology.orchestra.run.smithi086.stdout:creating /etc/ceph/ceph.client.0.keyring 2024-03-06T07:25:57.017 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.2 /etc/ceph/ceph.client.2.keyring && sudo chmod 0644 /etc/ceph/ceph.client.2.keyring 2024-03-06T07:25:57.140 INFO:teuthology.orchestra.run.smithi086.stdout:creating /etc/ceph/ceph.client.2.keyring 2024-03-06T07:25:57.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/ceph.client.1.keyring && sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-03-06T07:25:57.321 INFO:teuthology.orchestra.run.smithi194.stdout:creating /etc/ceph/ceph.client.1.keyring 2024-03-06T07:25:57.368 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.3 /etc/ceph/ceph.client.3.keyring && sudo chmod 0644 /etc/ceph/ceph.client.3.keyring 2024-03-06T07:25:57.485 INFO:teuthology.orchestra.run.smithi194.stdout:creating /etc/ceph/ceph.client.3.keyring 2024-03-06T07:25:57.542 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-03-06T07:25:57.542 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi086.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi194.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2024-03-06T07:25:57.543 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2024-03-06T07:25:57.641 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-06T07:25:57.641 INFO:tasks.ceph:role: osd.0 2024-03-06T07:25:57.641 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:25:57.641 DEBUG:teuthology.orchestra.run.smithi086:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:25:57.739 INFO:teuthology.orchestra.run.smithi086.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:25:58.267 INFO:teuthology.orchestra.run.smithi086.stdout:Discarding blocks...Done. 2024-03-06T07:25:58.276 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi086.front.sepia.ceph.com -o noatime 2024-03-06T07:25:58.276 DEBUG:teuthology.orchestra.run.smithi086:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2024-03-06T07:25:58.444 DEBUG:teuthology.orchestra.run.smithi086:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2024-03-06T07:25:58.549 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2024-03-06T07:25:58.629 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-06T07:25:58.630 INFO:tasks.ceph:role: osd.1 2024-03-06T07:25:58.630 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:25:58.630 DEBUG:teuthology.orchestra.run.smithi086:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:25:58.725 INFO:teuthology.orchestra.run.smithi086.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:25:58.726 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:25:58.726 INFO:teuthology.orchestra.run.smithi086.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:25:59.219 INFO:teuthology.orchestra.run.smithi086.stdout:Discarding blocks...Done. 2024-03-06T07:25:59.227 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi086.front.sepia.ceph.com -o noatime 2024-03-06T07:25:59.227 DEBUG:teuthology.orchestra.run.smithi086:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2024-03-06T07:25:59.362 DEBUG:teuthology.orchestra.run.smithi086:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2024-03-06T07:25:59.453 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2024-03-06T07:25:59.543 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-06T07:25:59.543 INFO:tasks.ceph:role: osd.2 2024-03-06T07:25:59.543 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:25:59.544 DEBUG:teuthology.orchestra.run.smithi086:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-03-06T07:25:59.641 INFO:teuthology.orchestra.run.smithi086.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:25:59.641 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:25:59.642 INFO:teuthology.orchestra.run.smithi086.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:26:00.172 INFO:teuthology.orchestra.run.smithi086.stdout:Discarding blocks...Done. 2024-03-06T07:26:00.179 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi086.front.sepia.ceph.com -o noatime 2024-03-06T07:26:00.179 DEBUG:teuthology.orchestra.run.smithi086:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2024-03-06T07:26:00.365 DEBUG:teuthology.orchestra.run.smithi086:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2024-03-06T07:26:00.457 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2024-03-06T07:26:00.541 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-06T07:26:00.541 INFO:tasks.ceph:role: osd.3 2024-03-06T07:26:00.541 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:26:00.541 DEBUG:teuthology.orchestra.run.smithi086:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:26:00.637 INFO:teuthology.orchestra.run.smithi086.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:26:00.638 INFO:teuthology.orchestra.run.smithi086.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:26:00.638 INFO:teuthology.orchestra.run.smithi086.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:26:01.108 INFO:teuthology.orchestra.run.smithi086.stdout:Discarding blocks...Done. 2024-03-06T07:26:01.114 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi086.front.sepia.ceph.com -o noatime 2024-03-06T07:26:01.114 DEBUG:teuthology.orchestra.run.smithi086:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2024-03-06T07:26:01.265 DEBUG:teuthology.orchestra.run.smithi086:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2024-03-06T07:26:01.362 DEBUG:teuthology.orchestra.run.smithi086:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:01.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:01.509+0000 7f14c1411640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2024-03-06T07:26:01.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:01.509+0000 7f14c1411640 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2024-03-06T07:26:01.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:01.510+0000 7f14c1411640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-03-06T07:26:01.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:01.510+0000 7f14c1411640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-03-06T07:26:01.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:01.510+0000 7f14c1411640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-03-06T07:26:01.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:01.520+0000 7f14c1411640 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2024-03-06T07:26:04.692 DEBUG:teuthology.orchestra.run.smithi086:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-06T07:26:04.790 DEBUG:teuthology.orchestra.run.smithi086:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:04.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:04.921+0000 7f8ab1d73640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2024-03-06T07:26:04.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:04.922+0000 7f8ab1d73640 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2024-03-06T07:26:04.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:04.922+0000 7f8ab1d73640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-03-06T07:26:04.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:04.922+0000 7f8ab1d73640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-03-06T07:26:04.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:04.922+0000 7f8ab1d73640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-03-06T07:26:04.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:04.924+0000 7f8ab1d73640 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2024-03-06T07:26:08.098 DEBUG:teuthology.orchestra.run.smithi086:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-06T07:26:08.189 DEBUG:teuthology.orchestra.run.smithi086:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:08.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:08.321+0000 7faef1d93640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2024-03-06T07:26:08.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:08.322+0000 7faef1d93640 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2024-03-06T07:26:08.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:08.322+0000 7faef1d93640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-03-06T07:26:08.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:08.322+0000 7faef1d93640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-03-06T07:26:08.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:08.322+0000 7faef1d93640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-03-06T07:26:08.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:08.324+0000 7faef1d93640 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2024-03-06T07:26:11.253 DEBUG:teuthology.orchestra.run.smithi086:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-06T07:26:11.352 DEBUG:teuthology.orchestra.run.smithi086:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:11.488+0000 7f21aa214640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2024-03-06T07:26:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:11.489+0000 7f21aa214640 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2024-03-06T07:26:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:11.489+0000 7f21aa214640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-03-06T07:26:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:11.489+0000 7f21aa214640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-03-06T07:26:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:11.489+0000 7f21aa214640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-03-06T07:26:11.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:11.491+0000 7f21aa214640 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2024-03-06T07:26:14.675 DEBUG:teuthology.orchestra.run.smithi086:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-06T07:26:14.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/osd/ceph-4 2024-03-06T07:26:14.865 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-06T07:26:14.865 INFO:tasks.ceph:role: osd.4 2024-03-06T07:26:14.865 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:26:14.865 DEBUG:teuthology.orchestra.run.smithi194:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-03-06T07:26:14.962 INFO:teuthology.orchestra.run.smithi194.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:26:14.963 INFO:teuthology.orchestra.run.smithi194.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:26:15.475 INFO:teuthology.orchestra.run.smithi194.stdout:Discarding blocks...Done. 2024-03-06T07:26:15.481 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi194.front.sepia.ceph.com -o noatime 2024-03-06T07:26:15.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-4 2024-03-06T07:26:15.646 DEBUG:teuthology.orchestra.run.smithi194:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-4 2024-03-06T07:26:15.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/osd/ceph-5 2024-03-06T07:26:15.844 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-06T07:26:15.844 INFO:tasks.ceph:role: osd.5 2024-03-06T07:26:15.844 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:26:15.844 DEBUG:teuthology.orchestra.run.smithi194:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-03-06T07:26:15.939 INFO:teuthology.orchestra.run.smithi194.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:26:15.939 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:26:15.940 INFO:teuthology.orchestra.run.smithi194.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:26:16.411 INFO:teuthology.orchestra.run.smithi194.stdout:Discarding blocks...Done. 2024-03-06T07:26:16.417 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi194.front.sepia.ceph.com -o noatime 2024-03-06T07:26:16.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-5 2024-03-06T07:26:16.565 DEBUG:teuthology.orchestra.run.smithi194:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-5 2024-03-06T07:26:16.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/osd/ceph-6 2024-03-06T07:26:16.751 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-06T07:26:16.751 INFO:tasks.ceph:role: osd.6 2024-03-06T07:26:16.751 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:26:16.751 DEBUG:teuthology.orchestra.run.smithi194:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-03-06T07:26:16.848 INFO:teuthology.orchestra.run.smithi194.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:26:16.849 INFO:teuthology.orchestra.run.smithi194.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:26:17.363 INFO:teuthology.orchestra.run.smithi194.stdout:Discarding blocks...Done. 2024-03-06T07:26:17.371 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi194.front.sepia.ceph.com -o noatime 2024-03-06T07:26:17.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-6 2024-03-06T07:26:17.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-6 2024-03-06T07:26:17.662 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/osd/ceph-7 2024-03-06T07:26:17.754 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-06T07:26:17.754 INFO:tasks.ceph:role: osd.7 2024-03-06T07:26:17.754 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:26:17.754 DEBUG:teuthology.orchestra.run.smithi194:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout: = sunit=0 swidth=0 blks 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-06T07:26:17.853 INFO:teuthology.orchestra.run.smithi194.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-06T07:26:18.304 INFO:teuthology.orchestra.run.smithi194.stdout:Discarding blocks...Done. 2024-03-06T07:26:18.310 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi194.front.sepia.ceph.com -o noatime 2024-03-06T07:26:18.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-7 2024-03-06T07:26:18.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-7 2024-03-06T07:26:18.562 DEBUG:teuthology.orchestra.run.smithi194:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:18.724 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:18.722+0000 7f82e180f640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-4/keyring: can't open /var/lib/ceph/osd/ceph-4/keyring: (2) No such file or directory 2024-03-06T07:26:18.727 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:18.725+0000 7f82e180f640 -1 created new key in keyring /var/lib/ceph/osd/ceph-4/keyring 2024-03-06T07:26:18.727 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:18.725+0000 7f82e180f640 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2024-03-06T07:26:18.727 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:18.726+0000 7f82e180f640 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2024-03-06T07:26:18.727 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:18.726+0000 7f82e180f640 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2024-03-06T07:26:18.732 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:18.730+0000 7f82e180f640 -1 bluestore(/var/lib/ceph/osd/ceph-4) _read_fsid unparsable uuid 2024-03-06T07:26:21.754 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:26:21.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:26:21.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-06T07:26:22.034 DEBUG:teuthology.orchestra.run.smithi194:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:22.167 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:22.166+0000 7fd3e0019640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-5/keyring: can't open /var/lib/ceph/osd/ceph-5/keyring: (2) No such file or directory 2024-03-06T07:26:22.168 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:22.166+0000 7fd3e0019640 -1 created new key in keyring /var/lib/ceph/osd/ceph-5/keyring 2024-03-06T07:26:22.168 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:22.166+0000 7fd3e0019640 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2024-03-06T07:26:22.168 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:22.166+0000 7fd3e0019640 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2024-03-06T07:26:22.168 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:22.166+0000 7fd3e0019640 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2024-03-06T07:26:22.170 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:22.169+0000 7fd3e0019640 -1 bluestore(/var/lib/ceph/osd/ceph-5) _read_fsid unparsable uuid 2024-03-06T07:26:25.334 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-06T07:26:25.432 DEBUG:teuthology.orchestra.run.smithi194:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 6 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:25.568 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:25.566+0000 7f69874a8640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-6/keyring: can't open /var/lib/ceph/osd/ceph-6/keyring: (2) No such file or directory 2024-03-06T07:26:25.569 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:25.567+0000 7f69874a8640 -1 created new key in keyring /var/lib/ceph/osd/ceph-6/keyring 2024-03-06T07:26:25.569 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:25.567+0000 7f69874a8640 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2024-03-06T07:26:25.569 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:25.567+0000 7f69874a8640 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2024-03-06T07:26:25.569 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:25.567+0000 7f69874a8640 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2024-03-06T07:26:25.571 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:25.570+0000 7f69874a8640 -1 bluestore(/var/lib/ceph/osd/ceph-6) _read_fsid unparsable uuid 2024-03-06T07:26:28.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-06T07:26:28.853 DEBUG:teuthology.orchestra.run.smithi194:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 7 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:28.989 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:28.988+0000 7fad4c815640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-7/keyring: can't open /var/lib/ceph/osd/ceph-7/keyring: (2) No such file or directory 2024-03-06T07:26:28.990 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:28.989+0000 7fad4c815640 -1 created new key in keyring /var/lib/ceph/osd/ceph-7/keyring 2024-03-06T07:26:28.990 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:28.989+0000 7fad4c815640 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2024-03-06T07:26:28.990 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:28.989+0000 7fad4c815640 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2024-03-06T07:26:28.990 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:28.989+0000 7fad4c815640 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2024-03-06T07:26:28.992 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:28.991+0000 7fad4c815640 -1 bluestore(/var/lib/ceph/osd/ceph-7) _read_fsid unparsable uuid 2024-03-06T07:26:32.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-06T07:26:32.251 INFO:tasks.ceph:Reading keys from all nodes... 2024-03-06T07:26:32.252 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:32.252 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2024-03-06T07:26:32.348 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:32.348 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/mds/ceph-a/keyring of=/dev/stdout 2024-03-06T07:26:32.443 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:32.443 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/mds/ceph-c/keyring of=/dev/stdout 2024-03-06T07:26:32.537 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:32.538 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2024-03-06T07:26:32.631 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:32.631 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2024-03-06T07:26:32.721 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:32.721 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2024-03-06T07:26:32.813 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:32.813 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2024-03-06T07:26:32.907 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:32.907 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2024-03-06T07:26:33.005 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/mds/ceph-b/keyring of=/dev/stdout 2024-03-06T07:26:33.098 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.099 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/mds/ceph-d/keyring of=/dev/stdout 2024-03-06T07:26:33.189 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.189 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-4/keyring of=/dev/stdout 2024-03-06T07:26:33.283 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-5/keyring of=/dev/stdout 2024-03-06T07:26:33.377 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.377 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-6/keyring of=/dev/stdout 2024-03-06T07:26:33.467 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-7/keyring of=/dev/stdout 2024-03-06T07:26:33.560 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:33.560 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/etc/ceph/ceph.client.0.keyring of=/dev/stdout 2024-03-06T07:26:33.619 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:33.619 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/etc/ceph/ceph.client.2.keyring of=/dev/stdout 2024-03-06T07:26:33.670 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.671 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/etc/ceph/ceph.client.1.keyring of=/dev/stdout 2024-03-06T07:26:33.728 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:33.728 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/etc/ceph/ceph.client.3.keyring of=/dev/stdout 2024-03-06T07:26:33.782 INFO:tasks.ceph:Adding keys to all mons... 2024-03-06T07:26:33.782 DEBUG:teuthology.orchestra.run.smithi086:> sudo tee -a /etc/ceph/ceph.keyring 2024-03-06T07:26:33.789 DEBUG:teuthology.orchestra.run.smithi194:> sudo tee -a /etc/ceph/ceph.keyring 2024-03-06T07:26:33.874 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr.y] 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ== 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw== 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA== 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.0] 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ== 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.1] 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w== 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.2] 2024-03-06T07:26:33.875 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA== 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.3] 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g== 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr.x] 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A== 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug== 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q== 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.4] 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA== 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.5] 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ== 2024-03-06T07:26:33.876 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.6] 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg== 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout:[osd.7] 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g== 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout:[client.2] 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w== 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout:[client.1] 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg== 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout:[client.3] 2024-03-06T07:26:33.877 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ== 2024-03-06T07:26:33.880 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr.y] 2024-03-06T07:26:33.880 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ== 2024-03-06T07:26:33.880 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:26:33.880 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw== 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA== 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.0] 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ== 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.1] 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w== 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.2] 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA== 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.3] 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g== 2024-03-06T07:26:33.881 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr.x] 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A== 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug== 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q== 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.4] 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA== 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.5] 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ== 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.6] 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg== 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout:[osd.7] 2024-03-06T07:26:33.882 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g== 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout:[client.0] 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout:[client.2] 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w== 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout:[client.1] 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg== 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout:[client.3] 2024-03-06T07:26:33.883 INFO:teuthology.orchestra.run.smithi194.stdout: key = AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ== 2024-03-06T07:26:33.885 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-06T07:26:33.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-06T07:26:34.069 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:34.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:34.218 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.c --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:34.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.c --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:34.360 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.505 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.510 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.646 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.651 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.794 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:34.950 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-06T07:26:34.957 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-06T07:26:35.096 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.b --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:35.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.b --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:35.247 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.d --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:35.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.d --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-06T07:26:35.399 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.549 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.696 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.837 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.845 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-06T07:26:35.983 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:35.990 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:36.136 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:36.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:36.281 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:36.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:36.428 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:36.433 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-06T07:26:36.573 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-03-06T07:26:36.573 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2024-03-06T07:26:36.665 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-03-06T07:26:36.821 DEBUG:teuthology.orchestra.run.smithi086:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2024-03-06T07:26:36.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2024-03-06T07:26:36.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-03-06T07:26:37.137 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2024-03-06T07:26:37.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2024-03-06T07:26:37.315 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-03-06T07:26:37.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2024-03-06T07:26:37.542 DEBUG:teuthology.orchestra.run.smithi086:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:37.550 DEBUG:teuthology.orchestra.run.smithi194:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-03-06T07:26:37.601 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2024-03-06T07:26:37.601 INFO:tasks.ceph.mon.a:Restarting daemon 2024-03-06T07:26:37.601 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-03-06T07:26:37.606 INFO:tasks.ceph.mon.a:Started 2024-03-06T07:26:37.606 INFO:tasks.ceph.mon.b:Restarting daemon 2024-03-06T07:26:37.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-03-06T07:26:37.611 INFO:tasks.ceph.mon.b:Started 2024-03-06T07:26:37.611 INFO:tasks.ceph.mon.c:Restarting daemon 2024-03-06T07:26:37.612 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-03-06T07:26:37.617 INFO:tasks.ceph.mon.c:Started 2024-03-06T07:26:37.617 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2024-03-06T07:26:37.617 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-03-06T07:26:37.617 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-03-06T07:26:37.622 INFO:tasks.ceph.mgr.y:Started 2024-03-06T07:26:37.622 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-03-06T07:26:37.622 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-03-06T07:26:37.628 INFO:tasks.ceph.mgr.x:Started 2024-03-06T07:26:37.628 INFO:tasks.ceph:Setting crush tunables to default 2024-03-06T07:26:37.628 DEBUG:teuthology.orchestra.run.smithi086:> sudo ceph --cluster ceph osd crush tunables default 2024-03-06T07:26:37.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.804+0000 7ffb27200640 1 Processor -- start 2024-03-06T07:26:37.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.804+0000 7ffb27200640 1 -- start start 2024-03-06T07:26:37.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb27200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:37.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb27200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffb200b46c0 con 0x7ffb200b3f70 2024-03-06T07:26:37.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb27200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffb200b49a0 con 0x7ffb2015fc10 2024-03-06T07:26:37.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb27200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb20072c90 con 0x7ffb20160700 2024-03-06T07:26:37.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb17400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ffb200b3f70 0x7ffb200ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57980/0 (socket says 172.21.15.86:57980) 2024-03-06T07:26:37.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 learned_addr learned my addr 172.21.15.86:0/696098468 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:26:37.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:37.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 15 2024-03-06T07:26:37.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:37.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.806+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:37.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:37.807+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-06T07:26:38.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.008+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:38.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.009+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 15 2024-03-06T07:26:38.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.009+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:38.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.009+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:38.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.009+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-06T07:26:38.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.410+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:38.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.411+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 15 2024-03-06T07:26:38.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.411+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:38.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.411+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:38.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:38.411+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-06T07:26:39.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:39.213+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:39.213+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 15 2024-03-06T07:26:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:39.213+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:39.213+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:39.213+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 msgr2=0x7ffb20070ff0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb20160700 0x7ffb20070ff0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> v1:172.21.15.194:6789/0 conn(0x7ffb2015fc10 legacy=0x7ffb2015fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> v1:172.21.15.86:6789/0 conn(0x7ffb200b3f70 legacy=0x7ffb200ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffb20072c90 con 0x7ffafc003710 2024-03-06T07:26:40.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.806+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffafc008d10 con 0x7ffafc0061c0 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:40.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:40.807+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.008+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.008+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:41.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-06T07:26:41.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.009+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.410+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.410+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-06T07:26:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:41.411+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.212+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-06T07:26:42.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:42.213+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-06T07:26:43.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.806+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 msgr2=0x7ffafc008660 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.806+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc0061c0 0x7ffafc008660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.806+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 msgr2=0x7ffafc005b70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.806+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003710 0x7ffafc005b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.806+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.806+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 reap_dead start 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.807+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffb2009dd80 con 0x7ffafc006b10 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffb20072c90 con 0x7ffafc003b20 2024-03-06T07:26:43.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffafc008d10 con 0x7ffafc009100 2024-03-06T07:26:43.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.807+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:43.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.807+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:43.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:43.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:43.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:43.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:43.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:43.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:43.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-06T07:26:43.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:43.808+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-06T07:26:44.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.009+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.009+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-06T07:26:44.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.010+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-06T07:26:44.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.411+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:44.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.411+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-06T07:26:44.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:44.412+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb1fe00640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:45.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-06T07:26:45.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:45.214+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-06T07:26:46.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 msgr2=0x7ffafc00b550 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-06T07:26:46.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.194:3300/0 conn(0x7ffafc009100 0x7ffafc00b550 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:46.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 msgr2=0x7ffafc0065d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-06T07:26:46.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> v2:172.21.15.86:3300/0 conn(0x7ffafc003b20 0x7ffafc0065d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:46.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 >> v1:172.21.15.86:6789/0 conn(0x7ffafc006b10 legacy=0x7ffafc006fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:46.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:46.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffafc008950 con 0x7ffafc00dcc0 2024-03-06T07:26:46.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.807+0000 7ffb1e000640 1 -- 172.21.15.86:0/696098468 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb20072c90 con 0x7ffafc011bb0 2024-03-06T07:26:46.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.808+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:46.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.808+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 msgr2=0x7ffafc013fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:46.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.809+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 msgr2=0x7ffafc013fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:46.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.809+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:46.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:46.809+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-06T07:26:47.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.010+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.011+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 msgr2=0x7ffafc013fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:47.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.011+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 msgr2=0x7ffafc013fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:47.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.011+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:47.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.011+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-06T07:26:47.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.412+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.412+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 msgr2=0x7ffafc013fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-06T07:26:47.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.412+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 msgr2=0x7ffafc013fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-06T07:26:47.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.412+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-06T07:26:47.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.413+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-06T07:26:47.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.871+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 700811517 0 0) 0x7ffafc008950 con 0x7ffafc00dcc0 2024-03-06T07:26:47.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.872+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ffb04003520 con 0x7ffafc00dcc0 2024-03-06T07:26:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.876+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 <== mon.1 v1:172.21.15.194:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 319525465 0 0) 0x7ffb04003520 con 0x7ffafc00dcc0 2024-03-06T07:26:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.876+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 msgr2=0x7ffafc013fa0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-06T07:26:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.876+0000 7ffb1f400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffafc011bb0 0x7ffafc013fa0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.876+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 --> v1:172.21.15.194:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffafc015540 con 0x7ffafc00dcc0 2024-03-06T07:26:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.876+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 <== mon.1 v1:172.21.15.194:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ffb10007da0 con 0x7ffafc00dcc0 2024-03-06T07:26:47.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.876+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 >> v1:172.21.15.194:6789/0 conn(0x7ffafc00dcc0 legacy=0x7ffafc00e0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:47.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.877+0000 7ffb25200640 1 -- 172.21.15.86:0/696098468 reap_dead start 2024-03-06T07:26:47.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.877+0000 7ffb1f400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb040039b0 0x7ffb04003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:47.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.877+0000 7ffb1f400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb04004800 0x7ffb04004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:47.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.877+0000 7ffb1f400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04005210 0x7ffb0400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:47.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.877+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb20072c90 con 0x7ffb040039b0 2024-03-06T07:26:47.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.877+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb0400c140 con 0x7ffb04005210 2024-03-06T07:26:47.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.877+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb0400c340 con 0x7ffb04004800 2024-03-06T07:26:47.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.878+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb04004800 0x7ffb04004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.878+0000 7ffb25200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04005210 0x7ffb0400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.878+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb040039b0 0x7ffb04003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.878+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04005210 msgr2=0x7ffb0400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.878+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04005210 0x7ffb0400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.879+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb040039b0 msgr2=0x7ffb04003db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.879+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb040039b0 0x7ffb04003db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.879+0000 7ffb17400640 1 -- 172.21.15.86:0/696098468 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb04003520 con 0x7ffb04004800 2024-03-06T07:26:47.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.883+0000 7ffb17400640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb04004800 0x7ffb04004c80 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7ffb08007c70 tx=0x7ffb04005830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=27d9de3e01b4a419 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:47.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.884+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb08036070 con 0x7ffb04004800 2024-03-06T07:26:47.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.884+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffb08032470 con 0x7ffb04004800 2024-03-06T07:26:47.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.884+0000 7ffb1f400640 1 -- 172.21.15.86:0/696098468 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb08005cc0 con 0x7ffb04004800 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 -- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb04004800 msgr2=0x7ffb04004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb04004800 0x7ffb04004c80 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7ffb08007c70 tx=0x7ffb04005830 comp rx=0 tx=0).stop 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 -- 172.21.15.86:0/696098468 shutdown_connections 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04005210 0x7ffb0400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb04004800 0x7ffb04004c80 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 --2- 172.21.15.86:0/696098468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb040039b0 0x7ffb04003db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 -- 172.21.15.86:0/696098468 >> 172.21.15.86:0/696098468 conn(0x7ffb200b1940 msgr2=0x7ffb200b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 -- 172.21.15.86:0/696098468 shutdown_connections 2024-03-06T07:26:47.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.885+0000 7ffb27200640 1 -- 172.21.15.86:0/696098468 wait complete. 2024-03-06T07:26:47.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.888+0000 7ffb27200640 1 Processor -- start 2024-03-06T07:26:47.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.889+0000 7ffb27200640 1 -- start start 2024-03-06T07:26:47.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb27200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb040039b0 0x7ffb2015f3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb27200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04004800 0x7ffb201519a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb27200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb04005210 0x7ffb200c5430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb27200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb2015c460 con 0x7ffb04005210 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb27200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb2015c2e0 con 0x7ffb04004800 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb27200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb20070ff0 con 0x7ffb040039b0 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb25200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb04005210 0x7ffb200c5430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb25200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb04005210 0x7ffb200c5430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36390/0 (socket says 172.21.15.86:36390) 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb25200640 1 -- 172.21.15.86:0/842159609 learned_addr learned my addr 172.21.15.86:0/842159609 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb17400640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04004800 0x7ffb201519a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.890+0000 7ffb1fe00640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb040039b0 0x7ffb2015f3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.891+0000 7ffb25200640 1 -- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb040039b0 msgr2=0x7ffb2015f3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.891+0000 7ffb25200640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb040039b0 0x7ffb2015f3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.891+0000 7ffb25200640 1 -- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04004800 msgr2=0x7ffb201519a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.891+0000 7ffb25200640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04004800 0x7ffb201519a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.891+0000 7ffb25200640 1 -- 172.21.15.86:0/842159609 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb2015e320 con 0x7ffb04005210 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.891+0000 7ffb25200640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb04005210 0x7ffb200c5430 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7ffb20070bc0 tx=0x7ffb18009f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.892+0000 7ffb1cc00640 1 -- 172.21.15.86:0/842159609 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb1801b030 con 0x7ffb04005210 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.892+0000 7ffb1cc00640 1 -- 172.21.15.86:0/842159609 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffb18014a20 con 0x7ffb04005210 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.892+0000 7ffb1cc00640 1 -- 172.21.15.86:0/842159609 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb18014e30 con 0x7ffb04005210 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.892+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffb2015e600 con 0x7ffb04005210 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.892+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb2015ebc0 con 0x7ffb04005210 2024-03-06T07:26:47.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.892+0000 7ffb1cc00640 1 -- 172.21.15.86:0/842159609 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7ffb18005d10 con 0x7ffb04005210 2024-03-06T07:26:47.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.892+0000 7ffb1cc00640 1 -- 172.21.15.86:0/842159609 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7ffb18020d10 con 0x7ffb04005210 2024-03-06T07:26:47.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.894+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffb2004f330 con 0x7ffb04005210 2024-03-06T07:26:47.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.897+0000 7ffb1cc00640 1 -- 172.21.15.86:0/842159609 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7ffb18010070 con 0x7ffb04005210 2024-03-06T07:26:47.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.971+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7ffb2006ef30 con 0x7ffb04005210 2024-03-06T07:26:47.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.974+0000 7ffb1cc00640 1 -- 172.21.15.86:0/842159609 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v2) v1 ==== 124+0+0 (secure 0 0 0) 0x7ffb18020410 con 0x7ffb04005210 2024-03-06T07:26:47.976 INFO:teuthology.orchestra.run.smithi086.stderr:adjusted tunables profile to default 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb04005210 msgr2=0x7ffb200c5430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb04005210 0x7ffb200c5430 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7ffb20070bc0 tx=0x7ffb18009f10 comp rx=0 tx=0).stop 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 shutdown_connections 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb04005210 0x7ffb200c5430 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb04004800 0x7ffb201519a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 --2- 172.21.15.86:0/842159609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb040039b0 0x7ffb2015f3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 >> 172.21.15.86:0/842159609 conn(0x7ffb200b1940 msgr2=0x7ffb200cbe30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 shutdown_connections 2024-03-06T07:26:47.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:47.976+0000 7ffb27200640 1 -- 172.21.15.86:0/842159609 wait complete. 2024-03-06T07:26:47.996 INFO:tasks.ceph:check_enable_crimson: False 2024-03-06T07:26:47.996 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2024-03-06T07:26:47.996 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:47.996 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2024-03-06T07:26:48.078 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:48.078 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2024-03-06T07:26:48.205 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:48.205 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2024-03-06T07:26:48.334 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:26:48.334 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2024-03-06T07:26:48.459 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:48.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-4/fsid of=/dev/stdout 2024-03-06T07:26:48.485 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:48.484+0000 7fc784c2f040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-06T07:26:48.488 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:48.486+0000 7fce54073040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-06T07:26:48.546 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:48.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-5/fsid of=/dev/stdout 2024-03-06T07:26:48.547 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:48.546+0000 7fc784c2f040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-06T07:26:48.557 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:48.555+0000 7fce54073040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-06T07:26:48.640 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:48.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-6/fsid of=/dev/stdout 2024-03-06T07:26:48.759 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:26:48.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/var/lib/ceph/osd/ceph-7/fsid of=/dev/stdout 2024-03-06T07:26:48.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new ca3d759d-401c-42df-893e-5c63b2bbab14 0 2024-03-06T07:26:49.053 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.050+0000 7f6d7ce00640 1 Processor -- start 2024-03-06T07:26:49.053 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.051+0000 7f6d7ce00640 1 -- start start 2024-03-06T07:26:49.055 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d7ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6d7815f490 0x7f6d780d08a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.055 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d7ce00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6d780ced50 0x7f6d780cf180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.055 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d7ce00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6d780d4c40 con 0x7f6d780b4de0 2024-03-06T07:26:49.055 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d7ce00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6d780d49f0 con 0x7f6d7815f490 2024-03-06T07:26:49.055 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d7ce00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6d780d24b0 con 0x7f6d780ced50 2024-03-06T07:26:49.055 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d76800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6d780ced50 0x7f6d780cf180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d76800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6d780ced50 0x7f6d780cf180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.194:43046/0 (socket says 172.21.15.194:43046) 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d76800640 1 -- 172.21.15.194:0/2698760306 learned_addr learned my addr 172.21.15.194:0/2698760306 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.052+0000 7f6d75e00640 1 --2- 172.21.15.194:0/2698760306 >> v2:172.21.15.86:3300/0 conn(0x7f6d7815f490 0x7f6d780d08a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.053+0000 7f6d76800640 1 -- 172.21.15.194:0/2698760306 >> v2:172.21.15.86:3300/0 conn(0x7f6d7815f490 msgr2=0x7f6d780d08a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.053+0000 7f6d76800640 1 --2- 172.21.15.194:0/2698760306 >> v2:172.21.15.86:3300/0 conn(0x7f6d7815f490 0x7f6d780d08a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.053+0000 7f6d76800640 1 -- 172.21.15.194:0/2698760306 >> v1:172.21.15.86:6789/0 conn(0x7f6d780b4de0 legacy=0x7f6d780d0de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.053+0000 7f6d76800640 1 -- 172.21.15.194:0/2698760306 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d780d2700 con 0x7f6d780ced50 2024-03-06T07:26:49.056 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.054+0000 7f6d76800640 1 --2- 172.21.15.194:0/2698760306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d780ced50 0x7f6d780cf180 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f6d780d4b70 tx=0x7f6d7002b880 comp rx=0 tx=0).ready entity=mon.1 client_cookie=44938cf8d3cb3645 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.057 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d75400640 1 -- 172.21.15.194:0/2698760306 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d7002e070 con 0x7f6d780ced50 2024-03-06T07:26:49.057 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d75400640 1 -- 172.21.15.194:0/2698760306 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d70037070 con 0x7f6d780ced50 2024-03-06T07:26:49.057 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d75400640 1 -- 172.21.15.194:0/2698760306 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d70031a00 con 0x7f6d780ced50 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d7ce00640 1 -- 172.21.15.194:0/2698760306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d780ced50 msgr2=0x7f6d780cf180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d7ce00640 1 --2- 172.21.15.194:0/2698760306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d780ced50 0x7f6d780cf180 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f6d780d4b70 tx=0x7f6d7002b880 comp rx=0 tx=0).stop 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d7ce00640 1 -- 172.21.15.194:0/2698760306 shutdown_connections 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d7ce00640 1 --2- 172.21.15.194:0/2698760306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d780ced50 0x7f6d780cf180 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d7ce00640 1 --2- 172.21.15.194:0/2698760306 >> v2:172.21.15.86:3300/0 conn(0x7f6d7815f490 0x7f6d780d08a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.055+0000 7f6d7ce00640 1 -- 172.21.15.194:0/2698760306 >> 172.21.15.194:0/2698760306 conn(0x7f6d780d3ed0 msgr2=0x7f6d780d42e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.056+0000 7f6d7ce00640 1 -- 172.21.15.194:0/2698760306 shutdown_connections 2024-03-06T07:26:49.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.056+0000 7f6d7ce00640 1 -- 172.21.15.194:0/2698760306 wait complete. 2024-03-06T07:26:49.059 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.057+0000 7f6d7ce00640 1 Processor -- start 2024-03-06T07:26:49.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.057+0000 7f6d7ce00640 1 -- start start 2024-03-06T07:26:49.061 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d7ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d780b4de0 0x7f6d7814e410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d7ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d780ced50 0x7f6d780cc5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d7ce00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d7815f490 0x7f6d780b8720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d7ce00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d7815fd50 con 0x7f6d780ced50 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d7ce00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d780d2cc0 con 0x7f6d7815f490 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d7ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6d7815f870 con 0x7f6d780b4de0 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d75e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d780b4de0 0x7f6d7814e410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d75e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d780b4de0 0x7f6d7814e410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41416/0 (socket says 172.21.15.194:41416) 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.058+0000 7f6d75e00640 1 -- 172.21.15.194:0/591589876 learned_addr learned my addr 172.21.15.194:0/591589876 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d76800640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d7815f490 0x7f6d780b8720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d6d400640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d780ced50 0x7f6d780cc5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d75e00640 1 -- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d7815f490 msgr2=0x7f6d780b8720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.062 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d75e00640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d7815f490 0x7f6d780b8720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.063 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d75e00640 1 -- 172.21.15.194:0/591589876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d780ced50 msgr2=0x7f6d780cc5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.063 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d75e00640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d780ced50 0x7f6d780cc5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.063 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d75e00640 1 -- 172.21.15.194:0/591589876 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d780b46f0 con 0x7f6d780b4de0 2024-03-06T07:26:49.063 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d76800640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d7815f490 0x7f6d780b8720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:26:49.063 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.059+0000 7f6d75e00640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d780b4de0 0x7f6d7814e410 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f6d68007ef0 tx=0x7f6d6800e4b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.060+0000 7f6d6ea00640 1 -- 172.21.15.194:0/591589876 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d68014620 con 0x7f6d780b4de0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.060+0000 7f6d6ea00640 1 -- 172.21.15.194:0/591589876 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d68019070 con 0x7f6d780b4de0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.060+0000 7f6d6ea00640 1 -- 172.21.15.194:0/591589876 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d68014a00 con 0x7f6d780b4de0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.060+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6d780b0c30 con 0x7f6d780b4de0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.060+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6d780b11f0 con 0x7f6d780b4de0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.060+0000 7f6d6ea00640 1 -- 172.21.15.194:0/591589876 <== mon.2 v2:172.21.15.194:3301/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f6d68014ba0 con 0x7f6d780b4de0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.061+0000 7f6d6ea00640 1 -- 172.21.15.194:0/591589876 <== mon.2 v2:172.21.15.194:3301/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 725+0+0 (secure 0 0 0) 0x7f6d6801d400 con 0x7f6d780b4de0 2024-03-06T07:26:49.064 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.061+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6d7804f3a0 con 0x7f6d780b4de0 2024-03-06T07:26:49.066 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.064+0000 7f6d6ea00640 1 -- 172.21.15.194:0/591589876 <== mon.2 v2:172.21.15.194:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f6d6801e460 con 0x7f6d780b4de0 2024-03-06T07:26:49.160 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.158+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "osd new", "uuid": "ca3d759d-401c-42df-893e-5c63b2bbab14", "id": 0} v 0) v1 -- 0x7f6d780c7140 con 0x7f6d780b4de0 2024-03-06T07:26:49.167 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.164+0000 7f6d6ea00640 1 -- 172.21.15.194:0/591589876 <== mon.2 v2:172.21.15.194:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "ca3d759d-401c-42df-893e-5c63b2bbab14", "id": 0}]=0 v3) v1 ==== 112+0+2 (secure 0 0 0) 0x7f6d6801e640 con 0x7f6d780b4de0 2024-03-06T07:26:49.167 INFO:teuthology.orchestra.run.smithi194.stdout:0 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d780b4de0 msgr2=0x7f6d7814e410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d780b4de0 0x7f6d7814e410 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f6d68007ef0 tx=0x7f6d6800e4b0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 shutdown_connections 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d7815f490 0x7f6d780b8720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d780ced50 0x7f6d780cc5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 --2- 172.21.15.194:0/591589876 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d780b4de0 0x7f6d7814e410 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 >> 172.21.15.194:0/591589876 conn(0x7f6d780d3ed0 msgr2=0x7f6d7806c510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 shutdown_connections 2024-03-06T07:26:49.169 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.166+0000 7f6d7ce00640 1 -- 172.21.15.194:0/591589876 wait complete. 2024-03-06T07:26:49.182 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.181+0000 7fc784c2f040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-06T07:26:49.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new a4b124ea-80de-4143-892b-1cbf191ab49e 1 2024-03-06T07:26:49.219 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.216+0000 7fce54073040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-06T07:26:49.245 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.244+0000 7fc784c2f040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-06T07:26:49.287 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.285+0000 7fce54073040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-06T07:26:49.296 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.294+0000 7fc784c2f040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-06T07:26:49.345 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.343+0000 7fce54073040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-06T07:26:49.349 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.347+0000 7f9cbc800640 1 Processor -- start 2024-03-06T07:26:49.349 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.347+0000 7f9cbc800640 1 -- start start 2024-03-06T07:26:49.351 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.348+0000 7f9cbc800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb415e240 0x7f9cb40d1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.351 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.348+0000 7f9cbc800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9cb40cfc30 con 0x7f9cb40d0e70 2024-03-06T07:26:49.351 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.348+0000 7f9cbc800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9cb40b34e0 con 0x7f9cb40d4950 2024-03-06T07:26:49.351 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.348+0000 7f9cbc800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9cb40cfed0 con 0x7f9cb415e240 2024-03-06T07:26:49.351 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.348+0000 7f9cb9400640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f9cb40d4950 0x7f9cb40d0760 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.194:40290/0 (socket says 172.21.15.194:40290) 2024-03-06T07:26:49.351 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.348+0000 7f9cb9400640 1 -- 172.21.15.194:0/3635207948 learned_addr learned my addr 172.21.15.194:0/3635207948 (peer_addr_for_me v1:172.21.15.194:0/0) 2024-03-06T07:26:49.351 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.348+0000 7f9cb9e00640 1 --2- 172.21.15.194:0/3635207948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb415e240 0x7f9cb40d1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.352 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.349+0000 7f9cb8a00640 1 -- 172.21.15.194:0/3635207948 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2283119239 0 0) 0x7f9cb40b34e0 con 0x7f9cb40d4950 2024-03-06T07:26:49.352 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.349+0000 7f9cb8a00640 1 -- 172.21.15.194:0/3635207948 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9c9c003520 con 0x7f9cb40d4950 2024-03-06T07:26:49.352 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.349+0000 7f9cb9e00640 1 -- 172.21.15.194:0/3635207948 >> v1:172.21.15.194:6789/0 conn(0x7f9cb40d4950 legacy=0x7f9cb40d0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.352 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.349+0000 7f9cb9e00640 1 -- 172.21.15.194:0/3635207948 >> v1:172.21.15.86:6789/0 conn(0x7f9cb40d0e70 legacy=0x7f9cb40b3e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.352 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.349+0000 7f9cb9e00640 1 -- 172.21.15.194:0/3635207948 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9cb40d01b0 con 0x7f9cb415e240 2024-03-06T07:26:49.353 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.350+0000 7f9cb9e00640 1 --2- 172.21.15.194:0/3635207948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb415e240 0x7f9cb40d1c10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f9ca80097d0 tx=0x7f9ca802b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=10f3d60b2d03d338 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.353 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.350+0000 7f9cb8a00640 1 -- 172.21.15.194:0/3635207948 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ca8035020 con 0x7f9cb415e240 2024-03-06T07:26:49.353 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.350+0000 7f9cb8a00640 1 -- 172.21.15.194:0/3635207948 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9ca8031890 con 0x7f9cb415e240 2024-03-06T07:26:49.353 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.350+0000 7f9cb8a00640 1 -- 172.21.15.194:0/3635207948 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ca8031c60 con 0x7f9cb415e240 2024-03-06T07:26:49.354 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.351+0000 7f9cbc800640 1 -- 172.21.15.194:0/3635207948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb415e240 msgr2=0x7f9cb40d1c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.354 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.351+0000 7f9cbc800640 1 --2- 172.21.15.194:0/3635207948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb415e240 0x7f9cb40d1c10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f9ca80097d0 tx=0x7f9ca802b930 comp rx=0 tx=0).stop 2024-03-06T07:26:49.354 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.351+0000 7f9cbc800640 1 -- 172.21.15.194:0/3635207948 shutdown_connections 2024-03-06T07:26:49.354 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.351+0000 7f9cbc800640 1 --2- 172.21.15.194:0/3635207948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb415e240 0x7f9cb40d1c10 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.354 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.351+0000 7f9cbc800640 1 -- 172.21.15.194:0/3635207948 >> 172.21.15.194:0/3635207948 conn(0x7f9cb40d3e30 msgr2=0x7f9cb40d4240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:49.354 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.352+0000 7f9cbc800640 1 -- 172.21.15.194:0/3635207948 shutdown_connections 2024-03-06T07:26:49.355 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.352+0000 7f9cbc800640 1 -- 172.21.15.194:0/3635207948 wait complete. 2024-03-06T07:26:49.356 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.354+0000 7f9cbc800640 1 Processor -- start 2024-03-06T07:26:49.356 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.354+0000 7f9cbc800640 1 -- start start 2024-03-06T07:26:49.358 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.356+0000 7fc784c2f040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-06T07:26:49.359 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cbc800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9cb40d0e70 0x7f9cb40ca080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.359 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cbc800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb40d4950 0x7f9cb40ca5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.359 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cbc800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9cb415e240 0x7f9cb40caf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.359 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cbc800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9cb415e7a0 con 0x7f9cb415e240 2024-03-06T07:26:49.359 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cbc800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9cb415e620 con 0x7f9cb40d0e70 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cbc800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9cb40b4540 con 0x7f9cb40d4950 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cb9400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb40d4950 0x7f9cb40ca5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cb9400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb40d4950 0x7f9cb40ca5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41446/0 (socket says 172.21.15.194:41446) 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cb9400640 1 -- 172.21.15.194:0/1596979556 learned_addr learned my addr 172.21.15.194:0/1596979556 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.356+0000 7f9cb9e00640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9cb40d0e70 0x7f9cb40ca080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cba800640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9cb415e240 0x7f9cb40caf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cb9400640 1 -- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9cb40d0e70 msgr2=0x7f9cb40ca080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cb9400640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9cb40d0e70 0x7f9cb40ca080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cb9400640 1 -- 172.21.15.194:0/1596979556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9cb415e240 msgr2=0x7f9cb40caf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cb9400640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9cb415e240 0x7f9cb40caf00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.360 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cb9400640 1 -- 172.21.15.194:0/1596979556 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9cb415f970 con 0x7f9cb40d4950 2024-03-06T07:26:49.361 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cb9400640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb40d4950 0x7f9cb40ca5c0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f9ca400c4d0 tx=0x7f9ca400c990 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cae000640 1 -- 172.21.15.194:0/1596979556 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ca4017070 con 0x7f9cb40d4950 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cae000640 1 -- 172.21.15.194:0/1596979556 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9ca4005ce0 con 0x7f9cb40d4950 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cae000640 1 -- 172.21.15.194:0/1596979556 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ca4007270 con 0x7f9cb40d4950 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.357+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9cb40cbbb0 con 0x7f9cb40d4950 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.358+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9cb40cc170 con 0x7f9cb40d4950 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.358+0000 7f9cae000640 1 -- 172.21.15.194:0/1596979556 <== mon.2 v2:172.21.15.194:3301/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f9ca401a070 con 0x7f9cb40d4950 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.358+0000 7f9cae000640 1 -- 172.21.15.194:0/1596979556 <== mon.2 v2:172.21.15.194:3301/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 840+0+0 (secure 0 0 0) 0x7f9ca4012400 con 0x7f9cb40d4950 2024-03-06T07:26:49.362 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.359+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9cb404f3a0 con 0x7f9cb40d4950 2024-03-06T07:26:49.365 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.363+0000 7f9cae000640 1 -- 172.21.15.194:0/1596979556 <== mon.2 v2:172.21.15.194:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f9ca4007410 con 0x7f9cb40d4950 2024-03-06T07:26:49.407 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.406+0000 7fc784c2f040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-06T07:26:49.415 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.412+0000 7fce54073040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-06T07:26:49.444 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.441+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "osd new", "uuid": "a4b124ea-80de-4143-892b-1cbf191ab49e", "id": 1} v 0) v1 -- 0x7f9cb40cd500 con 0x7f9cb40d4950 2024-03-06T07:26:49.450 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.448+0000 7f9cae000640 1 -- 172.21.15.194:0/1596979556 <== mon.2 v2:172.21.15.194:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "a4b124ea-80de-4143-892b-1cbf191ab49e", "id": 1}]=0 v4) v1 ==== 112+0+2 (secure 0 0 0) 0x7f9ca40075f0 con 0x7f9cb40d4950 2024-03-06T07:26:49.451 INFO:teuthology.orchestra.run.smithi194.stdout:1 2024-03-06T07:26:49.452 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.449+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb40d4950 msgr2=0x7f9cb40ca5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.449+0000 7f9cbc800640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb40d4950 0x7f9cb40ca5c0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f9ca400c4d0 tx=0x7f9ca400c990 comp rx=0 tx=0).stop 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.449+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 shutdown_connections 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.449+0000 7f9cbc800640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9cb415e240 0x7f9cb40caf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.450+0000 7f9cbc800640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9cb40d4950 0x7f9cb40ca5c0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.450+0000 7f9cbc800640 1 --2- 172.21.15.194:0/1596979556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9cb40d0e70 0x7f9cb40ca080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.450+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 >> 172.21.15.194:0/1596979556 conn(0x7f9cb40d3e30 msgr2=0x7f9cb40cec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.450+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 shutdown_connections 2024-03-06T07:26:49.453 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.450+0000 7f9cbc800640 1 -- 172.21.15.194:0/1596979556 wait complete. 2024-03-06T07:26:49.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new 52f80d9b-399d-40fa-ba7e-0119d56617a6 2 2024-03-06T07:26:49.476 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.473+0000 7fce54073040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-06T07:26:49.537 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.536+0000 7fc784c2f040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-06T07:26:49.616 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.613+0000 7fce54073040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-06T07:26:49.616 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.615+0000 7fc784c2f040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-06T07:26:49.656 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.653+0000 7f0464a00640 1 Processor -- start 2024-03-06T07:26:49.656 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.654+0000 7f0464a00640 1 -- start start 2024-03-06T07:26:49.658 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f0464a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f046015e100 0x7f04600d0750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.658 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f0464a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f04600b3120 0x7f04600d0c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.658 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f0464a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04600d4ce0 con 0x7f04600b2bd0 2024-03-06T07:26:49.658 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f0464a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04600d4a40 con 0x7f04600b3120 2024-03-06T07:26:49.658 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f0464a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f046015e4e0 con 0x7f046015e100 2024-03-06T07:26:49.658 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f045f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f046015e100 0x7f04600d0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.658 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f045fe00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f04600b2bd0 0x7f04600ce890 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.194:40298/0 (socket says 172.21.15.194:40298) 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f045f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f046015e100 0x7f04600d0750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41450/0 (socket says 172.21.15.194:41450) 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.655+0000 7f045fe00640 1 -- 172.21.15.194:0/958416867 learned_addr learned my addr 172.21.15.194:0/958416867 (peer_addr_for_me v1:172.21.15.194:0/0) 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.656+0000 7f0457e00640 1 --2- 172.21.15.194:0/958416867 >> v2:172.21.15.86:3300/0 conn(0x7f04600b3120 0x7f04600d0c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.656+0000 7f045f400640 1 -- 172.21.15.194:0/958416867 >> v2:172.21.15.86:3300/0 conn(0x7f04600b3120 msgr2=0x7f04600d0c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.656+0000 7f045f400640 1 --2- 172.21.15.194:0/958416867 >> v2:172.21.15.86:3300/0 conn(0x7f04600b3120 0x7f04600d0c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.656+0000 7f045f400640 1 -- 172.21.15.194:0/958416867 >> v1:172.21.15.194:6789/0 conn(0x7f04600b2bd0 legacy=0x7f04600ce890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.656+0000 7f045f400640 1 -- 172.21.15.194:0/958416867 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04600d2470 con 0x7f046015e100 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.656+0000 7f045ea00640 1 -- 172.21.15.194:0/958416867 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 609528305 0 0) 0x7f04600d4ce0 con 0x7f04600b2bd0 2024-03-06T07:26:49.659 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.657+0000 7f045f400640 1 --2- 172.21.15.194:0/958416867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f046015e100 0x7f04600d0750 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f04500097a0 tx=0x7f045002b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cca2080e10f8e070 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.660 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.657+0000 7f045ea00640 1 -- 172.21.15.194:0/958416867 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450035020 con 0x7f046015e100 2024-03-06T07:26:49.660 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f045ea00640 1 -- 172.21.15.194:0/958416867 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0450031830 con 0x7f046015e100 2024-03-06T07:26:49.660 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f045ea00640 1 -- 172.21.15.194:0/958416867 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450031c00 con 0x7f046015e100 2024-03-06T07:26:49.661 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 -- 172.21.15.194:0/958416867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f046015e100 msgr2=0x7f04600d0750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.661 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 --2- 172.21.15.194:0/958416867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f046015e100 0x7f04600d0750 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f04500097a0 tx=0x7f045002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.661 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 -- 172.21.15.194:0/958416867 shutdown_connections 2024-03-06T07:26:49.661 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 --2- 172.21.15.194:0/958416867 >> v2:172.21.15.86:3300/0 conn(0x7f04600b3120 0x7f04600d0c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.661 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 --2- 172.21.15.194:0/958416867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f046015e100 0x7f04600d0750 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.661 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 -- 172.21.15.194:0/958416867 >> 172.21.15.194:0/958416867 conn(0x7f04600d3d80 msgr2=0x7f04600d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:49.662 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 -- 172.21.15.194:0/958416867 shutdown_connections 2024-03-06T07:26:49.662 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.658+0000 7f0464a00640 1 -- 172.21.15.194:0/958416867 wait complete. 2024-03-06T07:26:49.663 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.661+0000 7f0464a00640 1 Processor -- start 2024-03-06T07:26:49.663 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.661+0000 7f0464a00640 1 -- start start 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0464a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04600b2bd0 0x7f046014f160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0464a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04600b3120 0x7f04600c78b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0464a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f046015e100 0x7f04600cae50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0464a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04600cfbd0 con 0x7f046015e100 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0464a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04600d2a30 con 0x7f04600b2bd0 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0464a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04600b4780 con 0x7f04600b3120 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0457e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04600b3120 0x7f04600c78b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f045f400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04600b2bd0 0x7f046014f160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.666 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0457e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04600b3120 0x7f04600c78b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41466/0 (socket says 172.21.15.194:41466) 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0457e00640 1 -- 172.21.15.194:0/542474719 learned_addr learned my addr 172.21.15.194:0/542474719 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f045fe00640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f046015e100 0x7f04600cae50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f045f400640 1 -- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04600b3120 msgr2=0x7f04600c78b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f045f400640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04600b3120 0x7f04600c78b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f045f400640 1 -- 172.21.15.194:0/542474719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f046015e100 msgr2=0x7f04600cae50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f045f400640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f046015e100 0x7f04600cae50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f045f400640 1 -- 172.21.15.194:0/542474719 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f046015c180 con 0x7f04600b2bd0 2024-03-06T07:26:49.667 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.663+0000 7f0457e00640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04600b3120 0x7f04600c78b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:26:49.668 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.664+0000 7f045f400640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04600b2bd0 0x7f046014f160 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f0450033000 tx=0x7f04500046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.668 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.664+0000 7f0457400640 1 -- 172.21.15.194:0/542474719 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450035040 con 0x7f04600b2bd0 2024-03-06T07:26:49.668 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.664+0000 7f0457400640 1 -- 172.21.15.194:0/542474719 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0450004250 con 0x7f04600b2bd0 2024-03-06T07:26:49.668 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.664+0000 7f0457400640 1 -- 172.21.15.194:0/542474719 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f045003a700 con 0x7f04600b2bd0 2024-03-06T07:26:49.669 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.664+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04600b7ae0 con 0x7f04600b2bd0 2024-03-06T07:26:49.669 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.664+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04600b80a0 con 0x7f04600b2bd0 2024-03-06T07:26:49.669 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.665+0000 7f0457400640 1 -- 172.21.15.194:0/542474719 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f0450038020 con 0x7f04600b2bd0 2024-03-06T07:26:49.669 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.665+0000 7f0457400640 1 -- 172.21.15.194:0/542474719 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 955+0+0 (secure 0 0 0) 0x7f045003b430 con 0x7f04600b2bd0 2024-03-06T07:26:49.669 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.666+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f04600c70f0 con 0x7f04600b2bd0 2024-03-06T07:26:49.670 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.669+0000 7fc784c2f040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-06T07:26:49.671 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.669+0000 7f0457400640 1 -- 172.21.15.194:0/542474719 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f0450053020 con 0x7f04600b2bd0 2024-03-06T07:26:49.704 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.701+0000 7fce54073040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-06T07:26:49.730 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.729+0000 7fc784c2f040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-06T07:26:49.756 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.753+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "osd new", "uuid": "52f80d9b-399d-40fa-ba7e-0119d56617a6", "id": 2} v 0) v1 -- 0x7f046006d470 con 0x7f04600b2bd0 2024-03-06T07:26:49.760 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.758+0000 7fce54073040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-06T07:26:49.763 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.760+0000 7f0457400640 1 -- 172.21.15.194:0/542474719 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "52f80d9b-399d-40fa-ba7e-0119d56617a6", "id": 2}]=0 v5) v1 ==== 112+0+2 (secure 0 0 0) 0x7f0450031800 con 0x7f04600b2bd0 2024-03-06T07:26:49.763 INFO:teuthology.orchestra.run.smithi194.stdout:2 2024-03-06T07:26:49.764 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.761+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04600b2bd0 msgr2=0x7f046014f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.764 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.761+0000 7f0464a00640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04600b2bd0 0x7f046014f160 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f0450033000 tx=0x7f04500046c0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.764 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.762+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 shutdown_connections 2024-03-06T07:26:49.765 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.762+0000 7f0464a00640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f046015e100 0x7f04600cae50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.765 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.762+0000 7f0464a00640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04600b3120 0x7f04600c78b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.765 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.762+0000 7f0464a00640 1 --2- 172.21.15.194:0/542474719 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04600b2bd0 0x7f046014f160 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.765 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.762+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 >> 172.21.15.194:0/542474719 conn(0x7f04600d3d80 msgr2=0x7f04600d1a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:49.765 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.762+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 shutdown_connections 2024-03-06T07:26:49.765 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.762+0000 7f0464a00640 1 -- 172.21.15.194:0/542474719 wait complete. 2024-03-06T07:26:49.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new 771a4b29-0323-4910-acff-3261da6fee51 3 2024-03-06T07:26:49.786 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:49.784+0000 7fc784c2f040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-06T07:26:49.827 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.824+0000 7fce54073040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-06T07:26:49.887 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:49.884+0000 7fce54073040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-06T07:26:49.959 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.956+0000 7f385ec00640 1 Processor -- start 2024-03-06T07:26:49.959 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.956+0000 7f385ec00640 1 -- start start 2024-03-06T07:26:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.958+0000 7f385ec00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f38580d0ea0 0x7f3858168f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.958+0000 7f385ec00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f38580d1d00 con 0x7f385815e1f0 2024-03-06T07:26:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.958+0000 7f385ec00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f38580d1fe0 con 0x7f38580d2da0 2024-03-06T07:26:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.958+0000 7f385ec00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f385815e5d0 con 0x7f38580d0ea0 2024-03-06T07:26:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.958+0000 7f384fe00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f38580d2da0 0x7f38580d0790 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.194:40306/0 (socket says 172.21.15.194:40306) 2024-03-06T07:26:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.958+0000 7f384fe00640 1 -- 172.21.15.194:0/2206717923 learned_addr learned my addr 172.21.15.194:0/2206717923 (peer_addr_for_me v1:172.21.15.194:0/0) 2024-03-06T07:26:49.961 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.958+0000 7f385cc00640 1 --2- 172.21.15.194:0/2206717923 >> v2:172.21.15.194:3300/0 conn(0x7f38580d0ea0 0x7f3858168f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.962 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.959+0000 7f3857400640 1 -- 172.21.15.194:0/2206717923 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 545759974 0 0) 0x7f38580d1fe0 con 0x7f38580d2da0 2024-03-06T07:26:49.962 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.959+0000 7f385cc00640 1 -- 172.21.15.194:0/2206717923 >> v1:172.21.15.194:6789/0 conn(0x7f38580d2da0 legacy=0x7f38580d0790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.962 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.959+0000 7f385cc00640 1 -- 172.21.15.194:0/2206717923 >> v1:172.21.15.86:6789/0 conn(0x7f385815e1f0 legacy=0x7f38580cf600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.962 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.959+0000 7f385cc00640 1 -- 172.21.15.194:0/2206717923 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38580d22c0 con 0x7f38580d0ea0 2024-03-06T07:26:49.963 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.960+0000 7f385cc00640 1 --2- 172.21.15.194:0/2206717923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f3858168f80 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f3850009d90 tx=0x7f385002beb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8cc0e6a365cab1af server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.963 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.960+0000 7f3857400640 1 -- 172.21.15.194:0/2206717923 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3850039070 con 0x7f38580d0ea0 2024-03-06T07:26:49.963 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.960+0000 7f3857400640 1 -- 172.21.15.194:0/2206717923 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f385002cc60 con 0x7f38580d0ea0 2024-03-06T07:26:49.963 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.960+0000 7f3857400640 1 -- 172.21.15.194:0/2206717923 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3850034660 con 0x7f38580d0ea0 2024-03-06T07:26:49.964 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.961+0000 7f385ec00640 1 -- 172.21.15.194:0/2206717923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 msgr2=0x7f3858168f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.964 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.961+0000 7f385ec00640 1 --2- 172.21.15.194:0/2206717923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f3858168f80 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f3850009d90 tx=0x7f385002beb0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.964 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.961+0000 7f385ec00640 1 -- 172.21.15.194:0/2206717923 shutdown_connections 2024-03-06T07:26:49.964 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.961+0000 7f385ec00640 1 --2- 172.21.15.194:0/2206717923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f3858168f80 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.964 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.961+0000 7f385ec00640 1 -- 172.21.15.194:0/2206717923 >> 172.21.15.194:0/2206717923 conn(0x7f38580d3ee0 msgr2=0x7f38580d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:49.964 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.961+0000 7f385ec00640 1 -- 172.21.15.194:0/2206717923 shutdown_connections 2024-03-06T07:26:49.964 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.961+0000 7f385ec00640 1 -- 172.21.15.194:0/2206717923 wait complete. 2024-03-06T07:26:49.966 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.964+0000 7f385ec00640 1 Processor -- start 2024-03-06T07:26:49.966 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.964+0000 7f385ec00640 1 -- start start 2024-03-06T07:26:49.969 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.965+0000 7f385ec00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f38580cdc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.969 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.965+0000 7f385ec00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38580d2da0 0x7f385815f160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.965+0000 7f385ec00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f385815e1f0 0x7f38580c79f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.965+0000 7f385ec00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38580b7d40 con 0x7f385815e1f0 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.965+0000 7f385ec00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38580d1ab0 con 0x7f38580d0ea0 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.965+0000 7f385ec00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38580d2880 con 0x7f38580d2da0 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f38580cdc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f384fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38580d2da0 0x7f385815f160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f38580cdc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.194:43092/0 (socket says 172.21.15.194:43092) 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 -- 172.21.15.194:0/3486429004 learned_addr learned my addr 172.21.15.194:0/3486429004 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f384fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38580d2da0 0x7f385815f160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41480/0 (socket says 172.21.15.194:41480) 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f385cc00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f385815e1f0 0x7f38580c79f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 -- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38580d2da0 msgr2=0x7f385815f160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38580d2da0 0x7f385815f160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 -- 172.21.15.194:0/3486429004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f385815e1f0 msgr2=0x7f38580c79f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f385815e1f0 0x7f38580c79f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:49.970 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 -- 172.21.15.194:0/3486429004 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f385815fa70 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f384fe00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38580d2da0 0x7f385815f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.966+0000 7f3857e00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f38580cdc60 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f384800c400 tx=0x7f384800c8c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.967+0000 7f3854c00640 1 -- 172.21.15.194:0/3486429004 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3848017070 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.967+0000 7f3854c00640 1 -- 172.21.15.194:0/3486429004 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f38480052d0 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.967+0000 7f3854c00640 1 -- 172.21.15.194:0/3486429004 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3848003830 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.967+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f385815fd50 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.967+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3858160310 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.967+0000 7f3854c00640 1 -- 172.21.15.194:0/3486429004 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f384801a070 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.967+0000 7f3854c00640 1 -- 172.21.15.194:0/3486429004 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1070+0+0 (secure 0 0 0) 0x7f3848011c20 con 0x7f38580d0ea0 2024-03-06T07:26:49.971 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.968+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f38580cb900 con 0x7f38580d0ea0 2024-03-06T07:26:49.974 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:49.972+0000 7f3854c00640 1 -- 172.21.15.194:0/3486429004 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f38580cb900 con 0x7f38580d0ea0 2024-03-06T07:26:50.051 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.048+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "osd new", "uuid": "771a4b29-0323-4910-acff-3261da6fee51", "id": 3} v 0) v1 -- 0x7f385804f3a0 con 0x7f38580d0ea0 2024-03-06T07:26:50.058 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.055+0000 7f3854c00640 1 -- 172.21.15.194:0/3486429004 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "771a4b29-0323-4910-acff-3261da6fee51", "id": 3}]=0 v6) v1 ==== 112+0+2 (secure 0 0 0) 0x7f3848021020 con 0x7f38580d0ea0 2024-03-06T07:26:50.058 INFO:teuthology.orchestra.run.smithi194.stdout:3 2024-03-06T07:26:50.059 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 msgr2=0x7f38580cdc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f38580cdc60 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f384800c400 tx=0x7f384800c8c0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 shutdown_connections 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f385815e1f0 0x7f38580c79f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38580d2da0 0x7f385815f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 --2- 172.21.15.194:0/3486429004 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38580d0ea0 0x7f38580cdc60 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 >> 172.21.15.194:0/3486429004 conn(0x7f38580d3ee0 msgr2=0x7f38580cf1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 shutdown_connections 2024-03-06T07:26:50.060 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.057+0000 7f385ec00640 1 -- 172.21.15.194:0/3486429004 wait complete. 2024-03-06T07:26:50.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new 79765dea-3964-41f3-a109-a064f2fb7c89 4 2024-03-06T07:26:50.234 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:50.233+0000 7fc784c2f040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-06T07:26:50.293 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.290+0000 7f1cc0600640 1 Processor -- start 2024-03-06T07:26:50.293 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.291+0000 7f1cc0600640 1 -- start start 2024-03-06T07:26:50.295 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cc0600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1cb80d0ea0 0x7f1cb8168f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.295 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cc0600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1cb80d1d00 con 0x7f1cb80d2da0 2024-03-06T07:26:50.296 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cc0600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1cb80d1fe0 con 0x7f1cb815e1f0 2024-03-06T07:26:50.296 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cc0600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1cb815e5d0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.296 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cbe600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1cb80d0ea0 0x7f1cb8168f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.296 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cbe600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1cb80d0ea0 0x7f1cb8168f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.194:43094/0 (socket says 172.21.15.194:43094) 2024-03-06T07:26:50.296 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cbe600640 1 -- 172.21.15.194:0/1800220613 learned_addr learned my addr 172.21.15.194:0/1800220613 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:50.296 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.292+0000 7f1cbdc00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f1cb815e1f0 0x7f1cb80cf600 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.194:40316/0 (socket says 172.21.15.194:40316) 2024-03-06T07:26:50.297 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.293+0000 7f1cbe600640 1 -- 172.21.15.194:0/1800220613 >> v1:172.21.15.194:6789/0 conn(0x7f1cb815e1f0 legacy=0x7f1cb80cf600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.297 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.293+0000 7f1cbe600640 1 -- 172.21.15.194:0/1800220613 >> v1:172.21.15.86:6789/0 conn(0x7f1cb80d2da0 legacy=0x7f1cb80d0790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:50.297 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.293+0000 7f1cbe600640 1 -- 172.21.15.194:0/1800220613 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cb80d22c0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.298 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.294+0000 7f1cbe600640 1 --2- 172.21.15.194:0/1800220613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb8168f80 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f1cb0009e50 tx=0x7f1cb002bee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9ce72678238334d2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:50.298 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.294+0000 7f1cb7e00640 1 -- 172.21.15.194:0/1800220613 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cb002ca80 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.294+0000 7f1cb7e00640 1 -- 172.21.15.194:0/1800220613 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cb002cc20 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.294+0000 7f1cb7e00640 1 -- 172.21.15.194:0/1800220613 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cb00343f0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.295+0000 7f1cc0600640 1 -- 172.21.15.194:0/1800220613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 msgr2=0x7f1cb8168f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.295+0000 7f1cc0600640 1 --2- 172.21.15.194:0/1800220613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb8168f80 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f1cb0009e50 tx=0x7f1cb002bee0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.295+0000 7f1cc0600640 1 -- 172.21.15.194:0/1800220613 shutdown_connections 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.295+0000 7f1cc0600640 1 --2- 172.21.15.194:0/1800220613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb8168f80 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.295+0000 7f1cc0600640 1 -- 172.21.15.194:0/1800220613 >> 172.21.15.194:0/1800220613 conn(0x7f1cb80d3ee0 msgr2=0x7f1cb80d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.295+0000 7f1cc0600640 1 -- 172.21.15.194:0/1800220613 shutdown_connections 2024-03-06T07:26:50.299 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.295+0000 7f1cc0600640 1 -- 172.21.15.194:0/1800220613 wait complete. 2024-03-06T07:26:50.302 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.296+0000 7f1cc0600640 1 Processor -- start 2024-03-06T07:26:50.302 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.296+0000 7f1cc0600640 1 -- start start 2024-03-06T07:26:50.302 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.297+0000 7f1cc0600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb80c8400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.302 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.297+0000 7f1cc0600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cb80d2da0 0x7f1cb80c9df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.302 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.297+0000 7f1cc0600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cb815e1f0 0x7f1cb8159340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.297+0000 7f1cc0600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cb80b7d40 con 0x7f1cb80d2da0 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.297+0000 7f1cc0600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cb80d1ab0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.297+0000 7f1cc0600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1cb80d2880 con 0x7f1cb815e1f0 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb80c8400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb80c8400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.194:43096/0 (socket says 172.21.15.194:43096) 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 -- 172.21.15.194:0/1614804637 learned_addr learned my addr 172.21.15.194:0/1614804637 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbe600640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cb815e1f0 0x7f1cb8159340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbd200640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cb80d2da0 0x7f1cb80c9df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 -- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cb815e1f0 msgr2=0x7f1cb8159340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cb815e1f0 0x7f1cb8159340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 -- 172.21.15.194:0/1614804637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cb80d2da0 msgr2=0x7f1cb80c9df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cb80d2da0 0x7f1cb80c9df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 -- 172.21.15.194:0/1614804637 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cb80ca330 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbd200640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cb80d2da0 0x7f1cb80c9df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:26:50.303 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbe600640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cb815e1f0 0x7f1cb8159340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.298+0000 7f1cbdc00640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb80c8400 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f1cac00c3c0 tx=0x7f1cac00c880 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.299+0000 7f1cb5600640 1 -- 172.21.15.194:0/1614804637 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cac0074e0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.299+0000 7f1cb5600640 1 -- 172.21.15.194:0/1614804637 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cac007680 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.299+0000 7f1cb5600640 1 -- 172.21.15.194:0/1614804637 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cac004a00 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.299+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1cb80cb0e0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.299+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1cb80cb6a0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.299+0000 7f1cb5600640 1 -- 172.21.15.194:0/1614804637 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f1cac01a020 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.299+0000 7f1cb5600640 1 -- 172.21.15.194:0/1614804637 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1185+0+0 (secure 0 0 0) 0x7f1cac013910 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.304 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.300+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1cb804f3a0 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.305 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.302+0000 7f1cb5600640 1 -- 172.21.15.194:0/1614804637 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f1cac021070 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.365 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:50.362+0000 7fce54073040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-06T07:26:50.384 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.382+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "osd new", "uuid": "79765dea-3964-41f3-a109-a064f2fb7c89", "id": 4} v 0) v1 -- 0x7f1cb808d940 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.391 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.388+0000 7f1cb5600640 1 -- 172.21.15.194:0/1614804637 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "79765dea-3964-41f3-a109-a064f2fb7c89", "id": 4}]=0 v7) v1 ==== 112+0+2 (secure 0 0 0) 0x7f1cac013c30 con 0x7f1cb80d0ea0 2024-03-06T07:26:50.391 INFO:teuthology.orchestra.run.smithi194.stdout:4 2024-03-06T07:26:50.392 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.389+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 msgr2=0x7f1cb80c8400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.389+0000 7f1cc0600640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb80c8400 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f1cac00c3c0 tx=0x7f1cac00c880 comp rx=0 tx=0).stop 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.390+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 shutdown_connections 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.390+0000 7f1cc0600640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cb815e1f0 0x7f1cb8159340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.390+0000 7f1cc0600640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cb80d2da0 0x7f1cb80c9df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.390+0000 7f1cc0600640 1 --2- 172.21.15.194:0/1614804637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cb80d0ea0 0x7f1cb80c8400 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.390+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 >> 172.21.15.194:0/1614804637 conn(0x7f1cb80d3ee0 msgr2=0x7f1cb80cf1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.390+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 shutdown_connections 2024-03-06T07:26:50.393 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.390+0000 7f1cc0600640 1 -- 172.21.15.194:0/1614804637 wait complete. 2024-03-06T07:26:50.393 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:50.391+0000 7fc784c2f040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-06T07:26:50.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new dbfaf0fd-5805-4e2d-8903-0b4a5c522443 5 2024-03-06T07:26:50.534 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:50.531+0000 7fce54073040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-06T07:26:50.540 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:50.539+0000 7fc784c2f040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-06T07:26:50.591 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.588+0000 7f67b2600640 1 Processor -- start 2024-03-06T07:26:50.591 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.588+0000 7f67b2600640 1 -- start start 2024-03-06T07:26:50.593 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67b2600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f67ac15e240 0x7f67ac0d1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.594 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67b2600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac0b3e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.594 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67b2600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67ac0cfc30 con 0x7f67ac0d4950 2024-03-06T07:26:50.594 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67b2600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f67ac0b34e0 con 0x7f67ac15e240 2024-03-06T07:26:50.594 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67b2600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67ac0cfed0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.594 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67abe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac0b3e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.594 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67abe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac0b3e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41496/0 (socket says 172.21.15.194:41496) 2024-03-06T07:26:50.595 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67abe00640 1 -- 172.21.15.194:0/3860524561 learned_addr learned my addr 172.21.15.194:0/3860524561 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:50.595 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.590+0000 7f67ab400640 1 --2- 172.21.15.194:0/3860524561 >> v2:172.21.15.86:3300/0 conn(0x7f67ac15e240 0x7f67ac0d1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.595 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.591+0000 7f67abe00640 1 -- 172.21.15.194:0/3860524561 >> v2:172.21.15.86:3300/0 conn(0x7f67ac15e240 msgr2=0x7f67ac0d1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:50.595 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.591+0000 7f67abe00640 1 --2- 172.21.15.194:0/3860524561 >> v2:172.21.15.86:3300/0 conn(0x7f67ac15e240 0x7f67ac0d1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.595 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.591+0000 7f67abe00640 1 -- 172.21.15.194:0/3860524561 >> v1:172.21.15.86:6789/0 conn(0x7f67ac0d4950 legacy=0x7f67ac0d0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:50.595 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.591+0000 7f67abe00640 1 -- 172.21.15.194:0/3860524561 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67ac0d01b0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.596 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:50.591+0000 7fc784c2f040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-06T07:26:50.601 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.592+0000 7f67abe00640 1 --2- 172.21.15.194:0/3860524561 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac0b3e30 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f67a4009e50 tx=0x7f67a402bed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=29ca0c4278ae40e3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:50.601 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.592+0000 7f67aaa00640 1 -- 172.21.15.194:0/3860524561 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67a402ca70 con 0x7f67ac0d0e70 2024-03-06T07:26:50.601 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.592+0000 7f67aaa00640 1 -- 172.21.15.194:0/3860524561 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f67a402cc10 con 0x7f67ac0d0e70 2024-03-06T07:26:50.601 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.592+0000 7f67aaa00640 1 -- 172.21.15.194:0/3860524561 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67a40343f0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.601 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 -- 172.21.15.194:0/3860524561 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 msgr2=0x7f67ac0b3e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 --2- 172.21.15.194:0/3860524561 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac0b3e30 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f67a4009e50 tx=0x7f67a402bed0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 -- 172.21.15.194:0/3860524561 shutdown_connections 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 --2- 172.21.15.194:0/3860524561 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac0b3e30 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 --2- 172.21.15.194:0/3860524561 >> v2:172.21.15.86:3300/0 conn(0x7f67ac15e240 0x7f67ac0d1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 -- 172.21.15.194:0/3860524561 >> 172.21.15.194:0/3860524561 conn(0x7f67ac0d3e30 msgr2=0x7f67ac0d4240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 -- 172.21.15.194:0/3860524561 shutdown_connections 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.593+0000 7f67b2600640 1 -- 172.21.15.194:0/3860524561 wait complete. 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.594+0000 7f67b2600640 1 Processor -- start 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.595+0000 7f67b2600640 1 -- start start 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.595+0000 7f67b2600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac157300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.595+0000 7f67b2600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67ac0d4950 0x7f67ac0c8de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.595+0000 7f67b2600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67ac15e240 0x7f67ac0c9320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.595+0000 7f67b2600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67ac15e7a0 con 0x7f67ac15e240 2024-03-06T07:26:50.602 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.595+0000 7f67b2600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67ac15e620 con 0x7f67ac0d4950 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.595+0000 7f67b2600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67ac0b4370 con 0x7f67ac0d0e70 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67a3e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67ac0d4950 0x7f67ac0c8de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67ab400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac157300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67a3e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67ac0d4950 0x7f67ac0c8de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.194:43102/0 (socket says 172.21.15.194:43102) 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67a3e00640 1 -- 172.21.15.194:0/3723356372 learned_addr learned my addr 172.21.15.194:0/3723356372 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67ab400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac157300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41508/0 (socket says 172.21.15.194:41508) 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67abe00640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67ac15e240 0x7f67ac0c9320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67ab400640 1 -- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67ac0d4950 msgr2=0x7f67ac0c8de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67ab400640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67ac0d4950 0x7f67ac0c8de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67ab400640 1 -- 172.21.15.194:0/3723356372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67ac15e240 msgr2=0x7f67ac0c9320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67ab400640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67ac15e240 0x7f67ac0c9320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67ab400640 1 -- 172.21.15.194:0/3723356372 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67ac0c9940 con 0x7f67ac0d0e70 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67a3e00640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67ac0d4950 0x7f67ac0c8de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:26:50.603 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.596+0000 7f67abe00640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67ac15e240 0x7f67ac0c9320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67ab400640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac157300 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f679c00d550 tx=0x7f679c00da10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67a3400640 1 -- 172.21.15.194:0/3723356372 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f679c0125f0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67a3400640 1 -- 172.21.15.194:0/3723356372 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f679c019070 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67a3400640 1 -- 172.21.15.194:0/3723356372 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f679c0129b0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f67ac0c9c20 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f67ac0ca1e0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67a3400640 1 -- 172.21.15.194:0/3723356372 <== mon.2 v2:172.21.15.194:3301/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f679c012b50 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.597+0000 7f67a3400640 1 -- 172.21.15.194:0/3723356372 <== mon.2 v2:172.21.15.194:3301/0 5 ==== osd_map(7..7 src has 1..7) v4 ==== 1300+0+0 (secure 0 0 0) 0x7f679c01e9e0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.598+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f67ac04f3a0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.604 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.601+0000 7f67a3400640 1 -- 172.21.15.194:0/3723356372 <== mon.2 v2:172.21.15.194:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f679c01e460 con 0x7f67ac0d0e70 2024-03-06T07:26:50.688 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:50.681+0000 7fce54073040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-06T07:26:50.698 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.695+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "osd new", "uuid": "dbfaf0fd-5805-4e2d-8903-0b4a5c522443", "id": 5} v 0) v1 -- 0x7f67ac06c4b0 con 0x7f67ac0d0e70 2024-03-06T07:26:50.706 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.703+0000 7f67a3400640 1 -- 172.21.15.194:0/3723356372 <== mon.2 v2:172.21.15.194:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "dbfaf0fd-5805-4e2d-8903-0b4a5c522443", "id": 5}]=0 v8) v1 ==== 112+0+2 (secure 0 0 0) 0x7f679c01e640 con 0x7f67ac0d0e70 2024-03-06T07:26:50.706 INFO:teuthology.orchestra.run.smithi194.stdout:5 2024-03-06T07:26:50.707 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 msgr2=0x7f67ac157300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac157300 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f679c00d550 tx=0x7f679c00da10 comp rx=0 tx=0).stop 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 shutdown_connections 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67ac15e240 0x7f67ac0c9320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67ac0d4950 0x7f67ac0c8de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 --2- 172.21.15.194:0/3723356372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67ac0d0e70 0x7f67ac157300 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 >> 172.21.15.194:0/3723356372 conn(0x7f67ac0d3e30 msgr2=0x7f67ac15d3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 shutdown_connections 2024-03-06T07:26:50.708 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.705+0000 7f67b2600640 1 -- 172.21.15.194:0/3723356372 wait complete. 2024-03-06T07:26:50.719 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:50.718+0000 7fc784c2f040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-06T07:26:50.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new 70fa6d13-7e1a-4720-9009-24c3c629e40f 6 2024-03-06T07:26:50.742 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:50.740+0000 7fce54073040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-06T07:26:50.829 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:50.828+0000 7fc784c2f040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-06T07:26:50.882 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:50.880+0000 7fce54073040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-06T07:26:50.905 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.902+0000 7fc107800640 1 Processor -- start 2024-03-06T07:26:50.905 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.903+0000 7fc107800640 1 -- start start 2024-03-06T07:26:50.907 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.904+0000 7fc107800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc1001455f0 0x7fc1000b2ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.907 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.904+0000 7fc107800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc1000ce070 con 0x7fc10015f990 2024-03-06T07:26:50.907 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.904+0000 7fc107800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc1001603e0 con 0x7fc1000d05c0 2024-03-06T07:26:50.907 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.904+0000 7fc107800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc1000ce680 con 0x7fc1001455f0 2024-03-06T07:26:50.907 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc105800640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7fc1000d05c0 0x7fc1000ceb10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.194:40332/0 (socket says 172.21.15.194:40332) 2024-03-06T07:26:50.907 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc105800640 1 -- 172.21.15.194:0/809878974 learned_addr learned my addr 172.21.15.194:0/809878974 (peer_addr_for_me v1:172.21.15.194:0/0) 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc104e00640 1 --2- 172.21.15.194:0/809878974 >> v2:172.21.15.86:3300/0 conn(0x7fc1001455f0 0x7fc1000b2ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3240507971 0 0) 0x7fc1001603e0 con 0x7fc1000d05c0 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc0e0003520 con 0x7fc1000d05c0 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 <== mon.1 v1:172.21.15.194:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2721154054 0 0) 0x7fc0e0003520 con 0x7fc1000d05c0 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 >> v2:172.21.15.86:3300/0 conn(0x7fc1001455f0 msgr2=0x7fc1000b2ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 --2- 172.21.15.194:0/809878974 >> v2:172.21.15.86:3300/0 conn(0x7fc1001455f0 0x7fc1000b2ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 >> v1:172.21.15.86:6789/0 conn(0x7fc10015f990 legacy=0x7fc1000cfeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 --> v1:172.21.15.194:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc1001606c0 con 0x7fc1000d05c0 2024-03-06T07:26:50.910 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 <== mon.1 v1:172.21.15.194:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc0f80019f0 con 0x7fc1000d05c0 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 >> v1:172.21.15.194:6789/0 conn(0x7fc1000d05c0 legacy=0x7fc1000ceb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc0e0003e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0e0004800 0x7fc0e0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.905+0000 7fc0ffe00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0e0005210 0x7fc0e000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1000ce680 con 0x7fc0e0004800 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0e000c140 con 0x7fc0e0003a70 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc0e000c340 con 0x7fc0e0005210 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc104e00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc0e0003e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc105800640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0e0005210 0x7fc0e000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc0f7e00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0e0004800 0x7fc0e0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.911 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc104e00640 1 -- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0e0005210 msgr2=0x7fc0e000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc104e00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0e0005210 0x7fc0e000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc104e00640 1 -- 172.21.15.194:0/809878974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0e0004800 msgr2=0x7fc0e0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc104e00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0e0004800 0x7fc0e0004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc104e00640 1 -- 172.21.15.194:0/809878974 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0e0003520 con 0x7fc0e0003a70 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.906+0000 7fc105800640 1 -- 172.21.15.194:0/809878974 reap_dead start 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.907+0000 7fc104e00640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc0e0003e90 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fc0f0009ac0 tx=0x7fc0e0005830 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7fe7d2fd99fcc2de server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.907+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0f0039070 con 0x7fc0e0003a70 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.907+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc0f002c6b0 con 0x7fc0e0003a70 2024-03-06T07:26:50.912 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.907+0000 7fc0ffe00640 1 -- 172.21.15.194:0/809878974 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0f0009690 con 0x7fc0e0003a70 2024-03-06T07:26:50.913 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.908+0000 7fc107800640 1 -- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 msgr2=0x7fc0e0003e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.913 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.908+0000 7fc107800640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc0e0003e90 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fc0f0009ac0 tx=0x7fc0e0005830 comp rx=0 tx=0).stop 2024-03-06T07:26:50.913 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.908+0000 7fc107800640 1 -- 172.21.15.194:0/809878974 shutdown_connections 2024-03-06T07:26:50.913 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.908+0000 7fc107800640 1 --2- 172.21.15.194:0/809878974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc0e0003e90 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.913 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.908+0000 7fc107800640 1 -- 172.21.15.194:0/809878974 >> 172.21.15.194:0/809878974 conn(0x7fc10006b3e0 msgr2=0x7fc10006b7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:50.913 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.908+0000 7fc107800640 1 -- 172.21.15.194:0/809878974 shutdown_connections 2024-03-06T07:26:50.913 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.908+0000 7fc107800640 1 -- 172.21.15.194:0/809878974 wait complete. 2024-03-06T07:26:50.914 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.910+0000 7fc107800640 1 Processor -- start 2024-03-06T07:26:50.914 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.910+0000 7fc107800640 1 -- start start 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc107800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc10015d4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc107800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc10015da20 0x7fc1000b07b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc107800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1001599b0 0x7fc1000b0cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc107800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1000b4b50 con 0x7fc1001599b0 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc107800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1001459d0 con 0x7fc0e0003a70 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc107800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc1000b4cd0 con 0x7fc10015da20 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc104e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc10015d4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc104e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc10015d4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.194:43118/0 (socket says 172.21.15.194:43118) 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc104e00640 1 -- 172.21.15.194:0/2813689614 learned_addr learned my addr 172.21.15.194:0/2813689614 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.911+0000 7fc0f7e00640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc10015da20 0x7fc1000b07b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc105800640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1001599b0 0x7fc1000b0cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc104e00640 1 -- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc10015da20 msgr2=0x7fc1000b07b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.916 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc104e00640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc10015da20 0x7fc1000b07b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc104e00640 1 -- 172.21.15.194:0/2813689614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1001599b0 msgr2=0x7fc1000b0cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc104e00640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1001599b0 0x7fc1000b0cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc104e00640 1 -- 172.21.15.194:0/2813689614 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc10015dfa0 con 0x7fc0e0003a70 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc105800640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1001599b0 0x7fc1000b0cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc104e00640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc10015d4e0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fc0f0037040 tx=0x7fc0f0004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc0fd600640 1 -- 172.21.15.194:0/2813689614 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0f0039050 con 0x7fc0e0003a70 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc0fd600640 1 -- 172.21.15.194:0/2813689614 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc0f0045070 con 0x7fc0e0003a70 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc0fd600640 1 -- 172.21.15.194:0/2813689614 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0f00044e0 con 0x7fc0e0003a70 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.912+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc10015e280 con 0x7fc0e0003a70 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.913+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc10015e840 con 0x7fc0e0003a70 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.913+0000 7fc0fd600640 1 -- 172.21.15.194:0/2813689614 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7fc0f0003a50 con 0x7fc0e0003a70 2024-03-06T07:26:50.917 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.913+0000 7fc0fd600640 1 -- 172.21.15.194:0/2813689614 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(8..8 src has 1..8) v4 ==== 1415+0+0 (secure 0 0 0) 0x7fc0f002cca0 con 0x7fc0e0003a70 2024-03-06T07:26:50.918 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.914+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc10004f3a0 con 0x7fc0e0003a70 2024-03-06T07:26:50.918 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.916+0000 7fc0fd600640 1 -- 172.21.15.194:0/2813689614 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7fc0f004f020 con 0x7fc0e0003a70 2024-03-06T07:26:50.994 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:50.992+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "osd new", "uuid": "70fa6d13-7e1a-4720-9009-24c3c629e40f", "id": 6} v 0) v1 -- 0x7fc100136b40 con 0x7fc0e0003a70 2024-03-06T07:26:51.004 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.001+0000 7fc0fd600640 1 -- 172.21.15.194:0/2813689614 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "70fa6d13-7e1a-4720-9009-24c3c629e40f", "id": 6}]=0 v9) v1 ==== 112+0+2 (secure 0 0 0) 0x7fc0f002c6b0 con 0x7fc0e0003a70 2024-03-06T07:26:51.004 INFO:teuthology.orchestra.run.smithi194.stdout:6 2024-03-06T07:26:51.005 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 msgr2=0x7fc10015d4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc10015d4e0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fc0f0037040 tx=0x7fc0f0004730 comp rx=0 tx=0).stop 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 shutdown_connections 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1001599b0 0x7fc1000b0cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc10015da20 0x7fc1000b07b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 --2- 172.21.15.194:0/2813689614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0e0003a70 0x7fc10015d4e0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 >> 172.21.15.194:0/2813689614 conn(0x7fc10006b3e0 msgr2=0x7fc10006d750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 shutdown_connections 2024-03-06T07:26:51.006 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.003+0000 7fc107800640 1 -- 172.21.15.194:0/2813689614 wait complete. 2024-03-06T07:26:51.006 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:51.004+0000 7fce54073040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-06T07:26:51.008 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:51.006+0000 7fc784c2f040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-06T07:26:51.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo ceph --cluster ceph osd new c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5 7 2024-03-06T07:26:51.057 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:51.055+0000 7fc784c2f040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-06T07:26:51.193 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:51.190+0000 7fce54073040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-06T07:26:51.229 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:51.227+0000 7fc784c2f040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-06T07:26:51.245 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:51.242+0000 7fce54073040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-06T07:26:51.256 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.253+0000 7f9b8c400640 1 Processor -- start 2024-03-06T07:26:51.256 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.253+0000 7f9b8c400640 1 -- start start 2024-03-06T07:26:51.258 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8c400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9b8415e240 0x7f9b840d1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.258 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8c400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b840b3e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.258 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8c400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b840cfc30 con 0x7f9b840d4950 2024-03-06T07:26:51.258 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8c400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9b840b34e0 con 0x7f9b8415e240 2024-03-06T07:26:51.258 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8c400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b840cfed0 con 0x7f9b840d0e70 2024-03-06T07:26:51.258 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8a400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b840b3e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.259 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8a400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b840b3e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41540/0 (socket says 172.21.15.194:41540) 2024-03-06T07:26:51.259 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b8a400640 1 -- 172.21.15.194:0/4220178803 learned_addr learned my addr 172.21.15.194:0/4220178803 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:51.259 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.255+0000 7f9b89a00640 1 --2- 172.21.15.194:0/4220178803 >> v2:172.21.15.86:3300/0 conn(0x7f9b8415e240 0x7f9b840d1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.259 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.256+0000 7f9b8a400640 1 -- 172.21.15.194:0/4220178803 >> v2:172.21.15.86:3300/0 conn(0x7f9b8415e240 msgr2=0x7f9b840d1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:51.259 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.256+0000 7f9b8a400640 1 --2- 172.21.15.194:0/4220178803 >> v2:172.21.15.86:3300/0 conn(0x7f9b8415e240 0x7f9b840d1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.259 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.256+0000 7f9b8a400640 1 -- 172.21.15.194:0/4220178803 >> v1:172.21.15.86:6789/0 conn(0x7f9b840d4950 legacy=0x7f9b840d0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:51.259 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.256+0000 7f9b8a400640 1 -- 172.21.15.194:0/4220178803 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b840d01b0 con 0x7f9b840d0e70 2024-03-06T07:26:51.260 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.257+0000 7f9b8a400640 1 --2- 172.21.15.194:0/4220178803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b840b3e30 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f9b7c009e50 tx=0x7f9b7c02bed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a020b4ad85ebb418 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:51.260 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.257+0000 7f9b83e00640 1 -- 172.21.15.194:0/4220178803 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b7c02ca70 con 0x7f9b840d0e70 2024-03-06T07:26:51.260 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.257+0000 7f9b83e00640 1 -- 172.21.15.194:0/4220178803 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b7c02cc10 con 0x7f9b840d0e70 2024-03-06T07:26:51.260 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.257+0000 7f9b83e00640 1 -- 172.21.15.194:0/4220178803 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b7c0343f0 con 0x7f9b840d0e70 2024-03-06T07:26:51.260 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 -- 172.21.15.194:0/4220178803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 msgr2=0x7f9b840b3e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.261 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 --2- 172.21.15.194:0/4220178803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b840b3e30 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f9b7c009e50 tx=0x7f9b7c02bed0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.261 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 -- 172.21.15.194:0/4220178803 shutdown_connections 2024-03-06T07:26:51.261 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 --2- 172.21.15.194:0/4220178803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b840b3e30 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.261 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 --2- 172.21.15.194:0/4220178803 >> v2:172.21.15.86:3300/0 conn(0x7f9b8415e240 0x7f9b840d1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.261 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 -- 172.21.15.194:0/4220178803 >> 172.21.15.194:0/4220178803 conn(0x7f9b840d3e30 msgr2=0x7f9b840d4240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:51.261 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 -- 172.21.15.194:0/4220178803 shutdown_connections 2024-03-06T07:26:51.261 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.258+0000 7f9b8c400640 1 -- 172.21.15.194:0/4220178803 wait complete. 2024-03-06T07:26:51.262 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.259+0000 7f9b8c400640 1 Processor -- start 2024-03-06T07:26:51.262 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.259+0000 7f9b8c400640 1 -- start start 2024-03-06T07:26:51.263 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.260+0000 7f9b8c400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b8415ed50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.260+0000 7f9b8c400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b840d4950 0x7f9b84161110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.260+0000 7f9b8c400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b8415e240 0x7f9b840b9310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.260+0000 7f9b8c400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b8415e7a0 con 0x7f9b840d4950 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.260+0000 7f9b8c400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b8415e620 con 0x7f9b8415e240 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.260+0000 7f9b8c400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b840b4370 con 0x7f9b840d0e70 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b8415ed50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b8a400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b8415e240 0x7f9b840b9310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b8415ed50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.194:41546/0 (socket says 172.21.15.194:41546) 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 -- 172.21.15.194:0/1420353014 learned_addr learned my addr 172.21.15.194:0/1420353014 (peer_addr_for_me v2:172.21.15.194:0/0) 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 -- 172.21.15.194:0/1420353014 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b8415e240 msgr2=0x7f9b840b9310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 --2- 172.21.15.194:0/1420353014 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b8415e240 0x7f9b840b9310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 -- 172.21.15.194:0/1420353014 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b840d4950 msgr2=0x7f9b84161110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 --2- 172.21.15.194:0/1420353014 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b840d4950 0x7f9b84161110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.264 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 -- 172.21.15.194:0/1420353014 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b840c7930 con 0x7f9b840d0e70 2024-03-06T07:26:51.265 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b89a00640 1 --2- 172.21.15.194:0/1420353014 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b8415ed50 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f9b7800d5f0 tx=0x7f9b7800dab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:51.265 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.261+0000 7f9b81600640 1 -- 172.21.15.194:0/1420353014 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b780125f0 con 0x7f9b840d0e70 2024-03-06T07:26:51.266 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.262+0000 7f9b81600640 1 -- 172.21.15.194:0/1420353014 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b78019070 con 0x7f9b840d0e70 2024-03-06T07:26:51.266 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.262+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b840c7c10 con 0x7f9b840d0e70 2024-03-06T07:26:51.266 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.262+0000 7f9b81600640 1 -- 172.21.15.194:0/1420353014 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b780129b0 con 0x7f9b840d0e70 2024-03-06T07:26:51.266 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.262+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b840b9850 con 0x7f9b840d0e70 2024-03-06T07:26:51.266 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.262+0000 7f9b81600640 1 -- 172.21.15.194:0/1420353014 <== mon.2 v2:172.21.15.194:3301/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f9b78012b50 con 0x7f9b840d0e70 2024-03-06T07:26:51.266 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.262+0000 7f9b81600640 1 -- 172.21.15.194:0/1420353014 <== mon.2 v2:172.21.15.194:3301/0 5 ==== osd_map(9..9 src has 1..9) v4 ==== 1530+0+0 (secure 0 0 0) 0x7f9b7801eac0 con 0x7f9b840d0e70 2024-03-06T07:26:51.266 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.263+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b8404f3a0 con 0x7f9b840d0e70 2024-03-06T07:26:51.269 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.267+0000 7f9b81600640 1 -- 172.21.15.194:0/1420353014 <== mon.2 v2:172.21.15.194:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f9b7801e460 con 0x7f9b840d0e70 2024-03-06T07:26:51.348 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.345+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "osd new", "uuid": "c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5", "id": 7} v 0) v1 -- 0x7f9b840d44f0 con 0x7f9b840d0e70 2024-03-06T07:26:51.355 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.353+0000 7f9b81600640 1 -- 172.21.15.194:0/1420353014 <== mon.2 v2:172.21.15.194:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5", "id": 7}]=0 v10) v1 ==== 112+0+2 (secure 0 0 0) 0x7f9b7801e640 con 0x7f9b840d0e70 2024-03-06T07:26:51.355 INFO:teuthology.orchestra.run.smithi194.stdout:7 2024-03-06T07:26:51.356 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 msgr2=0x7f9b8415ed50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 --2- 172.21.15.194:0/1420353014 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b8415ed50 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f9b7800d5f0 tx=0x7f9b7800dab0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 shutdown_connections 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 --2- 172.21.15.194:0/1420353014 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b8415e240 0x7f9b840b9310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 --2- 172.21.15.194:0/1420353014 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b840d4950 0x7f9b84161110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 --2- 172.21.15.194:0/1420353014 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b840d0e70 0x7f9b8415ed50 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 >> 172.21.15.194:0/1420353014 conn(0x7f9b840d3e30 msgr2=0x7f9b8415d510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 shutdown_connections 2024-03-06T07:26:51.357 INFO:teuthology.orchestra.run.smithi194.stderr:2024-03-06T07:26:51.354+0000 7f9b8c400640 1 -- 172.21.15.194:0/1420353014 wait complete. 2024-03-06T07:26:51.372 INFO:tasks.ceph.osd.0:Restarting daemon 2024-03-06T07:26:51.372 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-03-06T07:26:51.380 INFO:tasks.ceph.osd.0:Started 2024-03-06T07:26:51.380 INFO:tasks.ceph.osd.1:Restarting daemon 2024-03-06T07:26:51.380 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-03-06T07:26:51.387 INFO:tasks.ceph.osd.1:Started 2024-03-06T07:26:51.387 INFO:tasks.ceph.osd.2:Restarting daemon 2024-03-06T07:26:51.387 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-03-06T07:26:51.392 INFO:tasks.ceph.osd.2:Started 2024-03-06T07:26:51.392 INFO:tasks.ceph.osd.3:Restarting daemon 2024-03-06T07:26:51.392 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-03-06T07:26:51.398 INFO:tasks.ceph.osd.3:Started 2024-03-06T07:26:51.398 INFO:tasks.ceph.osd.4:Restarting daemon 2024-03-06T07:26:51.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 4 2024-03-06T07:26:51.405 INFO:tasks.ceph.osd.4:Started 2024-03-06T07:26:51.405 INFO:tasks.ceph.osd.5:Restarting daemon 2024-03-06T07:26:51.405 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 5 2024-03-06T07:26:51.410 INFO:tasks.ceph.osd.5:Started 2024-03-06T07:26:51.410 INFO:tasks.ceph.osd.6:Restarting daemon 2024-03-06T07:26:51.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 6 2024-03-06T07:26:51.415 INFO:tasks.ceph.osd.6:Started 2024-03-06T07:26:51.416 INFO:tasks.ceph.osd.7:Restarting daemon 2024-03-06T07:26:51.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 7 2024-03-06T07:26:51.420 INFO:tasks.ceph.osd.7:Started 2024-03-06T07:26:51.421 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:26:51.440 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:51.437+0000 7fce54073040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-06T07:26:51.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.629+0000 7f4e31000640 1 Processor -- start 2024-03-06T07:26:51.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.629+0000 7f4e31000640 1 -- start start 2024-03-06T07:26:51.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e31000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4e2c0d2360 0x7f4e2c0d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e31000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4e2c0b3ec0 0x7f4e2c0b4310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e31000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4e2c161960 con 0x7f4e2c161580 2024-03-06T07:26:51.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e31000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4e2c0d2ce0 con 0x7f4e2c0b3ec0 2024-03-06T07:26:51.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e31000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4e2c0cf6f0 con 0x7f4e2c0d2360 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e2be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4e2c0b3ec0 0x7f4e2c0b4310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e2be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4e2c0b3ec0 0x7f4e2c0b4310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36440/0 (socket says 172.21.15.86:36440) 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e2be00640 1 -- 172.21.15.86:0/2852965964 learned_addr learned my addr 172.21.15.86:0/2852965964 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.630+0000 7f4e22a00640 1 --2- 172.21.15.86:0/2852965964 >> v2:172.21.15.194:3300/0 conn(0x7f4e2c0d2360 0x7f4e2c0d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.631+0000 7f4e2be00640 1 -- 172.21.15.86:0/2852965964 >> v2:172.21.15.194:3300/0 conn(0x7f4e2c0d2360 msgr2=0x7f4e2c0d27a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.631+0000 7f4e2be00640 1 --2- 172.21.15.86:0/2852965964 >> v2:172.21.15.194:3300/0 conn(0x7f4e2c0d2360 0x7f4e2c0d27a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.631+0000 7f4e2be00640 1 -- 172.21.15.86:0/2852965964 >> v1:172.21.15.194:6789/0 conn(0x7f4e2c161580 legacy=0x7f4e2c0d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.631+0000 7f4e2be00640 1 -- 172.21.15.86:0/2852965964 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e2c0cf9d0 con 0x7f4e2c0b3ec0 2024-03-06T07:26:51.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.632+0000 7f4e2be00640 1 --2- 172.21.15.86:0/2852965964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0b3ec0 0x7f4e2c0b4310 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f4e2c0d2e60 tx=0x7f4e2402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f0b2de4b1b786bb6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:51.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.632+0000 7f4e2aa00640 1 -- 172.21.15.86:0/2852965964 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e24034070 con 0x7f4e2c0b3ec0 2024-03-06T07:26:51.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.632+0000 7f4e2aa00640 1 -- 172.21.15.86:0/2852965964 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4e24030860 con 0x7f4e2c0b3ec0 2024-03-06T07:26:51.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.632+0000 7f4e2aa00640 1 -- 172.21.15.86:0/2852965964 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e24030c30 con 0x7f4e2c0b3ec0 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.632+0000 7f4e31000640 1 -- 172.21.15.86:0/2852965964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0b3ec0 msgr2=0x7f4e2c0b4310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.632+0000 7f4e31000640 1 --2- 172.21.15.86:0/2852965964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0b3ec0 0x7f4e2c0b4310 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f4e2c0d2e60 tx=0x7f4e2402b9a0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.633+0000 7f4e31000640 1 -- 172.21.15.86:0/2852965964 shutdown_connections 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.633+0000 7f4e31000640 1 --2- 172.21.15.86:0/2852965964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0b3ec0 0x7f4e2c0b4310 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.633+0000 7f4e31000640 1 --2- 172.21.15.86:0/2852965964 >> v2:172.21.15.194:3300/0 conn(0x7f4e2c0d2360 0x7f4e2c0d27a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.633+0000 7f4e31000640 1 -- 172.21.15.86:0/2852965964 >> 172.21.15.86:0/2852965964 conn(0x7f4e2c0d3d80 msgr2=0x7f4e2c0d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.633+0000 7f4e31000640 1 -- 172.21.15.86:0/2852965964 shutdown_connections 2024-03-06T07:26:51.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.633+0000 7f4e31000640 1 -- 172.21.15.86:0/2852965964 wait complete. 2024-03-06T07:26:51.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.634+0000 7f4e31000640 1 Processor -- start 2024-03-06T07:26:51.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.635+0000 7f4e31000640 1 -- start start 2024-03-06T07:26:51.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.635+0000 7f4e31000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e2c0b3ec0 0x7f4e2c0cd070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.635+0000 7f4e31000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0d2360 0x7f4e2c1f3140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.635+0000 7f4e31000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e2c161580 0x7f4e2c0cb030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:51.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.635+0000 7f4e31000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e2c0b39a0 con 0x7f4e2c0d2360 2024-03-06T07:26:51.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.635+0000 7f4e31000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e2c0b36c0 con 0x7f4e2c0b3ec0 2024-03-06T07:26:51.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.635+0000 7f4e31000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4e2c0cf3a0 con 0x7f4e2c161580 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0d2360 0x7f4e2c1f3140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0d2360 0x7f4e2c1f3140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36456/0 (socket says 172.21.15.86:36456) 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 -- 172.21.15.86:0/1945731070 learned_addr learned my addr 172.21.15.86:0/1945731070 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e2b400640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e2c0b3ec0 0x7f4e2c0cd070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e2be00640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e2c161580 0x7f4e2c0cb030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 -- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e2c161580 msgr2=0x7f4e2c0cb030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e2c161580 0x7f4e2c0cb030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 -- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e2c0b3ec0 msgr2=0x7f4e2c0cd070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e2c0b3ec0 0x7f4e2c0cd070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 -- 172.21.15.86:0/1945731070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e2c160340 con 0x7f4e2c0d2360 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.636+0000 7f4e22a00640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0d2360 0x7f4e2c1f3140 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f4e1400e870 tx=0x7f4e1400ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.637+0000 7f4e23e00640 1 -- 172.21.15.86:0/1945731070 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e14017070 con 0x7f4e2c0d2360 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.637+0000 7f4e23e00640 1 -- 172.21.15.86:0/1945731070 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4e1400c940 con 0x7f4e2c0d2360 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.637+0000 7f4e23e00640 1 -- 172.21.15.86:0/1945731070 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e1400cd30 con 0x7f4e2c0d2360 2024-03-06T07:26:51.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.637+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4e2c0cd5b0 con 0x7f4e2c0d2360 2024-03-06T07:26:51.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.637+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4e2c0cdb50 con 0x7f4e2c0d2360 2024-03-06T07:26:51.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.637+0000 7f4e23e00640 1 -- 172.21.15.86:0/1945731070 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f4e1401a070 con 0x7f4e2c0d2360 2024-03-06T07:26:51.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.637+0000 7f4e23e00640 1 -- 172.21.15.86:0/1945731070 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(10..10 src has 1..10) v4 ==== 1645+0+0 (secure 0 0 0) 0x7f4e14006020 con 0x7f4e2c0d2360 2024-03-06T07:26:51.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.638+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4e2c04f330 con 0x7f4e2c0d2360 2024-03-06T07:26:51.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.640+0000 7f4e23e00640 1 -- 172.21.15.86:0/1945731070 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f4e14006480 con 0x7f4e2c0d2360 2024-03-06T07:26:51.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.705+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4e2c0d3b60 con 0x7f4e2c0d2360 2024-03-06T07:26:51.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.705+0000 7f4e23e00640 1 -- 172.21.15.86:0/1945731070 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v10) v1 ==== 74+0+7058 (secure 0 0 0) 0x7f4e14006660 con 0x7f4e2c0d2360 2024-03-06T07:26:51.707 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:26:51.707 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":10,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:26:51.348822+0000","last_up_change":"0.000000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:26:51.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.706+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0d2360 msgr2=0x7f4e2c1f3140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:51.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.706+0000 7f4e31000640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0d2360 0x7f4e2c1f3140 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f4e1400e870 tx=0x7f4e1400ed30 comp rx=0 tx=0).stop 2024-03-06T07:26:51.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.707+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 shutdown_connections 2024-03-06T07:26:51.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.707+0000 7f4e31000640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e2c161580 0x7f4e2c0cb030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.707+0000 7f4e31000640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e2c0d2360 0x7f4e2c1f3140 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.707+0000 7f4e31000640 1 --2- 172.21.15.86:0/1945731070 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e2c0b3ec0 0x7f4e2c0cd070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:51.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.707+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 >> 172.21.15.86:0/1945731070 conn(0x7f4e2c0d3d80 msgr2=0x7f4e2c06cf00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:51.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.707+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 shutdown_connections 2024-03-06T07:26:51.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:26:51.707+0000 7f4e31000640 1 -- 172.21.15.86:0/1945731070 wait complete. 2024-03-06T07:26:51.725 INFO:tasks.ceph.ceph_manager.ceph:[] 2024-03-06T07:26:51.726 INFO:tasks.ceph:Waiting for OSDs to come up 2024-03-06T07:26:51.908 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:26:51.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:26:52.079 INFO:tasks.ceph.osd.0.smithi086.stderr:2024-03-06T07:26:52.078+0000 7f1c71a11640 -1 Falling back to public interface 2024-03-06T07:26:52.092 INFO:tasks.ceph.osd.4.smithi194.stderr:2024-03-06T07:26:52.090+0000 7fbe7c8d3640 -1 Falling back to public interface 2024-03-06T07:26:52.096 INFO:tasks.ceph.osd.6.smithi194.stderr:2024-03-06T07:26:52.093+0000 7f2b96924640 -1 Falling back to public interface 2024-03-06T07:26:52.096 INFO:tasks.ceph.osd.1.smithi086.stderr:2024-03-06T07:26:52.095+0000 7fe55a0f9640 -1 Falling back to public interface 2024-03-06T07:26:52.096 INFO:tasks.ceph.osd.2.smithi086.stderr:2024-03-06T07:26:52.095+0000 7f24cd71d640 -1 Falling back to public interface 2024-03-06T07:26:52.097 INFO:tasks.ceph.osd.1.smithi086.stderr:2024-03-06T07:26:52.096+0000 7fe558c00640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6808/0: (98) Address already in use 2024-03-06T07:26:52.115 INFO:tasks.ceph.osd.5.smithi194.stderr:2024-03-06T07:26:52.113+0000 7f5c65ed7640 -1 Falling back to public interface 2024-03-06T07:26:52.119 INFO:tasks.ceph.osd.7.smithi194.stderr:2024-03-06T07:26:52.117+0000 7f10b7e0f640 -1 Falling back to public interface 2024-03-06T07:26:52.119 INFO:tasks.ceph.osd.3.smithi086.stderr:2024-03-06T07:26:52.118+0000 7ff3a5811640 -1 Falling back to public interface 2024-03-06T07:26:52.312 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:52.310+0000 7fc784c2f040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-06T07:26:52.531 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-06T07:26:52.588 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:52.585+0000 7fce54073040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-06T07:26:52.629 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:52.627+0000 7fc784c2f040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-06T07:26:52.683 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.681+0000 7fb54b000640 1 Processor -- start 2024-03-06T07:26:52.683 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.681+0000 7fb54b000640 1 -- start start 2024-03-06T07:26:52.685 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54b000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb54406be70 0x7fb5440af3f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:52.685 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54b000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb5440cc7d0 0x7fb5440b1f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:52.685 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54b000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5440620b0 con 0x7fb54415c000 2024-03-06T07:26:52.685 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54b000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5440b1ad0 con 0x7fb5440cc7d0 2024-03-06T07:26:52.685 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54b000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5440b14c0 con 0x7fb54406be70 2024-03-06T07:26:52.685 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54a600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb5440cc7d0 0x7fb5440b1f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:52.685 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54a600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb5440cc7d0 0x7fb5440b1f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36468/0 (socket says 172.21.15.86:36468) 2024-03-06T07:26:52.686 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb54a600640 1 -- 172.21.15.86:0/879134991 learned_addr learned my addr 172.21.15.86:0/879134991 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:26:52.686 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.683+0000 7fb549200640 1 --2- 172.21.15.86:0/879134991 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb54406be70 0x7fb5440af3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:52.686 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.684+0000 7fb54a600640 1 -- 172.21.15.86:0/879134991 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb54406be70 msgr2=0x7fb5440af3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:52.686 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.684+0000 7fb54a600640 1 --2- 172.21.15.86:0/879134991 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb54406be70 0x7fb5440af3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.686 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.684+0000 7fb54a600640 1 -- 172.21.15.86:0/879134991 >> v1:172.21.15.86:6789/0 conn(0x7fb54415c000 legacy=0x7fb54406b760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:52.686 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.684+0000 7fb54a600640 1 -- 172.21.15.86:0/879134991 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb544061dd0 con 0x7fb5440cc7d0 2024-03-06T07:26:52.686 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.685+0000 7fb54a600640 1 --2- 172.21.15.86:0/879134991 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5440cc7d0 0x7fb5440b1f60 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fb53c009e50 tx=0x7fb53c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b238aae083d50a65 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:52.687 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.685+0000 7fb543e00640 1 -- 172.21.15.86:0/879134991 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb53c02ca70 con 0x7fb5440cc7d0 2024-03-06T07:26:52.687 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb543e00640 1 -- 172.21.15.86:0/879134991 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb53c02cc10 con 0x7fb5440cc7d0 2024-03-06T07:26:52.687 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb543e00640 1 -- 172.21.15.86:0/879134991 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb53c0343f0 con 0x7fb5440cc7d0 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 -- 172.21.15.86:0/879134991 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5440cc7d0 msgr2=0x7fb5440b1f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 --2- 172.21.15.86:0/879134991 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5440cc7d0 0x7fb5440b1f60 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fb53c009e50 tx=0x7fb53c02bed0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 -- 172.21.15.86:0/879134991 shutdown_connections 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 --2- 172.21.15.86:0/879134991 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5440cc7d0 0x7fb5440b1f60 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 --2- 172.21.15.86:0/879134991 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb54406be70 0x7fb5440af3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 -- 172.21.15.86:0/879134991 >> 172.21.15.86:0/879134991 conn(0x7fb544071710 msgr2=0x7fb5440b5060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 -- 172.21.15.86:0/879134991 shutdown_connections 2024-03-06T07:26:52.688 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.686+0000 7fb54b000640 1 -- 172.21.15.86:0/879134991 wait complete. 2024-03-06T07:26:52.689 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.688+0000 7fb54b000640 1 Processor -- start 2024-03-06T07:26:52.689 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.688+0000 7fb54b000640 1 -- start start 2024-03-06T07:26:52.692 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb54b000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb54406be70 0x7fb5440cb3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:52.692 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb54b000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5440cc7d0 0x7fb54415cda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:52.692 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb54b000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb54415c000 0x7fb54414b4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:52.692 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb54b000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb544061b90 con 0x7fb54406be70 2024-03-06T07:26:52.692 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb54b000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb544061a10 con 0x7fb54415c000 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb54b000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb54406b2a0 con 0x7fb5440cc7d0 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb549c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb54406be70 0x7fb5440cb3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb549c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb54406be70 0x7fb5440cb3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36472/0 (socket says 172.21.15.86:36472) 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb549c00640 1 -- 172.21.15.86:0/2447637179 learned_addr learned my addr 172.21.15.86:0/2447637179 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb549200640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5440cc7d0 0x7fb54415cda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.689+0000 7fb54a600640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb54415c000 0x7fb54414b4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb549c00640 1 -- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5440cc7d0 msgr2=0x7fb54415cda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb549c00640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5440cc7d0 0x7fb54415cda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb549c00640 1 -- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb54415c000 msgr2=0x7fb54414b4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb549c00640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb54415c000 0x7fb54414b4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb549c00640 1 -- 172.21.15.86:0/2447637179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb54415d5d0 con 0x7fb54406be70 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb549c00640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb54406be70 0x7fb5440cb3b0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fb53800c3c0 tx=0x7fb53800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb541600640 1 -- 172.21.15.86:0/2447637179 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5380052d0 con 0x7fb54406be70 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb541600640 1 -- 172.21.15.86:0/2447637179 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb538005490 con 0x7fb54406be70 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb541600640 1 -- 172.21.15.86:0/2447637179 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb538007260 con 0x7fb54406be70 2024-03-06T07:26:52.693 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb54415d8b0 con 0x7fb54406be70 2024-03-06T07:26:52.694 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb54415de70 con 0x7fb54406be70 2024-03-06T07:26:52.694 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.690+0000 7fb541600640 1 -- 172.21.15.86:0/2447637179 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7fb538007400 con 0x7fb54406be70 2024-03-06T07:26:52.694 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.691+0000 7fb541600640 1 -- 172.21.15.86:0/2447637179 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(10..10 src has 1..10) v4 ==== 1645+0+0 (secure 0 0 0) 0x7fb538003a00 con 0x7fb54406be70 2024-03-06T07:26:52.694 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.691+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5440684e0 con 0x7fb54406be70 2024-03-06T07:26:52.694 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:52.693+0000 7fc784c2f040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-06T07:26:52.695 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.694+0000 7fb541600640 1 -- 172.21.15.86:0/2447637179 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7fb5380035f0 con 0x7fb54406be70 2024-03-06T07:26:52.762 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.761+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fb54406e750 con 0x7fb54406be70 2024-03-06T07:26:52.763 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.762+0000 7fb541600640 1 -- 172.21.15.86:0/2447637179 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v10) v1 ==== 74+0+7058 (secure 0 0 0) 0x7fb538003d20 con 0x7fb54406be70 2024-03-06T07:26:52.763 INFO:teuthology.misc.health.smithi086.stdout: 2024-03-06T07:26:52.763 INFO:teuthology.misc.health.smithi086.stdout:{"epoch":10,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:26:51.348822+0000","last_up_change":"0.000000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:26:52.764 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb54406be70 msgr2=0x7fb5440cb3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:52.764 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb54406be70 0x7fb5440cb3b0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fb53800c3c0 tx=0x7fb53800c880 comp rx=0 tx=0).stop 2024-03-06T07:26:52.765 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 shutdown_connections 2024-03-06T07:26:52.765 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb54415c000 0x7fb54414b4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.765 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5440cc7d0 0x7fb54415cda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.765 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 --2- 172.21.15.86:0/2447637179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb54406be70 0x7fb5440cb3b0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:52.765 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 >> 172.21.15.86:0/2447637179 conn(0x7fb544071710 msgr2=0x7fb54406e9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:52.765 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 shutdown_connections 2024-03-06T07:26:52.765 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:52.763+0000 7fb54b000640 1 -- 172.21.15.86:0/2447637179 wait complete. 2024-03-06T07:26:52.777 DEBUG:teuthology.misc:0 of 8 OSDs are up 2024-03-06T07:26:52.830 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:52.829+0000 7fc784c2f040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-06T07:26:52.911 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:52.908+0000 7fce54073040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-06T07:26:52.973 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:52.971+0000 7fce54073040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-06T07:26:53.103 INFO:tasks.ceph.mgr.x.smithi194.stderr:2024-03-06T07:26:53.100+0000 7fce54073040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-06T07:26:53.877 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:53.876+0000 7fc724e00640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-03-06T07:26:53.880 INFO:tasks.ceph.mgr.y.smithi086.stderr:2024-03-06T07:26:53.878+0000 7fc724e00640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-06T07:26:55.613 INFO:tasks.ceph.osd.0.smithi086.stderr:2024-03-06T07:26:55.612+0000 7f1c71a11640 -1 osd.0 0 log_to_monitors true 2024-03-06T07:26:55.639 INFO:tasks.ceph.osd.2.smithi086.stderr:2024-03-06T07:26:55.637+0000 7f24cd71d640 -1 osd.2 0 log_to_monitors true 2024-03-06T07:26:55.667 INFO:tasks.ceph.osd.4.smithi194.stderr:2024-03-06T07:26:55.665+0000 7fbe7c8d3640 -1 osd.4 0 log_to_monitors true 2024-03-06T07:26:55.695 INFO:tasks.ceph.osd.1.smithi086.stderr:2024-03-06T07:26:55.694+0000 7fe55a0f9640 -1 osd.1 0 log_to_monitors true 2024-03-06T07:26:55.718 INFO:tasks.ceph.osd.7.smithi194.stderr:2024-03-06T07:26:55.716+0000 7f10b7e0f640 -1 osd.7 0 log_to_monitors true 2024-03-06T07:26:55.722 INFO:tasks.ceph.osd.3.smithi086.stderr:2024-03-06T07:26:55.721+0000 7ff3a5811640 -1 osd.3 0 log_to_monitors true 2024-03-06T07:26:55.912 INFO:tasks.ceph.osd.6.smithi194.stderr:2024-03-06T07:26:55.909+0000 7f2b96924640 -1 osd.6 0 log_to_monitors true 2024-03-06T07:26:55.970 INFO:tasks.ceph.osd.5.smithi194.stderr:2024-03-06T07:26:55.967+0000 7f5c65ed7640 -1 osd.5 0 log_to_monitors true 2024-03-06T07:26:58.575 INFO:tasks.ceph.osd.7.smithi194.stderr:2024-03-06T07:26:58.572+0000 7f10b1c00640 -1 osd.7 0 waiting for initial osdmap 2024-03-06T07:26:58.590 INFO:tasks.ceph.osd.4.smithi194.stderr:2024-03-06T07:26:58.588+0000 7fbe76e00640 -1 osd.4 0 waiting for initial osdmap 2024-03-06T07:26:58.598 INFO:tasks.ceph.osd.7.smithi194.stderr:2024-03-06T07:26:58.596+0000 7f10a8600640 -1 osd.7 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:58.617 INFO:tasks.ceph.osd.4.smithi194.stderr:2024-03-06T07:26:58.615+0000 7fbe6d800640 -1 osd.4 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:59.168 INFO:tasks.ceph.osd.3.smithi086.stderr:2024-03-06T07:26:59.166+0000 7ff39f800640 -1 osd.3 0 waiting for initial osdmap 2024-03-06T07:26:59.178 INFO:tasks.ceph.osd.3.smithi086.stderr:2024-03-06T07:26:59.176+0000 7ff396200640 -1 osd.3 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:59.194 INFO:tasks.ceph.osd.0.smithi086.stderr:2024-03-06T07:26:59.193+0000 7f1c6b800640 -1 osd.0 0 waiting for initial osdmap 2024-03-06T07:26:59.201 INFO:tasks.ceph.osd.0.smithi086.stderr:2024-03-06T07:26:59.200+0000 7f1c62200640 -1 osd.0 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:59.246 INFO:tasks.ceph.osd.1.smithi086.stderr:2024-03-06T07:26:59.244+0000 7fe554600640 -1 osd.1 0 waiting for initial osdmap 2024-03-06T07:26:59.253 INFO:tasks.ceph.osd.1.smithi086.stderr:2024-03-06T07:26:59.251+0000 7fe54b000640 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:59.300 INFO:tasks.ceph.osd.2.smithi086.stderr:2024-03-06T07:26:59.299+0000 7f24c7c00640 -1 osd.2 0 waiting for initial osdmap 2024-03-06T07:26:59.310 INFO:tasks.ceph.osd.2.smithi086.stderr:2024-03-06T07:26:59.308+0000 7f24be600640 -1 osd.2 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:59.493 INFO:tasks.ceph.osd.5.smithi194.stderr:2024-03-06T07:26:59.490+0000 7f5c60400640 -1 osd.5 0 waiting for initial osdmap 2024-03-06T07:26:59.501 INFO:tasks.ceph.osd.5.smithi194.stderr:2024-03-06T07:26:59.498+0000 7f5c56e00640 -1 osd.5 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:59.548 INFO:tasks.ceph.osd.6.smithi194.stderr:2024-03-06T07:26:59.546+0000 7f2b92c00640 -1 osd.6 0 waiting for initial osdmap 2024-03-06T07:26:59.559 INFO:tasks.ceph.osd.6.smithi194.stderr:2024-03-06T07:26:59.556+0000 7f2b87800640 -1 osd.6 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-06T07:26:59.584 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-06T07:26:59.745 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.743+0000 7f033fa00640 1 Processor -- start 2024-03-06T07:26:59.745 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.743+0000 7f033fa00640 1 -- start start 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033fa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0338155480 0x7f0338071450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033fa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f03380cbba0 0x7f03380b34c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033fa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f03380cc010 con 0x7f0338161e70 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033fa00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0338162cf0 con 0x7f0338155480 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033fa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0338071a60 con 0x7f03380cbba0 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0338155480 0x7f0338071450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0338155480 0x7f0338071450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51802/0 (socket says 172.21.15.86:51802) 2024-03-06T07:26:59.747 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f033d000640 1 -- 172.21.15.86:0/313961375 learned_addr learned my addr 172.21.15.86:0/313961375 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:26:59.748 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.745+0000 7f032fe00640 1 --2- 172.21.15.86:0/313961375 >> v2:172.21.15.194:3300/0 conn(0x7f03380cbba0 0x7f03380b34c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:59.748 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.746+0000 7f033d000640 1 -- 172.21.15.86:0/313961375 >> v2:172.21.15.194:3300/0 conn(0x7f03380cbba0 msgr2=0x7f03380b34c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:26:59.749 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.746+0000 7f033d000640 1 --2- 172.21.15.86:0/313961375 >> v2:172.21.15.194:3300/0 conn(0x7f03380cbba0 0x7f03380b34c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.749 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.746+0000 7f033d000640 1 -- 172.21.15.86:0/313961375 >> v1:172.21.15.194:6789/0 conn(0x7f0338161e70 legacy=0x7f0338162280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:59.749 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.746+0000 7f033d000640 1 -- 172.21.15.86:0/313961375 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0338071d40 con 0x7f0338155480 2024-03-06T07:26:59.749 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.747+0000 7f033d000640 1 --2- 172.21.15.86:0/313961375 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f0338071450 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f0338071990 tx=0x7f032002b9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b054ea3f6c6cf6eb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:59.749 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.747+0000 7f0337e00640 1 -- 172.21.15.86:0/313961375 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0320034070 con 0x7f0338155480 2024-03-06T07:26:59.749 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.747+0000 7f0337e00640 1 -- 172.21.15.86:0/313961375 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0320030880 con 0x7f0338155480 2024-03-06T07:26:59.749 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.747+0000 7f0337e00640 1 -- 172.21.15.86:0/313961375 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0320030c50 con 0x7f0338155480 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 -- 172.21.15.86:0/313961375 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 msgr2=0x7f0338071450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 --2- 172.21.15.86:0/313961375 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f0338071450 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f0338071990 tx=0x7f032002b9c0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 -- 172.21.15.86:0/313961375 shutdown_connections 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 --2- 172.21.15.86:0/313961375 >> v2:172.21.15.194:3300/0 conn(0x7f03380cbba0 0x7f03380b34c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 --2- 172.21.15.86:0/313961375 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f0338071450 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 -- 172.21.15.86:0/313961375 >> 172.21.15.86:0/313961375 conn(0x7f03380b16b0 msgr2=0x7f03380b1ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 -- 172.21.15.86:0/313961375 shutdown_connections 2024-03-06T07:26:59.750 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.748+0000 7f033fa00640 1 -- 172.21.15.86:0/313961375 wait complete. 2024-03-06T07:26:59.751 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.749+0000 7f033fa00640 1 Processor -- start 2024-03-06T07:26:59.751 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.750+0000 7f033fa00640 1 -- start start 2024-03-06T07:26:59.753 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.750+0000 7f033fa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03380cbba0 0x7f033815dec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:59.753 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.750+0000 7f033fa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f033815e400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:59.753 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.750+0000 7f033fa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0338161e70 0x7f033815c410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:59.753 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.750+0000 7f033fa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f033806d0f0 con 0x7f0338155480 2024-03-06T07:26:59.753 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.750+0000 7f033fa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f033806cf70 con 0x7f0338161e70 2024-03-06T07:26:59.753 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.750+0000 7f033fa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03380b3a00 con 0x7f03380cbba0 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f033815e400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f033815e400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51806/0 (socket says 172.21.15.86:51806) 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 -- 172.21.15.86:0/719828241 learned_addr learned my addr 172.21.15.86:0/719828241 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 -- 172.21.15.86:0/719828241 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03380cbba0 msgr2=0x7f033815dec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03380cbba0 0x7f033815dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 -- 172.21.15.86:0/719828241 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0338161e70 msgr2=0x7f033815c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0338161e70 0x7f033815c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 -- 172.21.15.86:0/719828241 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03380b0f30 con 0x7f0338155480 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f032fe00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f033815e400 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f032800e9e0 tx=0x7f032800eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.751+0000 7f0335600640 1 -- 172.21.15.86:0/719828241 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0328017070 con 0x7f0338155480 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.752+0000 7f0335600640 1 -- 172.21.15.86:0/719828241 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f032800c9d0 con 0x7f0338155480 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.752+0000 7f0335600640 1 -- 172.21.15.86:0/719828241 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f032800cdc0 con 0x7f0338155480 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.752+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03381e9e20 con 0x7f0338155480 2024-03-06T07:26:59.754 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.752+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03381ea3e0 con 0x7f0338155480 2024-03-06T07:26:59.755 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.753+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f033804f330 con 0x7f0338155480 2024-03-06T07:26:59.755 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.754+0000 7f0335600640 1 -- 172.21.15.86:0/719828241 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7f032801a070 con 0x7f0338155480 2024-03-06T07:26:59.756 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.754+0000 7f0335600640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f031406b470 0x7f031406d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:26:59.756 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.755+0000 7f0335600640 1 -- 172.21.15.86:0/719828241 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(14..14 src has 1..14) v4 ==== 4471+0+0 (secure 0 0 0) 0x7f0328089fe0 con 0x7f0338155480 2024-03-06T07:26:59.756 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.755+0000 7f033d000640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f031406b470 0x7f031406d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:26:59.757 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.755+0000 7f033d000640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f031406b470 0x7f031406d930 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f0320036000 tx=0x7f032003b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:26:59.761 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.759+0000 7f0335600640 1 -- 172.21.15.86:0/719828241 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f03280593f0 con 0x7f0338155480 2024-03-06T07:26:59.947 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.944+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f033806e320 con 0x7f0338155480 2024-03-06T07:26:59.949 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.946+0000 7f0335600640 1 -- 172.21.15.86:0/719828241 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v14) v1 ==== 74+0+10921 (secure 0 0 0) 0x7f0328058cf0 con 0x7f0338155480 2024-03-06T07:26:59.949 INFO:teuthology.misc.health.smithi086.stdout: 2024-03-06T07:26:59.949 INFO:teuthology.misc.health.smithi086.stdout:{"epoch":14,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:26:59.589976+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f031406b470 msgr2=0x7f031406d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f031406b470 0x7f031406d930 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f0320036000 tx=0x7f032003b040 comp rx=0 tx=0).stop 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 msgr2=0x7f033815e400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f033815e400 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f032800e9e0 tx=0x7f032800eea0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 shutdown_connections 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f031406b470 0x7f031406d930 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0338161e70 0x7f033815c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0338155480 0x7f033815e400 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 --2- 172.21.15.86:0/719828241 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03380cbba0 0x7f033815dec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 >> 172.21.15.86:0/719828241 conn(0x7f03380b16b0 msgr2=0x7f0338153d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 shutdown_connections 2024-03-06T07:26:59.952 INFO:teuthology.misc.health.smithi086.stderr:2024-03-06T07:26:59.949+0000 7f033fa00640 1 -- 172.21.15.86:0/719828241 wait complete. 2024-03-06T07:26:59.968 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-03-06T07:26:59.968 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2024-03-06T07:26:59.969 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:26:59.969 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:26:59.974 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:26:59.974 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:26:59.974 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:26:59.981 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:26:59.981 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:26:59.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:26:59.989 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:26:59.989 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:26:59.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:26:59.996 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:26:59.996 INFO:tasks.ceph:Setting up CephFS filesystem(s)... 2024-03-06T07:26:59.999 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:00.117 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:27:00.124 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:27:00.152 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:27:00.153 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:27:00.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.174+0000 7f287ee00640 1 Processor -- start 2024-03-06T07:27:00.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.174+0000 7f287ee00640 1 -- start start 2024-03-06T07:27:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.175+0000 7f287ee00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f287815e0d0 0x7f28780d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.175+0000 7f287ee00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f287815f470 0x7f28780d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.175+0000 7f287ee00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28780d1200 con 0x7f28780d0df0 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.175+0000 7f287ee00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f28780d2f60 con 0x7f287815f470 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.175+0000 7f287ee00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f28780cea60 con 0x7f287815e0d0 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f287ce00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f28780d0df0 0x7f2878160dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39614/0 (socket says 172.21.15.86:39614) 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f2877400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f287815f470 0x7f28780d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f287ce00640 1 -- 172.21.15.86:0/484132267 learned_addr learned my addr 172.21.15.86:0/484132267 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f2877e00640 1 --2- 172.21.15.86:0/484132267 >> v2:172.21.15.194:3300/0 conn(0x7f287815e0d0 0x7f28780d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f2877400640 1 -- 172.21.15.86:0/484132267 >> v2:172.21.15.194:3300/0 conn(0x7f287815e0d0 msgr2=0x7f28780d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f2877400640 1 --2- 172.21.15.86:0/484132267 >> v2:172.21.15.194:3300/0 conn(0x7f287815e0d0 0x7f28780d1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f2877400640 1 -- 172.21.15.86:0/484132267 >> v1:172.21.15.86:6789/0 conn(0x7f28780d0df0 legacy=0x7f2878160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.176+0000 7f2877400640 1 -- 172.21.15.86:0/484132267 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28780d2350 con 0x7f287815f470 2024-03-06T07:27:00.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.177+0000 7f2877400640 1 --2- 172.21.15.86:0/484132267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28780d08b0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f28780b6a80 tx=0x7f286402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f49ac23ce62529ac server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:00.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.177+0000 7f2876a00640 1 -- 172.21.15.86:0/484132267 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2864034070 con 0x7f287815f470 2024-03-06T07:27:00.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.177+0000 7f2876a00640 1 -- 172.21.15.86:0/484132267 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2864030610 con 0x7f287815f470 2024-03-06T07:27:00.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.177+0000 7f2876a00640 1 -- 172.21.15.86:0/484132267 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28640309e0 con 0x7f287815f470 2024-03-06T07:27:00.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 -- 172.21.15.86:0/484132267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 msgr2=0x7f28780d08b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 --2- 172.21.15.86:0/484132267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28780d08b0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f28780b6a80 tx=0x7f286402b960 comp rx=0 tx=0).stop 2024-03-06T07:27:00.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 -- 172.21.15.86:0/484132267 shutdown_connections 2024-03-06T07:27:00.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 --2- 172.21.15.86:0/484132267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28780d08b0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 --2- 172.21.15.86:0/484132267 >> v2:172.21.15.194:3300/0 conn(0x7f287815e0d0 0x7f28780d1a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 -- 172.21.15.86:0/484132267 >> 172.21.15.86:0/484132267 conn(0x7f28780d3ec0 msgr2=0x7f28780d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:00.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 -- 172.21.15.86:0/484132267 shutdown_connections 2024-03-06T07:27:00.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.178+0000 7f287ee00640 1 -- 172.21.15.86:0/484132267 wait complete. 2024-03-06T07:27:00.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.180+0000 7f287ee00640 1 Processor -- start 2024-03-06T07:27:00.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.180+0000 7f287ee00640 1 -- start start 2024-03-06T07:27:00.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.180+0000 7f287ee00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28780d0df0 0x7f2878159310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ee00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f287815e0d0 0x7f28780b98e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ee00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28781f43e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ee00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28780d2bf0 con 0x7f287815f470 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ee00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28780d2910 con 0x7f287815e0d0 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ee00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f287815fb30 con 0x7f28780d0df0 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28781f43e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f2877e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28780d0df0 0x7f2878159310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28781f43e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51848/0 (socket says 172.21.15.86:51848) 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 -- 172.21.15.86:0/223356188 learned_addr learned my addr 172.21.15.86:0/223356188 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f2877400640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f287815e0d0 0x7f28780b98e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 -- 172.21.15.86:0/223356188 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28780d0df0 msgr2=0x7f2878159310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28780d0df0 0x7f2878159310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 -- 172.21.15.86:0/223356188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f287815e0d0 msgr2=0x7f28780b98e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f287815e0d0 0x7f28780b98e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 -- 172.21.15.86:0/223356188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28781f4cf0 con 0x7f287815f470 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.181+0000 7f287ce00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28781f43e0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f287000cab0 tx=0x7f287000cf70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.182+0000 7f286fe00640 1 -- 172.21.15.86:0/223356188 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2870017070 con 0x7f287815f470 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.182+0000 7f286fe00640 1 -- 172.21.15.86:0/223356188 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f287000ac30 con 0x7f287815f470 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.182+0000 7f286fe00640 1 -- 172.21.15.86:0/223356188 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2870012400 con 0x7f287815f470 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.182+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28781f4fd0 con 0x7f287815f470 2024-03-06T07:27:00.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.182+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28780cc5a0 con 0x7f287815f470 2024-03-06T07:27:00.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.183+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f287804f330 con 0x7f287815f470 2024-03-06T07:27:00.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.184+0000 7f286fe00640 1 -- 172.21.15.86:0/223356188 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7f287001a070 con 0x7f287815f470 2024-03-06T07:27:00.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.184+0000 7f286fe00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f285806b440 0x7f285806d900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.185+0000 7f286fe00640 1 -- 172.21.15.86:0/223356188 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(14..14 src has 1..14) v4 ==== 4471+0+0 (secure 0 0 0) 0x7f2870089810 con 0x7f287815f470 2024-03-06T07:27:00.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.185+0000 7f2877e00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f285806b440 0x7f285806d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.185+0000 7f2877e00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f285806b440 0x7f285806d900 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f2868009b40 tx=0x7f2868009190 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:00.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.189+0000 7f286fe00640 1 -- 172.21.15.86:0/223356188 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f28780d1200 con 0x7f287815f470 2024-03-06T07:27:00.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.370+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f287808d920 con 0x7f287815f470 2024-03-06T07:27:00.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.371+0000 7f286fe00640 1 -- 172.21.15.86:0/223356188 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v14) v1 ==== 74+0+10921 (secure 0 0 0) 0x7f28700585d0 con 0x7f287815f470 2024-03-06T07:27:00.374 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:00.374 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":14,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:26:59.589976+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:00.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.373+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f285806b440 msgr2=0x7f285806d900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.373+0000 7f287ee00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f285806b440 0x7f285806d900 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f2868009b40 tx=0x7f2868009190 comp rx=0 tx=0).stop 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.373+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 msgr2=0x7f28781f43e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.373+0000 7f287ee00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28781f43e0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f287000cab0 tx=0x7f287000cf70 comp rx=0 tx=0).stop 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 shutdown_connections 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f285806b440 0x7f285806d900 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f287815f470 0x7f28781f43e0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f287815e0d0 0x7f28780b98e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 --2- 172.21.15.86:0/223356188 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28780d0df0 0x7f2878159310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 >> 172.21.15.86:0/223356188 conn(0x7f28780d3ec0 msgr2=0x7f28780b5110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 shutdown_connections 2024-03-06T07:27:00.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.374+0000 7f287ee00640 1 -- 172.21.15.86:0/223356188 wait complete. 2024-03-06T07:27:00.393 INFO:tasks.cephfs.filesystem.ceph_manager:[] 2024-03-06T07:27:00.393 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:00.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.587+0000 7f2e9da00640 1 Processor -- start 2024-03-06T07:27:00.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.587+0000 7f2e9da00640 1 -- start start 2024-03-06T07:27:00.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e9da00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e98144c00 0x7f2e9806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e9da00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2e9815f9d0 0x7f2e9815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e9da00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2e9806ee70 con 0x7f2e980ce0a0 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e9da00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2e98160370 con 0x7f2e9815f9d0 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e9da00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e9806cef0 con 0x7f2e98144c00 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e8e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2e9815f9d0 0x7f2e9815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e8e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2e9815f9d0 0x7f2e9815fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51862/0 (socket says 172.21.15.86:51862) 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.588+0000 7f2e8e000640 1 -- 172.21.15.86:0/1852978484 learned_addr learned my addr 172.21.15.86:0/1852978484 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.589+0000 7f2e8e000640 1 -- 172.21.15.86:0/1852978484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e98144c00 msgr2=0x7f2e9806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.589+0000 7f2e8e000640 1 --2- 172.21.15.86:0/1852978484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e98144c00 0x7f2e9806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.589+0000 7f2e8e000640 1 -- 172.21.15.86:0/1852978484 >> v1:172.21.15.194:6789/0 conn(0x7f2e980ce0a0 legacy=0x7f2e980ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:00.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.589+0000 7f2e8e000640 1 -- 172.21.15.86:0/1852978484 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e9806d1d0 con 0x7f2e9815f9d0 2024-03-06T07:27:00.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.590+0000 7f2e8e000640 1 --2- 172.21.15.86:0/1852978484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e9815f9d0 0x7f2e9815fe30 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f2e80009870 tx=0x7f2e8002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=55f1275b3edbda0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.590+0000 7f2e96000640 1 -- 172.21.15.86:0/1852978484 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e80035020 con 0x7f2e9815f9d0 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.590+0000 7f2e96000640 1 -- 172.21.15.86:0/1852978484 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e80031640 con 0x7f2e9815f9d0 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.590+0000 7f2e96000640 1 -- 172.21.15.86:0/1852978484 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e8002d040 con 0x7f2e9815f9d0 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.590+0000 7f2e9da00640 1 -- 172.21.15.86:0/1852978484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e9815f9d0 msgr2=0x7f2e9815fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.590+0000 7f2e9da00640 1 --2- 172.21.15.86:0/1852978484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e9815f9d0 0x7f2e9815fe30 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f2e80009870 tx=0x7f2e8002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.591+0000 7f2e9da00640 1 -- 172.21.15.86:0/1852978484 shutdown_connections 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.591+0000 7f2e9da00640 1 --2- 172.21.15.86:0/1852978484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e9815f9d0 0x7f2e9815fe30 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.591+0000 7f2e9da00640 1 --2- 172.21.15.86:0/1852978484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e98144c00 0x7f2e9806e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.591+0000 7f2e9da00640 1 -- 172.21.15.86:0/1852978484 >> 172.21.15.86:0/1852978484 conn(0x7f2e980b1730 msgr2=0x7f2e980b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.591+0000 7f2e9da00640 1 -- 172.21.15.86:0/1852978484 shutdown_connections 2024-03-06T07:27:00.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.591+0000 7f2e9da00640 1 -- 172.21.15.86:0/1852978484 wait complete. 2024-03-06T07:27:00.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.592+0000 7f2e9da00640 1 Processor -- start 2024-03-06T07:27:00.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.592+0000 7f2e9da00640 1 -- start start 2024-03-06T07:27:00.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.593+0000 7f2e9da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e980ce0a0 0x7f2e981394e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.593+0000 7f2e9da00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e98144c00 0x7f2e9815e510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.593+0000 7f2e9da00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e9815f9d0 0x7f2e9815ea50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e9da00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e98160730 con 0x7f2e980ce0a0 2024-03-06T07:27:00.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e9da00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e9806d790 con 0x7f2e98144c00 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e9da00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e9806cb50 con 0x7f2e9815f9d0 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e980ce0a0 0x7f2e981394e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e980ce0a0 0x7f2e981394e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51864/0 (socket says 172.21.15.86:51864) 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 -- 172.21.15.86:0/2383630131 learned_addr learned my addr 172.21.15.86:0/2383630131 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 -- 172.21.15.86:0/2383630131 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e9815f9d0 msgr2=0x7f2e9815ea50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e9815f9d0 0x7f2e9815ea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 -- 172.21.15.86:0/2383630131 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e98144c00 msgr2=0x7f2e9815e510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e98144c00 0x7f2e9815e510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 -- 172.21.15.86:0/2383630131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e981f2220 con 0x7f2e980ce0a0 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.594+0000 7f2e96a00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e980ce0a0 0x7f2e981394e0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f2e8800b4d0 tx=0x7f2e8800b990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.595+0000 7f2e8f400640 1 -- 172.21.15.86:0/2383630131 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e88014070 con 0x7f2e980ce0a0 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.595+0000 7f2e8f400640 1 -- 172.21.15.86:0/2383630131 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e88010870 con 0x7f2e980ce0a0 2024-03-06T07:27:00.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.595+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2e9813a8e0 con 0x7f2e980ce0a0 2024-03-06T07:27:00.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.595+0000 7f2e8f400640 1 -- 172.21.15.86:0/2383630131 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e88010c60 con 0x7f2e980ce0a0 2024-03-06T07:27:00.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.595+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2e9813af30 con 0x7f2e980ce0a0 2024-03-06T07:27:00.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.596+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2e9804f330 con 0x7f2e980ce0a0 2024-03-06T07:27:00.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.597+0000 7f2e8f400640 1 -- 172.21.15.86:0/2383630131 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7f2e88017070 con 0x7f2e980ce0a0 2024-03-06T07:27:00.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.598+0000 7f2e8f400640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e7406b440 0x7f2e7406d900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:00.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.598+0000 7f2e8f400640 1 -- 172.21.15.86:0/2383630131 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(14..14 src has 1..14) v4 ==== 4471+0+0 (secure 0 0 0) 0x7f2e88089a30 con 0x7f2e980ce0a0 2024-03-06T07:27:00.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.598+0000 7f2e8e000640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e7406b440 0x7f2e7406d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.598+0000 7f2e8e000640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e7406b440 0x7f2e7406d900 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f2e80036000 tx=0x7f2e80030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.605+0000 7f2e8f400640 1 -- 172.21.15.86:0/2383630131 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2e88058e60 con 0x7f2e980ce0a0 2024-03-06T07:27:00.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.822+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2e980b1dc0 con 0x7f2e980ce0a0 2024-03-06T07:27:00.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.823+0000 7f2e8f400640 1 -- 172.21.15.86:0/2383630131 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) v1 ==== 93+0+1321 (secure 0 0 0) 0x7f2e88058760 con 0x7f2e980ce0a0 2024-03-06T07:27:00.826 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:00.826 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2}],"filesystems":[]} 2024-03-06T07:27:00.826 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 2 2024-03-06T07:27:00.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.825+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e7406b440 msgr2=0x7f2e7406d900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.825+0000 7f2e9da00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e7406b440 0x7f2e7406d900 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f2e80036000 tx=0x7f2e80030000 comp rx=0 tx=0).stop 2024-03-06T07:27:00.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.825+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e980ce0a0 msgr2=0x7f2e981394e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:00.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.825+0000 7f2e9da00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e980ce0a0 0x7f2e981394e0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f2e8800b4d0 tx=0x7f2e8800b990 comp rx=0 tx=0).stop 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 shutdown_connections 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e7406b440 0x7f2e7406d900 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e9815f9d0 0x7f2e9815ea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e98144c00 0x7f2e9815e510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 --2- 172.21.15.86:0/2383630131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e980ce0a0 0x7f2e981394e0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 >> 172.21.15.86:0/2383630131 conn(0x7f2e980b1730 msgr2=0x7f2e980b0fb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 shutdown_connections 2024-03-06T07:27:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:00.826+0000 7f2e9da00640 1 -- 172.21.15.86:0/2383630131 wait complete. 2024-03-06T07:27:02.846 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:03.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.050+0000 7fc4e3c00640 1 Processor -- start 2024-03-06T07:27:03.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.050+0000 7fc4e3c00640 1 -- start start 2024-03-06T07:27:03.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.051+0000 7fc4e3c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc4dc0d0fc0 0x7fc4dc160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.051+0000 7fc4e3c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc4dc0cea60 con 0x7fc4dc15e0d0 2024-03-06T07:27:03.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.051+0000 7fc4e3c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc4dc0d2350 con 0x7fc4dc15f470 2024-03-06T07:27:03.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.051+0000 7fc4e3c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc4dc0d2f60 con 0x7fc4dc0d0fc0 2024-03-06T07:27:03.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4e1200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc4dc15e0d0 0x7fc4dc0d1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39624/0 (socket says 172.21.15.86:39624) 2024-03-06T07:27:03.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4e1200640 1 -- 172.21.15.86:0/2963193632 learned_addr learned my addr 172.21.15.86:0/2963193632 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:03.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4e1c00640 1 --2- 172.21.15.86:0/2963193632 >> v2:172.21.15.194:3300/0 conn(0x7fc4dc0d0fc0 0x7fc4dc160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 962299569 0 0) 0x7fc4dc0cea60 con 0x7fc4dc15e0d0 2024-03-06T07:27:03.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc4c0003520 con 0x7fc4dc15e0d0 2024-03-06T07:27:03.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2167538722 0 0) 0x7fc4c0003520 con 0x7fc4dc15e0d0 2024-03-06T07:27:03.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 >> v2:172.21.15.194:3300/0 conn(0x7fc4dc0d0fc0 msgr2=0x7fc4dc160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 --2- 172.21.15.86:0/2963193632 >> v2:172.21.15.194:3300/0 conn(0x7fc4dc0d0fc0 0x7fc4dc160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 >> v1:172.21.15.194:6789/0 conn(0x7fc4dc15f470 legacy=0x7fc4dc0d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4dc0d2630 con 0x7fc4dc15e0d0 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.052+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc4cc001670 con 0x7fc4dc15e0d0 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 >> v1:172.21.15.86:6789/0 conn(0x7fc4dc15e0d0 legacy=0x7fc4dc0d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4dbe00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4c0003a00 0x7fc4c0003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4dbe00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4c0004800 0x7fc4c0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4dbe00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4c000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4c000c070 con 0x7fc4c00051c0 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4c000c220 con 0x7fc4c0004800 2024-03-06T07:27:03.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc4c000c420 con 0x7fc4c0003a00 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1c00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4c000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1200640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4c0003a00 0x7fc4c0003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4d3e00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4c0004800 0x7fc4c0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1c00640 1 -- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4c0003a00 msgr2=0x7fc4c0003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1c00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4c0003a00 0x7fc4c0003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1c00640 1 -- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4c0004800 msgr2=0x7fc4c0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1c00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4c0004800 0x7fc4c0004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1c00640 1 -- 172.21.15.86:0/2963193632 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4c0003520 con 0x7fc4c00051c0 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4d3e00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4c0004800 0x7fc4c0004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:03.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.053+0000 7fc4e1200640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4c0003a00 0x7fc4c0003e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.054+0000 7fc4e1c00640 1 -- 172.21.15.86:0/2963193632 reap_dead start 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.054+0000 7fc4e1c00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4c000b9c0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fc4d4009fc0 tx=0x7fc4c0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=377ffbde88e2a7c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4d4009c10 con 0x7fc4c00051c0 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc4d402cca0 con 0x7fc4c00051c0 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4dbe00640 1 -- 172.21.15.86:0/2963193632 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4d4003bd0 con 0x7fc4c00051c0 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4e3c00640 1 -- 172.21.15.86:0/2963193632 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 msgr2=0x7fc4c000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4c000b9c0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fc4d4009fc0 tx=0x7fc4c0005830 comp rx=0 tx=0).stop 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4e3c00640 1 -- 172.21.15.86:0/2963193632 shutdown_connections 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/2963193632 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4c000b9c0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.055+0000 7fc4e3c00640 1 -- 172.21.15.86:0/2963193632 >> 172.21.15.86:0/2963193632 conn(0x7fc4dc0d3ec0 msgr2=0x7fc4dc0d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.056+0000 7fc4e3c00640 1 -- 172.21.15.86:0/2963193632 shutdown_connections 2024-03-06T07:27:03.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.056+0000 7fc4e3c00640 1 -- 172.21.15.86:0/2963193632 wait complete. 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.057+0000 7fc4e3c00640 1 Processor -- start 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.058+0000 7fc4e3c00640 1 -- start start 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.058+0000 7fc4e3c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4dc0b2740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.058+0000 7fc4e3c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4dc1f3ca0 0x7fc4dc152540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.058+0000 7fc4e3c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4dc160ba0 0x7fc4dc161020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e3c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4dc15fb30 con 0x7fc4c00051c0 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e3c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4dc0d2bf0 con 0x7fc4dc160ba0 2024-03-06T07:27:03.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e3c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc4dc15e4b0 con 0x7fc4dc1f3ca0 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4dc0b2740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4dc0b2740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51908/0 (socket says 172.21.15.86:51908) 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 -- 172.21.15.86:0/3794205799 learned_addr learned my addr 172.21.15.86:0/3794205799 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4d3e00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4dc1f3ca0 0x7fc4dc152540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 -- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4dc1f3ca0 msgr2=0x7fc4dc152540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1c00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4dc160ba0 0x7fc4dc161020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4dc1f3ca0 0x7fc4dc152540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 -- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4dc160ba0 msgr2=0x7fc4dc161020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4dc160ba0 0x7fc4dc161020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 -- 172.21.15.86:0/3794205799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4dc1f4240 con 0x7fc4c00051c0 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4d3e00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4dc1f3ca0 0x7fc4dc152540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:03.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.059+0000 7fc4e1200640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4dc0b2740 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fc4cc00d770 tx=0x7fc4cc00dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.060+0000 7fc4d9600640 1 -- 172.21.15.86:0/3794205799 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4cc019070 con 0x7fc4c00051c0 2024-03-06T07:27:03.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.060+0000 7fc4d9600640 1 -- 172.21.15.86:0/3794205799 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc4cc0074e0 con 0x7fc4c00051c0 2024-03-06T07:27:03.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.060+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc4dc1f4520 con 0x7fc4c00051c0 2024-03-06T07:27:03.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.060+0000 7fc4d9600640 1 -- 172.21.15.86:0/3794205799 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4cc004a00 con 0x7fc4c00051c0 2024-03-06T07:27:03.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.060+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc4dc1f4ae0 con 0x7fc4c00051c0 2024-03-06T07:27:03.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.061+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc4dc04f330 con 0x7fc4c00051c0 2024-03-06T07:27:03.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.062+0000 7fc4d9600640 1 -- 172.21.15.86:0/3794205799 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7fc4cc01c070 con 0x7fc4c00051c0 2024-03-06T07:27:03.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.063+0000 7fc4d9600640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4b806b440 0x7fc4b806d900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.063+0000 7fc4d9600640 1 -- 172.21.15.86:0/3794205799 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fc4cc0897c0 con 0x7fc4c00051c0 2024-03-06T07:27:03.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.063+0000 7fc4d3e00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4b806b440 0x7fc4b806d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.064+0000 7fc4d3e00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4b806b440 0x7fc4b806d900 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fc4c4000c00 tx=0x7fc4c4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.068+0000 7fc4d9600640 1 -- 172.21.15.86:0/3794205799 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc4cc00de20 con 0x7fc4c00051c0 2024-03-06T07:27:03.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.317+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc4dc068500 con 0x7fc4c00051c0 2024-03-06T07:27:03.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.317+0000 7fc4d9600640 1 -- 172.21.15.86:0/3794205799 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) v1 ==== 93+0+3691 (secure 0 0 0) 0x7fc4cc058aa0 con 0x7fc4c00051c0 2024-03-06T07:27:03.320 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:03.320 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4255,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[]} 2024-03-06T07:27:03.320 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 3 2024-03-06T07:27:03.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.320+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4b806b440 msgr2=0x7fc4b806d900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.320+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4b806b440 0x7fc4b806d900 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fc4c4000c00 tx=0x7fc4c4009040 comp rx=0 tx=0).stop 2024-03-06T07:27:03.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.320+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 msgr2=0x7fc4dc0b2740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.320+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4dc0b2740 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fc4cc00d770 tx=0x7fc4cc00dc30 comp rx=0 tx=0).stop 2024-03-06T07:27:03.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 shutdown_connections 2024-03-06T07:27:03.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4b806b440 0x7fc4b806d900 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4dc160ba0 0x7fc4dc161020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4dc1f3ca0 0x7fc4dc152540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 --2- 172.21.15.86:0/3794205799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4c00051c0 0x7fc4dc0b2740 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 >> 172.21.15.86:0/3794205799 conn(0x7fc4dc0d3ec0 msgr2=0x7fc4dc0d42a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:03.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 shutdown_connections 2024-03-06T07:27:03.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.321+0000 7fc4e3c00640 1 -- 172.21.15.86:0/3794205799 wait complete. 2024-03-06T07:27:03.340 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:03.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.522+0000 7f1067c00640 1 Processor -- start 2024-03-06T07:27:03.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.522+0000 7f1067c00640 1 -- start start 2024-03-06T07:27:03.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.523+0000 7f1067c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f106015f9d0 0x7f106015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.523+0000 7f1067c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f106006cef0 con 0x7f1060070b10 2024-03-06T07:27:03.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.523+0000 7f1067c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f106006d1d0 con 0x7f1060144c00 2024-03-06T07:27:03.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.523+0000 7f1067c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f106006d4b0 con 0x7f106015f9d0 2024-03-06T07:27:03.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.523+0000 7f1057e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f106015f9d0 0x7f106015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.523+0000 7f1057e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f106015f9d0 0x7f106015fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51914/0 (socket says 172.21.15.86:51914) 2024-03-06T07:27:03.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.523+0000 7f1057e00640 1 -- 172.21.15.86:0/3645091243 learned_addr learned my addr 172.21.15.86:0/3645091243 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:03.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.524+0000 7f1057e00640 1 -- 172.21.15.86:0/3645091243 >> v1:172.21.15.194:6789/0 conn(0x7f1060144c00 legacy=0x7f106006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.524+0000 7f1057e00640 1 -- 172.21.15.86:0/3645091243 >> v1:172.21.15.86:6789/0 conn(0x7f1060070b10 legacy=0x7f1060070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.524+0000 7f1057e00640 1 -- 172.21.15.86:0/3645091243 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f106006d790 con 0x7f106015f9d0 2024-03-06T07:27:03.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.524+0000 7f105fe00640 1 -- 172.21.15.86:0/3645091243 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1430392928 0 0) 0x7f106006cef0 con 0x7f1060070b10 2024-03-06T07:27:03.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.525+0000 7f1057e00640 1 --2- 172.21.15.86:0/3645091243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f106015f9d0 0x7f106015fe60 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1048009870 tx=0x7f104802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=544255264cc06c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.525+0000 7f105fe00640 1 -- 172.21.15.86:0/3645091243 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1048034070 con 0x7f106015f9d0 2024-03-06T07:27:03.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.525+0000 7f105fe00640 1 -- 172.21.15.86:0/3645091243 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1048030860 con 0x7f106015f9d0 2024-03-06T07:27:03.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.525+0000 7f105fe00640 1 -- 172.21.15.86:0/3645091243 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1048030c30 con 0x7f106015f9d0 2024-03-06T07:27:03.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.526+0000 7f1067c00640 1 -- 172.21.15.86:0/3645091243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f106015f9d0 msgr2=0x7f106015fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.526+0000 7f1067c00640 1 --2- 172.21.15.86:0/3645091243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f106015f9d0 0x7f106015fe60 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1048009870 tx=0x7f104802b9a0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.526+0000 7f1067c00640 1 -- 172.21.15.86:0/3645091243 shutdown_connections 2024-03-06T07:27:03.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.526+0000 7f1067c00640 1 --2- 172.21.15.86:0/3645091243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f106015f9d0 0x7f106015fe60 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.526+0000 7f1067c00640 1 -- 172.21.15.86:0/3645091243 >> 172.21.15.86:0/3645091243 conn(0x7f10600b1730 msgr2=0x7f10600b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:03.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.526+0000 7f1067c00640 1 -- 172.21.15.86:0/3645091243 shutdown_connections 2024-03-06T07:27:03.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.526+0000 7f1067c00640 1 -- 172.21.15.86:0/3645091243 wait complete. 2024-03-06T07:27:03.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.527+0000 7f1067c00640 1 Processor -- start 2024-03-06T07:27:03.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.528+0000 7f1067c00640 1 -- start start 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.528+0000 7f1067c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1060070b10 0x7f1060072240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.528+0000 7f1067c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1060144c00 0x7f106012f790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.528+0000 7f1067c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f106015f9d0 0x7f10601f3700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.528+0000 7f1067c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f106006ccd0 con 0x7f1060144c00 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.528+0000 7f1067c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f106006cb50 con 0x7f106015f9d0 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.528+0000 7f1067c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f106006df90 con 0x7f1060070b10 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1060144c00 0x7f106012f790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1060144c00 0x7f106012f790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51924/0 (socket says 172.21.15.86:51924) 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 -- 172.21.15.86:0/1639022493 learned_addr learned my addr 172.21.15.86:0/1639022493 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1065200640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1060070b10 0x7f1060072240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1065c00640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f106015f9d0 0x7f10601f3700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 -- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1060070b10 msgr2=0x7f1060072240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1060070b10 0x7f1060072240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 -- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f106015f9d0 msgr2=0x7f10601f3700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f106015f9d0 0x7f10601f3700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 -- 172.21.15.86:0/1639022493 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f106012fcd0 con 0x7f1060144c00 2024-03-06T07:27:03.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.529+0000 7f1057e00640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1060144c00 0x7f106012f790 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f10480090b0 tx=0x7f10480046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.530+0000 7f105d600640 1 -- 172.21.15.86:0/1639022493 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1048034040 con 0x7f1060144c00 2024-03-06T07:27:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.530+0000 7f105d600640 1 -- 172.21.15.86:0/1639022493 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f104803ab90 con 0x7f1060144c00 2024-03-06T07:27:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.530+0000 7f105d600640 1 -- 172.21.15.86:0/1639022493 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1048004030 con 0x7f1060144c00 2024-03-06T07:27:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.530+0000 7f1067c00640 1 -- 172.21.15.86:0/1639022493 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f106012ff70 con 0x7f1060144c00 2024-03-06T07:27:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.530+0000 7f1067c00640 1 -- 172.21.15.86:0/1639022493 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1060130530 con 0x7f1060144c00 2024-03-06T07:27:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.531+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f106004f330 con 0x7f1060144c00 2024-03-06T07:27:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.532+0000 7f105d600640 1 -- 172.21.15.86:0/1639022493 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7f1048037070 con 0x7f1060144c00 2024-03-06T07:27:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.533+0000 7f105d600640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103c06b440 0x7f103c06d900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.533+0000 7f105d600640 1 -- 172.21.15.86:0/1639022493 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f10480abe70 con 0x7f1060144c00 2024-03-06T07:27:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.533+0000 7f1065200640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103c06b440 0x7f103c06d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.534+0000 7f1065200640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103c06b440 0x7f103c06d900 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f1050003d90 tx=0x7f1050009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.537+0000 7f105d600640 1 -- 172.21.15.86:0/1639022493 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f104803f070 con 0x7f1060144c00 2024-03-06T07:27:03.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.742+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1060072890 con 0x7f1060144c00 2024-03-06T07:27:03.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.744+0000 7f105d600640 1 -- 172.21.15.86:0/1639022493 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v18) v1 ==== 74+0+12896 (secure 0 0 0) 0x7f104807b1a0 con 0x7f1060144c00 2024-03-06T07:27:03.747 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:03.747 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":18,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:03.622341+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:03.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.746+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103c06b440 msgr2=0x7f103c06d900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.746+0000 7f1056000640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103c06b440 0x7f103c06d900 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f1050003d90 tx=0x7f1050009040 comp rx=0 tx=0).stop 2024-03-06T07:27:03.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.746+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1060144c00 msgr2=0x7f106012f790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.746+0000 7f1056000640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1060144c00 0x7f106012f790 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f10480090b0 tx=0x7f10480046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 shutdown_connections 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103c06b440 0x7f103c06d900 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f106015f9d0 0x7f10601f3700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1060144c00 0x7f106012f790 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 --2- 172.21.15.86:0/1639022493 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1060070b10 0x7f1060072240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 >> 172.21.15.86:0/1639022493 conn(0x7f10600b1730 msgr2=0x7f10600ae630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 shutdown_connections 2024-03-06T07:27:03.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.747+0000 7f1056000640 1 -- 172.21.15.86:0/1639022493 wait complete. 2024-03-06T07:27:03.767 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:03.767 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:03.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.955+0000 7fc478600640 1 Processor -- start 2024-03-06T07:27:03.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.955+0000 7fc478600640 1 -- start start 2024-03-06T07:27:03.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc478600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 0x7fc4700d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc478600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc47015e450 0x7fc4700b4480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc478600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc4700b3730 con 0x7fc4700d1bf0 2024-03-06T07:27:03.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc478600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc4700d4930 con 0x7fc47015e450 2024-03-06T07:27:03.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc478600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc470067b20 con 0x7fc4700d07f0 2024-03-06T07:27:03.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc476600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc4700d1bf0 0x7fc4700b49c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39638/0 (socket says 172.21.15.86:39638) 2024-03-06T07:27:03.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc476600640 1 -- 172.21.15.86:0/3821802005 learned_addr learned my addr 172.21.15.86:0/3821802005 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc475c00640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 0x7fc4700d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc475200640 1 --2- 172.21.15.86:0/3821802005 >> v2:172.21.15.194:3300/0 conn(0x7fc47015e450 0x7fc4700b4480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3107636414 0 0) 0x7fc4700b3730 con 0x7fc4700d1bf0 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.957+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc458003520 con 0x7fc4700d1bf0 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 517339337 0 0) 0x7fc458003520 con 0x7fc4700d1bf0 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 msgr2=0x7fc4700d0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 0x7fc4700d0bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 >> v2:172.21.15.194:3300/0 conn(0x7fc47015e450 msgr2=0x7fc4700b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 --2- 172.21.15.86:0/3821802005 >> v2:172.21.15.194:3300/0 conn(0x7fc47015e450 0x7fc4700b4480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4700d1110 con 0x7fc4700d1bf0 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc4680019f0 con 0x7fc4700d1bf0 2024-03-06T07:27:03.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc475200640 1 --2- 172.21.15.86:0/3821802005 >> v2:172.21.15.194:3300/0 conn(0x7fc47015e450 0x7fc4700b4480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 >> v1:172.21.15.86:6789/0 conn(0x7fc4700d1bf0 legacy=0x7fc4700b49c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4580039b0 0x7fc458003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 0x7fc458004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc458004d40 0x7fc4580051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4700d4930 con 0x7fc458004d40 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc470067b20 con 0x7fc4580039b0 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc45800bb20 con 0x7fc4700d07f0 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc476600640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc458004d40 0x7fc4580051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc475c00640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4580039b0 0x7fc458003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc475200640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 0x7fc458004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc476600640 1 -- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 msgr2=0x7fc458004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.958+0000 7fc476600640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700d07f0 0x7fc458004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.959+0000 7fc476600640 1 -- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4580039b0 msgr2=0x7fc458003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.959+0000 7fc476600640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4580039b0 0x7fc458003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.959+0000 7fc476600640 1 -- 172.21.15.86:0/3821802005 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc458003520 con 0x7fc458004d40 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc476600640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc458004d40 0x7fc4580051c0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fc46800a920 tx=0x7fc458005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b8d4590316f10ea5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc46800a4c0 con 0x7fc458004d40 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc468034650 con 0x7fc458004d40 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc46fe00640 1 -- 172.21.15.86:0/3821802005 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc468034a40 con 0x7fc458004d40 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc478600640 1 -- 172.21.15.86:0/3821802005 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc458004d40 msgr2=0x7fc4580051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc478600640 1 --2- 172.21.15.86:0/3821802005 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc458004d40 0x7fc4580051c0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fc46800a920 tx=0x7fc458005830 comp rx=0 tx=0).stop 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc476600640 1 -- 172.21.15.86:0/3821802005 reap_dead start 2024-03-06T07:27:03.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc478600640 1 -- 172.21.15.86:0/3821802005 shutdown_connections 2024-03-06T07:27:03.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.960+0000 7fc478600640 1 -- 172.21.15.86:0/3821802005 >> 172.21.15.86:0/3821802005 conn(0x7fc4700d3e10 msgr2=0x7fc4700d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:03.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.961+0000 7fc478600640 1 -- 172.21.15.86:0/3821802005 shutdown_connections 2024-03-06T07:27:03.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.961+0000 7fc478600640 1 -- 172.21.15.86:0/3821802005 wait complete. 2024-03-06T07:27:03.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.962+0000 7fc478600640 1 Processor -- start 2024-03-06T07:27:03.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.962+0000 7fc478600640 1 -- start start 2024-03-06T07:27:03.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.963+0000 7fc478600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc458004d40 0x7fc47006ccc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.963+0000 7fc478600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700c93a0 0x7fc4701f7490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.963+0000 7fc478600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4701f79d0 0x7fc4700c7940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.963+0000 7fc478600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4700b3c80 con 0x7fc4701f79d0 2024-03-06T07:27:03.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.963+0000 7fc478600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4700b39a0 con 0x7fc458004d40 2024-03-06T07:27:03.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.963+0000 7fc478600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc4700b3f60 con 0x7fc4700c93a0 2024-03-06T07:27:03.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4701f79d0 0x7fc4700c7940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4701f79d0 0x7fc4700c7940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51938/0 (socket says 172.21.15.86:51938) 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 -- 172.21.15.86:0/2237657447 learned_addr learned my addr 172.21.15.86:0/2237657447 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc475c00640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc458004d40 0x7fc47006ccc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc475200640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700c93a0 0x7fc4701f7490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 -- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700c93a0 msgr2=0x7fc4701f7490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700c93a0 0x7fc4701f7490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 -- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc458004d40 msgr2=0x7fc47006ccc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc458004d40 0x7fc47006ccc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 -- 172.21.15.86:0/2237657447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4701f3fa0 con 0x7fc4701f79d0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.964+0000 7fc476600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4701f79d0 0x7fc4700c7940 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fc46802ccb0 tx=0x7fc4700b09e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.965+0000 7fc46d600640 1 -- 172.21.15.86:0/2237657447 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc468002000 con 0x7fc4701f79d0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.965+0000 7fc46d600640 1 -- 172.21.15.86:0/2237657447 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc4680019c0 con 0x7fc4701f79d0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.965+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc4701f4280 con 0x7fc4701f79d0 2024-03-06T07:27:03.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.965+0000 7fc46d600640 1 -- 172.21.15.86:0/2237657447 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc468004b60 con 0x7fc4701f79d0 2024-03-06T07:27:03.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.965+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc4701f6f70 con 0x7fc4701f79d0 2024-03-06T07:27:03.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.966+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc4700b9760 con 0x7fc4701f79d0 2024-03-06T07:27:03.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.967+0000 7fc46d600640 1 -- 172.21.15.86:0/2237657447 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7fc468047020 con 0x7fc4701f79d0 2024-03-06T07:27:03.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.968+0000 7fc46d600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45006b440 0x7fc45006d900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:03.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.968+0000 7fc46d600640 1 -- 172.21.15.86:0/2237657447 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fc4680abbd0 con 0x7fc4701f79d0 2024-03-06T07:27:03.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.968+0000 7fc475c00640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45006b440 0x7fc45006d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:03.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.969+0000 7fc475c00640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45006b440 0x7fc45006d900 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fc464006fd0 tx=0x7fc464008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:03.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:03.973+0000 7fc46d600640 1 -- 172.21.15.86:0/2237657447 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc46807ad50 con 0x7fc4701f79d0 2024-03-06T07:27:04.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.170+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fc4701f6d90 con 0x7fc4701f79d0 2024-03-06T07:27:04.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.171+0000 7fc46d600640 1 -- 172.21.15.86:0/2237657447 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v18) v1 ==== 93+0+10 (secure 0 0 0) 0x7fc4701f6d90 con 0x7fc4701f79d0 2024-03-06T07:27:04.173 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:04.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.173+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45006b440 msgr2=0x7fc45006d900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.173+0000 7fc478600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45006b440 0x7fc45006d900 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fc464006fd0 tx=0x7fc464008040 comp rx=0 tx=0).stop 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4701f79d0 msgr2=0x7fc4700c7940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4701f79d0 0x7fc4700c7940 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fc46802ccb0 tx=0x7fc4700b09e0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 shutdown_connections 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45006b440 0x7fc45006d900 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4701f79d0 0x7fc4700c7940 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4700c93a0 0x7fc4701f7490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 --2- 172.21.15.86:0/2237657447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc458004d40 0x7fc47006ccc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 >> 172.21.15.86:0/2237657447 conn(0x7fc4700d3e10 msgr2=0x7fc470160870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 shutdown_connections 2024-03-06T07:27:04.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.174+0000 7fc478600640 1 -- 172.21.15.86:0/2237657447 wait complete. 2024-03-06T07:27:04.194 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:27:04.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.396+0000 7faaf2600640 1 Processor -- start 2024-03-06T07:27:04.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.396+0000 7faaf2600640 1 -- start start 2024-03-06T07:27:04.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.397+0000 7faaf2600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7faaec06ec00 0x7faaec0ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.397+0000 7faaf2600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7faaec15b090 con 0x7faaec0b4680 2024-03-06T07:27:04.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.397+0000 7faaf2600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7faaec0b2a30 con 0x7faaec15acb0 2024-03-06T07:27:04.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.397+0000 7faaf2600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faaec0b4360 con 0x7faaec06ec00 2024-03-06T07:27:04.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faae3e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7faaec0b4680 0x7faaec06e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39652/0 (socket says 172.21.15.86:39652) 2024-03-06T07:27:04.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faae3e00640 1 -- 172.21.15.86:0/1461880344 learned_addr learned my addr 172.21.15.86:0/1461880344 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:04.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaebe00640 1 --2- 172.21.15.86:0/1461880344 >> v2:172.21.15.194:3300/0 conn(0x7faaec06ec00 0x7faaec0ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2821576935 0 0) 0x7faaec15b090 con 0x7faaec0b4680 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7faad0003520 con 0x7faaec0b4680 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3137434296 0 0) 0x7faaec0b2a30 con 0x7faaec15acb0 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7faaec15b090 con 0x7faaec15acb0 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1543768406 0 0) 0x7faad0003520 con 0x7faaec0b4680 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 >> v2:172.21.15.194:3300/0 conn(0x7faaec06ec00 msgr2=0x7faaec0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 --2- 172.21.15.86:0/1461880344 >> v2:172.21.15.194:3300/0 conn(0x7faaec06ec00 0x7faaec0ccfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 >> v1:172.21.15.194:6789/0 conn(0x7faaec15acb0 legacy=0x7faaec0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faaec0b2cf0 con 0x7faaec0b4680 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7faad4001670 con 0x7faaec0b4680 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.398+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 >> v1:172.21.15.86:6789/0 conn(0x7faaec0b4680 legacy=0x7faaec06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeaa00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad0003b90 0x7faad0003f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeaa00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faad0004800 0x7faad0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeaa00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faad00051c0 0x7faad000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faaec0b4360 con 0x7faad00051c0 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faad000c090 con 0x7faad0004800 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faad000c290 con 0x7faad0003b90 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaebe00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faad00051c0 0x7faad000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faae3e00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faad0004800 0x7faad0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeb400640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad0003b90 0x7faad0003f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaebe00640 1 -- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad0003b90 msgr2=0x7faad0003f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaebe00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad0003b90 0x7faad0003f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaebe00640 1 -- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faad0004800 msgr2=0x7faad0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaebe00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faad0004800 0x7faad0004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaebe00640 1 -- 172.21.15.86:0/1461880344 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faad0003520 con 0x7faad00051c0 2024-03-06T07:27:04.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.399+0000 7faaeb400640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad0003b90 0x7faad0003f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.400+0000 7faaebe00640 1 -- 172.21.15.86:0/1461880344 reap_dead start 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.400+0000 7faaebe00640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faad00051c0 0x7faad000b9c0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7faae4009fc0 tx=0x7faad0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=24fd2e44bfdd168a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.400+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faae4009c10 con 0x7faad00051c0 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.400+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faae402cc70 con 0x7faad00051c0 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.400+0000 7faaeaa00640 1 -- 172.21.15.86:0/1461880344 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faae4003bd0 con 0x7faad00051c0 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.401+0000 7faaf2600640 1 -- 172.21.15.86:0/1461880344 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faad00051c0 msgr2=0x7faad000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.401+0000 7faaf2600640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faad00051c0 0x7faad000b9c0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7faae4009fc0 tx=0x7faad0005830 comp rx=0 tx=0).stop 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.401+0000 7faaf2600640 1 -- 172.21.15.86:0/1461880344 shutdown_connections 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.401+0000 7faaf2600640 1 --2- 172.21.15.86:0/1461880344 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faad00051c0 0x7faad000b9c0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.401+0000 7faaf2600640 1 -- 172.21.15.86:0/1461880344 >> 172.21.15.86:0/1461880344 conn(0x7faaec0b1730 msgr2=0x7faaec0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.401+0000 7faaf2600640 1 -- 172.21.15.86:0/1461880344 shutdown_connections 2024-03-06T07:27:04.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.401+0000 7faaf2600640 1 -- 172.21.15.86:0/1461880344 wait complete. 2024-03-06T07:27:04.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.403+0000 7faaf2600640 1 Processor -- start 2024-03-06T07:27:04.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.403+0000 7faaf2600640 1 -- start start 2024-03-06T07:27:04.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaf2600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad00051c0 0x7faaec146e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaf2600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faaec14e710 0x7faaec14bfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaf2600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faaec0cac60 0x7faaec0cb110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaf2600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faaec0b4010 con 0x7faaec0cac60 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaf2600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faaec0b32b0 con 0x7faaec14e710 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaf2600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faaec072340 con 0x7faad00051c0 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaebe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faaec0cac60 0x7faaec0cb110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaebe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faaec0cac60 0x7faaec0cb110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51964/0 (socket says 172.21.15.86:51964) 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaebe00640 1 -- 172.21.15.86:0/2301866341 learned_addr learned my addr 172.21.15.86:0/2301866341 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faae3e00640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faaec14e710 0x7faaec14bfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.404+0000 7faaeb400640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad00051c0 0x7faaec146e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaebe00640 1 -- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad00051c0 msgr2=0x7faaec146e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaebe00640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad00051c0 0x7faaec146e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaebe00640 1 -- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faaec14e710 msgr2=0x7faaec14bfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaebe00640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faaec14e710 0x7faaec14bfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaebe00640 1 -- 172.21.15.86:0/2301866341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faaec14b300 con 0x7faaec0cac60 2024-03-06T07:27:04.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaebe00640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faaec0cac60 0x7faaec0cb110 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7faae402c6a0 tx=0x7faae4009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:04.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faae3400640 1 -- 172.21.15.86:0/2301866341 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faae4009c10 con 0x7faaec0cac60 2024-03-06T07:27:04.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faae3400640 1 -- 172.21.15.86:0/2301866341 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faae4044070 con 0x7faaec0cac60 2024-03-06T07:27:04.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faae3400640 1 -- 172.21.15.86:0/2301866341 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faae4003b90 con 0x7faaec0cac60 2024-03-06T07:27:04.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faaec0ca180 con 0x7faaec0cac60 2024-03-06T07:27:04.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.405+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faaec0ca740 con 0x7faaec0cac60 2024-03-06T07:27:04.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.407+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faaec0ca940 con 0x7faaec0cac60 2024-03-06T07:27:04.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.407+0000 7faae3400640 1 -- 172.21.15.86:0/2301866341 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7faae4005d10 con 0x7faaec0cac60 2024-03-06T07:27:04.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.408+0000 7faae3400640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faac806b470 0x7faac806d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.408+0000 7faae3400640 1 -- 172.21.15.86:0/2301866341 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 4857+0+0 (secure 0 0 0) 0x7faae40ac650 con 0x7faaec0cac60 2024-03-06T07:27:04.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.408+0000 7faaeb400640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faac806b470 0x7faac806d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.409+0000 7faaeb400640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faac806b470 0x7faac806d930 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7faadc0017f0 tx=0x7faadc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:04.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.413+0000 7faae3400640 1 -- 172.21.15.86:0/2301866341 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faaec0ca940 con 0x7faaec0cac60 2024-03-06T07:27:04.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.611+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7faaec0cc9b0 con 0x7faaec0cac60 2024-03-06T07:27:04.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.612+0000 7faae3400640 1 -- 172.21.15.86:0/2301866341 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v18) v1 ==== 84+0+65 (secure 0 0 0) 0x7faae402ce20 con 0x7faaec0cac60 2024-03-06T07:27:04.614 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:04.614 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:27:04.614 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:27:04.614 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:27:04.614 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:27:04.614 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:27:04.615 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faac806b470 msgr2=0x7faac806d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faac806b470 0x7faac806d930 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7faadc0017f0 tx=0x7faadc009040 comp rx=0 tx=0).stop 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faaec0cac60 msgr2=0x7faaec0cb110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faaec0cac60 0x7faaec0cb110 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7faae402c6a0 tx=0x7faae4009750 comp rx=0 tx=0).stop 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 shutdown_connections 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faac806b470 0x7faac806d930 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faaec0cac60 0x7faaec0cb110 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faaec14e710 0x7faaec14bfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 --2- 172.21.15.86:0/2301866341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faad00051c0 0x7faaec146e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 >> 172.21.15.86:0/2301866341 conn(0x7faaec0b1730 msgr2=0x7faaec06d080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:04.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 shutdown_connections 2024-03-06T07:27:04.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.614+0000 7faaf2600640 1 -- 172.21.15.86:0/2301866341 wait complete. 2024-03-06T07:27:04.634 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:27:04.634 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:27:04.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.841+0000 7f0004c00640 1 Processor -- start 2024-03-06T07:27:04.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.841+0000 7f0004c00640 1 -- start start 2024-03-06T07:27:04.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.842+0000 7f0004c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f000015f1f0 0x7f0000165d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7f0004c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f000015b1d0 con 0x7f000015adf0 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7f0004c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f00000b2bd0 con 0x7f000006eab0 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7f0004c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f000015f5f0 con 0x7f000015f1f0 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7efffde00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f000015adf0 0x7f000006e3a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39660/0 (socket says 172.21.15.86:39660) 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7efffde00640 1 -- 172.21.15.86:0/448136655 learned_addr learned my addr 172.21.15.86:0/448136655 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7efffe800640 1 --2- 172.21.15.86:0/448136655 >> v2:172.21.15.194:3300/0 conn(0x7f000015f1f0 0x7f0000165d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7efffd400640 1 -- 172.21.15.86:0/448136655 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 988907678 0 0) 0x7f000015b1d0 con 0x7f000015adf0 2024-03-06T07:27:04.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.843+0000 7efffd400640 1 -- 172.21.15.86:0/448136655 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7effe0003520 con 0x7f000015adf0 2024-03-06T07:27:04.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.844+0000 7efffd400640 1 -- 172.21.15.86:0/448136655 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3854599219 0 0) 0x7effe0003520 con 0x7f000015adf0 2024-03-06T07:27:04.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.844+0000 7efffe800640 1 -- 172.21.15.86:0/448136655 >> v1:172.21.15.194:6789/0 conn(0x7f000006eab0 legacy=0x7f000015eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.844+0000 7efffe800640 1 -- 172.21.15.86:0/448136655 >> v1:172.21.15.86:6789/0 conn(0x7f000015adf0 legacy=0x7f000006e3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.844+0000 7efffe800640 1 -- 172.21.15.86:0/448136655 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f00000b2eb0 con 0x7f000015f1f0 2024-03-06T07:27:04.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.845+0000 7efffe800640 1 --2- 172.21.15.86:0/448136655 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000015f1f0 0x7f0000165d20 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7efff8009d90 tx=0x7efff802be10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=abed9b80c9447230 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:04.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.845+0000 7efffd400640 1 -- 172.21.15.86:0/448136655 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efff8039070 con 0x7f000015f1f0 2024-03-06T07:27:04.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.845+0000 7efffd400640 1 -- 172.21.15.86:0/448136655 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efff802c940 con 0x7f000015f1f0 2024-03-06T07:27:04.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.845+0000 7efffd400640 1 -- 172.21.15.86:0/448136655 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7effe8001670 con 0x7f000015f1f0 2024-03-06T07:27:04.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.846+0000 7f0004c00640 1 -- 172.21.15.86:0/448136655 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000015f1f0 msgr2=0x7f0000165d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.846+0000 7f0004c00640 1 --2- 172.21.15.86:0/448136655 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000015f1f0 0x7f0000165d20 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7efff8009d90 tx=0x7efff802be10 comp rx=0 tx=0).stop 2024-03-06T07:27:04.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.846+0000 7f0004c00640 1 -- 172.21.15.86:0/448136655 shutdown_connections 2024-03-06T07:27:04.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.846+0000 7f0004c00640 1 --2- 172.21.15.86:0/448136655 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000015f1f0 0x7f0000165d20 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.846+0000 7f0004c00640 1 -- 172.21.15.86:0/448136655 >> 172.21.15.86:0/448136655 conn(0x7f00000cfa80 msgr2=0x7f00000cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:04.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.846+0000 7f0004c00640 1 -- 172.21.15.86:0/448136655 shutdown_connections 2024-03-06T07:27:04.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.846+0000 7f0004c00640 1 -- 172.21.15.86:0/448136655 wait complete. 2024-03-06T07:27:04.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.848+0000 7f0004c00640 1 Processor -- start 2024-03-06T07:27:04.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.848+0000 7f0004c00640 1 -- start start 2024-03-06T07:27:04.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7f0004c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000006eab0 0x7f00000cd470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7f0004c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f000015adf0 0x7f0000152890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7f0004c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f000015f1f0 0x7f00001474e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7f0004c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f000015f850 con 0x7f000015f1f0 2024-03-06T07:27:04.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7f0004c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00000b3470 con 0x7f000006eab0 2024-03-06T07:27:04.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7f0004c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f00000b3990 con 0x7f000015adf0 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7efffe800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f000015f1f0 0x7f00001474e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7efffe800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f000015f1f0 0x7f00001474e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51980/0 (socket says 172.21.15.86:51980) 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7efffe800640 1 -- 172.21.15.86:0/3888347370 learned_addr learned my addr 172.21.15.86:0/3888347370 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7efff7e00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f000015adf0 0x7f0000152890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.849+0000 7efffde00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000006eab0 0x7f00000cd470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efffe800640 1 -- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f000015adf0 msgr2=0x7f0000152890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efffe800640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f000015adf0 0x7f0000152890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efffe800640 1 -- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000006eab0 msgr2=0x7f00000cd470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efffe800640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000006eab0 0x7f00000cd470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efffe800640 1 -- 172.21.15.86:0/3888347370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0000147a20 con 0x7f000015f1f0 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efff7e00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f000015adf0 0x7f0000152890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:04.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efffe800640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f000015f1f0 0x7f00001474e0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7efff8009eb0 tx=0x7efff8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:04.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efff6000640 1 -- 172.21.15.86:0/3888347370 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efff8044030 con 0x7f000015f1f0 2024-03-06T07:27:04.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efff6000640 1 -- 172.21.15.86:0/3888347370 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efff8004040 con 0x7f000015f1f0 2024-03-06T07:27:04.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0000152fc0 con 0x7f000015f1f0 2024-03-06T07:27:04.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7efff6000640 1 -- 172.21.15.86:0/3888347370 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efff8039040 con 0x7f000015f1f0 2024-03-06T07:27:04.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.850+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0000153580 con 0x7f000015f1f0 2024-03-06T07:27:04.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.852+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f00000b4150 con 0x7f000015f1f0 2024-03-06T07:27:04.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.853+0000 7efff6000640 1 -- 172.21.15.86:0/3888347370 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7efff8047070 con 0x7f000015f1f0 2024-03-06T07:27:04.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.853+0000 7efff6000640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7effdc06b470 0x7effdc06d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:04.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.854+0000 7efff6000640 1 -- 172.21.15.86:0/3888347370 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 4857+0+0 (secure 0 0 0) 0x7efff80abff0 con 0x7f000015f1f0 2024-03-06T07:27:04.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.854+0000 7efffde00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7effdc06b470 0x7effdc06d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:04.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.854+0000 7efffde00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7effdc06b470 0x7effdc06d930 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7effe80011e0 tx=0x7effe8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:04.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:04.858+0000 7efff6000640 1 -- 172.21.15.86:0/3888347370 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f00000b4150 con 0x7f000015f1f0 2024-03-06T07:27:05.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.054+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f000004f330 con 0x7f000015f1f0 2024-03-06T07:27:05.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.875+0000 7efff6000640 1 -- 172.21.15.86:0/3888347370 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v19) v1 ==== 138+0+0 (secure 0 0 0) 0x7f0000147a20 con 0x7f000015f1f0 2024-03-06T07:27:05.878 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:27:05.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7effdc06b470 msgr2=0x7effdc06d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:05.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7effdc06b470 0x7effdc06d930 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7effe80011e0 tx=0x7effe8009040 comp rx=0 tx=0).stop 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f000015f1f0 msgr2=0x7f00001474e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f000015f1f0 0x7f00001474e0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7efff8009eb0 tx=0x7efff8004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 shutdown_connections 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7effdc06b470 0x7effdc06d930 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f000015f1f0 0x7f00001474e0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f000015adf0 0x7f0000152890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 --2- 172.21.15.86:0/3888347370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f000006eab0 0x7f00000cd470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:05.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.879+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 >> 172.21.15.86:0/3888347370 conn(0x7f00000cfa80 msgr2=0x7f000006e2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:05.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.880+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 shutdown_connections 2024-03-06T07:27:05.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:05.880+0000 7f0004c00640 1 -- 172.21.15.86:0/3888347370 wait complete. 2024-03-06T07:27:05.911 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:27:06.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.098+0000 7f3be6000640 1 Processor -- start 2024-03-06T07:27:06.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.098+0000 7f3be6000640 1 -- start start 2024-03-06T07:27:06.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3be6000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3be00b3f70 0x7f3be0160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:06.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3be6000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3be0070ff0 0x7f3be00ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:06.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3be6000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3be0071440 con 0x7f3be015fc10 2024-03-06T07:27:06.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3be6000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3be00b46c0 con 0x7f3be00b3f70 2024-03-06T07:27:06.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3be6000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3be00b49a0 con 0x7f3be0070ff0 2024-03-06T07:27:06.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bdf400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3be015fc10 0x7f3be015fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59190/0 (socket says 172.21.15.86:59190) 2024-03-06T07:27:06.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bdf400640 1 -- 172.21.15.86:0/260583984 learned_addr learned my addr 172.21.15.86:0/260583984 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:06.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bd6a00640 1 --2- 172.21.15.86:0/260583984 >> v2:172.21.15.86:3300/0 conn(0x7f3be00b3f70 0x7f3be0160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:06.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bdfe00640 1 --2- 172.21.15.86:0/260583984 >> v2:172.21.15.194:3300/0 conn(0x7f3be0070ff0 0x7f3be00ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:06.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bdea00640 1 -- 172.21.15.86:0/260583984 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3779894649 0 0) 0x7f3be0071440 con 0x7f3be015fc10 2024-03-06T07:27:06.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bdea00640 1 -- 172.21.15.86:0/260583984 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3bc4003520 con 0x7f3be015fc10 2024-03-06T07:27:06.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bd6a00640 1 -- 172.21.15.86:0/260583984 >> v2:172.21.15.194:3300/0 conn(0x7f3be0070ff0 msgr2=0x7f3be00ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:06.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bd6a00640 1 --2- 172.21.15.86:0/260583984 >> v2:172.21.15.194:3300/0 conn(0x7f3be0070ff0 0x7f3be00ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.100+0000 7f3bd6a00640 1 -- 172.21.15.86:0/260583984 >> v1:172.21.15.86:6789/0 conn(0x7f3be015fc10 legacy=0x7f3be015fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:06.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.101+0000 7f3bd6a00640 1 -- 172.21.15.86:0/260583984 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3be0072c90 con 0x7f3be00b3f70 2024-03-06T07:27:06.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.102+0000 7f3bd6a00640 1 --2- 172.21.15.86:0/260583984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be0160700 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f3be00cf6a0 tx=0x7f3bc802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=89daa6cd3c917734 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:06.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.102+0000 7f3bdea00640 1 -- 172.21.15.86:0/260583984 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bc8035020 con 0x7f3be00b3f70 2024-03-06T07:27:06.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.102+0000 7f3bdea00640 1 -- 172.21.15.86:0/260583984 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3bc8031830 con 0x7f3be00b3f70 2024-03-06T07:27:06.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.102+0000 7f3bdea00640 1 -- 172.21.15.86:0/260583984 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bc8031be0 con 0x7f3be00b3f70 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 -- 172.21.15.86:0/260583984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 msgr2=0x7f3be0160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 --2- 172.21.15.86:0/260583984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be0160700 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f3be00cf6a0 tx=0x7f3bc802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 -- 172.21.15.86:0/260583984 shutdown_connections 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 --2- 172.21.15.86:0/260583984 >> v2:172.21.15.194:3300/0 conn(0x7f3be0070ff0 0x7f3be00ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 --2- 172.21.15.86:0/260583984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be0160700 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 -- 172.21.15.86:0/260583984 >> 172.21.15.86:0/260583984 conn(0x7f3be00b1940 msgr2=0x7f3be00b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 -- 172.21.15.86:0/260583984 shutdown_connections 2024-03-06T07:27:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.103+0000 7f3be6000640 1 -- 172.21.15.86:0/260583984 wait complete. 2024-03-06T07:27:06.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.105+0000 7f3be6000640 1 Processor -- start 2024-03-06T07:27:06.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.105+0000 7f3be6000640 1 -- start start 2024-03-06T07:27:06.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3be6000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be0070ff0 0x7f3be00caa40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:06.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3be6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be00c8fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:06.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3be6000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015fc10 0x7f3be01573a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:06.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3be6000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3be0072a30 con 0x7f3be00b3f70 2024-03-06T07:27:06.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3be6000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3be00728b0 con 0x7f3be0070ff0 2024-03-06T07:27:06.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3be6000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3be015c2e0 con 0x7f3be015fc10 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3bd6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be00c8fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3bd6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be00c8fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53724/0 (socket says 172.21.15.86:53724) 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3bd6a00640 1 -- 172.21.15.86:0/438091897 learned_addr learned my addr 172.21.15.86:0/438091897 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.106+0000 7f3bdf400640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be0070ff0 0x7f3be00caa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd6a00640 1 -- 172.21.15.86:0/438091897 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015fc10 msgr2=0x7f3be01573a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd6a00640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015fc10 0x7f3be01573a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd6a00640 1 -- 172.21.15.86:0/438091897 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be0070ff0 msgr2=0x7f3be00caa40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd6a00640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be0070ff0 0x7f3be00caa40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd6a00640 1 -- 172.21.15.86:0/438091897 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3be00c9a80 con 0x7f3be00b3f70 2024-03-06T07:27:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd6a00640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be00c8fc0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8000c00 tx=0x7f3bc80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:06.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd7e00640 1 -- 172.21.15.86:0/438091897 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bc8041070 con 0x7f3be00b3f70 2024-03-06T07:27:06.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd7e00640 1 -- 172.21.15.86:0/438091897 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3bc803b860 con 0x7f3be00b3f70 2024-03-06T07:27:06.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3be00c9d60 con 0x7f3be00b3f70 2024-03-06T07:27:06.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3bd7e00640 1 -- 172.21.15.86:0/438091897 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bc8035040 con 0x7f3be00b3f70 2024-03-06T07:27:06.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.107+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3be0144570 con 0x7f3be00b3f70 2024-03-06T07:27:06.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.109+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3be00ca280 con 0x7f3be00b3f70 2024-03-06T07:27:06.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.110+0000 7f3bd7e00640 1 -- 172.21.15.86:0/438091897 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3bc8038020 con 0x7f3be00b3f70 2024-03-06T07:27:06.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.111+0000 7f3bd7e00640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:06.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.111+0000 7f3bd7e00640 1 -- 172.21.15.86:0/438091897 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(19..19 src has 1..19) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f3bc80ab9b0 con 0x7f3be00b3f70 2024-03-06T07:27:06.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.111+0000 7f3bdf400640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:06.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.112+0000 7f3bdf400640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f3bd0005880 tx=0x7f3bd0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:06.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.116+0000 7f3bd7e00640 1 -- 172.21.15.86:0/438091897 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3be00ca280 con 0x7f3be00b3f70 2024-03-06T07:27:06.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.313+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f3be004f330 con 0x7f3be00b3f70 2024-03-06T07:27:06.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.890+0000 7f3bd7e00640 1 -- 172.21.15.86:0/438091897 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v20) v1 ==== 170+0+0 (secure 0 0 0) 0x7f3bc8031800 con 0x7f3be00b3f70 2024-03-06T07:27:06.893 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:27:06.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 msgr2=0x7f3bbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:06.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f3bd0005880 tx=0x7f3bd0009040 comp rx=0 tx=0).stop 2024-03-06T07:27:06.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 msgr2=0x7f3be00c8fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:06.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be00c8fc0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8000c00 tx=0x7f3bc80046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 shutdown_connections 2024-03-06T07:27:06.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015fc10 0x7f3be01573a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be00b3f70 0x7f3be00c8fc0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 --2- 172.21.15.86:0/438091897 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be0070ff0 0x7f3be00caa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:06.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.894+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 >> 172.21.15.86:0/438091897 conn(0x7f3be00b1940 msgr2=0x7f3be00adef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:06.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.895+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 shutdown_connections 2024-03-06T07:27:06.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:06.895+0000 7f3be6000640 1 -- 172.21.15.86:0/438091897 wait complete. 2024-03-06T07:27:06.931 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:27:07.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.111+0000 7f273ee00640 1 Processor -- start 2024-03-06T07:27:07.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.111+0000 7f273ee00640 1 -- start start 2024-03-06T07:27:07.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f273ee00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f27380b3530 0x7f273815b780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:07.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f273ee00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380adb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:07.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f273ee00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f273815bcc0 con 0x7f27380b3910 2024-03-06T07:27:07.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f273ee00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f27380ce690 con 0x7f27380b3530 2024-03-06T07:27:07.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f273ee00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f27380ce970 con 0x7f27380ad670 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f2737e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f27380b3530 0x7f273815b780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f2737400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f27380b3910 0x7f27380acf60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59202/0 (socket says 172.21.15.86:59202) 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.112+0000 7f2737400640 1 -- 172.21.15.86:0/1238556370 learned_addr learned my addr 172.21.15.86:0/1238556370 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.113+0000 7f273ce00640 1 --2- 172.21.15.86:0/1238556370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380adb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.113+0000 7f2737e00640 1 -- 172.21.15.86:0/1238556370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 msgr2=0x7f27380adb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.113+0000 7f2737e00640 1 --2- 172.21.15.86:0/1238556370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380adb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.113+0000 7f2737e00640 1 -- 172.21.15.86:0/1238556370 >> v1:172.21.15.86:6789/0 conn(0x7f27380b3910 legacy=0x7f27380acf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.113+0000 7f2737e00640 1 -- 172.21.15.86:0/1238556370 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27380cec50 con 0x7f27380b3530 2024-03-06T07:27:07.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.113+0000 7f2736a00640 1 -- 172.21.15.86:0/1238556370 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1250776410 0 0) 0x7f2724001110 con 0x7f27380b3910 2024-03-06T07:27:07.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.114+0000 7f2737e00640 1 --2- 172.21.15.86:0/1238556370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f273815b780 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f27280097a0 tx=0x7f272802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=259500fe9c952eea server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:07.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.115+0000 7f2736a00640 1 -- 172.21.15.86:0/1238556370 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2728035020 con 0x7f27380b3530 2024-03-06T07:27:07.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.115+0000 7f2736a00640 1 -- 172.21.15.86:0/1238556370 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2728031830 con 0x7f27380b3530 2024-03-06T07:27:07.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.115+0000 7f2736a00640 1 -- 172.21.15.86:0/1238556370 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2728031c00 con 0x7f27380b3530 2024-03-06T07:27:07.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.115+0000 7f273ee00640 1 -- 172.21.15.86:0/1238556370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 msgr2=0x7f273815b780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:07.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.115+0000 7f273ee00640 1 --2- 172.21.15.86:0/1238556370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f273815b780 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f27280097a0 tx=0x7f272802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.116+0000 7f273ee00640 1 -- 172.21.15.86:0/1238556370 shutdown_connections 2024-03-06T07:27:07.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.116+0000 7f273ee00640 1 --2- 172.21.15.86:0/1238556370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380adb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.116+0000 7f273ee00640 1 --2- 172.21.15.86:0/1238556370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f273815b780 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.116+0000 7f273ee00640 1 -- 172.21.15.86:0/1238556370 >> 172.21.15.86:0/1238556370 conn(0x7f27380b1940 msgr2=0x7f27380b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:07.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.116+0000 7f273ee00640 1 -- 172.21.15.86:0/1238556370 shutdown_connections 2024-03-06T07:27:07.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.116+0000 7f273ee00640 1 -- 172.21.15.86:0/1238556370 wait complete. 2024-03-06T07:27:07.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.118+0000 7f273ee00640 1 Processor -- start 2024-03-06T07:27:07.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.118+0000 7f273ee00640 1 -- start start 2024-03-06T07:27:07.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f273ee00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380ae5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:07.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f273ee00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f27380aeb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:07.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f273ee00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f27380b3910 0x7f2738149fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:07.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f273ee00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27380b3d90 con 0x7f27380b3530 2024-03-06T07:27:07.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f273ee00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27380cf210 con 0x7f27380b3910 2024-03-06T07:27:07.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f273ee00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f273815aa30 con 0x7f27380ad670 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f2737400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f27380aeb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f2737400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f27380aeb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53750/0 (socket says 172.21.15.86:53750) 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.119+0000 7f2737400640 1 -- 172.21.15.86:0/927898220 learned_addr learned my addr 172.21.15.86:0/927898220 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f2737e00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380ae5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f273ce00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f27380b3910 0x7f2738149fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f2737400640 1 -- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 msgr2=0x7f27380ae5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f2737400640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380ae5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f2737400640 1 -- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f27380b3910 msgr2=0x7f2738149fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f2737400640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f27380b3910 0x7f2738149fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f2737400640 1 -- 172.21.15.86:0/927898220 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f273806cbe0 con 0x7f27380b3530 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f2737400640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f27380aeb10 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f272400c460 tx=0x7f272400c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f272fe00640 1 -- 172.21.15.86:0/927898220 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2724018070 con 0x7f27380b3530 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f272fe00640 1 -- 172.21.15.86:0/927898220 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2724007500 con 0x7f27380b3530 2024-03-06T07:27:07.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f272fe00640 1 -- 172.21.15.86:0/927898220 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2724004a20 con 0x7f27380b3530 2024-03-06T07:27:07.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.120+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f273806cec0 con 0x7f27380b3530 2024-03-06T07:27:07.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.121+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f273806d480 con 0x7f27380b3530 2024-03-06T07:27:07.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.122+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f273804f330 con 0x7f27380b3530 2024-03-06T07:27:07.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.123+0000 7f272fe00640 1 -- 172.21.15.86:0/927898220 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f272401b070 con 0x7f27380b3530 2024-03-06T07:27:07.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.124+0000 7f272fe00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f271806b4d0 0x7f271806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:07.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.124+0000 7f272fe00640 1 -- 172.21.15.86:0/927898220 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f2724089480 con 0x7f27380b3530 2024-03-06T07:27:07.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.124+0000 7f2737e00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f271806b4d0 0x7f271806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:07.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.125+0000 7f2737e00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f271806b4d0 0x7f271806d990 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f2728036000 tx=0x7f2728008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:07.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.129+0000 7f272fe00640 1 -- 172.21.15.86:0/927898220 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f273815bcc0 con 0x7f27380b3530 2024-03-06T07:27:07.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.362+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f27380cd5e0 con 0x7f27380b3530 2024-03-06T07:27:07.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.906+0000 7f272fe00640 1 -- 172.21.15.86:0/927898220 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v4) v1 ==== 172+0+0 (secure 0 0 0) 0x7f2724058350 con 0x7f27380b3530 2024-03-06T07:27:07.911 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 2 and data pool 3 2024-03-06T07:27:07.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.915+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f271806b4d0 msgr2=0x7f271806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:07.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.915+0000 7f273ee00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f271806b4d0 0x7f271806d990 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f2728036000 tx=0x7f2728008030 comp rx=0 tx=0).stop 2024-03-06T07:27:07.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.915+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 msgr2=0x7f27380aeb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:07.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.915+0000 7f273ee00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f27380aeb10 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f272400c460 tx=0x7f272400c920 comp rx=0 tx=0).stop 2024-03-06T07:27:07.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 shutdown_connections 2024-03-06T07:27:07.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f271806b4d0 0x7f271806d990 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f27380b3910 0x7f2738149fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f27380b3530 0x7f27380aeb10 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 --2- 172.21.15.86:0/927898220 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f27380ad670 0x7f27380ae5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:07.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 >> 172.21.15.86:0/927898220 conn(0x7f27380b1940 msgr2=0x7f27380cbed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:07.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 shutdown_connections 2024-03-06T07:27:07.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:07.916+0000 7f273ee00640 1 -- 172.21.15.86:0/927898220 wait complete. 2024-03-06T07:27:07.950 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:08.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.124+0000 7f70aee00640 1 Processor -- start 2024-03-06T07:27:08.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.124+0000 7f70aee00640 1 -- start start 2024-03-06T07:27:08.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.126+0000 7f70aee00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 0x7f70a80d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.126+0000 7f70aee00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f70a80cea60 con 0x7f70a815f470 2024-03-06T07:27:08.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.126+0000 7f70aee00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f70a80d2350 con 0x7f70a80d0fc0 2024-03-06T07:27:08.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.126+0000 7f70aee00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70a80d2f60 con 0x7f70a815e0d0 2024-03-06T07:27:08.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.126+0000 7f709f400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f70a815f470 0x7f70a80d08b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59206/0 (socket says 172.21.15.86:59206) 2024-03-06T07:27:08.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.126+0000 7f709f400640 1 -- 172.21.15.86:0/2791596158 learned_addr learned my addr 172.21.15.86:0/2791596158 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:08.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2099937168 0 0) 0x7f70a80cea60 con 0x7f70a815f470 2024-03-06T07:27:08.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f708c003520 con 0x7f70a815f470 2024-03-06T07:27:08.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7e00640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 0x7f70a80d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4189638418 0 0) 0x7f708c003520 con 0x7f70a815f470 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 msgr2=0x7f70a80d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 0x7f70a80d1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 >> v1:172.21.15.194:6789/0 conn(0x7f70a80d0fc0 legacy=0x7f70a8160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70a80d2630 con 0x7f70a815f470 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7090001670 con 0x7f70a815f470 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 >> v1:172.21.15.86:6789/0 conn(0x7f70a815f470 legacy=0x7f70a80d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c0039e0 0x7f708c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f708c004800 0x7f708c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 0x7f708c005200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f708c00bb50 con 0x7f708c004800 2024-03-06T07:27:08.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f708c00bd00 con 0x7f708c0039e0 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f708c00bf00 con 0x7f70a815e0d0 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.127+0000 7f709f400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f708c004800 0x7f708c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f70a7e00640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c0039e0 0x7f708c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f70ace00640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 0x7f708c005200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f709f400640 1 -- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 msgr2=0x7f708c005200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f709f400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 0x7f708c005200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f709f400640 1 -- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c0039e0 msgr2=0x7f708c003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f709f400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c0039e0 0x7f708c003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f709f400640 1 -- 172.21.15.86:0/2791596158 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f708c003520 con 0x7f708c004800 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f70ace00640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a815e0d0 0x7f708c005200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.128+0000 7f70a7e00640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c0039e0 0x7f708c003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.129+0000 7f709f400640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f708c004800 0x7f708c004c80 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f709000c210 tx=0x7f708c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c2d97e459ea30e2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.129+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7090007d30 con 0x7f708c004800 2024-03-06T07:27:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.129+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7090030070 con 0x7f708c004800 2024-03-06T07:27:08.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.129+0000 7f70a7400640 1 -- 172.21.15.86:0/2791596158 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f709002cd60 con 0x7f708c004800 2024-03-06T07:27:08.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.129+0000 7f70aee00640 1 -- 172.21.15.86:0/2791596158 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f708c004800 msgr2=0x7f708c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.129+0000 7f70aee00640 1 --2- 172.21.15.86:0/2791596158 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f708c004800 0x7f708c004c80 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f709000c210 tx=0x7f708c005830 comp rx=0 tx=0).stop 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.130+0000 7f70ace00640 1 -- 172.21.15.86:0/2791596158 reap_dead start 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.130+0000 7f70aee00640 1 -- 172.21.15.86:0/2791596158 shutdown_connections 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.130+0000 7f70aee00640 1 -- 172.21.15.86:0/2791596158 >> 172.21.15.86:0/2791596158 conn(0x7f70a80d3ec0 msgr2=0x7f70a80d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.130+0000 7f70aee00640 1 -- 172.21.15.86:0/2791596158 shutdown_connections 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.130+0000 7f70aee00640 1 -- 172.21.15.86:0/2791596158 wait complete. 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.131+0000 7f70aee00640 1 Processor -- start 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.131+0000 7f70aee00640 1 -- start start 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.132+0000 7f70aee00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c004800 0x7f70a815c700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.132+0000 7f70aee00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70a809f040 0x7f70a80c97d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.132+0000 7f70aee00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a809f4c0 0x7f70a81f4e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.132+0000 7f70aee00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70a80cff10 con 0x7f70a809f040 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.132+0000 7f70aee00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70a80d2bf0 con 0x7f708c004800 2024-03-06T07:27:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.132+0000 7f70aee00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70a815fb30 con 0x7f70a809f4c0 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70a809f040 0x7f70a80c97d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70a809f040 0x7f70a80c97d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53772/0 (socket says 172.21.15.86:53772) 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 -- 172.21.15.86:0/2297650856 learned_addr learned my addr 172.21.15.86:0/2297650856 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f70ace00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a809f4c0 0x7f70a81f4e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f70a7e00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c004800 0x7f70a815c700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 -- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a809f4c0 msgr2=0x7f70a81f4e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a809f4c0 0x7f70a81f4e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 -- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c004800 msgr2=0x7f70a815c700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c004800 0x7f70a815c700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 -- 172.21.15.86:0/2297650856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70a809dfa0 con 0x7f70a809f040 2024-03-06T07:27:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.133+0000 7f709f400640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70a809f040 0x7f70a80c97d0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f709000c330 tx=0x7f7090001a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.134+0000 7f70a4c00640 1 -- 172.21.15.86:0/2297650856 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7090046070 con 0x7f70a809f040 2024-03-06T07:27:08.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.134+0000 7f70a4c00640 1 -- 172.21.15.86:0/2297650856 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7090004bf0 con 0x7f70a809f040 2024-03-06T07:27:08.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.134+0000 7f70a4c00640 1 -- 172.21.15.86:0/2297650856 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7090007d10 con 0x7f70a809f040 2024-03-06T07:27:08.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.134+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70a809e280 con 0x7f70a809f040 2024-03-06T07:27:08.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.134+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70a809e840 con 0x7f70a809f040 2024-03-06T07:27:08.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.135+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f70a804f330 con 0x7f70a809f040 2024-03-06T07:27:08.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.136+0000 7f70a4c00640 1 -- 172.21.15.86:0/2297650856 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7090007070 con 0x7f70a809f040 2024-03-06T07:27:08.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.137+0000 7f70a4c00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f708406b550 0x7f708406da10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.137+0000 7f70a4c00640 1 -- 172.21.15.86:0/2297650856 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(21..21 src has 1..21) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f70900ab6c0 con 0x7f70a809f040 2024-03-06T07:27:08.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.137+0000 7f70a7e00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f708406b550 0x7f708406da10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.137+0000 7f70a7e00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f708406b550 0x7f708406da10 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f7098006fd0 tx=0x7f7098008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.141+0000 7f70a4c00640 1 -- 172.21.15.86:0/2297650856 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f709007a520 con 0x7f70a809f040 2024-03-06T07:27:08.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.322+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f70a808d920 con 0x7f70a809f040 2024-03-06T07:27:08.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.325+0000 7f70a4c00640 1 -- 172.21.15.86:0/2297650856 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v21) v1 ==== 74+0+16630 (secure 0 0 0) 0x7f7090079e20 con 0x7f70a809f040 2024-03-06T07:27:08.328 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:08.328 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":21,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:07.364351+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:08.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f708406b550 msgr2=0x7f708406da10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f708406b550 0x7f708406da10 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f7098006fd0 tx=0x7f7098008040 comp rx=0 tx=0).stop 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70a809f040 msgr2=0x7f70a80c97d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70a809f040 0x7f70a80c97d0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f709000c330 tx=0x7f7090001a40 comp rx=0 tx=0).stop 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 shutdown_connections 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f708406b550 0x7f708406da10 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70a809f4c0 0x7f70a81f4e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70a809f040 0x7f70a80c97d0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 --2- 172.21.15.86:0/2297650856 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f708c004800 0x7f70a815c700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.327+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 >> 172.21.15.86:0/2297650856 conn(0x7f70a80d3ec0 msgr2=0x7f70a806d370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.328+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 shutdown_connections 2024-03-06T07:27:08.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.328+0000 7f70aee00640 1 -- 172.21.15.86:0/2297650856 wait complete. 2024-03-06T07:27:08.346 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:08.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.535+0000 7f03db600640 1 Processor -- start 2024-03-06T07:27:08.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.535+0000 7f03db600640 1 -- start start 2024-03-06T07:27:08.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.537+0000 7f03db600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f03d40af1c0 0x7f03d415eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.537+0000 7f03db600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d406e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.537+0000 7f03db600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f03d4071d30 con 0x7f03d4070d20 2024-03-06T07:27:08.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.537+0000 7f03db600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f03d40711d0 con 0x7f03d40af1c0 2024-03-06T07:27:08.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.537+0000 7f03db600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03d4071490 con 0x7f03d406e4d0 2024-03-06T07:27:08.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.538+0000 7f03d8c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f03d40af1c0 0x7f03d415eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.538+0000 7f03d9600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d406e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.538+0000 7f03d8c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f03d40af1c0 0x7f03d415eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51590/0 (socket says 172.21.15.86:51590) 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.538+0000 7f03d9600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d406e960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:54866/0 (socket says 172.21.15.86:54866) 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.538+0000 7f03d8c00640 1 -- 172.21.15.86:0/1905589888 learned_addr learned my addr 172.21.15.86:0/1905589888 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.539+0000 7f03d8c00640 1 -- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 msgr2=0x7f03d406e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.539+0000 7f03d8c00640 1 --2- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d406e960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.539+0000 7f03d8c00640 1 -- 172.21.15.86:0/1905589888 >> v1:172.21.15.194:6789/0 conn(0x7f03d4070d20 legacy=0x7f03d406cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.539+0000 7f03d3e00640 1 -- 172.21.15.86:0/1905589888 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1644430585 0 0) 0x7f03bc001110 con 0x7f03d4070d20 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.539+0000 7f03d8c00640 1 -- 172.21.15.86:0/1905589888 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03d4071770 con 0x7f03d40af1c0 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.539+0000 7f03d9600640 1 --2- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d406e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.540+0000 7f03d8c00640 1 --2- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d40af1c0 0x7f03d415eff0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f03c40098a0 tx=0x7f03c402b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2f06e38e9e70663a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.541+0000 7f03d3e00640 1 -- 172.21.15.86:0/1905589888 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03c4034070 con 0x7f03d40af1c0 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.541+0000 7f03d3e00640 1 -- 172.21.15.86:0/1905589888 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03c4030840 con 0x7f03d40af1c0 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.541+0000 7f03d3e00640 1 -- 172.21.15.86:0/1905589888 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03c4030c10 con 0x7f03d40af1c0 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 -- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d40af1c0 msgr2=0x7f03d415eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 --2- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d40af1c0 0x7f03d415eff0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f03c40098a0 tx=0x7f03c402b980 comp rx=0 tx=0).stop 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 -- 172.21.15.86:0/1905589888 shutdown_connections 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 --2- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d406e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 --2- 172.21.15.86:0/1905589888 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d40af1c0 0x7f03d415eff0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 -- 172.21.15.86:0/1905589888 >> 172.21.15.86:0/1905589888 conn(0x7f03d40b1730 msgr2=0x7f03d40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 -- 172.21.15.86:0/1905589888 shutdown_connections 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.542+0000 7f03db600640 1 -- 172.21.15.86:0/1905589888 wait complete. 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.544+0000 7f03db600640 1 Processor -- start 2024-03-06T07:27:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.544+0000 7f03db600640 1 -- start start 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03db600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d41e7680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03db600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d4070d20 0x7f03d414a7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03db600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03d40af1c0 0x7f03d414ad30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03db600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03d406d370 con 0x7f03d40af1c0 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03db600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03d40b30b0 con 0x7f03d4070d20 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03db600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03d40b35d0 con 0x7f03d406e4d0 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03d40af1c0 0x7f03d414ad30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03d40af1c0 0x7f03d414ad30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53786/0 (socket says 172.21.15.86:53786) 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 -- 172.21.15.86:0/2305799439 learned_addr learned my addr 172.21.15.86:0/2305799439 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03cbe00640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d4070d20 0x7f03d414a7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d8c00640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d41e7680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 -- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 msgr2=0x7f03d41e7680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d41e7680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 -- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d4070d20 msgr2=0x7f03d414a7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d4070d20 0x7f03d414a7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.545+0000 7f03d9600640 1 -- 172.21.15.86:0/2305799439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03d41f41c0 con 0x7f03d40af1c0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.546+0000 7f03d9600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03d40af1c0 0x7f03d414ad30 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f03cc009f00 tx=0x7f03cc00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.546+0000 7f03d1600640 1 -- 172.21.15.86:0/2305799439 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03cc018030 con 0x7f03d40af1c0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.546+0000 7f03d1600640 1 -- 172.21.15.86:0/2305799439 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03cc00ce50 con 0x7f03d40af1c0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.546+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03d41e6ac0 con 0x7f03d40af1c0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.546+0000 7f03d1600640 1 -- 172.21.15.86:0/2305799439 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03cc003bb0 con 0x7f03d40af1c0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.546+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03d414b670 con 0x7f03d40af1c0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.547+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f03d404f330 con 0x7f03d40af1c0 2024-03-06T07:27:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.548+0000 7f03d1600640 1 -- 172.21.15.86:0/2305799439 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f03cc005d10 con 0x7f03d40af1c0 2024-03-06T07:27:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.549+0000 7f03d1600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03b006b4d0 0x7f03b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.549+0000 7f03d1600640 1 -- 172.21.15.86:0/2305799439 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(21..21 src has 1..21) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f03cc08acd0 con 0x7f03d40af1c0 2024-03-06T07:27:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.549+0000 7f03d8c00640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03b006b4d0 0x7f03b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.550+0000 7f03d8c00640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03b006b4d0 0x7f03b006d990 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f03c4036000 tx=0x7f03c403b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.553+0000 7f03d1600640 1 -- 172.21.15.86:0/2305799439 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f03cc022070 con 0x7f03d40af1c0 2024-03-06T07:27:08.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.750+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f03d40af740 con 0x7f03d40af1c0 2024-03-06T07:27:08.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.751+0000 7f03d1600640 1 -- 172.21.15.86:0/2305799439 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v21) v1 ==== 74+0+16630 (secure 0 0 0) 0x7f03d41f41c0 con 0x7f03d40af1c0 2024-03-06T07:27:08.754 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:08.754 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":21,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:07.364351+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03b006b4d0 msgr2=0x7f03b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03b006b4d0 0x7f03b006d990 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f03c4036000 tx=0x7f03c403b040 comp rx=0 tx=0).stop 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03d40af1c0 msgr2=0x7f03d414ad30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03d40af1c0 0x7f03d414ad30 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f03cc009f00 tx=0x7f03cc00c490 comp rx=0 tx=0).stop 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 shutdown_connections 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03b006b4d0 0x7f03b006d990 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03d40af1c0 0x7f03d414ad30 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03d4070d20 0x7f03d414a7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 --2- 172.21.15.86:0/2305799439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03d406e4d0 0x7f03d41e7680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 >> 172.21.15.86:0/2305799439 conn(0x7f03d40b1730 msgr2=0x7f03d40b39b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:08.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 shutdown_connections 2024-03-06T07:27:08.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.754+0000 7f03db600640 1 -- 172.21.15.86:0/2305799439 wait complete. 2024-03-06T07:27:08.775 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:27:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.980+0000 7f7ea9200640 1 Processor -- start 2024-03-06T07:27:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.980+0000 7f7ea9200640 1 -- start start 2024-03-06T07:27:08.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.981+0000 7f7ea9200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ea415fc10 0x7f7ea4070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.981+0000 7f7ea9200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ea40b46c0 con 0x7f7ea40b3f70 2024-03-06T07:27:08.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.981+0000 7f7ea9200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ea40b49a0 con 0x7f7ea415fff0 2024-03-06T07:27:08.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.981+0000 7f7ea9200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea4072c90 con 0x7f7ea415fc10 2024-03-06T07:27:08.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea2800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7ea40b3f70 0x7f7ea40ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59218/0 (socket says 172.21.15.86:59218) 2024-03-06T07:27:08.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea2800640 1 -- 172.21.15.86:0/2417528192 learned_addr learned my addr 172.21.15.86:0/2417528192 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ea415fc10 0x7f7ea4070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2805209990 0 0) 0x7f7ea40b46c0 con 0x7f7ea40b3f70 2024-03-06T07:27:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7e88003520 con 0x7f7ea40b3f70 2024-03-06T07:27:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2436433113 0 0) 0x7f7e88003520 con 0x7f7ea40b3f70 2024-03-06T07:27:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ea415fc10 msgr2=0x7f7ea4070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ea415fc10 0x7f7ea4070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 >> v1:172.21.15.194:6789/0 conn(0x7f7ea415fff0 legacy=0x7f7ea4160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ea409dd80 con 0x7f7ea40b3f70 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.982+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7e9c0019f0 con 0x7f7ea40b3f70 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 >> v1:172.21.15.86:6789/0 conn(0x7f7ea40b3f70 legacy=0x7f7ea40ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1400640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e880039e0 0x7f7e88003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1400640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88004800 0x7f7e88004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1400640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ea415fc10 0x7f7e880051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea4072c90 con 0x7f7e880039e0 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e8800baa0 con 0x7f7e88004800 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e8800bca0 con 0x7f7ea415fc10 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e880039e0 0x7f7e88003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1e00640 1 -- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ea415fc10 msgr2=0x7f7e880051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ea415fc10 0x7f7e880051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1e00640 1 -- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88004800 msgr2=0x7f7e88004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88004800 0x7f7e88004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.983+0000 7f7ea1e00640 1 -- 172.21.15.86:0/2417528192 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e88003520 con 0x7f7e880039e0 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.984+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e880039e0 0x7f7e88003e00 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f7e94009a90 tx=0x7f7e88005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=23b41545f8f1a028 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.984+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e94039070 con 0x7f7e880039e0 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.984+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e9402c690 con 0x7f7e880039e0 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.984+0000 7f7ea1400640 1 -- 172.21.15.86:0/2417528192 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e94009690 con 0x7f7e880039e0 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.985+0000 7f7ea9200640 1 -- 172.21.15.86:0/2417528192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e880039e0 msgr2=0x7f7e88003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.985+0000 7f7ea9200640 1 --2- 172.21.15.86:0/2417528192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e880039e0 0x7f7e88003e00 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f7e94009a90 tx=0x7f7e88005830 comp rx=0 tx=0).stop 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.985+0000 7f7ea2800640 1 -- 172.21.15.86:0/2417528192 reap_dead start 2024-03-06T07:27:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.985+0000 7f7ea9200640 1 -- 172.21.15.86:0/2417528192 shutdown_connections 2024-03-06T07:27:08.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.985+0000 7f7ea9200640 1 -- 172.21.15.86:0/2417528192 >> 172.21.15.86:0/2417528192 conn(0x7f7ea40b1940 msgr2=0x7f7ea40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:08.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.985+0000 7f7ea9200640 1 -- 172.21.15.86:0/2417528192 shutdown_connections 2024-03-06T07:27:08.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.985+0000 7f7ea9200640 1 -- 172.21.15.86:0/2417528192 wait complete. 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.987+0000 7f7ea9200640 1 Processor -- start 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.987+0000 7f7ea9200640 1 -- start start 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea9200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e880039e0 0x7f7ea41f7be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea9200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ea4145090 0x7f7ea41f8120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ea41f41a0 0x7f7ea415a890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea9200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea415c460 con 0x7f7ea41f41a0 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea9200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea415c2e0 con 0x7f7ea4145090 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea9200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea4070ff0 con 0x7f7e880039e0 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ea41f41a0 0x7f7ea415a890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ea41f41a0 0x7f7ea415a890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53798/0 (socket says 172.21.15.86:53798) 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7ea2800640 1 -- 172.21.15.86:0/2453497698 learned_addr learned my addr 172.21.15.86:0/2453497698 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:08.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.988+0000 7f7e99400640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ea4145090 0x7f7ea41f8120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e880039e0 0x7f7ea41f7be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea2800640 1 -- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e880039e0 msgr2=0x7f7ea41f7be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea2800640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e880039e0 0x7f7ea41f7be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea2800640 1 -- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ea4145090 msgr2=0x7f7ea41f8120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea2800640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ea4145090 0x7f7ea41f8120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea2800640 1 -- 172.21.15.86:0/2453497698 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ea415b5c0 con 0x7f7ea41f41a0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea2800640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ea41f41a0 0x7f7ea415a890 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f7ea4070bc0 tx=0x7f7e9c00cf00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7e9aa00640 1 -- 172.21.15.86:0/2453497698 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e9c00a5b0 con 0x7f7ea41f41a0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7e9aa00640 1 -- 172.21.15.86:0/2453497698 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e9c00a770 con 0x7f7ea41f41a0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7ea415b8a0 con 0x7f7ea41f41a0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7e9aa00640 1 -- 172.21.15.86:0/2453497698 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e9c00ab60 con 0x7f7ea41f41a0 2024-03-06T07:27:08.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.989+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ea409b5c0 con 0x7f7ea41f41a0 2024-03-06T07:27:08.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.990+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7ea404f330 con 0x7f7ea41f41a0 2024-03-06T07:27:08.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.991+0000 7f7e9aa00640 1 -- 172.21.15.86:0/2453497698 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7e9c011070 con 0x7f7ea41f41a0 2024-03-06T07:27:08.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.992+0000 7f7e9aa00640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e8006b500 0x7f7e8006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:08.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.992+0000 7f7e9aa00640 1 -- 172.21.15.86:0/2453497698 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7e9c089ce0 con 0x7f7ea41f41a0 2024-03-06T07:27:08.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.992+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e8006b500 0x7f7e8006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:08.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.993+0000 7f7ea1e00640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e8006b500 0x7f7e8006d9c0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f7e94036000 tx=0x7f7e94030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:08.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:08.996+0000 7f7e9aa00640 1 -- 172.21.15.86:0/2453497698 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7e9c01a060 con 0x7f7ea41f41a0 2024-03-06T07:27:09.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.234+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f7ea406ef30 con 0x7f7ea41f41a0 2024-03-06T07:27:09.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.915+0000 7f7e9aa00640 1 -- 172.21.15.86:0/2453497698 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v7) v1 ==== 118+0+0 (secure 0 0 0) 0x7f7e9c00f260 con 0x7f7ea41f41a0 2024-03-06T07:27:09.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e8006b500 msgr2=0x7f7e8006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:09.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e8006b500 0x7f7e8006d9c0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f7e94036000 tx=0x7f7e94030000 comp rx=0 tx=0).stop 2024-03-06T07:27:09.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ea41f41a0 msgr2=0x7f7ea415a890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:09.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ea41f41a0 0x7f7ea415a890 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f7ea4070bc0 tx=0x7f7e9c00cf00 comp rx=0 tx=0).stop 2024-03-06T07:27:09.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 shutdown_connections 2024-03-06T07:27:09.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e8006b500 0x7f7e8006d9c0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:09.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ea41f41a0 0x7f7ea415a890 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:09.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ea4145090 0x7f7ea41f8120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:09.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 --2- 172.21.15.86:0/2453497698 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e880039e0 0x7f7ea41f7be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:09.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 >> 172.21.15.86:0/2453497698 conn(0x7f7ea40b1940 msgr2=0x7f7ea41574a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:09.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 shutdown_connections 2024-03-06T07:27:09.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:09.919+0000 7f7ea9200640 1 -- 172.21.15.86:0/2453497698 wait complete. 2024-03-06T07:27:09.942 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:27:09.942 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:27:10.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.152+0000 7f6679000640 1 Processor -- start 2024-03-06T07:27:10.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.153+0000 7f6679000640 1 -- start start 2024-03-06T07:27:10.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.154+0000 7f6679000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6674161b50 0x7f66740d05a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.154+0000 7f6679000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740d0ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.154+0000 7f6679000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f66740b5490 0x7f66740d1d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.154+0000 7f6679000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f66740cef40 con 0x7f66740b5490 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.154+0000 7f6679000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f66740cecd0 con 0x7f6674161b50 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.154+0000 7f6679000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66740cfc80 con 0x7f66740d4a00 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.154+0000 7f6672800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f66740b5490 0x7f66740d1d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6672800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f66740b5490 0x7f66740d1d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53800/0 (socket says 172.21.15.86:53800) 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6672800640 1 -- 172.21.15.86:0/3100543338 learned_addr learned my addr 172.21.15.86:0/3100543338 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6671400640 1 --2- 172.21.15.86:0/3100543338 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740d0ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6671e00640 1 --2- 172.21.15.86:0/3100543338 >> v2:172.21.15.194:3300/0 conn(0x7f6674161b50 0x7f66740d05a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:10.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6672800640 1 -- 172.21.15.86:0/3100543338 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 msgr2=0x7f66740d0ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:10.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6672800640 1 --2- 172.21.15.86:0/3100543338 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740d0ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6672800640 1 -- 172.21.15.86:0/3100543338 >> v2:172.21.15.194:3300/0 conn(0x7f6674161b50 msgr2=0x7f66740d05a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:10.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6672800640 1 --2- 172.21.15.86:0/3100543338 >> v2:172.21.15.194:3300/0 conn(0x7f6674161b50 0x7f66740d05a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6672800640 1 -- 172.21.15.86:0/3100543338 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66740c7b60 con 0x7f66740b5490 2024-03-06T07:27:10.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.155+0000 7f6671e00640 1 --2- 172.21.15.86:0/3100543338 >> v2:172.21.15.194:3300/0 conn(0x7f6674161b50 0x7f66740d05a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:10.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.157+0000 7f6672800640 1 --2- 172.21.15.86:0/3100543338 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66740b5490 0x7f66740d1d70 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f666c009870 tx=0x7f666c02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fdfca01a30f7add3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:10.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.157+0000 7f6670a00640 1 -- 172.21.15.86:0/3100543338 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f666c035020 con 0x7f66740b5490 2024-03-06T07:27:10.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.157+0000 7f6670a00640 1 -- 172.21.15.86:0/3100543338 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f666c031640 con 0x7f66740b5490 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.157+0000 7f6670a00640 1 -- 172.21.15.86:0/3100543338 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f666c02d040 con 0x7f66740b5490 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.157+0000 7f6679000640 1 -- 172.21.15.86:0/3100543338 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66740b5490 msgr2=0x7f66740d1d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.157+0000 7f6679000640 1 --2- 172.21.15.86:0/3100543338 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66740b5490 0x7f66740d1d70 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f666c009870 tx=0x7f666c02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.158+0000 7f6679000640 1 -- 172.21.15.86:0/3100543338 shutdown_connections 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.158+0000 7f6679000640 1 --2- 172.21.15.86:0/3100543338 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66740b5490 0x7f66740d1d70 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.158+0000 7f6679000640 1 --2- 172.21.15.86:0/3100543338 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740d0ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.158+0000 7f6679000640 1 --2- 172.21.15.86:0/3100543338 >> v2:172.21.15.194:3300/0 conn(0x7f6674161b50 0x7f66740d05a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.158+0000 7f6679000640 1 -- 172.21.15.86:0/3100543338 >> 172.21.15.86:0/3100543338 conn(0x7f66740d3e70 msgr2=0x7f66740d4280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.158+0000 7f6679000640 1 -- 172.21.15.86:0/3100543338 shutdown_connections 2024-03-06T07:27:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.158+0000 7f6679000640 1 -- 172.21.15.86:0/3100543338 wait complete. 2024-03-06T07:27:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.159+0000 7f6679000640 1 Processor -- start 2024-03-06T07:27:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.159+0000 7f6679000640 1 -- start start 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.160+0000 7f6679000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66740b5490 0x7f667415af10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.160+0000 7f6679000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740cc9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.160+0000 7f6679000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6674161b50 0x7f66740b8bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.160+0000 7f6679000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66740d1230 con 0x7f6674161b50 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.160+0000 7f6679000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66740d22b0 con 0x7f66740b5490 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.160+0000 7f6679000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f667415d760 con 0x7f66740d4a00 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6674161b50 0x7f66740b8bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6674161b50 0x7f66740b8bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53804/0 (socket says 172.21.15.86:53804) 2024-03-06T07:27:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 -- 172.21.15.86:0/3435794650 learned_addr learned my addr 172.21.15.86:0/3435794650 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6671400640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740cc9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6671e00640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66740b5490 0x7f667415af10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 -- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 msgr2=0x7f66740cc9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740cc9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 -- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66740b5490 msgr2=0x7f667415af10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66740b5490 0x7f667415af10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 -- 172.21.15.86:0/3435794650 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f667415da40 con 0x7f6674161b50 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6671e00640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66740b5490 0x7f667415af10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6671400640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740cc9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.161+0000 7f6672800640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6674161b50 0x7f66740b8bc0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f666c0097c0 tx=0x7f666c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.162+0000 7f666a000640 1 -- 172.21.15.86:0/3435794650 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f666c040070 con 0x7f6674161b50 2024-03-06T07:27:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.162+0000 7f666a000640 1 -- 172.21.15.86:0/3435794650 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f666c0399f0 con 0x7f6674161b50 2024-03-06T07:27:10.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.162+0000 7f666a000640 1 -- 172.21.15.86:0/3435794650 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f666c035040 con 0x7f6674161b50 2024-03-06T07:27:10.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.162+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f667415dd20 con 0x7f6674161b50 2024-03-06T07:27:10.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.162+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f667415e2e0 con 0x7f6674161b50 2024-03-06T07:27:10.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.163+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f667404f330 con 0x7f6674161b50 2024-03-06T07:27:10.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.164+0000 7f666a000640 1 -- 172.21.15.86:0/3435794650 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f666c04b060 con 0x7f6674161b50 2024-03-06T07:27:10.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.165+0000 7f666a000640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f665006b4d0 0x7f665006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:10.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.165+0000 7f666a000640 1 -- 172.21.15.86:0/3435794650 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f666c0abb30 con 0x7f6674161b50 2024-03-06T07:27:10.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.165+0000 7f6671e00640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f665006b4d0 0x7f665006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:10.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.166+0000 7f6671e00640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f665006b4d0 0x7f665006d990 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f665c006fd0 tx=0x7f665c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:10.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.169+0000 7f666a000640 1 -- 172.21.15.86:0/3435794650 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f666c07a910 con 0x7f6674161b50 2024-03-06T07:27:10.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.404+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f66740d16d0 con 0x7f6674161b50 2024-03-06T07:27:10.927 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.924+0000 7f666a000640 1 -- 172.21.15.86:0/3435794650 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v8) v1 ==== 122+0+0 (secure 0 0 0) 0x7f666c07a210 con 0x7f6674161b50 2024-03-06T07:27:10.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f665006b4d0 msgr2=0x7f665006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:10.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f665006b4d0 0x7f665006d990 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f665c006fd0 tx=0x7f665c008040 comp rx=0 tx=0).stop 2024-03-06T07:27:10.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6674161b50 msgr2=0x7f66740b8bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:10.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6674161b50 0x7f66740b8bc0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f666c0097c0 tx=0x7f666c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 shutdown_connections 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f665006b4d0 0x7f665006d990 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6674161b50 0x7f66740b8bc0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66740d4a00 0x7f66740cc9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 --2- 172.21.15.86:0/3435794650 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66740b5490 0x7f667415af10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.928+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 >> 172.21.15.86:0/3435794650 conn(0x7f66740d3e70 msgr2=0x7f667406b970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.929+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 shutdown_connections 2024-03-06T07:27:10.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:10.929+0000 7f6679000640 1 -- 172.21.15.86:0/3435794650 wait complete. 2024-03-06T07:27:10.953 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:27:10.953 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:11.147 INFO:tasks.ceph.mds.b.smithi194.stderr:2024-03-06T07:27:11.145+0000 7fcc76200640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:27:11.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.157+0000 7f72f5400640 1 Processor -- start 2024-03-06T07:27:11.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.157+0000 7f72f5400640 1 -- start start 2024-03-06T07:27:11.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72f5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f015e230 0x7f72f006edb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72f5400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f72f00ce550 0x7f72f00ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72f5400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72f00cf340 con 0x7f72f00b4320 2024-03-06T07:27:11.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72f5400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f72f015e610 con 0x7f72f00ce550 2024-03-06T07:27:11.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72f5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72f00cfe60 con 0x7f72f015e230 2024-03-06T07:27:11.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72ef200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f72f00ce550 0x7f72f00ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72ef200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f72f00ce550 0x7f72f00ce9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53816/0 (socket says 172.21.15.86:53816) 2024-03-06T07:27:11.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72ef200640 1 -- 172.21.15.86:0/4181237350 learned_addr learned my addr 172.21.15.86:0/4181237350 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:11.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.159+0000 7f72ee800640 1 --2- 172.21.15.86:0/4181237350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f015e230 0x7f72f006edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.160+0000 7f72ef200640 1 -- 172.21.15.86:0/4181237350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f015e230 msgr2=0x7f72f006edb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:11.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.160+0000 7f72ef200640 1 --2- 172.21.15.86:0/4181237350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f015e230 0x7f72f006edb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.160+0000 7f72ef200640 1 -- 172.21.15.86:0/4181237350 >> v1:172.21.15.194:6789/0 conn(0x7f72f00b4320 legacy=0x7f72f00d0be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.160+0000 7f72ef200640 1 -- 172.21.15.86:0/4181237350 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72f006f390 con 0x7f72f00ce550 2024-03-06T07:27:11.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.161+0000 7f72ef200640 1 --2- 172.21.15.86:0/4181237350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00ce550 0x7f72f00ce9e0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f72e8009e50 tx=0x7f72e802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5e12cfd1d1c9efcc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:11.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.161+0000 7f72ede00640 1 -- 172.21.15.86:0/4181237350 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e802ca70 con 0x7f72f00ce550 2024-03-06T07:27:11.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.161+0000 7f72ede00640 1 -- 172.21.15.86:0/4181237350 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72e802cc10 con 0x7f72f00ce550 2024-03-06T07:27:11.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.161+0000 7f72ede00640 1 -- 172.21.15.86:0/4181237350 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e80343f0 con 0x7f72f00ce550 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 -- 172.21.15.86:0/4181237350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00ce550 msgr2=0x7f72f00ce9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 --2- 172.21.15.86:0/4181237350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00ce550 0x7f72f00ce9e0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f72e8009e50 tx=0x7f72e802bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 -- 172.21.15.86:0/4181237350 shutdown_connections 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 --2- 172.21.15.86:0/4181237350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00ce550 0x7f72f00ce9e0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 --2- 172.21.15.86:0/4181237350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f015e230 0x7f72f006edb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 -- 172.21.15.86:0/4181237350 >> 172.21.15.86:0/4181237350 conn(0x7f72f00b3800 msgr2=0x7f72f00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 -- 172.21.15.86:0/4181237350 shutdown_connections 2024-03-06T07:27:11.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.162+0000 7f72f5400640 1 -- 172.21.15.86:0/4181237350 wait complete. 2024-03-06T07:27:11.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.163+0000 7f72f5400640 1 Processor -- start 2024-03-06T07:27:11.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.164+0000 7f72f5400640 1 -- start start 2024-03-06T07:27:11.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.164+0000 7f72f5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00b4320 0x7f72f00c6f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.164+0000 7f72f5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f00ce550 0x7f72f01f8780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.164+0000 7f72f5400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f015e230 0x7f72f01f3f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.164+0000 7f72f5400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72f00d06c0 con 0x7f72f00b4320 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.164+0000 7f72f5400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72f00d03e0 con 0x7f72f015e230 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.164+0000 7f72f5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72f00cc320 con 0x7f72f00ce550 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00b4320 0x7f72f00c6f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00b4320 0x7f72f00c6f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53832/0 (socket says 172.21.15.86:53832) 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 -- 172.21.15.86:0/4146730282 learned_addr learned my addr 172.21.15.86:0/4146730282 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ef200640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f015e230 0x7f72f01f3f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72e7e00640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f00ce550 0x7f72f01f8780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 -- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f00ce550 msgr2=0x7f72f01f8780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f00ce550 0x7f72f01f8780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 -- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f015e230 msgr2=0x7f72f01f3f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f015e230 0x7f72f01f3f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 -- 172.21.15.86:0/4146730282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72f00c76c0 con 0x7f72f00b4320 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.165+0000 7f72ee800640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00b4320 0x7f72f00c6f90 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f72e000d5f0 tx=0x7f72e000dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:11.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.166+0000 7f72e6a00640 1 -- 172.21.15.86:0/4146730282 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e00125f0 con 0x7f72f00b4320 2024-03-06T07:27:11.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.166+0000 7f72e6a00640 1 -- 172.21.15.86:0/4146730282 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72e0019070 con 0x7f72f00b4320 2024-03-06T07:27:11.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.166+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72f00c79a0 con 0x7f72f00b4320 2024-03-06T07:27:11.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.166+0000 7f72e6a00640 1 -- 172.21.15.86:0/4146730282 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e00129d0 con 0x7f72f00b4320 2024-03-06T07:27:11.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.166+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72f00c7f60 con 0x7f72f00b4320 2024-03-06T07:27:11.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.167+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f72f0070950 con 0x7f72f00b4320 2024-03-06T07:27:11.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.168+0000 7f72e6a00640 1 -- 172.21.15.86:0/4146730282 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f72e0012b70 con 0x7f72f00b4320 2024-03-06T07:27:11.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.169+0000 7f72e6a00640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72cc06b500 0x7f72cc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.169+0000 7f72e6a00640 1 -- 172.21.15.86:0/4146730282 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f72e008a530 con 0x7f72f00b4320 2024-03-06T07:27:11.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.169+0000 7f72e7e00640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72cc06b500 0x7f72cc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.170+0000 7f72e7e00640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72cc06b500 0x7f72cc06d9c0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f72f0152b80 tx=0x7f72d8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:11.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.173+0000 7f72e6a00640 1 -- 172.21.15.86:0/4146730282 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f72e0059420 con 0x7f72f00b4320 2024-03-06T07:27:11.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.409+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f72f004f330 con 0x7f72f00b4320 2024-03-06T07:27:11.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.411+0000 7f72e6a00640 1 -- 172.21.15.86:0/4146730282 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f72e0058d20 con 0x7f72f00b4320 2024-03-06T07:27:11.413 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:11.413 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:27:11.414 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72cc06b500 msgr2=0x7f72cc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72cc06b500 0x7f72cc06d9c0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f72f0152b80 tx=0x7f72d8009040 comp rx=0 tx=0).stop 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00b4320 msgr2=0x7f72f00c6f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00b4320 0x7f72f00c6f90 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f72e000d5f0 tx=0x7f72e000dab0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 shutdown_connections 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72cc06b500 0x7f72cc06d9c0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f015e230 0x7f72f01f3f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f00ce550 0x7f72f01f8780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 --2- 172.21.15.86:0/4146730282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f00b4320 0x7f72f00c6f90 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 >> 172.21.15.86:0/4146730282 conn(0x7f72f00b3800 msgr2=0x7f72f00d0be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:11.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.413+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 shutdown_connections 2024-03-06T07:27:11.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.414+0000 7f72f5400640 1 -- 172.21.15.86:0/4146730282 wait complete. 2024-03-06T07:27:11.435 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:11.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.641+0000 7f451cc00640 1 Processor -- start 2024-03-06T07:27:11.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.641+0000 7f451cc00640 1 -- start start 2024-03-06T07:27:11.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.643+0000 7f451cc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f451815fe60 0x7f4518071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.643+0000 7f451cc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f451806cef0 con 0x7f451815f9d0 2024-03-06T07:27:11.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.643+0000 7f451cc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f451806d1d0 con 0x7f4518144c00 2024-03-06T07:27:11.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.643+0000 7f451cc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f451806d4b0 con 0x7f451815fe60 2024-03-06T07:27:11.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.643+0000 7f450fe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f451815f9d0 0x7f4518070b10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59234/0 (socket says 172.21.15.86:59234) 2024-03-06T07:27:11.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.643+0000 7f450fe00640 1 -- 172.21.15.86:0/484115175 learned_addr learned my addr 172.21.15.86:0/484115175 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:11.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.643+0000 7f4516800640 1 --2- 172.21.15.86:0/484115175 >> v2:172.21.15.194:3300/0 conn(0x7f451815fe60 0x7f4518071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.644+0000 7f4515400640 1 -- 172.21.15.86:0/484115175 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4188408366 0 0) 0x7f451806cef0 con 0x7f451815f9d0 2024-03-06T07:27:11.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.644+0000 7f4515400640 1 -- 172.21.15.86:0/484115175 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f44fc003520 con 0x7f451815f9d0 2024-03-06T07:27:11.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.644+0000 7f4516800640 1 -- 172.21.15.86:0/484115175 >> v1:172.21.15.194:6789/0 conn(0x7f4518144c00 legacy=0x7f451806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.644+0000 7f4516800640 1 -- 172.21.15.86:0/484115175 >> v1:172.21.15.86:6789/0 conn(0x7f451815f9d0 legacy=0x7f4518070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.644+0000 7f4516800640 1 -- 172.21.15.86:0/484115175 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f451806d790 con 0x7f451815fe60 2024-03-06T07:27:11.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.645+0000 7f4516800640 1 --2- 172.21.15.86:0/484115175 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 0x7f4518071220 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f4510009d90 tx=0x7f451002bf50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3fe7082431f3cce6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:11.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.645+0000 7f4515400640 1 -- 172.21.15.86:0/484115175 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4510039070 con 0x7f451815fe60 2024-03-06T07:27:11.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f4515400640 1 -- 172.21.15.86:0/484115175 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f451002cd00 con 0x7f451815fe60 2024-03-06T07:27:11.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f4515400640 1 -- 172.21.15.86:0/484115175 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4510034640 con 0x7f451815fe60 2024-03-06T07:27:11.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f451cc00640 1 -- 172.21.15.86:0/484115175 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 msgr2=0x7f4518071220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f451cc00640 1 --2- 172.21.15.86:0/484115175 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 0x7f4518071220 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f4510009d90 tx=0x7f451002bf50 comp rx=0 tx=0).stop 2024-03-06T07:27:11.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f451cc00640 1 -- 172.21.15.86:0/484115175 shutdown_connections 2024-03-06T07:27:11.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f451cc00640 1 --2- 172.21.15.86:0/484115175 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 0x7f4518071220 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f451cc00640 1 -- 172.21.15.86:0/484115175 >> 172.21.15.86:0/484115175 conn(0x7f45180b1730 msgr2=0x7f45180b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:11.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f451cc00640 1 -- 172.21.15.86:0/484115175 shutdown_connections 2024-03-06T07:27:11.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.646+0000 7f451cc00640 1 -- 172.21.15.86:0/484115175 wait complete. 2024-03-06T07:27:11.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.648+0000 7f451cc00640 1 Processor -- start 2024-03-06T07:27:11.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.648+0000 7f451cc00640 1 -- start start 2024-03-06T07:27:11.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f451cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4518144c00 0x7f451813b620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f451cc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f451815f9d0 0x7f451813bcd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f451cc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 0x7f451815d4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f451cc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f451806cb50 con 0x7f4518144c00 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f451cc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4518160730 con 0x7f451815fe60 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f451cc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f451806ccd0 con 0x7f451815f9d0 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f4515e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4518144c00 0x7f451813b620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f4515e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4518144c00 0x7f451813b620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53848/0 (socket says 172.21.15.86:53848) 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.649+0000 7f4515e00640 1 -- 172.21.15.86:0/2639958941 learned_addr learned my addr 172.21.15.86:0/2639958941 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f450fe00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f451815f9d0 0x7f451813bcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f4515e00640 1 -- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f451815f9d0 msgr2=0x7f451813bcd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f4515e00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f451815f9d0 0x7f451813bcd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f4515e00640 1 -- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 msgr2=0x7f451815d4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f4515e00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 0x7f451815d4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f4515e00640 1 -- 172.21.15.86:0/2639958941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f451813c3d0 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f450fe00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f451815f9d0 0x7f451813bcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f4515e00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4518144c00 0x7f451813b620 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f450800c400 tx=0x7f450800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f450e000640 1 -- 172.21.15.86:0/2639958941 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4508017070 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f450e000640 1 -- 172.21.15.86:0/2639958941 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f45080052d0 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f451813c6b0 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f450e000640 1 -- 172.21.15.86:0/2639958941 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4508003830 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.650+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f451813cc70 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.651+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f451804f330 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.652+0000 7f450e000640 1 -- 172.21.15.86:0/2639958941 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f450801a070 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.653+0000 7f450e000640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44f406b4d0 0x7f44f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.653+0000 7f450e000640 1 -- 172.21.15.86:0/2639958941 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4508089520 con 0x7f4518144c00 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.653+0000 7f450fe00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44f406b4d0 0x7f44f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:11.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.654+0000 7f450fe00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44f406b4d0 0x7f44f406d990 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f45181f7350 tx=0x7f4500009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:11.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.657+0000 7f450e000640 1 -- 172.21.15.86:0/2639958941 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4508058390 con 0x7f4518144c00 2024-03-06T07:27:11.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.853+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f45180b0530 con 0x7f4518144c00 2024-03-06T07:27:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.855+0000 7f450e000640 1 -- 172.21.15.86:0/2639958941 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f4508057c90 con 0x7f4518144c00 2024-03-06T07:27:11.862 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:11.863 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44f406b4d0 msgr2=0x7f44f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44f406b4d0 0x7f44f406d990 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f45181f7350 tx=0x7f4500009040 comp rx=0 tx=0).stop 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4518144c00 msgr2=0x7f451813b620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4518144c00 0x7f451813b620 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f450800c400 tx=0x7f450800c8c0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 shutdown_connections 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44f406b4d0 0x7f44f406d990 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f451815fe60 0x7f451815d4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f451815f9d0 0x7f451813bcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 --2- 172.21.15.86:0/2639958941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4518144c00 0x7f451813b620 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 >> 172.21.15.86:0/2639958941 conn(0x7f45180b1730 msgr2=0x7f4518071590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 shutdown_connections 2024-03-06T07:27:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:11.858+0000 7f451cc00640 1 -- 172.21.15.86:0/2639958941 wait complete. 2024-03-06T07:27:11.879 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:27:11.879 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:27:12.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.087+0000 7f8cdca00640 1 Processor -- start 2024-03-06T07:27:12.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.087+0000 7f8cdca00640 1 -- start start 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cdca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 0x7f8cd8071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cdca00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8cd806cef0 con 0x7f8cd8144c00 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cdca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8cd806d1d0 con 0x7f8cd815f9d0 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cdca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8cd806d4b0 con 0x7f8cd815fe60 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cd5e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8cd8144c00 0x7f8cd806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59248/0 (socket says 172.21.15.86:59248) 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cd5e00640 1 -- 172.21.15.86:0/2554418246 learned_addr learned my addr 172.21.15.86:0/2554418246 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cd6800640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 0x7f8cd8071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1654187863 0 0) 0x7f8cd806cef0 con 0x7f8cd8144c00 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.089+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8cbc003520 con 0x7f8cd8144c00 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2620326127 0 0) 0x7f8cbc003520 con 0x7f8cd8144c00 2024-03-06T07:27:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 msgr2=0x7f8cd8071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 0x7f8cd8071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 >> v1:172.21.15.194:6789/0 conn(0x7f8cd815f9d0 legacy=0x7f8cd8070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8cd806d790 con 0x7f8cd8144c00 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8cc8001670 con 0x7f8cd8144c00 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 >> v1:172.21.15.86:6789/0 conn(0x7f8cd8144c00 legacy=0x7f8cd806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc0039e0 0x7f8cbc003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cbc004800 0x7f8cbc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 0x7f8cbc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cd806d4b0 con 0x7f8cbc004800 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cbc00baa0 con 0x7f8cbc0039e0 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8cbc00bca0 con 0x7f8cd815fe60 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd5400640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cbc004800 0x7f8cbc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd5e00640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc0039e0 0x7f8cbc003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.090+0000 7f8cd6800640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 0x7f8cbc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.091+0000 7f8cd5400640 1 -- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 msgr2=0x7f8cbc0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:12.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.091+0000 7f8cd5400640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815fe60 0x7f8cbc0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.091+0000 7f8cd5400640 1 -- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc0039e0 msgr2=0x7f8cbc003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:12.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.091+0000 7f8cd5400640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc0039e0 0x7f8cbc003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.091+0000 7f8cd5400640 1 -- 172.21.15.86:0/2554418246 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8cbc003520 con 0x7f8cbc004800 2024-03-06T07:27:12.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.091+0000 7f8cd5e00640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc0039e0 0x7f8cbc003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:12.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.092+0000 7f8cd5400640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cbc004800 0x7f8cbc004c80 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f8cc4001720 tx=0x7f8cbc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=945ee05e703f9a8d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:12.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.092+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8cc4001370 con 0x7f8cbc004800 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.092+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8cc40052d0 con 0x7f8cbc004800 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.092+0000 7f8cd4a00640 1 -- 172.21.15.86:0/2554418246 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8cc4003830 con 0x7f8cbc004800 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.092+0000 7f8cdca00640 1 -- 172.21.15.86:0/2554418246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cbc004800 msgr2=0x7f8cbc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.092+0000 7f8cdca00640 1 --2- 172.21.15.86:0/2554418246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cbc004800 0x7f8cbc004c80 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f8cc4001720 tx=0x7f8cbc005830 comp rx=0 tx=0).stop 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.093+0000 7f8cd6800640 1 -- 172.21.15.86:0/2554418246 reap_dead start 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.093+0000 7f8cdca00640 1 -- 172.21.15.86:0/2554418246 shutdown_connections 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.093+0000 7f8cdca00640 1 -- 172.21.15.86:0/2554418246 >> 172.21.15.86:0/2554418246 conn(0x7f8cd80b1730 msgr2=0x7f8cd80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.093+0000 7f8cdca00640 1 -- 172.21.15.86:0/2554418246 shutdown_connections 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.093+0000 7f8cdca00640 1 -- 172.21.15.86:0/2554418246 wait complete. 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.094+0000 7f8cdca00640 1 Processor -- start 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.094+0000 7f8cdca00640 1 -- start start 2024-03-06T07:27:12.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.095+0000 7f8cdca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc004800 0x7f8cd81f7350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.095+0000 7f8cdca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cd81f7890 0x7f8cd815e040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.095+0000 7f8cdca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815e580 0x7f8cd80cbbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.095+0000 7f8cdca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cd806cb50 con 0x7f8cd81f7890 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.095+0000 7f8cdca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cd8160730 con 0x7f8cbc004800 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.095+0000 7f8cdca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8cd806ccd0 con 0x7f8cd815e580 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cd81f7890 0x7f8cd815e040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cd81f7890 0x7f8cd815e040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53876/0 (socket says 172.21.15.86:53876) 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 -- 172.21.15.86:0/3419616634 learned_addr learned my addr 172.21.15.86:0/3419616634 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd6800640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815e580 0x7f8cd80cbbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5e00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc004800 0x7f8cd81f7350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 -- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815e580 msgr2=0x7f8cd80cbbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815e580 0x7f8cd80cbbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 -- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc004800 msgr2=0x7f8cd81f7350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc004800 0x7f8cd81f7350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 -- 172.21.15.86:0/3419616634 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8cd8130310 con 0x7f8cd81f7890 2024-03-06T07:27:12.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.096+0000 7f8cd5400640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cd81f7890 0x7f8cd815e040 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f8cc4001840 tx=0x7f8cc4001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:12.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.097+0000 7f8cce000640 1 -- 172.21.15.86:0/3419616634 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8cc4041070 con 0x7f8cd81f7890 2024-03-06T07:27:12.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.097+0000 7f8cce000640 1 -- 172.21.15.86:0/3419616634 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8cc4005480 con 0x7f8cd81f7890 2024-03-06T07:27:12.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.097+0000 7f8cce000640 1 -- 172.21.15.86:0/3419616634 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8cc4007a30 con 0x7f8cd81f7890 2024-03-06T07:27:12.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.097+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8cd81305f0 con 0x7f8cd81f7890 2024-03-06T07:27:12.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.097+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8cd815ea00 con 0x7f8cd81f7890 2024-03-06T07:27:12.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.098+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8cd804f330 con 0x7f8cd81f7890 2024-03-06T07:27:12.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.099+0000 7f8cce000640 1 -- 172.21.15.86:0/3419616634 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8cc4044070 con 0x7f8cd81f7890 2024-03-06T07:27:12.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.100+0000 7f8cce000640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8cb806b4d0 0x7f8cb806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.100+0000 7f8cce000640 1 -- 172.21.15.86:0/3419616634 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8cc40abde0 con 0x7f8cd81f7890 2024-03-06T07:27:12.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.100+0000 7f8cd5e00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8cb806b4d0 0x7f8cb806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.101+0000 7f8cd5e00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8cb806b4d0 0x7f8cb806d990 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f8cc8001200 tx=0x7f8cc8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:12.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.104+0000 7f8cce000640 1 -- 172.21.15.86:0/3419616634 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8cc407ac60 con 0x7f8cd81f7890 2024-03-06T07:27:12.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.296+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f8cd815ed70 con 0x7f8cb806b4d0 2024-03-06T07:27:12.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.305+0000 7f8cce000640 1 -- 172.21.15.86:0/3419616634 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251946 (secure 0 0 0) 0x7f8cd815ed70 con 0x7f8cb806b4d0 2024-03-06T07:27:12.310 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:12.312 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8cb806b4d0 msgr2=0x7f8cb806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8cb806b4d0 0x7f8cb806d990 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f8cc8001200 tx=0x7f8cc8009040 comp rx=0 tx=0).stop 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cd81f7890 msgr2=0x7f8cd815e040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cd81f7890 0x7f8cd815e040 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f8cc4001840 tx=0x7f8cc4001120 comp rx=0 tx=0).stop 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 shutdown_connections 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8cb806b4d0 0x7f8cb806d990 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8cd815e580 0x7f8cd80cbbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8cd81f7890 0x7f8cd815e040 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.309+0000 7f8cdca00640 1 --2- 172.21.15.86:0/3419616634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8cbc004800 0x7f8cd81f7350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.310+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 >> 172.21.15.86:0/3419616634 conn(0x7f8cd80b1730 msgr2=0x7f8cd8071750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:12.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.310+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 shutdown_connections 2024-03-06T07:27:12.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.310+0000 7f8cdca00640 1 -- 172.21.15.86:0/3419616634 wait complete. 2024-03-06T07:27:12.329 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-03-06T07:27:10.851640+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":109,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219200,"kb_used_data":2984,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755520,"statfs":{"total":773094113280,"available":772869652480,"internally_reserved":0,"allocated":3055616,"data_stored":1816228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":221236787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":42,"apply_latency_ms":42,"commit_latency_ns":42000000,"apply_latency_ns":42000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":59,"num_write_kb":276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.973683"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225431+0000","last_change":"2024-03-06T07:27:07.923330+0000","last_active":"2024-03-06T07:27:09.225431+0000","last_peered":"2024-03-06T07:27:09.225431+0000","last_clean":"2024-03-06T07:27:09.225431+0000","last_became_active":"2024-03-06T07:27:07.923084+0000","last_became_peered":"2024-03-06T07:27:07.923084+0000","last_unstale":"2024-03-06T07:27:09.225431+0000","last_undegraded":"2024-03-06T07:27:09.225431+0000","last_fullsized":"2024-03-06T07:27:09.225431+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:46:48.027042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918758+0000","last_change":"2024-03-06T07:27:07.921104+0000","last_active":"2024-03-06T07:27:08.918758+0000","last_peered":"2024-03-06T07:27:08.918758+0000","last_clean":"2024-03-06T07:27:08.918758+0000","last_became_active":"2024-03-06T07:27:07.920770+0000","last_became_peered":"2024-03-06T07:27:07.920770+0000","last_unstale":"2024-03-06T07:27:08.918758+0000","last_undegraded":"2024-03-06T07:27:08.918758+0000","last_fullsized":"2024-03-06T07:27:08.918758+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226481+0000","last_change":"2024-03-06T07:27:07.922511+0000","last_active":"2024-03-06T07:27:09.226481+0000","last_peered":"2024-03-06T07:27:09.226481+0000","last_clean":"2024-03-06T07:27:09.226481+0000","last_became_active":"2024-03-06T07:27:07.922242+0000","last_became_peered":"2024-03-06T07:27:07.922242+0000","last_unstale":"2024-03-06T07:27:09.226481+0000","last_undegraded":"2024-03-06T07:27:09.226481+0000","last_fullsized":"2024-03-06T07:27:09.226481+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922737+0000","last_change":"2024-03-06T07:27:07.945609+0000","last_active":"2024-03-06T07:27:08.922737+0000","last_peered":"2024-03-06T07:27:08.922737+0000","last_clean":"2024-03-06T07:27:08.922737+0000","last_became_active":"2024-03-06T07:27:07.945312+0000","last_became_peered":"2024-03-06T07:27:07.945312+0000","last_unstale":"2024-03-06T07:27:08.922737+0000","last_undegraded":"2024-03-06T07:27:08.922737+0000","last_fullsized":"2024-03-06T07:27:08.922737+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:38.769682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224858+0000","last_change":"2024-03-06T07:27:07.921341+0000","last_active":"2024-03-06T07:27:09.224858+0000","last_peered":"2024-03-06T07:27:09.224858+0000","last_clean":"2024-03-06T07:27:09.224858+0000","last_became_active":"2024-03-06T07:27:07.920986+0000","last_became_peered":"2024-03-06T07:27:07.920986+0000","last_unstale":"2024-03-06T07:27:09.224858+0000","last_undegraded":"2024-03-06T07:27:09.224858+0000","last_fullsized":"2024-03-06T07:27:09.224858+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922064+0000","last_change":"2024-03-06T07:27:07.934696+0000","last_active":"2024-03-06T07:27:08.922064+0000","last_peered":"2024-03-06T07:27:08.922064+0000","last_clean":"2024-03-06T07:27:08.922064+0000","last_became_active":"2024-03-06T07:27:07.934400+0000","last_became_peered":"2024-03-06T07:27:07.934400+0000","last_unstale":"2024-03-06T07:27:08.922064+0000","last_undegraded":"2024-03-06T07:27:08.922064+0000","last_fullsized":"2024-03-06T07:27:08.922064+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224278+0000","last_change":"2024-03-06T07:27:07.926117+0000","last_active":"2024-03-06T07:27:09.224278+0000","last_peered":"2024-03-06T07:27:09.224278+0000","last_clean":"2024-03-06T07:27:09.224278+0000","last_became_active":"2024-03-06T07:27:07.925805+0000","last_became_peered":"2024-03-06T07:27:07.925805+0000","last_unstale":"2024-03-06T07:27:09.224278+0000","last_undegraded":"2024-03-06T07:27:09.224278+0000","last_fullsized":"2024-03-06T07:27:09.224278+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223418+0000","last_change":"2024-03-06T07:27:07.946584+0000","last_active":"2024-03-06T07:27:09.223418+0000","last_peered":"2024-03-06T07:27:09.223418+0000","last_clean":"2024-03-06T07:27:09.223418+0000","last_became_active":"2024-03-06T07:27:07.945766+0000","last_became_peered":"2024-03-06T07:27:07.945766+0000","last_unstale":"2024-03-06T07:27:09.223418+0000","last_undegraded":"2024-03-06T07:27:09.223418+0000","last_fullsized":"2024-03-06T07:27:09.223418+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225430+0000","last_change":"2024-03-06T07:27:07.948853+0000","last_active":"2024-03-06T07:27:09.225430+0000","last_peered":"2024-03-06T07:27:09.225430+0000","last_clean":"2024-03-06T07:27:09.225430+0000","last_became_active":"2024-03-06T07:27:07.948523+0000","last_became_peered":"2024-03-06T07:27:07.948523+0000","last_unstale":"2024-03-06T07:27:09.225430+0000","last_undegraded":"2024-03-06T07:27:09.225430+0000","last_fullsized":"2024-03-06T07:27:09.225430+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:17:40.285874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933598+0000","last_change":"2024-03-06T07:27:07.946279+0000","last_active":"2024-03-06T07:27:08.933598+0000","last_peered":"2024-03-06T07:27:08.933598+0000","last_clean":"2024-03-06T07:27:08.933598+0000","last_became_active":"2024-03-06T07:27:07.945895+0000","last_became_peered":"2024-03-06T07:27:07.945895+0000","last_unstale":"2024-03-06T07:27:08.933598+0000","last_undegraded":"2024-03-06T07:27:08.933598+0000","last_fullsized":"2024-03-06T07:27:08.933598+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933628+0000","last_change":"2024-03-06T07:27:07.922684+0000","last_active":"2024-03-06T07:27:08.933628+0000","last_peered":"2024-03-06T07:27:08.933628+0000","last_clean":"2024-03-06T07:27:08.933628+0000","last_became_active":"2024-03-06T07:27:07.922348+0000","last_became_peered":"2024-03-06T07:27:07.922348+0000","last_unstale":"2024-03-06T07:27:08.933628+0000","last_undegraded":"2024-03-06T07:27:08.933628+0000","last_fullsized":"2024-03-06T07:27:08.933628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:38:38.919038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223388+0000","last_change":"2024-03-06T07:27:07.923877+0000","last_active":"2024-03-06T07:27:09.223388+0000","last_peered":"2024-03-06T07:27:09.223388+0000","last_clean":"2024-03-06T07:27:09.223388+0000","last_became_active":"2024-03-06T07:27:07.923410+0000","last_became_peered":"2024-03-06T07:27:07.923410+0000","last_unstale":"2024-03-06T07:27:09.223388+0000","last_undegraded":"2024-03-06T07:27:09.223388+0000","last_fullsized":"2024-03-06T07:27:09.223388+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:30.853873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227024+0000","last_change":"2024-03-06T07:27:07.930873+0000","last_active":"2024-03-06T07:27:09.227024+0000","last_peered":"2024-03-06T07:27:09.227024+0000","last_clean":"2024-03-06T07:27:09.227024+0000","last_became_active":"2024-03-06T07:27:07.930653+0000","last_became_peered":"2024-03-06T07:27:07.930653+0000","last_unstale":"2024-03-06T07:27:09.227024+0000","last_undegraded":"2024-03-06T07:27:09.227024+0000","last_fullsized":"2024-03-06T07:27:09.227024+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224966+0000","last_change":"2024-03-06T07:27:07.919903+0000","last_active":"2024-03-06T07:27:09.224966+0000","last_peered":"2024-03-06T07:27:09.224966+0000","last_clean":"2024-03-06T07:27:09.224966+0000","last_became_active":"2024-03-06T07:27:07.919585+0000","last_became_peered":"2024-03-06T07:27:07.919585+0000","last_unstale":"2024-03-06T07:27:09.224966+0000","last_undegraded":"2024-03-06T07:27:09.224966+0000","last_fullsized":"2024-03-06T07:27:09.224966+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:06.597617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917655+0000","last_change":"2024-03-06T07:27:07.908420+0000","last_active":"2024-03-06T07:27:08.917655+0000","last_peered":"2024-03-06T07:27:08.917655+0000","last_clean":"2024-03-06T07:27:08.917655+0000","last_became_active":"2024-03-06T07:27:07.907392+0000","last_became_peered":"2024-03-06T07:27:07.907392+0000","last_unstale":"2024-03-06T07:27:08.917655+0000","last_undegraded":"2024-03-06T07:27:08.917655+0000","last_fullsized":"2024-03-06T07:27:08.917655+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922133+0000","last_change":"2024-03-06T07:27:07.934890+0000","last_active":"2024-03-06T07:27:08.922133+0000","last_peered":"2024-03-06T07:27:08.922133+0000","last_clean":"2024-03-06T07:27:08.922133+0000","last_became_active":"2024-03-06T07:27:07.934568+0000","last_became_peered":"2024-03-06T07:27:07.934568+0000","last_unstale":"2024-03-06T07:27:08.922133+0000","last_undegraded":"2024-03-06T07:27:08.922133+0000","last_fullsized":"2024-03-06T07:27:08.922133+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923353+0000","last_change":"2024-03-06T07:27:07.927869+0000","last_active":"2024-03-06T07:27:08.923353+0000","last_peered":"2024-03-06T07:27:08.923353+0000","last_clean":"2024-03-06T07:27:08.923353+0000","last_became_active":"2024-03-06T07:27:07.927507+0000","last_became_peered":"2024-03-06T07:27:07.927507+0000","last_unstale":"2024-03-06T07:27:08.923353+0000","last_undegraded":"2024-03-06T07:27:08.923353+0000","last_fullsized":"2024-03-06T07:27:08.923353+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:26:20.914324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224317+0000","last_change":"2024-03-06T07:27:07.922930+0000","last_active":"2024-03-06T07:27:09.224317+0000","last_peered":"2024-03-06T07:27:09.224317+0000","last_clean":"2024-03-06T07:27:09.224317+0000","last_became_active":"2024-03-06T07:27:07.922608+0000","last_became_peered":"2024-03-06T07:27:07.922608+0000","last_unstale":"2024-03-06T07:27:09.224317+0000","last_undegraded":"2024-03-06T07:27:09.224317+0000","last_fullsized":"2024-03-06T07:27:09.224317+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225850+0000","last_change":"2024-03-06T07:27:07.927132+0000","last_active":"2024-03-06T07:27:09.225850+0000","last_peered":"2024-03-06T07:27:09.225850+0000","last_clean":"2024-03-06T07:27:09.225850+0000","last_became_active":"2024-03-06T07:27:07.926850+0000","last_became_peered":"2024-03-06T07:27:07.926850+0000","last_unstale":"2024-03-06T07:27:09.225850+0000","last_undegraded":"2024-03-06T07:27:09.225850+0000","last_fullsized":"2024-03-06T07:27:09.225850+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:23.735936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933468+0000","last_change":"2024-03-06T07:27:07.928126+0000","last_active":"2024-03-06T07:27:08.933468+0000","last_peered":"2024-03-06T07:27:08.933468+0000","last_clean":"2024-03-06T07:27:08.933468+0000","last_became_active":"2024-03-06T07:27:07.927650+0000","last_became_peered":"2024-03-06T07:27:07.927650+0000","last_unstale":"2024-03-06T07:27:08.933468+0000","last_undegraded":"2024-03-06T07:27:08.933468+0000","last_fullsized":"2024-03-06T07:27:08.933468+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:55.755072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225914+0000","last_change":"2024-03-06T07:27:07.927942+0000","last_active":"2024-03-06T07:27:09.225914+0000","last_peered":"2024-03-06T07:27:09.225914+0000","last_clean":"2024-03-06T07:27:09.225914+0000","last_became_active":"2024-03-06T07:27:07.927629+0000","last_became_peered":"2024-03-06T07:27:07.927629+0000","last_unstale":"2024-03-06T07:27:09.225914+0000","last_undegraded":"2024-03-06T07:27:09.225914+0000","last_fullsized":"2024-03-06T07:27:09.225914+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922498+0000","last_change":"2024-03-06T07:27:07.922273+0000","last_active":"2024-03-06T07:27:08.922498+0000","last_peered":"2024-03-06T07:27:08.922498+0000","last_clean":"2024-03-06T07:27:08.922498+0000","last_became_active":"2024-03-06T07:27:07.921988+0000","last_became_peered":"2024-03-06T07:27:07.921988+0000","last_unstale":"2024-03-06T07:27:08.922498+0000","last_undegraded":"2024-03-06T07:27:08.922498+0000","last_fullsized":"2024-03-06T07:27:08.922498+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:57.831357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223829+0000","last_change":"2024-03-06T07:27:07.930778+0000","last_active":"2024-03-06T07:27:09.223829+0000","last_peered":"2024-03-06T07:27:09.223829+0000","last_clean":"2024-03-06T07:27:09.223829+0000","last_became_active":"2024-03-06T07:27:07.930464+0000","last_became_peered":"2024-03-06T07:27:07.930464+0000","last_unstale":"2024-03-06T07:27:09.223829+0000","last_undegraded":"2024-03-06T07:27:09.223829+0000","last_fullsized":"2024-03-06T07:27:09.223829+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922134+0000","last_change":"2024-03-06T07:27:07.917391+0000","last_active":"2024-03-06T07:27:08.922134+0000","last_peered":"2024-03-06T07:27:08.922134+0000","last_clean":"2024-03-06T07:27:08.922134+0000","last_became_active":"2024-03-06T07:27:07.917107+0000","last_became_peered":"2024-03-06T07:27:07.917107+0000","last_unstale":"2024-03-06T07:27:08.922134+0000","last_undegraded":"2024-03-06T07:27:08.922134+0000","last_fullsized":"2024-03-06T07:27:08.922134+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:08.467879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227476+0000","last_change":"2024-03-06T07:27:07.930923+0000","last_active":"2024-03-06T07:27:09.227476+0000","last_peered":"2024-03-06T07:27:09.227476+0000","last_clean":"2024-03-06T07:27:09.227476+0000","last_became_active":"2024-03-06T07:27:07.930078+0000","last_became_peered":"2024-03-06T07:27:07.930078+0000","last_unstale":"2024-03-06T07:27:09.227476+0000","last_undegraded":"2024-03-06T07:27:09.227476+0000","last_fullsized":"2024-03-06T07:27:09.227476+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933695+0000","last_change":"2024-03-06T07:27:07.936206+0000","last_active":"2024-03-06T07:27:08.933695+0000","last_peered":"2024-03-06T07:27:08.933695+0000","last_clean":"2024-03-06T07:27:08.933695+0000","last_became_active":"2024-03-06T07:27:07.930820+0000","last_became_peered":"2024-03-06T07:27:07.930820+0000","last_unstale":"2024-03-06T07:27:08.933695+0000","last_undegraded":"2024-03-06T07:27:08.933695+0000","last_fullsized":"2024-03-06T07:27:08.933695+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917816+0000","last_change":"2024-03-06T07:27:07.918528+0000","last_active":"2024-03-06T07:27:08.917816+0000","last_peered":"2024-03-06T07:27:08.917816+0000","last_clean":"2024-03-06T07:27:08.917816+0000","last_became_active":"2024-03-06T07:27:07.918221+0000","last_became_peered":"2024-03-06T07:27:07.918221+0000","last_unstale":"2024-03-06T07:27:08.917816+0000","last_undegraded":"2024-03-06T07:27:08.917816+0000","last_fullsized":"2024-03-06T07:27:08.917816+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227107+0000","last_change":"2024-03-06T07:27:07.929588+0000","last_active":"2024-03-06T07:27:09.227107+0000","last_peered":"2024-03-06T07:27:09.227107+0000","last_clean":"2024-03-06T07:27:09.227107+0000","last_became_active":"2024-03-06T07:27:07.929207+0000","last_became_peered":"2024-03-06T07:27:07.929207+0000","last_unstale":"2024-03-06T07:27:09.227107+0000","last_undegraded":"2024-03-06T07:27:09.227107+0000","last_fullsized":"2024-03-06T07:27:09.227107+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226074+0000","last_change":"2024-03-06T07:27:07.925047+0000","last_active":"2024-03-06T07:27:09.226074+0000","last_peered":"2024-03-06T07:27:09.226074+0000","last_clean":"2024-03-06T07:27:09.226074+0000","last_became_active":"2024-03-06T07:27:07.924734+0000","last_became_peered":"2024-03-06T07:27:07.924734+0000","last_unstale":"2024-03-06T07:27:09.226074+0000","last_undegraded":"2024-03-06T07:27:09.226074+0000","last_fullsized":"2024-03-06T07:27:09.226074+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922314+0000","last_change":"2024-03-06T07:27:07.927022+0000","last_active":"2024-03-06T07:27:08.922314+0000","last_peered":"2024-03-06T07:27:08.922314+0000","last_clean":"2024-03-06T07:27:08.922314+0000","last_became_active":"2024-03-06T07:27:07.926748+0000","last_became_peered":"2024-03-06T07:27:07.926748+0000","last_unstale":"2024-03-06T07:27:08.922314+0000","last_undegraded":"2024-03-06T07:27:08.922314+0000","last_fullsized":"2024-03-06T07:27:08.922314+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:25.638818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223563+0000","last_change":"2024-03-06T07:27:07.934988+0000","last_active":"2024-03-06T07:27:09.223563+0000","last_peered":"2024-03-06T07:27:09.223563+0000","last_clean":"2024-03-06T07:27:09.223563+0000","last_became_active":"2024-03-06T07:27:07.934606+0000","last_became_peered":"2024-03-06T07:27:07.934606+0000","last_unstale":"2024-03-06T07:27:09.223563+0000","last_undegraded":"2024-03-06T07:27:09.223563+0000","last_fullsized":"2024-03-06T07:27:09.223563+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933081+0000","last_change":"2024-03-06T07:27:07.929352+0000","last_active":"2024-03-06T07:27:08.933081+0000","last_peered":"2024-03-06T07:27:08.933081+0000","last_clean":"2024-03-06T07:27:08.933081+0000","last_became_active":"2024-03-06T07:27:07.929001+0000","last_became_peered":"2024-03-06T07:27:07.929001+0000","last_unstale":"2024-03-06T07:27:08.933081+0000","last_undegraded":"2024-03-06T07:27:08.933081+0000","last_fullsized":"2024-03-06T07:27:08.933081+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:17:25.890956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.222941+0000","last_change":"2024-03-06T07:27:07.906813+0000","last_active":"2024-03-06T07:27:09.222941+0000","last_peered":"2024-03-06T07:27:09.222941+0000","last_clean":"2024-03-06T07:27:09.222941+0000","last_became_active":"2024-03-06T07:27:07.906019+0000","last_became_peered":"2024-03-06T07:27:07.906019+0000","last_unstale":"2024-03-06T07:27:09.222941+0000","last_undegraded":"2024-03-06T07:27:09.222941+0000","last_fullsized":"2024-03-06T07:27:09.222941+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:18.357063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226805+0000","last_change":"2024-03-06T07:27:06.929035+0000","last_active":"2024-03-06T07:27:09.226805+0000","last_peered":"2024-03-06T07:27:09.226805+0000","last_clean":"2024-03-06T07:27:09.226805+0000","last_became_active":"2024-03-06T07:27:06.928702+0000","last_became_peered":"2024-03-06T07:27:06.928702+0000","last_unstale":"2024-03-06T07:27:09.226805+0000","last_undegraded":"2024-03-06T07:27:09.226805+0000","last_fullsized":"2024-03-06T07:27:09.226805+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226233+0000","last_change":"2024-03-06T07:27:07.938823+0000","last_active":"2024-03-06T07:27:09.226233+0000","last_peered":"2024-03-06T07:27:09.226233+0000","last_clean":"2024-03-06T07:27:09.226233+0000","last_became_active":"2024-03-06T07:27:07.937638+0000","last_became_peered":"2024-03-06T07:27:07.937638+0000","last_unstale":"2024-03-06T07:27:09.226233+0000","last_undegraded":"2024-03-06T07:27:09.226233+0000","last_fullsized":"2024-03-06T07:27:09.226233+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932806+0000","last_change":"2024-03-06T07:27:06.929034+0000","last_active":"2024-03-06T07:27:08.932806+0000","last_peered":"2024-03-06T07:27:08.932806+0000","last_clean":"2024-03-06T07:27:08.932806+0000","last_became_active":"2024-03-06T07:27:06.927376+0000","last_became_peered":"2024-03-06T07:27:06.927376+0000","last_unstale":"2024-03-06T07:27:08.932806+0000","last_undegraded":"2024-03-06T07:27:08.932806+0000","last_fullsized":"2024-03-06T07:27:08.932806+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225115+0000","last_change":"2024-03-06T07:27:06.933665+0000","last_active":"2024-03-06T07:27:09.225115+0000","last_peered":"2024-03-06T07:27:09.225115+0000","last_clean":"2024-03-06T07:27:09.225115+0000","last_became_active":"2024-03-06T07:27:06.933367+0000","last_became_peered":"2024-03-06T07:27:06.933367+0000","last_unstale":"2024-03-06T07:27:09.225115+0000","last_undegraded":"2024-03-06T07:27:09.225115+0000","last_fullsized":"2024-03-06T07:27:09.225115+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223616+0000","last_change":"2024-03-06T07:27:07.927629+0000","last_active":"2024-03-06T07:27:09.223616+0000","last_peered":"2024-03-06T07:27:09.223616+0000","last_clean":"2024-03-06T07:27:09.223616+0000","last_became_active":"2024-03-06T07:27:07.925804+0000","last_became_peered":"2024-03-06T07:27:07.925804+0000","last_unstale":"2024-03-06T07:27:09.223616+0000","last_undegraded":"2024-03-06T07:27:09.223616+0000","last_fullsized":"2024-03-06T07:27:09.223616+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922349+0000","last_change":"2024-03-06T07:27:06.935442+0000","last_active":"2024-03-06T07:27:08.922349+0000","last_peered":"2024-03-06T07:27:08.922349+0000","last_clean":"2024-03-06T07:27:08.922349+0000","last_became_active":"2024-03-06T07:27:06.933244+0000","last_became_peered":"2024-03-06T07:27:06.933244+0000","last_unstale":"2024-03-06T07:27:08.922349+0000","last_undegraded":"2024-03-06T07:27:08.922349+0000","last_fullsized":"2024-03-06T07:27:08.922349+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917698+0000","last_change":"2024-03-06T07:27:07.922392+0000","last_active":"2024-03-06T07:27:08.917698+0000","last_peered":"2024-03-06T07:27:08.917698+0000","last_clean":"2024-03-06T07:27:08.917698+0000","last_became_active":"2024-03-06T07:27:07.922127+0000","last_became_peered":"2024-03-06T07:27:07.922127+0000","last_unstale":"2024-03-06T07:27:08.917698+0000","last_undegraded":"2024-03-06T07:27:08.917698+0000","last_fullsized":"2024-03-06T07:27:08.917698+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933693+0000","last_change":"2024-03-06T07:27:06.937146+0000","last_active":"2024-03-06T07:27:08.933693+0000","last_peered":"2024-03-06T07:27:08.933693+0000","last_clean":"2024-03-06T07:27:08.933693+0000","last_became_active":"2024-03-06T07:27:06.936876+0000","last_became_peered":"2024-03-06T07:27:06.936876+0000","last_unstale":"2024-03-06T07:27:08.933693+0000","last_undegraded":"2024-03-06T07:27:08.933693+0000","last_fullsized":"2024-03-06T07:27:08.933693+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.943897+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.943627+0000","last_became_peered":"2024-03-06T07:27:07.943627+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:01:28.762594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923212+0000","last_change":"2024-03-06T07:27:06.940833+0000","last_active":"2024-03-06T07:27:08.923212+0000","last_peered":"2024-03-06T07:27:08.923212+0000","last_clean":"2024-03-06T07:27:08.923212+0000","last_became_active":"2024-03-06T07:27:06.939734+0000","last_became_peered":"2024-03-06T07:27:06.939734+0000","last_unstale":"2024-03-06T07:27:08.923212+0000","last_undegraded":"2024-03-06T07:27:08.923212+0000","last_fullsized":"2024-03-06T07:27:08.923212+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227338+0000","last_change":"2024-03-06T07:27:07.946830+0000","last_active":"2024-03-06T07:27:09.227338+0000","last_peered":"2024-03-06T07:27:09.227338+0000","last_clean":"2024-03-06T07:27:09.227338+0000","last_became_active":"2024-03-06T07:27:07.946495+0000","last_became_peered":"2024-03-06T07:27:07.946495+0000","last_unstale":"2024-03-06T07:27:09.227338+0000","last_undegraded":"2024-03-06T07:27:09.227338+0000","last_fullsized":"2024-03-06T07:27:09.227338+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"21'3","reported_seq":17,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.937692+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.937338+0000","last_became_peered":"2024-03-06T07:27:06.937338+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223197+0000","last_change":"2024-03-06T07:27:07.922246+0000","last_active":"2024-03-06T07:27:09.223197+0000","last_peered":"2024-03-06T07:27:09.223197+0000","last_clean":"2024-03-06T07:27:09.223197+0000","last_became_active":"2024-03-06T07:27:07.921469+0000","last_became_peered":"2024-03-06T07:27:07.921469+0000","last_unstale":"2024-03-06T07:27:09.223197+0000","last_undegraded":"2024-03-06T07:27:09.223197+0000","last_fullsized":"2024-03-06T07:27:09.223197+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"21'6","reported_seq":20,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933200+0000","last_change":"2024-03-06T07:27:06.926732+0000","last_active":"2024-03-06T07:27:08.933200+0000","last_peered":"2024-03-06T07:27:08.933200+0000","last_clean":"2024-03-06T07:27:08.933200+0000","last_became_active":"2024-03-06T07:27:06.925706+0000","last_became_peered":"2024-03-06T07:27:06.925706+0000","last_unstale":"2024-03-06T07:27:08.933200+0000","last_undegraded":"2024-03-06T07:27:08.933200+0000","last_fullsized":"2024-03-06T07:27:08.933200+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225109+0000","last_change":"2024-03-06T07:27:07.935123+0000","last_active":"2024-03-06T07:27:09.225109+0000","last_peered":"2024-03-06T07:27:09.225109+0000","last_clean":"2024-03-06T07:27:09.225109+0000","last_became_active":"2024-03-06T07:27:07.934820+0000","last_became_peered":"2024-03-06T07:27:07.934820+0000","last_unstale":"2024-03-06T07:27:09.225109+0000","last_undegraded":"2024-03-06T07:27:09.225109+0000","last_fullsized":"2024-03-06T07:27:09.225109+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:29.926787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226343+0000","last_change":"2024-03-06T07:27:06.921406+0000","last_active":"2024-03-06T07:27:09.226343+0000","last_peered":"2024-03-06T07:27:09.226343+0000","last_clean":"2024-03-06T07:27:09.226343+0000","last_became_active":"2024-03-06T07:27:06.921070+0000","last_became_peered":"2024-03-06T07:27:06.921070+0000","last_unstale":"2024-03-06T07:27:09.226343+0000","last_undegraded":"2024-03-06T07:27:09.226343+0000","last_fullsized":"2024-03-06T07:27:09.226343+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225286+0000","last_change":"2024-03-06T07:27:07.926884+0000","last_active":"2024-03-06T07:27:09.225286+0000","last_peered":"2024-03-06T07:27:09.225286+0000","last_clean":"2024-03-06T07:27:09.225286+0000","last_became_active":"2024-03-06T07:27:07.926600+0000","last_became_peered":"2024-03-06T07:27:07.926600+0000","last_unstale":"2024-03-06T07:27:09.225286+0000","last_undegraded":"2024-03-06T07:27:09.225286+0000","last_fullsized":"2024-03-06T07:27:09.225286+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922438+0000","last_change":"2024-03-06T07:27:06.936248+0000","last_active":"2024-03-06T07:27:08.922438+0000","last_peered":"2024-03-06T07:27:08.922438+0000","last_clean":"2024-03-06T07:27:08.922438+0000","last_became_active":"2024-03-06T07:27:06.934671+0000","last_became_peered":"2024-03-06T07:27:06.934671+0000","last_unstale":"2024-03-06T07:27:08.922438+0000","last_undegraded":"2024-03-06T07:27:08.922438+0000","last_fullsized":"2024-03-06T07:27:08.922438+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224843+0000","last_change":"2024-03-06T07:27:07.941367+0000","last_active":"2024-03-06T07:27:09.224843+0000","last_peered":"2024-03-06T07:27:09.224843+0000","last_clean":"2024-03-06T07:27:09.224843+0000","last_became_active":"2024-03-06T07:27:07.938238+0000","last_became_peered":"2024-03-06T07:27:07.938238+0000","last_unstale":"2024-03-06T07:27:09.224843+0000","last_undegraded":"2024-03-06T07:27:09.224843+0000","last_fullsized":"2024-03-06T07:27:09.224843+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933197+0000","last_change":"2024-03-06T07:27:06.931453+0000","last_active":"2024-03-06T07:27:08.933197+0000","last_peered":"2024-03-06T07:27:08.933197+0000","last_clean":"2024-03-06T07:27:08.933197+0000","last_became_active":"2024-03-06T07:27:06.931181+0000","last_became_peered":"2024-03-06T07:27:06.931181+0000","last_unstale":"2024-03-06T07:27:08.933197+0000","last_undegraded":"2024-03-06T07:27:08.933197+0000","last_fullsized":"2024-03-06T07:27:08.933197+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933121+0000","last_change":"2024-03-06T07:27:07.932186+0000","last_active":"2024-03-06T07:27:08.933121+0000","last_peered":"2024-03-06T07:27:08.933121+0000","last_clean":"2024-03-06T07:27:08.933121+0000","last_became_active":"2024-03-06T07:27:07.923467+0000","last_became_peered":"2024-03-06T07:27:07.923467+0000","last_unstale":"2024-03-06T07:27:08.933121+0000","last_undegraded":"2024-03-06T07:27:08.933121+0000","last_fullsized":"2024-03-06T07:27:08.933121+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:27:55.371987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"21'8","reported_seq":22,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223314+0000","last_change":"2024-03-06T07:27:06.925620+0000","last_active":"2024-03-06T07:27:09.223314+0000","last_peered":"2024-03-06T07:27:09.223314+0000","last_clean":"2024-03-06T07:27:09.223314+0000","last_became_active":"2024-03-06T07:27:06.925324+0000","last_became_peered":"2024-03-06T07:27:06.925324+0000","last_unstale":"2024-03-06T07:27:09.223314+0000","last_undegraded":"2024-03-06T07:27:09.223314+0000","last_fullsized":"2024-03-06T07:27:09.223314+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224242+0000","last_change":"2024-03-06T07:27:07.935148+0000","last_active":"2024-03-06T07:27:09.224242+0000","last_peered":"2024-03-06T07:27:09.224242+0000","last_clean":"2024-03-06T07:27:09.224242+0000","last_became_active":"2024-03-06T07:27:07.933239+0000","last_became_peered":"2024-03-06T07:27:07.933239+0000","last_unstale":"2024-03-06T07:27:09.224242+0000","last_undegraded":"2024-03-06T07:27:09.224242+0000","last_fullsized":"2024-03-06T07:27:09.224242+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223549+0000","last_change":"2024-03-06T07:27:06.919221+0000","last_active":"2024-03-06T07:27:09.223549+0000","last_peered":"2024-03-06T07:27:09.223549+0000","last_clean":"2024-03-06T07:27:09.223549+0000","last_became_active":"2024-03-06T07:27:06.918433+0000","last_became_peered":"2024-03-06T07:27:06.918433+0000","last_unstale":"2024-03-06T07:27:09.223549+0000","last_undegraded":"2024-03-06T07:27:09.223549+0000","last_fullsized":"2024-03-06T07:27:09.223549+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:21:09.808244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225325+0000","last_change":"2024-03-06T07:27:07.933938+0000","last_active":"2024-03-06T07:27:09.225325+0000","last_peered":"2024-03-06T07:27:09.225325+0000","last_clean":"2024-03-06T07:27:09.225325+0000","last_became_active":"2024-03-06T07:27:07.933656+0000","last_became_peered":"2024-03-06T07:27:07.933656+0000","last_unstale":"2024-03-06T07:27:09.225325+0000","last_undegraded":"2024-03-06T07:27:09.225325+0000","last_fullsized":"2024-03-06T07:27:09.225325+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:23:36.112418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224591+0000","last_change":"2024-03-06T07:27:06.913843+0000","last_active":"2024-03-06T07:27:09.224591+0000","last_peered":"2024-03-06T07:27:09.224591+0000","last_clean":"2024-03-06T07:27:09.224591+0000","last_became_active":"2024-03-06T07:27:06.912393+0000","last_became_peered":"2024-03-06T07:27:06.912393+0000","last_unstale":"2024-03-06T07:27:09.224591+0000","last_undegraded":"2024-03-06T07:27:09.224591+0000","last_fullsized":"2024-03-06T07:27:09.224591+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225673+0000","last_change":"2024-03-06T07:27:07.927885+0000","last_active":"2024-03-06T07:27:09.225673+0000","last_peered":"2024-03-06T07:27:09.225673+0000","last_clean":"2024-03-06T07:27:09.225673+0000","last_became_active":"2024-03-06T07:27:07.927215+0000","last_became_peered":"2024-03-06T07:27:07.927215+0000","last_unstale":"2024-03-06T07:27:09.225673+0000","last_undegraded":"2024-03-06T07:27:09.225673+0000","last_fullsized":"2024-03-06T07:27:09.225673+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:36:30.188524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918041+0000","last_change":"2024-03-06T07:27:06.921613+0000","last_active":"2024-03-06T07:27:08.918041+0000","last_peered":"2024-03-06T07:27:08.918041+0000","last_clean":"2024-03-06T07:27:08.918041+0000","last_became_active":"2024-03-06T07:27:06.921225+0000","last_became_peered":"2024-03-06T07:27:06.921225+0000","last_unstale":"2024-03-06T07:27:08.918041+0000","last_undegraded":"2024-03-06T07:27:08.918041+0000","last_fullsized":"2024-03-06T07:27:08.918041+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922980+0000","last_change":"2024-03-06T07:27:07.924340+0000","last_active":"2024-03-06T07:27:08.922980+0000","last_peered":"2024-03-06T07:27:08.922980+0000","last_clean":"2024-03-06T07:27:08.922980+0000","last_became_active":"2024-03-06T07:27:07.924090+0000","last_became_peered":"2024-03-06T07:27:07.924090+0000","last_unstale":"2024-03-06T07:27:08.922980+0000","last_undegraded":"2024-03-06T07:27:08.922980+0000","last_fullsized":"2024-03-06T07:27:08.922980+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:54.697587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223461+0000","last_change":"2024-03-06T07:27:06.933112+0000","last_active":"2024-03-06T07:27:09.223461+0000","last_peered":"2024-03-06T07:27:09.223461+0000","last_clean":"2024-03-06T07:27:09.223461+0000","last_became_active":"2024-03-06T07:27:06.932764+0000","last_became_peered":"2024-03-06T07:27:06.932764+0000","last_unstale":"2024-03-06T07:27:09.223461+0000","last_undegraded":"2024-03-06T07:27:09.223461+0000","last_fullsized":"2024-03-06T07:27:09.223461+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225383+0000","last_change":"2024-03-06T07:27:07.940694+0000","last_active":"2024-03-06T07:27:09.225383+0000","last_peered":"2024-03-06T07:27:09.225383+0000","last_clean":"2024-03-06T07:27:09.225383+0000","last_became_active":"2024-03-06T07:27:07.940352+0000","last_became_peered":"2024-03-06T07:27:07.940352+0000","last_unstale":"2024-03-06T07:27:09.225383+0000","last_undegraded":"2024-03-06T07:27:09.225383+0000","last_fullsized":"2024-03-06T07:27:09.225383+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:06.241503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":170,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224024+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:27:09.224024+0000","last_peered":"2024-03-06T07:27:09.224024+0000","last_clean":"2024-03-06T07:27:09.224024+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:27:09.224024+0000","last_undegraded":"2024-03-06T07:27:09.224024+0000","last_fullsized":"2024-03-06T07:27:09.224024+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922420+0000","last_change":"2024-03-06T07:27:06.931167+0000","last_active":"2024-03-06T07:27:08.922420+0000","last_peered":"2024-03-06T07:27:08.922420+0000","last_clean":"2024-03-06T07:27:08.922420+0000","last_became_active":"2024-03-06T07:27:06.930895+0000","last_became_peered":"2024-03-06T07:27:06.930895+0000","last_unstale":"2024-03-06T07:27:08.922420+0000","last_undegraded":"2024-03-06T07:27:08.922420+0000","last_fullsized":"2024-03-06T07:27:08.922420+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225706+0000","last_change":"2024-03-06T07:27:07.946419+0000","last_active":"2024-03-06T07:27:09.225706+0000","last_peered":"2024-03-06T07:27:09.225706+0000","last_clean":"2024-03-06T07:27:09.225706+0000","last_became_active":"2024-03-06T07:27:07.945795+0000","last_became_peered":"2024-03-06T07:27:07.945795+0000","last_unstale":"2024-03-06T07:27:09.225706+0000","last_undegraded":"2024-03-06T07:27:09.225706+0000","last_fullsized":"2024-03-06T07:27:09.225706+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224064+0000","last_change":"2024-03-06T07:27:06.936519+0000","last_active":"2024-03-06T07:27:09.224064+0000","last_peered":"2024-03-06T07:27:09.224064+0000","last_clean":"2024-03-06T07:27:09.224064+0000","last_became_active":"2024-03-06T07:27:06.934384+0000","last_became_peered":"2024-03-06T07:27:06.934384+0000","last_unstale":"2024-03-06T07:27:09.224064+0000","last_undegraded":"2024-03-06T07:27:09.224064+0000","last_fullsized":"2024-03-06T07:27:09.224064+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:29.611416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224613+0000","last_change":"2024-03-06T07:27:07.926848+0000","last_active":"2024-03-06T07:27:09.224613+0000","last_peered":"2024-03-06T07:27:09.224613+0000","last_clean":"2024-03-06T07:27:09.224613+0000","last_became_active":"2024-03-06T07:27:07.926255+0000","last_became_peered":"2024-03-06T07:27:07.926255+0000","last_unstale":"2024-03-06T07:27:09.224613+0000","last_undegraded":"2024-03-06T07:27:09.224613+0000","last_fullsized":"2024-03-06T07:27:09.224613+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:43:44.555041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223817+0000","last_change":"2024-03-06T07:27:06.912373+0000","last_active":"2024-03-06T07:27:09.223817+0000","last_peered":"2024-03-06T07:27:09.223817+0000","last_clean":"2024-03-06T07:27:09.223817+0000","last_became_active":"2024-03-06T07:27:06.911823+0000","last_became_peered":"2024-03-06T07:27:06.911823+0000","last_unstale":"2024-03-06T07:27:09.223817+0000","last_undegraded":"2024-03-06T07:27:09.223817+0000","last_fullsized":"2024-03-06T07:27:09.223817+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917936+0000","last_change":"2024-03-06T07:27:07.917473+0000","last_active":"2024-03-06T07:27:08.917936+0000","last_peered":"2024-03-06T07:27:08.917936+0000","last_clean":"2024-03-06T07:27:08.917936+0000","last_became_active":"2024-03-06T07:27:07.914951+0000","last_became_peered":"2024-03-06T07:27:07.914951+0000","last_unstale":"2024-03-06T07:27:08.917936+0000","last_undegraded":"2024-03-06T07:27:08.917936+0000","last_fullsized":"2024-03-06T07:27:08.917936+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922608+0000","last_change":"2024-03-06T07:27:06.928866+0000","last_active":"2024-03-06T07:27:08.922608+0000","last_peered":"2024-03-06T07:27:08.922608+0000","last_clean":"2024-03-06T07:27:08.922608+0000","last_became_active":"2024-03-06T07:27:06.928456+0000","last_became_peered":"2024-03-06T07:27:06.928456+0000","last_unstale":"2024-03-06T07:27:08.922608+0000","last_undegraded":"2024-03-06T07:27:08.922608+0000","last_fullsized":"2024-03-06T07:27:08.922608+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223508+0000","last_change":"2024-03-06T07:27:07.933229+0000","last_active":"2024-03-06T07:27:09.223508+0000","last_peered":"2024-03-06T07:27:09.223508+0000","last_clean":"2024-03-06T07:27:09.223508+0000","last_became_active":"2024-03-06T07:27:07.932737+0000","last_became_peered":"2024-03-06T07:27:07.932737+0000","last_unstale":"2024-03-06T07:27:09.223508+0000","last_undegraded":"2024-03-06T07:27:09.223508+0000","last_fullsized":"2024-03-06T07:27:09.223508+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:20.889857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918205+0000","last_change":"2024-03-06T07:27:06.918955+0000","last_active":"2024-03-06T07:27:08.918205+0000","last_peered":"2024-03-06T07:27:08.918205+0000","last_clean":"2024-03-06T07:27:08.918205+0000","last_became_active":"2024-03-06T07:27:06.918655+0000","last_became_peered":"2024-03-06T07:27:06.918655+0000","last_unstale":"2024-03-06T07:27:08.918205+0000","last_undegraded":"2024-03-06T07:27:08.918205+0000","last_fullsized":"2024-03-06T07:27:08.918205+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922887+0000","last_change":"2024-03-06T07:27:07.924920+0000","last_active":"2024-03-06T07:27:08.922887+0000","last_peered":"2024-03-06T07:27:08.922887+0000","last_clean":"2024-03-06T07:27:08.922887+0000","last_became_active":"2024-03-06T07:27:07.924624+0000","last_became_peered":"2024-03-06T07:27:07.924624+0000","last_unstale":"2024-03-06T07:27:08.922887+0000","last_undegraded":"2024-03-06T07:27:08.922887+0000","last_fullsized":"2024-03-06T07:27:08.922887+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:59:43.062703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225644+0000","last_change":"2024-03-06T07:27:06.936617+0000","last_active":"2024-03-06T07:27:09.225644+0000","last_peered":"2024-03-06T07:27:09.225644+0000","last_clean":"2024-03-06T07:27:09.225644+0000","last_became_active":"2024-03-06T07:27:06.936328+0000","last_became_peered":"2024-03-06T07:27:06.936328+0000","last_unstale":"2024-03-06T07:27:09.225644+0000","last_undegraded":"2024-03-06T07:27:09.225644+0000","last_fullsized":"2024-03-06T07:27:09.225644+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:42:10.164751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918287+0000","last_change":"2024-03-06T07:27:07.916912+0000","last_active":"2024-03-06T07:27:08.918287+0000","last_peered":"2024-03-06T07:27:08.918287+0000","last_clean":"2024-03-06T07:27:08.918287+0000","last_became_active":"2024-03-06T07:27:07.916557+0000","last_became_peered":"2024-03-06T07:27:07.916557+0000","last_unstale":"2024-03-06T07:27:08.918287+0000","last_undegraded":"2024-03-06T07:27:08.918287+0000","last_fullsized":"2024-03-06T07:27:08.918287+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"22'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.931761+0000","last_change":"2024-03-06T07:27:06.934214+0000","last_active":"2024-03-06T07:27:08.931761+0000","last_peered":"2024-03-06T07:27:08.931761+0000","last_clean":"2024-03-06T07:27:08.931761+0000","last_became_active":"2024-03-06T07:27:06.933984+0000","last_became_peered":"2024-03-06T07:27:06.933984+0000","last_unstale":"2024-03-06T07:27:08.931761+0000","last_undegraded":"2024-03-06T07:27:08.931761+0000","last_fullsized":"2024-03-06T07:27:08.931761+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227628+0000","last_change":"2024-03-06T07:27:07.934955+0000","last_active":"2024-03-06T07:27:09.227628+0000","last_peered":"2024-03-06T07:27:09.227628+0000","last_clean":"2024-03-06T07:27:09.227628+0000","last_became_active":"2024-03-06T07:27:07.934602+0000","last_became_peered":"2024-03-06T07:27:07.934602+0000","last_unstale":"2024-03-06T07:27:09.227628+0000","last_undegraded":"2024-03-06T07:27:09.227628+0000","last_fullsized":"2024-03-06T07:27:09.227628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933574+0000","last_change":"2024-03-06T07:27:06.943288+0000","last_active":"2024-03-06T07:27:08.933574+0000","last_peered":"2024-03-06T07:27:08.933574+0000","last_clean":"2024-03-06T07:27:08.933574+0000","last_became_active":"2024-03-06T07:27:06.942813+0000","last_became_peered":"2024-03-06T07:27:06.942813+0000","last_unstale":"2024-03-06T07:27:08.933574+0000","last_undegraded":"2024-03-06T07:27:08.933574+0000","last_fullsized":"2024-03-06T07:27:08.933574+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226729+0000","last_change":"2024-03-06T07:27:07.920024+0000","last_active":"2024-03-06T07:27:09.226729+0000","last_peered":"2024-03-06T07:27:09.226729+0000","last_clean":"2024-03-06T07:27:09.226729+0000","last_became_active":"2024-03-06T07:27:07.919750+0000","last_became_peered":"2024-03-06T07:27:07.919750+0000","last_unstale":"2024-03-06T07:27:09.226729+0000","last_undegraded":"2024-03-06T07:27:09.226729+0000","last_fullsized":"2024-03-06T07:27:09.226729+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223933+0000","last_change":"2024-03-06T07:27:06.936885+0000","last_active":"2024-03-06T07:27:09.223933+0000","last_peered":"2024-03-06T07:27:09.223933+0000","last_clean":"2024-03-06T07:27:09.223933+0000","last_became_active":"2024-03-06T07:27:06.934829+0000","last_became_peered":"2024-03-06T07:27:06.934829+0000","last_unstale":"2024-03-06T07:27:09.223933+0000","last_undegraded":"2024-03-06T07:27:09.223933+0000","last_fullsized":"2024-03-06T07:27:09.223933+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223030+0000","last_change":"2024-03-06T07:27:07.916276+0000","last_active":"2024-03-06T07:27:09.223030+0000","last_peered":"2024-03-06T07:27:09.223030+0000","last_clean":"2024-03-06T07:27:09.223030+0000","last_became_active":"2024-03-06T07:27:07.915974+0000","last_became_peered":"2024-03-06T07:27:07.915974+0000","last_unstale":"2024-03-06T07:27:09.223030+0000","last_undegraded":"2024-03-06T07:27:09.223030+0000","last_fullsized":"2024-03-06T07:27:09.223030+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.935659+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.935373+0000","last_became_peered":"2024-03-06T07:27:06.935373+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223769+0000","last_change":"2024-03-06T07:27:07.910767+0000","last_active":"2024-03-06T07:27:09.223769+0000","last_peered":"2024-03-06T07:27:09.223769+0000","last_clean":"2024-03-06T07:27:09.223769+0000","last_became_active":"2024-03-06T07:27:07.910268+0000","last_became_peered":"2024-03-06T07:27:07.910268+0000","last_unstale":"2024-03-06T07:27:09.223769+0000","last_undegraded":"2024-03-06T07:27:09.223769+0000","last_fullsized":"2024-03-06T07:27:09.223769+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917633+0000","last_change":"2024-03-06T07:27:06.923056+0000","last_active":"2024-03-06T07:27:08.917633+0000","last_peered":"2024-03-06T07:27:08.917633+0000","last_clean":"2024-03-06T07:27:08.917633+0000","last_became_active":"2024-03-06T07:27:06.922710+0000","last_became_peered":"2024-03-06T07:27:06.922710+0000","last_unstale":"2024-03-06T07:27:08.917633+0000","last_undegraded":"2024-03-06T07:27:08.917633+0000","last_fullsized":"2024-03-06T07:27:08.917633+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226527+0000","last_change":"2024-03-06T07:27:07.922445+0000","last_active":"2024-03-06T07:27:09.226527+0000","last_peered":"2024-03-06T07:27:09.226527+0000","last_clean":"2024-03-06T07:27:09.226527+0000","last_became_active":"2024-03-06T07:27:07.922147+0000","last_became_peered":"2024-03-06T07:27:07.922147+0000","last_unstale":"2024-03-06T07:27:09.226527+0000","last_undegraded":"2024-03-06T07:27:09.226527+0000","last_fullsized":"2024-03-06T07:27:09.226527+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224979+0000","last_change":"2024-03-06T07:27:06.940633+0000","last_active":"2024-03-06T07:27:09.224979+0000","last_peered":"2024-03-06T07:27:09.224979+0000","last_clean":"2024-03-06T07:27:09.224979+0000","last_became_active":"2024-03-06T07:27:06.937434+0000","last_became_peered":"2024-03-06T07:27:06.937434+0000","last_unstale":"2024-03-06T07:27:09.224979+0000","last_undegraded":"2024-03-06T07:27:09.224979+0000","last_fullsized":"2024-03-06T07:27:09.224979+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932876+0000","last_change":"2024-03-06T07:27:07.942081+0000","last_active":"2024-03-06T07:27:08.932876+0000","last_peered":"2024-03-06T07:27:08.932876+0000","last_clean":"2024-03-06T07:27:08.932876+0000","last_became_active":"2024-03-06T07:27:07.941647+0000","last_became_peered":"2024-03-06T07:27:07.941647+0000","last_unstale":"2024-03-06T07:27:08.932876+0000","last_undegraded":"2024-03-06T07:27:08.932876+0000","last_fullsized":"2024-03-06T07:27:08.932876+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:47:21.103970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922929+0000","last_change":"2024-03-06T07:27:07.944160+0000","last_active":"2024-03-06T07:27:08.922929+0000","last_peered":"2024-03-06T07:27:08.922929+0000","last_clean":"2024-03-06T07:27:08.922929+0000","last_became_active":"2024-03-06T07:27:07.943238+0000","last_became_peered":"2024-03-06T07:27:07.943238+0000","last_unstale":"2024-03-06T07:27:08.922929+0000","last_undegraded":"2024-03-06T07:27:08.922929+0000","last_fullsized":"2024-03-06T07:27:08.922929+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:51.931654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224835+0000","last_change":"2024-03-06T07:27:06.940899+0000","last_active":"2024-03-06T07:27:09.224835+0000","last_peered":"2024-03-06T07:27:09.224835+0000","last_clean":"2024-03-06T07:27:09.224835+0000","last_became_active":"2024-03-06T07:27:06.940627+0000","last_became_peered":"2024-03-06T07:27:06.940627+0000","last_unstale":"2024-03-06T07:27:09.224835+0000","last_undegraded":"2024-03-06T07:27:09.224835+0000","last_fullsized":"2024-03-06T07:27:09.224835+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.933712+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.933437+0000","last_became_peered":"2024-03-06T07:27:07.933437+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:18.550912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225416+0000","last_change":"2024-03-06T07:27:06.938164+0000","last_active":"2024-03-06T07:27:09.225416+0000","last_peered":"2024-03-06T07:27:09.225416+0000","last_clean":"2024-03-06T07:27:09.225416+0000","last_became_active":"2024-03-06T07:27:06.937669+0000","last_became_peered":"2024-03-06T07:27:06.937669+0000","last_unstale":"2024-03-06T07:27:09.225416+0000","last_undegraded":"2024-03-06T07:27:09.225416+0000","last_fullsized":"2024-03-06T07:27:09.225416+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:19.647400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226963+0000","last_change":"2024-03-06T07:27:07.948380+0000","last_active":"2024-03-06T07:27:09.226963+0000","last_peered":"2024-03-06T07:27:09.226963+0000","last_clean":"2024-03-06T07:27:09.226963+0000","last_became_active":"2024-03-06T07:27:07.948110+0000","last_became_peered":"2024-03-06T07:27:07.948110+0000","last_unstale":"2024-03-06T07:27:09.226963+0000","last_undegraded":"2024-03-06T07:27:09.226963+0000","last_fullsized":"2024-03-06T07:27:09.226963+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227256+0000","last_change":"2024-03-06T07:27:06.911519+0000","last_active":"2024-03-06T07:27:09.227256+0000","last_peered":"2024-03-06T07:27:09.227256+0000","last_clean":"2024-03-06T07:27:09.227256+0000","last_became_active":"2024-03-06T07:27:06.911051+0000","last_became_peered":"2024-03-06T07:27:06.911051+0000","last_unstale":"2024-03-06T07:27:09.227256+0000","last_undegraded":"2024-03-06T07:27:09.227256+0000","last_fullsized":"2024-03-06T07:27:09.227256+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227138+0000","last_change":"2024-03-06T07:27:07.937255+0000","last_active":"2024-03-06T07:27:09.227138+0000","last_peered":"2024-03-06T07:27:09.227138+0000","last_clean":"2024-03-06T07:27:09.227138+0000","last_became_active":"2024-03-06T07:27:07.936713+0000","last_became_peered":"2024-03-06T07:27:07.936713+0000","last_unstale":"2024-03-06T07:27:09.227138+0000","last_undegraded":"2024-03-06T07:27:09.227138+0000","last_fullsized":"2024-03-06T07:27:09.227138+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"21'4","reported_seq":18,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227467+0000","last_change":"2024-03-06T07:27:06.935678+0000","last_active":"2024-03-06T07:27:09.227467+0000","last_peered":"2024-03-06T07:27:09.227467+0000","last_clean":"2024-03-06T07:27:09.227467+0000","last_became_active":"2024-03-06T07:27:06.935349+0000","last_became_peered":"2024-03-06T07:27:06.935349+0000","last_unstale":"2024-03-06T07:27:09.227467+0000","last_undegraded":"2024-03-06T07:27:09.227467+0000","last_fullsized":"2024-03-06T07:27:09.227467+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542149,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27836,"kb_used_data":804,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344004,"statfs":{"total":96636764160,"available":96608260096,"internally_reserved":0,"allocated":823296,"data_stored":669784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542148,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542149,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27244,"kb_used_data":224,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344596,"statfs":{"total":96636764160,"available":96608866304,"internally_reserved":0,"allocated":229376,"data_stored":78924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542148,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":78960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542148,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542149,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27248,"kb_used_data":220,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344592,"statfs":{"total":96636764160,"available":96608862208,"internally_reserved":0,"allocated":225280,"data_stored":78834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542150,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27256,"kb_used_data":228,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344584,"statfs":{"total":96636764160,"available":96608854016,"internally_reserved":0,"allocated":233472,"data_stored":78994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542147,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27844,"kb_used_data":812,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343996,"statfs":{"total":96636764160,"available":96608251904,"internally_reserved":0,"allocated":831488,"data_stored":669840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:27:12.333 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:27:12.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.578+0000 7f386de00640 1 Processor -- start 2024-03-06T07:27:12.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.578+0000 7f386de00640 1 -- start start 2024-03-06T07:27:12.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f386de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f386815fe60 0x7f3868071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f386de00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f386806cef0 con 0x7f3868144c00 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f386de00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f386806d1d0 con 0x7f386815f9d0 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f386de00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f386806d4b0 con 0x7f386815fe60 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f3866a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3868144c00 0x7f386806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59256/0 (socket says 172.21.15.86:59256) 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f3866a00640 1 -- 172.21.15.86:0/2340132035 learned_addr learned my addr 172.21.15.86:0/2340132035 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f3867400640 1 --2- 172.21.15.86:0/2340132035 >> v2:172.21.15.86:3300/0 conn(0x7f386815fe60 0x7f3868071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f3867400640 1 -- 172.21.15.86:0/2340132035 >> v1:172.21.15.194:6789/0 conn(0x7f386815f9d0 legacy=0x7f3868070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f3867400640 1 -- 172.21.15.86:0/2340132035 >> v1:172.21.15.86:6789/0 conn(0x7f3868144c00 legacy=0x7f386806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.580+0000 7f3867400640 1 -- 172.21.15.86:0/2340132035 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f386806d790 con 0x7f386815fe60 2024-03-06T07:27:12.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.582+0000 7f3867400640 1 --2- 172.21.15.86:0/2340132035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815fe60 0x7f3868071220 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f3860009d90 tx=0x7f386002beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c35cb710ea7db526 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:12.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.582+0000 7f3866000640 1 -- 172.21.15.86:0/2340132035 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3860039070 con 0x7f386815fe60 2024-03-06T07:27:12.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.582+0000 7f3866000640 1 -- 172.21.15.86:0/2340132035 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f386002cc60 con 0x7f386815fe60 2024-03-06T07:27:12.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.582+0000 7f3866000640 1 -- 172.21.15.86:0/2340132035 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3860034660 con 0x7f386815fe60 2024-03-06T07:27:12.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.582+0000 7f386de00640 1 -- 172.21.15.86:0/2340132035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815fe60 msgr2=0x7f3868071220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.582+0000 7f386de00640 1 --2- 172.21.15.86:0/2340132035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815fe60 0x7f3868071220 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f3860009d90 tx=0x7f386002beb0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.583+0000 7f386de00640 1 -- 172.21.15.86:0/2340132035 shutdown_connections 2024-03-06T07:27:12.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.583+0000 7f386de00640 1 --2- 172.21.15.86:0/2340132035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815fe60 0x7f3868071220 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.583+0000 7f386de00640 1 -- 172.21.15.86:0/2340132035 >> 172.21.15.86:0/2340132035 conn(0x7f38680b1730 msgr2=0x7f38680b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:12.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.583+0000 7f386de00640 1 -- 172.21.15.86:0/2340132035 shutdown_connections 2024-03-06T07:27:12.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.583+0000 7f386de00640 1 -- 172.21.15.86:0/2340132035 wait complete. 2024-03-06T07:27:12.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.584+0000 7f386de00640 1 Processor -- start 2024-03-06T07:27:12.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.585+0000 7f386de00640 1 -- start start 2024-03-06T07:27:12.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f386de00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3868144c00 0x7f3868140240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f386de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815f9d0 0x7f3868141cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f386de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f386815fe60 0x7f3868139040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f386de00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f386806cb50 con 0x7f386815f9d0 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f386de00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3868160730 con 0x7f3868144c00 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f386de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f386806ccd0 con 0x7f386815fe60 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815f9d0 0x7f3868141cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815f9d0 0x7f3868141cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53892/0 (socket says 172.21.15.86:53892) 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 -- 172.21.15.86:0/2090436250 learned_addr learned my addr 172.21.15.86:0/2090436250 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f3866a00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3868144c00 0x7f3868140240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f3867400640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f386815fe60 0x7f3868139040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 -- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f386815fe60 msgr2=0x7f3868139040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f386815fe60 0x7f3868139040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 -- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3868144c00 msgr2=0x7f3868140240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3868144c00 0x7f3868140240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f385fe00640 1 -- 172.21.15.86:0/2090436250 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3868140780 con 0x7f386815f9d0 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.586+0000 7f3866a00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3868144c00 0x7f3868140240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.587+0000 7f385fe00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815f9d0 0x7f3868141cc0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f385000c430 tx=0x7f385000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.587+0000 7f385f400640 1 -- 172.21.15.86:0/2090436250 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3850011790 con 0x7f386815f9d0 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.587+0000 7f385f400640 1 -- 172.21.15.86:0/2090436250 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3850017070 con 0x7f386815f9d0 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.587+0000 7f385f400640 1 -- 172.21.15.86:0/2090436250 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3850011bb0 con 0x7f386815f9d0 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.587+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f386813a2e0 con 0x7f386815f9d0 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.587+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f386813a8a0 con 0x7f386815f9d0 2024-03-06T07:27:12.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.588+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f386804f330 con 0x7f386815f9d0 2024-03-06T07:27:12.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.589+0000 7f385f400640 1 -- 172.21.15.86:0/2090436250 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f385001c460 con 0x7f386815f9d0 2024-03-06T07:27:12.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.590+0000 7f385f400640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f384406b4d0 0x7f384406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:12.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.590+0000 7f385f400640 1 -- 172.21.15.86:0/2090436250 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f385008a100 con 0x7f386815f9d0 2024-03-06T07:27:12.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.590+0000 7f3866a00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f384406b4d0 0x7f384406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:12.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.591+0000 7f3866a00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f384406b4d0 0x7f384406d990 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f3868142660 tx=0x7f3858009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:12.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.595+0000 7f385f400640 1 -- 172.21.15.86:0/2090436250 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3850058ee0 con 0x7f386815f9d0 2024-03-06T07:27:12.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.789+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f3868072890 con 0x7f384406b4d0 2024-03-06T07:27:12.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.797+0000 7f385f400640 1 -- 172.21.15.86:0/2090436250 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251946 (secure 0 0 0) 0x7f3868072890 con 0x7f384406b4d0 2024-03-06T07:27:12.803 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:12.804 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f384406b4d0 msgr2=0x7f384406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f384406b4d0 0x7f384406d990 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f3868142660 tx=0x7f3858009040 comp rx=0 tx=0).stop 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815f9d0 msgr2=0x7f3868141cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815f9d0 0x7f3868141cc0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f385000c430 tx=0x7f385000c8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 shutdown_connections 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f384406b4d0 0x7f384406d990 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f386815fe60 0x7f3868139040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f386815f9d0 0x7f3868141cc0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 --2- 172.21.15.86:0/2090436250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3868144c00 0x7f3868140240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 >> 172.21.15.86:0/2090436250 conn(0x7f38680b1730 msgr2=0x7f38680b06a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 shutdown_connections 2024-03-06T07:27:12.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:12.803+0000 7f386de00640 1 -- 172.21.15.86:0/2090436250 wait complete. 2024-03-06T07:27:12.825 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-03-06T07:27:10.851640+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":109,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219200,"kb_used_data":2984,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755520,"statfs":{"total":773094113280,"available":772869652480,"internally_reserved":0,"allocated":3055616,"data_stored":1816228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":221236787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":42,"apply_latency_ms":42,"commit_latency_ns":42000000,"apply_latency_ns":42000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":59,"num_write_kb":276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.973683"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225431+0000","last_change":"2024-03-06T07:27:07.923330+0000","last_active":"2024-03-06T07:27:09.225431+0000","last_peered":"2024-03-06T07:27:09.225431+0000","last_clean":"2024-03-06T07:27:09.225431+0000","last_became_active":"2024-03-06T07:27:07.923084+0000","last_became_peered":"2024-03-06T07:27:07.923084+0000","last_unstale":"2024-03-06T07:27:09.225431+0000","last_undegraded":"2024-03-06T07:27:09.225431+0000","last_fullsized":"2024-03-06T07:27:09.225431+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:46:48.027042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918758+0000","last_change":"2024-03-06T07:27:07.921104+0000","last_active":"2024-03-06T07:27:08.918758+0000","last_peered":"2024-03-06T07:27:08.918758+0000","last_clean":"2024-03-06T07:27:08.918758+0000","last_became_active":"2024-03-06T07:27:07.920770+0000","last_became_peered":"2024-03-06T07:27:07.920770+0000","last_unstale":"2024-03-06T07:27:08.918758+0000","last_undegraded":"2024-03-06T07:27:08.918758+0000","last_fullsized":"2024-03-06T07:27:08.918758+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226481+0000","last_change":"2024-03-06T07:27:07.922511+0000","last_active":"2024-03-06T07:27:09.226481+0000","last_peered":"2024-03-06T07:27:09.226481+0000","last_clean":"2024-03-06T07:27:09.226481+0000","last_became_active":"2024-03-06T07:27:07.922242+0000","last_became_peered":"2024-03-06T07:27:07.922242+0000","last_unstale":"2024-03-06T07:27:09.226481+0000","last_undegraded":"2024-03-06T07:27:09.226481+0000","last_fullsized":"2024-03-06T07:27:09.226481+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922737+0000","last_change":"2024-03-06T07:27:07.945609+0000","last_active":"2024-03-06T07:27:08.922737+0000","last_peered":"2024-03-06T07:27:08.922737+0000","last_clean":"2024-03-06T07:27:08.922737+0000","last_became_active":"2024-03-06T07:27:07.945312+0000","last_became_peered":"2024-03-06T07:27:07.945312+0000","last_unstale":"2024-03-06T07:27:08.922737+0000","last_undegraded":"2024-03-06T07:27:08.922737+0000","last_fullsized":"2024-03-06T07:27:08.922737+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:38.769682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224858+0000","last_change":"2024-03-06T07:27:07.921341+0000","last_active":"2024-03-06T07:27:09.224858+0000","last_peered":"2024-03-06T07:27:09.224858+0000","last_clean":"2024-03-06T07:27:09.224858+0000","last_became_active":"2024-03-06T07:27:07.920986+0000","last_became_peered":"2024-03-06T07:27:07.920986+0000","last_unstale":"2024-03-06T07:27:09.224858+0000","last_undegraded":"2024-03-06T07:27:09.224858+0000","last_fullsized":"2024-03-06T07:27:09.224858+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922064+0000","last_change":"2024-03-06T07:27:07.934696+0000","last_active":"2024-03-06T07:27:08.922064+0000","last_peered":"2024-03-06T07:27:08.922064+0000","last_clean":"2024-03-06T07:27:08.922064+0000","last_became_active":"2024-03-06T07:27:07.934400+0000","last_became_peered":"2024-03-06T07:27:07.934400+0000","last_unstale":"2024-03-06T07:27:08.922064+0000","last_undegraded":"2024-03-06T07:27:08.922064+0000","last_fullsized":"2024-03-06T07:27:08.922064+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224278+0000","last_change":"2024-03-06T07:27:07.926117+0000","last_active":"2024-03-06T07:27:09.224278+0000","last_peered":"2024-03-06T07:27:09.224278+0000","last_clean":"2024-03-06T07:27:09.224278+0000","last_became_active":"2024-03-06T07:27:07.925805+0000","last_became_peered":"2024-03-06T07:27:07.925805+0000","last_unstale":"2024-03-06T07:27:09.224278+0000","last_undegraded":"2024-03-06T07:27:09.224278+0000","last_fullsized":"2024-03-06T07:27:09.224278+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223418+0000","last_change":"2024-03-06T07:27:07.946584+0000","last_active":"2024-03-06T07:27:09.223418+0000","last_peered":"2024-03-06T07:27:09.223418+0000","last_clean":"2024-03-06T07:27:09.223418+0000","last_became_active":"2024-03-06T07:27:07.945766+0000","last_became_peered":"2024-03-06T07:27:07.945766+0000","last_unstale":"2024-03-06T07:27:09.223418+0000","last_undegraded":"2024-03-06T07:27:09.223418+0000","last_fullsized":"2024-03-06T07:27:09.223418+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225430+0000","last_change":"2024-03-06T07:27:07.948853+0000","last_active":"2024-03-06T07:27:09.225430+0000","last_peered":"2024-03-06T07:27:09.225430+0000","last_clean":"2024-03-06T07:27:09.225430+0000","last_became_active":"2024-03-06T07:27:07.948523+0000","last_became_peered":"2024-03-06T07:27:07.948523+0000","last_unstale":"2024-03-06T07:27:09.225430+0000","last_undegraded":"2024-03-06T07:27:09.225430+0000","last_fullsized":"2024-03-06T07:27:09.225430+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:17:40.285874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933598+0000","last_change":"2024-03-06T07:27:07.946279+0000","last_active":"2024-03-06T07:27:08.933598+0000","last_peered":"2024-03-06T07:27:08.933598+0000","last_clean":"2024-03-06T07:27:08.933598+0000","last_became_active":"2024-03-06T07:27:07.945895+0000","last_became_peered":"2024-03-06T07:27:07.945895+0000","last_unstale":"2024-03-06T07:27:08.933598+0000","last_undegraded":"2024-03-06T07:27:08.933598+0000","last_fullsized":"2024-03-06T07:27:08.933598+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933628+0000","last_change":"2024-03-06T07:27:07.922684+0000","last_active":"2024-03-06T07:27:08.933628+0000","last_peered":"2024-03-06T07:27:08.933628+0000","last_clean":"2024-03-06T07:27:08.933628+0000","last_became_active":"2024-03-06T07:27:07.922348+0000","last_became_peered":"2024-03-06T07:27:07.922348+0000","last_unstale":"2024-03-06T07:27:08.933628+0000","last_undegraded":"2024-03-06T07:27:08.933628+0000","last_fullsized":"2024-03-06T07:27:08.933628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:38:38.919038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223388+0000","last_change":"2024-03-06T07:27:07.923877+0000","last_active":"2024-03-06T07:27:09.223388+0000","last_peered":"2024-03-06T07:27:09.223388+0000","last_clean":"2024-03-06T07:27:09.223388+0000","last_became_active":"2024-03-06T07:27:07.923410+0000","last_became_peered":"2024-03-06T07:27:07.923410+0000","last_unstale":"2024-03-06T07:27:09.223388+0000","last_undegraded":"2024-03-06T07:27:09.223388+0000","last_fullsized":"2024-03-06T07:27:09.223388+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:30.853873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227024+0000","last_change":"2024-03-06T07:27:07.930873+0000","last_active":"2024-03-06T07:27:09.227024+0000","last_peered":"2024-03-06T07:27:09.227024+0000","last_clean":"2024-03-06T07:27:09.227024+0000","last_became_active":"2024-03-06T07:27:07.930653+0000","last_became_peered":"2024-03-06T07:27:07.930653+0000","last_unstale":"2024-03-06T07:27:09.227024+0000","last_undegraded":"2024-03-06T07:27:09.227024+0000","last_fullsized":"2024-03-06T07:27:09.227024+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224966+0000","last_change":"2024-03-06T07:27:07.919903+0000","last_active":"2024-03-06T07:27:09.224966+0000","last_peered":"2024-03-06T07:27:09.224966+0000","last_clean":"2024-03-06T07:27:09.224966+0000","last_became_active":"2024-03-06T07:27:07.919585+0000","last_became_peered":"2024-03-06T07:27:07.919585+0000","last_unstale":"2024-03-06T07:27:09.224966+0000","last_undegraded":"2024-03-06T07:27:09.224966+0000","last_fullsized":"2024-03-06T07:27:09.224966+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:06.597617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917655+0000","last_change":"2024-03-06T07:27:07.908420+0000","last_active":"2024-03-06T07:27:08.917655+0000","last_peered":"2024-03-06T07:27:08.917655+0000","last_clean":"2024-03-06T07:27:08.917655+0000","last_became_active":"2024-03-06T07:27:07.907392+0000","last_became_peered":"2024-03-06T07:27:07.907392+0000","last_unstale":"2024-03-06T07:27:08.917655+0000","last_undegraded":"2024-03-06T07:27:08.917655+0000","last_fullsized":"2024-03-06T07:27:08.917655+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922133+0000","last_change":"2024-03-06T07:27:07.934890+0000","last_active":"2024-03-06T07:27:08.922133+0000","last_peered":"2024-03-06T07:27:08.922133+0000","last_clean":"2024-03-06T07:27:08.922133+0000","last_became_active":"2024-03-06T07:27:07.934568+0000","last_became_peered":"2024-03-06T07:27:07.934568+0000","last_unstale":"2024-03-06T07:27:08.922133+0000","last_undegraded":"2024-03-06T07:27:08.922133+0000","last_fullsized":"2024-03-06T07:27:08.922133+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923353+0000","last_change":"2024-03-06T07:27:07.927869+0000","last_active":"2024-03-06T07:27:08.923353+0000","last_peered":"2024-03-06T07:27:08.923353+0000","last_clean":"2024-03-06T07:27:08.923353+0000","last_became_active":"2024-03-06T07:27:07.927507+0000","last_became_peered":"2024-03-06T07:27:07.927507+0000","last_unstale":"2024-03-06T07:27:08.923353+0000","last_undegraded":"2024-03-06T07:27:08.923353+0000","last_fullsized":"2024-03-06T07:27:08.923353+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:26:20.914324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224317+0000","last_change":"2024-03-06T07:27:07.922930+0000","last_active":"2024-03-06T07:27:09.224317+0000","last_peered":"2024-03-06T07:27:09.224317+0000","last_clean":"2024-03-06T07:27:09.224317+0000","last_became_active":"2024-03-06T07:27:07.922608+0000","last_became_peered":"2024-03-06T07:27:07.922608+0000","last_unstale":"2024-03-06T07:27:09.224317+0000","last_undegraded":"2024-03-06T07:27:09.224317+0000","last_fullsized":"2024-03-06T07:27:09.224317+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225850+0000","last_change":"2024-03-06T07:27:07.927132+0000","last_active":"2024-03-06T07:27:09.225850+0000","last_peered":"2024-03-06T07:27:09.225850+0000","last_clean":"2024-03-06T07:27:09.225850+0000","last_became_active":"2024-03-06T07:27:07.926850+0000","last_became_peered":"2024-03-06T07:27:07.926850+0000","last_unstale":"2024-03-06T07:27:09.225850+0000","last_undegraded":"2024-03-06T07:27:09.225850+0000","last_fullsized":"2024-03-06T07:27:09.225850+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:23.735936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933468+0000","last_change":"2024-03-06T07:27:07.928126+0000","last_active":"2024-03-06T07:27:08.933468+0000","last_peered":"2024-03-06T07:27:08.933468+0000","last_clean":"2024-03-06T07:27:08.933468+0000","last_became_active":"2024-03-06T07:27:07.927650+0000","last_became_peered":"2024-03-06T07:27:07.927650+0000","last_unstale":"2024-03-06T07:27:08.933468+0000","last_undegraded":"2024-03-06T07:27:08.933468+0000","last_fullsized":"2024-03-06T07:27:08.933468+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:55.755072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225914+0000","last_change":"2024-03-06T07:27:07.927942+0000","last_active":"2024-03-06T07:27:09.225914+0000","last_peered":"2024-03-06T07:27:09.225914+0000","last_clean":"2024-03-06T07:27:09.225914+0000","last_became_active":"2024-03-06T07:27:07.927629+0000","last_became_peered":"2024-03-06T07:27:07.927629+0000","last_unstale":"2024-03-06T07:27:09.225914+0000","last_undegraded":"2024-03-06T07:27:09.225914+0000","last_fullsized":"2024-03-06T07:27:09.225914+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922498+0000","last_change":"2024-03-06T07:27:07.922273+0000","last_active":"2024-03-06T07:27:08.922498+0000","last_peered":"2024-03-06T07:27:08.922498+0000","last_clean":"2024-03-06T07:27:08.922498+0000","last_became_active":"2024-03-06T07:27:07.921988+0000","last_became_peered":"2024-03-06T07:27:07.921988+0000","last_unstale":"2024-03-06T07:27:08.922498+0000","last_undegraded":"2024-03-06T07:27:08.922498+0000","last_fullsized":"2024-03-06T07:27:08.922498+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:57.831357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223829+0000","last_change":"2024-03-06T07:27:07.930778+0000","last_active":"2024-03-06T07:27:09.223829+0000","last_peered":"2024-03-06T07:27:09.223829+0000","last_clean":"2024-03-06T07:27:09.223829+0000","last_became_active":"2024-03-06T07:27:07.930464+0000","last_became_peered":"2024-03-06T07:27:07.930464+0000","last_unstale":"2024-03-06T07:27:09.223829+0000","last_undegraded":"2024-03-06T07:27:09.223829+0000","last_fullsized":"2024-03-06T07:27:09.223829+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922134+0000","last_change":"2024-03-06T07:27:07.917391+0000","last_active":"2024-03-06T07:27:08.922134+0000","last_peered":"2024-03-06T07:27:08.922134+0000","last_clean":"2024-03-06T07:27:08.922134+0000","last_became_active":"2024-03-06T07:27:07.917107+0000","last_became_peered":"2024-03-06T07:27:07.917107+0000","last_unstale":"2024-03-06T07:27:08.922134+0000","last_undegraded":"2024-03-06T07:27:08.922134+0000","last_fullsized":"2024-03-06T07:27:08.922134+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:08.467879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227476+0000","last_change":"2024-03-06T07:27:07.930923+0000","last_active":"2024-03-06T07:27:09.227476+0000","last_peered":"2024-03-06T07:27:09.227476+0000","last_clean":"2024-03-06T07:27:09.227476+0000","last_became_active":"2024-03-06T07:27:07.930078+0000","last_became_peered":"2024-03-06T07:27:07.930078+0000","last_unstale":"2024-03-06T07:27:09.227476+0000","last_undegraded":"2024-03-06T07:27:09.227476+0000","last_fullsized":"2024-03-06T07:27:09.227476+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933695+0000","last_change":"2024-03-06T07:27:07.936206+0000","last_active":"2024-03-06T07:27:08.933695+0000","last_peered":"2024-03-06T07:27:08.933695+0000","last_clean":"2024-03-06T07:27:08.933695+0000","last_became_active":"2024-03-06T07:27:07.930820+0000","last_became_peered":"2024-03-06T07:27:07.930820+0000","last_unstale":"2024-03-06T07:27:08.933695+0000","last_undegraded":"2024-03-06T07:27:08.933695+0000","last_fullsized":"2024-03-06T07:27:08.933695+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917816+0000","last_change":"2024-03-06T07:27:07.918528+0000","last_active":"2024-03-06T07:27:08.917816+0000","last_peered":"2024-03-06T07:27:08.917816+0000","last_clean":"2024-03-06T07:27:08.917816+0000","last_became_active":"2024-03-06T07:27:07.918221+0000","last_became_peered":"2024-03-06T07:27:07.918221+0000","last_unstale":"2024-03-06T07:27:08.917816+0000","last_undegraded":"2024-03-06T07:27:08.917816+0000","last_fullsized":"2024-03-06T07:27:08.917816+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227107+0000","last_change":"2024-03-06T07:27:07.929588+0000","last_active":"2024-03-06T07:27:09.227107+0000","last_peered":"2024-03-06T07:27:09.227107+0000","last_clean":"2024-03-06T07:27:09.227107+0000","last_became_active":"2024-03-06T07:27:07.929207+0000","last_became_peered":"2024-03-06T07:27:07.929207+0000","last_unstale":"2024-03-06T07:27:09.227107+0000","last_undegraded":"2024-03-06T07:27:09.227107+0000","last_fullsized":"2024-03-06T07:27:09.227107+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226074+0000","last_change":"2024-03-06T07:27:07.925047+0000","last_active":"2024-03-06T07:27:09.226074+0000","last_peered":"2024-03-06T07:27:09.226074+0000","last_clean":"2024-03-06T07:27:09.226074+0000","last_became_active":"2024-03-06T07:27:07.924734+0000","last_became_peered":"2024-03-06T07:27:07.924734+0000","last_unstale":"2024-03-06T07:27:09.226074+0000","last_undegraded":"2024-03-06T07:27:09.226074+0000","last_fullsized":"2024-03-06T07:27:09.226074+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922314+0000","last_change":"2024-03-06T07:27:07.927022+0000","last_active":"2024-03-06T07:27:08.922314+0000","last_peered":"2024-03-06T07:27:08.922314+0000","last_clean":"2024-03-06T07:27:08.922314+0000","last_became_active":"2024-03-06T07:27:07.926748+0000","last_became_peered":"2024-03-06T07:27:07.926748+0000","last_unstale":"2024-03-06T07:27:08.922314+0000","last_undegraded":"2024-03-06T07:27:08.922314+0000","last_fullsized":"2024-03-06T07:27:08.922314+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:25.638818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223563+0000","last_change":"2024-03-06T07:27:07.934988+0000","last_active":"2024-03-06T07:27:09.223563+0000","last_peered":"2024-03-06T07:27:09.223563+0000","last_clean":"2024-03-06T07:27:09.223563+0000","last_became_active":"2024-03-06T07:27:07.934606+0000","last_became_peered":"2024-03-06T07:27:07.934606+0000","last_unstale":"2024-03-06T07:27:09.223563+0000","last_undegraded":"2024-03-06T07:27:09.223563+0000","last_fullsized":"2024-03-06T07:27:09.223563+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933081+0000","last_change":"2024-03-06T07:27:07.929352+0000","last_active":"2024-03-06T07:27:08.933081+0000","last_peered":"2024-03-06T07:27:08.933081+0000","last_clean":"2024-03-06T07:27:08.933081+0000","last_became_active":"2024-03-06T07:27:07.929001+0000","last_became_peered":"2024-03-06T07:27:07.929001+0000","last_unstale":"2024-03-06T07:27:08.933081+0000","last_undegraded":"2024-03-06T07:27:08.933081+0000","last_fullsized":"2024-03-06T07:27:08.933081+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:17:25.890956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.222941+0000","last_change":"2024-03-06T07:27:07.906813+0000","last_active":"2024-03-06T07:27:09.222941+0000","last_peered":"2024-03-06T07:27:09.222941+0000","last_clean":"2024-03-06T07:27:09.222941+0000","last_became_active":"2024-03-06T07:27:07.906019+0000","last_became_peered":"2024-03-06T07:27:07.906019+0000","last_unstale":"2024-03-06T07:27:09.222941+0000","last_undegraded":"2024-03-06T07:27:09.222941+0000","last_fullsized":"2024-03-06T07:27:09.222941+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:18.357063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226805+0000","last_change":"2024-03-06T07:27:06.929035+0000","last_active":"2024-03-06T07:27:09.226805+0000","last_peered":"2024-03-06T07:27:09.226805+0000","last_clean":"2024-03-06T07:27:09.226805+0000","last_became_active":"2024-03-06T07:27:06.928702+0000","last_became_peered":"2024-03-06T07:27:06.928702+0000","last_unstale":"2024-03-06T07:27:09.226805+0000","last_undegraded":"2024-03-06T07:27:09.226805+0000","last_fullsized":"2024-03-06T07:27:09.226805+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226233+0000","last_change":"2024-03-06T07:27:07.938823+0000","last_active":"2024-03-06T07:27:09.226233+0000","last_peered":"2024-03-06T07:27:09.226233+0000","last_clean":"2024-03-06T07:27:09.226233+0000","last_became_active":"2024-03-06T07:27:07.937638+0000","last_became_peered":"2024-03-06T07:27:07.937638+0000","last_unstale":"2024-03-06T07:27:09.226233+0000","last_undegraded":"2024-03-06T07:27:09.226233+0000","last_fullsized":"2024-03-06T07:27:09.226233+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932806+0000","last_change":"2024-03-06T07:27:06.929034+0000","last_active":"2024-03-06T07:27:08.932806+0000","last_peered":"2024-03-06T07:27:08.932806+0000","last_clean":"2024-03-06T07:27:08.932806+0000","last_became_active":"2024-03-06T07:27:06.927376+0000","last_became_peered":"2024-03-06T07:27:06.927376+0000","last_unstale":"2024-03-06T07:27:08.932806+0000","last_undegraded":"2024-03-06T07:27:08.932806+0000","last_fullsized":"2024-03-06T07:27:08.932806+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225115+0000","last_change":"2024-03-06T07:27:06.933665+0000","last_active":"2024-03-06T07:27:09.225115+0000","last_peered":"2024-03-06T07:27:09.225115+0000","last_clean":"2024-03-06T07:27:09.225115+0000","last_became_active":"2024-03-06T07:27:06.933367+0000","last_became_peered":"2024-03-06T07:27:06.933367+0000","last_unstale":"2024-03-06T07:27:09.225115+0000","last_undegraded":"2024-03-06T07:27:09.225115+0000","last_fullsized":"2024-03-06T07:27:09.225115+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223616+0000","last_change":"2024-03-06T07:27:07.927629+0000","last_active":"2024-03-06T07:27:09.223616+0000","last_peered":"2024-03-06T07:27:09.223616+0000","last_clean":"2024-03-06T07:27:09.223616+0000","last_became_active":"2024-03-06T07:27:07.925804+0000","last_became_peered":"2024-03-06T07:27:07.925804+0000","last_unstale":"2024-03-06T07:27:09.223616+0000","last_undegraded":"2024-03-06T07:27:09.223616+0000","last_fullsized":"2024-03-06T07:27:09.223616+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922349+0000","last_change":"2024-03-06T07:27:06.935442+0000","last_active":"2024-03-06T07:27:08.922349+0000","last_peered":"2024-03-06T07:27:08.922349+0000","last_clean":"2024-03-06T07:27:08.922349+0000","last_became_active":"2024-03-06T07:27:06.933244+0000","last_became_peered":"2024-03-06T07:27:06.933244+0000","last_unstale":"2024-03-06T07:27:08.922349+0000","last_undegraded":"2024-03-06T07:27:08.922349+0000","last_fullsized":"2024-03-06T07:27:08.922349+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917698+0000","last_change":"2024-03-06T07:27:07.922392+0000","last_active":"2024-03-06T07:27:08.917698+0000","last_peered":"2024-03-06T07:27:08.917698+0000","last_clean":"2024-03-06T07:27:08.917698+0000","last_became_active":"2024-03-06T07:27:07.922127+0000","last_became_peered":"2024-03-06T07:27:07.922127+0000","last_unstale":"2024-03-06T07:27:08.917698+0000","last_undegraded":"2024-03-06T07:27:08.917698+0000","last_fullsized":"2024-03-06T07:27:08.917698+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933693+0000","last_change":"2024-03-06T07:27:06.937146+0000","last_active":"2024-03-06T07:27:08.933693+0000","last_peered":"2024-03-06T07:27:08.933693+0000","last_clean":"2024-03-06T07:27:08.933693+0000","last_became_active":"2024-03-06T07:27:06.936876+0000","last_became_peered":"2024-03-06T07:27:06.936876+0000","last_unstale":"2024-03-06T07:27:08.933693+0000","last_undegraded":"2024-03-06T07:27:08.933693+0000","last_fullsized":"2024-03-06T07:27:08.933693+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.943897+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.943627+0000","last_became_peered":"2024-03-06T07:27:07.943627+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:01:28.762594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923212+0000","last_change":"2024-03-06T07:27:06.940833+0000","last_active":"2024-03-06T07:27:08.923212+0000","last_peered":"2024-03-06T07:27:08.923212+0000","last_clean":"2024-03-06T07:27:08.923212+0000","last_became_active":"2024-03-06T07:27:06.939734+0000","last_became_peered":"2024-03-06T07:27:06.939734+0000","last_unstale":"2024-03-06T07:27:08.923212+0000","last_undegraded":"2024-03-06T07:27:08.923212+0000","last_fullsized":"2024-03-06T07:27:08.923212+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227338+0000","last_change":"2024-03-06T07:27:07.946830+0000","last_active":"2024-03-06T07:27:09.227338+0000","last_peered":"2024-03-06T07:27:09.227338+0000","last_clean":"2024-03-06T07:27:09.227338+0000","last_became_active":"2024-03-06T07:27:07.946495+0000","last_became_peered":"2024-03-06T07:27:07.946495+0000","last_unstale":"2024-03-06T07:27:09.227338+0000","last_undegraded":"2024-03-06T07:27:09.227338+0000","last_fullsized":"2024-03-06T07:27:09.227338+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"21'3","reported_seq":17,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.937692+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.937338+0000","last_became_peered":"2024-03-06T07:27:06.937338+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223197+0000","last_change":"2024-03-06T07:27:07.922246+0000","last_active":"2024-03-06T07:27:09.223197+0000","last_peered":"2024-03-06T07:27:09.223197+0000","last_clean":"2024-03-06T07:27:09.223197+0000","last_became_active":"2024-03-06T07:27:07.921469+0000","last_became_peered":"2024-03-06T07:27:07.921469+0000","last_unstale":"2024-03-06T07:27:09.223197+0000","last_undegraded":"2024-03-06T07:27:09.223197+0000","last_fullsized":"2024-03-06T07:27:09.223197+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"21'6","reported_seq":20,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933200+0000","last_change":"2024-03-06T07:27:06.926732+0000","last_active":"2024-03-06T07:27:08.933200+0000","last_peered":"2024-03-06T07:27:08.933200+0000","last_clean":"2024-03-06T07:27:08.933200+0000","last_became_active":"2024-03-06T07:27:06.925706+0000","last_became_peered":"2024-03-06T07:27:06.925706+0000","last_unstale":"2024-03-06T07:27:08.933200+0000","last_undegraded":"2024-03-06T07:27:08.933200+0000","last_fullsized":"2024-03-06T07:27:08.933200+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225109+0000","last_change":"2024-03-06T07:27:07.935123+0000","last_active":"2024-03-06T07:27:09.225109+0000","last_peered":"2024-03-06T07:27:09.225109+0000","last_clean":"2024-03-06T07:27:09.225109+0000","last_became_active":"2024-03-06T07:27:07.934820+0000","last_became_peered":"2024-03-06T07:27:07.934820+0000","last_unstale":"2024-03-06T07:27:09.225109+0000","last_undegraded":"2024-03-06T07:27:09.225109+0000","last_fullsized":"2024-03-06T07:27:09.225109+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:29.926787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226343+0000","last_change":"2024-03-06T07:27:06.921406+0000","last_active":"2024-03-06T07:27:09.226343+0000","last_peered":"2024-03-06T07:27:09.226343+0000","last_clean":"2024-03-06T07:27:09.226343+0000","last_became_active":"2024-03-06T07:27:06.921070+0000","last_became_peered":"2024-03-06T07:27:06.921070+0000","last_unstale":"2024-03-06T07:27:09.226343+0000","last_undegraded":"2024-03-06T07:27:09.226343+0000","last_fullsized":"2024-03-06T07:27:09.226343+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225286+0000","last_change":"2024-03-06T07:27:07.926884+0000","last_active":"2024-03-06T07:27:09.225286+0000","last_peered":"2024-03-06T07:27:09.225286+0000","last_clean":"2024-03-06T07:27:09.225286+0000","last_became_active":"2024-03-06T07:27:07.926600+0000","last_became_peered":"2024-03-06T07:27:07.926600+0000","last_unstale":"2024-03-06T07:27:09.225286+0000","last_undegraded":"2024-03-06T07:27:09.225286+0000","last_fullsized":"2024-03-06T07:27:09.225286+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922438+0000","last_change":"2024-03-06T07:27:06.936248+0000","last_active":"2024-03-06T07:27:08.922438+0000","last_peered":"2024-03-06T07:27:08.922438+0000","last_clean":"2024-03-06T07:27:08.922438+0000","last_became_active":"2024-03-06T07:27:06.934671+0000","last_became_peered":"2024-03-06T07:27:06.934671+0000","last_unstale":"2024-03-06T07:27:08.922438+0000","last_undegraded":"2024-03-06T07:27:08.922438+0000","last_fullsized":"2024-03-06T07:27:08.922438+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224843+0000","last_change":"2024-03-06T07:27:07.941367+0000","last_active":"2024-03-06T07:27:09.224843+0000","last_peered":"2024-03-06T07:27:09.224843+0000","last_clean":"2024-03-06T07:27:09.224843+0000","last_became_active":"2024-03-06T07:27:07.938238+0000","last_became_peered":"2024-03-06T07:27:07.938238+0000","last_unstale":"2024-03-06T07:27:09.224843+0000","last_undegraded":"2024-03-06T07:27:09.224843+0000","last_fullsized":"2024-03-06T07:27:09.224843+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933197+0000","last_change":"2024-03-06T07:27:06.931453+0000","last_active":"2024-03-06T07:27:08.933197+0000","last_peered":"2024-03-06T07:27:08.933197+0000","last_clean":"2024-03-06T07:27:08.933197+0000","last_became_active":"2024-03-06T07:27:06.931181+0000","last_became_peered":"2024-03-06T07:27:06.931181+0000","last_unstale":"2024-03-06T07:27:08.933197+0000","last_undegraded":"2024-03-06T07:27:08.933197+0000","last_fullsized":"2024-03-06T07:27:08.933197+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933121+0000","last_change":"2024-03-06T07:27:07.932186+0000","last_active":"2024-03-06T07:27:08.933121+0000","last_peered":"2024-03-06T07:27:08.933121+0000","last_clean":"2024-03-06T07:27:08.933121+0000","last_became_active":"2024-03-06T07:27:07.923467+0000","last_became_peered":"2024-03-06T07:27:07.923467+0000","last_unstale":"2024-03-06T07:27:08.933121+0000","last_undegraded":"2024-03-06T07:27:08.933121+0000","last_fullsized":"2024-03-06T07:27:08.933121+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:27:55.371987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"21'8","reported_seq":22,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223314+0000","last_change":"2024-03-06T07:27:06.925620+0000","last_active":"2024-03-06T07:27:09.223314+0000","last_peered":"2024-03-06T07:27:09.223314+0000","last_clean":"2024-03-06T07:27:09.223314+0000","last_became_active":"2024-03-06T07:27:06.925324+0000","last_became_peered":"2024-03-06T07:27:06.925324+0000","last_unstale":"2024-03-06T07:27:09.223314+0000","last_undegraded":"2024-03-06T07:27:09.223314+0000","last_fullsized":"2024-03-06T07:27:09.223314+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224242+0000","last_change":"2024-03-06T07:27:07.935148+0000","last_active":"2024-03-06T07:27:09.224242+0000","last_peered":"2024-03-06T07:27:09.224242+0000","last_clean":"2024-03-06T07:27:09.224242+0000","last_became_active":"2024-03-06T07:27:07.933239+0000","last_became_peered":"2024-03-06T07:27:07.933239+0000","last_unstale":"2024-03-06T07:27:09.224242+0000","last_undegraded":"2024-03-06T07:27:09.224242+0000","last_fullsized":"2024-03-06T07:27:09.224242+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223549+0000","last_change":"2024-03-06T07:27:06.919221+0000","last_active":"2024-03-06T07:27:09.223549+0000","last_peered":"2024-03-06T07:27:09.223549+0000","last_clean":"2024-03-06T07:27:09.223549+0000","last_became_active":"2024-03-06T07:27:06.918433+0000","last_became_peered":"2024-03-06T07:27:06.918433+0000","last_unstale":"2024-03-06T07:27:09.223549+0000","last_undegraded":"2024-03-06T07:27:09.223549+0000","last_fullsized":"2024-03-06T07:27:09.223549+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:21:09.808244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225325+0000","last_change":"2024-03-06T07:27:07.933938+0000","last_active":"2024-03-06T07:27:09.225325+0000","last_peered":"2024-03-06T07:27:09.225325+0000","last_clean":"2024-03-06T07:27:09.225325+0000","last_became_active":"2024-03-06T07:27:07.933656+0000","last_became_peered":"2024-03-06T07:27:07.933656+0000","last_unstale":"2024-03-06T07:27:09.225325+0000","last_undegraded":"2024-03-06T07:27:09.225325+0000","last_fullsized":"2024-03-06T07:27:09.225325+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:23:36.112418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224591+0000","last_change":"2024-03-06T07:27:06.913843+0000","last_active":"2024-03-06T07:27:09.224591+0000","last_peered":"2024-03-06T07:27:09.224591+0000","last_clean":"2024-03-06T07:27:09.224591+0000","last_became_active":"2024-03-06T07:27:06.912393+0000","last_became_peered":"2024-03-06T07:27:06.912393+0000","last_unstale":"2024-03-06T07:27:09.224591+0000","last_undegraded":"2024-03-06T07:27:09.224591+0000","last_fullsized":"2024-03-06T07:27:09.224591+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225673+0000","last_change":"2024-03-06T07:27:07.927885+0000","last_active":"2024-03-06T07:27:09.225673+0000","last_peered":"2024-03-06T07:27:09.225673+0000","last_clean":"2024-03-06T07:27:09.225673+0000","last_became_active":"2024-03-06T07:27:07.927215+0000","last_became_peered":"2024-03-06T07:27:07.927215+0000","last_unstale":"2024-03-06T07:27:09.225673+0000","last_undegraded":"2024-03-06T07:27:09.225673+0000","last_fullsized":"2024-03-06T07:27:09.225673+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:36:30.188524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918041+0000","last_change":"2024-03-06T07:27:06.921613+0000","last_active":"2024-03-06T07:27:08.918041+0000","last_peered":"2024-03-06T07:27:08.918041+0000","last_clean":"2024-03-06T07:27:08.918041+0000","last_became_active":"2024-03-06T07:27:06.921225+0000","last_became_peered":"2024-03-06T07:27:06.921225+0000","last_unstale":"2024-03-06T07:27:08.918041+0000","last_undegraded":"2024-03-06T07:27:08.918041+0000","last_fullsized":"2024-03-06T07:27:08.918041+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922980+0000","last_change":"2024-03-06T07:27:07.924340+0000","last_active":"2024-03-06T07:27:08.922980+0000","last_peered":"2024-03-06T07:27:08.922980+0000","last_clean":"2024-03-06T07:27:08.922980+0000","last_became_active":"2024-03-06T07:27:07.924090+0000","last_became_peered":"2024-03-06T07:27:07.924090+0000","last_unstale":"2024-03-06T07:27:08.922980+0000","last_undegraded":"2024-03-06T07:27:08.922980+0000","last_fullsized":"2024-03-06T07:27:08.922980+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:54.697587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223461+0000","last_change":"2024-03-06T07:27:06.933112+0000","last_active":"2024-03-06T07:27:09.223461+0000","last_peered":"2024-03-06T07:27:09.223461+0000","last_clean":"2024-03-06T07:27:09.223461+0000","last_became_active":"2024-03-06T07:27:06.932764+0000","last_became_peered":"2024-03-06T07:27:06.932764+0000","last_unstale":"2024-03-06T07:27:09.223461+0000","last_undegraded":"2024-03-06T07:27:09.223461+0000","last_fullsized":"2024-03-06T07:27:09.223461+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225383+0000","last_change":"2024-03-06T07:27:07.940694+0000","last_active":"2024-03-06T07:27:09.225383+0000","last_peered":"2024-03-06T07:27:09.225383+0000","last_clean":"2024-03-06T07:27:09.225383+0000","last_became_active":"2024-03-06T07:27:07.940352+0000","last_became_peered":"2024-03-06T07:27:07.940352+0000","last_unstale":"2024-03-06T07:27:09.225383+0000","last_undegraded":"2024-03-06T07:27:09.225383+0000","last_fullsized":"2024-03-06T07:27:09.225383+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:06.241503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":170,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224024+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:27:09.224024+0000","last_peered":"2024-03-06T07:27:09.224024+0000","last_clean":"2024-03-06T07:27:09.224024+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:27:09.224024+0000","last_undegraded":"2024-03-06T07:27:09.224024+0000","last_fullsized":"2024-03-06T07:27:09.224024+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922420+0000","last_change":"2024-03-06T07:27:06.931167+0000","last_active":"2024-03-06T07:27:08.922420+0000","last_peered":"2024-03-06T07:27:08.922420+0000","last_clean":"2024-03-06T07:27:08.922420+0000","last_became_active":"2024-03-06T07:27:06.930895+0000","last_became_peered":"2024-03-06T07:27:06.930895+0000","last_unstale":"2024-03-06T07:27:08.922420+0000","last_undegraded":"2024-03-06T07:27:08.922420+0000","last_fullsized":"2024-03-06T07:27:08.922420+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225706+0000","last_change":"2024-03-06T07:27:07.946419+0000","last_active":"2024-03-06T07:27:09.225706+0000","last_peered":"2024-03-06T07:27:09.225706+0000","last_clean":"2024-03-06T07:27:09.225706+0000","last_became_active":"2024-03-06T07:27:07.945795+0000","last_became_peered":"2024-03-06T07:27:07.945795+0000","last_unstale":"2024-03-06T07:27:09.225706+0000","last_undegraded":"2024-03-06T07:27:09.225706+0000","last_fullsized":"2024-03-06T07:27:09.225706+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224064+0000","last_change":"2024-03-06T07:27:06.936519+0000","last_active":"2024-03-06T07:27:09.224064+0000","last_peered":"2024-03-06T07:27:09.224064+0000","last_clean":"2024-03-06T07:27:09.224064+0000","last_became_active":"2024-03-06T07:27:06.934384+0000","last_became_peered":"2024-03-06T07:27:06.934384+0000","last_unstale":"2024-03-06T07:27:09.224064+0000","last_undegraded":"2024-03-06T07:27:09.224064+0000","last_fullsized":"2024-03-06T07:27:09.224064+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:29.611416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224613+0000","last_change":"2024-03-06T07:27:07.926848+0000","last_active":"2024-03-06T07:27:09.224613+0000","last_peered":"2024-03-06T07:27:09.224613+0000","last_clean":"2024-03-06T07:27:09.224613+0000","last_became_active":"2024-03-06T07:27:07.926255+0000","last_became_peered":"2024-03-06T07:27:07.926255+0000","last_unstale":"2024-03-06T07:27:09.224613+0000","last_undegraded":"2024-03-06T07:27:09.224613+0000","last_fullsized":"2024-03-06T07:27:09.224613+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:43:44.555041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223817+0000","last_change":"2024-03-06T07:27:06.912373+0000","last_active":"2024-03-06T07:27:09.223817+0000","last_peered":"2024-03-06T07:27:09.223817+0000","last_clean":"2024-03-06T07:27:09.223817+0000","last_became_active":"2024-03-06T07:27:06.911823+0000","last_became_peered":"2024-03-06T07:27:06.911823+0000","last_unstale":"2024-03-06T07:27:09.223817+0000","last_undegraded":"2024-03-06T07:27:09.223817+0000","last_fullsized":"2024-03-06T07:27:09.223817+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917936+0000","last_change":"2024-03-06T07:27:07.917473+0000","last_active":"2024-03-06T07:27:08.917936+0000","last_peered":"2024-03-06T07:27:08.917936+0000","last_clean":"2024-03-06T07:27:08.917936+0000","last_became_active":"2024-03-06T07:27:07.914951+0000","last_became_peered":"2024-03-06T07:27:07.914951+0000","last_unstale":"2024-03-06T07:27:08.917936+0000","last_undegraded":"2024-03-06T07:27:08.917936+0000","last_fullsized":"2024-03-06T07:27:08.917936+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922608+0000","last_change":"2024-03-06T07:27:06.928866+0000","last_active":"2024-03-06T07:27:08.922608+0000","last_peered":"2024-03-06T07:27:08.922608+0000","last_clean":"2024-03-06T07:27:08.922608+0000","last_became_active":"2024-03-06T07:27:06.928456+0000","last_became_peered":"2024-03-06T07:27:06.928456+0000","last_unstale":"2024-03-06T07:27:08.922608+0000","last_undegraded":"2024-03-06T07:27:08.922608+0000","last_fullsized":"2024-03-06T07:27:08.922608+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223508+0000","last_change":"2024-03-06T07:27:07.933229+0000","last_active":"2024-03-06T07:27:09.223508+0000","last_peered":"2024-03-06T07:27:09.223508+0000","last_clean":"2024-03-06T07:27:09.223508+0000","last_became_active":"2024-03-06T07:27:07.932737+0000","last_became_peered":"2024-03-06T07:27:07.932737+0000","last_unstale":"2024-03-06T07:27:09.223508+0000","last_undegraded":"2024-03-06T07:27:09.223508+0000","last_fullsized":"2024-03-06T07:27:09.223508+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:20.889857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918205+0000","last_change":"2024-03-06T07:27:06.918955+0000","last_active":"2024-03-06T07:27:08.918205+0000","last_peered":"2024-03-06T07:27:08.918205+0000","last_clean":"2024-03-06T07:27:08.918205+0000","last_became_active":"2024-03-06T07:27:06.918655+0000","last_became_peered":"2024-03-06T07:27:06.918655+0000","last_unstale":"2024-03-06T07:27:08.918205+0000","last_undegraded":"2024-03-06T07:27:08.918205+0000","last_fullsized":"2024-03-06T07:27:08.918205+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922887+0000","last_change":"2024-03-06T07:27:07.924920+0000","last_active":"2024-03-06T07:27:08.922887+0000","last_peered":"2024-03-06T07:27:08.922887+0000","last_clean":"2024-03-06T07:27:08.922887+0000","last_became_active":"2024-03-06T07:27:07.924624+0000","last_became_peered":"2024-03-06T07:27:07.924624+0000","last_unstale":"2024-03-06T07:27:08.922887+0000","last_undegraded":"2024-03-06T07:27:08.922887+0000","last_fullsized":"2024-03-06T07:27:08.922887+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:59:43.062703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225644+0000","last_change":"2024-03-06T07:27:06.936617+0000","last_active":"2024-03-06T07:27:09.225644+0000","last_peered":"2024-03-06T07:27:09.225644+0000","last_clean":"2024-03-06T07:27:09.225644+0000","last_became_active":"2024-03-06T07:27:06.936328+0000","last_became_peered":"2024-03-06T07:27:06.936328+0000","last_unstale":"2024-03-06T07:27:09.225644+0000","last_undegraded":"2024-03-06T07:27:09.225644+0000","last_fullsized":"2024-03-06T07:27:09.225644+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:42:10.164751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918287+0000","last_change":"2024-03-06T07:27:07.916912+0000","last_active":"2024-03-06T07:27:08.918287+0000","last_peered":"2024-03-06T07:27:08.918287+0000","last_clean":"2024-03-06T07:27:08.918287+0000","last_became_active":"2024-03-06T07:27:07.916557+0000","last_became_peered":"2024-03-06T07:27:07.916557+0000","last_unstale":"2024-03-06T07:27:08.918287+0000","last_undegraded":"2024-03-06T07:27:08.918287+0000","last_fullsized":"2024-03-06T07:27:08.918287+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"22'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.931761+0000","last_change":"2024-03-06T07:27:06.934214+0000","last_active":"2024-03-06T07:27:08.931761+0000","last_peered":"2024-03-06T07:27:08.931761+0000","last_clean":"2024-03-06T07:27:08.931761+0000","last_became_active":"2024-03-06T07:27:06.933984+0000","last_became_peered":"2024-03-06T07:27:06.933984+0000","last_unstale":"2024-03-06T07:27:08.931761+0000","last_undegraded":"2024-03-06T07:27:08.931761+0000","last_fullsized":"2024-03-06T07:27:08.931761+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227628+0000","last_change":"2024-03-06T07:27:07.934955+0000","last_active":"2024-03-06T07:27:09.227628+0000","last_peered":"2024-03-06T07:27:09.227628+0000","last_clean":"2024-03-06T07:27:09.227628+0000","last_became_active":"2024-03-06T07:27:07.934602+0000","last_became_peered":"2024-03-06T07:27:07.934602+0000","last_unstale":"2024-03-06T07:27:09.227628+0000","last_undegraded":"2024-03-06T07:27:09.227628+0000","last_fullsized":"2024-03-06T07:27:09.227628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933574+0000","last_change":"2024-03-06T07:27:06.943288+0000","last_active":"2024-03-06T07:27:08.933574+0000","last_peered":"2024-03-06T07:27:08.933574+0000","last_clean":"2024-03-06T07:27:08.933574+0000","last_became_active":"2024-03-06T07:27:06.942813+0000","last_became_peered":"2024-03-06T07:27:06.942813+0000","last_unstale":"2024-03-06T07:27:08.933574+0000","last_undegraded":"2024-03-06T07:27:08.933574+0000","last_fullsized":"2024-03-06T07:27:08.933574+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226729+0000","last_change":"2024-03-06T07:27:07.920024+0000","last_active":"2024-03-06T07:27:09.226729+0000","last_peered":"2024-03-06T07:27:09.226729+0000","last_clean":"2024-03-06T07:27:09.226729+0000","last_became_active":"2024-03-06T07:27:07.919750+0000","last_became_peered":"2024-03-06T07:27:07.919750+0000","last_unstale":"2024-03-06T07:27:09.226729+0000","last_undegraded":"2024-03-06T07:27:09.226729+0000","last_fullsized":"2024-03-06T07:27:09.226729+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223933+0000","last_change":"2024-03-06T07:27:06.936885+0000","last_active":"2024-03-06T07:27:09.223933+0000","last_peered":"2024-03-06T07:27:09.223933+0000","last_clean":"2024-03-06T07:27:09.223933+0000","last_became_active":"2024-03-06T07:27:06.934829+0000","last_became_peered":"2024-03-06T07:27:06.934829+0000","last_unstale":"2024-03-06T07:27:09.223933+0000","last_undegraded":"2024-03-06T07:27:09.223933+0000","last_fullsized":"2024-03-06T07:27:09.223933+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223030+0000","last_change":"2024-03-06T07:27:07.916276+0000","last_active":"2024-03-06T07:27:09.223030+0000","last_peered":"2024-03-06T07:27:09.223030+0000","last_clean":"2024-03-06T07:27:09.223030+0000","last_became_active":"2024-03-06T07:27:07.915974+0000","last_became_peered":"2024-03-06T07:27:07.915974+0000","last_unstale":"2024-03-06T07:27:09.223030+0000","last_undegraded":"2024-03-06T07:27:09.223030+0000","last_fullsized":"2024-03-06T07:27:09.223030+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.935659+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.935373+0000","last_became_peered":"2024-03-06T07:27:06.935373+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223769+0000","last_change":"2024-03-06T07:27:07.910767+0000","last_active":"2024-03-06T07:27:09.223769+0000","last_peered":"2024-03-06T07:27:09.223769+0000","last_clean":"2024-03-06T07:27:09.223769+0000","last_became_active":"2024-03-06T07:27:07.910268+0000","last_became_peered":"2024-03-06T07:27:07.910268+0000","last_unstale":"2024-03-06T07:27:09.223769+0000","last_undegraded":"2024-03-06T07:27:09.223769+0000","last_fullsized":"2024-03-06T07:27:09.223769+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917633+0000","last_change":"2024-03-06T07:27:06.923056+0000","last_active":"2024-03-06T07:27:08.917633+0000","last_peered":"2024-03-06T07:27:08.917633+0000","last_clean":"2024-03-06T07:27:08.917633+0000","last_became_active":"2024-03-06T07:27:06.922710+0000","last_became_peered":"2024-03-06T07:27:06.922710+0000","last_unstale":"2024-03-06T07:27:08.917633+0000","last_undegraded":"2024-03-06T07:27:08.917633+0000","last_fullsized":"2024-03-06T07:27:08.917633+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226527+0000","last_change":"2024-03-06T07:27:07.922445+0000","last_active":"2024-03-06T07:27:09.226527+0000","last_peered":"2024-03-06T07:27:09.226527+0000","last_clean":"2024-03-06T07:27:09.226527+0000","last_became_active":"2024-03-06T07:27:07.922147+0000","last_became_peered":"2024-03-06T07:27:07.922147+0000","last_unstale":"2024-03-06T07:27:09.226527+0000","last_undegraded":"2024-03-06T07:27:09.226527+0000","last_fullsized":"2024-03-06T07:27:09.226527+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224979+0000","last_change":"2024-03-06T07:27:06.940633+0000","last_active":"2024-03-06T07:27:09.224979+0000","last_peered":"2024-03-06T07:27:09.224979+0000","last_clean":"2024-03-06T07:27:09.224979+0000","last_became_active":"2024-03-06T07:27:06.937434+0000","last_became_peered":"2024-03-06T07:27:06.937434+0000","last_unstale":"2024-03-06T07:27:09.224979+0000","last_undegraded":"2024-03-06T07:27:09.224979+0000","last_fullsized":"2024-03-06T07:27:09.224979+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932876+0000","last_change":"2024-03-06T07:27:07.942081+0000","last_active":"2024-03-06T07:27:08.932876+0000","last_peered":"2024-03-06T07:27:08.932876+0000","last_clean":"2024-03-06T07:27:08.932876+0000","last_became_active":"2024-03-06T07:27:07.941647+0000","last_became_peered":"2024-03-06T07:27:07.941647+0000","last_unstale":"2024-03-06T07:27:08.932876+0000","last_undegraded":"2024-03-06T07:27:08.932876+0000","last_fullsized":"2024-03-06T07:27:08.932876+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:47:21.103970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922929+0000","last_change":"2024-03-06T07:27:07.944160+0000","last_active":"2024-03-06T07:27:08.922929+0000","last_peered":"2024-03-06T07:27:08.922929+0000","last_clean":"2024-03-06T07:27:08.922929+0000","last_became_active":"2024-03-06T07:27:07.943238+0000","last_became_peered":"2024-03-06T07:27:07.943238+0000","last_unstale":"2024-03-06T07:27:08.922929+0000","last_undegraded":"2024-03-06T07:27:08.922929+0000","last_fullsized":"2024-03-06T07:27:08.922929+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:51.931654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224835+0000","last_change":"2024-03-06T07:27:06.940899+0000","last_active":"2024-03-06T07:27:09.224835+0000","last_peered":"2024-03-06T07:27:09.224835+0000","last_clean":"2024-03-06T07:27:09.224835+0000","last_became_active":"2024-03-06T07:27:06.940627+0000","last_became_peered":"2024-03-06T07:27:06.940627+0000","last_unstale":"2024-03-06T07:27:09.224835+0000","last_undegraded":"2024-03-06T07:27:09.224835+0000","last_fullsized":"2024-03-06T07:27:09.224835+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.933712+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.933437+0000","last_became_peered":"2024-03-06T07:27:07.933437+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:18.550912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225416+0000","last_change":"2024-03-06T07:27:06.938164+0000","last_active":"2024-03-06T07:27:09.225416+0000","last_peered":"2024-03-06T07:27:09.225416+0000","last_clean":"2024-03-06T07:27:09.225416+0000","last_became_active":"2024-03-06T07:27:06.937669+0000","last_became_peered":"2024-03-06T07:27:06.937669+0000","last_unstale":"2024-03-06T07:27:09.225416+0000","last_undegraded":"2024-03-06T07:27:09.225416+0000","last_fullsized":"2024-03-06T07:27:09.225416+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:19.647400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226963+0000","last_change":"2024-03-06T07:27:07.948380+0000","last_active":"2024-03-06T07:27:09.226963+0000","last_peered":"2024-03-06T07:27:09.226963+0000","last_clean":"2024-03-06T07:27:09.226963+0000","last_became_active":"2024-03-06T07:27:07.948110+0000","last_became_peered":"2024-03-06T07:27:07.948110+0000","last_unstale":"2024-03-06T07:27:09.226963+0000","last_undegraded":"2024-03-06T07:27:09.226963+0000","last_fullsized":"2024-03-06T07:27:09.226963+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227256+0000","last_change":"2024-03-06T07:27:06.911519+0000","last_active":"2024-03-06T07:27:09.227256+0000","last_peered":"2024-03-06T07:27:09.227256+0000","last_clean":"2024-03-06T07:27:09.227256+0000","last_became_active":"2024-03-06T07:27:06.911051+0000","last_became_peered":"2024-03-06T07:27:06.911051+0000","last_unstale":"2024-03-06T07:27:09.227256+0000","last_undegraded":"2024-03-06T07:27:09.227256+0000","last_fullsized":"2024-03-06T07:27:09.227256+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227138+0000","last_change":"2024-03-06T07:27:07.937255+0000","last_active":"2024-03-06T07:27:09.227138+0000","last_peered":"2024-03-06T07:27:09.227138+0000","last_clean":"2024-03-06T07:27:09.227138+0000","last_became_active":"2024-03-06T07:27:07.936713+0000","last_became_peered":"2024-03-06T07:27:07.936713+0000","last_unstale":"2024-03-06T07:27:09.227138+0000","last_undegraded":"2024-03-06T07:27:09.227138+0000","last_fullsized":"2024-03-06T07:27:09.227138+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"21'4","reported_seq":18,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227467+0000","last_change":"2024-03-06T07:27:06.935678+0000","last_active":"2024-03-06T07:27:09.227467+0000","last_peered":"2024-03-06T07:27:09.227467+0000","last_clean":"2024-03-06T07:27:09.227467+0000","last_became_active":"2024-03-06T07:27:06.935349+0000","last_became_peered":"2024-03-06T07:27:06.935349+0000","last_unstale":"2024-03-06T07:27:09.227467+0000","last_undegraded":"2024-03-06T07:27:09.227467+0000","last_fullsized":"2024-03-06T07:27:09.227467+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542149,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27836,"kb_used_data":804,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344004,"statfs":{"total":96636764160,"available":96608260096,"internally_reserved":0,"allocated":823296,"data_stored":669784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542148,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542149,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27244,"kb_used_data":224,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344596,"statfs":{"total":96636764160,"available":96608866304,"internally_reserved":0,"allocated":229376,"data_stored":78924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542148,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":78960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542148,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542149,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27248,"kb_used_data":220,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344592,"statfs":{"total":96636764160,"available":96608862208,"internally_reserved":0,"allocated":225280,"data_stored":78834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542150,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27256,"kb_used_data":228,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344584,"statfs":{"total":96636764160,"available":96608854016,"internally_reserved":0,"allocated":233472,"data_stored":78994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542147,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27844,"kb_used_data":812,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343996,"statfs":{"total":96636764160,"available":96608251904,"internally_reserved":0,"allocated":831488,"data_stored":669840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:27:12.829 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:27:12.829 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-06T07:27:12.829 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-03-06T07:27:12.829 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-03-06T07:27:12.830 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-03-06T07:27:13.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.076+0000 7f5fe0600640 1 Processor -- start 2024-03-06T07:27:13.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.077+0000 7f5fe0600640 1 -- start start 2024-03-06T07:27:13.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.078+0000 7f5fe0600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5fd815fe60 0x7f5fd8071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.078+0000 7f5fe0600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5fd806cef0 con 0x7f5fd815f9d0 2024-03-06T07:27:13.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.078+0000 7f5fe0600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5fd806d1d0 con 0x7f5fd8144c00 2024-03-06T07:27:13.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.078+0000 7f5fe0600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5fd806d4b0 con 0x7f5fd815fe60 2024-03-06T07:27:13.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.078+0000 7f5fde600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5fd815fe60 0x7f5fd8071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.079+0000 7f5fde600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5fd815fe60 0x7f5fd8071220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53906/0 (socket says 172.21.15.86:53906) 2024-03-06T07:27:13.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.079+0000 7f5fde600640 1 -- 172.21.15.86:0/2219858748 learned_addr learned my addr 172.21.15.86:0/2219858748 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:13.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.079+0000 7f5fde600640 1 -- 172.21.15.86:0/2219858748 >> v1:172.21.15.194:6789/0 conn(0x7f5fd8144c00 legacy=0x7f5fd806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:13.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.079+0000 7f5fde600640 1 -- 172.21.15.86:0/2219858748 >> v1:172.21.15.86:6789/0 conn(0x7f5fd815f9d0 legacy=0x7f5fd8070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.079+0000 7f5fde600640 1 -- 172.21.15.86:0/2219858748 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5fd806d790 con 0x7f5fd815fe60 2024-03-06T07:27:13.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.080+0000 7f5fde600640 1 --2- 172.21.15.86:0/2219858748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd815fe60 0x7f5fd8071220 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f5fd0009e50 tx=0x7f5fd002bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c443ef1a7f0106f8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:13.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.080+0000 7f5fd7e00640 1 -- 172.21.15.86:0/2219858748 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fd002cb20 con 0x7f5fd815fe60 2024-03-06T07:27:13.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.080+0000 7f5fd7e00640 1 -- 172.21.15.86:0/2219858748 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5fd002ccc0 con 0x7f5fd815fe60 2024-03-06T07:27:13.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.080+0000 7f5fd7e00640 1 -- 172.21.15.86:0/2219858748 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fd0034610 con 0x7f5fd815fe60 2024-03-06T07:27:13.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.081+0000 7f5fe0600640 1 -- 172.21.15.86:0/2219858748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd815fe60 msgr2=0x7f5fd8071220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.081+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2219858748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd815fe60 0x7f5fd8071220 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f5fd0009e50 tx=0x7f5fd002bf80 comp rx=0 tx=0).stop 2024-03-06T07:27:13.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.081+0000 7f5fe0600640 1 -- 172.21.15.86:0/2219858748 shutdown_connections 2024-03-06T07:27:13.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.081+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2219858748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd815fe60 0x7f5fd8071220 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.081+0000 7f5fe0600640 1 -- 172.21.15.86:0/2219858748 >> 172.21.15.86:0/2219858748 conn(0x7f5fd80b1730 msgr2=0x7f5fd80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:13.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.081+0000 7f5fe0600640 1 -- 172.21.15.86:0/2219858748 shutdown_connections 2024-03-06T07:27:13.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.081+0000 7f5fe0600640 1 -- 172.21.15.86:0/2219858748 wait complete. 2024-03-06T07:27:13.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.083+0000 7f5fe0600640 1 Processor -- start 2024-03-06T07:27:13.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.083+0000 7f5fe0600640 1 -- start start 2024-03-06T07:27:13.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fe0600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd8144c00 0x7f5fd8138f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fe0600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5fd815f9d0 0x7f5fd813ba50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fe0600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5fd815fe60 0x7f5fd815a940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fe0600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5fd806cb50 con 0x7f5fd8144c00 2024-03-06T07:27:13.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fe0600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5fd8160730 con 0x7f5fd815fe60 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fe0600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5fd806ccd0 con 0x7f5fd815f9d0 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd8144c00 0x7f5fd8138f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd8144c00 0x7f5fd8138f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53918/0 (socket says 172.21.15.86:53918) 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 -- 172.21.15.86:0/2525275929 learned_addr learned my addr 172.21.15.86:0/2525275929 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 -- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5fd815f9d0 msgr2=0x7f5fd813ba50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fdd200640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5fd815f9d0 0x7f5fd813ba50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fde600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5fd815fe60 0x7f5fd815a940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5fd815f9d0 0x7f5fd813ba50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 -- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5fd815fe60 msgr2=0x7f5fd815a940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5fd815fe60 0x7f5fd815a940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.084+0000 7f5fddc00640 1 -- 172.21.15.86:0/2525275929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5fd8139a20 con 0x7f5fd8144c00 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.085+0000 7f5fddc00640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd8144c00 0x7f5fd8138f90 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f5fcc00b3f0 tx=0x7f5fcc00b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.085+0000 7f5fd5600640 1 -- 172.21.15.86:0/2525275929 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fcc015020 con 0x7f5fd8144c00 2024-03-06T07:27:13.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.085+0000 7f5fd5600640 1 -- 172.21.15.86:0/2525275929 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5fcc011830 con 0x7f5fd8144c00 2024-03-06T07:27:13.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.085+0000 7f5fd5600640 1 -- 172.21.15.86:0/2525275929 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fcc011c20 con 0x7f5fd8144c00 2024-03-06T07:27:13.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.085+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5fd81e57d0 con 0x7f5fd8144c00 2024-03-06T07:27:13.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.085+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5fd81e5d90 con 0x7f5fd8144c00 2024-03-06T07:27:13.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.086+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5fd804f330 con 0x7f5fd8144c00 2024-03-06T07:27:13.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.087+0000 7f5fd5600640 1 -- 172.21.15.86:0/2525275929 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5fcc018020 con 0x7f5fd8144c00 2024-03-06T07:27:13.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.088+0000 7f5fd5600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5fb806b4d0 0x7f5fb806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.088+0000 7f5fd5600640 1 -- 172.21.15.86:0/2525275929 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5fcc089e40 con 0x7f5fd8144c00 2024-03-06T07:27:13.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.088+0000 7f5fdd200640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5fb806b4d0 0x7f5fb806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.089+0000 7f5fdd200640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5fb806b4d0 0x7f5fb806d990 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f5fd813ce90 tx=0x7f5fc4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:13.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.093+0000 7f5fd5600640 1 -- 172.21.15.86:0/2525275929 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5fcc01f030 con 0x7f5fd8144c00 2024-03-06T07:27:13.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.337+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) v1 -- 0x7f5fd80b1e50 con 0x7f5fd8144c00 2024-03-06T07:27:13.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.342+0000 7f5fd5600640 1 -- 172.21.15.86:0/2525275929 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) v1 ==== 74+0+171784 (secure 0 0 0) 0x7f5fd8139a20 con 0x7f5fd8144c00 2024-03-06T07:27:13.347 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5fb806b4d0 msgr2=0x7f5fb806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5fb806b4d0 0x7f5fb806d990 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f5fd813ce90 tx=0x7f5fc4009040 comp rx=0 tx=0).stop 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd8144c00 msgr2=0x7f5fd8138f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd8144c00 0x7f5fd8138f90 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f5fcc00b3f0 tx=0x7f5fcc00b8b0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 shutdown_connections 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5fb806b4d0 0x7f5fb806d990 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5fd815fe60 0x7f5fd815a940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5fd815f9d0 0x7f5fd813ba50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 --2- 172.21.15.86:0/2525275929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5fd8144c00 0x7f5fd8138f90 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 >> 172.21.15.86:0/2525275929 conn(0x7f5fd80b1730 msgr2=0x7f5fd80b0770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 shutdown_connections 2024-03-06T07:27:13.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.347+0000 7f5fe0600640 1 -- 172.21.15.86:0/2525275929 wait complete. 2024-03-06T07:27:13.368 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6832","nonce":2881780473},{"type":"v1","addr":"172.21.15.86:6833","nonce":2881780473}]},"active_addr":"172.21.15.86:6833/2881780473","active_change":"2024-03-06T07:26:52.831562+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":4111,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":1062309594}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":558069895}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":1268493935}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":3404469691}]}]} 2024-03-06T07:27:13.371 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-03-06T07:27:13.371 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-03-06T07:27:13.371 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:13.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.576+0000 7fa5a3800640 1 Processor -- start 2024-03-06T07:27:13.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.576+0000 7fa5a3800640 1 -- start start 2024-03-06T07:27:13.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.577+0000 7fa5a3800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa59c0af1c0 0x7fa59c15eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.577+0000 7fa5a3800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 0x7fa59c06cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.577+0000 7fa5a3800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa59c06d1a0 0x7fa59c06e4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.577+0000 7fa5a3800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa59c071d30 con 0x7fa59c0af1c0 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.578+0000 7fa5a3800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa59c071490 con 0x7fa59c06d1a0 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.578+0000 7fa5a3800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa59c071770 con 0x7fa59c070d20 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.578+0000 7fa5a0e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa59c0af1c0 0x7fa59c15eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.578+0000 7fa5a0e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa59c0af1c0 0x7fa59c15eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53922/0 (socket says 172.21.15.86:53922) 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.578+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2276188690 learned_addr learned my addr 172.21.15.86:0/2276188690 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.578+0000 7fa59be00640 1 --2- 172.21.15.86:0/2276188690 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 0x7fa59c06cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.578+0000 7fa5a1800640 1 --2- 172.21.15.86:0/2276188690 >> v2:172.21.15.194:3300/0 conn(0x7fa59c06d1a0 0x7fa59c06e4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.579+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2276188690 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 msgr2=0x7fa59c06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:13.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.579+0000 7fa5a0e00640 1 --2- 172.21.15.86:0/2276188690 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 0x7fa59c06cc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.579+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2276188690 >> v2:172.21.15.194:3300/0 conn(0x7fa59c06d1a0 msgr2=0x7fa59c06e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:13.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.579+0000 7fa5a0e00640 1 --2- 172.21.15.86:0/2276188690 >> v2:172.21.15.194:3300/0 conn(0x7fa59c06d1a0 0x7fa59c06e4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.579+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2276188690 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa59c071a50 con 0x7fa59c0af1c0 2024-03-06T07:27:13.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.579+0000 7fa5a1800640 1 --2- 172.21.15.86:0/2276188690 >> v2:172.21.15.194:3300/0 conn(0x7fa59c06d1a0 0x7fa59c06e4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:13.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.580+0000 7fa5a0e00640 1 --2- 172.21.15.86:0/2276188690 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c0af1c0 0x7fa59c15eff0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fa59c0b2730 tx=0x7fa58402b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d8f00a5b0ab2fe51 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:13.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.581+0000 7fa59b400640 1 -- 172.21.15.86:0/2276188690 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa584035020 con 0x7fa59c0af1c0 2024-03-06T07:27:13.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.581+0000 7fa59b400640 1 -- 172.21.15.86:0/2276188690 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa58402d040 con 0x7fa59c0af1c0 2024-03-06T07:27:13.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.581+0000 7fa59b400640 1 -- 172.21.15.86:0/2276188690 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa584031a80 con 0x7fa59c0af1c0 2024-03-06T07:27:13.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.581+0000 7fa5a3800640 1 -- 172.21.15.86:0/2276188690 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c0af1c0 msgr2=0x7fa59c15eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.581+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2276188690 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c0af1c0 0x7fa59c15eff0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fa59c0b2730 tx=0x7fa58402b900 comp rx=0 tx=0).stop 2024-03-06T07:27:13.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.582+0000 7fa5a3800640 1 -- 172.21.15.86:0/2276188690 shutdown_connections 2024-03-06T07:27:13.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.582+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2276188690 >> v2:172.21.15.194:3300/0 conn(0x7fa59c06d1a0 0x7fa59c06e4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.582+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2276188690 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 0x7fa59c06cc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.582+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2276188690 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c0af1c0 0x7fa59c15eff0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.582+0000 7fa5a3800640 1 -- 172.21.15.86:0/2276188690 >> 172.21.15.86:0/2276188690 conn(0x7fa59c0b1730 msgr2=0x7fa59c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:13.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.582+0000 7fa5a3800640 1 -- 172.21.15.86:0/2276188690 shutdown_connections 2024-03-06T07:27:13.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.582+0000 7fa5a3800640 1 -- 172.21.15.86:0/2276188690 wait complete. 2024-03-06T07:27:13.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.584+0000 7fa5a3800640 1 Processor -- start 2024-03-06T07:27:13.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.584+0000 7fa5a3800640 1 -- start start 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a3800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c06d1a0 0x7fa59c14d2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a3800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 0x7fa59c1f7210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a3800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa59c0af1c0 0x7fa59c1f7750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a3800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa59c06ea10 con 0x7fa59c06d1a0 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a3800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa59c0711a0 con 0x7fa59c0af1c0 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a3800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa59c06eb90 con 0x7fa59c070d20 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a0e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c06d1a0 0x7fa59c14d2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a0e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c06d1a0 0x7fa59c14d2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53928/0 (socket says 172.21.15.86:53928) 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.585+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2089351189 learned_addr learned my addr 172.21.15.86:0/2089351189 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2089351189 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 msgr2=0x7fa59c1f7210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a0e00640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 0x7fa59c1f7210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2089351189 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa59c0af1c0 msgr2=0x7fa59c1f7750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a0e00640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa59c0af1c0 0x7fa59c1f7750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a0e00640 1 -- 172.21.15.86:0/2089351189 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa59c0cda30 con 0x7fa59c06d1a0 2024-03-06T07:27:13.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a0e00640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c06d1a0 0x7fa59c14d2d0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fa58402b940 tx=0x7fa5840046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:13.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa598c00640 1 -- 172.21.15.86:0/2089351189 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa584040070 con 0x7fa59c06d1a0 2024-03-06T07:27:13.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa598c00640 1 -- 172.21.15.86:0/2089351189 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa5840399f0 con 0x7fa59c06d1a0 2024-03-06T07:27:13.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa598c00640 1 -- 172.21.15.86:0/2089351189 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa584035040 con 0x7fa59c06d1a0 2024-03-06T07:27:13.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa59c0cdd10 con 0x7fa59c06d1a0 2024-03-06T07:27:13.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.586+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa59c1e67e0 con 0x7fa59c06d1a0 2024-03-06T07:27:13.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.587+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa59c04f330 con 0x7fa59c06d1a0 2024-03-06T07:27:13.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.588+0000 7fa598c00640 1 -- 172.21.15.86:0/2089351189 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa58404b060 con 0x7fa59c06d1a0 2024-03-06T07:27:13.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.589+0000 7fa598c00640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57806b4d0 0x7fa57806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:13.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.589+0000 7fa598c00640 1 -- 172.21.15.86:0/2089351189 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa5840abcb0 con 0x7fa59c06d1a0 2024-03-06T07:27:13.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.589+0000 7fa59be00640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57806b4d0 0x7fa57806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:13.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.590+0000 7fa59be00640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57806b4d0 0x7fa57806d990 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fa58c006a90 tx=0x7fa58c006a20 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:13.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.593+0000 7fa598c00640 1 -- 172.21.15.86:0/2089351189 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa58407abc0 con 0x7fa59c06d1a0 2024-03-06T07:27:13.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.791+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa59c06c780 con 0x7fa59c06d1a0 2024-03-06T07:27:13.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.794+0000 7fa598c00640 1 -- 172.21.15.86:0/2089351189 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7fa58407a4c0 con 0x7fa59c06d1a0 2024-03-06T07:27:13.797 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:13.797 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57806b4d0 msgr2=0x7fa57806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57806b4d0 0x7fa57806d990 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fa58c006a90 tx=0x7fa58c006a20 comp rx=0 tx=0).stop 2024-03-06T07:27:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c06d1a0 msgr2=0x7fa59c14d2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c06d1a0 0x7fa59c14d2d0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fa58402b940 tx=0x7fa5840046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 shutdown_connections 2024-03-06T07:27:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57806b4d0 0x7fa57806d990 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa59c0af1c0 0x7fa59c1f7750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59c070d20 0x7fa59c1f7210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 --2- 172.21.15.86:0/2089351189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59c06d1a0 0x7fa59c14d2d0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 >> 172.21.15.86:0/2089351189 conn(0x7fa59c0b1730 msgr2=0x7fa59c0b0f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 shutdown_connections 2024-03-06T07:27:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:13.797+0000 7fa5a3800640 1 -- 172.21.15.86:0/2089351189 wait complete. 2024-03-06T07:27:13.818 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-03-06T07:27:13.818 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:14.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.018+0000 7f4965200640 1 Processor -- start 2024-03-06T07:27:14.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.019+0000 7f4965200640 1 -- start start 2024-03-06T07:27:14.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.020+0000 7f4965200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f496015f9d0 0x7f496015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.020+0000 7f4965200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4960070b10 0x7f4960070f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.020+0000 7f4965200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f496006cef0 con 0x7f4960144c00 2024-03-06T07:27:14.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.020+0000 7f4965200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f496006d1d0 con 0x7f4960070b10 2024-03-06T07:27:14.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.020+0000 7f4965200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f496006d4b0 con 0x7f496015f9d0 2024-03-06T07:27:14.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4960070b10 0x7f4960070f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4960070b10 0x7f4960070f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53932/0 (socket says 172.21.15.86:53932) 2024-03-06T07:27:14.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495e800640 1 -- 172.21.15.86:0/2579821166 learned_addr learned my addr 172.21.15.86:0/2579821166 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495d400640 1 --2- 172.21.15.86:0/2579821166 >> v2:172.21.15.194:3300/0 conn(0x7f496015f9d0 0x7f496015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495e800640 1 -- 172.21.15.86:0/2579821166 >> v2:172.21.15.194:3300/0 conn(0x7f496015f9d0 msgr2=0x7f496015fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495e800640 1 --2- 172.21.15.86:0/2579821166 >> v2:172.21.15.194:3300/0 conn(0x7f496015f9d0 0x7f496015fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495e800640 1 -- 172.21.15.86:0/2579821166 >> v1:172.21.15.194:6789/0 conn(0x7f4960144c00 legacy=0x7f496006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.021+0000 7f495e800640 1 -- 172.21.15.86:0/2579821166 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f496006d790 con 0x7f4960070b10 2024-03-06T07:27:14.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.022+0000 7f495e800640 1 --2- 172.21.15.86:0/2579821166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f4960070f20 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f49600b26f0 tx=0x7f495802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5b078d012d57eae8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.023+0000 7f495ca00640 1 -- 172.21.15.86:0/2579821166 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f495802ca80 con 0x7f4960070b10 2024-03-06T07:27:14.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.023+0000 7f495ca00640 1 -- 172.21.15.86:0/2579821166 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f495802cc20 con 0x7f4960070b10 2024-03-06T07:27:14.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.023+0000 7f495ca00640 1 -- 172.21.15.86:0/2579821166 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f49580343f0 con 0x7f4960070b10 2024-03-06T07:27:14.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 -- 172.21.15.86:0/2579821166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 msgr2=0x7f4960070f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 --2- 172.21.15.86:0/2579821166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f4960070f20 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f49600b26f0 tx=0x7f495802bee0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 -- 172.21.15.86:0/2579821166 shutdown_connections 2024-03-06T07:27:14.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 --2- 172.21.15.86:0/2579821166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f4960070f20 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 --2- 172.21.15.86:0/2579821166 >> v2:172.21.15.194:3300/0 conn(0x7f496015f9d0 0x7f496015fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 -- 172.21.15.86:0/2579821166 >> 172.21.15.86:0/2579821166 conn(0x7f49600b1730 msgr2=0x7f49600b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 -- 172.21.15.86:0/2579821166 shutdown_connections 2024-03-06T07:27:14.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.024+0000 7f4965200640 1 -- 172.21.15.86:0/2579821166 wait complete. 2024-03-06T07:27:14.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.025+0000 7f4965200640 1 Processor -- start 2024-03-06T07:27:14.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.026+0000 7f4965200640 1 -- start start 2024-03-06T07:27:14.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.026+0000 7f4965200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f496013d060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.026+0000 7f4965200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4960144c00 0x7f496015e780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f4965200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f496015f9d0 0x7f496013fbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f4965200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f496006ccd0 con 0x7f4960070b10 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f4965200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f496006cb50 con 0x7f4960144c00 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f4965200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f496006df90 con 0x7f496015f9d0 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f496013d060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f496013d060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53946/0 (socket says 172.21.15.86:53946) 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 -- 172.21.15.86:0/159301057 learned_addr learned my addr 172.21.15.86:0/159301057 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 -- 172.21.15.86:0/159301057 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f496015f9d0 msgr2=0x7f496013fbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f496015f9d0 0x7f496013fbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 -- 172.21.15.86:0/159301057 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4960144c00 msgr2=0x7f496015e780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4960144c00 0x7f496015e780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 -- 172.21.15.86:0/159301057 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f496013ebc0 con 0x7f4960070b10 2024-03-06T07:27:14.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.027+0000 7f495de00640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f496013d060 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f495000c410 tx=0x7f495000c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.028+0000 7f4956000640 1 -- 172.21.15.86:0/159301057 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4950017070 con 0x7f4960070b10 2024-03-06T07:27:14.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.028+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f496013ee60 con 0x7f4960070b10 2024-03-06T07:27:14.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.028+0000 7f4956000640 1 -- 172.21.15.86:0/159301057 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f49500035f0 con 0x7f4960070b10 2024-03-06T07:27:14.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.028+0000 7f4956000640 1 -- 172.21.15.86:0/159301057 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f49500039e0 con 0x7f4960070b10 2024-03-06T07:27:14.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.028+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f496013f100 con 0x7f4960070b10 2024-03-06T07:27:14.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.029+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f496004f330 con 0x7f4960070b10 2024-03-06T07:27:14.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.030+0000 7f4956000640 1 -- 172.21.15.86:0/159301057 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4950003b80 con 0x7f4960070b10 2024-03-06T07:27:14.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.030+0000 7f4956000640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f494006b4d0 0x7f494006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.031+0000 7f4956000640 1 -- 172.21.15.86:0/159301057 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f49500894f0 con 0x7f4960070b10 2024-03-06T07:27:14.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.031+0000 7f495d400640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f494006b4d0 0x7f494006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.031+0000 7f495d400640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f494006b4d0 0x7f494006d990 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f494c006a90 tx=0x7f494c006a20 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.035+0000 7f4956000640 1 -- 172.21.15.86:0/159301057 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f49500582f0 con 0x7f4960070b10 2024-03-06T07:27:14.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.225+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f496013f800 con 0x7f4960070b10 2024-03-06T07:27:14.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.227+0000 7f4956000640 1 -- 172.21.15.86:0/159301057 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f4950057bf0 con 0x7f4960070b10 2024-03-06T07:27:14.230 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:14.230 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:14.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f494006b4d0 msgr2=0x7f494006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f494006b4d0 0x7f494006d990 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f494c006a90 tx=0x7f494c006a20 comp rx=0 tx=0).stop 2024-03-06T07:27:14.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 msgr2=0x7f496013d060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f496013d060 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f495000c410 tx=0x7f495000c8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 shutdown_connections 2024-03-06T07:27:14.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f494006b4d0 0x7f494006d990 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f496015f9d0 0x7f496013fbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4960144c00 0x7f496015e780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 --2- 172.21.15.86:0/159301057 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4960070b10 0x7f496013d060 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.229+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 >> 172.21.15.86:0/159301057 conn(0x7f49600b1730 msgr2=0x7f4960070ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.230+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 shutdown_connections 2024-03-06T07:27:14.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.230+0000 7f4965200640 1 -- 172.21.15.86:0/159301057 wait complete. 2024-03-06T07:27:14.250 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-03-06T07:27:14.250 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-03-06T07:27:14.319 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-03-06T07:27:14.319 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-03-06T07:27:14.320 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.4 flush_pg_stats 2024-03-06T07:27:14.320 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.5 flush_pg_stats 2024-03-06T07:27:14.320 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.6 flush_pg_stats 2024-03-06T07:27:14.321 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.7 flush_pg_stats 2024-03-06T07:27:14.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.606+0000 7f8e07c00640 1 Processor -- start 2024-03-06T07:27:14.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.607+0000 7f8e07c00640 1 -- start start 2024-03-06T07:27:14.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.609+0000 7f8e07c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8e000cd5b0 0x7f8e00145b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.610+0000 7f8e07c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8e000cd990 0x7f8e0015c560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.610+0000 7f8e07c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e00166ec0 con 0x7f8e0015caa0 2024-03-06T07:27:14.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.610+0000 7f8e07c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e0006cbe0 con 0x7f8e000cd990 2024-03-06T07:27:14.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.610+0000 7f8e07c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e0006cec0 con 0x7f8e000cd5b0 2024-03-06T07:27:14.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.610+0000 7f8e05200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8e000cd5b0 0x7f8e00145b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.611+0000 7f8e05200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8e000cd5b0 0x7f8e00145b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51756/0 (socket says 172.21.15.86:51756) 2024-03-06T07:27:14.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.611+0000 7f8e05200640 1 -- 172.21.15.86:0/1084804542 learned_addr learned my addr 172.21.15.86:0/1084804542 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.611+0000 7f8dffe00640 1 --2- 172.21.15.86:0/1084804542 >> v2:172.21.15.86:3300/0 conn(0x7f8e000cd990 0x7f8e0015c560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.611+0000 7fa815200640 1 Processor -- start 2024-03-06T07:27:14.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.611+0000 7fa815200640 1 -- start start 2024-03-06T07:27:14.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.613+0000 7fa815200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa810140370 0x7fa810070a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.613+0000 7fa815200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810072660 0x7fa8100cdbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.612+0000 7f8dff400640 1 -- 172.21.15.86:0/1084804542 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 354761156 0 0) 0x7f8e00166ec0 con 0x7f8e0015caa0 2024-03-06T07:27:14.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.613+0000 7fa815200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa810140750 con 0x7fa810070f60 2024-03-06T07:27:14.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.613+0000 7fa815200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa8100cd730 con 0x7fa810140370 2024-03-06T07:27:14.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.613+0000 7fa815200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa810071450 con 0x7fa810072660 2024-03-06T07:27:14.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.612+0000 7f8dff400640 1 -- 172.21.15.86:0/1084804542 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8de8003520 con 0x7f8e0015caa0 2024-03-06T07:27:14.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.612+0000 7f8dffe00640 1 -- 172.21.15.86:0/1084804542 >> v2:172.21.15.194:3300/0 conn(0x7f8e000cd5b0 msgr2=0x7f8e00145b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.612+0000 7f8dffe00640 1 --2- 172.21.15.86:0/1084804542 >> v2:172.21.15.194:3300/0 conn(0x7f8e000cd5b0 0x7f8e00145b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.612+0000 7f8dffe00640 1 -- 172.21.15.86:0/1084804542 >> v1:172.21.15.194:6789/0 conn(0x7f8e0015caa0 legacy=0x7f8e00162fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.612+0000 7f8dffe00640 1 -- 172.21.15.86:0/1084804542 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e0006d1a0 con 0x7f8e000cd990 2024-03-06T07:27:14.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.614+0000 7fa80de00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa810140370 0x7fa810070a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.614+0000 7fa80de00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa810140370 0x7fa810070a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51764/0 (socket says 172.21.15.86:51764) 2024-03-06T07:27:14.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.614+0000 7fa80de00640 1 -- 172.21.15.86:0/4168692823 learned_addr learned my addr 172.21.15.86:0/4168692823 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.614+0000 7fa80e800640 1 --2- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810072660 0x7fa8100cdbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.614+0000 7f8dffe00640 1 --2- 172.21.15.86:0/1084804542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd990 0x7f8e0015c560 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f8e00073670 tx=0x7f8dec02b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d928520a67a685c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.615+0000 7fa80de00640 1 -- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810072660 msgr2=0x7fa8100cdbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.615+0000 7fa80de00640 1 --2- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810072660 0x7fa8100cdbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.615+0000 7fa80de00640 1 -- 172.21.15.86:0/4168692823 >> v1:172.21.15.194:6789/0 conn(0x7fa810070f60 legacy=0x7fa810071f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.615+0000 7f8dff400640 1 -- 172.21.15.86:0/1084804542 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dec0317c0 con 0x7f8e000cd990 2024-03-06T07:27:14.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.615+0000 7f8dff400640 1 -- 172.21.15.86:0/1084804542 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8dec037070 con 0x7f8e000cd990 2024-03-06T07:27:14.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.615+0000 7f8dff400640 1 -- 172.21.15.86:0/1084804542 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dec031b60 con 0x7f8e000cd990 2024-03-06T07:27:14.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.616+0000 7f8e07c00640 1 -- 172.21.15.86:0/1084804542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd990 msgr2=0x7f8e0015c560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.616+0000 7f8e07c00640 1 --2- 172.21.15.86:0/1084804542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd990 0x7f8e0015c560 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f8e00073670 tx=0x7f8dec02b860 comp rx=0 tx=0).stop 2024-03-06T07:27:14.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.615+0000 7fa80de00640 1 -- 172.21.15.86:0/4168692823 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa810071730 con 0x7fa810140370 2024-03-06T07:27:14.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.617+0000 7f8e07c00640 1 -- 172.21.15.86:0/1084804542 shutdown_connections 2024-03-06T07:27:14.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.617+0000 7f8e07c00640 1 --2- 172.21.15.86:0/1084804542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd990 0x7f8e0015c560 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.617+0000 7f8e07c00640 1 --2- 172.21.15.86:0/1084804542 >> v2:172.21.15.194:3300/0 conn(0x7f8e000cd5b0 0x7f8e00145b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.617+0000 7f8e07c00640 1 -- 172.21.15.86:0/1084804542 >> 172.21.15.86:0/1084804542 conn(0x7f8e000cc4f0 msgr2=0x7f8e000cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.617+0000 7f8e07c00640 1 -- 172.21.15.86:0/1084804542 shutdown_connections 2024-03-06T07:27:14.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.617+0000 7f8e07c00640 1 -- 172.21.15.86:0/1084804542 wait complete. 2024-03-06T07:27:14.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.618+0000 7fa80de00640 1 --2- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810140370 0x7fa810070a20 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fa808009e50 tx=0x7fa80802c830 comp rx=0 tx=0).ready entity=mon.1 client_cookie=332486fdf9a3b72e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.620+0000 7fa80ca00640 1 -- 172.21.15.86:0/4168692823 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa808039070 con 0x7fa810140370 2024-03-06T07:27:14.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.620+0000 7fa80ca00640 1 -- 172.21.15.86:0/4168692823 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa808034680 con 0x7fa810140370 2024-03-06T07:27:14.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.620+0000 7fa80ca00640 1 -- 172.21.15.86:0/4168692823 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa808034a90 con 0x7fa810140370 2024-03-06T07:27:14.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 -- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810140370 msgr2=0x7fa810070a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 --2- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810140370 0x7fa810070a20 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fa808009e50 tx=0x7fa80802c830 comp rx=0 tx=0).stop 2024-03-06T07:27:14.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 -- 172.21.15.86:0/4168692823 shutdown_connections 2024-03-06T07:27:14.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 --2- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810072660 0x7fa8100cdbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 --2- 172.21.15.86:0/4168692823 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810140370 0x7fa810070a20 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 -- 172.21.15.86:0/4168692823 >> 172.21.15.86:0/4168692823 conn(0x7fa8100b1730 msgr2=0x7fa8100b1b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7f8e07c00640 1 Processor -- start 2024-03-06T07:27:14.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 -- 172.21.15.86:0/4168692823 shutdown_connections 2024-03-06T07:27:14.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7fa815200640 1 -- 172.21.15.86:0/4168692823 wait complete. 2024-03-06T07:27:14.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.621+0000 7f8e07c00640 1 -- start start 2024-03-06T07:27:14.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.622+0000 7f9e60e00640 1 Processor -- start 2024-03-06T07:27:14.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.622+0000 7f9e60e00640 1 -- start start 2024-03-06T07:27:14.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.623+0000 7f8e07c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd5b0 0x7f8e001f7650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.623+0000 7fa815200640 1 Processor -- start 2024-03-06T07:27:14.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7fa815200640 1 -- start start 2024-03-06T07:27:14.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.623+0000 7f8e07c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e000cd990 0x7f8e0014efa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8e05200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd5b0 0x7f8e001f7650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8e05200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd5b0 0x7f8e001f7650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53970/0 (socket says 172.21.15.86:53970) 2024-03-06T07:27:14.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8e05200640 1 -- 172.21.15.86:0/1500057796 learned_addr learned my addr 172.21.15.86:0/1500057796 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8e07c00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e0015caa0 0x7f8e001459c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e000cde40 con 0x7f8e000cd5b0 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e0015cf30 con 0x7f8e000cd990 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e001636c0 con 0x7f8e0015caa0 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f9e60e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9e5c06d560 0x7f9e5c0b1650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f9e60e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9e5c06d9c0 0x7f9e5c0b1b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f9e60e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9e5c0cd930 con 0x7f9e5c0cf320 2024-03-06T07:27:14.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f9e60e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9e5c0cdef0 con 0x7f9e5c06d560 2024-03-06T07:27:14.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f9e60e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9e5c0cdc10 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.624+0000 7f8dffe00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e000cd990 0x7f8e0014efa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.625+0000 7f8e05c00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e0015caa0 0x7f8e001459c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.625+0000 7f8e05200640 1 -- 172.21.15.86:0/1500057796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e0015caa0 msgr2=0x7f8e001459c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.625+0000 7f8e05200640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e0015caa0 0x7f8e001459c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.625+0000 7f8e05200640 1 -- 172.21.15.86:0/1500057796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e000cd990 msgr2=0x7f8e0014efa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.625+0000 7f8e05200640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e000cd990 0x7f8e0014efa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.625+0000 7f8e05200640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e0015d250 con 0x7f8e000cd5b0 2024-03-06T07:27:14.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.625+0000 7f9e59400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9e5c06d560 0x7f9e5c0b1650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa815200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa810070f60 0x7fa8101598f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa815200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810072660 0x7fa8100ac340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa815200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810140370 0x7fa8100accf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7f9e59400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9e5c06d560 0x7f9e5c0b1650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53974/0 (socket says 172.21.15.86:53974) 2024-03-06T07:27:14.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7f9e59400640 1 -- 172.21.15.86:0/1764707671 learned_addr learned my addr 172.21.15.86:0/1764707671 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7f9e5a800640 1 --2- 172.21.15.86:0/1764707671 >> v2:172.21.15.194:3300/0 conn(0x7f9e5c06d9c0 0x7f9e5c0b1b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa815200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8100cd460 con 0x7fa810070f60 2024-03-06T07:27:14.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa815200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8100cd2e0 con 0x7fa810072660 2024-03-06T07:27:14.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa815200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa81006d940 con 0x7fa810140370 2024-03-06T07:27:14.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7f0c64200640 1 Processor -- start 2024-03-06T07:27:14.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa80de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa810070f60 0x7fa8101598f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa80de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa810070f60 0x7fa8101598f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53982/0 (socket says 172.21.15.86:53982) 2024-03-06T07:27:14.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa80de00640 1 -- 172.21.15.86:0/2836735250 learned_addr learned my addr 172.21.15.86:0/2836735250 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7fa80d400640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810072660 0x7fa8100ac340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7f8e05200640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd5b0 0x7f8e001f7650 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f8df400e840 tx=0x7f8df400ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7fa80d400640 1 -- 172.21.15.86:0/2836735250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810140370 msgr2=0x7fa8100accf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7fa80d400640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa810140370 0x7fa8100accf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7fa80d400640 1 -- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa810070f60 msgr2=0x7fa8101598f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.626+0000 7f0c64200640 1 -- start start 2024-03-06T07:27:14.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7fa80d400640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa810070f60 0x7fa8101598f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7fa80d400640 1 -- 172.21.15.86:0/2836735250 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8100ab2a0 con 0x7fa810072660 2024-03-06T07:27:14.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7f77de200640 1 Processor -- start 2024-03-06T07:27:14.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7fa80d400640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810072660 0x7fa8100ac340 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fa7f800c390 tx=0x7fa7f800c850 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7f0c64200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0c5c0b6d90 0x7f0c5c0d0810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7f0c64200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0d0d50 0x7f0c5c0d2160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7f0c64200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0c5c0cf6d0 con 0x7f0c5c1617d0 2024-03-06T07:27:14.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7f0c64200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0c5c0d11a0 con 0x7f0c5c0b6d90 2024-03-06T07:27:14.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7f0c64200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0c5c0cff70 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7f9e59400640 1 -- 172.21.15.86:0/1764707671 >> v2:172.21.15.194:3300/0 conn(0x7f9e5c06d9c0 msgr2=0x7f9e5c0b1b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7f9e59400640 1 --2- 172.21.15.86:0/1764707671 >> v2:172.21.15.194:3300/0 conn(0x7f9e5c06d9c0 0x7f9e5c0b1b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c61800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0c5c1617d0 0x7f0c5c0ceb50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59268/0 (socket says 172.21.15.86:59268) 2024-03-06T07:27:14.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c61800640 1 -- 172.21.15.86:0/636322054 learned_addr learned my addr 172.21.15.86:0/636322054 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:14.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c60e00640 1 --2- 172.21.15.86:0/636322054 >> v2:172.21.15.86:3300/0 conn(0x7f0c5c0b6d90 0x7f0c5c0d0810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.628+0000 7f77de200640 1 -- start start 2024-03-06T07:27:14.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7f9e59400640 1 -- 172.21.15.86:0/1764707671 >> v1:172.21.15.194:6789/0 conn(0x7f9e5c0cf320 legacy=0x7f9e5c16bba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7f9e59400640 1 -- 172.21.15.86:0/1764707671 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e5c06cd60 con 0x7f9e5c06d560 2024-03-06T07:27:14.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c62200640 1 --2- 172.21.15.86:0/636322054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0d0d50 0x7f0c5c0d2160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c60e00640 1 -- 172.21.15.86:0/636322054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0d0d50 msgr2=0x7f0c5c0d2160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df4017070 con 0x7f8e000cd5b0 2024-03-06T07:27:14.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.627+0000 7f9e58a00640 1 -- 172.21.15.86:0/1764707671 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1995488031 0 0) 0x7f9e5c0cd930 con 0x7f9e5c0cf320 2024-03-06T07:27:14.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8df400c8e0 con 0x7f8e000cd5b0 2024-03-06T07:27:14.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df400ccd0 con 0x7f8e000cd5b0 2024-03-06T07:27:14.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7f8005220 con 0x7fa810072660 2024-03-06T07:27:14.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa7f80053c0 con 0x7fa810072660 2024-03-06T07:27:14.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7f8007270 con 0x7fa810072660 2024-03-06T07:27:14.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e0015fac0 con 0x7f8e000cd5b0 2024-03-06T07:27:14.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e0015d7d0 con 0x7f8e000cd5b0 2024-03-06T07:27:14.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f9e59400640 1 --2- 172.21.15.86:0/1764707671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d560 0x7f9e5c0b1650 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f9e5c16b120 tx=0x7f9e4c02b9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=456c945ca14f860c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f9e58a00640 1 -- 172.21.15.86:0/1764707671 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e4c02f810 con 0x7f9e5c06d560 2024-03-06T07:27:14.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f9e58a00640 1 -- 172.21.15.86:0/1764707671 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e4c02f9b0 con 0x7f9e5c06d560 2024-03-06T07:27:14.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7f9e58a00640 1 -- 172.21.15.86:0/1764707671 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e4c02fd60 con 0x7f9e5c06d560 2024-03-06T07:27:14.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c60e00640 1 --2- 172.21.15.86:0/636322054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0d0d50 0x7f0c5c0d2160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c60e00640 1 -- 172.21.15.86:0/636322054 >> v1:172.21.15.86:6789/0 conn(0x7f0c5c1617d0 legacy=0x7f0c5c0ceb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.629+0000 7f0c60e00640 1 -- 172.21.15.86:0/636322054 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c5c0cf9b0 con 0x7f0c5c0b6d90 2024-03-06T07:27:14.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7fa815200640 1 -- 172.21.15.86:0/2836735250 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa8100ab580 con 0x7fa810072660 2024-03-06T07:27:14.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.630+0000 7fa815200640 1 -- 172.21.15.86:0/2836735250 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8100abb40 con 0x7fa810072660 2024-03-06T07:27:14.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.631+0000 7f0c62200640 1 --2- 172.21.15.86:0/636322054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0d0d50 0x7f0c5c0d2160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:14.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.631+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f8e0004f330 con 0x7f8e000cd5b0 2024-03-06T07:27:14.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.631+0000 7f9e60e00640 1 -- 172.21.15.86:0/1764707671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d560 msgr2=0x7f9e5c0b1650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.631+0000 7f9e60e00640 1 --2- 172.21.15.86:0/1764707671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d560 0x7f9e5c0b1650 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f9e5c16b120 tx=0x7f9e4c02b9f0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f9e60e00640 1 -- 172.21.15.86:0/1764707671 shutdown_connections 2024-03-06T07:27:14.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f77de200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f77d8145b70 0x7f77d815c560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f77de200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f77d815caa0 0x7f77d8162fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f9e60e00640 1 --2- 172.21.15.86:0/1764707671 >> v2:172.21.15.194:3300/0 conn(0x7f9e5c06d9c0 0x7f9e5c0b1b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f9e60e00640 1 --2- 172.21.15.86:0/1764707671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d560 0x7f9e5c0b1650 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f9e60e00640 1 -- 172.21.15.86:0/1764707671 >> 172.21.15.86:0/1764707671 conn(0x7f9e5c0cfa70 msgr2=0x7f9e5c0cfe80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fa7e0001030 con 0x7fa810072660 2024-03-06T07:27:14.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f77de200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f77d8166ec0 con 0x7f77d80cd5b0 2024-03-06T07:27:14.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f77de200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f77d806cbe0 con 0x7f77d8145b70 2024-03-06T07:27:14.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f77de200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f77d806cec0 con 0x7f77d815caa0 2024-03-06T07:27:14.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.632+0000 7f9e60e00640 1 -- 172.21.15.86:0/1764707671 shutdown_connections 2024-03-06T07:27:14.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.633+0000 7f77d6a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f77d8145b70 0x7f77d815c560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.633+0000 7f77d7400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f77d80cd5b0 0x7f77d80cd990 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59284/0 (socket says 172.21.15.86:59284) 2024-03-06T07:27:14.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.633+0000 7f9e60e00640 1 -- 172.21.15.86:0/1764707671 wait complete. 2024-03-06T07:27:14.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.633+0000 7f0c60e00640 1 --2- 172.21.15.86:0/636322054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0b6d90 0x7f0c5c0d0810 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f0c4c0097a0 tx=0x7f0c4c02b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ee374786a59846dd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.633+0000 7f77d7e00640 1 --2- 172.21.15.86:0/566510975 >> v2:172.21.15.194:3300/0 conn(0x7f77d815caa0 0x7f77d8162fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.634+0000 7f77d7400640 1 -- 172.21.15.86:0/566510975 learned_addr learned my addr 172.21.15.86:0/566510975 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:14.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.634+0000 7f77d7e00640 1 -- 172.21.15.86:0/566510975 >> v2:172.21.15.86:3300/0 conn(0x7f77d8145b70 msgr2=0x7f77d815c560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.634+0000 7f77d7e00640 1 --2- 172.21.15.86:0/566510975 >> v2:172.21.15.86:3300/0 conn(0x7f77d8145b70 0x7f77d815c560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.634+0000 7f77d7e00640 1 -- 172.21.15.86:0/566510975 >> v1:172.21.15.86:6789/0 conn(0x7f77d80cd5b0 legacy=0x7f77d80cd990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.634+0000 7f0c5be00640 1 -- 172.21.15.86:0/636322054 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c4c0317c0 con 0x7f0c5c0b6d90 2024-03-06T07:27:14.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.635+0000 7f0c5be00640 1 -- 172.21.15.86:0/636322054 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0c4c037070 con 0x7f0c5c0b6d90 2024-03-06T07:27:14.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.635+0000 7f0c5be00640 1 -- 172.21.15.86:0/636322054 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c4c031bc0 con 0x7f0c5c0b6d90 2024-03-06T07:27:14.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.635+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa7f8019020 con 0x7fa810072660 2024-03-06T07:27:14.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.634+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8df401a070 con 0x7f8e000cd5b0 2024-03-06T07:27:14.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.635+0000 7f8dfcc00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de006b520 0x7f8de006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.635+0000 7fd1b3a00640 1 Processor -- start 2024-03-06T07:27:14.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.635+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8df408a060 con 0x7f8e000cd5b0 2024-03-06T07:27:14.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.634+0000 7f77d7e00640 1 -- 172.21.15.86:0/566510975 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77d806d1a0 con 0x7f77d815caa0 2024-03-06T07:27:14.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.635+0000 7f8dfcc00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f8de0073740 0x7f8de0075b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f8df4022e50 con 0x7f8de0073740 2024-03-06T07:27:14.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8df4091050 con 0x7f8e000cd5b0 2024-03-06T07:27:14.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 -- 172.21.15.86:0/636322054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0b6d90 msgr2=0x7f0c5c0d0810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f8dffe00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de006b520 0x7f8de006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 --2- 172.21.15.86:0/636322054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0b6d90 0x7f0c5c0d0810 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f0c4c0097a0 tx=0x7f0c4c02b860 comp rx=0 tx=0).stop 2024-03-06T07:27:14.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 -- 172.21.15.86:0/636322054 shutdown_connections 2024-03-06T07:27:14.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 --2- 172.21.15.86:0/636322054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0d0d50 0x7f0c5c0d2160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 --2- 172.21.15.86:0/636322054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0b6d90 0x7f0c5c0d0810 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 -- 172.21.15.86:0/636322054 >> 172.21.15.86:0/636322054 conn(0x7f0c5c09d210 msgr2=0x7f0c5c0d3f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7fa48fe00640 1 Processor -- start 2024-03-06T07:27:14.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 -- 172.21.15.86:0/636322054 shutdown_connections 2024-03-06T07:27:14.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7fd1b3a00640 1 -- start start 2024-03-06T07:27:14.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7fa806000640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7ec06b5a0 0x7fa7ec06da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f0c64200640 1 -- 172.21.15.86:0/636322054 wait complete. 2024-03-06T07:27:14.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7f77d7e00640 1 --2- 172.21.15.86:0/566510975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d815caa0 0x7f77d8162fb0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f77d8162ac0 tx=0x7f77c402b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d5114bd870d32120 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7f8dffe00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de006b520 0x7f8de006d9e0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f8dec036000 tx=0x7f8dec008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.636+0000 7fa48fe00640 1 -- start start 2024-03-06T07:27:14.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa7f8089500 con 0x7fa810072660 2024-03-06T07:27:14.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7f8e05c00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f8de0073740 0x7f8de0075b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7f9e60e00640 1 Processor -- start 2024-03-06T07:27:14.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7f9e60e00640 1 -- start start 2024-03-06T07:27:14.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7fa806000640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fa7ec0737c0 0x7fa7ec075be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fa7f800ee50 con 0x7fa7ec0737c0 2024-03-06T07:27:14.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.637+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa7f8090050 con 0x7fa810072660 2024-03-06T07:27:14.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.638+0000 7fa80de00640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7ec06b5a0 0x7fa7ec06da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.638+0000 7fa80e800640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fa7ec0737c0 0x7fa7ec075be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.638+0000 7f8e05c00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f8de0073740 0x7f8de0075b60 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.638+0000 7fa80de00640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7ec06b5a0 0x7fa7ec06da60 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fa80802c870 tx=0x7fa808009000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e60e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e5c06d560 0x7f9e5c0b4d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e60e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d9c0 0x7f9e5c16d910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e60e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e5c0cf320 0x7f9e5c1614a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e60e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e5c0cd5b0 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e60e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e5c06e670 con 0x7f9e5c06d560 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e60e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e5c08bc10 con 0x7f9e5c0cf320 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e59400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d9c0 0x7f9e5c16d910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e59400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d9c0 0x7f9e5c16d910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54012/0 (socket says 172.21.15.86:54012) 2024-03-06T07:27:14.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7fa48fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa4900b2e50 0x7fa4900d4850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7fa48fe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4900b5240 con 0x7fa4900d0c00 2024-03-06T07:27:14.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7fa48fe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4900cf4d0 con 0x7fa49015e200 2024-03-06T07:27:14.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7fa48fe00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4900d2ec0 con 0x7fa4900b2e50 2024-03-06T07:27:14.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== osd.0 v2:172.21.15.86:6800/1601033517 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f8df4022e50 con 0x7f8de0073740 2024-03-06T07:27:14.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.639+0000 7f9e59400640 1 -- 172.21.15.86:0/3268748831 learned_addr learned my addr 172.21.15.86:0/3268748831 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f77d6000640 1 -- 172.21.15.86:0/566510975 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77c40315a0 con 0x7f77d815caa0 2024-03-06T07:27:14.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f77d6000640 1 -- 172.21.15.86:0/566510975 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f77c4037070 con 0x7f77d815caa0 2024-03-06T07:27:14.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f77d6000640 1 -- 172.21.15.86:0/566510975 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77c4031960 con 0x7f77d815caa0 2024-03-06T07:27:14.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f0c64200640 1 Processor -- start 2024-03-06T07:27:14.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fa80e800640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fa7ec0737c0 0x7fa7ec075be0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fd1b3a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd5b0 0x7fd1ac145b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fd1b3a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd1ac0cd990 0x7fd1ac15c560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fd1b3a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1ac166ec0 con 0x7fd1ac15caa0 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fd1b3a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd1ac06cbe0 con 0x7fd1ac0cd990 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fd1b3a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1ac06cec0 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fa48ea00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7fa49015e200 0x7fa4900d04f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:36352/0 (socket says 172.21.15.86:36352) 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7fa48ea00640 1 -- 172.21.15.86:0/1724557943 learned_addr learned my addr 172.21.15.86:0/1724557943 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:14.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f0c64200640 1 -- start start 2024-03-06T07:27:14.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f9e5a800640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e5c0cf320 0x7f9e5c1614a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f77de200640 1 -- 172.21.15.86:0/566510975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d815caa0 msgr2=0x7f77d8162fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.640+0000 7f77de200640 1 --2- 172.21.15.86:0/566510975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d815caa0 0x7f77d8162fb0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f77d8162ac0 tx=0x7f77c402b840 comp rx=0 tx=0).stop 2024-03-06T07:27:14.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f77de200640 1 -- 172.21.15.86:0/566510975 shutdown_connections 2024-03-06T07:27:14.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f77de200640 1 --2- 172.21.15.86:0/566510975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d815caa0 0x7f77d8162fb0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f9e59e00640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e5c06d560 0x7f9e5c0b4d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f77de200640 1 --2- 172.21.15.86:0/566510975 >> v2:172.21.15.86:3300/0 conn(0x7f77d8145b70 0x7f77d815c560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f77de200640 1 -- 172.21.15.86:0/566510975 >> 172.21.15.86:0/566510975 conn(0x7f77d80cc4f0 msgr2=0x7f77d80cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7fd1b1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd5b0 0x7fd1ac145b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7fd1abe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd1ac0cd990 0x7fd1ac15c560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f77de200640 1 -- 172.21.15.86:0/566510975 shutdown_connections 2024-03-06T07:27:14.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f77de200640 1 -- 172.21.15.86:0/566510975 wait complete. 2024-03-06T07:27:14.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7fd1abe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd1ac0cd990 0x7fd1ac15c560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51826/0 (socket says 172.21.15.86:51826) 2024-03-06T07:27:14.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7fd1abe00640 1 -- 172.21.15.86:0/2102847685 learned_addr learned my addr 172.21.15.86:0/2102847685 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== osd.1 v2:172.21.15.86:6809/2555210107 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7fa7f800ee50 con 0x7fa7ec0737c0 2024-03-06T07:27:14.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fa48d600640 1 -- 172.21.15.86:0/1724557943 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3287795611 0 0) 0x7fa4900cf4d0 con 0x7fa49015e200 2024-03-06T07:27:14.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fa48d600640 1 -- 172.21.15.86:0/1724557943 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa474003520 con 0x7fa49015e200 2024-03-06T07:27:14.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fa48f400640 1 --2- 172.21.15.86:0/1724557943 >> v2:172.21.15.86:3300/0 conn(0x7fa4900b2e50 0x7fa4900d4850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f9e59400640 1 -- 172.21.15.86:0/3268748831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e5c0cf320 msgr2=0x7f9e5c1614a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f9e59400640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e5c0cf320 0x7f9e5c1614a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f9e59400640 1 -- 172.21.15.86:0/3268748831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e5c06d560 msgr2=0x7f9e5c0b4d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f9e59400640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e5c06d560 0x7f9e5c0b4d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.641+0000 7f9e59400640 1 -- 172.21.15.86:0/3268748831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e5c0b5270 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f0c64200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0b6d90 0x7f0c5c160a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f0c64200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0d0d50 0x7f0c5c0cd390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f0c64200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c5c1617d0 0x7f0c5c204340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f0c64200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c5c161bb0 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f0c64200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c5c0d1c40 con 0x7f0c5c1617d0 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f0c64200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0c5c0b5350 con 0x7f0c5c0b6d90 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0d0d50 0x7f0c5c0cd390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0d0d50 0x7f0c5c0cd390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54038/0 (socket says 172.21.15.86:54038) 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 -- 172.21.15.86:0/1325885917 learned_addr learned my addr 172.21.15.86:0/1325885917 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c62200640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c5c1617d0 0x7f0c5c204340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 -- 172.21.15.86:0/1325885917 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0b6d90 msgr2=0x7f0c5c160a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c61800640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0b6d90 0x7f0c5c160a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c5c0b6d90 0x7f0c5c160a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 -- 172.21.15.86:0/1325885917 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c5c1617d0 msgr2=0x7f0c5c204340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c5c1617d0 0x7f0c5c204340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f0c60e00640 1 -- 172.21.15.86:0/1325885917 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c5c0cbbc0 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f9e59400640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d9c0 0x7f9e5c16d910 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f9e4c02ba30 tx=0x7f9e4c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e4c048070 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e4c03d070 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.644+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e4c003610 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f77de200640 1 Processor -- start 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7f77de200640 1 -- start start 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7f77de200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77d80cd5b0 0x7f77d8142580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7f77de200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d8145b70 0x7f77d81ebee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7f77de200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77d815caa0 0x7f77d81432e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fd1abe00640 1 -- 172.21.15.86:0/2102847685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd5b0 msgr2=0x7fd1ac145b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fd1abe00640 1 --2- 172.21.15.86:0/2102847685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd5b0 0x7fd1ac145b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fd1abe00640 1 -- 172.21.15.86:0/2102847685 >> v1:172.21.15.194:6789/0 conn(0x7fd1ac15caa0 legacy=0x7fd1ac162fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fd1abe00640 1 -- 172.21.15.86:0/2102847685 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1ac06d1a0 con 0x7fd1ac0cd990 2024-03-06T07:27:14.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fd1abe00640 1 --2- 172.21.15.86:0/2102847685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd990 0x7fd1ac15c560 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fd198009870 tx=0x7fd19802b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=874ace9a635b1e7b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fa48f400640 1 -- 172.21.15.86:0/1724557943 >> v1:172.21.15.194:6789/0 conn(0x7fa49015e200 legacy=0x7fa4900d04f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.642+0000 7fa48d600640 1 -- 172.21.15.86:0/1724557943 <== mon.1 v1:172.21.15.194:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4092447683 0 0) 0x7fa474003520 con 0x7fa49015e200 2024-03-06T07:27:14.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7fa48f400640 1 -- 172.21.15.86:0/1724557943 >> v1:172.21.15.86:6789/0 conn(0x7fa4900d0c00 legacy=0x7fa4900b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.643+0000 7fa48f400640 1 -- 172.21.15.86:0/1724557943 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4900b5500 con 0x7fa4900b2e50 2024-03-06T07:27:14.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fa48f400640 1 --2- 172.21.15.86:0/1724557943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4900b2e50 0x7fa4900d4850 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fa488009d90 tx=0x7fa48802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b3716ad3e8762289 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fa48d600640 1 -- 172.21.15.86:0/1724557943 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa488039070 con 0x7fa4900b2e50 2024-03-06T07:27:14.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fd1ab400640 1 -- 172.21.15.86:0/2102847685 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd198034070 con 0x7fd1ac0cd990 2024-03-06T07:27:14.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fd1ab400640 1 -- 172.21.15.86:0/2102847685 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd198030840 con 0x7fd1ac0cd990 2024-03-06T07:27:14.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fa48d600640 1 -- 172.21.15.86:0/1724557943 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa48802ca00 con 0x7fa4900b2e50 2024-03-06T07:27:14.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fd1ab400640 1 -- 172.21.15.86:0/2102847685 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd198030c10 con 0x7fd1ac0cd990 2024-03-06T07:27:14.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7f0c60e00640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0d0d50 0x7f0c5c0cd390 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f0c4c0098c0 tx=0x7f0c4c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.645+0000 7fa48d600640 1 -- 172.21.15.86:0/1724557943 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa488039070 con 0x7fa4900b2e50 2024-03-06T07:27:14.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c4c032860 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7f9e60e00640 1 -- 172.21.15.86:0/3268748831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e5c072130 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7f9e60e00640 1 -- 172.21.15.86:0/3268748831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e5c0b2440 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0c4c032a00 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7f0c64200640 1 -- 172.21.15.86:0/1325885917 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c5c0cbea0 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7f0c64200640 1 -- 172.21.15.86:0/1325885917 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c5c0cc980 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7fd1b3a00640 1 -- 172.21.15.86:0/2102847685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd990 msgr2=0x7fd1ac15c560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7fd1b3a00640 1 --2- 172.21.15.86:0/2102847685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd990 0x7fd1ac15c560 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fd198009870 tx=0x7fd19802b980 comp rx=0 tx=0).stop 2024-03-06T07:27:14.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7fa48fe00640 1 -- 172.21.15.86:0/1724557943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4900b2e50 msgr2=0x7fa4900d4850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.646+0000 7fa48fe00640 1 --2- 172.21.15.86:0/1724557943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4900b2e50 0x7fa4900d4850 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fa488009d90 tx=0x7fa48802bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7fa48fe00640 1 -- 172.21.15.86:0/1724557943 shutdown_connections 2024-03-06T07:27:14.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7fa48fe00640 1 --2- 172.21.15.86:0/1724557943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4900b2e50 0x7fa4900d4850 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7fa48fe00640 1 -- 172.21.15.86:0/1724557943 >> 172.21.15.86:0/1724557943 conn(0x7fa4900d3d30 msgr2=0x7fa4900d4140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7fa48fe00640 1 -- 172.21.15.86:0/1724557943 shutdown_connections 2024-03-06T07:27:14.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c4c032860 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7fd1b3a00640 1 -- 172.21.15.86:0/2102847685 shutdown_connections 2024-03-06T07:27:14.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7fd1b3a00640 1 --2- 172.21.15.86:0/2102847685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd990 0x7fd1ac15c560 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7fd1b3a00640 1 --2- 172.21.15.86:0/2102847685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd5b0 0x7fd1ac145b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7fd1b3a00640 1 -- 172.21.15.86:0/2102847685 >> 172.21.15.86:0/2102847685 conn(0x7fd1ac0cc4f0 msgr2=0x7fd1ac0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f0c34001030 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.649+0000 7fd1b3a00640 1 -- 172.21.15.86:0/2102847685 shutdown_connections 2024-03-06T07:27:14.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7f77de200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77d81634f0 con 0x7f77d815caa0 2024-03-06T07:27:14.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7f77de200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77d815cef0 con 0x7f77d8145b70 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7f77de200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f77d8163670 con 0x7f77d80cd5b0 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7f77d7400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77d80cd5b0 0x7f77d8142580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77d80cd5b0 0x7f77d8142580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55042/0 (socket says 172.21.15.86:55042) 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7400640 1 -- 172.21.15.86:0/2702104554 learned_addr learned my addr 172.21.15.86:0/2702104554 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d6a00640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d8145b70 0x7f77d81ebee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7e00640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77d815caa0 0x7f77d81432e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7400640 1 -- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d8145b70 msgr2=0x7f77d81ebee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7400640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77d8145b70 0x7f77d81ebee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7400640 1 -- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77d815caa0 msgr2=0x7f77d81432e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7400640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77d815caa0 0x7f77d81432e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7400640 1 -- 172.21.15.86:0/2702104554 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77d81417c0 con 0x7f77d80cd5b0 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.648+0000 7f77d7e00640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77d815caa0 0x7f77d81432e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:14.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.649+0000 7f77d7400640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77d80cd5b0 0x7f77d8142580 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f77d000ba30 tx=0x7f77d000bef0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.647+0000 7fa48fe00640 1 -- 172.21.15.86:0/1724557943 wait complete. 2024-03-06T07:27:14.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.649+0000 7fa48fe00640 1 Processor -- start 2024-03-06T07:27:14.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.650+0000 7fa48fe00640 1 -- start start 2024-03-06T07:27:14.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.651+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77d002a070 con 0x7f77d80cd5b0 2024-03-06T07:27:14.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.651+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f77d001dc20 con 0x7f77d80cd5b0 2024-03-06T07:27:14.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.651+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77d0025440 con 0x7f77d80cd5b0 2024-03-06T07:27:14.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.649+0000 7fd1b3a00640 1 -- 172.21.15.86:0/2102847685 wait complete. 2024-03-06T07:27:14.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.651+0000 7f9f1ca00640 1 Processor -- start 2024-03-06T07:27:14.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.651+0000 7f9f1ca00640 1 -- start start 2024-03-06T07:27:14.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.651+0000 7fd1b3a00640 1 Processor -- start 2024-03-06T07:27:14.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.652+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9e4c0037b0 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.652+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f9e5c04f330 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.652+0000 7fd1b3a00640 1 -- start start 2024-03-06T07:27:14.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.652+0000 7fa48fe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4900b2e50 0x7fa4901571c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f77d8141aa0 con 0x7f77d80cd5b0 2024-03-06T07:27:14.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7fa48fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4900d0c00 0x7fa49015b040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f77d8071e70 con 0x7f77d80cd5b0 2024-03-06T07:27:14.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f9f1ca00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9f180d2360 0x7f9f180d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f9f1ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180b3ec0 0x7f9f180b4350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f9f1ca00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f18161960 con 0x7f9f18161580 2024-03-06T07:27:14.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f9f1ca00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f180d2ce0 con 0x7f9f180d2360 2024-03-06T07:27:14.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f9f1ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f180cf6f0 con 0x7f9f180b3ec0 2024-03-06T07:27:14.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7fa48fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa49015e200 0x7fa4900cb7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7fa48ea00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4900b2e50 0x7fa4901571c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7fa48ea00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4900b2e50 0x7fa4901571c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51860/0 (socket says 172.21.15.86:51860) 2024-03-06T07:27:14.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f9f15e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9f180d2360 0x7f9f180d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f9f15e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9f180d2360 0x7f9f180d27a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51862/0 (socket says 172.21.15.86:51862) 2024-03-06T07:27:14.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f9f15e00640 1 -- 172.21.15.86:0/1087152775 learned_addr learned my addr 172.21.15.86:0/1087152775 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f77d8141570 con 0x7f77d80cd5b0 2024-03-06T07:27:14.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7fd1b3a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd5b0 0x7fd1ac0b4af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7fd1b3a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd990 0x7fd1ac071ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7fa48e000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4900d0c00 0x7fa49015b040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7fa48ea00640 1 -- 172.21.15.86:0/1306669779 learned_addr learned my addr 172.21.15.86:0/1306669779 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f9f15e00640 1 -- 172.21.15.86:0/1087152775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180b3ec0 msgr2=0x7f9f180b4350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:14.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f9f15e00640 1 --2- 172.21.15.86:0/1087152775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180b3ec0 0x7f9f180b4350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4900cfd60 con 0x7fa49015e200 2024-03-06T07:27:14.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7fd1b3a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd1ac15caa0 0x7fd1ac072010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f9f15e00640 1 -- 172.21.15.86:0/1087152775 >> v1:172.21.15.86:6789/0 conn(0x7f9f18161580 legacy=0x7f9f180d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4900cfa80 con 0x7fa4900b2e50 2024-03-06T07:27:14.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fd1b1000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd5b0 0x7fd1ac0b4af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.653+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0c4c04c060 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.654+0000 7f0c59600640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c4006b6d0 0x7f0c4006db90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fd1b1000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd5b0 0x7fd1ac0b4af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51866/0 (socket says 172.21.15.86:51866) 2024-03-06T07:27:14.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fa48f400640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa49015e200 0x7fa4900cb7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fd1b1000640 1 -- 172.21.15.86:0/4237050161 learned_addr learned my addr 172.21.15.86:0/4237050161 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1ac0cde10 con 0x7fd1ac15caa0 2024-03-06T07:27:14.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fd1b1a00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd1ac15caa0 0x7fd1ac072010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa49015e5e0 con 0x7fa4900d0c00 2024-03-06T07:27:14.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7f9f15400640 1 -- 172.21.15.86:0/1087152775 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3888810296 0 0) 0x7f9f18161960 con 0x7f9f18161580 2024-03-06T07:27:14.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7f9f15e00640 1 -- 172.21.15.86:0/1087152775 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f180cf9d0 con 0x7f9f180d2360 2024-03-06T07:27:14.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1ac15cef0 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1ac1636c0 con 0x7fd1ac0cd990 2024-03-06T07:27:14.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7f0c61800640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c4006b6d0 0x7f0c4006db90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1abe00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd990 0x7fd1ac071ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1b1000640 1 -- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd990 msgr2=0x7fd1ac071ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1b1000640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd1ac0cd990 0x7fd1ac071ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1b1000640 1 -- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd1ac15caa0 msgr2=0x7fd1ac072010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1b1000640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd1ac15caa0 0x7fd1ac072010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1b1000640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1ac163cd0 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1b1a00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd1ac15caa0 0x7fd1ac072010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:14.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7f0c61800640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c4006b6d0 0x7f0c4006db90 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f0c50000f60 tx=0x7f0c50009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7fd1b1000640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd5b0 0x7fd1ac0b4af0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fd1a000e8a0 tx=0x7fd1a000ed60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0c4c078210 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7f9e52000640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e3c06b4d0 0x7f9e3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.655+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9e4c0abc50 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7f9e52000640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f9e3c073670 0x7f9e3c075a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f9e4c00af30 con 0x7f9e3c073670 2024-03-06T07:27:14.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9e4c0768d0 con 0x7f9e5c06d9c0 2024-03-06T07:27:14.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.656+0000 7f9f15e00640 1 --2- 172.21.15.86:0/1087152775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f180d2360 0x7f9f180d27a0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f9f04009870 tx=0x7f9f0402b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b81fee2e101f2d5a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f0c59600640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f0c40074400 0x7f0c400768a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f0c40076f70 con 0x7f0c40074400 2024-03-06T07:27:14.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0c4c076ba0 con 0x7f0c5c0d0d50 2024-03-06T07:27:14.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1a0017070 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd1a000c940 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1a000cd30 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f9f15400640 1 -- 172.21.15.86:0/1087152775 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f04034070 con 0x7f9f180d2360 2024-03-06T07:27:14.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f9f15400640 1 -- 172.21.15.86:0/1087152775 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f04030840 con 0x7f9f180d2360 2024-03-06T07:27:14.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f9f15400640 1 -- 172.21.15.86:0/1087152775 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f04030c10 con 0x7f9f180d2360 2024-03-06T07:27:14.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f9e5a800640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f9e3c073670 0x7f9e3c075a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7f9e59e00640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e3c06b4d0 0x7f9e3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd1ac163fb0 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.657+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd1ac164570 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7f0c62200640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f0c40074400 0x7f0c400768a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7f0c62200640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f0c40074400 0x7f0c400768a0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7f9e59e00640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e3c06b4d0 0x7f9e3c06d990 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f9e48000f60 tx=0x7f9e48009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== mon.2 v2:172.21.15.194:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f77d002d070 con 0x7f77d80cd5b0 2024-03-06T07:27:14.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7fa48ea00640 1 -- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4900d0c00 msgr2=0x7fa49015b040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7fa48ea00640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4900d0c00 0x7fa49015b040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7fa48ea00640 1 -- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa49015e200 msgr2=0x7fa4900cb7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.658+0000 7fa48ea00640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa49015e200 0x7fa4900cb7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.659+0000 7fa48ea00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa490157700 con 0x7fa4900b2e50 2024-03-06T07:27:14.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.659+0000 7fa48f400640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa49015e200 0x7fa4900cb7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:14.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.659+0000 7f9e5a800640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f9e3c073670 0x7f9e3c075a90 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.659+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fd188001030 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.659+0000 7fa48ea00640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4900b2e50 0x7fa4901571c0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fa48000c730 tx=0x7fa48000cbf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa480018070 con 0x7fa4900b2e50 2024-03-06T07:27:14.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f9f1ca00640 1 -- 172.21.15.86:0/1087152775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f180d2360 msgr2=0x7f9f180d27a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f9f1ca00640 1 --2- 172.21.15.86:0/1087152775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f180d2360 0x7f9f180d27a0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f9f04009870 tx=0x7f9f0402b980 comp rx=0 tx=0).stop 2024-03-06T07:27:14.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4900cc240 con 0x7fa4900b2e50 2024-03-06T07:27:14.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4900b6c00 con 0x7fa4900b2e50 2024-03-06T07:27:14.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa4800074e0 con 0x7fa4900b2e50 2024-03-06T07:27:14.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== osd.3 v2:172.21.15.86:6824/3540554891 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f9e4c00af30 con 0x7f9e3c073670 2024-03-06T07:27:14.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f77cf400640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f77b806b5a0 0x7f77b806da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== osd.2 v2:172.21.15.86:6808/2264489672 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f0c40076f70 con 0x7f0c40074400 2024-03-06T07:27:14.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f9f1ca00640 1 -- 172.21.15.86:0/1087152775 shutdown_connections 2024-03-06T07:27:14.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f9f1ca00640 1 --2- 172.21.15.86:0/1087152775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180b3ec0 0x7f9f180b4350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.660+0000 7f9f1ca00640 1 --2- 172.21.15.86:0/1087152775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f180d2360 0x7f9f180d27a0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f9f1ca00640 1 -- 172.21.15.86:0/1087152775 >> 172.21.15.86:0/1087152775 conn(0x7f9f180d3d80 msgr2=0x7f9f180d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f9f1ca00640 1 -- 172.21.15.86:0/1087152775 shutdown_connections 2024-03-06T07:27:14.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== mon.2 v2:172.21.15.194:3301/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f77d009b680 con 0x7f77d80cd5b0 2024-03-06T07:27:14.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f77cf400640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f77b80737c0 0x7f77b8075be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f77b8076290 con 0x7f77b80737c0 2024-03-06T07:27:14.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== mon.2 v2:172.21.15.194:3301/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7f77d009ba70 con 0x7f77d80cd5b0 2024-03-06T07:27:14.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa480018070 con 0x7fa4900b2e50 2024-03-06T07:27:14.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fa468001030 con 0x7fa4900b2e50 2024-03-06T07:27:14.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f77d6a00640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f77b806b5a0 0x7f77b806da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.662+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd1a001a070 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.662+0000 7f77d6a00640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f77b806b5a0 0x7f77b806da60 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f77d8144ef0 tx=0x7f77c8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.661+0000 7f9f1ca00640 1 -- 172.21.15.86:0/1087152775 wait complete. 2024-03-06T07:27:14.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.663+0000 7fd1a8c00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd18c06b4d0 0x7fd18c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.664+0000 7f77d7e00640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f77b80737c0 0x7f77b8075be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.664+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd1a008ae70 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.664+0000 7fd1a8c00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd18c0736f0 0x7fd18c075b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.664+0000 7fd1abe00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd18c06b4d0 0x7fd18c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7f77d7e00640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f77b80737c0 0x7f77b8075be0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fd18c0761c0 con 0x7fd18c0736f0 2024-03-06T07:27:14.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7f9f1ca00640 1 Processor -- start 2024-03-06T07:27:14.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7fd1b1a00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd18c0736f0 0x7fd18c075b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7fd1abe00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd18c06b4d0 0x7fd18c06d990 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fd198036000 tx=0x7fd19803b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd1a008b260 con 0x7fd1ac0cd5b0 2024-03-06T07:27:14.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7f9f1ca00640 1 -- start start 2024-03-06T07:27:14.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.664+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa4800047e0 con 0x7fa4900b2e50 2024-03-06T07:27:14.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.665+0000 7fa487400640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa46c06b6d0 0x7fa46c06db90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.666+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa4800895b0 con 0x7fa4900b2e50 2024-03-06T07:27:14.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.666+0000 7fa487400640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fa46c073870 0x7fa46c075c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.666+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fa46c076340 con 0x7fa46c073870 2024-03-06T07:27:14.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.666+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa480089a40 con 0x7fa4900b2e50 2024-03-06T07:27:14.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.666+0000 7fd1b1a00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd18c0736f0 0x7fd18c075b10 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.666+0000 7fa48e000640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa46c06b6d0 0x7fa46c06db90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.667+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== osd.4 v2:172.21.15.194:6800/2403584566 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f77b8076290 con 0x7f77b80737c0 2024-03-06T07:27:14.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.667+0000 7fa48f400640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fa46c073870 0x7fa46c075c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.667+0000 7fa48f400640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fa46c073870 0x7fa46c075c90 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f1ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f180b3ec0 0x7f9f180ccca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7fa48e000640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa46c06b6d0 0x7fa46c06db90 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa4900cad60 tx=0x7fa478009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f1ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180d2360 0x7f9f18156140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f1ca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f18161580 0x7f9f1815c4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f1ca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f180b39a0 con 0x7f9f180b3ec0 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f1ca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f180b36c0 con 0x7f9f18161580 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f1ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f180cf3a0 con 0x7f9f180d2360 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f16800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f180b3ec0 0x7f9f180ccca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f15e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180d2360 0x7f9f18156140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f15e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180d2360 0x7f9f18156140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55076/0 (socket says 172.21.15.86:55076) 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.668+0000 7f9f15e00640 1 -- 172.21.15.86:0/2693054479 learned_addr learned my addr 172.21.15.86:0/2693054479 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.669+0000 7f9f17200640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f18161580 0x7f9f1815c4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.669+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== osd.5 v2:172.21.15.194:6816/1278478503 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7fd18c0761c0 con 0x7fd18c0736f0 2024-03-06T07:27:14.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.669+0000 7f9f16800640 1 -- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180d2360 msgr2=0x7f9f18156140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.669+0000 7f9f16800640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f180d2360 0x7f9f18156140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.669+0000 7f9f16800640 1 -- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f18161580 msgr2=0x7f9f1815c4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.669+0000 7f9f16800640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f18161580 0x7f9f1815c4e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.669+0000 7f9f16800640 1 -- 172.21.15.86:0/2693054479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f180cd840 con 0x7f9f180b3ec0 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.670+0000 7f9f16800640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f180b3ec0 0x7f9f180ccca0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f9f0800c3e0 tx=0x7f9f0800c8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.671+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f08017070 con 0x7f9f180b3ec0 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.671+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f080035f0 con 0x7f9f180b3ec0 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.671+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f080039e0 con 0x7f9f180b3ec0 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.671+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f180cd280 con 0x7f9f180b3ec0 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.671+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f1815bce0 con 0x7f9f180b3ec0 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.673+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== osd.6 v2:172.21.15.194:6808/2611339787 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7fa46c076340 con 0x7fa46c073870 2024-03-06T07:27:14.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.672+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f9f1804f330 con 0x7f9f180b3ec0 2024-03-06T07:27:14.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.674+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f8e0006e8d0 con 0x7f8de0073740 2024-03-06T07:27:14.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.673+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fa7e0002dd0 con 0x7fa7ec0737c0 2024-03-06T07:27:14.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.674+0000 7f8dfcc00640 1 -- 172.21.15.86:0/1500057796 <== osd.0 v2:172.21.15.86:6800/1601033517 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f8e0006e8d0 con 0x7f8de0073740 2024-03-06T07:27:14.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.675+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9f0801a070 con 0x7f9f180b3ec0 2024-03-06T07:27:14.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.675+0000 7fa806000640 1 -- 172.21.15.86:0/2836735250 <== osd.1 v2:172.21.15.86:6809/2555210107 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7fa7e0002dd0 con 0x7fa7ec0737c0 2024-03-06T07:27:14.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.676+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f8de0073740 msgr2=0x7f8de0075b60 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.676+0000 7f8e07c00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f8de0073740 0x7f8de0075b60 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de006b520 msgr2=0x7f8de006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de006b520 0x7f8de006d9e0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f8dec036000 tx=0x7f8dec008030 comp rx=0 tx=0).stop 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd5b0 msgr2=0x7f8e001f7650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 --2- 172.21.15.86:0/1500057796 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e000cd5b0 0x7f8e001f7650 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f8df400e840 tx=0x7f8df400ed00 comp rx=0 tx=0).stop 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e05c00640 1 -- 172.21.15.86:0/1500057796 reap_dead start 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 shutdown_connections 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 >> 172.21.15.86:0/1500057796 conn(0x7f8e000cc4f0 msgr2=0x7f8e00162af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 shutdown_connections 2024-03-06T07:27:14.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f8e07c00640 1 -- 172.21.15.86:0/1500057796 wait complete. 2024-03-06T07:27:14.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f9f0ea00640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef806b5a0 0x7f9ef806da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9f08089560 con 0x7f9f180b3ec0 2024-03-06T07:27:14.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f9f0ea00640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x7f9ef80737c0 0x7f9ef8075be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:14.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f9f08015e70 con 0x7f9ef80737c0 2024-03-06T07:27:14.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.677+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9f08090050 con 0x7f9f180b3ec0 2024-03-06T07:27:14.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.678+0000 7f9f15e00640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef806b5a0 0x7f9ef806da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fa7ec0737c0 msgr2=0x7fa7ec075be0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7fa7ff400640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fa7ec0737c0 0x7fa7ec075be0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7ec06b5a0 msgr2=0x7fa7ec06da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7fa7ff400640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7ec06b5a0 0x7fa7ec06da60 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fa80802c870 tx=0x7fa808009000 comp rx=0 tx=0).stop 2024-03-06T07:27:14.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810072660 msgr2=0x7fa8100ac340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7fa7ff400640 1 --2- 172.21.15.86:0/2836735250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa810072660 0x7fa8100ac340 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fa7f800c390 tx=0x7fa7f800c850 comp rx=0 tx=0).stop 2024-03-06T07:27:14.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7fa80e800640 1 -- 172.21.15.86:0/2836735250 reap_dead start 2024-03-06T07:27:14.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.679+0000 7f9f15e00640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef806b5a0 0x7f9ef806da60 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f9f0402b9c0 tx=0x7f9f04008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.681+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 shutdown_connections 2024-03-06T07:27:14.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.681+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 >> 172.21.15.86:0/2836735250 conn(0x7fa8100b1730 msgr2=0x7fa8100b0b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.681+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 shutdown_connections 2024-03-06T07:27:14.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.681+0000 7fa7ff400640 1 -- 172.21.15.86:0/2836735250 wait complete. 2024-03-06T07:27:14.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.683+0000 7f9f17200640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x7f9ef80737c0 0x7f9ef8075be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:14.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.686+0000 7f9f17200640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x7f9ef80737c0 0x7f9ef8075be0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:14.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.688+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f0c34002dd0 con 0x7f0c40074400 2024-03-06T07:27:14.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.689+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== osd.7 v2:172.21.15.194:6819/3406102027 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f9f08015e70 con 0x7f9ef80737c0 2024-03-06T07:27:14.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.691+0000 7f0c59600640 1 -- 172.21.15.86:0/1325885917 <== osd.2 v2:172.21.15.86:6808/2264489672 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f0c34002dd0 con 0x7f0c40074400 2024-03-06T07:27:14.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.691+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f0c40074400 msgr2=0x7f0c400768a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.691+0000 7f0c4aa00640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f0c40074400 0x7f0c400768a0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.691+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c4006b6d0 msgr2=0x7f0c4006db90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.691+0000 7f0c4aa00640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c4006b6d0 0x7f0c4006db90 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f0c50000f60 tx=0x7f0c50009040 comp rx=0 tx=0).stop 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.691+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0d0d50 msgr2=0x7f0c5c0cd390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.691+0000 7f0c4aa00640 1 --2- 172.21.15.86:0/1325885917 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c5c0d0d50 0x7f0c5c0cd390 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f0c4c0098c0 tx=0x7f0c4c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.692+0000 7f0c62200640 1 -- 172.21.15.86:0/1325885917 reap_dead start 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.692+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 shutdown_connections 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.692+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 >> 172.21.15.86:0/1325885917 conn(0x7f0c5c09d210 msgr2=0x7f0c5c06c510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.692+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 shutdown_connections 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.692+0000 7f0c4aa00640 1 -- 172.21.15.86:0/1325885917 wait complete. 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.695+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f9e5c0cebe0 con 0x7f9e3c073670 2024-03-06T07:27:14.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.698+0000 7f9e52000640 1 -- 172.21.15.86:0/3268748831 <== osd.3 v2:172.21.15.86:6824/3540554891 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f9e5c0cebe0 con 0x7f9e3c073670 2024-03-06T07:27:14.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.698+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fd188002dd0 con 0x7fd18c0736f0 2024-03-06T07:27:14.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.699+0000 7fd1a8c00640 1 -- 172.21.15.86:0/4237050161 <== osd.5 v2:172.21.15.194:6816/1278478503 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7fd188002dd0 con 0x7fd18c0736f0 2024-03-06T07:27:14.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd18c0736f0 msgr2=0x7fd18c075b10 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7fd1b3a00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd18c0736f0 0x7fd18c075b10 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f9e3c073670 msgr2=0x7f9e3c075a90 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7f9e47400640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f9e3c073670 0x7f9e3c075a90 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd18c06b4d0 msgr2=0x7fd18c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7fd1b3a00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd18c06b4d0 0x7fd18c06d990 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fd198036000 tx=0x7fd19803b040 comp rx=0 tx=0).stop 2024-03-06T07:27:14.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e3c06b4d0 msgr2=0x7f9e3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7f9e47400640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e3c06b4d0 0x7f9e3c06d990 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f9e48000f60 tx=0x7f9e48009040 comp rx=0 tx=0).stop 2024-03-06T07:27:14.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd5b0 msgr2=0x7fd1ac0b4af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7fd1b3a00640 1 --2- 172.21.15.86:0/4237050161 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd1ac0cd5b0 0x7fd1ac0b4af0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fd1a000e8a0 tx=0x7fd1a000ed60 comp rx=0 tx=0).stop 2024-03-06T07:27:14.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.700+0000 7fd1b1a00640 1 -- 172.21.15.86:0/4237050161 reap_dead start 2024-03-06T07:27:14.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.701+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d9c0 msgr2=0x7f9e5c16d910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.701+0000 7f9e47400640 1 --2- 172.21.15.86:0/3268748831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e5c06d9c0 0x7f9e5c16d910 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f9e4c02ba30 tx=0x7f9e4c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.701+0000 7f9e5a800640 1 -- 172.21.15.86:0/3268748831 reap_dead start 2024-03-06T07:27:14.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.701+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 shutdown_connections 2024-03-06T07:27:14.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.701+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 >> 172.21.15.86:0/3268748831 conn(0x7f9e5c0cfa70 msgr2=0x7f9e5c0b1540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.701+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 shutdown_connections 2024-03-06T07:27:14.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.701+0000 7f9e47400640 1 -- 172.21.15.86:0/3268748831 wait complete. 2024-03-06T07:27:14.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.702+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 shutdown_connections 2024-03-06T07:27:14.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.702+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 >> 172.21.15.86:0/4237050161 conn(0x7fd1ac0cc4f0 msgr2=0x7fd1ac0c6360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.702+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 shutdown_connections 2024-03-06T07:27:14.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.702+0000 7fd1b3a00640 1 -- 172.21.15.86:0/4237050161 wait complete. 2024-03-06T07:27:14.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.703+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f77d804f330 con 0x7f77b80737c0 2024-03-06T07:27:14.756 INFO:teuthology.orchestra.run.smithi086.stdout:60129542148 2024-03-06T07:27:14.756 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-03-06T07:27:14.757 INFO:teuthology.orchestra.run.smithi086.stdout:60129542151 2024-03-06T07:27:14.757 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-03-06T07:27:14.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.706+0000 7f77cf400640 1 -- 172.21.15.86:0/2702104554 <== osd.4 v2:172.21.15.194:6800/2403584566 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f77d804f330 con 0x7f77b80737c0 2024-03-06T07:27:14.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.706+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f77b80737c0 msgr2=0x7f77b8075be0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.706+0000 7f77de200640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f77b80737c0 0x7f77b8075be0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.706+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f77b806b5a0 msgr2=0x7f77b806da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.706+0000 7f77de200640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f77b806b5a0 0x7f77b806da60 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f77d8144ef0 tx=0x7f77c8008040 comp rx=0 tx=0).stop 2024-03-06T07:27:14.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.706+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77d80cd5b0 msgr2=0x7f77d8142580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.706+0000 7f77de200640 1 --2- 172.21.15.86:0/2702104554 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77d80cd5b0 0x7f77d8142580 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f77d000ba30 tx=0x7f77d000bef0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.707+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fa468002dd0 con 0x7fa46c073870 2024-03-06T07:27:14.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.708+0000 7f77d7e00640 1 -- 172.21.15.86:0/2702104554 reap_dead start 2024-03-06T07:27:14.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.708+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 shutdown_connections 2024-03-06T07:27:14.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.708+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 >> 172.21.15.86:0/2702104554 conn(0x7f77d80cc4f0 msgr2=0x7f77d80cd990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.708+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 shutdown_connections 2024-03-06T07:27:14.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.708+0000 7f77de200640 1 -- 172.21.15.86:0/2702104554 wait complete. 2024-03-06T07:27:14.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.708+0000 7fa487400640 1 -- 172.21.15.86:0/1306669779 <== osd.6 v2:172.21.15.194:6808/2611339787 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7fa468002dd0 con 0x7fa46c073870 2024-03-06T07:27:14.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fa46c073870 msgr2=0x7fa46c075c90 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fa46c073870 0x7fa46c075c90 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa46c06b6d0 msgr2=0x7fa46c06db90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa46c06b6d0 0x7fa46c06db90 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa4900cad60 tx=0x7fa478009040 comp rx=0 tx=0).stop 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4900b2e50 msgr2=0x7fa4901571c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 --2- 172.21.15.86:0/1306669779 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4900b2e50 0x7fa4901571c0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fa48000c730 tx=0x7fa48000cbf0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48f400640 1 -- 172.21.15.86:0/1306669779 reap_dead start 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 shutdown_connections 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 >> 172.21.15.86:0/1306669779 conn(0x7fa4900d3d30 msgr2=0x7fa49006cec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 shutdown_connections 2024-03-06T07:27:14.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.711+0000 7fa48fe00640 1 -- 172.21.15.86:0/1306669779 wait complete. 2024-03-06T07:27:14.763 INFO:teuthology.orchestra.run.smithi086.stdout:60129542150 2024-03-06T07:27:14.763 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-03-06T07:27:14.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.719+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f9f180d3b60 con 0x7f9ef80737c0 2024-03-06T07:27:14.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.722+0000 7f9f0ea00640 1 -- 172.21.15.86:0/2693054479 <== osd.7 v2:172.21.15.194:6819/3406102027 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f9f180d3b60 con 0x7f9ef80737c0 2024-03-06T07:27:14.764 INFO:teuthology.orchestra.run.smithi086.stdout:60129542149 2024-03-06T07:27:14.765 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-03-06T07:27:14.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.722+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x7f9ef80737c0 msgr2=0x7f9ef8075be0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.722+0000 7f9f1ca00640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x7f9ef80737c0 0x7f9ef8075be0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef806b5a0 msgr2=0x7f9ef806da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef806b5a0 0x7f9ef806da60 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f9f0402b9c0 tx=0x7f9f04008030 comp rx=0 tx=0).stop 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f180b3ec0 msgr2=0x7f9f180ccca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 --2- 172.21.15.86:0/2693054479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f180b3ec0 0x7f9f180ccca0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f9f0800c3e0 tx=0x7f9f0800c8a0 comp rx=0 tx=0).stop 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f17200640 1 -- 172.21.15.86:0/2693054479 reap_dead start 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 shutdown_connections 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 >> 172.21.15.86:0/2693054479 conn(0x7f9f180d3d80 msgr2=0x7f9f180ca650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 shutdown_connections 2024-03-06T07:27:14.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:14.723+0000 7f9f1ca00640 1 -- 172.21.15.86:0/2693054479 wait complete. 2024-03-06T07:27:14.767 INFO:teuthology.orchestra.run.smithi086.stdout:60129542150 2024-03-06T07:27:14.767 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.5 2024-03-06T07:27:14.768 INFO:teuthology.orchestra.run.smithi086.stdout:60129542149 2024-03-06T07:27:14.768 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.6 2024-03-06T07:27:14.769 INFO:teuthology.orchestra.run.smithi086.stdout:60129542149 2024-03-06T07:27:14.769 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.4 2024-03-06T07:27:14.770 INFO:teuthology.orchestra.run.smithi086.stdout:60129542150 2024-03-06T07:27:14.770 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.7 2024-03-06T07:27:15.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.083+0000 7f4ba5000640 1 Processor -- start 2024-03-06T07:27:15.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.083+0000 7f4ba5000640 1 -- start start 2024-03-06T07:27:15.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.084+0000 7ff9a3200640 1 Processor -- start 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.084+0000 7ff9a3200640 1 -- start start 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7ff9a3200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff9a415e200 0x7ff9a40d4850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7ff9a3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a40b2e50 0x7ff9a40d04f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7ff9a3200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff9a415e5e0 0x7ff9a40d0a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a3200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff9a40d2ec0 con 0x7ff9a415e200 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a3200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff9a40b5240 con 0x7ff9a415e5e0 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9a40cf4d0 con 0x7ff9a40b2e50 2024-03-06T07:27:15.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a40b2e50 0x7ff9a40d04f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a1e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff9a415e200 0x7ff9a40d4850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a40b2e50 0x7ff9a40d04f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55088/0 (socket says 172.21.15.86:55088) 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a1400640 1 -- 172.21.15.86:0/828640771 learned_addr learned my addr 172.21.15.86:0/828640771 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7ff9a2800640 1 --2- 172.21.15.86:0/828640771 >> v2:172.21.15.194:3300/0 conn(0x7ff9a415e5e0 0x7ff9a40d0a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.085+0000 7f4ba5000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba00d0c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.085+0000 7f4ba5000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4ba00b2bc0 0x7f4ba00cf800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7f4ba5000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ba00cea40 con 0x7f4ba00d21e0 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7f4ba5000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4ba00ce810 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7f4ba5000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba00d4b70 con 0x7f4ba015e0d0 2024-03-06T07:27:15.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7f4b9fe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4ba00d21e0 0x7f4ba00b07d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59320/0 (socket says 172.21.15.86:59320) 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.086+0000 7f4b9fe00640 1 -- 172.21.15.86:0/2902752121 learned_addr learned my addr 172.21.15.86:0/2902752121 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7f4b9f400640 1 --2- 172.21.15.86:0/2902752121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba00d0c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.087+0000 7f4b9ea00640 1 --2- 172.21.15.86:0/2902752121 >> v2:172.21.15.86:3300/0 conn(0x7f4ba00b2bc0 0x7f4ba00cf800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7f4b9ea00640 1 -- 172.21.15.86:0/2902752121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 msgr2=0x7f4ba00d0c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7f4b9ea00640 1 --2- 172.21.15.86:0/2902752121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba00d0c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7f4b9ea00640 1 -- 172.21.15.86:0/2902752121 >> v1:172.21.15.86:6789/0 conn(0x7f4ba00d21e0 legacy=0x7f4ba00b07d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7f4b9ea00640 1 -- 172.21.15.86:0/2902752121 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ba00ced20 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7ff9a1e00640 1 -- 172.21.15.86:0/828640771 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a40b2e50 msgr2=0x7ff9a40d04f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7ff9a1e00640 1 --2- 172.21.15.86:0/828640771 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a40b2e50 0x7ff9a40d04f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7ff9a1e00640 1 -- 172.21.15.86:0/828640771 >> v2:172.21.15.194:3300/0 conn(0x7ff9a415e5e0 msgr2=0x7ff9a40d0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7ff9a1e00640 1 --2- 172.21.15.86:0/828640771 >> v2:172.21.15.194:3300/0 conn(0x7ff9a415e5e0 0x7ff9a40d0a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7ff9a1e00640 1 -- 172.21.15.86:0/828640771 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9a40b5500 con 0x7ff9a415e200 2024-03-06T07:27:15.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7ff9a1400640 1 --2- 172.21.15.86:0/828640771 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a40b2e50 0x7ff9a40d04f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:15.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.088+0000 7ff9a2800640 1 --2- 172.21.15.86:0/828640771 >> v2:172.21.15.194:3300/0 conn(0x7ff9a415e5e0 0x7ff9a40d0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:15.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.089+0000 7f4b9ea00640 1 --2- 172.21.15.86:0/2902752121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba00cf800 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f4b8c009a30 tx=0x7f4b8c02bb10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4ab1a401fe47c1cb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.090+0000 7ff9a1e00640 1 --2- 172.21.15.86:0/828640771 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a415e200 0x7ff9a40d4850 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7ff9a408d0b0 tx=0x7ff98c02bb20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4886161a14130281 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.090+0000 7f4b9e000640 1 -- 172.21.15.86:0/2902752121 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b8c02d6c0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.090+0000 7ff9a0a00640 1 -- 172.21.15.86:0/828640771 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff98c02d6d0 con 0x7ff9a415e200 2024-03-06T07:27:15.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.090+0000 7f4b9e000640 1 -- 172.21.15.86:0/2902752121 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b8c02d860 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.091+0000 7ff9a0a00640 1 -- 172.21.15.86:0/828640771 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff98c02d870 con 0x7ff9a415e200 2024-03-06T07:27:15.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.091+0000 7f4b9e000640 1 -- 172.21.15.86:0/2902752121 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b8c02d6c0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.091+0000 7f4ba5000640 1 -- 172.21.15.86:0/2902752121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 msgr2=0x7f4ba00cf800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.091+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2902752121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba00cf800 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f4b8c009a30 tx=0x7f4b8c02bb10 comp rx=0 tx=0).stop 2024-03-06T07:27:15.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a0a00640 1 -- 172.21.15.86:0/828640771 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff98c02dc40 con 0x7ff9a415e200 2024-03-06T07:27:15.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 -- 172.21.15.86:0/828640771 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a415e200 msgr2=0x7ff9a40d4850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 --2- 172.21.15.86:0/828640771 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a415e200 0x7ff9a40d4850 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7ff9a408d0b0 tx=0x7ff98c02bb20 comp rx=0 tx=0).stop 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 -- 172.21.15.86:0/828640771 shutdown_connections 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 --2- 172.21.15.86:0/828640771 >> v2:172.21.15.194:3300/0 conn(0x7ff9a415e5e0 0x7ff9a40d0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 --2- 172.21.15.86:0/828640771 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a40b2e50 0x7ff9a40d04f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 --2- 172.21.15.86:0/828640771 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a415e200 0x7ff9a40d4850 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 -- 172.21.15.86:0/828640771 >> 172.21.15.86:0/828640771 conn(0x7ff9a40d3d30 msgr2=0x7ff9a40d4140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 -- 172.21.15.86:0/828640771 shutdown_connections 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7ff9a3200640 1 -- 172.21.15.86:0/828640771 wait complete. 2024-03-06T07:27:15.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7f4ba5000640 1 -- 172.21.15.86:0/2902752121 shutdown_connections 2024-03-06T07:27:15.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2902752121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba00cf800 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2902752121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba00d0c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.092+0000 7f4ba5000640 1 -- 172.21.15.86:0/2902752121 >> 172.21.15.86:0/2902752121 conn(0x7f4ba00d3c00 msgr2=0x7f4ba00d4010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.093+0000 7f4ba5000640 1 -- 172.21.15.86:0/2902752121 shutdown_connections 2024-03-06T07:27:15.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.093+0000 7f4ba5000640 1 -- 172.21.15.86:0/2902752121 wait complete. 2024-03-06T07:27:15.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.095+0000 7f4ba5000640 1 Processor -- start 2024-03-06T07:27:15.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.096+0000 7ff9a3200640 1 Processor -- start 2024-03-06T07:27:15.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.096+0000 7ff9a3200640 1 -- start start 2024-03-06T07:27:15.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.096+0000 7f4ba5000640 1 -- start start 2024-03-06T07:27:15.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7f4ba5000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba015f270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7f4ba5000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ba00d21e0 0x7f4ba015fc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7f4ba5000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba015c6e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7ff9a3200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a40b2e50 0x7ff9a41f6dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7f4ba5000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba00cf2e0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7f4ba5000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba00cfd40 con 0x7f4ba00d21e0 2024-03-06T07:27:15.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7f4ba5000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba00d4720 con 0x7f4ba015e0d0 2024-03-06T07:27:15.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7ff9a3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a415e200 0x7ff9a40c7860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.098+0000 7ff9a1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a40b2e50 0x7ff9a41f6dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba015f270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a40b2e50 0x7ff9a41f6dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54094/0 (socket says 172.21.15.86:54094) 2024-03-06T07:27:15.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a1e00640 1 -- 172.21.15.86:0/2527419116 learned_addr learned my addr 172.21.15.86:0/2527419116 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a3200640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9a415e5e0 0x7ff9a40c7da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba015f270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54080/0 (socket says 172.21.15.86:54080) 2024-03-06T07:27:15.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 -- 172.21.15.86:0/2425175699 learned_addr learned my addr 172.21.15.86:0/2425175699 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9a40cfd60 con 0x7ff9a40b2e50 2024-03-06T07:27:15.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9fe00640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba015c6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9a40cfa80 con 0x7ff9a415e5e0 2024-03-06T07:27:15.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9a40d1c30 con 0x7ff9a415e200 2024-03-06T07:27:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a1400640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a415e200 0x7ff9a40c7860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7ff9a2800640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9a415e5e0 0x7ff9a40c7da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 -- 172.21.15.86:0/2425175699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 msgr2=0x7f4ba015c6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba015c6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 -- 172.21.15.86:0/2425175699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ba00d21e0 msgr2=0x7f4ba015fc70 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-06T07:27:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ba00d21e0 0x7f4ba015fc70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.099+0000 7f4b9f400640 1 -- 172.21.15.86:0/2425175699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ba00c9010 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.100+0000 7ff9a1400640 1 -- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9a415e5e0 msgr2=0x7ff9a40c7da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.100+0000 7ff9a1400640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9a415e5e0 0x7ff9a40c7da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.100+0000 7ff9a1400640 1 -- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a40b2e50 msgr2=0x7ff9a41f6dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.100+0000 7ff9a1400640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a40b2e50 0x7ff9a41f6dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.100+0000 7ff9a1400640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9a40b86a0 con 0x7ff9a415e200 2024-03-06T07:27:15.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.100+0000 7ff9a1e00640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a40b2e50 0x7ff9a41f6dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:15.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.101+0000 7f4b9f400640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba015f270 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f4b9000d620 tx=0x7f4b9000dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.101+0000 7ff9a1400640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a415e200 0x7ff9a40c7860 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7ff9a40d4d90 tx=0x7ff99400b480 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.102+0000 7f4b97400640 1 -- 172.21.15.86:0/2425175699 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b9000b620 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.102+0000 7f4b97400640 1 -- 172.21.15.86:0/2425175699 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b90019070 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.102+0000 7f4b97400640 1 -- 172.21.15.86:0/2425175699 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b9000ba00 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7ff99a000640 1 -- 172.21.15.86:0/2527419116 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff99401a070 con 0x7ff9a415e200 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7ff99a000640 1 -- 172.21.15.86:0/2527419116 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff99400bd10 con 0x7ff9a415e200 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7ff99a000640 1 -- 172.21.15.86:0/2527419116 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff994003bd0 con 0x7ff9a415e200 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff9a40b8980 con 0x7ff9a415e200 2024-03-06T07:27:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff9a40b8f40 con 0x7ff9a415e200 2024-03-06T07:27:15.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7ff425200640 1 Processor -- start 2024-03-06T07:27:15.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ba015e4b0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ba015ea70 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.103+0000 7ff425200640 1 -- start start 2024-03-06T07:27:15.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.105+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff97c005290 con 0x7ff9a415e200 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.105+0000 7ff425200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420155060 0x7ff420140e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.105+0000 7ff425200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff420155440 0x7ff420071c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.105+0000 7ff425200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff42015b7c0 con 0x7ff42015b290 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.105+0000 7ff425200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff420141360 con 0x7ff420155440 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.105+0000 7ff425200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff42015e880 con 0x7ff420155060 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.106+0000 7ff415400640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7ff42015b290 0x7ff42006eb30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:36376/0 (socket says 172.21.15.86:36376) 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.106+0000 7ff415400640 1 -- 172.21.15.86:0/1965365203 learned_addr learned my addr 172.21.15.86:0/1965365203 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.106+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ba015eeb0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.106+0000 7eff39200640 1 Processor -- start 2024-03-06T07:27:15.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.106+0000 7eff39200640 1 -- start start 2024-03-06T07:27:15.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.106+0000 7ff41e800640 1 --2- 172.21.15.86:0/1965365203 >> v2:172.21.15.86:3300/0 conn(0x7ff420155440 0x7ff420071c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.108+0000 7eff39200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340b1650 0x7eff340b1a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.108+0000 7eff39200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7eff340cd6d0 0x7eff340cdb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.108+0000 7eff39200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7eff340b1f70 con 0x7eff34165fc0 2024-03-06T07:27:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.108+0000 7eff39200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7eff3406dbf0 con 0x7eff340cd6d0 2024-03-06T07:27:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.108+0000 7eff39200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff34162ae0 con 0x7eff340b1650 2024-03-06T07:27:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.107+0000 7ff41de00640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420155060 0x7ff420140e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7eff31e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340b1650 0x7eff340b1a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7eff31e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340b1650 0x7eff340b1a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55116/0 (socket says 172.21.15.86:55116) 2024-03-06T07:27:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7eff31e00640 1 -- 172.21.15.86:0/1545063836 learned_addr learned my addr 172.21.15.86:0/1545063836 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2732578837 0 0) 0x7ff42015b7c0 con 0x7ff42015b290 2024-03-06T07:27:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff404003520 con 0x7ff42015b290 2024-03-06T07:27:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7ff99a000640 1 -- 172.21.15.86:0/2527419116 <== mon.2 v2:172.21.15.194:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff994005d10 con 0x7ff9a415e200 2024-03-06T07:27:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.108+0000 7f4b97400640 1 -- 172.21.15.86:0/2425175699 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b9000bba0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7f4b97400640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b8006b4d0 0x7f4b8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.109+0000 7f4b97400640 1 -- 172.21.15.86:0/2425175699 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4b9008a6e0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7eff31e00640 1 -- 172.21.15.86:0/1545063836 >> v2:172.21.15.86:3300/0 conn(0x7eff340cd6d0 msgr2=0x7eff340cdb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7eff31e00640 1 --2- 172.21.15.86:0/1545063836 >> v2:172.21.15.86:3300/0 conn(0x7eff340cd6d0 0x7eff340cdb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7eff31e00640 1 -- 172.21.15.86:0/1545063836 >> v1:172.21.15.86:6789/0 conn(0x7eff34165fc0 legacy=0x7eff34166450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7eff31e00640 1 -- 172.21.15.86:0/1545063836 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff34162dc0 con 0x7eff340b1650 2024-03-06T07:27:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 <== mon.1 v1:172.21.15.194:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3509013618 0 0) 0x7ff404003520 con 0x7ff42015b290 2024-03-06T07:27:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.111+0000 7f31f3e00640 1 Processor -- start 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420155060 msgr2=0x7ff420140e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420155060 0x7ff420140e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 >> v2:172.21.15.86:3300/0 conn(0x7ff420155440 msgr2=0x7ff420071c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 --2- 172.21.15.86:0/1965365203 >> v2:172.21.15.86:3300/0 conn(0x7ff420155440 0x7ff420071c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 --> v1:172.21.15.194:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff42015baa0 con 0x7ff42015b290 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 <== mon.1 v1:172.21.15.194:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff4100019f0 con 0x7ff42015b290 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 >> v1:172.21.15.194:6789/0 conn(0x7ff42015b290 legacy=0x7ff42006eb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4040039b0 0x7ff404003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff404004800 0x7ff404004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4040099b0 0x7ff4040051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff420141360 con 0x7ff4040039b0 2024-03-06T07:27:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff42015e880 con 0x7ff4040099b0 2024-03-06T07:27:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4040147a0 con 0x7ff404004800 2024-03-06T07:27:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.111+0000 7ff415400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff404004800 0x7ff404004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.111+0000 7ff99a000640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff98006b5a0 0x7ff98006da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.110+0000 7f4b9ea00640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b8006b4d0 0x7f4b8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.111+0000 7ff99a000640 1 -- 172.21.15.86:0/2527419116 <== mon.2 v2:172.21.15.194:3301/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff99408a430 con 0x7ff9a415e200 2024-03-06T07:27:15.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.111+0000 7f31f3e00640 1 -- start start 2024-03-06T07:27:15.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.112+0000 7ff9a1e00640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff98006b5a0 0x7ff98006da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.112+0000 7ff415400640 1 -- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4040099b0 msgr2=0x7ff4040051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.112+0000 7ff415400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4040099b0 0x7ff4040051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.112+0000 7ff415400640 1 -- 172.21.15.86:0/1965365203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4040039b0 msgr2=0x7ff404003dd0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-06T07:27:15.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.112+0000 7ff415400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4040039b0 0x7ff404003dd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.112+0000 7ff415400640 1 -- 172.21.15.86:0/1965365203 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff404003520 con 0x7ff404004800 2024-03-06T07:27:15.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.112+0000 7eff31e00640 1 --2- 172.21.15.86:0/1545063836 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340b1650 0x7eff340b1a30 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7eff24009870 tx=0x7eff2402b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3ed477a062a279a4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.113+0000 7f31f3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f31ec0069e0 0x7f31ec006e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.113+0000 7f31f3e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f31ec053720 con 0x7f31ec005e90 2024-03-06T07:27:15.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.113+0000 7f4b9ea00640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b8006b4d0 0x7f4b8006d990 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4b8c036000 tx=0x7f4b8c008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.113+0000 7f31f3e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f31ec053a00 con 0x7f31ec0ead00 2024-03-06T07:27:15.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.113+0000 7f31f3e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f31ec054330 con 0x7f31ec0069e0 2024-03-06T07:27:15.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.113+0000 7ff9a1e00640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff98006b5a0 0x7ff98006da60 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7ff98c02bb60 tx=0x7ff98c045040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.113+0000 7ff415400640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff404004800 0x7ff404004c80 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7ff41000a820 tx=0x7ff404005830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5a0741753c996f69 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.114+0000 7f31f2a00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f31ec0ead00 0x7f31ec04fa90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:36378/0 (socket says 172.21.15.86:36378) 2024-03-06T07:27:15.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.114+0000 7f31f2a00640 1 -- 172.21.15.86:0/2087390571 learned_addr learned my addr 172.21.15.86:0/2087390571 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:15.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.114+0000 7f31f2000640 1 --1- 172.21.15.86:0/2087390571 >> v1:172.21.15.86:6789/0 conn(0x7f31ec005e90 0x7f31ec0062d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59344/0 (socket says 172.21.15.86:59344) 2024-03-06T07:27:15.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.114+0000 7f31f3400640 1 --2- 172.21.15.86:0/2087390571 >> v2:172.21.15.86:3300/0 conn(0x7f31ec0069e0 0x7f31ec006e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.115+0000 7f31f3400640 1 -- 172.21.15.86:0/2087390571 >> v1:172.21.15.194:6789/0 conn(0x7f31ec0ead00 legacy=0x7f31ec04fa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.115+0000 7f31f3400640 1 -- 172.21.15.86:0/2087390571 >> v1:172.21.15.86:6789/0 conn(0x7f31ec005e90 legacy=0x7f31ec0062d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.115+0000 7f31f3400640 1 -- 172.21.15.86:0/2087390571 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f31ec053ce0 con 0x7f31ec0069e0 2024-03-06T07:27:15.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.115+0000 7f9557400640 1 Processor -- start 2024-03-06T07:27:15.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.115+0000 7f9557400640 1 -- start start 2024-03-06T07:27:15.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.116+0000 7f31f3400640 1 --2- 172.21.15.86:0/2087390571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec0069e0 0x7f31ec006e50 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f31f407ac70 tx=0x7f31f40a6ae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=363172fe782433f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f8d89a00640 1 Processor -- start 2024-03-06T07:27:15.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f9557400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f955015e360 0x7f95500cec10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f9557400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f9550161d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f9557400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9550162270 0x7f955015e740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f9557400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f95500cf2b0 con 0x7f9550162270 2024-03-06T07:27:15.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f9557400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f95500cf590 con 0x7f955015e360 2024-03-06T07:27:15.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f9557400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95500cf870 con 0x7f95500d12e0 2024-03-06T07:27:15.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f31f1600640 1 -- 172.21.15.86:0/2087390571 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31f40a8960 con 0x7f31ec0069e0 2024-03-06T07:27:15.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f31f1600640 1 -- 172.21.15.86:0/2087390571 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f31f40a8b00 con 0x7f31ec0069e0 2024-03-06T07:27:15.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f31f1600640 1 -- 172.21.15.86:0/2087390571 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31f40a9470 con 0x7f31ec0069e0 2024-03-06T07:27:15.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f8d89a00640 1 -- start start 2024-03-06T07:27:15.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.117+0000 7f9554a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f955015e360 0x7f95500cec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f4b97400640 1 -- 172.21.15.86:0/2425175699 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b900594c0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9554a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f955015e360 0x7f95500cec10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51924/0 (socket says 172.21.15.86:51924) 2024-03-06T07:27:15.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9554a00640 1 -- 172.21.15.86:0/2316662439 learned_addr learned my addr 172.21.15.86:0/2316662439 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f954fe00640 1 --2- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f9550161d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9555400640 1 --2- 172.21.15.86:0/2316662439 >> v2:172.21.15.86:3300/0 conn(0x7f9550162270 0x7f955015e740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9554a00640 1 -- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 msgr2=0x7f9550161d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9554a00640 1 --2- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f9550161d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9554a00640 1 -- 172.21.15.86:0/2316662439 >> v2:172.21.15.86:3300/0 conn(0x7f9550162270 msgr2=0x7f955015e740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f31f3e00640 1 -- 172.21.15.86:0/2087390571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec0069e0 msgr2=0x7f31ec006e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f31f3e00640 1 --2- 172.21.15.86:0/2087390571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec0069e0 0x7f31ec006e50 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f31f407ac70 tx=0x7f31f40a6ae0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9554a00640 1 --2- 172.21.15.86:0/2316662439 >> v2:172.21.15.86:3300/0 conn(0x7f9550162270 0x7f955015e740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.118+0000 7f9554a00640 1 -- 172.21.15.86:0/2316662439 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95500cfb50 con 0x7f955015e360 2024-03-06T07:27:15.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f9555400640 1 --2- 172.21.15.86:0/2316662439 >> v2:172.21.15.86:3300/0 conn(0x7f9550162270 0x7f955015e740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:15.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f31f3e00640 1 -- 172.21.15.86:0/2087390571 shutdown_connections 2024-03-06T07:27:15.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f31f3e00640 1 --2- 172.21.15.86:0/2087390571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec0069e0 0x7f31ec006e50 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f31f3e00640 1 -- 172.21.15.86:0/2087390571 >> 172.21.15.86:0/2087390571 conn(0x7f31ec009fb0 msgr2=0x7f31ec00a390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f8d89a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d8415f510 0x7f8d840b1410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f8d89a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8d840d2830 0x7f8d840b1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f31f3e00640 1 -- 172.21.15.86:0/2087390571 shutdown_connections 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f8d89a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8d840b32c0 con 0x7f8d840b1e90 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f8d89a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8d8415f8f0 con 0x7f8d840d2830 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f8d89a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8d840aeb60 con 0x7f8d8415f510 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f8d82000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8d840d2830 0x7f8d840b1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f8d83400640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f8d840b1e90 0x7f8d840cfaf0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:36394/0 (socket says 172.21.15.86:36394) 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f8d83400640 1 -- 172.21.15.86:0/1777703707 learned_addr learned my addr 172.21.15.86:0/1777703707 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f8d82a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d8415f510 0x7f8d840b1410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.119+0000 7f31f3e00640 1 -- 172.21.15.86:0/2087390571 wait complete. 2024-03-06T07:27:15.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f16b0a00640 1 Processor -- start 2024-03-06T07:27:15.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f8d82a00640 1 -- 172.21.15.86:0/1777703707 >> v2:172.21.15.194:3300/0 conn(0x7f8d840d2830 msgr2=0x7f8d840b1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f8d82a00640 1 --2- 172.21.15.86:0/1777703707 >> v2:172.21.15.194:3300/0 conn(0x7f8d840d2830 0x7f8d840b1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.120+0000 7f8d82a00640 1 -- 172.21.15.86:0/1777703707 >> v1:172.21.15.194:6789/0 conn(0x7f8d840b1e90 legacy=0x7f8d840cfaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f8d82a00640 1 -- 172.21.15.86:0/1777703707 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d840aee40 con 0x7f8d8415f510 2024-03-06T07:27:15.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f9554a00640 1 --2- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f955015e360 0x7f95500cec10 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f95500d4fc0 tx=0x7f954402b9a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=13dcd8e6c0d53dcd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f16b0a00640 1 -- start start 2024-03-06T07:27:15.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f954f400640 1 -- 172.21.15.86:0/2316662439 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9544034070 con 0x7f955015e360 2024-03-06T07:27:15.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f954f400640 1 -- 172.21.15.86:0/2316662439 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9544030860 con 0x7f955015e360 2024-03-06T07:27:15.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f954f400640 1 -- 172.21.15.86:0/2316662439 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9544030c30 con 0x7f955015e360 2024-03-06T07:27:15.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f31f3e00640 1 Processor -- start 2024-03-06T07:27:15.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.121+0000 7f31f3e00640 1 -- start start 2024-03-06T07:27:15.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f8d82a00640 1 --2- 172.21.15.86:0/1777703707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d8415f510 0x7f8d840b1410 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f8d740097a0 tx=0x7f8d7402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8b27acda86ee20f1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f16b0a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16ac1617d0 0x7f16ac0ceb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f31f3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec005e90 0x7f31ec0dc0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f31f3e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f31ec0069e0 0x7f31ec035640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f31f3e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31ec0ead00 0x7f31ec035b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f31f3e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31ec0547c0 con 0x7f31ec005e90 2024-03-06T07:27:15.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f16b0a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac0b6d90 0x7f16ac0cf090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f31f3e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31ec0501a0 con 0x7f31ec0069e0 2024-03-06T07:27:15.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f31f3e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f31ec054940 con 0x7f31ec0ead00 2024-03-06T07:27:15.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f16b0a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16ac0cf6d0 con 0x7f16ac0d0810 2024-03-06T07:27:15.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f16b0a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f16ac0cff70 con 0x7f16ac1617d0 2024-03-06T07:27:15.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.122+0000 7f16b0a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16ac0cf9b0 con 0x7f16ac0b6d90 2024-03-06T07:27:15.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16ac1617d0 0x7f16ac0ceb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16ac1617d0 0x7f16ac0ceb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54158/0 (socket says 172.21.15.86:54158) 2024-03-06T07:27:15.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9e00640 1 -- 172.21.15.86:0/2326304367 learned_addr learned my addr 172.21.15.86:0/2326304367 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f31f2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec005e90 0x7f31ec0dc0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f31f2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec005e90 0x7f31ec0dc0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54142/0 (socket says 172.21.15.86:54142) 2024-03-06T07:27:15.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9400640 1 --2- 172.21.15.86:0/2326304367 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac0b6d90 0x7f16ac0cf090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9e00640 1 -- 172.21.15.86:0/2326304367 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac0b6d90 msgr2=0x7f16ac0cf090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9e00640 1 --2- 172.21.15.86:0/2326304367 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac0b6d90 0x7f16ac0cf090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9e00640 1 -- 172.21.15.86:0/2326304367 >> v1:172.21.15.194:6789/0 conn(0x7f16ac0d0810 legacy=0x7f16ac0d2160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f9557400640 1 -- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f955015e360 msgr2=0x7f95500cec10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f16a9e00640 1 -- 172.21.15.86:0/2326304367 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16ac0cfc90 con 0x7f16ac1617d0 2024-03-06T07:27:15.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f31f2a00640 1 -- 172.21.15.86:0/840259159 learned_addr learned my addr 172.21.15.86:0/840259159 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f31f2000640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f31ec0069e0 0x7f31ec035640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f31f3400640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31ec0ead00 0x7f31ec035b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.123+0000 7f9557400640 1 --2- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f955015e360 0x7f95500cec10 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f95500d4fc0 tx=0x7f954402b9a0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f9557400640 1 -- 172.21.15.86:0/2316662439 shutdown_connections 2024-03-06T07:27:15.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f9557400640 1 --2- 172.21.15.86:0/2316662439 >> v2:172.21.15.86:3300/0 conn(0x7f9550162270 0x7f955015e740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f9557400640 1 --2- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f9550161d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f9557400640 1 --2- 172.21.15.86:0/2316662439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f955015e360 0x7f95500cec10 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f16a9400640 1 --2- 172.21.15.86:0/2326304367 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac0b6d90 0x7f16ac0cf090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:15.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f9557400640 1 -- 172.21.15.86:0/2316662439 >> 172.21.15.86:0/2316662439 conn(0x7f95500d3f50 msgr2=0x7f95500d4330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f31f2a00640 1 -- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31ec0ead00 msgr2=0x7f31ec035b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f31f2a00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31ec0ead00 0x7f31ec035b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f31f2a00640 1 -- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f31ec0069e0 msgr2=0x7f31ec035640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f31f2a00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f31ec0069e0 0x7f31ec035640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f31f2a00640 1 -- 172.21.15.86:0/840259159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f31ec0486a0 con 0x7f31ec005e90 2024-03-06T07:27:15.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f31f2a00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec005e90 0x7f31ec0dc0d0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f31e400c780 tx=0x7f31e400cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.124+0000 7f9557400640 1 -- 172.21.15.86:0/2316662439 shutdown_connections 2024-03-06T07:27:15.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f9557400640 1 -- 172.21.15.86:0/2316662439 wait complete. 2024-03-06T07:27:15.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f31eaa00640 1 -- 172.21.15.86:0/840259159 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31e4017070 con 0x7f31ec005e90 2024-03-06T07:27:15.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f31eaa00640 1 -- 172.21.15.86:0/840259159 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f31e40058b0 con 0x7f31ec005e90 2024-03-06T07:27:15.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f31eaa00640 1 -- 172.21.15.86:0/840259159 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31e4005d20 con 0x7f31ec005e90 2024-03-06T07:27:15.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f31ec048980 con 0x7f31ec005e90 2024-03-06T07:27:15.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f16a9e00640 1 --2- 172.21.15.86:0/2326304367 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac1617d0 0x7f16ac0ceb50 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f169c009870 tx=0x7f169c02b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=378fd97dabe114b7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31ec048f40 con 0x7f31ec005e90 2024-03-06T07:27:15.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f16a8a00640 1 -- 172.21.15.86:0/2326304367 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f169c0317c0 con 0x7f16ac1617d0 2024-03-06T07:27:15.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f16a8a00640 1 -- 172.21.15.86:0/2326304367 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f169c037070 con 0x7f16ac1617d0 2024-03-06T07:27:15.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.125+0000 7f16a8a00640 1 -- 172.21.15.86:0/2326304367 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f169c031b80 con 0x7f16ac1617d0 2024-03-06T07:27:15.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f16b0a00640 1 -- 172.21.15.86:0/2326304367 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac1617d0 msgr2=0x7f16ac0ceb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f16b0a00640 1 --2- 172.21.15.86:0/2326304367 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac1617d0 0x7f16ac0ceb50 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f169c009870 tx=0x7f169c02b860 comp rx=0 tx=0).stop 2024-03-06T07:27:15.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f16b0a00640 1 -- 172.21.15.86:0/2326304367 shutdown_connections 2024-03-06T07:27:15.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f16b0a00640 1 --2- 172.21.15.86:0/2326304367 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac0b6d90 0x7f16ac0cf090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f16b0a00640 1 --2- 172.21.15.86:0/2326304367 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac1617d0 0x7f16ac0ceb50 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f16b0a00640 1 -- 172.21.15.86:0/2326304367 >> 172.21.15.86:0/2326304367 conn(0x7f16ac09d210 msgr2=0x7f16ac0d3f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f8d81600640 1 -- 172.21.15.86:0/1777703707 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d74035020 con 0x7f8d8415f510 2024-03-06T07:27:15.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7eff30a00640 1 -- 172.21.15.86:0/1545063836 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff24035020 con 0x7eff340b1650 2024-03-06T07:27:15.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7eff30a00640 1 -- 172.21.15.86:0/1545063836 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff24031640 con 0x7eff340b1650 2024-03-06T07:27:15.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7eff30a00640 1 -- 172.21.15.86:0/1545063836 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff2402d040 con 0x7eff340b1650 2024-03-06T07:27:15.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f31ec005390 con 0x7f31ec005e90 2024-03-06T07:27:15.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.127+0000 7ff99a000640 1 -- 172.21.15.86:0/2527419116 <== mon.2 v2:172.21.15.194:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff994058c40 con 0x7ff9a415e200 2024-03-06T07:27:15.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.127+0000 7f9557400640 1 Processor -- start 2024-03-06T07:27:15.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.126+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff41000a410 con 0x7ff404004800 2024-03-06T07:27:15.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.127+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff41002ce50 con 0x7ff404004800 2024-03-06T07:27:15.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.127+0000 7ff41d400640 1 -- 172.21.15.86:0/1965365203 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff410034870 con 0x7ff404004800 2024-03-06T07:27:15.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.127+0000 7f8d81600640 1 -- 172.21.15.86:0/1777703707 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8d74031830 con 0x7f8d8415f510 2024-03-06T07:27:15.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.127+0000 7f9557400640 1 -- start start 2024-03-06T07:27:15.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.127+0000 7f16b0a00640 1 -- 172.21.15.86:0/2326304367 shutdown_connections 2024-03-06T07:27:15.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7f8d81600640 1 -- 172.21.15.86:0/1777703707 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d74031c00 con 0x7f8d8415f510 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7ff425200640 1 -- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff404004800 msgr2=0x7ff404004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7ff425200640 1 --2- 172.21.15.86:0/1965365203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff404004800 0x7ff404004c80 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7ff41000a820 tx=0x7ff404005830 comp rx=0 tx=0).stop 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7ff41e800640 1 -- 172.21.15.86:0/1965365203 reap_dead start 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7ff425200640 1 -- 172.21.15.86:0/1965365203 shutdown_connections 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7ff425200640 1 -- 172.21.15.86:0/1965365203 >> 172.21.15.86:0/1965365203 conn(0x7ff4200d0b90 msgr2=0x7ff4200d0f70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7ff425200640 1 -- 172.21.15.86:0/1965365203 shutdown_connections 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7f16b0a00640 1 -- 172.21.15.86:0/2326304367 wait complete. 2024-03-06T07:27:15.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7f8d89a00640 1 -- 172.21.15.86:0/1777703707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d8415f510 msgr2=0x7f8d840b1410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7f8d89a00640 1 --2- 172.21.15.86:0/1777703707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d8415f510 0x7f8d840b1410 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f8d740097a0 tx=0x7f8d7402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7eff39200640 1 -- 172.21.15.86:0/1545063836 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340b1650 msgr2=0x7eff340b1a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7eff39200640 1 --2- 172.21.15.86:0/1545063836 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340b1650 0x7eff340b1a30 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7eff24009870 tx=0x7eff2402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7f8d89a00640 1 -- 172.21.15.86:0/1777703707 shutdown_connections 2024-03-06T07:27:15.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7f8d89a00640 1 --2- 172.21.15.86:0/1777703707 >> v2:172.21.15.194:3300/0 conn(0x7f8d840d2830 0x7f8d840b1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7f8d89a00640 1 --2- 172.21.15.86:0/1777703707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d8415f510 0x7f8d840b1410 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7f8d89a00640 1 -- 172.21.15.86:0/1777703707 >> 172.21.15.86:0/1777703707 conn(0x7f8d840d1d10 msgr2=0x7f8d840d2120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7f31eaa00640 1 -- 172.21.15.86:0/840259159 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f31e401a070 con 0x7f31ec005e90 2024-03-06T07:27:15.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7eff39200640 1 -- 172.21.15.86:0/1545063836 shutdown_connections 2024-03-06T07:27:15.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7eff39200640 1 --2- 172.21.15.86:0/1545063836 >> v2:172.21.15.86:3300/0 conn(0x7eff340cd6d0 0x7eff340cdb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7eff39200640 1 --2- 172.21.15.86:0/1545063836 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340b1650 0x7eff340b1a30 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7eff39200640 1 -- 172.21.15.86:0/1545063836 >> 172.21.15.86:0/1545063836 conn(0x7eff340cfa70 msgr2=0x7eff340cfe80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.130+0000 7f31eaa00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31d406b5a0 0x7f31d406da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.130+0000 7f31eaa00640 1 -- 172.21.15.86:0/840259159 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f31e40894a0 con 0x7f31ec005e90 2024-03-06T07:27:15.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.130+0000 7f31f2000640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31d406b5a0 0x7f31d406da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.130+0000 7f8d89a00640 1 -- 172.21.15.86:0/1777703707 shutdown_connections 2024-03-06T07:27:15.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.129+0000 7eff39200640 1 -- 172.21.15.86:0/1545063836 shutdown_connections 2024-03-06T07:27:15.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.131+0000 7f31f2000640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31d406b5a0 0x7f31d406da60 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f31ec047940 tx=0x7f31e0000ed0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.128+0000 7ff425200640 1 -- 172.21.15.86:0/1965365203 wait complete. 2024-03-06T07:27:15.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.131+0000 7ff425200640 1 Processor -- start 2024-03-06T07:27:15.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.130+0000 7f16b0a00640 1 Processor -- start 2024-03-06T07:27:15.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.130+0000 7f16b0a00640 1 -- start start 2024-03-06T07:27:15.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f16b0a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16ac0b6d90 0x7f16ac0cbe70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f16b0a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac0d0810 0x7f16ac200ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f16b0a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac1617d0 0x7f16ac0b8710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f9557400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f95500c8710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f9557400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f955015e360 0x7f955015a800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f9557400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9550162270 0x7f95500b24d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f16b0a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16ac0b5350 con 0x7f16ac0d0810 2024-03-06T07:27:15.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f16b0a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16ac161bb0 con 0x7f16ac0b6d90 2024-03-06T07:27:15.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.132+0000 7f16b0a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16ac0b54d0 con 0x7f16ac1617d0 2024-03-06T07:27:15.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f16a9e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16ac0b6d90 0x7f16ac0cbe70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f16a9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac0d0810 0x7f16ac200ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f16a9e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16ac0b6d90 0x7f16ac0cbe70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51942/0 (socket says 172.21.15.86:51942) 2024-03-06T07:27:15.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f16a9e00640 1 -- 172.21.15.86:0/894787707 learned_addr learned my addr 172.21.15.86:0/894787707 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f16aa800640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac1617d0 0x7f16ac0b8710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f9554a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f95500c8710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f9554a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f95500c8710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55174/0 (socket says 172.21.15.86:55174) 2024-03-06T07:27:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f954fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f955015e360 0x7f955015a800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f954fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f955015e360 0x7f955015a800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54174/0 (socket says 172.21.15.86:54174) 2024-03-06T07:27:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f9554a00640 1 -- 172.21.15.86:0/748374981 learned_addr learned my addr 172.21.15.86:0/748374981 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f9555400640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9550162270 0x7f95500b24d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95500d18c0 con 0x7f955015e360 2024-03-06T07:27:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f16a9e00640 1 -- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac1617d0 msgr2=0x7f16ac0b8710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95500d1740 con 0x7f9550162270 2024-03-06T07:27:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7f8d89a00640 1 -- 172.21.15.86:0/1777703707 wait complete. 2024-03-06T07:27:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f16a9e00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac1617d0 0x7f16ac0b8710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f16a9e00640 1 -- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac0d0810 msgr2=0x7f16ac200ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f16a9e00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac0d0810 0x7f16ac200ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f16a9e00640 1 -- 172.21.15.86:0/894787707 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16ac0cb0b0 con 0x7f16ac0b6d90 2024-03-06T07:27:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f16a9400640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac0d0810 0x7f16ac200ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7f16a9e00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16ac0b6d90 0x7f16ac0cbe70 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f169c000c00 tx=0x7f169c0046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.131+0000 7eff39200640 1 -- 172.21.15.86:0/1545063836 wait complete. 2024-03-06T07:27:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.133+0000 7eff39200640 1 Processor -- start 2024-03-06T07:27:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7eff39200640 1 -- start start 2024-03-06T07:27:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7f16a2000640 1 -- 172.21.15.86:0/894787707 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f169c032860 con 0x7f16ac0b6d90 2024-03-06T07:27:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7f16a2000640 1 -- 172.21.15.86:0/894787707 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f169c032a00 con 0x7f16ac0b6d90 2024-03-06T07:27:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7f16a2000640 1 -- 172.21.15.86:0/894787707 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f169c032df0 con 0x7f16ac0b6d90 2024-03-06T07:27:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.134+0000 7ff425200640 1 -- start start 2024-03-06T07:27:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.136+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f955015ec80 con 0x7f95500d12e0 2024-03-06T07:27:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16ac0cb390 con 0x7f16ac0b6d90 2024-03-06T07:27:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16ac0cd2d0 con 0x7f16ac0b6d90 2024-03-06T07:27:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.136+0000 7f954fe00640 1 -- 172.21.15.86:0/748374981 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 msgr2=0x7f95500c8710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.136+0000 7f954fe00640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f95500c8710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.136+0000 7f954fe00640 1 -- 172.21.15.86:0/748374981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9550162270 msgr2=0x7f95500b24d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.136+0000 7f954fe00640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9550162270 0x7f95500b24d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.137+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16ac04f330 con 0x7f16ac0b6d90 2024-03-06T07:27:15.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.137+0000 7f954fe00640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95500b1a30 con 0x7f955015e360 2024-03-06T07:27:15.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.137+0000 7f8d89a00640 1 Processor -- start 2024-03-06T07:27:15.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.137+0000 7f954fe00640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f955015e360 0x7f955015a800 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f953c00e8a0 tx=0x7f953c00ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7ff425200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff404004800 0x7ff42013a630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7ff425200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420140790 0x7ff4201414e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7ff425200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420142e40 0x7ff4200cf6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7ff425200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff42015c060 con 0x7ff404004800 2024-03-06T07:27:15.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7ff425200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4200cc620 con 0x7ff420142e40 2024-03-06T07:27:15.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.135+0000 7ff425200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff42006f240 con 0x7ff420140790 2024-03-06T07:27:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.138+0000 7f954cc00640 1 -- 172.21.15.86:0/748374981 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f953c017070 con 0x7f955015e360 2024-03-06T07:27:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.138+0000 7f954cc00640 1 -- 172.21.15.86:0/748374981 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f953c00c940 con 0x7f955015e360 2024-03-06T07:27:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.138+0000 7f954cc00640 1 -- 172.21.15.86:0/748374981 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f953c00ccf0 con 0x7f955015e360 2024-03-06T07:27:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.138+0000 7eff39200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff340b1650 0x7eff34200730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.138+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95500ca5e0 con 0x7f955015e360 2024-03-06T07:27:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.138+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95500ca8c0 con 0x7f955015e360 2024-03-06T07:27:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7ff41de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff404004800 0x7ff42013a630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7ff41de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff404004800 0x7ff42013a630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54184/0 (socket says 172.21.15.86:54184) 2024-03-06T07:27:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7ff41de00640 1 -- 172.21.15.86:0/2002552653 learned_addr learned my addr 172.21.15.86:0/2002552653 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7ff41de00640 1 -- 172.21.15.86:0/2002552653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420140790 msgr2=0x7ff4201414e0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-06T07:27:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7ff41de00640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420140790 0x7ff4201414e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7ff41de00640 1 -- 172.21.15.86:0/2002552653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420142e40 msgr2=0x7ff4200cf6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7ff41de00640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420142e40 0x7ff4200cf6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.140+0000 7ff41de00640 1 -- 172.21.15.86:0/2002552653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff420141da0 con 0x7ff404004800 2024-03-06T07:27:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.140+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f955004f330 con 0x7f955015e360 2024-03-06T07:27:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7f16a2000640 1 -- 172.21.15.86:0/894787707 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f169c04c060 con 0x7f16ac0b6d90 2024-03-06T07:27:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.140+0000 7f16a2000640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f168c06b5d0 0x7f168c06da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.140+0000 7f16a2000640 1 -- 172.21.15.86:0/894787707 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f169c0abca0 con 0x7f16ac0b6d90 2024-03-06T07:27:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.139+0000 7f31eaa00640 1 -- 172.21.15.86:0/840259159 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f31e4057cb0 con 0x7f31ec005e90 2024-03-06T07:27:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff39200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340cd6d0 0x7eff340ca490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff39200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff34165fc0 0x7eff34071910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff39200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff34145a70 con 0x7eff340b1650 2024-03-06T07:27:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff39200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff34163380 con 0x7eff34165fc0 2024-03-06T07:27:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff39200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff34145bf0 con 0x7eff340cd6d0 2024-03-06T07:27:15.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.140+0000 7f8d89a00640 1 -- start start 2024-03-06T07:27:15.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff31e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff340b1650 0x7eff34200730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7f16a9400640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f168c06b5d0 0x7f168c06da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff31e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff340b1650 0x7eff34200730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54198/0 (socket says 172.21.15.86:54198) 2024-03-06T07:27:15.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff31e00640 1 -- 172.21.15.86:0/1862582796 learned_addr learned my addr 172.21.15.86:0/1862582796 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff31400640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340cd6d0 0x7eff340ca490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.141+0000 7eff32800640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff34165fc0 0x7eff34071910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f954cc00640 1 -- 172.21.15.86:0/748374981 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f953c01a070 con 0x7f955015e360 2024-03-06T07:27:15.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f16a9400640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f168c06b5d0 0x7f168c06da90 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f16ac068500 tx=0x7f1698008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff31e00640 1 -- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340cd6d0 msgr2=0x7eff340ca490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f8d89a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d840b1e90 0x7f8d840c8de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f8d89a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d840d2830 0x7f8d840caa90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f8d89a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d8415f510 0x7f8d840c5840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f8d89a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d840b0fc0 con 0x7f8d840d2830 2024-03-06T07:27:15.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f8d89a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d840b3560 con 0x7f8d8415f510 2024-03-06T07:27:15.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.142+0000 7f8d89a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8d840b1140 con 0x7f8d840b1e90 2024-03-06T07:27:15.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff31e00640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340cd6d0 0x7eff340ca490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff31e00640 1 -- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff34165fc0 msgr2=0x7eff34071910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff31e00640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff34165fc0 0x7eff34071910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff31e00640 1 -- 172.21.15.86:0/1862582796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff3415c770 con 0x7eff340b1650 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff31400640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340cd6d0 0x7eff340ca490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff32800640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff34165fc0 0x7eff34071910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7f8d83400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d8415f510 0x7f8d840c5840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7f8d83400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d8415f510 0x7f8d840c5840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:51982/0 (socket says 172.21.15.86:51982) 2024-03-06T07:27:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7f8d83400640 1 -- 172.21.15.86:0/2509201786 learned_addr learned my addr 172.21.15.86:0/2509201786 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7f954cc00640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f953006b520 0x7f953006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7f954cc00640 1 -- 172.21.15.86:0/748374981 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f953c089ff0 con 0x7f955015e360 2024-03-06T07:27:15.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.143+0000 7eff31e00640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff340b1650 0x7eff34200730 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7eff2402b930 tx=0x7eff240046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f9554a00640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f953006b520 0x7f953006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f8d82a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d840b1e90 0x7f8d840c8de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7ff41de00640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff404004800 0x7ff42013a630 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7ff40800d690 tx=0x7ff40800db50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f8d83400640 1 -- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d840b1e90 msgr2=0x7f8d840c8de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f8d83400640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d840b1e90 0x7f8d840c8de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f8d83400640 1 -- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d840d2830 msgr2=0x7f8d840caa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f8d83400640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d840d2830 0x7f8d840caa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f8d83400640 1 -- 172.21.15.86:0/2509201786 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d840cd080 con 0x7f8d8415f510 2024-03-06T07:27:15.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f8d82a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d840b1e90 0x7f8d840c8de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:15.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.144+0000 7f9554a00640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f953006b520 0x7f953006d9e0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f9544009630 tx=0x7f9544008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.145+0000 7f8d83400640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d8415f510 0x7f8d840c5840 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f8d7c00caa0 tx=0x7f8d7c00cf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.151+0000 7eff2a000640 1 -- 172.21.15.86:0/1862582796 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff2403ac60 con 0x7eff340b1650 2024-03-06T07:27:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.152+0000 7eff2a000640 1 -- 172.21.15.86:0/1862582796 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff2403ae20 con 0x7eff340b1650 2024-03-06T07:27:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.152+0000 7eff2a000640 1 -- 172.21.15.86:0/1862582796 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff24044630 con 0x7eff340b1650 2024-03-06T07:27:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.151+0000 7ff416a00640 1 -- 172.21.15.86:0/2002552653 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff408015070 con 0x7ff404004800 2024-03-06T07:27:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.151+0000 7ff416a00640 1 -- 172.21.15.86:0/2002552653 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff408007920 con 0x7ff404004800 2024-03-06T07:27:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.152+0000 7ff416a00640 1 -- 172.21.15.86:0/2002552653 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff408007d10 con 0x7ff404004800 2024-03-06T07:27:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.152+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff34142e80 con 0x7eff340b1650 2024-03-06T07:27:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.152+0000 7f954cc00640 1 -- 172.21.15.86:0/748374981 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f953c0588b0 con 0x7f955015e360 2024-03-06T07:27:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.152+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff3415bd10 con 0x7eff340b1650 2024-03-06T07:27:15.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.152+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff420142080 con 0x7ff404004800 2024-03-06T07:27:15.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.153+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff420142640 con 0x7ff404004800 2024-03-06T07:27:15.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.153+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7eff0c005290 con 0x7eff340b1650 2024-03-06T07:27:15.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.156+0000 7f16a2000640 1 -- 172.21.15.86:0/894787707 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f169c040070 con 0x7f16ac0b6d90 2024-03-06T07:27:15.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.156+0000 7ff416a00640 1 -- 172.21.15.86:0/2002552653 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff408018070 con 0x7ff404004800 2024-03-06T07:27:15.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.157+0000 7eff2a000640 1 -- 172.21.15.86:0/1862582796 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7eff2404b060 con 0x7eff340b1650 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.158+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff42004f330 con 0x7ff404004800 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.158+0000 7ff416a00640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff3fc06b520 0x7ff3fc06d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.158+0000 7ff416a00640 1 -- 172.21.15.86:0/2002552653 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff408089f50 con 0x7ff404004800 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.159+0000 7f8d7aa00640 1 -- 172.21.15.86:0/2509201786 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d7c017070 con 0x7f8d8415f510 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.159+0000 7f8d7aa00640 1 -- 172.21.15.86:0/2509201786 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8d7c00ac20 con 0x7f8d8415f510 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.159+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8d840c63b0 con 0x7f8d8415f510 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.159+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8d840c7100 con 0x7f8d8415f510 2024-03-06T07:27:15.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.159+0000 7f8d7aa00640 1 -- 172.21.15.86:0/2509201786 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d7c012400 con 0x7f8d8415f510 2024-03-06T07:27:15.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.159+0000 7eff2a000640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff1406b520 0x7eff1406d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.159+0000 7eff2a000640 1 -- 172.21.15.86:0/1862582796 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7eff240ac220 con 0x7eff340b1650 2024-03-06T07:27:15.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.160+0000 7ff415400640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff3fc06b520 0x7ff3fc06d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.161+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8d5c005290 con 0x7f8d8415f510 2024-03-06T07:27:15.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.162+0000 7eff31400640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff1406b520 0x7eff1406d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.163+0000 7eff31400640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff1406b520 0x7eff1406d9e0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7eff34162330 tx=0x7eff20009110 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.164+0000 7ff415400640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff3fc06b520 0x7ff3fc06d9e0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7ff410034000 tx=0x7ff410009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.164+0000 7f8d7aa00640 1 -- 172.21.15.86:0/2509201786 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8d7c01a070 con 0x7f8d8415f510 2024-03-06T07:27:15.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.166+0000 7f8d7aa00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d6406b5a0 0x7f8d6406da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.166+0000 7f8d7aa00640 1 -- 172.21.15.86:0/2509201786 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8d7c089ff0 con 0x7f8d8415f510 2024-03-06T07:27:15.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.167+0000 7f8d82a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d6406b5a0 0x7f8d6406da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.167+0000 7eff2a000640 1 -- 172.21.15.86:0/1862582796 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7eff2403f070 con 0x7eff340b1650 2024-03-06T07:27:15.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.168+0000 7f8d82a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d6406b5a0 0x7f8d6406da60 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f8d74036000 tx=0x7f8d7403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.174+0000 7ff416a00640 1 -- 172.21.15.86:0/2002552653 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff408058eb0 con 0x7ff404004800 2024-03-06T07:27:15.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.179+0000 7f8d7aa00640 1 -- 172.21.15.86:0/2509201786 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8d7c058dc0 con 0x7f8d8415f510 2024-03-06T07:27:15.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.541+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7ff97c005850 con 0x7ff9a415e200 2024-03-06T07:27:15.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.542+0000 7ff99a000640 1 -- 172.21.15.86:0/2527419116 <== mon.2 v2:172.21.15.194:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7ff99405cce0 con 0x7ff9a415e200 2024-03-06T07:27:15.545 INFO:teuthology.orchestra.run.smithi086.stdout:60129542148 2024-03-06T07:27:15.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.545+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff98006b5a0 msgr2=0x7ff98006da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff98006b5a0 0x7ff98006da60 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7ff98c02bb60 tx=0x7ff98c045040 comp rx=0 tx=0).stop 2024-03-06T07:27:15.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a415e200 msgr2=0x7ff9a40c7860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a415e200 0x7ff9a40c7860 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7ff9a40d4d90 tx=0x7ff99400b480 comp rx=0 tx=0).stop 2024-03-06T07:27:15.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 shutdown_connections 2024-03-06T07:27:15.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff98006b5a0 0x7ff98006da60 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9a415e5e0 0x7ff9a40c7da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9a415e200 0x7ff9a40c7860 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 --2- 172.21.15.86:0/2527419116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9a40b2e50 0x7ff9a41f6dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.546+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 >> 172.21.15.86:0/2527419116 conn(0x7ff9a40d3d30 msgr2=0x7ff9a40b47c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.547+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 shutdown_connections 2024-03-06T07:27:15.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.547+0000 7ff9a3200640 1 -- 172.21.15.86:0/2527419116 wait complete. 2024-03-06T07:27:15.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.548+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f4ba004f330 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.549+0000 7f4b97400640 1 -- 172.21.15.86:0/2425175699 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f4b90058dc0 con 0x7f4ba00b2bc0 2024-03-06T07:27:15.551 INFO:teuthology.orchestra.run.smithi086.stdout:60129542151 2024-03-06T07:27:15.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.553+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b8006b4d0 msgr2=0x7f4b8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.553+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b8006b4d0 0x7f4b8006d990 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4b8c036000 tx=0x7f4b8c008030 comp rx=0 tx=0).stop 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.553+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 msgr2=0x7f4ba015f270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.553+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba015f270 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f4b9000d620 tx=0x7f4b9000dae0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 shutdown_connections 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b8006b4d0 0x7f4b8006d990 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ba015e0d0 0x7f4ba015c6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ba00d21e0 0x7f4ba015fc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 --2- 172.21.15.86:0/2425175699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ba00b2bc0 0x7f4ba015f270 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 >> 172.21.15.86:0/2425175699 conn(0x7f4ba00d3c00 msgr2=0x7f4ba00b1270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 shutdown_connections 2024-03-06T07:27:15.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.554+0000 7f4ba5000640 1 -- 172.21.15.86:0/2425175699 wait complete. 2024-03-06T07:27:15.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.559+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 6} v 0) v1 -- 0x7f955008d9b0 con 0x7f955015e360 2024-03-06T07:27:15.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.560+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) v1 -- 0x7f31ec03a980 con 0x7f31ec005e90 2024-03-06T07:27:15.563 INFO:teuthology.orchestra.run.smithi086.stdout:60129542149 2024-03-06T07:27:15.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.560+0000 7f954cc00640 1 -- 172.21.15.86:0/748374981 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 6}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f953c05c950 con 0x7f955015e360 2024-03-06T07:27:15.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.561+0000 7f31eaa00640 1 -- 172.21.15.86:0/840259159 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f31e405bd50 con 0x7f31ec005e90 2024-03-06T07:27:15.566 INFO:teuthology.orchestra.run.smithi086.stdout:60129542150 2024-03-06T07:27:15.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f953006b520 msgr2=0x7f953006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f953006b520 0x7f953006d9e0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f9544009630 tx=0x7f9544008030 comp rx=0 tx=0).stop 2024-03-06T07:27:15.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f955015e360 msgr2=0x7f955015a800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f955015e360 0x7f955015a800 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f953c00e8a0 tx=0x7f953c00ed60 comp rx=0 tx=0).stop 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 shutdown_connections 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f953006b520 0x7f953006d9e0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9550162270 0x7f95500b24d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f955015e360 0x7f955015a800 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 --2- 172.21.15.86:0/748374981 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95500d12e0 0x7f95500c8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.565+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 >> 172.21.15.86:0/748374981 conn(0x7f95500d3f50 msgr2=0x7f955015b0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 shutdown_connections 2024-03-06T07:27:15.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31d406b5a0 msgr2=0x7f31d406da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31d406b5a0 0x7f31d406da60 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f31ec047940 tx=0x7f31e0000ed0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec005e90 msgr2=0x7f31ec0dc0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec005e90 0x7f31ec0dc0d0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f31e400c780 tx=0x7f31e400cc40 comp rx=0 tx=0).stop 2024-03-06T07:27:15.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 shutdown_connections 2024-03-06T07:27:15.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31d406b5a0 0x7f31d406da60 secure :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f31ec047940 tx=0x7f31e0000ed0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31ec0ead00 0x7f31ec035b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f31ec0069e0 0x7f31ec035640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 --2- 172.21.15.86:0/840259159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31ec005e90 0x7f31ec0dc0d0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 >> 172.21.15.86:0/840259159 conn(0x7f31ec009fb0 msgr2=0x7f31ec006dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.566+0000 7f9557400640 1 -- 172.21.15.86:0/748374981 wait complete. 2024-03-06T07:27:15.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.568+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 shutdown_connections 2024-03-06T07:27:15.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.568+0000 7f31f3e00640 1 -- 172.21.15.86:0/840259159 wait complete. 2024-03-06T07:27:15.576 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542148 got 60129542148 for osd.0 2024-03-06T07:27:15.576 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.577+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) v1 -- 0x7f16ac0d15c0 con 0x7f16ac0b6d90 2024-03-06T07:27:15.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.578+0000 7f16a2000640 1 -- 172.21.15.86:0/894787707 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f169c07a660 con 0x7f16ac0b6d90 2024-03-06T07:27:15.581 INFO:teuthology.orchestra.run.smithi086.stdout:60129542149 2024-03-06T07:27:15.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.581+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f168c06b5d0 msgr2=0x7f168c06da90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.581+0000 7f16b0a00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f168c06b5d0 0x7f168c06da90 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f16ac068500 tx=0x7f1698008040 comp rx=0 tx=0).stop 2024-03-06T07:27:15.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.581+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16ac0b6d90 msgr2=0x7f16ac0cbe70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.581+0000 7f16b0a00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16ac0b6d90 0x7f16ac0cbe70 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f169c000c00 tx=0x7f169c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 shutdown_connections 2024-03-06T07:27:15.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f168c06b5d0 0x7f168c06da90 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16ac1617d0 0x7f16ac0b8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16ac0d0810 0x7f16ac200ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 --2- 172.21.15.86:0/894787707 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16ac0b6d90 0x7f16ac0cbe70 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 >> 172.21.15.86:0/894787707 conn(0x7f16ac09d210 msgr2=0x7f16ac06ba90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.586 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542151 got 60129542151 for osd.1 2024-03-06T07:27:15.587 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 shutdown_connections 2024-03-06T07:27:15.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.583+0000 7f16b0a00640 1 -- 172.21.15.86:0/894787707 wait complete. 2024-03-06T07:27:15.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.584+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7eff0c005850 con 0x7eff340b1650 2024-03-06T07:27:15.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.585+0000 7eff2a000640 1 -- 172.21.15.86:0/1862582796 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7eff2407b180 con 0x7eff340b1650 2024-03-06T07:27:15.588 INFO:teuthology.orchestra.run.smithi086.stdout:60129542150 2024-03-06T07:27:15.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.587+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7ff420072e50 con 0x7ff404004800 2024-03-06T07:27:15.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff1406b520 msgr2=0x7eff1406d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff1406b520 0x7eff1406d9e0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7eff34162330 tx=0x7eff20009110 comp rx=0 tx=0).stop 2024-03-06T07:27:15.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff340b1650 msgr2=0x7eff34200730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff340b1650 0x7eff34200730 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7eff2402b930 tx=0x7eff240046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7ff416a00640 1 -- 172.21.15.86:0/2002552653 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7ff4080587b0 con 0x7ff404004800 2024-03-06T07:27:15.591 INFO:teuthology.orchestra.run.smithi086.stdout:60129542149 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 shutdown_connections 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff1406b520 0x7eff1406d9e0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff34165fc0 0x7eff34071910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff340cd6d0 0x7eff340ca490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 --2- 172.21.15.86:0/1862582796 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff340b1650 0x7eff34200730 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 >> 172.21.15.86:0/1862582796 conn(0x7eff340cfa70 msgr2=0x7eff340cfe50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 shutdown_connections 2024-03-06T07:27:15.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.588+0000 7eff39200640 1 -- 172.21.15.86:0/1862582796 wait complete. 2024-03-06T07:27:15.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.590+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff3fc06b520 msgr2=0x7ff3fc06d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.590+0000 7ff425200640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff3fc06b520 0x7ff3fc06d9e0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7ff410034000 tx=0x7ff410009030 comp rx=0 tx=0).stop 2024-03-06T07:27:15.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.590+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff404004800 msgr2=0x7ff42013a630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.590+0000 7ff425200640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff404004800 0x7ff42013a630 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7ff40800d690 tx=0x7ff40800db50 comp rx=0 tx=0).stop 2024-03-06T07:27:15.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.591+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 shutdown_connections 2024-03-06T07:27:15.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.591+0000 7ff425200640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff3fc06b520 0x7ff3fc06d9e0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.591+0000 7ff425200640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420142e40 0x7ff4200cf6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.591+0000 7ff425200640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff420140790 0x7ff4201414e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.591+0000 7ff425200640 1 --2- 172.21.15.86:0/2002552653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff404004800 0x7ff42013a630 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.591+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 >> 172.21.15.86:0/2002552653 conn(0x7ff4200d0b90 msgr2=0x7ff42013b6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.592+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 shutdown_connections 2024-03-06T07:27:15.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.592+0000 7ff425200640 1 -- 172.21.15.86:0/2002552653 wait complete. 2024-03-06T07:27:15.596 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542150 got 60129542150 for osd.5 2024-03-06T07:27:15.597 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.597 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542149 got 60129542149 for osd.6 2024-03-06T07:27:15.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.595+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 7} v 0) v1 -- 0x7f8d5c005580 con 0x7f8d8415f510 2024-03-06T07:27:15.598 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.595+0000 7f8d7aa00640 1 -- 172.21.15.86:0/2509201786 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 7}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f8d7c0586c0 con 0x7f8d8415f510 2024-03-06T07:27:15.598 INFO:teuthology.orchestra.run.smithi086.stdout:60129542150 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d6406b5a0 msgr2=0x7f8d6406da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d6406b5a0 0x7f8d6406da60 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f8d74036000 tx=0x7f8d7403c040 comp rx=0 tx=0).stop 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d8415f510 msgr2=0x7f8d840c5840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d8415f510 0x7f8d840c5840 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f8d7c00caa0 tx=0x7f8d7c00cf60 comp rx=0 tx=0).stop 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 shutdown_connections 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d6406b5a0 0x7f8d6406da60 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d8415f510 0x7f8d840c5840 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d840d2830 0x7f8d840caa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 --2- 172.21.15.86:0/2509201786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d840b1e90 0x7f8d840c8de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 >> 172.21.15.86:0/2509201786 conn(0x7f8d840d1d10 msgr2=0x7f8d8406ca20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 shutdown_connections 2024-03-06T07:27:15.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.598+0000 7f8d89a00640 1 -- 172.21.15.86:0/2509201786 wait complete. 2024-03-06T07:27:15.612 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542149 got 60129542149 for osd.4 2024-03-06T07:27:15.612 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.615 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542150 got 60129542150 for osd.2 2024-03-06T07:27:15.615 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.617 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542149 got 60129542149 for osd.3 2024-03-06T07:27:15.618 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.619 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542150 got 60129542150 for osd.7 2024-03-06T07:27:15.619 DEBUG:teuthology.parallel:result is None 2024-03-06T07:27:15.619 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-03-06T07:27:15.620 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:27:15.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.853+0000 7f63aec00640 1 Processor -- start 2024-03-06T07:27:15.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.853+0000 7f63aec00640 1 -- start start 2024-03-06T07:27:15.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.854+0000 7f63aec00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f63a8144c00 0x7f63a806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.854+0000 7f63aec00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f63a815fe60 0x7f63a80ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63aec00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63a806ee70 con 0x7f63a815f9d0 2024-03-06T07:27:15.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63aec00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63a806cef0 con 0x7f63a815fe60 2024-03-06T07:27:15.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63aec00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63a806d1d0 con 0x7f63a8144c00 2024-03-06T07:27:15.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63acc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f63a815fe60 0x7f63a80ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63acc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f63a815fe60 0x7f63a80ce7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54228/0 (socket says 172.21.15.86:54228) 2024-03-06T07:27:15.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63acc00640 1 -- 172.21.15.86:0/2448718303 learned_addr learned my addr 172.21.15.86:0/2448718303 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63a7e00640 1 --2- 172.21.15.86:0/2448718303 >> v2:172.21.15.194:3300/0 conn(0x7f63a8144c00 0x7f63a806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63acc00640 1 -- 172.21.15.86:0/2448718303 >> v2:172.21.15.194:3300/0 conn(0x7f63a8144c00 msgr2=0x7f63a806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:15.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63acc00640 1 --2- 172.21.15.86:0/2448718303 >> v2:172.21.15.194:3300/0 conn(0x7f63a8144c00 0x7f63a806e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63acc00640 1 -- 172.21.15.86:0/2448718303 >> v1:172.21.15.86:6789/0 conn(0x7f63a815f9d0 legacy=0x7f63a80ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.855+0000 7f63acc00640 1 -- 172.21.15.86:0/2448718303 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63a806d4b0 con 0x7f63a815fe60 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.856+0000 7f63acc00640 1 --2- 172.21.15.86:0/2448718303 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a80ce7b0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f63a80b25f0 tx=0x7f63a002bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f20a39fa32a37aac server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63a7400640 1 -- 172.21.15.86:0/2448718303 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a002cb20 con 0x7f63a815fe60 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63a7400640 1 -- 172.21.15.86:0/2448718303 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f63a002ccc0 con 0x7f63a815fe60 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63a7400640 1 -- 172.21.15.86:0/2448718303 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a0034610 con 0x7f63a815fe60 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 -- 172.21.15.86:0/2448718303 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 msgr2=0x7f63a80ce7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 --2- 172.21.15.86:0/2448718303 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a80ce7b0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f63a80b25f0 tx=0x7f63a002bf80 comp rx=0 tx=0).stop 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 -- 172.21.15.86:0/2448718303 shutdown_connections 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 --2- 172.21.15.86:0/2448718303 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a80ce7b0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 --2- 172.21.15.86:0/2448718303 >> v2:172.21.15.194:3300/0 conn(0x7f63a8144c00 0x7f63a806e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 -- 172.21.15.86:0/2448718303 >> 172.21.15.86:0/2448718303 conn(0x7f63a80b1730 msgr2=0x7f63a80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:15.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 -- 172.21.15.86:0/2448718303 shutdown_connections 2024-03-06T07:27:15.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.857+0000 7f63aec00640 1 -- 172.21.15.86:0/2448718303 wait complete. 2024-03-06T07:27:15.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.859+0000 7f63aec00640 1 Processor -- start 2024-03-06T07:27:15.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.859+0000 7f63aec00640 1 -- start start 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63aec00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63a8144c00 0x7f63a81f82d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63aec00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63a815f9d0 0x7f63a815e690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63aec00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a813cdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63aec00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63a8160730 con 0x7f63a815fe60 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63aec00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63a8160270 con 0x7f63a815f9d0 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63aec00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63a806cb50 con 0x7f63a8144c00 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63acc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a813cdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63acc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a813cdc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54230/0 (socket says 172.21.15.86:54230) 2024-03-06T07:27:15.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.860+0000 7f63acc00640 1 -- 172.21.15.86:0/2354370170 learned_addr learned my addr 172.21.15.86:0/2354370170 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:15.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63acc00640 1 -- 172.21.15.86:0/2354370170 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63a8144c00 msgr2=0x7f63a81f82d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63acc00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63a8144c00 0x7f63a81f82d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63acc00640 1 -- 172.21.15.86:0/2354370170 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63a815f9d0 msgr2=0x7f63a815e690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:15.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f639fe00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63a815f9d0 0x7f63a815e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63acc00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63a815f9d0 0x7f63a815e690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:15.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63acc00640 1 -- 172.21.15.86:0/2354370170 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63a81f47f0 con 0x7f63a815fe60 2024-03-06T07:27:15.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63acc00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a813cdc0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f63a0009f40 tx=0x7f63a0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63a4c00640 1 -- 172.21.15.86:0/2354370170 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a0043020 con 0x7f63a815fe60 2024-03-06T07:27:15.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63a4c00640 1 -- 172.21.15.86:0/2354370170 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f63a003e450 con 0x7f63a815fe60 2024-03-06T07:27:15.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63a4c00640 1 -- 172.21.15.86:0/2354370170 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a0009510 con 0x7f63a815fe60 2024-03-06T07:27:15.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63a81f4ad0 con 0x7f63a815fe60 2024-03-06T07:27:15.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.861+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63a81f5090 con 0x7f63a815fe60 2024-03-06T07:27:15.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.862+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f63a804f330 con 0x7f63a815fe60 2024-03-06T07:27:15.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.863+0000 7f63a4c00640 1 -- 172.21.15.86:0/2354370170 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f63a0046020 con 0x7f63a815fe60 2024-03-06T07:27:15.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.864+0000 7f63a4c00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f638406b4d0 0x7f638406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:15.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.864+0000 7f63a4c00640 1 -- 172.21.15.86:0/2354370170 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f63a00acad0 con 0x7f63a815fe60 2024-03-06T07:27:15.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.864+0000 7f63a7e00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f638406b4d0 0x7f638406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:15.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.865+0000 7f63a7e00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f638406b4d0 0x7f638406d990 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f63980098f0 tx=0x7f6398008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:15.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:15.868+0000 7f63a4c00640 1 -- 172.21.15.86:0/2354370170 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f63a004d030 con 0x7f63a815fe60 2024-03-06T07:27:16.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.053+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f63a8072890 con 0x7f638406b4d0 2024-03-06T07:27:16.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.059+0000 7f63a4c00640 1 -- 172.21.15.86:0/2354370170 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251838 (secure 0 0 0) 0x7f63a8072890 con 0x7f638406b4d0 2024-03-06T07:27:16.066 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:16.067 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:27:16.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f638406b4d0 msgr2=0x7f638406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f638406b4d0 0x7f638406d990 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f63980098f0 tx=0x7f6398008040 comp rx=0 tx=0).stop 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 msgr2=0x7f63a813cdc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a813cdc0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f63a0009f40 tx=0x7f63a0004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 shutdown_connections 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f638406b4d0 0x7f638406d990 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63a815fe60 0x7f63a813cdc0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63a815f9d0 0x7f63a815e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 --2- 172.21.15.86:0/2354370170 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63a8144c00 0x7f63a81f82d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.064+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 >> 172.21.15.86:0/2354370170 conn(0x7f63a80b1730 msgr2=0x7f63a80b0ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.065+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 shutdown_connections 2024-03-06T07:27:16.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.065+0000 7f63aec00640 1 -- 172.21.15.86:0/2354370170 wait complete. 2024-03-06T07:27:16.083 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":23,"stamp":"2024-03-06T07:27:14.853306+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219200,"kb_used_data":2984,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755520,"statfs":{"total":773094113280,"available":772869652480,"internally_reserved":0,"allocated":3055616,"data_stored":1816228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":221236787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":59,"num_write_kb":276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.959180"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225431+0000","last_change":"2024-03-06T07:27:07.923330+0000","last_active":"2024-03-06T07:27:09.225431+0000","last_peered":"2024-03-06T07:27:09.225431+0000","last_clean":"2024-03-06T07:27:09.225431+0000","last_became_active":"2024-03-06T07:27:07.923084+0000","last_became_peered":"2024-03-06T07:27:07.923084+0000","last_unstale":"2024-03-06T07:27:09.225431+0000","last_undegraded":"2024-03-06T07:27:09.225431+0000","last_fullsized":"2024-03-06T07:27:09.225431+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:46:48.027042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918758+0000","last_change":"2024-03-06T07:27:07.921104+0000","last_active":"2024-03-06T07:27:08.918758+0000","last_peered":"2024-03-06T07:27:08.918758+0000","last_clean":"2024-03-06T07:27:08.918758+0000","last_became_active":"2024-03-06T07:27:07.920770+0000","last_became_peered":"2024-03-06T07:27:07.920770+0000","last_unstale":"2024-03-06T07:27:08.918758+0000","last_undegraded":"2024-03-06T07:27:08.918758+0000","last_fullsized":"2024-03-06T07:27:08.918758+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226481+0000","last_change":"2024-03-06T07:27:07.922511+0000","last_active":"2024-03-06T07:27:09.226481+0000","last_peered":"2024-03-06T07:27:09.226481+0000","last_clean":"2024-03-06T07:27:09.226481+0000","last_became_active":"2024-03-06T07:27:07.922242+0000","last_became_peered":"2024-03-06T07:27:07.922242+0000","last_unstale":"2024-03-06T07:27:09.226481+0000","last_undegraded":"2024-03-06T07:27:09.226481+0000","last_fullsized":"2024-03-06T07:27:09.226481+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922737+0000","last_change":"2024-03-06T07:27:07.945609+0000","last_active":"2024-03-06T07:27:08.922737+0000","last_peered":"2024-03-06T07:27:08.922737+0000","last_clean":"2024-03-06T07:27:08.922737+0000","last_became_active":"2024-03-06T07:27:07.945312+0000","last_became_peered":"2024-03-06T07:27:07.945312+0000","last_unstale":"2024-03-06T07:27:08.922737+0000","last_undegraded":"2024-03-06T07:27:08.922737+0000","last_fullsized":"2024-03-06T07:27:08.922737+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:38.769682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224858+0000","last_change":"2024-03-06T07:27:07.921341+0000","last_active":"2024-03-06T07:27:09.224858+0000","last_peered":"2024-03-06T07:27:09.224858+0000","last_clean":"2024-03-06T07:27:09.224858+0000","last_became_active":"2024-03-06T07:27:07.920986+0000","last_became_peered":"2024-03-06T07:27:07.920986+0000","last_unstale":"2024-03-06T07:27:09.224858+0000","last_undegraded":"2024-03-06T07:27:09.224858+0000","last_fullsized":"2024-03-06T07:27:09.224858+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922064+0000","last_change":"2024-03-06T07:27:07.934696+0000","last_active":"2024-03-06T07:27:08.922064+0000","last_peered":"2024-03-06T07:27:08.922064+0000","last_clean":"2024-03-06T07:27:08.922064+0000","last_became_active":"2024-03-06T07:27:07.934400+0000","last_became_peered":"2024-03-06T07:27:07.934400+0000","last_unstale":"2024-03-06T07:27:08.922064+0000","last_undegraded":"2024-03-06T07:27:08.922064+0000","last_fullsized":"2024-03-06T07:27:08.922064+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224278+0000","last_change":"2024-03-06T07:27:07.926117+0000","last_active":"2024-03-06T07:27:09.224278+0000","last_peered":"2024-03-06T07:27:09.224278+0000","last_clean":"2024-03-06T07:27:09.224278+0000","last_became_active":"2024-03-06T07:27:07.925805+0000","last_became_peered":"2024-03-06T07:27:07.925805+0000","last_unstale":"2024-03-06T07:27:09.224278+0000","last_undegraded":"2024-03-06T07:27:09.224278+0000","last_fullsized":"2024-03-06T07:27:09.224278+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223418+0000","last_change":"2024-03-06T07:27:07.946584+0000","last_active":"2024-03-06T07:27:09.223418+0000","last_peered":"2024-03-06T07:27:09.223418+0000","last_clean":"2024-03-06T07:27:09.223418+0000","last_became_active":"2024-03-06T07:27:07.945766+0000","last_became_peered":"2024-03-06T07:27:07.945766+0000","last_unstale":"2024-03-06T07:27:09.223418+0000","last_undegraded":"2024-03-06T07:27:09.223418+0000","last_fullsized":"2024-03-06T07:27:09.223418+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225430+0000","last_change":"2024-03-06T07:27:07.948853+0000","last_active":"2024-03-06T07:27:09.225430+0000","last_peered":"2024-03-06T07:27:09.225430+0000","last_clean":"2024-03-06T07:27:09.225430+0000","last_became_active":"2024-03-06T07:27:07.948523+0000","last_became_peered":"2024-03-06T07:27:07.948523+0000","last_unstale":"2024-03-06T07:27:09.225430+0000","last_undegraded":"2024-03-06T07:27:09.225430+0000","last_fullsized":"2024-03-06T07:27:09.225430+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:17:40.285874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933598+0000","last_change":"2024-03-06T07:27:07.946279+0000","last_active":"2024-03-06T07:27:08.933598+0000","last_peered":"2024-03-06T07:27:08.933598+0000","last_clean":"2024-03-06T07:27:08.933598+0000","last_became_active":"2024-03-06T07:27:07.945895+0000","last_became_peered":"2024-03-06T07:27:07.945895+0000","last_unstale":"2024-03-06T07:27:08.933598+0000","last_undegraded":"2024-03-06T07:27:08.933598+0000","last_fullsized":"2024-03-06T07:27:08.933598+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933628+0000","last_change":"2024-03-06T07:27:07.922684+0000","last_active":"2024-03-06T07:27:08.933628+0000","last_peered":"2024-03-06T07:27:08.933628+0000","last_clean":"2024-03-06T07:27:08.933628+0000","last_became_active":"2024-03-06T07:27:07.922348+0000","last_became_peered":"2024-03-06T07:27:07.922348+0000","last_unstale":"2024-03-06T07:27:08.933628+0000","last_undegraded":"2024-03-06T07:27:08.933628+0000","last_fullsized":"2024-03-06T07:27:08.933628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:38:38.919038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223388+0000","last_change":"2024-03-06T07:27:07.923877+0000","last_active":"2024-03-06T07:27:09.223388+0000","last_peered":"2024-03-06T07:27:09.223388+0000","last_clean":"2024-03-06T07:27:09.223388+0000","last_became_active":"2024-03-06T07:27:07.923410+0000","last_became_peered":"2024-03-06T07:27:07.923410+0000","last_unstale":"2024-03-06T07:27:09.223388+0000","last_undegraded":"2024-03-06T07:27:09.223388+0000","last_fullsized":"2024-03-06T07:27:09.223388+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:30.853873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227024+0000","last_change":"2024-03-06T07:27:07.930873+0000","last_active":"2024-03-06T07:27:09.227024+0000","last_peered":"2024-03-06T07:27:09.227024+0000","last_clean":"2024-03-06T07:27:09.227024+0000","last_became_active":"2024-03-06T07:27:07.930653+0000","last_became_peered":"2024-03-06T07:27:07.930653+0000","last_unstale":"2024-03-06T07:27:09.227024+0000","last_undegraded":"2024-03-06T07:27:09.227024+0000","last_fullsized":"2024-03-06T07:27:09.227024+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224966+0000","last_change":"2024-03-06T07:27:07.919903+0000","last_active":"2024-03-06T07:27:09.224966+0000","last_peered":"2024-03-06T07:27:09.224966+0000","last_clean":"2024-03-06T07:27:09.224966+0000","last_became_active":"2024-03-06T07:27:07.919585+0000","last_became_peered":"2024-03-06T07:27:07.919585+0000","last_unstale":"2024-03-06T07:27:09.224966+0000","last_undegraded":"2024-03-06T07:27:09.224966+0000","last_fullsized":"2024-03-06T07:27:09.224966+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:06.597617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917655+0000","last_change":"2024-03-06T07:27:07.908420+0000","last_active":"2024-03-06T07:27:08.917655+0000","last_peered":"2024-03-06T07:27:08.917655+0000","last_clean":"2024-03-06T07:27:08.917655+0000","last_became_active":"2024-03-06T07:27:07.907392+0000","last_became_peered":"2024-03-06T07:27:07.907392+0000","last_unstale":"2024-03-06T07:27:08.917655+0000","last_undegraded":"2024-03-06T07:27:08.917655+0000","last_fullsized":"2024-03-06T07:27:08.917655+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922133+0000","last_change":"2024-03-06T07:27:07.934890+0000","last_active":"2024-03-06T07:27:08.922133+0000","last_peered":"2024-03-06T07:27:08.922133+0000","last_clean":"2024-03-06T07:27:08.922133+0000","last_became_active":"2024-03-06T07:27:07.934568+0000","last_became_peered":"2024-03-06T07:27:07.934568+0000","last_unstale":"2024-03-06T07:27:08.922133+0000","last_undegraded":"2024-03-06T07:27:08.922133+0000","last_fullsized":"2024-03-06T07:27:08.922133+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923353+0000","last_change":"2024-03-06T07:27:07.927869+0000","last_active":"2024-03-06T07:27:08.923353+0000","last_peered":"2024-03-06T07:27:08.923353+0000","last_clean":"2024-03-06T07:27:08.923353+0000","last_became_active":"2024-03-06T07:27:07.927507+0000","last_became_peered":"2024-03-06T07:27:07.927507+0000","last_unstale":"2024-03-06T07:27:08.923353+0000","last_undegraded":"2024-03-06T07:27:08.923353+0000","last_fullsized":"2024-03-06T07:27:08.923353+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:26:20.914324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224317+0000","last_change":"2024-03-06T07:27:07.922930+0000","last_active":"2024-03-06T07:27:09.224317+0000","last_peered":"2024-03-06T07:27:09.224317+0000","last_clean":"2024-03-06T07:27:09.224317+0000","last_became_active":"2024-03-06T07:27:07.922608+0000","last_became_peered":"2024-03-06T07:27:07.922608+0000","last_unstale":"2024-03-06T07:27:09.224317+0000","last_undegraded":"2024-03-06T07:27:09.224317+0000","last_fullsized":"2024-03-06T07:27:09.224317+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225850+0000","last_change":"2024-03-06T07:27:07.927132+0000","last_active":"2024-03-06T07:27:09.225850+0000","last_peered":"2024-03-06T07:27:09.225850+0000","last_clean":"2024-03-06T07:27:09.225850+0000","last_became_active":"2024-03-06T07:27:07.926850+0000","last_became_peered":"2024-03-06T07:27:07.926850+0000","last_unstale":"2024-03-06T07:27:09.225850+0000","last_undegraded":"2024-03-06T07:27:09.225850+0000","last_fullsized":"2024-03-06T07:27:09.225850+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:23.735936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933468+0000","last_change":"2024-03-06T07:27:07.928126+0000","last_active":"2024-03-06T07:27:08.933468+0000","last_peered":"2024-03-06T07:27:08.933468+0000","last_clean":"2024-03-06T07:27:08.933468+0000","last_became_active":"2024-03-06T07:27:07.927650+0000","last_became_peered":"2024-03-06T07:27:07.927650+0000","last_unstale":"2024-03-06T07:27:08.933468+0000","last_undegraded":"2024-03-06T07:27:08.933468+0000","last_fullsized":"2024-03-06T07:27:08.933468+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:55.755072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225914+0000","last_change":"2024-03-06T07:27:07.927942+0000","last_active":"2024-03-06T07:27:09.225914+0000","last_peered":"2024-03-06T07:27:09.225914+0000","last_clean":"2024-03-06T07:27:09.225914+0000","last_became_active":"2024-03-06T07:27:07.927629+0000","last_became_peered":"2024-03-06T07:27:07.927629+0000","last_unstale":"2024-03-06T07:27:09.225914+0000","last_undegraded":"2024-03-06T07:27:09.225914+0000","last_fullsized":"2024-03-06T07:27:09.225914+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922498+0000","last_change":"2024-03-06T07:27:07.922273+0000","last_active":"2024-03-06T07:27:08.922498+0000","last_peered":"2024-03-06T07:27:08.922498+0000","last_clean":"2024-03-06T07:27:08.922498+0000","last_became_active":"2024-03-06T07:27:07.921988+0000","last_became_peered":"2024-03-06T07:27:07.921988+0000","last_unstale":"2024-03-06T07:27:08.922498+0000","last_undegraded":"2024-03-06T07:27:08.922498+0000","last_fullsized":"2024-03-06T07:27:08.922498+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:57.831357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223829+0000","last_change":"2024-03-06T07:27:07.930778+0000","last_active":"2024-03-06T07:27:09.223829+0000","last_peered":"2024-03-06T07:27:09.223829+0000","last_clean":"2024-03-06T07:27:09.223829+0000","last_became_active":"2024-03-06T07:27:07.930464+0000","last_became_peered":"2024-03-06T07:27:07.930464+0000","last_unstale":"2024-03-06T07:27:09.223829+0000","last_undegraded":"2024-03-06T07:27:09.223829+0000","last_fullsized":"2024-03-06T07:27:09.223829+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922134+0000","last_change":"2024-03-06T07:27:07.917391+0000","last_active":"2024-03-06T07:27:08.922134+0000","last_peered":"2024-03-06T07:27:08.922134+0000","last_clean":"2024-03-06T07:27:08.922134+0000","last_became_active":"2024-03-06T07:27:07.917107+0000","last_became_peered":"2024-03-06T07:27:07.917107+0000","last_unstale":"2024-03-06T07:27:08.922134+0000","last_undegraded":"2024-03-06T07:27:08.922134+0000","last_fullsized":"2024-03-06T07:27:08.922134+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:08.467879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227476+0000","last_change":"2024-03-06T07:27:07.930923+0000","last_active":"2024-03-06T07:27:09.227476+0000","last_peered":"2024-03-06T07:27:09.227476+0000","last_clean":"2024-03-06T07:27:09.227476+0000","last_became_active":"2024-03-06T07:27:07.930078+0000","last_became_peered":"2024-03-06T07:27:07.930078+0000","last_unstale":"2024-03-06T07:27:09.227476+0000","last_undegraded":"2024-03-06T07:27:09.227476+0000","last_fullsized":"2024-03-06T07:27:09.227476+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933695+0000","last_change":"2024-03-06T07:27:07.936206+0000","last_active":"2024-03-06T07:27:08.933695+0000","last_peered":"2024-03-06T07:27:08.933695+0000","last_clean":"2024-03-06T07:27:08.933695+0000","last_became_active":"2024-03-06T07:27:07.930820+0000","last_became_peered":"2024-03-06T07:27:07.930820+0000","last_unstale":"2024-03-06T07:27:08.933695+0000","last_undegraded":"2024-03-06T07:27:08.933695+0000","last_fullsized":"2024-03-06T07:27:08.933695+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917816+0000","last_change":"2024-03-06T07:27:07.918528+0000","last_active":"2024-03-06T07:27:08.917816+0000","last_peered":"2024-03-06T07:27:08.917816+0000","last_clean":"2024-03-06T07:27:08.917816+0000","last_became_active":"2024-03-06T07:27:07.918221+0000","last_became_peered":"2024-03-06T07:27:07.918221+0000","last_unstale":"2024-03-06T07:27:08.917816+0000","last_undegraded":"2024-03-06T07:27:08.917816+0000","last_fullsized":"2024-03-06T07:27:08.917816+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227107+0000","last_change":"2024-03-06T07:27:07.929588+0000","last_active":"2024-03-06T07:27:09.227107+0000","last_peered":"2024-03-06T07:27:09.227107+0000","last_clean":"2024-03-06T07:27:09.227107+0000","last_became_active":"2024-03-06T07:27:07.929207+0000","last_became_peered":"2024-03-06T07:27:07.929207+0000","last_unstale":"2024-03-06T07:27:09.227107+0000","last_undegraded":"2024-03-06T07:27:09.227107+0000","last_fullsized":"2024-03-06T07:27:09.227107+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226074+0000","last_change":"2024-03-06T07:27:07.925047+0000","last_active":"2024-03-06T07:27:09.226074+0000","last_peered":"2024-03-06T07:27:09.226074+0000","last_clean":"2024-03-06T07:27:09.226074+0000","last_became_active":"2024-03-06T07:27:07.924734+0000","last_became_peered":"2024-03-06T07:27:07.924734+0000","last_unstale":"2024-03-06T07:27:09.226074+0000","last_undegraded":"2024-03-06T07:27:09.226074+0000","last_fullsized":"2024-03-06T07:27:09.226074+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922314+0000","last_change":"2024-03-06T07:27:07.927022+0000","last_active":"2024-03-06T07:27:08.922314+0000","last_peered":"2024-03-06T07:27:08.922314+0000","last_clean":"2024-03-06T07:27:08.922314+0000","last_became_active":"2024-03-06T07:27:07.926748+0000","last_became_peered":"2024-03-06T07:27:07.926748+0000","last_unstale":"2024-03-06T07:27:08.922314+0000","last_undegraded":"2024-03-06T07:27:08.922314+0000","last_fullsized":"2024-03-06T07:27:08.922314+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:25.638818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223563+0000","last_change":"2024-03-06T07:27:07.934988+0000","last_active":"2024-03-06T07:27:09.223563+0000","last_peered":"2024-03-06T07:27:09.223563+0000","last_clean":"2024-03-06T07:27:09.223563+0000","last_became_active":"2024-03-06T07:27:07.934606+0000","last_became_peered":"2024-03-06T07:27:07.934606+0000","last_unstale":"2024-03-06T07:27:09.223563+0000","last_undegraded":"2024-03-06T07:27:09.223563+0000","last_fullsized":"2024-03-06T07:27:09.223563+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933081+0000","last_change":"2024-03-06T07:27:07.929352+0000","last_active":"2024-03-06T07:27:08.933081+0000","last_peered":"2024-03-06T07:27:08.933081+0000","last_clean":"2024-03-06T07:27:08.933081+0000","last_became_active":"2024-03-06T07:27:07.929001+0000","last_became_peered":"2024-03-06T07:27:07.929001+0000","last_unstale":"2024-03-06T07:27:08.933081+0000","last_undegraded":"2024-03-06T07:27:08.933081+0000","last_fullsized":"2024-03-06T07:27:08.933081+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:17:25.890956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.222941+0000","last_change":"2024-03-06T07:27:07.906813+0000","last_active":"2024-03-06T07:27:09.222941+0000","last_peered":"2024-03-06T07:27:09.222941+0000","last_clean":"2024-03-06T07:27:09.222941+0000","last_became_active":"2024-03-06T07:27:07.906019+0000","last_became_peered":"2024-03-06T07:27:07.906019+0000","last_unstale":"2024-03-06T07:27:09.222941+0000","last_undegraded":"2024-03-06T07:27:09.222941+0000","last_fullsized":"2024-03-06T07:27:09.222941+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:18.357063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226805+0000","last_change":"2024-03-06T07:27:06.929035+0000","last_active":"2024-03-06T07:27:09.226805+0000","last_peered":"2024-03-06T07:27:09.226805+0000","last_clean":"2024-03-06T07:27:09.226805+0000","last_became_active":"2024-03-06T07:27:06.928702+0000","last_became_peered":"2024-03-06T07:27:06.928702+0000","last_unstale":"2024-03-06T07:27:09.226805+0000","last_undegraded":"2024-03-06T07:27:09.226805+0000","last_fullsized":"2024-03-06T07:27:09.226805+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226233+0000","last_change":"2024-03-06T07:27:07.938823+0000","last_active":"2024-03-06T07:27:09.226233+0000","last_peered":"2024-03-06T07:27:09.226233+0000","last_clean":"2024-03-06T07:27:09.226233+0000","last_became_active":"2024-03-06T07:27:07.937638+0000","last_became_peered":"2024-03-06T07:27:07.937638+0000","last_unstale":"2024-03-06T07:27:09.226233+0000","last_undegraded":"2024-03-06T07:27:09.226233+0000","last_fullsized":"2024-03-06T07:27:09.226233+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932806+0000","last_change":"2024-03-06T07:27:06.929034+0000","last_active":"2024-03-06T07:27:08.932806+0000","last_peered":"2024-03-06T07:27:08.932806+0000","last_clean":"2024-03-06T07:27:08.932806+0000","last_became_active":"2024-03-06T07:27:06.927376+0000","last_became_peered":"2024-03-06T07:27:06.927376+0000","last_unstale":"2024-03-06T07:27:08.932806+0000","last_undegraded":"2024-03-06T07:27:08.932806+0000","last_fullsized":"2024-03-06T07:27:08.932806+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225115+0000","last_change":"2024-03-06T07:27:06.933665+0000","last_active":"2024-03-06T07:27:09.225115+0000","last_peered":"2024-03-06T07:27:09.225115+0000","last_clean":"2024-03-06T07:27:09.225115+0000","last_became_active":"2024-03-06T07:27:06.933367+0000","last_became_peered":"2024-03-06T07:27:06.933367+0000","last_unstale":"2024-03-06T07:27:09.225115+0000","last_undegraded":"2024-03-06T07:27:09.225115+0000","last_fullsized":"2024-03-06T07:27:09.225115+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223616+0000","last_change":"2024-03-06T07:27:07.927629+0000","last_active":"2024-03-06T07:27:09.223616+0000","last_peered":"2024-03-06T07:27:09.223616+0000","last_clean":"2024-03-06T07:27:09.223616+0000","last_became_active":"2024-03-06T07:27:07.925804+0000","last_became_peered":"2024-03-06T07:27:07.925804+0000","last_unstale":"2024-03-06T07:27:09.223616+0000","last_undegraded":"2024-03-06T07:27:09.223616+0000","last_fullsized":"2024-03-06T07:27:09.223616+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922349+0000","last_change":"2024-03-06T07:27:06.935442+0000","last_active":"2024-03-06T07:27:08.922349+0000","last_peered":"2024-03-06T07:27:08.922349+0000","last_clean":"2024-03-06T07:27:08.922349+0000","last_became_active":"2024-03-06T07:27:06.933244+0000","last_became_peered":"2024-03-06T07:27:06.933244+0000","last_unstale":"2024-03-06T07:27:08.922349+0000","last_undegraded":"2024-03-06T07:27:08.922349+0000","last_fullsized":"2024-03-06T07:27:08.922349+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917698+0000","last_change":"2024-03-06T07:27:07.922392+0000","last_active":"2024-03-06T07:27:08.917698+0000","last_peered":"2024-03-06T07:27:08.917698+0000","last_clean":"2024-03-06T07:27:08.917698+0000","last_became_active":"2024-03-06T07:27:07.922127+0000","last_became_peered":"2024-03-06T07:27:07.922127+0000","last_unstale":"2024-03-06T07:27:08.917698+0000","last_undegraded":"2024-03-06T07:27:08.917698+0000","last_fullsized":"2024-03-06T07:27:08.917698+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933693+0000","last_change":"2024-03-06T07:27:06.937146+0000","last_active":"2024-03-06T07:27:08.933693+0000","last_peered":"2024-03-06T07:27:08.933693+0000","last_clean":"2024-03-06T07:27:08.933693+0000","last_became_active":"2024-03-06T07:27:06.936876+0000","last_became_peered":"2024-03-06T07:27:06.936876+0000","last_unstale":"2024-03-06T07:27:08.933693+0000","last_undegraded":"2024-03-06T07:27:08.933693+0000","last_fullsized":"2024-03-06T07:27:08.933693+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.943897+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.943627+0000","last_became_peered":"2024-03-06T07:27:07.943627+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:01:28.762594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923212+0000","last_change":"2024-03-06T07:27:06.940833+0000","last_active":"2024-03-06T07:27:08.923212+0000","last_peered":"2024-03-06T07:27:08.923212+0000","last_clean":"2024-03-06T07:27:08.923212+0000","last_became_active":"2024-03-06T07:27:06.939734+0000","last_became_peered":"2024-03-06T07:27:06.939734+0000","last_unstale":"2024-03-06T07:27:08.923212+0000","last_undegraded":"2024-03-06T07:27:08.923212+0000","last_fullsized":"2024-03-06T07:27:08.923212+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227338+0000","last_change":"2024-03-06T07:27:07.946830+0000","last_active":"2024-03-06T07:27:09.227338+0000","last_peered":"2024-03-06T07:27:09.227338+0000","last_clean":"2024-03-06T07:27:09.227338+0000","last_became_active":"2024-03-06T07:27:07.946495+0000","last_became_peered":"2024-03-06T07:27:07.946495+0000","last_unstale":"2024-03-06T07:27:09.227338+0000","last_undegraded":"2024-03-06T07:27:09.227338+0000","last_fullsized":"2024-03-06T07:27:09.227338+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"21'3","reported_seq":17,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.937692+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.937338+0000","last_became_peered":"2024-03-06T07:27:06.937338+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223197+0000","last_change":"2024-03-06T07:27:07.922246+0000","last_active":"2024-03-06T07:27:09.223197+0000","last_peered":"2024-03-06T07:27:09.223197+0000","last_clean":"2024-03-06T07:27:09.223197+0000","last_became_active":"2024-03-06T07:27:07.921469+0000","last_became_peered":"2024-03-06T07:27:07.921469+0000","last_unstale":"2024-03-06T07:27:09.223197+0000","last_undegraded":"2024-03-06T07:27:09.223197+0000","last_fullsized":"2024-03-06T07:27:09.223197+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"21'6","reported_seq":20,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933200+0000","last_change":"2024-03-06T07:27:06.926732+0000","last_active":"2024-03-06T07:27:08.933200+0000","last_peered":"2024-03-06T07:27:08.933200+0000","last_clean":"2024-03-06T07:27:08.933200+0000","last_became_active":"2024-03-06T07:27:06.925706+0000","last_became_peered":"2024-03-06T07:27:06.925706+0000","last_unstale":"2024-03-06T07:27:08.933200+0000","last_undegraded":"2024-03-06T07:27:08.933200+0000","last_fullsized":"2024-03-06T07:27:08.933200+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225109+0000","last_change":"2024-03-06T07:27:07.935123+0000","last_active":"2024-03-06T07:27:09.225109+0000","last_peered":"2024-03-06T07:27:09.225109+0000","last_clean":"2024-03-06T07:27:09.225109+0000","last_became_active":"2024-03-06T07:27:07.934820+0000","last_became_peered":"2024-03-06T07:27:07.934820+0000","last_unstale":"2024-03-06T07:27:09.225109+0000","last_undegraded":"2024-03-06T07:27:09.225109+0000","last_fullsized":"2024-03-06T07:27:09.225109+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:29.926787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226343+0000","last_change":"2024-03-06T07:27:06.921406+0000","last_active":"2024-03-06T07:27:09.226343+0000","last_peered":"2024-03-06T07:27:09.226343+0000","last_clean":"2024-03-06T07:27:09.226343+0000","last_became_active":"2024-03-06T07:27:06.921070+0000","last_became_peered":"2024-03-06T07:27:06.921070+0000","last_unstale":"2024-03-06T07:27:09.226343+0000","last_undegraded":"2024-03-06T07:27:09.226343+0000","last_fullsized":"2024-03-06T07:27:09.226343+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225286+0000","last_change":"2024-03-06T07:27:07.926884+0000","last_active":"2024-03-06T07:27:09.225286+0000","last_peered":"2024-03-06T07:27:09.225286+0000","last_clean":"2024-03-06T07:27:09.225286+0000","last_became_active":"2024-03-06T07:27:07.926600+0000","last_became_peered":"2024-03-06T07:27:07.926600+0000","last_unstale":"2024-03-06T07:27:09.225286+0000","last_undegraded":"2024-03-06T07:27:09.225286+0000","last_fullsized":"2024-03-06T07:27:09.225286+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922438+0000","last_change":"2024-03-06T07:27:06.936248+0000","last_active":"2024-03-06T07:27:08.922438+0000","last_peered":"2024-03-06T07:27:08.922438+0000","last_clean":"2024-03-06T07:27:08.922438+0000","last_became_active":"2024-03-06T07:27:06.934671+0000","last_became_peered":"2024-03-06T07:27:06.934671+0000","last_unstale":"2024-03-06T07:27:08.922438+0000","last_undegraded":"2024-03-06T07:27:08.922438+0000","last_fullsized":"2024-03-06T07:27:08.922438+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224843+0000","last_change":"2024-03-06T07:27:07.941367+0000","last_active":"2024-03-06T07:27:09.224843+0000","last_peered":"2024-03-06T07:27:09.224843+0000","last_clean":"2024-03-06T07:27:09.224843+0000","last_became_active":"2024-03-06T07:27:07.938238+0000","last_became_peered":"2024-03-06T07:27:07.938238+0000","last_unstale":"2024-03-06T07:27:09.224843+0000","last_undegraded":"2024-03-06T07:27:09.224843+0000","last_fullsized":"2024-03-06T07:27:09.224843+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933197+0000","last_change":"2024-03-06T07:27:06.931453+0000","last_active":"2024-03-06T07:27:08.933197+0000","last_peered":"2024-03-06T07:27:08.933197+0000","last_clean":"2024-03-06T07:27:08.933197+0000","last_became_active":"2024-03-06T07:27:06.931181+0000","last_became_peered":"2024-03-06T07:27:06.931181+0000","last_unstale":"2024-03-06T07:27:08.933197+0000","last_undegraded":"2024-03-06T07:27:08.933197+0000","last_fullsized":"2024-03-06T07:27:08.933197+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933121+0000","last_change":"2024-03-06T07:27:07.932186+0000","last_active":"2024-03-06T07:27:08.933121+0000","last_peered":"2024-03-06T07:27:08.933121+0000","last_clean":"2024-03-06T07:27:08.933121+0000","last_became_active":"2024-03-06T07:27:07.923467+0000","last_became_peered":"2024-03-06T07:27:07.923467+0000","last_unstale":"2024-03-06T07:27:08.933121+0000","last_undegraded":"2024-03-06T07:27:08.933121+0000","last_fullsized":"2024-03-06T07:27:08.933121+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:27:55.371987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"21'8","reported_seq":22,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223314+0000","last_change":"2024-03-06T07:27:06.925620+0000","last_active":"2024-03-06T07:27:09.223314+0000","last_peered":"2024-03-06T07:27:09.223314+0000","last_clean":"2024-03-06T07:27:09.223314+0000","last_became_active":"2024-03-06T07:27:06.925324+0000","last_became_peered":"2024-03-06T07:27:06.925324+0000","last_unstale":"2024-03-06T07:27:09.223314+0000","last_undegraded":"2024-03-06T07:27:09.223314+0000","last_fullsized":"2024-03-06T07:27:09.223314+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224242+0000","last_change":"2024-03-06T07:27:07.935148+0000","last_active":"2024-03-06T07:27:09.224242+0000","last_peered":"2024-03-06T07:27:09.224242+0000","last_clean":"2024-03-06T07:27:09.224242+0000","last_became_active":"2024-03-06T07:27:07.933239+0000","last_became_peered":"2024-03-06T07:27:07.933239+0000","last_unstale":"2024-03-06T07:27:09.224242+0000","last_undegraded":"2024-03-06T07:27:09.224242+0000","last_fullsized":"2024-03-06T07:27:09.224242+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223549+0000","last_change":"2024-03-06T07:27:06.919221+0000","last_active":"2024-03-06T07:27:09.223549+0000","last_peered":"2024-03-06T07:27:09.223549+0000","last_clean":"2024-03-06T07:27:09.223549+0000","last_became_active":"2024-03-06T07:27:06.918433+0000","last_became_peered":"2024-03-06T07:27:06.918433+0000","last_unstale":"2024-03-06T07:27:09.223549+0000","last_undegraded":"2024-03-06T07:27:09.223549+0000","last_fullsized":"2024-03-06T07:27:09.223549+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:21:09.808244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225325+0000","last_change":"2024-03-06T07:27:07.933938+0000","last_active":"2024-03-06T07:27:09.225325+0000","last_peered":"2024-03-06T07:27:09.225325+0000","last_clean":"2024-03-06T07:27:09.225325+0000","last_became_active":"2024-03-06T07:27:07.933656+0000","last_became_peered":"2024-03-06T07:27:07.933656+0000","last_unstale":"2024-03-06T07:27:09.225325+0000","last_undegraded":"2024-03-06T07:27:09.225325+0000","last_fullsized":"2024-03-06T07:27:09.225325+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:23:36.112418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224591+0000","last_change":"2024-03-06T07:27:06.913843+0000","last_active":"2024-03-06T07:27:09.224591+0000","last_peered":"2024-03-06T07:27:09.224591+0000","last_clean":"2024-03-06T07:27:09.224591+0000","last_became_active":"2024-03-06T07:27:06.912393+0000","last_became_peered":"2024-03-06T07:27:06.912393+0000","last_unstale":"2024-03-06T07:27:09.224591+0000","last_undegraded":"2024-03-06T07:27:09.224591+0000","last_fullsized":"2024-03-06T07:27:09.224591+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225673+0000","last_change":"2024-03-06T07:27:07.927885+0000","last_active":"2024-03-06T07:27:09.225673+0000","last_peered":"2024-03-06T07:27:09.225673+0000","last_clean":"2024-03-06T07:27:09.225673+0000","last_became_active":"2024-03-06T07:27:07.927215+0000","last_became_peered":"2024-03-06T07:27:07.927215+0000","last_unstale":"2024-03-06T07:27:09.225673+0000","last_undegraded":"2024-03-06T07:27:09.225673+0000","last_fullsized":"2024-03-06T07:27:09.225673+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:36:30.188524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918041+0000","last_change":"2024-03-06T07:27:06.921613+0000","last_active":"2024-03-06T07:27:08.918041+0000","last_peered":"2024-03-06T07:27:08.918041+0000","last_clean":"2024-03-06T07:27:08.918041+0000","last_became_active":"2024-03-06T07:27:06.921225+0000","last_became_peered":"2024-03-06T07:27:06.921225+0000","last_unstale":"2024-03-06T07:27:08.918041+0000","last_undegraded":"2024-03-06T07:27:08.918041+0000","last_fullsized":"2024-03-06T07:27:08.918041+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922980+0000","last_change":"2024-03-06T07:27:07.924340+0000","last_active":"2024-03-06T07:27:08.922980+0000","last_peered":"2024-03-06T07:27:08.922980+0000","last_clean":"2024-03-06T07:27:08.922980+0000","last_became_active":"2024-03-06T07:27:07.924090+0000","last_became_peered":"2024-03-06T07:27:07.924090+0000","last_unstale":"2024-03-06T07:27:08.922980+0000","last_undegraded":"2024-03-06T07:27:08.922980+0000","last_fullsized":"2024-03-06T07:27:08.922980+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:54.697587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223461+0000","last_change":"2024-03-06T07:27:06.933112+0000","last_active":"2024-03-06T07:27:09.223461+0000","last_peered":"2024-03-06T07:27:09.223461+0000","last_clean":"2024-03-06T07:27:09.223461+0000","last_became_active":"2024-03-06T07:27:06.932764+0000","last_became_peered":"2024-03-06T07:27:06.932764+0000","last_unstale":"2024-03-06T07:27:09.223461+0000","last_undegraded":"2024-03-06T07:27:09.223461+0000","last_fullsized":"2024-03-06T07:27:09.223461+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225383+0000","last_change":"2024-03-06T07:27:07.940694+0000","last_active":"2024-03-06T07:27:09.225383+0000","last_peered":"2024-03-06T07:27:09.225383+0000","last_clean":"2024-03-06T07:27:09.225383+0000","last_became_active":"2024-03-06T07:27:07.940352+0000","last_became_peered":"2024-03-06T07:27:07.940352+0000","last_unstale":"2024-03-06T07:27:09.225383+0000","last_undegraded":"2024-03-06T07:27:09.225383+0000","last_fullsized":"2024-03-06T07:27:09.225383+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:06.241503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":170,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224024+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:27:09.224024+0000","last_peered":"2024-03-06T07:27:09.224024+0000","last_clean":"2024-03-06T07:27:09.224024+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:27:09.224024+0000","last_undegraded":"2024-03-06T07:27:09.224024+0000","last_fullsized":"2024-03-06T07:27:09.224024+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922420+0000","last_change":"2024-03-06T07:27:06.931167+0000","last_active":"2024-03-06T07:27:08.922420+0000","last_peered":"2024-03-06T07:27:08.922420+0000","last_clean":"2024-03-06T07:27:08.922420+0000","last_became_active":"2024-03-06T07:27:06.930895+0000","last_became_peered":"2024-03-06T07:27:06.930895+0000","last_unstale":"2024-03-06T07:27:08.922420+0000","last_undegraded":"2024-03-06T07:27:08.922420+0000","last_fullsized":"2024-03-06T07:27:08.922420+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225706+0000","last_change":"2024-03-06T07:27:07.946419+0000","last_active":"2024-03-06T07:27:09.225706+0000","last_peered":"2024-03-06T07:27:09.225706+0000","last_clean":"2024-03-06T07:27:09.225706+0000","last_became_active":"2024-03-06T07:27:07.945795+0000","last_became_peered":"2024-03-06T07:27:07.945795+0000","last_unstale":"2024-03-06T07:27:09.225706+0000","last_undegraded":"2024-03-06T07:27:09.225706+0000","last_fullsized":"2024-03-06T07:27:09.225706+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224064+0000","last_change":"2024-03-06T07:27:06.936519+0000","last_active":"2024-03-06T07:27:09.224064+0000","last_peered":"2024-03-06T07:27:09.224064+0000","last_clean":"2024-03-06T07:27:09.224064+0000","last_became_active":"2024-03-06T07:27:06.934384+0000","last_became_peered":"2024-03-06T07:27:06.934384+0000","last_unstale":"2024-03-06T07:27:09.224064+0000","last_undegraded":"2024-03-06T07:27:09.224064+0000","last_fullsized":"2024-03-06T07:27:09.224064+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:29.611416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224613+0000","last_change":"2024-03-06T07:27:07.926848+0000","last_active":"2024-03-06T07:27:09.224613+0000","last_peered":"2024-03-06T07:27:09.224613+0000","last_clean":"2024-03-06T07:27:09.224613+0000","last_became_active":"2024-03-06T07:27:07.926255+0000","last_became_peered":"2024-03-06T07:27:07.926255+0000","last_unstale":"2024-03-06T07:27:09.224613+0000","last_undegraded":"2024-03-06T07:27:09.224613+0000","last_fullsized":"2024-03-06T07:27:09.224613+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:43:44.555041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223817+0000","last_change":"2024-03-06T07:27:06.912373+0000","last_active":"2024-03-06T07:27:09.223817+0000","last_peered":"2024-03-06T07:27:09.223817+0000","last_clean":"2024-03-06T07:27:09.223817+0000","last_became_active":"2024-03-06T07:27:06.911823+0000","last_became_peered":"2024-03-06T07:27:06.911823+0000","last_unstale":"2024-03-06T07:27:09.223817+0000","last_undegraded":"2024-03-06T07:27:09.223817+0000","last_fullsized":"2024-03-06T07:27:09.223817+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917936+0000","last_change":"2024-03-06T07:27:07.917473+0000","last_active":"2024-03-06T07:27:08.917936+0000","last_peered":"2024-03-06T07:27:08.917936+0000","last_clean":"2024-03-06T07:27:08.917936+0000","last_became_active":"2024-03-06T07:27:07.914951+0000","last_became_peered":"2024-03-06T07:27:07.914951+0000","last_unstale":"2024-03-06T07:27:08.917936+0000","last_undegraded":"2024-03-06T07:27:08.917936+0000","last_fullsized":"2024-03-06T07:27:08.917936+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922608+0000","last_change":"2024-03-06T07:27:06.928866+0000","last_active":"2024-03-06T07:27:08.922608+0000","last_peered":"2024-03-06T07:27:08.922608+0000","last_clean":"2024-03-06T07:27:08.922608+0000","last_became_active":"2024-03-06T07:27:06.928456+0000","last_became_peered":"2024-03-06T07:27:06.928456+0000","last_unstale":"2024-03-06T07:27:08.922608+0000","last_undegraded":"2024-03-06T07:27:08.922608+0000","last_fullsized":"2024-03-06T07:27:08.922608+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223508+0000","last_change":"2024-03-06T07:27:07.933229+0000","last_active":"2024-03-06T07:27:09.223508+0000","last_peered":"2024-03-06T07:27:09.223508+0000","last_clean":"2024-03-06T07:27:09.223508+0000","last_became_active":"2024-03-06T07:27:07.932737+0000","last_became_peered":"2024-03-06T07:27:07.932737+0000","last_unstale":"2024-03-06T07:27:09.223508+0000","last_undegraded":"2024-03-06T07:27:09.223508+0000","last_fullsized":"2024-03-06T07:27:09.223508+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:20.889857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918205+0000","last_change":"2024-03-06T07:27:06.918955+0000","last_active":"2024-03-06T07:27:08.918205+0000","last_peered":"2024-03-06T07:27:08.918205+0000","last_clean":"2024-03-06T07:27:08.918205+0000","last_became_active":"2024-03-06T07:27:06.918655+0000","last_became_peered":"2024-03-06T07:27:06.918655+0000","last_unstale":"2024-03-06T07:27:08.918205+0000","last_undegraded":"2024-03-06T07:27:08.918205+0000","last_fullsized":"2024-03-06T07:27:08.918205+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922887+0000","last_change":"2024-03-06T07:27:07.924920+0000","last_active":"2024-03-06T07:27:08.922887+0000","last_peered":"2024-03-06T07:27:08.922887+0000","last_clean":"2024-03-06T07:27:08.922887+0000","last_became_active":"2024-03-06T07:27:07.924624+0000","last_became_peered":"2024-03-06T07:27:07.924624+0000","last_unstale":"2024-03-06T07:27:08.922887+0000","last_undegraded":"2024-03-06T07:27:08.922887+0000","last_fullsized":"2024-03-06T07:27:08.922887+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:59:43.062703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225644+0000","last_change":"2024-03-06T07:27:06.936617+0000","last_active":"2024-03-06T07:27:09.225644+0000","last_peered":"2024-03-06T07:27:09.225644+0000","last_clean":"2024-03-06T07:27:09.225644+0000","last_became_active":"2024-03-06T07:27:06.936328+0000","last_became_peered":"2024-03-06T07:27:06.936328+0000","last_unstale":"2024-03-06T07:27:09.225644+0000","last_undegraded":"2024-03-06T07:27:09.225644+0000","last_fullsized":"2024-03-06T07:27:09.225644+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:42:10.164751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918287+0000","last_change":"2024-03-06T07:27:07.916912+0000","last_active":"2024-03-06T07:27:08.918287+0000","last_peered":"2024-03-06T07:27:08.918287+0000","last_clean":"2024-03-06T07:27:08.918287+0000","last_became_active":"2024-03-06T07:27:07.916557+0000","last_became_peered":"2024-03-06T07:27:07.916557+0000","last_unstale":"2024-03-06T07:27:08.918287+0000","last_undegraded":"2024-03-06T07:27:08.918287+0000","last_fullsized":"2024-03-06T07:27:08.918287+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"22'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.931761+0000","last_change":"2024-03-06T07:27:06.934214+0000","last_active":"2024-03-06T07:27:08.931761+0000","last_peered":"2024-03-06T07:27:08.931761+0000","last_clean":"2024-03-06T07:27:08.931761+0000","last_became_active":"2024-03-06T07:27:06.933984+0000","last_became_peered":"2024-03-06T07:27:06.933984+0000","last_unstale":"2024-03-06T07:27:08.931761+0000","last_undegraded":"2024-03-06T07:27:08.931761+0000","last_fullsized":"2024-03-06T07:27:08.931761+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227628+0000","last_change":"2024-03-06T07:27:07.934955+0000","last_active":"2024-03-06T07:27:09.227628+0000","last_peered":"2024-03-06T07:27:09.227628+0000","last_clean":"2024-03-06T07:27:09.227628+0000","last_became_active":"2024-03-06T07:27:07.934602+0000","last_became_peered":"2024-03-06T07:27:07.934602+0000","last_unstale":"2024-03-06T07:27:09.227628+0000","last_undegraded":"2024-03-06T07:27:09.227628+0000","last_fullsized":"2024-03-06T07:27:09.227628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933574+0000","last_change":"2024-03-06T07:27:06.943288+0000","last_active":"2024-03-06T07:27:08.933574+0000","last_peered":"2024-03-06T07:27:08.933574+0000","last_clean":"2024-03-06T07:27:08.933574+0000","last_became_active":"2024-03-06T07:27:06.942813+0000","last_became_peered":"2024-03-06T07:27:06.942813+0000","last_unstale":"2024-03-06T07:27:08.933574+0000","last_undegraded":"2024-03-06T07:27:08.933574+0000","last_fullsized":"2024-03-06T07:27:08.933574+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226729+0000","last_change":"2024-03-06T07:27:07.920024+0000","last_active":"2024-03-06T07:27:09.226729+0000","last_peered":"2024-03-06T07:27:09.226729+0000","last_clean":"2024-03-06T07:27:09.226729+0000","last_became_active":"2024-03-06T07:27:07.919750+0000","last_became_peered":"2024-03-06T07:27:07.919750+0000","last_unstale":"2024-03-06T07:27:09.226729+0000","last_undegraded":"2024-03-06T07:27:09.226729+0000","last_fullsized":"2024-03-06T07:27:09.226729+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223933+0000","last_change":"2024-03-06T07:27:06.936885+0000","last_active":"2024-03-06T07:27:09.223933+0000","last_peered":"2024-03-06T07:27:09.223933+0000","last_clean":"2024-03-06T07:27:09.223933+0000","last_became_active":"2024-03-06T07:27:06.934829+0000","last_became_peered":"2024-03-06T07:27:06.934829+0000","last_unstale":"2024-03-06T07:27:09.223933+0000","last_undegraded":"2024-03-06T07:27:09.223933+0000","last_fullsized":"2024-03-06T07:27:09.223933+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223030+0000","last_change":"2024-03-06T07:27:07.916276+0000","last_active":"2024-03-06T07:27:09.223030+0000","last_peered":"2024-03-06T07:27:09.223030+0000","last_clean":"2024-03-06T07:27:09.223030+0000","last_became_active":"2024-03-06T07:27:07.915974+0000","last_became_peered":"2024-03-06T07:27:07.915974+0000","last_unstale":"2024-03-06T07:27:09.223030+0000","last_undegraded":"2024-03-06T07:27:09.223030+0000","last_fullsized":"2024-03-06T07:27:09.223030+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.935659+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.935373+0000","last_became_peered":"2024-03-06T07:27:06.935373+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223769+0000","last_change":"2024-03-06T07:27:07.910767+0000","last_active":"2024-03-06T07:27:09.223769+0000","last_peered":"2024-03-06T07:27:09.223769+0000","last_clean":"2024-03-06T07:27:09.223769+0000","last_became_active":"2024-03-06T07:27:07.910268+0000","last_became_peered":"2024-03-06T07:27:07.910268+0000","last_unstale":"2024-03-06T07:27:09.223769+0000","last_undegraded":"2024-03-06T07:27:09.223769+0000","last_fullsized":"2024-03-06T07:27:09.223769+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917633+0000","last_change":"2024-03-06T07:27:06.923056+0000","last_active":"2024-03-06T07:27:08.917633+0000","last_peered":"2024-03-06T07:27:08.917633+0000","last_clean":"2024-03-06T07:27:08.917633+0000","last_became_active":"2024-03-06T07:27:06.922710+0000","last_became_peered":"2024-03-06T07:27:06.922710+0000","last_unstale":"2024-03-06T07:27:08.917633+0000","last_undegraded":"2024-03-06T07:27:08.917633+0000","last_fullsized":"2024-03-06T07:27:08.917633+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226527+0000","last_change":"2024-03-06T07:27:07.922445+0000","last_active":"2024-03-06T07:27:09.226527+0000","last_peered":"2024-03-06T07:27:09.226527+0000","last_clean":"2024-03-06T07:27:09.226527+0000","last_became_active":"2024-03-06T07:27:07.922147+0000","last_became_peered":"2024-03-06T07:27:07.922147+0000","last_unstale":"2024-03-06T07:27:09.226527+0000","last_undegraded":"2024-03-06T07:27:09.226527+0000","last_fullsized":"2024-03-06T07:27:09.226527+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224979+0000","last_change":"2024-03-06T07:27:06.940633+0000","last_active":"2024-03-06T07:27:09.224979+0000","last_peered":"2024-03-06T07:27:09.224979+0000","last_clean":"2024-03-06T07:27:09.224979+0000","last_became_active":"2024-03-06T07:27:06.937434+0000","last_became_peered":"2024-03-06T07:27:06.937434+0000","last_unstale":"2024-03-06T07:27:09.224979+0000","last_undegraded":"2024-03-06T07:27:09.224979+0000","last_fullsized":"2024-03-06T07:27:09.224979+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932876+0000","last_change":"2024-03-06T07:27:07.942081+0000","last_active":"2024-03-06T07:27:08.932876+0000","last_peered":"2024-03-06T07:27:08.932876+0000","last_clean":"2024-03-06T07:27:08.932876+0000","last_became_active":"2024-03-06T07:27:07.941647+0000","last_became_peered":"2024-03-06T07:27:07.941647+0000","last_unstale":"2024-03-06T07:27:08.932876+0000","last_undegraded":"2024-03-06T07:27:08.932876+0000","last_fullsized":"2024-03-06T07:27:08.932876+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:47:21.103970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922929+0000","last_change":"2024-03-06T07:27:07.944160+0000","last_active":"2024-03-06T07:27:08.922929+0000","last_peered":"2024-03-06T07:27:08.922929+0000","last_clean":"2024-03-06T07:27:08.922929+0000","last_became_active":"2024-03-06T07:27:07.943238+0000","last_became_peered":"2024-03-06T07:27:07.943238+0000","last_unstale":"2024-03-06T07:27:08.922929+0000","last_undegraded":"2024-03-06T07:27:08.922929+0000","last_fullsized":"2024-03-06T07:27:08.922929+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:51.931654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224835+0000","last_change":"2024-03-06T07:27:06.940899+0000","last_active":"2024-03-06T07:27:09.224835+0000","last_peered":"2024-03-06T07:27:09.224835+0000","last_clean":"2024-03-06T07:27:09.224835+0000","last_became_active":"2024-03-06T07:27:06.940627+0000","last_became_peered":"2024-03-06T07:27:06.940627+0000","last_unstale":"2024-03-06T07:27:09.224835+0000","last_undegraded":"2024-03-06T07:27:09.224835+0000","last_fullsized":"2024-03-06T07:27:09.224835+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.933712+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.933437+0000","last_became_peered":"2024-03-06T07:27:07.933437+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:18.550912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225416+0000","last_change":"2024-03-06T07:27:06.938164+0000","last_active":"2024-03-06T07:27:09.225416+0000","last_peered":"2024-03-06T07:27:09.225416+0000","last_clean":"2024-03-06T07:27:09.225416+0000","last_became_active":"2024-03-06T07:27:06.937669+0000","last_became_peered":"2024-03-06T07:27:06.937669+0000","last_unstale":"2024-03-06T07:27:09.225416+0000","last_undegraded":"2024-03-06T07:27:09.225416+0000","last_fullsized":"2024-03-06T07:27:09.225416+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:19.647400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226963+0000","last_change":"2024-03-06T07:27:07.948380+0000","last_active":"2024-03-06T07:27:09.226963+0000","last_peered":"2024-03-06T07:27:09.226963+0000","last_clean":"2024-03-06T07:27:09.226963+0000","last_became_active":"2024-03-06T07:27:07.948110+0000","last_became_peered":"2024-03-06T07:27:07.948110+0000","last_unstale":"2024-03-06T07:27:09.226963+0000","last_undegraded":"2024-03-06T07:27:09.226963+0000","last_fullsized":"2024-03-06T07:27:09.226963+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227256+0000","last_change":"2024-03-06T07:27:06.911519+0000","last_active":"2024-03-06T07:27:09.227256+0000","last_peered":"2024-03-06T07:27:09.227256+0000","last_clean":"2024-03-06T07:27:09.227256+0000","last_became_active":"2024-03-06T07:27:06.911051+0000","last_became_peered":"2024-03-06T07:27:06.911051+0000","last_unstale":"2024-03-06T07:27:09.227256+0000","last_undegraded":"2024-03-06T07:27:09.227256+0000","last_fullsized":"2024-03-06T07:27:09.227256+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227138+0000","last_change":"2024-03-06T07:27:07.937255+0000","last_active":"2024-03-06T07:27:09.227138+0000","last_peered":"2024-03-06T07:27:09.227138+0000","last_clean":"2024-03-06T07:27:09.227138+0000","last_became_active":"2024-03-06T07:27:07.936713+0000","last_became_peered":"2024-03-06T07:27:07.936713+0000","last_unstale":"2024-03-06T07:27:09.227138+0000","last_undegraded":"2024-03-06T07:27:09.227138+0000","last_fullsized":"2024-03-06T07:27:09.227138+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"21'4","reported_seq":18,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227467+0000","last_change":"2024-03-06T07:27:06.935678+0000","last_active":"2024-03-06T07:27:09.227467+0000","last_peered":"2024-03-06T07:27:09.227467+0000","last_clean":"2024-03-06T07:27:09.227467+0000","last_became_active":"2024-03-06T07:27:06.935349+0000","last_became_peered":"2024-03-06T07:27:06.935349+0000","last_unstale":"2024-03-06T07:27:09.227467+0000","last_undegraded":"2024-03-06T07:27:09.227467+0000","last_fullsized":"2024-03-06T07:27:09.227467+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542150,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27836,"kb_used_data":804,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344004,"statfs":{"total":96636764160,"available":96608260096,"internally_reserved":0,"allocated":823296,"data_stored":669784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542149,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542150,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27244,"kb_used_data":224,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344596,"statfs":{"total":96636764160,"available":96608866304,"internally_reserved":0,"allocated":229376,"data_stored":78924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542149,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":78960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542149,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542150,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27248,"kb_used_data":220,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344592,"statfs":{"total":96636764160,"available":96608862208,"internally_reserved":0,"allocated":225280,"data_stored":78834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542151,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27256,"kb_used_data":228,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344584,"statfs":{"total":96636764160,"available":96608854016,"internally_reserved":0,"allocated":233472,"data_stored":78994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542148,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27844,"kb_used_data":812,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343996,"statfs":{"total":96636764160,"available":96608251904,"internally_reserved":0,"allocated":831488,"data_stored":669840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:27:16.087 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:27:16.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.326+0000 7f5b1f200640 1 Processor -- start 2024-03-06T07:27:16.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.327+0000 7f5b1f200640 1 -- start start 2024-03-06T07:27:16.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.328+0000 7f5b1f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5b2015a940 0x7f5b200b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.328+0000 7f5b1f200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5b200cc2a0 0x7f5b200cc720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.328+0000 7f5b1f200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5b200b2b30 con 0x7f5b200cf370 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.328+0000 7f5b1f200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5b200b28e0 con 0x7f5b2015a940 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.328+0000 7f5b1f200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5b200b2e10 con 0x7f5b200cc2a0 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5b2015a940 0x7f5b200b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5b2015a940 0x7f5b200b45e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34360/0 (socket says 172.21.15.86:34360) 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1de00640 1 -- 172.21.15.86:0/2703819292 learned_addr learned my addr 172.21.15.86:0/2703819292 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1e800640 1 --2- 172.21.15.86:0/2703819292 >> v2:172.21.15.194:3300/0 conn(0x7f5b200cc2a0 0x7f5b200cc720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1de00640 1 -- 172.21.15.86:0/2703819292 >> v2:172.21.15.194:3300/0 conn(0x7f5b200cc2a0 msgr2=0x7f5b200cc720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1de00640 1 --2- 172.21.15.86:0/2703819292 >> v2:172.21.15.194:3300/0 conn(0x7f5b200cc2a0 0x7f5b200cc720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1de00640 1 -- 172.21.15.86:0/2703819292 >> v1:172.21.15.194:6789/0 conn(0x7f5b200cf370 legacy=0x7f5b2006d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:16.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.329+0000 7f5b1de00640 1 -- 172.21.15.86:0/2703819292 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b2006ca50 con 0x7f5b2015a940 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.330+0000 7f5b1de00640 1 --2- 172.21.15.86:0/2703819292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b200b45e0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5b2006ec20 tx=0x7f5b1002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84b72c0a187e2c91 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.330+0000 7f5b1d400640 1 -- 172.21.15.86:0/2703819292 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1002ca80 con 0x7f5b2015a940 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.330+0000 7f5b1d400640 1 -- 172.21.15.86:0/2703819292 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b1002cc20 con 0x7f5b2015a940 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.330+0000 7f5b1d400640 1 -- 172.21.15.86:0/2703819292 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b100343f0 con 0x7f5b2015a940 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 -- 172.21.15.86:0/2703819292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 msgr2=0x7f5b200b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 --2- 172.21.15.86:0/2703819292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b200b45e0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5b2006ec20 tx=0x7f5b1002bee0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 -- 172.21.15.86:0/2703819292 shutdown_connections 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 --2- 172.21.15.86:0/2703819292 >> v2:172.21.15.194:3300/0 conn(0x7f5b200cc2a0 0x7f5b200cc720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 --2- 172.21.15.86:0/2703819292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b200b45e0 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 -- 172.21.15.86:0/2703819292 >> 172.21.15.86:0/2703819292 conn(0x7f5b2006afc0 msgr2=0x7f5b2006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:16.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 -- 172.21.15.86:0/2703819292 shutdown_connections 2024-03-06T07:27:16.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.331+0000 7f5b1f200640 1 -- 172.21.15.86:0/2703819292 wait complete. 2024-03-06T07:27:16.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.333+0000 7f5b1f200640 1 Processor -- start 2024-03-06T07:27:16.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.333+0000 7f5b1f200640 1 -- start start 2024-03-06T07:27:16.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1f200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b200cc2a0 0x7f5b201454e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b200cf370 0x7f5b201f3b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b2015cf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1f200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b200b4b20 con 0x7f5b2015a940 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1f200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b200b30f0 con 0x7f5b200cc2a0 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b2015ad20 con 0x7f5b200cf370 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b2015cf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b2015cf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34366/0 (socket says 172.21.15.86:34366) 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1e800640 1 -- 172.21.15.86:0/4197743831 learned_addr learned my addr 172.21.15.86:0/4197743831 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1de00640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b200cc2a0 0x7f5b201454e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1e800640 1 -- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b200cf370 msgr2=0x7f5b201f3b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b15400640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b200cf370 0x7f5b201f3b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1e800640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b200cf370 0x7f5b201f3b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1e800640 1 -- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b200cc2a0 msgr2=0x7f5b201454e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.334+0000 7f5b1e800640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b200cc2a0 0x7f5b201454e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.335+0000 7f5b1e800640 1 -- 172.21.15.86:0/4197743831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b20145a20 con 0x7f5b2015a940 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.335+0000 7f5b1e800640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b2015cf60 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f5b1800d5d0 tx=0x7f5b1800da90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.335+0000 7f5b16a00640 1 -- 172.21.15.86:0/4197743831 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b180125f0 con 0x7f5b2015a940 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.335+0000 7f5b16a00640 1 -- 172.21.15.86:0/4197743831 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b18019070 con 0x7f5b2015a940 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.335+0000 7f5b16a00640 1 -- 172.21.15.86:0/4197743831 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b180129f0 con 0x7f5b2015a940 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.335+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b2015d9d0 con 0x7f5b2015a940 2024-03-06T07:27:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.335+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b20144ce0 con 0x7f5b2015a940 2024-03-06T07:27:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.336+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5b2015def0 con 0x7f5b2015a940 2024-03-06T07:27:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.337+0000 7f5b16a00640 1 -- 172.21.15.86:0/4197743831 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5b18012b90 con 0x7f5b2015a940 2024-03-06T07:27:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.338+0000 7f5b16a00640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5afc06b500 0x7f5afc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.338+0000 7f5b16a00640 1 -- 172.21.15.86:0/4197743831 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5b1808a600 con 0x7f5b2015a940 2024-03-06T07:27:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.338+0000 7f5b1de00640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5afc06b500 0x7f5afc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.339+0000 7f5b1de00640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5afc06b500 0x7f5afc06d9c0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f5b10035000 tx=0x7f5b10030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:16.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.343+0000 7f5b16a00640 1 -- 172.21.15.86:0/4197743831 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5b180593f0 con 0x7f5b2015a940 2024-03-06T07:27:16.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.519+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f5b200b3bc0 con 0x7f5afc06b500 2024-03-06T07:27:16.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.529+0000 7f5b16a00640 1 -- 172.21.15.86:0/4197743831 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251838 (secure 0 0 0) 0x7f5b200b3bc0 con 0x7f5afc06b500 2024-03-06T07:27:16.534 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:16.535 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:27:16.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5afc06b500 msgr2=0x7f5afc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5afc06b500 0x7f5afc06d9c0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f5b10035000 tx=0x7f5b10030000 comp rx=0 tx=0).stop 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 msgr2=0x7f5b2015cf60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b2015cf60 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f5b1800d5d0 tx=0x7f5b1800da90 comp rx=0 tx=0).stop 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 shutdown_connections 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5afc06b500 0x7f5afc06d9c0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b2015a940 0x7f5b2015cf60 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b200cf370 0x7f5b201f3b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 --2- 172.21.15.86:0/4197743831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b200cc2a0 0x7f5b201454e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 >> 172.21.15.86:0/4197743831 conn(0x7f5b2006afc0 msgr2=0x7f5b20071440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.533+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 shutdown_connections 2024-03-06T07:27:16.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.534+0000 7f5b1f200640 1 -- 172.21.15.86:0/4197743831 wait complete. 2024-03-06T07:27:16.554 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":23,"stamp":"2024-03-06T07:27:14.853306+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219200,"kb_used_data":2984,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755520,"statfs":{"total":773094113280,"available":772869652480,"internally_reserved":0,"allocated":3055616,"data_stored":1816228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":221236787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":59,"num_write_kb":276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.959180"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225431+0000","last_change":"2024-03-06T07:27:07.923330+0000","last_active":"2024-03-06T07:27:09.225431+0000","last_peered":"2024-03-06T07:27:09.225431+0000","last_clean":"2024-03-06T07:27:09.225431+0000","last_became_active":"2024-03-06T07:27:07.923084+0000","last_became_peered":"2024-03-06T07:27:07.923084+0000","last_unstale":"2024-03-06T07:27:09.225431+0000","last_undegraded":"2024-03-06T07:27:09.225431+0000","last_fullsized":"2024-03-06T07:27:09.225431+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:46:48.027042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918758+0000","last_change":"2024-03-06T07:27:07.921104+0000","last_active":"2024-03-06T07:27:08.918758+0000","last_peered":"2024-03-06T07:27:08.918758+0000","last_clean":"2024-03-06T07:27:08.918758+0000","last_became_active":"2024-03-06T07:27:07.920770+0000","last_became_peered":"2024-03-06T07:27:07.920770+0000","last_unstale":"2024-03-06T07:27:08.918758+0000","last_undegraded":"2024-03-06T07:27:08.918758+0000","last_fullsized":"2024-03-06T07:27:08.918758+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226481+0000","last_change":"2024-03-06T07:27:07.922511+0000","last_active":"2024-03-06T07:27:09.226481+0000","last_peered":"2024-03-06T07:27:09.226481+0000","last_clean":"2024-03-06T07:27:09.226481+0000","last_became_active":"2024-03-06T07:27:07.922242+0000","last_became_peered":"2024-03-06T07:27:07.922242+0000","last_unstale":"2024-03-06T07:27:09.226481+0000","last_undegraded":"2024-03-06T07:27:09.226481+0000","last_fullsized":"2024-03-06T07:27:09.226481+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922737+0000","last_change":"2024-03-06T07:27:07.945609+0000","last_active":"2024-03-06T07:27:08.922737+0000","last_peered":"2024-03-06T07:27:08.922737+0000","last_clean":"2024-03-06T07:27:08.922737+0000","last_became_active":"2024-03-06T07:27:07.945312+0000","last_became_peered":"2024-03-06T07:27:07.945312+0000","last_unstale":"2024-03-06T07:27:08.922737+0000","last_undegraded":"2024-03-06T07:27:08.922737+0000","last_fullsized":"2024-03-06T07:27:08.922737+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:38.769682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224858+0000","last_change":"2024-03-06T07:27:07.921341+0000","last_active":"2024-03-06T07:27:09.224858+0000","last_peered":"2024-03-06T07:27:09.224858+0000","last_clean":"2024-03-06T07:27:09.224858+0000","last_became_active":"2024-03-06T07:27:07.920986+0000","last_became_peered":"2024-03-06T07:27:07.920986+0000","last_unstale":"2024-03-06T07:27:09.224858+0000","last_undegraded":"2024-03-06T07:27:09.224858+0000","last_fullsized":"2024-03-06T07:27:09.224858+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922064+0000","last_change":"2024-03-06T07:27:07.934696+0000","last_active":"2024-03-06T07:27:08.922064+0000","last_peered":"2024-03-06T07:27:08.922064+0000","last_clean":"2024-03-06T07:27:08.922064+0000","last_became_active":"2024-03-06T07:27:07.934400+0000","last_became_peered":"2024-03-06T07:27:07.934400+0000","last_unstale":"2024-03-06T07:27:08.922064+0000","last_undegraded":"2024-03-06T07:27:08.922064+0000","last_fullsized":"2024-03-06T07:27:08.922064+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224278+0000","last_change":"2024-03-06T07:27:07.926117+0000","last_active":"2024-03-06T07:27:09.224278+0000","last_peered":"2024-03-06T07:27:09.224278+0000","last_clean":"2024-03-06T07:27:09.224278+0000","last_became_active":"2024-03-06T07:27:07.925805+0000","last_became_peered":"2024-03-06T07:27:07.925805+0000","last_unstale":"2024-03-06T07:27:09.224278+0000","last_undegraded":"2024-03-06T07:27:09.224278+0000","last_fullsized":"2024-03-06T07:27:09.224278+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223418+0000","last_change":"2024-03-06T07:27:07.946584+0000","last_active":"2024-03-06T07:27:09.223418+0000","last_peered":"2024-03-06T07:27:09.223418+0000","last_clean":"2024-03-06T07:27:09.223418+0000","last_became_active":"2024-03-06T07:27:07.945766+0000","last_became_peered":"2024-03-06T07:27:07.945766+0000","last_unstale":"2024-03-06T07:27:09.223418+0000","last_undegraded":"2024-03-06T07:27:09.223418+0000","last_fullsized":"2024-03-06T07:27:09.223418+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225430+0000","last_change":"2024-03-06T07:27:07.948853+0000","last_active":"2024-03-06T07:27:09.225430+0000","last_peered":"2024-03-06T07:27:09.225430+0000","last_clean":"2024-03-06T07:27:09.225430+0000","last_became_active":"2024-03-06T07:27:07.948523+0000","last_became_peered":"2024-03-06T07:27:07.948523+0000","last_unstale":"2024-03-06T07:27:09.225430+0000","last_undegraded":"2024-03-06T07:27:09.225430+0000","last_fullsized":"2024-03-06T07:27:09.225430+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:17:40.285874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933598+0000","last_change":"2024-03-06T07:27:07.946279+0000","last_active":"2024-03-06T07:27:08.933598+0000","last_peered":"2024-03-06T07:27:08.933598+0000","last_clean":"2024-03-06T07:27:08.933598+0000","last_became_active":"2024-03-06T07:27:07.945895+0000","last_became_peered":"2024-03-06T07:27:07.945895+0000","last_unstale":"2024-03-06T07:27:08.933598+0000","last_undegraded":"2024-03-06T07:27:08.933598+0000","last_fullsized":"2024-03-06T07:27:08.933598+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:53.264052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933628+0000","last_change":"2024-03-06T07:27:07.922684+0000","last_active":"2024-03-06T07:27:08.933628+0000","last_peered":"2024-03-06T07:27:08.933628+0000","last_clean":"2024-03-06T07:27:08.933628+0000","last_became_active":"2024-03-06T07:27:07.922348+0000","last_became_peered":"2024-03-06T07:27:07.922348+0000","last_unstale":"2024-03-06T07:27:08.933628+0000","last_undegraded":"2024-03-06T07:27:08.933628+0000","last_fullsized":"2024-03-06T07:27:08.933628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:38:38.919038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223388+0000","last_change":"2024-03-06T07:27:07.923877+0000","last_active":"2024-03-06T07:27:09.223388+0000","last_peered":"2024-03-06T07:27:09.223388+0000","last_clean":"2024-03-06T07:27:09.223388+0000","last_became_active":"2024-03-06T07:27:07.923410+0000","last_became_peered":"2024-03-06T07:27:07.923410+0000","last_unstale":"2024-03-06T07:27:09.223388+0000","last_undegraded":"2024-03-06T07:27:09.223388+0000","last_fullsized":"2024-03-06T07:27:09.223388+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:30.853873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227024+0000","last_change":"2024-03-06T07:27:07.930873+0000","last_active":"2024-03-06T07:27:09.227024+0000","last_peered":"2024-03-06T07:27:09.227024+0000","last_clean":"2024-03-06T07:27:09.227024+0000","last_became_active":"2024-03-06T07:27:07.930653+0000","last_became_peered":"2024-03-06T07:27:07.930653+0000","last_unstale":"2024-03-06T07:27:09.227024+0000","last_undegraded":"2024-03-06T07:27:09.227024+0000","last_fullsized":"2024-03-06T07:27:09.227024+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224966+0000","last_change":"2024-03-06T07:27:07.919903+0000","last_active":"2024-03-06T07:27:09.224966+0000","last_peered":"2024-03-06T07:27:09.224966+0000","last_clean":"2024-03-06T07:27:09.224966+0000","last_became_active":"2024-03-06T07:27:07.919585+0000","last_became_peered":"2024-03-06T07:27:07.919585+0000","last_unstale":"2024-03-06T07:27:09.224966+0000","last_undegraded":"2024-03-06T07:27:09.224966+0000","last_fullsized":"2024-03-06T07:27:09.224966+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:06.597617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917655+0000","last_change":"2024-03-06T07:27:07.908420+0000","last_active":"2024-03-06T07:27:08.917655+0000","last_peered":"2024-03-06T07:27:08.917655+0000","last_clean":"2024-03-06T07:27:08.917655+0000","last_became_active":"2024-03-06T07:27:07.907392+0000","last_became_peered":"2024-03-06T07:27:07.907392+0000","last_unstale":"2024-03-06T07:27:08.917655+0000","last_undegraded":"2024-03-06T07:27:08.917655+0000","last_fullsized":"2024-03-06T07:27:08.917655+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922133+0000","last_change":"2024-03-06T07:27:07.934890+0000","last_active":"2024-03-06T07:27:08.922133+0000","last_peered":"2024-03-06T07:27:08.922133+0000","last_clean":"2024-03-06T07:27:08.922133+0000","last_became_active":"2024-03-06T07:27:07.934568+0000","last_became_peered":"2024-03-06T07:27:07.934568+0000","last_unstale":"2024-03-06T07:27:08.922133+0000","last_undegraded":"2024-03-06T07:27:08.922133+0000","last_fullsized":"2024-03-06T07:27:08.922133+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923353+0000","last_change":"2024-03-06T07:27:07.927869+0000","last_active":"2024-03-06T07:27:08.923353+0000","last_peered":"2024-03-06T07:27:08.923353+0000","last_clean":"2024-03-06T07:27:08.923353+0000","last_became_active":"2024-03-06T07:27:07.927507+0000","last_became_peered":"2024-03-06T07:27:07.927507+0000","last_unstale":"2024-03-06T07:27:08.923353+0000","last_undegraded":"2024-03-06T07:27:08.923353+0000","last_fullsized":"2024-03-06T07:27:08.923353+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:26:20.914324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224317+0000","last_change":"2024-03-06T07:27:07.922930+0000","last_active":"2024-03-06T07:27:09.224317+0000","last_peered":"2024-03-06T07:27:09.224317+0000","last_clean":"2024-03-06T07:27:09.224317+0000","last_became_active":"2024-03-06T07:27:07.922608+0000","last_became_peered":"2024-03-06T07:27:07.922608+0000","last_unstale":"2024-03-06T07:27:09.224317+0000","last_undegraded":"2024-03-06T07:27:09.224317+0000","last_fullsized":"2024-03-06T07:27:09.224317+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225850+0000","last_change":"2024-03-06T07:27:07.927132+0000","last_active":"2024-03-06T07:27:09.225850+0000","last_peered":"2024-03-06T07:27:09.225850+0000","last_clean":"2024-03-06T07:27:09.225850+0000","last_became_active":"2024-03-06T07:27:07.926850+0000","last_became_peered":"2024-03-06T07:27:07.926850+0000","last_unstale":"2024-03-06T07:27:09.225850+0000","last_undegraded":"2024-03-06T07:27:09.225850+0000","last_fullsized":"2024-03-06T07:27:09.225850+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:23.735936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933468+0000","last_change":"2024-03-06T07:27:07.928126+0000","last_active":"2024-03-06T07:27:08.933468+0000","last_peered":"2024-03-06T07:27:08.933468+0000","last_clean":"2024-03-06T07:27:08.933468+0000","last_became_active":"2024-03-06T07:27:07.927650+0000","last_became_peered":"2024-03-06T07:27:07.927650+0000","last_unstale":"2024-03-06T07:27:08.933468+0000","last_undegraded":"2024-03-06T07:27:08.933468+0000","last_fullsized":"2024-03-06T07:27:08.933468+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:55.755072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225914+0000","last_change":"2024-03-06T07:27:07.927942+0000","last_active":"2024-03-06T07:27:09.225914+0000","last_peered":"2024-03-06T07:27:09.225914+0000","last_clean":"2024-03-06T07:27:09.225914+0000","last_became_active":"2024-03-06T07:27:07.927629+0000","last_became_peered":"2024-03-06T07:27:07.927629+0000","last_unstale":"2024-03-06T07:27:09.225914+0000","last_undegraded":"2024-03-06T07:27:09.225914+0000","last_fullsized":"2024-03-06T07:27:09.225914+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922498+0000","last_change":"2024-03-06T07:27:07.922273+0000","last_active":"2024-03-06T07:27:08.922498+0000","last_peered":"2024-03-06T07:27:08.922498+0000","last_clean":"2024-03-06T07:27:08.922498+0000","last_became_active":"2024-03-06T07:27:07.921988+0000","last_became_peered":"2024-03-06T07:27:07.921988+0000","last_unstale":"2024-03-06T07:27:08.922498+0000","last_undegraded":"2024-03-06T07:27:08.922498+0000","last_fullsized":"2024-03-06T07:27:08.922498+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:57.831357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223829+0000","last_change":"2024-03-06T07:27:07.930778+0000","last_active":"2024-03-06T07:27:09.223829+0000","last_peered":"2024-03-06T07:27:09.223829+0000","last_clean":"2024-03-06T07:27:09.223829+0000","last_became_active":"2024-03-06T07:27:07.930464+0000","last_became_peered":"2024-03-06T07:27:07.930464+0000","last_unstale":"2024-03-06T07:27:09.223829+0000","last_undegraded":"2024-03-06T07:27:09.223829+0000","last_fullsized":"2024-03-06T07:27:09.223829+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922134+0000","last_change":"2024-03-06T07:27:07.917391+0000","last_active":"2024-03-06T07:27:08.922134+0000","last_peered":"2024-03-06T07:27:08.922134+0000","last_clean":"2024-03-06T07:27:08.922134+0000","last_became_active":"2024-03-06T07:27:07.917107+0000","last_became_peered":"2024-03-06T07:27:07.917107+0000","last_unstale":"2024-03-06T07:27:08.922134+0000","last_undegraded":"2024-03-06T07:27:08.922134+0000","last_fullsized":"2024-03-06T07:27:08.922134+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:08.467879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227476+0000","last_change":"2024-03-06T07:27:07.930923+0000","last_active":"2024-03-06T07:27:09.227476+0000","last_peered":"2024-03-06T07:27:09.227476+0000","last_clean":"2024-03-06T07:27:09.227476+0000","last_became_active":"2024-03-06T07:27:07.930078+0000","last_became_peered":"2024-03-06T07:27:07.930078+0000","last_unstale":"2024-03-06T07:27:09.227476+0000","last_undegraded":"2024-03-06T07:27:09.227476+0000","last_fullsized":"2024-03-06T07:27:09.227476+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933695+0000","last_change":"2024-03-06T07:27:07.936206+0000","last_active":"2024-03-06T07:27:08.933695+0000","last_peered":"2024-03-06T07:27:08.933695+0000","last_clean":"2024-03-06T07:27:08.933695+0000","last_became_active":"2024-03-06T07:27:07.930820+0000","last_became_peered":"2024-03-06T07:27:07.930820+0000","last_unstale":"2024-03-06T07:27:08.933695+0000","last_undegraded":"2024-03-06T07:27:08.933695+0000","last_fullsized":"2024-03-06T07:27:08.933695+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:41.225025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917816+0000","last_change":"2024-03-06T07:27:07.918528+0000","last_active":"2024-03-06T07:27:08.917816+0000","last_peered":"2024-03-06T07:27:08.917816+0000","last_clean":"2024-03-06T07:27:08.917816+0000","last_became_active":"2024-03-06T07:27:07.918221+0000","last_became_peered":"2024-03-06T07:27:07.918221+0000","last_unstale":"2024-03-06T07:27:08.917816+0000","last_undegraded":"2024-03-06T07:27:08.917816+0000","last_fullsized":"2024-03-06T07:27:08.917816+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227107+0000","last_change":"2024-03-06T07:27:07.929588+0000","last_active":"2024-03-06T07:27:09.227107+0000","last_peered":"2024-03-06T07:27:09.227107+0000","last_clean":"2024-03-06T07:27:09.227107+0000","last_became_active":"2024-03-06T07:27:07.929207+0000","last_became_peered":"2024-03-06T07:27:07.929207+0000","last_unstale":"2024-03-06T07:27:09.227107+0000","last_undegraded":"2024-03-06T07:27:09.227107+0000","last_fullsized":"2024-03-06T07:27:09.227107+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226074+0000","last_change":"2024-03-06T07:27:07.925047+0000","last_active":"2024-03-06T07:27:09.226074+0000","last_peered":"2024-03-06T07:27:09.226074+0000","last_clean":"2024-03-06T07:27:09.226074+0000","last_became_active":"2024-03-06T07:27:07.924734+0000","last_became_peered":"2024-03-06T07:27:07.924734+0000","last_unstale":"2024-03-06T07:27:09.226074+0000","last_undegraded":"2024-03-06T07:27:09.226074+0000","last_fullsized":"2024-03-06T07:27:09.226074+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922314+0000","last_change":"2024-03-06T07:27:07.927022+0000","last_active":"2024-03-06T07:27:08.922314+0000","last_peered":"2024-03-06T07:27:08.922314+0000","last_clean":"2024-03-06T07:27:08.922314+0000","last_became_active":"2024-03-06T07:27:07.926748+0000","last_became_peered":"2024-03-06T07:27:07.926748+0000","last_unstale":"2024-03-06T07:27:08.922314+0000","last_undegraded":"2024-03-06T07:27:08.922314+0000","last_fullsized":"2024-03-06T07:27:08.922314+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:25.638818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223563+0000","last_change":"2024-03-06T07:27:07.934988+0000","last_active":"2024-03-06T07:27:09.223563+0000","last_peered":"2024-03-06T07:27:09.223563+0000","last_clean":"2024-03-06T07:27:09.223563+0000","last_became_active":"2024-03-06T07:27:07.934606+0000","last_became_peered":"2024-03-06T07:27:07.934606+0000","last_unstale":"2024-03-06T07:27:09.223563+0000","last_undegraded":"2024-03-06T07:27:09.223563+0000","last_fullsized":"2024-03-06T07:27:09.223563+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933081+0000","last_change":"2024-03-06T07:27:07.929352+0000","last_active":"2024-03-06T07:27:08.933081+0000","last_peered":"2024-03-06T07:27:08.933081+0000","last_clean":"2024-03-06T07:27:08.933081+0000","last_became_active":"2024-03-06T07:27:07.929001+0000","last_became_peered":"2024-03-06T07:27:07.929001+0000","last_unstale":"2024-03-06T07:27:08.933081+0000","last_undegraded":"2024-03-06T07:27:08.933081+0000","last_fullsized":"2024-03-06T07:27:08.933081+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:17:25.890956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.222941+0000","last_change":"2024-03-06T07:27:07.906813+0000","last_active":"2024-03-06T07:27:09.222941+0000","last_peered":"2024-03-06T07:27:09.222941+0000","last_clean":"2024-03-06T07:27:09.222941+0000","last_became_active":"2024-03-06T07:27:07.906019+0000","last_became_peered":"2024-03-06T07:27:07.906019+0000","last_unstale":"2024-03-06T07:27:09.222941+0000","last_undegraded":"2024-03-06T07:27:09.222941+0000","last_fullsized":"2024-03-06T07:27:09.222941+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:18.357063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226805+0000","last_change":"2024-03-06T07:27:06.929035+0000","last_active":"2024-03-06T07:27:09.226805+0000","last_peered":"2024-03-06T07:27:09.226805+0000","last_clean":"2024-03-06T07:27:09.226805+0000","last_became_active":"2024-03-06T07:27:06.928702+0000","last_became_peered":"2024-03-06T07:27:06.928702+0000","last_unstale":"2024-03-06T07:27:09.226805+0000","last_undegraded":"2024-03-06T07:27:09.226805+0000","last_fullsized":"2024-03-06T07:27:09.226805+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226233+0000","last_change":"2024-03-06T07:27:07.938823+0000","last_active":"2024-03-06T07:27:09.226233+0000","last_peered":"2024-03-06T07:27:09.226233+0000","last_clean":"2024-03-06T07:27:09.226233+0000","last_became_active":"2024-03-06T07:27:07.937638+0000","last_became_peered":"2024-03-06T07:27:07.937638+0000","last_unstale":"2024-03-06T07:27:09.226233+0000","last_undegraded":"2024-03-06T07:27:09.226233+0000","last_fullsized":"2024-03-06T07:27:09.226233+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932806+0000","last_change":"2024-03-06T07:27:06.929034+0000","last_active":"2024-03-06T07:27:08.932806+0000","last_peered":"2024-03-06T07:27:08.932806+0000","last_clean":"2024-03-06T07:27:08.932806+0000","last_became_active":"2024-03-06T07:27:06.927376+0000","last_became_peered":"2024-03-06T07:27:06.927376+0000","last_unstale":"2024-03-06T07:27:08.932806+0000","last_undegraded":"2024-03-06T07:27:08.932806+0000","last_fullsized":"2024-03-06T07:27:08.932806+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225115+0000","last_change":"2024-03-06T07:27:06.933665+0000","last_active":"2024-03-06T07:27:09.225115+0000","last_peered":"2024-03-06T07:27:09.225115+0000","last_clean":"2024-03-06T07:27:09.225115+0000","last_became_active":"2024-03-06T07:27:06.933367+0000","last_became_peered":"2024-03-06T07:27:06.933367+0000","last_unstale":"2024-03-06T07:27:09.225115+0000","last_undegraded":"2024-03-06T07:27:09.225115+0000","last_fullsized":"2024-03-06T07:27:09.225115+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223616+0000","last_change":"2024-03-06T07:27:07.927629+0000","last_active":"2024-03-06T07:27:09.223616+0000","last_peered":"2024-03-06T07:27:09.223616+0000","last_clean":"2024-03-06T07:27:09.223616+0000","last_became_active":"2024-03-06T07:27:07.925804+0000","last_became_peered":"2024-03-06T07:27:07.925804+0000","last_unstale":"2024-03-06T07:27:09.223616+0000","last_undegraded":"2024-03-06T07:27:09.223616+0000","last_fullsized":"2024-03-06T07:27:09.223616+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:20.979511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922349+0000","last_change":"2024-03-06T07:27:06.935442+0000","last_active":"2024-03-06T07:27:08.922349+0000","last_peered":"2024-03-06T07:27:08.922349+0000","last_clean":"2024-03-06T07:27:08.922349+0000","last_became_active":"2024-03-06T07:27:06.933244+0000","last_became_peered":"2024-03-06T07:27:06.933244+0000","last_unstale":"2024-03-06T07:27:08.922349+0000","last_undegraded":"2024-03-06T07:27:08.922349+0000","last_fullsized":"2024-03-06T07:27:08.922349+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917698+0000","last_change":"2024-03-06T07:27:07.922392+0000","last_active":"2024-03-06T07:27:08.917698+0000","last_peered":"2024-03-06T07:27:08.917698+0000","last_clean":"2024-03-06T07:27:08.917698+0000","last_became_active":"2024-03-06T07:27:07.922127+0000","last_became_peered":"2024-03-06T07:27:07.922127+0000","last_unstale":"2024-03-06T07:27:08.917698+0000","last_undegraded":"2024-03-06T07:27:08.917698+0000","last_fullsized":"2024-03-06T07:27:08.917698+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:08:06.222252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933693+0000","last_change":"2024-03-06T07:27:06.937146+0000","last_active":"2024-03-06T07:27:08.933693+0000","last_peered":"2024-03-06T07:27:08.933693+0000","last_clean":"2024-03-06T07:27:08.933693+0000","last_became_active":"2024-03-06T07:27:06.936876+0000","last_became_peered":"2024-03-06T07:27:06.936876+0000","last_unstale":"2024-03-06T07:27:08.933693+0000","last_undegraded":"2024-03-06T07:27:08.933693+0000","last_fullsized":"2024-03-06T07:27:08.933693+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.943897+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.943627+0000","last_became_peered":"2024-03-06T07:27:07.943627+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:01:28.762594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.923212+0000","last_change":"2024-03-06T07:27:06.940833+0000","last_active":"2024-03-06T07:27:08.923212+0000","last_peered":"2024-03-06T07:27:08.923212+0000","last_clean":"2024-03-06T07:27:08.923212+0000","last_became_active":"2024-03-06T07:27:06.939734+0000","last_became_peered":"2024-03-06T07:27:06.939734+0000","last_unstale":"2024-03-06T07:27:08.923212+0000","last_undegraded":"2024-03-06T07:27:08.923212+0000","last_fullsized":"2024-03-06T07:27:08.923212+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227338+0000","last_change":"2024-03-06T07:27:07.946830+0000","last_active":"2024-03-06T07:27:09.227338+0000","last_peered":"2024-03-06T07:27:09.227338+0000","last_clean":"2024-03-06T07:27:09.227338+0000","last_became_active":"2024-03-06T07:27:07.946495+0000","last_became_peered":"2024-03-06T07:27:07.946495+0000","last_unstale":"2024-03-06T07:27:09.227338+0000","last_undegraded":"2024-03-06T07:27:09.227338+0000","last_fullsized":"2024-03-06T07:27:09.227338+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:47.203711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"21'3","reported_seq":17,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.937692+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.937338+0000","last_became_peered":"2024-03-06T07:27:06.937338+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223197+0000","last_change":"2024-03-06T07:27:07.922246+0000","last_active":"2024-03-06T07:27:09.223197+0000","last_peered":"2024-03-06T07:27:09.223197+0000","last_clean":"2024-03-06T07:27:09.223197+0000","last_became_active":"2024-03-06T07:27:07.921469+0000","last_became_peered":"2024-03-06T07:27:07.921469+0000","last_unstale":"2024-03-06T07:27:09.223197+0000","last_undegraded":"2024-03-06T07:27:09.223197+0000","last_fullsized":"2024-03-06T07:27:09.223197+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:14.822199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"21'6","reported_seq":20,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933200+0000","last_change":"2024-03-06T07:27:06.926732+0000","last_active":"2024-03-06T07:27:08.933200+0000","last_peered":"2024-03-06T07:27:08.933200+0000","last_clean":"2024-03-06T07:27:08.933200+0000","last_became_active":"2024-03-06T07:27:06.925706+0000","last_became_peered":"2024-03-06T07:27:06.925706+0000","last_unstale":"2024-03-06T07:27:08.933200+0000","last_undegraded":"2024-03-06T07:27:08.933200+0000","last_fullsized":"2024-03-06T07:27:08.933200+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225109+0000","last_change":"2024-03-06T07:27:07.935123+0000","last_active":"2024-03-06T07:27:09.225109+0000","last_peered":"2024-03-06T07:27:09.225109+0000","last_clean":"2024-03-06T07:27:09.225109+0000","last_became_active":"2024-03-06T07:27:07.934820+0000","last_became_peered":"2024-03-06T07:27:07.934820+0000","last_unstale":"2024-03-06T07:27:09.225109+0000","last_undegraded":"2024-03-06T07:27:09.225109+0000","last_fullsized":"2024-03-06T07:27:09.225109+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:29.926787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226343+0000","last_change":"2024-03-06T07:27:06.921406+0000","last_active":"2024-03-06T07:27:09.226343+0000","last_peered":"2024-03-06T07:27:09.226343+0000","last_clean":"2024-03-06T07:27:09.226343+0000","last_became_active":"2024-03-06T07:27:06.921070+0000","last_became_peered":"2024-03-06T07:27:06.921070+0000","last_unstale":"2024-03-06T07:27:09.226343+0000","last_undegraded":"2024-03-06T07:27:09.226343+0000","last_fullsized":"2024-03-06T07:27:09.226343+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225286+0000","last_change":"2024-03-06T07:27:07.926884+0000","last_active":"2024-03-06T07:27:09.225286+0000","last_peered":"2024-03-06T07:27:09.225286+0000","last_clean":"2024-03-06T07:27:09.225286+0000","last_became_active":"2024-03-06T07:27:07.926600+0000","last_became_peered":"2024-03-06T07:27:07.926600+0000","last_unstale":"2024-03-06T07:27:09.225286+0000","last_undegraded":"2024-03-06T07:27:09.225286+0000","last_fullsized":"2024-03-06T07:27:09.225286+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:59.757026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922438+0000","last_change":"2024-03-06T07:27:06.936248+0000","last_active":"2024-03-06T07:27:08.922438+0000","last_peered":"2024-03-06T07:27:08.922438+0000","last_clean":"2024-03-06T07:27:08.922438+0000","last_became_active":"2024-03-06T07:27:06.934671+0000","last_became_peered":"2024-03-06T07:27:06.934671+0000","last_unstale":"2024-03-06T07:27:08.922438+0000","last_undegraded":"2024-03-06T07:27:08.922438+0000","last_fullsized":"2024-03-06T07:27:08.922438+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:08.101185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224843+0000","last_change":"2024-03-06T07:27:07.941367+0000","last_active":"2024-03-06T07:27:09.224843+0000","last_peered":"2024-03-06T07:27:09.224843+0000","last_clean":"2024-03-06T07:27:09.224843+0000","last_became_active":"2024-03-06T07:27:07.938238+0000","last_became_peered":"2024-03-06T07:27:07.938238+0000","last_unstale":"2024-03-06T07:27:09.224843+0000","last_undegraded":"2024-03-06T07:27:09.224843+0000","last_fullsized":"2024-03-06T07:27:09.224843+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:45:47.378843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933197+0000","last_change":"2024-03-06T07:27:06.931453+0000","last_active":"2024-03-06T07:27:08.933197+0000","last_peered":"2024-03-06T07:27:08.933197+0000","last_clean":"2024-03-06T07:27:08.933197+0000","last_became_active":"2024-03-06T07:27:06.931181+0000","last_became_peered":"2024-03-06T07:27:06.931181+0000","last_unstale":"2024-03-06T07:27:08.933197+0000","last_undegraded":"2024-03-06T07:27:08.933197+0000","last_fullsized":"2024-03-06T07:27:08.933197+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:36.400439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933121+0000","last_change":"2024-03-06T07:27:07.932186+0000","last_active":"2024-03-06T07:27:08.933121+0000","last_peered":"2024-03-06T07:27:08.933121+0000","last_clean":"2024-03-06T07:27:08.933121+0000","last_became_active":"2024-03-06T07:27:07.923467+0000","last_became_peered":"2024-03-06T07:27:07.923467+0000","last_unstale":"2024-03-06T07:27:08.933121+0000","last_undegraded":"2024-03-06T07:27:08.933121+0000","last_fullsized":"2024-03-06T07:27:08.933121+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:27:55.371987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"21'8","reported_seq":22,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223314+0000","last_change":"2024-03-06T07:27:06.925620+0000","last_active":"2024-03-06T07:27:09.223314+0000","last_peered":"2024-03-06T07:27:09.223314+0000","last_clean":"2024-03-06T07:27:09.223314+0000","last_became_active":"2024-03-06T07:27:06.925324+0000","last_became_peered":"2024-03-06T07:27:06.925324+0000","last_unstale":"2024-03-06T07:27:09.223314+0000","last_undegraded":"2024-03-06T07:27:09.223314+0000","last_fullsized":"2024-03-06T07:27:09.223314+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:29.581809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224242+0000","last_change":"2024-03-06T07:27:07.935148+0000","last_active":"2024-03-06T07:27:09.224242+0000","last_peered":"2024-03-06T07:27:09.224242+0000","last_clean":"2024-03-06T07:27:09.224242+0000","last_became_active":"2024-03-06T07:27:07.933239+0000","last_became_peered":"2024-03-06T07:27:07.933239+0000","last_unstale":"2024-03-06T07:27:09.224242+0000","last_undegraded":"2024-03-06T07:27:09.224242+0000","last_fullsized":"2024-03-06T07:27:09.224242+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:03.663936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223549+0000","last_change":"2024-03-06T07:27:06.919221+0000","last_active":"2024-03-06T07:27:09.223549+0000","last_peered":"2024-03-06T07:27:09.223549+0000","last_clean":"2024-03-06T07:27:09.223549+0000","last_became_active":"2024-03-06T07:27:06.918433+0000","last_became_peered":"2024-03-06T07:27:06.918433+0000","last_unstale":"2024-03-06T07:27:09.223549+0000","last_undegraded":"2024-03-06T07:27:09.223549+0000","last_fullsized":"2024-03-06T07:27:09.223549+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:21:09.808244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225325+0000","last_change":"2024-03-06T07:27:07.933938+0000","last_active":"2024-03-06T07:27:09.225325+0000","last_peered":"2024-03-06T07:27:09.225325+0000","last_clean":"2024-03-06T07:27:09.225325+0000","last_became_active":"2024-03-06T07:27:07.933656+0000","last_became_peered":"2024-03-06T07:27:07.933656+0000","last_unstale":"2024-03-06T07:27:09.225325+0000","last_undegraded":"2024-03-06T07:27:09.225325+0000","last_fullsized":"2024-03-06T07:27:09.225325+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:23:36.112418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224591+0000","last_change":"2024-03-06T07:27:06.913843+0000","last_active":"2024-03-06T07:27:09.224591+0000","last_peered":"2024-03-06T07:27:09.224591+0000","last_clean":"2024-03-06T07:27:09.224591+0000","last_became_active":"2024-03-06T07:27:06.912393+0000","last_became_peered":"2024-03-06T07:27:06.912393+0000","last_unstale":"2024-03-06T07:27:09.224591+0000","last_undegraded":"2024-03-06T07:27:09.224591+0000","last_fullsized":"2024-03-06T07:27:09.224591+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:36.573949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225673+0000","last_change":"2024-03-06T07:27:07.927885+0000","last_active":"2024-03-06T07:27:09.225673+0000","last_peered":"2024-03-06T07:27:09.225673+0000","last_clean":"2024-03-06T07:27:09.225673+0000","last_became_active":"2024-03-06T07:27:07.927215+0000","last_became_peered":"2024-03-06T07:27:07.927215+0000","last_unstale":"2024-03-06T07:27:09.225673+0000","last_undegraded":"2024-03-06T07:27:09.225673+0000","last_fullsized":"2024-03-06T07:27:09.225673+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:36:30.188524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918041+0000","last_change":"2024-03-06T07:27:06.921613+0000","last_active":"2024-03-06T07:27:08.918041+0000","last_peered":"2024-03-06T07:27:08.918041+0000","last_clean":"2024-03-06T07:27:08.918041+0000","last_became_active":"2024-03-06T07:27:06.921225+0000","last_became_peered":"2024-03-06T07:27:06.921225+0000","last_unstale":"2024-03-06T07:27:08.918041+0000","last_undegraded":"2024-03-06T07:27:08.918041+0000","last_fullsized":"2024-03-06T07:27:08.918041+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922980+0000","last_change":"2024-03-06T07:27:07.924340+0000","last_active":"2024-03-06T07:27:08.922980+0000","last_peered":"2024-03-06T07:27:08.922980+0000","last_clean":"2024-03-06T07:27:08.922980+0000","last_became_active":"2024-03-06T07:27:07.924090+0000","last_became_peered":"2024-03-06T07:27:07.924090+0000","last_unstale":"2024-03-06T07:27:08.922980+0000","last_undegraded":"2024-03-06T07:27:08.922980+0000","last_fullsized":"2024-03-06T07:27:08.922980+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:54.697587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223461+0000","last_change":"2024-03-06T07:27:06.933112+0000","last_active":"2024-03-06T07:27:09.223461+0000","last_peered":"2024-03-06T07:27:09.223461+0000","last_clean":"2024-03-06T07:27:09.223461+0000","last_became_active":"2024-03-06T07:27:06.932764+0000","last_became_peered":"2024-03-06T07:27:06.932764+0000","last_unstale":"2024-03-06T07:27:09.223461+0000","last_undegraded":"2024-03-06T07:27:09.223461+0000","last_fullsized":"2024-03-06T07:27:09.223461+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225383+0000","last_change":"2024-03-06T07:27:07.940694+0000","last_active":"2024-03-06T07:27:09.225383+0000","last_peered":"2024-03-06T07:27:09.225383+0000","last_clean":"2024-03-06T07:27:09.225383+0000","last_became_active":"2024-03-06T07:27:07.940352+0000","last_became_peered":"2024-03-06T07:27:07.940352+0000","last_unstale":"2024-03-06T07:27:09.225383+0000","last_undegraded":"2024-03-06T07:27:09.225383+0000","last_fullsized":"2024-03-06T07:27:09.225383+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:40:06.241503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":170,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224024+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:27:09.224024+0000","last_peered":"2024-03-06T07:27:09.224024+0000","last_clean":"2024-03-06T07:27:09.224024+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:27:09.224024+0000","last_undegraded":"2024-03-06T07:27:09.224024+0000","last_fullsized":"2024-03-06T07:27:09.224024+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922420+0000","last_change":"2024-03-06T07:27:06.931167+0000","last_active":"2024-03-06T07:27:08.922420+0000","last_peered":"2024-03-06T07:27:08.922420+0000","last_clean":"2024-03-06T07:27:08.922420+0000","last_became_active":"2024-03-06T07:27:06.930895+0000","last_became_peered":"2024-03-06T07:27:06.930895+0000","last_unstale":"2024-03-06T07:27:08.922420+0000","last_undegraded":"2024-03-06T07:27:08.922420+0000","last_fullsized":"2024-03-06T07:27:08.922420+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:26:47.129259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225706+0000","last_change":"2024-03-06T07:27:07.946419+0000","last_active":"2024-03-06T07:27:09.225706+0000","last_peered":"2024-03-06T07:27:09.225706+0000","last_clean":"2024-03-06T07:27:09.225706+0000","last_became_active":"2024-03-06T07:27:07.945795+0000","last_became_peered":"2024-03-06T07:27:07.945795+0000","last_unstale":"2024-03-06T07:27:09.225706+0000","last_undegraded":"2024-03-06T07:27:09.225706+0000","last_fullsized":"2024-03-06T07:27:09.225706+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224064+0000","last_change":"2024-03-06T07:27:06.936519+0000","last_active":"2024-03-06T07:27:09.224064+0000","last_peered":"2024-03-06T07:27:09.224064+0000","last_clean":"2024-03-06T07:27:09.224064+0000","last_became_active":"2024-03-06T07:27:06.934384+0000","last_became_peered":"2024-03-06T07:27:06.934384+0000","last_unstale":"2024-03-06T07:27:09.224064+0000","last_undegraded":"2024-03-06T07:27:09.224064+0000","last_fullsized":"2024-03-06T07:27:09.224064+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:29.611416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224613+0000","last_change":"2024-03-06T07:27:07.926848+0000","last_active":"2024-03-06T07:27:09.224613+0000","last_peered":"2024-03-06T07:27:09.224613+0000","last_clean":"2024-03-06T07:27:09.224613+0000","last_became_active":"2024-03-06T07:27:07.926255+0000","last_became_peered":"2024-03-06T07:27:07.926255+0000","last_unstale":"2024-03-06T07:27:09.224613+0000","last_undegraded":"2024-03-06T07:27:09.224613+0000","last_fullsized":"2024-03-06T07:27:09.224613+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:43:44.555041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223817+0000","last_change":"2024-03-06T07:27:06.912373+0000","last_active":"2024-03-06T07:27:09.223817+0000","last_peered":"2024-03-06T07:27:09.223817+0000","last_clean":"2024-03-06T07:27:09.223817+0000","last_became_active":"2024-03-06T07:27:06.911823+0000","last_became_peered":"2024-03-06T07:27:06.911823+0000","last_unstale":"2024-03-06T07:27:09.223817+0000","last_undegraded":"2024-03-06T07:27:09.223817+0000","last_fullsized":"2024-03-06T07:27:09.223817+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917936+0000","last_change":"2024-03-06T07:27:07.917473+0000","last_active":"2024-03-06T07:27:08.917936+0000","last_peered":"2024-03-06T07:27:08.917936+0000","last_clean":"2024-03-06T07:27:08.917936+0000","last_became_active":"2024-03-06T07:27:07.914951+0000","last_became_peered":"2024-03-06T07:27:07.914951+0000","last_unstale":"2024-03-06T07:27:08.917936+0000","last_undegraded":"2024-03-06T07:27:08.917936+0000","last_fullsized":"2024-03-06T07:27:08.917936+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922608+0000","last_change":"2024-03-06T07:27:06.928866+0000","last_active":"2024-03-06T07:27:08.922608+0000","last_peered":"2024-03-06T07:27:08.922608+0000","last_clean":"2024-03-06T07:27:08.922608+0000","last_became_active":"2024-03-06T07:27:06.928456+0000","last_became_peered":"2024-03-06T07:27:06.928456+0000","last_unstale":"2024-03-06T07:27:08.922608+0000","last_undegraded":"2024-03-06T07:27:08.922608+0000","last_fullsized":"2024-03-06T07:27:08.922608+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:40.927231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223508+0000","last_change":"2024-03-06T07:27:07.933229+0000","last_active":"2024-03-06T07:27:09.223508+0000","last_peered":"2024-03-06T07:27:09.223508+0000","last_clean":"2024-03-06T07:27:09.223508+0000","last_became_active":"2024-03-06T07:27:07.932737+0000","last_became_peered":"2024-03-06T07:27:07.932737+0000","last_unstale":"2024-03-06T07:27:09.223508+0000","last_undegraded":"2024-03-06T07:27:09.223508+0000","last_fullsized":"2024-03-06T07:27:09.223508+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:20.889857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918205+0000","last_change":"2024-03-06T07:27:06.918955+0000","last_active":"2024-03-06T07:27:08.918205+0000","last_peered":"2024-03-06T07:27:08.918205+0000","last_clean":"2024-03-06T07:27:08.918205+0000","last_became_active":"2024-03-06T07:27:06.918655+0000","last_became_peered":"2024-03-06T07:27:06.918655+0000","last_unstale":"2024-03-06T07:27:08.918205+0000","last_undegraded":"2024-03-06T07:27:08.918205+0000","last_fullsized":"2024-03-06T07:27:08.918205+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922887+0000","last_change":"2024-03-06T07:27:07.924920+0000","last_active":"2024-03-06T07:27:08.922887+0000","last_peered":"2024-03-06T07:27:08.922887+0000","last_clean":"2024-03-06T07:27:08.922887+0000","last_became_active":"2024-03-06T07:27:07.924624+0000","last_became_peered":"2024-03-06T07:27:07.924624+0000","last_unstale":"2024-03-06T07:27:08.922887+0000","last_undegraded":"2024-03-06T07:27:08.922887+0000","last_fullsized":"2024-03-06T07:27:08.922887+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:59:43.062703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225644+0000","last_change":"2024-03-06T07:27:06.936617+0000","last_active":"2024-03-06T07:27:09.225644+0000","last_peered":"2024-03-06T07:27:09.225644+0000","last_clean":"2024-03-06T07:27:09.225644+0000","last_became_active":"2024-03-06T07:27:06.936328+0000","last_became_peered":"2024-03-06T07:27:06.936328+0000","last_unstale":"2024-03-06T07:27:09.225644+0000","last_undegraded":"2024-03-06T07:27:09.225644+0000","last_fullsized":"2024-03-06T07:27:09.225644+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:42:10.164751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.918287+0000","last_change":"2024-03-06T07:27:07.916912+0000","last_active":"2024-03-06T07:27:08.918287+0000","last_peered":"2024-03-06T07:27:08.918287+0000","last_clean":"2024-03-06T07:27:08.918287+0000","last_became_active":"2024-03-06T07:27:07.916557+0000","last_became_peered":"2024-03-06T07:27:07.916557+0000","last_unstale":"2024-03-06T07:27:08.918287+0000","last_undegraded":"2024-03-06T07:27:08.918287+0000","last_fullsized":"2024-03-06T07:27:08.918287+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:09:42.503216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"22'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.931761+0000","last_change":"2024-03-06T07:27:06.934214+0000","last_active":"2024-03-06T07:27:08.931761+0000","last_peered":"2024-03-06T07:27:08.931761+0000","last_clean":"2024-03-06T07:27:08.931761+0000","last_became_active":"2024-03-06T07:27:06.933984+0000","last_became_peered":"2024-03-06T07:27:06.933984+0000","last_unstale":"2024-03-06T07:27:08.931761+0000","last_undegraded":"2024-03-06T07:27:08.931761+0000","last_fullsized":"2024-03-06T07:27:08.931761+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227628+0000","last_change":"2024-03-06T07:27:07.934955+0000","last_active":"2024-03-06T07:27:09.227628+0000","last_peered":"2024-03-06T07:27:09.227628+0000","last_clean":"2024-03-06T07:27:09.227628+0000","last_became_active":"2024-03-06T07:27:07.934602+0000","last_became_peered":"2024-03-06T07:27:07.934602+0000","last_unstale":"2024-03-06T07:27:09.227628+0000","last_undegraded":"2024-03-06T07:27:09.227628+0000","last_fullsized":"2024-03-06T07:27:09.227628+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:32:01.505526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.933574+0000","last_change":"2024-03-06T07:27:06.943288+0000","last_active":"2024-03-06T07:27:08.933574+0000","last_peered":"2024-03-06T07:27:08.933574+0000","last_clean":"2024-03-06T07:27:08.933574+0000","last_became_active":"2024-03-06T07:27:06.942813+0000","last_became_peered":"2024-03-06T07:27:06.942813+0000","last_unstale":"2024-03-06T07:27:08.933574+0000","last_undegraded":"2024-03-06T07:27:08.933574+0000","last_fullsized":"2024-03-06T07:27:08.933574+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:00.680476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226729+0000","last_change":"2024-03-06T07:27:07.920024+0000","last_active":"2024-03-06T07:27:09.226729+0000","last_peered":"2024-03-06T07:27:09.226729+0000","last_clean":"2024-03-06T07:27:09.226729+0000","last_became_active":"2024-03-06T07:27:07.919750+0000","last_became_peered":"2024-03-06T07:27:07.919750+0000","last_unstale":"2024-03-06T07:27:09.226729+0000","last_undegraded":"2024-03-06T07:27:09.226729+0000","last_fullsized":"2024-03-06T07:27:09.226729+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:06.141825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223933+0000","last_change":"2024-03-06T07:27:06.936885+0000","last_active":"2024-03-06T07:27:09.223933+0000","last_peered":"2024-03-06T07:27:09.223933+0000","last_clean":"2024-03-06T07:27:09.223933+0000","last_became_active":"2024-03-06T07:27:06.934829+0000","last_became_peered":"2024-03-06T07:27:06.934829+0000","last_unstale":"2024-03-06T07:27:09.223933+0000","last_undegraded":"2024-03-06T07:27:09.223933+0000","last_fullsized":"2024-03-06T07:27:09.223933+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:17.114605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223030+0000","last_change":"2024-03-06T07:27:07.916276+0000","last_active":"2024-03-06T07:27:09.223030+0000","last_peered":"2024-03-06T07:27:09.223030+0000","last_clean":"2024-03-06T07:27:09.223030+0000","last_became_active":"2024-03-06T07:27:07.915974+0000","last_became_peered":"2024-03-06T07:27:07.915974+0000","last_unstale":"2024-03-06T07:27:09.223030+0000","last_undegraded":"2024-03-06T07:27:09.223030+0000","last_fullsized":"2024-03-06T07:27:09.223030+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:37.360040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917927+0000","last_change":"2024-03-06T07:27:06.935659+0000","last_active":"2024-03-06T07:27:08.917927+0000","last_peered":"2024-03-06T07:27:08.917927+0000","last_clean":"2024-03-06T07:27:08.917927+0000","last_became_active":"2024-03-06T07:27:06.935373+0000","last_became_peered":"2024-03-06T07:27:06.935373+0000","last_unstale":"2024-03-06T07:27:08.917927+0000","last_undegraded":"2024-03-06T07:27:08.917927+0000","last_fullsized":"2024-03-06T07:27:08.917927+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.223769+0000","last_change":"2024-03-06T07:27:07.910767+0000","last_active":"2024-03-06T07:27:09.223769+0000","last_peered":"2024-03-06T07:27:09.223769+0000","last_clean":"2024-03-06T07:27:09.223769+0000","last_became_active":"2024-03-06T07:27:07.910268+0000","last_became_peered":"2024-03-06T07:27:07.910268+0000","last_unstale":"2024-03-06T07:27:09.223769+0000","last_undegraded":"2024-03-06T07:27:09.223769+0000","last_fullsized":"2024-03-06T07:27:09.223769+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:08:59.946124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.917633+0000","last_change":"2024-03-06T07:27:06.923056+0000","last_active":"2024-03-06T07:27:08.917633+0000","last_peered":"2024-03-06T07:27:08.917633+0000","last_clean":"2024-03-06T07:27:08.917633+0000","last_became_active":"2024-03-06T07:27:06.922710+0000","last_became_peered":"2024-03-06T07:27:06.922710+0000","last_unstale":"2024-03-06T07:27:08.917633+0000","last_undegraded":"2024-03-06T07:27:08.917633+0000","last_fullsized":"2024-03-06T07:27:08.917633+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:24.507549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226527+0000","last_change":"2024-03-06T07:27:07.922445+0000","last_active":"2024-03-06T07:27:09.226527+0000","last_peered":"2024-03-06T07:27:09.226527+0000","last_clean":"2024-03-06T07:27:09.226527+0000","last_became_active":"2024-03-06T07:27:07.922147+0000","last_became_peered":"2024-03-06T07:27:07.922147+0000","last_unstale":"2024-03-06T07:27:09.226527+0000","last_undegraded":"2024-03-06T07:27:09.226527+0000","last_fullsized":"2024-03-06T07:27:09.226527+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224979+0000","last_change":"2024-03-06T07:27:06.940633+0000","last_active":"2024-03-06T07:27:09.224979+0000","last_peered":"2024-03-06T07:27:09.224979+0000","last_clean":"2024-03-06T07:27:09.224979+0000","last_became_active":"2024-03-06T07:27:06.937434+0000","last_became_peered":"2024-03-06T07:27:06.937434+0000","last_unstale":"2024-03-06T07:27:09.224979+0000","last_undegraded":"2024-03-06T07:27:09.224979+0000","last_fullsized":"2024-03-06T07:27:09.224979+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:57:13.579742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.932876+0000","last_change":"2024-03-06T07:27:07.942081+0000","last_active":"2024-03-06T07:27:08.932876+0000","last_peered":"2024-03-06T07:27:08.932876+0000","last_clean":"2024-03-06T07:27:08.932876+0000","last_became_active":"2024-03-06T07:27:07.941647+0000","last_became_peered":"2024-03-06T07:27:07.941647+0000","last_unstale":"2024-03-06T07:27:08.932876+0000","last_undegraded":"2024-03-06T07:27:08.932876+0000","last_fullsized":"2024-03-06T07:27:08.932876+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:47:21.103970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922929+0000","last_change":"2024-03-06T07:27:07.944160+0000","last_active":"2024-03-06T07:27:08.922929+0000","last_peered":"2024-03-06T07:27:08.922929+0000","last_clean":"2024-03-06T07:27:08.922929+0000","last_became_active":"2024-03-06T07:27:07.943238+0000","last_became_peered":"2024-03-06T07:27:07.943238+0000","last_unstale":"2024-03-06T07:27:08.922929+0000","last_undegraded":"2024-03-06T07:27:08.922929+0000","last_fullsized":"2024-03-06T07:27:08.922929+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:51.931654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.224835+0000","last_change":"2024-03-06T07:27:06.940899+0000","last_active":"2024-03-06T07:27:09.224835+0000","last_peered":"2024-03-06T07:27:09.224835+0000","last_clean":"2024-03-06T07:27:09.224835+0000","last_became_active":"2024-03-06T07:27:06.940627+0000","last_became_peered":"2024-03-06T07:27:06.940627+0000","last_unstale":"2024-03-06T07:27:09.224835+0000","last_undegraded":"2024-03-06T07:27:09.224835+0000","last_fullsized":"2024-03-06T07:27:09.224835+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:36.117582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:08.922932+0000","last_change":"2024-03-06T07:27:07.933712+0000","last_active":"2024-03-06T07:27:08.922932+0000","last_peered":"2024-03-06T07:27:08.922932+0000","last_clean":"2024-03-06T07:27:08.922932+0000","last_became_active":"2024-03-06T07:27:07.933437+0000","last_became_peered":"2024-03-06T07:27:07.933437+0000","last_unstale":"2024-03-06T07:27:08.922932+0000","last_undegraded":"2024-03-06T07:27:08.922932+0000","last_fullsized":"2024-03-06T07:27:08.922932+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:18.550912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"21'2","reported_seq":16,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.225416+0000","last_change":"2024-03-06T07:27:06.938164+0000","last_active":"2024-03-06T07:27:09.225416+0000","last_peered":"2024-03-06T07:27:09.225416+0000","last_clean":"2024-03-06T07:27:09.225416+0000","last_became_active":"2024-03-06T07:27:06.937669+0000","last_became_peered":"2024-03-06T07:27:06.937669+0000","last_unstale":"2024-03-06T07:27:09.225416+0000","last_undegraded":"2024-03-06T07:27:09.225416+0000","last_fullsized":"2024-03-06T07:27:09.225416+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:19.647400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.226963+0000","last_change":"2024-03-06T07:27:07.948380+0000","last_active":"2024-03-06T07:27:09.226963+0000","last_peered":"2024-03-06T07:27:09.226963+0000","last_clean":"2024-03-06T07:27:09.226963+0000","last_became_active":"2024-03-06T07:27:07.948110+0000","last_became_peered":"2024-03-06T07:27:07.948110+0000","last_unstale":"2024-03-06T07:27:09.226963+0000","last_undegraded":"2024-03-06T07:27:09.226963+0000","last_fullsized":"2024-03-06T07:27:09.226963+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:22.975903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"21'1","reported_seq":15,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227256+0000","last_change":"2024-03-06T07:27:06.911519+0000","last_active":"2024-03-06T07:27:09.227256+0000","last_peered":"2024-03-06T07:27:09.227256+0000","last_clean":"2024-03-06T07:27:09.227256+0000","last_became_active":"2024-03-06T07:27:06.911051+0000","last_became_peered":"2024-03-06T07:27:06.911051+0000","last_unstale":"2024-03-06T07:27:09.227256+0000","last_undegraded":"2024-03-06T07:27:09.227256+0000","last_fullsized":"2024-03-06T07:27:09.227256+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:42:16.748967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227138+0000","last_change":"2024-03-06T07:27:07.937255+0000","last_active":"2024-03-06T07:27:09.227138+0000","last_peered":"2024-03-06T07:27:09.227138+0000","last_clean":"2024-03-06T07:27:09.227138+0000","last_became_active":"2024-03-06T07:27:07.936713+0000","last_became_peered":"2024-03-06T07:27:07.936713+0000","last_unstale":"2024-03-06T07:27:09.227138+0000","last_undegraded":"2024-03-06T07:27:09.227138+0000","last_fullsized":"2024-03-06T07:27:09.227138+0000","mapping_epoch":20,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":21,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:06.877246+0000","last_clean_scrub_stamp":"2024-03-06T07:27:06.877246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:20:47.888059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"21'4","reported_seq":18,"reported_epoch":22,"state":"active+clean","last_fresh":"2024-03-06T07:27:09.227467+0000","last_change":"2024-03-06T07:27:06.935678+0000","last_active":"2024-03-06T07:27:09.227467+0000","last_peered":"2024-03-06T07:27:09.227467+0000","last_clean":"2024-03-06T07:27:09.227467+0000","last_became_active":"2024-03-06T07:27:06.935349+0000","last_became_peered":"2024-03-06T07:27:06.935349+0000","last_unstale":"2024-03-06T07:27:09.227467+0000","last_undegraded":"2024-03-06T07:27:09.227467+0000","last_fullsized":"2024-03-06T07:27:09.227467+0000","mapping_epoch":19,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":20,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:05.634788+0000","last_clean_scrub_stamp":"2024-03-06T07:27:05.634788+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:41.343661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542150,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27836,"kb_used_data":804,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344004,"statfs":{"total":96636764160,"available":96608260096,"internally_reserved":0,"allocated":823296,"data_stored":669784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542149,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27252,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344588,"statfs":{"total":96636764160,"available":96608858112,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542150,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27244,"kb_used_data":224,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344596,"statfs":{"total":96636764160,"available":96608866304,"internally_reserved":0,"allocated":229376,"data_stored":78924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542149,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":78960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542149,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27260,"kb_used_data":232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344580,"statfs":{"total":96636764160,"available":96608849920,"internally_reserved":0,"allocated":237568,"data_stored":80446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542150,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27248,"kb_used_data":220,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344592,"statfs":{"total":96636764160,"available":96608862208,"internally_reserved":0,"allocated":225280,"data_stored":78834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542151,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27256,"kb_used_data":228,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344584,"statfs":{"total":96636764160,"available":96608854016,"internally_reserved":0,"allocated":233472,"data_stored":78994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542148,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27844,"kb_used_data":812,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343996,"statfs":{"total":96636764160,"available":96608251904,"internally_reserved":0,"allocated":831488,"data_stored":669840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:27:16.558 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-03-06T07:27:16.601 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-06T07:27:16.602 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-03-06T07:27:16.602 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-03-06T07:27:16.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.802+0000 7f1ddae00640 1 Processor -- start 2024-03-06T07:27:16.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.802+0000 7f1ddae00640 1 -- start start 2024-03-06T07:27:16.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1ddae00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1dd406dc80 0x7f1dd41625b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1ddae00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1dd4071e30 0x7f1dd415e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1ddae00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1dd4072260 con 0x7f1dd40af100 2024-03-06T07:27:16.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1ddae00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1dd40af590 con 0x7f1dd4071e30 2024-03-06T07:27:16.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1ddae00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1dd4161b60 con 0x7f1dd406dc80 2024-03-06T07:27:16.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1dd8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1dd4071e30 0x7f1dd415e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1dd8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1dd4071e30 0x7f1dd415e1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34382/0 (socket says 172.21.15.86:34382) 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.804+0000 7f1dd8e00640 1 -- 172.21.15.86:0/2673692891 learned_addr learned my addr 172.21.15.86:0/2673692891 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.805+0000 7f1dd3e00640 1 --2- 172.21.15.86:0/2673692891 >> v2:172.21.15.194:3300/0 conn(0x7f1dd406dc80 0x7f1dd41625b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.805+0000 7f1dd8e00640 1 -- 172.21.15.86:0/2673692891 >> v2:172.21.15.194:3300/0 conn(0x7f1dd406dc80 msgr2=0x7f1dd41625b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.805+0000 7f1dd8e00640 1 --2- 172.21.15.86:0/2673692891 >> v2:172.21.15.194:3300/0 conn(0x7f1dd406dc80 0x7f1dd41625b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.805+0000 7f1dd8e00640 1 -- 172.21.15.86:0/2673692891 >> v1:172.21.15.194:6789/0 conn(0x7f1dd40af100 legacy=0x7f1dd406e060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.805+0000 7f1dd8e00640 1 -- 172.21.15.86:0/2673692891 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1dd4072530 con 0x7f1dd4071e30 2024-03-06T07:27:16.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.805+0000 7f1dd3e00640 1 --2- 172.21.15.86:0/2673692891 >> v2:172.21.15.194:3300/0 conn(0x7f1dd406dc80 0x7f1dd41625b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:16.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.806+0000 7f1dd8e00640 1 --2- 172.21.15.86:0/2673692891 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd4071e30 0x7f1dd415e1d0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f1dd406cc20 tx=0x7f1dcc02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9f0d6b4dd7cc5d3d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:16.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.806+0000 7f1dd3400640 1 -- 172.21.15.86:0/2673692891 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dcc02ca80 con 0x7f1dd4071e30 2024-03-06T07:27:16.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.806+0000 7f1dd3400640 1 -- 172.21.15.86:0/2673692891 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1dcc02cc20 con 0x7f1dd4071e30 2024-03-06T07:27:16.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.806+0000 7f1dd3400640 1 -- 172.21.15.86:0/2673692891 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dcc0343f0 con 0x7f1dd4071e30 2024-03-06T07:27:16.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 -- 172.21.15.86:0/2673692891 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd4071e30 msgr2=0x7f1dd415e1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 --2- 172.21.15.86:0/2673692891 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd4071e30 0x7f1dd415e1d0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f1dd406cc20 tx=0x7f1dcc02bee0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 -- 172.21.15.86:0/2673692891 shutdown_connections 2024-03-06T07:27:16.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 --2- 172.21.15.86:0/2673692891 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd4071e30 0x7f1dd415e1d0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 --2- 172.21.15.86:0/2673692891 >> v2:172.21.15.194:3300/0 conn(0x7f1dd406dc80 0x7f1dd41625b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 -- 172.21.15.86:0/2673692891 >> 172.21.15.86:0/2673692891 conn(0x7f1dd40cfa30 msgr2=0x7f1dd40cfe40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:16.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 -- 172.21.15.86:0/2673692891 shutdown_connections 2024-03-06T07:27:16.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.807+0000 7f1ddae00640 1 -- 172.21.15.86:0/2673692891 wait complete. 2024-03-06T07:27:16.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.808+0000 7f1ddae00640 1 Processor -- start 2024-03-06T07:27:16.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.808+0000 7f1ddae00640 1 -- start start 2024-03-06T07:27:16.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1ddae00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd406dc80 0x7f1dd415ac90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1ddae00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dd4071e30 0x7f1dd4145860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1ddae00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dd40af100 0x7f1dd4156880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1ddae00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1dd4162090 con 0x7f1dd406dc80 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1ddae00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1dd4161db0 con 0x7f1dd40af100 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1ddae00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1dd415e710 con 0x7f1dd4071e30 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1dd3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd406dc80 0x7f1dd415ac90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1dd3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd406dc80 0x7f1dd415ac90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34394/0 (socket says 172.21.15.86:34394) 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.809+0000 7f1dd3e00640 1 -- 172.21.15.86:0/3862403177 learned_addr learned my addr 172.21.15.86:0/3862403177 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd3e00640 1 -- 172.21.15.86:0/3862403177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dd4071e30 msgr2=0x7f1dd4145860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd3e00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dd4071e30 0x7f1dd4145860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd3e00640 1 -- 172.21.15.86:0/3862403177 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dd40af100 msgr2=0x7f1dd4156880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd3e00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dd40af100 0x7f1dd4156880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd3e00640 1 -- 172.21.15.86:0/3862403177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1dd4159ed0 con 0x7f1dd406dc80 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd3e00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd406dc80 0x7f1dd415ac90 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f1dc400c7f0 tx=0x7f1dc400cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd0c00640 1 -- 172.21.15.86:0/3862403177 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dc4015070 con 0x7f1dd406dc80 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd0c00640 1 -- 172.21.15.86:0/3862403177 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1dc4007a30 con 0x7f1dd406dc80 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1dd0c00640 1 -- 172.21.15.86:0/3862403177 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dc4007e00 con 0x7f1dd406dc80 2024-03-06T07:27:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1dd415a1b0 con 0x7f1dd406dc80 2024-03-06T07:27:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.810+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1dd4161520 con 0x7f1dd406dc80 2024-03-06T07:27:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.811+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1dd404f330 con 0x7f1dd406dc80 2024-03-06T07:27:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.812+0000 7f1dd0c00640 1 -- 172.21.15.86:0/3862403177 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1dc4018070 con 0x7f1dd406dc80 2024-03-06T07:27:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.813+0000 7f1dd0c00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1db006b4d0 0x7f1db006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.813+0000 7f1dd0c00640 1 -- 172.21.15.86:0/3862403177 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1dc408a2a0 con 0x7f1dd406dc80 2024-03-06T07:27:16.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.813+0000 7f1dcbe00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1db006b4d0 0x7f1db006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:16.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.814+0000 7f1dcbe00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1db006b4d0 0x7f1db006d990 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f1dd41577d0 tx=0x7f1dbc006ba0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:16.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:16.817+0000 7f1dd0c00640 1 -- 172.21.15.86:0/3862403177 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1dc4059080 con 0x7f1dd406dc80 2024-03-06T07:27:17.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.048+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) v1 -- 0x7f1dd40aeb20 con 0x7f1dd406dc80 2024-03-06T07:27:17.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.049+0000 7f1dd0c00640 1 -- 172.21.15.86:0/3862403177 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) v1 ==== 72+0+46 (secure 0 0 0) 0x7f1dc4058980 con 0x7f1dd406dc80 2024-03-06T07:27:17.051 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:17.051 INFO:teuthology.orchestra.run.smithi086.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1db006b4d0 msgr2=0x7f1db006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1db006b4d0 0x7f1db006d990 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f1dd41577d0 tx=0x7f1dbc006ba0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd406dc80 msgr2=0x7f1dd415ac90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd406dc80 0x7f1dd415ac90 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f1dc400c7f0 tx=0x7f1dc400cbf0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 shutdown_connections 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1db006b4d0 0x7f1db006d990 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dd40af100 0x7f1dd4156880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dd4071e30 0x7f1dd4145860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 --2- 172.21.15.86:0/3862403177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dd406dc80 0x7f1dd415ac90 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 >> 172.21.15.86:0/3862403177 conn(0x7f1dd40cfa30 msgr2=0x7f1dd40714c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 shutdown_connections 2024-03-06T07:27:17.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.052+0000 7f1ddae00640 1 -- 172.21.15.86:0/3862403177 wait complete. 2024-03-06T07:27:17.073 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-03-06T07:27:17.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:17.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.276+0000 7f3a9f200640 1 Processor -- start 2024-03-06T07:27:17.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.276+0000 7f3a9f200640 1 -- start start 2024-03-06T07:27:17.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.277+0000 7f3a9f200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3aa015aab0 0x7f3aa00b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.277+0000 7f3a9f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 0x7f3aa00cd460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a9f200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3aa015ae90 con 0x7f3aa006d2e0 2024-03-06T07:27:17.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a9f200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3aa006ce50 con 0x7f3aa015aab0 2024-03-06T07:27:17.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a9f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa00b3160 con 0x7f3aa00cc9b0 2024-03-06T07:27:17.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a95400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3aa006d2e0 0x7f3aa00cc2a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36806/0 (socket says 172.21.15.86:36806) 2024-03-06T07:27:17.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a95400640 1 -- 172.21.15.86:0/1302181340 learned_addr learned my addr 172.21.15.86:0/1302181340 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:17.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a9de00640 1 --2- 172.21.15.86:0/1302181340 >> v2:172.21.15.194:3300/0 conn(0x7f3aa015aab0 0x7f3aa00b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a9e800640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 0x7f3aa00cd460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2457232690 0 0) 0x7f3aa015ae90 con 0x7f3aa006d2e0 2024-03-06T07:27:17.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.278+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3a80003520 con 0x7f3aa006d2e0 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1908387822 0 0) 0x7f3a80003520 con 0x7f3aa006d2e0 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 msgr2=0x7f3aa00cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 0x7f3aa00cd460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 >> v2:172.21.15.194:3300/0 conn(0x7f3aa015aab0 msgr2=0x7f3aa00b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 --2- 172.21.15.86:0/1302181340 >> v2:172.21.15.194:3300/0 conn(0x7f3aa015aab0 0x7f3aa00b45e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3aa00b28e0 con 0x7f3aa006d2e0 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3a88001690 con 0x7f3aa006d2e0 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 >> v1:172.21.15.86:6789/0 conn(0x7f3aa006d2e0 legacy=0x7f3aa00cc2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a800039b0 0x7f3a80003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a80004800 0x7f3a80004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 0x7f3a800051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa006ce50 con 0x7f3a80004800 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa00b3160 con 0x7f3a800039b0 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3a8000fdd0 con 0x7f3aa00cc9b0 2024-03-06T07:27:17.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a95400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a80004800 0x7f3a80004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9de00640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a800039b0 0x7f3a80003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a9e800640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 0x7f3a800051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a95400640 1 -- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 msgr2=0x7f3a800051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a95400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa00cc9b0 0x7f3a800051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a95400640 1 -- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a800039b0 msgr2=0x7f3a80003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a95400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a800039b0 0x7f3a80003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.279+0000 7f3a95400640 1 -- 172.21.15.86:0/1302181340 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a80003520 con 0x7f3a80004800 2024-03-06T07:27:17.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.280+0000 7f3a9de00640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a800039b0 0x7f3a80003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:17.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.280+0000 7f3a95400640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a80004800 0x7f3a80004c80 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f3aa006ec50 tx=0x7f3a80005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2993ff3dc064ee39 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:17.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a88007ca0 con 0x7f3a80004800 2024-03-06T07:27:17.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a88030070 con 0x7f3a80004800 2024-03-06T07:27:17.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9d400640 1 -- 172.21.15.86:0/1302181340 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a88005d10 con 0x7f3a80004800 2024-03-06T07:27:17.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9f200640 1 -- 172.21.15.86:0/1302181340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a80004800 msgr2=0x7f3a80004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9f200640 1 --2- 172.21.15.86:0/1302181340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a80004800 0x7f3a80004c80 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f3aa006ec50 tx=0x7f3a80005830 comp rx=0 tx=0).stop 2024-03-06T07:27:17.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9e800640 1 -- 172.21.15.86:0/1302181340 reap_dead start 2024-03-06T07:27:17.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9f200640 1 -- 172.21.15.86:0/1302181340 shutdown_connections 2024-03-06T07:27:17.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9f200640 1 -- 172.21.15.86:0/1302181340 >> 172.21.15.86:0/1302181340 conn(0x7f3aa006afc0 msgr2=0x7f3aa006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:17.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.281+0000 7f3a9f200640 1 -- 172.21.15.86:0/1302181340 shutdown_connections 2024-03-06T07:27:17.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.282+0000 7f3a9f200640 1 -- 172.21.15.86:0/1302181340 wait complete. 2024-03-06T07:27:17.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.283+0000 7f3a9f200640 1 Processor -- start 2024-03-06T07:27:17.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.283+0000 7f3a9f200640 1 -- start start 2024-03-06T07:27:17.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.284+0000 7f3a9f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a80004800 0x7f3aa01ef990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.284+0000 7f3a9f200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa0146290 0x7f3aa0157170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.284+0000 7f3a9f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa0146710 0x7f3aa01576b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.284+0000 7f3a9f200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa006cae0 con 0x7f3aa0146710 2024-03-06T07:27:17.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.284+0000 7f3a9f200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa00b2e80 con 0x7f3aa0146290 2024-03-06T07:27:17.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.284+0000 7f3a9f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa00b4b20 con 0x7f3a80004800 2024-03-06T07:27:17.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a80004800 0x7f3aa01ef990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa0146710 0x7f3aa01576b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a80004800 0x7f3aa01ef990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:54904/0 (socket says 172.21.15.86:54904) 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9de00640 1 -- 172.21.15.86:0/3411727486 learned_addr learned my addr 172.21.15.86:0/3411727486 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a95400640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa0146290 0x7f3aa0157170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9e800640 1 -- 172.21.15.86:0/3411727486 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a80004800 msgr2=0x7f3aa01ef990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9e800640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a80004800 0x7f3aa01ef990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9e800640 1 -- 172.21.15.86:0/3411727486 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa0146290 msgr2=0x7f3aa0157170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9e800640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa0146290 0x7f3aa0157170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.285+0000 7f3a9e800640 1 -- 172.21.15.86:0/3411727486 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3aa00b07a0 con 0x7f3aa0146710 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.286+0000 7f3a9e800640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa0146710 0x7f3aa01576b0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f3aa006eb60 tx=0x7f3a9800f4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.286+0000 7f3a96a00640 1 -- 172.21.15.86:0/3411727486 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a9801c070 con 0x7f3aa0146710 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.286+0000 7f3a96a00640 1 -- 172.21.15.86:0/3411727486 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a98017400 con 0x7f3aa0146710 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.286+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3aa00b0a10 con 0x7f3aa0146710 2024-03-06T07:27:17.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.286+0000 7f3a96a00640 1 -- 172.21.15.86:0/3411727486 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a980177f0 con 0x7f3aa0146710 2024-03-06T07:27:17.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.286+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3aa00b0fd0 con 0x7f3aa0146710 2024-03-06T07:27:17.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.287+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3aa004f330 con 0x7f3aa0146710 2024-03-06T07:27:17.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.288+0000 7f3a96a00640 1 -- 172.21.15.86:0/3411727486 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3a9801f070 con 0x7f3aa0146710 2024-03-06T07:27:17.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.289+0000 7f3a96a00640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a7c06b4d0 0x7f3a7c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.289+0000 7f3a96a00640 1 -- 172.21.15.86:0/3411727486 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3a9808a110 con 0x7f3aa0146710 2024-03-06T07:27:17.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.289+0000 7f3a9de00640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a7c06b4d0 0x7f3a7c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.290+0000 7f3a9de00640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a7c06b4d0 0x7f3a7c06d990 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f3a90009c90 tx=0x7f3a90008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:17.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.293+0000 7f3a96a00640 1 -- 172.21.15.86:0/3411727486 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3a98012070 con 0x7f3aa0146710 2024-03-06T07:27:17.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.491+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f3aa006e180 con 0x7f3aa0146710 2024-03-06T07:27:17.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.494+0000 7f3a96a00640 1 -- 172.21.15.86:0/3411727486 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f3a980589d0 con 0x7f3aa0146710 2024-03-06T07:27:17.497 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:17.497 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:17.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.496+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a7c06b4d0 msgr2=0x7f3a7c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.496+0000 7f3a9f200640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a7c06b4d0 0x7f3a7c06d990 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f3a90009c90 tx=0x7f3a90008040 comp rx=0 tx=0).stop 2024-03-06T07:27:17.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.496+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa0146710 msgr2=0x7f3aa01576b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.496+0000 7f3a9f200640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa0146710 0x7f3aa01576b0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f3aa006eb60 tx=0x7f3a9800f4a0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.496+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 shutdown_connections 2024-03-06T07:27:17.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.496+0000 7f3a9f200640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a7c06b4d0 0x7f3a7c06d990 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.497+0000 7f3a9f200640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa0146710 0x7f3aa01576b0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.497+0000 7f3a9f200640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa0146290 0x7f3aa0157170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.497+0000 7f3a9f200640 1 --2- 172.21.15.86:0/3411727486 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a80004800 0x7f3aa01ef990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.497+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 >> 172.21.15.86:0/3411727486 conn(0x7f3aa006afc0 msgr2=0x7f3aa015e510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:17.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.497+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 shutdown_connections 2024-03-06T07:27:17.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.497+0000 7f3a9f200640 1 -- 172.21.15.86:0/3411727486 wait complete. 2024-03-06T07:27:17.516 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:17.516 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:17.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.709+0000 7f5b71600640 1 Processor -- start 2024-03-06T07:27:17.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.710+0000 7f5b71600640 1 -- start start 2024-03-06T07:27:17.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.711+0000 7f5b71600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 0x7f5b6c0cea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.711+0000 7f5b71600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5b6c0b3730 con 0x7f5b6c0d07f0 2024-03-06T07:27:17.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.711+0000 7f5b71600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5b6c067b20 con 0x7f5b6c0b5490 2024-03-06T07:27:17.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.711+0000 7f5b71600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b6c0d4930 con 0x7f5b6c15e450 2024-03-06T07:27:17.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b6aa00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f5b6c0d07f0 0x7f5b6c0d0bd0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36822/0 (socket says 172.21.15.86:36822) 2024-03-06T07:27:17.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b6aa00640 1 -- 172.21.15.86:0/3365658662 learned_addr learned my addr 172.21.15.86:0/3365658662 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:17.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b6a000640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 0x7f5b6c0cea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2104236723 0 0) 0x7f5b6c0b3730 con 0x7f5b6c0d07f0 2024-03-06T07:27:17.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5b50003520 con 0x7f5b6c0d07f0 2024-03-06T07:27:17.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 45769107 0 0) 0x7f5b50003520 con 0x7f5b6c0d07f0 2024-03-06T07:27:17.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 msgr2=0x7f5b6c0cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 0x7f5b6c0cea20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 >> v1:172.21.15.194:6789/0 conn(0x7f5b6c0b5490 legacy=0x7f5b6c0cef60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b6c0d4b80 con 0x7f5b6c0d07f0 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.712+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f5b54001670 con 0x7f5b6c0d07f0 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 >> v1:172.21.15.86:6789/0 conn(0x7f5b6c0d07f0 legacy=0x7f5b6c0d0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b69600640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 0x7f5b500039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b69600640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b50003f60 0x7f5b50004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b69600640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b50004d40 0x7f5b500051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b6c0d4930 con 0x7f5b50004d40 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b5000bb50 con 0x7f5b50003f60 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b5000bd50 con 0x7f5b6c15e450 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6b400640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b50004d40 0x7f5b500051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6aa00640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 0x7f5b500039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6a000640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b50003f60 0x7f5b50004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6b400640 1 -- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 msgr2=0x7f5b500039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:17.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6b400640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 0x7f5b500039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6b400640 1 -- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b50003f60 msgr2=0x7f5b50004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:17.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6b400640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b50003f60 0x7f5b50004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6b400640 1 -- 172.21.15.86:0/3365658662 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b50003520 con 0x7f5b50004d40 2024-03-06T07:27:17.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.713+0000 7f5b6aa00640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b6c15e450 0x7f5b500039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.714+0000 7f5b6b400640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b50004d40 0x7f5b500051c0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f5b64001720 tx=0x7f5b50005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=998f91d75b23d7da server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b64001370 con 0x7f5b50004d40 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b640052d0 con 0x7f5b50004d40 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b69600640 1 -- 172.21.15.86:0/3365658662 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b64003830 con 0x7f5b50004d40 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b71600640 1 -- 172.21.15.86:0/3365658662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b50004d40 msgr2=0x7f5b500051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b71600640 1 --2- 172.21.15.86:0/3365658662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b50004d40 0x7f5b500051c0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f5b64001720 tx=0x7f5b50005830 comp rx=0 tx=0).stop 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b6b400640 1 -- 172.21.15.86:0/3365658662 reap_dead start 2024-03-06T07:27:17.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b71600640 1 -- 172.21.15.86:0/3365658662 shutdown_connections 2024-03-06T07:27:17.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b71600640 1 -- 172.21.15.86:0/3365658662 >> 172.21.15.86:0/3365658662 conn(0x7f5b6c0d3e10 msgr2=0x7f5b6c0d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:17.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.715+0000 7f5b71600640 1 -- 172.21.15.86:0/3365658662 shutdown_connections 2024-03-06T07:27:17.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.716+0000 7f5b71600640 1 -- 172.21.15.86:0/3365658662 wait complete. 2024-03-06T07:27:17.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.717+0000 7f5b71600640 1 Processor -- start 2024-03-06T07:27:17.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.717+0000 7f5b71600640 1 -- start start 2024-03-06T07:27:17.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.718+0000 7f5b71600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b50004d40 0x7f5b6c0c9f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.718+0000 7f5b71600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b6c156c60 0x7f5b6c0c8410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.718+0000 7f5b71600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b6c1570e0 0x7f5b6c0cca00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.718+0000 7f5b71600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b6c0b3f60 con 0x7f5b6c1570e0 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.718+0000 7f5b71600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b6c0b3c80 con 0x7f5b6c156c60 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.718+0000 7f5b71600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b6c0d1bf0 con 0x7f5b50004d40 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b6c1570e0 0x7f5b6c0cca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b6c1570e0 0x7f5b6c0cca00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34420/0 (socket says 172.21.15.86:34420) 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 -- 172.21.15.86:0/1356674029 learned_addr learned my addr 172.21.15.86:0/1356674029 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6aa00640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b50004d40 0x7f5b6c0c9f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6a000640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b6c156c60 0x7f5b6c0c8410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 -- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b50004d40 msgr2=0x7f5b6c0c9f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b50004d40 0x7f5b6c0c9f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 -- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b6c156c60 msgr2=0x7f5b6c0c8410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b6c156c60 0x7f5b6c0c8410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 -- 172.21.15.86:0/1356674029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b6c0ca5a0 con 0x7f5b6c1570e0 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6aa00640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b50004d40 0x7f5b6c0c9f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6a000640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b6c156c60 0x7f5b6c0c8410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:17.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.719+0000 7f5b6b400640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b6c1570e0 0x7f5b6c0cca00 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f5b64001840 tx=0x7f5b64001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:17.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.720+0000 7f5b62a00640 1 -- 172.21.15.86:0/1356674029 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b64001370 con 0x7f5b6c1570e0 2024-03-06T07:27:17.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.720+0000 7f5b62a00640 1 -- 172.21.15.86:0/1356674029 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b64044070 con 0x7f5b6c1570e0 2024-03-06T07:27:17.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.720+0000 7f5b62a00640 1 -- 172.21.15.86:0/1356674029 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b64007a30 con 0x7f5b6c1570e0 2024-03-06T07:27:17.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.720+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b6c0c7b20 con 0x7f5b6c1570e0 2024-03-06T07:27:17.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.720+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b6c0c80e0 con 0x7f5b6c1570e0 2024-03-06T07:27:17.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.721+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5b6c0c7940 con 0x7f5b6c1570e0 2024-03-06T07:27:17.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.722+0000 7f5b62a00640 1 -- 172.21.15.86:0/1356674029 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5b6403b040 con 0x7f5b6c1570e0 2024-03-06T07:27:17.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.723+0000 7f5b62a00640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b4806b4d0 0x7f5b4806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:17.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.723+0000 7f5b62a00640 1 -- 172.21.15.86:0/1356674029 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5b640ab800 con 0x7f5b6c1570e0 2024-03-06T07:27:17.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.723+0000 7f5b6aa00640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b4806b4d0 0x7f5b4806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:17.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.724+0000 7f5b6aa00640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b4806b4d0 0x7f5b4806d990 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f5b6c0b1f30 tx=0x7f5b54001df0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:17.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.728+0000 7f5b62a00640 1 -- 172.21.15.86:0/1356674029 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5b6c0c7940 con 0x7f5b6c1570e0 2024-03-06T07:27:17.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.923+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f5b6c0ce600 con 0x7f5b6c1570e0 2024-03-06T07:27:17.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.924+0000 7f5b62a00640 1 -- 172.21.15.86:0/1356674029 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f5b6c0ce600 con 0x7f5b6c1570e0 2024-03-06T07:27:17.927 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:17.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b4806b4d0 msgr2=0x7f5b4806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b4806b4d0 0x7f5b4806d990 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f5b6c0b1f30 tx=0x7f5b54001df0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b6c1570e0 msgr2=0x7f5b6c0cca00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:17.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b6c1570e0 0x7f5b6c0cca00 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f5b64001840 tx=0x7f5b64001120 comp rx=0 tx=0).stop 2024-03-06T07:27:17.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 shutdown_connections 2024-03-06T07:27:17.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b4806b4d0 0x7f5b4806d990 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5b6c1570e0 0x7f5b6c0cca00 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5b6c156c60 0x7f5b6c0c8410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 --2- 172.21.15.86:0/1356674029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5b50004d40 0x7f5b6c0c9f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:17.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.926+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 >> 172.21.15.86:0/1356674029 conn(0x7f5b6c0d3e10 msgr2=0x7f5b6c0d23f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:17.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.927+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 shutdown_connections 2024-03-06T07:27:17.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:17.927+0000 7f5b71600640 1 -- 172.21.15.86:0/1356674029 wait complete. 2024-03-06T07:27:17.948 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:18.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.159+0000 7f089fa00640 1 Processor -- start 2024-03-06T07:27:18.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.159+0000 7f089fa00640 1 -- start start 2024-03-06T07:27:18.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.161+0000 7f089fa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0898163800 0x7f08980b2c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.161+0000 7f089fa00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f089815cf70 con 0x7f089806e060 2024-03-06T07:27:18.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.161+0000 7f089fa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f08980b35b0 con 0x7f089815cb90 2024-03-06T07:27:18.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.161+0000 7f089fa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f08980b3890 con 0x7f0898163800 2024-03-06T07:27:18.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.161+0000 7f089da00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f089806e060 0x7f08980cda90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36826/0 (socket says 172.21.15.86:36826) 2024-03-06T07:27:18.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f089da00640 1 -- 172.21.15.86:0/2334857361 learned_addr learned my addr 172.21.15.86:0/2334857361 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:18.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897e00640 1 --2- 172.21.15.86:0/2334857361 >> v2:172.21.15.194:3300/0 conn(0x7f0898163800 0x7f08980b2c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3057329255 0 0) 0x7f089815cf70 con 0x7f089806e060 2024-03-06T07:27:18.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f087c003520 con 0x7f089806e060 2024-03-06T07:27:18.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1918862516 0 0) 0x7f087c003520 con 0x7f089806e060 2024-03-06T07:27:18.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 >> v2:172.21.15.194:3300/0 conn(0x7f0898163800 msgr2=0x7f08980b2c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 --2- 172.21.15.86:0/2334857361 >> v2:172.21.15.194:3300/0 conn(0x7f0898163800 0x7f08980b2c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 >> v1:172.21.15.194:6789/0 conn(0x7f089815cb90 legacy=0x7f089806d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f089806cbe0 con 0x7f089806e060 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f08900019f0 con 0x7f089806e060 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 >> v1:172.21.15.86:6789/0 conn(0x7f089806e060 legacy=0x7f08980cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.162+0000 7f0897400640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c0039b0 0x7f087c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897400640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f087c004800 0x7f087c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897400640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f087c0051c0 0x7f087c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08980b3890 con 0x7f087c004800 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f087c00c060 con 0x7f087c0039b0 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f087c00c260 con 0x7f087c0051c0 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897e00640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f087c004800 0x7f087c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f089d000640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c0039b0 0x7f087c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f089da00640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f087c0051c0 0x7f087c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897e00640 1 -- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f087c0051c0 msgr2=0x7f087c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:18.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897e00640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f087c0051c0 0x7f087c00b9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897e00640 1 -- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c0039b0 msgr2=0x7f087c003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:18.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897e00640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c0039b0 0x7f087c003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f0897e00640 1 -- 172.21.15.86:0/2334857361 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f087c003520 con 0x7f087c004800 2024-03-06T07:27:18.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.163+0000 7f089da00640 1 -- 172.21.15.86:0/2334857361 reap_dead start 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.164+0000 7f0897e00640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f087c004800 0x7f087c004c80 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f0880009ac0 tx=0x7f087c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2d8208d4b21bfce7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0880039070 con 0x7f087c004800 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f088002c6d0 con 0x7f087c004800 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f0897400640 1 -- 172.21.15.86:0/2334857361 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0880009690 con 0x7f087c004800 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f089fa00640 1 -- 172.21.15.86:0/2334857361 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f087c004800 msgr2=0x7f087c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f089fa00640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f087c004800 0x7f087c004c80 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f0880009ac0 tx=0x7f087c005830 comp rx=0 tx=0).stop 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f089fa00640 1 -- 172.21.15.86:0/2334857361 shutdown_connections 2024-03-06T07:27:18.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f089fa00640 1 --2- 172.21.15.86:0/2334857361 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f087c004800 0x7f087c004c80 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f089fa00640 1 -- 172.21.15.86:0/2334857361 >> 172.21.15.86:0/2334857361 conn(0x7f08980b1940 msgr2=0x7f08980b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:18.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f089fa00640 1 -- 172.21.15.86:0/2334857361 shutdown_connections 2024-03-06T07:27:18.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.165+0000 7f089fa00640 1 -- 172.21.15.86:0/2334857361 wait complete. 2024-03-06T07:27:18.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.167+0000 7f089fa00640 1 Processor -- start 2024-03-06T07:27:18.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.167+0000 7f089fa00640 1 -- start start 2024-03-06T07:27:18.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.168+0000 7f089fa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c004800 0x7f0898157910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.168+0000 7f089fa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f089814bb80 0x7f089815b790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.168+0000 7f089fa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08980adfd0 0x7f08980ae450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.168+0000 7f089fa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f089806d230 con 0x7f08980adfd0 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.168+0000 7f089fa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f089806c990 con 0x7f087c004800 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.168+0000 7f089fa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f089806e4d0 con 0x7f089814bb80 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.168+0000 7f089da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08980adfd0 0x7f08980ae450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08980adfd0 0x7f08980ae450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34444/0 (socket says 172.21.15.86:34444) 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 -- 172.21.15.86:0/2585076777 learned_addr learned my addr 172.21.15.86:0/2585076777 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089d000640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c004800 0x7f0898157910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f0897e00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f089814bb80 0x7f089815b790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 -- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f089814bb80 msgr2=0x7f089815b790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f089814bb80 0x7f089815b790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 -- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c004800 msgr2=0x7f0898157910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c004800 0x7f0898157910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 -- 172.21.15.86:0/2585076777 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0898156590 con 0x7f08980adfd0 2024-03-06T07:27:18.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f0897e00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f089814bb80 0x7f089815b790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:18.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.169+0000 7f089da00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08980adfd0 0x7f08980ae450 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0890007c40 tx=0x7f089000b4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:18.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.170+0000 7f0894c00640 1 -- 172.21.15.86:0/2585076777 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0890004d60 con 0x7f08980adfd0 2024-03-06T07:27:18.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.170+0000 7f0894c00640 1 -- 172.21.15.86:0/2585076777 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f089000bb20 con 0x7f08980adfd0 2024-03-06T07:27:18.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.170+0000 7f0894c00640 1 -- 172.21.15.86:0/2585076777 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0890007070 con 0x7f08980adfd0 2024-03-06T07:27:18.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.170+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0898156870 con 0x7f08980adfd0 2024-03-06T07:27:18.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.170+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0898156e30 con 0x7f08980adfd0 2024-03-06T07:27:18.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.171+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f089804f330 con 0x7f08980adfd0 2024-03-06T07:27:18.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.172+0000 7f0894c00640 1 -- 172.21.15.86:0/2585076777 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f089001b020 con 0x7f08980adfd0 2024-03-06T07:27:18.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.172+0000 7f0894c00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087406b500 0x7f087406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.173+0000 7f0894c00640 1 -- 172.21.15.86:0/2585076777 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f089008a390 con 0x7f08980adfd0 2024-03-06T07:27:18.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.173+0000 7f089d000640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087406b500 0x7f087406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.173+0000 7f089d000640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087406b500 0x7f087406d9c0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f0888007fd0 tx=0x7f0888009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:18.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.177+0000 7f0894c00640 1 -- 172.21.15.86:0/2585076777 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f08900590e0 con 0x7f08980adfd0 2024-03-06T07:27:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.373+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f08980b2530 con 0x7f08980adfd0 2024-03-06T07:27:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.374+0000 7f0894c00640 1 -- 172.21.15.86:0/2585076777 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7f08900589e0 con 0x7f08980adfd0 2024-03-06T07:27:18.376 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.376+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087406b500 msgr2=0x7f087406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.376+0000 7f089fa00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087406b500 0x7f087406d9c0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f0888007fd0 tx=0x7f0888009040 comp rx=0 tx=0).stop 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.376+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08980adfd0 msgr2=0x7f08980ae450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.376+0000 7f089fa00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08980adfd0 0x7f08980ae450 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0890007c40 tx=0x7f089000b4a0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 shutdown_connections 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087406b500 0x7f087406d9c0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08980adfd0 0x7f08980ae450 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f089814bb80 0x7f089815b790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 --2- 172.21.15.86:0/2585076777 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f087c004800 0x7f0898157910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 >> 172.21.15.86:0/2585076777 conn(0x7f08980b1940 msgr2=0x7f08980b0480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:18.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 shutdown_connections 2024-03-06T07:27:18.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.377+0000 7f089fa00640 1 -- 172.21.15.86:0/2585076777 wait complete. 2024-03-06T07:27:18.398 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:18.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.609+0000 7f03f3400640 1 Processor -- start 2024-03-06T07:27:18.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.609+0000 7f03f3400640 1 -- start start 2024-03-06T07:27:18.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03f3400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f03ec15fc10 0x7f03ec070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03f3400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f03ec15fff0 0x7f03ec160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03f3400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f03ec0b46c0 con 0x7f03ec0ce4a0 2024-03-06T07:27:18.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03f3400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f03ec0b49a0 con 0x7f03ec15fff0 2024-03-06T07:27:18.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03f3400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f03ec072c90 con 0x7f03ec15fc10 2024-03-06T07:27:18.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03ebe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f03ec15fff0 0x7f03ec160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03ebe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f03ec15fff0 0x7f03ec160450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34446/0 (socket says 172.21.15.86:34446) 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03ebe00640 1 -- 172.21.15.86:0/384284045 learned_addr learned my addr 172.21.15.86:0/384284045 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.611+0000 7f03f0a00640 1 --2- 172.21.15.86:0/384284045 >> v2:172.21.15.194:3300/0 conn(0x7f03ec15fc10 0x7f03ec070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.612+0000 7f03ebe00640 1 -- 172.21.15.86:0/384284045 >> v2:172.21.15.194:3300/0 conn(0x7f03ec15fc10 msgr2=0x7f03ec070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.612+0000 7f03ebe00640 1 --2- 172.21.15.86:0/384284045 >> v2:172.21.15.194:3300/0 conn(0x7f03ec15fc10 0x7f03ec070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.612+0000 7f03ebe00640 1 -- 172.21.15.86:0/384284045 >> v1:172.21.15.194:6789/0 conn(0x7f03ec0ce4a0 legacy=0x7f03ec0ce8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.612+0000 7f03ebe00640 1 -- 172.21.15.86:0/384284045 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03ec160990 con 0x7f03ec15fff0 2024-03-06T07:27:18.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.613+0000 7f03ebe00640 1 --2- 172.21.15.86:0/384284045 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec160450 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f03ec0cf770 tx=0x7f03dc02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=407f9d7f727ccc77 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:18.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.613+0000 7f03eb400640 1 -- 172.21.15.86:0/384284045 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03dc035020 con 0x7f03ec15fff0 2024-03-06T07:27:18.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.613+0000 7f03eb400640 1 -- 172.21.15.86:0/384284045 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03dc031870 con 0x7f03ec15fff0 2024-03-06T07:27:18.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.613+0000 7f03eb400640 1 -- 172.21.15.86:0/384284045 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03dc031c40 con 0x7f03ec15fff0 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 -- 172.21.15.86:0/384284045 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 msgr2=0x7f03ec160450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 --2- 172.21.15.86:0/384284045 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec160450 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f03ec0cf770 tx=0x7f03dc02b910 comp rx=0 tx=0).stop 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 -- 172.21.15.86:0/384284045 shutdown_connections 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 --2- 172.21.15.86:0/384284045 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec160450 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 --2- 172.21.15.86:0/384284045 >> v2:172.21.15.194:3300/0 conn(0x7f03ec15fc10 0x7f03ec070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 -- 172.21.15.86:0/384284045 >> 172.21.15.86:0/384284045 conn(0x7f03ec0b1940 msgr2=0x7f03ec0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 -- 172.21.15.86:0/384284045 shutdown_connections 2024-03-06T07:27:18.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.614+0000 7f03f3400640 1 -- 172.21.15.86:0/384284045 wait complete. 2024-03-06T07:27:18.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.616+0000 7f03f3400640 1 Processor -- start 2024-03-06T07:27:18.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.616+0000 7f03f3400640 1 -- start start 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f3400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03ec0ce4a0 0x7f03ec1518f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f3400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03ec15fc10 0x7f03ec06de20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec1e9e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f3400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03ec072a30 con 0x7f03ec15fff0 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f3400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03ec0728b0 con 0x7f03ec15fc10 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f3400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03ec15c2e0 con 0x7f03ec0ce4a0 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec1e9e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec1e9e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34450/0 (socket says 172.21.15.86:34450) 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.617+0000 7f03f1400640 1 -- 172.21.15.86:0/2272383752 learned_addr learned my addr 172.21.15.86:0/2272383752 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:18.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f0a00640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03ec0ce4a0 0x7f03ec1518f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03ebe00640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03ec15fc10 0x7f03ec06de20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f1400640 1 -- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03ec0ce4a0 msgr2=0x7f03ec1518f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f1400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03ec0ce4a0 0x7f03ec1518f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f1400640 1 -- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03ec15fc10 msgr2=0x7f03ec06de20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f1400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03ec15fc10 0x7f03ec06de20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f1400640 1 -- 172.21.15.86:0/2272383752 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03ec06cbe0 con 0x7f03ec15fff0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f1400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec1e9e00 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f03e400c7b0 tx=0x7f03e400cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03e8c00640 1 -- 172.21.15.86:0/2272383752 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03e4017070 con 0x7f03ec15fff0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03e8c00640 1 -- 172.21.15.86:0/2272383752 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03e4003da0 con 0x7f03ec15fff0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03ec06cec0 con 0x7f03ec15fff0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03e8c00640 1 -- 172.21.15.86:0/2272383752 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03e40052b0 con 0x7f03ec15fff0 2024-03-06T07:27:18.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.618+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03ec06d480 con 0x7f03ec15fff0 2024-03-06T07:27:18.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.619+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f03ec04f330 con 0x7f03ec15fff0 2024-03-06T07:27:18.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.620+0000 7f03e8c00640 1 -- 172.21.15.86:0/2272383752 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f03e401a070 con 0x7f03ec15fff0 2024-03-06T07:27:18.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.621+0000 7f03e8c00640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03d006b4d0 0x7f03d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:18.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.621+0000 7f03e8c00640 1 -- 172.21.15.86:0/2272383752 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f03e40898e0 con 0x7f03ec15fff0 2024-03-06T07:27:18.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.622+0000 7f03f0a00640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03d006b4d0 0x7f03d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:18.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.622+0000 7f03f0a00640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03d006b4d0 0x7f03d006d990 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f03e0006fd0 tx=0x7f03e0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:18.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.626+0000 7f03e8c00640 1 -- 172.21.15.86:0/2272383752 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f03e40586c0 con 0x7f03ec15fff0 2024-03-06T07:27:18.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.824+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f03ec0acec0 con 0x7f03ec15fff0 2024-03-06T07:27:18.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.825+0000 7f03e8c00640 1 -- 172.21.15.86:0/2272383752 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f03e4057fc0 con 0x7f03ec15fff0 2024-03-06T07:27:18.828 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03d006b4d0 msgr2=0x7f03d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03d006b4d0 0x7f03d006d990 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f03e0006fd0 tx=0x7f03e0008040 comp rx=0 tx=0).stop 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 msgr2=0x7f03ec1e9e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec1e9e00 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f03e400c7b0 tx=0x7f03e400cc70 comp rx=0 tx=0).stop 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 shutdown_connections 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03d006b4d0 0x7f03d006d990 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03ec15fff0 0x7f03ec1e9e00 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03ec15fc10 0x7f03ec06de20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 --2- 172.21.15.86:0/2272383752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f03ec0ce4a0 0x7f03ec1518f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 >> 172.21.15.86:0/2272383752 conn(0x7f03ec0b1940 msgr2=0x7f03ec0cc9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 shutdown_connections 2024-03-06T07:27:18.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:18.828+0000 7f03f3400640 1 -- 172.21.15.86:0/2272383752 wait complete. 2024-03-06T07:27:18.848 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:19.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.052+0000 7f7541200640 1 Processor -- start 2024-03-06T07:27:19.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.052+0000 7f7541200640 1 -- start start 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.053+0000 7f7541200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f753c15eff0 0x7f753c070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.053+0000 7f7541200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c06e4d0 0x7f753c06e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.053+0000 7f7541200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f753c071d30 con 0x7f753c0af1c0 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.053+0000 7f7541200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f753c071260 con 0x7f753c15eff0 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.053+0000 7f7541200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f753c15f4a0 con 0x7f753c06e4d0 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f7531400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f753c15eff0 0x7f753c070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f7531400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f753c15eff0 0x7f753c070d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34458/0 (socket says 172.21.15.86:34458) 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f7531400640 1 -- 172.21.15.86:0/1540402322 learned_addr learned my addr 172.21.15.86:0/1540402322 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f753a800640 1 --2- 172.21.15.86:0/1540402322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c06e4d0 0x7f753c06e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f7531400640 1 -- 172.21.15.86:0/1540402322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c06e4d0 msgr2=0x7f753c06e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f7531400640 1 --2- 172.21.15.86:0/1540402322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c06e4d0 0x7f753c06e960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f7531400640 1 -- 172.21.15.86:0/1540402322 >> v1:172.21.15.194:6789/0 conn(0x7f753c0af1c0 legacy=0x7f753c06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:19.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f7531400640 1 -- 172.21.15.86:0/1540402322 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f753c071490 con 0x7f753c15eff0 2024-03-06T07:27:19.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.054+0000 7f753a800640 1 --2- 172.21.15.86:0/1540402322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c06e4d0 0x7f753c06e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.055+0000 7f7531400640 1 --2- 172.21.15.86:0/1540402322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c070d20 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f75240098a0 tx=0x7f752402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7e0f687552ea1c04 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.055+0000 7f7539400640 1 -- 172.21.15.86:0/1540402322 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7524035020 con 0x7f753c15eff0 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7539400640 1 -- 172.21.15.86:0/1540402322 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7524031640 con 0x7f753c15eff0 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7539400640 1 -- 172.21.15.86:0/1540402322 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f752402d040 con 0x7f753c15eff0 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 -- 172.21.15.86:0/1540402322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 msgr2=0x7f753c070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 --2- 172.21.15.86:0/1540402322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c070d20 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f75240098a0 tx=0x7f752402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 -- 172.21.15.86:0/1540402322 shutdown_connections 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 --2- 172.21.15.86:0/1540402322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c06e4d0 0x7f753c06e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 --2- 172.21.15.86:0/1540402322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c070d20 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 -- 172.21.15.86:0/1540402322 >> 172.21.15.86:0/1540402322 conn(0x7f753c0b1730 msgr2=0x7f753c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:19.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 -- 172.21.15.86:0/1540402322 shutdown_connections 2024-03-06T07:27:19.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.056+0000 7f7541200640 1 -- 172.21.15.86:0/1540402322 wait complete. 2024-03-06T07:27:19.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.058+0000 7f7541200640 1 Processor -- start 2024-03-06T07:27:19.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.058+0000 7f7541200640 1 -- start start 2024-03-06T07:27:19.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f7541200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f753c06e4d0 0x7f753c14b340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f7541200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c0af1c0 0x7f753c0cdba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f7541200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c15cf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f7541200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f753c0b30b0 con 0x7f753c15eff0 2024-03-06T07:27:19.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f7541200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f753c071a50 con 0x7f753c06e4d0 2024-03-06T07:27:19.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f7541200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f753c06d370 con 0x7f753c0af1c0 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f753a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c15cf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f753a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c15cf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34468/0 (socket says 172.21.15.86:34468) 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.059+0000 7f753a800640 1 -- 172.21.15.86:0/528952732 learned_addr learned my addr 172.21.15.86:0/528952732 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f7531400640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c0af1c0 0x7f753c0cdba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f7539e00640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f753c06e4d0 0x7f753c14b340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f753a800640 1 -- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c0af1c0 msgr2=0x7f753c0cdba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f753a800640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c0af1c0 0x7f753c0cdba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f753a800640 1 -- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f753c06e4d0 msgr2=0x7f753c14b340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f753a800640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f753c06e4d0 0x7f753c14b340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f753a800640 1 -- 172.21.15.86:0/528952732 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f753c0cf700 con 0x7f753c15eff0 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f753a800640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c15cf00 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f753400cd50 tx=0x7f753400a650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f7532a00640 1 -- 172.21.15.86:0/528952732 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f753400f680 con 0x7f753c15eff0 2024-03-06T07:27:19.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f7532a00640 1 -- 172.21.15.86:0/528952732 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f753400f840 con 0x7f753c15eff0 2024-03-06T07:27:19.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f7532a00640 1 -- 172.21.15.86:0/528952732 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f753400fc30 con 0x7f753c15eff0 2024-03-06T07:27:19.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.060+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f753c0ce660 con 0x7f753c15eff0 2024-03-06T07:27:19.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.061+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f753c0cec20 con 0x7f753c15eff0 2024-03-06T07:27:19.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.062+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f753c04f330 con 0x7f753c15eff0 2024-03-06T07:27:19.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.062+0000 7f7532a00640 1 -- 172.21.15.86:0/528952732 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f753400e3e0 con 0x7f753c15eff0 2024-03-06T07:27:19.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.063+0000 7f7532a00640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f751806b4d0 0x7f751806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.063+0000 7f7532a00640 1 -- 172.21.15.86:0/528952732 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f753408b0a0 con 0x7f753c15eff0 2024-03-06T07:27:19.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.064+0000 7f7539e00640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f751806b4d0 0x7f751806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.064+0000 7f7539e00640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f751806b4d0 0x7f751806d990 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f752c004ea0 tx=0x7f752c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.068+0000 7f7532a00640 1 -- 172.21.15.86:0/528952732 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7534014030 con 0x7f753c15eff0 2024-03-06T07:27:19.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.303+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f753c0b1b10 con 0x7f753c15eff0 2024-03-06T07:27:19.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.304+0000 7f7532a00640 1 -- 172.21.15.86:0/528952732 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f7534059e80 con 0x7f753c15eff0 2024-03-06T07:27:19.307 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:19.307 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:27:19.307 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f751806b4d0 msgr2=0x7f751806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f751806b4d0 0x7f751806d990 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f752c004ea0 tx=0x7f752c009040 comp rx=0 tx=0).stop 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 msgr2=0x7f753c15cf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c15cf00 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f753400cd50 tx=0x7f753400a650 comp rx=0 tx=0).stop 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 shutdown_connections 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f751806b4d0 0x7f751806d990 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f753c15eff0 0x7f753c15cf00 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f753c0af1c0 0x7f753c0cdba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 --2- 172.21.15.86:0/528952732 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f753c06e4d0 0x7f753c14b340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 >> 172.21.15.86:0/528952732 conn(0x7f753c0b1730 msgr2=0x7f753c15e7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:19.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 shutdown_connections 2024-03-06T07:27:19.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.307+0000 7f7541200640 1 -- 172.21.15.86:0/528952732 wait complete. 2024-03-06T07:27:19.329 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:19.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.528+0000 7f2a47000640 1 Processor -- start 2024-03-06T07:27:19.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.528+0000 7f2a47000640 1 -- start start 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a47000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2a40144c00 0x7f2a4006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a47000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2a4006ee70 con 0x7f2a4015f9d0 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a47000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2a4006cef0 con 0x7f2a4015fe60 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a47000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2a4006d1d0 con 0x7f2a40144c00 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a3fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2a40144c00 0x7f2a4006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a3fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2a40144c00 0x7f2a4006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34474/0 (socket says 172.21.15.86:34474) 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a3fe00640 1 -- 172.21.15.86:0/3312532977 learned_addr learned my addr 172.21.15.86:0/3312532977 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a3fe00640 1 -- 172.21.15.86:0/3312532977 >> v1:172.21.15.194:6789/0 conn(0x7f2a4015fe60 legacy=0x7f2a400ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.530+0000 7f2a3fe00640 1 -- 172.21.15.86:0/3312532977 >> v1:172.21.15.86:6789/0 conn(0x7f2a4015f9d0 legacy=0x7f2a400ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.531+0000 7f2a3fe00640 1 -- 172.21.15.86:0/3312532977 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a4006d4b0 con 0x7f2a40144c00 2024-03-06T07:27:19.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.531+0000 7f2a3ea00640 1 -- 172.21.15.86:0/3312532977 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4235772147 0 0) 0x7f2a2c001110 con 0x7f2a4015f9d0 2024-03-06T07:27:19.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.532+0000 7f2a3fe00640 1 --2- 172.21.15.86:0/3312532977 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4006e930 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f2a30009870 tx=0x7f2a3002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5ef129fd5a3443c3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.532+0000 7f2a3ea00640 1 -- 172.21.15.86:0/3312532977 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a30034070 con 0x7f2a40144c00 2024-03-06T07:27:19.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.532+0000 7f2a3ea00640 1 -- 172.21.15.86:0/3312532977 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a30030860 con 0x7f2a40144c00 2024-03-06T07:27:19.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.532+0000 7f2a3ea00640 1 -- 172.21.15.86:0/3312532977 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a30030c30 con 0x7f2a40144c00 2024-03-06T07:27:19.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.532+0000 7f2a47000640 1 -- 172.21.15.86:0/3312532977 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 msgr2=0x7f2a4006e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.532+0000 7f2a47000640 1 --2- 172.21.15.86:0/3312532977 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4006e930 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f2a30009870 tx=0x7f2a3002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.533+0000 7f2a47000640 1 -- 172.21.15.86:0/3312532977 shutdown_connections 2024-03-06T07:27:19.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.533+0000 7f2a47000640 1 --2- 172.21.15.86:0/3312532977 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4006e930 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.533+0000 7f2a47000640 1 -- 172.21.15.86:0/3312532977 >> 172.21.15.86:0/3312532977 conn(0x7f2a400b1730 msgr2=0x7f2a400b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:19.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.533+0000 7f2a47000640 1 -- 172.21.15.86:0/3312532977 shutdown_connections 2024-03-06T07:27:19.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.533+0000 7f2a47000640 1 -- 172.21.15.86:0/3312532977 wait complete. 2024-03-06T07:27:19.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.534+0000 7f2a47000640 1 Processor -- start 2024-03-06T07:27:19.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.535+0000 7f2a47000640 1 -- start start 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.535+0000 7f2a47000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4015ee90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.535+0000 7f2a47000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a4015f9d0 0x7f2a401f8120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.535+0000 7f2a47000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a4015fe60 0x7f2a40070c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a47000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a40160730 con 0x7f2a40144c00 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a47000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a40160270 con 0x7f2a4015f9d0 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a47000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a400ceec0 con 0x7f2a4015fe60 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4015ee90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4015ee90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34490/0 (socket says 172.21.15.86:34490) 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 -- 172.21.15.86:0/216770492 learned_addr learned my addr 172.21.15.86:0/216770492 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3f400640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a4015f9d0 0x7f2a401f8120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 -- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a4015fe60 msgr2=0x7f2a40070c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a45000640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a4015fe60 0x7f2a40070c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a4015fe60 0x7f2a40070c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 -- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a4015f9d0 msgr2=0x7f2a401f8120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a4015f9d0 0x7f2a401f8120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 -- 172.21.15.86:0/216770492 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a4015c3e0 con 0x7f2a40144c00 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3f400640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a4015f9d0 0x7f2a401f8120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.536+0000 7f2a3fe00640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4015ee90 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f2a30000c00 tx=0x7f2a300046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.537+0000 7f2a37e00640 1 -- 172.21.15.86:0/216770492 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a30034040 con 0x7f2a40144c00 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.537+0000 7f2a37e00640 1 -- 172.21.15.86:0/216770492 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a30039d90 con 0x7f2a40144c00 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.537+0000 7f2a37e00640 1 -- 172.21.15.86:0/216770492 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a30043610 con 0x7f2a40144c00 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.537+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a4015c680 con 0x7f2a40144c00 2024-03-06T07:27:19.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.537+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a4015cc40 con 0x7f2a40144c00 2024-03-06T07:27:19.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.538+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2a40144740 con 0x7f2a40144c00 2024-03-06T07:27:19.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.539+0000 7f2a37e00640 1 -- 172.21.15.86:0/216770492 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2a3004a060 con 0x7f2a40144c00 2024-03-06T07:27:19.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.540+0000 7f2a37e00640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a2006b4d0 0x7f2a2006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.540+0000 7f2a37e00640 1 -- 172.21.15.86:0/216770492 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2a300ac460 con 0x7f2a40144c00 2024-03-06T07:27:19.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.540+0000 7f2a3f400640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a2006b4d0 0x7f2a2006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.541+0000 7f2a3f400640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a2006b4d0 0x7f2a2006d990 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f2a40071880 tx=0x7f2a2c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.544+0000 7f2a37e00640 1 -- 172.21.15.86:0/216770492 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2a3003e070 con 0x7f2a40144c00 2024-03-06T07:27:19.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.739+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2a4004f330 con 0x7f2a40144c00 2024-03-06T07:27:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.741+0000 7f2a37e00640 1 -- 172.21.15.86:0/216770492 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f2a30037360 con 0x7f2a40144c00 2024-03-06T07:27:19.745 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:19.745 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a2006b4d0 msgr2=0x7f2a2006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a2006b4d0 0x7f2a2006d990 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f2a40071880 tx=0x7f2a2c009040 comp rx=0 tx=0).stop 2024-03-06T07:27:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 msgr2=0x7f2a4015ee90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4015ee90 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f2a30000c00 tx=0x7f2a300046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 shutdown_connections 2024-03-06T07:27:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a2006b4d0 0x7f2a2006d990 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a4015fe60 0x7f2a40070c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a4015f9d0 0x7f2a401f8120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 --2- 172.21.15.86:0/216770492 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a40144c00 0x7f2a4015ee90 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 >> 172.21.15.86:0/216770492 conn(0x7f2a400b1730 msgr2=0x7f2a400ce060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:19.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.744+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 shutdown_connections 2024-03-06T07:27:19.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.745+0000 7f2a47000640 1 -- 172.21.15.86:0/216770492 wait complete. 2024-03-06T07:27:19.767 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:19.767 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:19.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.980+0000 7f42a5400640 1 Processor -- start 2024-03-06T07:27:19.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.980+0000 7f42a5400640 1 -- start start 2024-03-06T07:27:19.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f42a5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fc10 0x7f42a0070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f42a5400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f42a015fff0 0x7f42a0160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f42a5400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f42a00b46c0 con 0x7f42a00ce4a0 2024-03-06T07:27:19.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f42a5400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f42a00b49a0 con 0x7f42a015fff0 2024-03-06T07:27:19.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f42a5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f42a0072c90 con 0x7f42a015fc10 2024-03-06T07:27:19.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f429de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f42a015fff0 0x7f42a0160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f429de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f42a015fff0 0x7f42a0160450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34496/0 (socket says 172.21.15.86:34496) 2024-03-06T07:27:19.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f429de00640 1 -- 172.21.15.86:0/1375883079 learned_addr learned my addr 172.21.15.86:0/1375883079 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:19.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.982+0000 7f429e800640 1 --2- 172.21.15.86:0/1375883079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fc10 0x7f42a0070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.983+0000 7f429de00640 1 -- 172.21.15.86:0/1375883079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fc10 msgr2=0x7f42a0070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:19.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.983+0000 7f429de00640 1 --2- 172.21.15.86:0/1375883079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fc10 0x7f42a0070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.983+0000 7f429de00640 1 -- 172.21.15.86:0/1375883079 >> v1:172.21.15.86:6789/0 conn(0x7f42a00ce4a0 legacy=0x7f42a00ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.983+0000 7f429d400640 1 -- 172.21.15.86:0/1375883079 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1300992712 0 0) 0x7f42a00b46c0 con 0x7f42a00ce4a0 2024-03-06T07:27:19.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.983+0000 7f429de00640 1 -- 172.21.15.86:0/1375883079 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42a0160990 con 0x7f42a015fff0 2024-03-06T07:27:19.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.984+0000 7f429de00640 1 --2- 172.21.15.86:0/1375883079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a015fff0 0x7f42a0160450 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f428c009870 tx=0x7f428c02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d1487ed617df9305 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.984+0000 7f429d400640 1 -- 172.21.15.86:0/1375883079 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f428c034070 con 0x7f42a015fff0 2024-03-06T07:27:19.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.984+0000 7f429d400640 1 -- 172.21.15.86:0/1375883079 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f428c030840 con 0x7f42a015fff0 2024-03-06T07:27:19.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.984+0000 7f429d400640 1 -- 172.21.15.86:0/1375883079 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f428c030c10 con 0x7f42a015fff0 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 -- 172.21.15.86:0/1375883079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a015fff0 msgr2=0x7f42a0160450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 --2- 172.21.15.86:0/1375883079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a015fff0 0x7f42a0160450 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f428c009870 tx=0x7f428c02b980 comp rx=0 tx=0).stop 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 -- 172.21.15.86:0/1375883079 shutdown_connections 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 --2- 172.21.15.86:0/1375883079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a015fff0 0x7f42a0160450 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 --2- 172.21.15.86:0/1375883079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fc10 0x7f42a0070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 -- 172.21.15.86:0/1375883079 >> 172.21.15.86:0/1375883079 conn(0x7f42a00b1940 msgr2=0x7f42a00b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 -- 172.21.15.86:0/1375883079 shutdown_connections 2024-03-06T07:27:19.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.985+0000 7f42a5400640 1 -- 172.21.15.86:0/1375883079 wait complete. 2024-03-06T07:27:19.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.987+0000 7f42a5400640 1 Processor -- start 2024-03-06T07:27:19.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.987+0000 7f42a5400640 1 -- start start 2024-03-06T07:27:19.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f42a5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a00ce4a0 0x7f42a006e270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f42a5400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f42a015fc10 0x7f42a006d7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f42a5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fff0 0x7f42a009aad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f42a5400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42a0072a30 con 0x7f42a00ce4a0 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f42a5400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42a00728b0 con 0x7f42a015fc10 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f42a5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f42a015c2e0 con 0x7f42a015fff0 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f429e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a00ce4a0 0x7f42a006e270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f429e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a00ce4a0 0x7f42a006e270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34500/0 (socket says 172.21.15.86:34500) 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f429e800640 1 -- 172.21.15.86:0/1405390905 learned_addr learned my addr 172.21.15.86:0/1405390905 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f429de00640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f42a015fc10 0x7f42a006d7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.988+0000 7f429f200640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fff0 0x7f42a009aad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.989+0000 7f429e800640 1 -- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fff0 msgr2=0x7f42a009aad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.989+0000 7f429e800640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fff0 0x7f42a009aad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.989+0000 7f429e800640 1 -- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f42a015fc10 msgr2=0x7f42a006d7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:19.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.989+0000 7f429e800640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f42a015fc10 0x7f42a006d7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:19.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.989+0000 7f429e800640 1 -- 172.21.15.86:0/1405390905 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42a006ca30 con 0x7f42a00ce4a0 2024-03-06T07:27:19.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.989+0000 7f429f200640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fff0 0x7f42a009aad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:19.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.989+0000 7f429e800640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a00ce4a0 0x7f42a006e270 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f429000e9b0 tx=0x7f429000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:19.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.991+0000 7f4296a00640 1 -- 172.21.15.86:0/1405390905 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4290017070 con 0x7f42a00ce4a0 2024-03-06T07:27:19.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.991+0000 7f4296a00640 1 -- 172.21.15.86:0/1405390905 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f429000c9a0 con 0x7f42a00ce4a0 2024-03-06T07:27:19.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.991+0000 7f4296a00640 1 -- 172.21.15.86:0/1405390905 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f429000cd90 con 0x7f42a00ce4a0 2024-03-06T07:27:19.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.991+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42a006cd10 con 0x7f42a00ce4a0 2024-03-06T07:27:19.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.992+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42a006d2d0 con 0x7f42a00ce4a0 2024-03-06T07:27:19.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.993+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f42a004f330 con 0x7f42a00ce4a0 2024-03-06T07:27:19.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.994+0000 7f4296a00640 1 -- 172.21.15.86:0/1405390905 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f429001a070 con 0x7f42a00ce4a0 2024-03-06T07:27:19.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.994+0000 7f4296a00640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f428006b4d0 0x7f428006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:19.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.994+0000 7f4296a00640 1 -- 172.21.15.86:0/1405390905 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f429008a5e0 con 0x7f42a00ce4a0 2024-03-06T07:27:19.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.995+0000 7f429de00640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f428006b4d0 0x7f428006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:19.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.995+0000 7f429de00640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f428006b4d0 0x7f428006d990 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f428c036000 tx=0x7f428c03b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:20.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:19.999+0000 7f4296a00640 1 -- 172.21.15.86:0/1405390905 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4290059460 con 0x7f42a00ce4a0 2024-03-06T07:27:20.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.192+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f42a006ef30 con 0x7f42a00ce4a0 2024-03-06T07:27:20.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.193+0000 7f4296a00640 1 -- 172.21.15.86:0/1405390905 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f4290058d60 con 0x7f42a00ce4a0 2024-03-06T07:27:20.196 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.195+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f428006b4d0 msgr2=0x7f428006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.195+0000 7f42a5400640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f428006b4d0 0x7f428006d990 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f428c036000 tx=0x7f428c03b040 comp rx=0 tx=0).stop 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.195+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a00ce4a0 msgr2=0x7f42a006e270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.195+0000 7f42a5400640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a00ce4a0 0x7f42a006e270 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f429000e9b0 tx=0x7f429000ee70 comp rx=0 tx=0).stop 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 shutdown_connections 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f428006b4d0 0x7f428006d990 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f42a015fff0 0x7f42a009aad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f42a015fc10 0x7f42a006d7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 --2- 172.21.15.86:0/1405390905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f42a00ce4a0 0x7f42a006e270 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 >> 172.21.15.86:0/1405390905 conn(0x7f42a00b1940 msgr2=0x7f42a00acd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:20.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 shutdown_connections 2024-03-06T07:27:20.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.196+0000 7f42a5400640 1 -- 172.21.15.86:0/1405390905 wait complete. 2024-03-06T07:27:20.217 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.426+0000 7f7bf7200640 1 Processor -- start 2024-03-06T07:27:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.426+0000 7f7bf7200640 1 -- start start 2024-03-06T07:27:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.427+0000 7f7bf7200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7bf015be10 0x7f7bf00ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.427+0000 7f7bf7200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7bf015c1f0 con 0x7f7bf006da40 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.427+0000 7f7bf7200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7bf006cac0 con 0x7f7bf00aef20 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.427+0000 7f7bf7200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7bf00cd200 con 0x7f7bf015be10 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7bf015be10 0x7f7bf00ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf6800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7bf006da40 0x7f7bf0071ab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36852/0 (socket says 172.21.15.86:36852) 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7bf015be10 0x7f7bf00ccc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34510/0 (socket says 172.21.15.86:34510) 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf6800640 1 -- 172.21.15.86:0/2446752209 learned_addr learned my addr 172.21.15.86:0/2446752209 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf5e00640 1 -- 172.21.15.86:0/2446752209 >> v1:172.21.15.194:6789/0 conn(0x7f7bf00aef20 legacy=0x7f7bf006d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf5e00640 1 -- 172.21.15.86:0/2446752209 >> v1:172.21.15.86:6789/0 conn(0x7f7bf006da40 legacy=0x7f7bf0071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf5e00640 1 -- 172.21.15.86:0/2446752209 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7bf006cda0 con 0x7f7bf015be10 2024-03-06T07:27:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.428+0000 7f7bf4a00640 1 -- 172.21.15.86:0/2446752209 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 479149002 0 0) 0x7f7bf015c1f0 con 0x7f7bf006da40 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.429+0000 7f7bf5e00640 1 --2- 172.21.15.86:0/2446752209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ccc20 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f7be40097a0 tx=0x7f7be402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c1d780a54f3c9ae server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.429+0000 7f7bf4a00640 1 -- 172.21.15.86:0/2446752209 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7be4035020 con 0x7f7bf015be10 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.429+0000 7f7bf4a00640 1 -- 172.21.15.86:0/2446752209 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7be4031830 con 0x7f7bf015be10 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.429+0000 7f7bf4a00640 1 -- 172.21.15.86:0/2446752209 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7be4031c00 con 0x7f7bf015be10 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.430+0000 7f7bf7200640 1 -- 172.21.15.86:0/2446752209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 msgr2=0x7f7bf00ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.430+0000 7f7bf7200640 1 --2- 172.21.15.86:0/2446752209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ccc20 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f7be40097a0 tx=0x7f7be402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.430+0000 7f7bf7200640 1 -- 172.21.15.86:0/2446752209 shutdown_connections 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.430+0000 7f7bf7200640 1 --2- 172.21.15.86:0/2446752209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ccc20 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.430+0000 7f7bf7200640 1 -- 172.21.15.86:0/2446752209 >> 172.21.15.86:0/2446752209 conn(0x7f7bf006afc0 msgr2=0x7f7bf006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.430+0000 7f7bf7200640 1 -- 172.21.15.86:0/2446752209 shutdown_connections 2024-03-06T07:27:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.430+0000 7f7bf7200640 1 -- 172.21.15.86:0/2446752209 wait complete. 2024-03-06T07:27:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.432+0000 7f7bf7200640 1 Processor -- start 2024-03-06T07:27:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.432+0000 7f7bf7200640 1 -- start start 2024-03-06T07:27:20.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf7200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7bf006da40 0x7f7bf01496a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf7200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7bf00aef20 0x7f7bf015c1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ad860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf7200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7bf00cda90 con 0x7f7bf015be10 2024-03-06T07:27:20.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf7200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7bf006d080 con 0x7f7bf006da40 2024-03-06T07:27:20.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf7200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7bf00ce2b0 con 0x7f7bf00aef20 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ad860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ad860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34524/0 (socket says 172.21.15.86:34524) 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf6800640 1 -- 172.21.15.86:0/4179085465 learned_addr learned my addr 172.21.15.86:0/4179085465 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf5e00640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7bf006da40 0x7f7bf01496a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf6800640 1 -- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7bf00aef20 msgr2=0x7f7bf015c1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.433+0000 7f7bf5400640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7bf00aef20 0x7f7bf015c1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf6800640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7bf00aef20 0x7f7bf015c1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf6800640 1 -- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7bf006da40 msgr2=0x7f7bf01496a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf6800640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7bf006da40 0x7f7bf01496a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf6800640 1 -- 172.21.15.86:0/4179085465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7bf00b2a10 con 0x7f7bf015be10 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf5400640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7bf00aef20 0x7f7bf015c1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf6800640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ad860 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f7bec00c770 tx=0x7f7bec00cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bea000640 1 -- 172.21.15.86:0/4179085465 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7bec018070 con 0x7f7bf015be10 2024-03-06T07:27:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bea000640 1 -- 172.21.15.86:0/4179085465 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7bec00a640 con 0x7f7bf015be10 2024-03-06T07:27:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bea000640 1 -- 172.21.15.86:0/4179085465 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7bec00aa30 con 0x7f7bf015be10 2024-03-06T07:27:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7bf015ccb0 con 0x7f7bf015be10 2024-03-06T07:27:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.434+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7bf0146720 con 0x7f7bf015be10 2024-03-06T07:27:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.435+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7bf004f330 con 0x7f7bf015be10 2024-03-06T07:27:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.437+0000 7f7bea000640 1 -- 172.21.15.86:0/4179085465 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7bec00abd0 con 0x7f7bf015be10 2024-03-06T07:27:20.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.438+0000 7f7bea000640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7bd006b4d0 0x7f7bd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.438+0000 7f7bea000640 1 -- 172.21.15.86:0/4179085465 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7bec0897d0 con 0x7f7bf015be10 2024-03-06T07:27:20.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.438+0000 7f7bf5e00640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7bd006b4d0 0x7f7bd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.439+0000 7f7bf5e00640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7bd006b4d0 0x7f7bd006d990 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f7be4036000 tx=0x7f7be403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:20.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.442+0000 7f7bea000640 1 -- 172.21.15.86:0/4179085465 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7bec0586e0 con 0x7f7bf015be10 2024-03-06T07:27:20.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.639+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f7bf0160670 con 0x7f7bf015be10 2024-03-06T07:27:20.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.640+0000 7f7bea000640 1 -- 172.21.15.86:0/4179085465 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7f7bec057fe0 con 0x7f7bf015be10 2024-03-06T07:27:20.643 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.642+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7bd006b4d0 msgr2=0x7f7bd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.642+0000 7f7bf7200640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7bd006b4d0 0x7f7bd006d990 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f7be4036000 tx=0x7f7be403c040 comp rx=0 tx=0).stop 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.642+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 msgr2=0x7f7bf00ad860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.642+0000 7f7bf7200640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ad860 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f7bec00c770 tx=0x7f7bec00cc30 comp rx=0 tx=0).stop 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 shutdown_connections 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7bd006b4d0 0x7f7bd006d990 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7bf015be10 0x7f7bf00ad860 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7bf00aef20 0x7f7bf015c1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 --2- 172.21.15.86:0/4179085465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7bf006da40 0x7f7bf01496a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 >> 172.21.15.86:0/4179085465 conn(0x7f7bf006afc0 msgr2=0x7f7bf0156070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 shutdown_connections 2024-03-06T07:27:20.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.643+0000 7f7bf7200640 1 -- 172.21.15.86:0/4179085465 wait complete. 2024-03-06T07:27:20.663 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:20.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.874+0000 7f293ec00640 1 Processor -- start 2024-03-06T07:27:20.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.874+0000 7f293ec00640 1 -- start start 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ec00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f2938158570 0x7f29380ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ec00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158950 0x7f29380cf6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ec00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f29380aebf0 0x7f29380cfc10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ec00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f293815b130 con 0x7f29380aebf0 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ec00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f29380af170 con 0x7f2938158570 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ec00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f293815b3f0 con 0x7f2938158950 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f29380aebf0 0x7f29380cfc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f29380aebf0 0x7f29380cfc10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34532/0 (socket says 172.21.15.86:34532) 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 -- 172.21.15.86:0/3675193969 learned_addr learned my addr 172.21.15.86:0/3675193969 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293d800640 1 --2- 172.21.15.86:0/3675193969 >> v2:172.21.15.194:3300/0 conn(0x7f2938158570 0x7f29380ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ce00640 1 --2- 172.21.15.86:0/3675193969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158950 0x7f29380cf6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 -- 172.21.15.86:0/3675193969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158950 msgr2=0x7f29380cf6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 --2- 172.21.15.86:0/3675193969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158950 0x7f29380cf6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 -- 172.21.15.86:0/3675193969 >> v2:172.21.15.194:3300/0 conn(0x7f2938158570 msgr2=0x7f29380ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 --2- 172.21.15.86:0/3675193969 >> v2:172.21.15.194:3300/0 conn(0x7f2938158570 0x7f29380ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293e200640 1 -- 172.21.15.86:0/3675193969 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f293815b6d0 con 0x7f29380aebf0 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293ce00640 1 --2- 172.21.15.86:0/3675193969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158950 0x7f29380cf6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:20.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.875+0000 7f293d800640 1 --2- 172.21.15.86:0/3675193969 >> v2:172.21.15.194:3300/0 conn(0x7f2938158570 0x7f29380ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.876+0000 7f293e200640 1 --2- 172.21.15.86:0/3675193969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f29380cfc10 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f293806eb20 tx=0x7f293002be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b7f60e25165fcf95 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f2937e00640 1 -- 172.21.15.86:0/3675193969 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f293002c9f0 con 0x7f29380aebf0 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f2937e00640 1 -- 172.21.15.86:0/3675193969 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f293002cb90 con 0x7f29380aebf0 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f2937e00640 1 -- 172.21.15.86:0/3675193969 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29300343f0 con 0x7f29380aebf0 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f293ec00640 1 -- 172.21.15.86:0/3675193969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 msgr2=0x7f29380cfc10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f293ec00640 1 --2- 172.21.15.86:0/3675193969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f29380cfc10 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f293806eb20 tx=0x7f293002be50 comp rx=0 tx=0).stop 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f293ec00640 1 -- 172.21.15.86:0/3675193969 shutdown_connections 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f293ec00640 1 --2- 172.21.15.86:0/3675193969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f29380cfc10 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f293ec00640 1 --2- 172.21.15.86:0/3675193969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158950 0x7f29380cf6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f293ec00640 1 --2- 172.21.15.86:0/3675193969 >> v2:172.21.15.194:3300/0 conn(0x7f2938158570 0x7f29380ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.877+0000 7f293ec00640 1 -- 172.21.15.86:0/3675193969 >> 172.21.15.86:0/3675193969 conn(0x7f293806afc0 msgr2=0x7f293806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:20.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.878+0000 7f293ec00640 1 -- 172.21.15.86:0/3675193969 shutdown_connections 2024-03-06T07:27:20.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.878+0000 7f293ec00640 1 -- 172.21.15.86:0/3675193969 wait complete. 2024-03-06T07:27:20.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.879+0000 7f293ec00640 1 Processor -- start 2024-03-06T07:27:20.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.879+0000 7f293ec00640 1 -- start start 2024-03-06T07:27:20.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.880+0000 7f293ec00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f2938144200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.880+0000 7f293ec00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158570 0x7f2938156aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.880+0000 7f293ec00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2938158950 0x7f2938070970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.880+0000 7f293ec00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f293815bed0 con 0x7f29380aebf0 2024-03-06T07:27:20.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.880+0000 7f293ec00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29380b4950 con 0x7f2938158950 2024-03-06T07:27:20.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.880+0000 7f293ec00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f293815c050 con 0x7f2938158570 2024-03-06T07:27:20.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.880+0000 7f293d800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f2938144200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f2938144200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34548/0 (socket says 172.21.15.86:34548) 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 -- 172.21.15.86:0/1688460528 learned_addr learned my addr 172.21.15.86:0/1688460528 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293e200640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2938158950 0x7f2938070970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 -- 172.21.15.86:0/1688460528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158570 msgr2=0x7f2938156aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158570 0x7f2938156aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 -- 172.21.15.86:0/1688460528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2938158950 msgr2=0x7f2938070970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2938158950 0x7f2938070970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 -- 172.21.15.86:0/1688460528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2938154fa0 con 0x7f29380aebf0 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293d800640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f2938144200 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f292c00c7f0 tx=0x7f292c00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f2935600640 1 -- 172.21.15.86:0/1688460528 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f292c015070 con 0x7f29380aebf0 2024-03-06T07:27:20.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f2935600640 1 -- 172.21.15.86:0/1688460528 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f292c007820 con 0x7f29380aebf0 2024-03-06T07:27:20.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f2935600640 1 -- 172.21.15.86:0/1688460528 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f292c007bd0 con 0x7f29380aebf0 2024-03-06T07:27:20.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.881+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2938155280 con 0x7f29380aebf0 2024-03-06T07:27:20.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.882+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2938155560 con 0x7f29380aebf0 2024-03-06T07:27:20.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.883+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f293804f330 con 0x7f29380aebf0 2024-03-06T07:27:20.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.884+0000 7f2935600640 1 -- 172.21.15.86:0/1688460528 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f292c018070 con 0x7f29380aebf0 2024-03-06T07:27:20.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.885+0000 7f2935600640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f291806b4d0 0x7f291806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:20.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.885+0000 7f2935600640 1 -- 172.21.15.86:0/1688460528 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f292c08a380 con 0x7f29380aebf0 2024-03-06T07:27:20.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.885+0000 7f293ce00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f291806b4d0 0x7f291806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:20.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.886+0000 7f293ce00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f291806b4d0 0x7f291806d990 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f29381f7640 tx=0x7f2924008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:20.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:20.890+0000 7f2935600640 1 -- 172.21.15.86:0/1688460528 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f292c0591f0 con 0x7f29380aebf0 2024-03-06T07:27:21.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.090+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f29380ce750 con 0x7f29380aebf0 2024-03-06T07:27:21.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.091+0000 7f2935600640 1 -- 172.21.15.86:0/1688460528 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f292c058af0 con 0x7f29380aebf0 2024-03-06T07:27:21.093 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:21.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f291806b4d0 msgr2=0x7f291806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f291806b4d0 0x7f291806d990 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f29381f7640 tx=0x7f2924008040 comp rx=0 tx=0).stop 2024-03-06T07:27:21.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 msgr2=0x7f2938144200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f2938144200 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f292c00c7f0 tx=0x7f292c00ccb0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 shutdown_connections 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f291806b4d0 0x7f291806d990 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2938158950 0x7f2938070970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2938158570 0x7f2938156aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 --2- 172.21.15.86:0/1688460528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29380aebf0 0x7f2938144200 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 >> 172.21.15.86:0/1688460528 conn(0x7f293806afc0 msgr2=0x7f29380ad860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 shutdown_connections 2024-03-06T07:27:21.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.093+0000 7f293ec00640 1 -- 172.21.15.86:0/1688460528 wait complete. 2024-03-06T07:27:21.115 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:21.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.310+0000 7fa3d6e00640 1 Processor -- start 2024-03-06T07:27:21.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.310+0000 7fa3d6e00640 1 -- start start 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d0161580 0x7fa3d00d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa3d00d2a70 0x7fa3d00b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa3d0161960 con 0x7fa3d00d1e90 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa3d00cf6f0 con 0x7fa3d00d2a70 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa3d00cf9d0 con 0x7fa3d0161580 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa3d00d2a70 0x7fa3d00b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa3d00d2a70 0x7fa3d00b3ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34556/0 (socket says 172.21.15.86:34556) 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d6400640 1 -- 172.21.15.86:0/3901591054 learned_addr learned my addr 172.21.15.86:0/3901591054 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:21.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.311+0000 7fa3d5a00640 1 --2- 172.21.15.86:0/3901591054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d0161580 0x7fa3d00d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.312+0000 7fa3d6400640 1 -- 172.21.15.86:0/3901591054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d0161580 msgr2=0x7fa3d00d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:21.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.312+0000 7fa3d6400640 1 --2- 172.21.15.86:0/3901591054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d0161580 0x7fa3d00d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.312+0000 7fa3d6400640 1 -- 172.21.15.86:0/3901591054 >> v1:172.21.15.194:6789/0 conn(0x7fa3d00d1e90 legacy=0x7fa3d00d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.312+0000 7fa3d6400640 1 -- 172.21.15.86:0/3901591054 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3d00cfcb0 con 0x7fa3d00d2a70 2024-03-06T07:27:21.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6400640 1 --2- 172.21.15.86:0/3901591054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d2a70 0x7fa3d00b3ec0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fa3c8009e50 tx=0x7fa3c802bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b33f20da1e11d211 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:21.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3cfe00640 1 -- 172.21.15.86:0/3901591054 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3c802cb00 con 0x7fa3d00d2a70 2024-03-06T07:27:21.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3cfe00640 1 -- 172.21.15.86:0/3901591054 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa3c802cca0 con 0x7fa3d00d2a70 2024-03-06T07:27:21.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3cfe00640 1 -- 172.21.15.86:0/3901591054 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3c8034610 con 0x7fa3d00d2a70 2024-03-06T07:27:21.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 -- 172.21.15.86:0/3901591054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d2a70 msgr2=0x7fa3d00b3ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/3901591054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d2a70 0x7fa3d00b3ec0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fa3c8009e50 tx=0x7fa3c802bf60 comp rx=0 tx=0).stop 2024-03-06T07:27:21.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 -- 172.21.15.86:0/3901591054 shutdown_connections 2024-03-06T07:27:21.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/3901591054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d2a70 0x7fa3d00b3ec0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/3901591054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d0161580 0x7fa3d00d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 -- 172.21.15.86:0/3901591054 >> 172.21.15.86:0/3901591054 conn(0x7fa3d00d3d80 msgr2=0x7fa3d00d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:21.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 -- 172.21.15.86:0/3901591054 shutdown_connections 2024-03-06T07:27:21.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.314+0000 7fa3d6e00640 1 -- 172.21.15.86:0/3901591054 wait complete. 2024-03-06T07:27:21.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.315+0000 7fa3d6e00640 1 Processor -- start 2024-03-06T07:27:21.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.316+0000 7fa3d6e00640 1 -- start start 2024-03-06T07:27:21.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.316+0000 7fa3d6e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d1e90 0x7fa3d0151410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.316+0000 7fa3d6e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d00d2a70 0x7fa3d00c7410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.316+0000 7fa3d6e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3d0161580 0x7fa3d00cc2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.316+0000 7fa3d6e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3d00cf3a0 con 0x7fa3d00d1e90 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.316+0000 7fa3d6e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3d00b39a0 con 0x7fa3d0161580 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.316+0000 7fa3d6e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa3d00b4400 con 0x7fa3d00d2a70 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d1e90 0x7fa3d0151410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d1e90 0x7fa3d0151410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34562/0 (socket says 172.21.15.86:34562) 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 -- 172.21.15.86:0/2451764974 learned_addr learned my addr 172.21.15.86:0/2451764974 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 -- 172.21.15.86:0/2451764974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d00d2a70 msgr2=0x7fa3d00c7410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d00d2a70 0x7fa3d00c7410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 -- 172.21.15.86:0/2451764974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3d0161580 msgr2=0x7fa3d00cc2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3d0161580 0x7fa3d00cc2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 -- 172.21.15.86:0/2451764974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3d00ca9b0 con 0x7fa3d00d1e90 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3d5a00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d1e90 0x7fa3d0151410 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fa3c400d620 tx=0x7fa3c400dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.317+0000 7fa3cd600640 1 -- 172.21.15.86:0/2451764974 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3c400b620 con 0x7fa3d00d1e90 2024-03-06T07:27:21.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.318+0000 7fa3cd600640 1 -- 172.21.15.86:0/2451764974 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa3c4019070 con 0x7fa3d00d1e90 2024-03-06T07:27:21.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.318+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa3d00b7a80 con 0x7fa3d00d1e90 2024-03-06T07:27:21.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.318+0000 7fa3cd600640 1 -- 172.21.15.86:0/2451764974 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3c400ba00 con 0x7fa3d00d1e90 2024-03-06T07:27:21.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.318+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa3d0151950 con 0x7fa3d00d1e90 2024-03-06T07:27:21.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.319+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa3d004f330 con 0x7fa3d00d1e90 2024-03-06T07:27:21.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.319+0000 7fa3cd600640 1 -- 172.21.15.86:0/2451764974 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa3c400bba0 con 0x7fa3d00d1e90 2024-03-06T07:27:21.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.320+0000 7fa3cd600640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3b006b4d0 0x7fa3b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.320+0000 7fa3cd600640 1 -- 172.21.15.86:0/2451764974 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa3c408a6e0 con 0x7fa3d00d1e90 2024-03-06T07:27:21.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.321+0000 7fa3d5000640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3b006b4d0 0x7fa3b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.321+0000 7fa3d5000640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3b006b4d0 0x7fa3b006d990 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fa3bc000f60 tx=0x7fa3bc0079b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:21.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.325+0000 7fa3cd600640 1 -- 172.21.15.86:0/2451764974 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa3c40594c0 con 0x7fa3d00d1e90 2024-03-06T07:27:21.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.556+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa3d00d3b60 con 0x7fa3d00d1e90 2024-03-06T07:27:21.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.557+0000 7fa3cd600640 1 -- 172.21.15.86:0/2451764974 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7fa3c4058dc0 con 0x7fa3d00d1e90 2024-03-06T07:27:21.560 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:21.560 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:27:21.561 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3b006b4d0 msgr2=0x7fa3b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3b006b4d0 0x7fa3b006d990 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fa3bc000f60 tx=0x7fa3bc0079b0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d1e90 msgr2=0x7fa3d0151410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d1e90 0x7fa3d0151410 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fa3c400d620 tx=0x7fa3c400dae0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 shutdown_connections 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3b006b4d0 0x7fa3b006d990 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3d0161580 0x7fa3d00cc2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3d00d2a70 0x7fa3d00c7410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 --2- 172.21.15.86:0/2451764974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3d00d1e90 0x7fa3d0151410 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 >> 172.21.15.86:0/2451764974 conn(0x7fa3d00d3d80 msgr2=0x7fa3d00b47c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 shutdown_connections 2024-03-06T07:27:21.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.561+0000 7fa3d6e00640 1 -- 172.21.15.86:0/2451764974 wait complete. 2024-03-06T07:27:21.579 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:21.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.784+0000 7f54d1600640 1 Processor -- start 2024-03-06T07:27:21.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.784+0000 7f54d1600640 1 -- start start 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54d1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 0x7f54cc0affe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54d1600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f54cc0cef70 con 0x7f54cc15dfd0 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54d1600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f54cc15e3b0 con 0x7f54cc06b6d0 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54d1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f54cc162270 con 0x7f54cc162700 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54cbe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f54cc15dfd0 0x7f54cc06afc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36866/0 (socket says 172.21.15.86:36866) 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54cbe00640 1 -- 172.21.15.86:0/1536787537 learned_addr learned my addr 172.21.15.86:0/1536787537 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1101486237 0 0) 0x7f54cc0cef70 con 0x7f54cc15dfd0 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54d0c00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 0x7f54cc0affe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.785+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f54ac003520 con 0x7f54cc15dfd0 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2295833068 0 0) 0x7f54ac003520 con 0x7f54cc15dfd0 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 msgr2=0x7f54cc0affe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 0x7f54cc0affe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 >> v1:172.21.15.194:6789/0 conn(0x7f54cc06b6d0 legacy=0x7f54cc0d06f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54cc161c60 con 0x7f54cc15dfd0 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f54bc001670 con 0x7f54cc15dfd0 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 >> v1:172.21.15.86:6789/0 conn(0x7f54cc15dfd0 legacy=0x7f54cc06afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 0x7f54ac0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54ac004010 0x7f54ac004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54ac004d90 0x7f54ac005210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54cc162270 con 0x7f54ac004010 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54ac00bca0 con 0x7f54ac004d90 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f54ac00bea0 con 0x7f54cc162700 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54cb400640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54ac004010 0x7f54ac004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54cbe00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 0x7f54ac0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54d0c00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54ac004d90 0x7f54ac005210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54cb400640 1 -- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 msgr2=0x7f54ac0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54cb400640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 0x7f54ac0039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54cb400640 1 -- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54ac004d90 msgr2=0x7f54ac005210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54cb400640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54ac004d90 0x7f54ac005210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.786+0000 7f54cb400640 1 -- 172.21.15.86:0/1536787537 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54ac003520 con 0x7f54ac004010 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.787+0000 7f54cbe00640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54cc162700 0x7f54ac0039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.787+0000 7f54cb400640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54ac004010 0x7f54ac004800 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f54b4001e20 tx=0x7f54ac005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1cb4a0503a3fe7de server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:21.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54b4001a70 con 0x7f54ac004010 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f54b4003b70 con 0x7f54ac004010 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54caa00640 1 -- 172.21.15.86:0/1536787537 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54b40050d0 con 0x7f54ac004010 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54d1600640 1 -- 172.21.15.86:0/1536787537 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54ac004010 msgr2=0x7f54ac004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54d1600640 1 --2- 172.21.15.86:0/1536787537 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54ac004010 0x7f54ac004800 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f54b4001e20 tx=0x7f54ac005830 comp rx=0 tx=0).stop 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54d0c00640 1 -- 172.21.15.86:0/1536787537 reap_dead start 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54d1600640 1 -- 172.21.15.86:0/1536787537 shutdown_connections 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54d1600640 1 -- 172.21.15.86:0/1536787537 >> 172.21.15.86:0/1536787537 conn(0x7f54cc06c0d0 msgr2=0x7f54cc06c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54d1600640 1 -- 172.21.15.86:0/1536787537 shutdown_connections 2024-03-06T07:27:21.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.788+0000 7f54d1600640 1 -- 172.21.15.86:0/1536787537 wait complete. 2024-03-06T07:27:21.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.790+0000 7f54d1600640 1 Processor -- start 2024-03-06T07:27:21.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.790+0000 7f54d1600640 1 -- start start 2024-03-06T07:27:21.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54ac004010 0x7f54cc06e470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d1600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54cc06e9b0 0x7f54cc158ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54cc159420 0x7f54cc1e89a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d1600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54cc0ce8d0 con 0x7f54cc159420 2024-03-06T07:27:21.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d1600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54cc161f00 con 0x7f54cc06e9b0 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f54cc0cea50 con 0x7f54ac004010 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54cc159420 0x7f54cc1e89a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54cc159420 0x7f54cc1e89a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34574/0 (socket says 172.21.15.86:34574) 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.791+0000 7f54d0c00640 1 -- 172.21.15.86:0/2662339948 learned_addr learned my addr 172.21.15.86:0/2662339948 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54cbe00640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54ac004010 0x7f54cc06e470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54cb400640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54cc06e9b0 0x7f54cc158ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d0c00640 1 -- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54ac004010 msgr2=0x7f54cc06e470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d0c00640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54ac004010 0x7f54cc06e470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d0c00640 1 -- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54cc06e9b0 msgr2=0x7f54cc158ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d0c00640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54cc06e9b0 0x7f54cc158ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d0c00640 1 -- 172.21.15.86:0/2662339948 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54cc1e7900 con 0x7f54cc159420 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d0c00640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54cc159420 0x7f54cc1e89a0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f54c4006fd0 tx=0x7f54c400e8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54c3e00640 1 -- 172.21.15.86:0/2662339948 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54c400f040 con 0x7f54cc159420 2024-03-06T07:27:21.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54c3e00640 1 -- 172.21.15.86:0/2662339948 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f54c4019070 con 0x7f54cc159420 2024-03-06T07:27:21.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f54cc1e7be0 con 0x7f54cc159420 2024-03-06T07:27:21.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54c3e00640 1 -- 172.21.15.86:0/2662339948 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54c4013850 con 0x7f54cc159420 2024-03-06T07:27:21.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.792+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54cc1e81a0 con 0x7f54cc159420 2024-03-06T07:27:21.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.793+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f54cc04f330 con 0x7f54cc159420 2024-03-06T07:27:21.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.794+0000 7f54c3e00640 1 -- 172.21.15.86:0/2662339948 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f54c40139f0 con 0x7f54cc159420 2024-03-06T07:27:21.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.796+0000 7f54c3e00640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f54a806b500 0x7f54a806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:21.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.796+0000 7f54c3e00640 1 -- 172.21.15.86:0/2662339948 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f54c4089a50 con 0x7f54cc159420 2024-03-06T07:27:21.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.796+0000 7f54cbe00640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f54a806b500 0x7f54a806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:21.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.796+0000 7f54cbe00640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f54a806b500 0x7f54a806d9c0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f54bc001eb0 tx=0x7f54bc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:21.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.801+0000 7f54c3e00640 1 -- 172.21.15.86:0/2662339948 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f54c4058830 con 0x7f54cc159420 2024-03-06T07:27:22.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:21.999+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f54cc0cdea0 con 0x7f54cc159420 2024-03-06T07:27:22.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.000+0000 7f54c3e00640 1 -- 172.21.15.86:0/2662339948 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f54c4058130 con 0x7f54cc159420 2024-03-06T07:27:22.002 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:22.003 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f54a806b500 msgr2=0x7f54a806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f54a806b500 0x7f54a806d9c0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f54bc001eb0 tx=0x7f54bc009040 comp rx=0 tx=0).stop 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54cc159420 msgr2=0x7f54cc1e89a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54cc159420 0x7f54cc1e89a0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f54c4006fd0 tx=0x7f54c400e8b0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 shutdown_connections 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f54a806b500 0x7f54a806d9c0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54cc159420 0x7f54cc1e89a0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f54cc06e9b0 0x7f54cc158ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 --2- 172.21.15.86:0/2662339948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f54ac004010 0x7f54cc06e470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.003+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 >> 172.21.15.86:0/2662339948 conn(0x7f54cc06c0d0 msgr2=0x7f54cc1605c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:22.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.004+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 shutdown_connections 2024-03-06T07:27:22.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.004+0000 7f54d1600640 1 -- 172.21.15.86:0/2662339948 wait complete. 2024-03-06T07:27:22.024 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:22.041 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:27:22.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:27:22.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.222+0000 7f4eeb600640 1 Processor -- start 2024-03-06T07:27:22.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.222+0000 7f4eeb600640 1 -- start start 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4eeb600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4ee40b4680 0x7f4ee406e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4eeb600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4ee4072340 0x7f4ee406ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4eeb600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ee415b090 con 0x7f4ee415acb0 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4eeb600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4ee40b2a30 con 0x7f4ee40b4680 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4eeb600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4ee40b4360 con 0x7f4ee4072340 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4edbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4ee40b4680 0x7f4ee406e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4edbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4ee40b4680 0x7f4ee406e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34586/0 (socket says 172.21.15.86:34586) 2024-03-06T07:27:22.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4edbe00640 1 -- 172.21.15.86:0/570000625 learned_addr learned my addr 172.21.15.86:0/570000625 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:22.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4ee9600640 1 --2- 172.21.15.86:0/570000625 >> v2:172.21.15.194:3300/0 conn(0x7f4ee4072340 0x7f4ee406ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:22.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4edbe00640 1 -- 172.21.15.86:0/570000625 >> v2:172.21.15.194:3300/0 conn(0x7f4ee4072340 msgr2=0x7f4ee406ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:22.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4edbe00640 1 --2- 172.21.15.86:0/570000625 >> v2:172.21.15.194:3300/0 conn(0x7f4ee4072340 0x7f4ee406ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4edbe00640 1 -- 172.21.15.86:0/570000625 >> v1:172.21.15.86:6789/0 conn(0x7f4ee415acb0 legacy=0x7f4ee40b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4edbe00640 1 -- 172.21.15.86:0/570000625 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ee40b2cf0 con 0x7f4ee40b4680 2024-03-06T07:27:22.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.224+0000 7f4ee3e00640 1 -- 172.21.15.86:0/570000625 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 501484604 0 0) 0x7f4ee415b090 con 0x7f4ee415acb0 2024-03-06T07:27:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.225+0000 7f4edbe00640 1 --2- 172.21.15.86:0/570000625 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee40b4680 0x7f4ee406e4f0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f4ee40b4e60 tx=0x7f4ed002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=62fa91dfd51536e0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4ee3e00640 1 -- 172.21.15.86:0/570000625 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ed0034070 con 0x7f4ee40b4680 2024-03-06T07:27:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4ee3e00640 1 -- 172.21.15.86:0/570000625 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ed0030860 con 0x7f4ee40b4680 2024-03-06T07:27:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4ee3e00640 1 -- 172.21.15.86:0/570000625 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ed0030c30 con 0x7f4ee40b4680 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 -- 172.21.15.86:0/570000625 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee40b4680 msgr2=0x7f4ee406e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 --2- 172.21.15.86:0/570000625 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee40b4680 0x7f4ee406e4f0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f4ee40b4e60 tx=0x7f4ed002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 -- 172.21.15.86:0/570000625 shutdown_connections 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 --2- 172.21.15.86:0/570000625 >> v2:172.21.15.194:3300/0 conn(0x7f4ee4072340 0x7f4ee406ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 --2- 172.21.15.86:0/570000625 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee40b4680 0x7f4ee406e4f0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 -- 172.21.15.86:0/570000625 >> 172.21.15.86:0/570000625 conn(0x7f4ee40b1730 msgr2=0x7f4ee40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 -- 172.21.15.86:0/570000625 shutdown_connections 2024-03-06T07:27:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.226+0000 7f4eeb600640 1 -- 172.21.15.86:0/570000625 wait complete. 2024-03-06T07:27:22.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.229+0000 7f4eeb600640 1 Processor -- start 2024-03-06T07:27:22.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.229+0000 7f4eeb600640 1 -- start start 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4eeb600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ee4072340 0x7f4ee4149570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4eeb600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ee40b4680 0x7f4ee415bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4eeb600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee415acb0 0x7f4ee415c4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4eeb600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ee40b4010 con 0x7f4ee415acb0 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4eeb600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ee40b32b0 con 0x7f4ee4072340 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4eeb600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ee40ccfb0 con 0x7f4ee40b4680 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4ee9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee415acb0 0x7f4ee415c4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:22.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4ee9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee415acb0 0x7f4ee415c4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34596/0 (socket says 172.21.15.86:34596) 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4ee9600640 1 -- 172.21.15.86:0/271589468 learned_addr learned my addr 172.21.15.86:0/271589468 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.230+0000 7f4ee8c00640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ee4072340 0x7f4ee4149570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4edbe00640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ee40b4680 0x7f4ee415bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee9600640 1 -- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ee40b4680 msgr2=0x7f4ee415bf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee9600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ee40b4680 0x7f4ee415bf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee9600640 1 -- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ee4072340 msgr2=0x7f4ee4149570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee9600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ee4072340 0x7f4ee4149570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee9600640 1 -- 172.21.15.86:0/271589468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ee40ce950 con 0x7f4ee415acb0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4edbe00640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ee40b4680 0x7f4ee415bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee9600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee415acb0 0x7f4ee415c4c0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f4edc00d800 tx=0x7f4edc00dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee1600640 1 -- 172.21.15.86:0/271589468 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4edc00b6c0 con 0x7f4ee415acb0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee1600640 1 -- 172.21.15.86:0/271589468 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4edc019070 con 0x7f4ee415acb0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4ee1600640 1 -- 172.21.15.86:0/271589468 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4edc00bac0 con 0x7f4ee415acb0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.231+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ee40cec30 con 0x7f4ee415acb0 2024-03-06T07:27:22.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.232+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ee40cf1f0 con 0x7f4ee415acb0 2024-03-06T07:27:22.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.233+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ee404f330 con 0x7f4ee415acb0 2024-03-06T07:27:22.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.233+0000 7f4ee1600640 1 -- 172.21.15.86:0/271589468 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4edc00bc60 con 0x7f4ee415acb0 2024-03-06T07:27:22.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.235+0000 7f4ee1600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4ec406b500 0x7f4ec406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:22.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.235+0000 7f4ee1600640 1 -- 172.21.15.86:0/271589468 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4edc08a4f0 con 0x7f4ee415acb0 2024-03-06T07:27:22.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.235+0000 7f4ee8c00640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4ec406b500 0x7f4ec406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:22.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.236+0000 7f4ee8c00640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4ec406b500 0x7f4ec406d9c0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f4ed40012e0 tx=0x7f4ed4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:22.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.239+0000 7f4ee1600640 1 -- 172.21.15.86:0/271589468 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4edc059400 con 0x7f4ee415acb0 2024-03-06T07:27:22.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.457+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4ee406ce60 con 0x7f4ee415acb0 2024-03-06T07:27:22.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.458+0000 7f4ee1600640 1 -- 172.21.15.86:0/271589468 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f4edc058d00 con 0x7f4ee415acb0 2024-03-06T07:27:22.459 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:22.460 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:27:22.460 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:27:22.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4ec406b500 msgr2=0x7f4ec406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4ec406b500 0x7f4ec406d9c0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f4ed40012e0 tx=0x7f4ed4009040 comp rx=0 tx=0).stop 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee415acb0 msgr2=0x7f4ee415c4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee415acb0 0x7f4ee415c4c0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f4edc00d800 tx=0x7f4edc00dcc0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 shutdown_connections 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4ec406b500 0x7f4ec406d9c0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ee415acb0 0x7f4ee415c4c0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ee40b4680 0x7f4ee415bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 --2- 172.21.15.86:0/271589468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ee4072340 0x7f4ee4149570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.460+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 >> 172.21.15.86:0/271589468 conn(0x7f4ee40b1730 msgr2=0x7f4ee40b37d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.461+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 shutdown_connections 2024-03-06T07:27:22.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:22.461+0000 7f4eeb600640 1 -- 172.21.15.86:0/271589468 wait complete. 2024-03-06T07:27:22.882 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 8, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:27:07.365128+0000', 'modified': '2024-03-06T07:27:10.407141+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 4255}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_4255': {'gid': 4255, 'name': 'b', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.194:6834/1612687235', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.194:6832', 'nonce': 1612687235}, {'type': 'v1', 'addr': '172.21.15.194:6834', 'nonce': 1612687235}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 4255, 'qdb_cluster': [4255]} 2024-03-06T07:27:22.882 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:27:22.882 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-03-06T07:27:23.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.097+0000 7fe500600640 1 Processor -- start 2024-03-06T07:27:23.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.097+0000 7fe500600640 1 -- start start 2024-03-06T07:27:23.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.098+0000 7fe500600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ac90 0x7fe4f806e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.098+0000 7fe500600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe4f815b070 0x7fe4f815ea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.098+0000 7fe500600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4f80cc000 con 0x7fe4f815ef50 2024-03-06T07:27:23.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.098+0000 7fe500600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe4f80ce360 con 0x7fe4f815b070 2024-03-06T07:27:23.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.098+0000 7fe500600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4f80ce640 con 0x7fe4f815ac90 2024-03-06T07:27:23.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.099+0000 7fe4fdc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ac90 0x7fe4f806e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.099+0000 7fe4fdc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ac90 0x7fe4f806e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55086/0 (socket says 172.21.15.86:55086) 2024-03-06T07:27:23.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.099+0000 7fe4fdc00640 1 -- 172.21.15.86:0/3991693770 learned_addr learned my addr 172.21.15.86:0/3991693770 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:23.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.099+0000 7fe4fd200640 1 --2- 172.21.15.86:0/3991693770 >> v2:172.21.15.194:3300/0 conn(0x7fe4f815b070 0x7fe4f815ea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.100+0000 7fe4fdc00640 1 -- 172.21.15.86:0/3991693770 >> v2:172.21.15.194:3300/0 conn(0x7fe4f815b070 msgr2=0x7fe4f815ea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.100+0000 7fe4f7e00640 1 -- 172.21.15.86:0/3991693770 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 815998203 0 0) 0x7fe4f80cc000 con 0x7fe4f815ef50 2024-03-06T07:27:23.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.100+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/3991693770 >> v2:172.21.15.194:3300/0 conn(0x7fe4f815b070 0x7fe4f815ea10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.100+0000 7fe4fdc00640 1 -- 172.21.15.86:0/3991693770 >> v1:172.21.15.194:6789/0 conn(0x7fe4f815ef50 legacy=0x7fe4f80cac60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.100+0000 7fe4fdc00640 1 -- 172.21.15.86:0/3991693770 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4f80ce920 con 0x7fe4f815ac90 2024-03-06T07:27:23.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.101+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/3991693770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ac90 0x7fe4f806e580 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fe4ec009870 tx=0x7fe4ec02b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cd658d7ad6c439bb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:23.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe4f7e00640 1 -- 172.21.15.86:0/3991693770 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4ec035020 con 0x7fe4f815ac90 2024-03-06T07:27:23.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe4f7e00640 1 -- 172.21.15.86:0/3991693770 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4ec031640 con 0x7fe4f815ac90 2024-03-06T07:27:23.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe4f7e00640 1 -- 172.21.15.86:0/3991693770 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4ec02d040 con 0x7fe4f815ac90 2024-03-06T07:27:23.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 -- 172.21.15.86:0/3991693770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ac90 msgr2=0x7fe4f806e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 --2- 172.21.15.86:0/3991693770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ac90 0x7fe4f806e580 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fe4ec009870 tx=0x7fe4ec02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 -- 172.21.15.86:0/3991693770 shutdown_connections 2024-03-06T07:27:23.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 --2- 172.21.15.86:0/3991693770 >> v2:172.21.15.194:3300/0 conn(0x7fe4f815b070 0x7fe4f815ea10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 --2- 172.21.15.86:0/3991693770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ac90 0x7fe4f806e580 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 -- 172.21.15.86:0/3991693770 >> 172.21.15.86:0/3991693770 conn(0x7fe4f80cc4f0 msgr2=0x7fe4f80cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:23.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 -- 172.21.15.86:0/3991693770 shutdown_connections 2024-03-06T07:27:23.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.102+0000 7fe500600640 1 -- 172.21.15.86:0/3991693770 wait complete. 2024-03-06T07:27:23.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.104+0000 7fe500600640 1 Processor -- start 2024-03-06T07:27:23.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.104+0000 7fe500600640 1 -- start start 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe500600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4f815ac90 0x7fe4f80ae390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe500600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4f815b070 0x7fe4f81511c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe500600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ef50 0x7fe4f80aeb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe500600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4f80c5a20 con 0x7fe4f815b070 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe500600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4f80ceee0 con 0x7fe4f815ac90 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe500600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4f80c5ba0 con 0x7fe4f815ef50 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe4fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4f815b070 0x7fe4f81511c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe4fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4f815b070 0x7fe4f81511c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34608/0 (socket says 172.21.15.86:34608) 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.105+0000 7fe4fd200640 1 -- 172.21.15.86:0/300463635 learned_addr learned my addr 172.21.15.86:0/300463635 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:23.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fe600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ef50 0x7fe4f80aeb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4f815ac90 0x7fe4f80ae390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fd200640 1 -- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ef50 msgr2=0x7fe4f80aeb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fd200640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ef50 0x7fe4f80aeb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fd200640 1 -- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4f815ac90 msgr2=0x7fe4f80ae390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fd200640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4f815ac90 0x7fe4f80ae390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fd200640 1 -- 172.21.15.86:0/300463635 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4f80af0a0 con 0x7fe4f815b070 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4f815ac90 0x7fe4f80ae390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4fd200640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4f815b070 0x7fe4f81511c0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fe4e400e9e0 tx=0x7fe4e400eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4f5600640 1 -- 172.21.15.86:0/300463635 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4e4017070 con 0x7fe4f815b070 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4f5600640 1 -- 172.21.15.86:0/300463635 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4e400c9d0 con 0x7fe4f815b070 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4f80b4340 con 0x7fe4f815b070 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe4f5600640 1 -- 172.21.15.86:0/300463635 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4e400cdc0 con 0x7fe4f815b070 2024-03-06T07:27:23.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.106+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4f80b4900 con 0x7fe4f815b070 2024-03-06T07:27:23.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.108+0000 7fe4f5600640 1 -- 172.21.15.86:0/300463635 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe4e401a070 con 0x7fe4f815b070 2024-03-06T07:27:23.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.109+0000 7fe4f5600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4d806b4d0 0x7fe4d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.109+0000 7fe4f5600640 1 -- 172.21.15.86:0/300463635 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe4e408a6e0 con 0x7fe4f815b070 2024-03-06T07:27:23.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.110+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4d806b4d0 0x7fe4d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.110+0000 7fe503280740 1 Processor -- start 2024-03-06T07:27:23.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.110+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4d806b4d0 0x7fe4d806d990 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fe4ec036000 tx=0x7fe4ec008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:23.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.110+0000 7fe503280740 1 -- start start 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.111+0000 7fe503280740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92c1c750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.111+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 0x560d92c4c540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.111+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c1cc90 0x560d92c4ca80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.111+0000 7fe503280740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92b86600 con 0x560d92c497b0 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.111+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92b2f930 con 0x560d92c49b90 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.111+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d92b826a0 con 0x560d92c1cc90 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92c1c750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92c1c750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34624/0 (socket says 172.21.15.86:34624) 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 -- 172.21.15.86:0/2800818507 learned_addr learned my addr 172.21.15.86:0/2800818507 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 -- 172.21.15.86:0/2800818507 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c1cc90 msgr2=0x560d92c4ca80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fe600640 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c1cc90 0x560d92c4ca80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c1cc90 0x560d92c4ca80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 -- 172.21.15.86:0/2800818507 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 msgr2=0x560d92c4c540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 0x560d92c4c540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 -- 172.21.15.86:0/2800818507 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d9289b2d0 con 0x560d92c497b0 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.112+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92c1c750 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fe4ec000c00 tx=0x7fe4ec030000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=efcec2c19f71188d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe4ea000640 1 -- 172.21.15.86:0/2800818507 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4ec035040 con 0x560d92c497b0 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe4ea000640 1 -- 172.21.15.86:0/2800818507 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4ec031e50 con 0x560d92c497b0 2024-03-06T07:27:23.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe4ea000640 1 -- 172.21.15.86:0/2800818507 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4ec03a820 con 0x560d92c497b0 2024-03-06T07:27:23.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 -- 172.21.15.86:0/2800818507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 msgr2=0x560d92c1c750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92c1c750 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fe4ec000c00 tx=0x7fe4ec030000 comp rx=0 tx=0).stop 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 -- 172.21.15.86:0/2800818507 shutdown_connections 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c1cc90 0x560d92c4ca80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 0x560d92c4c540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 --2- 172.21.15.86:0/2800818507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92c1c750 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 -- 172.21.15.86:0/2800818507 >> 172.21.15.86:0/2800818507 conn(0x560d92c1e9c0 msgr2=0x560d92c67540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 -- 172.21.15.86:0/2800818507 shutdown_connections 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.113+0000 7fe503280740 1 -- 172.21.15.86:0/2800818507 wait complete. 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.114+0000 7fe503280740 1 -- start start 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.114+0000 7fe503280740 1 Processor -- start 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c497b0 0x560d92c65230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe503280740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c62780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 0x560d92c4e9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe503280740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92c20f60 con 0x560d92c49b90 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92b42d90 con 0x560d92c497b0 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d92c20350 con 0x560d92c48f00 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe503280740 10 client.0 _ll_register_callbacks cb 0x560d92b7e350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe4fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c62780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe4fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c62780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34634/0 (socket says 172.21.15.86:34634) 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.115+0000 7fe4fd200640 1 -- 172.21.15.86:0/299775773 learned_addr learned my addr 172.21.15.86:0/299775773 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4fd200640 1 -- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 msgr2=0x560d92c4e9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4fd200640 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 0x560d92c4e9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4fd200640 1 -- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c497b0 msgr2=0x560d92c65230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4fd200640 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c497b0 0x560d92c65230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4fd200640 1 -- 172.21.15.86:0/299775773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560d929656b0 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4fd200640 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c62780 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fe4e4055220 tx=0x7fe4e40594c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af1c6202a45e5d08 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4dea00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4dea00640 1 -- 172.21.15.86:0/299775773 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4e4021070 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4dea00640 1 -- 172.21.15.86:0/299775773 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe4e4058210 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe4ec031e50 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4dea00640 1 -- 172.21.15.86:0/299775773 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=8) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe4e40614d0 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe503280740 10 client.4590 fetch_fsmap learned FSMap version 8 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=8,monmap=2+}) v3 -- 0x560d92c65d60 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.116+0000 7fe4dea00640 1 -- 172.21.15.86:0/299775773 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4ec031e50 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe4dea00640 1 -- 172.21.15.86:0/299775773 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 8) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fe4e4070b50 con 0x560d92c49b90 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe503280740 10 client.4590 fetch_fsmap finished waiting for FSMap version 8 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe503280740 10 client.4590 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe503280740 20 client.4590 populate_metadata read hostname 'smithi086' 2024-03-06T07:27:23.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe503280740 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c7b6e0 0x560d92c63bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe503280740 4 client.4590 mds_command: new command op to 4255 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:27:23.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 --> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560d92b93b40 con 0x560d92c7b6e0 2024-03-06T07:27:23.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.117+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c7b6e0 0x560d92c63bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.118+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c7b6e0 0x560d92c63bb0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3815267b08cecd61 server_cookie=df1a7b258aaf8053 in_seq=0 out_seq=0 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.118+0000 7fe4dea00640 10 client.4590 ms_handle_connect on v2:172.21.15.194:6832/1612687235 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe4dea00640 1 -- 172.21.15.86:0/299775773 <== mds.0 v2:172.21.15.194:6832/1612687235 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x560d92b93b40 con 0x560d92c7b6e0 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe4dea00640 10 client.4590 handle_command_reply: tid=0 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 client.4590 shutdown 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 msgr2=0x560d92c62780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c62780 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fe4e4055220 tx=0x7fe4e40594c0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 shutdown_connections 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c7b6e0 0x560d92c63bb0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 0x560d92c4e9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c62780 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe4dea00640 0 client.4590 ms_handle_reset on v2:172.21.15.194:6832/1612687235 2024-03-06T07:27:23.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 --2- 172.21.15.86:0/299775773 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c497b0 0x560d92c65230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 >> 172.21.15.86:0/299775773 conn(0x560d92c1cc90 msgr2=0x560d92c4c530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:23.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.120+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 shutdown_connections 2024-03-06T07:27:23.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.121+0000 7fe503280740 1 -- 172.21.15.86:0/299775773 wait complete. 2024-03-06T07:27:23.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.121+0000 7fe503280740 20 client.4590 trim_cache size 0 max 16384 2024-03-06T07:27:23.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.138+0000 7fe503280740 1 Processor -- start 2024-03-06T07:27:23.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.138+0000 7fe503280740 1 -- start start 2024-03-06T07:27:23.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c48f00 0x560d92c64bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe503280740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c4c000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe503280740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4f81ff3c0 con 0x560d92c49b90 2024-03-06T07:27:23.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4f81fe7c0 con 0x560d92c48f00 2024-03-06T07:27:23.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4f81fedc0 con 0x560d92c497b0 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe4fe600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c4c000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe4fe600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c4c000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34650/0 (socket says 172.21.15.86:34650) 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe4fe600640 1 -- 172.21.15.86:0/2073300191 learned_addr learned my addr 172.21.15.86:0/2073300191 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c48f00 0x560d92c64bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.139+0000 7fe4fd200640 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4fe600640 1 -- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c497b0 msgr2=0x560d92cd73d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4fe600640 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4fe600640 1 -- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c48f00 msgr2=0x560d92c64bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4fe600640 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c48f00 0x560d92c64bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4fe600640 1 -- 172.21.15.86:0/2073300191 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d92c18050 con 0x560d92c49b90 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c48f00 0x560d92c64bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4fe600640 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c4c000 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fe4f0001ad0 tx=0x7fe4f0001f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e8620d2d593d3bb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4e9600640 1 -- 172.21.15.86:0/2073300191 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4f0017030 con 0x560d92c49b90 2024-03-06T07:27:23.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4e9600640 1 -- 172.21.15.86:0/2073300191 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4f000c070 con 0x560d92c49b90 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.140+0000 7fe4e9600640 1 -- 172.21.15.86:0/2073300191 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4f0006d90 con 0x560d92c49b90 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 -- 172.21.15.86:0/2073300191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 msgr2=0x560d92c4c000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c4c000 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fe4f0001ad0 tx=0x7fe4f0001f90 comp rx=0 tx=0).stop 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 -- 172.21.15.86:0/2073300191 shutdown_connections 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c49b90 0x560d92c4c000 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 --2- 172.21.15.86:0/2073300191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c48f00 0x560d92c64bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 -- 172.21.15.86:0/2073300191 >> 172.21.15.86:0/2073300191 conn(0x560d92c1cc90 msgr2=0x560d92c1b620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 -- 172.21.15.86:0/2073300191 shutdown_connections 2024-03-06T07:27:23.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.141+0000 7fe503280740 1 -- 172.21.15.86:0/2073300191 wait complete. 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.142+0000 7fe503280740 1 -- start start 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.142+0000 7fe503280740 1 Processor -- start 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 0x560d92c45f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe503280740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe503280740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 0x560d92c20770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe503280740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92c7b560 con 0x560d92c497b0 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92c64470 con 0x560d92c49b90 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe503280740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d92c7bb60 con 0x560d92c48f00 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe503280740 10 client.0 _ll_register_callbacks cb 0x7fe4e40046c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34654/0 (socket says 172.21.15.86:34654) 2024-03-06T07:27:23.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 -- 172.21.15.86:0/2730580674 learned_addr learned my addr 172.21.15.86:0/2730580674 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 -- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 msgr2=0x560d92c45f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 0x560d92c45f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 -- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 msgr2=0x560d92c20770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 0x560d92c20770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.143+0000 7fe4fd200640 1 -- 172.21.15.86:0/2730580674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560d92c7add0 con 0x560d92c497b0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe4fd200640 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92cd73d0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fe4e4055740 tx=0x7fe4e400efd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=534c6540abe6a22f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe4dea00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe4dea00640 1 -- 172.21.15.86:0/2730580674 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4e4021040 con 0x560d92c497b0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe4dea00640 1 -- 172.21.15.86:0/2730580674 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe4e4054b10 con 0x560d92c497b0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe4f000c070 con 0x560d92c497b0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe4dea00640 1 -- 172.21.15.86:0/2730580674 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=8) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe4e4054d10 con 0x560d92c497b0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 10 client.4596 fetch_fsmap learned FSMap version 8 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=8,monmap=2+}) v3 -- 0x560d92c1c4c0 con 0x560d92c497b0 2024-03-06T07:27:23.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe4dea00640 1 -- 172.21.15.86:0/2730580674 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4f000c070 con 0x560d92c497b0 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe4dea00640 1 -- 172.21.15.86:0/2730580674 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 8) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fe4e4077030 con 0x560d92c497b0 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 10 client.4596 fetch_fsmap finished waiting for FSMap version 8 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 10 client.4596 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 20 client.4596 populate_metadata read hostname 'smithi086' 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c690d0 0x560d92c694b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 4 client.4596 mds_command: new command op to 4255 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.144+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 --> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] -- command(tid 0: {"prefix": "status"}) v1 -- 0x560d92b8de90 con 0x560d92c690d0 2024-03-06T07:27:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.145+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c690d0 0x560d92c694b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:23.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.146+0000 7fe4dea00640 10 client.4596 ms_handle_connect on v2:172.21.15.194:6832/1612687235 2024-03-06T07:27:23.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.146+0000 7fe4fdc00640 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c690d0 0x560d92c694b0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2ccd45f07e20fa46 server_cookie=3c844cf4b1a05f32 in_seq=0 out_seq=0 2024-03-06T07:27:23.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.146+0000 7fe4dea00640 1 -- 172.21.15.86:0/2730580674 <== mds.0 v2:172.21.15.194:6832/1612687235 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+324 (crc 0 0 0) 0x560d92b8de90 con 0x560d92c690d0 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.146+0000 7fe4dea00640 10 client.4596 handle_command_reply: tid=0 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.146+0000 7fe503280740 1 client.4596 shutdown 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 msgr2=0x560d92cd73d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92cd73d0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fe4e4055740 tx=0x7fe4e400efd0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 shutdown_connections 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:6832/1612687235,v1:172.21.15.194:6834/1612687235] conn(0x560d92c690d0 0x560d92c694b0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d92c49b90 0x560d92c20770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92c497b0 0x560d92cd73d0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 --2- 172.21.15.86:0/2730580674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d92c48f00 0x560d92c45f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe4dea00640 0 client.4596 ms_handle_reset on v2:172.21.15.194:6832/1612687235 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 >> 172.21.15.86:0/2730580674 conn(0x560d92c1cc90 msgr2=0x560d92c1b620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 shutdown_connections 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 1 -- 172.21.15.86:0/2730580674 wait complete. 2024-03-06T07:27:23.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe503280740 20 client.4596 trim_cache size 0 max 16384 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 4255, 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 15.232176289, 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 8, 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 22, 2024-03-06T07:27:23.152 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 22, 2024-03-06T07:27:23.153 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 22.991774285000002 2024-03-06T07:27:23.153 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:27:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4d806b4d0 msgr2=0x7fe4d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe500600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4d806b4d0 0x7fe4d806d990 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fe4ec036000 tx=0x7fe4ec008030 comp rx=0 tx=0).stop 2024-03-06T07:27:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4f815b070 msgr2=0x7fe4f81511c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.147+0000 7fe500600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4f815b070 0x7fe4f81511c0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fe4e400e9e0 tx=0x7fe4e400eea0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 shutdown_connections 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4d806b4d0 0x7fe4d806d990 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4f815ef50 0x7fe4f80aeb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4f815b070 0x7fe4f81511c0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 --2- 172.21.15.86:0/300463635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4f815ac90 0x7fe4f80ae390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 >> 172.21.15.86:0/300463635 conn(0x7fe4f80cc4f0 msgr2=0x7fe4f80af530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 shutdown_connections 2024-03-06T07:27:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:23.148+0000 7fe500600640 1 -- 172.21.15.86:0/300463635 wait complete. 2024-03-06T07:27:23.165 INFO:teuthology.run_tasks:Running task kclient... 2024-03-06T07:27:23.189 INFO:tasks.kclient:Mounting kernel clients... 2024-03-06T07:27:23.189 INFO:tasks.kclient:config is {'client.0': {}, 'client.2': {}, 'client.1': {}, 'client.3': {}} 2024-03-06T07:27:23.189 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.189 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:27:23.237 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.237 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link delete ceph-brx 2024-03-06T07:27:23.333 INFO:teuthology.orchestra.run.smithi086.stderr:Cannot find device "ceph-brx" 2024-03-06T07:27:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:27:23.338 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.338 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:27:23.389 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.389 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link delete ceph-brx 2024-03-06T07:27:23.488 INFO:teuthology.orchestra.run.smithi086.stderr:Cannot find device "ceph-brx" 2024-03-06T07:27:23.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:27:23.493 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.493 DEBUG:teuthology.orchestra.run.smithi194:> ip netns list 2024-03-06T07:27:23.547 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link delete ceph-brx 2024-03-06T07:27:23.643 INFO:teuthology.orchestra.run.smithi194.stderr:Cannot find device "ceph-brx" 2024-03-06T07:27:23.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:27:23.649 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.650 DEBUG:teuthology.orchestra.run.smithi194:> ip netns list 2024-03-06T07:27:23.699 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:23.699 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link delete ceph-brx 2024-03-06T07:27:23.796 INFO:teuthology.orchestra.run.smithi194.stderr:Cannot find device "ceph-brx" 2024-03-06T07:27:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:27:23.800 INFO:tasks.kclient:client.0 config is {'syntax': 'v2'} 2024-03-06T07:27:23.801 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-06T07:27:23.801 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:27:23.801 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:27:24.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.020+0000 7f3816600640 1 Processor -- start 2024-03-06T07:27:24.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.020+0000 7f3816600640 1 -- start start 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.021+0000 7f3816600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f381015a940 0x7f38100b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3816600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f38100cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3816600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f38100b2b30 con 0x7f38100cf370 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3816600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38100b28e0 con 0x7f381015a940 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3816600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38100b2e10 con 0x7f38100cc2a0 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f381015a940 0x7f38100b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f381015a940 0x7f38100b45e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34668/0 (socket says 172.21.15.86:34668) 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815200640 1 -- 172.21.15.86:0/460907189 learned_addr learned my addr 172.21.15.86:0/460907189 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815c00640 1 --2- 172.21.15.86:0/460907189 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f38100cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815200640 1 -- 172.21.15.86:0/460907189 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 msgr2=0x7f38100cc750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815200640 1 --2- 172.21.15.86:0/460907189 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f38100cc750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815200640 1 -- 172.21.15.86:0/460907189 >> v1:172.21.15.194:6789/0 conn(0x7f38100cf370 legacy=0x7f381006d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:24.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.022+0000 7f3815200640 1 -- 172.21.15.86:0/460907189 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f381006ca50 con 0x7f381015a940 2024-03-06T07:27:24.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.023+0000 7f3815200640 1 --2- 172.21.15.86:0/460907189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100b45e0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f3800009870 tx=0x7f380002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eaf458acf405ec26 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:24.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f380fe00640 1 -- 172.21.15.86:0/460907189 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3800035020 con 0x7f381015a940 2024-03-06T07:27:24.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f380fe00640 1 -- 172.21.15.86:0/460907189 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3800031640 con 0x7f381015a940 2024-03-06T07:27:24.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f380fe00640 1 -- 172.21.15.86:0/460907189 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f380002d040 con 0x7f381015a940 2024-03-06T07:27:24.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f3816600640 1 -- 172.21.15.86:0/460907189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 msgr2=0x7f38100b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:24.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f3816600640 1 --2- 172.21.15.86:0/460907189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100b45e0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f3800009870 tx=0x7f380002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f3816600640 1 -- 172.21.15.86:0/460907189 shutdown_connections 2024-03-06T07:27:24.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f3816600640 1 --2- 172.21.15.86:0/460907189 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f38100cc750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f3816600640 1 --2- 172.21.15.86:0/460907189 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100b45e0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.024+0000 7f3816600640 1 -- 172.21.15.86:0/460907189 >> 172.21.15.86:0/460907189 conn(0x7f381006afc0 msgr2=0x7f381006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:24.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.025+0000 7f3816600640 1 -- 172.21.15.86:0/460907189 shutdown_connections 2024-03-06T07:27:24.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.025+0000 7f3816600640 1 -- 172.21.15.86:0/460907189 wait complete. 2024-03-06T07:27:24.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.026+0000 7f3816600640 1 Processor -- start 2024-03-06T07:27:24.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.026+0000 7f3816600640 1 -- start start 2024-03-06T07:27:24.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.027+0000 7f3816600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f3810146490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:24.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.027+0000 7f3816600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38100cf370 0x7f38100cf890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:24.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.027+0000 7f3816600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100cfdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:24.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.027+0000 7f3816600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38100b4b20 con 0x7f381015a940 2024-03-06T07:27:24.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.027+0000 7f3816600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38100b30f0 con 0x7f38100cf370 2024-03-06T07:27:24.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.027+0000 7f3816600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f381015ad20 con 0x7f38100cc2a0 2024-03-06T07:27:24.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100cfdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100cfdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34670/0 (socket says 172.21.15.86:34670) 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 -- 172.21.15.86:0/2771857558 learned_addr learned my addr 172.21.15.86:0/2771857558 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815200640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f3810146490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3807e00640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38100cf370 0x7f38100cf890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 -- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 msgr2=0x7f3810146490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f3810146490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 -- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38100cf370 msgr2=0x7f38100cf890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38100cf370 0x7f38100cf890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 -- 172.21.15.86:0/2771857558 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f381015be20 con 0x7f381015a940 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.028+0000 7f3815c00640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100cfdd0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f380800cc30 tx=0x7f380800a5c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.029+0000 7f380d600640 1 -- 172.21.15.86:0/2771857558 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f380800ce40 con 0x7f381015a940 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.029+0000 7f380d600640 1 -- 172.21.15.86:0/2771857558 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3808003990 con 0x7f381015a940 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.029+0000 7f380d600640 1 -- 172.21.15.86:0/2771857558 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3808003d40 con 0x7f381015a940 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.029+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f381015c100 con 0x7f381015a940 2024-03-06T07:27:24.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.029+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f381015c6c0 con 0x7f381015a940 2024-03-06T07:27:24.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.030+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f381004f330 con 0x7f381015a940 2024-03-06T07:27:24.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.031+0000 7f380d600640 1 -- 172.21.15.86:0/2771857558 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3808003ee0 con 0x7f381015a940 2024-03-06T07:27:24.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.032+0000 7f380d600640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f37ec06b4d0 0x7f37ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:24.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.032+0000 7f380d600640 1 -- 172.21.15.86:0/2771857558 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f380808a6d0 con 0x7f381015a940 2024-03-06T07:27:24.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.032+0000 7f3815200640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f37ec06b4d0 0x7f37ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:24.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.033+0000 7f3815200640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f37ec06b4d0 0x7f37ec06d990 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f3800036000 tx=0x7f380003c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.037+0000 7f380d600640 1 -- 172.21.15.86:0/2771857558 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3808014030 con 0x7f381015a940 2024-03-06T07:27:24.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.276+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f381006e180 con 0x7f381015a940 2024-03-06T07:27:24.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.277+0000 7f380d600640 1 -- 172.21.15.86:0/2771857558 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v8) v1 ==== 53+0+73 (secure 0 0 0) 0x7f380805de20 con 0x7f381015a940 2024-03-06T07:27:24.279 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f37ec06b4d0 msgr2=0x7f37ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f37ec06b4d0 0x7f37ec06d990 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f3800036000 tx=0x7f380003c040 comp rx=0 tx=0).stop 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 msgr2=0x7f38100cfdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100cfdd0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f380800cc30 tx=0x7f380800a5c0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 shutdown_connections 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f37ec06b4d0 0x7f37ec06d990 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f381015a940 0x7f38100cfdd0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38100cf370 0x7f38100cf890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 --2- 172.21.15.86:0/2771857558 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38100cc2a0 0x7f3810146490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:24.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 >> 172.21.15.86:0/2771857558 conn(0x7f381006afc0 msgr2=0x7f3810071ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:24.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 shutdown_connections 2024-03-06T07:27:24.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:24.280+0000 7f3816600640 1 -- 172.21.15.86:0/2771857558 wait complete. 2024-03-06T07:27:24.301 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-06T07:27:24.301 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:27:24.301 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:27:24.301 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:27:24.301 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:27:24.301 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-06T07:27:24.302 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-06T07:27:24.302 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:27:24.302 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:27:24.302 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:24.302 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:24.351 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42423sec preferred_lft 42423sec 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:27:24.352 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:24.353 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-03-06T07:27:24.353 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:24.353 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:24.353 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:24.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link add name ceph-brx type bridge 2024-03-06T07:27:24.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip addr flush dev ceph-brx 2024-03-06T07:27:24.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set ceph-brx up 2024-03-06T07:27:24.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-03-06T07:27:24.354 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:24.784 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:24.785 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:27:24.865 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:27:24.868 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:24.868 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:27:24.918 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:27:24.918 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:27:24.918 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:27:24.918 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:27:24.918 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:27:24.919 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:24.919 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:24.919 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:24.919 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:27:24.919 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:27:24.920 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:27:24.920 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:25.605 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:25.605 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:27:25.658 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:25.659 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list-id 2024-03-06T07:27:25.712 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:25.712 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:25.712 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:25.712 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:27:25.712 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.0 0 2024-03-06T07:27:25.713 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:25.845 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.0' with 192.168.0.1/16 2024-03-06T07:27:25.845 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.0 type veth peer name brx.0 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set veth0 up 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set lo up 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip route add default via 192.168.255.254 2024-03-06T07:27:25.845 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:26.112 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:26.113 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:26.113 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:26.113 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:27:26.113 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set dev brx.0 master ceph-brx 2024-03-06T07:27:26.113 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:26.267 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:26.267 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:27:26.311 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:27:26.311 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:26.311 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:27:26.405 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:27:26.405 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:26.405 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:26.538 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:26.538 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:27.164 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:27.167 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:27.167 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:27:27.167 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:27:27.167 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:27:27.168 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:27:27.168 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:27:27.168 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:27:27.168 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:27:27.168 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:27:27.172 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:27:27.172 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:27.172 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:27:27.261 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:27.261 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:27.337 INFO:teuthology.orchestra.run.smithi086.stdout:client4507 2024-03-06T07:27:27.337 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:27.337 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:27.388 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:27.388 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:27.459 INFO:teuthology.orchestra.run.smithi086.stdout:client4507 2024-03-06T07:27:27.460 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:27.460 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:27.536 INFO:teuthology.orchestra.run.smithi086.stdout:client4507 2024-03-06T07:27:27.536 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:27.536 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'sudo dd if=/sys/kernel/debug/ceph/013155b5-b812-4a77-8af3-f8c90b517e11.client4507/status') 2024-03-06T07:27:27.641 INFO:teuthology.orchestra.run.smithi086.stdout:instance: client.4507 (3)172.21.15.86:0/564231498 2024-03-06T07:27:27.641 INFO:teuthology.orchestra.run.smithi086.stdout:blocklisted: false 2024-03-06T07:27:27.641 INFO:teuthology.orchestra.run.smithi086.stderr:0+1 records in 2024-03-06T07:27:27.641 INFO:teuthology.orchestra.run.smithi086.stderr:0+1 records out 2024-03-06T07:27:27.641 INFO:teuthology.orchestra.run.smithi086.stderr:69 bytes copied, 6.6603e-05 s, 1.0 MB/s 2024-03-06T07:27:27.646 INFO:tasks.kclient:client.2 config is {'syntax': 'v2'} 2024-03-06T07:27:27.646 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-06T07:27:27.646 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.2 2024-03-06T07:27:27.646 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:27:27.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.849+0000 7f4101200640 1 Processor -- start 2024-03-06T07:27:27.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.850+0000 7f4101200640 1 -- start start 2024-03-06T07:27:27.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.851+0000 7f4101200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f40fc0cc9d0 0x7f40fc06d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.851+0000 7f4101200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f40fc15c240 con 0x7f40fc0aedc0 2024-03-06T07:27:27.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.851+0000 7f4101200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f40fc15c500 con 0x7f40fc1561f0 2024-03-06T07:27:27.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.851+0000 7f4101200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f40fc15ce30 con 0x7f40fc0cc9d0 2024-03-06T07:27:27.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.851+0000 7f40faa00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f40fc0aedc0 0x7f40fc0cc2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38610/0 (socket says 172.21.15.86:38610) 2024-03-06T07:27:27.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.851+0000 7f40faa00640 1 -- 172.21.15.86:0/2574466527 learned_addr learned my addr 172.21.15.86:0/2574466527 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fbe00640 1 --2- 172.21.15.86:0/2574466527 >> v2:172.21.15.194:3300/0 conn(0x7f40fc0cc9d0 0x7f40fc06d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2264859080 0 0) 0x7f40fc15c240 con 0x7f40fc0aedc0 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f40e0003520 con 0x7f40fc0aedc0 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 53160290 0 0) 0x7f40e0003520 con 0x7f40fc0aedc0 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 >> v2:172.21.15.194:3300/0 conn(0x7f40fc0cc9d0 msgr2=0x7f40fc06d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 --2- 172.21.15.86:0/2574466527 >> v2:172.21.15.194:3300/0 conn(0x7f40fc0cc9d0 0x7f40fc06d350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 >> v1:172.21.15.194:6789/0 conn(0x7f40fc1561f0 legacy=0x7f40fc0ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:27.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40fc15c7e0 con 0x7f40fc0aedc0 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f40e4001670 con 0x7f40fc0aedc0 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 >> v1:172.21.15.86:6789/0 conn(0x7f40fc0aedc0 legacy=0x7f40fc0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40e00039b0 0x7f40e0003df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40e0004800 0x7f40e0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0005210 0x7f40e000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40fc15ce30 con 0x7f40e0004800 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40e000c140 con 0x7f40e00039b0 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.852+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f40e000c340 con 0x7f40e0005210 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40faa00640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40e0004800 0x7f40e0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40fbe00640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0005210 0x7f40e000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40faa00640 1 -- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0005210 msgr2=0x7f40e000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40fb400640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40e00039b0 0x7f40e0003df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40faa00640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0005210 0x7f40e000b9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:27.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40faa00640 1 -- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40e00039b0 msgr2=0x7f40e0003df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:27.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40faa00640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40e00039b0 0x7f40e0003df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:27.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40fbe00640 1 -- 172.21.15.86:0/2574466527 reap_dead start 2024-03-06T07:27:27.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.853+0000 7f40faa00640 1 -- 172.21.15.86:0/2574466527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40e0003520 con 0x7f40e0004800 2024-03-06T07:27:27.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.854+0000 7f40faa00640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40e0004800 0x7f40e0004c80 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f40e400c240 tx=0x7f40e0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f682e04b3a9f4752 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:27.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.854+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40e4007d10 con 0x7f40e0004800 2024-03-06T07:27:27.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.854+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f40e4030070 con 0x7f40e0004800 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.854+0000 7f40fa000640 1 -- 172.21.15.86:0/2574466527 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40e4005d70 con 0x7f40e0004800 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.855+0000 7f4101200640 1 -- 172.21.15.86:0/2574466527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40e0004800 msgr2=0x7f40e0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.855+0000 7f4101200640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40e0004800 0x7f40e0004c80 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f40e400c240 tx=0x7f40e0005830 comp rx=0 tx=0).stop 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.855+0000 7f4101200640 1 -- 172.21.15.86:0/2574466527 shutdown_connections 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.855+0000 7f4101200640 1 --2- 172.21.15.86:0/2574466527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40e0004800 0x7f40e0004c80 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.855+0000 7f4101200640 1 -- 172.21.15.86:0/2574466527 >> 172.21.15.86:0/2574466527 conn(0x7f40fc06afc0 msgr2=0x7f40fc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.855+0000 7f4101200640 1 -- 172.21.15.86:0/2574466527 shutdown_connections 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.855+0000 7f4101200640 1 -- 172.21.15.86:0/2574466527 wait complete. 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.857+0000 7f4101200640 1 Processor -- start 2024-03-06T07:27:27.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.857+0000 7f4101200640 1 -- start start 2024-03-06T07:27:27.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f4101200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0004800 0x7f40fc1551e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f4101200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40fc0acb20 0x7f40fc0ce390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f4101200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40fc1f7980 0x7f40fc1f7e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f4101200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40fc156750 con 0x7f40fc0acb20 2024-03-06T07:27:27.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f4101200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40fc1565d0 con 0x7f40fc1f7980 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f4101200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f40fc0cc2c0 con 0x7f40e0004800 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f40faa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40fc0acb20 0x7f40fc0ce390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f40faa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40fc0acb20 0x7f40fc0ce390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40920/0 (socket says 172.21.15.86:40920) 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.858+0000 7f40faa00640 1 -- 172.21.15.86:0/649623831 learned_addr learned my addr 172.21.15.86:0/649623831 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40fb400640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0004800 0x7f40fc1551e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40fbe00640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40fc1f7980 0x7f40fc1f7e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40faa00640 1 -- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0004800 msgr2=0x7f40fc1551e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40faa00640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0004800 0x7f40fc1551e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40faa00640 1 -- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40fc1f7980 msgr2=0x7f40fc1f7e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40faa00640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40fc1f7980 0x7f40fc1f7e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40faa00640 1 -- 172.21.15.86:0/649623831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40fc1f87b0 con 0x7f40fc0acb20 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40faa00640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40fc0acb20 0x7f40fc0ce390 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f40e400c240 tx=0x7f40e402e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40f3400640 1 -- 172.21.15.86:0/649623831 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40e4007d10 con 0x7f40fc0acb20 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40f3400640 1 -- 172.21.15.86:0/649623831 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f40e4004bf0 con 0x7f40fc0acb20 2024-03-06T07:27:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f40fc0ce8d0 con 0x7f40fc0acb20 2024-03-06T07:27:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.859+0000 7f40f3400640 1 -- 172.21.15.86:0/649623831 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40e4007070 con 0x7f40fc0acb20 2024-03-06T07:27:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.860+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f40fc0ad2c0 con 0x7f40fc0acb20 2024-03-06T07:27:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.861+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f40fc04f330 con 0x7f40fc0acb20 2024-03-06T07:27:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.862+0000 7f40f3400640 1 -- 172.21.15.86:0/649623831 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f40e4048020 con 0x7f40fc0acb20 2024-03-06T07:27:27.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.862+0000 7f40f3400640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d806b4d0 0x7f40d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:27.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.862+0000 7f40f3400640 1 -- 172.21.15.86:0/649623831 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f40e40ab720 con 0x7f40fc0acb20 2024-03-06T07:27:27.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.863+0000 7f40fb400640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d806b4d0 0x7f40d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.863+0000 7f40fb400640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d806b4d0 0x7f40d806d990 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f40ec000c00 tx=0x7f40ec009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:27.867+0000 7f40f3400640 1 -- 172.21.15.86:0/649623831 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f40e407a500 con 0x7f40fc0acb20 2024-03-06T07:27:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.104+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f40fc0b2b10 con 0x7f40fc0acb20 2024-03-06T07:27:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.105+0000 7f40f3400640 1 -- 172.21.15.86:0/649623831 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v8) v1 ==== 53+0+73 (secure 0 0 0) 0x7f40e4079e00 con 0x7f40fc0acb20 2024-03-06T07:27:28.106 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d806b4d0 msgr2=0x7f40d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d806b4d0 0x7f40d806d990 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f40ec000c00 tx=0x7f40ec009040 comp rx=0 tx=0).stop 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40fc0acb20 msgr2=0x7f40fc0ce390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40fc0acb20 0x7f40fc0ce390 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f40e400c240 tx=0x7f40e402e070 comp rx=0 tx=0).stop 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 shutdown_connections 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d806b4d0 0x7f40d806d990 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40fc1f7980 0x7f40fc1f7e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40fc0acb20 0x7f40fc0ce390 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 --2- 172.21.15.86:0/649623831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40e0004800 0x7f40fc1551e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 >> 172.21.15.86:0/649623831 conn(0x7f40fc06afc0 msgr2=0x7f40fc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:28.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 shutdown_connections 2024-03-06T07:27:28.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:28.107+0000 7f4101200640 1 -- 172.21.15.86:0/649623831 wait complete. 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:self.client.name = client.2 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.2 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:27:28.129 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.2' 2024-03-06T07:27:28.129 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.130 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:27:28.184 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:28.184 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:28.184 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:28.184 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:27:28.185 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42419sec preferred_lft 42419sec 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:27:28.186 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:27:28.187 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:27:28.187 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.187 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.187 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:27:28.269 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:27:28.273 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.273 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:27:28.322 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:27:28.322 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:27:28.322 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:27:28.322 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:27:28.323 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:27:28.323 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.323 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:28.323 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:28.323 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:27:28.323 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:27:28.323 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:27:28.323 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:28.493 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.493 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:27:28.542 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:27:28.543 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.543 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list-id 2024-03-06T07:27:28.593 INFO:teuthology.orchestra.run.smithi086.stdout:nsid 0 (iproute2 netns name: ceph-ns--home-ubuntu-cephtest-mnt.0) 2024-03-06T07:27:28.594 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.594 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:28.594 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:28.594 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:27:28.594 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.2 1 2024-03-06T07:27:28.594 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:28.731 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.732 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host proto kernel_lo 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 46:60:3a:e7:07:8f brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::4460:3aff:fee7:78f/64 scope link proto kernel_ll 2024-03-06T07:27:28.816 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.820 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.820 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr 2024-03-06T07:27:28.916 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:28.916 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:28.916 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:28.916 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host proto kernel_lo 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 46:60:3a:e7:07:8f brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::4460:3aff:fee7:78f/64 scope link proto kernel_ll 2024-03-06T07:27:28.917 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:28.921 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.2' with 192.168.0.2/16 2024-03-06T07:27:28.921 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:28.921 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:28.921 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:28.921 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.2 type veth peer name brx.1 2024-03-06T07:27:28.922 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip addr add 192.168.0.2/16 brd 192.168.255.255 dev veth0 2024-03-06T07:27:28.922 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip link set veth0 up 2024-03-06T07:27:28.922 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip link set lo up 2024-03-06T07:27:28.922 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip route add default via 192.168.255.254 2024-03-06T07:27:28.922 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:29.199 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:29.200 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:27:29.200 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:27:29.200 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.1 up 2024-03-06T07:27:29.200 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set dev brx.1 master ceph-brx 2024-03-06T07:27:29.200 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:27:29.367 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:29.367 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.2 2024-03-06T07:27:29.414 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.2' 2024-03-06T07:27:29.415 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:29.415 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.2 2024-03-06T07:27:29.510 INFO:tasks.cephfs.kernel_mount:mounting using device: 2@.cephfs=/ 2024-03-06T07:27:29.510 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:29.511 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 2@.cephfs=/ /home/ubuntu/cephtest/mnt.2 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:29.649 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:29.649 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:29.687 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:29.687 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:29.687 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:27:29.687 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 2@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:27:29.687 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=2,ms_mode=prefer-crc,key=2,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:27:29.690 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.2 does not contain SELinux labels. 2024-03-06T07:27:29.690 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:27:29.690 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:27:29.690 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:27:29.690 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:27:29.694 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:27:29.694 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:29.694 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.2 2024-03-06T07:27:29.780 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:29.781 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.2 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:29.838 INFO:teuthology.orchestra.run.smithi086.stdout:client4516 2024-03-06T07:27:29.838 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:29.838 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:29.891 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:29.891 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.2 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:29.945 INFO:teuthology.orchestra.run.smithi086.stdout:client4516 2024-03-06T07:27:29.946 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:29.946 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.2 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:30.000 INFO:teuthology.orchestra.run.smithi086.stdout:client4516 2024-03-06T07:27:30.001 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:30.001 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.2 && exec bash -c 'sudo dd if=/sys/kernel/debug/ceph/013155b5-b812-4a77-8af3-f8c90b517e11.client4516/status') 2024-03-06T07:27:30.087 INFO:teuthology.orchestra.run.smithi086.stdout:instance: client.4516 (3)172.21.15.86:0/1413650770 2024-03-06T07:27:30.088 INFO:teuthology.orchestra.run.smithi086.stdout:blocklisted: false 2024-03-06T07:27:30.088 INFO:teuthology.orchestra.run.smithi086.stderr:0+1 records in 2024-03-06T07:27:30.088 INFO:teuthology.orchestra.run.smithi086.stderr:0+1 records out 2024-03-06T07:27:30.088 INFO:teuthology.orchestra.run.smithi086.stderr:70 bytes copied, 6.2815e-05 s, 1.1 MB/s 2024-03-06T07:27:30.091 INFO:tasks.kclient:client.1 config is {'syntax': 'v2'} 2024-03-06T07:27:30.092 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-06T07:27:30.092 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.1 2024-03-06T07:27:30.092 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:27:30.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.299+0000 7ff76d000640 1 Processor -- start 2024-03-06T07:27:30.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.299+0000 7ff76d000640 1 -- start start 2024-03-06T07:27:30.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.300+0000 7ff76d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff7680ae670 0x7ff76815e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.300+0000 7ff76d000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff7681620c0 0x7ff7680b28e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.300+0000 7ff76d000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff768161900 con 0x7ff7680b40c0 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.300+0000 7ff76d000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff76815ebd0 con 0x7ff7680ae670 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.300+0000 7ff76d000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff76815eeb0 con 0x7ff7681620c0 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff766a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff7680ae670 0x7ff76815e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff767400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff7680b40c0 0x7ff7680b44a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38620/0 (socket says 172.21.15.86:38620) 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff767400640 1 -- 172.21.15.86:0/2698851824 learned_addr learned my addr 172.21.15.86:0/2698851824 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff766a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff7680ae670 0x7ff76815e2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40934/0 (socket says 172.21.15.86:40934) 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff767e00640 1 --2- 172.21.15.86:0/2698851824 >> v2:172.21.15.194:3300/0 conn(0x7ff7681620c0 0x7ff7680b28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff766a00640 1 -- 172.21.15.86:0/2698851824 >> v2:172.21.15.194:3300/0 conn(0x7ff7681620c0 msgr2=0x7ff7680b28e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff766a00640 1 --2- 172.21.15.86:0/2698851824 >> v2:172.21.15.194:3300/0 conn(0x7ff7681620c0 0x7ff7680b28e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff766a00640 1 -- 172.21.15.86:0/2698851824 >> v1:172.21.15.86:6789/0 conn(0x7ff7680b40c0 legacy=0x7ff7680b44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:30.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff766a00640 1 -- 172.21.15.86:0/2698851824 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff76815f190 con 0x7ff7680ae670 2024-03-06T07:27:30.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.301+0000 7ff766000640 1 -- 172.21.15.86:0/2698851824 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 269729463 0 0) 0x7ff768161900 con 0x7ff7680b40c0 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.302+0000 7ff766a00640 1 --2- 172.21.15.86:0/2698851824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7680ae670 0x7ff76815e2c0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7ff768070800 tx=0x7ff75002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=39fddbe02ae62f2e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff766000640 1 -- 172.21.15.86:0/2698851824 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff750035020 con 0x7ff7680ae670 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff766000640 1 -- 172.21.15.86:0/2698851824 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff750031830 con 0x7ff7680ae670 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff766000640 1 -- 172.21.15.86:0/2698851824 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff750031c00 con 0x7ff7680ae670 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 -- 172.21.15.86:0/2698851824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7680ae670 msgr2=0x7ff76815e2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 --2- 172.21.15.86:0/2698851824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7680ae670 0x7ff76815e2c0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7ff768070800 tx=0x7ff75002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 -- 172.21.15.86:0/2698851824 shutdown_connections 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 --2- 172.21.15.86:0/2698851824 >> v2:172.21.15.194:3300/0 conn(0x7ff7681620c0 0x7ff7680b28e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 --2- 172.21.15.86:0/2698851824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7680ae670 0x7ff76815e2c0 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 -- 172.21.15.86:0/2698851824 >> 172.21.15.86:0/2698851824 conn(0x7ff76806afc0 msgr2=0x7ff76806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 -- 172.21.15.86:0/2698851824 shutdown_connections 2024-03-06T07:27:30.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.303+0000 7ff76d000640 1 -- 172.21.15.86:0/2698851824 wait complete. 2024-03-06T07:27:30.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.305+0000 7ff76d000640 1 Processor -- start 2024-03-06T07:27:30.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.305+0000 7ff76d000640 1 -- start start 2024-03-06T07:27:30.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.306+0000 7ff76d000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7680ae670 0x7ff768157950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:30.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.306+0000 7ff76d000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7680b40c0 0x7ff76814f4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:30.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.306+0000 7ff76d000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7681620c0 0x7ff76815b0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.306+0000 7ff76d000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff76815e980 con 0x7ff7681620c0 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.306+0000 7ff76d000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff76815e800 con 0x7ff7680ae670 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.306+0000 7ff76d000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff768158660 con 0x7ff7680b40c0 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.306+0000 7ff767e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7681620c0 0x7ff76815b0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7681620c0 0x7ff76815b0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40948/0 (socket says 172.21.15.86:40948) 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 -- 172.21.15.86:0/3738631288 learned_addr learned my addr 172.21.15.86:0/3738631288 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff766a00640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7680b40c0 0x7ff76814f4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767400640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7680ae670 0x7ff768157950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 -- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7680b40c0 msgr2=0x7ff76814f4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7680b40c0 0x7ff76814f4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 -- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7680ae670 msgr2=0x7ff768157950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7680ae670 0x7ff768157950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 -- 172.21.15.86:0/3738631288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff7681568b0 con 0x7ff7681620c0 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff766a00640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7680b40c0 0x7ff76814f4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:30.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff767e00640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7681620c0 0x7ff76815b0b0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7ff760007e40 tx=0x7ff76000e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:30.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.307+0000 7ff75f400640 1 -- 172.21.15.86:0/3738631288 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff760014620 con 0x7ff7681620c0 2024-03-06T07:27:30.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.308+0000 7ff75f400640 1 -- 172.21.15.86:0/3738631288 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff760019070 con 0x7ff7681620c0 2024-03-06T07:27:30.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.308+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff768156b90 con 0x7ff7681620c0 2024-03-06T07:27:30.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.308+0000 7ff75f400640 1 -- 172.21.15.86:0/3738631288 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff760014a20 con 0x7ff7681620c0 2024-03-06T07:27:30.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.308+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff768157150 con 0x7ff7681620c0 2024-03-06T07:27:30.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.309+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff7680ce860 con 0x7ff7681620c0 2024-03-06T07:27:30.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.310+0000 7ff75f400640 1 -- 172.21.15.86:0/3738631288 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff760014bc0 con 0x7ff7681620c0 2024-03-06T07:27:30.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.310+0000 7ff75f400640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff74406b4d0 0x7ff74406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:30.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.311+0000 7ff75f400640 1 -- 172.21.15.86:0/3738631288 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff76008ab70 con 0x7ff7681620c0 2024-03-06T07:27:30.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.311+0000 7ff767400640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff74406b4d0 0x7ff74406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:30.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.311+0000 7ff767400640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff74406b4d0 0x7ff74406d990 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7ff758000f60 tx=0x7ff758009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:30.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.315+0000 7ff75f400640 1 -- 172.21.15.86:0/3738631288 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff7680ce860 con 0x7ff7681620c0 2024-03-06T07:27:30.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.551+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7ff76804f330 con 0x7ff7681620c0 2024-03-06T07:27:30.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.552+0000 7ff75f400640 1 -- 172.21.15.86:0/3738631288 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v8) v1 ==== 53+0+73 (secure 0 0 0) 0x7ff760059950 con 0x7ff7681620c0 2024-03-06T07:27:30.553 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:27:30.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.554+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff74406b4d0 msgr2=0x7ff74406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:30.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.554+0000 7ff76d000640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff74406b4d0 0x7ff74406d990 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7ff758000f60 tx=0x7ff758009040 comp rx=0 tx=0).stop 2024-03-06T07:27:30.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.554+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7681620c0 msgr2=0x7ff76815b0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:30.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.554+0000 7ff76d000640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7681620c0 0x7ff76815b0b0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7ff760007e40 tx=0x7ff76000e4b0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 shutdown_connections 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff74406b4d0 0x7ff74406d990 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7681620c0 0x7ff76815b0b0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7680b40c0 0x7ff76814f4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 --2- 172.21.15.86:0/3738631288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7680ae670 0x7ff768157950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 >> 172.21.15.86:0/3738631288 conn(0x7ff76806afc0 msgr2=0x7ff76806d860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 shutdown_connections 2024-03-06T07:27:30.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:30.555+0000 7ff76d000640 1 -- 172.21.15.86:0/3738631288 wait complete. 2024-03-06T07:27:30.574 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-06T07:27:30.574 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:27:30.574 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi194.front.sepia.ceph.com 2024-03-06T07:27:30.575 INFO:tasks.cephfs.mount:self.client.name = client.1 2024-03-06T07:27:30.575 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.1 2024-03-06T07:27:30.575 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-06T07:27:30.575 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-06T07:27:30.575 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:27:30.575 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.1' 2024-03-06T07:27:30.575 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:30.575 DEBUG:teuthology.orchestra.run.smithi194:> ip addr 2024-03-06T07:27:30.632 INFO:teuthology.orchestra.run.smithi194.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:30.632 INFO:teuthology.orchestra.run.smithi194.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:30.632 INFO:teuthology.orchestra.run.smithi194.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:30.632 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 ::1/128 scope host 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:d9:3a:fe brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: altname enp7s0f0 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:d9:3a:ff brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: altname enp7s0f1 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:88:7e:a4 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:88:7e:a5 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: inet 172.21.15.194/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:27:30.633 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft 42613sec preferred_lft 42613sec 2024-03-06T07:27:30.634 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 fe80::ec4:7aff:fe88:7ea5/64 scope link proto kernel_ll 2024-03-06T07:27:30.634 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:30.634 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-03-06T07:27:30.634 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:30.634 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:30.634 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:30.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link add name ceph-brx type bridge 2024-03-06T07:27:30.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip addr flush dev ceph-brx 2024-03-06T07:27:30.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link set ceph-brx up 2024-03-06T07:27:30.634 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-03-06T07:27:30.634 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:31.068 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:31.068 DEBUG:teuthology.orchestra.run.smithi194:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:27:31.148 INFO:teuthology.orchestra.run.smithi194.stdout:1 2024-03-06T07:27:31.153 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:31.153 DEBUG:teuthology.orchestra.run.smithi194:> route 2024-03-06T07:27:31.215 INFO:teuthology.orchestra.run.smithi194.stdout:Kernel IP routing table 2024-03-06T07:27:31.215 INFO:teuthology.orchestra.run.smithi194.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:27:31.215 INFO:teuthology.orchestra.run.smithi194.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:27:31.215 INFO:teuthology.orchestra.run.smithi194.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:27:31.215 INFO:teuthology.orchestra.run.smithi194.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:27:31.216 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:31.216 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:31.216 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:31.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:27:31.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:27:31.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:27:31.217 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:31.889 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:31.889 DEBUG:teuthology.orchestra.run.smithi194:> ip netns list 2024-03-06T07:27:31.939 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:31.940 DEBUG:teuthology.orchestra.run.smithi194:> ip netns list-id 2024-03-06T07:27:31.984 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:31.985 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:31.985 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:31.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-06T07:27:31.985 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.1 0 2024-03-06T07:27:31.985 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:32.163 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.1' with 192.168.0.1/16 2024-03-06T07:27:32.163 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:32.163 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:32.163 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:32.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.1 type veth peer name brx.0 2024-03-06T07:27:32.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-03-06T07:27:32.163 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip link set veth0 up 2024-03-06T07:27:32.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip link set lo up 2024-03-06T07:27:32.164 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip route add default via 192.168.255.254 2024-03-06T07:27:32.164 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:32.434 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:32.434 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:32.434 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:32.434 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link set brx.0 up 2024-03-06T07:27:32.434 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link set dev brx.0 master ceph-brx 2024-03-06T07:27:32.435 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:32.584 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:32.584 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -v /home/ubuntu/cephtest/mnt.1 2024-03-06T07:27:32.627 INFO:teuthology.orchestra.run.smithi194.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.1' 2024-03-06T07:27:32.627 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:32.627 DEBUG:teuthology.orchestra.run.smithi194:> chmod 0000 /home/ubuntu/cephtest/mnt.1 2024-03-06T07:27:32.717 INFO:tasks.cephfs.kernel_mount:mounting using device: 1@.cephfs=/ 2024-03-06T07:27:32.717 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:32.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 1@.cephfs=/ /home/ubuntu/cephtest/mnt.1 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:32.847 INFO:teuthology.orchestra.run.smithi194.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:32.847 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:33.460 INFO:teuthology.orchestra.run.smithi194.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:33.460 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:33.460 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:27:33.460 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: trying mount with new device syntax: 1@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:27:33.461 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=1,ms_mode=prefer-crc,key=1,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:27:33.469 INFO:teuthology.orchestra.run.smithi194.stdout:mount: /home/ubuntu/cephtest/mnt.1 does not contain SELinux labels. 2024-03-06T07:27:33.469 INFO:teuthology.orchestra.run.smithi194.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:27:33.469 INFO:teuthology.orchestra.run.smithi194.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:27:33.469 INFO:teuthology.orchestra.run.smithi194.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:27:33.469 INFO:teuthology.orchestra.run.smithi194.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:27:33.473 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:27:33.473 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:33.473 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.1 2024-03-06T07:27:33.555 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:33.555 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.1 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:33.607 INFO:teuthology.orchestra.run.smithi194.stdout:client4620 2024-03-06T07:27:33.607 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:33.608 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:33.652 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:33.652 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.1 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:33.753 INFO:teuthology.orchestra.run.smithi194.stdout:client4620 2024-03-06T07:27:33.753 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:33.753 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.1 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:33.809 INFO:teuthology.orchestra.run.smithi194.stdout:client4620 2024-03-06T07:27:33.809 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:33.809 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.1 && exec bash -c 'sudo dd if=/sys/kernel/debug/ceph/013155b5-b812-4a77-8af3-f8c90b517e11.client4620/status') 2024-03-06T07:27:33.900 INFO:teuthology.orchestra.run.smithi194.stdout:instance: client.4620 (3)172.21.15.194:0/3064684512 2024-03-06T07:27:33.900 INFO:teuthology.orchestra.run.smithi194.stdout:blocklisted: false 2024-03-06T07:27:33.900 INFO:teuthology.orchestra.run.smithi194.stderr:0+1 records in 2024-03-06T07:27:33.900 INFO:teuthology.orchestra.run.smithi194.stderr:0+1 records out 2024-03-06T07:27:33.900 INFO:teuthology.orchestra.run.smithi194.stderr:71 bytes copied, 6.3651e-05 s, 1.1 MB/s 2024-03-06T07:27:33.904 INFO:tasks.kclient:client.3 config is {'syntax': 'v2'} 2024-03-06T07:27:33.904 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-06T07:27:33.904 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.3 2024-03-06T07:27:33.905 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:27:34.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.119+0000 7f530ba00640 1 Processor -- start 2024-03-06T07:27:34.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.119+0000 7f530ba00640 1 -- start start 2024-03-06T07:27:34.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.120+0000 7f530ba00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f53040cf430 0x7f5304070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:34.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.120+0000 7f530ba00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5304071260 0x7f530406cb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:34.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.120+0000 7f530ba00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f530406d4a0 con 0x7f530406d090 2024-03-06T07:27:34.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.120+0000 7f530ba00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f53040717c0 con 0x7f5304071260 2024-03-06T07:27:34.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.120+0000 7f530ba00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5304071aa0 con 0x7f53040cf430 2024-03-06T07:27:34.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.121+0000 7f52fbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5304071260 0x7f530406cb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.121+0000 7f52fbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5304071260 0x7f530406cb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40952/0 (socket says 172.21.15.86:40952) 2024-03-06T07:27:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.121+0000 7f52fbe00640 1 -- 172.21.15.86:0/808141237 learned_addr learned my addr 172.21.15.86:0/808141237 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.121+0000 7f52fbe00640 1 -- 172.21.15.86:0/808141237 >> v2:172.21.15.194:3300/0 conn(0x7f53040cf430 msgr2=0x7f5304070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.121+0000 7f52fbe00640 1 --2- 172.21.15.86:0/808141237 >> v2:172.21.15.194:3300/0 conn(0x7f53040cf430 0x7f5304070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.121+0000 7f52fbe00640 1 -- 172.21.15.86:0/808141237 >> v1:172.21.15.194:6789/0 conn(0x7f530406d090 legacy=0x7f5304165f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.121+0000 7f52fbe00640 1 -- 172.21.15.86:0/808141237 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5304071d80 con 0x7f5304071260 2024-03-06T07:27:34.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.122+0000 7f52fbe00640 1 --2- 172.21.15.86:0/808141237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530406cb50 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f530406bfd0 tx=0x7f52ec02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9bd987815d635fd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:34.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f5303e00640 1 -- 172.21.15.86:0/808141237 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f52ec035020 con 0x7f5304071260 2024-03-06T07:27:34.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f5303e00640 1 -- 172.21.15.86:0/808141237 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f52ec031870 con 0x7f5304071260 2024-03-06T07:27:34.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f5303e00640 1 -- 172.21.15.86:0/808141237 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f52ec031c40 con 0x7f5304071260 2024-03-06T07:27:34.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f530ba00640 1 -- 172.21.15.86:0/808141237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 msgr2=0x7f530406cb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:34.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f530ba00640 1 --2- 172.21.15.86:0/808141237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530406cb50 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f530406bfd0 tx=0x7f52ec02b910 comp rx=0 tx=0).stop 2024-03-06T07:27:34.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f530ba00640 1 -- 172.21.15.86:0/808141237 shutdown_connections 2024-03-06T07:27:34.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f530ba00640 1 --2- 172.21.15.86:0/808141237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530406cb50 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f530ba00640 1 --2- 172.21.15.86:0/808141237 >> v2:172.21.15.194:3300/0 conn(0x7f53040cf430 0x7f5304070d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.123+0000 7f530ba00640 1 -- 172.21.15.86:0/808141237 >> 172.21.15.86:0/808141237 conn(0x7f53040b1730 msgr2=0x7f53040b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:34.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.124+0000 7f530ba00640 1 -- 172.21.15.86:0/808141237 shutdown_connections 2024-03-06T07:27:34.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.124+0000 7f530ba00640 1 -- 172.21.15.86:0/808141237 wait complete. 2024-03-06T07:27:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.125+0000 7f530ba00640 1 Processor -- start 2024-03-06T07:27:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.125+0000 7f530ba00640 1 -- start start 2024-03-06T07:27:34.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.126+0000 7f530ba00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f530406d090 0x7f5304141350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.126+0000 7f530ba00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530415d010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.126+0000 7f530ba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53040cf430 0x7f5304163d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.126+0000 7f530ba00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5304162b70 con 0x7f5304071260 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.126+0000 7f530ba00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5304072340 con 0x7f530406d090 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.126+0000 7f530ba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5304162cf0 con 0x7f53040cf430 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530415d010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530415d010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40968/0 (socket says 172.21.15.86:40968) 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 -- 172.21.15.86:0/3163759791 learned_addr learned my addr 172.21.15.86:0/3163759791 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f5309000640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f530406d090 0x7f5304141350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 -- 172.21.15.86:0/3163759791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53040cf430 msgr2=0x7f5304163d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53040cf430 0x7f5304163d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 -- 172.21.15.86:0/3163759791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f530406d090 msgr2=0x7f5304141350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:34.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f530406d090 0x7f5304141350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 -- 172.21.15.86:0/3163759791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f53041601d0 con 0x7f5304071260 2024-03-06T07:27:34.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.127+0000 7f52fbe00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530415d010 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f52ec000c00 tx=0x7f52ec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:34.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.128+0000 7f5301600640 1 -- 172.21.15.86:0/3163759791 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f52ec03ad60 con 0x7f5304071260 2024-03-06T07:27:34.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.128+0000 7f5301600640 1 -- 172.21.15.86:0/3163759791 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f52ec0443f0 con 0x7f5304071260 2024-03-06T07:27:34.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.128+0000 7f5301600640 1 -- 172.21.15.86:0/3163759791 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f52ec0447e0 con 0x7f5304071260 2024-03-06T07:27:34.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.128+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f530415fef0 con 0x7f5304071260 2024-03-06T07:27:34.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.128+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5304164850 con 0x7f5304071260 2024-03-06T07:27:34.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.129+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f530404f330 con 0x7f5304071260 2024-03-06T07:27:34.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.130+0000 7f5301600640 1 -- 172.21.15.86:0/3163759791 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f52ec04b060 con 0x7f5304071260 2024-03-06T07:27:34.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.131+0000 7f5301600640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f52e006b500 0x7f52e006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:34.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.131+0000 7f5301600640 1 -- 172.21.15.86:0/3163759791 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f52ec0ac590 con 0x7f5304071260 2024-03-06T07:27:34.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.131+0000 7f5309000640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f52e006b500 0x7f52e006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:34.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.132+0000 7f5309000640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f52e006b500 0x7f52e006d9c0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f52f400b810 tx=0x7f52f4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:34.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.136+0000 7f5301600640 1 -- 172.21.15.86:0/3163759791 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f52ec03f070 con 0x7f5304071260 2024-03-06T07:27:34.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.373+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f53040b4a40 con 0x7f5304071260 2024-03-06T07:27:34.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.373+0000 7f5301600640 1 -- 172.21.15.86:0/3163759791 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v8) v1 ==== 53+0+73 (secure 0 0 0) 0x7f530406d4a0 con 0x7f5304071260 2024-03-06T07:27:34.375 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f52e006b500 msgr2=0x7f52e006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f52e006b500 0x7f52e006d9c0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f52f400b810 tx=0x7f52f4008040 comp rx=0 tx=0).stop 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 msgr2=0x7f530415d010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530415d010 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f52ec000c00 tx=0x7f52ec0046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 shutdown_connections 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f52e006b500 0x7f52e006d9c0 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53040cf430 0x7f5304163d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5304071260 0x7f530415d010 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 --2- 172.21.15.86:0/3163759791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f530406d090 0x7f5304141350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:34.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 >> 172.21.15.86:0/3163759791 conn(0x7f53040b1730 msgr2=0x7f5304158800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:34.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 shutdown_connections 2024-03-06T07:27:34.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:34.376+0000 7f530ba00640 1 -- 172.21.15.86:0/3163759791 wait complete. 2024-03-06T07:27:34.396 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-06T07:27:34.396 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:27:34.396 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi194.front.sepia.ceph.com 2024-03-06T07:27:34.396 INFO:tasks.cephfs.mount:self.client.name = client.3 2024-03-06T07:27:34.397 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.3 2024-03-06T07:27:34.397 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-06T07:27:34.397 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-06T07:27:34.397 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:27:34.397 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.3' 2024-03-06T07:27:34.397 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:34.397 DEBUG:teuthology.orchestra.run.smithi194:> ip addr 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 ::1/128 scope host 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:34.454 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:d9:3a:fe brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: altname enp7s0f0 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:d9:3a:ff brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: altname enp7s0f1 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:88:7e:a4 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 0c:c4:7a:88:7e:a5 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: inet 172.21.15.194/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft 42609sec preferred_lft 42609sec 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 fe80::ec4:7aff:fe88:7ea5/64 scope link proto kernel_ll 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:27:34.455 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 22:9c:1e:4a:ae:fc brd ff:ff:ff:ff:ff:ff 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 fe80::74b2:32ff:fe0f:1292/64 scope link proto kernel_ll 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 22:9c:1e:4a:ae:fc brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 fe80::209c:1eff:fe4a:aefc/64 scope link proto kernel_ll 2024-03-06T07:27:34.456 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:34.457 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:34.457 DEBUG:teuthology.orchestra.run.smithi194:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:27:34.538 INFO:teuthology.orchestra.run.smithi194.stdout:1 2024-03-06T07:27:34.543 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:34.543 DEBUG:teuthology.orchestra.run.smithi194:> route 2024-03-06T07:27:34.589 INFO:teuthology.orchestra.run.smithi194.stdout:Kernel IP routing table 2024-03-06T07:27:34.589 INFO:teuthology.orchestra.run.smithi194.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:27:34.590 INFO:teuthology.orchestra.run.smithi194.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:27:34.590 INFO:teuthology.orchestra.run.smithi194.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:27:34.590 INFO:teuthology.orchestra.run.smithi194.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:27:34.590 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:34.590 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:34.590 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:34.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:27:34.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:27:34.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:27:34.590 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:34.810 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:34.810 DEBUG:teuthology.orchestra.run.smithi194:> ip netns list 2024-03-06T07:27:34.861 INFO:teuthology.orchestra.run.smithi194.stdout:ceph-ns--home-ubuntu-cephtest-mnt.1 (id: 0) 2024-03-06T07:27:34.863 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:34.863 DEBUG:teuthology.orchestra.run.smithi194:> ip netns list-id 2024-03-06T07:27:34.914 INFO:teuthology.orchestra.run.smithi194.stdout:nsid 0 (iproute2 netns name: ceph-ns--home-ubuntu-cephtest-mnt.1) 2024-03-06T07:27:34.914 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:34.914 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:34.914 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:34.914 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.3 2024-03-06T07:27:34.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.3 1 2024-03-06T07:27:34.915 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:35.045 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:35.045 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip addr 2024-03-06T07:27:35.128 INFO:teuthology.orchestra.run.smithi194.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:35.128 INFO:teuthology.orchestra.run.smithi194.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:35.128 INFO:teuthology.orchestra.run.smithi194.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:35.128 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.128 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 ::1/128 scope host proto kernel_lo 2024-03-06T07:27:35.128 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.129 INFO:teuthology.orchestra.run.smithi194.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:27:35.129 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 76:a7:7f:fe:a9:b5 brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-06T07:27:35.129 INFO:teuthology.orchestra.run.smithi194.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-06T07:27:35.129 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.129 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 fe80::74a7:7fff:fefe:a9b5/64 scope link proto kernel_ll 2024-03-06T07:27:35.129 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.132 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:35.132 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip addr 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 ::1/128 scope host proto kernel_lo 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: link/ether 76:a7:7f:fe:a9:b5 brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-06T07:27:35.225 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.226 INFO:teuthology.orchestra.run.smithi194.stdout: inet6 fe80::74a7:7fff:fefe:a9b5/64 scope link proto kernel_ll 2024-03-06T07:27:35.226 INFO:teuthology.orchestra.run.smithi194.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:27:35.230 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.3' with 192.168.0.2/16 2024-03-06T07:27:35.230 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.3 type veth peer name brx.1 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip addr add 192.168.0.2/16 brd 192.168.255.255 dev veth0 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip link set veth0 up 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip link set lo up 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip route add default via 192.168.255.254 2024-03-06T07:27:35.230 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:35.495 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:35.495 DEBUG:teuthology.orchestra.run.smithi194:> (cd / && exec bash -c ' 2024-03-06T07:27:35.495 DEBUG:teuthology.orchestra.run.smithi194:> set -e 2024-03-06T07:27:35.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link set brx.1 up 2024-03-06T07:27:35.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo ip link set dev brx.1 master ceph-brx 2024-03-06T07:27:35.495 DEBUG:teuthology.orchestra.run.smithi194:> ') 2024-03-06T07:27:35.651 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:35.651 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -v /home/ubuntu/cephtest/mnt.3 2024-03-06T07:27:35.698 INFO:teuthology.orchestra.run.smithi194.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.3' 2024-03-06T07:27:35.698 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:27:35.698 DEBUG:teuthology.orchestra.run.smithi194:> chmod 0000 /home/ubuntu/cephtest/mnt.3 2024-03-06T07:27:35.787 INFO:tasks.cephfs.kernel_mount:mounting using device: 3@.cephfs=/ 2024-03-06T07:27:35.788 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:35.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 3@.cephfs=/ /home/ubuntu/cephtest/mnt.3 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:35.906 INFO:teuthology.orchestra.run.smithi194.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:35.906 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:35.940 INFO:teuthology.orchestra.run.smithi194.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:27:35.940 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:27:35.940 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:27:35.940 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: trying mount with new device syntax: 3@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:27:35.940 INFO:teuthology.orchestra.run.smithi194.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=3,ms_mode=prefer-crc,key=3,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:27:35.942 INFO:teuthology.orchestra.run.smithi194.stdout:mount: /home/ubuntu/cephtest/mnt.3 does not contain SELinux labels. 2024-03-06T07:27:35.942 INFO:teuthology.orchestra.run.smithi194.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:27:35.942 INFO:teuthology.orchestra.run.smithi194.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:27:35.942 INFO:teuthology.orchestra.run.smithi194.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:27:35.942 INFO:teuthology.orchestra.run.smithi194.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:27:35.946 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:27:35.946 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:35.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.3 2024-03-06T07:27:36.017 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:36.018 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.3 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:36.064 INFO:teuthology.orchestra.run.smithi194.stdout:client4525 2024-03-06T07:27:36.065 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:36.065 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:36.155 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:36.155 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.3 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:36.251 INFO:teuthology.orchestra.run.smithi194.stdout:client4525 2024-03-06T07:27:36.251 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:36.251 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.3 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:27:36.300 INFO:teuthology.orchestra.run.smithi194.stdout:client4525 2024-03-06T07:27:36.300 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:36.300 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest/mnt.3 && exec bash -c 'sudo dd if=/sys/kernel/debug/ceph/013155b5-b812-4a77-8af3-f8c90b517e11.client4525/status') 2024-03-06T07:27:36.375 INFO:teuthology.orchestra.run.smithi194.stdout:instance: client.4525 (3)192.168.0.2:0/117760668 2024-03-06T07:27:36.375 INFO:teuthology.orchestra.run.smithi194.stdout:blocklisted: false 2024-03-06T07:27:36.375 INFO:teuthology.orchestra.run.smithi194.stderr:0+1 records in 2024-03-06T07:27:36.375 INFO:teuthology.orchestra.run.smithi194.stderr:0+1 records out 2024-03-06T07:27:36.375 INFO:teuthology.orchestra.run.smithi194.stderr:68 bytes copied, 6.6193e-05 s, 1.0 MB/s 2024-03-06T07:27:36.378 INFO:teuthology.run_tasks:Running task cephfs_test_runner... 2024-03-06T07:27:36.395 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:36.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.613+0000 7f644b200640 1 Processor -- start 2024-03-06T07:27:36.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.613+0000 7f644b200640 1 -- start start 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f644b200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f644c15a940 0x7f644c0b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f644b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c06d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f644b200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f644c0b2b30 con 0x7f644c0cc2a0 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f644b200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f644c0b28e0 con 0x7f644c15a940 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f644b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f644c0b2e10 con 0x7f644c0cf370 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f6449e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f644c15a940 0x7f644c0b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f6449e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f644c15a940 0x7f644c0b45e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36394/0 (socket says 172.21.15.86:36394) 2024-03-06T07:27:36.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f6449e00640 1 -- 172.21.15.86:0/2550479076 learned_addr learned my addr 172.21.15.86:0/2550479076 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:36.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f6449e00640 1 -- 172.21.15.86:0/2550479076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 msgr2=0x7f644c06d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:36.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.615+0000 7f6449400640 1 --2- 172.21.15.86:0/2550479076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c06d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:36.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.616+0000 7f6449e00640 1 --2- 172.21.15.86:0/2550479076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c06d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.616+0000 7f6449e00640 1 -- 172.21.15.86:0/2550479076 >> v1:172.21.15.194:6789/0 conn(0x7f644c0cc2a0 legacy=0x7f644c0cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:36.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.616+0000 7f6449e00640 1 -- 172.21.15.86:0/2550479076 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f644c06ca50 con 0x7f644c15a940 2024-03-06T07:27:36.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.616+0000 7f6449400640 1 --2- 172.21.15.86:0/2550479076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c06d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:36.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.617+0000 7f6449e00640 1 --2- 172.21.15.86:0/2550479076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c15a940 0x7f644c0b45e0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f643c009e50 tx=0x7f643c02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f7b482ca791f3e41 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.617+0000 7f6448a00640 1 -- 172.21.15.86:0/2550479076 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f643c02ca80 con 0x7f644c15a940 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.617+0000 7f6448a00640 1 -- 172.21.15.86:0/2550479076 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f643c02cc20 con 0x7f644c15a940 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.617+0000 7f6448a00640 1 -- 172.21.15.86:0/2550479076 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f643c0343f0 con 0x7f644c15a940 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 -- 172.21.15.86:0/2550479076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c15a940 msgr2=0x7f644c0b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 --2- 172.21.15.86:0/2550479076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c15a940 0x7f644c0b45e0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f643c009e50 tx=0x7f643c02bee0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 -- 172.21.15.86:0/2550479076 shutdown_connections 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 --2- 172.21.15.86:0/2550479076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c06d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 --2- 172.21.15.86:0/2550479076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c15a940 0x7f644c0b45e0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 -- 172.21.15.86:0/2550479076 >> 172.21.15.86:0/2550479076 conn(0x7f644c06afc0 msgr2=0x7f644c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 -- 172.21.15.86:0/2550479076 shutdown_connections 2024-03-06T07:27:36.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.618+0000 7f644b200640 1 -- 172.21.15.86:0/2550479076 wait complete. 2024-03-06T07:27:36.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.620+0000 7f644b200640 1 Processor -- start 2024-03-06T07:27:36.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.620+0000 7f644b200640 1 -- start start 2024-03-06T07:27:36.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.621+0000 7f644b200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c0cc2a0 0x7f644c155cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:36.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.621+0000 7f644b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c071c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:36.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.621+0000 7f644b200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f644c15a940 0x7f644c0721c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:36.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.621+0000 7f644b200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f644c0b4b20 con 0x7f644c0cc2a0 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.621+0000 7f644b200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f644c0b30f0 con 0x7f644c15a940 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.621+0000 7f644b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f644c06d790 con 0x7f644c0cf370 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.621+0000 7f6449e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c0cc2a0 0x7f644c155cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c0cc2a0 0x7f644c155cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36402/0 (socket says 172.21.15.86:36402) 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 -- 172.21.15.86:0/1515811363 learned_addr learned my addr 172.21.15.86:0/1515811363 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449400640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c071c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f644a800640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f644c15a940 0x7f644c0721c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 -- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 msgr2=0x7f644c071c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c071c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 -- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f644c15a940 msgr2=0x7f644c0721c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f644c15a940 0x7f644c0721c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 -- 172.21.15.86:0/1515811363 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f644c1e6a60 con 0x7f644c0cc2a0 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449400640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c071c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:36.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.622+0000 7f6449e00640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c0cc2a0 0x7f644c155cf0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f643c009e50 tx=0x7f643c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:36.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.623+0000 7f6442000640 1 -- 172.21.15.86:0/1515811363 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f643c043020 con 0x7f644c0cc2a0 2024-03-06T07:27:36.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.623+0000 7f6442000640 1 -- 172.21.15.86:0/1515811363 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f643c03e450 con 0x7f644c0cc2a0 2024-03-06T07:27:36.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.623+0000 7f6442000640 1 -- 172.21.15.86:0/1515811363 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f643c009510 con 0x7f644c0cc2a0 2024-03-06T07:27:36.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.623+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f644c1e6d40 con 0x7f644c0cc2a0 2024-03-06T07:27:36.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.623+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f644c1f1d30 con 0x7f644c0cc2a0 2024-03-06T07:27:36.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.624+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f644c04f330 con 0x7f644c0cc2a0 2024-03-06T07:27:36.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.625+0000 7f6442000640 1 -- 172.21.15.86:0/1515811363 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f643c046020 con 0x7f644c0cc2a0 2024-03-06T07:27:36.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.626+0000 7f6442000640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f642c06b4d0 0x7f642c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:36.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.626+0000 7f6442000640 1 -- 172.21.15.86:0/1515811363 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f643c0acb80 con 0x7f644c0cc2a0 2024-03-06T07:27:36.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.626+0000 7f6449400640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f642c06b4d0 0x7f642c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:36.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.627+0000 7f6449400640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f642c06b4d0 0x7f642c06d990 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f644c159ae0 tx=0x7f6438008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:36.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.630+0000 7f6442000640 1 -- 172.21.15.86:0/1515811363 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f643c04d030 con 0x7f644c0cc2a0 2024-03-06T07:27:36.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.827+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f644c06e180 con 0x7f644c0cc2a0 2024-03-06T07:27:36.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.830+0000 7f6442000640 1 -- 172.21.15.86:0/1515811363 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f643c07b9e0 con 0x7f644c0cc2a0 2024-03-06T07:27:36.832 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:36.832 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:36.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.832+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f642c06b4d0 msgr2=0x7f642c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:36.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.832+0000 7f644b200640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f642c06b4d0 0x7f642c06d990 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f644c159ae0 tx=0x7f6438008040 comp rx=0 tx=0).stop 2024-03-06T07:27:36.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.832+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c0cc2a0 msgr2=0x7f644c155cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:36.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.832+0000 7f644b200640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c0cc2a0 0x7f644c155cf0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f643c009e50 tx=0x7f643c004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:36.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 shutdown_connections 2024-03-06T07:27:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f642c06b4d0 0x7f642c06d990 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f644c15a940 0x7f644c0721c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f644c0cf370 0x7f644c071c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 --2- 172.21.15.86:0/1515811363 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f644c0cc2a0 0x7f644c155cf0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 >> 172.21.15.86:0/1515811363 conn(0x7f644c06afc0 msgr2=0x7f644c071100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 shutdown_connections 2024-03-06T07:27:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:36.833+0000 7f644b200640 1 -- 172.21.15.86:0/1515811363 wait complete. 2024-03-06T07:27:36.854 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:36.854 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:37.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.061+0000 7f4906e00640 1 Processor -- start 2024-03-06T07:27:37.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.061+0000 7f4906e00640 1 -- start start 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f4906e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f490015fc10 0x7f4900070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f4906e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f490015fff0 0x7f4900160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f4906e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f49000b46c0 con 0x7f49000ce4a0 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f4906e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f49000b49a0 con 0x7f490015fc10 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f4906e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4900072c90 con 0x7f490015fff0 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f48ffe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f490015fc10 0x7f4900070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f48ffe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f490015fc10 0x7f4900070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36416/0 (socket says 172.21.15.86:36416) 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f48ffe00640 1 -- 172.21.15.86:0/4265816285 learned_addr learned my addr 172.21.15.86:0/4265816285 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:37.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.063+0000 7f48ff400640 1 --2- 172.21.15.86:0/4265816285 >> v2:172.21.15.194:3300/0 conn(0x7f490015fff0 0x7f4900160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.064+0000 7f48ffe00640 1 -- 172.21.15.86:0/4265816285 >> v2:172.21.15.194:3300/0 conn(0x7f490015fff0 msgr2=0x7f4900160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:37.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.064+0000 7f48ffe00640 1 --2- 172.21.15.86:0/4265816285 >> v2:172.21.15.194:3300/0 conn(0x7f490015fff0 0x7f4900160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.064+0000 7f48ffe00640 1 -- 172.21.15.86:0/4265816285 >> v1:172.21.15.194:6789/0 conn(0x7f49000ce4a0 legacy=0x7f49000ce8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:37.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.064+0000 7f48ffe00640 1 -- 172.21.15.86:0/4265816285 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4900160990 con 0x7f490015fc10 2024-03-06T07:27:37.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.065+0000 7f48ffe00640 1 --2- 172.21.15.86:0/4265816285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fc10 0x7f4900070ff0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f4900074a50 tx=0x7f48f002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a734ec466a371895 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.065+0000 7f48fea00640 1 -- 172.21.15.86:0/4265816285 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48f0035020 con 0x7f490015fc10 2024-03-06T07:27:37.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.065+0000 7f48fea00640 1 -- 172.21.15.86:0/4265816285 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f48f0031870 con 0x7f490015fc10 2024-03-06T07:27:37.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.065+0000 7f48fea00640 1 -- 172.21.15.86:0/4265816285 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48f0031c40 con 0x7f490015fc10 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 -- 172.21.15.86:0/4265816285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fc10 msgr2=0x7f4900070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 --2- 172.21.15.86:0/4265816285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fc10 0x7f4900070ff0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f4900074a50 tx=0x7f48f002b910 comp rx=0 tx=0).stop 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 -- 172.21.15.86:0/4265816285 shutdown_connections 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 --2- 172.21.15.86:0/4265816285 >> v2:172.21.15.194:3300/0 conn(0x7f490015fff0 0x7f4900160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 --2- 172.21.15.86:0/4265816285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fc10 0x7f4900070ff0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 -- 172.21.15.86:0/4265816285 >> 172.21.15.86:0/4265816285 conn(0x7f49000b1940 msgr2=0x7f49000b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 -- 172.21.15.86:0/4265816285 shutdown_connections 2024-03-06T07:27:37.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.066+0000 7f4906e00640 1 -- 172.21.15.86:0/4265816285 wait complete. 2024-03-06T07:27:37.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.068+0000 7f4906e00640 1 Processor -- start 2024-03-06T07:27:37.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.068+0000 7f4906e00640 1 -- start start 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4906e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f49000ce4a0 0x7f490015e7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4906e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f490015fc10 0x7f49000b0440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4906e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fff0 0x7f49000b0980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4906e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4900072a30 con 0x7f490015fff0 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4906e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49000728b0 con 0x7f49000ce4a0 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4906e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f490015c2e0 con 0x7f490015fc10 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4904e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fff0 0x7f49000b0980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4904e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fff0 0x7f49000b0980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36420/0 (socket says 172.21.15.86:36420) 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f4904e00640 1 -- 172.21.15.86:0/2974223674 learned_addr learned my addr 172.21.15.86:0/2974223674 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:37.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f48ff400640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f490015fc10 0x7f49000b0440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.069+0000 7f48ffe00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f49000ce4a0 0x7f490015e7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4904e00640 1 -- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f490015fc10 msgr2=0x7f49000b0440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4904e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f490015fc10 0x7f49000b0440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4904e00640 1 -- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f49000ce4a0 msgr2=0x7f490015e7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4904e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f49000ce4a0 0x7f490015e7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4904e00640 1 -- 172.21.15.86:0/2974223674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f490015ca90 con 0x7f490015fff0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f48ff400640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f490015fc10 0x7f49000b0440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f48ffe00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f49000ce4a0 0x7f490015e7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4904e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fff0 0x7f49000b0980 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f48f800c7b0 tx=0x7f48f800cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f48f7e00640 1 -- 172.21.15.86:0/2974223674 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48f8004b60 con 0x7f490015fff0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f48f7e00640 1 -- 172.21.15.86:0/2974223674 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f48f8004d00 con 0x7f490015fff0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f48f7e00640 1 -- 172.21.15.86:0/2974223674 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48f800aa90 con 0x7f490015fff0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f490015cd70 con 0x7f490015fff0 2024-03-06T07:27:37.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.070+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f490015d330 con 0x7f490015fff0 2024-03-06T07:27:37.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.071+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f490004f330 con 0x7f490015fff0 2024-03-06T07:27:37.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.072+0000 7f48f7e00640 1 -- 172.21.15.86:0/2974223674 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f48f800ac30 con 0x7f490015fff0 2024-03-06T07:27:37.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.073+0000 7f48f7e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f48e006b4d0 0x7f48e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.073+0000 7f48f7e00640 1 -- 172.21.15.86:0/2974223674 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f48f808a4f0 con 0x7f490015fff0 2024-03-06T07:27:37.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.073+0000 7f48ffe00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f48e006b4d0 0x7f48e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.074+0000 7f48ffe00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f48e006b4d0 0x7f48e006d990 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f48f0036000 tx=0x7f48f0008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.077+0000 7f48f7e00640 1 -- 172.21.15.86:0/2974223674 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f48f8020070 con 0x7f490015fff0 2024-03-06T07:27:37.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.272+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f49000b3ec0 con 0x7f490015fff0 2024-03-06T07:27:37.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.273+0000 7f48f7e00640 1 -- 172.21.15.86:0/2974223674 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f48f8088e20 con 0x7f490015fff0 2024-03-06T07:27:37.275 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f48e006b4d0 msgr2=0x7f48e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f48e006b4d0 0x7f48e006d990 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f48f0036000 tx=0x7f48f0008030 comp rx=0 tx=0).stop 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fff0 msgr2=0x7f49000b0980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fff0 0x7f49000b0980 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f48f800c7b0 tx=0x7f48f800cc70 comp rx=0 tx=0).stop 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 shutdown_connections 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f48e006b4d0 0x7f48e006d990 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f490015fff0 0x7f49000b0980 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f490015fc10 0x7f49000b0440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 --2- 172.21.15.86:0/2974223674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f49000ce4a0 0x7f490015e7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 >> 172.21.15.86:0/2974223674 conn(0x7f49000b1940 msgr2=0x7f49000afc40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:37.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 shutdown_connections 2024-03-06T07:27:37.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.276+0000 7f4906e00640 1 -- 172.21.15.86:0/2974223674 wait complete. 2024-03-06T07:27:37.297 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:37.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.497+0000 7f7e57400640 1 Processor -- start 2024-03-06T07:27:37.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.498+0000 7f7e57400640 1 -- start start 2024-03-06T07:27:37.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.499+0000 7f7e57400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 0x7f7e5006dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.499+0000 7f7e57400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e50071ea0 con 0x7f7e5015efa0 2024-03-06T07:27:37.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.499+0000 7f7e57400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e5006e380 con 0x7f7e5006afc0 2024-03-06T07:27:37.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.499+0000 7f7e57400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e5015bf70 con 0x7f7e5015bb90 2024-03-06T07:27:37.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.499+0000 7f7e4fe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7e5015efa0 0x7f7e500cec20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57534/0 (socket says 172.21.15.86:57534) 2024-03-06T07:27:37.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4fe00640 1 -- 172.21.15.86:0/3705423374 learned_addr learned my addr 172.21.15.86:0/3705423374 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:37.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e54a00640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 0x7f7e5006dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1928968537 0 0) 0x7f7e50071ea0 con 0x7f7e5015efa0 2024-03-06T07:27:37.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7e34003520 con 0x7f7e5015efa0 2024-03-06T07:27:37.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2791431854 0 0) 0x7f7e34003520 con 0x7f7e5015efa0 2024-03-06T07:27:37.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 msgr2=0x7f7e5006dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 0x7f7e5006dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 >> v1:172.21.15.194:6789/0 conn(0x7f7e5006afc0 legacy=0x7f7e500723a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e5006e660 con 0x7f7e5015efa0 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7e3c001670 con 0x7f7e5015efa0 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 >> v1:172.21.15.86:6789/0 conn(0x7f7e5015efa0 legacy=0x7f7e500cec20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.500+0000 7f7e4f400640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e340039e0 0x7f7e34003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e4f400640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e34004800 0x7f7e34004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e4f400640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 0x7f7e340051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e5015bf70 con 0x7f7e340039e0 2024-03-06T07:27:37.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e3400ba70 con 0x7f7e34004800 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e3400bc70 con 0x7f7e5015bb90 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e54a00640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e340039e0 0x7f7e34003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e55400640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 0x7f7e340051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e4fe00640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e34004800 0x7f7e34004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e54a00640 1 -- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 msgr2=0x7f7e340051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e54a00640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 0x7f7e340051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e54a00640 1 -- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e34004800 msgr2=0x7f7e34004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e54a00640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e34004800 0x7f7e34004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e54a00640 1 -- 172.21.15.86:0/3705423374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e34003520 con 0x7f7e340039e0 2024-03-06T07:27:37.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e4fe00640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e34004800 0x7f7e34004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.501+0000 7f7e55400640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e5015bb90 0x7f7e340051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.502+0000 7f7e54a00640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e340039e0 0x7f7e34003e00 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f7e40009a90 tx=0x7f7e34005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5cef590057bc6192 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e40039070 con 0x7f7e340039e0 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e4002c660 con 0x7f7e340039e0 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e4f400640 1 -- 172.21.15.86:0/3705423374 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e40009690 con 0x7f7e340039e0 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e57400640 1 -- 172.21.15.86:0/3705423374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e340039e0 msgr2=0x7f7e34003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e57400640 1 --2- 172.21.15.86:0/3705423374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e340039e0 0x7f7e34003e00 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f7e40009a90 tx=0x7f7e34005830 comp rx=0 tx=0).stop 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e55400640 1 -- 172.21.15.86:0/3705423374 reap_dead start 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e57400640 1 -- 172.21.15.86:0/3705423374 shutdown_connections 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e57400640 1 -- 172.21.15.86:0/3705423374 >> 172.21.15.86:0/3705423374 conn(0x7f7e500b27d0 msgr2=0x7f7e500b2bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e57400640 1 -- 172.21.15.86:0/3705423374 shutdown_connections 2024-03-06T07:27:37.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.503+0000 7f7e57400640 1 -- 172.21.15.86:0/3705423374 wait complete. 2024-03-06T07:27:37.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.505+0000 7f7e57400640 1 Processor -- start 2024-03-06T07:27:37.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.505+0000 7f7e57400640 1 -- start start 2024-03-06T07:27:37.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.506+0000 7f7e57400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e340039e0 0x7f7e5014c5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.506+0000 7f7e57400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e500cb9a0 0x7f7e500cbe20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.506+0000 7f7e57400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e500cc360 0x7f7e500b4a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.506+0000 7f7e57400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e5015f400 con 0x7f7e500cc360 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.506+0000 7f7e57400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e5006ec20 con 0x7f7e500cb9a0 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.506+0000 7f7e57400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e5006dcf0 con 0x7f7e340039e0 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.506+0000 7f7e55400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e500cc360 0x7f7e500b4a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e500cc360 0x7f7e500b4a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36436/0 (socket says 172.21.15.86:36436) 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 -- 172.21.15.86:0/2391139596 learned_addr learned my addr 172.21.15.86:0/2391139596 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e54a00640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e340039e0 0x7f7e5014c5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e4fe00640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e500cb9a0 0x7f7e500cbe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 -- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e340039e0 msgr2=0x7f7e5014c5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e340039e0 0x7f7e5014c5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 -- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e500cb9a0 msgr2=0x7f7e500cbe20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e500cb9a0 0x7f7e500cbe20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 -- 172.21.15.86:0/2391139596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e50150880 con 0x7f7e500cc360 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e54a00640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e340039e0 0x7f7e5014c5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e55400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e500cc360 0x7f7e500b4a30 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f7e500ce620 tx=0x7f7e4800dc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e4cc00640 1 -- 172.21.15.86:0/2391139596 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e48006d50 con 0x7f7e500cc360 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.507+0000 7f7e4cc00640 1 -- 172.21.15.86:0/2391139596 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e48003970 con 0x7f7e500cc360 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.508+0000 7f7e4cc00640 1 -- 172.21.15.86:0/2391139596 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e48003da0 con 0x7f7e500cc360 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.508+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7e500b3f50 con 0x7f7e500cc360 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.508+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e500b4510 con 0x7f7e500cc360 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.509+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7e5004f330 con 0x7f7e500cc360 2024-03-06T07:27:37.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.510+0000 7f7e4cc00640 1 -- 172.21.15.86:0/2391139596 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7e4801b020 con 0x7f7e500cc360 2024-03-06T07:27:37.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.510+0000 7f7e4cc00640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e3006b500 0x7f7e3006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.511+0000 7f7e4cc00640 1 -- 172.21.15.86:0/2391139596 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7e48089e10 con 0x7f7e500cc360 2024-03-06T07:27:37.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.511+0000 7f7e54a00640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e3006b500 0x7f7e3006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.511+0000 7f7e54a00640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e3006b500 0x7f7e3006d9c0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f7e40036000 tx=0x7f7e40030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.515+0000 7f7e4cc00640 1 -- 172.21.15.86:0/2391139596 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7e4808d030 con 0x7f7e500cc360 2024-03-06T07:27:37.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.714+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f7e500b46e0 con 0x7f7e500cc360 2024-03-06T07:27:37.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.715+0000 7f7e4cc00640 1 -- 172.21.15.86:0/2391139596 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7f7e48058b60 con 0x7f7e500cc360 2024-03-06T07:27:37.717 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e3006b500 msgr2=0x7f7e3006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e3006b500 0x7f7e3006d9c0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f7e40036000 tx=0x7f7e40030000 comp rx=0 tx=0).stop 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e500cc360 msgr2=0x7f7e500b4a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e500cc360 0x7f7e500b4a30 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f7e500ce620 tx=0x7f7e4800dc00 comp rx=0 tx=0).stop 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 shutdown_connections 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e3006b500 0x7f7e3006d9c0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e500cc360 0x7f7e500b4a30 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e500cb9a0 0x7f7e500cbe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 --2- 172.21.15.86:0/2391139596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e340039e0 0x7f7e5014c5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 >> 172.21.15.86:0/2391139596 conn(0x7f7e500b27d0 msgr2=0x7f7e5015c120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:37.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 shutdown_connections 2024-03-06T07:27:37.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.718+0000 7f7e57400640 1 -- 172.21.15.86:0/2391139596 wait complete. 2024-03-06T07:27:37.739 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:37.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.952+0000 7f5f37200640 1 Processor -- start 2024-03-06T07:27:37.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.952+0000 7f5f37200640 1 -- start start 2024-03-06T07:27:37.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.953+0000 7f5f37200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5f3006da40 0x7f5f300ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.953+0000 7f5f37200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f300aef20 0x7f5f300ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.953+0000 7f5f37200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5f3015c1f0 con 0x7f5f3015be10 2024-03-06T07:27:37.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.953+0000 7f5f37200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5f300cd200 con 0x7f5f3006da40 2024-03-06T07:27:37.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.953+0000 7f5f37200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f3006cac0 con 0x7f5f300aef20 2024-03-06T07:27:37.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.954+0000 7f5f35400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5f3006da40 0x7f5f300ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.954+0000 7f5f36800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f300aef20 0x7f5f300ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.954+0000 7f5f35e00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f5f3015be10 0x7f5f3006d330 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:41570/0 (socket says 172.21.15.86:41570) 2024-03-06T07:27:37.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.954+0000 7f5f35e00640 1 -- 172.21.15.86:0/1055318956 learned_addr learned my addr 172.21.15.86:0/1055318956 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:37.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.955+0000 7f5f36800640 1 -- 172.21.15.86:0/1055318956 >> v2:172.21.15.194:3300/0 conn(0x7f5f3006da40 msgr2=0x7f5f300ccc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:37.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.955+0000 7f5f36800640 1 --2- 172.21.15.86:0/1055318956 >> v2:172.21.15.194:3300/0 conn(0x7f5f3006da40 0x7f5f300ccc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.955+0000 7f5f36800640 1 -- 172.21.15.86:0/1055318956 >> v1:172.21.15.194:6789/0 conn(0x7f5f3015be10 legacy=0x7f5f3006d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.955+0000 7f5f36800640 1 -- 172.21.15.86:0/1055318956 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f3006cda0 con 0x7f5f300aef20 2024-03-06T07:27:37.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.956+0000 7f5f36800640 1 --2- 172.21.15.86:0/1055318956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f300aef20 0x7f5f300ce2b0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f5f2c009d90 tx=0x7f5f2c02bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=31f9e35ed1960d06 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f34a00640 1 -- 172.21.15.86:0/1055318956 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f2c039070 con 0x7f5f300aef20 2024-03-06T07:27:37.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f34a00640 1 -- 172.21.15.86:0/1055318956 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f2c02cc50 con 0x7f5f300aef20 2024-03-06T07:27:37.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f34a00640 1 -- 172.21.15.86:0/1055318956 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f2c034440 con 0x7f5f300aef20 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f37200640 1 -- 172.21.15.86:0/1055318956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f300aef20 msgr2=0x7f5f300ce2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f37200640 1 --2- 172.21.15.86:0/1055318956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f300aef20 0x7f5f300ce2b0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f5f2c009d90 tx=0x7f5f2c02bea0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f37200640 1 -- 172.21.15.86:0/1055318956 shutdown_connections 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f37200640 1 --2- 172.21.15.86:0/1055318956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f300aef20 0x7f5f300ce2b0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f37200640 1 --2- 172.21.15.86:0/1055318956 >> v2:172.21.15.194:3300/0 conn(0x7f5f3006da40 0x7f5f300ccc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.957+0000 7f5f37200640 1 -- 172.21.15.86:0/1055318956 >> 172.21.15.86:0/1055318956 conn(0x7f5f3006afc0 msgr2=0x7f5f3006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.958+0000 7f5f37200640 1 -- 172.21.15.86:0/1055318956 shutdown_connections 2024-03-06T07:27:37.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.958+0000 7f5f37200640 1 -- 172.21.15.86:0/1055318956 wait complete. 2024-03-06T07:27:37.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.959+0000 7f5f37200640 1 Processor -- start 2024-03-06T07:27:37.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.959+0000 7f5f37200640 1 -- start start 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.960+0000 7f5f37200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f3006da40 0x7f5f3013f1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.960+0000 7f5f37200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f300aef20 0x7f5f301580e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.960+0000 7f5f37200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f3015be10 0x7f5f3015a950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.960+0000 7f5f37200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f300cda90 con 0x7f5f300aef20 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.960+0000 7f5f37200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f3006d080 con 0x7f5f3006da40 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.960+0000 7f5f37200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f30071ab0 con 0x7f5f3015be10 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f300aef20 0x7f5f301580e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f300aef20 0x7f5f301580e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36442/0 (socket says 172.21.15.86:36442) 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 -- 172.21.15.86:0/383353443 learned_addr learned my addr 172.21.15.86:0/383353443 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35e00640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f3006da40 0x7f5f3013f1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 -- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f3015be10 msgr2=0x7f5f3015a950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f36800640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f3015be10 0x7f5f3015a950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f3015be10 0x7f5f3015a950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 -- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f3006da40 msgr2=0x7f5f3013f1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:37.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f3006da40 0x7f5f3013f1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:37.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.961+0000 7f5f35400640 1 -- 172.21.15.86:0/383353443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f301589e0 con 0x7f5f300aef20 2024-03-06T07:27:37.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.962+0000 7f5f35400640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f300aef20 0x7f5f301580e0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f5f2000e9e0 tx=0x7f5f2000eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.962+0000 7f5f2a000640 1 -- 172.21.15.86:0/383353443 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f20017070 con 0x7f5f300aef20 2024-03-06T07:27:37.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.962+0000 7f5f2a000640 1 -- 172.21.15.86:0/383353443 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f2000c9d0 con 0x7f5f300aef20 2024-03-06T07:27:37.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.962+0000 7f5f2a000640 1 -- 172.21.15.86:0/383353443 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f2000cdc0 con 0x7f5f300aef20 2024-03-06T07:27:37.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.962+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5f30158cc0 con 0x7f5f300aef20 2024-03-06T07:27:37.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.962+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5f30159560 con 0x7f5f300aef20 2024-03-06T07:27:37.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.963+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5f30158e90 con 0x7f5f300aef20 2024-03-06T07:27:37.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.964+0000 7f5f2a000640 1 -- 172.21.15.86:0/383353443 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5f2001a070 con 0x7f5f300aef20 2024-03-06T07:27:37.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.965+0000 7f5f2a000640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f1406b4d0 0x7f5f1406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:37.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.965+0000 7f5f2a000640 1 -- 172.21.15.86:0/383353443 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5f2008a640 con 0x7f5f300aef20 2024-03-06T07:27:37.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.965+0000 7f5f35e00640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f1406b4d0 0x7f5f1406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:37.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.966+0000 7f5f35e00640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f1406b4d0 0x7f5f1406d990 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f5f301f73a0 tx=0x7f5f24009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:37.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:37.970+0000 7f5f2a000640 1 -- 172.21.15.86:0/383353443 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5f200594c0 con 0x7f5f300aef20 2024-03-06T07:27:38.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.165+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f5f3004f330 con 0x7f5f300aef20 2024-03-06T07:27:38.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.166+0000 7f5f2a000640 1 -- 172.21.15.86:0/383353443 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f5f20058dc0 con 0x7f5f300aef20 2024-03-06T07:27:38.167 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:38.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f1406b4d0 msgr2=0x7f5f1406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f1406b4d0 0x7f5f1406d990 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f5f301f73a0 tx=0x7f5f24009040 comp rx=0 tx=0).stop 2024-03-06T07:27:38.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f300aef20 msgr2=0x7f5f301580e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f300aef20 0x7f5f301580e0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f5f2000e9e0 tx=0x7f5f2000eea0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 shutdown_connections 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f1406b4d0 0x7f5f1406d990 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f3015be10 0x7f5f3015a950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f300aef20 0x7f5f301580e0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 --2- 172.21.15.86:0/383353443 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f3006da40 0x7f5f3013f1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.168+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 >> 172.21.15.86:0/383353443 conn(0x7f5f3006afc0 msgr2=0x7f5f300b1da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.169+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 shutdown_connections 2024-03-06T07:27:38.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.169+0000 7f5f37200640 1 -- 172.21.15.86:0/383353443 wait complete. 2024-03-06T07:27:38.189 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:38.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.396+0000 7f8df3e00640 1 Processor -- start 2024-03-06T07:27:38.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.396+0000 7f8df3e00640 1 -- start start 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.397+0000 7f8df3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8df415a940 0x7f8df40b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.397+0000 7f8df3e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cf370 0x7f8df406d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.397+0000 7f8df3e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8df40cc2a0 0x7f8df40cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.397+0000 7f8df3e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8df40b28e0 con 0x7f8df415a940 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df3e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8df40b2b30 con 0x7f8df40cc2a0 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df3e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8df40b2e10 con 0x7f8df40cf370 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8df415a940 0x7f8df40b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8df415a940 0x7f8df40b45e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36458/0 (socket says 172.21.15.86:36458) 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 -- 172.21.15.86:0/1073603688 learned_addr learned my addr 172.21.15.86:0/1073603688 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df3400640 1 --2- 172.21.15.86:0/1073603688 >> v2:172.21.15.194:3300/0 conn(0x7f8df40cc2a0 0x7f8df40cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8debe00640 1 --2- 172.21.15.86:0/1073603688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cf370 0x7f8df406d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 -- 172.21.15.86:0/1073603688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cf370 msgr2=0x7f8df406d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:38.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 --2- 172.21.15.86:0/1073603688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cf370 0x7f8df406d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 -- 172.21.15.86:0/1073603688 >> v2:172.21.15.194:3300/0 conn(0x7f8df40cc2a0 msgr2=0x7f8df40cc750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:38.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 --2- 172.21.15.86:0/1073603688 >> v2:172.21.15.194:3300/0 conn(0x7f8df40cc2a0 0x7f8df40cc750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.398+0000 7f8df2a00640 1 -- 172.21.15.86:0/1073603688 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8df406ca50 con 0x7f8df415a940 2024-03-06T07:27:38.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.400+0000 7f8df2a00640 1 --2- 172.21.15.86:0/1073603688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40b45e0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f8df415f670 tx=0x7f8de402b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ebec0ab0fbb76e4c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:38.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.400+0000 7f8df2000640 1 -- 172.21.15.86:0/1073603688 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8de4035020 con 0x7f8df415a940 2024-03-06T07:27:38.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.400+0000 7f8df2000640 1 -- 172.21.15.86:0/1073603688 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8de402d040 con 0x7f8df415a940 2024-03-06T07:27:38.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.400+0000 7f8df2000640 1 -- 172.21.15.86:0/1073603688 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8de4031a80 con 0x7f8df415a940 2024-03-06T07:27:38.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 -- 172.21.15.86:0/1073603688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 msgr2=0x7f8df40b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 --2- 172.21.15.86:0/1073603688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40b45e0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f8df415f670 tx=0x7f8de402b900 comp rx=0 tx=0).stop 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 -- 172.21.15.86:0/1073603688 shutdown_connections 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 --2- 172.21.15.86:0/1073603688 >> v2:172.21.15.194:3300/0 conn(0x7f8df40cc2a0 0x7f8df40cc750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 --2- 172.21.15.86:0/1073603688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cf370 0x7f8df406d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 --2- 172.21.15.86:0/1073603688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40b45e0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 -- 172.21.15.86:0/1073603688 >> 172.21.15.86:0/1073603688 conn(0x7f8df406afc0 msgr2=0x7f8df406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 -- 172.21.15.86:0/1073603688 shutdown_connections 2024-03-06T07:27:38.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.401+0000 7f8df3e00640 1 -- 172.21.15.86:0/1073603688 wait complete. 2024-03-06T07:27:38.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.403+0000 7f8df3e00640 1 Processor -- start 2024-03-06T07:27:38.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.403+0000 7f8df3e00640 1 -- start start 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cc2a0 0x7f8df4150650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8df40cf370 0x7f8df40aee90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40af3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8df40b4b20 con 0x7f8df415a940 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8df40b30f0 con 0x7f8df40cf370 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8df406d790 con 0x7f8df40cc2a0 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40af3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40af3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36464/0 (socket says 172.21.15.86:36464) 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3400640 1 -- 172.21.15.86:0/578440586 learned_addr learned my addr 172.21.15.86:0/578440586 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df2a00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cc2a0 0x7f8df4150650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8debe00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8df40cf370 0x7f8df40aee90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.404+0000 7f8df3400640 1 -- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cc2a0 msgr2=0x7f8df4150650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8df3400640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cc2a0 0x7f8df4150650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8df3400640 1 -- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8df40cf370 msgr2=0x7f8df40aee90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8df3400640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8df40cf370 0x7f8df40aee90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8df3400640 1 -- 172.21.15.86:0/578440586 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8df4150b90 con 0x7f8df415a940 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8df3400640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40af3d0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f8dec007e70 tx=0x7f8dec00e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8deaa00640 1 -- 172.21.15.86:0/578440586 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dec014620 con 0x7f8df415a940 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8deaa00640 1 -- 172.21.15.86:0/578440586 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8dec019070 con 0x7f8df415a940 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8deaa00640 1 -- 172.21.15.86:0/578440586 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dec014a20 con 0x7f8df415a940 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8df415aec0 con 0x7f8df415a940 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.405+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8df415b480 con 0x7f8df415a940 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.406+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8df404f330 con 0x7f8df415a940 2024-03-06T07:27:38.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.408+0000 7f8deaa00640 1 -- 172.21.15.86:0/578440586 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8dec014bc0 con 0x7f8df415a940 2024-03-06T07:27:38.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.408+0000 7f8deaa00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8dd006b4d0 0x7f8dd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.409+0000 7f8deaa00640 1 -- 172.21.15.86:0/578440586 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8dec08ab10 con 0x7f8df415a940 2024-03-06T07:27:38.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.409+0000 7f8df2a00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8dd006b4d0 0x7f8dd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.409+0000 7f8df2a00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8dd006b4d0 0x7f8dd006d990 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f8de4036000 tx=0x7f8de403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:38.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.413+0000 7f8deaa00640 1 -- 172.21.15.86:0/578440586 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8dec059980 con 0x7f8df415a940 2024-03-06T07:27:38.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.608+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f8df406e180 con 0x7f8df415a940 2024-03-06T07:27:38.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.610+0000 7f8deaa00640 1 -- 172.21.15.86:0/578440586 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f8dec059280 con 0x7f8df415a940 2024-03-06T07:27:38.612 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:38.613 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8dd006b4d0 msgr2=0x7f8dd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8dd006b4d0 0x7f8dd006d990 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f8de4036000 tx=0x7f8de403c040 comp rx=0 tx=0).stop 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 msgr2=0x7f8df40af3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40af3d0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f8dec007e70 tx=0x7f8dec00e4b0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 shutdown_connections 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8dd006b4d0 0x7f8dd006d990 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8df415a940 0x7f8df40af3d0 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8df40cf370 0x7f8df40aee90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 --2- 172.21.15.86:0/578440586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8df40cc2a0 0x7f8df4150650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 >> 172.21.15.86:0/578440586 conn(0x7f8df406afc0 msgr2=0x7f8df40cf890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:38.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 shutdown_connections 2024-03-06T07:27:38.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.613+0000 7f8df3e00640 1 -- 172.21.15.86:0/578440586 wait complete. 2024-03-06T07:27:38.633 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:38.633 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:38.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.826+0000 7fdd6f000640 1 Processor -- start 2024-03-06T07:27:38.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.826+0000 7fdd6f000640 1 -- start start 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.827+0000 7fdd6f000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd68154dc0 0x7fdd68161b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.827+0000 7fdd6f000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdd680aeff0 0x7fdd680cebc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.827+0000 7fdd6f000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdd680cfb30 con 0x7fdd681551a0 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.827+0000 7fdd6f000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdd6815e540 con 0x7fdd680aeff0 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.827+0000 7fdd6f000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd680cfe10 con 0x7fdd68154dc0 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6d200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fdd681551a0 0x7fdd680ae8e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57546/0 (socket says 172.21.15.86:57546) 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6d200640 1 -- 172.21.15.86:0/1104565254 learned_addr learned my addr 172.21.15.86:0/1104565254 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:38.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6e600640 1 --2- 172.21.15.86:0/1104565254 >> v2:172.21.15.86:3300/0 conn(0x7fdd680aeff0 0x7fdd680cebc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6dc00640 1 --2- 172.21.15.86:0/1104565254 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd68154dc0 0x7fdd68161b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd67e00640 1 -- 172.21.15.86:0/1104565254 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1222934931 0 0) 0x7fdd680cfb30 con 0x7fdd681551a0 2024-03-06T07:27:38.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd67e00640 1 -- 172.21.15.86:0/1104565254 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fdd50003520 con 0x7fdd681551a0 2024-03-06T07:27:38.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6e600640 1 -- 172.21.15.86:0/1104565254 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd68154dc0 msgr2=0x7fdd68161b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:38.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6e600640 1 --2- 172.21.15.86:0/1104565254 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd68154dc0 0x7fdd68161b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6e600640 1 -- 172.21.15.86:0/1104565254 >> v1:172.21.15.86:6789/0 conn(0x7fdd681551a0 legacy=0x7fdd680ae8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.828+0000 7fdd6e600640 1 -- 172.21.15.86:0/1104565254 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd680d00f0 con 0x7fdd680aeff0 2024-03-06T07:27:38.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.830+0000 7fdd6e600640 1 --2- 172.21.15.86:0/1104565254 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd680aeff0 0x7fdd680cebc0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fdd60009d90 tx=0x7fdd6002bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9268fc388b6c338e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:38.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.830+0000 7fdd67e00640 1 -- 172.21.15.86:0/1104565254 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd60039070 con 0x7fdd680aeff0 2024-03-06T07:27:38.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.830+0000 7fdd67e00640 1 -- 172.21.15.86:0/1104565254 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd6002cc50 con 0x7fdd680aeff0 2024-03-06T07:27:38.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.830+0000 7fdd67e00640 1 -- 172.21.15.86:0/1104565254 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd60034440 con 0x7fdd680aeff0 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 -- 172.21.15.86:0/1104565254 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd680aeff0 msgr2=0x7fdd680cebc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 --2- 172.21.15.86:0/1104565254 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd680aeff0 0x7fdd680cebc0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fdd60009d90 tx=0x7fdd6002bea0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 -- 172.21.15.86:0/1104565254 shutdown_connections 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 --2- 172.21.15.86:0/1104565254 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd680aeff0 0x7fdd680cebc0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 --2- 172.21.15.86:0/1104565254 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd68154dc0 0x7fdd68161b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 -- 172.21.15.86:0/1104565254 >> 172.21.15.86:0/1104565254 conn(0x7fdd6806afc0 msgr2=0x7fdd6806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 -- 172.21.15.86:0/1104565254 shutdown_connections 2024-03-06T07:27:38.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.831+0000 7fdd6f000640 1 -- 172.21.15.86:0/1104565254 wait complete. 2024-03-06T07:27:38.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.833+0000 7fdd6f000640 1 Processor -- start 2024-03-06T07:27:38.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.833+0000 7fdd6f000640 1 -- start start 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6f000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd680aeff0 0x7fdd681f7520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6f000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd68154dc0 0x7fdd680afe70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6f000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd681551a0 0x7fdd680cee10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6f000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd68162070 con 0x7fdd68154dc0 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6f000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd680cf820 con 0x7fdd681551a0 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6f000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd681f83c0 con 0x7fdd680aeff0 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd68154dc0 0x7fdd680afe70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd68154dc0 0x7fdd680afe70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36490/0 (socket says 172.21.15.86:36490) 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6d200640 1 -- 172.21.15.86:0/263921604 learned_addr learned my addr 172.21.15.86:0/263921604 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.834+0000 7fdd6e600640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd681551a0 0x7fdd680cee10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6dc00640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd680aeff0 0x7fdd681f7520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6d200640 1 -- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd680aeff0 msgr2=0x7fdd681f7520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6d200640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd680aeff0 0x7fdd681f7520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6d200640 1 -- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd681551a0 msgr2=0x7fdd680cee10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6d200640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd681551a0 0x7fdd680cee10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6d200640 1 -- 172.21.15.86:0/263921604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd681f7b20 con 0x7fdd68154dc0 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6e600640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd681551a0 0x7fdd680cee10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6dc00640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd680aeff0 0x7fdd681f7520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6d200640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd68154dc0 0x7fdd680afe70 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fdd58001eb0 tx=0x7fdd5800c820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd65600640 1 -- 172.21.15.86:0/263921604 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd58017070 con 0x7fdd68154dc0 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd65600640 1 -- 172.21.15.86:0/263921604 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd58005d00 con 0x7fdd68154dc0 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd65600640 1 -- 172.21.15.86:0/263921604 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd58007310 con 0x7fdd68154dc0 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdd681f7e00 con 0x7fdd68154dc0 2024-03-06T07:27:38.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.835+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdd680b1510 con 0x7fdd68154dc0 2024-03-06T07:27:38.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.836+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdd6804f330 con 0x7fdd68154dc0 2024-03-06T07:27:38.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.837+0000 7fdd65600640 1 -- 172.21.15.86:0/263921604 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdd5801a070 con 0x7fdd68154dc0 2024-03-06T07:27:38.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.838+0000 7fdd65600640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd4c06b4d0 0x7fdd4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:38.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.839+0000 7fdd65600640 1 -- 172.21.15.86:0/263921604 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fdd580895e0 con 0x7fdd68154dc0 2024-03-06T07:27:38.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.839+0000 7fdd6dc00640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd4c06b4d0 0x7fdd4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:38.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.839+0000 7fdd6dc00640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd4c06b4d0 0x7fdd4c06d990 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fdd5c009ac0 tx=0x7fdd5c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:38.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:38.843+0000 7fdd65600640 1 -- 172.21.15.86:0/263921604 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdd58058450 con 0x7fdd68154dc0 2024-03-06T07:27:39.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.043+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fdd680add40 con 0x7fdd68154dc0 2024-03-06T07:27:39.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.044+0000 7fdd65600640 1 -- 172.21.15.86:0/263921604 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7fdd58057d50 con 0x7fdd68154dc0 2024-03-06T07:27:39.045 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:39.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd4c06b4d0 msgr2=0x7fdd4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd4c06b4d0 0x7fdd4c06d990 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fdd5c009ac0 tx=0x7fdd5c008040 comp rx=0 tx=0).stop 2024-03-06T07:27:39.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd68154dc0 msgr2=0x7fdd680afe70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd68154dc0 0x7fdd680afe70 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fdd58001eb0 tx=0x7fdd5800c820 comp rx=0 tx=0).stop 2024-03-06T07:27:39.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 shutdown_connections 2024-03-06T07:27:39.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd4c06b4d0 0x7fdd4c06d990 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd681551a0 0x7fdd680cee10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd68154dc0 0x7fdd680afe70 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 --2- 172.21.15.86:0/263921604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd680aeff0 0x7fdd681f7520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 >> 172.21.15.86:0/263921604 conn(0x7fdd6806afc0 msgr2=0x7fdd680b2660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:39.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.046+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 shutdown_connections 2024-03-06T07:27:39.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.047+0000 7fdd6f000640 1 -- 172.21.15.86:0/263921604 wait complete. 2024-03-06T07:27:39.066 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:39.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.273+0000 7f641b400640 1 Processor -- start 2024-03-06T07:27:39.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.274+0000 7f641b400640 1 -- start start 2024-03-06T07:27:39.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f641b400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f6414070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f641b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f641415fff0 0x7f6414160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f641b400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f64140b46c0 con 0x7f64140ce4a0 2024-03-06T07:27:39.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f641b400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f64140b49a0 con 0x7f641415fff0 2024-03-06T07:27:39.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f641b400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6414072c90 con 0x7f641415fc10 2024-03-06T07:27:39.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f6419400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f64140ce4a0 0x7f64140ce930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57562/0 (socket says 172.21.15.86:57562) 2024-03-06T07:27:39.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f6419400640 1 -- 172.21.15.86:0/2232014600 learned_addr learned my addr 172.21.15.86:0/2232014600 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:39.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.275+0000 7f640be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f641415fff0 0x7f6414160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.276+0000 7f6418a00640 1 --2- 172.21.15.86:0/2232014600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f6414070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.276+0000 7f640be00640 1 -- 172.21.15.86:0/2232014600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 msgr2=0x7f6414070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:39.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.276+0000 7f640be00640 1 --2- 172.21.15.86:0/2232014600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f6414070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.276+0000 7f640be00640 1 -- 172.21.15.86:0/2232014600 >> v1:172.21.15.86:6789/0 conn(0x7f64140ce4a0 legacy=0x7f64140ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.276+0000 7f640be00640 1 -- 172.21.15.86:0/2232014600 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6414160990 con 0x7f641415fff0 2024-03-06T07:27:39.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.276+0000 7f6413e00640 1 -- 172.21.15.86:0/2232014600 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3723550699 0 0) 0x7f64140b46c0 con 0x7f64140ce4a0 2024-03-06T07:27:39.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.277+0000 7f640be00640 1 --2- 172.21.15.86:0/2232014600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f6414160450 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f63fc0097a0 tx=0x7f63fc02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aaa698ce145c51c8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:39.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.277+0000 7f6413e00640 1 -- 172.21.15.86:0/2232014600 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63fc035020 con 0x7f641415fff0 2024-03-06T07:27:39.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.277+0000 7f6413e00640 1 -- 172.21.15.86:0/2232014600 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f63fc031830 con 0x7f641415fff0 2024-03-06T07:27:39.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.277+0000 7f6413e00640 1 -- 172.21.15.86:0/2232014600 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63fc031c00 con 0x7f641415fff0 2024-03-06T07:27:39.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 -- 172.21.15.86:0/2232014600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 msgr2=0x7f6414160450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 --2- 172.21.15.86:0/2232014600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f6414160450 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f63fc0097a0 tx=0x7f63fc02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 -- 172.21.15.86:0/2232014600 shutdown_connections 2024-03-06T07:27:39.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 --2- 172.21.15.86:0/2232014600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f6414160450 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 --2- 172.21.15.86:0/2232014600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f6414070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 -- 172.21.15.86:0/2232014600 >> 172.21.15.86:0/2232014600 conn(0x7f64140b1940 msgr2=0x7f64140b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:39.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 -- 172.21.15.86:0/2232014600 shutdown_connections 2024-03-06T07:27:39.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.278+0000 7f641b400640 1 -- 172.21.15.86:0/2232014600 wait complete. 2024-03-06T07:27:39.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.280+0000 7f641b400640 1 Processor -- start 2024-03-06T07:27:39.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.280+0000 7f641b400640 1 -- start start 2024-03-06T07:27:39.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f641b400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64140ce4a0 0x7f641414c200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f641b400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f64140b2f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f641b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f64140b34a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f641b400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6414072a30 con 0x7f641415fff0 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f641b400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f64140728b0 con 0x7f64140ce4a0 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f641b400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f641415c2e0 con 0x7f641415fc10 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f64140b34a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f64140b34a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36514/0 (socket says 172.21.15.86:36514) 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 -- 172.21.15.86:0/234128327 learned_addr learned my addr 172.21.15.86:0/234128327 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f640be00640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f64140b2f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 -- 172.21.15.86:0/234128327 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 msgr2=0x7f64140b2f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f64140b2f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 -- 172.21.15.86:0/234128327 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64140ce4a0 msgr2=0x7f641414c200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64140ce4a0 0x7f641414c200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.281+0000 7f6419400640 1 -- 172.21.15.86:0/234128327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64141f7b90 con 0x7f641415fff0 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.282+0000 7f6419400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f64140b34a0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f640c00c7d0 tx=0x7f640c00cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.282+0000 7f6411600640 1 -- 172.21.15.86:0/234128327 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f640c018070 con 0x7f641415fff0 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.282+0000 7f6411600640 1 -- 172.21.15.86:0/234128327 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f640c00a640 con 0x7f641415fff0 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.282+0000 7f6411600640 1 -- 172.21.15.86:0/234128327 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f640c00aa70 con 0x7f641415fff0 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.282+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f64141f7e30 con 0x7f641415fff0 2024-03-06T07:27:39.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.282+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64141f83f0 con 0x7f641415fff0 2024-03-06T07:27:39.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.283+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f64140b22c0 con 0x7f641415fff0 2024-03-06T07:27:39.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.284+0000 7f6411600640 1 -- 172.21.15.86:0/234128327 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f640c01b070 con 0x7f641415fff0 2024-03-06T07:27:39.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.285+0000 7f6411600640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f006b500 0x7f63f006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.285+0000 7f6411600640 1 -- 172.21.15.86:0/234128327 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f640c0897a0 con 0x7f641415fff0 2024-03-06T07:27:39.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.285+0000 7f6418a00640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f006b500 0x7f63f006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.286+0000 7f6418a00640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f006b500 0x7f63f006d9c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f6404006fd0 tx=0x7f6404008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:39.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.289+0000 7f6411600640 1 -- 172.21.15.86:0/234128327 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f64140b22c0 con 0x7f641415fff0 2024-03-06T07:27:39.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.485+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f64140cf870 con 0x7f641415fff0 2024-03-06T07:27:39.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.487+0000 7f6411600640 1 -- 172.21.15.86:0/234128327 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7f64140cf870 con 0x7f641415fff0 2024-03-06T07:27:39.488 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f006b500 msgr2=0x7f63f006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f006b500 0x7f63f006d9c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f6404006fd0 tx=0x7f6404008040 comp rx=0 tx=0).stop 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 msgr2=0x7f64140b34a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f64140b34a0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f640c00c7d0 tx=0x7f640c00cc90 comp rx=0 tx=0).stop 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 shutdown_connections 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f006b500 0x7f63f006d9c0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f641415fff0 0x7f64140b34a0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641415fc10 0x7f64140b2f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 --2- 172.21.15.86:0/234128327 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64140ce4a0 0x7f641414c200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 >> 172.21.15.86:0/234128327 conn(0x7f64140b1940 msgr2=0x7f64140b3e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:39.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 shutdown_connections 2024-03-06T07:27:39.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.489+0000 7f641b400640 1 -- 172.21.15.86:0/234128327 wait complete. 2024-03-06T07:27:39.509 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:39.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.725+0000 7fcd11400640 1 Processor -- start 2024-03-06T07:27:39.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.726+0000 7fcd11400640 1 -- start start 2024-03-06T07:27:39.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.727+0000 7fcd11400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 0x7fcd0c070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.727+0000 7fcd11400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcd0c0b46c0 con 0x7fcd0c15fff0 2024-03-06T07:27:39.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.727+0000 7fcd11400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcd0c0b49a0 con 0x7fcd0c0b3f70 2024-03-06T07:27:39.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.727+0000 7fcd11400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcd0c072c90 con 0x7fcd0c15fc10 2024-03-06T07:27:39.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.727+0000 7fcd09e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fcd0c15fff0 0x7fcd0c160450 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57568/0 (socket says 172.21.15.86:57568) 2024-03-06T07:27:39.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.727+0000 7fcd09e00640 1 -- 172.21.15.86:0/1954189585 learned_addr learned my addr 172.21.15.86:0/1954189585 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:39.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd0a800640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 0x7fcd0c070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3694805025 0 0) 0x7fcd0c0b46c0 con 0x7fcd0c15fff0 2024-03-06T07:27:39.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fccf0003520 con 0x7fcd0c15fff0 2024-03-06T07:27:39.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1977206814 0 0) 0x7fccf0003520 con 0x7fcd0c15fff0 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 msgr2=0x7fcd0c070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 0x7fcd0c070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 >> v1:172.21.15.194:6789/0 conn(0x7fcd0c0b3f70 legacy=0x7fcd0c0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcd0c09dd80 con 0x7fcd0c15fff0 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fccf8001670 con 0x7fcd0c15fff0 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 >> v1:172.21.15.86:6789/0 conn(0x7fcd0c15fff0 legacy=0x7fcd0c160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 0x7fccf00039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fccf0003f60 0x7fccf0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fccf0004d40 0x7fccf00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd0c072c90 con 0x7fccf0004d40 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fccf000baa0 con 0x7fccf0003f60 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.728+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fccf000bca0 con 0x7fcd0c15fc10 2024-03-06T07:27:39.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd0b200640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fccf0004d40 0x7fccf00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd0a800640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 0x7fccf00039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd09e00640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fccf0003f60 0x7fccf0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd0b200640 1 -- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 msgr2=0x7fccf00039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd0b200640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c15fc10 0x7fccf00039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd0b200640 1 -- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fccf0003f60 msgr2=0x7fccf0004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd0b200640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fccf0003f60 0x7fccf0004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.729+0000 7fcd0b200640 1 -- 172.21.15.86:0/1954189585 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fccf0003520 con 0x7fccf0004d40 2024-03-06T07:27:39.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.730+0000 7fcd0b200640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fccf0004d40 0x7fccf00051c0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fcd04001e50 tx=0x7fccf0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c033342ec2843694 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:39.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.730+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd04001a70 con 0x7fccf0004d40 2024-03-06T07:27:39.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.730+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcd04003b70 con 0x7fccf0004d40 2024-03-06T07:27:39.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.730+0000 7fcd09400640 1 -- 172.21.15.86:0/1954189585 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd040050d0 con 0x7fccf0004d40 2024-03-06T07:27:39.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.731+0000 7fcd11400640 1 -- 172.21.15.86:0/1954189585 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fccf0004d40 msgr2=0x7fccf00051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.731+0000 7fcd11400640 1 --2- 172.21.15.86:0/1954189585 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fccf0004d40 0x7fccf00051c0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fcd04001e50 tx=0x7fccf0005830 comp rx=0 tx=0).stop 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.731+0000 7fcd0b200640 1 -- 172.21.15.86:0/1954189585 reap_dead start 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.731+0000 7fcd11400640 1 -- 172.21.15.86:0/1954189585 shutdown_connections 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.731+0000 7fcd11400640 1 -- 172.21.15.86:0/1954189585 >> 172.21.15.86:0/1954189585 conn(0x7fcd0c0b1940 msgr2=0x7fcd0c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.731+0000 7fcd11400640 1 -- 172.21.15.86:0/1954189585 shutdown_connections 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.731+0000 7fcd11400640 1 -- 172.21.15.86:0/1954189585 wait complete. 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.733+0000 7fcd11400640 1 Processor -- start 2024-03-06T07:27:39.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.733+0000 7fcd11400640 1 -- start start 2024-03-06T07:27:39.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd11400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c09cef0 0x7fcd0c142cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd11400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcd0c14eb10 0x7fcd0c14ef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd11400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcd0c14f4b0 0x7fcd0c09ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd11400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd0c15c460 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd11400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd0c15c2e0 con 0x7fcd0c14eb10 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd11400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcd0c070ff0 con 0x7fcd0c09cef0 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd0b200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcd0c14f4b0 0x7fcd0c09ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd0b200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcd0c14f4b0 0x7fcd0c09ba40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36546/0 (socket says 172.21.15.86:36546) 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd0b200640 1 -- 172.21.15.86:0/1487013737 learned_addr learned my addr 172.21.15.86:0/1487013737 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.734+0000 7fcd0a800640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c09cef0 0x7fcd0c142cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd09e00640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcd0c14eb10 0x7fcd0c14ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd0b200640 1 -- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c09cef0 msgr2=0x7fcd0c142cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd0b200640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c09cef0 0x7fcd0c142cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd0b200640 1 -- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcd0c14eb10 msgr2=0x7fcd0c14ef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd0b200640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcd0c14eb10 0x7fcd0c14ef70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd0b200640 1 -- 172.21.15.86:0/1487013737 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcd0c14f930 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd0b200640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcd0c14f4b0 0x7fcd0c09ba40 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fcd0402c060 tx=0x7fcd040014a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:39.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd02a00640 1 -- 172.21.15.86:0/1487013737 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd04007db0 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd02a00640 1 -- 172.21.15.86:0/1487013737 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcd04003970 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcd0c14dd90 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd02a00640 1 -- 172.21.15.86:0/1487013737 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd0402f6a0 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.735+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcd0c14e030 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.736+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcd0c04f330 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.737+0000 7fcd02a00640 1 -- 172.21.15.86:0/1487013737 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcd04041030 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.738+0000 7fcd02a00640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fccec06b4d0 0x7fccec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:39.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.738+0000 7fcd02a00640 1 -- 172.21.15.86:0/1487013737 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcd040ab200 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.739+0000 7fcd0a800640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fccec06b4d0 0x7fccec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:39.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.739+0000 7fcd0a800640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fccec06b4d0 0x7fccec06d990 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fccfc009a10 tx=0x7fccfc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:39.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.743+0000 7fcd02a00640 1 -- 172.21.15.86:0/1487013737 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcd0c14f930 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.945+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fcd0c06ef30 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.946+0000 7fcd02a00640 1 -- 172.21.15.86:0/1487013737 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7fcd04033020 con 0x7fcd0c14f4b0 2024-03-06T07:27:39.948 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:39.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fccec06b4d0 msgr2=0x7fccec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fccec06b4d0 0x7fccec06d990 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fccfc009a10 tx=0x7fccfc008040 comp rx=0 tx=0).stop 2024-03-06T07:27:39.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcd0c14f4b0 msgr2=0x7fcd0c09ba40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:39.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcd0c14f4b0 0x7fcd0c09ba40 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fcd0402c060 tx=0x7fcd040014a0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 shutdown_connections 2024-03-06T07:27:39.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fccec06b4d0 0x7fccec06d990 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcd0c14f4b0 0x7fcd0c09ba40 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcd0c14eb10 0x7fcd0c14ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 --2- 172.21.15.86:0/1487013737 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcd0c09cef0 0x7fcd0c142cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:39.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 >> 172.21.15.86:0/1487013737 conn(0x7fcd0c0b1940 msgr2=0x7fcd0c0ae440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:39.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 shutdown_connections 2024-03-06T07:27:39.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:39.949+0000 7fcd11400640 1 -- 172.21.15.86:0/1487013737 wait complete. 2024-03-06T07:27:39.971 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:40.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.181+0000 7ff709200640 1 Processor -- start 2024-03-06T07:27:40.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.181+0000 7ff709200640 1 -- start start 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.182+0000 7ff709200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff7040cf370 0x7ff70406d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.182+0000 7ff709200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cc2a0 0x7ff7040cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.182+0000 7ff709200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff7040b2b30 con 0x7ff70415a940 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.182+0000 7ff709200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff7040b28e0 con 0x7ff7040cf370 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.182+0000 7ff709200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff7040b2e10 con 0x7ff7040cc2a0 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff702a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff7040cf370 0x7ff70406d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff702a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff7040cf370 0x7ff70406d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36560/0 (socket says 172.21.15.86:36560) 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff702a00640 1 -- 172.21.15.86:0/1734615669 learned_addr learned my addr 172.21.15.86:0/1734615669 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff703e00640 1 --2- 172.21.15.86:0/1734615669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cc2a0 0x7ff7040cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff702a00640 1 -- 172.21.15.86:0/1734615669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cc2a0 msgr2=0x7ff7040cc750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff702a00640 1 --2- 172.21.15.86:0/1734615669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cc2a0 0x7ff7040cc750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff702a00640 1 -- 172.21.15.86:0/1734615669 >> v1:172.21.15.194:6789/0 conn(0x7ff70415a940 legacy=0x7ff7040b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:40.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff702a00640 1 -- 172.21.15.86:0/1734615669 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff70406ca50 con 0x7ff7040cf370 2024-03-06T07:27:40.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.183+0000 7ff703e00640 1 --2- 172.21.15.86:0/1734615669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cc2a0 0x7ff7040cc750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:40.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.184+0000 7ff702a00640 1 --2- 172.21.15.86:0/1734615669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7040cf370 0x7ff70406d250 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7ff6f000bb10 tx=0x7ff6f002e0c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b392548bc9a96af server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:40.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff702000640 1 -- 172.21.15.86:0/1734615669 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6f003b070 con 0x7ff7040cf370 2024-03-06T07:27:40.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff702000640 1 -- 172.21.15.86:0/1734615669 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff6f0036440 con 0x7ff7040cf370 2024-03-06T07:27:40.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff702000640 1 -- 172.21.15.86:0/1734615669 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6f0036810 con 0x7ff7040cf370 2024-03-06T07:27:40.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff709200640 1 -- 172.21.15.86:0/1734615669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7040cf370 msgr2=0x7ff70406d250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff709200640 1 --2- 172.21.15.86:0/1734615669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7040cf370 0x7ff70406d250 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7ff6f000bb10 tx=0x7ff6f002e0c0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff709200640 1 -- 172.21.15.86:0/1734615669 shutdown_connections 2024-03-06T07:27:40.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff709200640 1 --2- 172.21.15.86:0/1734615669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cc2a0 0x7ff7040cc750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff709200640 1 --2- 172.21.15.86:0/1734615669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff7040cf370 0x7ff70406d250 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.185+0000 7ff709200640 1 -- 172.21.15.86:0/1734615669 >> 172.21.15.86:0/1734615669 conn(0x7ff70406afc0 msgr2=0x7ff70406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:40.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.186+0000 7ff709200640 1 -- 172.21.15.86:0/1734615669 shutdown_connections 2024-03-06T07:27:40.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.186+0000 7ff709200640 1 -- 172.21.15.86:0/1734615669 wait complete. 2024-03-06T07:27:40.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.187+0000 7ff709200640 1 Processor -- start 2024-03-06T07:27:40.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.187+0000 7ff709200640 1 -- start start 2024-03-06T07:27:40.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.188+0000 7ff709200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7040cc2a0 0x7ff704153a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.188+0000 7ff709200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cf370 0x7ff704071b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.188+0000 7ff709200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff70415a940 0x7ff704072050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.188+0000 7ff709200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff70406d790 con 0x7ff70415a940 2024-03-06T07:27:40.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.188+0000 7ff709200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff7040b30f0 con 0x7ff7040cc2a0 2024-03-06T07:27:40.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.188+0000 7ff709200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff7041f4080 con 0x7ff7040cf370 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.188+0000 7ff703e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff70415a940 0x7ff704072050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff70415a940 0x7ff704072050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36566/0 (socket says 172.21.15.86:36566) 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 -- 172.21.15.86:0/1891554281 learned_addr learned my addr 172.21.15.86:0/1891554281 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703400640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7040cc2a0 0x7ff704153a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff702a00640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cf370 0x7ff704071b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 -- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cf370 msgr2=0x7ff704071b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cf370 0x7ff704071b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 -- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7040cc2a0 msgr2=0x7ff704153a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7040cc2a0 0x7ff704153a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 -- 172.21.15.86:0/1891554281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff7041f4360 con 0x7ff70415a940 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.189+0000 7ff703e00640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff70415a940 0x7ff704072050 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc00cd50 tx=0x7ff6fc00a650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.190+0000 7ff6fb400640 1 -- 172.21.15.86:0/1891554281 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6fc00f680 con 0x7ff70415a940 2024-03-06T07:27:40.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.190+0000 7ff6fb400640 1 -- 172.21.15.86:0/1891554281 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff6fc00f840 con 0x7ff70415a940 2024-03-06T07:27:40.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.190+0000 7ff6fb400640 1 -- 172.21.15.86:0/1891554281 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6fc00fc30 con 0x7ff70415a940 2024-03-06T07:27:40.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.190+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff7041f4640 con 0x7ff70415a940 2024-03-06T07:27:40.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.190+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff7041f3d30 con 0x7ff70415a940 2024-03-06T07:27:40.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.191+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff70404f330 con 0x7ff70415a940 2024-03-06T07:27:40.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.192+0000 7ff6fb400640 1 -- 172.21.15.86:0/1891554281 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff6fc00e3e0 con 0x7ff70415a940 2024-03-06T07:27:40.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.193+0000 7ff6fb400640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6e406b4d0 0x7ff6e406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.193+0000 7ff6fb400640 1 -- 172.21.15.86:0/1891554281 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff6fc08b0a0 con 0x7ff70415a940 2024-03-06T07:27:40.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.193+0000 7ff703400640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6e406b4d0 0x7ff6e406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.194+0000 7ff703400640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6e406b4d0 0x7ff6e406d990 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7ff6f4004ea0 tx=0x7ff6f4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:40.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.197+0000 7ff6fb400640 1 -- 172.21.15.86:0/1891554281 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff6fc014030 con 0x7ff70415a940 2024-03-06T07:27:40.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.395+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7ff70406b4a0 con 0x7ff70415a940 2024-03-06T07:27:40.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.398+0000 7ff6fb400640 1 -- 172.21.15.86:0/1891554281 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7ff6fc059e80 con 0x7ff70415a940 2024-03-06T07:27:40.399 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:40.400 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6e406b4d0 msgr2=0x7ff6e406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6e406b4d0 0x7ff6e406d990 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7ff6f4004ea0 tx=0x7ff6f4009040 comp rx=0 tx=0).stop 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff70415a940 msgr2=0x7ff704072050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff70415a940 0x7ff704072050 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc00cd50 tx=0x7ff6fc00a650 comp rx=0 tx=0).stop 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 shutdown_connections 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6e406b4d0 0x7ff6e406d990 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff70415a940 0x7ff704072050 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff7040cf370 0x7ff704071b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 --2- 172.21.15.86:0/1891554281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff7040cc2a0 0x7ff704153a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.400+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 >> 172.21.15.86:0/1891554281 conn(0x7ff70406afc0 msgr2=0x7ff7040cf890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:40.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.401+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 shutdown_connections 2024-03-06T07:27:40.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.401+0000 7ff709200640 1 -- 172.21.15.86:0/1891554281 wait complete. 2024-03-06T07:27:40.420 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:40.420 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:40.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.625+0000 7f6c50a00640 1 Processor -- start 2024-03-06T07:27:40.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.625+0000 7f6c50a00640 1 -- start start 2024-03-06T07:27:40.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.626+0000 7f6c50a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 0x7f6c4c160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c50a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6c4c070ff0 0x7f6c4c0ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c50a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6c4c0b46c0 con 0x7f6c4c15fc10 2024-03-06T07:27:40.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c50a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c4c071480 con 0x7f6c4c070ff0 2024-03-06T07:27:40.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c50a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c4c0b49a0 con 0x7f6c4c0b3f70 2024-03-06T07:27:40.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c49e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6c4c15fc10 0x7f6c4c15fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57580/0 (socket says 172.21.15.86:57580) 2024-03-06T07:27:40.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c49e00640 1 -- 172.21.15.86:0/3414307945 learned_addr learned my addr 172.21.15.86:0/3414307945 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:40.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c49400640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 0x7f6c4c160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c4a800640 1 --2- 172.21.15.86:0/3414307945 >> v2:172.21.15.194:3300/0 conn(0x7f6c4c070ff0 0x7f6c4c0ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2639824784 0 0) 0x7f6c4c0b46c0 con 0x7f6c4c15fc10 2024-03-06T07:27:40.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.627+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6c30003520 con 0x7f6c4c15fc10 2024-03-06T07:27:40.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1919470377 0 0) 0x7f6c30003520 con 0x7f6c4c15fc10 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 msgr2=0x7f6c4c160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 0x7f6c4c160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 >> v2:172.21.15.194:3300/0 conn(0x7f6c4c070ff0 msgr2=0x7f6c4c0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 --2- 172.21.15.86:0/3414307945 >> v2:172.21.15.194:3300/0 conn(0x7f6c4c070ff0 0x7f6c4c0ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c4c072c90 con 0x7f6c4c15fc10 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6c3c001670 con 0x7f6c4c15fc10 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 >> v1:172.21.15.86:6789/0 conn(0x7f6c4c15fc10 legacy=0x7f6c4c15fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c30003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 0x7f6c30004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c30004d40 0x7f6c300051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c4c071480 con 0x7f6c300039b0 2024-03-06T07:27:40.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c4c0b49a0 con 0x7f6c30004d40 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c3000fdd0 con 0x7f6c4c0b3f70 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c49e00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c30003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.628+0000 7f6c49400640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 0x7f6c30004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.629+0000 7f6c49e00640 1 -- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 msgr2=0x7f6c30004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.629+0000 7f6c49e00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c0b3f70 0x7f6c30004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.629+0000 7f6c49e00640 1 -- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c30004d40 msgr2=0x7f6c300051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.629+0000 7f6c49e00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c30004d40 0x7f6c300051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.629+0000 7f6c49e00640 1 -- 172.21.15.86:0/3414307945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c30003520 con 0x7f6c300039b0 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c49e00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c30003dd0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f6c3c00c190 tx=0x7f6c30005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3b31f86eae935d1b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3c007c80 con 0x7f6c300039b0 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c3c030070 con 0x7f6c300039b0 2024-03-06T07:27:40.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c48a00640 1 -- 172.21.15.86:0/3414307945 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3c005d50 con 0x7f6c300039b0 2024-03-06T07:27:40.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c50a00640 1 -- 172.21.15.86:0/3414307945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 msgr2=0x7f6c30003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c50a00640 1 --2- 172.21.15.86:0/3414307945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c30003dd0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f6c3c00c190 tx=0x7f6c30005830 comp rx=0 tx=0).stop 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c4a800640 1 -- 172.21.15.86:0/3414307945 reap_dead start 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c50a00640 1 -- 172.21.15.86:0/3414307945 shutdown_connections 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.630+0000 7f6c50a00640 1 -- 172.21.15.86:0/3414307945 >> 172.21.15.86:0/3414307945 conn(0x7f6c4c0b1940 msgr2=0x7f6c4c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.631+0000 7f6c50a00640 1 -- 172.21.15.86:0/3414307945 shutdown_connections 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.631+0000 7f6c50a00640 1 -- 172.21.15.86:0/3414307945 wait complete. 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.632+0000 7f6c50a00640 1 Processor -- start 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.632+0000 7f6c50a00640 1 -- start start 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.633+0000 7f6c50a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c4c145b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.633+0000 7f6c50a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c14afd0 0x7f6c4c0cdac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.633+0000 7f6c50a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c4c1463e0 0x7f6c4c0ce000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.633+0000 7f6c50a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c4c072a30 con 0x7f6c300039b0 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.633+0000 7f6c50a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c4c0728b0 con 0x7f6c4c1463e0 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.633+0000 7f6c50a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c4c15c2e0 con 0x7f6c4c14afd0 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c4c145b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c4c145b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36580/0 (socket says 172.21.15.86:36580) 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 -- 172.21.15.86:0/3265693319 learned_addr learned my addr 172.21.15.86:0/3265693319 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49400640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c14afd0 0x7f6c4c0cdac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c4a800640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c4c1463e0 0x7f6c4c0ce000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 -- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c14afd0 msgr2=0x7f6c4c0cdac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c14afd0 0x7f6c4c0cdac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 -- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c4c1463e0 msgr2=0x7f6c4c0ce000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c4c1463e0 0x7f6c4c0ce000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 -- 172.21.15.86:0/3265693319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c4c09ab10 con 0x7f6c300039b0 2024-03-06T07:27:40.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.634+0000 7f6c49e00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c4c145b10 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f6c3c00c2b0 tx=0x7f6c3c001970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:40.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.635+0000 7f6c42000640 1 -- 172.21.15.86:0/3265693319 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3c045070 con 0x7f6c300039b0 2024-03-06T07:27:40.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.635+0000 7f6c42000640 1 -- 172.21.15.86:0/3265693319 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c3c004c10 con 0x7f6c300039b0 2024-03-06T07:27:40.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.635+0000 7f6c42000640 1 -- 172.21.15.86:0/3265693319 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3c007c80 con 0x7f6c300039b0 2024-03-06T07:27:40.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.635+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6c4c06ca30 con 0x7f6c300039b0 2024-03-06T07:27:40.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.635+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c4c06cfb0 con 0x7f6c300039b0 2024-03-06T07:27:40.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.636+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6c4c04f330 con 0x7f6c300039b0 2024-03-06T07:27:40.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.637+0000 7f6c42000640 1 -- 172.21.15.86:0/3265693319 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6c3c007070 con 0x7f6c300039b0 2024-03-06T07:27:40.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.638+0000 7f6c42000640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2c06b4d0 0x7f6c2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:40.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.638+0000 7f6c42000640 1 -- 172.21.15.86:0/3265693319 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6c3c0ab1e0 con 0x7f6c300039b0 2024-03-06T07:27:40.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.638+0000 7f6c49400640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2c06b4d0 0x7f6c2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:40.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.638+0000 7f6c49400640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2c06b4d0 0x7f6c2c06d990 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f6c4c0b1e30 tx=0x7f6c38008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:40.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.642+0000 7f6c42000640 1 -- 172.21.15.86:0/3265693319 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6c4c09ab10 con 0x7f6c300039b0 2024-03-06T07:27:40.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.840+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f6c4c06ef30 con 0x7f6c300039b0 2024-03-06T07:27:40.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.841+0000 7f6c42000640 1 -- 172.21.15.86:0/3265693319 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f6c3c07a040 con 0x7f6c300039b0 2024-03-06T07:27:40.843 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:40.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.843+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2c06b4d0 msgr2=0x7f6c2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.843+0000 7f6c50a00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2c06b4d0 0x7f6c2c06d990 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f6c4c0b1e30 tx=0x7f6c38008040 comp rx=0 tx=0).stop 2024-03-06T07:27:40.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.843+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 msgr2=0x7f6c4c145b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.843+0000 7f6c50a00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c4c145b10 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f6c3c00c2b0 tx=0x7f6c3c001970 comp rx=0 tx=0).stop 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 shutdown_connections 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2c06b4d0 0x7f6c2c06d990 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c4c1463e0 0x7f6c4c0ce000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c4c14afd0 0x7f6c4c0cdac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 --2- 172.21.15.86:0/3265693319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c300039b0 0x7f6c4c145b10 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 >> 172.21.15.86:0/3265693319 conn(0x7f6c4c0b1940 msgr2=0x7f6c4c14f080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 shutdown_connections 2024-03-06T07:27:40.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:40.844+0000 7f6c50a00640 1 -- 172.21.15.86:0/3265693319 wait complete. 2024-03-06T07:27:40.865 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:41.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.064+0000 7fd09ce00640 1 Processor -- start 2024-03-06T07:27:41.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.064+0000 7fd09ce00640 1 -- start start 2024-03-06T07:27:41.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.065+0000 7fd09ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd098158570 0x7fd0980ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.065+0000 7fd09ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd0980cfde0 0x7fd0980cd7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.065+0000 7fd09ce00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd09815b130 con 0x7fd098158950 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.065+0000 7fd09ce00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd0980af170 con 0x7fd0980cfde0 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.065+0000 7fd09ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd09815b3f0 con 0x7fd098158570 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.066+0000 7fd097e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd0980cfde0 0x7fd0980cd7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.066+0000 7fd097e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd0980cfde0 0x7fd0980cd7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36586/0 (socket says 172.21.15.86:36586) 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.066+0000 7fd097e00640 1 -- 172.21.15.86:0/4212383142 learned_addr learned my addr 172.21.15.86:0/4212383142 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.066+0000 7fd097e00640 1 -- 172.21.15.86:0/4212383142 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd098158570 msgr2=0x7fd0980ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.066+0000 7fd097e00640 1 --2- 172.21.15.86:0/4212383142 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd098158570 0x7fd0980ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.066+0000 7fd097e00640 1 -- 172.21.15.86:0/4212383142 >> v1:172.21.15.194:6789/0 conn(0x7fd098158950 legacy=0x7fd0980cf6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.066+0000 7fd097e00640 1 -- 172.21.15.86:0/4212383142 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd09815b6d0 con 0x7fd0980cfde0 2024-03-06T07:27:41.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.067+0000 7fd097e00640 1 --2- 172.21.15.86:0/4212383142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0980cfde0 0x7fd0980cd7c0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fd090009e50 tx=0x7fd09002be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea15cb8fd17b083a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.067+0000 7fd096a00640 1 -- 172.21.15.86:0/4212383142 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd09002c9f0 con 0x7fd0980cfde0 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.067+0000 7fd096a00640 1 -- 172.21.15.86:0/4212383142 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd09002cb90 con 0x7fd0980cfde0 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.067+0000 7fd096a00640 1 -- 172.21.15.86:0/4212383142 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0900343f0 con 0x7fd0980cfde0 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 -- 172.21.15.86:0/4212383142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0980cfde0 msgr2=0x7fd0980cd7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 --2- 172.21.15.86:0/4212383142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0980cfde0 0x7fd0980cd7c0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fd090009e50 tx=0x7fd09002be50 comp rx=0 tx=0).stop 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 -- 172.21.15.86:0/4212383142 shutdown_connections 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 --2- 172.21.15.86:0/4212383142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0980cfde0 0x7fd0980cd7c0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 --2- 172.21.15.86:0/4212383142 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd098158570 0x7fd0980ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 -- 172.21.15.86:0/4212383142 >> 172.21.15.86:0/4212383142 conn(0x7fd09806afc0 msgr2=0x7fd09806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 -- 172.21.15.86:0/4212383142 shutdown_connections 2024-03-06T07:27:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.068+0000 7fd09ce00640 1 -- 172.21.15.86:0/4212383142 wait complete. 2024-03-06T07:27:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.070+0000 7fd09ce00640 1 Processor -- start 2024-03-06T07:27:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.070+0000 7fd09ce00640 1 -- start start 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd09ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0980cfde0 0x7fd0981472f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd09ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd098158570 0x7fd0980cd930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd09ce00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd098158950 0x7fd0980cde70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd09ce00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd09815bed0 con 0x7fd098158570 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd09ce00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0980b4950 con 0x7fd098158950 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd09ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd09815c050 con 0x7fd0980cfde0 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd098158570 0x7fd0980cd930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd098158570 0x7fd0980cd930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36592/0 (socket says 172.21.15.86:36592) 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 -- 172.21.15.86:0/279497676 learned_addr learned my addr 172.21.15.86:0/279497676 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd097400640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0980cfde0 0x7fd0981472f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 -- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0980cfde0 msgr2=0x7fd0981472f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0980cfde0 0x7fd0981472f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 -- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd098158950 msgr2=0x7fd0980cde70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd098158950 0x7fd0980cde70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd08fe00640 1 -- 172.21.15.86:0/279497676 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd098153930 con 0x7fd098158570 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.071+0000 7fd097400640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0980cfde0 0x7fd0981472f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.072+0000 7fd08fe00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd098158570 0x7fd0980cd930 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fd08000b3e0 tx=0x7fd08000b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.072+0000 7fd08f400640 1 -- 172.21.15.86:0/279497676 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd080015020 con 0x7fd098158570 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.072+0000 7fd08f400640 1 -- 172.21.15.86:0/279497676 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd080011850 con 0x7fd098158570 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.072+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd098153c10 con 0x7fd098158570 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.072+0000 7fd08f400640 1 -- 172.21.15.86:0/279497676 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd080011c40 con 0x7fd098158570 2024-03-06T07:27:41.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.072+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0980a9730 con 0x7fd098158570 2024-03-06T07:27:41.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.073+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd09804f330 con 0x7fd098158570 2024-03-06T07:27:41.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.074+0000 7fd08f400640 1 -- 172.21.15.86:0/279497676 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd080018020 con 0x7fd098158570 2024-03-06T07:27:41.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.075+0000 7fd08f400640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd07406b4d0 0x7fd07406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.075+0000 7fd08f400640 1 -- 172.21.15.86:0/279497676 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd08008a210 con 0x7fd098158570 2024-03-06T07:27:41.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.075+0000 7fd097400640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd07406b4d0 0x7fd07406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.076+0000 7fd097400640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd07406b4d0 0x7fd07406d990 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fd088006fd0 tx=0x7fd088008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.079+0000 7fd08f400640 1 -- 172.21.15.86:0/279497676 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd08001f020 con 0x7fd098158570 2024-03-06T07:27:41.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.262+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fd0980b4730 con 0x7fd098158570 2024-03-06T07:27:41.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.262+0000 7fd08f400640 1 -- 172.21.15.86:0/279497676 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7fd0800172c0 con 0x7fd098158570 2024-03-06T07:27:41.264 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:41.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd07406b4d0 msgr2=0x7fd07406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd07406b4d0 0x7fd07406d990 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fd088006fd0 tx=0x7fd088008040 comp rx=0 tx=0).stop 2024-03-06T07:27:41.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd098158570 msgr2=0x7fd0980cd930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd098158570 0x7fd0980cd930 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fd08000b3e0 tx=0x7fd08000b8a0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 shutdown_connections 2024-03-06T07:27:41.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd07406b4d0 0x7fd07406d990 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd098158950 0x7fd0980cde70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd098158570 0x7fd0980cd930 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 --2- 172.21.15.86:0/279497676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0980cfde0 0x7fd0981472f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 >> 172.21.15.86:0/279497676 conn(0x7fd09806afc0 msgr2=0x7fd0981562d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:41.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 shutdown_connections 2024-03-06T07:27:41.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.265+0000 7fd09ce00640 1 -- 172.21.15.86:0/279497676 wait complete. 2024-03-06T07:27:41.284 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:41.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.487+0000 7f39cfa00640 1 Processor -- start 2024-03-06T07:27:41.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.488+0000 7f39cfa00640 1 -- start start 2024-03-06T07:27:41.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cfa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 0x7f39c815fea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cfa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f39c81603e0 0x7f39c80cc870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cfa00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39c80ce1e0 con 0x7f39c815c330 2024-03-06T07:27:41.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cfa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f39c8160830 con 0x7f39c81603e0 2024-03-06T07:27:41.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cfa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39c80cedf0 con 0x7f39c80b43b0 2024-03-06T07:27:41.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cd000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f39c815c330 0x7f39c80d0570 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57582/0 (socket says 172.21.15.86:57582) 2024-03-06T07:27:41.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cd000640 1 -- 172.21.15.86:0/3663992369 learned_addr learned my addr 172.21.15.86:0/3663992369 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:41.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39c7e00640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 0x7f39c815fea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.489+0000 7f39cda00640 1 --2- 172.21.15.86:0/3663992369 >> v2:172.21.15.194:3300/0 conn(0x7f39c81603e0 0x7f39c80cc870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 428794705 0 0) 0x7f39c80ce1e0 con 0x7f39c815c330 2024-03-06T07:27:41.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f39ac003520 con 0x7f39c815c330 2024-03-06T07:27:41.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4094430291 0 0) 0x7f39ac003520 con 0x7f39c815c330 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 msgr2=0x7f39c815fea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 0x7f39c815fea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 >> v2:172.21.15.194:3300/0 conn(0x7f39c81603e0 msgr2=0x7f39c80cc870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 --2- 172.21.15.86:0/3663992369 >> v2:172.21.15.194:3300/0 conn(0x7f39c81603e0 0x7f39c80cc870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39c80ce4c0 con 0x7f39c815c330 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f39b0001670 con 0x7f39c815c330 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 >> v1:172.21.15.86:6789/0 conn(0x7f39c815c330 legacy=0x7f39c80d0570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac0039b0 0x7f39ac003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 0x7f39ac004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39ac004d40 0x7f39ac0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39c8160830 con 0x7f39ac004d40 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39c80cedf0 con 0x7f39ac0039b0 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.490+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39ac00b9c0 con 0x7f39c80b43b0 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39cda00640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39ac004d40 0x7f39ac0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39c7e00640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 0x7f39ac004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39cd000640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac0039b0 0x7f39ac003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39cda00640 1 -- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 msgr2=0x7f39ac004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:41.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39cda00640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80b43b0 0x7f39ac004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39cda00640 1 -- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac0039b0 msgr2=0x7f39ac003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:41.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39cda00640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac0039b0 0x7f39ac003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.491+0000 7f39cda00640 1 -- 172.21.15.86:0/3663992369 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39ac003520 con 0x7f39ac004d40 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.492+0000 7f39cda00640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39ac004d40 0x7f39ac0051c0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f39c0009ac0 tx=0x7f39ac005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ab217ddd9b3ea65 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.492+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c0039070 con 0x7f39ac004d40 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.492+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f39c002c710 con 0x7f39ac004d40 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.492+0000 7f39c7400640 1 -- 172.21.15.86:0/3663992369 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c0009690 con 0x7f39ac004d40 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.493+0000 7f39cfa00640 1 -- 172.21.15.86:0/3663992369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39ac004d40 msgr2=0x7f39ac0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.493+0000 7f39cfa00640 1 --2- 172.21.15.86:0/3663992369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39ac004d40 0x7f39ac0051c0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f39c0009ac0 tx=0x7f39ac005830 comp rx=0 tx=0).stop 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.493+0000 7f39cda00640 1 -- 172.21.15.86:0/3663992369 reap_dead start 2024-03-06T07:27:41.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.493+0000 7f39cfa00640 1 -- 172.21.15.86:0/3663992369 shutdown_connections 2024-03-06T07:27:41.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.493+0000 7f39cfa00640 1 -- 172.21.15.86:0/3663992369 >> 172.21.15.86:0/3663992369 conn(0x7f39c80b3790 msgr2=0x7f39c80b3ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:41.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.493+0000 7f39cfa00640 1 -- 172.21.15.86:0/3663992369 shutdown_connections 2024-03-06T07:27:41.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.493+0000 7f39cfa00640 1 -- 172.21.15.86:0/3663992369 wait complete. 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.495+0000 7f39cfa00640 1 Processor -- start 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.495+0000 7f39cfa00640 1 -- start start 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.496+0000 7f39cfa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac004d40 0x7f39c815b120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.496+0000 7f39cfa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80ca180 0x7f39c80c8d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.496+0000 7f39cfa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39c80cad50 0x7f39c80cb1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.496+0000 7f39cfa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39c815d720 con 0x7f39c80cad50 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.496+0000 7f39cfa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39c80cea80 con 0x7f39ac004d40 2024-03-06T07:27:41.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.496+0000 7f39cfa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39c80cf280 con 0x7f39c80ca180 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.496+0000 7f39cda00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39c80cad50 0x7f39c80cb1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39c80cad50 0x7f39c80cb1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36624/0 (socket says 172.21.15.86:36624) 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 -- 172.21.15.86:0/2644780964 learned_addr learned my addr 172.21.15.86:0/2644780964 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39c7e00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80ca180 0x7f39c80c8d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cd000640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac004d40 0x7f39c815b120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 -- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80ca180 msgr2=0x7f39c80c8d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80ca180 0x7f39c80c8d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 -- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac004d40 msgr2=0x7f39c815b120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac004d40 0x7f39c815b120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 -- 172.21.15.86:0/2644780964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39c81f51c0 con 0x7f39c80cad50 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39cda00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39c80cad50 0x7f39c80cb1d0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f39c0009ac0 tx=0x7f39c0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.497+0000 7f39c4c00640 1 -- 172.21.15.86:0/2644780964 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c0039070 con 0x7f39c80cad50 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.498+0000 7f39c4c00640 1 -- 172.21.15.86:0/2644780964 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f39c0043070 con 0x7f39c80cad50 2024-03-06T07:27:41.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.498+0000 7f39c4c00640 1 -- 172.21.15.86:0/2644780964 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c0003a50 con 0x7f39c80cad50 2024-03-06T07:27:41.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.498+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39c80c96a0 con 0x7f39c80cad50 2024-03-06T07:27:41.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.498+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39c80c9c60 con 0x7f39c80cad50 2024-03-06T07:27:41.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.499+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f39c804f330 con 0x7f39c80cad50 2024-03-06T07:27:41.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.500+0000 7f39c4c00640 1 -- 172.21.15.86:0/2644780964 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f39c0003bf0 con 0x7f39c80cad50 2024-03-06T07:27:41.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.500+0000 7f39c4c00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f39a406b4d0 0x7f39a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.501+0000 7f39c4c00640 1 -- 172.21.15.86:0/2644780964 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f39c00ab490 con 0x7f39c80cad50 2024-03-06T07:27:41.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.501+0000 7f39cd000640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f39a406b4d0 0x7f39a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.501+0000 7f39cd000640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f39a406b4d0 0x7f39a406d990 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f39b0000c00 tx=0x7f39b0001430 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.505+0000 7f39c4c00640 1 -- 172.21.15.86:0/2644780964 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f39c004c020 con 0x7f39c80cad50 2024-03-06T07:27:41.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.700+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f39c80c4ec0 con 0x7f39c80cad50 2024-03-06T07:27:41.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.701+0000 7f39c4c00640 1 -- 172.21.15.86:0/2644780964 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f39c00044a0 con 0x7f39c80cad50 2024-03-06T07:27:41.703 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:41.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.704+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f39a406b4d0 msgr2=0x7f39a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.704+0000 7f39cfa00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f39a406b4d0 0x7f39a406d990 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f39b0000c00 tx=0x7f39b0001430 comp rx=0 tx=0).stop 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.704+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39c80cad50 msgr2=0x7f39c80cb1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.704+0000 7f39cfa00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39c80cad50 0x7f39c80cb1d0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f39c0009ac0 tx=0x7f39c0004730 comp rx=0 tx=0).stop 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 shutdown_connections 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f39a406b4d0 0x7f39a406d990 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39c80cad50 0x7f39c80cb1d0 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39c80ca180 0x7f39c80c8d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 --2- 172.21.15.86:0/2644780964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39ac004d40 0x7f39c815b120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 >> 172.21.15.86:0/2644780964 conn(0x7f39c80b3790 msgr2=0x7f39c806ec90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 shutdown_connections 2024-03-06T07:27:41.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.705+0000 7f39cfa00640 1 -- 172.21.15.86:0/2644780964 wait complete. 2024-03-06T07:27:41.725 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:41.928 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.927+0000 7f7ba5400640 1 Processor -- start 2024-03-06T07:27:41.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.927+0000 7f7ba5400640 1 -- start start 2024-03-06T07:27:41.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7ba5400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7ba0144c00 0x7f7ba006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7ba5400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7ba015f9d0 0x7f7ba015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7ba5400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ba006ee70 con 0x7f7ba00ce0a0 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7ba5400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7ba006cef0 con 0x7f7ba015f9d0 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7ba5400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7ba006d1d0 con 0x7f7ba0144c00 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7ba015f9d0 0x7f7ba015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9f200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7ba00ce0a0 0x7f7ba00ce4b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57598/0 (socket says 172.21.15.86:57598) 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9f200640 1 -- 172.21.15.86:0/3728459302 learned_addr learned my addr 172.21.15.86:0/3728459302 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9e800640 1 --2- 172.21.15.86:0/3728459302 >> v2:172.21.15.194:3300/0 conn(0x7f7ba0144c00 0x7f7ba006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9de00640 1 -- 172.21.15.86:0/3728459302 >> v2:172.21.15.194:3300/0 conn(0x7f7ba0144c00 msgr2=0x7f7ba006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9de00640 1 --2- 172.21.15.86:0/3728459302 >> v2:172.21.15.194:3300/0 conn(0x7f7ba0144c00 0x7f7ba006e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9de00640 1 -- 172.21.15.86:0/3728459302 >> v1:172.21.15.86:6789/0 conn(0x7f7ba00ce0a0 legacy=0x7f7ba00ce4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.930+0000 7f7b9de00640 1 -- 172.21.15.86:0/3728459302 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ba006d4b0 con 0x7f7ba015f9d0 2024-03-06T07:27:41.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.929+0000 7f7b9d400640 1 -- 172.21.15.86:0/3728459302 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3928739808 0 0) 0x7f7b98001490 con 0x7f7ba00ce0a0 2024-03-06T07:27:41.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.931+0000 7f7b9de00640 1 --2- 172.21.15.86:0/3728459302 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015fe60 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f7ba00b25f0 tx=0x7f7b8c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=df154a116111d2e0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.931+0000 7f7b9d400640 1 -- 172.21.15.86:0/3728459302 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b8c035020 con 0x7f7ba015f9d0 2024-03-06T07:27:41.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.931+0000 7f7b9d400640 1 -- 172.21.15.86:0/3728459302 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7b8c031830 con 0x7f7ba015f9d0 2024-03-06T07:27:41.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.931+0000 7f7b9d400640 1 -- 172.21.15.86:0/3728459302 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b8c031c00 con 0x7f7ba015f9d0 2024-03-06T07:27:41.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.931+0000 7f7ba5400640 1 -- 172.21.15.86:0/3728459302 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 msgr2=0x7f7ba015fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.931+0000 7f7ba5400640 1 --2- 172.21.15.86:0/3728459302 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015fe60 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f7ba00b25f0 tx=0x7f7b8c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.932+0000 7f7ba5400640 1 -- 172.21.15.86:0/3728459302 shutdown_connections 2024-03-06T07:27:41.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.932+0000 7f7ba5400640 1 --2- 172.21.15.86:0/3728459302 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015fe60 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.932+0000 7f7ba5400640 1 --2- 172.21.15.86:0/3728459302 >> v2:172.21.15.194:3300/0 conn(0x7f7ba0144c00 0x7f7ba006e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.932+0000 7f7ba5400640 1 -- 172.21.15.86:0/3728459302 >> 172.21.15.86:0/3728459302 conn(0x7f7ba00b1730 msgr2=0x7f7ba00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:41.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.932+0000 7f7ba5400640 1 -- 172.21.15.86:0/3728459302 shutdown_connections 2024-03-06T07:27:41.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.932+0000 7f7ba5400640 1 -- 172.21.15.86:0/3728459302 wait complete. 2024-03-06T07:27:41.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.933+0000 7f7ba5400640 1 Processor -- start 2024-03-06T07:27:41.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.933+0000 7f7ba5400640 1 -- start start 2024-03-06T07:27:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.934+0000 7f7ba5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ba00ce0a0 0x7f7ba013b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.934+0000 7f7ba5400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ba0144c00 0x7f7ba013c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.934+0000 7f7ba5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.934+0000 7f7ba5400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ba006cb50 con 0x7f7ba015f9d0 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.934+0000 7f7ba5400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ba0160730 con 0x7f7ba0144c00 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.934+0000 7f7ba5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ba006ccd0 con 0x7f7ba00ce0a0 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015e6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36642/0 (socket says 172.21.15.86:36642) 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 -- 172.21.15.86:0/2726885748 learned_addr learned my addr 172.21.15.86:0/2726885748 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9e800640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ba00ce0a0 0x7f7ba013b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9de00640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ba0144c00 0x7f7ba013c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 -- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ba00ce0a0 msgr2=0x7f7ba013b390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ba00ce0a0 0x7f7ba013b390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 -- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ba0144c00 msgr2=0x7f7ba013c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ba0144c00 0x7f7ba013c350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.935+0000 7f7b9f200640 1 -- 172.21.15.86:0/2726885748 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ba015c3e0 con 0x7f7ba015f9d0 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.936+0000 7f7b9f200640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015e6a0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f7b9800c730 tx=0x7f7b9800cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.936+0000 7f7b96a00640 1 -- 172.21.15.86:0/2726885748 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b98018070 con 0x7f7ba015f9d0 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.936+0000 7f7b96a00640 1 -- 172.21.15.86:0/2726885748 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7b9800ce40 con 0x7f7ba015f9d0 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.936+0000 7f7b96a00640 1 -- 172.21.15.86:0/2726885748 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b9800a860 con 0x7f7ba015f9d0 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.936+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7ba015c6c0 con 0x7f7ba015f9d0 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.936+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ba015cc80 con 0x7f7ba015f9d0 2024-03-06T07:27:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.937+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7ba0144740 con 0x7f7ba015f9d0 2024-03-06T07:27:41.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.938+0000 7f7b96a00640 1 -- 172.21.15.86:0/2726885748 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7b9800aa00 con 0x7f7ba015f9d0 2024-03-06T07:27:41.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.939+0000 7f7b96a00640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b8006b4d0 0x7f7b8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:41.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.939+0000 7f7b96a00640 1 -- 172.21.15.86:0/2726885748 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7b98089860 con 0x7f7ba015f9d0 2024-03-06T07:27:41.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.939+0000 7f7b9e800640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b8006b4d0 0x7f7b8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:41.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.940+0000 7f7b9e800640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b8006b4d0 0x7f7b8006d990 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f7b90006fd0 tx=0x7f7b90008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:41.943+0000 7f7b96a00640 1 -- 172.21.15.86:0/2726885748 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7ba006ee70 con 0x7f7ba015f9d0 2024-03-06T07:27:42.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.137+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7ba0072890 con 0x7f7ba015f9d0 2024-03-06T07:27:42.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.140+0000 7f7b96a00640 1 -- 172.21.15.86:0/2726885748 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f7b980586d0 con 0x7f7ba015f9d0 2024-03-06T07:27:42.142 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:42.142 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:42.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.142+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b8006b4d0 msgr2=0x7f7b8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b8006b4d0 0x7f7b8006d990 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f7b90006fd0 tx=0x7f7b90008040 comp rx=0 tx=0).stop 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 msgr2=0x7f7ba015e6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015e6a0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f7b9800c730 tx=0x7f7b9800cbf0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 shutdown_connections 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b8006b4d0 0x7f7b8006d990 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ba015f9d0 0x7f7ba015e6a0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ba0144c00 0x7f7ba013c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 --2- 172.21.15.86:0/2726885748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ba00ce0a0 0x7f7ba013b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 >> 172.21.15.86:0/2726885748 conn(0x7f7ba00b1730 msgr2=0x7f7ba00b0290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 shutdown_connections 2024-03-06T07:27:42.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.143+0000 7f7ba5400640 1 -- 172.21.15.86:0/2726885748 wait complete. 2024-03-06T07:27:42.164 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:42.164 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:42.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.374+0000 7fe10f200640 1 Processor -- start 2024-03-06T07:27:42.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.374+0000 7fe10f200640 1 -- start start 2024-03-06T07:27:42.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe11015be10 0x7fe1100ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10f200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe1100aef20 0x7fe1100ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10f200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe11015c1f0 con 0x7fe11006d330 2024-03-06T07:27:42.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10f200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe1100cd200 con 0x7fe11015be10 2024-03-06T07:27:42.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10f200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe11006cac0 con 0x7fe1100aef20 2024-03-06T07:27:42.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10e800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe11006d330 0x7fe110071ab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57612/0 (socket says 172.21.15.86:57612) 2024-03-06T07:27:42.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10e800640 1 -- 172.21.15.86:0/578080983 learned_addr learned my addr 172.21.15.86:0/578080983 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:42.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe10de00640 1 --2- 172.21.15.86:0/578080983 >> v2:172.21.15.86:3300/0 conn(0x7fe11015be10 0x7fe1100ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.376+0000 7fe107e00640 1 --2- 172.21.15.86:0/578080983 >> v2:172.21.15.194:3300/0 conn(0x7fe1100aef20 0x7fe1100ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.377+0000 7fe10d400640 1 -- 172.21.15.86:0/578080983 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3320604663 0 0) 0x7fe11015c1f0 con 0x7fe11006d330 2024-03-06T07:27:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.377+0000 7fe10d400640 1 -- 172.21.15.86:0/578080983 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe0f0003520 con 0x7fe11006d330 2024-03-06T07:27:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.377+0000 7fe10de00640 1 -- 172.21.15.86:0/578080983 >> v2:172.21.15.194:3300/0 conn(0x7fe1100aef20 msgr2=0x7fe1100ce2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.377+0000 7fe10de00640 1 --2- 172.21.15.86:0/578080983 >> v2:172.21.15.194:3300/0 conn(0x7fe1100aef20 0x7fe1100ce2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.377+0000 7fe10de00640 1 -- 172.21.15.86:0/578080983 >> v1:172.21.15.86:6789/0 conn(0x7fe11006d330 legacy=0x7fe110071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.377+0000 7fe10de00640 1 -- 172.21.15.86:0/578080983 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe11006cda0 con 0x7fe11015be10 2024-03-06T07:27:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.379+0000 7fe10de00640 1 --2- 172.21.15.86:0/578080983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11015be10 0x7fe1100ccc20 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fe11006ea00 tx=0x7fe10002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80c8186a9e57dded server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.379+0000 7fe10d400640 1 -- 172.21.15.86:0/578080983 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe100035020 con 0x7fe11015be10 2024-03-06T07:27:42.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.379+0000 7fe10d400640 1 -- 172.21.15.86:0/578080983 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe100031830 con 0x7fe11015be10 2024-03-06T07:27:42.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.379+0000 7fe10d400640 1 -- 172.21.15.86:0/578080983 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe100031c00 con 0x7fe11015be10 2024-03-06T07:27:42.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 -- 172.21.15.86:0/578080983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11015be10 msgr2=0x7fe1100ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 --2- 172.21.15.86:0/578080983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11015be10 0x7fe1100ccc20 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fe11006ea00 tx=0x7fe10002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 -- 172.21.15.86:0/578080983 shutdown_connections 2024-03-06T07:27:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 --2- 172.21.15.86:0/578080983 >> v2:172.21.15.194:3300/0 conn(0x7fe1100aef20 0x7fe1100ce2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 --2- 172.21.15.86:0/578080983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11015be10 0x7fe1100ccc20 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 -- 172.21.15.86:0/578080983 >> 172.21.15.86:0/578080983 conn(0x7fe11006afc0 msgr2=0x7fe11006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 -- 172.21.15.86:0/578080983 shutdown_connections 2024-03-06T07:27:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.380+0000 7fe10f200640 1 -- 172.21.15.86:0/578080983 wait complete. 2024-03-06T07:27:42.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.382+0000 7fe10f200640 1 Processor -- start 2024-03-06T07:27:42.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.382+0000 7fe10f200640 1 -- start start 2024-03-06T07:27:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11006d330 0x7fe11014d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10f200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1100aef20 0x7fe110150070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe11015be10 0x7fe11013c180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10f200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1100cda90 con 0x7fe11006d330 2024-03-06T07:27:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10f200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe11006d080 con 0x7fe1100aef20 2024-03-06T07:27:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe11006d7a0 con 0x7fe11015be10 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11006d330 0x7fe11014d1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11006d330 0x7fe11014d1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36662/0 (socket says 172.21.15.86:36662) 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe10de00640 1 -- 172.21.15.86:0/2416561791 learned_addr learned my addr 172.21.15.86:0/2416561791 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.383+0000 7fe107e00640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1100aef20 0x7fe110150070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10e800640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe11015be10 0x7fe11013c180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10de00640 1 -- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe11015be10 msgr2=0x7fe11013c180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10de00640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe11015be10 0x7fe11013c180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10de00640 1 -- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1100aef20 msgr2=0x7fe110150070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10de00640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1100aef20 0x7fe110150070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10de00640 1 -- 172.21.15.86:0/2416561791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe110146680 con 0x7fe11006d330 2024-03-06T07:27:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10de00640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11006d330 0x7fe11014d1d0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7fe100000c00 tx=0x7fe1000046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe106000640 1 -- 172.21.15.86:0/2416561791 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe100040070 con 0x7fe11006d330 2024-03-06T07:27:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe106000640 1 -- 172.21.15.86:0/2416561791 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe10003a910 con 0x7fe11006d330 2024-03-06T07:27:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe106000640 1 -- 172.21.15.86:0/2416561791 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe100035040 con 0x7fe11006d330 2024-03-06T07:27:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe1101523d0 con 0x7fe11006d330 2024-03-06T07:27:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.384+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe110152990 con 0x7fe11006d330 2024-03-06T07:27:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.386+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe11004f330 con 0x7fe11006d330 2024-03-06T07:27:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.386+0000 7fe106000640 1 -- 172.21.15.86:0/2416561791 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe100038020 con 0x7fe11006d330 2024-03-06T07:27:42.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.387+0000 7fe106000640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0ec06b4d0 0x7fe0ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.387+0000 7fe106000640 1 -- 172.21.15.86:0/2416561791 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe1000ab9d0 con 0x7fe11006d330 2024-03-06T07:27:42.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.387+0000 7fe107e00640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0ec06b4d0 0x7fe0ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.388+0000 7fe107e00640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0ec06b4d0 0x7fe0ec06d990 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fe11014dad0 tx=0x7fe0f8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:42.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.392+0000 7fe106000640 1 -- 172.21.15.86:0/2416561791 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe10007a840 con 0x7fe11006d330 2024-03-06T07:27:42.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.587+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fe1100cef60 con 0x7fe11006d330 2024-03-06T07:27:42.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.588+0000 7fe106000640 1 -- 172.21.15.86:0/2416561791 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7fe100080020 con 0x7fe11006d330 2024-03-06T07:27:42.590 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.590+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0ec06b4d0 msgr2=0x7fe0ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.590+0000 7fe10f200640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0ec06b4d0 0x7fe0ec06d990 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fe11014dad0 tx=0x7fe0f8008040 comp rx=0 tx=0).stop 2024-03-06T07:27:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11006d330 msgr2=0x7fe11014d1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11006d330 0x7fe11014d1d0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7fe100000c00 tx=0x7fe1000046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 shutdown_connections 2024-03-06T07:27:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0ec06b4d0 0x7fe0ec06d990 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe11015be10 0x7fe11013c180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1100aef20 0x7fe110150070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 --2- 172.21.15.86:0/2416561791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe11006d330 0x7fe11014d1d0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 >> 172.21.15.86:0/2416561791 conn(0x7fe11006afc0 msgr2=0x7fe1100af5e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 shutdown_connections 2024-03-06T07:27:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.591+0000 7fe10f200640 1 -- 172.21.15.86:0/2416561791 wait complete. 2024-03-06T07:27:42.611 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:42.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.816+0000 7f04bb800640 1 Processor -- start 2024-03-06T07:27:42.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.816+0000 7f04bb800640 1 -- start start 2024-03-06T07:27:42.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.817+0000 7f04bb800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f04b40b3df0 0x7f04b40727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.817+0000 7f04bb800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04b40719e0 con 0x7f04b40b3650 2024-03-06T07:27:42.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.817+0000 7f04bb800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04b4071cc0 con 0x7f04b40cda90 2024-03-06T07:27:42.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.817+0000 7f04bb800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04b4072310 con 0x7f04b40b3df0 2024-03-06T07:27:42.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.818+0000 7f04b3e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f04b40b3650 0x7f04b40b41d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57628/0 (socket says 172.21.15.86:57628) 2024-03-06T07:27:42.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.818+0000 7f04b3e00640 1 -- 172.21.15.86:0/990399628 learned_addr learned my addr 172.21.15.86:0/990399628 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:42.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.818+0000 7f04b8e00640 1 --2- 172.21.15.86:0/990399628 >> v2:172.21.15.194:3300/0 conn(0x7f04b40b3df0 0x7f04b40727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.818+0000 7f04b3400640 1 -- 172.21.15.86:0/990399628 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2682799274 0 0) 0x7f04b40719e0 con 0x7f04b40b3650 2024-03-06T07:27:42.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.818+0000 7f04b3400640 1 -- 172.21.15.86:0/990399628 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f049c003520 con 0x7f04b40b3650 2024-03-06T07:27:42.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.818+0000 7f04b8e00640 1 -- 172.21.15.86:0/990399628 >> v1:172.21.15.194:6789/0 conn(0x7f04b40cda90 legacy=0x7f04b406c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.818+0000 7f04b8e00640 1 -- 172.21.15.86:0/990399628 >> v1:172.21.15.86:6789/0 conn(0x7f04b40b3650 legacy=0x7f04b40b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.819+0000 7f04b3400640 1 -- 172.21.15.86:0/990399628 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2809284423 0 0) 0x7f049c003520 con 0x7f04b40b3650 2024-03-06T07:27:42.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.819+0000 7f04b8e00640 1 -- 172.21.15.86:0/990399628 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04b4071fa0 con 0x7f04b40b3df0 2024-03-06T07:27:42.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.820+0000 7f04b8e00640 1 --2- 172.21.15.86:0/990399628 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3df0 0x7f04b40727a0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f04a80097a0 tx=0x7f04a802b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c8128f4354b01014 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:42.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.820+0000 7f04b3400640 1 -- 172.21.15.86:0/990399628 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04a8038070 con 0x7f04b40b3df0 2024-03-06T07:27:42.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.820+0000 7f04b3400640 1 -- 172.21.15.86:0/990399628 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f04a8031500 con 0x7f04b40b3df0 2024-03-06T07:27:42.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.820+0000 7f04b3400640 1 -- 172.21.15.86:0/990399628 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04a0001670 con 0x7f04b40b3df0 2024-03-06T07:27:42.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.821+0000 7f04bb800640 1 -- 172.21.15.86:0/990399628 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3df0 msgr2=0x7f04b40727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.821+0000 7f04bb800640 1 --2- 172.21.15.86:0/990399628 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3df0 0x7f04b40727a0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f04a80097a0 tx=0x7f04a802b820 comp rx=0 tx=0).stop 2024-03-06T07:27:42.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.821+0000 7f04bb800640 1 -- 172.21.15.86:0/990399628 shutdown_connections 2024-03-06T07:27:42.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.821+0000 7f04bb800640 1 --2- 172.21.15.86:0/990399628 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3df0 0x7f04b40727a0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.821+0000 7f04bb800640 1 -- 172.21.15.86:0/990399628 >> 172.21.15.86:0/990399628 conn(0x7f04b40b1940 msgr2=0x7f04b40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:42.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.821+0000 7f04bb800640 1 -- 172.21.15.86:0/990399628 shutdown_connections 2024-03-06T07:27:42.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.821+0000 7f04bb800640 1 -- 172.21.15.86:0/990399628 wait complete. 2024-03-06T07:27:42.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.823+0000 7f04bb800640 1 Processor -- start 2024-03-06T07:27:42.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.823+0000 7f04bb800640 1 -- start start 2024-03-06T07:27:42.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04bb800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3650 0x7f04b41f4090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04bb800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04b40b3df0 0x7f04b409d480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04bb800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04b40cda90 0x7f04b4146e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04bb800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04b415b3c0 con 0x7f04b40b3df0 2024-03-06T07:27:42.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04bb800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04b415b240 con 0x7f04b40b3650 2024-03-06T07:27:42.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04bb800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04b40cf510 con 0x7f04b40cda90 2024-03-06T07:27:42.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04b3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04b40b3df0 0x7f04b409d480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04b3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04b40b3df0 0x7f04b409d480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36670/0 (socket says 172.21.15.86:36670) 2024-03-06T07:27:42.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04b3e00640 1 -- 172.21.15.86:0/3709872063 learned_addr learned my addr 172.21.15.86:0/3709872063 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04b8e00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3650 0x7f04b41f4090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.824+0000 7f04b9800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04b40cda90 0x7f04b4146e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b3e00640 1 -- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04b40cda90 msgr2=0x7f04b4146e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b3e00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04b40cda90 0x7f04b4146e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b3e00640 1 -- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3650 msgr2=0x7f04b41f4090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b3e00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3650 0x7f04b41f4090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b3e00640 1 -- 172.21.15.86:0/3709872063 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04b40ca170 con 0x7f04b40b3df0 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b8e00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3650 0x7f04b41f4090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b9800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04b40cda90 0x7f04b4146e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b3e00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04b40b3df0 0x7f04b409d480 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f04a000c700 tx=0x7f04a000cbc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:42.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b0c00640 1 -- 172.21.15.86:0/3709872063 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04a0005ad0 con 0x7f04b40b3df0 2024-03-06T07:27:42.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b0c00640 1 -- 172.21.15.86:0/3709872063 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f04a0005c70 con 0x7f04b40b3df0 2024-03-06T07:27:42.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04b40ca450 con 0x7f04b40b3df0 2024-03-06T07:27:42.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04b0c00640 1 -- 172.21.15.86:0/3709872063 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04a0005e60 con 0x7f04b40b3df0 2024-03-06T07:27:42.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.825+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04b40caa10 con 0x7f04b40b3df0 2024-03-06T07:27:42.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.827+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f04b404f330 con 0x7f04b40b3df0 2024-03-06T07:27:42.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.827+0000 7f04b0c00640 1 -- 172.21.15.86:0/3709872063 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f04a0007290 con 0x7f04b40b3df0 2024-03-06T07:27:42.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.828+0000 7f04b0c00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f049406b500 0x7f049406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:42.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.828+0000 7f04b0c00640 1 -- 172.21.15.86:0/3709872063 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f04a0089fe0 con 0x7f04b40b3df0 2024-03-06T07:27:42.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.829+0000 7f04b8e00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f049406b500 0x7f049406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:42.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.829+0000 7f04b8e00640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f049406b500 0x7f049406d9c0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f04a8035000 tx=0x7f04a8008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:42.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:42.833+0000 7f04b0c00640 1 -- 172.21.15.86:0/3709872063 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f04a0058de0 con 0x7f04b40b3df0 2024-03-06T07:27:43.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.032+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f04b40cc0e0 con 0x7f04b40b3df0 2024-03-06T07:27:43.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.033+0000 7f04b0c00640 1 -- 172.21.15.86:0/3709872063 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7f04a00586e0 con 0x7f04b40b3df0 2024-03-06T07:27:43.035 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f049406b500 msgr2=0x7f049406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f049406b500 0x7f049406d9c0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f04a8035000 tx=0x7f04a8008030 comp rx=0 tx=0).stop 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04b40b3df0 msgr2=0x7f04b409d480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04b40b3df0 0x7f04b409d480 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f04a000c700 tx=0x7f04a000cbc0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 shutdown_connections 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f049406b500 0x7f049406d9c0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04b40cda90 0x7f04b4146e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04b40b3df0 0x7f04b409d480 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 --2- 172.21.15.86:0/3709872063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04b40b3650 0x7f04b41f4090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 >> 172.21.15.86:0/3709872063 conn(0x7f04b40b1940 msgr2=0x7f04b40b03d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 shutdown_connections 2024-03-06T07:27:43.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.036+0000 7f04bb800640 1 -- 172.21.15.86:0/3709872063 wait complete. 2024-03-06T07:27:43.055 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:43.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.255+0000 7f76e0a00640 1 Processor -- start 2024-03-06T07:27:43.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.255+0000 7f76e0a00640 1 -- start start 2024-03-06T07:27:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.257+0000 7f76e0a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc160700 0x7f76dc070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.257+0000 7f76e0a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f76dc0b46c0 con 0x7f76dc0b3f70 2024-03-06T07:27:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.257+0000 7f76e0a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f76dc0b49a0 con 0x7f76dc15fc10 2024-03-06T07:27:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.257+0000 7f76e0a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76dc072c90 con 0x7f76dc160700 2024-03-06T07:27:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.257+0000 7f76d9400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f76dc0b3f70 0x7f76dc0ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57636/0 (socket says 172.21.15.86:57636) 2024-03-06T07:27:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.257+0000 7f76d9400640 1 -- 172.21.15.86:0/3642225390 learned_addr learned my addr 172.21.15.86:0/3642225390 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:43.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76da800640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc160700 0x7f76dc070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1105248587 0 0) 0x7f76dc0b46c0 con 0x7f76dc0b3f70 2024-03-06T07:27:43.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f76bc003520 con 0x7f76dc0b3f70 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1654850263 0 0) 0x7f76bc003520 con 0x7f76dc0b3f70 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc160700 msgr2=0x7f76dc070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc160700 0x7f76dc070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 >> v1:172.21.15.194:6789/0 conn(0x7f76dc15fc10 legacy=0x7f76dc15fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76dc09dd80 con 0x7f76dc0b3f70 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f76c40019f0 con 0x7f76dc0b3f70 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 >> v1:172.21.15.86:6789/0 conn(0x7f76dc0b3f70 legacy=0x7f76dc0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76bc0039e0 0x7f76bc003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76bc004800 0x7f76bc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76bc0051c0 0x7f76bc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76dc072c90 con 0x7f76bc0051c0 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76bc00c090 con 0x7f76bc004800 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76bc00c290 con 0x7f76bc0039e0 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.258+0000 7f76da800640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc160700 0x7f76dc070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76da800640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76bc0051c0 0x7f76bc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76d9400640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76bc004800 0x7f76bc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76d9e00640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76bc0039e0 0x7f76bc003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76da800640 1 -- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76bc0039e0 msgr2=0x7f76bc003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76da800640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76bc0039e0 0x7f76bc003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76da800640 1 -- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76bc004800 msgr2=0x7f76bc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76da800640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76bc004800 0x7f76bc004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76da800640 1 -- 172.21.15.86:0/3642225390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76bc003520 con 0x7f76bc0051c0 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76d9e00640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76bc0039e0 0x7f76bc003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.259+0000 7f76d9400640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76bc004800 0x7f76bc004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.260+0000 7f76da800640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76bc0051c0 0x7f76bc00b9c0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f76d4006fd0 tx=0x7f76bc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ff64f2aafb5f7e89 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.260+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76d402f050 con 0x7f76bc0051c0 2024-03-06T07:27:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.260+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76d4033630 con 0x7f76bc0051c0 2024-03-06T07:27:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.260+0000 7f76d8a00640 1 -- 172.21.15.86:0/3642225390 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76d400f040 con 0x7f76bc0051c0 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.261+0000 7f76e0a00640 1 -- 172.21.15.86:0/3642225390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76bc0051c0 msgr2=0x7f76bc00b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.261+0000 7f76e0a00640 1 --2- 172.21.15.86:0/3642225390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76bc0051c0 0x7f76bc00b9c0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f76d4006fd0 tx=0x7f76bc005830 comp rx=0 tx=0).stop 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.261+0000 7f76da800640 1 -- 172.21.15.86:0/3642225390 reap_dead start 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.261+0000 7f76e0a00640 1 -- 172.21.15.86:0/3642225390 shutdown_connections 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.261+0000 7f76e0a00640 1 -- 172.21.15.86:0/3642225390 >> 172.21.15.86:0/3642225390 conn(0x7f76dc0b1940 msgr2=0x7f76dc0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.261+0000 7f76e0a00640 1 -- 172.21.15.86:0/3642225390 shutdown_connections 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.261+0000 7f76e0a00640 1 -- 172.21.15.86:0/3642225390 wait complete. 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.263+0000 7f76e0a00640 1 Processor -- start 2024-03-06T07:27:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.263+0000 7f76e0a00640 1 -- start start 2024-03-06T07:27:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76e0a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76dc1f4180 0x7f76dc14ee90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76e0a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76dc1f4560 0x7f76dc14f3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76e0a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc140ec0 0x7f76dc1511b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76e0a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76dc15c460 con 0x7f76dc1f4560 2024-03-06T07:27:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76e0a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76dc15c2e0 con 0x7f76dc1f4180 2024-03-06T07:27:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76e0a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76dc070ff0 con 0x7f76dc140ec0 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76d9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76dc1f4560 0x7f76dc14f3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76d9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76dc1f4560 0x7f76dc14f3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36688/0 (socket says 172.21.15.86:36688) 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76d9400640 1 -- 172.21.15.86:0/61511021 learned_addr learned my addr 172.21.15.86:0/61511021 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.264+0000 7f76d9e00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76dc1f4180 0x7f76dc14ee90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76da800640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc140ec0 0x7f76dc1511b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d9400640 1 -- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc140ec0 msgr2=0x7f76dc1511b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d9400640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc140ec0 0x7f76dc1511b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d9400640 1 -- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76dc1f4180 msgr2=0x7f76dc14ee90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d9400640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76dc1f4180 0x7f76dc14ee90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d9400640 1 -- 172.21.15.86:0/61511021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76dc152200 con 0x7f76dc1f4560 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d9400640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76dc1f4560 0x7f76dc14f3d0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f76c400dfc0 tx=0x7f76c400b560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d2000640 1 -- 172.21.15.86:0/61511021 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76c4019070 con 0x7f76dc1f4560 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d2000640 1 -- 172.21.15.86:0/61511021 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76c40072d0 con 0x7f76dc1f4560 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76d2000640 1 -- 172.21.15.86:0/61511021 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76c40076c0 con 0x7f76dc1f4560 2024-03-06T07:27:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f76dc09b400 con 0x7f76dc1f4560 2024-03-06T07:27:43.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.265+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f76dc09b6a0 con 0x7f76dc1f4560 2024-03-06T07:27:43.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.267+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f76dc04f330 con 0x7f76dc1f4560 2024-03-06T07:27:43.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.268+0000 7f76d2000640 1 -- 172.21.15.86:0/61511021 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f76c401c070 con 0x7f76dc1f4560 2024-03-06T07:27:43.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.268+0000 7f76d2000640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f76b806b4d0 0x7f76b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.268+0000 7f76d2000640 1 -- 172.21.15.86:0/61511021 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f76c408a6b0 con 0x7f76dc1f4560 2024-03-06T07:27:43.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.269+0000 7f76d9e00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f76b806b4d0 0x7f76b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.269+0000 7f76d9e00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f76b806b4d0 0x7f76b806d990 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f76cc001750 tx=0x7f76cc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:43.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.273+0000 7f76d2000640 1 -- 172.21.15.86:0/61511021 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f76c4058f70 con 0x7f76dc1f4560 2024-03-06T07:27:43.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.469+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f76dc0b2d40 con 0x7f76dc1f4560 2024-03-06T07:27:43.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.470+0000 7f76d2000640 1 -- 172.21.15.86:0/61511021 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f76c405d010 con 0x7f76dc1f4560 2024-03-06T07:27:43.472 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f76b806b4d0 msgr2=0x7f76b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f76b806b4d0 0x7f76b806d990 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f76cc001750 tx=0x7f76cc009040 comp rx=0 tx=0).stop 2024-03-06T07:27:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76dc1f4560 msgr2=0x7f76dc14f3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76dc1f4560 0x7f76dc14f3d0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f76c400dfc0 tx=0x7f76c400b560 comp rx=0 tx=0).stop 2024-03-06T07:27:43.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 shutdown_connections 2024-03-06T07:27:43.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f76b806b4d0 0x7f76b806d990 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76dc140ec0 0x7f76dc1511b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76dc1f4560 0x7f76dc14f3d0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 --2- 172.21.15.86:0/61511021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76dc1f4180 0x7f76dc14ee90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 >> 172.21.15.86:0/61511021 conn(0x7f76dc0b1940 msgr2=0x7f76dc0ace90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:43.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 shutdown_connections 2024-03-06T07:27:43.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.473+0000 7f76e0a00640 1 -- 172.21.15.86:0/61511021 wait complete. 2024-03-06T07:27:43.507 INFO:tasks.cephfs_test_runner:Starting test: test_dir_deletion (tasks.cephfs.test_strays.TestStrays) That when deleting a bunch of dentries and the containing 2024-03-06T07:27:43.507 INFO:tasks.cephfs_test_runner:test_dir_deletion (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:27:43.508 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:43.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.723+0000 7f9776800640 1 Processor -- start 2024-03-06T07:27:43.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.723+0000 7f9776800640 1 -- start start 2024-03-06T07:27:43.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f9776800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f977015f9d0 0x7f977015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f9776800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f9770070f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f9776800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f977006cef0 con 0x7f9770144c00 2024-03-06T07:27:43.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f9776800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f977006d1d0 con 0x7f977015f9d0 2024-03-06T07:27:43.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f9776800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f977006d4b0 con 0x7f9770070b10 2024-03-06T07:27:43.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f977015f9d0 0x7f977015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976f400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9770144c00 0x7f977006e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57646/0 (socket says 172.21.15.86:57646) 2024-03-06T07:27:43.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976f400640 1 -- 172.21.15.86:0/1315141266 learned_addr learned my addr 172.21.15.86:0/1315141266 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:43.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976fe00640 1 --2- 172.21.15.86:0/1315141266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f9770070f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976ea00640 1 -- 172.21.15.86:0/1315141266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 msgr2=0x7f9770070f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:43.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976ea00640 1 --2- 172.21.15.86:0/1315141266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f9770070f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976ea00640 1 -- 172.21.15.86:0/1315141266 >> v1:172.21.15.86:6789/0 conn(0x7f9770144c00 legacy=0x7f977006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976ea00640 1 -- 172.21.15.86:0/1315141266 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f977006d790 con 0x7f977015f9d0 2024-03-06T07:27:43.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976e000640 1 -- 172.21.15.86:0/1315141266 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3190735660 0 0) 0x7f977006cef0 con 0x7f9770144c00 2024-03-06T07:27:43.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.725+0000 7f976fe00640 1 --2- 172.21.15.86:0/1315141266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f9770070f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:43.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.727+0000 7f976ea00640 1 --2- 172.21.15.86:0/1315141266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f977015fe60 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f975c0097a0 tx=0x7f975c02b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=35c4e166ca41973a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:43.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.727+0000 7f976e000640 1 -- 172.21.15.86:0/1315141266 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f975c035020 con 0x7f977015f9d0 2024-03-06T07:27:43.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.727+0000 7f976e000640 1 -- 172.21.15.86:0/1315141266 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f975c031810 con 0x7f977015f9d0 2024-03-06T07:27:43.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.727+0000 7f976e000640 1 -- 172.21.15.86:0/1315141266 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f975c031be0 con 0x7f977015f9d0 2024-03-06T07:27:43.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 -- 172.21.15.86:0/1315141266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 msgr2=0x7f977015fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 --2- 172.21.15.86:0/1315141266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f977015fe60 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f975c0097a0 tx=0x7f975c02b8b0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 -- 172.21.15.86:0/1315141266 shutdown_connections 2024-03-06T07:27:43.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 --2- 172.21.15.86:0/1315141266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f9770070f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 --2- 172.21.15.86:0/1315141266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f977015fe60 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 -- 172.21.15.86:0/1315141266 >> 172.21.15.86:0/1315141266 conn(0x7f97700b1730 msgr2=0x7f97700b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:43.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 -- 172.21.15.86:0/1315141266 shutdown_connections 2024-03-06T07:27:43.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.728+0000 7f9776800640 1 -- 172.21.15.86:0/1315141266 wait complete. 2024-03-06T07:27:43.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.729+0000 7f9776800640 1 Processor -- start 2024-03-06T07:27:43.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.730+0000 7f9776800640 1 -- start start 2024-03-06T07:27:43.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.730+0000 7f9776800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f97700aea90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.730+0000 7f9776800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9770144c00 0x7f97700ae150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f9776800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f9770141770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f9776800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f977006ccd0 con 0x7f977015f9d0 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f9776800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f977006cb50 con 0x7f9770144c00 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f9776800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f977006df90 con 0x7f9770070b10 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f9770141770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f9770141770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36716/0 (socket says 172.21.15.86:36716) 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 -- 172.21.15.86:0/3781929019 learned_addr learned my addr 172.21.15.86:0/3781929019 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976f400640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f97700aea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976ea00640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9770144c00 0x7f97700ae150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 -- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 msgr2=0x7f97700aea90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f97700aea90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 -- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9770144c00 msgr2=0x7f97700ae150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9770144c00 0x7f97700ae150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976fe00640 1 -- 172.21.15.86:0/3781929019 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97701e56f0 con 0x7f977015f9d0 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.731+0000 7f976f400640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f97700aea90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.732+0000 7f976fe00640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f9770141770 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f976800cd80 tx=0x7f976800a650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:43.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.732+0000 7f9767400640 1 -- 172.21.15.86:0/3781929019 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f976800f680 con 0x7f977015f9d0 2024-03-06T07:27:43.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.732+0000 7f9767400640 1 -- 172.21.15.86:0/3781929019 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f976800f840 con 0x7f977015f9d0 2024-03-06T07:27:43.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.732+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97701e59d0 con 0x7f977015f9d0 2024-03-06T07:27:43.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.732+0000 7f9767400640 1 -- 172.21.15.86:0/3781929019 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f976800fc30 con 0x7f977015f9d0 2024-03-06T07:27:43.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.732+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f977014a5a0 con 0x7f977015f9d0 2024-03-06T07:27:43.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.733+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f977004f330 con 0x7f977015f9d0 2024-03-06T07:27:43.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.734+0000 7f9767400640 1 -- 172.21.15.86:0/3781929019 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f97680213f0 con 0x7f977015f9d0 2024-03-06T07:27:43.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.735+0000 7f9767400640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f975006b4d0 0x7f975006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:43.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.736+0000 7f9767400640 1 -- 172.21.15.86:0/3781929019 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f976808b0c0 con 0x7f977015f9d0 2024-03-06T07:27:43.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.736+0000 7f976f400640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f975006b4d0 0x7f975006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:43.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.736+0000 7f976f400640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f975006b4d0 0x7f975006d990 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f9760000f60 tx=0x7f9760009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:43.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.740+0000 7f9767400640 1 -- 172.21.15.86:0/3781929019 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9768014030 con 0x7f977015f9d0 2024-03-06T07:27:43.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.938+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f97700cd4c0 con 0x7f977015f9d0 2024-03-06T07:27:43.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.939+0000 7f9767400640 1 -- 172.21.15.86:0/3781929019 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f9768059ea0 con 0x7f977015f9d0 2024-03-06T07:27:43.941 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:43.941 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f975006b4d0 msgr2=0x7f975006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f975006b4d0 0x7f975006d990 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f9760000f60 tx=0x7f9760009040 comp rx=0 tx=0).stop 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 msgr2=0x7f9770141770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f9770141770 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f976800cd80 tx=0x7f976800a650 comp rx=0 tx=0).stop 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 shutdown_connections 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f975006b4d0 0x7f975006d990 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f977015f9d0 0x7f9770141770 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9770144c00 0x7f97700ae150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 --2- 172.21.15.86:0/3781929019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9770070b10 0x7f97700aea90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:43.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 >> 172.21.15.86:0/3781929019 conn(0x7f97700b1730 msgr2=0x7f977006e550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:43.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 shutdown_connections 2024-03-06T07:27:43.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:43.942+0000 7f9776800640 1 -- 172.21.15.86:0/3781929019 wait complete. 2024-03-06T07:27:43.963 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:43.964 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:44.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.175+0000 7f0cf8600640 1 Processor -- start 2024-03-06T07:27:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.175+0000 7f0cf8600640 1 -- start start 2024-03-06T07:27:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cf8600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0cf00cda90 0x7f0cf00727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cf8600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0cf00719e0 con 0x7f0cf00b3df0 2024-03-06T07:27:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cf8600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0cf0071cc0 con 0x7f0cf00b3650 2024-03-06T07:27:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cf8600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0cf0072310 con 0x7f0cf00cda90 2024-03-06T07:27:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cf5c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0cf00b3df0 0x7f0cf00b41d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57648/0 (socket says 172.21.15.86:57648) 2024-03-06T07:27:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cf5c00640 1 -- 172.21.15.86:0/19324795 learned_addr learned my addr 172.21.15.86:0/19324795 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cf6600640 1 --2- 172.21.15.86:0/19324795 >> v2:172.21.15.194:3300/0 conn(0x7f0cf00cda90 0x7f0cf00727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.177+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3414013766 0 0) 0x7f0cf00719e0 con 0x7f0cf00b3df0 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0cd8005730 con 0x7f0cf00b3df0 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 731307931 0 0) 0x7f0cd8005730 con 0x7f0cf00b3df0 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 >> v2:172.21.15.194:3300/0 conn(0x7f0cf00cda90 msgr2=0x7f0cf00727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 --2- 172.21.15.86:0/19324795 >> v2:172.21.15.194:3300/0 conn(0x7f0cf00cda90 0x7f0cf00727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 >> v1:172.21.15.194:6789/0 conn(0x7f0cf00b3650 legacy=0x7f0cf006c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cf0071fa0 con 0x7f0cf00b3df0 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f0ce4001670 con 0x7f0cf00b3df0 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 >> v1:172.21.15.86:6789/0 conn(0x7f0cf00b3df0 legacy=0x7f0cf00b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd8005bc0 0x7f0cd8005fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cd8006a10 0x7f0cd8006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cd80073d0 0x7f0cd800dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cf0072310 con 0x7f0cd80073d0 2024-03-06T07:27:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cd800e220 con 0x7f0cd8006a10 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0cd800e420 con 0x7f0cd8005bc0 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.178+0000 7f0cf6600640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cd80073d0 0x7f0cd800dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf5200640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cd8006a10 0x7f0cd8006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf5c00640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd8005bc0 0x7f0cd8005fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf6600640 1 -- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd8005bc0 msgr2=0x7f0cd8005fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf6600640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd8005bc0 0x7f0cd8005fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf6600640 1 -- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cd8006a10 msgr2=0x7f0cd8006e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf6600640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cd8006a10 0x7f0cd8006e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf6600640 1 -- 172.21.15.86:0/19324795 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cd8005730 con 0x7f0cd80073d0 2024-03-06T07:27:44.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.179+0000 7f0cf5200640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cd8006a10 0x7f0cd8006e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.180+0000 7f0cf6600640 1 -- 172.21.15.86:0/19324795 reap_dead start 2024-03-06T07:27:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.180+0000 7f0cf6600640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cd80073d0 0x7f0cd800dbd0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f0ce8009fc0 tx=0x7f0cd8007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aa03acc4067af83d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.180+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ce8009c10 con 0x7f0cd80073d0 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.180+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ce802cc60 con 0x7f0cd80073d0 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.180+0000 7f0cefe00640 1 -- 172.21.15.86:0/19324795 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ce8003bd0 con 0x7f0cd80073d0 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.181+0000 7f0cf8600640 1 -- 172.21.15.86:0/19324795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cd80073d0 msgr2=0x7f0cd800dbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.181+0000 7f0cf8600640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cd80073d0 0x7f0cd800dbd0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f0ce8009fc0 tx=0x7f0cd8007a40 comp rx=0 tx=0).stop 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.181+0000 7f0cf8600640 1 -- 172.21.15.86:0/19324795 shutdown_connections 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.181+0000 7f0cf8600640 1 --2- 172.21.15.86:0/19324795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cd80073d0 0x7f0cd800dbd0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.181+0000 7f0cf8600640 1 -- 172.21.15.86:0/19324795 >> 172.21.15.86:0/19324795 conn(0x7f0cf00b1940 msgr2=0x7f0cf00b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.181+0000 7f0cf8600640 1 -- 172.21.15.86:0/19324795 shutdown_connections 2024-03-06T07:27:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.181+0000 7f0cf8600640 1 -- 172.21.15.86:0/19324795 wait complete. 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.183+0000 7f0cf8600640 1 Processor -- start 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.183+0000 7f0cf8600640 1 -- start start 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf8600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd80073d0 0x7f0cf015de10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf8600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cf014f4e0 0x7f0cf014fda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf8600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cf01502e0 0x7f0cf0150760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf8600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cf015b3c0 con 0x7f0cf01502e0 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf8600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cf015b240 con 0x7f0cf014f4e0 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf8600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0cf00cf510 con 0x7f0cd80073d0 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cf01502e0 0x7f0cf0150760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cf01502e0 0x7f0cf0150760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36742/0 (socket says 172.21.15.86:36742) 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 -- 172.21.15.86:0/964441513 learned_addr learned my addr 172.21.15.86:0/964441513 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf5200640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cf014f4e0 0x7f0cf014fda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf5c00640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd80073d0 0x7f0cf015de10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 -- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd80073d0 msgr2=0x7f0cf015de10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd80073d0 0x7f0cf015de10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 -- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cf014f4e0 msgr2=0x7f0cf014fda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cf014f4e0 0x7f0cf014fda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf6600640 1 -- 172.21.15.86:0/964441513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cf0152130 con 0x7f0cf01502e0 2024-03-06T07:27:44.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.184+0000 7f0cf5200640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cf014f4e0 0x7f0cf014fda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.185+0000 7f0cf6600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cf01502e0 0x7f0cf0150760 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f0ce802c6d0 tx=0x7f0ce8009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.185+0000 7f0ced600640 1 -- 172.21.15.86:0/964441513 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ce8009c10 con 0x7f0cf01502e0 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.185+0000 7f0ced600640 1 -- 172.21.15.86:0/964441513 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ce8044070 con 0x7f0cf01502e0 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.185+0000 7f0ced600640 1 -- 172.21.15.86:0/964441513 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ce8003b90 con 0x7f0cf01502e0 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.185+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0cf0152410 con 0x7f0cf01502e0 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.185+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0cf01529d0 con 0x7f0cf01502e0 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.186+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0cf004f330 con 0x7f0cf01502e0 2024-03-06T07:27:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.187+0000 7f0ced600640 1 -- 172.21.15.86:0/964441513 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0ce8005d10 con 0x7f0cf01502e0 2024-03-06T07:27:44.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.188+0000 7f0ced600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cd006b4d0 0x7f0cd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.188+0000 7f0ced600640 1 -- 172.21.15.86:0/964441513 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0ce80aca60 con 0x7f0cf01502e0 2024-03-06T07:27:44.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.188+0000 7f0cf5c00640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cd006b4d0 0x7f0cd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.189+0000 7f0cf5c00640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cd006b4d0 0x7f0cd006d990 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f0ce4001eb0 tx=0x7f0ce4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:44.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.192+0000 7f0ced600640 1 -- 172.21.15.86:0/964441513 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0ce80333a0 con 0x7f0cf01502e0 2024-03-06T07:27:44.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.381+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f0cf006e740 con 0x7f0cf01502e0 2024-03-06T07:27:44.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.382+0000 7f0ced600640 1 -- 172.21.15.86:0/964441513 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f0ce807b960 con 0x7f0cf01502e0 2024-03-06T07:27:44.383 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:44.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.384+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cd006b4d0 msgr2=0x7f0cd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.384+0000 7f0cf8600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cd006b4d0 0x7f0cd006d990 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f0ce4001eb0 tx=0x7f0ce4009040 comp rx=0 tx=0).stop 2024-03-06T07:27:44.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.384+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cf01502e0 msgr2=0x7f0cf0150760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.384+0000 7f0cf8600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cf01502e0 0x7f0cf0150760 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f0ce802c6d0 tx=0x7f0ce8009750 comp rx=0 tx=0).stop 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 shutdown_connections 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cd006b4d0 0x7f0cd006d990 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0cf01502e0 0x7f0cf0150760 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0cf014f4e0 0x7f0cf014fda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 --2- 172.21.15.86:0/964441513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0cd80073d0 0x7f0cf015de10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 >> 172.21.15.86:0/964441513 conn(0x7f0cf00b1940 msgr2=0x7f0cf00b41d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 shutdown_connections 2024-03-06T07:27:44.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.385+0000 7f0cf8600640 1 -- 172.21.15.86:0/964441513 wait complete. 2024-03-06T07:27:44.403 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:44.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.607+0000 7f5529e00640 1 Processor -- start 2024-03-06T07:27:44.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.608+0000 7f5529e00640 1 -- start start 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.609+0000 7f5529e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f552406d960 0x7f552415f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.609+0000 7f5529e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f552406e2f0 0x7f55240cda90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.609+0000 7f5529e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f552406cba0 con 0x7f55240b2c40 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.609+0000 7f5529e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f552415f700 con 0x7f552406d960 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.609+0000 7f5529e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f552406d4d0 con 0x7f552406e2f0 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.609+0000 7f5522a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f55240b2c40 0x7f55240b3020 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57664/0 (socket says 172.21.15.86:57664) 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5522a00640 1 -- 172.21.15.86:0/3655209685 learned_addr learned my addr 172.21.15.86:0/3655209685 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5522000640 1 --2- 172.21.15.86:0/3655209685 >> v2:172.21.15.86:3300/0 conn(0x7f552406d960 0x7f552415f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5523400640 1 --2- 172.21.15.86:0/3655209685 >> v2:172.21.15.194:3300/0 conn(0x7f552406e2f0 0x7f55240cda90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5521600640 1 -- 172.21.15.86:0/3655209685 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2131266395 0 0) 0x7f552406cba0 con 0x7f55240b2c40 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5521600640 1 -- 172.21.15.86:0/3655209685 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5508003520 con 0x7f55240b2c40 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5522000640 1 -- 172.21.15.86:0/3655209685 >> v2:172.21.15.194:3300/0 conn(0x7f552406e2f0 msgr2=0x7f55240cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5522000640 1 --2- 172.21.15.86:0/3655209685 >> v2:172.21.15.194:3300/0 conn(0x7f552406e2f0 0x7f55240cda90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5522000640 1 -- 172.21.15.86:0/3655209685 >> v1:172.21.15.86:6789/0 conn(0x7f55240b2c40 legacy=0x7f55240b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.610+0000 7f5522000640 1 -- 172.21.15.86:0/3655209685 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f552406ce80 con 0x7f552406d960 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.611+0000 7f5522000640 1 --2- 172.21.15.86:0/3655209685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f552406d960 0x7f552415f1c0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f5524074910 tx=0x7f551002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=284a2d5d9aa8a227 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5521600640 1 -- 172.21.15.86:0/3655209685 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5510035020 con 0x7f552406d960 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5521600640 1 -- 172.21.15.86:0/3655209685 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5510031830 con 0x7f552406d960 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5521600640 1 -- 172.21.15.86:0/3655209685 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5510031c00 con 0x7f552406d960 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 -- 172.21.15.86:0/3655209685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f552406d960 msgr2=0x7f552415f1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 --2- 172.21.15.86:0/3655209685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f552406d960 0x7f552415f1c0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f5524074910 tx=0x7f551002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 -- 172.21.15.86:0/3655209685 shutdown_connections 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 --2- 172.21.15.86:0/3655209685 >> v2:172.21.15.194:3300/0 conn(0x7f552406e2f0 0x7f55240cda90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 --2- 172.21.15.86:0/3655209685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f552406d960 0x7f552415f1c0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 -- 172.21.15.86:0/3655209685 >> 172.21.15.86:0/3655209685 conn(0x7f55240b1940 msgr2=0x7f55240b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 -- 172.21.15.86:0/3655209685 shutdown_connections 2024-03-06T07:27:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.612+0000 7f5529e00640 1 -- 172.21.15.86:0/3655209685 wait complete. 2024-03-06T07:27:44.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.614+0000 7f5529e00640 1 Processor -- start 2024-03-06T07:27:44.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.614+0000 7f5529e00640 1 -- start start 2024-03-06T07:27:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5529e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f552406d960 0x7f552415b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5529e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f552406e2f0 0x7f5524152cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5529e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55240b2c40 0x7f55241531f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5529e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f552406e900 con 0x7f55240b2c40 2024-03-06T07:27:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5529e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f552406e780 con 0x7f552406e2f0 2024-03-06T07:27:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5529e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55240cdfd0 con 0x7f552406d960 2024-03-06T07:27:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5523400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55240b2c40 0x7f55241531f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5523400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55240b2c40 0x7f55241531f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36772/0 (socket says 172.21.15.86:36772) 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.615+0000 7f5523400640 1 -- 172.21.15.86:0/3559598126 learned_addr learned my addr 172.21.15.86:0/3559598126 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5523400640 1 -- 172.21.15.86:0/3559598126 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f552406d960 msgr2=0x7f552415b6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5523400640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f552406d960 0x7f552415b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5523400640 1 -- 172.21.15.86:0/3559598126 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f552406e2f0 msgr2=0x7f5524152cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5523400640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f552406e2f0 0x7f5524152cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5523400640 1 -- 172.21.15.86:0/3559598126 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5524161780 con 0x7f55240b2c40 2024-03-06T07:27:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5523400640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55240b2c40 0x7f55241531f0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f551c00efc0 tx=0x7f551c00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f551b400640 1 -- 172.21.15.86:0/3559598126 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f551c018020 con 0x7f55240b2c40 2024-03-06T07:27:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f551b400640 1 -- 172.21.15.86:0/3559598126 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f551c014480 con 0x7f55240b2c40 2024-03-06T07:27:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5524161a20 con 0x7f55240b2c40 2024-03-06T07:27:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f551b400640 1 -- 172.21.15.86:0/3559598126 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f551c014870 con 0x7f55240b2c40 2024-03-06T07:27:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.616+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5524161fe0 con 0x7f55240b2c40 2024-03-06T07:27:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.617+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f552404f330 con 0x7f55240b2c40 2024-03-06T07:27:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.618+0000 7f551b400640 1 -- 172.21.15.86:0/3559598126 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f551c005d10 con 0x7f55240b2c40 2024-03-06T07:27:44.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.619+0000 7f551b400640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f550406b4d0 0x7f550406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:44.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.619+0000 7f551b400640 1 -- 172.21.15.86:0/3559598126 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f551c08acf0 con 0x7f55240b2c40 2024-03-06T07:27:44.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.619+0000 7f5522a00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f550406b4d0 0x7f550406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:44.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.620+0000 7f5522a00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f550406b4d0 0x7f550406d990 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f5514000f60 tx=0x7f55140078f0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:44.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.624+0000 7f551b400640 1 -- 172.21.15.86:0/3559598126 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f551c08f070 con 0x7f55240b2c40 2024-03-06T07:27:44.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.817+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f55240b2530 con 0x7f55240b2c40 2024-03-06T07:27:44.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.818+0000 7f551b400640 1 -- 172.21.15.86:0/3559598126 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7f551c022070 con 0x7f55240b2c40 2024-03-06T07:27:44.820 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f550406b4d0 msgr2=0x7f550406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f550406b4d0 0x7f550406d990 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f5514000f60 tx=0x7f55140078f0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55240b2c40 msgr2=0x7f55241531f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55240b2c40 0x7f55241531f0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f551c00efc0 tx=0x7f551c00c490 comp rx=0 tx=0).stop 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 shutdown_connections 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f550406b4d0 0x7f550406d990 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55240b2c40 0x7f55241531f0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f552406e2f0 0x7f5524152cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 --2- 172.21.15.86:0/3559598126 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f552406d960 0x7f552415b6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:44.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 >> 172.21.15.86:0/3559598126 conn(0x7f55240b1940 msgr2=0x7f5524070c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:44.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 shutdown_connections 2024-03-06T07:27:44.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:44.821+0000 7f5529e00640 1 -- 172.21.15.86:0/3559598126 wait complete. 2024-03-06T07:27:44.841 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:45.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.041+0000 7f777c000640 1 Processor -- start 2024-03-06T07:27:45.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.041+0000 7f777c000640 1 -- start start 2024-03-06T07:27:45.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.043+0000 7f777c000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77740b3f70 0x7f7774160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.043+0000 7f777c000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7774070ff0 0x7f77740ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.043+0000 7f777c000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f77740b46c0 con 0x7f777415fc10 2024-03-06T07:27:45.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.043+0000 7f777c000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7774071480 con 0x7f7774070ff0 2024-03-06T07:27:45.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.043+0000 7f777c000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f77740b49a0 con 0x7f77740b3f70 2024-03-06T07:27:45.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7779600640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f777415fc10 0x7f777415fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:41632/0 (socket says 172.21.15.86:41632) 2024-03-06T07:27:45.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7779600640 1 -- 172.21.15.86:0/3039145776 learned_addr learned my addr 172.21.15.86:0/3039145776 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:45.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f777a000640 1 --2- 172.21.15.86:0/3039145776 >> v2:172.21.15.194:3300/0 conn(0x7f7774070ff0 0x7f77740ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7778c00640 1 --2- 172.21.15.86:0/3039145776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77740b3f70 0x7f7774160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7778c00640 1 -- 172.21.15.86:0/3039145776 >> v2:172.21.15.194:3300/0 conn(0x7f7774070ff0 msgr2=0x7f77740ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:45.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7778c00640 1 --2- 172.21.15.86:0/3039145776 >> v2:172.21.15.194:3300/0 conn(0x7f7774070ff0 0x7f77740ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7778c00640 1 -- 172.21.15.86:0/3039145776 >> v1:172.21.15.194:6789/0 conn(0x7f777415fc10 legacy=0x7f777415fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7778c00640 1 -- 172.21.15.86:0/3039145776 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7774072c90 con 0x7f77740b3f70 2024-03-06T07:27:45.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.044+0000 7f7773e00640 1 -- 172.21.15.86:0/3039145776 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1304855356 0 0) 0x7f77740b46c0 con 0x7f777415fc10 2024-03-06T07:27:45.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.046+0000 7f7778c00640 1 --2- 172.21.15.86:0/3039145776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77740b3f70 0x7f7774160700 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f77680097a0 tx=0x7f776802b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=874656209869dd32 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:45.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.046+0000 7f7773e00640 1 -- 172.21.15.86:0/3039145776 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7768035020 con 0x7f77740b3f70 2024-03-06T07:27:45.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.046+0000 7f7773e00640 1 -- 172.21.15.86:0/3039145776 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7768031830 con 0x7f77740b3f70 2024-03-06T07:27:45.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.046+0000 7f7773e00640 1 -- 172.21.15.86:0/3039145776 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7768031c00 con 0x7f77740b3f70 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 -- 172.21.15.86:0/3039145776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77740b3f70 msgr2=0x7f7774160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 --2- 172.21.15.86:0/3039145776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77740b3f70 0x7f7774160700 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f77680097a0 tx=0x7f776802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 -- 172.21.15.86:0/3039145776 shutdown_connections 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 --2- 172.21.15.86:0/3039145776 >> v2:172.21.15.194:3300/0 conn(0x7f7774070ff0 0x7f77740ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 --2- 172.21.15.86:0/3039145776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f77740b3f70 0x7f7774160700 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 -- 172.21.15.86:0/3039145776 >> 172.21.15.86:0/3039145776 conn(0x7f77740b1940 msgr2=0x7f77740b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 -- 172.21.15.86:0/3039145776 shutdown_connections 2024-03-06T07:27:45.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.047+0000 7f777c000640 1 -- 172.21.15.86:0/3039145776 wait complete. 2024-03-06T07:27:45.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.049+0000 7f777c000640 1 Processor -- start 2024-03-06T07:27:45.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.049+0000 7f777c000640 1 -- start start 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777c000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7774070ff0 0x7f77741f7270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777c000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77740b3f70 0x7f777409d390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777c000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f777415fc10 0x7f7774146a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777c000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7774072a30 con 0x7f777415fc10 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777c000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77740728b0 con 0x7f77740b3f70 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777c000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f777415c2e0 con 0x7f7774070ff0 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f777415fc10 0x7f7774146a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f777415fc10 0x7f7774146a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36778/0 (socket says 172.21.15.86:36778) 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777a000640 1 -- 172.21.15.86:0/2062648898 learned_addr learned my addr 172.21.15.86:0/2062648898 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f7778c00640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77740b3f70 0x7f777409d390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f7779600640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7774070ff0 0x7f77741f7270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777a000640 1 -- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7774070ff0 msgr2=0x7f77741f7270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777a000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7774070ff0 0x7f77741f7270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777a000640 1 -- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77740b3f70 msgr2=0x7f777409d390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.050+0000 7f777a000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77740b3f70 0x7f777409d390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f777a000640 1 -- 172.21.15.86:0/2062648898 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7774146fc0 con 0x7f777415fc10 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f7779600640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7774070ff0 0x7f77741f7270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:45.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f777a000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f777415fc10 0x7f7774146a80 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f776c00efc0 tx=0x7f776c00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:45.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f7771600640 1 -- 172.21.15.86:0/2062648898 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f776c018020 con 0x7f777415fc10 2024-03-06T07:27:45.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f7771600640 1 -- 172.21.15.86:0/2062648898 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f776c014480 con 0x7f777415fc10 2024-03-06T07:27:45.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f77741f78c0 con 0x7f777415fc10 2024-03-06T07:27:45.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f7771600640 1 -- 172.21.15.86:0/2062648898 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f776c014870 con 0x7f777415fc10 2024-03-06T07:27:45.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.051+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f77741f7e80 con 0x7f777415fc10 2024-03-06T07:27:45.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.052+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f777404f330 con 0x7f777415fc10 2024-03-06T07:27:45.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.053+0000 7f7771600640 1 -- 172.21.15.86:0/2062648898 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f776c005d10 con 0x7f777415fc10 2024-03-06T07:27:45.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.054+0000 7f7771600640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775406b500 0x7f775406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.054+0000 7f7771600640 1 -- 172.21.15.86:0/2062648898 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f776c08ac10 con 0x7f777415fc10 2024-03-06T07:27:45.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.054+0000 7f7779600640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775406b500 0x7f775406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.055+0000 7f7779600640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775406b500 0x7f775406d9c0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f7760000f60 tx=0x7f7760009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:45.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.058+0000 7f7771600640 1 -- 172.21.15.86:0/2062648898 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f776c022070 con 0x7f777415fc10 2024-03-06T07:27:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.240+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f777406ef30 con 0x7f777415fc10 2024-03-06T07:27:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.241+0000 7f7771600640 1 -- 172.21.15.86:0/2062648898 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f776c010040 con 0x7f777415fc10 2024-03-06T07:27:45.243 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775406b500 msgr2=0x7f775406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775406b500 0x7f775406d9c0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f7760000f60 tx=0x7f7760009040 comp rx=0 tx=0).stop 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f777415fc10 msgr2=0x7f7774146a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f777415fc10 0x7f7774146a80 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f776c00efc0 tx=0x7f776c00c490 comp rx=0 tx=0).stop 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 shutdown_connections 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775406b500 0x7f775406d9c0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f777415fc10 0x7f7774146a80 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77740b3f70 0x7f777409d390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 --2- 172.21.15.86:0/2062648898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7774070ff0 0x7f77741f7270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 >> 172.21.15.86:0/2062648898 conn(0x7f77740b1940 msgr2=0x7f77740acd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 shutdown_connections 2024-03-06T07:27:45.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.244+0000 7f777c000640 1 -- 172.21.15.86:0/2062648898 wait complete. 2024-03-06T07:27:45.262 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dir_deletion' 2024-03-06T07:27:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.464+0000 7f1d14a00640 1 Processor -- start 2024-03-06T07:27:45.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.464+0000 7f1d14a00640 1 -- start start 2024-03-06T07:27:45.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d14a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d14a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1d100cc9d0 0x7f1d1015f830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d14a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1d1015c240 con 0x7f1d100aebf0 2024-03-06T07:27:45.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d14a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1d1015ce30 con 0x7f1d100cc9d0 2024-03-06T07:27:45.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d14a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1d1015c500 con 0x7f1d101561f0 2024-03-06T07:27:45.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d0f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d0fe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1d100cc9d0 0x7f1d1015f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d0f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100ae6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:60684/0 (socket says 172.21.15.86:60684) 2024-03-06T07:27:45.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.466+0000 7f1d0f400640 1 -- 172.21.15.86:0/3442308526 learned_addr learned my addr 172.21.15.86:0/3442308526 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:45.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.467+0000 7f1d0f400640 1 -- 172.21.15.86:0/3442308526 >> v2:172.21.15.194:3300/0 conn(0x7f1d100cc9d0 msgr2=0x7f1d1015f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:45.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.467+0000 7f1d0f400640 1 --2- 172.21.15.86:0/3442308526 >> v2:172.21.15.194:3300/0 conn(0x7f1d100cc9d0 0x7f1d1015f830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.467+0000 7f1d0f400640 1 -- 172.21.15.86:0/3442308526 >> v1:172.21.15.194:6789/0 conn(0x7f1d100aebf0 legacy=0x7f1d100cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.467+0000 7f1d0f400640 1 -- 172.21.15.86:0/3442308526 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d1015c7e0 con 0x7f1d101561f0 2024-03-06T07:27:45.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.468+0000 7f1d0f400640 1 --2- 172.21.15.86:0/3442308526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100ae6b0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1d00009e50 tx=0x7f1d0002bf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f05179372899c32 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:45.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.469+0000 7f1d0ea00640 1 -- 172.21.15.86:0/3442308526 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d0002cb00 con 0x7f1d101561f0 2024-03-06T07:27:45.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.469+0000 7f1d0ea00640 1 -- 172.21.15.86:0/3442308526 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1d0002cca0 con 0x7f1d101561f0 2024-03-06T07:27:45.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.469+0000 7f1d0ea00640 1 -- 172.21.15.86:0/3442308526 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d00034610 con 0x7f1d101561f0 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 -- 172.21.15.86:0/3442308526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 msgr2=0x7f1d100ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3442308526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100ae6b0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1d00009e50 tx=0x7f1d0002bf60 comp rx=0 tx=0).stop 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 -- 172.21.15.86:0/3442308526 shutdown_connections 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3442308526 >> v2:172.21.15.194:3300/0 conn(0x7f1d100cc9d0 0x7f1d1015f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3442308526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100ae6b0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 -- 172.21.15.86:0/3442308526 >> 172.21.15.86:0/3442308526 conn(0x7f1d1006afc0 msgr2=0x7f1d1006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 -- 172.21.15.86:0/3442308526 shutdown_connections 2024-03-06T07:27:45.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.470+0000 7f1d14a00640 1 -- 172.21.15.86:0/3442308526 wait complete. 2024-03-06T07:27:45.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.472+0000 7f1d14a00640 1 Processor -- start 2024-03-06T07:27:45.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.473+0000 7f1d14a00640 1 -- start start 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d14a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1d100aebf0 0x7f1d101492c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d14a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1d100cc9d0 0x7f1d100ad860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d14a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100adda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d14a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d10156750 con 0x7f1d100aebf0 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d14a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d101565d0 con 0x7f1d100cc9d0 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d14a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1d1015fd70 con 0x7f1d101561f0 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1d100aebf0 0x7f1d101492c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1d100aebf0 0x7f1d101492c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36788/0 (socket says 172.21.15.86:36788) 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 -- 172.21.15.86:0/3850985283 learned_addr learned my addr 172.21.15.86:0/3850985283 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d06a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1d100cc9d0 0x7f1d100ad860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0fe00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100adda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 -- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 msgr2=0x7f1d100adda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100adda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 -- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1d100cc9d0 msgr2=0x7f1d100ad860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1d100cc9d0 0x7f1d100ad860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.474+0000 7f1d0f400640 1 -- 172.21.15.86:0/3850985283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d101577f0 con 0x7f1d100aebf0 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.475+0000 7f1d06a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1d100cc9d0 0x7f1d100ad860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.475+0000 7f1d0f400640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1d100aebf0 0x7f1d101492c0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f1d00000f80 tx=0x7f1d00004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.475+0000 7f1d07e00640 1 -- 172.21.15.86:0/3850985283 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d00043020 con 0x7f1d100aebf0 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.475+0000 7f1d07e00640 1 -- 172.21.15.86:0/3850985283 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1d0002ce50 con 0x7f1d100aebf0 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.475+0000 7f1d07e00640 1 -- 172.21.15.86:0/3850985283 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d00009510 con 0x7f1d100aebf0 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.475+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d10157ad0 con 0x7f1d100aebf0 2024-03-06T07:27:45.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.475+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d10158090 con 0x7f1d100aebf0 2024-03-06T07:27:45.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.476+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1d1006b5d0 con 0x7f1d100aebf0 2024-03-06T07:27:45.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.478+0000 7f1d07e00640 1 -- 172.21.15.86:0/3850985283 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1d00046020 con 0x7f1d100aebf0 2024-03-06T07:27:45.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.478+0000 7f1d07e00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cec06b4d0 0x7f1cec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:45.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.479+0000 7f1d07e00640 1 -- 172.21.15.86:0/3850985283 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1d000ac800 con 0x7f1d100aebf0 2024-03-06T07:27:45.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.479+0000 7f1d06a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cec06b4d0 0x7f1cec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:45.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.479+0000 7f1d06a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cec06b4d0 0x7f1cec06d990 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f1d100cde60 tx=0x7f1cf8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:45.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.483+0000 7f1d07e00640 1 -- 172.21.15.86:0/3850985283 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1d1006b5d0 con 0x7f1d100aebf0 2024-03-06T07:27:45.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.741+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]} v 0) v1 -- 0x7f1d1004f330 con 0x7f1d100aebf0 2024-03-06T07:27:45.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.938+0000 7f1d07e00640 1 -- 172.21.15.86:0/3850985283 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]}]=0 v55) v1 ==== 135+0+0 (secure 0 0 0) 0x7f1d0004d030 con 0x7f1d100aebf0 2024-03-06T07:27:45.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cec06b4d0 msgr2=0x7f1cec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cec06b4d0 0x7f1cec06d990 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f1d100cde60 tx=0x7f1cf8009040 comp rx=0 tx=0).stop 2024-03-06T07:27:45.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1d100aebf0 msgr2=0x7f1d101492c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1d100aebf0 0x7f1d101492c0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f1d00000f80 tx=0x7f1d00004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 shutdown_connections 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cec06b4d0 0x7f1cec06d990 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1d101561f0 0x7f1d100adda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1d100cc9d0 0x7f1d100ad860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 --2- 172.21.15.86:0/3850985283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1d100aebf0 0x7f1d101492c0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 >> 172.21.15.86:0/3850985283 conn(0x7f1d1006afc0 msgr2=0x7f1d100725e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 shutdown_connections 2024-03-06T07:27:45.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:45.941+0000 7f1d14a00640 1 -- 172.21.15.86:0/3850985283 wait complete. 2024-03-06T07:27:45.962 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:27:46.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.164+0000 7f6669800640 1 Processor -- start 2024-03-06T07:27:46.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.164+0000 7f6669800640 1 -- start start 2024-03-06T07:27:46.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6669800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f666415fc10 0x7f6664070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6669800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f666415fff0 0x7f6664160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6669800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66640b46c0 con 0x7f66640ce4a0 2024-03-06T07:27:46.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6669800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f66640b49a0 con 0x7f666415fc10 2024-03-06T07:27:46.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6669800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6664072c90 con 0x7f666415fff0 2024-03-06T07:27:46.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6663400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f66640ce4a0 0x7f66640ce930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36926/0 (socket says 172.21.15.86:36926) 2024-03-06T07:27:46.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6663400640 1 -- 172.21.15.86:0/2281042147 learned_addr learned my addr 172.21.15.86:0/2281042147 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:46.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6662a00640 1 --2- 172.21.15.86:0/2281042147 >> v2:172.21.15.86:3300/0 conn(0x7f666415fc10 0x7f6664070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f665be00640 1 --2- 172.21.15.86:0/2281042147 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f666415fff0 0x7f6664160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.166+0000 7f6662000640 1 -- 172.21.15.86:0/2281042147 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2522766 0 0) 0x7f66640b46c0 con 0x7f66640ce4a0 2024-03-06T07:27:46.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.167+0000 7f6662000640 1 -- 172.21.15.86:0/2281042147 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6648003520 con 0x7f66640ce4a0 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.167+0000 7f6662000640 1 -- 172.21.15.86:0/2281042147 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1900848972 0 0) 0x7f6648003520 con 0x7f66640ce4a0 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.167+0000 7f6662a00640 1 -- 172.21.15.86:0/2281042147 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f666415fff0 msgr2=0x7f6664160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.167+0000 7f6662a00640 1 --2- 172.21.15.86:0/2281042147 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f666415fff0 0x7f6664160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.167+0000 7f6662a00640 1 -- 172.21.15.86:0/2281042147 >> v1:172.21.15.86:6789/0 conn(0x7f66640ce4a0 legacy=0x7f66640ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.167+0000 7f6662a00640 1 -- 172.21.15.86:0/2281042147 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6664160990 con 0x7f666415fc10 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.167+0000 7f665be00640 1 --2- 172.21.15.86:0/2281042147 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f666415fff0 0x7f6664160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.168+0000 7f6662a00640 1 --2- 172.21.15.86:0/2281042147 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f6664070ff0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f66540097a0 tx=0x7f665402b790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f1c03d8d4114fd84 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.168+0000 7f6662000640 1 -- 172.21.15.86:0/2281042147 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6654037070 con 0x7f666415fc10 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.168+0000 7f6662000640 1 -- 172.21.15.86:0/2281042147 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6654030510 con 0x7f666415fc10 2024-03-06T07:27:46.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.168+0000 7f6662000640 1 -- 172.21.15.86:0/2281042147 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f665c0019f0 con 0x7f666415fc10 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 -- 172.21.15.86:0/2281042147 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 msgr2=0x7f6664070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 --2- 172.21.15.86:0/2281042147 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f6664070ff0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f66540097a0 tx=0x7f665402b790 comp rx=0 tx=0).stop 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 -- 172.21.15.86:0/2281042147 shutdown_connections 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 --2- 172.21.15.86:0/2281042147 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f666415fff0 0x7f6664160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 --2- 172.21.15.86:0/2281042147 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f6664070ff0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 -- 172.21.15.86:0/2281042147 >> 172.21.15.86:0/2281042147 conn(0x7f66640b1940 msgr2=0x7f66640b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 -- 172.21.15.86:0/2281042147 shutdown_connections 2024-03-06T07:27:46.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.169+0000 7f6669800640 1 -- 172.21.15.86:0/2281042147 wait complete. 2024-03-06T07:27:46.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.170+0000 7f6669800640 1 Processor -- start 2024-03-06T07:27:46.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.171+0000 7f6669800640 1 -- start start 2024-03-06T07:27:46.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.171+0000 7f6669800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66640ce4a0 0x7f66641468d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.171+0000 7f6669800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f66640ace20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.171+0000 7f6669800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f666415fff0 0x7f66640ad360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.171+0000 7f6669800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6664072a30 con 0x7f666415fc10 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.171+0000 7f6669800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66640728b0 con 0x7f666415fff0 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.171+0000 7f6669800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f666415c2e0 con 0x7f66640ce4a0 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f66640ace20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f66640ace20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35260/0 (socket says 172.21.15.86:35260) 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 -- 172.21.15.86:0/3477145139 learned_addr learned my addr 172.21.15.86:0/3477145139 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 -- 172.21.15.86:0/3477145139 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66640ce4a0 msgr2=0x7f66641468d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66640ce4a0 0x7f66641468d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 -- 172.21.15.86:0/3477145139 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f666415fff0 msgr2=0x7f66640ad360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f666415fff0 0x7f66640ad360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 -- 172.21.15.86:0/3477145139 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f666415e280 con 0x7f666415fc10 2024-03-06T07:27:46.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.172+0000 7f665be00640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f66640ace20 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f664c00d7d0 tx=0x7f664c00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:46.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.173+0000 7f665aa00640 1 -- 172.21.15.86:0/3477145139 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f664c004030 con 0x7f666415fc10 2024-03-06T07:27:46.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.173+0000 7f665aa00640 1 -- 172.21.15.86:0/3477145139 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f664c0041d0 con 0x7f666415fc10 2024-03-06T07:27:46.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.173+0000 7f665aa00640 1 -- 172.21.15.86:0/3477145139 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f664c0096a0 con 0x7f666415fc10 2024-03-06T07:27:46.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.173+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6664147200 con 0x7f666415fc10 2024-03-06T07:27:46.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.173+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66641477c0 con 0x7f666415fc10 2024-03-06T07:27:46.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.174+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f666404f330 con 0x7f666415fc10 2024-03-06T07:27:46.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.175+0000 7f665aa00640 1 -- 172.21.15.86:0/3477145139 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f664c009840 con 0x7f666415fc10 2024-03-06T07:27:46.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.176+0000 7f665aa00640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f664006b520 0x7f664006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.176+0000 7f665aa00640 1 -- 172.21.15.86:0/3477145139 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f664c08a370 con 0x7f666415fc10 2024-03-06T07:27:46.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.176+0000 7f6662a00640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f664006b520 0x7f664006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.176+0000 7f6662a00640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f664006b520 0x7f664006d9e0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f6654034000 tx=0x7f665402d000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:46.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.180+0000 7f665aa00640 1 -- 172.21.15.86:0/3477145139 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f664c059360 con 0x7f666415fc10 2024-03-06T07:27:46.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.358+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f666415a320 con 0x7f666415fc10 2024-03-06T07:27:46.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.362+0000 7f665aa00640 1 -- 172.21.15.86:0/3477145139 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v10)=0 v10) v1 ==== 122+0+0 (secure 0 0 0) 0x7f664c058c60 con 0x7f666415fc10 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f664006b520 msgr2=0x7f664006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f664006b520 0x7f664006d9e0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f6654034000 tx=0x7f665402d000 comp rx=0 tx=0).stop 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 msgr2=0x7f66640ace20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f66640ace20 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f664c00d7d0 tx=0x7f664c00dc90 comp rx=0 tx=0).stop 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 shutdown_connections 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f664006b520 0x7f664006d9e0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f666415fff0 0x7f66640ad360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f666415fc10 0x7f66640ace20 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 --2- 172.21.15.86:0/3477145139 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66640ce4a0 0x7f66641468d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 >> 172.21.15.86:0/3477145139 conn(0x7f66640b1940 msgr2=0x7f66640cc690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:46.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 shutdown_connections 2024-03-06T07:27:46.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.365+0000 7f6669800640 1 -- 172.21.15.86:0/3477145139 wait complete. 2024-03-06T07:27:46.383 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:27:46.384 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:27:46.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:27:46.601 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:27:46.601 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:46.601 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:46.654 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:46.654 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:46.706 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:46.706 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:46.756 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:46.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.970+0000 7f9c26200640 1 Processor -- start 2024-03-06T07:27:46.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.970+0000 7f9c26200640 1 -- start start 2024-03-06T07:27:46.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c26200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9c200d24c0 0x7f9c200d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c26200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200b0aa0 0x7f9c200b0f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c26200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9c200d4c50 con 0x7f9c200b2de0 2024-03-06T07:27:46.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c26200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9c200d4a00 con 0x7f9c200d24c0 2024-03-06T07:27:46.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c26200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9c200cf7e0 con 0x7f9c200b0aa0 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c1f400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9c200d24c0 0x7f9c200d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c1f400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9c200d24c0 0x7f9c200d1cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:38706/0 (socket says 172.21.15.86:38706) 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.972+0000 7f9c1f400640 1 -- 172.21.15.86:0/1812723019 learned_addr learned my addr 172.21.15.86:0/1812723019 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.973+0000 7f9c1fe00640 1 --2- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200b0aa0 0x7f9c200b0f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.973+0000 7f9c1f400640 1 -- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200b0aa0 msgr2=0x7f9c200b0f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.973+0000 7f9c1f400640 1 --2- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200b0aa0 0x7f9c200b0f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.973+0000 7f9c1f400640 1 -- 172.21.15.86:0/1812723019 >> v1:172.21.15.194:6789/0 conn(0x7f9c200b2de0 legacy=0x7f9c200d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.973+0000 7f9c1f400640 1 -- 172.21.15.86:0/1812723019 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9c200cfa30 con 0x7f9c200d24c0 2024-03-06T07:27:46.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.974+0000 7f9c1f400640 1 --2- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200d24c0 0x7f9c200d1cc0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f9c08009870 tx=0x7f9c0802b8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b165ed5e6ff0bbf9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:46.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c1e000640 1 -- 172.21.15.86:0/1812723019 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c08035020 con 0x7f9c200d24c0 2024-03-06T07:27:46.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c1e000640 1 -- 172.21.15.86:0/1812723019 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9c08031640 con 0x7f9c200d24c0 2024-03-06T07:27:46.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c1e000640 1 -- 172.21.15.86:0/1812723019 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c0802d040 con 0x7f9c200d24c0 2024-03-06T07:27:46.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c26200640 1 -- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200d24c0 msgr2=0x7f9c200d1cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c26200640 1 --2- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200d24c0 0x7f9c200d1cc0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f9c08009870 tx=0x7f9c0802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c26200640 1 -- 172.21.15.86:0/1812723019 shutdown_connections 2024-03-06T07:27:46.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c26200640 1 --2- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200b0aa0 0x7f9c200b0f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c26200640 1 --2- 172.21.15.86:0/1812723019 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200d24c0 0x7f9c200d1cc0 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c26200640 1 -- 172.21.15.86:0/1812723019 >> 172.21.15.86:0/1812723019 conn(0x7f9c200d3ee0 msgr2=0x7f9c200d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:46.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.975+0000 7f9c26200640 1 -- 172.21.15.86:0/1812723019 shutdown_connections 2024-03-06T07:27:46.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.976+0000 7f9c26200640 1 -- 172.21.15.86:0/1812723019 wait complete. 2024-03-06T07:27:46.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.977+0000 7f9c26200640 1 Processor -- start 2024-03-06T07:27:46.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.977+0000 7f9c26200640 1 -- start start 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.978+0000 7f9c26200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9c200b0aa0 0x7f9c201532e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.978+0000 7f9c26200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200b2de0 0x7f9c200b3da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.978+0000 7f9c26200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200d24c0 0x7f9c20160770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.978+0000 7f9c26200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c2015e5b0 con 0x7f9c200b0aa0 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.978+0000 7f9c26200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c200d0080 con 0x7f9c200b2de0 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.978+0000 7f9c26200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9c2015e730 con 0x7f9c200d24c0 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9c200b0aa0 0x7f9c201532e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9c200b0aa0 0x7f9c201532e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35268/0 (socket says 172.21.15.86:35268) 2024-03-06T07:27:46.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 -- 172.21.15.86:0/2814573018 learned_addr learned my addr 172.21.15.86:0/2814573018 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 -- 172.21.15.86:0/2814573018 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200d24c0 msgr2=0x7f9c20160770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200d24c0 0x7f9c20160770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 -- 172.21.15.86:0/2814573018 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200b2de0 msgr2=0x7f9c200b3da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200b2de0 0x7f9c200b3da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 -- 172.21.15.86:0/2814573018 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9c2015fed0 con 0x7f9c200b0aa0 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.979+0000 7f9c1f400640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9c200b0aa0 0x7f9c201532e0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f9c08000c00 tx=0x7f9c080046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.980+0000 7f9c17400640 1 -- 172.21.15.86:0/2814573018 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c08040070 con 0x7f9c200b0aa0 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.980+0000 7f9c17400640 1 -- 172.21.15.86:0/2814573018 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9c080399f0 con 0x7f9c200b0aa0 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.980+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9c200b6ac0 con 0x7f9c200b0aa0 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.980+0000 7f9c17400640 1 -- 172.21.15.86:0/2814573018 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c08035040 con 0x7f9c200b0aa0 2024-03-06T07:27:46.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.980+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9c200b7080 con 0x7f9c200b0aa0 2024-03-06T07:27:46.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.981+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9c2004f330 con 0x7f9c200b0aa0 2024-03-06T07:27:46.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.982+0000 7f9c17400640 1 -- 172.21.15.86:0/2814573018 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9c0804b060 con 0x7f9c200b0aa0 2024-03-06T07:27:46.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.983+0000 7f9c17400640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9bfc06b4d0 0x7f9bfc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:46.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.983+0000 7f9c17400640 1 -- 172.21.15.86:0/2814573018 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9c080aba90 con 0x7f9c200b0aa0 2024-03-06T07:27:46.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.983+0000 7f9c1ea00640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9bfc06b4d0 0x7f9bfc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:46.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.984+0000 7f9c1ea00640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9bfc06b4d0 0x7f9bfc06d990 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f9c100012e0 tx=0x7f9c10007d50 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:46.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:46.987+0000 7f9c17400640 1 -- 172.21.15.86:0/2814573018 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9c0807a870 con 0x7f9c200b0aa0 2024-03-06T07:27:47.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.183+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f9c2006c990 con 0x7f9c200b0aa0 2024-03-06T07:27:47.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.185+0000 7f9c17400640 1 -- 172.21.15.86:0/2814573018 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f9c0807a170 con 0x7f9c200b0aa0 2024-03-06T07:27:47.188 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:47.188 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9bfc06b4d0 msgr2=0x7f9bfc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9bfc06b4d0 0x7f9bfc06d990 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f9c100012e0 tx=0x7f9c10007d50 comp rx=0 tx=0).stop 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9c200b0aa0 msgr2=0x7f9c201532e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9c200b0aa0 0x7f9c201532e0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f9c08000c00 tx=0x7f9c080046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 shutdown_connections 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9bfc06b4d0 0x7f9bfc06d990 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9c200d24c0 0x7f9c20160770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9c200b2de0 0x7f9c200b3da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 --2- 172.21.15.86:0/2814573018 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9c200b0aa0 0x7f9c201532e0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.188+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 >> 172.21.15.86:0/2814573018 conn(0x7f9c200d3ee0 msgr2=0x7f9c200cb4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:47.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.189+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 shutdown_connections 2024-03-06T07:27:47.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.189+0000 7f9c26200640 1 -- 172.21.15.86:0/2814573018 wait complete. 2024-03-06T07:27:47.210 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:47.211 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:47.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.422+0000 7f7785000640 1 Processor -- start 2024-03-06T07:27:47.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.422+0000 7f7785000640 1 -- start start 2024-03-06T07:27:47.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f7785000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778006da40 0x7f77800ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f7785000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f77800aef20 0x7f77800ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f7785000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f778015c1f0 con 0x7f778015be10 2024-03-06T07:27:47.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f7785000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f77800cd200 con 0x7f77800aef20 2024-03-06T07:27:47.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f7785000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f778006cac0 con 0x7f778006da40 2024-03-06T07:27:47.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f777fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f77800aef20 0x7f77800ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f777fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f77800aef20 0x7f77800ce2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35278/0 (socket says 172.21.15.86:35278) 2024-03-06T07:27:47.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f777fe00640 1 -- 172.21.15.86:0/3768943266 learned_addr learned my addr 172.21.15.86:0/3768943266 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:47.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.424+0000 7f7776a00640 1 --2- 172.21.15.86:0/3768943266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778006da40 0x7f77800ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.425+0000 7f777fe00640 1 -- 172.21.15.86:0/3768943266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778006da40 msgr2=0x7f77800ccc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:47.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.425+0000 7f777fe00640 1 --2- 172.21.15.86:0/3768943266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778006da40 0x7f77800ccc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.425+0000 7f777fe00640 1 -- 172.21.15.86:0/3768943266 >> v1:172.21.15.194:6789/0 conn(0x7f778015be10 legacy=0x7f778006d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:47.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.425+0000 7f777fe00640 1 -- 172.21.15.86:0/3768943266 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f778006cda0 con 0x7f77800aef20 2024-03-06T07:27:47.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.426+0000 7f777fe00640 1 --2- 172.21.15.86:0/3768943266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77800aef20 0x7f77800ce2b0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f7778009e50 tx=0x7f777802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e13869b3f869218a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:47.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.426+0000 7f777ea00640 1 -- 172.21.15.86:0/3768943266 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f777802ca70 con 0x7f77800aef20 2024-03-06T07:27:47.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.426+0000 7f777ea00640 1 -- 172.21.15.86:0/3768943266 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f777802cc10 con 0x7f77800aef20 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.426+0000 7f777ea00640 1 -- 172.21.15.86:0/3768943266 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77780343f0 con 0x7f77800aef20 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 -- 172.21.15.86:0/3768943266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77800aef20 msgr2=0x7f77800ce2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 --2- 172.21.15.86:0/3768943266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77800aef20 0x7f77800ce2b0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f7778009e50 tx=0x7f777802bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 -- 172.21.15.86:0/3768943266 shutdown_connections 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 --2- 172.21.15.86:0/3768943266 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f77800aef20 0x7f77800ce2b0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 --2- 172.21.15.86:0/3768943266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778006da40 0x7f77800ccc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 -- 172.21.15.86:0/3768943266 >> 172.21.15.86:0/3768943266 conn(0x7f778006afc0 msgr2=0x7f778006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 -- 172.21.15.86:0/3768943266 shutdown_connections 2024-03-06T07:27:47.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.427+0000 7f7785000640 1 -- 172.21.15.86:0/3768943266 wait complete. 2024-03-06T07:27:47.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.429+0000 7f7785000640 1 Processor -- start 2024-03-06T07:27:47.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.429+0000 7f7785000640 1 -- start start 2024-03-06T07:27:47.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f7785000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f778006da40 0x7f7780158550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f7785000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77800aef20 0x7f77801524d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f7785000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778015be10 0x7f7780152a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f7785000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77800cda90 con 0x7f778006da40 2024-03-06T07:27:47.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f7785000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f778006d080 con 0x7f77800aef20 2024-03-06T07:27:47.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f7785000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7780071ab0 con 0x7f778015be10 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f777f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f778006da40 0x7f7780158550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f777f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f778006da40 0x7f7780158550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35280/0 (socket says 172.21.15.86:35280) 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f777f400640 1 -- 172.21.15.86:0/2718161660 learned_addr learned my addr 172.21.15.86:0/2718161660 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f7776a00640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77800aef20 0x7f77801524d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.430+0000 7f777fe00640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778015be10 0x7f7780152a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f777f400640 1 -- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778015be10 msgr2=0x7f7780152a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f777f400640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778015be10 0x7f7780152a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f777f400640 1 -- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77800aef20 msgr2=0x7f77801524d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f777f400640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77800aef20 0x7f77801524d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f777f400640 1 -- 172.21.15.86:0/2718161660 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77801f5780 con 0x7f778006da40 2024-03-06T07:27:47.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f777fe00640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778015be10 0x7f7780152a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:47.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f777f400640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f778006da40 0x7f7780158550 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f777000b420 tx=0x7f777000b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:47.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f7777e00640 1 -- 172.21.15.86:0/2718161660 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7770011830 con 0x7f778006da40 2024-03-06T07:27:47.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f7777e00640 1 -- 172.21.15.86:0/2718161660 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7770016070 con 0x7f778006da40 2024-03-06T07:27:47.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f7777e00640 1 -- 172.21.15.86:0/2718161660 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7770011c10 con 0x7f778006da40 2024-03-06T07:27:47.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f77801f5a60 con 0x7f778006da40 2024-03-06T07:27:47.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.431+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f77801f6020 con 0x7f778006da40 2024-03-06T07:27:47.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.433+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f778004f330 con 0x7f778006da40 2024-03-06T07:27:47.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.434+0000 7f7777e00640 1 -- 172.21.15.86:0/2718161660 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f777001b460 con 0x7f778006da40 2024-03-06T07:27:47.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.434+0000 7f7777e00640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775c06b4d0 0x7f775c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.434+0000 7f7777e00640 1 -- 172.21.15.86:0/2718161660 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7770089dc0 con 0x7f778006da40 2024-03-06T07:27:47.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.435+0000 7f7776a00640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775c06b4d0 0x7f775c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.435+0000 7f7776a00640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775c06b4d0 0x7f775c06d990 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f7780157a70 tx=0x7f7768008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:47.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.439+0000 7f7777e00640 1 -- 172.21.15.86:0/2718161660 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7770020020 con 0x7f778006da40 2024-03-06T07:27:47.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.632+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f77800ae530 con 0x7f778006da40 2024-03-06T07:27:47.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.633+0000 7f7777e00640 1 -- 172.21.15.86:0/2718161660 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f7770058ba0 con 0x7f778006da40 2024-03-06T07:27:47.636 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:47.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775c06b4d0 msgr2=0x7f775c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775c06b4d0 0x7f775c06d990 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f7780157a70 tx=0x7f7768008040 comp rx=0 tx=0).stop 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f778006da40 msgr2=0x7f7780158550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f778006da40 0x7f7780158550 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f777000b420 tx=0x7f777000b8e0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 shutdown_connections 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f775c06b4d0 0x7f775c06d990 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f778015be10 0x7f7780152a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f77800aef20 0x7f77801524d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 --2- 172.21.15.86:0/2718161660 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f778006da40 0x7f7780158550 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 >> 172.21.15.86:0/2718161660 conn(0x7f778006afc0 msgr2=0x7f778006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 shutdown_connections 2024-03-06T07:27:47.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.636+0000 7f7785000640 1 -- 172.21.15.86:0/2718161660 wait complete. 2024-03-06T07:27:47.660 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:47.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.870+0000 7ff522800640 1 Processor -- start 2024-03-06T07:27:47.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.870+0000 7ff522800640 1 -- start start 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff522800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff51c06d960 0x7ff51c15f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff522800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff51c06e2f0 0x7ff51c0cda90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff522800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff51c06cba0 con 0x7ff51c0b2c40 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff522800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff51c15f700 con 0x7ff51c06e2f0 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff522800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff51c06d4d0 con 0x7ff51c06d960 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff51be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff51c06e2f0 0x7ff51c0cda90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff51be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff51c06e2f0 0x7ff51c0cda90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35288/0 (socket says 172.21.15.86:35288) 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff51be00640 1 -- 172.21.15.86:0/958836854 learned_addr learned my addr 172.21.15.86:0/958836854 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:47.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff51aa00640 1 --2- 172.21.15.86:0/958836854 >> v2:172.21.15.194:3300/0 conn(0x7ff51c06d960 0x7ff51c15f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff51be00640 1 -- 172.21.15.86:0/958836854 >> v2:172.21.15.194:3300/0 conn(0x7ff51c06d960 msgr2=0x7ff51c15f1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:47.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.872+0000 7ff51be00640 1 --2- 172.21.15.86:0/958836854 >> v2:172.21.15.194:3300/0 conn(0x7ff51c06d960 0x7ff51c15f1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.873+0000 7ff51be00640 1 -- 172.21.15.86:0/958836854 >> v1:172.21.15.194:6789/0 conn(0x7ff51c0b2c40 legacy=0x7ff51c0b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:47.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.873+0000 7ff51be00640 1 -- 172.21.15.86:0/958836854 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff51c06ce80 con 0x7ff51c06e2f0 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff51be00640 1 --2- 172.21.15.86:0/958836854 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06e2f0 0x7ff51c0cda90 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7ff51c074a50 tx=0x7ff51402bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=568244bf2ad73d31 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff51a000640 1 -- 172.21.15.86:0/958836854 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff51402ca80 con 0x7ff51c06e2f0 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff51a000640 1 -- 172.21.15.86:0/958836854 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff51402cc20 con 0x7ff51c06e2f0 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff51a000640 1 -- 172.21.15.86:0/958836854 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5140343f0 con 0x7ff51c06e2f0 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff522800640 1 -- 172.21.15.86:0/958836854 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06e2f0 msgr2=0x7ff51c0cda90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff522800640 1 --2- 172.21.15.86:0/958836854 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06e2f0 0x7ff51c0cda90 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7ff51c074a50 tx=0x7ff51402bee0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff522800640 1 -- 172.21.15.86:0/958836854 shutdown_connections 2024-03-06T07:27:47.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff522800640 1 --2- 172.21.15.86:0/958836854 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06e2f0 0x7ff51c0cda90 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff522800640 1 --2- 172.21.15.86:0/958836854 >> v2:172.21.15.194:3300/0 conn(0x7ff51c06d960 0x7ff51c15f1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.874+0000 7ff522800640 1 -- 172.21.15.86:0/958836854 >> 172.21.15.86:0/958836854 conn(0x7ff51c0b1940 msgr2=0x7ff51c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.875+0000 7ff522800640 1 -- 172.21.15.86:0/958836854 shutdown_connections 2024-03-06T07:27:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.875+0000 7ff522800640 1 -- 172.21.15.86:0/958836854 wait complete. 2024-03-06T07:27:47.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.876+0000 7ff522800640 1 Processor -- start 2024-03-06T07:27:47.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.876+0000 7ff522800640 1 -- start start 2024-03-06T07:27:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff522800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06d960 0x7ff51c09c160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff522800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff51c06e2f0 0x7ff51c0c8fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff522800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff51c0b2c40 0x7ff51c14c400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff522800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff51c06e900 con 0x7ff51c06d960 2024-03-06T07:27:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff522800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff51c06e780 con 0x7ff51c0b2c40 2024-03-06T07:27:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff522800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff51c0cdfd0 con 0x7ff51c06e2f0 2024-03-06T07:27:47.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff51b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06d960 0x7ff51c09c160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff51b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06d960 0x7ff51c09c160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35296/0 (socket says 172.21.15.86:35296) 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.877+0000 7ff51b400640 1 -- 172.21.15.86:0/3517475175 learned_addr learned my addr 172.21.15.86:0/3517475175 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff51b400640 1 -- 172.21.15.86:0/3517475175 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff51c06e2f0 msgr2=0x7ff51c0c8fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff51b400640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff51c06e2f0 0x7ff51c0c8fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff51b400640 1 -- 172.21.15.86:0/3517475175 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff51c0b2c40 msgr2=0x7ff51c14c400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff51b400640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff51c0b2c40 0x7ff51c14c400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff51b400640 1 -- 172.21.15.86:0/3517475175 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff51c0c9500 con 0x7ff51c06d960 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff51b400640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06d960 0x7ff51c09c160 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7ff50c00b410 tx=0x7ff50c00b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff513400640 1 -- 172.21.15.86:0/3517475175 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff50c015020 con 0x7ff51c06d960 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff513400640 1 -- 172.21.15.86:0/3517475175 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff50c011830 con 0x7ff51c06d960 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff513400640 1 -- 172.21.15.86:0/3517475175 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff50c011c20 con 0x7ff51c06d960 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.878+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff51c0c97a0 con 0x7ff51c06d960 2024-03-06T07:27:47.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.879+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff51c0c9d60 con 0x7ff51c06d960 2024-03-06T07:27:47.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.880+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff51c0c9fa0 con 0x7ff51c06d960 2024-03-06T07:27:47.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.881+0000 7ff513400640 1 -- 172.21.15.86:0/3517475175 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff50c018020 con 0x7ff51c06d960 2024-03-06T07:27:47.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.881+0000 7ff513400640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4fc06b4d0 0x7ff4fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:47.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.882+0000 7ff513400640 1 -- 172.21.15.86:0/3517475175 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff50c08a210 con 0x7ff51c06d960 2024-03-06T07:27:47.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.882+0000 7ff51aa00640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4fc06b4d0 0x7ff4fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:47.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.882+0000 7ff51aa00640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4fc06b4d0 0x7ff4fc06d990 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7ff51c15c7c0 tx=0x7ff508008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:47.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:47.886+0000 7ff513400640 1 -- 172.21.15.86:0/3517475175 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff50c01f020 con 0x7ff51c06d960 2024-03-06T07:27:48.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.073+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7ff51c04f330 con 0x7ff51c06d960 2024-03-06T07:27:48.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.074+0000 7ff513400640 1 -- 172.21.15.86:0/3517475175 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7ff50c0172c0 con 0x7ff51c06d960 2024-03-06T07:27:48.077 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:48.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.076+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4fc06b4d0 msgr2=0x7ff4fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.076+0000 7ff522800640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4fc06b4d0 0x7ff4fc06d990 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7ff51c15c7c0 tx=0x7ff508008040 comp rx=0 tx=0).stop 2024-03-06T07:27:48.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.076+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06d960 msgr2=0x7ff51c09c160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.076+0000 7ff522800640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06d960 0x7ff51c09c160 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7ff50c00b410 tx=0x7ff50c00b8d0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 shutdown_connections 2024-03-06T07:27:48.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4fc06b4d0 0x7ff4fc06d990 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff51c0b2c40 0x7ff51c14c400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff51c06e2f0 0x7ff51c0c8fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 --2- 172.21.15.86:0/3517475175 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff51c06d960 0x7ff51c09c160 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 >> 172.21.15.86:0/3517475175 conn(0x7ff51c0b1940 msgr2=0x7ff51c0aefd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:48.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 shutdown_connections 2024-03-06T07:27:48.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.077+0000 7ff522800640 1 -- 172.21.15.86:0/3517475175 wait complete. 2024-03-06T07:27:48.097 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:48.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.298+0000 7f11e8a00640 1 Processor -- start 2024-03-06T07:27:48.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.298+0000 7f11e8a00640 1 -- start start 2024-03-06T07:27:48.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e8a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f11e4160700 0x7f11e4070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e8a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f11e40b46c0 con 0x7f11e415fc10 2024-03-06T07:27:48.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e8a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f11e40b49a0 con 0x7f11e40b3f70 2024-03-06T07:27:48.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e8a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f11e4072c90 con 0x7f11e4160700 2024-03-06T07:27:48.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e1e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f11e415fc10 0x7f11e415fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36936/0 (socket says 172.21.15.86:36936) 2024-03-06T07:27:48.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e1e00640 1 -- 172.21.15.86:0/2995264397 learned_addr learned my addr 172.21.15.86:0/2995264397 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e2800640 1 --2- 172.21.15.86:0/2995264397 >> v2:172.21.15.194:3300/0 conn(0x7f11e4160700 0x7f11e4070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1553346458 0 0) 0x7f11e40b46c0 con 0x7f11e415fc10 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.300+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f11c8003520 con 0x7f11e415fc10 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 977389667 0 0) 0x7f11c8003520 con 0x7f11e415fc10 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 >> v2:172.21.15.194:3300/0 conn(0x7f11e4160700 msgr2=0x7f11e4070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 --2- 172.21.15.86:0/2995264397 >> v2:172.21.15.194:3300/0 conn(0x7f11e4160700 0x7f11e4070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 >> v1:172.21.15.194:6789/0 conn(0x7f11e40b3f70 legacy=0x7f11e40ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11e409dd80 con 0x7f11e415fc10 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f11d4001670 con 0x7f11e415fc10 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 >> v1:172.21.15.86:6789/0 conn(0x7f11e415fc10 legacy=0x7f11e415fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11c8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11c8004800 0x7f11c8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11c80051c0 0x7f11c800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11e4072c90 con 0x7f11c80039b0 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11c800c060 con 0x7f11c8004800 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11c800c260 con 0x7f11c80051c0 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.301+0000 7f11e1e00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11c8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e1400640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11c8004800 0x7f11c8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e2800640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11c80051c0 0x7f11c800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e1e00640 1 -- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11c80051c0 msgr2=0x7f11c800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e1e00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11c80051c0 0x7f11c800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e1e00640 1 -- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11c8004800 msgr2=0x7f11c8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e1e00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11c8004800 0x7f11c8004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e1e00640 1 -- 172.21.15.86:0/2995264397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11c8003520 con 0x7f11c80039b0 2024-03-06T07:27:48.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.302+0000 7f11e2800640 1 -- 172.21.15.86:0/2995264397 reap_dead start 2024-03-06T07:27:48.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.303+0000 7f11e1e00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11c8003dd0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f11d400c210 tx=0x7f11c8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad3dab70a292aa8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:48.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11d4007d30 con 0x7f11c80039b0 2024-03-06T07:27:48.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11d4030070 con 0x7f11c80039b0 2024-03-06T07:27:48.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e0a00640 1 -- 172.21.15.86:0/2995264397 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11d4005d70 con 0x7f11c80039b0 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e8a00640 1 -- 172.21.15.86:0/2995264397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 msgr2=0x7f11c8003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e8a00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11c8003dd0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f11d400c210 tx=0x7f11c8005830 comp rx=0 tx=0).stop 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e8a00640 1 -- 172.21.15.86:0/2995264397 shutdown_connections 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e8a00640 1 --2- 172.21.15.86:0/2995264397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11c8003dd0 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e8a00640 1 -- 172.21.15.86:0/2995264397 >> 172.21.15.86:0/2995264397 conn(0x7f11e40b1940 msgr2=0x7f11e40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e8a00640 1 -- 172.21.15.86:0/2995264397 shutdown_connections 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.304+0000 7f11e8a00640 1 -- 172.21.15.86:0/2995264397 wait complete. 2024-03-06T07:27:48.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.306+0000 7f11e8a00640 1 Processor -- start 2024-03-06T07:27:48.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.306+0000 7f11e8a00640 1 -- start start 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.307+0000 7f11e8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11e415dfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.307+0000 7f11e8a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11e4152470 0x7f11e415ee00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.307+0000 7f11e8a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11e409cc70 0x7f11e41450b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.307+0000 7f11e8a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11e415c460 con 0x7f11c80039b0 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.307+0000 7f11e8a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11e415c2e0 con 0x7f11e409cc70 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.307+0000 7f11e8a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11e4070ff0 con 0x7f11e4152470 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.307+0000 7f11e1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11e415dfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11e415dfc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35302/0 (socket says 172.21.15.86:35302) 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 -- 172.21.15.86:0/4106232394 learned_addr learned my addr 172.21.15.86:0/4106232394 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1400640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11e4152470 0x7f11e415ee00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e2800640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11e409cc70 0x7f11e41450b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 -- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11e4152470 msgr2=0x7f11e415ee00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11e4152470 0x7f11e415ee00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 -- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11e409cc70 msgr2=0x7f11e41450b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11e409cc70 0x7f11e41450b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 -- 172.21.15.86:0/4106232394 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11e40c4f20 con 0x7f11c80039b0 2024-03-06T07:27:48.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e2800640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11e409cc70 0x7f11e41450b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:48.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.308+0000 7f11e1e00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11e415dfc0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f11d400c330 tx=0x7f11d402e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:48.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.309+0000 7f11da000640 1 -- 172.21.15.86:0/4106232394 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11d4004bf0 con 0x7f11c80039b0 2024-03-06T07:27:48.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.309+0000 7f11da000640 1 -- 172.21.15.86:0/4106232394 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11d4004d90 con 0x7f11c80039b0 2024-03-06T07:27:48.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.309+0000 7f11da000640 1 -- 172.21.15.86:0/4106232394 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11d4007270 con 0x7f11c80039b0 2024-03-06T07:27:48.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.309+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f11e415d200 con 0x7f11c80039b0 2024-03-06T07:27:48.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.309+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f11e415d7c0 con 0x7f11c80039b0 2024-03-06T07:27:48.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.310+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f11e404f330 con 0x7f11c80039b0 2024-03-06T07:27:48.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.311+0000 7f11da000640 1 -- 172.21.15.86:0/4106232394 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f11d4048020 con 0x7f11c80039b0 2024-03-06T07:27:48.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.312+0000 7f11da000640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11c406b4d0 0x7f11c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.312+0000 7f11da000640 1 -- 172.21.15.86:0/4106232394 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f11d40ab570 con 0x7f11c80039b0 2024-03-06T07:27:48.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.312+0000 7f11e1400640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11c406b4d0 0x7f11c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.313+0000 7f11e1400640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11c406b4d0 0x7f11c406d990 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f11e4152cf0 tx=0x7f11d0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:48.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.316+0000 7f11da000640 1 -- 172.21.15.86:0/4106232394 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f11d407a400 con 0x7f11c80039b0 2024-03-06T07:27:48.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.503+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f11e40b16b0 con 0x7f11c80039b0 2024-03-06T07:27:48.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.503+0000 7f11da000640 1 -- 172.21.15.86:0/4106232394 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f11d4079d00 con 0x7f11c80039b0 2024-03-06T07:27:48.506 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:48.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11c406b4d0 msgr2=0x7f11c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11c406b4d0 0x7f11c406d990 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f11e4152cf0 tx=0x7f11d0009040 comp rx=0 tx=0).stop 2024-03-06T07:27:48.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 msgr2=0x7f11e415dfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11e415dfc0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f11d400c330 tx=0x7f11d402e070 comp rx=0 tx=0).stop 2024-03-06T07:27:48.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 shutdown_connections 2024-03-06T07:27:48.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11c406b4d0 0x7f11c406d990 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11e409cc70 0x7f11e41450b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f11e4152470 0x7f11e415ee00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 --2- 172.21.15.86:0/4106232394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f11c80039b0 0x7f11e415dfc0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.506+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 >> 172.21.15.86:0/4106232394 conn(0x7f11e40b1940 msgr2=0x7f11e40cc690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:48.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.507+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 shutdown_connections 2024-03-06T07:27:48.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.507+0000 7f11e8a00640 1 -- 172.21.15.86:0/4106232394 wait complete. 2024-03-06T07:27:48.527 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:27:48.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.724+0000 7f3e30600640 1 Processor -- start 2024-03-06T07:27:48.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.724+0000 7f3e30600640 1 -- start start 2024-03-06T07:27:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.725+0000 7f3e30600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3e2815ac90 0x7f3e2806e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e30600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e2815b070 0x7f3e2815ea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e30600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3e280cc000 con 0x7f3e2815ef50 2024-03-06T07:27:48.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e30600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e280ce360 con 0x7f3e2815b070 2024-03-06T07:27:48.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e30600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e280ce640 con 0x7f3e2815ac90 2024-03-06T07:27:48.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2d200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e2815b070 0x7f3e2815ea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2e600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3e2815ef50 0x7f3e280cac60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36944/0 (socket says 172.21.15.86:36944) 2024-03-06T07:27:48.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2e600640 1 -- 172.21.15.86:0/3240910557 learned_addr learned my addr 172.21.15.86:0/3240910557 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:48.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2dc00640 1 --2- 172.21.15.86:0/3240910557 >> v2:172.21.15.194:3300/0 conn(0x7f3e2815ac90 0x7f3e2806e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2d200640 1 -- 172.21.15.86:0/3240910557 >> v2:172.21.15.194:3300/0 conn(0x7f3e2815ac90 msgr2=0x7f3e2806e580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:48.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2d200640 1 --2- 172.21.15.86:0/3240910557 >> v2:172.21.15.194:3300/0 conn(0x7f3e2815ac90 0x7f3e2806e580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2d200640 1 -- 172.21.15.86:0/3240910557 >> v1:172.21.15.86:6789/0 conn(0x7f3e2815ef50 legacy=0x7f3e280cac60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e2d200640 1 -- 172.21.15.86:0/3240910557 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e280ce920 con 0x7f3e2815b070 2024-03-06T07:27:48.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.726+0000 7f3e27e00640 1 -- 172.21.15.86:0/3240910557 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1609469627 0 0) 0x7f3e280cc000 con 0x7f3e2815ef50 2024-03-06T07:27:48.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.727+0000 7f3e2d200640 1 --2- 172.21.15.86:0/3240910557 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815b070 0x7f3e2815ea10 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f3e28070510 tx=0x7f3e1402b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c843b54d095c2d2b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:48.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e27e00640 1 -- 172.21.15.86:0/3240910557 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e140315a0 con 0x7f3e2815b070 2024-03-06T07:27:48.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e27e00640 1 -- 172.21.15.86:0/3240910557 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e14037070 con 0x7f3e2815b070 2024-03-06T07:27:48.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e27e00640 1 -- 172.21.15.86:0/3240910557 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e14031960 con 0x7f3e2815b070 2024-03-06T07:27:48.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e30600640 1 -- 172.21.15.86:0/3240910557 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815b070 msgr2=0x7f3e2815ea10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e30600640 1 --2- 172.21.15.86:0/3240910557 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815b070 0x7f3e2815ea10 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f3e28070510 tx=0x7f3e1402b840 comp rx=0 tx=0).stop 2024-03-06T07:27:48.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e30600640 1 -- 172.21.15.86:0/3240910557 shutdown_connections 2024-03-06T07:27:48.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e30600640 1 --2- 172.21.15.86:0/3240910557 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815b070 0x7f3e2815ea10 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e30600640 1 --2- 172.21.15.86:0/3240910557 >> v2:172.21.15.194:3300/0 conn(0x7f3e2815ac90 0x7f3e2806e580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e30600640 1 -- 172.21.15.86:0/3240910557 >> 172.21.15.86:0/3240910557 conn(0x7f3e280cc4f0 msgr2=0x7f3e280cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:48.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.728+0000 7f3e30600640 1 -- 172.21.15.86:0/3240910557 shutdown_connections 2024-03-06T07:27:48.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.729+0000 7f3e30600640 1 -- 172.21.15.86:0/3240910557 wait complete. 2024-03-06T07:27:48.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.730+0000 7f3e30600640 1 Processor -- start 2024-03-06T07:27:48.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.730+0000 7f3e30600640 1 -- start start 2024-03-06T07:27:48.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e30600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815ac90 0x7f3e2806e510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e30600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e2815b070 0x7f3e28168840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e30600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e2815ef50 0x7f3e28149af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e30600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e280c5a20 con 0x7f3e2815ac90 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e30600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e280ceee0 con 0x7f3e2815ef50 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e30600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3e280c5ba0 con 0x7f3e2815b070 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e2dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815ac90 0x7f3e2806e510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e2dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815ac90 0x7f3e2806e510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35322/0 (socket says 172.21.15.86:35322) 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.731+0000 7f3e2dc00640 1 -- 172.21.15.86:0/629620234 learned_addr learned my addr 172.21.15.86:0/629620234 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e2dc00640 1 -- 172.21.15.86:0/629620234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e2815b070 msgr2=0x7f3e28168840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e2dc00640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e2815b070 0x7f3e28168840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e2dc00640 1 -- 172.21.15.86:0/629620234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e2815ef50 msgr2=0x7f3e28149af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e2dc00640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e2815ef50 0x7f3e28149af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e2dc00640 1 -- 172.21.15.86:0/629620234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e28167b80 con 0x7f3e2815ac90 2024-03-06T07:27:48.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e2dc00640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815ac90 0x7f3e2806e510 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f3e1c00d620 tx=0x7f3e1c00dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:48.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e25600640 1 -- 172.21.15.86:0/629620234 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e1c00b620 con 0x7f3e2815ac90 2024-03-06T07:27:48.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e25600640 1 -- 172.21.15.86:0/629620234 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e1c019070 con 0x7f3e2815ac90 2024-03-06T07:27:48.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e28168d80 con 0x7f3e2815ac90 2024-03-06T07:27:48.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e25600640 1 -- 172.21.15.86:0/629620234 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e1c00ba00 con 0x7f3e2815ac90 2024-03-06T07:27:48.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.732+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e2815de30 con 0x7f3e2815ac90 2024-03-06T07:27:48.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.734+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3e2804f330 con 0x7f3e2815ac90 2024-03-06T07:27:48.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.735+0000 7f3e25600640 1 -- 172.21.15.86:0/629620234 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3e1c00bba0 con 0x7f3e2815ac90 2024-03-06T07:27:48.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.735+0000 7f3e25600640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e0806b4d0 0x7f3e0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:48.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.736+0000 7f3e25600640 1 -- 172.21.15.86:0/629620234 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3e1c08a770 con 0x7f3e2815ac90 2024-03-06T07:27:48.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.736+0000 7f3e2d200640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e0806b4d0 0x7f3e0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:48.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.736+0000 7f3e2d200640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e0806b4d0 0x7f3e0806d990 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f3e14036000 tx=0x7f3e14030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:48.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.740+0000 7f3e25600640 1 -- 172.21.15.86:0/629620234 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3e1c0594c0 con 0x7f3e2815ac90 2024-03-06T07:27:48.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.951+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f3e280cd1c0 con 0x7f3e2815ac90 2024-03-06T07:27:48.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.951+0000 7f3e25600640 1 -- 172.21.15.86:0/629620234 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v22) v1 ==== 80+0+0 (secure 0 0 0) 0x7f3e1c058dc0 con 0x7f3e2815ac90 2024-03-06T07:27:48.954 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:27:48.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e0806b4d0 msgr2=0x7f3e0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e0806b4d0 0x7f3e0806d990 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f3e14036000 tx=0x7f3e14030000 comp rx=0 tx=0).stop 2024-03-06T07:27:48.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815ac90 msgr2=0x7f3e2806e510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815ac90 0x7f3e2806e510 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f3e1c00d620 tx=0x7f3e1c00dae0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 shutdown_connections 2024-03-06T07:27:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e0806b4d0 0x7f3e0806d990 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e2815ef50 0x7f3e28149af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e2815b070 0x7f3e28168840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 --2- 172.21.15.86:0/629620234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e2815ac90 0x7f3e2806e510 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:48.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.954+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 >> 172.21.15.86:0/629620234 conn(0x7f3e280cc4f0 msgr2=0x7f3e2814d820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:48.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.955+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 shutdown_connections 2024-03-06T07:27:48.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:48.955+0000 7f3e30600640 1 -- 172.21.15.86:0/629620234 wait complete. 2024-03-06T07:27:48.976 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:27:48.976 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:48.977 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 -f 2024-03-06T07:27:49.065 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:27:49.065 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:49.065 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:27:49.124 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:49.124 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:49.180 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:49.180 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:49.232 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:49.232 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:49.323 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:49.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.532+0000 7f1b69e00640 1 Processor -- start 2024-03-06T07:27:49.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.533+0000 7f1b69e00640 1 -- start start 2024-03-06T07:27:49.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.534+0000 7f1b69e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1b6406ebe0 0x7f1b64070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.534+0000 7f1b69e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b64071d30 con 0x7f1b640af1c0 2024-03-06T07:27:49.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.534+0000 7f1b69e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b6415f4a0 con 0x7f1b6415eff0 2024-03-06T07:27:49.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.534+0000 7f1b69e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1b64071260 con 0x7f1b6406ebe0 2024-03-06T07:27:49.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.534+0000 7f1b62a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1b640af1c0 0x7f1b6406cc60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36958/0 (socket says 172.21.15.86:36958) 2024-03-06T07:27:49.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.534+0000 7f1b62a00640 1 -- 172.21.15.86:0/967438569 learned_addr learned my addr 172.21.15.86:0/967438569 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:49.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3772684335 0 0) 0x7f1b54001110 con 0x7f1b640af1c0 2024-03-06T07:27:49.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1b48003520 con 0x7f1b640af1c0 2024-03-06T07:27:49.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b63400640 1 --2- 172.21.15.86:0/967438569 >> v2:172.21.15.194:3300/0 conn(0x7f1b6406ebe0 0x7f1b64070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1378888164 0 0) 0x7f1b48003520 con 0x7f1b640af1c0 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 >> v2:172.21.15.194:3300/0 conn(0x7f1b6406ebe0 msgr2=0x7f1b64070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 --2- 172.21.15.86:0/967438569 >> v2:172.21.15.194:3300/0 conn(0x7f1b6406ebe0 0x7f1b64070d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 >> v1:172.21.15.194:6789/0 conn(0x7f1b6415eff0 legacy=0x7f1b6406e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b64071490 con 0x7f1b640af1c0 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1b54001840 con 0x7f1b640af1c0 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b63400640 1 --2- 172.21.15.86:0/967438569 >> v2:172.21.15.194:3300/0 conn(0x7f1b6406ebe0 0x7f1b64070d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 >> v1:172.21.15.86:6789/0 conn(0x7f1b640af1c0 legacy=0x7f1b6406cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.535+0000 7f1b62000640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b48003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62000640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b48004800 0x7f1b48004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62000640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b48005210 0x7f1b4800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b4800c110 con 0x7f1b48003a00 2024-03-06T07:27:49.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b4800c2c0 con 0x7f1b48004800 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1b4800c4c0 con 0x7f1b48005210 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62a00640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b48003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b5a000640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b48004800 0x7f1b48004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b63400640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b48005210 0x7f1b4800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62a00640 1 -- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b48005210 msgr2=0x7f1b4800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62a00640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b48005210 0x7f1b4800b9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62a00640 1 -- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b48004800 msgr2=0x7f1b48004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62a00640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b48004800 0x7f1b48004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b62a00640 1 -- 172.21.15.86:0/967438569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b48003520 con 0x7f1b48003a00 2024-03-06T07:27:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.536+0000 7f1b63400640 1 -- 172.21.15.86:0/967438569 reap_dead start 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.537+0000 7f1b62a00640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b48003e00 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f1b5400c2f0 tx=0x7f1b48005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5210d14253f2df80 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b5402cbb0 con 0x7f1b48003a00 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b54030070 con 0x7f1b48003a00 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b62000640 1 -- 172.21.15.86:0/967438569 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b540058d0 con 0x7f1b48003a00 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b69e00640 1 -- 172.21.15.86:0/967438569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 msgr2=0x7f1b48003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b69e00640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b48003e00 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f1b5400c2f0 tx=0x7f1b48005830 comp rx=0 tx=0).stop 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b69e00640 1 -- 172.21.15.86:0/967438569 shutdown_connections 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b69e00640 1 --2- 172.21.15.86:0/967438569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b48003e00 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.538+0000 7f1b69e00640 1 -- 172.21.15.86:0/967438569 >> 172.21.15.86:0/967438569 conn(0x7f1b640b1730 msgr2=0x7f1b640b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.539+0000 7f1b69e00640 1 -- 172.21.15.86:0/967438569 shutdown_connections 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.539+0000 7f1b69e00640 1 -- 172.21.15.86:0/967438569 wait complete. 2024-03-06T07:27:49.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.540+0000 7f1b69e00640 1 Processor -- start 2024-03-06T07:27:49.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.540+0000 7f1b69e00640 1 -- start start 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.541+0000 7f1b69e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b6415d5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.541+0000 7f1b69e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b640c61c0 0x7f1b640c6640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.541+0000 7f1b69e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b6414c070 0x7f1b6414c500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.541+0000 7f1b69e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b640b30b0 con 0x7f1b48003a00 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.541+0000 7f1b69e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b64071a50 con 0x7f1b640c61c0 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.541+0000 7f1b69e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1b64070d20 con 0x7f1b6414c070 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b6415d5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b6415d5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35336/0 (socket says 172.21.15.86:35336) 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 -- 172.21.15.86:0/3935691107 learned_addr learned my addr 172.21.15.86:0/3935691107 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b5a000640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b640c61c0 0x7f1b640c6640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b63400640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b6414c070 0x7f1b6414c500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 -- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b6414c070 msgr2=0x7f1b6414c500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b6414c070 0x7f1b6414c500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 -- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b640c61c0 msgr2=0x7f1b640c6640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b640c61c0 0x7f1b640c6640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.542+0000 7f1b62a00640 1 -- 172.21.15.86:0/3935691107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b6414cf80 con 0x7f1b48003a00 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.543+0000 7f1b62a00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b6415d5b0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f1b5400c410 tx=0x7f1b5402e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.543+0000 7f1b5be00640 1 -- 172.21.15.86:0/3935691107 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b54004bf0 con 0x7f1b48003a00 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.543+0000 7f1b5be00640 1 -- 172.21.15.86:0/3935691107 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b54007e50 con 0x7f1b48003a00 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.543+0000 7f1b5be00640 1 -- 172.21.15.86:0/3935691107 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b54007070 con 0x7f1b48003a00 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.543+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b6414d260 con 0x7f1b48003a00 2024-03-06T07:27:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.543+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b6414d820 con 0x7f1b48003a00 2024-03-06T07:27:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.544+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1b6404f330 con 0x7f1b48003a00 2024-03-06T07:27:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.545+0000 7f1b5be00640 1 -- 172.21.15.86:0/3935691107 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1b54048020 con 0x7f1b48003a00 2024-03-06T07:27:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.546+0000 7f1b5be00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b4006b500 0x7f1b4006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.546+0000 7f1b5be00640 1 -- 172.21.15.86:0/3935691107 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1b540ab560 con 0x7f1b48003a00 2024-03-06T07:27:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.546+0000 7f1b5a000640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b4006b500 0x7f1b4006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.547+0000 7f1b5a000640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b4006b500 0x7f1b4006d9c0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f1b6415e020 tx=0x7f1b4c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:49.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.551+0000 7f1b5be00640 1 -- 172.21.15.86:0/3935691107 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1b64071d30 con 0x7f1b48003a00 2024-03-06T07:27:49.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.747+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1b640c6b80 con 0x7f1b48003a00 2024-03-06T07:27:49.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.750+0000 7f1b5be00640 1 -- 172.21.15.86:0/3935691107 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f1b5407a340 con 0x7f1b48003a00 2024-03-06T07:27:49.753 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:49.753 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b4006b500 msgr2=0x7f1b4006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b4006b500 0x7f1b4006d9c0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f1b6415e020 tx=0x7f1b4c009040 comp rx=0 tx=0).stop 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 msgr2=0x7f1b6415d5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b6415d5b0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f1b5400c410 tx=0x7f1b5402e070 comp rx=0 tx=0).stop 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 shutdown_connections 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b4006b500 0x7f1b4006d9c0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b6414c070 0x7f1b6414c500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b640c61c0 0x7f1b640c6640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 --2- 172.21.15.86:0/3935691107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b48003a00 0x7f1b6415d5b0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.752+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 >> 172.21.15.86:0/3935691107 conn(0x7f1b640b1730 msgr2=0x7f1b640b0b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:49.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.753+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 shutdown_connections 2024-03-06T07:27:49.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.753+0000 7f1b69e00640 1 -- 172.21.15.86:0/3935691107 wait complete. 2024-03-06T07:27:49.774 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:49.774 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:49.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.983+0000 7fa416e00640 1 Processor -- start 2024-03-06T07:27:49.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.983+0000 7fa416e00640 1 -- start start 2024-03-06T07:27:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.984+0000 7fa416e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa4100b3c40 0x7fa4100cda90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.984+0000 7fa416e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa410165330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.984+0000 7fa416e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4100ce690 con 0x7fa41015b780 2024-03-06T07:27:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.984+0000 7fa416e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4100ce970 con 0x7fa4100b3c40 2024-03-06T07:27:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.984+0000 7fa416e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4100cec50 con 0x7fa4100cdfd0 2024-03-06T07:27:49.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.985+0000 7fa414e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa410165330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.986+0000 7fa414e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa410165330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:38402/0 (socket says 172.21.15.86:38402) 2024-03-06T07:27:49.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.986+0000 7fa414e00640 1 -- 172.21.15.86:0/1850987215 learned_addr learned my addr 172.21.15.86:0/1850987215 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:49.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.986+0000 7fa407400640 1 --2- 172.21.15.86:0/1850987215 >> v2:172.21.15.194:3300/0 conn(0x7fa4100b3c40 0x7fa4100cda90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.986+0000 7fa414e00640 1 -- 172.21.15.86:0/1850987215 >> v2:172.21.15.194:3300/0 conn(0x7fa4100b3c40 msgr2=0x7fa4100cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:49.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.986+0000 7fa414e00640 1 --2- 172.21.15.86:0/1850987215 >> v2:172.21.15.194:3300/0 conn(0x7fa4100b3c40 0x7fa4100cda90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.986+0000 7fa414e00640 1 -- 172.21.15.86:0/1850987215 >> v1:172.21.15.194:6789/0 conn(0x7fa41015b780 legacy=0x7fa4100b3530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:49.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.986+0000 7fa414e00640 1 -- 172.21.15.86:0/1850987215 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4100cef30 con 0x7fa4100cdfd0 2024-03-06T07:27:49.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.988+0000 7fa414e00640 1 --2- 172.21.15.86:0/1850987215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa410165330 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fa408009e50 tx=0x7fa40802bed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cc3387a3655141b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:49.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.988+0000 7fa40f400640 1 -- 172.21.15.86:0/1850987215 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa40802ca70 con 0x7fa4100cdfd0 2024-03-06T07:27:49.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa40f400640 1 -- 172.21.15.86:0/1850987215 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa40802cc10 con 0x7fa4100cdfd0 2024-03-06T07:27:49.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa40f400640 1 -- 172.21.15.86:0/1850987215 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4080343f0 con 0x7fa4100cdfd0 2024-03-06T07:27:49.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa416e00640 1 -- 172.21.15.86:0/1850987215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 msgr2=0x7fa410165330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa416e00640 1 --2- 172.21.15.86:0/1850987215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa410165330 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fa408009e50 tx=0x7fa40802bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa416e00640 1 -- 172.21.15.86:0/1850987215 shutdown_connections 2024-03-06T07:27:49.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa416e00640 1 --2- 172.21.15.86:0/1850987215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa410165330 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa416e00640 1 --2- 172.21.15.86:0/1850987215 >> v2:172.21.15.194:3300/0 conn(0x7fa4100b3c40 0x7fa4100cda90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa416e00640 1 -- 172.21.15.86:0/1850987215 >> 172.21.15.86:0/1850987215 conn(0x7fa4100b1940 msgr2=0x7fa4100b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:49.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.989+0000 7fa416e00640 1 -- 172.21.15.86:0/1850987215 shutdown_connections 2024-03-06T07:27:49.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.990+0000 7fa416e00640 1 -- 172.21.15.86:0/1850987215 wait complete. 2024-03-06T07:27:49.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.992+0000 7fa416e00640 1 Processor -- start 2024-03-06T07:27:49.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.992+0000 7fa416e00640 1 -- start start 2024-03-06T07:27:49.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa416e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4100b3c40 0x7fa410154c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa416e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa4101f3a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa416e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41015b780 0x7fa41006de30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa416e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa41015abb0 con 0x7fa41015b780 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa416e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa41015aa30 con 0x7fa4100b3c40 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa416e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4100af120 con 0x7fa4100cdfd0 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa414e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41015b780 0x7fa41006de30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa414e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41015b780 0x7fa41006de30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35352/0 (socket says 172.21.15.86:35352) 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa414e00640 1 -- 172.21.15.86:0/3082348370 learned_addr learned my addr 172.21.15.86:0/3082348370 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa407400640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa4101f3a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.993+0000 7fa40fe00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4100b3c40 0x7fa410154c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa414e00640 1 -- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 msgr2=0x7fa4101f3a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa414e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa4101f3a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa414e00640 1 -- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4100b3c40 msgr2=0x7fa410154c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:49.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa414e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4100b3c40 0x7fa410154c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:49.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa414e00640 1 -- 172.21.15.86:0/3082348370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4100c9760 con 0x7fa41015b780 2024-03-06T07:27:49.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa407400640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa4101f3a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:49.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa40fe00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4100b3c40 0x7fa410154c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:49.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa414e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41015b780 0x7fa41006de30 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7fa408036040 tx=0x7fa408004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:49.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.994+0000 7fa40cc00640 1 -- 172.21.15.86:0/3082348370 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa408043020 con 0x7fa41015b780 2024-03-06T07:27:49.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.995+0000 7fa40cc00640 1 -- 172.21.15.86:0/3082348370 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa40803e450 con 0x7fa41015b780 2024-03-06T07:27:49.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.995+0000 7fa40cc00640 1 -- 172.21.15.86:0/3082348370 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa408009510 con 0x7fa41015b780 2024-03-06T07:27:49.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.995+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa410155310 con 0x7fa41015b780 2024-03-06T07:27:49.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.995+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4101558d0 con 0x7fa41015b780 2024-03-06T07:27:49.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.996+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa4100c9930 con 0x7fa41015b780 2024-03-06T07:27:49.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.997+0000 7fa40cc00640 1 -- 172.21.15.86:0/3082348370 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa408046020 con 0x7fa41015b780 2024-03-06T07:27:50.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.998+0000 7fa40cc00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3ec06b4d0 0x7fa3ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.998+0000 7fa40cc00640 1 -- 172.21.15.86:0/3082348370 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa4080acbb0 con 0x7fa41015b780 2024-03-06T07:27:50.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.998+0000 7fa40fe00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3ec06b4d0 0x7fa3ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:49.999+0000 7fa40fe00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3ec06b4d0 0x7fa3ec06d990 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fa400006fd0 tx=0x7fa400008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:50.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.002+0000 7fa40cc00640 1 -- 172.21.15.86:0/3082348370 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa40804d030 con 0x7fa41015b780 2024-03-06T07:27:50.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.203+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fa41004f330 con 0x7fa41015b780 2024-03-06T07:27:50.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.204+0000 7fa40cc00640 1 -- 172.21.15.86:0/3082348370 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7fa40807b900 con 0x7fa41015b780 2024-03-06T07:27:50.207 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.206+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3ec06b4d0 msgr2=0x7fa3ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.206+0000 7fa416e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3ec06b4d0 0x7fa3ec06d990 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fa400006fd0 tx=0x7fa400008040 comp rx=0 tx=0).stop 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.206+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41015b780 msgr2=0x7fa41006de30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.206+0000 7fa416e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41015b780 0x7fa41006de30 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7fa408036040 tx=0x7fa408004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 shutdown_connections 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3ec06b4d0 0x7fa3ec06d990 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41015b780 0x7fa41006de30 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4100cdfd0 0x7fa4101f3a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 --2- 172.21.15.86:0/3082348370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4100b3c40 0x7fa410154c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 >> 172.21.15.86:0/3082348370 conn(0x7fa4100b1940 msgr2=0x7fa4100b3120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:50.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 shutdown_connections 2024-03-06T07:27:50.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.207+0000 7fa416e00640 1 -- 172.21.15.86:0/3082348370 wait complete. 2024-03-06T07:27:50.228 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:50.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.438+0000 7fe968200640 1 Processor -- start 2024-03-06T07:27:50.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.438+0000 7fe968200640 1 -- start start 2024-03-06T07:27:50.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe968200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600b3df0 0x7fe9600727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe968200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe9600b3650 0x7fe9600b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe968200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe9600cda90 0x7fe96006c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe968200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe960072310 con 0x7fe9600b3650 2024-03-06T07:27:50.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe968200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe9600b4710 con 0x7fe9600cda90 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe968200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe9600719e0 con 0x7fe9600b3df0 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe964e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe9600b3650 0x7fe9600b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe964e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe9600b3650 0x7fe9600b41d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35366/0 (socket says 172.21.15.86:35366) 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe964e00640 1 -- 172.21.15.86:0/615239593 learned_addr learned my addr 172.21.15.86:0/615239593 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.440+0000 7fe965800640 1 --2- 172.21.15.86:0/615239593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600b3df0 0x7fe9600727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.441+0000 7fe966200640 1 --2- 172.21.15.86:0/615239593 >> v2:172.21.15.194:3300/0 conn(0x7fe9600cda90 0x7fe96006c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.441+0000 7fe964e00640 1 -- 172.21.15.86:0/615239593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600b3df0 msgr2=0x7fe9600727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.441+0000 7fe964e00640 1 --2- 172.21.15.86:0/615239593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600b3df0 0x7fe9600727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.441+0000 7fe964e00640 1 -- 172.21.15.86:0/615239593 >> v2:172.21.15.194:3300/0 conn(0x7fe9600cda90 msgr2=0x7fe96006c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.441+0000 7fe964e00640 1 --2- 172.21.15.86:0/615239593 >> v2:172.21.15.194:3300/0 conn(0x7fe9600cda90 0x7fe96006c990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.441+0000 7fe964e00640 1 -- 172.21.15.86:0/615239593 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe960071cc0 con 0x7fe9600b3650 2024-03-06T07:27:50.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.441+0000 7fe966200640 1 --2- 172.21.15.86:0/615239593 >> v2:172.21.15.194:3300/0 conn(0x7fe9600cda90 0x7fe96006c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.442+0000 7fe964e00640 1 --2- 172.21.15.86:0/615239593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9600b41d0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fe9600af840 tx=0x7fe94c02db20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d5e41d138ed9ad6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.442+0000 7fe95fe00640 1 -- 172.21.15.86:0/615239593 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe94c02f8e0 con 0x7fe9600b3650 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.442+0000 7fe95fe00640 1 -- 172.21.15.86:0/615239593 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe94c02fa80 con 0x7fe9600b3650 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.442+0000 7fe95fe00640 1 -- 172.21.15.86:0/615239593 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe94c02fe50 con 0x7fe9600b3650 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 -- 172.21.15.86:0/615239593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 msgr2=0x7fe9600b41d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 --2- 172.21.15.86:0/615239593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9600b41d0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fe9600af840 tx=0x7fe94c02db20 comp rx=0 tx=0).stop 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 -- 172.21.15.86:0/615239593 shutdown_connections 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 --2- 172.21.15.86:0/615239593 >> v2:172.21.15.194:3300/0 conn(0x7fe9600cda90 0x7fe96006c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 --2- 172.21.15.86:0/615239593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9600b41d0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 --2- 172.21.15.86:0/615239593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600b3df0 0x7fe9600727a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 -- 172.21.15.86:0/615239593 >> 172.21.15.86:0/615239593 conn(0x7fe9600b1940 msgr2=0x7fe9600b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 -- 172.21.15.86:0/615239593 shutdown_connections 2024-03-06T07:27:50.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.443+0000 7fe968200640 1 -- 172.21.15.86:0/615239593 wait complete. 2024-03-06T07:27:50.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.445+0000 7fe968200640 1 Processor -- start 2024-03-06T07:27:50.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.445+0000 7fe968200640 1 -- start start 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe968200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9601f5840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe968200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe9600b3df0 0x7fe96009ac00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe968200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600cda90 0x7fe960152c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe968200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe96015b240 con 0x7fe9600b3650 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe968200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe9600b3b30 con 0x7fe9600b3df0 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe968200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe96015b3c0 con 0x7fe9600cda90 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe965800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9601f5840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe965800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9601f5840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35374/0 (socket says 172.21.15.86:35374) 2024-03-06T07:27:50.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.446+0000 7fe965800640 1 -- 172.21.15.86:0/2352478606 learned_addr learned my addr 172.21.15.86:0/2352478606 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe966200640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600cda90 0x7fe960152c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe965800640 1 -- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600cda90 msgr2=0x7fe960152c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe964e00640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe9600b3df0 0x7fe96009ac00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe965800640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600cda90 0x7fe960152c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe965800640 1 -- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe9600b3df0 msgr2=0x7fe96009ac00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe965800640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe9600b3df0 0x7fe96009ac00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe965800640 1 -- 172.21.15.86:0/2352478606 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe96009b6c0 con 0x7fe9600b3650 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe965800640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9601f5840 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fe95400d800 tx=0x7fe95400dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe95d600640 1 -- 172.21.15.86:0/2352478606 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe954018030 con 0x7fe9600b3650 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe95d600640 1 -- 172.21.15.86:0/2352478606 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe954009640 con 0x7fe9600b3650 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe96009b9a0 con 0x7fe9600b3650 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe95d600640 1 -- 172.21.15.86:0/2352478606 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe954009810 con 0x7fe9600b3650 2024-03-06T07:27:50.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.447+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe96009bf60 con 0x7fe9600b3650 2024-03-06T07:27:50.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.449+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe96004f330 con 0x7fe9600b3650 2024-03-06T07:27:50.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.449+0000 7fe95d600640 1 -- 172.21.15.86:0/2352478606 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe95401b070 con 0x7fe9600b3650 2024-03-06T07:27:50.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.451+0000 7fe95d600640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe94006b4d0 0x7fe94006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.451+0000 7fe95d600640 1 -- 172.21.15.86:0/2352478606 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe95408a440 con 0x7fe9600b3650 2024-03-06T07:27:50.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.451+0000 7fe964e00640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe94006b4d0 0x7fe94006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.452+0000 7fe964e00640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe94006b4d0 0x7fe94006d990 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fe94c038000 tx=0x7fe94c008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:50.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.455+0000 7fe95d600640 1 -- 172.21.15.86:0/2352478606 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe9540592b0 con 0x7fe9600b3650 2024-03-06T07:27:50.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.650+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fe9600b15a0 con 0x7fe9600b3650 2024-03-06T07:27:50.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.651+0000 7fe95d600640 1 -- 172.21.15.86:0/2352478606 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7fe954058bb0 con 0x7fe9600b3650 2024-03-06T07:27:50.653 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:50.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe94006b4d0 msgr2=0x7fe94006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe94006b4d0 0x7fe94006d990 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fe94c038000 tx=0x7fe94c008030 comp rx=0 tx=0).stop 2024-03-06T07:27:50.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 msgr2=0x7fe9601f5840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9601f5840 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fe95400d800 tx=0x7fe95400dcc0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 shutdown_connections 2024-03-06T07:27:50.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe94006b4d0 0x7fe94006d990 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9600cda90 0x7fe960152c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe9600b3df0 0x7fe96009ac00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 --2- 172.21.15.86:0/2352478606 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9600b3650 0x7fe9601f5840 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.653+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 >> 172.21.15.86:0/2352478606 conn(0x7fe9600b1940 msgr2=0x7fe9600b0380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:50.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.654+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 shutdown_connections 2024-03-06T07:27:50.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.654+0000 7fe968200640 1 -- 172.21.15.86:0/2352478606 wait complete. 2024-03-06T07:27:50.675 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:50.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.885+0000 7f78dae00640 1 Processor -- start 2024-03-06T07:27:50.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.885+0000 7f78dae00640 1 -- start start 2024-03-06T07:27:50.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78dae00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f78d415e000 0x7f78d406f130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78dae00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f78d406fef0 0x7f78d40d0ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78dae00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f78d40d1260 con 0x7f78d415e3e0 2024-03-06T07:27:50.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78dae00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f78d40d1020 con 0x7f78d406fef0 2024-03-06T07:27:50.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78dae00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f78d406c0d0 con 0x7f78d415e000 2024-03-06T07:27:50.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78d8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f78d406fef0 0x7f78d40d0ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78d3e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f78d415e000 0x7f78d406f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78d8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f78d406fef0 0x7f78d40d0ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35376/0 (socket says 172.21.15.86:35376) 2024-03-06T07:27:50.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78d3e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f78d415e000 0x7f78d406f130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:38856/0 (socket says 172.21.15.86:38856) 2024-03-06T07:27:50.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.887+0000 7f78d8e00640 1 -- 172.21.15.86:0/1026341335 learned_addr learned my addr 172.21.15.86:0/1026341335 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:50.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.888+0000 7f78d8e00640 1 -- 172.21.15.86:0/1026341335 >> v2:172.21.15.194:3300/0 conn(0x7f78d415e000 msgr2=0x7f78d406f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:50.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.888+0000 7f78d8e00640 1 --2- 172.21.15.86:0/1026341335 >> v2:172.21.15.194:3300/0 conn(0x7f78d415e000 0x7f78d406f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.888+0000 7f78d8e00640 1 -- 172.21.15.86:0/1026341335 >> v1:172.21.15.194:6789/0 conn(0x7f78d415e3e0 legacy=0x7f78d406f7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.888+0000 7f78d8e00640 1 -- 172.21.15.86:0/1026341335 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78d406c320 con 0x7f78d406fef0 2024-03-06T07:27:50.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.889+0000 7f78d8e00640 1 --2- 172.21.15.86:0/1026341335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d406fef0 0x7f78d40d0ae0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f78d4071b20 tx=0x7f78cc02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1f26e844ea1ad48d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:50.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78d2a00640 1 -- 172.21.15.86:0/1026341335 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78cc034070 con 0x7f78d406fef0 2024-03-06T07:27:50.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78d2a00640 1 -- 172.21.15.86:0/1026341335 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f78cc030860 con 0x7f78d406fef0 2024-03-06T07:27:50.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78d2a00640 1 -- 172.21.15.86:0/1026341335 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78cc030c30 con 0x7f78d406fef0 2024-03-06T07:27:50.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78dae00640 1 -- 172.21.15.86:0/1026341335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d406fef0 msgr2=0x7f78d40d0ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78dae00640 1 --2- 172.21.15.86:0/1026341335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d406fef0 0x7f78d40d0ae0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f78d4071b20 tx=0x7f78cc02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78dae00640 1 -- 172.21.15.86:0/1026341335 shutdown_connections 2024-03-06T07:27:50.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78dae00640 1 --2- 172.21.15.86:0/1026341335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d406fef0 0x7f78d40d0ae0 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78dae00640 1 --2- 172.21.15.86:0/1026341335 >> v2:172.21.15.194:3300/0 conn(0x7f78d415e000 0x7f78d406f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.890+0000 7f78dae00640 1 -- 172.21.15.86:0/1026341335 >> 172.21.15.86:0/1026341335 conn(0x7f78d40b3800 msgr2=0x7f78d40b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:50.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.891+0000 7f78dae00640 1 -- 172.21.15.86:0/1026341335 shutdown_connections 2024-03-06T07:27:50.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.891+0000 7f78dae00640 1 -- 172.21.15.86:0/1026341335 wait complete. 2024-03-06T07:27:50.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.892+0000 7f78dae00640 1 Processor -- start 2024-03-06T07:27:50.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.892+0000 7f78dae00640 1 -- start start 2024-03-06T07:27:50.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.893+0000 7f78dae00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78d406fef0 0x7f78d40b4bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.893+0000 7f78dae00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d415e000 0x7f78d40b5110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.893+0000 7f78dae00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78d415e3e0 0x7f78d40b5650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.893+0000 7f78dae00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78d40ce4c0 con 0x7f78d415e000 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.893+0000 7f78dae00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78d40ce340 con 0x7f78d415e3e0 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.893+0000 7f78dae00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f78d40ce640 con 0x7f78d406fef0 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d415e000 0x7f78d40b5110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d415e000 0x7f78d40b5110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35380/0 (socket says 172.21.15.86:35380) 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 -- 172.21.15.86:0/3443586440 learned_addr learned my addr 172.21.15.86:0/3443586440 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3e00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78d406fef0 0x7f78d40b4bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d8e00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78d415e3e0 0x7f78d40b5650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 -- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78d406fef0 msgr2=0x7f78d40b4bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78d406fef0 0x7f78d40b4bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 -- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78d415e3e0 msgr2=0x7f78d40b5650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78d415e3e0 0x7f78d40b5650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.894+0000 7f78d3400640 1 -- 172.21.15.86:0/3443586440 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78d40c6640 con 0x7f78d415e000 2024-03-06T07:27:50.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.895+0000 7f78d3400640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d415e000 0x7f78d40b5110 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f78bc00c3c0 tx=0x7f78bc00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:50.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.895+0000 7f78cbe00640 1 -- 172.21.15.86:0/3443586440 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78bc0052d0 con 0x7f78d415e000 2024-03-06T07:27:50.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.895+0000 7f78cbe00640 1 -- 172.21.15.86:0/3443586440 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f78bc005490 con 0x7f78d415e000 2024-03-06T07:27:50.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.895+0000 7f78cbe00640 1 -- 172.21.15.86:0/3443586440 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78bc007260 con 0x7f78d415e000 2024-03-06T07:27:50.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.895+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78d40c6920 con 0x7f78d415e000 2024-03-06T07:27:50.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.895+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78d40c6ee0 con 0x7f78d415e000 2024-03-06T07:27:50.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.896+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f78d40cfbe0 con 0x7f78d415e000 2024-03-06T07:27:50.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.897+0000 7f78cbe00640 1 -- 172.21.15.86:0/3443586440 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f78bc007400 con 0x7f78d415e000 2024-03-06T07:27:50.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.898+0000 7f78cbe00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f78b006b4d0 0x7f78b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:50.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.898+0000 7f78cbe00640 1 -- 172.21.15.86:0/3443586440 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f78bc089650 con 0x7f78d415e000 2024-03-06T07:27:50.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.898+0000 7f78d3e00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f78b006b4d0 0x7f78b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:50.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.899+0000 7f78d3e00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f78b006b4d0 0x7f78b006d990 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f78c40074a0 tx=0x7f78c400b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:50.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:50.903+0000 7f78cbe00640 1 -- 172.21.15.86:0/3443586440 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f78d40cfbe0 con 0x7f78d415e000 2024-03-06T07:27:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.098+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f78d404f330 con 0x7f78d415e000 2024-03-06T07:27:51.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.099+0000 7f78cbe00640 1 -- 172.21.15.86:0/3443586440 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f78bc058560 con 0x7f78d415e000 2024-03-06T07:27:51.101 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f78b006b4d0 msgr2=0x7f78b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f78b006b4d0 0x7f78b006d990 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f78c40074a0 tx=0x7f78c400b040 comp rx=0 tx=0).stop 2024-03-06T07:27:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d415e000 msgr2=0x7f78d40b5110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d415e000 0x7f78d40b5110 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f78bc00c3c0 tx=0x7f78bc00c880 comp rx=0 tx=0).stop 2024-03-06T07:27:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 shutdown_connections 2024-03-06T07:27:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f78b006b4d0 0x7f78b006d990 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78d415e3e0 0x7f78d40b5650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78d415e000 0x7f78d40b5110 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 --2- 172.21.15.86:0/3443586440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78d406fef0 0x7f78d40b4bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 >> 172.21.15.86:0/3443586440 conn(0x7f78d40b3800 msgr2=0x7f78d40b0dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 shutdown_connections 2024-03-06T07:27:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.101+0000 7f78dae00640 1 -- 172.21.15.86:0/3443586440 wait complete. 2024-03-06T07:27:51.123 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:27:51.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.334+0000 7f22b0600640 1 Processor -- start 2024-03-06T07:27:51.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.334+0000 7f22b0600640 1 -- start start 2024-03-06T07:27:51.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.336+0000 7f22b0600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f22a80c4930 0x7f22a80c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.336+0000 7f22b0600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 0x7f22a8165b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.336+0000 7f22b0600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f22a8145b30 con 0x7f22a80c4d10 2024-03-06T07:27:51.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.336+0000 7f22b0600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f22a8142ce0 con 0x7f22a80c4930 2024-03-06T07:27:51.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.336+0000 7f22b0600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f22a80cbca0 con 0x7f22a80c5880 2024-03-06T07:27:51.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.336+0000 7f22ad200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f22a80c4d10 0x7f22a80c5170 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36964/0 (socket says 172.21.15.86:36964) 2024-03-06T07:27:51.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.336+0000 7f22ad200640 1 -- 172.21.15.86:0/2063971685 learned_addr learned my addr 172.21.15.86:0/2063971685 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22adc00640 1 --2- 172.21.15.86:0/2063971685 >> v2:172.21.15.194:3300/0 conn(0x7f22a80c4930 0x7f22a80c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22ae600640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 0x7f22a8165b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2403483339 0 0) 0x7f2294001110 con 0x7f22a80c4d10 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2290003520 con 0x7f22a80c4d10 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1172203859 0 0) 0x7f2290003520 con 0x7f22a80c4d10 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 msgr2=0x7f22a8165b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 0x7f22a8165b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 >> v2:172.21.15.194:3300/0 conn(0x7f22a80c4930 msgr2=0x7f22a80c6460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 --2- 172.21.15.86:0/2063971685 >> v2:172.21.15.194:3300/0 conn(0x7f22a80c4930 0x7f22a80c6460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22a80cbf80 con 0x7f22a80c4d10 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f2294001840 con 0x7f22a80c4d10 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22ae600640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 0x7f22a8165b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 >> v1:172.21.15.86:6789/0 conn(0x7f22a80c4d10 legacy=0x7f22a80c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22900039b0 0x7f2290003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 0x7f2290004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2290004d40 0x7f22900051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22a8142ce0 con 0x7f2290004d40 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22a80cbca0 con 0x7f22900039b0 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.337+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f229000ba10 con 0x7f22a80c5880 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.338+0000 7f22ae600640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2290004d40 0x7f22900051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.338+0000 7f22ae600640 1 -- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 msgr2=0x7f2290004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.338+0000 7f22ae600640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f22a80c5880 0x7f2290004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.338+0000 7f22adc00640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22900039b0 0x7f2290003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.338+0000 7f22ae600640 1 -- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22900039b0 msgr2=0x7f2290003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:51.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.338+0000 7f22ae600640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22900039b0 0x7f2290003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.338+0000 7f22ae600640 1 -- 172.21.15.86:0/2063971685 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2290003520 con 0x7f2290004d40 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.339+0000 7f22ae600640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2290004d40 0x7f22900051c0 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f22a00073e0 tx=0x7f2290005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1fa5f7bb93bf3276 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22a0039070 con 0x7f2290004d40 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f22a002cd70 con 0x7f2290004d40 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22a7e00640 1 -- 172.21.15.86:0/2063971685 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22a0009c80 con 0x7f2290004d40 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22b0600640 1 -- 172.21.15.86:0/2063971685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2290004d40 msgr2=0x7f22900051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22b0600640 1 --2- 172.21.15.86:0/2063971685 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2290004d40 0x7f22900051c0 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f22a00073e0 tx=0x7f2290005830 comp rx=0 tx=0).stop 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22ae600640 1 -- 172.21.15.86:0/2063971685 reap_dead start 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22b0600640 1 -- 172.21.15.86:0/2063971685 shutdown_connections 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22b0600640 1 -- 172.21.15.86:0/2063971685 >> 172.21.15.86:0/2063971685 conn(0x7f22a80cc4f0 msgr2=0x7f22a80cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22b0600640 1 -- 172.21.15.86:0/2063971685 shutdown_connections 2024-03-06T07:27:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.340+0000 7f22b0600640 1 -- 172.21.15.86:0/2063971685 wait complete. 2024-03-06T07:27:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.342+0000 7f22b0600640 1 Processor -- start 2024-03-06T07:27:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.342+0000 7f22b0600640 1 -- start start 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.343+0000 7f22b0600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2290004d40 0x7f22a813e580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.343+0000 7f22b0600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f22a8151ed0 0x7f22a80b4e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.343+0000 7f22b0600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22a80b5380 0x7f22a815d070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.343+0000 7f22b0600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22a806e600 con 0x7f22a8151ed0 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.343+0000 7f22b0600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22a80ca3b0 con 0x7f22a80b5380 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.343+0000 7f22b0600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f22a80cc260 con 0x7f2290004d40 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f22a8151ed0 0x7f22a80b4e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f22a8151ed0 0x7f22a80b4e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35402/0 (socket says 172.21.15.86:35402) 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 -- 172.21.15.86:0/1051831927 learned_addr learned my addr 172.21.15.86:0/1051831927 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ae600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22a80b5380 0x7f22a815d070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22adc00640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2290004d40 0x7f22a813e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 -- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2290004d40 msgr2=0x7f22a813e580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2290004d40 0x7f22a813e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 -- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22a80b5380 msgr2=0x7f22a815d070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22a80b5380 0x7f22a815d070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 -- 172.21.15.86:0/1051831927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22a813eac0 con 0x7f22a8151ed0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22ad200640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f22a8151ed0 0x7f22a80b4e40 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f229400c840 tx=0x7f229400cd00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22a5600640 1 -- 172.21.15.86:0/1051831927 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2294018070 con 0x7f22a8151ed0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.344+0000 7f22a5600640 1 -- 172.21.15.86:0/1051831927 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f22940074e0 con 0x7f22a8151ed0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.345+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f22a815bca0 con 0x7f22a8151ed0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.345+0000 7f22a5600640 1 -- 172.21.15.86:0/1051831927 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2294004a20 con 0x7f22a8151ed0 2024-03-06T07:27:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.345+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f22a80c8110 con 0x7f22a8151ed0 2024-03-06T07:27:51.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.346+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f22a804f330 con 0x7f22a8151ed0 2024-03-06T07:27:51.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.347+0000 7f22a5600640 1 -- 172.21.15.86:0/1051831927 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f229401b070 con 0x7f22a8151ed0 2024-03-06T07:27:51.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.347+0000 7f22a5600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f228806b4d0 0x7f228806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:51.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.348+0000 7f22a5600640 1 -- 172.21.15.86:0/1051831927 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2294089920 con 0x7f22a8151ed0 2024-03-06T07:27:51.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.348+0000 7f22adc00640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f228806b4d0 0x7f228806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:51.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.348+0000 7f22adc00640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f228806b4d0 0x7f228806d990 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f229c009990 tx=0x7f229c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:51.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.352+0000 7f22a5600640 1 -- 172.21.15.86:0/1051831927 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f22a8145b30 con 0x7f22a8151ed0 2024-03-06T07:27:51.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.562+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f22a8143a40 con 0x7f22a8151ed0 2024-03-06T07:27:51.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.562+0000 7f22a5600640 1 -- 172.21.15.86:0/1051831927 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v22) v1 ==== 80+0+0 (secure 0 0 0) 0x7f2294058700 con 0x7f22a8151ed0 2024-03-06T07:27:51.565 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f228806b4d0 msgr2=0x7f228806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f228806b4d0 0x7f228806d990 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f229c009990 tx=0x7f229c008040 comp rx=0 tx=0).stop 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f22a8151ed0 msgr2=0x7f22a80b4e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f22a8151ed0 0x7f22a80b4e40 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f229400c840 tx=0x7f229400cd00 comp rx=0 tx=0).stop 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 shutdown_connections 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f228806b4d0 0x7f228806d990 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f22a80b5380 0x7f22a815d070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f22a8151ed0 0x7f22a80b4e40 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 --2- 172.21.15.86:0/1051831927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2290004d40 0x7f22a813e580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 >> 172.21.15.86:0/1051831927 conn(0x7f22a80cc4f0 msgr2=0x7f22a806da70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:51.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 shutdown_connections 2024-03-06T07:27:51.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:51.565+0000 7f22b0600640 1 -- 172.21.15.86:0/1051831927 wait complete. 2024-03-06T07:27:51.587 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.1... 2024-03-06T07:27:51.587 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:51.587 DEBUG:teuthology.orchestra.run.smithi194:> sudo umount /home/ubuntu/cephtest/mnt.1 -f 2024-03-06T07:27:51.694 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:27:51.695 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:51.695 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-06T07:27:51.751 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:51.752 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:51.808 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:51.808 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:51.853 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:27:51.853 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:51.949 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:52.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.158+0000 7f4133a00640 1 Processor -- start 2024-03-06T07:27:52.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.158+0000 7f4133a00640 1 -- start start 2024-03-06T07:27:52.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.160+0000 7f4133a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f412c0af820 0x7f412c0b33b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.160+0000 7f4133a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f412c0b3da0 con 0x7f412c15e160 2024-03-06T07:27:52.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.160+0000 7f4133a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f412c06f1f0 con 0x7f412c0b38f0 2024-03-06T07:27:52.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.160+0000 7f4133a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f412c06b0f0 con 0x7f412c0af820 2024-03-06T07:27:52.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.160+0000 7f4131000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f412c15e160 0x7f412c162620 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36970/0 (socket says 172.21.15.86:36970) 2024-03-06T07:27:52.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.160+0000 7f4131000640 1 -- 172.21.15.86:0/154683103 learned_addr learned my addr 172.21.15.86:0/154683103 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f4123e00640 1 --2- 172.21.15.86:0/154683103 >> v2:172.21.15.194:3300/0 conn(0x7f412c0af820 0x7f412c0b33b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3269398331 0 0) 0x7f411c001110 con 0x7f412c15e160 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4110003520 con 0x7f412c15e160 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1740908552 0 0) 0x7f4110003520 con 0x7f412c15e160 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 >> v2:172.21.15.194:3300/0 conn(0x7f412c0af820 msgr2=0x7f412c0b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 --2- 172.21.15.86:0/154683103 >> v2:172.21.15.194:3300/0 conn(0x7f412c0af820 0x7f412c0b33b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 >> v1:172.21.15.194:6789/0 conn(0x7f412c0b38f0 legacy=0x7f412c06b580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f412c0afce0 con 0x7f412c15e160 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f411c001840 con 0x7f412c15e160 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 >> v1:172.21.15.86:6789/0 conn(0x7f412c15e160 legacy=0x7f412c162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4110003a00 0x7f4110003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4110004800 0x7f4110004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f411000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f412c06b0f0 con 0x7f4110005210 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f411000c160 con 0x7f4110003a00 2024-03-06T07:27:52.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.161+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f411000c360 con 0x7f4110004800 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4131a00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f411000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4131000640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4110003a00 0x7f4110003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4123e00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4110004800 0x7f4110004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4131a00640 1 -- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4110004800 msgr2=0x7f4110004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4131a00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4110004800 0x7f4110004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4131a00640 1 -- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4110003a00 msgr2=0x7f4110003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4131a00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4110003a00 0x7f4110003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.162+0000 7f4131a00640 1 -- 172.21.15.86:0/154683103 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4110003520 con 0x7f4110005210 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.163+0000 7f4131a00640 1 -- 172.21.15.86:0/154683103 reap_dead start 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.163+0000 7f4131a00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f411000b9c0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f4124001e20 tx=0x7f4110005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3b2d6b24d8f5a7b2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.163+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4124001a70 con 0x7f4110005210 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.163+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4124003b70 con 0x7f4110005210 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.163+0000 7f412be00640 1 -- 172.21.15.86:0/154683103 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41240050d0 con 0x7f4110005210 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.164+0000 7f4133a00640 1 -- 172.21.15.86:0/154683103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 msgr2=0x7f411000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.164+0000 7f4133a00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f411000b9c0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f4124001e20 tx=0x7f4110005830 comp rx=0 tx=0).stop 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.164+0000 7f4133a00640 1 -- 172.21.15.86:0/154683103 shutdown_connections 2024-03-06T07:27:52.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.164+0000 7f4133a00640 1 --2- 172.21.15.86:0/154683103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f411000b9c0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.164+0000 7f4133a00640 1 -- 172.21.15.86:0/154683103 >> 172.21.15.86:0/154683103 conn(0x7f412c0d0340 msgr2=0x7f412c0d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:52.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.164+0000 7f4133a00640 1 -- 172.21.15.86:0/154683103 shutdown_connections 2024-03-06T07:27:52.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.164+0000 7f4133a00640 1 -- 172.21.15.86:0/154683103 wait complete. 2024-03-06T07:27:52.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.166+0000 7f4133a00640 1 Processor -- start 2024-03-06T07:27:52.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.166+0000 7f4133a00640 1 -- start start 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4133a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f412c155070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4133a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f412c1555b0 0x7f412c1e7540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4133a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f412c1e7a80 0x7f412c13f470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4133a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f412c06ef10 con 0x7f4110005210 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4133a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f412c0b5620 con 0x7f412c1e7a80 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4133a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f412c0b33b0 con 0x7f412c1555b0 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4131000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f412c155070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4131000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f412c155070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35426/0 (socket says 172.21.15.86:35426) 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.167+0000 7f4131000640 1 -- 172.21.15.86:0/2415569478 learned_addr learned my addr 172.21.15.86:0/2415569478 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4123e00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f412c1555b0 0x7f412c1e7540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4131000640 1 -- 172.21.15.86:0/2415569478 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f412c1555b0 msgr2=0x7f412c1e7540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4131000640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f412c1555b0 0x7f412c1e7540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4131000640 1 -- 172.21.15.86:0/2415569478 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f412c1e7a80 msgr2=0x7f412c13f470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4131000640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f412c1e7a80 0x7f412c13f470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4131000640 1 -- 172.21.15.86:0/2415569478 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f412c1f4070 con 0x7f4110005210 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4131000640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f412c155070 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f411c001eb0 tx=0x7f411c00eb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4129600640 1 -- 172.21.15.86:0/2415569478 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f411c00c6c0 con 0x7f4110005210 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4129600640 1 -- 172.21.15.86:0/2415569478 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f411c018070 con 0x7f4110005210 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4129600640 1 -- 172.21.15.86:0/2415569478 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f411c00caa0 con 0x7f4110005210 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f412c155b60 con 0x7f4110005210 2024-03-06T07:27:52.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.168+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f412c1402d0 con 0x7f4110005210 2024-03-06T07:27:52.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.169+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f412c074ae0 con 0x7f4110005210 2024-03-06T07:27:52.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.170+0000 7f4129600640 1 -- 172.21.15.86:0/2415569478 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f411c00cc40 con 0x7f4110005210 2024-03-06T07:27:52.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.171+0000 7f4129600640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f410806b500 0x7f410806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.171+0000 7f4129600640 1 -- 172.21.15.86:0/2415569478 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f411c089ce0 con 0x7f4110005210 2024-03-06T07:27:52.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.172+0000 7f4123e00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f410806b500 0x7f410806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.172+0000 7f4123e00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f410806b500 0x7f410806d9c0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f412c1f76f0 tx=0x7f4114008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:52.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.176+0000 7f4129600640 1 -- 172.21.15.86:0/2415569478 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f412c0b3da0 con 0x7f4110005210 2024-03-06T07:27:52.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.374+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f412c04f330 con 0x7f4110005210 2024-03-06T07:27:52.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.376+0000 7f4129600640 1 -- 172.21.15.86:0/2415569478 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f411c058b50 con 0x7f4110005210 2024-03-06T07:27:52.379 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:52.379 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":22,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:08.902568+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f410806b500 msgr2=0x7f410806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f410806b500 0x7f410806d9c0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f412c1f76f0 tx=0x7f4114008040 comp rx=0 tx=0).stop 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 msgr2=0x7f412c155070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f412c155070 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f411c001eb0 tx=0x7f411c00eb90 comp rx=0 tx=0).stop 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 shutdown_connections 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f410806b500 0x7f410806d9c0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f412c1e7a80 0x7f412c13f470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f412c1555b0 0x7f412c1e7540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 --2- 172.21.15.86:0/2415569478 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4110005210 0x7f412c155070 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 >> 172.21.15.86:0/2415569478 conn(0x7f412c0d0340 msgr2=0x7f412c0ae7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:52.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.378+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 shutdown_connections 2024-03-06T07:27:52.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.379+0000 7f4133a00640 1 -- 172.21.15.86:0/2415569478 wait complete. 2024-03-06T07:27:52.399 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:52.399 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:52.566 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:27:52.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:27:52.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.593+0000 7f2d79e00640 1 Processor -- start 2024-03-06T07:27:52.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.593+0000 7f2d79e00640 1 -- start start 2024-03-06T07:27:52.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d79e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f2d7415e130 0x7f2d740cf5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d79e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740d25b0 0x7f2d740c7970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d79e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d740d13e0 con 0x7f2d740cfaf0 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d79e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d740cf120 con 0x7f2d7415e130 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d79e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d740cead0 con 0x7f2d740d25b0 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d73e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2d740cfaf0 0x7f2d740d1ea0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36978/0 (socket says 172.21.15.86:36978) 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d73e00640 1 -- 172.21.15.86:0/1827876292 learned_addr learned my addr 172.21.15.86:0/1827876292 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d79400640 1 --2- 172.21.15.86:0/1827876292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740d25b0 0x7f2d740c7970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.595+0000 7f2d78a00640 1 --2- 172.21.15.86:0/1827876292 >> v2:172.21.15.194:3300/0 conn(0x7f2d7415e130 0x7f2d740cf5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.596+0000 7f2d73400640 1 -- 172.21.15.86:0/1827876292 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3203078221 0 0) 0x7f2d740d13e0 con 0x7f2d740cfaf0 2024-03-06T07:27:52.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.596+0000 7f2d73400640 1 -- 172.21.15.86:0/1827876292 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d5c003520 con 0x7f2d740cfaf0 2024-03-06T07:27:52.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.596+0000 7f2d79400640 1 -- 172.21.15.86:0/1827876292 >> v2:172.21.15.194:3300/0 conn(0x7f2d7415e130 msgr2=0x7f2d740cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:52.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.596+0000 7f2d79400640 1 --2- 172.21.15.86:0/1827876292 >> v2:172.21.15.194:3300/0 conn(0x7f2d7415e130 0x7f2d740cf5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.596+0000 7f2d79400640 1 -- 172.21.15.86:0/1827876292 >> v1:172.21.15.86:6789/0 conn(0x7f2d740cfaf0 legacy=0x7f2d740d1ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.596+0000 7f2d79400640 1 -- 172.21.15.86:0/1827876292 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d740cedb0 con 0x7f2d740d25b0 2024-03-06T07:27:52.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.596+0000 7f2d78a00640 1 --2- 172.21.15.86:0/1827876292 >> v2:172.21.15.194:3300/0 conn(0x7f2d7415e130 0x7f2d740cf5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.597+0000 7f2d79400640 1 --2- 172.21.15.86:0/1827876292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740d25b0 0x7f2d740c7970 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f2d6c009d90 tx=0x7f2d6c02beb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=db8cebe09374ca6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.597+0000 7f2d73400640 1 -- 172.21.15.86:0/1827876292 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d6c039070 con 0x7f2d740d25b0 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d73400640 1 -- 172.21.15.86:0/1827876292 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d6c02cc60 con 0x7f2d740d25b0 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d73400640 1 -- 172.21.15.86:0/1827876292 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d6c034640 con 0x7f2d740d25b0 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 -- 172.21.15.86:0/1827876292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740d25b0 msgr2=0x7f2d740c7970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 --2- 172.21.15.86:0/1827876292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740d25b0 0x7f2d740c7970 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f2d6c009d90 tx=0x7f2d6c02beb0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 -- 172.21.15.86:0/1827876292 shutdown_connections 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 --2- 172.21.15.86:0/1827876292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740d25b0 0x7f2d740c7970 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 --2- 172.21.15.86:0/1827876292 >> v2:172.21.15.194:3300/0 conn(0x7f2d7415e130 0x7f2d740cf5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 -- 172.21.15.86:0/1827876292 >> 172.21.15.86:0/1827876292 conn(0x7f2d7409d070 msgr2=0x7f2d740d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:52.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 -- 172.21.15.86:0/1827876292 shutdown_connections 2024-03-06T07:27:52.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.598+0000 7f2d79e00640 1 -- 172.21.15.86:0/1827876292 wait complete. 2024-03-06T07:27:52.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.600+0000 7f2d79e00640 1 Processor -- start 2024-03-06T07:27:52.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.600+0000 7f2d79e00640 1 -- start start 2024-03-06T07:27:52.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.601+0000 7f2d79e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740cfaf0 0x7f2d74152310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.601+0000 7f2d79e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d740d25b0 0x7f2d741f3a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.601+0000 7f2d79e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d7415e130 0x7f2d740b73b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.601+0000 7f2d79e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d740c8030 con 0x7f2d7415e130 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.601+0000 7f2d79e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d740c7eb0 con 0x7f2d740d25b0 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.601+0000 7f2d79e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d740c81b0 con 0x7f2d740cfaf0 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d7415e130 0x7f2d740b73b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d7415e130 0x7f2d740b73b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35442/0 (socket says 172.21.15.86:35442) 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 -- 172.21.15.86:0/2609714050 learned_addr learned my addr 172.21.15.86:0/2609714050 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d740d25b0 0x7f2d741f3a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d78a00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740cfaf0 0x7f2d74152310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 -- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740cfaf0 msgr2=0x7f2d74152310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740cfaf0 0x7f2d74152310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 -- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d740d25b0 msgr2=0x7f2d741f3a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d740d25b0 0x7f2d741f3a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 -- 172.21.15.86:0/2609714050 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d740b7d20 con 0x7f2d7415e130 2024-03-06T07:27:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.602+0000 7f2d79400640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d7415e130 0x7f2d740b73b0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f2d6c037040 tx=0x7f2d6c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.603+0000 7f2d70c00640 1 -- 172.21.15.86:0/2609714050 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d6c042030 con 0x7f2d7415e130 2024-03-06T07:27:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.603+0000 7f2d70c00640 1 -- 172.21.15.86:0/2609714050 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d6c03a040 con 0x7f2d7415e130 2024-03-06T07:27:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.603+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d7415e640 con 0x7f2d7415e130 2024-03-06T07:27:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.603+0000 7f2d70c00640 1 -- 172.21.15.86:0/2609714050 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d6c039070 con 0x7f2d7415e130 2024-03-06T07:27:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.603+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d7415ec00 con 0x7f2d7415e130 2024-03-06T07:27:52.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.604+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d7404f330 con 0x7f2d7415e130 2024-03-06T07:27:52.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.605+0000 7f2d70c00640 1 -- 172.21.15.86:0/2609714050 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d6c045070 con 0x7f2d7415e130 2024-03-06T07:27:52.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.606+0000 7f2d70c00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b4d0 0x7f2d5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:52.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.606+0000 7f2d70c00640 1 -- 172.21.15.86:0/2609714050 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2d6c0abff0 con 0x7f2d7415e130 2024-03-06T07:27:52.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.607+0000 7f2d78a00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b4d0 0x7f2d5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:52.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.607+0000 7f2d78a00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b4d0 0x7f2d5406d990 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f2d68009560 tx=0x7f2d68008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:52.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.611+0000 7f2d70c00640 1 -- 172.21.15.86:0/2609714050 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d6c04c070 con 0x7f2d7415e130 2024-03-06T07:27:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.810+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f2d740d4cb0 con 0x7f2d7415e130 2024-03-06T07:27:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.811+0000 7f2d70c00640 1 -- 172.21.15.86:0/2609714050 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f2d6c07ae80 con 0x7f2d7415e130 2024-03-06T07:27:52.814 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b4d0 msgr2=0x7f2d5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b4d0 0x7f2d5406d990 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f2d68009560 tx=0x7f2d68008040 comp rx=0 tx=0).stop 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d7415e130 msgr2=0x7f2d740b73b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d7415e130 0x7f2d740b73b0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f2d6c037040 tx=0x7f2d6c004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 shutdown_connections 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b4d0 0x7f2d5406d990 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d7415e130 0x7f2d740b73b0 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d740d25b0 0x7f2d741f3a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 --2- 172.21.15.86:0/2609714050 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d740cfaf0 0x7f2d74152310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 >> 172.21.15.86:0/2609714050 conn(0x7f2d7409d070 msgr2=0x7f2d7406bd00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:52.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 shutdown_connections 2024-03-06T07:27:52.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:52.814+0000 7f2d79e00640 1 -- 172.21.15.86:0/2609714050 wait complete. 2024-03-06T07:27:52.835 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:53.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.025+0000 7fb5f9000640 1 Processor -- start 2024-03-06T07:27:53.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.026+0000 7fb5f9000640 1 -- start start 2024-03-06T07:27:53.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.027+0000 7fb5f9000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb5f40c8080 0x7fb5f40d2300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.027+0000 7fb5f9000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5f40d13e0 con 0x7fb5f415e130 2024-03-06T07:27:53.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.027+0000 7fb5f9000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5f40cead0 con 0x7fb5f40d1ea0 2024-03-06T07:27:53.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.027+0000 7fb5f9000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5f40d2840 con 0x7fb5f40c8080 2024-03-06T07:27:53.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.027+0000 7fb5f3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb5f40c8080 0x7fb5f40d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.027+0000 7fb5f3400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb5f415e130 0x7fb5f40cf5b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36982/0 (socket says 172.21.15.86:36982) 2024-03-06T07:27:53.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.027+0000 7fb5f3400640 1 -- 172.21.15.86:0/3039526221 learned_addr learned my addr 172.21.15.86:0/3039526221 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:53.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.028+0000 7fb5f3e00640 1 -- 172.21.15.86:0/3039526221 >> v1:172.21.15.194:6789/0 conn(0x7fb5f40d1ea0 legacy=0x7fb5f40c7970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.028+0000 7fb5f3e00640 1 -- 172.21.15.86:0/3039526221 >> v1:172.21.15.86:6789/0 conn(0x7fb5f415e130 legacy=0x7fb5f40cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.028+0000 7fb5f3e00640 1 -- 172.21.15.86:0/3039526221 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5f40cedb0 con 0x7fb5f40c8080 2024-03-06T07:27:53.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.028+0000 7fb5f2000640 1 -- 172.21.15.86:0/3039526221 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3306087811 0 0) 0x7fb5f40d13e0 con 0x7fb5f415e130 2024-03-06T07:27:53.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.029+0000 7fb5f3e00640 1 --2- 172.21.15.86:0/3039526221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f40c8080 0x7fb5f40d2300 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fb5ec009d90 tx=0x7fb5ec02bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1a7b0ce39f2ac2b9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.029+0000 7fb5f2000640 1 -- 172.21.15.86:0/3039526221 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5ec039070 con 0x7fb5f40c8080 2024-03-06T07:27:53.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.029+0000 7fb5f2000640 1 -- 172.21.15.86:0/3039526221 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5ec02cd00 con 0x7fb5f40c8080 2024-03-06T07:27:53.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.029+0000 7fb5f2000640 1 -- 172.21.15.86:0/3039526221 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5ec034660 con 0x7fb5f40c8080 2024-03-06T07:27:53.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.030+0000 7fb5f9000640 1 -- 172.21.15.86:0/3039526221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f40c8080 msgr2=0x7fb5f40d2300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.030+0000 7fb5f9000640 1 --2- 172.21.15.86:0/3039526221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f40c8080 0x7fb5f40d2300 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fb5ec009d90 tx=0x7fb5ec02bf50 comp rx=0 tx=0).stop 2024-03-06T07:27:53.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.030+0000 7fb5f9000640 1 -- 172.21.15.86:0/3039526221 shutdown_connections 2024-03-06T07:27:53.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.030+0000 7fb5f9000640 1 --2- 172.21.15.86:0/3039526221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f40c8080 0x7fb5f40d2300 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.030+0000 7fb5f9000640 1 -- 172.21.15.86:0/3039526221 >> 172.21.15.86:0/3039526221 conn(0x7fb5f409d070 msgr2=0x7fb5f40d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:53.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.030+0000 7fb5f9000640 1 -- 172.21.15.86:0/3039526221 shutdown_connections 2024-03-06T07:27:53.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.030+0000 7fb5f9000640 1 -- 172.21.15.86:0/3039526221 wait complete. 2024-03-06T07:27:53.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.032+0000 7fb5f9000640 1 Processor -- start 2024-03-06T07:27:53.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.032+0000 7fb5f9000640 1 -- start start 2024-03-06T07:27:53.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f9000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5f40c8080 0x7fb5f41f0fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f9000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5f40d1ea0 0x7fb5f40b7a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f415e130 0x7fb5f4161480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f9000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5f40d16c0 con 0x7fb5f415e130 2024-03-06T07:27:53.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f9000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5f40cf120 con 0x7fb5f40c8080 2024-03-06T07:27:53.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f9000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5f40d06b0 con 0x7fb5f40d1ea0 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f415e130 0x7fb5f4161480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f415e130 0x7fb5f4161480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35454/0 (socket says 172.21.15.86:35454) 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 -- 172.21.15.86:0/2683752228 learned_addr learned my addr 172.21.15.86:0/2683752228 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3400640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5f40c8080 0x7fb5f41f0fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f2a00640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5f40d1ea0 0x7fb5f40b7a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 -- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5f40d1ea0 msgr2=0x7fb5f40b7a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5f40d1ea0 0x7fb5f40b7a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 -- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5f40c8080 msgr2=0x7fb5f41f0fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5f40c8080 0x7fb5f41f0fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.033+0000 7fb5f3e00640 1 -- 172.21.15.86:0/2683752228 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5f41606c0 con 0x7fb5f415e130 2024-03-06T07:27:53.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.034+0000 7fb5f3e00640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f415e130 0x7fb5f4161480 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fb5ec009d90 tx=0x7fb5ec004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.034+0000 7fb5eb400640 1 -- 172.21.15.86:0/2683752228 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5ec039050 con 0x7fb5f415e130 2024-03-06T07:27:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.034+0000 7fb5eb400640 1 -- 172.21.15.86:0/2683752228 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5ec045070 con 0x7fb5f415e130 2024-03-06T07:27:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.034+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5f41609a0 con 0x7fb5f415e130 2024-03-06T07:27:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.034+0000 7fb5eb400640 1 -- 172.21.15.86:0/2683752228 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5ec03e830 con 0x7fb5f415e130 2024-03-06T07:27:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.034+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5f4160f60 con 0x7fb5f415e130 2024-03-06T07:27:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.036+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5f404f330 con 0x7fb5f415e130 2024-03-06T07:27:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.036+0000 7fb5eb400640 1 -- 172.21.15.86:0/2683752228 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb5ec03e9d0 con 0x7fb5f415e130 2024-03-06T07:27:53.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.037+0000 7fb5eb400640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5d406b4d0 0x7fb5d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.037+0000 7fb5eb400640 1 -- 172.21.15.86:0/2683752228 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(23..23 src has 1..23) v4 ==== 5833+0+0 (secure 0 0 0) 0x7fb5ec0ad0a0 con 0x7fb5f415e130 2024-03-06T07:27:53.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.037+0000 7fb5f3400640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5d406b4d0 0x7fb5d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.038+0000 7fb5f3400640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5d406b4d0 0x7fb5d406d990 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fb5e4000f60 tx=0x7fb5e4007d20 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.042+0000 7fb5eb400640 1 -- 172.21.15.86:0/2683752228 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb5ec07bea0 con 0x7fb5f415e130 2024-03-06T07:27:53.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.239+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fb5f4161130 con 0x7fb5f415e130 2024-03-06T07:27:53.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.240+0000 7fb5eb400640 1 -- 172.21.15.86:0/2683752228 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v23) v1 ==== 104+0+11 (secure 0 0 0) 0x7fb5ec07b7a0 con 0x7fb5f415e130 2024-03-06T07:27:53.243 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5d406b4d0 msgr2=0x7fb5d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5d406b4d0 0x7fb5d406d990 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fb5e4000f60 tx=0x7fb5e4007d20 comp rx=0 tx=0).stop 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f415e130 msgr2=0x7fb5f4161480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f415e130 0x7fb5f4161480 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fb5ec009d90 tx=0x7fb5ec004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 shutdown_connections 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5d406b4d0 0x7fb5d406d990 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5f415e130 0x7fb5f4161480 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5f40d1ea0 0x7fb5f40b7a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 --2- 172.21.15.86:0/2683752228 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5f40c8080 0x7fb5f41f0fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 >> 172.21.15.86:0/2683752228 conn(0x7fb5f409d070 msgr2=0x7fb5f40c8e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:53.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 shutdown_connections 2024-03-06T07:27:53.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.243+0000 7fb5f9000640 1 -- 172.21.15.86:0/2683752228 wait complete. 2024-03-06T07:27:53.264 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:53.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.459+0000 7f7efc000640 1 Processor -- start 2024-03-06T07:27:53.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.459+0000 7f7efc000640 1 -- start start 2024-03-06T07:27:53.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7efc000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 0x7f7ef40d2500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7efc000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ef40d4e00 con 0x7f7ef40d1060 2024-03-06T07:27:53.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7efc000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ef40b33b0 con 0x7f7ef40b5180 2024-03-06T07:27:53.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7efc000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef40b3ce0 con 0x7f7ef415e3f0 2024-03-06T07:27:53.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7ef8c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7ef40d1060 0x7f7ef40ceac0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36992/0 (socket says 172.21.15.86:36992) 2024-03-06T07:27:53.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7ef8c00640 1 -- 172.21.15.86:0/3259778790 learned_addr learned my addr 172.21.15.86:0/3259778790 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:53.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7ef9600640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 0x7f7ef40d2500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1415428001 0 0) 0x7f7ef40d4e00 con 0x7f7ef40d1060 2024-03-06T07:27:53.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.461+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7edc003520 con 0x7f7ef40d1060 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3251114784 0 0) 0x7f7edc003520 con 0x7f7ef40d1060 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 msgr2=0x7f7ef40d2500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 0x7f7ef40d2500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 >> v1:172.21.15.194:6789/0 conn(0x7f7ef40b5180 legacy=0x7f7ef40cf1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ef40b3690 con 0x7f7ef40d1060 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7ee0001670 con 0x7f7ef40d1060 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 >> v1:172.21.15.86:6789/0 conn(0x7f7ef40d1060 legacy=0x7f7ef40ceac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7edc0039e0 0x7f7edc003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7edc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 0x7f7edc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef40b3ce0 con 0x7f7edc004800 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7edc00ba70 con 0x7f7edc0039e0 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7edc00bc70 con 0x7f7ef415e3f0 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef8c00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7edc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7ef9600640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7edc0039e0 0x7f7edc003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.462+0000 7f7efa000640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 0x7f7edc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.463+0000 7f7ef8c00640 1 -- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 msgr2=0x7f7edc0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.463+0000 7f7ef8c00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef415e3f0 0x7f7edc0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.463+0000 7f7ef8c00640 1 -- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7edc0039e0 msgr2=0x7f7edc003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.463+0000 7f7ef8c00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7edc0039e0 0x7f7edc003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.463+0000 7f7ef8c00640 1 -- 172.21.15.86:0/3259778790 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7edc003520 con 0x7f7edc004800 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.464+0000 7f7ef8c00640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7edc004c80 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f7ee0001eb0 tx=0x7f7edc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b24cafe2c82ec4b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.464+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee0007d30 con 0x7f7edc004800 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.464+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ee0030070 con 0x7f7edc004800 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.464+0000 7f7ef3e00640 1 -- 172.21.15.86:0/3259778790 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee0005d30 con 0x7f7edc004800 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.465+0000 7f7efc000640 1 -- 172.21.15.86:0/3259778790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 msgr2=0x7f7edc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.465+0000 7f7efc000640 1 --2- 172.21.15.86:0/3259778790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7edc004c80 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f7ee0001eb0 tx=0x7f7edc005830 comp rx=0 tx=0).stop 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.465+0000 7f7efa000640 1 -- 172.21.15.86:0/3259778790 reap_dead start 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.465+0000 7f7efc000640 1 -- 172.21.15.86:0/3259778790 shutdown_connections 2024-03-06T07:27:53.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.465+0000 7f7efc000640 1 -- 172.21.15.86:0/3259778790 >> 172.21.15.86:0/3259778790 conn(0x7f7ef40d3f20 msgr2=0x7f7ef40d4330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.465+0000 7f7efc000640 1 -- 172.21.15.86:0/3259778790 shutdown_connections 2024-03-06T07:27:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.465+0000 7f7efc000640 1 -- 172.21.15.86:0/3259778790 wait complete. 2024-03-06T07:27:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.467+0000 7f7efc000640 1 Processor -- start 2024-03-06T07:27:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.467+0000 7f7efc000640 1 -- start start 2024-03-06T07:27:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7efc000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7ef414b550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7efc000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ef415f610 0x7f7ef40b6b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7efc000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef40b7080 0x7f7ef40ccfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7efc000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef40d4a40 con 0x7f7edc004800 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7efc000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef40cf8e0 con 0x7f7ef415f610 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7efc000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef40d4bc0 con 0x7f7ef40b7080 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7ef414b550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7ef414b550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35474/0 (socket says 172.21.15.86:35474) 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 -- 172.21.15.86:0/1841774285 learned_addr learned my addr 172.21.15.86:0/1841774285 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7efa000640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef40b7080 0x7f7ef40ccfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef8c00640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ef415f610 0x7f7ef40b6b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 -- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef40b7080 msgr2=0x7f7ef40ccfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef40b7080 0x7f7ef40ccfa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 -- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ef415f610 msgr2=0x7f7ef40b6b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ef415f610 0x7f7ef40b6b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.468+0000 7f7ef9600640 1 -- 172.21.15.86:0/1841774285 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ef40cc1e0 con 0x7f7edc004800 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.469+0000 7f7ef9600640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7ef414b550 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f7ee8011600 tx=0x7f7ee8011ac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.469+0000 7f7ef1600640 1 -- 172.21.15.86:0/1841774285 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee800f920 con 0x7f7edc004800 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.469+0000 7f7ef1600640 1 -- 172.21.15.86:0/1841774285 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ee800fac0 con 0x7f7edc004800 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.469+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7ef40cc4c0 con 0x7f7edc004800 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.469+0000 7f7ef1600640 1 -- 172.21.15.86:0/1841774285 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee800fe70 con 0x7f7edc004800 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.469+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ef40cca80 con 0x7f7edc004800 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.470+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7ef404f330 con 0x7f7edc004800 2024-03-06T07:27:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.471+0000 7f7ef1600640 1 -- 172.21.15.86:0/1841774285 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7ee8015460 con 0x7f7edc004800 2024-03-06T07:27:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.472+0000 7f7ef1600640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ed406b500 0x7f7ed406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.472+0000 7f7ef1600640 1 -- 172.21.15.86:0/1841774285 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(23..23 src has 1..23) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f7ee808acd0 con 0x7f7edc004800 2024-03-06T07:27:53.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.472+0000 7f7ef8c00640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ed406b500 0x7f7ed406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.473+0000 7f7ef8c00640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ed406b500 0x7f7ed406d9c0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f7ee0034000 tx=0x7f7ee0009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.477+0000 7f7ef1600640 1 -- 172.21.15.86:0/1841774285 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7ee8019030 con 0x7f7edc004800 2024-03-06T07:27:53.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.671+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f7ef40b7460 con 0x7f7edc004800 2024-03-06T07:27:53.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.672+0000 7f7ef1600640 1 -- 172.21.15.86:0/1841774285 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v23) v1 ==== 100+0+11 (secure 0 0 0) 0x7f7ee8059a30 con 0x7f7edc004800 2024-03-06T07:27:53.674 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:53.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ed406b500 msgr2=0x7f7ed406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ed406b500 0x7f7ed406d9c0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f7ee0034000 tx=0x7f7ee0009030 comp rx=0 tx=0).stop 2024-03-06T07:27:53.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 msgr2=0x7f7ef414b550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7ef414b550 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f7ee8011600 tx=0x7f7ee8011ac0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 shutdown_connections 2024-03-06T07:27:53.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ed406b500 0x7f7ed406d9c0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7ef40b7080 0x7f7ef40ccfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ef415f610 0x7f7ef40b6b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 --2- 172.21.15.86:0/1841774285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7edc004800 0x7f7ef414b550 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.674+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 >> 172.21.15.86:0/1841774285 conn(0x7f7ef40d3f20 msgr2=0x7f7ef406cfe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:53.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.675+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 shutdown_connections 2024-03-06T07:27:53.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.675+0000 7f7efc000640 1 -- 172.21.15.86:0/1841774285 wait complete. 2024-03-06T07:27:53.695 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:27:53.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.884+0000 7fa7d9a00640 1 Processor -- start 2024-03-06T07:27:53.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.884+0000 7fa7d9a00640 1 -- start start 2024-03-06T07:27:53.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.885+0000 7fa7d9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa7d415dfd0 0x7fa7d4162700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d406b6d0 0x7fa7d40d06f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d9a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa7d40cef70 con 0x7fa7d40affe0 2024-03-06T07:27:53.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d9a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa7d4162270 con 0x7fa7d415dfd0 2024-03-06T07:27:53.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d9a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa7d415e3b0 con 0x7fa7d406b6d0 2024-03-06T07:27:53.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa7d415dfd0 0x7fa7d4162700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa7d415dfd0 0x7fa7d4162700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35486/0 (socket says 172.21.15.86:35486) 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa7d40affe0 0x7fa7d406afc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37002/0 (socket says 172.21.15.86:37002) 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3e00640 1 -- 172.21.15.86:0/1218299555 learned_addr learned my addr 172.21.15.86:0/1218299555 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d9000640 1 --2- 172.21.15.86:0/1218299555 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d406b6d0 0x7fa7d40d06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3e00640 1 -- 172.21.15.86:0/1218299555 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d406b6d0 msgr2=0x7fa7d40d06f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3e00640 1 --2- 172.21.15.86:0/1218299555 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d406b6d0 0x7fa7d40d06f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3e00640 1 -- 172.21.15.86:0/1218299555 >> v1:172.21.15.86:6789/0 conn(0x7fa7d40affe0 legacy=0x7fa7d406afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d3e00640 1 -- 172.21.15.86:0/1218299555 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7d4161c60 con 0x7fa7d415dfd0 2024-03-06T07:27:53.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.886+0000 7fa7d2a00640 1 -- 172.21.15.86:0/1218299555 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3739993399 0 0) 0x7fa7d40cef70 con 0x7fa7d40affe0 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.888+0000 7fa7d3e00640 1 --2- 172.21.15.86:0/1218299555 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d4162700 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7fa7c4009870 tx=0x7fa7c402b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2baeb7068829eb7c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d2a00640 1 -- 172.21.15.86:0/1218299555 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7c40317c0 con 0x7fa7d415dfd0 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d2a00640 1 -- 172.21.15.86:0/1218299555 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa7c4037070 con 0x7fa7d415dfd0 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d2a00640 1 -- 172.21.15.86:0/1218299555 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7c4031b80 con 0x7fa7d415dfd0 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 -- 172.21.15.86:0/1218299555 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 msgr2=0x7fa7d4162700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/1218299555 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d4162700 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7fa7c4009870 tx=0x7fa7c402b860 comp rx=0 tx=0).stop 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 -- 172.21.15.86:0/1218299555 shutdown_connections 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/1218299555 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d406b6d0 0x7fa7d40d06f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/1218299555 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d4162700 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 -- 172.21.15.86:0/1218299555 >> 172.21.15.86:0/1218299555 conn(0x7fa7d406c0d0 msgr2=0x7fa7d406c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 -- 172.21.15.86:0/1218299555 shutdown_connections 2024-03-06T07:27:53.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.889+0000 7fa7d9a00640 1 -- 172.21.15.86:0/1218299555 wait complete. 2024-03-06T07:27:53.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.891+0000 7fa7d9a00640 1 Processor -- start 2024-03-06T07:27:53.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.891+0000 7fa7d9a00640 1 -- start start 2024-03-06T07:27:53.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7d406b6d0 0x7fa7d4148570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d40affe0 0x7fa7d40aeb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d40af090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7d40ce8d0 con 0x7fa7d415dfd0 2024-03-06T07:27:53.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7d4161f00 con 0x7fa7d406b6d0 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa7d40cea50 con 0x7fa7d40affe0 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d40af090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d40af090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35492/0 (socket says 172.21.15.86:35492) 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d9000640 1 -- 172.21.15.86:0/2116114724 learned_addr learned my addr 172.21.15.86:0/2116114724 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d3400640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d40affe0 0x7fa7d40aeb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.892+0000 7fa7d3e00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7d406b6d0 0x7fa7d4148570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9000640 1 -- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d40affe0 msgr2=0x7fa7d40aeb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9000640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d40affe0 0x7fa7d40aeb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9000640 1 -- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7d406b6d0 msgr2=0x7fa7d4148570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9000640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7d406b6d0 0x7fa7d4148570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9000640 1 -- 172.21.15.86:0/2116114724 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7d415e6c0 con 0x7fa7d415dfd0 2024-03-06T07:27:53.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9000640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d40af090 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fa7cc007d40 tx=0x7fa7cc00e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7cbe00640 1 -- 172.21.15.86:0/2116114724 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7cc014420 con 0x7fa7d415dfd0 2024-03-06T07:27:53.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7cbe00640 1 -- 172.21.15.86:0/2116114724 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa7cc019070 con 0x7fa7d415dfd0 2024-03-06T07:27:53.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7d415e9a0 con 0x7fa7d415dfd0 2024-03-06T07:27:53.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7cbe00640 1 -- 172.21.15.86:0/2116114724 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7cc014820 con 0x7fa7d415dfd0 2024-03-06T07:27:53.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.893+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7d415ef60 con 0x7fa7d415dfd0 2024-03-06T07:27:53.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.894+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa7d404f330 con 0x7fa7d415dfd0 2024-03-06T07:27:53.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.896+0000 7fa7cbe00640 1 -- 172.21.15.86:0/2116114724 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa7cc0149c0 con 0x7fa7d415dfd0 2024-03-06T07:27:53.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.897+0000 7fa7cbe00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7b406b500 0x7fa7b406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:53.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.897+0000 7fa7cbe00640 1 -- 172.21.15.86:0/2116114724 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(23..23 src has 1..23) v4 ==== 5833+0+0 (secure 0 0 0) 0x7fa7cc08aa90 con 0x7fa7d415dfd0 2024-03-06T07:27:53.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.897+0000 7fa7d3e00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7b406b500 0x7fa7b406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:53.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.898+0000 7fa7d3e00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7b406b500 0x7fa7b406d9c0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fa7c4036000 tx=0x7fa7c4008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:53.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:53.902+0000 7fa7cbe00640 1 -- 172.21.15.86:0/2116114724 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa7cc059890 con 0x7fa7d415dfd0 2024-03-06T07:27:54.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.118+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fa7d40b5e60 con 0x7fa7d415dfd0 2024-03-06T07:27:54.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.119+0000 7fa7cbe00640 1 -- 172.21.15.86:0/2116114724 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v24) v1 ==== 80+0+0 (secure 0 0 0) 0x7fa7cc059190 con 0x7fa7d415dfd0 2024-03-06T07:27:54.122 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7b406b500 msgr2=0x7fa7b406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7b406b500 0x7fa7b406d9c0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fa7c4036000 tx=0x7fa7c4008030 comp rx=0 tx=0).stop 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 msgr2=0x7fa7d40af090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d40af090 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fa7cc007d40 tx=0x7fa7cc00e4b0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 shutdown_connections 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa7b406b500 0x7fa7b406d9c0 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7d415dfd0 0x7fa7d40af090 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7d40affe0 0x7fa7d40aeb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 --2- 172.21.15.86:0/2116114724 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7d406b6d0 0x7fa7d4148570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 >> 172.21.15.86:0/2116114724 conn(0x7fa7d406c0d0 msgr2=0x7fa7d415d080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:54.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 shutdown_connections 2024-03-06T07:27:54.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.122+0000 7fa7d9a00640 1 -- 172.21.15.86:0/2116114724 wait complete. 2024-03-06T07:27:54.141 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.2... 2024-03-06T07:27:54.142 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:54.142 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.2 -f 2024-03-06T07:27:54.228 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:27:54.228 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:54.228 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-06T07:27:54.280 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:54.280 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:54.333 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:54.334 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:54.378 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:27:54.378 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:27:54.470 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:54.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.685+0000 7f0a34c00640 1 Processor -- start 2024-03-06T07:27:54.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.685+0000 7f0a34c00640 1 -- start start 2024-03-06T07:27:54.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.687+0000 7f0a34c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a3015f9d0 0x7f0a3015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:54.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.687+0000 7f0a34c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0a30070b10 0x7f0a30070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:54.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.687+0000 7f0a34c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0a3006cef0 con 0x7f0a30144c00 2024-03-06T07:27:54.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.687+0000 7f0a34c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a30160370 con 0x7f0a30070b10 2024-03-06T07:27:54.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.687+0000 7f0a34c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0a3006d1d0 con 0x7f0a3015f9d0 2024-03-06T07:27:54.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.688+0000 7f0a2e800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0a30070b10 0x7f0a30070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:54.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.688+0000 7f0a2e800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0a30070b10 0x7f0a30070f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:38986/0 (socket says 172.21.15.86:38986) 2024-03-06T07:27:54.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.688+0000 7f0a2e800640 1 -- 172.21.15.86:0/3939340485 learned_addr learned my addr 172.21.15.86:0/3939340485 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:54.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.688+0000 7f0a27e00640 1 --2- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a3015f9d0 0x7f0a3015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:54.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.689+0000 7f0a2e800640 1 -- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a3015f9d0 msgr2=0x7f0a3015fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:54.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.689+0000 7f0a2e800640 1 --2- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a3015f9d0 0x7f0a3015fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.689+0000 7f0a2e800640 1 -- 172.21.15.86:0/3939340485 >> v1:172.21.15.194:6789/0 conn(0x7f0a30144c00 legacy=0x7f0a3006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.689+0000 7f0a2e800640 1 -- 172.21.15.86:0/3939340485 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a3006d4b0 con 0x7f0a30070b10 2024-03-06T07:27:54.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.690+0000 7f0a2e800640 1 --2- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a30070f60 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f0a28009e50 tx=0x7f0a2802bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=29b3757ee300a1c9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:54.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.690+0000 7f0a2d400640 1 -- 172.21.15.86:0/3939340485 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a2802cb00 con 0x7f0a30070b10 2024-03-06T07:27:54.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a2d400640 1 -- 172.21.15.86:0/3939340485 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a2802cca0 con 0x7f0a30070b10 2024-03-06T07:27:54.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a2d400640 1 -- 172.21.15.86:0/3939340485 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a28034610 con 0x7f0a30070b10 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 -- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 msgr2=0x7f0a30070f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 --2- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a30070f60 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f0a28009e50 tx=0x7f0a2802bf60 comp rx=0 tx=0).stop 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 -- 172.21.15.86:0/3939340485 shutdown_connections 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 --2- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a30070f60 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 --2- 172.21.15.86:0/3939340485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a3015f9d0 0x7f0a3015fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 -- 172.21.15.86:0/3939340485 >> 172.21.15.86:0/3939340485 conn(0x7f0a300b1730 msgr2=0x7f0a300b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 -- 172.21.15.86:0/3939340485 shutdown_connections 2024-03-06T07:27:54.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.691+0000 7f0a34c00640 1 -- 172.21.15.86:0/3939340485 wait complete. 2024-03-06T07:27:54.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.693+0000 7f0a34c00640 1 Processor -- start 2024-03-06T07:27:54.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.693+0000 7f0a34c00640 1 -- start start 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.694+0000 7f0a34c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a3013a810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.694+0000 7f0a34c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a30144c00 0x7f0a300ce900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.694+0000 7f0a34c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a3015f9d0 0x7f0a300cee40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.694+0000 7f0a34c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a3006cb50 con 0x7f0a3015f9d0 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.694+0000 7f0a34c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a30160730 con 0x7f0a30070b10 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.694+0000 7f0a34c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0a3006ccd0 con 0x7f0a30144c00 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.694+0000 7f0a2e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a3015f9d0 0x7f0a300cee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a3015f9d0 0x7f0a300cee40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35494/0 (socket says 172.21.15.86:35494) 2024-03-06T07:27:54.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 -- 172.21.15.86:0/2919548860 learned_addr learned my addr 172.21.15.86:0/2919548860 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:54.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a27e00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a30144c00 0x7f0a300ce900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2de00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a3013a810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 -- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a30144c00 msgr2=0x7f0a300ce900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a30144c00 0x7f0a300ce900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 -- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 msgr2=0x7f0a3013a810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a3013a810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 -- 172.21.15.86:0/2919548860 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a3013b170 con 0x7f0a3015f9d0 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a27e00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a30144c00 0x7f0a300ce900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2de00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a3013a810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.695+0000 7f0a2e800640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a3015f9d0 0x7f0a300cee40 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f0a28009e50 tx=0x7f0a28004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.696+0000 7f0a26000640 1 -- 172.21.15.86:0/2919548860 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a28043020 con 0x7f0a3015f9d0 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.696+0000 7f0a26000640 1 -- 172.21.15.86:0/2919548860 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a2802ce50 con 0x7f0a3015f9d0 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.696+0000 7f0a26000640 1 -- 172.21.15.86:0/2919548860 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a28009510 con 0x7f0a3015f9d0 2024-03-06T07:27:54.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.696+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0a3013b450 con 0x7f0a3015f9d0 2024-03-06T07:27:54.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.696+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a3013bcf0 con 0x7f0a3015f9d0 2024-03-06T07:27:54.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.697+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0a3013b620 con 0x7f0a3015f9d0 2024-03-06T07:27:54.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.698+0000 7f0a26000640 1 -- 172.21.15.86:0/2919548860 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0a28046020 con 0x7f0a3015f9d0 2024-03-06T07:27:54.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.699+0000 7f0a26000640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a0c06b4d0 0x7f0a0c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:54.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.699+0000 7f0a26000640 1 -- 172.21.15.86:0/2919548860 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f0a280ac8f0 con 0x7f0a3015f9d0 2024-03-06T07:27:54.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.699+0000 7f0a2de00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a0c06b4d0 0x7f0a0c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:54.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.700+0000 7f0a2de00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a0c06b4d0 0x7f0a0c06d990 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f0a20001680 tx=0x7f0a20009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:54.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.704+0000 7f0a26000640 1 -- 172.21.15.86:0/2919548860 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0a2804d030 con 0x7f0a3015f9d0 2024-03-06T07:27:54.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.885+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0a3004f330 con 0x7f0a3015f9d0 2024-03-06T07:27:54.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.888+0000 7f0a26000640 1 -- 172.21.15.86:0/2919548860 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v24) v1 ==== 74+0+16885 (secure 0 0 0) 0x7f0a2803c350 con 0x7f0a3015f9d0 2024-03-06T07:27:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:54.891 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":24,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:53.981490+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.16","mappings":[{"from":5,"to":2},{"from":7,"to":1}]},{"pgid":"3.24","mappings":[{"from":3,"to":1}]},{"pgid":"3.28","mappings":[{"from":5,"to":2}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a0c06b4d0 msgr2=0x7f0a0c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a0c06b4d0 0x7f0a0c06d990 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f0a20001680 tx=0x7f0a20009040 comp rx=0 tx=0).stop 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a3015f9d0 msgr2=0x7f0a300cee40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a3015f9d0 0x7f0a300cee40 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f0a28009e50 tx=0x7f0a28004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 shutdown_connections 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a0c06b4d0 0x7f0a0c06d990 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a3015f9d0 0x7f0a300cee40 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a30144c00 0x7f0a300ce900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 --2- 172.21.15.86:0/2919548860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a30070b10 0x7f0a3013a810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:54.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 >> 172.21.15.86:0/2919548860 conn(0x7f0a300b1730 msgr2=0x7f0a30070ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:54.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 shutdown_connections 2024-03-06T07:27:54.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:54.891+0000 7f0a34c00640 1 -- 172.21.15.86:0/2919548860 wait complete. 2024-03-06T07:27:54.913 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:54.914 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:55.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.115+0000 7f0133c00640 1 Processor -- start 2024-03-06T07:27:55.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.115+0000 7f0133c00640 1 -- start start 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0133c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0133c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f012c15fff0 0x7f012c160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0133c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f012c0ce4a0 0x7f012c0ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0133c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f012c0b46c0 con 0x7f012c0ce4a0 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0133c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f012c0b49a0 con 0x7f012c15fff0 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0133c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f012c072c90 con 0x7f012c15fc10 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0131c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f012c0ce4a0 0x7f012c0ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0131c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f012c0ce4a0 0x7f012c0ce930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35510/0 (socket says 172.21.15.86:35510) 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0131c00640 1 -- 172.21.15.86:0/3523730377 learned_addr learned my addr 172.21.15.86:0/3523730377 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f012be00640 1 --2- 172.21.15.86:0/3523730377 >> v2:172.21.15.194:3300/0 conn(0x7f012c15fff0 0x7f012c160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.117+0000 7f0131200640 1 --2- 172.21.15.86:0/3523730377 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.118+0000 7f0131c00640 1 -- 172.21.15.86:0/3523730377 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 msgr2=0x7f012c070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.118+0000 7f0131c00640 1 --2- 172.21.15.86:0/3523730377 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.118+0000 7f0131c00640 1 -- 172.21.15.86:0/3523730377 >> v2:172.21.15.194:3300/0 conn(0x7f012c15fff0 msgr2=0x7f012c160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.118+0000 7f0131c00640 1 --2- 172.21.15.86:0/3523730377 >> v2:172.21.15.194:3300/0 conn(0x7f012c15fff0 0x7f012c160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.118+0000 7f0131c00640 1 -- 172.21.15.86:0/3523730377 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f012c160990 con 0x7f012c0ce4a0 2024-03-06T07:27:55.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.119+0000 7f0131c00640 1 --2- 172.21.15.86:0/3523730377 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c0ce4a0 0x7f012c0ce930 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f012c0749e0 tx=0x7f012402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b3c34e4a6787869e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:55.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.119+0000 7f012b400640 1 -- 172.21.15.86:0/3523730377 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f012402ca70 con 0x7f012c0ce4a0 2024-03-06T07:27:55.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.119+0000 7f012b400640 1 -- 172.21.15.86:0/3523730377 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f012402cc10 con 0x7f012c0ce4a0 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.119+0000 7f012b400640 1 -- 172.21.15.86:0/3523730377 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01240343f0 con 0x7f012c0ce4a0 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 -- 172.21.15.86:0/3523730377 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c0ce4a0 msgr2=0x7f012c0ce930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 --2- 172.21.15.86:0/3523730377 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c0ce4a0 0x7f012c0ce930 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f012c0749e0 tx=0x7f012402bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 -- 172.21.15.86:0/3523730377 shutdown_connections 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 --2- 172.21.15.86:0/3523730377 >> v2:172.21.15.194:3300/0 conn(0x7f012c15fff0 0x7f012c160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 --2- 172.21.15.86:0/3523730377 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c0ce4a0 0x7f012c0ce930 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 --2- 172.21.15.86:0/3523730377 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 -- 172.21.15.86:0/3523730377 >> 172.21.15.86:0/3523730377 conn(0x7f012c0b1940 msgr2=0x7f012c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 -- 172.21.15.86:0/3523730377 shutdown_connections 2024-03-06T07:27:55.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.120+0000 7f0133c00640 1 -- 172.21.15.86:0/3523730377 wait complete. 2024-03-06T07:27:55.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.122+0000 7f0133c00640 1 Processor -- start 2024-03-06T07:27:55.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.122+0000 7f0133c00640 1 -- start start 2024-03-06T07:27:55.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0133c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f012c0ce4a0 0x7f012c144540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0133c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c09bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0133c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c15fff0 0x7f012c09cd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0133c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f012c072a30 con 0x7f012c15fff0 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0133c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f012c0728b0 con 0x7f012c0ce4a0 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0133c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f012c15c2e0 con 0x7f012c15fc10 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c15fff0 0x7f012c09cd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c15fff0 0x7f012c09cd20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35522/0 (socket says 172.21.15.86:35522) 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 -- 172.21.15.86:0/3906465203 learned_addr learned my addr 172.21.15.86:0/3906465203 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131200640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f012c0ce4a0 0x7f012c144540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f012be00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c09bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 -- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 msgr2=0x7f012c09bf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c09bf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 -- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f012c0ce4a0 msgr2=0x7f012c144540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f012c0ce4a0 0x7f012c144540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131c00640 1 -- 172.21.15.86:0/3906465203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f012c09ae80 con 0x7f012c15fff0 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f0131200640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f012c0ce4a0 0x7f012c144540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.123+0000 7f012be00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c09bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.124+0000 7f0131c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c15fff0 0x7f012c09cd20 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f0124009f40 tx=0x7f0124004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.124+0000 7f0128c00640 1 -- 172.21.15.86:0/3906465203 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0124043020 con 0x7f012c15fff0 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.124+0000 7f0128c00640 1 -- 172.21.15.86:0/3906465203 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f012403e450 con 0x7f012c15fff0 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.124+0000 7f0128c00640 1 -- 172.21.15.86:0/3906465203 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0124009510 con 0x7f012c15fff0 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.124+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f012c09b160 con 0x7f012c15fff0 2024-03-06T07:27:55.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.124+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f012c09b720 con 0x7f012c15fff0 2024-03-06T07:27:55.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.125+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f012c04f330 con 0x7f012c15fff0 2024-03-06T07:27:55.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.126+0000 7f0128c00640 1 -- 172.21.15.86:0/3906465203 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0124046020 con 0x7f012c15fff0 2024-03-06T07:27:55.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.127+0000 7f0128c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010c06b4d0 0x7f010c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.127+0000 7f0128c00640 1 -- 172.21.15.86:0/3906465203 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f01240acf10 con 0x7f012c15fff0 2024-03-06T07:27:55.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.127+0000 7f0131200640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010c06b4d0 0x7f010c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.128+0000 7f0131200640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010c06b4d0 0x7f010c06d990 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0120006fd0 tx=0x7f0120008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:55.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.131+0000 7f0128c00640 1 -- 172.21.15.86:0/3906465203 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f012404d020 con 0x7f012c15fff0 2024-03-06T07:27:55.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.331+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f012c156710 con 0x7f012c15fff0 2024-03-06T07:27:55.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.332+0000 7f0128c00640 1 -- 172.21.15.86:0/3906465203 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v24) v1 ==== 93+0+10 (secure 0 0 0) 0x7f012407bbe0 con 0x7f012c15fff0 2024-03-06T07:27:55.335 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:55.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.334+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010c06b4d0 msgr2=0x7f010c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.334+0000 7f0133c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010c06b4d0 0x7f010c06d990 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0120006fd0 tx=0x7f0120008040 comp rx=0 tx=0).stop 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.334+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c15fff0 msgr2=0x7f012c09cd20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.334+0000 7f0133c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c15fff0 0x7f012c09cd20 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f0124009f40 tx=0x7f0124004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 shutdown_connections 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010c06b4d0 0x7f010c06d990 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f012c15fff0 0x7f012c09cd20 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f012c15fc10 0x7f012c09bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 --2- 172.21.15.86:0/3906465203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f012c0ce4a0 0x7f012c144540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 >> 172.21.15.86:0/3906465203 conn(0x7f012c0b1940 msgr2=0x7f012c1569a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 shutdown_connections 2024-03-06T07:27:55.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.335+0000 7f0133c00640 1 -- 172.21.15.86:0/3906465203 wait complete. 2024-03-06T07:27:55.355 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:55.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.563+0000 7fa62d600640 1 Processor -- start 2024-03-06T07:27:55.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.563+0000 7fa62d600640 1 -- start start 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa62d600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa6280b3f70 0x7fa628160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa62d600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa628070ff0 0x7fa6280ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa62d600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa628071440 con 0x7fa62815fc10 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa62d600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6280b46c0 con 0x7fa628070ff0 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa62d600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6280b49a0 con 0x7fa6280b3f70 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa627400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa628070ff0 0x7fa6280ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa627400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa628070ff0 0x7fa6280ce4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35534/0 (socket says 172.21.15.86:35534) 2024-03-06T07:27:55.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa627400640 1 -- 172.21.15.86:0/2479768428 learned_addr learned my addr 172.21.15.86:0/2479768428 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:55.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa626000640 1 --2- 172.21.15.86:0/2479768428 >> v2:172.21.15.194:3300/0 conn(0x7fa6280b3f70 0x7fa628160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa627400640 1 -- 172.21.15.86:0/2479768428 >> v2:172.21.15.194:3300/0 conn(0x7fa6280b3f70 msgr2=0x7fa628160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:55.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.565+0000 7fa627400640 1 --2- 172.21.15.86:0/2479768428 >> v2:172.21.15.194:3300/0 conn(0x7fa6280b3f70 0x7fa628160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.566+0000 7fa627400640 1 -- 172.21.15.86:0/2479768428 >> v1:172.21.15.194:6789/0 conn(0x7fa62815fc10 legacy=0x7fa62815fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:55.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.566+0000 7fa627400640 1 -- 172.21.15.86:0/2479768428 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa628072c90 con 0x7fa628070ff0 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa627400640 1 --2- 172.21.15.86:0/2479768428 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa628070ff0 0x7fa6280ce4a0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fa628074a50 tx=0x7fa62002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c94bf8140feb50c0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa625600640 1 -- 172.21.15.86:0/2479768428 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa62002ca80 con 0x7fa628070ff0 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa625600640 1 -- 172.21.15.86:0/2479768428 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa62002cc20 con 0x7fa628070ff0 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa625600640 1 -- 172.21.15.86:0/2479768428 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6200343f0 con 0x7fa628070ff0 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa62d600640 1 -- 172.21.15.86:0/2479768428 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa628070ff0 msgr2=0x7fa6280ce4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa62d600640 1 --2- 172.21.15.86:0/2479768428 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa628070ff0 0x7fa6280ce4a0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fa628074a50 tx=0x7fa62002bee0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa62d600640 1 -- 172.21.15.86:0/2479768428 shutdown_connections 2024-03-06T07:27:55.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa62d600640 1 --2- 172.21.15.86:0/2479768428 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa628070ff0 0x7fa6280ce4a0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa62d600640 1 --2- 172.21.15.86:0/2479768428 >> v2:172.21.15.194:3300/0 conn(0x7fa6280b3f70 0x7fa628160700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.567+0000 7fa62d600640 1 -- 172.21.15.86:0/2479768428 >> 172.21.15.86:0/2479768428 conn(0x7fa6280b1940 msgr2=0x7fa6280b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:55.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.568+0000 7fa62d600640 1 -- 172.21.15.86:0/2479768428 shutdown_connections 2024-03-06T07:27:55.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.568+0000 7fa62d600640 1 -- 172.21.15.86:0/2479768428 wait complete. 2024-03-06T07:27:55.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.569+0000 7fa62d600640 1 Processor -- start 2024-03-06T07:27:55.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.569+0000 7fa62d600640 1 -- start start 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa62d600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa628070ff0 0x7fa62815acc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa62d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6280b3f70 0x7fa62814b6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa62d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62815fc10 0x7fa6281473b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa62d600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa628072a30 con 0x7fa6280b3f70 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa62d600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6280728b0 con 0x7fa628070ff0 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa62d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa62815c2e0 con 0x7fa62815fc10 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa626000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6280b3f70 0x7fa62814b6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa626000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6280b3f70 0x7fa62814b6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35538/0 (socket says 172.21.15.86:35538) 2024-03-06T07:27:55.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.570+0000 7fa626000640 1 -- 172.21.15.86:0/3166205064 learned_addr learned my addr 172.21.15.86:0/3166205064 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:55.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa626000640 1 -- 172.21.15.86:0/3166205064 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62815fc10 msgr2=0x7fa6281473b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa626000640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62815fc10 0x7fa6281473b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa626000640 1 -- 172.21.15.86:0/3166205064 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa628070ff0 msgr2=0x7fa62815acc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa626000640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa628070ff0 0x7fa62815acc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa626000640 1 -- 172.21.15.86:0/3166205064 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa628146950 con 0x7fa6280b3f70 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa626000640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6280b3f70 0x7fa62814b6d0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fa61400d7d0 tx=0x7fa61400dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa61ea00640 1 -- 172.21.15.86:0/3166205064 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa61400b6c0 con 0x7fa6280b3f70 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa61ea00640 1 -- 172.21.15.86:0/3166205064 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa614019070 con 0x7fa6280b3f70 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa61ea00640 1 -- 172.21.15.86:0/3166205064 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa61400ba80 con 0x7fa6280b3f70 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa62814af20 con 0x7fa6280b3f70 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.571+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa628146bd0 con 0x7fa6280b3f70 2024-03-06T07:27:55.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.573+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa62804f330 con 0x7fa6280b3f70 2024-03-06T07:27:55.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.573+0000 7fa61ea00640 1 -- 172.21.15.86:0/3166205064 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa61400bc20 con 0x7fa6280b3f70 2024-03-06T07:27:55.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.574+0000 7fa61ea00640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b500 0x7fa60806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:55.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.574+0000 7fa61ea00640 1 -- 172.21.15.86:0/3166205064 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7fa61408a6f0 con 0x7fa6280b3f70 2024-03-06T07:27:55.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.575+0000 7fa626a00640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b500 0x7fa60806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:55.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.575+0000 7fa626a00640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b500 0x7fa60806d9c0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fa618006fd0 tx=0x7fa618006bc0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:55.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.579+0000 7fa61ea00640 1 -- 172.21.15.86:0/3166205064 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa628146950 con 0x7fa6280b3f70 2024-03-06T07:27:55.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.775+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fa62806e9b0 con 0x7fa6280b3f70 2024-03-06T07:27:55.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.776+0000 7fa61ea00640 1 -- 172.21.15.86:0/3166205064 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v24) v1 ==== 104+0+11 (secure 0 0 0) 0x7fa6140594f0 con 0x7fa6280b3f70 2024-03-06T07:27:55.779 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b500 msgr2=0x7fa60806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b500 0x7fa60806d9c0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fa618006fd0 tx=0x7fa618006bc0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6280b3f70 msgr2=0x7fa62814b6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6280b3f70 0x7fa62814b6d0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fa61400d7d0 tx=0x7fa61400dc90 comp rx=0 tx=0).stop 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 shutdown_connections 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b500 0x7fa60806d9c0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62815fc10 0x7fa6281473b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6280b3f70 0x7fa62814b6d0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 --2- 172.21.15.86:0/3166205064 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa628070ff0 0x7fa62815acc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:55.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 >> 172.21.15.86:0/3166205064 conn(0x7fa6280b1940 msgr2=0x7fa62815fff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:55.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 shutdown_connections 2024-03-06T07:27:55.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:55.779+0000 7fa62d600640 1 -- 172.21.15.86:0/3166205064 wait complete. 2024-03-06T07:27:55.800 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:56.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.006+0000 7ff4f1400640 1 Processor -- start 2024-03-06T07:27:56.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.007+0000 7ff4f1400640 1 -- start start 2024-03-06T07:27:56.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.009+0000 7ff4f1400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff4ec15fc10 0x7ff4ec070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.009+0000 7ff4f1400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4ec0b46c0 con 0x7ff4ec15fff0 2024-03-06T07:27:56.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.009+0000 7ff4f1400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4ec0b49a0 con 0x7ff4ec0b3f70 2024-03-06T07:27:56.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.009+0000 7ff4f1400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff4ec072c90 con 0x7ff4ec15fc10 2024-03-06T07:27:56.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.009+0000 7ff4e9e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff4ec15fff0 0x7ff4ec160450 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39776/0 (socket says 172.21.15.86:39776) 2024-03-06T07:27:56.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.009+0000 7ff4e9e00640 1 -- 172.21.15.86:0/8379860 learned_addr learned my addr 172.21.15.86:0/8379860 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:27:56.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.010+0000 7ff4ea800640 1 --2- 172.21.15.86:0/8379860 >> v2:172.21.15.194:3300/0 conn(0x7ff4ec15fc10 0x7ff4ec070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.010+0000 7ff4e9400640 1 -- 172.21.15.86:0/8379860 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2503572988 0 0) 0x7ff4ec0b46c0 con 0x7ff4ec15fff0 2024-03-06T07:27:56.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.010+0000 7ff4e9400640 1 -- 172.21.15.86:0/8379860 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff4d0003520 con 0x7ff4ec15fff0 2024-03-06T07:27:56.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.011+0000 7ff4e9400640 1 -- 172.21.15.86:0/8379860 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 736267136 0 0) 0x7ff4d0003520 con 0x7ff4ec15fff0 2024-03-06T07:27:56.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.011+0000 7ff4ea800640 1 -- 172.21.15.86:0/8379860 >> v1:172.21.15.194:6789/0 conn(0x7ff4ec0b3f70 legacy=0x7ff4ec0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.011+0000 7ff4ea800640 1 -- 172.21.15.86:0/8379860 >> v1:172.21.15.86:6789/0 conn(0x7ff4ec15fff0 legacy=0x7ff4ec160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.011+0000 7ff4ea800640 1 -- 172.21.15.86:0/8379860 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4ec09dd80 con 0x7ff4ec15fc10 2024-03-06T07:27:56.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.012+0000 7ff4ea800640 1 --2- 172.21.15.86:0/8379860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec070ff0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7ff4d40097a0 tx=0x7ff4d402b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1371e41066a0b3de server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:56.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.013+0000 7ff4e9400640 1 -- 172.21.15.86:0/8379860 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4d4038070 con 0x7ff4ec15fc10 2024-03-06T07:27:56.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.013+0000 7ff4e9400640 1 -- 172.21.15.86:0/8379860 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4d4031500 con 0x7ff4ec15fc10 2024-03-06T07:27:56.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.013+0000 7ff4e9400640 1 -- 172.21.15.86:0/8379860 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4dc001670 con 0x7ff4ec15fc10 2024-03-06T07:27:56.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.013+0000 7ff4f1400640 1 -- 172.21.15.86:0/8379860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec15fc10 msgr2=0x7ff4ec070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.013+0000 7ff4f1400640 1 --2- 172.21.15.86:0/8379860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec070ff0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7ff4d40097a0 tx=0x7ff4d402b820 comp rx=0 tx=0).stop 2024-03-06T07:27:56.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.014+0000 7ff4f1400640 1 -- 172.21.15.86:0/8379860 shutdown_connections 2024-03-06T07:27:56.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.014+0000 7ff4f1400640 1 --2- 172.21.15.86:0/8379860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec070ff0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.014+0000 7ff4f1400640 1 -- 172.21.15.86:0/8379860 >> 172.21.15.86:0/8379860 conn(0x7ff4ec0b1940 msgr2=0x7ff4ec0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:56.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.014+0000 7ff4f1400640 1 -- 172.21.15.86:0/8379860 shutdown_connections 2024-03-06T07:27:56.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.014+0000 7ff4f1400640 1 -- 172.21.15.86:0/8379860 wait complete. 2024-03-06T07:27:56.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.015+0000 7ff4f1400640 1 Processor -- start 2024-03-06T07:27:56.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.015+0000 7ff4f1400640 1 -- start start 2024-03-06T07:27:56.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.016+0000 7ff4f1400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec0b3f70 0x7ff4ec152690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.016+0000 7ff4f1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec0cb2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.016+0000 7ff4f1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4ec15fff0 0x7ff4ec0cb810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.016+0000 7ff4f1400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4ec15c460 con 0x7ff4ec15fc10 2024-03-06T07:27:56.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.016+0000 7ff4f1400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4ec15c2e0 con 0x7ff4ec0b3f70 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.016+0000 7ff4f1400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4ec0cebb0 con 0x7ff4ec15fff0 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec0cb2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec0cb2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50990/0 (socket says 172.21.15.86:50990) 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 -- 172.21.15.86:0/652917985 learned_addr learned my addr 172.21.15.86:0/652917985 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 -- 172.21.15.86:0/652917985 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4ec15fff0 msgr2=0x7ff4ec0cb810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4ea800640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec0b3f70 0x7ff4ec152690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4ec15fff0 0x7ff4ec0cb810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 -- 172.21.15.86:0/652917985 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec0b3f70 msgr2=0x7ff4ec152690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec0b3f70 0x7ff4ec152690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 -- 172.21.15.86:0/652917985 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4ec152bd0 con 0x7ff4ec15fc10 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e9e00640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec0cb2d0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff4dc00c700 tx=0x7ff4dc00cbc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e2a00640 1 -- 172.21.15.86:0/652917985 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4dc005ad0 con 0x7ff4ec15fc10 2024-03-06T07:27:56.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.017+0000 7ff4e2a00640 1 -- 172.21.15.86:0/652917985 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4dc005c70 con 0x7ff4ec15fc10 2024-03-06T07:27:56.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.018+0000 7ff4e2a00640 1 -- 172.21.15.86:0/652917985 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4dc007290 con 0x7ff4ec15fc10 2024-03-06T07:27:56.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.018+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4ec155600 con 0x7ff4ec15fc10 2024-03-06T07:27:56.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.018+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4ec155bc0 con 0x7ff4ec15fc10 2024-03-06T07:27:56.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.019+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4ec04f330 con 0x7ff4ec15fc10 2024-03-06T07:27:56.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.020+0000 7ff4e2a00640 1 -- 172.21.15.86:0/652917985 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff4dc007430 con 0x7ff4ec15fc10 2024-03-06T07:27:56.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.021+0000 7ff4e2a00640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4c806b4d0 0x7ff4c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.021+0000 7ff4e2a00640 1 -- 172.21.15.86:0/652917985 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7ff4dc089fb0 con 0x7ff4ec15fc10 2024-03-06T07:27:56.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.021+0000 7ff4ea800640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4c806b4d0 0x7ff4c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.022+0000 7ff4ea800640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4c806b4d0 0x7ff4c806d990 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7ff4d4035000 tx=0x7ff4d4008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:56.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.025+0000 7ff4e2a00640 1 -- 172.21.15.86:0/652917985 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff4dc010300 con 0x7ff4ec15fc10 2024-03-06T07:27:56.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.222+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7ff4ec06ef30 con 0x7ff4ec15fc10 2024-03-06T07:27:56.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.223+0000 7ff4e2a00640 1 -- 172.21.15.86:0/652917985 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v24) v1 ==== 100+0+11 (secure 0 0 0) 0x7ff4dc058e40 con 0x7ff4ec15fc10 2024-03-06T07:27:56.226 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:56.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4c806b4d0 msgr2=0x7ff4c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4c806b4d0 0x7ff4c806d990 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7ff4d4035000 tx=0x7ff4d4008030 comp rx=0 tx=0).stop 2024-03-06T07:27:56.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4ec15fc10 msgr2=0x7ff4ec0cb2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec0cb2d0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff4dc00c700 tx=0x7ff4dc00cbc0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 shutdown_connections 2024-03-06T07:27:56.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff4c806b4d0 0x7ff4c806d990 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4ec15fff0 0x7ff4ec0cb810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4ec15fc10 0x7ff4ec0cb2d0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 --2- 172.21.15.86:0/652917985 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4ec0b3f70 0x7ff4ec152690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.226+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 >> 172.21.15.86:0/652917985 conn(0x7ff4ec0b1940 msgr2=0x7ff4ec0ae2e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:56.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.227+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 shutdown_connections 2024-03-06T07:27:56.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.227+0000 7ff4f1400640 1 -- 172.21.15.86:0/652917985 wait complete. 2024-03-06T07:27:56.248 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:27:56.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.458+0000 7f8c39000640 1 Processor -- start 2024-03-06T07:27:56.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.458+0000 7f8c39000640 1 -- start start 2024-03-06T07:27:56.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c39000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8c340c4930 0x7f8c340c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c39000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340c5170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c39000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8c34145b30 con 0x7f8c340c56b0 2024-03-06T07:27:56.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c39000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8c34142ce0 con 0x7f8c340c4930 2024-03-06T07:27:56.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c39000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c340cbca0 con 0x7f8c340c4d10 2024-03-06T07:27:56.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c31e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8c340c4930 0x7f8c340c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c31e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8c340c4930 0x7f8c340c6460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:49028/0 (socket says 172.21.15.86:49028) 2024-03-06T07:27:56.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c31e00640 1 -- 172.21.15.86:0/266457193 learned_addr learned my addr 172.21.15.86:0/266457193 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:56.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.460+0000 7f8c2be00640 1 --2- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340c5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.461+0000 7f8c31e00640 1 -- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 msgr2=0x7f8c340c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:56.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.461+0000 7f8c31e00640 1 --2- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340c5170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.461+0000 7f8c31e00640 1 -- 172.21.15.86:0/266457193 >> v1:172.21.15.194:6789/0 conn(0x7f8c340c56b0 legacy=0x7f8c3406d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.461+0000 7f8c31e00640 1 -- 172.21.15.86:0/266457193 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c340cbf80 con 0x7f8c340c4930 2024-03-06T07:27:56.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.461+0000 7f8c2be00640 1 --2- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:27:56.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.461+0000 7f8c31400640 1 -- 172.21.15.86:0/266457193 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2636371543 0 0) 0x7f8c2c001490 con 0x7f8c340c56b0 2024-03-06T07:27:56.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.463+0000 7f8c31e00640 1 --2- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c4930 0x7f8c340c6460 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f8c24009870 tx=0x7f8c2402b8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=24f972ae55cad985 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:56.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.463+0000 7f8c31400640 1 -- 172.21.15.86:0/266457193 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c24035020 con 0x7f8c340c4930 2024-03-06T07:27:56.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.463+0000 7f8c31400640 1 -- 172.21.15.86:0/266457193 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c24031640 con 0x7f8c340c4930 2024-03-06T07:27:56.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.463+0000 7f8c31400640 1 -- 172.21.15.86:0/266457193 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c2402d040 con 0x7f8c340c4930 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 -- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c4930 msgr2=0x7f8c340c6460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 --2- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c4930 0x7f8c340c6460 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f8c24009870 tx=0x7f8c2402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 -- 172.21.15.86:0/266457193 shutdown_connections 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 --2- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 --2- 172.21.15.86:0/266457193 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c4930 0x7f8c340c6460 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 -- 172.21.15.86:0/266457193 >> 172.21.15.86:0/266457193 conn(0x7f8c340cc4f0 msgr2=0x7f8c340cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 -- 172.21.15.86:0/266457193 shutdown_connections 2024-03-06T07:27:56.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.464+0000 7f8c39000640 1 -- 172.21.15.86:0/266457193 wait complete. 2024-03-06T07:27:56.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.466+0000 7f8c39000640 1 Processor -- start 2024-03-06T07:27:56.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.466+0000 7f8c39000640 1 -- start start 2024-03-06T07:27:56.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.467+0000 7f8c39000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340c4930 0x7f8c34141c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.467+0000 7f8c39000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340b3200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.467+0000 7f8c39000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c56b0 0x7f8c340b3740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.467+0000 7f8c39000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c340cc260 con 0x7f8c340c4930 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.467+0000 7f8c39000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c340ca3b0 con 0x7f8c340c56b0 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.467+0000 7f8c39000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c340c5b40 con 0x7f8c340c4d10 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340c4930 0x7f8c34141c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340c4930 0x7f8c34141c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50992/0 (socket says 172.21.15.86:50992) 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 -- 172.21.15.86:0/3413370807 learned_addr learned my addr 172.21.15.86:0/3413370807 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c32800640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c56b0 0x7f8c340b3740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 -- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 msgr2=0x7f8c340b3200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c2be00640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340b3200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340b3200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 -- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c56b0 msgr2=0x7f8c340b3740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c56b0 0x7f8c340b3740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c31e00640 1 -- 172.21.15.86:0/3413370807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c340b40f0 con 0x7f8c340c4930 2024-03-06T07:27:56.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.468+0000 7f8c32800640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c56b0 0x7f8c340b3740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.469+0000 7f8c31e00640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340c4930 0x7f8c34141c00 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f8c240090b0 tx=0x7f8c240046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.469+0000 7f8c2a000640 1 -- 172.21.15.86:0/3413370807 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c24040070 con 0x7f8c340c4930 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.469+0000 7f8c2a000640 1 -- 172.21.15.86:0/3413370807 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c240399f0 con 0x7f8c340c4930 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.469+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c340ce840 con 0x7f8c340c4930 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.469+0000 7f8c2a000640 1 -- 172.21.15.86:0/3413370807 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c24035040 con 0x7f8c340c4930 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.469+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c340cee00 con 0x7f8c340c4930 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.470+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c3406e8c0 con 0x7f8c340c4930 2024-03-06T07:27:56.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.471+0000 7f8c2a000640 1 -- 172.21.15.86:0/3413370807 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8c24038020 con 0x7f8c340c4930 2024-03-06T07:27:56.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.472+0000 7f8c2a000640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c1006b4d0 0x7f8c1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:56.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.472+0000 7f8c2a000640 1 -- 172.21.15.86:0/3413370807 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f8c240abbd0 con 0x7f8c340c4930 2024-03-06T07:27:56.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.472+0000 7f8c2be00640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c1006b4d0 0x7f8c1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:56.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.473+0000 7f8c2be00640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c1006b4d0 0x7f8c1006d990 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f8c1c009560 tx=0x7f8c1c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:56.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.476+0000 7f8c2a000640 1 -- 172.21.15.86:0/3413370807 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8c3406e8c0 con 0x7f8c340c4930 2024-03-06T07:27:56.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.681+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f8c340cf500 con 0x7f8c340c4930 2024-03-06T07:27:56.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.682+0000 7f8c2a000640 1 -- 172.21.15.86:0/3413370807 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v24) v1 ==== 80+0+0 (secure 0 0 0) 0x7f8c2407a9d0 con 0x7f8c340c4930 2024-03-06T07:27:56.684 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:27:56.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c1006b4d0 msgr2=0x7f8c1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c1006b4d0 0x7f8c1006d990 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f8c1c009560 tx=0x7f8c1c008040 comp rx=0 tx=0).stop 2024-03-06T07:27:56.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340c4930 msgr2=0x7f8c34141c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:56.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340c4930 0x7f8c34141c00 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f8c240090b0 tx=0x7f8c240046c0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 shutdown_connections 2024-03-06T07:27:56.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c1006b4d0 0x7f8c1006d990 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c340c56b0 0x7f8c340b3740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340c4d10 0x7f8c340b3200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 --2- 172.21.15.86:0/3413370807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340c4930 0x7f8c34141c00 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:56.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.684+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 >> 172.21.15.86:0/3413370807 conn(0x7f8c340cc4f0 msgr2=0x7f8c341e52d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:56.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.685+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 shutdown_connections 2024-03-06T07:27:56.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:56.685+0000 7f8c39000640 1 -- 172.21.15.86:0/3413370807 wait complete. 2024-03-06T07:27:56.707 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.3... 2024-03-06T07:27:56.707 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:56.707 DEBUG:teuthology.orchestra.run.smithi194:> sudo umount /home/ubuntu/cephtest/mnt.3 -f 2024-03-06T07:27:56.809 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi194.front.sepia.ceph.com 2024-03-06T07:27:56.809 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:27:56.809 DEBUG:teuthology.orchestra.run.smithi194:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-06T07:27:56.856 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:27:56.856 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:57.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.070+0000 7f0b8d400640 1 Processor -- start 2024-03-06T07:27:57.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.070+0000 7f0b8d400640 1 -- start start 2024-03-06T07:27:57.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.072+0000 7f0b8d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b88144c00 0x7f0b8806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.072+0000 7f0b8d400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0b8815f9d0 0x7f0b8815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.072+0000 7f0b8d400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0b8806ee70 con 0x7f0b880ce0a0 2024-03-06T07:27:57.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.072+0000 7f0b8d400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0b88160370 con 0x7f0b8815f9d0 2024-03-06T07:27:57.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.072+0000 7f0b8d400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b8806cef0 con 0x7f0b88144c00 2024-03-06T07:27:57.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b86800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b88144c00 0x7f0b8806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b7de00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0b8815f9d0 0x7f0b8815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b86800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b88144c00 0x7f0b8806e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:46048/0 (socket says 172.21.15.86:46048) 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b86800640 1 -- 172.21.15.86:0/702331645 learned_addr learned my addr 172.21.15.86:0/702331645 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b86800640 1 -- 172.21.15.86:0/702331645 >> v2:172.21.15.194:3300/0 conn(0x7f0b8815f9d0 msgr2=0x7f0b8815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b86800640 1 --2- 172.21.15.86:0/702331645 >> v2:172.21.15.194:3300/0 conn(0x7f0b8815f9d0 0x7f0b8815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b86800640 1 -- 172.21.15.86:0/702331645 >> v1:172.21.15.194:6789/0 conn(0x7f0b880ce0a0 legacy=0x7f0b880ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.073+0000 7f0b86800640 1 -- 172.21.15.86:0/702331645 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b8806d1d0 con 0x7f0b88144c00 2024-03-06T07:27:57.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.074+0000 7f0b86800640 1 --2- 172.21.15.86:0/702331645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b88144c00 0x7f0b8806e930 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f0b78009870 tx=0x7f0b7802b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4336098794806335 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:57.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.075+0000 7f0b85e00640 1 -- 172.21.15.86:0/702331645 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b78034070 con 0x7f0b88144c00 2024-03-06T07:27:57.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.075+0000 7f0b85e00640 1 -- 172.21.15.86:0/702331645 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b78030840 con 0x7f0b88144c00 2024-03-06T07:27:57.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.075+0000 7f0b85e00640 1 -- 172.21.15.86:0/702331645 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b78030c10 con 0x7f0b88144c00 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.075+0000 7f0b8d400640 1 -- 172.21.15.86:0/702331645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b88144c00 msgr2=0x7f0b8806e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.075+0000 7f0b8d400640 1 --2- 172.21.15.86:0/702331645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b88144c00 0x7f0b8806e930 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f0b78009870 tx=0x7f0b7802b980 comp rx=0 tx=0).stop 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.076+0000 7f0b8d400640 1 -- 172.21.15.86:0/702331645 shutdown_connections 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.076+0000 7f0b8d400640 1 --2- 172.21.15.86:0/702331645 >> v2:172.21.15.194:3300/0 conn(0x7f0b8815f9d0 0x7f0b8815fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.076+0000 7f0b8d400640 1 --2- 172.21.15.86:0/702331645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b88144c00 0x7f0b8806e930 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.076+0000 7f0b8d400640 1 -- 172.21.15.86:0/702331645 >> 172.21.15.86:0/702331645 conn(0x7f0b880b1730 msgr2=0x7f0b880b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.076+0000 7f0b8d400640 1 -- 172.21.15.86:0/702331645 shutdown_connections 2024-03-06T07:27:57.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.076+0000 7f0b8d400640 1 -- 172.21.15.86:0/702331645 wait complete. 2024-03-06T07:27:57.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.078+0000 7f0b8d400640 1 Processor -- start 2024-03-06T07:27:57.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.078+0000 7f0b8d400640 1 -- start start 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b8d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b880ce0a0 0x7f0b881f1320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b8d400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b88144c00 0x7f0b8813a4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b8d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b8815f9d0 0x7f0b8815af60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b8d400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b88160730 con 0x7f0b880ce0a0 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b8d400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b8806d790 con 0x7f0b88144c00 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b8d400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b8806cb50 con 0x7f0b8815f9d0 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b86800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b880ce0a0 0x7f0b881f1320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b86800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b880ce0a0 0x7f0b881f1320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51004/0 (socket says 172.21.15.86:51004) 2024-03-06T07:27:57.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.079+0000 7f0b86800640 1 -- 172.21.15.86:0/2951417269 learned_addr learned my addr 172.21.15.86:0/2951417269 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:57.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b87200640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b8815f9d0 0x7f0b8815af60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b7de00640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b88144c00 0x7f0b8813a4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b86800640 1 -- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b8815f9d0 msgr2=0x7f0b8815af60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b86800640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b8815f9d0 0x7f0b8815af60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b86800640 1 -- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b88144c00 msgr2=0x7f0b8813a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b86800640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b88144c00 0x7f0b8813a4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b86800640 1 -- 172.21.15.86:0/2951417269 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b8815e130 con 0x7f0b880ce0a0 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.080+0000 7f0b86800640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b880ce0a0 0x7f0b881f1320 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f0b7802b9c0 tx=0x7f0b78004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.081+0000 7f0b7f400640 1 -- 172.21.15.86:0/2951417269 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b78048070 con 0x7f0b880ce0a0 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.081+0000 7f0b7f400640 1 -- 172.21.15.86:0/2951417269 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b78030640 con 0x7f0b880ce0a0 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.081+0000 7f0b7f400640 1 -- 172.21.15.86:0/2951417269 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b78034040 con 0x7f0b880ce0a0 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.081+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0b8815e410 con 0x7f0b880ce0a0 2024-03-06T07:27:57.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.081+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0b88139fa0 con 0x7f0b880ce0a0 2024-03-06T07:27:57.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.082+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0b8804f330 con 0x7f0b880ce0a0 2024-03-06T07:27:57.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.083+0000 7f0b7f400640 1 -- 172.21.15.86:0/2951417269 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0b7804b070 con 0x7f0b880ce0a0 2024-03-06T07:27:57.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.084+0000 7f0b7f400640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b6406b4d0 0x7f0b6406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.084+0000 7f0b7f400640 1 -- 172.21.15.86:0/2951417269 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f0b780ac1e0 con 0x7f0b880ce0a0 2024-03-06T07:27:57.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.084+0000 7f0b7de00640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b6406b4d0 0x7f0b6406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.085+0000 7f0b7de00640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b6406b4d0 0x7f0b6406d990 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f0b8813fa20 tx=0x7f0b70008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:57.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.088+0000 7f0b7f400640 1 -- 172.21.15.86:0/2951417269 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0b7807b170 con 0x7f0b880ce0a0 2024-03-06T07:27:57.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.326+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0b88072890 con 0x7f0b880ce0a0 2024-03-06T07:27:57.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.327+0000 7f0b7f400640 1 -- 172.21.15.86:0/2951417269 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f0b7807aa70 con 0x7f0b880ce0a0 2024-03-06T07:27:57.330 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:57.330 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:27:57.330 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b6406b4d0 msgr2=0x7f0b6406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b6406b4d0 0x7f0b6406d990 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f0b8813fa20 tx=0x7f0b70008040 comp rx=0 tx=0).stop 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b880ce0a0 msgr2=0x7f0b881f1320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b880ce0a0 0x7f0b881f1320 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f0b7802b9c0 tx=0x7f0b78004730 comp rx=0 tx=0).stop 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 shutdown_connections 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b6406b4d0 0x7f0b6406d990 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b8815f9d0 0x7f0b8815af60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b88144c00 0x7f0b8813a4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 --2- 172.21.15.86:0/2951417269 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b880ce0a0 0x7f0b881f1320 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 >> 172.21.15.86:0/2951417269 conn(0x7f0b880b1730 msgr2=0x7f0b880b01c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:57.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 shutdown_connections 2024-03-06T07:27:57.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.330+0000 7f0b8d400640 1 -- 172.21.15.86:0/2951417269 wait complete. 2024-03-06T07:27:57.352 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:57.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.553+0000 7efecc800640 1 Processor -- start 2024-03-06T07:27:57.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.553+0000 7efecc800640 1 -- start start 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efecc800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efecc800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 0x7efec415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efecc800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efec40ce0a0 0x7efec40ce4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efecc800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efec4160370 con 0x7efec40ce0a0 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efecc800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efec406ee70 con 0x7efec4144c00 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efecc800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efec406cef0 con 0x7efec415f9d0 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efeca800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efec40ce0a0 0x7efec40ce4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efeca800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efec40ce0a0 0x7efec40ce4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51008/0 (socket says 172.21.15.86:51008) 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efeca800640 1 -- 172.21.15.86:0/2128755656 learned_addr learned my addr 172.21.15.86:0/2128755656 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efec9400640 1 --2- 172.21.15.86:0/2128755656 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 0x7efec415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efec9e00640 1 --2- 172.21.15.86:0/2128755656 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efeca800640 1 -- 172.21.15.86:0/2128755656 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 msgr2=0x7efec415fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efeca800640 1 --2- 172.21.15.86:0/2128755656 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 0x7efec415fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efeca800640 1 -- 172.21.15.86:0/2128755656 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 msgr2=0x7efec406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.555+0000 7efeca800640 1 --2- 172.21.15.86:0/2128755656 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.556+0000 7efeca800640 1 -- 172.21.15.86:0/2128755656 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efec406d1d0 con 0x7efec40ce0a0 2024-03-06T07:27:57.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.557+0000 7efeca800640 1 --2- 172.21.15.86:0/2128755656 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec40ce4f0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7efec40b25f0 tx=0x7efec002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44dfed9c7f5311fc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:57.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.557+0000 7efec8a00640 1 -- 172.21.15.86:0/2128755656 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec002ca70 con 0x7efec40ce0a0 2024-03-06T07:27:57.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.557+0000 7efec8a00640 1 -- 172.21.15.86:0/2128755656 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efec002cc10 con 0x7efec40ce0a0 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.557+0000 7efec8a00640 1 -- 172.21.15.86:0/2128755656 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec00343f0 con 0x7efec40ce0a0 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 -- 172.21.15.86:0/2128755656 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 msgr2=0x7efec40ce4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 --2- 172.21.15.86:0/2128755656 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec40ce4f0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7efec40b25f0 tx=0x7efec002bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 -- 172.21.15.86:0/2128755656 shutdown_connections 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 --2- 172.21.15.86:0/2128755656 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec40ce4f0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 --2- 172.21.15.86:0/2128755656 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 0x7efec415fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 --2- 172.21.15.86:0/2128755656 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 -- 172.21.15.86:0/2128755656 >> 172.21.15.86:0/2128755656 conn(0x7efec40b1730 msgr2=0x7efec40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 -- 172.21.15.86:0/2128755656 shutdown_connections 2024-03-06T07:27:57.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.558+0000 7efecc800640 1 -- 172.21.15.86:0/2128755656 wait complete. 2024-03-06T07:27:57.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.560+0000 7efecc800640 1 Processor -- start 2024-03-06T07:27:57.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.560+0000 7efecc800640 1 -- start start 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efecc800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec40ce0a0 0x7efec4139b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efecc800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec4144c00 0x7efec41557a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efecc800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec4158cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efecc800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec4160730 con 0x7efec415f9d0 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efecc800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec406d790 con 0x7efec40ce0a0 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efecc800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efec406cb50 con 0x7efec4144c00 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec4158cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec4158cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51014/0 (socket says 172.21.15.86:51014) 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 -- 172.21.15.86:0/1750561397 learned_addr learned my addr 172.21.15.86:0/1750561397 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efec9e00640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec40ce0a0 0x7efec4139b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efec9400640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec4144c00 0x7efec41557a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 -- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec4144c00 msgr2=0x7efec41557a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec4144c00 0x7efec41557a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 -- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec40ce0a0 msgr2=0x7efec4139b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec40ce0a0 0x7efec4139b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efeca800640 1 -- 172.21.15.86:0/1750561397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efec413a220 con 0x7efec415f9d0 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.561+0000 7efec9400640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec4144c00 0x7efec41557a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.562+0000 7efeca800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec4158cb0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7efec0009f40 tx=0x7efec0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.562+0000 7efebe000640 1 -- 172.21.15.86:0/1750561397 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec0043020 con 0x7efec415f9d0 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.562+0000 7efebe000640 1 -- 172.21.15.86:0/1750561397 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efec003e450 con 0x7efec415f9d0 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.562+0000 7efebe000640 1 -- 172.21.15.86:0/1750561397 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec0009510 con 0x7efec415f9d0 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.562+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efec413a500 con 0x7efec415f9d0 2024-03-06T07:27:57.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.562+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efec413aac0 con 0x7efec415f9d0 2024-03-06T07:27:57.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.564+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efec404f330 con 0x7efec415f9d0 2024-03-06T07:27:57.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.564+0000 7efebe000640 1 -- 172.21.15.86:0/1750561397 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efec0046020 con 0x7efec415f9d0 2024-03-06T07:27:57.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.565+0000 7efebe000640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea806b4d0 0x7efea806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:57.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.565+0000 7efebe000640 1 -- 172.21.15.86:0/1750561397 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7efec00aca20 con 0x7efec415f9d0 2024-03-06T07:27:57.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.566+0000 7efec9e00640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea806b4d0 0x7efea806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:57.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.566+0000 7efec9e00640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea806b4d0 0x7efea806d990 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7efeb8006fd0 tx=0x7efeb8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:57.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.570+0000 7efebe000640 1 -- 172.21.15.86:0/1750561397 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efec004d030 con 0x7efec415f9d0 2024-03-06T07:27:57.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.764+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7efec4072890 con 0x7efec415f9d0 2024-03-06T07:27:57.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.766+0000 7efebe000640 1 -- 172.21.15.86:0/1750561397 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v24) v1 ==== 74+0+16885 (secure 0 0 0) 0x7efec007b8b0 con 0x7efec415f9d0 2024-03-06T07:27:57.769 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:57.769 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":24,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:53.981490+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.16","mappings":[{"from":5,"to":2},{"from":7,"to":1}]},{"pgid":"3.24","mappings":[{"from":3,"to":1}]},{"pgid":"3.28","mappings":[{"from":5,"to":2}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:27:57.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.768+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea806b4d0 msgr2=0x7efea806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.768+0000 7efecc800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea806b4d0 0x7efea806d990 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7efeb8006fd0 tx=0x7efeb8008040 comp rx=0 tx=0).stop 2024-03-06T07:27:57.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 msgr2=0x7efec4158cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:57.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec4158cb0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7efec0009f40 tx=0x7efec0004a40 comp rx=0 tx=0).stop 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 shutdown_connections 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea806b4d0 0x7efea806d990 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec4158cb0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec4144c00 0x7efec41557a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 --2- 172.21.15.86:0/1750561397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec40ce0a0 0x7efec4139b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 >> 172.21.15.86:0/1750561397 conn(0x7efec40b1730 msgr2=0x7efec40affe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 shutdown_connections 2024-03-06T07:27:57.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:57.769+0000 7efecc800640 1 -- 172.21.15.86:0/1750561397 wait complete. 2024-03-06T07:27:57.790 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:27:05.106298+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:27:06.363974+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:27:57.790 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:27:58.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.002+0000 7f2148a00640 1 Processor -- start 2024-03-06T07:27:58.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.002+0000 7f2148a00640 1 -- start start 2024-03-06T07:27:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.004+0000 7f2148a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f2144154dc0 0x7f2144161b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.004+0000 7f2148a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441551a0 0x7f21440ae8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.004+0000 7f2148a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21440cfb30 con 0x7f21440aee20 2024-03-06T07:27:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.004+0000 7f2148a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21440af270 con 0x7f2144154dc0 2024-03-06T07:27:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.004+0000 7f2148a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f214415e540 con 0x7f21441551a0 2024-03-06T07:27:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.005+0000 7f213be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441551a0 0x7f21440ae8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.005+0000 7f2142800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f2144154dc0 0x7f2144161b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.005+0000 7f213be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441551a0 0x7f21440ae8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:46092/0 (socket says 172.21.15.86:46092) 2024-03-06T07:27:58.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.005+0000 7f213be00640 1 -- 172.21.15.86:0/1303451286 learned_addr learned my addr 172.21.15.86:0/1303451286 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:58.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.006+0000 7f213be00640 1 -- 172.21.15.86:0/1303451286 >> v2:172.21.15.194:3300/0 conn(0x7f2144154dc0 msgr2=0x7f2144161b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:58.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.006+0000 7f213be00640 1 --2- 172.21.15.86:0/1303451286 >> v2:172.21.15.194:3300/0 conn(0x7f2144154dc0 0x7f2144161b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.006+0000 7f213be00640 1 -- 172.21.15.86:0/1303451286 >> v1:172.21.15.194:6789/0 conn(0x7f21440aee20 legacy=0x7f21440cebc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.006+0000 7f213be00640 1 -- 172.21.15.86:0/1303451286 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21440cfe10 con 0x7f21441551a0 2024-03-06T07:27:58.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.007+0000 7f213be00640 1 --2- 172.21.15.86:0/1303451286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441551a0 0x7f21440ae8e0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f212c009870 tx=0x7f212c02b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f22b20f39257eb6d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2141e00640 1 -- 172.21.15.86:0/1303451286 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f212c034070 con 0x7f21441551a0 2024-03-06T07:27:58.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2141e00640 1 -- 172.21.15.86:0/1303451286 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f212c030840 con 0x7f21441551a0 2024-03-06T07:27:58.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2141e00640 1 -- 172.21.15.86:0/1303451286 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f212c030c10 con 0x7f21441551a0 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 -- 172.21.15.86:0/1303451286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441551a0 msgr2=0x7f21440ae8e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 --2- 172.21.15.86:0/1303451286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441551a0 0x7f21440ae8e0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f212c009870 tx=0x7f212c02b980 comp rx=0 tx=0).stop 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 -- 172.21.15.86:0/1303451286 shutdown_connections 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 --2- 172.21.15.86:0/1303451286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441551a0 0x7f21440ae8e0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 --2- 172.21.15.86:0/1303451286 >> v2:172.21.15.194:3300/0 conn(0x7f2144154dc0 0x7f2144161b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 -- 172.21.15.86:0/1303451286 >> 172.21.15.86:0/1303451286 conn(0x7f214406afc0 msgr2=0x7f214406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 -- 172.21.15.86:0/1303451286 shutdown_connections 2024-03-06T07:27:58.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.008+0000 7f2148a00640 1 -- 172.21.15.86:0/1303451286 wait complete. 2024-03-06T07:27:58.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.010+0000 7f2148a00640 1 Processor -- start 2024-03-06T07:27:58.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.010+0000 7f2148a00640 1 -- start start 2024-03-06T07:27:58.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2148a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21440aee20 0x7f21440b19c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2148a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154dc0 0x7f2144158b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2148a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21441551a0 0x7f21440af5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2148a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21440cf820 con 0x7f21441551a0 2024-03-06T07:27:58.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2148a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21440d03d0 con 0x7f21440aee20 2024-03-06T07:27:58.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2148a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2144162070 con 0x7f2144154dc0 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2143200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21441551a0 0x7f21440af5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.011+0000 7f2143200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21441551a0 0x7f21440af5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51030/0 (socket says 172.21.15.86:51030) 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2143200640 1 -- 172.21.15.86:0/1410347943 learned_addr learned my addr 172.21.15.86:0/1410347943 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2142800640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21440aee20 0x7f21440b19c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f213be00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154dc0 0x7f2144158b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2143200640 1 -- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154dc0 msgr2=0x7f2144158b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2143200640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154dc0 0x7f2144158b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2143200640 1 -- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21440aee20 msgr2=0x7f21440b19c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2143200640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21440aee20 0x7f21440b19c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2143200640 1 -- 172.21.15.86:0/1410347943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f214415b470 con 0x7f21441551a0 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f213be00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154dc0 0x7f2144158b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f2143200640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21441551a0 0x7f21440af5a0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f213c00c7a0 tx=0x7f213c00cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.012+0000 7f213aa00640 1 -- 172.21.15.86:0/1410347943 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f213c003b70 con 0x7f21441551a0 2024-03-06T07:27:58.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.013+0000 7f213aa00640 1 -- 172.21.15.86:0/1410347943 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f213c003d30 con 0x7f21441551a0 2024-03-06T07:27:58.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.013+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f214415b750 con 0x7f21441551a0 2024-03-06T07:27:58.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.013+0000 7f213aa00640 1 -- 172.21.15.86:0/1410347943 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f213c00aa90 con 0x7f21441551a0 2024-03-06T07:27:58.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.013+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f214415bd10 con 0x7f21441551a0 2024-03-06T07:27:58.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.014+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f214404f330 con 0x7f21441551a0 2024-03-06T07:27:58.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.015+0000 7f213aa00640 1 -- 172.21.15.86:0/1410347943 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f213c00ac30 con 0x7f21441551a0 2024-03-06T07:27:58.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.016+0000 7f213aa00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.016+0000 7f213aa00640 1 -- 172.21.15.86:0/1410347943 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f213c08a270 con 0x7f21441551a0 2024-03-06T07:27:58.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.016+0000 7f2142800640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.017+0000 7f2142800640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f2134009ad0 tx=0x7f2134008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.021+0000 7f213aa00640 1 -- 172.21.15.86:0/1410347943 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f213c058fd0 con 0x7f21441551a0 2024-03-06T07:27:58.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.216+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f21440add40 con 0x7f21441551a0 2024-03-06T07:27:58.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.217+0000 7f213aa00640 1 -- 172.21.15.86:0/1410347943 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v24) v1 ==== 93+0+10 (secure 0 0 0) 0x7f213c0588d0 con 0x7f21441551a0 2024-03-06T07:27:58.220 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:27:58.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.219+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 msgr2=0x7f212006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.219+0000 7f2148a00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f2134009ad0 tx=0x7f2134008040 comp rx=0 tx=0).stop 2024-03-06T07:27:58.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.219+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21441551a0 msgr2=0x7f21440af5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.219+0000 7f2148a00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21441551a0 0x7f21440af5a0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f213c00c7a0 tx=0x7f213c00cc60 comp rx=0 tx=0).stop 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 shutdown_connections 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21441551a0 0x7f21440af5a0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154dc0 0x7f2144158b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 --2- 172.21.15.86:0/1410347943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21440aee20 0x7f21440b19c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 >> 172.21.15.86:0/1410347943 conn(0x7f214406afc0 msgr2=0x7f2144070970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 shutdown_connections 2024-03-06T07:27:58.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.220+0000 7f2148a00640 1 -- 172.21.15.86:0/1410347943 wait complete. 2024-03-06T07:27:58.242 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:27:58.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.454+0000 7fe934c00640 1 Processor -- start 2024-03-06T07:27:58.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.454+0000 7fe934c00640 1 -- start start 2024-03-06T07:27:58.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.456+0000 7fe934c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe93015bfe0 0x7fe9300d0900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.456+0000 7fe934c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe9300cec90 0x7fe9300cd810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.456+0000 7fe934c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe9300cf8a0 con 0x7fe9300cd3c0 2024-03-06T07:27:58.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.456+0000 7fe934c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe9300cfb40 con 0x7fe9300cec90 2024-03-06T07:27:58.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.456+0000 7fe934c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe9300cfe20 con 0x7fe93015bfe0 2024-03-06T07:27:58.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe9300cec90 0x7fe9300cd810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92de00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe93015bfe0 0x7fe9300d0900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe9300cec90 0x7fe9300cd810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51044/0 (socket says 172.21.15.86:51044) 2024-03-06T07:27:58.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92e800640 1 -- 172.21.15.86:0/1267023962 learned_addr learned my addr 172.21.15.86:0/1267023962 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:58.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92e800640 1 -- 172.21.15.86:0/1267023962 >> v2:172.21.15.194:3300/0 conn(0x7fe93015bfe0 msgr2=0x7fe9300d0900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:58.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92e800640 1 --2- 172.21.15.86:0/1267023962 >> v2:172.21.15.194:3300/0 conn(0x7fe93015bfe0 0x7fe9300d0900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92e800640 1 -- 172.21.15.86:0/1267023962 >> v1:172.21.15.194:6789/0 conn(0x7fe9300cd3c0 legacy=0x7fe9300ce580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.457+0000 7fe92e800640 1 -- 172.21.15.86:0/1267023962 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe9300d0100 con 0x7fe9300cec90 2024-03-06T07:27:58.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.458+0000 7fe92e800640 1 --2- 172.21.15.86:0/1267023962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cec90 0x7fe9300cd810 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fe930071b20 tx=0x7fe92802c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d8f4ee240cac2e3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.459+0000 7fe92ca00640 1 -- 172.21.15.86:0/1267023962 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe92802cbb0 con 0x7fe9300cec90 2024-03-06T07:27:58.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.459+0000 7fe92ca00640 1 -- 172.21.15.86:0/1267023962 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe92802cd50 con 0x7fe9300cec90 2024-03-06T07:27:58.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.459+0000 7fe92ca00640 1 -- 172.21.15.86:0/1267023962 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe928034610 con 0x7fe9300cec90 2024-03-06T07:27:58.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.459+0000 7fe934c00640 1 -- 172.21.15.86:0/1267023962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cec90 msgr2=0x7fe9300cd810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.459+0000 7fe934c00640 1 --2- 172.21.15.86:0/1267023962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cec90 0x7fe9300cd810 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fe930071b20 tx=0x7fe92802c010 comp rx=0 tx=0).stop 2024-03-06T07:27:58.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.460+0000 7fe934c00640 1 -- 172.21.15.86:0/1267023962 shutdown_connections 2024-03-06T07:27:58.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.460+0000 7fe934c00640 1 --2- 172.21.15.86:0/1267023962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cec90 0x7fe9300cd810 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.460+0000 7fe934c00640 1 --2- 172.21.15.86:0/1267023962 >> v2:172.21.15.194:3300/0 conn(0x7fe93015bfe0 0x7fe9300d0900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.460+0000 7fe934c00640 1 -- 172.21.15.86:0/1267023962 >> 172.21.15.86:0/1267023962 conn(0x7fe9300b3800 msgr2=0x7fe9300b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:58.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.460+0000 7fe934c00640 1 -- 172.21.15.86:0/1267023962 shutdown_connections 2024-03-06T07:27:58.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.460+0000 7fe934c00640 1 -- 172.21.15.86:0/1267023962 wait complete. 2024-03-06T07:27:58.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.461+0000 7fe934c00640 1 Processor -- start 2024-03-06T07:27:58.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.461+0000 7fe934c00640 1 -- start start 2024-03-06T07:27:58.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.462+0000 7fe934c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cd3c0 0x7fe9300c4b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.462+0000 7fe934c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9300cec90 0x7fe9300b54e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.462+0000 7fe934c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe93015bfe0 0x7fe9300c79b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.462+0000 7fe934c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe93015c3c0 con 0x7fe9300cd3c0 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.462+0000 7fe934c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe9300b4540 con 0x7fe93015bfe0 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.462+0000 7fe934c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe93015c540 con 0x7fe9300cec90 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cd3c0 0x7fe9300c4b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cd3c0 0x7fe9300c4b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51058/0 (socket says 172.21.15.86:51058) 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 -- 172.21.15.86:0/3129663271 learned_addr learned my addr 172.21.15.86:0/3129663271 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92d400640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9300cec90 0x7fe9300b54e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92e800640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe93015bfe0 0x7fe9300c79b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 -- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9300cec90 msgr2=0x7fe9300b54e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9300cec90 0x7fe9300b54e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 -- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe93015bfe0 msgr2=0x7fe9300c79b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe93015bfe0 0x7fe9300c79b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 -- 172.21.15.86:0/3129663271 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe9300b5e30 con 0x7fe9300cd3c0 2024-03-06T07:27:58.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92e800640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe93015bfe0 0x7fe9300c79b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:58.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.463+0000 7fe92de00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cd3c0 0x7fe9300c4b50 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fe92000e8a0 tx=0x7fe92000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.464+0000 7fe926000640 1 -- 172.21.15.86:0/3129663271 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe920017070 con 0x7fe9300cd3c0 2024-03-06T07:27:58.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.464+0000 7fe926000640 1 -- 172.21.15.86:0/3129663271 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe92000c940 con 0x7fe9300cd3c0 2024-03-06T07:27:58.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.464+0000 7fe926000640 1 -- 172.21.15.86:0/3129663271 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe92000cd30 con 0x7fe9300cd3c0 2024-03-06T07:27:58.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.464+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe9300b6110 con 0x7fe9300cd3c0 2024-03-06T07:27:58.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.464+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe9300b66d0 con 0x7fe9300cd3c0 2024-03-06T07:27:58.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.465+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe9300b68a0 con 0x7fe9300cd3c0 2024-03-06T07:27:58.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.466+0000 7fe926000640 1 -- 172.21.15.86:0/3129663271 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe92001a070 con 0x7fe9300cd3c0 2024-03-06T07:27:58.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.467+0000 7fe926000640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90c06b500 0x7fe90c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.467+0000 7fe926000640 1 -- 172.21.15.86:0/3129663271 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7fe92008a5e0 con 0x7fe9300cd3c0 2024-03-06T07:27:58.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.467+0000 7fe92d400640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90c06b500 0x7fe90c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.468+0000 7fe92d400640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90c06b500 0x7fe90c06d9c0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fe9301f74a0 tx=0x7fe918009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.472+0000 7fe926000640 1 -- 172.21.15.86:0/3129663271 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe9200592b0 con 0x7fe9300cd3c0 2024-03-06T07:27:58.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.666+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fe9300c5090 con 0x7fe9300cd3c0 2024-03-06T07:27:58.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.666+0000 7fe926000640 1 -- 172.21.15.86:0/3129663271 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v24) v1 ==== 104+0+11 (secure 0 0 0) 0x7fe9300c5090 con 0x7fe9300cd3c0 2024-03-06T07:27:58.669 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90c06b500 msgr2=0x7fe90c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90c06b500 0x7fe90c06d9c0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fe9301f74a0 tx=0x7fe918009040 comp rx=0 tx=0).stop 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cd3c0 msgr2=0x7fe9300c4b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cd3c0 0x7fe9300c4b50 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fe92000e8a0 tx=0x7fe92000ed60 comp rx=0 tx=0).stop 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 shutdown_connections 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90c06b500 0x7fe90c06d9c0 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe93015bfe0 0x7fe9300c79b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe9300cec90 0x7fe9300b54e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 --2- 172.21.15.86:0/3129663271 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe9300cd3c0 0x7fe9300c4b50 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 >> 172.21.15.86:0/3129663271 conn(0x7fe9300b3800 msgr2=0x7fe9300af400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:58.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 shutdown_connections 2024-03-06T07:27:58.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.669+0000 7fe934c00640 1 -- 172.21.15.86:0/3129663271 wait complete. 2024-03-06T07:27:58.690 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:27:58.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.882+0000 7fd211000640 1 Processor -- start 2024-03-06T07:27:58.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.882+0000 7fd211000640 1 -- start start 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.883+0000 7fd211000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd20c0b3df0 0x7fd20c0727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.883+0000 7fd211000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3650 0x7fd20c0b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd211000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd20c0b4710 con 0x7fd20c0cda90 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd211000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd20c072310 con 0x7fd20c0b3df0 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd211000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd20c0719e0 con 0x7fd20c0b3650 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd20c0b3df0 0x7fd20c0727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd20c0b3df0 0x7fd20c0727a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51070/0 (socket says 172.21.15.86:51070) 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209e00640 1 -- 172.21.15.86:0/35023780 learned_addr learned my addr 172.21.15.86:0/35023780 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209400640 1 --2- 172.21.15.86:0/35023780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3650 0x7fd20c0b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209e00640 1 -- 172.21.15.86:0/35023780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3650 msgr2=0x7fd20c0b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209e00640 1 --2- 172.21.15.86:0/35023780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3650 0x7fd20c0b41d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209e00640 1 -- 172.21.15.86:0/35023780 >> v1:172.21.15.194:6789/0 conn(0x7fd20c0cda90 legacy=0x7fd20c06c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209e00640 1 -- 172.21.15.86:0/35023780 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd20c071cc0 con 0x7fd20c0b3df0 2024-03-06T07:27:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.884+0000 7fd209400640 1 --2- 172.21.15.86:0/35023780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3650 0x7fd20c0b41d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:27:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.885+0000 7fd209e00640 1 --2- 172.21.15.86:0/35023780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0b3df0 0x7fd20c0727a0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fd1fc009870 tx=0x7fd1fc02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4faa83eb48ce453 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.886+0000 7fd208a00640 1 -- 172.21.15.86:0/35023780 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1fc035020 con 0x7fd20c0b3df0 2024-03-06T07:27:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.886+0000 7fd208a00640 1 -- 172.21.15.86:0/35023780 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd1fc031640 con 0x7fd20c0b3df0 2024-03-06T07:27:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.886+0000 7fd208a00640 1 -- 172.21.15.86:0/35023780 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1fc02d040 con 0x7fd20c0b3df0 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.886+0000 7fd211000640 1 -- 172.21.15.86:0/35023780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0b3df0 msgr2=0x7fd20c0727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.886+0000 7fd211000640 1 --2- 172.21.15.86:0/35023780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0b3df0 0x7fd20c0727a0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fd1fc009870 tx=0x7fd1fc02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.887+0000 7fd211000640 1 -- 172.21.15.86:0/35023780 shutdown_connections 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.887+0000 7fd211000640 1 --2- 172.21.15.86:0/35023780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3650 0x7fd20c0b41d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.887+0000 7fd211000640 1 --2- 172.21.15.86:0/35023780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0b3df0 0x7fd20c0727a0 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.887+0000 7fd211000640 1 -- 172.21.15.86:0/35023780 >> 172.21.15.86:0/35023780 conn(0x7fd20c0b1940 msgr2=0x7fd20c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.887+0000 7fd211000640 1 -- 172.21.15.86:0/35023780 shutdown_connections 2024-03-06T07:27:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.887+0000 7fd211000640 1 -- 172.21.15.86:0/35023780 wait complete. 2024-03-06T07:27:58.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.889+0000 7fd211000640 1 Processor -- start 2024-03-06T07:27:58.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.889+0000 7fd211000640 1 -- start start 2024-03-06T07:27:58.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd211000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd20c0b3650 0x7fd20c151ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd211000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3df0 0x7fd20c15be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd211000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0cda90 0x7fd20c15c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd211000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd20c15b240 con 0x7fd20c0cda90 2024-03-06T07:27:58.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd211000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd20c0b3b30 con 0x7fd20c0b3650 2024-03-06T07:27:58.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd211000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd20c15b3c0 con 0x7fd20c0b3df0 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd20a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0cda90 0x7fd20c15c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd20a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0cda90 0x7fd20c15c3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51074/0 (socket says 172.21.15.86:51074) 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd20a800640 1 -- 172.21.15.86:0/2801547027 learned_addr learned my addr 172.21.15.86:0/2801547027 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd209400640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3df0 0x7fd20c15be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.890+0000 7fd209e00640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd20c0b3650 0x7fd20c151ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd20a800640 1 -- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3df0 msgr2=0x7fd20c15be60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd20a800640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3df0 0x7fd20c15be60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd20a800640 1 -- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd20c0b3650 msgr2=0x7fd20c151ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd20a800640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd20c0b3650 0x7fd20c151ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd20a800640 1 -- 172.21.15.86:0/2801547027 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd20c152010 con 0x7fd20c0cda90 2024-03-06T07:27:58.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd20a800640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0cda90 0x7fd20c15c3a0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fd20400ba10 tx=0x7fd20400bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd202000640 1 -- 172.21.15.86:0/2801547027 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd204019070 con 0x7fd20c0cda90 2024-03-06T07:27:58.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd202000640 1 -- 172.21.15.86:0/2801547027 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd20400cb40 con 0x7fd20c0cda90 2024-03-06T07:27:58.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd20c0cac00 con 0x7fd20c0cda90 2024-03-06T07:27:58.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd202000640 1 -- 172.21.15.86:0/2801547027 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd204014440 con 0x7fd20c0cda90 2024-03-06T07:27:58.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.891+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd20c0cb1c0 con 0x7fd20c0cda90 2024-03-06T07:27:58.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.892+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd20c04f330 con 0x7fd20c0cda90 2024-03-06T07:27:58.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.894+0000 7fd202000640 1 -- 172.21.15.86:0/2801547027 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd20401c070 con 0x7fd20c0cda90 2024-03-06T07:27:58.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.894+0000 7fd202000640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd1ec06b4d0 0x7fd1ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:58.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.894+0000 7fd202000640 1 -- 172.21.15.86:0/2801547027 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7fd204089e70 con 0x7fd20c0cda90 2024-03-06T07:27:58.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.895+0000 7fd209e00640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd1ec06b4d0 0x7fd1ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:58.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.895+0000 7fd209e00640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd1ec06b4d0 0x7fd1ec06d990 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fd1fc036000 tx=0x7fd1fc03c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:58.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:58.900+0000 7fd202000640 1 -- 172.21.15.86:0/2801547027 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd204058bd0 con 0x7fd20c0cda90 2024-03-06T07:27:59.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.101+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fd20c0cc0e0 con 0x7fd20c0cda90 2024-03-06T07:27:59.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.102+0000 7fd202000640 1 -- 172.21.15.86:0/2801547027 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v24) v1 ==== 100+0+11 (secure 0 0 0) 0x7fd2040584d0 con 0x7fd20c0cda90 2024-03-06T07:27:59.105 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:27:59.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd1ec06b4d0 msgr2=0x7fd1ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd1ec06b4d0 0x7fd1ec06d990 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fd1fc036000 tx=0x7fd1fc03c040 comp rx=0 tx=0).stop 2024-03-06T07:27:59.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0cda90 msgr2=0x7fd20c15c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0cda90 0x7fd20c15c3a0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fd20400ba10 tx=0x7fd20400bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 shutdown_connections 2024-03-06T07:27:59.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd1ec06b4d0 0x7fd1ec06d990 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd20c0cda90 0x7fd20c15c3a0 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd20c0b3df0 0x7fd20c15be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 --2- 172.21.15.86:0/2801547027 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd20c0b3650 0x7fd20c151ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 >> 172.21.15.86:0/2801547027 conn(0x7fd20c0b1940 msgr2=0x7fd20c0b0110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:59.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 shutdown_connections 2024-03-06T07:27:59.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.105+0000 7fd211000640 1 -- 172.21.15.86:0/2801547027 wait complete. 2024-03-06T07:27:59.125 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:27:59.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.327+0000 7f7220800640 1 Processor -- start 2024-03-06T07:27:59.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.327+0000 7f7220800640 1 -- start start 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.328+0000 7f7220800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72180cf430 0x7f7218070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.328+0000 7f7220800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7218071260 0x7f721806cb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.328+0000 7f7220800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f721806d090 0x7f7218165f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.328+0000 7f7220800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f721806d4c0 con 0x7f721806d090 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.328+0000 7f7220800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f72180717c0 con 0x7f7218071260 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.328+0000 7f7220800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7218071aa0 con 0x7f72180cf430 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f721806d090 0x7f7218165f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f721806d090 0x7f7218165f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51086/0 (socket says 172.21.15.86:51086) 2024-03-06T07:27:59.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 -- 172.21.15.86:0/2355296921 learned_addr learned my addr 172.21.15.86:0/2355296921 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:59.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721de00640 1 --2- 172.21.15.86:0/2355296921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72180cf430 0x7f7218070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721d400640 1 --2- 172.21.15.86:0/2355296921 >> v2:172.21.15.194:3300/0 conn(0x7f7218071260 0x7f721806cb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 -- 172.21.15.86:0/2355296921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72180cf430 msgr2=0x7f7218070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:59.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 --2- 172.21.15.86:0/2355296921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72180cf430 0x7f7218070d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 -- 172.21.15.86:0/2355296921 >> v2:172.21.15.194:3300/0 conn(0x7f7218071260 msgr2=0x7f721806cb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:59.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 --2- 172.21.15.86:0/2355296921 >> v2:172.21.15.194:3300/0 conn(0x7f7218071260 0x7f721806cb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.329+0000 7f721e800640 1 -- 172.21.15.86:0/2355296921 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7218071d80 con 0x7f721806d090 2024-03-06T07:27:59.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.330+0000 7f721e800640 1 --2- 172.21.15.86:0/2355296921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218165f40 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f72180b27f0 tx=0x7f721402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2cabc3cccf119c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:59.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f721ca00640 1 -- 172.21.15.86:0/2355296921 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f721402ca70 con 0x7f721806d090 2024-03-06T07:27:59.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f721ca00640 1 -- 172.21.15.86:0/2355296921 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f721402cc10 con 0x7f721806d090 2024-03-06T07:27:59.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f721ca00640 1 -- 172.21.15.86:0/2355296921 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72140343f0 con 0x7f721806d090 2024-03-06T07:27:59.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 -- 172.21.15.86:0/2355296921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 msgr2=0x7f7218165f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 --2- 172.21.15.86:0/2355296921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218165f40 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f72180b27f0 tx=0x7f721402bed0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 -- 172.21.15.86:0/2355296921 shutdown_connections 2024-03-06T07:27:59.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 --2- 172.21.15.86:0/2355296921 >> v2:172.21.15.194:3300/0 conn(0x7f7218071260 0x7f721806cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 --2- 172.21.15.86:0/2355296921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218165f40 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 --2- 172.21.15.86:0/2355296921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72180cf430 0x7f7218070d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 -- 172.21.15.86:0/2355296921 >> 172.21.15.86:0/2355296921 conn(0x7f72180b1730 msgr2=0x7f72180b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:59.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.331+0000 7f7220800640 1 -- 172.21.15.86:0/2355296921 shutdown_connections 2024-03-06T07:27:59.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.332+0000 7f7220800640 1 -- 172.21.15.86:0/2355296921 wait complete. 2024-03-06T07:27:59.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.333+0000 7f7220800640 1 Processor -- start 2024-03-06T07:27:59.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.333+0000 7f7220800640 1 -- start start 2024-03-06T07:27:59.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.334+0000 7f7220800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218154e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.334+0000 7f7220800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7218071260 0x7f721815ee40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.334+0000 7f7220800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72180cf430 0x7f721815f380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.334+0000 7f7220800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7218162b70 con 0x7f721806d090 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.334+0000 7f7220800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7218072340 con 0x7f72180cf430 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.334+0000 7f7220800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7218162cf0 con 0x7f7218071260 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.334+0000 7f721de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218154e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218154e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51098/0 (socket says 172.21.15.86:51098) 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 -- 172.21.15.86:0/1267785786 learned_addr learned my addr 172.21.15.86:0/1267785786 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 -- 172.21.15.86:0/1267785786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7218071260 msgr2=0x7f721815ee40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721d400640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7218071260 0x7f721815ee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7218071260 0x7f721815ee40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 -- 172.21.15.86:0/1267785786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72180cf430 msgr2=0x7f721815f380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72180cf430 0x7f721815f380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 -- 172.21.15.86:0/1267785786 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7218155340 con 0x7f721806d090 2024-03-06T07:27:59.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f721de00640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218154e00 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f720c00d5f0 tx=0x7f720c00dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:59.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.335+0000 7f7212000640 1 -- 172.21.15.86:0/1267785786 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f720c0125f0 con 0x7f721806d090 2024-03-06T07:27:59.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.336+0000 7f7212000640 1 -- 172.21.15.86:0/1267785786 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f720c019070 con 0x7f721806d090 2024-03-06T07:27:59.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.336+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72181f7e60 con 0x7f721806d090 2024-03-06T07:27:59.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.336+0000 7f7212000640 1 -- 172.21.15.86:0/1267785786 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f720c0129d0 con 0x7f721806d090 2024-03-06T07:27:59.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.336+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f721815c010 con 0x7f721806d090 2024-03-06T07:27:59.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.337+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f721804f330 con 0x7f721806d090 2024-03-06T07:27:59.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.338+0000 7f7212000640 1 -- 172.21.15.86:0/1267785786 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f720c012b70 con 0x7f721806d090 2024-03-06T07:27:59.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.338+0000 7f7212000640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71fc06b4d0 0x7f71fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.339+0000 7f7212000640 1 -- 172.21.15.86:0/1267785786 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f720c08a690 con 0x7f721806d090 2024-03-06T07:27:59.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.339+0000 7f721d400640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71fc06b4d0 0x7f71fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.340+0000 7f721d400640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71fc06b4d0 0x7f71fc06d990 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f721806eef0 tx=0x7f7208008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:59.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.343+0000 7f7212000640 1 -- 172.21.15.86:0/1267785786 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f720c0593f0 con 0x7f721806d090 2024-03-06T07:27:59.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.579+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f721815cdd0 con 0x7f721806d090 2024-03-06T07:27:59.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.581+0000 7f7212000640 1 -- 172.21.15.86:0/1267785786 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f720c058cf0 con 0x7f721806d090 2024-03-06T07:27:59.584 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:27:59.584 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:27:59.584 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71fc06b4d0 msgr2=0x7f71fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71fc06b4d0 0x7f71fc06d990 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f721806eef0 tx=0x7f7208008040 comp rx=0 tx=0).stop 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 msgr2=0x7f7218154e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218154e00 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f720c00d5f0 tx=0x7f720c00dab0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 shutdown_connections 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71fc06b4d0 0x7f71fc06d990 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72180cf430 0x7f721815f380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7218071260 0x7f721815ee40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 --2- 172.21.15.86:0/1267785786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f721806d090 0x7f7218154e00 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 >> 172.21.15.86:0/1267785786 conn(0x7f72180b1730 msgr2=0x7f72180b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 shutdown_connections 2024-03-06T07:27:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.584+0000 7f7220800640 1 -- 172.21.15.86:0/1267785786 wait complete. 2024-03-06T07:27:59.605 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:27:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.801+0000 7f0dbaa00640 1 Processor -- start 2024-03-06T07:27:59.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.801+0000 7f0dbaa00640 1 -- start start 2024-03-06T07:27:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.802+0000 7f0dbaa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0db415acb0 0x7f0db40b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.802+0000 7f0dbaa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db40ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.802+0000 7f0dbaa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0db415b090 con 0x7f0db40b4680 2024-03-06T07:27:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.803+0000 7f0dbaa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0db40b2a30 con 0x7f0db415acb0 2024-03-06T07:27:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.803+0000 7f0dbaa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0db40b4360 con 0x7f0db406ec00 2024-03-06T07:27:59.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.803+0000 7f0db3e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0db415acb0 0x7f0db40b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.803+0000 7f0db8a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db40ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.803+0000 7f0db3e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0db415acb0 0x7f0db40b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:49126/0 (socket says 172.21.15.86:49126) 2024-03-06T07:27:59.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.803+0000 7f0db3e00640 1 -- 172.21.15.86:0/521002464 learned_addr learned my addr 172.21.15.86:0/521002464 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:59.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.803+0000 7f0db3400640 1 --1- 172.21.15.86:0/521002464 >> v1:172.21.15.194:6789/0 conn(0x7f0db40b4680 0x7f0db406e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:37016/0 (socket says 172.21.15.86:37016) 2024-03-06T07:27:59.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.804+0000 7f0db3e00640 1 -- 172.21.15.86:0/521002464 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 msgr2=0x7f0db40ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:27:59.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.804+0000 7f0db3e00640 1 --2- 172.21.15.86:0/521002464 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db40ccfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.804+0000 7f0db3e00640 1 -- 172.21.15.86:0/521002464 >> v1:172.21.15.194:6789/0 conn(0x7f0db40b4680 legacy=0x7f0db406e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.804+0000 7f0db3e00640 1 -- 172.21.15.86:0/521002464 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0db40b2cf0 con 0x7f0db415acb0 2024-03-06T07:27:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.805+0000 7f0db3e00640 1 --2- 172.21.15.86:0/521002464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 0x7f0db40b37d0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f0da4009870 tx=0x7f0da402b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bc378598d16cb77b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.806+0000 7f0db2a00640 1 -- 172.21.15.86:0/521002464 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0da4034070 con 0x7f0db415acb0 2024-03-06T07:27:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.806+0000 7f0db2a00640 1 -- 172.21.15.86:0/521002464 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0da4030840 con 0x7f0db415acb0 2024-03-06T07:27:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.806+0000 7f0db2a00640 1 -- 172.21.15.86:0/521002464 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0da4030c10 con 0x7f0db415acb0 2024-03-06T07:27:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 -- 172.21.15.86:0/521002464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 msgr2=0x7f0db40b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/521002464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 0x7f0db40b37d0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f0da4009870 tx=0x7f0da402b980 comp rx=0 tx=0).stop 2024-03-06T07:27:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 -- 172.21.15.86:0/521002464 shutdown_connections 2024-03-06T07:27:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/521002464 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db40ccfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/521002464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 0x7f0db40b37d0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 -- 172.21.15.86:0/521002464 >> 172.21.15.86:0/521002464 conn(0x7f0db40b1730 msgr2=0x7f0db40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:27:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 -- 172.21.15.86:0/521002464 shutdown_connections 2024-03-06T07:27:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.807+0000 7f0dbaa00640 1 -- 172.21.15.86:0/521002464 wait complete. 2024-03-06T07:27:59.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.809+0000 7f0dbaa00640 1 Processor -- start 2024-03-06T07:27:59.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.809+0000 7f0dbaa00640 1 -- start start 2024-03-06T07:27:59.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.810+0000 7f0dbaa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db41433f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.810+0000 7f0dbaa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0db40b4680 0x7f0db415c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.810+0000 7f0dbaa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 0x7f0db415cc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.810+0000 7f0dbaa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0db40b3d10 con 0x7f0db40b4680 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.810+0000 7f0dbaa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0db40b32b0 con 0x7f0db415acb0 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.810+0000 7f0dbaa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0db40b4010 con 0x7f0db406ec00 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0db40b4680 0x7f0db415c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0db40b4680 0x7f0db415c730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51106/0 (socket says 172.21.15.86:51106) 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 -- 172.21.15.86:0/4122139699 learned_addr learned my addr 172.21.15.86:0/4122139699 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db8a00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 0x7f0db415cc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3e00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db41433f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 -- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 msgr2=0x7f0db41433f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db41433f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 -- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 msgr2=0x7f0db415cc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:27:59.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 0x7f0db415cc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:27:59.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 -- 172.21.15.86:0/4122139699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0db4158e20 con 0x7f0db40b4680 2024-03-06T07:27:59.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.811+0000 7f0db3400640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0db40b4680 0x7f0db415c730 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f0da000c400 tx=0x7f0da000c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:59.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.812+0000 7f0dabe00640 1 -- 172.21.15.86:0/4122139699 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0da00074e0 con 0x7f0db40b4680 2024-03-06T07:27:59.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.812+0000 7f0dabe00640 1 -- 172.21.15.86:0/4122139699 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0da0007680 con 0x7f0db40b4680 2024-03-06T07:27:59.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.812+0000 7f0dabe00640 1 -- 172.21.15.86:0/4122139699 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0da0004a00 con 0x7f0db40b4680 2024-03-06T07:27:59.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.812+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0db4159100 con 0x7f0db40b4680 2024-03-06T07:27:59.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.812+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0db41596c0 con 0x7f0db40b4680 2024-03-06T07:27:59.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.813+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0db404f330 con 0x7f0db40b4680 2024-03-06T07:27:59.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.814+0000 7f0dabe00640 1 -- 172.21.15.86:0/4122139699 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0da001a020 con 0x7f0db40b4680 2024-03-06T07:27:59.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.815+0000 7f0dabe00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d9406b4d0 0x7f0d9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:27:59.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.815+0000 7f0dabe00640 1 -- 172.21.15.86:0/4122139699 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f0da0089a30 con 0x7f0db40b4680 2024-03-06T07:27:59.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.815+0000 7f0db3e00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d9406b4d0 0x7f0d9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:27:59.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.816+0000 7f0db3e00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d9406b4d0 0x7f0d9406d990 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f0da4036000 tx=0x7f0da403b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:27:59.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:27:59.819+0000 7f0dabe00640 1 -- 172.21.15.86:0/4122139699 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0da00588a0 con 0x7f0db40b4680 2024-03-06T07:28:00.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.020+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0db40d0500 con 0x7f0db40b4680 2024-03-06T07:28:00.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.022+0000 7f0dabe00640 1 -- 172.21.15.86:0/4122139699 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v24) v1 ==== 74+0+16885 (secure 0 0 0) 0x7f0da00581a0 con 0x7f0db40b4680 2024-03-06T07:28:00.025 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:00.025 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":24,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:53.981490+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.16","mappings":[{"from":5,"to":2},{"from":7,"to":1}]},{"pgid":"3.24","mappings":[{"from":3,"to":1}]},{"pgid":"3.28","mappings":[{"from":5,"to":2}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d9406b4d0 msgr2=0x7f0d9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d9406b4d0 0x7f0d9406d990 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f0da4036000 tx=0x7f0da403b040 comp rx=0 tx=0).stop 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0db40b4680 msgr2=0x7f0db415c730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0db40b4680 0x7f0db415c730 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f0da000c400 tx=0x7f0da000c8c0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 shutdown_connections 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d9406b4d0 0x7f0d9406d990 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0db415acb0 0x7f0db415cc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0db40b4680 0x7f0db415c730 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 --2- 172.21.15.86:0/4122139699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0db406ec00 0x7f0db41433f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.024+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 >> 172.21.15.86:0/4122139699 conn(0x7f0db40b1730 msgr2=0x7f0db40ce2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:00.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.025+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 shutdown_connections 2024-03-06T07:28:00.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.025+0000 7f0dbaa00640 1 -- 172.21.15.86:0/4122139699 wait complete. 2024-03-06T07:28:00.047 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:28:00.047 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:00.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.244+0000 7f707d600640 1 Processor -- start 2024-03-06T07:28:00.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.244+0000 7f707d600640 1 -- start start 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.245+0000 7f707d600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7078151660 0x7f7078071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.245+0000 7f707d600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f70780cf430 0x7f707806d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.245+0000 7f707d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f707806d920 0x7f707815dff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.245+0000 7f707d600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f70780b3ee0 con 0x7f70780cf430 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.245+0000 7f707d600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7078071a60 con 0x7f7078151660 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.245+0000 7f707d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70780b35b0 con 0x7f707806d920 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f70780cf430 0x7f707806d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f70780cf430 0x7f707806d3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51122/0 (socket says 172.21.15.86:51122) 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 -- 172.21.15.86:0/1318321616 learned_addr learned my addr 172.21.15.86:0/1318321616 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7077400640 1 --2- 172.21.15.86:0/1318321616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f707806d920 0x7f707815dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076a00640 1 --2- 172.21.15.86:0/1318321616 >> v2:172.21.15.194:3300/0 conn(0x7f7078151660 0x7f7078071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 -- 172.21.15.86:0/1318321616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f707806d920 msgr2=0x7f707815dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:00.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 --2- 172.21.15.86:0/1318321616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f707806d920 0x7f707815dff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 -- 172.21.15.86:0/1318321616 >> v2:172.21.15.194:3300/0 conn(0x7f7078151660 msgr2=0x7f7078071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:00.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 --2- 172.21.15.86:0/1318321616 >> v2:172.21.15.194:3300/0 conn(0x7f7078151660 0x7f7078071520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076000640 1 -- 172.21.15.86:0/1318321616 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70780b3890 con 0x7f70780cf430 2024-03-06T07:28:00.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7077400640 1 --2- 172.21.15.86:0/1318321616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f707806d920 0x7f707815dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:00.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.246+0000 7f7076a00640 1 --2- 172.21.15.86:0/1318321616 >> v2:172.21.15.194:3300/0 conn(0x7f7078151660 0x7f7078071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:00.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.247+0000 7f7076000640 1 --2- 172.21.15.86:0/1318321616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70780cf430 0x7f707806d3e0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f7078072fc0 tx=0x7f706402b7f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8bec2c0f4cd2c467 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:00.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f7075600640 1 -- 172.21.15.86:0/1318321616 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7064038070 con 0x7f70780cf430 2024-03-06T07:28:00.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f7075600640 1 -- 172.21.15.86:0/1318321616 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f706402f050 con 0x7f70780cf430 2024-03-06T07:28:00.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f7075600640 1 -- 172.21.15.86:0/1318321616 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7064031970 con 0x7f70780cf430 2024-03-06T07:28:00.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 -- 172.21.15.86:0/1318321616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70780cf430 msgr2=0x7f707806d3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 --2- 172.21.15.86:0/1318321616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70780cf430 0x7f707806d3e0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f7078072fc0 tx=0x7f706402b7f0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 -- 172.21.15.86:0/1318321616 shutdown_connections 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 --2- 172.21.15.86:0/1318321616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f707806d920 0x7f707815dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 --2- 172.21.15.86:0/1318321616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70780cf430 0x7f707806d3e0 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 --2- 172.21.15.86:0/1318321616 >> v2:172.21.15.194:3300/0 conn(0x7f7078151660 0x7f7078071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 -- 172.21.15.86:0/1318321616 >> 172.21.15.86:0/1318321616 conn(0x7f70780b1730 msgr2=0x7f70780b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 -- 172.21.15.86:0/1318321616 shutdown_connections 2024-03-06T07:28:00.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.248+0000 7f707d600640 1 -- 172.21.15.86:0/1318321616 wait complete. 2024-03-06T07:28:00.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.250+0000 7f707d600640 1 Processor -- start 2024-03-06T07:28:00.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.250+0000 7f707d600640 1 -- start start 2024-03-06T07:28:00.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f707d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f707806d920 0x7f707815c0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f707d600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70780cf430 0x7f707814ff70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f707d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7078151660 0x7f70780aad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f707d600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f707815e530 con 0x7f707806d920 2024-03-06T07:28:00.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f707d600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7078071d30 con 0x7f70780cf430 2024-03-06T07:28:00.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f707d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f707815e6b0 con 0x7f7078151660 2024-03-06T07:28:00.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f7076a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f707806d920 0x7f707815c0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f7076a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f707806d920 0x7f707815c0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51134/0 (socket says 172.21.15.86:51134) 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.251+0000 7f7076a00640 1 -- 172.21.15.86:0/3819868491 learned_addr learned my addr 172.21.15.86:0/3819868491 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7076000640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70780cf430 0x7f707814ff70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7077400640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7078151660 0x7f70780aad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7076a00640 1 -- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7078151660 msgr2=0x7f70780aad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7076a00640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7078151660 0x7f70780aad50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7076a00640 1 -- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70780cf430 msgr2=0x7f707814ff70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7076a00640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70780cf430 0x7f707814ff70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7076a00640 1 -- 172.21.15.86:0/3819868491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70781504b0 con 0x7f707806d920 2024-03-06T07:28:00.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f7076a00640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f707806d920 0x7f707815c0d0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f706800c820 tx=0x7f706800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:00.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f706ea00640 1 -- 172.21.15.86:0/3819868491 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7068015070 con 0x7f707806d920 2024-03-06T07:28:00.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f706ea00640 1 -- 172.21.15.86:0/3819868491 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7068007a30 con 0x7f707806d920 2024-03-06T07:28:00.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f706ea00640 1 -- 172.21.15.86:0/3819868491 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7068007de0 con 0x7f707806d920 2024-03-06T07:28:00.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70780cd450 con 0x7f707806d920 2024-03-06T07:28:00.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.252+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70780cda10 con 0x7f707806d920 2024-03-06T07:28:00.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.253+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f707804f330 con 0x7f707806d920 2024-03-06T07:28:00.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.254+0000 7f706ea00640 1 -- 172.21.15.86:0/3819868491 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7068018070 con 0x7f707806d920 2024-03-06T07:28:00.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.255+0000 7f706ea00640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f705806b500 0x7f705806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.255+0000 7f706ea00640 1 -- 172.21.15.86:0/3819868491 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f706808a1a0 con 0x7f707806d920 2024-03-06T07:28:00.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.256+0000 7f7076000640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f705806b500 0x7f705806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.256+0000 7f7076000640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f705806b500 0x7f705806d9c0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f7064035000 tx=0x7f7064008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:00.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.260+0000 7f706ea00640 1 -- 172.21.15.86:0/3819868491 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7068058f10 con 0x7f707806d920 2024-03-06T07:28:00.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.473+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f707815b590 con 0x7f707806d920 2024-03-06T07:28:00.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.475+0000 7f706ea00640 1 -- 172.21.15.86:0/3819868491 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f7068058810 con 0x7f707806d920 2024-03-06T07:28:00.477 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:00.478 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:28:00.478 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:28:00.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f705806b500 msgr2=0x7f705806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f705806b500 0x7f705806d9c0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f7064035000 tx=0x7f7064008030 comp rx=0 tx=0).stop 2024-03-06T07:28:00.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f707806d920 msgr2=0x7f707815c0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f707806d920 0x7f707815c0d0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f706800c820 tx=0x7f706800cce0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 shutdown_connections 2024-03-06T07:28:00.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f705806b500 0x7f705806d9c0 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7078151660 0x7f70780aad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70780cf430 0x7f707814ff70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 --2- 172.21.15.86:0/3819868491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f707806d920 0x7f707815c0d0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 >> 172.21.15.86:0/3819868491 conn(0x7f70780b1730 msgr2=0x7f707806dd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:00.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 shutdown_connections 2024-03-06T07:28:00.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.477+0000 7f707d600640 1 -- 172.21.15.86:0/3819868491 wait complete. 2024-03-06T07:28:00.498 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:00.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.696+0000 7f120a600640 1 Processor -- start 2024-03-06T07:28:00.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.696+0000 7f120a600640 1 -- start start 2024-03-06T07:28:00.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f120a600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f12040af1c0 0x7f120415eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f120a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 0x7f120406e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f120a600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1204071d30 con 0x7f1204070d20 2024-03-06T07:28:00.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f120a600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f12040711d0 con 0x7f12040af1c0 2024-03-06T07:28:00.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f120a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1204071490 con 0x7f120406e4d0 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f1203400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f12040af1c0 0x7f120415eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f1203400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f12040af1c0 0x7f120415eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51136/0 (socket says 172.21.15.86:51136) 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.698+0000 7f1203400640 1 -- 172.21.15.86:0/3059299368 learned_addr learned my addr 172.21.15.86:0/3059299368 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.699+0000 7f1203e00640 1 --2- 172.21.15.86:0/3059299368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 0x7f120406e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.699+0000 7f1203400640 1 -- 172.21.15.86:0/3059299368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 msgr2=0x7f120406e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.699+0000 7f1203400640 1 --2- 172.21.15.86:0/3059299368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 0x7f120406e960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.699+0000 7f1203400640 1 -- 172.21.15.86:0/3059299368 >> v1:172.21.15.194:6789/0 conn(0x7f1204070d20 legacy=0x7f120406cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:00.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.699+0000 7f1203400640 1 -- 172.21.15.86:0/3059299368 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1204071770 con 0x7f12040af1c0 2024-03-06T07:28:00.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.700+0000 7f1203400640 1 --2- 172.21.15.86:0/3059299368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12040af1c0 0x7f120415eff0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f11f40098a0 tx=0x7f11f402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1224ee2ea48ef934 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:00.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.700+0000 7f1202a00640 1 -- 172.21.15.86:0/3059299368 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11f4035020 con 0x7f12040af1c0 2024-03-06T07:28:00.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.700+0000 7f1202a00640 1 -- 172.21.15.86:0/3059299368 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11f4031640 con 0x7f12040af1c0 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.700+0000 7f1202a00640 1 -- 172.21.15.86:0/3059299368 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11f402d040 con 0x7f12040af1c0 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 -- 172.21.15.86:0/3059299368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12040af1c0 msgr2=0x7f120415eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 --2- 172.21.15.86:0/3059299368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12040af1c0 0x7f120415eff0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f11f40098a0 tx=0x7f11f402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 -- 172.21.15.86:0/3059299368 shutdown_connections 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 --2- 172.21.15.86:0/3059299368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 0x7f120406e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 --2- 172.21.15.86:0/3059299368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12040af1c0 0x7f120415eff0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 -- 172.21.15.86:0/3059299368 >> 172.21.15.86:0/3059299368 conn(0x7f12040b1730 msgr2=0x7f12040b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 -- 172.21.15.86:0/3059299368 shutdown_connections 2024-03-06T07:28:00.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.701+0000 7f120a600640 1 -- 172.21.15.86:0/3059299368 wait complete. 2024-03-06T07:28:00.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.702+0000 7f120a600640 1 Processor -- start 2024-03-06T07:28:00.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.702+0000 7f120a600640 1 -- start start 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.703+0000 7f120a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 0x7f120414b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.703+0000 7f120a600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1204070d20 0x7f12041e67e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.703+0000 7f120a600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12040af1c0 0x7f120415fdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.703+0000 7f120a600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f120406d370 con 0x7f1204070d20 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.703+0000 7f120a600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12040b30b0 con 0x7f12040af1c0 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.703+0000 7f120a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f12040b35d0 con 0x7f120406e4d0 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1204070d20 0x7f12041e67e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1204070d20 0x7f12041e67e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51144/0 (socket says 172.21.15.86:51144) 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 -- 172.21.15.86:0/2967576649 learned_addr learned my addr 172.21.15.86:0/2967576649 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f1203e00640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12040af1c0 0x7f120415fdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 -- 172.21.15.86:0/2967576649 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 msgr2=0x7f120414b3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 0x7f120414b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 -- 172.21.15.86:0/2967576649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12040af1c0 msgr2=0x7f120415fdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12040af1c0 0x7f120415fdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 -- 172.21.15.86:0/2967576649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12041e71c0 con 0x7f1204070d20 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fbe00640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1204070d20 0x7f12041e67e0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f11ec00c430 tx=0x7f11ec00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fb400640 1 -- 172.21.15.86:0/2967576649 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11ec011790 con 0x7f1204070d20 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fb400640 1 -- 172.21.15.86:0/2967576649 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11ec017070 con 0x7f1204070d20 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f12041e74a0 con 0x7f1204070d20 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.704+0000 7f11fb400640 1 -- 172.21.15.86:0/2967576649 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11ec011bb0 con 0x7f1204070d20 2024-03-06T07:28:00.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.705+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12041e7d40 con 0x7f1204070d20 2024-03-06T07:28:00.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.706+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f12041e7890 con 0x7f1204070d20 2024-03-06T07:28:00.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.707+0000 7f11fb400640 1 -- 172.21.15.86:0/2967576649 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f11ec01c460 con 0x7f1204070d20 2024-03-06T07:28:00.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.707+0000 7f11fb400640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11e006b4d0 0x7f11e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:00.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.708+0000 7f11fb400640 1 -- 172.21.15.86:0/2967576649 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f11ec08a180 con 0x7f1204070d20 2024-03-06T07:28:00.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.708+0000 7f1203400640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11e006b4d0 0x7f11e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:00.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.708+0000 7f1203400640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11e006b4d0 0x7f11e006d990 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f11f4036000 tx=0x7f11f4030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:00.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.712+0000 7f11fb400640 1 -- 172.21.15.86:0/2967576649 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f11ec058f70 con 0x7f1204070d20 2024-03-06T07:28:00.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.947+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f120404f330 con 0x7f1204070d20 2024-03-06T07:28:00.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.949+0000 7f11fb400640 1 -- 172.21.15.86:0/2967576649 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f11ec058870 con 0x7f1204070d20 2024-03-06T07:28:00.952 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:00.952 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":8,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4236,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/981723884","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":981723884},{"type":"v1","addr":"172.21.15.86:6835","nonce":981723884}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4239,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/877604601","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":877604601},{"type":"v1","addr":"172.21.15.86:6837","nonce":877604601}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4254,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/850011693","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":850011693},{"type":"v1","addr":"172.21.15.194:6835","nonce":850011693}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":8,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:27:07.365128+0000","modified":"2024-03-06T07:27:10.407141+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4255},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4255":{"gid":4255,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/1612687235","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1612687235},{"type":"v1","addr":"172.21.15.194:6834","nonce":1612687235}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4255,"qdb_cluster":[4255]},"id":1}]} 2024-03-06T07:28:00.952 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 8 2024-03-06T07:28:00.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11e006b4d0 msgr2=0x7f11e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11e006b4d0 0x7f11e006d990 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f11f4036000 tx=0x7f11f4030000 comp rx=0 tx=0).stop 2024-03-06T07:28:00.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1204070d20 msgr2=0x7f12041e67e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:00.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1204070d20 0x7f12041e67e0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f11ec00c430 tx=0x7f11ec00c8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 shutdown_connections 2024-03-06T07:28:00.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f11e006b4d0 0x7f11e006d990 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12040af1c0 0x7f120415fdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1204070d20 0x7f12041e67e0 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 --2- 172.21.15.86:0/2967576649 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f120406e4d0 0x7f120414b3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:00.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.951+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 >> 172.21.15.86:0/2967576649 conn(0x7f12040b1730 msgr2=0x7f12040cfa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:00.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.952+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 shutdown_connections 2024-03-06T07:28:00.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:00.952+0000 7f120a600640 1 -- 172.21.15.86:0/2967576649 wait complete. 2024-03-06T07:28:00.973 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:01.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.158+0000 7fc5bf200640 1 Processor -- start 2024-03-06T07:28:01.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.158+0000 7fc5bf200640 1 -- start start 2024-03-06T07:28:01.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.159+0000 7fc5bf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b815acb0 0x7fc5b80b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.159+0000 7fc5bf200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5b815b090 con 0x7fc5b806ec00 2024-03-06T07:28:01.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.159+0000 7fc5bf200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5b80b2a30 con 0x7fc5b80b4680 2024-03-06T07:28:01.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.159+0000 7fc5bf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b80b4360 con 0x7fc5b815acb0 2024-03-06T07:28:01.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5bd200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc5b806ec00 0x7fc5b80ccfb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39782/0 (socket says 172.21.15.86:39782) 2024-03-06T07:28:01.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5bd200640 1 -- 172.21.15.86:0/2891502605 learned_addr learned my addr 172.21.15.86:0/2891502605 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:01.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5b7e00640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b815acb0 0x7fc5b80b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 790631081 0 0) 0x7fc5b815b090 con 0x7fc5b806ec00 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc59c005730 con 0x7fc5b806ec00 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1763183570 0 0) 0x7fc59c005730 con 0x7fc5b806ec00 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b815acb0 msgr2=0x7fc5b80b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5b7400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b815acb0 0x7fc5b80b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.160+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 >> v1:172.21.15.194:6789/0 conn(0x7fc5b80b4680 legacy=0x7fc5b806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b80b2cf0 con 0x7fc5b806ec00 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc5b00019f0 con 0x7fc5b806ec00 2024-03-06T07:28:01.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 >> v1:172.21.15.86:6789/0 conn(0x7fc5b806ec00 legacy=0x7fc5b80ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b815acb0 0x7fc59c005bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc59c0061c0 0x7fc59c006a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc59c006f50 0x7fc59c0073d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b80b4360 con 0x7fc59c0061c0 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc59c00dd10 con 0x7fc59c006f50 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc59c00df10 con 0x7fc5b815acb0 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5af400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc59c0061c0 0x7fc59c006a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5af400640 1 -- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b815acb0 msgr2=0x7fc59c005bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5af400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b815acb0 0x7fc59c005bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5af400640 1 -- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc59c006f50 msgr2=0x7fc59c0073d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5af400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc59c006f50 0x7fc59c0073d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.161+0000 7fc5af400640 1 -- 172.21.15.86:0/2891502605 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc59c005730 con 0x7fc59c0061c0 2024-03-06T07:28:01.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.162+0000 7fc5af400640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc59c0061c0 0x7fc59c006a10 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fc5a0001750 tx=0x7fc59c007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=43bd614c7d8991b2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:01.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.162+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a0001370 con 0x7fc59c0061c0 2024-03-06T07:28:01.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.162+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5a00052d0 con 0x7fc59c0061c0 2024-03-06T07:28:01.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.162+0000 7fc5b7400640 1 -- 172.21.15.86:0/2891502605 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a0003830 con 0x7fc59c0061c0 2024-03-06T07:28:01.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.163+0000 7fc5bf200640 1 -- 172.21.15.86:0/2891502605 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc59c0061c0 msgr2=0x7fc59c006a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.163+0000 7fc5bf200640 1 --2- 172.21.15.86:0/2891502605 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc59c0061c0 0x7fc59c006a10 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fc5a0001750 tx=0x7fc59c007a40 comp rx=0 tx=0).stop 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.163+0000 7fc5bd200640 1 -- 172.21.15.86:0/2891502605 reap_dead start 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.163+0000 7fc5bf200640 1 -- 172.21.15.86:0/2891502605 shutdown_connections 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.163+0000 7fc5bf200640 1 -- 172.21.15.86:0/2891502605 >> 172.21.15.86:0/2891502605 conn(0x7fc5b80b1730 msgr2=0x7fc5b80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.163+0000 7fc5bf200640 1 -- 172.21.15.86:0/2891502605 shutdown_connections 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.163+0000 7fc5bf200640 1 -- 172.21.15.86:0/2891502605 wait complete. 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.165+0000 7fc5bf200640 1 Processor -- start 2024-03-06T07:28:01.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.165+0000 7fc5bf200640 1 -- start start 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5bf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc59c0061c0 0x7fc5b80ce1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5bf200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b8144c90 0x7fc5b80ce710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5bf200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b8158420 0x7fc5b81588a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5bf200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b80b4010 con 0x7fc5b8144c90 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5bf200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b80b32b0 con 0x7fc5b8158420 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5bf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b8072340 con 0x7fc59c0061c0 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5af400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b8144c90 0x7fc5b80ce710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5af400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b8144c90 0x7fc5b80ce710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51166/0 (socket says 172.21.15.86:51166) 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5af400640 1 -- 172.21.15.86:0/835623988 learned_addr learned my addr 172.21.15.86:0/835623988 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5b7e00640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc59c0061c0 0x7fc5b80ce1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.166+0000 7fc5bd200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b8158420 0x7fc5b81588a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5af400640 1 -- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc59c0061c0 msgr2=0x7fc5b80ce1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5af400640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc59c0061c0 0x7fc5b80ce1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5af400640 1 -- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b8158420 msgr2=0x7fc5b81588a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5af400640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b8158420 0x7fc5b81588a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5af400640 1 -- 172.21.15.86:0/835623988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b8159150 con 0x7fc5b8144c90 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5b7e00640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc59c0061c0 0x7fc5b80ce1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5bd200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b8158420 0x7fc5b81588a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5af400640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b8144c90 0x7fc5b80ce710 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7fc5a0005040 tx=0x7fc5a0001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5b4c00640 1 -- 172.21.15.86:0/835623988 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a0041070 con 0x7fc5b8144c90 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5b4c00640 1 -- 172.21.15.86:0/835623988 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5a0005480 con 0x7fc5b8144c90 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5b8159430 con 0x7fc5b8144c90 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5b4c00640 1 -- 172.21.15.86:0/835623988 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a0007a30 con 0x7fc5b8144c90 2024-03-06T07:28:01.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.167+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5b81599f0 con 0x7fc5b8144c90 2024-03-06T07:28:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.168+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5b804f330 con 0x7fc5b8144c90 2024-03-06T07:28:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.169+0000 7fc5b4c00640 1 -- 172.21.15.86:0/835623988 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc5a0044070 con 0x7fc5b8144c90 2024-03-06T07:28:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.170+0000 7fc5b4c00640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59406b500 0x7fc59406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.170+0000 7fc5b4c00640 1 -- 172.21.15.86:0/835623988 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7fc5a00abf20 con 0x7fc5b8144c90 2024-03-06T07:28:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.170+0000 7fc5b7e00640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59406b500 0x7fc59406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.171+0000 7fc5b7e00640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59406b500 0x7fc59406d9c0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fc5b81564f0 tx=0x7fc5a8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:01.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.174+0000 7fc5b4c00640 1 -- 172.21.15.86:0/835623988 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc5a007ad00 con 0x7fc5b8144c90 2024-03-06T07:28:01.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.366+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc5b806ce60 con 0x7fc5b8144c90 2024-03-06T07:28:01.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.369+0000 7fc5b4c00640 1 -- 172.21.15.86:0/835623988 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v24) v1 ==== 74+0+16885 (secure 0 0 0) 0x7fc5a007a600 con 0x7fc5b8144c90 2024-03-06T07:28:01.372 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:01.373 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":24,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:27:53.981490+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:27:05.106298+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-06T07:27:06.363974+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.16","mappings":[{"from":5,"to":2},{"from":7,"to":1}]},{"pgid":"3.24","mappings":[{"from":3,"to":1}]},{"pgid":"3.28","mappings":[{"from":5,"to":2}]},{"pgid":"3.36","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:01.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59406b500 msgr2=0x7fc59406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59406b500 0x7fc59406d9c0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fc5b81564f0 tx=0x7fc5a8008040 comp rx=0 tx=0).stop 2024-03-06T07:28:01.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b8144c90 msgr2=0x7fc5b80ce710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b8144c90 0x7fc5b80ce710 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7fc5a0005040 tx=0x7fc5a0001120 comp rx=0 tx=0).stop 2024-03-06T07:28:01.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 shutdown_connections 2024-03-06T07:28:01.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59406b500 0x7fc59406d9c0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b8158420 0x7fc5b81588a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b8144c90 0x7fc5b80ce710 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 --2- 172.21.15.86:0/835623988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc59c0061c0 0x7fc5b80ce1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 >> 172.21.15.86:0/835623988 conn(0x7fc5b80b1730 msgr2=0x7fc5b8071ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:01.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.372+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 shutdown_connections 2024-03-06T07:28:01.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.373+0000 7fc5bf200640 1 -- 172.21.15.86:0/835623988 wait complete. 2024-03-06T07:28:01.393 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:28:01.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.600+0000 7f2fa9a00640 1 Processor -- start 2024-03-06T07:28:01.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.600+0000 7f2fa9a00640 1 -- start start 2024-03-06T07:28:01.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.602+0000 7f2fa9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2fa4164b30 0x7f2fa4166f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.602+0000 7f2fa9a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2fa4145b30 con 0x7f2fa40c4930 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.602+0000 7f2fa9a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2fa4142ce0 con 0x7f2fa40c6460 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.602+0000 7f2fa9a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2fa40c68c0 con 0x7f2fa4164b30 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.602+0000 7f2fa2a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2fa40c4930 0x7f2fa40c4d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39784/0 (socket says 172.21.15.86:39784) 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.602+0000 7f2fa2a00640 1 -- 172.21.15.86:0/1430030122 learned_addr learned my addr 172.21.15.86:0/1430030122 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.602+0000 7f2fa3400640 1 --2- 172.21.15.86:0/1430030122 >> v2:172.21.15.86:3300/0 conn(0x7f2fa4164b30 0x7f2fa4166f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.603+0000 7f2fa1600640 1 -- 172.21.15.86:0/1430030122 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 437207621 0 0) 0x7f2f94001110 con 0x7f2fa40c4930 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.603+0000 7f2fa1600640 1 -- 172.21.15.86:0/1430030122 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2f88003520 con 0x7f2fa40c4930 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.603+0000 7f2fa3400640 1 -- 172.21.15.86:0/1430030122 >> v1:172.21.15.194:6789/0 conn(0x7f2fa40c6460 legacy=0x7f2fa40c5420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.603+0000 7f2fa3400640 1 -- 172.21.15.86:0/1430030122 >> v1:172.21.15.86:6789/0 conn(0x7f2fa40c4930 legacy=0x7f2fa40c4d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.603+0000 7f2fa3400640 1 -- 172.21.15.86:0/1430030122 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2fa40cbca0 con 0x7f2fa4164b30 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.604+0000 7f2fa3400640 1 --2- 172.21.15.86:0/1430030122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa4164b30 0x7f2fa4166f20 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f2f9c009d90 tx=0x7f2f9c02be20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ccf739bdc2060c02 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.604+0000 7f2fa1600640 1 -- 172.21.15.86:0/1430030122 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f9c039070 con 0x7f2fa4164b30 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.604+0000 7f2fa1600640 1 -- 172.21.15.86:0/1430030122 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f9c02cbd0 con 0x7f2fa4164b30 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.604+0000 7f2fa1600640 1 -- 172.21.15.86:0/1430030122 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f9c034440 con 0x7f2fa4164b30 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.605+0000 7f2fa9a00640 1 -- 172.21.15.86:0/1430030122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa4164b30 msgr2=0x7f2fa4166f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.605+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/1430030122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa4164b30 0x7f2fa4166f20 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f2f9c009d90 tx=0x7f2f9c02be20 comp rx=0 tx=0).stop 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.605+0000 7f2fa9a00640 1 -- 172.21.15.86:0/1430030122 shutdown_connections 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.605+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/1430030122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa4164b30 0x7f2fa4166f20 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.605+0000 7f2fa9a00640 1 -- 172.21.15.86:0/1430030122 >> 172.21.15.86:0/1430030122 conn(0x7f2fa40cc4f0 msgr2=0x7f2fa40cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.605+0000 7f2fa9a00640 1 -- 172.21.15.86:0/1430030122 shutdown_connections 2024-03-06T07:28:01.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.605+0000 7f2fa9a00640 1 -- 172.21.15.86:0/1430030122 wait complete. 2024-03-06T07:28:01.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.607+0000 7f2fa9a00640 1 Processor -- start 2024-03-06T07:28:01.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.607+0000 7f2fa9a00640 1 -- start start 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa9a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa40c4930 0x7f2fa40cf120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa9a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2fa40c6460 0x7f2fa413e330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2fa4164b30 0x7f2fa413bac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa9a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2fa40ca3b0 con 0x7f2fa40c4930 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa9a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2fa40ca0d0 con 0x7f2fa40c6460 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa9a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2fa406e600 con 0x7f2fa4164b30 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa40c4930 0x7f2fa40cf120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa40c4930 0x7f2fa40cf120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51174/0 (socket says 172.21.15.86:51174) 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa2a00640 1 -- 172.21.15.86:0/3325981324 learned_addr learned my addr 172.21.15.86:0/3325981324 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:01.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa3400640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2fa4164b30 0x7f2fa413bac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa2000640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2fa40c6460 0x7f2fa413e330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa2a00640 1 -- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2fa4164b30 msgr2=0x7f2fa413bac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa2a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2fa4164b30 0x7f2fa413bac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.608+0000 7f2fa2a00640 1 -- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2fa40c6460 msgr2=0x7f2fa413e330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2fa2a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2fa40c6460 0x7f2fa413e330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2fa2a00640 1 -- 172.21.15.86:0/3325981324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2fa40cea50 con 0x7f2fa40c4930 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2fa2000640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2fa40c6460 0x7f2fa413e330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2fa2a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa40c4930 0x7f2fa40cf120 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f2f9400c3d0 tx=0x7f2f9400c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2f9aa00640 1 -- 172.21.15.86:0/3325981324 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f94017070 con 0x7f2fa40c4930 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2f9aa00640 1 -- 172.21.15.86:0/3325981324 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f94005ce0 con 0x7f2fa40c4930 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2f9aa00640 1 -- 172.21.15.86:0/3325981324 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f94007310 con 0x7f2fa40c4930 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2fa40ce770 con 0x7f2fa40c4930 2024-03-06T07:28:01.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.609+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2fa4070e00 con 0x7f2fa40c4930 2024-03-06T07:28:01.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.610+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2fa404f330 con 0x7f2fa40c4930 2024-03-06T07:28:01.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.611+0000 7f2f9aa00640 1 -- 172.21.15.86:0/3325981324 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2f9401a070 con 0x7f2fa40c4930 2024-03-06T07:28:01.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.612+0000 7f2f9aa00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f8006b4d0 0x7f2f8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:01.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.612+0000 7f2f9aa00640 1 -- 172.21.15.86:0/3325981324 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5833+0+0 (secure 0 0 0) 0x7f2f94089700 con 0x7f2fa40c4930 2024-03-06T07:28:01.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.612+0000 7f2fa2000640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f8006b4d0 0x7f2f8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:01.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.613+0000 7f2fa2000640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f8006b4d0 0x7f2f8006d990 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f2fa41f74a0 tx=0x7f2f8c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:01.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.617+0000 7f2f9aa00640 1 -- 172.21.15.86:0/3325981324 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2fa4145b30 con 0x7f2fa40c4930 2024-03-06T07:28:01.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.845+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f2fa415cad0 con 0x7f2fa40c4930 2024-03-06T07:28:01.848 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:28:01.846+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:28:01.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.850+0000 7f2f9aa00640 1 -- 172.21.15.86:0/3325981324 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v9) v1 ==== 161+0+0 (secure 0 0 0) 0x7f2f940584f0 con 0x7f2fa40c4930 2024-03-06T07:28:01.852 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:28:01.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.852+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f8006b4d0 msgr2=0x7f2f8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.852+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f8006b4d0 0x7f2f8006d990 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f2fa41f74a0 tx=0x7f2f8c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:01.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.852+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa40c4930 msgr2=0x7f2fa40cf120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:01.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.852+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa40c4930 0x7f2fa40cf120 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f2f9400c3d0 tx=0x7f2f9400c890 comp rx=0 tx=0).stop 2024-03-06T07:28:01.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.852+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 shutdown_connections 2024-03-06T07:28:01.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.852+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f8006b4d0 0x7f2f8006d990 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.853+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2fa4164b30 0x7f2fa413bac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.853+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2fa40c6460 0x7f2fa413e330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.853+0000 7f2fa9a00640 1 --2- 172.21.15.86:0/3325981324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2fa40c4930 0x7f2fa40cf120 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:01.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.853+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 >> 172.21.15.86:0/3325981324 conn(0x7f2fa40cc4f0 msgr2=0x7f2fa415bd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:01.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.853+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 shutdown_connections 2024-03-06T07:28:01.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:01.853+0000 7f2fa9a00640 1 -- 172.21.15.86:0/3325981324 wait complete. 2024-03-06T07:28:01.874 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:28:01.893 INFO:tasks.ceph.mds.b.smithi194.stderr: -7618> 2024-03-06T07:27:11.145+0000 7fcc76200640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:28:01.982 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:28:02.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.077+0000 7f5a56200640 1 Processor -- start 2024-03-06T07:28:02.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.077+0000 7f5a56200640 1 -- start start 2024-03-06T07:28:02.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a56200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:02.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a56200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5a500b3650 0x7f5a500b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:02.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a56200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5a500b4710 con 0x7f5a500cda90 2024-03-06T07:28:02.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a56200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5a50072310 con 0x7f5a500b3650 2024-03-06T07:28:02.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a56200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a500719e0 con 0x7f5a500b3df0 2024-03-06T07:28:02.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a47e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5a500b3650 0x7f5a500b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:02.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a47e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5a500b3650 0x7f5a500b41d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:49200/0 (socket says 172.21.15.86:49200) 2024-03-06T07:28:02.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a47e00640 1 -- 172.21.15.86:0/3647720613 learned_addr learned my addr 172.21.15.86:0/3647720613 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:02.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.079+0000 7f5a4f400640 1 --2- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:02.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.080+0000 7f5a47e00640 1 -- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 msgr2=0x7f5a500727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:02.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.080+0000 7f5a47e00640 1 --2- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.080+0000 7f5a47e00640 1 -- 172.21.15.86:0/3647720613 >> v1:172.21.15.194:6789/0 conn(0x7f5a500cda90 legacy=0x7f5a5006c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:02.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.080+0000 7f5a47e00640 1 -- 172.21.15.86:0/3647720613 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a50071cc0 con 0x7f5a500b3650 2024-03-06T07:28:02.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.080+0000 7f5a4f400640 1 --2- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500727a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:28:02.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.082+0000 7f5a47e00640 1 --2- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500b3650 0x7f5a500b41d0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f5a3800bb10 tx=0x7f5a3802e150 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c768d49049167ea8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:02.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.082+0000 7f5a4ea00640 1 -- 172.21.15.86:0/3647720613 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a3803b070 con 0x7f5a500b3650 2024-03-06T07:28:02.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.082+0000 7f5a4ea00640 1 -- 172.21.15.86:0/3647720613 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a38036440 con 0x7f5a500b3650 2024-03-06T07:28:02.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.082+0000 7f5a4ea00640 1 -- 172.21.15.86:0/3647720613 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a38036810 con 0x7f5a500b3650 2024-03-06T07:28:02.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 -- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500b3650 msgr2=0x7f5a500b41d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:02.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 --2- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500b3650 0x7f5a500b41d0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f5a3800bb10 tx=0x7f5a3802e150 comp rx=0 tx=0).stop 2024-03-06T07:28:02.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 -- 172.21.15.86:0/3647720613 shutdown_connections 2024-03-06T07:28:02.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 --2- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500b3650 0x7f5a500b41d0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 --2- 172.21.15.86:0/3647720613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500727a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 -- 172.21.15.86:0/3647720613 >> 172.21.15.86:0/3647720613 conn(0x7f5a500b1940 msgr2=0x7f5a500b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:02.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 -- 172.21.15.86:0/3647720613 shutdown_connections 2024-03-06T07:28:02.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.083+0000 7f5a56200640 1 -- 172.21.15.86:0/3647720613 wait complete. 2024-03-06T07:28:02.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.085+0000 7f5a56200640 1 Processor -- start 2024-03-06T07:28:02.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.085+0000 7f5a56200640 1 -- start start 2024-03-06T07:28:02.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a56200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a500b3650 0x7f5a500caaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:02.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a56200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500c9940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:02.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a56200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500cda90 0x7f5a50151500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:02.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a56200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a5015b240 con 0x7f5a500b3650 2024-03-06T07:28:02.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a56200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a500b3b30 con 0x7f5a500cda90 2024-03-06T07:28:02.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a56200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a5015b3c0 con 0x7f5a500b3df0 2024-03-06T07:28:02.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a500b3650 0x7f5a500caaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:02.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a500b3650 0x7f5a500caaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51186/0 (socket says 172.21.15.86:51186) 2024-03-06T07:28:02.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 -- 172.21.15.86:0/3246484333 learned_addr learned my addr 172.21.15.86:0/3246484333 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:02.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a47e00640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500c9940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:02.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4fe00640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500cda90 0x7f5a50151500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:02.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 -- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 msgr2=0x7f5a500c9940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:02.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500c9940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 -- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500cda90 msgr2=0x7f5a50151500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500cda90 0x7f5a50151500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.086+0000 7f5a4f400640 1 -- 172.21.15.86:0/3246484333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a500cb030 con 0x7f5a500b3650 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.087+0000 7f5a4f400640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a500b3650 0x7f5a500caaf0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f5a4000ebe0 tx=0x7f5a4000c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.087+0000 7f5a47400640 1 -- 172.21.15.86:0/3246484333 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a40017070 con 0x7f5a500b3650 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.087+0000 7f5a47400640 1 -- 172.21.15.86:0/3246484333 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a4000cc30 con 0x7f5a500b3650 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.087+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5a500cb300 con 0x7f5a500b3650 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.087+0000 7f5a47400640 1 -- 172.21.15.86:0/3246484333 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a40013470 con 0x7f5a500b3650 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.087+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5a500c9140 con 0x7f5a500b3650 2024-03-06T07:28:02.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.088+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5a5004f330 con 0x7f5a500b3650 2024-03-06T07:28:02.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.089+0000 7f5a47400640 1 -- 172.21.15.86:0/3246484333 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5a4001a070 con 0x7f5a500b3650 2024-03-06T07:28:02.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.090+0000 7f5a47400640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5a2c06b4d0 0x7f5a2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:02.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.090+0000 7f5a47400640 1 -- 172.21.15.86:0/3246484333 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(25..25 src has 1..25) v4 ==== 5919+0+0 (secure 0 0 0) 0x7f5a4008a320 con 0x7f5a500b3650 2024-03-06T07:28:02.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.090+0000 7f5a47e00640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5a2c06b4d0 0x7f5a2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:02.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.091+0000 7f5a47e00640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5a2c06b4d0 0x7f5a2c06d990 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f5a38038000 tx=0x7f5a38032000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:02.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.094+0000 7f5a47400640 1 -- 172.21.15.86:0/3246484333 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5a40058fa0 con 0x7f5a500b3650 2024-03-06T07:28:02.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.315+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f5a500add90 con 0x7f5a500b3650 2024-03-06T07:28:02.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.854+0000 7f5a47400640 1 -- 172.21.15.86:0/3246484333 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v10) v1 ==== 104+0+0 (secure 0 0 0) 0x7f5a400588a0 con 0x7f5a500b3650 2024-03-06T07:28:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5a2c06b4d0 msgr2=0x7f5a2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5a2c06b4d0 0x7f5a2c06d990 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f5a38038000 tx=0x7f5a38032000 comp rx=0 tx=0).stop 2024-03-06T07:28:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a500b3650 msgr2=0x7f5a500caaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a500b3650 0x7f5a500caaf0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f5a4000ebe0 tx=0x7f5a4000c520 comp rx=0 tx=0).stop 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 shutdown_connections 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5a2c06b4d0 0x7f5a2c06d990 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a500cda90 0x7f5a50151500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a500b3df0 0x7f5a500c9940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 --2- 172.21.15.86:0/3246484333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a500b3650 0x7f5a500caaf0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.857+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 >> 172.21.15.86:0/3246484333 conn(0x7f5a500b1940 msgr2=0x7f5a500acd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.858+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 shutdown_connections 2024-03-06T07:28:02.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:02.858+0000 7f5a56200640 1 -- 172.21.15.86:0/3246484333 wait complete. 2024-03-06T07:28:02.884 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:28:03.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.098+0000 7fabf6c00640 1 Processor -- start 2024-03-06T07:28:03.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.098+0000 7fabf6c00640 1 -- start start 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.099+0000 7fabf6c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf015e200 0x7fabf00aed30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.099+0000 7fabf6c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fabf00cec70 0x7fabf00cc930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.099+0000 7fabf6c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fabf00cd550 con 0x7fabf00cc4d0 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.099+0000 7fabf6c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fabf00cce70 con 0x7fabf00cec70 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.099+0000 7fabf6c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fabf0162b50 con 0x7fabf015e200 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabf4c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fabf00cec70 0x7fabf00cc930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabf4c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fabf00cec70 0x7fabf00cc930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51188/0 (socket says 172.21.15.86:51188) 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabf4c00640 1 -- 172.21.15.86:0/1495459919 learned_addr learned my addr 172.21.15.86:0/1495459919 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabefe00640 1 --2- 172.21.15.86:0/1495459919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf015e200 0x7fabf00aed30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabf4c00640 1 -- 172.21.15.86:0/1495459919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf015e200 msgr2=0x7fabf00aed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabf4c00640 1 --2- 172.21.15.86:0/1495459919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf015e200 0x7fabf00aed30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabf4c00640 1 -- 172.21.15.86:0/1495459919 >> v1:172.21.15.86:6789/0 conn(0x7fabf00cc4d0 legacy=0x7fabf00ad860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabf4c00640 1 -- 172.21.15.86:0/1495459919 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fabf0161c60 con 0x7fabf00cec70 2024-03-06T07:28:03.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.100+0000 7fabefe00640 1 --2- 172.21.15.86:0/1495459919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf015e200 0x7fabf00aed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:03.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.101+0000 7fabf4c00640 1 --2- 172.21.15.86:0/1495459919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf00cec70 0x7fabf00cc930 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fabe8009e80 tx=0x7fabe802bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=521f9539667ee9f5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:03.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabef400640 1 -- 172.21.15.86:0/1495459919 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabe802caf0 con 0x7fabf00cec70 2024-03-06T07:28:03.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabef400640 1 -- 172.21.15.86:0/1495459919 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fabe802cc90 con 0x7fabf00cec70 2024-03-06T07:28:03.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabef400640 1 -- 172.21.15.86:0/1495459919 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabe8034610 con 0x7fabf00cec70 2024-03-06T07:28:03.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 -- 172.21.15.86:0/1495459919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf00cec70 msgr2=0x7fabf00cc930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:03.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 --2- 172.21.15.86:0/1495459919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf00cec70 0x7fabf00cc930 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fabe8009e80 tx=0x7fabe802bf50 comp rx=0 tx=0).stop 2024-03-06T07:28:03.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 -- 172.21.15.86:0/1495459919 shutdown_connections 2024-03-06T07:28:03.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 --2- 172.21.15.86:0/1495459919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf00cec70 0x7fabf00cc930 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:03.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 --2- 172.21.15.86:0/1495459919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf015e200 0x7fabf00aed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:03.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 -- 172.21.15.86:0/1495459919 >> 172.21.15.86:0/1495459919 conn(0x7fabf00cfa80 msgr2=0x7fabf00cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:03.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 -- 172.21.15.86:0/1495459919 shutdown_connections 2024-03-06T07:28:03.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.102+0000 7fabf6c00640 1 -- 172.21.15.86:0/1495459919 wait complete. 2024-03-06T07:28:03.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.104+0000 7fabf6c00640 1 Processor -- start 2024-03-06T07:28:03.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.105+0000 7fabf6c00640 1 -- start start 2024-03-06T07:28:03.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf6c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf00cc4d0 0x7fabf00c54b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:03.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf6c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabf00cec70 0x7fabf01610a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:03.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf6c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf015e200 0x7fabf01549d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:03.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf6c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fabf0162500 con 0x7fabf015e200 2024-03-06T07:28:03.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf6c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fabf0162220 con 0x7fabf00cec70 2024-03-06T07:28:03.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf6c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fabf01627e0 con 0x7fabf00cc4d0 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf015e200 0x7fabf01549d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf015e200 0x7fabf01549d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51200/0 (socket says 172.21.15.86:51200) 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 -- 172.21.15.86:0/3418092740 learned_addr learned my addr 172.21.15.86:0/3418092740 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabefe00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf00cc4d0 0x7fabf00c54b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabe7400640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabf00cec70 0x7fabf01610a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 -- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf00cc4d0 msgr2=0x7fabf00c54b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf00cc4d0 0x7fabf00c54b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 -- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabf00cec70 msgr2=0x7fabf01610a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabf00cec70 0x7fabf01610a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabf4c00640 1 -- 172.21.15.86:0/3418092740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fabf00c5bb0 con 0x7fabf015e200 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.106+0000 7fabe7400640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabf00cec70 0x7fabf01610a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:03.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.107+0000 7fabf4c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf015e200 0x7fabf01549d0 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fabe8009fa0 tx=0x7fabe8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:03.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.107+0000 7fabecc00640 1 -- 172.21.15.86:0/3418092740 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabe8043020 con 0x7fabf015e200 2024-03-06T07:28:03.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.107+0000 7fabecc00640 1 -- 172.21.15.86:0/3418092740 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fabe803e450 con 0x7fabf015e200 2024-03-06T07:28:03.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.107+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fabf00c5e90 con 0x7fabf015e200 2024-03-06T07:28:03.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.107+0000 7fabecc00640 1 -- 172.21.15.86:0/3418092740 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabe8009510 con 0x7fabf015e200 2024-03-06T07:28:03.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.107+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fabf015e710 con 0x7fabf015e200 2024-03-06T07:28:03.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.108+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fabf004f3a0 con 0x7fabf015e200 2024-03-06T07:28:03.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.109+0000 7fabecc00640 1 -- 172.21.15.86:0/3418092740 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fabe8046020 con 0x7fabf015e200 2024-03-06T07:28:03.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.110+0000 7fabecc00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fabcc06b4d0 0x7fabcc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:03.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.110+0000 7fabecc00640 1 -- 172.21.15.86:0/3418092740 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(25..25 src has 1..25) v4 ==== 5919+0+0 (secure 0 0 0) 0x7fabe80acb80 con 0x7fabf015e200 2024-03-06T07:28:03.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.110+0000 7fabefe00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fabcc06b4d0 0x7fabcc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:03.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.111+0000 7fabefe00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fabcc06b4d0 0x7fabcc06d990 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fabe000a950 tx=0x7fabe0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:03.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.115+0000 7fabecc00640 1 -- 172.21.15.86:0/3418092740 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fabe804d030 con 0x7fabf015e200 2024-03-06T07:28:03.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:03.306+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fabf00cbd40 con 0x7fabf015e200 2024-03-06T07:28:04.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.060+0000 7fabecc00640 1 -- 172.21.15.86:0/3418092740 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v26) v1 ==== 181+0+0 (secure 0 0 0) 0x7fabe807b920 con 0x7fabf015e200 2024-03-06T07:28:04.063 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:28:04.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.063+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fabcc06b4d0 msgr2=0x7fabcc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:04.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.063+0000 7fabf6c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fabcc06b4d0 0x7fabcc06d990 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fabe000a950 tx=0x7fabe0008040 comp rx=0 tx=0).stop 2024-03-06T07:28:04.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.063+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf015e200 msgr2=0x7fabf01549d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:04.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.063+0000 7fabf6c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf015e200 0x7fabf01549d0 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fabe8009fa0 tx=0x7fabe8004a40 comp rx=0 tx=0).stop 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.063+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 shutdown_connections 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.064+0000 7fabf6c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fabcc06b4d0 0x7fabcc06d990 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.064+0000 7fabf6c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabf015e200 0x7fabf01549d0 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.064+0000 7fabf6c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabf00cec70 0x7fabf01610a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.064+0000 7fabf6c00640 1 --2- 172.21.15.86:0/3418092740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabf00cc4d0 0x7fabf00c54b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.064+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 >> 172.21.15.86:0/3418092740 conn(0x7fabf00cfa80 msgr2=0x7fabf015ce10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.064+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 shutdown_connections 2024-03-06T07:28:04.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.064+0000 7fabf6c00640 1 -- 172.21.15.86:0/3418092740 wait complete. 2024-03-06T07:28:04.088 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:28:04.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.294+0000 7f483ea00640 1 Processor -- start 2024-03-06T07:28:04.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.294+0000 7f483ea00640 1 -- start start 2024-03-06T07:28:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f483ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f48380b7810 0x7f48380d0ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f483ea00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f48380d1020 0x7f483806c2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f483ea00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380b4b70 0x7f483806c820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f483ea00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f48380cc350 con 0x7f48380b7810 2024-03-06T07:28:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f483ea00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f48380cced0 con 0x7f48380d1020 2024-03-06T07:28:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f483ea00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f48380cc5a0 con 0x7f48380b4b70 2024-03-06T07:28:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f4837e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f48380b7810 0x7f48380d0ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f4837e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f48380b7810 0x7f48380d0ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51214/0 (socket says 172.21.15.86:51214) 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f4837e00640 1 -- 172.21.15.86:0/2510498871 learned_addr learned my addr 172.21.15.86:0/2510498871 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.296+0000 7f482f400640 1 --2- 172.21.15.86:0/2510498871 >> v2:172.21.15.194:3300/0 conn(0x7f48380d1020 0x7f483806c2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.297+0000 7f483ca00640 1 --2- 172.21.15.86:0/2510498871 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380b4b70 0x7f483806c820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.297+0000 7f4837e00640 1 -- 172.21.15.86:0/2510498871 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380b4b70 msgr2=0x7f483806c820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.297+0000 7f4837e00640 1 --2- 172.21.15.86:0/2510498871 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380b4b70 0x7f483806c820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.297+0000 7f4837e00640 1 -- 172.21.15.86:0/2510498871 >> v2:172.21.15.194:3300/0 conn(0x7f48380d1020 msgr2=0x7f483806c2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.297+0000 7f4837e00640 1 --2- 172.21.15.86:0/2510498871 >> v2:172.21.15.194:3300/0 conn(0x7f48380d1020 0x7f483806c2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.297+0000 7f4837e00640 1 -- 172.21.15.86:0/2510498871 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48380cc880 con 0x7f48380b7810 2024-03-06T07:28:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.297+0000 7f483ca00640 1 --2- 172.21.15.86:0/2510498871 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380b4b70 0x7f483806c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:04.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.298+0000 7f4837e00640 1 --2- 172.21.15.86:0/2510498871 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b7810 0x7f48380d0ae0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f4838071a00 tx=0x7f482802b990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bb7e735fe79c548 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.298+0000 7f4837400640 1 -- 172.21.15.86:0/2510498871 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4828034070 con 0x7f48380b7810 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.298+0000 7f4837400640 1 -- 172.21.15.86:0/2510498871 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4828030850 con 0x7f48380b7810 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.298+0000 7f4837400640 1 -- 172.21.15.86:0/2510498871 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4828030c20 con 0x7f48380b7810 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 -- 172.21.15.86:0/2510498871 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b7810 msgr2=0x7f48380d0ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 --2- 172.21.15.86:0/2510498871 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b7810 0x7f48380d0ae0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f4838071a00 tx=0x7f482802b990 comp rx=0 tx=0).stop 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 -- 172.21.15.86:0/2510498871 shutdown_connections 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 --2- 172.21.15.86:0/2510498871 >> v2:172.21.15.194:3300/0 conn(0x7f48380d1020 0x7f483806c2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 --2- 172.21.15.86:0/2510498871 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380b4b70 0x7f483806c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 --2- 172.21.15.86:0/2510498871 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b7810 0x7f48380d0ae0 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 -- 172.21.15.86:0/2510498871 >> 172.21.15.86:0/2510498871 conn(0x7f48380b3870 msgr2=0x7f48380b3c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 -- 172.21.15.86:0/2510498871 shutdown_connections 2024-03-06T07:28:04.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.299+0000 7f483ea00640 1 -- 172.21.15.86:0/2510498871 wait complete. 2024-03-06T07:28:04.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.301+0000 7f483ea00640 1 Processor -- start 2024-03-06T07:28:04.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.301+0000 7f483ea00640 1 -- start start 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f483ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b4b70 0x7f48381e84a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f483ea00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48380b7810 0x7f48380cac20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f483ea00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380d1020 0x7f48380cb160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f483ea00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f483806cf60 con 0x7f48380b4b70 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f483ea00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48380b7bf0 con 0x7f48380b7810 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f483ea00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f483806d0e0 con 0x7f48380d1020 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b4b70 0x7f48381e84a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b4b70 0x7f48381e84a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51228/0 (socket says 172.21.15.86:51228) 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 -- 172.21.15.86:0/1629060118 learned_addr learned my addr 172.21.15.86:0/1629060118 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f483ca00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380d1020 0x7f48380cb160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:04.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 -- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380d1020 msgr2=0x7f48380cb160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:04.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f482f400640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48380b7810 0x7f48380cac20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:04.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380d1020 0x7f48380cb160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 -- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48380b7810 msgr2=0x7f48380cac20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:04.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48380b7810 0x7f48380cac20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:04.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.302+0000 7f4837e00640 1 -- 172.21.15.86:0/1629060118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48381e78e0 con 0x7f48380b4b70 2024-03-06T07:28:04.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.303+0000 7f4837e00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b4b70 0x7f48381e84a0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f4828000c00 tx=0x7f4828004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:04.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.303+0000 7f4834c00640 1 -- 172.21.15.86:0/1629060118 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4828048070 con 0x7f48380b4b70 2024-03-06T07:28:04.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.303+0000 7f4834c00640 1 -- 172.21.15.86:0/1629060118 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4828030650 con 0x7f48380b4b70 2024-03-06T07:28:04.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.303+0000 7f4834c00640 1 -- 172.21.15.86:0/1629060118 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4828034040 con 0x7f48380b4b70 2024-03-06T07:28:04.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.303+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f48381e6b20 con 0x7f48380b4b70 2024-03-06T07:28:04.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.303+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48381e70e0 con 0x7f48380b4b70 2024-03-06T07:28:04.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.304+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f483804f330 con 0x7f48380b4b70 2024-03-06T07:28:04.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.305+0000 7f4834c00640 1 -- 172.21.15.86:0/1629060118 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f482804b070 con 0x7f48380b4b70 2024-03-06T07:28:04.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.306+0000 7f4834c00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f481406b4d0 0x7f481406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:04.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.306+0000 7f4834c00640 1 -- 172.21.15.86:0/1629060118 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(26..26 src has 1..26) v4 ==== 5481+0+0 (secure 0 0 0) 0x7f48280ac5a0 con 0x7f48380b4b70 2024-03-06T07:28:04.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.306+0000 7f482f400640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f481406b4d0 0x7f481406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:04.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.307+0000 7f482f400640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f481406b4d0 0x7f481406d990 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f4820006fd0 tx=0x7f4820008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:04.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.311+0000 7f4834c00640 1 -- 172.21.15.86:0/1629060118 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f482807b580 con 0x7f48380b4b70 2024-03-06T07:28:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:04.520+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f48380b3650 con 0x7f48380b4b70 2024-03-06T07:28:05.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.071+0000 7f4834c00640 1 -- 172.21.15.86:0/1629060118 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v27) v1 ==== 169+0+0 (secure 0 0 0) 0x7f482807ae80 con 0x7f48380b4b70 2024-03-06T07:28:05.074 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:28:05.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.074+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f481406b4d0 msgr2=0x7f481406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:05.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f481406b4d0 0x7f481406d990 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f4820006fd0 tx=0x7f4820008040 comp rx=0 tx=0).stop 2024-03-06T07:28:05.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b4b70 msgr2=0x7f48381e84a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b4b70 0x7f48381e84a0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f4828000c00 tx=0x7f4828004730 comp rx=0 tx=0).stop 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 shutdown_connections 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f481406b4d0 0x7f481406d990 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48380d1020 0x7f48380cb160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48380b7810 0x7f48380cac20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 --2- 172.21.15.86:0/1629060118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48380b4b70 0x7f48381e84a0 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 >> 172.21.15.86:0/1629060118 conn(0x7f48380b3870 msgr2=0x7f48380cffd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 shutdown_connections 2024-03-06T07:28:05.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.075+0000 7f483ea00640 1 -- 172.21.15.86:0/1629060118 wait complete. 2024-03-06T07:28:05.102 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:28:05.103 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:28:05.103 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:28:05.103 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:05.103 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:28:05.103 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:28:05.103 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:28:05.103 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:05.104 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:28:05.104 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:28:05.104 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:28:05.104 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:05.104 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:28:05.104 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:28:05.104 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:28:05.105 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:05.121 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:28:05.121 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:28:05.122 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:28:05.122 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:28:05.125 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:28:05.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:28:05.133 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:28:05.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:28:05.168 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:28:05.168 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:05.173 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:28:05.174 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:05.179 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:28:05.179 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:05.185 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:28:05.185 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:05.185 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:28:05.321 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:28:05.322 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:28:05.340 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:28:05.340 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:28:05.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.365+0000 7ff1d4c00640 1 Processor -- start 2024-03-06T07:28:05.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.366+0000 7ff1d4c00640 1 -- start start 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1d4c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c4930 0x7ff1d00c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1d4c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff1d00c4d10 0x7ff1d00c5170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1d4c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff1d00c56b0 0x7ff1d006d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1d4c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff1d0145b30 con 0x7ff1d00c56b0 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1d4c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff1d0142ce0 con 0x7ff1d00c4d10 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1d4c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff1d00cbca0 con 0x7ff1d00c4930 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1ce800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff1d00c56b0 0x7ff1d006d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1ce800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff1d00c56b0 0x7ff1d006d5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51272/0 (socket says 172.21.15.86:51272) 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1ce800640 1 -- 172.21.15.86:0/1256187262 learned_addr learned my addr 172.21.15.86:0/1256187262 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1cde00640 1 --2- 172.21.15.86:0/1256187262 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c4930 0x7ff1d00c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:05.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1cd400640 1 --2- 172.21.15.86:0/1256187262 >> v2:172.21.15.194:3300/0 conn(0x7ff1d00c4d10 0x7ff1d00c5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:05.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1ce800640 1 -- 172.21.15.86:0/1256187262 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c4930 msgr2=0x7ff1d00c6460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:05.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1ce800640 1 --2- 172.21.15.86:0/1256187262 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c4930 0x7ff1d00c6460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1ce800640 1 -- 172.21.15.86:0/1256187262 >> v2:172.21.15.194:3300/0 conn(0x7ff1d00c4d10 msgr2=0x7ff1d00c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:05.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.367+0000 7ff1ce800640 1 --2- 172.21.15.86:0/1256187262 >> v2:172.21.15.194:3300/0 conn(0x7ff1d00c4d10 0x7ff1d00c5170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.368+0000 7ff1ce800640 1 -- 172.21.15.86:0/1256187262 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1d00cbf80 con 0x7ff1d00c56b0 2024-03-06T07:28:05.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1ce800640 1 --2- 172.21.15.86:0/1256187262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c56b0 0x7ff1d006d5f0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7ff1d00d0aa0 tx=0x7ff1c802be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=77fa7f7ddda817bc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:05.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1cca00640 1 -- 172.21.15.86:0/1256187262 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1c802c9e0 con 0x7ff1d00c56b0 2024-03-06T07:28:05.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1cca00640 1 -- 172.21.15.86:0/1256187262 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff1c802cb80 con 0x7ff1d00c56b0 2024-03-06T07:28:05.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1cca00640 1 -- 172.21.15.86:0/1256187262 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1c80343f0 con 0x7ff1d00c56b0 2024-03-06T07:28:05.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1d4c00640 1 -- 172.21.15.86:0/1256187262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c56b0 msgr2=0x7ff1d006d5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:05.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/1256187262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c56b0 0x7ff1d006d5f0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7ff1d00d0aa0 tx=0x7ff1c802be40 comp rx=0 tx=0).stop 2024-03-06T07:28:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1d4c00640 1 -- 172.21.15.86:0/1256187262 shutdown_connections 2024-03-06T07:28:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/1256187262 >> v2:172.21.15.194:3300/0 conn(0x7ff1d00c4d10 0x7ff1d00c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/1256187262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c56b0 0x7ff1d006d5f0 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/1256187262 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c4930 0x7ff1d00c6460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.369+0000 7ff1d4c00640 1 -- 172.21.15.86:0/1256187262 >> 172.21.15.86:0/1256187262 conn(0x7ff1d00cc4f0 msgr2=0x7ff1d00cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.370+0000 7ff1d4c00640 1 -- 172.21.15.86:0/1256187262 shutdown_connections 2024-03-06T07:28:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.370+0000 7ff1d4c00640 1 -- 172.21.15.86:0/1256187262 wait complete. 2024-03-06T07:28:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.371+0000 7ff1d4c00640 1 Processor -- start 2024-03-06T07:28:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.371+0000 7ff1d4c00640 1 -- start start 2024-03-06T07:28:05.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.372+0000 7ff1d4c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1d00c4930 0x7ff1d013c4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:05.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.372+0000 7ff1d4c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c4d10 0x7ff1d00ad860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:05.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.372+0000 7ff1d4c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c56b0 0x7ff1d013d820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:05.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.372+0000 7ff1d4c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1d00cc260 con 0x7ff1d00c4d10 2024-03-06T07:28:05.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.372+0000 7ff1d4c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1d00ca3b0 con 0x7ff1d00c4930 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.372+0000 7ff1d4c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff1d00c5b40 con 0x7ff1d00c56b0 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c4d10 0x7ff1d00ad860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c4d10 0x7ff1d00ad860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51284/0 (socket says 172.21.15.86:51284) 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 -- 172.21.15.86:0/182968398 learned_addr learned my addr 172.21.15.86:0/182968398 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cde00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1d00c4930 0x7ff1d013c4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1ce800640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c56b0 0x7ff1d013d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 -- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c56b0 msgr2=0x7ff1d013d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c56b0 0x7ff1d013d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 -- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1d00c4930 msgr2=0x7ff1d013c4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1d00c4930 0x7ff1d013c4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 -- 172.21.15.86:0/182968398 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1d00adda0 con 0x7ff1d00c4d10 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.373+0000 7ff1cd400640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c4d10 0x7ff1d00ad860 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7ff1bc00d620 tx=0x7ff1bc00dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.374+0000 7ff1c6000640 1 -- 172.21.15.86:0/182968398 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1bc00b620 con 0x7ff1d00c4d10 2024-03-06T07:28:05.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.374+0000 7ff1c6000640 1 -- 172.21.15.86:0/182968398 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff1bc019070 con 0x7ff1d00c4d10 2024-03-06T07:28:05.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.374+0000 7ff1c6000640 1 -- 172.21.15.86:0/182968398 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1bc00ba00 con 0x7ff1d00c4d10 2024-03-06T07:28:05.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.374+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff1d00ae070 con 0x7ff1d00c4d10 2024-03-06T07:28:05.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.374+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1d00ae630 con 0x7ff1d00c4d10 2024-03-06T07:28:05.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.375+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff1d00ae830 con 0x7ff1d00c4d10 2024-03-06T07:28:05.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.376+0000 7ff1c6000640 1 -- 172.21.15.86:0/182968398 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff1bc00bba0 con 0x7ff1d00c4d10 2024-03-06T07:28:05.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.377+0000 7ff1c6000640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1b006b4d0 0x7ff1b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:05.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.377+0000 7ff1c6000640 1 -- 172.21.15.86:0/182968398 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(27..27 src has 1..27) v4 ==== 4943+0+0 (secure 0 0 0) 0x7ff1bc08a260 con 0x7ff1d00c4d10 2024-03-06T07:28:05.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.377+0000 7ff1cde00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1b006b4d0 0x7ff1b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:05.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.378+0000 7ff1cde00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1b006b4d0 0x7ff1b006d990 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7ff1c0003610 tx=0x7ff1c0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:05.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.381+0000 7ff1c6000640 1 -- 172.21.15.86:0/182968398 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff1bc058cf0 con 0x7ff1d00c4d10 2024-03-06T07:28:05.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:05.586+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7ff1d004f330 con 0x7ff1d00c4d10 2024-03-06T07:28:06.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.103+0000 7ff1c6000640 1 -- 172.21.15.86:0/182968398 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v29) v1 ==== 97+0+0 (secure 0 0 0) 0x7ff1bc05cd90 con 0x7ff1d00c4d10 2024-03-06T07:28:06.106 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1b006b4d0 msgr2=0x7ff1b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1b006b4d0 0x7ff1b006d990 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7ff1c0003610 tx=0x7ff1c0008040 comp rx=0 tx=0).stop 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c4d10 msgr2=0x7ff1d00ad860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c4d10 0x7ff1d00ad860 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7ff1bc00d620 tx=0x7ff1bc00dae0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 shutdown_connections 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1b006b4d0 0x7ff1b006d990 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1d00c56b0 0x7ff1d013d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1d00c4d10 0x7ff1d00ad860 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 --2- 172.21.15.86:0/182968398 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1d00c4930 0x7ff1d013c4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 >> 172.21.15.86:0/182968398 conn(0x7ff1d00cc4f0 msgr2=0x7ff1d0140220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 shutdown_connections 2024-03-06T07:28:06.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.107+0000 7ff1d4c00640 1 -- 172.21.15.86:0/182968398 wait complete. 2024-03-06T07:28:06.134 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:28:06.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.322+0000 7f3fbca00640 1 Processor -- start 2024-03-06T07:28:06.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.322+0000 7f3fbca00640 1 -- start start 2024-03-06T07:28:06.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.323+0000 7f3fbca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb815e230 0x7f3fb806edb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.323+0000 7f3fbca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3fb80b4320 0x7f3fb80d0be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.323+0000 7f3fbca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3fb80cf340 con 0x7f3fb80ce550 2024-03-06T07:28:06.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.323+0000 7f3fbca00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3fb815e610 con 0x7f3fb80b4320 2024-03-06T07:28:06.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.323+0000 7f3fbca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3fb80cfe60 con 0x7f3fb815e230 2024-03-06T07:28:06.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fafe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3fb80b4320 0x7f3fb80d0be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fafe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3fb80b4320 0x7f3fb80d0be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55508/0 (socket says 172.21.15.86:55508) 2024-03-06T07:28:06.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fafe00640 1 -- 172.21.15.86:0/2569227902 learned_addr learned my addr 172.21.15.86:0/2569227902 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:06.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fafe00640 1 -- 172.21.15.86:0/2569227902 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb815e230 msgr2=0x7f3fb806edb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:06.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fb5e00640 1 --2- 172.21.15.86:0/2569227902 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb815e230 0x7f3fb806edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fafe00640 1 --2- 172.21.15.86:0/2569227902 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb815e230 0x7f3fb806edb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fafe00640 1 -- 172.21.15.86:0/2569227902 >> v1:172.21.15.194:6789/0 conn(0x7f3fb80ce550 legacy=0x7f3fb80cc320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:06.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.324+0000 7f3fafe00640 1 -- 172.21.15.86:0/2569227902 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3fb806f390 con 0x7f3fb80b4320 2024-03-06T07:28:06.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.325+0000 7f3fafe00640 1 --2- 172.21.15.86:0/2569227902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80d0be0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f3fa0009870 tx=0x7f3fa002b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd750e5a6a31e94f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:06.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.325+0000 7f3fb5400640 1 -- 172.21.15.86:0/2569227902 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fa00317c0 con 0x7f3fb80b4320 2024-03-06T07:28:06.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fb5400640 1 -- 172.21.15.86:0/2569227902 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3fa0037070 con 0x7f3fb80b4320 2024-03-06T07:28:06.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fb5400640 1 -- 172.21.15.86:0/2569227902 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fa0031b80 con 0x7f3fb80b4320 2024-03-06T07:28:06.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 -- 172.21.15.86:0/2569227902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 msgr2=0x7f3fb80d0be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2569227902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80d0be0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f3fa0009870 tx=0x7f3fa002b860 comp rx=0 tx=0).stop 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 -- 172.21.15.86:0/2569227902 shutdown_connections 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2569227902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80d0be0 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2569227902 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb815e230 0x7f3fb806edb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 -- 172.21.15.86:0/2569227902 >> 172.21.15.86:0/2569227902 conn(0x7f3fb80b3800 msgr2=0x7f3fb80b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 -- 172.21.15.86:0/2569227902 shutdown_connections 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.326+0000 7f3fbca00640 1 -- 172.21.15.86:0/2569227902 wait complete. 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.328+0000 7f3fbca00640 1 Processor -- start 2024-03-06T07:28:06.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.328+0000 7f3fbca00640 1 -- start start 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fbca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80b6ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fbca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb80ce550 0x7f3fb815b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fbca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3fb815e230 0x7f3fb80af640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fbca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3fb80d06c0 con 0x7f3fb80b4320 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fbca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3fb80d03e0 con 0x7f3fb815e230 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fbca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3fb80d1120 con 0x7f3fb80ce550 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fb5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80b6ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fb5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80b6ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55510/0 (socket says 172.21.15.86:55510) 2024-03-06T07:28:06.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.329+0000 7f3fb5e00640 1 -- 172.21.15.86:0/2468634085 learned_addr learned my addr 172.21.15.86:0/2468634085 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:06.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fb5e00640 1 -- 172.21.15.86:0/2468634085 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb80ce550 msgr2=0x7f3fb815b390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fb5e00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb80ce550 0x7f3fb815b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fb5e00640 1 -- 172.21.15.86:0/2468634085 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3fb815e230 msgr2=0x7f3fb80af640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fb5e00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3fb815e230 0x7f3fb80af640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fb5e00640 1 -- 172.21.15.86:0/2468634085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3fb81f7cb0 con 0x7f3fb80b4320 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fb5e00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80b6ed0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f3fa800c7f0 tx=0x7f3fa800ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fae000640 1 -- 172.21.15.86:0/2468634085 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fa8015070 con 0x7f3fb80b4320 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fae000640 1 -- 172.21.15.86:0/2468634085 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3fa8004220 con 0x7f3fb80b4320 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3fb81f7f90 con 0x7f3fb80b4320 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fae000640 1 -- 172.21.15.86:0/2468634085 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fa8007620 con 0x7f3fb80b4320 2024-03-06T07:28:06.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.330+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3fb80b7410 con 0x7f3fb80b4320 2024-03-06T07:28:06.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.332+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3fb804f330 con 0x7f3fb80b4320 2024-03-06T07:28:06.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.332+0000 7f3fae000640 1 -- 172.21.15.86:0/2468634085 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3fa80079d0 con 0x7f3fb80b4320 2024-03-06T07:28:06.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.334+0000 7f3fae000640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f9406b4d0 0x7f3f9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.334+0000 7f3fae000640 1 -- 172.21.15.86:0/2468634085 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f3fa8089f00 con 0x7f3fb80b4320 2024-03-06T07:28:06.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.334+0000 7f3fafe00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f9406b4d0 0x7f3f9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.335+0000 7f3fafe00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f9406b4d0 0x7f3f9406d990 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f3fa0036000 tx=0x7f3fa0030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:06.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.338+0000 7f3fae000640 1 -- 172.21.15.86:0/2468634085 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3fa8059030 con 0x7f3fb80b4320 2024-03-06T07:28:06.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.605+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f3fb80b7c50 con 0x7f3fb80b4320 2024-03-06T07:28:06.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.606+0000 7f3fae000640 1 -- 172.21.15.86:0/2468634085 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v6) v1 ==== 80+0+6645 (secure 0 0 0) 0x7f3fa8058930 con 0x7f3fb80b4320 2024-03-06T07:28:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:06.608 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:28:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:06.610 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.611 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.612 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.613 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.614 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.615 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:06.616 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.617 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.618 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:28:06.619 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.620 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.bootstrap-mds", 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQC3Guhldp6pMRAAE+FUhirHbimKmvACarL8Lw==", 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile bootstrap-mds" 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.bootstrap-mgr", 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQC3GuhlXLipMRAANeJOZCdPRQtC1vc/QGORLw==", 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile bootstrap-mgr" 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.621 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.bootstrap-osd", 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQC3GuhlHNCpMRAATnYJgJExHaoXLZt9zM7Z2g==", 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile bootstrap-osd" 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.bootstrap-rbd", 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQC3GuhlGOepMRAArjPn4v1OK14DqFC+HwyQbg==", 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile bootstrap-rbd" 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.622 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.bootstrap-rbd-mirror", 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQC3Guhl9f6pMRAAXWTnxBKyzndwDjZ8UsTauw==", 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile bootstrap-rbd-mirror" 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.bootstrap-rgw", 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQC3Guhl0BeqMRAAkC5vxKfw4vizm/iPnw5yCw==", 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile bootstrap-rgw" 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.623 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:28:06.624 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:28:06.625 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:28:06.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f9406b4d0 msgr2=0x7f3f9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f9406b4d0 0x7f3f9406d990 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f3fa0036000 tx=0x7f3fa0030000 comp rx=0 tx=0).stop 2024-03-06T07:28:06.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 msgr2=0x7f3fb80b6ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80b6ed0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f3fa800c7f0 tx=0x7f3fa800ccb0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 shutdown_connections 2024-03-06T07:28:06.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f9406b4d0 0x7f3f9406d990 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3fb815e230 0x7f3fb80af640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3fb80ce550 0x7f3fb815b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 --2- 172.21.15.86:0/2468634085 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3fb80b4320 0x7f3fb80b6ed0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 >> 172.21.15.86:0/2468634085 conn(0x7f3fb80b3800 msgr2=0x7f3fb815d240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:06.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 shutdown_connections 2024-03-06T07:28:06.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.608+0000 7f3fbca00640 1 -- 172.21.15.86:0/2468634085 wait complete. 2024-03-06T07:28:06.628 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-mds 2024-03-06T07:28:06.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.833+0000 7f8b7a000640 1 Processor -- start 2024-03-06T07:28:06.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.833+0000 7f8b7a000640 1 -- start start 2024-03-06T07:28:06.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.835+0000 7f8b7a000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8b740d1660 0x7f8b740d0470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.835+0000 7f8b7a000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8b740cf3a0 con 0x7f8b740ce590 2024-03-06T07:28:06.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.835+0000 7f8b7a000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8b7406cd30 con 0x7f8b740cd470 2024-03-06T07:28:06.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.835+0000 7f8b7a000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8b7406cae0 con 0x7f8b740d1660 2024-03-06T07:28:06.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.835+0000 7f8b73400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8b740ce590 0x7f8b740ce970 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50162/0 (socket says 172.21.15.86:50162) 2024-03-06T07:28:06.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.835+0000 7f8b73400640 1 -- 172.21.15.86:0/1890242051 learned_addr learned my addr 172.21.15.86:0/1890242051 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:06.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.835+0000 7f8b73e00640 1 --2- 172.21.15.86:0/1890242051 >> v2:172.21.15.194:3300/0 conn(0x7f8b740d1660 0x7f8b740d0470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3765535826 0 0) 0x7f8b740cf3a0 con 0x7f8b740ce590 2024-03-06T07:28:06.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8b58003520 con 0x7f8b740ce590 2024-03-06T07:28:06.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 809687915 0 0) 0x7f8b58003520 con 0x7f8b740ce590 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 >> v2:172.21.15.194:3300/0 conn(0x7f8b740d1660 msgr2=0x7f8b740d0470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 --2- 172.21.15.86:0/1890242051 >> v2:172.21.15.194:3300/0 conn(0x7f8b740d1660 0x7f8b740d0470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 >> v1:172.21.15.194:6789/0 conn(0x7f8b740cd470 legacy=0x7f8b740cfd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b740b0e30 con 0x7f8b740ce590 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8b64001670 con 0x7f8b740ce590 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 >> v1:172.21.15.86:6789/0 conn(0x7f8b740ce590 legacy=0x7f8b740ce970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b58003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b58004800 0x7f8b58004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b580051c0 0x7f8b5800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b7406cae0 con 0x7f8b580039b0 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b5800c060 con 0x7f8b58004800 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.836+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8b5800c260 con 0x7f8b580051c0 2024-03-06T07:28:06.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b73400640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b58003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b6be00640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b58004800 0x7f8b58004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b73400640 1 -- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b580051c0 msgr2=0x7f8b5800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b73400640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b580051c0 0x7f8b5800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b73400640 1 -- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b58004800 msgr2=0x7f8b58004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b73400640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b58004800 0x7f8b58004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b73400640 1 -- 172.21.15.86:0/1890242051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b58003520 con 0x7f8b580039b0 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b6be00640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b58004800 0x7f8b58004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.837+0000 7f8b73e00640 1 -- 172.21.15.86:0/1890242051 reap_dead start 2024-03-06T07:28:06.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.838+0000 7f8b73400640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b58003dd0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f8b6400c210 tx=0x7f8b58005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c67d2cf36e32b36 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:06.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.838+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b64007d30 con 0x7f8b580039b0 2024-03-06T07:28:06.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.838+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b64030070 con 0x7f8b580039b0 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.838+0000 7f8b72a00640 1 -- 172.21.15.86:0/1890242051 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b64005d70 con 0x7f8b580039b0 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.839+0000 7f8b7a000640 1 -- 172.21.15.86:0/1890242051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 msgr2=0x7f8b58003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.839+0000 7f8b7a000640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b58003dd0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f8b6400c210 tx=0x7f8b58005830 comp rx=0 tx=0).stop 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.839+0000 7f8b7a000640 1 -- 172.21.15.86:0/1890242051 shutdown_connections 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.839+0000 7f8b7a000640 1 --2- 172.21.15.86:0/1890242051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b58003dd0 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.839+0000 7f8b7a000640 1 -- 172.21.15.86:0/1890242051 >> 172.21.15.86:0/1890242051 conn(0x7f8b740d1d10 msgr2=0x7f8b740d2120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.839+0000 7f8b7a000640 1 -- 172.21.15.86:0/1890242051 shutdown_connections 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.839+0000 7f8b7a000640 1 -- 172.21.15.86:0/1890242051 wait complete. 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.841+0000 7f8b7a000640 1 Processor -- start 2024-03-06T07:28:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.841+0000 7f8b7a000640 1 -- start start 2024-03-06T07:28:06.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b7a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b740b3250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b7a000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b740c8de0 0x7f8b740ca440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b7a000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b7414ffb0 0x7f8b740cafd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b7a000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b7406d0a0 con 0x7f8b580039b0 2024-03-06T07:28:06.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b7a000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b74161b90 con 0x7f8b7414ffb0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b7a000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8b740ce970 con 0x7f8b740c8de0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b73400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b740b3250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b73400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b740b3250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55520/0 (socket says 172.21.15.86:55520) 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.842+0000 7f8b73400640 1 -- 172.21.15.86:0/942366297 learned_addr learned my addr 172.21.15.86:0/942366297 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b6be00640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b740c8de0 0x7f8b740ca440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b73e00640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b7414ffb0 0x7f8b740cafd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b73400640 1 -- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b740c8de0 msgr2=0x7f8b740ca440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b73400640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b740c8de0 0x7f8b740ca440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b73400640 1 -- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b7414ffb0 msgr2=0x7f8b740cafd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b73400640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b7414ffb0 0x7f8b740cafd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b73400640 1 -- 172.21.15.86:0/942366297 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b741594e0 con 0x7f8b580039b0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b73400640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b740b3250 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f8b6400c330 tx=0x7f8b6402e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b6b400640 1 -- 172.21.15.86:0/942366297 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b64007d30 con 0x7f8b580039b0 2024-03-06T07:28:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b6b400640 1 -- 172.21.15.86:0/942366297 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b64004bf0 con 0x7f8b580039b0 2024-03-06T07:28:06.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b6b400640 1 -- 172.21.15.86:0/942366297 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b64007070 con 0x7f8b580039b0 2024-03-06T07:28:06.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8b740c93a0 con 0x7f8b580039b0 2024-03-06T07:28:06.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.843+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8b740c9960 con 0x7f8b580039b0 2024-03-06T07:28:06.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.845+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8b7404f330 con 0x7f8b580039b0 2024-03-06T07:28:06.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.845+0000 7f8b6b400640 1 -- 172.21.15.86:0/942366297 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8b64048020 con 0x7f8b580039b0 2024-03-06T07:28:06.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.846+0000 7f8b6b400640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b5006b4d0 0x7f8b5006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:06.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.846+0000 7f8b6b400640 1 -- 172.21.15.86:0/942366297 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f8b640ab2e0 con 0x7f8b580039b0 2024-03-06T07:28:06.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.847+0000 7f8b6be00640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b5006b4d0 0x7f8b5006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.847+0000 7f8b6be00640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b5006b4d0 0x7f8b5006d990 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f8b5c000c00 tx=0x7f8b5c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:06.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:06.851+0000 7f8b6b400640 1 -- 172.21.15.86:0/942366297 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8b6407a4a0 con 0x7f8b580039b0 2024-03-06T07:28:07.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.122+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-mds"} v 0) v1 -- 0x7f8b740c8590 con 0x7f8b580039b0 2024-03-06T07:28:07.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.125+0000 7f8b6b400640 1 -- 172.21.15.86:0/942366297 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-mds"}]=0 v7) v1 ==== 90+0+0 (secure 0 0 0) 0x7f8b64079da0 con 0x7f8b580039b0 2024-03-06T07:28:07.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.128+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b5006b4d0 msgr2=0x7f8b5006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.128+0000 7f8b7a000640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b5006b4d0 0x7f8b5006d990 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f8b5c000c00 tx=0x7f8b5c009040 comp rx=0 tx=0).stop 2024-03-06T07:28:07.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.128+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 msgr2=0x7f8b740b3250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.128+0000 7f8b7a000640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b740b3250 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f8b6400c330 tx=0x7f8b6402e070 comp rx=0 tx=0).stop 2024-03-06T07:28:07.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.128+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 shutdown_connections 2024-03-06T07:28:07.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.129+0000 7f8b7a000640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b5006b4d0 0x7f8b5006d990 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.129+0000 7f8b7a000640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b7414ffb0 0x7f8b740cafd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.129+0000 7f8b7a000640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b740c8de0 0x7f8b740ca440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.129+0000 7f8b7a000640 1 --2- 172.21.15.86:0/942366297 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b580039b0 0x7f8b740b3250 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.129+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 >> 172.21.15.86:0/942366297 conn(0x7f8b740d1d10 msgr2=0x7f8b7406f9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:07.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.129+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 shutdown_connections 2024-03-06T07:28:07.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.129+0000 7f8b7a000640 1 -- 172.21.15.86:0/942366297 wait complete. 2024-03-06T07:28:07.150 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-mgr 2024-03-06T07:28:07.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.345+0000 7fdd7be00640 1 Processor -- start 2024-03-06T07:28:07.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.345+0000 7fdd7be00640 1 -- start start 2024-03-06T07:28:07.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.346+0000 7fdd7be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 0x7fdd7c0cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.346+0000 7fdd7be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdd7c0cc800 0x7fdd7c15f830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.346+0000 7fdd7be00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdd7c15c240 con 0x7fdd7c1561f0 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.346+0000 7fdd7be00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdd7c15ce30 con 0x7fdd7c0cc800 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.346+0000 7fdd7be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd7c15c500 con 0x7fdd7c0aedc0 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdd7c0cc800 0x7fdd7c15f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdd7c0cc800 0x7fdd7c15f830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55536/0 (socket says 172.21.15.86:55536) 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7b400640 1 -- 172.21.15.86:0/595680034 learned_addr learned my addr 172.21.15.86:0/595680034 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7aa00640 1 --1- 172.21.15.86:0/595680034 >> v1:172.21.15.86:6789/0 conn(0x7fdd7c1561f0 0x7fdd7c0ae6b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50170/0 (socket says 172.21.15.86:50170) 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7a000640 1 --2- 172.21.15.86:0/595680034 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 0x7fdd7c0cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7b400640 1 -- 172.21.15.86:0/595680034 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 msgr2=0x7fdd7c0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7b400640 1 --2- 172.21.15.86:0/595680034 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 0x7fdd7c0cc2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7b400640 1 -- 172.21.15.86:0/595680034 >> v1:172.21.15.86:6789/0 conn(0x7fdd7c1561f0 legacy=0x7fdd7c0ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.347+0000 7fdd7b400640 1 -- 172.21.15.86:0/595680034 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd7c15c7e0 con 0x7fdd7c0cc800 2024-03-06T07:28:07.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.348+0000 7fdd7b400640 1 --2- 172.21.15.86:0/595680034 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c15f830 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7fdd74009e50 tx=0x7fdd7402bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1047442dac7d5000 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.348+0000 7fdd79600640 1 -- 172.21.15.86:0/595680034 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd7402cb00 con 0x7fdd7c0cc800 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd79600640 1 -- 172.21.15.86:0/595680034 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd7402cca0 con 0x7fdd7c0cc800 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd79600640 1 -- 172.21.15.86:0/595680034 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd74034610 con 0x7fdd7c0cc800 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 -- 172.21.15.86:0/595680034 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 msgr2=0x7fdd7c15f830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 --2- 172.21.15.86:0/595680034 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c15f830 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7fdd74009e50 tx=0x7fdd7402bf60 comp rx=0 tx=0).stop 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 -- 172.21.15.86:0/595680034 shutdown_connections 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 --2- 172.21.15.86:0/595680034 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c15f830 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 --2- 172.21.15.86:0/595680034 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 0x7fdd7c0cc2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 -- 172.21.15.86:0/595680034 >> 172.21.15.86:0/595680034 conn(0x7fdd7c06afc0 msgr2=0x7fdd7c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 -- 172.21.15.86:0/595680034 shutdown_connections 2024-03-06T07:28:07.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.349+0000 7fdd7be00640 1 -- 172.21.15.86:0/595680034 wait complete. 2024-03-06T07:28:07.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.351+0000 7fdd7be00640 1 Processor -- start 2024-03-06T07:28:07.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.351+0000 7fdd7be00640 1 -- start start 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 0x7fdd7c14d5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c1582d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd7c1561f0 0x7fdd7c15afe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd7c1565d0 con 0x7fdd7c0cc800 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd7c0ccc90 con 0x7fdd7c1561f0 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd7c156750 con 0x7fdd7c0aedc0 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c1582d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c1582d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55546/0 (socket says 172.21.15.86:55546) 2024-03-06T07:28:07.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.352+0000 7fdd7a000640 1 -- 172.21.15.86:0/1083422055 learned_addr learned my addr 172.21.15.86:0/1083422055 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:07.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7a000640 1 -- 172.21.15.86:0/1083422055 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 msgr2=0x7fdd7c14d5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7a000640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 0x7fdd7c14d5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7a000640 1 -- 172.21.15.86:0/1083422055 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd7c1561f0 msgr2=0x7fdd7c15afe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7a000640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd7c1561f0 0x7fdd7c15afe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7a000640 1 -- 172.21.15.86:0/1083422055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd7c14daf0 con 0x7fdd7c0cc800 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7a000640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c1582d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7fdd6400d800 tx=0x7fdd6400dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd72a00640 1 -- 172.21.15.86:0/1083422055 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd6400b6c0 con 0x7fdd7c0cc800 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd72a00640 1 -- 172.21.15.86:0/1083422055 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd64019070 con 0x7fdd7c0cc800 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdd7c158dd0 con 0x7fdd7c0cc800 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd72a00640 1 -- 172.21.15.86:0/1083422055 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd6400ba80 con 0x7fdd7c0cc800 2024-03-06T07:28:07.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.353+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdd7c159390 con 0x7fdd7c0cc800 2024-03-06T07:28:07.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.355+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdd7c04f330 con 0x7fdd7c0cc800 2024-03-06T07:28:07.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.355+0000 7fdd72a00640 1 -- 172.21.15.86:0/1083422055 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdd6400bc20 con 0x7fdd7c0cc800 2024-03-06T07:28:07.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.356+0000 7fdd72a00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd5806b4d0 0x7fdd5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.356+0000 7fdd72a00640 1 -- 172.21.15.86:0/1083422055 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fdd6408a380 con 0x7fdd7c0cc800 2024-03-06T07:28:07.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.356+0000 7fdd7aa00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd5806b4d0 0x7fdd5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.357+0000 7fdd7aa00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd5806b4d0 0x7fdd5806d990 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fdd6c000f60 tx=0x7fdd6c007a00 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:07.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.361+0000 7fdd72a00640 1 -- 172.21.15.86:0/1083422055 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdd64059530 con 0x7fdd7c0cc800 2024-03-06T07:28:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.627+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-mgr"} v 0) v1 -- 0x7fdd7c06ce70 con 0x7fdd7c0cc800 2024-03-06T07:28:07.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.632+0000 7fdd72a00640 1 -- 172.21.15.86:0/1083422055 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-mgr"}]=0 v8) v1 ==== 90+0+0 (secure 0 0 0) 0x7fdd64058e30 con 0x7fdd7c0cc800 2024-03-06T07:28:07.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd5806b4d0 msgr2=0x7fdd5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd5806b4d0 0x7fdd5806d990 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fdd6c000f60 tx=0x7fdd6c007a00 comp rx=0 tx=0).stop 2024-03-06T07:28:07.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 msgr2=0x7fdd7c1582d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c1582d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7fdd6400d800 tx=0x7fdd6400dcc0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 shutdown_connections 2024-03-06T07:28:07.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdd5806b4d0 0x7fdd5806d990 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd7c1561f0 0x7fdd7c15afe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd7c0cc800 0x7fdd7c1582d0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 --2- 172.21.15.86:0/1083422055 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd7c0aedc0 0x7fdd7c14d5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 >> 172.21.15.86:0/1083422055 conn(0x7fdd7c06afc0 msgr2=0x7fdd7c1552e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:07.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.634+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 shutdown_connections 2024-03-06T07:28:07.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.635+0000 7fdd7be00640 1 -- 172.21.15.86:0/1083422055 wait complete. 2024-03-06T07:28:07.654 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-osd 2024-03-06T07:28:07.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.848+0000 7f6f60e00640 1 Processor -- start 2024-03-06T07:28:07.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.848+0000 7f6f60e00640 1 -- start start 2024-03-06T07:28:07.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.849+0000 7f6f60e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5c15a940 0x7f6f5c0b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.849+0000 7f6f60e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0cf370 0x7f6f5c06d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.849+0000 7f6f60e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6f5c0b2b30 con 0x7f6f5c0cc2a0 2024-03-06T07:28:07.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.849+0000 7f6f60e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6f5c0b28e0 con 0x7f6f5c15a940 2024-03-06T07:28:07.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.849+0000 7f6f60e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6f5c0b2e10 con 0x7f6f5c0cf370 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5c15a940 0x7f6f5c0b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6f5c0cc2a0 0x7f6f5c0cd460 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50180/0 (socket says 172.21.15.86:50180) 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5be00640 1 -- 172.21.15.86:0/327609410 learned_addr learned my addr 172.21.15.86:0/327609410 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5aa00640 1 --2- 172.21.15.86:0/327609410 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0cf370 0x7f6f5c06d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5b400640 1 -- 172.21.15.86:0/327609410 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0cf370 msgr2=0x7f6f5c06d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5b400640 1 --2- 172.21.15.86:0/327609410 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0cf370 0x7f6f5c06d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5b400640 1 -- 172.21.15.86:0/327609410 >> v1:172.21.15.86:6789/0 conn(0x7f6f5c0cc2a0 legacy=0x7f6f5c0cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5b400640 1 -- 172.21.15.86:0/327609410 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f5c06ca50 con 0x7f6f5c15a940 2024-03-06T07:28:07.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.850+0000 7f6f5a000640 1 -- 172.21.15.86:0/327609410 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 695744665 0 0) 0x7f6f5c0b2b30 con 0x7f6f5c0cc2a0 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.851+0000 7f6f5b400640 1 --2- 172.21.15.86:0/327609410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c15a940 0x7f6f5c0b45e0 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f6f5c068400 tx=0x7f6f4c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1936638a1ea9010d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f5a000640 1 -- 172.21.15.86:0/327609410 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f4c035020 con 0x7f6f5c15a940 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f5a000640 1 -- 172.21.15.86:0/327609410 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6f4c031830 con 0x7f6f5c15a940 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f5a000640 1 -- 172.21.15.86:0/327609410 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f4c031c00 con 0x7f6f5c15a940 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 -- 172.21.15.86:0/327609410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c15a940 msgr2=0x7f6f5c0b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 --2- 172.21.15.86:0/327609410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c15a940 0x7f6f5c0b45e0 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f6f5c068400 tx=0x7f6f4c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 -- 172.21.15.86:0/327609410 shutdown_connections 2024-03-06T07:28:07.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 --2- 172.21.15.86:0/327609410 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0cf370 0x7f6f5c06d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 --2- 172.21.15.86:0/327609410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c15a940 0x7f6f5c0b45e0 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 -- 172.21.15.86:0/327609410 >> 172.21.15.86:0/327609410 conn(0x7f6f5c06afc0 msgr2=0x7f6f5c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:07.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 -- 172.21.15.86:0/327609410 shutdown_connections 2024-03-06T07:28:07.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.852+0000 7f6f60e00640 1 -- 172.21.15.86:0/327609410 wait complete. 2024-03-06T07:28:07.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.854+0000 7f6f60e00640 1 Processor -- start 2024-03-06T07:28:07.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.854+0000 7f6f60e00640 1 -- start start 2024-03-06T07:28:07.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f60e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c0cc2a0 0x7f6f5c15d740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f60e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0cf370 0x7f6f5c0cdc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f60e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c15a940 0x7f6f5c0ce1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f60e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f5c0b4b20 con 0x7f6f5c0cf370 2024-03-06T07:28:07.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f60e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f5c0b30f0 con 0x7f6f5c15a940 2024-03-06T07:28:07.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f60e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6f5c06d790 con 0x7f6f5c0cc2a0 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f5aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0cf370 0x7f6f5c0cdc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f5aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0cf370 0x7f6f5c0cdc90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55572/0 (socket says 172.21.15.86:55572) 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.855+0000 7f6f5aa00640 1 -- 172.21.15.86:0/2984909073 learned_addr learned my addr 172.21.15.86:0/2984909073 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f5aa00640 1 -- 172.21.15.86:0/2984909073 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c0cc2a0 msgr2=0x7f6f5c15d740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f5aa00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c0cc2a0 0x7f6f5c15d740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f5aa00640 1 -- 172.21.15.86:0/2984909073 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c15a940 msgr2=0x7f6f5c0ce1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f5aa00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c15a940 0x7f6f5c0ce1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f5aa00640 1 -- 172.21.15.86:0/2984909073 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f5c15dc80 con 0x7f6f5c0cf370 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f5aa00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0cf370 0x7f6f5c0cdc90 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f6f4800d800 tx=0x7f6f4800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:07.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f53400640 1 -- 172.21.15.86:0/2984909073 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f4800b6c0 con 0x7f6f5c0cf370 2024-03-06T07:28:07.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f53400640 1 -- 172.21.15.86:0/2984909073 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6f48019070 con 0x7f6f5c0cf370 2024-03-06T07:28:07.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f53400640 1 -- 172.21.15.86:0/2984909073 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f4800ba80 con 0x7f6f5c0cf370 2024-03-06T07:28:07.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6f5c150650 con 0x7f6f5c0cf370 2024-03-06T07:28:07.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.856+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6f5c150c10 con 0x7f6f5c0cf370 2024-03-06T07:28:07.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.857+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6f5c04f330 con 0x7f6f5c0cf370 2024-03-06T07:28:07.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.858+0000 7f6f53400640 1 -- 172.21.15.86:0/2984909073 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6f4800bc20 con 0x7f6f5c0cf370 2024-03-06T07:28:07.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.859+0000 7f6f53400640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:07.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.859+0000 7f6f53400640 1 -- 172.21.15.86:0/2984909073 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f6f4808a380 con 0x7f6f5c0cf370 2024-03-06T07:28:07.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.860+0000 7f6f5b400640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:07.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.860+0000 7f6f5b400640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f6f4c036000 tx=0x7f6f4c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:07.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:07.864+0000 7f6f53400640 1 -- 172.21.15.86:0/2984909073 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6f48059530 con 0x7f6f5c0cf370 2024-03-06T07:28:08.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.127+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-osd"} v 0) v1 -- 0x7f6f5c0cd740 con 0x7f6f5c0cf370 2024-03-06T07:28:08.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.130+0000 7f6f53400640 1 -- 172.21.15.86:0/2984909073 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-osd"}]=0 v9) v1 ==== 90+0+0 (secure 0 0 0) 0x7f6f48058e30 con 0x7f6f5c0cf370 2024-03-06T07:28:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 msgr2=0x7f6f3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f6f4c036000 tx=0x7f6f4c030000 comp rx=0 tx=0).stop 2024-03-06T07:28:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0cf370 msgr2=0x7f6f5c0cdc90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0cf370 0x7f6f5c0cdc90 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f6f4800d800 tx=0x7f6f4800dcc0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 shutdown_connections 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c15a940 0x7f6f5c0ce1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0cf370 0x7f6f5c0cdc90 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 --2- 172.21.15.86:0/2984909073 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c0cc2a0 0x7f6f5c15d740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 >> 172.21.15.86:0/2984909073 conn(0x7f6f5c06afc0 msgr2=0x7f6f5c0cf890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 shutdown_connections 2024-03-06T07:28:08.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.133+0000 7f6f60e00640 1 -- 172.21.15.86:0/2984909073 wait complete. 2024-03-06T07:28:08.154 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-rbd 2024-03-06T07:28:08.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.355+0000 7f3048200640 1 Processor -- start 2024-03-06T07:28:08.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.355+0000 7f3048200640 1 -- start start 2024-03-06T07:28:08.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3048200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f30400c4930 0x7f30400c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3048200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f30400c5880 0x7f3040165b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3048200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3040145b30 con 0x7f30400c4d10 2024-03-06T07:28:08.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3048200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3040142ce0 con 0x7f30400c5880 2024-03-06T07:28:08.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3048200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f30400cbca0 con 0x7f30400c4930 2024-03-06T07:28:08.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3046200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f30400c5880 0x7f3040165b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3046200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f30400c5880 0x7f3040165b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55586/0 (socket says 172.21.15.86:55586) 2024-03-06T07:28:08.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3046200640 1 -- 172.21.15.86:0/1781443501 learned_addr learned my addr 172.21.15.86:0/1781443501 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:08.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.356+0000 7f3045800640 1 --2- 172.21.15.86:0/1781443501 >> v2:172.21.15.194:3300/0 conn(0x7f30400c4930 0x7f30400c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.357+0000 7f3046200640 1 -- 172.21.15.86:0/1781443501 >> v2:172.21.15.194:3300/0 conn(0x7f30400c4930 msgr2=0x7f30400c6460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:08.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.357+0000 7f3046200640 1 --2- 172.21.15.86:0/1781443501 >> v2:172.21.15.194:3300/0 conn(0x7f30400c4930 0x7f30400c6460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.357+0000 7f3046200640 1 -- 172.21.15.86:0/1781443501 >> v1:172.21.15.194:6789/0 conn(0x7f30400c4d10 legacy=0x7f30400c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.357+0000 7f3046200640 1 -- 172.21.15.86:0/1781443501 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30400cbf80 con 0x7f30400c5880 2024-03-06T07:28:08.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.358+0000 7f3046200640 1 --2- 172.21.15.86:0/1781443501 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c5880 0x7f3040165b30 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f30400cf950 tx=0x7f303802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3baa9cd98d9563df server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:08.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.358+0000 7f303fe00640 1 -- 172.21.15.86:0/1781443501 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f303802ca80 con 0x7f30400c5880 2024-03-06T07:28:08.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.358+0000 7f303fe00640 1 -- 172.21.15.86:0/1781443501 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f303802cc20 con 0x7f30400c5880 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.358+0000 7f303fe00640 1 -- 172.21.15.86:0/1781443501 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30380343f0 con 0x7f30400c5880 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 -- 172.21.15.86:0/1781443501 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c5880 msgr2=0x7f3040165b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 --2- 172.21.15.86:0/1781443501 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c5880 0x7f3040165b30 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f30400cf950 tx=0x7f303802bee0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 -- 172.21.15.86:0/1781443501 shutdown_connections 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 --2- 172.21.15.86:0/1781443501 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c5880 0x7f3040165b30 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 --2- 172.21.15.86:0/1781443501 >> v2:172.21.15.194:3300/0 conn(0x7f30400c4930 0x7f30400c6460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 -- 172.21.15.86:0/1781443501 >> 172.21.15.86:0/1781443501 conn(0x7f30400cc4f0 msgr2=0x7f30400cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 -- 172.21.15.86:0/1781443501 shutdown_connections 2024-03-06T07:28:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.359+0000 7f3048200640 1 -- 172.21.15.86:0/1781443501 wait complete. 2024-03-06T07:28:08.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.361+0000 7f3048200640 1 Processor -- start 2024-03-06T07:28:08.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.361+0000 7f3048200640 1 -- start start 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3048200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f30400c4930 0x7f30401e0b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3048200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c4d10 0x7f3040071b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3048200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f30400c5880 0x7f3040072060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3048200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f304006e600 con 0x7f30400c4d10 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3048200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30400ca3b0 con 0x7f30400c5880 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3048200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f30400cc260 con 0x7f30400c4930 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c4d10 0x7f3040071b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c4d10 0x7f3040071b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55602/0 (socket says 172.21.15.86:55602) 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 -- 172.21.15.86:0/2124505095 learned_addr learned my addr 172.21.15.86:0/2124505095 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 -- 172.21.15.86:0/2124505095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f30400c4930 msgr2=0x7f30401e0b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f30400c4930 0x7f30401e0b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 -- 172.21.15.86:0/2124505095 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f30400c5880 msgr2=0x7f3040072060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f30400c5880 0x7f3040072060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 -- 172.21.15.86:0/2124505095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30401e1690 con 0x7f30400c4d10 2024-03-06T07:28:08.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.362+0000 7f3044e00640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c4d10 0x7f3040071b20 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f302c00c400 tx=0x7f302c00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:08.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.363+0000 7f303d600640 1 -- 172.21.15.86:0/2124505095 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f302c0052d0 con 0x7f30400c4d10 2024-03-06T07:28:08.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.363+0000 7f303d600640 1 -- 172.21.15.86:0/2124505095 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f302c005490 con 0x7f30400c4d10 2024-03-06T07:28:08.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.363+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f30401e4410 con 0x7f30400c4d10 2024-03-06T07:28:08.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.363+0000 7f303d600640 1 -- 172.21.15.86:0/2124505095 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f302c007260 con 0x7f30400c4d10 2024-03-06T07:28:08.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.363+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f30401e4cb0 con 0x7f30400c4d10 2024-03-06T07:28:08.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.364+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f30401e4800 con 0x7f30400c4d10 2024-03-06T07:28:08.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.365+0000 7f303d600640 1 -- 172.21.15.86:0/2124505095 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f302c007400 con 0x7f30400c4d10 2024-03-06T07:28:08.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.366+0000 7f303d600640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f302006b4d0 0x7f302006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.366+0000 7f303d600640 1 -- 172.21.15.86:0/2124505095 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f302c089410 con 0x7f30400c4d10 2024-03-06T07:28:08.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.366+0000 7f3045800640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f302006b4d0 0x7f302006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.367+0000 7f3045800640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f302006b4d0 0x7f302006d990 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f3034006fd0 tx=0x7f3034008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:08.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.371+0000 7f303d600640 1 -- 172.21.15.86:0/2124505095 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f30401e1690 con 0x7f30400c4d10 2024-03-06T07:28:08.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.642+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-rbd"} v 0) v1 -- 0x7f3040143f30 con 0x7f30400c4d10 2024-03-06T07:28:08.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.646+0000 7f303d600640 1 -- 172.21.15.86:0/2124505095 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-rbd"}]=0 v10) v1 ==== 90+0+0 (secure 0 0 0) 0x7f3040145b30 con 0x7f30400c4d10 2024-03-06T07:28:08.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f302006b4d0 msgr2=0x7f302006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f302006b4d0 0x7f302006d990 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f3034006fd0 tx=0x7f3034008040 comp rx=0 tx=0).stop 2024-03-06T07:28:08.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c4d10 msgr2=0x7f3040071b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c4d10 0x7f3040071b20 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f302c00c400 tx=0x7f302c00c8c0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 shutdown_connections 2024-03-06T07:28:08.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f302006b4d0 0x7f302006d990 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f30400c5880 0x7f3040072060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f30400c4d10 0x7f3040071b20 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 --2- 172.21.15.86:0/2124505095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f30400c4930 0x7f30401e0b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.648+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 >> 172.21.15.86:0/2124505095 conn(0x7f30400cc4f0 msgr2=0x7f30400c5150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:08.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.649+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 shutdown_connections 2024-03-06T07:28:08.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.649+0000 7f3048200640 1 -- 172.21.15.86:0/2124505095 wait complete. 2024-03-06T07:28:08.669 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-rbd-mirror 2024-03-06T07:28:08.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.870+0000 7fc721600640 1 Processor -- start 2024-03-06T07:28:08.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.870+0000 7fc721600640 1 -- start start 2024-03-06T07:28:08.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.872+0000 7fc721600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc71c15a940 0x7fc71c0b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.872+0000 7fc721600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc71c0b2b30 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.872+0000 7fc721600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc71c0b2e10 con 0x7fc71c0cf370 2024-03-06T07:28:08.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.872+0000 7fc721600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc71c0b28e0 con 0x7fc71c15a940 2024-03-06T07:28:08.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.872+0000 7fc71be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc71c15a940 0x7fc71c0b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.872+0000 7fc720c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc71c0cc2a0 0x7fc71c0cd460 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50186/0 (socket says 172.21.15.86:50186) 2024-03-06T07:28:08.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.872+0000 7fc720c00640 1 -- 172.21.15.86:0/2356360442 learned_addr learned my addr 172.21.15.86:0/2356360442 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:08.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.873+0000 7fc71b400640 1 -- 172.21.15.86:0/2356360442 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1976072929 0 0) 0x7fc71c0b2b30 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.873+0000 7fc71b400640 1 -- 172.21.15.86:0/2356360442 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc700003520 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.873+0000 7fc71be00640 1 -- 172.21.15.86:0/2356360442 >> v1:172.21.15.194:6789/0 conn(0x7fc71c0cf370 legacy=0x7fc71c06d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:08.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.873+0000 7fc71be00640 1 -- 172.21.15.86:0/2356360442 >> v1:172.21.15.86:6789/0 conn(0x7fc71c0cc2a0 legacy=0x7fc71c0cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.873+0000 7fc71be00640 1 -- 172.21.15.86:0/2356360442 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc71c06ca50 con 0x7fc71c15a940 2024-03-06T07:28:08.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.874+0000 7fc71be00640 1 --2- 172.21.15.86:0/2356360442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c15a940 0x7fc71c0b45e0 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fc70c0097a0 tx=0x7fc70c02b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=334f8fed0ba69970 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:08.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.874+0000 7fc71b400640 1 -- 172.21.15.86:0/2356360442 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc70c034070 con 0x7fc71c15a940 2024-03-06T07:28:08.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.874+0000 7fc71b400640 1 -- 172.21.15.86:0/2356360442 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc70c030610 con 0x7fc71c15a940 2024-03-06T07:28:08.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.874+0000 7fc71b400640 1 -- 172.21.15.86:0/2356360442 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc70c0309e0 con 0x7fc71c15a940 2024-03-06T07:28:08.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.875+0000 7fc721600640 1 -- 172.21.15.86:0/2356360442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c15a940 msgr2=0x7fc71c0b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.875+0000 7fc721600640 1 --2- 172.21.15.86:0/2356360442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c15a940 0x7fc71c0b45e0 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fc70c0097a0 tx=0x7fc70c02b960 comp rx=0 tx=0).stop 2024-03-06T07:28:08.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.875+0000 7fc721600640 1 -- 172.21.15.86:0/2356360442 shutdown_connections 2024-03-06T07:28:08.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.875+0000 7fc721600640 1 --2- 172.21.15.86:0/2356360442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c15a940 0x7fc71c0b45e0 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.875+0000 7fc721600640 1 -- 172.21.15.86:0/2356360442 >> 172.21.15.86:0/2356360442 conn(0x7fc71c06afc0 msgr2=0x7fc71c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:08.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.875+0000 7fc721600640 1 -- 172.21.15.86:0/2356360442 shutdown_connections 2024-03-06T07:28:08.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.875+0000 7fc721600640 1 -- 172.21.15.86:0/2356360442 wait complete. 2024-03-06T07:28:08.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.877+0000 7fc721600640 1 Processor -- start 2024-03-06T07:28:08.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.877+0000 7fc721600640 1 -- start start 2024-03-06T07:28:08.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc721600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c0cc2a0 0x7fc71c1435f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc721600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc71c0cf370 0x7fc71c071ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc721600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc71c15a940 0x7fc71c072410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc721600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc71c0b4b20 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc721600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc71c0b30f0 con 0x7fc71c0cf370 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc721600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc71c15ad20 con 0x7fc71c15a940 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc71be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c0cc2a0 0x7fc71c1435f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc71be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c0cc2a0 0x7fc71c1435f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55614/0 (socket says 172.21.15.86:55614) 2024-03-06T07:28:08.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.878+0000 7fc71be00640 1 -- 172.21.15.86:0/3545411908 learned_addr learned my addr 172.21.15.86:0/3545411908 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:08.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc71be00640 1 -- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc71c15a940 msgr2=0x7fc71c072410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc71be00640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc71c15a940 0x7fc71c072410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc71be00640 1 -- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc71c0cf370 msgr2=0x7fc71c071ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:08.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc713400640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc71c0cf370 0x7fc71c071ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc71be00640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc71c0cf370 0x7fc71c071ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc71be00640 1 -- 172.21.15.86:0/3545411908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc71c143b30 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc713400640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc71c0cf370 0x7fc71c071ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc71be00640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c0cc2a0 0x7fc71c1435f0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fc70c02bf70 tx=0x7fc70c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc718c00640 1 -- 172.21.15.86:0/3545411908 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc70c034040 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc718c00640 1 -- 172.21.15.86:0/3545411908 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc70c03aaa0 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc718c00640 1 -- 172.21.15.86:0/3545411908 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc70c0443f0 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc71c1462e0 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.879+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc71c1468a0 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.881+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc71c04f330 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.881+0000 7fc718c00640 1 -- 172.21.15.86:0/3545411908 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc70c04b060 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.882+0000 7fc718c00640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6f806b500 0x7fc6f806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:08.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.882+0000 7fc718c00640 1 -- 172.21.15.86:0/3545411908 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fc70c0abb80 con 0x7fc71c0cc2a0 2024-03-06T07:28:08.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.882+0000 7fc713400640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6f806b500 0x7fc6f806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:08.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.883+0000 7fc713400640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6f806b500 0x7fc6f806d9c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fc71c15d6a0 tx=0x7fc704009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:08.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:08.886+0000 7fc718c00640 1 -- 172.21.15.86:0/3545411908 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc70c03f070 con 0x7fc71c0cc2a0 2024-03-06T07:28:09.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.153+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-rbd-mirror"} v 0) v1 -- 0x7fc71c074a50 con 0x7fc71c0cc2a0 2024-03-06T07:28:09.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.158+0000 7fc718c00640 1 -- 172.21.15.86:0/3545411908 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-rbd-mirror"}]=0 v11) v1 ==== 97+0+0 (secure 0 0 0) 0x7fc70c04b350 con 0x7fc71c0cc2a0 2024-03-06T07:28:09.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.160+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6f806b500 msgr2=0x7fc6f806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.160+0000 7fc721600640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6f806b500 0x7fc6f806d9c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fc71c15d6a0 tx=0x7fc704009040 comp rx=0 tx=0).stop 2024-03-06T07:28:09.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.160+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c0cc2a0 msgr2=0x7fc71c1435f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.160+0000 7fc721600640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c0cc2a0 0x7fc71c1435f0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fc70c02bf70 tx=0x7fc70c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 shutdown_connections 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6f806b500 0x7fc6f806d9c0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc71c15a940 0x7fc71c072410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc71c0cf370 0x7fc71c071ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 --2- 172.21.15.86:0/3545411908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc71c0cc2a0 0x7fc71c1435f0 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 >> 172.21.15.86:0/3545411908 conn(0x7fc71c06afc0 msgr2=0x7fc71c0cbad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 shutdown_connections 2024-03-06T07:28:09.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.161+0000 7fc721600640 1 -- 172.21.15.86:0/3545411908 wait complete. 2024-03-06T07:28:09.181 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-rgw 2024-03-06T07:28:09.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.380+0000 7f5440800640 1 Processor -- start 2024-03-06T07:28:09.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.380+0000 7f5440800640 1 -- start start 2024-03-06T07:28:09.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f5440800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543813ef20 0x7f5438160400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f5440800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f543815c350 0x7f543815c7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f5440800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f543815f940 con 0x7f543806afc0 2024-03-06T07:28:09.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f5440800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f543806e9c0 con 0x7f543815c350 2024-03-06T07:28:09.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f5440800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f543815ce80 con 0x7f543813ef20 2024-03-06T07:28:09.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f543d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f543815c350 0x7f543815c7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f543d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f543815c350 0x7f543815c7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55618/0 (socket says 172.21.15.86:55618) 2024-03-06T07:28:09.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f543d400640 1 -- 172.21.15.86:0/2976492237 learned_addr learned my addr 172.21.15.86:0/2976492237 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:09.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.382+0000 7f543de00640 1 --2- 172.21.15.86:0/2976492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543813ef20 0x7f5438160400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.383+0000 7f543d400640 1 -- 172.21.15.86:0/2976492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543813ef20 msgr2=0x7f5438160400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:09.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.383+0000 7f543d400640 1 --2- 172.21.15.86:0/2976492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543813ef20 0x7f5438160400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.383+0000 7f543d400640 1 -- 172.21.15.86:0/2976492237 >> v1:172.21.15.194:6789/0 conn(0x7f543806afc0 legacy=0x7f54381414b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:09.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.383+0000 7f543d400640 1 -- 172.21.15.86:0/2976492237 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f543815d160 con 0x7f543815c350 2024-03-06T07:28:09.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.383+0000 7f543de00640 1 --2- 172.21.15.86:0/2976492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543813ef20 0x7f5438160400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:09.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.384+0000 7f543d400640 1 --2- 172.21.15.86:0/2976492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543815c350 0x7f543815c7b0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f5428009870 tx=0x7f542802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e060589da935ee51 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:09.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.384+0000 7f543ca00640 1 -- 172.21.15.86:0/2976492237 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5428035020 con 0x7f543815c350 2024-03-06T07:28:09.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.384+0000 7f543ca00640 1 -- 172.21.15.86:0/2976492237 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5428031640 con 0x7f543815c350 2024-03-06T07:28:09.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.384+0000 7f543ca00640 1 -- 172.21.15.86:0/2976492237 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f542802d040 con 0x7f543815c350 2024-03-06T07:28:09.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 -- 172.21.15.86:0/2976492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543815c350 msgr2=0x7f543815c7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 --2- 172.21.15.86:0/2976492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543815c350 0x7f543815c7b0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f5428009870 tx=0x7f542802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 -- 172.21.15.86:0/2976492237 shutdown_connections 2024-03-06T07:28:09.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 --2- 172.21.15.86:0/2976492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543815c350 0x7f543815c7b0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 --2- 172.21.15.86:0/2976492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543813ef20 0x7f5438160400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 -- 172.21.15.86:0/2976492237 >> 172.21.15.86:0/2976492237 conn(0x7f54380d0b90 msgr2=0x7f54380d0f70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:09.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 -- 172.21.15.86:0/2976492237 shutdown_connections 2024-03-06T07:28:09.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.385+0000 7f5440800640 1 -- 172.21.15.86:0/2976492237 wait complete. 2024-03-06T07:28:09.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.387+0000 7f5440800640 1 Processor -- start 2024-03-06T07:28:09.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.387+0000 7f5440800640 1 -- start start 2024-03-06T07:28:09.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f5440800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f543806afc0 0x7f543813e690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f5440800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543813ef20 0x7f543813f320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f5440800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543815c350 0x7f543813f860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f5440800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54380700e0 con 0x7f543813ef20 2024-03-06T07:28:09.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f5440800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f543815fee0 con 0x7f543806afc0 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f5440800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f543806b450 con 0x7f543815c350 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f543d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543813ef20 0x7f543813f320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f543d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543813ef20 0x7f543813f320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55632/0 (socket says 172.21.15.86:55632) 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.388+0000 7f543d400640 1 -- 172.21.15.86:0/1464876637 learned_addr learned my addr 172.21.15.86:0/1464876637 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543de00640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f543806afc0 0x7f543813e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543e800640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543815c350 0x7f543813f860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543d400640 1 -- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543815c350 msgr2=0x7f543813f860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543d400640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543815c350 0x7f543813f860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543d400640 1 -- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f543806afc0 msgr2=0x7f543813e690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543d400640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f543806afc0 0x7f543813e690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543d400640 1 -- 172.21.15.86:0/1464876637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54380b5200 con 0x7f543813ef20 2024-03-06T07:28:09.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f543d400640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543813ef20 0x7f543813f320 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f5428000c00 tx=0x7f54280046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:09.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f5432000640 1 -- 172.21.15.86:0/1464876637 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5428040070 con 0x7f543813ef20 2024-03-06T07:28:09.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f5432000640 1 -- 172.21.15.86:0/1464876637 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5428039c10 con 0x7f543813ef20 2024-03-06T07:28:09.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f5432000640 1 -- 172.21.15.86:0/1464876637 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5428035040 con 0x7f543813ef20 2024-03-06T07:28:09.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5438140630 con 0x7f543813ef20 2024-03-06T07:28:09.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.389+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54380b4720 con 0x7f543813ef20 2024-03-06T07:28:09.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.390+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f543806eb50 con 0x7f543813ef20 2024-03-06T07:28:09.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.391+0000 7f5432000640 1 -- 172.21.15.86:0/1464876637 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f542804b060 con 0x7f543813ef20 2024-03-06T07:28:09.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.393+0000 7f5432000640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f541c06b500 0x7f541c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.393+0000 7f5432000640 1 -- 172.21.15.86:0/1464876637 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f54280ab980 con 0x7f543813ef20 2024-03-06T07:28:09.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.393+0000 7f543de00640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f541c06b500 0x7f541c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.393+0000 7f543de00640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f541c06b500 0x7f541c06d9c0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f54380cbd40 tx=0x7f542c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:09.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.397+0000 7f5432000640 1 -- 172.21.15.86:0/1464876637 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f543806eb50 con 0x7f543813ef20 2024-03-06T07:28:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.666+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-rgw"} v 0) v1 -- 0x7f5438067830 con 0x7f543813ef20 2024-03-06T07:28:09.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.671+0000 7f5432000640 1 -- 172.21.15.86:0/1464876637 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-rgw"}]=0 v12) v1 ==== 90+0+0 (secure 0 0 0) 0x7f5438067830 con 0x7f543813ef20 2024-03-06T07:28:09.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.673+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f541c06b500 msgr2=0x7f541c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.673+0000 7f5440800640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f541c06b500 0x7f541c06d9c0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f54380cbd40 tx=0x7f542c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:09.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.673+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543813ef20 msgr2=0x7f543813f320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.673+0000 7f5440800640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543813ef20 0x7f543813f320 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f5428000c00 tx=0x7f54280046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 shutdown_connections 2024-03-06T07:28:09.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f541c06b500 0x7f541c06d9c0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f543815c350 0x7f543813f860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f543813ef20 0x7f543813f320 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 --2- 172.21.15.86:0/1464876637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f543806afc0 0x7f543813e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 >> 172.21.15.86:0/1464876637 conn(0x7f54380d0b90 msgr2=0x7f54380b1c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:09.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 shutdown_connections 2024-03-06T07:28:09.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.674+0000 7f5440800640 1 -- 172.21.15.86:0/1464876637 wait complete. 2024-03-06T07:28:09.693 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:09.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.897+0000 7f7b9bc00640 1 Processor -- start 2024-03-06T07:28:09.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.897+0000 7f7b9bc00640 1 -- start start 2024-03-06T07:28:09.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.899+0000 7f7b9bc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7b94144660 0x7f7b940b3b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.899+0000 7f7b9bc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b94071c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.899+0000 7f7b9bc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7b940cfe30 con 0x7f7b940cf5b0 2024-03-06T07:28:09.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.899+0000 7f7b9bc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7b940cfb50 con 0x7f7b94144660 2024-03-06T07:28:09.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.899+0000 7f7b9bc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7b9406ed90 con 0x7f7b940cf170 2024-03-06T07:28:09.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b99200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7b94144660 0x7f7b940b3b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b8be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b94071c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b8be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b94071c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:42820/0 (socket says 172.21.15.86:42820) 2024-03-06T07:28:09.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b8be00640 1 -- 172.21.15.86:0/237162746 learned_addr learned my addr 172.21.15.86:0/237162746 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:09.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b8be00640 1 -- 172.21.15.86:0/237162746 >> v2:172.21.15.194:3300/0 conn(0x7f7b94144660 msgr2=0x7f7b940b3b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:09.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b8be00640 1 --2- 172.21.15.86:0/237162746 >> v2:172.21.15.194:3300/0 conn(0x7f7b94144660 0x7f7b940b3b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b8be00640 1 -- 172.21.15.86:0/237162746 >> v1:172.21.15.194:6789/0 conn(0x7f7b940cf5b0 legacy=0x7f7b9406f870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.900+0000 7f7b8be00640 1 -- 172.21.15.86:0/237162746 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b9406f070 con 0x7f7b940cf170 2024-03-06T07:28:09.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.902+0000 7f7b8be00640 1 --2- 172.21.15.86:0/237162746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b94071c00 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f7b7c009870 tx=0x7f7b7c02b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a8211b118a6e8135 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:09.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b93e00640 1 -- 172.21.15.86:0/237162746 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b7c035020 con 0x7f7b940cf170 2024-03-06T07:28:09.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b93e00640 1 -- 172.21.15.86:0/237162746 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7b7c031640 con 0x7f7b940cf170 2024-03-06T07:28:09.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b93e00640 1 -- 172.21.15.86:0/237162746 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b7c02d040 con 0x7f7b940cf170 2024-03-06T07:28:09.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b9bc00640 1 -- 172.21.15.86:0/237162746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 msgr2=0x7f7b94071c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/237162746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b94071c00 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f7b7c009870 tx=0x7f7b7c02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b9bc00640 1 -- 172.21.15.86:0/237162746 shutdown_connections 2024-03-06T07:28:09.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/237162746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b94071c00 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/237162746 >> v2:172.21.15.194:3300/0 conn(0x7f7b94144660 0x7f7b940b3b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b9bc00640 1 -- 172.21.15.86:0/237162746 >> 172.21.15.86:0/237162746 conn(0x7f7b940d0b90 msgr2=0x7f7b940d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:09.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.903+0000 7f7b9bc00640 1 -- 172.21.15.86:0/237162746 shutdown_connections 2024-03-06T07:28:09.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.904+0000 7f7b9bc00640 1 -- 172.21.15.86:0/237162746 wait complete. 2024-03-06T07:28:09.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.905+0000 7f7b9bc00640 1 Processor -- start 2024-03-06T07:28:09.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.905+0000 7f7b9bc00640 1 -- start start 2024-03-06T07:28:09.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b9bc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b941672f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b9bc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7b940cf5b0 0x7f7b9415bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b9bc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7b94144660 0x7f7b940b0b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b9bc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b940cc6d0 con 0x7f7b940cf5b0 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b9bc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b940cc550 con 0x7f7b94144660 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b9bc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7b94072140 con 0x7f7b940cf170 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b8be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7b940cf5b0 0x7f7b9415bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b8be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7b940cf5b0 0x7f7b9415bf20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55646/0 (socket says 172.21.15.86:55646) 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.906+0000 7f7b8be00640 1 -- 172.21.15.86:0/2751183490 learned_addr learned my addr 172.21.15.86:0/2751183490 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b99c00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7b94144660 0x7f7b940b0b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b99200640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b941672f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b8be00640 1 -- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 msgr2=0x7f7b941672f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b8be00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b941672f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b8be00640 1 -- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7b94144660 msgr2=0x7f7b940b0b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:09.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b8be00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7b94144660 0x7f7b940b0b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:09.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b8be00640 1 -- 172.21.15.86:0/2751183490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b94167830 con 0x7f7b940cf5b0 2024-03-06T07:28:09.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b8be00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7b940cf5b0 0x7f7b9415bf20 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f7b7c000c00 tx=0x7f7b7c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:09.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.907+0000 7f7b91600640 1 -- 172.21.15.86:0/2751183490 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b7c040070 con 0x7f7b940cf5b0 2024-03-06T07:28:09.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.908+0000 7f7b91600640 1 -- 172.21.15.86:0/2751183490 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7b7c0399f0 con 0x7f7b940cf5b0 2024-03-06T07:28:09.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.908+0000 7f7b91600640 1 -- 172.21.15.86:0/2751183490 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b7c035040 con 0x7f7b940cf5b0 2024-03-06T07:28:09.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.908+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7b94168100 con 0x7f7b940cf5b0 2024-03-06T07:28:09.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.908+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7b941686c0 con 0x7f7b940cf5b0 2024-03-06T07:28:09.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.909+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7b9404f330 con 0x7f7b940cf5b0 2024-03-06T07:28:09.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.910+0000 7f7b91600640 1 -- 172.21.15.86:0/2751183490 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7b7c04b060 con 0x7f7b940cf5b0 2024-03-06T07:28:09.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.911+0000 7f7b91600640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b7006b4d0 0x7f7b7006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:09.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.911+0000 7f7b91600640 1 -- 172.21.15.86:0/2751183490 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f7b7c0ab750 con 0x7f7b940cf5b0 2024-03-06T07:28:09.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.911+0000 7f7b99200640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b7006b4d0 0x7f7b7006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:09.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.912+0000 7f7b99200640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b7006b4d0 0x7f7b7006d990 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f7b94166ed0 tx=0x7f7b8400b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:09.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:09.915+0000 7f7b91600640 1 -- 172.21.15.86:0/2751183490 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7b7c07a920 con 0x7f7b940cf5b0 2024-03-06T07:28:10.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.100+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7b940b2000 con 0x7f7b940cf5b0 2024-03-06T07:28:10.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.102+0000 7f7b91600640 1 -- 172.21.15.86:0/2751183490 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v31) v1 ==== 74+0+12903 (secure 0 0 0) 0x7f7b7c07a220 con 0x7f7b940cf5b0 2024-03-06T07:28:10.104 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:10.104 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":31,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:06.124847+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":23,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:10.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b7006b4d0 msgr2=0x7f7b7006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b7006b4d0 0x7f7b7006d990 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f7b94166ed0 tx=0x7f7b8400b040 comp rx=0 tx=0).stop 2024-03-06T07:28:10.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7b940cf5b0 msgr2=0x7f7b9415bf20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7b940cf5b0 0x7f7b9415bf20 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f7b7c000c00 tx=0x7f7b7c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 shutdown_connections 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7b7006b4d0 0x7f7b7006d990 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7b94144660 0x7f7b940b0b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7b940cf5b0 0x7f7b9415bf20 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 --2- 172.21.15.86:0/2751183490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7b940cf170 0x7f7b941672f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.104+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 >> 172.21.15.86:0/2751183490 conn(0x7f7b940d0b90 msgr2=0x7f7b9406afc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.105+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 shutdown_connections 2024-03-06T07:28:10.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.105+0000 7f7b9bc00640 1 -- 172.21.15.86:0/2751183490 wait complete. 2024-03-06T07:28:10.125 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:28:10.125 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:28:10.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.328+0000 7f8bfaa00640 1 Processor -- start 2024-03-06T07:28:10.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.328+0000 7f8bfaa00640 1 -- start start 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.329+0000 7f8bfaa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8bf40cff60 0x7f8bf406d4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.329+0000 7f8bfaa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf406da20 0x7f8bf40ce3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.329+0000 7f8bfaa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8bf40b3560 con 0x7f8bf4159570 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.329+0000 7f8bfaa00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8bf40cc480 con 0x7f8bf40cff60 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.329+0000 7f8bfaa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8bf40cc760 con 0x7f8bf406da20 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bf8c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8bf40cff60 0x7f8bf406d4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bf8c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8bf40cff60 0x7f8bf406d4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55652/0 (socket says 172.21.15.86:55652) 2024-03-06T07:28:10.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bf8c00640 1 -- 172.21.15.86:0/3744567797 learned_addr learned my addr 172.21.15.86:0/3744567797 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:10.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bfa000640 1 --2- 172.21.15.86:0/3744567797 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf406da20 0x7f8bf40ce3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bf8c00640 1 -- 172.21.15.86:0/3744567797 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf406da20 msgr2=0x7f8bf40ce3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:10.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bf8c00640 1 --2- 172.21.15.86:0/3744567797 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf406da20 0x7f8bf40ce3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bf8c00640 1 -- 172.21.15.86:0/3744567797 >> v1:172.21.15.194:6789/0 conn(0x7f8bf4159570 legacy=0x7f8bf40cf850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:10.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.330+0000 7f8bf8c00640 1 -- 172.21.15.86:0/3744567797 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8bf40cca40 con 0x7f8bf40cff60 2024-03-06T07:28:10.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.331+0000 7f8bf8c00640 1 --2- 172.21.15.86:0/3744567797 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf40cff60 0x7f8bf406d4e0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f8be0009870 tx=0x7f8be002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6319329b0a2f9fb3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:10.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.331+0000 7f8bf3e00640 1 -- 172.21.15.86:0/3744567797 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8be0035020 con 0x7f8bf40cff60 2024-03-06T07:28:10.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.331+0000 7f8bf3e00640 1 -- 172.21.15.86:0/3744567797 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8be0031640 con 0x7f8bf40cff60 2024-03-06T07:28:10.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.331+0000 7f8bf3e00640 1 -- 172.21.15.86:0/3744567797 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8be002d040 con 0x7f8bf40cff60 2024-03-06T07:28:10.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 -- 172.21.15.86:0/3744567797 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf40cff60 msgr2=0x7f8bf406d4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/3744567797 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf40cff60 0x7f8bf406d4e0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f8be0009870 tx=0x7f8be002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 -- 172.21.15.86:0/3744567797 shutdown_connections 2024-03-06T07:28:10.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/3744567797 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf406da20 0x7f8bf40ce3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/3744567797 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf40cff60 0x7f8bf406d4e0 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 -- 172.21.15.86:0/3744567797 >> 172.21.15.86:0/3744567797 conn(0x7f8bf406afc0 msgr2=0x7f8bf406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:10.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 -- 172.21.15.86:0/3744567797 shutdown_connections 2024-03-06T07:28:10.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.332+0000 7f8bfaa00640 1 -- 172.21.15.86:0/3744567797 wait complete. 2024-03-06T07:28:10.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.334+0000 7f8bfaa00640 1 Processor -- start 2024-03-06T07:28:10.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.334+0000 7f8bfaa00640 1 -- start start 2024-03-06T07:28:10.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfaa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8bf406da20 0x7f8bf4141eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfaa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf40cff60 0x7f8bf4152f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfaa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf4159570 0x7f8bf41534c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfaa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8bf40cea90 con 0x7f8bf4159570 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfaa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8bf40ce910 con 0x7f8bf406da20 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfaa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8bf40cec10 con 0x7f8bf40cff60 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfa000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf4159570 0x7f8bf41534c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfa000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf4159570 0x7f8bf41534c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55656/0 (socket says 172.21.15.86:55656) 2024-03-06T07:28:10.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bfa000640 1 -- 172.21.15.86:0/476999356 learned_addr learned my addr 172.21.15.86:0/476999356 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:10.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.335+0000 7f8bf8c00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf40cff60 0x7f8bf4152f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfa000640 1 -- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf40cff60 msgr2=0x7f8bf4152f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfa000640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf40cff60 0x7f8bf4152f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfa000640 1 -- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8bf406da20 msgr2=0x7f8bf4141eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bf9600640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8bf406da20 0x7f8bf4141eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfa000640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8bf406da20 0x7f8bf4141eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfa000640 1 -- 172.21.15.86:0/476999356 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8bf41ecb80 con 0x7f8bf4159570 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bf8c00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf40cff60 0x7f8bf4152f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bf9600640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8bf406da20 0x7f8bf4141eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfa000640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf4159570 0x7f8bf41534c0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f8bec007e40 tx=0x7f8bec00e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bf1600640 1 -- 172.21.15.86:0/476999356 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8bec014620 con 0x7f8bf4159570 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bf1600640 1 -- 172.21.15.86:0/476999356 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8bec019070 con 0x7f8bf4159570 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bf1600640 1 -- 172.21.15.86:0/476999356 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8bec014a20 con 0x7f8bf4159570 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8bf41ece60 con 0x7f8bf4159570 2024-03-06T07:28:10.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.336+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8bf41ed420 con 0x7f8bf4159570 2024-03-06T07:28:10.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.337+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8bf406b8d0 con 0x7f8bf4159570 2024-03-06T07:28:10.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.338+0000 7f8bf1600640 1 -- 172.21.15.86:0/476999356 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8bec014bc0 con 0x7f8bf4159570 2024-03-06T07:28:10.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.339+0000 7f8bf1600640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bd406b4d0 0x7f8bd406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.339+0000 7f8bf1600640 1 -- 172.21.15.86:0/476999356 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f8bec08a890 con 0x7f8bf4159570 2024-03-06T07:28:10.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.339+0000 7f8bf9600640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bd406b4d0 0x7f8bd406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.340+0000 7f8bf9600640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bd406b4d0 0x7f8bd406d990 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f8be800a830 tx=0x7f8be8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:10.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.344+0000 7f8bf1600640 1 -- 172.21.15.86:0/476999356 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8bf406b8d0 con 0x7f8bf4159570 2024-03-06T07:28:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.524+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f8bf40b2f80 con 0x7f8bf4159570 2024-03-06T07:28:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.525+0000 7f8bf1600640 1 -- 172.21.15.86:0/476999356 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v31) v1 ==== 93+0+10 (secure 0 0 0) 0x7f8bec0599c0 con 0x7f8bf4159570 2024-03-06T07:28:10.526 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bd406b4d0 msgr2=0x7f8bd406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bd406b4d0 0x7f8bd406d990 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f8be800a830 tx=0x7f8be8008040 comp rx=0 tx=0).stop 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf4159570 msgr2=0x7f8bf41534c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf4159570 0x7f8bf41534c0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f8bec007e40 tx=0x7f8bec00e4b0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 shutdown_connections 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bd406b4d0 0x7f8bd406d990 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8bf4159570 0x7f8bf41534c0 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8bf40cff60 0x7f8bf4152f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 --2- 172.21.15.86:0/476999356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8bf406da20 0x7f8bf4141eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 >> 172.21.15.86:0/476999356 conn(0x7f8bf406afc0 msgr2=0x7f8bf4154b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 shutdown_connections 2024-03-06T07:28:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.528+0000 7f8bfaa00640 1 -- 172.21.15.86:0/476999356 wait complete. 2024-03-06T07:28:10.548 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:28:10.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.750+0000 7f9013000640 1 Processor -- start 2024-03-06T07:28:10.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.750+0000 7f9013000640 1 -- start start 2024-03-06T07:28:10.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f9013000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f900c144c00 0x7f900c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f9013000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f900c06ee70 con 0x7f900c15fe60 2024-03-06T07:28:10.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f9013000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f900c06cef0 con 0x7f900c15f9d0 2024-03-06T07:28:10.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f9013000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f900c06d1d0 con 0x7f900c144c00 2024-03-06T07:28:10.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f9011000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f900c15fe60 0x7f900c0ce7b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50188/0 (socket says 172.21.15.86:50188) 2024-03-06T07:28:10.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f9011000640 1 -- 172.21.15.86:0/3379254653 learned_addr learned my addr 172.21.15.86:0/3379254653 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:10.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f900be00640 1 --2- 172.21.15.86:0/3379254653 >> v2:172.21.15.86:3300/0 conn(0x7f900c144c00 0x7f900c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f900b400640 1 -- 172.21.15.86:0/3379254653 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 108800363 0 0) 0x7f9004001490 con 0x7f900c15fe60 2024-03-06T07:28:10.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.752+0000 7f900b400640 1 -- 172.21.15.86:0/3379254653 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8ff0003520 con 0x7f900c15fe60 2024-03-06T07:28:10.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.753+0000 7f900be00640 1 -- 172.21.15.86:0/3379254653 >> v1:172.21.15.194:6789/0 conn(0x7f900c15f9d0 legacy=0x7f900c0ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:10.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.753+0000 7f900be00640 1 -- 172.21.15.86:0/3379254653 >> v1:172.21.15.86:6789/0 conn(0x7f900c15fe60 legacy=0x7f900c0ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.753+0000 7f900be00640 1 -- 172.21.15.86:0/3379254653 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f900c06d4b0 con 0x7f900c144c00 2024-03-06T07:28:10.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.754+0000 7f900be00640 1 --2- 172.21.15.86:0/3379254653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c144c00 0x7f900c06e930 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f8ffc0097a0 tx=0x7f8ffc02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=57a8699b04ba3851 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:10.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.754+0000 7f900b400640 1 -- 172.21.15.86:0/3379254653 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ffc035020 con 0x7f900c144c00 2024-03-06T07:28:10.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.754+0000 7f900b400640 1 -- 172.21.15.86:0/3379254653 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ffc031830 con 0x7f900c144c00 2024-03-06T07:28:10.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.754+0000 7f900b400640 1 -- 172.21.15.86:0/3379254653 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ffc031be0 con 0x7f900c144c00 2024-03-06T07:28:10.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.755+0000 7f9013000640 1 -- 172.21.15.86:0/3379254653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c144c00 msgr2=0x7f900c06e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.755+0000 7f9013000640 1 --2- 172.21.15.86:0/3379254653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c144c00 0x7f900c06e930 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f8ffc0097a0 tx=0x7f8ffc02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.755+0000 7f9013000640 1 -- 172.21.15.86:0/3379254653 shutdown_connections 2024-03-06T07:28:10.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.755+0000 7f9013000640 1 --2- 172.21.15.86:0/3379254653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c144c00 0x7f900c06e930 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.755+0000 7f9013000640 1 -- 172.21.15.86:0/3379254653 >> 172.21.15.86:0/3379254653 conn(0x7f900c0b1730 msgr2=0x7f900c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:10.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.755+0000 7f9013000640 1 -- 172.21.15.86:0/3379254653 shutdown_connections 2024-03-06T07:28:10.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.755+0000 7f9013000640 1 -- 172.21.15.86:0/3379254653 wait complete. 2024-03-06T07:28:10.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.756+0000 7f9013000640 1 Processor -- start 2024-03-06T07:28:10.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.757+0000 7f9013000640 1 -- start start 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.757+0000 7f9013000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f900c144c00 0x7f900c140750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.757+0000 7f9013000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c15f9d0 0x7f900c0c49a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9013000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f900c15fe60 0x7f900c070b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9013000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f900c160730 con 0x7f900c15f9d0 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9013000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f900c160270 con 0x7f900c144c00 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9013000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f900c0ceec0 con 0x7f900c15fe60 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c15f9d0 0x7f900c0c49a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c15f9d0 0x7f900c0c49a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55682/0 (socket says 172.21.15.86:55682) 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 -- 172.21.15.86:0/3419468799 learned_addr learned my addr 172.21.15.86:0/3419468799 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 -- 172.21.15.86:0/3419468799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f900c15fe60 msgr2=0x7f900c070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f900c15fe60 0x7f900c070b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f900be00640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f900c144c00 0x7f900c140750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 -- 172.21.15.86:0/3419468799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f900c144c00 msgr2=0x7f900c140750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f900c144c00 0x7f900c140750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 -- 172.21.15.86:0/3419468799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f900c15ee80 con 0x7f900c15f9d0 2024-03-06T07:28:10.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.758+0000 7f9003400640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c15f9d0 0x7f900c0c49a0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f8ff400c430 tx=0x7f8ff400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:10.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.759+0000 7f9008c00640 1 -- 172.21.15.86:0/3419468799 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ff4011790 con 0x7f900c15f9d0 2024-03-06T07:28:10.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.759+0000 7f9008c00640 1 -- 172.21.15.86:0/3419468799 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ff4017070 con 0x7f900c15f9d0 2024-03-06T07:28:10.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.759+0000 7f9008c00640 1 -- 172.21.15.86:0/3419468799 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ff4011bb0 con 0x7f900c15f9d0 2024-03-06T07:28:10.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.759+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f900c15d610 con 0x7f900c15f9d0 2024-03-06T07:28:10.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.759+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f900c15dbd0 con 0x7f900c15f9d0 2024-03-06T07:28:10.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.760+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f900c0cce60 con 0x7f900c15f9d0 2024-03-06T07:28:10.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.761+0000 7f9008c00640 1 -- 172.21.15.86:0/3419468799 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8ff401c460 con 0x7f900c15f9d0 2024-03-06T07:28:10.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.762+0000 7f9008c00640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8fe806b4d0 0x7f8fe806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:10.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.762+0000 7f9008c00640 1 -- 172.21.15.86:0/3419468799 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f8ff4089da0 con 0x7f900c15f9d0 2024-03-06T07:28:10.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.762+0000 7f900be00640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8fe806b4d0 0x7f8fe806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:10.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.762+0000 7f900be00640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8fe806b4d0 0x7f8fe806d990 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f8ffc036000 tx=0x7f8ffc008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:10.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.766+0000 7f9008c00640 1 -- 172.21.15.86:0/3419468799 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f900c0cce60 con 0x7f900c15f9d0 2024-03-06T07:28:10.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.965+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f900c145820 con 0x7f900c15f9d0 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.966+0000 7f9008c00640 1 -- 172.21.15.86:0/3419468799 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v31) v1 ==== 84+0+65 (secure 0 0 0) 0x7f8ff4058f60 con 0x7f900c15f9d0 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:10.968 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:28:10.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8fe806b4d0 msgr2=0x7f8fe806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8fe806b4d0 0x7f8fe806d990 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f8ffc036000 tx=0x7f8ffc008030 comp rx=0 tx=0).stop 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c15f9d0 msgr2=0x7f900c0c49a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c15f9d0 0x7f900c0c49a0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f8ff400c430 tx=0x7f8ff400c8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 shutdown_connections 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8fe806b4d0 0x7f8fe806d990 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f900c15fe60 0x7f900c070b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f900c15f9d0 0x7f900c0c49a0 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 --2- 172.21.15.86:0/3419468799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f900c144c00 0x7f900c140750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 >> 172.21.15.86:0/3419468799 conn(0x7f900c0b1730 msgr2=0x7f900c143870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 shutdown_connections 2024-03-06T07:28:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:10.969+0000 7f9013000640 1 -- 172.21.15.86:0/3419468799 wait complete. 2024-03-06T07:28:10.989 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:28:10.990 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:28:11.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.190+0000 7f71d9e00640 1 Processor -- start 2024-03-06T07:28:11.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.190+0000 7f71d9e00640 1 -- start start 2024-03-06T07:28:11.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.192+0000 7f71d9e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f71d406afc0 0x7f71d406b420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.192+0000 7f71d9e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71d415e880 con 0x7f71d406cd00 2024-03-06T07:28:11.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.192+0000 7f71d9e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71d415eb20 con 0x7f71d415bb30 2024-03-06T07:28:11.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.192+0000 7f71d9e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f71d406b960 con 0x7f71d406afc0 2024-03-06T07:28:11.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.192+0000 7f71d9400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f71d406cd00 0x7f71d406d190 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50200/0 (socket says 172.21.15.86:50200) 2024-03-06T07:28:11.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.192+0000 7f71d9400640 1 -- 172.21.15.86:0/10000542 learned_addr learned my addr 172.21.15.86:0/10000542 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.192+0000 7f71d3e00640 1 --2- 172.21.15.86:0/10000542 >> v2:172.21.15.194:3300/0 conn(0x7f71d406afc0 0x7f71d406b420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3169572705 0 0) 0x7f71d415e880 con 0x7f71d406cd00 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f71bc005730 con 0x7f71d406cd00 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4171251734 0 0) 0x7f71bc005730 con 0x7f71d406cd00 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 >> v2:172.21.15.194:3300/0 conn(0x7f71d406afc0 msgr2=0x7f71d406b420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 --2- 172.21.15.86:0/10000542 >> v2:172.21.15.194:3300/0 conn(0x7f71d406afc0 0x7f71d406b420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 >> v1:172.21.15.194:6789/0 conn(0x7f71d415bb30 legacy=0x7f71d4070970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71d415ee00 con 0x7f71d406cd00 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f71cc0019f0 con 0x7f71d406cd00 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 >> v1:172.21.15.86:6789/0 conn(0x7f71d406cd00 legacy=0x7f71d406d190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71bc006030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71bc006a10 0x7f71bc006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71bc007420 0x7f71bc00dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71bc00e340 con 0x7f71bc005c10 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71bc00e4f0 con 0x7f71bc007420 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f71bc00e6f0 con 0x7f71bc006a10 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.193+0000 7f71d8a00640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71bc006030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d9400640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71bc007420 0x7f71bc00dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d3e00640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71bc006a10 0x7f71bc006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d8a00640 1 -- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71bc006a10 msgr2=0x7f71bc006e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d8a00640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71bc006a10 0x7f71bc006e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d8a00640 1 -- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71bc007420 msgr2=0x7f71bc00dbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d8a00640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71bc007420 0x7f71bc00dbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d8a00640 1 -- 172.21.15.86:0/10000542 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71bc005730 con 0x7f71bc005c10 2024-03-06T07:28:11.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.194+0000 7f71d9400640 1 -- 172.21.15.86:0/10000542 reap_dead start 2024-03-06T07:28:11.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.195+0000 7f71d8a00640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71bc006030 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f71c8001720 tx=0x7f71bc007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4967f10ae865a013 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:11.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.195+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71c8001370 con 0x7f71bc005c10 2024-03-06T07:28:11.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.195+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f71c80052d0 con 0x7f71bc005c10 2024-03-06T07:28:11.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.195+0000 7f71d3400640 1 -- 172.21.15.86:0/10000542 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71c8003830 con 0x7f71bc005c10 2024-03-06T07:28:11.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.196+0000 7f71d9e00640 1 -- 172.21.15.86:0/10000542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 msgr2=0x7f71bc006030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:11.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.196+0000 7f71d9e00640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71bc006030 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f71c8001720 tx=0x7f71bc007a40 comp rx=0 tx=0).stop 2024-03-06T07:28:11.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.196+0000 7f71d9e00640 1 -- 172.21.15.86:0/10000542 shutdown_connections 2024-03-06T07:28:11.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.196+0000 7f71d9e00640 1 --2- 172.21.15.86:0/10000542 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71bc006030 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:11.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.196+0000 7f71d9e00640 1 -- 172.21.15.86:0/10000542 >> 172.21.15.86:0/10000542 conn(0x7f71d40b18a0 msgr2=0x7f71d40b1cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:11.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.196+0000 7f71d9e00640 1 -- 172.21.15.86:0/10000542 shutdown_connections 2024-03-06T07:28:11.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.196+0000 7f71d9e00640 1 -- 172.21.15.86:0/10000542 wait complete. 2024-03-06T07:28:11.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.198+0000 7f71d9e00640 1 Processor -- start 2024-03-06T07:28:11.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.198+0000 7f71d9e00640 1 -- start start 2024-03-06T07:28:11.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.199+0000 7f71d9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71d414e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.199+0000 7f71d9e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71d4153120 0x7f71d41535a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.199+0000 7f71d9e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71d40afd00 0x7f71d40b0180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.199+0000 7f71d9e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71d415f600 con 0x7f71bc005c10 2024-03-06T07:28:11.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.199+0000 7f71d9e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71d4071430 con 0x7f71d4153120 2024-03-06T07:28:11.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.199+0000 7f71d9e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f71d415bf10 con 0x7f71d40afd00 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71d414e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71d414e2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55708/0 (socket says 172.21.15.86:55708) 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 -- 172.21.15.86:0/2294435741 learned_addr learned my addr 172.21.15.86:0/2294435741 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d3e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71d4153120 0x7f71d41535a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d9400640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71d40afd00 0x7f71d40b0180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 -- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71d40afd00 msgr2=0x7f71d40b0180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71d40afd00 0x7f71d40b0180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 -- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71d4153120 msgr2=0x7f71d41535a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71d4153120 0x7f71d41535a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 -- 172.21.15.86:0/2294435741 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71d4153c70 con 0x7f71bc005c10 2024-03-06T07:28:11.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.200+0000 7f71d8a00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71d414e2f0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f71c8005040 tx=0x7f71c8001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:11.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.201+0000 7f71d0c00640 1 -- 172.21.15.86:0/2294435741 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71c8041070 con 0x7f71bc005c10 2024-03-06T07:28:11.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.201+0000 7f71d0c00640 1 -- 172.21.15.86:0/2294435741 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f71c8005480 con 0x7f71bc005c10 2024-03-06T07:28:11.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.201+0000 7f71d0c00640 1 -- 172.21.15.86:0/2294435741 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71c8007a30 con 0x7f71bc005c10 2024-03-06T07:28:11.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.201+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f71d4153f50 con 0x7f71bc005c10 2024-03-06T07:28:11.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.201+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f71d4154510 con 0x7f71bc005c10 2024-03-06T07:28:11.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.202+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f71d404f330 con 0x7f71bc005c10 2024-03-06T07:28:11.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.203+0000 7f71d0c00640 1 -- 172.21.15.86:0/2294435741 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f71c8044070 con 0x7f71bc005c10 2024-03-06T07:28:11.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.204+0000 7f71d0c00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71b406b500 0x7f71b406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:11.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.204+0000 7f71d0c00640 1 -- 172.21.15.86:0/2294435741 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f71c80ab9f0 con 0x7f71bc005c10 2024-03-06T07:28:11.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.205+0000 7f71d3e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71b406b500 0x7f71b406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:11.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.205+0000 7f71d3e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71b406b500 0x7f71b406d9c0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f71d414eef0 tx=0x7f71c0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:11.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.209+0000 7f71d0c00640 1 -- 172.21.15.86:0/2294435741 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f71c80ade20 con 0x7f71bc005c10 2024-03-06T07:28:11.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:11.391+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f71d40b1220 con 0x7f71bc005c10 2024-03-06T07:28:12.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.138+0000 7f71d0c00640 1 -- 172.21.15.86:0/2294435741 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v32) v1 ==== 138+0+0 (secure 0 0 0) 0x7f71c807ac30 con 0x7f71bc005c10 2024-03-06T07:28:12.140 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:28:12.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.142+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71b406b500 msgr2=0x7f71b406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:12.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.142+0000 7f71d9e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71b406b500 0x7f71b406d9c0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f71d414eef0 tx=0x7f71c0008040 comp rx=0 tx=0).stop 2024-03-06T07:28:12.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.142+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 msgr2=0x7f71d414e2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:12.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.142+0000 7f71d9e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71d414e2f0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f71c8005040 tx=0x7f71c8001120 comp rx=0 tx=0).stop 2024-03-06T07:28:12.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.142+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 shutdown_connections 2024-03-06T07:28:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.142+0000 7f71d9e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f71b406b500 0x7f71b406d9c0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.143+0000 7f71d9e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71d40afd00 0x7f71d40b0180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.143+0000 7f71d9e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71d4153120 0x7f71d41535a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.143+0000 7f71d9e00640 1 --2- 172.21.15.86:0/2294435741 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71bc005c10 0x7f71d414e2f0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.143+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 >> 172.21.15.86:0/2294435741 conn(0x7f71d40b18a0 msgr2=0x7f71d40cce50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.143+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 shutdown_connections 2024-03-06T07:28:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.143+0000 7f71d9e00640 1 -- 172.21.15.86:0/2294435741 wait complete. 2024-03-06T07:28:12.169 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:28:12.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.363+0000 7f72ffc00640 1 Processor -- start 2024-03-06T07:28:12.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.363+0000 7f72ffc00640 1 -- start start 2024-03-06T07:28:12.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72ffc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f72f815cb90 0x7f72f806d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:12.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72ffc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 0x7f72f806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:12.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72ffc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72f815cf70 con 0x7f72f8163800 2024-03-06T07:28:12.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72ffc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f72f80b3890 con 0x7f72f815cb90 2024-03-06T07:28:12.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72ffc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72f806c990 con 0x7f72f806e060 2024-03-06T07:28:12.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72fd200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f72f815cb90 0x7f72f806d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:12.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72fd200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f72f815cb90 0x7f72f806d6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55714/0 (socket says 172.21.15.86:55714) 2024-03-06T07:28:12.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72fd200640 1 -- 172.21.15.86:0/577418783 learned_addr learned my addr 172.21.15.86:0/577418783 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:12.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.365+0000 7f72fdc00640 1 --2- 172.21.15.86:0/577418783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 0x7f72f806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:12.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.366+0000 7f72fd200640 1 -- 172.21.15.86:0/577418783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 msgr2=0x7f72f806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:12.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.366+0000 7f72fd200640 1 --2- 172.21.15.86:0/577418783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 0x7f72f806e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.366+0000 7f72fd200640 1 -- 172.21.15.86:0/577418783 >> v1:172.21.15.194:6789/0 conn(0x7f72f8163800 legacy=0x7f72f80b2c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:12.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.366+0000 7f72fd200640 1 -- 172.21.15.86:0/577418783 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72f80b35b0 con 0x7f72f815cb90 2024-03-06T07:28:12.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.367+0000 7f72fd200640 1 --2- 172.21.15.86:0/577418783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f806d6c0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f72e8009870 tx=0x7f72e802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=324887d8267b8350 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:12.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.367+0000 7f72f7e00640 1 -- 172.21.15.86:0/577418783 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e8035020 con 0x7f72f815cb90 2024-03-06T07:28:12.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.367+0000 7f72f7e00640 1 -- 172.21.15.86:0/577418783 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72e8031640 con 0x7f72f815cb90 2024-03-06T07:28:12.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.367+0000 7f72f7e00640 1 -- 172.21.15.86:0/577418783 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e802d040 con 0x7f72f815cb90 2024-03-06T07:28:12.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 -- 172.21.15.86:0/577418783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 msgr2=0x7f72f806d6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:12.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 --2- 172.21.15.86:0/577418783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f806d6c0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f72e8009870 tx=0x7f72e802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 -- 172.21.15.86:0/577418783 shutdown_connections 2024-03-06T07:28:12.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 --2- 172.21.15.86:0/577418783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 0x7f72f806e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 --2- 172.21.15.86:0/577418783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f806d6c0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 -- 172.21.15.86:0/577418783 >> 172.21.15.86:0/577418783 conn(0x7f72f80b1940 msgr2=0x7f72f80b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:12.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 -- 172.21.15.86:0/577418783 shutdown_connections 2024-03-06T07:28:12.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.368+0000 7f72ffc00640 1 -- 172.21.15.86:0/577418783 wait complete. 2024-03-06T07:28:12.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.370+0000 7f72ffc00640 1 Processor -- start 2024-03-06T07:28:12.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.370+0000 7f72ffc00640 1 -- start start 2024-03-06T07:28:12.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72ffc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 0x7f72f814ba10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:12.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72ffc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f80ae1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72ffc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f8163800 0x7f72f80ae730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72ffc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72f80b3350 con 0x7f72f815cb90 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72ffc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72f806d230 con 0x7f72f8163800 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72ffc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72f806cec0 con 0x7f72f806e060 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72efe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f80ae1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72efe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f80ae1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55728/0 (socket says 172.21.15.86:55728) 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.371+0000 7f72efe00640 1 -- 172.21.15.86:0/126633527 learned_addr learned my addr 172.21.15.86:0/126633527 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72efe00640 1 -- 172.21.15.86:0/126633527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 msgr2=0x7f72f814ba10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72efe00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 0x7f72f814ba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72fdc00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f8163800 0x7f72f80ae730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72efe00640 1 -- 172.21.15.86:0/126633527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f8163800 msgr2=0x7f72f80ae730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72efe00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f8163800 0x7f72f80ae730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72efe00640 1 -- 172.21.15.86:0/126633527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72f8157410 con 0x7f72f815cb90 2024-03-06T07:28:12.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72efe00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f80ae1f0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f72e000b3f0 tx=0x7f72e000b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:12.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.372+0000 7f72f5600640 1 -- 172.21.15.86:0/126633527 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e0015020 con 0x7f72f815cb90 2024-03-06T07:28:12.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.373+0000 7f72f5600640 1 -- 172.21.15.86:0/126633527 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72e0011850 con 0x7f72f815cb90 2024-03-06T07:28:12.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.373+0000 7f72f5600640 1 -- 172.21.15.86:0/126633527 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72e0011c40 con 0x7f72f815cb90 2024-03-06T07:28:12.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.373+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72f81576f0 con 0x7f72f815cb90 2024-03-06T07:28:12.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.373+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72f8157cb0 con 0x7f72f815cb90 2024-03-06T07:28:12.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.374+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f72f804f330 con 0x7f72f815cb90 2024-03-06T07:28:12.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.375+0000 7f72f5600640 1 -- 172.21.15.86:0/126633527 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f72e0018020 con 0x7f72f815cb90 2024-03-06T07:28:12.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.376+0000 7f72f5600640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72d406b4d0 0x7f72d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:12.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.376+0000 7f72f5600640 1 -- 172.21.15.86:0/126633527 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(32..32 src has 1..32) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f72e008a5f0 con 0x7f72f815cb90 2024-03-06T07:28:12.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.376+0000 7f72fd200640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72d406b4d0 0x7f72d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:12.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.377+0000 7f72fd200640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72d406b4d0 0x7f72d406d990 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f72e8036000 tx=0x7f72e8030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:12.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.380+0000 7f72f5600640 1 -- 172.21.15.86:0/126633527 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f72e001f020 con 0x7f72f815cb90 2024-03-06T07:28:12.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:12.574+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f72f80b2530 con 0x7f72f815cb90 2024-03-06T07:28:13.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.152+0000 7f72f5600640 1 -- 172.21.15.86:0/126633527 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v33) v1 ==== 170+0+0 (secure 0 0 0) 0x7f72e000f2e0 con 0x7f72f815cb90 2024-03-06T07:28:13.155 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:28:13.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72d406b4d0 msgr2=0x7f72d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72d406b4d0 0x7f72d406d990 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f72e8036000 tx=0x7f72e8030000 comp rx=0 tx=0).stop 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 msgr2=0x7f72f80ae1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f80ae1f0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f72e000b3f0 tx=0x7f72e000b8b0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 shutdown_connections 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f72d406b4d0 0x7f72d406d990 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f72f8163800 0x7f72f80ae730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f72f815cb90 0x7f72f80ae1f0 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 --2- 172.21.15.86:0/126633527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f72f806e060 0x7f72f814ba10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.156+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 >> 172.21.15.86:0/126633527 conn(0x7f72f80b1940 msgr2=0x7f72f8164c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:13.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.157+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 shutdown_connections 2024-03-06T07:28:13.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.157+0000 7f72ffc00640 1 -- 172.21.15.86:0/126633527 wait complete. 2024-03-06T07:28:13.193 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:28:13.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.373+0000 7f5f1fc00640 1 Processor -- start 2024-03-06T07:28:13.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.373+0000 7f5f1fc00640 1 -- start start 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.374+0000 7f5f1fc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5f1815cb90 0x7f5f1806d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.374+0000 7f5f1fc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 0x7f5f1806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.374+0000 7f5f1fc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5f1815cf70 con 0x7f5f18163800 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.374+0000 7f5f1fc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5f180b3890 con 0x7f5f1815cb90 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.374+0000 7f5f1fc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f1806c990 con 0x7f5f1806e060 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.375+0000 7f5f17e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f5f18163800 0x7f5f180b2c40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50214/0 (socket says 172.21.15.86:50214) 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.375+0000 7f5f17e00640 1 -- 172.21.15.86:0/1318492237 learned_addr learned my addr 172.21.15.86:0/1318492237 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.375+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 0x7f5f1806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.375+0000 7f5f1d200640 1 --2- 172.21.15.86:0/1318492237 >> v2:172.21.15.194:3300/0 conn(0x7f5f1815cb90 0x7f5f1806d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.375+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 231286655 0 0) 0x7f5f1815cf70 con 0x7f5f18163800 2024-03-06T07:28:13.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.375+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5f00003520 con 0x7f5f18163800 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3608375504 0 0) 0x7f5f00003520 con 0x7f5f18163800 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 msgr2=0x7f5f1806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 0x7f5f1806e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 >> v2:172.21.15.194:3300/0 conn(0x7f5f1815cb90 msgr2=0x7f5f1806d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 --2- 172.21.15.86:0/1318492237 >> v2:172.21.15.194:3300/0 conn(0x7f5f1815cb90 0x7f5f1806d6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f180b35b0 con 0x7f5f18163800 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f5f04001670 con 0x7f5f18163800 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 >> v1:172.21.15.86:6789/0 conn(0x7f5f18163800 legacy=0x7f5f180b2c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f000039b0 0x7f5f00003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 0x7f5f00004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f00004d40 0x7f5f000051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f180b3890 con 0x7f5f00004d40 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f1806c990 con 0x7f5f000039b0 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f0000b9c0 con 0x7f5f1806e060 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.376+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f00004d40 0x7f5f000051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f17e00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 0x7f5f00004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f1d200640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f000039b0 0x7f5f00003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 msgr2=0x7f5f00004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 0x7f5f00004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f000039b0 msgr2=0x7f5f00003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f000039b0 0x7f5f00003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1318492237 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f00003520 con 0x7f5f00004d40 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f17e00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f1806e060 0x7f5f00004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:13.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.377+0000 7f5f1d200640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f000039b0 0x7f5f00003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:13.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.378+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f00004d40 0x7f5f000051c0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f5f10009fc0 tx=0x7f5f00005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=95d52fbd67102a83 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:13.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.378+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f10039070 con 0x7f5f00004d40 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.378+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f1002cc20 con 0x7f5f00004d40 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.378+0000 7f5f17400640 1 -- 172.21.15.86:0/1318492237 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f10009c10 con 0x7f5f00004d40 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.379+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1318492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f00004d40 msgr2=0x7f5f000051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.379+0000 7f5f1fc00640 1 --2- 172.21.15.86:0/1318492237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f00004d40 0x7f5f000051c0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f5f10009fc0 tx=0x7f5f00005830 comp rx=0 tx=0).stop 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.379+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1318492237 reap_dead start 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.379+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1318492237 shutdown_connections 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.379+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1318492237 >> 172.21.15.86:0/1318492237 conn(0x7f5f180b1940 msgr2=0x7f5f180b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.379+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1318492237 shutdown_connections 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.379+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1318492237 wait complete. 2024-03-06T07:28:13.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.381+0000 7f5f1fc00640 1 Processor -- start 2024-03-06T07:28:13.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.381+0000 7f5f1fc00640 1 -- start start 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1fc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f00004d40 0x7f5f181529f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1fc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f180adcf0 0x7f5f1815af40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1fc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f1809c050 0x7f5f1809c4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1fc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f180b3350 con 0x7f5f1809c050 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1fc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f1806d230 con 0x7f5f00004d40 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1fc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f1806cec0 con 0x7f5f180adcf0 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f1809c050 0x7f5f1809c4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f1809c050 0x7f5f1809c4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55734/0 (socket says 172.21.15.86:55734) 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1706024993 learned_addr learned my addr 172.21.15.86:0/1706024993 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:13.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1d200640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f00004d40 0x7f5f181529f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1706024993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f180adcf0 msgr2=0x7f5f1815af40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:13.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f180adcf0 0x7f5f1815af40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1706024993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f00004d40 msgr2=0x7f5f181529f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:13.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f00004d40 0x7f5f181529f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:13.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.382+0000 7f5f1dc00640 1 -- 172.21.15.86:0/1706024993 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f18152f30 con 0x7f5f1809c050 2024-03-06T07:28:13.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.383+0000 7f5f1dc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f1809c050 0x7f5f1809c4d0 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f5f10037040 tx=0x7f5f10009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:13.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.383+0000 7f5f14c00640 1 -- 172.21.15.86:0/1706024993 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f10039070 con 0x7f5f1809c050 2024-03-06T07:28:13.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.383+0000 7f5f14c00640 1 -- 172.21.15.86:0/1706024993 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f1003a040 con 0x7f5f1809c050 2024-03-06T07:28:13.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.383+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5f180ae220 con 0x7f5f1809c050 2024-03-06T07:28:13.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.383+0000 7f5f14c00640 1 -- 172.21.15.86:0/1706024993 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f10009c10 con 0x7f5f1809c050 2024-03-06T07:28:13.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.383+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5f180ae7e0 con 0x7f5f1809c050 2024-03-06T07:28:13.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.384+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5f1804f330 con 0x7f5f1809c050 2024-03-06T07:28:13.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.385+0000 7f5f14c00640 1 -- 172.21.15.86:0/1706024993 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5f10003bb0 con 0x7f5f1809c050 2024-03-06T07:28:13.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.387+0000 7f5f14c00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5ef806b4d0 0x7f5ef806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:13.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.387+0000 7f5f14c00640 1 -- 172.21.15.86:0/1706024993 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f5f100ac610 con 0x7f5f1809c050 2024-03-06T07:28:13.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.387+0000 7f5f1d200640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5ef806b4d0 0x7f5ef806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:13.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.387+0000 7f5f1d200640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5ef806b4d0 0x7f5ef806d990 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f5f0c009a40 tx=0x7f5f0c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:13.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.392+0000 7f5f14c00640 1 -- 172.21.15.86:0/1706024993 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5f1007b4e0 con 0x7f5f1809c050 2024-03-06T07:28:13.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:13.616+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f5f180b2530 con 0x7f5f1809c050 2024-03-06T07:28:14.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.157+0000 7f5f14c00640 1 -- 172.21.15.86:0/1706024993 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 4 and data pool 5 v13) v1 ==== 172+0+0 (secure 0 0 0) 0x7f5f1007ade0 con 0x7f5f1809c050 2024-03-06T07:28:14.159 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 4 and data pool 5 2024-03-06T07:28:14.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.161+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5ef806b4d0 msgr2=0x7f5ef806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5ef806b4d0 0x7f5ef806d990 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f5f0c009a40 tx=0x7f5f0c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f1809c050 msgr2=0x7f5f1809c4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f1809c050 0x7f5f1809c4d0 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f5f10037040 tx=0x7f5f10009750 comp rx=0 tx=0).stop 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 shutdown_connections 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5ef806b4d0 0x7f5ef806d990 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f1809c050 0x7f5f1809c4d0 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f180adcf0 0x7f5f1815af40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 --2- 172.21.15.86:0/1706024993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f00004d40 0x7f5f181529f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 >> 172.21.15.86:0/1706024993 conn(0x7f5f180b1940 msgr2=0x7f5f180b29d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 shutdown_connections 2024-03-06T07:28:14.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.162+0000 7f5f1fc00640 1 -- 172.21.15.86:0/1706024993 wait complete. 2024-03-06T07:28:14.207 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:14.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.381+0000 7fc084000640 1 Processor -- start 2024-03-06T07:28:14.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.381+0000 7fc084000640 1 -- start start 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.382+0000 7fc084000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc07c144c00 0x7fc07c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.382+0000 7fc084000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc07c06ee70 con 0x7fc07c15fe60 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.382+0000 7fc084000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc07c06cef0 con 0x7fc07c15f9d0 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.382+0000 7fc084000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc07c06d1d0 con 0x7fc07c144c00 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.382+0000 7fc081600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc07c144c00 0x7fc07c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.383+0000 7fc082000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc07c15fe60 0x7fc07c0ce7b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50220/0 (socket says 172.21.15.86:50220) 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.383+0000 7fc081600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc07c144c00 0x7fc07c06e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55740/0 (socket says 172.21.15.86:55740) 2024-03-06T07:28:14.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.383+0000 7fc082000640 1 -- 172.21.15.86:0/444584107 learned_addr learned my addr 172.21.15.86:0/444584107 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:14.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.383+0000 7fc081600640 1 -- 172.21.15.86:0/444584107 >> v1:172.21.15.194:6789/0 conn(0x7fc07c15f9d0 legacy=0x7fc07c0ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:14.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.383+0000 7fc081600640 1 -- 172.21.15.86:0/444584107 >> v1:172.21.15.86:6789/0 conn(0x7fc07c15fe60 legacy=0x7fc07c0ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.383+0000 7fc081600640 1 -- 172.21.15.86:0/444584107 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc07c06d4b0 con 0x7fc07c144c00 2024-03-06T07:28:14.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.384+0000 7fc081600640 1 --2- 172.21.15.86:0/444584107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c144c00 0x7fc07c06e930 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7fc0700097a0 tx=0x7fc07002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d6cd9fcd505329f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:14.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.384+0000 7fc07be00640 1 -- 172.21.15.86:0/444584107 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc070035020 con 0x7fc07c144c00 2024-03-06T07:28:14.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.384+0000 7fc07be00640 1 -- 172.21.15.86:0/444584107 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc070031830 con 0x7fc07c144c00 2024-03-06T07:28:14.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.384+0000 7fc07be00640 1 -- 172.21.15.86:0/444584107 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc070031c00 con 0x7fc07c144c00 2024-03-06T07:28:14.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.385+0000 7fc084000640 1 -- 172.21.15.86:0/444584107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c144c00 msgr2=0x7fc07c06e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.385+0000 7fc084000640 1 --2- 172.21.15.86:0/444584107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c144c00 0x7fc07c06e930 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7fc0700097a0 tx=0x7fc07002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.385+0000 7fc084000640 1 -- 172.21.15.86:0/444584107 shutdown_connections 2024-03-06T07:28:14.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.385+0000 7fc084000640 1 --2- 172.21.15.86:0/444584107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c144c00 0x7fc07c06e930 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.385+0000 7fc084000640 1 -- 172.21.15.86:0/444584107 >> 172.21.15.86:0/444584107 conn(0x7fc07c0b1730 msgr2=0x7fc07c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:14.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.385+0000 7fc084000640 1 -- 172.21.15.86:0/444584107 shutdown_connections 2024-03-06T07:28:14.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.385+0000 7fc084000640 1 -- 172.21.15.86:0/444584107 wait complete. 2024-03-06T07:28:14.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.387+0000 7fc084000640 1 Processor -- start 2024-03-06T07:28:14.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.387+0000 7fc084000640 1 -- start start 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc084000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc07c144c00 0x7fc07c139ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc084000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c15f9d0 0x7fc07c15b540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc084000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc07c15fe60 0x7fc07c141300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc084000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc07c160730 con 0x7fc07c15f9d0 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc084000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc07c160270 con 0x7fc07c15fe60 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc084000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc07c0ceec0 con 0x7fc07c144c00 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc080c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c15f9d0 0x7fc07c15b540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc080c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c15f9d0 0x7fc07c15b540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55756/0 (socket says 172.21.15.86:55756) 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc080c00640 1 -- 172.21.15.86:0/3413762295 learned_addr learned my addr 172.21.15.86:0/3413762295 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc081600640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc07c144c00 0x7fc07c139ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.388+0000 7fc080c00640 1 -- 172.21.15.86:0/3413762295 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc07c144c00 msgr2=0x7fc07c139ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc080c00640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc07c144c00 0x7fc07c139ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc080c00640 1 -- 172.21.15.86:0/3413762295 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc07c15fe60 msgr2=0x7fc07c141300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc080c00640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc07c15fe60 0x7fc07c141300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc080c00640 1 -- 172.21.15.86:0/3413762295 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc07c13a3a0 con 0x7fc07c15f9d0 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc080c00640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c15f9d0 0x7fc07c15b540 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fc06800c410 tx=0x7fc06800c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc079600640 1 -- 172.21.15.86:0/3413762295 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc068017070 con 0x7fc07c15f9d0 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc079600640 1 -- 172.21.15.86:0/3413762295 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc0680035f0 con 0x7fc07c15f9d0 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc07c13a680 con 0x7fc07c15f9d0 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc079600640 1 -- 172.21.15.86:0/3413762295 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc068003a20 con 0x7fc07c15f9d0 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.389+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc07c13ac40 con 0x7fc07c15f9d0 2024-03-06T07:28:14.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.390+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc07c04f330 con 0x7fc07c15f9d0 2024-03-06T07:28:14.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.392+0000 7fc079600640 1 -- 172.21.15.86:0/3413762295 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc06801a070 con 0x7fc07c15f9d0 2024-03-06T07:28:14.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.392+0000 7fc079600640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc05c06b4d0 0x7fc05c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.392+0000 7fc079600640 1 -- 172.21.15.86:0/3413762295 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(34..34 src has 1..34) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc068089700 con 0x7fc07c15f9d0 2024-03-06T07:28:14.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.393+0000 7fc081600640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc05c06b4d0 0x7fc05c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.393+0000 7fc081600640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc05c06b4d0 0x7fc05c06d990 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fc070036000 tx=0x7fc07003c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:14.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.397+0000 7fc079600640 1 -- 172.21.15.86:0/3413762295 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc068058570 con 0x7fc07c15f9d0 2024-03-06T07:28:14.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.577+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc07c1452e0 con 0x7fc07c15f9d0 2024-03-06T07:28:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.579+0000 7fc079600640 1 -- 172.21.15.86:0/3413762295 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v34) v1 ==== 74+0+16694 (secure 0 0 0) 0x7fc068057e70 con 0x7fc07c15f9d0 2024-03-06T07:28:14.581 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:14.581 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:13.617951+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:11.444335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:12.623953+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.581+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc05c06b4d0 msgr2=0x7fc05c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.581+0000 7fc084000640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc05c06b4d0 0x7fc05c06d990 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fc070036000 tx=0x7fc07003c040 comp rx=0 tx=0).stop 2024-03-06T07:28:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.581+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c15f9d0 msgr2=0x7fc07c15b540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.581+0000 7fc084000640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c15f9d0 0x7fc07c15b540 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fc06800c410 tx=0x7fc06800c8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 shutdown_connections 2024-03-06T07:28:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc05c06b4d0 0x7fc05c06d990 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc07c15fe60 0x7fc07c141300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc07c15f9d0 0x7fc07c15b540 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 --2- 172.21.15.86:0/3413762295 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc07c144c00 0x7fc07c139ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 >> 172.21.15.86:0/3413762295 conn(0x7fc07c0b1730 msgr2=0x7fc07c071dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 shutdown_connections 2024-03-06T07:28:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.582+0000 7fc084000640 1 -- 172.21.15.86:0/3413762295 wait complete. 2024-03-06T07:28:14.601 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:14.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.803+0000 7f12d3200640 1 Processor -- start 2024-03-06T07:28:14.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.803+0000 7f12d3200640 1 -- start start 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.804+0000 7f12d3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d4161580 0x7f12d40d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d3200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f12d40d2a70 0x7f12d40b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d3200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f12d4161960 con 0x7f12d40d1e90 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d3200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f12d40cf6f0 con 0x7f12d40d2a70 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f12d40cf9d0 con 0x7f12d4161580 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d1400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f12d40d1e90 0x7f12d40d2360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50232/0 (socket says 172.21.15.86:50232) 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d1400640 1 -- 172.21.15.86:0/1408604180 learned_addr learned my addr 172.21.15.86:0/1408604180 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d2800640 1 --2- 172.21.15.86:0/1408604180 >> v2:172.21.15.86:3300/0 conn(0x7f12d40d2a70 0x7f12d40b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d1e00640 1 --2- 172.21.15.86:0/1408604180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d4161580 0x7f12d40d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d2800640 1 -- 172.21.15.86:0/1408604180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d4161580 msgr2=0x7f12d40d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d2800640 1 --2- 172.21.15.86:0/1408604180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d4161580 0x7f12d40d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d2800640 1 -- 172.21.15.86:0/1408604180 >> v1:172.21.15.86:6789/0 conn(0x7f12d40d1e90 legacy=0x7f12d40d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d2800640 1 -- 172.21.15.86:0/1408604180 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12d40cfcb0 con 0x7f12d40d2a70 2024-03-06T07:28:14.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.805+0000 7f12d0a00640 1 -- 172.21.15.86:0/1408604180 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 238634725 0 0) 0x7f12d4161960 con 0x7f12d40d1e90 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.806+0000 7f12d2800640 1 --2- 172.21.15.86:0/1408604180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d40d2a70 0x7f12d40b3ec0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f12cc009d90 tx=0x7f12cc02bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f294cebbc4d1c14f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d0a00640 1 -- 172.21.15.86:0/1408604180 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12cc039070 con 0x7f12d40d2a70 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d0a00640 1 -- 172.21.15.86:0/1408604180 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f12cc02cc50 con 0x7f12d40d2a70 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d0a00640 1 -- 172.21.15.86:0/1408604180 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12cc034440 con 0x7f12d40d2a70 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 -- 172.21.15.86:0/1408604180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d40d2a70 msgr2=0x7f12d40b3ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 --2- 172.21.15.86:0/1408604180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d40d2a70 0x7f12d40b3ec0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f12cc009d90 tx=0x7f12cc02bea0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 -- 172.21.15.86:0/1408604180 shutdown_connections 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 --2- 172.21.15.86:0/1408604180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d40d2a70 0x7f12d40b3ec0 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 --2- 172.21.15.86:0/1408604180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d4161580 0x7f12d40d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 -- 172.21.15.86:0/1408604180 >> 172.21.15.86:0/1408604180 conn(0x7f12d40d3d80 msgr2=0x7f12d40d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:14.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 -- 172.21.15.86:0/1408604180 shutdown_connections 2024-03-06T07:28:14.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.807+0000 7f12d3200640 1 -- 172.21.15.86:0/1408604180 wait complete. 2024-03-06T07:28:14.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.809+0000 7f12d3200640 1 Processor -- start 2024-03-06T07:28:14.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.809+0000 7f12d3200640 1 -- start start 2024-03-06T07:28:14.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.810+0000 7f12d3200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12d40d1e90 0x7f12d40b7830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.810+0000 7f12d3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d40d2a70 0x7f12d415ada0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.810+0000 7f12d3200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d4161580 0x7f12d41f7220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.810+0000 7f12d3200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12d40cf3a0 con 0x7f12d4161580 2024-03-06T07:28:14.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.810+0000 7f12d3200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12d40b39a0 con 0x7f12d40d1e90 2024-03-06T07:28:14.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.810+0000 7f12d3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f12d40b4400 con 0x7f12d40d2a70 2024-03-06T07:28:14.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d4161580 0x7f12d41f7220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d4161580 0x7f12d41f7220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55780/0 (socket says 172.21.15.86:55780) 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 -- 172.21.15.86:0/15793912 learned_addr learned my addr 172.21.15.86:0/15793912 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d1e00640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12d40d1e90 0x7f12d40b7830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d1400640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d40d2a70 0x7f12d415ada0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 -- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d40d2a70 msgr2=0x7f12d415ada0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d40d2a70 0x7f12d415ada0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 -- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12d40d1e90 msgr2=0x7f12d40b7830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12d40d1e90 0x7f12d40b7830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 -- 172.21.15.86:0/15793912 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12d415b2e0 con 0x7f12d4161580 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.811+0000 7f12d2800640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d4161580 0x7f12d41f7220 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f12cc009d90 tx=0x7f12cc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.812+0000 7f12ca000640 1 -- 172.21.15.86:0/15793912 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12cc039050 con 0x7f12d4161580 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.812+0000 7f12ca000640 1 -- 172.21.15.86:0/15793912 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f12cc045070 con 0x7f12d4161580 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.812+0000 7f12ca000640 1 -- 172.21.15.86:0/15793912 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12cc03e610 con 0x7f12d4161580 2024-03-06T07:28:14.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.812+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f12d415b5a0 con 0x7f12d4161580 2024-03-06T07:28:14.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.812+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12d415bb60 con 0x7f12d4161580 2024-03-06T07:28:14.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.813+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f12d404f330 con 0x7f12d4161580 2024-03-06T07:28:14.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.814+0000 7f12ca000640 1 -- 172.21.15.86:0/15793912 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f12cc03e7b0 con 0x7f12d4161580 2024-03-06T07:28:14.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.814+0000 7f12ca000640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f12b406b4d0 0x7f12b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:14.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.815+0000 7f12ca000640 1 -- 172.21.15.86:0/15793912 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(34..34 src has 1..34) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f12cc0acd80 con 0x7f12d4161580 2024-03-06T07:28:14.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.815+0000 7f12d1e00640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f12b406b4d0 0x7f12b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:14.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.815+0000 7f12d1e00640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f12b406b4d0 0x7f12b406d990 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f12c4006fd0 tx=0x7f12c4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:14.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:14.819+0000 7f12ca000640 1 -- 172.21.15.86:0/15793912 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f12cc07bc00 con 0x7f12d4161580 2024-03-06T07:28:15.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.014+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f12d40d3b60 con 0x7f12d4161580 2024-03-06T07:28:15.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.017+0000 7f12ca000640 1 -- 172.21.15.86:0/15793912 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v34) v1 ==== 74+0+16694 (secure 0 0 0) 0x7f12cc07b500 con 0x7f12d4161580 2024-03-06T07:28:15.019 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:15.020 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:13.617951+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:11.444335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:12.623953+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f12b406b4d0 msgr2=0x7f12b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f12b406b4d0 0x7f12b406d990 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f12c4006fd0 tx=0x7f12c4008040 comp rx=0 tx=0).stop 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d4161580 msgr2=0x7f12d41f7220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d4161580 0x7f12d41f7220 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f12cc009d90 tx=0x7f12cc004a40 comp rx=0 tx=0).stop 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 shutdown_connections 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f12b406b4d0 0x7f12b406d990 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12d4161580 0x7f12d41f7220 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12d40d2a70 0x7f12d415ada0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 --2- 172.21.15.86:0/15793912 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12d40d1e90 0x7f12d40b7830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 >> 172.21.15.86:0/15793912 conn(0x7f12d40d3d80 msgr2=0x7f12d40d4160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 shutdown_connections 2024-03-06T07:28:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.020+0000 7f12d3200640 1 -- 172.21.15.86:0/15793912 wait complete. 2024-03-06T07:28:15.041 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:28:15.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.246+0000 7f8e0c600640 1 Processor -- start 2024-03-06T07:28:15.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.247+0000 7f8e0c600640 1 -- start start 2024-03-06T07:28:15.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.248+0000 7f8e0c600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8e0415fc10 0x7f8e04070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.248+0000 7f8e0c600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e040b46c0 con 0x7f8e040b3f70 2024-03-06T07:28:15.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.248+0000 7f8e0c600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e040b49a0 con 0x7f8e0415fff0 2024-03-06T07:28:15.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.248+0000 7f8e0c600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e04072c90 con 0x7f8e0415fc10 2024-03-06T07:28:15.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e0a600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8e040b3f70 0x7f8e040ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50234/0 (socket says 172.21.15.86:50234) 2024-03-06T07:28:15.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e0a600640 1 -- 172.21.15.86:0/1985356341 learned_addr learned my addr 172.21.15.86:0/1985356341 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e09c00640 1 --2- 172.21.15.86:0/1985356341 >> v2:172.21.15.194:3300/0 conn(0x7f8e0415fc10 0x7f8e04070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1035111468 0 0) 0x7f8e040b46c0 con 0x7f8e040b3f70 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8dec003520 con 0x7f8e040b3f70 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2488281362 0 0) 0x7f8dec003520 con 0x7f8e040b3f70 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 >> v2:172.21.15.194:3300/0 conn(0x7f8e0415fc10 msgr2=0x7f8e04070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 --2- 172.21.15.86:0/1985356341 >> v2:172.21.15.194:3300/0 conn(0x7f8e0415fc10 0x7f8e04070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 >> v1:172.21.15.194:6789/0 conn(0x7f8e0415fff0 legacy=0x7f8e04160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e0409dd80 con 0x7f8e040b3f70 2024-03-06T07:28:15.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8dfc0019f0 con 0x7f8e040b3f70 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 >> v1:172.21.15.86:6789/0 conn(0x7f8e040b3f70 legacy=0x7f8e040ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8dec0039b0 0x7f8dec003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8dec004800 0x7f8dec004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.249+0000 7f8e03e00640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0051c0 0x7f8dec00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e04072c90 con 0x7f8dec0039b0 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8dec00c040 con 0x7f8dec0051c0 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8dec00c240 con 0x7f8dec004800 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09c00640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8dec0039b0 0x7f8dec003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e0a600640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0051c0 0x7f8dec00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09200640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8dec004800 0x7f8dec004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09c00640 1 -- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8dec004800 msgr2=0x7f8dec004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09c00640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8dec004800 0x7f8dec004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09c00640 1 -- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0051c0 msgr2=0x7f8dec00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09c00640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0051c0 0x7f8dec00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09c00640 1 -- 172.21.15.86:0/1985356341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8dec003520 con 0x7f8dec0039b0 2024-03-06T07:28:15.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e09200640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8dec004800 0x7f8dec004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:15.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e0a600640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0051c0 0x7f8dec00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:28:15.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.250+0000 7f8e0a600640 1 -- 172.21.15.86:0/1985356341 reap_dead start 2024-03-06T07:28:15.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.251+0000 7f8e09c00640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8dec0039b0 0x7f8dec003db0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f8df8009ac0 tx=0x7f8dec005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=25616583a51a4c27 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df8039070 con 0x7f8dec0039b0 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8df802c6b0 con 0x7f8dec0039b0 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e03e00640 1 -- 172.21.15.86:0/1985356341 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df8009690 con 0x7f8dec0039b0 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e0c600640 1 -- 172.21.15.86:0/1985356341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8dec0039b0 msgr2=0x7f8dec003db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e0c600640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8dec0039b0 0x7f8dec003db0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f8df8009ac0 tx=0x7f8dec005830 comp rx=0 tx=0).stop 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e0c600640 1 -- 172.21.15.86:0/1985356341 shutdown_connections 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e0c600640 1 --2- 172.21.15.86:0/1985356341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8dec0039b0 0x7f8dec003db0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e0c600640 1 -- 172.21.15.86:0/1985356341 >> 172.21.15.86:0/1985356341 conn(0x7f8e040b1940 msgr2=0x7f8e040b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e0c600640 1 -- 172.21.15.86:0/1985356341 shutdown_connections 2024-03-06T07:28:15.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.252+0000 7f8e0c600640 1 -- 172.21.15.86:0/1985356341 wait complete. 2024-03-06T07:28:15.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.254+0000 7f8e0c600640 1 Processor -- start 2024-03-06T07:28:15.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.254+0000 7f8e0c600640 1 -- start start 2024-03-06T07:28:15.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e0c600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0039b0 0x7f8e041f8400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e0c600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e04153840 0x7f8e041503b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e0c600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e040ce810 0x7f8e040cec90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e0c600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e0415c460 con 0x7f8e04153840 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e0c600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e0415c2e0 con 0x7f8dec0039b0 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e0c600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e04070ff0 con 0x7f8e040ce810 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e04153840 0x7f8e041503b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e04153840 0x7f8e041503b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55798/0 (socket says 172.21.15.86:55798) 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 -- 172.21.15.86:0/2961760978 learned_addr learned my addr 172.21.15.86:0/2961760978 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09c00640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0039b0 0x7f8e041f8400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 -- 172.21.15.86:0/2961760978 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e040ce810 msgr2=0x7f8e040cec90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e040ce810 0x7f8e040cec90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 -- 172.21.15.86:0/2961760978 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0039b0 msgr2=0x7f8e041f8400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0039b0 0x7f8e041f8400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:15.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.255+0000 7f8e09200640 1 -- 172.21.15.86:0/2961760978 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e04153d50 con 0x7f8e04153840 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.256+0000 7f8e09200640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e04153840 0x7f8e041503b0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f8df000d3b0 tx=0x7f8df000d870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.256+0000 7f8e01600640 1 -- 172.21.15.86:0/2961760978 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df0016030 con 0x7f8e04153840 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.256+0000 7f8e01600640 1 -- 172.21.15.86:0/2961760978 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8df0007020 con 0x7f8e04153840 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.256+0000 7f8e01600640 1 -- 172.21.15.86:0/2961760978 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df0007410 con 0x7f8e04153840 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.256+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e04151d00 con 0x7f8e04153840 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.256+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e04151620 con 0x7f8e04153840 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.257+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e0404f330 con 0x7f8e04153840 2024-03-06T07:28:15.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.258+0000 7f8e01600640 1 -- 172.21.15.86:0/2961760978 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8df0019070 con 0x7f8e04153840 2024-03-06T07:28:15.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.259+0000 7f8e01600640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de406b4d0 0x7f8de406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:15.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.259+0000 7f8e01600640 1 -- 172.21.15.86:0/2961760978 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8df0089f00 con 0x7f8e04153840 2024-03-06T07:28:15.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.259+0000 7f8e09c00640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de406b4d0 0x7f8de406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:15.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.260+0000 7f8e09c00640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de406b4d0 0x7f8de406d990 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f8df8036000 tx=0x7f8df8030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:15.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.263+0000 7f8e01600640 1 -- 172.21.15.86:0/2961760978 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8df0058ce0 con 0x7f8e04153840 2024-03-06T07:28:15.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:15.508+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f8e0406c990 con 0x7f8e04153840 2024-03-06T07:28:16.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.193+0000 7f8e01600640 1 -- 172.21.15.86:0/2961760978 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v16) v1 ==== 118+0+0 (secure 0 0 0) 0x7f8df00585e0 con 0x7f8e04153840 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.196+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de406b4d0 msgr2=0x7f8de406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.196+0000 7f8e0c600640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de406b4d0 0x7f8de406d990 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f8df8036000 tx=0x7f8df8030000 comp rx=0 tx=0).stop 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.196+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e04153840 msgr2=0x7f8e041503b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.196+0000 7f8e0c600640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e04153840 0x7f8e041503b0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f8df000d3b0 tx=0x7f8df000d870 comp rx=0 tx=0).stop 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 shutdown_connections 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8de406b4d0 0x7f8de406d990 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e040ce810 0x7f8e040cec90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e04153840 0x7f8e041503b0 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 --2- 172.21.15.86:0/2961760978 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8dec0039b0 0x7f8e041f8400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 >> 172.21.15.86:0/2961760978 conn(0x7f8e040b1940 msgr2=0x7f8e040b0810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:16.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 shutdown_connections 2024-03-06T07:28:16.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.197+0000 7f8e0c600640 1 -- 172.21.15.86:0/2961760978 wait complete. 2024-03-06T07:28:16.221 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:28:16.221 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:28:16.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.434+0000 7fd10f600640 1 Processor -- start 2024-03-06T07:28:16.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.434+0000 7fd10f600640 1 -- start start 2024-03-06T07:28:16.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.436+0000 7fd10f600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 0x7fd108070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.436+0000 7fd10f600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1080b46c0 con 0x7fd10815fff0 2024-03-06T07:28:16.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.436+0000 7fd10f600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1080b49a0 con 0x7fd1080b3f70 2024-03-06T07:28:16.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.436+0000 7fd10f600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd108072c90 con 0x7fd10815fc10 2024-03-06T07:28:16.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.436+0000 7fd0ffe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd10815fff0 0x7fd108160450 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49982/0 (socket says 172.21.15.86:49982) 2024-03-06T07:28:16.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.436+0000 7fd0ffe00640 1 -- 172.21.15.86:0/3765690603 learned_addr learned my addr 172.21.15.86:0/3765690603 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd10cc00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 0x7fd108070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1273209926 0 0) 0x7fd1080b46c0 con 0x7fd10815fff0 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd0ec003520 con 0x7fd10815fff0 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1676846415 0 0) 0x7fd0ec003520 con 0x7fd10815fff0 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 msgr2=0x7fd108070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 0x7fd108070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 >> v1:172.21.15.194:6789/0 conn(0x7fd1080b3f70 legacy=0x7fd1080ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:16.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd10809dd80 con 0x7fd10815fff0 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd0f0001670 con 0x7fd10815fff0 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 >> v1:172.21.15.86:6789/0 conn(0x7fd10815fff0 legacy=0x7fd108160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec0039e0 0x7fd0ec003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0ec004800 0x7fd0ec004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 0x7fd0ec0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd108072c90 con 0x7fd0ec004800 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0ec00ba70 con 0x7fd0ec0039e0 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.437+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0ec00bc70 con 0x7fd10815fc10 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd0ffe00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0ec004800 0x7fd0ec004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd10cc00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec0039e0 0x7fd0ec003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd10d600640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 0x7fd0ec0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd0ffe00640 1 -- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 msgr2=0x7fd0ec0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd0ffe00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10815fc10 0x7fd0ec0051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd0ffe00640 1 -- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec0039e0 msgr2=0x7fd0ec003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd0ffe00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec0039e0 0x7fd0ec003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd0ffe00640 1 -- 172.21.15.86:0/3765690603 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0ec003520 con 0x7fd0ec004800 2024-03-06T07:28:16.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.438+0000 7fd10cc00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec0039e0 0x7fd0ec003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.439+0000 7fd0ffe00640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0ec004800 0x7fd0ec004c80 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fd0f0001eb0 tx=0x7fd0ec005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf02862ba1666fe7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.439+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0f0007d30 con 0x7fd0ec004800 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd0f0030070 con 0x7fd0ec004800 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd107e00640 1 -- 172.21.15.86:0/3765690603 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0f0005d30 con 0x7fd0ec004800 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd10f600640 1 -- 172.21.15.86:0/3765690603 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0ec004800 msgr2=0x7fd0ec004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd10f600640 1 --2- 172.21.15.86:0/3765690603 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0ec004800 0x7fd0ec004c80 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fd0f0001eb0 tx=0x7fd0ec005830 comp rx=0 tx=0).stop 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd10d600640 1 -- 172.21.15.86:0/3765690603 reap_dead start 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd10f600640 1 -- 172.21.15.86:0/3765690603 shutdown_connections 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd10f600640 1 -- 172.21.15.86:0/3765690603 >> 172.21.15.86:0/3765690603 conn(0x7fd1080b1940 msgr2=0x7fd1080b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd10f600640 1 -- 172.21.15.86:0/3765690603 shutdown_connections 2024-03-06T07:28:16.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.440+0000 7fd10f600640 1 -- 172.21.15.86:0/3765690603 wait complete. 2024-03-06T07:28:16.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.442+0000 7fd10f600640 1 Processor -- start 2024-03-06T07:28:16.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.442+0000 7fd10f600640 1 -- start start 2024-03-06T07:28:16.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd10f600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec004800 0x7fd10809cdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd10f600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd10814d410 0x7fd10806d590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd10f600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10806dad0 0x7fd1081f4480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd10f600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd10815c460 con 0x7fd10814d410 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd10f600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd10815c2e0 con 0x7fd0ec004800 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd10f600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd108070ff0 con 0x7fd10806dad0 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd0ffe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd10814d410 0x7fd10806d590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd0ffe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd10814d410 0x7fd10806d590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60658/0 (socket says 172.21.15.86:60658) 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.443+0000 7fd0ffe00640 1 -- 172.21.15.86:0/214336033 learned_addr learned my addr 172.21.15.86:0/214336033 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd10cc00640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec004800 0x7fd10809cdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd10d600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10806dad0 0x7fd1081f4480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd0ffe00640 1 -- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10806dad0 msgr2=0x7fd1081f4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd0ffe00640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10806dad0 0x7fd1081f4480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd0ffe00640 1 -- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec004800 msgr2=0x7fd10809cdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd0ffe00640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec004800 0x7fd10809cdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd0ffe00640 1 -- 172.21.15.86:0/214336033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd10809ba40 con 0x7fd10814d410 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd10d600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10806dad0 0x7fd1081f4480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:16.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd0ffe00640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd10814d410 0x7fd10806d590 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fd0f000c300 tx=0x7fd0f0001a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:16.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd105600640 1 -- 172.21.15.86:0/214336033 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0f0045070 con 0x7fd10814d410 2024-03-06T07:28:16.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd105600640 1 -- 172.21.15.86:0/214336033 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd0f0000ed0 con 0x7fd10814d410 2024-03-06T07:28:16.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd10809bd20 con 0x7fd10814d410 2024-03-06T07:28:16.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.444+0000 7fd105600640 1 -- 172.21.15.86:0/214336033 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0f0004df0 con 0x7fd10814d410 2024-03-06T07:28:16.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.445+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd10809c2e0 con 0x7fd10814d410 2024-03-06T07:28:16.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.446+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd10804f330 con 0x7fd10814d410 2024-03-06T07:28:16.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.447+0000 7fd105600640 1 -- 172.21.15.86:0/214336033 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd0f0007070 con 0x7fd10814d410 2024-03-06T07:28:16.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.448+0000 7fd105600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0e406b4d0 0x7fd0e406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:16.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.448+0000 7fd105600640 1 -- 172.21.15.86:0/214336033 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd0f00ab4a0 con 0x7fd10814d410 2024-03-06T07:28:16.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.448+0000 7fd10cc00640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0e406b4d0 0x7fd0e406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:16.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.449+0000 7fd10cc00640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0e406b4d0 0x7fd0e406d990 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fd1081f8b10 tx=0x7fd0f8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:16.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.452+0000 7fd105600640 1 -- 172.21.15.86:0/214336033 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd0f007a400 con 0x7fd10814d410 2024-03-06T07:28:16.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:16.687+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7fd10806ef30 con 0x7fd10814d410 2024-03-06T07:28:17.117 INFO:tasks.ceph.mds.d.smithi194.stderr:2024-03-06T07:28:17.115+0000 7fa2de600640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:28:17.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.211+0000 7fd105600640 1 -- 172.21.15.86:0/214336033 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v17) v1 ==== 122+0+0 (secure 0 0 0) 0x7fd0f0079d00 con 0x7fd10814d410 2024-03-06T07:28:17.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0e406b4d0 msgr2=0x7fd0e406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0e406b4d0 0x7fd0e406d990 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fd1081f8b10 tx=0x7fd0f8008040 comp rx=0 tx=0).stop 2024-03-06T07:28:17.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd10814d410 msgr2=0x7fd10806d590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd10814d410 0x7fd10806d590 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fd0f000c300 tx=0x7fd0f0001a40 comp rx=0 tx=0).stop 2024-03-06T07:28:17.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 shutdown_connections 2024-03-06T07:28:17.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0e406b4d0 0x7fd0e406d990 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd10806dad0 0x7fd1081f4480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd10814d410 0x7fd10806d590 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 --2- 172.21.15.86:0/214336033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0ec004800 0x7fd10809cdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 >> 172.21.15.86:0/214336033 conn(0x7fd1080b1940 msgr2=0x7fd1080ae200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:17.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 shutdown_connections 2024-03-06T07:28:17.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.216+0000 7fd10f600640 1 -- 172.21.15.86:0/214336033 wait complete. 2024-03-06T07:28:17.241 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:28:17.241 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:17.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.450+0000 7f04b2a00640 1 Processor -- start 2024-03-06T07:28:17.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.450+0000 7f04b2a00640 1 -- start start 2024-03-06T07:28:17.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04b2a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f04ac144c00 0x7f04ac06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04b2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f04ac15f9d0 0x7f04ac15fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04b2a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04ac06ee70 con 0x7f04ac0ce0a0 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04b2a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04ac06cef0 con 0x7f04ac15f9d0 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04b2a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04ac06d1d0 con 0x7f04ac144c00 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04b0a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f04ac0ce0a0 0x7f04ac0ce4b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49994/0 (socket says 172.21.15.86:49994) 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04b0a00640 1 -- 172.21.15.86:0/2132645410 learned_addr learned my addr 172.21.15.86:0/2132645410 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.452+0000 7f04ab400640 1 --2- 172.21.15.86:0/2132645410 >> v2:172.21.15.86:3300/0 conn(0x7f04ac15f9d0 0x7f04ac15fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.453+0000 7f04abe00640 1 --2- 172.21.15.86:0/2132645410 >> v2:172.21.15.194:3300/0 conn(0x7f04ac144c00 0x7f04ac06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.453+0000 7f04aaa00640 1 -- 172.21.15.86:0/2132645410 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1062301767 0 0) 0x7f04a4001490 con 0x7f04ac0ce0a0 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.453+0000 7f04ab400640 1 -- 172.21.15.86:0/2132645410 >> v2:172.21.15.194:3300/0 conn(0x7f04ac144c00 msgr2=0x7f04ac06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.453+0000 7f04ab400640 1 --2- 172.21.15.86:0/2132645410 >> v2:172.21.15.194:3300/0 conn(0x7f04ac144c00 0x7f04ac06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.453+0000 7f04ab400640 1 -- 172.21.15.86:0/2132645410 >> v1:172.21.15.86:6789/0 conn(0x7f04ac0ce0a0 legacy=0x7f04ac0ce4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.453+0000 7f04ab400640 1 -- 172.21.15.86:0/2132645410 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04ac06d4b0 con 0x7f04ac15f9d0 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.454+0000 7f04ab400640 1 --2- 172.21.15.86:0/2132645410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac15fe60 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f04ac0b25f0 tx=0x7f049802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6dd6732dfcef8e35 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.454+0000 7f04aaa00640 1 -- 172.21.15.86:0/2132645410 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0498035020 con 0x7f04ac15f9d0 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.454+0000 7f04aaa00640 1 -- 172.21.15.86:0/2132645410 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0498031830 con 0x7f04ac15f9d0 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.454+0000 7f04aaa00640 1 -- 172.21.15.86:0/2132645410 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0498031c00 con 0x7f04ac15f9d0 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 -- 172.21.15.86:0/2132645410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 msgr2=0x7f04ac15fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 --2- 172.21.15.86:0/2132645410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac15fe60 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f04ac0b25f0 tx=0x7f049802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 -- 172.21.15.86:0/2132645410 shutdown_connections 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 --2- 172.21.15.86:0/2132645410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac15fe60 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 --2- 172.21.15.86:0/2132645410 >> v2:172.21.15.194:3300/0 conn(0x7f04ac144c00 0x7f04ac06e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 -- 172.21.15.86:0/2132645410 >> 172.21.15.86:0/2132645410 conn(0x7f04ac0b1730 msgr2=0x7f04ac0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 -- 172.21.15.86:0/2132645410 shutdown_connections 2024-03-06T07:28:17.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.455+0000 7f04b2a00640 1 -- 172.21.15.86:0/2132645410 wait complete. 2024-03-06T07:28:17.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.457+0000 7f04b2a00640 1 Processor -- start 2024-03-06T07:28:17.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.457+0000 7f04b2a00640 1 -- start start 2024-03-06T07:28:17.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b2a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04ac0ce0a0 0x7f04ac15a800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b2a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04ac144c00 0x7f04ac13ef00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac139040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b2a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04ac06cb50 con 0x7f04ac15f9d0 2024-03-06T07:28:17.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b2a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04ac160730 con 0x7f04ac0ce0a0 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b2a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04ac06ccd0 con 0x7f04ac144c00 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b0a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac139040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b0a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac139040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60662/0 (socket says 172.21.15.86:60662) 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04b0a00640 1 -- 172.21.15.86:0/4045843669 learned_addr learned my addr 172.21.15.86:0/4045843669 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04ab400640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04ac144c00 0x7f04ac13ef00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.458+0000 7f04abe00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04ac0ce0a0 0x7f04ac15a800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b0a00640 1 -- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04ac144c00 msgr2=0x7f04ac13ef00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b0a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04ac144c00 0x7f04ac13ef00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b0a00640 1 -- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04ac0ce0a0 msgr2=0x7f04ac15a800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b0a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04ac0ce0a0 0x7f04ac15a800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b0a00640 1 -- 172.21.15.86:0/4045843669 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04ac13f8e0 con 0x7f04ac15f9d0 2024-03-06T07:28:17.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b0a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac139040 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f04a400c700 tx=0x7f04a400cbc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:17.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04a3e00640 1 -- 172.21.15.86:0/4045843669 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04a4018070 con 0x7f04ac15f9d0 2024-03-06T07:28:17.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04a3e00640 1 -- 172.21.15.86:0/4045843669 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f04a400ce10 con 0x7f04ac15f9d0 2024-03-06T07:28:17.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04a3e00640 1 -- 172.21.15.86:0/4045843669 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04a400a860 con 0x7f04ac15f9d0 2024-03-06T07:28:17.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04ac13fbc0 con 0x7f04ac15f9d0 2024-03-06T07:28:17.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.459+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04ac140180 con 0x7f04ac15f9d0 2024-03-06T07:28:17.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.460+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f04ac04f330 con 0x7f04ac15f9d0 2024-03-06T07:28:17.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.462+0000 7f04a3e00640 1 -- 172.21.15.86:0/4045843669 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f04a400aa00 con 0x7f04ac15f9d0 2024-03-06T07:28:17.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.463+0000 7f04a3e00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f048c06b4d0 0x7f048c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.463+0000 7f04a3e00640 1 -- 172.21.15.86:0/4045843669 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f04a40897f0 con 0x7f04ac15f9d0 2024-03-06T07:28:17.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.463+0000 7f04abe00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f048c06b4d0 0x7f048c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.464+0000 7f04abe00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f048c06b4d0 0x7f048c06d990 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f049c006fd0 tx=0x7f049c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:17.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.467+0000 7f04a3e00640 1 -- 172.21.15.86:0/4045843669 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f04ac06ee70 con 0x7f04ac15f9d0 2024-03-06T07:28:17.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.701+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f04ac0b1f20 con 0x7f04ac15f9d0 2024-03-06T07:28:17.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.702+0000 7f04a3e00640 1 -- 172.21.15.86:0/4045843669 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f04a4057f80 con 0x7f04ac15f9d0 2024-03-06T07:28:17.704 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:17.704 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:17.704 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:17.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.704+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f048c06b4d0 msgr2=0x7f048c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.704+0000 7f04b2a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f048c06b4d0 0x7f048c06d990 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f049c006fd0 tx=0x7f049c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:17.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.704+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 msgr2=0x7f04ac139040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.704+0000 7f04b2a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac139040 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f04a400c700 tx=0x7f04a400cbc0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 shutdown_connections 2024-03-06T07:28:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f048c06b4d0 0x7f048c06d990 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f04ac15f9d0 0x7f04ac139040 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f04ac144c00 0x7f04ac13ef00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 --2- 172.21.15.86:0/4045843669 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f04ac0ce0a0 0x7f04ac15a800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 >> 172.21.15.86:0/4045843669 conn(0x7f04ac0b1730 msgr2=0x7f04ac1434e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 shutdown_connections 2024-03-06T07:28:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.705+0000 7f04b2a00640 1 -- 172.21.15.86:0/4045843669 wait complete. 2024-03-06T07:28:17.726 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:17.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.932+0000 7f35dd000640 1 Processor -- start 2024-03-06T07:28:17.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.932+0000 7f35dd000640 1 -- start start 2024-03-06T07:28:17.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.933+0000 7f35dd000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f35d80af1c0 0x7f35d815eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.933+0000 7f35dd000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d8070d20 0x7f35d806cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.933+0000 7f35dd000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35d806d1a0 0x7f35d806e4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.933+0000 7f35dd000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35d8071d30 con 0x7f35d806d1a0 2024-03-06T07:28:17.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.933+0000 7f35dd000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35d8071490 con 0x7f35d80af1c0 2024-03-06T07:28:17.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.933+0000 7f35dd000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35d8071770 con 0x7f35d8070d20 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35d806d1a0 0x7f35d806e4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35d806d1a0 0x7f35d806e4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60674/0 (socket says 172.21.15.86:60674) 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 -- 172.21.15.86:0/4201034006 learned_addr learned my addr 172.21.15.86:0/4201034006 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35cfe00640 1 --2- 172.21.15.86:0/4201034006 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d8070d20 0x7f35d806cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d5e00640 1 --2- 172.21.15.86:0/4201034006 >> v2:172.21.15.194:3300/0 conn(0x7f35d80af1c0 0x7f35d815eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 -- 172.21.15.86:0/4201034006 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d8070d20 msgr2=0x7f35d806cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 --2- 172.21.15.86:0/4201034006 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d8070d20 0x7f35d806cc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 -- 172.21.15.86:0/4201034006 >> v2:172.21.15.194:3300/0 conn(0x7f35d80af1c0 msgr2=0x7f35d815eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 --2- 172.21.15.86:0/4201034006 >> v2:172.21.15.194:3300/0 conn(0x7f35d80af1c0 0x7f35d815eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.934+0000 7f35d6800640 1 -- 172.21.15.86:0/4201034006 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35d8071a50 con 0x7f35d806d1a0 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.936+0000 7f35d6800640 1 --2- 172.21.15.86:0/4201034006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d806e4d0 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f35d80d0ac0 tx=0x7f35d002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=82904bb11473c5a2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.936+0000 7f35d5400640 1 -- 172.21.15.86:0/4201034006 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35d002ca80 con 0x7f35d806d1a0 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.936+0000 7f35d5400640 1 -- 172.21.15.86:0/4201034006 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35d002cc20 con 0x7f35d806d1a0 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.936+0000 7f35d5400640 1 -- 172.21.15.86:0/4201034006 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35d00343f0 con 0x7f35d806d1a0 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 -- 172.21.15.86:0/4201034006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 msgr2=0x7f35d806e4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 --2- 172.21.15.86:0/4201034006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d806e4d0 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f35d80d0ac0 tx=0x7f35d002bee0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 -- 172.21.15.86:0/4201034006 shutdown_connections 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 --2- 172.21.15.86:0/4201034006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d806e4d0 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 --2- 172.21.15.86:0/4201034006 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d8070d20 0x7f35d806cc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 --2- 172.21.15.86:0/4201034006 >> v2:172.21.15.194:3300/0 conn(0x7f35d80af1c0 0x7f35d815eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 -- 172.21.15.86:0/4201034006 >> 172.21.15.86:0/4201034006 conn(0x7f35d80b1730 msgr2=0x7f35d80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 -- 172.21.15.86:0/4201034006 shutdown_connections 2024-03-06T07:28:17.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.937+0000 7f35dd000640 1 -- 172.21.15.86:0/4201034006 wait complete. 2024-03-06T07:28:17.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.938+0000 7f35dd000640 1 Processor -- start 2024-03-06T07:28:17.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.939+0000 7f35dd000640 1 -- start start 2024-03-06T07:28:17.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.939+0000 7f35dd000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d815efb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.939+0000 7f35dd000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d8070d20 0x7f35d81e8fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.939+0000 7f35dd000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d80af1c0 0x7f35d8148f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.939+0000 7f35dd000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35d806ea10 con 0x7f35d806d1a0 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.939+0000 7f35dd000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35d80711a0 con 0x7f35d8070d20 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.939+0000 7f35dd000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35d806eb90 con 0x7f35d80af1c0 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d815efb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d815efb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60686/0 (socket says 172.21.15.86:60686) 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 -- 172.21.15.86:0/3459764900 learned_addr learned my addr 172.21.15.86:0/3459764900 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35cfe00640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d8070d20 0x7f35d81e8fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d6800640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d80af1c0 0x7f35d8148f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 -- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d80af1c0 msgr2=0x7f35d8148f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d80af1c0 0x7f35d8148f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 -- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d8070d20 msgr2=0x7f35d81e8fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:17.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d8070d20 0x7f35d81e8fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 -- 172.21.15.86:0/3459764900 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35d815f900 con 0x7f35d806d1a0 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35cfe00640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d8070d20 0x7f35d81e8fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.940+0000 7f35d5e00640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d815efb0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f35c800d620 tx=0x7f35c800dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.941+0000 7f35ce000640 1 -- 172.21.15.86:0/3459764900 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35c800b620 con 0x7f35d806d1a0 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.941+0000 7f35ce000640 1 -- 172.21.15.86:0/3459764900 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35c8019070 con 0x7f35d806d1a0 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.941+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35d815fbe0 con 0x7f35d806d1a0 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.941+0000 7f35ce000640 1 -- 172.21.15.86:0/3459764900 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35c800ba00 con 0x7f35d806d1a0 2024-03-06T07:28:17.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.941+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35d8147150 con 0x7f35d806d1a0 2024-03-06T07:28:17.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.942+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f35d804f330 con 0x7f35d806d1a0 2024-03-06T07:28:17.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.943+0000 7f35ce000640 1 -- 172.21.15.86:0/3459764900 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f35c800bba0 con 0x7f35d806d1a0 2024-03-06T07:28:17.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.944+0000 7f35ce000640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:17.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.944+0000 7f35ce000640 1 -- 172.21.15.86:0/3459764900 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f35c808a580 con 0x7f35d806d1a0 2024-03-06T07:28:17.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.944+0000 7f35cfe00640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:17.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.945+0000 7f35cfe00640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f35d80b3e60 tx=0x7f35c0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:17.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:17.949+0000 7f35ce000640 1 -- 172.21.15.86:0/3459764900 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f35c8059400 con 0x7f35d806d1a0 2024-03-06T07:28:18.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.145+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f35d815b900 con 0x7f35d806d1a0 2024-03-06T07:28:18.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.148+0000 7f35ce000640 1 -- 172.21.15.86:0/3459764900 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v35) v1 ==== 74+0+16681 (secure 0 0 0) 0x7f35c8058d00 con 0x7f35d806d1a0 2024-03-06T07:28:18.150 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:18.150 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":35,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:15.157270+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:11.444335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:12.623953+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"35","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:18.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.150+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 msgr2=0x7f35b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.150+0000 7f35dd000640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f35d80b3e60 tx=0x7f35c0008040 comp rx=0 tx=0).stop 2024-03-06T07:28:18.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.150+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 msgr2=0x7f35d815efb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.150+0000 7f35dd000640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d815efb0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f35c800d620 tx=0x7f35c800dae0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 shutdown_connections 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d80af1c0 0x7f35d8148f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d8070d20 0x7f35d81e8fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 --2- 172.21.15.86:0/3459764900 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d806d1a0 0x7f35d815efb0 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 >> 172.21.15.86:0/3459764900 conn(0x7f35d80b1730 msgr2=0x7f35d80af5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 shutdown_connections 2024-03-06T07:28:18.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.151+0000 7f35dd000640 1 -- 172.21.15.86:0/3459764900 wait complete. 2024-03-06T07:28:18.169 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:28:18.169 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:28:18.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.364+0000 7feb66e00640 1 Processor -- start 2024-03-06T07:28:18.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.365+0000 7feb66e00640 1 -- start start 2024-03-06T07:28:18.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.366+0000 7feb66e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb6015acb0 0x7feb600b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.366+0000 7feb66e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7feb600b4680 0x7feb6006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.366+0000 7feb66e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feb6015b090 con 0x7feb60072340 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.366+0000 7feb66e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feb600b2a30 con 0x7feb600b4680 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.366+0000 7feb66e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feb600b4360 con 0x7feb6015acb0 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb57400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7feb600b4680 0x7feb6006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb57400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7feb600b4680 0x7feb6006e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60692/0 (socket says 172.21.15.86:60692) 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb57400640 1 -- 172.21.15.86:0/1138808639 learned_addr learned my addr 172.21.15.86:0/1138808639 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb5fe00640 1 --2- 172.21.15.86:0/1138808639 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb6015acb0 0x7feb600b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb57400640 1 -- 172.21.15.86:0/1138808639 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb6015acb0 msgr2=0x7feb600b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb57400640 1 --2- 172.21.15.86:0/1138808639 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb6015acb0 0x7feb600b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb57400640 1 -- 172.21.15.86:0/1138808639 >> v1:172.21.15.194:6789/0 conn(0x7feb60072340 legacy=0x7feb600ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:18.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.367+0000 7feb57400640 1 -- 172.21.15.86:0/1138808639 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feb600b2cf0 con 0x7feb600b4680 2024-03-06T07:28:18.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.368+0000 7feb57400640 1 --2- 172.21.15.86:0/1138808639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb600b4680 0x7feb6006e4f0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7feb48009870 tx=0x7feb4802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e257026cf367f32a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:18.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.368+0000 7feb5f400640 1 -- 172.21.15.86:0/1138808639 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb48035020 con 0x7feb600b4680 2024-03-06T07:28:18.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.368+0000 7feb5f400640 1 -- 172.21.15.86:0/1138808639 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feb48031640 con 0x7feb600b4680 2024-03-06T07:28:18.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb5f400640 1 -- 172.21.15.86:0/1138808639 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb4802d040 con 0x7feb600b4680 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 -- 172.21.15.86:0/1138808639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb600b4680 msgr2=0x7feb6006e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 --2- 172.21.15.86:0/1138808639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb600b4680 0x7feb6006e4f0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7feb48009870 tx=0x7feb4802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 -- 172.21.15.86:0/1138808639 shutdown_connections 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 --2- 172.21.15.86:0/1138808639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb600b4680 0x7feb6006e4f0 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 --2- 172.21.15.86:0/1138808639 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb6015acb0 0x7feb600b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 -- 172.21.15.86:0/1138808639 >> 172.21.15.86:0/1138808639 conn(0x7feb600b1730 msgr2=0x7feb600b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 -- 172.21.15.86:0/1138808639 shutdown_connections 2024-03-06T07:28:18.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.369+0000 7feb66e00640 1 -- 172.21.15.86:0/1138808639 wait complete. 2024-03-06T07:28:18.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.371+0000 7feb66e00640 1 Processor -- start 2024-03-06T07:28:18.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.371+0000 7feb66e00640 1 -- start start 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb66e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feb60072340 0x7feb6015b260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb66e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb600b4680 0x7feb6014e630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb66e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb6015acb0 0x7feb600cecc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb66e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feb600b3d10 con 0x7feb6015acb0 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb66e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feb600b32b0 con 0x7feb60072340 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb66e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feb600b4010 con 0x7feb600b4680 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb64e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb6015acb0 0x7feb600cecc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb64e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb6015acb0 0x7feb600cecc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60694/0 (socket says 172.21.15.86:60694) 2024-03-06T07:28:18.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.372+0000 7feb64e00640 1 -- 172.21.15.86:0/597910426 learned_addr learned my addr 172.21.15.86:0/597910426 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb5fe00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feb60072340 0x7feb6015b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb57400640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb600b4680 0x7feb6014e630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb64e00640 1 -- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb600b4680 msgr2=0x7feb6014e630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb64e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb600b4680 0x7feb6014e630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb64e00640 1 -- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feb60072340 msgr2=0x7feb6015b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb64e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feb60072340 0x7feb6015b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb64e00640 1 -- 172.21.15.86:0/597910426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feb6015b7a0 con 0x7feb6015acb0 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb5fe00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feb60072340 0x7feb6015b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb64e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb6015acb0 0x7feb600cecc0 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7feb5800b410 tx=0x7feb5800b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:18.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb5cc00640 1 -- 172.21.15.86:0/597910426 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb58015030 con 0x7feb6015acb0 2024-03-06T07:28:18.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb5cc00640 1 -- 172.21.15.86:0/597910426 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feb58004020 con 0x7feb6015acb0 2024-03-06T07:28:18.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb5cc00640 1 -- 172.21.15.86:0/597910426 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb58004410 con 0x7feb6015acb0 2024-03-06T07:28:18.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.373+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feb6014eb70 con 0x7feb6015acb0 2024-03-06T07:28:18.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.374+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feb600aebc0 con 0x7feb6015acb0 2024-03-06T07:28:18.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.375+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feb6006cba0 con 0x7feb6015acb0 2024-03-06T07:28:18.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.376+0000 7feb5cc00640 1 -- 172.21.15.86:0/597910426 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feb58018070 con 0x7feb6015acb0 2024-03-06T07:28:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.377+0000 7feb5cc00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feb3c06b4d0 0x7feb3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.377+0000 7feb5cc00640 1 -- 172.21.15.86:0/597910426 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7feb58089d20 con 0x7feb6015acb0 2024-03-06T07:28:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.377+0000 7feb5fe00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feb3c06b4d0 0x7feb3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.378+0000 7feb5fe00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feb3c06b4d0 0x7feb3c06d990 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7feb601f8a80 tx=0x7feb500074c0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:18.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.382+0000 7feb5cc00640 1 -- 172.21.15.86:0/597910426 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feb6006cba0 con 0x7feb6015acb0 2024-03-06T07:28:18.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.559+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7feb6004f330 con 0x7feb3c06b4d0 2024-03-06T07:28:18.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.570+0000 7feb5cc00640 1 -- 172.21.15.86:0/597910426 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251882 (secure 0 0 0) 0x7feb6004f330 con 0x7feb3c06b4d0 2024-03-06T07:28:18.575 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:18.576 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:28:18.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feb3c06b4d0 msgr2=0x7feb3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feb3c06b4d0 0x7feb3c06d990 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7feb601f8a80 tx=0x7feb500074c0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb6015acb0 msgr2=0x7feb600cecc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb6015acb0 0x7feb600cecc0 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7feb5800b410 tx=0x7feb5800b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 shutdown_connections 2024-03-06T07:28:18.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feb3c06b4d0 0x7feb3c06d990 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feb6015acb0 0x7feb600cecc0 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feb600b4680 0x7feb6014e630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 --2- 172.21.15.86:0/597910426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feb60072340 0x7feb6015b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 >> 172.21.15.86:0/597910426 conn(0x7feb600b1730 msgr2=0x7feb600af730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:18.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 shutdown_connections 2024-03-06T07:28:18.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.575+0000 7feb66e00640 1 -- 172.21.15.86:0/597910426 wait complete. 2024-03-06T07:28:18.593 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":67,"stamp":"2024-03-06T07:28:16.871737+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":103,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":220328,"kb_used_data":4156,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754754392,"statfs":{"total":773094113280,"available":772868497408,"internally_reserved":0,"allocated":4255744,"data_stored":2356330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":221236787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":32,"apply_latency_ms":32,"commit_latency_ns":32000000,"apply_latency_ns":32000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.731890"},"pg_stats":[{"pgid":"5.3f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200134+0000","last_change":"2024-03-06T07:28:14.191001+0000","last_active":"2024-03-06T07:28:14.200134+0000","last_peered":"2024-03-06T07:28:14.200134+0000","last_clean":"2024-03-06T07:28:14.200134+0000","last_became_active":"2024-03-06T07:28:14.189845+0000","last_became_peered":"2024-03-06T07:28:14.189845+0000","last_unstale":"2024-03-06T07:28:14.200134+0000","last_undegraded":"2024-03-06T07:28:14.200134+0000","last_fullsized":"2024-03-06T07:28:14.200134+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:15:19.053850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.3e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192888+0000","last_change":"2024-03-06T07:28:14.184519+0000","last_active":"2024-03-06T07:28:14.192888+0000","last_peered":"2024-03-06T07:28:14.192888+0000","last_clean":"2024-03-06T07:28:14.192888+0000","last_became_active":"2024-03-06T07:28:14.183689+0000","last_became_peered":"2024-03-06T07:28:14.183689+0000","last_unstale":"2024-03-06T07:28:14.192888+0000","last_undegraded":"2024-03-06T07:28:14.192888+0000","last_fullsized":"2024-03-06T07:28:14.192888+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:28.005358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203278+0000","last_change":"2024-03-06T07:28:14.198745+0000","last_active":"2024-03-06T07:28:14.203278+0000","last_peered":"2024-03-06T07:28:14.203278+0000","last_clean":"2024-03-06T07:28:14.203278+0000","last_became_active":"2024-03-06T07:28:14.198172+0000","last_became_peered":"2024-03-06T07:28:14.198172+0000","last_unstale":"2024-03-06T07:28:14.203278+0000","last_undegraded":"2024-03-06T07:28:14.203278+0000","last_fullsized":"2024-03-06T07:28:14.203278+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:02.619913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.3c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.195416+0000","last_change":"2024-03-06T07:28:14.188617+0000","last_active":"2024-03-06T07:28:14.195416+0000","last_peered":"2024-03-06T07:28:14.195416+0000","last_clean":"2024-03-06T07:28:14.195416+0000","last_became_active":"2024-03-06T07:28:14.184746+0000","last_became_peered":"2024-03-06T07:28:14.184746+0000","last_unstale":"2024-03-06T07:28:14.195416+0000","last_undegraded":"2024-03-06T07:28:14.195416+0000","last_fullsized":"2024-03-06T07:28:14.195416+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.197113+0000","last_change":"2024-03-06T07:28:14.194169+0000","last_active":"2024-03-06T07:28:14.197113+0000","last_peered":"2024-03-06T07:28:14.197113+0000","last_clean":"2024-03-06T07:28:14.197113+0000","last_became_active":"2024-03-06T07:28:14.193912+0000","last_became_peered":"2024-03-06T07:28:14.193912+0000","last_unstale":"2024-03-06T07:28:14.197113+0000","last_undegraded":"2024-03-06T07:28:14.197113+0000","last_fullsized":"2024-03-06T07:28:14.197113+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:40.876752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194501+0000","last_change":"2024-03-06T07:28:14.190905+0000","last_active":"2024-03-06T07:28:14.194501+0000","last_peered":"2024-03-06T07:28:14.194501+0000","last_clean":"2024-03-06T07:28:14.194501+0000","last_became_active":"2024-03-06T07:28:14.190627+0000","last_became_peered":"2024-03-06T07:28:14.190627+0000","last_unstale":"2024-03-06T07:28:14.194501+0000","last_undegraded":"2024-03-06T07:28:14.194501+0000","last_fullsized":"2024-03-06T07:28:14.194501+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:15.014804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.39","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205487+0000","last_change":"2024-03-06T07:28:14.203837+0000","last_active":"2024-03-06T07:28:14.205487+0000","last_peered":"2024-03-06T07:28:14.205487+0000","last_clean":"2024-03-06T07:28:14.205487+0000","last_became_active":"2024-03-06T07:28:14.203030+0000","last_became_peered":"2024-03-06T07:28:14.203030+0000","last_unstale":"2024-03-06T07:28:14.205487+0000","last_undegraded":"2024-03-06T07:28:14.205487+0000","last_fullsized":"2024-03-06T07:28:14.205487+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:20.764385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.38","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189466+0000","last_change":"2024-03-06T07:28:14.187580+0000","last_active":"2024-03-06T07:28:14.189466+0000","last_peered":"2024-03-06T07:28:14.189466+0000","last_clean":"2024-03-06T07:28:14.189466+0000","last_became_active":"2024-03-06T07:28:14.187267+0000","last_became_peered":"2024-03-06T07:28:14.187267+0000","last_unstale":"2024-03-06T07:28:14.189466+0000","last_undegraded":"2024-03-06T07:28:14.189466+0000","last_fullsized":"2024-03-06T07:28:14.189466+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.37","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189581+0000","last_change":"2024-03-06T07:28:14.181623+0000","last_active":"2024-03-06T07:28:14.189581+0000","last_peered":"2024-03-06T07:28:14.189581+0000","last_clean":"2024-03-06T07:28:14.189581+0000","last_became_active":"2024-03-06T07:28:14.180098+0000","last_became_peered":"2024-03-06T07:28:14.180098+0000","last_unstale":"2024-03-06T07:28:14.189581+0000","last_undegraded":"2024-03-06T07:28:14.189581+0000","last_fullsized":"2024-03-06T07:28:14.189581+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:22:31.310932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.36","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.198707+0000","last_change":"2024-03-06T07:28:14.196148+0000","last_active":"2024-03-06T07:28:14.198707+0000","last_peered":"2024-03-06T07:28:14.198707+0000","last_clean":"2024-03-06T07:28:14.198707+0000","last_became_active":"2024-03-06T07:28:14.190722+0000","last_became_peered":"2024-03-06T07:28:14.190722+0000","last_unstale":"2024-03-06T07:28:14.198707+0000","last_undegraded":"2024-03-06T07:28:14.198707+0000","last_fullsized":"2024-03-06T07:28:14.198707+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:50:57.453127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.35","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192828+0000","last_change":"2024-03-06T07:28:14.190124+0000","last_active":"2024-03-06T07:28:14.192828+0000","last_peered":"2024-03-06T07:28:14.192828+0000","last_clean":"2024-03-06T07:28:14.192828+0000","last_became_active":"2024-03-06T07:28:14.189749+0000","last_became_peered":"2024-03-06T07:28:14.189749+0000","last_unstale":"2024-03-06T07:28:14.192828+0000","last_undegraded":"2024-03-06T07:28:14.192828+0000","last_fullsized":"2024-03-06T07:28:14.192828+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.34","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203831+0000","last_change":"2024-03-06T07:28:14.195299+0000","last_active":"2024-03-06T07:28:14.203831+0000","last_peered":"2024-03-06T07:28:14.203831+0000","last_clean":"2024-03-06T07:28:14.203831+0000","last_became_active":"2024-03-06T07:28:14.195010+0000","last_became_peered":"2024-03-06T07:28:14.195010+0000","last_unstale":"2024-03-06T07:28:14.203831+0000","last_undegraded":"2024-03-06T07:28:14.203831+0000","last_fullsized":"2024-03-06T07:28:14.203831+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:59:40.916517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.33","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196044+0000","last_change":"2024-03-06T07:28:14.191112+0000","last_active":"2024-03-06T07:28:14.196044+0000","last_peered":"2024-03-06T07:28:14.196044+0000","last_clean":"2024-03-06T07:28:14.196044+0000","last_became_active":"2024-03-06T07:28:14.190515+0000","last_became_peered":"2024-03-06T07:28:14.190515+0000","last_unstale":"2024-03-06T07:28:14.196044+0000","last_undegraded":"2024-03-06T07:28:14.196044+0000","last_fullsized":"2024-03-06T07:28:14.196044+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:51:37.872171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.32","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194482+0000","last_change":"2024-03-06T07:28:14.189138+0000","last_active":"2024-03-06T07:28:14.194482+0000","last_peered":"2024-03-06T07:28:14.194482+0000","last_clean":"2024-03-06T07:28:14.194482+0000","last_became_active":"2024-03-06T07:28:14.188816+0000","last_became_peered":"2024-03-06T07:28:14.188816+0000","last_unstale":"2024-03-06T07:28:14.194482+0000","last_undegraded":"2024-03-06T07:28:14.194482+0000","last_fullsized":"2024-03-06T07:28:14.194482+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:52:20.286727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.31","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204409+0000","last_change":"2024-03-06T07:28:14.201952+0000","last_active":"2024-03-06T07:28:14.204409+0000","last_peered":"2024-03-06T07:28:14.204409+0000","last_clean":"2024-03-06T07:28:14.204409+0000","last_became_active":"2024-03-06T07:28:14.201594+0000","last_became_peered":"2024-03-06T07:28:14.201594+0000","last_unstale":"2024-03-06T07:28:14.204409+0000","last_undegraded":"2024-03-06T07:28:14.204409+0000","last_fullsized":"2024-03-06T07:28:14.204409+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:42:16.656971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.30","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200630+0000","last_change":"2024-03-06T07:28:14.197261+0000","last_active":"2024-03-06T07:28:14.200630+0000","last_peered":"2024-03-06T07:28:14.200630+0000","last_clean":"2024-03-06T07:28:14.200630+0000","last_became_active":"2024-03-06T07:28:14.197007+0000","last_became_peered":"2024-03-06T07:28:14.197007+0000","last_unstale":"2024-03-06T07:28:14.200630+0000","last_undegraded":"2024-03-06T07:28:14.200630+0000","last_fullsized":"2024-03-06T07:28:14.200630+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:16.521083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204930+0000","last_change":"2024-03-06T07:28:14.200979+0000","last_active":"2024-03-06T07:28:14.204930+0000","last_peered":"2024-03-06T07:28:14.204930+0000","last_clean":"2024-03-06T07:28:14.204930+0000","last_became_active":"2024-03-06T07:28:14.200679+0000","last_became_peered":"2024-03-06T07:28:14.200679+0000","last_unstale":"2024-03-06T07:28:14.204930+0000","last_undegraded":"2024-03-06T07:28:14.204930+0000","last_fullsized":"2024-03-06T07:28:14.204930+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:28.005358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205737+0000","last_change":"2024-03-06T07:28:14.198175+0000","last_active":"2024-03-06T07:28:14.205737+0000","last_peered":"2024-03-06T07:28:14.205737+0000","last_clean":"2024-03-06T07:28:14.205737+0000","last_became_active":"2024-03-06T07:28:14.197840+0000","last_became_peered":"2024-03-06T07:28:14.197840+0000","last_unstale":"2024-03-06T07:28:14.205737+0000","last_undegraded":"2024-03-06T07:28:14.205737+0000","last_fullsized":"2024-03-06T07:28:14.205737+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:55:01.352602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190339+0000","last_change":"2024-03-06T07:28:14.179175+0000","last_active":"2024-03-06T07:28:14.190339+0000","last_peered":"2024-03-06T07:28:14.190339+0000","last_clean":"2024-03-06T07:28:14.190339+0000","last_became_active":"2024-03-06T07:28:14.178900+0000","last_became_peered":"2024-03-06T07:28:14.178900+0000","last_unstale":"2024-03-06T07:28:14.190339+0000","last_undegraded":"2024-03-06T07:28:14.190339+0000","last_fullsized":"2024-03-06T07:28:14.190339+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:50:57.453127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194040+0000","last_change":"2024-03-06T07:28:14.190326+0000","last_active":"2024-03-06T07:28:14.194040+0000","last_peered":"2024-03-06T07:28:14.194040+0000","last_clean":"2024-03-06T07:28:14.194040+0000","last_became_active":"2024-03-06T07:28:14.189406+0000","last_became_peered":"2024-03-06T07:28:14.189406+0000","last_unstale":"2024-03-06T07:28:14.194040+0000","last_undegraded":"2024-03-06T07:28:14.194040+0000","last_fullsized":"2024-03-06T07:28:14.194040+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:52:16.900125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.2b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.173495+0000","last_change":"2024-03-06T07:28:14.169764+0000","last_active":"2024-03-06T07:28:14.173495+0000","last_peered":"2024-03-06T07:28:14.173495+0000","last_clean":"2024-03-06T07:28:14.173495+0000","last_became_active":"2024-03-06T07:28:14.169463+0000","last_became_peered":"2024-03-06T07:28:14.169463+0000","last_unstale":"2024-03-06T07:28:14.173495+0000","last_undegraded":"2024-03-06T07:28:14.173495+0000","last_fullsized":"2024-03-06T07:28:14.173495+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:06.673900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.2a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189055+0000","last_change":"2024-03-06T07:28:14.182211+0000","last_active":"2024-03-06T07:28:14.189055+0000","last_peered":"2024-03-06T07:28:14.189055+0000","last_clean":"2024-03-06T07:28:14.189055+0000","last_became_active":"2024-03-06T07:28:14.181920+0000","last_became_peered":"2024-03-06T07:28:14.181920+0000","last_unstale":"2024-03-06T07:28:14.189055+0000","last_undegraded":"2024-03-06T07:28:14.189055+0000","last_fullsized":"2024-03-06T07:28:14.189055+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:44.859980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.29","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.180863+0000","last_change":"2024-03-06T07:28:14.177839+0000","last_active":"2024-03-06T07:28:14.180863+0000","last_peered":"2024-03-06T07:28:14.180863+0000","last_clean":"2024-03-06T07:28:14.180863+0000","last_became_active":"2024-03-06T07:28:14.177447+0000","last_became_peered":"2024-03-06T07:28:14.177447+0000","last_unstale":"2024-03-06T07:28:14.180863+0000","last_undegraded":"2024-03-06T07:28:14.180863+0000","last_fullsized":"2024-03-06T07:28:14.180863+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.28","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.188947+0000","last_change":"2024-03-06T07:28:14.186095+0000","last_active":"2024-03-06T07:28:14.188947+0000","last_peered":"2024-03-06T07:28:14.188947+0000","last_clean":"2024-03-06T07:28:14.188947+0000","last_became_active":"2024-03-06T07:28:14.185801+0000","last_became_peered":"2024-03-06T07:28:14.185801+0000","last_unstale":"2024-03-06T07:28:14.188947+0000","last_undegraded":"2024-03-06T07:28:14.188947+0000","last_fullsized":"2024-03-06T07:28:14.188947+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:02:50.946267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.27","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196286+0000","last_change":"2024-03-06T07:28:14.181564+0000","last_active":"2024-03-06T07:28:14.196286+0000","last_peered":"2024-03-06T07:28:14.196286+0000","last_clean":"2024-03-06T07:28:14.196286+0000","last_became_active":"2024-03-06T07:28:14.181275+0000","last_became_peered":"2024-03-06T07:28:14.181275+0000","last_unstale":"2024-03-06T07:28:14.196286+0000","last_undegraded":"2024-03-06T07:28:14.196286+0000","last_fullsized":"2024-03-06T07:28:14.196286+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:44.859980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.26","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190613+0000","last_change":"2024-03-06T07:28:14.180935+0000","last_active":"2024-03-06T07:28:14.190613+0000","last_peered":"2024-03-06T07:28:14.190613+0000","last_clean":"2024-03-06T07:28:14.190613+0000","last_became_active":"2024-03-06T07:28:14.180676+0000","last_became_peered":"2024-03-06T07:28:14.180676+0000","last_unstale":"2024-03-06T07:28:14.190613+0000","last_undegraded":"2024-03-06T07:28:14.190613+0000","last_fullsized":"2024-03-06T07:28:14.190613+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:16.521083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.25","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196258+0000","last_change":"2024-03-06T07:28:14.192629+0000","last_active":"2024-03-06T07:28:14.196258+0000","last_peered":"2024-03-06T07:28:14.196258+0000","last_clean":"2024-03-06T07:28:14.196258+0000","last_became_active":"2024-03-06T07:28:14.192143+0000","last_became_peered":"2024-03-06T07:28:14.192143+0000","last_unstale":"2024-03-06T07:28:14.196258+0000","last_undegraded":"2024-03-06T07:28:14.196258+0000","last_fullsized":"2024-03-06T07:28:14.196258+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:35.371286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.24","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.188879+0000","last_change":"2024-03-06T07:28:14.185982+0000","last_active":"2024-03-06T07:28:14.188879+0000","last_peered":"2024-03-06T07:28:14.188879+0000","last_clean":"2024-03-06T07:28:14.188879+0000","last_became_active":"2024-03-06T07:28:14.185693+0000","last_became_peered":"2024-03-06T07:28:14.185693+0000","last_unstale":"2024-03-06T07:28:14.188879+0000","last_undegraded":"2024-03-06T07:28:14.188879+0000","last_fullsized":"2024-03-06T07:28:14.188879+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:25:19.031189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.23","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192566+0000","last_change":"2024-03-06T07:28:14.186863+0000","last_active":"2024-03-06T07:28:14.192566+0000","last_peered":"2024-03-06T07:28:14.192566+0000","last_clean":"2024-03-06T07:28:14.192566+0000","last_became_active":"2024-03-06T07:28:14.186663+0000","last_became_peered":"2024-03-06T07:28:14.186663+0000","last_unstale":"2024-03-06T07:28:14.192566+0000","last_undegraded":"2024-03-06T07:28:14.192566+0000","last_fullsized":"2024-03-06T07:28:14.192566+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.22","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.198013+0000","last_change":"2024-03-06T07:28:14.194770+0000","last_active":"2024-03-06T07:28:14.198013+0000","last_peered":"2024-03-06T07:28:14.198013+0000","last_clean":"2024-03-06T07:28:14.198013+0000","last_became_active":"2024-03-06T07:28:14.194428+0000","last_became_peered":"2024-03-06T07:28:14.194428+0000","last_unstale":"2024-03-06T07:28:14.198013+0000","last_undegraded":"2024-03-06T07:28:14.198013+0000","last_fullsized":"2024-03-06T07:28:14.198013+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:54.596749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.21","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.201186+0000","last_change":"2024-03-06T07:28:14.195293+0000","last_active":"2024-03-06T07:28:14.201186+0000","last_peered":"2024-03-06T07:28:14.201186+0000","last_clean":"2024-03-06T07:28:14.201186+0000","last_became_active":"2024-03-06T07:28:14.194851+0000","last_became_peered":"2024-03-06T07:28:14.194851+0000","last_unstale":"2024-03-06T07:28:14.201186+0000","last_undegraded":"2024-03-06T07:28:14.201186+0000","last_fullsized":"2024-03-06T07:28:14.201186+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:25:48.043980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.20","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196749+0000","last_change":"2024-03-06T07:28:14.192412+0000","last_active":"2024-03-06T07:28:14.196749+0000","last_peered":"2024-03-06T07:28:14.196749+0000","last_clean":"2024-03-06T07:28:14.196749+0000","last_became_active":"2024-03-06T07:28:14.192108+0000","last_became_peered":"2024-03-06T07:28:14.192108+0000","last_unstale":"2024-03-06T07:28:14.196749+0000","last_undegraded":"2024-03-06T07:28:14.196749+0000","last_fullsized":"2024-03-06T07:28:14.196749+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.183160+0000","last_change":"2024-03-06T07:28:14.178093+0000","last_active":"2024-03-06T07:28:14.183160+0000","last_peered":"2024-03-06T07:28:14.183160+0000","last_clean":"2024-03-06T07:28:14.183160+0000","last_became_active":"2024-03-06T07:28:14.177261+0000","last_became_peered":"2024-03-06T07:28:14.177261+0000","last_unstale":"2024-03-06T07:28:14.183160+0000","last_undegraded":"2024-03-06T07:28:14.183160+0000","last_fullsized":"2024-03-06T07:28:14.183160+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"34'4","reported_seq":16,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205516+0000","last_change":"2024-03-06T07:28:13.198209+0000","last_active":"2024-03-06T07:28:14.205516+0000","last_peered":"2024-03-06T07:28:14.205516+0000","last_clean":"2024-03-06T07:28:14.205516+0000","last_became_active":"2024-03-06T07:28:13.197761+0000","last_became_peered":"2024-03-06T07:28:13.197761+0000","last_unstale":"2024-03-06T07:28:14.205516+0000","last_undegraded":"2024-03-06T07:28:14.205516+0000","last_fullsized":"2024-03-06T07:28:14.205516+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:54:41.878583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.1f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.191144+0000","last_change":"2024-03-06T07:28:14.179839+0000","last_active":"2024-03-06T07:28:14.191144+0000","last_peered":"2024-03-06T07:28:14.191144+0000","last_clean":"2024-03-06T07:28:14.191144+0000","last_became_active":"2024-03-06T07:28:14.175864+0000","last_became_peered":"2024-03-06T07:28:14.175864+0000","last_unstale":"2024-03-06T07:28:14.191144+0000","last_undegraded":"2024-03-06T07:28:14.191144+0000","last_fullsized":"2024-03-06T07:28:14.191144+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:14:29.282408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1e","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203930+0000","last_change":"2024-03-06T07:28:13.186932+0000","last_active":"2024-03-06T07:28:14.203930+0000","last_peered":"2024-03-06T07:28:14.203930+0000","last_clean":"2024-03-06T07:28:14.203930+0000","last_became_active":"2024-03-06T07:28:13.185983+0000","last_became_peered":"2024-03-06T07:28:13.185983+0000","last_unstale":"2024-03-06T07:28:14.203930+0000","last_undegraded":"2024-03-06T07:28:14.203930+0000","last_fullsized":"2024-03-06T07:28:14.203930+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:33:14.195257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.188505+0000","last_change":"2024-03-06T07:28:14.185352+0000","last_active":"2024-03-06T07:28:14.188505+0000","last_peered":"2024-03-06T07:28:14.188505+0000","last_clean":"2024-03-06T07:28:14.188505+0000","last_became_active":"2024-03-06T07:28:14.184956+0000","last_became_peered":"2024-03-06T07:28:14.184956+0000","last_unstale":"2024-03-06T07:28:14.188505+0000","last_undegraded":"2024-03-06T07:28:14.188505+0000","last_fullsized":"2024-03-06T07:28:14.188505+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.211419+0000","last_change":"2024-03-06T07:28:13.186253+0000","last_active":"2024-03-06T07:28:14.211419+0000","last_peered":"2024-03-06T07:28:14.211419+0000","last_clean":"2024-03-06T07:28:14.211419+0000","last_became_active":"2024-03-06T07:28:13.185850+0000","last_became_peered":"2024-03-06T07:28:13.185850+0000","last_unstale":"2024-03-06T07:28:14.211419+0000","last_undegraded":"2024-03-06T07:28:14.211419+0000","last_fullsized":"2024-03-06T07:28:14.211419+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:24.344579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196232+0000","last_change":"2024-03-06T07:28:14.192387+0000","last_active":"2024-03-06T07:28:14.196232+0000","last_peered":"2024-03-06T07:28:14.196232+0000","last_clean":"2024-03-06T07:28:14.196232+0000","last_became_active":"2024-03-06T07:28:14.191766+0000","last_became_peered":"2024-03-06T07:28:14.191766+0000","last_unstale":"2024-03-06T07:28:14.196232+0000","last_undegraded":"2024-03-06T07:28:14.196232+0000","last_fullsized":"2024-03-06T07:28:14.196232+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206232+0000","last_change":"2024-03-06T07:28:13.184473+0000","last_active":"2024-03-06T07:28:14.206232+0000","last_peered":"2024-03-06T07:28:14.206232+0000","last_clean":"2024-03-06T07:28:14.206232+0000","last_became_active":"2024-03-06T07:28:13.182588+0000","last_became_peered":"2024-03-06T07:28:13.182588+0000","last_unstale":"2024-03-06T07:28:14.206232+0000","last_undegraded":"2024-03-06T07:28:14.206232+0000","last_fullsized":"2024-03-06T07:28:14.206232+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:28.980631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187980+0000","last_change":"2024-03-06T07:28:14.183508+0000","last_active":"2024-03-06T07:28:14.187980+0000","last_peered":"2024-03-06T07:28:14.187980+0000","last_clean":"2024-03-06T07:28:14.187980+0000","last_became_active":"2024-03-06T07:28:14.183208+0000","last_became_peered":"2024-03-06T07:28:14.183208+0000","last_unstale":"2024-03-06T07:28:14.187980+0000","last_undegraded":"2024-03-06T07:28:14.187980+0000","last_fullsized":"2024-03-06T07:28:14.187980+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:44.859980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206544+0000","last_change":"2024-03-06T07:28:13.185609+0000","last_active":"2024-03-06T07:28:14.206544+0000","last_peered":"2024-03-06T07:28:14.206544+0000","last_clean":"2024-03-06T07:28:14.206544+0000","last_became_active":"2024-03-06T07:28:13.185284+0000","last_became_peered":"2024-03-06T07:28:13.185284+0000","last_unstale":"2024-03-06T07:28:14.206544+0000","last_undegraded":"2024-03-06T07:28:14.206544+0000","last_fullsized":"2024-03-06T07:28:14.206544+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:35:52.290057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187619+0000","last_change":"2024-03-06T07:28:14.182280+0000","last_active":"2024-03-06T07:28:14.187619+0000","last_peered":"2024-03-06T07:28:14.187619+0000","last_clean":"2024-03-06T07:28:14.187619+0000","last_became_active":"2024-03-06T07:28:14.181917+0000","last_became_peered":"2024-03-06T07:28:14.181917+0000","last_unstale":"2024-03-06T07:28:14.187619+0000","last_undegraded":"2024-03-06T07:28:14.187619+0000","last_fullsized":"2024-03-06T07:28:14.187619+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:25:48.043980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205951+0000","last_change":"2024-03-06T07:28:13.182637+0000","last_active":"2024-03-06T07:28:14.205951+0000","last_peered":"2024-03-06T07:28:14.205951+0000","last_clean":"2024-03-06T07:28:14.205951+0000","last_became_active":"2024-03-06T07:28:13.181939+0000","last_became_peered":"2024-03-06T07:28:13.181939+0000","last_unstale":"2024-03-06T07:28:14.205951+0000","last_undegraded":"2024-03-06T07:28:14.205951+0000","last_fullsized":"2024-03-06T07:28:14.205951+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:51:21.126378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200180+0000","last_change":"2024-03-06T07:28:14.191492+0000","last_active":"2024-03-06T07:28:14.200180+0000","last_peered":"2024-03-06T07:28:14.200180+0000","last_clean":"2024-03-06T07:28:14.200180+0000","last_became_active":"2024-03-06T07:28:14.189261+0000","last_became_peered":"2024-03-06T07:28:14.189261+0000","last_unstale":"2024-03-06T07:28:14.200180+0000","last_undegraded":"2024-03-06T07:28:14.200180+0000","last_fullsized":"2024-03-06T07:28:14.200180+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:35.371286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.170463+0000","last_change":"2024-03-06T07:28:13.179224+0000","last_active":"2024-03-06T07:28:14.170463+0000","last_peered":"2024-03-06T07:28:14.170463+0000","last_clean":"2024-03-06T07:28:14.170463+0000","last_became_active":"2024-03-06T07:28:13.178961+0000","last_became_peered":"2024-03-06T07:28:13.178961+0000","last_unstale":"2024-03-06T07:28:14.170463+0000","last_undegraded":"2024-03-06T07:28:14.170463+0000","last_fullsized":"2024-03-06T07:28:14.170463+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:11.304727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190379+0000","last_change":"2024-03-06T07:28:14.187088+0000","last_active":"2024-03-06T07:28:14.190379+0000","last_peered":"2024-03-06T07:28:14.190379+0000","last_clean":"2024-03-06T07:28:14.190379+0000","last_became_active":"2024-03-06T07:28:14.186757+0000","last_became_peered":"2024-03-06T07:28:14.186757+0000","last_unstale":"2024-03-06T07:28:14.190379+0000","last_undegraded":"2024-03-06T07:28:14.190379+0000","last_fullsized":"2024-03-06T07:28:14.190379+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.166260+0000","last_change":"2024-03-06T07:28:13.190240+0000","last_active":"2024-03-06T07:28:14.166260+0000","last_peered":"2024-03-06T07:28:14.166260+0000","last_clean":"2024-03-06T07:28:14.166260+0000","last_became_active":"2024-03-06T07:28:13.189917+0000","last_became_peered":"2024-03-06T07:28:13.189917+0000","last_unstale":"2024-03-06T07:28:14.166260+0000","last_undegraded":"2024-03-06T07:28:14.166260+0000","last_fullsized":"2024-03-06T07:28:14.166260+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:45:03.516517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.16","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.207965+0000","last_change":"2024-03-06T07:28:14.203544+0000","last_active":"2024-03-06T07:28:14.207965+0000","last_peered":"2024-03-06T07:28:14.207965+0000","last_clean":"2024-03-06T07:28:14.207965+0000","last_became_active":"2024-03-06T07:28:14.203111+0000","last_became_peered":"2024-03-06T07:28:14.203111+0000","last_unstale":"2024-03-06T07:28:14.207965+0000","last_undegraded":"2024-03-06T07:28:14.207965+0000","last_fullsized":"2024-03-06T07:28:14.207965+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:42:06.295071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.171220+0000","last_change":"2024-03-06T07:28:13.191733+0000","last_active":"2024-03-06T07:28:14.171220+0000","last_peered":"2024-03-06T07:28:14.171220+0000","last_clean":"2024-03-06T07:28:14.171220+0000","last_became_active":"2024-03-06T07:28:13.191479+0000","last_became_peered":"2024-03-06T07:28:13.191479+0000","last_unstale":"2024-03-06T07:28:14.171220+0000","last_undegraded":"2024-03-06T07:28:14.171220+0000","last_fullsized":"2024-03-06T07:28:14.171220+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:57.071852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.17","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206263+0000","last_change":"2024-03-06T07:28:14.202604+0000","last_active":"2024-03-06T07:28:14.206263+0000","last_peered":"2024-03-06T07:28:14.206263+0000","last_clean":"2024-03-06T07:28:14.206263+0000","last_became_active":"2024-03-06T07:28:14.196754+0000","last_became_peered":"2024-03-06T07:28:14.196754+0000","last_unstale":"2024-03-06T07:28:14.206263+0000","last_undegraded":"2024-03-06T07:28:14.206263+0000","last_fullsized":"2024-03-06T07:28:14.206263+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:11:39.747997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.162265+0000","last_change":"2024-03-06T07:28:13.183746+0000","last_active":"2024-03-06T07:28:14.162265+0000","last_peered":"2024-03-06T07:28:14.162265+0000","last_clean":"2024-03-06T07:28:14.162265+0000","last_became_active":"2024-03-06T07:28:13.183227+0000","last_became_peered":"2024-03-06T07:28:13.183227+0000","last_unstale":"2024-03-06T07:28:14.162265+0000","last_undegraded":"2024-03-06T07:28:14.162265+0000","last_fullsized":"2024-03-06T07:28:14.162265+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:45:03.516517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.14","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190997+0000","last_change":"2024-03-06T07:28:14.185905+0000","last_active":"2024-03-06T07:28:14.190997+0000","last_peered":"2024-03-06T07:28:14.190997+0000","last_clean":"2024-03-06T07:28:14.190997+0000","last_became_active":"2024-03-06T07:28:14.183318+0000","last_became_peered":"2024-03-06T07:28:14.183318+0000","last_unstale":"2024-03-06T07:28:14.190997+0000","last_undegraded":"2024-03-06T07:28:14.190997+0000","last_fullsized":"2024-03-06T07:28:14.190997+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:20.764385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"34'3","reported_seq":15,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.210697+0000","last_change":"2024-03-06T07:28:13.195049+0000","last_active":"2024-03-06T07:28:14.210697+0000","last_peered":"2024-03-06T07:28:14.210697+0000","last_clean":"2024-03-06T07:28:14.210697+0000","last_became_active":"2024-03-06T07:28:13.194767+0000","last_became_peered":"2024-03-06T07:28:13.194767+0000","last_unstale":"2024-03-06T07:28:14.210697+0000","last_undegraded":"2024-03-06T07:28:14.210697+0000","last_fullsized":"2024-03-06T07:28:14.210697+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:15:18.036832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.15","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203292+0000","last_change":"2024-03-06T07:28:14.198814+0000","last_active":"2024-03-06T07:28:14.203292+0000","last_peered":"2024-03-06T07:28:14.203292+0000","last_clean":"2024-03-06T07:28:14.203292+0000","last_became_active":"2024-03-06T07:28:14.198475+0000","last_became_peered":"2024-03-06T07:28:14.198475+0000","last_unstale":"2024-03-06T07:28:14.203292+0000","last_undegraded":"2024-03-06T07:28:14.203292+0000","last_fullsized":"2024-03-06T07:28:14.203292+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:09.283825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"34'6","reported_seq":18,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.213867+0000","last_change":"2024-03-06T07:28:13.184116+0000","last_active":"2024-03-06T07:28:14.213867+0000","last_peered":"2024-03-06T07:28:14.213867+0000","last_clean":"2024-03-06T07:28:14.213867+0000","last_became_active":"2024-03-06T07:28:13.183617+0000","last_became_peered":"2024-03-06T07:28:13.183617+0000","last_unstale":"2024-03-06T07:28:14.213867+0000","last_undegraded":"2024-03-06T07:28:14.213867+0000","last_fullsized":"2024-03-06T07:28:14.213867+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:53.579731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.12","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.197741+0000","last_change":"2024-03-06T07:28:14.195034+0000","last_active":"2024-03-06T07:28:14.197741+0000","last_peered":"2024-03-06T07:28:14.197741+0000","last_clean":"2024-03-06T07:28:14.197741+0000","last_became_active":"2024-03-06T07:28:14.187691+0000","last_became_peered":"2024-03-06T07:28:14.187691+0000","last_unstale":"2024-03-06T07:28:14.197741+0000","last_undegraded":"2024-03-06T07:28:14.197741+0000","last_fullsized":"2024-03-06T07:28:14.197741+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:54.596749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.202415+0000","last_change":"2024-03-06T07:28:13.193486+0000","last_active":"2024-03-06T07:28:14.202415+0000","last_peered":"2024-03-06T07:28:14.202415+0000","last_clean":"2024-03-06T07:28:14.202415+0000","last_became_active":"2024-03-06T07:28:13.192611+0000","last_became_peered":"2024-03-06T07:28:13.192611+0000","last_unstale":"2024-03-06T07:28:14.202415+0000","last_undegraded":"2024-03-06T07:28:14.202415+0000","last_fullsized":"2024-03-06T07:28:14.202415+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:02:27.890526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194358+0000","last_change":"2024-03-06T07:28:14.186524+0000","last_active":"2024-03-06T07:28:14.194358+0000","last_peered":"2024-03-06T07:28:14.194358+0000","last_clean":"2024-03-06T07:28:14.194358+0000","last_became_active":"2024-03-06T07:28:14.185772+0000","last_became_peered":"2024-03-06T07:28:14.185772+0000","last_unstale":"2024-03-06T07:28:14.194358+0000","last_undegraded":"2024-03-06T07:28:14.194358+0000","last_fullsized":"2024-03-06T07:28:14.194358+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:26:33.379773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.169359+0000","last_change":"2024-03-06T07:28:13.190800+0000","last_active":"2024-03-06T07:28:14.169359+0000","last_peered":"2024-03-06T07:28:14.169359+0000","last_clean":"2024-03-06T07:28:14.169359+0000","last_became_active":"2024-03-06T07:28:13.190497+0000","last_became_peered":"2024-03-06T07:28:13.190497+0000","last_unstale":"2024-03-06T07:28:14.169359+0000","last_undegraded":"2024-03-06T07:28:14.169359+0000","last_fullsized":"2024-03-06T07:28:14.169359+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:33:14.195257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.10","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.197954+0000","last_change":"2024-03-06T07:28:14.192648+0000","last_active":"2024-03-06T07:28:14.197954+0000","last_peered":"2024-03-06T07:28:14.197954+0000","last_clean":"2024-03-06T07:28:14.197954+0000","last_became_active":"2024-03-06T07:28:14.192308+0000","last_became_peered":"2024-03-06T07:28:14.192308+0000","last_unstale":"2024-03-06T07:28:14.197954+0000","last_undegraded":"2024-03-06T07:28:14.197954+0000","last_fullsized":"2024-03-06T07:28:14.197954+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:35.371286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.173861+0000","last_change":"2024-03-06T07:28:13.192013+0000","last_active":"2024-03-06T07:28:14.173861+0000","last_peered":"2024-03-06T07:28:14.173861+0000","last_clean":"2024-03-06T07:28:14.173861+0000","last_became_active":"2024-03-06T07:28:13.191204+0000","last_became_peered":"2024-03-06T07:28:13.191204+0000","last_unstale":"2024-03-06T07:28:14.173861+0000","last_undegraded":"2024-03-06T07:28:14.173861+0000","last_fullsized":"2024-03-06T07:28:14.173861+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:34.354268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194666+0000","last_change":"2024-03-06T07:28:14.176017+0000","last_active":"2024-03-06T07:28:14.194666+0000","last_peered":"2024-03-06T07:28:14.194666+0000","last_clean":"2024-03-06T07:28:14.194666+0000","last_became_active":"2024-03-06T07:28:14.174689+0000","last_became_peered":"2024-03-06T07:28:14.174689+0000","last_unstale":"2024-03-06T07:28:14.194666+0000","last_undegraded":"2024-03-06T07:28:14.194666+0000","last_fullsized":"2024-03-06T07:28:14.194666+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:17.463217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"34'8","reported_seq":20,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.211353+0000","last_change":"2024-03-06T07:28:13.176218+0000","last_active":"2024-03-06T07:28:14.211353+0000","last_peered":"2024-03-06T07:28:14.211353+0000","last_clean":"2024-03-06T07:28:14.211353+0000","last_became_active":"2024-03-06T07:28:13.175859+0000","last_became_peered":"2024-03-06T07:28:13.175859+0000","last_unstale":"2024-03-06T07:28:14.211353+0000","last_undegraded":"2024-03-06T07:28:14.211353+0000","last_fullsized":"2024-03-06T07:28:14.211353+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:02.443572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.199068+0000","last_change":"2024-03-06T07:28:14.190503+0000","last_active":"2024-03-06T07:28:14.199068+0000","last_peered":"2024-03-06T07:28:14.199068+0000","last_clean":"2024-03-06T07:28:14.199068+0000","last_became_active":"2024-03-06T07:28:14.190215+0000","last_became_peered":"2024-03-06T07:28:14.190215+0000","last_unstale":"2024-03-06T07:28:14.199068+0000","last_undegraded":"2024-03-06T07:28:14.199068+0000","last_fullsized":"2024-03-06T07:28:14.199068+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206524+0000","last_change":"2024-03-06T07:28:13.182358+0000","last_active":"2024-03-06T07:28:14.206524+0000","last_peered":"2024-03-06T07:28:14.206524+0000","last_clean":"2024-03-06T07:28:14.206524+0000","last_became_active":"2024-03-06T07:28:13.182044+0000","last_became_peered":"2024-03-06T07:28:13.182044+0000","last_unstale":"2024-03-06T07:28:14.206524+0000","last_undegraded":"2024-03-06T07:28:14.206524+0000","last_fullsized":"2024-03-06T07:28:14.206524+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:45:03.516517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.199453+0000","last_change":"2024-03-06T07:28:14.193046+0000","last_active":"2024-03-06T07:28:14.199453+0000","last_peered":"2024-03-06T07:28:14.199453+0000","last_clean":"2024-03-06T07:28:14.199453+0000","last_became_active":"2024-03-06T07:28:14.190609+0000","last_became_peered":"2024-03-06T07:28:14.190609+0000","last_unstale":"2024-03-06T07:28:14.199453+0000","last_undegraded":"2024-03-06T07:28:14.199453+0000","last_fullsized":"2024-03-06T07:28:14.199453+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:58:24.229417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.202580+0000","last_change":"2024-03-06T07:28:13.190077+0000","last_active":"2024-03-06T07:28:14.202580+0000","last_peered":"2024-03-06T07:28:14.202580+0000","last_clean":"2024-03-06T07:28:14.202580+0000","last_became_active":"2024-03-06T07:28:13.189717+0000","last_became_peered":"2024-03-06T07:28:13.189717+0000","last_unstale":"2024-03-06T07:28:14.202580+0000","last_undegraded":"2024-03-06T07:28:14.202580+0000","last_fullsized":"2024-03-06T07:28:14.202580+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:33:14.195257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.195395+0000","last_change":"2024-03-06T07:28:14.186496+0000","last_active":"2024-03-06T07:28:14.195395+0000","last_peered":"2024-03-06T07:28:14.195395+0000","last_clean":"2024-03-06T07:28:14.195395+0000","last_became_active":"2024-03-06T07:28:14.184383+0000","last_became_peered":"2024-03-06T07:28:14.184383+0000","last_unstale":"2024-03-06T07:28:14.195395+0000","last_undegraded":"2024-03-06T07:28:14.195395+0000","last_fullsized":"2024-03-06T07:28:14.195395+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204973+0000","last_change":"2024-03-06T07:28:13.180100+0000","last_active":"2024-03-06T07:28:14.204973+0000","last_peered":"2024-03-06T07:28:14.204973+0000","last_clean":"2024-03-06T07:28:14.204973+0000","last_became_active":"2024-03-06T07:28:13.179659+0000","last_became_peered":"2024-03-06T07:28:13.179659+0000","last_unstale":"2024-03-06T07:28:14.204973+0000","last_undegraded":"2024-03-06T07:28:14.204973+0000","last_fullsized":"2024-03-06T07:28:14.204973+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:41:52.569686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196960+0000","last_change":"2024-03-06T07:28:14.192826+0000","last_active":"2024-03-06T07:28:14.196960+0000","last_peered":"2024-03-06T07:28:14.196960+0000","last_clean":"2024-03-06T07:28:14.196960+0000","last_became_active":"2024-03-06T07:28:14.192397+0000","last_became_peered":"2024-03-06T07:28:14.192397+0000","last_unstale":"2024-03-06T07:28:14.196960+0000","last_undegraded":"2024-03-06T07:28:14.196960+0000","last_fullsized":"2024-03-06T07:28:14.196960+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:49.056762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.165638+0000","last_change":"2024-03-06T07:28:13.183894+0000","last_active":"2024-03-06T07:28:14.165638+0000","last_peered":"2024-03-06T07:28:14.165638+0000","last_clean":"2024-03-06T07:28:14.165638+0000","last_became_active":"2024-03-06T07:28:13.181717+0000","last_became_peered":"2024-03-06T07:28:13.181717+0000","last_unstale":"2024-03-06T07:28:14.165638+0000","last_undegraded":"2024-03-06T07:28:14.165638+0000","last_fullsized":"2024-03-06T07:28:14.165638+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:48:06.392322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204911+0000","last_change":"2024-03-06T07:28:14.198849+0000","last_active":"2024-03-06T07:28:14.204911+0000","last_peered":"2024-03-06T07:28:14.204911+0000","last_clean":"2024-03-06T07:28:14.204911+0000","last_became_active":"2024-03-06T07:28:14.198586+0000","last_became_peered":"2024-03-06T07:28:14.198586+0000","last_unstale":"2024-03-06T07:28:14.204911+0000","last_undegraded":"2024-03-06T07:28:14.204911+0000","last_fullsized":"2024-03-06T07:28:14.204911+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:51.884783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204503+0000","last_change":"2024-03-06T07:28:13.186583+0000","last_active":"2024-03-06T07:28:14.204503+0000","last_peered":"2024-03-06T07:28:14.204503+0000","last_clean":"2024-03-06T07:28:14.204503+0000","last_became_active":"2024-03-06T07:28:13.180468+0000","last_became_peered":"2024-03-06T07:28:13.180468+0000","last_unstale":"2024-03-06T07:28:14.204503+0000","last_undegraded":"2024-03-06T07:28:14.204503+0000","last_fullsized":"2024-03-06T07:28:14.204503+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:05.656882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187756+0000","last_change":"2024-03-06T07:28:14.185820+0000","last_active":"2024-03-06T07:28:14.187756+0000","last_peered":"2024-03-06T07:28:14.187756+0000","last_clean":"2024-03-06T07:28:14.187756+0000","last_became_active":"2024-03-06T07:28:14.185239+0000","last_became_peered":"2024-03-06T07:28:14.185239+0000","last_unstale":"2024-03-06T07:28:14.187756+0000","last_undegraded":"2024-03-06T07:28:14.187756+0000","last_fullsized":"2024-03-06T07:28:14.187756+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:07.389377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.172678+0000","last_change":"2024-03-06T07:28:13.192355+0000","last_active":"2024-03-06T07:28:14.172678+0000","last_peered":"2024-03-06T07:28:14.172678+0000","last_clean":"2024-03-06T07:28:14.172678+0000","last_became_active":"2024-03-06T07:28:13.191972+0000","last_became_peered":"2024-03-06T07:28:13.191972+0000","last_unstale":"2024-03-06T07:28:14.172678+0000","last_undegraded":"2024-03-06T07:28:14.172678+0000","last_fullsized":"2024-03-06T07:28:14.172678+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:52.712300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187615+0000","last_change":"2024-03-06T07:28:14.184594+0000","last_active":"2024-03-06T07:28:14.187615+0000","last_peered":"2024-03-06T07:28:14.187615+0000","last_clean":"2024-03-06T07:28:14.187615+0000","last_became_active":"2024-03-06T07:28:14.184309+0000","last_became_peered":"2024-03-06T07:28:14.184309+0000","last_unstale":"2024-03-06T07:28:14.187615+0000","last_undegraded":"2024-03-06T07:28:14.187615+0000","last_fullsized":"2024-03-06T07:28:14.187615+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:15:05.420328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.164642+0000","last_change":"2024-03-06T07:28:13.182647+0000","last_active":"2024-03-06T07:28:14.164642+0000","last_peered":"2024-03-06T07:28:14.164642+0000","last_clean":"2024-03-06T07:28:14.164642+0000","last_became_active":"2024-03-06T07:28:13.182323+0000","last_became_peered":"2024-03-06T07:28:13.182323+0000","last_unstale":"2024-03-06T07:28:14.164642+0000","last_undegraded":"2024-03-06T07:28:14.164642+0000","last_fullsized":"2024-03-06T07:28:14.164642+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:28.246598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.9","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200408+0000","last_change":"2024-03-06T07:28:14.196038+0000","last_active":"2024-03-06T07:28:14.200408+0000","last_peered":"2024-03-06T07:28:14.200408+0000","last_clean":"2024-03-06T07:28:14.200408+0000","last_became_active":"2024-03-06T07:28:14.195710+0000","last_became_peered":"2024-03-06T07:28:14.195710+0000","last_unstale":"2024-03-06T07:28:14.200408+0000","last_undegraded":"2024-03-06T07:28:14.200408+0000","last_fullsized":"2024-03-06T07:28:14.200408+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:50:57.453127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.177691+0000","last_change":"2024-03-06T07:28:13.194684+0000","last_active":"2024-03-06T07:28:14.177691+0000","last_peered":"2024-03-06T07:28:14.177691+0000","last_clean":"2024-03-06T07:28:14.177691+0000","last_became_active":"2024-03-06T07:28:13.194336+0000","last_became_peered":"2024-03-06T07:28:13.194336+0000","last_unstale":"2024-03-06T07:28:14.177691+0000","last_undegraded":"2024-03-06T07:28:14.177691+0000","last_fullsized":"2024-03-06T07:28:14.177691+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:37:27.060079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.193911+0000","last_change":"2024-03-06T07:28:14.189760+0000","last_active":"2024-03-06T07:28:14.193911+0000","last_peered":"2024-03-06T07:28:14.193911+0000","last_clean":"2024-03-06T07:28:14.193911+0000","last_became_active":"2024-03-06T07:28:14.189445+0000","last_became_peered":"2024-03-06T07:28:14.189445+0000","last_unstale":"2024-03-06T07:28:14.193911+0000","last_undegraded":"2024-03-06T07:28:14.193911+0000","last_fullsized":"2024-03-06T07:28:14.193911+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:41.111808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204608+0000","last_change":"2024-03-06T07:28:13.190209+0000","last_active":"2024-03-06T07:28:14.204608+0000","last_peered":"2024-03-06T07:28:14.204608+0000","last_clean":"2024-03-06T07:28:14.204608+0000","last_became_active":"2024-03-06T07:28:13.189931+0000","last_became_peered":"2024-03-06T07:28:13.189931+0000","last_unstale":"2024-03-06T07:28:14.204608+0000","last_undegraded":"2024-03-06T07:28:14.204608+0000","last_fullsized":"2024-03-06T07:28:14.204608+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:05.656882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204267+0000","last_change":"2024-03-06T07:28:14.201039+0000","last_active":"2024-03-06T07:28:14.204267+0000","last_peered":"2024-03-06T07:28:14.204267+0000","last_clean":"2024-03-06T07:28:14.204267+0000","last_became_active":"2024-03-06T07:28:14.199423+0000","last_became_peered":"2024-03-06T07:28:14.199423+0000","last_unstale":"2024-03-06T07:28:14.204267+0000","last_undegraded":"2024-03-06T07:28:14.204267+0000","last_fullsized":"2024-03-06T07:28:14.204267+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:28.736878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.162415+0000","last_change":"2024-03-06T07:28:13.183496+0000","last_active":"2024-03-06T07:28:14.162415+0000","last_peered":"2024-03-06T07:28:14.162415+0000","last_clean":"2024-03-06T07:28:14.162415+0000","last_became_active":"2024-03-06T07:28:13.182729+0000","last_became_peered":"2024-03-06T07:28:13.182729+0000","last_unstale":"2024-03-06T07:28:14.162415+0000","last_undegraded":"2024-03-06T07:28:14.162415+0000","last_fullsized":"2024-03-06T07:28:14.162415+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:22:30.293914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.183206+0000","last_change":"2024-03-06T07:28:14.178461+0000","last_active":"2024-03-06T07:28:14.183206+0000","last_peered":"2024-03-06T07:28:14.183206+0000","last_clean":"2024-03-06T07:28:14.183206+0000","last_became_active":"2024-03-06T07:28:14.178179+0000","last_became_peered":"2024-03-06T07:28:14.178179+0000","last_unstale":"2024-03-06T07:28:14.183206+0000","last_undegraded":"2024-03-06T07:28:14.183206+0000","last_fullsized":"2024-03-06T07:28:14.183206+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.168858+0000","last_change":"2024-03-06T07:28:13.180700+0000","last_active":"2024-03-06T07:28:14.168858+0000","last_peered":"2024-03-06T07:28:14.168858+0000","last_clean":"2024-03-06T07:28:14.168858+0000","last_became_active":"2024-03-06T07:28:13.180342+0000","last_became_peered":"2024-03-06T07:28:13.180342+0000","last_unstale":"2024-03-06T07:28:14.168858+0000","last_undegraded":"2024-03-06T07:28:14.168858+0000","last_fullsized":"2024-03-06T07:28:14.168858+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:48:06.392322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190276+0000","last_change":"2024-03-06T07:28:14.186617+0000","last_active":"2024-03-06T07:28:14.190276+0000","last_peered":"2024-03-06T07:28:14.190276+0000","last_clean":"2024-03-06T07:28:14.190276+0000","last_became_active":"2024-03-06T07:28:14.186220+0000","last_became_peered":"2024-03-06T07:28:14.186220+0000","last_unstale":"2024-03-06T07:28:14.190276+0000","last_undegraded":"2024-03-06T07:28:14.190276+0000","last_fullsized":"2024-03-06T07:28:14.190276+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:37:28.077097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.162422+0000","last_change":"2024-03-06T07:28:13.188937+0000","last_active":"2024-03-06T07:28:14.162422+0000","last_peered":"2024-03-06T07:28:14.162422+0000","last_clean":"2024-03-06T07:28:14.162422+0000","last_became_active":"2024-03-06T07:28:13.188321+0000","last_became_peered":"2024-03-06T07:28:13.188321+0000","last_unstale":"2024-03-06T07:28:14.162422+0000","last_undegraded":"2024-03-06T07:28:14.162422+0000","last_fullsized":"2024-03-06T07:28:14.162422+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:43.842962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.191209+0000","last_change":"2024-03-06T07:28:14.188810+0000","last_active":"2024-03-06T07:28:14.191209+0000","last_peered":"2024-03-06T07:28:14.191209+0000","last_clean":"2024-03-06T07:28:14.191209+0000","last_became_active":"2024-03-06T07:28:14.188447+0000","last_became_peered":"2024-03-06T07:28:14.188447+0000","last_unstale":"2024-03-06T07:28:14.191209+0000","last_undegraded":"2024-03-06T07:28:14.191209+0000","last_fullsized":"2024-03-06T07:28:14.191209+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.175805+0000","last_change":"2024-03-06T07:28:13.194050+0000","last_active":"2024-03-06T07:28:14.175805+0000","last_peered":"2024-03-06T07:28:14.175805+0000","last_clean":"2024-03-06T07:28:14.175805+0000","last_became_active":"2024-03-06T07:28:13.193790+0000","last_became_peered":"2024-03-06T07:28:13.193790+0000","last_unstale":"2024-03-06T07:28:14.175805+0000","last_undegraded":"2024-03-06T07:28:14.175805+0000","last_fullsized":"2024-03-06T07:28:14.175805+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:22:30.293914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206461+0000","last_change":"2024-03-06T07:28:14.198564+0000","last_active":"2024-03-06T07:28:14.206461+0000","last_peered":"2024-03-06T07:28:14.206461+0000","last_clean":"2024-03-06T07:28:14.206461+0000","last_became_active":"2024-03-06T07:28:14.198303+0000","last_became_peered":"2024-03-06T07:28:14.198303+0000","last_unstale":"2024-03-06T07:28:14.206461+0000","last_undegraded":"2024-03-06T07:28:14.206461+0000","last_fullsized":"2024-03-06T07:28:14.206461+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:24.047894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.164801+0000","last_change":"2024-03-06T07:28:13.188740+0000","last_active":"2024-03-06T07:28:14.164801+0000","last_peered":"2024-03-06T07:28:14.164801+0000","last_clean":"2024-03-06T07:28:14.164801+0000","last_became_active":"2024-03-06T07:28:13.188384+0000","last_became_peered":"2024-03-06T07:28:13.188384+0000","last_unstale":"2024-03-06T07:28:14.164801+0000","last_undegraded":"2024-03-06T07:28:14.164801+0000","last_fullsized":"2024-03-06T07:28:14.164801+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:21:28.545540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192855+0000","last_change":"2024-03-06T07:28:14.190488+0000","last_active":"2024-03-06T07:28:14.192855+0000","last_peered":"2024-03-06T07:28:14.192855+0000","last_clean":"2024-03-06T07:28:14.192855+0000","last_became_active":"2024-03-06T07:28:14.187848+0000","last_became_peered":"2024-03-06T07:28:14.187848+0000","last_unstale":"2024-03-06T07:28:14.192855+0000","last_undegraded":"2024-03-06T07:28:14.192855+0000","last_fullsized":"2024-03-06T07:28:14.192855+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.168674+0000","last_change":"2024-03-06T07:28:13.190992+0000","last_active":"2024-03-06T07:28:14.168674+0000","last_peered":"2024-03-06T07:28:14.168674+0000","last_clean":"2024-03-06T07:28:14.168674+0000","last_became_active":"2024-03-06T07:28:13.190093+0000","last_became_peered":"2024-03-06T07:28:13.190093+0000","last_unstale":"2024-03-06T07:28:14.168674+0000","last_undegraded":"2024-03-06T07:28:14.168674+0000","last_fullsized":"2024-03-06T07:28:14.168674+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:43.842962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189770+0000","last_change":"2024-03-06T07:28:14.187410+0000","last_active":"2024-03-06T07:28:14.189770+0000","last_peered":"2024-03-06T07:28:14.189770+0000","last_clean":"2024-03-06T07:28:14.189770+0000","last_became_active":"2024-03-06T07:28:14.186885+0000","last_became_peered":"2024-03-06T07:28:14.186885+0000","last_unstale":"2024-03-06T07:28:14.189770+0000","last_undegraded":"2024-03-06T07:28:14.189770+0000","last_fullsized":"2024-03-06T07:28:14.189770+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.172162+0000","last_change":"2024-03-06T07:28:13.196175+0000","last_active":"2024-03-06T07:28:14.172162+0000","last_peered":"2024-03-06T07:28:14.172162+0000","last_clean":"2024-03-06T07:28:14.172162+0000","last_became_active":"2024-03-06T07:28:13.195443+0000","last_became_peered":"2024-03-06T07:28:13.195443+0000","last_unstale":"2024-03-06T07:28:14.172162+0000","last_undegraded":"2024-03-06T07:28:14.172162+0000","last_fullsized":"2024-03-06T07:28:14.172162+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:52:44.404427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":190,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.174823+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:28:14.174823+0000","last_peered":"2024-03-06T07:28:14.174823+0000","last_clean":"2024-03-06T07:28:14.174823+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:28:14.174823+0000","last_undegraded":"2024-03-06T07:28:14.174823+0000","last_fullsized":"2024-03-06T07:28:14.174823+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542163,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27960,"kb_used_data":940,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343880,"statfs":{"total":96636764160,"available":96608133120,"internally_reserved":0,"allocated":962560,"data_stored":731849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542162,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27400,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344440,"statfs":{"total":96636764160,"available":96608706560,"internally_reserved":0,"allocated":389120,"data_stored":148872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542163,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":389120,"data_stored":148928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542162,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27404,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344436,"statfs":{"total":96636764160,"available":96608702464,"internally_reserved":0,"allocated":389120,"data_stored":148360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542162,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27380,"kb_used_data":360,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344460,"statfs":{"total":96636764160,"available":96608727040,"internally_reserved":0,"allocated":368640,"data_stored":141481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542163,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27400,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344440,"statfs":{"total":96636764160,"available":96608706560,"internally_reserved":0,"allocated":385024,"data_stored":148326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542164,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27412,"kb_used_data":388,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344428,"statfs":{"total":96636764160,"available":96608694272,"internally_reserved":0,"allocated":397312,"data_stored":149890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542161,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27976,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343864,"statfs":{"total":96636764160,"available":96608116736,"internally_reserved":0,"allocated":974848,"data_stored":738624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:28:18.597 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:28:18.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.842+0000 7f17aa200640 1 Processor -- start 2024-03-06T07:28:18.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.843+0000 7f17aa200640 1 -- start start 2024-03-06T07:28:18.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f17aa200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f17a415f9d0 0x7f17a415fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f17aa200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a4070f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f17aa200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17a406cef0 con 0x7f17a4144c00 2024-03-06T07:28:18.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f17aa200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17a406d1d0 con 0x7f17a415f9d0 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f17aa200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17a406d4b0 con 0x7f17a4070b10 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f179aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f17a415f9d0 0x7f17a415fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f179aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f17a415f9d0 0x7f17a415fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60708/0 (socket says 172.21.15.86:60708) 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.844+0000 7f179aa00640 1 -- 172.21.15.86:0/383410925 learned_addr learned my addr 172.21.15.86:0/383410925 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.845+0000 7f17a3e00640 1 --2- 172.21.15.86:0/383410925 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a4070f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.845+0000 7f179aa00640 1 -- 172.21.15.86:0/383410925 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 msgr2=0x7f17a4070f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.845+0000 7f179aa00640 1 --2- 172.21.15.86:0/383410925 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a4070f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.845+0000 7f179aa00640 1 -- 172.21.15.86:0/383410925 >> v1:172.21.15.194:6789/0 conn(0x7f17a4144c00 legacy=0x7f17a406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:18.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.845+0000 7f179aa00640 1 -- 172.21.15.86:0/383410925 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17a406d790 con 0x7f17a415f9d0 2024-03-06T07:28:18.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.846+0000 7f179aa00640 1 --2- 172.21.15.86:0/383410925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a415f9d0 0x7f17a415fe60 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f178c009870 tx=0x7f178c02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=662d4c1a370afe91 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:18.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.846+0000 7f17a2a00640 1 -- 172.21.15.86:0/383410925 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f178c035020 con 0x7f17a415f9d0 2024-03-06T07:28:18.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.846+0000 7f17a2a00640 1 -- 172.21.15.86:0/383410925 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f178c031640 con 0x7f17a415f9d0 2024-03-06T07:28:18.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.846+0000 7f17a2a00640 1 -- 172.21.15.86:0/383410925 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f178c02d040 con 0x7f17a415f9d0 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 -- 172.21.15.86:0/383410925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a415f9d0 msgr2=0x7f17a415fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 --2- 172.21.15.86:0/383410925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a415f9d0 0x7f17a415fe60 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f178c009870 tx=0x7f178c02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 -- 172.21.15.86:0/383410925 shutdown_connections 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 --2- 172.21.15.86:0/383410925 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a4070f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 --2- 172.21.15.86:0/383410925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a415f9d0 0x7f17a415fe60 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 -- 172.21.15.86:0/383410925 >> 172.21.15.86:0/383410925 conn(0x7f17a40b1730 msgr2=0x7f17a40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 -- 172.21.15.86:0/383410925 shutdown_connections 2024-03-06T07:28:18.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.847+0000 7f17aa200640 1 -- 172.21.15.86:0/383410925 wait complete. 2024-03-06T07:28:18.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.849+0000 7f17aa200640 1 Processor -- start 2024-03-06T07:28:18.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.850+0000 7f17aa200640 1 -- start start 2024-03-06T07:28:18.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17aa200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a415e8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17aa200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a4144c00 0x7f17a41e6a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17aa200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17a415f9d0 0x7f17a40cc270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17aa200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17a406ccd0 con 0x7f17a4144c00 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17aa200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17a406cb50 con 0x7f17a415f9d0 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17aa200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17a406df90 con 0x7f17a4070b10 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f179aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a4144c00 0x7f17a41e6a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f179aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a4144c00 0x7f17a41e6a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60710/0 (socket says 172.21.15.86:60710) 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f179aa00640 1 -- 172.21.15.86:0/1636630365 learned_addr learned my addr 172.21.15.86:0/1636630365 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17a3400640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a415e8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.851+0000 7f17a3e00640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17a415f9d0 0x7f17a40cc270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179aa00640 1 -- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 msgr2=0x7f17a415e8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179aa00640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a415e8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179aa00640 1 -- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17a415f9d0 msgr2=0x7f17a40cc270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179aa00640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17a415f9d0 0x7f17a40cc270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:18.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179aa00640 1 -- 172.21.15.86:0/1636630365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17a41e72a0 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f17a3e00640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17a415f9d0 0x7f17a40cc270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179aa00640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a4144c00 0x7f17a41e6a70 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f178c02b930 tx=0x7f178c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179be00640 1 -- 172.21.15.86:0/1636630365 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f178c040070 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179be00640 1 -- 172.21.15.86:0/1636630365 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f178c039bf0 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17a41e7580 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f179be00640 1 -- 172.21.15.86:0/1636630365 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f178c035040 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.852+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17a41e7b40 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.854+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f17a404f330 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.854+0000 7f179be00640 1 -- 172.21.15.86:0/1636630365 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f178c04b060 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.855+0000 7f179be00640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f178006b4d0 0x7f178006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.855+0000 7f179be00640 1 -- 172.21.15.86:0/1636630365 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f178c0abd00 con 0x7f17a4144c00 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.856+0000 7f17a3400640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f178006b4d0 0x7f178006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:18.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.856+0000 7f17a3400640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f178006b4d0 0x7f178006d990 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f1794004ea0 tx=0x7f1794009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:18.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:18.860+0000 7f179be00640 1 -- 172.21.15.86:0/1636630365 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f178c07ab70 con 0x7f17a4144c00 2024-03-06T07:28:19.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.053+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f17a4072890 con 0x7f178006b4d0 2024-03-06T07:28:19.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.066+0000 7f179be00640 1 -- 172.21.15.86:0/1636630365 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251882 (secure 0 0 0) 0x7f17a4072890 con 0x7f178006b4d0 2024-03-06T07:28:19.071 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:19.072 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f178006b4d0 msgr2=0x7f178006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f178006b4d0 0x7f178006d990 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f1794004ea0 tx=0x7f1794009040 comp rx=0 tx=0).stop 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a4144c00 msgr2=0x7f17a41e6a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a4144c00 0x7f17a41e6a70 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f178c02b930 tx=0x7f178c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 shutdown_connections 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f178006b4d0 0x7f178006d990 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17a415f9d0 0x7f17a40cc270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17a4144c00 0x7f17a41e6a70 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 --2- 172.21.15.86:0/1636630365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17a4070b10 0x7f17a415e8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 >> 172.21.15.86:0/1636630365 conn(0x7f17a40b1730 msgr2=0x7f17a40ae580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:19.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 shutdown_connections 2024-03-06T07:28:19.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.072+0000 7f17aa200640 1 -- 172.21.15.86:0/1636630365 wait complete. 2024-03-06T07:28:19.094 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2024-03-06T07:28:18.872277+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":103,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":220328,"kb_used_data":4156,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754754392,"statfs":{"total":773094113280,"available":772868497408,"internally_reserved":0,"allocated":4255744,"data_stored":2356330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":221236787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":32,"apply_latency_ms":32,"commit_latency_ns":32000000,"apply_latency_ns":32000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001660"},"pg_stats":[{"pgid":"5.3f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200134+0000","last_change":"2024-03-06T07:28:14.191001+0000","last_active":"2024-03-06T07:28:14.200134+0000","last_peered":"2024-03-06T07:28:14.200134+0000","last_clean":"2024-03-06T07:28:14.200134+0000","last_became_active":"2024-03-06T07:28:14.189845+0000","last_became_peered":"2024-03-06T07:28:14.189845+0000","last_unstale":"2024-03-06T07:28:14.200134+0000","last_undegraded":"2024-03-06T07:28:14.200134+0000","last_fullsized":"2024-03-06T07:28:14.200134+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:15:19.053850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.3e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192888+0000","last_change":"2024-03-06T07:28:14.184519+0000","last_active":"2024-03-06T07:28:14.192888+0000","last_peered":"2024-03-06T07:28:14.192888+0000","last_clean":"2024-03-06T07:28:14.192888+0000","last_became_active":"2024-03-06T07:28:14.183689+0000","last_became_peered":"2024-03-06T07:28:14.183689+0000","last_unstale":"2024-03-06T07:28:14.192888+0000","last_undegraded":"2024-03-06T07:28:14.192888+0000","last_fullsized":"2024-03-06T07:28:14.192888+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:28.005358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203278+0000","last_change":"2024-03-06T07:28:14.198745+0000","last_active":"2024-03-06T07:28:14.203278+0000","last_peered":"2024-03-06T07:28:14.203278+0000","last_clean":"2024-03-06T07:28:14.203278+0000","last_became_active":"2024-03-06T07:28:14.198172+0000","last_became_peered":"2024-03-06T07:28:14.198172+0000","last_unstale":"2024-03-06T07:28:14.203278+0000","last_undegraded":"2024-03-06T07:28:14.203278+0000","last_fullsized":"2024-03-06T07:28:14.203278+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:02.619913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.3c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.195416+0000","last_change":"2024-03-06T07:28:14.188617+0000","last_active":"2024-03-06T07:28:14.195416+0000","last_peered":"2024-03-06T07:28:14.195416+0000","last_clean":"2024-03-06T07:28:14.195416+0000","last_became_active":"2024-03-06T07:28:14.184746+0000","last_became_peered":"2024-03-06T07:28:14.184746+0000","last_unstale":"2024-03-06T07:28:14.195416+0000","last_undegraded":"2024-03-06T07:28:14.195416+0000","last_fullsized":"2024-03-06T07:28:14.195416+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.197113+0000","last_change":"2024-03-06T07:28:14.194169+0000","last_active":"2024-03-06T07:28:14.197113+0000","last_peered":"2024-03-06T07:28:14.197113+0000","last_clean":"2024-03-06T07:28:14.197113+0000","last_became_active":"2024-03-06T07:28:14.193912+0000","last_became_peered":"2024-03-06T07:28:14.193912+0000","last_unstale":"2024-03-06T07:28:14.197113+0000","last_undegraded":"2024-03-06T07:28:14.197113+0000","last_fullsized":"2024-03-06T07:28:14.197113+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:13:40.876752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194501+0000","last_change":"2024-03-06T07:28:14.190905+0000","last_active":"2024-03-06T07:28:14.194501+0000","last_peered":"2024-03-06T07:28:14.194501+0000","last_clean":"2024-03-06T07:28:14.194501+0000","last_became_active":"2024-03-06T07:28:14.190627+0000","last_became_peered":"2024-03-06T07:28:14.190627+0000","last_unstale":"2024-03-06T07:28:14.194501+0000","last_undegraded":"2024-03-06T07:28:14.194501+0000","last_fullsized":"2024-03-06T07:28:14.194501+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:15.014804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.39","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205487+0000","last_change":"2024-03-06T07:28:14.203837+0000","last_active":"2024-03-06T07:28:14.205487+0000","last_peered":"2024-03-06T07:28:14.205487+0000","last_clean":"2024-03-06T07:28:14.205487+0000","last_became_active":"2024-03-06T07:28:14.203030+0000","last_became_peered":"2024-03-06T07:28:14.203030+0000","last_unstale":"2024-03-06T07:28:14.205487+0000","last_undegraded":"2024-03-06T07:28:14.205487+0000","last_fullsized":"2024-03-06T07:28:14.205487+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:20.764385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.38","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189466+0000","last_change":"2024-03-06T07:28:14.187580+0000","last_active":"2024-03-06T07:28:14.189466+0000","last_peered":"2024-03-06T07:28:14.189466+0000","last_clean":"2024-03-06T07:28:14.189466+0000","last_became_active":"2024-03-06T07:28:14.187267+0000","last_became_peered":"2024-03-06T07:28:14.187267+0000","last_unstale":"2024-03-06T07:28:14.189466+0000","last_undegraded":"2024-03-06T07:28:14.189466+0000","last_fullsized":"2024-03-06T07:28:14.189466+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.37","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189581+0000","last_change":"2024-03-06T07:28:14.181623+0000","last_active":"2024-03-06T07:28:14.189581+0000","last_peered":"2024-03-06T07:28:14.189581+0000","last_clean":"2024-03-06T07:28:14.189581+0000","last_became_active":"2024-03-06T07:28:14.180098+0000","last_became_peered":"2024-03-06T07:28:14.180098+0000","last_unstale":"2024-03-06T07:28:14.189581+0000","last_undegraded":"2024-03-06T07:28:14.189581+0000","last_fullsized":"2024-03-06T07:28:14.189581+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:22:31.310932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.36","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.198707+0000","last_change":"2024-03-06T07:28:14.196148+0000","last_active":"2024-03-06T07:28:14.198707+0000","last_peered":"2024-03-06T07:28:14.198707+0000","last_clean":"2024-03-06T07:28:14.198707+0000","last_became_active":"2024-03-06T07:28:14.190722+0000","last_became_peered":"2024-03-06T07:28:14.190722+0000","last_unstale":"2024-03-06T07:28:14.198707+0000","last_undegraded":"2024-03-06T07:28:14.198707+0000","last_fullsized":"2024-03-06T07:28:14.198707+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:50:57.453127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.35","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192828+0000","last_change":"2024-03-06T07:28:14.190124+0000","last_active":"2024-03-06T07:28:14.192828+0000","last_peered":"2024-03-06T07:28:14.192828+0000","last_clean":"2024-03-06T07:28:14.192828+0000","last_became_active":"2024-03-06T07:28:14.189749+0000","last_became_peered":"2024-03-06T07:28:14.189749+0000","last_unstale":"2024-03-06T07:28:14.192828+0000","last_undegraded":"2024-03-06T07:28:14.192828+0000","last_fullsized":"2024-03-06T07:28:14.192828+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.34","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203831+0000","last_change":"2024-03-06T07:28:14.195299+0000","last_active":"2024-03-06T07:28:14.203831+0000","last_peered":"2024-03-06T07:28:14.203831+0000","last_clean":"2024-03-06T07:28:14.203831+0000","last_became_active":"2024-03-06T07:28:14.195010+0000","last_became_peered":"2024-03-06T07:28:14.195010+0000","last_unstale":"2024-03-06T07:28:14.203831+0000","last_undegraded":"2024-03-06T07:28:14.203831+0000","last_fullsized":"2024-03-06T07:28:14.203831+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:59:40.916517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.33","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196044+0000","last_change":"2024-03-06T07:28:14.191112+0000","last_active":"2024-03-06T07:28:14.196044+0000","last_peered":"2024-03-06T07:28:14.196044+0000","last_clean":"2024-03-06T07:28:14.196044+0000","last_became_active":"2024-03-06T07:28:14.190515+0000","last_became_peered":"2024-03-06T07:28:14.190515+0000","last_unstale":"2024-03-06T07:28:14.196044+0000","last_undegraded":"2024-03-06T07:28:14.196044+0000","last_fullsized":"2024-03-06T07:28:14.196044+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:51:37.872171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.32","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194482+0000","last_change":"2024-03-06T07:28:14.189138+0000","last_active":"2024-03-06T07:28:14.194482+0000","last_peered":"2024-03-06T07:28:14.194482+0000","last_clean":"2024-03-06T07:28:14.194482+0000","last_became_active":"2024-03-06T07:28:14.188816+0000","last_became_peered":"2024-03-06T07:28:14.188816+0000","last_unstale":"2024-03-06T07:28:14.194482+0000","last_undegraded":"2024-03-06T07:28:14.194482+0000","last_fullsized":"2024-03-06T07:28:14.194482+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:52:20.286727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.31","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204409+0000","last_change":"2024-03-06T07:28:14.201952+0000","last_active":"2024-03-06T07:28:14.204409+0000","last_peered":"2024-03-06T07:28:14.204409+0000","last_clean":"2024-03-06T07:28:14.204409+0000","last_became_active":"2024-03-06T07:28:14.201594+0000","last_became_peered":"2024-03-06T07:28:14.201594+0000","last_unstale":"2024-03-06T07:28:14.204409+0000","last_undegraded":"2024-03-06T07:28:14.204409+0000","last_fullsized":"2024-03-06T07:28:14.204409+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:42:16.656971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.30","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200630+0000","last_change":"2024-03-06T07:28:14.197261+0000","last_active":"2024-03-06T07:28:14.200630+0000","last_peered":"2024-03-06T07:28:14.200630+0000","last_clean":"2024-03-06T07:28:14.200630+0000","last_became_active":"2024-03-06T07:28:14.197007+0000","last_became_peered":"2024-03-06T07:28:14.197007+0000","last_unstale":"2024-03-06T07:28:14.200630+0000","last_undegraded":"2024-03-06T07:28:14.200630+0000","last_fullsized":"2024-03-06T07:28:14.200630+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:16.521083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204930+0000","last_change":"2024-03-06T07:28:14.200979+0000","last_active":"2024-03-06T07:28:14.204930+0000","last_peered":"2024-03-06T07:28:14.204930+0000","last_clean":"2024-03-06T07:28:14.204930+0000","last_became_active":"2024-03-06T07:28:14.200679+0000","last_became_peered":"2024-03-06T07:28:14.200679+0000","last_unstale":"2024-03-06T07:28:14.204930+0000","last_undegraded":"2024-03-06T07:28:14.204930+0000","last_fullsized":"2024-03-06T07:28:14.204930+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:28.005358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205737+0000","last_change":"2024-03-06T07:28:14.198175+0000","last_active":"2024-03-06T07:28:14.205737+0000","last_peered":"2024-03-06T07:28:14.205737+0000","last_clean":"2024-03-06T07:28:14.205737+0000","last_became_active":"2024-03-06T07:28:14.197840+0000","last_became_peered":"2024-03-06T07:28:14.197840+0000","last_unstale":"2024-03-06T07:28:14.205737+0000","last_undegraded":"2024-03-06T07:28:14.205737+0000","last_fullsized":"2024-03-06T07:28:14.205737+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:55:01.352602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190339+0000","last_change":"2024-03-06T07:28:14.179175+0000","last_active":"2024-03-06T07:28:14.190339+0000","last_peered":"2024-03-06T07:28:14.190339+0000","last_clean":"2024-03-06T07:28:14.190339+0000","last_became_active":"2024-03-06T07:28:14.178900+0000","last_became_peered":"2024-03-06T07:28:14.178900+0000","last_unstale":"2024-03-06T07:28:14.190339+0000","last_undegraded":"2024-03-06T07:28:14.190339+0000","last_fullsized":"2024-03-06T07:28:14.190339+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:50:57.453127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194040+0000","last_change":"2024-03-06T07:28:14.190326+0000","last_active":"2024-03-06T07:28:14.194040+0000","last_peered":"2024-03-06T07:28:14.194040+0000","last_clean":"2024-03-06T07:28:14.194040+0000","last_became_active":"2024-03-06T07:28:14.189406+0000","last_became_peered":"2024-03-06T07:28:14.189406+0000","last_unstale":"2024-03-06T07:28:14.194040+0000","last_undegraded":"2024-03-06T07:28:14.194040+0000","last_fullsized":"2024-03-06T07:28:14.194040+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:52:16.900125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.2b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.173495+0000","last_change":"2024-03-06T07:28:14.169764+0000","last_active":"2024-03-06T07:28:14.173495+0000","last_peered":"2024-03-06T07:28:14.173495+0000","last_clean":"2024-03-06T07:28:14.173495+0000","last_became_active":"2024-03-06T07:28:14.169463+0000","last_became_peered":"2024-03-06T07:28:14.169463+0000","last_unstale":"2024-03-06T07:28:14.173495+0000","last_undegraded":"2024-03-06T07:28:14.173495+0000","last_fullsized":"2024-03-06T07:28:14.173495+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:06.673900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.2a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189055+0000","last_change":"2024-03-06T07:28:14.182211+0000","last_active":"2024-03-06T07:28:14.189055+0000","last_peered":"2024-03-06T07:28:14.189055+0000","last_clean":"2024-03-06T07:28:14.189055+0000","last_became_active":"2024-03-06T07:28:14.181920+0000","last_became_peered":"2024-03-06T07:28:14.181920+0000","last_unstale":"2024-03-06T07:28:14.189055+0000","last_undegraded":"2024-03-06T07:28:14.189055+0000","last_fullsized":"2024-03-06T07:28:14.189055+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:44.859980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.29","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.180863+0000","last_change":"2024-03-06T07:28:14.177839+0000","last_active":"2024-03-06T07:28:14.180863+0000","last_peered":"2024-03-06T07:28:14.180863+0000","last_clean":"2024-03-06T07:28:14.180863+0000","last_became_active":"2024-03-06T07:28:14.177447+0000","last_became_peered":"2024-03-06T07:28:14.177447+0000","last_unstale":"2024-03-06T07:28:14.180863+0000","last_undegraded":"2024-03-06T07:28:14.180863+0000","last_fullsized":"2024-03-06T07:28:14.180863+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.28","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.188947+0000","last_change":"2024-03-06T07:28:14.186095+0000","last_active":"2024-03-06T07:28:14.188947+0000","last_peered":"2024-03-06T07:28:14.188947+0000","last_clean":"2024-03-06T07:28:14.188947+0000","last_became_active":"2024-03-06T07:28:14.185801+0000","last_became_peered":"2024-03-06T07:28:14.185801+0000","last_unstale":"2024-03-06T07:28:14.188947+0000","last_undegraded":"2024-03-06T07:28:14.188947+0000","last_fullsized":"2024-03-06T07:28:14.188947+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:02:50.946267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.27","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196286+0000","last_change":"2024-03-06T07:28:14.181564+0000","last_active":"2024-03-06T07:28:14.196286+0000","last_peered":"2024-03-06T07:28:14.196286+0000","last_clean":"2024-03-06T07:28:14.196286+0000","last_became_active":"2024-03-06T07:28:14.181275+0000","last_became_peered":"2024-03-06T07:28:14.181275+0000","last_unstale":"2024-03-06T07:28:14.196286+0000","last_undegraded":"2024-03-06T07:28:14.196286+0000","last_fullsized":"2024-03-06T07:28:14.196286+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:44.859980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.26","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190613+0000","last_change":"2024-03-06T07:28:14.180935+0000","last_active":"2024-03-06T07:28:14.190613+0000","last_peered":"2024-03-06T07:28:14.190613+0000","last_clean":"2024-03-06T07:28:14.190613+0000","last_became_active":"2024-03-06T07:28:14.180676+0000","last_became_peered":"2024-03-06T07:28:14.180676+0000","last_unstale":"2024-03-06T07:28:14.190613+0000","last_undegraded":"2024-03-06T07:28:14.190613+0000","last_fullsized":"2024-03-06T07:28:14.190613+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:16.521083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.25","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196258+0000","last_change":"2024-03-06T07:28:14.192629+0000","last_active":"2024-03-06T07:28:14.196258+0000","last_peered":"2024-03-06T07:28:14.196258+0000","last_clean":"2024-03-06T07:28:14.196258+0000","last_became_active":"2024-03-06T07:28:14.192143+0000","last_became_peered":"2024-03-06T07:28:14.192143+0000","last_unstale":"2024-03-06T07:28:14.196258+0000","last_undegraded":"2024-03-06T07:28:14.196258+0000","last_fullsized":"2024-03-06T07:28:14.196258+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:35.371286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.24","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.188879+0000","last_change":"2024-03-06T07:28:14.185982+0000","last_active":"2024-03-06T07:28:14.188879+0000","last_peered":"2024-03-06T07:28:14.188879+0000","last_clean":"2024-03-06T07:28:14.188879+0000","last_became_active":"2024-03-06T07:28:14.185693+0000","last_became_peered":"2024-03-06T07:28:14.185693+0000","last_unstale":"2024-03-06T07:28:14.188879+0000","last_undegraded":"2024-03-06T07:28:14.188879+0000","last_fullsized":"2024-03-06T07:28:14.188879+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:25:19.031189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.23","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192566+0000","last_change":"2024-03-06T07:28:14.186863+0000","last_active":"2024-03-06T07:28:14.192566+0000","last_peered":"2024-03-06T07:28:14.192566+0000","last_clean":"2024-03-06T07:28:14.192566+0000","last_became_active":"2024-03-06T07:28:14.186663+0000","last_became_peered":"2024-03-06T07:28:14.186663+0000","last_unstale":"2024-03-06T07:28:14.192566+0000","last_undegraded":"2024-03-06T07:28:14.192566+0000","last_fullsized":"2024-03-06T07:28:14.192566+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.22","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.198013+0000","last_change":"2024-03-06T07:28:14.194770+0000","last_active":"2024-03-06T07:28:14.198013+0000","last_peered":"2024-03-06T07:28:14.198013+0000","last_clean":"2024-03-06T07:28:14.198013+0000","last_became_active":"2024-03-06T07:28:14.194428+0000","last_became_peered":"2024-03-06T07:28:14.194428+0000","last_unstale":"2024-03-06T07:28:14.198013+0000","last_undegraded":"2024-03-06T07:28:14.198013+0000","last_fullsized":"2024-03-06T07:28:14.198013+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:54.596749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.21","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.201186+0000","last_change":"2024-03-06T07:28:14.195293+0000","last_active":"2024-03-06T07:28:14.201186+0000","last_peered":"2024-03-06T07:28:14.201186+0000","last_clean":"2024-03-06T07:28:14.201186+0000","last_became_active":"2024-03-06T07:28:14.194851+0000","last_became_peered":"2024-03-06T07:28:14.194851+0000","last_unstale":"2024-03-06T07:28:14.201186+0000","last_undegraded":"2024-03-06T07:28:14.201186+0000","last_fullsized":"2024-03-06T07:28:14.201186+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:25:48.043980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.20","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196749+0000","last_change":"2024-03-06T07:28:14.192412+0000","last_active":"2024-03-06T07:28:14.196749+0000","last_peered":"2024-03-06T07:28:14.196749+0000","last_clean":"2024-03-06T07:28:14.196749+0000","last_became_active":"2024-03-06T07:28:14.192108+0000","last_became_peered":"2024-03-06T07:28:14.192108+0000","last_unstale":"2024-03-06T07:28:14.196749+0000","last_undegraded":"2024-03-06T07:28:14.196749+0000","last_fullsized":"2024-03-06T07:28:14.196749+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.183160+0000","last_change":"2024-03-06T07:28:14.178093+0000","last_active":"2024-03-06T07:28:14.183160+0000","last_peered":"2024-03-06T07:28:14.183160+0000","last_clean":"2024-03-06T07:28:14.183160+0000","last_became_active":"2024-03-06T07:28:14.177261+0000","last_became_peered":"2024-03-06T07:28:14.177261+0000","last_unstale":"2024-03-06T07:28:14.183160+0000","last_undegraded":"2024-03-06T07:28:14.183160+0000","last_fullsized":"2024-03-06T07:28:14.183160+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"34'4","reported_seq":16,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205516+0000","last_change":"2024-03-06T07:28:13.198209+0000","last_active":"2024-03-06T07:28:14.205516+0000","last_peered":"2024-03-06T07:28:14.205516+0000","last_clean":"2024-03-06T07:28:14.205516+0000","last_became_active":"2024-03-06T07:28:13.197761+0000","last_became_peered":"2024-03-06T07:28:13.197761+0000","last_unstale":"2024-03-06T07:28:14.205516+0000","last_undegraded":"2024-03-06T07:28:14.205516+0000","last_fullsized":"2024-03-06T07:28:14.205516+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:54:41.878583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.1f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.191144+0000","last_change":"2024-03-06T07:28:14.179839+0000","last_active":"2024-03-06T07:28:14.191144+0000","last_peered":"2024-03-06T07:28:14.191144+0000","last_clean":"2024-03-06T07:28:14.191144+0000","last_became_active":"2024-03-06T07:28:14.175864+0000","last_became_peered":"2024-03-06T07:28:14.175864+0000","last_unstale":"2024-03-06T07:28:14.191144+0000","last_undegraded":"2024-03-06T07:28:14.191144+0000","last_fullsized":"2024-03-06T07:28:14.191144+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:14:29.282408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1e","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203930+0000","last_change":"2024-03-06T07:28:13.186932+0000","last_active":"2024-03-06T07:28:14.203930+0000","last_peered":"2024-03-06T07:28:14.203930+0000","last_clean":"2024-03-06T07:28:14.203930+0000","last_became_active":"2024-03-06T07:28:13.185983+0000","last_became_peered":"2024-03-06T07:28:13.185983+0000","last_unstale":"2024-03-06T07:28:14.203930+0000","last_undegraded":"2024-03-06T07:28:14.203930+0000","last_fullsized":"2024-03-06T07:28:14.203930+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:33:14.195257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.188505+0000","last_change":"2024-03-06T07:28:14.185352+0000","last_active":"2024-03-06T07:28:14.188505+0000","last_peered":"2024-03-06T07:28:14.188505+0000","last_clean":"2024-03-06T07:28:14.188505+0000","last_became_active":"2024-03-06T07:28:14.184956+0000","last_became_peered":"2024-03-06T07:28:14.184956+0000","last_unstale":"2024-03-06T07:28:14.188505+0000","last_undegraded":"2024-03-06T07:28:14.188505+0000","last_fullsized":"2024-03-06T07:28:14.188505+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.211419+0000","last_change":"2024-03-06T07:28:13.186253+0000","last_active":"2024-03-06T07:28:14.211419+0000","last_peered":"2024-03-06T07:28:14.211419+0000","last_clean":"2024-03-06T07:28:14.211419+0000","last_became_active":"2024-03-06T07:28:13.185850+0000","last_became_peered":"2024-03-06T07:28:13.185850+0000","last_unstale":"2024-03-06T07:28:14.211419+0000","last_undegraded":"2024-03-06T07:28:14.211419+0000","last_fullsized":"2024-03-06T07:28:14.211419+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:24.344579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196232+0000","last_change":"2024-03-06T07:28:14.192387+0000","last_active":"2024-03-06T07:28:14.196232+0000","last_peered":"2024-03-06T07:28:14.196232+0000","last_clean":"2024-03-06T07:28:14.196232+0000","last_became_active":"2024-03-06T07:28:14.191766+0000","last_became_peered":"2024-03-06T07:28:14.191766+0000","last_unstale":"2024-03-06T07:28:14.196232+0000","last_undegraded":"2024-03-06T07:28:14.196232+0000","last_fullsized":"2024-03-06T07:28:14.196232+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206232+0000","last_change":"2024-03-06T07:28:13.184473+0000","last_active":"2024-03-06T07:28:14.206232+0000","last_peered":"2024-03-06T07:28:14.206232+0000","last_clean":"2024-03-06T07:28:14.206232+0000","last_became_active":"2024-03-06T07:28:13.182588+0000","last_became_peered":"2024-03-06T07:28:13.182588+0000","last_unstale":"2024-03-06T07:28:14.206232+0000","last_undegraded":"2024-03-06T07:28:14.206232+0000","last_fullsized":"2024-03-06T07:28:14.206232+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:28.980631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187980+0000","last_change":"2024-03-06T07:28:14.183508+0000","last_active":"2024-03-06T07:28:14.187980+0000","last_peered":"2024-03-06T07:28:14.187980+0000","last_clean":"2024-03-06T07:28:14.187980+0000","last_became_active":"2024-03-06T07:28:14.183208+0000","last_became_peered":"2024-03-06T07:28:14.183208+0000","last_unstale":"2024-03-06T07:28:14.187980+0000","last_undegraded":"2024-03-06T07:28:14.187980+0000","last_fullsized":"2024-03-06T07:28:14.187980+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:44.859980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206544+0000","last_change":"2024-03-06T07:28:13.185609+0000","last_active":"2024-03-06T07:28:14.206544+0000","last_peered":"2024-03-06T07:28:14.206544+0000","last_clean":"2024-03-06T07:28:14.206544+0000","last_became_active":"2024-03-06T07:28:13.185284+0000","last_became_peered":"2024-03-06T07:28:13.185284+0000","last_unstale":"2024-03-06T07:28:14.206544+0000","last_undegraded":"2024-03-06T07:28:14.206544+0000","last_fullsized":"2024-03-06T07:28:14.206544+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:35:52.290057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187619+0000","last_change":"2024-03-06T07:28:14.182280+0000","last_active":"2024-03-06T07:28:14.187619+0000","last_peered":"2024-03-06T07:28:14.187619+0000","last_clean":"2024-03-06T07:28:14.187619+0000","last_became_active":"2024-03-06T07:28:14.181917+0000","last_became_peered":"2024-03-06T07:28:14.181917+0000","last_unstale":"2024-03-06T07:28:14.187619+0000","last_undegraded":"2024-03-06T07:28:14.187619+0000","last_fullsized":"2024-03-06T07:28:14.187619+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:25:48.043980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.205951+0000","last_change":"2024-03-06T07:28:13.182637+0000","last_active":"2024-03-06T07:28:14.205951+0000","last_peered":"2024-03-06T07:28:14.205951+0000","last_clean":"2024-03-06T07:28:14.205951+0000","last_became_active":"2024-03-06T07:28:13.181939+0000","last_became_peered":"2024-03-06T07:28:13.181939+0000","last_unstale":"2024-03-06T07:28:14.205951+0000","last_undegraded":"2024-03-06T07:28:14.205951+0000","last_fullsized":"2024-03-06T07:28:14.205951+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:51:21.126378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200180+0000","last_change":"2024-03-06T07:28:14.191492+0000","last_active":"2024-03-06T07:28:14.200180+0000","last_peered":"2024-03-06T07:28:14.200180+0000","last_clean":"2024-03-06T07:28:14.200180+0000","last_became_active":"2024-03-06T07:28:14.189261+0000","last_became_peered":"2024-03-06T07:28:14.189261+0000","last_unstale":"2024-03-06T07:28:14.200180+0000","last_undegraded":"2024-03-06T07:28:14.200180+0000","last_fullsized":"2024-03-06T07:28:14.200180+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:35.371286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.170463+0000","last_change":"2024-03-06T07:28:13.179224+0000","last_active":"2024-03-06T07:28:14.170463+0000","last_peered":"2024-03-06T07:28:14.170463+0000","last_clean":"2024-03-06T07:28:14.170463+0000","last_became_active":"2024-03-06T07:28:13.178961+0000","last_became_peered":"2024-03-06T07:28:13.178961+0000","last_unstale":"2024-03-06T07:28:14.170463+0000","last_undegraded":"2024-03-06T07:28:14.170463+0000","last_fullsized":"2024-03-06T07:28:14.170463+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:11.304727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190379+0000","last_change":"2024-03-06T07:28:14.187088+0000","last_active":"2024-03-06T07:28:14.190379+0000","last_peered":"2024-03-06T07:28:14.190379+0000","last_clean":"2024-03-06T07:28:14.190379+0000","last_became_active":"2024-03-06T07:28:14.186757+0000","last_became_peered":"2024-03-06T07:28:14.186757+0000","last_unstale":"2024-03-06T07:28:14.190379+0000","last_undegraded":"2024-03-06T07:28:14.190379+0000","last_fullsized":"2024-03-06T07:28:14.190379+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:12.321745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.166260+0000","last_change":"2024-03-06T07:28:13.190240+0000","last_active":"2024-03-06T07:28:14.166260+0000","last_peered":"2024-03-06T07:28:14.166260+0000","last_clean":"2024-03-06T07:28:14.166260+0000","last_became_active":"2024-03-06T07:28:13.189917+0000","last_became_peered":"2024-03-06T07:28:13.189917+0000","last_unstale":"2024-03-06T07:28:14.166260+0000","last_undegraded":"2024-03-06T07:28:14.166260+0000","last_fullsized":"2024-03-06T07:28:14.166260+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:45:03.516517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.16","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.207965+0000","last_change":"2024-03-06T07:28:14.203544+0000","last_active":"2024-03-06T07:28:14.207965+0000","last_peered":"2024-03-06T07:28:14.207965+0000","last_clean":"2024-03-06T07:28:14.207965+0000","last_became_active":"2024-03-06T07:28:14.203111+0000","last_became_peered":"2024-03-06T07:28:14.203111+0000","last_unstale":"2024-03-06T07:28:14.207965+0000","last_undegraded":"2024-03-06T07:28:14.207965+0000","last_fullsized":"2024-03-06T07:28:14.207965+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:42:06.295071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.171220+0000","last_change":"2024-03-06T07:28:13.191733+0000","last_active":"2024-03-06T07:28:14.171220+0000","last_peered":"2024-03-06T07:28:14.171220+0000","last_clean":"2024-03-06T07:28:14.171220+0000","last_became_active":"2024-03-06T07:28:13.191479+0000","last_became_peered":"2024-03-06T07:28:13.191479+0000","last_unstale":"2024-03-06T07:28:14.171220+0000","last_undegraded":"2024-03-06T07:28:14.171220+0000","last_fullsized":"2024-03-06T07:28:14.171220+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:57.071852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.17","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206263+0000","last_change":"2024-03-06T07:28:14.202604+0000","last_active":"2024-03-06T07:28:14.206263+0000","last_peered":"2024-03-06T07:28:14.206263+0000","last_clean":"2024-03-06T07:28:14.206263+0000","last_became_active":"2024-03-06T07:28:14.196754+0000","last_became_peered":"2024-03-06T07:28:14.196754+0000","last_unstale":"2024-03-06T07:28:14.206263+0000","last_undegraded":"2024-03-06T07:28:14.206263+0000","last_fullsized":"2024-03-06T07:28:14.206263+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:11:39.747997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.162265+0000","last_change":"2024-03-06T07:28:13.183746+0000","last_active":"2024-03-06T07:28:14.162265+0000","last_peered":"2024-03-06T07:28:14.162265+0000","last_clean":"2024-03-06T07:28:14.162265+0000","last_became_active":"2024-03-06T07:28:13.183227+0000","last_became_peered":"2024-03-06T07:28:13.183227+0000","last_unstale":"2024-03-06T07:28:14.162265+0000","last_undegraded":"2024-03-06T07:28:14.162265+0000","last_fullsized":"2024-03-06T07:28:14.162265+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:45:03.516517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.14","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190997+0000","last_change":"2024-03-06T07:28:14.185905+0000","last_active":"2024-03-06T07:28:14.190997+0000","last_peered":"2024-03-06T07:28:14.190997+0000","last_clean":"2024-03-06T07:28:14.190997+0000","last_became_active":"2024-03-06T07:28:14.183318+0000","last_became_peered":"2024-03-06T07:28:14.183318+0000","last_unstale":"2024-03-06T07:28:14.190997+0000","last_undegraded":"2024-03-06T07:28:14.190997+0000","last_fullsized":"2024-03-06T07:28:14.190997+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:20.764385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"34'3","reported_seq":15,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.210697+0000","last_change":"2024-03-06T07:28:13.195049+0000","last_active":"2024-03-06T07:28:14.210697+0000","last_peered":"2024-03-06T07:28:14.210697+0000","last_clean":"2024-03-06T07:28:14.210697+0000","last_became_active":"2024-03-06T07:28:13.194767+0000","last_became_peered":"2024-03-06T07:28:13.194767+0000","last_unstale":"2024-03-06T07:28:14.210697+0000","last_undegraded":"2024-03-06T07:28:14.210697+0000","last_fullsized":"2024-03-06T07:28:14.210697+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:15:18.036832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.15","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.203292+0000","last_change":"2024-03-06T07:28:14.198814+0000","last_active":"2024-03-06T07:28:14.203292+0000","last_peered":"2024-03-06T07:28:14.203292+0000","last_clean":"2024-03-06T07:28:14.203292+0000","last_became_active":"2024-03-06T07:28:14.198475+0000","last_became_peered":"2024-03-06T07:28:14.198475+0000","last_unstale":"2024-03-06T07:28:14.203292+0000","last_undegraded":"2024-03-06T07:28:14.203292+0000","last_fullsized":"2024-03-06T07:28:14.203292+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:13:09.283825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"34'6","reported_seq":18,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.213867+0000","last_change":"2024-03-06T07:28:13.184116+0000","last_active":"2024-03-06T07:28:14.213867+0000","last_peered":"2024-03-06T07:28:14.213867+0000","last_clean":"2024-03-06T07:28:14.213867+0000","last_became_active":"2024-03-06T07:28:13.183617+0000","last_became_peered":"2024-03-06T07:28:13.183617+0000","last_unstale":"2024-03-06T07:28:14.213867+0000","last_undegraded":"2024-03-06T07:28:14.213867+0000","last_fullsized":"2024-03-06T07:28:14.213867+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:53.579731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.12","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.197741+0000","last_change":"2024-03-06T07:28:14.195034+0000","last_active":"2024-03-06T07:28:14.197741+0000","last_peered":"2024-03-06T07:28:14.197741+0000","last_clean":"2024-03-06T07:28:14.197741+0000","last_became_active":"2024-03-06T07:28:14.187691+0000","last_became_peered":"2024-03-06T07:28:14.187691+0000","last_unstale":"2024-03-06T07:28:14.197741+0000","last_undegraded":"2024-03-06T07:28:14.197741+0000","last_fullsized":"2024-03-06T07:28:14.197741+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:54.596749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.202415+0000","last_change":"2024-03-06T07:28:13.193486+0000","last_active":"2024-03-06T07:28:14.202415+0000","last_peered":"2024-03-06T07:28:14.202415+0000","last_clean":"2024-03-06T07:28:14.202415+0000","last_became_active":"2024-03-06T07:28:13.192611+0000","last_became_peered":"2024-03-06T07:28:13.192611+0000","last_unstale":"2024-03-06T07:28:14.202415+0000","last_undegraded":"2024-03-06T07:28:14.202415+0000","last_fullsized":"2024-03-06T07:28:14.202415+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:02:27.890526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194358+0000","last_change":"2024-03-06T07:28:14.186524+0000","last_active":"2024-03-06T07:28:14.194358+0000","last_peered":"2024-03-06T07:28:14.194358+0000","last_clean":"2024-03-06T07:28:14.194358+0000","last_became_active":"2024-03-06T07:28:14.185772+0000","last_became_peered":"2024-03-06T07:28:14.185772+0000","last_unstale":"2024-03-06T07:28:14.194358+0000","last_undegraded":"2024-03-06T07:28:14.194358+0000","last_fullsized":"2024-03-06T07:28:14.194358+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:26:33.379773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.169359+0000","last_change":"2024-03-06T07:28:13.190800+0000","last_active":"2024-03-06T07:28:14.169359+0000","last_peered":"2024-03-06T07:28:14.169359+0000","last_clean":"2024-03-06T07:28:14.169359+0000","last_became_active":"2024-03-06T07:28:13.190497+0000","last_became_peered":"2024-03-06T07:28:13.190497+0000","last_unstale":"2024-03-06T07:28:14.169359+0000","last_undegraded":"2024-03-06T07:28:14.169359+0000","last_fullsized":"2024-03-06T07:28:14.169359+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:33:14.195257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.10","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.197954+0000","last_change":"2024-03-06T07:28:14.192648+0000","last_active":"2024-03-06T07:28:14.197954+0000","last_peered":"2024-03-06T07:28:14.197954+0000","last_clean":"2024-03-06T07:28:14.197954+0000","last_became_active":"2024-03-06T07:28:14.192308+0000","last_became_peered":"2024-03-06T07:28:14.192308+0000","last_unstale":"2024-03-06T07:28:14.197954+0000","last_undegraded":"2024-03-06T07:28:14.197954+0000","last_fullsized":"2024-03-06T07:28:14.197954+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:35.371286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.173861+0000","last_change":"2024-03-06T07:28:13.192013+0000","last_active":"2024-03-06T07:28:14.173861+0000","last_peered":"2024-03-06T07:28:14.173861+0000","last_clean":"2024-03-06T07:28:14.173861+0000","last_became_active":"2024-03-06T07:28:13.191204+0000","last_became_peered":"2024-03-06T07:28:13.191204+0000","last_unstale":"2024-03-06T07:28:14.173861+0000","last_undegraded":"2024-03-06T07:28:14.173861+0000","last_fullsized":"2024-03-06T07:28:14.173861+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:53:34.354268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.194666+0000","last_change":"2024-03-06T07:28:14.176017+0000","last_active":"2024-03-06T07:28:14.194666+0000","last_peered":"2024-03-06T07:28:14.194666+0000","last_clean":"2024-03-06T07:28:14.194666+0000","last_became_active":"2024-03-06T07:28:14.174689+0000","last_became_peered":"2024-03-06T07:28:14.174689+0000","last_unstale":"2024-03-06T07:28:14.194666+0000","last_undegraded":"2024-03-06T07:28:14.194666+0000","last_fullsized":"2024-03-06T07:28:14.194666+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:17.463217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"34'8","reported_seq":20,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.211353+0000","last_change":"2024-03-06T07:28:13.176218+0000","last_active":"2024-03-06T07:28:14.211353+0000","last_peered":"2024-03-06T07:28:14.211353+0000","last_clean":"2024-03-06T07:28:14.211353+0000","last_became_active":"2024-03-06T07:28:13.175859+0000","last_became_peered":"2024-03-06T07:28:13.175859+0000","last_unstale":"2024-03-06T07:28:14.211353+0000","last_undegraded":"2024-03-06T07:28:14.211353+0000","last_fullsized":"2024-03-06T07:28:14.211353+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:02.443572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.e","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.199068+0000","last_change":"2024-03-06T07:28:14.190503+0000","last_active":"2024-03-06T07:28:14.199068+0000","last_peered":"2024-03-06T07:28:14.199068+0000","last_clean":"2024-03-06T07:28:14.199068+0000","last_became_active":"2024-03-06T07:28:14.190215+0000","last_became_peered":"2024-03-06T07:28:14.190215+0000","last_unstale":"2024-03-06T07:28:14.199068+0000","last_undegraded":"2024-03-06T07:28:14.199068+0000","last_fullsized":"2024-03-06T07:28:14.199068+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206524+0000","last_change":"2024-03-06T07:28:13.182358+0000","last_active":"2024-03-06T07:28:14.206524+0000","last_peered":"2024-03-06T07:28:14.206524+0000","last_clean":"2024-03-06T07:28:14.206524+0000","last_became_active":"2024-03-06T07:28:13.182044+0000","last_became_peered":"2024-03-06T07:28:13.182044+0000","last_unstale":"2024-03-06T07:28:14.206524+0000","last_undegraded":"2024-03-06T07:28:14.206524+0000","last_fullsized":"2024-03-06T07:28:14.206524+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:45:03.516517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.f","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.199453+0000","last_change":"2024-03-06T07:28:14.193046+0000","last_active":"2024-03-06T07:28:14.199453+0000","last_peered":"2024-03-06T07:28:14.199453+0000","last_clean":"2024-03-06T07:28:14.199453+0000","last_became_active":"2024-03-06T07:28:14.190609+0000","last_became_peered":"2024-03-06T07:28:14.190609+0000","last_unstale":"2024-03-06T07:28:14.199453+0000","last_undegraded":"2024-03-06T07:28:14.199453+0000","last_fullsized":"2024-03-06T07:28:14.199453+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:58:24.229417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.202580+0000","last_change":"2024-03-06T07:28:13.190077+0000","last_active":"2024-03-06T07:28:14.202580+0000","last_peered":"2024-03-06T07:28:14.202580+0000","last_clean":"2024-03-06T07:28:14.202580+0000","last_became_active":"2024-03-06T07:28:13.189717+0000","last_became_peered":"2024-03-06T07:28:13.189717+0000","last_unstale":"2024-03-06T07:28:14.202580+0000","last_undegraded":"2024-03-06T07:28:14.202580+0000","last_fullsized":"2024-03-06T07:28:14.202580+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:33:14.195257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.195395+0000","last_change":"2024-03-06T07:28:14.186496+0000","last_active":"2024-03-06T07:28:14.195395+0000","last_peered":"2024-03-06T07:28:14.195395+0000","last_clean":"2024-03-06T07:28:14.195395+0000","last_became_active":"2024-03-06T07:28:14.184383+0000","last_became_peered":"2024-03-06T07:28:14.184383+0000","last_unstale":"2024-03-06T07:28:14.195395+0000","last_undegraded":"2024-03-06T07:28:14.195395+0000","last_fullsized":"2024-03-06T07:28:14.195395+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"34'1","reported_seq":13,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204973+0000","last_change":"2024-03-06T07:28:13.180100+0000","last_active":"2024-03-06T07:28:14.204973+0000","last_peered":"2024-03-06T07:28:14.204973+0000","last_clean":"2024-03-06T07:28:14.204973+0000","last_became_active":"2024-03-06T07:28:13.179659+0000","last_became_peered":"2024-03-06T07:28:13.179659+0000","last_unstale":"2024-03-06T07:28:14.204973+0000","last_undegraded":"2024-03-06T07:28:14.204973+0000","last_fullsized":"2024-03-06T07:28:14.204973+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:41:52.569686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.196960+0000","last_change":"2024-03-06T07:28:14.192826+0000","last_active":"2024-03-06T07:28:14.196960+0000","last_peered":"2024-03-06T07:28:14.196960+0000","last_clean":"2024-03-06T07:28:14.196960+0000","last_became_active":"2024-03-06T07:28:14.192397+0000","last_became_peered":"2024-03-06T07:28:14.192397+0000","last_unstale":"2024-03-06T07:28:14.196960+0000","last_undegraded":"2024-03-06T07:28:14.196960+0000","last_fullsized":"2024-03-06T07:28:14.196960+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:49.056762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.165638+0000","last_change":"2024-03-06T07:28:13.183894+0000","last_active":"2024-03-06T07:28:14.165638+0000","last_peered":"2024-03-06T07:28:14.165638+0000","last_clean":"2024-03-06T07:28:14.165638+0000","last_became_active":"2024-03-06T07:28:13.181717+0000","last_became_peered":"2024-03-06T07:28:13.181717+0000","last_unstale":"2024-03-06T07:28:14.165638+0000","last_undegraded":"2024-03-06T07:28:14.165638+0000","last_fullsized":"2024-03-06T07:28:14.165638+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:48:06.392322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204911+0000","last_change":"2024-03-06T07:28:14.198849+0000","last_active":"2024-03-06T07:28:14.204911+0000","last_peered":"2024-03-06T07:28:14.204911+0000","last_clean":"2024-03-06T07:28:14.204911+0000","last_became_active":"2024-03-06T07:28:14.198586+0000","last_became_peered":"2024-03-06T07:28:14.198586+0000","last_unstale":"2024-03-06T07:28:14.204911+0000","last_undegraded":"2024-03-06T07:28:14.204911+0000","last_fullsized":"2024-03-06T07:28:14.204911+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:51.884783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204503+0000","last_change":"2024-03-06T07:28:13.186583+0000","last_active":"2024-03-06T07:28:14.204503+0000","last_peered":"2024-03-06T07:28:14.204503+0000","last_clean":"2024-03-06T07:28:14.204503+0000","last_became_active":"2024-03-06T07:28:13.180468+0000","last_became_peered":"2024-03-06T07:28:13.180468+0000","last_unstale":"2024-03-06T07:28:14.204503+0000","last_undegraded":"2024-03-06T07:28:14.204503+0000","last_fullsized":"2024-03-06T07:28:14.204503+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:05.656882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187756+0000","last_change":"2024-03-06T07:28:14.185820+0000","last_active":"2024-03-06T07:28:14.187756+0000","last_peered":"2024-03-06T07:28:14.187756+0000","last_clean":"2024-03-06T07:28:14.187756+0000","last_became_active":"2024-03-06T07:28:14.185239+0000","last_became_peered":"2024-03-06T07:28:14.185239+0000","last_unstale":"2024-03-06T07:28:14.187756+0000","last_undegraded":"2024-03-06T07:28:14.187756+0000","last_fullsized":"2024-03-06T07:28:14.187756+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:07.389377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.172678+0000","last_change":"2024-03-06T07:28:13.192355+0000","last_active":"2024-03-06T07:28:14.172678+0000","last_peered":"2024-03-06T07:28:14.172678+0000","last_clean":"2024-03-06T07:28:14.172678+0000","last_became_active":"2024-03-06T07:28:13.191972+0000","last_became_peered":"2024-03-06T07:28:13.191972+0000","last_unstale":"2024-03-06T07:28:14.172678+0000","last_undegraded":"2024-03-06T07:28:14.172678+0000","last_fullsized":"2024-03-06T07:28:14.172678+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:52.712300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.187615+0000","last_change":"2024-03-06T07:28:14.184594+0000","last_active":"2024-03-06T07:28:14.187615+0000","last_peered":"2024-03-06T07:28:14.187615+0000","last_clean":"2024-03-06T07:28:14.187615+0000","last_became_active":"2024-03-06T07:28:14.184309+0000","last_became_peered":"2024-03-06T07:28:14.184309+0000","last_unstale":"2024-03-06T07:28:14.187615+0000","last_undegraded":"2024-03-06T07:28:14.187615+0000","last_fullsized":"2024-03-06T07:28:14.187615+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:15:05.420328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.164642+0000","last_change":"2024-03-06T07:28:13.182647+0000","last_active":"2024-03-06T07:28:14.164642+0000","last_peered":"2024-03-06T07:28:14.164642+0000","last_clean":"2024-03-06T07:28:14.164642+0000","last_became_active":"2024-03-06T07:28:13.182323+0000","last_became_peered":"2024-03-06T07:28:13.182323+0000","last_unstale":"2024-03-06T07:28:14.164642+0000","last_undegraded":"2024-03-06T07:28:14.164642+0000","last_fullsized":"2024-03-06T07:28:14.164642+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:28.246598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.9","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.200408+0000","last_change":"2024-03-06T07:28:14.196038+0000","last_active":"2024-03-06T07:28:14.200408+0000","last_peered":"2024-03-06T07:28:14.200408+0000","last_clean":"2024-03-06T07:28:14.200408+0000","last_became_active":"2024-03-06T07:28:14.195710+0000","last_became_peered":"2024-03-06T07:28:14.195710+0000","last_unstale":"2024-03-06T07:28:14.200408+0000","last_undegraded":"2024-03-06T07:28:14.200408+0000","last_fullsized":"2024-03-06T07:28:14.200408+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:50:57.453127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.177691+0000","last_change":"2024-03-06T07:28:13.194684+0000","last_active":"2024-03-06T07:28:14.177691+0000","last_peered":"2024-03-06T07:28:14.177691+0000","last_clean":"2024-03-06T07:28:14.177691+0000","last_became_active":"2024-03-06T07:28:13.194336+0000","last_became_peered":"2024-03-06T07:28:13.194336+0000","last_unstale":"2024-03-06T07:28:14.177691+0000","last_undegraded":"2024-03-06T07:28:14.177691+0000","last_fullsized":"2024-03-06T07:28:14.177691+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:37:27.060079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.193911+0000","last_change":"2024-03-06T07:28:14.189760+0000","last_active":"2024-03-06T07:28:14.193911+0000","last_peered":"2024-03-06T07:28:14.193911+0000","last_clean":"2024-03-06T07:28:14.193911+0000","last_became_active":"2024-03-06T07:28:14.189445+0000","last_became_peered":"2024-03-06T07:28:14.189445+0000","last_unstale":"2024-03-06T07:28:14.193911+0000","last_undegraded":"2024-03-06T07:28:14.193911+0000","last_fullsized":"2024-03-06T07:28:14.193911+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:41.111808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"34'2","reported_seq":14,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204608+0000","last_change":"2024-03-06T07:28:13.190209+0000","last_active":"2024-03-06T07:28:14.204608+0000","last_peered":"2024-03-06T07:28:14.204608+0000","last_clean":"2024-03-06T07:28:14.204608+0000","last_became_active":"2024-03-06T07:28:13.189931+0000","last_became_peered":"2024-03-06T07:28:13.189931+0000","last_unstale":"2024-03-06T07:28:14.204608+0000","last_undegraded":"2024-03-06T07:28:14.204608+0000","last_fullsized":"2024-03-06T07:28:14.204608+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:05.656882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.204267+0000","last_change":"2024-03-06T07:28:14.201039+0000","last_active":"2024-03-06T07:28:14.204267+0000","last_peered":"2024-03-06T07:28:14.204267+0000","last_clean":"2024-03-06T07:28:14.204267+0000","last_became_active":"2024-03-06T07:28:14.199423+0000","last_became_peered":"2024-03-06T07:28:14.199423+0000","last_unstale":"2024-03-06T07:28:14.204267+0000","last_undegraded":"2024-03-06T07:28:14.204267+0000","last_fullsized":"2024-03-06T07:28:14.204267+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:28.736878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.162415+0000","last_change":"2024-03-06T07:28:13.183496+0000","last_active":"2024-03-06T07:28:14.162415+0000","last_peered":"2024-03-06T07:28:14.162415+0000","last_clean":"2024-03-06T07:28:14.162415+0000","last_became_active":"2024-03-06T07:28:13.182729+0000","last_became_peered":"2024-03-06T07:28:13.182729+0000","last_unstale":"2024-03-06T07:28:14.162415+0000","last_undegraded":"2024-03-06T07:28:14.162415+0000","last_fullsized":"2024-03-06T07:28:14.162415+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:22:30.293914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.183206+0000","last_change":"2024-03-06T07:28:14.178461+0000","last_active":"2024-03-06T07:28:14.183206+0000","last_peered":"2024-03-06T07:28:14.183206+0000","last_clean":"2024-03-06T07:28:14.183206+0000","last_became_active":"2024-03-06T07:28:14.178179+0000","last_became_peered":"2024-03-06T07:28:14.178179+0000","last_unstale":"2024-03-06T07:28:14.183206+0000","last_undegraded":"2024-03-06T07:28:14.183206+0000","last_fullsized":"2024-03-06T07:28:14.183206+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.168858+0000","last_change":"2024-03-06T07:28:13.180700+0000","last_active":"2024-03-06T07:28:14.168858+0000","last_peered":"2024-03-06T07:28:14.168858+0000","last_clean":"2024-03-06T07:28:14.168858+0000","last_became_active":"2024-03-06T07:28:13.180342+0000","last_became_peered":"2024-03-06T07:28:13.180342+0000","last_unstale":"2024-03-06T07:28:14.168858+0000","last_undegraded":"2024-03-06T07:28:14.168858+0000","last_fullsized":"2024-03-06T07:28:14.168858+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:48:06.392322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.190276+0000","last_change":"2024-03-06T07:28:14.186617+0000","last_active":"2024-03-06T07:28:14.190276+0000","last_peered":"2024-03-06T07:28:14.190276+0000","last_clean":"2024-03-06T07:28:14.190276+0000","last_became_active":"2024-03-06T07:28:14.186220+0000","last_became_peered":"2024-03-06T07:28:14.186220+0000","last_unstale":"2024-03-06T07:28:14.190276+0000","last_undegraded":"2024-03-06T07:28:14.190276+0000","last_fullsized":"2024-03-06T07:28:14.190276+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:37:28.077097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.162422+0000","last_change":"2024-03-06T07:28:13.188937+0000","last_active":"2024-03-06T07:28:14.162422+0000","last_peered":"2024-03-06T07:28:14.162422+0000","last_clean":"2024-03-06T07:28:14.162422+0000","last_became_active":"2024-03-06T07:28:13.188321+0000","last_became_peered":"2024-03-06T07:28:13.188321+0000","last_unstale":"2024-03-06T07:28:14.162422+0000","last_undegraded":"2024-03-06T07:28:14.162422+0000","last_fullsized":"2024-03-06T07:28:14.162422+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:43.842962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.191209+0000","last_change":"2024-03-06T07:28:14.188810+0000","last_active":"2024-03-06T07:28:14.191209+0000","last_peered":"2024-03-06T07:28:14.191209+0000","last_clean":"2024-03-06T07:28:14.191209+0000","last_became_active":"2024-03-06T07:28:14.188447+0000","last_became_peered":"2024-03-06T07:28:14.188447+0000","last_unstale":"2024-03-06T07:28:14.191209+0000","last_undegraded":"2024-03-06T07:28:14.191209+0000","last_fullsized":"2024-03-06T07:28:14.191209+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:58.088870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.175805+0000","last_change":"2024-03-06T07:28:13.194050+0000","last_active":"2024-03-06T07:28:14.175805+0000","last_peered":"2024-03-06T07:28:14.175805+0000","last_clean":"2024-03-06T07:28:14.175805+0000","last_became_active":"2024-03-06T07:28:13.193790+0000","last_became_peered":"2024-03-06T07:28:13.193790+0000","last_unstale":"2024-03-06T07:28:14.175805+0000","last_undegraded":"2024-03-06T07:28:14.175805+0000","last_fullsized":"2024-03-06T07:28:14.175805+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:22:30.293914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.206461+0000","last_change":"2024-03-06T07:28:14.198564+0000","last_active":"2024-03-06T07:28:14.206461+0000","last_peered":"2024-03-06T07:28:14.206461+0000","last_clean":"2024-03-06T07:28:14.206461+0000","last_became_active":"2024-03-06T07:28:14.198303+0000","last_became_peered":"2024-03-06T07:28:14.198303+0000","last_unstale":"2024-03-06T07:28:14.206461+0000","last_undegraded":"2024-03-06T07:28:14.206461+0000","last_fullsized":"2024-03-06T07:28:14.206461+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:16:24.047894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.164801+0000","last_change":"2024-03-06T07:28:13.188740+0000","last_active":"2024-03-06T07:28:14.164801+0000","last_peered":"2024-03-06T07:28:14.164801+0000","last_clean":"2024-03-06T07:28:14.164801+0000","last_became_active":"2024-03-06T07:28:13.188384+0000","last_became_peered":"2024-03-06T07:28:13.188384+0000","last_unstale":"2024-03-06T07:28:14.164801+0000","last_undegraded":"2024-03-06T07:28:14.164801+0000","last_fullsized":"2024-03-06T07:28:14.164801+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:21:28.545540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.192855+0000","last_change":"2024-03-06T07:28:14.190488+0000","last_active":"2024-03-06T07:28:14.192855+0000","last_peered":"2024-03-06T07:28:14.192855+0000","last_clean":"2024-03-06T07:28:14.192855+0000","last_became_active":"2024-03-06T07:28:14.187848+0000","last_became_peered":"2024-03-06T07:28:14.187848+0000","last_unstale":"2024-03-06T07:28:14.192855+0000","last_undegraded":"2024-03-06T07:28:14.192855+0000","last_fullsized":"2024-03-06T07:28:14.192855+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:18:25.361597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.168674+0000","last_change":"2024-03-06T07:28:13.190992+0000","last_active":"2024-03-06T07:28:14.168674+0000","last_peered":"2024-03-06T07:28:14.168674+0000","last_clean":"2024-03-06T07:28:14.168674+0000","last_became_active":"2024-03-06T07:28:13.190093+0000","last_became_peered":"2024-03-06T07:28:13.190093+0000","last_unstale":"2024-03-06T07:28:14.168674+0000","last_undegraded":"2024-03-06T07:28:14.168674+0000","last_fullsized":"2024-03-06T07:28:14.168674+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:59:43.842962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":9,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.189770+0000","last_change":"2024-03-06T07:28:14.187410+0000","last_active":"2024-03-06T07:28:14.189770+0000","last_peered":"2024-03-06T07:28:14.189770+0000","last_clean":"2024-03-06T07:28:14.189770+0000","last_became_active":"2024-03-06T07:28:14.186885+0000","last_became_peered":"2024-03-06T07:28:14.186885+0000","last_unstale":"2024-03-06T07:28:14.189770+0000","last_undegraded":"2024-03-06T07:28:14.189770+0000","last_fullsized":"2024-03-06T07:28:14.189770+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":33,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:13.138958+0000","last_clean_scrub_stamp":"2024-03-06T07:28:13.138958+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:03.460590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.172162+0000","last_change":"2024-03-06T07:28:13.196175+0000","last_active":"2024-03-06T07:28:14.172162+0000","last_peered":"2024-03-06T07:28:14.172162+0000","last_clean":"2024-03-06T07:28:14.172162+0000","last_became_active":"2024-03-06T07:28:13.195443+0000","last_became_peered":"2024-03-06T07:28:13.195443+0000","last_unstale":"2024-03-06T07:28:14.172162+0000","last_undegraded":"2024-03-06T07:28:14.172162+0000","last_fullsized":"2024-03-06T07:28:14.172162+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":32,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:12.121940+0000","last_clean_scrub_stamp":"2024-03-06T07:28:12.121940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:52:44.404427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":190,"reported_epoch":34,"state":"active+clean","last_fresh":"2024-03-06T07:28:14.174823+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:28:14.174823+0000","last_peered":"2024-03-06T07:28:14.174823+0000","last_clean":"2024-03-06T07:28:14.174823+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:28:14.174823+0000","last_undegraded":"2024-03-06T07:28:14.174823+0000","last_fullsized":"2024-03-06T07:28:14.174823+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542163,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27960,"kb_used_data":940,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343880,"statfs":{"total":96636764160,"available":96608133120,"internally_reserved":0,"allocated":962560,"data_stored":731849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542162,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27400,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344440,"statfs":{"total":96636764160,"available":96608706560,"internally_reserved":0,"allocated":389120,"data_stored":148872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542163,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":389120,"data_stored":148928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542162,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27404,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344436,"statfs":{"total":96636764160,"available":96608702464,"internally_reserved":0,"allocated":389120,"data_stored":148360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542162,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27380,"kb_used_data":360,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344460,"statfs":{"total":96636764160,"available":96608727040,"internally_reserved":0,"allocated":368640,"data_stored":141481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542163,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27400,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344440,"statfs":{"total":96636764160,"available":96608706560,"internally_reserved":0,"allocated":385024,"data_stored":148326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542164,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27412,"kb_used_data":388,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344428,"statfs":{"total":96636764160,"available":96608694272,"internally_reserved":0,"allocated":397312,"data_stored":149890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542161,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27976,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343864,"statfs":{"total":96636764160,"available":96608116736,"internally_reserved":0,"allocated":974848,"data_stored":738624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:28:19.098 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:28:19.098 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:28:19.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.360+0000 7f66d0200640 1 Processor -- start 2024-03-06T07:28:19.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.361+0000 7f66d0200640 1 -- start start 2024-03-06T07:28:19.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.362+0000 7f66d0200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f66c80b3f70 0x7f66c8160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.362+0000 7f66d0200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f66c8070ff0 0x7f66c80ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.362+0000 7f66d0200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66c8071440 con 0x7f66c815fc10 2024-03-06T07:28:19.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.362+0000 7f66d0200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f66c80b46c0 con 0x7f66c80b3f70 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.362+0000 7f66d0200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f66c80b49a0 con 0x7f66c8070ff0 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66cce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f66c80b3f70 0x7f66c8160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66cce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f66c80b3f70 0x7f66c8160700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60716/0 (socket says 172.21.15.86:60716) 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66cce00640 1 -- 172.21.15.86:0/943301846 learned_addr learned my addr 172.21.15.86:0/943301846 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66ce200640 1 --2- 172.21.15.86:0/943301846 >> v2:172.21.15.194:3300/0 conn(0x7f66c8070ff0 0x7f66c80ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66cce00640 1 -- 172.21.15.86:0/943301846 >> v2:172.21.15.194:3300/0 conn(0x7f66c8070ff0 msgr2=0x7f66c80ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66cce00640 1 --2- 172.21.15.86:0/943301846 >> v2:172.21.15.194:3300/0 conn(0x7f66c8070ff0 0x7f66c80ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66cce00640 1 -- 172.21.15.86:0/943301846 >> v1:172.21.15.194:6789/0 conn(0x7f66c815fc10 legacy=0x7f66c815fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.363+0000 7f66cce00640 1 -- 172.21.15.86:0/943301846 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66c8072c90 con 0x7f66c80b3f70 2024-03-06T07:28:19.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.365+0000 7f66cce00640 1 --2- 172.21.15.86:0/943301846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80b3f70 0x7f66c8160700 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f66c80cf6a0 tx=0x7f66b402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ca29467d8f4c8c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:19.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.365+0000 7f66c7e00640 1 -- 172.21.15.86:0/943301846 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66b4034070 con 0x7f66c80b3f70 2024-03-06T07:28:19.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.365+0000 7f66c7e00640 1 -- 172.21.15.86:0/943301846 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66b4030860 con 0x7f66c80b3f70 2024-03-06T07:28:19.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.365+0000 7f66c7e00640 1 -- 172.21.15.86:0/943301846 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66b4030c30 con 0x7f66c80b3f70 2024-03-06T07:28:19.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 -- 172.21.15.86:0/943301846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80b3f70 msgr2=0x7f66c8160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 --2- 172.21.15.86:0/943301846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80b3f70 0x7f66c8160700 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f66c80cf6a0 tx=0x7f66b402b9a0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 -- 172.21.15.86:0/943301846 shutdown_connections 2024-03-06T07:28:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 --2- 172.21.15.86:0/943301846 >> v2:172.21.15.194:3300/0 conn(0x7f66c8070ff0 0x7f66c80ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 --2- 172.21.15.86:0/943301846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80b3f70 0x7f66c8160700 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 -- 172.21.15.86:0/943301846 >> 172.21.15.86:0/943301846 conn(0x7f66c80b1940 msgr2=0x7f66c80b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 -- 172.21.15.86:0/943301846 shutdown_connections 2024-03-06T07:28:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.366+0000 7f66d0200640 1 -- 172.21.15.86:0/943301846 wait complete. 2024-03-06T07:28:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.368+0000 7f66d0200640 1 Processor -- start 2024-03-06T07:28:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.368+0000 7f66d0200640 1 -- start start 2024-03-06T07:28:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.369+0000 7f66d0200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8070ff0 0x7f66c8151dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.369+0000 7f66d0200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66c80b3f70 0x7f66c814dc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.369+0000 7f66d0200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c815fc10 0x7f66c815c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.369+0000 7f66d0200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66c8072a30 con 0x7f66c815fc10 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.369+0000 7f66d0200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66c80728b0 con 0x7f66c8070ff0 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.369+0000 7f66d0200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66c815c2e0 con 0x7f66c80b3f70 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c815fc10 0x7f66c815c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c815fc10 0x7f66c815c5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60728/0 (socket says 172.21.15.86:60728) 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 -- 172.21.15.86:0/1638076850 learned_addr learned my addr 172.21.15.86:0/1638076850 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66cd800640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8070ff0 0x7f66c8151dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66cce00640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66c80b3f70 0x7f66c814dc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 -- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66c80b3f70 msgr2=0x7f66c814dc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66c80b3f70 0x7f66c814dc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 -- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8070ff0 msgr2=0x7f66c8151dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8070ff0 0x7f66c8151dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 -- 172.21.15.86:0/1638076850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66c806cf50 con 0x7f66c815fc10 2024-03-06T07:28:19.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.370+0000 7f66ce200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c815fc10 0x7f66c815c5e0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f66c0009dd0 tx=0x7f66c000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.371+0000 7f66c5600640 1 -- 172.21.15.86:0/1638076850 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66c0018020 con 0x7f66c815fc10 2024-03-06T07:28:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.371+0000 7f66c5600640 1 -- 172.21.15.86:0/1638076850 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66c0014480 con 0x7f66c815fc10 2024-03-06T07:28:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.371+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66c806d230 con 0x7f66c815fc10 2024-03-06T07:28:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.371+0000 7f66c5600640 1 -- 172.21.15.86:0/1638076850 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66c0014870 con 0x7f66c815fc10 2024-03-06T07:28:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.371+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66c806d7f0 con 0x7f66c815fc10 2024-03-06T07:28:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.372+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f66c804f330 con 0x7f66c815fc10 2024-03-06T07:28:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.373+0000 7f66c5600640 1 -- 172.21.15.86:0/1638076850 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f66c0005d10 con 0x7f66c815fc10 2024-03-06T07:28:19.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.374+0000 7f66c5600640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66a806b500 0x7f66a806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.374+0000 7f66c5600640 1 -- 172.21.15.86:0/1638076850 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f66c008acb0 con 0x7f66c815fc10 2024-03-06T07:28:19.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.374+0000 7f66cd800640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66a806b500 0x7f66a806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.375+0000 7f66cd800640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66a806b500 0x7f66a806d9c0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f66b8007fd0 tx=0x7f66b8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:19.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.379+0000 7f66c5600640 1 -- 172.21.15.86:0/1638076850 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f66c0022070 con 0x7f66c815fc10 2024-03-06T07:28:19.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.642+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f66c80b2d40 con 0x7f66c815fc10 2024-03-06T07:28:19.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.647+0000 7f66c5600640 1 -- 172.21.15.86:0/1638076850 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v13) v1 ==== 190+0+147 (secure 0 0 0) 0x7f66c0010040 con 0x7f66c815fc10 2024-03-06T07:28:19.649 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:28:19.649 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:28:19.649 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:28:19.649 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:28:19.649 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:28:19.649 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66a806b500 msgr2=0x7f66a806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66a806b500 0x7f66a806d9c0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f66b8007fd0 tx=0x7f66b8009040 comp rx=0 tx=0).stop 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c815fc10 msgr2=0x7f66c815c5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c815fc10 0x7f66c815c5e0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f66c0009dd0 tx=0x7f66c000c490 comp rx=0 tx=0).stop 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 shutdown_connections 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66a806b500 0x7f66a806d9c0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c815fc10 0x7f66c815c5e0 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66c80b3f70 0x7f66c814dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 --2- 172.21.15.86:0/1638076850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8070ff0 0x7f66c8151dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.649+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 >> 172.21.15.86:0/1638076850 conn(0x7f66c80b1940 msgr2=0x7f66c80ce8f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:19.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.650+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 shutdown_connections 2024-03-06T07:28:19.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.650+0000 7f66d0200640 1 -- 172.21.15.86:0/1638076850 wait complete. 2024-03-06T07:28:19.670 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:19.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.869+0000 7f61a4800640 1 Processor -- start 2024-03-06T07:28:19.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.870+0000 7f61a4800640 1 -- start start 2024-03-06T07:28:19.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.871+0000 7f61a4800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f619c15f470 0x7f619c0d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.871+0000 7f61a4800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f619c0d1200 con 0x7f619c15e0d0 2024-03-06T07:28:19.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.871+0000 7f61a4800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f619c0cea60 con 0x7f619c0d0df0 2024-03-06T07:28:19.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.871+0000 7f61a4800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f619c0d2f60 con 0x7f619c15f470 2024-03-06T07:28:19.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.871+0000 7f61a1e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f619c15e0d0 0x7f619c0d1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50002/0 (socket says 172.21.15.86:50002) 2024-03-06T07:28:19.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.871+0000 7f61a1e00640 1 -- 172.21.15.86:0/4099006037 learned_addr learned my addr 172.21.15.86:0/4099006037 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a1400640 1 --2- 172.21.15.86:0/4099006037 >> v2:172.21.15.194:3300/0 conn(0x7f619c15f470 0x7f619c0d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 570841991 0 0) 0x7f6190001110 con 0x7f619c15e0d0 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6184003520 con 0x7f619c15e0d0 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4188416040 0 0) 0x7f6184003520 con 0x7f619c15e0d0 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 >> v2:172.21.15.194:3300/0 conn(0x7f619c15f470 msgr2=0x7f619c0d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 --2- 172.21.15.86:0/4099006037 >> v2:172.21.15.194:3300/0 conn(0x7f619c15f470 0x7f619c0d08b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 >> v1:172.21.15.194:6789/0 conn(0x7f619c0d0df0 legacy=0x7f619c160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f619c0d2350 con 0x7f619c15e0d0 2024-03-06T07:28:19.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6190001840 con 0x7f619c15e0d0 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 >> v1:172.21.15.86:6789/0 conn(0x7f619c15e0d0 legacy=0x7f619c0d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61840039b0 0x7f6184003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6184004800 0x7f6184004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840051c0 0x7f618400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f618400c0c0 con 0x7f61840039b0 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f618400c270 con 0x7f6184004800 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.872+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f618400c470 con 0x7f61840051c0 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a1e00640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61840039b0 0x7f6184003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a1400640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6184004800 0x7f6184004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a2800640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840051c0 0x7f618400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a1e00640 1 -- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840051c0 msgr2=0x7f618400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a1e00640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840051c0 0x7f618400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a1e00640 1 -- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6184004800 msgr2=0x7f6184004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:19.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a1e00640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6184004800 0x7f6184004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a1e00640 1 -- 172.21.15.86:0/4099006037 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6184003520 con 0x7f61840039b0 2024-03-06T07:28:19.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a2800640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840051c0 0x7f618400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:19.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.873+0000 7f61a2800640 1 -- 172.21.15.86:0/4099006037 reap_dead start 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.874+0000 7f61a1e00640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61840039b0 0x7f6184003dd0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f619000c4c0 tx=0x7f6184005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c16883226369b28 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.874+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f619000c0e0 con 0x7f61840039b0 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6190030070 con 0x7f61840039b0 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a0a00640 1 -- 172.21.15.86:0/4099006037 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61900058d0 con 0x7f61840039b0 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a4800640 1 -- 172.21.15.86:0/4099006037 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61840039b0 msgr2=0x7f6184003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a4800640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61840039b0 0x7f6184003dd0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f619000c4c0 tx=0x7f6184005830 comp rx=0 tx=0).stop 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a4800640 1 -- 172.21.15.86:0/4099006037 shutdown_connections 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a4800640 1 --2- 172.21.15.86:0/4099006037 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61840039b0 0x7f6184003dd0 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a4800640 1 -- 172.21.15.86:0/4099006037 >> 172.21.15.86:0/4099006037 conn(0x7f619c0d3ec0 msgr2=0x7f619c0d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:19.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a4800640 1 -- 172.21.15.86:0/4099006037 shutdown_connections 2024-03-06T07:28:19.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.875+0000 7f61a4800640 1 -- 172.21.15.86:0/4099006037 wait complete. 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.877+0000 7f61a4800640 1 Processor -- start 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.877+0000 7f61a4800640 1 -- start start 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a4800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840039b0 0x7f619c1525f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a4800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f619c1e8a20 0x7f619c1e8ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a4800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f619c15fdb0 0x7f619c160260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a4800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f619c0d2bf0 con 0x7f619c1e8a20 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a4800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f619c0d2910 con 0x7f619c15fdb0 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a4800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f619c15fb30 con 0x7f61840039b0 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f619c1e8a20 0x7f619c1e8ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f619c1e8a20 0x7f619c1e8ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60744/0 (socket says 172.21.15.86:60744) 2024-03-06T07:28:19.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.878+0000 7f61a1400640 1 -- 172.21.15.86:0/2402632323 learned_addr learned my addr 172.21.15.86:0/2402632323 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a1e00640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840039b0 0x7f619c1525f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a1400640 1 -- 172.21.15.86:0/2402632323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840039b0 msgr2=0x7f619c1525f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a1400640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840039b0 0x7f619c1525f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a1400640 1 -- 172.21.15.86:0/2402632323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f619c15fdb0 msgr2=0x7f619c160260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a1400640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f619c15fdb0 0x7f619c160260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a1400640 1 -- 172.21.15.86:0/2402632323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f619c1e94c0 con 0x7f619c1e8a20 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a1400640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f619c1e8a20 0x7f619c1e8ea0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f618800f660 tx=0x7f618800fb20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f6196000640 1 -- 172.21.15.86:0/2402632323 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6188019020 con 0x7f619c1e8a20 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f6196000640 1 -- 172.21.15.86:0/2402632323 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f618800d8e0 con 0x7f619c1e8a20 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f6196000640 1 -- 172.21.15.86:0/2402632323 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f618800dc90 con 0x7f619c1e8a20 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.879+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f619c0c8740 con 0x7f619c1e8a20 2024-03-06T07:28:19.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.880+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f619c0c8440 con 0x7f619c1e8a20 2024-03-06T07:28:19.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.881+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f619c04f330 con 0x7f619c1e8a20 2024-03-06T07:28:19.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.881+0000 7f6196000640 1 -- 172.21.15.86:0/2402632323 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f618801c020 con 0x7f619c1e8a20 2024-03-06T07:28:19.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.882+0000 7f6196000640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b500 0x7f617c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:19.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.882+0000 7f6196000640 1 -- 172.21.15.86:0/2402632323 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f618808a930 con 0x7f619c1e8a20 2024-03-06T07:28:19.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.883+0000 7f61a1e00640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b500 0x7f617c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:19.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.883+0000 7f61a1e00640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b500 0x7f617c06d9c0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f619000c490 tx=0x7f6190039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:19.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:19.887+0000 7f6196000640 1 -- 172.21.15.86:0/2402632323 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6188023020 con 0x7f619c1e8a20 2024-03-06T07:28:20.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.121+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f619c0b2dc0 con 0x7f619c1e8a20 2024-03-06T07:28:20.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.122+0000 7f6196000640 1 -- 172.21.15.86:0/2402632323 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f61880597a0 con 0x7f619c1e8a20 2024-03-06T07:28:20.124 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:20.124 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:20.125 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:20.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b500 msgr2=0x7f617c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b500 0x7f617c06d9c0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f619000c490 tx=0x7f6190039040 comp rx=0 tx=0).stop 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f619c1e8a20 msgr2=0x7f619c1e8ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f619c1e8a20 0x7f619c1e8ea0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f618800f660 tx=0x7f618800fb20 comp rx=0 tx=0).stop 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 shutdown_connections 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b500 0x7f617c06d9c0 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f619c15fdb0 0x7f619c160260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f619c1e8a20 0x7f619c1e8ea0 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 --2- 172.21.15.86:0/2402632323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61840039b0 0x7f619c1525f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 >> 172.21.15.86:0/2402632323 conn(0x7f619c0d3ec0 msgr2=0x7f619c0b45a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:20.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 shutdown_connections 2024-03-06T07:28:20.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.125+0000 7f61a4800640 1 -- 172.21.15.86:0/2402632323 wait complete. 2024-03-06T07:28:20.548 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 17, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:28:13.618810+0000', 'modified': '2024-03-06T07:28:16.688890+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 4995}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_4995': {'gid': 4995, 'name': 'd', 'rank': 0, 'incarnation': 13, 'state': 'up:active', 'state_seq': 4, 'addr': '172.21.15.194:6835/902800806', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.194:6833', 'nonce': 902800806}, {'type': 'v1', 'addr': '172.21.15.194:6835', 'nonce': 902800806}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [5], 'metadata_pool': 4, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 4995, 'qdb_cluster': [4995]} 2024-03-06T07:28:20.548 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:28:20.548 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.d status 2024-03-06T07:28:20.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.763+0000 7fe3abc00640 1 Processor -- start 2024-03-06T07:28:20.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.763+0000 7fe3abc00640 1 -- start start 2024-03-06T07:28:20.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3abc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe3a415e140 0x7fe3a40b3420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3abc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe3a406bfd0 0x7fe3a406c410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3abc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe3a415e520 con 0x7fe3a40b3960 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3abc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe3a406c950 con 0x7fe3a406bfd0 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3abc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe3a40b3e00 con 0x7fe3a415e140 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3a9c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe3a406bfd0 0x7fe3a406c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3a9c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe3a406bfd0 0x7fe3a406c410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60746/0 (socket says 172.21.15.86:60746) 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.765+0000 7fe3a9c00640 1 -- 172.21.15.86:0/583268068 learned_addr learned my addr 172.21.15.86:0/583268068 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.766+0000 7fe3a9200640 1 --2- 172.21.15.86:0/583268068 >> v2:172.21.15.194:3300/0 conn(0x7fe3a415e140 0x7fe3a40b3420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.766+0000 7fe3a9c00640 1 -- 172.21.15.86:0/583268068 >> v2:172.21.15.194:3300/0 conn(0x7fe3a415e140 msgr2=0x7fe3a40b3420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.766+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/583268068 >> v2:172.21.15.194:3300/0 conn(0x7fe3a415e140 0x7fe3a40b3420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.766+0000 7fe3a9c00640 1 -- 172.21.15.86:0/583268068 >> v1:172.21.15.86:6789/0 conn(0x7fe3a40b3960 legacy=0x7fe3a406d2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.766+0000 7fe3a9c00640 1 -- 172.21.15.86:0/583268068 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3a40b40e0 con 0x7fe3a406bfd0 2024-03-06T07:28:20.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.766+0000 7fe3a3400640 1 -- 172.21.15.86:0/583268068 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2904517400 0 0) 0x7fe3a415e520 con 0x7fe3a40b3960 2024-03-06T07:28:20.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.767+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/583268068 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a406bfd0 0x7fe3a406c410 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7fe3a4072ba0 tx=0x7fe39c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e59f7608f9827746 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:20.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.767+0000 7fe3a3400640 1 -- 172.21.15.86:0/583268068 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39c034070 con 0x7fe3a406bfd0 2024-03-06T07:28:20.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.767+0000 7fe3a3400640 1 -- 172.21.15.86:0/583268068 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe39c030860 con 0x7fe3a406bfd0 2024-03-06T07:28:20.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.767+0000 7fe3a3400640 1 -- 172.21.15.86:0/583268068 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39c030c30 con 0x7fe3a406bfd0 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 -- 172.21.15.86:0/583268068 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a406bfd0 msgr2=0x7fe3a406c410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 --2- 172.21.15.86:0/583268068 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a406bfd0 0x7fe3a406c410 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7fe3a4072ba0 tx=0x7fe39c02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 -- 172.21.15.86:0/583268068 shutdown_connections 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 --2- 172.21.15.86:0/583268068 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a406bfd0 0x7fe3a406c410 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 --2- 172.21.15.86:0/583268068 >> v2:172.21.15.194:3300/0 conn(0x7fe3a415e140 0x7fe3a40b3420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 -- 172.21.15.86:0/583268068 >> 172.21.15.86:0/583268068 conn(0x7fe3a4071380 msgr2=0x7fe3a4071760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 -- 172.21.15.86:0/583268068 shutdown_connections 2024-03-06T07:28:20.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.768+0000 7fe3abc00640 1 -- 172.21.15.86:0/583268068 wait complete. 2024-03-06T07:28:20.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.770+0000 7fe3abc00640 1 Processor -- start 2024-03-06T07:28:20.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.770+0000 7fe3abc00640 1 -- start start 2024-03-06T07:28:20.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3abc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe3a406bfd0 0x7fe3a414d730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3abc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe3a40b3960 0x7fe3a40ce600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3abc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a415e140 0x7fe3a41595c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3abc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a40b4980 con 0x7fe3a415e140 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3abc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a40b46a0 con 0x7fe3a406bfd0 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3abc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a406d9f0 con 0x7fe3a40b3960 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3a9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a415e140 0x7fe3a41595c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3a9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a415e140 0x7fe3a41595c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60756/0 (socket says 172.21.15.86:60756) 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2010053083 learned_addr learned my addr 172.21.15.86:0/2010053083 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3a9200640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe3a406bfd0 0x7fe3a414d730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.771+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe3a40b3960 0x7fe3a40ce600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe3a40b3960 msgr2=0x7fe3a40ce600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe3a40b3960 0x7fe3a40ce600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe3a406bfd0 msgr2=0x7fe3a414d730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe3a406bfd0 0x7fe3a414d730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2010053083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3a4158520 con 0x7fe3a415e140 2024-03-06T07:28:20.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a415e140 0x7fe3a41595c0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7fe39c009960 tx=0x7fe39c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:20.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a0c00640 1 -- 172.21.15.86:0/2010053083 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39c034040 con 0x7fe3a415e140 2024-03-06T07:28:20.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a0c00640 1 -- 172.21.15.86:0/2010053083 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe39c030660 con 0x7fe3a415e140 2024-03-06T07:28:20.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe3a4158800 con 0x7fe3a415e140 2024-03-06T07:28:20.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.772+0000 7fe3a0c00640 1 -- 172.21.15.86:0/2010053083 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39c039c80 con 0x7fe3a415e140 2024-03-06T07:28:20.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.773+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe3a4158dc0 con 0x7fe3a415e140 2024-03-06T07:28:20.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.775+0000 7fe3a0c00640 1 -- 172.21.15.86:0/2010053083 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe39c037070 con 0x7fe3a415e140 2024-03-06T07:28:20.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.775+0000 7fe3a0c00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe38006b4d0 0x7fe38006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.776+0000 7fe3a0c00640 1 -- 172.21.15.86:0/2010053083 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe39c0ac0c0 con 0x7fe3a415e140 2024-03-06T07:28:20.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.776+0000 7fe3a9200640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe38006b4d0 0x7fe38006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.776+0000 7fe3ae9ab740 1 Processor -- start 2024-03-06T07:28:20.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.776+0000 7fe3a9200640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe38006b4d0 0x7fe38006d990 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fe3940099c0 tx=0x7fe394003990 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:20.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.776+0000 7fe3ae9ab740 1 -- start start 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.777+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1c60750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.777+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1c90540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.777+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c60c90 0x5638a1c90a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.777+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638a1bca600 con 0x5638a1c8db90 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.777+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638a1b73930 con 0x5638a1c60c90 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.777+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5638a1bc66a0 con 0x5638a1c8d7b0 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1c90540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1c90540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60760/0 (socket says 172.21.15.86:60760) 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3176258134 learned_addr learned my addr 172.21.15.86:0/3176258134 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:20.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a9200640 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1c60750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c60c90 0x5638a1c90a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 msgr2=0x5638a1c60750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1c60750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c60c90 msgr2=0x5638a1c90a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c60c90 0x5638a1c90a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3176258134 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5638a18df2d0 con 0x5638a1c8db90 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c60c90 0x5638a1c90a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.778+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1c90540 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7fe38c0015c0 tx=0x7fe38c001a80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=99d8eab378d01bbd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe399600640 1 -- 172.21.15.86:0/3176258134 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe38c019070 con 0x5638a1c8db90 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe399600640 1 -- 172.21.15.86:0/3176258134 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe38c0050d0 con 0x5638a1c8db90 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe399600640 1 -- 172.21.15.86:0/3176258134 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe38c00e740 con 0x5638a1c8db90 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3176258134 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 msgr2=0x5638a1c90540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1c90540 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7fe38c0015c0 tx=0x7fe38c001a80 comp rx=0 tx=0).stop 2024-03-06T07:28:20.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3176258134 shutdown_connections 2024-03-06T07:28:20.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c60c90 0x5638a1c90a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1c90540 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3176258134 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1c60750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3176258134 >> 172.21.15.86:0/3176258134 conn(0x5638a1c629c0 msgr2=0x5638a1cab540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:20.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3176258134 shutdown_connections 2024-03-06T07:28:20.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.779+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3176258134 wait complete. 2024-03-06T07:28:20.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.780+0000 7fe3ae9ab740 1 -- start start 2024-03-06T07:28:20.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.780+0000 7fe3ae9ab740 1 Processor -- start 2024-03-06T07:28:20.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.781+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1ca5e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.781+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8db90 0x5638a1c931c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.781+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1c93700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.781+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638a1c64f60 con 0x5638a1ca63b0 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.781+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638a1b86d90 con 0x5638a1c8db90 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.781+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5638a1c64350 con 0x5638a1c8d7b0 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.781+0000 7fe3ae9ab740 10 client.0 _ll_register_callbacks cb 0x5638a1bc2350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1c93700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1c93700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60772/0 (socket says 172.21.15.86:60772) 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 -- 172.21.15.86:0/294614390 learned_addr learned my addr 172.21.15.86:0/294614390 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9200640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1ca5e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8db90 0x5638a1c931c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 -- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 msgr2=0x5638a1ca5e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1ca5e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 -- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8db90 msgr2=0x5638a1c931c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8db90 0x5638a1c931c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 -- 172.21.15.86:0/294614390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5638a19a96b0 con 0x5638a1ca63b0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9200640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1ca5e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1c93700 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7fe39c082980 tx=0x7fe39c07af50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a27741fdfd49fca server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.782+0000 7fe392000640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe392000640 1 -- 172.21.15.86:0/294614390 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39c082eb0 con 0x5638a1ca63b0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe392000640 1 -- 172.21.15.86:0/294614390 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe39c08fe10 con 0x5638a1ca63b0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe38c0050d0 con 0x5638a1ca63b0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe392000640 1 -- 172.21.15.86:0/294614390 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe39c0921b0 con 0x5638a1ca63b0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 10 client.5172 fetch_fsmap learned FSMap version 17 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=17,monmap=2+}) v3 -- 0x5638a1ca3d80 con 0x5638a1ca63b0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe392000640 1 -- 172.21.15.86:0/294614390 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe38c0050d0 con 0x5638a1ca63b0 2024-03-06T07:28:20.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe392000640 1 -- 172.21.15.86:0/294614390 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 17) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fe39c079b60 con 0x5638a1ca63b0 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 10 client.5172 fetch_fsmap finished waiting for FSMap version 17 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 10 client.5172 resolve_mds: resolved name 'd' to daemon mds.d 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 20 client.5172 populate_metadata read hostname 'smithi086' 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1ca4bd0 0x5638a1cbf7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 4 client.5172 mds_command: new command op to 4995 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.783+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 --> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5638a1bd7b40 con 0x5638a1ca4bd0 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.784+0000 7fe3a9200640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1ca4bd0 0x5638a1cbf7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.785+0000 7fe3a9200640 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1ca4bd0 0x5638a1cbf7b0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=16e674b68c57fcc server_cookie=368410c856c42ead in_seq=0 out_seq=0 2024-03-06T07:28:20.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.785+0000 7fe392000640 10 client.5172 ms_handle_connect on v2:172.21.15.194:6833/902800806 2024-03-06T07:28:20.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe392000640 1 -- 172.21.15.86:0/294614390 <== mds.0 v2:172.21.15.194:6833/902800806 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x5638a1bd7b40 con 0x5638a1ca4bd0 2024-03-06T07:28:20.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe392000640 10 client.5172 handle_command_reply: tid=0 2024-03-06T07:28:20.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 client.5172 shutdown 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 msgr2=0x5638a1c93700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1c93700 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7fe39c082980 tx=0x7fe39c07af50 comp rx=0 tx=0).stop 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 shutdown_connections 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1ca4bd0 0x5638a1cbf7b0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1c93700 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8db90 0x5638a1c931c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe392000640 0 client.5172 ms_handle_reset on v2:172.21.15.194:6833/902800806 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/294614390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8d7b0 0x5638a1ca5e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 >> 172.21.15.86:0/294614390 conn(0x5638a1c60c90 msgr2=0x5638a1c90530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 shutdown_connections 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.787+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/294614390 wait complete. 2024-03-06T07:28:20.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.788+0000 7fe3ae9ab740 20 client.5172 trim_cache size 0 max 16384 2024-03-06T07:28:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.806+0000 7fe3ae9ab740 1 Processor -- start 2024-03-06T07:28:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.806+0000 7fe3ae9ab740 1 -- start start 2024-03-06T07:28:20.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cbeb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1ca2480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1ca63b0 0x5638a1ca29c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a41ff2a0 con 0x5638a1c8db90 2024-03-06T07:28:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a41fe770 con 0x5638a1c8d7b0 2024-03-06T07:28:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a41feca0 con 0x5638a1ca63b0 2024-03-06T07:28:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1ca2480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1ca2480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60784/0 (socket says 172.21.15.86:60784) 2024-03-06T07:28:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3088873434 learned_addr learned my addr 172.21.15.86:0/3088873434 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:20.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a9200640 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cbeb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1ca63b0 0x5638a1ca29c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1ca63b0 msgr2=0x5638a1ca29c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1ca63b0 0x5638a1ca29c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 msgr2=0x5638a1cbeb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cbeb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.807+0000 7fe3a3e00640 1 -- 172.21.15.86:0/3088873434 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5638a1c5af60 con 0x5638a1c8db90 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.808+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1ca63b0 0x5638a1ca29c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.808+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1ca2480 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7fe38c007ce0 tx=0x7fe38c00a000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d9d585b13b2bcfb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.808+0000 7fe398c00640 1 -- 172.21.15.86:0/3088873434 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe38c023070 con 0x5638a1c8db90 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.808+0000 7fe398c00640 1 -- 172.21.15.86:0/3088873434 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe38c0052f0 con 0x5638a1c8db90 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.808+0000 7fe398c00640 1 -- 172.21.15.86:0/3088873434 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe38c019040 con 0x5638a1c8db90 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3088873434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 msgr2=0x5638a1ca2480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1ca2480 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7fe38c007ce0 tx=0x7fe38c00a000 comp rx=0 tx=0).stop 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3088873434 shutdown_connections 2024-03-06T07:28:20.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1ca63b0 0x5638a1ca29c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1c8db90 0x5638a1ca2480 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/3088873434 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cbeb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3088873434 >> 172.21.15.86:0/3088873434 conn(0x5638a1c60c90 msgr2=0x5638a1c67770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:20.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3088873434 shutdown_connections 2024-03-06T07:28:20.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/3088873434 wait complete. 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.809+0000 7fe3ae9ab740 1 -- start start 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.810+0000 7fe3ae9ab740 1 Processor -- start 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.810+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cb85e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8db90 0x5638a1d1ab00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3ae9ab740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1d1b040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638a1ca4a50 con 0x5638a1ca63b0 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638a1c64f60 con 0x5638a1c8d7b0 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3ae9ab740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5638a1ca4ff0 con 0x5638a1c8db90 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3ae9ab740 10 client.0 _ll_register_callbacks cb 0x7fe39c009550 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:28:20.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1d1b040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1d1b040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60790/0 (socket says 172.21.15.86:60790) 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2207637778 learned_addr learned my addr 172.21.15.86:0/2207637778 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9200640 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cb85e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8db90 msgr2=0x5638a1d1ab00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a3e00640 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8db90 0x5638a1d1ab00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8db90 0x5638a1d1ab00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 msgr2=0x5638a1cb85e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cb85e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 -- 172.21.15.86:0/2207637778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5638a1c940d0 con 0x5638a1ca63b0 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe3a9c00640 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1d1b040 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7fe39c0090b0 tx=0x7fe39c0aa420 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e5b7b70e69cb1376 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.811+0000 7fe392000640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe392000640 1 -- 172.21.15.86:0/2207637778 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39c099060 con 0x5638a1ca63b0 2024-03-06T07:28:20.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe392000640 1 -- 172.21.15.86:0/2207637778 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe39c076670 con 0x5638a1ca63b0 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe38c0052f0 con 0x5638a1ca63b0 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe392000640 1 -- 172.21.15.86:0/2207637778 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe39c09f050 con 0x5638a1ca63b0 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 10 client.5178 fetch_fsmap learned FSMap version 17 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=17,monmap=2+}) v3 -- 0x5638a1c940d0 con 0x5638a1ca63b0 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe392000640 1 -- 172.21.15.86:0/2207637778 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe38c0052f0 con 0x5638a1ca63b0 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe392000640 1 -- 172.21.15.86:0/2207637778 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 17) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fe39c0a2020 con 0x5638a1ca63b0 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 10 client.5178 fetch_fsmap finished waiting for FSMap version 17 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 10 client.5178 resolve_mds: resolved name 'd' to daemon mds.d 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 20 client.5178 populate_metadata read hostname 'smithi086' 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1d1edc0 0x5638a1d1f1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 4 client.5178 mds_command: new command op to 4995 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.812+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 --> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5638a1ca3550 con 0x5638a1d1edc0 2024-03-06T07:28:20.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.813+0000 7fe3a9200640 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1d1edc0 0x5638a1d1f1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.814+0000 7fe3a9200640 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1d1edc0 0x5638a1d1f1a0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cc09a437e9d581f8 server_cookie=aa7563cd5803bdf1 in_seq=0 out_seq=0 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.814+0000 7fe392000640 10 client.5178 ms_handle_connect on v2:172.21.15.194:6833/902800806 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.814+0000 7fe392000640 1 -- 172.21.15.86:0/2207637778 <== mds.0 v2:172.21.15.194:6833/902800806 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x5638a1ca3550 con 0x5638a1d1edc0 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.814+0000 7fe392000640 10 client.5178 handle_command_reply: tid=0 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.814+0000 7fe3ae9ab740 1 client.5178 shutdown 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 msgr2=0x5638a1d1b040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1d1b040 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7fe39c0090b0 tx=0x7fe39c0aa420 comp rx=0 tx=0).stop 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 shutdown_connections 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:6833/902800806,v1:172.21.15.194:6835/902800806] conn(0x5638a1d1edc0 0x5638a1d1f1a0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5638a1ca63b0 0x5638a1d1b040 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5638a1c8db90 0x5638a1d1ab00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 --2- 172.21.15.86:0/2207637778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5638a1c8d7b0 0x5638a1cb85e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 >> 172.21.15.86:0/2207637778 conn(0x5638a1c60c90 msgr2=0x5638a1c67770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe392000640 0 client.5178 ms_handle_reset on v2:172.21.15.194:6833/902800806 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 shutdown_connections 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 1 -- 172.21.15.86:0/2207637778 wait complete. 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.815+0000 7fe3ae9ab740 20 client.5178 trim_cache size 0 max 16384 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 4995, 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:28:20.820 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 6.6557363560000002, 2024-03-06T07:28:20.821 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 17, 2024-03-06T07:28:20.821 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 35, 2024-03-06T07:28:20.821 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 35, 2024-03-06T07:28:20.821 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 15.471978077999999 2024-03-06T07:28:20.821 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:28:20.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe38006b4d0 msgr2=0x7fe38006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe38006b4d0 0x7fe38006d990 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fe3940099c0 tx=0x7fe394003990 comp rx=0 tx=0).stop 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a415e140 msgr2=0x7fe3a41595c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a415e140 0x7fe3a41595c0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7fe39c009960 tx=0x7fe39c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 shutdown_connections 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe38006b4d0 0x7fe38006d990 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe3a415e140 0x7fe3a41595c0 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe3a40b3960 0x7fe3a40ce600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 --2- 172.21.15.86:0/2010053083 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe3a406bfd0 0x7fe3a414d730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 >> 172.21.15.86:0/2010053083 conn(0x7fe3a4071380 msgr2=0x7fe3a406f3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 shutdown_connections 2024-03-06T07:28:20.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:20.816+0000 7fe3abc00640 1 -- 172.21.15.86:0/2010053083 wait complete. 2024-03-06T07:28:20.835 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:28:21.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.030+0000 7fcbe9000640 1 Processor -- start 2024-03-06T07:28:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.030+0000 7fcbe9000640 1 -- start start 2024-03-06T07:28:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe9000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe9000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fcbe415f9d0 0x7fcbe415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe9000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcbe406ee70 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe9000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcbe4160370 con 0x7fcbe415f9d0 2024-03-06T07:28:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe9000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe406cef0 con 0x7fcbe4144c00 2024-03-06T07:28:21.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe2800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fcbe40ce0a0 0x7fcbe40ce4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50020/0 (socket says 172.21.15.86:50020) 2024-03-06T07:28:21.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe2800640 1 -- 172.21.15.86:0/2824327727 learned_addr learned my addr 172.21.15.86:0/2824327727 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:21.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe1e00640 1 --2- 172.21.15.86:0/2824327727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.032+0000 7fcbe1400640 1 --2- 172.21.15.86:0/2824327727 >> v2:172.21.15.194:3300/0 conn(0x7fcbe415f9d0 0x7fcbe415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.033+0000 7fcbe0a00640 1 -- 172.21.15.86:0/2824327727 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3948149203 0 0) 0x7fcbdc001490 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.033+0000 7fcbe0a00640 1 -- 172.21.15.86:0/2824327727 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcbc8003520 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.033+0000 7fcbe1e00640 1 -- 172.21.15.86:0/2824327727 >> v2:172.21.15.194:3300/0 conn(0x7fcbe415f9d0 msgr2=0x7fcbe415fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.033+0000 7fcbe1e00640 1 --2- 172.21.15.86:0/2824327727 >> v2:172.21.15.194:3300/0 conn(0x7fcbe415f9d0 0x7fcbe415fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.033+0000 7fcbe1e00640 1 -- 172.21.15.86:0/2824327727 >> v1:172.21.15.86:6789/0 conn(0x7fcbe40ce0a0 legacy=0x7fcbe40ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.033+0000 7fcbe1e00640 1 -- 172.21.15.86:0/2824327727 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbe406d1d0 con 0x7fcbe4144c00 2024-03-06T07:28:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.034+0000 7fcbe1e00640 1 --2- 172.21.15.86:0/2824327727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe406e930 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fcbd40097a0 tx=0x7fcbd402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c7d08c44fa133865 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe0a00640 1 -- 172.21.15.86:0/2824327727 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd4035020 con 0x7fcbe4144c00 2024-03-06T07:28:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe0a00640 1 -- 172.21.15.86:0/2824327727 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcbd4031830 con 0x7fcbe4144c00 2024-03-06T07:28:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe0a00640 1 -- 172.21.15.86:0/2824327727 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd4031be0 con 0x7fcbe4144c00 2024-03-06T07:28:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe9000640 1 -- 172.21.15.86:0/2824327727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 msgr2=0x7fcbe406e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe9000640 1 --2- 172.21.15.86:0/2824327727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe406e930 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fcbd40097a0 tx=0x7fcbd402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe9000640 1 -- 172.21.15.86:0/2824327727 shutdown_connections 2024-03-06T07:28:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe9000640 1 --2- 172.21.15.86:0/2824327727 >> v2:172.21.15.194:3300/0 conn(0x7fcbe415f9d0 0x7fcbe415fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe9000640 1 --2- 172.21.15.86:0/2824327727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe406e930 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.035+0000 7fcbe9000640 1 -- 172.21.15.86:0/2824327727 >> 172.21.15.86:0/2824327727 conn(0x7fcbe40b1730 msgr2=0x7fcbe40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.036+0000 7fcbe9000640 1 -- 172.21.15.86:0/2824327727 shutdown_connections 2024-03-06T07:28:21.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.036+0000 7fcbe9000640 1 -- 172.21.15.86:0/2824327727 wait complete. 2024-03-06T07:28:21.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.037+0000 7fcbe9000640 1 Processor -- start 2024-03-06T07:28:21.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.037+0000 7fcbe9000640 1 -- start start 2024-03-06T07:28:21.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.038+0000 7fcbe9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcbe40ce0a0 0x7fcbe41581c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:21.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.038+0000 7fcbe9000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe40ad860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:21.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.038+0000 7fcbe9000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcbe415f9d0 0x7fcbe41f7820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:21.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.038+0000 7fcbe9000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe4160730 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.038+0000 7fcbe9000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe406d790 con 0x7fcbe415f9d0 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.038+0000 7fcbe9000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe406cb50 con 0x7fcbe4144c00 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcbe40ce0a0 0x7fcbe41581c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcbe40ce0a0 0x7fcbe41581c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60806/0 (socket says 172.21.15.86:60806) 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 -- 172.21.15.86:0/3458670405 learned_addr learned my addr 172.21.15.86:0/3458670405 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1400640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe40ad860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe2800640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcbe415f9d0 0x7fcbe41f7820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 -- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 msgr2=0x7fcbe40ad860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe40ad860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 -- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcbe415f9d0 msgr2=0x7fcbe41f7820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcbe415f9d0 0x7fcbe41f7820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 -- 172.21.15.86:0/3458670405 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbe41f7040 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1400640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe40ad860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:28:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.039+0000 7fcbe1e00640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcbe40ce0a0 0x7fcbe41581c0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fcbd402b910 tx=0x7fcbd40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.040+0000 7fcbda000640 1 -- 172.21.15.86:0/3458670405 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd4035040 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.040+0000 7fcbda000640 1 -- 172.21.15.86:0/3458670405 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcbd4004450 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.040+0000 7fcbda000640 1 -- 172.21.15.86:0/3458670405 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd4003810 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.040+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbe40ae090 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.040+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbe40ae650 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.041+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbe404f330 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.042+0000 7fcbda000640 1 -- 172.21.15.86:0/3458670405 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcbd4038020 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.043+0000 7fcbda000640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcbc406b4d0 0x7fcbc406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:21.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.043+0000 7fcbda000640 1 -- 172.21.15.86:0/3458670405 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcbd40ac3c0 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.043+0000 7fcbe1400640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcbc406b4d0 0x7fcbc406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:21.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.044+0000 7fcbe1400640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcbc406b4d0 0x7fcbc406d990 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fcbe40b0ed0 tx=0x7fcbd0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:21.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.047+0000 7fcbda000640 1 -- 172.21.15.86:0/3458670405 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcbd407b1a0 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.293+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fcbe40b2de0 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.294+0000 7fcbda000640 1 -- 172.21.15.86:0/3458670405 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v17) v1 ==== 53+0+73 (secure 0 0 0) 0x7fcbd407aaa0 con 0x7fcbe40ce0a0 2024-03-06T07:28:21.296 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:28:21.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcbc406b4d0 msgr2=0x7fcbc406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:21.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcbc406b4d0 0x7fcbc406d990 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fcbe40b0ed0 tx=0x7fcbd0008040 comp rx=0 tx=0).stop 2024-03-06T07:28:21.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcbe40ce0a0 msgr2=0x7fcbe41581c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:21.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcbe40ce0a0 0x7fcbe41581c0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fcbd402b910 tx=0x7fcbd40046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 shutdown_connections 2024-03-06T07:28:21.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcbc406b4d0 0x7fcbc406d990 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcbe415f9d0 0x7fcbe41f7820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcbe4144c00 0x7fcbe40ad860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 --2- 172.21.15.86:0/3458670405 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcbe40ce0a0 0x7fcbe41581c0 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:21.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 >> 172.21.15.86:0/3458670405 conn(0x7fcbe40b1730 msgr2=0x7fcbe40afec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:21.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 shutdown_connections 2024-03-06T07:28:21.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:21.297+0000 7fcbe9000640 1 -- 172.21.15.86:0/3458670405 wait complete. 2024-03-06T07:28:21.316 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:28:21.317 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:28:21.317 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:21.317 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:28:21.364 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42365sec preferred_lft 42365sec 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:28:21.365 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:28:21.366 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:28:21.367 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:21.367 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:28:21.494 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:28:21.498 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:21.499 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:28:21.551 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:28:21.551 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:28:21.551 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:28:21.551 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:28:21.551 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:28:21.552 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:21.552 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:28:21.552 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:28:21.552 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:28:21.552 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:28:21.552 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:28:21.552 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:28:21.734 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:21.734 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:28:21.785 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:28:21.785 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:28:21.786 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:28:21.786 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:21.786 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:28:21.879 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:28:21.879 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:28:21.931 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:28:21.931 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:28:21.931 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:28:21.980 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:28:21.980 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:21.980 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:28:22.119 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:28:22.119 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:28:22.152 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:28:22.152 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:28:22.153 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:28:22.153 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:28:22.153 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:28:22.155 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:28:22.155 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:28:22.155 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:28:22.155 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:28:22.155 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:28:22.158 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:28:22.158 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:22.158 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:28:22.239 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:22.239 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:28:22.316 INFO:teuthology.orchestra.run.smithi086.stdout:client4577 2024-03-06T07:28:22.316 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:28:22.316 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> subdir = "delete_me" 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> size = 1024 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> file_count = 1000 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> os.mkdir(os.path.join(mountpoint, subdir)) 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> for i in range(0, file_count): 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> filename = "{0}_{1}.bin".format(i, size) 2024-03-06T07:28:22.369 DEBUG:teuthology.orchestra.run.smithi086:> with open(os.path.join(mountpoint, subdir, filename), '"'"'w'"'"') as f: 2024-03-06T07:28:22.370 DEBUG:teuthology.orchestra.run.smithi086:> f.write(size * '"'"'x'"'"') 2024-03-06T07:28:22.370 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:28:22.716 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:28:22.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:28:23.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:28:23.548 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:23.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.759+0000 7fba6bc00640 1 Processor -- start 2024-03-06T07:28:23.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.759+0000 7fba6bc00640 1 -- start start 2024-03-06T07:28:23.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba6bc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fba640af1c0 0x7fba6415eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba6bc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fba64070d20 0x7fba6406cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba6bc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fba64071d30 con 0x7fba6406d1a0 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba6bc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fba640711d0 con 0x7fba64070d20 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba6bc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fba64071490 con 0x7fba640af1c0 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba63e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fba64070d20 0x7fba6406cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba63e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fba64070d20 0x7fba6406cc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60816/0 (socket says 172.21.15.86:60816) 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba69c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fba6406d1a0 0x7fba6406e4d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50030/0 (socket says 172.21.15.86:50030) 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba63e00640 1 -- 172.21.15.86:0/2586878258 learned_addr learned my addr 172.21.15.86:0/2586878258 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:23.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba69200640 1 --2- 172.21.15.86:0/2586878258 >> v2:172.21.15.194:3300/0 conn(0x7fba640af1c0 0x7fba6415eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:23.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba63e00640 1 -- 172.21.15.86:0/2586878258 >> v2:172.21.15.194:3300/0 conn(0x7fba640af1c0 msgr2=0x7fba6415eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:23.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba63e00640 1 --2- 172.21.15.86:0/2586878258 >> v2:172.21.15.194:3300/0 conn(0x7fba640af1c0 0x7fba6415eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:23.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba63e00640 1 -- 172.21.15.86:0/2586878258 >> v1:172.21.15.86:6789/0 conn(0x7fba6406d1a0 legacy=0x7fba6406e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:23.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.761+0000 7fba63e00640 1 -- 172.21.15.86:0/2586878258 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba64071770 con 0x7fba64070d20 2024-03-06T07:28:23.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.763+0000 7fba63e00640 1 --2- 172.21.15.86:0/2586878258 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba64070d20 0x7fba6406cc60 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fba50009870 tx=0x7fba5002b990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=19e1879563930dea server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:23.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.763+0000 7fba63400640 1 -- 172.21.15.86:0/2586878258 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba50034070 con 0x7fba64070d20 2024-03-06T07:28:23.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.763+0000 7fba63400640 1 -- 172.21.15.86:0/2586878258 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba50030850 con 0x7fba64070d20 2024-03-06T07:28:23.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.763+0000 7fba63400640 1 -- 172.21.15.86:0/2586878258 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba50030c20 con 0x7fba64070d20 2024-03-06T07:28:23.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.763+0000 7fba6bc00640 1 -- 172.21.15.86:0/2586878258 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba64070d20 msgr2=0x7fba6406cc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:23.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.763+0000 7fba6bc00640 1 --2- 172.21.15.86:0/2586878258 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba64070d20 0x7fba6406cc60 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fba50009870 tx=0x7fba5002b990 comp rx=0 tx=0).stop 2024-03-06T07:28:23.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.764+0000 7fba6bc00640 1 -- 172.21.15.86:0/2586878258 shutdown_connections 2024-03-06T07:28:23.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.764+0000 7fba6bc00640 1 --2- 172.21.15.86:0/2586878258 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba64070d20 0x7fba6406cc60 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:23.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.764+0000 7fba6bc00640 1 --2- 172.21.15.86:0/2586878258 >> v2:172.21.15.194:3300/0 conn(0x7fba640af1c0 0x7fba6415eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:23.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.764+0000 7fba6bc00640 1 -- 172.21.15.86:0/2586878258 >> 172.21.15.86:0/2586878258 conn(0x7fba640b1730 msgr2=0x7fba640b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:23.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.764+0000 7fba6bc00640 1 -- 172.21.15.86:0/2586878258 shutdown_connections 2024-03-06T07:28:23.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.764+0000 7fba6bc00640 1 -- 172.21.15.86:0/2586878258 wait complete. 2024-03-06T07:28:23.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.765+0000 7fba6bc00640 1 Processor -- start 2024-03-06T07:28:23.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.765+0000 7fba6bc00640 1 -- start start 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.766+0000 7fba6bc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fba6406d1a0 0x7fba641484d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.766+0000 7fba6bc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fba64070d20 0x7fba640cce00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.766+0000 7fba6bc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba640af1c0 0x7fba64152410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.766+0000 7fba6bc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba6406ebe0 con 0x7fba640af1c0 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.766+0000 7fba6bc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba640b30b0 con 0x7fba64070d20 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.766+0000 7fba6bc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fba6406ed60 con 0x7fba6406d1a0 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba640af1c0 0x7fba64152410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba640af1c0 0x7fba64152410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60822/0 (socket says 172.21.15.86:60822) 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 -- 172.21.15.86:0/604165457 learned_addr learned my addr 172.21.15.86:0/604165457 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69200640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fba6406d1a0 0x7fba641484d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba63e00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fba64070d20 0x7fba640cce00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:23.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 -- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fba6406d1a0 msgr2=0x7fba641484d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:23.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fba6406d1a0 0x7fba641484d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:23.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 -- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fba64070d20 msgr2=0x7fba640cce00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:23.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fba64070d20 0x7fba640cce00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:23.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.767+0000 7fba69c00640 1 -- 172.21.15.86:0/604165457 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba640cd340 con 0x7fba640af1c0 2024-03-06T07:28:23.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.768+0000 7fba69c00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba640af1c0 0x7fba64152410 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fba5c00c760 tx=0x7fba5c00cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:23.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.768+0000 7fba60c00640 1 -- 172.21.15.86:0/604165457 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba5c017070 con 0x7fba640af1c0 2024-03-06T07:28:23.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.768+0000 7fba60c00640 1 -- 172.21.15.86:0/604165457 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba5c00a640 con 0x7fba640af1c0 2024-03-06T07:28:23.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.768+0000 7fba60c00640 1 -- 172.21.15.86:0/604165457 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba5c00a9f0 con 0x7fba640af1c0 2024-03-06T07:28:23.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.768+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fba641552e0 con 0x7fba640af1c0 2024-03-06T07:28:23.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.768+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fba64158e90 con 0x7fba640af1c0 2024-03-06T07:28:23.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.769+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fba6404f330 con 0x7fba640af1c0 2024-03-06T07:28:23.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.770+0000 7fba60c00640 1 -- 172.21.15.86:0/604165457 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fba5c01a070 con 0x7fba640af1c0 2024-03-06T07:28:23.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.771+0000 7fba60c00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fba4406b4d0 0x7fba4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:23.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.771+0000 7fba60c00640 1 -- 172.21.15.86:0/604165457 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fba5c0896e0 con 0x7fba640af1c0 2024-03-06T07:28:23.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.771+0000 7fba69200640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fba4406b4d0 0x7fba4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:23.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.772+0000 7fba69200640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fba4406b4d0 0x7fba4406d990 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fba58006fd0 tx=0x7fba58008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:23.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:23.776+0000 7fba60c00640 1 -- 172.21.15.86:0/604165457 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fba64071d30 con 0x7fba640af1c0 2024-03-06T07:28:24.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.011+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fba6415ec20 con 0x7fba640af1c0 2024-03-06T07:28:24.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.012+0000 7fba60c00640 1 -- 172.21.15.86:0/604165457 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fba5c057e70 con 0x7fba640af1c0 2024-03-06T07:28:24.014 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:24.014 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:24.014 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fba4406b4d0 msgr2=0x7fba4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fba4406b4d0 0x7fba4406d990 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fba58006fd0 tx=0x7fba58008040 comp rx=0 tx=0).stop 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba640af1c0 msgr2=0x7fba64152410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba640af1c0 0x7fba64152410 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fba5c00c760 tx=0x7fba5c00cc20 comp rx=0 tx=0).stop 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 shutdown_connections 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fba4406b4d0 0x7fba4406d990 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fba640af1c0 0x7fba64152410 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fba64070d20 0x7fba640cce00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 --2- 172.21.15.86:0/604165457 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fba6406d1a0 0x7fba641484d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 >> 172.21.15.86:0/604165457 conn(0x7fba640b1730 msgr2=0x7fba640afa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:24.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.015+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 shutdown_connections 2024-03-06T07:28:24.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.016+0000 7fba6bc00640 1 -- 172.21.15.86:0/604165457 wait complete. 2024-03-06T07:28:24.037 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json flush journal 2024-03-06T07:28:24.635 INFO:teuthology.orchestra.run.smithi194.stdout:{"message":"","return_code":0} 2024-03-06T07:28:24.635 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:28:24.636 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:28:24.636 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:28:24.636 DEBUG:teuthology.orchestra.run.smithi086:> import stat 2024-03-06T07:28:24.636 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:28:24.636 DEBUG:teuthology.orchestra.run.smithi086:> print(os.stat("/home/ubuntu/cephtest/mnt.0/delete_me").st_ino) 2024-03-06T07:28:24.636 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:28:24.745 INFO:teuthology.orchestra.run.smithi086.stdout:1099511627776 2024-03-06T07:28:24.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:28:24.757 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_metadata stat 10000000000.00000000 2024-03-06T07:28:24.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.904+0000 7f5b45b86f40 1 Processor -- start 2024-03-06T07:28:24.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.904+0000 7f5b45b86f40 1 -- start start 2024-03-06T07:28:24.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.905+0000 7f5b45b86f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed726c80 0x5588ed7b7030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.905+0000 7f5b45b86f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed728740 0x5588ed7bc620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.905+0000 7f5b45b86f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 0x5588ed7b0af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.905+0000 7f5b45b86f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5588ed72c5e0 con 0x5588ed728740 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.905+0000 7f5b45b86f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5588ed689fd0 con 0x5588ed7ba630 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.905+0000 7f5b45b86f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5588ed72c880 con 0x5588ed726c80 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed728740 0x5588ed7bc620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed728740 0x5588ed7bc620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60826/0 (socket says 172.21.15.86:60826) 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 -- 172.21.15.86:0/1170533909 learned_addr learned my addr 172.21.15.86:0/1170533909 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42e00640 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed726c80 0x5588ed7b7030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b43800640 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 0x5588ed7b0af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 -- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed726c80 msgr2=0x5588ed7b7030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:24.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed726c80 0x5588ed7b7030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 -- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 msgr2=0x5588ed7b0af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:24.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 0x5588ed7b0af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 -- 172.21.15.86:0/1170533909 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5588ed70cdc0 con 0x5588ed728740 2024-03-06T07:28:24.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.906+0000 7f5b42400640 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed728740 0x5588ed7bc620 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f5b30009850 tx=0x7f5b30005ac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=652dcdc32f89f365 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:24.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b41000640 1 -- 172.21.15.86:0/1170533909 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b300088d0 con 0x5588ed728740 2024-03-06T07:28:24.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b41000640 1 -- 172.21.15.86:0/1170533909 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b30018070 con 0x5588ed728740 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b41000640 1 -- 172.21.15.86:0/1170533909 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b30008c90 con 0x5588ed728740 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 -- 172.21.15.86:0/1170533909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed728740 msgr2=0x5588ed7bc620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed728740 0x5588ed7bc620 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f5b30009850 tx=0x7f5b30005ac0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 -- 172.21.15.86:0/1170533909 shutdown_connections 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 0x5588ed7b0af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed728740 0x5588ed7bc620 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/1170533909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed726c80 0x5588ed7b7030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 -- 172.21.15.86:0/1170533909 >> 172.21.15.86:0/1170533909 conn(0x5588ed710bf0 msgr2=0x5588ed7bd0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 -- 172.21.15.86:0/1170533909 shutdown_connections 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.907+0000 7f5b45b86f40 1 -- 172.21.15.86:0/1170533909 wait complete. 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.908+0000 7f5b45b86f40 1 Processor -- start 2024-03-06T07:28:24.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.908+0000 7f5b45b86f40 1 -- start start 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b45b86f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed726c80 0x5588ed7b97b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b45b86f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed728740 0x5588ed7a80f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b45b86f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 0x5588ed6e4bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b45b86f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5588ed70d060 con 0x5588ed726c80 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b45b86f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5588ed7b2bc0 con 0x5588ed7ba630 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b45b86f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5588ed7b7570 con 0x5588ed728740 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b42e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed726c80 0x5588ed7b97b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b42e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed726c80 0x5588ed7b97b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60842/0 (socket says 172.21.15.86:60842) 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.909+0000 7f5b42e00640 1 -- 172.21.15.86:0/4019406906 learned_addr learned my addr 172.21.15.86:0/4019406906 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b42400640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed728740 0x5588ed7a80f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b43800640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 0x5588ed6e4bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b42e00640 1 -- 172.21.15.86:0/4019406906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed728740 msgr2=0x5588ed7a80f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b42e00640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5588ed728740 0x5588ed7a80f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b42e00640 1 -- 172.21.15.86:0/4019406906 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 msgr2=0x5588ed6e4bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b42e00640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5588ed7ba630 0x5588ed6e4bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b42e00640 1 -- 172.21.15.86:0/4019406906 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5588ed6e50f0 con 0x5588ed726c80 2024-03-06T07:28:24.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b42e00640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed726c80 0x5588ed7b97b0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f5b3400a4f0 tx=0x7f5b34002fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:24.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b3b400640 1 -- 172.21.15.86:0/4019406906 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b34061020 con 0x5588ed726c80 2024-03-06T07:28:24.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b3b400640 1 -- 172.21.15.86:0/4019406906 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b34052990 con 0x5588ed726c80 2024-03-06T07:28:24.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x5588ed6e5360 con 0x5588ed726c80 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.910+0000 7f5b3b400640 1 -- 172.21.15.86:0/4019406906 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b34057040 con 0x5588ed726c80 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.911+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5588ed6e18b0 con 0x5588ed726c80 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.913+0000 7f5b3b400640 1 -- 172.21.15.86:0/4019406906 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5b34064020 con 0x5588ed726c80 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.913+0000 7f5b3b400640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b24069f00 0x7f5b2406c3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.914+0000 7f5b3b400640 1 -- 172.21.15.86:0/4019406906 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5b340ca5c0 con 0x5588ed726c80 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.914+0000 7f5b42400640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b24069f00 0x7f5b2406c3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.914+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x5588ed8545c0 0x5588ed8549c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:24.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.914+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:1 4.10 4:0cf6ad0f:::10000000000.00000000:head [stat] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e35) v9 -- 0x5588ed7b0ab0 con 0x5588ed8545c0 2024-03-06T07:28:24.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.914+0000 7f5b43800640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x5588ed8545c0 0x5588ed8549c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:24.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.914+0000 7f5b42400640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b24069f00 0x7f5b2406c3c0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x5588ed7b9fc0 tx=0x7f5b30012000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:24.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.915+0000 7f5b43800640 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x5588ed8545c0 0x5588ed8549c0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:24.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.915+0000 7f5b43800640 1 -- 172.21.15.86:0/4019406906 <== osd.3 v2:172.21.15.86:6824/3540554891 1 ==== osd_op_reply(1 10000000000.00000000 [stat out=16b] v0'0 uv10 ondisk = 0) v8 ==== 164+0+16 (crc 0 0 0) 0x7f5b3c006d50 con 0x5588ed8545c0 2024-03-06T07:28:24.917 INFO:teuthology.orchestra.run.smithi086.stdout:cephfs_metadata/10000000000.00000000 mtime 2024-03-06T07:28:24.000000+0000, size 0 2024-03-06T07:28:24.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.915+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x5588ed8545c0 msgr2=0x5588ed8549c0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.915+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x5588ed8545c0 0x5588ed8549c0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b24069f00 msgr2=0x7f5b2406c3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5b24069f00 0x7f5b2406c3c0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x5588ed7b9fc0 tx=0x7f5b30012000 comp rx=0 tx=0).stop 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed726c80 msgr2=0x5588ed7b97b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 --2- 172.21.15.86:0/4019406906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5588ed726c80 0x5588ed7b97b0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f5b3400a4f0 tx=0x7f5b34002fb0 comp rx=0 tx=0).stop 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b43800640 1 -- 172.21.15.86:0/4019406906 reap_dead start 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 shutdown_connections 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 >> 172.21.15.86:0/4019406906 conn(0x5588ed710bf0 msgr2=0x5588ed7a8c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 shutdown_connections 2024-03-06T07:28:24.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:24.916+0000 7f5b45b86f40 1 -- 172.21.15.86:0/4019406906 wait complete. 2024-03-06T07:28:24.922 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:24.922 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -rf delete_me) 2024-03-06T07:28:29.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:29.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.218+0000 7fc75ec00640 1 Processor -- start 2024-03-06T07:28:29.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.218+0000 7fc75ec00640 1 -- start start 2024-03-06T07:28:29.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.219+0000 7fc75ec00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc75815eff0 0x7fc758070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.219+0000 7fc75ec00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc758071d30 con 0x7fc7580af1c0 2024-03-06T07:28:29.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.219+0000 7fc75ec00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc75815f4a0 con 0x7fc75806e4d0 2024-03-06T07:28:29.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.219+0000 7fc75ec00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc758071260 con 0x7fc75815eff0 2024-03-06T07:28:29.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.219+0000 7fc757e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc7580af1c0 0x7fc75806cc60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:60310/0 (socket says 172.21.15.86:60310) 2024-03-06T07:28:29.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757e00640 1 -- 172.21.15.86:0/2753853076 learned_addr learned my addr 172.21.15.86:0/2753853076 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:29.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc74f400640 1 --2- 172.21.15.86:0/2753853076 >> v2:172.21.15.194:3300/0 conn(0x7fc75815eff0 0x7fc758070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2081702901 0 0) 0x7fc748001110 con 0x7fc7580af1c0 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc73c003520 con 0x7fc7580af1c0 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2775866702 0 0) 0x7fc73c003520 con 0x7fc7580af1c0 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 >> v2:172.21.15.194:3300/0 conn(0x7fc75815eff0 msgr2=0x7fc758070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 --2- 172.21.15.86:0/2753853076 >> v2:172.21.15.194:3300/0 conn(0x7fc75815eff0 0x7fc758070d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 >> v1:172.21.15.194:6789/0 conn(0x7fc75806e4d0 legacy=0x7fc7580b35d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc758071490 con 0x7fc7580af1c0 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc748001840 con 0x7fc7580af1c0 2024-03-06T07:28:29.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 >> v1:172.21.15.86:6789/0 conn(0x7fc7580af1c0 legacy=0x7fc75806cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc73c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.220+0000 7fc757400640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc73c004800 0x7fc73c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757400640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc73c0051c0 0x7fc73c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc73c00c090 con 0x7fc73c0039e0 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc73c00c210 con 0x7fc73c0051c0 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc73c00c410 con 0x7fc73c004800 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757e00640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc73c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757e00640 1 -- 172.21.15.86:0/2753853076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc73c004800 msgr2=0x7fc73c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757e00640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc73c004800 0x7fc73c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757e00640 1 -- 172.21.15.86:0/2753853076 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc73c0051c0 msgr2=0x7fc73c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757e00640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc73c0051c0 0x7fc73c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc757e00640 1 -- 172.21.15.86:0/2753853076 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc73c003520 con 0x7fc73c0039e0 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.221+0000 7fc75cc00640 1 -- 172.21.15.86:0/2753853076 reap_dead start 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.222+0000 7fc757e00640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc73c003e00 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fc74800c4c0 tx=0x7fc73c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cda5dbbfd16828ef server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.222+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc74800c0e0 con 0x7fc73c0039e0 2024-03-06T07:28:29.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.222+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc748030070 con 0x7fc73c0039e0 2024-03-06T07:28:29.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.222+0000 7fc757400640 1 -- 172.21.15.86:0/2753853076 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7480058d0 con 0x7fc73c0039e0 2024-03-06T07:28:29.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.223+0000 7fc75ec00640 1 -- 172.21.15.86:0/2753853076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 msgr2=0x7fc73c003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:29.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.223+0000 7fc75ec00640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc73c003e00 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fc74800c4c0 tx=0x7fc73c005830 comp rx=0 tx=0).stop 2024-03-06T07:28:29.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.223+0000 7fc75ec00640 1 -- 172.21.15.86:0/2753853076 shutdown_connections 2024-03-06T07:28:29.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.223+0000 7fc75ec00640 1 --2- 172.21.15.86:0/2753853076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc73c003e00 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.223+0000 7fc75ec00640 1 -- 172.21.15.86:0/2753853076 >> 172.21.15.86:0/2753853076 conn(0x7fc7580b1730 msgr2=0x7fc7580b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:29.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.223+0000 7fc75ec00640 1 -- 172.21.15.86:0/2753853076 shutdown_connections 2024-03-06T07:28:29.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.223+0000 7fc75ec00640 1 -- 172.21.15.86:0/2753853076 wait complete. 2024-03-06T07:28:29.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.225+0000 7fc75ec00640 1 Processor -- start 2024-03-06T07:28:29.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.225+0000 7fc75ec00640 1 -- start start 2024-03-06T07:28:29.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc75ec00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc758157c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc75ec00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc758152430 0x7fc7580ce260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc75ec00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc7580b0710 0x7fc7581490d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc75ec00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7580b30b0 con 0x7fc73c0039e0 2024-03-06T07:28:29.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc75ec00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc758071a50 con 0x7fc758152430 2024-03-06T07:28:29.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc75ec00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc758070d20 con 0x7fc7580b0710 2024-03-06T07:28:29.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc757e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc758157c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc757e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc758157c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58216/0 (socket says 172.21.15.86:58216) 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.226+0000 7fc757e00640 1 -- 172.21.15.86:0/504940824 learned_addr learned my addr 172.21.15.86:0/504940824 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc74f400640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc758152430 0x7fc7580ce260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc757e00640 1 -- 172.21.15.86:0/504940824 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc7580b0710 msgr2=0x7fc7581490d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc757e00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc7580b0710 0x7fc7581490d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc757e00640 1 -- 172.21.15.86:0/504940824 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc758152430 msgr2=0x7fc7580ce260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc757e00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc758152430 0x7fc7580ce260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc757e00640 1 -- 172.21.15.86:0/504940824 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc758156e90 con 0x7fc73c0039e0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc757e00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc758157c50 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fc74800c5e0 tx=0x7fc748001c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc754c00640 1 -- 172.21.15.86:0/504940824 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc748045070 con 0x7fc73c0039e0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc754c00640 1 -- 172.21.15.86:0/504940824 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc748001110 con 0x7fc73c0039e0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc754c00640 1 -- 172.21.15.86:0/504940824 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc748007270 con 0x7fc73c0039e0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.227+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc758157170 con 0x7fc73c0039e0 2024-03-06T07:28:29.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.228+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc758157730 con 0x7fc73c0039e0 2024-03-06T07:28:29.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.229+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc75804f330 con 0x7fc73c0039e0 2024-03-06T07:28:29.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.229+0000 7fc754c00640 1 -- 172.21.15.86:0/504940824 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc748007410 con 0x7fc73c0039e0 2024-03-06T07:28:29.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.231+0000 7fc754c00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc73406b4d0 0x7fc73406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:29.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.231+0000 7fc754c00640 1 -- 172.21.15.86:0/504940824 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc7480ab670 con 0x7fc73c0039e0 2024-03-06T07:28:29.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.231+0000 7fc74f400640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc73406b4d0 0x7fc73406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:29.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.232+0000 7fc74f400640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc73406b4d0 0x7fc73406d990 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fc7580b1280 tx=0x7fc740008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:29.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.236+0000 7fc754c00640 1 -- 172.21.15.86:0/504940824 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc758071d30 con 0x7fc73c0039e0 2024-03-06T07:28:29.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.578+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc75815ded0 con 0x7fc73c0039e0 2024-03-06T07:28:29.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.579+0000 7fc754c00640 1 -- 172.21.15.86:0/504940824 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fc74807a3c0 con 0x7fc73c0039e0 2024-03-06T07:28:29.580 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:29.580 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:29.581 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:29.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc73406b4d0 msgr2=0x7fc73406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:29.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc73406b4d0 0x7fc73406d990 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fc7580b1280 tx=0x7fc740008040 comp rx=0 tx=0).stop 2024-03-06T07:28:29.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 msgr2=0x7fc758157c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:29.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc758157c50 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fc74800c5e0 tx=0x7fc748001c10 comp rx=0 tx=0).stop 2024-03-06T07:28:29.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 shutdown_connections 2024-03-06T07:28:29.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc73406b4d0 0x7fc73406d990 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc7580b0710 0x7fc7581490d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc758152430 0x7fc7580ce260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 --2- 172.21.15.86:0/504940824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc73c0039e0 0x7fc758157c50 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:29.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 >> 172.21.15.86:0/504940824 conn(0x7fc7580b1730 msgr2=0x7fc7580ae1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:29.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 shutdown_connections 2024-03-06T07:28:29.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:29.581+0000 7fc75ec00640 1 -- 172.21.15.86:0/504940824 wait complete. 2024-03-06T07:28:29.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json flush journal 2024-03-06T07:28:29.953 INFO:teuthology.orchestra.run.smithi194.stdout:{"message":"","return_code":0} 2024-03-06T07:28:29.953 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:28:29.953 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:30.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.140+0000 7fe92ac00640 1 Processor -- start 2024-03-06T07:28:30.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.140+0000 7fe92ac00640 1 -- start start 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe92ac00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe92415f9d0 0x7fe92415fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe92ac00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe924070b10 0x7fe924070f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe92ac00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe92406cef0 con 0x7fe924144c00 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe92ac00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe92406d1d0 con 0x7fe924070b10 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe92ac00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe92406d4b0 con 0x7fe92415f9d0 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe923e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe924144c00 0x7fe92406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:60326/0 (socket says 172.21.15.86:60326) 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe923e00640 1 -- 172.21.15.86:0/2715738392 learned_addr learned my addr 172.21.15.86:0/2715738392 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:30.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.142+0000 7fe928c00640 1 --2- 172.21.15.86:0/2715738392 >> v2:172.21.15.86:3300/0 conn(0x7fe924070b10 0x7fe924070f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.143+0000 7fe91b400640 1 --2- 172.21.15.86:0/2715738392 >> v2:172.21.15.194:3300/0 conn(0x7fe92415f9d0 0x7fe92415fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.143+0000 7fe923400640 1 -- 172.21.15.86:0/2715738392 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4082701734 0 0) 0x7fe92406cef0 con 0x7fe924144c00 2024-03-06T07:28:30.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.143+0000 7fe928c00640 1 -- 172.21.15.86:0/2715738392 >> v2:172.21.15.194:3300/0 conn(0x7fe92415f9d0 msgr2=0x7fe92415fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:30.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.143+0000 7fe928c00640 1 --2- 172.21.15.86:0/2715738392 >> v2:172.21.15.194:3300/0 conn(0x7fe92415f9d0 0x7fe92415fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.143+0000 7fe928c00640 1 -- 172.21.15.86:0/2715738392 >> v1:172.21.15.86:6789/0 conn(0x7fe924144c00 legacy=0x7fe92406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.143+0000 7fe928c00640 1 -- 172.21.15.86:0/2715738392 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe92406d790 con 0x7fe924070b10 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.144+0000 7fe928c00640 1 --2- 172.21.15.86:0/2715738392 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924070b10 0x7fe924070f20 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7fe9240b26f0 tx=0x7fe91c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3afb787e3b5a4fde server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe923400640 1 -- 172.21.15.86:0/2715738392 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe91c035020 con 0x7fe924070b10 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe923400640 1 -- 172.21.15.86:0/2715738392 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe91c031830 con 0x7fe924070b10 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe923400640 1 -- 172.21.15.86:0/2715738392 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe91c031c00 con 0x7fe924070b10 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe92ac00640 1 -- 172.21.15.86:0/2715738392 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924070b10 msgr2=0x7fe924070f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2715738392 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924070b10 0x7fe924070f20 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7fe9240b26f0 tx=0x7fe91c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe92ac00640 1 -- 172.21.15.86:0/2715738392 shutdown_connections 2024-03-06T07:28:30.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2715738392 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924070b10 0x7fe924070f20 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2715738392 >> v2:172.21.15.194:3300/0 conn(0x7fe92415f9d0 0x7fe92415fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.145+0000 7fe92ac00640 1 -- 172.21.15.86:0/2715738392 >> 172.21.15.86:0/2715738392 conn(0x7fe9240b1730 msgr2=0x7fe9240b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:30.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.146+0000 7fe92ac00640 1 -- 172.21.15.86:0/2715738392 shutdown_connections 2024-03-06T07:28:30.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.146+0000 7fe92ac00640 1 -- 172.21.15.86:0/2715738392 wait complete. 2024-03-06T07:28:30.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.147+0000 7fe92ac00640 1 Processor -- start 2024-03-06T07:28:30.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.147+0000 7fe92ac00640 1 -- start start 2024-03-06T07:28:30.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.148+0000 7fe92ac00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe924070b10 0x7fe924142ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.148+0000 7fe92ac00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924144c00 0x7fe9241e7230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.148+0000 7fe92ac00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe92415f9d0 0x7fe9241f5010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.148+0000 7fe92ac00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe92406ccd0 con 0x7fe924144c00 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.148+0000 7fe92ac00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe92406cb50 con 0x7fe92415f9d0 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.148+0000 7fe92ac00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe92406df90 con 0x7fe924070b10 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924144c00 0x7fe9241e7230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924144c00 0x7fe9241e7230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58242/0 (socket says 172.21.15.86:58242) 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 -- 172.21.15.86:0/2536748046 learned_addr learned my addr 172.21.15.86:0/2536748046 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe923e00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe924070b10 0x7fe924142ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe928c00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe92415f9d0 0x7fe9241f5010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 -- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe924070b10 msgr2=0x7fe924142ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe924070b10 0x7fe924142ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 -- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe92415f9d0 msgr2=0x7fe9241f5010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe92415f9d0 0x7fe9241f5010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 -- 172.21.15.86:0/2536748046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe92415d430 con 0x7fe924144c00 2024-03-06T07:28:30.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.149+0000 7fe91b400640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924144c00 0x7fe9241e7230 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7fe90c00d620 tx=0x7fe90c00dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:30.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.150+0000 7fe920c00640 1 -- 172.21.15.86:0/2536748046 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe90c00b620 con 0x7fe924144c00 2024-03-06T07:28:30.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.150+0000 7fe920c00640 1 -- 172.21.15.86:0/2536748046 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe90c019070 con 0x7fe924144c00 2024-03-06T07:28:30.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.150+0000 7fe920c00640 1 -- 172.21.15.86:0/2536748046 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe90c00ba00 con 0x7fe924144c00 2024-03-06T07:28:30.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.150+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe92415d6a0 con 0x7fe924144c00 2024-03-06T07:28:30.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.150+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe92415dc60 con 0x7fe924144c00 2024-03-06T07:28:30.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.151+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe92404f330 con 0x7fe924144c00 2024-03-06T07:28:30.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.152+0000 7fe920c00640 1 -- 172.21.15.86:0/2536748046 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe90c00bba0 con 0x7fe924144c00 2024-03-06T07:28:30.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.153+0000 7fe920c00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90006b4d0 0x7fe90006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.153+0000 7fe920c00640 1 -- 172.21.15.86:0/2536748046 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe90c08a6e0 con 0x7fe924144c00 2024-03-06T07:28:30.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.153+0000 7fe923e00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90006b4d0 0x7fe90006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.154+0000 7fe923e00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90006b4d0 0x7fe90006d990 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fe9140012e0 tx=0x7fe914007ef0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:30.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.157+0000 7fe920c00640 1 -- 172.21.15.86:0/2536748046 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe90c0594c0 con 0x7fe924144c00 2024-03-06T07:28:30.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.409+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe9241e6600 con 0x7fe924144c00 2024-03-06T07:28:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.410+0000 7fe920c00640 1 -- 172.21.15.86:0/2536748046 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fe90c058dc0 con 0x7fe924144c00 2024-03-06T07:28:30.412 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:30.412 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:30.413 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:30.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90006b4d0 msgr2=0x7fe90006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90006b4d0 0x7fe90006d990 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fe9140012e0 tx=0x7fe914007ef0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924144c00 msgr2=0x7fe9241e7230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924144c00 0x7fe9241e7230 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7fe90c00d620 tx=0x7fe90c00dae0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 shutdown_connections 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe90006b4d0 0x7fe90006d990 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe92415f9d0 0x7fe9241f5010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe924144c00 0x7fe9241e7230 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 --2- 172.21.15.86:0/2536748046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe924070b10 0x7fe924142ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 >> 172.21.15.86:0/2536748046 conn(0x7fe9240b1730 msgr2=0x7fe9240af5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 shutdown_connections 2024-03-06T07:28:30.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.413+0000 7fe92ac00640 1 -- 172.21.15.86:0/2536748046 wait complete. 2024-03-06T07:28:30.431 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json perf dump mds_cache strays_created 2024-03-06T07:28:30.652 INFO:teuthology.orchestra.run.smithi194.stdout:{"mds_cache":{"strays_created":1001}} 2024-03-06T07:28:30.652 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 1001 } } 2024-03-06T07:28:30.652 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:30.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.872+0000 7f38c5600640 1 Processor -- start 2024-03-06T07:28:30.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.872+0000 7f38c5600640 1 -- start start 2024-03-06T07:28:30.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.874+0000 7f38c5600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38c0144c00 0x7f38c006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.874+0000 7f38c5600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f38c015f9d0 0x7f38c015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.874+0000 7f38c5600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f38c006ee70 con 0x7f38c00ce0a0 2024-03-06T07:28:30.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.874+0000 7f38c5600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38c006cef0 con 0x7f38c0144c00 2024-03-06T07:28:30.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.874+0000 7f38c5600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38c006d1d0 con 0x7f38c015f9d0 2024-03-06T07:28:30.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.874+0000 7f38bea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38c0144c00 0x7f38c006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.875+0000 7f38b7e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f38c015f9d0 0x7f38c015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.875+0000 7f38bea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38c0144c00 0x7f38c006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58248/0 (socket says 172.21.15.86:58248) 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.875+0000 7f38bea00640 1 -- 172.21.15.86:0/421164564 learned_addr learned my addr 172.21.15.86:0/421164564 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.875+0000 7f38bea00640 1 -- 172.21.15.86:0/421164564 >> v2:172.21.15.194:3300/0 conn(0x7f38c015f9d0 msgr2=0x7f38c015fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.875+0000 7f38bea00640 1 --2- 172.21.15.86:0/421164564 >> v2:172.21.15.194:3300/0 conn(0x7f38c015f9d0 0x7f38c015fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.875+0000 7f38bea00640 1 -- 172.21.15.86:0/421164564 >> v1:172.21.15.194:6789/0 conn(0x7f38c00ce0a0 legacy=0x7f38c00ce4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.875+0000 7f38bea00640 1 -- 172.21.15.86:0/421164564 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38c006d4b0 con 0x7f38c0144c00 2024-03-06T07:28:30.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.876+0000 7f38bea00640 1 --2- 172.21.15.86:0/421164564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c0144c00 0x7f38c006e930 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f38c00b26f0 tx=0x7f38b002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c1d7059f900b95ed server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:30.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.876+0000 7f38be000640 1 -- 172.21.15.86:0/421164564 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38b0034070 con 0x7f38c0144c00 2024-03-06T07:28:30.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.876+0000 7f38be000640 1 -- 172.21.15.86:0/421164564 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f38b0030860 con 0x7f38c0144c00 2024-03-06T07:28:30.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.876+0000 7f38be000640 1 -- 172.21.15.86:0/421164564 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38b0030c30 con 0x7f38c0144c00 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 -- 172.21.15.86:0/421164564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c0144c00 msgr2=0x7f38c006e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 --2- 172.21.15.86:0/421164564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c0144c00 0x7f38c006e930 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f38c00b26f0 tx=0x7f38b002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 -- 172.21.15.86:0/421164564 shutdown_connections 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 --2- 172.21.15.86:0/421164564 >> v2:172.21.15.194:3300/0 conn(0x7f38c015f9d0 0x7f38c015fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 --2- 172.21.15.86:0/421164564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c0144c00 0x7f38c006e930 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 -- 172.21.15.86:0/421164564 >> 172.21.15.86:0/421164564 conn(0x7f38c00b1730 msgr2=0x7f38c00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 -- 172.21.15.86:0/421164564 shutdown_connections 2024-03-06T07:28:30.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.877+0000 7f38c5600640 1 -- 172.21.15.86:0/421164564 wait complete. 2024-03-06T07:28:30.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.879+0000 7f38c5600640 1 Processor -- start 2024-03-06T07:28:30.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.879+0000 7f38c5600640 1 -- start start 2024-03-06T07:28:30.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38c5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c00ce0a0 0x7f38c00b4780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38c5600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38c0144c00 0x7f38c015a210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38c5600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38c015f9d0 0x7f38c013c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38c5600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38c006cb50 con 0x7f38c00ce0a0 2024-03-06T07:28:30.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38c5600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38c0160730 con 0x7f38c0144c00 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38c5600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38c006ccd0 con 0x7f38c015f9d0 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38bea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c00ce0a0 0x7f38c00b4780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38bea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c00ce0a0 0x7f38c00b4780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58256/0 (socket says 172.21.15.86:58256) 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38bea00640 1 -- 172.21.15.86:0/2792424706 learned_addr learned my addr 172.21.15.86:0/2792424706 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.880+0000 7f38bf400640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38c015f9d0 0x7f38c013c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38b7e00640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38c0144c00 0x7f38c015a210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38bea00640 1 -- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38c015f9d0 msgr2=0x7f38c013c5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38bea00640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38c015f9d0 0x7f38c013c5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38bea00640 1 -- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38c0144c00 msgr2=0x7f38c015a210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38bea00640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38c0144c00 0x7f38c015a210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38bea00640 1 -- 172.21.15.86:0/2792424706 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38c013d260 con 0x7f38c00ce0a0 2024-03-06T07:28:30.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38bea00640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c00ce0a0 0x7f38c00b4780 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f38b0000c00 tx=0x7f38b0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:30.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38b6a00640 1 -- 172.21.15.86:0/2792424706 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38b0048070 con 0x7f38c00ce0a0 2024-03-06T07:28:30.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38b6a00640 1 -- 172.21.15.86:0/2792424706 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f38b0030660 con 0x7f38c00ce0a0 2024-03-06T07:28:30.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38b6a00640 1 -- 172.21.15.86:0/2792424706 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38b0034040 con 0x7f38c00ce0a0 2024-03-06T07:28:30.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f38c00b3790 con 0x7f38c00ce0a0 2024-03-06T07:28:30.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.881+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f38c013d520 con 0x7f38c00ce0a0 2024-03-06T07:28:30.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.883+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f38c00cdc80 con 0x7f38c00ce0a0 2024-03-06T07:28:30.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.883+0000 7f38b6a00640 1 -- 172.21.15.86:0/2792424706 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f38b004b070 con 0x7f38c00ce0a0 2024-03-06T07:28:30.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.884+0000 7f38b6a00640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f389c06b4d0 0x7f389c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:30.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.884+0000 7f38b6a00640 1 -- 172.21.15.86:0/2792424706 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f38b00ac300 con 0x7f38c00ce0a0 2024-03-06T07:28:30.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.885+0000 7f38b7e00640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f389c06b4d0 0x7f389c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:30.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.885+0000 7f38b7e00640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f389c06b4d0 0x7f389c06d990 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f38c0139c90 tx=0x7f38a8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:30.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:30.889+0000 7f38b6a00640 1 -- 172.21.15.86:0/2792424706 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f38c00cdc80 con 0x7f38c00ce0a0 2024-03-06T07:28:31.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.121+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f38c004f330 con 0x7f38c00ce0a0 2024-03-06T07:28:31.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.122+0000 7f38b6a00640 1 -- 172.21.15.86:0/2792424706 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f38b007b200 con 0x7f38c00ce0a0 2024-03-06T07:28:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:31.124 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:31.124 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:31.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.124+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f389c06b4d0 msgr2=0x7f389c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:31.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.124+0000 7f38c5600640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f389c06b4d0 0x7f389c06d990 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f38c0139c90 tx=0x7f38a8008040 comp rx=0 tx=0).stop 2024-03-06T07:28:31.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.124+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c00ce0a0 msgr2=0x7f38c00b4780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:31.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.124+0000 7f38c5600640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c00ce0a0 0x7f38c00b4780 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f38b0000c00 tx=0x7f38b0004730 comp rx=0 tx=0).stop 2024-03-06T07:28:31.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 shutdown_connections 2024-03-06T07:28:31.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f389c06b4d0 0x7f389c06d990 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:31.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38c015f9d0 0x7f38c013c5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:31.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38c0144c00 0x7f38c015a210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:31.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 --2- 172.21.15.86:0/2792424706 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38c00ce0a0 0x7f38c00b4780 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:31.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 >> 172.21.15.86:0/2792424706 conn(0x7f38c00b1730 msgr2=0x7f38c00b0510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:31.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 shutdown_connections 2024-03-06T07:28:31.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:31.125+0000 7f38c5600640 1 -- 172.21.15.86:0/2792424706 wait complete. 2024-03-06T07:28:31.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:28:31.376 INFO:teuthology.orchestra.run.smithi194.stdout:{"mds_cache":{"strays_enqueued":1000}} 2024-03-06T07:28:31.376 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 1000 } } 2024-03-06T07:28:31.376 DEBUG:tasks.ceph_test_case:wait_until_equal: 1000 != 1001, waiting (timeout=600)... 2024-03-06T07:28:36.377 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:36.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.591+0000 7f0ae1200640 1 Processor -- start 2024-03-06T07:28:36.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.591+0000 7f0ae1200640 1 -- start start 2024-03-06T07:28:36.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.592+0000 7f0ae1200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0adc15acb0 0x7f0adc0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:36.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.592+0000 7f0ae1200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0adc0b4680 0x7f0adc06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:36.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.592+0000 7f0ae1200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0adc15b090 con 0x7f0adc072340 2024-03-06T07:28:36.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.592+0000 7f0ae1200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0adc0b2a30 con 0x7f0adc0b4680 2024-03-06T07:28:36.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.592+0000 7f0ae1200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0adc0b4360 con 0x7f0adc15acb0 2024-03-06T07:28:36.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0adc0b4680 0x7f0adc06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:36.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0adc0b4680 0x7f0adc06e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36184/0 (socket says 172.21.15.86:36184) 2024-03-06T07:28:36.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad3e00640 1 -- 172.21.15.86:0/34853876 learned_addr learned my addr 172.21.15.86:0/34853876 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:36.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad9e00640 1 --2- 172.21.15.86:0/34853876 >> v2:172.21.15.194:3300/0 conn(0x7f0adc15acb0 0x7f0adc0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:36.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad3e00640 1 -- 172.21.15.86:0/34853876 >> v2:172.21.15.194:3300/0 conn(0x7f0adc15acb0 msgr2=0x7f0adc0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:36.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad3e00640 1 --2- 172.21.15.86:0/34853876 >> v2:172.21.15.194:3300/0 conn(0x7f0adc15acb0 0x7f0adc0b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad3e00640 1 -- 172.21.15.86:0/34853876 >> v1:172.21.15.86:6789/0 conn(0x7f0adc072340 legacy=0x7f0adc0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:36.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad3e00640 1 -- 172.21.15.86:0/34853876 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0adc0b2cf0 con 0x7f0adc0b4680 2024-03-06T07:28:36.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.593+0000 7f0ad9400640 1 -- 172.21.15.86:0/34853876 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1564542035 0 0) 0x7f0adc15b090 con 0x7f0adc072340 2024-03-06T07:28:36.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.594+0000 7f0ad3e00640 1 --2- 172.21.15.86:0/34853876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc0b4680 0x7f0adc06e4f0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f0adc0b27f0 tx=0x7f0ac402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd3e698e6411e21f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ad9400640 1 -- 172.21.15.86:0/34853876 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ac4034070 con 0x7f0adc0b4680 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ad9400640 1 -- 172.21.15.86:0/34853876 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ac4030860 con 0x7f0adc0b4680 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ad9400640 1 -- 172.21.15.86:0/34853876 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ac4030c30 con 0x7f0adc0b4680 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 -- 172.21.15.86:0/34853876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc0b4680 msgr2=0x7f0adc06e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 --2- 172.21.15.86:0/34853876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc0b4680 0x7f0adc06e4f0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f0adc0b27f0 tx=0x7f0ac402b9a0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 -- 172.21.15.86:0/34853876 shutdown_connections 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 --2- 172.21.15.86:0/34853876 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc0b4680 0x7f0adc06e4f0 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 --2- 172.21.15.86:0/34853876 >> v2:172.21.15.194:3300/0 conn(0x7f0adc15acb0 0x7f0adc0b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 -- 172.21.15.86:0/34853876 >> 172.21.15.86:0/34853876 conn(0x7f0adc0b1730 msgr2=0x7f0adc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:36.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 -- 172.21.15.86:0/34853876 shutdown_connections 2024-03-06T07:28:36.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.595+0000 7f0ae1200640 1 -- 172.21.15.86:0/34853876 wait complete. 2024-03-06T07:28:36.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.597+0000 7f0ae1200640 1 Processor -- start 2024-03-06T07:28:36.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.597+0000 7f0ae1200640 1 -- start start 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ae1200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc072340 0x7f0adc155dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ae1200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0adc0b4680 0x7f0adc149670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ae1200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0adc15acb0 0x7f0adc15e770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ae1200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0adc0b3d10 con 0x7f0adc072340 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ae1200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0adc0b32b0 con 0x7f0adc0b4680 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ae1200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0adc0b4010 con 0x7f0adc15acb0 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ad9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc072340 0x7f0adc155dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ad9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc072340 0x7f0adc155dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36186/0 (socket says 172.21.15.86:36186) 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.598+0000 7f0ad9e00640 1 -- 172.21.15.86:0/964478745 learned_addr learned my addr 172.21.15.86:0/964478745 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad9e00640 1 -- 172.21.15.86:0/964478745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0adc15acb0 msgr2=0x7f0adc15e770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad9e00640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0adc15acb0 0x7f0adc15e770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad9e00640 1 -- 172.21.15.86:0/964478745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0adc0b4680 msgr2=0x7f0adc149670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad9e00640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0adc0b4680 0x7f0adc149670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad9e00640 1 -- 172.21.15.86:0/964478745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0adc156300 con 0x7f0adc072340 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad9e00640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc072340 0x7f0adc155dc0 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f0acc00c7f0 tx=0x7f0acc00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad2000640 1 -- 172.21.15.86:0/964478745 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0acc015070 con 0x7f0adc072340 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad2000640 1 -- 172.21.15.86:0/964478745 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0acc004240 con 0x7f0adc072340 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ad2000640 1 -- 172.21.15.86:0/964478745 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0acc007620 con 0x7f0adc072340 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0adc15b650 con 0x7f0adc072340 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.599+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0adc15bc10 con 0x7f0adc072340 2024-03-06T07:28:36.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.600+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0adc04f330 con 0x7f0adc072340 2024-03-06T07:28:36.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.602+0000 7f0ad2000640 1 -- 172.21.15.86:0/964478745 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0acc0079d0 con 0x7f0adc072340 2024-03-06T07:28:36.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.603+0000 7f0ad2000640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ab806b500 0x7f0ab806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:36.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.603+0000 7f0ad2000640 1 -- 172.21.15.86:0/964478745 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0acc08a270 con 0x7f0adc072340 2024-03-06T07:28:36.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.603+0000 7f0ad3e00640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ab806b500 0x7f0ab806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:36.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.604+0000 7f0ad3e00640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ab806b500 0x7f0ab806d9c0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f0ac4036000 tx=0x7f0ac4030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:36.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.608+0000 7f0ad2000640 1 -- 172.21.15.86:0/964478745 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0acc0590e0 con 0x7f0adc072340 2024-03-06T07:28:36.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.830+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0adc0b1510 con 0x7f0adc072340 2024-03-06T07:28:36.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.832+0000 7f0ad2000640 1 -- 172.21.15.86:0/964478745 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f0acc0589e0 con 0x7f0adc072340 2024-03-06T07:28:36.834 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:36.834 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:36.835 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:36.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ab806b500 msgr2=0x7f0ab806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:36.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ab806b500 0x7f0ab806d9c0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f0ac4036000 tx=0x7f0ac4030000 comp rx=0 tx=0).stop 2024-03-06T07:28:36.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc072340 msgr2=0x7f0adc155dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:36.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc072340 0x7f0adc155dc0 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f0acc00c7f0 tx=0x7f0acc00ccb0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 shutdown_connections 2024-03-06T07:28:36.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ab806b500 0x7f0ab806d9c0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0adc15acb0 0x7f0adc15e770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0adc0b4680 0x7f0adc149670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 --2- 172.21.15.86:0/964478745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0adc072340 0x7f0adc155dc0 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:36.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.834+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 >> 172.21.15.86:0/964478745 conn(0x7f0adc0b1730 msgr2=0x7f0adc0aee30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:36.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.835+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 shutdown_connections 2024-03-06T07:28:36.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:36.835+0000 7f0ae1200640 1 -- 172.21.15.86:0/964478745 wait complete. 2024-03-06T07:28:36.855 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:28:37.084 INFO:teuthology.orchestra.run.smithi194.stdout:{"mds_cache":{"strays_enqueued":1001}} 2024-03-06T07:28:37.084 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 1001 } } 2024-03-06T07:28:37.084 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:37.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.292+0000 7f1a20a00640 1 Processor -- start 2024-03-06T07:28:37.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.292+0000 7f1a20a00640 1 -- start start 2024-03-06T07:28:37.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.293+0000 7f1a20a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c144c00 0x7f1a1c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.293+0000 7f1a20a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1a1c15f9d0 0x7f1a1c15fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.293+0000 7f1a20a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1a1c06ee70 con 0x7f1a1c0ce0a0 2024-03-06T07:28:37.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.293+0000 7f1a20a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1a1c160370 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.293+0000 7f1a20a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a1c06cef0 con 0x7f1a1c144c00 2024-03-06T07:28:37.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.294+0000 7f1a11400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1a1c15f9d0 0x7f1a1c15fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.294+0000 7f1a19e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c144c00 0x7f1a1c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.294+0000 7f1a11400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1a1c15f9d0 0x7f1a1c15fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:43624/0 (socket says 172.21.15.86:43624) 2024-03-06T07:28:37.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.294+0000 7f1a11400640 1 -- 172.21.15.86:0/1746585246 learned_addr learned my addr 172.21.15.86:0/1746585246 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:37.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.295+0000 7f1a11400640 1 -- 172.21.15.86:0/1746585246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c144c00 msgr2=0x7f1a1c06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:37.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.295+0000 7f1a11400640 1 --2- 172.21.15.86:0/1746585246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c144c00 0x7f1a1c06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.295+0000 7f1a11400640 1 -- 172.21.15.86:0/1746585246 >> v1:172.21.15.194:6789/0 conn(0x7f1a1c0ce0a0 legacy=0x7f1a1c0ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.295+0000 7f1a11400640 1 -- 172.21.15.86:0/1746585246 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a1c06d1d0 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.297+0000 7f1a11400640 1 --2- 172.21.15.86:0/1746585246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c15fe30 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f1a04009870 tx=0x7f1a0402b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bed2ce4e8884e0c4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:37.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.297+0000 7f1a19400640 1 -- 172.21.15.86:0/1746585246 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a04034070 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.297+0000 7f1a19400640 1 -- 172.21.15.86:0/1746585246 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a04030840 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.297+0000 7f1a19400640 1 -- 172.21.15.86:0/1746585246 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a04030c10 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 -- 172.21.15.86:0/1746585246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c15f9d0 msgr2=0x7f1a1c15fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 --2- 172.21.15.86:0/1746585246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c15fe30 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f1a04009870 tx=0x7f1a0402b980 comp rx=0 tx=0).stop 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 -- 172.21.15.86:0/1746585246 shutdown_connections 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 --2- 172.21.15.86:0/1746585246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c15fe30 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 --2- 172.21.15.86:0/1746585246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c144c00 0x7f1a1c06e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 -- 172.21.15.86:0/1746585246 >> 172.21.15.86:0/1746585246 conn(0x7f1a1c0b1730 msgr2=0x7f1a1c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 -- 172.21.15.86:0/1746585246 shutdown_connections 2024-03-06T07:28:37.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.298+0000 7f1a20a00640 1 -- 172.21.15.86:0/1746585246 wait complete. 2024-03-06T07:28:37.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.300+0000 7f1a20a00640 1 Processor -- start 2024-03-06T07:28:37.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.300+0000 7f1a20a00640 1 -- start start 2024-03-06T07:28:37.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.301+0000 7f1a20a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c0ce0a0 0x7f1a1c15dd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.301+0000 7f1a20a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c144c00 0x7f1a1c130cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.301+0000 7f1a20a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c1311f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.301+0000 7f1a20a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a1c160730 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.301+0000 7f1a20a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a1c06d790 con 0x7f1a1c144c00 2024-03-06T07:28:37.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.301+0000 7f1a20a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a1c06cb50 con 0x7f1a1c0ce0a0 2024-03-06T07:28:37.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c1311f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c1311f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36188/0 (socket says 172.21.15.86:36188) 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 -- 172.21.15.86:0/500928115 learned_addr learned my addr 172.21.15.86:0/500928115 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a11400640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c144c00 0x7f1a1c130cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a19e00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c0ce0a0 0x7f1a1c15dd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 -- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c0ce0a0 msgr2=0x7f1a1c15dd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c0ce0a0 0x7f1a1c15dd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 -- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c144c00 msgr2=0x7f1a1c130cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c144c00 0x7f1a1c130cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 -- 172.21.15.86:0/500928115 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a1c15e630 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a19e00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c0ce0a0 0x7f1a1c15dd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.302+0000 7f1a1a800640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c1311f0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f1a1400c730 tx=0x7f1a1400cb30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.303+0000 7f1a12a00640 1 -- 172.21.15.86:0/500928115 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a14016020 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.303+0000 7f1a12a00640 1 -- 172.21.15.86:0/500928115 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a14006d50 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.303+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1a1c15e910 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.303+0000 7f1a12a00640 1 -- 172.21.15.86:0/500928115 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a14003bb0 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.303+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a1c140230 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.304+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1a1c04f330 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.305+0000 7f1a12a00640 1 -- 172.21.15.86:0/500928115 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1a140050d0 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.306+0000 7f1a12a00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f19f806b500 0x7f19f806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.306+0000 7f1a12a00640 1 -- 172.21.15.86:0/500928115 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1a14089dc0 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.306+0000 7f1a19e00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f19f806b500 0x7f19f806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.306+0000 7f1a19e00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f19f806b500 0x7f19f806d9c0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f1a0c009ac0 tx=0x7f1a0c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:37.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.310+0000 7f1a12a00640 1 -- 172.21.15.86:0/500928115 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1a14020020 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.527+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1a1c0723b0 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.528+0000 7f1a12a00640 1 -- 172.21.15.86:0/500928115 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f1a1c06ee70 con 0x7f1a1c15f9d0 2024-03-06T07:28:37.531 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:37.531 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f19f806b500 msgr2=0x7f19f806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f19f806b500 0x7f19f806d9c0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f1a0c009ac0 tx=0x7f1a0c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a1c15f9d0 msgr2=0x7f1a1c1311f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c1311f0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f1a1400c730 tx=0x7f1a1400cb30 comp rx=0 tx=0).stop 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 shutdown_connections 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f19f806b500 0x7f19f806d9c0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a1c15f9d0 0x7f1a1c1311f0 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a1c144c00 0x7f1a1c130cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 --2- 172.21.15.86:0/500928115 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a1c0ce0a0 0x7f1a1c15dd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 >> 172.21.15.86:0/500928115 conn(0x7f1a1c0b1730 msgr2=0x7f1a1c145040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:37.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 shutdown_connections 2024-03-06T07:28:37.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.531+0000 7f1a20a00640 1 -- 172.21.15.86:0/500928115 wait complete. 2024-03-06T07:28:37.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json perf dump purge_queue pq_executed 2024-03-06T07:28:37.773 INFO:teuthology.orchestra.run.smithi194.stdout:{"purge_queue":{"pq_executed":1001}} 2024-03-06T07:28:37.773 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 1001 } } 2024-03-06T07:28:37.773 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_metadata stat 10000000000.00000000 2024-03-06T07:28:37.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.878+0000 7f1cd8435f40 1 Processor -- start 2024-03-06T07:28:37.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.878+0000 7f1cd8435f40 1 -- start start 2024-03-06T07:28:37.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd8435f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 0x55e73b6339f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd8435f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b62c7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd8435f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b649090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd8435f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e73b573fb0 con 0x55e73b574d70 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd8435f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e73b573d10 con 0x55e73b6327e0 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd8435f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55e73b574250 con 0x55e73b574990 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b62c7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b62c7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36204/0 (socket says 172.21.15.86:36204) 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 -- 172.21.15.86:0/657031241 learned_addr learned my addr 172.21.15.86:0/657031241 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4e00640 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 0x55e73b6339f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd5800640 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b649090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 -- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 msgr2=0x55e73b6339f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 0x55e73b6339f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 -- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 msgr2=0x55e73b649090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b649090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.879+0000 7f1cd4400640 1 -- 172.21.15.86:0/657031241 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e73b4a0ad0 con 0x55e73b574d70 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.880+0000 7f1cd4400640 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b62c7e0 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f1cc00095d0 tx=0x7f1cc0009a90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e421cb60bcdae789 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.880+0000 7f1cd3000640 1 -- 172.21.15.86:0/657031241 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cc0017070 con 0x55e73b574d70 2024-03-06T07:28:37.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.880+0000 7f1cd3000640 1 -- 172.21.15.86:0/657031241 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cc0006600 con 0x55e73b574d70 2024-03-06T07:28:37.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.880+0000 7f1cd3000640 1 -- 172.21.15.86:0/657031241 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cc00069d0 con 0x55e73b574d70 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 -- 172.21.15.86:0/657031241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 msgr2=0x55e73b62c7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b62c7e0 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f1cc00095d0 tx=0x7f1cc0009a90 comp rx=0 tx=0).stop 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 -- 172.21.15.86:0/657031241 shutdown_connections 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b649090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b62c7e0 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/657031241 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 0x55e73b6339f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 -- 172.21.15.86:0/657031241 >> 172.21.15.86:0/657031241 conn(0x55e73b5a3c50 msgr2=0x55e73b631d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 -- 172.21.15.86:0/657031241 shutdown_connections 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 -- 172.21.15.86:0/657031241 wait complete. 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.881+0000 7f1cd8435f40 1 Processor -- start 2024-03-06T07:28:37.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.882+0000 7f1cd8435f40 1 -- start start 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.882+0000 7f1cd8435f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 0x55e73b640d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd8435f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b6306a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd8435f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b644d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd8435f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e73b62c290 con 0x55e73b574d70 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd8435f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e73b5a29a0 con 0x55e73b6327e0 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd8435f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55e73b62c4f0 con 0x55e73b574990 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b6306a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b6306a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36206/0 (socket says 172.21.15.86:36206) 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 -- 172.21.15.86:0/1605372085 learned_addr learned my addr 172.21.15.86:0/1605372085 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd5800640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b644d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 -- 172.21.15.86:0/1605372085 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 msgr2=0x55e73b640d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e73b574990 0x55e73b640d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 -- 172.21.15.86:0/1605372085 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 msgr2=0x55e73b644d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b644d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd4400640 1 -- 172.21.15.86:0/1605372085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e73b6412c0 con 0x55e73b574d70 2024-03-06T07:28:37.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.883+0000 7f1cd5800640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e73b6327e0 0x55e73b644d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:28:37.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.884+0000 7f1cd4400640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b6306a0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f1cc0009ad0 tx=0x7f1cc0003020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:37.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.884+0000 7f1cd1c00640 1 -- 172.21.15.86:0/1605372085 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cc0017040 con 0x55e73b574d70 2024-03-06T07:28:37.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.884+0000 7f1cd1c00640 1 -- 172.21.15.86:0/1605372085 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cc0022070 con 0x55e73b574d70 2024-03-06T07:28:37.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.884+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x55e73b64af60 con 0x55e73b574d70 2024-03-06T07:28:37.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.884+0000 7f1cd1c00640 1 -- 172.21.15.86:0/1605372085 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cc001d880 con 0x55e73b574d70 2024-03-06T07:28:37.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.884+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55e73b624c80 con 0x55e73b574d70 2024-03-06T07:28:37.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.887+0000 7f1cd1c00640 1 -- 172.21.15.86:0/1605372085 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1cc001da20 con 0x55e73b574d70 2024-03-06T07:28:37.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.888+0000 7f1cd1c00640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cb4069eb0 0x7f1cb406c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.888+0000 7f1cd1c00640 1 -- 172.21.15.86:0/1605372085 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1cc0089ea0 con 0x55e73b574d70 2024-03-06T07:28:37.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.888+0000 7f1cd4e00640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cb4069eb0 0x7f1cb406c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.888+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x55e73b645750 0x55e73b642f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:37.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.889+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:1 4.10 4:0cf6ad0f:::10000000000.00000000:head [stat] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e35) v9 -- 0x55e73b62e530 con 0x55e73b645750 2024-03-06T07:28:37.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.889+0000 7f1cd5800640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x55e73b645750 0x55e73b642f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:37.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.889+0000 7f1cd4e00640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cb4069eb0 0x7f1cb406c370 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f1cc4052000 tx=0x7f1cc404c000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.889+0000 7f1cd5800640 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x55e73b645750 0x55e73b642f90 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd5800640 1 -- 172.21.15.86:0/1605372085 <== osd.3 v2:172.21.15.86:6824/3540554891 1 ==== osd_op_reply(1 10000000000.00000000 [stat] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 ==== 164+0+0 (crc 0 0 0) 0x7f1ccc009870 con 0x55e73b645750 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr: error stat-ing cephfs_metadata/10000000000.00000000: (2) No such file or directory 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x55e73b645750 msgr2=0x55e73b642f90 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x55e73b645750 0x55e73b642f90 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cb4069eb0 msgr2=0x7f1cb406c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cb4069eb0 0x7f1cb406c370 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f1cc4052000 tx=0x7f1cc404c000 comp rx=0 tx=0).stop 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 msgr2=0x55e73b6306a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 --2- 172.21.15.86:0/1605372085 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e73b574d70 0x55e73b6306a0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f1cc0009ad0 tx=0x7f1cc0003020 comp rx=0 tx=0).stop 2024-03-06T07:28:37.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd5800640 1 -- 172.21.15.86:0/1605372085 reap_dead start 2024-03-06T07:28:37.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 shutdown_connections 2024-03-06T07:28:37.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 >> 172.21.15.86:0/1605372085 conn(0x55e73b5a3c50 msgr2=0x55e73b62ee80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:37.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 shutdown_connections 2024-03-06T07:28:37.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:37.890+0000 7f1cd8435f40 1 -- 172.21.15.86:0/1605372085 wait complete. 2024-03-06T07:28:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:28:37.898 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph df --format=json-pretty 2024-03-06T07:28:38.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.088+0000 7f012ca00640 1 Processor -- start 2024-03-06T07:28:38.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.088+0000 7f012ca00640 1 -- start start 2024-03-06T07:28:38.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.089+0000 7f012ca00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f012815b9c0 0x7f0128160190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f012ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 0x7f01280ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f012ca00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f012806d460 con 0x7f012815bda0 2024-03-06T07:28:38.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f012ca00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f01280aebf0 con 0x7f012815b9c0 2024-03-06T07:28:38.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f012ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0128072310 con 0x7f01280cff10 2024-03-06T07:28:38.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f011fe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f012815bda0 0x7f01280cf800 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38632/0 (socket says 172.21.15.86:38632) 2024-03-06T07:28:38.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f011fe00640 1 -- 172.21.15.86:0/1761965795 learned_addr learned my addr 172.21.15.86:0/1761965795 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f0126800640 1 --2- 172.21.15.86:0/1761965795 >> v2:172.21.15.194:3300/0 conn(0x7f012815b9c0 0x7f0128160190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f0127200640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 0x7f01280ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2919494854 0 0) 0x7f012806d460 con 0x7f012815bda0 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.090+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f010c003520 con 0x7f012815bda0 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 752335758 0 0) 0x7f010c003520 con 0x7f012815bda0 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 msgr2=0x7f01280ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 0x7f01280ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 >> v2:172.21.15.194:3300/0 conn(0x7f012815b9c0 msgr2=0x7f0128160190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 --2- 172.21.15.86:0/1761965795 >> v2:172.21.15.194:3300/0 conn(0x7f012815b9c0 0x7f0128160190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f012806d740 con 0x7f012815bda0 2024-03-06T07:28:38.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f0110001670 con 0x7f012815bda0 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 >> v1:172.21.15.86:6789/0 conn(0x7f012815bda0 legacy=0x7f01280cf800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 0x7f010c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c003f80 0x7f010c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f010c004d40 0x7f010c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01280aebf0 con 0x7f010c004d40 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0128072310 con 0x7f010c003f80 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f010c00ba70 con 0x7f01280cff10 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0127200640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f010c004d40 0x7f010c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0126800640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 0x7f010c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0127200640 1 -- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 msgr2=0x7f010c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0127200640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01280cff10 0x7f010c0039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f011fe00640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c003f80 0x7f010c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.091+0000 7f0127200640 1 -- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c003f80 msgr2=0x7f010c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:38.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.092+0000 7f0127200640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c003f80 0x7f010c004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.092+0000 7f0127200640 1 -- 172.21.15.86:0/1761965795 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f010c003520 con 0x7f010c004d40 2024-03-06T07:28:38.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.093+0000 7f0127200640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f010c004d40 0x7f010c0051c0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f0120009fc0 tx=0x7f010c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=620f5587b134bc7f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.093+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0120039070 con 0x7f010c004d40 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.093+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f012002cc20 con 0x7f010c004d40 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.093+0000 7f0125e00640 1 -- 172.21.15.86:0/1761965795 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0120009c10 con 0x7f010c004d40 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.093+0000 7f012ca00640 1 -- 172.21.15.86:0/1761965795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f010c004d40 msgr2=0x7f010c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.093+0000 7f012ca00640 1 --2- 172.21.15.86:0/1761965795 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f010c004d40 0x7f010c0051c0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f0120009fc0 tx=0x7f010c005830 comp rx=0 tx=0).stop 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.093+0000 7f0127200640 1 -- 172.21.15.86:0/1761965795 reap_dead start 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.094+0000 7f012ca00640 1 -- 172.21.15.86:0/1761965795 shutdown_connections 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.094+0000 7f012ca00640 1 -- 172.21.15.86:0/1761965795 >> 172.21.15.86:0/1761965795 conn(0x7f012806afc0 msgr2=0x7f012806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.094+0000 7f012ca00640 1 -- 172.21.15.86:0/1761965795 shutdown_connections 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.094+0000 7f012ca00640 1 -- 172.21.15.86:0/1761965795 wait complete. 2024-03-06T07:28:38.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.095+0000 7f012ca00640 1 Processor -- start 2024-03-06T07:28:38.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.095+0000 7f012ca00640 1 -- start start 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.096+0000 7f012ca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c004d40 0x7f012814a4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.096+0000 7f012ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f01280b0af0 0x7f01280b0f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.096+0000 7f012ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01281e6b70 0x7f01281e6ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.096+0000 7f012ca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f012806e220 con 0x7f01280b0af0 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.096+0000 7f012ca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f012806dd00 con 0x7f010c004d40 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.096+0000 7f012ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f012806e3a0 con 0x7f01281e6b70 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.096+0000 7f011fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f01280b0af0 0x7f01280b0f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f01280b0af0 0x7f01280b0f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36230/0 (socket says 172.21.15.86:36230) 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 -- 172.21.15.86:0/4114459106 learned_addr learned my addr 172.21.15.86:0/4114459106 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f0126800640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c004d40 0x7f012814a4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f0127200640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01281e6b70 0x7f01281e6ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 -- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01281e6b70 msgr2=0x7f01281e6ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01281e6b70 0x7f01281e6ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 -- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c004d40 msgr2=0x7f012814a4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c004d40 0x7f012814a4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 -- 172.21.15.86:0/4114459106 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01281e7910 con 0x7f01280b0af0 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.097+0000 7f011fe00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f01280b0af0 0x7f01280b0f70 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f011000d7e0 tx=0x7f011000dca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.098+0000 7f011ea00640 1 -- 172.21.15.86:0/4114459106 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0110019070 con 0x7f01280b0af0 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.098+0000 7f011ea00640 1 -- 172.21.15.86:0/4114459106 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f01100074e0 con 0x7f01280b0af0 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.098+0000 7f011ea00640 1 -- 172.21.15.86:0/4114459106 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01100049e0 con 0x7f01280b0af0 2024-03-06T07:28:38.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.098+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01281e7bf0 con 0x7f01280b0af0 2024-03-06T07:28:38.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.098+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f01281e81b0 con 0x7f01280b0af0 2024-03-06T07:28:38.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.099+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f012804f330 con 0x7f01280b0af0 2024-03-06T07:28:38.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.100+0000 7f011ea00640 1 -- 172.21.15.86:0/4114459106 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0110004b80 con 0x7f01280b0af0 2024-03-06T07:28:38.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.101+0000 7f011ea00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010406b500 0x7f010406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.101+0000 7f011ea00640 1 -- 172.21.15.86:0/4114459106 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0110089b70 con 0x7f01280b0af0 2024-03-06T07:28:38.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.101+0000 7f0126800640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010406b500 0x7f010406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.102+0000 7f0126800640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010406b500 0x7f010406d9c0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f0118006fd0 tx=0x7f0118008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:38.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.106+0000 7f011ea00640 1 -- 172.21.15.86:0/4114459106 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0110058a80 con 0x7f01280b0af0 2024-03-06T07:28:38.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.364+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "df", "format": "json-pretty"} v 0) v1 -- 0x7f01280cf0f0 con 0x7f01280b0af0 2024-03-06T07:28:38.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.365+0000 7f011ea00640 1 -- 172.21.15.86:0/4114459106 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "df", "format": "json-pretty"}]=0 v0) v1 ==== 75+0+1657 (secure 0 0 0) 0x7f0110058380 con 0x7f01280b0af0 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "total_bytes": 773094113280, 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "total_avail_bytes": 772828471296, 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_bytes": 265641984, 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_bytes": 265641984, 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_ratio": 0.00034360884455963969, 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "num_osds": 8, 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "num_per_pool_osds": 8, 2024-03-06T07:28:38.368 INFO:teuthology.orchestra.run.smithi086.stdout: "num_per_pool_omap_osds": 8 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "stats_by_class": { 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "ssd": { 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "total_bytes": 773094113280, 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "total_avail_bytes": 772828471296, 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_bytes": 265641984, 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_bytes": 265641984, 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_ratio": 0.00034360884455963969 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "pools": [ 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "name": ".mgr", 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 1, 2024-03-06T07:28:38.369 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "stored": 590368, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "objects": 2, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "kb_used": 1160, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "bytes_used": 1187840, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "percent_used": 1.6180575812541065e-06, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "max_avail": 367056814080 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "cephfs_metadata", 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 4, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "stored": 896795, 2024-03-06T07:28:38.370 INFO:teuthology.orchestra.run.smithi086.stdout: "objects": 23, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "kb_used": 1808, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "bytes_used": 1851392, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "percent_used": 2.5219358121830737e-06, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "max_avail": 367056814080 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "cephfs_data", 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 5, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "stored": 0, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "objects": 0, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "kb_used": 0, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "bytes_used": 0, 2024-03-06T07:28:38.371 INFO:teuthology.orchestra.run.smithi086.stdout: "percent_used": 0, 2024-03-06T07:28:38.372 INFO:teuthology.orchestra.run.smithi086.stdout: "max_avail": 367056814080 2024-03-06T07:28:38.372 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:38.372 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:38.372 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:28:38.372 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010406b500 msgr2=0x7f010406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010406b500 0x7f010406d9c0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f0118006fd0 tx=0x7f0118008040 comp rx=0 tx=0).stop 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f01280b0af0 msgr2=0x7f01280b0f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f01280b0af0 0x7f01280b0f70 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f011000d7e0 tx=0x7f011000dca0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 shutdown_connections 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f010406b500 0x7f010406d9c0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f01281e6b70 0x7f01281e6ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f01280b0af0 0x7f01280b0f70 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 --2- 172.21.15.86:0/4114459106 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f010c004d40 0x7f012814a4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 >> 172.21.15.86:0/4114459106 conn(0x7f012806afc0 msgr2=0x7f01280b2580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 shutdown_connections 2024-03-06T07:28:38.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.368+0000 7f012ca00640 1 -- 172.21.15.86:0/4114459106 wait complete. 2024-03-06T07:28:38.388 DEBUG:tasks.ceph_test_case:wait_until_true: success in 0s and 0 retries 2024-03-06T07:28:38.388 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:28:38.389 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:28:38.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:28:38.612 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:28:38.612 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:28:38.613 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:38.669 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:28:38.669 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:38.720 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:38.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.928+0000 7f1ff0800640 1 Processor -- start 2024-03-06T07:28:38.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.928+0000 7f1ff0800640 1 -- start start 2024-03-06T07:28:38.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.929+0000 7f1ff0800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1fe815acb0 0x7f1fe80b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.929+0000 7f1ff0800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe80b4680 0x7f1fe806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.929+0000 7f1ff0800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1fe815b090 con 0x7f1fe8072340 2024-03-06T07:28:38.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.929+0000 7f1ff0800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1fe80b2a30 con 0x7f1fe815acb0 2024-03-06T07:28:38.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.929+0000 7f1ff0800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1fe80b4360 con 0x7f1fe80b4680 2024-03-06T07:28:38.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.930+0000 7f1fede00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1fe815acb0 0x7f1fe80b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.930+0000 7f1fede00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1fe815acb0 0x7f1fe80b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:43702/0 (socket says 172.21.15.86:43702) 2024-03-06T07:28:38.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.930+0000 7f1fede00640 1 -- 172.21.15.86:0/220248024 learned_addr learned my addr 172.21.15.86:0/220248024 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:38.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.930+0000 7f1fed400640 1 --2- 172.21.15.86:0/220248024 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe80b4680 0x7f1fe806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.931+0000 7f1fed400640 1 -- 172.21.15.86:0/220248024 >> v2:172.21.15.194:3300/0 conn(0x7f1fe815acb0 msgr2=0x7f1fe80b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:38.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.931+0000 7f1fed400640 1 --2- 172.21.15.86:0/220248024 >> v2:172.21.15.194:3300/0 conn(0x7f1fe815acb0 0x7f1fe80b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.931+0000 7f1fed400640 1 -- 172.21.15.86:0/220248024 >> v1:172.21.15.194:6789/0 conn(0x7f1fe8072340 legacy=0x7f1fe80ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.931+0000 7f1fed400640 1 -- 172.21.15.86:0/220248024 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1fe80b2cf0 con 0x7f1fe80b4680 2024-03-06T07:28:38.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.931+0000 7f1fede00640 1 --2- 172.21.15.86:0/220248024 >> v2:172.21.15.194:3300/0 conn(0x7f1fe815acb0 0x7f1fe80b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:28:38.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.932+0000 7f1fed400640 1 --2- 172.21.15.86:0/220248024 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe80b4680 0x7f1fe806e4f0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f1fd80097a0 tx=0x7f1fd802b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e3c445a84664d654 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:38.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.933+0000 7f1feca00640 1 -- 172.21.15.86:0/220248024 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fd8035020 con 0x7f1fe80b4680 2024-03-06T07:28:38.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.933+0000 7f1feca00640 1 -- 172.21.15.86:0/220248024 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1fd8031830 con 0x7f1fe80b4680 2024-03-06T07:28:38.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.933+0000 7f1feca00640 1 -- 172.21.15.86:0/220248024 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fd8031c00 con 0x7f1fe80b4680 2024-03-06T07:28:38.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 -- 172.21.15.86:0/220248024 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe80b4680 msgr2=0x7f1fe806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 --2- 172.21.15.86:0/220248024 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe80b4680 0x7f1fe806e4f0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f1fd80097a0 tx=0x7f1fd802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 -- 172.21.15.86:0/220248024 shutdown_connections 2024-03-06T07:28:38.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 --2- 172.21.15.86:0/220248024 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe80b4680 0x7f1fe806e4f0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 --2- 172.21.15.86:0/220248024 >> v2:172.21.15.194:3300/0 conn(0x7f1fe815acb0 0x7f1fe80b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 -- 172.21.15.86:0/220248024 >> 172.21.15.86:0/220248024 conn(0x7f1fe80b1730 msgr2=0x7f1fe80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:38.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 -- 172.21.15.86:0/220248024 shutdown_connections 2024-03-06T07:28:38.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.934+0000 7f1ff0800640 1 -- 172.21.15.86:0/220248024 wait complete. 2024-03-06T07:28:38.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.935+0000 7f1ff0800640 1 Processor -- start 2024-03-06T07:28:38.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.936+0000 7f1ff0800640 1 -- start start 2024-03-06T07:28:38.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.936+0000 7f1ff0800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1fe8072340 0x7f1fe814b500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.936+0000 7f1ff0800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1fe80b4680 0x7f1fe81e5eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1ff0800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe815acb0 0x7f1fe81e63f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1ff0800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1fe80b3d10 con 0x7f1fe8072340 2024-03-06T07:28:38.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1ff0800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1fe80b32b0 con 0x7f1fe80b4680 2024-03-06T07:28:38.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1ff0800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1fe80b4010 con 0x7f1fe815acb0 2024-03-06T07:28:38.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1fe8072340 0x7f1fe814b500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1fe8072340 0x7f1fe814b500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36242/0 (socket says 172.21.15.86:36242) 2024-03-06T07:28:38.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 -- 172.21.15.86:0/1296451946 learned_addr learned my addr 172.21.15.86:0/1296451946 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:38.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 -- 172.21.15.86:0/1296451946 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe815acb0 msgr2=0x7f1fe81e63f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe815acb0 0x7f1fe81e63f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 -- 172.21.15.86:0/1296451946 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1fe80b4680 msgr2=0x7f1fe81e5eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:38.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fed400640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1fe80b4680 0x7f1fe81e5eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1fe80b4680 0x7f1fe81e5eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:38.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.937+0000 7f1fede00640 1 -- 172.21.15.86:0/1296451946 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1fe814ba40 con 0x7f1fe8072340 2024-03-06T07:28:38.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.938+0000 7f1fede00640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1fe8072340 0x7f1fe814b500 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f1fdc00e9d0 tx=0x7f1fdc00ee90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:38.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.938+0000 7f1fe2000640 1 -- 172.21.15.86:0/1296451946 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fdc017070 con 0x7f1fe8072340 2024-03-06T07:28:38.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.938+0000 7f1fe2000640 1 -- 172.21.15.86:0/1296451946 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1fdc00ca60 con 0x7f1fe8072340 2024-03-06T07:28:38.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.938+0000 7f1fe2000640 1 -- 172.21.15.86:0/1296451946 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fdc00ce10 con 0x7f1fe8072340 2024-03-06T07:28:38.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.938+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1fe81f3c40 con 0x7f1fe8072340 2024-03-06T07:28:38.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.938+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1fe815c4e0 con 0x7f1fe8072340 2024-03-06T07:28:38.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.939+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1fe804f330 con 0x7f1fe8072340 2024-03-06T07:28:38.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.940+0000 7f1fe2000640 1 -- 172.21.15.86:0/1296451946 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1fdc01a070 con 0x7f1fe8072340 2024-03-06T07:28:38.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.941+0000 7f1fe2000640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1fcc06b4d0 0x7f1fcc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:38.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.941+0000 7f1fe2000640 1 -- 172.21.15.86:0/1296451946 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1fdc08a190 con 0x7f1fe8072340 2024-03-06T07:28:38.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.941+0000 7f1fed400640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1fcc06b4d0 0x7f1fcc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:38.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.942+0000 7f1fed400640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1fcc06b4d0 0x7f1fcc06d990 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f1fd8036000 tx=0x7f1fd8008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:38.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:38.946+0000 7f1fe2000640 1 -- 172.21.15.86:0/1296451946 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1fe814ba40 con 0x7f1fe8072340 2024-03-06T07:28:39.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.145+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1fe806dfb0 con 0x7f1fe8072340 2024-03-06T07:28:39.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.147+0000 7f1fe2000640 1 -- 172.21.15.86:0/1296451946 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v35) v1 ==== 74+0+16681 (secure 0 0 0) 0x7f1fdc059000 con 0x7f1fe8072340 2024-03-06T07:28:39.150 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:39.150 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":35,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:15.157270+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:11.444335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:12.623953+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"35","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1fcc06b4d0 msgr2=0x7f1fcc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1fcc06b4d0 0x7f1fcc06d990 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f1fd8036000 tx=0x7f1fd8008030 comp rx=0 tx=0).stop 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1fe8072340 msgr2=0x7f1fe814b500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1fe8072340 0x7f1fe814b500 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f1fdc00e9d0 tx=0x7f1fdc00ee90 comp rx=0 tx=0).stop 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 shutdown_connections 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1fcc06b4d0 0x7f1fcc06d990 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1fe815acb0 0x7f1fe81e63f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1fe80b4680 0x7f1fe81e5eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 --2- 172.21.15.86:0/1296451946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1fe8072340 0x7f1fe814b500 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 >> 172.21.15.86:0/1296451946 conn(0x7f1fe80b1730 msgr2=0x7f1fe80ad8f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 shutdown_connections 2024-03-06T07:28:39.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.150+0000 7f1ff0800640 1 -- 172.21.15.86:0/1296451946 wait complete. 2024-03-06T07:28:39.171 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:28:11.444335+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '34', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 5, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:28:12.623953+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '35', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:28:39.172 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:28:39.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.377+0000 7fa8b4a00640 1 Processor -- start 2024-03-06T07:28:39.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.378+0000 7fa8b4a00640 1 -- start start 2024-03-06T07:28:39.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.379+0000 7fa8b4a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa8b015cb90 0x7fa8b006d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.379+0000 7fa8b4a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa8b015cf70 con 0x7fa8b006e060 2024-03-06T07:28:39.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.379+0000 7fa8b4a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa8b00b35b0 con 0x7fa8b0163800 2024-03-06T07:28:39.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.379+0000 7fa8b4a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa8b00b3890 con 0x7fa8b015cb90 2024-03-06T07:28:39.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ae800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa8b006e060 0x7fa8b00cda90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38640/0 (socket says 172.21.15.86:38640) 2024-03-06T07:28:39.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ae800640 1 -- 172.21.15.86:0/2444314384 learned_addr learned my addr 172.21.15.86:0/2444314384 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:39.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ade00640 1 --2- 172.21.15.86:0/2444314384 >> v2:172.21.15.194:3300/0 conn(0x7fa8b015cb90 0x7fa8b006d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 472560131 0 0) 0x7fa8b015cf70 con 0x7fa8b006e060 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa894003520 con 0x7fa8b006e060 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1190146176 0 0) 0x7fa894003520 con 0x7fa8b006e060 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 >> v2:172.21.15.194:3300/0 conn(0x7fa8b015cb90 msgr2=0x7fa8b006d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 --2- 172.21.15.86:0/2444314384 >> v2:172.21.15.194:3300/0 conn(0x7fa8b015cb90 0x7fa8b006d6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 >> v1:172.21.15.194:6789/0 conn(0x7fa8b0163800 legacy=0x7fa8b00b2c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8b006cbe0 con 0x7fa8b006e060 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fa8a80019f0 con 0x7fa8b006e060 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 >> v1:172.21.15.86:6789/0 conn(0x7fa8b006e060 legacy=0x7fa8b00cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8940039b0 0x7fa894003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa894004800 0x7fa894004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8940051c0 0x7fa89400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8b00b3890 con 0x7fa8940039b0 2024-03-06T07:28:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa89400c040 con 0x7fa8940051c0 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.380+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa89400c240 con 0x7fa894004800 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.381+0000 7fa8ade00640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8940039b0 0x7fa894003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.381+0000 7fa8ade00640 1 -- 172.21.15.86:0/2444314384 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa894004800 msgr2=0x7fa894004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.381+0000 7fa8ade00640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa894004800 0x7fa894004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.381+0000 7fa8ade00640 1 -- 172.21.15.86:0/2444314384 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8940051c0 msgr2=0x7fa89400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.381+0000 7fa8ade00640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8940051c0 0x7fa89400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.381+0000 7fa8ade00640 1 -- 172.21.15.86:0/2444314384 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa894003520 con 0x7fa8940039b0 2024-03-06T07:28:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.381+0000 7fa8ae800640 1 -- 172.21.15.86:0/2444314384 reap_dead start 2024-03-06T07:28:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.382+0000 7fa8ade00640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8940039b0 0x7fa894003db0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7fa8a0009ac0 tx=0x7fa894005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f7fb06cc230e6d1e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.382+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8a0039070 con 0x7fa8940039b0 2024-03-06T07:28:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.382+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8a002c710 con 0x7fa8940039b0 2024-03-06T07:28:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.382+0000 7fa8ad400640 1 -- 172.21.15.86:0/2444314384 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8a0009690 con 0x7fa8940039b0 2024-03-06T07:28:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.383+0000 7fa8b4a00640 1 -- 172.21.15.86:0/2444314384 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8940039b0 msgr2=0x7fa894003db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.383+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8940039b0 0x7fa894003db0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7fa8a0009ac0 tx=0x7fa894005830 comp rx=0 tx=0).stop 2024-03-06T07:28:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.383+0000 7fa8b4a00640 1 -- 172.21.15.86:0/2444314384 shutdown_connections 2024-03-06T07:28:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.383+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/2444314384 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8940039b0 0x7fa894003db0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.383+0000 7fa8b4a00640 1 -- 172.21.15.86:0/2444314384 >> 172.21.15.86:0/2444314384 conn(0x7fa8b00b1940 msgr2=0x7fa8b00b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.383+0000 7fa8b4a00640 1 -- 172.21.15.86:0/2444314384 shutdown_connections 2024-03-06T07:28:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.383+0000 7fa8b4a00640 1 -- 172.21.15.86:0/2444314384 wait complete. 2024-03-06T07:28:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.385+0000 7fa8b4a00640 1 Processor -- start 2024-03-06T07:28:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.385+0000 7fa8b4a00640 1 -- start start 2024-03-06T07:28:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8b4a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa8940039b0 0x7fa8b014be40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8b4a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8b015b0b0 0x7fa8b00add70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8b4a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8b00ae2b0 0x7fa8b01fbad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8b4a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8b006d230 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8b4a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8b006c990 con 0x7fa8b015b0b0 2024-03-06T07:28:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8b4a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa8b006e4d0 con 0x7fa8940039b0 2024-03-06T07:28:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8ae800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8b00ae2b0 0x7fa8b01fbad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8ae800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8b00ae2b0 0x7fa8b01fbad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36260/0 (socket says 172.21.15.86:36260) 2024-03-06T07:28:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.386+0000 7fa8ae800640 1 -- 172.21.15.86:0/862124937 learned_addr learned my addr 172.21.15.86:0/862124937 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ade00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa8940039b0 0x7fa8b014be40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8a5400640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8b015b0b0 0x7fa8b00add70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ae800640 1 -- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa8940039b0 msgr2=0x7fa8b014be40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ae800640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa8940039b0 0x7fa8b014be40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ae800640 1 -- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8b015b0b0 msgr2=0x7fa8b00add70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ae800640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8b015b0b0 0x7fa8b00add70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ae800640 1 -- 172.21.15.86:0/862124937 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8b009b0d0 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ade00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa8940039b0 0x7fa8b014be40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8ae800640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8b00ae2b0 0x7fa8b01fbad0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7fa8b006ec20 tx=0x7fa8a800ccd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8a6a00640 1 -- 172.21.15.86:0/862124937 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8a800a610 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8a6a00640 1 -- 172.21.15.86:0/862124937 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8a800a7b0 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8a6a00640 1 -- 172.21.15.86:0/862124937 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8a800aba0 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa8b0156750 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.387+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8b0156d10 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.388+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa8b004f330 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.390+0000 7fa8a6a00640 1 -- 172.21.15.86:0/862124937 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa8a8011070 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.390+0000 7fa8a6a00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa88c06b4d0 0x7fa88c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.390+0000 7fa8a6a00640 1 -- 172.21.15.86:0/862124937 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa8a8089c40 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.391+0000 7fa8ade00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa88c06b4d0 0x7fa88c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.391+0000 7fa8ade00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa88c06b4d0 0x7fa88c06d990 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fa8a0036000 tx=0x7fa8a0030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:39.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.395+0000 7fa8a6a00640 1 -- 172.21.15.86:0/862124937 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa8a801a060 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.588+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fa8b00b2530 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.589+0000 7fa8a6a00640 1 -- 172.21.15.86:0/862124937 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v35) v1 ==== 93+0+10 (secure 0 0 0) 0x7fa8b009b0d0 con 0x7fa8b00ae2b0 2024-03-06T07:28:39.592 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:28:39.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa88c06b4d0 msgr2=0x7fa88c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa88c06b4d0 0x7fa88c06d990 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fa8a0036000 tx=0x7fa8a0030000 comp rx=0 tx=0).stop 2024-03-06T07:28:39.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8b00ae2b0 msgr2=0x7fa8b01fbad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8b00ae2b0 0x7fa8b01fbad0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7fa8b006ec20 tx=0x7fa8a800ccd0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 shutdown_connections 2024-03-06T07:28:39.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa88c06b4d0 0x7fa88c06d990 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa8b00ae2b0 0x7fa8b01fbad0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa8b015b0b0 0x7fa8b00add70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 --2- 172.21.15.86:0/862124937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa8940039b0 0x7fa8b014be40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 >> 172.21.15.86:0/862124937 conn(0x7fa8b00b1940 msgr2=0x7fa8b0164b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:39.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 shutdown_connections 2024-03-06T07:28:39.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.592+0000 7fa8b4a00640 1 -- 172.21.15.86:0/862124937 wait complete. 2024-03-06T07:28:39.613 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:28:39.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.819+0000 7fd49be00640 1 Processor -- start 2024-03-06T07:28:39.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.820+0000 7fd49be00640 1 -- start start 2024-03-06T07:28:39.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.821+0000 7fd49be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd49c15e130 0x7fd49c0cf5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.821+0000 7fd49be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0cfaf0 0x7fd49c0d1ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.821+0000 7fd49be00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd49c0d13e0 con 0x7fd49c0d23e0 2024-03-06T07:28:39.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.821+0000 7fd49be00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd49c0cf120 con 0x7fd49c15e130 2024-03-06T07:28:39.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.821+0000 7fd49be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd49c0cead0 con 0x7fd49c0cfaf0 2024-03-06T07:28:39.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.821+0000 7fd49b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd49c0d23e0 0x7fd49c0c7970 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38642/0 (socket says 172.21.15.86:38642) 2024-03-06T07:28:39.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.821+0000 7fd49b400640 1 -- 172.21.15.86:0/4279334742 learned_addr learned my addr 172.21.15.86:0/4279334742 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd49aa00640 1 --2- 172.21.15.86:0/4279334742 >> v2:172.21.15.194:3300/0 conn(0x7fd49c15e130 0x7fd49c0cf5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd49a000640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0cfaf0 0x7fd49c0d1ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 567528286 0 0) 0x7fd49c0d13e0 con 0x7fd49c0d23e0 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd480003520 con 0x7fd49c0d23e0 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1752518633 0 0) 0x7fd480003520 con 0x7fd49c0d23e0 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0cfaf0 msgr2=0x7fd49c0d1ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0cfaf0 0x7fd49c0d1ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 >> v2:172.21.15.194:3300/0 conn(0x7fd49c15e130 msgr2=0x7fd49c0cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 --2- 172.21.15.86:0/4279334742 >> v2:172.21.15.194:3300/0 conn(0x7fd49c15e130 0x7fd49c0cf5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd49c0cedb0 con 0x7fd49c0d23e0 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd4940019f0 con 0x7fd49c0d23e0 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 >> v1:172.21.15.86:6789/0 conn(0x7fd49c0d23e0 legacy=0x7fd49c0c7970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4800039b0 0x7fd480003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd480004800 0x7fd480004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0cfaf0 0x7fd4800051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd49c0cf120 con 0x7fd4800039b0 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd49c0cead0 con 0x7fd480004800 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.822+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd48000b9c0 con 0x7fd49c0cfaf0 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49aa00640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4800039b0 0x7fd480003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49a000640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd480004800 0x7fd480004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49aa00640 1 -- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0cfaf0 msgr2=0x7fd4800051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49aa00640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0cfaf0 0x7fd4800051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49aa00640 1 -- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd480004800 msgr2=0x7fd480004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:39.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49aa00640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd480004800 0x7fd480004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49aa00640 1 -- 172.21.15.86:0/4279334742 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd480003520 con 0x7fd4800039b0 2024-03-06T07:28:39.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.823+0000 7fd49a000640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd480004800 0x7fd480004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:39.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.824+0000 7fd49aa00640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4800039b0 0x7fd480003dd0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7fd48c009ac0 tx=0x7fd480005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=552bd0040a1d8b1e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:39.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.824+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd48c039070 con 0x7fd4800039b0 2024-03-06T07:28:39.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.824+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd48c02c710 con 0x7fd4800039b0 2024-03-06T07:28:39.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.824+0000 7fd499600640 1 -- 172.21.15.86:0/4279334742 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd48c009690 con 0x7fd4800039b0 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.825+0000 7fd49be00640 1 -- 172.21.15.86:0/4279334742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4800039b0 msgr2=0x7fd480003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.825+0000 7fd49be00640 1 --2- 172.21.15.86:0/4279334742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4800039b0 0x7fd480003dd0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7fd48c009ac0 tx=0x7fd480005830 comp rx=0 tx=0).stop 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.825+0000 7fd49b400640 1 -- 172.21.15.86:0/4279334742 reap_dead start 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.825+0000 7fd49be00640 1 -- 172.21.15.86:0/4279334742 shutdown_connections 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.825+0000 7fd49be00640 1 -- 172.21.15.86:0/4279334742 >> 172.21.15.86:0/4279334742 conn(0x7fd49c09d070 msgr2=0x7fd49c0d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.825+0000 7fd49be00640 1 -- 172.21.15.86:0/4279334742 shutdown_connections 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.825+0000 7fd49be00640 1 -- 172.21.15.86:0/4279334742 wait complete. 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.827+0000 7fd49be00640 1 Processor -- start 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.827+0000 7fd49be00640 1 -- start start 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4800039b0 0x7fd49c0caef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0b8e80 0x7fd49c15f560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd49c0b9300 0x7fd49c15faa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd49c0c8200 con 0x7fd49c0b9300 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd49c0c8080 con 0x7fd4800039b0 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd49c0d1ea0 con 0x7fd49c0b8e80 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd49c0b9300 0x7fd49c15faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd49c0b9300 0x7fd49c15faa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36284/0 (socket says 172.21.15.86:36284) 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49b400640 1 -- 172.21.15.86:0/997434885 learned_addr learned my addr 172.21.15.86:0/997434885 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49a000640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0b8e80 0x7fd49c15f560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.828+0000 7fd49aa00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4800039b0 0x7fd49c0caef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49b400640 1 -- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0b8e80 msgr2=0x7fd49c15f560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49b400640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0b8e80 0x7fd49c15f560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49b400640 1 -- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4800039b0 msgr2=0x7fd49c0caef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49b400640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4800039b0 0x7fd49c0caef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49b400640 1 -- 172.21.15.86:0/997434885 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd49c0cb4d0 con 0x7fd49c0b9300 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49b400640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd49c0b9300 0x7fd49c15faa0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7fd49c08e2d0 tx=0x7fd49400ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd492a00640 1 -- 172.21.15.86:0/997434885 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd49400a5b0 con 0x7fd49c0b9300 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd492a00640 1 -- 172.21.15.86:0/997434885 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd49400a770 con 0x7fd49c0b9300 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd492a00640 1 -- 172.21.15.86:0/997434885 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd49400ab60 con 0x7fd49c0b9300 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd49c0cb810 con 0x7fd49c0b9300 2024-03-06T07:28:39.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.829+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd49c0cc050 con 0x7fd49c0b9300 2024-03-06T07:28:39.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.831+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd49c0cc250 con 0x7fd49c0b9300 2024-03-06T07:28:39.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.831+0000 7fd492a00640 1 -- 172.21.15.86:0/997434885 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd49400ad00 con 0x7fd49c0b9300 2024-03-06T07:28:39.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.832+0000 7fd492a00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd47c06b4d0 0x7fd47c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:39.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.832+0000 7fd492a00640 1 -- 172.21.15.86:0/997434885 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd494089b00 con 0x7fd49c0b9300 2024-03-06T07:28:39.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.833+0000 7fd49aa00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd47c06b4d0 0x7fd47c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:39.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.833+0000 7fd49aa00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd47c06b4d0 0x7fd47c06d990 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fd48c036000 tx=0x7fd48c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:39.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:39.837+0000 7fd492a00640 1 -- 172.21.15.86:0/997434885 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd49c0cc250 con 0x7fd49c0b9300 2024-03-06T07:28:40.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.035+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fd49c04f330 con 0x7fd49c0b9300 2024-03-06T07:28:40.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.036+0000 7fd492a00640 1 -- 172.21.15.86:0/997434885 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v35) v1 ==== 104+0+11 (secure 0 0 0) 0x7fd49c0cb810 con 0x7fd49c0b9300 2024-03-06T07:28:40.038 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd47c06b4d0 msgr2=0x7fd47c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd47c06b4d0 0x7fd47c06d990 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fd48c036000 tx=0x7fd48c030000 comp rx=0 tx=0).stop 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd49c0b9300 msgr2=0x7fd49c15faa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd49c0b9300 0x7fd49c15faa0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7fd49c08e2d0 tx=0x7fd49400ccb0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 shutdown_connections 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd47c06b4d0 0x7fd47c06d990 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd49c0b9300 0x7fd49c15faa0 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd49c0b8e80 0x7fd49c15f560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 --2- 172.21.15.86:0/997434885 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4800039b0 0x7fd49c0caef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 >> 172.21.15.86:0/997434885 conn(0x7fd49c09d070 msgr2=0x7fd49c0c7950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:40.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 shutdown_connections 2024-03-06T07:28:40.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.039+0000 7fd49be00640 1 -- 172.21.15.86:0/997434885 wait complete. 2024-03-06T07:28:40.060 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:28:40.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.260+0000 7f760b200640 1 Processor -- start 2024-03-06T07:28:40.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.260+0000 7f760b200640 1 -- start start 2024-03-06T07:28:40.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f760b200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7604145690 0x7f7604160470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f760b200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f760406ef10 0x7f760406afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f760b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406b500 0x7f760406dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f760b200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f76040cce60 con 0x7f7604145690 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f760b200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f760406f6d0 con 0x7f760406ef10 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f760b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f760406f9b0 con 0x7f760406b500 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f7603e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7604145690 0x7f7604160470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f7603e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7604145690 0x7f7604160470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36290/0 (socket says 172.21.15.86:36290) 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f7603e00640 1 -- 172.21.15.86:0/239933586 learned_addr learned my addr 172.21.15.86:0/239933586 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f7609200640 1 --2- 172.21.15.86:0/239933586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406b500 0x7f760406dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f7603400640 1 --2- 172.21.15.86:0/239933586 >> v2:172.21.15.194:3300/0 conn(0x7f760406ef10 0x7f760406afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.262+0000 7f7603e00640 1 -- 172.21.15.86:0/239933586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406b500 msgr2=0x7f760406dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.263+0000 7f7603e00640 1 --2- 172.21.15.86:0/239933586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406b500 0x7f760406dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.263+0000 7f7603e00640 1 -- 172.21.15.86:0/239933586 >> v2:172.21.15.194:3300/0 conn(0x7f760406ef10 msgr2=0x7f760406afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:40.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.263+0000 7f7603e00640 1 --2- 172.21.15.86:0/239933586 >> v2:172.21.15.194:3300/0 conn(0x7f760406ef10 0x7f760406afc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.263+0000 7f7603e00640 1 -- 172.21.15.86:0/239933586 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76040ccaf0 con 0x7f7604145690 2024-03-06T07:28:40.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.263+0000 7f7603400640 1 --2- 172.21.15.86:0/239933586 >> v2:172.21.15.194:3300/0 conn(0x7f760406ef10 0x7f760406afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:40.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.263+0000 7f7609200640 1 --2- 172.21.15.86:0/239933586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406b500 0x7f760406dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.264+0000 7f7603e00640 1 --2- 172.21.15.86:0/239933586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f7604160470 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f76040cb100 tx=0x7f75f402b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=159ef44896f40e49 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.264+0000 7f7602a00640 1 -- 172.21.15.86:0/239933586 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75f4035020 con 0x7f7604145690 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.264+0000 7f7602a00640 1 -- 172.21.15.86:0/239933586 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75f402d040 con 0x7f7604145690 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.264+0000 7f7602a00640 1 -- 172.21.15.86:0/239933586 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75f4031a80 con 0x7f7604145690 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 -- 172.21.15.86:0/239933586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 msgr2=0x7f7604160470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 --2- 172.21.15.86:0/239933586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f7604160470 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f76040cb100 tx=0x7f75f402b900 comp rx=0 tx=0).stop 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 -- 172.21.15.86:0/239933586 shutdown_connections 2024-03-06T07:28:40.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 --2- 172.21.15.86:0/239933586 >> v2:172.21.15.194:3300/0 conn(0x7f760406ef10 0x7f760406afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 --2- 172.21.15.86:0/239933586 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406b500 0x7f760406dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 --2- 172.21.15.86:0/239933586 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f7604160470 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 -- 172.21.15.86:0/239933586 >> 172.21.15.86:0/239933586 conn(0x7f76040b27d0 msgr2=0x7f76040b2bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:40.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 -- 172.21.15.86:0/239933586 shutdown_connections 2024-03-06T07:28:40.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.265+0000 7f760b200640 1 -- 172.21.15.86:0/239933586 wait complete. 2024-03-06T07:28:40.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.267+0000 7f760b200640 1 Processor -- start 2024-03-06T07:28:40.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.267+0000 7f760b200640 1 -- start start 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.268+0000 7f760b200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f760406b500 0x7f76041f37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.268+0000 7f760b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406ef10 0x7f76040cf850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.268+0000 7f760b200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f76041429b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.268+0000 7f760b200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76040cc7a0 con 0x7f7604145690 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.268+0000 7f760b200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f760406f3a0 con 0x7f760406b500 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.268+0000 7f760b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76040b3ad0 con 0x7f760406ef10 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f76041429b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f76041429b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36292/0 (socket says 172.21.15.86:36292) 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 -- 172.21.15.86:0/3390901812 learned_addr learned my addr 172.21.15.86:0/3390901812 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7603400640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406ef10 0x7f76040cf850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7603e00640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f760406b500 0x7f76041f37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 -- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406ef10 msgr2=0x7f76040cf850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406ef10 0x7f76040cf850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 -- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f760406b500 msgr2=0x7f76041f37f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f760406b500 0x7f76041f37f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7609200640 1 -- 172.21.15.86:0/3390901812 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7604141870 con 0x7f7604145690 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.269+0000 7f7603e00640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f760406b500 0x7f76041f37f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.270+0000 7f7609200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f76041429b0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f75fc00cd50 tx=0x7f75fc00a650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.270+0000 7f75fbe00640 1 -- 172.21.15.86:0/3390901812 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75fc00f680 con 0x7f7604145690 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.270+0000 7f75fbe00640 1 -- 172.21.15.86:0/3390901812 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75fc00f840 con 0x7f7604145690 2024-03-06T07:28:40.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.270+0000 7f75fbe00640 1 -- 172.21.15.86:0/3390901812 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75fc00fc30 con 0x7f7604145690 2024-03-06T07:28:40.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.270+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7604141b10 con 0x7f7604145690 2024-03-06T07:28:40.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.270+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f76041420d0 con 0x7f7604145690 2024-03-06T07:28:40.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.271+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f760404f330 con 0x7f7604145690 2024-03-06T07:28:40.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.272+0000 7f75fbe00640 1 -- 172.21.15.86:0/3390901812 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f75fc0213f0 con 0x7f7604145690 2024-03-06T07:28:40.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.273+0000 7f75fbe00640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75e406b4d0 0x7f75e406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.274+0000 7f75fbe00640 1 -- 172.21.15.86:0/3390901812 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f75fc08af80 con 0x7f7604145690 2024-03-06T07:28:40.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.274+0000 7f7603e00640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75e406b4d0 0x7f75e406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.275+0000 7f7603e00640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75e406b4d0 0x7f75e406d990 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f75f4036000 tx=0x7f75f4008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.279+0000 7f75fbe00640 1 -- 172.21.15.86:0/3390901812 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f75fc014030 con 0x7f7604145690 2024-03-06T07:28:40.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.474+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f7604072380 con 0x7f7604145690 2024-03-06T07:28:40.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.475+0000 7f75fbe00640 1 -- 172.21.15.86:0/3390901812 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v35) v1 ==== 100+0+11 (secure 0 0 0) 0x7f75fc059790 con 0x7f7604145690 2024-03-06T07:28:40.477 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:28:40.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75e406b4d0 msgr2=0x7f75e406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75e406b4d0 0x7f75e406d990 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f75f4036000 tx=0x7f75f4008030 comp rx=0 tx=0).stop 2024-03-06T07:28:40.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 msgr2=0x7f76041429b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f76041429b0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f75fc00cd50 tx=0x7f75fc00a650 comp rx=0 tx=0).stop 2024-03-06T07:28:40.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 shutdown_connections 2024-03-06T07:28:40.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75e406b4d0 0x7f75e406d990 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7604145690 0x7f76041429b0 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f760406ef10 0x7f76040cf850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 --2- 172.21.15.86:0/3390901812 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f760406b500 0x7f76041f37f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 >> 172.21.15.86:0/3390901812 conn(0x7f76040b27d0 msgr2=0x7f76040ca530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:40.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.477+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 shutdown_connections 2024-03-06T07:28:40.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.478+0000 7f760b200640 1 -- 172.21.15.86:0/3390901812 wait complete. 2024-03-06T07:28:40.498 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:28:40.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.702+0000 7eff53800640 1 Processor -- start 2024-03-06T07:28:40.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.702+0000 7eff53800640 1 -- start start 2024-03-06T07:28:40.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.703+0000 7eff53800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c0c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.703+0000 7eff53800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7eff4c0c5880 0x7eff4c165b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.703+0000 7eff53800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7eff4c145b30 con 0x7eff4c0c4d10 2024-03-06T07:28:40.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.703+0000 7eff53800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7eff4c142ce0 con 0x7eff4c0c5880 2024-03-06T07:28:40.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.703+0000 7eff53800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff4c0cbca0 con 0x7eff4c0c4930 2024-03-06T07:28:40.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.704+0000 7eff50e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c0c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.704+0000 7eff51800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7eff4c0c5880 0x7eff4c165b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.704+0000 7eff50e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c0c6460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:37646/0 (socket says 172.21.15.86:37646) 2024-03-06T07:28:40.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.704+0000 7eff51800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7eff4c0c5880 0x7eff4c165b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:43802/0 (socket says 172.21.15.86:43802) 2024-03-06T07:28:40.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.704+0000 7eff50e00640 1 -- 172.21.15.86:0/1004840305 learned_addr learned my addr 172.21.15.86:0/1004840305 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:40.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.705+0000 7eff50e00640 1 -- 172.21.15.86:0/1004840305 >> v2:172.21.15.194:3300/0 conn(0x7eff4c0c5880 msgr2=0x7eff4c165b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:40.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.705+0000 7eff50e00640 1 --2- 172.21.15.86:0/1004840305 >> v2:172.21.15.194:3300/0 conn(0x7eff4c0c5880 0x7eff4c165b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.705+0000 7eff50e00640 1 -- 172.21.15.86:0/1004840305 >> v1:172.21.15.194:6789/0 conn(0x7eff4c0c4d10 legacy=0x7eff4c0c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.705+0000 7eff50e00640 1 -- 172.21.15.86:0/1004840305 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff4c0cbf80 con 0x7eff4c0c4930 2024-03-06T07:28:40.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.706+0000 7eff50e00640 1 --2- 172.21.15.86:0/1004840305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c0c6460 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7eff3c009870 tx=0x7eff3c02b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d96694f5d5e46099 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:40.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.707+0000 7eff4be00640 1 -- 172.21.15.86:0/1004840305 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff3c035020 con 0x7eff4c0c4930 2024-03-06T07:28:40.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.707+0000 7eff4be00640 1 -- 172.21.15.86:0/1004840305 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff3c031640 con 0x7eff4c0c4930 2024-03-06T07:28:40.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.707+0000 7eff4be00640 1 -- 172.21.15.86:0/1004840305 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff3c02d040 con 0x7eff4c0c4930 2024-03-06T07:28:40.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.707+0000 7eff53800640 1 -- 172.21.15.86:0/1004840305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 msgr2=0x7eff4c0c6460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.707+0000 7eff53800640 1 --2- 172.21.15.86:0/1004840305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c0c6460 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7eff3c009870 tx=0x7eff3c02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.708+0000 7eff53800640 1 -- 172.21.15.86:0/1004840305 shutdown_connections 2024-03-06T07:28:40.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.708+0000 7eff53800640 1 --2- 172.21.15.86:0/1004840305 >> v2:172.21.15.194:3300/0 conn(0x7eff4c0c5880 0x7eff4c165b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.708+0000 7eff53800640 1 --2- 172.21.15.86:0/1004840305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c0c6460 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.708+0000 7eff53800640 1 -- 172.21.15.86:0/1004840305 >> 172.21.15.86:0/1004840305 conn(0x7eff4c0cc4f0 msgr2=0x7eff4c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:40.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.708+0000 7eff53800640 1 -- 172.21.15.86:0/1004840305 shutdown_connections 2024-03-06T07:28:40.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.708+0000 7eff53800640 1 -- 172.21.15.86:0/1004840305 wait complete. 2024-03-06T07:28:40.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.710+0000 7eff53800640 1 Processor -- start 2024-03-06T07:28:40.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.710+0000 7eff53800640 1 -- start start 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.711+0000 7eff53800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c13d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.711+0000 7eff53800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff4c0c4d10 0x7eff4c1e5d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.711+0000 7eff53800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff4c0c5880 0x7eff4c1e6280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.711+0000 7eff53800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff4c06e600 con 0x7eff4c0c4d10 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.711+0000 7eff53800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff4c0ca3b0 con 0x7eff4c0c5880 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.711+0000 7eff53800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff4c0cc260 con 0x7eff4c0c4930 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff4c0c4d10 0x7eff4c1e5d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff4c0c4d10 0x7eff4c1e5d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36300/0 (socket says 172.21.15.86:36300) 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 -- 172.21.15.86:0/1890748814 learned_addr learned my addr 172.21.15.86:0/1890748814 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff50e00640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c13d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff51800640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff4c0c5880 0x7eff4c1e6280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 -- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 msgr2=0x7eff4c13d8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c13d8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 -- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff4c0c5880 msgr2=0x7eff4c1e6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff4c0c5880 0x7eff4c1e6280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 -- 172.21.15.86:0/1890748814 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff4c13f510 con 0x7eff4c0c4d10 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.712+0000 7eff43e00640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff4c0c4d10 0x7eff4c1e5d40 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7eff3400c430 tx=0x7eff3400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.713+0000 7eff49600640 1 -- 172.21.15.86:0/1890748814 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff340052d0 con 0x7eff4c0c4d10 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.713+0000 7eff49600640 1 -- 172.21.15.86:0/1890748814 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff34005490 con 0x7eff4c0c4d10 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.713+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff4c13f7f0 con 0x7eff4c0c4d10 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.713+0000 7eff49600640 1 -- 172.21.15.86:0/1890748814 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff34007260 con 0x7eff4c0c4d10 2024-03-06T07:28:40.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.713+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff4c13fdb0 con 0x7eff4c0c4d10 2024-03-06T07:28:40.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.714+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7eff4c04f330 con 0x7eff4c0c4d10 2024-03-06T07:28:40.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.715+0000 7eff49600640 1 -- 172.21.15.86:0/1890748814 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7eff34007400 con 0x7eff4c0c4d10 2024-03-06T07:28:40.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.716+0000 7eff49600640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff2806b4d0 0x7eff2806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:40.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.716+0000 7eff49600640 1 -- 172.21.15.86:0/1890748814 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7eff34089790 con 0x7eff4c0c4d10 2024-03-06T07:28:40.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.716+0000 7eff50e00640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff2806b4d0 0x7eff2806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:40.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.717+0000 7eff50e00640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff2806b4d0 0x7eff2806d990 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7eff3c036000 tx=0x7eff3c03c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:40.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.721+0000 7eff49600640 1 -- 172.21.15.86:0/1890748814 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7eff4c145b30 con 0x7eff4c0c4d10 2024-03-06T07:28:40.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.931+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7eff4c0c6660 con 0x7eff4c0c4d10 2024-03-06T07:28:40.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.932+0000 7eff49600640 1 -- 172.21.15.86:0/1890748814 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v35) v1 ==== 80+0+0 (secure 0 0 0) 0x7eff34058570 con 0x7eff4c0c4d10 2024-03-06T07:28:40.934 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:28:40.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.934+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff2806b4d0 msgr2=0x7eff2806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.934+0000 7eff53800640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff2806b4d0 0x7eff2806d990 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7eff3c036000 tx=0x7eff3c03c040 comp rx=0 tx=0).stop 2024-03-06T07:28:40.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.934+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff4c0c4d10 msgr2=0x7eff4c1e5d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.934+0000 7eff53800640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff4c0c4d10 0x7eff4c1e5d40 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7eff3400c430 tx=0x7eff3400c8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.934+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 shutdown_connections 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.934+0000 7eff53800640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7eff2806b4d0 0x7eff2806d990 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.934+0000 7eff53800640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7eff4c0c5880 0x7eff4c1e6280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.935+0000 7eff53800640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7eff4c0c4d10 0x7eff4c1e5d40 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.935+0000 7eff53800640 1 --2- 172.21.15.86:0/1890748814 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7eff4c0c4930 0x7eff4c13d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.935+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 >> 172.21.15.86:0/1890748814 conn(0x7eff4c0cc4f0 msgr2=0x7eff4c143fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.935+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 shutdown_connections 2024-03-06T07:28:40.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:40.935+0000 7eff53800640 1 -- 172.21.15.86:0/1890748814 wait complete. 2024-03-06T07:28:40.956 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:28:40.956 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:40.956 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:28:41.084 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:28:41.084 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:28:41.084 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:28:41.123 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:28:41.124 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:41.161 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:28:41.161 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:28:41.161 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:41.207 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:28:41.207 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:28:41.207 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:41.249 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:28:41.250 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:41.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.461+0000 7f631d200640 1 Processor -- start 2024-03-06T07:28:41.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.461+0000 7f631d200640 1 -- start start 2024-03-06T07:28:41.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.462+0000 7f631d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f631806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.462+0000 7f631d200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6318072340 0x7f631806ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.462+0000 7f631d200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f631815b090 con 0x7f631815acb0 2024-03-06T07:28:41.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.462+0000 7f631d200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63180b2a30 con 0x7f6318072340 2024-03-06T07:28:41.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.462+0000 7f631d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63180b4360 con 0x7f63180b4680 2024-03-06T07:28:41.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.463+0000 7f6316800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6318072340 0x7f631806ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.463+0000 7f630d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f631806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.463+0000 7f6316800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6318072340 0x7f631806ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:43810/0 (socket says 172.21.15.86:43810) 2024-03-06T07:28:41.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.463+0000 7f630d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f631806e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:37654/0 (socket says 172.21.15.86:37654) 2024-03-06T07:28:41.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.463+0000 7f6316800640 1 -- 172.21.15.86:0/2466612290 learned_addr learned my addr 172.21.15.86:0/2466612290 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:41.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.464+0000 7f630d400640 1 -- 172.21.15.86:0/2466612290 >> v2:172.21.15.194:3300/0 conn(0x7f6318072340 msgr2=0x7f631806ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:41.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.464+0000 7f630d400640 1 --2- 172.21.15.86:0/2466612290 >> v2:172.21.15.194:3300/0 conn(0x7f6318072340 0x7f631806ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.464+0000 7f630d400640 1 -- 172.21.15.86:0/2466612290 >> v1:172.21.15.194:6789/0 conn(0x7f631815acb0 legacy=0x7f63180b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.464+0000 7f630d400640 1 -- 172.21.15.86:0/2466612290 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63180b2cf0 con 0x7f63180b4680 2024-03-06T07:28:41.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.465+0000 7f630d400640 1 --2- 172.21.15.86:0/2466612290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f631806e4f0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f63000097a0 tx=0x7f630002b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c3940e3f4c3d9c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:41.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f6315400640 1 -- 172.21.15.86:0/2466612290 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6300035020 con 0x7f63180b4680 2024-03-06T07:28:41.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f6315400640 1 -- 172.21.15.86:0/2466612290 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6300031830 con 0x7f63180b4680 2024-03-06T07:28:41.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f6315400640 1 -- 172.21.15.86:0/2466612290 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6300031c00 con 0x7f63180b4680 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 -- 172.21.15.86:0/2466612290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 msgr2=0x7f631806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 --2- 172.21.15.86:0/2466612290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f631806e4f0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f63000097a0 tx=0x7f630002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 -- 172.21.15.86:0/2466612290 shutdown_connections 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 --2- 172.21.15.86:0/2466612290 >> v2:172.21.15.194:3300/0 conn(0x7f6318072340 0x7f631806ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 --2- 172.21.15.86:0/2466612290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f631806e4f0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 -- 172.21.15.86:0/2466612290 >> 172.21.15.86:0/2466612290 conn(0x7f63180b1730 msgr2=0x7f63180b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 -- 172.21.15.86:0/2466612290 shutdown_connections 2024-03-06T07:28:41.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.466+0000 7f631d200640 1 -- 172.21.15.86:0/2466612290 wait complete. 2024-03-06T07:28:41.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.468+0000 7f631d200640 1 Processor -- start 2024-03-06T07:28:41.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.468+0000 7f631d200640 1 -- start start 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.469+0000 7f631d200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6318072340 0x7f631815bbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.469+0000 7f631d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f6318147590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.469+0000 7f631d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f631815acb0 0x7f631815d5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.469+0000 7f631d200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63180b4010 con 0x7f631815acb0 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.469+0000 7f631d200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63180b32b0 con 0x7f6318072340 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.469+0000 7f631d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63180ccfb0 con 0x7f63180b4680 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f631815acb0 0x7f631815d5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f631815acb0 0x7f631815d5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36306/0 (socket says 172.21.15.86:36306) 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 -- 172.21.15.86:0/1249935995 learned_addr learned my addr 172.21.15.86:0/1249935995 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f630d400640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f6318147590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6315e00640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6318072340 0x7f631815bbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 -- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 msgr2=0x7f6318147590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f6318147590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 -- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6318072340 msgr2=0x7f631815bbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6318072340 0x7f631815bbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 -- 172.21.15.86:0/1249935995 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f631815c2e0 con 0x7f631815acb0 2024-03-06T07:28:41.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6315e00640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6318072340 0x7f631815bbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:28:41.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.470+0000 7f6316800640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f631815acb0 0x7f631815d5a0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f631000e870 tx=0x7f631000ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:41.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.471+0000 7f630ea00640 1 -- 172.21.15.86:0/1249935995 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6310017070 con 0x7f631815acb0 2024-03-06T07:28:41.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.471+0000 7f630ea00640 1 -- 172.21.15.86:0/1249935995 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f631000c940 con 0x7f631815acb0 2024-03-06T07:28:41.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.471+0000 7f630ea00640 1 -- 172.21.15.86:0/1249935995 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f631000cd30 con 0x7f631815acb0 2024-03-06T07:28:41.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.471+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f631815cac0 con 0x7f631815acb0 2024-03-06T07:28:41.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.471+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f631815d080 con 0x7f631815acb0 2024-03-06T07:28:41.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.472+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f631804f330 con 0x7f631815acb0 2024-03-06T07:28:41.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.473+0000 7f630ea00640 1 -- 172.21.15.86:0/1249935995 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f631001a070 con 0x7f631815acb0 2024-03-06T07:28:41.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.474+0000 7f630ea00640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62f406b4d0 0x7f62f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.474+0000 7f630ea00640 1 -- 172.21.15.86:0/1249935995 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f631008a460 con 0x7f631815acb0 2024-03-06T07:28:41.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.474+0000 7f6315e00640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62f406b4d0 0x7f62f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.475+0000 7f6315e00640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62f406b4d0 0x7f62f406d990 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f6308001d60 tx=0x7f6308009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:41.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.478+0000 7f630ea00640 1 -- 172.21.15.86:0/1249935995 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6310058d20 con 0x7f631815acb0 2024-03-06T07:28:41.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.706+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6318151ab0 con 0x7f631815acb0 2024-03-06T07:28:41.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.707+0000 7f630ea00640 1 -- 172.21.15.86:0/1249935995 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f631005cdc0 con 0x7f631815acb0 2024-03-06T07:28:41.710 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:41.710 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:41.710 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:41.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62f406b4d0 msgr2=0x7f62f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62f406b4d0 0x7f62f406d990 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f6308001d60 tx=0x7f6308009040 comp rx=0 tx=0).stop 2024-03-06T07:28:41.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f631815acb0 msgr2=0x7f631815d5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f631815acb0 0x7f631815d5a0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f631000e870 tx=0x7f631000ed30 comp rx=0 tx=0).stop 2024-03-06T07:28:41.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 shutdown_connections 2024-03-06T07:28:41.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62f406b4d0 0x7f62f406d990 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f631815acb0 0x7f631815d5a0 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63180b4680 0x7f6318147590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 --2- 172.21.15.86:0/1249935995 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6318072340 0x7f631815bbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.709+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 >> 172.21.15.86:0/1249935995 conn(0x7f63180b1730 msgr2=0x7f63180af6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:41.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.710+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 shutdown_connections 2024-03-06T07:28:41.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.710+0000 7f631d200640 1 -- 172.21.15.86:0/1249935995 wait complete. 2024-03-06T07:28:41.730 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:41.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.932+0000 7fed61000640 1 Processor -- start 2024-03-06T07:28:41.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.932+0000 7fed61000640 1 -- start start 2024-03-06T07:28:41.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.933+0000 7fed61000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fed5c0b4680 0x7fed5c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.933+0000 7fed61000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fed5c072340 0x7fed5c06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.933+0000 7fed61000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fed5c15b090 con 0x7fed5c15acb0 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.933+0000 7fed61000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fed5c0b2a30 con 0x7fed5c072340 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.933+0000 7fed61000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fed5c0b4360 con 0x7fed5c0b4680 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed5a800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fed5c072340 0x7fed5c06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed5a800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fed5c072340 0x7fed5c06ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36316/0 (socket says 172.21.15.86:36316) 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed5a800640 1 -- 172.21.15.86:0/3350312682 learned_addr learned my addr 172.21.15.86:0/3350312682 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed51400640 1 --2- 172.21.15.86:0/3350312682 >> v2:172.21.15.194:3300/0 conn(0x7fed5c0b4680 0x7fed5c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed5a800640 1 -- 172.21.15.86:0/3350312682 >> v2:172.21.15.194:3300/0 conn(0x7fed5c0b4680 msgr2=0x7fed5c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed5a800640 1 --2- 172.21.15.86:0/3350312682 >> v2:172.21.15.194:3300/0 conn(0x7fed5c0b4680 0x7fed5c06e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed5a800640 1 -- 172.21.15.86:0/3350312682 >> v1:172.21.15.194:6789/0 conn(0x7fed5c15acb0 legacy=0x7fed5c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:41.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.934+0000 7fed5a800640 1 -- 172.21.15.86:0/3350312682 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed5c0b2cf0 con 0x7fed5c072340 2024-03-06T07:28:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.935+0000 7fed5a800640 1 --2- 172.21.15.86:0/3350312682 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c072340 0x7fed5c06ea30 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fed5c068560 tx=0x7fed5402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6f476a40360530e3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.935+0000 7fed59400640 1 -- 172.21.15.86:0/3350312682 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed54035020 con 0x7fed5c072340 2024-03-06T07:28:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed59400640 1 -- 172.21.15.86:0/3350312682 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fed54031870 con 0x7fed5c072340 2024-03-06T07:28:41.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed59400640 1 -- 172.21.15.86:0/3350312682 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed54031c40 con 0x7fed5c072340 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 -- 172.21.15.86:0/3350312682 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c072340 msgr2=0x7fed5c06ea30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 --2- 172.21.15.86:0/3350312682 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c072340 0x7fed5c06ea30 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fed5c068560 tx=0x7fed5402b910 comp rx=0 tx=0).stop 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 -- 172.21.15.86:0/3350312682 shutdown_connections 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 --2- 172.21.15.86:0/3350312682 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c072340 0x7fed5c06ea30 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 --2- 172.21.15.86:0/3350312682 >> v2:172.21.15.194:3300/0 conn(0x7fed5c0b4680 0x7fed5c06e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 -- 172.21.15.86:0/3350312682 >> 172.21.15.86:0/3350312682 conn(0x7fed5c0b1730 msgr2=0x7fed5c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 -- 172.21.15.86:0/3350312682 shutdown_connections 2024-03-06T07:28:41.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.936+0000 7fed61000640 1 -- 172.21.15.86:0/3350312682 wait complete. 2024-03-06T07:28:41.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.938+0000 7fed61000640 1 Processor -- start 2024-03-06T07:28:41.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.939+0000 7fed61000640 1 -- start start 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed61000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed5c072340 0x7fed5c1476c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed61000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c0b4680 0x7fed5c1510a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed61000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed5c15acb0 0x7fed5c159420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed61000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed5c0b4010 con 0x7fed5c0b4680 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed61000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed5c0b32b0 con 0x7fed5c15acb0 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed61000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fed5c0ccfb0 con 0x7fed5c072340 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c0b4680 0x7fed5c1510a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c0b4680 0x7fed5c1510a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36324/0 (socket says 172.21.15.86:36324) 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 -- 172.21.15.86:0/2727089352 learned_addr learned my addr 172.21.15.86:0/2727089352 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed59e00640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed5c072340 0x7fed5c1476c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed5a800640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed5c15acb0 0x7fed5c159420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 -- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed5c072340 msgr2=0x7fed5c1476c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed5c072340 0x7fed5c1476c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 -- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed5c15acb0 msgr2=0x7fed5c159420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed5c15acb0 0x7fed5c159420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed51400640 1 -- 172.21.15.86:0/2727089352 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed5c146af0 con 0x7fed5c0b4680 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.940+0000 7fed59e00640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed5c072340 0x7fed5c1476c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.941+0000 7fed51400640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c0b4680 0x7fed5c1510a0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fed4400ce10 tx=0x7fed4400a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.941+0000 7fed52a00640 1 -- 172.21.15.86:0/2727089352 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed44003b90 con 0x7fed5c0b4680 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.941+0000 7fed52a00640 1 -- 172.21.15.86:0/2727089352 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fed44003d30 con 0x7fed5c0b4680 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.941+0000 7fed52a00640 1 -- 172.21.15.86:0/2727089352 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed44004240 con 0x7fed5c0b4680 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.941+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fed5c149920 con 0x7fed5c0b4680 2024-03-06T07:28:41.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.941+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fed5c149ee0 con 0x7fed5c0b4680 2024-03-06T07:28:41.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.942+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fed5c04f330 con 0x7fed5c0b4680 2024-03-06T07:28:41.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.943+0000 7fed52a00640 1 -- 172.21.15.86:0/2727089352 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fed440043e0 con 0x7fed5c0b4680 2024-03-06T07:28:41.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.944+0000 7fed52a00640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed3806b4d0 0x7fed3806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:41.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.944+0000 7fed52a00640 1 -- 172.21.15.86:0/2727089352 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fed4408ad20 con 0x7fed5c0b4680 2024-03-06T07:28:41.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.944+0000 7fed59e00640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed3806b4d0 0x7fed3806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:41.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.945+0000 7fed59e00640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed3806b4d0 0x7fed3806d990 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fed4c00a830 tx=0x7fed4c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:41.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:41.948+0000 7fed52a00640 1 -- 172.21.15.86:0/2727089352 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fed4401d340 con 0x7fed5c0b4680 2024-03-06T07:28:42.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.144+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fed5c0b03f0 con 0x7fed5c0b4680 2024-03-06T07:28:42.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.147+0000 7fed52a00640 1 -- 172.21.15.86:0/2727089352 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v35) v1 ==== 74+0+16681 (secure 0 0 0) 0x7fed44059ba0 con 0x7fed5c0b4680 2024-03-06T07:28:42.150 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:42.150 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":35,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:15.157270+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:11.444335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:12.623953+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"35","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:42.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.149+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed3806b4d0 msgr2=0x7fed3806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.149+0000 7fed61000640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed3806b4d0 0x7fed3806d990 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fed4c00a830 tx=0x7fed4c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.149+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c0b4680 msgr2=0x7fed5c1510a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.149+0000 7fed61000640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c0b4680 0x7fed5c1510a0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fed4400ce10 tx=0x7fed4400a660 comp rx=0 tx=0).stop 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 shutdown_connections 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed3806b4d0 0x7fed3806d990 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed5c15acb0 0x7fed5c159420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed5c0b4680 0x7fed5c1510a0 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 --2- 172.21.15.86:0/2727089352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed5c072340 0x7fed5c1476c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 >> 172.21.15.86:0/2727089352 conn(0x7fed5c0b1730 msgr2=0x7fed5c0b0bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 shutdown_connections 2024-03-06T07:28:42.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.150+0000 7fed61000640 1 -- 172.21.15.86:0/2727089352 wait complete. 2024-03-06T07:28:42.170 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:28:11.444335+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '34', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 5, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:28:12.623953+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '35', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:28:42.171 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:28:42.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.372+0000 7fd571000640 1 Processor -- start 2024-03-06T07:28:42.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.372+0000 7fd571000640 1 -- start start 2024-03-06T07:28:42.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.373+0000 7fd571000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd56c0b3f70 0x7fd56c160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.374+0000 7fd571000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd56c071440 con 0x7fd56c070ff0 2024-03-06T07:28:42.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.374+0000 7fd571000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd56c0b46c0 con 0x7fd56c15fc10 2024-03-06T07:28:42.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.374+0000 7fd571000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd56c0b49a0 con 0x7fd56c0b3f70 2024-03-06T07:28:42.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.374+0000 7fd56a800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd56c070ff0 0x7fd56c0ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38648/0 (socket says 172.21.15.86:38648) 2024-03-06T07:28:42.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.374+0000 7fd56a800640 1 -- 172.21.15.86:0/909759691 learned_addr learned my addr 172.21.15.86:0/909759691 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:42.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.374+0000 7fd569400640 1 --2- 172.21.15.86:0/909759691 >> v2:172.21.15.194:3300/0 conn(0x7fd56c0b3f70 0x7fd56c160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.374+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1897039614 0 0) 0x7fd56c071440 con 0x7fd56c070ff0 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd550003520 con 0x7fd56c070ff0 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4281980991 0 0) 0x7fd550003520 con 0x7fd56c070ff0 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 >> v2:172.21.15.194:3300/0 conn(0x7fd56c0b3f70 msgr2=0x7fd56c160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 --2- 172.21.15.86:0/909759691 >> v2:172.21.15.194:3300/0 conn(0x7fd56c0b3f70 0x7fd56c160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 >> v1:172.21.15.194:6789/0 conn(0x7fd56c15fc10 legacy=0x7fd56c15fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd56c072c90 con 0x7fd56c070ff0 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd5640019f0 con 0x7fd56c070ff0 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 >> v1:172.21.15.86:6789/0 conn(0x7fd56c070ff0 legacy=0x7fd56c0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd550003a00 0x7fd550003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd550004800 0x7fd550004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5500051c0 0x7fd55000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd56c0b49a0 con 0x7fd5500051c0 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd55000c0c0 con 0x7fd550004800 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd55000c2c0 con 0x7fd550003a00 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd569400640 1 --2- 172.21.15.86:0/909759691 >> v2:172.21.15.194:3300/0 conn(0x7fd56c0b3f70 0x7fd56c160700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.375+0000 7fd56a800640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5500051c0 0x7fd55000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.376+0000 7fd569e00640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd550003a00 0x7fd550003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.376+0000 7fd56a800640 1 -- 172.21.15.86:0/909759691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd550003a00 msgr2=0x7fd550003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.376+0000 7fd56a800640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd550003a00 0x7fd550003e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.376+0000 7fd56a800640 1 -- 172.21.15.86:0/909759691 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd550004800 msgr2=0x7fd550004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.376+0000 7fd56a800640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd550004800 0x7fd550004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.376+0000 7fd56a800640 1 -- 172.21.15.86:0/909759691 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd550003520 con 0x7fd5500051c0 2024-03-06T07:28:42.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.377+0000 7fd56a800640 1 -- 172.21.15.86:0/909759691 reap_dead start 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.377+0000 7fd56a800640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5500051c0 0x7fd55000b9c0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7fd56400a7b0 tx=0x7fd550005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f33988ff3fc023fb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.377+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd56400a3d0 con 0x7fd5500051c0 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.377+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd564034650 con 0x7fd5500051c0 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.377+0000 7fd568a00640 1 -- 172.21.15.86:0/909759691 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd564034a40 con 0x7fd5500051c0 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.378+0000 7fd571000640 1 -- 172.21.15.86:0/909759691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5500051c0 msgr2=0x7fd55000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.378+0000 7fd571000640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5500051c0 0x7fd55000b9c0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7fd56400a7b0 tx=0x7fd550005830 comp rx=0 tx=0).stop 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.378+0000 7fd571000640 1 -- 172.21.15.86:0/909759691 shutdown_connections 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.378+0000 7fd571000640 1 --2- 172.21.15.86:0/909759691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5500051c0 0x7fd55000b9c0 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.378+0000 7fd571000640 1 -- 172.21.15.86:0/909759691 >> 172.21.15.86:0/909759691 conn(0x7fd56c0b1940 msgr2=0x7fd56c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.378+0000 7fd571000640 1 -- 172.21.15.86:0/909759691 shutdown_connections 2024-03-06T07:28:42.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.378+0000 7fd571000640 1 -- 172.21.15.86:0/909759691 wait complete. 2024-03-06T07:28:42.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.380+0000 7fd571000640 1 Processor -- start 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.380+0000 7fd571000640 1 -- start start 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd571000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5500051c0 0x7fd56c1f8230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd571000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd56c1450b0 0x7fd56c15d660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd571000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd56c0adcf0 0x7fd56c1420f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd571000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd56c072a30 con 0x7fd56c1450b0 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd571000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd56c0728b0 con 0x7fd56c0adcf0 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd571000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd56c15c2e0 con 0x7fd5500051c0 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd569400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd56c1450b0 0x7fd56c15d660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd569400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd56c1450b0 0x7fd56c15d660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36344/0 (socket says 172.21.15.86:36344) 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd569400640 1 -- 172.21.15.86:0/3516850169 learned_addr learned my addr 172.21.15.86:0/3516850169 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:42.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd569e00640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5500051c0 0x7fd56c1f8230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.381+0000 7fd56a800640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd56c0adcf0 0x7fd56c1420f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd569400640 1 -- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5500051c0 msgr2=0x7fd56c1f8230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd569400640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5500051c0 0x7fd56c1f8230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd569400640 1 -- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd56c0adcf0 msgr2=0x7fd56c1420f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd569400640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd56c0adcf0 0x7fd56c1420f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd569400640 1 -- 172.21.15.86:0/3516850169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd56c145890 con 0x7fd56c1450b0 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd569400640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd56c1450b0 0x7fd56c15d660 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7fd558006c00 tx=0x7fd55800ed90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd562000640 1 -- 172.21.15.86:0/3516850169 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd558004030 con 0x7fd56c1450b0 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd562000640 1 -- 172.21.15.86:0/3516850169 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd5580041f0 con 0x7fd56c1450b0 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd56c145b70 con 0x7fd56c1450b0 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd562000640 1 -- 172.21.15.86:0/3516850169 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd55800c940 con 0x7fd56c1450b0 2024-03-06T07:28:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.382+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd56c145530 con 0x7fd56c1450b0 2024-03-06T07:28:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.383+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd56c04f330 con 0x7fd56c1450b0 2024-03-06T07:28:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.384+0000 7fd562000640 1 -- 172.21.15.86:0/3516850169 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd55800cae0 con 0x7fd56c1450b0 2024-03-06T07:28:42.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.385+0000 7fd562000640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd54c06b4d0 0x7fd54c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.385+0000 7fd562000640 1 -- 172.21.15.86:0/3516850169 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd55808a7d0 con 0x7fd56c1450b0 2024-03-06T07:28:42.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.385+0000 7fd569e00640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd54c06b4d0 0x7fd54c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.386+0000 7fd569e00640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd54c06b4d0 0x7fd54c06d990 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fd55c000c00 tx=0x7fd55c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:42.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.390+0000 7fd562000640 1 -- 172.21.15.86:0/3516850169 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd558017020 con 0x7fd56c1450b0 2024-03-06T07:28:42.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.569+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd56c0b16b0 con 0x7fd56c1450b0 2024-03-06T07:28:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.570+0000 7fd562000640 1 -- 172.21.15.86:0/3516850169 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v35) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd5580595b0 con 0x7fd56c1450b0 2024-03-06T07:28:42.573 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:28:42.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.572+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd54c06b4d0 msgr2=0x7fd54c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.572+0000 7fd571000640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd54c06b4d0 0x7fd54c06d990 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fd55c000c00 tx=0x7fd55c009040 comp rx=0 tx=0).stop 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.572+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd56c1450b0 msgr2=0x7fd56c15d660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.572+0000 7fd571000640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd56c1450b0 0x7fd56c15d660 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7fd558006c00 tx=0x7fd55800ed90 comp rx=0 tx=0).stop 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 shutdown_connections 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd54c06b4d0 0x7fd54c06d990 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd56c0adcf0 0x7fd56c1420f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd56c1450b0 0x7fd56c15d660 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 --2- 172.21.15.86:0/3516850169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5500051c0 0x7fd56c1f8230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 >> 172.21.15.86:0/3516850169 conn(0x7fd56c0b1940 msgr2=0x7fd56c09ca00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 shutdown_connections 2024-03-06T07:28:42.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.573+0000 7fd571000640 1 -- 172.21.15.86:0/3516850169 wait complete. 2024-03-06T07:28:42.594 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:28:42.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.799+0000 7f4d62400640 1 Processor -- start 2024-03-06T07:28:42.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.799+0000 7f4d62400640 1 -- start start 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.801+0000 7f4d62400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 0x7f4d5c070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.801+0000 7f4d62400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4d5c0b46c0 con 0x7f4d5c15fff0 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.801+0000 7f4d62400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4d5c0b49a0 con 0x7f4d5c0b3f70 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.801+0000 7f4d62400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d5c072c90 con 0x7f4d5c15fc10 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.801+0000 7f4d5aa00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4d5c15fff0 0x7f4d5c160450 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38658/0 (socket says 172.21.15.86:38658) 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.801+0000 7f4d5aa00640 1 -- 172.21.15.86:0/2502649305 learned_addr learned my addr 172.21.15.86:0/2502649305 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5b400640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 0x7f4d5c070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4251049409 0 0) 0x7f4d5c0b46c0 con 0x7f4d5c15fff0 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4d40003520 con 0x7f4d5c15fff0 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3703004023 0 0) 0x7f4d40003520 con 0x7f4d5c15fff0 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 msgr2=0x7f4d5c070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 0x7f4d5c070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 >> v1:172.21.15.194:6789/0 conn(0x7f4d5c0b3f70 legacy=0x7f4d5c0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d5c09dd80 con 0x7f4d5c15fff0 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f4d48001670 con 0x7f4d5c15fff0 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 >> v1:172.21.15.86:6789/0 conn(0x7f4d5c15fff0 legacy=0x7f4d5c160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 0x7f4d400039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d40003f60 0x7f4d40004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d40004d40 0x7f4d400051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d5c072c90 con 0x7f4d40003f60 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d4000baa0 con 0x7f4d40004d40 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.802+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d4000bca0 con 0x7f4d5c15fc10 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5aa00640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d40003f60 0x7f4d40004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5b400640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 0x7f4d400039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5be00640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d40004d40 0x7f4d400051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5aa00640 1 -- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 msgr2=0x7f4d400039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5aa00640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 0x7f4d400039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5aa00640 1 -- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d40004d40 msgr2=0x7f4d400051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:42.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5aa00640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d40004d40 0x7f4d400051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5aa00640 1 -- 172.21.15.86:0/2502649305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d40003520 con 0x7f4d40003f60 2024-03-06T07:28:42.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5b400640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d5c15fc10 0x7f4d400039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:42.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.803+0000 7f4d5be00640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d40004d40 0x7f4d400051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.804+0000 7f4d5aa00640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d40003f60 0x7f4d40004800 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f4d4800c210 tx=0x7f4d40005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=789a31e5c41ce270 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.804+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d48007d30 con 0x7f4d40003f60 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.804+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d48030070 con 0x7f4d40003f60 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.804+0000 7f4d5a000640 1 -- 172.21.15.86:0/2502649305 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d4802cda0 con 0x7f4d40003f60 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.805+0000 7f4d62400640 1 -- 172.21.15.86:0/2502649305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d40003f60 msgr2=0x7f4d40004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.805+0000 7f4d62400640 1 --2- 172.21.15.86:0/2502649305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d40003f60 0x7f4d40004800 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f4d4800c210 tx=0x7f4d40005830 comp rx=0 tx=0).stop 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.805+0000 7f4d5be00640 1 -- 172.21.15.86:0/2502649305 reap_dead start 2024-03-06T07:28:42.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.805+0000 7f4d62400640 1 -- 172.21.15.86:0/2502649305 shutdown_connections 2024-03-06T07:28:42.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.805+0000 7f4d62400640 1 -- 172.21.15.86:0/2502649305 >> 172.21.15.86:0/2502649305 conn(0x7f4d5c0b1940 msgr2=0x7f4d5c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:42.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.805+0000 7f4d62400640 1 -- 172.21.15.86:0/2502649305 shutdown_connections 2024-03-06T07:28:42.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.805+0000 7f4d62400640 1 -- 172.21.15.86:0/2502649305 wait complete. 2024-03-06T07:28:42.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.807+0000 7f4d62400640 1 Processor -- start 2024-03-06T07:28:42.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.807+0000 7f4d62400640 1 -- start start 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d62400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d40003f60 0x7f4d5c09ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d62400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d5c14ec50 0x7f4d5c14f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d62400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d5c06d9e0 0x7f4d5c06de60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d62400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d5c15c460 con 0x7f4d5c14ec50 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d62400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d5c15c2e0 con 0x7f4d5c06d9e0 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d62400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d5c070ff0 con 0x7f4d40003f60 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d5c14ec50 0x7f4d5c14f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d5c14ec50 0x7f4d5c14f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36370/0 (socket says 172.21.15.86:36370) 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 -- 172.21.15.86:0/287853945 learned_addr learned my addr 172.21.15.86:0/287853945 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:42.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5b400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d40003f60 0x7f4d5c09ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 -- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d40003f60 msgr2=0x7f4d5c09ba40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d40003f60 0x7f4d5c09ba40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 -- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d5c06d9e0 msgr2=0x7f4d5c06de60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5be00640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d5c06d9e0 0x7f4d5c06de60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d5c06d9e0 0x7f4d5c06de60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5aa00640 1 -- 172.21.15.86:0/287853945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d5c14de90 con 0x7f4d5c14ec50 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.808+0000 7f4d5b400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d40003f60 0x7f4d5c09ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.809+0000 7f4d5aa00640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d5c14ec50 0x7f4d5c14f0d0 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f4d4800c330 tx=0x7f4d48001a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.809+0000 7f4d53400640 1 -- 172.21.15.86:0/287853945 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d48045070 con 0x7f4d5c14ec50 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.809+0000 7f4d53400640 1 -- 172.21.15.86:0/287853945 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d48004bf0 con 0x7f4d5c14ec50 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.809+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d5c14e170 con 0x7f4d5c14ec50 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.809+0000 7f4d53400640 1 -- 172.21.15.86:0/287853945 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d48007d10 con 0x7f4d5c14ec50 2024-03-06T07:28:42.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.809+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4d5c06e910 con 0x7f4d5c14ec50 2024-03-06T07:28:42.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.810+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4d5c04f330 con 0x7f4d5c14ec50 2024-03-06T07:28:42.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.811+0000 7f4d53400640 1 -- 172.21.15.86:0/287853945 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4d48007070 con 0x7f4d5c14ec50 2024-03-06T07:28:42.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.812+0000 7f4d53400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d3c06b500 0x7f4d3c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:42.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.812+0000 7f4d53400640 1 -- 172.21.15.86:0/287853945 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4d480ab6c0 con 0x7f4d5c14ec50 2024-03-06T07:28:42.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.812+0000 7f4d5b400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d3c06b500 0x7f4d3c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:42.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.813+0000 7f4d5b400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d3c06b500 0x7f4d3c06d9c0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f4d4c009aa0 tx=0x7f4d4c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:42.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:42.817+0000 7f4d53400640 1 -- 172.21.15.86:0/287853945 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4d4807a520 con 0x7f4d5c14ec50 2024-03-06T07:28:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.013+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f4d5c06ef30 con 0x7f4d5c14ec50 2024-03-06T07:28:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.014+0000 7f4d53400640 1 -- 172.21.15.86:0/287853945 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v35) v1 ==== 104+0+11 (secure 0 0 0) 0x7f4d48079e20 con 0x7f4d5c14ec50 2024-03-06T07:28:43.017 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:28:43.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d3c06b500 msgr2=0x7f4d3c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d3c06b500 0x7f4d3c06d9c0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f4d4c009aa0 tx=0x7f4d4c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d5c14ec50 msgr2=0x7f4d5c14f0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d5c14ec50 0x7f4d5c14f0d0 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f4d4800c330 tx=0x7f4d48001a40 comp rx=0 tx=0).stop 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 shutdown_connections 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d3c06b500 0x7f4d3c06d9c0 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d5c06d9e0 0x7f4d5c06de60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d5c14ec50 0x7f4d5c14f0d0 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 --2- 172.21.15.86:0/287853945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d40003f60 0x7f4d5c09ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 >> 172.21.15.86:0/287853945 conn(0x7f4d5c0b1940 msgr2=0x7f4d5c156d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 shutdown_connections 2024-03-06T07:28:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.017+0000 7f4d62400640 1 -- 172.21.15.86:0/287853945 wait complete. 2024-03-06T07:28:43.039 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:28:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.252+0000 7f2d49000640 1 Processor -- start 2024-03-06T07:28:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.252+0000 7f2d49000640 1 -- start start 2024-03-06T07:28:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.253+0000 7f2d49000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d4415fc10 0x7f2d44070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.253+0000 7f2d49000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f2d440b3f70 0x7f2d440ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.253+0000 7f2d49000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d440b46c0 con 0x7f2d4415fff0 2024-03-06T07:28:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.254+0000 7f2d49000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d440b49a0 con 0x7f2d440b3f70 2024-03-06T07:28:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.254+0000 7f2d49000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d44072c90 con 0x7f2d4415fc10 2024-03-06T07:28:43.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.254+0000 7f2d41e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d4415fc10 0x7f2d44070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.254+0000 7f2d3be00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f2d4415fff0 0x7f2d44160450 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:47110/0 (socket says 172.21.15.86:47110) 2024-03-06T07:28:43.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.254+0000 7f2d3be00640 1 -- 172.21.15.86:0/1569504848 learned_addr learned my addr 172.21.15.86:0/1569504848 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:43.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.254+0000 7f2d42800640 1 --2- 172.21.15.86:0/1569504848 >> v2:172.21.15.194:3300/0 conn(0x7f2d440b3f70 0x7f2d440ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.255+0000 7f2d41e00640 1 -- 172.21.15.86:0/1569504848 >> v2:172.21.15.194:3300/0 conn(0x7f2d440b3f70 msgr2=0x7f2d440ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:43.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.255+0000 7f2d41e00640 1 --2- 172.21.15.86:0/1569504848 >> v2:172.21.15.194:3300/0 conn(0x7f2d440b3f70 0x7f2d440ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.255+0000 7f2d41e00640 1 -- 172.21.15.86:0/1569504848 >> v1:172.21.15.194:6789/0 conn(0x7f2d4415fff0 legacy=0x7f2d44160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.255+0000 7f2d41400640 1 -- 172.21.15.86:0/1569504848 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2754597888 0 0) 0x7f2d440b46c0 con 0x7f2d4415fff0 2024-03-06T07:28:43.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.255+0000 7f2d41e00640 1 -- 172.21.15.86:0/1569504848 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d4409dd80 con 0x7f2d4415fc10 2024-03-06T07:28:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.256+0000 7f2d41e00640 1 --2- 172.21.15.86:0/1569504848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d4415fc10 0x7f2d44070ff0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f2d340097a0 tx=0x7f2d3402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fa4dd02e2c09cbd4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.257+0000 7f2d41400640 1 -- 172.21.15.86:0/1569504848 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d34035020 con 0x7f2d4415fc10 2024-03-06T07:28:43.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.257+0000 7f2d41400640 1 -- 172.21.15.86:0/1569504848 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d34031830 con 0x7f2d4415fc10 2024-03-06T07:28:43.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.257+0000 7f2d41400640 1 -- 172.21.15.86:0/1569504848 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d34031c00 con 0x7f2d4415fc10 2024-03-06T07:28:43.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 -- 172.21.15.86:0/1569504848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d4415fc10 msgr2=0x7f2d44070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 --2- 172.21.15.86:0/1569504848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d4415fc10 0x7f2d44070ff0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f2d340097a0 tx=0x7f2d3402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 -- 172.21.15.86:0/1569504848 shutdown_connections 2024-03-06T07:28:43.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 --2- 172.21.15.86:0/1569504848 >> v2:172.21.15.194:3300/0 conn(0x7f2d440b3f70 0x7f2d440ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 --2- 172.21.15.86:0/1569504848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d4415fc10 0x7f2d44070ff0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 -- 172.21.15.86:0/1569504848 >> 172.21.15.86:0/1569504848 conn(0x7f2d440b1940 msgr2=0x7f2d440b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:43.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 -- 172.21.15.86:0/1569504848 shutdown_connections 2024-03-06T07:28:43.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.258+0000 7f2d49000640 1 -- 172.21.15.86:0/1569504848 wait complete. 2024-03-06T07:28:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.259+0000 7f2d49000640 1 Processor -- start 2024-03-06T07:28:43.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.260+0000 7f2d49000640 1 -- start start 2024-03-06T07:28:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.260+0000 7f2d49000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d440b3f70 0x7f2d44150850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.260+0000 7f2d49000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d4415fc10 0x7f2d4409c450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.260+0000 7f2d49000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d4415fff0 0x7f2d441511e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.260+0000 7f2d49000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d4415c460 con 0x7f2d4415fc10 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.260+0000 7f2d49000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d4415c2e0 con 0x7f2d4415fff0 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.260+0000 7f2d49000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d440ce9e0 con 0x7f2d440b3f70 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d4415fc10 0x7f2d4409c450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d4415fc10 0x7f2d4409c450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36376/0 (socket says 172.21.15.86:36376) 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 -- 172.21.15.86:0/1773981956 learned_addr learned my addr 172.21.15.86:0/1773981956 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d41e00640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d440b3f70 0x7f2d44150850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d42800640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d4415fff0 0x7f2d441511e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 -- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d440b3f70 msgr2=0x7f2d44150850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d440b3f70 0x7f2d44150850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 -- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d4415fff0 msgr2=0x7f2d441511e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d4415fff0 0x7f2d441511e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 -- 172.21.15.86:0/1773981956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d4415e310 con 0x7f2d4415fc10 2024-03-06T07:28:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.261+0000 7f2d3be00640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d4415fc10 0x7f2d4409c450 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f2d2c00c430 tx=0x7f2d2c00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:43.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.262+0000 7f2d3a000640 1 -- 172.21.15.86:0/1773981956 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d2c0074e0 con 0x7f2d4415fc10 2024-03-06T07:28:43.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.262+0000 7f2d3a000640 1 -- 172.21.15.86:0/1773981956 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d2c0047e0 con 0x7f2d4415fc10 2024-03-06T07:28:43.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.262+0000 7f2d3a000640 1 -- 172.21.15.86:0/1773981956 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d2c004bd0 con 0x7f2d4415fc10 2024-03-06T07:28:43.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.262+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d4415e5f0 con 0x7f2d4415fc10 2024-03-06T07:28:43.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.262+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d4409c990 con 0x7f2d4415fc10 2024-03-06T07:28:43.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.263+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d4404f330 con 0x7f2d4415fc10 2024-03-06T07:28:43.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.264+0000 7f2d3a000640 1 -- 172.21.15.86:0/1773981956 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d2c011070 con 0x7f2d4415fc10 2024-03-06T07:28:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.265+0000 7f2d3a000640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d2006b4d0 0x7f2d2006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.265+0000 7f2d3a000640 1 -- 172.21.15.86:0/1773981956 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2d2c089ed0 con 0x7f2d4415fc10 2024-03-06T07:28:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.265+0000 7f2d41e00640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d2006b4d0 0x7f2d2006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.266+0000 7f2d41e00640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d2006b4d0 0x7f2d2006d990 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f2d34036000 tx=0x7f2d34008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:43.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.270+0000 7f2d3a000640 1 -- 172.21.15.86:0/1773981956 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d2c00e280 con 0x7f2d4415fc10 2024-03-06T07:28:43.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.467+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f2d4415e7c0 con 0x7f2d4415fc10 2024-03-06T07:28:43.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.468+0000 7f2d3a000640 1 -- 172.21.15.86:0/1773981956 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v35) v1 ==== 100+0+11 (secure 0 0 0) 0x7f2d2c058d30 con 0x7f2d4415fc10 2024-03-06T07:28:43.471 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:28:43.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d2006b4d0 msgr2=0x7f2d2006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d2006b4d0 0x7f2d2006d990 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f2d34036000 tx=0x7f2d34008030 comp rx=0 tx=0).stop 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d4415fc10 msgr2=0x7f2d4409c450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d4415fc10 0x7f2d4409c450 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f2d2c00c430 tx=0x7f2d2c00c8f0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 shutdown_connections 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d2006b4d0 0x7f2d2006d990 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d4415fff0 0x7f2d441511e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d4415fc10 0x7f2d4409c450 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 --2- 172.21.15.86:0/1773981956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d440b3f70 0x7f2d44150850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 >> 172.21.15.86:0/1773981956 conn(0x7f2d440b1940 msgr2=0x7f2d440cc690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 shutdown_connections 2024-03-06T07:28:43.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.471+0000 7f2d49000640 1 -- 172.21.15.86:0/1773981956 wait complete. 2024-03-06T07:28:43.492 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:43.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.698+0000 7f58b1200640 1 Processor -- start 2024-03-06T07:28:43.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.699+0000 7f58b1200640 1 -- start start 2024-03-06T07:28:43.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.700+0000 7f58b1200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f58ac144c00 0x7f58ac06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.700+0000 7f58b1200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f58ac15fe60 0x7f58ac0ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.700+0000 7f58b1200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f58ac06ee70 con 0x7f58ac15f9d0 2024-03-06T07:28:43.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.700+0000 7f58b1200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f58ac06cef0 con 0x7f58ac15fe60 2024-03-06T07:28:43.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.700+0000 7f58b1200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f58ac06d1d0 con 0x7f58ac144c00 2024-03-06T07:28:43.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58a9400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f58ac15f9d0 0x7f58ac0ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38662/0 (socket says 172.21.15.86:38662) 2024-03-06T07:28:43.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58aa800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f58ac15fe60 0x7f58ac0ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58a9400640 1 -- 172.21.15.86:0/4034505192 learned_addr learned my addr 172.21.15.86:0/4034505192 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:43.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58a9e00640 1 --2- 172.21.15.86:0/4034505192 >> v2:172.21.15.194:3300/0 conn(0x7f58ac144c00 0x7f58ac06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58aa800640 1 -- 172.21.15.86:0/4034505192 >> v2:172.21.15.194:3300/0 conn(0x7f58ac144c00 msgr2=0x7f58ac06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:43.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58aa800640 1 --2- 172.21.15.86:0/4034505192 >> v2:172.21.15.194:3300/0 conn(0x7f58ac144c00 0x7f58ac06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58aa800640 1 -- 172.21.15.86:0/4034505192 >> v1:172.21.15.86:6789/0 conn(0x7f58ac15f9d0 legacy=0x7f58ac0ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58aa800640 1 -- 172.21.15.86:0/4034505192 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58ac06d4b0 con 0x7f58ac15fe60 2024-03-06T07:28:43.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.701+0000 7f58a8a00640 1 -- 172.21.15.86:0/4034505192 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4240832642 0 0) 0x7f5898001110 con 0x7f58ac15f9d0 2024-03-06T07:28:43.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.702+0000 7f58aa800640 1 --2- 172.21.15.86:0/4034505192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15fe60 0x7f58ac0ce7b0 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f58ac0b25f0 tx=0x7f58a402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da9dbfa5eaaec8b8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:43.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58a8a00640 1 -- 172.21.15.86:0/4034505192 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f58a4035020 con 0x7f58ac15fe60 2024-03-06T07:28:43.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58a8a00640 1 -- 172.21.15.86:0/4034505192 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f58a4031830 con 0x7f58ac15fe60 2024-03-06T07:28:43.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58a8a00640 1 -- 172.21.15.86:0/4034505192 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f58a4031c00 con 0x7f58ac15fe60 2024-03-06T07:28:43.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 -- 172.21.15.86:0/4034505192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15fe60 msgr2=0x7f58ac0ce7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 --2- 172.21.15.86:0/4034505192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15fe60 0x7f58ac0ce7b0 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f58ac0b25f0 tx=0x7f58a402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 -- 172.21.15.86:0/4034505192 shutdown_connections 2024-03-06T07:28:43.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 --2- 172.21.15.86:0/4034505192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15fe60 0x7f58ac0ce7b0 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 --2- 172.21.15.86:0/4034505192 >> v2:172.21.15.194:3300/0 conn(0x7f58ac144c00 0x7f58ac06e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 -- 172.21.15.86:0/4034505192 >> 172.21.15.86:0/4034505192 conn(0x7f58ac0b1730 msgr2=0x7f58ac0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:43.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 -- 172.21.15.86:0/4034505192 shutdown_connections 2024-03-06T07:28:43.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.703+0000 7f58b1200640 1 -- 172.21.15.86:0/4034505192 wait complete. 2024-03-06T07:28:43.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.705+0000 7f58b1200640 1 Processor -- start 2024-03-06T07:28:43.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.705+0000 7f58b1200640 1 -- start start 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58b1200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58ac144c00 0x7f58ac15e540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58b1200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15f9d0 0x7f58ac15c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58b1200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58ac15fe60 0x7f58ac13e170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58b1200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f58ac160730 con 0x7f58ac15f9d0 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58b1200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f58ac160270 con 0x7f58ac15fe60 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58b1200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f58ac06cb50 con 0x7f58ac144c00 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58a9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15f9d0 0x7f58ac15c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58a9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15f9d0 0x7f58ac15c340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36394/0 (socket says 172.21.15.86:36394) 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.706+0000 7f58a9400640 1 -- 172.21.15.86:0/353507082 learned_addr learned my addr 172.21.15.86:0/353507082 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:43.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58aa800640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58ac15fe60 0x7f58ac13e170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a9e00640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58ac144c00 0x7f58ac15e540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a9400640 1 -- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58ac144c00 msgr2=0x7f58ac15e540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a9400640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58ac144c00 0x7f58ac15e540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a9400640 1 -- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58ac15fe60 msgr2=0x7f58ac13e170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a9400640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58ac15fe60 0x7f58ac13e170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a9400640 1 -- 172.21.15.86:0/353507082 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58ac15cc20 con 0x7f58ac15f9d0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a9400640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15f9d0 0x7f58ac15c340 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f589800c7b0 tx=0x7f589800cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a2000640 1 -- 172.21.15.86:0/353507082 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5898018070 con 0x7f58ac15f9d0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a2000640 1 -- 172.21.15.86:0/353507082 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f58980072d0 con 0x7f58ac15f9d0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58a2000640 1 -- 172.21.15.86:0/353507082 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f58980047e0 con 0x7f58ac15f9d0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f58ac15cf00 con 0x7f58ac15f9d0 2024-03-06T07:28:43.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.707+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f58ac15d4c0 con 0x7f58ac15f9d0 2024-03-06T07:28:43.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.709+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f58ac04f330 con 0x7f58ac15f9d0 2024-03-06T07:28:43.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.710+0000 7f58a2000640 1 -- 172.21.15.86:0/353507082 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f589801b070 con 0x7f58ac15f9d0 2024-03-06T07:28:43.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.710+0000 7f58a2000640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f588c06b4d0 0x7f588c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:43.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.711+0000 7f58a2000640 1 -- 172.21.15.86:0/353507082 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f58980894c0 con 0x7f58ac15f9d0 2024-03-06T07:28:43.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.711+0000 7f58a9e00640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f588c06b4d0 0x7f588c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:43.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.711+0000 7f58a9e00640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f588c06b4d0 0x7f588c06d990 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f589c009e90 tx=0x7f589c00b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:43.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.715+0000 7f58a2000640 1 -- 172.21.15.86:0/353507082 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f58ac06ee70 con 0x7f58ac15f9d0 2024-03-06T07:28:43.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.944+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f58ac072890 con 0x7f58ac15f9d0 2024-03-06T07:28:43.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.945+0000 7f58a2000640 1 -- 172.21.15.86:0/353507082 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f58980583d0 con 0x7f58ac15f9d0 2024-03-06T07:28:43.948 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:43.948 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:43.948 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:43.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f588c06b4d0 msgr2=0x7f588c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f588c06b4d0 0x7f588c06d990 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f589c009e90 tx=0x7f589c00b040 comp rx=0 tx=0).stop 2024-03-06T07:28:43.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15f9d0 msgr2=0x7f58ac15c340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:43.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15f9d0 0x7f58ac15c340 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f589800c7b0 tx=0x7f589800cc70 comp rx=0 tx=0).stop 2024-03-06T07:28:43.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 shutdown_connections 2024-03-06T07:28:43.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f588c06b4d0 0x7f588c06d990 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58ac15fe60 0x7f58ac13e170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58ac15f9d0 0x7f58ac15c340 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 --2- 172.21.15.86:0/353507082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58ac144c00 0x7f58ac15e540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:43.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 >> 172.21.15.86:0/353507082 conn(0x7f58ac0b1730 msgr2=0x7f58ac0719f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:43.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 shutdown_connections 2024-03-06T07:28:43.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:43.948+0000 7f58b1200640 1 -- 172.21.15.86:0/353507082 wait complete. 2024-03-06T07:28:43.967 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:44.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.170+0000 7f13e3800640 1 Processor -- start 2024-03-06T07:28:44.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.170+0000 7f13e3800640 1 -- start start 2024-03-06T07:28:44.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e3800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 0x7f13dc15fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e3800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f13dc070b10 0x7f13dc070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e3800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f13dc06cef0 con 0x7f13dc144c00 2024-03-06T07:28:44.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e3800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f13dc160370 con 0x7f13dc070b10 2024-03-06T07:28:44.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e3800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f13dc06d1d0 con 0x7f13dc15f9d0 2024-03-06T07:28:44.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e0e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f13dc144c00 0x7f13dc06e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38676/0 (socket says 172.21.15.86:38676) 2024-03-06T07:28:44.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e0e00640 1 -- 172.21.15.86:0/1575922397 learned_addr learned my addr 172.21.15.86:0/1575922397 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.172+0000 7f13e1800640 1 --2- 172.21.15.86:0/1575922397 >> v2:172.21.15.194:3300/0 conn(0x7f13dc070b10 0x7f13dc070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13dbe00640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 0x7f13dc15fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1906930206 0 0) 0x7f13dc06cef0 con 0x7f13dc144c00 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f13c0003520 con 0x7f13dc144c00 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1952605914 0 0) 0x7f13c0003520 con 0x7f13dc144c00 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 msgr2=0x7f13dc15fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 0x7f13dc15fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 >> v2:172.21.15.194:3300/0 conn(0x7f13dc070b10 msgr2=0x7f13dc070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 --2- 172.21.15.86:0/1575922397 >> v2:172.21.15.194:3300/0 conn(0x7f13dc070b10 0x7f13dc070f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13dc06d4b0 con 0x7f13dc144c00 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f13cc001670 con 0x7f13dc144c00 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 >> v1:172.21.15.86:6789/0 conn(0x7f13dc144c00 legacy=0x7f13dc06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13c00039b0 0x7f13c0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 0x7f13c0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13c0004d40 0x7f13c00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13dc06d1d0 con 0x7f13c0004d40 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13c000b9c0 con 0x7f13c00039b0 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.173+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f13c000bb40 con 0x7f13dc15f9d0 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e1800640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13c0004d40 0x7f13c00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e0e00640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13c00039b0 0x7f13c0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13dbe00640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 0x7f13c0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e1800640 1 -- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 msgr2=0x7f13c0004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e1800640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc15f9d0 0x7f13c0004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e1800640 1 -- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13c00039b0 msgr2=0x7f13c0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e1800640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13c00039b0 0x7f13c0003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e1800640 1 -- 172.21.15.86:0/1575922397 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13c0003520 con 0x7f13c0004d40 2024-03-06T07:28:44.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.174+0000 7f13e0e00640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13c00039b0 0x7f13c0003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.175+0000 7f13e1800640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13c0004d40 0x7f13c00051c0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f13d400e170 tx=0x7f13c0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f811a2743b3e63c5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.175+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13d4039070 con 0x7f13c0004d40 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.175+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f13d402cc20 con 0x7f13c0004d40 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.175+0000 7f13db400640 1 -- 172.21.15.86:0/1575922397 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13d4009c10 con 0x7f13c0004d40 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.176+0000 7f13e3800640 1 -- 172.21.15.86:0/1575922397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13c0004d40 msgr2=0x7f13c00051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.176+0000 7f13e3800640 1 --2- 172.21.15.86:0/1575922397 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13c0004d40 0x7f13c00051c0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f13d400e170 tx=0x7f13c0005830 comp rx=0 tx=0).stop 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.176+0000 7f13e1800640 1 -- 172.21.15.86:0/1575922397 reap_dead start 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.176+0000 7f13e3800640 1 -- 172.21.15.86:0/1575922397 shutdown_connections 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.176+0000 7f13e3800640 1 -- 172.21.15.86:0/1575922397 >> 172.21.15.86:0/1575922397 conn(0x7f13dc0b1730 msgr2=0x7f13dc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.176+0000 7f13e3800640 1 -- 172.21.15.86:0/1575922397 shutdown_connections 2024-03-06T07:28:44.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.176+0000 7f13e3800640 1 -- 172.21.15.86:0/1575922397 wait complete. 2024-03-06T07:28:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.178+0000 7f13e3800640 1 Processor -- start 2024-03-06T07:28:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.178+0000 7f13e3800640 1 -- start start 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e3800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13dc1426e0 0x7f13dc15dda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e3800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc1e81e0 0x7f13dc1e8640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e3800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13dc1e8b80 0x7f13dc1e6150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e3800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13dc06cb50 con 0x7f13dc1e8b80 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e3800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13dc160730 con 0x7f13dc1426e0 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e3800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f13dc06ccd0 con 0x7f13dc1e81e0 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e1800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13dc1e8b80 0x7f13dc1e6150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e1800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13dc1e8b80 0x7f13dc1e6150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36406/0 (socket says 172.21.15.86:36406) 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.179+0000 7f13e1800640 1 -- 172.21.15.86:0/2208350686 learned_addr learned my addr 172.21.15.86:0/2208350686 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e0e00640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13dc1426e0 0x7f13dc15dda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e1800640 1 -- 172.21.15.86:0/2208350686 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc1e81e0 msgr2=0x7f13dc1e8640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e1800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc1e81e0 0x7f13dc1e8640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e1800640 1 -- 172.21.15.86:0/2208350686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13dc1426e0 msgr2=0x7f13dc15dda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e1800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13dc1426e0 0x7f13dc15dda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e1800640 1 -- 172.21.15.86:0/2208350686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13dc1436c0 con 0x7f13dc1e8b80 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e1800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13dc1e8b80 0x7f13dc1e6150 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f13d4037040 tx=0x7f13d4009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13d8c00640 1 -- 172.21.15.86:0/2208350686 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13d4039070 con 0x7f13dc1e8b80 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13d8c00640 1 -- 172.21.15.86:0/2208350686 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f13d403a040 con 0x7f13dc1e8b80 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13d8c00640 1 -- 172.21.15.86:0/2208350686 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13d4009c10 con 0x7f13dc1e8b80 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f13dc1e5c50 con 0x7f13dc1e8b80 2024-03-06T07:28:44.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.180+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13dc15e660 con 0x7f13dc1e8b80 2024-03-06T07:28:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.181+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f13dc1e5e20 con 0x7f13dc1e8b80 2024-03-06T07:28:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.182+0000 7f13d8c00640 1 -- 172.21.15.86:0/2208350686 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f13d4046030 con 0x7f13dc1e8b80 2024-03-06T07:28:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.183+0000 7f13d8c00640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f13bc06b4d0 0x7f13bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.183+0000 7f13d8c00640 1 -- 172.21.15.86:0/2208350686 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f13d40abc40 con 0x7f13dc1e8b80 2024-03-06T07:28:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.183+0000 7f13e0e00640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f13bc06b4d0 0x7f13bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.184+0000 7f13e0e00640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f13bc06b4d0 0x7f13bc06d990 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f13cc0017e0 tx=0x7f13cc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:44.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.188+0000 7f13d8c00640 1 -- 172.21.15.86:0/2208350686 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f13d404d060 con 0x7f13dc1e8b80 2024-03-06T07:28:44.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.382+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f13dc072890 con 0x7f13dc1e8b80 2024-03-06T07:28:44.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.383+0000 7f13d8c00640 1 -- 172.21.15.86:0/2208350686 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v35) v1 ==== 74+0+16681 (secure 0 0 0) 0x7f13dc1436c0 con 0x7f13dc1e8b80 2024-03-06T07:28:44.386 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:44.386 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":35,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:15.157270+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:11.444335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:12.623953+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"35","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f13bc06b4d0 msgr2=0x7f13bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f13bc06b4d0 0x7f13bc06d990 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f13cc0017e0 tx=0x7f13cc009040 comp rx=0 tx=0).stop 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13dc1e8b80 msgr2=0x7f13dc1e6150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13dc1e8b80 0x7f13dc1e6150 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f13d4037040 tx=0x7f13d4009750 comp rx=0 tx=0).stop 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 shutdown_connections 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f13bc06b4d0 0x7f13bc06d990 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13dc1e8b80 0x7f13dc1e6150 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13dc1e81e0 0x7f13dc1e8640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 --2- 172.21.15.86:0/2208350686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f13dc1426e0 0x7f13dc15dda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 >> 172.21.15.86:0/2208350686 conn(0x7f13dc0b1730 msgr2=0x7f13dc0ae150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 shutdown_connections 2024-03-06T07:28:44.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.386+0000 7f13e3800640 1 -- 172.21.15.86:0/2208350686 wait complete. 2024-03-06T07:28:44.408 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:28:44.408 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:44.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.610+0000 7f8083000640 1 Processor -- start 2024-03-06T07:28:44.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.610+0000 7f8083000640 1 -- start start 2024-03-06T07:28:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.611+0000 7f8083000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f807c15acb0 0x7f807c0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.611+0000 7f8083000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c0b4680 0x7f807c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.611+0000 7f8083000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f807c15b090 con 0x7f807c072340 2024-03-06T07:28:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.611+0000 7f8083000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f807c0b2a30 con 0x7f807c15acb0 2024-03-06T07:28:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.611+0000 7f8083000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f807c0b4360 con 0x7f807c0b4680 2024-03-06T07:28:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f8081000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f807c072340 0x7f807c0ccfb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38682/0 (socket says 172.21.15.86:38682) 2024-03-06T07:28:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f8081000640 1 -- 172.21.15.86:0/2076781463 learned_addr learned my addr 172.21.15.86:0/2076781463 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807be00640 1 --2- 172.21.15.86:0/2076781463 >> v2:172.21.15.194:3300/0 conn(0x7f807c15acb0 0x7f807c0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f8073400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c0b4680 0x7f807c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2609404863 0 0) 0x7f807c15b090 con 0x7f807c072340 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8060003520 con 0x7f807c072340 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3215423134 0 0) 0x7f8060003520 con 0x7f807c072340 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c0b4680 msgr2=0x7f807c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c0b4680 0x7f807c06e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 >> v2:172.21.15.194:3300/0 conn(0x7f807c15acb0 msgr2=0x7f807c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 --2- 172.21.15.86:0/2076781463 >> v2:172.21.15.194:3300/0 conn(0x7f807c15acb0 0x7f807c0b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f807c0b2cf0 con 0x7f807c072340 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f80740019f0 con 0x7f807c072340 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 >> v1:172.21.15.86:6789/0 conn(0x7f807c072340 legacy=0x7f807c0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600039b0 0x7f8060003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.612+0000 7f807b400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80600043a0 0x7f8060004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f807b400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c0b4680 0x7f8060004f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807c0b2a30 con 0x7f80600043a0 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807c0b4360 con 0x7f80600039b0 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f806000dbd0 con 0x7f807c0b4680 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f8073400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80600043a0 0x7f8060004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f8073400640 1 -- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c0b4680 msgr2=0x7f8060004f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f8073400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c0b4680 0x7f8060004f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f8073400640 1 -- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600039b0 msgr2=0x7f8060003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f8073400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600039b0 0x7f8060003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.613+0000 7f8073400640 1 -- 172.21.15.86:0/2076781463 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f807c0b2cf0 con 0x7f80600043a0 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.614+0000 7f8073400640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80600043a0 0x7f8060004880 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f8064009ac0 tx=0x7f8060005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c3dee6b62e812c03 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.614+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8064039070 con 0x7f80600043a0 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.614+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f806402c710 con 0x7f80600043a0 2024-03-06T07:28:44.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.614+0000 7f807b400640 1 -- 172.21.15.86:0/2076781463 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8064009690 con 0x7f80600043a0 2024-03-06T07:28:44.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.615+0000 7f8083000640 1 -- 172.21.15.86:0/2076781463 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80600043a0 msgr2=0x7f8060004880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.615+0000 7f8083000640 1 --2- 172.21.15.86:0/2076781463 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80600043a0 0x7f8060004880 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f8064009ac0 tx=0x7f8060005830 comp rx=0 tx=0).stop 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.615+0000 7f8081000640 1 -- 172.21.15.86:0/2076781463 reap_dead start 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.615+0000 7f8083000640 1 -- 172.21.15.86:0/2076781463 shutdown_connections 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.615+0000 7f8083000640 1 -- 172.21.15.86:0/2076781463 >> 172.21.15.86:0/2076781463 conn(0x7f807c0b1730 msgr2=0x7f807c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.615+0000 7f8083000640 1 -- 172.21.15.86:0/2076781463 shutdown_connections 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.615+0000 7f8083000640 1 -- 172.21.15.86:0/2076781463 wait complete. 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.617+0000 7f8083000640 1 Processor -- start 2024-03-06T07:28:44.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.617+0000 7f8083000640 1 -- start start 2024-03-06T07:28:44.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8083000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600043a0 0x7f807c0cd500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8083000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c153ab0 0x7f807c0ceec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8083000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f807c151780 0x7f807c15ba10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8083000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807c0b3d10 con 0x7f807c151780 2024-03-06T07:28:44.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8083000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807c0b32b0 con 0x7f80600043a0 2024-03-06T07:28:44.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8083000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f807c0b4010 con 0x7f807c153ab0 2024-03-06T07:28:44.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f807c151780 0x7f807c15ba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f807c151780 0x7f807c15ba10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36430/0 (socket says 172.21.15.86:36430) 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 -- 172.21.15.86:0/4080891035 learned_addr learned my addr 172.21.15.86:0/4080891035 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8073400640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c153ab0 0x7f807c0ceec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f807be00640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600043a0 0x7f807c0cd500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 -- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c153ab0 msgr2=0x7f807c0ceec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c153ab0 0x7f807c0ceec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 -- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600043a0 msgr2=0x7f807c0cd500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600043a0 0x7f807c0cd500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8081000640 1 -- 172.21.15.86:0/4080891035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f807c152cf0 con 0x7f807c151780 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.618+0000 7f8073400640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c153ab0 0x7f807c0ceec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.619+0000 7f8081000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f807c151780 0x7f807c15ba10 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f807c073730 tx=0x7f807400cf00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.619+0000 7f8078c00640 1 -- 172.21.15.86:0/4080891035 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f807400a5b0 con 0x7f807c151780 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.619+0000 7f8078c00640 1 -- 172.21.15.86:0/4080891035 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f807400a750 con 0x7f807c151780 2024-03-06T07:28:44.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.619+0000 7f8078c00640 1 -- 172.21.15.86:0/4080891035 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f807400ab40 con 0x7f807c151780 2024-03-06T07:28:44.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.619+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f807c152fd0 con 0x7f807c151780 2024-03-06T07:28:44.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.619+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f807c151c30 con 0x7f807c151780 2024-03-06T07:28:44.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.620+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f807c151ec0 con 0x7f807c151780 2024-03-06T07:28:44.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.621+0000 7f8078c00640 1 -- 172.21.15.86:0/4080891035 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f807400ace0 con 0x7f807c151780 2024-03-06T07:28:44.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.622+0000 7f8078c00640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805806b500 0x7f805806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:44.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.622+0000 7f8078c00640 1 -- 172.21.15.86:0/4080891035 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8074089c90 con 0x7f807c151780 2024-03-06T07:28:44.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.622+0000 7f807be00640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805806b500 0x7f805806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:44.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.623+0000 7f807be00640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805806b500 0x7f805806d9c0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f806c004770 tx=0x7f806c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:44.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.627+0000 7f8078c00640 1 -- 172.21.15.86:0/4080891035 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8074058a70 con 0x7f807c151780 2024-03-06T07:28:44.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.859+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f807c04f330 con 0x7f807c151780 2024-03-06T07:28:44.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.860+0000 7f8078c00640 1 -- 172.21.15.86:0/4080891035 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f8074058370 con 0x7f807c151780 2024-03-06T07:28:44.863 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:44.863 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:44.864 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:44.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.863+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805806b500 msgr2=0x7f805806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.863+0000 7f8083000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805806b500 0x7f805806d9c0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f806c004770 tx=0x7f806c008040 comp rx=0 tx=0).stop 2024-03-06T07:28:44.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.863+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f807c151780 msgr2=0x7f807c15ba10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.863+0000 7f8083000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f807c151780 0x7f807c15ba10 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f807c073730 tx=0x7f807400cf00 comp rx=0 tx=0).stop 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 shutdown_connections 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805806b500 0x7f805806d9c0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f807c151780 0x7f807c15ba10 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f807c153ab0 0x7f807c0ceec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 --2- 172.21.15.86:0/4080891035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f80600043a0 0x7f807c0cd500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 >> 172.21.15.86:0/4080891035 conn(0x7f807c0b1730 msgr2=0x7f807c0af830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 shutdown_connections 2024-03-06T07:28:44.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:44.864+0000 7f8083000640 1 -- 172.21.15.86:0/4080891035 wait complete. 2024-03-06T07:28:44.884 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:45.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.075+0000 7f0e47e00640 1 Processor -- start 2024-03-06T07:28:45.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.075+0000 7f0e47e00640 1 -- start start 2024-03-06T07:28:45.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.076+0000 7f0e47e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 0x7f0e4006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.076+0000 7f0e47e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0e4015f9d0 0x7f0e4015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.076+0000 7f0e47e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0e400ce0a0 0x7f0e400ce4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.076+0000 7f0e47e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0e40160370 con 0x7f0e400ce0a0 2024-03-06T07:28:45.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.076+0000 7f0e47e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0e4006ee70 con 0x7f0e4015f9d0 2024-03-06T07:28:45.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.076+0000 7f0e47e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0e4006cef0 con 0x7f0e40144c00 2024-03-06T07:28:45.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0e400ce0a0 0x7f0e400ce4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0e400ce0a0 0x7f0e400ce4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36434/0 (socket says 172.21.15.86:36434) 2024-03-06T07:28:45.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 -- 172.21.15.86:0/1629285365 learned_addr learned my addr 172.21.15.86:0/1629285365 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 -- 172.21.15.86:0/1629285365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 msgr2=0x7f0e4006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e44a00640 1 --2- 172.21.15.86:0/1629285365 >> v2:172.21.15.194:3300/0 conn(0x7f0e4015f9d0 0x7f0e4015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 --2- 172.21.15.86:0/1629285365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 0x7f0e4006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 -- 172.21.15.86:0/1629285365 >> v2:172.21.15.194:3300/0 conn(0x7f0e4015f9d0 msgr2=0x7f0e4015fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 --2- 172.21.15.86:0/1629285365 >> v2:172.21.15.194:3300/0 conn(0x7f0e4015f9d0 0x7f0e4015fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.077+0000 7f0e45e00640 1 -- 172.21.15.86:0/1629285365 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0e4006d1d0 con 0x7f0e400ce0a0 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.078+0000 7f0e45e00640 1 --2- 172.21.15.86:0/1629285365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e400ce0a0 0x7f0e400ce4f0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f0e400b27f0 tx=0x7f0e3802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bf495ce055be8bcc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.078+0000 7f0e3fe00640 1 -- 172.21.15.86:0/1629285365 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0e3802ca70 con 0x7f0e400ce0a0 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.078+0000 7f0e3fe00640 1 -- 172.21.15.86:0/1629285365 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0e3802cc10 con 0x7f0e400ce0a0 2024-03-06T07:28:45.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.078+0000 7f0e3fe00640 1 -- 172.21.15.86:0/1629285365 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0e380343f0 con 0x7f0e400ce0a0 2024-03-06T07:28:45.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 -- 172.21.15.86:0/1629285365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e400ce0a0 msgr2=0x7f0e400ce4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 --2- 172.21.15.86:0/1629285365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e400ce0a0 0x7f0e400ce4f0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f0e400b27f0 tx=0x7f0e3802bed0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 -- 172.21.15.86:0/1629285365 shutdown_connections 2024-03-06T07:28:45.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 --2- 172.21.15.86:0/1629285365 >> v2:172.21.15.194:3300/0 conn(0x7f0e4015f9d0 0x7f0e4015fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 --2- 172.21.15.86:0/1629285365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e400ce0a0 0x7f0e400ce4f0 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 --2- 172.21.15.86:0/1629285365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 0x7f0e4006e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 -- 172.21.15.86:0/1629285365 >> 172.21.15.86:0/1629285365 conn(0x7f0e400b1730 msgr2=0x7f0e400b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:45.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 -- 172.21.15.86:0/1629285365 shutdown_connections 2024-03-06T07:28:45.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.079+0000 7f0e47e00640 1 -- 172.21.15.86:0/1629285365 wait complete. 2024-03-06T07:28:45.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.081+0000 7f0e47e00640 1 Processor -- start 2024-03-06T07:28:45.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.081+0000 7f0e47e00640 1 -- start start 2024-03-06T07:28:45.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e47e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0e400ce0a0 0x7f0e40142ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e47e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 0x7f0e401e5bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e47e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e4015f9d0 0x7f0e401e60f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e47e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0e40160730 con 0x7f0e4015f9d0 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e47e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0e4006d790 con 0x7f0e400ce0a0 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e47e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0e4006cb50 con 0x7f0e40144c00 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e4015f9d0 0x7f0e401e60f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e4015f9d0 0x7f0e401e60f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36438/0 (socket says 172.21.15.86:36438) 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 -- 172.21.15.86:0/3510803734 learned_addr learned my addr 172.21.15.86:0/3510803734 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45400640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0e400ce0a0 0x7f0e40142ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e44a00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 0x7f0e401e5bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 -- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 msgr2=0x7f0e401e5bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 0x7f0e401e5bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 -- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0e400ce0a0 msgr2=0x7f0e40142ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0e400ce0a0 0x7f0e40142ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45e00640 1 -- 172.21.15.86:0/3510803734 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0e401437c0 con 0x7f0e4015f9d0 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.082+0000 7f0e45400640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0e400ce0a0 0x7f0e40142ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:45.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.083+0000 7f0e45e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e4015f9d0 0x7f0e401e60f0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f0e38036040 tx=0x7f0e38004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.083+0000 7f0e3d600640 1 -- 172.21.15.86:0/3510803734 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0e38043020 con 0x7f0e4015f9d0 2024-03-06T07:28:45.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.083+0000 7f0e3d600640 1 -- 172.21.15.86:0/3510803734 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0e3803e450 con 0x7f0e4015f9d0 2024-03-06T07:28:45.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.083+0000 7f0e3d600640 1 -- 172.21.15.86:0/3510803734 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0e38009510 con 0x7f0e4015f9d0 2024-03-06T07:28:45.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.083+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0e40143220 con 0x7f0e4015f9d0 2024-03-06T07:28:45.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.083+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0e401e6a30 con 0x7f0e4015f9d0 2024-03-06T07:28:45.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.084+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0e4004f330 con 0x7f0e4015f9d0 2024-03-06T07:28:45.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.085+0000 7f0e3d600640 1 -- 172.21.15.86:0/3510803734 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0e38046020 con 0x7f0e4015f9d0 2024-03-06T07:28:45.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.086+0000 7f0e3d600640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0e2006b500 0x7f0e2006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.086+0000 7f0e3d600640 1 -- 172.21.15.86:0/3510803734 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0e380acbb0 con 0x7f0e4015f9d0 2024-03-06T07:28:45.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.086+0000 7f0e45400640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0e2006b500 0x7f0e2006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.087+0000 7f0e45400640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0e2006b500 0x7f0e2006d9c0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f0e34006fd0 tx=0x7f0e34008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.091+0000 7f0e3d600640 1 -- 172.21.15.86:0/3510803734 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0e3804d030 con 0x7f0e4015f9d0 2024-03-06T07:28:45.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.324+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0e400b1dd0 con 0x7f0e4015f9d0 2024-03-06T07:28:45.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.325+0000 7f0e3d600640 1 -- 172.21.15.86:0/3510803734 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f0e401437c0 con 0x7f0e4015f9d0 2024-03-06T07:28:45.327 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:45.328 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":17,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:13.618810+0000","modified":"2024-03-06T07:28:16.688890+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4995},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4995":{"gid":4995,"name":"d","rank":0,"incarnation":13,"state":"up:active","state_seq":4,"addr":"172.21.15.194:6835/902800806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":902800806},{"type":"v1","addr":"172.21.15.194:6835","nonce":902800806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4995,"qdb_cluster":[4995]},"id":2}]} 2024-03-06T07:28:45.328 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 17 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0e2006b500 msgr2=0x7f0e2006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0e2006b500 0x7f0e2006d9c0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f0e34006fd0 tx=0x7f0e34008040 comp rx=0 tx=0).stop 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e4015f9d0 msgr2=0x7f0e401e60f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e4015f9d0 0x7f0e401e60f0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f0e38036040 tx=0x7f0e38004a40 comp rx=0 tx=0).stop 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 shutdown_connections 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0e2006b500 0x7f0e2006d9c0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0e4015f9d0 0x7f0e401e60f0 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0e40144c00 0x7f0e401e5bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 --2- 172.21.15.86:0/3510803734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0e400ce0a0 0x7f0e40142ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 >> 172.21.15.86:0/3510803734 conn(0x7f0e400b1730 msgr2=0x7f0e400717c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:45.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 shutdown_connections 2024-03-06T07:28:45.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.327+0000 7f0e47e00640 1 -- 172.21.15.86:0/3510803734 wait complete. 2024-03-06T07:28:45.349 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:45.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.541+0000 7f3da1e00640 1 Processor -- start 2024-03-06T07:28:45.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.541+0000 7f3da1e00640 1 -- start start 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3da1e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d9c15acb0 0x7f3d9c0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3da1e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3d9c0b4680 0x7f3d9c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3da1e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c072340 0x7f3d9c06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3da1e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d9c15b090 con 0x7f3d9c15acb0 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3da1e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d9c0b2a30 con 0x7f3d9c0b4680 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3da1e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d9c0b4360 con 0x7f3d9c072340 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3d9aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d9c15acb0 0x7f3d9c0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3d9aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d9c15acb0 0x7f3d9c0b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36446/0 (socket says 172.21.15.86:36446) 2024-03-06T07:28:45.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.543+0000 7f3d9aa00640 1 -- 172.21.15.86:0/258463895 learned_addr learned my addr 172.21.15.86:0/258463895 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:45.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9a000640 1 --2- 172.21.15.86:0/258463895 >> v2:172.21.15.194:3300/0 conn(0x7f3d9c0b4680 0x7f3d9c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9b400640 1 --2- 172.21.15.86:0/258463895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c072340 0x7f3d9c06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9aa00640 1 -- 172.21.15.86:0/258463895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c072340 msgr2=0x7f3d9c06ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:45.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9aa00640 1 --2- 172.21.15.86:0/258463895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c072340 0x7f3d9c06ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9aa00640 1 -- 172.21.15.86:0/258463895 >> v2:172.21.15.194:3300/0 conn(0x7f3d9c0b4680 msgr2=0x7f3d9c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:45.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9aa00640 1 --2- 172.21.15.86:0/258463895 >> v2:172.21.15.194:3300/0 conn(0x7f3d9c0b4680 0x7f3d9c06e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9aa00640 1 -- 172.21.15.86:0/258463895 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d9c0b2cf0 con 0x7f3d9c15acb0 2024-03-06T07:28:45.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9b400640 1 --2- 172.21.15.86:0/258463895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c072340 0x7f3d9c06ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:45.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.544+0000 7f3d9a000640 1 --2- 172.21.15.86:0/258463895 >> v2:172.21.15.194:3300/0 conn(0x7f3d9c0b4680 0x7f3d9c06e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:45.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.545+0000 7f3d9aa00640 1 --2- 172.21.15.86:0/258463895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c15acb0 0x7f3d9c0b37d0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f3d9c068560 tx=0x7f3d8c02e0d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c1e66156f0743ae server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.545+0000 7f3d99600640 1 -- 172.21.15.86:0/258463895 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d8c03b070 con 0x7f3d9c15acb0 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.545+0000 7f3d99600640 1 -- 172.21.15.86:0/258463895 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d8c036440 con 0x7f3d9c15acb0 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.545+0000 7f3d99600640 1 -- 172.21.15.86:0/258463895 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d8c036810 con 0x7f3d9c15acb0 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 -- 172.21.15.86:0/258463895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c15acb0 msgr2=0x7f3d9c0b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 --2- 172.21.15.86:0/258463895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c15acb0 0x7f3d9c0b37d0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f3d9c068560 tx=0x7f3d8c02e0d0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 -- 172.21.15.86:0/258463895 shutdown_connections 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 --2- 172.21.15.86:0/258463895 >> v2:172.21.15.194:3300/0 conn(0x7f3d9c0b4680 0x7f3d9c06e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 --2- 172.21.15.86:0/258463895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c072340 0x7f3d9c06ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 --2- 172.21.15.86:0/258463895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c15acb0 0x7f3d9c0b37d0 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 -- 172.21.15.86:0/258463895 >> 172.21.15.86:0/258463895 conn(0x7f3d9c0b1730 msgr2=0x7f3d9c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 -- 172.21.15.86:0/258463895 shutdown_connections 2024-03-06T07:28:45.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.546+0000 7f3da1e00640 1 -- 172.21.15.86:0/258463895 wait complete. 2024-03-06T07:28:45.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.548+0000 7f3da1e00640 1 Processor -- start 2024-03-06T07:28:45.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.548+0000 7f3da1e00640 1 -- start start 2024-03-06T07:28:45.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3da1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c072340 0x7f3d9c14e750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3da1e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d9c0b4680 0x7f3d9c15b240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3da1e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c15acb0 0x7f3d9c15b780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3da1e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d9c0b3d10 con 0x7f3d9c072340 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3da1e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d9c0b32b0 con 0x7f3d9c0b4680 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3da1e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d9c0b4010 con 0x7f3d9c15acb0 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c072340 0x7f3d9c14e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c072340 0x7f3d9c14e750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36454/0 (socket says 172.21.15.86:36454) 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 -- 172.21.15.86:0/4035619195 learned_addr learned my addr 172.21.15.86:0/4035619195 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 -- 172.21.15.86:0/4035619195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c15acb0 msgr2=0x7f3d9c15b780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c15acb0 0x7f3d9c15b780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 -- 172.21.15.86:0/4035619195 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d9c0b4680 msgr2=0x7f3d9c15b240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d9c0b4680 0x7f3d9c15b240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.549+0000 7f3d9aa00640 1 -- 172.21.15.86:0/4035619195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d9c15bdb0 con 0x7f3d9c072340 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.550+0000 7f3d9aa00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c072340 0x7f3d9c14e750 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f3d8c000c00 tx=0x7f3d8c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.550+0000 7f3d93400640 1 -- 172.21.15.86:0/4035619195 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d8c03b030 con 0x7f3d9c072340 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.550+0000 7f3d93400640 1 -- 172.21.15.86:0/4035619195 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d8c03d070 con 0x7f3d9c072340 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.550+0000 7f3d93400640 1 -- 172.21.15.86:0/4035619195 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d8c034ba0 con 0x7f3d9c072340 2024-03-06T07:28:45.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.550+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d9c15c090 con 0x7f3d9c072340 2024-03-06T07:28:45.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.550+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d9c15c650 con 0x7f3d9c072340 2024-03-06T07:28:45.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.551+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d9c04f330 con 0x7f3d9c072340 2024-03-06T07:28:45.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.552+0000 7f3d93400640 1 -- 172.21.15.86:0/4035619195 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3d8c03f030 con 0x7f3d9c072340 2024-03-06T07:28:45.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.553+0000 7f3d93400640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d7c06b4d0 0x7f3d7c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.553+0000 7f3d93400640 1 -- 172.21.15.86:0/4035619195 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3d8c0ac080 con 0x7f3d9c072340 2024-03-06T07:28:45.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.553+0000 7f3d9a000640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d7c06b4d0 0x7f3d7c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.554+0000 7f3d9a000640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d7c06b4d0 0x7f3d7c06d990 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f3d88006fd0 tx=0x7f3d88008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.557+0000 7f3d93400640 1 -- 172.21.15.86:0/4035619195 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3d8c07ae60 con 0x7f3d9c072340 2024-03-06T07:28:45.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.741+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f3d9c0d07f0 con 0x7f3d9c072340 2024-03-06T07:28:45.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.744+0000 7f3d93400640 1 -- 172.21.15.86:0/4035619195 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v35) v1 ==== 74+0+16681 (secure 0 0 0) 0x7f3d8c07a760 con 0x7f3d9c072340 2024-03-06T07:28:45.747 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:45.747 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":35,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:15.157270+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:11.444335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"34","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:12.623953+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"35","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:45.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.746+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d7c06b4d0 msgr2=0x7f3d7c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.746+0000 7f3da1e00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d7c06b4d0 0x7f3d7c06d990 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f3d88006fd0 tx=0x7f3d88008040 comp rx=0 tx=0).stop 2024-03-06T07:28:45.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c072340 msgr2=0x7f3d9c14e750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c072340 0x7f3d9c14e750 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f3d8c000c00 tx=0x7f3d8c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 shutdown_connections 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d7c06b4d0 0x7f3d7c06d990 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d9c15acb0 0x7f3d9c15b780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d9c0b4680 0x7f3d9c15b240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 --2- 172.21.15.86:0/4035619195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d9c072340 0x7f3d9c14e750 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 >> 172.21.15.86:0/4035619195 conn(0x7f3d9c0b1730 msgr2=0x7f3d9c15e4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 shutdown_connections 2024-03-06T07:28:45.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.747+0000 7f3da1e00640 1 -- 172.21.15.86:0/4035619195 wait complete. 2024-03-06T07:28:45.767 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:28:45.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.972+0000 7fda1c400640 1 Processor -- start 2024-03-06T07:28:45.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.972+0000 7fda1c400640 1 -- start start 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.974+0000 7fda1c400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fda140c6460 0x7fda140c5420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.974+0000 7fda1c400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fda14145b30 con 0x7fda140c5960 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.974+0000 7fda1c400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fda14142ce0 con 0x7fda140c4930 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.974+0000 7fda1c400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fda140c68c0 con 0x7fda140c6460 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.974+0000 7fda1a400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fda140c5960 0x7fda14165b30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54698/0 (socket says 172.21.15.86:54698) 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.974+0000 7fda1a400640 1 -- 172.21.15.86:0/4099706970 learned_addr learned my addr 172.21.15.86:0/4099706970 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.974+0000 7fda19000640 1 --2- 172.21.15.86:0/4099706970 >> v2:172.21.15.86:3300/0 conn(0x7fda140c6460 0x7fda140c5420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.975+0000 7fda19000640 1 -- 172.21.15.86:0/4099706970 >> v1:172.21.15.194:6789/0 conn(0x7fda140c4930 legacy=0x7fda140c4d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.975+0000 7fda19000640 1 -- 172.21.15.86:0/4099706970 >> v1:172.21.15.86:6789/0 conn(0x7fda140c5960 legacy=0x7fda14165b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.975+0000 7fda19000640 1 -- 172.21.15.86:0/4099706970 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fda140cbca0 con 0x7fda140c6460 2024-03-06T07:28:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.976+0000 7fda19000640 1 --2- 172.21.15.86:0/4099706970 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c6460 0x7fda140c5420 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fda000097a0 tx=0x7fda0002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cd1cf96e9e070853 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.976+0000 7fda13e00640 1 -- 172.21.15.86:0/4099706970 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda000315a0 con 0x7fda140c6460 2024-03-06T07:28:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.976+0000 7fda13e00640 1 -- 172.21.15.86:0/4099706970 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fda00037070 con 0x7fda140c6460 2024-03-06T07:28:45.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.976+0000 7fda13e00640 1 -- 172.21.15.86:0/4099706970 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda00031960 con 0x7fda140c6460 2024-03-06T07:28:45.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.976+0000 7fda1c400640 1 -- 172.21.15.86:0/4099706970 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c6460 msgr2=0x7fda140c5420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.976+0000 7fda1c400640 1 --2- 172.21.15.86:0/4099706970 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c6460 0x7fda140c5420 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fda000097a0 tx=0x7fda0002b840 comp rx=0 tx=0).stop 2024-03-06T07:28:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.977+0000 7fda1c400640 1 -- 172.21.15.86:0/4099706970 shutdown_connections 2024-03-06T07:28:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.977+0000 7fda1c400640 1 --2- 172.21.15.86:0/4099706970 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c6460 0x7fda140c5420 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.977+0000 7fda1c400640 1 -- 172.21.15.86:0/4099706970 >> 172.21.15.86:0/4099706970 conn(0x7fda140cc4f0 msgr2=0x7fda140cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.977+0000 7fda1c400640 1 -- 172.21.15.86:0/4099706970 shutdown_connections 2024-03-06T07:28:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.977+0000 7fda1c400640 1 -- 172.21.15.86:0/4099706970 wait complete. 2024-03-06T07:28:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.978+0000 7fda1c400640 1 Processor -- start 2024-03-06T07:28:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.979+0000 7fda1c400640 1 -- start start 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.979+0000 7fda1c400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c4930 0x7fda14071490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.979+0000 7fda1c400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fda140c5960 0x7fda141e2100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.979+0000 7fda1c400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fda140c6460 0x7fda14072200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.979+0000 7fda1c400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda140ca3b0 con 0x7fda140c4930 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.979+0000 7fda1c400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda140ca0d0 con 0x7fda140c6460 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.979+0000 7fda1c400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fda1406e600 con 0x7fda140c5960 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c4930 0x7fda14071490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c4930 0x7fda14071490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57668/0 (socket says 172.21.15.86:57668) 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 -- 172.21.15.86:0/1370080803 learned_addr learned my addr 172.21.15.86:0/1370080803 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda1a400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fda140c6460 0x7fda14072200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19000640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fda140c5960 0x7fda141e2100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 -- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fda140c5960 msgr2=0x7fda141e2100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fda140c5960 0x7fda141e2100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 -- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fda140c6460 msgr2=0x7fda14072200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fda140c6460 0x7fda14072200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda19a00640 1 -- 172.21.15.86:0/1370080803 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fda141d4a80 con 0x7fda140c4930 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.980+0000 7fda1a400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fda140c6460 0x7fda14072200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.981+0000 7fda19a00640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c4930 0x7fda14071490 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7fda0800b3f0 tx=0x7fda0800b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.981+0000 7fda11600640 1 -- 172.21.15.86:0/1370080803 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda08015020 con 0x7fda140c4930 2024-03-06T07:28:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.981+0000 7fda11600640 1 -- 172.21.15.86:0/1370080803 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fda08011830 con 0x7fda140c4930 2024-03-06T07:28:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.981+0000 7fda11600640 1 -- 172.21.15.86:0/1370080803 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda08011c20 con 0x7fda140c4930 2024-03-06T07:28:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.981+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fda141d4d60 con 0x7fda140c4930 2024-03-06T07:28:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.981+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fda141d5320 con 0x7fda140c4930 2024-03-06T07:28:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.982+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fda140c5f70 con 0x7fda140c4930 2024-03-06T07:28:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.983+0000 7fda11600640 1 -- 172.21.15.86:0/1370080803 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fda08018020 con 0x7fda140c4930 2024-03-06T07:28:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.984+0000 7fda11600640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd9f406b4d0 0x7fd9f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.984+0000 7fda11600640 1 -- 172.21.15.86:0/1370080803 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fda08089ec0 con 0x7fda140c4930 2024-03-06T07:28:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.984+0000 7fda19000640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd9f406b4d0 0x7fd9f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:45.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.985+0000 7fda19000640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd9f406b4d0 0x7fd9f406d990 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fda00036000 tx=0x7fda00008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:45.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:45.988+0000 7fda11600640 1 -- 172.21.15.86:0/1370080803 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fda140c5f70 con 0x7fda140c4930 2024-03-06T07:28:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.223+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7fda1404f330 con 0x7fda140c4930 2024-03-06T07:28:46.227 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:28:46.225+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:28:46.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.228+0000 7fda11600640 1 -- 172.21.15.86:0/1370080803 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v18) v1 ==== 161+0+0 (secure 0 0 0) 0x7fda0801f030 con 0x7fda140c4930 2024-03-06T07:28:46.231 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:28:46.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd9f406b4d0 msgr2=0x7fd9f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:46.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd9f406b4d0 0x7fd9f406d990 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fda00036000 tx=0x7fda00008030 comp rx=0 tx=0).stop 2024-03-06T07:28:46.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c4930 msgr2=0x7fda14071490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:46.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c4930 0x7fda14071490 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7fda0800b3f0 tx=0x7fda0800b8b0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 shutdown_connections 2024-03-06T07:28:46.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd9f406b4d0 0x7fd9f406d990 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fda140c6460 0x7fda14072200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fda140c5960 0x7fda141e2100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 --2- 172.21.15.86:0/1370080803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fda140c4930 0x7fda14071490 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.232+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 >> 172.21.15.86:0/1370080803 conn(0x7fda140cc4f0 msgr2=0x7fda141e38e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:46.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.233+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 shutdown_connections 2024-03-06T07:28:46.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.233+0000 7fda1c400640 1 -- 172.21.15.86:0/1370080803 wait complete. 2024-03-06T07:28:46.252 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:28:46.361 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:28:46.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.439+0000 7fcecd800640 1 Processor -- start 2024-03-06T07:28:46.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.440+0000 7fcecd800640 1 -- start start 2024-03-06T07:28:46.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.441+0000 7fcecd800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec80d07f0 0x7fcec80d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:46.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.441+0000 7fcecd800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcec80b3730 con 0x7fcec80d1bf0 2024-03-06T07:28:46.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.441+0000 7fcecd800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcec8067b20 con 0x7fcec815e450 2024-03-06T07:28:46.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.441+0000 7fcecd800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcec80d4930 con 0x7fcec80d07f0 2024-03-06T07:28:46.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.441+0000 7fcec7400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fcec80d1bf0 0x7fcec80b4b90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54700/0 (socket says 172.21.15.86:54700) 2024-03-06T07:28:46.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.441+0000 7fcec7400640 1 -- 172.21.15.86:0/1629984627 learned_addr learned my addr 172.21.15.86:0/1629984627 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:46.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.441+0000 7fcec6a00640 1 --2- 172.21.15.86:0/1629984627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec80d07f0 0x7fcec80d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:46.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.442+0000 7fcec5600640 1 -- 172.21.15.86:0/1629984627 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1879885813 0 0) 0x7fcec80b3730 con 0x7fcec80d1bf0 2024-03-06T07:28:46.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.442+0000 7fcec5600640 1 -- 172.21.15.86:0/1629984627 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcea8003520 con 0x7fcec80d1bf0 2024-03-06T07:28:46.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.442+0000 7fcec6a00640 1 -- 172.21.15.86:0/1629984627 >> v1:172.21.15.194:6789/0 conn(0x7fcec815e450 legacy=0x7fcec80b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:46.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.442+0000 7fcec6a00640 1 -- 172.21.15.86:0/1629984627 >> v1:172.21.15.86:6789/0 conn(0x7fcec80d1bf0 legacy=0x7fcec80b4b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:46.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.442+0000 7fcec5600640 1 -- 172.21.15.86:0/1629984627 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1592030293 0 0) 0x7fcea8003520 con 0x7fcec80d1bf0 2024-03-06T07:28:46.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.442+0000 7fcec6a00640 1 -- 172.21.15.86:0/1629984627 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcec80d1110 con 0x7fcec80d07f0 2024-03-06T07:28:46.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.443+0000 7fcec6a00640 1 --2- 172.21.15.86:0/1629984627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec80d07f0 0x7fcec80d0bd0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fceb80097d0 tx=0x7fceb802b770 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ff8832b60f55e481 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:46.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.444+0000 7fcec5600640 1 -- 172.21.15.86:0/1629984627 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fceb8037070 con 0x7fcec80d07f0 2024-03-06T07:28:46.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.444+0000 7fcec5600640 1 -- 172.21.15.86:0/1629984627 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fceb8030510 con 0x7fcec80d07f0 2024-03-06T07:28:46.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.444+0000 7fcec5600640 1 -- 172.21.15.86:0/1629984627 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcec00019f0 con 0x7fcec80d07f0 2024-03-06T07:28:46.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.445+0000 7fcecd800640 1 -- 172.21.15.86:0/1629984627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec80d07f0 msgr2=0x7fcec80d0bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:46.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.445+0000 7fcecd800640 1 --2- 172.21.15.86:0/1629984627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec80d07f0 0x7fcec80d0bd0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fceb80097d0 tx=0x7fceb802b770 comp rx=0 tx=0).stop 2024-03-06T07:28:46.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.445+0000 7fcecd800640 1 -- 172.21.15.86:0/1629984627 shutdown_connections 2024-03-06T07:28:46.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.445+0000 7fcecd800640 1 --2- 172.21.15.86:0/1629984627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec80d07f0 0x7fcec80d0bd0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.445+0000 7fcecd800640 1 -- 172.21.15.86:0/1629984627 >> 172.21.15.86:0/1629984627 conn(0x7fcec80d3e10 msgr2=0x7fcec80d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:46.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.445+0000 7fcecd800640 1 -- 172.21.15.86:0/1629984627 shutdown_connections 2024-03-06T07:28:46.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.445+0000 7fcecd800640 1 -- 172.21.15.86:0/1629984627 wait complete. 2024-03-06T07:28:46.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.446+0000 7fcecd800640 1 Processor -- start 2024-03-06T07:28:46.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.447+0000 7fcecd800640 1 -- start start 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.447+0000 7fcecd800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcec80d07f0 0x7fcec80c9380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.447+0000 7fcecd800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcec80d1bf0 0x7fcec806c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcecd800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec815e450 0x7fcec806c930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcecd800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcec80b3c80 con 0x7fcec80d1bf0 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcecd800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcec80b39a0 con 0x7fcec80d07f0 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcecd800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcec80b3f60 con 0x7fcec815e450 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcec80d1bf0 0x7fcec806c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcec80d1bf0 0x7fcec806c3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57682/0 (socket says 172.21.15.86:57682) 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 -- 172.21.15.86:0/3080386780 learned_addr learned my addr 172.21.15.86:0/3080386780 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:46.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6a00640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcec80d07f0 0x7fcec80c9380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec7400640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec815e450 0x7fcec806c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 -- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec815e450 msgr2=0x7fcec806c930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec815e450 0x7fcec806c930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 -- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcec80d07f0 msgr2=0x7fcec80c9380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcec80d07f0 0x7fcec80c9380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec6000640 1 -- 172.21.15.86:0/3080386780 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcec80b89d0 con 0x7fcec80d1bf0 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.448+0000 7fcec7400640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec815e450 0x7fcec806c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.449+0000 7fcec6000640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcec80d1bf0 0x7fcec806c3f0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fceb000c3c0 tx=0x7fceb000c7c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.449+0000 7fcebea00640 1 -- 172.21.15.86:0/3080386780 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fceb00052d0 con 0x7fcec80d1bf0 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.449+0000 7fcebea00640 1 -- 172.21.15.86:0/3080386780 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fceb0005490 con 0x7fcec80d1bf0 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.449+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcec80b8cb0 con 0x7fcec80d1bf0 2024-03-06T07:28:46.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.449+0000 7fcebea00640 1 -- 172.21.15.86:0/3080386780 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fceb0007260 con 0x7fcec80d1bf0 2024-03-06T07:28:46.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.449+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcec80b9270 con 0x7fcec80d1bf0 2024-03-06T07:28:46.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.450+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcec804f330 con 0x7fcec80d1bf0 2024-03-06T07:28:46.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.451+0000 7fcebea00640 1 -- 172.21.15.86:0/3080386780 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fceb0007400 con 0x7fcec80d1bf0 2024-03-06T07:28:46.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.452+0000 7fcebea00640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcea406b4d0 0x7fcea406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:46.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.452+0000 7fcebea00640 1 -- 172.21.15.86:0/3080386780 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 5795+0+0 (secure 0 0 0) 0x7fceb0089770 con 0x7fcec80d1bf0 2024-03-06T07:28:46.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.452+0000 7fcec6a00640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcea406b4d0 0x7fcea406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:46.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.453+0000 7fcec6a00640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcea406b4d0 0x7fcea406d990 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7fceb8034000 tx=0x7fceb8008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:46.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.456+0000 7fcebea00640 1 -- 172.21.15.86:0/3080386780 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fceb0058590 con 0x7fcec80d1bf0 2024-03-06T07:28:46.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:46.692+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7fcec80d2190 con 0x7fcec80d1bf0 2024-03-06T07:28:47.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.241+0000 7fcebea00640 1 -- 172.21.15.86:0/3080386780 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v19) v1 ==== 104+0+0 (secure 0 0 0) 0x7fceb0057e90 con 0x7fcec80d1bf0 2024-03-06T07:28:47.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcea406b4d0 msgr2=0x7fcea406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:47.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcea406b4d0 0x7fcea406d990 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7fceb8034000 tx=0x7fceb8008030 comp rx=0 tx=0).stop 2024-03-06T07:28:47.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcec80d1bf0 msgr2=0x7fcec806c3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:47.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcec80d1bf0 0x7fcec806c3f0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fceb000c3c0 tx=0x7fceb000c7c0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 shutdown_connections 2024-03-06T07:28:47.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcea406b4d0 0x7fcea406d990 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcec815e450 0x7fcec806c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcec80d1bf0 0x7fcec806c3f0 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 --2- 172.21.15.86:0/3080386780 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcec80d07f0 0x7fcec80c9380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 >> 172.21.15.86:0/3080386780 conn(0x7fcec80d3e10 msgr2=0x7fcec806ba00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:47.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 shutdown_connections 2024-03-06T07:28:47.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.245+0000 7fcecd800640 1 -- 172.21.15.86:0/3080386780 wait complete. 2024-03-06T07:28:47.268 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:28:47.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.485+0000 7f0893e00640 1 Processor -- start 2024-03-06T07:28:47.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.485+0000 7f0893e00640 1 -- start start 2024-03-06T07:28:47.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.487+0000 7f0893e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0894145420 0x7f089406c6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.487+0000 7f0893e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 0x7f089415cc40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.487+0000 7f0893e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f089406cdb0 con 0x7f089406b3e0 2024-03-06T07:28:47.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.487+0000 7f0893e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f08940b2a80 con 0x7f0894145420 2024-03-06T07:28:47.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.487+0000 7f0893e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f089406d090 con 0x7f08940cdd00 2024-03-06T07:28:47.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.487+0000 7f0893400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f089406b3e0 0x7f089415f740 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54712/0 (socket says 172.21.15.86:54712) 2024-03-06T07:28:47.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.487+0000 7f0893400640 1 -- 172.21.15.86:0/1537118243 learned_addr learned my addr 172.21.15.86:0/1537118243 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:47.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1994334723 0 0) 0x7f089406cdb0 con 0x7f089406b3e0 2024-03-06T07:28:47.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892a00640 1 --2- 172.21.15.86:0/1537118243 >> v2:172.21.15.194:3300/0 conn(0x7f0894145420 0x7f089406c6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0878003520 con 0x7f089406b3e0 2024-03-06T07:28:47.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f088a000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 0x7f089415cc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1568594103 0 0) 0x7f0878003520 con 0x7f089406b3e0 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 msgr2=0x7f089415cc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 0x7f089415cc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 >> v2:172.21.15.194:3300/0 conn(0x7f0894145420 msgr2=0x7f089406c6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 --2- 172.21.15.86:0/1537118243 >> v2:172.21.15.194:3300/0 conn(0x7f0894145420 0x7f089406c6f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f089406d370 con 0x7f089406b3e0 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f088c001670 con 0x7f089406b3e0 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 >> v1:172.21.15.86:6789/0 conn(0x7f089406b3e0 legacy=0x7f089415f740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08780039b0 0x7f0878003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0878004800 0x7f0878004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 0x7f08780051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08940b2a80 con 0x7f0878004800 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f089406d090 con 0x7f08780039b0 2024-03-06T07:28:47.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.488+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f087800b9c0 con 0x7f08940cdd00 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f088a000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0878004800 0x7f0878004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f0892a00640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08780039b0 0x7f0878003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f0893400640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 0x7f08780051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f088a000640 1 -- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 msgr2=0x7f08780051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f088a000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08940cdd00 0x7f08780051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f088a000640 1 -- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08780039b0 msgr2=0x7f0878003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f088a000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08780039b0 0x7f0878003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.489+0000 7f088a000640 1 -- 172.21.15.86:0/1537118243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0878003520 con 0x7f0878004800 2024-03-06T07:28:47.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.490+0000 7f088a000640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0878004800 0x7f0878004c80 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f087c00ba30 tx=0x7f0878005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f377046a47b8dab server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:47.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.490+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f087c036020 con 0x7f0878004800 2024-03-06T07:28:47.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.490+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f087c032640 con 0x7f0878004800 2024-03-06T07:28:47.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.490+0000 7f0892000640 1 -- 172.21.15.86:0/1537118243 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f087c00b600 con 0x7f0878004800 2024-03-06T07:28:47.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.491+0000 7f0893e00640 1 -- 172.21.15.86:0/1537118243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0878004800 msgr2=0x7f0878004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:47.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.491+0000 7f0893e00640 1 --2- 172.21.15.86:0/1537118243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0878004800 0x7f0878004c80 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f087c00ba30 tx=0x7f0878005830 comp rx=0 tx=0).stop 2024-03-06T07:28:47.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.491+0000 7f0893400640 1 -- 172.21.15.86:0/1537118243 reap_dead start 2024-03-06T07:28:47.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.491+0000 7f0893e00640 1 -- 172.21.15.86:0/1537118243 shutdown_connections 2024-03-06T07:28:47.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.491+0000 7f0893e00640 1 -- 172.21.15.86:0/1537118243 >> 172.21.15.86:0/1537118243 conn(0x7f08940b1a70 msgr2=0x7f08940b1e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:47.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.491+0000 7f0893e00640 1 -- 172.21.15.86:0/1537118243 shutdown_connections 2024-03-06T07:28:47.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.491+0000 7f0893e00640 1 -- 172.21.15.86:0/1537118243 wait complete. 2024-03-06T07:28:47.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.493+0000 7f0893e00640 1 Processor -- start 2024-03-06T07:28:47.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.493+0000 7f0893e00640 1 -- start start 2024-03-06T07:28:47.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0878004800 0x7f08940aee40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0894070850 0x7f0894143ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f089414dc10 0x7f089415d800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f089406b870 con 0x7f089414dc10 2024-03-06T07:28:47.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0894145800 con 0x7f0894070850 2024-03-06T07:28:47.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f089406b9f0 con 0x7f0878004800 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f089414dc10 0x7f089415d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f089414dc10 0x7f089415d800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57702/0 (socket says 172.21.15.86:57702) 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.494+0000 7f0893400640 1 -- 172.21.15.86:0/3597902410 learned_addr learned my addr 172.21.15.86:0/3597902410 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f088a000640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0894070850 0x7f0894143ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893400640 1 -- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0878004800 msgr2=0x7f08940aee40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0892a00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0878004800 0x7f08940aee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893400640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0878004800 0x7f08940aee40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893400640 1 -- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0894070850 msgr2=0x7f0894143ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893400640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0894070850 0x7f0894143ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893400640 1 -- 172.21.15.86:0/3597902410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0894144020 con 0x7f089414dc10 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f088a000640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0894070850 0x7f0894143ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893400640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f089414dc10 0x7f089415d800 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f08940b26c0 tx=0x7f088c00d950 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f088b400640 1 -- 172.21.15.86:0/3597902410 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f088c007440 con 0x7f089414dc10 2024-03-06T07:28:47.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f088b400640 1 -- 172.21.15.86:0/3597902410 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f088c007620 con 0x7f089414dc10 2024-03-06T07:28:47.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0894070ce0 con 0x7f089414dc10 2024-03-06T07:28:47.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f088b400640 1 -- 172.21.15.86:0/3597902410 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f088c004a20 con 0x7f089414dc10 2024-03-06T07:28:47.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.495+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08940712a0 con 0x7f089414dc10 2024-03-06T07:28:47.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.496+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f089404f330 con 0x7f089414dc10 2024-03-06T07:28:47.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.498+0000 7f088b400640 1 -- 172.21.15.86:0/3597902410 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f088c004bc0 con 0x7f089414dc10 2024-03-06T07:28:47.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.498+0000 7f088b400640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087006b500 0x7f087006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:47.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.499+0000 7f088b400640 1 -- 172.21.15.86:0/3597902410 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f088c089e50 con 0x7f089414dc10 2024-03-06T07:28:47.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.499+0000 7f0892a00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087006b500 0x7f087006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:47.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.500+0000 7f0892a00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087006b500 0x7f087006d9c0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f0884007380 tx=0x7f088400b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:47.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.503+0000 7f088b400640 1 -- 172.21.15.86:0/3597902410 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f088c01a060 con 0x7f089414dc10 2024-03-06T07:28:47.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:47.697+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f089406dfc0 con 0x7f089414dc10 2024-03-06T07:28:48.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.244+0000 7f088b400640 1 -- 172.21.15.86:0/3597902410 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v37) v1 ==== 181+0+0 (secure 0 0 0) 0x7f0894144020 con 0x7f089414dc10 2024-03-06T07:28:48.247 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:28:48.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.247+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087006b500 msgr2=0x7f087006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:48.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.247+0000 7f0893e00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087006b500 0x7f087006d9c0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f0884007380 tx=0x7f088400b040 comp rx=0 tx=0).stop 2024-03-06T07:28:48.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.247+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f089414dc10 msgr2=0x7f089415d800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:48.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.247+0000 7f0893e00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f089414dc10 0x7f089415d800 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f08940b26c0 tx=0x7f088c00d950 comp rx=0 tx=0).stop 2024-03-06T07:28:48.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 shutdown_connections 2024-03-06T07:28:48.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f087006b500 0x7f087006d9c0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f089414dc10 0x7f089415d800 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0894070850 0x7f0894143ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 --2- 172.21.15.86:0/3597902410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0878004800 0x7f08940aee40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 >> 172.21.15.86:0/3597902410 conn(0x7f08940b1a70 msgr2=0x7f08940b0a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:48.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 shutdown_connections 2024-03-06T07:28:48.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.248+0000 7f0893e00640 1 -- 172.21.15.86:0/3597902410 wait complete. 2024-03-06T07:28:48.274 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:28:48.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.477+0000 7fd0ed800640 1 Processor -- start 2024-03-06T07:28:48.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.477+0000 7fd0ed800640 1 -- start start 2024-03-06T07:28:48.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.478+0000 7fd0ed800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd0e8145980 0x7fd0e815fa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0ed800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e815ff40 0x7fd0e806dec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0ed800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd0e80cf1e0 con 0x7fd0e80ced20 2024-03-06T07:28:48.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0ed800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd0e8070e40 con 0x7fd0e8145980 2024-03-06T07:28:48.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0ed800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0e8160370 con 0x7fd0e815ff40 2024-03-06T07:28:48.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0e6000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd0e80ced20 0x7fd0e80712d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54716/0 (socket says 172.21.15.86:54716) 2024-03-06T07:28:48.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0e6000640 1 -- 172.21.15.86:0/2955083593 learned_addr learned my addr 172.21.15.86:0/2955083593 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:48.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/2955083593 >> v2:172.21.15.194:3300/0 conn(0x7fd0e8145980 0x7fd0e815fa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.479+0000 7fd0e7400640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e815ff40 0x7fd0e806dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1179622527 0 0) 0x7fd0e80cf1e0 con 0x7fd0e80ced20 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd0cc003520 con 0x7fd0e80ced20 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2257247455 0 0) 0x7fd0cc003520 con 0x7fd0e80ced20 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e815ff40 msgr2=0x7fd0e806dec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e815ff40 0x7fd0e806dec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 >> v2:172.21.15.194:3300/0 conn(0x7fd0e8145980 msgr2=0x7fd0e815fa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 --2- 172.21.15.86:0/2955083593 >> v2:172.21.15.194:3300/0 conn(0x7fd0e8145980 0x7fd0e815fa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0e80cf4c0 con 0x7fd0e80ced20 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd0d4001670 con 0x7fd0e80ced20 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 >> v1:172.21.15.86:6789/0 conn(0x7fd0e80ced20 legacy=0x7fd0e80712d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0cc003a20 0x7fd0cc003e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc004800 0x7fd0cc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e815ff40 0x7fd0cc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0e8070e40 con 0x7fd0cc003a20 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0e8160370 con 0x7fd0cc004800 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0cc00b9c0 con 0x7fd0e815ff40 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.480+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0cc003a20 0x7fd0cc003e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.481+0000 7fd0e6000640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc004800 0x7fd0cc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.481+0000 7fd0e6a00640 1 -- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e815ff40 msgr2=0x7fd0cc0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.481+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e815ff40 0x7fd0cc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.481+0000 7fd0e6a00640 1 -- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc004800 msgr2=0x7fd0cc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.481+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc004800 0x7fd0cc004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.481+0000 7fd0e6a00640 1 -- 172.21.15.86:0/2955083593 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0cc003520 con 0x7fd0cc003a20 2024-03-06T07:28:48.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.482+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0cc003a20 0x7fd0cc003e40 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7fd0d8009af0 tx=0x7fd0cc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6eedf9d5d7a473e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.482+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0d8039070 con 0x7fd0cc003a20 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.482+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd0d802c710 con 0x7fd0cc003a20 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.482+0000 7fd0e5600640 1 -- 172.21.15.86:0/2955083593 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0d8009690 con 0x7fd0cc003a20 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.483+0000 7fd0ed800640 1 -- 172.21.15.86:0/2955083593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0cc003a20 msgr2=0x7fd0cc003e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.483+0000 7fd0ed800640 1 --2- 172.21.15.86:0/2955083593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0cc003a20 0x7fd0cc003e40 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7fd0d8009af0 tx=0x7fd0cc005830 comp rx=0 tx=0).stop 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.483+0000 7fd0e7400640 1 -- 172.21.15.86:0/2955083593 reap_dead start 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.483+0000 7fd0ed800640 1 -- 172.21.15.86:0/2955083593 shutdown_connections 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.483+0000 7fd0ed800640 1 -- 172.21.15.86:0/2955083593 >> 172.21.15.86:0/2955083593 conn(0x7fd0e80cfa80 msgr2=0x7fd0e80cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.483+0000 7fd0ed800640 1 -- 172.21.15.86:0/2955083593 shutdown_connections 2024-03-06T07:28:48.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.483+0000 7fd0ed800640 1 -- 172.21.15.86:0/2955083593 wait complete. 2024-03-06T07:28:48.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.485+0000 7fd0ed800640 1 Processor -- start 2024-03-06T07:28:48.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.485+0000 7fd0ed800640 1 -- start start 2024-03-06T07:28:48.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.486+0000 7fd0ed800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc003a20 0x7fd0e8140560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0ed800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0e8142580 0x7fd0e814f4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0ed800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e81444b0 0x7fd0e8144930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0ed800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0e8160740 con 0x7fd0e8142580 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0ed800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0e8070ad0 con 0x7fd0cc003a20 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0ed800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0e81608c0 con 0x7fd0e81444b0 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0e6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0e8142580 0x7fd0e814f4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0e6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0e8142580 0x7fd0e814f4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57724/0 (socket says 172.21.15.86:57724) 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0e6000640 1 -- 172.21.15.86:0/72086936 learned_addr learned my addr 172.21.15.86:0/72086936 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc003a20 0x7fd0e8140560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.487+0000 7fd0e7400640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e81444b0 0x7fd0e8144930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0e6000640 1 -- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e81444b0 msgr2=0x7fd0e8144930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:48.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0e6000640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e81444b0 0x7fd0e8144930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0e6000640 1 -- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc003a20 msgr2=0x7fd0e8140560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0e6000640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc003a20 0x7fd0e8140560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0e6000640 1 -- 172.21.15.86:0/72086936 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0e8143870 con 0x7fd0e8142580 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0e6000640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0e8142580 0x7fd0e814f4b0 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7fd0d400e860 tx=0x7fd0d400ec60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0dea00640 1 -- 172.21.15.86:0/72086936 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0d400c8d0 con 0x7fd0e8142580 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0dea00640 1 -- 172.21.15.86:0/72086936 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd0d4018070 con 0x7fd0e8142580 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0dea00640 1 -- 172.21.15.86:0/72086936 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0d400ccb0 con 0x7fd0e8142580 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd0e8142cf0 con 0x7fd0e8142580 2024-03-06T07:28:48.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.488+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0e81432b0 con 0x7fd0e8142580 2024-03-06T07:28:48.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.489+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd0e804f330 con 0x7fd0e8142580 2024-03-06T07:28:48.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.490+0000 7fd0dea00640 1 -- 172.21.15.86:0/72086936 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd0d401d460 con 0x7fd0e8142580 2024-03-06T07:28:48.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.491+0000 7fd0dea00640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0c806b4d0 0x7fd0c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:48.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.491+0000 7fd0dea00640 1 -- 172.21.15.86:0/72086936 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(37..37 src has 1..37) v4 ==== 5357+0+0 (secure 0 0 0) 0x7fd0d4089ff0 con 0x7fd0e8142580 2024-03-06T07:28:48.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.491+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0c806b4d0 0x7fd0c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:48.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.492+0000 7fd0e6a00640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0c806b4d0 0x7fd0c806d990 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd0d8036000 tx=0x7fd0d8030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:48.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.496+0000 7fd0dea00640 1 -- 172.21.15.86:0/72086936 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd0d4058f50 con 0x7fd0e8142580 2024-03-06T07:28:48.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:48.699+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fd0e8072920 con 0x7fd0e8142580 2024-03-06T07:28:49.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.255+0000 7fd0dea00640 1 -- 172.21.15.86:0/72086936 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v38) v1 ==== 169+0+0 (secure 0 0 0) 0x7fd0d4058850 con 0x7fd0e8142580 2024-03-06T07:28:49.258 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:28:49.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0c806b4d0 msgr2=0x7fd0c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0c806b4d0 0x7fd0c806d990 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd0d8036000 tx=0x7fd0d8030000 comp rx=0 tx=0).stop 2024-03-06T07:28:49.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0e8142580 msgr2=0x7fd0e814f4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0e8142580 0x7fd0e814f4b0 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7fd0d400e860 tx=0x7fd0d400ec60 comp rx=0 tx=0).stop 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 shutdown_connections 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0c806b4d0 0x7fd0c806d990 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0e81444b0 0x7fd0e8144930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0e8142580 0x7fd0e814f4b0 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 --2- 172.21.15.86:0/72086936 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0cc003a20 0x7fd0e8140560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.259+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 >> 172.21.15.86:0/72086936 conn(0x7fd0e80cfa80 msgr2=0x7fd0e80cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.260+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 shutdown_connections 2024-03-06T07:28:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.260+0000 7fd0ed800640 1 -- 172.21.15.86:0/72086936 wait complete. 2024-03-06T07:28:49.287 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-06T07:28:49.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.488+0000 7f314d800640 1 Processor -- start 2024-03-06T07:28:49.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.488+0000 7f314d800640 1 -- start start 2024-03-06T07:28:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f314d800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f31480b3f70 0x7f3148160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f314d800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3148070ff0 0x7f31480ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f314d800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3148071440 con 0x7f314815fc10 2024-03-06T07:28:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f314d800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f31480b46c0 con 0x7f3148070ff0 2024-03-06T07:28:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f314d800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f31480b49a0 con 0x7f31480b3f70 2024-03-06T07:28:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f3147400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3148070ff0 0x7f31480ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f3147400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3148070ff0 0x7f31480ce4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57732/0 (socket says 172.21.15.86:57732) 2024-03-06T07:28:49.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f3147400640 1 -- 172.21.15.86:0/4035374524 learned_addr learned my addr 172.21.15.86:0/4035374524 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:49.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f3147400640 1 -- 172.21.15.86:0/4035374524 >> v2:172.21.15.194:3300/0 conn(0x7f31480b3f70 msgr2=0x7f3148160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:49.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f3147400640 1 --2- 172.21.15.86:0/4035374524 >> v2:172.21.15.194:3300/0 conn(0x7f31480b3f70 0x7f3148160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f3147400640 1 -- 172.21.15.86:0/4035374524 >> v1:172.21.15.194:6789/0 conn(0x7f314815fc10 legacy=0x7f314815fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:49.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.490+0000 7f3147400640 1 -- 172.21.15.86:0/4035374524 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3148072c90 con 0x7f3148070ff0 2024-03-06T07:28:49.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.491+0000 7f3147400640 1 --2- 172.21.15.86:0/4035374524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f31480ce4a0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f3148074a50 tx=0x7f314002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=37e3b4e92d6684a0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:49.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f3146000640 1 -- 172.21.15.86:0/4035374524 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3140035020 con 0x7f3148070ff0 2024-03-06T07:28:49.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f3146000640 1 -- 172.21.15.86:0/4035374524 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3140031870 con 0x7f3148070ff0 2024-03-06T07:28:49.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f3146000640 1 -- 172.21.15.86:0/4035374524 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3140031c40 con 0x7f3148070ff0 2024-03-06T07:28:49.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f314d800640 1 -- 172.21.15.86:0/4035374524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 msgr2=0x7f31480ce4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f314d800640 1 --2- 172.21.15.86:0/4035374524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f31480ce4a0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f3148074a50 tx=0x7f314002b910 comp rx=0 tx=0).stop 2024-03-06T07:28:49.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f314d800640 1 -- 172.21.15.86:0/4035374524 shutdown_connections 2024-03-06T07:28:49.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f314d800640 1 --2- 172.21.15.86:0/4035374524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f31480ce4a0 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f314d800640 1 --2- 172.21.15.86:0/4035374524 >> v2:172.21.15.194:3300/0 conn(0x7f31480b3f70 0x7f3148160700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.492+0000 7f314d800640 1 -- 172.21.15.86:0/4035374524 >> 172.21.15.86:0/4035374524 conn(0x7f31480b1940 msgr2=0x7f31480b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:49.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.493+0000 7f314d800640 1 -- 172.21.15.86:0/4035374524 shutdown_connections 2024-03-06T07:28:49.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.493+0000 7f314d800640 1 -- 172.21.15.86:0/4035374524 wait complete. 2024-03-06T07:28:49.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.494+0000 7f314d800640 1 Processor -- start 2024-03-06T07:28:49.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.494+0000 7f314d800640 1 -- start start 2024-03-06T07:28:49.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.495+0000 7f314d800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f3148151c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.495+0000 7f314d800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31480b3f70 0x7f314809b620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.495+0000 7f314d800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314815fc10 0x7f31480caa20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.495+0000 7f314d800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3148072a30 con 0x7f3148070ff0 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.495+0000 7f314d800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31480728b0 con 0x7f314815fc10 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.495+0000 7f314d800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f314815c2e0 con 0x7f31480b3f70 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f3148151c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f3148151c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57740/0 (socket says 172.21.15.86:57740) 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 -- 172.21.15.86:0/1463983652 learned_addr learned my addr 172.21.15.86:0/1463983652 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3147400640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314815fc10 0x7f31480caa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 -- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31480b3f70 msgr2=0x7f314809b620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31480b3f70 0x7f314809b620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 -- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314815fc10 msgr2=0x7f31480caa20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314815fc10 0x7f31480caa20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 -- 172.21.15.86:0/1463983652 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f314806cdb0 con 0x7f3148070ff0 2024-03-06T07:28:49.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3147400640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314815fc10 0x7f31480caa20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:49.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.496+0000 7f3146a00640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f3148151c70 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7f313800ba50 tx=0x7f313800bf10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:49.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.497+0000 7f313f400640 1 -- 172.21.15.86:0/1463983652 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3138019070 con 0x7f3148070ff0 2024-03-06T07:28:49.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.497+0000 7f313f400640 1 -- 172.21.15.86:0/1463983652 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f313800cc10 con 0x7f3148070ff0 2024-03-06T07:28:49.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.497+0000 7f313f400640 1 -- 172.21.15.86:0/1463983652 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3138014440 con 0x7f3148070ff0 2024-03-06T07:28:49.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.497+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f314806d090 con 0x7f3148070ff0 2024-03-06T07:28:49.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.497+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f314806d650 con 0x7f3148070ff0 2024-03-06T07:28:49.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.498+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f314804f330 con 0x7f3148070ff0 2024-03-06T07:28:49.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.499+0000 7f313f400640 1 -- 172.21.15.86:0/1463983652 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f313801c070 con 0x7f3148070ff0 2024-03-06T07:28:49.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.500+0000 7f313f400640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312406b4d0 0x7f312406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.500+0000 7f313f400640 1 -- 172.21.15.86:0/1463983652 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f3138089c50 con 0x7f3148070ff0 2024-03-06T07:28:49.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.501+0000 7f313e000640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312406b4d0 0x7f312406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.501+0000 7f313e000640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312406b4d0 0x7f312406d990 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f3148152880 tx=0x7f3130006c90 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:49.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.505+0000 7f313f400640 1 -- 172.21.15.86:0/1463983652 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3138058e40 con 0x7f3148070ff0 2024-03-06T07:28:49.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.703+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f314806ef30 con 0x7f3148070ff0 2024-03-06T07:28:49.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.710+0000 7f313f400640 1 -- 172.21.15.86:0/1463983652 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v11) v1 ==== 104+0+0 (secure 0 0 0) 0x7f3138058740 con 0x7f3148070ff0 2024-03-06T07:28:49.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312406b4d0 msgr2=0x7f312406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312406b4d0 0x7f312406d990 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f3148152880 tx=0x7f3130006c90 comp rx=0 tx=0).stop 2024-03-06T07:28:49.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 msgr2=0x7f3148151c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f3148151c70 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7f313800ba50 tx=0x7f313800bf10 comp rx=0 tx=0).stop 2024-03-06T07:28:49.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 shutdown_connections 2024-03-06T07:28:49.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312406b4d0 0x7f312406d990 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314815fc10 0x7f31480caa20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f31480b3f70 0x7f314809b620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 --2- 172.21.15.86:0/1463983652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3148070ff0 0x7f3148151c70 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.712+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 >> 172.21.15.86:0/1463983652 conn(0x7f31480b1940 msgr2=0x7f31480ad120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:49.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.713+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 shutdown_connections 2024-03-06T07:28:49.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.713+0000 7f314d800640 1 -- 172.21.15.86:0/1463983652 wait complete. 2024-03-06T07:28:49.734 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dir_deletion' 2024-03-06T07:28:49.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.927+0000 7f62e1400640 1 Processor -- start 2024-03-06T07:28:49.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.927+0000 7f62e1400640 1 -- start start 2024-03-06T07:28:49.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62e1400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f62dc0cf430 0x7f62dc06d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62e1400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f62dc06d920 0x7f62dc15dff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62e1400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f62dc0b35b0 con 0x7f62dc151660 2024-03-06T07:28:49.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62e1400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62dc0b3ee0 con 0x7f62dc0cf430 2024-03-06T07:28:49.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62e1400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62dc0b3890 con 0x7f62dc06d920 2024-03-06T07:28:49.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62da800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f62dc151660 0x7f62dc071520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54730/0 (socket says 172.21.15.86:54730) 2024-03-06T07:28:49.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62da800640 1 -- 172.21.15.86:0/1462909128 learned_addr learned my addr 172.21.15.86:0/1462909128 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:49.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62d9e00640 1 --2- 172.21.15.86:0/1462909128 >> v2:172.21.15.86:3300/0 conn(0x7f62dc0cf430 0x7f62dc06d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62d9400640 1 -- 172.21.15.86:0/1462909128 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1052578066 0 0) 0x7f62dc0b35b0 con 0x7f62dc151660 2024-03-06T07:28:49.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62db200640 1 --2- 172.21.15.86:0/1462909128 >> v2:172.21.15.194:3300/0 conn(0x7f62dc06d920 0x7f62dc15dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62d9400640 1 -- 172.21.15.86:0/1462909128 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f62c0005730 con 0x7f62dc151660 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62d9e00640 1 -- 172.21.15.86:0/1462909128 >> v2:172.21.15.194:3300/0 conn(0x7f62dc06d920 msgr2=0x7f62dc15dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62d9e00640 1 --2- 172.21.15.86:0/1462909128 >> v2:172.21.15.194:3300/0 conn(0x7f62dc06d920 0x7f62dc15dff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62d9e00640 1 -- 172.21.15.86:0/1462909128 >> v1:172.21.15.86:6789/0 conn(0x7f62dc151660 legacy=0x7f62dc071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62d9e00640 1 -- 172.21.15.86:0/1462909128 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62dc0b3b70 con 0x7f62dc0cf430 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.929+0000 7f62db200640 1 --2- 172.21.15.86:0/1462909128 >> v2:172.21.15.194:3300/0 conn(0x7f62dc06d920 0x7f62dc15dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.931+0000 7f62d9e00640 1 --2- 172.21.15.86:0/1462909128 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc06d3e0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7f62dc154fc0 tx=0x7f62c402bdb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9c972090195bc770 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.931+0000 7f62d9400640 1 -- 172.21.15.86:0/1462909128 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62c4039070 con 0x7f62dc0cf430 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.931+0000 7f62d9400640 1 -- 172.21.15.86:0/1462909128 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62c402cb60 con 0x7f62dc0cf430 2024-03-06T07:28:49.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.931+0000 7f62d9400640 1 -- 172.21.15.86:0/1462909128 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62c4034440 con 0x7f62dc0cf430 2024-03-06T07:28:49.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 -- 172.21.15.86:0/1462909128 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 msgr2=0x7f62dc06d3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 --2- 172.21.15.86:0/1462909128 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc06d3e0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7f62dc154fc0 tx=0x7f62c402bdb0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 -- 172.21.15.86:0/1462909128 shutdown_connections 2024-03-06T07:28:49.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 --2- 172.21.15.86:0/1462909128 >> v2:172.21.15.194:3300/0 conn(0x7f62dc06d920 0x7f62dc15dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 --2- 172.21.15.86:0/1462909128 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc06d3e0 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 -- 172.21.15.86:0/1462909128 >> 172.21.15.86:0/1462909128 conn(0x7f62dc0b1730 msgr2=0x7f62dc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:49.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 -- 172.21.15.86:0/1462909128 shutdown_connections 2024-03-06T07:28:49.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.932+0000 7f62e1400640 1 -- 172.21.15.86:0/1462909128 wait complete. 2024-03-06T07:28:49.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.934+0000 7f62e1400640 1 Processor -- start 2024-03-06T07:28:49.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.934+0000 7f62e1400640 1 -- start start 2024-03-06T07:28:49.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62e1400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62dc06d920 0x7f62dc144040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62e1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc152a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62e1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62dc151660 0x7f62dc159340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62e1400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62dc15e6b0 con 0x7f62dc0cf430 2024-03-06T07:28:49.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62e1400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62dc15e530 con 0x7f62dc06d920 2024-03-06T07:28:49.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62e1400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62dc15e830 con 0x7f62dc151660 2024-03-06T07:28:49.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc152a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc152a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57758/0 (socket says 172.21.15.86:57758) 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 -- 172.21.15.86:0/3062355915 learned_addr learned my addr 172.21.15.86:0/3062355915 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62da800640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62dc06d920 0x7f62dc144040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 -- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62dc151660 msgr2=0x7f62dc159340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62db200640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62dc151660 0x7f62dc159340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62dc151660 0x7f62dc159340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 -- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62dc06d920 msgr2=0x7f62dc144040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62dc06d920 0x7f62dc144040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62d9e00640 1 -- 172.21.15.86:0/3062355915 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62dc144680 con 0x7f62dc0cf430 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.935+0000 7f62da800640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62dc06d920 0x7f62dc144040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.936+0000 7f62d9e00640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc152a90 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7f62c402c3c0 tx=0x7f62c4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:49.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.936+0000 7f62d2a00640 1 -- 172.21.15.86:0/3062355915 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62c4039050 con 0x7f62dc0cf430 2024-03-06T07:28:49.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.936+0000 7f62d2a00640 1 -- 172.21.15.86:0/3062355915 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62c4045070 con 0x7f62dc0cf430 2024-03-06T07:28:49.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.936+0000 7f62d2a00640 1 -- 172.21.15.86:0/3062355915 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62c4009450 con 0x7f62dc0cf430 2024-03-06T07:28:49.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.936+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f62dc144940 con 0x7f62dc0cf430 2024-03-06T07:28:49.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.936+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62dc144f00 con 0x7f62dc0cf430 2024-03-06T07:28:49.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.937+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f62dc04f330 con 0x7f62dc0cf430 2024-03-06T07:28:49.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.938+0000 7f62d2a00640 1 -- 172.21.15.86:0/3062355915 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f62c402cd10 con 0x7f62dc0cf430 2024-03-06T07:28:49.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.939+0000 7f62d2a00640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62b806b4d0 0x7f62b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:49.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.939+0000 7f62d2a00640 1 -- 172.21.15.86:0/3062355915 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f62c40ac6f0 con 0x7f62dc0cf430 2024-03-06T07:28:49.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.939+0000 7f62da800640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62b806b4d0 0x7f62b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:49.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.940+0000 7f62da800640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62b806b4d0 0x7f62b806d990 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f62d4005880 tx=0x7f62d4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:49.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:49.944+0000 7f62d2a00640 1 -- 172.21.15.86:0/3062355915 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f62c404f020 con 0x7f62dc0cf430 2024-03-06T07:28:50.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.215+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]} v 0) v1 -- 0x7f62dc0742e0 con 0x7f62dc0cf430 2024-03-06T07:28:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.258+0000 7f62d2a00640 1 -- 172.21.15.86:0/3062355915 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]}]=0 v115) v1 ==== 132+0+0 (secure 0 0 0) 0x7f62c4032e20 con 0x7f62dc0cf430 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.262+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62b806b4d0 msgr2=0x7f62b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.262+0000 7f62e1400640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62b806b4d0 0x7f62b806d990 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f62d4005880 tx=0x7f62d4009040 comp rx=0 tx=0).stop 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.262+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 msgr2=0x7f62dc152a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.262+0000 7f62e1400640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc152a90 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7f62c402c3c0 tx=0x7f62c4004a40 comp rx=0 tx=0).stop 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 shutdown_connections 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62b806b4d0 0x7f62b806d990 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62dc151660 0x7f62dc159340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62dc0cf430 0x7f62dc152a90 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 --2- 172.21.15.86:0/3062355915 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62dc06d920 0x7f62dc144040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 >> 172.21.15.86:0/3062355915 conn(0x7f62dc0b1730 msgr2=0x7f62dc15cae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:50.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 shutdown_connections 2024-03-06T07:28:50.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.263+0000 7f62e1400640 1 -- 172.21.15.86:0/3062355915 wait complete. 2024-03-06T07:28:50.363 INFO:tasks.cephfs_test_runner:That when deleting a bunch of dentries and the containing ... ok 2024-03-06T07:28:50.363 INFO:tasks.cephfs_test_runner:Starting test: test_dirfrag_limit (tasks.cephfs.test_strays.TestStrays) That the directory fragment size cannot exceed mds_bal_fragment_size_max (using a limit of 50 in all configurations). 2024-03-06T07:28:50.363 INFO:tasks.cephfs_test_runner:test_dirfrag_limit (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:28:50.363 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:50.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.570+0000 7f2f25400640 1 Processor -- start 2024-03-06T07:28:50.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.571+0000 7f2f25400640 1 -- start start 2024-03-06T07:28:50.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.572+0000 7f2f25400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f2f2015f9d0 0x7f2f2015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.572+0000 7f2f25400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2f2006cef0 con 0x7f2f20070b10 2024-03-06T07:28:50.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.572+0000 7f2f25400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2f2006d1d0 con 0x7f2f20144c00 2024-03-06T07:28:50.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.572+0000 7f2f25400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2f2006d4b0 con 0x7f2f2015f9d0 2024-03-06T07:28:50.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.572+0000 7f2f1f200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2f20070b10 0x7f2f20070f20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54746/0 (socket says 172.21.15.86:54746) 2024-03-06T07:28:50.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.572+0000 7f2f1f200640 1 -- 172.21.15.86:0/2075757187 learned_addr learned my addr 172.21.15.86:0/2075757187 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:50.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3837858134 0 0) 0x7f2f2006cef0 con 0x7f2f20070b10 2024-03-06T07:28:50.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2f04003520 con 0x7f2f20070b10 2024-03-06T07:28:50.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1de00640 1 --2- 172.21.15.86:0/2075757187 >> v2:172.21.15.194:3300/0 conn(0x7f2f2015f9d0 0x7f2f2015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:50.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1229846518 0 0) 0x7f2f04003520 con 0x7f2f20070b10 2024-03-06T07:28:50.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 >> v2:172.21.15.194:3300/0 conn(0x7f2f2015f9d0 msgr2=0x7f2f2015fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:50.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 --2- 172.21.15.86:0/2075757187 >> v2:172.21.15.194:3300/0 conn(0x7f2f2015f9d0 0x7f2f2015fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 >> v1:172.21.15.194:6789/0 conn(0x7f2f20144c00 legacy=0x7f2f2006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f2006d790 con 0x7f2f20070b10 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f2f180019f0 con 0x7f2f20070b10 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 >> v1:172.21.15.86:6789/0 conn(0x7f2f20070b10 legacy=0x7f2f20070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.573+0000 7f2f1d400640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f04003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1d400640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f04004800 0x7f2f04004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1d400640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f040051c0 0x7f2f0400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f2006d4b0 con 0x7f2f040039b0 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f0400c110 con 0x7f2f040051c0 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2f0400c310 con 0x7f2f04004800 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1e800640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f04003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1de00640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f04004800 0x7f2f04004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1e800640 1 -- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f04004800 msgr2=0x7f2f04004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1e800640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f04004800 0x7f2f04004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1e800640 1 -- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f040051c0 msgr2=0x7f2f0400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:50.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1e800640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f040051c0 0x7f2f0400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1e800640 1 -- 172.21.15.86:0/2075757187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f04003520 con 0x7f2f040039b0 2024-03-06T07:28:50.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1f200640 1 -- 172.21.15.86:0/2075757187 reap_dead start 2024-03-06T07:28:50.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.574+0000 7f2f1de00640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f04004800 0x7f2f04004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.575+0000 7f2f1e800640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f04003dd0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f2f10001750 tx=0x7f2f04005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2c7b6943828aa8f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.575+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f10001370 con 0x7f2f040039b0 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f100052d0 con 0x7f2f040039b0 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f1d400640 1 -- 172.21.15.86:0/2075757187 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f10003830 con 0x7f2f040039b0 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f25400640 1 -- 172.21.15.86:0/2075757187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 msgr2=0x7f2f04003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f25400640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f04003dd0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f2f10001750 tx=0x7f2f04005830 comp rx=0 tx=0).stop 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f25400640 1 -- 172.21.15.86:0/2075757187 shutdown_connections 2024-03-06T07:28:50.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f25400640 1 --2- 172.21.15.86:0/2075757187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f04003dd0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f25400640 1 -- 172.21.15.86:0/2075757187 >> 172.21.15.86:0/2075757187 conn(0x7f2f200b1730 msgr2=0x7f2f200b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:50.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f25400640 1 -- 172.21.15.86:0/2075757187 shutdown_connections 2024-03-06T07:28:50.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.576+0000 7f2f25400640 1 -- 172.21.15.86:0/2075757187 wait complete. 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.578+0000 7f2f25400640 1 Processor -- start 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.578+0000 7f2f25400640 1 -- start start 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f25400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f2015e280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f25400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2015cd50 0x7f2f2013cc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f25400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f2015ab40 0x7f2f2013e9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f25400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f2006ccd0 con 0x7f2f040039b0 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f25400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f2006cb50 con 0x7f2f2015ab40 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f25400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2f2006df90 con 0x7f2f2015cd50 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f1e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f2015e280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f1e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f2015e280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57780/0 (socket says 172.21.15.86:57780) 2024-03-06T07:28:50.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.579+0000 7f2f1e800640 1 -- 172.21.15.86:0/1605697168 learned_addr learned my addr 172.21.15.86:0/1605697168 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:50.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f1e800640 1 -- 172.21.15.86:0/1605697168 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2015cd50 msgr2=0x7f2f2013cc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f1e800640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2015cd50 0x7f2f2013cc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f1e800640 1 -- 172.21.15.86:0/1605697168 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f2015ab40 msgr2=0x7f2f2013e9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f1e800640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f2015ab40 0x7f2f2013e9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f1e800640 1 -- 172.21.15.86:0/1605697168 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f2015d1e0 con 0x7f2f040039b0 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f1e800640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f2015e280 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7f2f10000c00 tx=0x7f2f10001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f16a00640 1 -- 172.21.15.86:0/1605697168 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f10041020 con 0x7f2f040039b0 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f16a00640 1 -- 172.21.15.86:0/1605697168 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f10005480 con 0x7f2f040039b0 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f16a00640 1 -- 172.21.15.86:0/1605697168 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f10001370 con 0x7f2f040039b0 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.580+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f2015d4c0 con 0x7f2f040039b0 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.581+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f2015da80 con 0x7f2f040039b0 2024-03-06T07:28:50.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.582+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2f2004f330 con 0x7f2f040039b0 2024-03-06T07:28:50.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.582+0000 7f2f16a00640 1 -- 172.21.15.86:0/1605697168 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2f10044020 con 0x7f2f040039b0 2024-03-06T07:28:50.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.583+0000 7f2f16a00640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2efc06b470 0x7f2efc06d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:50.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.584+0000 7f2f16a00640 1 -- 172.21.15.86:0/1605697168 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f2f100abac0 con 0x7f2f040039b0 2024-03-06T07:28:50.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.584+0000 7f2f1de00640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2efc06b470 0x7f2efc06d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:50.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.584+0000 7f2f1de00640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2efc06b470 0x7f2efc06d930 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f2f200b4ea0 tx=0x7f2f08009260 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:50.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.588+0000 7f2f16a00640 1 -- 172.21.15.86:0/1605697168 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2f1004b030 con 0x7f2f040039b0 2024-03-06T07:28:50.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.783+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2f20072890 con 0x7f2f040039b0 2024-03-06T07:28:50.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.784+0000 7f2f16a00640 1 -- 172.21.15.86:0/1605697168 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v38) v1 ==== 74+0+13032 (secure 0 0 0) 0x7f2f100303c0 con 0x7f2f040039b0 2024-03-06T07:28:50.787 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:50.787 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":38,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:49.246340+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.194:6835/902800806":"2024-03-07T07:28:46.225986+0000","172.21.15.194:6833/902800806":"2024-03-07T07:28:46.225986+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2efc06b470 msgr2=0x7f2efc06d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2efc06b470 0x7f2efc06d930 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f2f200b4ea0 tx=0x7f2f08009260 comp rx=0 tx=0).stop 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 msgr2=0x7f2f2015e280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f2015e280 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7f2f10000c00 tx=0x7f2f10001120 comp rx=0 tx=0).stop 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 shutdown_connections 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2efc06b470 0x7f2efc06d930 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f2015ab40 0x7f2f2013e9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2015cd50 0x7f2f2013cc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 --2- 172.21.15.86:0/1605697168 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f040039b0 0x7f2f2015e280 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:50.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.787+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 >> 172.21.15.86:0/1605697168 conn(0x7f2f200b1730 msgr2=0x7f2f200afe90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:50.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.788+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 shutdown_connections 2024-03-06T07:28:50.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:50.788+0000 7f2f25400640 1 -- 172.21.15.86:0/1605697168 wait complete. 2024-03-06T07:28:50.808 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:28:50.809 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:28:51.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.016+0000 7f6ffa400640 1 Processor -- start 2024-03-06T07:28:51.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.016+0000 7f6ffa400640 1 -- start start 2024-03-06T07:28:51.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ffa400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6ff40b3f70 0x7f6ff4160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ffa400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6ff4070ff0 0x7f6ff40ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ffa400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6ff40b46c0 con 0x7f6ff415fc10 2024-03-06T07:28:51.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ffa400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ff4071480 con 0x7f6ff40b3f70 2024-03-06T07:28:51.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ffa400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ff40b49a0 con 0x7f6ff4070ff0 2024-03-06T07:28:51.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ff3400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6ff415fc10 0x7f6ff415fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54754/0 (socket says 172.21.15.86:54754) 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ff3400640 1 -- 172.21.15.86:0/3858704465 learned_addr learned my addr 172.21.15.86:0/3858704465 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.018+0000 7f6ff2a00640 1 --2- 172.21.15.86:0/3858704465 >> v2:172.21.15.194:3300/0 conn(0x7f6ff40b3f70 0x7f6ff4160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff3e00640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6ff4070ff0 0x7f6ff40ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1893378917 0 0) 0x7f6ff40b46c0 con 0x7f6ff415fc10 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6fd8003520 con 0x7f6ff415fc10 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 450902913 0 0) 0x7f6fd8003520 con 0x7f6ff415fc10 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6ff4070ff0 msgr2=0x7f6ff40ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6ff4070ff0 0x7f6ff40ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 >> v2:172.21.15.194:3300/0 conn(0x7f6ff40b3f70 msgr2=0x7f6ff4160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 --2- 172.21.15.86:0/3858704465 >> v2:172.21.15.194:3300/0 conn(0x7f6ff40b3f70 0x7f6ff4160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ff4072c90 con 0x7f6ff415fc10 2024-03-06T07:28:51.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6fe4001670 con 0x7f6ff415fc10 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 >> v1:172.21.15.86:6789/0 conn(0x7f6ff415fc10 legacy=0x7f6ff415fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6fd80039b0 0x7f6fd8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6fd8004800 0x7f6fd8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6ff4070ff0 0x7f6fd80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ff4071480 con 0x7f6fd80039b0 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ff40b49a0 con 0x7f6fd8004800 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.019+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd800b9c0 con 0x7f6ff4070ff0 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.020+0000 7f6ff3400640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6fd80039b0 0x7f6fd8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.020+0000 7f6ff3400640 1 -- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6ff4070ff0 msgr2=0x7f6fd80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.020+0000 7f6ff3400640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6ff4070ff0 0x7f6fd80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.020+0000 7f6ff3400640 1 -- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6fd8004800 msgr2=0x7f6fd8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.020+0000 7f6ff3400640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6fd8004800 0x7f6fd8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.020+0000 7f6ff3400640 1 -- 172.21.15.86:0/3858704465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fd8003520 con 0x7f6fd80039b0 2024-03-06T07:28:51.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.021+0000 7f6ff3400640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6fd80039b0 0x7f6fd8003dd0 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7f6fe400c210 tx=0x7f6fd8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c94aded3ffae443f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.021+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fe4007d50 con 0x7f6fd80039b0 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.021+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fe4030070 con 0x7f6fd80039b0 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.021+0000 7f6ff2000640 1 -- 172.21.15.86:0/3858704465 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fe402cd80 con 0x7f6fd80039b0 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.022+0000 7f6ffa400640 1 -- 172.21.15.86:0/3858704465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6fd80039b0 msgr2=0x7f6fd8003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.022+0000 7f6ffa400640 1 --2- 172.21.15.86:0/3858704465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6fd80039b0 0x7f6fd8003dd0 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7f6fe400c210 tx=0x7f6fd8005830 comp rx=0 tx=0).stop 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.022+0000 7f6ff3e00640 1 -- 172.21.15.86:0/3858704465 reap_dead start 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.022+0000 7f6ffa400640 1 -- 172.21.15.86:0/3858704465 shutdown_connections 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.022+0000 7f6ffa400640 1 -- 172.21.15.86:0/3858704465 >> 172.21.15.86:0/3858704465 conn(0x7f6ff40b1940 msgr2=0x7f6ff40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.022+0000 7f6ffa400640 1 -- 172.21.15.86:0/3858704465 shutdown_connections 2024-03-06T07:28:51.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.022+0000 7f6ffa400640 1 -- 172.21.15.86:0/3858704465 wait complete. 2024-03-06T07:28:51.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.024+0000 7f6ffa400640 1 Processor -- start 2024-03-06T07:28:51.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.024+0000 7f6ffa400640 1 -- start start 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.025+0000 7f6ffa400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6fd80039b0 0x7f6ff414e190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.025+0000 7f6ffa400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6ff41f6820 0x7f6ff4154d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.025+0000 7f6ffa400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6ff4155240 0x7f6ff41f25f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.025+0000 7f6ffa400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ff4072a30 con 0x7f6ff41f6820 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.025+0000 7f6ffa400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ff40728b0 con 0x7f6ff4155240 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.025+0000 7f6ffa400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ff415c2e0 con 0x7f6fd80039b0 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6ff41f6820 0x7f6ff4154d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6ff41f6820 0x7f6ff4154d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57800/0 (socket says 172.21.15.86:57800) 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 -- 172.21.15.86:0/972532108 learned_addr learned my addr 172.21.15.86:0/972532108 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff3400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6fd80039b0 0x7f6ff414e190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff3e00640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6ff4155240 0x7f6ff41f25f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 -- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6fd80039b0 msgr2=0x7f6ff414e190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6fd80039b0 0x7f6ff414e190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 -- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6ff4155240 msgr2=0x7f6ff41f25f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6ff4155240 0x7f6ff41f25f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 -- 172.21.15.86:0/972532108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ff41f3570 con 0x7f6ff41f6820 2024-03-06T07:28:51.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.026+0000 7f6ff2a00640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6ff41f6820 0x7f6ff4154d00 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f6fdc00e7c0 tx=0x7f6fdc00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:51.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.027+0000 7f6feb400640 1 -- 172.21.15.86:0/972532108 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fdc017070 con 0x7f6ff41f6820 2024-03-06T07:28:51.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.027+0000 7f6feb400640 1 -- 172.21.15.86:0/972532108 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fdc00c8c0 con 0x7f6ff41f6820 2024-03-06T07:28:51.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.027+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ff41f3290 con 0x7f6ff41f6820 2024-03-06T07:28:51.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.027+0000 7f6feb400640 1 -- 172.21.15.86:0/972532108 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fdc00ccb0 con 0x7f6ff41f6820 2024-03-06T07:28:51.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.027+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ff41f6020 con 0x7f6ff41f6820 2024-03-06T07:28:51.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.028+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ff415f960 con 0x7f6ff41f6820 2024-03-06T07:28:51.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.029+0000 7f6feb400640 1 -- 172.21.15.86:0/972532108 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6fdc01a070 con 0x7f6ff41f6820 2024-03-06T07:28:51.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.030+0000 7f6feb400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6fd006b4d0 0x7f6fd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.030+0000 7f6feb400640 1 -- 172.21.15.86:0/972532108 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f6fdc08a0c0 con 0x7f6ff41f6820 2024-03-06T07:28:51.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.030+0000 7f6ff3400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6fd006b4d0 0x7f6fd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.031+0000 7f6ff3400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6fd006b4d0 0x7f6fd006d990 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f6fe4034000 tx=0x7f6fe4007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:51.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.034+0000 7f6feb400640 1 -- 172.21.15.86:0/972532108 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6ff415f960 con 0x7f6ff41f6820 2024-03-06T07:28:51.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.228+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f6ff40b43e0 con 0x7f6ff41f6820 2024-03-06T07:28:51.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.229+0000 7f6feb400640 1 -- 172.21.15.86:0/972532108 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v38) v1 ==== 93+0+10 (secure 0 0 0) 0x7f6ff40b43e0 con 0x7f6ff41f6820 2024-03-06T07:28:51.231 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:28:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6fd006b4d0 msgr2=0x7f6fd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6fd006b4d0 0x7f6fd006d990 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f6fe4034000 tx=0x7f6fe4007000 comp rx=0 tx=0).stop 2024-03-06T07:28:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6ff41f6820 msgr2=0x7f6ff4154d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6ff41f6820 0x7f6ff4154d00 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f6fdc00e7c0 tx=0x7f6fdc00ec80 comp rx=0 tx=0).stop 2024-03-06T07:28:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 shutdown_connections 2024-03-06T07:28:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6fd006b4d0 0x7f6fd006d990 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6ff4155240 0x7f6ff41f25f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6ff41f6820 0x7f6ff4154d00 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 --2- 172.21.15.86:0/972532108 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6fd80039b0 0x7f6ff414e190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 >> 172.21.15.86:0/972532108 conn(0x7f6ff40b1940 msgr2=0x7f6ff415fff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:51.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.231+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 shutdown_connections 2024-03-06T07:28:51.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.232+0000 7f6ffa400640 1 -- 172.21.15.86:0/972532108 wait complete. 2024-03-06T07:28:51.250 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit' 2024-03-06T07:28:51.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.448+0000 7fb61be00640 1 Processor -- start 2024-03-06T07:28:51.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.448+0000 7fb61be00640 1 -- start start 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb61be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb614144c00 0x7fb61406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb61be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb6140ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb61be00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb61406ee70 con 0x7fb61415f9d0 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb61be00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb61406cef0 con 0x7fb614144c00 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb61be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb61406d1d0 con 0x7fb61415fe60 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb619400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb614144c00 0x7fb61406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb618a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb61415f9d0 0x7fb6140ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54768/0 (socket says 172.21.15.86:54768) 2024-03-06T07:28:51.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.450+0000 7fb618a00640 1 -- 172.21.15.86:0/2911317092 learned_addr learned my addr 172.21.15.86:0/2911317092 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:51.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.451+0000 7fb619e00640 1 --2- 172.21.15.86:0/2911317092 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb6140ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.451+0000 7fb619400640 1 -- 172.21.15.86:0/2911317092 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 msgr2=0x7fb6140ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:51.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.451+0000 7fb619400640 1 --2- 172.21.15.86:0/2911317092 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb6140ce7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.451+0000 7fb619400640 1 -- 172.21.15.86:0/2911317092 >> v1:172.21.15.86:6789/0 conn(0x7fb61415f9d0 legacy=0x7fb6140ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.451+0000 7fb619400640 1 -- 172.21.15.86:0/2911317092 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb61406d4b0 con 0x7fb614144c00 2024-03-06T07:28:51.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.451+0000 7fb613e00640 1 -- 172.21.15.86:0/2911317092 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 963260324 0 0) 0x7fb600001110 con 0x7fb61415f9d0 2024-03-06T07:28:51.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.452+0000 7fb619400640 1 --2- 172.21.15.86:0/2911317092 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb614144c00 0x7fb61406e930 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7fb6080097a0 tx=0x7fb60802b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9667f7e8adec4a2d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:51.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.452+0000 7fb613e00640 1 -- 172.21.15.86:0/2911317092 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb608035020 con 0x7fb614144c00 2024-03-06T07:28:51.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.452+0000 7fb613e00640 1 -- 172.21.15.86:0/2911317092 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb608031810 con 0x7fb614144c00 2024-03-06T07:28:51.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.452+0000 7fb613e00640 1 -- 172.21.15.86:0/2911317092 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb608031be0 con 0x7fb614144c00 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 -- 172.21.15.86:0/2911317092 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb614144c00 msgr2=0x7fb61406e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 --2- 172.21.15.86:0/2911317092 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb614144c00 0x7fb61406e930 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7fb6080097a0 tx=0x7fb60802b8b0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 -- 172.21.15.86:0/2911317092 shutdown_connections 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 --2- 172.21.15.86:0/2911317092 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb6140ce7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 --2- 172.21.15.86:0/2911317092 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb614144c00 0x7fb61406e930 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 -- 172.21.15.86:0/2911317092 >> 172.21.15.86:0/2911317092 conn(0x7fb6140b1730 msgr2=0x7fb6140b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 -- 172.21.15.86:0/2911317092 shutdown_connections 2024-03-06T07:28:51.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.453+0000 7fb61be00640 1 -- 172.21.15.86:0/2911317092 wait complete. 2024-03-06T07:28:51.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.454+0000 7fb61be00640 1 Processor -- start 2024-03-06T07:28:51.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.455+0000 7fb61be00640 1 -- start start 2024-03-06T07:28:51.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb61be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb614144c00 0x7fb6141392b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb61be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb61415f9d0 0x7fb61413d440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb61be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb61413c090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb61be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb614160730 con 0x7fb61415f9d0 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb61be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb614160290 con 0x7fb614144c00 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb61be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb61406cb50 con 0x7fb61415fe60 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb61415f9d0 0x7fb61413d440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb61415f9d0 0x7fb61413d440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57810/0 (socket says 172.21.15.86:57810) 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 -- 172.21.15.86:0/2024006524 learned_addr learned my addr 172.21.15.86:0/2024006524 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb619400640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb614144c00 0x7fb6141392b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 -- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 msgr2=0x7fb61413c090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb619e00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb61413c090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb61413c090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 -- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb614144c00 msgr2=0x7fb6141392b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb614144c00 0x7fb6141392b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:51.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.456+0000 7fb618a00640 1 -- 172.21.15.86:0/2024006524 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb614140790 con 0x7fb61415f9d0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.457+0000 7fb618a00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb61415f9d0 0x7fb61413d440 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fb60000c780 tx=0x7fb60000cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.457+0000 7fb611600640 1 -- 172.21.15.86:0/2024006524 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb600018070 con 0x7fb61415f9d0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.457+0000 7fb611600640 1 -- 172.21.15.86:0/2024006524 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6000072d0 con 0x7fb61415f9d0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.457+0000 7fb611600640 1 -- 172.21.15.86:0/2024006524 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6000047e0 con 0x7fb61415f9d0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.457+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6141401d0 con 0x7fb61415f9d0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.457+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb61414d450 con 0x7fb61415f9d0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.458+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb61404f330 con 0x7fb61415f9d0 2024-03-06T07:28:51.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.459+0000 7fb611600640 1 -- 172.21.15.86:0/2024006524 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb600004980 con 0x7fb61415f9d0 2024-03-06T07:28:51.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.460+0000 7fb611600640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5f406b4d0 0x7fb5f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:51.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.461+0000 7fb611600640 1 -- 172.21.15.86:0/2024006524 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fb600089240 con 0x7fb61415f9d0 2024-03-06T07:28:51.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.461+0000 7fb619400640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5f406b4d0 0x7fb5f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:51.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.461+0000 7fb619400640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5f406b4d0 0x7fb5f406d990 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fb608036000 tx=0x7fb608008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:51.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.465+0000 7fb611600640 1 -- 172.21.15.86:0/2024006524 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb61406ee70 con 0x7fb61415f9d0 2024-03-06T07:28:51.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:51.732+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]} v 0) v1 -- 0x7fb614072890 con 0x7fb61415f9d0 2024-03-06T07:28:52.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.271+0000 7fb611600640 1 -- 172.21.15.86:0/2024006524 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]}]=0 v117) v1 ==== 136+0+0 (secure 0 0 0) 0x7fb600058450 con 0x7fb61415f9d0 2024-03-06T07:28:52.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5f406b4d0 msgr2=0x7fb5f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5f406b4d0 0x7fb5f406d990 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fb608036000 tx=0x7fb608008030 comp rx=0 tx=0).stop 2024-03-06T07:28:52.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb61415f9d0 msgr2=0x7fb61413d440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb61415f9d0 0x7fb61413d440 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fb60000c780 tx=0x7fb60000cc40 comp rx=0 tx=0).stop 2024-03-06T07:28:52.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 shutdown_connections 2024-03-06T07:28:52.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5f406b4d0 0x7fb5f406d990 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb61415fe60 0x7fb61413c090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb61415f9d0 0x7fb61413d440 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 --2- 172.21.15.86:0/2024006524 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb614144c00 0x7fb6141392b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 >> 172.21.15.86:0/2024006524 conn(0x7fb6140b1730 msgr2=0x7fb6140cd0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:52.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.275+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 shutdown_connections 2024-03-06T07:28:52.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.276+0000 7fb61be00640 1 -- 172.21.15.86:0/2024006524 wait complete. 2024-03-06T07:28:52.300 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:28:52.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.510+0000 7f46c4200640 1 Processor -- start 2024-03-06T07:28:52.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.510+0000 7f46c4200640 1 -- start start 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.511+0000 7f46c4200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f46bc0cdd70 0x7f46bc06d170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.511+0000 7f46c4200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc141850 0x7f46bc06d6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.511+0000 7f46c4200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f46bc15e730 con 0x7f46bc15bdc0 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.511+0000 7f46c4200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46bc15ea10 con 0x7f46bc0cdd70 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.511+0000 7f46c4200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46bc15ecf0 con 0x7f46bc141850 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c0e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f46bc0cdd70 0x7f46bc06d170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c0e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f46bc0cdd70 0x7f46bc06d170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57814/0 (socket says 172.21.15.86:57814) 2024-03-06T07:28:52.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c0e00640 1 -- 172.21.15.86:0/329938731 learned_addr learned my addr 172.21.15.86:0/329938731 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:52.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c2200640 1 --2- 172.21.15.86:0/329938731 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc141850 0x7f46bc06d6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:52.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c0e00640 1 -- 172.21.15.86:0/329938731 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc141850 msgr2=0x7f46bc06d6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:52.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c0e00640 1 --2- 172.21.15.86:0/329938731 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc141850 0x7f46bc06d6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c0e00640 1 -- 172.21.15.86:0/329938731 >> v1:172.21.15.86:6789/0 conn(0x7f46bc15bdc0 legacy=0x7f46bc0cd660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46c0e00640 1 -- 172.21.15.86:0/329938731 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46bc15efd0 con 0x7f46bc0cdd70 2024-03-06T07:28:52.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.512+0000 7f46bbe00640 1 -- 172.21.15.86:0/329938731 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 474802612 0 0) 0x7f46bc15e730 con 0x7f46bc15bdc0 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.513+0000 7f46c0e00640 1 --2- 172.21.15.86:0/329938731 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc0cdd70 0x7f46bc06d170 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7f46b0009870 tx=0x7f46b002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f7886cdb5d2799fe server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46bbe00640 1 -- 172.21.15.86:0/329938731 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46b0034070 con 0x7f46bc0cdd70 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46bbe00640 1 -- 172.21.15.86:0/329938731 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46b0030840 con 0x7f46bc0cdd70 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46bbe00640 1 -- 172.21.15.86:0/329938731 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46b0030c10 con 0x7f46bc0cdd70 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 -- 172.21.15.86:0/329938731 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc0cdd70 msgr2=0x7f46bc06d170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 --2- 172.21.15.86:0/329938731 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc0cdd70 0x7f46bc06d170 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7f46b0009870 tx=0x7f46b002b980 comp rx=0 tx=0).stop 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 -- 172.21.15.86:0/329938731 shutdown_connections 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 --2- 172.21.15.86:0/329938731 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc141850 0x7f46bc06d6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 --2- 172.21.15.86:0/329938731 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc0cdd70 0x7f46bc06d170 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 -- 172.21.15.86:0/329938731 >> 172.21.15.86:0/329938731 conn(0x7f46bc0b2890 msgr2=0x7f46bc0ccf50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:52.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 -- 172.21.15.86:0/329938731 shutdown_connections 2024-03-06T07:28:52.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.514+0000 7f46c4200640 1 -- 172.21.15.86:0/329938731 wait complete. 2024-03-06T07:28:52.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.516+0000 7f46c4200640 1 Processor -- start 2024-03-06T07:28:52.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.516+0000 7f46c4200640 1 -- start start 2024-03-06T07:28:52.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c4200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46bc0cdd70 0x7f46bc1f4600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c4200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc141850 0x7f46bc140c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c4200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc143f40 0x7f46bc0ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c4200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46bc1439e0 con 0x7f46bc141850 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c4200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46bc06dbf0 con 0x7f46bc0cdd70 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c4200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46bc143b60 con 0x7f46bc143f40 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc141850 0x7f46bc140c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc141850 0x7f46bc140c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57818/0 (socket says 172.21.15.86:57818) 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 -- 172.21.15.86:0/1497611294 learned_addr learned my addr 172.21.15.86:0/1497611294 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 -- 172.21.15.86:0/1497611294 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc143f40 msgr2=0x7f46bc0ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc143f40 0x7f46bc0ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 -- 172.21.15.86:0/1497611294 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46bc0cdd70 msgr2=0x7f46bc1f4600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46bc0cdd70 0x7f46bc1f4600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.517+0000 7f46c0e00640 1 -- 172.21.15.86:0/1497611294 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46bc0b01b0 con 0x7f46bc141850 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.518+0000 7f46c0e00640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc141850 0x7f46bc140c10 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f46b0000c00 tx=0x7f46b00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.518+0000 7f46b9600640 1 -- 172.21.15.86:0/1497611294 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46b0034040 con 0x7f46bc141850 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.518+0000 7f46b9600640 1 -- 172.21.15.86:0/1497611294 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46b0039b00 con 0x7f46bc141850 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.518+0000 7f46b9600640 1 -- 172.21.15.86:0/1497611294 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46b0004030 con 0x7f46bc141850 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.518+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46bc144490 con 0x7f46bc141850 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.518+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46bc144a50 con 0x7f46bc141850 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.519+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f46bc04f330 con 0x7f46bc141850 2024-03-06T07:28:52.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.520+0000 7f46b9600640 1 -- 172.21.15.86:0/1497611294 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f46b004b060 con 0x7f46bc141850 2024-03-06T07:28:52.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.521+0000 7f46b9600640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f469c06b4d0 0x7f469c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:52.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.521+0000 7f46b9600640 1 -- 172.21.15.86:0/1497611294 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f46b00abe60 con 0x7f46bc141850 2024-03-06T07:28:52.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.521+0000 7f46c1800640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f469c06b4d0 0x7f469c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:52.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.522+0000 7f46c1800640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f469c06b4d0 0x7f469c06d990 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f46bc0cda80 tx=0x7f46a8007930 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:52.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.526+0000 7f46b9600640 1 -- 172.21.15.86:0/1497611294 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f46bc0b01b0 con 0x7f46bc141850 2024-03-06T07:28:52.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.721+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f46bc06c4b0 con 0x7f46bc141850 2024-03-06T07:28:52.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.727+0000 7f46b9600640 1 -- 172.21.15.86:0/1497611294 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v12)=0 v12) v1 ==== 122+0+0 (secure 0 0 0) 0x7f46b004ce90 con 0x7f46bc141850 2024-03-06T07:28:52.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f469c06b4d0 msgr2=0x7f469c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f469c06b4d0 0x7f469c06d990 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f46bc0cda80 tx=0x7f46a8007930 comp rx=0 tx=0).stop 2024-03-06T07:28:52.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc141850 msgr2=0x7f46bc140c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:52.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc141850 0x7f46bc140c10 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f46b0000c00 tx=0x7f46b00046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 shutdown_connections 2024-03-06T07:28:52.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f469c06b4d0 0x7f469c06d990 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46bc143f40 0x7f46bc0ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46bc141850 0x7f46bc140c10 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 --2- 172.21.15.86:0/1497611294 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46bc0cdd70 0x7f46bc1f4600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:52.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 >> 172.21.15.86:0/1497611294 conn(0x7f46bc0b2890 msgr2=0x7f46bc06afc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:52.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 shutdown_connections 2024-03-06T07:28:52.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:52.730+0000 7f46c4200640 1 -- 172.21.15.86:0/1497611294 wait complete. 2024-03-06T07:28:52.751 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:28:52.752 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:28:52.839 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:28:52.844 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:28:52.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:28:52.996 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:28:52.996 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:28:52.996 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:53.055 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:28:53.055 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:28:53.055 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:53.108 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:28:53.109 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:28:53.109 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:53.165 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:28:53.165 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:28:53.165 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:28:53.220 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:28:53.220 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:28:53.220 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:28:53.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.433+0000 7f1101400640 1 Processor -- start 2024-03-06T07:28:53.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.433+0000 7f1101400640 1 -- start start 2024-03-06T07:28:53.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.435+0000 7f1101400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 0x7f10fc0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.435+0000 7f1101400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f10fc15b090 con 0x7f10fc0b4680 2024-03-06T07:28:53.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.435+0000 7f1101400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f10fc0b2a30 con 0x7f10fc06ec00 2024-03-06T07:28:53.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.435+0000 7f1101400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10fc0b4360 con 0x7f10fc15acb0 2024-03-06T07:28:53.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.435+0000 7f10f1e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f10fc0b4680 0x7f10fc06e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54800/0 (socket says 172.21.15.86:54800) 2024-03-06T07:28:53.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.435+0000 7f10f1e00640 1 -- 172.21.15.86:0/11451234 learned_addr learned my addr 172.21.15.86:0/11451234 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:53.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4004922967 0 0) 0x7f10fc15b090 con 0x7f10fc0b4680 2024-03-06T07:28:53.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f10e0005730 con 0x7f10fc0b4680 2024-03-06T07:28:53.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10fa800640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 0x7f10fc0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 303230614 0 0) 0x7f10e0005730 con 0x7f10fc0b4680 2024-03-06T07:28:53.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 msgr2=0x7f10fc0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 0x7f10fc0b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 >> v1:172.21.15.194:6789/0 conn(0x7f10fc06ec00 legacy=0x7f10fc0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10fc0b2cf0 con 0x7f10fc0b4680 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f10e4001670 con 0x7f10fc0b4680 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 >> v1:172.21.15.86:6789/0 conn(0x7f10fc0b4680 legacy=0x7f10fc06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10e0005bf0 0x7f10e0006010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 0x7f10e0006a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10e0006f50 0x7f10e00073d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10fc0b4360 con 0x7f10e0005bf0 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10e000dd60 con 0x7f10e0006f50 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10e000df30 con 0x7f10fc15acb0 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.436+0000 7f10fa800640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10e0005bf0 0x7f10e0006010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10fb200640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10e0006f50 0x7f10e00073d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10f1e00640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 0x7f10e0006a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10fa800640 1 -- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 msgr2=0x7f10e0006a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10fa800640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10fc15acb0 0x7f10e0006a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10fa800640 1 -- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10e0006f50 msgr2=0x7f10e00073d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10fa800640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10e0006f50 0x7f10e00073d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10fa800640 1 -- 172.21.15.86:0/11451234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10e0005730 con 0x7f10e0005bf0 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.437+0000 7f10fb200640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10e0006f50 0x7f10e00073d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:53.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.438+0000 7f10fa800640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10e0005bf0 0x7f10e0006010 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7f10ec007390 tx=0x7f10e0007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=77023a008a54c7f6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:53.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.438+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10ec039070 con 0x7f10e0005bf0 2024-03-06T07:28:53.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.438+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f10ec02ccb0 con 0x7f10e0005bf0 2024-03-06T07:28:53.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.438+0000 7f10f9e00640 1 -- 172.21.15.86:0/11451234 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10ec009be0 con 0x7f10e0005bf0 2024-03-06T07:28:53.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.439+0000 7f1101400640 1 -- 172.21.15.86:0/11451234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10e0005bf0 msgr2=0x7f10e0006010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.439+0000 7f1101400640 1 --2- 172.21.15.86:0/11451234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10e0005bf0 0x7f10e0006010 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7f10ec007390 tx=0x7f10e0007a40 comp rx=0 tx=0).stop 2024-03-06T07:28:53.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.439+0000 7f10fb200640 1 -- 172.21.15.86:0/11451234 reap_dead start 2024-03-06T07:28:53.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.439+0000 7f1101400640 1 -- 172.21.15.86:0/11451234 shutdown_connections 2024-03-06T07:28:53.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.439+0000 7f1101400640 1 -- 172.21.15.86:0/11451234 >> 172.21.15.86:0/11451234 conn(0x7f10fc0b1730 msgr2=0x7f10fc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:53.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.439+0000 7f1101400640 1 -- 172.21.15.86:0/11451234 shutdown_connections 2024-03-06T07:28:53.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.439+0000 7f1101400640 1 -- 172.21.15.86:0/11451234 wait complete. 2024-03-06T07:28:53.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.441+0000 7f1101400640 1 Processor -- start 2024-03-06T07:28:53.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.441+0000 7f1101400640 1 -- start start 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f1101400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10e0005bf0 0x7f10fc156050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f1101400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10fc147850 0x7f10fc15dd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f1101400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10fc153bd0 0x7f10fc159260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f1101400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10fc0b4010 con 0x7f10fc153bd0 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f1101400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10fc0b32b0 con 0x7f10fc147850 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f1101400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10fc072340 con 0x7f10e0005bf0 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f10fb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10fc153bd0 0x7f10fc159260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f10fb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10fc153bd0 0x7f10fc159260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57826/0 (socket says 172.21.15.86:57826) 2024-03-06T07:28:53.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.442+0000 7f10fb200640 1 -- 172.21.15.86:0/1289554056 learned_addr learned my addr 172.21.15.86:0/1289554056 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10fa800640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10e0005bf0 0x7f10fc156050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10f1e00640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10fc147850 0x7f10fc15dd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10fb200640 1 -- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10e0005bf0 msgr2=0x7f10fc156050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10fb200640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10e0005bf0 0x7f10fc156050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10fb200640 1 -- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10fc147850 msgr2=0x7f10fc15dd20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10fb200640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10fc147850 0x7f10fc15dd20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10fb200640 1 -- 172.21.15.86:0/1289554056 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10fc154420 con 0x7f10fc153bd0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10fb200640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10fc153bd0 0x7f10fc159260 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f10fc068580 tx=0x7f10f400d890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10f3400640 1 -- 172.21.15.86:0/1289554056 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10f4007030 con 0x7f10fc153bd0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10f3400640 1 -- 172.21.15.86:0/1289554056 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f10f40071d0 con 0x7f10fc153bd0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f10f3400640 1 -- 172.21.15.86:0/1289554056 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10f4007600 con 0x7f10fc153bd0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.443+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10fc146d70 con 0x7f10fc153bd0 2024-03-06T07:28:53.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.444+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10fc147330 con 0x7f10fc153bd0 2024-03-06T07:28:53.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.445+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f10fc04f330 con 0x7f10fc153bd0 2024-03-06T07:28:53.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.446+0000 7f10f3400640 1 -- 172.21.15.86:0/1289554056 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f10f401b020 con 0x7f10fc153bd0 2024-03-06T07:28:53.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.446+0000 7f10f3400640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f10d806b4d0 0x7f10d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.447+0000 7f10f3400640 1 -- 172.21.15.86:0/1289554056 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f10f4089890 con 0x7f10fc153bd0 2024-03-06T07:28:53.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.447+0000 7f10fa800640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f10d806b4d0 0x7f10d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.447+0000 7f10fa800640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f10d806b4d0 0x7f10d806d990 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f10ec036000 tx=0x7f10ec030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:53.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.451+0000 7f10f3400640 1 -- 172.21.15.86:0/1289554056 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f10f4058aa0 con 0x7f10fc153bd0 2024-03-06T07:28:53.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.689+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f10fc06ce60 con 0x7f10fc153bd0 2024-03-06T07:28:53.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.690+0000 7f10f3400640 1 -- 172.21.15.86:0/1289554056 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 19 v19) v1 ==== 94+0+3705 (secure 0 0 0) 0x7f10f40583a0 con 0x7f10fc153bd0 2024-03-06T07:28:53.693 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:53.693 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":19,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4780,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/3969706704","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3969706704},{"type":"v1","addr":"172.21.15.194:6834","nonce":3969706704}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4977,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1789084811","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1789084811},{"type":"v1","addr":"172.21.15.86:6836","nonce":1789084811}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":4980,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3912588369","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3912588369},{"type":"v1","addr":"172.21.15.86:6837","nonce":3912588369}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":12},{"gid":4990,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1180814764","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1180814764},{"type":"v1","addr":"172.21.15.194:6835","nonce":1180814764}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19}],"filesystems":[]} 2024-03-06T07:28:53.693 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 19 2024-03-06T07:28:53.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.692+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f10d806b4d0 msgr2=0x7f10d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f10d806b4d0 0x7f10d806d990 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f10ec036000 tx=0x7f10ec030000 comp rx=0 tx=0).stop 2024-03-06T07:28:53.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10fc153bd0 msgr2=0x7f10fc159260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10fc153bd0 0x7f10fc159260 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f10fc068580 tx=0x7f10f400d890 comp rx=0 tx=0).stop 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 shutdown_connections 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f10d806b4d0 0x7f10d806d990 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10fc153bd0 0x7f10fc159260 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f10fc147850 0x7f10fc15dd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 --2- 172.21.15.86:0/1289554056 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10e0005bf0 0x7f10fc156050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 >> 172.21.15.86:0/1289554056 conn(0x7f10fc0b1730 msgr2=0x7f10fc0ccf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 shutdown_connections 2024-03-06T07:28:53.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.693+0000 7f1101400640 1 -- 172.21.15.86:0/1289554056 wait complete. 2024-03-06T07:28:53.713 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:28:53.713 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:28:53.713 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:28:53.713 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:53.713 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:28:53.713 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:28:53.714 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:28:53.714 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:53.714 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:28:53.714 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:28:53.714 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:28:53.714 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:53.714 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:28:53.714 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:28:53.715 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:28:53.715 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:28:53.732 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:28:53.732 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:28:53.733 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:28:53.734 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:28:53.736 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:28:53.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:28:53.743 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:28:53.743 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:28:53.780 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:28:53.780 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:53.786 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:28:53.786 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:53.790 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:28:53.790 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:53.796 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:28:53.796 DEBUG:teuthology.parallel:result is None 2024-03-06T07:28:53.796 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:28:53.934 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:28:53.938 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:28:53.951 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:28:53.958 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:28:53.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.973+0000 7f8d36800640 1 Processor -- start 2024-03-06T07:28:53.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.974+0000 7f8d36800640 1 -- start start 2024-03-06T07:28:53.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d36800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8d3015ac90 0x7f8d3006e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d36800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8d300cc000 con 0x7f8d3015b070 2024-03-06T07:28:53.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d36800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8d300ce360 con 0x7f8d3015f120 2024-03-06T07:28:53.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d36800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8d300ce640 con 0x7f8d3015ac90 2024-03-06T07:28:53.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d2ea00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8d3015b070 0x7f8d3015ea10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54822/0 (socket says 172.21.15.86:54822) 2024-03-06T07:28:53.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d2ea00640 1 -- 172.21.15.86:0/1090958302 learned_addr learned my addr 172.21.15.86:0/1090958302 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:53.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d2f400640 1 --2- 172.21.15.86:0/1090958302 >> v2:172.21.15.194:3300/0 conn(0x7f8d3015ac90 0x7f8d3006e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d2e000640 1 -- 172.21.15.86:0/1090958302 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4285499728 0 0) 0x7f8d300cc000 con 0x7f8d3015b070 2024-03-06T07:28:53.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.975+0000 7f8d2e000640 1 -- 172.21.15.86:0/1090958302 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8d14003520 con 0x7f8d3015b070 2024-03-06T07:28:53.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.976+0000 7f8d2e000640 1 -- 172.21.15.86:0/1090958302 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3119654618 0 0) 0x7f8d14003520 con 0x7f8d3015b070 2024-03-06T07:28:53.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.976+0000 7f8d2f400640 1 -- 172.21.15.86:0/1090958302 >> v1:172.21.15.194:6789/0 conn(0x7f8d3015f120 legacy=0x7f8d300c4a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.976+0000 7f8d2f400640 1 -- 172.21.15.86:0/1090958302 >> v1:172.21.15.86:6789/0 conn(0x7f8d3015b070 legacy=0x7f8d3015ea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.976+0000 7f8d2f400640 1 -- 172.21.15.86:0/1090958302 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d300ce920 con 0x7f8d3015ac90 2024-03-06T07:28:53.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.977+0000 7f8d2f400640 1 --2- 172.21.15.86:0/1090958302 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3006e580 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f8d200097a0 tx=0x7f8d2002b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3aa2fe7a5b594d4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:53.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.977+0000 7f8d2e000640 1 -- 172.21.15.86:0/1090958302 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d20037070 con 0x7f8d3015ac90 2024-03-06T07:28:53.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.977+0000 7f8d2e000640 1 -- 172.21.15.86:0/1090958302 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8d20030510 con 0x7f8d3015ac90 2024-03-06T07:28:53.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.977+0000 7f8d2e000640 1 -- 172.21.15.86:0/1090958302 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d1c001670 con 0x7f8d3015ac90 2024-03-06T07:28:53.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.978+0000 7f8d36800640 1 -- 172.21.15.86:0/1090958302 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 msgr2=0x7f8d3006e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.978+0000 7f8d36800640 1 --2- 172.21.15.86:0/1090958302 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3006e580 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f8d200097a0 tx=0x7f8d2002b790 comp rx=0 tx=0).stop 2024-03-06T07:28:53.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.978+0000 7f8d36800640 1 -- 172.21.15.86:0/1090958302 shutdown_connections 2024-03-06T07:28:53.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.978+0000 7f8d36800640 1 --2- 172.21.15.86:0/1090958302 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3006e580 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.978+0000 7f8d36800640 1 -- 172.21.15.86:0/1090958302 >> 172.21.15.86:0/1090958302 conn(0x7f8d300cc4f0 msgr2=0x7f8d300cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:53.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.978+0000 7f8d36800640 1 -- 172.21.15.86:0/1090958302 shutdown_connections 2024-03-06T07:28:53.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.978+0000 7f8d36800640 1 -- 172.21.15.86:0/1090958302 wait complete. 2024-03-06T07:28:53.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.980+0000 7f8d36800640 1 Processor -- start 2024-03-06T07:28:53.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.980+0000 7f8d36800640 1 -- start start 2024-03-06T07:28:53.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d36800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3015cbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d36800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d3015b070 0x7f8d3015e9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d36800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d3015f120 0x7f8d3015bf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d36800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d3015f5b0 con 0x7f8d3015f120 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d36800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d300ceee0 con 0x7f8d3015ac90 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d36800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8d3015f730 con 0x7f8d3015b070 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d3015f120 0x7f8d3015bf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d3015f120 0x7f8d3015bf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57888/0 (socket says 172.21.15.86:57888) 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 -- 172.21.15.86:0/1611524197 learned_addr learned my addr 172.21.15.86:0/1611524197 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2ea00640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d3015b070 0x7f8d3015e9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2f400640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3015cbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 -- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d3015b070 msgr2=0x7f8d3015e9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d3015b070 0x7f8d3015e9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 -- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 msgr2=0x7f8d3015cbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3015cbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:53.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2fe00640 1 -- 172.21.15.86:0/1611524197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d301f3100 con 0x7f8d3015f120 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2f400640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3015cbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.981+0000 7f8d2ea00640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d3015b070 0x7f8d3015e9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.982+0000 7f8d2fe00640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d3015f120 0x7f8d3015bf00 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f8d2800c730 tx=0x7f8d2800cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.982+0000 7f8d27400640 1 -- 172.21.15.86:0/1611524197 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d28003b70 con 0x7f8d3015f120 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.982+0000 7f8d27400640 1 -- 172.21.15.86:0/1611524197 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8d28003d30 con 0x7f8d3015f120 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.982+0000 7f8d27400640 1 -- 172.21.15.86:0/1611524197 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d28005310 con 0x7f8d3015f120 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.982+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8d301f33e0 con 0x7f8d3015f120 2024-03-06T07:28:53.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.982+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8d301f39a0 con 0x7f8d3015f120 2024-03-06T07:28:53.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.983+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8d3004f330 con 0x7f8d3015f120 2024-03-06T07:28:53.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.984+0000 7f8d27400640 1 -- 172.21.15.86:0/1611524197 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8d280054b0 con 0x7f8d3015f120 2024-03-06T07:28:53.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.985+0000 7f8d27400640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d1006b4d0 0x7f8d1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:53.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.986+0000 7f8d27400640 1 -- 172.21.15.86:0/1611524197 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(38..38 src has 1..38) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f8d28089d90 con 0x7f8d3015f120 2024-03-06T07:28:53.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.986+0000 7f8d2f400640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d1006b4d0 0x7f8d1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:53.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.986+0000 7f8d2f400640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d1006b4d0 0x7f8d1006d990 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f8d20034000 tx=0x7f8d2002f000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:53.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:53.990+0000 7f8d27400640 1 -- 172.21.15.86:0/1611524197 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8d28058f00 con 0x7f8d3015f120 2024-03-06T07:28:54.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:54.188+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f8d300cd1c0 con 0x7f8d3015f120 2024-03-06T07:28:55.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.315+0000 7f8d27400640 1 -- 172.21.15.86:0/1611524197 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v43) v1 ==== 97+0+0 (secure 0 0 0) 0x7f8d28058800 con 0x7f8d3015f120 2024-03-06T07:28:55.318 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.319+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d1006b4d0 msgr2=0x7f8d1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.319+0000 7f8d36800640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d1006b4d0 0x7f8d1006d990 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f8d20034000 tx=0x7f8d2002f000 comp rx=0 tx=0).stop 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.319+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d3015f120 msgr2=0x7f8d3015bf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.319+0000 7f8d36800640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d3015f120 0x7f8d3015bf00 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f8d2800c730 tx=0x7f8d2800cbf0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 shutdown_connections 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8d1006b4d0 0x7f8d1006d990 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8d3015f120 0x7f8d3015bf00 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8d3015b070 0x7f8d3015e9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 --2- 172.21.15.86:0/1611524197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8d3015ac90 0x7f8d3015cbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 >> 172.21.15.86:0/1611524197 conn(0x7f8d300cc4f0 msgr2=0x7f8d300b0530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 shutdown_connections 2024-03-06T07:28:55.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.320+0000 7f8d36800640 1 -- 172.21.15.86:0/1611524197 wait complete. 2024-03-06T07:28:55.341 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:28:55.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.547+0000 7fdd22600640 1 Processor -- start 2024-03-06T07:28:55.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.547+0000 7fdd22600640 1 -- start start 2024-03-06T07:28:55.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd22600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdd1c0b4680 0x7fdd1c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:55.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd22600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c072340 0x7fdd1c06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:55.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd22600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdd1c15b090 con 0x7fdd1c15acb0 2024-03-06T07:28:55.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd22600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdd1c0b2a30 con 0x7fdd1c0b4680 2024-03-06T07:28:55.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd22600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd1c0b4360 con 0x7fdd1c072340 2024-03-06T07:28:55.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd1aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdd1c0b4680 0x7fdd1c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd1b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fdd1c15acb0 0x7fdd1c0b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54836/0 (socket says 172.21.15.86:54836) 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd1aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdd1c0b4680 0x7fdd1c06e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57902/0 (socket says 172.21.15.86:57902) 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.549+0000 7fdd1b400640 1 -- 172.21.15.86:0/985218187 learned_addr learned my addr 172.21.15.86:0/985218187 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.550+0000 7fdd1be00640 1 --2- 172.21.15.86:0/985218187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c072340 0x7fdd1c06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.550+0000 7fdd1aa00640 1 -- 172.21.15.86:0/985218187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c072340 msgr2=0x7fdd1c06ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.550+0000 7fdd1aa00640 1 --2- 172.21.15.86:0/985218187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c072340 0x7fdd1c06ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.550+0000 7fdd1aa00640 1 -- 172.21.15.86:0/985218187 >> v1:172.21.15.86:6789/0 conn(0x7fdd1c15acb0 legacy=0x7fdd1c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.550+0000 7fdd1aa00640 1 -- 172.21.15.86:0/985218187 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd1c0b2cf0 con 0x7fdd1c0b4680 2024-03-06T07:28:55.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.551+0000 7fdd1aa00640 1 --2- 172.21.15.86:0/985218187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c06e4f0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7fdd080097a0 tx=0x7fdd0802b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7c32434b9ef91f9c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:55.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.551+0000 7fdd1a000640 1 -- 172.21.15.86:0/985218187 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd08035020 con 0x7fdd1c0b4680 2024-03-06T07:28:55.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.551+0000 7fdd1a000640 1 -- 172.21.15.86:0/985218187 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd08031810 con 0x7fdd1c0b4680 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.551+0000 7fdd1a000640 1 -- 172.21.15.86:0/985218187 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd08031be0 con 0x7fdd1c0b4680 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 -- 172.21.15.86:0/985218187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 msgr2=0x7fdd1c06e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 --2- 172.21.15.86:0/985218187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c06e4f0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7fdd080097a0 tx=0x7fdd0802b8b0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 -- 172.21.15.86:0/985218187 shutdown_connections 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 --2- 172.21.15.86:0/985218187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c072340 0x7fdd1c06ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 --2- 172.21.15.86:0/985218187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c06e4f0 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 -- 172.21.15.86:0/985218187 >> 172.21.15.86:0/985218187 conn(0x7fdd1c0b1730 msgr2=0x7fdd1c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 -- 172.21.15.86:0/985218187 shutdown_connections 2024-03-06T07:28:55.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.552+0000 7fdd22600640 1 -- 172.21.15.86:0/985218187 wait complete. 2024-03-06T07:28:55.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.553+0000 7fdd22600640 1 Processor -- start 2024-03-06T07:28:55.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.553+0000 7fdd22600640 1 -- start start 2024-03-06T07:28:55.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.554+0000 7fdd22600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd1c072340 0x7fdd1c159430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:55.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.554+0000 7fdd22600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c14ffa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:55.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.554+0000 7fdd22600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c15acb0 0x7fdd1c0b0910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.554+0000 7fdd22600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd1c0b4010 con 0x7fdd1c0b4680 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.554+0000 7fdd22600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd1c0b32b0 con 0x7fdd1c072340 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.554+0000 7fdd22600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd1c0ccfb0 con 0x7fdd1c15acb0 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c14ffa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c14ffa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57910/0 (socket says 172.21.15.86:57910) 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 -- 172.21.15.86:0/186636880 learned_addr learned my addr 172.21.15.86:0/186636880 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 -- 172.21.15.86:0/186636880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c15acb0 msgr2=0x7fdd1c0b0910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c15acb0 0x7fdd1c0b0910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 -- 172.21.15.86:0/186636880 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd1c072340 msgr2=0x7fdd1c159430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd1c072340 0x7fdd1c159430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 -- 172.21.15.86:0/186636880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd1c15b260 con 0x7fdd1c0b4680 2024-03-06T07:28:55.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.555+0000 7fdd1aa00640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c14ffa0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7fdd08009890 tx=0x7fdd080046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:55.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.556+0000 7fdd13400640 1 -- 172.21.15.86:0/186636880 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd08041070 con 0x7fdd1c0b4680 2024-03-06T07:28:55.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.556+0000 7fdd13400640 1 -- 172.21.15.86:0/186636880 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd0803f070 con 0x7fdd1c0b4680 2024-03-06T07:28:55.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.556+0000 7fdd13400640 1 -- 172.21.15.86:0/186636880 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd08035040 con 0x7fdd1c0b4680 2024-03-06T07:28:55.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.556+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdd1c1504e0 con 0x7fdd1c0b4680 2024-03-06T07:28:55.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.556+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdd1c15b7c0 con 0x7fdd1c0b4680 2024-03-06T07:28:55.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.557+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdd1c04f330 con 0x7fdd1c0b4680 2024-03-06T07:28:55.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.558+0000 7fdd13400640 1 -- 172.21.15.86:0/186636880 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdd08038020 con 0x7fdd1c0b4680 2024-03-06T07:28:55.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.558+0000 7fdd13400640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdcfc06b4d0 0x7fdcfc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:55.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.559+0000 7fdd13400640 1 -- 172.21.15.86:0/186636880 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fdd080abcb0 con 0x7fdd1c0b4680 2024-03-06T07:28:55.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.559+0000 7fdd1b400640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdcfc06b4d0 0x7fdcfc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:55.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.559+0000 7fdd1b400640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdcfc06b4d0 0x7fdcfc06d990 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fdd0c0012e0 tx=0x7fdd0c001590 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:55.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.564+0000 7fdd13400640 1 -- 172.21.15.86:0/186636880 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdd1c15b260 con 0x7fdd1c0b4680 2024-03-06T07:28:55.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.807+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7fdd1c0b3a20 con 0x7fdd1c0b4680 2024-03-06T07:28:55.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.808+0000 7fdd13400640 1 -- 172.21.15.86:0/186636880 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v13) v1 ==== 80+0+5304 (secure 0 0 0) 0x7fdd0807af10 con 0x7fdd1c0b4680 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.811 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:28:55.812 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:55.813 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:28:55.814 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.815 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:28:55.816 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.817 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:28:55.818 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow r", 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:28:55.819 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.820 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:28:55.821 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:28:55.822 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:28:55.823 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.824 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:55.824 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:28:55.824 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:28:55.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdcfc06b4d0 msgr2=0x7fdcfc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdcfc06b4d0 0x7fdcfc06d990 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fdd0c0012e0 tx=0x7fdd0c001590 comp rx=0 tx=0).stop 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 msgr2=0x7fdd1c14ffa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c14ffa0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7fdd08009890 tx=0x7fdd080046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 shutdown_connections 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdcfc06b4d0 0x7fdcfc06d990 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd1c15acb0 0x7fdd1c0b0910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd1c0b4680 0x7fdd1c14ffa0 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 --2- 172.21.15.86:0/186636880 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd1c072340 0x7fdd1c159430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.810+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 >> 172.21.15.86:0/186636880 conn(0x7fdd1c0b1730 msgr2=0x7fdd1c071180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.811+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 shutdown_connections 2024-03-06T07:28:55.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:55.811+0000 7fdd22600640 1 -- 172.21.15.86:0/186636880 wait complete. 2024-03-06T07:28:55.832 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:28:56.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.037+0000 7f5410000640 1 Processor -- start 2024-03-06T07:28:56.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.037+0000 7f5410000640 1 -- start start 2024-03-06T07:28:56.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f5410000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5408144c00 0x7f540806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f5410000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f540815f9d0 0x7f540815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f5410000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f54080ce0a0 0x7f54080ce4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f5410000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5408160370 con 0x7f54080ce0a0 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f5410000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f540806ee70 con 0x7f5408144c00 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f5410000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f540806cef0 con 0x7f540815f9d0 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f540e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f54080ce0a0 0x7f54080ce4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f540e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f54080ce0a0 0x7f54080ce4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59688/0 (socket says 172.21.15.86:59688) 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f540e000640 1 -- 172.21.15.86:0/2075177007 learned_addr learned my addr 172.21.15.86:0/2075177007 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f540d600640 1 --2- 172.21.15.86:0/2075177007 >> v2:172.21.15.194:3300/0 conn(0x7f5408144c00 0x7f540806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.038+0000 7f540cc00640 1 --2- 172.21.15.86:0/2075177007 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f540815f9d0 0x7f540815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.039+0000 7f540e000640 1 -- 172.21.15.86:0/2075177007 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f540815f9d0 msgr2=0x7f540815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:56.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.039+0000 7f540e000640 1 --2- 172.21.15.86:0/2075177007 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f540815f9d0 0x7f540815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.039+0000 7f540e000640 1 -- 172.21.15.86:0/2075177007 >> v2:172.21.15.194:3300/0 conn(0x7f5408144c00 msgr2=0x7f540806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:56.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.039+0000 7f540e000640 1 --2- 172.21.15.86:0/2075177007 >> v2:172.21.15.194:3300/0 conn(0x7f5408144c00 0x7f540806e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.039+0000 7f540e000640 1 -- 172.21.15.86:0/2075177007 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f540806d1d0 con 0x7f54080ce0a0 2024-03-06T07:28:56.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.040+0000 7f540e000640 1 --2- 172.21.15.86:0/2075177007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f54080ce4f0 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f54080b25f0 tx=0x7f540002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7c720410d10ba7cf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.040+0000 7f5407e00640 1 -- 172.21.15.86:0/2075177007 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f540002ca80 con 0x7f54080ce0a0 2024-03-06T07:28:56.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.040+0000 7f5407e00640 1 -- 172.21.15.86:0/2075177007 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f540002cc20 con 0x7f54080ce0a0 2024-03-06T07:28:56.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.040+0000 7f5407e00640 1 -- 172.21.15.86:0/2075177007 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54000343f0 con 0x7f54080ce0a0 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 -- 172.21.15.86:0/2075177007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 msgr2=0x7f54080ce4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 --2- 172.21.15.86:0/2075177007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f54080ce4f0 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f54080b25f0 tx=0x7f540002bee0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 -- 172.21.15.86:0/2075177007 shutdown_connections 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 --2- 172.21.15.86:0/2075177007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f54080ce4f0 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 --2- 172.21.15.86:0/2075177007 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f540815f9d0 0x7f540815fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 --2- 172.21.15.86:0/2075177007 >> v2:172.21.15.194:3300/0 conn(0x7f5408144c00 0x7f540806e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 -- 172.21.15.86:0/2075177007 >> 172.21.15.86:0/2075177007 conn(0x7f54080b1730 msgr2=0x7f54080b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 -- 172.21.15.86:0/2075177007 shutdown_connections 2024-03-06T07:28:56.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.041+0000 7f5410000640 1 -- 172.21.15.86:0/2075177007 wait complete. 2024-03-06T07:28:56.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.043+0000 7f5410000640 1 Processor -- start 2024-03-06T07:28:56.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.043+0000 7f5410000640 1 -- start start 2024-03-06T07:28:56.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f5410000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f540813d080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f5410000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5408144c00 0x7f540813da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f5410000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f540815f9d0 0x7f540813fda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f5410000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5408160730 con 0x7f54080ce0a0 2024-03-06T07:28:56.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f5410000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f540806d790 con 0x7f540815f9d0 2024-03-06T07:28:56.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f5410000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f540806cb50 con 0x7f5408144c00 2024-03-06T07:28:56.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f540813d080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f540813d080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59704/0 (socket says 172.21.15.86:59704) 2024-03-06T07:28:56.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 -- 172.21.15.86:0/1788792415 learned_addr learned my addr 172.21.15.86:0/1788792415 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:56.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540cc00640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5408144c00 0x7f540813da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540e000640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f540815f9d0 0x7f540813fda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 -- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5408144c00 msgr2=0x7f540813da90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5408144c00 0x7f540813da90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 -- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f540815f9d0 msgr2=0x7f540813fda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f540815f9d0 0x7f540813fda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.044+0000 7f540d600640 1 -- 172.21.15.86:0/1788792415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f540813e120 con 0x7f54080ce0a0 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.045+0000 7f540d600640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f540813d080 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f53fc00d800 tx=0x7f53fc00dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.045+0000 7f5405600640 1 -- 172.21.15.86:0/1788792415 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53fc018020 con 0x7f54080ce0a0 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.045+0000 7f5405600640 1 -- 172.21.15.86:0/1788792415 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f53fc0096a0 con 0x7f54080ce0a0 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.045+0000 7f5405600640 1 -- 172.21.15.86:0/1788792415 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53fc009a70 con 0x7f54080ce0a0 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.045+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f54080ad900 con 0x7f54080ce0a0 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.045+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54080adec0 con 0x7f54080ce0a0 2024-03-06T07:28:56.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.046+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f540804f330 con 0x7f54080ce0a0 2024-03-06T07:28:56.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.047+0000 7f5405600640 1 -- 172.21.15.86:0/1788792415 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f53fc01b020 con 0x7f54080ce0a0 2024-03-06T07:28:56.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.048+0000 7f5405600640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f53e806b4d0 0x7f53e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.048+0000 7f5405600640 1 -- 172.21.15.86:0/1788792415 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f53fc08a4d0 con 0x7f54080ce0a0 2024-03-06T07:28:56.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.048+0000 7f540cc00640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f53e806b4d0 0x7f53e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.049+0000 7f540cc00640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f53e806b4d0 0x7f53e806d990 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f540813d5c0 tx=0x7f53f4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.054+0000 7f5405600640 1 -- 172.21.15.86:0/1788792415 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f53fc022020 con 0x7f54080ce0a0 2024-03-06T07:28:56.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.245+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f5408072890 con 0x7f54080ce0a0 2024-03-06T07:28:56.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.247+0000 7f5405600640 1 -- 172.21.15.86:0/1788792415 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v43) v1 ==== 74+0+12903 (secure 0 0 0) 0x7f53fc058fb0 con 0x7f54080ce0a0 2024-03-06T07:28:56.249 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:56.250 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":43,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:55.312971+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f53e806b4d0 msgr2=0x7f53e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f53e806b4d0 0x7f53e806d990 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f540813d5c0 tx=0x7f53f4008040 comp rx=0 tx=0).stop 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 msgr2=0x7f540813d080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f540813d080 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f53fc00d800 tx=0x7f53fc00dcc0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 shutdown_connections 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f53e806b4d0 0x7f53e806d990 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f540815f9d0 0x7f540813fda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5408144c00 0x7f540813da90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 --2- 172.21.15.86:0/1788792415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f54080ce0a0 0x7f540813d080 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 >> 172.21.15.86:0/1788792415 conn(0x7f54080b1730 msgr2=0x7f5408142a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:56.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 shutdown_connections 2024-03-06T07:28:56.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.250+0000 7f5410000640 1 -- 172.21.15.86:0/1788792415 wait complete. 2024-03-06T07:28:56.270 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:28:56.271 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:28:56.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.473+0000 7f3692000640 1 Processor -- start 2024-03-06T07:28:56.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.473+0000 7f3692000640 1 -- start start 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.473+0000 7f3692000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f368c15be10 0x7f368c0ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.473+0000 7f3692000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c0aef20 0x7f368c0ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3692000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f368c06d330 0x7f368c06d7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3692000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f368c0cd200 con 0x7f368c15be10 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3692000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f368c15c1f0 con 0x7f368c06d330 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3692000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f368c06cac0 con 0x7f368c0aef20 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3690c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f368c15be10 0x7f368c0ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3690c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f368c15be10 0x7f368c0ccc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59720/0 (socket says 172.21.15.86:59720) 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3690c00640 1 -- 172.21.15.86:0/293549808 learned_addr learned my addr 172.21.15.86:0/293549808 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f368be00640 1 --2- 172.21.15.86:0/293549808 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c0aef20 0x7f368c0ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3691600640 1 --2- 172.21.15.86:0/293549808 >> v2:172.21.15.194:3300/0 conn(0x7f368c06d330 0x7f368c06d7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3690c00640 1 -- 172.21.15.86:0/293549808 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c0aef20 msgr2=0x7f368c0ce2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3690c00640 1 --2- 172.21.15.86:0/293549808 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c0aef20 0x7f368c0ce2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3690c00640 1 -- 172.21.15.86:0/293549808 >> v2:172.21.15.194:3300/0 conn(0x7f368c06d330 msgr2=0x7f368c06d7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.474+0000 7f3690c00640 1 --2- 172.21.15.86:0/293549808 >> v2:172.21.15.194:3300/0 conn(0x7f368c06d330 0x7f368c06d7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.475+0000 7f3690c00640 1 -- 172.21.15.86:0/293549808 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f368c06cda0 con 0x7f368c15be10 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.475+0000 7f368be00640 1 --2- 172.21.15.86:0/293549808 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c0aef20 0x7f368c0ce2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:56.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.475+0000 7f3691600640 1 --2- 172.21.15.86:0/293549808 >> v2:172.21.15.194:3300/0 conn(0x7f368c06d330 0x7f368c06d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.476+0000 7f3690c00640 1 --2- 172.21.15.86:0/293549808 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c15be10 0x7f368c0ccc20 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f368c0b2900 tx=0x7f368002b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb8656b7f3390d53 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.476+0000 7f368b400640 1 -- 172.21.15.86:0/293549808 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3680035020 con 0x7f368c15be10 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.476+0000 7f368b400640 1 -- 172.21.15.86:0/293549808 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f368002d040 con 0x7f368c15be10 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.476+0000 7f368b400640 1 -- 172.21.15.86:0/293549808 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3680031a80 con 0x7f368c15be10 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 -- 172.21.15.86:0/293549808 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c15be10 msgr2=0x7f368c0ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 --2- 172.21.15.86:0/293549808 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c15be10 0x7f368c0ccc20 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f368c0b2900 tx=0x7f368002b900 comp rx=0 tx=0).stop 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 -- 172.21.15.86:0/293549808 shutdown_connections 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 --2- 172.21.15.86:0/293549808 >> v2:172.21.15.194:3300/0 conn(0x7f368c06d330 0x7f368c06d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 --2- 172.21.15.86:0/293549808 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c0aef20 0x7f368c0ce2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 --2- 172.21.15.86:0/293549808 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c15be10 0x7f368c0ccc20 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 -- 172.21.15.86:0/293549808 >> 172.21.15.86:0/293549808 conn(0x7f368c06afc0 msgr2=0x7f368c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 -- 172.21.15.86:0/293549808 shutdown_connections 2024-03-06T07:28:56.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.477+0000 7f3692000640 1 -- 172.21.15.86:0/293549808 wait complete. 2024-03-06T07:28:56.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.478+0000 7f3692000640 1 Processor -- start 2024-03-06T07:28:56.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.479+0000 7f3692000640 1 -- start start 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.479+0000 7f3692000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c06d330 0x7f368c0b07f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.479+0000 7f3692000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f368c0aef20 0x7f368c14c2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.479+0000 7f3692000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c15be10 0x7f368c14c810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.479+0000 7f3692000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f368c0cda90 con 0x7f368c06d330 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.479+0000 7f3692000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f368c06d080 con 0x7f368c0aef20 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3692000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f368c071ab0 con 0x7f368c15be10 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c06d330 0x7f368c0b07f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c06d330 0x7f368c0b07f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59732/0 (socket says 172.21.15.86:59732) 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 -- 172.21.15.86:0/4171426007 learned_addr learned my addr 172.21.15.86:0/4171426007 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 -- 172.21.15.86:0/4171426007 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c15be10 msgr2=0x7f368c14c810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f368be00640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f368c0aef20 0x7f368c14c2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c15be10 0x7f368c14c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 -- 172.21.15.86:0/4171426007 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f368c0aef20 msgr2=0x7f368c14c2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f368c0aef20 0x7f368c14c2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 -- 172.21.15.86:0/4171426007 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f368c158390 con 0x7f368c06d330 2024-03-06T07:28:56.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.480+0000 7f3690c00640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c06d330 0x7f368c0b07f0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f3680000c00 tx=0x7f36800046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.481+0000 7f3688c00640 1 -- 172.21.15.86:0/4171426007 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3680040070 con 0x7f368c06d330 2024-03-06T07:28:56.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.481+0000 7f3688c00640 1 -- 172.21.15.86:0/4171426007 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3680039bf0 con 0x7f368c06d330 2024-03-06T07:28:56.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.481+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f368c0b41d0 con 0x7f368c06d330 2024-03-06T07:28:56.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.481+0000 7f3688c00640 1 -- 172.21.15.86:0/4171426007 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3680035040 con 0x7f368c06d330 2024-03-06T07:28:56.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.481+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f368c0b4790 con 0x7f368c06d330 2024-03-06T07:28:56.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.482+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f368c04f330 con 0x7f368c06d330 2024-03-06T07:28:56.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.483+0000 7f3688c00640 1 -- 172.21.15.86:0/4171426007 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f368004b060 con 0x7f368c06d330 2024-03-06T07:28:56.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.485+0000 7f3688c00640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f366c06b4d0 0x7f366c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.485+0000 7f3688c00640 1 -- 172.21.15.86:0/4171426007 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f36800abc70 con 0x7f368c06d330 2024-03-06T07:28:56.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.485+0000 7f368be00640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f366c06b4d0 0x7f366c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.485+0000 7f368be00640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f366c06b4d0 0x7f366c06d990 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f368c0b3240 tx=0x7f3678008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.489+0000 7f3688c00640 1 -- 172.21.15.86:0/4171426007 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f368002f310 con 0x7f368c06d330 2024-03-06T07:28:56.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.668+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f368c0cf270 con 0x7f368c06d330 2024-03-06T07:28:56.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.670+0000 7f3688c00640 1 -- 172.21.15.86:0/4171426007 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v43) v1 ==== 93+0+10 (secure 0 0 0) 0x7f368007ada0 con 0x7f368c06d330 2024-03-06T07:28:56.672 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:28:56.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.672+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f366c06b4d0 msgr2=0x7f366c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.672+0000 7f3692000640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f366c06b4d0 0x7f366c06d990 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f368c0b3240 tx=0x7f3678008040 comp rx=0 tx=0).stop 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.672+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c06d330 msgr2=0x7f368c0b07f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.672+0000 7f3692000640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c06d330 0x7f368c0b07f0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f3680000c00 tx=0x7f36800046c0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 shutdown_connections 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f366c06b4d0 0x7f366c06d990 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f368c15be10 0x7f368c14c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f368c0aef20 0x7f368c14c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 --2- 172.21.15.86:0/4171426007 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f368c06d330 0x7f368c0b07f0 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 >> 172.21.15.86:0/4171426007 conn(0x7f368c06afc0 msgr2=0x7f368c1567a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 shutdown_connections 2024-03-06T07:28:56.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.673+0000 7f3692000640 1 -- 172.21.15.86:0/4171426007 wait complete. 2024-03-06T07:28:56.693 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:28:56.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.896+0000 7fefc6e00640 1 Processor -- start 2024-03-06T07:28:56.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.897+0000 7fefc6e00640 1 -- start start 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.898+0000 7fefc6e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fefc00d07b0 0x7fefc00cea80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.898+0000 7fefc6e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc00d20b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.898+0000 7fefc6e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fefc00cf140 con 0x7fefc015f110 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.898+0000 7fefc6e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fefc00cfd50 con 0x7fefc00d07b0 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.898+0000 7fefc6e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fefc00cf420 con 0x7fefc00d1c20 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fefc00d07b0 0x7fefc00cea80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fefc00d07b0 0x7fefc00cea80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59734/0 (socket says 172.21.15.86:59734) 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbfe00640 1 -- 172.21.15.86:0/369251978 learned_addr learned my addr 172.21.15.86:0/369251978 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbf400640 1 --1- 172.21.15.86:0/369251978 >> v1:172.21.15.86:6789/0 conn(0x7fefc015f110 0x7fefc00d0b90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35736/0 (socket says 172.21.15.86:35736) 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefc4e00640 1 --2- 172.21.15.86:0/369251978 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc00d20b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbfe00640 1 -- 172.21.15.86:0/369251978 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 msgr2=0x7fefc00d20b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:56.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbfe00640 1 --2- 172.21.15.86:0/369251978 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc00d20b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbfe00640 1 -- 172.21.15.86:0/369251978 >> v1:172.21.15.86:6789/0 conn(0x7fefc015f110 legacy=0x7fefc00d0b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbfe00640 1 -- 172.21.15.86:0/369251978 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fefc00cf700 con 0x7fefc00d07b0 2024-03-06T07:28:56.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.899+0000 7fefbea00640 1 -- 172.21.15.86:0/369251978 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1914880663 0 0) 0x7fefc00cf140 con 0x7fefc015f110 2024-03-06T07:28:56.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.901+0000 7fefbfe00640 1 --2- 172.21.15.86:0/369251978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc00d07b0 0x7fefc00cea80 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7fefb0009870 tx=0x7fefb002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8267206a77682f97 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.901+0000 7fefbea00640 1 -- 172.21.15.86:0/369251978 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb0034070 con 0x7fefc00d07b0 2024-03-06T07:28:56.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.901+0000 7fefbea00640 1 -- 172.21.15.86:0/369251978 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fefb0030840 con 0x7fefc00d07b0 2024-03-06T07:28:56.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.901+0000 7fefbea00640 1 -- 172.21.15.86:0/369251978 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb0030c10 con 0x7fefc00d07b0 2024-03-06T07:28:56.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 -- 172.21.15.86:0/369251978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc00d07b0 msgr2=0x7fefc00cea80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 --2- 172.21.15.86:0/369251978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc00d07b0 0x7fefc00cea80 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7fefb0009870 tx=0x7fefb002b980 comp rx=0 tx=0).stop 2024-03-06T07:28:56.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 -- 172.21.15.86:0/369251978 shutdown_connections 2024-03-06T07:28:56.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 --2- 172.21.15.86:0/369251978 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc00d20b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 --2- 172.21.15.86:0/369251978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc00d07b0 0x7fefc00cea80 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 -- 172.21.15.86:0/369251978 >> 172.21.15.86:0/369251978 conn(0x7fefc00d3ee0 msgr2=0x7fefc00d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:56.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 -- 172.21.15.86:0/369251978 shutdown_connections 2024-03-06T07:28:56.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.902+0000 7fefc6e00640 1 -- 172.21.15.86:0/369251978 wait complete. 2024-03-06T07:28:56.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.904+0000 7fefc6e00640 1 Processor -- start 2024-03-06T07:28:56.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.904+0000 7fefc6e00640 1 -- start start 2024-03-06T07:28:56.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.905+0000 7fefc6e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fefc00d07b0 0x7fefc01549d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.905+0000 7fefc6e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc01f8620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.905+0000 7fefc6e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc015f110 0x7fefc015a950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.905+0000 7fefc6e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefc015e390 con 0x7fefc015f110 2024-03-06T07:28:56.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.905+0000 7fefc6e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefc015f570 con 0x7fefc00d07b0 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.905+0000 7fefc6e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fefc015e510 con 0x7fefc00d1c20 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc015f110 0x7fefc015a950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc015f110 0x7fefc015a950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59744/0 (socket says 172.21.15.86:59744) 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 -- 172.21.15.86:0/588864477 learned_addr learned my addr 172.21.15.86:0/588864477 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefbf400640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc01f8620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefbfe00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fefc00d07b0 0x7fefc01549d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 -- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 msgr2=0x7fefc01f8620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc01f8620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 -- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fefc00d07b0 msgr2=0x7fefc01549d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fefc00d07b0 0x7fefc01549d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefc4e00640 1 -- 172.21.15.86:0/588864477 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fefc0159f40 con 0x7fefc015f110 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.906+0000 7fefbfe00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fefc00d07b0 0x7fefc01549d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.907+0000 7fefc4e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc015f110 0x7fefc015a950 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7fefb800d800 tx=0x7fefb800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.907+0000 7fefb7e00640 1 -- 172.21.15.86:0/588864477 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb800b660 con 0x7fefc015f110 2024-03-06T07:28:56.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.907+0000 7fefb7e00640 1 -- 172.21.15.86:0/588864477 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fefb8019070 con 0x7fefc015f110 2024-03-06T07:28:56.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.907+0000 7fefb7e00640 1 -- 172.21.15.86:0/588864477 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb800ba60 con 0x7fefc015f110 2024-03-06T07:28:56.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.907+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fefc015a220 con 0x7fefc015f110 2024-03-06T07:28:56.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.907+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fefc01f7b40 con 0x7fefc015f110 2024-03-06T07:28:56.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.908+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fefc00d36f0 con 0x7fefc015f110 2024-03-06T07:28:56.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.909+0000 7fefb7e00640 1 -- 172.21.15.86:0/588864477 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fefb800bc00 con 0x7fefc015f110 2024-03-06T07:28:56.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.910+0000 7fefb7e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fefa006b4d0 0x7fefa006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:56.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.910+0000 7fefb7e00640 1 -- 172.21.15.86:0/588864477 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fefb808a2d0 con 0x7fefc015f110 2024-03-06T07:28:56.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.910+0000 7fefbfe00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fefa006b4d0 0x7fefa006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:56.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.911+0000 7fefbfe00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fefa006b4d0 0x7fefa006d990 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fefb0036000 tx=0x7fefb003b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:56.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:56.915+0000 7fefb7e00640 1 -- 172.21.15.86:0/588864477 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fefc00d36f0 con 0x7fefc015f110 2024-03-06T07:28:57.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.115+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7fefc004f330 con 0x7fefc015f110 2024-03-06T07:28:57.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.116+0000 7fefb7e00640 1 -- 172.21.15.86:0/588864477 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v43) v1 ==== 84+0+65 (secure 0 0 0) 0x7fefb80594a0 con 0x7fefc015f110 2024-03-06T07:28:57.118 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:28:57.118 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:28:57.118 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:28:57.118 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:28:57.118 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:28:57.118 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:28:57.119 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:28:57.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fefa006b4d0 msgr2=0x7fefa006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fefa006b4d0 0x7fefa006d990 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fefb0036000 tx=0x7fefb003b040 comp rx=0 tx=0).stop 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc015f110 msgr2=0x7fefc015a950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc015f110 0x7fefc015a950 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7fefb800d800 tx=0x7fefb800dcc0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 shutdown_connections 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fefa006b4d0 0x7fefa006d990 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fefc015f110 0x7fefc015a950 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fefc00d1c20 0x7fefc01f8620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 --2- 172.21.15.86:0/588864477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fefc00d07b0 0x7fefc01549d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 >> 172.21.15.86:0/588864477 conn(0x7fefc00d3ee0 msgr2=0x7fefc006d370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 shutdown_connections 2024-03-06T07:28:57.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.119+0000 7fefc6e00640 1 -- 172.21.15.86:0/588864477 wait complete. 2024-03-06T07:28:57.137 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:28:57.137 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:28:57.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.341+0000 7f8283e00640 1 Processor -- start 2024-03-06T07:28:57.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.342+0000 7f8283e00640 1 -- start start 2024-03-06T07:28:57.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.343+0000 7f8283e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f828415bb30 0x7f8284070970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.343+0000 7f8283e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f828415e880 con 0x7f8284070eb0 2024-03-06T07:28:57.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.343+0000 7f8283e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f828415eb20 con 0x7f828406b6d0 2024-03-06T07:28:57.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.343+0000 7f8283e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f828415ee00 con 0x7f828415bb30 2024-03-06T07:28:57.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f827be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8284070eb0 0x7f828406afc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35740/0 (socket says 172.21.15.86:35740) 2024-03-06T07:28:57.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f827be00640 1 -- 172.21.15.86:0/3827277425 learned_addr learned my addr 172.21.15.86:0/3827277425 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:57.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282a00640 1 --2- 172.21.15.86:0/3827277425 >> v2:172.21.15.194:3300/0 conn(0x7f828415bb30 0x7f8284070970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:57.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3828468313 0 0) 0x7f828415e880 con 0x7f8284070eb0 2024-03-06T07:28:57.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8268003520 con 0x7f8284070eb0 2024-03-06T07:28:57.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2619390294 0 0) 0x7f8268003520 con 0x7f8284070eb0 2024-03-06T07:28:57.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 >> v2:172.21.15.194:3300/0 conn(0x7f828415bb30 msgr2=0x7f8284070970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:57.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 --2- 172.21.15.86:0/3827277425 >> v2:172.21.15.194:3300/0 conn(0x7f828415bb30 0x7f8284070970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 >> v1:172.21.15.194:6789/0 conn(0x7f828406b6d0 legacy=0x7f828406cd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f828415f0e0 con 0x7f8284070eb0 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.344+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f826c001670 con 0x7f8284070eb0 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 >> v1:172.21.15.86:6789/0 conn(0x7f8284070eb0 legacy=0x7f828406afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282000640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82680039b0 0x7f8268003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282000640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8268004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282000640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f82680051c0 0x7f826800dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f828415ee00 con 0x7f8268004800 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f826800e370 con 0x7f82680039b0 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f826800e570 con 0x7f82680051c0 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f827be00640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8268004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f827be00640 1 -- 172.21.15.86:0/3827277425 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f82680051c0 msgr2=0x7f826800dbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:57.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f827be00640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f82680051c0 0x7f826800dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f827be00640 1 -- 172.21.15.86:0/3827277425 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82680039b0 msgr2=0x7f8268003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:57.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8282a00640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82680039b0 0x7f8268003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:57.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f827be00640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82680039b0 0x7f8268003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f827be00640 1 -- 172.21.15.86:0/3827277425 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8268003520 con 0x7f8268004800 2024-03-06T07:28:57.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.345+0000 7f8283400640 1 -- 172.21.15.86:0/3827277425 reap_dead start 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.346+0000 7f827be00640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8268004c80 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f826c00c240 tx=0x7f8268005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a4c23c3a84fa0e9a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f826c007d10 con 0x7f8268004800 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f826c030070 con 0x7f8268004800 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8282000640 1 -- 172.21.15.86:0/3827277425 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f826c02cda0 con 0x7f8268004800 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3827277425 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 msgr2=0x7f8268004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8268004c80 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f826c00c240 tx=0x7f8268005830 comp rx=0 tx=0).stop 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3827277425 shutdown_connections 2024-03-06T07:28:57.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3827277425 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8268004c80 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3827277425 >> 172.21.15.86:0/3827277425 conn(0x7f82840b18a0 msgr2=0x7f82840b1cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:57.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.348+0000 7f8283e00640 1 -- 172.21.15.86:0/3827277425 shutdown_connections 2024-03-06T07:28:57.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.348+0000 7f8283e00640 1 -- 172.21.15.86:0/3827277425 wait complete. 2024-03-06T07:28:57.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.349+0000 7f8283e00640 1 Processor -- start 2024-03-06T07:28:57.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.349+0000 7f8283e00640 1 -- start start 2024-03-06T07:28:57.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.350+0000 7f8283e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8284159c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.350+0000 7f8283e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f828415a170 0x7f82840aa4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.350+0000 7f8283e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82840aa9e0 0x7f82840aae70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.350+0000 7f8283e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f828415bf10 con 0x7f8268004800 2024-03-06T07:28:57.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.350+0000 7f8283e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f828415f600 con 0x7f82840aa9e0 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.350+0000 7f8283e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f828415c090 con 0x7f828415a170 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.350+0000 7f8282a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8284159c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8284159c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59752/0 (socket says 172.21.15.86:59752) 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 -- 172.21.15.86:0/3393209174 learned_addr learned my addr 172.21.15.86:0/3393209174 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f827be00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f828415a170 0x7f82840aa4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 -- 172.21.15.86:0/3393209174 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f828415a170 msgr2=0x7f82840aa4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f828415a170 0x7f82840aa4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 -- 172.21.15.86:0/3393209174 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82840aa9e0 msgr2=0x7f82840aae70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82840aa9e0 0x7f82840aae70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 -- 172.21.15.86:0/3393209174 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82840ac2a0 con 0x7f8268004800 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8282a00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8284159c30 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f82740107c0 tx=0x7f8274010c80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f827aa00640 1 -- 172.21.15.86:0/3393209174 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f827400f040 con 0x7f8268004800 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f827aa00640 1 -- 172.21.15.86:0/3393209174 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f827400eb00 con 0x7f8268004800 2024-03-06T07:28:57.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f827aa00640 1 -- 172.21.15.86:0/3393209174 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f827401c400 con 0x7f8268004800 2024-03-06T07:28:57.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.351+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f82840ac560 con 0x7f8268004800 2024-03-06T07:28:57.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.352+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f82840acb20 con 0x7f8268004800 2024-03-06T07:28:57.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.353+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f828404f330 con 0x7f8268004800 2024-03-06T07:28:57.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.354+0000 7f827aa00640 1 -- 172.21.15.86:0/3393209174 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f827401c5a0 con 0x7f8268004800 2024-03-06T07:28:57.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.354+0000 7f827aa00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f826006b4d0 0x7f826006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:57.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.355+0000 7f827aa00640 1 -- 172.21.15.86:0/3393209174 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f827408a560 con 0x7f8268004800 2024-03-06T07:28:57.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.355+0000 7f827be00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f826006b4d0 0x7f826006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:57.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.355+0000 7f827be00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f826006b4d0 0x7f826006d990 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f826c034000 tx=0x7f826c039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:57.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.359+0000 7f827aa00640 1 -- 172.21.15.86:0/3393209174 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8274018030 con 0x7f8268004800 2024-03-06T07:28:57.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:57.556+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f828415a550 con 0x7f8268004800 2024-03-06T07:28:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.343+0000 7f827aa00640 1 -- 172.21.15.86:0/3393209174 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v44) v1 ==== 138+0+0 (secure 0 0 0) 0x7f8274059600 con 0x7f8268004800 2024-03-06T07:28:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f826006b4d0 msgr2=0x7f826006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f826006b4d0 0x7f826006d990 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f826c034000 tx=0x7f826c039040 comp rx=0 tx=0).stop 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 msgr2=0x7f8284159c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8284159c30 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f82740107c0 tx=0x7f8274010c80 comp rx=0 tx=0).stop 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 shutdown_connections 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f826006b4d0 0x7f826006d990 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f82840aa9e0 0x7f82840aae70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f828415a170 0x7f82840aa4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 --2- 172.21.15.86:0/3393209174 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8268004800 0x7f8284159c30 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 >> 172.21.15.86:0/3393209174 conn(0x7f82840b18a0 msgr2=0x7f82840b47c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 shutdown_connections 2024-03-06T07:28:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.347+0000 7f8283e00640 1 -- 172.21.15.86:0/3393209174 wait complete. 2024-03-06T07:28:58.374 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:28:58.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.577+0000 7fc730200640 1 Processor -- start 2024-03-06T07:28:58.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.577+0000 7fc730200640 1 -- start start 2024-03-06T07:28:58.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.579+0000 7fc730200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc72815fc10 0x7fc728070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.579+0000 7fc730200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc7280b46c0 con 0x7fc7280b3f70 2024-03-06T07:28:58.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.579+0000 7fc730200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc7280b49a0 con 0x7fc72815fff0 2024-03-06T07:28:58.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.579+0000 7fc730200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc728072c90 con 0x7fc72815fc10 2024-03-06T07:28:58.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.579+0000 7fc72e200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc7280b3f70 0x7fc7280ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35750/0 (socket says 172.21.15.86:35750) 2024-03-06T07:28:58.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.579+0000 7fc72e200640 1 -- 172.21.15.86:0/1039958166 learned_addr learned my addr 172.21.15.86:0/1039958166 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:28:58.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc72d800640 1 --2- 172.21.15.86:0/1039958166 >> v2:172.21.15.194:3300/0 conn(0x7fc72815fc10 0x7fc728070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:58.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2527678862 0 0) 0x7fc7280b46c0 con 0x7fc7280b3f70 2024-03-06T07:28:58.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc710003520 con 0x7fc7280b3f70 2024-03-06T07:28:58.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1804592798 0 0) 0x7fc710003520 con 0x7fc7280b3f70 2024-03-06T07:28:58.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 >> v2:172.21.15.194:3300/0 conn(0x7fc72815fc10 msgr2=0x7fc728070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:58.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 --2- 172.21.15.86:0/1039958166 >> v2:172.21.15.194:3300/0 conn(0x7fc72815fc10 0x7fc728070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 >> v1:172.21.15.194:6789/0 conn(0x7fc72815fff0 legacy=0x7fc728160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc72809dd80 con 0x7fc7280b3f70 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc7200019f0 con 0x7fc7280b3f70 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 >> v1:172.21.15.86:6789/0 conn(0x7fc7280b3f70 legacy=0x7fc7280ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7100039b0 0x7fc710003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc710004800 0x7fc710004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100051c0 0x7fc71000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc728072c90 con 0x7fc7100039b0 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc71000c060 con 0x7fc7100051c0 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.580+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc71000c260 con 0x7fc710004800 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.581+0000 7fc72d800640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7100039b0 0x7fc710003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.581+0000 7fc72d800640 1 -- 172.21.15.86:0/1039958166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc710004800 msgr2=0x7fc710004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:58.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.581+0000 7fc72d800640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc710004800 0x7fc710004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.581+0000 7fc72d800640 1 -- 172.21.15.86:0/1039958166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100051c0 msgr2=0x7fc71000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:58.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.581+0000 7fc72d800640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100051c0 0x7fc71000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.581+0000 7fc72d800640 1 -- 172.21.15.86:0/1039958166 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc710003520 con 0x7fc7100039b0 2024-03-06T07:28:58.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.581+0000 7fc72e200640 1 -- 172.21.15.86:0/1039958166 reap_dead start 2024-03-06T07:28:58.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.582+0000 7fc72d800640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7100039b0 0x7fc710003dd0 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7fc71c009ac0 tx=0x7fc710005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=147e3d0b85bade7f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:58.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.582+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc71c039070 con 0x7fc7100039b0 2024-03-06T07:28:58.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.582+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc71c02c6d0 con 0x7fc7100039b0 2024-03-06T07:28:58.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.582+0000 7fc727e00640 1 -- 172.21.15.86:0/1039958166 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc71c009690 con 0x7fc7100039b0 2024-03-06T07:28:58.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.583+0000 7fc730200640 1 -- 172.21.15.86:0/1039958166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7100039b0 msgr2=0x7fc710003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:58.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.583+0000 7fc730200640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7100039b0 0x7fc710003dd0 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7fc71c009ac0 tx=0x7fc710005830 comp rx=0 tx=0).stop 2024-03-06T07:28:58.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.583+0000 7fc730200640 1 -- 172.21.15.86:0/1039958166 shutdown_connections 2024-03-06T07:28:58.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.583+0000 7fc730200640 1 --2- 172.21.15.86:0/1039958166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7100039b0 0x7fc710003dd0 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.583+0000 7fc730200640 1 -- 172.21.15.86:0/1039958166 >> 172.21.15.86:0/1039958166 conn(0x7fc7280b1940 msgr2=0x7fc7280b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:58.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.583+0000 7fc730200640 1 -- 172.21.15.86:0/1039958166 shutdown_connections 2024-03-06T07:28:58.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.583+0000 7fc730200640 1 -- 172.21.15.86:0/1039958166 wait complete. 2024-03-06T07:28:58.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.585+0000 7fc730200640 1 Processor -- start 2024-03-06T07:28:58.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.585+0000 7fc730200640 1 -- start start 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc730200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100039b0 0x7fc72806e580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc730200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7280c8fc0 0x7fc72814d540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc730200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc72814dd80 0x7fc7280b08f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc730200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc72815c460 con 0x7fc7280c8fc0 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc730200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc72815c2e0 con 0x7fc7100039b0 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc730200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc728070ff0 con 0x7fc72814dd80 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7280c8fc0 0x7fc72814d540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:58.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7280c8fc0 0x7fc72814d540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59768/0 (socket says 172.21.15.86:59768) 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 -- 172.21.15.86:0/3389964442 learned_addr learned my addr 172.21.15.86:0/3389964442 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72d800640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100039b0 0x7fc72806e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72e200640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc72814dd80 0x7fc7280b08f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 -- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc72814dd80 msgr2=0x7fc7280b08f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc72814dd80 0x7fc7280b08f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 -- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100039b0 msgr2=0x7fc72806e580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100039b0 0x7fc72806e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.586+0000 7fc72ce00640 1 -- 172.21.15.86:0/3389964442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7280c9d60 con 0x7fc7280c8fc0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.587+0000 7fc72ce00640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7280c8fc0 0x7fc72814d540 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7fc71800c390 tx=0x7fc71800c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.587+0000 7fc725600640 1 -- 172.21.15.86:0/3389964442 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7180052d0 con 0x7fc7280c8fc0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.587+0000 7fc725600640 1 -- 172.21.15.86:0/3389964442 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc718005490 con 0x7fc7280c8fc0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.587+0000 7fc725600640 1 -- 172.21.15.86:0/3389964442 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc718007260 con 0x7fc7280c8fc0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.587+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc7280c97a0 con 0x7fc7280c8fc0 2024-03-06T07:28:58.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.587+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc72806cb10 con 0x7fc7280c8fc0 2024-03-06T07:28:58.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.588+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc72804f330 con 0x7fc7280c8fc0 2024-03-06T07:28:58.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.590+0000 7fc725600640 1 -- 172.21.15.86:0/3389964442 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc718007400 con 0x7fc7280c8fc0 2024-03-06T07:28:58.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.591+0000 7fc725600640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc70c06b4d0 0x7fc70c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:58.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.591+0000 7fc725600640 1 -- 172.21.15.86:0/3389964442 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5227+0+0 (secure 0 0 0) 0x7fc718089a70 con 0x7fc7280c8fc0 2024-03-06T07:28:58.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.591+0000 7fc72d800640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc70c06b4d0 0x7fc70c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:58.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.592+0000 7fc72d800640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc70c06b4d0 0x7fc70c06d990 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fc71c036000 tx=0x7fc71c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:58.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.594+0000 7fc725600640 1 -- 172.21.15.86:0/3389964442 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc718058ab0 con 0x7fc7280c8fc0 2024-03-06T07:28:58.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:58.790+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fc72806ef30 con 0x7fc7280c8fc0 2024-03-06T07:28:59.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.360+0000 7fc725600640 1 -- 172.21.15.86:0/3389964442 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v45) v1 ==== 170+0+0 (secure 0 0 0) 0x7fc7180583b0 con 0x7fc7280c8fc0 2024-03-06T07:28:59.362 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:28:59.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.365+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc70c06b4d0 msgr2=0x7fc70c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:59.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.365+0000 7fc730200640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc70c06b4d0 0x7fc70c06d990 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fc71c036000 tx=0x7fc71c030000 comp rx=0 tx=0).stop 2024-03-06T07:28:59.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.365+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7280c8fc0 msgr2=0x7fc72814d540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:59.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.365+0000 7fc730200640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7280c8fc0 0x7fc72814d540 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7fc71800c390 tx=0x7fc71800c850 comp rx=0 tx=0).stop 2024-03-06T07:28:59.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.366+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 shutdown_connections 2024-03-06T07:28:59.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.366+0000 7fc730200640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc70c06b4d0 0x7fc70c06d990 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.366+0000 7fc730200640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc72814dd80 0x7fc7280b08f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.366+0000 7fc730200640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7280c8fc0 0x7fc72814d540 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.366+0000 7fc730200640 1 --2- 172.21.15.86:0/3389964442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7100039b0 0x7fc72806e580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.366+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 >> 172.21.15.86:0/3389964442 conn(0x7fc7280b1940 msgr2=0x7fc7280acf30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:59.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.367+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 shutdown_connections 2024-03-06T07:28:59.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.368+0000 7fc730200640 1 -- 172.21.15.86:0/3389964442 wait complete. 2024-03-06T07:28:59.403 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:28:59.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.582+0000 7f4f63e00640 1 Processor -- start 2024-03-06T07:28:59.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.583+0000 7f4f63e00640 1 -- start start 2024-03-06T07:28:59.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f63e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4f64154dc0 0x7f4f64161b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f63e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4f641551a0 0x7f4f640ae8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f63e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f640aee20 0x7f4f640cebc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f63e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f640af290 con 0x7f4f64154dc0 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f63e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f6415e540 con 0x7f4f641551a0 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f63e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f640cfb30 con 0x7f4f640aee20 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f62a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4f64154dc0 0x7f4f64161b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f62a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4f64154dc0 0x7f4f64161b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59780/0 (socket says 172.21.15.86:59780) 2024-03-06T07:28:59.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.584+0000 7f4f62a00640 1 -- 172.21.15.86:0/658452362 learned_addr learned my addr 172.21.15.86:0/658452362 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.585+0000 7f4f5be00640 1 --2- 172.21.15.86:0/658452362 >> v2:172.21.15.194:3300/0 conn(0x7f4f641551a0 0x7f4f640ae8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.585+0000 7f4f62a00640 1 -- 172.21.15.86:0/658452362 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f640aee20 msgr2=0x7f4f640cebc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.585+0000 7f4f62a00640 1 --2- 172.21.15.86:0/658452362 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f640aee20 0x7f4f640cebc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.585+0000 7f4f62a00640 1 -- 172.21.15.86:0/658452362 >> v2:172.21.15.194:3300/0 conn(0x7f4f641551a0 msgr2=0x7f4f640ae8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:28:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.585+0000 7f4f62a00640 1 --2- 172.21.15.86:0/658452362 >> v2:172.21.15.194:3300/0 conn(0x7f4f641551a0 0x7f4f640ae8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.585+0000 7f4f62a00640 1 -- 172.21.15.86:0/658452362 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f640cfe10 con 0x7f4f64154dc0 2024-03-06T07:28:59.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.586+0000 7f4f62a00640 1 --2- 172.21.15.86:0/658452362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f64154dc0 0x7f4f64161b30 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f4f64070920 tx=0x7f4f5402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=edf559a0d5cd11c3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.586+0000 7f4f62000640 1 -- 172.21.15.86:0/658452362 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f54035020 con 0x7f4f64154dc0 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.586+0000 7f4f62000640 1 -- 172.21.15.86:0/658452362 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f54031870 con 0x7f4f64154dc0 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.586+0000 7f4f62000640 1 -- 172.21.15.86:0/658452362 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f54031c40 con 0x7f4f64154dc0 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 -- 172.21.15.86:0/658452362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f64154dc0 msgr2=0x7f4f64161b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 --2- 172.21.15.86:0/658452362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f64154dc0 0x7f4f64161b30 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f4f64070920 tx=0x7f4f5402b910 comp rx=0 tx=0).stop 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 -- 172.21.15.86:0/658452362 shutdown_connections 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 --2- 172.21.15.86:0/658452362 >> v2:172.21.15.194:3300/0 conn(0x7f4f641551a0 0x7f4f640ae8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 --2- 172.21.15.86:0/658452362 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f640aee20 0x7f4f640cebc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 --2- 172.21.15.86:0/658452362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f64154dc0 0x7f4f64161b30 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 -- 172.21.15.86:0/658452362 >> 172.21.15.86:0/658452362 conn(0x7f4f6406afc0 msgr2=0x7f4f6406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 -- 172.21.15.86:0/658452362 shutdown_connections 2024-03-06T07:28:59.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.587+0000 7f4f63e00640 1 -- 172.21.15.86:0/658452362 wait complete. 2024-03-06T07:28:59.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.589+0000 7f4f63e00640 1 Processor -- start 2024-03-06T07:28:59.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.589+0000 7f4f63e00640 1 -- start start 2024-03-06T07:28:59.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f640aee20 0x7f4f641f4ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f64154dc0 0x7f4f641f7910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f641551a0 0x7f4f640b1600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f640cf820 con 0x7f4f641551a0 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f640d03d0 con 0x7f4f640aee20 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f64162070 con 0x7f4f64154dc0 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f641551a0 0x7f4f640b1600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f641551a0 0x7f4f640b1600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59782/0 (socket says 172.21.15.86:59782) 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 -- 172.21.15.86:0/1252063358 learned_addr learned my addr 172.21.15.86:0/1252063358 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f62a00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f640aee20 0x7f4f641f4ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f5be00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f64154dc0 0x7f4f641f7910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 -- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f64154dc0 msgr2=0x7f4f641f7910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f64154dc0 0x7f4f641f7910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 -- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f640aee20 msgr2=0x7f4f641f4ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f640aee20 0x7f4f641f4ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.590+0000 7f4f63400640 1 -- 172.21.15.86:0/1252063358 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f640b2180 con 0x7f4f641551a0 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.591+0000 7f4f63400640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f641551a0 0x7f4f640b1600 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f4f5c00b9a0 tx=0x7f4f5c00be60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.591+0000 7f4f5aa00640 1 -- 172.21.15.86:0/1252063358 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f5c019070 con 0x7f4f641551a0 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.591+0000 7f4f5aa00640 1 -- 172.21.15.86:0/1252063358 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f5c00cbc0 con 0x7f4f641551a0 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.591+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f6415f3f0 con 0x7f4f641551a0 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.591+0000 7f4f5aa00640 1 -- 172.21.15.86:0/1252063358 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f5c014440 con 0x7f4f641551a0 2024-03-06T07:28:59.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.591+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f6415f9b0 con 0x7f4f641551a0 2024-03-06T07:28:59.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.592+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f6404f330 con 0x7f4f641551a0 2024-03-06T07:28:59.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.593+0000 7f4f5aa00640 1 -- 172.21.15.86:0/1252063358 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4f5c01c070 con 0x7f4f641551a0 2024-03-06T07:28:59.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.594+0000 7f4f5aa00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f4006b4d0 0x7f4f4006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:28:59.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.594+0000 7f4f5aa00640 1 -- 172.21.15.86:0/1252063358 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(45..45 src has 1..45) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f4f5c089e90 con 0x7f4f641551a0 2024-03-06T07:28:59.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.594+0000 7f4f62a00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f4006b4d0 0x7f4f4006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:28:59.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.595+0000 7f4f62a00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f4006b4d0 0x7f4f4006d990 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f4f54036000 tx=0x7f4f5403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:28:59.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.599+0000 7f4f5aa00640 1 -- 172.21.15.86:0/1252063358 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4f5c058e00 con 0x7f4f641551a0 2024-03-06T07:28:59.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:28:59.830+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f4f6415fb80 con 0x7f4f641551a0 2024-03-06T07:29:00.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.381+0000 7f4f5aa00640 1 -- 172.21.15.86:0/1252063358 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 6 and data pool 7 v21) v1 ==== 172+0+0 (secure 0 0 0) 0x7f4f5c058700 con 0x7f4f641551a0 2024-03-06T07:29:00.384 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 6 and data pool 7 2024-03-06T07:29:00.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f4006b4d0 msgr2=0x7f4f4006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:00.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f4006b4d0 0x7f4f4006d990 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f4f54036000 tx=0x7f4f5403c040 comp rx=0 tx=0).stop 2024-03-06T07:29:00.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f641551a0 msgr2=0x7f4f640b1600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:00.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f641551a0 0x7f4f640b1600 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f4f5c00b9a0 tx=0x7f4f5c00be60 comp rx=0 tx=0).stop 2024-03-06T07:29:00.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 shutdown_connections 2024-03-06T07:29:00.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f4006b4d0 0x7f4f4006d990 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f641551a0 0x7f4f640b1600 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f64154dc0 0x7f4f641f7910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 --2- 172.21.15.86:0/1252063358 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f640aee20 0x7f4f641f4ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.387+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 >> 172.21.15.86:0/1252063358 conn(0x7f4f6406afc0 msgr2=0x7f4f640b27d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:00.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.388+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 shutdown_connections 2024-03-06T07:29:00.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.388+0000 7f4f63e00640 1 -- 172.21.15.86:0/1252063358 wait complete. 2024-03-06T07:29:00.423 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:00.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.597+0000 7fc666000640 1 Processor -- start 2024-03-06T07:29:00.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.597+0000 7fc666000640 1 -- start start 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc666000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc6600b4680 0x7fc66006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc666000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc660072340 0x7fc66006ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc666000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc66015b090 con 0x7fc66015acb0 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc666000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc6600b2a30 con 0x7fc6600b4680 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc666000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc6600b4360 con 0x7fc660072340 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc656a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc6600b4680 0x7fc66006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc656a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc6600b4680 0x7fc66006e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59792/0 (socket says 172.21.15.86:59792) 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.599+0000 7fc656a00640 1 -- 172.21.15.86:0/1296126040 learned_addr learned my addr 172.21.15.86:0/1296126040 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:00.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.600+0000 7fc65fe00640 1 --2- 172.21.15.86:0/1296126040 >> v2:172.21.15.194:3300/0 conn(0x7fc660072340 0x7fc66006ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:00.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.600+0000 7fc656a00640 1 -- 172.21.15.86:0/1296126040 >> v2:172.21.15.194:3300/0 conn(0x7fc660072340 msgr2=0x7fc66006ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:00.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.600+0000 7fc656a00640 1 --2- 172.21.15.86:0/1296126040 >> v2:172.21.15.194:3300/0 conn(0x7fc660072340 0x7fc66006ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.600+0000 7fc656a00640 1 -- 172.21.15.86:0/1296126040 >> v1:172.21.15.194:6789/0 conn(0x7fc66015acb0 legacy=0x7fc6600b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:00.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.600+0000 7fc656a00640 1 -- 172.21.15.86:0/1296126040 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6600b2cf0 con 0x7fc6600b4680 2024-03-06T07:29:00.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.601+0000 7fc656a00640 1 --2- 172.21.15.86:0/1296126040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6600b4680 0x7fc66006e4f0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7fc6600b4e60 tx=0x7fc64802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8593534e0039531b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:00.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.601+0000 7fc65ea00640 1 -- 172.21.15.86:0/1296126040 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc648035020 con 0x7fc6600b4680 2024-03-06T07:29:00.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.601+0000 7fc65ea00640 1 -- 172.21.15.86:0/1296126040 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc648031870 con 0x7fc6600b4680 2024-03-06T07:29:00.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.601+0000 7fc65ea00640 1 -- 172.21.15.86:0/1296126040 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc648031c40 con 0x7fc6600b4680 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 -- 172.21.15.86:0/1296126040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6600b4680 msgr2=0x7fc66006e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 --2- 172.21.15.86:0/1296126040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6600b4680 0x7fc66006e4f0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7fc6600b4e60 tx=0x7fc64802b910 comp rx=0 tx=0).stop 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 -- 172.21.15.86:0/1296126040 shutdown_connections 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 --2- 172.21.15.86:0/1296126040 >> v2:172.21.15.194:3300/0 conn(0x7fc660072340 0x7fc66006ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 --2- 172.21.15.86:0/1296126040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6600b4680 0x7fc66006e4f0 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 -- 172.21.15.86:0/1296126040 >> 172.21.15.86:0/1296126040 conn(0x7fc6600b1730 msgr2=0x7fc6600b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 -- 172.21.15.86:0/1296126040 shutdown_connections 2024-03-06T07:29:00.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.602+0000 7fc666000640 1 -- 172.21.15.86:0/1296126040 wait complete. 2024-03-06T07:29:00.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 Processor -- start 2024-03-06T07:29:00.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 -- start start 2024-03-06T07:29:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc660072340 0x7fc6600ce690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6600b4680 0x7fc6601434f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc66015acb0 0x7fc6600b00f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6600b4010 con 0x7fc660072340 2024-03-06T07:29:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6600b32b0 con 0x7fc66015acb0 2024-03-06T07:29:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.604+0000 7fc666000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc6600ccfb0 con 0x7fc6600b4680 2024-03-06T07:29:00.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc660072340 0x7fc6600ce690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:00.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc660072340 0x7fc6600ce690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59800/0 (socket says 172.21.15.86:59800) 2024-03-06T07:29:00.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 -- 172.21.15.86:0/508316553 learned_addr learned my addr 172.21.15.86:0/508316553 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65fe00640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc66015acb0 0x7fc6600b00f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 -- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6600b4680 msgr2=0x7fc6601434f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc656a00640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6600b4680 0x7fc6601434f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6600b4680 0x7fc6601434f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 -- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc66015acb0 msgr2=0x7fc6600b00f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc66015acb0 0x7fc6600b00f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65f400640 1 -- 172.21.15.86:0/508316553 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6601f0020 con 0x7fc660072340 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.605+0000 7fc65fe00640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc66015acb0 0x7fc6600b00f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.606+0000 7fc65f400640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc660072340 0x7fc6600ce690 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7fc65000b3f0 tx=0x7fc65000b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.606+0000 7fc657e00640 1 -- 172.21.15.86:0/508316553 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc650015020 con 0x7fc660072340 2024-03-06T07:29:00.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.606+0000 7fc657e00640 1 -- 172.21.15.86:0/508316553 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc650011830 con 0x7fc660072340 2024-03-06T07:29:00.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.606+0000 7fc657e00640 1 -- 172.21.15.86:0/508316553 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc650011c20 con 0x7fc660072340 2024-03-06T07:29:00.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.606+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc6601f0300 con 0x7fc660072340 2024-03-06T07:29:00.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.606+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc6601f08c0 con 0x7fc660072340 2024-03-06T07:29:00.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.607+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc66004f330 con 0x7fc660072340 2024-03-06T07:29:00.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.608+0000 7fc657e00640 1 -- 172.21.15.86:0/508316553 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc650018020 con 0x7fc660072340 2024-03-06T07:29:00.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.609+0000 7fc657e00640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc63c06b4d0 0x7fc63c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:00.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.609+0000 7fc657e00640 1 -- 172.21.15.86:0/508316553 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc650089ed0 con 0x7fc660072340 2024-03-06T07:29:00.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.609+0000 7fc656a00640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc63c06b4d0 0x7fc63c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:00.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.610+0000 7fc656a00640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc63c06b4d0 0x7fc63c06d990 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fc648036000 tx=0x7fc648008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:00.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.613+0000 7fc657e00640 1 -- 172.21.15.86:0/508316553 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc65001f030 con 0x7fc660072340 2024-03-06T07:29:00.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.808+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc660158000 con 0x7fc660072340 2024-03-06T07:29:00.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.810+0000 7fc657e00640 1 -- 172.21.15.86:0/508316553 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v46) v1 ==== 74+0+16638 (secure 0 0 0) 0x7fc6500172c0 con 0x7fc660072340 2024-03-06T07:29:00.812 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:00.812 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":46,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:59.831717+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:57.609038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:58.840885+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:00.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc63c06b4d0 msgr2=0x7fc63c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc63c06b4d0 0x7fc63c06d990 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fc648036000 tx=0x7fc648008030 comp rx=0 tx=0).stop 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc660072340 msgr2=0x7fc6600ce690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc660072340 0x7fc6600ce690 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7fc65000b3f0 tx=0x7fc65000b8b0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 shutdown_connections 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc63c06b4d0 0x7fc63c06d990 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc66015acb0 0x7fc6600b00f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6600b4680 0x7fc6601434f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 --2- 172.21.15.86:0/508316553 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc660072340 0x7fc6600ce690 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 >> 172.21.15.86:0/508316553 conn(0x7fc6600b1730 msgr2=0x7fc6600af6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 shutdown_connections 2024-03-06T07:29:00.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:00.813+0000 7fc666000640 1 -- 172.21.15.86:0/508316553 wait complete. 2024-03-06T07:29:00.832 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:01.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.041+0000 7f4406400640 1 Processor -- start 2024-03-06T07:29:01.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.041+0000 7f4406400640 1 -- start start 2024-03-06T07:29:01.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.042+0000 7f4406400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f44000ae670 0x7f440015e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.042+0000 7f4406400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f44000b28e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.042+0000 7f4406400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4400161900 con 0x7f44000b40c0 2024-03-06T07:29:01.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.042+0000 7f4406400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f440015ebd0 con 0x7f44000ae670 2024-03-06T07:29:01.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.042+0000 7f4406400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f440015eeb0 con 0x7f44001620c0 2024-03-06T07:29:01.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.043+0000 7f43f7e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f44000ae670 0x7f440015e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.043+0000 7f43f7e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f44000ae670 0x7f440015e2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:47292/0 (socket says 172.21.15.86:47292) 2024-03-06T07:29:01.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.043+0000 7f43f7e00640 1 -- 172.21.15.86:0/3773019759 learned_addr learned my addr 172.21.15.86:0/3773019759 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:01.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.043+0000 7f4405a00640 1 --2- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f44000b28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.044+0000 7f43f7e00640 1 -- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 msgr2=0x7f44000b28e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:01.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.044+0000 7f43f7e00640 1 --2- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f44000b28e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.044+0000 7f43f7e00640 1 -- 172.21.15.86:0/3773019759 >> v1:172.21.15.194:6789/0 conn(0x7f44000b40c0 legacy=0x7f44000b44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.044+0000 7f43f7e00640 1 -- 172.21.15.86:0/3773019759 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f440015f190 con 0x7f44000ae670 2024-03-06T07:29:01.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.045+0000 7f43f7e00640 1 --2- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000ae670 0x7f440015e2c0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f43e8009870 tx=0x7f43e802b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7fb89e25e44cf43d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:01.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.045+0000 7f43ffe00640 1 -- 172.21.15.86:0/3773019759 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43e8034070 con 0x7f44000ae670 2024-03-06T07:29:01.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.045+0000 7f43ffe00640 1 -- 172.21.15.86:0/3773019759 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43e8030840 con 0x7f44000ae670 2024-03-06T07:29:01.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.045+0000 7f43ffe00640 1 -- 172.21.15.86:0/3773019759 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43e8030c10 con 0x7f44000ae670 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 -- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000ae670 msgr2=0x7f440015e2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 --2- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000ae670 0x7f440015e2c0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f43e8009870 tx=0x7f43e802b980 comp rx=0 tx=0).stop 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 -- 172.21.15.86:0/3773019759 shutdown_connections 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 --2- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f44000b28e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 --2- 172.21.15.86:0/3773019759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000ae670 0x7f440015e2c0 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 -- 172.21.15.86:0/3773019759 >> 172.21.15.86:0/3773019759 conn(0x7f440006afc0 msgr2=0x7f440006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 -- 172.21.15.86:0/3773019759 shutdown_connections 2024-03-06T07:29:01.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.046+0000 7f4406400640 1 -- 172.21.15.86:0/3773019759 wait complete. 2024-03-06T07:29:01.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.048+0000 7f4406400640 1 Processor -- start 2024-03-06T07:29:01.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.048+0000 7f4406400640 1 -- start start 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4406400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44000ae670 0x7f4400159660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4406400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000b40c0 0x7f4400160650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4406400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f4400150020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4406400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f440015e980 con 0x7f44000ae670 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4406400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f440015e800 con 0x7f44000b40c0 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4406400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4400158660 con 0x7f44001620c0 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4405000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44000ae670 0x7f4400159660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4405000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44000ae670 0x7f4400159660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59804/0 (socket says 172.21.15.86:59804) 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.049+0000 7f4405000640 1 -- 172.21.15.86:0/1197729935 learned_addr learned my addr 172.21.15.86:0/1197729935 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:01.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4405a00640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f4400150020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f43f7e00640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000b40c0 0x7f4400160650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4405000640 1 -- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 msgr2=0x7f4400150020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4405000640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f4400150020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4405000640 1 -- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000b40c0 msgr2=0x7f4400160650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4405000640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000b40c0 0x7f4400160650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4405000640 1 -- 172.21.15.86:0/1197729935 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f440015a630 con 0x7f44000ae670 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f43f7e00640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000b40c0 0x7f4400160650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4405000640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44000ae670 0x7f4400159660 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f43f000c400 tx=0x7f43f000c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f43fd600640 1 -- 172.21.15.86:0/1197729935 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43f00052d0 con 0x7f44000ae670 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f43fd600640 1 -- 172.21.15.86:0/1197729935 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43f0005490 con 0x7f44000ae670 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f43fd600640 1 -- 172.21.15.86:0/1197729935 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43f0007260 con 0x7f44000ae670 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.050+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f440015a910 con 0x7f44000ae670 2024-03-06T07:29:01.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.051+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f440015aed0 con 0x7f44000ae670 2024-03-06T07:29:01.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.052+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4400162570 con 0x7f44000ae670 2024-03-06T07:29:01.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.053+0000 7f43fd600640 1 -- 172.21.15.86:0/1197729935 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f43f0007400 con 0x7f44000ae670 2024-03-06T07:29:01.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.053+0000 7f43fd600640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43dc06b500 0x7f43dc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.054+0000 7f43fd600640 1 -- 172.21.15.86:0/1197729935 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f43f0089780 con 0x7f44000ae670 2024-03-06T07:29:01.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.054+0000 7f43f7e00640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43dc06b500 0x7f43dc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.054+0000 7f43f7e00640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43dc06b500 0x7f43dc06d9c0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f43e8036000 tx=0x7f43e803b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:01.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.058+0000 7f43fd600640 1 -- 172.21.15.86:0/1197729935 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4400162570 con 0x7f44000ae670 2024-03-06T07:29:01.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.254+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f440004f330 con 0x7f44000ae670 2024-03-06T07:29:01.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.255+0000 7f43fd600640 1 -- 172.21.15.86:0/1197729935 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v46) v1 ==== 74+0+16638 (secure 0 0 0) 0x7f43f0058560 con 0x7f44000ae670 2024-03-06T07:29:01.257 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:01.257 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":46,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:28:59.831717+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:57.609038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:58.840885+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.257+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43dc06b500 msgr2=0x7f43dc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.257+0000 7f4406400640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43dc06b500 0x7f43dc06d9c0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f43e8036000 tx=0x7f43e803b040 comp rx=0 tx=0).stop 2024-03-06T07:29:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.257+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44000ae670 msgr2=0x7f4400159660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.257+0000 7f4406400640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44000ae670 0x7f4400159660 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f43f000c400 tx=0x7f43f000c8c0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 shutdown_connections 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43dc06b500 0x7f43dc06d9c0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44001620c0 0x7f4400150020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44000b40c0 0x7f4400160650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 --2- 172.21.15.86:0/1197729935 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44000ae670 0x7f4400159660 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 >> 172.21.15.86:0/1197729935 conn(0x7f440006afc0 msgr2=0x7f44000ce940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 shutdown_connections 2024-03-06T07:29:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.258+0000 7f4406400640 1 -- 172.21.15.86:0/1197729935 wait complete. 2024-03-06T07:29:01.277 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:29:01.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.491+0000 7f3a44e00640 1 Processor -- start 2024-03-06T07:29:01.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.491+0000 7f3a44e00640 1 -- start start 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.492+0000 7f3a44e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3a400b2c40 0x7f3a4015f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.492+0000 7f3a44e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3a4006d960 0x7f3a400b3020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a44e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3a400b3560 con 0x7f3a4006e2f0 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a44e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a4015f700 con 0x7f3a4006d960 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a44e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a4006d4d0 con 0x7f3a400b2c40 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a37e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3a4006d960 0x7f3a400b3020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a3e800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3a4006e2f0 0x7f3a40161d50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35758/0 (socket says 172.21.15.86:35758) 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a3e800640 1 -- 172.21.15.86:0/1523289494 learned_addr learned my addr 172.21.15.86:0/1523289494 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a3de00640 1 --2- 172.21.15.86:0/1523289494 >> v2:172.21.15.194:3300/0 conn(0x7f3a400b2c40 0x7f3a4015f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a37e00640 1 -- 172.21.15.86:0/1523289494 >> v2:172.21.15.194:3300/0 conn(0x7f3a400b2c40 msgr2=0x7f3a4015f1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a37e00640 1 --2- 172.21.15.86:0/1523289494 >> v2:172.21.15.194:3300/0 conn(0x7f3a400b2c40 0x7f3a4015f1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a37e00640 1 -- 172.21.15.86:0/1523289494 >> v1:172.21.15.86:6789/0 conn(0x7f3a4006e2f0 legacy=0x7f3a40161d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.493+0000 7f3a37e00640 1 -- 172.21.15.86:0/1523289494 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a4006cba0 con 0x7f3a4006d960 2024-03-06T07:29:01.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.494+0000 7f3a37e00640 1 --2- 172.21.15.86:0/1523289494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006d960 0x7f3a400b3020 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f3a4006c6e0 tx=0x7f3a2802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b004a1a3c0106726 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:01.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a3d400640 1 -- 172.21.15.86:0/1523289494 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a28035020 con 0x7f3a4006d960 2024-03-06T07:29:01.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a3d400640 1 -- 172.21.15.86:0/1523289494 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a28031830 con 0x7f3a4006d960 2024-03-06T07:29:01.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a3d400640 1 -- 172.21.15.86:0/1523289494 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a28031c00 con 0x7f3a4006d960 2024-03-06T07:29:01.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a44e00640 1 -- 172.21.15.86:0/1523289494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006d960 msgr2=0x7f3a400b3020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a44e00640 1 --2- 172.21.15.86:0/1523289494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006d960 0x7f3a400b3020 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f3a4006c6e0 tx=0x7f3a2802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a44e00640 1 -- 172.21.15.86:0/1523289494 shutdown_connections 2024-03-06T07:29:01.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a44e00640 1 --2- 172.21.15.86:0/1523289494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006d960 0x7f3a400b3020 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a44e00640 1 --2- 172.21.15.86:0/1523289494 >> v2:172.21.15.194:3300/0 conn(0x7f3a400b2c40 0x7f3a4015f1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.495+0000 7f3a44e00640 1 -- 172.21.15.86:0/1523289494 >> 172.21.15.86:0/1523289494 conn(0x7f3a400b1940 msgr2=0x7f3a400b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:01.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.496+0000 7f3a44e00640 1 -- 172.21.15.86:0/1523289494 shutdown_connections 2024-03-06T07:29:01.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.496+0000 7f3a44e00640 1 -- 172.21.15.86:0/1523289494 wait complete. 2024-03-06T07:29:01.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.497+0000 7f3a44e00640 1 Processor -- start 2024-03-06T07:29:01.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.497+0000 7f3a44e00640 1 -- start start 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.498+0000 7f3a44e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a4006d960 0x7f3a4014d7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.498+0000 7f3a44e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006e2f0 0x7f3a400c9b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.498+0000 7f3a44e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a400b2c40 0x7f3a4009d080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.498+0000 7f3a44e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a4006e780 con 0x7f3a4006e2f0 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.498+0000 7f3a44e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a4006d160 con 0x7f3a4006d960 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.498+0000 7f3a44e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3a4006e900 con 0x7f3a400b2c40 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006e2f0 0x7f3a400c9b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006e2f0 0x7f3a400c9b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59836/0 (socket says 172.21.15.86:59836) 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 -- 172.21.15.86:0/2431279043 learned_addr learned my addr 172.21.15.86:0/2431279043 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a3de00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a4006d960 0x7f3a4014d7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a3e800640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a400b2c40 0x7f3a4009d080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 -- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a400b2c40 msgr2=0x7f3a4009d080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a400b2c40 0x7f3a4009d080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 -- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a4006d960 msgr2=0x7f3a4014d7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a4006d960 0x7f3a4014d7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:01.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a37e00640 1 -- 172.21.15.86:0/2431279043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a4009bfe0 con 0x7f3a4006e2f0 2024-03-06T07:29:01.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.499+0000 7f3a3e800640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a400b2c40 0x7f3a4009d080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:01.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.500+0000 7f3a37e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006e2f0 0x7f3a400c9b60 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f3a28009890 tx=0x7f3a280046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.500+0000 7f3a36000640 1 -- 172.21.15.86:0/2431279043 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a28040070 con 0x7f3a4006e2f0 2024-03-06T07:29:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.500+0000 7f3a36000640 1 -- 172.21.15.86:0/2431279043 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a2803e070 con 0x7f3a4006e2f0 2024-03-06T07:29:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.500+0000 7f3a36000640 1 -- 172.21.15.86:0/2431279043 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a28035040 con 0x7f3a4006e2f0 2024-03-06T07:29:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.500+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a4009c2c0 con 0x7f3a4006e2f0 2024-03-06T07:29:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.500+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a4009c880 con 0x7f3a4006e2f0 2024-03-06T07:29:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.501+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a4004f330 con 0x7f3a4006e2f0 2024-03-06T07:29:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.502+0000 7f3a36000640 1 -- 172.21.15.86:0/2431279043 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3a28038020 con 0x7f3a4006e2f0 2024-03-06T07:29:01.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.503+0000 7f3a36000640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a1c06b4d0 0x7f3a1c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:01.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.503+0000 7f3a36000640 1 -- 172.21.15.86:0/2431279043 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3a280abce0 con 0x7f3a4006e2f0 2024-03-06T07:29:01.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.503+0000 7f3a3de00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a1c06b4d0 0x7f3a1c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:01.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.504+0000 7f3a3de00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a1c06b4d0 0x7f3a1c06d990 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f3a30006fd0 tx=0x7f3a30008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:01.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.508+0000 7f3a36000640 1 -- 172.21.15.86:0/2431279043 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3a2807aac0 con 0x7f3a4006e2f0 2024-03-06T07:29:01.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:01.745+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f3a40074cc0 con 0x7f3a4006e2f0 2024-03-06T07:29:02.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.392+0000 7f3a36000640 1 -- 172.21.15.86:0/2431279043 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v24) v1 ==== 118+0+0 (secure 0 0 0) 0x7f3a280317f0 con 0x7f3a4006e2f0 2024-03-06T07:29:02.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.395+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a1c06b4d0 msgr2=0x7f3a1c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:02.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.395+0000 7f3a44e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a1c06b4d0 0x7f3a1c06d990 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f3a30006fd0 tx=0x7f3a30008040 comp rx=0 tx=0).stop 2024-03-06T07:29:02.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.395+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006e2f0 msgr2=0x7f3a400c9b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:02.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.395+0000 7f3a44e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006e2f0 0x7f3a400c9b60 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f3a28009890 tx=0x7f3a280046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 shutdown_connections 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a1c06b4d0 0x7f3a1c06d990 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a400b2c40 0x7f3a4009d080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a4006e2f0 0x7f3a400c9b60 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 --2- 172.21.15.86:0/2431279043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a4006d960 0x7f3a4014d7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 >> 172.21.15.86:0/2431279043 conn(0x7f3a400b1940 msgr2=0x7f3a401573b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 shutdown_connections 2024-03-06T07:29:02.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.396+0000 7f3a44e00640 1 -- 172.21.15.86:0/2431279043 wait complete. 2024-03-06T07:29:02.418 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:29:02.418 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:29:02.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.626+0000 7fc5d3a00640 1 Processor -- start 2024-03-06T07:29:02.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.626+0000 7fc5d3a00640 1 -- start start 2024-03-06T07:29:02.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.628+0000 7fc5d3a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc15f1c0 0x7fc5cc1690e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.628+0000 7fc5d3a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5cc06cba0 con 0x7fc5cc06d960 2024-03-06T07:29:02.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.628+0000 7fc5d3a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5cc06ce80 con 0x7fc5cc0b2c40 2024-03-06T07:29:02.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.628+0000 7fc5d3a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5cc06ea00 con 0x7fc5cc15f1c0 2024-03-06T07:29:02.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.628+0000 7fc5cbe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc5cc06d960 0x7fc5cc06e2f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35764/0 (socket says 172.21.15.86:35764) 2024-03-06T07:29:02.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.628+0000 7fc5cbe00640 1 -- 172.21.15.86:0/2145631860 learned_addr learned my addr 172.21.15.86:0/2145631860 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:02.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.629+0000 7fc5d1a00640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc15f1c0 0x7fc5cc1690e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.629+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2039985137 0 0) 0x7fc5cc06cba0 con 0x7fc5cc06d960 2024-03-06T07:29:02.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.629+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc5b4003520 con 0x7fc5cc06d960 2024-03-06T07:29:02.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.629+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2152508060 0 0) 0x7fc5b4003520 con 0x7fc5cc06d960 2024-03-06T07:29:02.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.629+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc15f1c0 msgr2=0x7fc5cc1690e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:02.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.629+0000 7fc5cb400640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc15f1c0 0x7fc5cc1690e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.629+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 >> v1:172.21.15.194:6789/0 conn(0x7fc5cc0b2c40 legacy=0x7fc5cc0b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:02.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5cc06d160 con 0x7fc5cc06d960 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc5b8001670 con 0x7fc5cc06d960 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 >> v1:172.21.15.86:6789/0 conn(0x7fc5cc06d960 legacy=0x7fc5cc06e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b40039e0 0x7fc5b4003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b4004800 0x7fc5b4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b40051c0 0x7fc5b400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5cc06ea00 con 0x7fc5b40039e0 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b400c090 con 0x7fc5b4004800 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b400c210 con 0x7fc5b40051c0 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5d1000640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b40039e0 0x7fc5b4003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5d1a00640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b40051c0 0x7fc5b400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5d1000640 1 -- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b40051c0 msgr2=0x7fc5b400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5d1000640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b40051c0 0x7fc5b400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5d1000640 1 -- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b4004800 msgr2=0x7fc5b4004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cbe00640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b4004800 0x7fc5b4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5d1000640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b4004800 0x7fc5b4004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5d1000640 1 -- 172.21.15.86:0/2145631860 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b4003520 con 0x7fc5b40039e0 2024-03-06T07:29:02.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.630+0000 7fc5cbe00640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b4004800 0x7fc5b4004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.631+0000 7fc5d1000640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b40039e0 0x7fc5b4003e00 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fc5c0001780 tx=0x7fc5b4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=49b35c251600e3e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c0001370 con 0x7fc5b40039e0 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5c00052d0 con 0x7fc5b40039e0 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5cb400640 1 -- 172.21.15.86:0/2145631860 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c0003830 con 0x7fc5b40039e0 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5d3a00640 1 -- 172.21.15.86:0/2145631860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b40039e0 msgr2=0x7fc5b4003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5d3a00640 1 --2- 172.21.15.86:0/2145631860 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b40039e0 0x7fc5b4003e00 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fc5c0001780 tx=0x7fc5b4005830 comp rx=0 tx=0).stop 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5d1a00640 1 -- 172.21.15.86:0/2145631860 reap_dead start 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5d3a00640 1 -- 172.21.15.86:0/2145631860 shutdown_connections 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5d3a00640 1 -- 172.21.15.86:0/2145631860 >> 172.21.15.86:0/2145631860 conn(0x7fc5cc0b1940 msgr2=0x7fc5cc0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5d3a00640 1 -- 172.21.15.86:0/2145631860 shutdown_connections 2024-03-06T07:29:02.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.632+0000 7fc5d3a00640 1 -- 172.21.15.86:0/2145631860 wait complete. 2024-03-06T07:29:02.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.634+0000 7fc5d3a00640 1 Processor -- start 2024-03-06T07:29:02.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.634+0000 7fc5d3a00640 1 -- start start 2024-03-06T07:29:02.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.635+0000 7fc5d3a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b40039e0 0x7fc5cc09a9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.635+0000 7fc5d3a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5cc15a3a0 0x7fc5cc1568e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.635+0000 7fc5d3a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc156e20 0x7fc5cc1589a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.635+0000 7fc5d3a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5cc1670d0 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.635+0000 7fc5d3a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5cc06d4d0 con 0x7fc5b40039e0 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.635+0000 7fc5d3a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5cc167250 con 0x7fc5cc156e20 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.635+0000 7fc5cbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5cc15a3a0 0x7fc5cc1568e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5cc15a3a0 0x7fc5cc1568e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59862/0 (socket says 172.21.15.86:59862) 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 -- 172.21.15.86:0/3383510567 learned_addr learned my addr 172.21.15.86:0/3383510567 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5d1a00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc156e20 0x7fc5cc1589a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5d1000640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b40039e0 0x7fc5cc09a9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 -- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc156e20 msgr2=0x7fc5cc1589a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc156e20 0x7fc5cc1589a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 -- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b40039e0 msgr2=0x7fc5cc09a9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b40039e0 0x7fc5cc09a9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 -- 172.21.15.86:0/3383510567 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5cc1624d0 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5d1000640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b40039e0 0x7fc5cc09a9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.636+0000 7fc5cbe00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5cc15a3a0 0x7fc5cc1568e0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7fc5b800d7a0 tx=0x7fc5b800dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.637+0000 7fc5c8c00640 1 -- 172.21.15.86:0/3383510567 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5b8019070 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.637+0000 7fc5c8c00640 1 -- 172.21.15.86:0/3383510567 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5b80074e0 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.637+0000 7fc5c8c00640 1 -- 172.21.15.86:0/3383510567 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5b8004a00 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.637+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5cc155840 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.637+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5cc155e00 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.638+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5cc04f330 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.639+0000 7fc5c8c00640 1 -- 172.21.15.86:0/3383510567 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc5b8004ba0 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.640+0000 7fc5c8c00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc5ac06b4d0 0x7fc5ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:02.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.640+0000 7fc5c8c00640 1 -- 172.21.15.86:0/3383510567 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc5b8089ce0 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.640+0000 7fc5d1000640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc5ac06b4d0 0x7fc5ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:02.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.641+0000 7fc5d1000640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc5ac06b4d0 0x7fc5ac06d990 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fc5c0033000 tx=0x7fc5c0009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:02.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.644+0000 7fc5c8c00640 1 -- 172.21.15.86:0/3383510567 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc5cc1624d0 con 0x7fc5cc15a3a0 2024-03-06T07:29:02.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:02.882+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7fc5cc0b2530 con 0x7fc5cc15a3a0 2024-03-06T07:29:03.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.405+0000 7fc5c8c00640 1 -- 172.21.15.86:0/3383510567 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v25) v1 ==== 122+0+0 (secure 0 0 0) 0x7fc5b8058ac0 con 0x7fc5cc15a3a0 2024-03-06T07:29:03.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc5ac06b4d0 msgr2=0x7fc5ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:03.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc5ac06b4d0 0x7fc5ac06d990 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fc5c0033000 tx=0x7fc5c0009030 comp rx=0 tx=0).stop 2024-03-06T07:29:03.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5cc15a3a0 msgr2=0x7fc5cc1568e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:03.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5cc15a3a0 0x7fc5cc1568e0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7fc5b800d7a0 tx=0x7fc5b800dc60 comp rx=0 tx=0).stop 2024-03-06T07:29:03.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 shutdown_connections 2024-03-06T07:29:03.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc5ac06b4d0 0x7fc5ac06d990 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5cc156e20 0x7fc5cc1589a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5cc15a3a0 0x7fc5cc1568e0 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 --2- 172.21.15.86:0/3383510567 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b40039e0 0x7fc5cc09a9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.409+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 >> 172.21.15.86:0/3383510567 conn(0x7fc5cc0b1940 msgr2=0x7fc5cc06e2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:03.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.410+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 shutdown_connections 2024-03-06T07:29:03.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.410+0000 7fc5d3a00640 1 -- 172.21.15.86:0/3383510567 wait complete. 2024-03-06T07:29:03.430 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:29:03.430 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:03.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.635+0000 7fca99600640 1 Processor -- start 2024-03-06T07:29:03.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.635+0000 7fca99600640 1 -- start start 2024-03-06T07:29:03.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.637+0000 7fca99600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fca9415f930 0x7fca94066f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:03.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.637+0000 7fca99600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9406ed70 0x7fca9415fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:03.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.638+0000 7fca99600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fca9415c2f0 0x7fca9415c780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:03.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.638+0000 7fca99600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fca940ccd90 con 0x7fca9415c2f0 2024-03-06T07:29:03.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.638+0000 7fca99600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fca940afc00 con 0x7fca9415f930 2024-03-06T07:29:03.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.638+0000 7fca99600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca9406f1f0 con 0x7fca9406ed70 2024-03-06T07:29:03.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.638+0000 7fca93400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fca9415c2f0 0x7fca9415c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:03.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.638+0000 7fca93400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fca9415c2f0 0x7fca9415c780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59872/0 (socket says 172.21.15.86:59872) 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.638+0000 7fca93400640 1 -- 172.21.15.86:0/830615221 learned_addr learned my addr 172.21.15.86:0/830615221 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.639+0000 7fca92000640 1 --2- 172.21.15.86:0/830615221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9406ed70 0x7fca9415fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.639+0000 7fca92a00640 1 --2- 172.21.15.86:0/830615221 >> v2:172.21.15.194:3300/0 conn(0x7fca9415f930 0x7fca94066f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.639+0000 7fca93400640 1 -- 172.21.15.86:0/830615221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9406ed70 msgr2=0x7fca9415fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.639+0000 7fca93400640 1 --2- 172.21.15.86:0/830615221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9406ed70 0x7fca9415fd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.639+0000 7fca93400640 1 -- 172.21.15.86:0/830615221 >> v2:172.21.15.194:3300/0 conn(0x7fca9415f930 msgr2=0x7fca94066f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.639+0000 7fca93400640 1 --2- 172.21.15.86:0/830615221 >> v2:172.21.15.194:3300/0 conn(0x7fca9415f930 0x7fca94066f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.639+0000 7fca93400640 1 -- 172.21.15.86:0/830615221 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca9406e5b0 con 0x7fca9415c2f0 2024-03-06T07:29:03.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.640+0000 7fca93400640 1 --2- 172.21.15.86:0/830615221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415c2f0 0x7fca9415c780 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fca94071050 tx=0x7fca8c02be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6101813dcf938006 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:03.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.640+0000 7fca91600640 1 -- 172.21.15.86:0/830615221 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca8c02c9e0 con 0x7fca9415c2f0 2024-03-06T07:29:03.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.640+0000 7fca91600640 1 -- 172.21.15.86:0/830615221 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca8c02cb80 con 0x7fca9415c2f0 2024-03-06T07:29:03.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.640+0000 7fca91600640 1 -- 172.21.15.86:0/830615221 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca8c0343f0 con 0x7fca9415c2f0 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 -- 172.21.15.86:0/830615221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415c2f0 msgr2=0x7fca9415c780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 --2- 172.21.15.86:0/830615221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415c2f0 0x7fca9415c780 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fca94071050 tx=0x7fca8c02be40 comp rx=0 tx=0).stop 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 -- 172.21.15.86:0/830615221 shutdown_connections 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 --2- 172.21.15.86:0/830615221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415c2f0 0x7fca9415c780 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 --2- 172.21.15.86:0/830615221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9406ed70 0x7fca9415fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 --2- 172.21.15.86:0/830615221 >> v2:172.21.15.194:3300/0 conn(0x7fca9415f930 0x7fca94066f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 -- 172.21.15.86:0/830615221 >> 172.21.15.86:0/830615221 conn(0x7fca940d0340 msgr2=0x7fca940d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 -- 172.21.15.86:0/830615221 shutdown_connections 2024-03-06T07:29:03.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.641+0000 7fca99600640 1 -- 172.21.15.86:0/830615221 wait complete. 2024-03-06T07:29:03.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.643+0000 7fca99600640 1 Processor -- start 2024-03-06T07:29:03.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.644+0000 7fca99600640 1 -- start start 2024-03-06T07:29:03.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca99600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca9406ed70 0x7fca941556d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:03.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca99600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9415c2f0 0x7fca940ae9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:03.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca99600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415f930 0x7fca940aef20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:03.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca99600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca940cd2c0 con 0x7fca9415f930 2024-03-06T07:29:03.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca99600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca940ccfe0 con 0x7fca9406ed70 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca99600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca9406ddb0 con 0x7fca9415c2f0 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca93400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415f930 0x7fca940aef20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca93400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415f930 0x7fca940aef20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59886/0 (socket says 172.21.15.86:59886) 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.645+0000 7fca93400640 1 -- 172.21.15.86:0/1945141564 learned_addr learned my addr 172.21.15.86:0/1945141564 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca92000640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9415c2f0 0x7fca940ae9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca92a00640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca9406ed70 0x7fca941556d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca93400640 1 -- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9415c2f0 msgr2=0x7fca940ae9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca93400640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9415c2f0 0x7fca940ae9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca93400640 1 -- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca9406ed70 msgr2=0x7fca941556d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca93400640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca9406ed70 0x7fca941556d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca93400640 1 -- 172.21.15.86:0/1945141564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca941f7360 con 0x7fca9415f930 2024-03-06T07:29:03.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca93400640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415f930 0x7fca940aef20 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fca8c036040 tx=0x7fca8c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:03.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca8aa00640 1 -- 172.21.15.86:0/1945141564 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca8c043020 con 0x7fca9415f930 2024-03-06T07:29:03.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca8aa00640 1 -- 172.21.15.86:0/1945141564 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca8c03e450 con 0x7fca9415f930 2024-03-06T07:29:03.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca941f7640 con 0x7fca9415f930 2024-03-06T07:29:03.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca8aa00640 1 -- 172.21.15.86:0/1945141564 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca8c009510 con 0x7fca9415f930 2024-03-06T07:29:03.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.646+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca941f7c00 con 0x7fca9415f930 2024-03-06T07:29:03.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.647+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca9404f330 con 0x7fca9415f930 2024-03-06T07:29:03.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.649+0000 7fca8aa00640 1 -- 172.21.15.86:0/1945141564 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fca8c046020 con 0x7fca9415f930 2024-03-06T07:29:03.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.650+0000 7fca8aa00640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca7406b4d0 0x7fca7406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:03.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.650+0000 7fca8aa00640 1 -- 172.21.15.86:0/1945141564 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fca8c0acab0 con 0x7fca9415f930 2024-03-06T07:29:03.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.650+0000 7fca92a00640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca7406b4d0 0x7fca7406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:03.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.651+0000 7fca92a00640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca7406b4d0 0x7fca7406d990 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fca8400b7c0 tx=0x7fca84008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:03.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.654+0000 7fca8aa00640 1 -- 172.21.15.86:0/1945141564 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fca8c04d030 con 0x7fca9415f930 2024-03-06T07:29:03.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.886+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fca940d0fa0 con 0x7fca9415f930 2024-03-06T07:29:03.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.887+0000 7fca8aa00640 1 -- 172.21.15.86:0/1945141564 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) v1 ==== 94+0+4983 (secure 0 0 0) 0x7fca8c07b920 con 0x7fca9415f930 2024-03-06T07:29:03.889 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:03.889 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":25,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5041,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/3678578793","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":3678578793},{"type":"v1","addr":"172.21.15.194:6835","nonce":3678578793}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/4206282845","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":4206282845},{"type":"v1","addr":"172.21.15.86:6835","nonce":4206282845}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5427,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3500830166","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3500830166},{"type":"v1","addr":"172.21.15.86:6837","nonce":3500830166}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":25,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:59.832195+0000","modified":"2024-03-06T07:29:02.883187+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5439},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5439":{"gid":5439,"name":"b","rank":0,"incarnation":21,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6833/4064823241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":4064823241},{"type":"v1","addr":"172.21.15.194:6833","nonce":4064823241}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5439,"qdb_cluster":[5439]},"id":3}]} 2024-03-06T07:29:03.889 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 25 2024-03-06T07:29:03.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.889+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca7406b4d0 msgr2=0x7fca7406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:03.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.889+0000 7fca99600640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca7406b4d0 0x7fca7406d990 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fca8400b7c0 tx=0x7fca84008040 comp rx=0 tx=0).stop 2024-03-06T07:29:03.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415f930 msgr2=0x7fca940aef20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:03.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415f930 0x7fca940aef20 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fca8c036040 tx=0x7fca8c004a40 comp rx=0 tx=0).stop 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 shutdown_connections 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca7406b4d0 0x7fca7406d990 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca9415f930 0x7fca940aef20 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca9415c2f0 0x7fca940ae9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 --2- 172.21.15.86:0/1945141564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca9406ed70 0x7fca941556d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 >> 172.21.15.86:0/1945141564 conn(0x7fca940d0340 msgr2=0x7fca9415c6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 shutdown_connections 2024-03-06T07:29:03.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:03.890+0000 7fca99600640 1 -- 172.21.15.86:0/1945141564 wait complete. 2024-03-06T07:29:03.908 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:04.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.079+0000 7fbadf200640 1 Processor -- start 2024-03-06T07:29:04.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.079+0000 7fbadf200640 1 -- start start 2024-03-06T07:29:04.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 0x7fbad80d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadf200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbad80b3ec0 0x7fbad80b4350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadf200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbad8161960 con 0x7fbad8161580 2024-03-06T07:29:04.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadf200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbad80d2ce0 con 0x7fbad80b3ec0 2024-03-06T07:29:04.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbad80cf6f0 con 0x7fbad80d2360 2024-03-06T07:29:04.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadde00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fbad8161580 0x7fbad80d1950 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35778/0 (socket says 172.21.15.86:35778) 2024-03-06T07:29:04.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadde00640 1 -- 172.21.15.86:0/2241241938 learned_addr learned my addr 172.21.15.86:0/2241241938 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadd400640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 0x7fbad80d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbade800640 1 --2- 172.21.15.86:0/2241241938 >> v2:172.21.15.194:3300/0 conn(0x7fbad80b3ec0 0x7fbad80b4350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.081+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3177234931 0 0) 0x7fbad8161960 con 0x7fbad8161580 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbac0003520 con 0x7fbad8161580 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3449399489 0 0) 0x7fbac0003520 con 0x7fbad8161580 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 msgr2=0x7fbad80d27a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 0x7fbad80d27a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 >> v2:172.21.15.194:3300/0 conn(0x7fbad80b3ec0 msgr2=0x7fbad80b4350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 --2- 172.21.15.86:0/2241241938 >> v2:172.21.15.194:3300/0 conn(0x7fbad80b3ec0 0x7fbad80b4350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbad80cf9d0 con 0x7fbad8161580 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fbacc001670 con 0x7fbad8161580 2024-03-06T07:29:04.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 >> v1:172.21.15.86:6789/0 conn(0x7fbad8161580 legacy=0x7fbad80d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 0x7fbac00039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbac0003f30 0x7fbac0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbac00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbad80d2ce0 con 0x7fbac0004d40 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbad80cf6f0 con 0x7fbac0003f30 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbac000b9c0 con 0x7fbad80d2360 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.082+0000 7fbade800640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbac00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.083+0000 7fbade800640 1 -- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 msgr2=0x7fbac00039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.083+0000 7fbadde00640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 0x7fbac00039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.083+0000 7fbade800640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80d2360 0x7fbac00039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.083+0000 7fbade800640 1 -- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbac0003f30 msgr2=0x7fbac0004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.083+0000 7fbade800640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbac0003f30 0x7fbac0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.083+0000 7fbade800640 1 -- 172.21.15.86:0/2241241938 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbac0003520 con 0x7fbac0004d40 2024-03-06T07:29:04.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.084+0000 7fbade800640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbac00051c0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fbad4009fc0 tx=0x7fbac0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0e96ff3d7c71c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.084+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbad4039070 con 0x7fbac0004d40 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.084+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbad402cc40 con 0x7fbac0004d40 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.084+0000 7fbadca00640 1 -- 172.21.15.86:0/2241241938 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbad4009c10 con 0x7fbac0004d40 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.084+0000 7fbadf200640 1 -- 172.21.15.86:0/2241241938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 msgr2=0x7fbac00051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.084+0000 7fbadf200640 1 --2- 172.21.15.86:0/2241241938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbac00051c0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fbad4009fc0 tx=0x7fbac0005830 comp rx=0 tx=0).stop 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.084+0000 7fbade800640 1 -- 172.21.15.86:0/2241241938 reap_dead start 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.085+0000 7fbadf200640 1 -- 172.21.15.86:0/2241241938 shutdown_connections 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.085+0000 7fbadf200640 1 -- 172.21.15.86:0/2241241938 >> 172.21.15.86:0/2241241938 conn(0x7fbad80d3d80 msgr2=0x7fbad80d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.085+0000 7fbadf200640 1 -- 172.21.15.86:0/2241241938 shutdown_connections 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.085+0000 7fbadf200640 1 -- 172.21.15.86:0/2241241938 wait complete. 2024-03-06T07:29:04.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.086+0000 7fbadf200640 1 Processor -- start 2024-03-06T07:29:04.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.086+0000 7fbadf200640 1 -- start start 2024-03-06T07:29:04.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.087+0000 7fbadf200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbad815b270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.087+0000 7fbadf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80b6970 0x7fbad815f550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.087+0000 7fbadf200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbad80b6df0 0x7fbad815fa90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.087+0000 7fbadf200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbad80b39a0 con 0x7fbac0004d40 2024-03-06T07:29:04.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.087+0000 7fbadf200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbad80b36c0 con 0x7fbad80b6df0 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.087+0000 7fbadf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbad80cf3a0 con 0x7fbad80b6970 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbad815b270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbad815b270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59904/0 (socket says 172.21.15.86:59904) 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 -- 172.21.15.86:0/789956466 learned_addr learned my addr 172.21.15.86:0/789956466 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 -- 172.21.15.86:0/789956466 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80b6970 msgr2=0x7fbad815f550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80b6970 0x7fbad815f550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 -- 172.21.15.86:0/789956466 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbad80b6df0 msgr2=0x7fbad815fa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbad80b6df0 0x7fbad815fa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 -- 172.21.15.86:0/789956466 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbad80c7860 con 0x7fbac0004d40 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbadde00640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbad815b270 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fbacc00d8f0 tx=0x7fbacc00ddb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.088+0000 7fbad2000640 1 -- 172.21.15.86:0/789956466 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbacc019070 con 0x7fbac0004d40 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.089+0000 7fbad2000640 1 -- 172.21.15.86:0/789956466 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbacc0074e0 con 0x7fbac0004d40 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.089+0000 7fbad2000640 1 -- 172.21.15.86:0/789956466 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbacc004a00 con 0x7fbac0004d40 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.089+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbad80c7b40 con 0x7fbac0004d40 2024-03-06T07:29:04.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.089+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbad80c8100 con 0x7fbac0004d40 2024-03-06T07:29:04.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.090+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbad804f330 con 0x7fbac0004d40 2024-03-06T07:29:04.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.091+0000 7fbad2000640 1 -- 172.21.15.86:0/789956466 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbacc01c070 con 0x7fbac0004d40 2024-03-06T07:29:04.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.091+0000 7fbad2000640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbabc06b4d0 0x7fbabc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.092+0000 7fbad2000640 1 -- 172.21.15.86:0/789956466 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbacc089e70 con 0x7fbac0004d40 2024-03-06T07:29:04.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.092+0000 7fbadd400640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbabc06b4d0 0x7fbabc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.092+0000 7fbadd400640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbabc06b4d0 0x7fbabc06d990 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fbad806cf80 tx=0x7fbac80094e0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:04.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.096+0000 7fbad2000640 1 -- 172.21.15.86:0/789956466 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbacc058c60 con 0x7fbac0004d40 2024-03-06T07:29:04.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.275+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fbad80d3b60 con 0x7fbac0004d40 2024-03-06T07:29:04.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.278+0000 7fbad2000640 1 -- 172.21.15.86:0/789956466 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v47) v1 ==== 74+0+16625 (secure 0 0 0) 0x7fbacc058560 con 0x7fbac0004d40 2024-03-06T07:29:04.280 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:04.281 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":47,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:01.377099+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:57.609038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:58.840885+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbabc06b4d0 msgr2=0x7fbabc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbabc06b4d0 0x7fbabc06d990 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fbad806cf80 tx=0x7fbac80094e0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 msgr2=0x7fbad815b270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbad815b270 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fbacc00d8f0 tx=0x7fbacc00ddb0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 shutdown_connections 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbabc06b4d0 0x7fbabc06d990 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbad80b6df0 0x7fbad815fa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbad80b6970 0x7fbad815f550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 --2- 172.21.15.86:0/789956466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbac0004d40 0x7fbad815b270 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 >> 172.21.15.86:0/789956466 conn(0x7fbad80d3d80 msgr2=0x7fbad80b1aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 shutdown_connections 2024-03-06T07:29:04.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.281+0000 7fbadf200640 1 -- 172.21.15.86:0/789956466 wait complete. 2024-03-06T07:29:04.301 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:29:04.301 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:29:04.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.502+0000 7f16e2800640 1 Processor -- start 2024-03-06T07:29:04.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.502+0000 7f16e2800640 1 -- start start 2024-03-06T07:29:04.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.503+0000 7f16e2800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc0af1c0 0x7f16dc15eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16e2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16dc070d20 0x7f16dc06cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16e2800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16dc071d30 con 0x7f16dc06d1a0 2024-03-06T07:29:04.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16e2800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f16dc071490 con 0x7f16dc070d20 2024-03-06T07:29:04.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16e2800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16dc071770 con 0x7f16dc0af1c0 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16d2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16dc070d20 0x7f16dc06cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16d2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16dc070d20 0x7f16dc06cc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59912/0 (socket says 172.21.15.86:59912) 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16d2a00640 1 -- 172.21.15.86:0/3363626200 learned_addr learned my addr 172.21.15.86:0/3363626200 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16db400640 1 --2- 172.21.15.86:0/3363626200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc0af1c0 0x7f16dc15eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16d2a00640 1 -- 172.21.15.86:0/3363626200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc0af1c0 msgr2=0x7f16dc15eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16d2a00640 1 --2- 172.21.15.86:0/3363626200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc0af1c0 0x7f16dc15eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16d2a00640 1 -- 172.21.15.86:0/3363626200 >> v1:172.21.15.194:6789/0 conn(0x7f16dc06d1a0 legacy=0x7f16dc06e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:04.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.504+0000 7f16d2a00640 1 -- 172.21.15.86:0/3363626200 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16dc071a50 con 0x7f16dc070d20 2024-03-06T07:29:04.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.505+0000 7f16d2a00640 1 --2- 172.21.15.86:0/3363626200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc06cc60 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f16c4009870 tx=0x7f16c402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3734b9fa7069add6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:04.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.506+0000 7f16daa00640 1 -- 172.21.15.86:0/3363626200 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c4035020 con 0x7f16dc070d20 2024-03-06T07:29:04.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.506+0000 7f16daa00640 1 -- 172.21.15.86:0/3363626200 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16c4031640 con 0x7f16dc070d20 2024-03-06T07:29:04.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.506+0000 7f16daa00640 1 -- 172.21.15.86:0/3363626200 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c402d040 con 0x7f16dc070d20 2024-03-06T07:29:04.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.506+0000 7f16e2800640 1 -- 172.21.15.86:0/3363626200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 msgr2=0x7f16dc06cc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.506+0000 7f16e2800640 1 --2- 172.21.15.86:0/3363626200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc06cc60 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f16c4009870 tx=0x7f16c402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.507+0000 7f16e2800640 1 -- 172.21.15.86:0/3363626200 shutdown_connections 2024-03-06T07:29:04.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.507+0000 7f16e2800640 1 --2- 172.21.15.86:0/3363626200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc06cc60 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.507+0000 7f16e2800640 1 --2- 172.21.15.86:0/3363626200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc0af1c0 0x7f16dc15eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.507+0000 7f16e2800640 1 -- 172.21.15.86:0/3363626200 >> 172.21.15.86:0/3363626200 conn(0x7f16dc0b1730 msgr2=0x7f16dc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:04.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.507+0000 7f16e2800640 1 -- 172.21.15.86:0/3363626200 shutdown_connections 2024-03-06T07:29:04.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.507+0000 7f16e2800640 1 -- 172.21.15.86:0/3363626200 wait complete. 2024-03-06T07:29:04.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.508+0000 7f16e2800640 1 Processor -- start 2024-03-06T07:29:04.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.508+0000 7f16e2800640 1 -- start start 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16e2800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc06d1a0 0x7f16dc0cc4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16e2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc0cef10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16e2800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16dc0af1c0 0x7f16dc1f3d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16e2800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16dc06ebe0 con 0x7f16dc070d20 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16e2800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16dc0711a0 con 0x7f16dc0af1c0 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16e2800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16dc06ed60 con 0x7f16dc06d1a0 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16d2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc0cef10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16d2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc0cef10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59914/0 (socket says 172.21.15.86:59914) 2024-03-06T07:29:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.509+0000 7f16d2a00640 1 -- 172.21.15.86:0/1863985194 learned_addr learned my addr 172.21.15.86:0/1863985194 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:04.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16db400640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc06d1a0 0x7f16dc0cc4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d2a00640 1 -- 172.21.15.86:0/1863985194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc06d1a0 msgr2=0x7f16dc0cc4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d2a00640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc06d1a0 0x7f16dc0cc4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d2a00640 1 -- 172.21.15.86:0/1863985194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16dc0af1c0 msgr2=0x7f16dc1f3d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d2a00640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16dc0af1c0 0x7f16dc1f3d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d2a00640 1 -- 172.21.15.86:0/1863985194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16dc0ccbc0 con 0x7f16dc070d20 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d2a00640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc0cef10 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f16c402b930 tx=0x7f16c40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d3e00640 1 -- 172.21.15.86:0/1863985194 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c4040070 con 0x7f16dc070d20 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d3e00640 1 -- 172.21.15.86:0/1863985194 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16c40399f0 con 0x7f16dc070d20 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.510+0000 7f16d3e00640 1 -- 172.21.15.86:0/1863985194 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c4035040 con 0x7f16dc070d20 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.511+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16dc0ccea0 con 0x7f16dc070d20 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.511+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16dc0cd460 con 0x7f16dc070d20 2024-03-06T07:29:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.512+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16dc04f330 con 0x7f16dc070d20 2024-03-06T07:29:04.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.513+0000 7f16d3e00640 1 -- 172.21.15.86:0/1863985194 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f16c404b060 con 0x7f16dc070d20 2024-03-06T07:29:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.513+0000 7f16d3e00640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16b806b4d0 0x7f16b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.514+0000 7f16d3e00640 1 -- 172.21.15.86:0/1863985194 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f16c40abc70 con 0x7f16dc070d20 2024-03-06T07:29:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.514+0000 7f16db400640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16b806b4d0 0x7f16b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.514+0000 7f16db400640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16b806b4d0 0x7f16b806d990 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f16dc072c40 tx=0x7f16cc00b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:04.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.518+0000 7f16d3e00640 1 -- 172.21.15.86:0/1863985194 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f16dc071d30 con 0x7f16dc070d20 2024-03-06T07:29:04.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.706+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f16dc0b1d60 con 0x7f16b806b4d0 2024-03-06T07:29:04.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.725+0000 7f16d3e00640 1 -- 172.21.15.86:0/1863985194 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7f16dc0b1d60 con 0x7f16b806b4d0 2024-03-06T07:29:04.730 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:29:04.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16b806b4d0 msgr2=0x7f16b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16b806b4d0 0x7f16b806d990 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f16dc072c40 tx=0x7f16cc00b040 comp rx=0 tx=0).stop 2024-03-06T07:29:04.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 msgr2=0x7f16dc0cef10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:04.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc0cef10 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f16c402b930 tx=0x7f16c40046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 shutdown_connections 2024-03-06T07:29:04.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16b806b4d0 0x7f16b806d990 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16dc0af1c0 0x7f16dc1f3d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16dc070d20 0x7f16dc0cef10 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 --2- 172.21.15.86:0/1863985194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16dc06d1a0 0x7f16dc0cc4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:04.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 >> 172.21.15.86:0/1863985194 conn(0x7f16dc0b1730 msgr2=0x7f16dc0b35d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:04.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 shutdown_connections 2024-03-06T07:29:04.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:04.730+0000 7f16e2800640 1 -- 172.21.15.86:0/1863985194 wait complete. 2024-03-06T07:29:04.748 INFO:tasks.ceph.mds.b.smithi194.stderr:2024-03-06T07:29:04.747+0000 7f4c8a400640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:29:04.751 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":102,"stamp":"2024-03-06T07:29:02.884865+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":43,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":254672,"kb_used_data":5220,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754720048,"statfs":{"total":773094113280,"available":772833329152,"internally_reserved":0,"allocated":5345280,"data_stored":2841839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.539951"},"pg_stats":[{"pgid":"7.3f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381377+0000","last_change":"2024-03-06T07:28:59.381377+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381377+0000","last_undegraded":"2024-03-06T07:28:59.381377+0000","last_fullsized":"2024-03-06T07:28:59.381377+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.383263+0000","last_change":"2024-03-06T07:28:59.383263+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.383263+0000","last_undegraded":"2024-03-06T07:28:59.383263+0000","last_fullsized":"2024-03-06T07:28:59.383263+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381839+0000","last_change":"2024-03-06T07:28:59.381839+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381839+0000","last_undegraded":"2024-03-06T07:28:59.381839+0000","last_fullsized":"2024-03-06T07:28:59.381839+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376428+0000","last_change":"2024-03-06T07:28:59.376428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376428+0000","last_undegraded":"2024-03-06T07:28:59.376428+0000","last_fullsized":"2024-03-06T07:28:59.376428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381584+0000","last_change":"2024-03-06T07:28:59.381584+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381584+0000","last_undegraded":"2024-03-06T07:28:59.381584+0000","last_fullsized":"2024-03-06T07:28:59.381584+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371720+0000","last_change":"2024-03-06T07:28:59.371720+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371720+0000","last_undegraded":"2024-03-06T07:28:59.371720+0000","last_fullsized":"2024-03-06T07:28:59.371720+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.39","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375651+0000","last_change":"2024-03-06T07:28:59.375651+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375651+0000","last_undegraded":"2024-03-06T07:28:59.375651+0000","last_fullsized":"2024-03-06T07:28:59.375651+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.38","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377314+0000","last_change":"2024-03-06T07:28:59.377314+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377314+0000","last_undegraded":"2024-03-06T07:28:59.377314+0000","last_fullsized":"2024-03-06T07:28:59.377314+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.37","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378074+0000","last_change":"2024-03-06T07:28:59.378074+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378074+0000","last_undegraded":"2024-03-06T07:28:59.378074+0000","last_fullsized":"2024-03-06T07:28:59.378074+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.36","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377833+0000","last_change":"2024-03-06T07:28:59.377833+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377833+0000","last_undegraded":"2024-03-06T07:28:59.377833+0000","last_fullsized":"2024-03-06T07:28:59.377833+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.35","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378278+0000","last_change":"2024-03-06T07:28:59.378278+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378278+0000","last_undegraded":"2024-03-06T07:28:59.378278+0000","last_fullsized":"2024-03-06T07:28:59.378278+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.34","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384955+0000","last_change":"2024-03-06T07:28:59.384955+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384955+0000","last_undegraded":"2024-03-06T07:28:59.384955+0000","last_fullsized":"2024-03-06T07:28:59.384955+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.33","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370349+0000","last_change":"2024-03-06T07:28:59.370349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370349+0000","last_undegraded":"2024-03-06T07:28:59.370349+0000","last_fullsized":"2024-03-06T07:28:59.370349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.32","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376562+0000","last_change":"2024-03-06T07:28:59.376562+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376562+0000","last_undegraded":"2024-03-06T07:28:59.376562+0000","last_fullsized":"2024-03-06T07:28:59.376562+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.31","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370581+0000","last_change":"2024-03-06T07:28:59.370581+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370581+0000","last_undegraded":"2024-03-06T07:28:59.370581+0000","last_fullsized":"2024-03-06T07:28:59.370581+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.30","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377473+0000","last_change":"2024-03-06T07:28:59.377473+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377473+0000","last_undegraded":"2024-03-06T07:28:59.377473+0000","last_fullsized":"2024-03-06T07:28:59.377473+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375727+0000","last_change":"2024-03-06T07:28:59.375727+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375727+0000","last_undegraded":"2024-03-06T07:28:59.375727+0000","last_fullsized":"2024-03-06T07:28:59.375727+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382638+0000","last_change":"2024-03-06T07:28:59.382638+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382638+0000","last_undegraded":"2024-03-06T07:28:59.382638+0000","last_fullsized":"2024-03-06T07:28:59.382638+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370250+0000","last_change":"2024-03-06T07:28:59.370250+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370250+0000","last_undegraded":"2024-03-06T07:28:59.370250+0000","last_fullsized":"2024-03-06T07:28:59.370250+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370202+0000","last_change":"2024-03-06T07:28:59.370202+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370202+0000","last_undegraded":"2024-03-06T07:28:59.370202+0000","last_fullsized":"2024-03-06T07:28:59.370202+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382289+0000","last_change":"2024-03-06T07:28:59.382289+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382289+0000","last_undegraded":"2024-03-06T07:28:59.382289+0000","last_fullsized":"2024-03-06T07:28:59.382289+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375975+0000","last_change":"2024-03-06T07:28:59.375975+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375975+0000","last_undegraded":"2024-03-06T07:28:59.375975+0000","last_fullsized":"2024-03-06T07:28:59.375975+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.29","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370161+0000","last_change":"2024-03-06T07:28:59.370161+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370161+0000","last_undegraded":"2024-03-06T07:28:59.370161+0000","last_fullsized":"2024-03-06T07:28:59.370161+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.28","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369918+0000","last_change":"2024-03-06T07:28:59.369918+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369918+0000","last_undegraded":"2024-03-06T07:28:59.369918+0000","last_fullsized":"2024-03-06T07:28:59.369918+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.27","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376467+0000","last_change":"2024-03-06T07:28:59.376467+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376467+0000","last_undegraded":"2024-03-06T07:28:59.376467+0000","last_fullsized":"2024-03-06T07:28:59.376467+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.26","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370426+0000","last_change":"2024-03-06T07:28:59.370426+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370426+0000","last_undegraded":"2024-03-06T07:28:59.370426+0000","last_fullsized":"2024-03-06T07:28:59.370426+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.25","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375607+0000","last_change":"2024-03-06T07:28:59.375607+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375607+0000","last_undegraded":"2024-03-06T07:28:59.375607+0000","last_fullsized":"2024-03-06T07:28:59.375607+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.24","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384526+0000","last_change":"2024-03-06T07:28:59.384526+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384526+0000","last_undegraded":"2024-03-06T07:28:59.384526+0000","last_fullsized":"2024-03-06T07:28:59.384526+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.23","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371404+0000","last_change":"2024-03-06T07:28:59.371404+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371404+0000","last_undegraded":"2024-03-06T07:28:59.371404+0000","last_fullsized":"2024-03-06T07:28:59.371404+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.22","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376239+0000","last_change":"2024-03-06T07:28:59.376239+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376239+0000","last_undegraded":"2024-03-06T07:28:59.376239+0000","last_fullsized":"2024-03-06T07:28:59.376239+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.21","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370076+0000","last_change":"2024-03-06T07:28:59.370076+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370076+0000","last_undegraded":"2024-03-06T07:28:59.370076+0000","last_fullsized":"2024-03-06T07:28:59.370076+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.20","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370885+0000","last_change":"2024-03-06T07:28:59.370885+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370885+0000","last_undegraded":"2024-03-06T07:28:59.370885+0000","last_fullsized":"2024-03-06T07:28:59.370885+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375428+0000","last_change":"2024-03-06T07:28:59.375428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375428+0000","last_undegraded":"2024-03-06T07:28:59.375428+0000","last_fullsized":"2024-03-06T07:28:59.375428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398367+0000","last_change":"2024-03-06T07:28:59.394295+0000","last_active":"2024-03-06T07:28:59.398367+0000","last_peered":"2024-03-06T07:28:59.398367+0000","last_clean":"2024-03-06T07:28:59.398367+0000","last_became_active":"2024-03-06T07:28:59.393954+0000","last_became_peered":"2024-03-06T07:28:59.393954+0000","last_unstale":"2024-03-06T07:28:59.398367+0000","last_undegraded":"2024-03-06T07:28:59.398367+0000","last_fullsized":"2024-03-06T07:28:59.398367+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:31:25.596355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375212+0000","last_change":"2024-03-06T07:28:59.375212+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375212+0000","last_undegraded":"2024-03-06T07:28:59.375212+0000","last_fullsized":"2024-03-06T07:28:59.375212+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398826+0000","last_change":"2024-03-06T07:28:59.394280+0000","last_active":"2024-03-06T07:28:59.398826+0000","last_peered":"2024-03-06T07:28:59.398826+0000","last_clean":"2024-03-06T07:28:59.398826+0000","last_became_active":"2024-03-06T07:28:59.393796+0000","last_became_peered":"2024-03-06T07:28:59.393796+0000","last_unstale":"2024-03-06T07:28:59.398826+0000","last_undegraded":"2024-03-06T07:28:59.398826+0000","last_fullsized":"2024-03-06T07:28:59.398826+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.1c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384089+0000","last_change":"2024-03-06T07:28:59.384089+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384089+0000","last_undegraded":"2024-03-06T07:28:59.384089+0000","last_fullsized":"2024-03-06T07:28:59.384089+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400925+0000","last_change":"2024-03-06T07:28:59.399825+0000","last_active":"2024-03-06T07:28:59.400925+0000","last_peered":"2024-03-06T07:28:59.400925+0000","last_clean":"2024-03-06T07:28:59.400925+0000","last_became_active":"2024-03-06T07:28:59.399545+0000","last_became_peered":"2024-03-06T07:28:59.399545+0000","last_unstale":"2024-03-06T07:28:59.400925+0000","last_undegraded":"2024-03-06T07:28:59.400925+0000","last_fullsized":"2024-03-06T07:28:59.400925+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374114+0000","last_change":"2024-03-06T07:28:59.374114+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374114+0000","last_undegraded":"2024-03-06T07:28:59.374114+0000","last_fullsized":"2024-03-06T07:28:59.374114+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.403876+0000","last_change":"2024-03-06T07:28:59.400350+0000","last_active":"2024-03-06T07:28:59.403876+0000","last_peered":"2024-03-06T07:28:59.403876+0000","last_clean":"2024-03-06T07:28:59.403876+0000","last_became_active":"2024-03-06T07:28:59.399807+0000","last_became_peered":"2024-03-06T07:28:59.399807+0000","last_unstale":"2024-03-06T07:28:59.403876+0000","last_undegraded":"2024-03-06T07:28:59.403876+0000","last_fullsized":"2024-03-06T07:28:59.403876+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:41:52.673252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.1a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374817+0000","last_change":"2024-03-06T07:28:59.374817+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374817+0000","last_undegraded":"2024-03-06T07:28:59.374817+0000","last_fullsized":"2024-03-06T07:28:59.374817+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392184+0000","last_change":"2024-03-06T07:28:59.389474+0000","last_active":"2024-03-06T07:28:59.392184+0000","last_peered":"2024-03-06T07:28:59.392184+0000","last_clean":"2024-03-06T07:28:59.392184+0000","last_became_active":"2024-03-06T07:28:59.389044+0000","last_became_peered":"2024-03-06T07:28:59.389044+0000","last_unstale":"2024-03-06T07:28:59.392184+0000","last_undegraded":"2024-03-06T07:28:59.392184+0000","last_fullsized":"2024-03-06T07:28:59.392184+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:10:59.969773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378890+0000","last_change":"2024-03-06T07:28:59.378890+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378890+0000","last_undegraded":"2024-03-06T07:28:59.378890+0000","last_fullsized":"2024-03-06T07:28:59.378890+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398793+0000","last_change":"2024-03-06T07:28:59.396587+0000","last_active":"2024-03-06T07:28:59.398793+0000","last_peered":"2024-03-06T07:28:59.398793+0000","last_clean":"2024-03-06T07:28:59.398793+0000","last_became_active":"2024-03-06T07:28:59.396300+0000","last_became_peered":"2024-03-06T07:28:59.396300+0000","last_unstale":"2024-03-06T07:28:59.398793+0000","last_undegraded":"2024-03-06T07:28:59.398793+0000","last_fullsized":"2024-03-06T07:28:59.398793+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376200+0000","last_change":"2024-03-06T07:28:59.376200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376200+0000","last_undegraded":"2024-03-06T07:28:59.376200+0000","last_fullsized":"2024-03-06T07:28:59.376200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399000+0000","last_change":"2024-03-06T07:28:59.396760+0000","last_active":"2024-03-06T07:28:59.399000+0000","last_peered":"2024-03-06T07:28:59.399000+0000","last_clean":"2024-03-06T07:28:59.399000+0000","last_became_active":"2024-03-06T07:28:59.396437+0000","last_became_peered":"2024-03-06T07:28:59.396437+0000","last_unstale":"2024-03-06T07:28:59.399000+0000","last_undegraded":"2024-03-06T07:28:59.399000+0000","last_fullsized":"2024-03-06T07:28:59.399000+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:30.328438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376059+0000","last_change":"2024-03-06T07:28:59.376059+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376059+0000","last_undegraded":"2024-03-06T07:28:59.376059+0000","last_fullsized":"2024-03-06T07:28:59.376059+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.18","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391172+0000","last_change":"2024-03-06T07:28:59.384938+0000","last_active":"2024-03-06T07:28:59.391172+0000","last_peered":"2024-03-06T07:28:59.391172+0000","last_clean":"2024-03-06T07:28:59.391172+0000","last_became_active":"2024-03-06T07:28:59.384506+0000","last_became_peered":"2024-03-06T07:28:59.384506+0000","last_unstale":"2024-03-06T07:28:59.391172+0000","last_undegraded":"2024-03-06T07:28:59.391172+0000","last_fullsized":"2024-03-06T07:28:59.391172+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:35.877223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.16","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375820+0000","last_change":"2024-03-06T07:28:59.375820+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375820+0000","last_undegraded":"2024-03-06T07:28:59.375820+0000","last_fullsized":"2024-03-06T07:28:59.375820+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.17","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392813+0000","last_change":"2024-03-06T07:28:59.390136+0000","last_active":"2024-03-06T07:28:59.392813+0000","last_peered":"2024-03-06T07:28:59.392813+0000","last_clean":"2024-03-06T07:28:59.392813+0000","last_became_active":"2024-03-06T07:28:59.387355+0000","last_became_peered":"2024-03-06T07:28:59.387355+0000","last_unstale":"2024-03-06T07:28:59.392813+0000","last_undegraded":"2024-03-06T07:28:59.392813+0000","last_fullsized":"2024-03-06T07:28:59.392813+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380200+0000","last_change":"2024-03-06T07:28:59.380200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380200+0000","last_undegraded":"2024-03-06T07:28:59.380200+0000","last_fullsized":"2024-03-06T07:28:59.380200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.16","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391374+0000","last_change":"2024-03-06T07:28:59.387731+0000","last_active":"2024-03-06T07:28:59.391374+0000","last_peered":"2024-03-06T07:28:59.391374+0000","last_clean":"2024-03-06T07:28:59.391374+0000","last_became_active":"2024-03-06T07:28:59.387440+0000","last_became_peered":"2024-03-06T07:28:59.387440+0000","last_unstale":"2024-03-06T07:28:59.391374+0000","last_undegraded":"2024-03-06T07:28:59.391374+0000","last_fullsized":"2024-03-06T07:28:59.391374+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:07:04.179315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381152+0000","last_change":"2024-03-06T07:28:59.381152+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381152+0000","last_undegraded":"2024-03-06T07:28:59.381152+0000","last_fullsized":"2024-03-06T07:28:59.381152+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.401208+0000","last_change":"2024-03-06T07:28:59.399848+0000","last_active":"2024-03-06T07:28:59.401208+0000","last_peered":"2024-03-06T07:28:59.401208+0000","last_clean":"2024-03-06T07:28:59.401208+0000","last_became_active":"2024-03-06T07:28:59.399515+0000","last_became_peered":"2024-03-06T07:28:59.399515+0000","last_unstale":"2024-03-06T07:28:59.401208+0000","last_undegraded":"2024-03-06T07:28:59.401208+0000","last_fullsized":"2024-03-06T07:28:59.401208+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:14:43.766404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380656+0000","last_change":"2024-03-06T07:28:59.380656+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380656+0000","last_undegraded":"2024-03-06T07:28:59.380656+0000","last_fullsized":"2024-03-06T07:28:59.380656+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.383916+0000","last_change":"2024-03-06T07:28:59.381955+0000","last_active":"2024-03-06T07:28:59.383916+0000","last_peered":"2024-03-06T07:28:59.383916+0000","last_clean":"2024-03-06T07:28:59.383916+0000","last_became_active":"2024-03-06T07:28:59.381628+0000","last_became_peered":"2024-03-06T07:28:59.381628+0000","last_unstale":"2024-03-06T07:28:59.383916+0000","last_undegraded":"2024-03-06T07:28:59.383916+0000","last_fullsized":"2024-03-06T07:28:59.383916+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:53:05.477963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.12","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372349+0000","last_change":"2024-03-06T07:28:59.372349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372349+0000","last_undegraded":"2024-03-06T07:28:59.372349+0000","last_fullsized":"2024-03-06T07:28:59.372349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392142+0000","last_change":"2024-03-06T07:28:59.388270+0000","last_active":"2024-03-06T07:28:59.392142+0000","last_peered":"2024-03-06T07:28:59.392142+0000","last_clean":"2024-03-06T07:28:59.392142+0000","last_became_active":"2024-03-06T07:28:59.387990+0000","last_became_peered":"2024-03-06T07:28:59.387990+0000","last_unstale":"2024-03-06T07:28:59.392142+0000","last_undegraded":"2024-03-06T07:28:59.392142+0000","last_fullsized":"2024-03-06T07:28:59.392142+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:07.389322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378302+0000","last_change":"2024-03-06T07:28:59.378302+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378302+0000","last_undegraded":"2024-03-06T07:28:59.378302+0000","last_fullsized":"2024-03-06T07:28:59.378302+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.12","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398882+0000","last_change":"2024-03-06T07:28:59.396602+0000","last_active":"2024-03-06T07:28:59.398882+0000","last_peered":"2024-03-06T07:28:59.398882+0000","last_clean":"2024-03-06T07:28:59.398882+0000","last_became_active":"2024-03-06T07:28:59.396165+0000","last_became_peered":"2024-03-06T07:28:59.396165+0000","last_unstale":"2024-03-06T07:28:59.398882+0000","last_undegraded":"2024-03-06T07:28:59.398882+0000","last_fullsized":"2024-03-06T07:28:59.398882+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:55:11.215461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.10","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373888+0000","last_change":"2024-03-06T07:28:59.373888+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373888+0000","last_undegraded":"2024-03-06T07:28:59.373888+0000","last_fullsized":"2024-03-06T07:28:59.373888+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.11","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392124+0000","last_change":"2024-03-06T07:28:59.387732+0000","last_active":"2024-03-06T07:28:59.392124+0000","last_peered":"2024-03-06T07:28:59.392124+0000","last_clean":"2024-03-06T07:28:59.392124+0000","last_became_active":"2024-03-06T07:28:59.387392+0000","last_became_peered":"2024-03-06T07:28:59.387392+0000","last_unstale":"2024-03-06T07:28:59.392124+0000","last_undegraded":"2024-03-06T07:28:59.392124+0000","last_fullsized":"2024-03-06T07:28:59.392124+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:50:58.093927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374592+0000","last_change":"2024-03-06T07:28:59.374592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374592+0000","last_undegraded":"2024-03-06T07:28:59.374592+0000","last_fullsized":"2024-03-06T07:28:59.374592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400169+0000","last_change":"2024-03-06T07:28:59.399605+0000","last_active":"2024-03-06T07:28:59.400169+0000","last_peered":"2024-03-06T07:28:59.400169+0000","last_clean":"2024-03-06T07:28:59.400169+0000","last_became_active":"2024-03-06T07:28:59.398958+0000","last_became_peered":"2024-03-06T07:28:59.398958+0000","last_unstale":"2024-03-06T07:28:59.400169+0000","last_undegraded":"2024-03-06T07:28:59.400169+0000","last_fullsized":"2024-03-06T07:28:59.400169+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369832+0000","last_change":"2024-03-06T07:28:59.369832+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369832+0000","last_undegraded":"2024-03-06T07:28:59.369832+0000","last_fullsized":"2024-03-06T07:28:59.369832+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395429+0000","last_change":"2024-03-06T07:28:59.389645+0000","last_active":"2024-03-06T07:28:59.395429+0000","last_peered":"2024-03-06T07:28:59.395429+0000","last_clean":"2024-03-06T07:28:59.395429+0000","last_became_active":"2024-03-06T07:28:59.389355+0000","last_became_peered":"2024-03-06T07:28:59.389355+0000","last_unstale":"2024-03-06T07:28:59.395429+0000","last_undegraded":"2024-03-06T07:28:59.395429+0000","last_fullsized":"2024-03-06T07:28:59.395429+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:00.206040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376005+0000","last_change":"2024-03-06T07:28:59.376005+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376005+0000","last_undegraded":"2024-03-06T07:28:59.376005+0000","last_fullsized":"2024-03-06T07:28:59.376005+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.390195+0000","last_change":"2024-03-06T07:28:59.386058+0000","last_active":"2024-03-06T07:28:59.390195+0000","last_peered":"2024-03-06T07:28:59.390195+0000","last_clean":"2024-03-06T07:28:59.390195+0000","last_became_active":"2024-03-06T07:28:59.384967+0000","last_became_peered":"2024-03-06T07:28:59.384967+0000","last_unstale":"2024-03-06T07:28:59.390195+0000","last_undegraded":"2024-03-06T07:28:59.390195+0000","last_fullsized":"2024-03-06T07:28:59.390195+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371865+0000","last_change":"2024-03-06T07:28:59.371865+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371865+0000","last_undegraded":"2024-03-06T07:28:59.371865+0000","last_fullsized":"2024-03-06T07:28:59.371865+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398928+0000","last_change":"2024-03-06T07:28:59.397008+0000","last_active":"2024-03-06T07:28:59.398928+0000","last_peered":"2024-03-06T07:28:59.398928+0000","last_clean":"2024-03-06T07:28:59.398928+0000","last_became_active":"2024-03-06T07:28:59.396486+0000","last_became_peered":"2024-03-06T07:28:59.396486+0000","last_unstale":"2024-03-06T07:28:59.398928+0000","last_undegraded":"2024-03-06T07:28:59.398928+0000","last_fullsized":"2024-03-06T07:28:59.398928+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:12.726533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380771+0000","last_change":"2024-03-06T07:28:59.380771+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380771+0000","last_undegraded":"2024-03-06T07:28:59.380771+0000","last_fullsized":"2024-03-06T07:28:59.380771+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398440+0000","last_change":"2024-03-06T07:28:59.394320+0000","last_active":"2024-03-06T07:28:59.398440+0000","last_peered":"2024-03-06T07:28:59.398440+0000","last_clean":"2024-03-06T07:28:59.398440+0000","last_became_active":"2024-03-06T07:28:59.394031+0000","last_became_peered":"2024-03-06T07:28:59.394031+0000","last_unstale":"2024-03-06T07:28:59.398440+0000","last_undegraded":"2024-03-06T07:28:59.398440+0000","last_fullsized":"2024-03-06T07:28:59.398440+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:26.578812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371219+0000","last_change":"2024-03-06T07:28:59.371219+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371219+0000","last_undegraded":"2024-03-06T07:28:59.371219+0000","last_fullsized":"2024-03-06T07:28:59.371219+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392167+0000","last_change":"2024-03-06T07:28:59.389453+0000","last_active":"2024-03-06T07:28:59.392167+0000","last_peered":"2024-03-06T07:28:59.392167+0000","last_clean":"2024-03-06T07:28:59.392167+0000","last_became_active":"2024-03-06T07:28:59.389059+0000","last_became_peered":"2024-03-06T07:28:59.389059+0000","last_unstale":"2024-03-06T07:28:59.392167+0000","last_undegraded":"2024-03-06T07:28:59.392167+0000","last_fullsized":"2024-03-06T07:28:59.392167+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:14.687420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373283+0000","last_change":"2024-03-06T07:28:59.373283+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373283+0000","last_undegraded":"2024-03-06T07:28:59.373283+0000","last_fullsized":"2024-03-06T07:28:59.373283+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399097+0000","last_change":"2024-03-06T07:28:59.397527+0000","last_active":"2024-03-06T07:28:59.399097+0000","last_peered":"2024-03-06T07:28:59.399097+0000","last_clean":"2024-03-06T07:28:59.399097+0000","last_became_active":"2024-03-06T07:28:59.397178+0000","last_became_peered":"2024-03-06T07:28:59.397178+0000","last_unstale":"2024-03-06T07:28:59.399097+0000","last_undegraded":"2024-03-06T07:28:59.399097+0000","last_fullsized":"2024-03-06T07:28:59.399097+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:25.019648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369845+0000","last_change":"2024-03-06T07:28:59.369845+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369845+0000","last_undegraded":"2024-03-06T07:28:59.369845+0000","last_fullsized":"2024-03-06T07:28:59.369845+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391255+0000","last_change":"2024-03-06T07:28:59.387387+0000","last_active":"2024-03-06T07:28:59.391255+0000","last_peered":"2024-03-06T07:28:59.391255+0000","last_clean":"2024-03-06T07:28:59.391255+0000","last_became_active":"2024-03-06T07:28:59.387111+0000","last_became_peered":"2024-03-06T07:28:59.387111+0000","last_unstale":"2024-03-06T07:28:59.391255+0000","last_undegraded":"2024-03-06T07:28:59.391255+0000","last_fullsized":"2024-03-06T07:28:59.391255+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:37.347867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380592+0000","last_change":"2024-03-06T07:28:59.380592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380592+0000","last_undegraded":"2024-03-06T07:28:59.380592+0000","last_fullsized":"2024-03-06T07:28:59.380592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399304+0000","last_change":"2024-03-06T07:28:59.396862+0000","last_active":"2024-03-06T07:28:59.399304+0000","last_peered":"2024-03-06T07:28:59.399304+0000","last_clean":"2024-03-06T07:28:59.399304+0000","last_became_active":"2024-03-06T07:28:59.396358+0000","last_became_peered":"2024-03-06T07:28:59.396358+0000","last_unstale":"2024-03-06T07:28:59.399304+0000","last_undegraded":"2024-03-06T07:28:59.399304+0000","last_fullsized":"2024-03-06T07:28:59.399304+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:06.154352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373843+0000","last_change":"2024-03-06T07:28:59.373843+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373843+0000","last_undegraded":"2024-03-06T07:28:59.373843+0000","last_fullsized":"2024-03-06T07:28:59.373843+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395679+0000","last_change":"2024-03-06T07:28:59.392267+0000","last_active":"2024-03-06T07:28:59.395679+0000","last_peered":"2024-03-06T07:28:59.395679+0000","last_clean":"2024-03-06T07:28:59.395679+0000","last_became_active":"2024-03-06T07:28:59.391787+0000","last_became_peered":"2024-03-06T07:28:59.391787+0000","last_unstale":"2024-03-06T07:28:59.395679+0000","last_undegraded":"2024-03-06T07:28:59.395679+0000","last_fullsized":"2024-03-06T07:28:59.395679+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:55.602966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375838+0000","last_change":"2024-03-06T07:28:59.375838+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375838+0000","last_undegraded":"2024-03-06T07:28:59.375838+0000","last_fullsized":"2024-03-06T07:28:59.375838+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.404330+0000","last_change":"2024-03-06T07:28:59.401945+0000","last_active":"2024-03-06T07:28:59.404330+0000","last_peered":"2024-03-06T07:28:59.404330+0000","last_clean":"2024-03-06T07:28:59.404330+0000","last_became_active":"2024-03-06T07:28:59.401690+0000","last_became_peered":"2024-03-06T07:28:59.401690+0000","last_unstale":"2024-03-06T07:28:59.404330+0000","last_undegraded":"2024-03-06T07:28:59.404330+0000","last_fullsized":"2024-03-06T07:28:59.404330+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:18:50.468500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.379717+0000","last_change":"2024-03-06T07:28:59.379717+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.379717+0000","last_undegraded":"2024-03-06T07:28:59.379717+0000","last_fullsized":"2024-03-06T07:28:59.379717+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400919+0000","last_change":"2024-03-06T07:28:59.399872+0000","last_active":"2024-03-06T07:28:59.400919+0000","last_peered":"2024-03-06T07:28:59.400919+0000","last_clean":"2024-03-06T07:28:59.400919+0000","last_became_active":"2024-03-06T07:28:59.399598+0000","last_became_peered":"2024-03-06T07:28:59.399598+0000","last_unstale":"2024-03-06T07:28:59.400919+0000","last_undegraded":"2024-03-06T07:28:59.400919+0000","last_fullsized":"2024-03-06T07:28:59.400919+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370762+0000","last_change":"2024-03-06T07:28:59.370762+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370762+0000","last_undegraded":"2024-03-06T07:28:59.370762+0000","last_fullsized":"2024-03-06T07:28:59.370762+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398849+0000","last_change":"2024-03-06T07:28:59.395941+0000","last_active":"2024-03-06T07:28:59.398849+0000","last_peered":"2024-03-06T07:28:59.398849+0000","last_clean":"2024-03-06T07:28:59.398849+0000","last_became_active":"2024-03-06T07:28:59.395072+0000","last_became_peered":"2024-03-06T07:28:59.395072+0000","last_unstale":"2024-03-06T07:28:59.398849+0000","last_undegraded":"2024-03-06T07:28:59.398849+0000","last_fullsized":"2024-03-06T07:28:59.398849+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:49:21.534076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372479+0000","last_change":"2024-03-06T07:28:59.372479+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372479+0000","last_undegraded":"2024-03-06T07:28:59.372479+0000","last_fullsized":"2024-03-06T07:28:59.372479+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398776+0000","last_change":"2024-03-06T07:28:59.394472+0000","last_active":"2024-03-06T07:28:59.398776+0000","last_peered":"2024-03-06T07:28:59.398776+0000","last_clean":"2024-03-06T07:28:59.398776+0000","last_became_active":"2024-03-06T07:28:59.394029+0000","last_became_peered":"2024-03-06T07:28:59.394029+0000","last_unstale":"2024-03-06T07:28:59.398776+0000","last_undegraded":"2024-03-06T07:28:59.398776+0000","last_fullsized":"2024-03-06T07:28:59.398776+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.386812+0000","last_change":"2024-03-06T07:28:59.386812+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.386812+0000","last_undegraded":"2024-03-06T07:28:59.386812+0000","last_fullsized":"2024-03-06T07:28:59.386812+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398202+0000","last_change":"2024-03-06T07:28:59.393871+0000","last_active":"2024-03-06T07:28:59.398202+0000","last_peered":"2024-03-06T07:28:59.398202+0000","last_clean":"2024-03-06T07:28:59.398202+0000","last_became_active":"2024-03-06T07:28:59.393554+0000","last_became_peered":"2024-03-06T07:28:59.393554+0000","last_unstale":"2024-03-06T07:28:59.398202+0000","last_undegraded":"2024-03-06T07:28:59.398202+0000","last_fullsized":"2024-03-06T07:28:59.398202+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374029+0000","last_change":"2024-03-06T07:28:59.374029+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374029+0000","last_undegraded":"2024-03-06T07:28:59.374029+0000","last_fullsized":"2024-03-06T07:28:59.374029+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399185+0000","last_change":"2024-03-06T07:28:59.392741+0000","last_active":"2024-03-06T07:28:59.399185+0000","last_peered":"2024-03-06T07:28:59.399185+0000","last_clean":"2024-03-06T07:28:59.399185+0000","last_became_active":"2024-03-06T07:28:59.392415+0000","last_became_peered":"2024-03-06T07:28:59.392415+0000","last_unstale":"2024-03-06T07:28:59.399185+0000","last_undegraded":"2024-03-06T07:28:59.399185+0000","last_fullsized":"2024-03-06T07:28:59.399185+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376957+0000","last_change":"2024-03-06T07:28:59.376957+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376957+0000","last_undegraded":"2024-03-06T07:28:59.376957+0000","last_fullsized":"2024-03-06T07:28:59.376957+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398958+0000","last_change":"2024-03-06T07:28:59.393942+0000","last_active":"2024-03-06T07:28:59.398958+0000","last_peered":"2024-03-06T07:28:59.398958+0000","last_clean":"2024-03-06T07:28:59.398958+0000","last_became_active":"2024-03-06T07:28:59.393597+0000","last_became_peered":"2024-03-06T07:28:59.393597+0000","last_unstale":"2024-03-06T07:28:59.398958+0000","last_undegraded":"2024-03-06T07:28:59.398958+0000","last_fullsized":"2024-03-06T07:28:59.398958+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:00:31.863535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":208,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.388216+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:28:59.388216+0000","last_peered":"2024-03-06T07:28:59.388216+0000","last_clean":"2024-03-06T07:28:59.388216+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:28:59.388216+0000","last_undegraded":"2024-03-06T07:28:59.388216+0000","last_fullsized":"2024-03-06T07:28:59.388216+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":6,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542172,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":794089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542172,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542171,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542172,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542173,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542170,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32276,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339564,"statfs":{"total":96636764160,"available":96603713536,"internally_reserved":0,"allocated":1118208,"data_stored":800356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:29:04.755 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:29:05.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.000+0000 7f69c7200640 1 Processor -- start 2024-03-06T07:29:05.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.000+0000 7f69c7200640 1 -- start start 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69c7200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c015acb0 0x7f69c00b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69c7200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f69c006ec00 0x7f69c00ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69c7200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f69c015b090 con 0x7f69c00b4680 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69c7200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f69c00b2a30 con 0x7f69c006ec00 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69c7200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f69c00b4360 con 0x7f69c015acb0 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69c5200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f69c006ec00 0x7f69c00ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69b7400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f69c00b4680 0x7f69c006e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35788/0 (socket says 172.21.15.86:35788) 2024-03-06T07:29:05.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.002+0000 7f69b7400640 1 -- 172.21.15.86:0/3014480389 learned_addr learned my addr 172.21.15.86:0/3014480389 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.003+0000 7f69c5200640 1 -- 172.21.15.86:0/3014480389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c015acb0 msgr2=0x7f69c00b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.003+0000 7f69bfe00640 1 --2- 172.21.15.86:0/3014480389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c015acb0 0x7f69c00b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.003+0000 7f69c5200640 1 --2- 172.21.15.86:0/3014480389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c015acb0 0x7f69c00b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.003+0000 7f69bf400640 1 -- 172.21.15.86:0/3014480389 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3779380000 0 0) 0x7f69c015b090 con 0x7f69c00b4680 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.003+0000 7f69c5200640 1 -- 172.21.15.86:0/3014480389 >> v1:172.21.15.86:6789/0 conn(0x7f69c00b4680 legacy=0x7f69c006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.003+0000 7f69c5200640 1 -- 172.21.15.86:0/3014480389 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f69c00b2cf0 con 0x7f69c006ec00 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.004+0000 7f69c5200640 1 --2- 172.21.15.86:0/3014480389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c006ec00 0x7f69c00ccfb0 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f69b8009d90 tx=0x7f69b802beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52f46ffa8ff6e999 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:05.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.004+0000 7f69bf400640 1 -- 172.21.15.86:0/3014480389 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69b8039070 con 0x7f69c006ec00 2024-03-06T07:29:05.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.004+0000 7f69bf400640 1 -- 172.21.15.86:0/3014480389 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f69b802cc60 con 0x7f69c006ec00 2024-03-06T07:29:05.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.004+0000 7f69bf400640 1 -- 172.21.15.86:0/3014480389 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69b8034660 con 0x7f69c006ec00 2024-03-06T07:29:05.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 -- 172.21.15.86:0/3014480389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c006ec00 msgr2=0x7f69c00ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 --2- 172.21.15.86:0/3014480389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c006ec00 0x7f69c00ccfb0 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f69b8009d90 tx=0x7f69b802beb0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 -- 172.21.15.86:0/3014480389 shutdown_connections 2024-03-06T07:29:05.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 --2- 172.21.15.86:0/3014480389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c006ec00 0x7f69c00ccfb0 unknown :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 --2- 172.21.15.86:0/3014480389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c015acb0 0x7f69c00b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 -- 172.21.15.86:0/3014480389 >> 172.21.15.86:0/3014480389 conn(0x7f69c00b1730 msgr2=0x7f69c00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:05.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 -- 172.21.15.86:0/3014480389 shutdown_connections 2024-03-06T07:29:05.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.005+0000 7f69c7200640 1 -- 172.21.15.86:0/3014480389 wait complete. 2024-03-06T07:29:05.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.006+0000 7f69c7200640 1 Processor -- start 2024-03-06T07:29:05.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.007+0000 7f69c7200640 1 -- start start 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.007+0000 7f69c7200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c006ec00 0x7f69c01591d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.007+0000 7f69c7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c00b4680 0x7f69c01f3de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.007+0000 7f69c7200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f69c015acb0 0x7f69c01f4320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.007+0000 7f69c7200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c00b3d10 con 0x7f69c00b4680 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.007+0000 7f69c7200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c00b32b0 con 0x7f69c015acb0 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.007+0000 7f69c7200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f69c00b4010 con 0x7f69c006ec00 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c00b4680 0x7f69c01f3de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c00b4680 0x7f69c01f3de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59926/0 (socket says 172.21.15.86:59926) 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 -- 172.21.15.86:0/3298774381 learned_addr learned my addr 172.21.15.86:0/3298774381 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69bfe00640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c006ec00 0x7f69c01591d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 -- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c006ec00 msgr2=0x7f69c01591d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69c5200640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f69c015acb0 0x7f69c01f4320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c006ec00 0x7f69c01591d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 -- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f69c015acb0 msgr2=0x7f69c01f4320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f69c015acb0 0x7f69c01f4320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 -- 172.21.15.86:0/3298774381 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f69c00ccf70 con 0x7f69c00b4680 2024-03-06T07:29:05.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.008+0000 7f69b7400640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c00b4680 0x7f69c01f3de0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f69a800c4d0 tx=0x7f69a800c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:05.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.009+0000 7f69bcc00640 1 -- 172.21.15.86:0/3298774381 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69a8018070 con 0x7f69c00b4680 2024-03-06T07:29:05.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.009+0000 7f69bcc00640 1 -- 172.21.15.86:0/3298774381 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f69a8007500 con 0x7f69c00b4680 2024-03-06T07:29:05.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.009+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f69c015d560 con 0x7f69c00b4680 2024-03-06T07:29:05.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.009+0000 7f69bcc00640 1 -- 172.21.15.86:0/3298774381 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69a8004a20 con 0x7f69c00b4680 2024-03-06T07:29:05.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.009+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69c015cfa0 con 0x7f69c00b4680 2024-03-06T07:29:05.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.010+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f69c004f330 con 0x7f69c00b4680 2024-03-06T07:29:05.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.011+0000 7f69bcc00640 1 -- 172.21.15.86:0/3298774381 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f69a801b070 con 0x7f69c00b4680 2024-03-06T07:29:05.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.012+0000 7f69bcc00640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f699c06b4d0 0x7f699c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.012+0000 7f69bcc00640 1 -- 172.21.15.86:0/3298774381 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f69a8089520 con 0x7f69c00b4680 2024-03-06T07:29:05.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.012+0000 7f69bfe00640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f699c06b4d0 0x7f699c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.013+0000 7f69bfe00640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f699c06b4d0 0x7f699c06d990 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f69c00af3c0 tx=0x7f69b0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:05.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.016+0000 7f69bcc00640 1 -- 172.21.15.86:0/3298774381 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f69a8058390 con 0x7f69c00b4680 2024-03-06T07:29:05.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.210+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f69c006ce60 con 0x7f699c06b4d0 2024-03-06T07:29:05.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.220+0000 7f69bcc00640 1 -- 172.21.15.86:0/3298774381 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7f69c006ce60 con 0x7f699c06b4d0 2024-03-06T07:29:05.225 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:05.226 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:29:05.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f699c06b4d0 msgr2=0x7f699c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f699c06b4d0 0x7f699c06d990 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f69c00af3c0 tx=0x7f69b0008040 comp rx=0 tx=0).stop 2024-03-06T07:29:05.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c00b4680 msgr2=0x7f69c01f3de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c00b4680 0x7f69c01f3de0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f69a800c4d0 tx=0x7f69a800c990 comp rx=0 tx=0).stop 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 shutdown_connections 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f699c06b4d0 0x7f699c06d990 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f69c015acb0 0x7f69c01f4320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f69c00b4680 0x7f69c01f3de0 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 --2- 172.21.15.86:0/3298774381 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f69c006ec00 0x7f69c01591d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.224+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 >> 172.21.15.86:0/3298774381 conn(0x7f69c00b1730 msgr2=0x7f69c006dc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.225+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 shutdown_connections 2024-03-06T07:29:05.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.225+0000 7f69c7200640 1 -- 172.21.15.86:0/3298774381 wait complete. 2024-03-06T07:29:05.245 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":103,"stamp":"2024-03-06T07:29:04.885500+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":43,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":254672,"kb_used_data":5220,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754720048,"statfs":{"total":773094113280,"available":772833329152,"internally_reserved":0,"allocated":5345280,"data_stored":2841839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001530"},"pg_stats":[{"pgid":"7.3f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381377+0000","last_change":"2024-03-06T07:28:59.381377+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381377+0000","last_undegraded":"2024-03-06T07:28:59.381377+0000","last_fullsized":"2024-03-06T07:28:59.381377+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.383263+0000","last_change":"2024-03-06T07:28:59.383263+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.383263+0000","last_undegraded":"2024-03-06T07:28:59.383263+0000","last_fullsized":"2024-03-06T07:28:59.383263+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381839+0000","last_change":"2024-03-06T07:28:59.381839+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381839+0000","last_undegraded":"2024-03-06T07:28:59.381839+0000","last_fullsized":"2024-03-06T07:28:59.381839+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376428+0000","last_change":"2024-03-06T07:28:59.376428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376428+0000","last_undegraded":"2024-03-06T07:28:59.376428+0000","last_fullsized":"2024-03-06T07:28:59.376428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381584+0000","last_change":"2024-03-06T07:28:59.381584+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381584+0000","last_undegraded":"2024-03-06T07:28:59.381584+0000","last_fullsized":"2024-03-06T07:28:59.381584+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371720+0000","last_change":"2024-03-06T07:28:59.371720+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371720+0000","last_undegraded":"2024-03-06T07:28:59.371720+0000","last_fullsized":"2024-03-06T07:28:59.371720+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.39","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375651+0000","last_change":"2024-03-06T07:28:59.375651+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375651+0000","last_undegraded":"2024-03-06T07:28:59.375651+0000","last_fullsized":"2024-03-06T07:28:59.375651+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.38","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377314+0000","last_change":"2024-03-06T07:28:59.377314+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377314+0000","last_undegraded":"2024-03-06T07:28:59.377314+0000","last_fullsized":"2024-03-06T07:28:59.377314+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.37","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378074+0000","last_change":"2024-03-06T07:28:59.378074+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378074+0000","last_undegraded":"2024-03-06T07:28:59.378074+0000","last_fullsized":"2024-03-06T07:28:59.378074+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.36","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377833+0000","last_change":"2024-03-06T07:28:59.377833+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377833+0000","last_undegraded":"2024-03-06T07:28:59.377833+0000","last_fullsized":"2024-03-06T07:28:59.377833+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.35","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378278+0000","last_change":"2024-03-06T07:28:59.378278+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378278+0000","last_undegraded":"2024-03-06T07:28:59.378278+0000","last_fullsized":"2024-03-06T07:28:59.378278+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.34","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384955+0000","last_change":"2024-03-06T07:28:59.384955+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384955+0000","last_undegraded":"2024-03-06T07:28:59.384955+0000","last_fullsized":"2024-03-06T07:28:59.384955+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.33","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370349+0000","last_change":"2024-03-06T07:28:59.370349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370349+0000","last_undegraded":"2024-03-06T07:28:59.370349+0000","last_fullsized":"2024-03-06T07:28:59.370349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.32","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376562+0000","last_change":"2024-03-06T07:28:59.376562+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376562+0000","last_undegraded":"2024-03-06T07:28:59.376562+0000","last_fullsized":"2024-03-06T07:28:59.376562+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.31","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370581+0000","last_change":"2024-03-06T07:28:59.370581+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370581+0000","last_undegraded":"2024-03-06T07:28:59.370581+0000","last_fullsized":"2024-03-06T07:28:59.370581+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.30","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377473+0000","last_change":"2024-03-06T07:28:59.377473+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377473+0000","last_undegraded":"2024-03-06T07:28:59.377473+0000","last_fullsized":"2024-03-06T07:28:59.377473+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375727+0000","last_change":"2024-03-06T07:28:59.375727+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375727+0000","last_undegraded":"2024-03-06T07:28:59.375727+0000","last_fullsized":"2024-03-06T07:28:59.375727+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382638+0000","last_change":"2024-03-06T07:28:59.382638+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382638+0000","last_undegraded":"2024-03-06T07:28:59.382638+0000","last_fullsized":"2024-03-06T07:28:59.382638+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370250+0000","last_change":"2024-03-06T07:28:59.370250+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370250+0000","last_undegraded":"2024-03-06T07:28:59.370250+0000","last_fullsized":"2024-03-06T07:28:59.370250+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370202+0000","last_change":"2024-03-06T07:28:59.370202+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370202+0000","last_undegraded":"2024-03-06T07:28:59.370202+0000","last_fullsized":"2024-03-06T07:28:59.370202+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382289+0000","last_change":"2024-03-06T07:28:59.382289+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382289+0000","last_undegraded":"2024-03-06T07:28:59.382289+0000","last_fullsized":"2024-03-06T07:28:59.382289+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375975+0000","last_change":"2024-03-06T07:28:59.375975+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375975+0000","last_undegraded":"2024-03-06T07:28:59.375975+0000","last_fullsized":"2024-03-06T07:28:59.375975+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.29","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370161+0000","last_change":"2024-03-06T07:28:59.370161+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370161+0000","last_undegraded":"2024-03-06T07:28:59.370161+0000","last_fullsized":"2024-03-06T07:28:59.370161+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.28","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369918+0000","last_change":"2024-03-06T07:28:59.369918+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369918+0000","last_undegraded":"2024-03-06T07:28:59.369918+0000","last_fullsized":"2024-03-06T07:28:59.369918+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.27","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376467+0000","last_change":"2024-03-06T07:28:59.376467+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376467+0000","last_undegraded":"2024-03-06T07:28:59.376467+0000","last_fullsized":"2024-03-06T07:28:59.376467+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.26","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370426+0000","last_change":"2024-03-06T07:28:59.370426+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370426+0000","last_undegraded":"2024-03-06T07:28:59.370426+0000","last_fullsized":"2024-03-06T07:28:59.370426+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.25","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375607+0000","last_change":"2024-03-06T07:28:59.375607+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375607+0000","last_undegraded":"2024-03-06T07:28:59.375607+0000","last_fullsized":"2024-03-06T07:28:59.375607+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.24","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384526+0000","last_change":"2024-03-06T07:28:59.384526+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384526+0000","last_undegraded":"2024-03-06T07:28:59.384526+0000","last_fullsized":"2024-03-06T07:28:59.384526+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.23","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371404+0000","last_change":"2024-03-06T07:28:59.371404+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371404+0000","last_undegraded":"2024-03-06T07:28:59.371404+0000","last_fullsized":"2024-03-06T07:28:59.371404+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.22","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376239+0000","last_change":"2024-03-06T07:28:59.376239+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376239+0000","last_undegraded":"2024-03-06T07:28:59.376239+0000","last_fullsized":"2024-03-06T07:28:59.376239+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.21","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370076+0000","last_change":"2024-03-06T07:28:59.370076+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370076+0000","last_undegraded":"2024-03-06T07:28:59.370076+0000","last_fullsized":"2024-03-06T07:28:59.370076+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.20","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370885+0000","last_change":"2024-03-06T07:28:59.370885+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370885+0000","last_undegraded":"2024-03-06T07:28:59.370885+0000","last_fullsized":"2024-03-06T07:28:59.370885+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375428+0000","last_change":"2024-03-06T07:28:59.375428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375428+0000","last_undegraded":"2024-03-06T07:28:59.375428+0000","last_fullsized":"2024-03-06T07:28:59.375428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398367+0000","last_change":"2024-03-06T07:28:59.394295+0000","last_active":"2024-03-06T07:28:59.398367+0000","last_peered":"2024-03-06T07:28:59.398367+0000","last_clean":"2024-03-06T07:28:59.398367+0000","last_became_active":"2024-03-06T07:28:59.393954+0000","last_became_peered":"2024-03-06T07:28:59.393954+0000","last_unstale":"2024-03-06T07:28:59.398367+0000","last_undegraded":"2024-03-06T07:28:59.398367+0000","last_fullsized":"2024-03-06T07:28:59.398367+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:31:25.596355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375212+0000","last_change":"2024-03-06T07:28:59.375212+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375212+0000","last_undegraded":"2024-03-06T07:28:59.375212+0000","last_fullsized":"2024-03-06T07:28:59.375212+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398826+0000","last_change":"2024-03-06T07:28:59.394280+0000","last_active":"2024-03-06T07:28:59.398826+0000","last_peered":"2024-03-06T07:28:59.398826+0000","last_clean":"2024-03-06T07:28:59.398826+0000","last_became_active":"2024-03-06T07:28:59.393796+0000","last_became_peered":"2024-03-06T07:28:59.393796+0000","last_unstale":"2024-03-06T07:28:59.398826+0000","last_undegraded":"2024-03-06T07:28:59.398826+0000","last_fullsized":"2024-03-06T07:28:59.398826+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.1c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384089+0000","last_change":"2024-03-06T07:28:59.384089+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384089+0000","last_undegraded":"2024-03-06T07:28:59.384089+0000","last_fullsized":"2024-03-06T07:28:59.384089+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400925+0000","last_change":"2024-03-06T07:28:59.399825+0000","last_active":"2024-03-06T07:28:59.400925+0000","last_peered":"2024-03-06T07:28:59.400925+0000","last_clean":"2024-03-06T07:28:59.400925+0000","last_became_active":"2024-03-06T07:28:59.399545+0000","last_became_peered":"2024-03-06T07:28:59.399545+0000","last_unstale":"2024-03-06T07:28:59.400925+0000","last_undegraded":"2024-03-06T07:28:59.400925+0000","last_fullsized":"2024-03-06T07:28:59.400925+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374114+0000","last_change":"2024-03-06T07:28:59.374114+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374114+0000","last_undegraded":"2024-03-06T07:28:59.374114+0000","last_fullsized":"2024-03-06T07:28:59.374114+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.403876+0000","last_change":"2024-03-06T07:28:59.400350+0000","last_active":"2024-03-06T07:28:59.403876+0000","last_peered":"2024-03-06T07:28:59.403876+0000","last_clean":"2024-03-06T07:28:59.403876+0000","last_became_active":"2024-03-06T07:28:59.399807+0000","last_became_peered":"2024-03-06T07:28:59.399807+0000","last_unstale":"2024-03-06T07:28:59.403876+0000","last_undegraded":"2024-03-06T07:28:59.403876+0000","last_fullsized":"2024-03-06T07:28:59.403876+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:41:52.673252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.1a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374817+0000","last_change":"2024-03-06T07:28:59.374817+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374817+0000","last_undegraded":"2024-03-06T07:28:59.374817+0000","last_fullsized":"2024-03-06T07:28:59.374817+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392184+0000","last_change":"2024-03-06T07:28:59.389474+0000","last_active":"2024-03-06T07:28:59.392184+0000","last_peered":"2024-03-06T07:28:59.392184+0000","last_clean":"2024-03-06T07:28:59.392184+0000","last_became_active":"2024-03-06T07:28:59.389044+0000","last_became_peered":"2024-03-06T07:28:59.389044+0000","last_unstale":"2024-03-06T07:28:59.392184+0000","last_undegraded":"2024-03-06T07:28:59.392184+0000","last_fullsized":"2024-03-06T07:28:59.392184+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:10:59.969773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378890+0000","last_change":"2024-03-06T07:28:59.378890+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378890+0000","last_undegraded":"2024-03-06T07:28:59.378890+0000","last_fullsized":"2024-03-06T07:28:59.378890+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398793+0000","last_change":"2024-03-06T07:28:59.396587+0000","last_active":"2024-03-06T07:28:59.398793+0000","last_peered":"2024-03-06T07:28:59.398793+0000","last_clean":"2024-03-06T07:28:59.398793+0000","last_became_active":"2024-03-06T07:28:59.396300+0000","last_became_peered":"2024-03-06T07:28:59.396300+0000","last_unstale":"2024-03-06T07:28:59.398793+0000","last_undegraded":"2024-03-06T07:28:59.398793+0000","last_fullsized":"2024-03-06T07:28:59.398793+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376200+0000","last_change":"2024-03-06T07:28:59.376200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376200+0000","last_undegraded":"2024-03-06T07:28:59.376200+0000","last_fullsized":"2024-03-06T07:28:59.376200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399000+0000","last_change":"2024-03-06T07:28:59.396760+0000","last_active":"2024-03-06T07:28:59.399000+0000","last_peered":"2024-03-06T07:28:59.399000+0000","last_clean":"2024-03-06T07:28:59.399000+0000","last_became_active":"2024-03-06T07:28:59.396437+0000","last_became_peered":"2024-03-06T07:28:59.396437+0000","last_unstale":"2024-03-06T07:28:59.399000+0000","last_undegraded":"2024-03-06T07:28:59.399000+0000","last_fullsized":"2024-03-06T07:28:59.399000+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:30.328438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376059+0000","last_change":"2024-03-06T07:28:59.376059+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376059+0000","last_undegraded":"2024-03-06T07:28:59.376059+0000","last_fullsized":"2024-03-06T07:28:59.376059+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.18","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391172+0000","last_change":"2024-03-06T07:28:59.384938+0000","last_active":"2024-03-06T07:28:59.391172+0000","last_peered":"2024-03-06T07:28:59.391172+0000","last_clean":"2024-03-06T07:28:59.391172+0000","last_became_active":"2024-03-06T07:28:59.384506+0000","last_became_peered":"2024-03-06T07:28:59.384506+0000","last_unstale":"2024-03-06T07:28:59.391172+0000","last_undegraded":"2024-03-06T07:28:59.391172+0000","last_fullsized":"2024-03-06T07:28:59.391172+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:35.877223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.16","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375820+0000","last_change":"2024-03-06T07:28:59.375820+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375820+0000","last_undegraded":"2024-03-06T07:28:59.375820+0000","last_fullsized":"2024-03-06T07:28:59.375820+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.17","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392813+0000","last_change":"2024-03-06T07:28:59.390136+0000","last_active":"2024-03-06T07:28:59.392813+0000","last_peered":"2024-03-06T07:28:59.392813+0000","last_clean":"2024-03-06T07:28:59.392813+0000","last_became_active":"2024-03-06T07:28:59.387355+0000","last_became_peered":"2024-03-06T07:28:59.387355+0000","last_unstale":"2024-03-06T07:28:59.392813+0000","last_undegraded":"2024-03-06T07:28:59.392813+0000","last_fullsized":"2024-03-06T07:28:59.392813+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380200+0000","last_change":"2024-03-06T07:28:59.380200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380200+0000","last_undegraded":"2024-03-06T07:28:59.380200+0000","last_fullsized":"2024-03-06T07:28:59.380200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.16","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391374+0000","last_change":"2024-03-06T07:28:59.387731+0000","last_active":"2024-03-06T07:28:59.391374+0000","last_peered":"2024-03-06T07:28:59.391374+0000","last_clean":"2024-03-06T07:28:59.391374+0000","last_became_active":"2024-03-06T07:28:59.387440+0000","last_became_peered":"2024-03-06T07:28:59.387440+0000","last_unstale":"2024-03-06T07:28:59.391374+0000","last_undegraded":"2024-03-06T07:28:59.391374+0000","last_fullsized":"2024-03-06T07:28:59.391374+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:07:04.179315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381152+0000","last_change":"2024-03-06T07:28:59.381152+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381152+0000","last_undegraded":"2024-03-06T07:28:59.381152+0000","last_fullsized":"2024-03-06T07:28:59.381152+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.401208+0000","last_change":"2024-03-06T07:28:59.399848+0000","last_active":"2024-03-06T07:28:59.401208+0000","last_peered":"2024-03-06T07:28:59.401208+0000","last_clean":"2024-03-06T07:28:59.401208+0000","last_became_active":"2024-03-06T07:28:59.399515+0000","last_became_peered":"2024-03-06T07:28:59.399515+0000","last_unstale":"2024-03-06T07:28:59.401208+0000","last_undegraded":"2024-03-06T07:28:59.401208+0000","last_fullsized":"2024-03-06T07:28:59.401208+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:14:43.766404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380656+0000","last_change":"2024-03-06T07:28:59.380656+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380656+0000","last_undegraded":"2024-03-06T07:28:59.380656+0000","last_fullsized":"2024-03-06T07:28:59.380656+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.383916+0000","last_change":"2024-03-06T07:28:59.381955+0000","last_active":"2024-03-06T07:28:59.383916+0000","last_peered":"2024-03-06T07:28:59.383916+0000","last_clean":"2024-03-06T07:28:59.383916+0000","last_became_active":"2024-03-06T07:28:59.381628+0000","last_became_peered":"2024-03-06T07:28:59.381628+0000","last_unstale":"2024-03-06T07:28:59.383916+0000","last_undegraded":"2024-03-06T07:28:59.383916+0000","last_fullsized":"2024-03-06T07:28:59.383916+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:53:05.477963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.12","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372349+0000","last_change":"2024-03-06T07:28:59.372349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372349+0000","last_undegraded":"2024-03-06T07:28:59.372349+0000","last_fullsized":"2024-03-06T07:28:59.372349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392142+0000","last_change":"2024-03-06T07:28:59.388270+0000","last_active":"2024-03-06T07:28:59.392142+0000","last_peered":"2024-03-06T07:28:59.392142+0000","last_clean":"2024-03-06T07:28:59.392142+0000","last_became_active":"2024-03-06T07:28:59.387990+0000","last_became_peered":"2024-03-06T07:28:59.387990+0000","last_unstale":"2024-03-06T07:28:59.392142+0000","last_undegraded":"2024-03-06T07:28:59.392142+0000","last_fullsized":"2024-03-06T07:28:59.392142+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:07.389322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378302+0000","last_change":"2024-03-06T07:28:59.378302+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378302+0000","last_undegraded":"2024-03-06T07:28:59.378302+0000","last_fullsized":"2024-03-06T07:28:59.378302+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.12","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398882+0000","last_change":"2024-03-06T07:28:59.396602+0000","last_active":"2024-03-06T07:28:59.398882+0000","last_peered":"2024-03-06T07:28:59.398882+0000","last_clean":"2024-03-06T07:28:59.398882+0000","last_became_active":"2024-03-06T07:28:59.396165+0000","last_became_peered":"2024-03-06T07:28:59.396165+0000","last_unstale":"2024-03-06T07:28:59.398882+0000","last_undegraded":"2024-03-06T07:28:59.398882+0000","last_fullsized":"2024-03-06T07:28:59.398882+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:55:11.215461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.10","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373888+0000","last_change":"2024-03-06T07:28:59.373888+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373888+0000","last_undegraded":"2024-03-06T07:28:59.373888+0000","last_fullsized":"2024-03-06T07:28:59.373888+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.11","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392124+0000","last_change":"2024-03-06T07:28:59.387732+0000","last_active":"2024-03-06T07:28:59.392124+0000","last_peered":"2024-03-06T07:28:59.392124+0000","last_clean":"2024-03-06T07:28:59.392124+0000","last_became_active":"2024-03-06T07:28:59.387392+0000","last_became_peered":"2024-03-06T07:28:59.387392+0000","last_unstale":"2024-03-06T07:28:59.392124+0000","last_undegraded":"2024-03-06T07:28:59.392124+0000","last_fullsized":"2024-03-06T07:28:59.392124+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:50:58.093927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374592+0000","last_change":"2024-03-06T07:28:59.374592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374592+0000","last_undegraded":"2024-03-06T07:28:59.374592+0000","last_fullsized":"2024-03-06T07:28:59.374592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400169+0000","last_change":"2024-03-06T07:28:59.399605+0000","last_active":"2024-03-06T07:28:59.400169+0000","last_peered":"2024-03-06T07:28:59.400169+0000","last_clean":"2024-03-06T07:28:59.400169+0000","last_became_active":"2024-03-06T07:28:59.398958+0000","last_became_peered":"2024-03-06T07:28:59.398958+0000","last_unstale":"2024-03-06T07:28:59.400169+0000","last_undegraded":"2024-03-06T07:28:59.400169+0000","last_fullsized":"2024-03-06T07:28:59.400169+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369832+0000","last_change":"2024-03-06T07:28:59.369832+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369832+0000","last_undegraded":"2024-03-06T07:28:59.369832+0000","last_fullsized":"2024-03-06T07:28:59.369832+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395429+0000","last_change":"2024-03-06T07:28:59.389645+0000","last_active":"2024-03-06T07:28:59.395429+0000","last_peered":"2024-03-06T07:28:59.395429+0000","last_clean":"2024-03-06T07:28:59.395429+0000","last_became_active":"2024-03-06T07:28:59.389355+0000","last_became_peered":"2024-03-06T07:28:59.389355+0000","last_unstale":"2024-03-06T07:28:59.395429+0000","last_undegraded":"2024-03-06T07:28:59.395429+0000","last_fullsized":"2024-03-06T07:28:59.395429+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:00.206040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376005+0000","last_change":"2024-03-06T07:28:59.376005+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376005+0000","last_undegraded":"2024-03-06T07:28:59.376005+0000","last_fullsized":"2024-03-06T07:28:59.376005+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.390195+0000","last_change":"2024-03-06T07:28:59.386058+0000","last_active":"2024-03-06T07:28:59.390195+0000","last_peered":"2024-03-06T07:28:59.390195+0000","last_clean":"2024-03-06T07:28:59.390195+0000","last_became_active":"2024-03-06T07:28:59.384967+0000","last_became_peered":"2024-03-06T07:28:59.384967+0000","last_unstale":"2024-03-06T07:28:59.390195+0000","last_undegraded":"2024-03-06T07:28:59.390195+0000","last_fullsized":"2024-03-06T07:28:59.390195+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371865+0000","last_change":"2024-03-06T07:28:59.371865+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371865+0000","last_undegraded":"2024-03-06T07:28:59.371865+0000","last_fullsized":"2024-03-06T07:28:59.371865+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398928+0000","last_change":"2024-03-06T07:28:59.397008+0000","last_active":"2024-03-06T07:28:59.398928+0000","last_peered":"2024-03-06T07:28:59.398928+0000","last_clean":"2024-03-06T07:28:59.398928+0000","last_became_active":"2024-03-06T07:28:59.396486+0000","last_became_peered":"2024-03-06T07:28:59.396486+0000","last_unstale":"2024-03-06T07:28:59.398928+0000","last_undegraded":"2024-03-06T07:28:59.398928+0000","last_fullsized":"2024-03-06T07:28:59.398928+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:12.726533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380771+0000","last_change":"2024-03-06T07:28:59.380771+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380771+0000","last_undegraded":"2024-03-06T07:28:59.380771+0000","last_fullsized":"2024-03-06T07:28:59.380771+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398440+0000","last_change":"2024-03-06T07:28:59.394320+0000","last_active":"2024-03-06T07:28:59.398440+0000","last_peered":"2024-03-06T07:28:59.398440+0000","last_clean":"2024-03-06T07:28:59.398440+0000","last_became_active":"2024-03-06T07:28:59.394031+0000","last_became_peered":"2024-03-06T07:28:59.394031+0000","last_unstale":"2024-03-06T07:28:59.398440+0000","last_undegraded":"2024-03-06T07:28:59.398440+0000","last_fullsized":"2024-03-06T07:28:59.398440+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:26.578812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371219+0000","last_change":"2024-03-06T07:28:59.371219+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371219+0000","last_undegraded":"2024-03-06T07:28:59.371219+0000","last_fullsized":"2024-03-06T07:28:59.371219+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392167+0000","last_change":"2024-03-06T07:28:59.389453+0000","last_active":"2024-03-06T07:28:59.392167+0000","last_peered":"2024-03-06T07:28:59.392167+0000","last_clean":"2024-03-06T07:28:59.392167+0000","last_became_active":"2024-03-06T07:28:59.389059+0000","last_became_peered":"2024-03-06T07:28:59.389059+0000","last_unstale":"2024-03-06T07:28:59.392167+0000","last_undegraded":"2024-03-06T07:28:59.392167+0000","last_fullsized":"2024-03-06T07:28:59.392167+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:14.687420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373283+0000","last_change":"2024-03-06T07:28:59.373283+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373283+0000","last_undegraded":"2024-03-06T07:28:59.373283+0000","last_fullsized":"2024-03-06T07:28:59.373283+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399097+0000","last_change":"2024-03-06T07:28:59.397527+0000","last_active":"2024-03-06T07:28:59.399097+0000","last_peered":"2024-03-06T07:28:59.399097+0000","last_clean":"2024-03-06T07:28:59.399097+0000","last_became_active":"2024-03-06T07:28:59.397178+0000","last_became_peered":"2024-03-06T07:28:59.397178+0000","last_unstale":"2024-03-06T07:28:59.399097+0000","last_undegraded":"2024-03-06T07:28:59.399097+0000","last_fullsized":"2024-03-06T07:28:59.399097+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:25.019648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369845+0000","last_change":"2024-03-06T07:28:59.369845+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369845+0000","last_undegraded":"2024-03-06T07:28:59.369845+0000","last_fullsized":"2024-03-06T07:28:59.369845+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391255+0000","last_change":"2024-03-06T07:28:59.387387+0000","last_active":"2024-03-06T07:28:59.391255+0000","last_peered":"2024-03-06T07:28:59.391255+0000","last_clean":"2024-03-06T07:28:59.391255+0000","last_became_active":"2024-03-06T07:28:59.387111+0000","last_became_peered":"2024-03-06T07:28:59.387111+0000","last_unstale":"2024-03-06T07:28:59.391255+0000","last_undegraded":"2024-03-06T07:28:59.391255+0000","last_fullsized":"2024-03-06T07:28:59.391255+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:37.347867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380592+0000","last_change":"2024-03-06T07:28:59.380592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380592+0000","last_undegraded":"2024-03-06T07:28:59.380592+0000","last_fullsized":"2024-03-06T07:28:59.380592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399304+0000","last_change":"2024-03-06T07:28:59.396862+0000","last_active":"2024-03-06T07:28:59.399304+0000","last_peered":"2024-03-06T07:28:59.399304+0000","last_clean":"2024-03-06T07:28:59.399304+0000","last_became_active":"2024-03-06T07:28:59.396358+0000","last_became_peered":"2024-03-06T07:28:59.396358+0000","last_unstale":"2024-03-06T07:28:59.399304+0000","last_undegraded":"2024-03-06T07:28:59.399304+0000","last_fullsized":"2024-03-06T07:28:59.399304+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:06.154352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373843+0000","last_change":"2024-03-06T07:28:59.373843+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373843+0000","last_undegraded":"2024-03-06T07:28:59.373843+0000","last_fullsized":"2024-03-06T07:28:59.373843+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395679+0000","last_change":"2024-03-06T07:28:59.392267+0000","last_active":"2024-03-06T07:28:59.395679+0000","last_peered":"2024-03-06T07:28:59.395679+0000","last_clean":"2024-03-06T07:28:59.395679+0000","last_became_active":"2024-03-06T07:28:59.391787+0000","last_became_peered":"2024-03-06T07:28:59.391787+0000","last_unstale":"2024-03-06T07:28:59.395679+0000","last_undegraded":"2024-03-06T07:28:59.395679+0000","last_fullsized":"2024-03-06T07:28:59.395679+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:55.602966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375838+0000","last_change":"2024-03-06T07:28:59.375838+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375838+0000","last_undegraded":"2024-03-06T07:28:59.375838+0000","last_fullsized":"2024-03-06T07:28:59.375838+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.404330+0000","last_change":"2024-03-06T07:28:59.401945+0000","last_active":"2024-03-06T07:28:59.404330+0000","last_peered":"2024-03-06T07:28:59.404330+0000","last_clean":"2024-03-06T07:28:59.404330+0000","last_became_active":"2024-03-06T07:28:59.401690+0000","last_became_peered":"2024-03-06T07:28:59.401690+0000","last_unstale":"2024-03-06T07:28:59.404330+0000","last_undegraded":"2024-03-06T07:28:59.404330+0000","last_fullsized":"2024-03-06T07:28:59.404330+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:18:50.468500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.379717+0000","last_change":"2024-03-06T07:28:59.379717+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.379717+0000","last_undegraded":"2024-03-06T07:28:59.379717+0000","last_fullsized":"2024-03-06T07:28:59.379717+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400919+0000","last_change":"2024-03-06T07:28:59.399872+0000","last_active":"2024-03-06T07:28:59.400919+0000","last_peered":"2024-03-06T07:28:59.400919+0000","last_clean":"2024-03-06T07:28:59.400919+0000","last_became_active":"2024-03-06T07:28:59.399598+0000","last_became_peered":"2024-03-06T07:28:59.399598+0000","last_unstale":"2024-03-06T07:28:59.400919+0000","last_undegraded":"2024-03-06T07:28:59.400919+0000","last_fullsized":"2024-03-06T07:28:59.400919+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370762+0000","last_change":"2024-03-06T07:28:59.370762+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370762+0000","last_undegraded":"2024-03-06T07:28:59.370762+0000","last_fullsized":"2024-03-06T07:28:59.370762+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398849+0000","last_change":"2024-03-06T07:28:59.395941+0000","last_active":"2024-03-06T07:28:59.398849+0000","last_peered":"2024-03-06T07:28:59.398849+0000","last_clean":"2024-03-06T07:28:59.398849+0000","last_became_active":"2024-03-06T07:28:59.395072+0000","last_became_peered":"2024-03-06T07:28:59.395072+0000","last_unstale":"2024-03-06T07:28:59.398849+0000","last_undegraded":"2024-03-06T07:28:59.398849+0000","last_fullsized":"2024-03-06T07:28:59.398849+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:49:21.534076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372479+0000","last_change":"2024-03-06T07:28:59.372479+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372479+0000","last_undegraded":"2024-03-06T07:28:59.372479+0000","last_fullsized":"2024-03-06T07:28:59.372479+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398776+0000","last_change":"2024-03-06T07:28:59.394472+0000","last_active":"2024-03-06T07:28:59.398776+0000","last_peered":"2024-03-06T07:28:59.398776+0000","last_clean":"2024-03-06T07:28:59.398776+0000","last_became_active":"2024-03-06T07:28:59.394029+0000","last_became_peered":"2024-03-06T07:28:59.394029+0000","last_unstale":"2024-03-06T07:28:59.398776+0000","last_undegraded":"2024-03-06T07:28:59.398776+0000","last_fullsized":"2024-03-06T07:28:59.398776+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.386812+0000","last_change":"2024-03-06T07:28:59.386812+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.386812+0000","last_undegraded":"2024-03-06T07:28:59.386812+0000","last_fullsized":"2024-03-06T07:28:59.386812+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398202+0000","last_change":"2024-03-06T07:28:59.393871+0000","last_active":"2024-03-06T07:28:59.398202+0000","last_peered":"2024-03-06T07:28:59.398202+0000","last_clean":"2024-03-06T07:28:59.398202+0000","last_became_active":"2024-03-06T07:28:59.393554+0000","last_became_peered":"2024-03-06T07:28:59.393554+0000","last_unstale":"2024-03-06T07:28:59.398202+0000","last_undegraded":"2024-03-06T07:28:59.398202+0000","last_fullsized":"2024-03-06T07:28:59.398202+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374029+0000","last_change":"2024-03-06T07:28:59.374029+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374029+0000","last_undegraded":"2024-03-06T07:28:59.374029+0000","last_fullsized":"2024-03-06T07:28:59.374029+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399185+0000","last_change":"2024-03-06T07:28:59.392741+0000","last_active":"2024-03-06T07:28:59.399185+0000","last_peered":"2024-03-06T07:28:59.399185+0000","last_clean":"2024-03-06T07:28:59.399185+0000","last_became_active":"2024-03-06T07:28:59.392415+0000","last_became_peered":"2024-03-06T07:28:59.392415+0000","last_unstale":"2024-03-06T07:28:59.399185+0000","last_undegraded":"2024-03-06T07:28:59.399185+0000","last_fullsized":"2024-03-06T07:28:59.399185+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376957+0000","last_change":"2024-03-06T07:28:59.376957+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376957+0000","last_undegraded":"2024-03-06T07:28:59.376957+0000","last_fullsized":"2024-03-06T07:28:59.376957+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398958+0000","last_change":"2024-03-06T07:28:59.393942+0000","last_active":"2024-03-06T07:28:59.398958+0000","last_peered":"2024-03-06T07:28:59.398958+0000","last_clean":"2024-03-06T07:28:59.398958+0000","last_became_active":"2024-03-06T07:28:59.393597+0000","last_became_peered":"2024-03-06T07:28:59.393597+0000","last_unstale":"2024-03-06T07:28:59.398958+0000","last_undegraded":"2024-03-06T07:28:59.398958+0000","last_fullsized":"2024-03-06T07:28:59.398958+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:00:31.863535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":208,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.388216+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:28:59.388216+0000","last_peered":"2024-03-06T07:28:59.388216+0000","last_clean":"2024-03-06T07:28:59.388216+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:28:59.388216+0000","last_undegraded":"2024-03-06T07:28:59.388216+0000","last_fullsized":"2024-03-06T07:28:59.388216+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":6,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542172,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":794089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542172,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542171,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542172,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542173,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542170,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32276,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339564,"statfs":{"total":96636764160,"available":96603713536,"internally_reserved":0,"allocated":1118208,"data_stored":800356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:29:05.249 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:29:05.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.499+0000 7fbc57e00640 1 Processor -- start 2024-03-06T07:29:05.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.499+0000 7fbc57e00640 1 -- start start 2024-03-06T07:29:05.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.500+0000 7fbc57e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbc5815a940 0x7fbc580b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.500+0000 7fbc57e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbc580cc2a0 0x7fbc580cc720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.500+0000 7fbc57e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbc580b2b30 con 0x7fbc580cf370 2024-03-06T07:29:05.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.500+0000 7fbc57e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbc580b28e0 con 0x7fbc580cc2a0 2024-03-06T07:29:05.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.500+0000 7fbc57e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbc580b2e10 con 0x7fbc5815a940 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc57400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbc580cc2a0 0x7fbc580cc720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc57400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbc580cc2a0 0x7fbc580cc720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59936/0 (socket says 172.21.15.86:59936) 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc57400640 1 -- 172.21.15.86:0/2082663626 learned_addr learned my addr 172.21.15.86:0/2082663626 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc56a00640 1 --2- 172.21.15.86:0/2082663626 >> v2:172.21.15.194:3300/0 conn(0x7fbc5815a940 0x7fbc580b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc57400640 1 -- 172.21.15.86:0/2082663626 >> v2:172.21.15.194:3300/0 conn(0x7fbc5815a940 msgr2=0x7fbc580b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc57400640 1 --2- 172.21.15.86:0/2082663626 >> v2:172.21.15.194:3300/0 conn(0x7fbc5815a940 0x7fbc580b45e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc57400640 1 -- 172.21.15.86:0/2082663626 >> v1:172.21.15.194:6789/0 conn(0x7fbc580cf370 legacy=0x7fbc5806d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:05.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.501+0000 7fbc57400640 1 -- 172.21.15.86:0/2082663626 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc5806ca50 con 0x7fbc580cc2a0 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.502+0000 7fbc57400640 1 --2- 172.21.15.86:0/2082663626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc580cc720 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7fbc58068510 tx=0x7fbc5002e0f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4d480658b21274f3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc56000640 1 -- 172.21.15.86:0/2082663626 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc5002ea90 con 0x7fbc580cc2a0 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc56000640 1 -- 172.21.15.86:0/2082663626 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc5002ec30 con 0x7fbc580cc2a0 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc56000640 1 -- 172.21.15.86:0/2082663626 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc50009600 con 0x7fbc580cc2a0 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 -- 172.21.15.86:0/2082663626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 msgr2=0x7fbc580cc720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 --2- 172.21.15.86:0/2082663626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc580cc720 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7fbc58068510 tx=0x7fbc5002e0f0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 -- 172.21.15.86:0/2082663626 shutdown_connections 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 --2- 172.21.15.86:0/2082663626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc580cc720 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 --2- 172.21.15.86:0/2082663626 >> v2:172.21.15.194:3300/0 conn(0x7fbc5815a940 0x7fbc580b45e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 -- 172.21.15.86:0/2082663626 >> 172.21.15.86:0/2082663626 conn(0x7fbc5806afc0 msgr2=0x7fbc5806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 -- 172.21.15.86:0/2082663626 shutdown_connections 2024-03-06T07:29:05.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.503+0000 7fbc57e00640 1 -- 172.21.15.86:0/2082663626 wait complete. 2024-03-06T07:29:05.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.505+0000 7fbc57e00640 1 Processor -- start 2024-03-06T07:29:05.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.505+0000 7fbc57e00640 1 -- start start 2024-03-06T07:29:05.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc57e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc581599f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc57e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc580cf370 0x7fbc581e9580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc57e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc5815a940 0x7fbc581e9ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc57e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc580b4b20 con 0x7fbc580cc2a0 2024-03-06T07:29:05.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc57e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc580b30f0 con 0x7fbc5815a940 2024-03-06T07:29:05.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc57e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbc5815ad20 con 0x7fbc580cf370 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc56a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc581599f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc56a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc581599f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:59948/0 (socket says 172.21.15.86:59948) 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.506+0000 7fbc56a00640 1 -- 172.21.15.86:0/3942667533 learned_addr learned my addr 172.21.15.86:0/3942667533 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc56a00640 1 -- 172.21.15.86:0/3942667533 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc580cf370 msgr2=0x7fbc581e9580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc56a00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc580cf370 0x7fbc581e9580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc56a00640 1 -- 172.21.15.86:0/3942667533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc5815a940 msgr2=0x7fbc581e9ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc56a00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc5815a940 0x7fbc581e9ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc56a00640 1 -- 172.21.15.86:0/3942667533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc58159f30 con 0x7fbc580cc2a0 2024-03-06T07:29:05.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc56a00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc581599f0 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7fbc4800e9b0 tx=0x7fbc4800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:05.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc4f400640 1 -- 172.21.15.86:0/3942667533 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc48017070 con 0x7fbc580cc2a0 2024-03-06T07:29:05.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc4f400640 1 -- 172.21.15.86:0/3942667533 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc4800c9a0 con 0x7fbc580cc2a0 2024-03-06T07:29:05.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc4f400640 1 -- 172.21.15.86:0/3942667533 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc4800cd90 con 0x7fbc580cc2a0 2024-03-06T07:29:05.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc581f7730 con 0x7fbc580cc2a0 2024-03-06T07:29:05.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.507+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc581f79d0 con 0x7fbc580cc2a0 2024-03-06T07:29:05.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.508+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc580b3bc0 con 0x7fbc580cc2a0 2024-03-06T07:29:05.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.509+0000 7fbc4f400640 1 -- 172.21.15.86:0/3942667533 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbc4801a070 con 0x7fbc580cc2a0 2024-03-06T07:29:05.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.511+0000 7fbc4f400640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc3406b4d0 0x7fbc3406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.511+0000 7fbc4f400640 1 -- 172.21.15.86:0/3942667533 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbc4808a290 con 0x7fbc580cc2a0 2024-03-06T07:29:05.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.511+0000 7fbc4e000640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc3406b4d0 0x7fbc3406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.511+0000 7fbc4e000640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc3406b4d0 0x7fbc3406d990 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fbc580d0900 tx=0x7fbc400079b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:05.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.515+0000 7fbc4f400640 1 -- 172.21.15.86:0/3942667533 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbc580b3bc0 con 0x7fbc580cc2a0 2024-03-06T07:29:05.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.707+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fbc5804f330 con 0x7fbc3406b4d0 2024-03-06T07:29:05.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.717+0000 7fbc4f400640 1 -- 172.21.15.86:0/3942667533 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7fbc5804f330 con 0x7fbc3406b4d0 2024-03-06T07:29:05.721 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:05.723 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:29:05.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc3406b4d0 msgr2=0x7fbc3406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc3406b4d0 0x7fbc3406d990 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fbc580d0900 tx=0x7fbc400079b0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 msgr2=0x7fbc581599f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc581599f0 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7fbc4800e9b0 tx=0x7fbc4800ee70 comp rx=0 tx=0).stop 2024-03-06T07:29:05.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 shutdown_connections 2024-03-06T07:29:05.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc3406b4d0 0x7fbc3406d990 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc5815a940 0x7fbc581e9ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc580cf370 0x7fbc581e9580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 --2- 172.21.15.86:0/3942667533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc580cc2a0 0x7fbc581599f0 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.723+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 >> 172.21.15.86:0/3942667533 conn(0x7fbc5806afc0 msgr2=0x7fbc58157680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:05.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.724+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 shutdown_connections 2024-03-06T07:29:05.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.724+0000 7fbc57e00640 1 -- 172.21.15.86:0/3942667533 wait complete. 2024-03-06T07:29:05.743 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":103,"stamp":"2024-03-06T07:29:04.885500+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":43,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":254672,"kb_used_data":5220,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754720048,"statfs":{"total":773094113280,"available":772833329152,"internally_reserved":0,"allocated":5345280,"data_stored":2841839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001530"},"pg_stats":[{"pgid":"7.3f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381377+0000","last_change":"2024-03-06T07:28:59.381377+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381377+0000","last_undegraded":"2024-03-06T07:28:59.381377+0000","last_fullsized":"2024-03-06T07:28:59.381377+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.383263+0000","last_change":"2024-03-06T07:28:59.383263+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.383263+0000","last_undegraded":"2024-03-06T07:28:59.383263+0000","last_fullsized":"2024-03-06T07:28:59.383263+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381839+0000","last_change":"2024-03-06T07:28:59.381839+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381839+0000","last_undegraded":"2024-03-06T07:28:59.381839+0000","last_fullsized":"2024-03-06T07:28:59.381839+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376428+0000","last_change":"2024-03-06T07:28:59.376428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376428+0000","last_undegraded":"2024-03-06T07:28:59.376428+0000","last_fullsized":"2024-03-06T07:28:59.376428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381584+0000","last_change":"2024-03-06T07:28:59.381584+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381584+0000","last_undegraded":"2024-03-06T07:28:59.381584+0000","last_fullsized":"2024-03-06T07:28:59.381584+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371720+0000","last_change":"2024-03-06T07:28:59.371720+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371720+0000","last_undegraded":"2024-03-06T07:28:59.371720+0000","last_fullsized":"2024-03-06T07:28:59.371720+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.39","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375651+0000","last_change":"2024-03-06T07:28:59.375651+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375651+0000","last_undegraded":"2024-03-06T07:28:59.375651+0000","last_fullsized":"2024-03-06T07:28:59.375651+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.38","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377314+0000","last_change":"2024-03-06T07:28:59.377314+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377314+0000","last_undegraded":"2024-03-06T07:28:59.377314+0000","last_fullsized":"2024-03-06T07:28:59.377314+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.37","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378074+0000","last_change":"2024-03-06T07:28:59.378074+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378074+0000","last_undegraded":"2024-03-06T07:28:59.378074+0000","last_fullsized":"2024-03-06T07:28:59.378074+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.36","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377833+0000","last_change":"2024-03-06T07:28:59.377833+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377833+0000","last_undegraded":"2024-03-06T07:28:59.377833+0000","last_fullsized":"2024-03-06T07:28:59.377833+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.35","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378278+0000","last_change":"2024-03-06T07:28:59.378278+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378278+0000","last_undegraded":"2024-03-06T07:28:59.378278+0000","last_fullsized":"2024-03-06T07:28:59.378278+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.34","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384955+0000","last_change":"2024-03-06T07:28:59.384955+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384955+0000","last_undegraded":"2024-03-06T07:28:59.384955+0000","last_fullsized":"2024-03-06T07:28:59.384955+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.33","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370349+0000","last_change":"2024-03-06T07:28:59.370349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370349+0000","last_undegraded":"2024-03-06T07:28:59.370349+0000","last_fullsized":"2024-03-06T07:28:59.370349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.32","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376562+0000","last_change":"2024-03-06T07:28:59.376562+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376562+0000","last_undegraded":"2024-03-06T07:28:59.376562+0000","last_fullsized":"2024-03-06T07:28:59.376562+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.31","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370581+0000","last_change":"2024-03-06T07:28:59.370581+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370581+0000","last_undegraded":"2024-03-06T07:28:59.370581+0000","last_fullsized":"2024-03-06T07:28:59.370581+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.30","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377473+0000","last_change":"2024-03-06T07:28:59.377473+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377473+0000","last_undegraded":"2024-03-06T07:28:59.377473+0000","last_fullsized":"2024-03-06T07:28:59.377473+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375727+0000","last_change":"2024-03-06T07:28:59.375727+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375727+0000","last_undegraded":"2024-03-06T07:28:59.375727+0000","last_fullsized":"2024-03-06T07:28:59.375727+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382638+0000","last_change":"2024-03-06T07:28:59.382638+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382638+0000","last_undegraded":"2024-03-06T07:28:59.382638+0000","last_fullsized":"2024-03-06T07:28:59.382638+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370250+0000","last_change":"2024-03-06T07:28:59.370250+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370250+0000","last_undegraded":"2024-03-06T07:28:59.370250+0000","last_fullsized":"2024-03-06T07:28:59.370250+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370202+0000","last_change":"2024-03-06T07:28:59.370202+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370202+0000","last_undegraded":"2024-03-06T07:28:59.370202+0000","last_fullsized":"2024-03-06T07:28:59.370202+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382289+0000","last_change":"2024-03-06T07:28:59.382289+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382289+0000","last_undegraded":"2024-03-06T07:28:59.382289+0000","last_fullsized":"2024-03-06T07:28:59.382289+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375975+0000","last_change":"2024-03-06T07:28:59.375975+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375975+0000","last_undegraded":"2024-03-06T07:28:59.375975+0000","last_fullsized":"2024-03-06T07:28:59.375975+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.29","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370161+0000","last_change":"2024-03-06T07:28:59.370161+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370161+0000","last_undegraded":"2024-03-06T07:28:59.370161+0000","last_fullsized":"2024-03-06T07:28:59.370161+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.28","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369918+0000","last_change":"2024-03-06T07:28:59.369918+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369918+0000","last_undegraded":"2024-03-06T07:28:59.369918+0000","last_fullsized":"2024-03-06T07:28:59.369918+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.27","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376467+0000","last_change":"2024-03-06T07:28:59.376467+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376467+0000","last_undegraded":"2024-03-06T07:28:59.376467+0000","last_fullsized":"2024-03-06T07:28:59.376467+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.26","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370426+0000","last_change":"2024-03-06T07:28:59.370426+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370426+0000","last_undegraded":"2024-03-06T07:28:59.370426+0000","last_fullsized":"2024-03-06T07:28:59.370426+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.25","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375607+0000","last_change":"2024-03-06T07:28:59.375607+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375607+0000","last_undegraded":"2024-03-06T07:28:59.375607+0000","last_fullsized":"2024-03-06T07:28:59.375607+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.24","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384526+0000","last_change":"2024-03-06T07:28:59.384526+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384526+0000","last_undegraded":"2024-03-06T07:28:59.384526+0000","last_fullsized":"2024-03-06T07:28:59.384526+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.23","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371404+0000","last_change":"2024-03-06T07:28:59.371404+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371404+0000","last_undegraded":"2024-03-06T07:28:59.371404+0000","last_fullsized":"2024-03-06T07:28:59.371404+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.22","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376239+0000","last_change":"2024-03-06T07:28:59.376239+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376239+0000","last_undegraded":"2024-03-06T07:28:59.376239+0000","last_fullsized":"2024-03-06T07:28:59.376239+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.21","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370076+0000","last_change":"2024-03-06T07:28:59.370076+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370076+0000","last_undegraded":"2024-03-06T07:28:59.370076+0000","last_fullsized":"2024-03-06T07:28:59.370076+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.20","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370885+0000","last_change":"2024-03-06T07:28:59.370885+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370885+0000","last_undegraded":"2024-03-06T07:28:59.370885+0000","last_fullsized":"2024-03-06T07:28:59.370885+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375428+0000","last_change":"2024-03-06T07:28:59.375428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375428+0000","last_undegraded":"2024-03-06T07:28:59.375428+0000","last_fullsized":"2024-03-06T07:28:59.375428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398367+0000","last_change":"2024-03-06T07:28:59.394295+0000","last_active":"2024-03-06T07:28:59.398367+0000","last_peered":"2024-03-06T07:28:59.398367+0000","last_clean":"2024-03-06T07:28:59.398367+0000","last_became_active":"2024-03-06T07:28:59.393954+0000","last_became_peered":"2024-03-06T07:28:59.393954+0000","last_unstale":"2024-03-06T07:28:59.398367+0000","last_undegraded":"2024-03-06T07:28:59.398367+0000","last_fullsized":"2024-03-06T07:28:59.398367+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:31:25.596355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375212+0000","last_change":"2024-03-06T07:28:59.375212+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375212+0000","last_undegraded":"2024-03-06T07:28:59.375212+0000","last_fullsized":"2024-03-06T07:28:59.375212+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398826+0000","last_change":"2024-03-06T07:28:59.394280+0000","last_active":"2024-03-06T07:28:59.398826+0000","last_peered":"2024-03-06T07:28:59.398826+0000","last_clean":"2024-03-06T07:28:59.398826+0000","last_became_active":"2024-03-06T07:28:59.393796+0000","last_became_peered":"2024-03-06T07:28:59.393796+0000","last_unstale":"2024-03-06T07:28:59.398826+0000","last_undegraded":"2024-03-06T07:28:59.398826+0000","last_fullsized":"2024-03-06T07:28:59.398826+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.1c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384089+0000","last_change":"2024-03-06T07:28:59.384089+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384089+0000","last_undegraded":"2024-03-06T07:28:59.384089+0000","last_fullsized":"2024-03-06T07:28:59.384089+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400925+0000","last_change":"2024-03-06T07:28:59.399825+0000","last_active":"2024-03-06T07:28:59.400925+0000","last_peered":"2024-03-06T07:28:59.400925+0000","last_clean":"2024-03-06T07:28:59.400925+0000","last_became_active":"2024-03-06T07:28:59.399545+0000","last_became_peered":"2024-03-06T07:28:59.399545+0000","last_unstale":"2024-03-06T07:28:59.400925+0000","last_undegraded":"2024-03-06T07:28:59.400925+0000","last_fullsized":"2024-03-06T07:28:59.400925+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374114+0000","last_change":"2024-03-06T07:28:59.374114+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374114+0000","last_undegraded":"2024-03-06T07:28:59.374114+0000","last_fullsized":"2024-03-06T07:28:59.374114+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.403876+0000","last_change":"2024-03-06T07:28:59.400350+0000","last_active":"2024-03-06T07:28:59.403876+0000","last_peered":"2024-03-06T07:28:59.403876+0000","last_clean":"2024-03-06T07:28:59.403876+0000","last_became_active":"2024-03-06T07:28:59.399807+0000","last_became_peered":"2024-03-06T07:28:59.399807+0000","last_unstale":"2024-03-06T07:28:59.403876+0000","last_undegraded":"2024-03-06T07:28:59.403876+0000","last_fullsized":"2024-03-06T07:28:59.403876+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:41:52.673252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.1a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374817+0000","last_change":"2024-03-06T07:28:59.374817+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374817+0000","last_undegraded":"2024-03-06T07:28:59.374817+0000","last_fullsized":"2024-03-06T07:28:59.374817+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392184+0000","last_change":"2024-03-06T07:28:59.389474+0000","last_active":"2024-03-06T07:28:59.392184+0000","last_peered":"2024-03-06T07:28:59.392184+0000","last_clean":"2024-03-06T07:28:59.392184+0000","last_became_active":"2024-03-06T07:28:59.389044+0000","last_became_peered":"2024-03-06T07:28:59.389044+0000","last_unstale":"2024-03-06T07:28:59.392184+0000","last_undegraded":"2024-03-06T07:28:59.392184+0000","last_fullsized":"2024-03-06T07:28:59.392184+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:10:59.969773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378890+0000","last_change":"2024-03-06T07:28:59.378890+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378890+0000","last_undegraded":"2024-03-06T07:28:59.378890+0000","last_fullsized":"2024-03-06T07:28:59.378890+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398793+0000","last_change":"2024-03-06T07:28:59.396587+0000","last_active":"2024-03-06T07:28:59.398793+0000","last_peered":"2024-03-06T07:28:59.398793+0000","last_clean":"2024-03-06T07:28:59.398793+0000","last_became_active":"2024-03-06T07:28:59.396300+0000","last_became_peered":"2024-03-06T07:28:59.396300+0000","last_unstale":"2024-03-06T07:28:59.398793+0000","last_undegraded":"2024-03-06T07:28:59.398793+0000","last_fullsized":"2024-03-06T07:28:59.398793+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376200+0000","last_change":"2024-03-06T07:28:59.376200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376200+0000","last_undegraded":"2024-03-06T07:28:59.376200+0000","last_fullsized":"2024-03-06T07:28:59.376200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399000+0000","last_change":"2024-03-06T07:28:59.396760+0000","last_active":"2024-03-06T07:28:59.399000+0000","last_peered":"2024-03-06T07:28:59.399000+0000","last_clean":"2024-03-06T07:28:59.399000+0000","last_became_active":"2024-03-06T07:28:59.396437+0000","last_became_peered":"2024-03-06T07:28:59.396437+0000","last_unstale":"2024-03-06T07:28:59.399000+0000","last_undegraded":"2024-03-06T07:28:59.399000+0000","last_fullsized":"2024-03-06T07:28:59.399000+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:30.328438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376059+0000","last_change":"2024-03-06T07:28:59.376059+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376059+0000","last_undegraded":"2024-03-06T07:28:59.376059+0000","last_fullsized":"2024-03-06T07:28:59.376059+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.18","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391172+0000","last_change":"2024-03-06T07:28:59.384938+0000","last_active":"2024-03-06T07:28:59.391172+0000","last_peered":"2024-03-06T07:28:59.391172+0000","last_clean":"2024-03-06T07:28:59.391172+0000","last_became_active":"2024-03-06T07:28:59.384506+0000","last_became_peered":"2024-03-06T07:28:59.384506+0000","last_unstale":"2024-03-06T07:28:59.391172+0000","last_undegraded":"2024-03-06T07:28:59.391172+0000","last_fullsized":"2024-03-06T07:28:59.391172+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:35.877223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.16","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375820+0000","last_change":"2024-03-06T07:28:59.375820+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375820+0000","last_undegraded":"2024-03-06T07:28:59.375820+0000","last_fullsized":"2024-03-06T07:28:59.375820+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.17","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392813+0000","last_change":"2024-03-06T07:28:59.390136+0000","last_active":"2024-03-06T07:28:59.392813+0000","last_peered":"2024-03-06T07:28:59.392813+0000","last_clean":"2024-03-06T07:28:59.392813+0000","last_became_active":"2024-03-06T07:28:59.387355+0000","last_became_peered":"2024-03-06T07:28:59.387355+0000","last_unstale":"2024-03-06T07:28:59.392813+0000","last_undegraded":"2024-03-06T07:28:59.392813+0000","last_fullsized":"2024-03-06T07:28:59.392813+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380200+0000","last_change":"2024-03-06T07:28:59.380200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380200+0000","last_undegraded":"2024-03-06T07:28:59.380200+0000","last_fullsized":"2024-03-06T07:28:59.380200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.16","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391374+0000","last_change":"2024-03-06T07:28:59.387731+0000","last_active":"2024-03-06T07:28:59.391374+0000","last_peered":"2024-03-06T07:28:59.391374+0000","last_clean":"2024-03-06T07:28:59.391374+0000","last_became_active":"2024-03-06T07:28:59.387440+0000","last_became_peered":"2024-03-06T07:28:59.387440+0000","last_unstale":"2024-03-06T07:28:59.391374+0000","last_undegraded":"2024-03-06T07:28:59.391374+0000","last_fullsized":"2024-03-06T07:28:59.391374+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:07:04.179315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381152+0000","last_change":"2024-03-06T07:28:59.381152+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381152+0000","last_undegraded":"2024-03-06T07:28:59.381152+0000","last_fullsized":"2024-03-06T07:28:59.381152+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.401208+0000","last_change":"2024-03-06T07:28:59.399848+0000","last_active":"2024-03-06T07:28:59.401208+0000","last_peered":"2024-03-06T07:28:59.401208+0000","last_clean":"2024-03-06T07:28:59.401208+0000","last_became_active":"2024-03-06T07:28:59.399515+0000","last_became_peered":"2024-03-06T07:28:59.399515+0000","last_unstale":"2024-03-06T07:28:59.401208+0000","last_undegraded":"2024-03-06T07:28:59.401208+0000","last_fullsized":"2024-03-06T07:28:59.401208+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:14:43.766404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380656+0000","last_change":"2024-03-06T07:28:59.380656+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380656+0000","last_undegraded":"2024-03-06T07:28:59.380656+0000","last_fullsized":"2024-03-06T07:28:59.380656+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.383916+0000","last_change":"2024-03-06T07:28:59.381955+0000","last_active":"2024-03-06T07:28:59.383916+0000","last_peered":"2024-03-06T07:28:59.383916+0000","last_clean":"2024-03-06T07:28:59.383916+0000","last_became_active":"2024-03-06T07:28:59.381628+0000","last_became_peered":"2024-03-06T07:28:59.381628+0000","last_unstale":"2024-03-06T07:28:59.383916+0000","last_undegraded":"2024-03-06T07:28:59.383916+0000","last_fullsized":"2024-03-06T07:28:59.383916+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:53:05.477963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.12","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372349+0000","last_change":"2024-03-06T07:28:59.372349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372349+0000","last_undegraded":"2024-03-06T07:28:59.372349+0000","last_fullsized":"2024-03-06T07:28:59.372349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392142+0000","last_change":"2024-03-06T07:28:59.388270+0000","last_active":"2024-03-06T07:28:59.392142+0000","last_peered":"2024-03-06T07:28:59.392142+0000","last_clean":"2024-03-06T07:28:59.392142+0000","last_became_active":"2024-03-06T07:28:59.387990+0000","last_became_peered":"2024-03-06T07:28:59.387990+0000","last_unstale":"2024-03-06T07:28:59.392142+0000","last_undegraded":"2024-03-06T07:28:59.392142+0000","last_fullsized":"2024-03-06T07:28:59.392142+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:07.389322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378302+0000","last_change":"2024-03-06T07:28:59.378302+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378302+0000","last_undegraded":"2024-03-06T07:28:59.378302+0000","last_fullsized":"2024-03-06T07:28:59.378302+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.12","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398882+0000","last_change":"2024-03-06T07:28:59.396602+0000","last_active":"2024-03-06T07:28:59.398882+0000","last_peered":"2024-03-06T07:28:59.398882+0000","last_clean":"2024-03-06T07:28:59.398882+0000","last_became_active":"2024-03-06T07:28:59.396165+0000","last_became_peered":"2024-03-06T07:28:59.396165+0000","last_unstale":"2024-03-06T07:28:59.398882+0000","last_undegraded":"2024-03-06T07:28:59.398882+0000","last_fullsized":"2024-03-06T07:28:59.398882+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:55:11.215461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.10","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373888+0000","last_change":"2024-03-06T07:28:59.373888+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373888+0000","last_undegraded":"2024-03-06T07:28:59.373888+0000","last_fullsized":"2024-03-06T07:28:59.373888+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.11","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392124+0000","last_change":"2024-03-06T07:28:59.387732+0000","last_active":"2024-03-06T07:28:59.392124+0000","last_peered":"2024-03-06T07:28:59.392124+0000","last_clean":"2024-03-06T07:28:59.392124+0000","last_became_active":"2024-03-06T07:28:59.387392+0000","last_became_peered":"2024-03-06T07:28:59.387392+0000","last_unstale":"2024-03-06T07:28:59.392124+0000","last_undegraded":"2024-03-06T07:28:59.392124+0000","last_fullsized":"2024-03-06T07:28:59.392124+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:50:58.093927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374592+0000","last_change":"2024-03-06T07:28:59.374592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374592+0000","last_undegraded":"2024-03-06T07:28:59.374592+0000","last_fullsized":"2024-03-06T07:28:59.374592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400169+0000","last_change":"2024-03-06T07:28:59.399605+0000","last_active":"2024-03-06T07:28:59.400169+0000","last_peered":"2024-03-06T07:28:59.400169+0000","last_clean":"2024-03-06T07:28:59.400169+0000","last_became_active":"2024-03-06T07:28:59.398958+0000","last_became_peered":"2024-03-06T07:28:59.398958+0000","last_unstale":"2024-03-06T07:28:59.400169+0000","last_undegraded":"2024-03-06T07:28:59.400169+0000","last_fullsized":"2024-03-06T07:28:59.400169+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369832+0000","last_change":"2024-03-06T07:28:59.369832+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369832+0000","last_undegraded":"2024-03-06T07:28:59.369832+0000","last_fullsized":"2024-03-06T07:28:59.369832+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395429+0000","last_change":"2024-03-06T07:28:59.389645+0000","last_active":"2024-03-06T07:28:59.395429+0000","last_peered":"2024-03-06T07:28:59.395429+0000","last_clean":"2024-03-06T07:28:59.395429+0000","last_became_active":"2024-03-06T07:28:59.389355+0000","last_became_peered":"2024-03-06T07:28:59.389355+0000","last_unstale":"2024-03-06T07:28:59.395429+0000","last_undegraded":"2024-03-06T07:28:59.395429+0000","last_fullsized":"2024-03-06T07:28:59.395429+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:00.206040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376005+0000","last_change":"2024-03-06T07:28:59.376005+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376005+0000","last_undegraded":"2024-03-06T07:28:59.376005+0000","last_fullsized":"2024-03-06T07:28:59.376005+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.390195+0000","last_change":"2024-03-06T07:28:59.386058+0000","last_active":"2024-03-06T07:28:59.390195+0000","last_peered":"2024-03-06T07:28:59.390195+0000","last_clean":"2024-03-06T07:28:59.390195+0000","last_became_active":"2024-03-06T07:28:59.384967+0000","last_became_peered":"2024-03-06T07:28:59.384967+0000","last_unstale":"2024-03-06T07:28:59.390195+0000","last_undegraded":"2024-03-06T07:28:59.390195+0000","last_fullsized":"2024-03-06T07:28:59.390195+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371865+0000","last_change":"2024-03-06T07:28:59.371865+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371865+0000","last_undegraded":"2024-03-06T07:28:59.371865+0000","last_fullsized":"2024-03-06T07:28:59.371865+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398928+0000","last_change":"2024-03-06T07:28:59.397008+0000","last_active":"2024-03-06T07:28:59.398928+0000","last_peered":"2024-03-06T07:28:59.398928+0000","last_clean":"2024-03-06T07:28:59.398928+0000","last_became_active":"2024-03-06T07:28:59.396486+0000","last_became_peered":"2024-03-06T07:28:59.396486+0000","last_unstale":"2024-03-06T07:28:59.398928+0000","last_undegraded":"2024-03-06T07:28:59.398928+0000","last_fullsized":"2024-03-06T07:28:59.398928+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:12.726533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380771+0000","last_change":"2024-03-06T07:28:59.380771+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380771+0000","last_undegraded":"2024-03-06T07:28:59.380771+0000","last_fullsized":"2024-03-06T07:28:59.380771+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398440+0000","last_change":"2024-03-06T07:28:59.394320+0000","last_active":"2024-03-06T07:28:59.398440+0000","last_peered":"2024-03-06T07:28:59.398440+0000","last_clean":"2024-03-06T07:28:59.398440+0000","last_became_active":"2024-03-06T07:28:59.394031+0000","last_became_peered":"2024-03-06T07:28:59.394031+0000","last_unstale":"2024-03-06T07:28:59.398440+0000","last_undegraded":"2024-03-06T07:28:59.398440+0000","last_fullsized":"2024-03-06T07:28:59.398440+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:26.578812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371219+0000","last_change":"2024-03-06T07:28:59.371219+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371219+0000","last_undegraded":"2024-03-06T07:28:59.371219+0000","last_fullsized":"2024-03-06T07:28:59.371219+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392167+0000","last_change":"2024-03-06T07:28:59.389453+0000","last_active":"2024-03-06T07:28:59.392167+0000","last_peered":"2024-03-06T07:28:59.392167+0000","last_clean":"2024-03-06T07:28:59.392167+0000","last_became_active":"2024-03-06T07:28:59.389059+0000","last_became_peered":"2024-03-06T07:28:59.389059+0000","last_unstale":"2024-03-06T07:28:59.392167+0000","last_undegraded":"2024-03-06T07:28:59.392167+0000","last_fullsized":"2024-03-06T07:28:59.392167+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:14.687420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373283+0000","last_change":"2024-03-06T07:28:59.373283+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373283+0000","last_undegraded":"2024-03-06T07:28:59.373283+0000","last_fullsized":"2024-03-06T07:28:59.373283+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399097+0000","last_change":"2024-03-06T07:28:59.397527+0000","last_active":"2024-03-06T07:28:59.399097+0000","last_peered":"2024-03-06T07:28:59.399097+0000","last_clean":"2024-03-06T07:28:59.399097+0000","last_became_active":"2024-03-06T07:28:59.397178+0000","last_became_peered":"2024-03-06T07:28:59.397178+0000","last_unstale":"2024-03-06T07:28:59.399097+0000","last_undegraded":"2024-03-06T07:28:59.399097+0000","last_fullsized":"2024-03-06T07:28:59.399097+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:25.019648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369845+0000","last_change":"2024-03-06T07:28:59.369845+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369845+0000","last_undegraded":"2024-03-06T07:28:59.369845+0000","last_fullsized":"2024-03-06T07:28:59.369845+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391255+0000","last_change":"2024-03-06T07:28:59.387387+0000","last_active":"2024-03-06T07:28:59.391255+0000","last_peered":"2024-03-06T07:28:59.391255+0000","last_clean":"2024-03-06T07:28:59.391255+0000","last_became_active":"2024-03-06T07:28:59.387111+0000","last_became_peered":"2024-03-06T07:28:59.387111+0000","last_unstale":"2024-03-06T07:28:59.391255+0000","last_undegraded":"2024-03-06T07:28:59.391255+0000","last_fullsized":"2024-03-06T07:28:59.391255+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:37.347867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380592+0000","last_change":"2024-03-06T07:28:59.380592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380592+0000","last_undegraded":"2024-03-06T07:28:59.380592+0000","last_fullsized":"2024-03-06T07:28:59.380592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399304+0000","last_change":"2024-03-06T07:28:59.396862+0000","last_active":"2024-03-06T07:28:59.399304+0000","last_peered":"2024-03-06T07:28:59.399304+0000","last_clean":"2024-03-06T07:28:59.399304+0000","last_became_active":"2024-03-06T07:28:59.396358+0000","last_became_peered":"2024-03-06T07:28:59.396358+0000","last_unstale":"2024-03-06T07:28:59.399304+0000","last_undegraded":"2024-03-06T07:28:59.399304+0000","last_fullsized":"2024-03-06T07:28:59.399304+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:06.154352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373843+0000","last_change":"2024-03-06T07:28:59.373843+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373843+0000","last_undegraded":"2024-03-06T07:28:59.373843+0000","last_fullsized":"2024-03-06T07:28:59.373843+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395679+0000","last_change":"2024-03-06T07:28:59.392267+0000","last_active":"2024-03-06T07:28:59.395679+0000","last_peered":"2024-03-06T07:28:59.395679+0000","last_clean":"2024-03-06T07:28:59.395679+0000","last_became_active":"2024-03-06T07:28:59.391787+0000","last_became_peered":"2024-03-06T07:28:59.391787+0000","last_unstale":"2024-03-06T07:28:59.395679+0000","last_undegraded":"2024-03-06T07:28:59.395679+0000","last_fullsized":"2024-03-06T07:28:59.395679+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:55.602966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375838+0000","last_change":"2024-03-06T07:28:59.375838+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375838+0000","last_undegraded":"2024-03-06T07:28:59.375838+0000","last_fullsized":"2024-03-06T07:28:59.375838+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.404330+0000","last_change":"2024-03-06T07:28:59.401945+0000","last_active":"2024-03-06T07:28:59.404330+0000","last_peered":"2024-03-06T07:28:59.404330+0000","last_clean":"2024-03-06T07:28:59.404330+0000","last_became_active":"2024-03-06T07:28:59.401690+0000","last_became_peered":"2024-03-06T07:28:59.401690+0000","last_unstale":"2024-03-06T07:28:59.404330+0000","last_undegraded":"2024-03-06T07:28:59.404330+0000","last_fullsized":"2024-03-06T07:28:59.404330+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:18:50.468500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.379717+0000","last_change":"2024-03-06T07:28:59.379717+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.379717+0000","last_undegraded":"2024-03-06T07:28:59.379717+0000","last_fullsized":"2024-03-06T07:28:59.379717+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400919+0000","last_change":"2024-03-06T07:28:59.399872+0000","last_active":"2024-03-06T07:28:59.400919+0000","last_peered":"2024-03-06T07:28:59.400919+0000","last_clean":"2024-03-06T07:28:59.400919+0000","last_became_active":"2024-03-06T07:28:59.399598+0000","last_became_peered":"2024-03-06T07:28:59.399598+0000","last_unstale":"2024-03-06T07:28:59.400919+0000","last_undegraded":"2024-03-06T07:28:59.400919+0000","last_fullsized":"2024-03-06T07:28:59.400919+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370762+0000","last_change":"2024-03-06T07:28:59.370762+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370762+0000","last_undegraded":"2024-03-06T07:28:59.370762+0000","last_fullsized":"2024-03-06T07:28:59.370762+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398849+0000","last_change":"2024-03-06T07:28:59.395941+0000","last_active":"2024-03-06T07:28:59.398849+0000","last_peered":"2024-03-06T07:28:59.398849+0000","last_clean":"2024-03-06T07:28:59.398849+0000","last_became_active":"2024-03-06T07:28:59.395072+0000","last_became_peered":"2024-03-06T07:28:59.395072+0000","last_unstale":"2024-03-06T07:28:59.398849+0000","last_undegraded":"2024-03-06T07:28:59.398849+0000","last_fullsized":"2024-03-06T07:28:59.398849+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:49:21.534076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372479+0000","last_change":"2024-03-06T07:28:59.372479+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372479+0000","last_undegraded":"2024-03-06T07:28:59.372479+0000","last_fullsized":"2024-03-06T07:28:59.372479+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398776+0000","last_change":"2024-03-06T07:28:59.394472+0000","last_active":"2024-03-06T07:28:59.398776+0000","last_peered":"2024-03-06T07:28:59.398776+0000","last_clean":"2024-03-06T07:28:59.398776+0000","last_became_active":"2024-03-06T07:28:59.394029+0000","last_became_peered":"2024-03-06T07:28:59.394029+0000","last_unstale":"2024-03-06T07:28:59.398776+0000","last_undegraded":"2024-03-06T07:28:59.398776+0000","last_fullsized":"2024-03-06T07:28:59.398776+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.386812+0000","last_change":"2024-03-06T07:28:59.386812+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.386812+0000","last_undegraded":"2024-03-06T07:28:59.386812+0000","last_fullsized":"2024-03-06T07:28:59.386812+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398202+0000","last_change":"2024-03-06T07:28:59.393871+0000","last_active":"2024-03-06T07:28:59.398202+0000","last_peered":"2024-03-06T07:28:59.398202+0000","last_clean":"2024-03-06T07:28:59.398202+0000","last_became_active":"2024-03-06T07:28:59.393554+0000","last_became_peered":"2024-03-06T07:28:59.393554+0000","last_unstale":"2024-03-06T07:28:59.398202+0000","last_undegraded":"2024-03-06T07:28:59.398202+0000","last_fullsized":"2024-03-06T07:28:59.398202+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374029+0000","last_change":"2024-03-06T07:28:59.374029+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374029+0000","last_undegraded":"2024-03-06T07:28:59.374029+0000","last_fullsized":"2024-03-06T07:28:59.374029+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399185+0000","last_change":"2024-03-06T07:28:59.392741+0000","last_active":"2024-03-06T07:28:59.399185+0000","last_peered":"2024-03-06T07:28:59.399185+0000","last_clean":"2024-03-06T07:28:59.399185+0000","last_became_active":"2024-03-06T07:28:59.392415+0000","last_became_peered":"2024-03-06T07:28:59.392415+0000","last_unstale":"2024-03-06T07:28:59.399185+0000","last_undegraded":"2024-03-06T07:28:59.399185+0000","last_fullsized":"2024-03-06T07:28:59.399185+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376957+0000","last_change":"2024-03-06T07:28:59.376957+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376957+0000","last_undegraded":"2024-03-06T07:28:59.376957+0000","last_fullsized":"2024-03-06T07:28:59.376957+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398958+0000","last_change":"2024-03-06T07:28:59.393942+0000","last_active":"2024-03-06T07:28:59.398958+0000","last_peered":"2024-03-06T07:28:59.398958+0000","last_clean":"2024-03-06T07:28:59.398958+0000","last_became_active":"2024-03-06T07:28:59.393597+0000","last_became_peered":"2024-03-06T07:28:59.393597+0000","last_unstale":"2024-03-06T07:28:59.398958+0000","last_undegraded":"2024-03-06T07:28:59.398958+0000","last_fullsized":"2024-03-06T07:28:59.398958+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:00:31.863535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":208,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.388216+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:28:59.388216+0000","last_peered":"2024-03-06T07:28:59.388216+0000","last_clean":"2024-03-06T07:28:59.388216+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:28:59.388216+0000","last_undegraded":"2024-03-06T07:28:59.388216+0000","last_fullsized":"2024-03-06T07:28:59.388216+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":6,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542172,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":794089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542172,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542171,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542172,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542173,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542170,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32276,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339564,"statfs":{"total":96636764160,"available":96603713536,"internally_reserved":0,"allocated":1118208,"data_stored":800356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:29:05.747 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.3f is not active+clean 2024-03-06T07:29:05.747 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.3f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.381377+0000', 'last_change': '2024-03-06T07:28:59.381377+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.381377+0000', 'last_undegraded': '2024-03-06T07:28:59.381377+0000', 'last_fullsized': '2024-03-06T07:28:59.381377+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4], 'acting': [7, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-06T07:29:05.747 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.3e is not active+clean 2024-03-06T07:29:05.747 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.3e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.383263+0000', 'last_change': '2024-03-06T07:28:59.383263+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.383263+0000', 'last_undegraded': '2024-03-06T07:28:59.383263+0000', 'last_fullsized': '2024-03-06T07:28:59.383263+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 0], 'acting': [5, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-06T07:29:05.747 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.3d is not active+clean 2024-03-06T07:29:05.747 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.3d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.381839+0000', 'last_change': '2024-03-06T07:28:59.381839+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.381839+0000', 'last_undegraded': '2024-03-06T07:28:59.381839+0000', 'last_fullsized': '2024-03-06T07:28:59.381839+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 2], 'acting': [4, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.747 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.3c is not active+clean 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.3c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376428+0000', 'last_change': '2024-03-06T07:28:59.376428+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376428+0000', 'last_undegraded': '2024-03-06T07:28:59.376428+0000', 'last_fullsized': '2024-03-06T07:28:59.376428+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.3b is not active+clean 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.3b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.381584+0000', 'last_change': '2024-03-06T07:28:59.381584+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.381584+0000', 'last_undegraded': '2024-03-06T07:28:59.381584+0000', 'last_fullsized': '2024-03-06T07:28:59.381584+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 0], 'acting': [5, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.3a is not active+clean 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.3a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.371720+0000', 'last_change': '2024-03-06T07:28:59.371720+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.371720+0000', 'last_undegraded': '2024-03-06T07:28:59.371720+0000', 'last_fullsized': '2024-03-06T07:28:59.371720+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 7], 'acting': [3, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.39 is not active+clean 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.39', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375651+0000', 'last_change': '2024-03-06T07:28:59.375651+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375651+0000', 'last_undegraded': '2024-03-06T07:28:59.375651+0000', 'last_fullsized': '2024-03-06T07:28:59.375651+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 1], 'acting': [7, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.38 is not active+clean 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.38', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.377314+0000', 'last_change': '2024-03-06T07:28:59.377314+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.377314+0000', 'last_undegraded': '2024-03-06T07:28:59.377314+0000', 'last_fullsized': '2024-03-06T07:28:59.377314+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.37 is not active+clean 2024-03-06T07:29:05.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.37', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.378074+0000', 'last_change': '2024-03-06T07:28:59.378074+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.378074+0000', 'last_undegraded': '2024-03-06T07:28:59.378074+0000', 'last_fullsized': '2024-03-06T07:28:59.378074+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 1], 'acting': [0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.36 is not active+clean 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.36', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.377833+0000', 'last_change': '2024-03-06T07:28:59.377833+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.377833+0000', 'last_undegraded': '2024-03-06T07:28:59.377833+0000', 'last_fullsized': '2024-03-06T07:28:59.377833+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.35 is not active+clean 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.35', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.378278+0000', 'last_change': '2024-03-06T07:28:59.378278+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.378278+0000', 'last_undegraded': '2024-03-06T07:28:59.378278+0000', 'last_fullsized': '2024-03-06T07:28:59.378278+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 1], 'acting': [0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.34 is not active+clean 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.34', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.384955+0000', 'last_change': '2024-03-06T07:28:59.384955+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.384955+0000', 'last_undegraded': '2024-03-06T07:28:59.384955+0000', 'last_fullsized': '2024-03-06T07:28:59.384955+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 2], 'acting': [5, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.33 is not active+clean 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.33', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370349+0000', 'last_change': '2024-03-06T07:28:59.370349+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370349+0000', 'last_undegraded': '2024-03-06T07:28:59.370349+0000', 'last_fullsized': '2024-03-06T07:28:59.370349+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 3], 'acting': [2, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.32 is not active+clean 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.32', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376562+0000', 'last_change': '2024-03-06T07:28:59.376562+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376562+0000', 'last_undegraded': '2024-03-06T07:28:59.376562+0000', 'last_fullsized': '2024-03-06T07:28:59.376562+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 1], 'acting': [0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.31 is not active+clean 2024-03-06T07:29:05.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.31', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370581+0000', 'last_change': '2024-03-06T07:28:59.370581+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370581+0000', 'last_undegraded': '2024-03-06T07:28:59.370581+0000', 'last_fullsized': '2024-03-06T07:28:59.370581+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 4], 'acting': [3, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.30 is not active+clean 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.30', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.377473+0000', 'last_change': '2024-03-06T07:28:59.377473+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.377473+0000', 'last_undegraded': '2024-03-06T07:28:59.377473+0000', 'last_fullsized': '2024-03-06T07:28:59.377473+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0], 'acting': [6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.2f is not active+clean 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.2f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375727+0000', 'last_change': '2024-03-06T07:28:59.375727+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375727+0000', 'last_undegraded': '2024-03-06T07:28:59.375727+0000', 'last_fullsized': '2024-03-06T07:28:59.375727+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 5], 'acting': [4, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.2e is not active+clean 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.2e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.382638+0000', 'last_change': '2024-03-06T07:28:59.382638+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.382638+0000', 'last_undegraded': '2024-03-06T07:28:59.382638+0000', 'last_fullsized': '2024-03-06T07:28:59.382638+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 0], 'acting': [4, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.2d is not active+clean 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.2d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370250+0000', 'last_change': '2024-03-06T07:28:59.370250+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370250+0000', 'last_undegraded': '2024-03-06T07:28:59.370250+0000', 'last_fullsized': '2024-03-06T07:28:59.370250+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0], 'acting': [6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.2c is not active+clean 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.2c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370202+0000', 'last_change': '2024-03-06T07:28:59.370202+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370202+0000', 'last_undegraded': '2024-03-06T07:28:59.370202+0000', 'last_fullsized': '2024-03-06T07:28:59.370202+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 0], 'acting': [2, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-06T07:29:05.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.2b is not active+clean 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.2b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.382289+0000', 'last_change': '2024-03-06T07:28:59.382289+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.382289+0000', 'last_undegraded': '2024-03-06T07:28:59.382289+0000', 'last_fullsized': '2024-03-06T07:28:59.382289+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 6], 'acting': [3, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.2a is not active+clean 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.2a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375975+0000', 'last_change': '2024-03-06T07:28:59.375975+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375975+0000', 'last_undegraded': '2024-03-06T07:28:59.375975+0000', 'last_fullsized': '2024-03-06T07:28:59.375975+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.29 is not active+clean 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.29', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370161+0000', 'last_change': '2024-03-06T07:28:59.370161+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370161+0000', 'last_undegraded': '2024-03-06T07:28:59.370161+0000', 'last_fullsized': '2024-03-06T07:28:59.370161+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 7], 'acting': [6, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.28 is not active+clean 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.28', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.369918+0000', 'last_change': '2024-03-06T07:28:59.369918+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.369918+0000', 'last_undegraded': '2024-03-06T07:28:59.369918+0000', 'last_fullsized': '2024-03-06T07:28:59.369918+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 7], 'acting': [6, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.27 is not active+clean 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.27', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376467+0000', 'last_change': '2024-03-06T07:28:59.376467+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376467+0000', 'last_undegraded': '2024-03-06T07:28:59.376467+0000', 'last_fullsized': '2024-03-06T07:28:59.376467+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 1], 'acting': [6, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.26 is not active+clean 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.26', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370426+0000', 'last_change': '2024-03-06T07:28:59.370426+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370426+0000', 'last_undegraded': '2024-03-06T07:28:59.370426+0000', 'last_fullsized': '2024-03-06T07:28:59.370426+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 3], 'acting': [2, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.25 is not active+clean 2024-03-06T07:29:05.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.25', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375607+0000', 'last_change': '2024-03-06T07:28:59.375607+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375607+0000', 'last_undegraded': '2024-03-06T07:28:59.375607+0000', 'last_fullsized': '2024-03-06T07:28:59.375607+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 5], 'acting': [4, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.24 is not active+clean 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.24', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.384526+0000', 'last_change': '2024-03-06T07:28:59.384526+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.384526+0000', 'last_undegraded': '2024-03-06T07:28:59.384526+0000', 'last_fullsized': '2024-03-06T07:28:59.384526+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 2], 'acting': [5, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.23 is not active+clean 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.23', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.371404+0000', 'last_change': '2024-03-06T07:28:59.371404+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.371404+0000', 'last_undegraded': '2024-03-06T07:28:59.371404+0000', 'last_fullsized': '2024-03-06T07:28:59.371404+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 1], 'acting': [6, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.22 is not active+clean 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.22', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376239+0000', 'last_change': '2024-03-06T07:28:59.376239+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376239+0000', 'last_undegraded': '2024-03-06T07:28:59.376239+0000', 'last_fullsized': '2024-03-06T07:28:59.376239+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 7], 'acting': [4, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.21 is not active+clean 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.21', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370076+0000', 'last_change': '2024-03-06T07:28:59.370076+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370076+0000', 'last_undegraded': '2024-03-06T07:28:59.370076+0000', 'last_fullsized': '2024-03-06T07:28:59.370076+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 5], 'acting': [3, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.20 is not active+clean 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.20', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370885+0000', 'last_change': '2024-03-06T07:28:59.370885+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370885+0000', 'last_undegraded': '2024-03-06T07:28:59.370885+0000', 'last_fullsized': '2024-03-06T07:28:59.370885+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-06T07:29:05.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.1e is not active+clean 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.1e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375428+0000', 'last_change': '2024-03-06T07:28:59.375428+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375428+0000', 'last_undegraded': '2024-03-06T07:28:59.375428+0000', 'last_fullsized': '2024-03-06T07:28:59.375428+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 3], 'acting': [1, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.1f is not active+clean 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.1f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375212+0000', 'last_change': '2024-03-06T07:28:59.375212+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375212+0000', 'last_undegraded': '2024-03-06T07:28:59.375212+0000', 'last_fullsized': '2024-03-06T07:28:59.375212+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 4], 'acting': [6, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.1c is not active+clean 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.1c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.384089+0000', 'last_change': '2024-03-06T07:28:59.384089+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.384089+0000', 'last_undegraded': '2024-03-06T07:28:59.384089+0000', 'last_fullsized': '2024-03-06T07:28:59.384089+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 3], 'acting': [5, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.1d is not active+clean 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.1d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.374114+0000', 'last_change': '2024-03-06T07:28:59.374114+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.374114+0000', 'last_undegraded': '2024-03-06T07:28:59.374114+0000', 'last_fullsized': '2024-03-06T07:28:59.374114+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 6], 'acting': [1, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.1a is not active+clean 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.1a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.374817+0000', 'last_change': '2024-03-06T07:28:59.374817+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.374817+0000', 'last_undegraded': '2024-03-06T07:28:59.374817+0000', 'last_fullsized': '2024-03-06T07:28:59.374817+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 6], 'acting': [4, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.1b is not active+clean 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.1b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.378890+0000', 'last_change': '2024-03-06T07:28:59.378890+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.378890+0000', 'last_undegraded': '2024-03-06T07:28:59.378890+0000', 'last_fullsized': '2024-03-06T07:28:59.378890+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 2], 'acting': [0, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.18 is not active+clean 2024-03-06T07:29:05.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.18', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376200+0000', 'last_change': '2024-03-06T07:28:59.376200+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376200+0000', 'last_undegraded': '2024-03-06T07:28:59.376200+0000', 'last_fullsized': '2024-03-06T07:28:59.376200+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.19 is not active+clean 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.19', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376059+0000', 'last_change': '2024-03-06T07:28:59.376059+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376059+0000', 'last_undegraded': '2024-03-06T07:28:59.376059+0000', 'last_fullsized': '2024-03-06T07:28:59.376059+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 7], 'acting': [6, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.16 is not active+clean 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.16', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375820+0000', 'last_change': '2024-03-06T07:28:59.375820+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375820+0000', 'last_undegraded': '2024-03-06T07:28:59.375820+0000', 'last_fullsized': '2024-03-06T07:28:59.375820+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 0], 'acting': [4, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.17 is not active+clean 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.17', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.380200+0000', 'last_change': '2024-03-06T07:28:59.380200+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.380200+0000', 'last_undegraded': '2024-03-06T07:28:59.380200+0000', 'last_fullsized': '2024-03-06T07:28:59.380200+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 5], 'acting': [7, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.14 is not active+clean 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.14', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.381152+0000', 'last_change': '2024-03-06T07:28:59.381152+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.381152+0000', 'last_undegraded': '2024-03-06T07:28:59.381152+0000', 'last_fullsized': '2024-03-06T07:28:59.381152+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 2], 'acting': [3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.15 is not active+clean 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.15', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.380656+0000', 'last_change': '2024-03-06T07:28:59.380656+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.380656+0000', 'last_undegraded': '2024-03-06T07:28:59.380656+0000', 'last_fullsized': '2024-03-06T07:28:59.380656+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 1], 'acting': [7, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.12 is not active+clean 2024-03-06T07:29:05.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.12', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.372349+0000', 'last_change': '2024-03-06T07:28:59.372349+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.372349+0000', 'last_undegraded': '2024-03-06T07:28:59.372349+0000', 'last_fullsized': '2024-03-06T07:28:59.372349+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0], 'acting': [6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.13 is not active+clean 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.13', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.378302+0000', 'last_change': '2024-03-06T07:28:59.378302+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.378302+0000', 'last_undegraded': '2024-03-06T07:28:59.378302+0000', 'last_fullsized': '2024-03-06T07:28:59.378302+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 7], 'acting': [0, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.10 is not active+clean 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.10', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.373888+0000', 'last_change': '2024-03-06T07:28:59.373888+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.373888+0000', 'last_undegraded': '2024-03-06T07:28:59.373888+0000', 'last_fullsized': '2024-03-06T07:28:59.373888+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 2], 'acting': [3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.11 is not active+clean 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.11', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.374592+0000', 'last_change': '2024-03-06T07:28:59.374592+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.374592+0000', 'last_undegraded': '2024-03-06T07:28:59.374592+0000', 'last_fullsized': '2024-03-06T07:28:59.374592+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 6], 'acting': [7, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.e is not active+clean 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.369832+0000', 'last_change': '2024-03-06T07:28:59.369832+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.369832+0000', 'last_undegraded': '2024-03-06T07:28:59.369832+0000', 'last_fullsized': '2024-03-06T07:28:59.369832+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 5], 'acting': [2, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.f is not active+clean 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376005+0000', 'last_change': '2024-03-06T07:28:59.376005+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376005+0000', 'last_undegraded': '2024-03-06T07:28:59.376005+0000', 'last_fullsized': '2024-03-06T07:28:59.376005+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 3], 'acting': [0, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.c is not active+clean 2024-03-06T07:29:05.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.371865+0000', 'last_change': '2024-03-06T07:28:59.371865+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.371865+0000', 'last_undegraded': '2024-03-06T07:28:59.371865+0000', 'last_fullsized': '2024-03-06T07:28:59.371865+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 3], 'acting': [6, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.d is not active+clean 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.380771+0000', 'last_change': '2024-03-06T07:28:59.380771+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.380771+0000', 'last_undegraded': '2024-03-06T07:28:59.380771+0000', 'last_fullsized': '2024-03-06T07:28:59.380771+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 1], 'acting': [6, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.a is not active+clean 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.371219+0000', 'last_change': '2024-03-06T07:28:59.371219+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.371219+0000', 'last_undegraded': '2024-03-06T07:28:59.371219+0000', 'last_fullsized': '2024-03-06T07:28:59.371219+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 2], 'acting': [3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.b is not active+clean 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.373283+0000', 'last_change': '2024-03-06T07:28:59.373283+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.373283+0000', 'last_undegraded': '2024-03-06T07:28:59.373283+0000', 'last_fullsized': '2024-03-06T07:28:59.373283+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 5], 'acting': [1, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.8 is not active+clean 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.8', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.369845+0000', 'last_change': '2024-03-06T07:28:59.369845+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.369845+0000', 'last_undegraded': '2024-03-06T07:28:59.369845+0000', 'last_fullsized': '2024-03-06T07:28:59.369845+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 7], 'acting': [2, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.9 is not active+clean 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.9', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.380592+0000', 'last_change': '2024-03-06T07:28:59.380592+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.380592+0000', 'last_undegraded': '2024-03-06T07:28:59.380592+0000', 'last_fullsized': '2024-03-06T07:28:59.380592+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 1], 'acting': [0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.7 is not active+clean 2024-03-06T07:29:05.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.7', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.373843+0000', 'last_change': '2024-03-06T07:28:59.373843+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.373843+0000', 'last_undegraded': '2024-03-06T07:28:59.373843+0000', 'last_fullsized': '2024-03-06T07:28:59.373843+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0], 'acting': [6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.4 is not active+clean 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.4', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.375838+0000', 'last_change': '2024-03-06T07:28:59.375838+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.375838+0000', 'last_undegraded': '2024-03-06T07:28:59.375838+0000', 'last_fullsized': '2024-03-06T07:28:59.375838+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 4], 'acting': [0, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.5 is not active+clean 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.5', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.379717+0000', 'last_change': '2024-03-06T07:28:59.379717+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.379717+0000', 'last_undegraded': '2024-03-06T07:28:59.379717+0000', 'last_fullsized': '2024-03-06T07:28:59.379717+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 2], 'acting': [7, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.2 is not active+clean 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.2', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.370762+0000', 'last_change': '2024-03-06T07:28:59.370762+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.370762+0000', 'last_undegraded': '2024-03-06T07:28:59.370762+0000', 'last_fullsized': '2024-03-06T07:28:59.370762+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 1], 'acting': [3, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.3 is not active+clean 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.3', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.372479+0000', 'last_change': '2024-03-06T07:28:59.372479+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.372479+0000', 'last_undegraded': '2024-03-06T07:28:59.372479+0000', 'last_fullsized': '2024-03-06T07:28:59.372479+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 2], 'acting': [3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.0 is not active+clean 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.0', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.386812+0000', 'last_change': '2024-03-06T07:28:59.386812+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.386812+0000', 'last_undegraded': '2024-03-06T07:28:59.386812+0000', 'last_fullsized': '2024-03-06T07:28:59.386812+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 1], 'acting': [5, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.1 is not active+clean 2024-03-06T07:29:05.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.1', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.374029+0000', 'last_change': '2024-03-06T07:28:59.374029+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.374029+0000', 'last_undegraded': '2024-03-06T07:28:59.374029+0000', 'last_fullsized': '2024-03-06T07:28:59.374029+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 3], 'acting': [4, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-06T07:29:05.758 INFO:tasks.cephfs.filesystem.ceph_manager:PG 7.6 is not active+clean 2024-03-06T07:29:05.758 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '7.6', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 45, 'state': 'creating+peering', 'last_fresh': '2024-03-06T07:28:59.376957+0000', 'last_change': '2024-03-06T07:28:59.376957+0000', 'last_active': '2024-03-06T07:28:59.344665+0000', 'last_peered': '2024-03-06T07:28:59.344665+0000', 'last_clean': '2024-03-06T07:28:59.344665+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-06T07:28:59.376957+0000', 'last_undegraded': '2024-03-06T07:28:59.376957+0000', 'last_fullsized': '2024-03-06T07:28:59.376957+0000', 'mapping_epoch': 45, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 45, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'last_clean_scrub_stamp': '2024-03-06T07:28:59.344665+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 7], 'acting': [0, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-06T07:29:05.758 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json 2024-03-06T07:29:05.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.991+0000 7fae31a00640 1 Processor -- start 2024-03-06T07:29:05.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.991+0000 7fae31a00640 1 -- start start 2024-03-06T07:29:05.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae31a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fae2c071d20 0x7fae2c0b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae31a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c0b5310 0x7fae2c0ce070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:05.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae31a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fae2c071280 con 0x7fae2c1566a0 2024-03-06T07:29:05.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae31a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fae2c0b5780 con 0x7fae2c071d20 2024-03-06T07:29:05.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae31a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fae2c072df0 con 0x7fae2c0b5310 2024-03-06T07:29:05.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae23e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fae2c071d20 0x7fae2c0b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae2b400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c0b5310 0x7fae2c0ce070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:05.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae23e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fae2c071d20 0x7fae2c0b4dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:35326/0 (socket says 172.21.15.86:35326) 2024-03-06T07:29:05.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.993+0000 7fae23e00640 1 -- 172.21.15.86:0/3051416039 learned_addr learned my addr 172.21.15.86:0/3051416039 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:05.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.994+0000 7fae2b400640 1 -- 172.21.15.86:0/3051416039 >> v2:172.21.15.194:3300/0 conn(0x7fae2c071d20 msgr2=0x7fae2c0b4dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:05.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.994+0000 7fae2b400640 1 --2- 172.21.15.86:0/3051416039 >> v2:172.21.15.194:3300/0 conn(0x7fae2c071d20 0x7fae2c0b4dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.994+0000 7fae2b400640 1 -- 172.21.15.86:0/3051416039 >> v1:172.21.15.194:6789/0 conn(0x7fae2c1566a0 legacy=0x7fae2c15ce60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.994+0000 7fae2b400640 1 -- 172.21.15.86:0/3051416039 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae2c0ccbd0 con 0x7fae2c0b5310 2024-03-06T07:29:05.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.995+0000 7fae2b400640 1 --2- 172.21.15.86:0/3051416039 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c0b5310 0x7fae2c0ce070 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fae24009d90 tx=0x7fae2402bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aca4e1a947257421 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:05.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.996+0000 7fae2a000640 1 -- 172.21.15.86:0/3051416039 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae24039070 con 0x7fae2c0b5310 2024-03-06T07:29:05.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.996+0000 7fae2a000640 1 -- 172.21.15.86:0/3051416039 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fae2402cc50 con 0x7fae2c0b5310 2024-03-06T07:29:05.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.996+0000 7fae2a000640 1 -- 172.21.15.86:0/3051416039 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae24034440 con 0x7fae2c0b5310 2024-03-06T07:29:05.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 -- 172.21.15.86:0/3051416039 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c0b5310 msgr2=0x7fae2c0ce070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:05.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 --2- 172.21.15.86:0/3051416039 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c0b5310 0x7fae2c0ce070 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fae24009d90 tx=0x7fae2402bea0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 -- 172.21.15.86:0/3051416039 shutdown_connections 2024-03-06T07:29:05.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 --2- 172.21.15.86:0/3051416039 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c0b5310 0x7fae2c0ce070 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 --2- 172.21.15.86:0/3051416039 >> v2:172.21.15.194:3300/0 conn(0x7fae2c071d20 0x7fae2c0b4dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:05.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 -- 172.21.15.86:0/3051416039 >> 172.21.15.86:0/3051416039 conn(0x7fae2c0cfa30 msgr2=0x7fae2c0cfe40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:05.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 -- 172.21.15.86:0/3051416039 shutdown_connections 2024-03-06T07:29:05.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.997+0000 7fae31a00640 1 -- 172.21.15.86:0/3051416039 wait complete. 2024-03-06T07:29:06.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.998+0000 7fae31a00640 1 Processor -- start 2024-03-06T07:29:06.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.999+0000 7fae31a00640 1 -- start start 2024-03-06T07:29:06.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.999+0000 7fae31a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae2c071d20 0x7fae2c1f84c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.999+0000 7fae31a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae2c0b5310 0x7fae2c0adf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.999+0000 7fae31a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c1566a0 0x7fae2c1f3990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.999+0000 7fae31a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae2c071800 con 0x7fae2c0b5310 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.999+0000 7fae31a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae2c071520 con 0x7fae2c071d20 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:05.999+0000 7fae31a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fae2c0cc4a0 con 0x7fae2c1566a0 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae2c0b5310 0x7fae2c0adf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae2c0b5310 0x7fae2c0adf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38018/0 (socket says 172.21.15.86:38018) 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 -- 172.21.15.86:0/22440831 learned_addr learned my addr 172.21.15.86:0/22440831 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae2aa00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae2c071d20 0x7fae2c1f84c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae2b400640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c1566a0 0x7fae2c1f3990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 -- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c1566a0 msgr2=0x7fae2c1f3990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c1566a0 0x7fae2c1f3990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 -- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae2c071d20 msgr2=0x7fae2c1f84c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae2c071d20 0x7fae2c1f84c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 -- 172.21.15.86:0/22440831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae2c0ae4a0 con 0x7fae2c0b5310 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae2b400640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c1566a0 0x7fae2c1f3990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:06.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.000+0000 7fae23e00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae2c0b5310 0x7fae2c0adf60 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7fae14009fd0 tx=0x7fae1400ed40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:06.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.001+0000 7fae22a00640 1 -- 172.21.15.86:0/22440831 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae14017070 con 0x7fae2c0b5310 2024-03-06T07:29:06.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.001+0000 7fae22a00640 1 -- 172.21.15.86:0/22440831 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fae1400c940 con 0x7fae2c0b5310 2024-03-06T07:29:06.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.001+0000 7fae22a00640 1 -- 172.21.15.86:0/22440831 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae1400cd10 con 0x7fae2c0b5310 2024-03-06T07:29:06.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.001+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fae2c0ae770 con 0x7fae2c0b5310 2024-03-06T07:29:06.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.001+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae2c0aed30 con 0x7fae2c0b5310 2024-03-06T07:29:06.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.002+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fae2c04f330 con 0x7fae2c0b5310 2024-03-06T07:29:06.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.003+0000 7fae22a00640 1 -- 172.21.15.86:0/22440831 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fae1401a070 con 0x7fae2c0b5310 2024-03-06T07:29:06.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.004+0000 7fae22a00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae0806b4d0 0x7fae0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.004+0000 7fae22a00640 1 -- 172.21.15.86:0/22440831 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fae1408a100 con 0x7fae2c0b5310 2024-03-06T07:29:06.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.004+0000 7fae2aa00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae0806b4d0 0x7fae0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.004+0000 7fae2aa00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae0806b4d0 0x7fae0806d990 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fae1c007fd0 tx=0x7fae1c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:06.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.008+0000 7fae22a00640 1 -- 172.21.15.86:0/22440831 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fae14059010 con 0x7fae2c0b5310 2024-03-06T07:29:06.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.255+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "status", "format": "json"} v 0) v1 -- 0x7fae2c0af470 con 0x7fae2c0b5310 2024-03-06T07:29:06.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.256+0000 7fae22a00640 1 -- 172.21.15.86:0/22440831 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json"}]=0 v0) v1 ==== 72+0+1453 (secure 0 0 0) 0x7fae14058910 con 0x7fae2c0b5310 2024-03-06T07:29:06.257 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:06.257 INFO:teuthology.orchestra.run.smithi086.stdout:{"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":6,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":138,"monmap":{"epoch":1,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":47,"num_osds":8,"num_up_osds":8,"osd_up_since":1709710019,"num_in_osds":8,"osd_in_since":1709710011,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"creating+peering","count":64},{"state_name":"active+clean","count":33}],"num_pgs":97,"num_pools":3,"num_objects":2,"data_bytes":590368,"bytes_used":260784128,"bytes_avail":772833329152,"bytes_total":773094113280,"inactive_pgs_ratio":0.65979379415512085},"fsmap":{"epoch":25,"id":3,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":3,"rank":0,"name":"b","status":"up:active","gid":5439}],"up:standby":3},"mgrmap":{"available":true,"num_standbys":1,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2024-03-06T07:28:46.881258+0000","services":{"mds":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"b":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"c":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae0806b4d0 msgr2=0x7fae0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae0806b4d0 0x7fae0806d990 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fae1c007fd0 tx=0x7fae1c009040 comp rx=0 tx=0).stop 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae2c0b5310 msgr2=0x7fae2c0adf60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae2c0b5310 0x7fae2c0adf60 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7fae14009fd0 tx=0x7fae1400ed40 comp rx=0 tx=0).stop 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 shutdown_connections 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae0806b4d0 0x7fae0806d990 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae2c1566a0 0x7fae2c1f3990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae2c0b5310 0x7fae2c0adf60 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 --2- 172.21.15.86:0/22440831 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae2c071d20 0x7fae2c1f84c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 >> 172.21.15.86:0/22440831 conn(0x7fae2c0cfa30 msgr2=0x7fae2c15ae90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:06.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 shutdown_connections 2024-03-06T07:29:06.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.258+0000 7fae31a00640 1 -- 172.21.15.86:0/22440831 wait complete. 2024-03-06T07:29:06.278 INFO:tasks.cephfs.filesystem.ceph_manager:no progress seen, keeping timeout for now 2024-03-06T07:29:06.278 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:29:06.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.480+0000 7f7fdf200640 1 Processor -- start 2024-03-06T07:29:06.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.480+0000 7f7fdf200640 1 -- start start 2024-03-06T07:29:06.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.481+0000 7f7fdf200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7fd815f9d0 0x7f7fd815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.481+0000 7f7fdf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 0x7f7fd8070f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.481+0000 7f7fdf200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7fd806cef0 con 0x7f7fd8144c00 2024-03-06T07:29:06.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.481+0000 7f7fdf200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7fd806d1d0 con 0x7f7fd815f9d0 2024-03-06T07:29:06.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.481+0000 7f7fdf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7fd806d4b0 con 0x7f7fd8070b10 2024-03-06T07:29:06.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.481+0000 7f7fd7e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7fd8144c00 0x7f7fd806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59002/0 (socket says 172.21.15.86:59002) 2024-03-06T07:29:06.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.481+0000 7f7fd7e00640 1 -- 172.21.15.86:0/1474621619 learned_addr learned my addr 172.21.15.86:0/1474621619 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fdd200640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 0x7f7fd8070f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd7400640 1 --2- 172.21.15.86:0/1474621619 >> v2:172.21.15.194:3300/0 conn(0x7f7fd815f9d0 0x7f7fd815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2408545706 0 0) 0x7f7fd806cef0 con 0x7f7fd8144c00 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7fbc003520 con 0x7f7fd8144c00 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 282231372 0 0) 0x7f7fbc003520 con 0x7f7fd8144c00 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 msgr2=0x7f7fd8070f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 0x7f7fd8070f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 >> v2:172.21.15.194:3300/0 conn(0x7f7fd815f9d0 msgr2=0x7f7fd815fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:06.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 --2- 172.21.15.86:0/1474621619 >> v2:172.21.15.194:3300/0 conn(0x7f7fd815f9d0 0x7f7fd815fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7fd806d790 con 0x7f7fd8144c00 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7fc8001670 con 0x7f7fd8144c00 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd7400640 1 --2- 172.21.15.86:0/1474621619 >> v2:172.21.15.194:3300/0 conn(0x7f7fd815f9d0 0x7f7fd815fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 >> v1:172.21.15.86:6789/0 conn(0x7f7fd8144c00 legacy=0x7f7fd806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fbc0039b0 0x7f7fbc003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 0x7f7fbc004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc004d40 0x7f7fbc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7fd806d1d0 con 0x7f7fbc0039b0 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7fd806d4b0 con 0x7f7fbc004d40 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.482+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7fbc00b9c0 con 0x7f7fd8070b10 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fd7e00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fbc0039b0 0x7f7fbc003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fdd200640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc004d40 0x7f7fbc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fd7400640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 0x7f7fbc004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fd7e00640 1 -- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 msgr2=0x7f7fbc004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fd7e00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd8070b10 0x7f7fbc004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fd7e00640 1 -- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc004d40 msgr2=0x7f7fbc0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:06.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fd7e00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc004d40 0x7f7fbc0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fd7e00640 1 -- 172.21.15.86:0/1474621619 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7fbc003520 con 0x7f7fbc0039b0 2024-03-06T07:29:06.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.483+0000 7f7fdd200640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc004d40 0x7f7fbc0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:06.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.484+0000 7f7fd7e00640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fbc0039b0 0x7f7fbc003dd0 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f7fc800c240 tx=0x7f7fbc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5b86ad84dfba9bed server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:06.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fc8007d50 con 0x7f7fbc0039b0 2024-03-06T07:29:06.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7fc8030070 con 0x7f7fbc0039b0 2024-03-06T07:29:06.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fd6a00640 1 -- 172.21.15.86:0/1474621619 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fc802cd80 con 0x7f7fbc0039b0 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fdf200640 1 -- 172.21.15.86:0/1474621619 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fbc0039b0 msgr2=0x7f7fbc003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fdf200640 1 --2- 172.21.15.86:0/1474621619 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fbc0039b0 0x7f7fbc003dd0 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f7fc800c240 tx=0x7f7fbc005830 comp rx=0 tx=0).stop 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fdd200640 1 -- 172.21.15.86:0/1474621619 reap_dead start 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fdf200640 1 -- 172.21.15.86:0/1474621619 shutdown_connections 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fdf200640 1 -- 172.21.15.86:0/1474621619 >> 172.21.15.86:0/1474621619 conn(0x7f7fd80b1730 msgr2=0x7f7fd80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fdf200640 1 -- 172.21.15.86:0/1474621619 shutdown_connections 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.485+0000 7f7fdf200640 1 -- 172.21.15.86:0/1474621619 wait complete. 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.487+0000 7f7fdf200640 1 Processor -- start 2024-03-06T07:29:06.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.487+0000 7f7fdf200640 1 -- start start 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fdf200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc0039b0 0x7f7fd80aed90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fdf200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fd815a0d0 0x7f7fd815a550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fdf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd813d840 0x7f7fd813dcc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fdf200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7fd806ccd0 con 0x7f7fd815a0d0 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fdf200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7fd806cb50 con 0x7f7fbc0039b0 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fdf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7fd806df90 con 0x7f7fd813d840 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fd7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fd815a0d0 0x7f7fd815a550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fd7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fd815a0d0 0x7f7fd815a550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38030/0 (socket says 172.21.15.86:38030) 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.488+0000 7f7fd7400640 1 -- 172.21.15.86:0/4261044167 learned_addr learned my addr 172.21.15.86:0/4261044167 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fd7e00640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc0039b0 0x7f7fd80aed90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fdd200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd813d840 0x7f7fd813dcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fd7400640 1 -- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd813d840 msgr2=0x7f7fd813dcc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fd7400640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd813d840 0x7f7fd813dcc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fd7400640 1 -- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc0039b0 msgr2=0x7f7fd80aed90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fd7400640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc0039b0 0x7f7fd80aed90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fd7400640 1 -- 172.21.15.86:0/4261044167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7fd813e420 con 0x7f7fd815a0d0 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fdd200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd813d840 0x7f7fd813dcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fd7400640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fd815a0d0 0x7f7fd815a550 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f7fc400f580 tx=0x7f7fc400fa40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fcfe00640 1 -- 172.21.15.86:0/4261044167 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fc4014890 con 0x7f7fd815a0d0 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fcfe00640 1 -- 172.21.15.86:0/4261044167 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7fc4014a30 con 0x7f7fd815a0d0 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fcfe00640 1 -- 172.21.15.86:0/4261044167 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fc4014de0 con 0x7f7fd815a0d0 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.489+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7fd813e700 con 0x7f7fd815a0d0 2024-03-06T07:29:06.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.490+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7fd81f3b10 con 0x7f7fd815a0d0 2024-03-06T07:29:06.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.491+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7fd804f330 con 0x7f7fd815a0d0 2024-03-06T07:29:06.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.492+0000 7f7fcfe00640 1 -- 172.21.15.86:0/4261044167 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7fc4012070 con 0x7f7fd815a0d0 2024-03-06T07:29:06.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.493+0000 7f7fcfe00640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fb806b500 0x7f7fb806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:06.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.493+0000 7f7fcfe00640 1 -- 172.21.15.86:0/4261044167 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7fc408a470 con 0x7f7fd815a0d0 2024-03-06T07:29:06.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.493+0000 7f7fd7e00640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fb806b500 0x7f7fb806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:06.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.494+0000 7f7fd7e00640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fb806b500 0x7f7fb806d9c0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f7fc8034000 tx=0x7f7fc8007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:06.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.497+0000 7f7fcfe00640 1 -- 172.21.15.86:0/4261044167 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7fc4018020 con 0x7f7fd815a0d0 2024-03-06T07:29:06.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.675+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f7fd80725f0 con 0x7f7fb806b500 2024-03-06T07:29:06.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.686+0000 7f7fcfe00640 1 -- 172.21.15.86:0/4261044167 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7f7fd80725f0 con 0x7f7fb806b500 2024-03-06T07:29:06.691 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:06.693 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:29:06.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fb806b500 msgr2=0x7f7fb806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fb806b500 0x7f7fb806d9c0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f7fc8034000 tx=0x7f7fc8007000 comp rx=0 tx=0).stop 2024-03-06T07:29:06.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fd815a0d0 msgr2=0x7f7fd815a550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:06.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fd815a0d0 0x7f7fd815a550 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f7fc400f580 tx=0x7f7fc400fa40 comp rx=0 tx=0).stop 2024-03-06T07:29:06.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 shutdown_connections 2024-03-06T07:29:06.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fb806b500 0x7f7fb806d9c0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7fd813d840 0x7f7fd813dcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7fd815a0d0 0x7f7fd815a550 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 --2- 172.21.15.86:0/4261044167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7fbc0039b0 0x7f7fd80aed90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:06.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 >> 172.21.15.86:0/4261044167 conn(0x7f7fd80b1730 msgr2=0x7f7fd8143ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:06.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 shutdown_connections 2024-03-06T07:29:06.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:06.692+0000 7f7fdf200640 1 -- 172.21.15.86:0/4261044167 wait complete. 2024-03-06T07:29:06.711 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":103,"stamp":"2024-03-06T07:29:04.885500+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":43,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":254672,"kb_used_data":5220,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754720048,"statfs":{"total":773094113280,"available":772833329152,"internally_reserved":0,"allocated":5345280,"data_stored":2841839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001530"},"pg_stats":[{"pgid":"7.3f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381377+0000","last_change":"2024-03-06T07:28:59.381377+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381377+0000","last_undegraded":"2024-03-06T07:28:59.381377+0000","last_fullsized":"2024-03-06T07:28:59.381377+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.383263+0000","last_change":"2024-03-06T07:28:59.383263+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.383263+0000","last_undegraded":"2024-03-06T07:28:59.383263+0000","last_fullsized":"2024-03-06T07:28:59.383263+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381839+0000","last_change":"2024-03-06T07:28:59.381839+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381839+0000","last_undegraded":"2024-03-06T07:28:59.381839+0000","last_fullsized":"2024-03-06T07:28:59.381839+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376428+0000","last_change":"2024-03-06T07:28:59.376428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376428+0000","last_undegraded":"2024-03-06T07:28:59.376428+0000","last_fullsized":"2024-03-06T07:28:59.376428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381584+0000","last_change":"2024-03-06T07:28:59.381584+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381584+0000","last_undegraded":"2024-03-06T07:28:59.381584+0000","last_fullsized":"2024-03-06T07:28:59.381584+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371720+0000","last_change":"2024-03-06T07:28:59.371720+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371720+0000","last_undegraded":"2024-03-06T07:28:59.371720+0000","last_fullsized":"2024-03-06T07:28:59.371720+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.39","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375651+0000","last_change":"2024-03-06T07:28:59.375651+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375651+0000","last_undegraded":"2024-03-06T07:28:59.375651+0000","last_fullsized":"2024-03-06T07:28:59.375651+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.38","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377314+0000","last_change":"2024-03-06T07:28:59.377314+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377314+0000","last_undegraded":"2024-03-06T07:28:59.377314+0000","last_fullsized":"2024-03-06T07:28:59.377314+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.37","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378074+0000","last_change":"2024-03-06T07:28:59.378074+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378074+0000","last_undegraded":"2024-03-06T07:28:59.378074+0000","last_fullsized":"2024-03-06T07:28:59.378074+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.36","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377833+0000","last_change":"2024-03-06T07:28:59.377833+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377833+0000","last_undegraded":"2024-03-06T07:28:59.377833+0000","last_fullsized":"2024-03-06T07:28:59.377833+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.35","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378278+0000","last_change":"2024-03-06T07:28:59.378278+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378278+0000","last_undegraded":"2024-03-06T07:28:59.378278+0000","last_fullsized":"2024-03-06T07:28:59.378278+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.34","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384955+0000","last_change":"2024-03-06T07:28:59.384955+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384955+0000","last_undegraded":"2024-03-06T07:28:59.384955+0000","last_fullsized":"2024-03-06T07:28:59.384955+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.33","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370349+0000","last_change":"2024-03-06T07:28:59.370349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370349+0000","last_undegraded":"2024-03-06T07:28:59.370349+0000","last_fullsized":"2024-03-06T07:28:59.370349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.32","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376562+0000","last_change":"2024-03-06T07:28:59.376562+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376562+0000","last_undegraded":"2024-03-06T07:28:59.376562+0000","last_fullsized":"2024-03-06T07:28:59.376562+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.31","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370581+0000","last_change":"2024-03-06T07:28:59.370581+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370581+0000","last_undegraded":"2024-03-06T07:28:59.370581+0000","last_fullsized":"2024-03-06T07:28:59.370581+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.30","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.377473+0000","last_change":"2024-03-06T07:28:59.377473+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.377473+0000","last_undegraded":"2024-03-06T07:28:59.377473+0000","last_fullsized":"2024-03-06T07:28:59.377473+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375727+0000","last_change":"2024-03-06T07:28:59.375727+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375727+0000","last_undegraded":"2024-03-06T07:28:59.375727+0000","last_fullsized":"2024-03-06T07:28:59.375727+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382638+0000","last_change":"2024-03-06T07:28:59.382638+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382638+0000","last_undegraded":"2024-03-06T07:28:59.382638+0000","last_fullsized":"2024-03-06T07:28:59.382638+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370250+0000","last_change":"2024-03-06T07:28:59.370250+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370250+0000","last_undegraded":"2024-03-06T07:28:59.370250+0000","last_fullsized":"2024-03-06T07:28:59.370250+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370202+0000","last_change":"2024-03-06T07:28:59.370202+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370202+0000","last_undegraded":"2024-03-06T07:28:59.370202+0000","last_fullsized":"2024-03-06T07:28:59.370202+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.382289+0000","last_change":"2024-03-06T07:28:59.382289+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.382289+0000","last_undegraded":"2024-03-06T07:28:59.382289+0000","last_fullsized":"2024-03-06T07:28:59.382289+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375975+0000","last_change":"2024-03-06T07:28:59.375975+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375975+0000","last_undegraded":"2024-03-06T07:28:59.375975+0000","last_fullsized":"2024-03-06T07:28:59.375975+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.29","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370161+0000","last_change":"2024-03-06T07:28:59.370161+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370161+0000","last_undegraded":"2024-03-06T07:28:59.370161+0000","last_fullsized":"2024-03-06T07:28:59.370161+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.28","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369918+0000","last_change":"2024-03-06T07:28:59.369918+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369918+0000","last_undegraded":"2024-03-06T07:28:59.369918+0000","last_fullsized":"2024-03-06T07:28:59.369918+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.27","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376467+0000","last_change":"2024-03-06T07:28:59.376467+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376467+0000","last_undegraded":"2024-03-06T07:28:59.376467+0000","last_fullsized":"2024-03-06T07:28:59.376467+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.26","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370426+0000","last_change":"2024-03-06T07:28:59.370426+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370426+0000","last_undegraded":"2024-03-06T07:28:59.370426+0000","last_fullsized":"2024-03-06T07:28:59.370426+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.25","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375607+0000","last_change":"2024-03-06T07:28:59.375607+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375607+0000","last_undegraded":"2024-03-06T07:28:59.375607+0000","last_fullsized":"2024-03-06T07:28:59.375607+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.24","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384526+0000","last_change":"2024-03-06T07:28:59.384526+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384526+0000","last_undegraded":"2024-03-06T07:28:59.384526+0000","last_fullsized":"2024-03-06T07:28:59.384526+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.23","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371404+0000","last_change":"2024-03-06T07:28:59.371404+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371404+0000","last_undegraded":"2024-03-06T07:28:59.371404+0000","last_fullsized":"2024-03-06T07:28:59.371404+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.22","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376239+0000","last_change":"2024-03-06T07:28:59.376239+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376239+0000","last_undegraded":"2024-03-06T07:28:59.376239+0000","last_fullsized":"2024-03-06T07:28:59.376239+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.21","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370076+0000","last_change":"2024-03-06T07:28:59.370076+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370076+0000","last_undegraded":"2024-03-06T07:28:59.370076+0000","last_fullsized":"2024-03-06T07:28:59.370076+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.20","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370885+0000","last_change":"2024-03-06T07:28:59.370885+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370885+0000","last_undegraded":"2024-03-06T07:28:59.370885+0000","last_fullsized":"2024-03-06T07:28:59.370885+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375428+0000","last_change":"2024-03-06T07:28:59.375428+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375428+0000","last_undegraded":"2024-03-06T07:28:59.375428+0000","last_fullsized":"2024-03-06T07:28:59.375428+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398367+0000","last_change":"2024-03-06T07:28:59.394295+0000","last_active":"2024-03-06T07:28:59.398367+0000","last_peered":"2024-03-06T07:28:59.398367+0000","last_clean":"2024-03-06T07:28:59.398367+0000","last_became_active":"2024-03-06T07:28:59.393954+0000","last_became_peered":"2024-03-06T07:28:59.393954+0000","last_unstale":"2024-03-06T07:28:59.398367+0000","last_undegraded":"2024-03-06T07:28:59.398367+0000","last_fullsized":"2024-03-06T07:28:59.398367+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:31:25.596355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375212+0000","last_change":"2024-03-06T07:28:59.375212+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375212+0000","last_undegraded":"2024-03-06T07:28:59.375212+0000","last_fullsized":"2024-03-06T07:28:59.375212+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398826+0000","last_change":"2024-03-06T07:28:59.394280+0000","last_active":"2024-03-06T07:28:59.398826+0000","last_peered":"2024-03-06T07:28:59.398826+0000","last_clean":"2024-03-06T07:28:59.398826+0000","last_became_active":"2024-03-06T07:28:59.393796+0000","last_became_peered":"2024-03-06T07:28:59.393796+0000","last_unstale":"2024-03-06T07:28:59.398826+0000","last_undegraded":"2024-03-06T07:28:59.398826+0000","last_fullsized":"2024-03-06T07:28:59.398826+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.1c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.384089+0000","last_change":"2024-03-06T07:28:59.384089+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.384089+0000","last_undegraded":"2024-03-06T07:28:59.384089+0000","last_fullsized":"2024-03-06T07:28:59.384089+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400925+0000","last_change":"2024-03-06T07:28:59.399825+0000","last_active":"2024-03-06T07:28:59.400925+0000","last_peered":"2024-03-06T07:28:59.400925+0000","last_clean":"2024-03-06T07:28:59.400925+0000","last_became_active":"2024-03-06T07:28:59.399545+0000","last_became_peered":"2024-03-06T07:28:59.399545+0000","last_unstale":"2024-03-06T07:28:59.400925+0000","last_undegraded":"2024-03-06T07:28:59.400925+0000","last_fullsized":"2024-03-06T07:28:59.400925+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374114+0000","last_change":"2024-03-06T07:28:59.374114+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374114+0000","last_undegraded":"2024-03-06T07:28:59.374114+0000","last_fullsized":"2024-03-06T07:28:59.374114+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.403876+0000","last_change":"2024-03-06T07:28:59.400350+0000","last_active":"2024-03-06T07:28:59.403876+0000","last_peered":"2024-03-06T07:28:59.403876+0000","last_clean":"2024-03-06T07:28:59.403876+0000","last_became_active":"2024-03-06T07:28:59.399807+0000","last_became_peered":"2024-03-06T07:28:59.399807+0000","last_unstale":"2024-03-06T07:28:59.403876+0000","last_undegraded":"2024-03-06T07:28:59.403876+0000","last_fullsized":"2024-03-06T07:28:59.403876+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:41:52.673252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.1a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374817+0000","last_change":"2024-03-06T07:28:59.374817+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374817+0000","last_undegraded":"2024-03-06T07:28:59.374817+0000","last_fullsized":"2024-03-06T07:28:59.374817+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392184+0000","last_change":"2024-03-06T07:28:59.389474+0000","last_active":"2024-03-06T07:28:59.392184+0000","last_peered":"2024-03-06T07:28:59.392184+0000","last_clean":"2024-03-06T07:28:59.392184+0000","last_became_active":"2024-03-06T07:28:59.389044+0000","last_became_peered":"2024-03-06T07:28:59.389044+0000","last_unstale":"2024-03-06T07:28:59.392184+0000","last_undegraded":"2024-03-06T07:28:59.392184+0000","last_fullsized":"2024-03-06T07:28:59.392184+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:10:59.969773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378890+0000","last_change":"2024-03-06T07:28:59.378890+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378890+0000","last_undegraded":"2024-03-06T07:28:59.378890+0000","last_fullsized":"2024-03-06T07:28:59.378890+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398793+0000","last_change":"2024-03-06T07:28:59.396587+0000","last_active":"2024-03-06T07:28:59.398793+0000","last_peered":"2024-03-06T07:28:59.398793+0000","last_clean":"2024-03-06T07:28:59.398793+0000","last_became_active":"2024-03-06T07:28:59.396300+0000","last_became_peered":"2024-03-06T07:28:59.396300+0000","last_unstale":"2024-03-06T07:28:59.398793+0000","last_undegraded":"2024-03-06T07:28:59.398793+0000","last_fullsized":"2024-03-06T07:28:59.398793+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376200+0000","last_change":"2024-03-06T07:28:59.376200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376200+0000","last_undegraded":"2024-03-06T07:28:59.376200+0000","last_fullsized":"2024-03-06T07:28:59.376200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399000+0000","last_change":"2024-03-06T07:28:59.396760+0000","last_active":"2024-03-06T07:28:59.399000+0000","last_peered":"2024-03-06T07:28:59.399000+0000","last_clean":"2024-03-06T07:28:59.399000+0000","last_became_active":"2024-03-06T07:28:59.396437+0000","last_became_peered":"2024-03-06T07:28:59.396437+0000","last_unstale":"2024-03-06T07:28:59.399000+0000","last_undegraded":"2024-03-06T07:28:59.399000+0000","last_fullsized":"2024-03-06T07:28:59.399000+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:30.328438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376059+0000","last_change":"2024-03-06T07:28:59.376059+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376059+0000","last_undegraded":"2024-03-06T07:28:59.376059+0000","last_fullsized":"2024-03-06T07:28:59.376059+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.18","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391172+0000","last_change":"2024-03-06T07:28:59.384938+0000","last_active":"2024-03-06T07:28:59.391172+0000","last_peered":"2024-03-06T07:28:59.391172+0000","last_clean":"2024-03-06T07:28:59.391172+0000","last_became_active":"2024-03-06T07:28:59.384506+0000","last_became_peered":"2024-03-06T07:28:59.384506+0000","last_unstale":"2024-03-06T07:28:59.391172+0000","last_undegraded":"2024-03-06T07:28:59.391172+0000","last_fullsized":"2024-03-06T07:28:59.391172+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:35.877223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.16","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375820+0000","last_change":"2024-03-06T07:28:59.375820+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375820+0000","last_undegraded":"2024-03-06T07:28:59.375820+0000","last_fullsized":"2024-03-06T07:28:59.375820+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.17","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392813+0000","last_change":"2024-03-06T07:28:59.390136+0000","last_active":"2024-03-06T07:28:59.392813+0000","last_peered":"2024-03-06T07:28:59.392813+0000","last_clean":"2024-03-06T07:28:59.392813+0000","last_became_active":"2024-03-06T07:28:59.387355+0000","last_became_peered":"2024-03-06T07:28:59.387355+0000","last_unstale":"2024-03-06T07:28:59.392813+0000","last_undegraded":"2024-03-06T07:28:59.392813+0000","last_fullsized":"2024-03-06T07:28:59.392813+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380200+0000","last_change":"2024-03-06T07:28:59.380200+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380200+0000","last_undegraded":"2024-03-06T07:28:59.380200+0000","last_fullsized":"2024-03-06T07:28:59.380200+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.16","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391374+0000","last_change":"2024-03-06T07:28:59.387731+0000","last_active":"2024-03-06T07:28:59.391374+0000","last_peered":"2024-03-06T07:28:59.391374+0000","last_clean":"2024-03-06T07:28:59.391374+0000","last_became_active":"2024-03-06T07:28:59.387440+0000","last_became_peered":"2024-03-06T07:28:59.387440+0000","last_unstale":"2024-03-06T07:28:59.391374+0000","last_undegraded":"2024-03-06T07:28:59.391374+0000","last_fullsized":"2024-03-06T07:28:59.391374+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:07:04.179315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.381152+0000","last_change":"2024-03-06T07:28:59.381152+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.381152+0000","last_undegraded":"2024-03-06T07:28:59.381152+0000","last_fullsized":"2024-03-06T07:28:59.381152+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.401208+0000","last_change":"2024-03-06T07:28:59.399848+0000","last_active":"2024-03-06T07:28:59.401208+0000","last_peered":"2024-03-06T07:28:59.401208+0000","last_clean":"2024-03-06T07:28:59.401208+0000","last_became_active":"2024-03-06T07:28:59.399515+0000","last_became_peered":"2024-03-06T07:28:59.399515+0000","last_unstale":"2024-03-06T07:28:59.401208+0000","last_undegraded":"2024-03-06T07:28:59.401208+0000","last_fullsized":"2024-03-06T07:28:59.401208+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:14:43.766404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380656+0000","last_change":"2024-03-06T07:28:59.380656+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380656+0000","last_undegraded":"2024-03-06T07:28:59.380656+0000","last_fullsized":"2024-03-06T07:28:59.380656+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.383916+0000","last_change":"2024-03-06T07:28:59.381955+0000","last_active":"2024-03-06T07:28:59.383916+0000","last_peered":"2024-03-06T07:28:59.383916+0000","last_clean":"2024-03-06T07:28:59.383916+0000","last_became_active":"2024-03-06T07:28:59.381628+0000","last_became_peered":"2024-03-06T07:28:59.381628+0000","last_unstale":"2024-03-06T07:28:59.383916+0000","last_undegraded":"2024-03-06T07:28:59.383916+0000","last_fullsized":"2024-03-06T07:28:59.383916+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:53:05.477963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.12","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372349+0000","last_change":"2024-03-06T07:28:59.372349+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372349+0000","last_undegraded":"2024-03-06T07:28:59.372349+0000","last_fullsized":"2024-03-06T07:28:59.372349+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392142+0000","last_change":"2024-03-06T07:28:59.388270+0000","last_active":"2024-03-06T07:28:59.392142+0000","last_peered":"2024-03-06T07:28:59.392142+0000","last_clean":"2024-03-06T07:28:59.392142+0000","last_became_active":"2024-03-06T07:28:59.387990+0000","last_became_peered":"2024-03-06T07:28:59.387990+0000","last_unstale":"2024-03-06T07:28:59.392142+0000","last_undegraded":"2024-03-06T07:28:59.392142+0000","last_fullsized":"2024-03-06T07:28:59.392142+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:07.389322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.378302+0000","last_change":"2024-03-06T07:28:59.378302+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.378302+0000","last_undegraded":"2024-03-06T07:28:59.378302+0000","last_fullsized":"2024-03-06T07:28:59.378302+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.12","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398882+0000","last_change":"2024-03-06T07:28:59.396602+0000","last_active":"2024-03-06T07:28:59.398882+0000","last_peered":"2024-03-06T07:28:59.398882+0000","last_clean":"2024-03-06T07:28:59.398882+0000","last_became_active":"2024-03-06T07:28:59.396165+0000","last_became_peered":"2024-03-06T07:28:59.396165+0000","last_unstale":"2024-03-06T07:28:59.398882+0000","last_undegraded":"2024-03-06T07:28:59.398882+0000","last_fullsized":"2024-03-06T07:28:59.398882+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:55:11.215461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.10","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373888+0000","last_change":"2024-03-06T07:28:59.373888+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373888+0000","last_undegraded":"2024-03-06T07:28:59.373888+0000","last_fullsized":"2024-03-06T07:28:59.373888+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.11","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392124+0000","last_change":"2024-03-06T07:28:59.387732+0000","last_active":"2024-03-06T07:28:59.392124+0000","last_peered":"2024-03-06T07:28:59.392124+0000","last_clean":"2024-03-06T07:28:59.392124+0000","last_became_active":"2024-03-06T07:28:59.387392+0000","last_became_peered":"2024-03-06T07:28:59.387392+0000","last_unstale":"2024-03-06T07:28:59.392124+0000","last_undegraded":"2024-03-06T07:28:59.392124+0000","last_fullsized":"2024-03-06T07:28:59.392124+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:50:58.093927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374592+0000","last_change":"2024-03-06T07:28:59.374592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374592+0000","last_undegraded":"2024-03-06T07:28:59.374592+0000","last_fullsized":"2024-03-06T07:28:59.374592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400169+0000","last_change":"2024-03-06T07:28:59.399605+0000","last_active":"2024-03-06T07:28:59.400169+0000","last_peered":"2024-03-06T07:28:59.400169+0000","last_clean":"2024-03-06T07:28:59.400169+0000","last_became_active":"2024-03-06T07:28:59.398958+0000","last_became_peered":"2024-03-06T07:28:59.398958+0000","last_unstale":"2024-03-06T07:28:59.400169+0000","last_undegraded":"2024-03-06T07:28:59.400169+0000","last_fullsized":"2024-03-06T07:28:59.400169+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:30.882808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369832+0000","last_change":"2024-03-06T07:28:59.369832+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369832+0000","last_undegraded":"2024-03-06T07:28:59.369832+0000","last_fullsized":"2024-03-06T07:28:59.369832+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395429+0000","last_change":"2024-03-06T07:28:59.389645+0000","last_active":"2024-03-06T07:28:59.395429+0000","last_peered":"2024-03-06T07:28:59.395429+0000","last_clean":"2024-03-06T07:28:59.395429+0000","last_became_active":"2024-03-06T07:28:59.389355+0000","last_became_peered":"2024-03-06T07:28:59.389355+0000","last_unstale":"2024-03-06T07:28:59.395429+0000","last_undegraded":"2024-03-06T07:28:59.395429+0000","last_fullsized":"2024-03-06T07:28:59.395429+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:00.206040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376005+0000","last_change":"2024-03-06T07:28:59.376005+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376005+0000","last_undegraded":"2024-03-06T07:28:59.376005+0000","last_fullsized":"2024-03-06T07:28:59.376005+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.390195+0000","last_change":"2024-03-06T07:28:59.386058+0000","last_active":"2024-03-06T07:28:59.390195+0000","last_peered":"2024-03-06T07:28:59.390195+0000","last_clean":"2024-03-06T07:28:59.390195+0000","last_became_active":"2024-03-06T07:28:59.384967+0000","last_became_peered":"2024-03-06T07:28:59.384967+0000","last_unstale":"2024-03-06T07:28:59.390195+0000","last_undegraded":"2024-03-06T07:28:59.390195+0000","last_fullsized":"2024-03-06T07:28:59.390195+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371865+0000","last_change":"2024-03-06T07:28:59.371865+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371865+0000","last_undegraded":"2024-03-06T07:28:59.371865+0000","last_fullsized":"2024-03-06T07:28:59.371865+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398928+0000","last_change":"2024-03-06T07:28:59.397008+0000","last_active":"2024-03-06T07:28:59.398928+0000","last_peered":"2024-03-06T07:28:59.398928+0000","last_clean":"2024-03-06T07:28:59.398928+0000","last_became_active":"2024-03-06T07:28:59.396486+0000","last_became_peered":"2024-03-06T07:28:59.396486+0000","last_unstale":"2024-03-06T07:28:59.398928+0000","last_undegraded":"2024-03-06T07:28:59.398928+0000","last_fullsized":"2024-03-06T07:28:59.398928+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:12.726533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380771+0000","last_change":"2024-03-06T07:28:59.380771+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380771+0000","last_undegraded":"2024-03-06T07:28:59.380771+0000","last_fullsized":"2024-03-06T07:28:59.380771+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398440+0000","last_change":"2024-03-06T07:28:59.394320+0000","last_active":"2024-03-06T07:28:59.398440+0000","last_peered":"2024-03-06T07:28:59.398440+0000","last_clean":"2024-03-06T07:28:59.398440+0000","last_became_active":"2024-03-06T07:28:59.394031+0000","last_became_peered":"2024-03-06T07:28:59.394031+0000","last_unstale":"2024-03-06T07:28:59.398440+0000","last_undegraded":"2024-03-06T07:28:59.398440+0000","last_fullsized":"2024-03-06T07:28:59.398440+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:26.578812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.371219+0000","last_change":"2024-03-06T07:28:59.371219+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.371219+0000","last_undegraded":"2024-03-06T07:28:59.371219+0000","last_fullsized":"2024-03-06T07:28:59.371219+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.392167+0000","last_change":"2024-03-06T07:28:59.389453+0000","last_active":"2024-03-06T07:28:59.392167+0000","last_peered":"2024-03-06T07:28:59.392167+0000","last_clean":"2024-03-06T07:28:59.392167+0000","last_became_active":"2024-03-06T07:28:59.389059+0000","last_became_peered":"2024-03-06T07:28:59.389059+0000","last_unstale":"2024-03-06T07:28:59.392167+0000","last_undegraded":"2024-03-06T07:28:59.392167+0000","last_fullsized":"2024-03-06T07:28:59.392167+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:14.687420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373283+0000","last_change":"2024-03-06T07:28:59.373283+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373283+0000","last_undegraded":"2024-03-06T07:28:59.373283+0000","last_fullsized":"2024-03-06T07:28:59.373283+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399097+0000","last_change":"2024-03-06T07:28:59.397527+0000","last_active":"2024-03-06T07:28:59.399097+0000","last_peered":"2024-03-06T07:28:59.399097+0000","last_clean":"2024-03-06T07:28:59.399097+0000","last_became_active":"2024-03-06T07:28:59.397178+0000","last_became_peered":"2024-03-06T07:28:59.397178+0000","last_unstale":"2024-03-06T07:28:59.399097+0000","last_undegraded":"2024-03-06T07:28:59.399097+0000","last_fullsized":"2024-03-06T07:28:59.399097+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:25.019648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.369845+0000","last_change":"2024-03-06T07:28:59.369845+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.369845+0000","last_undegraded":"2024-03-06T07:28:59.369845+0000","last_fullsized":"2024-03-06T07:28:59.369845+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.391255+0000","last_change":"2024-03-06T07:28:59.387387+0000","last_active":"2024-03-06T07:28:59.391255+0000","last_peered":"2024-03-06T07:28:59.391255+0000","last_clean":"2024-03-06T07:28:59.391255+0000","last_became_active":"2024-03-06T07:28:59.387111+0000","last_became_peered":"2024-03-06T07:28:59.387111+0000","last_unstale":"2024-03-06T07:28:59.391255+0000","last_undegraded":"2024-03-06T07:28:59.391255+0000","last_fullsized":"2024-03-06T07:28:59.391255+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:37.347867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.380592+0000","last_change":"2024-03-06T07:28:59.380592+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.380592+0000","last_undegraded":"2024-03-06T07:28:59.380592+0000","last_fullsized":"2024-03-06T07:28:59.380592+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399304+0000","last_change":"2024-03-06T07:28:59.396862+0000","last_active":"2024-03-06T07:28:59.399304+0000","last_peered":"2024-03-06T07:28:59.399304+0000","last_clean":"2024-03-06T07:28:59.399304+0000","last_became_active":"2024-03-06T07:28:59.396358+0000","last_became_peered":"2024-03-06T07:28:59.396358+0000","last_unstale":"2024-03-06T07:28:59.399304+0000","last_undegraded":"2024-03-06T07:28:59.399304+0000","last_fullsized":"2024-03-06T07:28:59.399304+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:06.154352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.373843+0000","last_change":"2024-03-06T07:28:59.373843+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.373843+0000","last_undegraded":"2024-03-06T07:28:59.373843+0000","last_fullsized":"2024-03-06T07:28:59.373843+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.395679+0000","last_change":"2024-03-06T07:28:59.392267+0000","last_active":"2024-03-06T07:28:59.395679+0000","last_peered":"2024-03-06T07:28:59.395679+0000","last_clean":"2024-03-06T07:28:59.395679+0000","last_became_active":"2024-03-06T07:28:59.391787+0000","last_became_peered":"2024-03-06T07:28:59.391787+0000","last_unstale":"2024-03-06T07:28:59.395679+0000","last_undegraded":"2024-03-06T07:28:59.395679+0000","last_fullsized":"2024-03-06T07:28:59.395679+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:55.602966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.375838+0000","last_change":"2024-03-06T07:28:59.375838+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.375838+0000","last_undegraded":"2024-03-06T07:28:59.375838+0000","last_fullsized":"2024-03-06T07:28:59.375838+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.404330+0000","last_change":"2024-03-06T07:28:59.401945+0000","last_active":"2024-03-06T07:28:59.404330+0000","last_peered":"2024-03-06T07:28:59.404330+0000","last_clean":"2024-03-06T07:28:59.404330+0000","last_became_active":"2024-03-06T07:28:59.401690+0000","last_became_peered":"2024-03-06T07:28:59.401690+0000","last_unstale":"2024-03-06T07:28:59.404330+0000","last_undegraded":"2024-03-06T07:28:59.404330+0000","last_fullsized":"2024-03-06T07:28:59.404330+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:18:50.468500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.379717+0000","last_change":"2024-03-06T07:28:59.379717+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.379717+0000","last_undegraded":"2024-03-06T07:28:59.379717+0000","last_fullsized":"2024-03-06T07:28:59.379717+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.400919+0000","last_change":"2024-03-06T07:28:59.399872+0000","last_active":"2024-03-06T07:28:59.400919+0000","last_peered":"2024-03-06T07:28:59.400919+0000","last_clean":"2024-03-06T07:28:59.400919+0000","last_became_active":"2024-03-06T07:28:59.399598+0000","last_became_peered":"2024-03-06T07:28:59.399598+0000","last_unstale":"2024-03-06T07:28:59.400919+0000","last_undegraded":"2024-03-06T07:28:59.400919+0000","last_fullsized":"2024-03-06T07:28:59.400919+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.370762+0000","last_change":"2024-03-06T07:28:59.370762+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.370762+0000","last_undegraded":"2024-03-06T07:28:59.370762+0000","last_fullsized":"2024-03-06T07:28:59.370762+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398849+0000","last_change":"2024-03-06T07:28:59.395941+0000","last_active":"2024-03-06T07:28:59.398849+0000","last_peered":"2024-03-06T07:28:59.398849+0000","last_clean":"2024-03-06T07:28:59.398849+0000","last_became_active":"2024-03-06T07:28:59.395072+0000","last_became_peered":"2024-03-06T07:28:59.395072+0000","last_unstale":"2024-03-06T07:28:59.398849+0000","last_undegraded":"2024-03-06T07:28:59.398849+0000","last_fullsized":"2024-03-06T07:28:59.398849+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:49:21.534076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.372479+0000","last_change":"2024-03-06T07:28:59.372479+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.372479+0000","last_undegraded":"2024-03-06T07:28:59.372479+0000","last_fullsized":"2024-03-06T07:28:59.372479+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398776+0000","last_change":"2024-03-06T07:28:59.394472+0000","last_active":"2024-03-06T07:28:59.398776+0000","last_peered":"2024-03-06T07:28:59.398776+0000","last_clean":"2024-03-06T07:28:59.398776+0000","last_became_active":"2024-03-06T07:28:59.394029+0000","last_became_peered":"2024-03-06T07:28:59.394029+0000","last_unstale":"2024-03-06T07:28:59.398776+0000","last_undegraded":"2024-03-06T07:28:59.398776+0000","last_fullsized":"2024-03-06T07:28:59.398776+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.386812+0000","last_change":"2024-03-06T07:28:59.386812+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.386812+0000","last_undegraded":"2024-03-06T07:28:59.386812+0000","last_fullsized":"2024-03-06T07:28:59.386812+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398202+0000","last_change":"2024-03-06T07:28:59.393871+0000","last_active":"2024-03-06T07:28:59.398202+0000","last_peered":"2024-03-06T07:28:59.398202+0000","last_clean":"2024-03-06T07:28:59.398202+0000","last_became_active":"2024-03-06T07:28:59.393554+0000","last_became_peered":"2024-03-06T07:28:59.393554+0000","last_unstale":"2024-03-06T07:28:59.398202+0000","last_undegraded":"2024-03-06T07:28:59.398202+0000","last_fullsized":"2024-03-06T07:28:59.398202+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:01:22.351289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.374029+0000","last_change":"2024-03-06T07:28:59.374029+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.374029+0000","last_undegraded":"2024-03-06T07:28:59.374029+0000","last_fullsized":"2024-03-06T07:28:59.374029+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.399185+0000","last_change":"2024-03-06T07:28:59.392741+0000","last_active":"2024-03-06T07:28:59.399185+0000","last_peered":"2024-03-06T07:28:59.399185+0000","last_clean":"2024-03-06T07:28:59.399185+0000","last_became_active":"2024-03-06T07:28:59.392415+0000","last_became_peered":"2024-03-06T07:28:59.392415+0000","last_unstale":"2024-03-06T07:28:59.399185+0000","last_undegraded":"2024-03-06T07:28:59.399185+0000","last_fullsized":"2024-03-06T07:28:59.399185+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:27.271800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":1,"reported_epoch":45,"state":"creating+peering","last_fresh":"2024-03-06T07:28:59.376957+0000","last_change":"2024-03-06T07:28:59.376957+0000","last_active":"2024-03-06T07:28:59.344665+0000","last_peered":"2024-03-06T07:28:59.344665+0000","last_clean":"2024-03-06T07:28:59.344665+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:28:59.376957+0000","last_undegraded":"2024-03-06T07:28:59.376957+0000","last_fullsized":"2024-03-06T07:28:59.376957+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":9,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.398958+0000","last_change":"2024-03-06T07:28:59.393942+0000","last_active":"2024-03-06T07:28:59.398958+0000","last_peered":"2024-03-06T07:28:59.398958+0000","last_clean":"2024-03-06T07:28:59.398958+0000","last_became_active":"2024-03-06T07:28:59.393597+0000","last_became_peered":"2024-03-06T07:28:59.393597+0000","last_unstale":"2024-03-06T07:28:59.398958+0000","last_undegraded":"2024-03-06T07:28:59.398958+0000","last_fullsized":"2024-03-06T07:28:59.398958+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:00:31.863535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":208,"reported_epoch":45,"state":"active+clean","last_fresh":"2024-03-06T07:28:59.388216+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:28:59.388216+0000","last_peered":"2024-03-06T07:28:59.388216+0000","last_clean":"2024-03-06T07:28:59.388216+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:28:59.388216+0000","last_undegraded":"2024-03-06T07:28:59.388216+0000","last_fullsized":"2024-03-06T07:28:59.388216+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":6,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542172,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":794089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542172,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542171,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542171,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542172,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":524288,"data_stored":209988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542173,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31680,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340160,"statfs":{"total":96636764160,"available":96604323840,"internally_reserved":0,"allocated":512000,"data_stored":203721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542170,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32276,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339564,"statfs":{"total":96636764160,"available":96603713536,"internally_reserved":0,"allocated":1118208,"data_stored":800356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:29:09.715 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:29:09.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.931+0000 7f242ac00640 1 Processor -- start 2024-03-06T07:29:09.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.931+0000 7f242ac00640 1 -- start start 2024-03-06T07:29:09.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.933+0000 7f242ac00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 0x7f2424071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.933+0000 7f242ac00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f242406cef0 con 0x7f2424144c00 2024-03-06T07:29:09.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.933+0000 7f242ac00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f242406d1d0 con 0x7f242415f9d0 2024-03-06T07:29:09.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.933+0000 7f242ac00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f242406d4b0 con 0x7f242415fe60 2024-03-06T07:29:09.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.933+0000 7f2423e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2424144c00 0x7f242406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59012/0 (socket says 172.21.15.86:59012) 2024-03-06T07:29:09.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.933+0000 7f2423e00640 1 -- 172.21.15.86:0/1872345209 learned_addr learned my addr 172.21.15.86:0/1872345209 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:09.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2428c00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 0x7f2424071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:09.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2390267849 0 0) 0x7f242406cef0 con 0x7f2424144c00 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2408003520 con 0x7f2424144c00 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4258682550 0 0) 0x7f2408003520 con 0x7f2424144c00 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 msgr2=0x7f2424071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 0x7f2424071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 >> v1:172.21.15.194:6789/0 conn(0x7f242415f9d0 legacy=0x7f2424070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f242406d790 con 0x7f2424144c00 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f2414001670 con 0x7f2424144c00 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 >> v1:172.21.15.86:6789/0 conn(0x7f2424144c00 legacy=0x7f242406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 0x7f24080039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2408003fb0 0x7f2408004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2408004d40 0x7f24080051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f242406d4b0 con 0x7f2408004d40 2024-03-06T07:29:09.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f240800bb00 con 0x7f2408003fb0 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.934+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f240800bd00 con 0x7f242415fe60 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2428c00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2408004d40 0x7f24080051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2423400640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2408003fb0 0x7f2408004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2423e00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 0x7f24080039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2428c00640 1 -- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 msgr2=0x7f24080039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2428c00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 0x7f24080039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2428c00640 1 -- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2408003fb0 msgr2=0x7f2408004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2428c00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2408003fb0 0x7f2408004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2428c00640 1 -- 172.21.15.86:0/1872345209 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2408003520 con 0x7f2408004d40 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2423e00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f242415fe60 0x7f24080039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.935+0000 7f2423400640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2408003fb0 0x7f2408004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:09.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.936+0000 7f2428c00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2408004d40 0x7f24080051c0 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f241c009fc0 tx=0x7f2408005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c383206f9d83c7c4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:09.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f241c039070 con 0x7f2408004d40 2024-03-06T07:29:09.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f241c02cc20 con 0x7f2408004d40 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f2422a00640 1 -- 172.21.15.86:0/1872345209 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f241c009c10 con 0x7f2408004d40 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f242ac00640 1 -- 172.21.15.86:0/1872345209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2408004d40 msgr2=0x7f24080051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f242ac00640 1 --2- 172.21.15.86:0/1872345209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2408004d40 0x7f24080051c0 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f241c009fc0 tx=0x7f2408005830 comp rx=0 tx=0).stop 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f2428c00640 1 -- 172.21.15.86:0/1872345209 reap_dead start 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f242ac00640 1 -- 172.21.15.86:0/1872345209 shutdown_connections 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f242ac00640 1 -- 172.21.15.86:0/1872345209 >> 172.21.15.86:0/1872345209 conn(0x7f24240b1730 msgr2=0x7f24240b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f242ac00640 1 -- 172.21.15.86:0/1872345209 shutdown_connections 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.937+0000 7f242ac00640 1 -- 172.21.15.86:0/1872345209 wait complete. 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.939+0000 7f242ac00640 1 Processor -- start 2024-03-06T07:29:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.939+0000 7f242ac00640 1 -- start start 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f242ac00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2408004d40 0x7f242415e690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f242ac00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f242415d7a0 0x7f242415b460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f242ac00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f242415b9a0 0x7f2424071a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f242ac00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f242406cb50 con 0x7f242415d7a0 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f242ac00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2424160730 con 0x7f242415b9a0 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f242ac00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f242406ccd0 con 0x7f2408004d40 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f2423400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f242415d7a0 0x7f242415b460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f2423400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f242415d7a0 0x7f242415b460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38036/0 (socket says 172.21.15.86:38036) 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.940+0000 7f2423400640 1 -- 172.21.15.86:0/849461176 learned_addr learned my addr 172.21.15.86:0/849461176 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:09.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f2423400640 1 -- 172.21.15.86:0/849461176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2408004d40 msgr2=0x7f242415e690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f2423400640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2408004d40 0x7f242415e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f2423400640 1 -- 172.21.15.86:0/849461176 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f242415b9a0 msgr2=0x7f2424071a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f2423400640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f242415b9a0 0x7f2424071a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f2423400640 1 -- 172.21.15.86:0/849461176 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f242415c700 con 0x7f242415d7a0 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f2423400640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f242415d7a0 0x7f242415b460 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f241000d800 tx=0x7f241000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f241be00640 1 -- 172.21.15.86:0/849461176 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2410005070 con 0x7f242415d7a0 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f241be00640 1 -- 172.21.15.86:0/849461176 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2410005210 con 0x7f242415d7a0 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f241be00640 1 -- 172.21.15.86:0/849461176 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2410007030 con 0x7f242415d7a0 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.941+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f242415c9e0 con 0x7f242415d7a0 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.942+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f242415cfa0 con 0x7f242415d7a0 2024-03-06T07:29:09.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.943+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f242404f330 con 0x7f242415d7a0 2024-03-06T07:29:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.944+0000 7f241be00640 1 -- 172.21.15.86:0/849461176 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f241001a030 con 0x7f242415d7a0 2024-03-06T07:29:09.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.945+0000 7f241be00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f240406b500 0x7f240406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:09.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.945+0000 7f241be00640 1 -- 172.21.15.86:0/849461176 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f24100898b0 con 0x7f242415d7a0 2024-03-06T07:29:09.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.945+0000 7f2423e00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f240406b500 0x7f240406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:09.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.946+0000 7f2423e00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f240406b500 0x7f240406d9c0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f2414007a70 tx=0x7f2414009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:09.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:09.950+0000 7f241be00640 1 -- 172.21.15.86:0/849461176 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2410021020 con 0x7f242415d7a0 2024-03-06T07:29:10.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.144+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f24240cee80 con 0x7f240406b500 2024-03-06T07:29:10.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.154+0000 7f241be00640 1 -- 172.21.15.86:0/849461176 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251960 (secure 0 0 0) 0x7f24240cee80 con 0x7f240406b500 2024-03-06T07:29:10.159 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:10.160 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:29:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f240406b500 msgr2=0x7f240406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f240406b500 0x7f240406d9c0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f2414007a70 tx=0x7f2414009040 comp rx=0 tx=0).stop 2024-03-06T07:29:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f242415d7a0 msgr2=0x7f242415b460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f242415d7a0 0x7f242415b460 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f241000d800 tx=0x7f241000dcc0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 shutdown_connections 2024-03-06T07:29:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f240406b500 0x7f240406d9c0 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f242415b9a0 0x7f2424071a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f242415d7a0 0x7f242415b460 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 --2- 172.21.15.86:0/849461176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2408004d40 0x7f242415e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.159+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 >> 172.21.15.86:0/849461176 conn(0x7f24240b1730 msgr2=0x7f24240b0960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.160+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 shutdown_connections 2024-03-06T07:29:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.160+0000 7f242ac00640 1 -- 172.21.15.86:0/849461176 wait complete. 2024-03-06T07:29:10.178 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":105,"stamp":"2024-03-06T07:29:08.887249+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":185,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":254968,"kb_used_data":5512,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754719752,"statfs":{"total":773094113280,"available":772833026048,"internally_reserved":0,"allocated":5644288,"data_stored":2970060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":38,"apply_latency_ms":38,"commit_latency_ns":38000000,"apply_latency_ns":38000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.532368"},"pg_stats":[{"pgid":"7.3f","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.511541+0000","last_change":"2024-03-06T07:29:00.384897+0000","last_active":"2024-03-06T07:29:01.511541+0000","last_peered":"2024-03-06T07:29:01.511541+0000","last_clean":"2024-03-06T07:29:01.511541+0000","last_became_active":"2024-03-06T07:29:00.384597+0000","last_became_peered":"2024-03-06T07:29:00.384597+0000","last_unstale":"2024-03-06T07:29:01.511541+0000","last_undegraded":"2024-03-06T07:29:01.511541+0000","last_fullsized":"2024-03-06T07:29:01.511541+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:32.273620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3e","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395841+0000","last_change":"2024-03-06T07:29:00.403079+0000","last_active":"2024-03-06T07:29:01.395841+0000","last_peered":"2024-03-06T07:29:01.395841+0000","last_clean":"2024-03-06T07:29:01.395841+0000","last_became_active":"2024-03-06T07:29:00.402830+0000","last_became_peered":"2024-03-06T07:29:00.402830+0000","last_unstale":"2024-03-06T07:29:01.395841+0000","last_undegraded":"2024-03-06T07:29:01.395841+0000","last_fullsized":"2024-03-06T07:29:01.395841+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:14:50.907364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3d","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417529+0000","last_change":"2024-03-06T07:29:00.370288+0000","last_active":"2024-03-06T07:29:01.417529+0000","last_peered":"2024-03-06T07:29:01.417529+0000","last_clean":"2024-03-06T07:29:01.417529+0000","last_became_active":"2024-03-06T07:29:00.369536+0000","last_became_peered":"2024-03-06T07:29:00.369536+0000","last_unstale":"2024-03-06T07:29:01.417529+0000","last_undegraded":"2024-03-06T07:29:01.417529+0000","last_fullsized":"2024-03-06T07:29:01.417529+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:05.257516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3c","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508270+0000","last_change":"2024-03-06T07:29:00.398452+0000","last_active":"2024-03-06T07:29:01.508270+0000","last_peered":"2024-03-06T07:29:01.508270+0000","last_clean":"2024-03-06T07:29:01.508270+0000","last_became_active":"2024-03-06T07:29:00.397211+0000","last_became_peered":"2024-03-06T07:29:00.397211+0000","last_unstale":"2024-03-06T07:29:01.508270+0000","last_undegraded":"2024-03-06T07:29:01.508270+0000","last_fullsized":"2024-03-06T07:29:01.508270+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:15:32.273620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3b","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395108+0000","last_change":"2024-03-06T07:29:00.399673+0000","last_active":"2024-03-06T07:29:01.395108+0000","last_peered":"2024-03-06T07:29:01.395108+0000","last_clean":"2024-03-06T07:29:01.395108+0000","last_became_active":"2024-03-06T07:29:00.399300+0000","last_became_peered":"2024-03-06T07:29:00.399300+0000","last_unstale":"2024-03-06T07:29:01.395108+0000","last_undegraded":"2024-03-06T07:29:01.395108+0000","last_fullsized":"2024-03-06T07:29:01.395108+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:46:32.024075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3a","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408472+0000","last_change":"2024-03-06T07:29:00.389170+0000","last_active":"2024-03-06T07:29:01.408472+0000","last_peered":"2024-03-06T07:29:01.408472+0000","last_clean":"2024-03-06T07:29:01.408472+0000","last_became_active":"2024-03-06T07:29:00.388000+0000","last_became_peered":"2024-03-06T07:29:00.388000+0000","last_unstale":"2024-03-06T07:29:01.408472+0000","last_undegraded":"2024-03-06T07:29:01.408472+0000","last_fullsized":"2024-03-06T07:29:01.408472+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:31:44.246079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.39","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510865+0000","last_change":"2024-03-06T07:29:00.390354+0000","last_active":"2024-03-06T07:29:01.510865+0000","last_peered":"2024-03-06T07:29:01.510865+0000","last_clean":"2024-03-06T07:29:01.510865+0000","last_became_active":"2024-03-06T07:29:00.389160+0000","last_became_peered":"2024-03-06T07:29:00.389160+0000","last_unstale":"2024-03-06T07:29:01.510865+0000","last_undegraded":"2024-03-06T07:29:01.510865+0000","last_fullsized":"2024-03-06T07:29:01.510865+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:27:54.726570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.38","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.507783+0000","last_change":"2024-03-06T07:29:00.401314+0000","last_active":"2024-03-06T07:29:01.507783+0000","last_peered":"2024-03-06T07:29:01.507783+0000","last_clean":"2024-03-06T07:29:01.507783+0000","last_became_active":"2024-03-06T07:29:00.400959+0000","last_became_peered":"2024-03-06T07:29:00.400959+0000","last_unstale":"2024-03-06T07:29:01.507783+0000","last_undegraded":"2024-03-06T07:29:01.507783+0000","last_fullsized":"2024-03-06T07:29:01.507783+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:32.948127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.37","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508997+0000","last_change":"2024-03-06T07:29:00.398111+0000","last_active":"2024-03-06T07:29:01.508997+0000","last_peered":"2024-03-06T07:29:01.508997+0000","last_clean":"2024-03-06T07:29:01.508997+0000","last_became_active":"2024-03-06T07:29:00.397786+0000","last_became_peered":"2024-03-06T07:29:00.397786+0000","last_unstale":"2024-03-06T07:29:01.508997+0000","last_undegraded":"2024-03-06T07:29:01.508997+0000","last_fullsized":"2024-03-06T07:29:01.508997+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:56:01.877834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.36","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508685+0000","last_change":"2024-03-06T07:29:00.395230+0000","last_active":"2024-03-06T07:29:01.508685+0000","last_peered":"2024-03-06T07:29:01.508685+0000","last_clean":"2024-03-06T07:29:01.508685+0000","last_became_active":"2024-03-06T07:29:00.393644+0000","last_became_peered":"2024-03-06T07:29:00.393644+0000","last_unstale":"2024-03-06T07:29:01.508685+0000","last_undegraded":"2024-03-06T07:29:01.508685+0000","last_fullsized":"2024-03-06T07:29:01.508685+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:01.215206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.35","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508704+0000","last_change":"2024-03-06T07:29:00.401992+0000","last_active":"2024-03-06T07:29:01.508704+0000","last_peered":"2024-03-06T07:29:01.508704+0000","last_clean":"2024-03-06T07:29:01.508704+0000","last_became_active":"2024-03-06T07:29:00.401703+0000","last_became_peered":"2024-03-06T07:29:00.401703+0000","last_unstale":"2024-03-06T07:29:01.508704+0000","last_undegraded":"2024-03-06T07:29:01.508704+0000","last_fullsized":"2024-03-06T07:29:01.508704+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:35:21.645656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.34","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395864+0000","last_change":"2024-03-06T07:29:00.390311+0000","last_active":"2024-03-06T07:29:01.395864+0000","last_peered":"2024-03-06T07:29:01.395864+0000","last_clean":"2024-03-06T07:29:01.395864+0000","last_became_active":"2024-03-06T07:29:00.390002+0000","last_became_peered":"2024-03-06T07:29:00.390002+0000","last_unstale":"2024-03-06T07:29:01.395864+0000","last_undegraded":"2024-03-06T07:29:01.395864+0000","last_fullsized":"2024-03-06T07:29:01.395864+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:32:38.346968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.33","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.509162+0000","last_change":"2024-03-06T07:29:00.386054+0000","last_active":"2024-03-06T07:29:01.509162+0000","last_peered":"2024-03-06T07:29:01.509162+0000","last_clean":"2024-03-06T07:29:01.509162+0000","last_became_active":"2024-03-06T07:29:00.385801+0000","last_became_peered":"2024-03-06T07:29:00.385801+0000","last_unstale":"2024-03-06T07:29:01.509162+0000","last_undegraded":"2024-03-06T07:29:01.509162+0000","last_fullsized":"2024-03-06T07:29:01.509162+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:38.362338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.32","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.507958+0000","last_change":"2024-03-06T07:29:00.399385+0000","last_active":"2024-03-06T07:29:01.507958+0000","last_peered":"2024-03-06T07:29:01.507958+0000","last_clean":"2024-03-06T07:29:01.507958+0000","last_became_active":"2024-03-06T07:29:00.399122+0000","last_became_peered":"2024-03-06T07:29:00.399122+0000","last_unstale":"2024-03-06T07:29:01.507958+0000","last_undegraded":"2024-03-06T07:29:01.507958+0000","last_fullsized":"2024-03-06T07:29:01.507958+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:00.028883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.31","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408300+0000","last_change":"2024-03-06T07:29:00.385860+0000","last_active":"2024-03-06T07:29:01.408300+0000","last_peered":"2024-03-06T07:29:01.408300+0000","last_clean":"2024-03-06T07:29:01.408300+0000","last_became_active":"2024-03-06T07:29:00.384206+0000","last_became_peered":"2024-03-06T07:29:00.384206+0000","last_unstale":"2024-03-06T07:29:01.408300+0000","last_undegraded":"2024-03-06T07:29:01.408300+0000","last_fullsized":"2024-03-06T07:29:01.408300+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:04:42.314914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.30","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510993+0000","last_change":"2024-03-06T07:29:00.404320+0000","last_active":"2024-03-06T07:29:01.510993+0000","last_peered":"2024-03-06T07:29:01.510993+0000","last_clean":"2024-03-06T07:29:01.510993+0000","last_became_active":"2024-03-06T07:29:00.404033+0000","last_became_peered":"2024-03-06T07:29:00.404033+0000","last_unstale":"2024-03-06T07:29:01.510993+0000","last_undegraded":"2024-03-06T07:29:01.510993+0000","last_fullsized":"2024-03-06T07:29:01.510993+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:45:25.642660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2f","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417980+0000","last_change":"2024-03-06T07:29:00.395667+0000","last_active":"2024-03-06T07:29:01.417980+0000","last_peered":"2024-03-06T07:29:01.417980+0000","last_clean":"2024-03-06T07:29:01.417980+0000","last_became_active":"2024-03-06T07:29:00.394878+0000","last_became_peered":"2024-03-06T07:29:00.394878+0000","last_unstale":"2024-03-06T07:29:01.417980+0000","last_undegraded":"2024-03-06T07:29:01.417980+0000","last_fullsized":"2024-03-06T07:29:01.417980+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:00.028883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2e","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417747+0000","last_change":"2024-03-06T07:29:00.397746+0000","last_active":"2024-03-06T07:29:01.417747+0000","last_peered":"2024-03-06T07:29:01.417747+0000","last_clean":"2024-03-06T07:29:01.417747+0000","last_became_active":"2024-03-06T07:29:00.397470+0000","last_became_peered":"2024-03-06T07:29:00.397470+0000","last_unstale":"2024-03-06T07:29:01.417747+0000","last_undegraded":"2024-03-06T07:29:01.417747+0000","last_fullsized":"2024-03-06T07:29:01.417747+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:29:25.328079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2d","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.513219+0000","last_change":"2024-03-06T07:29:00.403975+0000","last_active":"2024-03-06T07:29:01.513219+0000","last_peered":"2024-03-06T07:29:01.513219+0000","last_clean":"2024-03-06T07:29:01.513219+0000","last_became_active":"2024-03-06T07:29:00.403532+0000","last_became_peered":"2024-03-06T07:29:00.403532+0000","last_unstale":"2024-03-06T07:29:01.513219+0000","last_undegraded":"2024-03-06T07:29:01.513219+0000","last_fullsized":"2024-03-06T07:29:01.513219+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:13.741004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2c","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.509108+0000","last_change":"2024-03-06T07:29:00.399441+0000","last_active":"2024-03-06T07:29:01.509108+0000","last_peered":"2024-03-06T07:29:01.509108+0000","last_clean":"2024-03-06T07:29:01.509108+0000","last_became_active":"2024-03-06T07:29:00.398082+0000","last_became_peered":"2024-03-06T07:29:00.398082+0000","last_unstale":"2024-03-06T07:29:01.509108+0000","last_undegraded":"2024-03-06T07:29:01.509108+0000","last_fullsized":"2024-03-06T07:29:01.509108+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:16:36.891695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2b","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408603+0000","last_change":"2024-03-06T07:29:00.399928+0000","last_active":"2024-03-06T07:29:01.408603+0000","last_peered":"2024-03-06T07:29:01.408603+0000","last_clean":"2024-03-06T07:29:01.408603+0000","last_became_active":"2024-03-06T07:29:00.397991+0000","last_became_peered":"2024-03-06T07:29:00.397991+0000","last_unstale":"2024-03-06T07:29:01.408603+0000","last_undegraded":"2024-03-06T07:29:01.408603+0000","last_fullsized":"2024-03-06T07:29:01.408603+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:09:44.482346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2a","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508048+0000","last_change":"2024-03-06T07:29:00.390736+0000","last_active":"2024-03-06T07:29:01.508048+0000","last_peered":"2024-03-06T07:29:01.508048+0000","last_clean":"2024-03-06T07:29:01.508048+0000","last_became_active":"2024-03-06T07:29:00.390413+0000","last_became_peered":"2024-03-06T07:29:00.390413+0000","last_unstale":"2024-03-06T07:29:01.508048+0000","last_undegraded":"2024-03-06T07:29:01.508048+0000","last_fullsized":"2024-03-06T07:29:01.508048+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:27:54.726570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.29","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.512004+0000","last_change":"2024-03-06T07:29:00.391071+0000","last_active":"2024-03-06T07:29:01.512004+0000","last_peered":"2024-03-06T07:29:01.512004+0000","last_clean":"2024-03-06T07:29:01.512004+0000","last_became_active":"2024-03-06T07:29:00.390641+0000","last_became_peered":"2024-03-06T07:29:00.390641+0000","last_unstale":"2024-03-06T07:29:01.512004+0000","last_undegraded":"2024-03-06T07:29:01.512004+0000","last_fullsized":"2024-03-06T07:29:01.512004+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:03:20.742057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.28","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.511285+0000","last_change":"2024-03-06T07:29:00.390434+0000","last_active":"2024-03-06T07:29:01.511285+0000","last_peered":"2024-03-06T07:29:01.511285+0000","last_clean":"2024-03-06T07:29:01.511285+0000","last_became_active":"2024-03-06T07:29:00.390126+0000","last_became_peered":"2024-03-06T07:29:00.390126+0000","last_unstale":"2024-03-06T07:29:01.511285+0000","last_undegraded":"2024-03-06T07:29:01.511285+0000","last_fullsized":"2024-03-06T07:29:01.511285+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:47:47.967450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.27","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.513439+0000","last_change":"2024-03-06T07:29:00.399020+0000","last_active":"2024-03-06T07:29:01.513439+0000","last_peered":"2024-03-06T07:29:01.513439+0000","last_clean":"2024-03-06T07:29:01.513439+0000","last_became_active":"2024-03-06T07:29:00.397060+0000","last_became_peered":"2024-03-06T07:29:00.397060+0000","last_unstale":"2024-03-06T07:29:01.513439+0000","last_undegraded":"2024-03-06T07:29:01.513439+0000","last_fullsized":"2024-03-06T07:29:01.513439+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:57:09.998372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.26","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.509491+0000","last_change":"2024-03-06T07:29:00.387564+0000","last_active":"2024-03-06T07:29:01.509491+0000","last_peered":"2024-03-06T07:29:01.509491+0000","last_clean":"2024-03-06T07:29:01.509491+0000","last_became_active":"2024-03-06T07:29:00.387254+0000","last_became_peered":"2024-03-06T07:29:00.387254+0000","last_unstale":"2024-03-06T07:29:01.509491+0000","last_undegraded":"2024-03-06T07:29:01.509491+0000","last_fullsized":"2024-03-06T07:29:01.509491+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:07:05.193786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.25","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417667+0000","last_change":"2024-03-06T07:29:00.395808+0000","last_active":"2024-03-06T07:29:01.417667+0000","last_peered":"2024-03-06T07:29:01.417667+0000","last_clean":"2024-03-06T07:29:01.417667+0000","last_became_active":"2024-03-06T07:29:00.394944+0000","last_became_peered":"2024-03-06T07:29:00.394944+0000","last_unstale":"2024-03-06T07:29:01.417667+0000","last_undegraded":"2024-03-06T07:29:01.417667+0000","last_fullsized":"2024-03-06T07:29:01.417667+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:32.948127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.24","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.396084+0000","last_change":"2024-03-06T07:29:00.388750+0000","last_active":"2024-03-06T07:29:01.396084+0000","last_peered":"2024-03-06T07:29:01.396084+0000","last_clean":"2024-03-06T07:29:01.396084+0000","last_became_active":"2024-03-06T07:29:00.388246+0000","last_became_peered":"2024-03-06T07:29:00.388246+0000","last_unstale":"2024-03-06T07:29:01.396084+0000","last_undegraded":"2024-03-06T07:29:01.396084+0000","last_fullsized":"2024-03-06T07:29:01.396084+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:47:26.156888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.23","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.512459+0000","last_change":"2024-03-06T07:29:00.389198+0000","last_active":"2024-03-06T07:29:01.512459+0000","last_peered":"2024-03-06T07:29:01.512459+0000","last_clean":"2024-03-06T07:29:01.512459+0000","last_became_active":"2024-03-06T07:29:00.385020+0000","last_became_peered":"2024-03-06T07:29:00.385020+0000","last_unstale":"2024-03-06T07:29:01.512459+0000","last_undegraded":"2024-03-06T07:29:01.512459+0000","last_fullsized":"2024-03-06T07:29:01.512459+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:00:32.878007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.22","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417178+0000","last_change":"2024-03-06T07:29:00.382101+0000","last_active":"2024-03-06T07:29:01.417178+0000","last_peered":"2024-03-06T07:29:01.417178+0000","last_clean":"2024-03-06T07:29:01.417178+0000","last_became_active":"2024-03-06T07:29:00.381775+0000","last_became_peered":"2024-03-06T07:29:00.381775+0000","last_unstale":"2024-03-06T07:29:01.417178+0000","last_undegraded":"2024-03-06T07:29:01.417178+0000","last_fullsized":"2024-03-06T07:29:01.417178+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:31:26.610827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.21","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408259+0000","last_change":"2024-03-06T07:29:00.396286+0000","last_active":"2024-03-06T07:29:01.408259+0000","last_peered":"2024-03-06T07:29:01.408259+0000","last_clean":"2024-03-06T07:29:01.408259+0000","last_became_active":"2024-03-06T07:29:00.395993+0000","last_became_peered":"2024-03-06T07:29:00.395993+0000","last_unstale":"2024-03-06T07:29:01.408259+0000","last_undegraded":"2024-03-06T07:29:01.408259+0000","last_fullsized":"2024-03-06T07:29:01.408259+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:22:14.763100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.20","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508487+0000","last_change":"2024-03-06T07:29:00.387589+0000","last_active":"2024-03-06T07:29:01.508487+0000","last_peered":"2024-03-06T07:29:01.508487+0000","last_clean":"2024-03-06T07:29:01.508487+0000","last_became_active":"2024-03-06T07:29:00.387278+0000","last_became_peered":"2024-03-06T07:29:00.387278+0000","last_unstale":"2024-03-06T07:29:01.508487+0000","last_undegraded":"2024-03-06T07:29:01.508487+0000","last_fullsized":"2024-03-06T07:29:01.508487+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:15:44.091956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1e","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395896+0000","last_change":"2024-03-06T07:29:00.388985+0000","last_active":"2024-03-06T07:29:01.395896+0000","last_peered":"2024-03-06T07:29:01.395896+0000","last_clean":"2024-03-06T07:29:01.395896+0000","last_became_active":"2024-03-06T07:29:00.388575+0000","last_became_peered":"2024-03-06T07:29:00.388575+0000","last_unstale":"2024-03-06T07:29:01.395896+0000","last_undegraded":"2024-03-06T07:29:01.395896+0000","last_fullsized":"2024-03-06T07:29:01.395896+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:19:08.554122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1f","version":"46'4","reported_seq":18,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.409166+0000","last_change":"2024-03-06T07:28:59.394295+0000","last_active":"2024-03-06T07:29:01.409166+0000","last_peered":"2024-03-06T07:29:01.409166+0000","last_clean":"2024-03-06T07:29:01.409166+0000","last_became_active":"2024-03-06T07:28:59.393954+0000","last_became_peered":"2024-03-06T07:28:59.393954+0000","last_unstale":"2024-03-06T07:29:01.409166+0000","last_undegraded":"2024-03-06T07:29:01.409166+0000","last_fullsized":"2024-03-06T07:29:01.409166+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:16:09.060898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1f","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.513266+0000","last_change":"2024-03-06T07:29:00.391739+0000","last_active":"2024-03-06T07:29:01.513266+0000","last_peered":"2024-03-06T07:29:01.513266+0000","last_clean":"2024-03-06T07:29:01.513266+0000","last_became_active":"2024-03-06T07:29:00.388261+0000","last_became_peered":"2024-03-06T07:29:00.388261+0000","last_unstale":"2024-03-06T07:29:01.513266+0000","last_undegraded":"2024-03-06T07:29:01.513266+0000","last_fullsized":"2024-03-06T07:29:01.513266+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:26.034120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1e","version":"46'1","reported_seq":15,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.418048+0000","last_change":"2024-03-06T07:28:59.394280+0000","last_active":"2024-03-06T07:29:01.418048+0000","last_peered":"2024-03-06T07:29:01.418048+0000","last_clean":"2024-03-06T07:29:01.418048+0000","last_became_active":"2024-03-06T07:28:59.393796+0000","last_became_peered":"2024-03-06T07:28:59.393796+0000","last_unstale":"2024-03-06T07:29:01.418048+0000","last_undegraded":"2024-03-06T07:29:01.418048+0000","last_fullsized":"2024-03-06T07:29:01.418048+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:06.154352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.1c","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.396022+0000","last_change":"2024-03-06T07:29:00.394801+0000","last_active":"2024-03-06T07:29:01.396022+0000","last_peered":"2024-03-06T07:29:01.396022+0000","last_clean":"2024-03-06T07:29:01.396022+0000","last_became_active":"2024-03-06T07:29:00.394487+0000","last_became_peered":"2024-03-06T07:29:00.394487+0000","last_unstale":"2024-03-06T07:29:01.396022+0000","last_undegraded":"2024-03-06T07:29:01.396022+0000","last_fullsized":"2024-03-06T07:29:01.396022+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:23:05.375027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1d","version":"46'2","reported_seq":16,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.396357+0000","last_change":"2024-03-06T07:28:59.399825+0000","last_active":"2024-03-06T07:29:01.396357+0000","last_peered":"2024-03-06T07:29:01.396357+0000","last_clean":"2024-03-06T07:29:01.396357+0000","last_became_active":"2024-03-06T07:28:59.399545+0000","last_became_peered":"2024-03-06T07:28:59.399545+0000","last_unstale":"2024-03-06T07:29:01.396357+0000","last_undegraded":"2024-03-06T07:29:01.396357+0000","last_fullsized":"2024-03-06T07:29:01.396357+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:49:21.534076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1d","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.396234+0000","last_change":"2024-03-06T07:29:00.393907+0000","last_active":"2024-03-06T07:29:01.396234+0000","last_peered":"2024-03-06T07:29:01.396234+0000","last_clean":"2024-03-06T07:29:01.396234+0000","last_became_active":"2024-03-06T07:29:00.393618+0000","last_became_peered":"2024-03-06T07:29:00.393618+0000","last_unstale":"2024-03-06T07:29:01.396234+0000","last_undegraded":"2024-03-06T07:29:01.396234+0000","last_fullsized":"2024-03-06T07:29:01.396234+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:50:59.108399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"46'2","reported_seq":16,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510442+0000","last_change":"2024-03-06T07:28:59.400350+0000","last_active":"2024-03-06T07:29:01.510442+0000","last_peered":"2024-03-06T07:29:01.510442+0000","last_clean":"2024-03-06T07:29:01.510442+0000","last_became_active":"2024-03-06T07:28:59.399807+0000","last_became_peered":"2024-03-06T07:28:59.399807+0000","last_unstale":"2024-03-06T07:29:01.510442+0000","last_undegraded":"2024-03-06T07:29:01.510442+0000","last_fullsized":"2024-03-06T07:29:01.510442+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:58:07.389322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.1a","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417280+0000","last_change":"2024-03-06T07:29:00.391380+0000","last_active":"2024-03-06T07:29:01.417280+0000","last_peered":"2024-03-06T07:29:01.417280+0000","last_clean":"2024-03-06T07:29:01.417280+0000","last_became_active":"2024-03-06T07:29:00.391125+0000","last_became_peered":"2024-03-06T07:29:00.391125+0000","last_unstale":"2024-03-06T07:29:01.417280+0000","last_undegraded":"2024-03-06T07:29:01.417280+0000","last_fullsized":"2024-03-06T07:29:01.417280+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:50:20.748885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"46'1","reported_seq":15,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408875+0000","last_change":"2024-03-06T07:28:59.389474+0000","last_active":"2024-03-06T07:29:01.408875+0000","last_peered":"2024-03-06T07:29:01.408875+0000","last_clean":"2024-03-06T07:29:01.408875+0000","last_became_active":"2024-03-06T07:28:59.389044+0000","last_became_peered":"2024-03-06T07:28:59.389044+0000","last_unstale":"2024-03-06T07:29:01.408875+0000","last_undegraded":"2024-03-06T07:29:01.408875+0000","last_fullsized":"2024-03-06T07:29:01.408875+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:04.243045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1b","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508343+0000","last_change":"2024-03-06T07:29:00.390637+0000","last_active":"2024-03-06T07:29:01.508343+0000","last_peered":"2024-03-06T07:29:01.508343+0000","last_clean":"2024-03-06T07:29:01.508343+0000","last_became_active":"2024-03-06T07:29:00.390315+0000","last_became_peered":"2024-03-06T07:29:00.390315+0000","last_unstale":"2024-03-06T07:29:01.508343+0000","last_undegraded":"2024-03-06T07:29:01.508343+0000","last_fullsized":"2024-03-06T07:29:01.508343+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:22:49.093062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1a","version":"46'1","reported_seq":15,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417402+0000","last_change":"2024-03-06T07:28:59.396587+0000","last_active":"2024-03-06T07:29:01.417402+0000","last_peered":"2024-03-06T07:29:01.417402+0000","last_clean":"2024-03-06T07:29:01.417402+0000","last_became_active":"2024-03-06T07:28:59.396300+0000","last_became_peered":"2024-03-06T07:28:59.396300+0000","last_unstale":"2024-03-06T07:29:01.417402+0000","last_undegraded":"2024-03-06T07:29:01.417402+0000","last_fullsized":"2024-03-06T07:29:01.417402+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:55:11.215461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.507837+0000","last_change":"2024-03-06T07:29:00.397364+0000","last_active":"2024-03-06T07:29:01.507837+0000","last_peered":"2024-03-06T07:29:01.507837+0000","last_clean":"2024-03-06T07:29:01.507837+0000","last_became_active":"2024-03-06T07:29:00.397064+0000","last_became_peered":"2024-03-06T07:29:00.397064+0000","last_unstale":"2024-03-06T07:29:01.507837+0000","last_undegraded":"2024-03-06T07:29:01.507837+0000","last_fullsized":"2024-03-06T07:29:01.507837+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:29:25.328079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.394709+0000","last_change":"2024-03-06T07:28:59.396760+0000","last_active":"2024-03-06T07:29:01.394709+0000","last_peered":"2024-03-06T07:29:01.394709+0000","last_clean":"2024-03-06T07:29:01.394709+0000","last_became_active":"2024-03-06T07:28:59.396437+0000","last_became_peered":"2024-03-06T07:28:59.396437+0000","last_unstale":"2024-03-06T07:29:01.394709+0000","last_undegraded":"2024-03-06T07:29:01.394709+0000","last_fullsized":"2024-03-06T07:29:01.394709+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:57:08.983901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.511965+0000","last_change":"2024-03-06T07:29:00.390861+0000","last_active":"2024-03-06T07:29:01.511965+0000","last_peered":"2024-03-06T07:29:01.511965+0000","last_clean":"2024-03-06T07:29:01.511965+0000","last_became_active":"2024-03-06T07:29:00.390193+0000","last_became_peered":"2024-03-06T07:29:00.390193+0000","last_unstale":"2024-03-06T07:29:01.511965+0000","last_undegraded":"2024-03-06T07:29:01.511965+0000","last_fullsized":"2024-03-06T07:29:01.511965+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:15:36.733119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.18","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.507903+0000","last_change":"2024-03-06T07:28:59.384938+0000","last_active":"2024-03-06T07:29:01.507903+0000","last_peered":"2024-03-06T07:29:01.507903+0000","last_clean":"2024-03-06T07:29:01.507903+0000","last_became_active":"2024-03-06T07:28:59.384506+0000","last_became_peered":"2024-03-06T07:28:59.384506+0000","last_unstale":"2024-03-06T07:29:01.507903+0000","last_undegraded":"2024-03-06T07:29:01.507903+0000","last_fullsized":"2024-03-06T07:29:01.507903+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:59:47.148508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.16","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.418165+0000","last_change":"2024-03-06T07:29:00.397814+0000","last_active":"2024-03-06T07:29:01.418165+0000","last_peered":"2024-03-06T07:29:01.418165+0000","last_clean":"2024-03-06T07:29:01.418165+0000","last_became_active":"2024-03-06T07:29:00.397510+0000","last_became_peered":"2024-03-06T07:29:00.397510+0000","last_unstale":"2024-03-06T07:29:01.418165+0000","last_undegraded":"2024-03-06T07:29:01.418165+0000","last_fullsized":"2024-03-06T07:29:01.418165+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:27:54.726570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.17","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.418075+0000","last_change":"2024-03-06T07:28:59.390136+0000","last_active":"2024-03-06T07:29:01.418075+0000","last_peered":"2024-03-06T07:29:01.418075+0000","last_clean":"2024-03-06T07:29:01.418075+0000","last_became_active":"2024-03-06T07:28:59.387355+0000","last_became_peered":"2024-03-06T07:28:59.387355+0000","last_unstale":"2024-03-06T07:29:01.418075+0000","last_undegraded":"2024-03-06T07:29:01.418075+0000","last_fullsized":"2024-03-06T07:29:01.418075+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:41:52.673252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.511519+0000","last_change":"2024-03-06T07:29:00.394314+0000","last_active":"2024-03-06T07:29:01.511519+0000","last_peered":"2024-03-06T07:29:01.511519+0000","last_clean":"2024-03-06T07:29:01.511519+0000","last_became_active":"2024-03-06T07:29:00.393893+0000","last_became_peered":"2024-03-06T07:29:00.393893+0000","last_unstale":"2024-03-06T07:29:01.511519+0000","last_undegraded":"2024-03-06T07:29:01.511519+0000","last_fullsized":"2024-03-06T07:29:01.511519+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:48:03.804667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.16","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508788+0000","last_change":"2024-03-06T07:28:59.387731+0000","last_active":"2024-03-06T07:29:01.508788+0000","last_peered":"2024-03-06T07:29:01.508788+0000","last_clean":"2024-03-06T07:29:01.508788+0000","last_became_active":"2024-03-06T07:28:59.387440+0000","last_became_peered":"2024-03-06T07:28:59.387440+0000","last_unstale":"2024-03-06T07:29:01.508788+0000","last_undegraded":"2024-03-06T07:29:01.508788+0000","last_fullsized":"2024-03-06T07:29:01.508788+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:46:21.204440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408639+0000","last_change":"2024-03-06T07:29:00.401831+0000","last_active":"2024-03-06T07:29:01.408639+0000","last_peered":"2024-03-06T07:29:01.408639+0000","last_clean":"2024-03-06T07:29:01.408639+0000","last_became_active":"2024-03-06T07:29:00.401510+0000","last_became_peered":"2024-03-06T07:29:00.401510+0000","last_unstale":"2024-03-06T07:29:01.408639+0000","last_undegraded":"2024-03-06T07:29:01.408639+0000","last_fullsized":"2024-03-06T07:29:01.408639+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:35:09.106158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"46'3","reported_seq":17,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.511067+0000","last_change":"2024-03-06T07:28:59.399848+0000","last_active":"2024-03-06T07:29:01.511067+0000","last_peered":"2024-03-06T07:29:01.511067+0000","last_clean":"2024-03-06T07:29:01.511067+0000","last_became_active":"2024-03-06T07:28:59.399515+0000","last_became_peered":"2024-03-06T07:28:59.399515+0000","last_unstale":"2024-03-06T07:29:01.511067+0000","last_undegraded":"2024-03-06T07:29:01.511067+0000","last_fullsized":"2024-03-06T07:29:01.511067+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:14.687420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510633+0000","last_change":"2024-03-06T07:29:00.394880+0000","last_active":"2024-03-06T07:29:01.510633+0000","last_peered":"2024-03-06T07:29:01.510633+0000","last_clean":"2024-03-06T07:29:01.510633+0000","last_became_active":"2024-03-06T07:29:00.394588+0000","last_became_peered":"2024-03-06T07:29:00.394588+0000","last_unstale":"2024-03-06T07:29:01.510633+0000","last_undegraded":"2024-03-06T07:29:01.510633+0000","last_fullsized":"2024-03-06T07:29:01.510633+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:29:25.328079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"46'6","reported_seq":20,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.509166+0000","last_change":"2024-03-06T07:28:59.381955+0000","last_active":"2024-03-06T07:29:01.509166+0000","last_peered":"2024-03-06T07:29:01.509166+0000","last_clean":"2024-03-06T07:29:01.509166+0000","last_became_active":"2024-03-06T07:28:59.381628+0000","last_became_peered":"2024-03-06T07:28:59.381628+0000","last_unstale":"2024-03-06T07:29:01.509166+0000","last_undegraded":"2024-03-06T07:29:01.509166+0000","last_fullsized":"2024-03-06T07:29:01.509166+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:12:24.939233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.12","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.512089+0000","last_change":"2024-03-06T07:29:00.404089+0000","last_active":"2024-03-06T07:29:01.512089+0000","last_peered":"2024-03-06T07:29:01.512089+0000","last_clean":"2024-03-06T07:29:01.512089+0000","last_became_active":"2024-03-06T07:29:00.403714+0000","last_became_peered":"2024-03-06T07:29:00.403714+0000","last_unstale":"2024-03-06T07:29:01.512089+0000","last_undegraded":"2024-03-06T07:29:01.512089+0000","last_fullsized":"2024-03-06T07:29:01.512089+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:19:15.745009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"46'2","reported_seq":16,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.409085+0000","last_change":"2024-03-06T07:28:59.388270+0000","last_active":"2024-03-06T07:29:01.409085+0000","last_peered":"2024-03-06T07:29:01.409085+0000","last_clean":"2024-03-06T07:29:01.409085+0000","last_became_active":"2024-03-06T07:28:59.387990+0000","last_became_peered":"2024-03-06T07:28:59.387990+0000","last_unstale":"2024-03-06T07:29:01.409085+0000","last_undegraded":"2024-03-06T07:29:01.409085+0000","last_fullsized":"2024-03-06T07:29:01.409085+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:31.933656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508242+0000","last_change":"2024-03-06T07:29:00.386244+0000","last_active":"2024-03-06T07:29:01.508242+0000","last_peered":"2024-03-06T07:29:01.508242+0000","last_clean":"2024-03-06T07:29:01.508242+0000","last_became_active":"2024-03-06T07:29:00.385719+0000","last_became_peered":"2024-03-06T07:29:00.385719+0000","last_unstale":"2024-03-06T07:29:01.508242+0000","last_undegraded":"2024-03-06T07:29:01.508242+0000","last_fullsized":"2024-03-06T07:29:01.508242+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:48:03.804667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.12","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510745+0000","last_change":"2024-03-06T07:28:59.396602+0000","last_active":"2024-03-06T07:29:01.510745+0000","last_peered":"2024-03-06T07:29:01.510745+0000","last_clean":"2024-03-06T07:29:01.510745+0000","last_became_active":"2024-03-06T07:28:59.396165+0000","last_became_peered":"2024-03-06T07:28:59.396165+0000","last_unstale":"2024-03-06T07:29:01.510745+0000","last_undegraded":"2024-03-06T07:29:01.510745+0000","last_fullsized":"2024-03-06T07:29:01.510745+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:19:07.539651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.10","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408304+0000","last_change":"2024-03-06T07:29:00.391547+0000","last_active":"2024-03-06T07:29:01.408304+0000","last_peered":"2024-03-06T07:29:01.408304+0000","last_clean":"2024-03-06T07:29:01.408304+0000","last_became_active":"2024-03-06T07:29:00.390818+0000","last_became_peered":"2024-03-06T07:29:00.390818+0000","last_unstale":"2024-03-06T07:29:01.408304+0000","last_undegraded":"2024-03-06T07:29:01.408304+0000","last_fullsized":"2024-03-06T07:29:01.408304+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:19:52.344019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.11","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408815+0000","last_change":"2024-03-06T07:28:59.387732+0000","last_active":"2024-03-06T07:29:01.408815+0000","last_peered":"2024-03-06T07:29:01.408815+0000","last_clean":"2024-03-06T07:29:01.408815+0000","last_became_active":"2024-03-06T07:28:59.387392+0000","last_became_peered":"2024-03-06T07:28:59.387392+0000","last_unstale":"2024-03-06T07:29:01.408815+0000","last_undegraded":"2024-03-06T07:29:01.408815+0000","last_fullsized":"2024-03-06T07:29:01.408815+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:50:19.734414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.511136+0000","last_change":"2024-03-06T07:29:00.394002+0000","last_active":"2024-03-06T07:29:01.511136+0000","last_peered":"2024-03-06T07:29:01.511136+0000","last_clean":"2024-03-06T07:29:01.511136+0000","last_became_active":"2024-03-06T07:29:00.393632+0000","last_became_peered":"2024-03-06T07:29:00.393632+0000","last_unstale":"2024-03-06T07:29:01.511136+0000","last_undegraded":"2024-03-06T07:29:01.511136+0000","last_fullsized":"2024-03-06T07:29:01.511136+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:16:10.075370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"46'8","reported_seq":22,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.507952+0000","last_change":"2024-03-06T07:28:59.399605+0000","last_active":"2024-03-06T07:29:01.507952+0000","last_peered":"2024-03-06T07:29:01.507952+0000","last_clean":"2024-03-06T07:29:01.507952+0000","last_became_active":"2024-03-06T07:28:59.398958+0000","last_became_peered":"2024-03-06T07:28:59.398958+0000","last_unstale":"2024-03-06T07:29:01.507952+0000","last_undegraded":"2024-03-06T07:29:01.507952+0000","last_fullsized":"2024-03-06T07:29:01.507952+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:49:21.534076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.509639+0000","last_change":"2024-03-06T07:29:00.394436+0000","last_active":"2024-03-06T07:29:01.509639+0000","last_peered":"2024-03-06T07:29:01.509639+0000","last_clean":"2024-03-06T07:29:01.509639+0000","last_became_active":"2024-03-06T07:29:00.393972+0000","last_became_peered":"2024-03-06T07:29:00.393972+0000","last_unstale":"2024-03-06T07:29:01.509639+0000","last_undegraded":"2024-03-06T07:29:01.509639+0000","last_fullsized":"2024-03-06T07:29:01.509639+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:28.286271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"46'1","reported_seq":15,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.509705+0000","last_change":"2024-03-06T07:28:59.389645+0000","last_active":"2024-03-06T07:29:01.509705+0000","last_peered":"2024-03-06T07:29:01.509705+0000","last_clean":"2024-03-06T07:29:01.509705+0000","last_became_active":"2024-03-06T07:28:59.389355+0000","last_became_peered":"2024-03-06T07:28:59.389355+0000","last_unstale":"2024-03-06T07:29:01.509705+0000","last_undegraded":"2024-03-06T07:29:01.509705+0000","last_fullsized":"2024-03-06T07:29:01.509705+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:17:09.239129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.509062+0000","last_change":"2024-03-06T07:29:00.398291+0000","last_active":"2024-03-06T07:29:01.509062+0000","last_peered":"2024-03-06T07:29:01.509062+0000","last_clean":"2024-03-06T07:29:01.509062+0000","last_became_active":"2024-03-06T07:29:00.397805+0000","last_became_peered":"2024-03-06T07:29:00.397805+0000","last_unstale":"2024-03-06T07:29:01.509062+0000","last_undegraded":"2024-03-06T07:29:01.509062+0000","last_fullsized":"2024-03-06T07:29:01.509062+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:52:44.519882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"46'2","reported_seq":16,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.418239+0000","last_change":"2024-03-06T07:28:59.386058+0000","last_active":"2024-03-06T07:29:01.418239+0000","last_peered":"2024-03-06T07:29:01.418239+0000","last_clean":"2024-03-06T07:29:01.418239+0000","last_became_active":"2024-03-06T07:28:59.384967+0000","last_became_peered":"2024-03-06T07:28:59.384967+0000","last_unstale":"2024-03-06T07:29:01.418239+0000","last_undegraded":"2024-03-06T07:29:01.418239+0000","last_fullsized":"2024-03-06T07:29:01.418239+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:14:43.766404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.512300+0000","last_change":"2024-03-06T07:29:00.390067+0000","last_active":"2024-03-06T07:29:01.512300+0000","last_peered":"2024-03-06T07:29:01.512300+0000","last_clean":"2024-03-06T07:29:01.512300+0000","last_became_active":"2024-03-06T07:29:00.388405+0000","last_became_peered":"2024-03-06T07:29:00.388405+0000","last_unstale":"2024-03-06T07:29:01.512300+0000","last_undegraded":"2024-03-06T07:29:01.512300+0000","last_fullsized":"2024-03-06T07:29:01.512300+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:18:10.223012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"46'1","reported_seq":15,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395470+0000","last_change":"2024-03-06T07:28:59.397008+0000","last_active":"2024-03-06T07:29:01.395470+0000","last_peered":"2024-03-06T07:29:01.395470+0000","last_clean":"2024-03-06T07:29:01.395470+0000","last_became_active":"2024-03-06T07:28:59.396486+0000","last_became_peered":"2024-03-06T07:28:59.396486+0000","last_unstale":"2024-03-06T07:29:01.395470+0000","last_undegraded":"2024-03-06T07:29:01.395470+0000","last_fullsized":"2024-03-06T07:29:01.395470+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:56.681458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.513099+0000","last_change":"2024-03-06T07:29:00.398932+0000","last_active":"2024-03-06T07:29:01.513099+0000","last_peered":"2024-03-06T07:29:01.513099+0000","last_clean":"2024-03-06T07:29:01.513099+0000","last_became_active":"2024-03-06T07:29:00.398547+0000","last_became_peered":"2024-03-06T07:29:00.398547+0000","last_unstale":"2024-03-06T07:29:01.513099+0000","last_undegraded":"2024-03-06T07:29:01.513099+0000","last_fullsized":"2024-03-06T07:29:01.513099+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:31.342909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408751+0000","last_change":"2024-03-06T07:28:59.394320+0000","last_active":"2024-03-06T07:29:01.408751+0000","last_peered":"2024-03-06T07:29:01.408751+0000","last_clean":"2024-03-06T07:29:01.408751+0000","last_became_active":"2024-03-06T07:28:59.394031+0000","last_became_peered":"2024-03-06T07:28:59.394031+0000","last_unstale":"2024-03-06T07:29:01.408751+0000","last_undegraded":"2024-03-06T07:29:01.408751+0000","last_fullsized":"2024-03-06T07:29:01.408751+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:14:59.014411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408609+0000","last_change":"2024-03-06T07:29:00.399235+0000","last_active":"2024-03-06T07:29:01.408609+0000","last_peered":"2024-03-06T07:29:01.408609+0000","last_clean":"2024-03-06T07:29:01.408609+0000","last_became_active":"2024-03-06T07:29:00.398923+0000","last_became_peered":"2024-03-06T07:29:00.398923+0000","last_unstale":"2024-03-06T07:29:01.408609+0000","last_undegraded":"2024-03-06T07:29:01.408609+0000","last_fullsized":"2024-03-06T07:29:01.408609+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:31.133248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"46'2","reported_seq":16,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.409351+0000","last_change":"2024-03-06T07:28:59.389453+0000","last_active":"2024-03-06T07:29:01.409351+0000","last_peered":"2024-03-06T07:29:01.409351+0000","last_clean":"2024-03-06T07:29:01.409351+0000","last_became_active":"2024-03-06T07:28:59.389059+0000","last_became_peered":"2024-03-06T07:28:59.389059+0000","last_unstale":"2024-03-06T07:29:01.409351+0000","last_undegraded":"2024-03-06T07:29:01.409351+0000","last_fullsized":"2024-03-06T07:29:01.409351+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:29:24.313608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395683+0000","last_change":"2024-03-06T07:29:00.394761+0000","last_active":"2024-03-06T07:29:01.395683+0000","last_peered":"2024-03-06T07:29:01.395683+0000","last_clean":"2024-03-06T07:29:01.395683+0000","last_became_active":"2024-03-06T07:29:00.394341+0000","last_became_peered":"2024-03-06T07:29:00.394341+0000","last_unstale":"2024-03-06T07:29:01.395683+0000","last_undegraded":"2024-03-06T07:29:01.395683+0000","last_fullsized":"2024-03-06T07:29:01.395683+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:59:48.162979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.394984+0000","last_change":"2024-03-06T07:28:59.397527+0000","last_active":"2024-03-06T07:29:01.394984+0000","last_peered":"2024-03-06T07:29:01.394984+0000","last_clean":"2024-03-06T07:29:01.394984+0000","last_became_active":"2024-03-06T07:28:59.397178+0000","last_became_peered":"2024-03-06T07:28:59.397178+0000","last_unstale":"2024-03-06T07:29:01.394984+0000","last_undegraded":"2024-03-06T07:29:01.394984+0000","last_fullsized":"2024-03-06T07:29:01.394984+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:38:01.047295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508739+0000","last_change":"2024-03-06T07:29:00.379404+0000","last_active":"2024-03-06T07:29:01.508739+0000","last_peered":"2024-03-06T07:29:01.508739+0000","last_clean":"2024-03-06T07:29:01.508739+0000","last_became_active":"2024-03-06T07:29:00.378713+0000","last_became_peered":"2024-03-06T07:29:00.378713+0000","last_unstale":"2024-03-06T07:29:01.508739+0000","last_undegraded":"2024-03-06T07:29:01.508739+0000","last_fullsized":"2024-03-06T07:29:01.508739+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:02:03.120470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.507877+0000","last_change":"2024-03-06T07:28:59.387387+0000","last_active":"2024-03-06T07:29:01.507877+0000","last_peered":"2024-03-06T07:29:01.507877+0000","last_clean":"2024-03-06T07:29:01.507877+0000","last_became_active":"2024-03-06T07:28:59.387111+0000","last_became_peered":"2024-03-06T07:28:59.387111+0000","last_unstale":"2024-03-06T07:29:01.507877+0000","last_undegraded":"2024-03-06T07:29:01.507877+0000","last_fullsized":"2024-03-06T07:29:01.507877+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:04:58.992842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508198+0000","last_change":"2024-03-06T07:29:00.396571+0000","last_active":"2024-03-06T07:29:01.508198+0000","last_peered":"2024-03-06T07:29:01.508198+0000","last_clean":"2024-03-06T07:29:01.508198+0000","last_became_active":"2024-03-06T07:29:00.396165+0000","last_became_peered":"2024-03-06T07:29:00.396165+0000","last_unstale":"2024-03-06T07:29:01.508198+0000","last_undegraded":"2024-03-06T07:29:01.508198+0000","last_fullsized":"2024-03-06T07:29:01.508198+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:16:10.075370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510302+0000","last_change":"2024-03-06T07:28:59.396862+0000","last_active":"2024-03-06T07:29:01.510302+0000","last_peered":"2024-03-06T07:29:01.510302+0000","last_clean":"2024-03-06T07:29:01.510302+0000","last_became_active":"2024-03-06T07:28:59.396358+0000","last_became_peered":"2024-03-06T07:28:59.396358+0000","last_unstale":"2024-03-06T07:29:01.510302+0000","last_undegraded":"2024-03-06T07:29:01.510302+0000","last_fullsized":"2024-03-06T07:29:01.510302+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:59:47.148508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.513624+0000","last_change":"2024-03-06T07:29:00.401102+0000","last_active":"2024-03-06T07:29:01.513624+0000","last_peered":"2024-03-06T07:29:01.513624+0000","last_clean":"2024-03-06T07:29:01.513624+0000","last_became_active":"2024-03-06T07:29:00.400872+0000","last_became_peered":"2024-03-06T07:29:00.400872+0000","last_unstale":"2024-03-06T07:29:01.513624+0000","last_undegraded":"2024-03-06T07:29:01.513624+0000","last_fullsized":"2024-03-06T07:29:01.513624+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:38:02.061766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.6","version":"47'1","reported_seq":15,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.409198+0000","last_change":"2024-03-06T07:28:59.392267+0000","last_active":"2024-03-06T07:29:01.409198+0000","last_peered":"2024-03-06T07:29:01.409198+0000","last_clean":"2024-03-06T07:29:01.409198+0000","last_became_active":"2024-03-06T07:28:59.391787+0000","last_became_peered":"2024-03-06T07:28:59.391787+0000","last_unstale":"2024-03-06T07:29:01.409198+0000","last_undegraded":"2024-03-06T07:29:01.409198+0000","last_fullsized":"2024-03-06T07:29:01.409198+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:27:53.712099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508490+0000","last_change":"2024-03-06T07:29:00.396960+0000","last_active":"2024-03-06T07:29:01.508490+0000","last_peered":"2024-03-06T07:29:01.508490+0000","last_clean":"2024-03-06T07:29:01.508490+0000","last_became_active":"2024-03-06T07:29:00.396647+0000","last_became_peered":"2024-03-06T07:29:00.396647+0000","last_unstale":"2024-03-06T07:29:01.508490+0000","last_undegraded":"2024-03-06T07:29:01.508490+0000","last_fullsized":"2024-03-06T07:29:01.508490+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:26:09.653788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.511228+0000","last_change":"2024-03-06T07:28:59.401945+0000","last_active":"2024-03-06T07:29:01.511228+0000","last_peered":"2024-03-06T07:29:01.511228+0000","last_clean":"2024-03-06T07:29:01.511228+0000","last_became_active":"2024-03-06T07:28:59.401690+0000","last_became_peered":"2024-03-06T07:28:59.401690+0000","last_unstale":"2024-03-06T07:29:01.511228+0000","last_undegraded":"2024-03-06T07:29:01.511228+0000","last_fullsized":"2024-03-06T07:29:01.511228+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:10:59.969773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510895+0000","last_change":"2024-03-06T07:29:00.392765+0000","last_active":"2024-03-06T07:29:01.510895+0000","last_peered":"2024-03-06T07:29:01.510895+0000","last_clean":"2024-03-06T07:29:01.510895+0000","last_became_active":"2024-03-06T07:29:00.392441+0000","last_became_peered":"2024-03-06T07:29:00.392441+0000","last_unstale":"2024-03-06T07:29:01.510895+0000","last_undegraded":"2024-03-06T07:29:01.510895+0000","last_fullsized":"2024-03-06T07:29:01.510895+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:00.028883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395405+0000","last_change":"2024-03-06T07:28:59.399872+0000","last_active":"2024-03-06T07:29:01.395405+0000","last_peered":"2024-03-06T07:29:01.395405+0000","last_clean":"2024-03-06T07:29:01.395405+0000","last_became_active":"2024-03-06T07:28:59.399598+0000","last_became_peered":"2024-03-06T07:28:59.399598+0000","last_unstale":"2024-03-06T07:29:01.395405+0000","last_undegraded":"2024-03-06T07:29:01.395405+0000","last_fullsized":"2024-03-06T07:29:01.395405+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:06.154352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.409072+0000","last_change":"2024-03-06T07:29:00.392493+0000","last_active":"2024-03-06T07:29:01.409072+0000","last_peered":"2024-03-06T07:29:01.409072+0000","last_clean":"2024-03-06T07:29:01.409072+0000","last_became_active":"2024-03-06T07:29:00.392186+0000","last_became_peered":"2024-03-06T07:29:00.392186+0000","last_unstale":"2024-03-06T07:29:01.409072+0000","last_undegraded":"2024-03-06T07:29:01.409072+0000","last_fullsized":"2024-03-06T07:29:01.409072+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:04:40.192377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510877+0000","last_change":"2024-03-06T07:28:59.395941+0000","last_active":"2024-03-06T07:29:01.510877+0000","last_peered":"2024-03-06T07:29:01.510877+0000","last_clean":"2024-03-06T07:29:01.510877+0000","last_became_active":"2024-03-06T07:28:59.395072+0000","last_became_peered":"2024-03-06T07:28:59.395072+0000","last_unstale":"2024-03-06T07:29:01.510877+0000","last_undegraded":"2024-03-06T07:29:01.510877+0000","last_fullsized":"2024-03-06T07:29:01.510877+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:50:58.093927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.408825+0000","last_change":"2024-03-06T07:29:00.401054+0000","last_active":"2024-03-06T07:29:01.408825+0000","last_peered":"2024-03-06T07:29:01.408825+0000","last_clean":"2024-03-06T07:29:01.408825+0000","last_became_active":"2024-03-06T07:29:00.400692+0000","last_became_peered":"2024-03-06T07:29:00.400692+0000","last_unstale":"2024-03-06T07:29:01.408825+0000","last_undegraded":"2024-03-06T07:29:01.408825+0000","last_fullsized":"2024-03-06T07:29:01.408825+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:25:18.315615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417547+0000","last_change":"2024-03-06T07:28:59.394472+0000","last_active":"2024-03-06T07:29:01.417547+0000","last_peered":"2024-03-06T07:29:01.417547+0000","last_clean":"2024-03-06T07:29:01.417547+0000","last_became_active":"2024-03-06T07:28:59.394029+0000","last_became_peered":"2024-03-06T07:28:59.394029+0000","last_unstale":"2024-03-06T07:29:01.417547+0000","last_undegraded":"2024-03-06T07:29:01.417547+0000","last_fullsized":"2024-03-06T07:29:01.417547+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:49:21.534076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.394894+0000","last_change":"2024-03-06T07:29:00.389305+0000","last_active":"2024-03-06T07:29:01.394894+0000","last_peered":"2024-03-06T07:29:01.394894+0000","last_clean":"2024-03-06T07:29:01.394894+0000","last_became_active":"2024-03-06T07:29:00.389015+0000","last_became_peered":"2024-03-06T07:29:00.389015+0000","last_unstale":"2024-03-06T07:29:01.394894+0000","last_undegraded":"2024-03-06T07:29:01.394894+0000","last_fullsized":"2024-03-06T07:29:01.394894+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:57:51.144912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.395251+0000","last_change":"2024-03-06T07:28:59.393871+0000","last_active":"2024-03-06T07:29:01.395251+0000","last_peered":"2024-03-06T07:29:01.395251+0000","last_clean":"2024-03-06T07:29:01.395251+0000","last_became_active":"2024-03-06T07:28:59.393554+0000","last_became_peered":"2024-03-06T07:28:59.393554+0000","last_unstale":"2024-03-06T07:29:01.395251+0000","last_undegraded":"2024-03-06T07:29:01.395251+0000","last_fullsized":"2024-03-06T07:29:01.395251+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:55:11.215461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.417204+0000","last_change":"2024-03-06T07:29:00.392365+0000","last_active":"2024-03-06T07:29:01.417204+0000","last_peered":"2024-03-06T07:29:01.417204+0000","last_clean":"2024-03-06T07:29:01.417204+0000","last_became_active":"2024-03-06T07:29:00.391962+0000","last_became_peered":"2024-03-06T07:29:00.391962+0000","last_unstale":"2024-03-06T07:29:01.417204+0000","last_undegraded":"2024-03-06T07:29:01.417204+0000","last_fullsized":"2024-03-06T07:29:01.417204+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:27.593283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":14,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508067+0000","last_change":"2024-03-06T07:28:59.392741+0000","last_active":"2024-03-06T07:29:01.508067+0000","last_peered":"2024-03-06T07:29:01.508067+0000","last_clean":"2024-03-06T07:29:01.508067+0000","last_became_active":"2024-03-06T07:28:59.392415+0000","last_became_peered":"2024-03-06T07:28:59.392415+0000","last_unstale":"2024-03-06T07:29:01.508067+0000","last_undegraded":"2024-03-06T07:29:01.508067+0000","last_fullsized":"2024-03-06T07:29:01.508067+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:55:11.215461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":12,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.508707+0000","last_change":"2024-03-06T07:29:00.396758+0000","last_active":"2024-03-06T07:29:01.508707+0000","last_peered":"2024-03-06T07:29:01.508707+0000","last_clean":"2024-03-06T07:29:01.508707+0000","last_became_active":"2024-03-06T07:29:00.388553+0000","last_became_peered":"2024-03-06T07:29:00.388553+0000","last_unstale":"2024-03-06T07:29:01.508707+0000","last_undegraded":"2024-03-06T07:29:01.508707+0000","last_fullsized":"2024-03-06T07:29:01.508707+0000","mapping_epoch":45,"log_start":"0'0","ondisk_log_start":"0'0","created":45,"last_epoch_clean":46,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:59.344665+0000","last_clean_scrub_stamp":"2024-03-06T07:28:59.344665+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:58:33.674863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"46'2","reported_seq":16,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.396109+0000","last_change":"2024-03-06T07:28:59.393942+0000","last_active":"2024-03-06T07:29:01.396109+0000","last_peered":"2024-03-06T07:29:01.396109+0000","last_clean":"2024-03-06T07:29:01.396109+0000","last_became_active":"2024-03-06T07:28:59.393597+0000","last_became_peered":"2024-03-06T07:28:59.393597+0000","last_unstale":"2024-03-06T07:29:01.396109+0000","last_undegraded":"2024-03-06T07:29:01.396109+0000","last_fullsized":"2024-03-06T07:29:01.396109+0000","mapping_epoch":44,"log_start":"0'0","ondisk_log_start":"0'0","created":44,"last_epoch_clean":45,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:28:58.330194+0000","last_clean_scrub_stamp":"2024-03-06T07:28:58.330194+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:55:05.742596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":213,"reported_epoch":47,"state":"active+clean","last_fresh":"2024-03-06T07:29:01.510379+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:29:01.510379+0000","last_peered":"2024-03-06T07:29:01.510379+0000","last_clean":"2024-03-06T07:29:01.510379+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:29:01.510379+0000","last_undegraded":"2024-03-06T07:29:01.510379+0000","last_fullsized":"2024-03-06T07:29:01.510379+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":6,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542173,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32300,"kb_used_data":1120,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339540,"statfs":{"total":96636764160,"available":96603688960,"internally_reserved":0,"allocated":1146880,"data_stored":813521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542172,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31732,"kb_used_data":544,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340108,"statfs":{"total":96636764160,"available":96604270592,"internally_reserved":0,"allocated":557056,"data_stored":223735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542173,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31728,"kb_used_data":540,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340112,"statfs":{"total":96636764160,"available":96604274688,"internally_reserved":0,"allocated":552960,"data_stored":223133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542172,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31728,"kb_used_data":548,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340112,"statfs":{"total":96636764160,"available":96604274688,"internally_reserved":0,"allocated":561152,"data_stored":224675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542172,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31728,"kb_used_data":548,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340112,"statfs":{"total":96636764160,"available":96604274688,"internally_reserved":0,"allocated":561152,"data_stored":223701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542173,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31732,"kb_used_data":552,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340108,"statfs":{"total":96636764160,"available":96604270592,"internally_reserved":0,"allocated":565248,"data_stored":224697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542174,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31720,"kb_used_data":540,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340120,"statfs":{"total":96636764160,"available":96604282880,"internally_reserved":0,"allocated":552960,"data_stored":223133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542171,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32300,"kb_used_data":1120,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339540,"statfs":{"total":96636764160,"available":96603688960,"internally_reserved":0,"allocated":1146880,"data_stored":813465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:29:10.182 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:29:10.182 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:29:10.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.441+0000 7f8738c00640 1 Processor -- start 2024-03-06T07:29:10.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.441+0000 7f8738c00640 1 -- start start 2024-03-06T07:29:10.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.443+0000 7f8738c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 0x7f87340d1860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.443+0000 7f8738c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f87340cf240 0x7f87340b3c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.443+0000 7f8738c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f87340b3190 con 0x7f873415e2d0 2024-03-06T07:29:10.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.443+0000 7f8738c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f87340cfae0 con 0x7f87340cf240 2024-03-06T07:29:10.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.443+0000 7f8738c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f87340cf800 con 0x7f87340d1f20 2024-03-06T07:29:10.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.443+0000 7f8732a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f873415e2d0 0x7f87340d2300 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59014/0 (socket says 172.21.15.86:59014) 2024-03-06T07:29:10.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.443+0000 7f8732a00640 1 -- 172.21.15.86:0/198164889 learned_addr learned my addr 172.21.15.86:0/198164889 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8733400640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 0x7f87340d1860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8733e00640 1 --2- 172.21.15.86:0/198164889 >> v2:172.21.15.194:3300/0 conn(0x7f87340cf240 0x7f87340b3c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2670758373 0 0) 0x7f87340b3190 con 0x7f873415e2d0 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8718003520 con 0x7f873415e2d0 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1843543139 0 0) 0x7f8718003520 con 0x7f873415e2d0 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 msgr2=0x7f87340d1860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 0x7f87340d1860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 >> v2:172.21.15.194:3300/0 conn(0x7f87340cf240 msgr2=0x7f87340b3c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 --2- 172.21.15.86:0/198164889 >> v2:172.21.15.194:3300/0 conn(0x7f87340cf240 0x7f87340b3c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87340b3430 con 0x7f873415e2d0 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f871c001670 con 0x7f873415e2d0 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 >> v1:172.21.15.86:6789/0 conn(0x7f873415e2d0 legacy=0x7f87340d2300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 0x7f87180039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8718003f30 0x7f8718004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8718004d40 0x7f87180051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87340cfae0 con 0x7f8718004d40 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87340cf800 con 0x7f8718003f30 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f871800ba10 con 0x7f87340d1f20 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.444+0000 7f8733e00640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8718004d40 0x7f87180051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.445+0000 7f8733400640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 0x7f87180039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.445+0000 7f8732a00640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8718003f30 0x7f8718004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.445+0000 7f8733e00640 1 -- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 msgr2=0x7f87180039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.445+0000 7f8733e00640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87340d1f20 0x7f87180039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.445+0000 7f8733e00640 1 -- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8718003f30 msgr2=0x7f8718004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.445+0000 7f8733e00640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8718003f30 0x7f8718004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.445+0000 7f8733e00640 1 -- 172.21.15.86:0/198164889 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8718003520 con 0x7f8718004d40 2024-03-06T07:29:10.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.446+0000 7f8733e00640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8718004d40 0x7f87180051c0 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f872c00ba30 tx=0x7f8718005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e20569f09aca6aff server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:10.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.446+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872c036020 con 0x7f8718004d40 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.446+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f872c032680 con 0x7f8718004d40 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.446+0000 7f8732000640 1 -- 172.21.15.86:0/198164889 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872c00b600 con 0x7f8718004d40 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.447+0000 7f8738c00640 1 -- 172.21.15.86:0/198164889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8718004d40 msgr2=0x7f87180051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.447+0000 7f8738c00640 1 --2- 172.21.15.86:0/198164889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8718004d40 0x7f87180051c0 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f872c00ba30 tx=0x7f8718005830 comp rx=0 tx=0).stop 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.447+0000 7f8733e00640 1 -- 172.21.15.86:0/198164889 reap_dead start 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.447+0000 7f8738c00640 1 -- 172.21.15.86:0/198164889 shutdown_connections 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.447+0000 7f8738c00640 1 -- 172.21.15.86:0/198164889 >> 172.21.15.86:0/198164889 conn(0x7f87340d3c90 msgr2=0x7f87340d40a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.447+0000 7f8738c00640 1 -- 172.21.15.86:0/198164889 shutdown_connections 2024-03-06T07:29:10.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.447+0000 7f8738c00640 1 -- 172.21.15.86:0/198164889 wait complete. 2024-03-06T07:29:10.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.449+0000 7f8738c00640 1 Processor -- start 2024-03-06T07:29:10.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.449+0000 7f8738c00640 1 -- start start 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8738c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8718004d40 0x7f87341f6820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8738c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873414d6d0 0x7f87340b8fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8738c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87341e33f0 0x7f87341e3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8738c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87340d19e0 con 0x7f87341e33f0 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8738c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87340d1860 con 0x7f873414d6d0 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8738c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f87340d1b60 con 0x7f8718004d40 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87341e33f0 0x7f87341e3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87341e33f0 0x7f87341e3880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38054/0 (socket says 172.21.15.86:38054) 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733e00640 1 -- 172.21.15.86:0/2441574644 learned_addr learned my addr 172.21.15.86:0/2441574644 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733400640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8718004d40 0x7f87341f6820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8732a00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873414d6d0 0x7f87340b8fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733e00640 1 -- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8718004d40 msgr2=0x7f87341f6820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733e00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8718004d40 0x7f87341f6820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733e00640 1 -- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873414d6d0 msgr2=0x7f87340b8fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.450+0000 7f8733e00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873414d6d0 0x7f87340b8fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.451+0000 7f8733e00640 1 -- 172.21.15.86:0/2441574644 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87341e5ac0 con 0x7f87341e33f0 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.451+0000 7f8733e00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87341e33f0 0x7f87341e3880 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f872c00bb50 tx=0x7f872c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.451+0000 7f872b400640 1 -- 172.21.15.86:0/2441574644 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872c041070 con 0x7f87341e33f0 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.451+0000 7f872b400640 1 -- 172.21.15.86:0/2441574644 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f872c004020 con 0x7f87341e33f0 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.451+0000 7f872b400640 1 -- 172.21.15.86:0/2441574644 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872c036040 con 0x7f87341e33f0 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.451+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f87341e8280 con 0x7f87341e33f0 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.451+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f87341e3dc0 con 0x7f87341e33f0 2024-03-06T07:29:10.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.453+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f87341e5da0 con 0x7f87341e33f0 2024-03-06T07:29:10.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.454+0000 7f872b400640 1 -- 172.21.15.86:0/2441574644 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f872c039020 con 0x7f87341e33f0 2024-03-06T07:29:10.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.454+0000 7f872b400640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b500 0x7f871006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.455+0000 7f872b400640 1 -- 172.21.15.86:0/2441574644 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f872c0abcb0 con 0x7f87341e33f0 2024-03-06T07:29:10.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.455+0000 7f8733400640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b500 0x7f871006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.455+0000 7f8733400640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b500 0x7f871006d9c0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f873406d310 tx=0x7f872400d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:10.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.459+0000 7f872b400640 1 -- 172.21.15.86:0/2441574644 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f872c07aa00 con 0x7f87341e33f0 2024-03-06T07:29:10.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.730+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f87340b5870 con 0x7f87341e33f0 2024-03-06T07:29:10.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.734+0000 7f872b400640 1 -- 172.21.15.86:0/2441574644 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v14) v1 ==== 190+0+147 (secure 0 0 0) 0x7f872c07a300 con 0x7f87341e33f0 2024-03-06T07:29:10.736 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:29:10.736 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:29:10.736 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:29:10.736 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:29:10.736 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:29:10.737 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b500 msgr2=0x7f871006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b500 0x7f871006d9c0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f873406d310 tx=0x7f872400d040 comp rx=0 tx=0).stop 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87341e33f0 msgr2=0x7f87341e3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87341e33f0 0x7f87341e3880 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f872c00bb50 tx=0x7f872c004730 comp rx=0 tx=0).stop 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 shutdown_connections 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b500 0x7f871006d9c0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87341e33f0 0x7f87341e3880 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873414d6d0 0x7f87340b8fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 --2- 172.21.15.86:0/2441574644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8718004d40 0x7f87341f6820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 >> 172.21.15.86:0/2441574644 conn(0x7f87340d3c90 msgr2=0x7f87340b1b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:10.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.737+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 shutdown_connections 2024-03-06T07:29:10.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.738+0000 7f8738c00640 1 -- 172.21.15.86:0/2441574644 wait complete. 2024-03-06T07:29:10.758 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:10.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.965+0000 7f40ff200640 1 Processor -- start 2024-03-06T07:29:10.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.965+0000 7f40ff200640 1 -- start start 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40ff200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f40f815f9d0 0x7f40f815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40ff200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f40f8070b10 0x7f40f8070f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40ff200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f40f806cef0 con 0x7f40f8144c00 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40ff200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f40f806d1d0 con 0x7f40f8070b10 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40ff200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f40f806d4b0 con 0x7f40f815f9d0 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40fd200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f40f8070b10 0x7f40f8070f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40f7e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f40f8144c00 0x7f40f806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59026/0 (socket says 172.21.15.86:59026) 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.967+0000 7f40f7e00640 1 -- 172.21.15.86:0/2321913115 learned_addr learned my addr 172.21.15.86:0/2321913115 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:10.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.968+0000 7f40ef400640 1 --2- 172.21.15.86:0/2321913115 >> v2:172.21.15.194:3300/0 conn(0x7f40f815f9d0 0x7f40f815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.968+0000 7f40fd200640 1 -- 172.21.15.86:0/2321913115 >> v2:172.21.15.194:3300/0 conn(0x7f40f815f9d0 msgr2=0x7f40f815fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:10.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.968+0000 7f40fd200640 1 --2- 172.21.15.86:0/2321913115 >> v2:172.21.15.194:3300/0 conn(0x7f40f815f9d0 0x7f40f815fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.968+0000 7f40fd200640 1 -- 172.21.15.86:0/2321913115 >> v1:172.21.15.86:6789/0 conn(0x7f40f8144c00 legacy=0x7f40f806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.968+0000 7f40fd200640 1 -- 172.21.15.86:0/2321913115 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40f806d790 con 0x7f40f8070b10 2024-03-06T07:29:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.968+0000 7f40f7400640 1 -- 172.21.15.86:0/2321913115 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3216547694 0 0) 0x7f40f806cef0 con 0x7f40f8144c00 2024-03-06T07:29:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.969+0000 7f40fd200640 1 --2- 172.21.15.86:0/2321913115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f8070f20 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f40f80b26f0 tx=0x7f40f002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fd5a6a7980a2c9a2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.969+0000 7f40f7400640 1 -- 172.21.15.86:0/2321913115 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40f0035020 con 0x7f40f8070b10 2024-03-06T07:29:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.969+0000 7f40f7400640 1 -- 172.21.15.86:0/2321913115 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f40f0031830 con 0x7f40f8070b10 2024-03-06T07:29:10.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.969+0000 7f40f7400640 1 -- 172.21.15.86:0/2321913115 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40f0031c00 con 0x7f40f8070b10 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 -- 172.21.15.86:0/2321913115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 msgr2=0x7f40f8070f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 --2- 172.21.15.86:0/2321913115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f8070f20 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f40f80b26f0 tx=0x7f40f002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 -- 172.21.15.86:0/2321913115 shutdown_connections 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 --2- 172.21.15.86:0/2321913115 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f8070f20 unknown :-1 s=CLOSED pgs=530 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 --2- 172.21.15.86:0/2321913115 >> v2:172.21.15.194:3300/0 conn(0x7f40f815f9d0 0x7f40f815fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 -- 172.21.15.86:0/2321913115 >> 172.21.15.86:0/2321913115 conn(0x7f40f80b1730 msgr2=0x7f40f80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 -- 172.21.15.86:0/2321913115 shutdown_connections 2024-03-06T07:29:10.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.970+0000 7f40ff200640 1 -- 172.21.15.86:0/2321913115 wait complete. 2024-03-06T07:29:10.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.971+0000 7f40ff200640 1 Processor -- start 2024-03-06T07:29:10.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.972+0000 7f40ff200640 1 -- start start 2024-03-06T07:29:10.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.972+0000 7f40ff200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f815b220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.972+0000 7f40ff200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40f8144c00 0x7f40f813c400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40ff200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40f815f9d0 0x7f40f813c940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40ff200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40f806ccd0 con 0x7f40f8070b10 2024-03-06T07:29:10.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40ff200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40f806cb50 con 0x7f40f815f9d0 2024-03-06T07:29:10.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40ff200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f40f806df90 con 0x7f40f8144c00 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f815b220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f815b220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38078/0 (socket says 172.21.15.86:38078) 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 -- 172.21.15.86:0/3843939319 learned_addr learned my addr 172.21.15.86:0/3843939319 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 -- 172.21.15.86:0/3843939319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40f8144c00 msgr2=0x7f40f813c400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40f8144c00 0x7f40f813c400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 -- 172.21.15.86:0/3843939319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40f815f9d0 msgr2=0x7f40f813c940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40f815f9d0 0x7f40f813c940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 -- 172.21.15.86:0/3843939319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40f815b760 con 0x7f40f8070b10 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.973+0000 7f40f7e00640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f815b220 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f40e800c890 tx=0x7f40e800cd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.974+0000 7f40f4c00640 1 -- 172.21.15.86:0/3843939319 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40e8018070 con 0x7f40f8070b10 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.974+0000 7f40f4c00640 1 -- 172.21.15.86:0/3843939319 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f40e800a640 con 0x7f40f8070b10 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.974+0000 7f40f4c00640 1 -- 172.21.15.86:0/3843939319 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40e800aa30 con 0x7f40f8070b10 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.974+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f40f813ac20 con 0x7f40f8070b10 2024-03-06T07:29:10.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.974+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f40f813b1a0 con 0x7f40f8070b10 2024-03-06T07:29:10.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.975+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f40f804f330 con 0x7f40f8070b10 2024-03-06T07:29:10.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.976+0000 7f40f4c00640 1 -- 172.21.15.86:0/3843939319 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f40e800abd0 con 0x7f40f8070b10 2024-03-06T07:29:10.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.977+0000 7f40f4c00640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d406b4d0 0x7f40d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:10.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.978+0000 7f40f4c00640 1 -- 172.21.15.86:0/3843939319 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f40e8089830 con 0x7f40f8070b10 2024-03-06T07:29:10.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.978+0000 7f40ef400640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d406b4d0 0x7f40d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:10.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.978+0000 7f40ef400640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d406b4d0 0x7f40d406d990 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f40e0006fd0 tx=0x7f40e0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:10.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:10.982+0000 7f40f4c00640 1 -- 172.21.15.86:0/3843939319 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f40f815b760 con 0x7f40f8070b10 2024-03-06T07:29:11.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.215+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f40f815f690 con 0x7f40f8070b10 2024-03-06T07:29:11.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.216+0000 7f40f4c00640 1 -- 172.21.15.86:0/3843939319 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) v1 ==== 94+0+4983 (secure 0 0 0) 0x7f40e80586a0 con 0x7f40f8070b10 2024-03-06T07:29:11.218 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:11.218 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":25,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5041,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/3678578793","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":3678578793},{"type":"v1","addr":"172.21.15.194:6835","nonce":3678578793}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/4206282845","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":4206282845},{"type":"v1","addr":"172.21.15.86:6835","nonce":4206282845}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5427,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3500830166","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3500830166},{"type":"v1","addr":"172.21.15.86:6837","nonce":3500830166}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":25,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:59.832195+0000","modified":"2024-03-06T07:29:02.883187+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5439},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5439":{"gid":5439,"name":"b","rank":0,"incarnation":21,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6833/4064823241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":4064823241},{"type":"v1","addr":"172.21.15.194:6833","nonce":4064823241}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5439,"qdb_cluster":[5439]},"id":3}]} 2024-03-06T07:29:11.218 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 25 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d406b4d0 msgr2=0x7f40d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d406b4d0 0x7f40d406d990 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f40e0006fd0 tx=0x7f40e0008040 comp rx=0 tx=0).stop 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 msgr2=0x7f40f815b220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f815b220 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f40e800c890 tx=0x7f40e800cd50 comp rx=0 tx=0).stop 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 shutdown_connections 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40d406b4d0 0x7f40d406d990 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40f815f9d0 0x7f40f813c940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f40f8144c00 0x7f40f813c400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 --2- 172.21.15.86:0/3843939319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40f8070b10 0x7f40f815b220 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 >> 172.21.15.86:0/3843939319 conn(0x7f40f80b1730 msgr2=0x7f40f81431a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:11.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 shutdown_connections 2024-03-06T07:29:11.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.219+0000 7f40ff200640 1 -- 172.21.15.86:0/3843939319 wait complete. 2024-03-06T07:29:11.641 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 25, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:28:59.832195+0000', 'modified': '2024-03-06T07:29:02.883187+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 5439}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_5439': {'gid': 5439, 'name': 'b', 'rank': 0, 'incarnation': 21, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.194:6833/4064823241', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.194:6832', 'nonce': 4064823241}, {'type': 'v1', 'addr': '172.21.15.194:6833', 'nonce': 4064823241}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [7], 'metadata_pool': 6, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 5439, 'qdb_cluster': [5439]} 2024-03-06T07:29:11.641 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:29:11.641 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-03-06T07:29:11.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.858+0000 7ff957200640 1 Processor -- start 2024-03-06T07:29:11.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.858+0000 7ff957200640 1 -- start start 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff957200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff95815e090 0x7ff9580cf8f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff957200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9580d1270 0x7ff9580d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff957200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff9580d2340 0x7ff9580d27d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff957200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff95815e470 con 0x7ff95815e090 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff957200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff9580d2180 con 0x7ff9580d2340 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff957200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9580cf460 con 0x7ff9580d1270 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff955e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff95815e090 0x7ff9580cf8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff955e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff95815e090 0x7ff9580cf8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38094/0 (socket says 172.21.15.86:38094) 2024-03-06T07:29:11.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.860+0000 7ff955e00640 1 -- 172.21.15.86:0/2283767821 learned_addr learned my addr 172.21.15.86:0/2283767821 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:11.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.861+0000 7ff955e00640 1 -- 172.21.15.86:0/2283767821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9580d1270 msgr2=0x7ff9580d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.861+0000 7ff955e00640 1 --2- 172.21.15.86:0/2283767821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9580d1270 0x7ff9580d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.861+0000 7ff955e00640 1 -- 172.21.15.86:0/2283767821 >> v2:172.21.15.194:3300/0 conn(0x7ff9580d2340 msgr2=0x7ff9580d27d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.861+0000 7ff955e00640 1 --2- 172.21.15.86:0/2283767821 >> v2:172.21.15.194:3300/0 conn(0x7ff9580d2340 0x7ff9580d27d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.861+0000 7ff955e00640 1 -- 172.21.15.86:0/2283767821 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9580ce890 con 0x7ff95815e090 2024-03-06T07:29:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.862+0000 7ff955e00640 1 --2- 172.21.15.86:0/2283767821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff95815e090 0x7ff9580cf8f0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7ff9580d0360 tx=0x7ff94802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b1a085137c823587 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.862+0000 7ff955400640 1 -- 172.21.15.86:0/2283767821 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948035020 con 0x7ff95815e090 2024-03-06T07:29:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.862+0000 7ff955400640 1 -- 172.21.15.86:0/2283767821 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff948031870 con 0x7ff95815e090 2024-03-06T07:29:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.862+0000 7ff955400640 1 -- 172.21.15.86:0/2283767821 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948031c40 con 0x7ff95815e090 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.862+0000 7ff957200640 1 -- 172.21.15.86:0/2283767821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff95815e090 msgr2=0x7ff9580cf8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.862+0000 7ff957200640 1 --2- 172.21.15.86:0/2283767821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff95815e090 0x7ff9580cf8f0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7ff9580d0360 tx=0x7ff94802b910 comp rx=0 tx=0).stop 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.863+0000 7ff957200640 1 -- 172.21.15.86:0/2283767821 shutdown_connections 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.863+0000 7ff957200640 1 --2- 172.21.15.86:0/2283767821 >> v2:172.21.15.194:3300/0 conn(0x7ff9580d2340 0x7ff9580d27d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.863+0000 7ff957200640 1 --2- 172.21.15.86:0/2283767821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9580d1270 0x7ff9580d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.863+0000 7ff957200640 1 --2- 172.21.15.86:0/2283767821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff95815e090 0x7ff9580cf8f0 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.863+0000 7ff957200640 1 -- 172.21.15.86:0/2283767821 >> 172.21.15.86:0/2283767821 conn(0x7ff9580d3d80 msgr2=0x7ff9580d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.863+0000 7ff957200640 1 -- 172.21.15.86:0/2283767821 shutdown_connections 2024-03-06T07:29:11.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.863+0000 7ff957200640 1 -- 172.21.15.86:0/2283767821 wait complete. 2024-03-06T07:29:11.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.864+0000 7ff957200640 1 Processor -- start 2024-03-06T07:29:11.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.864+0000 7ff957200640 1 -- start start 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.865+0000 7ff957200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9580d1270 0x7ff9580cb0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.865+0000 7ff957200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9580d2340 0x7ff95814a9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.865+0000 7ff957200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff95815e090 0x7ff95814aef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.865+0000 7ff957200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9580cf0f0 con 0x7ff9580d1270 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.865+0000 7ff957200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9580cee10 con 0x7ff9580d2340 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.865+0000 7ff957200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9580d1e90 con 0x7ff95815e090 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9580d1270 0x7ff9580cb0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9580d1270 0x7ff9580cb0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38096/0 (socket says 172.21.15.86:38096) 2024-03-06T07:29:11.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 -- 172.21.15.86:0/2763119257 learned_addr learned my addr 172.21.15.86:0/2763119257 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff94d400640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9580d2340 0x7ff95814a9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff956800640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff95815e090 0x7ff95814aef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 -- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff95815e090 msgr2=0x7ff95814aef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff95815e090 0x7ff95814aef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 -- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9580d2340 msgr2=0x7ff95814a9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9580d2340 0x7ff95814a9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 -- 172.21.15.86:0/2763119257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff95814b510 con 0x7ff9580d1270 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff94d400640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9580d2340 0x7ff95814a9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.866+0000 7ff955e00640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9580d1270 0x7ff9580cb0c0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7ff948000c00 tx=0x7ff9480046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.867+0000 7ff94ea00640 1 -- 172.21.15.86:0/2763119257 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948040070 con 0x7ff9580d1270 2024-03-06T07:29:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.867+0000 7ff94ea00640 1 -- 172.21.15.86:0/2763119257 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff94803a420 con 0x7ff9580d1270 2024-03-06T07:29:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.867+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff95814b7f0 con 0x7ff9580d1270 2024-03-06T07:29:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.867+0000 7ff94ea00640 1 -- 172.21.15.86:0/2763119257 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948035040 con 0x7ff9580d1270 2024-03-06T07:29:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.867+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff9580b7f20 con 0x7ff9580d1270 2024-03-06T07:29:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.869+0000 7ff94ea00640 1 -- 172.21.15.86:0/2763119257 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff94804b060 con 0x7ff9580d1270 2024-03-06T07:29:11.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.870+0000 7ff94ea00640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff93406b4d0 0x7ff93406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.870+0000 7ff94ea00640 1 -- 172.21.15.86:0/2763119257 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff9480ac1a0 con 0x7ff9580d1270 2024-03-06T07:29:11.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.870+0000 7ff94d400640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff93406b4d0 0x7ff93406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.871+0000 7ff94d400640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff93406b4d0 0x7ff93406d990 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7ff9580b2f60 tx=0x7ff940008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:11.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.871+0000 7ff95ffd1740 1 Processor -- start 2024-03-06T07:29:11.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.871+0000 7ff95ffd1740 1 -- start start 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf19930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 0x5578cdf49720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf19e70 0x5578cdf49c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5578cde83600 con 0x5578cdf46990 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5578cde2c930 con 0x5578cdf19e70 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5578cde7f6a0 con 0x5578cdf46d70 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf19930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf19930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38100/0 (socket says 172.21.15.86:38100) 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 -- 172.21.15.86:0/2588124779 learned_addr learned my addr 172.21.15.86:0/2588124779 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 -- 172.21.15.86:0/2588124779 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 msgr2=0x5578cdf49720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 --2- 172.21.15.86:0/2588124779 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 0x5578cdf49720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 -- 172.21.15.86:0/2588124779 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf19e70 msgr2=0x5578cdf49c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 --2- 172.21.15.86:0/2588124779 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf19e70 0x5578cdf49c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.872+0000 7ff955e00640 1 -- 172.21.15.86:0/2588124779 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5578cdb982d0 con 0x5578cdf46990 2024-03-06T07:29:11.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.873+0000 7ff955e00640 1 --2- 172.21.15.86:0/2588124779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf19930 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7ff94807a810 tx=0x7ff94807af80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6869f762683a9d52 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:11.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.873+0000 7ff946a00640 1 -- 172.21.15.86:0/2588124779 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948082f90 con 0x5578cdf46990 2024-03-06T07:29:11.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.873+0000 7ff946a00640 1 -- 172.21.15.86:0/2588124779 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff948078b80 con 0x5578cdf46990 2024-03-06T07:29:11.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.873+0000 7ff946a00640 1 -- 172.21.15.86:0/2588124779 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948076ef0 con 0x5578cdf46990 2024-03-06T07:29:11.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2588124779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 msgr2=0x5578cdf19930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2588124779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf19930 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7ff94807a810 tx=0x7ff94807af80 comp rx=0 tx=0).stop 2024-03-06T07:29:11.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2588124779 shutdown_connections 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2588124779 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf19e70 0x5578cdf49c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2588124779 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 0x5578cdf49720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2588124779 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf19930 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2588124779 >> 172.21.15.86:0/2588124779 conn(0x5578cdf1bba0 msgr2=0x5578cdf64720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2588124779 shutdown_connections 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2588124779 wait complete. 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.874+0000 7ff95ffd1740 1 -- start start 2024-03-06T07:29:11.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.875+0000 7ff95ffd1740 1 Processor -- start 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf74cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 0x5578cdf518d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 0x5578cdf4c910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5578cde2c930 con 0x5578cdf46990 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5578cde3fd90 con 0x5578cdf46d70 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5578cde83600 con 0x5578cdf458c0 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff95ffd1740 10 client.0 _ll_register_callbacks cb 0x5578cde7b350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf74cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf74cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38112/0 (socket says 172.21.15.86:38112) 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 -- 172.21.15.86:0/2112291192 learned_addr learned my addr 172.21.15.86:0/2112291192 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:11.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 -- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 msgr2=0x5578cdf4c910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 0x5578cdf4c910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 -- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 msgr2=0x5578cdf518d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 0x5578cdf518d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 -- 172.21.15.86:0/2112291192 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5578cdc626b0 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff955e00640 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf74cf0 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7ff94807aa40 tx=0x7ff9480ac040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=55110b26cfa38c21 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.876+0000 7ff93b400640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff93b400640 1 -- 172.21.15.86:0/2112291192 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948082f90 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff93b400640 1 -- 172.21.15.86:0/2112291192 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff9480aa050 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff948078b80 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff93b400640 1 -- 172.21.15.86:0/2112291192 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff948031730 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 10 client.5595 fetch_fsmap learned FSMap version 25 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=2+}) v3 -- 0x5578cdf4c3f0 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff93b400640 1 -- 172.21.15.86:0/2112291192 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff948078b80 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff93b400640 1 -- 172.21.15.86:0/2112291192 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 25) v1 ==== 3314+0+0 (secure 0 0 0) 0x7ff9480992b0 con 0x5578cdf46990 2024-03-06T07:29:11.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 10 client.5595 fetch_fsmap finished waiting for FSMap version 25 2024-03-06T07:29:11.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 10 client.5595 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-06T07:29:11.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 20 client.5595 populate_metadata read hostname 'smithi086' 2024-03-06T07:29:11.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4f180 0x5578cdf77940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 4 client.5595 mds_command: new command op to 5439 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:29:11.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.877+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 --> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5578cde90b40 con 0x5578cdf4f180 2024-03-06T07:29:11.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.878+0000 7ff94d400640 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4f180 0x5578cdf77940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.879+0000 7ff94d400640 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4f180 0x5578cdf77940 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9e723df042ba6717 server_cookie=ede71304038a5152 in_seq=0 out_seq=0 2024-03-06T07:29:11.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.879+0000 7ff93b400640 10 client.5595 ms_handle_connect on v2:172.21.15.194:6832/4064823241 2024-03-06T07:29:11.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff93b400640 1 -- 172.21.15.86:0/2112291192 <== mds.0 v2:172.21.15.194:6832/4064823241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x5578cde90b40 con 0x5578cdf4f180 2024-03-06T07:29:11.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff93b400640 10 client.5595 handle_command_reply: tid=0 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 client.5595 shutdown 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 msgr2=0x5578cdf74cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf74cf0 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7ff94807aa40 tx=0x7ff9480ac040 comp rx=0 tx=0).stop 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 shutdown_connections 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4f180 0x5578cdf77940 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 0x5578cdf4c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 0x5578cdf518d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2112291192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf74cf0 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff93b400640 0 client.5595 ms_handle_reset on v2:172.21.15.194:6832/4064823241 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.881+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 >> 172.21.15.86:0/2112291192 conn(0x5578cdf19e70 msgr2=0x5578cdf19050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.882+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 shutdown_connections 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.882+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2112291192 wait complete. 2024-03-06T07:29:11.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.882+0000 7ff95ffd1740 20 client.5595 trim_cache size 0 max 16384 2024-03-06T07:29:11.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.900+0000 7ff95ffd1740 1 Processor -- start 2024-03-06T07:29:11.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.900+0000 7ff95ffd1740 1 -- start start 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 0x5578cdf4b020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf1abb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 0x5578cdf4e050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9581f85c0 con 0x5578cdf46990 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9581f79c0 con 0x5578cdf46d70 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9581f7fc0 con 0x5578cdf458c0 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf1abb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf1abb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38118/0 (socket says 172.21.15.86:38118) 2024-03-06T07:29:11.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 -- 172.21.15.86:0/884019252 learned_addr learned my addr 172.21.15.86:0/884019252 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:11.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 -- 172.21.15.86:0/884019252 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 msgr2=0x5578cdf4b020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 --2- 172.21.15.86:0/884019252 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 0x5578cdf4b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 -- 172.21.15.86:0/884019252 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 msgr2=0x5578cdf4e050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 --2- 172.21.15.86:0/884019252 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 0x5578cdf4e050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.901+0000 7ff94d400640 1 -- 172.21.15.86:0/884019252 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5578cdf18440 con 0x5578cdf46990 2024-03-06T07:29:11.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.902+0000 7ff94d400640 1 --2- 172.21.15.86:0/884019252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf1abb0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7ff9580c7cb0 tx=0x7ff940004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=986ba855a64512c0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:11.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.902+0000 7ff946000640 1 -- 172.21.15.86:0/884019252 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9400220f0 con 0x5578cdf46990 2024-03-06T07:29:11.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.902+0000 7ff946000640 1 -- 172.21.15.86:0/884019252 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff940022290 con 0x5578cdf46990 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.902+0000 7ff946000640 1 -- 172.21.15.86:0/884019252 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff940013640 con 0x5578cdf46990 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 -- 172.21.15.86:0/884019252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 msgr2=0x5578cdf1abb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/884019252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf1abb0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7ff9580c7cb0 tx=0x7ff940004730 comp rx=0 tx=0).stop 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 -- 172.21.15.86:0/884019252 shutdown_connections 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/884019252 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46d70 0x5578cdf4e050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/884019252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf46990 0x5578cdf1abb0 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/884019252 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf458c0 0x5578cdf4b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 -- 172.21.15.86:0/884019252 >> 172.21.15.86:0/884019252 conn(0x5578cdf19e70 msgr2=0x5578cdf516e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 -- 172.21.15.86:0/884019252 shutdown_connections 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 -- 172.21.15.86:0/884019252 wait complete. 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.903+0000 7ff95ffd1740 1 -- start start 2024-03-06T07:29:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.904+0000 7ff95ffd1740 1 Processor -- start 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf458c0 0x5578cdfc7050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46990 0x5578cdf41270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff95ffd1740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 0x5578cdf417b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5578cde2c930 con 0x5578cdf458c0 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5578cde3fd90 con 0x5578cdf46990 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff95ffd1740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5578cdf44cd0 con 0x5578cdf46d70 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff95ffd1740 10 client.0 _ll_register_callbacks cb 0x7ff94807a450 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf458c0 0x5578cdfc7050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf458c0 0x5578cdfc7050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38120/0 (socket says 172.21.15.86:38120) 2024-03-06T07:29:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 -- 172.21.15.86:0/2481627304 learned_addr learned my addr 172.21.15.86:0/2481627304 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff94d400640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46990 0x5578cdf41270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff956800640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 0x5578cdf417b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 -- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 msgr2=0x5578cdf417b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 0x5578cdf417b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 -- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46990 msgr2=0x5578cdf41270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46990 0x5578cdf41270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff955e00640 1 -- 172.21.15.86:0/2481627304 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5578cdf42260 con 0x5578cdf458c0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.905+0000 7ff94d400640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46990 0x5578cdf41270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff955e00640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf458c0 0x5578cdfc7050 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7ff948082e10 tx=0x7ff9480aa500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4a67eaa92f1fa2c5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff93b400640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff93b400640 1 -- 172.21.15.86:0/2481627304 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff948082f90 con 0x5578cdf458c0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff93b400640 1 -- 172.21.15.86:0/2481627304 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff948078b80 con 0x5578cdf458c0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff940022290 con 0x5578cdf458c0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff93b400640 1 -- 172.21.15.86:0/2481627304 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff94807e9d0 con 0x5578cdf458c0 2024-03-06T07:29:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 10 client.5601 fetch_fsmap learned FSMap version 25 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=2+}) v3 -- 0x5578cdf42260 con 0x5578cdf458c0 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff93b400640 1 -- 172.21.15.86:0/2481627304 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff940022290 con 0x5578cdf458c0 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff93b400640 1 -- 172.21.15.86:0/2481627304 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 25) v1 ==== 3314+0+0 (secure 0 0 0) 0x7ff94807bbb0 con 0x5578cdf458c0 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 10 client.5601 fetch_fsmap finished waiting for FSMap version 25 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 10 client.5601 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 20 client.5601 populate_metadata read hostname 'smithi086' 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4abf0 0x5578cdfd3dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 4 client.5601 mds_command: new command op to 5439 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.906+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 --> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5578cdf151e0 con 0x5578cdf4abf0 2024-03-06T07:29:11.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.907+0000 7ff94d400640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4abf0 0x5578cdfd3dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:11.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.908+0000 7ff94d400640 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4abf0 0x5578cdfd3dc0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2383cbde356c2fe6 server_cookie=c936b4eea869b753 in_seq=0 out_seq=0 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.908+0000 7ff93b400640 10 client.5601 ms_handle_connect on v2:172.21.15.194:6832/4064823241 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff93b400640 1 -- 172.21.15.86:0/2481627304 <== mds.0 v2:172.21.15.194:6832/4064823241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x5578cdf151e0 con 0x5578cdf4abf0 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff93b400640 10 client.5601 handle_command_reply: tid=0 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 client.5601 shutdown 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf458c0 msgr2=0x5578cdfc7050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf458c0 0x5578cdfc7050 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7ff948082e10 tx=0x7ff9480aa500 comp rx=0 tx=0).stop 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 shutdown_connections 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:6832/4064823241,v1:172.21.15.194:6833/4064823241] conn(0x5578cdf4abf0 0x5578cdfd3dc0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5578cdf46d70 0x5578cdf417b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5578cdf46990 0x5578cdf41270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 --2- 172.21.15.86:0/2481627304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5578cdf458c0 0x5578cdfc7050 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff93b400640 0 client.5601 ms_handle_reset on v2:172.21.15.194:6832/4064823241 2024-03-06T07:29:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 >> 172.21.15.86:0/2481627304 conn(0x5578cdf19e70 msgr2=0x5578cdf4e050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 shutdown_connections 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 1 -- 172.21.15.86:0/2481627304 wait complete. 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.909+0000 7ff95ffd1740 20 client.5601 trim_cache size 0 max 16384 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 5439, 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:29:11.914 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 11.530843941000001, 2024-03-06T07:29:11.915 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 25, 2024-03-06T07:29:11.915 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 47, 2024-03-06T07:29:11.915 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 47, 2024-03-06T07:29:11.915 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 17.954956801000002 2024-03-06T07:29:11.915 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:29:11.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff93406b4d0 msgr2=0x7ff93406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff93406b4d0 0x7ff93406d990 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7ff9580b2f60 tx=0x7ff940008040 comp rx=0 tx=0).stop 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9580d1270 msgr2=0x7ff9580cb0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9580d1270 0x7ff9580cb0c0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7ff948000c00 tx=0x7ff9480046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 shutdown_connections 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff93406b4d0 0x7ff93406d990 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff95815e090 0x7ff95814aef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9580d2340 0x7ff95814a9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 --2- 172.21.15.86:0/2763119257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9580d1270 0x7ff9580cb0c0 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 >> 172.21.15.86:0/2763119257 conn(0x7ff9580d3d80 msgr2=0x7ff95815e680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 shutdown_connections 2024-03-06T07:29:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:11.910+0000 7ff957200640 1 -- 172.21.15.86:0/2763119257 wait complete. 2024-03-06T07:29:11.931 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:29:12.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.139+0000 7f47bda00640 1 Processor -- start 2024-03-06T07:29:12.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.139+0000 7f47bda00640 1 -- start start 2024-03-06T07:29:12.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.140+0000 7f47bda00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 0x7f47b80b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.140+0000 7f47bda00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f47b815b090 con 0x7f47b806ec00 2024-03-06T07:29:12.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.140+0000 7f47bda00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f47b80b2a30 con 0x7f47b80b4680 2024-03-06T07:29:12.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.140+0000 7f47bda00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47b80b4360 con 0x7f47b815acb0 2024-03-06T07:29:12.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b7400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f47b806ec00 0x7f47b80ccfb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59040/0 (socket says 172.21.15.86:59040) 2024-03-06T07:29:12.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b7400640 1 -- 172.21.15.86:0/1634889305 learned_addr learned my addr 172.21.15.86:0/1634889305 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:12.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b6a00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 0x7f47b80b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 175878108 0 0) 0x7f47b815b090 con 0x7f47b806ec00 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f479c003520 con 0x7f47b806ec00 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2674919330 0 0) 0x7f479c003520 con 0x7f47b806ec00 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 msgr2=0x7f47b80b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b6000640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 0x7f47b80b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.141+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 >> v1:172.21.15.194:6789/0 conn(0x7f47b80b4680 legacy=0x7f47b806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47b80b2cf0 con 0x7f47b806ec00 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f47b00019f0 con 0x7f47b806ec00 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 >> v1:172.21.15.86:6789/0 conn(0x7f47b806ec00 legacy=0x7f47b80ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f479c0039e0 0x7f479c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c004800 0x7f479c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 0x7f479c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47b80b4360 con 0x7f479c0039e0 2024-03-06T07:29:12.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f479c00ba70 con 0x7f479c004800 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f479c00bc70 con 0x7f47b815acb0 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6a00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f479c0039e0 0x7f479c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b7400640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 0x7f479c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47afe00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c004800 0x7f479c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6a00640 1 -- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 msgr2=0x7f479c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6a00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b815acb0 0x7f479c0051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6a00640 1 -- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c004800 msgr2=0x7f479c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6a00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c004800 0x7f479c004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47b6a00640 1 -- 172.21.15.86:0/1634889305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f479c003520 con 0x7f479c0039e0 2024-03-06T07:29:12.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.142+0000 7f47afe00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c004800 0x7f479c004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:12.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.144+0000 7f47b6a00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f479c0039e0 0x7f479c003e00 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f47a80093c0 tx=0x7f479c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c718422ef6e48b4c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:12.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.144+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47a803b070 con 0x7f479c0039e0 2024-03-06T07:29:12.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.144+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f47a802e890 con 0x7f479c0039e0 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.144+0000 7f47b6000640 1 -- 172.21.15.86:0/1634889305 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47a8009690 con 0x7f479c0039e0 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.144+0000 7f47bda00640 1 -- 172.21.15.86:0/1634889305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f479c0039e0 msgr2=0x7f479c003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.144+0000 7f47bda00640 1 --2- 172.21.15.86:0/1634889305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f479c0039e0 0x7f479c003e00 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f47a80093c0 tx=0x7f479c005830 comp rx=0 tx=0).stop 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.144+0000 7f47b7400640 1 -- 172.21.15.86:0/1634889305 reap_dead start 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.145+0000 7f47bda00640 1 -- 172.21.15.86:0/1634889305 shutdown_connections 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.145+0000 7f47bda00640 1 -- 172.21.15.86:0/1634889305 >> 172.21.15.86:0/1634889305 conn(0x7f47b80b1730 msgr2=0x7f47b80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.145+0000 7f47bda00640 1 -- 172.21.15.86:0/1634889305 shutdown_connections 2024-03-06T07:29:12.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.145+0000 7f47bda00640 1 -- 172.21.15.86:0/1634889305 wait complete. 2024-03-06T07:29:12.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.146+0000 7f47bda00640 1 Processor -- start 2024-03-06T07:29:12.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.146+0000 7f47bda00640 1 -- start start 2024-03-06T07:29:12.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.147+0000 7f47bda00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c0039e0 0x7f47b80cb740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.147+0000 7f47bda00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b81f7be0 0x7f47b81e5d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.147+0000 7f47bda00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f47b814e170 0x7f47b814e600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.147+0000 7f47bda00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47b80b4010 con 0x7f47b814e170 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.147+0000 7f47bda00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47b80b32b0 con 0x7f479c0039e0 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.147+0000 7f47bda00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47b8072340 con 0x7f47b81f7be0 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f47b814e170 0x7f47b814e600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f47b814e170 0x7f47b814e600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38150/0 (socket says 172.21.15.86:38150) 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 -- 172.21.15.86:0/369107270 learned_addr learned my addr 172.21.15.86:0/369107270 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b6a00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c0039e0 0x7f47b80cb740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47afe00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b81f7be0 0x7f47b81e5d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 -- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b81f7be0 msgr2=0x7f47b81e5d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b81f7be0 0x7f47b81e5d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 -- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c0039e0 msgr2=0x7f47b80cb740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c0039e0 0x7f47b80cb740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 -- 172.21.15.86:0/369107270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47b81f8140 con 0x7f47b814e170 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.148+0000 7f47b7400640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f47b814e170 0x7f47b814e600 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f47b8072f10 tx=0x7f47b000b4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.149+0000 7f47aea00640 1 -- 172.21.15.86:0/369107270 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47b000bb80 con 0x7f47b814e170 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.149+0000 7f47aea00640 1 -- 172.21.15.86:0/369107270 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f47b000bd20 con 0x7f47b814e170 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.149+0000 7f47aea00640 1 -- 172.21.15.86:0/369107270 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47b0007290 con 0x7f47b814e170 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.149+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47b81f8420 con 0x7f47b814e170 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.149+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47b81f89e0 con 0x7f47b814e170 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.150+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f47b804f330 con 0x7f47b814e170 2024-03-06T07:29:12.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.151+0000 7f47aea00640 1 -- 172.21.15.86:0/369107270 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f47b001b020 con 0x7f47b814e170 2024-03-06T07:29:12.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.152+0000 7f47aea00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f479406b500 0x7f479406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:12.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.152+0000 7f47aea00640 1 -- 172.21.15.86:0/369107270 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f47b008a450 con 0x7f47b814e170 2024-03-06T07:29:12.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.152+0000 7f47b6a00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f479406b500 0x7f479406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:12.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.153+0000 7f47b6a00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f479406b500 0x7f479406d9c0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f47a8038000 tx=0x7f47a8032000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:12.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.156+0000 7f47aea00640 1 -- 172.21.15.86:0/369107270 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f47b0059230 con 0x7f47b814e170 2024-03-06T07:29:12.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.396+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f47b806ce60 con 0x7f47b814e170 2024-03-06T07:29:12.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.396+0000 7f47aea00640 1 -- 172.21.15.86:0/369107270 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v25) v1 ==== 53+0+73 (secure 0 0 0) 0x7f47b0058b30 con 0x7f47b814e170 2024-03-06T07:29:12.398 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f479406b500 msgr2=0x7f479406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f479406b500 0x7f479406d9c0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f47a8038000 tx=0x7f47a8032000 comp rx=0 tx=0).stop 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f47b814e170 msgr2=0x7f47b814e600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f47b814e170 0x7f47b814e600 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f47b8072f10 tx=0x7f47b000b4a0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 shutdown_connections 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f479406b500 0x7f479406d9c0 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f47b814e170 0x7f47b814e600 unknown :-1 s=CLOSED pgs=540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f47b81f7be0 0x7f47b81e5d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 --2- 172.21.15.86:0/369107270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f479c0039e0 0x7f47b80cb740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 >> 172.21.15.86:0/369107270 conn(0x7f47b80b1730 msgr2=0x7f47b81546b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:12.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 shutdown_connections 2024-03-06T07:29:12.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:12.399+0000 7f47bda00640 1 -- 172.21.15.86:0/369107270 wait complete. 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:29:12.420 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:29:12.420 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:12.421 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:29:12.475 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:29:12.475 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:29:12.475 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:29:12.475 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:12.475 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:29:12.475 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42314sec preferred_lft 42314sec 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:29:12.476 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:29:12.477 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:12.478 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:12.479 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:29:12.566 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:29:12.571 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:12.571 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:29:12.624 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:29:12.625 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:29:12.625 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:29:12.625 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:29:12.625 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:29:12.626 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:12.626 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:29:12.626 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:29:12.626 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:29:12.626 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:29:12.626 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:29:12.626 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:29:12.805 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:12.805 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:29:12.855 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:29:12.855 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:29:12.856 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:29:12.856 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:12.856 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:29:12.946 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:29:12.947 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:12.996 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:29:12.997 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:29:12.997 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:13.045 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:29:13.045 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:13.045 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:29:13.179 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:29:13.180 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:29:13.213 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:29:13.213 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:29:13.214 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:29:13.214 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:29:13.214 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:29:13.216 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:29:13.216 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:29:13.216 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:29:13.216 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:29:13.216 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:29:13.219 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:29:13.220 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:13.220 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:13.304 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:13.304 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:29:13.378 INFO:teuthology.orchestra.run.smithi086.stdout:client5610 2024-03-06T07:29:13.378 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:13.378 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:13.433 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_fragment_size_max 50 2024-03-06T07:29:13.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.651+0000 7f2661e00640 1 Processor -- start 2024-03-06T07:29:13.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.651+0000 7f2661e00640 1 -- start start 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f2661e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c0b3910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f2661e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f265c162310 0x7f265c164700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f2661e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f265c0b3e50 con 0x7f265c0b3530 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f2661e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f265c0ce690 con 0x7f265c162310 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f2661e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f265c0ce970 con 0x7f265c0adf70 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f265b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f265c162310 0x7f265c164700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f265b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f265c162310 0x7f265c164700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38166/0 (socket says 172.21.15.86:38166) 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f265b400640 1 -- 172.21.15.86:0/1055298285 learned_addr learned my addr 172.21.15.86:0/1055298285 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.653+0000 7f2653e00640 1 --2- 172.21.15.86:0/1055298285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c0b3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:13.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.654+0000 7f265b400640 1 -- 172.21.15.86:0/1055298285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 msgr2=0x7f265c0b3910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:13.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.654+0000 7f265b400640 1 --2- 172.21.15.86:0/1055298285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c0b3910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.654+0000 7f265b400640 1 -- 172.21.15.86:0/1055298285 >> v1:172.21.15.194:6789/0 conn(0x7f265c0b3530 legacy=0x7f265c15b780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:13.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.654+0000 7f265b400640 1 -- 172.21.15.86:0/1055298285 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f265c0cec50 con 0x7f265c162310 2024-03-06T07:29:13.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.655+0000 7f265b400640 1 --2- 172.21.15.86:0/1055298285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c164700 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7f265400d150 tx=0x7f265402e160 comp rx=0 tx=0).ready entity=mon.0 client_cookie=47e24bdf143e5bcb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:13.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.655+0000 7f265a000640 1 -- 172.21.15.86:0/1055298285 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f265400b0d0 con 0x7f265c162310 2024-03-06T07:29:13.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.655+0000 7f265a000640 1 -- 172.21.15.86:0/1055298285 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f265400b270 con 0x7f265c162310 2024-03-06T07:29:13.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.655+0000 7f265a000640 1 -- 172.21.15.86:0/1055298285 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2654009600 con 0x7f265c162310 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 -- 172.21.15.86:0/1055298285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 msgr2=0x7f265c164700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 --2- 172.21.15.86:0/1055298285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c164700 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7f265400d150 tx=0x7f265402e160 comp rx=0 tx=0).stop 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 -- 172.21.15.86:0/1055298285 shutdown_connections 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 --2- 172.21.15.86:0/1055298285 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c164700 unknown :-1 s=CLOSED pgs=542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 --2- 172.21.15.86:0/1055298285 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c0b3910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 -- 172.21.15.86:0/1055298285 >> 172.21.15.86:0/1055298285 conn(0x7f265c0b1940 msgr2=0x7f265c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 -- 172.21.15.86:0/1055298285 shutdown_connections 2024-03-06T07:29:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.656+0000 7f2661e00640 1 -- 172.21.15.86:0/1055298285 wait complete. 2024-03-06T07:29:13.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.659+0000 7f2661e00640 1 Processor -- start 2024-03-06T07:29:13.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.659+0000 7f2661e00640 1 -- start start 2024-03-06T07:29:13.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2661e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c1558c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:13.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2661e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f265c0b3530 0x7f265c09b850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:13.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2661e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c156000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:13.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2661e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f265c15aa30 con 0x7f265c162310 2024-03-06T07:29:13.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2661e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f265c0cf210 con 0x7f265c0b3530 2024-03-06T07:29:13.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2661e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f265c15abb0 con 0x7f265c0adf70 2024-03-06T07:29:13.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c156000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:13.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c156000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38178/0 (socket says 172.21.15.86:38178) 2024-03-06T07:29:13.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 -- 172.21.15.86:0/248375961 learned_addr learned my addr 172.21.15.86:0/248375961 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:13.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265aa00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c1558c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:13.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2653e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f265c0b3530 0x7f265c09b850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:13.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 -- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 msgr2=0x7f265c1558c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:13.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c1558c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 -- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f265c0b3530 msgr2=0x7f265c09b850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:13.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f265c0b3530 0x7f265c09b850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265b400640 1 -- 172.21.15.86:0/248375961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f265c09c2c0 con 0x7f265c162310 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f265aa00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c1558c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.661+0000 7f2653e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f265c0b3530 0x7f265c09b850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.662+0000 7f265b400640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c156000 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f265400d270 tx=0x7f2654004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.662+0000 7f2653400640 1 -- 172.21.15.86:0/248375961 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2654044020 con 0x7f265c162310 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.662+0000 7f2653400640 1 -- 172.21.15.86:0/248375961 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f26540369b0 con 0x7f265c162310 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.662+0000 7f2653400640 1 -- 172.21.15.86:0/248375961 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f265400b0d0 con 0x7f265c162310 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.662+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f265c09c5a0 con 0x7f265c162310 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.662+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f265c09cb60 con 0x7f265c162310 2024-03-06T07:29:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.663+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f265c0b4760 con 0x7f265c162310 2024-03-06T07:29:13.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.664+0000 7f2653400640 1 -- 172.21.15.86:0/248375961 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2654047020 con 0x7f265c162310 2024-03-06T07:29:13.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.665+0000 7f2653400640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f263806b4d0 0x7f263806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:13.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.665+0000 7f2653400640 1 -- 172.21.15.86:0/248375961 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f26540ad1f0 con 0x7f265c162310 2024-03-06T07:29:13.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.666+0000 7f265aa00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f263806b4d0 0x7f263806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:13.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.666+0000 7f265aa00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f263806b4d0 0x7f263806d990 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f264c001680 tx=0x7f264c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:13.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.670+0000 7f2653400640 1 -- 172.21.15.86:0/248375961 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f265c0b4760 con 0x7f265c162310 2024-03-06T07:29:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.868+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mds_bal_fragment_size_max}] v 0) v1 -- 0x7f265c157930 con 0x7f265c162310 2024-03-06T07:29:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.874+0000 7f2653400640 1 -- 172.21.15.86:0/248375961 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_fragment_size_max}]=0 v13)=0 v13) v1 ==== 124+0+0 (secure 0 0 0) 0x7f265404e030 con 0x7f265c162310 2024-03-06T07:29:13.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f263806b4d0 msgr2=0x7f263806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:13.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f263806b4d0 0x7f263806d990 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f264c001680 tx=0x7f264c009040 comp rx=0 tx=0).stop 2024-03-06T07:29:13.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 msgr2=0x7f265c156000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:13.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c156000 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f265400d270 tx=0x7f2654004a40 comp rx=0 tx=0).stop 2024-03-06T07:29:13.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 shutdown_connections 2024-03-06T07:29:13.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f263806b4d0 0x7f263806d990 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f265c162310 0x7f265c156000 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f265c0b3530 0x7f265c09b850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 --2- 172.21.15.86:0/248375961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f265c0adf70 0x7f265c1558c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:13.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 >> 172.21.15.86:0/248375961 conn(0x7f265c0b1940 msgr2=0x7f265c156d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:13.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 shutdown_connections 2024-03-06T07:29:13.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:13.877+0000 7f2661e00640 1 -- 172.21.15.86:0/248375961 wait complete. 2024-03-06T07:29:22.974 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:29:22.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:29:23.900 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:23.900 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:23.958 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:29:23.958 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:29:23.958 DEBUG:teuthology.orchestra.run.smithi086:> import uuid 2024-03-06T07:29:23.958 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:29:23.958 DEBUG:teuthology.orchestra.run.smithi086:> n = 51 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = False 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> if 0 > 0: 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = True 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> dpath = os.path.dirname(path) 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> fnameprefix = os.path.basename(path) 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> os.makedirs(dpath, exist_ok=True) 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> try: 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> for i in range(n): 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> with open(fpath, '"'"'w'"'"') as f: 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> f.write(f"{i}") 2024-03-06T07:29:23.959 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> f.flush() 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(f.fileno()) 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> os.unlink(fpath) 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> if create_hard_links: 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> for j in range(0): 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> if True: 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> finally: 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> os.close(dirfd) 2024-03-06T07:29:23.960 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:29:24.187 INFO:teuthology.orchestra.run.smithi086.stderr:Traceback (most recent call last): 2024-03-06T07:29:24.187 INFO:teuthology.orchestra.run.smithi086.stderr: File "", line 33, in 2024-03-06T07:29:24.187 INFO:teuthology.orchestra.run.smithi086.stderr:OSError: [Errno 28] No space left on device 2024-03-06T07:29:24.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24 stdin-killer INFO: command exited with status 1: exiting normally with same code! 2024-03-06T07:29:24.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:29:24.206 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:29:24.207 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:29:24.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:29:24.473 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:29:24.473 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:24.473 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:24.528 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:24.528 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:24.574 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:24.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.826+0000 7ff441e00640 1 Processor -- start 2024-03-06T07:29:24.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.826+0000 7ff441e00640 1 -- start start 2024-03-06T07:29:24.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.827+0000 7ff441e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff43c1620c0 0x7ff43c0b28e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.827+0000 7ff441e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff43c161900 con 0x7ff43c0ae670 2024-03-06T07:29:24.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.827+0000 7ff441e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff43c15ebd0 con 0x7ff43c0b40c0 2024-03-06T07:29:24.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.827+0000 7ff441e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff43c15eeb0 con 0x7ff43c1620c0 2024-03-06T07:29:24.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff433e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff43c0ae670 0x7ff43c15e2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36770/0 (socket says 172.21.15.86:36770) 2024-03-06T07:29:24.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff433e00640 1 -- 172.21.15.86:0/3244770119 learned_addr learned my addr 172.21.15.86:0/3244770119 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:24.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff441400640 1 --2- 172.21.15.86:0/3244770119 >> v2:172.21.15.194:3300/0 conn(0x7ff43c1620c0 0x7ff43c0b28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2352537119 0 0) 0x7ff43c161900 con 0x7ff43c0ae670 2024-03-06T07:29:24.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff420003520 con 0x7ff43c0ae670 2024-03-06T07:29:24.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3532371423 0 0) 0x7ff420003520 con 0x7ff43c0ae670 2024-03-06T07:29:24.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 >> v2:172.21.15.194:3300/0 conn(0x7ff43c1620c0 msgr2=0x7ff43c0b28e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:24.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 --2- 172.21.15.86:0/3244770119 >> v2:172.21.15.194:3300/0 conn(0x7ff43c1620c0 0x7ff43c0b28e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:24.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 >> v1:172.21.15.194:6789/0 conn(0x7ff43c0b40c0 legacy=0x7ff43c0b44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:24.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff43c15f190 con 0x7ff43c0ae670 2024-03-06T07:29:24.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.828+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff424001670 con 0x7ff43c0ae670 2024-03-06T07:29:24.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff441400640 1 --2- 172.21.15.86:0/3244770119 >> v2:172.21.15.194:3300/0 conn(0x7ff43c1620c0 0x7ff43c0b28e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 >> v1:172.21.15.86:6789/0 conn(0x7ff43c0ae670 legacy=0x7ff43c15e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff43be00640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200039b0 0x7ff420003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff43be00640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420004800 0x7ff420004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff43be00640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4200051c0 0x7ff42000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff43c15eeb0 con 0x7ff4200051c0 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff42000c010 con 0x7ff420004800 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff42000c210 con 0x7ff4200039b0 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff441400640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4200051c0 0x7ff42000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff440a00640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200039b0 0x7ff420003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff433e00640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420004800 0x7ff420004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff441400640 1 -- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200039b0 msgr2=0x7ff420003d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff441400640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200039b0 0x7ff420003d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff441400640 1 -- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420004800 msgr2=0x7ff420004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:24.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff441400640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff420004800 0x7ff420004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:24.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.829+0000 7ff441400640 1 -- 172.21.15.86:0/3244770119 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff420003520 con 0x7ff4200051c0 2024-03-06T07:29:24.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.830+0000 7ff441400640 1 -- 172.21.15.86:0/3244770119 reap_dead start 2024-03-06T07:29:24.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.830+0000 7ff441400640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4200051c0 0x7ff42000b9c0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7ff43400e350 tx=0x7ff420005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c87ac11ac48245 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:24.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff434039070 con 0x7ff4200051c0 2024-03-06T07:29:24.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff43402ce20 con 0x7ff4200051c0 2024-03-06T07:29:24.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff43be00640 1 -- 172.21.15.86:0/3244770119 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff434009de0 con 0x7ff4200051c0 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff441e00640 1 -- 172.21.15.86:0/3244770119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4200051c0 msgr2=0x7ff42000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff441e00640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4200051c0 0x7ff42000b9c0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7ff43400e350 tx=0x7ff420005830 comp rx=0 tx=0).stop 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff441e00640 1 -- 172.21.15.86:0/3244770119 shutdown_connections 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff441e00640 1 --2- 172.21.15.86:0/3244770119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4200051c0 0x7ff42000b9c0 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff441e00640 1 -- 172.21.15.86:0/3244770119 >> 172.21.15.86:0/3244770119 conn(0x7ff43c06afc0 msgr2=0x7ff43c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff441e00640 1 -- 172.21.15.86:0/3244770119 shutdown_connections 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.831+0000 7ff441e00640 1 -- 172.21.15.86:0/3244770119 wait complete. 2024-03-06T07:29:24.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.833+0000 7ff441e00640 1 Processor -- start 2024-03-06T07:29:24.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.833+0000 7ff441e00640 1 -- start start 2024-03-06T07:29:24.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.834+0000 7ff441e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200051c0 0x7ff43c1411a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.834+0000 7ff441e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff43c071900 0x7ff43c071d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.834+0000 7ff441e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff43c1f71d0 0x7ff43c1f7680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.834+0000 7ff441e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff43c1587e0 con 0x7ff43c071900 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.834+0000 7ff441e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff43c158660 con 0x7ff43c1f71d0 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.834+0000 7ff441e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff43c162570 con 0x7ff4200051c0 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.834+0000 7ff433e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff43c071900 0x7ff43c071d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff43c071900 0x7ff43c071d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42438/0 (socket says 172.21.15.86:42438) 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 -- 172.21.15.86:0/2037951987 learned_addr learned my addr 172.21.15.86:0/2037951987 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff441400640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff43c1f71d0 0x7ff43c1f7680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff440a00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200051c0 0x7ff43c1411a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 -- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200051c0 msgr2=0x7ff43c1411a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200051c0 0x7ff43c1411a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 -- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff43c1f71d0 msgr2=0x7ff43c1f7680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff43c1f71d0 0x7ff43c1f7680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:24.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 -- 172.21.15.86:0/2037951987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff43c0b1810 con 0x7ff43c071900 2024-03-06T07:29:24.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff433e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff43c071900 0x7ff43c071d80 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7ff42400e700 tx=0x7ff42400ebc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:24.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff439600640 1 -- 172.21.15.86:0/2037951987 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff42400c6c0 con 0x7ff43c071900 2024-03-06T07:29:24.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.835+0000 7ff439600640 1 -- 172.21.15.86:0/2037951987 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff424018070 con 0x7ff43c071900 2024-03-06T07:29:24.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.836+0000 7ff439600640 1 -- 172.21.15.86:0/2037951987 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff42400caa0 con 0x7ff43c071900 2024-03-06T07:29:24.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.836+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff43c0b1af0 con 0x7ff43c071900 2024-03-06T07:29:24.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.836+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff43c0b20b0 con 0x7ff43c071900 2024-03-06T07:29:24.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.837+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff43c0722c0 con 0x7ff43c071900 2024-03-06T07:29:24.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.838+0000 7ff439600640 1 -- 172.21.15.86:0/2037951987 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff42400cc40 con 0x7ff43c071900 2024-03-06T07:29:24.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.839+0000 7ff439600640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff41806b4d0 0x7ff41806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:24.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.839+0000 7ff439600640 1 -- 172.21.15.86:0/2037951987 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff424089f90 con 0x7ff43c071900 2024-03-06T07:29:24.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.839+0000 7ff440a00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff41806b4d0 0x7ff41806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:24.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.839+0000 7ff440a00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff41806b4d0 0x7ff41806d990 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7ff42c0054c0 tx=0x7ff42c007b90 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:24.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:24.843+0000 7ff439600640 1 -- 172.21.15.86:0/2037951987 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff424058e00 con 0x7ff43c071900 2024-03-06T07:29:25.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.046+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7ff43c04f330 con 0x7ff43c071900 2024-03-06T07:29:25.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.049+0000 7ff439600640 1 -- 172.21.15.86:0/2037951987 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v47) v1 ==== 74+0+16625 (secure 0 0 0) 0x7ff424058700 con 0x7ff43c071900 2024-03-06T07:29:25.052 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:25.053 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":47,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:01.377099+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:57.609038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:58.840885+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:25.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.052+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff41806b4d0 msgr2=0x7ff41806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.052+0000 7ff441e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff41806b4d0 0x7ff41806d990 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7ff42c0054c0 tx=0x7ff42c007b90 comp rx=0 tx=0).stop 2024-03-06T07:29:25.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.052+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff43c071900 msgr2=0x7ff43c071d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.052+0000 7ff441e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff43c071900 0x7ff43c071d80 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7ff42400e700 tx=0x7ff42400ebc0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 shutdown_connections 2024-03-06T07:29:25.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff41806b4d0 0x7ff41806d990 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff43c1f71d0 0x7ff43c1f7680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff43c071900 0x7ff43c071d80 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 --2- 172.21.15.86:0/2037951987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4200051c0 0x7ff43c1411a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 >> 172.21.15.86:0/2037951987 conn(0x7ff43c06afc0 msgr2=0x7ff43c0ceed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:25.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 shutdown_connections 2024-03-06T07:29:25.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.053+0000 7ff441e00640 1 -- 172.21.15.86:0/2037951987 wait complete. 2024-03-06T07:29:25.076 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 6, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:28:57.609038+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '46', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 7, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:28:58.840885+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '47', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:29:25.076 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:29:25.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.272+0000 7fc426600640 1 Processor -- start 2024-03-06T07:29:25.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.272+0000 7fc426600640 1 -- start start 2024-03-06T07:29:25.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc426600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc420160470 0x7fc42006afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc426600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc42006b500 0x7fc4200cd2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc426600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc42006f6d0 con 0x7fc420145690 2024-03-06T07:29:25.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc426600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc4200cce60 con 0x7fc420160470 2024-03-06T07:29:25.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc426600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc42006f9b0 con 0x7fc42006b500 2024-03-06T07:29:25.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc41f400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc420145690 0x7fc42006ef10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36780/0 (socket says 172.21.15.86:36780) 2024-03-06T07:29:25.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc416a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc420160470 0x7fc42006afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc41f400640 1 -- 172.21.15.86:0/462918025 learned_addr learned my addr 172.21.15.86:0/462918025 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:25.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.274+0000 7fc41fe00640 1 --2- 172.21.15.86:0/462918025 >> v2:172.21.15.194:3300/0 conn(0x7fc42006b500 0x7fc4200cd2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.275+0000 7fc416a00640 1 -- 172.21.15.86:0/462918025 >> v2:172.21.15.194:3300/0 conn(0x7fc42006b500 msgr2=0x7fc4200cd2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:25.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.275+0000 7fc416a00640 1 --2- 172.21.15.86:0/462918025 >> v2:172.21.15.194:3300/0 conn(0x7fc42006b500 0x7fc4200cd2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.275+0000 7fc416a00640 1 -- 172.21.15.86:0/462918025 >> v1:172.21.15.86:6789/0 conn(0x7fc420145690 legacy=0x7fc42006ef10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.275+0000 7fc416a00640 1 -- 172.21.15.86:0/462918025 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4200ccaf0 con 0x7fc420160470 2024-03-06T07:29:25.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.275+0000 7fc41ea00640 1 -- 172.21.15.86:0/462918025 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2173477160 0 0) 0x7fc42006f6d0 con 0x7fc420145690 2024-03-06T07:29:25.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.276+0000 7fc416a00640 1 --2- 172.21.15.86:0/462918025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42006afc0 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7fc42015f8a0 tx=0x7fc40802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb14f7a44ecc96bc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:25.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.276+0000 7fc41ea00640 1 -- 172.21.15.86:0/462918025 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc408035020 con 0x7fc420160470 2024-03-06T07:29:25.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.276+0000 7fc41ea00640 1 -- 172.21.15.86:0/462918025 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc408031830 con 0x7fc420160470 2024-03-06T07:29:25.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.276+0000 7fc41ea00640 1 -- 172.21.15.86:0/462918025 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc408031c00 con 0x7fc420160470 2024-03-06T07:29:25.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 -- 172.21.15.86:0/462918025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 msgr2=0x7fc42006afc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 --2- 172.21.15.86:0/462918025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42006afc0 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7fc42015f8a0 tx=0x7fc40802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 -- 172.21.15.86:0/462918025 shutdown_connections 2024-03-06T07:29:25.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 --2- 172.21.15.86:0/462918025 >> v2:172.21.15.194:3300/0 conn(0x7fc42006b500 0x7fc4200cd2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 --2- 172.21.15.86:0/462918025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42006afc0 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 -- 172.21.15.86:0/462918025 >> 172.21.15.86:0/462918025 conn(0x7fc4200b27d0 msgr2=0x7fc4200b2bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:25.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 -- 172.21.15.86:0/462918025 shutdown_connections 2024-03-06T07:29:25.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.277+0000 7fc426600640 1 -- 172.21.15.86:0/462918025 wait complete. 2024-03-06T07:29:25.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.279+0000 7fc426600640 1 Processor -- start 2024-03-06T07:29:25.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.279+0000 7fc426600640 1 -- start start 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc426600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc42006b500 0x7fc4201e6bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc426600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc420145690 0x7fc42015f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc426600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42015b030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc426600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4200b3ad0 con 0x7fc420160470 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc426600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4200cc7a0 con 0x7fc42006b500 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc426600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc4200cd830 con 0x7fc420145690 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42015b030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42015b030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42460/0 (socket says 172.21.15.86:42460) 2024-03-06T07:29:25.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 -- 172.21.15.86:0/3009855960 learned_addr learned my addr 172.21.15.86:0/3009855960 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41f400640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc42006b500 0x7fc4201e6bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc416a00640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc420145690 0x7fc42015f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 -- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc420145690 msgr2=0x7fc42015f140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc420145690 0x7fc42015f140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 -- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc42006b500 msgr2=0x7fc4201e6bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc42006b500 0x7fc4201e6bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc41fe00640 1 -- 172.21.15.86:0/3009855960 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4201e5b50 con 0x7fc420160470 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.280+0000 7fc416a00640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc420145690 0x7fc42015f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:25.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.281+0000 7fc41fe00640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42015b030 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fc41800d800 tx=0x7fc41800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:25.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.281+0000 7fc417e00640 1 -- 172.21.15.86:0/3009855960 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc41800b6c0 con 0x7fc420160470 2024-03-06T07:29:25.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.281+0000 7fc417e00640 1 -- 172.21.15.86:0/3009855960 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc418019070 con 0x7fc420160470 2024-03-06T07:29:25.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.281+0000 7fc417e00640 1 -- 172.21.15.86:0/3009855960 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc41800bac0 con 0x7fc420160470 2024-03-06T07:29:25.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.281+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc4201e5e30 con 0x7fc420160470 2024-03-06T07:29:25.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.281+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc4201e63f0 con 0x7fc420160470 2024-03-06T07:29:25.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.282+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc42004f330 con 0x7fc420160470 2024-03-06T07:29:25.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.283+0000 7fc417e00640 1 -- 172.21.15.86:0/3009855960 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc41800bc60 con 0x7fc420160470 2024-03-06T07:29:25.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.284+0000 7fc417e00640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3fc06b4d0 0x7fc3fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.284+0000 7fc417e00640 1 -- 172.21.15.86:0/3009855960 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc41808a710 con 0x7fc420160470 2024-03-06T07:29:25.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.284+0000 7fc41f400640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3fc06b4d0 0x7fc3fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.285+0000 7fc41f400640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3fc06b4d0 0x7fc3fc06d990 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7fc4100012e0 tx=0x7fc410007ef0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:25.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.289+0000 7fc417e00640 1 -- 172.21.15.86:0/3009855960 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc418059580 con 0x7fc420160470 2024-03-06T07:29:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.483+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fc4200d06c0 con 0x7fc420160470 2024-03-06T07:29:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.484+0000 7fc417e00640 1 -- 172.21.15.86:0/3009855960 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v47) v1 ==== 93+0+10 (secure 0 0 0) 0x7fc418058e80 con 0x7fc420160470 2024-03-06T07:29:25.487 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:29:25.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3fc06b4d0 msgr2=0x7fc3fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3fc06b4d0 0x7fc3fc06d990 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7fc4100012e0 tx=0x7fc410007ef0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 msgr2=0x7fc42015b030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42015b030 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fc41800d800 tx=0x7fc41800dcc0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 shutdown_connections 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3fc06b4d0 0x7fc3fc06d990 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc420160470 0x7fc42015b030 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc420145690 0x7fc42015f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 --2- 172.21.15.86:0/3009855960 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc42006b500 0x7fc4201e6bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 >> 172.21.15.86:0/3009855960 conn(0x7fc4200b27d0 msgr2=0x7fc4200b09e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 shutdown_connections 2024-03-06T07:29:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.487+0000 7fc426600640 1 -- 172.21.15.86:0/3009855960 wait complete. 2024-03-06T07:29:25.508 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:29:25.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.717+0000 7ff761200640 1 Processor -- start 2024-03-06T07:29:25.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.717+0000 7ff761200640 1 -- start start 2024-03-06T07:29:25.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff761200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff75c0b3650 0x7ff75c0727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff761200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 0x7ff75c06c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff761200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff75c0719e0 con 0x7ff75c0b3df0 2024-03-06T07:29:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff761200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff75c072310 con 0x7ff75c0b3650 2024-03-06T07:29:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff761200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff75c071cc0 con 0x7ff75c0cda90 2024-03-06T07:29:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff759e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff75c0b3df0 0x7ff75c0b41d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36782/0 (socket says 172.21.15.86:36782) 2024-03-06T07:29:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff759e00640 1 -- 172.21.15.86:0/927533745 learned_addr learned my addr 172.21.15.86:0/927533745 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff75a800640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 0x7ff75c06c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.719+0000 7ff759400640 1 --2- 172.21.15.86:0/927533745 >> v2:172.21.15.194:3300/0 conn(0x7ff75c0b3650 0x7ff75c0727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3916481668 0 0) 0x7ff75c0719e0 con 0x7ff75c0b3df0 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff740005730 con 0x7ff75c0b3df0 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1124225298 0 0) 0x7ff740005730 con 0x7ff75c0b3df0 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 msgr2=0x7ff75c06c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 0x7ff75c06c990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 >> v2:172.21.15.194:3300/0 conn(0x7ff75c0b3650 msgr2=0x7ff75c0727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 --2- 172.21.15.86:0/927533745 >> v2:172.21.15.194:3300/0 conn(0x7ff75c0b3650 0x7ff75c0727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff75c071fa0 con 0x7ff75c0b3df0 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff74c001670 con 0x7ff75c0b3df0 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 >> v1:172.21.15.86:6789/0 conn(0x7ff75c0b3df0 legacy=0x7ff75c0b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 0x7ff740005bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff740006190 0x7ff740006a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006f50 0x7ff7400073d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff75c072310 con 0x7ff740006190 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff75c071cc0 con 0x7ff740006f50 2024-03-06T07:29:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff74000dbd0 con 0x7ff75c0cda90 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.720+0000 7ff759400640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff740006190 0x7ff740006a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.721+0000 7ff759e00640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 0x7ff740005bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.721+0000 7ff75a800640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006f50 0x7ff7400073d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.721+0000 7ff759400640 1 -- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 msgr2=0x7ff740005bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.721+0000 7ff759400640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c0cda90 0x7ff740005bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.721+0000 7ff759400640 1 -- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006f50 msgr2=0x7ff7400073d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.721+0000 7ff759400640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006f50 0x7ff7400073d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.721+0000 7ff759400640 1 -- 172.21.15.86:0/927533745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff740005730 con 0x7ff740006190 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.722+0000 7ff759400640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff740006190 0x7ff740006a10 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7ff748009af0 tx=0x7ff740007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d738e91955bedb20 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.722+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff748039070 con 0x7ff740006190 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.722+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff74802c710 con 0x7ff740006190 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.722+0000 7ff758a00640 1 -- 172.21.15.86:0/927533745 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff748009690 con 0x7ff740006190 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.723+0000 7ff761200640 1 -- 172.21.15.86:0/927533745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff740006190 msgr2=0x7ff740006a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.723+0000 7ff761200640 1 --2- 172.21.15.86:0/927533745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff740006190 0x7ff740006a10 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7ff748009af0 tx=0x7ff740007a40 comp rx=0 tx=0).stop 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.723+0000 7ff75a800640 1 -- 172.21.15.86:0/927533745 reap_dead start 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.723+0000 7ff761200640 1 -- 172.21.15.86:0/927533745 shutdown_connections 2024-03-06T07:29:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.723+0000 7ff761200640 1 -- 172.21.15.86:0/927533745 >> 172.21.15.86:0/927533745 conn(0x7ff75c0b1940 msgr2=0x7ff75c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:25.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.723+0000 7ff761200640 1 -- 172.21.15.86:0/927533745 shutdown_connections 2024-03-06T07:29:25.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.723+0000 7ff761200640 1 -- 172.21.15.86:0/927533745 wait complete. 2024-03-06T07:29:25.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.725+0000 7ff761200640 1 Processor -- start 2024-03-06T07:29:25.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.725+0000 7ff761200640 1 -- start start 2024-03-06T07:29:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff761200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006190 0x7ff75c150b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff761200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff75c151d70 0x7ff75c0affc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff761200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c147d00 0x7ff75c148180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff761200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff75c15b3c0 con 0x7ff75c151d70 2024-03-06T07:29:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff761200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff75c15b240 con 0x7ff740006190 2024-03-06T07:29:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff761200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff75c0cf510 con 0x7ff75c147d00 2024-03-06T07:29:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff759400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff75c151d70 0x7ff75c0affc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff759400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff75c151d70 0x7ff75c0affc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42470/0 (socket says 172.21.15.86:42470) 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.726+0000 7ff759400640 1 -- 172.21.15.86:0/3562710333 learned_addr learned my addr 172.21.15.86:0/3562710333 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff759e00640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006190 0x7ff75c150b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff75a800640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c147d00 0x7ff75c148180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff759400640 1 -- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c147d00 msgr2=0x7ff75c148180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff759400640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c147d00 0x7ff75c148180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff759400640 1 -- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006190 msgr2=0x7ff75c150b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff759400640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006190 0x7ff75c150b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff759400640 1 -- 172.21.15.86:0/3562710333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff75c152200 con 0x7ff75c151d70 2024-03-06T07:29:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff759400640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff75c151d70 0x7ff75c0affc0 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7ff748037040 tx=0x7ff748004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.727+0000 7ff752000640 1 -- 172.21.15.86:0/3562710333 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff748039050 con 0x7ff75c151d70 2024-03-06T07:29:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.728+0000 7ff752000640 1 -- 172.21.15.86:0/3562710333 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff748045070 con 0x7ff75c151d70 2024-03-06T07:29:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.728+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff75c1524e0 con 0x7ff75c151d70 2024-03-06T07:29:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.728+0000 7ff752000640 1 -- 172.21.15.86:0/3562710333 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff7480044e0 con 0x7ff75c151d70 2024-03-06T07:29:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.728+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff75c152aa0 con 0x7ff75c151d70 2024-03-06T07:29:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.729+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff75c04f330 con 0x7ff75c151d70 2024-03-06T07:29:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.730+0000 7ff752000640 1 -- 172.21.15.86:0/3562710333 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff748003830 con 0x7ff75c151d70 2024-03-06T07:29:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.731+0000 7ff752000640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff73c06b4d0 0x7ff73c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.731+0000 7ff752000640 1 -- 172.21.15.86:0/3562710333 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff7480ab890 con 0x7ff75c151d70 2024-03-06T07:29:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.731+0000 7ff759e00640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff73c06b4d0 0x7ff73c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:25.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.732+0000 7ff759e00640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff73c06b4d0 0x7ff73c06d990 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7ff74c001050 tx=0x7ff74c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:25.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.735+0000 7ff752000640 1 -- 172.21.15.86:0/3562710333 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff74807a700 con 0x7ff75c151d70 2024-03-06T07:29:25.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.934+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7ff75c0b4650 con 0x7ff75c151d70 2024-03-06T07:29:25.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.935+0000 7ff752000640 1 -- 172.21.15.86:0/3562710333 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v47) v1 ==== 104+0+11 (secure 0 0 0) 0x7ff74807a000 con 0x7ff75c151d70 2024-03-06T07:29:25.937 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:29:25.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.937+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff73c06b4d0 msgr2=0x7ff73c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.937+0000 7ff761200640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff73c06b4d0 0x7ff73c06d990 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7ff74c001050 tx=0x7ff74c009040 comp rx=0 tx=0).stop 2024-03-06T07:29:25.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.937+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff75c151d70 msgr2=0x7ff75c0affc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:25.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.937+0000 7ff761200640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff75c151d70 0x7ff75c0affc0 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7ff748037040 tx=0x7ff748004730 comp rx=0 tx=0).stop 2024-03-06T07:29:25.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 shutdown_connections 2024-03-06T07:29:25.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff73c06b4d0 0x7ff73c06d990 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff75c147d00 0x7ff75c148180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff75c151d70 0x7ff75c0affc0 unknown :-1 s=CLOSED pgs=552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 --2- 172.21.15.86:0/3562710333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff740006190 0x7ff75c150b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:25.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 >> 172.21.15.86:0/3562710333 conn(0x7ff75c0b1940 msgr2=0x7ff75c0b2a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:25.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 shutdown_connections 2024-03-06T07:29:25.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:25.938+0000 7ff761200640 1 -- 172.21.15.86:0/3562710333 wait complete. 2024-03-06T07:29:25.960 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:29:26.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.169+0000 7f99c8e00640 1 Processor -- start 2024-03-06T07:29:26.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.169+0000 7f99c8e00640 1 -- start start 2024-03-06T07:29:26.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.170+0000 7f99c8e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 0x7f99c40ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.170+0000 7f99c8e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f99c415c1f0 con 0x7f99c406da40 2024-03-06T07:29:26.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.170+0000 7f99c8e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f99c406cac0 con 0x7f99c415be10 2024-03-06T07:29:26.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.170+0000 7f99c8e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99c40cd200 con 0x7f99c40aef20 2024-03-06T07:29:26.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.171+0000 7f99baa00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f99c406da40 0x7f99c4071ab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47618/0 (socket says 172.21.15.86:47618) 2024-03-06T07:29:26.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.171+0000 7f99baa00640 1 -- 172.21.15.86:0/2857872950 learned_addr learned my addr 172.21.15.86:0/2857872950 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:26.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.171+0000 7f99c3e00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 0x7f99c40ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.171+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 40839396 0 0) 0x7f99c415c1f0 con 0x7f99c406da40 2024-03-06T07:29:26.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.171+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f99a8003520 con 0x7f99c406da40 2024-03-06T07:29:26.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.171+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1958715627 0 0) 0x7f99a8003520 con 0x7f99c406da40 2024-03-06T07:29:26.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 msgr2=0x7f99c40ccc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 0x7f99c40ccc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 >> v1:172.21.15.194:6789/0 conn(0x7f99c415be10 legacy=0x7f99c406d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99c406cda0 con 0x7f99c406da40 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f99ac001670 con 0x7f99c406da40 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 >> v1:172.21.15.86:6789/0 conn(0x7f99c406da40 legacy=0x7f99c4071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99a80039e0 0x7f99a8003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a8004800 0x7f99a8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 0x7f99a80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99c40cd200 con 0x7f99a80039e0 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99a800ba70 con 0x7f99a8004800 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99a800bc70 con 0x7f99c40aef20 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c3400640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99a80039e0 0x7f99a8003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c3e00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 0x7f99a80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99baa00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a8004800 0x7f99a8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c3400640 1 -- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 msgr2=0x7f99a80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:26.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c3400640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c40aef20 0x7f99a80051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c3400640 1 -- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a8004800 msgr2=0x7f99a8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:26.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c3400640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a8004800 0x7f99a8004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.172+0000 7f99c3400640 1 -- 172.21.15.86:0/2857872950 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99a8003520 con 0x7f99a80039e0 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.173+0000 7f99c3400640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99a80039e0 0x7f99a8003e00 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7f99b4001720 tx=0x7f99a8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d4eb69de033c9f10 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99b4001370 con 0x7f99a80039e0 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99b40052d0 con 0x7f99a80039e0 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c2a00640 1 -- 172.21.15.86:0/2857872950 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99b4003830 con 0x7f99a80039e0 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c8e00640 1 -- 172.21.15.86:0/2857872950 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99a80039e0 msgr2=0x7f99a8003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c8e00640 1 --2- 172.21.15.86:0/2857872950 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99a80039e0 0x7f99a8003e00 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7f99b4001720 tx=0x7f99a8005830 comp rx=0 tx=0).stop 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c3e00640 1 -- 172.21.15.86:0/2857872950 reap_dead start 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c8e00640 1 -- 172.21.15.86:0/2857872950 shutdown_connections 2024-03-06T07:29:26.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c8e00640 1 -- 172.21.15.86:0/2857872950 >> 172.21.15.86:0/2857872950 conn(0x7f99c406afc0 msgr2=0x7f99c406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:26.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c8e00640 1 -- 172.21.15.86:0/2857872950 shutdown_connections 2024-03-06T07:29:26.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.174+0000 7f99c8e00640 1 -- 172.21.15.86:0/2857872950 wait complete. 2024-03-06T07:29:26.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.176+0000 7f99c8e00640 1 Processor -- start 2024-03-06T07:29:26.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.176+0000 7f99c8e00640 1 -- start start 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.177+0000 7f99c8e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a80039e0 0x7f99c41473f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.177+0000 7f99c8e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c41506b0 0x7f99c4150b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.177+0000 7f99c8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99c40b39c0 0x7f99c40b3e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.177+0000 7f99c8e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99c40cda90 con 0x7f99c40b39c0 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.177+0000 7f99c8e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99c406d080 con 0x7f99a80039e0 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.177+0000 7f99c8e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99c40ccc20 con 0x7f99c41506b0 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99c40b39c0 0x7f99c40b3e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99c40b39c0 0x7f99c40b3e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50726/0 (socket says 172.21.15.86:50726) 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 -- 172.21.15.86:0/1810629908 learned_addr learned my addr 172.21.15.86:0/1810629908 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3400640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a80039e0 0x7f99c41473f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99baa00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c41506b0 0x7f99c4150b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 -- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c41506b0 msgr2=0x7f99c4150b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c41506b0 0x7f99c4150b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 -- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a80039e0 msgr2=0x7f99c41473f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a80039e0 0x7f99c41473f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 -- 172.21.15.86:0/1810629908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99c41f3f10 con 0x7f99c40b39c0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99c3e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99c40b39c0 0x7f99c40b3e40 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f99c4071110 tx=0x7f99bc00f4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.178+0000 7f99bbe00640 1 -- 172.21.15.86:0/1810629908 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99bc01c070 con 0x7f99c40b39c0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.179+0000 7f99bbe00640 1 -- 172.21.15.86:0/1810629908 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99bc017400 con 0x7f99c40b39c0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.179+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99c41f41f0 con 0x7f99c40b39c0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.179+0000 7f99bbe00640 1 -- 172.21.15.86:0/1810629908 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99bc0177f0 con 0x7f99c40b39c0 2024-03-06T07:29:26.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.179+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99c41f47b0 con 0x7f99c40b39c0 2024-03-06T07:29:26.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.180+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f99c40b4380 con 0x7f99c40b39c0 2024-03-06T07:29:26.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.181+0000 7f99bbe00640 1 -- 172.21.15.86:0/1810629908 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f99bc01f070 con 0x7f99c40b39c0 2024-03-06T07:29:26.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.181+0000 7f99bbe00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99a006b500 0x7f99a006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.182+0000 7f99bbe00640 1 -- 172.21.15.86:0/1810629908 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f99bc08a0f0 con 0x7f99c40b39c0 2024-03-06T07:29:26.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.182+0000 7f99c3400640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99a006b500 0x7f99a006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.183+0000 7f99c3400640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99a006b500 0x7f99a006d9c0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f99b4033000 tx=0x7f99b4009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:26.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.186+0000 7f99bbe00640 1 -- 172.21.15.86:0/1810629908 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f99bc012070 con 0x7f99c40b39c0 2024-03-06T07:29:26.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.381+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f99c404f330 con 0x7f99c40b39c0 2024-03-06T07:29:26.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.382+0000 7f99bbe00640 1 -- 172.21.15.86:0/1810629908 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v47) v1 ==== 100+0+11 (secure 0 0 0) 0x7f99bc058ed0 con 0x7f99c40b39c0 2024-03-06T07:29:26.384 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:29:26.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99a006b500 msgr2=0x7f99a006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99a006b500 0x7f99a006d9c0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f99b4033000 tx=0x7f99b4009030 comp rx=0 tx=0).stop 2024-03-06T07:29:26.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99c40b39c0 msgr2=0x7f99c40b3e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99c40b39c0 0x7f99c40b3e40 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f99c4071110 tx=0x7f99bc00f4a0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 shutdown_connections 2024-03-06T07:29:26.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99a006b500 0x7f99a006d9c0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99c40b39c0 0x7f99c40b3e40 unknown :-1 s=CLOSED pgs=555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99c41506b0 0x7f99c4150b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 --2- 172.21.15.86:0/1810629908 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99a80039e0 0x7f99c41473f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 >> 172.21.15.86:0/1810629908 conn(0x7f99c406afc0 msgr2=0x7f99c40cf5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 shutdown_connections 2024-03-06T07:29:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.385+0000 7f99c8e00640 1 -- 172.21.15.86:0/1810629908 wait complete. 2024-03-06T07:29:26.408 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:29:26.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.614+0000 7f3414000640 1 Processor -- start 2024-03-06T07:29:26.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.614+0000 7f3414000640 1 -- start start 2024-03-06T07:29:26.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3414000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f340c15ac90 0x7f340c06e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3414000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f340c15f120 0x7f340c0c4a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3414000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f340c0cc000 con 0x7f340c15b070 2024-03-06T07:29:26.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3414000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f340c0ce360 con 0x7f340c15ac90 2024-03-06T07:29:26.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3414000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f340c0ce640 con 0x7f340c15f120 2024-03-06T07:29:26.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3411600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f340c15ac90 0x7f340c06e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3410c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f340c15b070 0x7f340c15ea10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47626/0 (socket says 172.21.15.86:47626) 2024-03-06T07:29:26.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.616+0000 7f3410c00640 1 -- 172.21.15.86:0/3660784978 learned_addr learned my addr 172.21.15.86:0/3660784978 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:26.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.617+0000 7f3412000640 1 --2- 172.21.15.86:0/3660784978 >> v2:172.21.15.194:3300/0 conn(0x7f340c15f120 0x7f340c0c4a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.617+0000 7f3411600640 1 -- 172.21.15.86:0/3660784978 >> v2:172.21.15.194:3300/0 conn(0x7f340c15f120 msgr2=0x7f340c0c4a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:26.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.617+0000 7f3411600640 1 --2- 172.21.15.86:0/3660784978 >> v2:172.21.15.194:3300/0 conn(0x7f340c15f120 0x7f340c0c4a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.617+0000 7f3411600640 1 -- 172.21.15.86:0/3660784978 >> v1:172.21.15.86:6789/0 conn(0x7f340c15b070 legacy=0x7f340c15ea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.617+0000 7f3411600640 1 -- 172.21.15.86:0/3660784978 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f340c0ce920 con 0x7f340c15ac90 2024-03-06T07:29:26.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.617+0000 7f340be00640 1 -- 172.21.15.86:0/3660784978 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2911385298 0 0) 0x7f340c0cc000 con 0x7f340c15b070 2024-03-06T07:29:26.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.618+0000 7f3411600640 1 --2- 172.21.15.86:0/3660784978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15ac90 0x7f340c06e580 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f340c0cf2e0 tx=0x7f340002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d511736f8520d7c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:26.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.618+0000 7f340be00640 1 -- 172.21.15.86:0/3660784978 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34000315a0 con 0x7f340c15ac90 2024-03-06T07:29:26.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.618+0000 7f340be00640 1 -- 172.21.15.86:0/3660784978 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3400037070 con 0x7f340c15ac90 2024-03-06T07:29:26.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.618+0000 7f340be00640 1 -- 172.21.15.86:0/3660784978 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3400031960 con 0x7f340c15ac90 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 -- 172.21.15.86:0/3660784978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15ac90 msgr2=0x7f340c06e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 --2- 172.21.15.86:0/3660784978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15ac90 0x7f340c06e580 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f340c0cf2e0 tx=0x7f340002b840 comp rx=0 tx=0).stop 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 -- 172.21.15.86:0/3660784978 shutdown_connections 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 --2- 172.21.15.86:0/3660784978 >> v2:172.21.15.194:3300/0 conn(0x7f340c15f120 0x7f340c0c4a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 --2- 172.21.15.86:0/3660784978 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15ac90 0x7f340c06e580 unknown :-1 s=CLOSED pgs=557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 -- 172.21.15.86:0/3660784978 >> 172.21.15.86:0/3660784978 conn(0x7f340c0cc4f0 msgr2=0x7f340c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 -- 172.21.15.86:0/3660784978 shutdown_connections 2024-03-06T07:29:26.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.619+0000 7f3414000640 1 -- 172.21.15.86:0/3660784978 wait complete. 2024-03-06T07:29:26.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.621+0000 7f3414000640 1 Processor -- start 2024-03-06T07:29:26.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.621+0000 7f3414000640 1 -- start start 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3414000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f340c15ac90 0x7f340c0c71e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3414000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15b070 0x7f340c0c4930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3414000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f340c15f120 0x7f340c0c4e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3414000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f340c15f5b0 con 0x7f340c15b070 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3414000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f340c0ceee0 con 0x7f340c15ac90 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3414000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f340c15f730 con 0x7f340c15f120 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3410c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15b070 0x7f340c0c4930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3410c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15b070 0x7f340c0c4930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50744/0 (socket says 172.21.15.86:50744) 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.622+0000 7f3410c00640 1 -- 172.21.15.86:0/975704461 learned_addr learned my addr 172.21.15.86:0/975704461 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3411600640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f340c15ac90 0x7f340c0c71e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3412000640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f340c15f120 0x7f340c0c4e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3410c00640 1 -- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f340c15f120 msgr2=0x7f340c0c4e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3410c00640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f340c15f120 0x7f340c0c4e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3410c00640 1 -- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f340c15ac90 msgr2=0x7f340c0c71e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3410c00640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f340c15ac90 0x7f340c0c71e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3410c00640 1 -- 172.21.15.86:0/975704461 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f340c15de90 con 0x7f340c15b070 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3411600640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f340c15ac90 0x7f340c0c71e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3410c00640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15b070 0x7f340c0c4930 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f33fc00c820 tx=0x7f33fc00cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3409600640 1 -- 172.21.15.86:0/975704461 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f33fc018070 con 0x7f340c15b070 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3409600640 1 -- 172.21.15.86:0/975704461 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f33fc00a640 con 0x7f340c15b070 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3409600640 1 -- 172.21.15.86:0/975704461 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f33fc00aa30 con 0x7f340c15b070 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f340c15e170 con 0x7f340c15b070 2024-03-06T07:29:26.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.623+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f340c0c9b90 con 0x7f340c15b070 2024-03-06T07:29:26.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.625+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f340c15e340 con 0x7f340c15b070 2024-03-06T07:29:26.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.626+0000 7f3409600640 1 -- 172.21.15.86:0/975704461 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f33fc00abd0 con 0x7f340c15b070 2024-03-06T07:29:26.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.627+0000 7f3409600640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f33f006b4d0 0x7f33f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:26.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.627+0000 7f3409600640 1 -- 172.21.15.86:0/975704461 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f33fc0899e0 con 0x7f340c15b070 2024-03-06T07:29:26.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.627+0000 7f3411600640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f33f006b4d0 0x7f33f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:26.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.628+0000 7f3411600640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f33f006b4d0 0x7f33f006d990 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f3400036000 tx=0x7f3400008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:26.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.631+0000 7f3409600640 1 -- 172.21.15.86:0/975704461 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f340c15e340 con 0x7f340c15b070 2024-03-06T07:29:26.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.830+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f340c0c6ab0 con 0x7f340c15b070 2024-03-06T07:29:26.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.831+0000 7f3409600640 1 -- 172.21.15.86:0/975704461 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v47) v1 ==== 80+0+0 (secure 0 0 0) 0x7f33fc0587c0 con 0x7f340c15b070 2024-03-06T07:29:26.833 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f33f006b4d0 msgr2=0x7f33f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f33f006b4d0 0x7f33f006d990 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f3400036000 tx=0x7f3400008030 comp rx=0 tx=0).stop 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15b070 msgr2=0x7f340c0c4930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15b070 0x7f340c0c4930 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f33fc00c820 tx=0x7f33fc00cce0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 shutdown_connections 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f33f006b4d0 0x7f33f006d990 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f340c15f120 0x7f340c0c4e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f340c15b070 0x7f340c0c4930 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 --2- 172.21.15.86:0/975704461 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f340c15ac90 0x7f340c0c71e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 >> 172.21.15.86:0/975704461 conn(0x7f340c0cc4f0 msgr2=0x7f340c0c5d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 shutdown_connections 2024-03-06T07:29:26.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:26.833+0000 7f3414000640 1 -- 172.21.15.86:0/975704461 wait complete. 2024-03-06T07:29:26.852 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:29:26.853 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:26.853 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:27.032 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:29:27.033 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:27.033 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:29:27.073 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:29:27.073 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:27.110 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:29:27.110 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:27.110 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:27.152 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:29:27.152 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:29:27.152 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:27.192 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:29:27.192 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:27.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.386+0000 7f4cd1c00640 1 Processor -- start 2024-03-06T07:29:27.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.386+0000 7f4cd1c00640 1 -- start start 2024-03-06T07:29:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.387+0000 7f4cd1c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4ccc15f9d0 0x7f4ccc15fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.387+0000 7f4cd1c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ccc06cef0 con 0x7f4ccc070b10 2024-03-06T07:29:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.387+0000 7f4cd1c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ccc06d1d0 con 0x7f4ccc144c00 2024-03-06T07:29:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.387+0000 7f4cd1c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4ccc06d4b0 con 0x7f4ccc15f9d0 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.388+0000 7f4ccb400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4ccc070b10 0x7f4ccc070f20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47632/0 (socket says 172.21.15.86:47632) 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.388+0000 7f4ccb400640 1 -- 172.21.15.86:0/911898927 learned_addr learned my addr 172.21.15.86:0/911898927 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.388+0000 7f4cca000640 1 --2- 172.21.15.86:0/911898927 >> v2:172.21.15.194:3300/0 conn(0x7f4ccc15f9d0 0x7f4ccc15fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.388+0000 7f4cc9600640 1 -- 172.21.15.86:0/911898927 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1259012276 0 0) 0x7f4ccc06cef0 con 0x7f4ccc070b10 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.388+0000 7f4cc9600640 1 -- 172.21.15.86:0/911898927 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4cb0003520 con 0x7f4ccc070b10 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.388+0000 7f4cc9600640 1 -- 172.21.15.86:0/911898927 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1163837543 0 0) 0x7f4cb0003520 con 0x7f4ccc070b10 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.388+0000 7f4cca000640 1 -- 172.21.15.86:0/911898927 >> v1:172.21.15.194:6789/0 conn(0x7f4ccc144c00 legacy=0x7f4ccc06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.389+0000 7f4cca000640 1 -- 172.21.15.86:0/911898927 >> v1:172.21.15.86:6789/0 conn(0x7f4ccc070b10 legacy=0x7f4ccc070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.389+0000 7f4cca000640 1 -- 172.21.15.86:0/911898927 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ccc06d790 con 0x7f4ccc15f9d0 2024-03-06T07:29:27.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.390+0000 7f4cca000640 1 --2- 172.21.15.86:0/911898927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc15f9d0 0x7f4ccc15fe60 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f4cb80097a0 tx=0x7f4cb802b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6f8ce8fc59f64928 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.390+0000 7f4cc9600640 1 -- 172.21.15.86:0/911898927 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cb8038070 con 0x7f4ccc15f9d0 2024-03-06T07:29:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.390+0000 7f4cc9600640 1 -- 172.21.15.86:0/911898927 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cb8031500 con 0x7f4ccc15f9d0 2024-03-06T07:29:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.390+0000 7f4cc9600640 1 -- 172.21.15.86:0/911898927 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cc40019f0 con 0x7f4ccc15f9d0 2024-03-06T07:29:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.391+0000 7f4cd1c00640 1 -- 172.21.15.86:0/911898927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc15f9d0 msgr2=0x7f4ccc15fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.391+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/911898927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc15f9d0 0x7f4ccc15fe60 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f4cb80097a0 tx=0x7f4cb802b820 comp rx=0 tx=0).stop 2024-03-06T07:29:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.391+0000 7f4cd1c00640 1 -- 172.21.15.86:0/911898927 shutdown_connections 2024-03-06T07:29:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.391+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/911898927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc15f9d0 0x7f4ccc15fe60 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.391+0000 7f4cd1c00640 1 -- 172.21.15.86:0/911898927 >> 172.21.15.86:0/911898927 conn(0x7f4ccc0b1730 msgr2=0x7f4ccc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.391+0000 7f4cd1c00640 1 -- 172.21.15.86:0/911898927 shutdown_connections 2024-03-06T07:29:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.391+0000 7f4cd1c00640 1 -- 172.21.15.86:0/911898927 wait complete. 2024-03-06T07:29:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.393+0000 7f4cd1c00640 1 Processor -- start 2024-03-06T07:29:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.393+0000 7f4cd1c00640 1 -- start start 2024-03-06T07:29:27.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4cd1c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ccc070b10 0x7f4ccc141810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4cd1c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc144c00 0x7f4ccc141d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4cd1c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ccc15f9d0 0x7f4ccc15e140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4cd1c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ccc06ccd0 con 0x7f4ccc070b10 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4cd1c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ccc06cb50 con 0x7f4ccc144c00 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4cd1c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ccc06df90 con 0x7f4ccc15f9d0 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4ccaa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ccc070b10 0x7f4ccc141810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4ccaa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ccc070b10 0x7f4ccc141810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50752/0 (socket says 172.21.15.86:50752) 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4ccaa00640 1 -- 172.21.15.86:0/2075898526 learned_addr learned my addr 172.21.15.86:0/2075898526 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4cca000640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc144c00 0x7f4ccc141d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4ccaa00640 1 -- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ccc15f9d0 msgr2=0x7f4ccc15e140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4ccaa00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ccc15f9d0 0x7f4ccc15e140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4ccaa00640 1 -- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc144c00 msgr2=0x7f4ccc141d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.394+0000 7f4ccaa00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc144c00 0x7f4ccc141d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4ccaa00640 1 -- 172.21.15.86:0/2075898526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ccc1425e0 con 0x7f4ccc070b10 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4cca000640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc144c00 0x7f4ccc141d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4ccaa00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ccc070b10 0x7f4ccc141810 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f4cbc00c400 tx=0x7f4cbc00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4cc2a00640 1 -- 172.21.15.86:0/2075898526 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cbc0052d0 con 0x7f4ccc070b10 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4cc2a00640 1 -- 172.21.15.86:0/2075898526 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cbc005490 con 0x7f4ccc070b10 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4cc2a00640 1 -- 172.21.15.86:0/2075898526 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cbc007260 con 0x7f4ccc070b10 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ccc1428c0 con 0x7f4ccc070b10 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.395+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ccc13d6c0 con 0x7f4ccc070b10 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.396+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ccc142a90 con 0x7f4ccc070b10 2024-03-06T07:29:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.397+0000 7f4cc2a00640 1 -- 172.21.15.86:0/2075898526 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4cbc007400 con 0x7f4ccc070b10 2024-03-06T07:29:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.398+0000 7f4cc2a00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cac06b500 0x7f4cac06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.398+0000 7f4cc2a00640 1 -- 172.21.15.86:0/2075898526 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4cbc0896b0 con 0x7f4ccc070b10 2024-03-06T07:29:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.398+0000 7f4cca000640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cac06b500 0x7f4cac06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.399+0000 7f4cca000640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cac06b500 0x7f4cac06d9c0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f4cb8035000 tx=0x7f4cb8008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:27.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.402+0000 7f4cc2a00640 1 -- 172.21.15.86:0/2075898526 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4cbc058520 con 0x7f4ccc070b10 2024-03-06T07:29:27.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.636+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4ccc04f330 con 0x7f4ccc070b10 2024-03-06T07:29:27.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.637+0000 7f4cc2a00640 1 -- 172.21.15.86:0/2075898526 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) v1 ==== 94+0+4983 (secure 0 0 0) 0x7f4cbc057e20 con 0x7f4ccc070b10 2024-03-06T07:29:27.640 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:27.640 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":25,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5041,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/3678578793","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":3678578793},{"type":"v1","addr":"172.21.15.194:6835","nonce":3678578793}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/4206282845","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":4206282845},{"type":"v1","addr":"172.21.15.86:6835","nonce":4206282845}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5427,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3500830166","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3500830166},{"type":"v1","addr":"172.21.15.86:6837","nonce":3500830166}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":25,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:59.832195+0000","modified":"2024-03-06T07:29:02.883187+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5439},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5439":{"gid":5439,"name":"b","rank":0,"incarnation":21,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6833/4064823241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":4064823241},{"type":"v1","addr":"172.21.15.194:6833","nonce":4064823241}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5439,"qdb_cluster":[5439]},"id":3}]} 2024-03-06T07:29:27.640 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 25 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cac06b500 msgr2=0x7f4cac06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cac06b500 0x7f4cac06d9c0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f4cb8035000 tx=0x7f4cb8008030 comp rx=0 tx=0).stop 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ccc070b10 msgr2=0x7f4ccc141810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ccc070b10 0x7f4ccc141810 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f4cbc00c400 tx=0x7f4cbc00c8c0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 shutdown_connections 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cac06b500 0x7f4cac06d9c0 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4ccc15f9d0 0x7f4ccc15e140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4ccc144c00 0x7f4ccc141d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 --2- 172.21.15.86:0/2075898526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4ccc070b10 0x7f4ccc141810 unknown :-1 s=CLOSED pgs=560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 >> 172.21.15.86:0/2075898526 conn(0x7f4ccc0b1730 msgr2=0x7f4ccc0ae380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 shutdown_connections 2024-03-06T07:29:27.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.640+0000 7f4cd1c00640 1 -- 172.21.15.86:0/2075898526 wait complete. 2024-03-06T07:29:27.661 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:27.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.858+0000 7f100fa00640 1 Processor -- start 2024-03-06T07:29:27.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.858+0000 7f100fa00640 1 -- start start 2024-03-06T07:29:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f100fa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f100815acb0 0x7f10080b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f100fa00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f100815b090 con 0x7f10080b4680 2024-03-06T07:29:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f100fa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f10080b2a30 con 0x7f100806ec00 2024-03-06T07:29:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f100fa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f10080b4360 con 0x7f100815acb0 2024-03-06T07:29:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f1007e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f10080b4680 0x7f100806e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47648/0 (socket says 172.21.15.86:47648) 2024-03-06T07:29:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f1007e00640 1 -- 172.21.15.86:0/2041909096 learned_addr learned my addr 172.21.15.86:0/2041909096 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:27.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f100d000640 1 --2- 172.21.15.86:0/2041909096 >> v2:172.21.15.194:3300/0 conn(0x7f100815acb0 0x7f10080b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1651393038 0 0) 0x7f100815b090 con 0x7f10080b4680 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.860+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0fec005730 con 0x7f10080b4680 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2748012637 0 0) 0x7f0fec005730 con 0x7f10080b4680 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 >> v2:172.21.15.194:3300/0 conn(0x7f100815acb0 msgr2=0x7f10080b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 --2- 172.21.15.86:0/2041909096 >> v2:172.21.15.194:3300/0 conn(0x7f100815acb0 0x7f10080b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 >> v1:172.21.15.194:6789/0 conn(0x7f100806ec00 legacy=0x7f10080ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10080b2cf0 con 0x7f10080b4680 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f0ff4001670 con 0x7f10080b4680 2024-03-06T07:29:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 >> v1:172.21.15.86:6789/0 conn(0x7f10080b4680 legacy=0x7f100806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec005c10 0x7f0fec006030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fec006a10 0x7f0fec006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fec0073d0 0x7f0fec00dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10080b4360 con 0x7f0fec0073d0 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fec00e2d0 con 0x7f0fec005c10 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0fec00e4d0 con 0x7f0fec006a10 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f100da00640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fec0073d0 0x7f0fec00dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.861+0000 7f100d000640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec005c10 0x7f0fec006030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.862+0000 7f1007e00640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fec006a10 0x7f0fec006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.862+0000 7f100da00640 1 -- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fec006a10 msgr2=0x7f0fec006e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.862+0000 7f100da00640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fec006a10 0x7f0fec006e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.862+0000 7f100da00640 1 -- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec005c10 msgr2=0x7f0fec006030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.862+0000 7f100da00640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec005c10 0x7f0fec006030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.862+0000 7f100da00640 1 -- 172.21.15.86:0/2041909096 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0fec005730 con 0x7f0fec0073d0 2024-03-06T07:29:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.863+0000 7f100da00640 1 -- 172.21.15.86:0/2041909096 reap_dead start 2024-03-06T07:29:27.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.863+0000 7f100da00640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fec0073d0 0x7f0fec00dbd0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f1000001e50 tx=0x7f0fec007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c26594e82c9bd30 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.863+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1000001a70 con 0x7f0fec0073d0 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.863+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1000003bb0 con 0x7f0fec0073d0 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.863+0000 7f1007400640 1 -- 172.21.15.86:0/2041909096 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10000050d0 con 0x7f0fec0073d0 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.863+0000 7f100fa00640 1 -- 172.21.15.86:0/2041909096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fec0073d0 msgr2=0x7f0fec00dbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.863+0000 7f100fa00640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fec0073d0 0x7f0fec00dbd0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f1000001e50 tx=0x7f0fec007a40 comp rx=0 tx=0).stop 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.864+0000 7f100fa00640 1 -- 172.21.15.86:0/2041909096 shutdown_connections 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.864+0000 7f100fa00640 1 --2- 172.21.15.86:0/2041909096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fec0073d0 0x7f0fec00dbd0 unknown :-1 s=CLOSED pgs=562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.864+0000 7f100fa00640 1 -- 172.21.15.86:0/2041909096 >> 172.21.15.86:0/2041909096 conn(0x7f10080b1730 msgr2=0x7f10080b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.864+0000 7f100fa00640 1 -- 172.21.15.86:0/2041909096 shutdown_connections 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.864+0000 7f100fa00640 1 -- 172.21.15.86:0/2041909096 wait complete. 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.865+0000 7f100fa00640 1 Processor -- start 2024-03-06T07:29:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.865+0000 7f100fa00640 1 -- start start 2024-03-06T07:29:27.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.866+0000 7f100fa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec0073d0 0x7f10081494e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.866+0000 7f100fa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10080c77b0 0x7f10080c7c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.866+0000 7f100fa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10080c5210 0x7f10080c5690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.866+0000 7f100fa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10080b4010 con 0x7f10080c5210 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.866+0000 7f100fa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10080b32b0 con 0x7f0fec0073d0 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.866+0000 7f100fa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1008072340 con 0x7f10080c77b0 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10080c5210 0x7f10080c5690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10080c5210 0x7f10080c5690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50760/0 (socket says 172.21.15.86:50760) 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 -- 172.21.15.86:0/138277534 learned_addr learned my addr 172.21.15.86:0/138277534 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100d000640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec0073d0 0x7f10081494e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f1007e00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10080c77b0 0x7f10080c7c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 -- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10080c77b0 msgr2=0x7f10080c7c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10080c77b0 0x7f10080c7c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 -- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec0073d0 msgr2=0x7f10081494e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:27.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec0073d0 0x7f10081494e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 -- 172.21.15.86:0/138277534 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10080c5f80 con 0x7f10080c5210 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.867+0000 7f100da00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10080c5210 0x7f10080c5690 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f100002c060 tx=0x7f10000014a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.868+0000 7f1004c00640 1 -- 172.21.15.86:0/138277534 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1000006d60 con 0x7f10080c5210 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.868+0000 7f1004c00640 1 -- 172.21.15.86:0/138277534 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1000039480 con 0x7f10080c5210 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.868+0000 7f1004c00640 1 -- 172.21.15.86:0/138277534 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1000039830 con 0x7f10080c5210 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.868+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10080c6260 con 0x7f10080c5210 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.868+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10080c6820 con 0x7f10080c5210 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.869+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f100804f330 con 0x7f10080c5210 2024-03-06T07:29:27.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.870+0000 7f1004c00640 1 -- 172.21.15.86:0/138277534 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f10000399d0 con 0x7f10080c5210 2024-03-06T07:29:27.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.871+0000 7f1004c00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fe806b500 0x7f0fe806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:27.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.871+0000 7f1004c00640 1 -- 172.21.15.86:0/138277534 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f10000abe80 con 0x7f10080c5210 2024-03-06T07:29:27.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.871+0000 7f100d000640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fe806b500 0x7f0fe806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:27.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.872+0000 7f100d000640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fe806b500 0x7f0fe806d9c0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f0ff8009990 tx=0x7f0ff8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:27.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:27.876+0000 7f1004c00640 1 -- 172.21.15.86:0/138277534 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f100003e070 con 0x7f10080c5210 2024-03-06T07:29:28.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.060+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f100806dfe0 con 0x7f10080c5210 2024-03-06T07:29:28.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.062+0000 7f1004c00640 1 -- 172.21.15.86:0/138277534 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v47) v1 ==== 74+0+16625 (secure 0 0 0) 0x7f1000033030 con 0x7f10080c5210 2024-03-06T07:29:28.065 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:28.065 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":47,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:01.377099+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:57.609038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:58.840885+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fe806b500 msgr2=0x7f0fe806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fe806b500 0x7f0fe806d9c0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f0ff8009990 tx=0x7f0ff8008040 comp rx=0 tx=0).stop 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10080c5210 msgr2=0x7f10080c5690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10080c5210 0x7f10080c5690 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f100002c060 tx=0x7f10000014a0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 shutdown_connections 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fe806b500 0x7f0fe806d9c0 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f10080c5210 0x7f10080c5690 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10080c77b0 0x7f10080c7c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 --2- 172.21.15.86:0/138277534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fec0073d0 0x7f10081494e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 >> 172.21.15.86:0/138277534 conn(0x7f10080b1730 msgr2=0x7f10080ccf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 shutdown_connections 2024-03-06T07:29:28.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.065+0000 7f100fa00640 1 -- 172.21.15.86:0/138277534 wait complete. 2024-03-06T07:29:28.086 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 6, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:28:57.609038+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '46', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 7, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:28:58.840885+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '47', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:29:28.086 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:29:28.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.289+0000 7f1433e00640 1 Processor -- start 2024-03-06T07:29:28.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.289+0000 7f1433e00640 1 -- start start 2024-03-06T07:29:28.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f1433e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f143406da40 0x7f14340ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f1433e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f14340aef20 0x7f14340ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f1433e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f143415c1f0 con 0x7f143415be10 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f1433e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14340cd200 con 0x7f143406da40 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f1433e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f143406cac0 con 0x7f14340aef20 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f142a000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f143406da40 0x7f14340ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f142a000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f143406da40 0x7f14340ccc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50776/0 (socket says 172.21.15.86:50776) 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f142a000640 1 -- 172.21.15.86:0/1729739576 learned_addr learned my addr 172.21.15.86:0/1729739576 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f1432a00640 1 --1- 172.21.15.86:0/1729739576 >> v1:172.21.15.86:6789/0 conn(0x7f143415be10 0x7f143406d330 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47662/0 (socket says 172.21.15.86:47662) 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.291+0000 7f1433400640 1 --2- 172.21.15.86:0/1729739576 >> v2:172.21.15.194:3300/0 conn(0x7f14340aef20 0x7f14340ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.292+0000 7f142a000640 1 -- 172.21.15.86:0/1729739576 >> v2:172.21.15.194:3300/0 conn(0x7f14340aef20 msgr2=0x7f14340ce2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.292+0000 7f142a000640 1 --2- 172.21.15.86:0/1729739576 >> v2:172.21.15.194:3300/0 conn(0x7f14340aef20 0x7f14340ce2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.292+0000 7f142a000640 1 -- 172.21.15.86:0/1729739576 >> v1:172.21.15.86:6789/0 conn(0x7f143415be10 legacy=0x7f143406d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.292+0000 7f142a000640 1 -- 172.21.15.86:0/1729739576 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f143406cda0 con 0x7f143406da40 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.292+0000 7f1432000640 1 -- 172.21.15.86:0/1729739576 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1563046678 0 0) 0x7f143415c1f0 con 0x7f143415be10 2024-03-06T07:29:28.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.293+0000 7f142a000640 1 --2- 172.21.15.86:0/1729739576 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f14340ccc20 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f14340af490 tx=0x7f141c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=63b04e6e091aa44c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:28.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.293+0000 7f1432000640 1 -- 172.21.15.86:0/1729739576 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f141c035020 con 0x7f143406da40 2024-03-06T07:29:28.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.293+0000 7f1432000640 1 -- 172.21.15.86:0/1729739576 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f141c031870 con 0x7f143406da40 2024-03-06T07:29:28.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.293+0000 7f1432000640 1 -- 172.21.15.86:0/1729739576 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f141c031c40 con 0x7f143406da40 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.293+0000 7f1433e00640 1 -- 172.21.15.86:0/1729739576 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 msgr2=0x7f14340ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.293+0000 7f1433e00640 1 --2- 172.21.15.86:0/1729739576 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f14340ccc20 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f14340af490 tx=0x7f141c02b910 comp rx=0 tx=0).stop 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.294+0000 7f1433e00640 1 -- 172.21.15.86:0/1729739576 shutdown_connections 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.294+0000 7f1433e00640 1 --2- 172.21.15.86:0/1729739576 >> v2:172.21.15.194:3300/0 conn(0x7f14340aef20 0x7f14340ce2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.294+0000 7f1433e00640 1 --2- 172.21.15.86:0/1729739576 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f14340ccc20 unknown :-1 s=CLOSED pgs=565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.294+0000 7f1433e00640 1 -- 172.21.15.86:0/1729739576 >> 172.21.15.86:0/1729739576 conn(0x7f143406afc0 msgr2=0x7f143406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.294+0000 7f1433e00640 1 -- 172.21.15.86:0/1729739576 shutdown_connections 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.294+0000 7f1433e00640 1 -- 172.21.15.86:0/1729739576 wait complete. 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.295+0000 7f1433e00640 1 Processor -- start 2024-03-06T07:29:28.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.295+0000 7f1433e00640 1 -- start start 2024-03-06T07:29:28.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.296+0000 7f1433e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f143415ee30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.296+0000 7f1433e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14340aef20 0x7f1434152cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.296+0000 7f1433e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f143415be10 0x7f143414af90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.296+0000 7f1433e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14340cda90 con 0x7f143406da40 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.296+0000 7f1433e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f143406d080 con 0x7f143415be10 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.296+0000 7f1433e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1434071ab0 con 0x7f14340aef20 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f143415ee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f143415ee30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50792/0 (socket says 172.21.15.86:50792) 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 -- 172.21.15.86:0/270087996 learned_addr learned my addr 172.21.15.86:0/270087996 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1433400640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f143415be10 0x7f143414af90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f142a000640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14340aef20 0x7f1434152cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 -- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14340aef20 msgr2=0x7f1434152cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14340aef20 0x7f1434152cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 -- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f143415be10 msgr2=0x7f143414af90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f143415be10 0x7f143414af90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 -- 172.21.15.86:0/270087996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1434151c10 con 0x7f143406da40 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1433400640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f143415be10 0x7f143414af90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:28.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.297+0000 7f1432a00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f143415ee30 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7f142400c460 tx=0x7f142400c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:28.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.298+0000 7f142b400640 1 -- 172.21.15.86:0/270087996 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1424018070 con 0x7f143406da40 2024-03-06T07:29:28.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.298+0000 7f142b400640 1 -- 172.21.15.86:0/270087996 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1424007500 con 0x7f143406da40 2024-03-06T07:29:28.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.298+0000 7f142b400640 1 -- 172.21.15.86:0/270087996 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1424004a20 con 0x7f143406da40 2024-03-06T07:29:28.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.298+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1434151ef0 con 0x7f143406da40 2024-03-06T07:29:28.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.298+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14341524b0 con 0x7f143406da40 2024-03-06T07:29:28.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.299+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f143404f330 con 0x7f143406da40 2024-03-06T07:29:28.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.300+0000 7f142b400640 1 -- 172.21.15.86:0/270087996 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1424004bc0 con 0x7f143406da40 2024-03-06T07:29:28.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.301+0000 7f142b400640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f141006b4d0 0x7f141006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.301+0000 7f142b400640 1 -- 172.21.15.86:0/270087996 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1424089640 con 0x7f143406da40 2024-03-06T07:29:28.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.301+0000 7f142a000640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f141006b4d0 0x7f141006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.302+0000 7f142a000640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f141006b4d0 0x7f141006d990 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f141c036000 tx=0x7f141c008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:28.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.306+0000 7f142b400640 1 -- 172.21.15.86:0/270087996 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1424058390 con 0x7f143406da40 2024-03-06T07:29:28.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.501+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f143415b410 con 0x7f143406da40 2024-03-06T07:29:28.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.502+0000 7f142b400640 1 -- 172.21.15.86:0/270087996 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v47) v1 ==== 93+0+10 (secure 0 0 0) 0x7f1424057c90 con 0x7f143406da40 2024-03-06T07:29:28.504 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f141006b4d0 msgr2=0x7f141006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f141006b4d0 0x7f141006d990 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f141c036000 tx=0x7f141c008030 comp rx=0 tx=0).stop 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 msgr2=0x7f143415ee30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f143415ee30 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7f142400c460 tx=0x7f142400c920 comp rx=0 tx=0).stop 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 shutdown_connections 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f141006b4d0 0x7f141006d990 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f143415be10 0x7f143414af90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14340aef20 0x7f1434152cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 --2- 172.21.15.86:0/270087996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f143406da40 0x7f143415ee30 unknown :-1 s=CLOSED pgs=566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 >> 172.21.15.86:0/270087996 conn(0x7f143406afc0 msgr2=0x7f143406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 shutdown_connections 2024-03-06T07:29:28.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.504+0000 7f1433e00640 1 -- 172.21.15.86:0/270087996 wait complete. 2024-03-06T07:29:28.526 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:29:28.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.735+0000 7fc01c200640 1 Processor -- start 2024-03-06T07:29:28.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.735+0000 7fc01c200640 1 -- start start 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.737+0000 7fc01c200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc014163800 0x7fc0140b2c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.737+0000 7fc01c200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc01406e060 0x7fc01406e4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.737+0000 7fc01c200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc01415cf70 con 0x7fc01415cb90 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.737+0000 7fc01c200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc0140b3890 con 0x7fc014163800 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.737+0000 7fc01c200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc01406ea10 con 0x7fc01406e060 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.737+0000 7fc018e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc014163800 0x7fc0140b2c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.738+0000 7fc018e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc014163800 0x7fc0140b2c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50798/0 (socket says 172.21.15.86:50798) 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.738+0000 7fc018e00640 1 -- 172.21.15.86:0/1116042280 learned_addr learned my addr 172.21.15.86:0/1116042280 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.738+0000 7fc01a200640 1 --2- 172.21.15.86:0/1116042280 >> v2:172.21.15.194:3300/0 conn(0x7fc01406e060 0x7fc01406e4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.738+0000 7fc018e00640 1 -- 172.21.15.86:0/1116042280 >> v2:172.21.15.194:3300/0 conn(0x7fc01406e060 msgr2=0x7fc01406e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:28.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.738+0000 7fc018e00640 1 --2- 172.21.15.86:0/1116042280 >> v2:172.21.15.194:3300/0 conn(0x7fc01406e060 0x7fc01406e4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.738+0000 7fc018e00640 1 -- 172.21.15.86:0/1116042280 >> v1:172.21.15.194:6789/0 conn(0x7fc01415cb90 legacy=0x7fc01406d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.738+0000 7fc018e00640 1 -- 172.21.15.86:0/1116042280 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0140b35b0 con 0x7fc014163800 2024-03-06T07:29:28.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.739+0000 7fc018e00640 1 --2- 172.21.15.86:0/1116042280 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc014163800 0x7fc0140b2c40 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7fc01406c670 tx=0x7fc00002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e8df9e553ead0493 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:28.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.740+0000 7fc013e00640 1 -- 172.21.15.86:0/1116042280 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc000034070 con 0x7fc014163800 2024-03-06T07:29:28.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.740+0000 7fc013e00640 1 -- 172.21.15.86:0/1116042280 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc000030860 con 0x7fc014163800 2024-03-06T07:29:28.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.740+0000 7fc013e00640 1 -- 172.21.15.86:0/1116042280 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc000030c30 con 0x7fc014163800 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.740+0000 7fc01c200640 1 -- 172.21.15.86:0/1116042280 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc014163800 msgr2=0x7fc0140b2c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.740+0000 7fc01c200640 1 --2- 172.21.15.86:0/1116042280 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc014163800 0x7fc0140b2c40 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7fc01406c670 tx=0x7fc00002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.741+0000 7fc01c200640 1 -- 172.21.15.86:0/1116042280 shutdown_connections 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.741+0000 7fc01c200640 1 --2- 172.21.15.86:0/1116042280 >> v2:172.21.15.194:3300/0 conn(0x7fc01406e060 0x7fc01406e4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.741+0000 7fc01c200640 1 --2- 172.21.15.86:0/1116042280 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc014163800 0x7fc0140b2c40 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.741+0000 7fc01c200640 1 -- 172.21.15.86:0/1116042280 >> 172.21.15.86:0/1116042280 conn(0x7fc0140b1940 msgr2=0x7fc0140b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.741+0000 7fc01c200640 1 -- 172.21.15.86:0/1116042280 shutdown_connections 2024-03-06T07:29:28.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.741+0000 7fc01c200640 1 -- 172.21.15.86:0/1116042280 wait complete. 2024-03-06T07:29:28.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.742+0000 7fc01c200640 1 Processor -- start 2024-03-06T07:29:28.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.742+0000 7fc01c200640 1 -- start start 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.743+0000 7fc01c200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc01406e060 0x7fc014155c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.743+0000 7fc01c200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc01415cb90 0x7fc01409c820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.743+0000 7fc01c200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc014163800 0x7fc014158c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.743+0000 7fc01c200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc01406d230 con 0x7fc01406e060 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.743+0000 7fc01c200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc01406c990 con 0x7fc014163800 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.743+0000 7fc01c200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc01406cec0 con 0x7fc01415cb90 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc01406e060 0x7fc014155c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc01406e060 0x7fc014155c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50814/0 (socket says 172.21.15.86:50814) 2024-03-06T07:29:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 -- 172.21.15.86:0/2161713447 learned_addr learned my addr 172.21.15.86:0/2161713447 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 -- 172.21.15.86:0/2161713447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc01415cb90 msgr2=0x7fc01409c820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc01415cb90 0x7fc01409c820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 -- 172.21.15.86:0/2161713447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc014163800 msgr2=0x7fc014158c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc01a200640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc014163800 0x7fc014158c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc014163800 0x7fc014158c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 -- 172.21.15.86:0/2161713447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc01409d200 con 0x7fc01406e060 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.744+0000 7fc019800640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc01406e060 0x7fc014155c30 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7fc00800c430 tx=0x7fc00800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.745+0000 7fc011600640 1 -- 172.21.15.86:0/2161713447 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0080052d0 con 0x7fc01406e060 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.745+0000 7fc011600640 1 -- 172.21.15.86:0/2161713447 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc008005490 con 0x7fc01406e060 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.745+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc01409d4e0 con 0x7fc01406e060 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.745+0000 7fc011600640 1 -- 172.21.15.86:0/2161713447 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc008007260 con 0x7fc01406e060 2024-03-06T07:29:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.745+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc014156910 con 0x7fc01406e060 2024-03-06T07:29:28.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.746+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc01404f330 con 0x7fc01406e060 2024-03-06T07:29:28.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.747+0000 7fc011600640 1 -- 172.21.15.86:0/2161713447 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc008007400 con 0x7fc01406e060 2024-03-06T07:29:28.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.748+0000 7fc011600640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbff406b4d0 0x7fbff406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:28.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.748+0000 7fc011600640 1 -- 172.21.15.86:0/2161713447 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc008089700 con 0x7fc01406e060 2024-03-06T07:29:28.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.748+0000 7fc018e00640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbff406b4d0 0x7fbff406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:28.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.749+0000 7fc018e00640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbff406b4d0 0x7fbff406d990 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fc000036000 tx=0x7fc000030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:28.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.752+0000 7fc011600640 1 -- 172.21.15.86:0/2161713447 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc008058570 con 0x7fc01406e060 2024-03-06T07:29:28.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.951+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fc0140afcb0 con 0x7fc01406e060 2024-03-06T07:29:28.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.952+0000 7fc011600640 1 -- 172.21.15.86:0/2161713447 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v47) v1 ==== 104+0+11 (secure 0 0 0) 0x7fc008057e70 con 0x7fc01406e060 2024-03-06T07:29:28.954 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:29:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbff406b4d0 msgr2=0x7fbff406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbff406b4d0 0x7fbff406d990 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fc000036000 tx=0x7fc000030000 comp rx=0 tx=0).stop 2024-03-06T07:29:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc01406e060 msgr2=0x7fc014155c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc01406e060 0x7fc014155c30 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7fc00800c430 tx=0x7fc00800c8f0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 shutdown_connections 2024-03-06T07:29:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbff406b4d0 0x7fbff406d990 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc014163800 0x7fc014158c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc01415cb90 0x7fc01409c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 --2- 172.21.15.86:0/2161713447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc01406e060 0x7fc014155c30 unknown :-1 s=CLOSED pgs=568 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.954+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 >> 172.21.15.86:0/2161713447 conn(0x7fc0140b1940 msgr2=0x7fc014164fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.955+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 shutdown_connections 2024-03-06T07:29:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:28.955+0000 7fc01c200640 1 -- 172.21.15.86:0/2161713447 wait complete. 2024-03-06T07:29:28.975 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:29:29.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.182+0000 7fd502e00640 1 Processor -- start 2024-03-06T07:29:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.182+0000 7fd502e00640 1 -- start start 2024-03-06T07:29:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.183+0000 7fd502e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd4fc06da40 0x7fd4fc0ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.183+0000 7fd502e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd4fc15c1f0 con 0x7fd4fc15be10 2024-03-06T07:29:29.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.183+0000 7fd502e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd4fc06cac0 con 0x7fd4fc0aef20 2024-03-06T07:29:29.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.183+0000 7fd502e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd4fc0cd200 con 0x7fd4fc06da40 2024-03-06T07:29:29.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.183+0000 7fd501a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd4fc15be10 0x7fd4fc06d330 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47664/0 (socket says 172.21.15.86:47664) 2024-03-06T07:29:29.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.183+0000 7fd501a00640 1 -- 172.21.15.86:0/2643763030 learned_addr learned my addr 172.21.15.86:0/2643763030 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:29.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.183+0000 7fd501000640 1 --2- 172.21.15.86:0/2643763030 >> v2:172.21.15.86:3300/0 conn(0x7fd4fc06da40 0x7fd4fc0ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.184+0000 7fd4fbe00640 1 -- 172.21.15.86:0/2643763030 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3347660028 0 0) 0x7fd4fc15c1f0 con 0x7fd4fc15be10 2024-03-06T07:29:29.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.184+0000 7fd501000640 1 -- 172.21.15.86:0/2643763030 >> v1:172.21.15.194:6789/0 conn(0x7fd4fc0aef20 legacy=0x7fd4fc071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:29.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.184+0000 7fd501000640 1 -- 172.21.15.86:0/2643763030 >> v1:172.21.15.86:6789/0 conn(0x7fd4fc15be10 legacy=0x7fd4fc06d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.184+0000 7fd501000640 1 -- 172.21.15.86:0/2643763030 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4fc06cda0 con 0x7fd4fc06da40 2024-03-06T07:29:29.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.185+0000 7fd501000640 1 --2- 172.21.15.86:0/2643763030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc06da40 0x7fd4fc0ccc20 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7fd4e80097a0 tx=0x7fd4e802b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9b7947dc80d70642 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:29.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd4fbe00640 1 -- 172.21.15.86:0/2643763030 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4e80315a0 con 0x7fd4fc06da40 2024-03-06T07:29:29.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd4fbe00640 1 -- 172.21.15.86:0/2643763030 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd4e8037070 con 0x7fd4fc06da40 2024-03-06T07:29:29.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd4fbe00640 1 -- 172.21.15.86:0/2643763030 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4e8031960 con 0x7fd4fc06da40 2024-03-06T07:29:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd502e00640 1 -- 172.21.15.86:0/2643763030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc06da40 msgr2=0x7fd4fc0ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd502e00640 1 --2- 172.21.15.86:0/2643763030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc06da40 0x7fd4fc0ccc20 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7fd4e80097a0 tx=0x7fd4e802b840 comp rx=0 tx=0).stop 2024-03-06T07:29:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd502e00640 1 -- 172.21.15.86:0/2643763030 shutdown_connections 2024-03-06T07:29:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd502e00640 1 --2- 172.21.15.86:0/2643763030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc06da40 0x7fd4fc0ccc20 unknown :-1 s=CLOSED pgs=570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd502e00640 1 -- 172.21.15.86:0/2643763030 >> 172.21.15.86:0/2643763030 conn(0x7fd4fc06afc0 msgr2=0x7fd4fc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd502e00640 1 -- 172.21.15.86:0/2643763030 shutdown_connections 2024-03-06T07:29:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.186+0000 7fd502e00640 1 -- 172.21.15.86:0/2643763030 wait complete. 2024-03-06T07:29:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.188+0000 7fd502e00640 1 Processor -- start 2024-03-06T07:29:29.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.188+0000 7fd502e00640 1 -- start start 2024-03-06T07:29:29.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd502e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd4fc06da40 0x7fd4fc1f40e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd502e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc0aef20 0x7fd4fc14d290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd502e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4fc15be10 0x7fd4fc0ac780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd502e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4fc0cda90 con 0x7fd4fc0aef20 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd502e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4fc06d080 con 0x7fd4fc15be10 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd502e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd4fc0ce2b0 con 0x7fd4fc06da40 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd501000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc0aef20 0x7fd4fc14d290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd501000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc0aef20 0x7fd4fc14d290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50834/0 (socket says 172.21.15.86:50834) 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.189+0000 7fd501000640 1 -- 172.21.15.86:0/1717968418 learned_addr learned my addr 172.21.15.86:0/1717968418 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd502400640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4fc15be10 0x7fd4fc0ac780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501a00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd4fc06da40 0x7fd4fc1f40e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501000640 1 -- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd4fc06da40 msgr2=0x7fd4fc1f40e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501000640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd4fc06da40 0x7fd4fc1f40e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501000640 1 -- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4fc15be10 msgr2=0x7fd4fc0ac780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501000640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4fc15be10 0x7fd4fc0ac780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501000640 1 -- 172.21.15.86:0/1717968418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4fc0aa1e0 con 0x7fd4fc0aef20 2024-03-06T07:29:29.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501a00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd4fc06da40 0x7fd4fc1f40e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:29.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.190+0000 7fd501000640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc0aef20 0x7fd4fc14d290 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7fd4e80097a0 tx=0x7fd4e80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:29.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.191+0000 7fd4f9600640 1 -- 172.21.15.86:0/1717968418 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4e8004030 con 0x7fd4fc0aef20 2024-03-06T07:29:29.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.191+0000 7fd4f9600640 1 -- 172.21.15.86:0/1717968418 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd4e80041f0 con 0x7fd4fc0aef20 2024-03-06T07:29:29.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.191+0000 7fd4f9600640 1 -- 172.21.15.86:0/1717968418 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4e8032640 con 0x7fd4fc0aef20 2024-03-06T07:29:29.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.191+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4fc0aa4c0 con 0x7fd4fc0aef20 2024-03-06T07:29:29.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.191+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd4fc0aaa80 con 0x7fd4fc0aef20 2024-03-06T07:29:29.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.192+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd4fc068350 con 0x7fd4fc0aef20 2024-03-06T07:29:29.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.193+0000 7fd4f9600640 1 -- 172.21.15.86:0/1717968418 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd4e80327e0 con 0x7fd4fc0aef20 2024-03-06T07:29:29.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.194+0000 7fd4f9600640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd4dc06b4d0 0x7fd4dc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.194+0000 7fd4f9600640 1 -- 172.21.15.86:0/1717968418 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd4e802f030 con 0x7fd4fc0aef20 2024-03-06T07:29:29.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.194+0000 7fd501a00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd4dc06b4d0 0x7fd4dc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.195+0000 7fd501a00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd4dc06b4d0 0x7fd4dc06d990 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fd4f0000f60 tx=0x7fd4f0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:29.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.199+0000 7fd4f9600640 1 -- 172.21.15.86:0/1717968418 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd4fc068350 con 0x7fd4fc0aef20 2024-03-06T07:29:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.394+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fd4fc0cdf90 con 0x7fd4fc0aef20 2024-03-06T07:29:29.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.395+0000 7fd4f9600640 1 -- 172.21.15.86:0/1717968418 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v47) v1 ==== 100+0+11 (secure 0 0 0) 0x7fd4fc0cdf90 con 0x7fd4fc0aef20 2024-03-06T07:29:29.398 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd4dc06b4d0 msgr2=0x7fd4dc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd4dc06b4d0 0x7fd4dc06d990 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fd4f0000f60 tx=0x7fd4f0009040 comp rx=0 tx=0).stop 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc0aef20 msgr2=0x7fd4fc14d290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc0aef20 0x7fd4fc14d290 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7fd4e80097a0 tx=0x7fd4e80046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 shutdown_connections 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd4dc06b4d0 0x7fd4dc06d990 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd4fc15be10 0x7fd4fc0ac780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd4fc0aef20 0x7fd4fc14d290 unknown :-1 s=CLOSED pgs=571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 --2- 172.21.15.86:0/1717968418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd4fc06da40 0x7fd4fc1f40e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 >> 172.21.15.86:0/1717968418 conn(0x7fd4fc06afc0 msgr2=0x7fd4fc0cbf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 shutdown_connections 2024-03-06T07:29:29.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.398+0000 7fd502e00640 1 -- 172.21.15.86:0/1717968418 wait complete. 2024-03-06T07:29:29.419 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:29.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.610+0000 7ff607800640 1 Processor -- start 2024-03-06T07:29:29.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.611+0000 7ff607800640 1 -- start start 2024-03-06T07:29:29.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.612+0000 7ff607800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff60015eff0 0x7ff600070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.612+0000 7ff607800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff600071d30 con 0x7ff6000af1c0 2024-03-06T07:29:29.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.612+0000 7ff607800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff60015f4a0 con 0x7ff60006e4d0 2024-03-06T07:29:29.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.612+0000 7ff607800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff600071260 con 0x7ff60015eff0 2024-03-06T07:29:29.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff604e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff6000af1c0 0x7ff60006cc60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47674/0 (socket says 172.21.15.86:47674) 2024-03-06T07:29:29.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff604e00640 1 -- 172.21.15.86:0/3808880758 learned_addr learned my addr 172.21.15.86:0/3808880758 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:29.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/3808880758 >> v2:172.21.15.194:3300/0 conn(0x7ff60015eff0 0x7ff600070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2490091450 0 0) 0x7ff5f0001110 con 0x7ff6000af1c0 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff5e4003520 con 0x7ff6000af1c0 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2006684581 0 0) 0x7ff5e4003520 con 0x7ff6000af1c0 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 >> v2:172.21.15.194:3300/0 conn(0x7ff60015eff0 msgr2=0x7ff600070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 --2- 172.21.15.86:0/3808880758 >> v2:172.21.15.194:3300/0 conn(0x7ff60015eff0 0x7ff600070d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 >> v1:172.21.15.194:6789/0 conn(0x7ff60006e4d0 legacy=0x7ff6000b35d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff600071490 con 0x7ff6000af1c0 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff5f0001840 con 0x7ff6000af1c0 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 >> v1:172.21.15.86:6789/0 conn(0x7ff6000af1c0 legacy=0x7ff60006cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.613+0000 7ff5ffe00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4003a30 0x7ff5e4003e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5ffe00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e4004800 0x7ff5e4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5ffe00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e40051c0 0x7ff5e400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5e400c090 con 0x7ff5e4004800 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5e400c240 con 0x7ff5e40051c0 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5e400c440 con 0x7ff5e4003a30 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e4004800 0x7ff5e4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff604e00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4003a30 0x7ff5e4003e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5f7e00640 1 -- 172.21.15.86:0/3808880758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4003a30 msgr2=0x7ff5e4003e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4003a30 0x7ff5e4003e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5f7e00640 1 -- 172.21.15.86:0/3808880758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e40051c0 msgr2=0x7ff5e400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e40051c0 0x7ff5e400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff5f7e00640 1 -- 172.21.15.86:0/3808880758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5e4003520 con 0x7ff5e4004800 2024-03-06T07:29:29.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.614+0000 7ff605800640 1 -- 172.21.15.86:0/3808880758 reap_dead start 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.615+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e4004800 0x7ff5e4004c80 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7ff5e8009ac0 tx=0x7ff5e4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=99464bc0c718118e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.615+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5e8039070 con 0x7ff5e4004800 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.615+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5e802c710 con 0x7ff5e4004800 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.615+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3808880758 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5e8009690 con 0x7ff5e4004800 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.616+0000 7ff607800640 1 -- 172.21.15.86:0/3808880758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e4004800 msgr2=0x7ff5e4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.616+0000 7ff607800640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e4004800 0x7ff5e4004c80 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7ff5e8009ac0 tx=0x7ff5e4005830 comp rx=0 tx=0).stop 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.616+0000 7ff607800640 1 -- 172.21.15.86:0/3808880758 shutdown_connections 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.616+0000 7ff607800640 1 --2- 172.21.15.86:0/3808880758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e4004800 0x7ff5e4004c80 unknown :-1 s=CLOSED pgs=573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.616+0000 7ff607800640 1 -- 172.21.15.86:0/3808880758 >> 172.21.15.86:0/3808880758 conn(0x7ff6000b1730 msgr2=0x7ff6000b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.616+0000 7ff607800640 1 -- 172.21.15.86:0/3808880758 shutdown_connections 2024-03-06T07:29:29.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.616+0000 7ff607800640 1 -- 172.21.15.86:0/3808880758 wait complete. 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.618+0000 7ff607800640 1 Processor -- start 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.618+0000 7ff607800640 1 -- start start 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff607800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4004800 0x7ff6001475c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff607800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6000b34d0 0x7ff6000b3950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff607800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff600147fd0 0x7ff600148460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff607800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6000b30b0 con 0x7ff6000b34d0 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff607800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff600071a50 con 0x7ff600147fd0 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff607800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff600070d20 con 0x7ff5e4004800 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6000b34d0 0x7ff6000b3950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6000b34d0 0x7ff6000b3950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50846/0 (socket says 172.21.15.86:50846) 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 -- 172.21.15.86:0/2328379983 learned_addr learned my addr 172.21.15.86:0/2328379983 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff604e00640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4004800 0x7ff6001475c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 -- 172.21.15.86:0/2328379983 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4004800 msgr2=0x7ff6001475c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4004800 0x7ff6001475c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 -- 172.21.15.86:0/2328379983 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff600147fd0 msgr2=0x7ff600148460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff600147fd0 0x7ff600148460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.619+0000 7ff5f7e00640 1 -- 172.21.15.86:0/2328379983 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff600147b00 con 0x7ff6000b34d0 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.620+0000 7ff5f7e00640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6000b34d0 0x7ff6000b3950 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7ff5e8009ac0 tx=0x7ff5e8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.620+0000 7ff5fd600640 1 -- 172.21.15.86:0/2328379983 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5e8042020 con 0x7ff6000b34d0 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.620+0000 7ff5fd600640 1 -- 172.21.15.86:0/2328379983 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5e803a040 con 0x7ff6000b34d0 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.620+0000 7ff5fd600640 1 -- 172.21.15.86:0/2328379983 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5e8039070 con 0x7ff6000b34d0 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.620+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff600148d20 con 0x7ff6000b34d0 2024-03-06T07:29:29.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.620+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff6001492e0 con 0x7ff6000b34d0 2024-03-06T07:29:29.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.621+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff6000b3e90 con 0x7ff6000b34d0 2024-03-06T07:29:29.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.622+0000 7ff5fd600640 1 -- 172.21.15.86:0/2328379983 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff5e8045020 con 0x7ff6000b34d0 2024-03-06T07:29:29.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.623+0000 7ff5fd600640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5dc06b500 0x7ff5dc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:29.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.623+0000 7ff5fd600640 1 -- 172.21.15.86:0/2328379983 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff5e80ab660 con 0x7ff6000b34d0 2024-03-06T07:29:29.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.623+0000 7ff604e00640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5dc06b500 0x7ff5dc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:29.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.624+0000 7ff604e00640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5dc06b500 0x7ff5dc06d9c0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7ff60015bc40 tx=0x7ff5f0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:29.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.628+0000 7ff5fd600640 1 -- 172.21.15.86:0/2328379983 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff5e804c070 con 0x7ff6000b34d0 2024-03-06T07:29:29.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.842+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff600161840 con 0x7ff6000b34d0 2024-03-06T07:29:29.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.844+0000 7ff5fd600640 1 -- 172.21.15.86:0/2328379983 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) v1 ==== 94+0+4983 (secure 0 0 0) 0x7ff600161840 con 0x7ff6000b34d0 2024-03-06T07:29:29.846 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:29.847 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":25,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5041,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/3678578793","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":3678578793},{"type":"v1","addr":"172.21.15.194:6835","nonce":3678578793}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/4206282845","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":4206282845},{"type":"v1","addr":"172.21.15.86:6835","nonce":4206282845}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5427,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3500830166","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3500830166},{"type":"v1","addr":"172.21.15.86:6837","nonce":3500830166}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":25,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:59.832195+0000","modified":"2024-03-06T07:29:02.883187+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5439},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5439":{"gid":5439,"name":"b","rank":0,"incarnation":21,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6833/4064823241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":4064823241},{"type":"v1","addr":"172.21.15.194:6833","nonce":4064823241}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5439,"qdb_cluster":[5439]},"id":3}]} 2024-03-06T07:29:29.847 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 25 2024-03-06T07:29:29.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5dc06b500 msgr2=0x7ff5dc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5dc06b500 0x7ff5dc06d9c0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7ff60015bc40 tx=0x7ff5f0009040 comp rx=0 tx=0).stop 2024-03-06T07:29:29.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6000b34d0 msgr2=0x7ff6000b3950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:29.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6000b34d0 0x7ff6000b3950 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7ff5e8009ac0 tx=0x7ff5e8004730 comp rx=0 tx=0).stop 2024-03-06T07:29:29.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 shutdown_connections 2024-03-06T07:29:29.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5dc06b500 0x7ff5dc06d9c0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff600147fd0 0x7ff600148460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6000b34d0 0x7ff6000b3950 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.846+0000 7ff607800640 1 --2- 172.21.15.86:0/2328379983 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e4004800 0x7ff6001475c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:29.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.847+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 >> 172.21.15.86:0/2328379983 conn(0x7ff6000b1730 msgr2=0x7ff6000ad940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:29.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.847+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 shutdown_connections 2024-03-06T07:29:29.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:29.847+0000 7ff607800640 1 -- 172.21.15.86:0/2328379983 wait complete. 2024-03-06T07:29:29.866 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:30.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.042+0000 7fe5ff400640 1 Processor -- start 2024-03-06T07:29:30.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.042+0000 7fe5ff400640 1 -- start start 2024-03-06T07:29:30.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5ff400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe5f815fe60 0x7fe5f8071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5ff400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe5f806cef0 con 0x7fe5f8144c00 2024-03-06T07:29:30.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5ff400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe5f806d1d0 con 0x7fe5f815f9d0 2024-03-06T07:29:30.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5ff400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe5f806d4b0 con 0x7fe5f815fe60 2024-03-06T07:29:30.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5fca00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe5f8144c00 0x7fe5f806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47680/0 (socket says 172.21.15.86:47680) 2024-03-06T07:29:30.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5fca00640 1 -- 172.21.15.86:0/4018404073 learned_addr learned my addr 172.21.15.86:0/4018404073 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:30.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2545114981 0 0) 0x7fe5f806cef0 con 0x7fe5f8144c00 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe5dc003520 con 0x7fe5f8144c00 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5fd400640 1 --2- 172.21.15.86:0/4018404073 >> v2:172.21.15.194:3300/0 conn(0x7fe5f815fe60 0x7fe5f8071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4043499098 0 0) 0x7fe5dc003520 con 0x7fe5f8144c00 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 >> v2:172.21.15.194:3300/0 conn(0x7fe5f815fe60 msgr2=0x7fe5f8071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.044+0000 7fe5f7e00640 1 --2- 172.21.15.86:0/4018404073 >> v2:172.21.15.194:3300/0 conn(0x7fe5f815fe60 0x7fe5f8071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 >> v1:172.21.15.194:6789/0 conn(0x7fe5f815f9d0 legacy=0x7fe5f8070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5f806d790 con 0x7fe5f8144c00 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fe5e8001670 con 0x7fe5f8144c00 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 >> v1:172.21.15.86:6789/0 conn(0x7fe5f8144c00 legacy=0x7fe5f806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5dc0039b0 0x7fe5dc003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5dc004800 0x7fe5dc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5dc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f806d4b0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5dc00c0f0 con 0x7fe5dc0039b0 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5dc00c2f0 con 0x7fe5dc004800 2024-03-06T07:29:30.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5fd400640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5dc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5fca00640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5dc0039b0 0x7fe5dc003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5fd400640 1 -- 172.21.15.86:0/4018404073 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5dc004800 msgr2=0x7fe5dc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5fd400640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5dc004800 0x7fe5dc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5fd400640 1 -- 172.21.15.86:0/4018404073 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5dc0039b0 msgr2=0x7fe5dc003db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5fd400640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5dc0039b0 0x7fe5dc003db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.045+0000 7fe5fd400640 1 -- 172.21.15.86:0/4018404073 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5dc003520 con 0x7fe5dc0051c0 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.046+0000 7fe5fd400640 1 -- 172.21.15.86:0/4018404073 reap_dead start 2024-03-06T07:29:30.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.046+0000 7fe5fd400640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5dc00b9c0 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7fe5f0007330 tx=0x7fe5dc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5b17c677ff702ab9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5f0009be0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5f002ccd0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5f7e00640 1 -- 172.21.15.86:0/4018404073 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5f0003bd0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5ff400640 1 -- 172.21.15.86:0/4018404073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 msgr2=0x7fe5dc00b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5ff400640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5dc00b9c0 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7fe5f0007330 tx=0x7fe5dc005830 comp rx=0 tx=0).stop 2024-03-06T07:29:30.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5ff400640 1 -- 172.21.15.86:0/4018404073 shutdown_connections 2024-03-06T07:29:30.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5ff400640 1 --2- 172.21.15.86:0/4018404073 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5dc00b9c0 unknown :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5ff400640 1 -- 172.21.15.86:0/4018404073 >> 172.21.15.86:0/4018404073 conn(0x7fe5f80b1730 msgr2=0x7fe5f80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:30.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5ff400640 1 -- 172.21.15.86:0/4018404073 shutdown_connections 2024-03-06T07:29:30.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.047+0000 7fe5ff400640 1 -- 172.21.15.86:0/4018404073 wait complete. 2024-03-06T07:29:30.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.049+0000 7fe5ff400640 1 Processor -- start 2024-03-06T07:29:30.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.049+0000 7fe5ff400640 1 -- start start 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5ff400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5f813c420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5ff400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f813ebc0 0x7fe5f813afa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5ff400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f815b400 0x7fe5f815b860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5ff400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f806cb50 con 0x7fe5dc0051c0 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5ff400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f8160730 con 0x7fe5f813ebc0 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5ff400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f806ccd0 con 0x7fe5f815b400 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5fca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5f813c420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5fca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5f813c420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50864/0 (socket says 172.21.15.86:50864) 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5fca00640 1 -- 172.21.15.86:0/1573988694 learned_addr learned my addr 172.21.15.86:0/1573988694 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.050+0000 7fe5efe00640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f813ebc0 0x7fe5f813afa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5fd400640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f815b400 0x7fe5f815b860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5fca00640 1 -- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f815b400 msgr2=0x7fe5f815b860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5fca00640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f815b400 0x7fe5f815b860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5fca00640 1 -- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f813ebc0 msgr2=0x7fe5f813afa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5fca00640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f813ebc0 0x7fe5f813afa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5fca00640 1 -- 172.21.15.86:0/1573988694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5f815bda0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5fca00640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5f813c420 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fe5e800e730 tx=0x7fe5e800ebf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5f5600640 1 -- 172.21.15.86:0/1573988694 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5e800ee40 con 0x7fe5dc0051c0 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5f5600640 1 -- 172.21.15.86:0/1573988694 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5e8018070 con 0x7fe5dc0051c0 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5f5600640 1 -- 172.21.15.86:0/1573988694 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5e800cb10 con 0x7fe5dc0051c0 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe5f815edd0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.051+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5f813e3c0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.052+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe5f804f330 con 0x7fe5dc0051c0 2024-03-06T07:29:30.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.053+0000 7fe5f5600640 1 -- 172.21.15.86:0/1573988694 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe5e800ccb0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.054+0000 7fe5f5600640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5d806b500 0x7fe5d806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.054+0000 7fe5f5600640 1 -- 172.21.15.86:0/1573988694 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe5e808a040 con 0x7fe5dc0051c0 2024-03-06T07:29:30.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.054+0000 7fe5efe00640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5d806b500 0x7fe5d806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.055+0000 7fe5efe00640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5d806b500 0x7fe5d806d9c0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fe5f815ea30 tx=0x7fe5e4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.058+0000 7fe5f5600640 1 -- 172.21.15.86:0/1573988694 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe5e8058ec0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.251+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe5f80b1e90 con 0x7fe5dc0051c0 2024-03-06T07:29:30.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.253+0000 7fe5f5600640 1 -- 172.21.15.86:0/1573988694 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v47) v1 ==== 74+0+16625 (secure 0 0 0) 0x7fe5e80587c0 con 0x7fe5dc0051c0 2024-03-06T07:29:30.256 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:30.256 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":47,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:01.377099+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:57.609038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:58.840885+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:30.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5d806b500 msgr2=0x7fe5d806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5d806b500 0x7fe5d806d9c0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fe5f815ea30 tx=0x7fe5e4009040 comp rx=0 tx=0).stop 2024-03-06T07:29:30.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 msgr2=0x7fe5f813c420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5f813c420 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fe5e800e730 tx=0x7fe5e800ebf0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 shutdown_connections 2024-03-06T07:29:30.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5d806b500 0x7fe5d806d9c0 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f815b400 0x7fe5f815b860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f813ebc0 0x7fe5f813afa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 --2- 172.21.15.86:0/1573988694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5dc0051c0 0x7fe5f813c420 unknown :-1 s=CLOSED pgs=577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.255+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 >> 172.21.15.86:0/1573988694 conn(0x7fe5f80b1730 msgr2=0x7fe5f8072060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:30.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.256+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 shutdown_connections 2024-03-06T07:29:30.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.256+0000 7fe5ff400640 1 -- 172.21.15.86:0/1573988694 wait complete. 2024-03-06T07:29:30.275 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:29:30.275 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:30.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.477+0000 7f346f000640 1 Processor -- start 2024-03-06T07:29:30.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.477+0000 7f346f000640 1 -- start start 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.479+0000 7f346f000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3468144c00 0x7f346806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.479+0000 7f346f000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f346815f9d0 0x7f346815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.479+0000 7f346f000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f346806ee70 con 0x7f34680ce0a0 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.479+0000 7f346f000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f346806cef0 con 0x7f3468144c00 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.479+0000 7f346f000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f346806d1d0 con 0x7f346815f9d0 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.479+0000 7f3467e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3468144c00 0x7f346806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.480+0000 7f3467e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3468144c00 0x7f346806e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50876/0 (socket says 172.21.15.86:50876) 2024-03-06T07:29:30.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.480+0000 7f3467e00640 1 -- 172.21.15.86:0/2057386917 learned_addr learned my addr 172.21.15.86:0/2057386917 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:30.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.480+0000 7f345f400640 1 --2- 172.21.15.86:0/2057386917 >> v2:172.21.15.194:3300/0 conn(0x7f346815f9d0 0x7f346815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.480+0000 7f3467e00640 1 -- 172.21.15.86:0/2057386917 >> v2:172.21.15.194:3300/0 conn(0x7f346815f9d0 msgr2=0x7f346815fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.480+0000 7f3467e00640 1 --2- 172.21.15.86:0/2057386917 >> v2:172.21.15.194:3300/0 conn(0x7f346815f9d0 0x7f346815fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.480+0000 7f3467e00640 1 -- 172.21.15.86:0/2057386917 >> v1:172.21.15.194:6789/0 conn(0x7f34680ce0a0 legacy=0x7f34680ce4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.480+0000 7f3467e00640 1 -- 172.21.15.86:0/2057386917 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f346806d4b0 con 0x7f3468144c00 2024-03-06T07:29:30.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.482+0000 7f3467e00640 1 --2- 172.21.15.86:0/2057386917 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f346806e930 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f34680b26f0 tx=0x7f345802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a66ffea3ebb03f52 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.482+0000 7f3467400640 1 -- 172.21.15.86:0/2057386917 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3458035020 con 0x7f3468144c00 2024-03-06T07:29:30.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.482+0000 7f3467400640 1 -- 172.21.15.86:0/2057386917 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3458031870 con 0x7f3468144c00 2024-03-06T07:29:30.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.482+0000 7f3467400640 1 -- 172.21.15.86:0/2057386917 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3458031c40 con 0x7f3468144c00 2024-03-06T07:29:30.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 -- 172.21.15.86:0/2057386917 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 msgr2=0x7f346806e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 --2- 172.21.15.86:0/2057386917 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f346806e930 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f34680b26f0 tx=0x7f345802b910 comp rx=0 tx=0).stop 2024-03-06T07:29:30.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 -- 172.21.15.86:0/2057386917 shutdown_connections 2024-03-06T07:29:30.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 --2- 172.21.15.86:0/2057386917 >> v2:172.21.15.194:3300/0 conn(0x7f346815f9d0 0x7f346815fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 --2- 172.21.15.86:0/2057386917 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f346806e930 unknown :-1 s=CLOSED pgs=578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 -- 172.21.15.86:0/2057386917 >> 172.21.15.86:0/2057386917 conn(0x7f34680b1730 msgr2=0x7f34680b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:30.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 -- 172.21.15.86:0/2057386917 shutdown_connections 2024-03-06T07:29:30.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.483+0000 7f346f000640 1 -- 172.21.15.86:0/2057386917 wait complete. 2024-03-06T07:29:30.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.485+0000 7f346f000640 1 Processor -- start 2024-03-06T07:29:30.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.485+0000 7f346f000640 1 -- start start 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f346f000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34680ce0a0 0x7f34681f40d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f346f000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f34681f6e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f346f000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346815f9d0 0x7f346813c190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f346f000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f346806cb50 con 0x7f3468144c00 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f346f000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3468160730 con 0x7f34680ce0a0 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f346f000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f346806ccd0 con 0x7f346815f9d0 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f345f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f34681f6e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f345f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f34681f6e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50888/0 (socket says 172.21.15.86:50888) 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f345f400640 1 -- 172.21.15.86:0/565056112 learned_addr learned my addr 172.21.15.86:0/565056112 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f346d000640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346815f9d0 0x7f346813c190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.486+0000 7f3467e00640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34680ce0a0 0x7f34681f40d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f345f400640 1 -- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346815f9d0 msgr2=0x7f346813c190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f345f400640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346815f9d0 0x7f346813c190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f345f400640 1 -- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34680ce0a0 msgr2=0x7f34681f40d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f345f400640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34680ce0a0 0x7f34681f40d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f345f400640 1 -- 172.21.15.86:0/565056112 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34680ae490 con 0x7f3468144c00 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f3467e00640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34680ce0a0 0x7f34681f40d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f345f400640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f34681f6e40 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f345000d800 tx=0x7f345000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f3464c00640 1 -- 172.21.15.86:0/565056112 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f345000b660 con 0x7f3468144c00 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f3464c00640 1 -- 172.21.15.86:0/565056112 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3450019070 con 0x7f3468144c00 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f34680ae770 con 0x7f3468144c00 2024-03-06T07:29:30.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f3464c00640 1 -- 172.21.15.86:0/565056112 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f345000ba20 con 0x7f3468144c00 2024-03-06T07:29:30.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.487+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f34680b2e10 con 0x7f3468144c00 2024-03-06T07:29:30.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.489+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f346804f330 con 0x7f3468144c00 2024-03-06T07:29:30.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.489+0000 7f3464c00640 1 -- 172.21.15.86:0/565056112 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f345000bbc0 con 0x7f3468144c00 2024-03-06T07:29:30.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.490+0000 7f3464c00640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.490+0000 7f3464c00640 1 -- 172.21.15.86:0/565056112 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f345008a660 con 0x7f3468144c00 2024-03-06T07:29:30.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.491+0000 7f3467e00640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.491+0000 7f3467e00640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f3458036000 tx=0x7f3458008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.494+0000 7f3464c00640 1 -- 172.21.15.86:0/565056112 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f34500594e0 con 0x7f3468144c00 2024-03-06T07:29:30.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.722+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3468072890 con 0x7f3468144c00 2024-03-06T07:29:30.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.723+0000 7f3464c00640 1 -- 172.21.15.86:0/565056112 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) v1 ==== 94+0+4983 (secure 0 0 0) 0x7f3450058de0 con 0x7f3468144c00 2024-03-06T07:29:30.726 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:30.726 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":25,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5041,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/3678578793","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":3678578793},{"type":"v1","addr":"172.21.15.194:6835","nonce":3678578793}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/4206282845","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":4206282845},{"type":"v1","addr":"172.21.15.86:6835","nonce":4206282845}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5427,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3500830166","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3500830166},{"type":"v1","addr":"172.21.15.86:6837","nonce":3500830166}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":25,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:59.832195+0000","modified":"2024-03-06T07:29:02.883187+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5439},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5439":{"gid":5439,"name":"b","rank":0,"incarnation":21,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6833/4064823241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":4064823241},{"type":"v1","addr":"172.21.15.194:6833","nonce":4064823241}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5439,"qdb_cluster":[5439]},"id":3}]} 2024-03-06T07:29:30.726 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 25 2024-03-06T07:29:30.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 msgr2=0x7f344406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f3458036000 tx=0x7f3458008030 comp rx=0 tx=0).stop 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 msgr2=0x7f34681f6e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f34681f6e40 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f345000d800 tx=0x7f345000dcc0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 shutdown_connections 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346815f9d0 0x7f346813c190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3468144c00 0x7f34681f6e40 unknown :-1 s=CLOSED pgs=579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 --2- 172.21.15.86:0/565056112 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34680ce0a0 0x7f34681f40d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 >> 172.21.15.86:0/565056112 conn(0x7f34680b1730 msgr2=0x7f34680af5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 shutdown_connections 2024-03-06T07:29:30.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.726+0000 7f346f000640 1 -- 172.21.15.86:0/565056112 wait complete. 2024-03-06T07:29:30.744 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:30.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.944+0000 7f2d7a600640 1 Processor -- start 2024-03-06T07:29:30.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.944+0000 7f2d7a600640 1 -- start start 2024-03-06T07:29:30.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.945+0000 7f2d7a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 0x7f2d740ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.946+0000 7f2d7a600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d7415c240 con 0x7f2d740cc9d0 2024-03-06T07:29:30.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.946+0000 7f2d7a600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d7415c500 con 0x7f2d740aebf0 2024-03-06T07:29:30.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.946+0000 7f2d7a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d7415ce30 con 0x7f2d741561f0 2024-03-06T07:29:30.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.946+0000 7f2d79c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2d740cc9d0 0x7f2d7415f830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47684/0 (socket says 172.21.15.86:47684) 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.946+0000 7f2d79c00640 1 -- 172.21.15.86:0/2253309974 learned_addr learned my addr 172.21.15.86:0/2253309974 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.946+0000 7f2d79200640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 0x7f2d740ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.946+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2314331432 0 0) 0x7f2d7415c240 con 0x7f2d740cc9d0 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d5c003520 con 0x7f2d740cc9d0 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2876762436 0 0) 0x7f2d5c003520 con 0x7f2d740cc9d0 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 msgr2=0x7f2d740ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 0x7f2d740ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 >> v1:172.21.15.194:6789/0 conn(0x7f2d740aebf0 legacy=0x7f2d740cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d7415c7e0 con 0x7f2d740cc9d0 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f2d6c0019f0 con 0x7f2d740cc9d0 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 >> v1:172.21.15.86:6789/0 conn(0x7f2d740cc9d0 legacy=0x7f2d7415f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d5c0039e0 0x7f2d5c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d5c004800 0x7f2d5c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 0x7f2d5c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d7415ce30 con 0x7f2d5c004800 2024-03-06T07:29:30.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d5c00ba70 con 0x7f2d5c0039e0 2024-03-06T07:29:30.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d5c00bc70 con 0x7f2d741561f0 2024-03-06T07:29:30.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d5c004800 0x7f2d5c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.947+0000 7f2d79200640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d5c0039e0 0x7f2d5c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.948+0000 7f2d79c00640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 0x7f2d5c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.948+0000 7f2d73e00640 1 -- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 msgr2=0x7f2d5c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.948+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 0x7f2d5c0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.948+0000 7f2d73e00640 1 -- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d5c0039e0 msgr2=0x7f2d5c003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.948+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d5c0039e0 0x7f2d5c003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.948+0000 7f2d73e00640 1 -- 172.21.15.86:0/2253309974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d5c003520 con 0x7f2d5c004800 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.948+0000 7f2d79c00640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d741561f0 0x7f2d5c0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.949+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d5c004800 0x7f2d5c004c80 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f2d60001720 tx=0x7f2d5c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ff1f36c0380e2cf9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.949+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d60001370 con 0x7f2d5c004800 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.949+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d600052d0 con 0x7f2d5c004800 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.949+0000 7f2d73400640 1 -- 172.21.15.86:0/2253309974 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d60003830 con 0x7f2d5c004800 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.950+0000 7f2d7a600640 1 -- 172.21.15.86:0/2253309974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d5c004800 msgr2=0x7f2d5c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.950+0000 7f2d7a600640 1 --2- 172.21.15.86:0/2253309974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d5c004800 0x7f2d5c004c80 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f2d60001720 tx=0x7f2d5c005830 comp rx=0 tx=0).stop 2024-03-06T07:29:30.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.950+0000 7f2d79c00640 1 -- 172.21.15.86:0/2253309974 reap_dead start 2024-03-06T07:29:30.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.950+0000 7f2d7a600640 1 -- 172.21.15.86:0/2253309974 shutdown_connections 2024-03-06T07:29:30.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.950+0000 7f2d7a600640 1 -- 172.21.15.86:0/2253309974 >> 172.21.15.86:0/2253309974 conn(0x7f2d7406afc0 msgr2=0x7f2d7406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:30.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.950+0000 7f2d7a600640 1 -- 172.21.15.86:0/2253309974 shutdown_connections 2024-03-06T07:29:30.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.950+0000 7f2d7a600640 1 -- 172.21.15.86:0/2253309974 wait complete. 2024-03-06T07:29:30.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.951+0000 7f2d7a600640 1 Processor -- start 2024-03-06T07:29:30.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.952+0000 7f2d7a600640 1 -- start start 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.952+0000 7f2d7a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d5c004800 0x7f2d741e5380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.952+0000 7f2d7a600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d740cd410 0x7f2d741f3ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.952+0000 7f2d7a600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d741f4410 0x7f2d741f48c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.952+0000 7f2d7a600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d741565d0 con 0x7f2d740cd410 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.952+0000 7f2d7a600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d7406e360 con 0x7f2d741f4410 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.952+0000 7f2d7a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d74156750 con 0x7f2d5c004800 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d740cd410 0x7f2d741f3ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d740cd410 0x7f2d741f3ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50906/0 (socket says 172.21.15.86:50906) 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 -- 172.21.15.86:0/2755146667 learned_addr learned my addr 172.21.15.86:0/2755146667 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d79200640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d5c004800 0x7f2d741e5380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d79c00640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d741f4410 0x7f2d741f48c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 -- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d5c004800 msgr2=0x7f2d741e5380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d5c004800 0x7f2d741e5380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 -- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d741f4410 msgr2=0x7f2d741f48c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d741f4410 0x7f2d741f48c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d73e00640 1 -- 172.21.15.86:0/2755146667 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d740cdbf0 con 0x7f2d740cd410 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.953+0000 7f2d79c00640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d741f4410 0x7f2d741f48c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.954+0000 7f2d73e00640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d740cd410 0x7f2d741f3ed0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f2d60005040 tx=0x7f2d60001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.954+0000 7f2d70c00640 1 -- 172.21.15.86:0/2755146667 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d60041070 con 0x7f2d740cd410 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.954+0000 7f2d70c00640 1 -- 172.21.15.86:0/2755146667 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d60005480 con 0x7f2d740cd410 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.954+0000 7f2d70c00640 1 -- 172.21.15.86:0/2755146667 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d60007a30 con 0x7f2d740cd410 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.954+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d740cded0 con 0x7f2d740cd410 2024-03-06T07:29:30.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.954+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d740ce490 con 0x7f2d740cd410 2024-03-06T07:29:30.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.955+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d7404f330 con 0x7f2d740cd410 2024-03-06T07:29:30.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.956+0000 7f2d70c00640 1 -- 172.21.15.86:0/2755146667 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d60044070 con 0x7f2d740cd410 2024-03-06T07:29:30.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.957+0000 7f2d70c00640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b500 0x7f2d5406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:30.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.957+0000 7f2d70c00640 1 -- 172.21.15.86:0/2755146667 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2d600abd70 con 0x7f2d740cd410 2024-03-06T07:29:30.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.957+0000 7f2d79200640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b500 0x7f2d5406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:30.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.958+0000 7f2d79200640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b500 0x7f2d5406d9c0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f2d741f7480 tx=0x7f2d68008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:30.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:30.961+0000 7f2d70c00640 1 -- 172.21.15.86:0/2755146667 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d600052a0 con 0x7f2d740cd410 2024-03-06T07:29:31.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.196+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2d740b2b10 con 0x7f2d740cd410 2024-03-06T07:29:31.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.197+0000 7f2d70c00640 1 -- 172.21.15.86:0/2755146667 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 25 v25) v1 ==== 94+0+4983 (secure 0 0 0) 0x7f2d6007ab50 con 0x7f2d740cd410 2024-03-06T07:29:31.200 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:31.200 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":25,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5041,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/3678578793","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":3678578793},{"type":"v1","addr":"172.21.15.194:6835","nonce":3678578793}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/4206282845","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":4206282845},{"type":"v1","addr":"172.21.15.86:6835","nonce":4206282845}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5427,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3500830166","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3500830166},{"type":"v1","addr":"172.21.15.86:6837","nonce":3500830166}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20}],"filesystems":[{"mdsmap":{"epoch":25,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:28:59.832195+0000","modified":"2024-03-06T07:29:02.883187+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5439},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5439":{"gid":5439,"name":"b","rank":0,"incarnation":21,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6833/4064823241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":4064823241},{"type":"v1","addr":"172.21.15.194:6833","nonce":4064823241}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5439,"qdb_cluster":[5439]},"id":3}]} 2024-03-06T07:29:31.200 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 25 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b500 msgr2=0x7f2d5406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b500 0x7f2d5406d9c0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f2d741f7480 tx=0x7f2d68008040 comp rx=0 tx=0).stop 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d740cd410 msgr2=0x7f2d741f3ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d740cd410 0x7f2d741f3ed0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f2d60005040 tx=0x7f2d60001120 comp rx=0 tx=0).stop 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 shutdown_connections 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2d5406b500 0x7f2d5406d9c0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2d741f4410 0x7f2d741f48c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2d740cd410 0x7f2d741f3ed0 unknown :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 --2- 172.21.15.86:0/2755146667 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2d5c004800 0x7f2d741e5380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 >> 172.21.15.86:0/2755146667 conn(0x7f2d7406afc0 msgr2=0x7f2d740af550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:31.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 shutdown_connections 2024-03-06T07:29:31.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.200+0000 7f2d7a600640 1 -- 172.21.15.86:0/2755146667 wait complete. 2024-03-06T07:29:31.220 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:31.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.432+0000 7f1da9a00640 1 Processor -- start 2024-03-06T07:29:31.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.433+0000 7f1da9a00640 1 -- start start 2024-03-06T07:29:31.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1da40cc9d0 0x7f1da406d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da9a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1da415c240 con 0x7f1da40aedc0 2024-03-06T07:29:31.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da9a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1da415c500 con 0x7f1da41561f0 2024-03-06T07:29:31.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da9a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1da415ce30 con 0x7f1da40cc9d0 2024-03-06T07:29:31.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da9000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1da40cc9d0 0x7f1da406d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da9000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1da40cc9d0 0x7f1da406d350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50910/0 (socket says 172.21.15.86:50910) 2024-03-06T07:29:31.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da3400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1da40aedc0 0x7f1da40cc2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47688/0 (socket says 172.21.15.86:47688) 2024-03-06T07:29:31.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.434+0000 7f1da9000640 1 -- 172.21.15.86:0/1898937362 learned_addr learned my addr 172.21.15.86:0/1898937362 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:31.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.435+0000 7f1da9000640 1 -- 172.21.15.86:0/1898937362 >> v1:172.21.15.194:6789/0 conn(0x7f1da41561f0 legacy=0x7f1da40ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:31.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.435+0000 7f1da9000640 1 -- 172.21.15.86:0/1898937362 >> v1:172.21.15.86:6789/0 conn(0x7f1da40aedc0 legacy=0x7f1da40cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.435+0000 7f1da9000640 1 -- 172.21.15.86:0/1898937362 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1da415c7e0 con 0x7f1da40cc9d0 2024-03-06T07:29:31.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.435+0000 7f1da2a00640 1 -- 172.21.15.86:0/1898937362 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4031710408 0 0) 0x7f1da415c240 con 0x7f1da40aedc0 2024-03-06T07:29:31.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da9000640 1 --2- 172.21.15.86:0/1898937362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da40cc9d0 0x7f1da406d350 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f1da415cfb0 tx=0x7f1d9c02bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71f608abd79042dc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:31.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da2a00640 1 -- 172.21.15.86:0/1898937362 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d9c02caf0 con 0x7f1da40cc9d0 2024-03-06T07:29:31.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da2a00640 1 -- 172.21.15.86:0/1898937362 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1d9c02cc90 con 0x7f1da40cc9d0 2024-03-06T07:29:31.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da2a00640 1 -- 172.21.15.86:0/1898937362 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d9c034610 con 0x7f1da40cc9d0 2024-03-06T07:29:31.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da9a00640 1 -- 172.21.15.86:0/1898937362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da40cc9d0 msgr2=0x7f1da406d350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1898937362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da40cc9d0 0x7f1da406d350 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f1da415cfb0 tx=0x7f1d9c02bf50 comp rx=0 tx=0).stop 2024-03-06T07:29:31.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da9a00640 1 -- 172.21.15.86:0/1898937362 shutdown_connections 2024-03-06T07:29:31.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1898937362 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da40cc9d0 0x7f1da406d350 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.436+0000 7f1da9a00640 1 -- 172.21.15.86:0/1898937362 >> 172.21.15.86:0/1898937362 conn(0x7f1da406afc0 msgr2=0x7f1da406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:31.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.437+0000 7f1da9a00640 1 -- 172.21.15.86:0/1898937362 shutdown_connections 2024-03-06T07:29:31.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.437+0000 7f1da9a00640 1 -- 172.21.15.86:0/1898937362 wait complete. 2024-03-06T07:29:31.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.438+0000 7f1da9a00640 1 Processor -- start 2024-03-06T07:29:31.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.438+0000 7f1da9a00640 1 -- start start 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1da40aedc0 0x7f1da41587d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1da40cc9d0 0x7f1da4152600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da41561f0 0x7f1da415e770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1da4156750 con 0x7f1da41561f0 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1da41565d0 con 0x7f1da40aedc0 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1da406d890 con 0x7f1da40cc9d0 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da41561f0 0x7f1da415e770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da41561f0 0x7f1da415e770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50924/0 (socket says 172.21.15.86:50924) 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.439+0000 7f1da9000640 1 -- 172.21.15.86:0/1998314929 learned_addr learned my addr 172.21.15.86:0/1998314929 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da3400640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1da40cc9d0 0x7f1da4152600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da9000640 1 -- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1da40cc9d0 msgr2=0x7f1da4152600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da9000640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1da40cc9d0 0x7f1da4152600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da9000640 1 -- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1da40aedc0 msgr2=0x7f1da41587d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da3e00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1da40aedc0 0x7f1da41587d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da9000640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1da40aedc0 0x7f1da41587d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da9000640 1 -- 172.21.15.86:0/1998314929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1da4157730 con 0x7f1da41561f0 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da3e00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1da40aedc0 0x7f1da41587d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da9000640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da41561f0 0x7f1da415e770 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f1d9c036040 tx=0x7f1d9c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1d9be00640 1 -- 172.21.15.86:0/1998314929 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d9c043020 con 0x7f1da41561f0 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1d9be00640 1 -- 172.21.15.86:0/1998314929 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1d9c03e450 con 0x7f1da41561f0 2024-03-06T07:29:31.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1da4157a10 con 0x7f1da41561f0 2024-03-06T07:29:31.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.440+0000 7f1d9be00640 1 -- 172.21.15.86:0/1998314929 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d9c03e820 con 0x7f1da41561f0 2024-03-06T07:29:31.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.441+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1da41582b0 con 0x7f1da41561f0 2024-03-06T07:29:31.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.442+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1da404f330 con 0x7f1da41561f0 2024-03-06T07:29:31.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.442+0000 7f1d9be00640 1 -- 172.21.15.86:0/1998314929 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1d9c005d10 con 0x7f1da41561f0 2024-03-06T07:29:31.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.443+0000 7f1d9be00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1d8406b500 0x7f1d8406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.443+0000 7f1d9be00640 1 -- 172.21.15.86:0/1998314929 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1d9c0ac810 con 0x7f1da41561f0 2024-03-06T07:29:31.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.443+0000 7f1da3e00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1d8406b500 0x7f1d8406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.444+0000 7f1da3e00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1d8406b500 0x7f1d8406d9c0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f1d9400a830 tx=0x7f1d94008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:31.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.447+0000 7f1d9be00640 1 -- 172.21.15.86:0/1998314929 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1d9c04d020 con 0x7f1da41561f0 2024-03-06T07:29:31.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.637+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1da406ce70 con 0x7f1da41561f0 2024-03-06T07:29:31.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.640+0000 7f1d9be00640 1 -- 172.21.15.86:0/1998314929 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v47) v1 ==== 74+0+16625 (secure 0 0 0) 0x7f1d9c07b5f0 con 0x7f1da41561f0 2024-03-06T07:29:31.643 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:31.643 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":47,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:01.377099+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:28:57.609038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"46","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-06T07:28:58.840885+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1d8406b500 msgr2=0x7f1d8406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1d8406b500 0x7f1d8406d9c0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f1d9400a830 tx=0x7f1d94008040 comp rx=0 tx=0).stop 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da41561f0 msgr2=0x7f1da415e770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da41561f0 0x7f1da415e770 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f1d9c036040 tx=0x7f1d9c004a40 comp rx=0 tx=0).stop 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 shutdown_connections 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1d8406b500 0x7f1d8406d9c0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1da41561f0 0x7f1da415e770 unknown :-1 s=CLOSED pgs=585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1da40cc9d0 0x7f1da4152600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 --2- 172.21.15.86:0/1998314929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1da40aedc0 0x7f1da41587d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 >> 172.21.15.86:0/1998314929 conn(0x7f1da406afc0 msgr2=0x7f1da4071ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:31.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 shutdown_connections 2024-03-06T07:29:31.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.642+0000 7f1da9a00640 1 -- 172.21.15.86:0/1998314929 wait complete. 2024-03-06T07:29:31.661 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:29:31.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.851+0000 7fc1b6400640 1 Processor -- start 2024-03-06T07:29:31.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.851+0000 7fc1b6400640 1 -- start start 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.852+0000 7fc1b6400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc1b00c4930 0x7fc1b00c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1b6400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00c5170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1b6400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc1b0145b30 con 0x7fc1b00c56b0 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1b6400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc1b0142ce0 con 0x7fc1b00c4930 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1b6400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc1b00cbca0 con 0x7fc1b00c4d10 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1af400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc1b00c4930 0x7fc1b00c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1af400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc1b00c4930 0x7fc1b00c6460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50940/0 (socket says 172.21.15.86:50940) 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1af400640 1 -- 172.21.15.86:0/3663267742 learned_addr learned my addr 172.21.15.86:0/3663267742 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:31.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1a6a00640 1 --2- 172.21.15.86:0/3663267742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00c5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1af400640 1 -- 172.21.15.86:0/3663267742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 msgr2=0x7fc1b00c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:31.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1af400640 1 --2- 172.21.15.86:0/3663267742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00c5170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1af400640 1 -- 172.21.15.86:0/3663267742 >> v1:172.21.15.194:6789/0 conn(0x7fc1b00c56b0 legacy=0x7fc1b006d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:31.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1af400640 1 -- 172.21.15.86:0/3663267742 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc1b00cbf80 con 0x7fc1b00c4930 2024-03-06T07:29:31.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.853+0000 7fc1a6a00640 1 --2- 172.21.15.86:0/3663267742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:31.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.855+0000 7fc1af400640 1 --2- 172.21.15.86:0/3663267742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c4930 0x7fc1b00c6460 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7fc1a0009870 tx=0x7fc1a002b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f0f34f66a7fa306b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:31.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.855+0000 7fc1aea00640 1 -- 172.21.15.86:0/3663267742 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc1a00317c0 con 0x7fc1b00c4930 2024-03-06T07:29:31.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.855+0000 7fc1aea00640 1 -- 172.21.15.86:0/3663267742 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc1a0037070 con 0x7fc1b00c4930 2024-03-06T07:29:31.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.855+0000 7fc1aea00640 1 -- 172.21.15.86:0/3663267742 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc1a0031b80 con 0x7fc1b00c4930 2024-03-06T07:29:31.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.855+0000 7fc1b6400640 1 -- 172.21.15.86:0/3663267742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c4930 msgr2=0x7fc1b00c6460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.855+0000 7fc1b6400640 1 --2- 172.21.15.86:0/3663267742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c4930 0x7fc1b00c6460 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7fc1a0009870 tx=0x7fc1a002b860 comp rx=0 tx=0).stop 2024-03-06T07:29:31.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.856+0000 7fc1b6400640 1 -- 172.21.15.86:0/3663267742 shutdown_connections 2024-03-06T07:29:31.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.856+0000 7fc1b6400640 1 --2- 172.21.15.86:0/3663267742 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.856+0000 7fc1b6400640 1 --2- 172.21.15.86:0/3663267742 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c4930 0x7fc1b00c6460 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.856+0000 7fc1b6400640 1 -- 172.21.15.86:0/3663267742 >> 172.21.15.86:0/3663267742 conn(0x7fc1b00cc4f0 msgr2=0x7fc1b00cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:31.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.856+0000 7fc1b6400640 1 -- 172.21.15.86:0/3663267742 shutdown_connections 2024-03-06T07:29:31.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.856+0000 7fc1b6400640 1 -- 172.21.15.86:0/3663267742 wait complete. 2024-03-06T07:29:31.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.857+0000 7fc1b6400640 1 Processor -- start 2024-03-06T07:29:31.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.857+0000 7fc1b6400640 1 -- start start 2024-03-06T07:29:31.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.858+0000 7fc1b6400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc1b00c4930 0x7fc1b00aef40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.858+0000 7fc1b6400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00b0580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.858+0000 7fc1b6400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c56b0 0x7fc1b013e480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.858+0000 7fc1b6400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1b00cc260 con 0x7fc1b00c56b0 2024-03-06T07:29:31.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.858+0000 7fc1b6400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1b00ca3b0 con 0x7fc1b00c4930 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.858+0000 7fc1b6400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc1b00c5b40 con 0x7fc1b00c4d10 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c56b0 0x7fc1b013e480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c56b0 0x7fc1b013e480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50952/0 (socket says 172.21.15.86:50952) 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 -- 172.21.15.86:0/2594489569 learned_addr learned my addr 172.21.15.86:0/2594489569 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1af400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc1b00c4930 0x7fc1b00aef40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1a6a00640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00b0580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 -- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 msgr2=0x7fc1b00b0580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00b0580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 -- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc1b00c4930 msgr2=0x7fc1b00aef40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc1b00c4930 0x7fc1b00aef40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 -- 172.21.15.86:0/2594489569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc1b013e9c0 con 0x7fc1b00c56b0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.859+0000 7fc1afe00640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c56b0 0x7fc1b013e480 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7fc1a800c7b0 tx=0x7fc1a800cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.860+0000 7fc1a7e00640 1 -- 172.21.15.86:0/2594489569 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc1a800ce60 con 0x7fc1b00c56b0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.860+0000 7fc1a7e00640 1 -- 172.21.15.86:0/2594489569 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc1a8004b10 con 0x7fc1b00c56b0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.860+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc1b00b0cd0 con 0x7fc1b00c56b0 2024-03-06T07:29:31.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.860+0000 7fc1a7e00640 1 -- 172.21.15.86:0/2594489569 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc1a800a850 con 0x7fc1b00c56b0 2024-03-06T07:29:31.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.860+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc1b00b1290 con 0x7fc1b00c56b0 2024-03-06T07:29:31.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.861+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc1b004f330 con 0x7fc1b00c56b0 2024-03-06T07:29:31.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.862+0000 7fc1a7e00640 1 -- 172.21.15.86:0/2594489569 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc1a800a9f0 con 0x7fc1b00c56b0 2024-03-06T07:29:31.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.862+0000 7fc1a7e00640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc18c06b4d0 0x7fc18c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:31.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.863+0000 7fc1a7e00640 1 -- 172.21.15.86:0/2594489569 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc1a808a3b0 con 0x7fc1b00c56b0 2024-03-06T07:29:31.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.863+0000 7fc1af400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc18c06b4d0 0x7fc18c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:31.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.863+0000 7fc1af400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc18c06b4d0 0x7fc18c06d990 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fc1a0036000 tx=0x7fc1a003d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:31.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:31.867+0000 7fc1a7e00640 1 -- 172.21.15.86:0/2594489569 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc1a8020070 con 0x7fc1b00c56b0 2024-03-06T07:29:32.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.101+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7fc1b01443b0 con 0x7fc1b00c56b0 2024-03-06T07:29:32.104 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:29:32.101+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:29:32.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.105+0000 7fc1a7e00640 1 -- 172.21.15.86:0/2594489569 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v26) v1 ==== 161+0+0 (secure 0 0 0) 0x7fc1b013e9c0 con 0x7fc1b00c56b0 2024-03-06T07:29:32.107 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:29:32.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.108+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc18c06b4d0 msgr2=0x7fc18c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:32.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.108+0000 7fc1b6400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc18c06b4d0 0x7fc18c06d990 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fc1a0036000 tx=0x7fc1a003d040 comp rx=0 tx=0).stop 2024-03-06T07:29:32.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.108+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c56b0 msgr2=0x7fc1b013e480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.108+0000 7fc1b6400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c56b0 0x7fc1b013e480 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7fc1a800c7b0 tx=0x7fc1a800cc70 comp rx=0 tx=0).stop 2024-03-06T07:29:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 shutdown_connections 2024-03-06T07:29:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc18c06b4d0 0x7fc18c06d990 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1b00c56b0 0x7fc1b013e480 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1b00c4d10 0x7fc1b00b0580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 --2- 172.21.15.86:0/2594489569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc1b00c4930 0x7fc1b00aef40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 >> 172.21.15.86:0/2594489569 conn(0x7fc1b00cc4f0 msgr2=0x7fc1b01448d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:32.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 shutdown_connections 2024-03-06T07:29:32.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.109+0000 7fc1b6400640 1 -- 172.21.15.86:0/2594489569 wait complete. 2024-03-06T07:29:32.130 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:29:32.239 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:29:32.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.325+0000 7ff1ca400640 1 Processor -- start 2024-03-06T07:29:32.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.325+0000 7ff1ca400640 1 -- start start 2024-03-06T07:29:32.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1ca400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff1c40cda90 0x7ff1c40727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1ca400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff1c40719e0 con 0x7ff1c40b3650 2024-03-06T07:29:32.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1ca400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff1c4071cc0 con 0x7ff1c40b3df0 2024-03-06T07:29:32.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1ca400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff1c4072310 con 0x7ff1c40cda90 2024-03-06T07:29:32.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1c2a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff1c40b3650 0x7ff1c406c990 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47696/0 (socket says 172.21.15.86:47696) 2024-03-06T07:29:32.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1c2a00640 1 -- 172.21.15.86:0/374020262 learned_addr learned my addr 172.21.15.86:0/374020262 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:32.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3310636350 0 0) 0x7ff1c40719e0 con 0x7ff1c40b3650 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1c3e00640 1 --2- 172.21.15.86:0/374020262 >> v2:172.21.15.194:3300/0 conn(0x7ff1c40cda90 0x7ff1c40727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.327+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff1a8005730 con 0x7ff1c40b3650 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3984026823 0 0) 0x7ff1a8005730 con 0x7ff1c40b3650 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 >> v2:172.21.15.194:3300/0 conn(0x7ff1c40cda90 msgr2=0x7ff1c40727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 --2- 172.21.15.86:0/374020262 >> v2:172.21.15.194:3300/0 conn(0x7ff1c40cda90 0x7ff1c40727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 >> v1:172.21.15.194:6789/0 conn(0x7ff1c40b3df0 legacy=0x7ff1c40b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1c4071fa0 con 0x7ff1c40b3650 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff1b0001670 con 0x7ff1c40b3650 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 >> v1:172.21.15.86:6789/0 conn(0x7ff1c40b3650 legacy=0x7ff1c406c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1a8005bc0 0x7ff1a8005fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8006a10 0x7ff1a8006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1a80073d0 0x7ff1a800dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1c4072310 con 0x7ff1a8005bc0 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1a800e270 con 0x7ff1a80073d0 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff1a800e470 con 0x7ff1a8006a10 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.328+0000 7ff1c3400640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1a8005bc0 0x7ff1a8005fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.329+0000 7ff1c3400640 1 -- 172.21.15.86:0/374020262 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8006a10 msgr2=0x7ff1a8006e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.329+0000 7ff1c3400640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8006a10 0x7ff1a8006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.329+0000 7ff1c3e00640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1a80073d0 0x7ff1a800dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.329+0000 7ff1c3400640 1 -- 172.21.15.86:0/374020262 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1a80073d0 msgr2=0x7ff1a800dbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.329+0000 7ff1c3400640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1a80073d0 0x7ff1a800dbd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.329+0000 7ff1c3400640 1 -- 172.21.15.86:0/374020262 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1a8005730 con 0x7ff1a8005bc0 2024-03-06T07:29:32.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.329+0000 7ff1c3e00640 1 -- 172.21.15.86:0/374020262 reap_dead start 2024-03-06T07:29:32.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.330+0000 7ff1c3400640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1a8005bc0 0x7ff1a8005fe0 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7ff1b4001e50 tx=0x7ff1a8007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2b4e5ad99d243f17 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:32.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.330+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1b4001a70 con 0x7ff1a8005bc0 2024-03-06T07:29:32.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.330+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff1b4003b70 con 0x7ff1a8005bc0 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.330+0000 7ff1c2000640 1 -- 172.21.15.86:0/374020262 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1b40050d0 con 0x7ff1a8005bc0 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.330+0000 7ff1ca400640 1 -- 172.21.15.86:0/374020262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1a8005bc0 msgr2=0x7ff1a8005fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.330+0000 7ff1ca400640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1a8005bc0 0x7ff1a8005fe0 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7ff1b4001e50 tx=0x7ff1a8007a40 comp rx=0 tx=0).stop 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.331+0000 7ff1ca400640 1 -- 172.21.15.86:0/374020262 shutdown_connections 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.331+0000 7ff1ca400640 1 --2- 172.21.15.86:0/374020262 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1a8005bc0 0x7ff1a8005fe0 unknown :-1 s=CLOSED pgs=590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.331+0000 7ff1ca400640 1 -- 172.21.15.86:0/374020262 >> 172.21.15.86:0/374020262 conn(0x7ff1c40b1940 msgr2=0x7ff1c40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.331+0000 7ff1ca400640 1 -- 172.21.15.86:0/374020262 shutdown_connections 2024-03-06T07:29:32.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.331+0000 7ff1ca400640 1 -- 172.21.15.86:0/374020262 wait complete. 2024-03-06T07:29:32.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.332+0000 7ff1ca400640 1 Processor -- start 2024-03-06T07:29:32.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.333+0000 7ff1ca400640 1 -- start start 2024-03-06T07:29:32.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.333+0000 7ff1ca400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8005bc0 0x7ff1c415dfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.333+0000 7ff1ca400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1c41502b0 0x7ff1c40caa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.333+0000 7ff1ca400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1c40caf50 0x7ff1c40cb3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.333+0000 7ff1ca400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1c415b3c0 con 0x7ff1c41502b0 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.333+0000 7ff1ca400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1c415b240 con 0x7ff1c40caf50 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.333+0000 7ff1ca400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff1c40cf510 con 0x7ff1a8005bc0 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1c41502b0 0x7ff1c40caa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1c41502b0 0x7ff1c40caa10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50970/0 (socket says 172.21.15.86:50970) 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 -- 172.21.15.86:0/3861077107 learned_addr learned my addr 172.21.15.86:0/3861077107 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c3400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8005bc0 0x7ff1c415dfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c3e00640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1c40caf50 0x7ff1c40cb3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 -- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8005bc0 msgr2=0x7ff1c415dfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8005bc0 0x7ff1c415dfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 -- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1c40caf50 msgr2=0x7ff1c40cb3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:32.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1c40caf50 0x7ff1c40cb3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:32.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 -- 172.21.15.86:0/3861077107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1c40ca2d0 con 0x7ff1c41502b0 2024-03-06T07:29:32.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.334+0000 7ff1c2a00640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1c41502b0 0x7ff1c40caa10 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7ff1b000c730 tx=0x7ff1b000cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:32.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.335+0000 7ff1bb400640 1 -- 172.21.15.86:0/3861077107 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1b0018070 con 0x7ff1c41502b0 2024-03-06T07:29:32.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.335+0000 7ff1bb400640 1 -- 172.21.15.86:0/3861077107 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff1b00074e0 con 0x7ff1c41502b0 2024-03-06T07:29:32.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.335+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff1c409c950 con 0x7ff1c41502b0 2024-03-06T07:29:32.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.335+0000 7ff1bb400640 1 -- 172.21.15.86:0/3861077107 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1b0004a20 con 0x7ff1c41502b0 2024-03-06T07:29:32.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.335+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1c409cf10 con 0x7ff1c41502b0 2024-03-06T07:29:32.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.336+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff1c404f330 con 0x7ff1c41502b0 2024-03-06T07:29:32.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.337+0000 7ff1bb400640 1 -- 172.21.15.86:0/3861077107 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff1b001b070 con 0x7ff1c41502b0 2024-03-06T07:29:32.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.338+0000 7ff1bb400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1a406b4d0 0x7ff1a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:32.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.338+0000 7ff1bb400640 1 -- 172.21.15.86:0/3861077107 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(48..48 src has 1..48) v4 ==== 5795+0+0 (secure 0 0 0) 0x7ff1b0089740 con 0x7ff1c41502b0 2024-03-06T07:29:32.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.338+0000 7ff1c3400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1a406b4d0 0x7ff1a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:32.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.339+0000 7ff1c3400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1a406b4d0 0x7ff1a406d990 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7ff1b4034000 tx=0x7ff1b4009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:32.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.342+0000 7ff1bb400640 1 -- 172.21.15.86:0/3861077107 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff1b0058550 con 0x7ff1c41502b0 2024-03-06T07:29:32.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:32.582+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7ff1c40cc0e0 con 0x7ff1c41502b0 2024-03-06T07:29:33.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.117+0000 7ff1bb400640 1 -- 172.21.15.86:0/3861077107 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v27) v1 ==== 104+0+0 (secure 0 0 0) 0x7ff1b0057e50 con 0x7ff1c41502b0 2024-03-06T07:29:33.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1a406b4d0 msgr2=0x7ff1a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:33.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1a406b4d0 0x7ff1a406d990 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7ff1b4034000 tx=0x7ff1b4009030 comp rx=0 tx=0).stop 2024-03-06T07:29:33.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1c41502b0 msgr2=0x7ff1c40caa10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1c41502b0 0x7ff1c40caa10 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7ff1b000c730 tx=0x7ff1b000cbf0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 shutdown_connections 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff1a406b4d0 0x7ff1a406d990 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff1c40caf50 0x7ff1c40cb3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1c41502b0 0x7ff1c40caa10 unknown :-1 s=CLOSED pgs=591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 --2- 172.21.15.86:0/3861077107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff1a8005bc0 0x7ff1c415dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.120+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 >> 172.21.15.86:0/3861077107 conn(0x7ff1c40b1940 msgr2=0x7ff1c40727a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.121+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 shutdown_connections 2024-03-06T07:29:33.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.121+0000 7ff1ca400640 1 -- 172.21.15.86:0/3861077107 wait complete. 2024-03-06T07:29:33.145 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:29:33.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.358+0000 7fe4c5000640 1 Processor -- start 2024-03-06T07:29:33.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.359+0000 7fe4c5000640 1 -- start start 2024-03-06T07:29:33.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.360+0000 7fe4c5000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe4c015e200 0x7fe4c00aed30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:33.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.360+0000 7fe4c5000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c00cc4d0 0x7fe4c00cc930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:33.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.360+0000 7fe4c5000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4c00cd550 con 0x7fe4c00ad860 2024-03-06T07:29:33.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.360+0000 7fe4c5000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe4c00cce70 con 0x7fe4c015e200 2024-03-06T07:29:33.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.360+0000 7fe4c5000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4c0162b50 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4be800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe4c00ad860 0x7fe4c00adcf0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47708/0 (socket says 172.21.15.86:47708) 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4bde00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe4c015e200 0x7fe4c00aed30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4be800640 1 -- 172.21.15.86:0/3744589803 learned_addr learned my addr 172.21.15.86:0/3744589803 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4b5400640 1 --2- 172.21.15.86:0/3744589803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c00cc4d0 0x7fe4c00cc930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4bd400640 1 -- 172.21.15.86:0/3744589803 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2365670875 0 0) 0x7fe4c00cd550 con 0x7fe4c00ad860 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4bde00640 1 -- 172.21.15.86:0/3744589803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c00cc4d0 msgr2=0x7fe4c00cc930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4bde00640 1 --2- 172.21.15.86:0/3744589803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c00cc4d0 0x7fe4c00cc930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4bde00640 1 -- 172.21.15.86:0/3744589803 >> v1:172.21.15.86:6789/0 conn(0x7fe4c00ad860 legacy=0x7fe4c00adcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:33.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.361+0000 7fe4bde00640 1 -- 172.21.15.86:0/3744589803 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4c0161c60 con 0x7fe4c015e200 2024-03-06T07:29:33.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.362+0000 7fe4bde00640 1 --2- 172.21.15.86:0/3744589803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c015e200 0x7fe4c00aed30 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7fe4b00097d0 tx=0x7fe4b002b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2b4759ed85a536d8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:33.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4bd400640 1 -- 172.21.15.86:0/3744589803 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4b0035020 con 0x7fe4c015e200 2024-03-06T07:29:33.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4bd400640 1 -- 172.21.15.86:0/3744589803 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4b0031800 con 0x7fe4c015e200 2024-03-06T07:29:33.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4bd400640 1 -- 172.21.15.86:0/3744589803 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4b0031bd0 con 0x7fe4c015e200 2024-03-06T07:29:33.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 -- 172.21.15.86:0/3744589803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c015e200 msgr2=0x7fe4c00aed30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:33.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3744589803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c015e200 0x7fe4c00aed30 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7fe4b00097d0 tx=0x7fe4b002b8a0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 -- 172.21.15.86:0/3744589803 shutdown_connections 2024-03-06T07:29:33.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3744589803 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c00cc4d0 0x7fe4c00cc930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3744589803 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c015e200 0x7fe4c00aed30 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 -- 172.21.15.86:0/3744589803 >> 172.21.15.86:0/3744589803 conn(0x7fe4c00cfa80 msgr2=0x7fe4c00cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:33.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 -- 172.21.15.86:0/3744589803 shutdown_connections 2024-03-06T07:29:33.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.363+0000 7fe4c5000640 1 -- 172.21.15.86:0/3744589803 wait complete. 2024-03-06T07:29:33.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.365+0000 7fe4c5000640 1 Processor -- start 2024-03-06T07:29:33.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.365+0000 7fe4c5000640 1 -- start start 2024-03-06T07:29:33.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.366+0000 7fe4c5000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4c00ad860 0x7fe4c00b3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:33.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.366+0000 7fe4c5000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c00cc4d0 0x7fe4c01f35f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.366+0000 7fe4c5000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c015e200 0x7fe4c01f3b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.366+0000 7fe4c5000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4c0162500 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.366+0000 7fe4c5000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4c0162220 con 0x7fe4c00ad860 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.366+0000 7fe4c5000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4c01627e0 con 0x7fe4c015e200 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c00cc4d0 0x7fe4c01f35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c00cc4d0 0x7fe4c01f35f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50986/0 (socket says 172.21.15.86:50986) 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 -- 172.21.15.86:0/3209126221 learned_addr learned my addr 172.21.15.86:0/3209126221 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4be800640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c015e200 0x7fe4c01f3b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4bde00640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4c00ad860 0x7fe4c00b3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 -- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c015e200 msgr2=0x7fe4c01f3b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c015e200 0x7fe4c01f3b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 -- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4c00ad860 msgr2=0x7fe4c00b3660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:33.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4c00ad860 0x7fe4c00b3660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:33.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.367+0000 7fe4b5400640 1 -- 172.21.15.86:0/3209126221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4c00b23c0 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.368+0000 7fe4b5400640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c00cc4d0 0x7fe4c01f35f0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7fe4a800d800 tx=0x7fe4a800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:33.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.368+0000 7fe4b6a00640 1 -- 172.21.15.86:0/3209126221 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4a800b6c0 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.368+0000 7fe4b6a00640 1 -- 172.21.15.86:0/3209126221 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4a8019070 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.368+0000 7fe4b6a00640 1 -- 172.21.15.86:0/3209126221 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4a800ba80 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.368+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4c00b26a0 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.368+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4c00b2c60 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.369+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe4c004f3a0 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.370+0000 7fe4b6a00640 1 -- 172.21.15.86:0/3209126221 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe4a800bc20 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.371+0000 7fe4b6a00640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe49c06b4d0 0x7fe49c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:33.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.371+0000 7fe4b6a00640 1 -- 172.21.15.86:0/3209126221 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(48..48 src has 1..48) v4 ==== 5795+0+0 (secure 0 0 0) 0x7fe4a808a7b0 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.371+0000 7fe4bde00640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe49c06b4d0 0x7fe49c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:33.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.372+0000 7fe4bde00640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe49c06b4d0 0x7fe49c06d990 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fe4b0036000 tx=0x7fe4b0008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:33.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.376+0000 7fe4b6a00640 1 -- 172.21.15.86:0/3209126221 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe4a8059530 con 0x7fe4c00cc4d0 2024-03-06T07:29:33.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:33.571+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fe4c01f4070 con 0x7fe4c00cc4d0 2024-03-06T07:29:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.120+0000 7fe4b6a00640 1 -- 172.21.15.86:0/3209126221 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v49) v1 ==== 181+0+0 (secure 0 0 0) 0x7fe4a8058e30 con 0x7fe4c00cc4d0 2024-03-06T07:29:34.123 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe49c06b4d0 msgr2=0x7fe49c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe49c06b4d0 0x7fe49c06d990 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fe4b0036000 tx=0x7fe4b0008030 comp rx=0 tx=0).stop 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c00cc4d0 msgr2=0x7fe4c01f35f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c00cc4d0 0x7fe4c01f35f0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7fe4a800d800 tx=0x7fe4a800dcc0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 shutdown_connections 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe49c06b4d0 0x7fe49c06d990 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe4c015e200 0x7fe4c01f3b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe4c00cc4d0 0x7fe4c01f35f0 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 --2- 172.21.15.86:0/3209126221 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe4c00ad860 0x7fe4c00b3660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 >> 172.21.15.86:0/3209126221 conn(0x7fe4c00cfa80 msgr2=0x7fe4c015d430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:34.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.124+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 shutdown_connections 2024-03-06T07:29:34.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.126+0000 7fe4c5000640 1 -- 172.21.15.86:0/3209126221 wait complete. 2024-03-06T07:29:34.150 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:29:34.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.355+0000 7f44f6800640 1 Processor -- start 2024-03-06T07:29:34.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.355+0000 7f44f6800640 1 -- start start 2024-03-06T07:29:34.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.356+0000 7f44f6800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f44f015e100 0x7f44f00d4a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:34.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.356+0000 7f44f6800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44f015e4e0 0x7f44f00b0ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:34.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.356+0000 7f44f6800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f44f00d12c0 con 0x7f44f00b0ff0 2024-03-06T07:29:34.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.356+0000 7f44f6800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f44f00ceb00 con 0x7f44f015e4e0 2024-03-06T07:29:34.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.356+0000 7f44f6800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f44f00d1e00 con 0x7f44f015e100 2024-03-06T07:29:34.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44e6a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44f015e4e0 0x7f44f00b0ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:34.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44e6a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44f015e4e0 0x7f44f00b0ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50994/0 (socket says 172.21.15.86:50994) 2024-03-06T07:29:34.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44e6a00640 1 -- 172.21.15.86:0/2870138581 learned_addr learned my addr 172.21.15.86:0/2870138581 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:34.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44ef400640 1 --2- 172.21.15.86:0/2870138581 >> v2:172.21.15.194:3300/0 conn(0x7f44f015e100 0x7f44f00d4a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:34.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44e6a00640 1 -- 172.21.15.86:0/2870138581 >> v2:172.21.15.194:3300/0 conn(0x7f44f015e100 msgr2=0x7f44f00d4a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:34.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44e6a00640 1 --2- 172.21.15.86:0/2870138581 >> v2:172.21.15.194:3300/0 conn(0x7f44f015e100 0x7f44f00d4a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44e6a00640 1 -- 172.21.15.86:0/2870138581 >> v1:172.21.15.194:6789/0 conn(0x7f44f00b0ff0 legacy=0x7f44f00cf580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:34.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.357+0000 7f44e6a00640 1 -- 172.21.15.86:0/2870138581 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44f00d20a0 con 0x7f44f015e4e0 2024-03-06T07:29:34.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.359+0000 7f44e6a00640 1 --2- 172.21.15.86:0/2870138581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e4e0 0x7f44f00b0ab0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f44f006df30 tx=0x7f44d802b7f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=637b7c8a726e8dd3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:34.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.359+0000 7f44eea00640 1 -- 172.21.15.86:0/2870138581 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44d8038070 con 0x7f44f015e4e0 2024-03-06T07:29:34.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.359+0000 7f44eea00640 1 -- 172.21.15.86:0/2870138581 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f44d802f050 con 0x7f44f015e4e0 2024-03-06T07:29:34.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.359+0000 7f44eea00640 1 -- 172.21.15.86:0/2870138581 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44d8031970 con 0x7f44f015e4e0 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 -- 172.21.15.86:0/2870138581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e4e0 msgr2=0x7f44f00b0ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 --2- 172.21.15.86:0/2870138581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e4e0 0x7f44f00b0ab0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f44f006df30 tx=0x7f44d802b7f0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 -- 172.21.15.86:0/2870138581 shutdown_connections 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 --2- 172.21.15.86:0/2870138581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e4e0 0x7f44f00b0ab0 unknown :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 --2- 172.21.15.86:0/2870138581 >> v2:172.21.15.194:3300/0 conn(0x7f44f015e100 0x7f44f00d4a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 -- 172.21.15.86:0/2870138581 >> 172.21.15.86:0/2870138581 conn(0x7f44f00d3ef0 msgr2=0x7f44f00d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 -- 172.21.15.86:0/2870138581 shutdown_connections 2024-03-06T07:29:34.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.360+0000 7f44f6800640 1 -- 172.21.15.86:0/2870138581 wait complete. 2024-03-06T07:29:34.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.362+0000 7f44f6800640 1 Processor -- start 2024-03-06T07:29:34.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.362+0000 7f44f6800640 1 -- start start 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44f6800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44f00b0ff0 0x7f44f0150d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44f6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e100 0x7f44f014da40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44f6800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44f015e4e0 0x7f44f006bff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44f6800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44f00cfc90 con 0x7f44f015e100 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44f6800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44f00cf060 con 0x7f44f015e4e0 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44f6800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f44f00cfe10 con 0x7f44f00b0ff0 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44e6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e100 0x7f44f014da40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:34.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44e6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e100 0x7f44f014da40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51000/0 (socket says 172.21.15.86:51000) 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.363+0000 7f44e6a00640 1 -- 172.21.15.86:0/452045020 learned_addr learned my addr 172.21.15.86:0/452045020 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44ef400640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44f00b0ff0 0x7f44f0150d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e6a00640 1 -- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44f00b0ff0 msgr2=0x7f44f0150d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e6a00640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44f00b0ff0 0x7f44f0150d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e6a00640 1 -- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44f015e4e0 msgr2=0x7f44f006bff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e6a00640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44f015e4e0 0x7f44f006bff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e6a00640 1 -- 172.21.15.86:0/452045020 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44f00ca790 con 0x7f44f015e100 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44ef400640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44f00b0ff0 0x7f44f0150d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e6a00640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e100 0x7f44f014da40 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7f44d8000c00 tx=0x7f44d80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e7e00640 1 -- 172.21.15.86:0/452045020 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44d8042070 con 0x7f44f015e100 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e7e00640 1 -- 172.21.15.86:0/452045020 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f44d803d870 con 0x7f44f015e100 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44e7e00640 1 -- 172.21.15.86:0/452045020 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44d8038040 con 0x7f44f015e100 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.364+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44f00b53d0 con 0x7f44f015e100 2024-03-06T07:29:34.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.365+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44f00b5990 con 0x7f44f015e100 2024-03-06T07:29:34.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.366+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f44f004f330 con 0x7f44f015e100 2024-03-06T07:29:34.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.367+0000 7f44e7e00640 1 -- 172.21.15.86:0/452045020 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f44d8045070 con 0x7f44f015e100 2024-03-06T07:29:34.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.367+0000 7f44e7e00640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44cc06b4d0 0x7f44cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:34.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.368+0000 7f44e7e00640 1 -- 172.21.15.86:0/452045020 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(49..49 src has 1..49) v4 ==== 5357+0+0 (secure 0 0 0) 0x7f44d80abf80 con 0x7f44f015e100 2024-03-06T07:29:34.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.368+0000 7f44ef400640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44cc06b4d0 0x7f44cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:34.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.368+0000 7f44ef400640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44cc06b4d0 0x7f44cc06d990 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f44e0006fd0 tx=0x7f44e0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:34.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.372+0000 7f44e7e00640 1 -- 172.21.15.86:0/452045020 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f44f00d12c0 con 0x7f44f015e100 2024-03-06T07:29:34.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:34.568+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f44f0073070 con 0x7f44f015e100 2024-03-06T07:29:35.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.133+0000 7f44e7e00640 1 -- 172.21.15.86:0/452045020 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v50) v1 ==== 169+0+0 (secure 0 0 0) 0x7f44d807b110 con 0x7f44f015e100 2024-03-06T07:29:35.136 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:29:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.138+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44cc06b4d0 msgr2=0x7f44cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.138+0000 7f44f6800640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44cc06b4d0 0x7f44cc06d990 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f44e0006fd0 tx=0x7f44e0008040 comp rx=0 tx=0).stop 2024-03-06T07:29:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.138+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e100 msgr2=0x7f44f014da40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.138+0000 7f44f6800640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e100 0x7f44f014da40 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7f44d8000c00 tx=0x7f44d80046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.139+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 shutdown_connections 2024-03-06T07:29:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.139+0000 7f44f6800640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f44cc06b4d0 0x7f44cc06d990 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.139+0000 7f44f6800640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44f015e4e0 0x7f44f006bff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.139+0000 7f44f6800640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44f015e100 0x7f44f014da40 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.139+0000 7f44f6800640 1 --2- 172.21.15.86:0/452045020 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44f00b0ff0 0x7f44f0150d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.139+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 >> 172.21.15.86:0/452045020 conn(0x7f44f00d3ef0 msgr2=0x7f44f015ebe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.143+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 shutdown_connections 2024-03-06T07:29:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.143+0000 7f44f6800640 1 -- 172.21.15.86:0/452045020 wait complete. 2024-03-06T07:29:35.170 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-06T07:29:35.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.358+0000 7fadbd400640 1 Processor -- start 2024-03-06T07:29:35.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.358+0000 7fadbd400640 1 -- start start 2024-03-06T07:29:35.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.360+0000 7fadbd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 0x7fadb8070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.360+0000 7fadbd400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fadb80b46c0 con 0x7fadb80b3f70 2024-03-06T07:29:35.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.360+0000 7fadbd400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fadb80b49a0 con 0x7fadb815fc10 2024-03-06T07:29:35.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.360+0000 7fadbd400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fadb8072c90 con 0x7fadb8160700 2024-03-06T07:29:35.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.360+0000 7fadafe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fadb80b3f70 0x7fadb80ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47722/0 (socket says 172.21.15.86:47722) 2024-03-06T07:29:35.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.360+0000 7fadafe00640 1 -- 172.21.15.86:0/483652032 learned_addr learned my addr 172.21.15.86:0/483652032 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:35.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb7200640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 0x7fadb8070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2328043697 0 0) 0x7fadb80b46c0 con 0x7fadb80b3f70 2024-03-06T07:29:35.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fad9c003520 con 0x7fadb80b3f70 2024-03-06T07:29:35.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3191764601 0 0) 0x7fad9c003520 con 0x7fadb80b3f70 2024-03-06T07:29:35.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 msgr2=0x7fadb8070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:35.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 0x7fadb8070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 >> v1:172.21.15.194:6789/0 conn(0x7fadb815fc10 legacy=0x7fadb815fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fadb809dd80 con 0x7fadb80b3f70 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fada0001670 con 0x7fadb80b3f70 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 >> v1:172.21.15.86:6789/0 conn(0x7fadb80b3f70 legacy=0x7fadb80ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fad9c0039e0 0x7fad9c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 0x7fad9c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fad9c004d40 0x7fad9c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fadb8072c90 con 0x7fad9c004d40 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad9c00bba0 con 0x7fad9c0039e0 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fad9c00bda0 con 0x7fadb8160700 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.361+0000 7fadb7200640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fad9c004d40 0x7fad9c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.362+0000 7fadafe00640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 0x7fad9c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.362+0000 7fadb7200640 1 -- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 msgr2=0x7fad9c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.362+0000 7fadb7200640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8160700 0x7fad9c004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.362+0000 7fadb7200640 1 -- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fad9c0039e0 msgr2=0x7fad9c003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:35.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.362+0000 7fadb7200640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fad9c0039e0 0x7fad9c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.362+0000 7fadb7200640 1 -- 172.21.15.86:0/483652032 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fad9c003520 con 0x7fad9c004d40 2024-03-06T07:29:35.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.363+0000 7fadb7200640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fad9c004d40 0x7fad9c0051c0 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7fadb0007360 tx=0x7fad9c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=894849cbacaacfaf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:35.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.363+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fadb0039070 con 0x7fad9c004d40 2024-03-06T07:29:35.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.363+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fadb002ccd0 con 0x7fad9c004d40 2024-03-06T07:29:35.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.363+0000 7fadb5e00640 1 -- 172.21.15.86:0/483652032 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fadb0009be0 con 0x7fad9c004d40 2024-03-06T07:29:35.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.364+0000 7fadbd400640 1 -- 172.21.15.86:0/483652032 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fad9c004d40 msgr2=0x7fad9c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.364+0000 7fadbd400640 1 --2- 172.21.15.86:0/483652032 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fad9c004d40 0x7fad9c0051c0 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7fadb0007360 tx=0x7fad9c005830 comp rx=0 tx=0).stop 2024-03-06T07:29:35.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.364+0000 7fadb7200640 1 -- 172.21.15.86:0/483652032 reap_dead start 2024-03-06T07:29:35.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.364+0000 7fadbd400640 1 -- 172.21.15.86:0/483652032 shutdown_connections 2024-03-06T07:29:35.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.364+0000 7fadbd400640 1 -- 172.21.15.86:0/483652032 >> 172.21.15.86:0/483652032 conn(0x7fadb80b1940 msgr2=0x7fadb80b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:35.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.364+0000 7fadbd400640 1 -- 172.21.15.86:0/483652032 shutdown_connections 2024-03-06T07:29:35.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.364+0000 7fadbd400640 1 -- 172.21.15.86:0/483652032 wait complete. 2024-03-06T07:29:35.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.366+0000 7fadbd400640 1 Processor -- start 2024-03-06T07:29:35.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.366+0000 7fadbd400640 1 -- start start 2024-03-06T07:29:35.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadbd400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fad9c004d40 0x7fadb81f1790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadbd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8146150 0x7fadb8158370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadbd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fadb81465d0 0x7fadb81588b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadbd400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fadb815c460 con 0x7fadb81465d0 2024-03-06T07:29:35.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadbd400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fadb815c2e0 con 0x7fad9c004d40 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadbd400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fadb8070ff0 con 0x7fadb8146150 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fadb81465d0 0x7fadb81588b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fadb81465d0 0x7fadb81588b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51018/0 (socket says 172.21.15.86:51018) 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 -- 172.21.15.86:0/1633845602 learned_addr learned my addr 172.21.15.86:0/1633845602 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 -- 172.21.15.86:0/1633845602 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8146150 msgr2=0x7fadb8158370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8146150 0x7fadb8158370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 -- 172.21.15.86:0/1633845602 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fad9c004d40 msgr2=0x7fadb81f1790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fad9c004d40 0x7fadb81f1790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.367+0000 7fadb7200640 1 -- 172.21.15.86:0/1633845602 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fadb815d7b0 con 0x7fadb81465d0 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.368+0000 7fadb7200640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fadb81465d0 0x7fadb81588b0 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7fadb0037040 tx=0x7fadb0004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.368+0000 7fadaea00640 1 -- 172.21.15.86:0/1633845602 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fadb0039030 con 0x7fadb81465d0 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.368+0000 7fadaea00640 1 -- 172.21.15.86:0/1633845602 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fadb0043070 con 0x7fadb81465d0 2024-03-06T07:29:35.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.368+0000 7fadaea00640 1 -- 172.21.15.86:0/1633845602 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fadb0009be0 con 0x7fadb81465d0 2024-03-06T07:29:35.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.368+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fadb815da90 con 0x7fadb81465d0 2024-03-06T07:29:35.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.368+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fadb81f1cd0 con 0x7fadb81465d0 2024-03-06T07:29:35.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.369+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fadb804f330 con 0x7fadb81465d0 2024-03-06T07:29:35.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.370+0000 7fadaea00640 1 -- 172.21.15.86:0/1633845602 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fadb003d7d0 con 0x7fadb81465d0 2024-03-06T07:29:35.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.371+0000 7fadaea00640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fad9406b4d0 0x7fad9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.371+0000 7fadaea00640 1 -- 172.21.15.86:0/1633845602 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fadb00ac190 con 0x7fadb81465d0 2024-03-06T07:29:35.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.371+0000 7fadb6800640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fad9406b4d0 0x7fad9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.372+0000 7fadb6800640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fad9406b4d0 0x7fad9406d990 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fada8001590 tx=0x7fada8007990 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:35.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.376+0000 7fadaea00640 1 -- 172.21.15.86:0/1633845602 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fadb007b310 con 0x7fadb81465d0 2024-03-06T07:29:35.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.569+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7fadb806ef30 con 0x7fadb81465d0 2024-03-06T07:29:35.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.590+0000 7fadaea00640 1 -- 172.21.15.86:0/1633845602 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v14) v1 ==== 104+0+0 (secure 0 0 0) 0x7fadb007ac10 con 0x7fadb81465d0 2024-03-06T07:29:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fad9406b4d0 msgr2=0x7fad9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fad9406b4d0 0x7fad9406d990 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fada8001590 tx=0x7fada8007990 comp rx=0 tx=0).stop 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fadb81465d0 msgr2=0x7fadb81588b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fadb81465d0 0x7fadb81588b0 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7fadb0037040 tx=0x7fadb0004ab0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 shutdown_connections 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fad9406b4d0 0x7fad9406d990 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fadb81465d0 0x7fadb81588b0 unknown :-1 s=CLOSED pgs=599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fadb8146150 0x7fadb8158370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 --2- 172.21.15.86:0/1633845602 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fad9c004d40 0x7fadb81f1790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.593+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 >> 172.21.15.86:0/1633845602 conn(0x7fadb80b1940 msgr2=0x7fadb80adc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.594+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 shutdown_connections 2024-03-06T07:29:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.594+0000 7fadbd400640 1 -- 172.21.15.86:0/1633845602 wait complete. 2024-03-06T07:29:35.616 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_fragment_size_max 2024-03-06T07:29:35.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.822+0000 7fbe0f200640 1 Processor -- start 2024-03-06T07:29:35.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.822+0000 7fbe0f200640 1 -- start start 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.823+0000 7fbe0f200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbe080b3df0 0x7fbe080727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe0f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe080b3650 0x7fbe080b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe0f200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe080b4710 con 0x7fbe080cda90 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe0f200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbe08072310 con 0x7fbe080b3650 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe0f200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbe080719e0 con 0x7fbe080b3df0 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbdff400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe080b3650 0x7fbe080b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbdff400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe080b3650 0x7fbe080b41d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51022/0 (socket says 172.21.15.86:51022) 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbdff400640 1 -- 172.21.15.86:0/2435196485 learned_addr learned my addr 172.21.15.86:0/2435196485 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe0d200640 1 --1- 172.21.15.86:0/2435196485 >> v1:172.21.15.86:6789/0 conn(0x7fbe080cda90 0x7fbe0806c990 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47732/0 (socket says 172.21.15.86:47732) 2024-03-06T07:29:35.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbdff400640 1 -- 172.21.15.86:0/2435196485 >> v2:172.21.15.194:3300/0 conn(0x7fbe080b3df0 msgr2=0x7fbe080727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:35.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe07e00640 1 --2- 172.21.15.86:0/2435196485 >> v2:172.21.15.194:3300/0 conn(0x7fbe080b3df0 0x7fbe080727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbdff400640 1 --2- 172.21.15.86:0/2435196485 >> v2:172.21.15.194:3300/0 conn(0x7fbe080b3df0 0x7fbe080727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbdff400640 1 -- 172.21.15.86:0/2435196485 >> v1:172.21.15.86:6789/0 conn(0x7fbe080cda90 legacy=0x7fbe0806c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbdff400640 1 -- 172.21.15.86:0/2435196485 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe08071cc0 con 0x7fbe080b3650 2024-03-06T07:29:35.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe07e00640 1 --2- 172.21.15.86:0/2435196485 >> v2:172.21.15.194:3300/0 conn(0x7fbe080b3df0 0x7fbe080727a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:35.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.824+0000 7fbe07400640 1 -- 172.21.15.86:0/2435196485 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1783418990 0 0) 0x7fbe080b4710 con 0x7fbe080cda90 2024-03-06T07:29:35.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbdff400640 1 --2- 172.21.15.86:0/2435196485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080b3650 0x7fbe080b41d0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fbe080b3dc0 tx=0x7fbdf002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7b620ecfd7ee7eee server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:35.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe07400640 1 -- 172.21.15.86:0/2435196485 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdf0034070 con 0x7fbe080b3650 2024-03-06T07:29:35.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe07400640 1 -- 172.21.15.86:0/2435196485 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbdf0030860 con 0x7fbe080b3650 2024-03-06T07:29:35.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe07400640 1 -- 172.21.15.86:0/2435196485 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdf0030c30 con 0x7fbe080b3650 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe0f200640 1 -- 172.21.15.86:0/2435196485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080b3650 msgr2=0x7fbe080b41d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe0f200640 1 --2- 172.21.15.86:0/2435196485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080b3650 0x7fbe080b41d0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fbe080b3dc0 tx=0x7fbdf002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe0f200640 1 -- 172.21.15.86:0/2435196485 shutdown_connections 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe0f200640 1 --2- 172.21.15.86:0/2435196485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080b3650 0x7fbe080b41d0 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe0f200640 1 --2- 172.21.15.86:0/2435196485 >> v2:172.21.15.194:3300/0 conn(0x7fbe080b3df0 0x7fbe080727a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.826+0000 7fbe0f200640 1 -- 172.21.15.86:0/2435196485 >> 172.21.15.86:0/2435196485 conn(0x7fbe080b1940 msgr2=0x7fbe080b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.827+0000 7fbe0f200640 1 -- 172.21.15.86:0/2435196485 shutdown_connections 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.827+0000 7fbe0f200640 1 -- 172.21.15.86:0/2435196485 wait complete. 2024-03-06T07:29:35.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.828+0000 7fbe0f200640 1 Processor -- start 2024-03-06T07:29:35.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.828+0000 7fbe0f200640 1 -- start start 2024-03-06T07:29:35.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.829+0000 7fbe0f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe080b3650 0x7fbe081484f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.829+0000 7fbe0f200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe080b3df0 0x7fbe08154890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.829+0000 7fbe0f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080cda90 0x7fbe08154dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.829+0000 7fbe0f200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe0815b240 con 0x7fbe080cda90 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.829+0000 7fbe0f200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe080b3b30 con 0x7fbe080b3df0 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.829+0000 7fbe0f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe0815b3c0 con 0x7fbe080b3650 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080cda90 0x7fbe08154dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080cda90 0x7fbe08154dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51038/0 (socket says 172.21.15.86:51038) 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 -- 172.21.15.86:0/167853527 learned_addr learned my addr 172.21.15.86:0/167853527 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbdff400640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe080b3df0 0x7fbe08154890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 -- 172.21.15.86:0/167853527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe080b3650 msgr2=0x7fbe081484f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe080b3650 0x7fbe081484f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 -- 172.21.15.86:0/167853527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe080b3df0 msgr2=0x7fbe08154890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe080b3df0 0x7fbe08154890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 -- 172.21.15.86:0/167853527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe08148a30 con 0x7fbe080cda90 2024-03-06T07:29:35.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe0d200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080cda90 0x7fbe08154dd0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7fbe0000c700 tx=0x7fbe0000cbc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:35.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe04c00640 1 -- 172.21.15.86:0/167853527 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe00018070 con 0x7fbe080cda90 2024-03-06T07:29:35.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe04c00640 1 -- 172.21.15.86:0/167853527 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe0000ce10 con 0x7fbe080cda90 2024-03-06T07:29:35.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.830+0000 7fbe04c00640 1 -- 172.21.15.86:0/167853527 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe0000a860 con 0x7fbe080cda90 2024-03-06T07:29:35.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.831+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbe080adb50 con 0x7fbe080cda90 2024-03-06T07:29:35.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.831+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe080ae110 con 0x7fbe080cda90 2024-03-06T07:29:35.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.832+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbe0804f330 con 0x7fbe080cda90 2024-03-06T07:29:35.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.833+0000 7fbe04c00640 1 -- 172.21.15.86:0/167853527 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbe0000aa00 con 0x7fbe080cda90 2024-03-06T07:29:35.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.834+0000 7fbe04c00640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbde406b500 0x7fbde406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:35.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.834+0000 7fbe04c00640 1 -- 172.21.15.86:0/167853527 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fbe00089380 con 0x7fbe080cda90 2024-03-06T07:29:35.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.834+0000 7fbe07e00640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbde406b500 0x7fbde406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:35.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.835+0000 7fbe07e00640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbde406b500 0x7fbde406d9c0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fbdf8006fd0 tx=0x7fbdf8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:35.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:35.839+0000 7fbe04c00640 1 -- 172.21.15.86:0/167853527 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbe08148a30 con 0x7fbe080cda90 2024-03-06T07:29:36.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.047+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"} v 0) v1 -- 0x7fbe08156770 con 0x7fbe080cda90 2024-03-06T07:29:36.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.054+0000 7fbe04c00640 1 -- 172.21.15.86:0/167853527 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"}]=0 v15) v1 ==== 108+0+0 (secure 0 0 0) 0x7fbe00058590 con 0x7fbe080cda90 2024-03-06T07:29:36.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbde406b500 msgr2=0x7fbde406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:36.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbde406b500 0x7fbde406d9c0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fbdf8006fd0 tx=0x7fbdf8008040 comp rx=0 tx=0).stop 2024-03-06T07:29:36.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080cda90 msgr2=0x7fbe08154dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:36.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080cda90 0x7fbe08154dd0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7fbe0000c700 tx=0x7fbe0000cbc0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 shutdown_connections 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbde406b500 0x7fbde406d9c0 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe080cda90 0x7fbe08154dd0 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe080b3df0 0x7fbe08154890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 --2- 172.21.15.86:0/167853527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe080b3650 0x7fbe081484f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 >> 172.21.15.86:0/167853527 conn(0x7fbe080b1940 msgr2=0x7fbe080b0be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.057+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 shutdown_connections 2024-03-06T07:29:36.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.058+0000 7fbe0f200640 1 -- 172.21.15.86:0/167853527 wait complete. 2024-03-06T07:29:36.079 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit' 2024-03-06T07:29:36.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.279+0000 7faa9f400640 1 Processor -- start 2024-03-06T07:29:36.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.279+0000 7faa9f400640 1 -- start start 2024-03-06T07:29:36.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.281+0000 7faa9f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa980af1c0 0x7faa9815eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:36.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.281+0000 7faa9f400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7faa9806e4d0 0x7faa9806e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:36.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.281+0000 7faa9f400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7faa98071d30 con 0x7faa98070d20 2024-03-06T07:29:36.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.281+0000 7faa9f400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faa98071490 con 0x7faa9806e4d0 2024-03-06T07:29:36.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.281+0000 7faa9f400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faa98071770 con 0x7faa980af1c0 2024-03-06T07:29:36.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa980af1c0 0x7faa9815eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:36.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9d400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7faa9806e4d0 0x7faa9806e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:36.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9d400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7faa9806e4d0 0x7faa9806e960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:52842/0 (socket says 172.21.15.86:52842) 2024-03-06T07:29:36.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9d400640 1 -- 172.21.15.86:0/402019510 learned_addr learned my addr 172.21.15.86:0/402019510 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:36.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa97e00640 1 --1- 172.21.15.86:0/402019510 >> v1:172.21.15.194:6789/0 conn(0x7faa98070d20 0x7faa9806cc60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:43838/0 (socket says 172.21.15.86:43838) 2024-03-06T07:29:36.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9d400640 1 -- 172.21.15.86:0/402019510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa980af1c0 msgr2=0x7faa9815eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:36.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9d400640 1 --2- 172.21.15.86:0/402019510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa980af1c0 0x7faa9815eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9d400640 1 -- 172.21.15.86:0/402019510 >> v1:172.21.15.194:6789/0 conn(0x7faa98070d20 legacy=0x7faa9806cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:36.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.282+0000 7faa9d400640 1 -- 172.21.15.86:0/402019510 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa98071a50 con 0x7faa9806e4d0 2024-03-06T07:29:36.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.284+0000 7faa9d400640 1 --2- 172.21.15.86:0/402019510 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 0x7faa9806e960 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7faa90009e50 tx=0x7faa9002bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8f7667838f349536 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:36.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa97400640 1 -- 172.21.15.86:0/402019510 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa9002cb00 con 0x7faa9806e4d0 2024-03-06T07:29:36.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa97400640 1 -- 172.21.15.86:0/402019510 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faa9002cca0 con 0x7faa9806e4d0 2024-03-06T07:29:36.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa97400640 1 -- 172.21.15.86:0/402019510 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa90034610 con 0x7faa9806e4d0 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa9f400640 1 -- 172.21.15.86:0/402019510 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 msgr2=0x7faa9806e960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa9f400640 1 --2- 172.21.15.86:0/402019510 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 0x7faa9806e960 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7faa90009e50 tx=0x7faa9002bf60 comp rx=0 tx=0).stop 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa9f400640 1 -- 172.21.15.86:0/402019510 shutdown_connections 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa9f400640 1 --2- 172.21.15.86:0/402019510 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 0x7faa9806e960 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa9f400640 1 --2- 172.21.15.86:0/402019510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa980af1c0 0x7faa9815eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.285+0000 7faa9f400640 1 -- 172.21.15.86:0/402019510 >> 172.21.15.86:0/402019510 conn(0x7faa980b1730 msgr2=0x7faa980b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.286+0000 7faa9f400640 1 -- 172.21.15.86:0/402019510 shutdown_connections 2024-03-06T07:29:36.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.286+0000 7faa9f400640 1 -- 172.21.15.86:0/402019510 wait complete. 2024-03-06T07:29:36.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.288+0000 7faa9f400640 1 Processor -- start 2024-03-06T07:29:36.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.288+0000 7faa9f400640 1 -- start start 2024-03-06T07:29:36.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9f400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 0x7faa98157e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:36.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa98070d20 0x7faa981562a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:36.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa980af1c0 0x7faa98152d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9f400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa9806d370 con 0x7faa980af1c0 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9f400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa980711a0 con 0x7faa9806e4d0 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9f400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faa980b35d0 con 0x7faa98070d20 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa980af1c0 0x7faa98152d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa980af1c0 0x7faa98152d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39246/0 (socket says 172.21.15.86:39246) 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9d400640 1 -- 172.21.15.86:0/1345686403 learned_addr learned my addr 172.21.15.86:0/1345686403 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.289+0000 7faa9ca00640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 0x7faa98157e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa97e00640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa98070d20 0x7faa981562a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa9d400640 1 -- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa98070d20 msgr2=0x7faa981562a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa9d400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa98070d20 0x7faa981562a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa9d400640 1 -- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 msgr2=0x7faa98157e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa9d400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 0x7faa98157e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa9d400640 1 -- 172.21.15.86:0/1345686403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa98158340 con 0x7faa980af1c0 2024-03-06T07:29:36.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa97e00640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa98070d20 0x7faa981562a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:36.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa9d400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa980af1c0 0x7faa98152d20 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7faa90036040 tx=0x7faa90004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:36.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa94c00640 1 -- 172.21.15.86:0/1345686403 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa90043020 con 0x7faa980af1c0 2024-03-06T07:29:36.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa94c00640 1 -- 172.21.15.86:0/1345686403 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faa9002ce50 con 0x7faa980af1c0 2024-03-06T07:29:36.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa94c00640 1 -- 172.21.15.86:0/1345686403 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa90009510 con 0x7faa980af1c0 2024-03-06T07:29:36.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.290+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faa98152ad0 con 0x7faa980af1c0 2024-03-06T07:29:36.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.291+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faa98157040 con 0x7faa980af1c0 2024-03-06T07:29:36.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.292+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faa9804f330 con 0x7faa980af1c0 2024-03-06T07:29:36.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.293+0000 7faa94c00640 1 -- 172.21.15.86:0/1345686403 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faa90046020 con 0x7faa980af1c0 2024-03-06T07:29:36.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.293+0000 7faa94c00640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa7806b4d0 0x7faa7806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:36.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.293+0000 7faa94c00640 1 -- 172.21.15.86:0/1345686403 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7faa900ac520 con 0x7faa980af1c0 2024-03-06T07:29:36.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.294+0000 7faa9ca00640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa7806b4d0 0x7faa7806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:36.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.294+0000 7faa9ca00640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa7806b4d0 0x7faa7806d990 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7faa8c009a10 tx=0x7faa8c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:36.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.298+0000 7faa94c00640 1 -- 172.21.15.86:0/1345686403 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faa9004d030 con 0x7faa980af1c0 2024-03-06T07:29:36.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:36.572+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]} v 0) v1 -- 0x7faa9815b850 con 0x7faa980af1c0 2024-03-06T07:29:37.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.146+0000 7faa94c00640 1 -- 172.21.15.86:0/1345686403 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]}]=0 v156) v1 ==== 133+0+0 (secure 0 0 0) 0x7faa9007b600 con 0x7faa980af1c0 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa7806b4d0 msgr2=0x7faa7806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa7806b4d0 0x7faa7806d990 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7faa8c009a10 tx=0x7faa8c008040 comp rx=0 tx=0).stop 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa980af1c0 msgr2=0x7faa98152d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa980af1c0 0x7faa98152d20 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7faa90036040 tx=0x7faa90004a40 comp rx=0 tx=0).stop 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 shutdown_connections 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa7806b4d0 0x7faa7806d990 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa980af1c0 0x7faa98152d20 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa98070d20 0x7faa981562a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 --2- 172.21.15.86:0/1345686403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa9806e4d0 0x7faa98157e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 >> 172.21.15.86:0/1345686403 conn(0x7faa980b1730 msgr2=0x7faa980b0fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:37.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 shutdown_connections 2024-03-06T07:29:37.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.149+0000 7faa9f400640 1 -- 172.21.15.86:0/1345686403 wait complete. 2024-03-06T07:29:37.171 INFO:tasks.cephfs_test_runner:That the directory fragment size cannot exceed mds_bal_fragment_size_max (using a limit of 50 in all configurations). ... ok 2024-03-06T07:29:37.171 INFO:tasks.cephfs_test_runner:Starting test: test_dirfrag_limit_fragmented (tasks.cephfs.test_strays.TestStrays) That fragmentation (forced) will allow more entries to be created. 2024-03-06T07:29:37.171 INFO:tasks.cephfs_test_runner:test_dirfrag_limit_fragmented (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:29:37.171 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:37.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.366+0000 7f6f61400640 1 Processor -- start 2024-03-06T07:29:37.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.367+0000 7f6f61400640 1 -- start start 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.368+0000 7f6f61400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5c161580 0x7f6f5c0d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.368+0000 7f6f61400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0d2a70 0x7f6f5c0b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.368+0000 7f6f61400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6f5c161960 con 0x7f6f5c0d1e90 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.368+0000 7f6f61400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6f5c0cf6f0 con 0x7f6f5c161580 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.368+0000 7f6f61400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6f5c0cf9d0 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f5be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5c161580 0x7f6f5c0d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f5be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5c161580 0x7f6f5c0d1950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39254/0 (socket says 172.21.15.86:39254) 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f5be00640 1 -- 172.21.15.86:0/734927879 learned_addr learned my addr 172.21.15.86:0/734927879 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f60a00640 1 --2- 172.21.15.86:0/734927879 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0d2a70 0x7f6f5c0b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f5be00640 1 -- 172.21.15.86:0/734927879 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0d2a70 msgr2=0x7f6f5c0b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f5be00640 1 --2- 172.21.15.86:0/734927879 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0d2a70 0x7f6f5c0b3ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f5be00640 1 -- 172.21.15.86:0/734927879 >> v1:172.21.15.194:6789/0 conn(0x7f6f5c0d1e90 legacy=0x7f6f5c0d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:37.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.369+0000 7f6f5be00640 1 -- 172.21.15.86:0/734927879 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f5c0cfcb0 con 0x7f6f5c161580 2024-03-06T07:29:37.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.371+0000 7f6f5be00640 1 --2- 172.21.15.86:0/734927879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c161580 0x7f6f5c0d1950 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7f6f5c08d180 tx=0x7f6f4c02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4e0bb4e18cf30456 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:37.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.371+0000 7f6f5aa00640 1 -- 172.21.15.86:0/734927879 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f4c02ca80 con 0x7f6f5c161580 2024-03-06T07:29:37.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.371+0000 7f6f5aa00640 1 -- 172.21.15.86:0/734927879 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6f4c02cc20 con 0x7f6f5c161580 2024-03-06T07:29:37.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.371+0000 7f6f5aa00640 1 -- 172.21.15.86:0/734927879 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f4c0343f0 con 0x7f6f5c161580 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 -- 172.21.15.86:0/734927879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c161580 msgr2=0x7f6f5c0d1950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 --2- 172.21.15.86:0/734927879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c161580 0x7f6f5c0d1950 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7f6f5c08d180 tx=0x7f6f4c02bee0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 -- 172.21.15.86:0/734927879 shutdown_connections 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 --2- 172.21.15.86:0/734927879 >> v2:172.21.15.194:3300/0 conn(0x7f6f5c0d2a70 0x7f6f5c0b3ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 --2- 172.21.15.86:0/734927879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c161580 0x7f6f5c0d1950 unknown :-1 s=CLOSED pgs=604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 -- 172.21.15.86:0/734927879 >> 172.21.15.86:0/734927879 conn(0x7f6f5c0d3d80 msgr2=0x7f6f5c0d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 -- 172.21.15.86:0/734927879 shutdown_connections 2024-03-06T07:29:37.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.372+0000 7f6f61400640 1 -- 172.21.15.86:0/734927879 wait complete. 2024-03-06T07:29:37.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.374+0000 7f6f61400640 1 Processor -- start 2024-03-06T07:29:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.374+0000 7f6f61400640 1 -- start start 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f61400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c0d1e90 0x7f6f5c14a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f61400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0d2a70 0x7f6f5c14b040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f61400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c161580 0x7f6f5c0b9370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f61400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f5c0cf3a0 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f61400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f5c0b39a0 con 0x7f6f5c0d1e90 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f61400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6f5c0b4400 con 0x7f6f5c161580 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f5b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0d2a70 0x7f6f5c14b040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f5b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0d2a70 0x7f6f5c14b040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39260/0 (socket says 172.21.15.86:39260) 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f5b400640 1 -- 172.21.15.86:0/99432956 learned_addr learned my addr 172.21.15.86:0/99432956 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f5be00640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c0d1e90 0x7f6f5c14a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.376+0000 7f6f60a00640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c161580 0x7f6f5c0b9370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f5b400640 1 -- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c161580 msgr2=0x7f6f5c0b9370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f5b400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c161580 0x7f6f5c0b9370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f5b400640 1 -- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c0d1e90 msgr2=0x7f6f5c14a8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f5b400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c0d1e90 0x7f6f5c14a8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f5b400640 1 -- 172.21.15.86:0/99432956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f5c1e8980 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f5b400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0d2a70 0x7f6f5c14b040 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7f6f4800c460 tx=0x7f6f4800c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f53e00640 1 -- 172.21.15.86:0/99432956 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f48016020 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f53e00640 1 -- 172.21.15.86:0/99432956 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6f48012870 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6f5c1e8c60 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f53e00640 1 -- 172.21.15.86:0/99432956 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f48012c60 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.377+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6f5c1e9220 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.379+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6f5c04f330 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.379+0000 7f6f53e00640 1 -- 172.21.15.86:0/99432956 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6f48019020 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.380+0000 7f6f53e00640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.380+0000 7f6f53e00640 1 -- 172.21.15.86:0/99432956 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f6f48089f40 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.380+0000 7f6f5be00640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.381+0000 7f6f5be00640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f6f4c035000 tx=0x7f6f4c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:37.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.384+0000 7f6f53e00640 1 -- 172.21.15.86:0/99432956 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6f48020020 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.585+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6f5c0d3b60 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.586+0000 7f6f53e00640 1 -- 172.21.15.86:0/99432956 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v50) v1 ==== 74+0+13034 (secure 0 0 0) 0x7f6f480182c0 con 0x7f6f5c0d2a70 2024-03-06T07:29:37.589 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:37.589 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:35.122496+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.194:6833/4064823241":"2024-03-07T07:29:32.103185+0000","172.21.15.194:6832/4064823241":"2024-03-07T07:29:32.103185+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:37.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 msgr2=0x7f6f3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f6f4c035000 tx=0x7f6f4c030000 comp rx=0 tx=0).stop 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0d2a70 msgr2=0x7f6f5c14b040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0d2a70 0x7f6f5c14b040 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7f6f4800c460 tx=0x7f6f4800c920 comp rx=0 tx=0).stop 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 shutdown_connections 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f3c06b4d0 0x7f6f3c06d990 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5c161580 0x7f6f5c0b9370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5c0d2a70 0x7f6f5c14b040 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 --2- 172.21.15.86:0/99432956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f5c0d1e90 0x7f6f5c14a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 >> 172.21.15.86:0/99432956 conn(0x7f6f5c0d3d80 msgr2=0x7f6f5c06c1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 shutdown_connections 2024-03-06T07:29:37.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.589+0000 7f6f61400640 1 -- 172.21.15.86:0/99432956 wait complete. 2024-03-06T07:29:37.610 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:29:37.610 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:29:37.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.810+0000 7f873a200640 1 Processor -- start 2024-03-06T07:29:37.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.810+0000 7f873a200640 1 -- start start 2024-03-06T07:29:37.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.811+0000 7f873a200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f8734070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.811+0000 7f873a200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f87340b3f70 0x7f87340ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.811+0000 7f873a200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f87340b46c0 con 0x7f873415fff0 2024-03-06T07:29:37.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.811+0000 7f873a200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f87340b49a0 con 0x7f87340b3f70 2024-03-06T07:29:37.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.811+0000 7f873a200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8734072c90 con 0x7f873415fc10 2024-03-06T07:29:37.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.812+0000 7f8733e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f87340b3f70 0x7f87340ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.812+0000 7f872aa00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f873415fff0 0x7f8734160450 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:43850/0 (socket says 172.21.15.86:43850) 2024-03-06T07:29:37.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.812+0000 7f8733400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f8734070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.812+0000 7f872aa00640 1 -- 172.21.15.86:0/107448404 learned_addr learned my addr 172.21.15.86:0/107448404 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:37.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.813+0000 7f8733e00640 1 -- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 msgr2=0x7f8734070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:37.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.813+0000 7f8733e00640 1 --2- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f8734070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.813+0000 7f8733e00640 1 -- 172.21.15.86:0/107448404 >> v1:172.21.15.194:6789/0 conn(0x7f873415fff0 legacy=0x7f8734160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.813+0000 7f8733e00640 1 -- 172.21.15.86:0/107448404 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f873409dd80 con 0x7f87340b3f70 2024-03-06T07:29:37.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.813+0000 7f8733400640 1 --2- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f8734070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:29:37.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.815+0000 7f8733e00640 1 --2- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f87340b3f70 0x7f87340ce4a0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f872c0097a0 tx=0x7f872c02b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fbe9c92bf3897f41 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:37.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.815+0000 7f8732a00640 1 -- 172.21.15.86:0/107448404 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872c035020 con 0x7f87340b3f70 2024-03-06T07:29:37.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.815+0000 7f8732a00640 1 -- 172.21.15.86:0/107448404 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f872c031810 con 0x7f87340b3f70 2024-03-06T07:29:37.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.815+0000 7f8732a00640 1 -- 172.21.15.86:0/107448404 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872c031be0 con 0x7f87340b3f70 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 -- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f87340b3f70 msgr2=0x7f87340ce4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 --2- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f87340b3f70 0x7f87340ce4a0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f872c0097a0 tx=0x7f872c02b8b0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 -- 172.21.15.86:0/107448404 shutdown_connections 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 --2- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f87340b3f70 0x7f87340ce4a0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 --2- 172.21.15.86:0/107448404 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f8734070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 -- 172.21.15.86:0/107448404 >> 172.21.15.86:0/107448404 conn(0x7f87340b1940 msgr2=0x7f87340b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 -- 172.21.15.86:0/107448404 shutdown_connections 2024-03-06T07:29:37.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.816+0000 7f873a200640 1 -- 172.21.15.86:0/107448404 wait complete. 2024-03-06T07:29:37.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.818+0000 7f873a200640 1 Processor -- start 2024-03-06T07:29:37.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.818+0000 7f873a200640 1 -- start start 2024-03-06T07:29:37.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f873a200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87340b3f70 0x7f87340af890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f873a200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f873406e370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f873a200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873415fff0 0x7f873415c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f873a200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f873415c460 con 0x7f87340b3f70 2024-03-06T07:29:37.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f873a200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f873415c2e0 con 0x7f873415fff0 2024-03-06T07:29:37.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f873a200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f87340ce9e0 con 0x7f873415fc10 2024-03-06T07:29:37.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f8733400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87340b3f70 0x7f87340af890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f8733400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87340b3f70 0x7f87340af890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39268/0 (socket says 172.21.15.86:39268) 2024-03-06T07:29:37.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f8733400640 1 -- 172.21.15.86:0/2888983095 learned_addr learned my addr 172.21.15.86:0/2888983095 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:37.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f872aa00640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f873406e370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f8733400640 1 -- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 msgr2=0x7f873406e370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.819+0000 7f8733e00640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873415fff0 0x7f873415c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f8733400640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f873406e370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f8733400640 1 -- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873415fff0 msgr2=0x7f873415c5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f8733400640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873415fff0 0x7f873415c5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f8733400640 1 -- 172.21.15.86:0/2888983095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87340afe70 con 0x7f87340b3f70 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f872aa00640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f873406e370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f8733400640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87340b3f70 0x7f87340af890 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f8724009f60 tx=0x7f8724009f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f872be00640 1 -- 172.21.15.86:0/2888983095 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8724018020 con 0x7f87340b3f70 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f872be00640 1 -- 172.21.15.86:0/2888983095 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8724014680 con 0x7f87340b3f70 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f87340b0150 con 0x7f87340b3f70 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f872be00640 1 -- 172.21.15.86:0/2888983095 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8724014a50 con 0x7f87340b3f70 2024-03-06T07:29:37.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.820+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f873406cef0 con 0x7f87340b3f70 2024-03-06T07:29:37.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.821+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f873404f330 con 0x7f87340b3f70 2024-03-06T07:29:37.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.822+0000 7f872be00640 1 -- 172.21.15.86:0/2888983095 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8724005d10 con 0x7f87340b3f70 2024-03-06T07:29:37.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.823+0000 7f872be00640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b4d0 0x7f871006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:37.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.823+0000 7f872be00640 1 -- 172.21.15.86:0/2888983095 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f872408adb0 con 0x7f87340b3f70 2024-03-06T07:29:37.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.823+0000 7f872aa00640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b4d0 0x7f871006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:37.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.824+0000 7f872aa00640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b4d0 0x7f871006d990 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f873406eb70 tx=0x7f871c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:37.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:37.828+0000 7f872be00640 1 -- 172.21.15.86:0/2888983095 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f872408f070 con 0x7f87340b3f70 2024-03-06T07:29:38.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.026+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f873415b6a0 con 0x7f87340b3f70 2024-03-06T07:29:38.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.027+0000 7f872be00640 1 -- 172.21.15.86:0/2888983095 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v50) v1 ==== 93+0+10 (secure 0 0 0) 0x7f8724022030 con 0x7f87340b3f70 2024-03-06T07:29:38.029 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:29:38.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b4d0 msgr2=0x7f871006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:38.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b4d0 0x7f871006d990 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f873406eb70 tx=0x7f871c009040 comp rx=0 tx=0).stop 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87340b3f70 msgr2=0x7f87340af890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87340b3f70 0x7f87340af890 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f8724009f60 tx=0x7f8724009f90 comp rx=0 tx=0).stop 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 shutdown_connections 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f871006b4d0 0x7f871006d990 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f873415fff0 0x7f873415c5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f873415fc10 0x7f873406e370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.029+0000 7f873a200640 1 --2- 172.21.15.86:0/2888983095 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87340b3f70 0x7f87340af890 unknown :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.030+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 >> 172.21.15.86:0/2888983095 conn(0x7f87340b1940 msgr2=0x7f87340b0e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.030+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 shutdown_connections 2024-03-06T07:29:38.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.030+0000 7f873a200640 1 -- 172.21.15.86:0/2888983095 wait complete. 2024-03-06T07:29:38.049 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented' 2024-03-06T07:29:38.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.250+0000 7fea95c00640 1 Processor -- start 2024-03-06T07:29:38.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.250+0000 7fea95c00640 1 -- start start 2024-03-06T07:29:38.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.252+0000 7fea95c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fea9015f9d0 0x7fea9015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.252+0000 7fea95c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fea9006cef0 con 0x7fea90070b10 2024-03-06T07:29:38.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.252+0000 7fea95c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fea9006d1d0 con 0x7fea90144c00 2024-03-06T07:29:38.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.252+0000 7fea95c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fea9006d4b0 con 0x7fea9015f9d0 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.252+0000 7fea8f400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fea90070b10 0x7fea90070f20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39968/0 (socket says 172.21.15.86:39968) 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.252+0000 7fea8f400640 1 -- 172.21.15.86:0/4203616556 learned_addr learned my addr 172.21.15.86:0/4203616556 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1132503023 0 0) 0x7fea9006cef0 con 0x7fea90070b10 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8e000640 1 --2- 172.21.15.86:0/4203616556 >> v2:172.21.15.194:3300/0 conn(0x7fea9015f9d0 0x7fea9015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fea74003520 con 0x7fea90070b10 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 345197200 0 0) 0x7fea74003520 con 0x7fea90070b10 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 >> v2:172.21.15.194:3300/0 conn(0x7fea9015f9d0 msgr2=0x7fea9015fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 --2- 172.21.15.86:0/4203616556 >> v2:172.21.15.194:3300/0 conn(0x7fea9015f9d0 0x7fea9015fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 >> v1:172.21.15.194:6789/0 conn(0x7fea90144c00 legacy=0x7fea9006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea9006d790 con 0x7fea90070b10 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fea880019f0 con 0x7fea90070b10 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 >> v1:172.21.15.86:6789/0 conn(0x7fea90070b10 legacy=0x7fea90070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea740039b0 0x7fea74003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea74004800 0x7fea74004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea740051c0 0x7fea7400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9006d4b0 con 0x7fea74004800 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea7400c110 con 0x7fea740051c0 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.253+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fea7400c310 con 0x7fea740039b0 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8e000640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea74004800 0x7fea74004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8ea00640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea740039b0 0x7fea74003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8f400640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea740051c0 0x7fea7400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8e000640 1 -- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea740039b0 msgr2=0x7fea74003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8e000640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea740039b0 0x7fea74003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8e000640 1 -- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea740051c0 msgr2=0x7fea7400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8e000640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea740051c0 0x7fea7400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8e000640 1 -- 172.21.15.86:0/4203616556 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea74003520 con 0x7fea74004800 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8ea00640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea740039b0 0x7fea74003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8f400640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea740051c0 0x7fea7400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:38.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.254+0000 7fea8f400640 1 -- 172.21.15.86:0/4203616556 reap_dead start 2024-03-06T07:29:38.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.255+0000 7fea8e000640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea74004800 0x7fea74004c80 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7fea7c00ba30 tx=0x7fea74005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0161a08f5b98789 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:38.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.255+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea7c036020 con 0x7fea74004800 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.255+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fea7c032640 con 0x7fea74004800 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.255+0000 7fea8d600640 1 -- 172.21.15.86:0/4203616556 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea7c00b600 con 0x7fea74004800 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.256+0000 7fea95c00640 1 -- 172.21.15.86:0/4203616556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea74004800 msgr2=0x7fea74004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.256+0000 7fea95c00640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea74004800 0x7fea74004c80 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7fea7c00ba30 tx=0x7fea74005830 comp rx=0 tx=0).stop 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.256+0000 7fea95c00640 1 -- 172.21.15.86:0/4203616556 shutdown_connections 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.256+0000 7fea95c00640 1 --2- 172.21.15.86:0/4203616556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea74004800 0x7fea74004c80 unknown :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.256+0000 7fea95c00640 1 -- 172.21.15.86:0/4203616556 >> 172.21.15.86:0/4203616556 conn(0x7fea900b1730 msgr2=0x7fea900b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.256+0000 7fea95c00640 1 -- 172.21.15.86:0/4203616556 shutdown_connections 2024-03-06T07:29:38.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.256+0000 7fea95c00640 1 -- 172.21.15.86:0/4203616556 wait complete. 2024-03-06T07:29:38.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.258+0000 7fea95c00640 1 Processor -- start 2024-03-06T07:29:38.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.258+0000 7fea95c00640 1 -- start start 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea95c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea74004800 0x7fea901e5c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea95c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea901e8890 0x7fea900cbe60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea95c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea901e8cd0 0x7fea900cc3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea95c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9006ccd0 con 0x7fea901e8cd0 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea95c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9006cb50 con 0x7fea901e8890 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea95c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fea9006df90 con 0x7fea74004800 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea8f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea901e8cd0 0x7fea900cc3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea8f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea901e8cd0 0x7fea900cc3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39288/0 (socket says 172.21.15.86:39288) 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.259+0000 7fea8f400640 1 -- 172.21.15.86:0/4110154787 learned_addr learned my addr 172.21.15.86:0/4110154787 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea8ea00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea74004800 0x7fea901e5c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea8f400640 1 -- 172.21.15.86:0/4110154787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea74004800 msgr2=0x7fea901e5c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea8f400640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea74004800 0x7fea901e5c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea8f400640 1 -- 172.21.15.86:0/4110154787 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea901e8890 msgr2=0x7fea900cbe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea8f400640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea901e8890 0x7fea900cbe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea8f400640 1 -- 172.21.15.86:0/4110154787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea9012fe30 con 0x7fea901e8cd0 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea8f400640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea901e8cd0 0x7fea900cc3a0 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7fea900737b0 tx=0x7fea8800b4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea86a00640 1 -- 172.21.15.86:0/4110154787 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea880019d0 con 0x7fea901e8cd0 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea86a00640 1 -- 172.21.15.86:0/4110154787 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fea8800bb80 con 0x7fea901e8cd0 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea90130110 con 0x7fea901e8cd0 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea86a00640 1 -- 172.21.15.86:0/4110154787 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea88007070 con 0x7fea901e8cd0 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.260+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea901306d0 con 0x7fea901e8cd0 2024-03-06T07:29:38.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.261+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fea9004f330 con 0x7fea901e8cd0 2024-03-06T07:29:38.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.262+0000 7fea86a00640 1 -- 172.21.15.86:0/4110154787 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fea88007210 con 0x7fea901e8cd0 2024-03-06T07:29:38.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.263+0000 7fea86a00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fea7006b4d0 0x7fea7006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:38.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.263+0000 7fea86a00640 1 -- 172.21.15.86:0/4110154787 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fea88089e70 con 0x7fea901e8cd0 2024-03-06T07:29:38.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.264+0000 7fea8ea00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fea7006b4d0 0x7fea7006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:38.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.264+0000 7fea8ea00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fea7006b4d0 0x7fea7006d990 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fea900686c0 tx=0x7fea80009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:38.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.268+0000 7fea86a00640 1 -- 172.21.15.86:0/4110154787 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fea88058ff0 con 0x7fea901e8cd0 2024-03-06T07:29:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:38.536+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]} v 0) v1 -- 0x7fea90072890 con 0x7fea901e8cd0 2024-03-06T07:29:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.159+0000 7fea86a00640 1 -- 172.21.15.86:0/4110154787 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]}]=0 v158) v1 ==== 147+0+0 (secure 0 0 0) 0x7fea880588f0 con 0x7fea901e8cd0 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.162+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fea7006b4d0 msgr2=0x7fea7006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.162+0000 7fea95c00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fea7006b4d0 0x7fea7006d990 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fea900686c0 tx=0x7fea80009040 comp rx=0 tx=0).stop 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea901e8cd0 msgr2=0x7fea900cc3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea901e8cd0 0x7fea900cc3a0 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7fea900737b0 tx=0x7fea8800b4a0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 shutdown_connections 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fea7006b4d0 0x7fea7006d990 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fea901e8cd0 0x7fea900cc3a0 unknown :-1 s=CLOSED pgs=609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fea901e8890 0x7fea900cbe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 --2- 172.21.15.86:0/4110154787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fea74004800 0x7fea901e5c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 >> 172.21.15.86:0/4110154787 conn(0x7fea900b1730 msgr2=0x7fea900ae830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 shutdown_connections 2024-03-06T07:29:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.163+0000 7fea95c00640 1 -- 172.21.15.86:0/4110154787 wait complete. 2024-03-06T07:29:39.185 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:29:39.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.397+0000 7ff563000640 1 Processor -- start 2024-03-06T07:29:39.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.398+0000 7ff563000640 1 -- start start 2024-03-06T07:29:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.399+0000 7ff563000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c0b3df0 0x7ff55c0727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.399+0000 7ff563000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff55c0b3650 0x7ff55c0b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.399+0000 7ff563000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff55c0b4710 con 0x7ff55c0cda90 2024-03-06T07:29:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.399+0000 7ff563000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff55c072310 con 0x7ff55c0b3650 2024-03-06T07:29:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.399+0000 7ff563000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff55c0719e0 con 0x7ff55c0b3df0 2024-03-06T07:29:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.400+0000 7ff561000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff55c0cda90 0x7ff55c06c990 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39976/0 (socket says 172.21.15.86:39976) 2024-03-06T07:29:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.400+0000 7ff561000640 1 -- 172.21.15.86:0/21678590 learned_addr learned my addr 172.21.15.86:0/21678590 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:39.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.400+0000 7ff553400640 1 --2- 172.21.15.86:0/21678590 >> v2:172.21.15.194:3300/0 conn(0x7ff55c0b3650 0x7ff55c0b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.400+0000 7ff55be00640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c0b3df0 0x7ff55c0727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.400+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1838122627 0 0) 0x7ff55c0b4710 con 0x7ff55c0cda90 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.400+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff540003520 con 0x7ff55c0cda90 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2462426290 0 0) 0x7ff540003520 con 0x7ff55c0cda90 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c0b3df0 msgr2=0x7ff55c0727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c0b3df0 0x7ff55c0727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 >> v2:172.21.15.194:3300/0 conn(0x7ff55c0b3650 msgr2=0x7ff55c0b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 --2- 172.21.15.86:0/21678590 >> v2:172.21.15.194:3300/0 conn(0x7ff55c0b3650 0x7ff55c0b41d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff55c071cc0 con 0x7ff55c0cda90 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff5540019f0 con 0x7ff55c0cda90 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 >> v1:172.21.15.86:6789/0 conn(0x7ff55c0cda90 legacy=0x7ff55c06c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5400039b0 0x7ff540003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff540004800 0x7ff540004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c0b3df0 0x7ff5400051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff55c072310 con 0x7ff5400039b0 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff55c0719e0 con 0x7ff540004800 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff54000b9c0 con 0x7ff55c0b3df0 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55be00640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5400039b0 0x7ff540003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55be00640 1 -- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c0b3df0 msgr2=0x7ff5400051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55be00640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c0b3df0 0x7ff5400051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55be00640 1 -- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff540004800 msgr2=0x7ff540004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55be00640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff540004800 0x7ff540004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.401+0000 7ff55be00640 1 -- 172.21.15.86:0/21678590 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff540003520 con 0x7ff5400039b0 2024-03-06T07:29:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.402+0000 7ff55be00640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5400039b0 0x7ff540003dd0 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7ff54c009ac0 tx=0x7ff540005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bb09bae50b76023d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff54c039070 con 0x7ff5400039b0 2024-03-06T07:29:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff54c02c6d0 con 0x7ff5400039b0 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff55b400640 1 -- 172.21.15.86:0/21678590 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff54c009690 con 0x7ff5400039b0 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff563000640 1 -- 172.21.15.86:0/21678590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5400039b0 msgr2=0x7ff540003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff563000640 1 --2- 172.21.15.86:0/21678590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5400039b0 0x7ff540003dd0 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7ff54c009ac0 tx=0x7ff540005830 comp rx=0 tx=0).stop 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff561000640 1 -- 172.21.15.86:0/21678590 reap_dead start 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff563000640 1 -- 172.21.15.86:0/21678590 shutdown_connections 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.403+0000 7ff563000640 1 -- 172.21.15.86:0/21678590 >> 172.21.15.86:0/21678590 conn(0x7ff55c0b1940 msgr2=0x7ff55c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.404+0000 7ff563000640 1 -- 172.21.15.86:0/21678590 shutdown_connections 2024-03-06T07:29:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.404+0000 7ff563000640 1 -- 172.21.15.86:0/21678590 wait complete. 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.405+0000 7ff563000640 1 Processor -- start 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.405+0000 7ff563000640 1 -- start start 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.406+0000 7ff563000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5400039b0 0x7ff55c14d0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.406+0000 7ff563000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff55c1f4e10 0x7ff55c1f5270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.406+0000 7ff563000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c150b40 0x7ff55c150fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.406+0000 7ff563000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff55c15b240 con 0x7ff55c1f4e10 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.406+0000 7ff563000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff55c0b3b30 con 0x7ff5400039b0 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.406+0000 7ff563000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff55c15b3c0 con 0x7ff55c150b40 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff55c1f4e10 0x7ff55c1f5270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff55c1f4e10 0x7ff55c1f5270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39306/0 (socket says 172.21.15.86:39306) 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 -- 172.21.15.86:0/389201630 learned_addr learned my addr 172.21.15.86:0/389201630 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 -- 172.21.15.86:0/389201630 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c150b40 msgr2=0x7ff55c150fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c150b40 0x7ff55c150fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 -- 172.21.15.86:0/389201630 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5400039b0 msgr2=0x7ff55c14d0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff55be00640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5400039b0 0x7ff55c14d0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:39.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5400039b0 0x7ff55c14d0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 -- 172.21.15.86:0/389201630 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff55c1f7dd0 con 0x7ff55c1f4e10 2024-03-06T07:29:39.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.407+0000 7ff553400640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff55c1f4e10 0x7ff55c1f5270 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7ff54400e890 tx=0x7ff54400ed50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:39.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.408+0000 7ff558c00640 1 -- 172.21.15.86:0/389201630 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff544017070 con 0x7ff55c1f4e10 2024-03-06T07:29:39.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.408+0000 7ff558c00640 1 -- 172.21.15.86:0/389201630 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff54400c940 con 0x7ff55c1f4e10 2024-03-06T07:29:39.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.408+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff55c1f80b0 con 0x7ff55c1f4e10 2024-03-06T07:29:39.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.408+0000 7ff558c00640 1 -- 172.21.15.86:0/389201630 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff54400cd10 con 0x7ff55c1f4e10 2024-03-06T07:29:39.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.408+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff55c1f8670 con 0x7ff55c1f4e10 2024-03-06T07:29:39.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.409+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff55c1f8870 con 0x7ff55c1f4e10 2024-03-06T07:29:39.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.410+0000 7ff558c00640 1 -- 172.21.15.86:0/389201630 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff54401a070 con 0x7ff55c1f4e10 2024-03-06T07:29:39.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.411+0000 7ff558c00640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff53806b500 0x7ff53806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:39.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.411+0000 7ff558c00640 1 -- 172.21.15.86:0/389201630 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7ff54408a0d0 con 0x7ff55c1f4e10 2024-03-06T07:29:39.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.411+0000 7ff55be00640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff53806b500 0x7ff53806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:39.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.412+0000 7ff55be00640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff53806b500 0x7ff53806d9c0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7ff54c036000 tx=0x7ff54c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:39.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.415+0000 7ff558c00640 1 -- 172.21.15.86:0/389201630 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff544059240 con 0x7ff55c1f4e10 2024-03-06T07:29:39.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.615+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7ff55c074780 con 0x7ff55c1f4e10 2024-03-06T07:29:39.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.621+0000 7ff558c00640 1 -- 172.21.15.86:0/389201630 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v16)=0 v16) v1 ==== 122+0+0 (secure 0 0 0) 0x7ff55c074780 con 0x7ff55c1f4e10 2024-03-06T07:29:39.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff53806b500 msgr2=0x7ff53806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff53806b500 0x7ff53806d9c0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7ff54c036000 tx=0x7ff54c030000 comp rx=0 tx=0).stop 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff55c1f4e10 msgr2=0x7ff55c1f5270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff55c1f4e10 0x7ff55c1f5270 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7ff54400e890 tx=0x7ff54400ed50 comp rx=0 tx=0).stop 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 shutdown_connections 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff53806b500 0x7ff53806d9c0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff55c150b40 0x7ff55c150fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff55c1f4e10 0x7ff55c1f5270 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 --2- 172.21.15.86:0/389201630 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5400039b0 0x7ff55c14d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 >> 172.21.15.86:0/389201630 conn(0x7ff55c0b1940 msgr2=0x7ff55c0b0370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 shutdown_connections 2024-03-06T07:29:39.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:39.624+0000 7ff563000640 1 -- 172.21.15.86:0/389201630 wait complete. 2024-03-06T07:29:39.645 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:29:39.645 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:29:39.754 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:29:39.879 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:29:39.879 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:39.879 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:39.936 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:29:39.936 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:29:39.936 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:39.992 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:29:39.992 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:39.992 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:40.049 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:29:40.049 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:29:40.049 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:40.104 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:29:40.104 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:29:40.105 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:40.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.320+0000 7f4b4e400640 1 Processor -- start 2024-03-06T07:29:40.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.320+0000 7f4b4e400640 1 -- start start 2024-03-06T07:29:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b4e400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4b4806cb50 0x7f4b4806cfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b4e400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4b480b2d10 0x7f4b480b3120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b4e400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b480717c0 con 0x7f4b480cf430 2024-03-06T07:29:40.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b4e400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b48071aa0 con 0x7f4b480b2d10 2024-03-06T07:29:40.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b4e400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b48071d80 con 0x7f4b4806cb50 2024-03-06T07:29:40.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b47400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4b480cf430 0x7f4b48070d20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39984/0 (socket says 172.21.15.86:39984) 2024-03-06T07:29:40.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b47400640 1 -- 172.21.15.86:0/2955893075 learned_addr learned my addr 172.21.15.86:0/2955893075 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:40.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b47e00640 1 --2- 172.21.15.86:0/2955893075 >> v2:172.21.15.86:3300/0 conn(0x7f4b480b2d10 0x7f4b480b3120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b46a00640 1 --2- 172.21.15.86:0/2955893075 >> v2:172.21.15.194:3300/0 conn(0x7f4b4806cb50 0x7f4b4806cfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.322+0000 7f4b46000640 1 -- 172.21.15.86:0/2955893075 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2773261755 0 0) 0x7f4b480717c0 con 0x7f4b480cf430 2024-03-06T07:29:40.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.323+0000 7f4b46000640 1 -- 172.21.15.86:0/2955893075 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4b2c003520 con 0x7f4b480cf430 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.323+0000 7f4b47e00640 1 -- 172.21.15.86:0/2955893075 >> v2:172.21.15.194:3300/0 conn(0x7f4b4806cb50 msgr2=0x7f4b4806cfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.323+0000 7f4b47e00640 1 --2- 172.21.15.86:0/2955893075 >> v2:172.21.15.194:3300/0 conn(0x7f4b4806cb50 0x7f4b4806cfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.323+0000 7f4b47e00640 1 -- 172.21.15.86:0/2955893075 >> v1:172.21.15.86:6789/0 conn(0x7f4b480cf430 legacy=0x7f4b48070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.323+0000 7f4b47e00640 1 -- 172.21.15.86:0/2955893075 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b48072060 con 0x7f4b480b2d10 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.324+0000 7f4b47e00640 1 --2- 172.21.15.86:0/2955893075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b480b2d10 0x7f4b480b3120 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f4b48160520 tx=0x7f4b4002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3e3adcc91bffb1e1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.324+0000 7f4b46000640 1 -- 172.21.15.86:0/2955893075 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b40039070 con 0x7f4b480b2d10 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.324+0000 7f4b46000640 1 -- 172.21.15.86:0/2955893075 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b4002cc80 con 0x7f4b480b2d10 2024-03-06T07:29:40.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.324+0000 7f4b46000640 1 -- 172.21.15.86:0/2955893075 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b40034660 con 0x7f4b480b2d10 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.324+0000 7f4b4e400640 1 -- 172.21.15.86:0/2955893075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b480b2d10 msgr2=0x7f4b480b3120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.324+0000 7f4b4e400640 1 --2- 172.21.15.86:0/2955893075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b480b2d10 0x7f4b480b3120 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f4b48160520 tx=0x7f4b4002bed0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.325+0000 7f4b4e400640 1 -- 172.21.15.86:0/2955893075 shutdown_connections 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.325+0000 7f4b4e400640 1 --2- 172.21.15.86:0/2955893075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b480b2d10 0x7f4b480b3120 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.325+0000 7f4b4e400640 1 --2- 172.21.15.86:0/2955893075 >> v2:172.21.15.194:3300/0 conn(0x7f4b4806cb50 0x7f4b4806cfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.325+0000 7f4b4e400640 1 -- 172.21.15.86:0/2955893075 >> 172.21.15.86:0/2955893075 conn(0x7f4b480b1730 msgr2=0x7f4b480b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.325+0000 7f4b4e400640 1 -- 172.21.15.86:0/2955893075 shutdown_connections 2024-03-06T07:29:40.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.325+0000 7f4b4e400640 1 -- 172.21.15.86:0/2955893075 wait complete. 2024-03-06T07:29:40.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.326+0000 7f4b4e400640 1 Processor -- start 2024-03-06T07:29:40.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.326+0000 7f4b4e400640 1 -- start start 2024-03-06T07:29:40.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.327+0000 7f4b4e400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b4806cb50 0x7f4b4814ef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.327+0000 7f4b4e400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b480b2d10 0x7f4b4814f500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.327+0000 7f4b4e400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b480cf430 0x7f4b481fbdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.327+0000 7f4b4e400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b48162b70 con 0x7f4b4806cb50 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.327+0000 7f4b4e400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b48071430 con 0x7f4b480b2d10 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.327+0000 7f4b4e400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b48162cf0 con 0x7f4b480cf430 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b4806cb50 0x7f4b4814ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b4806cb50 0x7f4b4814ef70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39330/0 (socket says 172.21.15.86:39330) 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 -- 172.21.15.86:0/3128687184 learned_addr learned my addr 172.21.15.86:0/3128687184 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b46a00640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b480b2d10 0x7f4b4814f500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 -- 172.21.15.86:0/3128687184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b480cf430 msgr2=0x7f4b481fbdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b480cf430 0x7f4b481fbdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 -- 172.21.15.86:0/3128687184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b480b2d10 msgr2=0x7f4b4814f500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b480b2d10 0x7f4b4814f500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 -- 172.21.15.86:0/3128687184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b4814fbe0 con 0x7f4b4806cb50 2024-03-06T07:29:40.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.328+0000 7f4b47400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b4806cb50 0x7f4b4814ef70 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f4b3800c390 tx=0x7f4b3800c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:40.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.329+0000 7f4b3f400640 1 -- 172.21.15.86:0/3128687184 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b38017070 con 0x7f4b4806cb50 2024-03-06T07:29:40.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.329+0000 7f4b3f400640 1 -- 172.21.15.86:0/3128687184 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b38005ce0 con 0x7f4b4806cb50 2024-03-06T07:29:40.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.329+0000 7f4b3f400640 1 -- 172.21.15.86:0/3128687184 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b380072f0 con 0x7f4b4806cb50 2024-03-06T07:29:40.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.329+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b4814fec0 con 0x7f4b4806cb50 2024-03-06T07:29:40.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.329+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b481fb8d0 con 0x7f4b4806cb50 2024-03-06T07:29:40.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.330+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b481fbad0 con 0x7f4b4806cb50 2024-03-06T07:29:40.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.331+0000 7f4b3f400640 1 -- 172.21.15.86:0/3128687184 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b3801a070 con 0x7f4b4806cb50 2024-03-06T07:29:40.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.332+0000 7f4b3f400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b2806b4d0 0x7f4b2806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.332+0000 7f4b3f400640 1 -- 172.21.15.86:0/3128687184 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f4b38089270 con 0x7f4b4806cb50 2024-03-06T07:29:40.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.332+0000 7f4b46a00640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b2806b4d0 0x7f4b2806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.333+0000 7f4b46a00640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b2806b4d0 0x7f4b2806d990 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f4b480af4f0 tx=0x7f4b34008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:40.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.337+0000 7f4b3f400640 1 -- 172.21.15.86:0/3128687184 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b481fbad0 con 0x7f4b4806cb50 2024-03-06T07:29:40.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.571+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4b4804f330 con 0x7f4b4806cb50 2024-03-06T07:29:40.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.572+0000 7f4b3f400640 1 -- 172.21.15.86:0/3128687184 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+3705 (secure 0 0 0) 0x7f4b38058480 con 0x7f4b4806cb50 2024-03-06T07:29:40.575 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:40.576 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":27,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5041,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/3678578793","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":3678578793},{"type":"v1","addr":"172.21.15.194:6835","nonce":3678578793}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5200,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/1276096740","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":1276096740},{"type":"v1","addr":"172.21.15.194:6833","nonce":1276096740}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":27},{"gid":5424,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/4206282845","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":4206282845},{"type":"v1","addr":"172.21.15.86:6835","nonce":4206282845}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20},{"gid":5427,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3500830166","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3500830166},{"type":"v1","addr":"172.21.15.86:6837","nonce":3500830166}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":20}],"filesystems":[]} 2024-03-06T07:29:40.576 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 27 2024-03-06T07:29:40.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b2806b4d0 msgr2=0x7f4b2806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b2806b4d0 0x7f4b2806d990 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f4b480af4f0 tx=0x7f4b34008040 comp rx=0 tx=0).stop 2024-03-06T07:29:40.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b4806cb50 msgr2=0x7f4b4814ef70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b4806cb50 0x7f4b4814ef70 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f4b3800c390 tx=0x7f4b3800c850 comp rx=0 tx=0).stop 2024-03-06T07:29:40.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 shutdown_connections 2024-03-06T07:29:40.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b2806b4d0 0x7f4b2806d990 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b480cf430 0x7f4b481fbdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b480b2d10 0x7f4b4814f500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 --2- 172.21.15.86:0/3128687184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b4806cb50 0x7f4b4814ef70 unknown :-1 s=CLOSED pgs=615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.575+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 >> 172.21.15.86:0/3128687184 conn(0x7f4b480b1730 msgr2=0x7f4b48070d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:40.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.576+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 shutdown_connections 2024-03-06T07:29:40.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.576+0000 7f4b4e400640 1 -- 172.21.15.86:0/3128687184 wait complete. 2024-03-06T07:29:40.598 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:29:40.598 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:29:40.599 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:29:40.599 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:29:40.599 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:29:40.599 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:29:40.599 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:29:40.599 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:29:40.599 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:29:40.600 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:29:40.600 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:29:40.600 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:29:40.600 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:29:40.600 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:29:40.600 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:29:40.600 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:29:40.617 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:29:40.617 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:29:40.623 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:29:40.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:29:40.624 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:29:40.624 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:29:40.625 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:29:40.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:29:40.671 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:29:40.672 DEBUG:teuthology.parallel:result is None 2024-03-06T07:29:40.673 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:29:40.673 DEBUG:teuthology.parallel:result is None 2024-03-06T07:29:40.677 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:29:40.678 DEBUG:teuthology.parallel:result is None 2024-03-06T07:29:40.679 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:29:40.679 DEBUG:teuthology.parallel:result is None 2024-03-06T07:29:40.679 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:29:40.822 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:29:40.823 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:29:40.837 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:29:40.840 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:29:40.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.854+0000 7f1864600640 1 Processor -- start 2024-03-06T07:29:40.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.854+0000 7f1864600640 1 -- start start 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.855+0000 7f1864600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f185c15b070 0x7f185c06e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.855+0000 7f1864600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 0x7f185c0c4a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.855+0000 7f1864600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f185c0cc000 con 0x7f185c15ac90 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.855+0000 7f1864600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f185c0ce360 con 0x7f185c15b070 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.855+0000 7f1864600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f185c0ce640 con 0x7f185c15f120 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f1861c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f185c15ac90 0x7f185c15ea10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40004/0 (socket says 172.21.15.86:40004) 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f1861c00640 1 -- 172.21.15.86:0/2507481333 learned_addr learned my addr 172.21.15.86:0/2507481333 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:40.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f1861200640 1 --2- 172.21.15.86:0/2507481333 >> v2:172.21.15.86:3300/0 conn(0x7f185c15b070 0x7f185c06e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2688372882 0 0) 0x7f185c0cc000 con 0x7f185c15ac90 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1844003520 con 0x7f185c15ac90 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f1862600640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 0x7f185c0c4a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3220661750 0 0) 0x7f1844003520 con 0x7f185c15ac90 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 msgr2=0x7f185c0c4a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 0x7f185c0c4a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 >> v2:172.21.15.86:3300/0 conn(0x7f185c15b070 msgr2=0x7f185c06e580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 --2- 172.21.15.86:0/2507481333 >> v2:172.21.15.86:3300/0 conn(0x7f185c15b070 0x7f185c06e580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f185c0ce920 con 0x7f185c15ac90 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1850001670 con 0x7f185c15ac90 2024-03-06T07:29:40.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 >> v1:172.21.15.86:6789/0 conn(0x7f185c15ac90 legacy=0x7f185c15ea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 0x7f18440039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844003f30 0x7f1844004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1844004d40 0x7f18440051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f185c0ce360 con 0x7f1844004d40 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f185c0ce640 con 0x7f1844003f30 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.856+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f184400dc20 con 0x7f185c15f120 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1862600640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1844004d40 0x7f18440051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1861c00640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 0x7f18440039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1862600640 1 -- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 msgr2=0x7f18440039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1861200640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844003f30 0x7f1844004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1862600640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 0x7f18440039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1862600640 1 -- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844003f30 msgr2=0x7f1844004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1862600640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844003f30 0x7f1844004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1862600640 1 -- 172.21.15.86:0/2507481333 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1844003520 con 0x7f1844004d40 2024-03-06T07:29:40.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1861c00640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c15f120 0x7f18440039b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:40.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.857+0000 7f1861200640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844003f30 0x7f1844004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.858+0000 7f1862600640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1844004d40 0x7f18440051c0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f185400e170 tx=0x7f1844005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=999c3b4ad982edca server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.858+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1854039070 con 0x7f1844004d40 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f185402cbf0 con 0x7f1844004d40 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f185be00640 1 -- 172.21.15.86:0/2507481333 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1854009c10 con 0x7f1844004d40 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f1864600640 1 -- 172.21.15.86:0/2507481333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1844004d40 msgr2=0x7f18440051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f1864600640 1 --2- 172.21.15.86:0/2507481333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1844004d40 0x7f18440051c0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f185400e170 tx=0x7f1844005830 comp rx=0 tx=0).stop 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f1862600640 1 -- 172.21.15.86:0/2507481333 reap_dead start 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f1864600640 1 -- 172.21.15.86:0/2507481333 shutdown_connections 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f1864600640 1 -- 172.21.15.86:0/2507481333 >> 172.21.15.86:0/2507481333 conn(0x7f185c0cc4f0 msgr2=0x7f185c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f1864600640 1 -- 172.21.15.86:0/2507481333 shutdown_connections 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.859+0000 7f1864600640 1 -- 172.21.15.86:0/2507481333 wait complete. 2024-03-06T07:29:40.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.861+0000 7f1864600640 1 Processor -- start 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.861+0000 7f1864600640 1 -- start start 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1864600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844004d40 0x7f185c14d790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1864600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f185c15b4f0 0x7f185c14c090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1864600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c14b900 0x7f185c06db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1864600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f185c15f580 con 0x7f185c15b4f0 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1864600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f185c0ceee0 con 0x7f1844004d40 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1864600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f185c15f700 con 0x7f185c14b900 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1861200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f185c15b4f0 0x7f185c14c090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1861200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f185c15b4f0 0x7f185c14c090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39372/0 (socket says 172.21.15.86:39372) 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1861200640 1 -- 172.21.15.86:0/2165277184 learned_addr learned my addr 172.21.15.86:0/2165277184 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1861c00640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844004d40 0x7f185c14d790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1862600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c14b900 0x7f185c06db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.862+0000 7f1861200640 1 -- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c14b900 msgr2=0x7f185c06db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1861200640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c14b900 0x7f185c06db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1861200640 1 -- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844004d40 msgr2=0x7f185c14d790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:40.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1861200640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844004d40 0x7f185c14d790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:40.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1861200640 1 -- 172.21.15.86:0/2165277184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f185c14dd70 con 0x7f185c15b4f0 2024-03-06T07:29:40.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1861200640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f185c15b4f0 0x7f185c14c090 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f1848010a00 tx=0x7f1848010ec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:40.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1859600640 1 -- 172.21.15.86:0/2165277184 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f184800f040 con 0x7f185c15b4f0 2024-03-06T07:29:40.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1859600640 1 -- 172.21.15.86:0/2165277184 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f184800ec20 con 0x7f185c15b4f0 2024-03-06T07:29:40.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1859600640 1 -- 172.21.15.86:0/2165277184 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f184801c400 con 0x7f185c15b4f0 2024-03-06T07:29:40.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.863+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f185c0c9200 con 0x7f185c15b4f0 2024-03-06T07:29:40.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.864+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f185c0c97c0 con 0x7f185c15b4f0 2024-03-06T07:29:40.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.865+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f185c04f330 con 0x7f185c15b4f0 2024-03-06T07:29:40.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.866+0000 7f1859600640 1 -- 172.21.15.86:0/2165277184 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f184801c5a0 con 0x7f185c15b4f0 2024-03-06T07:29:40.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.867+0000 7f1859600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f183c06b500 0x7f183c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:40.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.867+0000 7f1859600640 1 -- 172.21.15.86:0/2165277184 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(50..50 src has 1..50) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f184808a4d0 con 0x7f185c15b4f0 2024-03-06T07:29:40.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.867+0000 7f1861c00640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f183c06b500 0x7f183c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:40.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.868+0000 7f1861c00640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f183c06b500 0x7f183c06d9c0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f1850000f60 tx=0x7f1850009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:40.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:40.871+0000 7f1859600640 1 -- 172.21.15.86:0/2165277184 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1848018030 con 0x7f185c15b4f0 2024-03-06T07:29:41.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.063+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f185c070cd0 con 0x7f185c15b4f0 2024-03-06T07:29:41.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.182+0000 7f1859600640 1 -- 172.21.15.86:0/2165277184 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v52) v1 ==== 97+0+0 (secure 0 0 0) 0x7f1848058f60 con 0x7f185c15b4f0 2024-03-06T07:29:41.185 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:29:41.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.184+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f183c06b500 msgr2=0x7f183c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.184+0000 7f1864600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f183c06b500 0x7f183c06d9c0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f1850000f60 tx=0x7f1850009040 comp rx=0 tx=0).stop 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.184+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f185c15b4f0 msgr2=0x7f185c14c090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.184+0000 7f1864600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f185c15b4f0 0x7f185c14c090 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f1848010a00 tx=0x7f1848010ec0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 shutdown_connections 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f183c06b500 0x7f183c06d9c0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f185c14b900 0x7f185c06db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f185c15b4f0 0x7f185c14c090 unknown :-1 s=CLOSED pgs=625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 --2- 172.21.15.86:0/2165277184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1844004d40 0x7f185c14d790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 >> 172.21.15.86:0/2165277184 conn(0x7f185c0cc4f0 msgr2=0x7f185c15e810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 shutdown_connections 2024-03-06T07:29:41.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.185+0000 7f1864600640 1 -- 172.21.15.86:0/2165277184 wait complete. 2024-03-06T07:29:41.208 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:29:41.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.396+0000 7f9697200640 1 Processor -- start 2024-03-06T07:29:41.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.397+0000 7f9697200640 1 -- start start 2024-03-06T07:29:41.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.398+0000 7f9697200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 0x7f96980cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.398+0000 7f9697200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f969815c240 con 0x7f96981561f0 2024-03-06T07:29:41.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.398+0000 7f9697200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f969815c500 con 0x7f96980cc800 2024-03-06T07:29:41.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.398+0000 7f9697200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f969815ce30 con 0x7f96980aedc0 2024-03-06T07:29:41.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f96981561f0 0x7f96980ae6b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40020/0 (socket says 172.21.15.86:40020) 2024-03-06T07:29:41.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695e00640 1 -- 172.21.15.86:0/2082497194 learned_addr learned my addr 172.21.15.86:0/2082497194 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:41.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2360831604 0 0) 0x7f969815c240 con 0x7f96981561f0 2024-03-06T07:29:41.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f967c003520 con 0x7f96981561f0 2024-03-06T07:29:41.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f968fe00640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 0x7f96980cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1146354597 0 0) 0x7f967c003520 con 0x7f96981561f0 2024-03-06T07:29:41.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 msgr2=0x7f96980cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:41.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 0x7f96980cc2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 >> v1:172.21.15.194:6789/0 conn(0x7f96980cc800 legacy=0x7f969815f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f969815c7e0 con 0x7f96981561f0 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.399+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9688001670 con 0x7f96981561f0 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 >> v1:172.21.15.86:6789/0 conn(0x7f96981561f0 legacy=0x7f96980ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695400640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 0x7f967c0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695400640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f967c004060 0x7f967c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695400640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f967c004d40 0x7f967c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f969815ce30 con 0x7f967c004d40 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f967c00bc00 con 0x7f967c004060 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f967c00be00 con 0x7f96980aedc0 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9696800640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f967c004d40 0x7f967c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9695e00640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 0x7f967c0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f968fe00640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f967c004060 0x7f967c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9696800640 1 -- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 msgr2=0x7f967c0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9696800640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96980aedc0 0x7f967c0039e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9696800640 1 -- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f967c004060 msgr2=0x7f967c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9696800640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f967c004060 0x7f967c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f9696800640 1 -- 172.21.15.86:0/2082497194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f967c003520 con 0x7f967c004d40 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.400+0000 7f968fe00640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f967c004060 0x7f967c004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.401+0000 7f9696800640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f967c004d40 0x7f967c0051c0 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f9690001e50 tx=0x7f967c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6015cacced14ef6a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9690001a70 con 0x7f967c004d40 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9690003b70 con 0x7f967c004d40 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9695400640 1 -- 172.21.15.86:0/2082497194 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96900050d0 con 0x7f967c004d40 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9697200640 1 -- 172.21.15.86:0/2082497194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f967c004d40 msgr2=0x7f967c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9697200640 1 --2- 172.21.15.86:0/2082497194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f967c004d40 0x7f967c0051c0 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f9690001e50 tx=0x7f967c005830 comp rx=0 tx=0).stop 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9696800640 1 -- 172.21.15.86:0/2082497194 reap_dead start 2024-03-06T07:29:41.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9697200640 1 -- 172.21.15.86:0/2082497194 shutdown_connections 2024-03-06T07:29:41.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9697200640 1 -- 172.21.15.86:0/2082497194 >> 172.21.15.86:0/2082497194 conn(0x7f969806afc0 msgr2=0x7f969806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:41.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9697200640 1 -- 172.21.15.86:0/2082497194 shutdown_connections 2024-03-06T07:29:41.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.402+0000 7f9697200640 1 -- 172.21.15.86:0/2082497194 wait complete. 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.404+0000 7f9697200640 1 Processor -- start 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.404+0000 7f9697200640 1 -- start start 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.405+0000 7f9697200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f967c004d40 0x7f9698140190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.405+0000 7f9697200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96980af3b0 0x7f96980af810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.405+0000 7f9697200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f96980afd50 0x7f9698142b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.405+0000 7f9697200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f969806e360 con 0x7f96980afd50 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.405+0000 7f9697200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96980ccc90 con 0x7f96980af3b0 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.405+0000 7f9697200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96981565d0 con 0x7f967c004d40 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f96980afd50 0x7f9698142b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f96980afd50 0x7f9698142b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39392/0 (socket says 172.21.15.86:39392) 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 -- 172.21.15.86:0/3670029560 learned_addr learned my addr 172.21.15.86:0/3670029560 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:41.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9695e00640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f967c004d40 0x7f9698140190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f968fe00640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96980af3b0 0x7f96980af810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 -- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f967c004d40 msgr2=0x7f9698140190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f967c004d40 0x7f9698140190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 -- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96980af3b0 msgr2=0x7f96980af810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96980af3b0 0x7f96980af810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 -- 172.21.15.86:0/3670029560 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9698156e60 con 0x7f96980afd50 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f968fe00640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96980af3b0 0x7f96980af810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.406+0000 7f9696800640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f96980afd50 0x7f9698142b90 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f969002c060 tx=0x7f96900014a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.407+0000 7f968e000640 1 -- 172.21.15.86:0/3670029560 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9690007db0 con 0x7f96980afd50 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.407+0000 7f968e000640 1 -- 172.21.15.86:0/3670029560 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9690003970 con 0x7f96980afd50 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.407+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96980b01d0 con 0x7f96980afd50 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.407+0000 7f968e000640 1 -- 172.21.15.86:0/3670029560 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969002f6a0 con 0x7f96980afd50 2024-03-06T07:29:41.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.407+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9698140d50 con 0x7f96980afd50 2024-03-06T07:29:41.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.408+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f969804f330 con 0x7f96980afd50 2024-03-06T07:29:41.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.409+0000 7f968e000640 1 -- 172.21.15.86:0/3670029560 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9690041030 con 0x7f96980afd50 2024-03-06T07:29:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.410+0000 7f968e000640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f967406b500 0x7f967406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.410+0000 7f968e000640 1 -- 172.21.15.86:0/3670029560 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f96900aae20 con 0x7f96980afd50 2024-03-06T07:29:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.411+0000 7f9695e00640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f967406b500 0x7f967406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.411+0000 7f9695e00640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f967406b500 0x7f967406d9c0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f9688000c00 tx=0x7f9688009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:41.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.415+0000 7f968e000640 1 -- 172.21.15.86:0/3670029560 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9698156e60 con 0x7f96980afd50 2024-03-06T07:29:41.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.680+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f96980b2b10 con 0x7f96980afd50 2024-03-06T07:29:41.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.681+0000 7f968e000640 1 -- 172.21.15.86:0/3670029560 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v14) v1 ==== 80+0+5304 (secure 0 0 0) 0x7f9690033020 con 0x7f96980afd50 2024-03-06T07:29:41.684 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.685 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.686 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.687 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:29:41.688 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.689 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.690 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.691 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:29:41.692 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow r", 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:29:41.693 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:29:41.694 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.695 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:29:41.696 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:29:41.697 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:29:41.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f967406b500 msgr2=0x7f967406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f967406b500 0x7f967406d9c0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f9688000c00 tx=0x7f9688009040 comp rx=0 tx=0).stop 2024-03-06T07:29:41.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f96980afd50 msgr2=0x7f9698142b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f96980afd50 0x7f9698142b90 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f969002c060 tx=0x7f96900014a0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 shutdown_connections 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f967406b500 0x7f967406d9c0 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f96980afd50 0x7f9698142b90 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96980af3b0 0x7f96980af810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 --2- 172.21.15.86:0/3670029560 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f967c004d40 0x7f9698140190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.684+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 >> 172.21.15.86:0/3670029560 conn(0x7f969806afc0 msgr2=0x7f9698159e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.685+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 shutdown_connections 2024-03-06T07:29:41.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.685+0000 7f9697200640 1 -- 172.21.15.86:0/3670029560 wait complete. 2024-03-06T07:29:41.705 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:41.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.904+0000 7fbee8e00640 1 Processor -- start 2024-03-06T07:29:41.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.904+0000 7fbee8e00640 1 -- start start 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.906+0000 7fbee8e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 0x7fbee4071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.906+0000 7fbee8e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbee406cef0 con 0x7fbee4144c00 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.906+0000 7fbee8e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbee406d1d0 con 0x7fbee415f9d0 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.906+0000 7fbee8e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbee406d4b0 con 0x7fbee415fe60 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.906+0000 7fbee1e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fbee4144c00 0x7fbee406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40036/0 (socket says 172.21.15.86:40036) 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.906+0000 7fbee1e00640 1 -- 172.21.15.86:0/3670235616 learned_addr learned my addr 172.21.15.86:0/3670235616 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee2800640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 0x7fbee4071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3360201820 0 0) 0x7fbee406cef0 con 0x7fbee4144c00 2024-03-06T07:29:41.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbec8003520 con 0x7fbee4144c00 2024-03-06T07:29:41.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3936342158 0 0) 0x7fbec8003520 con 0x7fbee4144c00 2024-03-06T07:29:41.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 msgr2=0x7fbee4071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:41.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 0x7fbee4071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 >> v1:172.21.15.194:6789/0 conn(0x7fbee415f9d0 legacy=0x7fbee4070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbee406d790 con 0x7fbee4144c00 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fbed4001670 con 0x7fbee4144c00 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 >> v1:172.21.15.86:6789/0 conn(0x7fbee4144c00 legacy=0x7fbee406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbec8003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbec8004800 0x7fbec8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 0x7fbec8005200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbee406d4b0 con 0x7fbec80039e0 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbec800baf0 con 0x7fbec8004800 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbec800bcf0 con 0x7fbee415fe60 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.907+0000 7fbee1e00640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbec8003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbee2800640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 0x7fbec8005200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbed9400640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbec8004800 0x7fbec8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbee1e00640 1 -- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 msgr2=0x7fbec8005200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:41.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbee1e00640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415fe60 0x7fbec8005200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbee1e00640 1 -- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbec8004800 msgr2=0x7fbec8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:41.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbee1e00640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbec8004800 0x7fbec8004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbee1e00640 1 -- 172.21.15.86:0/3670235616 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbec8003520 con 0x7fbec80039e0 2024-03-06T07:29:41.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.908+0000 7fbed9400640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbec8004800 0x7fbec8004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:41.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.909+0000 7fbee1e00640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbec8003e00 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7fbed400c210 tx=0x7fbec8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a7be99fb05e6accf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.909+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbed4007d30 con 0x7fbec80039e0 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.909+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbed4030070 con 0x7fbec80039e0 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.909+0000 7fbee1400640 1 -- 172.21.15.86:0/3670235616 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbed402cd60 con 0x7fbec80039e0 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.910+0000 7fbee8e00640 1 -- 172.21.15.86:0/3670235616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 msgr2=0x7fbec8003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.910+0000 7fbee8e00640 1 --2- 172.21.15.86:0/3670235616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbec8003e00 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7fbed400c210 tx=0x7fbec8005830 comp rx=0 tx=0).stop 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.910+0000 7fbee2800640 1 -- 172.21.15.86:0/3670235616 reap_dead start 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.910+0000 7fbee8e00640 1 -- 172.21.15.86:0/3670235616 shutdown_connections 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.910+0000 7fbee8e00640 1 -- 172.21.15.86:0/3670235616 >> 172.21.15.86:0/3670235616 conn(0x7fbee40b1730 msgr2=0x7fbee40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.910+0000 7fbee8e00640 1 -- 172.21.15.86:0/3670235616 shutdown_connections 2024-03-06T07:29:41.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.910+0000 7fbee8e00640 1 -- 172.21.15.86:0/3670235616 wait complete. 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.912+0000 7fbee8e00640 1 Processor -- start 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.912+0000 7fbee8e00640 1 -- start start 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbee41f4690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee8e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbee4143060 0x7fbee4142790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee8e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415d390 0x7fbee415d820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee8e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbee406cb50 con 0x7fbec80039e0 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee8e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbee4160730 con 0x7fbee4143060 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee8e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbee406ccd0 con 0x7fbee415d390 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbee41f4690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbee41f4690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39402/0 (socket says 172.21.15.86:39402) 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 -- 172.21.15.86:0/3967922088 learned_addr learned my addr 172.21.15.86:0/3967922088 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:41.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 -- 172.21.15.86:0/3967922088 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415d390 msgr2=0x7fbee415d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415d390 0x7fbee415d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 -- 172.21.15.86:0/3967922088 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbee4143060 msgr2=0x7fbee4142790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:41.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbee4143060 0x7fbee4142790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:41.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.913+0000 7fbee1e00640 1 -- 172.21.15.86:0/3967922088 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbee41f4bd0 con 0x7fbec80039e0 2024-03-06T07:29:41.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.914+0000 7fbee1e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbee41f4690 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7fbed4001970 tx=0x7fbed4001a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:41.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.914+0000 7fbedaa00640 1 -- 172.21.15.86:0/3967922088 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbed4004bf0 con 0x7fbec80039e0 2024-03-06T07:29:41.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.914+0000 7fbedaa00640 1 -- 172.21.15.86:0/3967922088 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbed4004d90 con 0x7fbec80039e0 2024-03-06T07:29:41.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.914+0000 7fbedaa00640 1 -- 172.21.15.86:0/3967922088 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbed4007290 con 0x7fbec80039e0 2024-03-06T07:29:41.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.914+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbee41e5f90 con 0x7fbec80039e0 2024-03-06T07:29:41.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.914+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbee41e6870 con 0x7fbec80039e0 2024-03-06T07:29:41.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.915+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbee404f330 con 0x7fbec80039e0 2024-03-06T07:29:41.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.916+0000 7fbedaa00640 1 -- 172.21.15.86:0/3967922088 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbed4007430 con 0x7fbec80039e0 2024-03-06T07:29:41.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.917+0000 7fbedaa00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbec006b4d0 0x7fbec006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:41.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.917+0000 7fbedaa00640 1 -- 172.21.15.86:0/3967922088 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fbed40ab1a0 con 0x7fbec80039e0 2024-03-06T07:29:41.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.917+0000 7fbed9400640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbec006b4d0 0x7fbec006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:41.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.918+0000 7fbed9400640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbec006b4d0 0x7fbec006d990 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fbee4143480 tx=0x7fbecc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:41.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:41.921+0000 7fbedaa00640 1 -- 172.21.15.86:0/3967922088 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbee41f4bd0 con 0x7fbec80039e0 2024-03-06T07:29:42.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.117+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fbee4072890 con 0x7fbec80039e0 2024-03-06T07:29:42.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.119+0000 7fbedaa00640 1 -- 172.21.15.86:0/3967922088 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v54) v1 ==== 74+0+12903 (secure 0 0 0) 0x7fbed407a370 con 0x7fbec80039e0 2024-03-06T07:29:42.122 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:42.122 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":54,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:41.193963+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:42.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbec006b4d0 msgr2=0x7fbec006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbec006b4d0 0x7fbec006d990 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fbee4143480 tx=0x7fbecc009040 comp rx=0 tx=0).stop 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 msgr2=0x7fbee41f4690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbee41f4690 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7fbed4001970 tx=0x7fbed4001a40 comp rx=0 tx=0).stop 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 shutdown_connections 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbec006b4d0 0x7fbec006d990 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbee415d390 0x7fbee415d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbee4143060 0x7fbee4142790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 --2- 172.21.15.86:0/3967922088 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbec80039e0 0x7fbee41f4690 unknown :-1 s=CLOSED pgs=631 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 >> 172.21.15.86:0/3967922088 conn(0x7fbee40b1730 msgr2=0x7fbee406e550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 shutdown_connections 2024-03-06T07:29:42.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.122+0000 7fbee8e00640 1 -- 172.21.15.86:0/3967922088 wait complete. 2024-03-06T07:29:42.143 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:29:42.143 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:29:42.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.347+0000 7fad01400640 1 Processor -- start 2024-03-06T07:29:42.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.348+0000 7fad01400640 1 -- start start 2024-03-06T07:29:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7fad01400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7facfc15fc10 0x7facfc070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7fad01400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7facfc15fff0 0x7facfc160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7fad01400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7facfc0b46c0 con 0x7facfc0ce4a0 2024-03-06T07:29:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7fad01400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7facfc0b49a0 con 0x7facfc15fff0 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7fad01400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7facfc072c90 con 0x7facfc15fc10 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7facf3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7facfc15fff0 0x7facfc160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7facf3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7facfc15fff0 0x7facfc160450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39410/0 (socket says 172.21.15.86:39410) 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.349+0000 7facf3e00640 1 -- 172.21.15.86:0/2588654753 learned_addr learned my addr 172.21.15.86:0/2588654753 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.350+0000 7facf3e00640 1 -- 172.21.15.86:0/2588654753 >> v2:172.21.15.194:3300/0 conn(0x7facfc15fc10 msgr2=0x7facfc070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.350+0000 7facf3e00640 1 --2- 172.21.15.86:0/2588654753 >> v2:172.21.15.194:3300/0 conn(0x7facfc15fc10 0x7facfc070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.350+0000 7facf3e00640 1 -- 172.21.15.86:0/2588654753 >> v1:172.21.15.194:6789/0 conn(0x7facfc0ce4a0 legacy=0x7facfc0ce8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.350+0000 7facf3e00640 1 -- 172.21.15.86:0/2588654753 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facfc160990 con 0x7facfc15fff0 2024-03-06T07:29:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.351+0000 7facf3e00640 1 --2- 172.21.15.86:0/2588654753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc160450 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7facfc0cf770 tx=0x7face402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c2eb499b3eb0f03b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.351+0000 7facf9e00640 1 -- 172.21.15.86:0/2588654753 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4035020 con 0x7facfc15fff0 2024-03-06T07:29:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.351+0000 7facf9e00640 1 -- 172.21.15.86:0/2588654753 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7face4031870 con 0x7facfc15fff0 2024-03-06T07:29:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.351+0000 7facf9e00640 1 -- 172.21.15.86:0/2588654753 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4031c40 con 0x7facfc15fff0 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 -- 172.21.15.86:0/2588654753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 msgr2=0x7facfc160450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 --2- 172.21.15.86:0/2588654753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc160450 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7facfc0cf770 tx=0x7face402b910 comp rx=0 tx=0).stop 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 -- 172.21.15.86:0/2588654753 shutdown_connections 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 --2- 172.21.15.86:0/2588654753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc160450 unknown :-1 s=CLOSED pgs=632 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 --2- 172.21.15.86:0/2588654753 >> v2:172.21.15.194:3300/0 conn(0x7facfc15fc10 0x7facfc070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 -- 172.21.15.86:0/2588654753 >> 172.21.15.86:0/2588654753 conn(0x7facfc0b1940 msgr2=0x7facfc0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 -- 172.21.15.86:0/2588654753 shutdown_connections 2024-03-06T07:29:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.352+0000 7fad01400640 1 -- 172.21.15.86:0/2588654753 wait complete. 2024-03-06T07:29:42.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.354+0000 7fad01400640 1 Processor -- start 2024-03-06T07:29:42.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.354+0000 7fad01400640 1 -- start start 2024-03-06T07:29:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7fad01400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7facfc0ce4a0 0x7facfc15ae00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7fad01400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7facfc15fc10 0x7facfc152200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7fad01400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc14c580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7fad01400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facfc072a30 con 0x7facfc15fff0 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7fad01400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facfc0728b0 con 0x7facfc0ce4a0 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7fad01400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facfc15c2e0 con 0x7facfc15fc10 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7facfb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc14c580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7facfb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc14c580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39426/0 (socket says 172.21.15.86:39426) 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.355+0000 7facfb200640 1 -- 172.21.15.86:0/4072153454 learned_addr learned my addr 172.21.15.86:0/4072153454 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facf3e00640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7facfc15fc10 0x7facfc152200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facfa800640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7facfc0ce4a0 0x7facfc15ae00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facfb200640 1 -- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7facfc15fc10 msgr2=0x7facfc152200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facfb200640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7facfc15fc10 0x7facfc152200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facfb200640 1 -- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7facfc0ce4a0 msgr2=0x7facfc15ae00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facfb200640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7facfc0ce4a0 0x7facfc15ae00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facfb200640 1 -- 172.21.15.86:0/4072153454 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facfc159d60 con 0x7facfc15fff0 2024-03-06T07:29:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facfb200640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc14c580 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7facf400c7b0 tx=0x7facf400cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facf2a00640 1 -- 172.21.15.86:0/4072153454 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facf4017070 con 0x7facfc15fff0 2024-03-06T07:29:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facf2a00640 1 -- 172.21.15.86:0/4072153454 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7facf4003da0 con 0x7facfc15fff0 2024-03-06T07:29:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7facf2a00640 1 -- 172.21.15.86:0/4072153454 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facf40052b0 con 0x7facfc15fff0 2024-03-06T07:29:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7facfc15a040 con 0x7facfc15fff0 2024-03-06T07:29:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.356+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7facfc15a600 con 0x7facfc15fff0 2024-03-06T07:29:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.357+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7facfc04f330 con 0x7facfc15fff0 2024-03-06T07:29:42.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.358+0000 7facf2a00640 1 -- 172.21.15.86:0/4072153454 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7facf401a070 con 0x7facfc15fff0 2024-03-06T07:29:42.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.359+0000 7facf2a00640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7facd806b4d0 0x7facd806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.359+0000 7facf2a00640 1 -- 172.21.15.86:0/4072153454 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 4857+0+0 (secure 0 0 0) 0x7facf4089500 con 0x7facfc15fff0 2024-03-06T07:29:42.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.359+0000 7facfa800640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7facd806b4d0 0x7facd806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.360+0000 7facfa800640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7facd806b4d0 0x7facd806d990 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7facfc15b5b0 tx=0x7facec008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:42.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.363+0000 7facf2a00640 1 -- 172.21.15.86:0/4072153454 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7facf40586c0 con 0x7facfc15fff0 2024-03-06T07:29:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.561+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7facfc152850 con 0x7facfc15fff0 2024-03-06T07:29:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.562+0000 7facf2a00640 1 -- 172.21.15.86:0/4072153454 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v54) v1 ==== 93+0+10 (secure 0 0 0) 0x7facf4057fc0 con 0x7facfc15fff0 2024-03-06T07:29:42.565 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:29:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.564+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7facd806b4d0 msgr2=0x7facd806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7facd806b4d0 0x7facd806d990 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7facfc15b5b0 tx=0x7facec008040 comp rx=0 tx=0).stop 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 msgr2=0x7facfc14c580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc14c580 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7facf400c7b0 tx=0x7facf400cc70 comp rx=0 tx=0).stop 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 shutdown_connections 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7facd806b4d0 0x7facd806d990 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7facfc15fff0 0x7facfc14c580 unknown :-1 s=CLOSED pgs=633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7facfc15fc10 0x7facfc152200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 --2- 172.21.15.86:0/4072153454 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7facfc0ce4a0 0x7facfc15ae00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 >> 172.21.15.86:0/4072153454 conn(0x7facfc0b1940 msgr2=0x7facfc0b3200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 shutdown_connections 2024-03-06T07:29:42.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.565+0000 7fad01400640 1 -- 172.21.15.86:0/4072153454 wait complete. 2024-03-06T07:29:42.586 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:29:42.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.792+0000 7fbe38200640 1 Processor -- start 2024-03-06T07:29:42.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.792+0000 7fbe38200640 1 -- start start 2024-03-06T07:29:42.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe38200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe3015f9d0 0x7fbe3015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe38200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe30070b10 0x7fbe30070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe38200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe3006cef0 con 0x7fbe30144c00 2024-03-06T07:29:42.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe38200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbe30160370 con 0x7fbe30070b10 2024-03-06T07:29:42.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe38200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe3006d1d0 con 0x7fbe3015f9d0 2024-03-06T07:29:42.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe36200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe30070b10 0x7fbe30070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe36200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe30070b10 0x7fbe30070f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39440/0 (socket says 172.21.15.86:39440) 2024-03-06T07:29:42.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.794+0000 7fbe36200640 1 -- 172.21.15.86:0/3164745691 learned_addr learned my addr 172.21.15.86:0/3164745691 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:42.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.795+0000 7fbe34e00640 1 --2- 172.21.15.86:0/3164745691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe3015f9d0 0x7fbe3015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.795+0000 7fbe36200640 1 -- 172.21.15.86:0/3164745691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe3015f9d0 msgr2=0x7fbe3015fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:42.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.795+0000 7fbe36200640 1 --2- 172.21.15.86:0/3164745691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe3015f9d0 0x7fbe3015fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.795+0000 7fbe36200640 1 -- 172.21.15.86:0/3164745691 >> v1:172.21.15.194:6789/0 conn(0x7fbe30144c00 legacy=0x7fbe3006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:42.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.795+0000 7fbe36200640 1 -- 172.21.15.86:0/3164745691 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe3006d4b0 con 0x7fbe30070b10 2024-03-06T07:29:42.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.796+0000 7fbe36200640 1 --2- 172.21.15.86:0/3164745691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30070b10 0x7fbe30070f60 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7fbe28009e50 tx=0x7fbe2802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=91db5579e626846c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:42.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.796+0000 7fbe2fe00640 1 -- 172.21.15.86:0/3164745691 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe2802ca70 con 0x7fbe30070b10 2024-03-06T07:29:42.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.796+0000 7fbe2fe00640 1 -- 172.21.15.86:0/3164745691 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe2802cc10 con 0x7fbe30070b10 2024-03-06T07:29:42.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.796+0000 7fbe2fe00640 1 -- 172.21.15.86:0/3164745691 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe280343f0 con 0x7fbe30070b10 2024-03-06T07:29:42.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 -- 172.21.15.86:0/3164745691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30070b10 msgr2=0x7fbe30070f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 --2- 172.21.15.86:0/3164745691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30070b10 0x7fbe30070f60 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7fbe28009e50 tx=0x7fbe2802bed0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 -- 172.21.15.86:0/3164745691 shutdown_connections 2024-03-06T07:29:42.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 --2- 172.21.15.86:0/3164745691 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30070b10 0x7fbe30070f60 unknown :-1 s=CLOSED pgs=634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 --2- 172.21.15.86:0/3164745691 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe3015f9d0 0x7fbe3015fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 -- 172.21.15.86:0/3164745691 >> 172.21.15.86:0/3164745691 conn(0x7fbe300b1730 msgr2=0x7fbe300b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:42.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 -- 172.21.15.86:0/3164745691 shutdown_connections 2024-03-06T07:29:42.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.797+0000 7fbe38200640 1 -- 172.21.15.86:0/3164745691 wait complete. 2024-03-06T07:29:42.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.798+0000 7fbe38200640 1 Processor -- start 2024-03-06T07:29:42.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.799+0000 7fbe38200640 1 -- start start 2024-03-06T07:29:42.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.799+0000 7fbe38200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe30070b10 0x7fbe30139670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.799+0000 7fbe38200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30144c00 0x7fbe300ae270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.799+0000 7fbe38200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe3015f9d0 0x7fbe3014c5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.799+0000 7fbe38200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe3006cb50 con 0x7fbe30144c00 2024-03-06T07:29:42.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.799+0000 7fbe38200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe30160730 con 0x7fbe3015f9d0 2024-03-06T07:29:42.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.799+0000 7fbe38200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe3006ccd0 con 0x7fbe30070b10 2024-03-06T07:29:42.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30144c00 0x7fbe300ae270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30144c00 0x7fbe300ae270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39446/0 (socket says 172.21.15.86:39446) 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 -- 172.21.15.86:0/1964148616 learned_addr learned my addr 172.21.15.86:0/1964148616 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe35800640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe30070b10 0x7fbe30139670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 -- 172.21.15.86:0/1964148616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe30070b10 msgr2=0x7fbe30139670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe30070b10 0x7fbe30139670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 -- 172.21.15.86:0/1964148616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe3015f9d0 msgr2=0x7fbe3014c5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe3015f9d0 0x7fbe3014c5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 -- 172.21.15.86:0/1964148616 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe3014d0d0 con 0x7fbe30144c00 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.800+0000 7fbe34e00640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30144c00 0x7fbe300ae270 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c00c7f0 tx=0x7fbe1c00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.801+0000 7fbe2d600640 1 -- 172.21.15.86:0/1964148616 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1c015070 con 0x7fbe30144c00 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.801+0000 7fbe2d600640 1 -- 172.21.15.86:0/1964148616 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe1c004240 con 0x7fbe30144c00 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.801+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbe3014d3b0 con 0x7fbe30144c00 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.801+0000 7fbe2d600640 1 -- 172.21.15.86:0/1964148616 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1c003610 con 0x7fbe30144c00 2024-03-06T07:29:42.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.801+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe300add50 con 0x7fbe30144c00 2024-03-06T07:29:42.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.802+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbe300b1b10 con 0x7fbe30144c00 2024-03-06T07:29:42.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.803+0000 7fbe2d600640 1 -- 172.21.15.86:0/1964148616 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbe1c0039c0 con 0x7fbe30144c00 2024-03-06T07:29:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.804+0000 7fbe2d600640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbe1006b4d0 0x7fbe1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.804+0000 7fbe2d600640 1 -- 172.21.15.86:0/1964148616 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fbe1c08a380 con 0x7fbe30144c00 2024-03-06T07:29:42.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.804+0000 7fbe35800640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbe1006b4d0 0x7fbe1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:42.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.805+0000 7fbe35800640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbe1006b4d0 0x7fbe1006d990 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fbe3014a450 tx=0x7fbe24009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:42.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:42.808+0000 7fbe2d600640 1 -- 172.21.15.86:0/1964148616 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbe300b1b10 con 0x7fbe30144c00 2024-03-06T07:29:43.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.008+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7fbe300ad860 con 0x7fbe30144c00 2024-03-06T07:29:43.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.009+0000 7fbe2d600640 1 -- 172.21.15.86:0/1964148616 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v54) v1 ==== 84+0+65 (secure 0 0 0) 0x7fbe1c05de20 con 0x7fbe30144c00 2024-03-06T07:29:43.011 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:43.011 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:29:43.012 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:29:43.012 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:29:43.012 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:29:43.012 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:29:43.012 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:29:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbe1006b4d0 msgr2=0x7fbe1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbe1006b4d0 0x7fbe1006d990 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fbe3014a450 tx=0x7fbe24009040 comp rx=0 tx=0).stop 2024-03-06T07:29:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30144c00 msgr2=0x7fbe300ae270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30144c00 0x7fbe300ae270 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c00c7f0 tx=0x7fbe1c00ccb0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 shutdown_connections 2024-03-06T07:29:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbe1006b4d0 0x7fbe1006d990 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe3015f9d0 0x7fbe3014c5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe30144c00 0x7fbe300ae270 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 --2- 172.21.15.86:0/1964148616 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe30070b10 0x7fbe30139670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.011+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 >> 172.21.15.86:0/1964148616 conn(0x7fbe300b1730 msgr2=0x7fbe30070ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.012+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 shutdown_connections 2024-03-06T07:29:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.012+0000 7fbe38200640 1 -- 172.21.15.86:0/1964148616 wait complete. 2024-03-06T07:29:43.032 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:29:43.032 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:29:43.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.242+0000 7feed5a00640 1 Processor -- start 2024-03-06T07:29:43.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.242+0000 7feed5a00640 1 -- start start 2024-03-06T07:29:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feed5a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0144940 0x7feed0070d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feed5a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7feed0160470 0x7feed0071860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feed5a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7feed006afc0 0x7feed006b450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feed5a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feed015ffe0 con 0x7feed0160470 2024-03-06T07:29:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feed5a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feed01608d0 con 0x7feed006afc0 2024-03-06T07:29:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feed5a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feed0072090 con 0x7feed0144940 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7feed0160470 0x7feed0071860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7feed0160470 0x7feed0071860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39448/0 (socket says 172.21.15.86:39448) 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 -- 172.21.15.86:0/589892291 learned_addr learned my addr 172.21.15.86:0/589892291 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 -- 172.21.15.86:0/589892291 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0144940 msgr2=0x7feed0070d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecfe00640 1 --2- 172.21.15.86:0/589892291 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0144940 0x7feed0070d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 --2- 172.21.15.86:0/589892291 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0144940 0x7feed0070d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 -- 172.21.15.86:0/589892291 >> v2:172.21.15.194:3300/0 conn(0x7feed006afc0 msgr2=0x7feed006b450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feed5000640 1 --2- 172.21.15.86:0/589892291 >> v2:172.21.15.194:3300/0 conn(0x7feed006afc0 0x7feed006b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 --2- 172.21.15.86:0/589892291 >> v2:172.21.15.194:3300/0 conn(0x7feed006afc0 0x7feed006b450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.244+0000 7feecf400640 1 -- 172.21.15.86:0/589892291 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feed0072370 con 0x7feed0160470 2024-03-06T07:29:43.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.245+0000 7feecf400640 1 --2- 172.21.15.86:0/589892291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0160470 0x7feed0071860 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7feed015c3a0 tx=0x7feec002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=145d0a71ff3db17e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:43.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.245+0000 7feecea00640 1 -- 172.21.15.86:0/589892291 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feec0035020 con 0x7feed0160470 2024-03-06T07:29:43.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.245+0000 7feecea00640 1 -- 172.21.15.86:0/589892291 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feec0031870 con 0x7feed0160470 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.245+0000 7feecea00640 1 -- 172.21.15.86:0/589892291 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feec0031c40 con 0x7feed0160470 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 -- 172.21.15.86:0/589892291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0160470 msgr2=0x7feed0071860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 --2- 172.21.15.86:0/589892291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0160470 0x7feed0071860 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7feed015c3a0 tx=0x7feec002b910 comp rx=0 tx=0).stop 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 -- 172.21.15.86:0/589892291 shutdown_connections 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 --2- 172.21.15.86:0/589892291 >> v2:172.21.15.194:3300/0 conn(0x7feed006afc0 0x7feed006b450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 --2- 172.21.15.86:0/589892291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0160470 0x7feed0071860 unknown :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 --2- 172.21.15.86:0/589892291 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0144940 0x7feed0070d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 -- 172.21.15.86:0/589892291 >> 172.21.15.86:0/589892291 conn(0x7feed00b18a0 msgr2=0x7feed00b1cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 -- 172.21.15.86:0/589892291 shutdown_connections 2024-03-06T07:29:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.246+0000 7feed5a00640 1 -- 172.21.15.86:0/589892291 wait complete. 2024-03-06T07:29:43.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.248+0000 7feed5a00640 1 Processor -- start 2024-03-06T07:29:43.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.248+0000 7feed5a00640 1 -- start start 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feed5a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feed006afc0 0x7feed013d730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feed5a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0144940 0x7feed0150b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feed5a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0160470 0x7feed00aa930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feed5a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feed0071da0 con 0x7feed0144940 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feed5a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feed015fc70 con 0x7feed006afc0 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feed5a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feed0144d20 con 0x7feed0160470 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0144940 0x7feed0150b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0144940 0x7feed0150b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39464/0 (socket says 172.21.15.86:39464) 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 -- 172.21.15.86:0/49277862 learned_addr learned my addr 172.21.15.86:0/49277862 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecfe00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feed006afc0 0x7feed013d730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feed5000640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0160470 0x7feed00aa930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:43.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 -- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0160470 msgr2=0x7feed00aa930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:43.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0160470 0x7feed00aa930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 -- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feed006afc0 msgr2=0x7feed013d730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:43.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feed006afc0 0x7feed013d730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:43.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecf400640 1 -- 172.21.15.86:0/49277862 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feed015c2b0 con 0x7feed0144940 2024-03-06T07:29:43.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.249+0000 7feecfe00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feed006afc0 0x7feed013d730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.250+0000 7feecf400640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0144940 0x7feed0150b70 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7feec002b950 tx=0x7feec00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.250+0000 7feec7e00640 1 -- 172.21.15.86:0/49277862 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feec0040070 con 0x7feed0144940 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.250+0000 7feec7e00640 1 -- 172.21.15.86:0/49277862 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feec003a420 con 0x7feed0144940 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.250+0000 7feec7e00640 1 -- 172.21.15.86:0/49277862 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feec0035040 con 0x7feed0144940 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.250+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feed015c590 con 0x7feed0144940 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.250+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feed015cb50 con 0x7feed0144940 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.252+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feed004f330 con 0x7feed0144940 2024-03-06T07:29:43.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.252+0000 7feec7e00640 1 -- 172.21.15.86:0/49277862 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feec004b060 con 0x7feed0144940 2024-03-06T07:29:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.253+0000 7feec7e00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feeac06b4d0 0x7feeac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.253+0000 7feec7e00640 1 -- 172.21.15.86:0/49277862 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 4857+0+0 (secure 0 0 0) 0x7feec00abea0 con 0x7feed0144940 2024-03-06T07:29:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.253+0000 7feecfe00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feeac06b4d0 0x7feeac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:43.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.254+0000 7feecfe00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feeac06b4d0 0x7feeac06d990 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7feeb8008890 tx=0x7feeb8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:43.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.259+0000 7feec7e00640 1 -- 172.21.15.86:0/49277862 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feec007b060 con 0x7feed0144940 2024-03-06T07:29:43.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:43.452+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7feed0074a50 con 0x7feed0144940 2024-03-06T07:29:44.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.218+0000 7feec7e00640 1 -- 172.21.15.86:0/49277862 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v55) v1 ==== 138+0+0 (secure 0 0 0) 0x7feec007a960 con 0x7feed0144940 2024-03-06T07:29:44.220 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feeac06b4d0 msgr2=0x7feeac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feeac06b4d0 0x7feeac06d990 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7feeb8008890 tx=0x7feeb8008040 comp rx=0 tx=0).stop 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0144940 msgr2=0x7feed0150b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0144940 0x7feed0150b70 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7feec002b950 tx=0x7feec00046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 shutdown_connections 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7feeac06b4d0 0x7feeac06d990 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feed0160470 0x7feed00aa930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7feed0144940 0x7feed0150b70 unknown :-1 s=CLOSED pgs=637 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 --2- 172.21.15.86:0/49277862 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feed006afc0 0x7feed013d730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.222+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 >> 172.21.15.86:0/49277862 conn(0x7feed00b18a0 msgr2=0x7feed00b1c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:44.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.223+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 shutdown_connections 2024-03-06T07:29:44.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.223+0000 7feed5a00640 1 -- 172.21.15.86:0/49277862 wait complete. 2024-03-06T07:29:44.253 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:29:44.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.447+0000 7f2e7a800640 1 Processor -- start 2024-03-06T07:29:44.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.447+0000 7f2e7a800640 1 -- start start 2024-03-06T07:29:44.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.449+0000 7f2e7a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3650 0x7f2e740727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:44.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.449+0000 7f2e7a800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f2e740cda90 0x7f2e7406c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:44.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.449+0000 7f2e7a800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2e740719e0 con 0x7f2e740b3df0 2024-03-06T07:29:44.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.449+0000 7f2e7a800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2e74072310 con 0x7f2e740cda90 2024-03-06T07:29:44.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.449+0000 7f2e7a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e74071cc0 con 0x7f2e740b3650 2024-03-06T07:29:44.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.450+0000 7f2e6aa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3650 0x7f2e740727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:44.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.450+0000 7f2e6aa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3650 0x7f2e740727a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55696/0 (socket says 172.21.15.86:55696) 2024-03-06T07:29:44.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.450+0000 7f2e6aa00640 1 -- 172.21.15.86:0/2640438988 learned_addr learned my addr 172.21.15.86:0/2640438988 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:44.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.450+0000 7f2e73e00640 1 --2- 172.21.15.86:0/2640438988 >> v2:172.21.15.194:3300/0 conn(0x7f2e740cda90 0x7f2e7406c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:44.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.451+0000 7f2e6aa00640 1 -- 172.21.15.86:0/2640438988 >> v2:172.21.15.194:3300/0 conn(0x7f2e740cda90 msgr2=0x7f2e7406c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:44.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.451+0000 7f2e6aa00640 1 --2- 172.21.15.86:0/2640438988 >> v2:172.21.15.194:3300/0 conn(0x7f2e740cda90 0x7f2e7406c990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.451+0000 7f2e6aa00640 1 -- 172.21.15.86:0/2640438988 >> v1:172.21.15.194:6789/0 conn(0x7f2e740b3df0 legacy=0x7f2e740b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:44.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.451+0000 7f2e6aa00640 1 -- 172.21.15.86:0/2640438988 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e74071fa0 con 0x7f2e740b3650 2024-03-06T07:29:44.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.452+0000 7f2e6aa00640 1 --2- 172.21.15.86:0/2640438988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3650 0x7f2e740727a0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c009870 tx=0x7f2e5c02b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d307abffed3f20f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:44.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.452+0000 7f2e72a00640 1 -- 172.21.15.86:0/2640438988 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e5c034070 con 0x7f2e740b3650 2024-03-06T07:29:44.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.452+0000 7f2e72a00640 1 -- 172.21.15.86:0/2640438988 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e5c030840 con 0x7f2e740b3650 2024-03-06T07:29:44.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.452+0000 7f2e72a00640 1 -- 172.21.15.86:0/2640438988 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e5c030c10 con 0x7f2e740b3650 2024-03-06T07:29:44.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 -- 172.21.15.86:0/2640438988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3650 msgr2=0x7f2e740727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:44.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 --2- 172.21.15.86:0/2640438988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3650 0x7f2e740727a0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c009870 tx=0x7f2e5c02b980 comp rx=0 tx=0).stop 2024-03-06T07:29:44.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 -- 172.21.15.86:0/2640438988 shutdown_connections 2024-03-06T07:29:44.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 --2- 172.21.15.86:0/2640438988 >> v2:172.21.15.194:3300/0 conn(0x7f2e740cda90 0x7f2e7406c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 --2- 172.21.15.86:0/2640438988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3650 0x7f2e740727a0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 -- 172.21.15.86:0/2640438988 >> 172.21.15.86:0/2640438988 conn(0x7f2e740b1940 msgr2=0x7f2e740b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:44.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 -- 172.21.15.86:0/2640438988 shutdown_connections 2024-03-06T07:29:44.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.453+0000 7f2e7a800640 1 -- 172.21.15.86:0/2640438988 wait complete. 2024-03-06T07:29:44.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.455+0000 7f2e7a800640 1 Processor -- start 2024-03-06T07:29:44.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.455+0000 7f2e7a800640 1 -- start start 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.456+0000 7f2e7a800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e740b3650 0x7f2e741514e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.456+0000 7f2e7a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3df0 0x7f2e7414d9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.456+0000 7f2e7a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e740cda90 0x7f2e740afae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e7a800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e7415b3c0 con 0x7f2e740cda90 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e7a800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e7415b240 con 0x7f2e740b3650 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e7a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e740cf510 con 0x7f2e740b3df0 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e740cda90 0x7f2e740afae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:44.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e740cda90 0x7f2e740afae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39472/0 (socket says 172.21.15.86:39472) 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 -- 172.21.15.86:0/4289064953 learned_addr learned my addr 172.21.15.86:0/4289064953 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73400640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e740b3650 0x7f2e741514e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e6aa00640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3df0 0x7f2e7414d9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 -- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3df0 msgr2=0x7f2e7414d9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3df0 0x7f2e7414d9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 -- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e740b3650 msgr2=0x7f2e741514e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e740b3650 0x7f2e741514e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.457+0000 7f2e73e00640 1 -- 172.21.15.86:0/4289064953 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e74152680 con 0x7f2e740cda90 2024-03-06T07:29:44.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.458+0000 7f2e73e00640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e740cda90 0x7f2e740afae0 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f2e6c00efc0 tx=0x7f2e6c00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:44.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.458+0000 7f2e6be00640 1 -- 172.21.15.86:0/4289064953 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e6c018020 con 0x7f2e740cda90 2024-03-06T07:29:44.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.458+0000 7f2e6be00640 1 -- 172.21.15.86:0/4289064953 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e6c014480 con 0x7f2e740cda90 2024-03-06T07:29:44.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.458+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2e7414e030 con 0x7f2e740cda90 2024-03-06T07:29:44.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.458+0000 7f2e6be00640 1 -- 172.21.15.86:0/4289064953 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e6c014870 con 0x7f2e740cda90 2024-03-06T07:29:44.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.458+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2e7414e8d0 con 0x7f2e740cda90 2024-03-06T07:29:44.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.459+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2e7414e420 con 0x7f2e740cda90 2024-03-06T07:29:44.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.460+0000 7f2e6be00640 1 -- 172.21.15.86:0/4289064953 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2e6c005d10 con 0x7f2e740cda90 2024-03-06T07:29:44.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.461+0000 7f2e6be00640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e5006b4d0 0x7f2e5006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:44.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.461+0000 7f2e6be00640 1 -- 172.21.15.86:0/4289064953 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f2e6c08b430 con 0x7f2e740cda90 2024-03-06T07:29:44.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.462+0000 7f2e73400640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e5006b4d0 0x7f2e5006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:44.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.462+0000 7f2e73400640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e5006b4d0 0x7f2e5006d990 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f2e64007fd0 tx=0x7f2e64009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:44.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.466+0000 7f2e6be00640 1 -- 172.21.15.86:0/4289064953 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2e6c022030 con 0x7f2e740cda90 2024-03-06T07:29:44.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:44.656+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f2e7404f330 con 0x7f2e740cda90 2024-03-06T07:29:45.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.224+0000 7f2e6be00640 1 -- 172.21.15.86:0/4289064953 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v56) v1 ==== 170+0+0 (secure 0 0 0) 0x7f2e6c010040 con 0x7f2e740cda90 2024-03-06T07:29:45.226 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:29:45.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.228+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e5006b4d0 msgr2=0x7f2e5006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:45.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.228+0000 7f2e7a800640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e5006b4d0 0x7f2e5006d990 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f2e64007fd0 tx=0x7f2e64009040 comp rx=0 tx=0).stop 2024-03-06T07:29:45.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.228+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e740cda90 msgr2=0x7f2e740afae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:45.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.228+0000 7f2e7a800640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e740cda90 0x7f2e740afae0 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f2e6c00efc0 tx=0x7f2e6c00c490 comp rx=0 tx=0).stop 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.228+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 shutdown_connections 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.229+0000 7f2e7a800640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e5006b4d0 0x7f2e5006d990 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.229+0000 7f2e7a800640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e740cda90 0x7f2e740afae0 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.229+0000 7f2e7a800640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e740b3df0 0x7f2e7414d9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.229+0000 7f2e7a800640 1 --2- 172.21.15.86:0/4289064953 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e740b3650 0x7f2e741514e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.229+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 >> 172.21.15.86:0/4289064953 conn(0x7f2e740b1940 msgr2=0x7f2e7414fdf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.229+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 shutdown_connections 2024-03-06T07:29:45.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.229+0000 7f2e7a800640 1 -- 172.21.15.86:0/4289064953 wait complete. 2024-03-06T07:29:45.261 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:29:45.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.443+0000 7f3158800640 1 Processor -- start 2024-03-06T07:29:45.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.444+0000 7f3158800640 1 -- start start 2024-03-06T07:29:45.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.445+0000 7f3158800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f3150070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:45.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.445+0000 7f3158800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f315015fff0 0x7f3150160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:45.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.445+0000 7f3158800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f31500ce4a0 0x7f31500ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:45.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.445+0000 7f3158800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f31500b46c0 con 0x7f31500ce4a0 2024-03-06T07:29:45.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.445+0000 7f3158800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f31500b49a0 con 0x7f315015fff0 2024-03-06T07:29:45.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.445+0000 7f3158800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3150072c90 con 0x7f315015fc10 2024-03-06T07:29:45.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f31500ce4a0 0x7f31500ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f31500ce4a0 0x7f31500ce930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39480/0 (socket says 172.21.15.86:39480) 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 -- 172.21.15.86:0/1966218164 learned_addr learned my addr 172.21.15.86:0/1966218164 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3155e00640 1 --2- 172.21.15.86:0/1966218164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f3150070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3155400640 1 --2- 172.21.15.86:0/1966218164 >> v2:172.21.15.194:3300/0 conn(0x7f315015fff0 0x7f3150160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 -- 172.21.15.86:0/1966218164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 msgr2=0x7f3150070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 --2- 172.21.15.86:0/1966218164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f3150070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 -- 172.21.15.86:0/1966218164 >> v2:172.21.15.194:3300/0 conn(0x7f315015fff0 msgr2=0x7f3150160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 --2- 172.21.15.86:0/1966218164 >> v2:172.21.15.194:3300/0 conn(0x7f315015fff0 0x7f3150160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.446+0000 7f3156800640 1 -- 172.21.15.86:0/1966218164 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3150160990 con 0x7f31500ce4a0 2024-03-06T07:29:45.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.447+0000 7f3156800640 1 --2- 172.21.15.86:0/1966218164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f31500ce930 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f31500749e0 tx=0x7f314c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4f7d9d7441e337aa server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.447+0000 7f3154a00640 1 -- 172.21.15.86:0/1966218164 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f314c02ca70 con 0x7f31500ce4a0 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3154a00640 1 -- 172.21.15.86:0/1966218164 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f314c02cc10 con 0x7f31500ce4a0 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3154a00640 1 -- 172.21.15.86:0/1966218164 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f314c0343f0 con 0x7f31500ce4a0 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 -- 172.21.15.86:0/1966218164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 msgr2=0x7f31500ce930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 --2- 172.21.15.86:0/1966218164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f31500ce930 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f31500749e0 tx=0x7f314c02bed0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 -- 172.21.15.86:0/1966218164 shutdown_connections 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 --2- 172.21.15.86:0/1966218164 >> v2:172.21.15.194:3300/0 conn(0x7f315015fff0 0x7f3150160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 --2- 172.21.15.86:0/1966218164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f31500ce930 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 --2- 172.21.15.86:0/1966218164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f3150070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 -- 172.21.15.86:0/1966218164 >> 172.21.15.86:0/1966218164 conn(0x7f31500b1940 msgr2=0x7f31500b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 -- 172.21.15.86:0/1966218164 shutdown_connections 2024-03-06T07:29:45.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.448+0000 7f3158800640 1 -- 172.21.15.86:0/1966218164 wait complete. 2024-03-06T07:29:45.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.450+0000 7f3158800640 1 Processor -- start 2024-03-06T07:29:45.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.450+0000 7f3158800640 1 -- start start 2024-03-06T07:29:45.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.451+0000 7f3158800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f315015ec90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:45.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.451+0000 7f3158800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f315006c990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:45.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.451+0000 7f3158800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f315015fff0 0x7f31501f7830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:45.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.451+0000 7f3158800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3150072a30 con 0x7f31500ce4a0 2024-03-06T07:29:45.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.451+0000 7f3158800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31500728b0 con 0x7f315015fff0 2024-03-06T07:29:45.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.451+0000 7f3158800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f315015c2e0 con 0x7f315015fc10 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.451+0000 7f3155e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f315015ec90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f315015ec90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39492/0 (socket says 172.21.15.86:39492) 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 -- 172.21.15.86:0/120763975 learned_addr learned my addr 172.21.15.86:0/120763975 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155400640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f315006c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3156800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f315015fff0 0x7f31501f7830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 -- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 msgr2=0x7f315006c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f315006c990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 -- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f315015fff0 msgr2=0x7f31501f7830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f315015fff0 0x7f31501f7830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 -- 172.21.15.86:0/120763975 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f315015f1d0 con 0x7f31500ce4a0 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3156800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f315015fff0 0x7f31501f7830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155400640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f315006c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.452+0000 7f3155e00640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f315015ec90 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f314400d830 tx=0x7f314400dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:45.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.453+0000 7f314a000640 1 -- 172.21.15.86:0/120763975 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3144018020 con 0x7f31500ce4a0 2024-03-06T07:29:45.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.453+0000 7f314a000640 1 -- 172.21.15.86:0/120763975 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f31440096a0 con 0x7f31500ce4a0 2024-03-06T07:29:45.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.453+0000 7f314a000640 1 -- 172.21.15.86:0/120763975 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3144009a70 con 0x7f31500ce4a0 2024-03-06T07:29:45.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.453+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f315006d450 con 0x7f31500ce4a0 2024-03-06T07:29:45.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.453+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f315006da10 con 0x7f31500ce4a0 2024-03-06T07:29:45.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.454+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f315004f330 con 0x7f31500ce4a0 2024-03-06T07:29:45.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.455+0000 7f314a000640 1 -- 172.21.15.86:0/120763975 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f314401b020 con 0x7f31500ce4a0 2024-03-06T07:29:45.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.456+0000 7f314a000640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f313406b4d0 0x7f313406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:45.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.456+0000 7f314a000640 1 -- 172.21.15.86:0/120763975 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(56..56 src has 1..56) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f314408ae60 con 0x7f31500ce4a0 2024-03-06T07:29:45.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.456+0000 7f3155400640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f313406b4d0 0x7f313406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:45.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.457+0000 7f3155400640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f313406b4d0 0x7f313406d990 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f3140006fd0 tx=0x7f3140008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:45.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.460+0000 7f314a000640 1 -- 172.21.15.86:0/120763975 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3144022030 con 0x7f31500ce4a0 2024-03-06T07:29:45.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:45.691+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f3150156710 con 0x7f31500ce4a0 2024-03-06T07:29:46.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.246+0000 7f314a000640 1 -- 172.21.15.86:0/120763975 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 8 and data pool 9 v30) v1 ==== 172+0+0 (secure 0 0 0) 0x7f3144011340 con 0x7f31500ce4a0 2024-03-06T07:29:46.249 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 8 and data pool 9 2024-03-06T07:29:46.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.251+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f313406b4d0 msgr2=0x7f313406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.251+0000 7f3158800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f313406b4d0 0x7f313406d990 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f3140006fd0 tx=0x7f3140008040 comp rx=0 tx=0).stop 2024-03-06T07:29:46.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.251+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 msgr2=0x7f315015ec90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.251+0000 7f3158800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f315015ec90 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f314400d830 tx=0x7f314400dcf0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.252+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 shutdown_connections 2024-03-06T07:29:46.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.252+0000 7f3158800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f313406b4d0 0x7f313406d990 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.252+0000 7f3158800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f315015fff0 0x7f31501f7830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.252+0000 7f3158800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f315015fc10 0x7f315006c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.252+0000 7f3158800640 1 --2- 172.21.15.86:0/120763975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f31500ce4a0 0x7f315015ec90 unknown :-1 s=CLOSED pgs=640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.252+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 >> 172.21.15.86:0/120763975 conn(0x7f31500b1940 msgr2=0x7f31501569a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:46.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.254+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 shutdown_connections 2024-03-06T07:29:46.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.254+0000 7f3158800640 1 -- 172.21.15.86:0/120763975 wait complete. 2024-03-06T07:29:46.304 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:46.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.475+0000 7f8527200640 1 Processor -- start 2024-03-06T07:29:46.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.475+0000 7f8527200640 1 -- start start 2024-03-06T07:29:46.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.476+0000 7f8527200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f85280b2a60 0x7f85280d4830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.476+0000 7f8527200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 0x7f85280d1d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.476+0000 7f8527200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f852815f030 con 0x7f85280d0600 2024-03-06T07:29:46.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.476+0000 7f8527200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f85280cf090 con 0x7f85280b2a60 2024-03-06T07:29:46.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.476+0000 7f8527200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f852815f310 con 0x7f85280d18e0 2024-03-06T07:29:46.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.477+0000 7f8525e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f85280d0600 0x7f85280d09e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50006/0 (socket says 172.21.15.86:50006) 2024-03-06T07:29:46.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.477+0000 7f8525e00640 1 -- 172.21.15.86:0/138565933 learned_addr learned my addr 172.21.15.86:0/138565933 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.477+0000 7f8526800640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 0x7f85280d1d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.477+0000 7f8525400640 1 --2- 172.21.15.86:0/138565933 >> v2:172.21.15.194:3300/0 conn(0x7f85280b2a60 0x7f85280d4830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.477+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1905826072 0 0) 0x7f852815f030 con 0x7f85280d0600 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.477+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f850c003520 con 0x7f85280d0600 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4104847555 0 0) 0x7f850c003520 con 0x7f85280d0600 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 msgr2=0x7f85280d1d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 0x7f85280d1d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 >> v2:172.21.15.194:3300/0 conn(0x7f85280b2a60 msgr2=0x7f85280d4830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 --2- 172.21.15.86:0/138565933 >> v2:172.21.15.194:3300/0 conn(0x7f85280b2a60 0x7f85280d4830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f85280cea40 con 0x7f85280d0600 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f85180019f0 con 0x7f85280d0600 2024-03-06T07:29:46.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 >> v1:172.21.15.86:6789/0 conn(0x7f85280d0600 legacy=0x7f85280d09e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 0x7f850c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f850c003f30 0x7f850c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c004d40 0x7f850c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85280cf090 con 0x7f850c003f30 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f852815f310 con 0x7f850c004d40 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f850c00b9c0 con 0x7f85280d18e0 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525400640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f850c003f30 0x7f850c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525e00640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 0x7f850c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525400640 1 -- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 msgr2=0x7f850c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525400640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 0x7f850c0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525400640 1 -- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c004d40 msgr2=0x7f850c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525400640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c004d40 0x7f850c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525400640 1 -- 172.21.15.86:0/138565933 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f850c003520 con 0x7f850c003f30 2024-03-06T07:29:46.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.478+0000 7f8525e00640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280d18e0 0x7f850c0039b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:46.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8525400640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f850c003f30 0x7f850c004800 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7f8514009ac0 tx=0x7f850c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4b190bc98646d860 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8514039070 con 0x7f850c003f30 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f851402c6d0 con 0x7f850c003f30 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8524a00640 1 -- 172.21.15.86:0/138565933 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8514009690 con 0x7f850c003f30 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8527200640 1 -- 172.21.15.86:0/138565933 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f850c003f30 msgr2=0x7f850c004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8527200640 1 --2- 172.21.15.86:0/138565933 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f850c003f30 0x7f850c004800 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7f8514009ac0 tx=0x7f850c005830 comp rx=0 tx=0).stop 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8526800640 1 -- 172.21.15.86:0/138565933 reap_dead start 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8527200640 1 -- 172.21.15.86:0/138565933 shutdown_connections 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8527200640 1 -- 172.21.15.86:0/138565933 >> 172.21.15.86:0/138565933 conn(0x7f85280d3d10 msgr2=0x7f85280d4120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:46.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8527200640 1 -- 172.21.15.86:0/138565933 shutdown_connections 2024-03-06T07:29:46.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.480+0000 7f8527200640 1 -- 172.21.15.86:0/138565933 wait complete. 2024-03-06T07:29:46.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.482+0000 7f8527200640 1 Processor -- start 2024-03-06T07:29:46.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.482+0000 7f8527200640 1 -- start start 2024-03-06T07:29:46.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8527200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c003f30 0x7f852814d250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8527200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85281565a0 0x7f85280cc2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8527200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280cc7f0 0x7f852806cad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8527200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f852815ecf0 con 0x7f85281565a0 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8527200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85280b3250 con 0x7f850c003f30 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8527200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f85280d1d70 con 0x7f85280cc7f0 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8525400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85281565a0 0x7f85280cc2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8525400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85281565a0 0x7f85280cc2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57326/0 (socket says 172.21.15.86:57326) 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.483+0000 7f8525400640 1 -- 172.21.15.86:0/2015434339 learned_addr learned my addr 172.21.15.86:0/2015434339 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8525e00640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c003f30 0x7f852814d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8526800640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280cc7f0 0x7f852806cad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8525400640 1 -- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280cc7f0 msgr2=0x7f852806cad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8525400640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280cc7f0 0x7f852806cad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8525400640 1 -- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c003f30 msgr2=0x7f852814d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8525400640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c003f30 0x7f852814d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8525400640 1 -- 172.21.15.86:0/2015434339 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f85280cccb0 con 0x7f85281565a0 2024-03-06T07:29:46.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8525400640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85281565a0 0x7f85280cc2b0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7f8514000c00 tx=0x7f8514004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:46.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f851e000640 1 -- 172.21.15.86:0/2015434339 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8514039050 con 0x7f85281565a0 2024-03-06T07:29:46.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f851e000640 1 -- 172.21.15.86:0/2015434339 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8514045070 con 0x7f85281565a0 2024-03-06T07:29:46.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f851e000640 1 -- 172.21.15.86:0/2015434339 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8514003a50 con 0x7f85281565a0 2024-03-06T07:29:46.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.484+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f85280ccf90 con 0x7f85281565a0 2024-03-06T07:29:46.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.485+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f85280cd550 con 0x7f85281565a0 2024-03-06T07:29:46.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.486+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f852804f330 con 0x7f85281565a0 2024-03-06T07:29:46.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.487+0000 7f851e000640 1 -- 172.21.15.86:0/2015434339 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8514003bf0 con 0x7f85281565a0 2024-03-06T07:29:46.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.487+0000 7f851e000640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f850806b4d0 0x7f850806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.488+0000 7f851e000640 1 -- 172.21.15.86:0/2015434339 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(57..57 src has 1..57) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f85140abe00 con 0x7f85281565a0 2024-03-06T07:29:46.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.488+0000 7f8525e00640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f850806b4d0 0x7f850806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.488+0000 7f8525e00640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f850806b4d0 0x7f850806d990 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f85180012e0 tx=0x7f8518009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:46.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.492+0000 7f851e000640 1 -- 172.21.15.86:0/2015434339 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f851404e020 con 0x7f85281565a0 2024-03-06T07:29:46.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.679+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f852808d7e0 con 0x7f85281565a0 2024-03-06T07:29:46.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.682+0000 7f851e000640 1 -- 172.21.15.86:0/2015434339 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v57) v1 ==== 74+0+16690 (secure 0 0 0) 0x7f851407abe0 con 0x7f85281565a0 2024-03-06T07:29:46.684 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:46.684 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":57,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:45.693177+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:46.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.684+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f850806b4d0 msgr2=0x7f850806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.684+0000 7f8527200640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f850806b4d0 0x7f850806d990 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f85180012e0 tx=0x7f8518009040 comp rx=0 tx=0).stop 2024-03-06T07:29:46.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.684+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85281565a0 msgr2=0x7f85280cc2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.684+0000 7f8527200640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85281565a0 0x7f85280cc2b0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7f8514000c00 tx=0x7f8514004730 comp rx=0 tx=0).stop 2024-03-06T07:29:46.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 shutdown_connections 2024-03-06T07:29:46.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f850806b4d0 0x7f850806d990 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f85280cc7f0 0x7f852806cad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85281565a0 0x7f85280cc2b0 unknown :-1 s=CLOSED pgs=643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 --2- 172.21.15.86:0/2015434339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f850c003f30 0x7f852814d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 >> 172.21.15.86:0/2015434339 conn(0x7f85280d3d10 msgr2=0x7f85280d4120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:46.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 shutdown_connections 2024-03-06T07:29:46.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.685+0000 7f8527200640 1 -- 172.21.15.86:0/2015434339 wait complete. 2024-03-06T07:29:46.703 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:46.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.901+0000 7fdad9800640 1 Processor -- start 2024-03-06T07:29:46.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.901+0000 7fdad9800640 1 -- start start 2024-03-06T07:29:46.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.902+0000 7fdad9800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fdad415f470 0x7fdad40d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.902+0000 7fdad9800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdad40d1200 con 0x7fdad40d0df0 2024-03-06T07:29:46.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.902+0000 7fdad9800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdad40cea60 con 0x7fdad415e0d0 2024-03-06T07:29:46.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.902+0000 7fdad9800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdad40d2f60 con 0x7fdad415f470 2024-03-06T07:29:46.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.903+0000 7fdad3400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fdad40d0df0 0x7fdad4160dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50020/0 (socket says 172.21.15.86:50020) 2024-03-06T07:29:46.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.903+0000 7fdad3400640 1 -- 172.21.15.86:0/2794644438 learned_addr learned my addr 172.21.15.86:0/2794644438 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:46.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.903+0000 7fdad2000640 1 --2- 172.21.15.86:0/2794644438 >> v2:172.21.15.194:3300/0 conn(0x7fdad415f470 0x7fdad40d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.903+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3243175288 0 0) 0x7fdacc001110 con 0x7fdad40d0df0 2024-03-06T07:29:46.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.903+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fdab8003520 con 0x7fdad40d0df0 2024-03-06T07:29:46.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2767560361 0 0) 0x7fdab8003520 con 0x7fdad40d0df0 2024-03-06T07:29:46.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 >> v2:172.21.15.194:3300/0 conn(0x7fdad415f470 msgr2=0x7fdad40d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 --2- 172.21.15.86:0/2794644438 >> v2:172.21.15.194:3300/0 conn(0x7fdad415f470 0x7fdad40d08b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 >> v1:172.21.15.194:6789/0 conn(0x7fdad415e0d0 legacy=0x7fdad40d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdad40d2350 con 0x7fdad40d0df0 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fdacc001840 con 0x7fdad40d0df0 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 >> v1:172.21.15.86:6789/0 conn(0x7fdad40d0df0 legacy=0x7fdad4160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdab8003a00 0x7fdab8003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdab8004800 0x7fdab8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdab800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdab800c070 con 0x7fdab80051c0 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdab800c220 con 0x7fdab8003a00 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdab800c420 con 0x7fdab8004800 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad3400640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdab800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.904+0000 7fdad2a00640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdab8003a00 0x7fdab8003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.905+0000 7fdad3400640 1 -- 172.21.15.86:0/2794644438 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdab8004800 msgr2=0x7fdab8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.905+0000 7fdad3400640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdab8004800 0x7fdab8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.905+0000 7fdad3400640 1 -- 172.21.15.86:0/2794644438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdab8003a00 msgr2=0x7fdab8003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:46.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.905+0000 7fdad3400640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdab8003a00 0x7fdab8003e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.905+0000 7fdad3400640 1 -- 172.21.15.86:0/2794644438 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdab8003520 con 0x7fdab80051c0 2024-03-06T07:29:46.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.905+0000 7fdad3400640 1 -- 172.21.15.86:0/2794644438 reap_dead start 2024-03-06T07:29:46.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.906+0000 7fdad3400640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdab800b9c0 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fdacc007fd0 tx=0x7fdab8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bb07c1608835146e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:46.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.906+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdacc007e30 con 0x7fdab80051c0 2024-03-06T07:29:46.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.906+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdacc030070 con 0x7fdab80051c0 2024-03-06T07:29:46.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.906+0000 7fdad1600640 1 -- 172.21.15.86:0/2794644438 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdacc02ce50 con 0x7fdab80051c0 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.906+0000 7fdad9800640 1 -- 172.21.15.86:0/2794644438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 msgr2=0x7fdab800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.906+0000 7fdad9800640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdab800b9c0 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fdacc007fd0 tx=0x7fdab8005830 comp rx=0 tx=0).stop 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.907+0000 7fdad9800640 1 -- 172.21.15.86:0/2794644438 shutdown_connections 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.907+0000 7fdad9800640 1 --2- 172.21.15.86:0/2794644438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdab800b9c0 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.907+0000 7fdad9800640 1 -- 172.21.15.86:0/2794644438 >> 172.21.15.86:0/2794644438 conn(0x7fdad40d3ec0 msgr2=0x7fdad40d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.907+0000 7fdad9800640 1 -- 172.21.15.86:0/2794644438 shutdown_connections 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.907+0000 7fdad9800640 1 -- 172.21.15.86:0/2794644438 wait complete. 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.908+0000 7fdad9800640 1 Processor -- start 2024-03-06T07:29:46.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.909+0000 7fdad9800640 1 -- start start 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad9800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdad40b1800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad9800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdad40cb580 0x7fdad40b1d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad9800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdad41602c0 0x7fdad4160740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad9800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdad40d2bf0 con 0x7fdab80051c0 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad9800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdad40d2910 con 0x7fdad41602c0 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad9800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdad415fb30 con 0x7fdad40cb580 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdad40b1800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdad40b1800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57352/0 (socket says 172.21.15.86:57352) 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad2a00640 1 -- 172.21.15.86:0/1564939190 learned_addr learned my addr 172.21.15.86:0/1564939190 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad2a00640 1 -- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdad40cb580 msgr2=0x7fdad40b1d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad2000640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdad40cb580 0x7fdad40b1d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad2a00640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdad40cb580 0x7fdad40b1d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad3400640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdad41602c0 0x7fdad4160740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.910+0000 7fdad2a00640 1 -- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdad41602c0 msgr2=0x7fdad4160740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdad2a00640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdad41602c0 0x7fdad4160740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdad2a00640 1 -- 172.21.15.86:0/1564939190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdad40cbb40 con 0x7fdab80051c0 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdad3400640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdad41602c0 0x7fdad4160740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdad2a00640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdad40b1800 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00d5a0 tx=0x7fdabc00da60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdacaa00640 1 -- 172.21.15.86:0/1564939190 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdabc016070 con 0x7fdab80051c0 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdacaa00640 1 -- 172.21.15.86:0/1564939190 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdabc012800 con 0x7fdab80051c0 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdacaa00640 1 -- 172.21.15.86:0/1564939190 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdabc012bd0 con 0x7fdab80051c0 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdad40cbe20 con 0x7fdab80051c0 2024-03-06T07:29:46.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.911+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdad40cc3e0 con 0x7fdab80051c0 2024-03-06T07:29:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.913+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdad415f850 con 0x7fdab80051c0 2024-03-06T07:29:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.913+0000 7fdacaa00640 1 -- 172.21.15.86:0/1564939190 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdabc019070 con 0x7fdab80051c0 2024-03-06T07:29:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.914+0000 7fdacaa00640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdab006b500 0x7fdab006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.914+0000 7fdacaa00640 1 -- 172.21.15.86:0/1564939190 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(57..57 src has 1..57) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fdabc089ff0 con 0x7fdab80051c0 2024-03-06T07:29:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.914+0000 7fdad2000640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdab006b500 0x7fdab006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.915+0000 7fdad2000640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdab006b500 0x7fdab006d9c0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7fdad40b25d0 tx=0x7fdac400b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:46.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:46.919+0000 7fdacaa00640 1 -- 172.21.15.86:0/1564939190 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdad415f850 con 0x7fdab80051c0 2024-03-06T07:29:47.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.101+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fdad404f330 con 0x7fdab80051c0 2024-03-06T07:29:47.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.104+0000 7fdacaa00640 1 -- 172.21.15.86:0/1564939190 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v57) v1 ==== 74+0+16690 (secure 0 0 0) 0x7fdabc058f00 con 0x7fdab80051c0 2024-03-06T07:29:47.106 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:47.106 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":57,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:45.693177+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:47.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.106+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdab006b500 msgr2=0x7fdab006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:47.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.106+0000 7fdad9800640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdab006b500 0x7fdab006d9c0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7fdad40b25d0 tx=0x7fdac400b040 comp rx=0 tx=0).stop 2024-03-06T07:29:47.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.106+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 msgr2=0x7fdad40b1800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:47.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.106+0000 7fdad9800640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdad40b1800 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00d5a0 tx=0x7fdabc00da60 comp rx=0 tx=0).stop 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 shutdown_connections 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdab006b500 0x7fdab006d9c0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdad41602c0 0x7fdad4160740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdad40cb580 0x7fdad40b1d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 --2- 172.21.15.86:0/1564939190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdab80051c0 0x7fdad40b1800 unknown :-1 s=CLOSED pgs=646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 >> 172.21.15.86:0/1564939190 conn(0x7fdad40d3ec0 msgr2=0x7fdad40b38a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 shutdown_connections 2024-03-06T07:29:47.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.107+0000 7fdad9800640 1 -- 172.21.15.86:0/1564939190 wait complete. 2024-03-06T07:29:47.126 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:29:47.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.335+0000 7f99abe00640 1 Processor -- start 2024-03-06T07:29:47.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.335+0000 7f99abe00640 1 -- start start 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99abe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f99ac0d1ea0 0x7f99ac0d2300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99abe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f99ac0c7970 0x7f99ac0c7e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99abe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f99ac0d13e0 con 0x7f99ac15e130 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99abe00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f99ac0d2840 con 0x7f99ac0d1ea0 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99abe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f99ac0cf120 con 0x7f99ac0c7970 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99a3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f99ac0d1ea0 0x7f99ac0d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99a3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f99ac0d1ea0 0x7f99ac0d2300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57358/0 (socket says 172.21.15.86:57358) 2024-03-06T07:29:47.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.336+0000 7f99a3e00640 1 -- 172.21.15.86:0/168673566 learned_addr learned my addr 172.21.15.86:0/168673566 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:47.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.337+0000 7f99a3e00640 1 -- 172.21.15.86:0/168673566 >> v2:172.21.15.194:3300/0 conn(0x7f99ac0c7970 msgr2=0x7f99ac0c7e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:47.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.337+0000 7f99a3e00640 1 --2- 172.21.15.86:0/168673566 >> v2:172.21.15.194:3300/0 conn(0x7f99ac0c7970 0x7f99ac0c7e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.337+0000 7f99a3e00640 1 -- 172.21.15.86:0/168673566 >> v1:172.21.15.194:6789/0 conn(0x7f99ac15e130 legacy=0x7f99ac0cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:47.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.337+0000 7f99a3e00640 1 -- 172.21.15.86:0/168673566 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99ac0cead0 con 0x7f99ac0d1ea0 2024-03-06T07:29:47.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.338+0000 7f99a3e00640 1 --2- 172.21.15.86:0/168673566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac0d1ea0 0x7f99ac0d2300 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7f99ac162bb0 tx=0x7f999402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c5a098ce4ea0f0c0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.338+0000 7f99aa000640 1 -- 172.21.15.86:0/168673566 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9994035020 con 0x7f99ac0d1ea0 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.338+0000 7f99aa000640 1 -- 172.21.15.86:0/168673566 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9994031870 con 0x7f99ac0d1ea0 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.338+0000 7f99aa000640 1 -- 172.21.15.86:0/168673566 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9994031c40 con 0x7f99ac0d1ea0 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 -- 172.21.15.86:0/168673566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac0d1ea0 msgr2=0x7f99ac0d2300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 --2- 172.21.15.86:0/168673566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac0d1ea0 0x7f99ac0d2300 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7f99ac162bb0 tx=0x7f999402b910 comp rx=0 tx=0).stop 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 -- 172.21.15.86:0/168673566 shutdown_connections 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 --2- 172.21.15.86:0/168673566 >> v2:172.21.15.194:3300/0 conn(0x7f99ac0c7970 0x7f99ac0c7e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 --2- 172.21.15.86:0/168673566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac0d1ea0 0x7f99ac0d2300 unknown :-1 s=CLOSED pgs=647 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 -- 172.21.15.86:0/168673566 >> 172.21.15.86:0/168673566 conn(0x7f99ac09d070 msgr2=0x7f99ac0d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 -- 172.21.15.86:0/168673566 shutdown_connections 2024-03-06T07:29:47.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.339+0000 7f99abe00640 1 -- 172.21.15.86:0/168673566 wait complete. 2024-03-06T07:29:47.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.341+0000 7f99abe00640 1 Processor -- start 2024-03-06T07:29:47.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.341+0000 7f99abe00640 1 -- start start 2024-03-06T07:29:47.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99abe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99ac0c7970 0x7f99ac150150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:47.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99abe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99ac0d1ea0 0x7f99ac160b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99abe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac15e130 0x7f99ac0cd0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99abe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99ac0cfcc0 con 0x7f99ac15e130 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99abe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99ac0c8340 con 0x7f99ac0c7970 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99abe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99ac0d16c0 con 0x7f99ac0d1ea0 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac15e130 0x7f99ac0cd0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac15e130 0x7f99ac0cd0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57362/0 (socket says 172.21.15.86:57362) 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 -- 172.21.15.86:0/3542453097 learned_addr learned my addr 172.21.15.86:0/3542453097 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 -- 172.21.15.86:0/3542453097 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99ac0d1ea0 msgr2=0x7f99ac160b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99ac0d1ea0 0x7f99ac160b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 -- 172.21.15.86:0/3542453097 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99ac0c7970 msgr2=0x7f99ac150150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99aaa00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99ac0c7970 0x7f99ac150150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99ac0c7970 0x7f99ac150150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.342+0000 7f99ab400640 1 -- 172.21.15.86:0/3542453097 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99ac1e7af0 con 0x7f99ac15e130 2024-03-06T07:29:47.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.343+0000 7f99ab400640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac15e130 0x7f99ac0cd0c0 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f99a400b970 tx=0x7f99a400be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:47.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.343+0000 7f99a2a00640 1 -- 172.21.15.86:0/3542453097 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99a4019070 con 0x7f99ac15e130 2024-03-06T07:29:47.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.343+0000 7f99a2a00640 1 -- 172.21.15.86:0/3542453097 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99a400cb60 con 0x7f99ac15e130 2024-03-06T07:29:47.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.343+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99ac1e7dd0 con 0x7f99ac15e130 2024-03-06T07:29:47.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.343+0000 7f99a2a00640 1 -- 172.21.15.86:0/3542453097 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99a4014440 con 0x7f99ac15e130 2024-03-06T07:29:47.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.343+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99ac15d3c0 con 0x7f99ac15e130 2024-03-06T07:29:47.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.344+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f99ac04f330 con 0x7f99ac15e130 2024-03-06T07:29:47.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.345+0000 7f99a2a00640 1 -- 172.21.15.86:0/3542453097 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f99a401c070 con 0x7f99ac15e130 2024-03-06T07:29:47.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.347+0000 7f99a2a00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f998806b500 0x7f998806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:47.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.347+0000 7f99a2a00640 1 -- 172.21.15.86:0/3542453097 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f99a4089f00 con 0x7f99ac15e130 2024-03-06T07:29:47.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.347+0000 7f99aaa00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f998806b500 0x7f998806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:47.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.347+0000 7f99aaa00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f998806b500 0x7f998806d9c0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f999c0015d0 tx=0x7f999c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:47.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.350+0000 7f99a2a00640 1 -- 172.21.15.86:0/3542453097 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f99a4058e10 con 0x7f99ac15e130 2024-03-06T07:29:47.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:47.575+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f99ac09d690 con 0x7f99ac15e130 2024-03-06T07:29:48.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.292+0000 7f99a2a00640 1 -- 172.21.15.86:0/3542453097 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v33) v1 ==== 118+0+0 (secure 0 0 0) 0x7f99a4058710 con 0x7f99ac15e130 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.295+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f998806b500 msgr2=0x7f998806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f998806b500 0x7f998806d9c0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f999c0015d0 tx=0x7f999c009040 comp rx=0 tx=0).stop 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac15e130 msgr2=0x7f99ac0cd0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac15e130 0x7f99ac0cd0c0 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f99a400b970 tx=0x7f99a400be30 comp rx=0 tx=0).stop 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 shutdown_connections 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f998806b500 0x7f998806d9c0 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f99ac15e130 0x7f99ac0cd0c0 unknown :-1 s=CLOSED pgs=648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99ac0d1ea0 0x7f99ac160b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 --2- 172.21.15.86:0/3542453097 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f99ac0c7970 0x7f99ac150150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 >> 172.21.15.86:0/3542453097 conn(0x7f99ac09d070 msgr2=0x7f99ac162310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 shutdown_connections 2024-03-06T07:29:48.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.296+0000 7f99abe00640 1 -- 172.21.15.86:0/3542453097 wait complete. 2024-03-06T07:29:48.321 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:29:48.321 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:29:48.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.542+0000 7fd5a7e00640 1 Processor -- start 2024-03-06T07:29:48.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.542+0000 7fd5a7e00640 1 -- start start 2024-03-06T07:29:48.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd5a7e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd5a80d1ea0 0x7fd5a80d2300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:48.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd5a7e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd5a80c7970 0x7fd5a80c7e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:48.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd5a7e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd5a80d13e0 con 0x7fd5a815e130 2024-03-06T07:29:48.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd5a7e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd5a80d2840 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd5a7e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd5a80cf120 con 0x7fd5a80c7970 2024-03-06T07:29:48.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd59e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd5a80d1ea0 0x7fd5a80d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:48.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd59e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd5a80d1ea0 0x7fd5a80d2300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57366/0 (socket says 172.21.15.86:57366) 2024-03-06T07:29:48.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.545+0000 7fd59e000640 1 -- 172.21.15.86:0/4161112000 learned_addr learned my addr 172.21.15.86:0/4161112000 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:48.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.546+0000 7fd59e000640 1 -- 172.21.15.86:0/4161112000 >> v2:172.21.15.194:3300/0 conn(0x7fd5a80c7970 msgr2=0x7fd5a80c7e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:48.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.546+0000 7fd59e000640 1 --2- 172.21.15.86:0/4161112000 >> v2:172.21.15.194:3300/0 conn(0x7fd5a80c7970 0x7fd5a80c7e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.546+0000 7fd59e000640 1 -- 172.21.15.86:0/4161112000 >> v1:172.21.15.194:6789/0 conn(0x7fd5a815e130 legacy=0x7fd5a80cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:48.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.546+0000 7fd59e000640 1 -- 172.21.15.86:0/4161112000 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5a80cead0 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.547+0000 7fd59e000640 1 --2- 172.21.15.86:0/4161112000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a80d2300 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8162bb0 tx=0x7fd59002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c35ae1c079ce1c05 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:48.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.547+0000 7fd5a6000640 1 -- 172.21.15.86:0/4161112000 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd590035020 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.547+0000 7fd5a6000640 1 -- 172.21.15.86:0/4161112000 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd590031870 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.547+0000 7fd5a6000640 1 -- 172.21.15.86:0/4161112000 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd590031c40 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 -- 172.21.15.86:0/4161112000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 msgr2=0x7fd5a80d2300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/4161112000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a80d2300 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8162bb0 tx=0x7fd59002b910 comp rx=0 tx=0).stop 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 -- 172.21.15.86:0/4161112000 shutdown_connections 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/4161112000 >> v2:172.21.15.194:3300/0 conn(0x7fd5a80c7970 0x7fd5a80c7e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/4161112000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a80d2300 unknown :-1 s=CLOSED pgs=649 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 -- 172.21.15.86:0/4161112000 >> 172.21.15.86:0/4161112000 conn(0x7fd5a809d070 msgr2=0x7fd5a80d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 -- 172.21.15.86:0/4161112000 shutdown_connections 2024-03-06T07:29:48.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.548+0000 7fd5a7e00640 1 -- 172.21.15.86:0/4161112000 wait complete. 2024-03-06T07:29:48.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.550+0000 7fd5a7e00640 1 Processor -- start 2024-03-06T07:29:48.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.550+0000 7fd5a7e00640 1 -- start start 2024-03-06T07:29:48.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd5a7e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5a80c7970 0x7fd5a8150150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:48.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd5a7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a8160b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:48.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd5a7e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5a815e130 0x7fd5a80cd0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:48.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd5a7e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5a80cfcc0 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd5a7e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5a80c8340 con 0x7fd5a80c7970 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd5a7e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd5a80d16c0 con 0x7fd5a815e130 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd59e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a8160b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd59e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a8160b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57376/0 (socket says 172.21.15.86:57376) 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.551+0000 7fd59e000640 1 -- 172.21.15.86:0/392566758 learned_addr learned my addr 172.21.15.86:0/392566758 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd5a6a00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5a80c7970 0x7fd5a8150150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59e000640 1 -- 172.21.15.86:0/392566758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5a815e130 msgr2=0x7fd5a80cd0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59e000640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5a815e130 0x7fd5a80cd0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59e000640 1 -- 172.21.15.86:0/392566758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5a80c7970 msgr2=0x7fd5a8150150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59e000640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5a80c7970 0x7fd5a8150150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59e000640 1 -- 172.21.15.86:0/392566758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5a81e7af0 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59e000640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a8160b50 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7fd590000c00 tx=0x7fd5900046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:48.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59fe00640 1 -- 172.21.15.86:0/392566758 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5900443f0 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59fe00640 1 -- 172.21.15.86:0/392566758 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd590044590 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd5a81e7dd0 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd59fe00640 1 -- 172.21.15.86:0/392566758 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd590044980 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.552+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd5a815d3c0 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.554+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd5a804f330 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.555+0000 7fd59fe00640 1 -- 172.21.15.86:0/392566758 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd59004b060 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.556+0000 7fd59fe00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd58406b500 0x7fd58406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:48.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.556+0000 7fd59fe00640 1 -- 172.21.15.86:0/392566758 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd5900ac450 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.556+0000 7fd5a6a00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd58406b500 0x7fd58406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:48.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.557+0000 7fd5a6a00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd58406b500 0x7fd58406d9c0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fd5980015d0 tx=0x7fd598009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:48.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.561+0000 7fd59fe00640 1 -- 172.21.15.86:0/392566758 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd59003f070 con 0x7fd5a80d1ea0 2024-03-06T07:29:48.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:48.795+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7fd5a80d1c80 con 0x7fd5a80d1ea0 2024-03-06T07:29:49.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.306+0000 7fd59fe00640 1 -- 172.21.15.86:0/392566758 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v34) v1 ==== 122+0+0 (secure 0 0 0) 0x7fd59007b360 con 0x7fd5a80d1ea0 2024-03-06T07:29:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.309+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd58406b500 msgr2=0x7fd58406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.309+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd58406b500 0x7fd58406d9c0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fd5980015d0 tx=0x7fd598009040 comp rx=0 tx=0).stop 2024-03-06T07:29:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.309+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 msgr2=0x7fd5a8160b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.309+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a8160b50 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7fd590000c00 tx=0x7fd5900046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 shutdown_connections 2024-03-06T07:29:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd58406b500 0x7fd58406d9c0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5a815e130 0x7fd5a80cd0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5a80d1ea0 0x7fd5a8160b50 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 --2- 172.21.15.86:0/392566758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5a80c7970 0x7fd5a8150150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 >> 172.21.15.86:0/392566758 conn(0x7fd5a809d070 msgr2=0x7fd5a8162310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:49.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 shutdown_connections 2024-03-06T07:29:49.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.310+0000 7fd5a7e00640 1 -- 172.21.15.86:0/392566758 wait complete. 2024-03-06T07:29:49.337 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:29:49.337 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.540+0000 7f8435600640 1 Processor -- start 2024-03-06T07:29:49.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.540+0000 7f8435600640 1 -- start start 2024-03-06T07:29:49.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.543+0000 7f8435600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f843015fe60 0x7f8430071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.543+0000 7f8435600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f843006cef0 con 0x7f843015f9d0 2024-03-06T07:29:49.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.543+0000 7f8435600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f843006d1d0 con 0x7f8430144c00 2024-03-06T07:29:49.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.543+0000 7f8435600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f843006d4b0 con 0x7f843015fe60 2024-03-06T07:29:49.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.543+0000 7f842e000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f843015f9d0 0x7f8430070b10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50026/0 (socket says 172.21.15.86:50026) 2024-03-06T07:29:49.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.543+0000 7f842e000640 1 -- 172.21.15.86:0/668736541 learned_addr learned my addr 172.21.15.86:0/668736541 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.543+0000 7f842f400640 1 --2- 172.21.15.86:0/668736541 >> v2:172.21.15.194:3300/0 conn(0x7f843015fe60 0x7f8430071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:49.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 105745328 0 0) 0x7f843006cef0 con 0x7f843015f9d0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8414003520 con 0x7f843015f9d0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3234571770 0 0) 0x7f8414003520 con 0x7f843015f9d0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 >> v2:172.21.15.194:3300/0 conn(0x7f843015fe60 msgr2=0x7f8430071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 --2- 172.21.15.86:0/668736541 >> v2:172.21.15.194:3300/0 conn(0x7f843015fe60 0x7f8430071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 >> v1:172.21.15.194:6789/0 conn(0x7f8430144c00 legacy=0x7f843006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f843006d790 con 0x7f843015f9d0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f841c001670 con 0x7f843015f9d0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 >> v1:172.21.15.86:6789/0 conn(0x7f843015f9d0 legacy=0x7f8430070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84140039b0 0x7f8414003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8414004800 0x7f8414004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84140051c0 0x7f841400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f843006d4b0 con 0x7f84140051c0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f841400c060 con 0x7f84140039b0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.544+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f841400c260 con 0x7f8414004800 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.545+0000 7f842f400640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84140051c0 0x7f841400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:49.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.545+0000 7f842e000640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8414004800 0x7f8414004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.545+0000 7f842f400640 1 -- 172.21.15.86:0/668736541 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8414004800 msgr2=0x7f8414004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.545+0000 7f842f400640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8414004800 0x7f8414004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.545+0000 7f842f400640 1 -- 172.21.15.86:0/668736541 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84140039b0 msgr2=0x7f8414003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.545+0000 7f842f400640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84140039b0 0x7f8414003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.545+0000 7f842f400640 1 -- 172.21.15.86:0/668736541 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8414003520 con 0x7f84140051c0 2024-03-06T07:29:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.546+0000 7f842f400640 1 -- 172.21.15.86:0/668736541 reap_dead start 2024-03-06T07:29:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.546+0000 7f842f400640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84140051c0 0x7f841400b9c0 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f8428009fc0 tx=0x7f8414005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98708d2928c77268 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.546+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8428009c10 con 0x7f84140051c0 2024-03-06T07:29:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.546+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f842802cc60 con 0x7f84140051c0 2024-03-06T07:29:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.546+0000 7f842d600640 1 -- 172.21.15.86:0/668736541 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8428003bd0 con 0x7f84140051c0 2024-03-06T07:29:49.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.547+0000 7f8435600640 1 -- 172.21.15.86:0/668736541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84140051c0 msgr2=0x7f841400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.547+0000 7f8435600640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84140051c0 0x7f841400b9c0 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f8428009fc0 tx=0x7f8414005830 comp rx=0 tx=0).stop 2024-03-06T07:29:49.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.547+0000 7f8435600640 1 -- 172.21.15.86:0/668736541 shutdown_connections 2024-03-06T07:29:49.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.547+0000 7f8435600640 1 --2- 172.21.15.86:0/668736541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84140051c0 0x7f841400b9c0 unknown :-1 s=CLOSED pgs=652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.547+0000 7f8435600640 1 -- 172.21.15.86:0/668736541 >> 172.21.15.86:0/668736541 conn(0x7f84300b1730 msgr2=0x7f84300b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:49.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.547+0000 7f8435600640 1 -- 172.21.15.86:0/668736541 shutdown_connections 2024-03-06T07:29:49.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.547+0000 7f8435600640 1 -- 172.21.15.86:0/668736541 wait complete. 2024-03-06T07:29:49.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.550+0000 7f8435600640 1 Processor -- start 2024-03-06T07:29:49.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.550+0000 7f8435600640 1 -- start start 2024-03-06T07:29:49.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f8435600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84140051c0 0x7f84301f5ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f8435600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84300c49a0 0x7f84301f51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f8435600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84300c4e20 0x7f843013f3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f8435600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f843006cb50 con 0x7f84300c4e20 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f8435600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8430160730 con 0x7f84300c49a0 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f8435600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f843006ccd0 con 0x7f84140051c0 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84300c4e20 0x7f843013f3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84300c4e20 0x7f843013f3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57392/0 (socket says 172.21.15.86:57392) 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 -- 172.21.15.86:0/4230207419 learned_addr learned my addr 172.21.15.86:0/4230207419 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842ea00640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84140051c0 0x7f84301f5ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842e000640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84300c49a0 0x7f84301f51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 -- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84140051c0 msgr2=0x7f84301f5ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84140051c0 0x7f84301f5ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 -- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84300c49a0 msgr2=0x7f84301f51b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84300c49a0 0x7f84301f51b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.552+0000 7f842f400640 1 -- 172.21.15.86:0/4230207419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84300aa540 con 0x7f84300c4e20 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.553+0000 7f842f400640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84300c4e20 0x7f843013f3c0 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7f842802c6d0 tx=0x7f8428009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.553+0000 7f8426a00640 1 -- 172.21.15.86:0/4230207419 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8428003990 con 0x7f84300c4e20 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.553+0000 7f8426a00640 1 -- 172.21.15.86:0/4230207419 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8428044070 con 0x7f84300c4e20 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.553+0000 7f8426a00640 1 -- 172.21.15.86:0/4230207419 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8428003d50 con 0x7f84300c4e20 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.553+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84300aa820 con 0x7f84300c4e20 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.553+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84300aade0 con 0x7f84300c4e20 2024-03-06T07:29:49.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.555+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8430160520 con 0x7f84300c4e20 2024-03-06T07:29:49.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.556+0000 7f8426a00640 1 -- 172.21.15.86:0/4230207419 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8428005d10 con 0x7f84300c4e20 2024-03-06T07:29:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.557+0000 7f8426a00640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f841006b4d0 0x7f841006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.557+0000 7f8426a00640 1 -- 172.21.15.86:0/4230207419 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f84280ac810 con 0x7f84300c4e20 2024-03-06T07:29:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.557+0000 7f842ea00640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f841006b4d0 0x7f841006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.558+0000 7f842ea00640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f841006b4d0 0x7f841006d990 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f8420001680 tx=0x7f8420009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:49.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.561+0000 7f8426a00640 1 -- 172.21.15.86:0/4230207419 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8430160520 con 0x7f84300c4e20 2024-03-06T07:29:49.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.796+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8430072300 con 0x7f84300c4e20 2024-03-06T07:29:49.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.797+0000 7f8426a00640 1 -- 172.21.15.86:0/4230207419 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f842807b7b0 con 0x7f84300c4e20 2024-03-06T07:29:49.799 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:49.799 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:29:49.800 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f841006b4d0 msgr2=0x7f841006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f841006b4d0 0x7f841006d990 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f8420001680 tx=0x7f8420009040 comp rx=0 tx=0).stop 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84300c4e20 msgr2=0x7f843013f3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84300c4e20 0x7f843013f3c0 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7f842802c6d0 tx=0x7f8428009750 comp rx=0 tx=0).stop 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 shutdown_connections 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f841006b4d0 0x7f841006d990 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84300c4e20 0x7f843013f3c0 unknown :-1 s=CLOSED pgs=653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84300c49a0 0x7f84301f51b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 --2- 172.21.15.86:0/4230207419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84140051c0 0x7f84301f5ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:49.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 >> 172.21.15.86:0/4230207419 conn(0x7f84300b1730 msgr2=0x7f84301441d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:49.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 shutdown_connections 2024-03-06T07:29:49.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:49.800+0000 7f8435600640 1 -- 172.21.15.86:0/4230207419 wait complete. 2024-03-06T07:29:49.828 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:29:50.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.038+0000 7efc0da00640 1 Processor -- start 2024-03-06T07:29:50.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.038+0000 7efc0da00640 1 -- start start 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc0da00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efc08144c00 0x7efc0806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc0da00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7efc0815f9d0 0x7efc0815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc0da00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efc0806ee70 con 0x7efc080ce0a0 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc0da00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efc0806cef0 con 0x7efc08144c00 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc0da00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efc0806d1d0 con 0x7efc0815f9d0 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc06a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efc08144c00 0x7efc0806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc06a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efc08144c00 0x7efc0806e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57400/0 (socket says 172.21.15.86:57400) 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc06a00640 1 -- 172.21.15.86:0/2980498658 learned_addr learned my addr 172.21.15.86:0/2980498658 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:50.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc06a00640 1 -- 172.21.15.86:0/2980498658 >> v2:172.21.15.194:3300/0 conn(0x7efc0815f9d0 msgr2=0x7efc0815fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc06a00640 1 --2- 172.21.15.86:0/2980498658 >> v2:172.21.15.194:3300/0 conn(0x7efc0815f9d0 0x7efc0815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc06a00640 1 -- 172.21.15.86:0/2980498658 >> v1:172.21.15.194:6789/0 conn(0x7efc080ce0a0 legacy=0x7efc080ce4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.041+0000 7efc06a00640 1 -- 172.21.15.86:0/2980498658 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc0806d4b0 con 0x7efc08144c00 2024-03-06T07:29:50.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.042+0000 7efc06a00640 1 --2- 172.21.15.86:0/2980498658 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc08144c00 0x7efc0806e930 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7efc080b26f0 tx=0x7efbf802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80fa7c2675f1a4d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.043+0000 7efc05600640 1 -- 172.21.15.86:0/2980498658 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efbf8035020 con 0x7efc08144c00 2024-03-06T07:29:50.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.043+0000 7efc05600640 1 -- 172.21.15.86:0/2980498658 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efbf8031870 con 0x7efc08144c00 2024-03-06T07:29:50.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.043+0000 7efc05600640 1 -- 172.21.15.86:0/2980498658 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efbf8031c40 con 0x7efc08144c00 2024-03-06T07:29:50.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.043+0000 7efc0da00640 1 -- 172.21.15.86:0/2980498658 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc08144c00 msgr2=0x7efc0806e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.043+0000 7efc0da00640 1 --2- 172.21.15.86:0/2980498658 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc08144c00 0x7efc0806e930 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7efc080b26f0 tx=0x7efbf802b910 comp rx=0 tx=0).stop 2024-03-06T07:29:50.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.044+0000 7efc0da00640 1 -- 172.21.15.86:0/2980498658 shutdown_connections 2024-03-06T07:29:50.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.044+0000 7efc0da00640 1 --2- 172.21.15.86:0/2980498658 >> v2:172.21.15.194:3300/0 conn(0x7efc0815f9d0 0x7efc0815fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.044+0000 7efc0da00640 1 --2- 172.21.15.86:0/2980498658 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc08144c00 0x7efc0806e930 unknown :-1 s=CLOSED pgs=654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.044+0000 7efc0da00640 1 -- 172.21.15.86:0/2980498658 >> 172.21.15.86:0/2980498658 conn(0x7efc080b1730 msgr2=0x7efc080b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:50.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.044+0000 7efc0da00640 1 -- 172.21.15.86:0/2980498658 shutdown_connections 2024-03-06T07:29:50.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.044+0000 7efc0da00640 1 -- 172.21.15.86:0/2980498658 wait complete. 2024-03-06T07:29:50.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.045+0000 7efc0da00640 1 Processor -- start 2024-03-06T07:29:50.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.045+0000 7efc0da00640 1 -- start start 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.046+0000 7efc0da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc080ce0a0 0x7efc0815f120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.046+0000 7efc0da00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efc08144c00 0x7efc0813a360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.046+0000 7efc0da00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efc0815f9d0 0x7efc08140ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.046+0000 7efc0da00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc0806cb50 con 0x7efc080ce0a0 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.046+0000 7efc0da00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc08160730 con 0x7efc08144c00 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.046+0000 7efc0da00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efc0806ccd0 con 0x7efc0815f9d0 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc080ce0a0 0x7efc0815f120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc080ce0a0 0x7efc0815f120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57404/0 (socket says 172.21.15.86:57404) 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 -- 172.21.15.86:0/3201290152 learned_addr learned my addr 172.21.15.86:0/3201290152 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc07400640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efc0815f9d0 0x7efc08140ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 -- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efc0815f9d0 msgr2=0x7efc08140ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efc0815f9d0 0x7efc08140ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06000640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efc08144c00 0x7efc0813a360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 -- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efc08144c00 msgr2=0x7efc0813a360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efc08144c00 0x7efc0813a360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06a00640 1 -- 172.21.15.86:0/3201290152 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc0815f660 con 0x7efc080ce0a0 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.047+0000 7efc06000640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efc08144c00 0x7efc0813a360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.048+0000 7efc06a00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc080ce0a0 0x7efc0815f120 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7efbf8000c00 tx=0x7efbf80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.048+0000 7efbfea00640 1 -- 172.21.15.86:0/3201290152 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efbf803ad60 con 0x7efc080ce0a0 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.048+0000 7efbfea00640 1 -- 172.21.15.86:0/3201290152 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efbf80443f0 con 0x7efc080ce0a0 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.048+0000 7efbfea00640 1 -- 172.21.15.86:0/3201290152 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efbf80447e0 con 0x7efc080ce0a0 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.048+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc0815e080 con 0x7efc080ce0a0 2024-03-06T07:29:50.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.048+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc0815e640 con 0x7efc080ce0a0 2024-03-06T07:29:50.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.050+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc0804f330 con 0x7efc080ce0a0 2024-03-06T07:29:50.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.050+0000 7efbfea00640 1 -- 172.21.15.86:0/3201290152 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efbf8038020 con 0x7efc080ce0a0 2024-03-06T07:29:50.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.052+0000 7efbfea00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efbe806b4d0 0x7efbe806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.052+0000 7efbfea00640 1 -- 172.21.15.86:0/3201290152 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7efbf80ac580 con 0x7efc080ce0a0 2024-03-06T07:29:50.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.052+0000 7efc06000640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efbe806b4d0 0x7efbe806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.053+0000 7efc06000640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efbe806b4d0 0x7efbe806d990 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7efbf4006fd0 tx=0x7efbf4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.056+0000 7efbfea00640 1 -- 172.21.15.86:0/3201290152 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efbf803f070 con 0x7efc080ce0a0 2024-03-06T07:29:50.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.251+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7efc08072890 con 0x7efc080ce0a0 2024-03-06T07:29:50.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.253+0000 7efbfea00640 1 -- 172.21.15.86:0/3201290152 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v58) v1 ==== 74+0+16677 (secure 0 0 0) 0x7efc0806ee70 con 0x7efc080ce0a0 2024-03-06T07:29:50.255 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:50.255 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":58,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:47.263155+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:29:50.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efbe806b4d0 msgr2=0x7efbe806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efbe806b4d0 0x7efbe806d990 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7efbf4006fd0 tx=0x7efbf4008040 comp rx=0 tx=0).stop 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc080ce0a0 msgr2=0x7efc0815f120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc080ce0a0 0x7efc0815f120 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7efbf8000c00 tx=0x7efbf80046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 shutdown_connections 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efbe806b4d0 0x7efbe806d990 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efc0815f9d0 0x7efc08140ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efc08144c00 0x7efc0813a360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 --2- 172.21.15.86:0/3201290152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efc080ce0a0 0x7efc0815f120 unknown :-1 s=CLOSED pgs=655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 >> 172.21.15.86:0/3201290152 conn(0x7efc080b1730 msgr2=0x7efc080afe30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 shutdown_connections 2024-03-06T07:29:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.256+0000 7efc0da00640 1 -- 172.21.15.86:0/3201290152 wait complete. 2024-03-06T07:29:50.277 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:29:50.277 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:29:50.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.471+0000 7fcdf7200640 1 Processor -- start 2024-03-06T07:29:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.471+0000 7fcdf7200640 1 -- start start 2024-03-06T07:29:50.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.474+0000 7fcdf7200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fcdf0151660 0x7fcdf0071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.474+0000 7fcdf7200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdf00cf430 0x7fcdf006d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.474+0000 7fcdf7200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcdf0071a60 con 0x7fcdf006d920 2024-03-06T07:29:50.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.474+0000 7fcdf7200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcdf00b3ee0 con 0x7fcdf0151660 2024-03-06T07:29:50.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.474+0000 7fcdf7200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcdf00b35b0 con 0x7fcdf00cf430 2024-03-06T07:29:50.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.474+0000 7fcdf5200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fcdf006d920 0x7fcdf015dff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50034/0 (socket says 172.21.15.86:50034) 2024-03-06T07:29:50.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.474+0000 7fcdf5200640 1 -- 172.21.15.86:0/1074432822 learned_addr learned my addr 172.21.15.86:0/1074432822 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdef400640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdf00cf430 0x7fcdf006d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdefe00640 1 --2- 172.21.15.86:0/1074432822 >> v2:172.21.15.194:3300/0 conn(0x7fcdf0151660 0x7fcdf0071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 786135981 0 0) 0x7fcdf0071a60 con 0x7fcdf006d920 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcdd4003520 con 0x7fcdf006d920 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3433809438 0 0) 0x7fcdd4003520 con 0x7fcdf006d920 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdf00cf430 msgr2=0x7fcdf006d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdf00cf430 0x7fcdf006d3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 >> v2:172.21.15.194:3300/0 conn(0x7fcdf0151660 msgr2=0x7fcdf0071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 --2- 172.21.15.86:0/1074432822 >> v2:172.21.15.194:3300/0 conn(0x7fcdf0151660 0x7fcdf0071520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcdf00b3890 con 0x7fcdf006d920 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.475+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fcde80019f0 con 0x7fcdf006d920 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 >> v1:172.21.15.86:6789/0 conn(0x7fcdf006d920 legacy=0x7fcdf015dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdeea00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdd40039b0 0x7fcdd4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdeea00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdf00cf430 0x7fcdd4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdeea00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdd4004d40 0x7fcdd40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdf00b3ee0 con 0x7fcdd40039b0 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdf00b35b0 con 0x7fcdd4004d40 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcdd400ba40 con 0x7fcdf00cf430 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdefe00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdd40039b0 0x7fcdd4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdefe00640 1 -- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdf00cf430 msgr2=0x7fcdd4004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdefe00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdf00cf430 0x7fcdd4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdefe00640 1 -- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdd4004d40 msgr2=0x7fcdd40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdefe00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdd4004d40 0x7fcdd40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.476+0000 7fcdefe00640 1 -- 172.21.15.86:0/1074432822 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcdd4003520 con 0x7fcdd40039b0 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.478+0000 7fcdefe00640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdd40039b0 0x7fcdd4003dd0 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fcde000aac0 tx=0x7fcdd4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=50129959a9da791a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.478+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcde000b040 con 0x7fcdd40039b0 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.478+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcde0034600 con 0x7fcdd40039b0 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.478+0000 7fcdeea00640 1 -- 172.21.15.86:0/1074432822 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcde00349d0 con 0x7fcdd40039b0 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.479+0000 7fcdf7200640 1 -- 172.21.15.86:0/1074432822 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdd40039b0 msgr2=0x7fcdd4003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.479+0000 7fcdf7200640 1 --2- 172.21.15.86:0/1074432822 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdd40039b0 0x7fcdd4003dd0 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fcde000aac0 tx=0x7fcdd4005830 comp rx=0 tx=0).stop 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.479+0000 7fcdf5200640 1 -- 172.21.15.86:0/1074432822 reap_dead start 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.479+0000 7fcdf7200640 1 -- 172.21.15.86:0/1074432822 shutdown_connections 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.479+0000 7fcdf7200640 1 -- 172.21.15.86:0/1074432822 >> 172.21.15.86:0/1074432822 conn(0x7fcdf00b1730 msgr2=0x7fcdf00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.479+0000 7fcdf7200640 1 -- 172.21.15.86:0/1074432822 shutdown_connections 2024-03-06T07:29:50.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.479+0000 7fcdf7200640 1 -- 172.21.15.86:0/1074432822 wait complete. 2024-03-06T07:29:50.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.482+0000 7fcdf7200640 1 Processor -- start 2024-03-06T07:29:50.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.482+0000 7fcdf7200640 1 -- start start 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdf7200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdd40039b0 0x7fcdf01f5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdf7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdf00b0250 0x7fcdf00b06d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdf7200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdf0155f90 0x7fcdf0156410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdf7200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdf015e700 con 0x7fcdf00b0250 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdf7200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdf0071d30 con 0x7fcdf0155f90 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdf7200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcdf015e880 con 0x7fcdd40039b0 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdf00b0250 0x7fcdf00b06d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdf00b0250 0x7fcdf00b06d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57428/0 (socket says 172.21.15.86:57428) 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 -- 172.21.15.86:0/3365097906 learned_addr learned my addr 172.21.15.86:0/3365097906 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdefe00640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdd40039b0 0x7fcdf01f5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 -- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdd40039b0 msgr2=0x7fcdf01f5160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdf5200640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdf0155f90 0x7fcdf0156410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdd40039b0 0x7fcdf01f5160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 -- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdf0155f90 msgr2=0x7fcdf0156410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdf0155f90 0x7fcdf0156410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.484+0000 7fcdef400640 1 -- 172.21.15.86:0/3365097906 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcdf0156cc0 con 0x7fcdf00b0250 2024-03-06T07:29:50.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.485+0000 7fcdef400640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdf00b0250 0x7fcdf00b06d0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7fcddc00e7c0 tx=0x7fcddc00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.485+0000 7fcde7e00640 1 -- 172.21.15.86:0/3365097906 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcddc017070 con 0x7fcdf00b0250 2024-03-06T07:29:50.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.485+0000 7fcde7e00640 1 -- 172.21.15.86:0/3365097906 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcddc00c940 con 0x7fcdf00b0250 2024-03-06T07:29:50.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.485+0000 7fcde7e00640 1 -- 172.21.15.86:0/3365097906 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcddc00cd10 con 0x7fcdf00b0250 2024-03-06T07:29:50.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.485+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcdf0156fa0 con 0x7fcdf00b0250 2024-03-06T07:29:50.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.485+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcdf0157560 con 0x7fcdf00b0250 2024-03-06T07:29:50.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.486+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcdf004f330 con 0x7fcdf00b0250 2024-03-06T07:29:50.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.487+0000 7fcde7e00640 1 -- 172.21.15.86:0/3365097906 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcddc01a070 con 0x7fcdf00b0250 2024-03-06T07:29:50.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.488+0000 7fcde7e00640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcdd006b4d0 0x7fcdd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.488+0000 7fcde7e00640 1 -- 172.21.15.86:0/3365097906 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcddc08a480 con 0x7fcdf00b0250 2024-03-06T07:29:50.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.488+0000 7fcdefe00640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcdd006b4d0 0x7fcdd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.489+0000 7fcdefe00640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcdd006b4d0 0x7fcdd006d990 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fcde0037000 tx=0x7fcde0032000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.492+0000 7fcde7e00640 1 -- 172.21.15.86:0/3365097906 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcddc0592e0 con 0x7fcdf00b0250 2024-03-06T07:29:50.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.683+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fcdf00cc240 con 0x7fcdd006b4d0 2024-03-06T07:29:50.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.694+0000 7fcde7e00640 1 -- 172.21.15.86:0/3365097906 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+239859 (secure 0 0 0) 0x7fcdf00cc240 con 0x7fcdd006b4d0 2024-03-06T07:29:50.699 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:50.700 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcdd006b4d0 msgr2=0x7fcdd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcdd006b4d0 0x7fcdd006d990 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fcde0037000 tx=0x7fcde0032000 comp rx=0 tx=0).stop 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdf00b0250 msgr2=0x7fcdf00b06d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdf00b0250 0x7fcdf00b06d0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7fcddc00e7c0 tx=0x7fcddc00ec80 comp rx=0 tx=0).stop 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 shutdown_connections 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcdd006b4d0 0x7fcdd006d990 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcdf0155f90 0x7fcdf0156410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcdf00b0250 0x7fcdf00b06d0 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 --2- 172.21.15.86:0/3365097906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcdd40039b0 0x7fcdf01f5160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 >> 172.21.15.86:0/3365097906 conn(0x7fcdf00b1730 msgr2=0x7fcdf00ce920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:50.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 shutdown_connections 2024-03-06T07:29:50.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.699+0000 7fcdf7200640 1 -- 172.21.15.86:0/3365097906 wait complete. 2024-03-06T07:29:50.716 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":136,"stamp":"2024-03-06T07:29:48.899894+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":66,"acting":66,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":8,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":7,"kb":754974720,"kb_used":255716,"kb_used_data":6268,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754719004,"statfs":{"total":773094113280,"available":772832260096,"internally_reserved":0,"allocated":6418432,"data_stored":3310223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":24,"apply_latency_ms":24,"commit_latency_ns":24000000,"apply_latency_ns":24000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.679893"},"pg_stats":[{"pgid":"9.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.3e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.31","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.25","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"9.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.1f","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.229240+0000","last_change":"2024-03-06T07:29:44.229240+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.229240+0000","last_undegraded":"2024-03-06T07:29:44.229240+0000","last_fullsized":"2024-03-06T07:29:44.229240+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.1e","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.229422+0000","last_change":"2024-03-06T07:29:44.229422+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.229422+0000","last_undegraded":"2024-03-06T07:29:44.229422+0000","last_fullsized":"2024-03-06T07:29:44.229422+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.1d","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.228229+0000","last_change":"2024-03-06T07:29:44.228229+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.228229+0000","last_undegraded":"2024-03-06T07:29:44.228229+0000","last_fullsized":"2024-03-06T07:29:44.228229+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.1c","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.227378+0000","last_change":"2024-03-06T07:29:44.227378+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.227378+0000","last_undegraded":"2024-03-06T07:29:44.227378+0000","last_fullsized":"2024-03-06T07:29:44.227378+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.1b","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.233968+0000","last_change":"2024-03-06T07:29:44.233968+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.233968+0000","last_undegraded":"2024-03-06T07:29:44.233968+0000","last_fullsized":"2024-03-06T07:29:44.233968+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.1a","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.228004+0000","last_change":"2024-03-06T07:29:44.228004+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.228004+0000","last_undegraded":"2024-03-06T07:29:44.228004+0000","last_fullsized":"2024-03-06T07:29:44.228004+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.19","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.227522+0000","last_change":"2024-03-06T07:29:44.227522+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.227522+0000","last_undegraded":"2024-03-06T07:29:44.227522+0000","last_fullsized":"2024-03-06T07:29:44.227522+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.18","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.233781+0000","last_change":"2024-03-06T07:29:44.233781+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.233781+0000","last_undegraded":"2024-03-06T07:29:44.233781+0000","last_fullsized":"2024-03-06T07:29:44.233781+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.17","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.232727+0000","last_change":"2024-03-06T07:29:44.232727+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.232727+0000","last_undegraded":"2024-03-06T07:29:44.232727+0000","last_fullsized":"2024-03-06T07:29:44.232727+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.16","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.228871+0000","last_change":"2024-03-06T07:29:44.228871+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.228871+0000","last_undegraded":"2024-03-06T07:29:44.228871+0000","last_fullsized":"2024-03-06T07:29:44.228871+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.15","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.232345+0000","last_change":"2024-03-06T07:29:44.232345+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.232345+0000","last_undegraded":"2024-03-06T07:29:44.232345+0000","last_fullsized":"2024-03-06T07:29:44.232345+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.14","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.237369+0000","last_change":"2024-03-06T07:29:44.237369+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.237369+0000","last_undegraded":"2024-03-06T07:29:44.237369+0000","last_fullsized":"2024-03-06T07:29:44.237369+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.13","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.235087+0000","last_change":"2024-03-06T07:29:44.235087+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.235087+0000","last_undegraded":"2024-03-06T07:29:44.235087+0000","last_fullsized":"2024-03-06T07:29:44.235087+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.12","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.238834+0000","last_change":"2024-03-06T07:29:44.238834+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.238834+0000","last_undegraded":"2024-03-06T07:29:44.238834+0000","last_fullsized":"2024-03-06T07:29:44.238834+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.11","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.225919+0000","last_change":"2024-03-06T07:29:44.225919+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.225919+0000","last_undegraded":"2024-03-06T07:29:44.225919+0000","last_fullsized":"2024-03-06T07:29:44.225919+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.10","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.232105+0000","last_change":"2024-03-06T07:29:44.232105+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.232105+0000","last_undegraded":"2024-03-06T07:29:44.232105+0000","last_fullsized":"2024-03-06T07:29:44.232105+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.229180+0000","last_change":"2024-03-06T07:29:44.229180+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.229180+0000","last_undegraded":"2024-03-06T07:29:44.229180+0000","last_fullsized":"2024-03-06T07:29:44.229180+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.229695+0000","last_change":"2024-03-06T07:29:44.229695+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.229695+0000","last_undegraded":"2024-03-06T07:29:44.229695+0000","last_fullsized":"2024-03-06T07:29:44.229695+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.225952+0000","last_change":"2024-03-06T07:29:44.225952+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.225952+0000","last_undegraded":"2024-03-06T07:29:44.225952+0000","last_fullsized":"2024-03-06T07:29:44.225952+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.228048+0000","last_change":"2024-03-06T07:29:44.228048+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.228048+0000","last_undegraded":"2024-03-06T07:29:44.228048+0000","last_fullsized":"2024-03-06T07:29:44.228048+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.228889+0000","last_change":"2024-03-06T07:29:44.228889+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.228889+0000","last_undegraded":"2024-03-06T07:29:44.228889+0000","last_fullsized":"2024-03-06T07:29:44.228889+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.225121+0000","last_change":"2024-03-06T07:29:44.225121+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.225121+0000","last_undegraded":"2024-03-06T07:29:44.225121+0000","last_fullsized":"2024-03-06T07:29:44.225121+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.225904+0000","last_change":"2024-03-06T07:29:44.225904+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.225904+0000","last_undegraded":"2024-03-06T07:29:44.225904+0000","last_fullsized":"2024-03-06T07:29:44.225904+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.225242+0000","last_change":"2024-03-06T07:29:44.225242+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.225242+0000","last_undegraded":"2024-03-06T07:29:44.225242+0000","last_fullsized":"2024-03-06T07:29:44.225242+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.228070+0000","last_change":"2024-03-06T07:29:44.228070+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.228070+0000","last_undegraded":"2024-03-06T07:29:44.228070+0000","last_fullsized":"2024-03-06T07:29:44.228070+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.239100+0000","last_change":"2024-03-06T07:29:44.239100+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.239100+0000","last_undegraded":"2024-03-06T07:29:44.239100+0000","last_fullsized":"2024-03-06T07:29:44.239100+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.225895+0000","last_change":"2024-03-06T07:29:44.225895+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.225895+0000","last_undegraded":"2024-03-06T07:29:44.225895+0000","last_fullsized":"2024-03-06T07:29:44.225895+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.224951+0000","last_change":"2024-03-06T07:29:44.224951+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.224951+0000","last_undegraded":"2024-03-06T07:29:44.224951+0000","last_fullsized":"2024-03-06T07:29:44.224951+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.235966+0000","last_change":"2024-03-06T07:29:44.235966+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.235966+0000","last_undegraded":"2024-03-06T07:29:44.235966+0000","last_fullsized":"2024-03-06T07:29:44.235966+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.224988+0000","last_change":"2024-03-06T07:29:44.224988+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.224988+0000","last_undegraded":"2024-03-06T07:29:44.224988+0000","last_fullsized":"2024-03-06T07:29:44.224988+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.228846+0000","last_change":"2024-03-06T07:29:44.228846+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.228846+0000","last_undegraded":"2024-03-06T07:29:44.228846+0000","last_fullsized":"2024-03-06T07:29:44.228846+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:29:45.218494+0000","last_change":"2024-03-06T07:29:45.218494+0000","last_active":"2024-03-06T07:29:45.218494+0000","last_peered":"2024-03-06T07:29:45.218494+0000","last_clean":"2024-03-06T07:29:45.218494+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:45.218494+0000","last_undegraded":"2024-03-06T07:29:45.218494+0000","last_fullsized":"2024-03-06T07:29:45.218494+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":1,"reported_epoch":55,"state":"creating+peering","last_fresh":"2024-03-06T07:29:44.237190+0000","last_change":"2024-03-06T07:29:44.237190+0000","last_active":"2024-03-06T07:29:44.202758+0000","last_peered":"2024-03-06T07:29:44.202758+0000","last_clean":"2024-03-06T07:29:44.202758+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:29:44.237190+0000","last_undegraded":"2024-03-06T07:29:44.237190+0000","last_fullsized":"2024-03-06T07:29:44.237190+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":227,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-06T07:29:44.237750+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:29:44.237750+0000","last_peered":"2024-03-06T07:29:44.237750+0000","last_clean":"2024-03-06T07:29:44.237750+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:29:44.237750+0000","last_undegraded":"2024-03-06T07:29:44.237750+0000","last_fullsized":"2024-03-06T07:29:44.237750+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":8,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542181,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32400,"kb_used_data":1220,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339440,"statfs":{"total":96636764160,"available":96603586560,"internally_reserved":0,"allocated":1249280,"data_stored":857281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542180,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31824,"kb_used_data":640,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340016,"statfs":{"total":96636764160,"available":96604176384,"internally_reserved":0,"allocated":655360,"data_stored":266913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542181,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31820,"kb_used_data":640,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340020,"statfs":{"total":96636764160,"available":96604180480,"internally_reserved":0,"allocated":655360,"data_stored":266913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542180,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31820,"kb_used_data":640,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340020,"statfs":{"total":96636764160,"available":96604180480,"internally_reserved":0,"allocated":655360,"data_stored":266913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542180,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31820,"kb_used_data":640,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340020,"statfs":{"total":96636764160,"available":96604180480,"internally_reserved":0,"allocated":655360,"data_stored":266913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542181,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":31820,"kb_used_data":640,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340020,"statfs":{"total":96636764160,"available":96604180480,"internally_reserved":0,"allocated":655360,"data_stored":266913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542182,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31820,"kb_used_data":640,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340020,"statfs":{"total":96636764160,"available":96604180480,"internally_reserved":0,"allocated":655360,"data_stored":266913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542179,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32392,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339448,"statfs":{"total":96636764160,"available":96603594752,"internally_reserved":0,"allocated":1236992,"data_stored":851464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:29:50.721 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:29:50.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.962+0000 7f5563e00640 1 Processor -- start 2024-03-06T07:29:50.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.962+0000 7f5563e00640 1 -- start start 2024-03-06T07:29:50.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f5563e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f55640d27a0 0x7f55640b45d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f5563e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5564161960 con 0x7f55640d2360 2024-03-06T07:29:50.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f5563e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55640cf6f0 con 0x7f5564161580 2024-03-06T07:29:50.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f5563e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f55640cf9d0 con 0x7f55640d27a0 2024-03-06T07:29:50.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f555be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f55640d2360 0x7f55640b3ec0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50044/0 (socket says 172.21.15.86:50044) 2024-03-06T07:29:50.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f555be00640 1 -- 172.21.15.86:0/752590399 learned_addr learned my addr 172.21.15.86:0/752590399 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:50.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f5563400640 1 --2- 172.21.15.86:0/752590399 >> v2:172.21.15.194:3300/0 conn(0x7f55640d27a0 0x7f55640b45d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.964+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 353164117 0 0) 0x7f5564161960 con 0x7f55640d2360 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5548003520 con 0x7f55640d2360 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 948543957 0 0) 0x7f5548003520 con 0x7f55640d2360 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 >> v2:172.21.15.194:3300/0 conn(0x7f55640d27a0 msgr2=0x7f55640b45d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 --2- 172.21.15.86:0/752590399 >> v2:172.21.15.194:3300/0 conn(0x7f55640d27a0 0x7f55640b45d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 >> v1:172.21.15.194:6789/0 conn(0x7f5564161580 legacy=0x7f55640d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55640cfcb0 con 0x7f55640d2360 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f554c001670 con 0x7f55640d2360 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 >> v1:172.21.15.86:6789/0 conn(0x7f55640d2360 legacy=0x7f55640b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55480039b0 0x7f5548003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5548004800 0x7f5548004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55480051c0 0x7f554800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55640cf9d0 con 0x7f55480051c0 2024-03-06T07:29:50.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f554800c110 con 0x7f5548004800 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f554800c310 con 0x7f55480039b0 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.965+0000 7f5563400640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55480051c0 0x7f554800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.966+0000 7f5563400640 1 -- 172.21.15.86:0/752590399 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55480039b0 msgr2=0x7f5548003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.966+0000 7f5562a00640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55480039b0 0x7f5548003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.966+0000 7f5563400640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55480039b0 0x7f5548003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.966+0000 7f5563400640 1 -- 172.21.15.86:0/752590399 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5548004800 msgr2=0x7f5548004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.966+0000 7f5563400640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5548004800 0x7f5548004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.966+0000 7f5563400640 1 -- 172.21.15.86:0/752590399 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5548003520 con 0x7f55480051c0 2024-03-06T07:29:50.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.967+0000 7f5563400640 1 -- 172.21.15.86:0/752590399 reap_dead start 2024-03-06T07:29:50.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.967+0000 7f5563400640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55480051c0 0x7f554800b9c0 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f555c007360 tx=0x7f5548005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=57a16dc96fd9393b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f555c009be0 con 0x7f55480051c0 2024-03-06T07:29:50.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f555c02cd30 con 0x7f55480051c0 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5562000640 1 -- 172.21.15.86:0/752590399 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f555c003bd0 con 0x7f55480051c0 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5563e00640 1 -- 172.21.15.86:0/752590399 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55480051c0 msgr2=0x7f554800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5563e00640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55480051c0 0x7f554800b9c0 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f555c007360 tx=0x7f5548005830 comp rx=0 tx=0).stop 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5563e00640 1 -- 172.21.15.86:0/752590399 shutdown_connections 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5563e00640 1 --2- 172.21.15.86:0/752590399 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55480051c0 0x7f554800b9c0 unknown :-1 s=CLOSED pgs=660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.968+0000 7f5563e00640 1 -- 172.21.15.86:0/752590399 >> 172.21.15.86:0/752590399 conn(0x7f55640d3d80 msgr2=0x7f55640d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.969+0000 7f5563e00640 1 -- 172.21.15.86:0/752590399 shutdown_connections 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.969+0000 7f5563e00640 1 -- 172.21.15.86:0/752590399 wait complete. 2024-03-06T07:29:50.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.970+0000 7f5563e00640 1 Processor -- start 2024-03-06T07:29:50.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.970+0000 7f5563e00640 1 -- start start 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.971+0000 7f5563e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55480051c0 0x7f55640c7840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.971+0000 7f5563e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55640cdce0 0x7f55640c9c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.971+0000 7f5563e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55641f3f10 0x7f55641f4390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.971+0000 7f5563e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55640d2bf0 con 0x7f55641f3f10 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.971+0000 7f5563e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55640b39a0 con 0x7f55480051c0 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.971+0000 7f5563e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55640cf3a0 con 0x7f55640cdce0 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.971+0000 7f5563400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55641f3f10 0x7f55641f4390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55641f3f10 0x7f55641f4390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57436/0 (socket says 172.21.15.86:57436) 2024-03-06T07:29:50.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 -- 172.21.15.86:0/347201490 learned_addr learned my addr 172.21.15.86:0/347201490 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f555be00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55640cdce0 0x7f55640c9c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 -- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55640cdce0 msgr2=0x7f55640c9c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55640cdce0 0x7f55640c9c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5562a00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55480051c0 0x7f55640c7840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 -- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55480051c0 msgr2=0x7f55640c7840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55480051c0 0x7f55640c7840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 -- 172.21.15.86:0/347201490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55640ca160 con 0x7f55641f3f10 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5562a00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55480051c0 0x7f55640c7840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.972+0000 7f5563400640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55641f3f10 0x7f55641f4390 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f555c007450 tx=0x7f555c004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.973+0000 7f555aa00640 1 -- 172.21.15.86:0/347201490 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f555c009be0 con 0x7f55641f3f10 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.973+0000 7f555aa00640 1 -- 172.21.15.86:0/347201490 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f555c003e30 con 0x7f55641f3f10 2024-03-06T07:29:50.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.973+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f55640ca400 con 0x7f55641f3f10 2024-03-06T07:29:50.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.973+0000 7f555aa00640 1 -- 172.21.15.86:0/347201490 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f555c034440 con 0x7f55641f3f10 2024-03-06T07:29:50.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.973+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55640ca9c0 con 0x7f55641f3f10 2024-03-06T07:29:50.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.974+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f556404f330 con 0x7f55641f3f10 2024-03-06T07:29:50.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.976+0000 7f555aa00640 1 -- 172.21.15.86:0/347201490 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f555c005d10 con 0x7f55641f3f10 2024-03-06T07:29:50.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.977+0000 7f555aa00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f554006b500 0x7f554006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:50.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.977+0000 7f555aa00640 1 -- 172.21.15.86:0/347201490 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f555c0ab9d0 con 0x7f55641f3f10 2024-03-06T07:29:50.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.977+0000 7f5562a00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f554006b500 0x7f554006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:50.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.978+0000 7f5562a00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f554006b500 0x7f554006d9c0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f55540015f0 tx=0x7f5554009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:50.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:50.982+0000 7f555aa00640 1 -- 172.21.15.86:0/347201490 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f555c07a850 con 0x7f55641f3f10 2024-03-06T07:29:51.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.164+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f556404f330 con 0x7f554006b500 2024-03-06T07:29:51.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.177+0000 7f555aa00640 1 -- 172.21.15.86:0/347201490 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251955 (secure 0 0 0) 0x7f556404f330 con 0x7f554006b500 2024-03-06T07:29:51.182 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:51.184 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:29:51.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f554006b500 msgr2=0x7f554006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f554006b500 0x7f554006d9c0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f55540015f0 tx=0x7f5554009040 comp rx=0 tx=0).stop 2024-03-06T07:29:51.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55641f3f10 msgr2=0x7f55641f4390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55641f3f10 0x7f55641f4390 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f555c007450 tx=0x7f555c004ab0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 shutdown_connections 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f554006b500 0x7f554006d9c0 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55641f3f10 0x7f55641f4390 unknown :-1 s=CLOSED pgs=661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55640cdce0 0x7f55640c9c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 --2- 172.21.15.86:0/347201490 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55480051c0 0x7f55640c7840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 >> 172.21.15.86:0/347201490 conn(0x7f55640d3d80 msgr2=0x7f55640b4ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 shutdown_connections 2024-03-06T07:29:51.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.183+0000 7f5563e00640 1 -- 172.21.15.86:0/347201490 wait complete. 2024-03-06T07:29:51.210 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":137,"stamp":"2024-03-06T07:29:50.900502+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":256096,"kb_used_data":6632,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754718624,"statfs":{"total":773094113280,"available":772831870976,"internally_reserved":0,"allocated":6791168,"data_stored":3475596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":44,"apply_latency_ms":44,"commit_latency_ns":44000000,"apply_latency_ns":44000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001847"},"pg_stats":[{"pgid":"9.3f","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626469+0000","last_change":"2024-03-06T07:29:46.276521+0000","last_active":"2024-03-06T07:29:47.626469+0000","last_peered":"2024-03-06T07:29:47.626469+0000","last_clean":"2024-03-06T07:29:47.626469+0000","last_became_active":"2024-03-06T07:29:46.276253+0000","last_became_peered":"2024-03-06T07:29:46.276253+0000","last_unstale":"2024-03-06T07:29:47.626469+0000","last_undegraded":"2024-03-06T07:29:47.626469+0000","last_fullsized":"2024-03-06T07:29:47.626469+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:30:11.201907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3e","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624995+0000","last_change":"2024-03-06T07:29:46.277826+0000","last_active":"2024-03-06T07:29:47.624995+0000","last_peered":"2024-03-06T07:29:47.624995+0000","last_clean":"2024-03-06T07:29:47.624995+0000","last_became_active":"2024-03-06T07:29:46.277528+0000","last_became_peered":"2024-03-06T07:29:46.277528+0000","last_unstale":"2024-03-06T07:29:47.624995+0000","last_undegraded":"2024-03-06T07:29:47.624995+0000","last_fullsized":"2024-03-06T07:29:47.624995+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:59:48.130870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.3d","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627134+0000","last_change":"2024-03-06T07:29:46.297017+0000","last_active":"2024-03-06T07:29:47.627134+0000","last_peered":"2024-03-06T07:29:47.627134+0000","last_clean":"2024-03-06T07:29:47.627134+0000","last_became_active":"2024-03-06T07:29:46.296759+0000","last_became_peered":"2024-03-06T07:29:46.296759+0000","last_unstale":"2024-03-06T07:29:47.627134+0000","last_undegraded":"2024-03-06T07:29:47.627134+0000","last_fullsized":"2024-03-06T07:29:47.627134+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:22:21.744551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.3c","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.521958+0000","last_change":"2024-03-06T07:29:46.288062+0000","last_active":"2024-03-06T07:29:47.521958+0000","last_peered":"2024-03-06T07:29:47.521958+0000","last_clean":"2024-03-06T07:29:47.521958+0000","last_became_active":"2024-03-06T07:29:46.287751+0000","last_became_peered":"2024-03-06T07:29:46.287751+0000","last_unstale":"2024-03-06T07:29:47.521958+0000","last_undegraded":"2024-03-06T07:29:47.521958+0000","last_fullsized":"2024-03-06T07:29:47.521958+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:39:41.791515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.3b","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626038+0000","last_change":"2024-03-06T07:29:46.294618+0000","last_active":"2024-03-06T07:29:47.626038+0000","last_peered":"2024-03-06T07:29:47.626038+0000","last_clean":"2024-03-06T07:29:47.626038+0000","last_became_active":"2024-03-06T07:29:46.294330+0000","last_became_peered":"2024-03-06T07:29:46.294330+0000","last_unstale":"2024-03-06T07:29:47.626038+0000","last_undegraded":"2024-03-06T07:29:47.626038+0000","last_fullsized":"2024-03-06T07:29:47.626038+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:15:18.821955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3a","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623093+0000","last_change":"2024-03-06T07:29:46.269579+0000","last_active":"2024-03-06T07:29:47.623093+0000","last_peered":"2024-03-06T07:29:47.623093+0000","last_clean":"2024-03-06T07:29:47.623093+0000","last_became_active":"2024-03-06T07:29:46.269276+0000","last_became_peered":"2024-03-06T07:29:46.269276+0000","last_unstale":"2024-03-06T07:29:47.623093+0000","last_undegraded":"2024-03-06T07:29:47.623093+0000","last_fullsized":"2024-03-06T07:29:47.623093+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:05:56.818895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.39","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273951+0000","last_change":"2024-03-06T07:29:46.285306+0000","last_active":"2024-03-06T07:29:47.273951+0000","last_peered":"2024-03-06T07:29:47.273951+0000","last_clean":"2024-03-06T07:29:47.273951+0000","last_became_active":"2024-03-06T07:29:46.284989+0000","last_became_peered":"2024-03-06T07:29:46.284989+0000","last_unstale":"2024-03-06T07:29:47.273951+0000","last_undegraded":"2024-03-06T07:29:47.273951+0000","last_fullsized":"2024-03-06T07:29:47.273951+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:24:28.993452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.38","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623998+0000","last_change":"2024-03-06T07:29:46.290801+0000","last_active":"2024-03-06T07:29:47.623998+0000","last_peered":"2024-03-06T07:29:47.623998+0000","last_clean":"2024-03-06T07:29:47.623998+0000","last_became_active":"2024-03-06T07:29:46.290490+0000","last_became_peered":"2024-03-06T07:29:46.290490+0000","last_unstale":"2024-03-06T07:29:47.623998+0000","last_undegraded":"2024-03-06T07:29:47.623998+0000","last_fullsized":"2024-03-06T07:29:47.623998+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:57:16.041791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.37","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.274798+0000","last_change":"2024-03-06T07:29:46.267841+0000","last_active":"2024-03-06T07:29:47.274798+0000","last_peered":"2024-03-06T07:29:47.274798+0000","last_clean":"2024-03-06T07:29:47.274798+0000","last_became_active":"2024-03-06T07:29:46.267108+0000","last_became_peered":"2024-03-06T07:29:46.267108+0000","last_unstale":"2024-03-06T07:29:47.274798+0000","last_undegraded":"2024-03-06T07:29:47.274798+0000","last_fullsized":"2024-03-06T07:29:47.274798+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:11:01.922386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.36","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273580+0000","last_change":"2024-03-06T07:29:46.283065+0000","last_active":"2024-03-06T07:29:47.273580+0000","last_peered":"2024-03-06T07:29:47.273580+0000","last_clean":"2024-03-06T07:29:47.273580+0000","last_became_active":"2024-03-06T07:29:46.282819+0000","last_became_peered":"2024-03-06T07:29:46.282819+0000","last_unstale":"2024-03-06T07:29:47.273580+0000","last_undegraded":"2024-03-06T07:29:47.273580+0000","last_fullsized":"2024-03-06T07:29:47.273580+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:05:26.066205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.35","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273736+0000","last_change":"2024-03-06T07:29:46.278627+0000","last_active":"2024-03-06T07:29:47.273736+0000","last_peered":"2024-03-06T07:29:47.273736+0000","last_clean":"2024-03-06T07:29:47.273736+0000","last_became_active":"2024-03-06T07:29:46.278353+0000","last_became_peered":"2024-03-06T07:29:46.278353+0000","last_unstale":"2024-03-06T07:29:47.273736+0000","last_undegraded":"2024-03-06T07:29:47.273736+0000","last_fullsized":"2024-03-06T07:29:47.273736+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:40:20.546649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.34","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522106+0000","last_change":"2024-03-06T07:29:46.292016+0000","last_active":"2024-03-06T07:29:47.522106+0000","last_peered":"2024-03-06T07:29:47.522106+0000","last_clean":"2024-03-06T07:29:47.522106+0000","last_became_active":"2024-03-06T07:29:46.291543+0000","last_became_peered":"2024-03-06T07:29:46.291543+0000","last_unstale":"2024-03-06T07:29:47.522106+0000","last_undegraded":"2024-03-06T07:29:47.522106+0000","last_fullsized":"2024-03-06T07:29:47.522106+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:16:20.714797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.33","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625610+0000","last_change":"2024-03-06T07:29:46.290503+0000","last_active":"2024-03-06T07:29:47.625610+0000","last_peered":"2024-03-06T07:29:47.625610+0000","last_clean":"2024-03-06T07:29:47.625610+0000","last_became_active":"2024-03-06T07:29:46.290116+0000","last_became_peered":"2024-03-06T07:29:46.290116+0000","last_unstale":"2024-03-06T07:29:47.625610+0000","last_undegraded":"2024-03-06T07:29:47.625610+0000","last_fullsized":"2024-03-06T07:29:47.625610+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:10:07.110179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.32","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.274608+0000","last_change":"2024-03-06T07:29:46.272081+0000","last_active":"2024-03-06T07:29:47.274608+0000","last_peered":"2024-03-06T07:29:47.274608+0000","last_clean":"2024-03-06T07:29:47.274608+0000","last_became_active":"2024-03-06T07:29:46.270892+0000","last_became_peered":"2024-03-06T07:29:46.270892+0000","last_unstale":"2024-03-06T07:29:47.274608+0000","last_undegraded":"2024-03-06T07:29:47.274608+0000","last_fullsized":"2024-03-06T07:29:47.274608+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:23:58.869841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.31","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.274162+0000","last_change":"2024-03-06T07:29:46.277246+0000","last_active":"2024-03-06T07:29:47.274162+0000","last_peered":"2024-03-06T07:29:47.274162+0000","last_clean":"2024-03-06T07:29:47.274162+0000","last_became_active":"2024-03-06T07:29:46.276540+0000","last_became_peered":"2024-03-06T07:29:46.276540+0000","last_unstale":"2024-03-06T07:29:47.274162+0000","last_undegraded":"2024-03-06T07:29:47.274162+0000","last_fullsized":"2024-03-06T07:29:47.274162+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:10:21.457295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.30","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623350+0000","last_change":"2024-03-06T07:29:46.292301+0000","last_active":"2024-03-06T07:29:47.623350+0000","last_peered":"2024-03-06T07:29:47.623350+0000","last_clean":"2024-03-06T07:29:47.623350+0000","last_became_active":"2024-03-06T07:29:46.292009+0000","last_became_peered":"2024-03-06T07:29:46.292009+0000","last_unstale":"2024-03-06T07:29:47.623350+0000","last_undegraded":"2024-03-06T07:29:47.623350+0000","last_fullsized":"2024-03-06T07:29:47.623350+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:08:41.720309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.2f","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624972+0000","last_change":"2024-03-06T07:29:46.286920+0000","last_active":"2024-03-06T07:29:47.624972+0000","last_peered":"2024-03-06T07:29:47.624972+0000","last_clean":"2024-03-06T07:29:47.624972+0000","last_became_active":"2024-03-06T07:29:46.285039+0000","last_became_peered":"2024-03-06T07:29:46.285039+0000","last_unstale":"2024-03-06T07:29:47.624972+0000","last_undegraded":"2024-03-06T07:29:47.624972+0000","last_fullsized":"2024-03-06T07:29:47.624972+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:30:17.825031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2e","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626966+0000","last_change":"2024-03-06T07:29:46.285057+0000","last_active":"2024-03-06T07:29:47.626966+0000","last_peered":"2024-03-06T07:29:47.626966+0000","last_clean":"2024-03-06T07:29:47.626966+0000","last_became_active":"2024-03-06T07:29:46.284730+0000","last_became_peered":"2024-03-06T07:29:46.284730+0000","last_unstale":"2024-03-06T07:29:47.626966+0000","last_undegraded":"2024-03-06T07:29:47.626966+0000","last_fullsized":"2024-03-06T07:29:47.626966+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:34:18.681880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2d","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626582+0000","last_change":"2024-03-06T07:29:46.279506+0000","last_active":"2024-03-06T07:29:47.626582+0000","last_peered":"2024-03-06T07:29:47.626582+0000","last_clean":"2024-03-06T07:29:47.626582+0000","last_became_active":"2024-03-06T07:29:46.279240+0000","last_became_peered":"2024-03-06T07:29:46.279240+0000","last_unstale":"2024-03-06T07:29:47.626582+0000","last_undegraded":"2024-03-06T07:29:47.626582+0000","last_fullsized":"2024-03-06T07:29:47.626582+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:45.902711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.2c","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624425+0000","last_change":"2024-03-06T07:29:46.270344+0000","last_active":"2024-03-06T07:29:47.624425+0000","last_peered":"2024-03-06T07:29:47.624425+0000","last_clean":"2024-03-06T07:29:47.624425+0000","last_became_active":"2024-03-06T07:29:46.270050+0000","last_became_peered":"2024-03-06T07:29:46.270050+0000","last_unstale":"2024-03-06T07:29:47.624425+0000","last_undegraded":"2024-03-06T07:29:47.624425+0000","last_fullsized":"2024-03-06T07:29:47.624425+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:19:22.000324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2b","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626472+0000","last_change":"2024-03-06T07:29:46.289577+0000","last_active":"2024-03-06T07:29:47.626472+0000","last_peered":"2024-03-06T07:29:47.626472+0000","last_clean":"2024-03-06T07:29:47.626472+0000","last_became_active":"2024-03-06T07:29:46.289002+0000","last_became_peered":"2024-03-06T07:29:46.289002+0000","last_unstale":"2024-03-06T07:29:47.626472+0000","last_undegraded":"2024-03-06T07:29:47.626472+0000","last_fullsized":"2024-03-06T07:29:47.626472+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:02:03.792947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2a","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623875+0000","last_change":"2024-03-06T07:29:46.273962+0000","last_active":"2024-03-06T07:29:47.623875+0000","last_peered":"2024-03-06T07:29:47.623875+0000","last_clean":"2024-03-06T07:29:47.623875+0000","last_became_active":"2024-03-06T07:29:46.273669+0000","last_became_peered":"2024-03-06T07:29:46.273669+0000","last_unstale":"2024-03-06T07:29:47.623875+0000","last_undegraded":"2024-03-06T07:29:47.623875+0000","last_fullsized":"2024-03-06T07:29:47.623875+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:54:15.218884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.29","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626595+0000","last_change":"2024-03-06T07:29:46.288371+0000","last_active":"2024-03-06T07:29:47.626595+0000","last_peered":"2024-03-06T07:29:47.626595+0000","last_clean":"2024-03-06T07:29:47.626595+0000","last_became_active":"2024-03-06T07:29:46.288073+0000","last_became_peered":"2024-03-06T07:29:46.288073+0000","last_unstale":"2024-03-06T07:29:47.626595+0000","last_undegraded":"2024-03-06T07:29:47.626595+0000","last_fullsized":"2024-03-06T07:29:47.626595+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:10:33.506643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.28","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625102+0000","last_change":"2024-03-06T07:29:46.296393+0000","last_active":"2024-03-06T07:29:47.625102+0000","last_peered":"2024-03-06T07:29:47.625102+0000","last_clean":"2024-03-06T07:29:47.625102+0000","last_became_active":"2024-03-06T07:29:46.296079+0000","last_became_peered":"2024-03-06T07:29:46.296079+0000","last_unstale":"2024-03-06T07:29:47.625102+0000","last_undegraded":"2024-03-06T07:29:47.625102+0000","last_fullsized":"2024-03-06T07:29:47.625102+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:10:33.911540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.27","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626690+0000","last_change":"2024-03-06T07:29:46.280567+0000","last_active":"2024-03-06T07:29:47.626690+0000","last_peered":"2024-03-06T07:29:47.626690+0000","last_clean":"2024-03-06T07:29:47.626690+0000","last_became_active":"2024-03-06T07:29:46.280300+0000","last_became_peered":"2024-03-06T07:29:46.280300+0000","last_unstale":"2024-03-06T07:29:47.626690+0000","last_undegraded":"2024-03-06T07:29:47.626690+0000","last_fullsized":"2024-03-06T07:29:47.626690+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:16:55.949198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.26","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522081+0000","last_change":"2024-03-06T07:29:46.294840+0000","last_active":"2024-03-06T07:29:47.522081+0000","last_peered":"2024-03-06T07:29:47.522081+0000","last_clean":"2024-03-06T07:29:47.522081+0000","last_became_active":"2024-03-06T07:29:46.294454+0000","last_became_peered":"2024-03-06T07:29:46.294454+0000","last_unstale":"2024-03-06T07:29:47.522081+0000","last_undegraded":"2024-03-06T07:29:47.522081+0000","last_fullsized":"2024-03-06T07:29:47.522081+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:33:59.510360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.25","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623432+0000","last_change":"2024-03-06T07:29:46.292009+0000","last_active":"2024-03-06T07:29:47.623432+0000","last_peered":"2024-03-06T07:29:47.623432+0000","last_clean":"2024-03-06T07:29:47.623432+0000","last_became_active":"2024-03-06T07:29:46.289988+0000","last_became_peered":"2024-03-06T07:29:46.289988+0000","last_unstale":"2024-03-06T07:29:47.623432+0000","last_undegraded":"2024-03-06T07:29:47.623432+0000","last_fullsized":"2024-03-06T07:29:47.623432+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:58:36.649724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.24","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626628+0000","last_change":"2024-03-06T07:29:46.289433+0000","last_active":"2024-03-06T07:29:47.626628+0000","last_peered":"2024-03-06T07:29:47.626628+0000","last_clean":"2024-03-06T07:29:47.626628+0000","last_became_active":"2024-03-06T07:29:46.288949+0000","last_became_peered":"2024-03-06T07:29:46.288949+0000","last_unstale":"2024-03-06T07:29:47.626628+0000","last_undegraded":"2024-03-06T07:29:47.626628+0000","last_fullsized":"2024-03-06T07:29:47.626628+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:28:40.600399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.23","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624503+0000","last_change":"2024-03-06T07:29:46.285503+0000","last_active":"2024-03-06T07:29:47.624503+0000","last_peered":"2024-03-06T07:29:47.624503+0000","last_clean":"2024-03-06T07:29:47.624503+0000","last_became_active":"2024-03-06T07:29:46.284730+0000","last_became_peered":"2024-03-06T07:29:46.284730+0000","last_unstale":"2024-03-06T07:29:47.624503+0000","last_undegraded":"2024-03-06T07:29:47.624503+0000","last_fullsized":"2024-03-06T07:29:47.624503+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:07:11.604833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.22","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626608+0000","last_change":"2024-03-06T07:29:46.292184+0000","last_active":"2024-03-06T07:29:47.626608+0000","last_peered":"2024-03-06T07:29:47.626608+0000","last_clean":"2024-03-06T07:29:47.626608+0000","last_became_active":"2024-03-06T07:29:46.291647+0000","last_became_peered":"2024-03-06T07:29:46.291647+0000","last_unstale":"2024-03-06T07:29:47.626608+0000","last_undegraded":"2024-03-06T07:29:47.626608+0000","last_fullsized":"2024-03-06T07:29:47.626608+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:50:43.509141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.21","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625899+0000","last_change":"2024-03-06T07:29:46.278272+0000","last_active":"2024-03-06T07:29:47.625899+0000","last_peered":"2024-03-06T07:29:47.625899+0000","last_clean":"2024-03-06T07:29:47.625899+0000","last_became_active":"2024-03-06T07:29:46.278022+0000","last_became_peered":"2024-03-06T07:29:46.278022+0000","last_unstale":"2024-03-06T07:29:47.625899+0000","last_undegraded":"2024-03-06T07:29:47.625899+0000","last_fullsized":"2024-03-06T07:29:47.625899+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:51.131344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.20","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.622931+0000","last_change":"2024-03-06T07:29:46.266559+0000","last_active":"2024-03-06T07:29:47.622931+0000","last_peered":"2024-03-06T07:29:47.622931+0000","last_clean":"2024-03-06T07:29:47.622931+0000","last_became_active":"2024-03-06T07:29:46.266208+0000","last_became_peered":"2024-03-06T07:29:46.266208+0000","last_unstale":"2024-03-06T07:29:47.622931+0000","last_undegraded":"2024-03-06T07:29:47.622931+0000","last_fullsized":"2024-03-06T07:29:47.622931+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:28:31.506024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1e","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623859+0000","last_change":"2024-03-06T07:29:46.291556+0000","last_active":"2024-03-06T07:29:47.623859+0000","last_peered":"2024-03-06T07:29:47.623859+0000","last_clean":"2024-03-06T07:29:47.623859+0000","last_became_active":"2024-03-06T07:29:46.287619+0000","last_became_peered":"2024-03-06T07:29:46.287619+0000","last_unstale":"2024-03-06T07:29:47.623859+0000","last_undegraded":"2024-03-06T07:29:47.623859+0000","last_fullsized":"2024-03-06T07:29:47.623859+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:48:28.399406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1f","version":"57'4","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623334+0000","last_change":"2024-03-06T07:29:45.266926+0000","last_active":"2024-03-06T07:29:47.623334+0000","last_peered":"2024-03-06T07:29:47.623334+0000","last_clean":"2024-03-06T07:29:47.623334+0000","last_became_active":"2024-03-06T07:29:45.266604+0000","last_became_peered":"2024-03-06T07:29:45.266604+0000","last_unstale":"2024-03-06T07:29:47.623334+0000","last_undegraded":"2024-03-06T07:29:47.623334+0000","last_fullsized":"2024-03-06T07:29:47.623334+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:24:27.977716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1f","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.274097+0000","last_change":"2024-03-06T07:29:46.285940+0000","last_active":"2024-03-06T07:29:47.274097+0000","last_peered":"2024-03-06T07:29:47.274097+0000","last_clean":"2024-03-06T07:29:47.274097+0000","last_became_active":"2024-03-06T07:29:46.285691+0000","last_became_peered":"2024-03-06T07:29:46.285691+0000","last_unstale":"2024-03-06T07:29:47.274097+0000","last_undegraded":"2024-03-06T07:29:47.274097+0000","last_fullsized":"2024-03-06T07:29:47.274097+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:31.759467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1e","version":"57'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625115+0000","last_change":"2024-03-06T07:29:45.245401+0000","last_active":"2024-03-06T07:29:47.625115+0000","last_peered":"2024-03-06T07:29:47.625115+0000","last_clean":"2024-03-06T07:29:47.625115+0000","last_became_active":"2024-03-06T07:29:45.244831+0000","last_became_peered":"2024-03-06T07:29:45.244831+0000","last_unstale":"2024-03-06T07:29:47.625115+0000","last_undegraded":"2024-03-06T07:29:47.625115+0000","last_fullsized":"2024-03-06T07:29:47.625115+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:40.027465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623205+0000","last_change":"2024-03-06T07:29:46.277994+0000","last_active":"2024-03-06T07:29:47.623205+0000","last_peered":"2024-03-06T07:29:47.623205+0000","last_clean":"2024-03-06T07:29:47.623205+0000","last_became_active":"2024-03-06T07:29:46.277720+0000","last_became_peered":"2024-03-06T07:29:46.277720+0000","last_unstale":"2024-03-06T07:29:47.623205+0000","last_undegraded":"2024-03-06T07:29:47.623205+0000","last_fullsized":"2024-03-06T07:29:47.623205+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:22:55.802032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1d","version":"57'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623573+0000","last_change":"2024-03-06T07:29:45.260409+0000","last_active":"2024-03-06T07:29:47.623573+0000","last_peered":"2024-03-06T07:29:47.623573+0000","last_clean":"2024-03-06T07:29:47.623573+0000","last_became_active":"2024-03-06T07:29:45.260091+0000","last_became_peered":"2024-03-06T07:29:45.260091+0000","last_unstale":"2024-03-06T07:29:47.623573+0000","last_undegraded":"2024-03-06T07:29:47.623573+0000","last_fullsized":"2024-03-06T07:29:47.623573+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:11:00.906651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1d","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623490+0000","last_change":"2024-03-06T07:29:46.292383+0000","last_active":"2024-03-06T07:29:47.623490+0000","last_peered":"2024-03-06T07:29:47.623490+0000","last_clean":"2024-03-06T07:29:47.623490+0000","last_became_active":"2024-03-06T07:29:46.292025+0000","last_became_peered":"2024-03-06T07:29:46.292025+0000","last_unstale":"2024-03-06T07:29:47.623490+0000","last_undegraded":"2024-03-06T07:29:47.623490+0000","last_fullsized":"2024-03-06T07:29:47.623490+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:35:25.805109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1c","version":"57'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522204+0000","last_change":"2024-03-06T07:29:45.267525+0000","last_active":"2024-03-06T07:29:47.522204+0000","last_peered":"2024-03-06T07:29:47.522204+0000","last_clean":"2024-03-06T07:29:47.522204+0000","last_became_active":"2024-03-06T07:29:45.266817+0000","last_became_peered":"2024-03-06T07:29:45.266817+0000","last_unstale":"2024-03-06T07:29:47.522204+0000","last_undegraded":"2024-03-06T07:29:47.522204+0000","last_fullsized":"2024-03-06T07:29:47.522204+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:28:20.090621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1a","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624169+0000","last_change":"2024-03-06T07:29:46.270049+0000","last_active":"2024-03-06T07:29:47.624169+0000","last_peered":"2024-03-06T07:29:47.624169+0000","last_clean":"2024-03-06T07:29:47.624169+0000","last_became_active":"2024-03-06T07:29:46.269772+0000","last_became_peered":"2024-03-06T07:29:46.269772+0000","last_unstale":"2024-03-06T07:29:47.624169+0000","last_undegraded":"2024-03-06T07:29:47.624169+0000","last_fullsized":"2024-03-06T07:29:47.624169+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:35:31.922757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1b","version":"57'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273928+0000","last_change":"2024-03-06T07:29:45.261989+0000","last_active":"2024-03-06T07:29:47.273928+0000","last_peered":"2024-03-06T07:29:47.273928+0000","last_clean":"2024-03-06T07:29:47.273928+0000","last_became_active":"2024-03-06T07:29:45.261622+0000","last_became_peered":"2024-03-06T07:29:45.261622+0000","last_unstale":"2024-03-06T07:29:47.273928+0000","last_undegraded":"2024-03-06T07:29:47.273928+0000","last_fullsized":"2024-03-06T07:29:47.273928+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:05:25.050469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1b","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624229+0000","last_change":"2024-03-06T07:29:46.275204+0000","last_active":"2024-03-06T07:29:47.624229+0000","last_peered":"2024-03-06T07:29:47.624229+0000","last_clean":"2024-03-06T07:29:47.624229+0000","last_became_active":"2024-03-06T07:29:46.273274+0000","last_became_peered":"2024-03-06T07:29:46.273274+0000","last_unstale":"2024-03-06T07:29:47.624229+0000","last_undegraded":"2024-03-06T07:29:47.624229+0000","last_fullsized":"2024-03-06T07:29:47.624229+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:58:23.709676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1a","version":"57'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623160+0000","last_change":"2024-03-06T07:29:45.264719+0000","last_active":"2024-03-06T07:29:47.623160+0000","last_peered":"2024-03-06T07:29:47.623160+0000","last_clean":"2024-03-06T07:29:47.623160+0000","last_became_active":"2024-03-06T07:29:45.264440+0000","last_became_peered":"2024-03-06T07:29:45.264440+0000","last_unstale":"2024-03-06T07:29:47.623160+0000","last_undegraded":"2024-03-06T07:29:47.623160+0000","last_fullsized":"2024-03-06T07:29:47.623160+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:23:09.710855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.18","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626759+0000","last_change":"2024-03-06T07:29:46.292602+0000","last_active":"2024-03-06T07:29:47.626759+0000","last_peered":"2024-03-06T07:29:47.626759+0000","last_clean":"2024-03-06T07:29:47.626759+0000","last_became_active":"2024-03-06T07:29:46.287862+0000","last_became_peered":"2024-03-06T07:29:46.287862+0000","last_unstale":"2024-03-06T07:29:47.626759+0000","last_undegraded":"2024-03-06T07:29:47.626759+0000","last_fullsized":"2024-03-06T07:29:47.626759+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:16:39.766854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.521316+0000","last_change":"2024-03-06T07:29:45.264075+0000","last_active":"2024-03-06T07:29:47.521316+0000","last_peered":"2024-03-06T07:29:47.521316+0000","last_clean":"2024-03-06T07:29:47.521316+0000","last_became_active":"2024-03-06T07:29:45.263199+0000","last_became_peered":"2024-03-06T07:29:45.263199+0000","last_unstale":"2024-03-06T07:29:47.521316+0000","last_undegraded":"2024-03-06T07:29:47.521316+0000","last_fullsized":"2024-03-06T07:29:47.521316+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:22:48.479697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624194+0000","last_change":"2024-03-06T07:29:46.286155+0000","last_active":"2024-03-06T07:29:47.624194+0000","last_peered":"2024-03-06T07:29:47.624194+0000","last_clean":"2024-03-06T07:29:47.624194+0000","last_became_active":"2024-03-06T07:29:46.285424+0000","last_became_peered":"2024-03-06T07:29:46.285424+0000","last_unstale":"2024-03-06T07:29:47.624194+0000","last_undegraded":"2024-03-06T07:29:47.624194+0000","last_fullsized":"2024-03-06T07:29:47.624194+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:31:42.028949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.18","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273675+0000","last_change":"2024-03-06T07:29:45.256026+0000","last_active":"2024-03-06T07:29:47.273675+0000","last_peered":"2024-03-06T07:29:47.273675+0000","last_clean":"2024-03-06T07:29:47.273675+0000","last_became_active":"2024-03-06T07:29:45.255338+0000","last_became_peered":"2024-03-06T07:29:45.255338+0000","last_unstale":"2024-03-06T07:29:47.273675+0000","last_undegraded":"2024-03-06T07:29:47.273675+0000","last_fullsized":"2024-03-06T07:29:47.273675+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:40:19.530914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.16","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.275024+0000","last_change":"2024-03-06T07:29:46.279517+0000","last_active":"2024-03-06T07:29:47.275024+0000","last_peered":"2024-03-06T07:29:47.275024+0000","last_clean":"2024-03-06T07:29:47.275024+0000","last_became_active":"2024-03-06T07:29:46.278727+0000","last_became_peered":"2024-03-06T07:29:46.278727+0000","last_unstale":"2024-03-06T07:29:47.275024+0000","last_undegraded":"2024-03-06T07:29:47.275024+0000","last_fullsized":"2024-03-06T07:29:47.275024+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:04:11.205672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.17","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627319+0000","last_change":"2024-03-06T07:29:45.255793+0000","last_active":"2024-03-06T07:29:47.627319+0000","last_peered":"2024-03-06T07:29:47.627319+0000","last_clean":"2024-03-06T07:29:47.627319+0000","last_became_active":"2024-03-06T07:29:45.255482+0000","last_became_peered":"2024-03-06T07:29:45.255482+0000","last_unstale":"2024-03-06T07:29:47.627319+0000","last_undegraded":"2024-03-06T07:29:47.627319+0000","last_fullsized":"2024-03-06T07:29:47.627319+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:30.743731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.17","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627532+0000","last_change":"2024-03-06T07:29:46.291120+0000","last_active":"2024-03-06T07:29:47.627532+0000","last_peered":"2024-03-06T07:29:47.627532+0000","last_clean":"2024-03-06T07:29:47.627532+0000","last_became_active":"2024-03-06T07:29:46.290828+0000","last_became_peered":"2024-03-06T07:29:46.290828+0000","last_unstale":"2024-03-06T07:29:47.627532+0000","last_undegraded":"2024-03-06T07:29:47.627532+0000","last_fullsized":"2024-03-06T07:29:47.627532+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:28:31.506024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.16","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625279+0000","last_change":"2024-03-06T07:29:45.263080+0000","last_active":"2024-03-06T07:29:47.625279+0000","last_peered":"2024-03-06T07:29:47.625279+0000","last_clean":"2024-03-06T07:29:47.625279+0000","last_became_active":"2024-03-06T07:29:45.262391+0000","last_became_peered":"2024-03-06T07:29:45.262391+0000","last_unstale":"2024-03-06T07:29:47.625279+0000","last_undegraded":"2024-03-06T07:29:47.625279+0000","last_fullsized":"2024-03-06T07:29:47.625279+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:25:10.665498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.272944+0000","last_change":"2024-03-06T07:29:46.294203+0000","last_active":"2024-03-06T07:29:47.272944+0000","last_peered":"2024-03-06T07:29:47.272944+0000","last_clean":"2024-03-06T07:29:47.272944+0000","last_became_active":"2024-03-06T07:29:46.293452+0000","last_became_peered":"2024-03-06T07:29:46.293452+0000","last_unstale":"2024-03-06T07:29:47.272944+0000","last_undegraded":"2024-03-06T07:29:47.272944+0000","last_fullsized":"2024-03-06T07:29:47.272944+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:26:04.189444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.15","version":"57'3","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627672+0000","last_change":"2024-03-06T07:29:45.262885+0000","last_active":"2024-03-06T07:29:47.627672+0000","last_peered":"2024-03-06T07:29:47.627672+0000","last_clean":"2024-03-06T07:29:47.627672+0000","last_became_active":"2024-03-06T07:29:45.262540+0000","last_became_peered":"2024-03-06T07:29:45.262540+0000","last_unstale":"2024-03-06T07:29:47.627672+0000","last_undegraded":"2024-03-06T07:29:47.627672+0000","last_fullsized":"2024-03-06T07:29:47.627672+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:01:22.323221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.15","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627543+0000","last_change":"2024-03-06T07:29:46.285784+0000","last_active":"2024-03-06T07:29:47.627543+0000","last_peered":"2024-03-06T07:29:47.627543+0000","last_clean":"2024-03-06T07:29:47.627543+0000","last_became_active":"2024-03-06T07:29:46.285481+0000","last_became_peered":"2024-03-06T07:29:46.285481+0000","last_unstale":"2024-03-06T07:29:47.627543+0000","last_undegraded":"2024-03-06T07:29:47.627543+0000","last_fullsized":"2024-03-06T07:29:47.627543+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:05:56.818895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.14","version":"57'6","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.274379+0000","last_change":"2024-03-06T07:29:45.261696+0000","last_active":"2024-03-06T07:29:47.274379+0000","last_peered":"2024-03-06T07:29:47.274379+0000","last_clean":"2024-03-06T07:29:47.274379+0000","last_became_active":"2024-03-06T07:29:45.261393+0000","last_became_peered":"2024-03-06T07:29:45.261393+0000","last_unstale":"2024-03-06T07:29:47.274379+0000","last_undegraded":"2024-03-06T07:29:47.274379+0000","last_fullsized":"2024-03-06T07:29:47.274379+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:14:30.743731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.12","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.521863+0000","last_change":"2024-03-06T07:29:46.289106+0000","last_active":"2024-03-06T07:29:47.521863+0000","last_peered":"2024-03-06T07:29:47.521863+0000","last_clean":"2024-03-06T07:29:47.521863+0000","last_became_active":"2024-03-06T07:29:46.288816+0000","last_became_peered":"2024-03-06T07:29:46.288816+0000","last_unstale":"2024-03-06T07:29:47.521863+0000","last_undegraded":"2024-03-06T07:29:47.521863+0000","last_fullsized":"2024-03-06T07:29:47.521863+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:35:58.823111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.13","version":"57'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624282+0000","last_change":"2024-03-06T07:29:45.262105+0000","last_active":"2024-03-06T07:29:47.624282+0000","last_peered":"2024-03-06T07:29:47.624282+0000","last_clean":"2024-03-06T07:29:47.624282+0000","last_became_active":"2024-03-06T07:29:45.261432+0000","last_became_peered":"2024-03-06T07:29:45.261432+0000","last_unstale":"2024-03-06T07:29:47.624282+0000","last_undegraded":"2024-03-06T07:29:47.624282+0000","last_fullsized":"2024-03-06T07:29:47.624282+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:16:22.997714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627231+0000","last_change":"2024-03-06T07:29:46.284815+0000","last_active":"2024-03-06T07:29:47.627231+0000","last_peered":"2024-03-06T07:29:47.627231+0000","last_clean":"2024-03-06T07:29:47.627231+0000","last_became_active":"2024-03-06T07:29:46.284459+0000","last_became_peered":"2024-03-06T07:29:46.284459+0000","last_unstale":"2024-03-06T07:29:47.627231+0000","last_undegraded":"2024-03-06T07:29:47.627231+0000","last_fullsized":"2024-03-06T07:29:47.627231+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:25:11.681234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.12","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625458+0000","last_change":"2024-03-06T07:29:45.264777+0000","last_active":"2024-03-06T07:29:47.625458+0000","last_peered":"2024-03-06T07:29:47.625458+0000","last_clean":"2024-03-06T07:29:47.625458+0000","last_became_active":"2024-03-06T07:29:45.264447+0000","last_became_peered":"2024-03-06T07:29:45.264447+0000","last_unstale":"2024-03-06T07:29:47.625458+0000","last_undegraded":"2024-03-06T07:29:47.625458+0000","last_fullsized":"2024-03-06T07:29:47.625458+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:33.728030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.10","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273775+0000","last_change":"2024-03-06T07:29:46.287238+0000","last_active":"2024-03-06T07:29:47.273775+0000","last_peered":"2024-03-06T07:29:47.273775+0000","last_clean":"2024-03-06T07:29:47.273775+0000","last_became_active":"2024-03-06T07:29:46.286960+0000","last_became_peered":"2024-03-06T07:29:46.286960+0000","last_unstale":"2024-03-06T07:29:47.273775+0000","last_undegraded":"2024-03-06T07:29:47.273775+0000","last_fullsized":"2024-03-06T07:29:47.273775+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:54.409839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.11","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.521383+0000","last_change":"2024-03-06T07:29:45.266165+0000","last_active":"2024-03-06T07:29:47.521383+0000","last_peered":"2024-03-06T07:29:47.521383+0000","last_clean":"2024-03-06T07:29:47.521383+0000","last_became_active":"2024-03-06T07:29:45.265903+0000","last_became_peered":"2024-03-06T07:29:45.265903+0000","last_unstale":"2024-03-06T07:29:47.521383+0000","last_undegraded":"2024-03-06T07:29:47.521383+0000","last_fullsized":"2024-03-06T07:29:47.521383+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:16:35.505263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627451+0000","last_change":"2024-03-06T07:29:46.289892+0000","last_active":"2024-03-06T07:29:47.627451+0000","last_peered":"2024-03-06T07:29:47.627451+0000","last_clean":"2024-03-06T07:29:47.627451+0000","last_became_active":"2024-03-06T07:29:46.288925+0000","last_became_peered":"2024-03-06T07:29:46.288925+0000","last_unstale":"2024-03-06T07:29:47.627451+0000","last_undegraded":"2024-03-06T07:29:47.627451+0000","last_fullsized":"2024-03-06T07:29:47.627451+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:16:24.013449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.10","version":"57'8","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625658+0000","last_change":"2024-03-06T07:29:45.262432+0000","last_active":"2024-03-06T07:29:47.625658+0000","last_peered":"2024-03-06T07:29:47.625658+0000","last_clean":"2024-03-06T07:29:47.625658+0000","last_became_active":"2024-03-06T07:29:45.261908+0000","last_became_peered":"2024-03-06T07:29:45.261908+0000","last_unstale":"2024-03-06T07:29:47.625658+0000","last_undegraded":"2024-03-06T07:29:47.625658+0000","last_fullsized":"2024-03-06T07:29:47.625658+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:40:19.530914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627103+0000","last_change":"2024-03-06T07:29:46.292138+0000","last_active":"2024-03-06T07:29:47.627103+0000","last_peered":"2024-03-06T07:29:47.627103+0000","last_clean":"2024-03-06T07:29:47.627103+0000","last_became_active":"2024-03-06T07:29:46.291693+0000","last_became_peered":"2024-03-06T07:29:46.291693+0000","last_unstale":"2024-03-06T07:29:47.627103+0000","last_undegraded":"2024-03-06T07:29:47.627103+0000","last_fullsized":"2024-03-06T07:29:47.627103+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:40:41.043201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"57'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624979+0000","last_change":"2024-03-06T07:29:45.262039+0000","last_active":"2024-03-06T07:29:47.624979+0000","last_peered":"2024-03-06T07:29:47.624979+0000","last_clean":"2024-03-06T07:29:47.624979+0000","last_became_active":"2024-03-06T07:29:45.261583+0000","last_became_peered":"2024-03-06T07:29:45.261583+0000","last_unstale":"2024-03-06T07:29:47.624979+0000","last_undegraded":"2024-03-06T07:29:47.624979+0000","last_fullsized":"2024-03-06T07:29:47.624979+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:24:49.461238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522198+0000","last_change":"2024-03-06T07:29:46.291073+0000","last_active":"2024-03-06T07:29:47.522198+0000","last_peered":"2024-03-06T07:29:47.522198+0000","last_clean":"2024-03-06T07:29:47.522198+0000","last_became_active":"2024-03-06T07:29:46.290625+0000","last_became_peered":"2024-03-06T07:29:46.290625+0000","last_unstale":"2024-03-06T07:29:47.522198+0000","last_undegraded":"2024-03-06T07:29:47.522198+0000","last_fullsized":"2024-03-06T07:29:47.522198+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:26:36.339876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"57'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623796+0000","last_change":"2024-03-06T07:29:45.261449+0000","last_active":"2024-03-06T07:29:47.623796+0000","last_peered":"2024-03-06T07:29:47.623796+0000","last_clean":"2024-03-06T07:29:47.623796+0000","last_became_active":"2024-03-06T07:29:45.261138+0000","last_became_peered":"2024-03-06T07:29:45.261138+0000","last_unstale":"2024-03-06T07:29:47.623796+0000","last_undegraded":"2024-03-06T07:29:47.623796+0000","last_fullsized":"2024-03-06T07:29:47.623796+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:10:20.441560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624752+0000","last_change":"2024-03-06T07:29:46.287694+0000","last_active":"2024-03-06T07:29:47.624752+0000","last_peered":"2024-03-06T07:29:47.624752+0000","last_clean":"2024-03-06T07:29:47.624752+0000","last_became_active":"2024-03-06T07:29:46.287370+0000","last_became_peered":"2024-03-06T07:29:46.287370+0000","last_unstale":"2024-03-06T07:29:47.624752+0000","last_undegraded":"2024-03-06T07:29:47.624752+0000","last_fullsized":"2024-03-06T07:29:47.624752+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:17:03.780486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"57'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522125+0000","last_change":"2024-03-06T07:29:45.267283+0000","last_active":"2024-03-06T07:29:47.522125+0000","last_peered":"2024-03-06T07:29:47.522125+0000","last_clean":"2024-03-06T07:29:47.522125+0000","last_became_active":"2024-03-06T07:29:45.266942+0000","last_became_peered":"2024-03-06T07:29:45.266942+0000","last_unstale":"2024-03-06T07:29:47.522125+0000","last_undegraded":"2024-03-06T07:29:47.522125+0000","last_fullsized":"2024-03-06T07:29:47.522125+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:36.724025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.627675+0000","last_change":"2024-03-06T07:29:46.286843+0000","last_active":"2024-03-06T07:29:47.627675+0000","last_peered":"2024-03-06T07:29:47.627675+0000","last_clean":"2024-03-06T07:29:47.627675+0000","last_became_active":"2024-03-06T07:29:46.286328+0000","last_became_peered":"2024-03-06T07:29:46.286328+0000","last_unstale":"2024-03-06T07:29:47.627675+0000","last_undegraded":"2024-03-06T07:29:47.627675+0000","last_fullsized":"2024-03-06T07:29:47.627675+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:22:55.802032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623249+0000","last_change":"2024-03-06T07:29:45.258062+0000","last_active":"2024-03-06T07:29:47.623249+0000","last_peered":"2024-03-06T07:29:47.623249+0000","last_clean":"2024-03-06T07:29:47.623249+0000","last_became_active":"2024-03-06T07:29:45.257051+0000","last_became_peered":"2024-03-06T07:29:45.257051+0000","last_unstale":"2024-03-06T07:29:47.623249+0000","last_undegraded":"2024-03-06T07:29:47.623249+0000","last_fullsized":"2024-03-06T07:29:47.623249+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:53.394103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.521541+0000","last_change":"2024-03-06T07:29:46.291251+0000","last_active":"2024-03-06T07:29:47.521541+0000","last_peered":"2024-03-06T07:29:47.521541+0000","last_clean":"2024-03-06T07:29:47.521541+0000","last_became_active":"2024-03-06T07:29:46.290977+0000","last_became_peered":"2024-03-06T07:29:46.290977+0000","last_unstale":"2024-03-06T07:29:47.521541+0000","last_undegraded":"2024-03-06T07:29:47.521541+0000","last_fullsized":"2024-03-06T07:29:47.521541+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:53:10.817334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"57'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624562+0000","last_change":"2024-03-06T07:29:45.260505+0000","last_active":"2024-03-06T07:29:47.624562+0000","last_peered":"2024-03-06T07:29:47.624562+0000","last_clean":"2024-03-06T07:29:47.624562+0000","last_became_active":"2024-03-06T07:29:45.258349+0000","last_became_peered":"2024-03-06T07:29:45.258349+0000","last_unstale":"2024-03-06T07:29:47.624562+0000","last_undegraded":"2024-03-06T07:29:47.624562+0000","last_fullsized":"2024-03-06T07:29:47.624562+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:08:40.704573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623294+0000","last_change":"2024-03-06T07:29:46.291845+0000","last_active":"2024-03-06T07:29:47.623294+0000","last_peered":"2024-03-06T07:29:47.623294+0000","last_clean":"2024-03-06T07:29:47.623294+0000","last_became_active":"2024-03-06T07:29:46.287834+0000","last_became_peered":"2024-03-06T07:29:46.287834+0000","last_unstale":"2024-03-06T07:29:47.623294+0000","last_undegraded":"2024-03-06T07:29:47.623294+0000","last_fullsized":"2024-03-06T07:29:47.623294+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:24:50.476974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.272675+0000","last_change":"2024-03-06T07:29:45.255608+0000","last_active":"2024-03-06T07:29:47.272675+0000","last_peered":"2024-03-06T07:29:47.272675+0000","last_clean":"2024-03-06T07:29:47.272675+0000","last_became_active":"2024-03-06T07:29:45.255301+0000","last_became_peered":"2024-03-06T07:29:45.255301+0000","last_unstale":"2024-03-06T07:29:47.272675+0000","last_undegraded":"2024-03-06T07:29:47.272675+0000","last_fullsized":"2024-03-06T07:29:47.272675+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:59:18.532955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623609+0000","last_change":"2024-03-06T07:29:46.291692+0000","last_active":"2024-03-06T07:29:47.623609+0000","last_peered":"2024-03-06T07:29:47.623609+0000","last_clean":"2024-03-06T07:29:47.623609+0000","last_became_active":"2024-03-06T07:29:46.288133+0000","last_became_peered":"2024-03-06T07:29:46.288133+0000","last_unstale":"2024-03-06T07:29:47.623609+0000","last_undegraded":"2024-03-06T07:29:47.623609+0000","last_fullsized":"2024-03-06T07:29:47.623609+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:53:28.261282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.521193+0000","last_change":"2024-03-06T07:29:45.265278+0000","last_active":"2024-03-06T07:29:47.521193+0000","last_peered":"2024-03-06T07:29:47.521193+0000","last_clean":"2024-03-06T07:29:47.521193+0000","last_became_active":"2024-03-06T07:29:45.264798+0000","last_became_peered":"2024-03-06T07:29:45.264798+0000","last_unstale":"2024-03-06T07:29:47.521193+0000","last_undegraded":"2024-03-06T07:29:47.521193+0000","last_fullsized":"2024-03-06T07:29:47.521193+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:20:50.654248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522490+0000","last_change":"2024-03-06T07:29:46.288141+0000","last_active":"2024-03-06T07:29:47.522490+0000","last_peered":"2024-03-06T07:29:47.522490+0000","last_clean":"2024-03-06T07:29:47.522490+0000","last_became_active":"2024-03-06T07:29:46.287867+0000","last_became_peered":"2024-03-06T07:29:46.287867+0000","last_unstale":"2024-03-06T07:29:47.522490+0000","last_undegraded":"2024-03-06T07:29:47.522490+0000","last_fullsized":"2024-03-06T07:29:47.522490+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:28:16.139804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.7","version":"57'2","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273885+0000","last_change":"2024-03-06T07:29:45.257504+0000","last_active":"2024-03-06T07:29:47.273885+0000","last_peered":"2024-03-06T07:29:47.273885+0000","last_clean":"2024-03-06T07:29:47.273885+0000","last_became_active":"2024-03-06T07:29:45.257197+0000","last_became_peered":"2024-03-06T07:29:45.257197+0000","last_unstale":"2024-03-06T07:29:47.273885+0000","last_undegraded":"2024-03-06T07:29:47.273885+0000","last_fullsized":"2024-03-06T07:29:47.273885+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:53:29.377974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522403+0000","last_change":"2024-03-06T07:29:46.294229+0000","last_active":"2024-03-06T07:29:47.522403+0000","last_peered":"2024-03-06T07:29:47.522403+0000","last_clean":"2024-03-06T07:29:47.522403+0000","last_became_active":"2024-03-06T07:29:46.293927+0000","last_became_peered":"2024-03-06T07:29:46.293927+0000","last_unstale":"2024-03-06T07:29:47.522403+0000","last_undegraded":"2024-03-06T07:29:47.522403+0000","last_fullsized":"2024-03-06T07:29:47.522403+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:03:59.081732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.6","version":"58'1","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623943+0000","last_change":"2024-03-06T07:29:45.264239+0000","last_active":"2024-03-06T07:29:47.623943+0000","last_peered":"2024-03-06T07:29:47.623943+0000","last_clean":"2024-03-06T07:29:47.623943+0000","last_became_active":"2024-03-06T07:29:45.263934+0000","last_became_peered":"2024-03-06T07:29:45.263934+0000","last_unstale":"2024-03-06T07:29:47.623943+0000","last_undegraded":"2024-03-06T07:29:47.623943+0000","last_fullsized":"2024-03-06T07:29:47.623943+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:23:57.854105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.273097+0000","last_change":"2024-03-06T07:29:46.293193+0000","last_active":"2024-03-06T07:29:47.273097+0000","last_peered":"2024-03-06T07:29:47.273097+0000","last_clean":"2024-03-06T07:29:47.273097+0000","last_became_active":"2024-03-06T07:29:46.292944+0000","last_became_peered":"2024-03-06T07:29:46.292944+0000","last_unstale":"2024-03-06T07:29:47.273097+0000","last_undegraded":"2024-03-06T07:29:47.273097+0000","last_fullsized":"2024-03-06T07:29:47.273097+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:32:30.119908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625505+0000","last_change":"2024-03-06T07:29:45.261758+0000","last_active":"2024-03-06T07:29:47.625505+0000","last_peered":"2024-03-06T07:29:47.625505+0000","last_clean":"2024-03-06T07:29:47.625505+0000","last_became_active":"2024-03-06T07:29:45.261424+0000","last_became_peered":"2024-03-06T07:29:45.261424+0000","last_unstale":"2024-03-06T07:29:47.625505+0000","last_undegraded":"2024-03-06T07:29:47.625505+0000","last_fullsized":"2024-03-06T07:29:47.625505+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:36:27.908330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.623579+0000","last_change":"2024-03-06T07:29:46.292237+0000","last_active":"2024-03-06T07:29:47.623579+0000","last_peered":"2024-03-06T07:29:47.623579+0000","last_clean":"2024-03-06T07:29:47.623579+0000","last_became_active":"2024-03-06T07:29:46.291189+0000","last_became_peered":"2024-03-06T07:29:46.291189+0000","last_unstale":"2024-03-06T07:29:47.623579+0000","last_undegraded":"2024-03-06T07:29:47.623579+0000","last_fullsized":"2024-03-06T07:29:47.623579+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:30:14.770207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.522246+0000","last_change":"2024-03-06T07:29:45.268080+0000","last_active":"2024-03-06T07:29:47.522246+0000","last_peered":"2024-03-06T07:29:47.522246+0000","last_clean":"2024-03-06T07:29:47.522246+0000","last_became_active":"2024-03-06T07:29:45.267134+0000","last_became_peered":"2024-03-06T07:29:45.267134+0000","last_unstale":"2024-03-06T07:29:47.522246+0000","last_undegraded":"2024-03-06T07:29:47.522246+0000","last_fullsized":"2024-03-06T07:29:47.522246+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:45:27.305490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624386+0000","last_change":"2024-03-06T07:29:46.277972+0000","last_active":"2024-03-06T07:29:47.624386+0000","last_peered":"2024-03-06T07:29:47.624386+0000","last_clean":"2024-03-06T07:29:47.624386+0000","last_became_active":"2024-03-06T07:29:46.277687+0000","last_became_peered":"2024-03-06T07:29:46.277687+0000","last_unstale":"2024-03-06T07:29:47.624386+0000","last_undegraded":"2024-03-06T07:29:47.624386+0000","last_fullsized":"2024-03-06T07:29:47.624386+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:50.497211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.272251+0000","last_change":"2024-03-06T07:29:45.266944+0000","last_active":"2024-03-06T07:29:47.272251+0000","last_peered":"2024-03-06T07:29:47.272251+0000","last_clean":"2024-03-06T07:29:47.272251+0000","last_became_active":"2024-03-06T07:29:45.266586+0000","last_became_peered":"2024-03-06T07:29:45.266586+0000","last_unstale":"2024-03-06T07:29:47.272251+0000","last_undegraded":"2024-03-06T07:29:47.272251+0000","last_fullsized":"2024-03-06T07:29:47.272251+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:56:46.735926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.626962+0000","last_change":"2024-03-06T07:29:46.287062+0000","last_active":"2024-03-06T07:29:47.626962+0000","last_peered":"2024-03-06T07:29:47.626962+0000","last_clean":"2024-03-06T07:29:47.626962+0000","last_became_active":"2024-03-06T07:29:46.286831+0000","last_became_peered":"2024-03-06T07:29:46.286831+0000","last_unstale":"2024-03-06T07:29:47.626962+0000","last_undegraded":"2024-03-06T07:29:47.626962+0000","last_fullsized":"2024-03-06T07:29:47.626962+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:29:06.326096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.274772+0000","last_change":"2024-03-06T07:29:45.259640+0000","last_active":"2024-03-06T07:29:47.274772+0000","last_peered":"2024-03-06T07:29:47.274772+0000","last_clean":"2024-03-06T07:29:47.274772+0000","last_became_active":"2024-03-06T07:29:45.259100+0000","last_became_peered":"2024-03-06T07:29:45.259100+0000","last_unstale":"2024-03-06T07:29:47.274772+0000","last_undegraded":"2024-03-06T07:29:47.274772+0000","last_fullsized":"2024-03-06T07:29:47.274772+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:01:22.323221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.272225+0000","last_change":"2024-03-06T07:29:46.294305+0000","last_active":"2024-03-06T07:29:47.272225+0000","last_peered":"2024-03-06T07:29:47.272225+0000","last_clean":"2024-03-06T07:29:47.272225+0000","last_became_active":"2024-03-06T07:29:46.293733+0000","last_became_peered":"2024-03-06T07:29:46.293733+0000","last_unstale":"2024-03-06T07:29:47.272225+0000","last_undegraded":"2024-03-06T07:29:47.272225+0000","last_fullsized":"2024-03-06T07:29:47.272225+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:36:17.007076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.272397+0000","last_change":"2024-03-06T07:29:45.257827+0000","last_active":"2024-03-06T07:29:47.272397+0000","last_peered":"2024-03-06T07:29:47.272397+0000","last_clean":"2024-03-06T07:29:47.272397+0000","last_became_active":"2024-03-06T07:29:45.257519+0000","last_became_peered":"2024-03-06T07:29:45.257519+0000","last_unstale":"2024-03-06T07:29:47.272397+0000","last_undegraded":"2024-03-06T07:29:47.272397+0000","last_fullsized":"2024-03-06T07:29:47.272397+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:36:06.503749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624392+0000","last_change":"2024-03-06T07:29:46.286591+0000","last_active":"2024-03-06T07:29:47.624392+0000","last_peered":"2024-03-06T07:29:47.624392+0000","last_clean":"2024-03-06T07:29:47.624392+0000","last_became_active":"2024-03-06T07:29:46.286226+0000","last_became_peered":"2024-03-06T07:29:46.286226+0000","last_unstale":"2024-03-06T07:29:47.624392+0000","last_undegraded":"2024-03-06T07:29:47.624392+0000","last_fullsized":"2024-03-06T07:29:47.624392+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:12:30.317099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624153+0000","last_change":"2024-03-06T07:29:45.262015+0000","last_active":"2024-03-06T07:29:47.624153+0000","last_peered":"2024-03-06T07:29:47.624153+0000","last_clean":"2024-03-06T07:29:47.624153+0000","last_became_active":"2024-03-06T07:29:45.260676+0000","last_became_peered":"2024-03-06T07:29:45.260676+0000","last_unstale":"2024-03-06T07:29:47.624153+0000","last_undegraded":"2024-03-06T07:29:47.624153+0000","last_fullsized":"2024-03-06T07:29:47.624153+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:22:20.728815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":12,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.624056+0000","last_change":"2024-03-06T07:29:46.270242+0000","last_active":"2024-03-06T07:29:47.624056+0000","last_peered":"2024-03-06T07:29:47.624056+0000","last_clean":"2024-03-06T07:29:47.624056+0000","last_became_active":"2024-03-06T07:29:46.269974+0000","last_became_peered":"2024-03-06T07:29:46.269974+0000","last_unstale":"2024-03-06T07:29:47.624056+0000","last_undegraded":"2024-03-06T07:29:47.624056+0000","last_fullsized":"2024-03-06T07:29:47.624056+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:45.218494+0000","last_clean_scrub_stamp":"2024-03-06T07:29:45.218494+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:06:59.749246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625113+0000","last_change":"2024-03-06T07:29:45.266385+0000","last_active":"2024-03-06T07:29:47.625113+0000","last_peered":"2024-03-06T07:29:47.625113+0000","last_clean":"2024-03-06T07:29:47.625113+0000","last_became_active":"2024-03-06T07:29:45.266005+0000","last_became_peered":"2024-03-06T07:29:45.266005+0000","last_unstale":"2024-03-06T07:29:47.625113+0000","last_undegraded":"2024-03-06T07:29:47.625113+0000","last_fullsized":"2024-03-06T07:29:47.625113+0000","mapping_epoch":55,"log_start":"0'0","ondisk_log_start":"0'0","created":55,"last_epoch_clean":56,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:29:44.202758+0000","last_clean_scrub_stamp":"2024-03-06T07:29:44.202758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:07:02.407994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":233,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-03-06T07:29:47.625861+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:29:47.625861+0000","last_peered":"2024-03-06T07:29:47.625861+0000","last_clean":"2024-03-06T07:29:47.625861+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:29:47.625861+0000","last_undegraded":"2024-03-06T07:29:47.625861+0000","last_fullsized":"2024-03-06T07:29:47.625861+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":8,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542182,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32448,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339392,"statfs":{"total":96636764160,"available":96603537408,"internally_reserved":0,"allocated":1294336,"data_stored":876783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542181,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31856,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339984,"statfs":{"total":96636764160,"available":96604143616,"internally_reserved":0,"allocated":692224,"data_stored":286255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542182,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31864,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339976,"statfs":{"total":96636764160,"available":96604135424,"internally_reserved":0,"allocated":700416,"data_stored":286907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542181,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31872,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339968,"statfs":{"total":96636764160,"available":96604127232,"internally_reserved":0,"allocated":704512,"data_stored":287867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542181,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31868,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339972,"statfs":{"total":96636764160,"available":96604131328,"internally_reserved":0,"allocated":700416,"data_stored":286311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542182,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31868,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339972,"statfs":{"total":96636764160,"available":96604131328,"internally_reserved":0,"allocated":700416,"data_stored":286311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542183,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31864,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339976,"statfs":{"total":96636764160,"available":96604135424,"internally_reserved":0,"allocated":696320,"data_stored":286345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542180,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32456,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339384,"statfs":{"total":96636764160,"available":96603529216,"internally_reserved":0,"allocated":1302528,"data_stored":878817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:29:51.215 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:29:51.215 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:29:51.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.469+0000 7f0aa7a00640 1 Processor -- start 2024-03-06T07:29:51.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.470+0000 7f0aa7a00640 1 -- start start 2024-03-06T07:29:51.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.471+0000 7f0aa7a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00b3530 0x7f0aa015b780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.471+0000 7f0aa7a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0aa00b3910 0x7f0aa00acf60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.471+0000 7f0aa7a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0aa00ad4a0 0x7f0aa00ad930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.471+0000 7f0aa7a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0aa015bcc0 con 0x7f0aa00b3910 2024-03-06T07:29:51.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.471+0000 7f0aa7a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0aa00ce690 con 0x7f0aa00ad4a0 2024-03-06T07:29:51.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.471+0000 7f0aa7a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0aa00ce970 con 0x7f0aa00b3530 2024-03-06T07:29:51.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0aa00b3910 0x7f0aa00acf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0aa00b3910 0x7f0aa00acf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57448/0 (socket says 172.21.15.86:57448) 2024-03-06T07:29:51.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 -- 172.21.15.86:0/4011100730 learned_addr learned my addr 172.21.15.86:0/4011100730 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:51.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0aa5000640 1 --2- 172.21.15.86:0/4011100730 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00b3530 0x7f0aa015b780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0aa5a00640 1 --2- 172.21.15.86:0/4011100730 >> v2:172.21.15.194:3300/0 conn(0x7f0aa00ad4a0 0x7f0aa00ad930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 -- 172.21.15.86:0/4011100730 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00b3530 msgr2=0x7f0aa015b780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:51.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 --2- 172.21.15.86:0/4011100730 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00b3530 0x7f0aa015b780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 -- 172.21.15.86:0/4011100730 >> v2:172.21.15.194:3300/0 conn(0x7f0aa00ad4a0 msgr2=0x7f0aa00ad930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:51.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 --2- 172.21.15.86:0/4011100730 >> v2:172.21.15.194:3300/0 conn(0x7f0aa00ad4a0 0x7f0aa00ad930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.472+0000 7f0a9fe00640 1 -- 172.21.15.86:0/4011100730 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0aa00cec50 con 0x7f0aa00b3910 2024-03-06T07:29:51.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.473+0000 7f0a9fe00640 1 --2- 172.21.15.86:0/4011100730 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3910 0x7f0aa00acf60 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f0a8c00bb10 tx=0x7f0a8c02e0c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6f54914074c7b896 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:51.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.474+0000 7f0a9f400640 1 -- 172.21.15.86:0/4011100730 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a8c03b070 con 0x7f0aa00b3910 2024-03-06T07:29:51.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.474+0000 7f0a9f400640 1 -- 172.21.15.86:0/4011100730 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a8c036440 con 0x7f0aa00b3910 2024-03-06T07:29:51.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.474+0000 7f0a9f400640 1 -- 172.21.15.86:0/4011100730 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a8c036810 con 0x7f0aa00b3910 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 -- 172.21.15.86:0/4011100730 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3910 msgr2=0x7f0aa00acf60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/4011100730 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3910 0x7f0aa00acf60 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f0a8c00bb10 tx=0x7f0a8c02e0c0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 -- 172.21.15.86:0/4011100730 shutdown_connections 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/4011100730 >> v2:172.21.15.194:3300/0 conn(0x7f0aa00ad4a0 0x7f0aa00ad930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/4011100730 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3910 0x7f0aa00acf60 unknown :-1 s=CLOSED pgs=662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/4011100730 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00b3530 0x7f0aa015b780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 -- 172.21.15.86:0/4011100730 >> 172.21.15.86:0/4011100730 conn(0x7f0aa00b1940 msgr2=0x7f0aa00b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 -- 172.21.15.86:0/4011100730 shutdown_connections 2024-03-06T07:29:51.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.475+0000 7f0aa7a00640 1 -- 172.21.15.86:0/4011100730 wait complete. 2024-03-06T07:29:51.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.477+0000 7f0aa7a00640 1 Processor -- start 2024-03-06T07:29:51.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.477+0000 7f0aa7a00640 1 -- start start 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa7a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00ad4a0 0x7f0aa009b330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa7a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3530 0x7f0aa01f5350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa7a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0aa00b3910 0x7f0aa009d630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa7a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0aa00b3d90 con 0x7f0aa00b3530 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa7a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0aa00cf210 con 0x7f0aa00b3910 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa7a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0aa015aa30 con 0x7f0aa00ad4a0 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3530 0x7f0aa01f5350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3530 0x7f0aa01f5350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57450/0 (socket says 172.21.15.86:57450) 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 -- 172.21.15.86:0/2522573043 learned_addr learned my addr 172.21.15.86:0/2522573043 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:51.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa5a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0aa00b3910 0x7f0aa009d630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 -- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00ad4a0 msgr2=0x7f0aa009b330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00ad4a0 0x7f0aa009b330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 -- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0aa00b3910 msgr2=0x7f0aa009d630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0aa00b3910 0x7f0aa009d630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0a9fe00640 1 -- 172.21.15.86:0/2522573043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0aa01f7bf0 con 0x7f0aa00b3530 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.479+0000 7f0aa5a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0aa00b3910 0x7f0aa009d630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.480+0000 7f0a9fe00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3530 0x7f0aa01f5350 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f0a8c000c00 tx=0x7f0a8c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.480+0000 7f0a9cc00640 1 -- 172.21.15.86:0/2522573043 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a8c046070 con 0x7f0aa00b3530 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.480+0000 7f0a9cc00640 1 -- 172.21.15.86:0/2522573043 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a8c03d050 con 0x7f0aa00b3530 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.480+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0aa01f7ed0 con 0x7f0aa00b3530 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.480+0000 7f0a9cc00640 1 -- 172.21.15.86:0/2522573043 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a8c03b030 con 0x7f0aa00b3530 2024-03-06T07:29:51.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.480+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0aa01f8490 con 0x7f0aa00b3530 2024-03-06T07:29:51.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.482+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0aa004f330 con 0x7f0aa00b3530 2024-03-06T07:29:51.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.482+0000 7f0a9cc00640 1 -- 172.21.15.86:0/2522573043 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0a8c049070 con 0x7f0aa00b3530 2024-03-06T07:29:51.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.484+0000 7f0a9cc00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a8006b500 0x7f0a8006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.484+0000 7f0a9cc00640 1 -- 172.21.15.86:0/2522573043 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0a8c0abca0 con 0x7f0aa00b3530 2024-03-06T07:29:51.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.484+0000 7f0aa5000640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a8006b500 0x7f0a8006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.485+0000 7f0aa5000640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a8006b500 0x7f0a8006d9c0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f0aa00c98a0 tx=0x7f0a94008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:51.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.488+0000 7f0a9cc00640 1 -- 172.21.15.86:0/2522573043 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0a8c07ab20 con 0x7f0aa00b3530 2024-03-06T07:29:51.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.755+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f0aa01f5890 con 0x7f0aa00b3530 2024-03-06T07:29:51.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.760+0000 7f0a9cc00640 1 -- 172.21.15.86:0/2522573043 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v15) v1 ==== 190+0+147 (secure 0 0 0) 0x7f0a8c07a420 con 0x7f0aa00b3530 2024-03-06T07:29:51.762 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:29:51.762 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:29:51.762 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:29:51.762 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:29:51.762 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:29:51.762 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:29:51.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a8006b500 msgr2=0x7f0a8006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a8006b500 0x7f0a8006d9c0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f0aa00c98a0 tx=0x7f0a94008040 comp rx=0 tx=0).stop 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3530 msgr2=0x7f0aa01f5350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3530 0x7f0aa01f5350 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f0a8c000c00 tx=0x7f0a8c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 shutdown_connections 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a8006b500 0x7f0a8006d9c0 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0aa00b3910 0x7f0aa009d630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0aa00b3530 0x7f0aa01f5350 unknown :-1 s=CLOSED pgs=663 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 --2- 172.21.15.86:0/2522573043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0aa00ad4a0 0x7f0aa009b330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 >> 172.21.15.86:0/2522573043 conn(0x7f0aa00b1940 msgr2=0x7f0aa00b0550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 shutdown_connections 2024-03-06T07:29:51.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.762+0000 7f0aa7a00640 1 -- 172.21.15.86:0/2522573043 wait complete. 2024-03-06T07:29:51.782 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:29:51.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.978+0000 7fd7fd200640 1 Processor -- start 2024-03-06T07:29:51.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.978+0000 7fd7fd200640 1 -- start start 2024-03-06T07:29:51.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7fd200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd7f8147310 0x7fd7f8072340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7fd200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd7f806daa0 0x7fd7f806cc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7fd200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd7f80b39c0 con 0x7fd7f806d1c0 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7fd200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7f806df30 con 0x7fd7f806daa0 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7fd200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7f80ad9e0 con 0x7fd7f8147310 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7ed400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd7f806daa0 0x7fd7f806cc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7ed400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd7f806daa0 0x7fd7f806cc80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57452/0 (socket says 172.21.15.86:57452) 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.980+0000 7fd7ed400640 1 -- 172.21.15.86:0/1585982830 learned_addr learned my addr 172.21.15.86:0/1585982830 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.981+0000 7fd7ed400640 1 -- 172.21.15.86:0/1585982830 >> v2:172.21.15.194:3300/0 conn(0x7fd7f8147310 msgr2=0x7fd7f8072340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.981+0000 7fd7ed400640 1 --2- 172.21.15.86:0/1585982830 >> v2:172.21.15.194:3300/0 conn(0x7fd7f8147310 0x7fd7f8072340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.981+0000 7fd7ed400640 1 -- 172.21.15.86:0/1585982830 >> v1:172.21.15.194:6789/0 conn(0x7fd7f806d1c0 legacy=0x7fd7f8161be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:51.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.981+0000 7fd7ed400640 1 -- 172.21.15.86:0/1585982830 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7f80adcc0 con 0x7fd7f806daa0 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.982+0000 7fd7ed400640 1 --2- 172.21.15.86:0/1585982830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f806cc80 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7fd7f8074ba0 tx=0x7fd7e002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b8f524bee8c648b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.982+0000 7fd7f5400640 1 -- 172.21.15.86:0/1585982830 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7e0035020 con 0x7fd7f806daa0 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.982+0000 7fd7f5400640 1 -- 172.21.15.86:0/1585982830 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd7e0031870 con 0x7fd7f806daa0 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.982+0000 7fd7f5400640 1 -- 172.21.15.86:0/1585982830 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7e0031c40 con 0x7fd7f806daa0 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 -- 172.21.15.86:0/1585982830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 msgr2=0x7fd7f806cc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 --2- 172.21.15.86:0/1585982830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f806cc80 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7fd7f8074ba0 tx=0x7fd7e002b910 comp rx=0 tx=0).stop 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 -- 172.21.15.86:0/1585982830 shutdown_connections 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 --2- 172.21.15.86:0/1585982830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f806cc80 unknown :-1 s=CLOSED pgs=664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 --2- 172.21.15.86:0/1585982830 >> v2:172.21.15.194:3300/0 conn(0x7fd7f8147310 0x7fd7f8072340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 -- 172.21.15.86:0/1585982830 >> 172.21.15.86:0/1585982830 conn(0x7fd7f80b1be0 msgr2=0x7fd7f80b1ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 -- 172.21.15.86:0/1585982830 shutdown_connections 2024-03-06T07:29:51.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.983+0000 7fd7fd200640 1 -- 172.21.15.86:0/1585982830 wait complete. 2024-03-06T07:29:51.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.985+0000 7fd7fd200640 1 Processor -- start 2024-03-06T07:29:51.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.985+0000 7fd7fd200640 1 -- start start 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7fd200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7f806d1c0 0x7fd7f8160100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f813e650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7fd200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7f8147310 0x7fd7f813eb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7fd200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7f81476f0 con 0x7fd7f806daa0 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7fd200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7f80ae280 con 0x7fd7f806d1c0 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7fd200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7f80af7b0 con 0x7fd7f8147310 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7ed400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f813e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7ed400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f813e650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57454/0 (socket says 172.21.15.86:57454) 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.986+0000 7fd7ed400640 1 -- 172.21.15.86:0/934308767 learned_addr learned my addr 172.21.15.86:0/934308767 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7f5e00640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7f806d1c0 0x7fd7f8160100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7ed400640 1 -- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7f8147310 msgr2=0x7fd7f813eb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7ed400640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7f8147310 0x7fd7f813eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7ed400640 1 -- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7f806d1c0 msgr2=0x7fd7f8160100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7ed400640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7f806d1c0 0x7fd7f8160100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7ed400640 1 -- 172.21.15.86:0/934308767 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7f8160a50 con 0x7fd7f806daa0 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7f5e00640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7f806d1c0 0x7fd7f8160100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7ed400640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f813e650 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7fd7e002b950 tx=0x7fd7e00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7eea00640 1 -- 172.21.15.86:0/934308767 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7e003ad60 con 0x7fd7f806daa0 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7eea00640 1 -- 172.21.15.86:0/934308767 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd7e00443f0 con 0x7fd7f806daa0 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7eea00640 1 -- 172.21.15.86:0/934308767 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7e00447e0 con 0x7fd7f806daa0 2024-03-06T07:29:51.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd7f8160d30 con 0x7fd7f806daa0 2024-03-06T07:29:51.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.987+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd7f81612f0 con 0x7fd7f806daa0 2024-03-06T07:29:51.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.989+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd7f804f330 con 0x7fd7f806daa0 2024-03-06T07:29:51.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.989+0000 7fd7eea00640 1 -- 172.21.15.86:0/934308767 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd7e0038020 con 0x7fd7f806daa0 2024-03-06T07:29:51.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.990+0000 7fd7eea00640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd7d406b4d0 0x7fd7d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:51.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.990+0000 7fd7eea00640 1 -- 172.21.15.86:0/934308767 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd7e00ac590 con 0x7fd7f806daa0 2024-03-06T07:29:51.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.991+0000 7fd7f5e00640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd7d406b4d0 0x7fd7d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:51.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.991+0000 7fd7f5e00640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd7d406b4d0 0x7fd7d406d990 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fd7f8161890 tx=0x7fd7e80073e0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:51.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:51.995+0000 7fd7eea00640 1 -- 172.21.15.86:0/934308767 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd7e003f070 con 0x7fd7f806daa0 2024-03-06T07:29:52.199 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:29:52.197+0000 7f2132a00640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:29:52.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.259+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd7f80b3000 con 0x7fd7f806daa0 2024-03-06T07:29:52.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.260+0000 7fd7eea00640 1 -- 172.21.15.86:0/934308767 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fd7f80b39c0 con 0x7fd7f806daa0 2024-03-06T07:29:52.262 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:29:52.262 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:29:52.262 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd7d406b4d0 msgr2=0x7fd7d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd7d406b4d0 0x7fd7d406d990 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fd7f8161890 tx=0x7fd7e80073e0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 msgr2=0x7fd7f813e650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f813e650 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7fd7e002b950 tx=0x7fd7e00046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 shutdown_connections 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd7d406b4d0 0x7fd7d406d990 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7f8147310 0x7fd7f813eb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd7f806daa0 0x7fd7f813e650 unknown :-1 s=CLOSED pgs=665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 --2- 172.21.15.86:0/934308767 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7f806d1c0 0x7fd7f8160100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 >> 172.21.15.86:0/934308767 conn(0x7fd7f80b1be0 msgr2=0x7fd7f80b1200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:52.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 shutdown_connections 2024-03-06T07:29:52.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.263+0000 7fd7fd200640 1 -- 172.21.15.86:0/934308767 wait complete. 2024-03-06T07:29:52.684 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 34, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:29:45.693670+0000', 'modified': '2024-03-06T07:29:48.797087+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 5820}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_5820': {'gid': 5820, 'name': 'c', 'rank': 0, 'incarnation': 30, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/3356338385', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3356338385}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 3356338385}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [9], 'metadata_pool': 8, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 5820, 'qdb_cluster': [5820]} 2024-03-06T07:29:52.684 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:29:52.684 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-06T07:29:52.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.887+0000 7f6b1ba00640 1 Processor -- start 2024-03-06T07:29:52.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.887+0000 7f6b1ba00640 1 -- start start 2024-03-06T07:29:52.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.888+0000 7f6b1ba00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6b14162440 0x7f6b14169060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.888+0000 7f6b1ba00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b140cf2b0 con 0x7f6b140cec10 2024-03-06T07:29:52.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.888+0000 7f6b1ba00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b140cf590 con 0x7f6b1415e360 2024-03-06T07:29:52.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.888+0000 7f6b1ba00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b140cf870 con 0x7f6b14162440 2024-03-06T07:29:52.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.889+0000 7f6b13e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6b140cec10 0x7f6b14161d30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50058/0 (socket says 172.21.15.86:50058) 2024-03-06T07:29:52.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.889+0000 7f6b13e00640 1 -- 172.21.15.86:0/2266601575 learned_addr learned my addr 172.21.15.86:0/2266601575 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:52.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.889+0000 7f6b19a00640 1 --2- 172.21.15.86:0/2266601575 >> v2:172.21.15.194:3300/0 conn(0x7f6b14162440 0x7f6b14169060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.889+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 86489950 0 0) 0x7f6b140cf2b0 con 0x7f6b140cec10 2024-03-06T07:29:52.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.889+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6afc003520 con 0x7f6b140cec10 2024-03-06T07:29:52.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.889+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3532618686 0 0) 0x7f6b140cf590 con 0x7f6b1415e360 2024-03-06T07:29:52.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.889+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6b140cf2b0 con 0x7f6b1415e360 2024-03-06T07:29:52.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.890+0000 7f6b19a00640 1 -- 172.21.15.86:0/2266601575 >> v1:172.21.15.194:6789/0 conn(0x7f6b1415e360 legacy=0x7f6b140d12e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.890+0000 7f6b19a00640 1 -- 172.21.15.86:0/2266601575 >> v1:172.21.15.86:6789/0 conn(0x7f6b140cec10 legacy=0x7f6b14161d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.890+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2876721352 0 0) 0x7f6afc003520 con 0x7f6b140cec10 2024-03-06T07:29:52.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.890+0000 7f6b19a00640 1 -- 172.21.15.86:0/2266601575 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b140cfb50 con 0x7f6b14162440 2024-03-06T07:29:52.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.891+0000 7f6b19a00640 1 --2- 172.21.15.86:0/2266601575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b14162440 0x7f6b14169060 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f6b0c009d90 tx=0x7f6b0c02bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=10a9b4ada8835565 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:52.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.892+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b0c039070 con 0x7f6b14162440 2024-03-06T07:29:52.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.892+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b0c02c9d0 con 0x7f6b14162440 2024-03-06T07:29:52.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.892+0000 7f6b13400640 1 -- 172.21.15.86:0/2266601575 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b00001670 con 0x7f6b14162440 2024-03-06T07:29:52.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.892+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2266601575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b14162440 msgr2=0x7f6b14169060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.892+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2266601575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b14162440 0x7f6b14169060 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f6b0c009d90 tx=0x7f6b0c02bea0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.893+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2266601575 shutdown_connections 2024-03-06T07:29:52.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.893+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2266601575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b14162440 0x7f6b14169060 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.893+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2266601575 >> 172.21.15.86:0/2266601575 conn(0x7f6b140d3f50 msgr2=0x7f6b140d4330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:52.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.893+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2266601575 shutdown_connections 2024-03-06T07:29:52.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.893+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2266601575 wait complete. 2024-03-06T07:29:52.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.894+0000 7f6b1ba00640 1 Processor -- start 2024-03-06T07:29:52.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.895+0000 7f6b1ba00640 1 -- start start 2024-03-06T07:29:52.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.895+0000 7f6b1ba00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b140cec10 0x7f6b140b1580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.895+0000 7f6b1ba00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b1415e360 0x7f6b140cd0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b1ba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b14162440 0x7f6b140cd600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b1ba00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b140d2e20 con 0x7f6b140cec10 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b1ba00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b140d2ca0 con 0x7f6b1415e360 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b1ba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b140d2fa0 con 0x7f6b14162440 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b140cec10 0x7f6b140b1580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b140cec10 0x7f6b140b1580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57456/0 (socket says 172.21.15.86:57456) 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 -- 172.21.15.86:0/2359716896 learned_addr learned my addr 172.21.15.86:0/2359716896 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b13e00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b1415e360 0x7f6b140cd0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19a00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b14162440 0x7f6b140cd600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 -- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b14162440 msgr2=0x7f6b140cd600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b14162440 0x7f6b140cd600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 -- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b1415e360 msgr2=0x7f6b140cd0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b1415e360 0x7f6b140cd0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.896+0000 7f6b19000640 1 -- 172.21.15.86:0/2359716896 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b1415f180 con 0x7f6b140cec10 2024-03-06T07:29:52.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.897+0000 7f6b19000640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b140cec10 0x7f6b140b1580 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f6b0800c530 tx=0x7f6b0800c9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:52.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.897+0000 7f6b10c00640 1 -- 172.21.15.86:0/2359716896 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b080052d0 con 0x7f6b140cec10 2024-03-06T07:29:52.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.897+0000 7f6b10c00640 1 -- 172.21.15.86:0/2359716896 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b08005490 con 0x7f6b140cec10 2024-03-06T07:29:52.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.897+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b140cdd30 con 0x7f6b140cec10 2024-03-06T07:29:52.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.897+0000 7f6b10c00640 1 -- 172.21.15.86:0/2359716896 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b08012870 con 0x7f6b140cec10 2024-03-06T07:29:52.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.897+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b1414b920 con 0x7f6b140cec10 2024-03-06T07:29:52.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.899+0000 7f6b10c00640 1 -- 172.21.15.86:0/2359716896 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b08012a10 con 0x7f6b140cec10 2024-03-06T07:29:52.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.901+0000 7f6b10c00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6af406b4d0 0x7f6af406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.901+0000 7f6b10c00640 1 -- 172.21.15.86:0/2359716896 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6b08089e60 con 0x7f6b140cec10 2024-03-06T07:29:52.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.901+0000 7f6b13e00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6af406b4d0 0x7f6af406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.901+0000 7f6b1e835740 1 Processor -- start 2024-03-06T07:29:52.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.902+0000 7f6b1e835740 1 -- start start 2024-03-06T07:29:52.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.902+0000 7f6b13e00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6af406b4d0 0x7f6af406d990 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f6b1415f5d0 tx=0x7f6b00009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:52.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d930157b0 0x560d92fe8750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d93018540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92fe8c90 0x560d93018a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b1e835740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92f52600 con 0x560d92fe8c90 2024-03-06T07:29:52.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92efb930 con 0x560d93015b90 2024-03-06T07:29:52.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d92f4e6a0 con 0x560d930157b0 2024-03-06T07:29:52.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92fe8c90 0x560d93018a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92fe8c90 0x560d93018a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57468/0 (socket says 172.21.15.86:57468) 2024-03-06T07:29:52.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 -- 172.21.15.86:0/3307630139 learned_addr learned my addr 172.21.15.86:0/3307630139 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:52.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19000640 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d930157b0 0x560d92fe8750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 -- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d930157b0 msgr2=0x560d92fe8750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b13e00640 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d93018540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d930157b0 0x560d92fe8750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 -- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 msgr2=0x560d93018540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d93018540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.903+0000 7f6b19a00640 1 -- 172.21.15.86:0/3307630139 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d92c672d0 con 0x560d92fe8c90 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.904+0000 7f6b19a00640 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92fe8c90 0x560d93018a80 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7f6b0c02c4b0 tx=0x7f6b0c02e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=902b57e0da81e977 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.904+0000 7f6b05600640 1 -- 172.21.15.86:0/3307630139 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b0c039040 con 0x560d92fe8c90 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.904+0000 7f6b05600640 1 -- 172.21.15.86:0/3307630139 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b0c02c9d0 con 0x560d92fe8c90 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.904+0000 7f6b05600640 1 -- 172.21.15.86:0/3307630139 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b0c02cdc0 con 0x560d92fe8c90 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 -- 172.21.15.86:0/3307630139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92fe8c90 msgr2=0x560d93018a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92fe8c90 0x560d93018a80 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7f6b0c02c4b0 tx=0x7f6b0c02e000 comp rx=0 tx=0).stop 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 -- 172.21.15.86:0/3307630139 shutdown_connections 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d92fe8c90 0x560d93018a80 unknown :-1 s=CLOSED pgs=668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d93018540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3307630139 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d930157b0 0x560d92fe8750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 -- 172.21.15.86:0/3307630139 >> 172.21.15.86:0/3307630139 conn(0x560d92fea9c0 msgr2=0x560d93033540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:52.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 -- 172.21.15.86:0/3307630139 shutdown_connections 2024-03-06T07:29:52.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 -- 172.21.15.86:0/3307630139 wait complete. 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.905+0000 7f6b1e835740 1 -- start start 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.906+0000 7f6b1e835740 1 Processor -- start 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93006b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d9302a060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 0x560d9301abd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b1e835740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92fecf60 con 0x560d930157b0 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d92f0ed90 con 0x560d93015b90 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d92fec350 con 0x560d9302a5a0 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b1e835740 10 client.0 _ll_register_callbacks cb 0x560d92f4a350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93006b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93006b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57476/0 (socket says 172.21.15.86:57476) 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 -- 172.21.15.86:0/1319159639 learned_addr learned my addr 172.21.15.86:0/1319159639 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:52.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 -- 172.21.15.86:0/1319159639 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 msgr2=0x560d9301abd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 0x560d9301abd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 -- 172.21.15.86:0/1319159639 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 msgr2=0x560d9302a060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d9302a060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.907+0000 7f6b19000640 1 -- 172.21.15.86:0/1319159639 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560d92d316b0 con 0x560d930157b0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b19000640 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93006b70 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f6b08088390 tx=0x7f6b080585f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ee5ed6b96f753c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6afa000640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6afa000640 1 -- 172.21.15.86:0/1319159639 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b08070180 con 0x560d930157b0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6afa000640 1 -- 172.21.15.86:0/1319159639 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6b08054c60 con 0x560d930157b0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6b0c02c9d0 con 0x560d930157b0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6afa000640 1 -- 172.21.15.86:0/1319159639 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=34) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6b08060870 con 0x560d930157b0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 10 client.5952 fetch_fsmap learned FSMap version 34 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=34,monmap=2+}) v3 -- 0x560d9302b900 con 0x560d930157b0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6afa000640 1 -- 172.21.15.86:0/1319159639 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b0c02c9d0 con 0x560d930157b0 2024-03-06T07:29:52.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6afa000640 1 -- 172.21.15.86:0/1319159639 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 34) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f6b08057a40 con 0x560d930157b0 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 10 client.5952 fetch_fsmap finished waiting for FSMap version 34 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 10 client.5952 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 20 client.5952 populate_metadata read hostname 'smithi086' 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d930470d0 0x560d9302c7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 4 client.5952 mds_command: new command op to 5820 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.908+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 --> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560d92f5fb40 con 0x560d930470d0 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.909+0000 7f6b13e00640 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d930470d0 0x560d9302c7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.909+0000 7f6b13e00640 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d930470d0 0x560d9302c7b0 crc :-1 s=READY pgs=11 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=17779ce17ddb1768 server_cookie=7f30ab52abbec9b4 in_seq=0 out_seq=0 2024-03-06T07:29:52.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.909+0000 7f6afa000640 10 client.5952 ms_handle_connect on v2:172.21.15.86:6834/3356338385 2024-03-06T07:29:52.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6afa000640 1 -- 172.21.15.86:0/1319159639 <== mds.0 v2:172.21.15.86:6834/3356338385 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x560d92f5fb40 con 0x560d930470d0 2024-03-06T07:29:52.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6afa000640 10 client.5952 handle_command_reply: tid=0 2024-03-06T07:29:52.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 client.5952 shutdown 2024-03-06T07:29:52.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 msgr2=0x560d93006b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93006b70 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f6b08088390 tx=0x7f6b080585f0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 shutdown_connections 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d930470d0 0x560d9302c7b0 crc :-1 s=READY pgs=11 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 0x560d9301abd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d9302a060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1319159639 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93006b70 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 >> 172.21.15.86:0/1319159639 conn(0x560d92fe8c90 msgr2=0x560d93018530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.910+0000 7f6afa000640 0 client.5952 ms_handle_reset on v2:172.21.15.86:6834/3356338385 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.911+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 shutdown_connections 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.911+0000 7f6b1e835740 1 -- 172.21.15.86:0/1319159639 wait complete. 2024-03-06T07:29:52.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.911+0000 7f6b1e835740 20 client.5952 trim_cache size 0 max 16384 2024-03-06T07:29:52.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.929+0000 7f6b1e835740 1 Processor -- start 2024-03-06T07:29:52.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.929+0000 7f6b1e835740 1 -- start start 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d9302c780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d93018170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 0x560d93034790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b1e835740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b1414d080 con 0x560d930157b0 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b1414e310 con 0x560d93015b90 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b1414ca80 con 0x560d9302a5a0 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d9302c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d9302c780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57492/0 (socket says 172.21.15.86:57492) 2024-03-06T07:29:52.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.930+0000 7f6b19000640 1 -- 172.21.15.86:0/1676667858 learned_addr learned my addr 172.21.15.86:0/1676667858 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b19000640 1 -- 172.21.15.86:0/1676667858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 msgr2=0x560d93034790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b19000640 1 --2- 172.21.15.86:0/1676667858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 0x560d93034790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b19000640 1 -- 172.21.15.86:0/1676667858 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 msgr2=0x560d93018170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b19000640 1 --2- 172.21.15.86:0/1676667858 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d93018170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b19000640 1 -- 172.21.15.86:0/1676667858 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d930147e0 con 0x560d930157b0 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b19000640 1 --2- 172.21.15.86:0/1676667858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d9302c780 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7f6b080607d0 tx=0x7f6b0801d000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8316cf6690bbb923 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b04c00640 1 -- 172.21.15.86:0/1676667858 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b08070180 con 0x560d930157b0 2024-03-06T07:29:52.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b04c00640 1 -- 172.21.15.86:0/1676667858 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b0807b020 con 0x560d930157b0 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.931+0000 7f6b04c00640 1 -- 172.21.15.86:0/1676667858 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b08057f50 con 0x560d930157b0 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 -- 172.21.15.86:0/1676667858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 msgr2=0x560d9302c780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1676667858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d9302c780 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7f6b080607d0 tx=0x7f6b0801d000 comp rx=0 tx=0).stop 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 -- 172.21.15.86:0/1676667858 shutdown_connections 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1676667858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d9302a5a0 0x560d93034790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1676667858 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d93015b90 0x560d93018170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 --2- 172.21.15.86:0/1676667858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d9302c780 unknown :-1 s=CLOSED pgs=670 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 -- 172.21.15.86:0/1676667858 >> 172.21.15.86:0/1676667858 conn(0x560d92fe8c90 msgr2=0x560d9302f230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 -- 172.21.15.86:0/1676667858 shutdown_connections 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 -- 172.21.15.86:0/1676667858 wait complete. 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.932+0000 7f6b1e835740 1 -- start start 2024-03-06T07:29:52.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.933+0000 7f6b1e835740 1 Processor -- start 2024-03-06T07:29:52.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93019bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d93015b90 0x560d9302d420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b1e835740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d9302a5a0 0x560d92feda20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b1e835740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d93046f50 con 0x560d930157b0 2024-03-06T07:29:52.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d9302bc40 con 0x560d9302a5a0 2024-03-06T07:29:52.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b1e835740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d93047540 con 0x560d93015b90 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b1e835740 10 client.0 _ll_register_callbacks cb 0x7f6b080583c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93019bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93019bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57498/0 (socket says 172.21.15.86:57498) 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 -- 172.21.15.86:0/3702404378 learned_addr learned my addr 172.21.15.86:0/3702404378 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 -- 172.21.15.86:0/3702404378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d93015b90 msgr2=0x560d9302d420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d93015b90 0x560d9302d420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 -- 172.21.15.86:0/3702404378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d9302a5a0 msgr2=0x560d92feda20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19a00640 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d9302a5a0 0x560d92feda20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d9302a5a0 0x560d92feda20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.934+0000 7f6b19000640 1 -- 172.21.15.86:0/3702404378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560d93013300 con 0x560d930157b0 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b19000640 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93019bd0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f6b08058580 tx=0x7f6b080882f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=27a2514f8487db6f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6afa000640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:29:52.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6afa000640 1 -- 172.21.15.86:0/3702404378 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b08057f50 con 0x560d930157b0 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6afa000640 1 -- 172.21.15.86:0/3702404378 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6b08054c60 con 0x560d930157b0 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6b0807b020 con 0x560d930157b0 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6afa000640 1 -- 172.21.15.86:0/3702404378 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=34) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6b08070180 con 0x560d930157b0 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 10 client.5958 fetch_fsmap learned FSMap version 34 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=34,monmap=2+}) v3 -- 0x560d93013300 con 0x560d930157b0 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6afa000640 1 -- 172.21.15.86:0/3702404378 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b0807b020 con 0x560d930157b0 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6afa000640 1 -- 172.21.15.86:0/3702404378 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 34) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f6b08054340 con 0x560d930157b0 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 10 client.5958 fetch_fsmap finished waiting for FSMap version 34 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 10 client.5958 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 20 client.5958 populate_metadata read hostname 'smithi086' 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d9302f450 0x560d92fedf60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 4 client.5958 mds_command: new command op to 5820 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.935+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 --> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] -- command(tid 0: {"prefix": "status"}) v1 -- 0x560d92f59e90 con 0x560d9302f450 2024-03-06T07:29:52.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.936+0000 7f6b13e00640 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d9302f450 0x560d92fedf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.936+0000 7f6afa000640 10 client.5958 ms_handle_connect on v2:172.21.15.86:6834/3356338385 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.936+0000 7f6b13e00640 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d9302f450 0x560d92fedf60 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=87a5e186985d5b10 server_cookie=ad98314b0f3a5fef in_seq=0 out_seq=0 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6afa000640 1 -- 172.21.15.86:0/3702404378 <== mds.0 v2:172.21.15.86:6834/3356338385 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x560d92f59e90 con 0x560d9302f450 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6afa000640 10 client.5958 handle_command_reply: tid=0 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 client.5958 shutdown 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 msgr2=0x560d93019bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93019bd0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f6b08058580 tx=0x7f6b080882f0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 shutdown_connections 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:6834/3356338385,v1:172.21.15.86:6837/3356338385] conn(0x560d9302f450 0x560d92fedf60 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560d9302a5a0 0x560d92feda20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560d93015b90 0x560d9302d420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 --2- 172.21.15.86:0/3702404378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560d930157b0 0x560d93019bd0 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 >> 172.21.15.86:0/3702404378 conn(0x560d92fe8c90 msgr2=0x560d93035f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:52.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6afa000640 0 client.5958 ms_handle_reset on v2:172.21.15.86:6834/3356338385 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 shutdown_connections 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 1 -- 172.21.15.86:0/3702404378 wait complete. 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.937+0000 7f6b1e835740 20 client.5958 trim_cache size 0 max 16384 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 5820, 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:29:52.943 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 6.6886029450000004, 2024-03-06T07:29:52.944 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 34, 2024-03-06T07:29:52.944 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 58, 2024-03-06T07:29:52.944 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 58, 2024-03-06T07:29:52.944 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 12.109845119999999 2024-03-06T07:29:52.944 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:29:52.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6af406b4d0 msgr2=0x7f6af406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6af406b4d0 0x7f6af406d990 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f6b1415f5d0 tx=0x7f6b00009040 comp rx=0 tx=0).stop 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b140cec10 msgr2=0x7f6b140b1580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b140cec10 0x7f6b140b1580 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f6b0800c530 tx=0x7f6b0800c9f0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 shutdown_connections 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6af406b4d0 0x7f6af406d990 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b14162440 0x7f6b140cd600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b1415e360 0x7f6b140cd0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 --2- 172.21.15.86:0/2359716896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b140cec10 0x7f6b140b1580 unknown :-1 s=CLOSED pgs=667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 >> 172.21.15.86:0/2359716896 conn(0x7f6b140d3f50 msgr2=0x7f6b140b0b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 shutdown_connections 2024-03-06T07:29:52.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:52.938+0000 7f6b1ba00640 1 -- 172.21.15.86:0/2359716896 wait complete. 2024-03-06T07:29:52.956 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:29:53.114 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:29:53.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:29:53.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.148+0000 7faa5f400640 1 Processor -- start 2024-03-06T07:29:53.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.149+0000 7faa5f400640 1 -- start start 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.150+0000 7faa5f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7faa58144c00 0x7faa5806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.150+0000 7faa5f400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7faa5815f9d0 0x7faa5815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.150+0000 7faa5f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa580ce0a0 0x7faa580ce4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.150+0000 7faa5f400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faa5806ee70 con 0x7faa58144c00 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.150+0000 7faa5f400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faa5806cef0 con 0x7faa5815f9d0 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.150+0000 7faa5f400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faa5806d1d0 con 0x7faa580ce0a0 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7faa58144c00 0x7faa5806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7faa58144c00 0x7faa5806e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57500/0 (socket says 172.21.15.86:57500) 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 -- 172.21.15.86:0/1068153906 learned_addr learned my addr 172.21.15.86:0/1068153906 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5d400640 1 --2- 172.21.15.86:0/1068153906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa580ce0a0 0x7faa580ce4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:53.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa57e00640 1 --2- 172.21.15.86:0/1068153906 >> v2:172.21.15.194:3300/0 conn(0x7faa5815f9d0 0x7faa5815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:53.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 -- 172.21.15.86:0/1068153906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa580ce0a0 msgr2=0x7faa580ce4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:53.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 --2- 172.21.15.86:0/1068153906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa580ce0a0 0x7faa580ce4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 -- 172.21.15.86:0/1068153906 >> v2:172.21.15.194:3300/0 conn(0x7faa5815f9d0 msgr2=0x7faa5815fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:53.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 --2- 172.21.15.86:0/1068153906 >> v2:172.21.15.194:3300/0 conn(0x7faa5815f9d0 0x7faa5815fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.151+0000 7faa5ca00640 1 -- 172.21.15.86:0/1068153906 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa5806d4b0 con 0x7faa58144c00 2024-03-06T07:29:53.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.153+0000 7faa5ca00640 1 --2- 172.21.15.86:0/1068153906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa58144c00 0x7faa5806e930 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7faa580b26f0 tx=0x7faa4c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=797621238f1c8398 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:53.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.153+0000 7faa57400640 1 -- 172.21.15.86:0/1068153906 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa4c035020 con 0x7faa58144c00 2024-03-06T07:29:53.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.153+0000 7faa57400640 1 -- 172.21.15.86:0/1068153906 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faa4c031870 con 0x7faa58144c00 2024-03-06T07:29:53.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.153+0000 7faa57400640 1 -- 172.21.15.86:0/1068153906 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa4c031c40 con 0x7faa58144c00 2024-03-06T07:29:53.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.153+0000 7faa5f400640 1 -- 172.21.15.86:0/1068153906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa58144c00 msgr2=0x7faa5806e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.153+0000 7faa5f400640 1 --2- 172.21.15.86:0/1068153906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa58144c00 0x7faa5806e930 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7faa580b26f0 tx=0x7faa4c02b910 comp rx=0 tx=0).stop 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.154+0000 7faa5f400640 1 -- 172.21.15.86:0/1068153906 shutdown_connections 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.154+0000 7faa5f400640 1 --2- 172.21.15.86:0/1068153906 >> v2:172.21.15.194:3300/0 conn(0x7faa5815f9d0 0x7faa5815fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.154+0000 7faa5f400640 1 --2- 172.21.15.86:0/1068153906 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa580ce0a0 0x7faa580ce4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.154+0000 7faa5f400640 1 --2- 172.21.15.86:0/1068153906 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa58144c00 0x7faa5806e930 unknown :-1 s=CLOSED pgs=672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.154+0000 7faa5f400640 1 -- 172.21.15.86:0/1068153906 >> 172.21.15.86:0/1068153906 conn(0x7faa580b1730 msgr2=0x7faa580b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.154+0000 7faa5f400640 1 -- 172.21.15.86:0/1068153906 shutdown_connections 2024-03-06T07:29:53.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.154+0000 7faa5f400640 1 -- 172.21.15.86:0/1068153906 wait complete. 2024-03-06T07:29:53.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.156+0000 7faa5f400640 1 Processor -- start 2024-03-06T07:29:53.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.156+0000 7faa5f400640 1 -- start start 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa580ce0a0 0x7faa581f5200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5f400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa58144c00 0x7faa5815e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa5815f9d0 0x7faa5815e8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5f400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa5806cb50 con 0x7faa580ce0a0 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5f400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa58160730 con 0x7faa58144c00 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5f400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faa5806ccd0 con 0x7faa5815f9d0 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa580ce0a0 0x7faa581f5200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa580ce0a0 0x7faa581f5200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57510/0 (socket says 172.21.15.86:57510) 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 -- 172.21.15.86:0/1018216597 learned_addr learned my addr 172.21.15.86:0/1018216597 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa57e00640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa58144c00 0x7faa5815e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5d400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa5815f9d0 0x7faa5815e8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:53.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 -- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa5815f9d0 msgr2=0x7faa5815e8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa5815f9d0 0x7faa5815e8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 -- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa58144c00 msgr2=0x7faa5815e3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa58144c00 0x7faa5815e3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5ca00640 1 -- 172.21.15.86:0/1018216597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa5815d5e0 con 0x7faa580ce0a0 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.157+0000 7faa5d400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa5815f9d0 0x7faa5815e8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.158+0000 7faa5ca00640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa580ce0a0 0x7faa581f5200 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7faa4c000c00 tx=0x7faa4c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.158+0000 7faa54c00640 1 -- 172.21.15.86:0/1018216597 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa4c035040 con 0x7faa580ce0a0 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.158+0000 7faa54c00640 1 -- 172.21.15.86:0/1018216597 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faa4c030d00 con 0x7faa580ce0a0 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.158+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faa5815d8c0 con 0x7faa580ce0a0 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.158+0000 7faa54c00640 1 -- 172.21.15.86:0/1018216597 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa4c0456a0 con 0x7faa580ce0a0 2024-03-06T07:29:53.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.158+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faa58143310 con 0x7faa580ce0a0 2024-03-06T07:29:53.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.160+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faa5804f330 con 0x7faa580ce0a0 2024-03-06T07:29:53.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.160+0000 7faa54c00640 1 -- 172.21.15.86:0/1018216597 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faa4c038020 con 0x7faa580ce0a0 2024-03-06T07:29:53.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.161+0000 7faa54c00640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa3806b4d0 0x7faa3806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:53.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.161+0000 7faa54c00640 1 -- 172.21.15.86:0/1018216597 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7faa4c0ac530 con 0x7faa580ce0a0 2024-03-06T07:29:53.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.161+0000 7faa57e00640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa3806b4d0 0x7faa3806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:53.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.162+0000 7faa57e00640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa3806b4d0 0x7faa3806d990 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7faa581f7590 tx=0x7faa44009190 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:53.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.166+0000 7faa54c00640 1 -- 172.21.15.86:0/1018216597 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faa4c07b390 con 0x7faa580ce0a0 2024-03-06T07:29:53.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.415+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7faa58072890 con 0x7faa580ce0a0 2024-03-06T07:29:53.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.416+0000 7faa54c00640 1 -- 172.21.15.86:0/1018216597 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v34) v1 ==== 53+0+73 (secure 0 0 0) 0x7faa4c07ac90 con 0x7faa580ce0a0 2024-03-06T07:29:53.418 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:29:53.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa3806b4d0 msgr2=0x7faa3806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:53.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa3806b4d0 0x7faa3806d990 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7faa581f7590 tx=0x7faa44009190 comp rx=0 tx=0).stop 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa580ce0a0 msgr2=0x7faa581f5200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa580ce0a0 0x7faa581f5200 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7faa4c000c00 tx=0x7faa4c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 shutdown_connections 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa3806b4d0 0x7faa3806d990 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faa5815f9d0 0x7faa5815e8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa58144c00 0x7faa5815e3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 --2- 172.21.15.86:0/1018216597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa580ce0a0 0x7faa581f5200 unknown :-1 s=CLOSED pgs=673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 >> 172.21.15.86:0/1018216597 conn(0x7faa580b1730 msgr2=0x7faa58145040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 shutdown_connections 2024-03-06T07:29:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:53.419+0000 7faa5f400640 1 -- 172.21.15.86:0/1018216597 wait complete. 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:29:53.439 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:29:53.440 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:29:53.440 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:53.440 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:53.487 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42273sec preferred_lft 42273sec 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:53.488 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:29:53.489 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:29:53.490 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:53.490 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:29:53.620 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:29:53.627 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:53.627 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:29:53.676 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:29:53.676 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:29:53.676 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:29:53.676 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:29:53.676 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:29:53.677 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:53.677 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:29:53.677 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:29:53.677 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:29:53.677 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:29:53.677 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:29:53.677 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:29:53.842 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:53.842 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:29:53.888 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:29:53.888 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:29:53.889 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:29:53.889 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:53.890 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:29:54.016 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:29:54.016 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:54.066 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:29:54.067 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:29:54.067 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:54.118 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:29:54.119 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:54.119 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:29:54.249 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:29:54.249 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:29:54.282 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:29:54.282 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:29:54.282 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:29:54.283 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:29:54.283 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:29:54.285 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:29:54.286 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:29:54.286 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:29:54.286 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:29:54.286 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:29:54.291 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:29:54.291 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:54.291 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:29:54.368 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:29:54.368 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:29:54.420 INFO:teuthology.orchestra.run.smithi086.stdout:client5335 2024-03-06T07:29:54.420 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:29:54.421 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:29:54.465 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_fragment_size_max 50 2024-03-06T07:29:54.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.721+0000 7fd308c00640 1 Processor -- start 2024-03-06T07:29:54.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.721+0000 7fd308c00640 1 -- start start 2024-03-06T07:29:54.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.722+0000 7fd308c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd3040d07f0 0x7fd3040d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.722+0000 7fd308c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd30415e450 0x7fd3040b4480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd308c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3040b3730 con 0x7fd3040d1bf0 2024-03-06T07:29:54.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd308c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3040d4930 con 0x7fd3040d07f0 2024-03-06T07:29:54.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd308c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd304067b20 con 0x7fd30415e450 2024-03-06T07:29:54.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd302800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd3040d1bf0 0x7fd3040b49c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:50064/0 (socket says 172.21.15.86:50064) 2024-03-06T07:29:54.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd302800640 1 -- 172.21.15.86:0/2885160113 learned_addr learned my addr 172.21.15.86:0/2885160113 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:54.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd301e00640 1 --2- 172.21.15.86:0/2885160113 >> v2:172.21.15.194:3300/0 conn(0x7fd3040d07f0 0x7fd3040d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd301400640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd30415e450 0x7fd3040b4480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.723+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4206034003 0 0) 0x7fd3040b3730 con 0x7fd3040d1bf0 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd2e8003520 con 0x7fd3040d1bf0 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2874024864 0 0) 0x7fd2e8003520 con 0x7fd3040d1bf0 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd30415e450 msgr2=0x7fd3040b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd30415e450 0x7fd3040b4480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 >> v2:172.21.15.194:3300/0 conn(0x7fd3040d07f0 msgr2=0x7fd3040d0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 --2- 172.21.15.86:0/2885160113 >> v2:172.21.15.194:3300/0 conn(0x7fd3040d07f0 0x7fd3040d0bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3040d1110 con 0x7fd3040d1bf0 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd2fc0019f0 con 0x7fd3040d1bf0 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 >> v1:172.21.15.86:6789/0 conn(0x7fd3040d1bf0 legacy=0x7fd3040b49c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd2e80039b0 0x7fd2e8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd30415e450 0x7fd2e8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd2e80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3040d4930 con 0x7fd2e8004d40 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd304067b20 con 0x7fd2e80039b0 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd2e800b9c0 con 0x7fd30415e450 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.724+0000 7fd302800640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd2e80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.725+0000 7fd301e00640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd2e80039b0 0x7fd2e8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.725+0000 7fd302800640 1 -- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd30415e450 msgr2=0x7fd2e8004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.725+0000 7fd302800640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd30415e450 0x7fd2e8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.725+0000 7fd302800640 1 -- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd2e80039b0 msgr2=0x7fd2e8003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.725+0000 7fd302800640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd2e80039b0 0x7fd2e8003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.725+0000 7fd302800640 1 -- 172.21.15.86:0/2885160113 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd2e8003520 con 0x7fd2e8004d40 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.726+0000 7fd302800640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd2e80051c0 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7fd2fc00a920 tx=0x7fd2e8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=27bf079e3e1eb9bd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.726+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd2fc00a4c0 con 0x7fd2e8004d40 2024-03-06T07:29:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.726+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd2fc034650 con 0x7fd2e8004d40 2024-03-06T07:29:54.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.726+0000 7fd300a00640 1 -- 172.21.15.86:0/2885160113 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd2fc034a40 con 0x7fd2e8004d40 2024-03-06T07:29:54.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.727+0000 7fd308c00640 1 -- 172.21.15.86:0/2885160113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 msgr2=0x7fd2e80051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:54.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.727+0000 7fd308c00640 1 --2- 172.21.15.86:0/2885160113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd2e80051c0 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7fd2fc00a920 tx=0x7fd2e8005830 comp rx=0 tx=0).stop 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.727+0000 7fd302800640 1 -- 172.21.15.86:0/2885160113 reap_dead start 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.727+0000 7fd308c00640 1 -- 172.21.15.86:0/2885160113 shutdown_connections 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.727+0000 7fd308c00640 1 -- 172.21.15.86:0/2885160113 >> 172.21.15.86:0/2885160113 conn(0x7fd3040d3e10 msgr2=0x7fd3040d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.727+0000 7fd308c00640 1 -- 172.21.15.86:0/2885160113 shutdown_connections 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.727+0000 7fd308c00640 1 -- 172.21.15.86:0/2885160113 wait complete. 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.728+0000 7fd308c00640 1 Processor -- start 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.729+0000 7fd308c00640 1 -- start start 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.729+0000 7fd308c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd3040ccb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.729+0000 7fd308c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3040b24c0 0x7fd3040c9df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd308c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3040b2920 0x7fd3040ca330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd308c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3040b3c80 con 0x7fd2e8004d40 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd308c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3040b39a0 con 0x7fd3040b24c0 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd308c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3040b3f60 con 0x7fd3040b2920 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd3040ccb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd3040ccb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57536/0 (socket says 172.21.15.86:57536) 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 -- 172.21.15.86:0/3078511528 learned_addr learned my addr 172.21.15.86:0/3078511528 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd302800640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3040b2920 0x7fd3040ca330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301400640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3040b24c0 0x7fd3040c9df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 -- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3040b2920 msgr2=0x7fd3040ca330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3040b2920 0x7fd3040ca330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 -- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3040b24c0 msgr2=0x7fd3040c9df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3040b24c0 0x7fd3040c9df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd301e00640 1 -- 172.21.15.86:0/3078511528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3040cba70 con 0x7fd2e8004d40 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.730+0000 7fd302800640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3040b2920 0x7fd3040ca330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.731+0000 7fd301e00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd3040ccb10 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fd2f40108e0 tx=0x7fd2f4010da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.731+0000 7fd2fa000640 1 -- 172.21.15.86:0/3078511528 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd2f400f040 con 0x7fd2e8004d40 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.731+0000 7fd2fa000640 1 -- 172.21.15.86:0/3078511528 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd2f400eb60 con 0x7fd2e8004d40 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.731+0000 7fd2fa000640 1 -- 172.21.15.86:0/3078511528 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd2f401c400 con 0x7fd2e8004d40 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.731+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3040cbd50 con 0x7fd2e8004d40 2024-03-06T07:29:54.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.731+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3040cc310 con 0x7fd2e8004d40 2024-03-06T07:29:54.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.732+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd30404f330 con 0x7fd2e8004d40 2024-03-06T07:29:54.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.734+0000 7fd2fa000640 1 -- 172.21.15.86:0/3078511528 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd2f401c5a0 con 0x7fd2e8004d40 2024-03-06T07:29:54.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.735+0000 7fd2fa000640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd2e406b500 0x7fd2e406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:54.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.735+0000 7fd2fa000640 1 -- 172.21.15.86:0/3078511528 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd2f408aa50 con 0x7fd2e8004d40 2024-03-06T07:29:54.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.735+0000 7fd301400640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd2e406b500 0x7fd2e406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:54.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.736+0000 7fd301400640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd2e406b500 0x7fd2e406d9c0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fd2f0004770 tx=0x7fd2f0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:54.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.741+0000 7fd2fa000640 1 -- 172.21.15.86:0/3078511528 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd2f4018030 con 0x7fd2e8004d40 2024-03-06T07:29:54.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.987+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mds_bal_fragment_size_max}] v 0) v1 -- 0x7fd3040d21d0 con 0x7fd2e8004d40 2024-03-06T07:29:54.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.993+0000 7fd2fa000640 1 -- 172.21.15.86:0/3078511528 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_fragment_size_max}]=0 v17)=0 v17) v1 ==== 124+0+0 (secure 0 0 0) 0x7fd2f40591f0 con 0x7fd2e8004d40 2024-03-06T07:29:54.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd2e406b500 msgr2=0x7fd2e406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:54.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd2e406b500 0x7fd2e406d9c0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fd2f0004770 tx=0x7fd2f0008040 comp rx=0 tx=0).stop 2024-03-06T07:29:54.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 msgr2=0x7fd3040ccb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:54.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd3040ccb10 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fd2f40108e0 tx=0x7fd2f4010da0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 shutdown_connections 2024-03-06T07:29:54.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd2e406b500 0x7fd2e406d9c0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3040b2920 0x7fd3040ca330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3040b24c0 0x7fd3040c9df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 --2- 172.21.15.86:0/3078511528 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd2e8004d40 0x7fd3040ccb10 unknown :-1 s=CLOSED pgs=676 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:54.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 >> 172.21.15.86:0/3078511528 conn(0x7fd3040d3e10 msgr2=0x7fd3040b6030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:54.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.996+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 shutdown_connections 2024-03-06T07:29:54.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:54.997+0000 7fd308c00640 1 -- 172.21.15.86:0/3078511528 wait complete. 2024-03-06T07:29:55.017 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_merge_size 1 2024-03-06T07:29:55.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.213+0000 7f584d800640 1 Processor -- start 2024-03-06T07:29:55.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.213+0000 7f584d800640 1 -- start start 2024-03-06T07:29:55.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.215+0000 7f584d800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f58480d07f0 0x7f58480d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:55.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.215+0000 7f584d800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d1bf0 0x7f58480cfa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:55.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.215+0000 7f584d800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f58480b3730 con 0x7f584815e450 2024-03-06T07:29:55.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.215+0000 7f584d800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f58480d4930 con 0x7f58480d07f0 2024-03-06T07:29:55.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.215+0000 7f584d800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5848067b20 con 0x7f58480d1bf0 2024-03-06T07:29:55.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.216+0000 7f5846a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f58480d07f0 0x7f58480d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:55.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.216+0000 7f583e000640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f584815e450 0x7f58480b4480 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:45636/0 (socket says 172.21.15.86:45636) 2024-03-06T07:29:55.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.216+0000 7f5846a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f58480d07f0 0x7f58480d0bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:35752/0 (socket says 172.21.15.86:35752) 2024-03-06T07:29:55.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.216+0000 7f583e000640 1 -- 172.21.15.86:0/2281778646 learned_addr learned my addr 172.21.15.86:0/2281778646 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:29:55.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.216+0000 7f5847400640 1 --2- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d1bf0 0x7f58480cfa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:55.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.217+0000 7f5846a00640 1 -- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d1bf0 msgr2=0x7f58480cfa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:29:55.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.217+0000 7f5846a00640 1 --2- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d1bf0 0x7f58480cfa30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.217+0000 7f5846a00640 1 -- 172.21.15.86:0/2281778646 >> v1:172.21.15.194:6789/0 conn(0x7f584815e450 legacy=0x7f58480b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:55.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.217+0000 7f5846a00640 1 -- 172.21.15.86:0/2281778646 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58480d1110 con 0x7f58480d07f0 2024-03-06T07:29:55.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.218+0000 7f5846a00640 1 --2- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d07f0 0x7f58480d0bd0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f58380097a0 tx=0x7f583802b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fe8151f6be56ed98 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.218+0000 7f5846000640 1 -- 172.21.15.86:0/2281778646 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f58380315a0 con 0x7f58480d07f0 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.218+0000 7f5846000640 1 -- 172.21.15.86:0/2281778646 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5838037070 con 0x7f58480d07f0 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.218+0000 7f5846000640 1 -- 172.21.15.86:0/2281778646 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5838031960 con 0x7f58480d07f0 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 -- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d07f0 msgr2=0x7f58480d0bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 --2- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d07f0 0x7f58480d0bd0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f58380097a0 tx=0x7f583802b840 comp rx=0 tx=0).stop 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 -- 172.21.15.86:0/2281778646 shutdown_connections 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 --2- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d1bf0 0x7f58480cfa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 --2- 172.21.15.86:0/2281778646 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d07f0 0x7f58480d0bd0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 -- 172.21.15.86:0/2281778646 >> 172.21.15.86:0/2281778646 conn(0x7f58480d3e10 msgr2=0x7f58480d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 -- 172.21.15.86:0/2281778646 shutdown_connections 2024-03-06T07:29:55.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.219+0000 7f584d800640 1 -- 172.21.15.86:0/2281778646 wait complete. 2024-03-06T07:29:55.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.221+0000 7f584d800640 1 Processor -- start 2024-03-06T07:29:55.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.221+0000 7f584d800640 1 -- start start 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f584d800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d07f0 0x7f5848154e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f584d800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d1bf0 0x7f58480b91f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f584d800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f584815e450 0x7f58480b6a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f584d800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f58480b3c80 con 0x7f584815e450 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f584d800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f58480b39a0 con 0x7f58480d1bf0 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f584d800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f58480b3f60 con 0x7f58480d07f0 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f5847400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f584815e450 0x7f58480b6a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f5847400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f584815e450 0x7f58480b6a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57544/0 (socket says 172.21.15.86:57544) 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f5847400640 1 -- 172.21.15.86:0/1453670598 learned_addr learned my addr 172.21.15.86:0/1453670598 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f5846a00640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d07f0 0x7f5848154e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:55.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.222+0000 7f583e000640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d1bf0 0x7f58480b91f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f5847400640 1 -- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d07f0 msgr2=0x7f5848154e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f5847400640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d07f0 0x7f5848154e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f5847400640 1 -- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d1bf0 msgr2=0x7f58480b91f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f5847400640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d1bf0 0x7f58480b91f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f5847400640 1 -- 172.21.15.86:0/1453670598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58480b7910 con 0x7f584815e450 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f5847400640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f584815e450 0x7f58480b6a00 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7f5840009e30 tx=0x7f584000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f583f400640 1 -- 172.21.15.86:0/1453670598 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5840018020 con 0x7f584815e450 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f583f400640 1 -- 172.21.15.86:0/1453670598 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5840014480 con 0x7f584815e450 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f58480b7bf0 con 0x7f584815e450 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f583f400640 1 -- 172.21.15.86:0/1453670598 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5840014870 con 0x7f584815e450 2024-03-06T07:29:55.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.223+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5848158ff0 con 0x7f584815e450 2024-03-06T07:29:55.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.225+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f584804f330 con 0x7f584815e450 2024-03-06T07:29:55.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.226+0000 7f583f400640 1 -- 172.21.15.86:0/1453670598 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5840005d10 con 0x7f584815e450 2024-03-06T07:29:55.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.227+0000 7f583f400640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f582406b4d0 0x7f582406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:29:55.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.227+0000 7f583f400640 1 -- 172.21.15.86:0/1453670598 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f584008b150 con 0x7f584815e450 2024-03-06T07:29:55.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.227+0000 7f5846a00640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f582406b4d0 0x7f582406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:29:55.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.228+0000 7f5846a00640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f582406b4d0 0x7f582406d990 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f5838036000 tx=0x7f583803d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:29:55.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.232+0000 7f583f400640 1 -- 172.21.15.86:0/1453670598 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5840022030 con 0x7f584815e450 2024-03-06T07:29:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.432+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mds_bal_merge_size}] v 0) v1 -- 0x7f58481e74f0 con 0x7f584815e450 2024-03-06T07:29:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.437+0000 7f583f400640 1 -- 172.21.15.86:0/1453670598 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_merge_size}]=0 v18)=0 v18) v1 ==== 116+0+0 (secure 0 0 0) 0x7f5840010040 con 0x7f584815e450 2024-03-06T07:29:55.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.440+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f582406b4d0 msgr2=0x7f582406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.440+0000 7f584d800640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f582406b4d0 0x7f582406d990 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f5838036000 tx=0x7f583803d040 comp rx=0 tx=0).stop 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.440+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f584815e450 msgr2=0x7f58480b6a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.440+0000 7f584d800640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f584815e450 0x7f58480b6a00 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7f5840009e30 tx=0x7f584000c490 comp rx=0 tx=0).stop 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 shutdown_connections 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f582406b4d0 0x7f582406d990 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f584815e450 0x7f58480b6a00 unknown :-1 s=CLOSED pgs=677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58480d1bf0 0x7f58480b91f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 --2- 172.21.15.86:0/1453670598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f58480d07f0 0x7f5848154e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 >> 172.21.15.86:0/1453670598 conn(0x7f58480d3e10 msgr2=0x7f58480b09e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 shutdown_connections 2024-03-06T07:29:55.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:29:55.441+0000 7f584d800640 1 -- 172.21.15.86:0/1453670598 wait complete. 2024-03-06T07:30:05.463 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:05.463 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> import uuid 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> n = 50 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = False 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> if 0 > 0: 2024-03-06T07:30:05.521 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = True 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> dpath = os.path.dirname(path) 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> fnameprefix = os.path.basename(path) 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> os.makedirs(dpath, exist_ok=True) 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> try: 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> for i in range(n): 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> with open(fpath, '"'"'w'"'"') as f: 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> f.write(f"{i}") 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> f.flush() 2024-03-06T07:30:05.522 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(f.fileno()) 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> os.unlink(fpath) 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> if create_hard_links: 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> for j in range(0): 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> if True: 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> finally: 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> os.close(dirfd) 2024-03-06T07:30:05.523 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:30:05.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:05 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:30:05.801 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:05.802 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:05.892 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:05.892 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:05.985 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:06.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.204+0000 7f29dbe00640 1 Processor -- start 2024-03-06T07:30:06.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.204+0000 7f29dbe00640 1 -- start start 2024-03-06T07:30:06.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.206+0000 7f29dbe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f29dc1620c0 0x7f29dc0b28e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.206+0000 7f29dbe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29dc161900 con 0x7f29dc0ae670 2024-03-06T07:30:06.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.206+0000 7f29dbe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29dc15ebd0 con 0x7f29dc0b40c0 2024-03-06T07:30:06.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.206+0000 7f29dbe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f29dc15eeb0 con 0x7f29dc1620c0 2024-03-06T07:30:06.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.206+0000 7f29d2000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f29dc0ae670 0x7f29dc15e2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54970/0 (socket says 172.21.15.86:54970) 2024-03-06T07:30:06.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.206+0000 7f29d2000640 1 -- 172.21.15.86:0/1390463580 learned_addr learned my addr 172.21.15.86:0/1390463580 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:06.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29db400640 1 --2- 172.21.15.86:0/1390463580 >> v2:172.21.15.194:3300/0 conn(0x7f29dc1620c0 0x7f29dc0b28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3810250632 0 0) 0x7f29dc161900 con 0x7f29dc0ae670 2024-03-06T07:30:06.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f29c0003520 con 0x7f29dc0ae670 2024-03-06T07:30:06.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3577528494 0 0) 0x7f29c0003520 con 0x7f29dc0ae670 2024-03-06T07:30:06.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 >> v2:172.21.15.194:3300/0 conn(0x7f29dc1620c0 msgr2=0x7f29dc0b28e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:06.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 --2- 172.21.15.86:0/1390463580 >> v2:172.21.15.194:3300/0 conn(0x7f29dc1620c0 0x7f29dc0b28e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 >> v1:172.21.15.194:6789/0 conn(0x7f29dc0b40c0 legacy=0x7f29dc0b44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29dc15f190 con 0x7f29dc0ae670 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f29c4001670 con 0x7f29dc0ae670 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 >> v1:172.21.15.86:6789/0 conn(0x7f29dc0ae670 legacy=0x7f29dc15e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29c00039b0 0x7f29c0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29c0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29c00051c0 0x7f29c000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc15eeb0 con 0x7f29c0004800 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29c000c110 con 0x7f29c00051c0 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.207+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29c000c310 con 0x7f29c00039b0 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29d2000640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29c0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29daa00640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29c00039b0 0x7f29c0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29d2000640 1 -- 172.21.15.86:0/1390463580 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29c00039b0 msgr2=0x7f29c0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29d2000640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29c00039b0 0x7f29c0003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29d2000640 1 -- 172.21.15.86:0/1390463580 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29c00051c0 msgr2=0x7f29c000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:06.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29d2000640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29c00051c0 0x7f29c000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29d2000640 1 -- 172.21.15.86:0/1390463580 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29c0003520 con 0x7f29c0004800 2024-03-06T07:30:06.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.208+0000 7f29db400640 1 -- 172.21.15.86:0/1390463580 reap_dead start 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.209+0000 7f29d2000640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29c0004c80 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f29dc0aeb20 tx=0x7f29c0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af0f6c797efa4861 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.209+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29c4007d30 con 0x7f29c0004800 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.209+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29c4030070 con 0x7f29c0004800 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.209+0000 7f29da000640 1 -- 172.21.15.86:0/1390463580 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29c4005d70 con 0x7f29c0004800 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.210+0000 7f29dbe00640 1 -- 172.21.15.86:0/1390463580 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 msgr2=0x7f29c0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.210+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29c0004c80 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f29dc0aeb20 tx=0x7f29c0005830 comp rx=0 tx=0).stop 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.210+0000 7f29dbe00640 1 -- 172.21.15.86:0/1390463580 shutdown_connections 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.210+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1390463580 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29c0004c80 unknown :-1 s=CLOSED pgs=679 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.210+0000 7f29dbe00640 1 -- 172.21.15.86:0/1390463580 >> 172.21.15.86:0/1390463580 conn(0x7f29dc06afc0 msgr2=0x7f29dc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:06.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.210+0000 7f29dbe00640 1 -- 172.21.15.86:0/1390463580 shutdown_connections 2024-03-06T07:30:06.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.210+0000 7f29dbe00640 1 -- 172.21.15.86:0/1390463580 wait complete. 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.212+0000 7f29dbe00640 1 Processor -- start 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.212+0000 7f29dbe00640 1 -- start start 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29dbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29dc1f8090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29dbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29dc1409e0 0x7f29dc15a870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29dbe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29dc140e60 0x7f29dc15adb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29dbe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc1587e0 con 0x7f29c0004800 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29dbe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc158660 con 0x7f29dc140e60 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29dbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc162570 con 0x7f29dc1409e0 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29daa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29dc1f8090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29daa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29dc1f8090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49552/0 (socket says 172.21.15.86:49552) 2024-03-06T07:30:06.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.213+0000 7f29daa00640 1 -- 172.21.15.86:0/1740119184 learned_addr learned my addr 172.21.15.86:0/1740119184 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:06.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29db400640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29dc140e60 0x7f29dc15adb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29d2000640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29dc1409e0 0x7f29dc15a870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29daa00640 1 -- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29dc1409e0 msgr2=0x7f29dc15a870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29daa00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29dc1409e0 0x7f29dc15a870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29daa00640 1 -- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29dc140e60 msgr2=0x7f29dc15adb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29daa00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29dc140e60 0x7f29dc15adb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29daa00640 1 -- 172.21.15.86:0/1740119184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29dc15b3c0 con 0x7f29c0004800 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29daa00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29dc1f8090 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f29cc00d830 tx=0x7f29cc00dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29d3400640 1 -- 172.21.15.86:0/1740119184 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29cc003da0 con 0x7f29c0004800 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29d3400640 1 -- 172.21.15.86:0/1740119184 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29cc017070 con 0x7f29c0004800 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29d3400640 1 -- 172.21.15.86:0/1740119184 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29cc005290 con 0x7f29c0004800 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29dc15b6a0 con 0x7f29c0004800 2024-03-06T07:30:06.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.214+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f29dc1f7890 con 0x7f29c0004800 2024-03-06T07:30:06.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.216+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f29dc04f330 con 0x7f29c0004800 2024-03-06T07:30:06.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.216+0000 7f29d3400640 1 -- 172.21.15.86:0/1740119184 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f29cc005430 con 0x7f29c0004800 2024-03-06T07:30:06.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.217+0000 7f29d3400640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f29b806b500 0x7f29b806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.217+0000 7f29d3400640 1 -- 172.21.15.86:0/1740119184 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f29cc08a3b0 con 0x7f29c0004800 2024-03-06T07:30:06.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.217+0000 7f29d2000640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f29b806b500 0x7f29b806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.218+0000 7f29d2000640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f29b806b500 0x7f29b806d9c0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f29c4034000 tx=0x7f29c4039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:06.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.226+0000 7f29d3400640 1 -- 172.21.15.86:0/1740119184 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f29cc0591c0 con 0x7f29c0004800 2024-03-06T07:30:06.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.422+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f29dc0adf40 con 0x7f29c0004800 2024-03-06T07:30:06.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.425+0000 7f29d3400640 1 -- 172.21.15.86:0/1740119184 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v58) v1 ==== 74+0+16677 (secure 0 0 0) 0x7f29cc058ac0 con 0x7f29c0004800 2024-03-06T07:30:06.427 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:06.427 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":58,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:47.263155+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:06.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f29b806b500 msgr2=0x7f29b806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f29b806b500 0x7f29b806d9c0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f29c4034000 tx=0x7f29c4039040 comp rx=0 tx=0).stop 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 msgr2=0x7f29dc1f8090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29dc1f8090 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f29cc00d830 tx=0x7f29cc00dcf0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 shutdown_connections 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f29b806b500 0x7f29b806d9c0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29dc140e60 0x7f29dc15adb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29dc1409e0 0x7f29dc15a870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 --2- 172.21.15.86:0/1740119184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f29c0004800 0x7f29dc1f8090 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 >> 172.21.15.86:0/1740119184 conn(0x7f29dc06afc0 msgr2=0x7f29dc15c7c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 shutdown_connections 2024-03-06T07:30:06.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.428+0000 7f29dbe00640 1 -- 172.21.15.86:0/1740119184 wait complete. 2024-03-06T07:30:06.447 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 8, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:29:43.503869+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '57', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 9, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:29:44.707119+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '58', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:30:06.447 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:30:06.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.648+0000 7fa241a00640 1 Processor -- start 2024-03-06T07:30:06.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.648+0000 7fa241a00640 1 -- start start 2024-03-06T07:30:06.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa241a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa23c15fc10 0x7fa23c070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa241a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 0x7fa23c160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa241a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa23c0b46c0 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa241a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa23c0b49a0 con 0x7fa23c15fc10 2024-03-06T07:30:06.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa241a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa23c072c90 con 0x7fa23c15fff0 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa23aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa23c15fc10 0x7fa23c070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa23aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa23c15fc10 0x7fa23c070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49556/0 (socket says 172.21.15.86:49556) 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.650+0000 7fa23aa00640 1 -- 172.21.15.86:0/2879616447 learned_addr learned my addr 172.21.15.86:0/2879616447 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.651+0000 7fa233e00640 1 --2- 172.21.15.86:0/2879616447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 0x7fa23c160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.651+0000 7fa23aa00640 1 -- 172.21.15.86:0/2879616447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 msgr2=0x7fa23c160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.651+0000 7fa23aa00640 1 --2- 172.21.15.86:0/2879616447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 0x7fa23c160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.651+0000 7fa23aa00640 1 -- 172.21.15.86:0/2879616447 >> v1:172.21.15.194:6789/0 conn(0x7fa23c0ce4a0 legacy=0x7fa23c0ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.651+0000 7fa23aa00640 1 -- 172.21.15.86:0/2879616447 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa23c160990 con 0x7fa23c15fc10 2024-03-06T07:30:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.653+0000 7fa23aa00640 1 --2- 172.21.15.86:0/2879616447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c15fc10 0x7fa23c070ff0 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7fa22c009870 tx=0x7fa22c02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d2b4a400d82753e9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.653+0000 7fa23a000640 1 -- 172.21.15.86:0/2879616447 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa22c035020 con 0x7fa23c15fc10 2024-03-06T07:30:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.653+0000 7fa23a000640 1 -- 172.21.15.86:0/2879616447 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa22c031640 con 0x7fa23c15fc10 2024-03-06T07:30:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.653+0000 7fa23a000640 1 -- 172.21.15.86:0/2879616447 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa22c02d040 con 0x7fa23c15fc10 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 -- 172.21.15.86:0/2879616447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c15fc10 msgr2=0x7fa23c070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 --2- 172.21.15.86:0/2879616447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c15fc10 0x7fa23c070ff0 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7fa22c009870 tx=0x7fa22c02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 -- 172.21.15.86:0/2879616447 shutdown_connections 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 --2- 172.21.15.86:0/2879616447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 0x7fa23c160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 --2- 172.21.15.86:0/2879616447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c15fc10 0x7fa23c070ff0 unknown :-1 s=CLOSED pgs=681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 -- 172.21.15.86:0/2879616447 >> 172.21.15.86:0/2879616447 conn(0x7fa23c0b1940 msgr2=0x7fa23c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 -- 172.21.15.86:0/2879616447 shutdown_connections 2024-03-06T07:30:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.654+0000 7fa241a00640 1 -- 172.21.15.86:0/2879616447 wait complete. 2024-03-06T07:30:06.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.657+0000 7fa241a00640 1 Processor -- start 2024-03-06T07:30:06.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.657+0000 7fa241a00640 1 -- start start 2024-03-06T07:30:06.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa241a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c0ce4a0 0x7fa23c14b740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa241a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa23c15fc10 0x7fa23c0ae2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa241a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 0x7fa23c09cbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa241a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa23c072a30 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa241a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa23c0728b0 con 0x7fa23c15fc10 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa241a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa23c15c2e0 con 0x7fa23c15fff0 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c0ce4a0 0x7fa23c14b740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c0ce4a0 0x7fa23c14b740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49564/0 (socket says 172.21.15.86:49564) 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 -- 172.21.15.86:0/453399374 learned_addr learned my addr 172.21.15.86:0/453399374 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 -- 172.21.15.86:0/453399374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 msgr2=0x7fa23c09cbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 0x7fa23c09cbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 -- 172.21.15.86:0/453399374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa23c15fc10 msgr2=0x7fa23c0ae2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa23c15fc10 0x7fa23c0ae2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.659+0000 7fa23aa00640 1 -- 172.21.15.86:0/453399374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa23c0b0ea0 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.660+0000 7fa23aa00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c0ce4a0 0x7fa23c14b740 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7fa22c000c00 tx=0x7fa22c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.660+0000 7fa232a00640 1 -- 172.21.15.86:0/453399374 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa22c040070 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.660+0000 7fa232a00640 1 -- 172.21.15.86:0/453399374 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa22c0399f0 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.660+0000 7fa232a00640 1 -- 172.21.15.86:0/453399374 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa22c035040 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.660+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa23c0b1180 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.660+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa23c15e720 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.662+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa23c04f330 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.662+0000 7fa232a00640 1 -- 172.21.15.86:0/453399374 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa22c04b060 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.663+0000 7fa232a00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa21806b4d0 0x7fa21806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:06.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.663+0000 7fa232a00640 1 -- 172.21.15.86:0/453399374 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa22c0abc70 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.663+0000 7fa233e00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa21806b4d0 0x7fa21806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:06.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.664+0000 7fa233e00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa21806b4d0 0x7fa21806d990 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7fa224006fd0 tx=0x7fa224008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:06.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.672+0000 7fa232a00640 1 -- 172.21.15.86:0/453399374 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa22c02f300 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.931+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fa23c06ef30 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.932+0000 7fa232a00640 1 -- 172.21.15.86:0/453399374 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v58) v1 ==== 93+0+10 (secure 0 0 0) 0x7fa22c07aa50 con 0x7fa23c0ce4a0 2024-03-06T07:30:06.933 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa21806b4d0 msgr2=0x7fa21806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa21806b4d0 0x7fa21806d990 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7fa224006fd0 tx=0x7fa224008040 comp rx=0 tx=0).stop 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c0ce4a0 msgr2=0x7fa23c14b740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c0ce4a0 0x7fa23c14b740 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7fa22c000c00 tx=0x7fa22c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 shutdown_connections 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa21806b4d0 0x7fa21806d990 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa23c15fff0 0x7fa23c09cbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa23c15fc10 0x7fa23c0ae2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 --2- 172.21.15.86:0/453399374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa23c0ce4a0 0x7fa23c14b740 unknown :-1 s=CLOSED pgs=682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:06.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 >> 172.21.15.86:0/453399374 conn(0x7fa23c0b1940 msgr2=0x7fa23c151350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:06.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.934+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 shutdown_connections 2024-03-06T07:30:06.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:06.935+0000 7fa241a00640 1 -- 172.21.15.86:0/453399374 wait complete. 2024-03-06T07:30:06.963 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:30:07.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.169+0000 7f0df2c00640 1 Processor -- start 2024-03-06T07:30:07.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.169+0000 7f0df2c00640 1 -- start start 2024-03-06T07:30:07.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.170+0000 7f0df2c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 0x7f0dec0ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.170+0000 7f0df2c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0dec15c1f0 con 0x7f0dec06da40 2024-03-06T07:30:07.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.170+0000 7f0df2c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0dec06cac0 con 0x7f0dec0aef20 2024-03-06T07:30:07.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.170+0000 7f0df2c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec0cd200 con 0x7f0dec15be10 2024-03-06T07:30:07.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.171+0000 7f0df2200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0dec06da40 0x7f0dec071ab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54980/0 (socket says 172.21.15.86:54980) 2024-03-06T07:30:07.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.171+0000 7f0df2200640 1 -- 172.21.15.86:0/2367067799 learned_addr learned my addr 172.21.15.86:0/2367067799 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:07.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.171+0000 7f0df1800640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 0x7f0dec0ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.171+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2357350676 0 0) 0x7f0dec15c1f0 con 0x7f0dec06da40 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.171+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0dd4003520 con 0x7f0dec06da40 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2761436920 0 0) 0x7f0dd4003520 con 0x7f0dec06da40 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 msgr2=0x7f0dec0ccc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 0x7f0dec0ccc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 >> v1:172.21.15.194:6789/0 conn(0x7f0dec0aef20 legacy=0x7f0dec06d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0dec06cda0 con 0x7f0dec06da40 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f0de4001670 con 0x7f0dec06da40 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 >> v1:172.21.15.86:6789/0 conn(0x7f0dec06da40 legacy=0x7f0dec071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dd4003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dd4004800 0x7f0dd4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 0x7f0dd40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec0cd200 con 0x7f0dd40039e0 2024-03-06T07:30:07.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dd400baa0 con 0x7f0dd4004800 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0dd400bca0 con 0x7f0dec15be10 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0df1800640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dd4003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0df0e00640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dd4004800 0x7f0dd4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0df2200640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 0x7f0dd40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.172+0000 7f0df1800640 1 -- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 msgr2=0x7f0dd40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.173+0000 7f0df1800640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec15be10 0x7f0dd40051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.173+0000 7f0df1800640 1 -- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dd4004800 msgr2=0x7f0dd4004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.173+0000 7f0df1800640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dd4004800 0x7f0dd4004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.173+0000 7f0df1800640 1 -- 172.21.15.86:0/2367067799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0dd4003520 con 0x7f0dd40039e0 2024-03-06T07:30:07.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0df1800640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dd4003e00 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f0de0009f90 tx=0x7f0dd4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e0b03a0ba0a8626f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:07.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de0039070 con 0x7f0dd40039e0 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0de002cbd0 con 0x7f0dd40039e0 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0debe00640 1 -- 172.21.15.86:0/2367067799 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de0009c10 con 0x7f0dd40039e0 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0df2c00640 1 -- 172.21.15.86:0/2367067799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 msgr2=0x7f0dd4003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0df2c00640 1 --2- 172.21.15.86:0/2367067799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dd4003e00 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f0de0009f90 tx=0x7f0dd4005830 comp rx=0 tx=0).stop 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0df2200640 1 -- 172.21.15.86:0/2367067799 reap_dead start 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0df2c00640 1 -- 172.21.15.86:0/2367067799 shutdown_connections 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.174+0000 7f0df2c00640 1 -- 172.21.15.86:0/2367067799 >> 172.21.15.86:0/2367067799 conn(0x7f0dec06afc0 msgr2=0x7f0dec06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.175+0000 7f0df2c00640 1 -- 172.21.15.86:0/2367067799 shutdown_connections 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.175+0000 7f0df2c00640 1 -- 172.21.15.86:0/2367067799 wait complete. 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.176+0000 7f0df2c00640 1 Processor -- start 2024-03-06T07:30:07.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.176+0000 7f0df2c00640 1 -- start start 2024-03-06T07:30:07.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.177+0000 7f0df2c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dec1540e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.177+0000 7f0df2c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec1559c0 0x7f0dec15c620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.177+0000 7f0df2c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dec15d570 0x7f0dec0ceb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.177+0000 7f0df2c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec0cda90 con 0x7f0dd40039e0 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.177+0000 7f0df2c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec06d080 con 0x7f0dec15d570 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.177+0000 7f0df2c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec0ccc20 con 0x7f0dec1559c0 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.177+0000 7f0df1800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dec1540e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dec1540e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49588/0 (socket says 172.21.15.86:49588) 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 -- 172.21.15.86:0/2930949959 learned_addr learned my addr 172.21.15.86:0/2930949959 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 -- 172.21.15.86:0/2930949959 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec1559c0 msgr2=0x7f0dec15c620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec1559c0 0x7f0dec15c620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 -- 172.21.15.86:0/2930949959 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dec15d570 msgr2=0x7f0dec0ceb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df2200640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dec15d570 0x7f0dec0ceb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dec15d570 0x7f0dec0ceb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 -- 172.21.15.86:0/2930949959 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0dec1546c0 con 0x7f0dd40039e0 2024-03-06T07:30:07.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.178+0000 7f0df1800640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dec1540e0 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f0de002c640 tx=0x7f0de0009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:07.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.179+0000 7f0de9600640 1 -- 172.21.15.86:0/2930949959 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de0042020 con 0x7f0dd40039e0 2024-03-06T07:30:07.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.179+0000 7f0de9600640 1 -- 172.21.15.86:0/2930949959 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0de003a040 con 0x7f0dd40039e0 2024-03-06T07:30:07.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.179+0000 7f0de9600640 1 -- 172.21.15.86:0/2930949959 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de0039070 con 0x7f0dd40039e0 2024-03-06T07:30:07.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.179+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0dec1549a0 con 0x7f0dd40039e0 2024-03-06T07:30:07.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.179+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0dec154f60 con 0x7f0dd40039e0 2024-03-06T07:30:07.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.180+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0dec04f330 con 0x7f0dd40039e0 2024-03-06T07:30:07.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.181+0000 7f0de9600640 1 -- 172.21.15.86:0/2930949959 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0de0005d10 con 0x7f0dd40039e0 2024-03-06T07:30:07.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.182+0000 7f0de9600640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dcc06b500 0x7f0dcc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.182+0000 7f0de9600640 1 -- 172.21.15.86:0/2930949959 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0de00ac710 con 0x7f0dd40039e0 2024-03-06T07:30:07.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.182+0000 7f0df0e00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dcc06b500 0x7f0dcc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.183+0000 7f0df0e00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dcc06b500 0x7f0dcc06d9c0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f0dd8007fd0 tx=0x7f0dd8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:07.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.186+0000 7f0de9600640 1 -- 172.21.15.86:0/2930949959 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0de007b570 con 0x7f0dd40039e0 2024-03-06T07:30:07.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.381+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f0dec0ce830 con 0x7f0dd40039e0 2024-03-06T07:30:07.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.382+0000 7f0de9600640 1 -- 172.21.15.86:0/2930949959 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v58) v1 ==== 104+0+11 (secure 0 0 0) 0x7f0de007ae70 con 0x7f0dd40039e0 2024-03-06T07:30:07.384 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dcc06b500 msgr2=0x7f0dcc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dcc06b500 0x7f0dcc06d9c0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f0dd8007fd0 tx=0x7f0dd8009040 comp rx=0 tx=0).stop 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 msgr2=0x7f0dec1540e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dec1540e0 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f0de002c640 tx=0x7f0de0009750 comp rx=0 tx=0).stop 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 shutdown_connections 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dcc06b500 0x7f0dcc06d9c0 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0dec15d570 0x7f0dec0ceb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0dec1559c0 0x7f0dec15c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 --2- 172.21.15.86:0/2930949959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0dd40039e0 0x7f0dec1540e0 unknown :-1 s=CLOSED pgs=685 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 >> 172.21.15.86:0/2930949959 conn(0x7f0dec06afc0 msgr2=0x7f0dec0b0bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 shutdown_connections 2024-03-06T07:30:07.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.385+0000 7f0df2c00640 1 -- 172.21.15.86:0/2930949959 wait complete. 2024-03-06T07:30:07.411 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:30:07.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.619+0000 7f0f0c800640 1 Processor -- start 2024-03-06T07:30:07.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.620+0000 7f0f0c800640 1 -- start start 2024-03-06T07:30:07.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.622+0000 7f0f0c800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0f04163320 0x7f0f04165710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.622+0000 7f0f0c800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0f040ce690 con 0x7f0f040adf70 2024-03-06T07:30:07.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.622+0000 7f0f0c800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0f040ce970 con 0x7f0f040b3530 2024-03-06T07:30:07.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.622+0000 7f0f0c800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0f040cec50 con 0x7f0f04163320 2024-03-06T07:30:07.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.622+0000 7f0f09400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0f040adf70 0x7f0f040b3910 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54990/0 (socket says 172.21.15.86:54990) 2024-03-06T07:30:07.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.622+0000 7f0f09400640 1 -- 172.21.15.86:0/2550425151 learned_addr learned my addr 172.21.15.86:0/2550425151 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:07.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.622+0000 7f0f0a800640 1 --2- 172.21.15.86:0/2550425151 >> v2:172.21.15.86:3300/0 conn(0x7f0f04163320 0x7f0f04165710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.623+0000 7f0f08a00640 1 -- 172.21.15.86:0/2550425151 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 178575553 0 0) 0x7f0f040ce690 con 0x7f0f040adf70 2024-03-06T07:30:07.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.623+0000 7f0f0a800640 1 -- 172.21.15.86:0/2550425151 >> v1:172.21.15.194:6789/0 conn(0x7f0f040b3530 legacy=0x7f0f0415b780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.623+0000 7f0f0a800640 1 -- 172.21.15.86:0/2550425151 >> v1:172.21.15.86:6789/0 conn(0x7f0f040adf70 legacy=0x7f0f040b3910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.623+0000 7f0f0a800640 1 -- 172.21.15.86:0/2550425151 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0f040cef30 con 0x7f0f04163320 2024-03-06T07:30:07.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.624+0000 7f0f0a800640 1 --2- 172.21.15.86:0/2550425151 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f04165710 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f0f00009d90 tx=0x7f0f0002bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9bdde275190801e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:07.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.624+0000 7f0f08a00640 1 -- 172.21.15.86:0/2550425151 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f00039070 con 0x7f0f04163320 2024-03-06T07:30:07.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.624+0000 7f0f08a00640 1 -- 172.21.15.86:0/2550425151 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0f0002cd00 con 0x7f0f04163320 2024-03-06T07:30:07.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.624+0000 7f0f08a00640 1 -- 172.21.15.86:0/2550425151 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f00034660 con 0x7f0f04163320 2024-03-06T07:30:07.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.625+0000 7f0f0c800640 1 -- 172.21.15.86:0/2550425151 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 msgr2=0x7f0f04165710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.625+0000 7f0f0c800640 1 --2- 172.21.15.86:0/2550425151 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f04165710 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f0f00009d90 tx=0x7f0f0002bf50 comp rx=0 tx=0).stop 2024-03-06T07:30:07.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.625+0000 7f0f0c800640 1 -- 172.21.15.86:0/2550425151 shutdown_connections 2024-03-06T07:30:07.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.625+0000 7f0f0c800640 1 --2- 172.21.15.86:0/2550425151 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f04165710 unknown :-1 s=CLOSED pgs=687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.625+0000 7f0f0c800640 1 -- 172.21.15.86:0/2550425151 >> 172.21.15.86:0/2550425151 conn(0x7f0f040b1940 msgr2=0x7f0f040b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:07.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.625+0000 7f0f0c800640 1 -- 172.21.15.86:0/2550425151 shutdown_connections 2024-03-06T07:30:07.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.625+0000 7f0f0c800640 1 -- 172.21.15.86:0/2550425151 wait complete. 2024-03-06T07:30:07.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.627+0000 7f0f0c800640 1 Processor -- start 2024-03-06T07:30:07.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.627+0000 7f0f0c800640 1 -- start start 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0c800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0f040adf70 0x7f0f0409b2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0c800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0f040b3530 0x7f0f041420a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0c800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f0409ccc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0c800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f0415abb0 con 0x7f0f04163320 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0c800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f0415aa30 con 0x7f0f040b3530 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0c800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0f040af120 con 0x7f0f040adf70 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f0409ccc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f0409ccc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49616/0 (socket says 172.21.15.86:49616) 2024-03-06T07:30:07.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.628+0000 7f0f0a800640 1 -- 172.21.15.86:0/1431156163 learned_addr learned my addr 172.21.15.86:0/1431156163 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:07.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f0a800640 1 -- 172.21.15.86:0/1431156163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0f040adf70 msgr2=0x7f0f0409b2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f0a800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0f040adf70 0x7f0f0409b2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f0a800640 1 -- 172.21.15.86:0/1431156163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0f040b3530 msgr2=0x7f0f041420a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f09400640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0f040b3530 0x7f0f041420a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f0a800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0f040b3530 0x7f0f041420a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f0a800640 1 -- 172.21.15.86:0/1431156163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0f04144fd0 con 0x7f0f04163320 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f0a800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f0409ccc0 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f0f00009d90 tx=0x7f0f00004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0efe000640 1 -- 172.21.15.86:0/1431156163 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f00044020 con 0x7f0f04163320 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0efe000640 1 -- 172.21.15.86:0/1431156163 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0f0003f470 con 0x7f0f04163320 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.629+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0f0406ca30 con 0x7f0f04163320 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.630+0000 7f0efe000640 1 -- 172.21.15.86:0/1431156163 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f00039040 con 0x7f0f04163320 2024-03-06T07:30:07.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.630+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0f0406d2d0 con 0x7f0f04163320 2024-03-06T07:30:07.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.631+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0f0404f330 con 0x7f0f04163320 2024-03-06T07:30:07.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.632+0000 7f0efe000640 1 -- 172.21.15.86:0/1431156163 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0f00047020 con 0x7f0f04163320 2024-03-06T07:30:07.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.633+0000 7f0efe000640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ee806b4d0 0x7f0ee806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:07.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.633+0000 7f0efe000640 1 -- 172.21.15.86:0/1431156163 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0f000acea0 con 0x7f0f04163320 2024-03-06T07:30:07.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.633+0000 7f0f09e00640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ee806b4d0 0x7f0ee806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:07.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.634+0000 7f0f09e00640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ee806b4d0 0x7f0ee806d990 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f0ef80015d0 tx=0x7f0ef8007950 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:07.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.642+0000 7f0efe000640 1 -- 172.21.15.86:0/1431156163 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0f0004e030 con 0x7f0f04163320 2024-03-06T07:30:07.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.837+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f0f04156b60 con 0x7f0f04163320 2024-03-06T07:30:07.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.838+0000 7f0efe000640 1 -- 172.21.15.86:0/1431156163 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v58) v1 ==== 100+0+11 (secure 0 0 0) 0x7f0f0007bd20 con 0x7f0f04163320 2024-03-06T07:30:07.840 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:30:07.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ee806b4d0 msgr2=0x7f0ee806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ee806b4d0 0x7f0ee806d990 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f0ef80015d0 tx=0x7f0ef8007950 comp rx=0 tx=0).stop 2024-03-06T07:30:07.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 msgr2=0x7f0f0409ccc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f0409ccc0 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f0f00009d90 tx=0x7f0f00004a40 comp rx=0 tx=0).stop 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 shutdown_connections 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ee806b4d0 0x7f0ee806d990 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0f04163320 0x7f0f0409ccc0 unknown :-1 s=CLOSED pgs=688 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0f040b3530 0x7f0f041420a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 --2- 172.21.15.86:0/1431156163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0f040adf70 0x7f0f0409b2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 >> 172.21.15.86:0/1431156163 conn(0x7f0f040b1940 msgr2=0x7f0f040cda90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 shutdown_connections 2024-03-06T07:30:07.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:07.841+0000 7f0f0c800640 1 -- 172.21.15.86:0/1431156163 wait complete. 2024-03-06T07:30:07.862 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:30:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.101+0000 7fbff2400640 1 Processor -- start 2024-03-06T07:30:08.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.101+0000 7fbff2400640 1 -- start start 2024-03-06T07:30:08.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.102+0000 7fbff2400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec156320 0x7fbfec15f8f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff2400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbfec06afc0 0x7fbfec06b450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff2400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbfec15fed0 con 0x7fbfec15cd30 2024-03-06T07:30:08.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff2400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbfec071dc0 con 0x7fbfec06afc0 2024-03-06T07:30:08.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff2400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbfec1601b0 con 0x7fbfec156320 2024-03-06T07:30:08.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff1a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbfec06afc0 0x7fbfec06b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec156320 0x7fbfec15f8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff1a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbfec06afc0 0x7fbfec06b450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:57230/0 (socket says 172.21.15.86:57230) 2024-03-06T07:30:08.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.103+0000 7fbff1a00640 1 -- 172.21.15.86:0/4191280054 learned_addr learned my addr 172.21.15.86:0/4191280054 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:08.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.104+0000 7fbff1a00640 1 -- 172.21.15.86:0/4191280054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec156320 msgr2=0x7fbfec15f8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:08.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.104+0000 7fbff1a00640 1 --2- 172.21.15.86:0/4191280054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec156320 0x7fbfec15f8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.104+0000 7fbff1a00640 1 -- 172.21.15.86:0/4191280054 >> v1:172.21.15.194:6789/0 conn(0x7fbfec15cd30 legacy=0x7fbfec0ce8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.104+0000 7fbff1a00640 1 -- 172.21.15.86:0/4191280054 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbfec160490 con 0x7fbfec06afc0 2024-03-06T07:30:08.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.105+0000 7fbff1a00640 1 --2- 172.21.15.86:0/4191280054 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec06afc0 0x7fbfec06b450 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fbfe4009e50 tx=0x7fbfe402bed0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=33f379e09bbfd4fb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:08.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.106+0000 7fbfeb400640 1 -- 172.21.15.86:0/4191280054 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfe402ca70 con 0x7fbfec06afc0 2024-03-06T07:30:08.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.106+0000 7fbfeb400640 1 -- 172.21.15.86:0/4191280054 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbfe402cc10 con 0x7fbfec06afc0 2024-03-06T07:30:08.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.106+0000 7fbfeb400640 1 -- 172.21.15.86:0/4191280054 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfe40343f0 con 0x7fbfec06afc0 2024-03-06T07:30:08.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.106+0000 7fbff2400640 1 -- 172.21.15.86:0/4191280054 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec06afc0 msgr2=0x7fbfec06b450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.106+0000 7fbff2400640 1 --2- 172.21.15.86:0/4191280054 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec06afc0 0x7fbfec06b450 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fbfe4009e50 tx=0x7fbfe402bed0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.106+0000 7fbff2400640 1 -- 172.21.15.86:0/4191280054 shutdown_connections 2024-03-06T07:30:08.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.107+0000 7fbff2400640 1 --2- 172.21.15.86:0/4191280054 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec06afc0 0x7fbfec06b450 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.107+0000 7fbff2400640 1 --2- 172.21.15.86:0/4191280054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec156320 0x7fbfec15f8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.107+0000 7fbff2400640 1 -- 172.21.15.86:0/4191280054 >> 172.21.15.86:0/4191280054 conn(0x7fbfec06c0d0 msgr2=0x7fbfec06c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:08.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.107+0000 7fbff2400640 1 -- 172.21.15.86:0/4191280054 shutdown_connections 2024-03-06T07:30:08.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.107+0000 7fbff2400640 1 -- 172.21.15.86:0/4191280054 wait complete. 2024-03-06T07:30:08.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.108+0000 7fbff2400640 1 Processor -- start 2024-03-06T07:30:08.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.108+0000 7fbff2400640 1 -- start start 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.109+0000 7fbff2400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec06afc0 0x7fbfec140580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.109+0000 7fbff2400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfec156320 0x7fbfec1442c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.109+0000 7fbff2400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec15cd30 0x7fbfec0c58d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.109+0000 7fbff2400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbfec15d1b0 con 0x7fbfec156320 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.109+0000 7fbff2400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbfec156700 con 0x7fbfec15cd30 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.109+0000 7fbff2400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbfec072250 con 0x7fbfec06afc0 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfec156320 0x7fbfec1442c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfec156320 0x7fbfec1442c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49628/0 (socket says 172.21.15.86:49628) 2024-03-06T07:30:08.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 -- 172.21.15.86:0/1266725393 learned_addr learned my addr 172.21.15.86:0/1266725393 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbff1a00640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec15cd30 0x7fbfec0c58d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbff1000640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec06afc0 0x7fbfec140580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 -- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec06afc0 msgr2=0x7fbfec140580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec06afc0 0x7fbfec140580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 -- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec15cd30 msgr2=0x7fbfec0c58d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec15cd30 0x7fbfec0c58d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 -- 172.21.15.86:0/1266725393 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbfec151140 con 0x7fbfec156320 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbff1000640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec06afc0 0x7fbfec140580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbff1a00640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec15cd30 0x7fbfec0c58d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.110+0000 7fbfebe00640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfec156320 0x7fbfec1442c0 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7fbfd800c400 tx=0x7fbfd800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.111+0000 7fbfe8c00640 1 -- 172.21.15.86:0/1266725393 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfd8017070 con 0x7fbfec156320 2024-03-06T07:30:08.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.111+0000 7fbfe8c00640 1 -- 172.21.15.86:0/1266725393 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbfd80052d0 con 0x7fbfec156320 2024-03-06T07:30:08.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.111+0000 7fbfe8c00640 1 -- 172.21.15.86:0/1266725393 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfd8003830 con 0x7fbfec156320 2024-03-06T07:30:08.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.111+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbfec159ba0 con 0x7fbfec156320 2024-03-06T07:30:08.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.111+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbfec15a160 con 0x7fbfec156320 2024-03-06T07:30:08.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.112+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbfec04f330 con 0x7fbfec156320 2024-03-06T07:30:08.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.113+0000 7fbfe8c00640 1 -- 172.21.15.86:0/1266725393 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbfd801a070 con 0x7fbfec156320 2024-03-06T07:30:08.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.114+0000 7fbfe8c00640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbfcc06b500 0x7fbfcc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.114+0000 7fbfe8c00640 1 -- 172.21.15.86:0/1266725393 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbfd80894f0 con 0x7fbfec156320 2024-03-06T07:30:08.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.114+0000 7fbff1000640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbfcc06b500 0x7fbfcc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.115+0000 7fbff1000640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbfcc06b500 0x7fbfcc06d9c0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fbfe0009ac0 tx=0x7fbfe0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:08.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.122+0000 7fbfe8c00640 1 -- 172.21.15.86:0/1266725393 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbfd8058360 con 0x7fbfec156320 2024-03-06T07:30:08.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.332+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fbfec0cd3f0 con 0x7fbfec156320 2024-03-06T07:30:08.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.332+0000 7fbfe8c00640 1 -- 172.21.15.86:0/1266725393 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v58) v1 ==== 80+0+0 (secure 0 0 0) 0x7fbfd8057c60 con 0x7fbfec156320 2024-03-06T07:30:08.334 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbfcc06b500 msgr2=0x7fbfcc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbfcc06b500 0x7fbfcc06d9c0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fbfe0009ac0 tx=0x7fbfe0008040 comp rx=0 tx=0).stop 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfec156320 msgr2=0x7fbfec1442c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfec156320 0x7fbfec1442c0 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7fbfd800c400 tx=0x7fbfd800c8c0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 shutdown_connections 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbfcc06b500 0x7fbfcc06d9c0 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfec15cd30 0x7fbfec0c58d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfec156320 0x7fbfec1442c0 unknown :-1 s=CLOSED pgs=689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 --2- 172.21.15.86:0/1266725393 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfec06afc0 0x7fbfec140580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 >> 172.21.15.86:0/1266725393 conn(0x7fbfec06c0d0 msgr2=0x7fbfec154270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:08.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 shutdown_connections 2024-03-06T07:30:08.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.336+0000 7fbff2400640 1 -- 172.21.15.86:0/1266725393 wait complete. 2024-03-06T07:30:08.363 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:30:08.363 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:08.363 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:08.552 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:30:08.552 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:08.553 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:30:08.590 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:08.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.806+0000 7f5dff800640 1 Processor -- start 2024-03-06T07:30:08.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.806+0000 7f5dff800640 1 -- start start 2024-03-06T07:30:08.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.807+0000 7f5dff800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5df8144c00 0x7f5df806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.807+0000 7f5dff800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5df815f9d0 0x7f5df815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.807+0000 7f5dff800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df80ce0a0 0x7f5df80ce4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.807+0000 7f5dff800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5df806ee70 con 0x7f5df8144c00 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.807+0000 7f5dff800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5df806cef0 con 0x7f5df815f9d0 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.807+0000 7f5dff800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5df806d1d0 con 0x7f5df80ce0a0 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5df8144c00 0x7f5df806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5df8144c00 0x7f5df806e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49638/0 (socket says 172.21.15.86:49638) 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 -- 172.21.15.86:0/2013700048 learned_addr learned my addr 172.21.15.86:0/2013700048 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5defe00640 1 --2- 172.21.15.86:0/2013700048 >> v2:172.21.15.194:3300/0 conn(0x7f5df815f9d0 0x7f5df815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 -- 172.21.15.86:0/2013700048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df80ce0a0 msgr2=0x7f5df80ce4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 --2- 172.21.15.86:0/2013700048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df80ce0a0 0x7f5df80ce4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 -- 172.21.15.86:0/2013700048 >> v2:172.21.15.194:3300/0 conn(0x7f5df815f9d0 msgr2=0x7f5df815fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 --2- 172.21.15.86:0/2013700048 >> v2:172.21.15.194:3300/0 conn(0x7f5df815f9d0 0x7f5df815fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5dfce00640 1 -- 172.21.15.86:0/2013700048 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5df806d4b0 con 0x7f5df8144c00 2024-03-06T07:30:08.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.808+0000 7f5defe00640 1 --2- 172.21.15.86:0/2013700048 >> v2:172.21.15.194:3300/0 conn(0x7f5df815f9d0 0x7f5df815fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.809+0000 7f5dfce00640 1 --2- 172.21.15.86:0/2013700048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df806e930 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7f5df80b26f0 tx=0x7f5de802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71c0d65b8f7085e0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.809+0000 7f5df7e00640 1 -- 172.21.15.86:0/2013700048 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de8035020 con 0x7f5df8144c00 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.809+0000 7f5df7e00640 1 -- 172.21.15.86:0/2013700048 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5de8031870 con 0x7f5df8144c00 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.809+0000 7f5df7e00640 1 -- 172.21.15.86:0/2013700048 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de8031c40 con 0x7f5df8144c00 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 -- 172.21.15.86:0/2013700048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 msgr2=0x7f5df806e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 --2- 172.21.15.86:0/2013700048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df806e930 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7f5df80b26f0 tx=0x7f5de802b910 comp rx=0 tx=0).stop 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 -- 172.21.15.86:0/2013700048 shutdown_connections 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 --2- 172.21.15.86:0/2013700048 >> v2:172.21.15.194:3300/0 conn(0x7f5df815f9d0 0x7f5df815fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 --2- 172.21.15.86:0/2013700048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df80ce0a0 0x7f5df80ce4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 --2- 172.21.15.86:0/2013700048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df806e930 unknown :-1 s=CLOSED pgs=690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 -- 172.21.15.86:0/2013700048 >> 172.21.15.86:0/2013700048 conn(0x7f5df80b1730 msgr2=0x7f5df80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 -- 172.21.15.86:0/2013700048 shutdown_connections 2024-03-06T07:30:08.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.810+0000 7f5dff800640 1 -- 172.21.15.86:0/2013700048 wait complete. 2024-03-06T07:30:08.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.812+0000 7f5dff800640 1 Processor -- start 2024-03-06T07:30:08.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.812+0000 7f5dff800640 1 -- start start 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5dff800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5df80ce0a0 0x7f5df80aefd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5dff800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df81f7030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5dff800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df815f9d0 0x7f5df813c470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5dff800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5df806cb50 con 0x7f5df8144c00 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5dff800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5df8160730 con 0x7f5df80ce0a0 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5dff800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5df806ccd0 con 0x7f5df815f9d0 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5defe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df81f7030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5defe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df81f7030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49640/0 (socket says 172.21.15.86:49640) 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5defe00640 1 -- 172.21.15.86:0/133107987 learned_addr learned my addr 172.21.15.86:0/133107987 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:08.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.813+0000 7f5dfce00640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5df80ce0a0 0x7f5df80aefd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5dfd800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df815f9d0 0x7f5df813c470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5defe00640 1 -- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df815f9d0 msgr2=0x7f5df813c470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5defe00640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df815f9d0 0x7f5df813c470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5defe00640 1 -- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5df80ce0a0 msgr2=0x7f5df80aefd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5defe00640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5df80ce0a0 0x7f5df80aefd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5defe00640 1 -- 172.21.15.86:0/133107987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5df813fab0 con 0x7f5df8144c00 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5dfd800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df815f9d0 0x7f5df813c470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5defe00640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df81f7030 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7f5de000c820 tx=0x7f5de000cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5df5600640 1 -- 172.21.15.86:0/133107987 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de0015070 con 0x7f5df8144c00 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5df5600640 1 -- 172.21.15.86:0/133107987 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5de0007a30 con 0x7f5df8144c00 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5df5600640 1 -- 172.21.15.86:0/133107987 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de0007e00 con 0x7f5df8144c00 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5df813fd90 con 0x7f5df8144c00 2024-03-06T07:30:08.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.814+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5df8153ec0 con 0x7f5df8144c00 2024-03-06T07:30:08.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.816+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5df804f330 con 0x7f5df8144c00 2024-03-06T07:30:08.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.817+0000 7f5df5600640 1 -- 172.21.15.86:0/133107987 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5de0018070 con 0x7f5df8144c00 2024-03-06T07:30:08.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.818+0000 7f5df5600640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5dd406b4d0 0x7f5dd406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:08.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.818+0000 7f5df5600640 1 -- 172.21.15.86:0/133107987 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5de008a1e0 con 0x7f5df8144c00 2024-03-06T07:30:08.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.818+0000 7f5dfce00640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5dd406b4d0 0x7f5dd406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:08.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.819+0000 7f5dfce00640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5dd406b4d0 0x7f5dd406d990 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f5de8036000 tx=0x7f5de803c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:08.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:08.822+0000 7f5df5600640 1 -- 172.21.15.86:0/133107987 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5de0059060 con 0x7f5df8144c00 2024-03-06T07:30:09.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.050+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5df80afa20 con 0x7f5df8144c00 2024-03-06T07:30:09.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.051+0000 7f5df5600640 1 -- 172.21.15.86:0/133107987 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f5de0058960 con 0x7f5df8144c00 2024-03-06T07:30:09.053 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:09.053 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:30:09.053 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5dd406b4d0 msgr2=0x7f5dd406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5dd406b4d0 0x7f5dd406d990 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f5de8036000 tx=0x7f5de803c040 comp rx=0 tx=0).stop 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 msgr2=0x7f5df81f7030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df81f7030 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7f5de000c820 tx=0x7f5de000cc20 comp rx=0 tx=0).stop 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 shutdown_connections 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5dd406b4d0 0x7f5dd406d990 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5df815f9d0 0x7f5df813c470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5df8144c00 0x7f5df81f7030 unknown :-1 s=CLOSED pgs=691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 --2- 172.21.15.86:0/133107987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5df80ce0a0 0x7f5df80aefd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 >> 172.21.15.86:0/133107987 conn(0x7f5df80b1730 msgr2=0x7f5df8071790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:09.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 shutdown_connections 2024-03-06T07:30:09.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.054+0000 7f5dff800640 1 -- 172.21.15.86:0/133107987 wait complete. 2024-03-06T07:30:09.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json dirfrag split /subdir 0/0 1 2024-03-06T07:30:09.293 DEBUG:tasks.cephfs.filesystem:_json_asok output empty 2024-03-06T07:30:09.293 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:09.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.485+0000 7f7e90200640 1 Processor -- start 2024-03-06T07:30:09.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.485+0000 7f7e90200640 1 -- start start 2024-03-06T07:30:09.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e90200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7e8815fe60 0x7f7e88071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:09.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e90200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e8806cef0 con 0x7f7e8815f9d0 2024-03-06T07:30:09.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e90200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e8806d1d0 con 0x7f7e88144c00 2024-03-06T07:30:09.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e90200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7e8806d4b0 con 0x7f7e8815fe60 2024-03-06T07:30:09.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e8ce00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7e8815f9d0 0x7f7e88070b10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55004/0 (socket says 172.21.15.86:55004) 2024-03-06T07:30:09.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e8ce00640 1 -- 172.21.15.86:0/1153908785 learned_addr learned my addr 172.21.15.86:0/1153908785 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:09.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e8e200640 1 --2- 172.21.15.86:0/1153908785 >> v2:172.21.15.86:3300/0 conn(0x7f7e8815fe60 0x7f7e88071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:09.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.487+0000 7f7e87e00640 1 -- 172.21.15.86:0/1153908785 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3050770469 0 0) 0x7f7e8806cef0 con 0x7f7e8815f9d0 2024-03-06T07:30:09.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.488+0000 7f7e87e00640 1 -- 172.21.15.86:0/1153908785 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7e70003520 con 0x7f7e8815f9d0 2024-03-06T07:30:09.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.488+0000 7f7e8e200640 1 -- 172.21.15.86:0/1153908785 >> v1:172.21.15.194:6789/0 conn(0x7f7e88144c00 legacy=0x7f7e8806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:09.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.488+0000 7f7e8e200640 1 -- 172.21.15.86:0/1153908785 >> v1:172.21.15.86:6789/0 conn(0x7f7e8815f9d0 legacy=0x7f7e88070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.488+0000 7f7e8e200640 1 -- 172.21.15.86:0/1153908785 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e8806d790 con 0x7f7e8815fe60 2024-03-06T07:30:09.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.489+0000 7f7e8e200640 1 --2- 172.21.15.86:0/1153908785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815fe60 0x7f7e88071220 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f7e80009d90 tx=0x7f7e8002bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6938747e4e20d58 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:09.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.489+0000 7f7e87e00640 1 -- 172.21.15.86:0/1153908785 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e80039070 con 0x7f7e8815fe60 2024-03-06T07:30:09.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.489+0000 7f7e87e00640 1 -- 172.21.15.86:0/1153908785 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e8002cd00 con 0x7f7e8815fe60 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.489+0000 7f7e87e00640 1 -- 172.21.15.86:0/1153908785 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e80034660 con 0x7f7e8815fe60 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.489+0000 7f7e90200640 1 -- 172.21.15.86:0/1153908785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815fe60 msgr2=0x7f7e88071220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.489+0000 7f7e90200640 1 --2- 172.21.15.86:0/1153908785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815fe60 0x7f7e88071220 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f7e80009d90 tx=0x7f7e8002bf50 comp rx=0 tx=0).stop 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.490+0000 7f7e90200640 1 -- 172.21.15.86:0/1153908785 shutdown_connections 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.490+0000 7f7e90200640 1 --2- 172.21.15.86:0/1153908785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815fe60 0x7f7e88071220 unknown :-1 s=CLOSED pgs=693 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.490+0000 7f7e90200640 1 -- 172.21.15.86:0/1153908785 >> 172.21.15.86:0/1153908785 conn(0x7f7e880b1730 msgr2=0x7f7e880b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.490+0000 7f7e90200640 1 -- 172.21.15.86:0/1153908785 shutdown_connections 2024-03-06T07:30:09.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.490+0000 7f7e90200640 1 -- 172.21.15.86:0/1153908785 wait complete. 2024-03-06T07:30:09.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.491+0000 7f7e90200640 1 Processor -- start 2024-03-06T07:30:09.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.491+0000 7f7e90200640 1 -- start start 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.492+0000 7f7e90200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88144c00 0x7f7e88153880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.492+0000 7f7e90200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815f9d0 0x7f7e8813be40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.492+0000 7f7e90200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e8815fe60 0x7f7e8813c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.492+0000 7f7e90200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e8806cb50 con 0x7f7e8815f9d0 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.492+0000 7f7e90200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e88160730 con 0x7f7e88144c00 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.492+0000 7f7e90200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e8806ccd0 con 0x7f7e8815fe60 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815f9d0 0x7f7e8813be40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815f9d0 0x7f7e8813be40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49656/0 (socket says 172.21.15.86:49656) 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 -- 172.21.15.86:0/1492779922 learned_addr learned my addr 172.21.15.86:0/1492779922 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 -- 172.21.15.86:0/1492779922 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e8815fe60 msgr2=0x7f7e8813c380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e8815fe60 0x7f7e8813c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 -- 172.21.15.86:0/1492779922 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88144c00 msgr2=0x7f7e88153880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8d800640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88144c00 0x7f7e88153880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:09.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88144c00 0x7f7e88153880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 -- 172.21.15.86:0/1492779922 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e8815d600 con 0x7f7e8815f9d0 2024-03-06T07:30:09.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.493+0000 7f7e8ce00640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815f9d0 0x7f7e8813be40 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f7e7400c500 tx=0x7f7e7400c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:09.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.495+0000 7f7e85600640 1 -- 172.21.15.86:0/1492779922 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e74017070 con 0x7f7e8815f9d0 2024-03-06T07:30:09.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.495+0000 7f7e85600640 1 -- 172.21.15.86:0/1492779922 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e74005ce0 con 0x7f7e8815f9d0 2024-03-06T07:30:09.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.495+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7e8813c9b0 con 0x7f7e8815f9d0 2024-03-06T07:30:09.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.495+0000 7f7e85600640 1 -- 172.21.15.86:0/1492779922 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e74007330 con 0x7f7e8815f9d0 2024-03-06T07:30:09.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.495+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e8813cf70 con 0x7f7e8815f9d0 2024-03-06T07:30:09.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.497+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7e881458b0 con 0x7f7e8815f9d0 2024-03-06T07:30:09.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.497+0000 7f7e85600640 1 -- 172.21.15.86:0/1492779922 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7e7401a070 con 0x7f7e8815f9d0 2024-03-06T07:30:09.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.499+0000 7f7e85600640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e6806b4d0 0x7f7e6806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:09.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.499+0000 7f7e85600640 1 -- 172.21.15.86:0/1492779922 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7e740896e0 con 0x7f7e8815f9d0 2024-03-06T07:30:09.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.499+0000 7f7e8d800640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e6806b4d0 0x7f7e6806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:09.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.500+0000 7f7e8d800640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e6806b4d0 0x7f7e6806d990 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f7e7c000f60 tx=0x7f7e7c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:09.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.503+0000 7f7e85600640 1 -- 172.21.15.86:0/1492779922 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7e881458b0 con 0x7f7e8815f9d0 2024-03-06T07:30:09.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.780+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7e8804f330 con 0x7f7e8815f9d0 2024-03-06T07:30:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.781+0000 7f7e85600640 1 -- 172.21.15.86:0/1492779922 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f7e74058610 con 0x7f7e8815f9d0 2024-03-06T07:30:09.784 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:09.784 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:30:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e6806b4d0 msgr2=0x7f7e6806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e6806b4d0 0x7f7e6806d990 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f7e7c000f60 tx=0x7f7e7c009040 comp rx=0 tx=0).stop 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815f9d0 msgr2=0x7f7e8813be40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815f9d0 0x7f7e8813be40 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f7e7400c500 tx=0x7f7e7400c9c0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 shutdown_connections 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e6806b4d0 0x7f7e6806d990 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e8815fe60 0x7f7e8813c380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e8815f9d0 0x7f7e8813be40 unknown :-1 s=CLOSED pgs=694 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 --2- 172.21.15.86:0/1492779922 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e88144c00 0x7f7e88153880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 >> 172.21.15.86:0/1492779922 conn(0x7f7e880b1730 msgr2=0x7f7e880aec50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 shutdown_connections 2024-03-06T07:30:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:09.784+0000 7f7e90200640 1 -- 172.21.15.86:0/1492779922 wait complete. 2024-03-06T07:30:09.804 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:30:10.057 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:30:10.057 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:30:10.058 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:30:10.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.235+0000 7f3b9be00640 1 Processor -- start 2024-03-06T07:30:10.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.236+0000 7f3b9be00640 1 -- start start 2024-03-06T07:30:10.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.238+0000 7f3b9be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3b9415f470 0x7f3b940d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.238+0000 7f3b9be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 0x7f3b94160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.238+0000 7f3b9be00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3b940cea60 con 0x7f3b9415e0d0 2024-03-06T07:30:10.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.238+0000 7f3b9be00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3b940d2f60 con 0x7f3b9415f470 2024-03-06T07:30:10.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.238+0000 7f3b9be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b940d1240 con 0x7f3b940d0df0 2024-03-06T07:30:10.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.238+0000 7f3b99400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3b9415e0d0 0x7f3b940d1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55010/0 (socket says 172.21.15.86:55010) 2024-03-06T07:30:10.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.238+0000 7f3b99400640 1 -- 172.21.15.86:0/1257985609 learned_addr learned my addr 172.21.15.86:0/1257985609 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:10.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1404897407 0 0) 0x7f3b940cea60 con 0x7f3b9415e0d0 2024-03-06T07:30:10.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3b7c003520 con 0x7f3b9415e0d0 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b98a00640 1 --2- 172.21.15.86:0/1257985609 >> v2:172.21.15.194:3300/0 conn(0x7f3b9415f470 0x7f3b940d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b99e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 0x7f3b94160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1453128637 0 0) 0x7f3b7c003520 con 0x7f3b9415e0d0 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 msgr2=0x7f3b94160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 0x7f3b94160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 >> v2:172.21.15.194:3300/0 conn(0x7f3b9415f470 msgr2=0x7f3b940d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 --2- 172.21.15.86:0/1257985609 >> v2:172.21.15.194:3300/0 conn(0x7f3b9415f470 0x7f3b940d08b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b940d2350 con 0x7f3b9415e0d0 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3b88001670 con 0x7f3b9415e0d0 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 >> v1:172.21.15.86:6789/0 conn(0x7f3b9415e0d0 legacy=0x7f3b940d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 0x7f3b7c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b7c003fe0 0x7f3b7c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b7c004d40 0x7f3b7c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b940d1240 con 0x7f3b7c004d40 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b7c00bad0 con 0x7f3b7c003fe0 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.239+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b7c00bcd0 con 0x7f3b940d0df0 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b99e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b7c004d40 0x7f3b7c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b98a00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b7c003fe0 0x7f3b7c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b99e00640 1 -- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 msgr2=0x7f3b7c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b99400640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 0x7f3b7c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b99e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b940d0df0 0x7f3b7c0039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b99e00640 1 -- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b7c003fe0 msgr2=0x7f3b7c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b99e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b7c003fe0 0x7f3b7c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b99e00640 1 -- 172.21.15.86:0/1257985609 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b7c003520 con 0x7f3b7c004d40 2024-03-06T07:30:10.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.240+0000 7f3b98a00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b7c003fe0 0x7f3b7c004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.242+0000 7f3b99e00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b7c004d40 0x7f3b7c0051c0 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f3b8c007360 tx=0x7f3b7c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=75a8383a9767f157 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.242+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b8c039070 con 0x7f3b7c004d40 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.242+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b8c02ccd0 con 0x7f3b7c004d40 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.242+0000 7f3b93e00640 1 -- 172.21.15.86:0/1257985609 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b8c009be0 con 0x7f3b7c004d40 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.242+0000 7f3b9be00640 1 -- 172.21.15.86:0/1257985609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b7c004d40 msgr2=0x7f3b7c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.242+0000 7f3b9be00640 1 --2- 172.21.15.86:0/1257985609 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b7c004d40 0x7f3b7c0051c0 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f3b8c007360 tx=0x7f3b7c005830 comp rx=0 tx=0).stop 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.242+0000 7f3b99e00640 1 -- 172.21.15.86:0/1257985609 reap_dead start 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.243+0000 7f3b9be00640 1 -- 172.21.15.86:0/1257985609 shutdown_connections 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.243+0000 7f3b9be00640 1 -- 172.21.15.86:0/1257985609 >> 172.21.15.86:0/1257985609 conn(0x7f3b940d3ec0 msgr2=0x7f3b940d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.243+0000 7f3b9be00640 1 -- 172.21.15.86:0/1257985609 shutdown_connections 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.243+0000 7f3b9be00640 1 -- 172.21.15.86:0/1257985609 wait complete. 2024-03-06T07:30:10.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.244+0000 7f3b9be00640 1 Processor -- start 2024-03-06T07:30:10.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.244+0000 7f3b9be00640 1 -- start start 2024-03-06T07:30:10.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.245+0000 7f3b9be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b94152440 0x7f3b940b8c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.245+0000 7f3b9be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b94162560 0x7f3b940caf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.245+0000 7f3b9be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b940cb4d0 0x7f3b941613a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.245+0000 7f3b9be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b940d2bf0 con 0x7f3b940cb4d0 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.245+0000 7f3b9be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b940d2910 con 0x7f3b94162560 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.245+0000 7f3b9be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b9415fb30 con 0x7f3b94152440 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b940cb4d0 0x7f3b941613a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b940cb4d0 0x7f3b941613a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49664/0 (socket says 172.21.15.86:49664) 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 -- 172.21.15.86:0/3319281319 learned_addr learned my addr 172.21.15.86:0/3319281319 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99400640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b94152440 0x7f3b940b8c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b98a00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b94162560 0x7f3b940caf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 -- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b94152440 msgr2=0x7f3b940b8c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b94152440 0x7f3b940b8c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 -- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b94162560 msgr2=0x7f3b940caf90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b94162560 0x7f3b940caf90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 -- 172.21.15.86:0/3319281319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b940b9500 con 0x7f3b940cb4d0 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b99e00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b940cb4d0 0x7f3b941613a0 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f3b8c007480 tx=0x7f3b8c004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:10.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.246+0000 7f3b91600640 1 -- 172.21.15.86:0/3319281319 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b8c039070 con 0x7f3b940cb4d0 2024-03-06T07:30:10.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.247+0000 7f3b91600640 1 -- 172.21.15.86:0/3319281319 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b8c03a040 con 0x7f3b940cb4d0 2024-03-06T07:30:10.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.247+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b940b97e0 con 0x7f3b940cb4d0 2024-03-06T07:30:10.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.247+0000 7f3b91600640 1 -- 172.21.15.86:0/3319281319 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b8c009be0 con 0x7f3b940cb4d0 2024-03-06T07:30:10.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.247+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b94152820 con 0x7f3b940cb4d0 2024-03-06T07:30:10.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.248+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3b9404f330 con 0x7f3b940cb4d0 2024-03-06T07:30:10.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.249+0000 7f3b91600640 1 -- 172.21.15.86:0/3319281319 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3b8c046030 con 0x7f3b940cb4d0 2024-03-06T07:30:10.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.251+0000 7f3b91600640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b7406b4d0 0x7f3b7406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:10.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.251+0000 7f3b91600640 1 -- 172.21.15.86:0/3319281319 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3b8c0ab800 con 0x7f3b940cb4d0 2024-03-06T07:30:10.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.251+0000 7f3b99400640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b7406b4d0 0x7f3b7406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:10.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.252+0000 7f3b99400640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b7406b4d0 0x7f3b7406d990 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f3b88001eb0 tx=0x7f3b88009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:10.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.255+0000 7f3b91600640 1 -- 172.21.15.86:0/3319281319 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3b8c04d060 con 0x7f3b940cb4d0 2024-03-06T07:30:10.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.581+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f3b9408d920 con 0x7f3b940cb4d0 2024-03-06T07:30:10.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.582+0000 7f3b91600640 1 -- 172.21.15.86:0/3319281319 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v34) v1 ==== 53+0+73 (secure 0 0 0) 0x7f3b8c07a6b0 con 0x7f3b940cb4d0 2024-03-06T07:30:10.584 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b7406b4d0 msgr2=0x7f3b7406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b7406b4d0 0x7f3b7406d990 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f3b88001eb0 tx=0x7f3b88009040 comp rx=0 tx=0).stop 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b940cb4d0 msgr2=0x7f3b941613a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b940cb4d0 0x7f3b941613a0 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f3b8c007480 tx=0x7f3b8c004ab0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 shutdown_connections 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b7406b4d0 0x7f3b7406d990 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b940cb4d0 0x7f3b941613a0 unknown :-1 s=CLOSED pgs=697 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b94162560 0x7f3b940caf90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 --2- 172.21.15.86:0/3319281319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b94152440 0x7f3b940b8c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 >> 172.21.15.86:0/3319281319 conn(0x7f3b940d3ec0 msgr2=0x7f3b940b1e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 shutdown_connections 2024-03-06T07:30:10.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:10.585+0000 7f3b9be00640 1 -- 172.21.15.86:0/3319281319 wait complete. 2024-03-06T07:30:10.612 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:30:10.612 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:30:10.612 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:30:10.612 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:30:10.612 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:10.612 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:30:10.612 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:30:10.613 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:30:10.613 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:30:10.613 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:10.613 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:10.658 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42256sec preferred_lft 42256sec 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:30:10.659 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:30:10.660 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:10.661 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:10.661 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:30:10.781 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:30:10.784 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:10.784 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:30:10.828 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:30:10.828 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:30:10.828 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:30:10.828 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:30:10.828 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:30:10.829 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:10.829 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:30:10.830 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:30:10.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:30:10.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:30:10.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:30:10.830 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:30:11.042 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:11.043 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:30:11.089 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:30:11.089 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:30:11.090 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:30:11.090 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:11.090 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:30:11.167 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:30:11.168 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:11.216 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:30:11.216 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:30:11.216 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:11.260 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:30:11.261 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:11.261 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:30:11.436 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:30:11.436 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:30:11.467 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:30:11.467 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:30:11.467 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:30:11.467 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:30:11.467 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:30:11.469 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:30:11.469 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:30:11.469 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:30:11.469 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:30:11.469 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:30:11.472 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:30:11.472 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:11.473 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:11.556 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:11.556 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:30:11.607 INFO:teuthology.orchestra.run.smithi086.stdout:client5374 2024-03-06T07:30:11.607 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:11.607 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:11.654 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:11.654 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> import uuid 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> n = 75 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = False 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> if 0 > 0: 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = True 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> dpath = os.path.dirname(path) 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> fnameprefix = os.path.basename(path) 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> os.makedirs(dpath, exist_ok=True) 2024-03-06T07:30:11.748 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> try: 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> for i in range(n): 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> with open(fpath, '"'"'w'"'"') as f: 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> f.write(f"{i}") 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> f.flush() 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(f.fileno()) 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> os.unlink(fpath) 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> if create_hard_links: 2024-03-06T07:30:11.749 DEBUG:teuthology.orchestra.run.smithi086:> for j in range(0): 2024-03-06T07:30:11.750 DEBUG:teuthology.orchestra.run.smithi086:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-06T07:30:11.750 DEBUG:teuthology.orchestra.run.smithi086:> if True: 2024-03-06T07:30:11.750 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:30:11.750 DEBUG:teuthology.orchestra.run.smithi086:> finally: 2024-03-06T07:30:11.750 DEBUG:teuthology.orchestra.run.smithi086:> os.close(dirfd) 2024-03-06T07:30:11.750 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:30:12.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:12 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:30:12.822 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:30:12.822 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:30:12.971 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:30:13.104 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:30:13.105 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:13.105 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:13.160 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:13.160 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:13.212 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:13.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.409+0000 7f56c2600640 1 Processor -- start 2024-03-06T07:30:13.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.409+0000 7f56c2600640 1 -- start start 2024-03-06T07:30:13.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.410+0000 7f56c2600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f56bc15eff0 0x7f56bc070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.410+0000 7f56c2600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f56bc071d30 con 0x7f56bc0af1c0 2024-03-06T07:30:13.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.410+0000 7f56c2600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f56bc15f4a0 con 0x7f56bc06e4d0 2024-03-06T07:30:13.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.410+0000 7f56c2600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f56bc071260 con 0x7f56bc15eff0 2024-03-06T07:30:13.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.411+0000 7f56b2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f56bc15eff0 0x7f56bc070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.411+0000 7f56b2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f56bc15eff0 0x7f56bc070d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49674/0 (socket says 172.21.15.86:49674) 2024-03-06T07:30:13.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.411+0000 7f56b2a00640 1 -- 172.21.15.86:0/3999974027 learned_addr learned my addr 172.21.15.86:0/3999974027 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.411+0000 7f56b2a00640 1 -- 172.21.15.86:0/3999974027 >> v1:172.21.15.194:6789/0 conn(0x7f56bc06e4d0 legacy=0x7f56bc0b35d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.411+0000 7f56b2a00640 1 -- 172.21.15.86:0/3999974027 >> v1:172.21.15.86:6789/0 conn(0x7f56bc0af1c0 legacy=0x7f56bc06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.411+0000 7f56b2a00640 1 -- 172.21.15.86:0/3999974027 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56bc071490 con 0x7f56bc15eff0 2024-03-06T07:30:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.413+0000 7f56b2a00640 1 --2- 172.21.15.86:0/3999974027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc15eff0 0x7f56bc070d20 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7f56a40098a0 tx=0x7f56a402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=974f87526f3ebc19 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:13.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.413+0000 7f56baa00640 1 -- 172.21.15.86:0/3999974027 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56a4034070 con 0x7f56bc15eff0 2024-03-06T07:30:13.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.413+0000 7f56baa00640 1 -- 172.21.15.86:0/3999974027 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f56a4030840 con 0x7f56bc15eff0 2024-03-06T07:30:13.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.413+0000 7f56baa00640 1 -- 172.21.15.86:0/3999974027 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56a4030c10 con 0x7f56bc15eff0 2024-03-06T07:30:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.414+0000 7f56c2600640 1 -- 172.21.15.86:0/3999974027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc15eff0 msgr2=0x7f56bc070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.414+0000 7f56c2600640 1 --2- 172.21.15.86:0/3999974027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc15eff0 0x7f56bc070d20 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7f56a40098a0 tx=0x7f56a402b980 comp rx=0 tx=0).stop 2024-03-06T07:30:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.414+0000 7f56c2600640 1 -- 172.21.15.86:0/3999974027 shutdown_connections 2024-03-06T07:30:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.414+0000 7f56c2600640 1 --2- 172.21.15.86:0/3999974027 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc15eff0 0x7f56bc070d20 unknown :-1 s=CLOSED pgs=699 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.414+0000 7f56c2600640 1 -- 172.21.15.86:0/3999974027 >> 172.21.15.86:0/3999974027 conn(0x7f56bc0b1730 msgr2=0x7f56bc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.414+0000 7f56c2600640 1 -- 172.21.15.86:0/3999974027 shutdown_connections 2024-03-06T07:30:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.414+0000 7f56c2600640 1 -- 172.21.15.86:0/3999974027 wait complete. 2024-03-06T07:30:13.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.416+0000 7f56c2600640 1 Processor -- start 2024-03-06T07:30:13.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.416+0000 7f56c2600640 1 -- start start 2024-03-06T07:30:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56c2600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f56bc06e4d0 0x7f56bc14b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56c2600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc0af1c0 0x7f56bc1e67e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56c2600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f56bc15eff0 0x7f56bc0ced70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56c2600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56bc0b30b0 con 0x7f56bc0af1c0 2024-03-06T07:30:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56c2600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56bc071a50 con 0x7f56bc06e4d0 2024-03-06T07:30:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56c2600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f56bc06d370 con 0x7f56bc15eff0 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56b2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc0af1c0 0x7f56bc1e67e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56b2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc0af1c0 0x7f56bc1e67e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49678/0 (socket says 172.21.15.86:49678) 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.417+0000 7f56b2a00640 1 -- 172.21.15.86:0/2954584610 learned_addr learned my addr 172.21.15.86:0/2954584610 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56bbe00640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f56bc15eff0 0x7f56bc0ced70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56bb400640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f56bc06e4d0 0x7f56bc14b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56b2a00640 1 -- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f56bc15eff0 msgr2=0x7f56bc0ced70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56b2a00640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f56bc15eff0 0x7f56bc0ced70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56b2a00640 1 -- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f56bc06e4d0 msgr2=0x7f56bc14b3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56b2a00640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f56bc06e4d0 0x7f56bc14b3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56b2a00640 1 -- 172.21.15.86:0/2954584610 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56bc14b910 con 0x7f56bc0af1c0 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56bbe00640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f56bc15eff0 0x7f56bc0ced70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56bb400640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f56bc06e4d0 0x7f56bc14b3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.418+0000 7f56b2a00640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc0af1c0 0x7f56bc1e67e0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f56a4000c00 tx=0x7f56a40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.419+0000 7f56b3e00640 1 -- 172.21.15.86:0/2954584610 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56a4034040 con 0x7f56bc0af1c0 2024-03-06T07:30:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.419+0000 7f56b3e00640 1 -- 172.21.15.86:0/2954584610 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f56a40443f0 con 0x7f56bc0af1c0 2024-03-06T07:30:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.419+0000 7f56b3e00640 1 -- 172.21.15.86:0/2954584610 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56a40447e0 con 0x7f56bc0af1c0 2024-03-06T07:30:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.419+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56bc0cdcd0 con 0x7f56bc0af1c0 2024-03-06T07:30:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.419+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56bc0ce290 con 0x7f56bc0af1c0 2024-03-06T07:30:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.420+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56bc04f330 con 0x7f56bc0af1c0 2024-03-06T07:30:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.421+0000 7f56b3e00640 1 -- 172.21.15.86:0/2954584610 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f56a404b060 con 0x7f56bc0af1c0 2024-03-06T07:30:13.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.422+0000 7f56b3e00640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f569806b550 0x7f569806da10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.422+0000 7f56b3e00640 1 -- 172.21.15.86:0/2954584610 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f56a40ac4e0 con 0x7f56bc0af1c0 2024-03-06T07:30:13.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.422+0000 7f56bb400640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f569806b550 0x7f569806da10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.423+0000 7f56bb400640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f569806b550 0x7f569806da10 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f56ac001d70 tx=0x7f56ac009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:13.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.427+0000 7f56b3e00640 1 -- 172.21.15.86:0/2954584610 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f56a403f070 con 0x7f56bc0af1c0 2024-03-06T07:30:13.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.624+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f56bc0b1b10 con 0x7f56bc0af1c0 2024-03-06T07:30:13.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.627+0000 7f56b3e00640 1 -- 172.21.15.86:0/2954584610 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v58) v1 ==== 74+0+16677 (secure 0 0 0) 0x7f56bc15f4a0 con 0x7f56bc0af1c0 2024-03-06T07:30:13.630 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:13.630 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":58,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:47.263155+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f569806b550 msgr2=0x7f569806da10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f569806b550 0x7f569806da10 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f56ac001d70 tx=0x7f56ac009040 comp rx=0 tx=0).stop 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc0af1c0 msgr2=0x7f56bc1e67e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc0af1c0 0x7f56bc1e67e0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f56a4000c00 tx=0x7f56a40046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 shutdown_connections 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f569806b550 0x7f569806da10 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f56bc15eff0 0x7f56bc0ced70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f56bc0af1c0 0x7f56bc1e67e0 unknown :-1 s=CLOSED pgs=700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 --2- 172.21.15.86:0/2954584610 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f56bc06e4d0 0x7f56bc14b3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 >> 172.21.15.86:0/2954584610 conn(0x7f56bc0b1730 msgr2=0x7f56bc15ff20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:13.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 shutdown_connections 2024-03-06T07:30:13.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.630+0000 7f56c2600640 1 -- 172.21.15.86:0/2954584610 wait complete. 2024-03-06T07:30:13.652 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 8, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:29:43.503869+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '57', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 9, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:29:44.707119+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '58', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:30:13.652 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:30:13.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.862+0000 7f95f5200640 1 Processor -- start 2024-03-06T07:30:13.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.863+0000 7f95f5200640 1 -- start start 2024-03-06T07:30:13.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.864+0000 7f95f5200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f95f015e3e0 0x7f95f006f130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.864+0000 7f95f5200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f95f00d1260 con 0x7f95f006fef0 2024-03-06T07:30:13.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.864+0000 7f95f5200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f95f006c320 con 0x7f95f015e000 2024-03-06T07:30:13.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.864+0000 7f95f5200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f95f006c0d0 con 0x7f95f015e3e0 2024-03-06T07:30:13.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.864+0000 7f95e5400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f95f006fef0 0x7f95f00ce340 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55030/0 (socket says 172.21.15.86:55030) 2024-03-06T07:30:13.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.864+0000 7f95e5400640 1 -- 172.21.15.86:0/3737664571 learned_addr learned my addr 172.21.15.86:0/3737664571 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:13.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.864+0000 7f95ee800640 1 --2- 172.21.15.86:0/3737664571 >> v2:172.21.15.194:3300/0 conn(0x7f95f015e3e0 0x7f95f006f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3680255815 0 0) 0x7f95f00d1260 con 0x7f95f006fef0 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f95d4003520 con 0x7f95f006fef0 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2161126553 0 0) 0x7f95d4003520 con 0x7f95f006fef0 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 >> v2:172.21.15.194:3300/0 conn(0x7f95f015e3e0 msgr2=0x7f95f006f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 --2- 172.21.15.86:0/3737664571 >> v2:172.21.15.194:3300/0 conn(0x7f95f015e3e0 0x7f95f006f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 >> v1:172.21.15.194:6789/0 conn(0x7f95f015e000 legacy=0x7f95f006f7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95f006c600 con 0x7f95f006fef0 2024-03-06T07:30:13.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f95e0001670 con 0x7f95f006fef0 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 >> v1:172.21.15.86:6789/0 conn(0x7f95f006fef0 legacy=0x7f95f00ce340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95d40039b0 0x7f95d4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95d4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95d40051c0 0x7f95d400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95f006c0d0 con 0x7f95d4004800 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95d400c060 con 0x7f95d40039b0 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.865+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95d400c260 con 0x7f95d40051c0 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95e5400640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95d4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95ede00640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95d40039b0 0x7f95d4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95ee800640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95d40051c0 0x7f95d400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95e5400640 1 -- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95d40051c0 msgr2=0x7f95d400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95e5400640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95d40051c0 0x7f95d400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95e5400640 1 -- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95d40039b0 msgr2=0x7f95d4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:13.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95e5400640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95d40039b0 0x7f95d4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95e5400640 1 -- 172.21.15.86:0/3737664571 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95d4003520 con 0x7f95d4004800 2024-03-06T07:30:13.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95ee800640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95d40051c0 0x7f95d400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:13.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.866+0000 7f95ee800640 1 -- 172.21.15.86:0/3737664571 reap_dead start 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.867+0000 7f95e5400640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95d4004c80 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7f95e000c210 tx=0x7f95d4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b944b4cfb67e1da server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.867+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95e0007d30 con 0x7f95d4004800 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.867+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95e0030070 con 0x7f95d4004800 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.867+0000 7f95ed400640 1 -- 172.21.15.86:0/3737664571 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95e0005d70 con 0x7f95d4004800 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.868+0000 7f95f5200640 1 -- 172.21.15.86:0/3737664571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 msgr2=0x7f95d4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.868+0000 7f95f5200640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95d4004c80 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7f95e000c210 tx=0x7f95d4005830 comp rx=0 tx=0).stop 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.868+0000 7f95f5200640 1 -- 172.21.15.86:0/3737664571 shutdown_connections 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.868+0000 7f95f5200640 1 --2- 172.21.15.86:0/3737664571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95d4004c80 unknown :-1 s=CLOSED pgs=702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.868+0000 7f95f5200640 1 -- 172.21.15.86:0/3737664571 >> 172.21.15.86:0/3737664571 conn(0x7f95f00b3800 msgr2=0x7f95f00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.868+0000 7f95f5200640 1 -- 172.21.15.86:0/3737664571 shutdown_connections 2024-03-06T07:30:13.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.868+0000 7f95f5200640 1 -- 172.21.15.86:0/3737664571 wait complete. 2024-03-06T07:30:13.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.870+0000 7f95f5200640 1 Processor -- start 2024-03-06T07:30:13.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.870+0000 7f95f5200640 1 -- start start 2024-03-06T07:30:13.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95f5200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95f014cf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95f5200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95f015c820 0x7f95f014d640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95f5200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95f00c7740 0x7f95f00c7bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95f5200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95f006f4c0 con 0x7f95d4004800 2024-03-06T07:30:13.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95f5200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95f006f340 con 0x7f95f015c820 2024-03-06T07:30:13.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95f5200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95f00d0ae0 con 0x7f95f00c7740 2024-03-06T07:30:13.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95ede00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95f014cf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95ede00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95f014cf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49700/0 (socket says 172.21.15.86:49700) 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.871+0000 7f95ede00640 1 -- 172.21.15.86:0/1823087210 learned_addr learned my addr 172.21.15.86:0/1823087210 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95ede00640 1 -- 172.21.15.86:0/1823087210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95f00c7740 msgr2=0x7f95f00c7bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95ede00640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95f00c7740 0x7f95f00c7bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95ede00640 1 -- 172.21.15.86:0/1823087210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95f015c820 msgr2=0x7f95f014d640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95ede00640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95f015c820 0x7f95f014d640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95ede00640 1 -- 172.21.15.86:0/1823087210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95f00c7260 con 0x7f95d4004800 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95ede00640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95f014cf80 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f95d800e500 tx=0x7f95d800e9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95e6a00640 1 -- 172.21.15.86:0/1823087210 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95d80053a0 con 0x7f95d4004800 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95e6a00640 1 -- 172.21.15.86:0/1823087210 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95d8018070 con 0x7f95d4004800 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95e6a00640 1 -- 172.21.15.86:0/1823087210 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95d8003810 con 0x7f95d4004800 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95f00cba50 con 0x7f95d4004800 2024-03-06T07:30:13.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.872+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95f01615a0 con 0x7f95d4004800 2024-03-06T07:30:13.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.874+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f95f004f330 con 0x7f95d4004800 2024-03-06T07:30:13.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.875+0000 7f95e6a00640 1 -- 172.21.15.86:0/1823087210 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f95d80039b0 con 0x7f95d4004800 2024-03-06T07:30:13.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.875+0000 7f95e6a00640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95cc06b500 0x7f95cc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:13.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.875+0000 7f95e6a00640 1 -- 172.21.15.86:0/1823087210 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f95d8089e60 con 0x7f95d4004800 2024-03-06T07:30:13.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.876+0000 7f95e5400640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95cc06b500 0x7f95cc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:13.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.876+0000 7f95e5400640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95cc06b500 0x7f95cc06d9c0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f95e0034000 tx=0x7f95e0007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:13.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:13.884+0000 7f95e6a00640 1 -- 172.21.15.86:0/1823087210 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f95d8058ce0 con 0x7f95d4004800 2024-03-06T07:30:14.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.080+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f95f01610a0 con 0x7f95d4004800 2024-03-06T07:30:14.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.081+0000 7f95e6a00640 1 -- 172.21.15.86:0/1823087210 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v58) v1 ==== 93+0+10 (secure 0 0 0) 0x7f95d80585e0 con 0x7f95d4004800 2024-03-06T07:30:14.084 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:30:14.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95cc06b500 msgr2=0x7f95cc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95cc06b500 0x7f95cc06d9c0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f95e0034000 tx=0x7f95e0007000 comp rx=0 tx=0).stop 2024-03-06T07:30:14.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 msgr2=0x7f95f014cf80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95f014cf80 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f95d800e500 tx=0x7f95d800e9c0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 shutdown_connections 2024-03-06T07:30:14.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95cc06b500 0x7f95cc06d9c0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95f00c7740 0x7f95f00c7bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95f015c820 0x7f95f014d640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 --2- 172.21.15.86:0/1823087210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95d4004800 0x7f95f014cf80 unknown :-1 s=CLOSED pgs=703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 >> 172.21.15.86:0/1823087210 conn(0x7f95f00b3800 msgr2=0x7f95f006f7e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:14.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 shutdown_connections 2024-03-06T07:30:14.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.085+0000 7f95f5200640 1 -- 172.21.15.86:0/1823087210 wait complete. 2024-03-06T07:30:14.114 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:30:14.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.320+0000 7fa91dc00640 1 Processor -- start 2024-03-06T07:30:14.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.320+0000 7fa91dc00640 1 -- start start 2024-03-06T07:30:14.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.322+0000 7fa91dc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa91815df80 0x7fa91806afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.322+0000 7fa91dc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa9180b01a0 0x7fa91806b500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.322+0000 7fa91dc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa9180b5c90 con 0x7fa918161c10 2024-03-06T07:30:14.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.322+0000 7fa91dc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa918072a00 con 0x7fa9180b01a0 2024-03-06T07:30:14.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.322+0000 7fa91dc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9180d1820 con 0x7fa91815df80 2024-03-06T07:30:14.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.323+0000 7fa917e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa91815df80 0x7fa91806afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.323+0000 7fa917400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa9180b01a0 0x7fa91806b500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.323+0000 7fa917e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa91815df80 0x7fa91806afc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:35388/0 (socket says 172.21.15.86:35388) 2024-03-06T07:30:14.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.323+0000 7fa917e00640 1 -- 172.21.15.86:0/3551680860 learned_addr learned my addr 172.21.15.86:0/3551680860 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:14.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.324+0000 7fa917e00640 1 -- 172.21.15.86:0/3551680860 >> v2:172.21.15.194:3300/0 conn(0x7fa9180b01a0 msgr2=0x7fa91806b500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:14.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.324+0000 7fa917e00640 1 --2- 172.21.15.86:0/3551680860 >> v2:172.21.15.194:3300/0 conn(0x7fa9180b01a0 0x7fa91806b500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.324+0000 7fa917e00640 1 -- 172.21.15.86:0/3551680860 >> v1:172.21.15.194:6789/0 conn(0x7fa918161c10 legacy=0x7fa9181620a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.324+0000 7fa917e00640 1 -- 172.21.15.86:0/3551680860 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9180d1540 con 0x7fa91815df80 2024-03-06T07:30:14.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.324+0000 7fa916a00640 1 -- 172.21.15.86:0/3551680860 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1898046583 0 0) 0x7fa9180b5c90 con 0x7fa918161c10 2024-03-06T07:30:14.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.324+0000 7fa917400640 1 --2- 172.21.15.86:0/3551680860 >> v2:172.21.15.194:3300/0 conn(0x7fa9180b01a0 0x7fa91806b500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:30:14.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.326+0000 7fa917e00640 1 --2- 172.21.15.86:0/3551680860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa91815df80 0x7fa91806afc0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fa908009870 tx=0x7fa90802b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a3cce88a343c8b19 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:14.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.326+0000 7fa916a00640 1 -- 172.21.15.86:0/3551680860 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa908034070 con 0x7fa91815df80 2024-03-06T07:30:14.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.326+0000 7fa916a00640 1 -- 172.21.15.86:0/3551680860 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa908030840 con 0x7fa91815df80 2024-03-06T07:30:14.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.326+0000 7fa916a00640 1 -- 172.21.15.86:0/3551680860 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa908030c10 con 0x7fa91815df80 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 -- 172.21.15.86:0/3551680860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa91815df80 msgr2=0x7fa91806afc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 --2- 172.21.15.86:0/3551680860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa91815df80 0x7fa91806afc0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fa908009870 tx=0x7fa90802b980 comp rx=0 tx=0).stop 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 -- 172.21.15.86:0/3551680860 shutdown_connections 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 --2- 172.21.15.86:0/3551680860 >> v2:172.21.15.194:3300/0 conn(0x7fa9180b01a0 0x7fa91806b500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 --2- 172.21.15.86:0/3551680860 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa91815df80 0x7fa91806afc0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 -- 172.21.15.86:0/3551680860 >> 172.21.15.86:0/3551680860 conn(0x7fa9180d0a20 msgr2=0x7fa9180d0e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 -- 172.21.15.86:0/3551680860 shutdown_connections 2024-03-06T07:30:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.327+0000 7fa91dc00640 1 -- 172.21.15.86:0/3551680860 wait complete. 2024-03-06T07:30:14.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.329+0000 7fa91dc00640 1 Processor -- start 2024-03-06T07:30:14.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.330+0000 7fa91dc00640 1 -- start start 2024-03-06T07:30:14.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa91dc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9180b01a0 0x7fa9181443f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa91dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa91815df80 0x7fa9181f3b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa91dc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa918161c10 0x7fa9180b0850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa91dc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9180b5770 con 0x7fa91815df80 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa91dc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9180b5490 con 0x7fa918161c10 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa91dc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa91815e360 con 0x7fa9180b01a0 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa917400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa91815df80 0x7fa9181f3b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa917400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa91815df80 0x7fa9181f3b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49708/0 (socket says 172.21.15.86:49708) 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa917400640 1 -- 172.21.15.86:0/1045757608 learned_addr learned my addr 172.21.15.86:0/1045757608 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa917e00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9180b01a0 0x7fa9181443f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.331+0000 7fa91d200640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa918161c10 0x7fa9180b0850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa917400640 1 -- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9180b01a0 msgr2=0x7fa9181443f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa917400640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9180b01a0 0x7fa9181443f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa917400640 1 -- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa918161c10 msgr2=0x7fa9180b0850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa917400640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa918161c10 0x7fa9180b0850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa917400640 1 -- 172.21.15.86:0/1045757608 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9181f4440 con 0x7fa91815df80 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa917400640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa91815df80 0x7fa9181f3b50 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7fa90400ebe0 tx=0x7fa90400c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa90fe00640 1 -- 172.21.15.86:0/1045757608 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa904017070 con 0x7fa91815df80 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa90fe00640 1 -- 172.21.15.86:0/1045757608 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa90400cc30 con 0x7fa91815df80 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa90fe00640 1 -- 172.21.15.86:0/1045757608 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa904013470 con 0x7fa91815df80 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa9181f4720 con 0x7fa91815df80 2024-03-06T07:30:14.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.332+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa91815e690 con 0x7fa91815df80 2024-03-06T07:30:14.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.333+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa91804f330 con 0x7fa91815df80 2024-03-06T07:30:14.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.334+0000 7fa90fe00640 1 -- 172.21.15.86:0/1045757608 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa90401a070 con 0x7fa91815df80 2024-03-06T07:30:14.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.335+0000 7fa90fe00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f806b4d0 0x7fa8f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.335+0000 7fa90fe00640 1 -- 172.21.15.86:0/1045757608 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa90408a5a0 con 0x7fa91815df80 2024-03-06T07:30:14.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.336+0000 7fa917e00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f806b4d0 0x7fa8f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.336+0000 7fa917e00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f806b4d0 0x7fa8f806d990 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7fa908036000 tx=0x7fa90803b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:14.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.340+0000 7fa90fe00640 1 -- 172.21.15.86:0/1045757608 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa904059380 con 0x7fa91815df80 2024-03-06T07:30:14.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.554+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fa9180cd950 con 0x7fa91815df80 2024-03-06T07:30:14.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.555+0000 7fa90fe00640 1 -- 172.21.15.86:0/1045757608 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v58) v1 ==== 104+0+11 (secure 0 0 0) 0x7fa904058c80 con 0x7fa91815df80 2024-03-06T07:30:14.558 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f806b4d0 msgr2=0x7fa8f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f806b4d0 0x7fa8f806d990 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7fa908036000 tx=0x7fa90803b040 comp rx=0 tx=0).stop 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa91815df80 msgr2=0x7fa9181f3b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa91815df80 0x7fa9181f3b50 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7fa90400ebe0 tx=0x7fa90400c520 comp rx=0 tx=0).stop 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 shutdown_connections 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f806b4d0 0x7fa8f806d990 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa918161c10 0x7fa9180b0850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa91815df80 0x7fa9181f3b50 unknown :-1 s=CLOSED pgs=704 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 --2- 172.21.15.86:0/1045757608 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9180b01a0 0x7fa9181443f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 >> 172.21.15.86:0/1045757608 conn(0x7fa9180d0a20 msgr2=0x7fa918158800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:14.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 shutdown_connections 2024-03-06T07:30:14.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.558+0000 7fa91dc00640 1 -- 172.21.15.86:0/1045757608 wait complete. 2024-03-06T07:30:14.579 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:30:14.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.783+0000 7f807d200640 1 Processor -- start 2024-03-06T07:30:14.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.783+0000 7f807d200640 1 -- start start 2024-03-06T07:30:14.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.784+0000 7f807d200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f807815bb90 0x7f807806dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f807d200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8078071ea0 con 0x7f807815efa0 2024-03-06T07:30:14.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f807d200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f807806e380 con 0x7f807806afc0 2024-03-06T07:30:14.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f807d200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f807815bf70 con 0x7f807815bb90 2024-03-06T07:30:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f806fe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f807815efa0 0x7f80780cec20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55044/0 (socket says 172.21.15.86:55044) 2024-03-06T07:30:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f806fe00640 1 -- 172.21.15.86:0/411609861 learned_addr learned my addr 172.21.15.86:0/411609861 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f8075e00640 1 --2- 172.21.15.86:0/411609861 >> v2:172.21.15.194:3300/0 conn(0x7f807815bb90 0x7f807806dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3069546997 0 0) 0x7f8078071ea0 con 0x7f807815efa0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.785+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f805c003520 con 0x7f807815efa0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3582220426 0 0) 0x7f805c003520 con 0x7f807815efa0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 >> v2:172.21.15.194:3300/0 conn(0x7f807815bb90 msgr2=0x7f807806dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 --2- 172.21.15.86:0/411609861 >> v2:172.21.15.194:3300/0 conn(0x7f807815bb90 0x7f807806dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 >> v1:172.21.15.194:6789/0 conn(0x7f807806afc0 legacy=0x7f80780723a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f807806e660 con 0x7f807815efa0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8060001670 con 0x7f807815efa0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 >> v1:172.21.15.86:6789/0 conn(0x7f807815efa0 legacy=0x7f80780cec20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f805c0039b0 0x7f805c003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c004800 0x7f805c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f805c0051c0 0x7f805c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807815bf70 con 0x7f805c0039b0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f805c00c040 con 0x7f805c0051c0 2024-03-06T07:30:14.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f805c00c240 con 0x7f805c004800 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.786+0000 7f8075e00640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f805c0039b0 0x7f805c003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.787+0000 7f8075e00640 1 -- 172.21.15.86:0/411609861 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c004800 msgr2=0x7f805c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.787+0000 7f8075e00640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c004800 0x7f805c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.787+0000 7f8075e00640 1 -- 172.21.15.86:0/411609861 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f805c0051c0 msgr2=0x7f805c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.787+0000 7f8076800640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f805c0051c0 0x7f805c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.787+0000 7f8075e00640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f805c0051c0 0x7f805c00b9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.787+0000 7f8075e00640 1 -- 172.21.15.86:0/411609861 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f805c003520 con 0x7f805c0039b0 2024-03-06T07:30:14.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.787+0000 7f8076800640 1 -- 172.21.15.86:0/411609861 reap_dead start 2024-03-06T07:30:14.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.788+0000 7f8075e00640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f805c0039b0 0x7f805c003db0 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f8068009ac0 tx=0x7f805c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad8e7308100d1787 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:14.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.788+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8068039070 con 0x7f805c0039b0 2024-03-06T07:30:14.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.788+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f806802c6b0 con 0x7f805c0039b0 2024-03-06T07:30:14.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.788+0000 7f8075400640 1 -- 172.21.15.86:0/411609861 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8068009690 con 0x7f805c0039b0 2024-03-06T07:30:14.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.789+0000 7f807d200640 1 -- 172.21.15.86:0/411609861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f805c0039b0 msgr2=0x7f805c003db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.789+0000 7f807d200640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f805c0039b0 0x7f805c003db0 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f8068009ac0 tx=0x7f805c005830 comp rx=0 tx=0).stop 2024-03-06T07:30:14.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.789+0000 7f807d200640 1 -- 172.21.15.86:0/411609861 shutdown_connections 2024-03-06T07:30:14.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.789+0000 7f807d200640 1 --2- 172.21.15.86:0/411609861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f805c0039b0 0x7f805c003db0 unknown :-1 s=CLOSED pgs=706 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.789+0000 7f807d200640 1 -- 172.21.15.86:0/411609861 >> 172.21.15.86:0/411609861 conn(0x7f80780b27d0 msgr2=0x7f80780b2bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:14.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.789+0000 7f807d200640 1 -- 172.21.15.86:0/411609861 shutdown_connections 2024-03-06T07:30:14.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.789+0000 7f807d200640 1 -- 172.21.15.86:0/411609861 wait complete. 2024-03-06T07:30:14.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.791+0000 7f807d200640 1 Processor -- start 2024-03-06T07:30:14.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.791+0000 7f807d200640 1 -- start start 2024-03-06T07:30:14.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f807d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c0039b0 0x7f80781497d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f807d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80780cadd0 0x7f80780cd150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f807d200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8078072490 0x7f8078072910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f807d200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807815f430 con 0x7f80780cadd0 2024-03-06T07:30:14.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f807d200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807806ec20 con 0x7f8078072490 2024-03-06T07:30:14.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f807d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f807806dcf0 con 0x7f805c0039b0 2024-03-06T07:30:14.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80780cadd0 0x7f80780cd150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80780cadd0 0x7f80780cd150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49720/0 (socket says 172.21.15.86:49720) 2024-03-06T07:30:14.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 -- 172.21.15.86:0/275623521 learned_addr learned my addr 172.21.15.86:0/275623521 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:14.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f8075e00640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c0039b0 0x7f80781497d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f8076800640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8078072490 0x7f8078072910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 -- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c0039b0 msgr2=0x7f80781497d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c0039b0 0x7f80781497d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 -- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8078072490 msgr2=0x7f8078072910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:14.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8078072490 0x7f8078072910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:14.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f806fe00640 1 -- 172.21.15.86:0/275623521 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80780cd980 con 0x7f80780cadd0 2024-03-06T07:30:14.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.792+0000 7f8076800640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8078072490 0x7f8078072910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:14.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.793+0000 7f806fe00640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80780cadd0 0x7f80780cd150 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f806000c760 tx=0x7f806000cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:14.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.793+0000 7f806e000640 1 -- 172.21.15.86:0/275623521 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8060018070 con 0x7f80780cadd0 2024-03-06T07:30:14.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.793+0000 7f806e000640 1 -- 172.21.15.86:0/275623521 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f80600074e0 con 0x7f80780cadd0 2024-03-06T07:30:14.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.793+0000 7f806e000640 1 -- 172.21.15.86:0/275623521 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8060004a20 con 0x7f80780cadd0 2024-03-06T07:30:14.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.793+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80780cdc60 con 0x7f80780cadd0 2024-03-06T07:30:14.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.793+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80781581a0 con 0x7f80780cadd0 2024-03-06T07:30:14.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.794+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f807804f330 con 0x7f80780cadd0 2024-03-06T07:30:14.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.795+0000 7f806e000640 1 -- 172.21.15.86:0/275623521 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f806001b070 con 0x7f80780cadd0 2024-03-06T07:30:14.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.797+0000 7f806e000640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805406b500 0x7f805406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:14.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.797+0000 7f806e000640 1 -- 172.21.15.86:0/275623521 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8060089620 con 0x7f80780cadd0 2024-03-06T07:30:14.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.797+0000 7f8075e00640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805406b500 0x7f805406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:14.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.798+0000 7f8075e00640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805406b500 0x7f805406d9c0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f8068036000 tx=0x7f8068030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:14.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:14.801+0000 7f806e000640 1 -- 172.21.15.86:0/275623521 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f80600584a0 con 0x7f80780cadd0 2024-03-06T07:30:15.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.001+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f80780b68f0 con 0x7f80780cadd0 2024-03-06T07:30:15.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.002+0000 7f806e000640 1 -- 172.21.15.86:0/275623521 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v58) v1 ==== 100+0+11 (secure 0 0 0) 0x7f8060057da0 con 0x7f80780cadd0 2024-03-06T07:30:15.004 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805406b500 msgr2=0x7f805406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805406b500 0x7f805406d9c0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f8068036000 tx=0x7f8068030000 comp rx=0 tx=0).stop 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80780cadd0 msgr2=0x7f80780cd150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80780cadd0 0x7f80780cd150 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f806000c760 tx=0x7f806000cc20 comp rx=0 tx=0).stop 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 shutdown_connections 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f805406b500 0x7f805406d9c0 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8078072490 0x7f8078072910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80780cadd0 0x7f80780cd150 unknown :-1 s=CLOSED pgs=707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 --2- 172.21.15.86:0/275623521 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f805c0039b0 0x7f80781497d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 >> 172.21.15.86:0/275623521 conn(0x7f80780b27d0 msgr2=0x7f80780cf8e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:15.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 shutdown_connections 2024-03-06T07:30:15.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.004+0000 7f807d200640 1 -- 172.21.15.86:0/275623521 wait complete. 2024-03-06T07:30:15.026 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:30:15.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.221+0000 7fa501e00640 1 Processor -- start 2024-03-06T07:30:15.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.221+0000 7fa501e00640 1 -- start start 2024-03-06T07:30:15.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa501e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc0d2360 0x7fa4fc0d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:15.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa501e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa4fc0b3ec0 0x7fa4fc0b4350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:15.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa501e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4fc161960 con 0x7fa4fc161580 2024-03-06T07:30:15.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa501e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4fc0d2ce0 con 0x7fa4fc0b3ec0 2024-03-06T07:30:15.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa501e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4fc0cf6f0 con 0x7fa4fc0d2360 2024-03-06T07:30:15.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa4fbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc0d2360 0x7fa4fc0d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa501400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa4fc0b3ec0 0x7fa4fc0b4350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa4fbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc0d2360 0x7fa4fc0d27a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:35420/0 (socket says 172.21.15.86:35420) 2024-03-06T07:30:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.224+0000 7fa4fbe00640 1 -- 172.21.15.86:0/2660084887 learned_addr learned my addr 172.21.15.86:0/2660084887 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.225+0000 7fa4fbe00640 1 -- 172.21.15.86:0/2660084887 >> v2:172.21.15.194:3300/0 conn(0x7fa4fc0b3ec0 msgr2=0x7fa4fc0b4350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.225+0000 7fa4fbe00640 1 --2- 172.21.15.86:0/2660084887 >> v2:172.21.15.194:3300/0 conn(0x7fa4fc0b3ec0 0x7fa4fc0b4350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.225+0000 7fa4fbe00640 1 -- 172.21.15.86:0/2660084887 >> v1:172.21.15.194:6789/0 conn(0x7fa4fc161580 legacy=0x7fa4fc0d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.225+0000 7fa4fbe00640 1 -- 172.21.15.86:0/2660084887 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4fc0cf9d0 con 0x7fa4fc0d2360 2024-03-06T07:30:15.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.227+0000 7fa4fbe00640 1 --2- 172.21.15.86:0/2660084887 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc0d2360 0x7fa4fc0d27a0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fa4e8009870 tx=0x7fa4e802b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a9ad2fd4ab8c4c29 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:15.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.227+0000 7fa4fb400640 1 -- 172.21.15.86:0/2660084887 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4e8034070 con 0x7fa4fc0d2360 2024-03-06T07:30:15.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.227+0000 7fa4fb400640 1 -- 172.21.15.86:0/2660084887 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa4e8030840 con 0x7fa4fc0d2360 2024-03-06T07:30:15.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.227+0000 7fa4fb400640 1 -- 172.21.15.86:0/2660084887 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4e8030c10 con 0x7fa4fc0d2360 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 -- 172.21.15.86:0/2660084887 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc0d2360 msgr2=0x7fa4fc0d27a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 --2- 172.21.15.86:0/2660084887 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc0d2360 0x7fa4fc0d27a0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fa4e8009870 tx=0x7fa4e802b980 comp rx=0 tx=0).stop 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 -- 172.21.15.86:0/2660084887 shutdown_connections 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 --2- 172.21.15.86:0/2660084887 >> v2:172.21.15.194:3300/0 conn(0x7fa4fc0b3ec0 0x7fa4fc0b4350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 --2- 172.21.15.86:0/2660084887 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc0d2360 0x7fa4fc0d27a0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 -- 172.21.15.86:0/2660084887 >> 172.21.15.86:0/2660084887 conn(0x7fa4fc0d3d80 msgr2=0x7fa4fc0d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 -- 172.21.15.86:0/2660084887 shutdown_connections 2024-03-06T07:30:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.228+0000 7fa501e00640 1 -- 172.21.15.86:0/2660084887 wait complete. 2024-03-06T07:30:15.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.232+0000 7fa501e00640 1 Processor -- start 2024-03-06T07:30:15.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.232+0000 7fa501e00640 1 -- start start 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa501e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4fc0b3ec0 0x7fa4fc0b7f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa501e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4fc0d2360 0x7fa4fc0cafe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa501e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc161580 0x7fa4fc152190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa501e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4fc0b39a0 con 0x7fa4fc0d2360 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa501e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4fc0b36c0 con 0x7fa4fc0b3ec0 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa501e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4fc0cf3a0 con 0x7fa4fc161580 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa4fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4fc0d2360 0x7fa4fc0cafe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa4fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4fc0d2360 0x7fa4fc0cafe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49722/0 (socket says 172.21.15.86:49722) 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.234+0000 7fa4fbe00640 1 -- 172.21.15.86:0/1757793319 learned_addr learned my addr 172.21.15.86:0/1757793319 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:15.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa500a00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4fc0b3ec0 0x7fa4fc0b7f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa501400640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc161580 0x7fa4fc152190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa4fbe00640 1 -- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc161580 msgr2=0x7fa4fc152190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa4fbe00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc161580 0x7fa4fc152190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa4fbe00640 1 -- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4fc0b3ec0 msgr2=0x7fa4fc0b7f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa4fbe00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4fc0b3ec0 0x7fa4fc0b7f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa4fbe00640 1 -- 172.21.15.86:0/1757793319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4fc0b8500 con 0x7fa4fc0d2360 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa501400640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc161580 0x7fa4fc152190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.235+0000 7fa4fbe00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4fc0d2360 0x7fa4fc0cafe0 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7fa4e8000c00 tx=0x7fa4e80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.236+0000 7fa4f8c00640 1 -- 172.21.15.86:0/1757793319 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4e8034040 con 0x7fa4fc0d2360 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.236+0000 7fa4f8c00640 1 -- 172.21.15.86:0/1757793319 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa4e8039d90 con 0x7fa4fc0d2360 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.236+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4fc0b87e0 con 0x7fa4fc0d2360 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.236+0000 7fa4f8c00640 1 -- 172.21.15.86:0/1757793319 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4e8043630 con 0x7fa4fc0d2360 2024-03-06T07:30:15.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.236+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4fc0b8da0 con 0x7fa4fc0d2360 2024-03-06T07:30:15.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.237+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa4fc04f330 con 0x7fa4fc0d2360 2024-03-06T07:30:15.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.238+0000 7fa4f8c00640 1 -- 172.21.15.86:0/1757793319 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa4e804a060 con 0x7fa4fc0d2360 2024-03-06T07:30:15.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.239+0000 7fa4f8c00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa4dc06b500 0x7fa4dc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:15.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.239+0000 7fa4f8c00640 1 -- 172.21.15.86:0/1757793319 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa4e80ac180 con 0x7fa4fc0d2360 2024-03-06T07:30:15.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.239+0000 7fa500a00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa4dc06b500 0x7fa4dc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:15.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.240+0000 7fa500a00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa4dc06b500 0x7fa4dc06d9c0 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fa4f0000f60 tx=0x7fa4f0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:15.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.244+0000 7fa4f8c00640 1 -- 172.21.15.86:0/1757793319 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa4e803e070 con 0x7fa4fc0d2360 2024-03-06T07:30:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.453+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fa4fc0d3b60 con 0x7fa4fc0d2360 2024-03-06T07:30:15.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.453+0000 7fa4f8c00640 1 -- 172.21.15.86:0/1757793319 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v58) v1 ==== 80+0+0 (secure 0 0 0) 0x7fa4e802d330 con 0x7fa4fc0d2360 2024-03-06T07:30:15.456 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:30:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa4dc06b500 msgr2=0x7fa4dc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa4dc06b500 0x7fa4dc06d9c0 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fa4f0000f60 tx=0x7fa4f0009040 comp rx=0 tx=0).stop 2024-03-06T07:30:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4fc0d2360 msgr2=0x7fa4fc0cafe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4fc0d2360 0x7fa4fc0cafe0 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7fa4e8000c00 tx=0x7fa4e80046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 shutdown_connections 2024-03-06T07:30:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa4dc06b500 0x7fa4dc06d9c0 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa4fc161580 0x7fa4fc152190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa4fc0d2360 0x7fa4fc0cafe0 unknown :-1 s=CLOSED pgs=708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 --2- 172.21.15.86:0/1757793319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa4fc0b3ec0 0x7fa4fc0b7f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:15.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.456+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 >> 172.21.15.86:0/1757793319 conn(0x7fa4fc0d3d80 msgr2=0x7fa4fc0b55a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:15.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.457+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 shutdown_connections 2024-03-06T07:30:15.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:15.457+0000 7fa501e00640 1 -- 172.21.15.86:0/1757793319 wait complete. 2024-03-06T07:30:15.477 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:30:15.477 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:15.478 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:15.697 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:30:15.697 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:15.697 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:30:15.741 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:30:15.741 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:15.783 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:30:15.783 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:15.783 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:15.820 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:30:15.821 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:30:15.821 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:15.874 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:30:15.874 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:16.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.075+0000 7f6aaa600640 1 Processor -- start 2024-03-06T07:30:16.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.076+0000 7f6aaa600640 1 -- start start 2024-03-06T07:30:16.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.078+0000 7f6aaa600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6aa406d390 0x7f6aa4072340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.078+0000 7f6aaa600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6aa406def0 con 0x7f6aa406daa0 2024-03-06T07:30:16.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.078+0000 7f6aaa600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6aa40b39c0 con 0x7f6aa4147310 2024-03-06T07:30:16.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.078+0000 7f6aaa600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6aa40ad9e0 con 0x7f6aa406d390 2024-03-06T07:30:16.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.079+0000 7f6aa3e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6aa406daa0 0x7f6aa40ae7a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37356/0 (socket says 172.21.15.86:37356) 2024-03-06T07:30:16.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.079+0000 7f6aa3e00640 1 -- 172.21.15.86:0/3575778763 learned_addr learned my addr 172.21.15.86:0/3575778763 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:16.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.079+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/3575778763 >> v2:172.21.15.194:3300/0 conn(0x7f6aa406d390 0x7f6aa4072340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.079+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1483683904 0 0) 0x7f6aa406def0 con 0x7f6aa406daa0 2024-03-06T07:30:16.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.079+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6a88003520 con 0x7f6aa406daa0 2024-03-06T07:30:16.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3030069344 0 0) 0x7f6a88003520 con 0x7f6aa406daa0 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 >> v2:172.21.15.194:3300/0 conn(0x7f6aa406d390 msgr2=0x7f6aa4072340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 --2- 172.21.15.86:0/3575778763 >> v2:172.21.15.194:3300/0 conn(0x7f6aa406d390 0x7f6aa4072340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 >> v1:172.21.15.194:6789/0 conn(0x7f6aa4147310 legacy=0x7f6aa406cc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6aa40adcc0 con 0x7f6aa406daa0 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6a9c0019f0 con 0x7f6aa406daa0 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 >> v1:172.21.15.86:6789/0 conn(0x7f6aa406daa0 legacy=0x7f6aa40ae7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a88003a00 0x7f6a88003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a88004800 0x7f6a88004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a880051c0 0x7f6a8800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6aa40ad9e0 con 0x7f6a88004800 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a8800c0c0 con 0x7f6a88003a00 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a8800c2c0 con 0x7f6a880051c0 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.080+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a88004800 0x7f6a88004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.081+0000 7f6aa2a00640 1 -- 172.21.15.86:0/3575778763 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a880051c0 msgr2=0x7f6a8800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:16.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.081+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a880051c0 0x7f6a8800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.081+0000 7f6aa2a00640 1 -- 172.21.15.86:0/3575778763 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a88003a00 msgr2=0x7f6a88003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:16.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.081+0000 7f6aa3400640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a88003a00 0x7f6a88003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.081+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a88003a00 0x7f6a88003e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.081+0000 7f6aa2a00640 1 -- 172.21.15.86:0/3575778763 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a88003520 con 0x7f6a88004800 2024-03-06T07:30:16.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.081+0000 7f6aa3e00640 1 -- 172.21.15.86:0/3575778763 reap_dead start 2024-03-06T07:30:16.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.082+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a88004800 0x7f6a88004c80 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7f6a90009ac0 tx=0x7f6a88005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8184dd8aedc839ee server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:16.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.083+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a90039070 con 0x7f6a88004800 2024-03-06T07:30:16.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.083+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a9002c6b0 con 0x7f6a88004800 2024-03-06T07:30:16.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.083+0000 7f6aa2000640 1 -- 172.21.15.86:0/3575778763 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a90009690 con 0x7f6a88004800 2024-03-06T07:30:16.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.084+0000 7f6aaa600640 1 -- 172.21.15.86:0/3575778763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a88004800 msgr2=0x7f6a88004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.084+0000 7f6aaa600640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a88004800 0x7f6a88004c80 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7f6a90009ac0 tx=0x7f6a88005830 comp rx=0 tx=0).stop 2024-03-06T07:30:16.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.084+0000 7f6aaa600640 1 -- 172.21.15.86:0/3575778763 shutdown_connections 2024-03-06T07:30:16.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.084+0000 7f6aaa600640 1 --2- 172.21.15.86:0/3575778763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a88004800 0x7f6a88004c80 unknown :-1 s=CLOSED pgs=710 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.084+0000 7f6aaa600640 1 -- 172.21.15.86:0/3575778763 >> 172.21.15.86:0/3575778763 conn(0x7f6aa40b1be0 msgr2=0x7f6aa40b1ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:16.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.084+0000 7f6aaa600640 1 -- 172.21.15.86:0/3575778763 shutdown_connections 2024-03-06T07:30:16.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.084+0000 7f6aaa600640 1 -- 172.21.15.86:0/3575778763 wait complete. 2024-03-06T07:30:16.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.086+0000 7f6aaa600640 1 Processor -- start 2024-03-06T07:30:16.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.086+0000 7f6aaa600640 1 -- start start 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aaa600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a88004800 0x7f6aa413d360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aaa600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6aa41f51d0 0x7f6aa415cad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aaa600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6aa40b0f20 0x7f6aa40b13a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aaa600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6aa41476f0 con 0x7f6aa41f51d0 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aaa600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6aa40ae280 con 0x7f6aa40b0f20 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aaa600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6aa4072340 con 0x7f6a88004800 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6aa41f51d0 0x7f6aa415cad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6aa41f51d0 0x7f6aa415cad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33444/0 (socket says 172.21.15.86:33444) 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 -- 172.21.15.86:0/2154170514 learned_addr learned my addr 172.21.15.86:0/2154170514 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa3400640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a88004800 0x7f6aa413d360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa3e00640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6aa40b0f20 0x7f6aa40b13a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 -- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a88004800 msgr2=0x7f6aa413d360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a88004800 0x7f6aa413d360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 -- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6aa40b0f20 msgr2=0x7f6aa40b13a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6aa40b0f20 0x7f6aa40b13a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.088+0000 7f6aa2a00640 1 -- 172.21.15.86:0/2154170514 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6aa413d9f0 con 0x7f6aa41f51d0 2024-03-06T07:30:16.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.089+0000 7f6aa2a00640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6aa41f51d0 0x7f6aa415cad0 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f6a90000c00 tx=0x7f6a90004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:16.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.089+0000 7f6a9b400640 1 -- 172.21.15.86:0/2154170514 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a90042020 con 0x7f6aa41f51d0 2024-03-06T07:30:16.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.089+0000 7f6a9b400640 1 -- 172.21.15.86:0/2154170514 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a9003a040 con 0x7f6aa41f51d0 2024-03-06T07:30:16.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.089+0000 7f6a9b400640 1 -- 172.21.15.86:0/2154170514 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a90039070 con 0x7f6aa41f51d0 2024-03-06T07:30:16.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.089+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6aa40b0160 con 0x7f6aa41f51d0 2024-03-06T07:30:16.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.089+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6aa40b0720 con 0x7f6aa41f51d0 2024-03-06T07:30:16.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.091+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6aa404f330 con 0x7f6aa41f51d0 2024-03-06T07:30:16.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.091+0000 7f6a9b400640 1 -- 172.21.15.86:0/2154170514 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6a90045020 con 0x7f6aa41f51d0 2024-03-06T07:30:16.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.093+0000 7f6a9b400640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a8406b500 0x7f6a8406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.093+0000 7f6a9b400640 1 -- 172.21.15.86:0/2154170514 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6a900ab630 con 0x7f6aa41f51d0 2024-03-06T07:30:16.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.093+0000 7f6aa3400640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a8406b500 0x7f6a8406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.094+0000 7f6aa3400640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a8406b500 0x7f6a8406d9c0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7f6aa40686c0 tx=0x7f6a94009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:16.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.097+0000 7f6a9b400640 1 -- 172.21.15.86:0/2154170514 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6a9004c070 con 0x7f6aa41f51d0 2024-03-06T07:30:16.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.333+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6aa415ece0 con 0x7f6aa41f51d0 2024-03-06T07:30:16.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.334+0000 7f6a9b400640 1 -- 172.21.15.86:0/2154170514 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f6a9007ee20 con 0x7f6aa41f51d0 2024-03-06T07:30:16.337 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:16.337 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:30:16.337 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:30:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a8406b500 msgr2=0x7f6a8406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a8406b500 0x7f6a8406d9c0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7f6aa40686c0 tx=0x7f6a94009040 comp rx=0 tx=0).stop 2024-03-06T07:30:16.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6aa41f51d0 msgr2=0x7f6aa415cad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6aa41f51d0 0x7f6aa415cad0 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f6a90000c00 tx=0x7f6a90004730 comp rx=0 tx=0).stop 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 shutdown_connections 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a8406b500 0x7f6a8406d9c0 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6aa40b0f20 0x7f6aa40b13a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6aa41f51d0 0x7f6aa415cad0 unknown :-1 s=CLOSED pgs=711 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 --2- 172.21.15.86:0/2154170514 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a88004800 0x7f6aa413d360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 >> 172.21.15.86:0/2154170514 conn(0x7f6aa40b1be0 msgr2=0x7f6aa4071630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 shutdown_connections 2024-03-06T07:30:16.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.337+0000 7f6aaa600640 1 -- 172.21.15.86:0/2154170514 wait complete. 2024-03-06T07:30:16.357 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:16.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.550+0000 7f4549600640 1 Processor -- start 2024-03-06T07:30:16.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.550+0000 7f4549600640 1 -- start start 2024-03-06T07:30:16.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4549600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f45440b0300 0x7f454406f5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4549600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454406afc0 0x7f454406b470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4549600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f45440b6090 con 0x7f4544147430 2024-03-06T07:30:16.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4549600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f45440b5db0 con 0x7f45440b0300 2024-03-06T07:30:16.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4549600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f45440d09c0 con 0x7f454406afc0 2024-03-06T07:30:16.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4542000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4544147430 0x7f45440d14a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37370/0 (socket says 172.21.15.86:37370) 2024-03-06T07:30:16.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4542000640 1 -- 172.21.15.86:0/1758742368 learned_addr learned my addr 172.21.15.86:0/1758742368 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:16.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4543400640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454406afc0 0x7f454406b470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4542a00640 1 --2- 172.21.15.86:0/1758742368 >> v2:172.21.15.194:3300/0 conn(0x7f45440b0300 0x7f454406f5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.552+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3262894346 0 0) 0x7f45440b6090 con 0x7f4544147430 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4528003520 con 0x7f4544147430 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3348619151 0 0) 0x7f4528003520 con 0x7f4544147430 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454406afc0 msgr2=0x7f454406b470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454406afc0 0x7f454406b470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 >> v2:172.21.15.194:3300/0 conn(0x7f45440b0300 msgr2=0x7f454406f5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 --2- 172.21.15.86:0/1758742368 >> v2:172.21.15.194:3300/0 conn(0x7f45440b0300 0x7f454406f5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f45440d0ca0 con 0x7f4544147430 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f4530001670 con 0x7f4544147430 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 >> v1:172.21.15.86:6789/0 conn(0x7f4544147430 legacy=0x7f45440d14a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45280039b0 0x7f4528003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454406afc0 0x7f4528004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4528004d40 0x7f45280051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45440b5db0 con 0x7f4528004d40 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45440d09c0 con 0x7f45280039b0 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f452800b9c0 con 0x7f454406afc0 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.553+0000 7f4543400640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4528004d40 0x7f45280051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.554+0000 7f4543400640 1 -- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454406afc0 msgr2=0x7f4528004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.554+0000 7f4543400640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454406afc0 0x7f4528004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.554+0000 7f4543400640 1 -- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45280039b0 msgr2=0x7f4528003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.554+0000 7f4543400640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45280039b0 0x7f4528003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.554+0000 7f4543400640 1 -- 172.21.15.86:0/1758742368 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4528003520 con 0x7f4528004d40 2024-03-06T07:30:16.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.555+0000 7f4543400640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4528004d40 0x7f45280051c0 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7f453c00e170 tx=0x7f4528005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c9268df4dfccc6a4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:16.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.555+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f453c039070 con 0x7f4528004d40 2024-03-06T07:30:16.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.555+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f453c02cc20 con 0x7f4528004d40 2024-03-06T07:30:16.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.555+0000 7f4541600640 1 -- 172.21.15.86:0/1758742368 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f453c009c10 con 0x7f4528004d40 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.556+0000 7f4549600640 1 -- 172.21.15.86:0/1758742368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4528004d40 msgr2=0x7f45280051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.556+0000 7f4549600640 1 --2- 172.21.15.86:0/1758742368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4528004d40 0x7f45280051c0 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7f453c00e170 tx=0x7f4528005830 comp rx=0 tx=0).stop 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.556+0000 7f4543400640 1 -- 172.21.15.86:0/1758742368 reap_dead start 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.556+0000 7f4549600640 1 -- 172.21.15.86:0/1758742368 shutdown_connections 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.556+0000 7f4549600640 1 -- 172.21.15.86:0/1758742368 >> 172.21.15.86:0/1758742368 conn(0x7f454406c0d0 msgr2=0x7f454406c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.556+0000 7f4549600640 1 -- 172.21.15.86:0/1758742368 shutdown_connections 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.556+0000 7f4549600640 1 -- 172.21.15.86:0/1758742368 wait complete. 2024-03-06T07:30:16.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.558+0000 7f4549600640 1 Processor -- start 2024-03-06T07:30:16.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.558+0000 7f4549600640 1 -- start start 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.558+0000 7f4549600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454415db10 0x7f4544161cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4549600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45441455d0 0x7f4544145a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4549600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45441f7270 0x7f45441f76f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4549600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45440b5b50 con 0x7f45441f7270 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4549600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45440b59d0 con 0x7f45441455d0 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4549600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f454406b470 con 0x7f454415db10 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45441f7270 0x7f45441f76f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45441f7270 0x7f45441f76f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33456/0 (socket says 172.21.15.86:33456) 2024-03-06T07:30:16.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 -- 172.21.15.86:0/208442849 learned_addr learned my addr 172.21.15.86:0/208442849 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4542a00640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454415db10 0x7f4544161cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 -- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454415db10 msgr2=0x7f4544161cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454415db10 0x7f4544161cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 -- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45441455d0 msgr2=0x7f4544145a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4542000640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45441455d0 0x7f4544145a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45441455d0 0x7f4544145a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4543400640 1 -- 172.21.15.86:0/208442849 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f454415f700 con 0x7f45441f7270 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.559+0000 7f4542000640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45441455d0 0x7f4544145a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:16.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.560+0000 7f4543400640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45441f7270 0x7f45441f76f0 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f453c037040 tx=0x7f453c009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:16.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.560+0000 7f453aa00640 1 -- 172.21.15.86:0/208442849 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f453c044070 con 0x7f45441f7270 2024-03-06T07:30:16.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.560+0000 7f453aa00640 1 -- 172.21.15.86:0/208442849 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f453c03a040 con 0x7f45441f7270 2024-03-06T07:30:16.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.560+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f454415f9e0 con 0x7f45441f7270 2024-03-06T07:30:16.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.560+0000 7f453aa00640 1 -- 172.21.15.86:0/208442849 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f453c039030 con 0x7f45441f7270 2024-03-06T07:30:16.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.560+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f454415ffa0 con 0x7f45441f7270 2024-03-06T07:30:16.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.561+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f454404f330 con 0x7f45441f7270 2024-03-06T07:30:16.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.562+0000 7f453aa00640 1 -- 172.21.15.86:0/208442849 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f453c047070 con 0x7f45441f7270 2024-03-06T07:30:16.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.563+0000 7f453aa00640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f452406b4d0 0x7f452406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:16.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.563+0000 7f453aa00640 1 -- 172.21.15.86:0/208442849 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f453c0abaa0 con 0x7f45441f7270 2024-03-06T07:30:16.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.563+0000 7f4542a00640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f452406b4d0 0x7f452406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:16.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.564+0000 7f4542a00640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f452406b4d0 0x7f452406d990 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f4534004770 tx=0x7f4534008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:16.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.567+0000 7f453aa00640 1 -- 172.21.15.86:0/208442849 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f453c07a230 con 0x7f45441f7270 2024-03-06T07:30:16.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.849+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4544072890 con 0x7f45441f7270 2024-03-06T07:30:16.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.851+0000 7f453aa00640 1 -- 172.21.15.86:0/208442849 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v58) v1 ==== 74+0+16677 (secure 0 0 0) 0x7f453c07e2d0 con 0x7f45441f7270 2024-03-06T07:30:16.853 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:16.854 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":58,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:47.263155+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.853+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f452406b4d0 msgr2=0x7f452406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.853+0000 7f4549600640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f452406b4d0 0x7f452406d990 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f4534004770 tx=0x7f4534008040 comp rx=0 tx=0).stop 2024-03-06T07:30:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.853+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45441f7270 msgr2=0x7f45441f76f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.853+0000 7f4549600640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45441f7270 0x7f45441f76f0 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f453c037040 tx=0x7f453c009750 comp rx=0 tx=0).stop 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 shutdown_connections 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f452406b4d0 0x7f452406d990 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45441f7270 0x7f45441f76f0 unknown :-1 s=CLOSED pgs=714 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45441455d0 0x7f4544145a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 --2- 172.21.15.86:0/208442849 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f454415db10 0x7f4544161cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 >> 172.21.15.86:0/208442849 conn(0x7f454406c0d0 msgr2=0x7f454406e7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 shutdown_connections 2024-03-06T07:30:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:16.854+0000 7f4549600640 1 -- 172.21.15.86:0/208442849 wait complete. 2024-03-06T07:30:16.877 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 8, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:29:43.503869+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '57', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 9, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:29:44.707119+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '58', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:30:16.878 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:30:17.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.086+0000 7fb6ffc00640 1 Processor -- start 2024-03-06T07:30:17.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.086+0000 7fb6ffc00640 1 -- start start 2024-03-06T07:30:17.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.087+0000 7fb6ffc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb6f80b3f70 0x7fb6f8160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6ffc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb6f8070ff0 0x7fb6f80ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6ffc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6f8071440 con 0x7fb6f815fc10 2024-03-06T07:30:17.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6ffc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb6f80b46c0 con 0x7fb6f8070ff0 2024-03-06T07:30:17.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6ffc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb6f80b49a0 con 0x7fb6f80b3f70 2024-03-06T07:30:17.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6fdc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb6f8070ff0 0x7fb6f80ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6fdc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb6f8070ff0 0x7fb6f80ce4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33466/0 (socket says 172.21.15.86:33466) 2024-03-06T07:30:17.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6fdc00640 1 -- 172.21.15.86:0/1301388212 learned_addr learned my addr 172.21.15.86:0/1301388212 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:17.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6f7e00640 1 --2- 172.21.15.86:0/1301388212 >> v2:172.21.15.194:3300/0 conn(0x7fb6f80b3f70 0x7fb6f8160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6fdc00640 1 -- 172.21.15.86:0/1301388212 >> v2:172.21.15.194:3300/0 conn(0x7fb6f80b3f70 msgr2=0x7fb6f8160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:17.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6fdc00640 1 --2- 172.21.15.86:0/1301388212 >> v2:172.21.15.194:3300/0 conn(0x7fb6f80b3f70 0x7fb6f8160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.088+0000 7fb6fdc00640 1 -- 172.21.15.86:0/1301388212 >> v1:172.21.15.194:6789/0 conn(0x7fb6f815fc10 legacy=0x7fb6f815fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:17.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.089+0000 7fb6fdc00640 1 -- 172.21.15.86:0/1301388212 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6f8072c90 con 0x7fb6f8070ff0 2024-03-06T07:30:17.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.090+0000 7fb6fdc00640 1 --2- 172.21.15.86:0/1301388212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f8070ff0 0x7fb6f80ce4a0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7fb6f8074a50 tx=0x7fb6f002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=61c210a6109f6e81 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:17.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.090+0000 7fb6f7400640 1 -- 172.21.15.86:0/1301388212 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f002ca80 con 0x7fb6f8070ff0 2024-03-06T07:30:17.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.090+0000 7fb6f7400640 1 -- 172.21.15.86:0/1301388212 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6f002cc20 con 0x7fb6f8070ff0 2024-03-06T07:30:17.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.090+0000 7fb6f7400640 1 -- 172.21.15.86:0/1301388212 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f00343f0 con 0x7fb6f8070ff0 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 -- 172.21.15.86:0/1301388212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f8070ff0 msgr2=0x7fb6f80ce4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/1301388212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f8070ff0 0x7fb6f80ce4a0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7fb6f8074a50 tx=0x7fb6f002bee0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 -- 172.21.15.86:0/1301388212 shutdown_connections 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/1301388212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f8070ff0 0x7fb6f80ce4a0 unknown :-1 s=CLOSED pgs=715 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/1301388212 >> v2:172.21.15.194:3300/0 conn(0x7fb6f80b3f70 0x7fb6f8160700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 -- 172.21.15.86:0/1301388212 >> 172.21.15.86:0/1301388212 conn(0x7fb6f80b1940 msgr2=0x7fb6f80b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 -- 172.21.15.86:0/1301388212 shutdown_connections 2024-03-06T07:30:17.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.091+0000 7fb6ffc00640 1 -- 172.21.15.86:0/1301388212 wait complete. 2024-03-06T07:30:17.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.093+0000 7fb6ffc00640 1 Processor -- start 2024-03-06T07:30:17.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.093+0000 7fb6ffc00640 1 -- start start 2024-03-06T07:30:17.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6ffc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6f8070ff0 0x7fb6f809a9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6ffc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6f80b3f70 0x7fb6f81481a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6ffc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f815fc10 0x7fb6f815b6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6ffc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6f8072a30 con 0x7fb6f815fc10 2024-03-06T07:30:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6ffc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6f80728b0 con 0x7fb6f8070ff0 2024-03-06T07:30:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6ffc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6f815c2e0 con 0x7fb6f80b3f70 2024-03-06T07:30:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fdc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f815fc10 0x7fb6f815b6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fdc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f815fc10 0x7fb6f815b6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33470/0 (socket says 172.21.15.86:33470) 2024-03-06T07:30:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fdc00640 1 -- 172.21.15.86:0/2066442431 learned_addr learned my addr 172.21.15.86:0/2066442431 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:17.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fd200640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6f8070ff0 0x7fb6f809a9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6f7e00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6f80b3f70 0x7fb6f81481a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fdc00640 1 -- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6f80b3f70 msgr2=0x7fb6f81481a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fdc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6f80b3f70 0x7fb6f81481a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fdc00640 1 -- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6f8070ff0 msgr2=0x7fb6f809a9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.094+0000 7fb6fdc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6f8070ff0 0x7fb6f809a9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.095+0000 7fb6fdc00640 1 -- 172.21.15.86:0/2066442431 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6f809c780 con 0x7fb6f815fc10 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.095+0000 7fb6f7e00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6f80b3f70 0x7fb6f81481a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.095+0000 7fb6fdc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f815fc10 0x7fb6f815b6b0 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7fb6f0036040 tx=0x7fb6f0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.095+0000 7fb6f4c00640 1 -- 172.21.15.86:0/2066442431 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f0043020 con 0x7fb6f815fc10 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.095+0000 7fb6f4c00640 1 -- 172.21.15.86:0/2066442431 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6f003e450 con 0x7fb6f815fc10 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.095+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6f809ca60 con 0x7fb6f815fc10 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.095+0000 7fb6f4c00640 1 -- 172.21.15.86:0/2066442431 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f0009510 con 0x7fb6f815fc10 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.096+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6f809c470 con 0x7fb6f815fc10 2024-03-06T07:30:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.097+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb6f804f330 con 0x7fb6f815fc10 2024-03-06T07:30:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.099+0000 7fb6f4c00640 1 -- 172.21.15.86:0/2066442431 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb6f0046020 con 0x7fb6f815fc10 2024-03-06T07:30:17.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.100+0000 7fb6f4c00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6d806b4d0 0x7fb6d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.100+0000 7fb6f4c00640 1 -- 172.21.15.86:0/2066442431 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb6f00accd0 con 0x7fb6f815fc10 2024-03-06T07:30:17.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.100+0000 7fb6fd200640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6d806b4d0 0x7fb6d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.101+0000 7fb6fd200640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6d806b4d0 0x7fb6d806d990 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7fb6ec006fd0 tx=0x7fb6ec008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:17.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.105+0000 7fb6f4c00640 1 -- 172.21.15.86:0/2066442431 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb6f004d020 con 0x7fb6f815fc10 2024-03-06T07:30:17.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.303+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fb6f80cf4a0 con 0x7fb6f815fc10 2024-03-06T07:30:17.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.304+0000 7fb6f4c00640 1 -- 172.21.15.86:0/2066442431 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v58) v1 ==== 93+0+10 (secure 0 0 0) 0x7fb6f007bbe0 con 0x7fb6f815fc10 2024-03-06T07:30:17.307 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6d806b4d0 msgr2=0x7fb6d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6d806b4d0 0x7fb6d806d990 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7fb6ec006fd0 tx=0x7fb6ec008040 comp rx=0 tx=0).stop 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f815fc10 msgr2=0x7fb6f815b6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f815fc10 0x7fb6f815b6b0 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7fb6f0036040 tx=0x7fb6f0004a40 comp rx=0 tx=0).stop 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 shutdown_connections 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6d806b4d0 0x7fb6d806d990 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6f815fc10 0x7fb6f815b6b0 unknown :-1 s=CLOSED pgs=716 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6f80b3f70 0x7fb6f81481a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 --2- 172.21.15.86:0/2066442431 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6f8070ff0 0x7fb6f809a9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 >> 172.21.15.86:0/2066442431 conn(0x7fb6f80b1940 msgr2=0x7fb6f80b0650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 shutdown_connections 2024-03-06T07:30:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.308+0000 7fb6ffc00640 1 -- 172.21.15.86:0/2066442431 wait complete. 2024-03-06T07:30:17.338 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:30:17.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.562+0000 7f4b1ce00640 1 Processor -- start 2024-03-06T07:30:17.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.562+0000 7f4b1ce00640 1 -- start start 2024-03-06T07:30:17.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.563+0000 7f4b1ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4b1815e130 0x7f4b180cf5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.563+0000 7f4b1ce00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4b180cfaf0 0x7f4b180d1ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.563+0000 7f4b1ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180d23e0 0x7f4b180c7970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.563+0000 7f4b1ce00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b180cf120 con 0x7f4b1815e130 2024-03-06T07:30:17.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.563+0000 7f4b1ce00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b180d13e0 con 0x7f4b180cfaf0 2024-03-06T07:30:17.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.563+0000 7f4b1ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b180cead0 con 0x7f4b180d23e0 2024-03-06T07:30:17.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.563+0000 7f4b17400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4b1815e130 0x7f4b180cf5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4b1815e130 0x7f4b180cf5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33474/0 (socket says 172.21.15.86:33474) 2024-03-06T07:30:17.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17400640 1 -- 172.21.15.86:0/899185722 learned_addr learned my addr 172.21.15.86:0/899185722 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:17.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b0ea00640 1 --2- 172.21.15.86:0/899185722 >> v2:172.21.15.194:3300/0 conn(0x7f4b180cfaf0 0x7f4b180d1ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17e00640 1 --2- 172.21.15.86:0/899185722 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180d23e0 0x7f4b180c7970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17400640 1 -- 172.21.15.86:0/899185722 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180d23e0 msgr2=0x7f4b180c7970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:17.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17400640 1 --2- 172.21.15.86:0/899185722 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180d23e0 0x7f4b180c7970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17400640 1 -- 172.21.15.86:0/899185722 >> v2:172.21.15.194:3300/0 conn(0x7f4b180cfaf0 msgr2=0x7f4b180d1ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:17.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17400640 1 --2- 172.21.15.86:0/899185722 >> v2:172.21.15.194:3300/0 conn(0x7f4b180cfaf0 0x7f4b180d1ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.564+0000 7f4b17400640 1 -- 172.21.15.86:0/899185722 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b180cedb0 con 0x7f4b1815e130 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.565+0000 7f4b17400640 1 --2- 172.21.15.86:0/899185722 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cf5b0 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f4b180b9ed0 tx=0x7f4b0802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=faaa043ef5554a77 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.565+0000 7f4b16a00640 1 -- 172.21.15.86:0/899185722 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b08035020 con 0x7f4b1815e130 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b16a00640 1 -- 172.21.15.86:0/899185722 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b08031870 con 0x7f4b1815e130 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b16a00640 1 -- 172.21.15.86:0/899185722 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b08031c40 con 0x7f4b1815e130 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 -- 172.21.15.86:0/899185722 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 msgr2=0x7f4b180cf5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/899185722 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cf5b0 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f4b180b9ed0 tx=0x7f4b0802b910 comp rx=0 tx=0).stop 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 -- 172.21.15.86:0/899185722 shutdown_connections 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/899185722 >> v2:172.21.15.194:3300/0 conn(0x7f4b180cfaf0 0x7f4b180d1ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/899185722 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180d23e0 0x7f4b180c7970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/899185722 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cf5b0 unknown :-1 s=CLOSED pgs=717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 -- 172.21.15.86:0/899185722 >> 172.21.15.86:0/899185722 conn(0x7f4b1809d070 msgr2=0x7f4b180d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:17.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 -- 172.21.15.86:0/899185722 shutdown_connections 2024-03-06T07:30:17.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.566+0000 7f4b1ce00640 1 -- 172.21.15.86:0/899185722 wait complete. 2024-03-06T07:30:17.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.568+0000 7f4b1ce00640 1 Processor -- start 2024-03-06T07:30:17.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.568+0000 7f4b1ce00640 1 -- start start 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.569+0000 7f4b1ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180cfaf0 0x7f4b1814f270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.569+0000 7f4b1ce00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b180d23e0 0x7f4b180b8aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.569+0000 7f4b1ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cc560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.569+0000 7f4b1ce00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b180c8030 con 0x7f4b1815e130 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.569+0000 7f4b1ce00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b180c7eb0 con 0x7f4b180d23e0 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.569+0000 7f4b1ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b180c81b0 con 0x7f4b180cfaf0 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cc560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cc560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33476/0 (socket says 172.21.15.86:33476) 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 -- 172.21.15.86:0/825784080 learned_addr learned my addr 172.21.15.86:0/825784080 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17400640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180cfaf0 0x7f4b1814f270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b0ea00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b180d23e0 0x7f4b180b8aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 -- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180cfaf0 msgr2=0x7f4b1814f270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180cfaf0 0x7f4b1814f270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 -- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b180d23e0 msgr2=0x7f4b180b8aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b180d23e0 0x7f4b180b8aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 -- 172.21.15.86:0/825784080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b1814f7b0 con 0x7f4b1815e130 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.570+0000 7f4b17e00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cc560 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7f4b1000cc30 tx=0x7f4b1000a5c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.571+0000 7f4b0fe00640 1 -- 172.21.15.86:0/825784080 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b1000ce40 con 0x7f4b1815e130 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.571+0000 7f4b0fe00640 1 -- 172.21.15.86:0/825784080 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b10003990 con 0x7f4b1815e130 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.571+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b180b7970 con 0x7f4b1815e130 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.571+0000 7f4b0fe00640 1 -- 172.21.15.86:0/825784080 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b10003d40 con 0x7f4b1815e130 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.571+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b180b7f30 con 0x7f4b1815e130 2024-03-06T07:30:17.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.572+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b1815ec50 con 0x7f4b1815e130 2024-03-06T07:30:17.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.574+0000 7f4b0fe00640 1 -- 172.21.15.86:0/825784080 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b10003ee0 con 0x7f4b1815e130 2024-03-06T07:30:17.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.575+0000 7f4b0fe00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4af406b4d0 0x7f4af406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:17.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.575+0000 7f4b0fe00640 1 -- 172.21.15.86:0/825784080 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4b1008a670 con 0x7f4b1815e130 2024-03-06T07:30:17.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.575+0000 7f4b17400640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4af406b4d0 0x7f4af406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:17.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.576+0000 7f4b17400640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4af406b4d0 0x7f4af406d990 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f4b08036000 tx=0x7f4b0803c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:17.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.580+0000 7f4b0fe00640 1 -- 172.21.15.86:0/825784080 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b1814f7b0 con 0x7f4b1815e130 2024-03-06T07:30:17.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.778+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f4b1804f330 con 0x7f4b1815e130 2024-03-06T07:30:17.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.779+0000 7f4b0fe00640 1 -- 172.21.15.86:0/825784080 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v58) v1 ==== 104+0+11 (secure 0 0 0) 0x7f4b10014030 con 0x7f4b1815e130 2024-03-06T07:30:17.782 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4af406b4d0 msgr2=0x7f4af406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4af406b4d0 0x7f4af406d990 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f4b08036000 tx=0x7f4b0803c040 comp rx=0 tx=0).stop 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 msgr2=0x7f4b180cc560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cc560 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7f4b1000cc30 tx=0x7f4b1000a5c0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 shutdown_connections 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4af406b4d0 0x7f4af406d990 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b1815e130 0x7f4b180cc560 unknown :-1 s=CLOSED pgs=718 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b180d23e0 0x7f4b180b8aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 --2- 172.21.15.86:0/825784080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b180cfaf0 0x7f4b1814f270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 >> 172.21.15.86:0/825784080 conn(0x7f4b1809d070 msgr2=0x7f4b18161a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 shutdown_connections 2024-03-06T07:30:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.781+0000 7f4b1ce00640 1 -- 172.21.15.86:0/825784080 wait complete. 2024-03-06T07:30:17.801 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:30:18.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:17.999+0000 7f3930000640 1 Processor -- start 2024-03-06T07:30:18.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.000+0000 7f3930000640 1 -- start start 2024-03-06T07:30:18.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.001+0000 7f3930000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f39280b3df0 0x7f39280727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.001+0000 7f3930000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.001+0000 7f3930000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39280719e0 con 0x7f39280b3650 2024-03-06T07:30:18.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.001+0000 7f3930000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3928072310 con 0x7f39280b3df0 2024-03-06T07:30:18.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.001+0000 7f3930000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3928071cc0 con 0x7f39280cda90 2024-03-06T07:30:18.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.002+0000 7f392e000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.002+0000 7f392cc00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f39280b3650 0x7f39280b41d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:45926/0 (socket says 172.21.15.86:45926) 2024-03-06T07:30:18.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.002+0000 7f392cc00640 1 -- 172.21.15.86:0/2841329067 learned_addr learned my addr 172.21.15.86:0/2841329067 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:18.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.002+0000 7f392d600640 1 --2- 172.21.15.86:0/2841329067 >> v2:172.21.15.194:3300/0 conn(0x7f39280b3df0 0x7f39280727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.003+0000 7f392d600640 1 -- 172.21.15.86:0/2841329067 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 msgr2=0x7f392806c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:18.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.003+0000 7f392d600640 1 --2- 172.21.15.86:0/2841329067 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806c990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.003+0000 7f392d600640 1 -- 172.21.15.86:0/2841329067 >> v1:172.21.15.194:6789/0 conn(0x7f39280b3650 legacy=0x7f39280b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.003+0000 7f392d600640 1 -- 172.21.15.86:0/2841329067 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3928071fa0 con 0x7f39280b3df0 2024-03-06T07:30:18.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.004+0000 7f392d600640 1 --2- 172.21.15.86:0/2841329067 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3df0 0x7f39280727a0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f391c0097a0 tx=0x7f391c02b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=562917bfef097c7a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.006+0000 7f3927e00640 1 -- 172.21.15.86:0/2841329067 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f391c035020 con 0x7f39280b3df0 2024-03-06T07:30:18.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.007+0000 7f3927e00640 1 -- 172.21.15.86:0/2841329067 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f391c031810 con 0x7f39280b3df0 2024-03-06T07:30:18.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.007+0000 7f3927e00640 1 -- 172.21.15.86:0/2841329067 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f391c031be0 con 0x7f39280b3df0 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.007+0000 7f3930000640 1 -- 172.21.15.86:0/2841329067 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3df0 msgr2=0x7f39280727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.007+0000 7f3930000640 1 --2- 172.21.15.86:0/2841329067 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3df0 0x7f39280727a0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f391c0097a0 tx=0x7f391c02b8b0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.008+0000 7f3930000640 1 -- 172.21.15.86:0/2841329067 shutdown_connections 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.008+0000 7f3930000640 1 --2- 172.21.15.86:0/2841329067 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.008+0000 7f3930000640 1 --2- 172.21.15.86:0/2841329067 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3df0 0x7f39280727a0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.008+0000 7f3930000640 1 -- 172.21.15.86:0/2841329067 >> 172.21.15.86:0/2841329067 conn(0x7f39280b1940 msgr2=0x7f39280b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.008+0000 7f3930000640 1 -- 172.21.15.86:0/2841329067 shutdown_connections 2024-03-06T07:30:18.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.008+0000 7f3930000640 1 -- 172.21.15.86:0/2841329067 wait complete. 2024-03-06T07:30:18.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.011+0000 7f3930000640 1 Processor -- start 2024-03-06T07:30:18.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.011+0000 7f3930000640 1 -- start start 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.012+0000 7f3930000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3650 0x7f392815daf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.012+0000 7f3930000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39280b3df0 0x7f392814e900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.012+0000 7f3930000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.012+0000 7f3930000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f392815b3c0 con 0x7f39280b3df0 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.012+0000 7f3930000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f392815b240 con 0x7f39280b3650 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.012+0000 7f3930000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39280cf510 con 0x7f39280cda90 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39280b3df0 0x7f392814e900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39280b3df0 0x7f392814e900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33482/0 (socket says 172.21.15.86:33482) 2024-03-06T07:30:18.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 -- 172.21.15.86:0/3958694769 learned_addr learned my addr 172.21.15.86:0/3958694769 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392d600640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3650 0x7f392815daf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392e000640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 -- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 msgr2=0x7f392806d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806d9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 -- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3650 msgr2=0x7f392815daf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3650 0x7f392815daf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.013+0000 7f392cc00640 1 -- 172.21.15.86:0/3958694769 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f392814ee40 con 0x7f39280b3df0 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.014+0000 7f392cc00640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39280b3df0 0x7f392814e900 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7f391400c3c0 tx=0x7f391400c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.014+0000 7f3925600640 1 -- 172.21.15.86:0/3958694769 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3914017070 con 0x7f39280b3df0 2024-03-06T07:30:18.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.014+0000 7f3925600640 1 -- 172.21.15.86:0/3958694769 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3914005ce0 con 0x7f39280b3df0 2024-03-06T07:30:18.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.014+0000 7f3925600640 1 -- 172.21.15.86:0/3958694769 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39140072f0 con 0x7f39280b3df0 2024-03-06T07:30:18.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.014+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39280ceed0 con 0x7f39280b3df0 2024-03-06T07:30:18.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.014+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f392815c210 con 0x7f39280b3df0 2024-03-06T07:30:18.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.016+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f39280b4ab0 con 0x7f39280b3df0 2024-03-06T07:30:18.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.016+0000 7f3925600640 1 -- 172.21.15.86:0/3958694769 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f391401a070 con 0x7f39280b3df0 2024-03-06T07:30:18.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.017+0000 7f3925600640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.017+0000 7f3925600640 1 -- 172.21.15.86:0/3958694769 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3914089600 con 0x7f39280b3df0 2024-03-06T07:30:18.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.017+0000 7f392d600640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.018+0000 7f392d600640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f391c036000 tx=0x7f391c03c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.022+0000 7f3925600640 1 -- 172.21.15.86:0/3958694769 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f39280b4ab0 con 0x7f39280b3df0 2024-03-06T07:30:18.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.215+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f392804f330 con 0x7f39280b3df0 2024-03-06T07:30:18.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.215+0000 7f3925600640 1 -- 172.21.15.86:0/3958694769 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v58) v1 ==== 100+0+11 (secure 0 0 0) 0x7f39140583e0 con 0x7f39280b3df0 2024-03-06T07:30:18.218 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:30:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 msgr2=0x7f390806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f391c036000 tx=0x7f391c03c040 comp rx=0 tx=0).stop 2024-03-06T07:30:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39280b3df0 msgr2=0x7f392814e900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39280b3df0 0x7f392814e900 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7f391400c3c0 tx=0x7f391400c880 comp rx=0 tx=0).stop 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 shutdown_connections 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39280cda90 0x7f392806d9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39280b3df0 0x7f392814e900 unknown :-1 s=CLOSED pgs=719 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 --2- 172.21.15.86:0/3958694769 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39280b3650 0x7f392815daf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 >> 172.21.15.86:0/3958694769 conn(0x7f39280b1940 msgr2=0x7f39280b1230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 shutdown_connections 2024-03-06T07:30:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.218+0000 7f3930000640 1 -- 172.21.15.86:0/3958694769 wait complete. 2024-03-06T07:30:18.238 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:18.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.431+0000 7fb094c00640 1 Processor -- start 2024-03-06T07:30:18.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.431+0000 7fb094c00640 1 -- start start 2024-03-06T07:30:18.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.432+0000 7fb094c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0900b4680 0x7fb09006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb094c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb090072340 0x7fb09006ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb094c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb09015b090 con 0x7fb09015acb0 2024-03-06T07:30:18.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb094c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0900b2a30 con 0x7fb090072340 2024-03-06T07:30:18.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb094c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0900b4360 con 0x7fb0900b4680 2024-03-06T07:30:18.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb090072340 0x7fb09006ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08e800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb090072340 0x7fb09006ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33498/0 (socket says 172.21.15.86:33498) 2024-03-06T07:30:18.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08e800640 1 -- 172.21.15.86:0/730677233 learned_addr learned my addr 172.21.15.86:0/730677233 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:18.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08d400640 1 --2- 172.21.15.86:0/730677233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0900b4680 0x7fb09006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08e800640 1 -- 172.21.15.86:0/730677233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0900b4680 msgr2=0x7fb09006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:18.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08e800640 1 --2- 172.21.15.86:0/730677233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0900b4680 0x7fb09006e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08e800640 1 -- 172.21.15.86:0/730677233 >> v1:172.21.15.194:6789/0 conn(0x7fb09015acb0 legacy=0x7fb0900b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:18.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08e800640 1 -- 172.21.15.86:0/730677233 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0900b2cf0 con 0x7fb090072340 2024-03-06T07:30:18.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.433+0000 7fb08d400640 1 --2- 172.21.15.86:0/730677233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0900b4680 0x7fb09006e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.434+0000 7fb08e800640 1 --2- 172.21.15.86:0/730677233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb090072340 0x7fb09006ea30 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7fb088009e50 tx=0x7fb08802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e7b49f18e7a38809 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb08ca00640 1 -- 172.21.15.86:0/730677233 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb08802ca80 con 0x7fb090072340 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb08ca00640 1 -- 172.21.15.86:0/730677233 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb08802cc20 con 0x7fb090072340 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb08ca00640 1 -- 172.21.15.86:0/730677233 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0880343f0 con 0x7fb090072340 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb094c00640 1 -- 172.21.15.86:0/730677233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb090072340 msgr2=0x7fb09006ea30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb094c00640 1 --2- 172.21.15.86:0/730677233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb090072340 0x7fb09006ea30 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7fb088009e50 tx=0x7fb08802bee0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb094c00640 1 -- 172.21.15.86:0/730677233 shutdown_connections 2024-03-06T07:30:18.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb094c00640 1 --2- 172.21.15.86:0/730677233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb090072340 0x7fb09006ea30 unknown :-1 s=CLOSED pgs=720 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb094c00640 1 --2- 172.21.15.86:0/730677233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0900b4680 0x7fb09006e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb094c00640 1 -- 172.21.15.86:0/730677233 >> 172.21.15.86:0/730677233 conn(0x7fb0900b1730 msgr2=0x7fb0900b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:18.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.435+0000 7fb094c00640 1 -- 172.21.15.86:0/730677233 shutdown_connections 2024-03-06T07:30:18.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.436+0000 7fb094c00640 1 -- 172.21.15.86:0/730677233 wait complete. 2024-03-06T07:30:18.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.437+0000 7fb094c00640 1 Processor -- start 2024-03-06T07:30:18.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.437+0000 7fb094c00640 1 -- start start 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.438+0000 7fb094c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb090072340 0x7fb09014e420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.438+0000 7fb094c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0900b4680 0x7fb09014c930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.438+0000 7fb094c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb09015acb0 0x7fb09015d4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.438+0000 7fb094c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0900b4010 con 0x7fb0900b4680 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.438+0000 7fb094c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0900b32b0 con 0x7fb090072340 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.438+0000 7fb094c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0900ccfb0 con 0x7fb09015acb0 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0900b4680 0x7fb09014c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0900b4680 0x7fb09014c930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33508/0 (socket says 172.21.15.86:33508) 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 -- 172.21.15.86:0/3184859684 learned_addr learned my addr 172.21.15.86:0/3184859684 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08e800640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb09015acb0 0x7fb09015d4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 -- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb09015acb0 msgr2=0x7fb09015d4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08de00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb090072340 0x7fb09014e420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb09015acb0 0x7fb09015d4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 -- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb090072340 msgr2=0x7fb09014e420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb090072340 0x7fb09014e420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 -- 172.21.15.86:0/3184859684 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb09015d9f0 con 0x7fb0900b4680 2024-03-06T07:30:18.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08e800640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb09015acb0 0x7fb09015d4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:18.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08de00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb090072340 0x7fb09014e420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:18.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.439+0000 7fb08d400640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0900b4680 0x7fb09014c930 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7fb07c00c7f0 tx=0x7fb07c00cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.440+0000 7fb086000640 1 -- 172.21.15.86:0/3184859684 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb07c015070 con 0x7fb0900b4680 2024-03-06T07:30:18.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.440+0000 7fb086000640 1 -- 172.21.15.86:0/3184859684 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb07c007a30 con 0x7fb0900b4680 2024-03-06T07:30:18.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.440+0000 7fb086000640 1 -- 172.21.15.86:0/3184859684 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb07c007e00 con 0x7fb0900b4680 2024-03-06T07:30:18.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.440+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb09015bec0 con 0x7fb0900b4680 2024-03-06T07:30:18.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.440+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb09015c480 con 0x7fb0900b4680 2024-03-06T07:30:18.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.441+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb09004f330 con 0x7fb0900b4680 2024-03-06T07:30:18.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.442+0000 7fb086000640 1 -- 172.21.15.86:0/3184859684 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb07c018070 con 0x7fb0900b4680 2024-03-06T07:30:18.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.443+0000 7fb086000640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb07006b4d0 0x7fb07006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.443+0000 7fb086000640 1 -- 172.21.15.86:0/3184859684 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb07c08a210 con 0x7fb0900b4680 2024-03-06T07:30:18.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.443+0000 7fb08de00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb07006b4d0 0x7fb07006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.444+0000 7fb08de00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb07006b4d0 0x7fb07006d990 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fb09015d3e0 tx=0x7fb080008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.452+0000 7fb086000640 1 -- 172.21.15.86:0/3184859684 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb07c058ff0 con 0x7fb0900b4680 2024-03-06T07:30:18.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.685+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb09006ce60 con 0x7fb0900b4680 2024-03-06T07:30:18.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.687+0000 7fb086000640 1 -- 172.21.15.86:0/3184859684 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fb07c0588f0 con 0x7fb0900b4680 2024-03-06T07:30:18.689 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:18.689 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:30:18.690 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb07006b4d0 msgr2=0x7fb07006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb07006b4d0 0x7fb07006d990 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fb09015d3e0 tx=0x7fb080008040 comp rx=0 tx=0).stop 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0900b4680 msgr2=0x7fb09014c930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0900b4680 0x7fb09014c930 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7fb07c00c7f0 tx=0x7fb07c00cbf0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 shutdown_connections 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb07006b4d0 0x7fb07006d990 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb09015acb0 0x7fb09015d4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0900b4680 0x7fb09014c930 unknown :-1 s=CLOSED pgs=721 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 --2- 172.21.15.86:0/3184859684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb090072340 0x7fb09014e420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.689+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 >> 172.21.15.86:0/3184859684 conn(0x7fb0900b1730 msgr2=0x7fb090156e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:18.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.690+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 shutdown_connections 2024-03-06T07:30:18.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.690+0000 7fb094c00640 1 -- 172.21.15.86:0/3184859684 wait complete. 2024-03-06T07:30:18.711 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:18.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.908+0000 7fe1e1600640 1 Processor -- start 2024-03-06T07:30:18.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.909+0000 7fe1e1600640 1 -- start start 2024-03-06T07:30:18.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.910+0000 7fe1e1600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe1dc0d0fc0 0x7fe1dc160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.910+0000 7fe1e1600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe1dc0cea60 con 0x7fe1dc15e0d0 2024-03-06T07:30:18.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.910+0000 7fe1e1600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe1dc0d2350 con 0x7fe1dc15f470 2024-03-06T07:30:18.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.910+0000 7fe1e1600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe1dc0d2f60 con 0x7fe1dc0d0fc0 2024-03-06T07:30:18.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.910+0000 7fe1daa00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe1dc15e0d0 0x7fe1dc0d1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37382/0 (socket says 172.21.15.86:37382) 2024-03-06T07:30:18.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.910+0000 7fe1daa00640 1 -- 172.21.15.86:0/417955250 learned_addr learned my addr 172.21.15.86:0/417955250 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1370476378 0 0) 0x7fe1dc0cea60 con 0x7fe1dc15e0d0 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe1c0003520 con 0x7fe1dc15e0d0 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1db400640 1 --2- 172.21.15.86:0/417955250 >> v2:172.21.15.194:3300/0 conn(0x7fe1dc0d0fc0 0x7fe1dc160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1074777884 0 0) 0x7fe1c0003520 con 0x7fe1dc15e0d0 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 >> v2:172.21.15.194:3300/0 conn(0x7fe1dc0d0fc0 msgr2=0x7fe1dc160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 --2- 172.21.15.86:0/417955250 >> v2:172.21.15.194:3300/0 conn(0x7fe1dc0d0fc0 0x7fe1dc160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 >> v1:172.21.15.194:6789/0 conn(0x7fe1dc15f470 legacy=0x7fe1dc0d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1dc0d2630 con 0x7fe1dc15e0d0 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fe1cc001670 con 0x7fe1dc15e0d0 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 >> v1:172.21.15.86:6789/0 conn(0x7fe1dc15e0d0 legacy=0x7fe1dc0d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1c00039b0 0x7fe1c0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1c0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1c00051c0 0x7fe1c000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1c000c0c0 con 0x7fe1c0004800 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1c000c270 con 0x7fe1c00039b0 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.911+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe1c000c470 con 0x7fe1c00051c0 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1d3e00640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1c0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1daa00640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1c00039b0 0x7fe1c0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1d3e00640 1 -- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1c00051c0 msgr2=0x7fe1c000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1d3e00640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1c00051c0 0x7fe1c000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1d3e00640 1 -- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1c00039b0 msgr2=0x7fe1c0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1d3e00640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1c00039b0 0x7fe1c0003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1db400640 1 -- 172.21.15.86:0/417955250 reap_dead start 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1d3e00640 1 -- 172.21.15.86:0/417955250 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1c0003520 con 0x7fe1c0004800 2024-03-06T07:30:18.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.912+0000 7fe1daa00640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1c00039b0 0x7fe1c0003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:18.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.913+0000 7fe1d3e00640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1c0004c80 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7fe1c4001720 tx=0x7fe1c0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=225cd304eed0f358 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.913+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe1c4001370 con 0x7fe1c0004800 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.913+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe1c40052d0 con 0x7fe1c0004800 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.913+0000 7fe1da000640 1 -- 172.21.15.86:0/417955250 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe1c4003830 con 0x7fe1c0004800 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.914+0000 7fe1e1600640 1 -- 172.21.15.86:0/417955250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 msgr2=0x7fe1c0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.914+0000 7fe1e1600640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1c0004c80 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7fe1c4001720 tx=0x7fe1c0005830 comp rx=0 tx=0).stop 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.914+0000 7fe1e1600640 1 -- 172.21.15.86:0/417955250 shutdown_connections 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.914+0000 7fe1e1600640 1 --2- 172.21.15.86:0/417955250 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1c0004c80 unknown :-1 s=CLOSED pgs=723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.914+0000 7fe1e1600640 1 -- 172.21.15.86:0/417955250 >> 172.21.15.86:0/417955250 conn(0x7fe1dc0d3ec0 msgr2=0x7fe1dc0d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.914+0000 7fe1e1600640 1 -- 172.21.15.86:0/417955250 shutdown_connections 2024-03-06T07:30:18.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.914+0000 7fe1e1600640 1 -- 172.21.15.86:0/417955250 wait complete. 2024-03-06T07:30:18.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.916+0000 7fe1e1600640 1 Processor -- start 2024-03-06T07:30:18.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.916+0000 7fe1e1600640 1 -- start start 2024-03-06T07:30:18.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1e1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1dc0b6ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1e1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1dc0b8ec0 0x7fe1dc0b9320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1e1600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1dc0c9840 0x7fe1dc0c9cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1e1600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1dc15fb30 con 0x7fe1c0004800 2024-03-06T07:30:18.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1e1600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1dc0d2bf0 con 0x7fe1dc0c9840 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1e1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe1dc15e4b0 con 0x7fe1dc0b8ec0 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1daa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1dc0b6ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1daa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1dc0b6ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33534/0 (socket says 172.21.15.86:33534) 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.917+0000 7fe1daa00640 1 -- 172.21.15.86:0/3829612838 learned_addr learned my addr 172.21.15.86:0/3829612838 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1daa00640 1 -- 172.21.15.86:0/3829612838 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1dc0b8ec0 msgr2=0x7fe1dc0b9320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1daa00640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1dc0b8ec0 0x7fe1dc0b9320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1daa00640 1 -- 172.21.15.86:0/3829612838 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1dc0c9840 msgr2=0x7fe1dc0c9cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1daa00640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1dc0c9840 0x7fe1dc0c9cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1daa00640 1 -- 172.21.15.86:0/3829612838 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1dc0b70e0 con 0x7fe1c0004800 2024-03-06T07:30:18.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1daa00640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1dc0b6ae0 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7fe1cc00d7a0 tx=0x7fe1cc00dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1d2a00640 1 -- 172.21.15.86:0/3829612838 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe1cc019070 con 0x7fe1c0004800 2024-03-06T07:30:18.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1d2a00640 1 -- 172.21.15.86:0/3829612838 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe1cc0074e0 con 0x7fe1c0004800 2024-03-06T07:30:18.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1d2a00640 1 -- 172.21.15.86:0/3829612838 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe1cc004a20 con 0x7fe1c0004800 2024-03-06T07:30:18.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe1dc0b73c0 con 0x7fe1c0004800 2024-03-06T07:30:18.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.918+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe1dc0c75f0 con 0x7fe1c0004800 2024-03-06T07:30:18.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.919+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe1dc04f330 con 0x7fe1c0004800 2024-03-06T07:30:18.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.921+0000 7fe1d2a00640 1 -- 172.21.15.86:0/3829612838 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe1cc004bc0 con 0x7fe1c0004800 2024-03-06T07:30:18.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.922+0000 7fe1d2a00640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe1b806b4d0 0x7fe1b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:18.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.922+0000 7fe1d2a00640 1 -- 172.21.15.86:0/3829612838 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe1cc0899d0 con 0x7fe1c0004800 2024-03-06T07:30:18.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.923+0000 7fe1d3e00640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe1b806b4d0 0x7fe1b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:18.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.923+0000 7fe1d3e00640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe1b806b4d0 0x7fe1b806d990 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7fe1c4033000 tx=0x7fe1c402f000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:18.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:18.927+0000 7fe1d2a00640 1 -- 172.21.15.86:0/3829612838 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe1cc058270 con 0x7fe1c0004800 2024-03-06T07:30:19.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.127+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe1dc0ce2d0 con 0x7fe1c0004800 2024-03-06T07:30:19.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.130+0000 7fe1d2a00640 1 -- 172.21.15.86:0/3829612838 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v58) v1 ==== 74+0+16677 (secure 0 0 0) 0x7fe1cc05c310 con 0x7fe1c0004800 2024-03-06T07:30:19.133 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:19.133 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":58,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:47.263155+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:19.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.132+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe1b806b4d0 msgr2=0x7fe1b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.132+0000 7fe1e1600640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe1b806b4d0 0x7fe1b806d990 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7fe1c4033000 tx=0x7fe1c402f000 comp rx=0 tx=0).stop 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 msgr2=0x7fe1dc0b6ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1dc0b6ae0 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7fe1cc00d7a0 tx=0x7fe1cc00dc60 comp rx=0 tx=0).stop 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 shutdown_connections 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe1b806b4d0 0x7fe1b806d990 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1dc0c9840 0x7fe1dc0c9cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1dc0b8ec0 0x7fe1dc0b9320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 --2- 172.21.15.86:0/3829612838 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1c0004800 0x7fe1dc0b6ae0 unknown :-1 s=CLOSED pgs=724 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 >> 172.21.15.86:0/3829612838 conn(0x7fe1dc0d3ec0 msgr2=0x7fe1dc0b3820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 shutdown_connections 2024-03-06T07:30:19.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.133+0000 7fe1e1600640 1 -- 172.21.15.86:0/3829612838 wait complete. 2024-03-06T07:30:19.161 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:30:19.161 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:19.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.363+0000 7fe732000640 1 Processor -- start 2024-03-06T07:30:19.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.363+0000 7fe732000640 1 -- start start 2024-03-06T07:30:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe732000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c15e160 0x7fe72c162620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe732000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe72c0af820 0x7fe72c0b33b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe732000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe72c0b38f0 0x7fe72c06d3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe732000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe72c06b0f0 con 0x7fe72c0b38f0 2024-03-06T07:30:19.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe732000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe72c0b5620 con 0x7fe72c0af820 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe732000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe72c162b60 con 0x7fe72c15e160 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe72be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe72c0b38f0 0x7fe72c06d3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe72be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe72c0b38f0 0x7fe72c06d3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33540/0 (socket says 172.21.15.86:33540) 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.365+0000 7fe72be00640 1 -- 172.21.15.86:0/4206165811 learned_addr learned my addr 172.21.15.86:0/4206165811 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.366+0000 7fe72b400640 1 --2- 172.21.15.86:0/4206165811 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c15e160 0x7fe72c162620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.366+0000 7fe723e00640 1 --2- 172.21.15.86:0/4206165811 >> v2:172.21.15.194:3300/0 conn(0x7fe72c0af820 0x7fe72c0b33b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.366+0000 7fe72be00640 1 -- 172.21.15.86:0/4206165811 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c15e160 msgr2=0x7fe72c162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.366+0000 7fe72be00640 1 --2- 172.21.15.86:0/4206165811 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c15e160 0x7fe72c162620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.366+0000 7fe72be00640 1 -- 172.21.15.86:0/4206165811 >> v2:172.21.15.194:3300/0 conn(0x7fe72c0af820 msgr2=0x7fe72c0b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.366+0000 7fe72be00640 1 --2- 172.21.15.86:0/4206165811 >> v2:172.21.15.194:3300/0 conn(0x7fe72c0af820 0x7fe72c0b33b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.366+0000 7fe72be00640 1 -- 172.21.15.86:0/4206165811 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe72c0b3d70 con 0x7fe72c0b38f0 2024-03-06T07:30:19.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.367+0000 7fe72be00640 1 --2- 172.21.15.86:0/4206165811 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c0b38f0 0x7fe72c06d3a0 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7fe72c067a50 tx=0x7fe72402be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e538853a2e8d574 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:19.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.367+0000 7fe72aa00640 1 -- 172.21.15.86:0/4206165811 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe72402c9e0 con 0x7fe72c0b38f0 2024-03-06T07:30:19.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.367+0000 7fe72aa00640 1 -- 172.21.15.86:0/4206165811 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe72402cb80 con 0x7fe72c0b38f0 2024-03-06T07:30:19.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.367+0000 7fe72aa00640 1 -- 172.21.15.86:0/4206165811 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe7240343f0 con 0x7fe72c0b38f0 2024-03-06T07:30:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 -- 172.21.15.86:0/4206165811 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c0b38f0 msgr2=0x7fe72c06d3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 --2- 172.21.15.86:0/4206165811 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c0b38f0 0x7fe72c06d3a0 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7fe72c067a50 tx=0x7fe72402be40 comp rx=0 tx=0).stop 2024-03-06T07:30:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 -- 172.21.15.86:0/4206165811 shutdown_connections 2024-03-06T07:30:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 --2- 172.21.15.86:0/4206165811 >> v2:172.21.15.194:3300/0 conn(0x7fe72c0af820 0x7fe72c0b33b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 --2- 172.21.15.86:0/4206165811 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c0b38f0 0x7fe72c06d3a0 unknown :-1 s=CLOSED pgs=725 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 --2- 172.21.15.86:0/4206165811 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c15e160 0x7fe72c162620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 -- 172.21.15.86:0/4206165811 >> 172.21.15.86:0/4206165811 conn(0x7fe72c0d0340 msgr2=0x7fe72c0d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 -- 172.21.15.86:0/4206165811 shutdown_connections 2024-03-06T07:30:19.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.368+0000 7fe732000640 1 -- 172.21.15.86:0/4206165811 wait complete. 2024-03-06T07:30:19.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.371+0000 7fe732000640 1 Processor -- start 2024-03-06T07:30:19.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.371+0000 7fe732000640 1 -- start start 2024-03-06T07:30:19.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe732000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c0af820 0x7fe72c0ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe732000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe72c0b38f0 0x7fe72c160ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe732000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c15e160 0x7fe72c158190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe732000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe72c06ec30 con 0x7fe72c15e160 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe732000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe72c0afce0 con 0x7fe72c0b38f0 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe732000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe72c06ef10 con 0x7fe72c0af820 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe72be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c15e160 0x7fe72c158190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe72be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c15e160 0x7fe72c158190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33542/0 (socket says 172.21.15.86:33542) 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.373+0000 7fe72be00640 1 -- 172.21.15.86:0/3349155944 learned_addr learned my addr 172.21.15.86:0/3349155944 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe72b400640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c0af820 0x7fe72c0ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe723e00640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe72c0b38f0 0x7fe72c160ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe72be00640 1 -- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c0af820 msgr2=0x7fe72c0ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe72be00640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c0af820 0x7fe72c0ccbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe72be00640 1 -- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe72c0b38f0 msgr2=0x7fe72c160ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe72be00640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe72c0b38f0 0x7fe72c160ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe72be00640 1 -- 172.21.15.86:0/3349155944 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe72c1534e0 con 0x7fe72c15e160 2024-03-06T07:30:19.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.374+0000 7fe72be00640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c15e160 0x7fe72c158190 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7fe724036040 tx=0x7fe724004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:19.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.375+0000 7fe723400640 1 -- 172.21.15.86:0/3349155944 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe724043020 con 0x7fe72c15e160 2024-03-06T07:30:19.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.375+0000 7fe723400640 1 -- 172.21.15.86:0/3349155944 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe72403e450 con 0x7fe72c15e160 2024-03-06T07:30:19.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.375+0000 7fe723400640 1 -- 172.21.15.86:0/3349155944 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe724009510 con 0x7fe72c15e160 2024-03-06T07:30:19.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.375+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe72c0cd1a0 con 0x7fe72c15e160 2024-03-06T07:30:19.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.375+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe72c0cfbb0 con 0x7fe72c15e160 2024-03-06T07:30:19.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.376+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe72c157a80 con 0x7fe72c15e160 2024-03-06T07:30:19.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.377+0000 7fe723400640 1 -- 172.21.15.86:0/3349155944 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe724046020 con 0x7fe72c15e160 2024-03-06T07:30:19.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.378+0000 7fe723400640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe70806b4d0 0x7fe70806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.379+0000 7fe723400640 1 -- 172.21.15.86:0/3349155944 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe7240aca20 con 0x7fe72c15e160 2024-03-06T07:30:19.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.379+0000 7fe72b400640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe70806b4d0 0x7fe70806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.379+0000 7fe72b400640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe70806b4d0 0x7fe70806d990 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7fe71c006fd0 tx=0x7fe71c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:19.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.382+0000 7fe723400640 1 -- 172.21.15.86:0/3349155944 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe72404d030 con 0x7fe72c15e160 2024-03-06T07:30:19.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.615+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe72c0d11e0 con 0x7fe72c15e160 2024-03-06T07:30:19.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.615+0000 7fe723400640 1 -- 172.21.15.86:0/3349155944 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fe72407b920 con 0x7fe72c15e160 2024-03-06T07:30:19.618 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:19.618 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:30:19.618 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:30:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe70806b4d0 msgr2=0x7fe70806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe70806b4d0 0x7fe70806d990 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7fe71c006fd0 tx=0x7fe71c008040 comp rx=0 tx=0).stop 2024-03-06T07:30:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c15e160 msgr2=0x7fe72c158190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c15e160 0x7fe72c158190 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7fe724036040 tx=0x7fe724004a40 comp rx=0 tx=0).stop 2024-03-06T07:30:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 shutdown_connections 2024-03-06T07:30:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe70806b4d0 0x7fe70806d990 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe72c15e160 0x7fe72c158190 unknown :-1 s=CLOSED pgs=726 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe72c0b38f0 0x7fe72c160ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 --2- 172.21.15.86:0/3349155944 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe72c0af820 0x7fe72c0ccbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.618+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 >> 172.21.15.86:0/3349155944 conn(0x7fe72c0d0340 msgr2=0x7fe72c15bf20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.619+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 shutdown_connections 2024-03-06T07:30:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.619+0000 7fe732000640 1 -- 172.21.15.86:0/3349155944 wait complete. 2024-03-06T07:30:19.638 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:19.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.837+0000 7f1a4d000640 1 Processor -- start 2024-03-06T07:30:19.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.837+0000 7f1a4d000640 1 -- start start 2024-03-06T07:30:19.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.838+0000 7f1a4d000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1a48144c00 0x7f1a4806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.838+0000 7f1a4d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1a4815f9d0 0x7f1a4815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.838+0000 7f1a4d000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a480ce0a0 0x7f1a480ce4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.838+0000 7f1a4d000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1a4806ee70 con 0x7f1a4815f9d0 2024-03-06T07:30:19.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.839+0000 7f1a4d000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1a4806cef0 con 0x7f1a48144c00 2024-03-06T07:30:19.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.839+0000 7f1a4d000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a4806d1d0 con 0x7f1a480ce0a0 2024-03-06T07:30:19.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.839+0000 7f1a45400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1a4815f9d0 0x7f1a4815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.839+0000 7f1a46800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a480ce0a0 0x7f1a480ce4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.839+0000 7f1a45400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1a4815f9d0 0x7f1a4815fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33548/0 (socket says 172.21.15.86:33548) 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.839+0000 7f1a45400640 1 -- 172.21.15.86:0/2628211416 learned_addr learned my addr 172.21.15.86:0/2628211416 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.839+0000 7f1a45e00640 1 --2- 172.21.15.86:0/2628211416 >> v2:172.21.15.194:3300/0 conn(0x7f1a48144c00 0x7f1a4806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.840+0000 7f1a45400640 1 -- 172.21.15.86:0/2628211416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a480ce0a0 msgr2=0x7f1a480ce4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.840+0000 7f1a45400640 1 --2- 172.21.15.86:0/2628211416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a480ce0a0 0x7f1a480ce4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.840+0000 7f1a45400640 1 -- 172.21.15.86:0/2628211416 >> v2:172.21.15.194:3300/0 conn(0x7f1a48144c00 msgr2=0x7f1a4806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.840+0000 7f1a45400640 1 --2- 172.21.15.86:0/2628211416 >> v2:172.21.15.194:3300/0 conn(0x7f1a48144c00 0x7f1a4806e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.840+0000 7f1a45400640 1 -- 172.21.15.86:0/2628211416 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a4806d4b0 con 0x7f1a4815f9d0 2024-03-06T07:30:19.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.841+0000 7f1a45400640 1 --2- 172.21.15.86:0/2628211416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a4815f9d0 0x7f1a4815fe60 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f1a480b25f0 tx=0x7f1a3402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=df9012f9d95ee013 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.841+0000 7f1a44a00640 1 -- 172.21.15.86:0/2628211416 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a34035020 con 0x7f1a4815f9d0 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.841+0000 7f1a44a00640 1 -- 172.21.15.86:0/2628211416 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a34031870 con 0x7f1a4815f9d0 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.841+0000 7f1a44a00640 1 -- 172.21.15.86:0/2628211416 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a34031c40 con 0x7f1a4815f9d0 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 -- 172.21.15.86:0/2628211416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a4815f9d0 msgr2=0x7f1a4815fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2628211416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a4815f9d0 0x7f1a4815fe60 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f1a480b25f0 tx=0x7f1a3402b910 comp rx=0 tx=0).stop 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 -- 172.21.15.86:0/2628211416 shutdown_connections 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2628211416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a480ce0a0 0x7f1a480ce4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2628211416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a4815f9d0 0x7f1a4815fe60 unknown :-1 s=CLOSED pgs=727 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2628211416 >> v2:172.21.15.194:3300/0 conn(0x7f1a48144c00 0x7f1a4806e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 -- 172.21.15.86:0/2628211416 >> 172.21.15.86:0/2628211416 conn(0x7f1a480b1730 msgr2=0x7f1a480b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 -- 172.21.15.86:0/2628211416 shutdown_connections 2024-03-06T07:30:19.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.842+0000 7f1a4d000640 1 -- 172.21.15.86:0/2628211416 wait complete. 2024-03-06T07:30:19.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.844+0000 7f1a4d000640 1 Processor -- start 2024-03-06T07:30:19.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.844+0000 7f1a4d000640 1 -- start start 2024-03-06T07:30:19.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a4d000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a480ce0a0 0x7f1a48138fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a4d000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a48144c00 0x7f1a4815a9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a4d000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a4815f9d0 0x7f1a48155530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a4d000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a4806cb50 con 0x7f1a480ce0a0 2024-03-06T07:30:19.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a4d000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a48160730 con 0x7f1a4815f9d0 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a4d000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a4806ccd0 con 0x7f1a48144c00 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a45e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a480ce0a0 0x7f1a48138fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a45e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a480ce0a0 0x7f1a48138fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33560/0 (socket says 172.21.15.86:33560) 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a45e00640 1 -- 172.21.15.86:0/2383219519 learned_addr learned my addr 172.21.15.86:0/2383219519 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a45400640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a48144c00 0x7f1a4815a9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a46800640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a4815f9d0 0x7f1a48155530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a45e00640 1 -- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a48144c00 msgr2=0x7f1a4815a9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.845+0000 7f1a45e00640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a48144c00 0x7f1a4815a9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a45e00640 1 -- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a4815f9d0 msgr2=0x7f1a48155530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a45e00640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a4815f9d0 0x7f1a48155530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a45e00640 1 -- 172.21.15.86:0/2383219519 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a48138260 con 0x7f1a480ce0a0 2024-03-06T07:30:19.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a45e00640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a480ce0a0 0x7f1a48138fe0 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7f1a3800d800 tx=0x7f1a3800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:19.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a3e000640 1 -- 172.21.15.86:0/2383219519 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a3800b6c0 con 0x7f1a480ce0a0 2024-03-06T07:30:19.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a3e000640 1 -- 172.21.15.86:0/2383219519 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a38019070 con 0x7f1a480ce0a0 2024-03-06T07:30:19.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a3e000640 1 -- 172.21.15.86:0/2383219519 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a3800ba80 con 0x7f1a480ce0a0 2024-03-06T07:30:19.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1a4813ae60 con 0x7f1a480ce0a0 2024-03-06T07:30:19.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.846+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a481387e0 con 0x7f1a480ce0a0 2024-03-06T07:30:19.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.848+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1a4804f330 con 0x7f1a480ce0a0 2024-03-06T07:30:19.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.848+0000 7f1a3e000640 1 -- 172.21.15.86:0/2383219519 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1a3800bc20 con 0x7f1a480ce0a0 2024-03-06T07:30:19.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.849+0000 7f1a3e000640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a2806b500 0x7f1a2806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:19.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.849+0000 7f1a3e000640 1 -- 172.21.15.86:0/2383219519 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1a3808a6c0 con 0x7f1a480ce0a0 2024-03-06T07:30:19.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.850+0000 7f1a45400640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a2806b500 0x7f1a2806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:19.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.850+0000 7f1a45400640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a2806b500 0x7f1a2806d9c0 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f1a34036000 tx=0x7f1a3403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:19.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:19.854+0000 7f1a3e000640 1 -- 172.21.15.86:0/2383219519 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1a3801c360 con 0x7f1a480ce0a0 2024-03-06T07:30:20.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.173+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1a48074be0 con 0x7f1a480ce0a0 2024-03-06T07:30:20.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.173+0000 7f1a3e000640 1 -- 172.21.15.86:0/2383219519 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 34 v34) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f1a38059540 con 0x7f1a480ce0a0 2024-03-06T07:30:20.176 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:20.176 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":34,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28}],"filesystems":[{"mdsmap":{"epoch":34,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:29:45.693670+0000","modified":"2024-03-06T07:29:48.797087+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5820},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5820":{"gid":5820,"name":"c","rank":0,"incarnation":30,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/3356338385","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3356338385},{"type":"v1","addr":"172.21.15.86:6837","nonce":3356338385}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5820,"qdb_cluster":[5820]},"id":4}]} 2024-03-06T07:30:20.176 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 34 2024-03-06T07:30:20.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.176+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a2806b500 msgr2=0x7f1a2806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.176+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a2806b500 0x7f1a2806d9c0 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f1a34036000 tx=0x7f1a3403c040 comp rx=0 tx=0).stop 2024-03-06T07:30:20.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a480ce0a0 msgr2=0x7f1a48138fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a480ce0a0 0x7f1a48138fe0 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7f1a3800d800 tx=0x7f1a3800dcc0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 shutdown_connections 2024-03-06T07:30:20.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a2806b500 0x7f1a2806d9c0 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1a4815f9d0 0x7f1a48155530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1a48144c00 0x7f1a4815a9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 --2- 172.21.15.86:0/2383219519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1a480ce0a0 0x7f1a48138fe0 unknown :-1 s=CLOSED pgs=728 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 >> 172.21.15.86:0/2383219519 conn(0x7f1a480b1730 msgr2=0x7f1a480aff30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:20.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 shutdown_connections 2024-03-06T07:30:20.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.177+0000 7f1a4d000640 1 -- 172.21.15.86:0/2383219519 wait complete. 2024-03-06T07:30:20.199 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:20.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.396+0000 7f9b4f400640 1 Processor -- start 2024-03-06T07:30:20.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.396+0000 7f9b4f400640 1 -- start start 2024-03-06T07:30:20.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.397+0000 7f9b4f400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9b4815acb0 0x7f9b480b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.397+0000 7f9b4f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9b4806ec00 0x7f9b480ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.397+0000 7f9b4f400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b4815b090 con 0x7f9b480b4680 2024-03-06T07:30:20.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.397+0000 7f9b4f400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9b480b2a30 con 0x7f9b4806ec00 2024-03-06T07:30:20.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.397+0000 7f9b4f400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9b480b4360 con 0x7f9b4815acb0 2024-03-06T07:30:20.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9b4806ec00 0x7f9b480ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9b4806ec00 0x7f9b480ccfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33570/0 (socket says 172.21.15.86:33570) 2024-03-06T07:30:20.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4d400640 1 -- 172.21.15.86:0/3626516210 learned_addr learned my addr 172.21.15.86:0/3626516210 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:20.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4ca00640 1 --2- 172.21.15.86:0/3626516210 >> v2:172.21.15.194:3300/0 conn(0x7f9b4815acb0 0x7f9b480b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4d400640 1 -- 172.21.15.86:0/3626516210 >> v2:172.21.15.194:3300/0 conn(0x7f9b4815acb0 msgr2=0x7f9b480b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:20.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4d400640 1 --2- 172.21.15.86:0/3626516210 >> v2:172.21.15.194:3300/0 conn(0x7f9b4815acb0 0x7f9b480b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4d400640 1 -- 172.21.15.86:0/3626516210 >> v1:172.21.15.194:6789/0 conn(0x7f9b480b4680 legacy=0x7f9b4806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:20.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.398+0000 7f9b4d400640 1 -- 172.21.15.86:0/3626516210 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b480b2cf0 con 0x7f9b4806ec00 2024-03-06T07:30:20.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.400+0000 7f9b4d400640 1 --2- 172.21.15.86:0/3626516210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b4806ec00 0x7f9b480ccfb0 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f9b480b27f0 tx=0x7f9b4002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d4a62d1a8137d68 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:20.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.400+0000 7f9b47e00640 1 -- 172.21.15.86:0/3626516210 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b4002ca80 con 0x7f9b4806ec00 2024-03-06T07:30:20.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.400+0000 7f9b47e00640 1 -- 172.21.15.86:0/3626516210 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b4002cc20 con 0x7f9b4806ec00 2024-03-06T07:30:20.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.400+0000 7f9b47e00640 1 -- 172.21.15.86:0/3626516210 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b400343f0 con 0x7f9b4806ec00 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 -- 172.21.15.86:0/3626516210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b4806ec00 msgr2=0x7f9b480ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 --2- 172.21.15.86:0/3626516210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b4806ec00 0x7f9b480ccfb0 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f9b480b27f0 tx=0x7f9b4002bee0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 -- 172.21.15.86:0/3626516210 shutdown_connections 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 --2- 172.21.15.86:0/3626516210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b4806ec00 0x7f9b480ccfb0 unknown :-1 s=CLOSED pgs=729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 --2- 172.21.15.86:0/3626516210 >> v2:172.21.15.194:3300/0 conn(0x7f9b4815acb0 0x7f9b480b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 -- 172.21.15.86:0/3626516210 >> 172.21.15.86:0/3626516210 conn(0x7f9b480b1730 msgr2=0x7f9b480b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 -- 172.21.15.86:0/3626516210 shutdown_connections 2024-03-06T07:30:20.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.401+0000 7f9b4f400640 1 -- 172.21.15.86:0/3626516210 wait complete. 2024-03-06T07:30:20.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.403+0000 7f9b4f400640 1 Processor -- start 2024-03-06T07:30:20.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.403+0000 7f9b4f400640 1 -- start start 2024-03-06T07:30:20.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b4f400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b4806ec00 0x7f9b481e4620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b4f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b480b4680 0x7f9b481e4b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b4f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b4815acb0 0x7f9b481412c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b4f400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b480b3d10 con 0x7f9b480b4680 2024-03-06T07:30:20.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b4f400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b480b32b0 con 0x7f9b4806ec00 2024-03-06T07:30:20.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b4f400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b480b4010 con 0x7f9b4815acb0 2024-03-06T07:30:20.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b480b4680 0x7f9b481e4b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b480b4680 0x7f9b481e4b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33580/0 (socket says 172.21.15.86:33580) 2024-03-06T07:30:20.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.404+0000 7f9b3fe00640 1 -- 172.21.15.86:0/76761415 learned_addr learned my addr 172.21.15.86:0/76761415 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:20.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b4d400640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b4815acb0 0x7f9b481412c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b4ca00640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b4806ec00 0x7f9b481e4620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b3fe00640 1 -- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b4815acb0 msgr2=0x7f9b481412c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b3fe00640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b4815acb0 0x7f9b481412c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b3fe00640 1 -- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b4806ec00 msgr2=0x7f9b481e4620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b3fe00640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b4806ec00 0x7f9b481e4620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b3fe00640 1 -- 172.21.15.86:0/76761415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b480ce840 con 0x7f9b480b4680 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b3fe00640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b480b4680 0x7f9b481e4b60 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f9b3000b420 tx=0x7f9b3000b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b45600640 1 -- 172.21.15.86:0/76761415 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b30011830 con 0x7f9b480b4680 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b45600640 1 -- 172.21.15.86:0/76761415 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b30016070 con 0x7f9b480b4680 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b4813f2d0 con 0x7f9b480b4680 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b45600640 1 -- 172.21.15.86:0/76761415 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b30011c10 con 0x7f9b480b4680 2024-03-06T07:30:20.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.405+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b4813f890 con 0x7f9b480b4680 2024-03-06T07:30:20.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.407+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b4804f330 con 0x7f9b480b4680 2024-03-06T07:30:20.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.408+0000 7f9b45600640 1 -- 172.21.15.86:0/76761415 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9b3001b460 con 0x7f9b480b4680 2024-03-06T07:30:20.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.408+0000 7f9b45600640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9b2406b4d0 0x7f9b2406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.409+0000 7f9b45600640 1 -- 172.21.15.86:0/76761415 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9b30089c50 con 0x7f9b480b4680 2024-03-06T07:30:20.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.409+0000 7f9b4ca00640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9b2406b4d0 0x7f9b2406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.409+0000 7f9b4ca00640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9b2406b4d0 0x7f9b2406d990 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f9b38006fd0 tx=0x7f9b38008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:20.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.414+0000 7f9b45600640 1 -- 172.21.15.86:0/76761415 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9b3008f020 con 0x7f9b480b4680 2024-03-06T07:30:20.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.609+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f9b4806ce60 con 0x7f9b480b4680 2024-03-06T07:30:20.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.612+0000 7f9b45600640 1 -- 172.21.15.86:0/76761415 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v58) v1 ==== 74+0+16677 (secure 0 0 0) 0x7f9b30020020 con 0x7f9b480b4680 2024-03-06T07:30:20.615 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:20.615 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":58,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:29:47.263155+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:29:43.503869+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-06T07:29:44.707119+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:20.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.614+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9b2406b4d0 msgr2=0x7f9b2406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.614+0000 7f9b4f400640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9b2406b4d0 0x7f9b2406d990 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f9b38006fd0 tx=0x7f9b38008040 comp rx=0 tx=0).stop 2024-03-06T07:30:20.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.614+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b480b4680 msgr2=0x7f9b481e4b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b480b4680 0x7f9b481e4b60 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f9b3000b420 tx=0x7f9b3000b8e0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 shutdown_connections 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9b2406b4d0 0x7f9b2406d990 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b4815acb0 0x7f9b481412c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b480b4680 0x7f9b481e4b60 unknown :-1 s=CLOSED pgs=730 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 --2- 172.21.15.86:0/76761415 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b4806ec00 0x7f9b481e4620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 >> 172.21.15.86:0/76761415 conn(0x7f9b480b1730 msgr2=0x7f9b480af6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 shutdown_connections 2024-03-06T07:30:20.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.615+0000 7f9b4f400640 1 -- 172.21.15.86:0/76761415 wait complete. 2024-03-06T07:30:20.634 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:30:20.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.824+0000 7f1145400640 1 Processor -- start 2024-03-06T07:30:20.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.824+0000 7f1145400640 1 -- start start 2024-03-06T07:30:20.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.826+0000 7f1145400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f114015e360 0x7f11400cec10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.826+0000 7f1145400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 0x7f11400d1c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.826+0000 7f1145400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f11400cf2b0 con 0x7f11400d12e0 2024-03-06T07:30:20.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.826+0000 7f1145400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f11400cf590 con 0x7f114015e360 2024-03-06T07:30:20.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.826+0000 7f1145400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11400cf870 con 0x7f1140162440 2024-03-06T07:30:20.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.826+0000 7f1135e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f11400d12e0 0x7f1140161d30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37388/0 (socket says 172.21.15.86:37388) 2024-03-06T07:30:20.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.826+0000 7f1135e00640 1 -- 172.21.15.86:0/3674149842 learned_addr learned my addr 172.21.15.86:0/3674149842 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:20.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113f200640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 0x7f11400d1c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113e800640 1 --2- 172.21.15.86:0/3674149842 >> v2:172.21.15.194:3300/0 conn(0x7f114015e360 0x7f11400cec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3076863010 0 0) 0x7f11400cf2b0 con 0x7f11400d12e0 2024-03-06T07:30:20.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1124003520 con 0x7f11400d12e0 2024-03-06T07:30:20.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2316639879 0 0) 0x7f1124003520 con 0x7f11400d12e0 2024-03-06T07:30:20.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 msgr2=0x7f11400d1c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 0x7f11400d1c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 >> v2:172.21.15.194:3300/0 conn(0x7f114015e360 msgr2=0x7f11400cec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 --2- 172.21.15.86:0/3674149842 >> v2:172.21.15.194:3300/0 conn(0x7f114015e360 0x7f11400cec10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11400cfb50 con 0x7f11400d12e0 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1128001690 con 0x7f11400d12e0 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113e800640 1 --2- 172.21.15.86:0/3674149842 >> v2:172.21.15.194:3300/0 conn(0x7f114015e360 0x7f11400cec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 >> v1:172.21.15.86:6789/0 conn(0x7f11400d12e0 legacy=0x7f1140161d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11240039b0 0x7f1124003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 0x7f1124004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f11240051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11400cf590 con 0x7f1124004d40 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11400cf870 con 0x7f11240039b0 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.827+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f112400bb00 con 0x7f1140162440 2024-03-06T07:30:20.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f113f200640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f11240051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f1135e00640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 0x7f1124004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f113e800640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11240039b0 0x7f1124003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f113f200640 1 -- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 msgr2=0x7f1124004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f113f200640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 0x7f1124004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f113f200640 1 -- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11240039b0 msgr2=0x7f1124003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f113f200640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f11240039b0 0x7f1124003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f113f200640 1 -- 172.21.15.86:0/3674149842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1124003520 con 0x7f1124004d40 2024-03-06T07:30:20.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.828+0000 7f1135e00640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1140162440 0x7f1124004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:20.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.829+0000 7f113f200640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f11240051c0 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f114008d370 tx=0x7f1124005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b3a1fbb850a4f248 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:20.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.829+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1138039070 con 0x7f1124004d40 2024-03-06T07:30:20.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.829+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f113802c640 con 0x7f1124004d40 2024-03-06T07:30:20.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.829+0000 7f113de00640 1 -- 172.21.15.86:0/3674149842 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1138009690 con 0x7f1124004d40 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.830+0000 7f1145400640 1 -- 172.21.15.86:0/3674149842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 msgr2=0x7f11240051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.830+0000 7f1145400640 1 --2- 172.21.15.86:0/3674149842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f11240051c0 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f114008d370 tx=0x7f1124005830 comp rx=0 tx=0).stop 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.830+0000 7f113f200640 1 -- 172.21.15.86:0/3674149842 reap_dead start 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.830+0000 7f1145400640 1 -- 172.21.15.86:0/3674149842 shutdown_connections 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.830+0000 7f1145400640 1 -- 172.21.15.86:0/3674149842 >> 172.21.15.86:0/3674149842 conn(0x7f11400d3f50 msgr2=0x7f11400d4330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.830+0000 7f1145400640 1 -- 172.21.15.86:0/3674149842 shutdown_connections 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.830+0000 7f1145400640 1 -- 172.21.15.86:0/3674149842 wait complete. 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.832+0000 7f1145400640 1 Processor -- start 2024-03-06T07:30:20.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.832+0000 7f1145400640 1 -- start start 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f1145400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f114014dc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f1145400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f114014a9a0 0x7f1140160ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f1145400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f114014b050 0x7f11400ca2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f1145400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11400d18f0 con 0x7f1124004d40 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f1145400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11400d1770 con 0x7f114014a9a0 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f1145400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11400d1c90 con 0x7f114014b050 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f113e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f114014dc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f113e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f114014dc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33600/0 (socket says 172.21.15.86:33600) 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.833+0000 7f113e800640 1 -- 172.21.15.86:0/3915207564 learned_addr learned my addr 172.21.15.86:0/3915207564 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f1135e00640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f114014a9a0 0x7f1140160ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f113e800640 1 -- 172.21.15.86:0/3915207564 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f114014b050 msgr2=0x7f11400ca2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f113e800640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f114014b050 0x7f11400ca2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f113e800640 1 -- 172.21.15.86:0/3915207564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f114014a9a0 msgr2=0x7f1140160ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f113e800640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f114014a9a0 0x7f1140160ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f113e800640 1 -- 172.21.15.86:0/3915207564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11400c94e0 con 0x7f1124004d40 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f113e800640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f114014dc60 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f113000e140 tx=0x7f113000e600 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f1137400640 1 -- 172.21.15.86:0/3915207564 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1130015610 con 0x7f1124004d40 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f1137400640 1 -- 172.21.15.86:0/3915207564 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f113001a070 con 0x7f1124004d40 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f11400c97c0 con 0x7f1124004d40 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f1137400640 1 -- 172.21.15.86:0/3915207564 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11300159d0 con 0x7f1124004d40 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.834+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1140161410 con 0x7f1124004d40 2024-03-06T07:30:20.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.835+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f114014ad80 con 0x7f1124004d40 2024-03-06T07:30:20.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.837+0000 7f1137400640 1 -- 172.21.15.86:0/3915207564 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1130012030 con 0x7f1124004d40 2024-03-06T07:30:20.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.838+0000 7f1137400640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f111c06b4d0 0x7f111c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:20.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.839+0000 7f1137400640 1 -- 172.21.15.86:0/3915207564 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f113008aea0 con 0x7f1124004d40 2024-03-06T07:30:20.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.839+0000 7f1135e00640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f111c06b4d0 0x7f111c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:20.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.840+0000 7f1135e00640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f111c06b4d0 0x7f111c06d990 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f1128001800 tx=0x7f1128009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:20.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:20.843+0000 7f1137400640 1 -- 172.21.15.86:0/3915207564 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1130010040 con 0x7f1124004d40 2024-03-06T07:30:21.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.134+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f114008d9b0 con 0x7f1124004d40 2024-03-06T07:30:21.138 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:30:21.135+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:30:21.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.139+0000 7f1137400640 1 -- 172.21.15.86:0/3915207564 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v35) v1 ==== 161+0+0 (secure 0 0 0) 0x7f1130059c80 con 0x7f1124004d40 2024-03-06T07:30:21.142 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f111c06b4d0 msgr2=0x7f111c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f111c06b4d0 0x7f111c06d990 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f1128001800 tx=0x7f1128009040 comp rx=0 tx=0).stop 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 msgr2=0x7f114014dc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f114014dc60 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f113000e140 tx=0x7f113000e600 comp rx=0 tx=0).stop 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 shutdown_connections 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f111c06b4d0 0x7f111c06d990 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f114014b050 0x7f11400ca2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f114014a9a0 0x7f1140160ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 --2- 172.21.15.86:0/3915207564 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1124004d40 0x7f114014dc60 unknown :-1 s=CLOSED pgs=733 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 >> 172.21.15.86:0/3915207564 conn(0x7f11400d3f50 msgr2=0x7f11400d2e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:21.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.142+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 shutdown_connections 2024-03-06T07:30:21.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.143+0000 7f1145400640 1 -- 172.21.15.86:0/3915207564 wait complete. 2024-03-06T07:30:21.165 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:30:21.231 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:30:21.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.343+0000 7f5a1be00640 1 Processor -- start 2024-03-06T07:30:21.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.343+0000 7f5a1be00640 1 -- start start 2024-03-06T07:30:21.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.344+0000 7f5a1be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a140b3f70 0x7f5a14160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:21.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.344+0000 7f5a1be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5a14070ff0 0x7f5a140ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:21.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.344+0000 7f5a1be00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5a140b46c0 con 0x7f5a1415fc10 2024-03-06T07:30:21.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.344+0000 7f5a1be00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5a14071480 con 0x7f5a14070ff0 2024-03-06T07:30:21.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.344+0000 7f5a1be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a140b49a0 con 0x7f5a140b3f70 2024-03-06T07:30:21.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.345+0000 7f5a18a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a140b3f70 0x7f5a14160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:21.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.345+0000 7f5a18a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a140b3f70 0x7f5a14160700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:38684/0 (socket says 172.21.15.86:38684) 2024-03-06T07:30:21.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.345+0000 7f5a18a00640 1 -- 172.21.15.86:0/3880998242 learned_addr learned my addr 172.21.15.86:0/3880998242 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:21.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.345+0000 7f5a19e00640 1 --2- 172.21.15.86:0/3880998242 >> v2:172.21.15.194:3300/0 conn(0x7f5a14070ff0 0x7f5a140ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:21.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.346+0000 7f5a18a00640 1 -- 172.21.15.86:0/3880998242 >> v2:172.21.15.194:3300/0 conn(0x7f5a14070ff0 msgr2=0x7f5a140ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:21.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.346+0000 7f5a18a00640 1 --2- 172.21.15.86:0/3880998242 >> v2:172.21.15.194:3300/0 conn(0x7f5a14070ff0 0x7f5a140ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.346+0000 7f5a18a00640 1 -- 172.21.15.86:0/3880998242 >> v1:172.21.15.194:6789/0 conn(0x7f5a1415fc10 legacy=0x7f5a1415fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:21.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.346+0000 7f5a18a00640 1 -- 172.21.15.86:0/3880998242 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a14072c90 con 0x7f5a140b3f70 2024-03-06T07:30:21.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.347+0000 7f5a18a00640 1 --2- 172.21.15.86:0/3880998242 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a140b3f70 0x7f5a14160700 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f5a00009870 tx=0x7f5a0002b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=84fb32117b3dc182 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:21.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.347+0000 7f5a13e00640 1 -- 172.21.15.86:0/3880998242 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a00034070 con 0x7f5a140b3f70 2024-03-06T07:30:21.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a13e00640 1 -- 172.21.15.86:0/3880998242 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a00030840 con 0x7f5a140b3f70 2024-03-06T07:30:21.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a13e00640 1 -- 172.21.15.86:0/3880998242 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a00030c10 con 0x7f5a140b3f70 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 -- 172.21.15.86:0/3880998242 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a140b3f70 msgr2=0x7f5a14160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 --2- 172.21.15.86:0/3880998242 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a140b3f70 0x7f5a14160700 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f5a00009870 tx=0x7f5a0002b980 comp rx=0 tx=0).stop 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 -- 172.21.15.86:0/3880998242 shutdown_connections 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 --2- 172.21.15.86:0/3880998242 >> v2:172.21.15.194:3300/0 conn(0x7f5a14070ff0 0x7f5a140ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 --2- 172.21.15.86:0/3880998242 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a140b3f70 0x7f5a14160700 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 -- 172.21.15.86:0/3880998242 >> 172.21.15.86:0/3880998242 conn(0x7f5a140b1940 msgr2=0x7f5a140b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 -- 172.21.15.86:0/3880998242 shutdown_connections 2024-03-06T07:30:21.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.348+0000 7f5a1be00640 1 -- 172.21.15.86:0/3880998242 wait complete. 2024-03-06T07:30:21.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.350+0000 7f5a1be00640 1 Processor -- start 2024-03-06T07:30:21.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.350+0000 7f5a1be00640 1 -- start start 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a1be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a14070ff0 0x7f5a1414b860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a1be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a140b3f70 0x7f5a14159900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a1be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a1415fc10 0x7f5a1409c890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a1be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a14072a30 con 0x7f5a140b3f70 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a1be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a140728b0 con 0x7f5a14070ff0 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a1be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a1415c2e0 con 0x7f5a1415fc10 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a140b3f70 0x7f5a14159900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:21.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a140b3f70 0x7f5a14159900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33636/0 (socket says 172.21.15.86:33636) 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 -- 172.21.15.86:0/134120483 learned_addr learned my addr 172.21.15.86:0/134120483 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a19e00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a1415fc10 0x7f5a1409c890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a19400640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a14070ff0 0x7f5a1414b860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 -- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a1415fc10 msgr2=0x7f5a1409c890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a1415fc10 0x7f5a1409c890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 -- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a14070ff0 msgr2=0x7f5a1414b860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a14070ff0 0x7f5a1414b860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.352+0000 7f5a18a00640 1 -- 172.21.15.86:0/134120483 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a141f8d90 con 0x7f5a140b3f70 2024-03-06T07:30:21.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.353+0000 7f5a18a00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a140b3f70 0x7f5a14159900 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7f5a00000c00 tx=0x7f5a00004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:21.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.353+0000 7f5a11600640 1 -- 172.21.15.86:0/134120483 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a00048070 con 0x7f5a140b3f70 2024-03-06T07:30:21.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.353+0000 7f5a11600640 1 -- 172.21.15.86:0/134120483 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a00030640 con 0x7f5a140b3f70 2024-03-06T07:30:21.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.353+0000 7f5a11600640 1 -- 172.21.15.86:0/134120483 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a00034040 con 0x7f5a140b3f70 2024-03-06T07:30:21.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.353+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5a1409be10 con 0x7f5a140b3f70 2024-03-06T07:30:21.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.353+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5a1406d610 con 0x7f5a140b3f70 2024-03-06T07:30:21.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.354+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5a1404f330 con 0x7f5a140b3f70 2024-03-06T07:30:21.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.355+0000 7f5a11600640 1 -- 172.21.15.86:0/134120483 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5a00037070 con 0x7f5a140b3f70 2024-03-06T07:30:21.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.356+0000 7f5a11600640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f59f406b4d0 0x7f59f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:21.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.356+0000 7f5a11600640 1 -- 172.21.15.86:0/134120483 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(59..59 src has 1..59) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f5a000ac200 con 0x7f5a140b3f70 2024-03-06T07:30:21.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.356+0000 7f5a19400640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f59f406b4d0 0x7f59f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:21.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.357+0000 7f5a19400640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f59f406b4d0 0x7f59f406d990 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f5a1415b5f0 tx=0x7f5a08009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:21.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.360+0000 7f5a11600640 1 -- 172.21.15.86:0/134120483 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5a1409be10 con 0x7f5a140b3f70 2024-03-06T07:30:21.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:21.577+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f5a1406ef30 con 0x7f5a140b3f70 2024-03-06T07:30:22.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.143+0000 7f5a11600640 1 -- 172.21.15.86:0/134120483 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v36) v1 ==== 104+0+0 (secure 0 0 0) 0x7f5a0007b130 con 0x7f5a140b3f70 2024-03-06T07:30:22.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f59f406b4d0 msgr2=0x7f59f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:22.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f59f406b4d0 0x7f59f406d990 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f5a1415b5f0 tx=0x7f5a08009040 comp rx=0 tx=0).stop 2024-03-06T07:30:22.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a140b3f70 msgr2=0x7f5a14159900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a140b3f70 0x7f5a14159900 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7f5a00000c00 tx=0x7f5a00004730 comp rx=0 tx=0).stop 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 shutdown_connections 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f59f406b4d0 0x7f59f406d990 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5a1415fc10 0x7f5a1409c890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5a140b3f70 0x7f5a14159900 unknown :-1 s=CLOSED pgs=737 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 --2- 172.21.15.86:0/134120483 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5a14070ff0 0x7f5a1414b860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.146+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 >> 172.21.15.86:0/134120483 conn(0x7f5a140b1940 msgr2=0x7f5a140ad2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.147+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 shutdown_connections 2024-03-06T07:30:22.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.147+0000 7f5a1be00640 1 -- 172.21.15.86:0/134120483 wait complete. 2024-03-06T07:30:22.172 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:30:22.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.379+0000 7f17c8c00640 1 Processor -- start 2024-03-06T07:30:22.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.379+0000 7f17c8c00640 1 -- start start 2024-03-06T07:30:22.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.380+0000 7f17c8c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f17c406dd30 0x7f17c4166ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:22.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.380+0000 7f17c8c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17c4072970 con 0x7f17c40cd450 2024-03-06T07:30:22.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.380+0000 7f17c8c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17c40cdbd0 con 0x7f17c4145750 2024-03-06T07:30:22.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17c8c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17c40cd980 con 0x7f17c406dd30 2024-03-06T07:30:22.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17b9400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f17c40cd450 0x7f17c406d620 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37392/0 (socket says 172.21.15.86:37392) 2024-03-06T07:30:22.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17b9400640 1 -- 172.21.15.86:0/604057099 learned_addr learned my addr 172.21.15.86:0/604057099 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:22.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17c2800640 1 --2- 172.21.15.86:0/604057099 >> v2:172.21.15.86:3300/0 conn(0x7f17c406dd30 0x7f17c4166ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:22.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17c1400640 1 -- 172.21.15.86:0/604057099 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 786056162 0 0) 0x7f17c4072970 con 0x7f17c40cd450 2024-03-06T07:30:22.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17c1400640 1 -- 172.21.15.86:0/604057099 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f17a8003520 con 0x7f17c40cd450 2024-03-06T07:30:22.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17c2800640 1 -- 172.21.15.86:0/604057099 >> v1:172.21.15.194:6789/0 conn(0x7f17c4145750 legacy=0x7f17c4072260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:22.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17c2800640 1 -- 172.21.15.86:0/604057099 >> v1:172.21.15.86:6789/0 conn(0x7f17c40cd450 legacy=0x7f17c406d620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:22.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.381+0000 7f17c2800640 1 -- 172.21.15.86:0/604057099 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17c40cdeb0 con 0x7f17c406dd30 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.382+0000 7f17c2800640 1 --2- 172.21.15.86:0/604057099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c406dd30 0x7f17c4166ac0 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f17bc009d90 tx=0x7f17bc02beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a26e654171d4569 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c1400640 1 -- 172.21.15.86:0/604057099 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17bc039070 con 0x7f17c406dd30 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c1400640 1 -- 172.21.15.86:0/604057099 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17bc02cc60 con 0x7f17c406dd30 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c1400640 1 -- 172.21.15.86:0/604057099 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17bc034640 con 0x7f17c406dd30 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c8c00640 1 -- 172.21.15.86:0/604057099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c406dd30 msgr2=0x7f17c4166ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c8c00640 1 --2- 172.21.15.86:0/604057099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c406dd30 0x7f17c4166ac0 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f17bc009d90 tx=0x7f17bc02beb0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c8c00640 1 -- 172.21.15.86:0/604057099 shutdown_connections 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c8c00640 1 --2- 172.21.15.86:0/604057099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c406dd30 0x7f17c4166ac0 unknown :-1 s=CLOSED pgs=739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c8c00640 1 -- 172.21.15.86:0/604057099 >> 172.21.15.86:0/604057099 conn(0x7f17c40cfa80 msgr2=0x7f17c40cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:22.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.383+0000 7f17c8c00640 1 -- 172.21.15.86:0/604057099 shutdown_connections 2024-03-06T07:30:22.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.384+0000 7f17c8c00640 1 -- 172.21.15.86:0/604057099 wait complete. 2024-03-06T07:30:22.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.385+0000 7f17c8c00640 1 Processor -- start 2024-03-06T07:30:22.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.385+0000 7f17c8c00640 1 -- start start 2024-03-06T07:30:22.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17c8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17c406dd30 0x7f17c413c560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:22.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17c8c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c40cd450 0x7f17c41575a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:22.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17c8c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17c4145750 0x7f17c40b3290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:22.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17c8c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17c415c440 con 0x7f17c40cd450 2024-03-06T07:30:22.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17c8c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17c40ce470 con 0x7f17c4145750 2024-03-06T07:30:22.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17c8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17c415c5c0 con 0x7f17c406dd30 2024-03-06T07:30:22.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17b9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c40cd450 0x7f17c41575a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17b9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c40cd450 0x7f17c41575a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33646/0 (socket says 172.21.15.86:33646) 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.386+0000 7f17b9400640 1 -- 172.21.15.86:0/817031965 learned_addr learned my addr 172.21.15.86:0/817031965 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17b9400640 1 -- 172.21.15.86:0/817031965 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17c406dd30 msgr2=0x7f17c413c560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17b9400640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17c406dd30 0x7f17c413c560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17b9400640 1 -- 172.21.15.86:0/817031965 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17c4145750 msgr2=0x7f17c40b3290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17b9400640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17c4145750 0x7f17c40b3290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17b9400640 1 -- 172.21.15.86:0/817031965 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17c4158170 con 0x7f17c40cd450 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17b9400640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c40cd450 0x7f17c41575a0 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f17ac00c500 tx=0x7f17ac00c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17baa00640 1 -- 172.21.15.86:0/817031965 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17ac017070 con 0x7f17c40cd450 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17baa00640 1 -- 172.21.15.86:0/817031965 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17ac005ce0 con 0x7f17c40cd450 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17baa00640 1 -- 172.21.15.86:0/817031965 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17ac007330 con 0x7f17c40cd450 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17c4158450 con 0x7f17c40cd450 2024-03-06T07:30:22.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.387+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17c4158a10 con 0x7f17c40cd450 2024-03-06T07:30:22.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.389+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f17c404f3a0 con 0x7f17c40cd450 2024-03-06T07:30:22.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.389+0000 7f17baa00640 1 -- 172.21.15.86:0/817031965 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f17ac01a070 con 0x7f17c40cd450 2024-03-06T07:30:22.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.390+0000 7f17baa00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17a006b4d0 0x7f17a006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:22.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.390+0000 7f17baa00640 1 -- 172.21.15.86:0/817031965 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(59..59 src has 1..59) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f17ac089730 con 0x7f17c40cd450 2024-03-06T07:30:22.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.391+0000 7f17c1e00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17a006b4d0 0x7f17a006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:22.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.391+0000 7f17c1e00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17a006b4d0 0x7f17a006d990 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f17b4000f60 tx=0x7f17b40079b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:22.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.395+0000 7f17baa00640 1 -- 172.21.15.86:0/817031965 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f17ac058600 con 0x7f17c40cd450 2024-03-06T07:30:22.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:22.592+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f17c40726c0 con 0x7f17c40cd450 2024-03-06T07:30:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.146+0000 7f17baa00640 1 -- 172.21.15.86:0/817031965 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v60) v1 ==== 181+0+0 (secure 0 0 0) 0x7f17ac057f00 con 0x7f17c40cd450 2024-03-06T07:30:23.149 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:30:23.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17a006b4d0 msgr2=0x7f17a006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:23.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17a006b4d0 0x7f17a006d990 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f17b4000f60 tx=0x7f17b40079b0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c40cd450 msgr2=0x7f17c41575a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c40cd450 0x7f17c41575a0 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f17ac00c500 tx=0x7f17ac00c9c0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 shutdown_connections 2024-03-06T07:30:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17a006b4d0 0x7f17a006d990 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17c4145750 0x7f17c40b3290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17c40cd450 0x7f17c41575a0 unknown :-1 s=CLOSED pgs=740 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 --2- 172.21.15.86:0/817031965 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17c406dd30 0x7f17c413c560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.150+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 >> 172.21.15.86:0/817031965 conn(0x7f17c40cfa80 msgr2=0x7f17c415b840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:23.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.151+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 shutdown_connections 2024-03-06T07:30:23.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.152+0000 7f17c8c00640 1 -- 172.21.15.86:0/817031965 wait complete. 2024-03-06T07:30:23.185 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:30:23.256 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:30:23.260 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:30:23.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.369+0000 7f4191e00640 1 Processor -- start 2024-03-06T07:30:23.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.369+0000 7f4191e00640 1 -- start start 2024-03-06T07:30:23.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.371+0000 7f4191e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f418c15df10 0x7f418c0b4300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:23.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.371+0000 7f4191e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c06cab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:23.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.371+0000 7f4191e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f418c06d170 con 0x7f418c161c30 2024-03-06T07:30:23.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.371+0000 7f4191e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f418c06d450 con 0x7f418c15df10 2024-03-06T07:30:23.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.371+0000 7f4191e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f418c0b4c10 con 0x7f418c15e2f0 2024-03-06T07:30:23.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f4182000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c06cab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:23.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f418aa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f418c15df10 0x7f418c0b4300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:23.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f418b400640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f418c161c30 0x7f418c0cfe90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:45980/0 (socket says 172.21.15.86:45980) 2024-03-06T07:30:23.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f418b400640 1 -- 172.21.15.86:0/1207768743 learned_addr learned my addr 172.21.15.86:0/1207768743 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:23.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f418aa00640 1 -- 172.21.15.86:0/1207768743 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 msgr2=0x7f418c06cab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:23.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f418aa00640 1 --2- 172.21.15.86:0/1207768743 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c06cab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f418aa00640 1 -- 172.21.15.86:0/1207768743 >> v1:172.21.15.194:6789/0 conn(0x7f418c161c30 legacy=0x7f418c0cfe90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:23.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.372+0000 7f418aa00640 1 -- 172.21.15.86:0/1207768743 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f418c0b4ef0 con 0x7f418c15df10 2024-03-06T07:30:23.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.374+0000 7f418aa00640 1 --2- 172.21.15.86:0/1207768743 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c15df10 0x7f418c0b4300 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f417c009d90 tx=0x7f417c02bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c5e5b6d0c0f6e540 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:23.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.374+0000 7f418a000640 1 -- 172.21.15.86:0/1207768743 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f417c039070 con 0x7f418c15df10 2024-03-06T07:30:23.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.374+0000 7f418a000640 1 -- 172.21.15.86:0/1207768743 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f417c02cc50 con 0x7f418c15df10 2024-03-06T07:30:23.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.374+0000 7f418a000640 1 -- 172.21.15.86:0/1207768743 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f417c034440 con 0x7f418c15df10 2024-03-06T07:30:23.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 -- 172.21.15.86:0/1207768743 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c15df10 msgr2=0x7f418c0b4300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:23.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 --2- 172.21.15.86:0/1207768743 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c15df10 0x7f418c0b4300 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f417c009d90 tx=0x7f417c02bea0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 -- 172.21.15.86:0/1207768743 shutdown_connections 2024-03-06T07:30:23.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 --2- 172.21.15.86:0/1207768743 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c06cab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 --2- 172.21.15.86:0/1207768743 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c15df10 0x7f418c0b4300 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 -- 172.21.15.86:0/1207768743 >> 172.21.15.86:0/1207768743 conn(0x7f418c0b37e0 msgr2=0x7f418c0b3bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:23.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 -- 172.21.15.86:0/1207768743 shutdown_connections 2024-03-06T07:30:23.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.375+0000 7f4191e00640 1 -- 172.21.15.86:0/1207768743 wait complete. 2024-03-06T07:30:23.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.377+0000 7f4191e00640 1 Processor -- start 2024-03-06T07:30:23.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.377+0000 7f4191e00640 1 -- start start 2024-03-06T07:30:23.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f4191e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f418c15df10 0x7f418c0d25e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:23.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f4191e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c1591f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:23.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f4191e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c161c30 0x7f418c0d0f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f4191e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f418c0b6880 con 0x7f418c15df10 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f4191e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f418c0b6700 con 0x7f418c161c30 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f4191e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f418c0b6a00 con 0x7f418c15e2f0 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f418aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f418c15df10 0x7f418c0d25e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f418aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f418c15df10 0x7f418c0d25e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33652/0 (socket says 172.21.15.86:33652) 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.378+0000 7f418aa00640 1 -- 172.21.15.86:0/936302784 learned_addr learned my addr 172.21.15.86:0/936302784 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f4182000640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c1591f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f418aa00640 1 -- 172.21.15.86:0/936302784 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 msgr2=0x7f418c1591f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f418aa00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c1591f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f418aa00640 1 -- 172.21.15.86:0/936302784 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c161c30 msgr2=0x7f418c0d0f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f418aa00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c161c30 0x7f418c0d0f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f418aa00640 1 -- 172.21.15.86:0/936302784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f418c0cfef0 con 0x7f418c15df10 2024-03-06T07:30:23.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f418aa00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f418c15df10 0x7f418c0d25e0 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f417c009d90 tx=0x7f417c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:23.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f4183e00640 1 -- 172.21.15.86:0/936302784 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f417c044020 con 0x7f418c15df10 2024-03-06T07:30:23.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f4183e00640 1 -- 172.21.15.86:0/936302784 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f417c02ca50 con 0x7f418c15df10 2024-03-06T07:30:23.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.379+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f418c0d01d0 con 0x7f418c15df10 2024-03-06T07:30:23.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.380+0000 7f4183e00640 1 -- 172.21.15.86:0/936302784 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f417c039040 con 0x7f418c15df10 2024-03-06T07:30:23.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.380+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f418c0d0790 con 0x7f418c15df10 2024-03-06T07:30:23.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.381+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f418c04f330 con 0x7f418c15df10 2024-03-06T07:30:23.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.382+0000 7f4183e00640 1 -- 172.21.15.86:0/936302784 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f417c047020 con 0x7f418c15df10 2024-03-06T07:30:23.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.384+0000 7f4183e00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f416806b4d0 0x7f416806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:23.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.384+0000 7f4183e00640 1 -- 172.21.15.86:0/936302784 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(60..60 src has 1..60) v4 ==== 5357+0+0 (secure 0 0 0) 0x7f417c0aca80 con 0x7f418c15df10 2024-03-06T07:30:23.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.384+0000 7f4182000640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f416806b4d0 0x7f416806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:23.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.384+0000 7f4182000640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f416806b4d0 0x7f416806d990 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f418c151b90 tx=0x7f4174008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:23.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.390+0000 7f4183e00640 1 -- 172.21.15.86:0/936302784 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f417c04e030 con 0x7f418c15df10 2024-03-06T07:30:23.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:23.589+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f418c072fd0 con 0x7f418c15df10 2024-03-06T07:30:24.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.160+0000 7f4183e00640 1 -- 172.21.15.86:0/936302784 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v61) v1 ==== 169+0+0 (secure 0 0 0) 0x7f417c02cc20 con 0x7f418c15df10 2024-03-06T07:30:24.164 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f416806b4d0 msgr2=0x7f416806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f416806b4d0 0x7f416806d990 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f418c151b90 tx=0x7f4174008040 comp rx=0 tx=0).stop 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f418c15df10 msgr2=0x7f418c0d25e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f418c15df10 0x7f418c0d25e0 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f417c009d90 tx=0x7f417c004a40 comp rx=0 tx=0).stop 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 shutdown_connections 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f416806b4d0 0x7f416806d990 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f418c161c30 0x7f418c0d0f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f418c15e2f0 0x7f418c1591f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 --2- 172.21.15.86:0/936302784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f418c15df10 0x7f418c0d25e0 unknown :-1 s=CLOSED pgs=741 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.164+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 >> 172.21.15.86:0/936302784 conn(0x7f418c0b37e0 msgr2=0x7f418c0b3bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:24.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.165+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 shutdown_connections 2024-03-06T07:30:24.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.165+0000 7f4191e00640 1 -- 172.21.15.86:0/936302784 wait complete. 2024-03-06T07:30:24.193 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-06T07:30:24.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.434+0000 7f9bcbc00640 1 Processor -- start 2024-03-06T07:30:24.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.434+0000 7f9bcbc00640 1 -- start start 2024-03-06T07:30:24.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.436+0000 7f9bcbc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 0x7f9bc4070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.436+0000 7f9bcbc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9bc40b46c0 con 0x7f9bc40b3f70 2024-03-06T07:30:24.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.436+0000 7f9bcbc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9bc40b49a0 con 0x7f9bc415fc10 2024-03-06T07:30:24.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.436+0000 7f9bcbc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc4072c90 con 0x7f9bc4160700 2024-03-06T07:30:24.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.436+0000 7f9bc3e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9bc40b3f70 0x7f9bc40ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37398/0 (socket says 172.21.15.86:37398) 2024-03-06T07:30:24.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.436+0000 7f9bc3e00640 1 -- 172.21.15.86:0/3132577821 learned_addr learned my addr 172.21.15.86:0/3132577821 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:24.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc9c00640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 0x7f9bc4070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3234880880 0 0) 0x7f9bc40b46c0 con 0x7f9bc40b3f70 2024-03-06T07:30:24.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9ba8003520 con 0x7f9bc40b3f70 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4207216587 0 0) 0x7f9ba8003520 con 0x7f9bc40b3f70 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 msgr2=0x7f9bc4070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 0x7f9bc4070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 >> v1:172.21.15.194:6789/0 conn(0x7f9bc415fc10 legacy=0x7f9bc415fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9bc409dd80 con 0x7f9bc40b3f70 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9bb0001670 con 0x7f9bc40b3f70 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 >> v1:172.21.15.86:6789/0 conn(0x7f9bc40b3f70 legacy=0x7f9bc40ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 0x7f9ba80039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba8003f60 0x7f9ba8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9ba80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc4072c90 con 0x7f9ba8004d40 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ba800baa0 con 0x7f9ba8003f60 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.437+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9ba800bca0 con 0x7f9bc4160700 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9c00640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9ba80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9c00640 1 -- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 msgr2=0x7f9ba80039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9200640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 0x7f9ba80039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc3e00640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba8003f60 0x7f9ba8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9c00640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 0x7f9ba80039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9c00640 1 -- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba8003f60 msgr2=0x7f9ba8004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:24.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9c00640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba8003f60 0x7f9ba8004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9c00640 1 -- 172.21.15.86:0/3132577821 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ba8003520 con 0x7f9ba8004d40 2024-03-06T07:30:24.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.438+0000 7f9bc9200640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160700 0x7f9ba80039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:24.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.439+0000 7f9bc9c00640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9ba80051c0 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f9bbc009ac0 tx=0x7f9ba8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6067f8241209d18a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:24.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.439+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bbc039070 con 0x7f9ba8004d40 2024-03-06T07:30:24.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.439+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9bbc02c6b0 con 0x7f9ba8004d40 2024-03-06T07:30:24.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.439+0000 7f9bc3400640 1 -- 172.21.15.86:0/3132577821 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bbc009690 con 0x7f9ba8004d40 2024-03-06T07:30:24.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.440+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3132577821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 msgr2=0x7f9ba80051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.440+0000 7f9bcbc00640 1 --2- 172.21.15.86:0/3132577821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9ba80051c0 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f9bbc009ac0 tx=0x7f9ba8005830 comp rx=0 tx=0).stop 2024-03-06T07:30:24.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.440+0000 7f9bc9c00640 1 -- 172.21.15.86:0/3132577821 reap_dead start 2024-03-06T07:30:24.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.440+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3132577821 shutdown_connections 2024-03-06T07:30:24.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.440+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3132577821 >> 172.21.15.86:0/3132577821 conn(0x7f9bc40b1940 msgr2=0x7f9bc40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:24.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.440+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3132577821 shutdown_connections 2024-03-06T07:30:24.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.440+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3132577821 wait complete. 2024-03-06T07:30:24.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.442+0000 7f9bcbc00640 1 Processor -- start 2024-03-06T07:30:24.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.442+0000 7f9bcbc00640 1 -- start start 2024-03-06T07:30:24.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bcbc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9bc415f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bcbc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4154670 0x7f9bc4154af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bcbc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9bc409bdc0 0x7f9bc409c240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bcbc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc415c460 con 0x7f9ba8004d40 2024-03-06T07:30:24.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bcbc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc415c2e0 con 0x7f9bc409bdc0 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bcbc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc4070ff0 con 0x7f9bc4154670 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bc9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9bc415f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bc9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9bc415f170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33668/0 (socket says 172.21.15.86:33668) 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.443+0000 7f9bc9200640 1 -- 172.21.15.86:0/3158742627 learned_addr learned my addr 172.21.15.86:0/3158742627 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc3e00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4154670 0x7f9bc4154af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc9200640 1 -- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4154670 msgr2=0x7f9bc4154af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc9200640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4154670 0x7f9bc4154af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc9200640 1 -- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9bc409bdc0 msgr2=0x7f9bc409c240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc9c00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9bc409bdc0 0x7f9bc409c240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc9200640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9bc409bdc0 0x7f9bc409c240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc9200640 1 -- 172.21.15.86:0/3158742627 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9bc409d740 con 0x7f9ba8004d40 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc9200640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9bc415f170 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f9bb400db40 tx=0x7f9bb40020c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:24.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc0c00640 1 -- 172.21.15.86:0/3158742627 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bb4018070 con 0x7f9ba8004d40 2024-03-06T07:30:24.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc0c00640 1 -- 172.21.15.86:0/3158742627 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9bb4003da0 con 0x7f9ba8004d40 2024-03-06T07:30:24.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9bc409cb30 con 0x7f9ba8004d40 2024-03-06T07:30:24.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.444+0000 7f9bc0c00640 1 -- 172.21.15.86:0/3158742627 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bb4006d70 con 0x7f9ba8004d40 2024-03-06T07:30:24.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.445+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bc409d0f0 con 0x7f9ba8004d40 2024-03-06T07:30:24.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.446+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9bc404f330 con 0x7f9ba8004d40 2024-03-06T07:30:24.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.446+0000 7f9bc0c00640 1 -- 172.21.15.86:0/3158742627 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9bb401b070 con 0x7f9ba8004d40 2024-03-06T07:30:24.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.447+0000 7f9bc0c00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba406b4d0 0x7f9ba406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.447+0000 7f9bc0c00640 1 -- 172.21.15.86:0/3158742627 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f9bb4089770 con 0x7f9ba8004d40 2024-03-06T07:30:24.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.448+0000 7f9bc3e00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba406b4d0 0x7f9ba406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.448+0000 7f9bc3e00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba406b4d0 0x7f9ba406d990 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f9bb0001200 tx=0x7f9bb0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:24.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.452+0000 7f9bc0c00640 1 -- 172.21.15.86:0/3158742627 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9bb40588f0 con 0x7f9ba8004d40 2024-03-06T07:30:24.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.647+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f9bc406ef30 con 0x7f9ba8004d40 2024-03-06T07:30:24.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.654+0000 7f9bc0c00640 1 -- 172.21.15.86:0/3158742627 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v19) v1 ==== 104+0+0 (secure 0 0 0) 0x7f9bb40581f0 con 0x7f9ba8004d40 2024-03-06T07:30:24.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba406b4d0 msgr2=0x7f9ba406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba406b4d0 0x7f9ba406d990 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f9bb0001200 tx=0x7f9bb0009040 comp rx=0 tx=0).stop 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 msgr2=0x7f9bc415f170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9bc415f170 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f9bb400db40 tx=0x7f9bb40020c0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 shutdown_connections 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba406b4d0 0x7f9ba406d990 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9bc409bdc0 0x7f9bc409c240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4154670 0x7f9bc4154af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 --2- 172.21.15.86:0/3158742627 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba8004d40 0x7f9bc415f170 unknown :-1 s=CLOSED pgs=744 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 >> 172.21.15.86:0/3158742627 conn(0x7f9bc40b1940 msgr2=0x7f9bc40cc690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 shutdown_connections 2024-03-06T07:30:24.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.657+0000 7f9bcbc00640 1 -- 172.21.15.86:0/3158742627 wait complete. 2024-03-06T07:30:24.678 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_merge_size 2024-03-06T07:30:24.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.883+0000 7fd911200640 1 Processor -- start 2024-03-06T07:30:24.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.883+0000 7fd911200640 1 -- start start 2024-03-06T07:30:24.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.885+0000 7fd911200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 0x7fd90c070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.885+0000 7fd911200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd90c15fff0 0x7fd90c160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.885+0000 7fd911200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd90c0b46c0 con 0x7fd90c0ce4a0 2024-03-06T07:30:24.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.885+0000 7fd911200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd90c0b49a0 con 0x7fd90c15fff0 2024-03-06T07:30:24.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.885+0000 7fd911200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd90c072c90 con 0x7fd90c15fc10 2024-03-06T07:30:24.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.885+0000 7fd90a800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd90c0ce4a0 0x7fd90c0ce930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37404/0 (socket says 172.21.15.86:37404) 2024-03-06T07:30:24.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.885+0000 7fd90a800640 1 -- 172.21.15.86:0/3781315418 learned_addr learned my addr 172.21.15.86:0/3781315418 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:24.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd901400640 1 --2- 172.21.15.86:0/3781315418 >> v2:172.21.15.194:3300/0 conn(0x7fd90c15fff0 0x7fd90c160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909e00640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 0x7fd90c070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3972466372 0 0) 0x7fd90c0b46c0 con 0x7fd90c0ce4a0 2024-03-06T07:30:24.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd8f0003520 con 0x7fd90c0ce4a0 2024-03-06T07:30:24.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1194335426 0 0) 0x7fd8f0003520 con 0x7fd90c0ce4a0 2024-03-06T07:30:24.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 msgr2=0x7fd90c070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:24.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 0x7fd90c070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 >> v2:172.21.15.194:3300/0 conn(0x7fd90c15fff0 msgr2=0x7fd90c160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 --2- 172.21.15.86:0/3781315418 >> v2:172.21.15.194:3300/0 conn(0x7fd90c15fff0 0x7fd90c160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd90c160990 con 0x7fd90c0ce4a0 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd904001670 con 0x7fd90c0ce4a0 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909e00640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 0x7fd90c070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 >> v1:172.21.15.86:6789/0 conn(0x7fd90c0ce4a0 legacy=0x7fd90c0ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.886+0000 7fd909400640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd8f00039b0 0x7fd8f0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd909400640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 0x7fd8f0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd909400640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd8f0004d40 0x7fd8f00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd90c0b49a0 con 0x7fd8f0004d40 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd90c072c90 con 0x7fd8f00039b0 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd8f000b9c0 con 0x7fd90c15fc10 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd90a800640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd8f0004d40 0x7fd8f00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd909e00640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd8f00039b0 0x7fd8f0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd901400640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 0x7fd8f0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd90a800640 1 -- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 msgr2=0x7fd8f0004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd90a800640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd90c15fc10 0x7fd8f0004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd90a800640 1 -- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd8f00039b0 msgr2=0x7fd8f0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd90a800640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd8f00039b0 0x7fd8f0003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.887+0000 7fd90a800640 1 -- 172.21.15.86:0/3781315418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8f0003520 con 0x7fd8f0004d40 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.888+0000 7fd90a800640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd8f0004d40 0x7fd8f00051c0 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7fd90400c210 tx=0x7fd8f0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9f5c404745919a2d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd904007d50 con 0x7fd8f0004d40 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd904030070 con 0x7fd8f0004d40 2024-03-06T07:30:24.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd909400640 1 -- 172.21.15.86:0/3781315418 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd90402cd80 con 0x7fd8f0004d40 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd911200640 1 -- 172.21.15.86:0/3781315418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd8f0004d40 msgr2=0x7fd8f00051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd911200640 1 --2- 172.21.15.86:0/3781315418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd8f0004d40 0x7fd8f00051c0 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7fd90400c210 tx=0x7fd8f0005830 comp rx=0 tx=0).stop 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd90a800640 1 -- 172.21.15.86:0/3781315418 reap_dead start 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd911200640 1 -- 172.21.15.86:0/3781315418 shutdown_connections 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd911200640 1 -- 172.21.15.86:0/3781315418 >> 172.21.15.86:0/3781315418 conn(0x7fd90c0b1940 msgr2=0x7fd90c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd911200640 1 -- 172.21.15.86:0/3781315418 shutdown_connections 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.889+0000 7fd911200640 1 -- 172.21.15.86:0/3781315418 wait complete. 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.891+0000 7fd911200640 1 Processor -- start 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.891+0000 7fd911200640 1 -- start start 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd911200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8f0004d40 0x7fd90c157630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd911200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd90c15d570 0x7fd90c15d9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd911200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd90c15b310 0x7fd90c15b790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd911200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd90c072a30 con 0x7fd90c15b310 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd911200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd90c0728b0 con 0x7fd90c15d570 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd911200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd90c15c2e0 con 0x7fd8f0004d40 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd90a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd90c15b310 0x7fd90c15b790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd90a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd90c15b310 0x7fd90c15b790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33688/0 (socket says 172.21.15.86:33688) 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.892+0000 7fd90a800640 1 -- 172.21.15.86:0/3016907255 learned_addr learned my addr 172.21.15.86:0/3016907255 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd901400640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd90c15d570 0x7fd90c15d9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd909e00640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8f0004d40 0x7fd90c157630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd90a800640 1 -- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8f0004d40 msgr2=0x7fd90c157630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd90a800640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8f0004d40 0x7fd90c157630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd90a800640 1 -- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd90c15d570 msgr2=0x7fd90c15d9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd90a800640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd90c15d570 0x7fd90c15d9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd90a800640 1 -- 172.21.15.86:0/3016907255 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd90c157b70 con 0x7fd90c15b310 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd909e00640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8f0004d40 0x7fd90c157630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:24.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd901400640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd90c15d570 0x7fd90c15d9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:24.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.893+0000 7fd90a800640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd90c15b310 0x7fd90c15b790 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7fd90402c590 tx=0x7fd904001ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:24.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.894+0000 7fd902a00640 1 -- 172.21.15.86:0/3016907255 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd904007d50 con 0x7fd90c15b310 2024-03-06T07:30:24.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.894+0000 7fd902a00640 1 -- 172.21.15.86:0/3016907255 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd904004bf0 con 0x7fd90c15b310 2024-03-06T07:30:24.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.894+0000 7fd902a00640 1 -- 172.21.15.86:0/3016907255 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd904001c90 con 0x7fd90c15b310 2024-03-06T07:30:24.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.894+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd90c15e0e0 con 0x7fd90c15b310 2024-03-06T07:30:24.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.894+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd90c15e6a0 con 0x7fd90c15b310 2024-03-06T07:30:24.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.895+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd90c04f330 con 0x7fd90c15b310 2024-03-06T07:30:24.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.896+0000 7fd902a00640 1 -- 172.21.15.86:0/3016907255 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd904007070 con 0x7fd90c15b310 2024-03-06T07:30:24.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.897+0000 7fd902a00640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd8e806b4d0 0x7fd8e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:24.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.897+0000 7fd902a00640 1 -- 172.21.15.86:0/3016907255 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fd9040aab90 con 0x7fd90c15b310 2024-03-06T07:30:24.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.897+0000 7fd909e00640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd8e806b4d0 0x7fd8e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:24.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.898+0000 7fd909e00640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd8e806b4d0 0x7fd8e806d990 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fd8fc0095c0 tx=0x7fd8fc00b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:24.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:24.904+0000 7fd902a00640 1 -- 172.21.15.86:0/3016907255 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd904048070 con 0x7fd90c15b310 2024-03-06T07:30:25.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.103+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_merge_size"} v 0) v1 -- 0x7fd90c06ef30 con 0x7fd90c15b310 2024-03-06T07:30:25.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.110+0000 7fd902a00640 1 -- 172.21.15.86:0/3016907255 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_merge_size"}]=0 v20) v1 ==== 101+0+0 (secure 0 0 0) 0x7fd904079c70 con 0x7fd90c15b310 2024-03-06T07:30:25.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd8e806b4d0 msgr2=0x7fd8e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd8e806b4d0 0x7fd8e806d990 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fd8fc0095c0 tx=0x7fd8fc00b040 comp rx=0 tx=0).stop 2024-03-06T07:30:25.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd90c15b310 msgr2=0x7fd90c15b790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd90c15b310 0x7fd90c15b790 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7fd90402c590 tx=0x7fd904001ab0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 shutdown_connections 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd8e806b4d0 0x7fd8e806d990 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd90c15b310 0x7fd90c15b790 unknown :-1 s=CLOSED pgs=747 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd90c15d570 0x7fd90c15d9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 --2- 172.21.15.86:0/3016907255 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8f0004d40 0x7fd90c157630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.113+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 >> 172.21.15.86:0/3016907255 conn(0x7fd90c0b1940 msgr2=0x7fd90c0adbf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.114+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 shutdown_connections 2024-03-06T07:30:25.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.114+0000 7fd911200640 1 -- 172.21.15.86:0/3016907255 wait complete. 2024-03-06T07:30:25.135 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_fragment_size_max 2024-03-06T07:30:25.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.341+0000 7f35d9a00640 1 Processor -- start 2024-03-06T07:30:25.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.341+0000 7f35d9a00640 1 -- start start 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d40ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35d40cfde0 0x7f35d40cd7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f35d415b130 con 0x7f35d4158950 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35d40af170 con 0x7f35d40cfde0 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35d415b3f0 con 0x7f35d4158570 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35d40cfde0 0x7f35d40cd7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35d40cfde0 0x7f35d40cd7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33690/0 (socket says 172.21.15.86:33690) 2024-03-06T07:30:25.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.343+0000 7f35d9000640 1 -- 172.21.15.86:0/3862071858 learned_addr learned my addr 172.21.15.86:0/3862071858 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:25.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.344+0000 7f35d9000640 1 -- 172.21.15.86:0/3862071858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 msgr2=0x7f35d40ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:25.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.344+0000 7f35d3e00640 1 --2- 172.21.15.86:0/3862071858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d40ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.344+0000 7f35d9000640 1 --2- 172.21.15.86:0/3862071858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d40ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.344+0000 7f35d9000640 1 -- 172.21.15.86:0/3862071858 >> v1:172.21.15.194:6789/0 conn(0x7f35d4158950 legacy=0x7f35d40cf6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:25.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.344+0000 7f35d9000640 1 -- 172.21.15.86:0/3862071858 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35d415b6d0 con 0x7f35d40cfde0 2024-03-06T07:30:25.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.344+0000 7f35d3e00640 1 --2- 172.21.15.86:0/3862071858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d40ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:25.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.345+0000 7f35d9000640 1 --2- 172.21.15.86:0/3862071858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d40cfde0 0x7f35d40cd7c0 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f35cc009e50 tx=0x7f35cc02be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e7d6a09f583f8ae7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:25.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d2a00640 1 -- 172.21.15.86:0/3862071858 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35cc02c9e0 con 0x7f35d40cfde0 2024-03-06T07:30:25.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d2a00640 1 -- 172.21.15.86:0/3862071858 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35cc02cb80 con 0x7f35d40cfde0 2024-03-06T07:30:25.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d2a00640 1 -- 172.21.15.86:0/3862071858 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35cc0343f0 con 0x7f35d40cfde0 2024-03-06T07:30:25.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d9a00640 1 -- 172.21.15.86:0/3862071858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d40cfde0 msgr2=0x7f35d40cd7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d9a00640 1 --2- 172.21.15.86:0/3862071858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d40cfde0 0x7f35d40cd7c0 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f35cc009e50 tx=0x7f35cc02be40 comp rx=0 tx=0).stop 2024-03-06T07:30:25.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d9a00640 1 -- 172.21.15.86:0/3862071858 shutdown_connections 2024-03-06T07:30:25.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d9a00640 1 --2- 172.21.15.86:0/3862071858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d40cfde0 0x7f35d40cd7c0 unknown :-1 s=CLOSED pgs=748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d9a00640 1 --2- 172.21.15.86:0/3862071858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d40ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.346+0000 7f35d9a00640 1 -- 172.21.15.86:0/3862071858 >> 172.21.15.86:0/3862071858 conn(0x7f35d406afc0 msgr2=0x7f35d406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:25.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.347+0000 7f35d9a00640 1 -- 172.21.15.86:0/3862071858 shutdown_connections 2024-03-06T07:30:25.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.347+0000 7f35d9a00640 1 -- 172.21.15.86:0/3862071858 wait complete. 2024-03-06T07:30:25.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.348+0000 7f35d9a00640 1 Processor -- start 2024-03-06T07:30:25.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.348+0000 7f35d9a00640 1 -- start start 2024-03-06T07:30:25.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.349+0000 7f35d9a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d40cfde0 0x7f35d4149410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.349+0000 7f35d9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d4159310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.349+0000 7f35d9a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d4158950 0x7f35d4153810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.349+0000 7f35d9a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35d415bed0 con 0x7f35d4158950 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.349+0000 7f35d9a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35d40b4950 con 0x7f35d40cfde0 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.349+0000 7f35d9a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35d415c050 con 0x7f35d4158570 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d4158950 0x7f35d4153810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d4158950 0x7f35d4153810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33704/0 (socket says 172.21.15.86:33704) 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 -- 172.21.15.86:0/1249919940 learned_addr learned my addr 172.21.15.86:0/1249919940 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 -- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 msgr2=0x7f35d4159310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d3400640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d4159310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d4159310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d3e00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d40cfde0 0x7f35d4149410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 -- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d40cfde0 msgr2=0x7f35d4149410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d40cfde0 0x7f35d4149410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d9000640 1 -- 172.21.15.86:0/1249919940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35d415c400 con 0x7f35d4158950 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d3400640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d4159310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:25.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.350+0000 7f35d3e00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d40cfde0 0x7f35d4149410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.351+0000 7f35d9000640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d4158950 0x7f35d4153810 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7f35cc009f70 tx=0x7f35cc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.351+0000 7f35cbe00640 1 -- 172.21.15.86:0/1249919940 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35cc043020 con 0x7f35d4158950 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.351+0000 7f35cbe00640 1 -- 172.21.15.86:0/1249919940 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35cc03e450 con 0x7f35d4158950 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.351+0000 7f35cbe00640 1 -- 172.21.15.86:0/1249919940 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35cc009510 con 0x7f35d4158950 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.351+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35d415c6e0 con 0x7f35d4158950 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.351+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35d4149950 con 0x7f35d4158950 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.352+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f35d404f330 con 0x7f35d4158950 2024-03-06T07:30:25.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.353+0000 7f35cbe00640 1 -- 172.21.15.86:0/1249919940 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f35cc046020 con 0x7f35d4158950 2024-03-06T07:30:25.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.354+0000 7f35cbe00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.354+0000 7f35cbe00640 1 -- 172.21.15.86:0/1249919940 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f35cc0acb00 con 0x7f35d4158950 2024-03-06T07:30:25.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.354+0000 7f35d3e00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.355+0000 7f35d3e00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f35c4006fd0 tx=0x7f35c4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:25.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.358+0000 7f35cbe00640 1 -- 172.21.15.86:0/1249919940 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f35cc04d020 con 0x7f35d4158950 2024-03-06T07:30:25.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.556+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"} v 0) v1 -- 0x7f35d40cfab0 con 0x7f35d4158950 2024-03-06T07:30:25.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.561+0000 7f35cbe00640 1 -- 172.21.15.86:0/1249919940 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"}]=0 v21) v1 ==== 108+0+0 (secure 0 0 0) 0x7f35cc0ace20 con 0x7f35d4158950 2024-03-06T07:30:25.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.564+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 msgr2=0x7f35b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.564+0000 7f35d9a00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f35c4006fd0 tx=0x7f35c4008040 comp rx=0 tx=0).stop 2024-03-06T07:30:25.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.564+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d4158950 msgr2=0x7f35d4153810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.564+0000 7f35d9a00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d4158950 0x7f35d4153810 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7f35cc009f70 tx=0x7f35cc004a40 comp rx=0 tx=0).stop 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 shutdown_connections 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f35b406b4d0 0x7f35b406d990 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35d4158950 0x7f35d4153810 unknown :-1 s=CLOSED pgs=749 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35d4158570 0x7f35d4159310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 --2- 172.21.15.86:0/1249919940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35d40cfde0 0x7f35d4149410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 >> 172.21.15.86:0/1249919940 conn(0x7f35d406afc0 msgr2=0x7f35d406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 shutdown_connections 2024-03-06T07:30:25.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.565+0000 7f35d9a00640 1 -- 172.21.15.86:0/1249919940 wait complete. 2024-03-06T07:30:25.587 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented' 2024-03-06T07:30:25.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.783+0000 7f81cfc00640 1 Processor -- start 2024-03-06T07:30:25.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.783+0000 7f81cfc00640 1 -- start start 2024-03-06T07:30:25.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.785+0000 7f81cfc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 0x7f81c80c9ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.785+0000 7f81cfc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81c80cabf0 con 0x7f81c80cf3e0 2024-03-06T07:30:25.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.785+0000 7f81cfc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81c80ce900 con 0x7f81c8144660 2024-03-06T07:30:25.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.785+0000 7f81cfc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81c80cebe0 con 0x7f81c8072680 2024-03-06T07:30:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81cdc00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f81c80cf3e0 0x7f81c80ca400 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37406/0 (socket says 172.21.15.86:37406) 2024-03-06T07:30:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81cdc00640 1 -- 172.21.15.86:0/1449806066 learned_addr learned my addr 172.21.15.86:0/1449806066 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7e00640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 0x7f81c80c9ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3265509123 0 0) 0x7f81c80cabf0 con 0x7f81c80cf3e0 2024-03-06T07:30:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f81a8003520 con 0x7f81c80cf3e0 2024-03-06T07:30:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3583132648 0 0) 0x7f81a8003520 con 0x7f81c80cf3e0 2024-03-06T07:30:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 msgr2=0x7f81c80c9ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 0x7f81c80c9ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 >> v1:172.21.15.194:6789/0 conn(0x7f81c8144660 legacy=0x7f81c8071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81c80ceec0 con 0x7f81c80cf3e0 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.786+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f81c00019f0 con 0x7f81c80cf3e0 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 >> v1:172.21.15.86:6789/0 conn(0x7f81c80cf3e0 legacy=0x7f81c80ca400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7400640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 0x7f81a80039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7400640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a8003fb0 0x7f81a8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7400640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a8004d40 0x7f81a80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81c80cebe0 con 0x7f81a8004d40 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81a800bb00 con 0x7f81a8003fb0 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81a800bd00 con 0x7f81c8072680 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81cdc00640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a8004d40 0x7f81a80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81cd200640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 0x7f81a80039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81c7e00640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a8003fb0 0x7f81a8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81cdc00640 1 -- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 msgr2=0x7f81a80039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:25.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81cdc00640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c8072680 0x7f81a80039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81cdc00640 1 -- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a8003fb0 msgr2=0x7f81a8004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:25.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81cdc00640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a8003fb0 0x7f81a8004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.787+0000 7f81cdc00640 1 -- 172.21.15.86:0/1449806066 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81a8003520 con 0x7f81a8004d40 2024-03-06T07:30:25.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.788+0000 7f81cdc00640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a8004d40 0x7f81a80051c0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7f81c000a770 tx=0x7f81a8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59526fce810c5f69 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:25.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81c000a390 con 0x7f81a8004d40 2024-03-06T07:30:25.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81c0034650 con 0x7f81a8004d40 2024-03-06T07:30:25.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81c7400640 1 -- 172.21.15.86:0/1449806066 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81c0034a80 con 0x7f81a8004d40 2024-03-06T07:30:25.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81cfc00640 1 -- 172.21.15.86:0/1449806066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a8004d40 msgr2=0x7f81a80051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81cfc00640 1 --2- 172.21.15.86:0/1449806066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a8004d40 0x7f81a80051c0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7f81c000a770 tx=0x7f81a8005830 comp rx=0 tx=0).stop 2024-03-06T07:30:25.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81cdc00640 1 -- 172.21.15.86:0/1449806066 reap_dead start 2024-03-06T07:30:25.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81cfc00640 1 -- 172.21.15.86:0/1449806066 shutdown_connections 2024-03-06T07:30:25.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81cfc00640 1 -- 172.21.15.86:0/1449806066 >> 172.21.15.86:0/1449806066 conn(0x7f81c80d0b90 msgr2=0x7f81c80d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:25.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81cfc00640 1 -- 172.21.15.86:0/1449806066 shutdown_connections 2024-03-06T07:30:25.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.789+0000 7f81cfc00640 1 -- 172.21.15.86:0/1449806066 wait complete. 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.791+0000 7f81cfc00640 1 Processor -- start 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.791+0000 7f81cfc00640 1 -- start start 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cfc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a8004d40 0x7f81c813d050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cfc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c813ba60 0x7f81c8154420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cfc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c813fcd0 0x7f81c8140150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cfc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81c80ca040 con 0x7f81c813fcd0 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cfc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81c80c9ec0 con 0x7f81c813ba60 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cfc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81c80ca1c0 con 0x7f81a8004d40 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cdc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c813fcd0 0x7f81c8140150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cdc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c813fcd0 0x7f81c8140150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33714/0 (socket says 172.21.15.86:33714) 2024-03-06T07:30:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.792+0000 7f81cdc00640 1 -- 172.21.15.86:0/89698467 learned_addr learned my addr 172.21.15.86:0/89698467 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:25.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81cdc00640 1 -- 172.21.15.86:0/89698467 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a8004d40 msgr2=0x7f81c813d050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81cdc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a8004d40 0x7f81c813d050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81cdc00640 1 -- 172.21.15.86:0/89698467 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c813ba60 msgr2=0x7f81c8154420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81cdc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c813ba60 0x7f81c8154420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81cdc00640 1 -- 172.21.15.86:0/89698467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81c813eed0 con 0x7f81c813fcd0 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81cdc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c813fcd0 0x7f81c8140150 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7f81c000a770 tx=0x7f81c80cead0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81c4c00640 1 -- 172.21.15.86:0/89698467 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81c00017f0 con 0x7f81c813fcd0 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81c4c00640 1 -- 172.21.15.86:0/89698467 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81c0044070 con 0x7f81c813fcd0 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81c813af80 con 0x7f81c813fcd0 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.793+0000 7f81c4c00640 1 -- 172.21.15.86:0/89698467 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81c0034800 con 0x7f81c813fcd0 2024-03-06T07:30:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.794+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81c815d710 con 0x7f81c813fcd0 2024-03-06T07:30:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.795+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81c804f330 con 0x7f81c813fcd0 2024-03-06T07:30:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.796+0000 7f81c4c00640 1 -- 172.21.15.86:0/89698467 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f81c0034bb0 con 0x7f81c813fcd0 2024-03-06T07:30:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.796+0000 7f81c4c00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f81a406b4d0 0x7f81a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.797+0000 7f81c4c00640 1 -- 172.21.15.86:0/89698467 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f81c00ac430 con 0x7f81c813fcd0 2024-03-06T07:30:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.797+0000 7f81cd200640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f81a406b4d0 0x7f81a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.797+0000 7f81cd200640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f81a406b4d0 0x7f81a406d990 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f81b8007fd0 tx=0x7f81b8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:25.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:25.801+0000 7f81c4c00640 1 -- 172.21.15.86:0/89698467 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f81c007b6c0 con 0x7f81c813fcd0 2024-03-06T07:30:26.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.060+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]} v 0) v1 -- 0x7f81c80cab50 con 0x7f81c813fcd0 2024-03-06T07:30:26.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.166+0000 7f81c4c00640 1 -- 172.21.15.86:0/89698467 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]}]=0 v201) v1 ==== 144+0+0 (secure 0 0 0) 0x7f81c007afc0 con 0x7f81c813fcd0 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f81a406b4d0 msgr2=0x7f81a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f81a406b4d0 0x7f81a406d990 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f81b8007fd0 tx=0x7f81b8009040 comp rx=0 tx=0).stop 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c813fcd0 msgr2=0x7f81c8140150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c813fcd0 0x7f81c8140150 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7f81c000a770 tx=0x7f81c80cead0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 shutdown_connections 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f81a406b4d0 0x7f81a406d990 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c813fcd0 0x7f81c8140150 unknown :-1 s=CLOSED pgs=752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c813ba60 0x7f81c8154420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 --2- 172.21.15.86:0/89698467 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a8004d40 0x7f81c813d050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 >> 172.21.15.86:0/89698467 conn(0x7f81c80d0b90 msgr2=0x7f81c80cd210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 shutdown_connections 2024-03-06T07:30:26.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.169+0000 7f81cfc00640 1 -- 172.21.15.86:0/89698467 wait complete. 2024-03-06T07:30:26.189 INFO:tasks.cephfs_test_runner:That fragmentation (forced) will allow more entries to be created. ... ok 2024-03-06T07:30:26.190 INFO:tasks.cephfs_test_runner:Starting test: test_dirfrag_limit_strays (tasks.cephfs.test_strays.TestStrays) That unlinking fails when the stray directory fragment becomes too 2024-03-06T07:30:26.190 INFO:tasks.cephfs_test_runner:test_dirfrag_limit_strays (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:30:26.190 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:26.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.385+0000 7f8a21600640 1 Processor -- start 2024-03-06T07:30:26.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.385+0000 7f8a21600640 1 -- start start 2024-03-06T07:30:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.386+0000 7f8a21600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8a1c1561f0 0x7f8a1c0ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.386+0000 7f8a21600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8a1c15c240 con 0x7f8a1c0cc9d0 2024-03-06T07:30:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.386+0000 7f8a21600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8a1c15c500 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.386+0000 7f8a21600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8a1c15ce30 con 0x7f8a1c1561f0 2024-03-06T07:30:26.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.387+0000 7f8a1be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8a1c1561f0 0x7f8a1c0ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.387+0000 7f8a20c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8a1c0cc9d0 0x7f8a1c15f830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57100/0 (socket says 172.21.15.86:57100) 2024-03-06T07:30:26.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.387+0000 7f8a20c00640 1 -- 172.21.15.86:0/666540674 learned_addr learned my addr 172.21.15.86:0/666540674 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:26.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.387+0000 7f8a1be00640 1 -- 172.21.15.86:0/666540674 >> v1:172.21.15.194:6789/0 conn(0x7f8a1c0aebf0 legacy=0x7f8a1c0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:26.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.387+0000 7f8a1be00640 1 -- 172.21.15.86:0/666540674 >> v1:172.21.15.86:6789/0 conn(0x7f8a1c0cc9d0 legacy=0x7f8a1c15f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.387+0000 7f8a1be00640 1 -- 172.21.15.86:0/666540674 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a1c15c7e0 con 0x7f8a1c1561f0 2024-03-06T07:30:26.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.388+0000 7f8a1be00640 1 --2- 172.21.15.86:0/666540674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c1561f0 0x7f8a1c0ae6b0 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c0097a0 tx=0x7f8a0c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5ddafd02b0c712bd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:26.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a1b400640 1 -- 172.21.15.86:0/666540674 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c035020 con 0x7f8a1c1561f0 2024-03-06T07:30:26.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a1b400640 1 -- 172.21.15.86:0/666540674 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8a0c031830 con 0x7f8a1c1561f0 2024-03-06T07:30:26.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a1b400640 1 -- 172.21.15.86:0/666540674 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c031c00 con 0x7f8a1c1561f0 2024-03-06T07:30:26.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a21600640 1 -- 172.21.15.86:0/666540674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c1561f0 msgr2=0x7f8a1c0ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a21600640 1 --2- 172.21.15.86:0/666540674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c1561f0 0x7f8a1c0ae6b0 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c0097a0 tx=0x7f8a0c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a21600640 1 -- 172.21.15.86:0/666540674 shutdown_connections 2024-03-06T07:30:26.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a21600640 1 --2- 172.21.15.86:0/666540674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c1561f0 0x7f8a1c0ae6b0 unknown :-1 s=CLOSED pgs=754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a21600640 1 -- 172.21.15.86:0/666540674 >> 172.21.15.86:0/666540674 conn(0x7f8a1c06afc0 msgr2=0x7f8a1c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:26.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a21600640 1 -- 172.21.15.86:0/666540674 shutdown_connections 2024-03-06T07:30:26.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.389+0000 7f8a21600640 1 -- 172.21.15.86:0/666540674 wait complete. 2024-03-06T07:30:26.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.391+0000 7f8a21600640 1 Processor -- start 2024-03-06T07:30:26.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.391+0000 7f8a21600640 1 -- start start 2024-03-06T07:30:26.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a21600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c0aebf0 0x7f8a1c154240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a21600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a1c0cc9d0 0x7f8a1c0ce710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a21600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a1c1561f0 0x7f8a1c0cf7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a21600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a1c1565d0 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a21600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a1c06e360 con 0x7f8a1c1561f0 2024-03-06T07:30:26.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a21600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8a1c156750 con 0x7f8a1c0cc9d0 2024-03-06T07:30:26.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a1be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c0aebf0 0x7f8a1c154240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a1be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c0aebf0 0x7f8a1c154240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55868/0 (socket says 172.21.15.86:55868) 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.392+0000 7f8a1be00640 1 -- 172.21.15.86:0/3823497201 learned_addr learned my addr 172.21.15.86:0/3823497201 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a1be00640 1 -- 172.21.15.86:0/3823497201 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a1c0cc9d0 msgr2=0x7f8a1c0ce710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a1be00640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a1c0cc9d0 0x7f8a1c0ce710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a1be00640 1 -- 172.21.15.86:0/3823497201 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a1c1561f0 msgr2=0x7f8a1c0cf7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a1be00640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a1c1561f0 0x7f8a1c0cf7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a1be00640 1 -- 172.21.15.86:0/3823497201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a1c154780 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a1be00640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c0aebf0 0x7f8a1c154240 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c000c00 tx=0x7f8a0c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:26.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a18c00640 1 -- 172.21.15.86:0/3823497201 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c040070 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a18c00640 1 -- 172.21.15.86:0/3823497201 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8a0c039a00 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a18c00640 1 -- 172.21.15.86:0/3823497201 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c035040 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8a1c0cf1d0 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.393+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a1c1f7be0 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.394+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8a1c04f330 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.395+0000 7f8a18c00640 1 -- 172.21.15.86:0/3823497201 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8a0c04b060 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.397+0000 7f8a18c00640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f89f806b4d0 0x7f89f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.397+0000 7f8a18c00640 1 -- 172.21.15.86:0/3823497201 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f8a0c0ab890 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.397+0000 7f8a13400640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f89f806b4d0 0x7f89f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.397+0000 7f8a13400640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f89f806b4d0 0x7f89f806d990 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c0ce6a0 tx=0x7f8a040079d0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:26.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.401+0000 7f8a18c00640 1 -- 172.21.15.86:0/3823497201 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8a0c07aaa0 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.593+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f8a1c0b2b10 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.595+0000 7f8a18c00640 1 -- 172.21.15.86:0/3823497201 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v61) v1 ==== 74+0+13032 (secure 0 0 0) 0x7f8a0c07a3a0 con 0x7f8a1c0aebf0 2024-03-06T07:30:26.597 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:26.598 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":61,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:24.158881+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/3356338385":"2024-03-07T07:30:21.137139+0000","172.21.15.86:6834/3356338385":"2024-03-07T07:30:21.137139+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:26.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f89f806b4d0 msgr2=0x7f89f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f89f806b4d0 0x7f89f806d990 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c0ce6a0 tx=0x7f8a040079d0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c0aebf0 msgr2=0x7f8a1c154240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c0aebf0 0x7f8a1c154240 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c000c00 tx=0x7f8a0c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 shutdown_connections 2024-03-06T07:30:26.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f89f806b4d0 0x7f89f806d990 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a1c1561f0 0x7f8a1c0cf7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a1c0cc9d0 0x7f8a1c0ce710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 --2- 172.21.15.86:0/3823497201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a1c0aebf0 0x7f8a1c154240 unknown :-1 s=CLOSED pgs=755 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.597+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 >> 172.21.15.86:0/3823497201 conn(0x7f8a1c06afc0 msgr2=0x7f8a1c072600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:26.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.598+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 shutdown_connections 2024-03-06T07:30:26.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.598+0000 7f8a21600640 1 -- 172.21.15.86:0/3823497201 wait complete. 2024-03-06T07:30:26.618 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:30:26.618 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:30:26.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.815+0000 7f6213e00640 1 Processor -- start 2024-03-06T07:30:26.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.815+0000 7f6213e00640 1 -- start start 2024-03-06T07:30:26.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6213e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c0b2c40 0x7f620c15f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6213e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f620c06d960 0x7f620c0b3020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6213e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f620c0b3560 con 0x7f620c06e2f0 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6213e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f620c15f700 con 0x7f620c06d960 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6213e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f620c06d4d0 con 0x7f620c0b2c40 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6211e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f620c06e2f0 0x7f620c161d50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57114/0 (socket says 172.21.15.86:57114) 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6211e00640 1 -- 172.21.15.86:0/2251705644 learned_addr learned my addr 172.21.15.86:0/2251705644 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6211400640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c0b2c40 0x7f620c15f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f6210a00640 1 --2- 172.21.15.86:0/2251705644 >> v2:172.21.15.194:3300/0 conn(0x7f620c06d960 0x7f620c0b3020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.817+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2495157997 0 0) 0x7f620c0b3560 con 0x7f620c06e2f0 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f61f4003520 con 0x7f620c06e2f0 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1537416755 0 0) 0x7f61f4003520 con 0x7f620c06e2f0 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c0b2c40 msgr2=0x7f620c15f1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c0b2c40 0x7f620c15f1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 >> v2:172.21.15.194:3300/0 conn(0x7f620c06d960 msgr2=0x7f620c0b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 --2- 172.21.15.86:0/2251705644 >> v2:172.21.15.194:3300/0 conn(0x7f620c06d960 0x7f620c0b3020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f620c06cba0 con 0x7f620c06e2f0 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6204001670 con 0x7f620c06e2f0 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 >> v1:172.21.15.86:6789/0 conn(0x7f620c06e2f0 legacy=0x7f620c161d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f40039b0 0x7f61f4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c0b2c40 0x7f61f4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61f4004d40 0x7f61f40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f620c06d4d0 con 0x7f61f4004d40 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61f400b9c0 con 0x7f61f40039b0 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f61f400bb40 con 0x7f620c0b2c40 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.818+0000 7f6211e00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61f4004d40 0x7f61f40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.819+0000 7f6211400640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f40039b0 0x7f61f4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.819+0000 7f6211e00640 1 -- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c0b2c40 msgr2=0x7f61f4004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.819+0000 7f6211e00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c0b2c40 0x7f61f4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.819+0000 7f6211e00640 1 -- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f40039b0 msgr2=0x7f61f4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.819+0000 7f6211e00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f40039b0 0x7f61f4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.819+0000 7f6211e00640 1 -- 172.21.15.86:0/2251705644 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61f4003520 con 0x7f61f4004d40 2024-03-06T07:30:26.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.819+0000 7f6211400640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f40039b0 0x7f61f4003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:26.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.820+0000 7f6211e00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61f4004d40 0x7f61f40051c0 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f6204001050 tx=0x7f61f4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=455f7341e5932330 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:26.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.820+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6204007d70 con 0x7f61f4004d40 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.820+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6204030070 con 0x7f61f4004d40 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.820+0000 7f620be00640 1 -- 172.21.15.86:0/2251705644 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f620402cd80 con 0x7f61f4004d40 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.820+0000 7f6213e00640 1 -- 172.21.15.86:0/2251705644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61f4004d40 msgr2=0x7f61f40051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.820+0000 7f6213e00640 1 --2- 172.21.15.86:0/2251705644 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61f4004d40 0x7f61f40051c0 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f6204001050 tx=0x7f61f4005830 comp rx=0 tx=0).stop 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.820+0000 7f6211e00640 1 -- 172.21.15.86:0/2251705644 reap_dead start 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.821+0000 7f6213e00640 1 -- 172.21.15.86:0/2251705644 shutdown_connections 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.821+0000 7f6213e00640 1 -- 172.21.15.86:0/2251705644 >> 172.21.15.86:0/2251705644 conn(0x7f620c0b1940 msgr2=0x7f620c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.821+0000 7f6213e00640 1 -- 172.21.15.86:0/2251705644 shutdown_connections 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.821+0000 7f6213e00640 1 -- 172.21.15.86:0/2251705644 wait complete. 2024-03-06T07:30:26.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.822+0000 7f6213e00640 1 Processor -- start 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.822+0000 7f6213e00640 1 -- start start 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.823+0000 7f6213e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f4004d40 0x7f620c0ad730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.823+0000 7f6213e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c1f3cf0 0x7f620c1f7610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.823+0000 7f6213e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f620c1f7b50 0x7f620c149c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.823+0000 7f6213e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f620c06e750 con 0x7f620c1f7b50 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.823+0000 7f6213e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f620c06d160 con 0x7f61f4004d40 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.823+0000 7f6213e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f620c06e8d0 con 0x7f620c1f3cf0 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f620c1f7b50 0x7f620c149c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f620c1f7b50 0x7f620c149c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55878/0 (socket says 172.21.15.86:55878) 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 -- 172.21.15.86:0/3826317842 learned_addr learned my addr 172.21.15.86:0/3826317842 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211400640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f4004d40 0x7f620c0ad730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 -- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c1f3cf0 msgr2=0x7f620c1f7610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c1f3cf0 0x7f620c1f7610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 -- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f4004d40 msgr2=0x7f620c0ad730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f4004d40 0x7f620c0ad730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 -- 172.21.15.86:0/3826317842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f620c0adea0 con 0x7f620c1f7b50 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211400640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f4004d40 0x7f620c0ad730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6211e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f620c1f7b50 0x7f620c149c20 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7f6204000c00 tx=0x7f6204001a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.824+0000 7f6209600640 1 -- 172.21.15.86:0/3826317842 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6204004bf0 con 0x7f620c1f7b50 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.825+0000 7f6209600640 1 -- 172.21.15.86:0/3826317842 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6204004d90 con 0x7f620c1f7b50 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.825+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f620c0ae180 con 0x7f620c1f7b50 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.825+0000 7f6209600640 1 -- 172.21.15.86:0/3826317842 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6204007270 con 0x7f620c1f7b50 2024-03-06T07:30:26.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.825+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f620c1e7d10 con 0x7f620c1f7b50 2024-03-06T07:30:26.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.826+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f620c0ae6a0 con 0x7f620c1f7b50 2024-03-06T07:30:26.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.827+0000 7f6209600640 1 -- 172.21.15.86:0/3826317842 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6204007410 con 0x7f620c1f7b50 2024-03-06T07:30:26.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.827+0000 7f6209600640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f61ec06b4d0 0x7f61ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:26.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.828+0000 7f6209600640 1 -- 172.21.15.86:0/3826317842 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f62040ab480 con 0x7f620c1f7b50 2024-03-06T07:30:26.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.828+0000 7f6211400640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f61ec06b4d0 0x7f61ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:26.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.828+0000 7f6211400640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f61ec06b4d0 0x7f61ec06d990 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f6200007540 tx=0x7f620000d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:26.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:26.832+0000 7f6209600640 1 -- 172.21.15.86:0/3826317842 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f620c0ae6a0 con 0x7f620c1f7b50 2024-03-06T07:30:27.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.014+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f620c1e7f10 con 0x7f620c1f7b50 2024-03-06T07:30:27.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.015+0000 7f6209600640 1 -- 172.21.15.86:0/3826317842 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v61) v1 ==== 93+0+10 (secure 0 0 0) 0x7f620c1e7f10 con 0x7f620c1f7b50 2024-03-06T07:30:27.017 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f61ec06b4d0 msgr2=0x7f61ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f61ec06b4d0 0x7f61ec06d990 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f6200007540 tx=0x7f620000d040 comp rx=0 tx=0).stop 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f620c1f7b50 msgr2=0x7f620c149c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f620c1f7b50 0x7f620c149c20 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7f6204000c00 tx=0x7f6204001a40 comp rx=0 tx=0).stop 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 shutdown_connections 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f61ec06b4d0 0x7f61ec06d990 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f620c1f7b50 0x7f620c149c20 unknown :-1 s=CLOSED pgs=758 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f620c1f3cf0 0x7f620c1f7610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 --2- 172.21.15.86:0/3826317842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61f4004d40 0x7f620c0ad730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 >> 172.21.15.86:0/3826317842 conn(0x7f620c0b1940 msgr2=0x7f620c070c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:27.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 shutdown_connections 2024-03-06T07:30:27.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.018+0000 7f6213e00640 1 -- 172.21.15.86:0/3826317842 wait complete. 2024-03-06T07:30:27.041 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays' 2024-03-06T07:30:27.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.242+0000 7f9055000640 1 Processor -- start 2024-03-06T07:30:27.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.242+0000 7f9055000640 1 -- start start 2024-03-06T07:30:27.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.243+0000 7f9055000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9050144c00 0x7f905006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:27.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.243+0000 7f9055000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f905015f9d0 0x7f905015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:27.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.244+0000 7f9055000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f905006ee70 con 0x7f90500ce0a0 2024-03-06T07:30:27.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.244+0000 7f9055000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9050160370 con 0x7f9050144c00 2024-03-06T07:30:27.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.244+0000 7f9055000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f905006cef0 con 0x7f905015f9d0 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.244+0000 7f904d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f905015f9d0 0x7f905015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.244+0000 7f904d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f905015f9d0 0x7f905015fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:37100/0 (socket says 172.21.15.86:37100) 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.244+0000 7f904d400640 1 -- 172.21.15.86:0/3597214652 learned_addr learned my addr 172.21.15.86:0/3597214652 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.244+0000 7f904de00640 1 --2- 172.21.15.86:0/3597214652 >> v2:172.21.15.194:3300/0 conn(0x7f9050144c00 0x7f905006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.245+0000 7f904d400640 1 -- 172.21.15.86:0/3597214652 >> v2:172.21.15.194:3300/0 conn(0x7f9050144c00 msgr2=0x7f905006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.245+0000 7f904d400640 1 --2- 172.21.15.86:0/3597214652 >> v2:172.21.15.194:3300/0 conn(0x7f9050144c00 0x7f905006e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.245+0000 7f904d400640 1 -- 172.21.15.86:0/3597214652 >> v1:172.21.15.194:6789/0 conn(0x7f90500ce0a0 legacy=0x7f90500ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.245+0000 7f904d400640 1 -- 172.21.15.86:0/3597214652 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f905006d1d0 con 0x7f905015f9d0 2024-03-06T07:30:27.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.247+0000 7f904d400640 1 --2- 172.21.15.86:0/3597214652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f905015f9d0 0x7f905015fe30 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f903c009870 tx=0x7f903c02b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7c6098eae22e3cc1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:27.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f904ca00640 1 -- 172.21.15.86:0/3597214652 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f903c034070 con 0x7f905015f9d0 2024-03-06T07:30:27.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f904ca00640 1 -- 172.21.15.86:0/3597214652 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f903c030840 con 0x7f905015f9d0 2024-03-06T07:30:27.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f904ca00640 1 -- 172.21.15.86:0/3597214652 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f903c030c10 con 0x7f905015f9d0 2024-03-06T07:30:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f9055000640 1 -- 172.21.15.86:0/3597214652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f905015f9d0 msgr2=0x7f905015fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f9055000640 1 --2- 172.21.15.86:0/3597214652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f905015f9d0 0x7f905015fe30 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f903c009870 tx=0x7f903c02b980 comp rx=0 tx=0).stop 2024-03-06T07:30:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f9055000640 1 -- 172.21.15.86:0/3597214652 shutdown_connections 2024-03-06T07:30:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f9055000640 1 --2- 172.21.15.86:0/3597214652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f905015f9d0 0x7f905015fe30 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f9055000640 1 --2- 172.21.15.86:0/3597214652 >> v2:172.21.15.194:3300/0 conn(0x7f9050144c00 0x7f905006e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.248+0000 7f9055000640 1 -- 172.21.15.86:0/3597214652 >> 172.21.15.86:0/3597214652 conn(0x7f90500b1730 msgr2=0x7f90500b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:27.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.249+0000 7f9055000640 1 -- 172.21.15.86:0/3597214652 shutdown_connections 2024-03-06T07:30:27.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.249+0000 7f9055000640 1 -- 172.21.15.86:0/3597214652 wait complete. 2024-03-06T07:30:27.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.250+0000 7f9055000640 1 Processor -- start 2024-03-06T07:30:27.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.250+0000 7f9055000640 1 -- start start 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.251+0000 7f9055000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90500ce0a0 0x7f9050139020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.251+0000 7f9055000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9050144c00 0x7f905015e550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.251+0000 7f9055000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f905015f9d0 0x7f905015aca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.251+0000 7f9055000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9050160730 con 0x7f905015f9d0 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.251+0000 7f9055000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f905006d790 con 0x7f9050144c00 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.251+0000 7f9055000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f905006cb50 con 0x7f90500ce0a0 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.251+0000 7f904e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f905015f9d0 0x7f905015aca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f905015f9d0 0x7f905015aca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55886/0 (socket says 172.21.15.86:55886) 2024-03-06T07:30:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 -- 172.21.15.86:0/3149135901 learned_addr learned my addr 172.21.15.86:0/3149135901 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904de00640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90500ce0a0 0x7f9050139020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904d400640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9050144c00 0x7f905015e550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 -- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90500ce0a0 msgr2=0x7f9050139020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90500ce0a0 0x7f9050139020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 -- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9050144c00 msgr2=0x7f905015e550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9050144c00 0x7f905015e550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 -- 172.21.15.86:0/3149135901 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90501e6590 con 0x7f905015f9d0 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904d400640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9050144c00 0x7f905015e550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.252+0000 7f904e800640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f905015f9d0 0x7f905015aca0 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f904800c6b0 tx=0x7f904800cb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.253+0000 7f9046000640 1 -- 172.21.15.86:0/3149135901 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9048003b70 con 0x7f905015f9d0 2024-03-06T07:30:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.253+0000 7f9046000640 1 -- 172.21.15.86:0/3149135901 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9048003d30 con 0x7f905015f9d0 2024-03-06T07:30:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.253+0000 7f9046000640 1 -- 172.21.15.86:0/3149135901 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9048005310 con 0x7f905015f9d0 2024-03-06T07:30:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.253+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90501e6870 con 0x7f905015f9d0 2024-03-06T07:30:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.253+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90501e6e30 con 0x7f905015f9d0 2024-03-06T07:30:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.254+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f905004f330 con 0x7f905015f9d0 2024-03-06T07:30:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.255+0000 7f9046000640 1 -- 172.21.15.86:0/3149135901 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f90480054b0 con 0x7f905015f9d0 2024-03-06T07:30:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.255+0000 7f9046000640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f903006b4d0 0x7f903006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.256+0000 7f9046000640 1 -- 172.21.15.86:0/3149135901 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f9048089e90 con 0x7f905015f9d0 2024-03-06T07:30:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.256+0000 7f904de00640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f903006b4d0 0x7f903006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.256+0000 7f904de00640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f903006b4d0 0x7f903006d990 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f90501f51e0 tx=0x7f9040008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:27.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.260+0000 7f9046000640 1 -- 172.21.15.86:0/3149135901 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f905006ee70 con 0x7f905015f9d0 2024-03-06T07:30:27.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:27.525+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]} v 0) v1 -- 0x7f9050145580 con 0x7f905015f9d0 2024-03-06T07:30:28.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.179+0000 7f9046000640 1 -- 172.21.15.86:0/3149135901 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]}]=0 v203) v1 ==== 143+0+0 (secure 0 0 0) 0x7f904808b020 con 0x7f905015f9d0 2024-03-06T07:30:28.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f903006b4d0 msgr2=0x7f903006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f903006b4d0 0x7f903006d990 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f90501f51e0 tx=0x7f9040008040 comp rx=0 tx=0).stop 2024-03-06T07:30:28.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f905015f9d0 msgr2=0x7f905015aca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f905015f9d0 0x7f905015aca0 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f904800c6b0 tx=0x7f904800cb70 comp rx=0 tx=0).stop 2024-03-06T07:30:28.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 shutdown_connections 2024-03-06T07:30:28.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f903006b4d0 0x7f903006d990 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f905015f9d0 0x7f905015aca0 unknown :-1 s=CLOSED pgs=759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9050144c00 0x7f905015e550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 --2- 172.21.15.86:0/3149135901 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90500ce0a0 0x7f9050139020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 >> 172.21.15.86:0/3149135901 conn(0x7f90500b1730 msgr2=0x7f90500cd060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:28.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.183+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 shutdown_connections 2024-03-06T07:30:28.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.184+0000 7f9055000640 1 -- 172.21.15.86:0/3149135901 wait complete. 2024-03-06T07:30:28.206 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:30:28.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.418+0000 7f409a600640 1 Processor -- start 2024-03-06T07:30:28.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.418+0000 7f409a600640 1 -- start start 2024-03-06T07:30:28.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f409a600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f40940b2c40 0x7f409415f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:28.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f409a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 0x7f40940cda90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:28.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f409a600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f409406cba0 con 0x7f409406d960 2024-03-06T07:30:28.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f409a600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f409415f700 con 0x7f40940b2c40 2024-03-06T07:30:28.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f409a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f409406d4d0 con 0x7f409406e2f0 2024-03-06T07:30:28.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f4093400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f40940b2c40 0x7f409415f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f4092a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f409406d960 0x7f40940b3020 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57116/0 (socket says 172.21.15.86:57116) 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f4092a00640 1 -- 172.21.15.86:0/3232004961 learned_addr learned my addr 172.21.15.86:0/3232004961 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.420+0000 7f4093e00640 1 --2- 172.21.15.86:0/3232004961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 0x7f40940cda90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.421+0000 7f4093400640 1 -- 172.21.15.86:0/3232004961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 msgr2=0x7f40940cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.421+0000 7f4093400640 1 --2- 172.21.15.86:0/3232004961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 0x7f40940cda90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.421+0000 7f4093400640 1 -- 172.21.15.86:0/3232004961 >> v1:172.21.15.86:6789/0 conn(0x7f409406d960 legacy=0x7f40940b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.421+0000 7f4092000640 1 -- 172.21.15.86:0/3232004961 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2876470087 0 0) 0x7f409406cba0 con 0x7f409406d960 2024-03-06T07:30:28.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.421+0000 7f4093400640 1 -- 172.21.15.86:0/3232004961 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f409406ce80 con 0x7f40940b2c40 2024-03-06T07:30:28.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.422+0000 7f4093400640 1 --2- 172.21.15.86:0/3232004961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40940b2c40 0x7f409415f1c0 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f40840097a0 tx=0x7f408402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d1bfe94555db663b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:28.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.422+0000 7f4092000640 1 -- 172.21.15.86:0/3232004961 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4084035020 con 0x7f40940b2c40 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.422+0000 7f4092000640 1 -- 172.21.15.86:0/3232004961 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4084031830 con 0x7f40940b2c40 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.422+0000 7f4092000640 1 -- 172.21.15.86:0/3232004961 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4084031c00 con 0x7f40940b2c40 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 -- 172.21.15.86:0/3232004961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40940b2c40 msgr2=0x7f409415f1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 --2- 172.21.15.86:0/3232004961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40940b2c40 0x7f409415f1c0 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f40840097a0 tx=0x7f408402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 -- 172.21.15.86:0/3232004961 shutdown_connections 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 --2- 172.21.15.86:0/3232004961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 0x7f40940cda90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 --2- 172.21.15.86:0/3232004961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f40940b2c40 0x7f409415f1c0 unknown :-1 s=CLOSED pgs=761 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 -- 172.21.15.86:0/3232004961 >> 172.21.15.86:0/3232004961 conn(0x7f40940b1940 msgr2=0x7f40940b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 -- 172.21.15.86:0/3232004961 shutdown_connections 2024-03-06T07:30:28.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.423+0000 7f409a600640 1 -- 172.21.15.86:0/3232004961 wait complete. 2024-03-06T07:30:28.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.424+0000 7f409a600640 1 Processor -- start 2024-03-06T07:30:28.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.425+0000 7f409a600640 1 -- start start 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.425+0000 7f409a600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f409406d960 0x7f4094155150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.425+0000 7f409a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 0x7f409409d320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.425+0000 7f409a600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40940b2c40 0x7f40940add70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.425+0000 7f409a600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f409406e900 con 0x7f409406d960 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.425+0000 7f409a600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f409406e780 con 0x7f40940b2c40 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.425+0000 7f409a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f40940cdfd0 con 0x7f409406e2f0 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f409406d960 0x7f4094155150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f409406d960 0x7f4094155150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55910/0 (socket says 172.21.15.86:55910) 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 -- 172.21.15.86:0/4015643054 learned_addr learned my addr 172.21.15.86:0/4015643054 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 -- 172.21.15.86:0/4015643054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 msgr2=0x7f409409d320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 0x7f409409d320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 -- 172.21.15.86:0/4015643054 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40940b2c40 msgr2=0x7f40940add70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40940b2c40 0x7f40940add70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 -- 172.21.15.86:0/4015643054 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40941f4570 con 0x7f409406d960 2024-03-06T07:30:28.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.426+0000 7f4093400640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f409406d960 0x7f4094155150 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f4084034040 tx=0x7f40840046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:28.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.427+0000 7f408b400640 1 -- 172.21.15.86:0/4015643054 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4084041070 con 0x7f409406d960 2024-03-06T07:30:28.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.427+0000 7f408b400640 1 -- 172.21.15.86:0/4015643054 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4084004250 con 0x7f409406d960 2024-03-06T07:30:28.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.427+0000 7f408b400640 1 -- 172.21.15.86:0/4015643054 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4084035040 con 0x7f409406d960 2024-03-06T07:30:28.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.427+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f40941f4850 con 0x7f409406d960 2024-03-06T07:30:28.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.427+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f40941f4e10 con 0x7f409406d960 2024-03-06T07:30:28.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.428+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f409404f330 con 0x7f409406d960 2024-03-06T07:30:28.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.429+0000 7f408b400640 1 -- 172.21.15.86:0/4015643054 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4084038020 con 0x7f409406d960 2024-03-06T07:30:28.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.430+0000 7f408b400640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f407406b4d0 0x7f407406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:28.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.430+0000 7f408b400640 1 -- 172.21.15.86:0/4015643054 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f40840aba20 con 0x7f409406d960 2024-03-06T07:30:28.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.430+0000 7f4092a00640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f407406b4d0 0x7f407406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:28.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.431+0000 7f4092a00640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f407406b4d0 0x7f407406d990 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f4080007fd0 tx=0x7f4080007920 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:28.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.435+0000 7f408b400640 1 -- 172.21.15.86:0/4015643054 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f408407aba0 con 0x7f409406d960 2024-03-06T07:30:28.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.629+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f40940cd590 con 0x7f409406d960 2024-03-06T07:30:28.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.635+0000 7f408b400640 1 -- 172.21.15.86:0/4015643054 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v22)=0 v22) v1 ==== 122+0+0 (secure 0 0 0) 0x7f408407a4a0 con 0x7f409406d960 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f407406b4d0 msgr2=0x7f407406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f407406b4d0 0x7f407406d990 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f4080007fd0 tx=0x7f4080007920 comp rx=0 tx=0).stop 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f409406d960 msgr2=0x7f4094155150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f409406d960 0x7f4094155150 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f4084034040 tx=0x7f40840046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 shutdown_connections 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f407406b4d0 0x7f407406d990 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f40940b2c40 0x7f40940add70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f409406e2f0 0x7f409409d320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 --2- 172.21.15.86:0/4015643054 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f409406d960 0x7f4094155150 unknown :-1 s=CLOSED pgs=762 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:28.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 >> 172.21.15.86:0/4015643054 conn(0x7f40940b1940 msgr2=0x7f40940b38b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:28.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 shutdown_connections 2024-03-06T07:30:28.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:28.638+0000 7f409a600640 1 -- 172.21.15.86:0/4015643054 wait complete. 2024-03-06T07:30:28.661 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:30:28.661 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:30:28.772 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:30:28.892 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:30:28.892 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:28.892 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:28.950 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:30:28.950 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:30:28.950 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:29.004 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:30:29.004 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:29.004 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:29.062 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:30:29.062 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:30:29.062 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:29.117 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:30:29.117 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:30:29.117 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:29.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.331+0000 7fc6c0c00640 1 Processor -- start 2024-03-06T07:30:29.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.331+0000 7fc6c0c00640 1 -- start start 2024-03-06T07:30:29.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.332+0000 7fc6c0c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc0cf370 0x7fc6bc06d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.332+0000 7fc6c0c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc6bc0cc2a0 0x7fc6bc0cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.332+0000 7fc6c0c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc6bc0b2b30 con 0x7fc6bc15a940 2024-03-06T07:30:29.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.332+0000 7fc6c0c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc6bc0b28e0 con 0x7fc6bc0cc2a0 2024-03-06T07:30:29.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.332+0000 7fc6c0c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc6bc0b2e10 con 0x7fc6bc0cf370 2024-03-06T07:30:29.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.333+0000 7fc6bb400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc6bc15a940 0x7fc6bc0b45e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57132/0 (socket says 172.21.15.86:57132) 2024-03-06T07:30:29.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.333+0000 7fc6bb400640 1 -- 172.21.15.86:0/4267630223 learned_addr learned my addr 172.21.15.86:0/4267630223 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:29.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.333+0000 7fc6bbe00640 1 --2- 172.21.15.86:0/4267630223 >> v2:172.21.15.194:3300/0 conn(0x7fc6bc0cc2a0 0x7fc6bc0cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.333+0000 7fc6b2a00640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc0cf370 0x7fc6bc06d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.333+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1638039076 0 0) 0x7fc6bc0b2b30 con 0x7fc6bc15a940 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.333+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc6a0003520 con 0x7fc6bc15a940 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2231612689 0 0) 0x7fc6a0003520 con 0x7fc6bc15a940 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc0cf370 msgr2=0x7fc6bc06d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc0cf370 0x7fc6bc06d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 >> v2:172.21.15.194:3300/0 conn(0x7fc6bc0cc2a0 msgr2=0x7fc6bc0cc750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 --2- 172.21.15.86:0/4267630223 >> v2:172.21.15.194:3300/0 conn(0x7fc6bc0cc2a0 0x7fc6bc0cc750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6bc06ca50 con 0x7fc6bc15a940 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc6ac0019f0 con 0x7fc6bc15a940 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 >> v1:172.21.15.86:6789/0 conn(0x7fc6bc15a940 legacy=0x7fc6bc0b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6a00039b0 0x7fc6a0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc0cf370 0x7fc6a0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a0004d40 0x7fc6a00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6bc0b28e0 con 0x7fc6a00039b0 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6bc0b2e10 con 0x7fc6a0004d40 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc6a0005990 con 0x7fc6bc0cf370 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6bb400640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6a00039b0 0x7fc6a0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.334+0000 7fc6bb400640 1 -- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc0cf370 msgr2=0x7fc6a0004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.335+0000 7fc6bb400640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc0cf370 0x7fc6a0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.335+0000 7fc6bb400640 1 -- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a0004d40 msgr2=0x7fc6a00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.335+0000 7fc6bb400640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a0004d40 0x7fc6a00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.335+0000 7fc6bb400640 1 -- 172.21.15.86:0/4267630223 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6a0003520 con 0x7fc6a00039b0 2024-03-06T07:30:29.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6bb400640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6a00039b0 0x7fc6a0003dd0 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7fc6ac00a880 tx=0x7fc6a0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=338d779ccb1aa290 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6ac00a4a0 con 0x7fc6a00039b0 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc6ac034650 con 0x7fc6a00039b0 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6baa00640 1 -- 172.21.15.86:0/4267630223 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6ac034a40 con 0x7fc6a00039b0 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6c0c00640 1 -- 172.21.15.86:0/4267630223 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6a00039b0 msgr2=0x7fc6a0003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6c0c00640 1 --2- 172.21.15.86:0/4267630223 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6a00039b0 0x7fc6a0003dd0 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7fc6ac00a880 tx=0x7fc6a0005830 comp rx=0 tx=0).stop 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6bbe00640 1 -- 172.21.15.86:0/4267630223 reap_dead start 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6c0c00640 1 -- 172.21.15.86:0/4267630223 shutdown_connections 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6c0c00640 1 -- 172.21.15.86:0/4267630223 >> 172.21.15.86:0/4267630223 conn(0x7fc6bc06afc0 msgr2=0x7fc6bc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6c0c00640 1 -- 172.21.15.86:0/4267630223 shutdown_connections 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.336+0000 7fc6c0c00640 1 -- 172.21.15.86:0/4267630223 wait complete. 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.338+0000 7fc6c0c00640 1 Processor -- start 2024-03-06T07:30:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.338+0000 7fc6c0c00640 1 -- start start 2024-03-06T07:30:29.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.339+0000 7fc6c0c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a00039b0 0x7fc6bc15e560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.339+0000 7fc6c0c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc13eea0 0x7fc6bc13f320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.339+0000 7fc6c0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6bc14ced0 0x7fc6bc14d350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.339+0000 7fc6c0c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6bc15ad20 con 0x7fc6bc14ced0 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.339+0000 7fc6c0c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6bc0b30f0 con 0x7fc6a00039b0 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.339+0000 7fc6c0c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc6bc15aea0 con 0x7fc6bc13eea0 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.339+0000 7fc6bbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6bc14ced0 0x7fc6bc14d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6bc14ced0 0x7fc6bc14d350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55936/0 (socket says 172.21.15.86:55936) 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 -- 172.21.15.86:0/2893772790 learned_addr learned my addr 172.21.15.86:0/2893772790 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bb400640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a00039b0 0x7fc6bc15e560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 -- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc13eea0 msgr2=0x7fc6bc13f320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6b2a00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc13eea0 0x7fc6bc13f320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc13eea0 0x7fc6bc13f320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 -- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a00039b0 msgr2=0x7fc6bc15e560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a00039b0 0x7fc6bc15e560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 -- 172.21.15.86:0/2893772790 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6bc14db80 con 0x7fc6bc14ced0 2024-03-06T07:30:29.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6bbe00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6bc14ced0 0x7fc6bc14d350 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7fc6bc070960 tx=0x7fc6b400ec00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:29.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6b3e00640 1 -- 172.21.15.86:0/2893772790 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6b400c680 con 0x7fc6bc14ced0 2024-03-06T07:30:29.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6b3e00640 1 -- 172.21.15.86:0/2893772790 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc6b401a070 con 0x7fc6bc14ced0 2024-03-06T07:30:29.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.340+0000 7fc6b3e00640 1 -- 172.21.15.86:0/2893772790 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6b400ca60 con 0x7fc6bc14ced0 2024-03-06T07:30:29.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.341+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc6bc14de60 con 0x7fc6bc14ced0 2024-03-06T07:30:29.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.341+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc6bc15b300 con 0x7fc6bc14ced0 2024-03-06T07:30:29.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.342+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc6bc04f330 con 0x7fc6bc14ced0 2024-03-06T07:30:29.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.343+0000 7fc6b3e00640 1 -- 172.21.15.86:0/2893772790 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc6b400cc00 con 0x7fc6bc14ced0 2024-03-06T07:30:29.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.344+0000 7fc6b3e00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc69806b500 0x7fc69806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.344+0000 7fc6b3e00640 1 -- 172.21.15.86:0/2893772790 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fc6b4089910 con 0x7fc6bc14ced0 2024-03-06T07:30:29.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.344+0000 7fc6bb400640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc69806b500 0x7fc69806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.345+0000 7fc6bb400640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc69806b500 0x7fc69806d9c0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7fc6ac034000 tx=0x7fc6ac009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:29.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.349+0000 7fc6b3e00640 1 -- 172.21.15.86:0/2893772790 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc6b40583a0 con 0x7fc6bc14ced0 2024-03-06T07:30:29.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.590+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc6bc06e180 con 0x7fc6bc14ced0 2024-03-06T07:30:29.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.591+0000 7fc6b3e00640 1 -- 172.21.15.86:0/2893772790 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+3702 (secure 0 0 0) 0x7fc6b405c440 con 0x7fc6bc14ced0 2024-03-06T07:30:29.594 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:29.594 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":36,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5266,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/914865944","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":914865944},{"type":"v1","addr":"172.21.15.194:6833","nonce":914865944}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5269,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1920094758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":1920094758},{"type":"v1","addr":"172.21.15.194:6835","nonce":1920094758}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":29},{"gid":5817,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2998850432","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2998850432},{"type":"v1","addr":"172.21.15.86:6836","nonce":2998850432}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":28},{"gid":6123,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/1601376009","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":1601376009},{"type":"v1","addr":"172.21.15.86:6837","nonce":1601376009}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":36}],"filesystems":[]} 2024-03-06T07:30:29.594 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 36 2024-03-06T07:30:29.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.593+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc69806b500 msgr2=0x7fc69806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.593+0000 7fc6c0c00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc69806b500 0x7fc69806d9c0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7fc6ac034000 tx=0x7fc6ac009030 comp rx=0 tx=0).stop 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.593+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6bc14ced0 msgr2=0x7fc6bc14d350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.593+0000 7fc6c0c00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6bc14ced0 0x7fc6bc14d350 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7fc6bc070960 tx=0x7fc6b400ec00 comp rx=0 tx=0).stop 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 shutdown_connections 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc69806b500 0x7fc69806d9c0 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6bc14ced0 0x7fc6bc14d350 unknown :-1 s=CLOSED pgs=765 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6bc13eea0 0x7fc6bc13f320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 --2- 172.21.15.86:0/2893772790 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6a00039b0 0x7fc6bc15e560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 >> 172.21.15.86:0/2893772790 conn(0x7fc6bc06afc0 msgr2=0x7fc6bc155b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 shutdown_connections 2024-03-06T07:30:29.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.594+0000 7fc6c0c00640 1 -- 172.21.15.86:0/2893772790 wait complete. 2024-03-06T07:30:29.615 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:30:29.616 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:30:29.616 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:30:29.616 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:30:29.616 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:30:29.616 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:30:29.616 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:30:29.617 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:30:29.617 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:30:29.617 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:30:29.617 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:30:29.617 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:30:29.617 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:30:29.617 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:30:29.618 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:30:29.618 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:30:29.635 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:30:29.635 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:30:29.638 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:30:29.638 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:30:29.639 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:30:29.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:30:29.647 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:30:29.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:30:29.685 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:30:29.685 DEBUG:teuthology.parallel:result is None 2024-03-06T07:30:29.691 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:30:29.691 DEBUG:teuthology.parallel:result is None 2024-03-06T07:30:29.694 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:30:29.694 DEBUG:teuthology.parallel:result is None 2024-03-06T07:30:29.700 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:30:29.700 DEBUG:teuthology.parallel:result is None 2024-03-06T07:30:29.701 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:30:29.839 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:30:29.841 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:30:29.861 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:30:29.861 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:30:29.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.883+0000 7fabd1800640 1 Processor -- start 2024-03-06T07:30:29.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.883+0000 7fabd1800640 1 -- start start 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.884+0000 7fabd1800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fabcc161580 0x7fabcc0d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.884+0000 7fabd1800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc0d1e90 0x7fabcc0d2360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabd1800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fabcc161960 con 0x7fabcc0d28a0 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabd1800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fabcc0cf6f0 con 0x7fabcc161580 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabd1800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fabcc0cf9d0 con 0x7fabcc0d1e90 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabcbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fabcc161580 0x7fabcc0d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabd0e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fabcc0d28a0 0x7fabcc0d06e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57142/0 (socket says 172.21.15.86:57142) 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabcbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fabcc161580 0x7fabcc0d1950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55974/0 (socket says 172.21.15.86:55974) 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabd0e00640 1 -- 172.21.15.86:0/634258880 learned_addr learned my addr 172.21.15.86:0/634258880 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:29.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabc3e00640 1 --2- 172.21.15.86:0/634258880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc0d1e90 0x7fabcc0d2360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabcbe00640 1 -- 172.21.15.86:0/634258880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc0d1e90 msgr2=0x7fabcc0d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:29.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabcbe00640 1 --2- 172.21.15.86:0/634258880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc0d1e90 0x7fabcc0d2360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabcbe00640 1 -- 172.21.15.86:0/634258880 >> v1:172.21.15.86:6789/0 conn(0x7fabcc0d28a0 legacy=0x7fabcc0d06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabcbe00640 1 -- 172.21.15.86:0/634258880 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fabcc0cfcb0 con 0x7fabcc161580 2024-03-06T07:30:29.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.885+0000 7fabc3e00640 1 --2- 172.21.15.86:0/634258880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc0d1e90 0x7fabcc0d2360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:29.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.886+0000 7fabcbe00640 1 --2- 172.21.15.86:0/634258880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc161580 0x7fabcc0d1950 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7fabbc0097a0 tx=0x7fabbc02b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b0c906be78923118 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:29.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabcb400640 1 -- 172.21.15.86:0/634258880 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabbc035020 con 0x7fabcc161580 2024-03-06T07:30:29.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabcb400640 1 -- 172.21.15.86:0/634258880 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fabbc031810 con 0x7fabcc161580 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabcb400640 1 -- 172.21.15.86:0/634258880 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabbc031be0 con 0x7fabcc161580 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 -- 172.21.15.86:0/634258880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc161580 msgr2=0x7fabcc0d1950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 --2- 172.21.15.86:0/634258880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc161580 0x7fabcc0d1950 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7fabbc0097a0 tx=0x7fabbc02b8b0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 -- 172.21.15.86:0/634258880 shutdown_connections 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 --2- 172.21.15.86:0/634258880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc0d1e90 0x7fabcc0d2360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 --2- 172.21.15.86:0/634258880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc161580 0x7fabcc0d1950 unknown :-1 s=CLOSED pgs=771 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 -- 172.21.15.86:0/634258880 >> 172.21.15.86:0/634258880 conn(0x7fabcc0d3d80 msgr2=0x7fabcc0d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 -- 172.21.15.86:0/634258880 shutdown_connections 2024-03-06T07:30:29.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.887+0000 7fabd1800640 1 -- 172.21.15.86:0/634258880 wait complete. 2024-03-06T07:30:29.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.889+0000 7fabd1800640 1 Processor -- start 2024-03-06T07:30:29.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.889+0000 7fabd1800640 1 -- start start 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabd1800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabcc0d1e90 0x7fabcc152200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabd1800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc0d28a0 0x7fabcc15d1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabd1800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc161580 0x7fabcc15d700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabd1800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fabcc0cf3a0 con 0x7fabcc0d28a0 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabd1800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fabcc0b39a0 con 0x7fabcc0d1e90 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabd1800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fabcc0b4ed0 con 0x7fabcc161580 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabc3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc0d28a0 0x7fabcc15d1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabc3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc0d28a0 0x7fabcc15d1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55984/0 (socket says 172.21.15.86:55984) 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabc3e00640 1 -- 172.21.15.86:0/237608225 learned_addr learned my addr 172.21.15.86:0/237608225 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabd0e00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc161580 0x7fabcc15d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.890+0000 7fabcbe00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabcc0d1e90 0x7fabcc152200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc3e00640 1 -- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc161580 msgr2=0x7fabcc15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc3e00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc161580 0x7fabcc15d700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc3e00640 1 -- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabcc0d1e90 msgr2=0x7fabcc152200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:29.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc3e00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabcc0d1e90 0x7fabcc152200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc3e00640 1 -- 172.21.15.86:0/237608225 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fabcc152740 con 0x7fabcc0d28a0 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc3e00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc0d28a0 0x7fabcc15d1c0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7fabb400c820 tx=0x7fabb400cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc8c00640 1 -- 172.21.15.86:0/237608225 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabb4015070 con 0x7fabcc0d28a0 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc8c00640 1 -- 172.21.15.86:0/237608225 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fabb4007a30 con 0x7fabcc0d28a0 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabc8c00640 1 -- 172.21.15.86:0/237608225 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabb4007e00 con 0x7fabcc0d28a0 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fabcc0cae50 con 0x7fabcc0d28a0 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.891+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fabcc0cb410 con 0x7fabcc0d28a0 2024-03-06T07:30:29.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.893+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fabcc04f330 con 0x7fabcc0d28a0 2024-03-06T07:30:29.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.894+0000 7fabc8c00640 1 -- 172.21.15.86:0/237608225 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fabb4018070 con 0x7fabcc0d28a0 2024-03-06T07:30:29.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.895+0000 7fabc8c00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faba806b4d0 0x7faba806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:29.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.895+0000 7fabc8c00640 1 -- 172.21.15.86:0/237608225 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fabb4089e80 con 0x7fabcc0d28a0 2024-03-06T07:30:29.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.895+0000 7fabcbe00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faba806b4d0 0x7faba806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:29.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.896+0000 7fabcbe00640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faba806b4d0 0x7faba806d990 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7fabbc036000 tx=0x7fabbc03c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:29.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:29.900+0000 7fabc8c00640 1 -- 172.21.15.86:0/237608225 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fabcc152740 con 0x7fabcc0d28a0 2024-03-06T07:30:30.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:30.110+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7fabcc0d3b60 con 0x7fabcc0d28a0 2024-03-06T07:30:31.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.221+0000 7fabc8c00640 1 -- 172.21.15.86:0/237608225 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v66) v1 ==== 97+0+0 (secure 0 0 0) 0x7fabb4058910 con 0x7fabcc0d28a0 2024-03-06T07:30:31.223 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faba806b4d0 msgr2=0x7faba806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faba806b4d0 0x7faba806d990 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7fabbc036000 tx=0x7fabbc03c040 comp rx=0 tx=0).stop 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc0d28a0 msgr2=0x7fabcc15d1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc0d28a0 0x7fabcc15d1c0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7fabb400c820 tx=0x7fabb400cc20 comp rx=0 tx=0).stop 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 shutdown_connections 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faba806b4d0 0x7faba806d990 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fabcc161580 0x7fabcc15d700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fabcc0d28a0 0x7fabcc15d1c0 unknown :-1 s=CLOSED pgs=772 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 --2- 172.21.15.86:0/237608225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fabcc0d1e90 0x7fabcc152200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 >> 172.21.15.86:0/237608225 conn(0x7fabcc0d3d80 msgr2=0x7fabcc0b55a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:31.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 shutdown_connections 2024-03-06T07:30:31.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.225+0000 7fabd1800640 1 -- 172.21.15.86:0/237608225 wait complete. 2024-03-06T07:30:31.247 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:30:31.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.452+0000 7f84a0e00640 1 Processor -- start 2024-03-06T07:30:31.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.452+0000 7f84a0e00640 1 -- start start 2024-03-06T07:30:31.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.453+0000 7f84a0e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f849c144c00 0x7f849c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.453+0000 7f84a0e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f849c15f9d0 0x7f849c15fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.453+0000 7f84a0e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c0ce0a0 0x7f849c0ce4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.453+0000 7f84a0e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f849c06ee70 con 0x7f849c144c00 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f84a0e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f849c06cef0 con 0x7f849c15f9d0 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f84a0e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f849c06d1d0 con 0x7f849c0ce0a0 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f849c144c00 0x7f849c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f849c144c00 0x7f849c06e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56000/0 (socket says 172.21.15.86:56000) 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 -- 172.21.15.86:0/981703042 learned_addr learned my addr 172.21.15.86:0/981703042 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 -- 172.21.15.86:0/981703042 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c0ce0a0 msgr2=0x7f849c0ce4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 --2- 172.21.15.86:0/981703042 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c0ce0a0 0x7f849c0ce4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 -- 172.21.15.86:0/981703042 >> v2:172.21.15.194:3300/0 conn(0x7f849c15f9d0 msgr2=0x7f849c15fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 --2- 172.21.15.86:0/981703042 >> v2:172.21.15.194:3300/0 conn(0x7f849c15f9d0 0x7f849c15fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.454+0000 7f8499e00640 1 -- 172.21.15.86:0/981703042 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f849c06d4b0 con 0x7f849c144c00 2024-03-06T07:30:31.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.455+0000 7f8499e00640 1 --2- 172.21.15.86:0/981703042 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c144c00 0x7f849c06e930 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7f849c0b26f0 tx=0x7f848c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7e28dccb60cec5a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:31.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f8498a00640 1 -- 172.21.15.86:0/981703042 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f848c035020 con 0x7f849c144c00 2024-03-06T07:30:31.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f8498a00640 1 -- 172.21.15.86:0/981703042 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f848c031870 con 0x7f849c144c00 2024-03-06T07:30:31.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f8498a00640 1 -- 172.21.15.86:0/981703042 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f848c031c40 con 0x7f849c144c00 2024-03-06T07:30:31.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f84a0e00640 1 -- 172.21.15.86:0/981703042 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c144c00 msgr2=0x7f849c06e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f84a0e00640 1 --2- 172.21.15.86:0/981703042 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c144c00 0x7f849c06e930 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7f849c0b26f0 tx=0x7f848c02b910 comp rx=0 tx=0).stop 2024-03-06T07:30:31.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f84a0e00640 1 -- 172.21.15.86:0/981703042 shutdown_connections 2024-03-06T07:30:31.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f84a0e00640 1 --2- 172.21.15.86:0/981703042 >> v2:172.21.15.194:3300/0 conn(0x7f849c15f9d0 0x7f849c15fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f84a0e00640 1 --2- 172.21.15.86:0/981703042 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c0ce0a0 0x7f849c0ce4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.456+0000 7f84a0e00640 1 --2- 172.21.15.86:0/981703042 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c144c00 0x7f849c06e930 unknown :-1 s=CLOSED pgs=773 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.457+0000 7f84a0e00640 1 -- 172.21.15.86:0/981703042 >> 172.21.15.86:0/981703042 conn(0x7f849c0b1730 msgr2=0x7f849c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:31.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.457+0000 7f84a0e00640 1 -- 172.21.15.86:0/981703042 shutdown_connections 2024-03-06T07:30:31.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.457+0000 7f84a0e00640 1 -- 172.21.15.86:0/981703042 wait complete. 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.458+0000 7f84a0e00640 1 Processor -- start 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.458+0000 7f84a0e00640 1 -- start start 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.459+0000 7f84a0e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c0ce0a0 0x7f849c13c150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.459+0000 7f84a0e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c144c00 0x7f849c15c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.459+0000 7f84a0e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f849c15f9d0 0x7f849c15de10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.459+0000 7f84a0e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f849c06cb50 con 0x7f849c0ce0a0 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.459+0000 7f84a0e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f849c160730 con 0x7f849c15f9d0 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.459+0000 7f84a0e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f849c06ccd0 con 0x7f849c144c00 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c0ce0a0 0x7f849c13c150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c0ce0a0 0x7f849c13c150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56014/0 (socket says 172.21.15.86:56014) 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 -- 172.21.15.86:0/1180700122 learned_addr learned my addr 172.21.15.86:0/1180700122 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:31.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 -- 172.21.15.86:0/1180700122 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c144c00 msgr2=0x7f849c15c340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c144c00 0x7f849c15c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 -- 172.21.15.86:0/1180700122 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f849c15f9d0 msgr2=0x7f849c15de10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f849c15f9d0 0x7f849c15de10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 -- 172.21.15.86:0/1180700122 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f849c1400d0 con 0x7f849c0ce0a0 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.460+0000 7f8499e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c0ce0a0 0x7f849c13c150 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f848c000c00 tx=0x7f848c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.461+0000 7f8492000640 1 -- 172.21.15.86:0/1180700122 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f848c03ad60 con 0x7f849c0ce0a0 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.461+0000 7f8492000640 1 -- 172.21.15.86:0/1180700122 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f848c0443f0 con 0x7f849c0ce0a0 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.461+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f849c15ec50 con 0x7f849c0ce0a0 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.461+0000 7f8492000640 1 -- 172.21.15.86:0/1180700122 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f848c0447e0 con 0x7f849c0ce0a0 2024-03-06T07:30:31.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.461+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f849c15f210 con 0x7f849c0ce0a0 2024-03-06T07:30:31.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.462+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f849c04f330 con 0x7f849c0ce0a0 2024-03-06T07:30:31.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.463+0000 7f8492000640 1 -- 172.21.15.86:0/1180700122 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f848c038020 con 0x7f849c0ce0a0 2024-03-06T07:30:31.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.464+0000 7f8492000640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f847c06b4d0 0x7f847c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.465+0000 7f8492000640 1 -- 172.21.15.86:0/1180700122 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f848c0ac1b0 con 0x7f849c0ce0a0 2024-03-06T07:30:31.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.465+0000 7f8499400640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f847c06b4d0 0x7f847c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:31.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.465+0000 7f8499400640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f847c06b4d0 0x7f847c06d990 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f849c1f7430 tx=0x7f8488006c90 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:31.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.469+0000 7f8492000640 1 -- 172.21.15.86:0/1180700122 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f849c1400d0 con 0x7f849c0ce0a0 2024-03-06T07:30:31.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.741+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f849c15be20 con 0x7f849c0ce0a0 2024-03-06T07:30:31.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.741+0000 7f8492000640 1 -- 172.21.15.86:0/1180700122 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v15) v1 ==== 80+0+5304 (secure 0 0 0) 0x7f848c03f070 con 0x7f849c0ce0a0 2024-03-06T07:30:31.743 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:31.743 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:30:31.743 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:30:31.743 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.745 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.746 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.747 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.748 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.749 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.750 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow r", 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.751 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:30:31.752 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:30:31.753 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:30:31.754 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:30:31.755 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:30:31.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f847c06b4d0 msgr2=0x7f847c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f847c06b4d0 0x7f847c06d990 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f849c1f7430 tx=0x7f8488006c90 comp rx=0 tx=0).stop 2024-03-06T07:30:31.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c0ce0a0 msgr2=0x7f849c13c150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c0ce0a0 0x7f849c13c150 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f848c000c00 tx=0x7f848c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 shutdown_connections 2024-03-06T07:30:31.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f847c06b4d0 0x7f847c06d990 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f849c15f9d0 0x7f849c15de10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f849c144c00 0x7f849c15c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 --2- 172.21.15.86:0/1180700122 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f849c0ce0a0 0x7f849c13c150 unknown :-1 s=CLOSED pgs=774 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.744+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 >> 172.21.15.86:0/1180700122 conn(0x7f849c0b1730 msgr2=0x7f849c071cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:31.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.745+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 shutdown_connections 2024-03-06T07:30:31.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.745+0000 7f84a0e00640 1 -- 172.21.15.86:0/1180700122 wait complete. 2024-03-06T07:30:31.763 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:31.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.971+0000 7fa66f400640 1 Processor -- start 2024-03-06T07:30:31.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.971+0000 7fa66f400640 1 -- start start 2024-03-06T07:30:31.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.972+0000 7fa66f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa6680b4680 0x7fa66806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.972+0000 7fa66f400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa668072340 0x7fa66806ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.972+0000 7fa66f400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa66815b090 con 0x7fa66815acb0 2024-03-06T07:30:31.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.972+0000 7fa66f400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6680b2a30 con 0x7fa6680b4680 2024-03-06T07:30:31.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.972+0000 7fa66f400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6680b4360 con 0x7fa668072340 2024-03-06T07:30:31.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.973+0000 7fa667e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa6680b4680 0x7fa66806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:31.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.973+0000 7fa667e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa6680b4680 0x7fa66806e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56020/0 (socket says 172.21.15.86:56020) 2024-03-06T07:30:31.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.973+0000 7fa667e00640 1 -- 172.21.15.86:0/2697018252 learned_addr learned my addr 172.21.15.86:0/2697018252 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:31.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.973+0000 7fa667e00640 1 -- 172.21.15.86:0/2697018252 >> v2:172.21.15.194:3300/0 conn(0x7fa668072340 msgr2=0x7fa66806ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:31.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.973+0000 7fa667e00640 1 --2- 172.21.15.86:0/2697018252 >> v2:172.21.15.194:3300/0 conn(0x7fa668072340 0x7fa66806ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.973+0000 7fa667e00640 1 -- 172.21.15.86:0/2697018252 >> v1:172.21.15.194:6789/0 conn(0x7fa66815acb0 legacy=0x7fa6680b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:31.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.973+0000 7fa667e00640 1 -- 172.21.15.86:0/2697018252 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6680b2cf0 con 0x7fa6680b4680 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.974+0000 7fa667e00640 1 --2- 172.21.15.86:0/2697018252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6680b4680 0x7fa66806e4f0 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7fa6680b4e60 tx=0x7fa65402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=32c409d77db4d62a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa667400640 1 -- 172.21.15.86:0/2697018252 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa654035020 con 0x7fa6680b4680 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa667400640 1 -- 172.21.15.86:0/2697018252 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa654031870 con 0x7fa6680b4680 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa667400640 1 -- 172.21.15.86:0/2697018252 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa654031c40 con 0x7fa6680b4680 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 -- 172.21.15.86:0/2697018252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6680b4680 msgr2=0x7fa66806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 --2- 172.21.15.86:0/2697018252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6680b4680 0x7fa66806e4f0 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7fa6680b4e60 tx=0x7fa65402b910 comp rx=0 tx=0).stop 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 -- 172.21.15.86:0/2697018252 shutdown_connections 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 --2- 172.21.15.86:0/2697018252 >> v2:172.21.15.194:3300/0 conn(0x7fa668072340 0x7fa66806ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 --2- 172.21.15.86:0/2697018252 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6680b4680 0x7fa66806e4f0 unknown :-1 s=CLOSED pgs=775 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 -- 172.21.15.86:0/2697018252 >> 172.21.15.86:0/2697018252 conn(0x7fa6680b1730 msgr2=0x7fa6680b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:31.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 -- 172.21.15.86:0/2697018252 shutdown_connections 2024-03-06T07:30:31.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.975+0000 7fa66f400640 1 -- 172.21.15.86:0/2697018252 wait complete. 2024-03-06T07:30:31.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.977+0000 7fa66f400640 1 Processor -- start 2024-03-06T07:30:31.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.977+0000 7fa66f400640 1 -- start start 2024-03-06T07:30:31.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66f400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa668072340 0x7fa668140320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66f400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6680b4680 0x7fa6680cebd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa66815acb0 0x7fa668155960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66f400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6680b4010 con 0x7fa66815acb0 2024-03-06T07:30:31.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66f400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6680b32b0 con 0x7fa668072340 2024-03-06T07:30:31.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66f400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6680ccfb0 con 0x7fa6680b4680 2024-03-06T07:30:31.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa66815acb0 0x7fa668155960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:31.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa66815acb0 0x7fa668155960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56024/0 (socket says 172.21.15.86:56024) 2024-03-06T07:30:31.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.978+0000 7fa66d400640 1 -- 172.21.15.86:0/4250102215 learned_addr learned my addr 172.21.15.86:0/4250102215 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:31.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66d400640 1 -- 172.21.15.86:0/4250102215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6680b4680 msgr2=0x7fa6680cebd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66d400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6680b4680 0x7fa6680cebd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66d400640 1 -- 172.21.15.86:0/4250102215 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa668072340 msgr2=0x7fa668140320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:31.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66d400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa668072340 0x7fa668140320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:31.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66d400640 1 -- 172.21.15.86:0/4250102215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6680cd770 con 0x7fa66815acb0 2024-03-06T07:30:31.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66d400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa66815acb0 0x7fa668155960 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7fa66000b970 tx=0x7fa66000be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:31.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa664c00640 1 -- 172.21.15.86:0/4250102215 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa660019070 con 0x7fa66815acb0 2024-03-06T07:30:31.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa664c00640 1 -- 172.21.15.86:0/4250102215 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa66000cb60 con 0x7fa66815acb0 2024-03-06T07:30:31.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa6680cda50 con 0x7fa66815acb0 2024-03-06T07:30:31.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa664c00640 1 -- 172.21.15.86:0/4250102215 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa660014440 con 0x7fa66815acb0 2024-03-06T07:30:31.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.979+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6680ce010 con 0x7fa66815acb0 2024-03-06T07:30:31.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.980+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa6680ce250 con 0x7fa66815acb0 2024-03-06T07:30:31.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.981+0000 7fa664c00640 1 -- 172.21.15.86:0/4250102215 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa66001c070 con 0x7fa66815acb0 2024-03-06T07:30:31.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.982+0000 7fa664c00640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa64806b500 0x7fa64806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:31.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.982+0000 7fa664c00640 1 -- 172.21.15.86:0/4250102215 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fa660089c80 con 0x7fa66815acb0 2024-03-06T07:30:31.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.982+0000 7fa66ca00640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa64806b500 0x7fa64806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:31.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.983+0000 7fa66ca00640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa64806b500 0x7fa64806d9c0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7fa65c009650 tx=0x7fa65c006ba0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:31.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:31.986+0000 7fa664c00640 1 -- 172.21.15.86:0/4250102215 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa6680ce250 con 0x7fa66815acb0 2024-03-06T07:30:32.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.181+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa66806ce60 con 0x7fa66815acb0 2024-03-06T07:30:32.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.183+0000 7fa664c00640 1 -- 172.21.15.86:0/4250102215 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v66) v1 ==== 74+0+12903 (secure 0 0 0) 0x7fa660058db0 con 0x7fa66815acb0 2024-03-06T07:30:32.185 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:32.185 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":66,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:31.218049+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:32.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa64806b500 msgr2=0x7fa64806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa64806b500 0x7fa64806d9c0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7fa65c009650 tx=0x7fa65c006ba0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa66815acb0 msgr2=0x7fa668155960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa66815acb0 0x7fa668155960 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7fa66000b970 tx=0x7fa66000be30 comp rx=0 tx=0).stop 2024-03-06T07:30:32.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 shutdown_connections 2024-03-06T07:30:32.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa64806b500 0x7fa64806d9c0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa66815acb0 0x7fa668155960 unknown :-1 s=CLOSED pgs=776 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6680b4680 0x7fa6680cebd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 --2- 172.21.15.86:0/4250102215 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa668072340 0x7fa668140320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 >> 172.21.15.86:0/4250102215 conn(0x7fa6680b1730 msgr2=0x7fa6680b37d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:32.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 shutdown_connections 2024-03-06T07:30:32.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.186+0000 7fa66f400640 1 -- 172.21.15.86:0/4250102215 wait complete. 2024-03-06T07:30:32.206 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:30:32.206 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:30:32.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.404+0000 7f97b4400640 1 Processor -- start 2024-03-06T07:30:32.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.405+0000 7f97b4400640 1 -- start start 2024-03-06T07:30:32.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.406+0000 7f97b4400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f97ac15df80 0x7f97ac06c0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.406+0000 7f97b4400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 0x7f97ac0cfb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.406+0000 7f97b4400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f97ac0d00d0 0x7f97ac161d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.406+0000 7f97b4400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97ac0d2a20 con 0x7f97ac15df80 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.406+0000 7f97b4400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97ac0d1fb0 con 0x7f97ac0d00d0 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.406+0000 7f97b4400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97ac09b100 con 0x7f97ac15e360 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.406+0000 7f97b1a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f97ac15df80 0x7f97ac06c0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.407+0000 7f97b1a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f97ac15df80 0x7f97ac06c0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56034/0 (socket says 172.21.15.86:56034) 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.407+0000 7f97b1a00640 1 -- 172.21.15.86:0/4144842675 learned_addr learned my addr 172.21.15.86:0/4144842675 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.407+0000 7f97b1a00640 1 -- 172.21.15.86:0/4144842675 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 msgr2=0x7f97ac0cfb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.407+0000 7f97b1a00640 1 --2- 172.21.15.86:0/4144842675 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 0x7f97ac0cfb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.407+0000 7f97b1a00640 1 -- 172.21.15.86:0/4144842675 >> v2:172.21.15.194:3300/0 conn(0x7f97ac0d00d0 msgr2=0x7f97ac161d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.407+0000 7f97b1a00640 1 --2- 172.21.15.86:0/4144842675 >> v2:172.21.15.194:3300/0 conn(0x7f97ac0d00d0 0x7f97ac161d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.407+0000 7f97b1a00640 1 -- 172.21.15.86:0/4144842675 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97ac09b3a0 con 0x7f97ac15df80 2024-03-06T07:30:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.408+0000 7f97b1a00640 1 --2- 172.21.15.86:0/4144842675 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac06c0d0 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f97ac0cfa60 tx=0x7f979c02b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=16ee439b0d7834fa server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.408+0000 7f97abe00640 1 -- 172.21.15.86:0/4144842675 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c035020 con 0x7f97ac15df80 2024-03-06T07:30:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97abe00640 1 -- 172.21.15.86:0/4144842675 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f979c02d040 con 0x7f97ac15df80 2024-03-06T07:30:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97abe00640 1 -- 172.21.15.86:0/4144842675 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c031a80 con 0x7f97ac15df80 2024-03-06T07:30:32.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 -- 172.21.15.86:0/4144842675 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 msgr2=0x7f97ac06c0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 --2- 172.21.15.86:0/4144842675 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac06c0d0 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f97ac0cfa60 tx=0x7f979c02b900 comp rx=0 tx=0).stop 2024-03-06T07:30:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 -- 172.21.15.86:0/4144842675 shutdown_connections 2024-03-06T07:30:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 --2- 172.21.15.86:0/4144842675 >> v2:172.21.15.194:3300/0 conn(0x7f97ac0d00d0 0x7f97ac161d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 --2- 172.21.15.86:0/4144842675 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 0x7f97ac0cfb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 --2- 172.21.15.86:0/4144842675 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac06c0d0 unknown :-1 s=CLOSED pgs=777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 -- 172.21.15.86:0/4144842675 >> 172.21.15.86:0/4144842675 conn(0x7f97ac0b7320 msgr2=0x7f97ac09a9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 -- 172.21.15.86:0/4144842675 shutdown_connections 2024-03-06T07:30:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.409+0000 7f97b4400640 1 -- 172.21.15.86:0/4144842675 wait complete. 2024-03-06T07:30:32.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.411+0000 7f97b4400640 1 Processor -- start 2024-03-06T07:30:32.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.411+0000 7f97b4400640 1 -- start start 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b4400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97ac0d00d0 0x7f97ac0b6770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b4400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac155170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b4400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 0x7f97ac0afc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b4400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97ac06c610 con 0x7f97ac15df80 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b4400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97ac0d2500 con 0x7f97ac0d00d0 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b4400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97ac09b680 con 0x7f97ac15e360 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac155170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac155170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56040/0 (socket says 172.21.15.86:56040) 2024-03-06T07:30:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.412+0000 7f97b1000640 1 -- 172.21.15.86:0/1769171188 learned_addr learned my addr 172.21.15.86:0/1769171188 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b1a00640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97ac0d00d0 0x7f97ac0b6770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b1000640 1 -- 172.21.15.86:0/1769171188 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 msgr2=0x7f97ac0afc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b1000640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 0x7f97ac0afc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b1000640 1 -- 172.21.15.86:0/1769171188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97ac0d00d0 msgr2=0x7f97ac0b6770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b1000640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97ac0d00d0 0x7f97ac0b6770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b1000640 1 -- 172.21.15.86:0/1769171188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97ac0b4f20 con 0x7f97ac15df80 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b1000640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac155170 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f979800b450 tx=0x7f979800b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97a9600640 1 -- 172.21.15.86:0/1769171188 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9798015020 con 0x7f97ac15df80 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97a9600640 1 -- 172.21.15.86:0/1769171188 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9798011890 con 0x7f97ac15df80 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97a9600640 1 -- 172.21.15.86:0/1769171188 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9798011c80 con 0x7f97ac15df80 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.413+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97ac0b5200 con 0x7f97ac15df80 2024-03-06T07:30:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.414+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f97ac0b57c0 con 0x7f97ac15df80 2024-03-06T07:30:32.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.415+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f97ac04f330 con 0x7f97ac15df80 2024-03-06T07:30:32.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.416+0000 7f97a9600640 1 -- 172.21.15.86:0/1769171188 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9798018020 con 0x7f97ac15df80 2024-03-06T07:30:32.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.417+0000 7f97a9600640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f978c06b4d0 0x7f978c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.417+0000 7f97a9600640 1 -- 172.21.15.86:0/1769171188 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f979808a0a0 con 0x7f97ac15df80 2024-03-06T07:30:32.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.417+0000 7f97b1a00640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f978c06b4d0 0x7f978c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.418+0000 7f97b1a00640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f978c06b4d0 0x7f978c06d990 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f979c036000 tx=0x7f979c03c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:32.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.421+0000 7f97a9600640 1 -- 172.21.15.86:0/1769171188 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f979801f020 con 0x7f97ac15df80 2024-03-06T07:30:32.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.615+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f97ac0b1850 con 0x7f97ac15df80 2024-03-06T07:30:32.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.616+0000 7f97a9600640 1 -- 172.21.15.86:0/1769171188 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v66) v1 ==== 93+0+10 (secure 0 0 0) 0x7f979800d040 con 0x7f97ac15df80 2024-03-06T07:30:32.618 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:30:32.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.618+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f978c06b4d0 msgr2=0x7f978c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.618+0000 7f97b4400640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f978c06b4d0 0x7f978c06d990 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f979c036000 tx=0x7f979c03c040 comp rx=0 tx=0).stop 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 msgr2=0x7f97ac155170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac155170 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f979800b450 tx=0x7f979800b910 comp rx=0 tx=0).stop 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 shutdown_connections 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f978c06b4d0 0x7f978c06d990 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97ac15e360 0x7f97ac0afc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97ac15df80 0x7f97ac155170 unknown :-1 s=CLOSED pgs=778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 --2- 172.21.15.86:0/1769171188 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97ac0d00d0 0x7f97ac0b6770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 >> 172.21.15.86:0/1769171188 conn(0x7f97ac0b7320 msgr2=0x7f97ac06f6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 shutdown_connections 2024-03-06T07:30:32.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.619+0000 7f97b4400640 1 -- 172.21.15.86:0/1769171188 wait complete. 2024-03-06T07:30:32.639 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:30:32.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.848+0000 7f481f800640 1 Processor -- start 2024-03-06T07:30:32.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.848+0000 7f481f800640 1 -- start start 2024-03-06T07:30:32.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.849+0000 7f481f800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f48180b4680 0x7f481806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.849+0000 7f481f800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f481815b090 con 0x7f481815acb0 2024-03-06T07:30:32.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.849+0000 7f481f800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f48180b2a30 con 0x7f4818072340 2024-03-06T07:30:32.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.849+0000 7f481f800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f48180b4360 con 0x7f48180b4680 2024-03-06T07:30:32.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f481ce00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f481815acb0 0x7f48180b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57158/0 (socket says 172.21.15.86:57158) 2024-03-06T07:30:32.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f481ce00640 1 -- 172.21.15.86:0/3064392777 learned_addr learned my addr 172.21.15.86:0/3064392777 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:32.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f4817e00640 1 --2- 172.21.15.86:0/3064392777 >> v2:172.21.15.86:3300/0 conn(0x7f48180b4680 0x7f481806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f4817400640 1 -- 172.21.15.86:0/3064392777 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1730841153 0 0) 0x7f481815b090 con 0x7f481815acb0 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f4817400640 1 -- 172.21.15.86:0/3064392777 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4800005730 con 0x7f481815acb0 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f4817e00640 1 -- 172.21.15.86:0/3064392777 >> v1:172.21.15.194:6789/0 conn(0x7f4818072340 legacy=0x7f48180ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f4817e00640 1 -- 172.21.15.86:0/3064392777 >> v1:172.21.15.86:6789/0 conn(0x7f481815acb0 legacy=0x7f48180b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.850+0000 7f4817e00640 1 -- 172.21.15.86:0/3064392777 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48180b2cf0 con 0x7f48180b4680 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.851+0000 7f4817e00640 1 --2- 172.21.15.86:0/3064392777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f481806e4f0 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7f48080097a0 tx=0x7f480802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=57a04690d871f965 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.851+0000 7f4817400640 1 -- 172.21.15.86:0/3064392777 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808035020 con 0x7f48180b4680 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f4817400640 1 -- 172.21.15.86:0/3064392777 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4808031830 con 0x7f48180b4680 2024-03-06T07:30:32.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f4817400640 1 -- 172.21.15.86:0/3064392777 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808031be0 con 0x7f48180b4680 2024-03-06T07:30:32.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f481f800640 1 -- 172.21.15.86:0/3064392777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 msgr2=0x7f481806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f481f800640 1 --2- 172.21.15.86:0/3064392777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f481806e4f0 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7f48080097a0 tx=0x7f480802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f481f800640 1 -- 172.21.15.86:0/3064392777 shutdown_connections 2024-03-06T07:30:32.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f481f800640 1 --2- 172.21.15.86:0/3064392777 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f481806e4f0 unknown :-1 s=CLOSED pgs=780 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f481f800640 1 -- 172.21.15.86:0/3064392777 >> 172.21.15.86:0/3064392777 conn(0x7f48180b1730 msgr2=0x7f48180b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:32.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f481f800640 1 -- 172.21.15.86:0/3064392777 shutdown_connections 2024-03-06T07:30:32.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.852+0000 7f481f800640 1 -- 172.21.15.86:0/3064392777 wait complete. 2024-03-06T07:30:32.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.854+0000 7f481f800640 1 Processor -- start 2024-03-06T07:30:32.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.854+0000 7f481f800640 1 -- start start 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f481f800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4818072340 0x7f48180cd7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f481f800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f4818156a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f481f800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f481815acb0 0x7f4818156fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f481f800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48180b4010 con 0x7f48180b4680 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f481f800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48180b32b0 con 0x7f4818072340 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f481f800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f481806ea30 con 0x7f481815acb0 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f4817e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f4818156a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f4817e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f4818156a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56056/0 (socket says 172.21.15.86:56056) 2024-03-06T07:30:32.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.855+0000 7f4817e00640 1 -- 172.21.15.86:0/1000262971 learned_addr learned my addr 172.21.15.86:0/1000262971 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:32.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f481d800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f481815acb0 0x7f4818156fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f4817e00640 1 -- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f481815acb0 msgr2=0x7f4818156fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f4817e00640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f481815acb0 0x7f4818156fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f4817e00640 1 -- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4818072340 msgr2=0x7f48180cd7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f4817e00640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4818072340 0x7f48180cd7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f4817e00640 1 -- 172.21.15.86:0/1000262971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48180ce1d0 con 0x7f48180b4680 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f481d800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f481815acb0 0x7f4818156fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f4817e00640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f4818156a80 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f480802b910 tx=0x7f48080046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.856+0000 7f4814c00640 1 -- 172.21.15.86:0/1000262971 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808035040 con 0x7f48180b4680 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.857+0000 7f4814c00640 1 -- 172.21.15.86:0/1000262971 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4808003610 con 0x7f48180b4680 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.857+0000 7f4814c00640 1 -- 172.21.15.86:0/1000262971 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48080039c0 con 0x7f48180b4680 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.857+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f48180ce4b0 con 0x7f48180b4680 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.857+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48180cea70 con 0x7f48180b4680 2024-03-06T07:30:32.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.858+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f481804f330 con 0x7f48180b4680 2024-03-06T07:30:32.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.859+0000 7f4814c00640 1 -- 172.21.15.86:0/1000262971 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4808038020 con 0x7f48180b4680 2024-03-06T07:30:32.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.860+0000 7f4814c00640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f47fc06b4d0 0x7f47fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:32.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.860+0000 7f4814c00640 1 -- 172.21.15.86:0/1000262971 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f48080ac0b0 con 0x7f48180b4680 2024-03-06T07:30:32.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.860+0000 7f481ce00640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f47fc06b4d0 0x7f47fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:32.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.861+0000 7f481ce00640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f47fc06b4d0 0x7f47fc06d990 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f480c000f60 tx=0x7f480c0078f0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:32.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:32.864+0000 7f4814c00640 1 -- 172.21.15.86:0/1000262971 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f480807b280 con 0x7f48180b4680 2024-03-06T07:30:33.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.066+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f48180b0c10 con 0x7f48180b4680 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.066+0000 7f4814c00640 1 -- 172.21.15.86:0/1000262971 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v66) v1 ==== 84+0+65 (secure 0 0 0) 0x7f480807ab80 con 0x7f48180b4680 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:30:33.068 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f47fc06b4d0 msgr2=0x7f47fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f47fc06b4d0 0x7f47fc06d990 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f480c000f60 tx=0x7f480c0078f0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 msgr2=0x7f4818156a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f4818156a80 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f480802b910 tx=0x7f48080046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 shutdown_connections 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f47fc06b4d0 0x7f47fc06d990 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f481815acb0 0x7f4818156fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48180b4680 0x7f4818156a80 unknown :-1 s=CLOSED pgs=781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 --2- 172.21.15.86:0/1000262971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4818072340 0x7f48180cd7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 >> 172.21.15.86:0/1000262971 conn(0x7f48180b1730 msgr2=0x7f48180ae900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:33.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 shutdown_connections 2024-03-06T07:30:33.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.069+0000 7f481f800640 1 -- 172.21.15.86:0/1000262971 wait complete. 2024-03-06T07:30:33.090 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:30:33.091 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:30:33.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.301+0000 7f3334200640 1 Processor -- start 2024-03-06T07:30:33.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.301+0000 7f3334200640 1 -- start start 2024-03-06T07:30:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.303+0000 7f3334200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f332c15f1f0 0x7f332c165d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.303+0000 7f3334200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f332c15b1d0 con 0x7f332c06eab0 2024-03-06T07:30:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.303+0000 7f3334200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f332c0b2bd0 con 0x7f332c15adf0 2024-03-06T07:30:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.303+0000 7f3334200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f332c15f5f0 con 0x7f332c15f1f0 2024-03-06T07:30:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.303+0000 7f3330e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f332c06eab0 0x7f332c15eae0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57168/0 (socket says 172.21.15.86:57168) 2024-03-06T07:30:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.303+0000 7f3330e00640 1 -- 172.21.15.86:0/2036726728 learned_addr learned my addr 172.21.15.86:0/2036726728 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:33.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.303+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2414951555 0 0) 0x7f332c15b1d0 con 0x7f332c06eab0 2024-03-06T07:30:33.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3314003520 con 0x7f332c06eab0 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f3332200640 1 --2- 172.21.15.86:0/2036726728 >> v2:172.21.15.194:3300/0 conn(0x7f332c15f1f0 0x7f332c165d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1118594013 0 0) 0x7f3314003520 con 0x7f332c06eab0 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 >> v2:172.21.15.194:3300/0 conn(0x7f332c15f1f0 msgr2=0x7f332c165d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 --2- 172.21.15.86:0/2036726728 >> v2:172.21.15.194:3300/0 conn(0x7f332c15f1f0 0x7f332c165d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 >> v1:172.21.15.194:6789/0 conn(0x7f332c15adf0 legacy=0x7f332c06e3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f332c0b2eb0 con 0x7f332c06eab0 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3318001670 con 0x7f332c06eab0 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 >> v1:172.21.15.86:6789/0 conn(0x7f332c06eab0 legacy=0x7f332c15eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3314003a20 0x7f3314003e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3314004800 0x7f3314004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314005230 0x7f331400dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f332c15f5f0 con 0x7f3314004800 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f331400e3a0 con 0x7f3314003a20 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f331400e5a0 con 0x7f3314005230 2024-03-06T07:30:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.304+0000 7f3330e00640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3314004800 0x7f3314004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.305+0000 7f3331800640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3314003a20 0x7f3314003e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.305+0000 7f3330e00640 1 -- 172.21.15.86:0/2036726728 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314005230 msgr2=0x7f331400dbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.305+0000 7f3330e00640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314005230 0x7f331400dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.305+0000 7f3330e00640 1 -- 172.21.15.86:0/2036726728 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3314003a20 msgr2=0x7f3314003e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.305+0000 7f3330e00640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3314003a20 0x7f3314003e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.305+0000 7f3330e00640 1 -- 172.21.15.86:0/2036726728 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3314003520 con 0x7f3314004800 2024-03-06T07:30:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.305+0000 7f3332200640 1 -- 172.21.15.86:0/2036726728 reap_dead start 2024-03-06T07:30:33.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.306+0000 7f3330e00640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3314004800 0x7f3314004c80 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7f331800c240 tx=0x7f3314005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dc5a00f9c087a8c8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:33.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.306+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3318007d10 con 0x7f3314004800 2024-03-06T07:30:33.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.306+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3318030070 con 0x7f3314004800 2024-03-06T07:30:33.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.306+0000 7f332be00640 1 -- 172.21.15.86:0/2036726728 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f331802cda0 con 0x7f3314004800 2024-03-06T07:30:33.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.307+0000 7f3334200640 1 -- 172.21.15.86:0/2036726728 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3314004800 msgr2=0x7f3314004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:33.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.307+0000 7f3334200640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3314004800 0x7f3314004c80 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7f331800c240 tx=0x7f3314005830 comp rx=0 tx=0).stop 2024-03-06T07:30:33.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.307+0000 7f3334200640 1 -- 172.21.15.86:0/2036726728 shutdown_connections 2024-03-06T07:30:33.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.307+0000 7f3334200640 1 --2- 172.21.15.86:0/2036726728 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3314004800 0x7f3314004c80 unknown :-1 s=CLOSED pgs=783 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.307+0000 7f3334200640 1 -- 172.21.15.86:0/2036726728 >> 172.21.15.86:0/2036726728 conn(0x7f332c0cfa80 msgr2=0x7f332c0cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:33.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.307+0000 7f3334200640 1 -- 172.21.15.86:0/2036726728 shutdown_connections 2024-03-06T07:30:33.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.307+0000 7f3334200640 1 -- 172.21.15.86:0/2036726728 wait complete. 2024-03-06T07:30:33.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.309+0000 7f3334200640 1 Processor -- start 2024-03-06T07:30:33.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.309+0000 7f3334200640 1 -- start start 2024-03-06T07:30:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3334200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314004800 0x7f332c1472b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3334200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f332c071e70 0x7f332c0c4aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3334200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f332c0c4fe0 0x7f332c0c5460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3334200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f332c15f850 con 0x7f332c071e70 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3334200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f332c0b3470 con 0x7f332c0c4fe0 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3334200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f332c0b3990 con 0x7f3314004800 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3330e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f332c071e70 0x7f332c0c4aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3330e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f332c071e70 0x7f332c0c4aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56080/0 (socket says 172.21.15.86:56080) 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.310+0000 7f3330e00640 1 -- 172.21.15.86:0/93379430 learned_addr learned my addr 172.21.15.86:0/93379430 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3331800640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314004800 0x7f332c1472b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3332200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f332c0c4fe0 0x7f332c0c5460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3330e00640 1 -- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314004800 msgr2=0x7f332c1472b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3330e00640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314004800 0x7f332c1472b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3330e00640 1 -- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f332c0c4fe0 msgr2=0x7f332c0c5460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3330e00640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f332c0c4fe0 0x7f332c0c5460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3330e00640 1 -- 172.21.15.86:0/93379430 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f332c0c6470 con 0x7f332c071e70 2024-03-06T07:30:33.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3331800640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314004800 0x7f332c1472b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:33.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3332200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f332c0c4fe0 0x7f332c0c5460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:33.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3330e00640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f332c071e70 0x7f332c0c4aa0 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7f331800c360 tx=0x7f331802e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:33.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3329600640 1 -- 172.21.15.86:0/93379430 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3318007d10 con 0x7f332c071e70 2024-03-06T07:30:33.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3329600640 1 -- 172.21.15.86:0/93379430 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3318004bf0 con 0x7f332c071e70 2024-03-06T07:30:33.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.311+0000 7f3329600640 1 -- 172.21.15.86:0/93379430 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3318007070 con 0x7f332c071e70 2024-03-06T07:30:33.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.312+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f332c157d30 con 0x7f332c071e70 2024-03-06T07:30:33.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.312+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f332c157fd0 con 0x7f332c071e70 2024-03-06T07:30:33.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.313+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f332c04f330 con 0x7f332c071e70 2024-03-06T07:30:33.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.314+0000 7f3329600640 1 -- 172.21.15.86:0/93379430 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3318048020 con 0x7f332c071e70 2024-03-06T07:30:33.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.315+0000 7f3329600640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f330c06b4d0 0x7f330c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:33.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.315+0000 7f3329600640 1 -- 172.21.15.86:0/93379430 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f33180ab440 con 0x7f332c071e70 2024-03-06T07:30:33.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.315+0000 7f3331800640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f330c06b4d0 0x7f330c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:33.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.316+0000 7f3331800640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f330c06b4d0 0x7f330c06d990 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f3320000c00 tx=0x7f3320009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:33.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.320+0000 7f3329600640 1 -- 172.21.15.86:0/93379430 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f331807a570 con 0x7f332c071e70 2024-03-06T07:30:33.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:33.514+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f332c15e700 con 0x7f332c071e70 2024-03-06T07:30:34.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.248+0000 7f3329600640 1 -- 172.21.15.86:0/93379430 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v67) v1 ==== 138+0+0 (secure 0 0 0) 0x7f3318079e70 con 0x7f332c071e70 2024-03-06T07:30:34.250 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:30:34.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f330c06b4d0 msgr2=0x7f330c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:34.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f330c06b4d0 0x7f330c06d990 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f3320000c00 tx=0x7f3320009040 comp rx=0 tx=0).stop 2024-03-06T07:30:34.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f332c071e70 msgr2=0x7f332c0c4aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:34.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f332c071e70 0x7f332c0c4aa0 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7f331800c360 tx=0x7f331802e070 comp rx=0 tx=0).stop 2024-03-06T07:30:34.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 shutdown_connections 2024-03-06T07:30:34.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f330c06b4d0 0x7f330c06d990 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f332c0c4fe0 0x7f332c0c5460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f332c071e70 0x7f332c0c4aa0 secure :-1 s=CLOSED pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7f331800c360 tx=0x7f331802e070 comp rx=0 tx=0).stop 2024-03-06T07:30:34.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 --2- 172.21.15.86:0/93379430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3314004800 0x7f332c1472b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.252+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 >> 172.21.15.86:0/93379430 conn(0x7f332c0cfa80 msgr2=0x7f332c06cf30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:34.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.253+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 shutdown_connections 2024-03-06T07:30:34.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.253+0000 7f3334200640 1 -- 172.21.15.86:0/93379430 wait complete. 2024-03-06T07:30:34.282 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:30:34.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.476+0000 7fbcd3e00640 1 Processor -- start 2024-03-06T07:30:34.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.476+0000 7fbcd3e00640 1 -- start start 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbcd4145320 0x7fbcd40b48a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd3e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4160420 0x7fbcd406b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd3e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbcd406c940 con 0x7fbcd406b920 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd3e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbcd406c6f0 con 0x7fbcd4145320 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd3e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbcd406cc20 con 0x7fbcd4160420 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbcd4145320 0x7fbcd40b48a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd2a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbcd4145320 0x7fbcd40b48a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56086/0 (socket says 172.21.15.86:56086) 2024-03-06T07:30:34.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd2a00640 1 -- 172.21.15.86:0/131569748 learned_addr learned my addr 172.21.15.86:0/131569748 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:34.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbccbe00640 1 --2- 172.21.15.86:0/131569748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4160420 0x7fbcd406b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:34.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd2a00640 1 -- 172.21.15.86:0/131569748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4160420 msgr2=0x7fbcd406b3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:34.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd2a00640 1 --2- 172.21.15.86:0/131569748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4160420 0x7fbcd406b3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd2a00640 1 -- 172.21.15.86:0/131569748 >> v1:172.21.15.194:6789/0 conn(0x7fbcd406b920 legacy=0x7fbcd40cb9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:34.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbcd2a00640 1 -- 172.21.15.86:0/131569748 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbcd406cf00 con 0x7fbcd4145320 2024-03-06T07:30:34.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.478+0000 7fbccbe00640 1 --2- 172.21.15.86:0/131569748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4160420 0x7fbcd406b3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:34.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.479+0000 7fbcd2a00640 1 --2- 172.21.15.86:0/131569748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4145320 0x7fbcd40b48a0 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7fbcc4009870 tx=0x7fbcc402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=77cb95204e2a8005 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:34.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd2000640 1 -- 172.21.15.86:0/131569748 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcc4035020 con 0x7fbcd4145320 2024-03-06T07:30:34.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd2000640 1 -- 172.21.15.86:0/131569748 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbcc4031640 con 0x7fbcd4145320 2024-03-06T07:30:34.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd2000640 1 -- 172.21.15.86:0/131569748 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcc402d040 con 0x7fbcd4145320 2024-03-06T07:30:34.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd3e00640 1 -- 172.21.15.86:0/131569748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4145320 msgr2=0x7fbcd40b48a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/131569748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4145320 0x7fbcd40b48a0 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7fbcc4009870 tx=0x7fbcc402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd3e00640 1 -- 172.21.15.86:0/131569748 shutdown_connections 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/131569748 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4160420 0x7fbcd406b3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/131569748 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4145320 0x7fbcd40b48a0 unknown :-1 s=CLOSED pgs=785 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.480+0000 7fbcd3e00640 1 -- 172.21.15.86:0/131569748 >> 172.21.15.86:0/131569748 conn(0x7fbcd40b1a70 msgr2=0x7fbcd40b1e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.481+0000 7fbcd3e00640 1 -- 172.21.15.86:0/131569748 shutdown_connections 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.481+0000 7fbcd3e00640 1 -- 172.21.15.86:0/131569748 wait complete. 2024-03-06T07:30:34.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.482+0000 7fbcd3e00640 1 Processor -- start 2024-03-06T07:30:34.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.482+0000 7fbcd3e00640 1 -- start start 2024-03-06T07:30:34.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.483+0000 7fbcd3e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbcd406b920 0x7fbcd4153880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:34.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.483+0000 7fbcd3e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4145320 0x7fbcd4142650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:34.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.483+0000 7fbcd3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4142b90 0x7fbcd41589c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.483+0000 7fbcd3e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbcd40b4de0 con 0x7fbcd4142b90 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.483+0000 7fbcd3e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbcd406d4c0 con 0x7fbcd406b920 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.483+0000 7fbcd3e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbcd4145700 con 0x7fbcd4145320 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4142b90 0x7fbcd41589c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4142b90 0x7fbcd41589c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56098/0 (socket says 172.21.15.86:56098) 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 -- 172.21.15.86:0/2829199508 learned_addr learned my addr 172.21.15.86:0/2829199508 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbccbe00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4145320 0x7fbcd4142650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd2a00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbcd406b920 0x7fbcd4153880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 -- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4145320 msgr2=0x7fbcd4142650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4145320 0x7fbcd4142650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 -- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbcd406b920 msgr2=0x7fbcd4153880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbcd406b920 0x7fbcd4153880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:34.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 -- 172.21.15.86:0/2829199508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbcd41f7b70 con 0x7fbcd4142b90 2024-03-06T07:30:34.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.484+0000 7fbcd3400640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4142b90 0x7fbcd41589c0 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7fbccc00b970 tx=0x7fbccc00be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:34.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.485+0000 7fbccaa00640 1 -- 172.21.15.86:0/2829199508 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbccc019070 con 0x7fbcd4142b90 2024-03-06T07:30:34.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.485+0000 7fbccaa00640 1 -- 172.21.15.86:0/2829199508 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbccc00cb40 con 0x7fbcd4142b90 2024-03-06T07:30:34.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.485+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbcd41f7e50 con 0x7fbcd4142b90 2024-03-06T07:30:34.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.485+0000 7fbccaa00640 1 -- 172.21.15.86:0/2829199508 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbccc014440 con 0x7fbcd4142b90 2024-03-06T07:30:34.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.485+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbcd41f8410 con 0x7fbcd4142b90 2024-03-06T07:30:34.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.486+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbcd4075480 con 0x7fbcd4142b90 2024-03-06T07:30:34.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.487+0000 7fbccaa00640 1 -- 172.21.15.86:0/2829199508 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbccc01c070 con 0x7fbcd4142b90 2024-03-06T07:30:34.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.488+0000 7fbccaa00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbcb006b4d0 0x7fbcb006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:34.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.488+0000 7fbccaa00640 1 -- 172.21.15.86:0/2829199508 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(67..67 src has 1..67) v4 ==== 5227+0+0 (secure 0 0 0) 0x7fbccc089ff0 con 0x7fbcd4142b90 2024-03-06T07:30:34.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.488+0000 7fbcd2a00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbcb006b4d0 0x7fbcb006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:34.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.489+0000 7fbcd2a00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbcb006b4d0 0x7fbcb006d990 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7fbcc4036000 tx=0x7fbcc403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:34.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.492+0000 7fbccaa00640 1 -- 172.21.15.86:0/2829199508 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbccc059090 con 0x7fbcd4142b90 2024-03-06T07:30:34.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:34.673+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fbcd406e6f0 con 0x7fbcd4142b90 2024-03-06T07:30:35.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.263+0000 7fbccaa00640 1 -- 172.21.15.86:0/2829199508 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v68) v1 ==== 170+0+0 (secure 0 0 0) 0x7fbccc058990 con 0x7fbcd4142b90 2024-03-06T07:30:35.265 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:30:35.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbcb006b4d0 msgr2=0x7fbcb006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:35.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbcb006b4d0 0x7fbcb006d990 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7fbcc4036000 tx=0x7fbcc403c040 comp rx=0 tx=0).stop 2024-03-06T07:30:35.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4142b90 msgr2=0x7fbcd41589c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:35.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4142b90 0x7fbcd41589c0 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7fbccc00b970 tx=0x7fbccc00be30 comp rx=0 tx=0).stop 2024-03-06T07:30:35.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 shutdown_connections 2024-03-06T07:30:35.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbcb006b4d0 0x7fbcb006d990 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:35.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbcd4142b90 0x7fbcd41589c0 unknown :-1 s=CLOSED pgs=786 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:35.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbcd4145320 0x7fbcd4142650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:35.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 --2- 172.21.15.86:0/2829199508 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbcd406b920 0x7fbcd4153880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:35.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.267+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 >> 172.21.15.86:0/2829199508 conn(0x7fbcd40b1a70 msgr2=0x7fbcd40713b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:35.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.268+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 shutdown_connections 2024-03-06T07:30:35.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.268+0000 7fbcd3e00640 1 -- 172.21.15.86:0/2829199508 wait complete. 2024-03-06T07:30:35.306 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:30:35.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.484+0000 7f1ae8c00640 1 Processor -- start 2024-03-06T07:30:35.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.484+0000 7f1ae8c00640 1 -- start start 2024-03-06T07:30:35.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.485+0000 7f1ae8c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1ae4158950 0x7f1ae40cf6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:35.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1ae8c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ae415b130 con 0x7f1ae4158570 2024-03-06T07:30:35.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1ae8c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ae40af170 con 0x7f1ae40cfc10 2024-03-06T07:30:35.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1ae8c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1ae415b3f0 con 0x7f1ae4158950 2024-03-06T07:30:35.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1ae3400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1ae4158570 0x7f1ae40ae6b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57172/0 (socket says 172.21.15.86:57172) 2024-03-06T07:30:35.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1ae3400640 1 -- 172.21.15.86:0/1448779519 learned_addr learned my addr 172.21.15.86:0/1448779519 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:35.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1adbe00640 1 --2- 172.21.15.86:0/1448779519 >> v2:172.21.15.86:3300/0 conn(0x7f1ae4158950 0x7f1ae40cf6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:35.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1ae2a00640 1 -- 172.21.15.86:0/1448779519 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 650014287 0 0) 0x7f1ae415b130 con 0x7f1ae4158570 2024-03-06T07:30:35.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1ae2a00640 1 -- 172.21.15.86:0/1448779519 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1ac8003520 con 0x7f1ae4158570 2024-03-06T07:30:35.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1adbe00640 1 -- 172.21.15.86:0/1448779519 >> v1:172.21.15.194:6789/0 conn(0x7f1ae40cfc10 legacy=0x7f1ae40cd7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:35.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1adbe00640 1 -- 172.21.15.86:0/1448779519 >> v1:172.21.15.86:6789/0 conn(0x7f1ae4158570 legacy=0x7f1ae40ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:35.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.486+0000 7f1adbe00640 1 -- 172.21.15.86:0/1448779519 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ae415b6d0 con 0x7f1ae4158950 2024-03-06T07:30:35.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.487+0000 7f1adbe00640 1 --2- 172.21.15.86:0/1448779519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae4158950 0x7f1ae40cf6d0 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7f1acc0097a0 tx=0x7f1acc02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f31eb3a47cd0e691 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:35.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae2a00640 1 -- 172.21.15.86:0/1448779519 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1acc0315a0 con 0x7f1ae4158950 2024-03-06T07:30:35.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae2a00640 1 -- 172.21.15.86:0/1448779519 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1acc037070 con 0x7f1ae4158950 2024-03-06T07:30:35.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae2a00640 1 -- 172.21.15.86:0/1448779519 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1acc031960 con 0x7f1ae4158950 2024-03-06T07:30:35.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae8c00640 1 -- 172.21.15.86:0/1448779519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae4158950 msgr2=0x7f1ae40cf6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:35.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/1448779519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae4158950 0x7f1ae40cf6d0 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7f1acc0097a0 tx=0x7f1acc02b840 comp rx=0 tx=0).stop 2024-03-06T07:30:35.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae8c00640 1 -- 172.21.15.86:0/1448779519 shutdown_connections 2024-03-06T07:30:35.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/1448779519 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae4158950 0x7f1ae40cf6d0 unknown :-1 s=CLOSED pgs=788 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:35.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.488+0000 7f1ae8c00640 1 -- 172.21.15.86:0/1448779519 >> 172.21.15.86:0/1448779519 conn(0x7f1ae406afc0 msgr2=0x7f1ae406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:35.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.489+0000 7f1ae8c00640 1 -- 172.21.15.86:0/1448779519 shutdown_connections 2024-03-06T07:30:35.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.489+0000 7f1ae8c00640 1 -- 172.21.15.86:0/1448779519 wait complete. 2024-03-06T07:30:35.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.490+0000 7f1ae8c00640 1 Processor -- start 2024-03-06T07:30:35.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.491+0000 7f1ae8c00640 1 -- start start 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.491+0000 7f1ae8c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae40cfc10 0x7f1ae4155620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.491+0000 7f1ae8c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae4158570 0x7f1ae4145bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae4158950 0x7f1ae40ae0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae8c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae40aedc0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae8c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae40b4950 con 0x7f1ae4158570 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae4155b60 con 0x7f1ae4158950 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae40cfc10 0x7f1ae4155620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae40cfc10 0x7f1ae4155620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56110/0 (socket says 172.21.15.86:56110) 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 -- 172.21.15.86:0/903427209 learned_addr learned my addr 172.21.15.86:0/903427209 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:35.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 -- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae4158950 msgr2=0x7f1ae40ae0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1adbe00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae4158570 0x7f1ae4145bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3e00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae4158950 0x7f1ae40ae0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae4158950 0x7f1ae40ae0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 -- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae4158570 msgr2=0x7f1ae4145bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae4158570 0x7f1ae4145bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1ae3400640 1 -- 172.21.15.86:0/903427209 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ae41464d0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.492+0000 7f1adbe00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae4158570 0x7f1ae4145bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:35.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.493+0000 7f1ae3400640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae40cfc10 0x7f1ae4155620 secure :-1 s=READY pgs=789 cs=0 l=1 rev1=1 crypto rx=0x7f1ad400c430 tx=0x7f1ad400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:35.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.493+0000 7f1adb400640 1 -- 172.21.15.86:0/903427209 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4017070 con 0x7f1ae40cfc10 2024-03-06T07:30:35.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.493+0000 7f1adb400640 1 -- 172.21.15.86:0/903427209 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ad4005ce0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.493+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ae41467b0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.493+0000 7f1adb400640 1 -- 172.21.15.86:0/903427209 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad40072f0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.493+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ae4143ec0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.494+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1ae415bed0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.495+0000 7f1adb400640 1 -- 172.21.15.86:0/903427209 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1ad401a070 con 0x7f1ae40cfc10 2024-03-06T07:30:35.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.496+0000 7f1adb400640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:35.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.496+0000 7f1adb400640 1 -- 172.21.15.86:0/903427209 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(68..68 src has 1..68) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f1ad4089670 con 0x7f1ae40cfc10 2024-03-06T07:30:35.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.496+0000 7f1adbe00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:35.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.497+0000 7f1adbe00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f1acc036000 tx=0x7f1acc008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:35.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.501+0000 7f1adb400640 1 -- 172.21.15.86:0/903427209 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1ae415bed0 con 0x7f1ae40cfc10 2024-03-06T07:30:35.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:35.738+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f1ae404f330 con 0x7f1ae40cfc10 2024-03-06T07:30:36.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.293+0000 7f1adb400640 1 -- 172.21.15.86:0/903427209 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 10 and data pool 11 v38) v1 ==== 174+0+0 (secure 0 0 0) 0x7f1ad4058540 con 0x7f1ae40cfc10 2024-03-06T07:30:36.299 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 10 and data pool 11 2024-03-06T07:30:36.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.300+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 msgr2=0x7f1ac006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.300+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f1acc036000 tx=0x7f1acc008030 comp rx=0 tx=0).stop 2024-03-06T07:30:36.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.300+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae40cfc10 msgr2=0x7f1ae4155620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.300+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae40cfc10 0x7f1ae4155620 secure :-1 s=READY pgs=789 cs=0 l=1 rev1=1 crypto rx=0x7f1ad400c430 tx=0x7f1ad400c8f0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.300+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 shutdown_connections 2024-03-06T07:30:36.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.301+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.301+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae4158950 0x7f1ae40ae0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.301+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae4158570 0x7f1ae4145bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.301+0000 7f1ae8c00640 1 --2- 172.21.15.86:0/903427209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ae40cfc10 0x7f1ae4155620 unknown :-1 s=CLOSED pgs=789 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.301+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 >> 172.21.15.86:0/903427209 conn(0x7f1ae406afc0 msgr2=0x7f1ae40ad860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:36.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.301+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 shutdown_connections 2024-03-06T07:30:36.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.301+0000 7f1ae8c00640 1 -- 172.21.15.86:0/903427209 wait complete. 2024-03-06T07:30:36.336 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:36.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.513+0000 7ff608400640 1 Processor -- start 2024-03-06T07:30:36.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.514+0000 7ff608400640 1 -- start start 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff608400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff60015f4f0 0x7ff6000706d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff608400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff600070c10 0x7ff6000cc700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff608400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff600160a20 con 0x7ff60015f8d0 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff608400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff60006ef50 con 0x7ff600070c10 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff608400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff60006f230 con 0x7ff60015f4f0 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff605000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff600070c10 0x7ff6000cc700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff605000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff600070c10 0x7ff6000cc700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55646/0 (socket says 172.21.15.86:55646) 2024-03-06T07:30:36.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.515+0000 7ff605000640 1 -- 172.21.15.86:0/3304375720 learned_addr learned my addr 172.21.15.86:0/3304375720 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:36.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.516+0000 7ff605000640 1 -- 172.21.15.86:0/3304375720 >> v2:172.21.15.194:3300/0 conn(0x7ff60015f4f0 msgr2=0x7ff6000706d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:36.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.516+0000 7ff605000640 1 --2- 172.21.15.86:0/3304375720 >> v2:172.21.15.194:3300/0 conn(0x7ff60015f4f0 0x7ff6000706d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.516+0000 7ff605000640 1 -- 172.21.15.86:0/3304375720 >> v1:172.21.15.194:6789/0 conn(0x7ff60015f8d0 legacy=0x7ff60015fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:36.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.516+0000 7ff605000640 1 -- 172.21.15.86:0/3304375720 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff600160460 con 0x7ff600070c10 2024-03-06T07:30:36.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.517+0000 7ff605000640 1 --2- 172.21.15.86:0/3304375720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff600070c10 0x7ff6000cc700 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7ff60006bf30 tx=0x7ff5ec02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ac7cd991bf6f16ad server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:36.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.517+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3304375720 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5ec035020 con 0x7ff600070c10 2024-03-06T07:30:36.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.517+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3304375720 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5ec031870 con 0x7ff600070c10 2024-03-06T07:30:36.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.517+0000 7ff5ffe00640 1 -- 172.21.15.86:0/3304375720 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5ec031c40 con 0x7ff600070c10 2024-03-06T07:30:36.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 -- 172.21.15.86:0/3304375720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff600070c10 msgr2=0x7ff6000cc700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 --2- 172.21.15.86:0/3304375720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff600070c10 0x7ff6000cc700 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7ff60006bf30 tx=0x7ff5ec02b910 comp rx=0 tx=0).stop 2024-03-06T07:30:36.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 -- 172.21.15.86:0/3304375720 shutdown_connections 2024-03-06T07:30:36.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 --2- 172.21.15.86:0/3304375720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff600070c10 0x7ff6000cc700 unknown :-1 s=CLOSED pgs=790 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 --2- 172.21.15.86:0/3304375720 >> v2:172.21.15.194:3300/0 conn(0x7ff60015f4f0 0x7ff6000706d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 -- 172.21.15.86:0/3304375720 >> 172.21.15.86:0/3304375720 conn(0x7ff6000b3800 msgr2=0x7ff6000b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:36.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 -- 172.21.15.86:0/3304375720 shutdown_connections 2024-03-06T07:30:36.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.518+0000 7ff608400640 1 -- 172.21.15.86:0/3304375720 wait complete. 2024-03-06T07:30:36.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.520+0000 7ff608400640 1 Processor -- start 2024-03-06T07:30:36.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.520+0000 7ff608400640 1 -- start start 2024-03-06T07:30:36.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff608400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff60015f4f0 0x7ff6001f7410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff608400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff60015f8d0 0x7ff6000c8da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff608400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff60006ddc0 0x7ff6000c92e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff608400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6000ccdc0 con 0x7ff60006ddc0 2024-03-06T07:30:36.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff608400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6000ccc40 con 0x7ff60015f4f0 2024-03-06T07:30:36.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff608400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff6000ccf40 con 0x7ff60015f8d0 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff606400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff60006ddc0 0x7ff6000c92e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff606400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff60006ddc0 0x7ff6000c92e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55652/0 (socket says 172.21.15.86:55652) 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff606400640 1 -- 172.21.15.86:0/2212795011 learned_addr learned my addr 172.21.15.86:0/2212795011 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff605a00640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff60015f4f0 0x7ff6001f7410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff605000640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff60015f8d0 0x7ff6000c8da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff606400640 1 -- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff60015f8d0 msgr2=0x7ff6000c8da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.521+0000 7ff606400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff60015f8d0 0x7ff6000c8da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff606400640 1 -- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff60015f4f0 msgr2=0x7ff6001f7410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff606400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff60015f4f0 0x7ff6001f7410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff606400640 1 -- 172.21.15.86:0/2212795011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6000cb0c0 con 0x7ff60006ddc0 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff605000640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff60015f8d0 0x7ff6000c8da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:36.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff606400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff60006ddc0 0x7ff6000c92e0 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7ff5f800c7e0 tx=0x7ff5f800cca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:36.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff5fd600640 1 -- 172.21.15.86:0/2212795011 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5f8017070 con 0x7ff60006ddc0 2024-03-06T07:30:36.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff5fd600640 1 -- 172.21.15.86:0/2212795011 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5f8003da0 con 0x7ff60006ddc0 2024-03-06T07:30:36.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff5fd600640 1 -- 172.21.15.86:0/2212795011 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5f80052b0 con 0x7ff60006ddc0 2024-03-06T07:30:36.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff6000cb3a0 con 0x7ff60006ddc0 2024-03-06T07:30:36.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.522+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff6000c9820 con 0x7ff60006ddc0 2024-03-06T07:30:36.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.523+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff60004f330 con 0x7ff60006ddc0 2024-03-06T07:30:36.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.524+0000 7ff5fd600640 1 -- 172.21.15.86:0/2212795011 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff5f8005450 con 0x7ff60006ddc0 2024-03-06T07:30:36.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.525+0000 7ff5fd600640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5e006b500 0x7ff5e006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.525+0000 7ff5fd600640 1 -- 172.21.15.86:0/2212795011 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(69..69 src has 1..69) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff5f8089610 con 0x7ff60006ddc0 2024-03-06T07:30:36.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.525+0000 7ff605a00640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5e006b500 0x7ff5e006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.526+0000 7ff605a00640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5e006b500 0x7ff5e006d9c0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7ff6000ccbc0 tx=0x7ff5f4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:36.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.530+0000 7ff5fd600640 1 -- 172.21.15.86:0/2212795011 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff5f8058490 con 0x7ff60006ddc0 2024-03-06T07:30:36.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.721+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7ff6000c8b60 con 0x7ff60006ddc0 2024-03-06T07:30:36.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.724+0000 7ff5fd600640 1 -- 172.21.15.86:0/2212795011 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v69) v1 ==== 74+0+16693 (secure 0 0 0) 0x7ff5f8057d90 con 0x7ff60006ddc0 2024-03-06T07:30:36.726 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:36.726 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:35.739699+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:30:33.566852+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-06T07:30:34.722690+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5e006b500 msgr2=0x7ff5e006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5e006b500 0x7ff5e006d9c0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7ff6000ccbc0 tx=0x7ff5f4008040 comp rx=0 tx=0).stop 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff60006ddc0 msgr2=0x7ff6000c92e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff60006ddc0 0x7ff6000c92e0 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7ff5f800c7e0 tx=0x7ff5f800cca0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 shutdown_connections 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5e006b500 0x7ff5e006d9c0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff60006ddc0 0x7ff6000c92e0 unknown :-1 s=CLOSED pgs=791 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff60015f8d0 0x7ff6000c8da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 --2- 172.21.15.86:0/2212795011 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff60015f4f0 0x7ff6001f7410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 >> 172.21.15.86:0/2212795011 conn(0x7ff6000b3800 msgr2=0x7ff60006f750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:36.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 shutdown_connections 2024-03-06T07:30:36.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.727+0000 7ff608400640 1 -- 172.21.15.86:0/2212795011 wait complete. 2024-03-06T07:30:36.747 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:36.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.949+0000 7fe722000640 1 Processor -- start 2024-03-06T07:30:36.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.949+0000 7fe722000640 1 -- start start 2024-03-06T07:30:36.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.950+0000 7fe722000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c0b4680 0x7fe71c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.950+0000 7fe722000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe71c072340 0x7fe71c06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.950+0000 7fe722000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe71c15b090 con 0x7fe71c15acb0 2024-03-06T07:30:36.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.950+0000 7fe722000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe71c0b2a30 con 0x7fe71c072340 2024-03-06T07:30:36.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.950+0000 7fe722000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe71c0b4360 con 0x7fe71c0b4680 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe71be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe71c072340 0x7fe71c06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe71be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe71c072340 0x7fe71c06ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:49186/0 (socket says 172.21.15.86:49186) 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe71be00640 1 -- 172.21.15.86:0/2685654706 learned_addr learned my addr 172.21.15.86:0/2685654706 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe713e00640 1 --2- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c0b4680 0x7fe71c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe71be00640 1 -- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c0b4680 msgr2=0x7fe71c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe71be00640 1 --2- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c0b4680 0x7fe71c06e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe71be00640 1 -- 172.21.15.86:0/2685654706 >> v1:172.21.15.194:6789/0 conn(0x7fe71c15acb0 legacy=0x7fe71c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.951+0000 7fe71be00640 1 -- 172.21.15.86:0/2685654706 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe71c0b2cf0 con 0x7fe71c072340 2024-03-06T07:30:36.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.953+0000 7fe71be00640 1 --2- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c072340 0x7fe71c06ea30 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fe714009e50 tx=0x7fe71402bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a6a6a1f9711036c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe71aa00640 1 -- 172.21.15.86:0/2685654706 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe71402cb00 con 0x7fe71c072340 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe71aa00640 1 -- 172.21.15.86:0/2685654706 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe71402cca0 con 0x7fe71c072340 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe71aa00640 1 -- 172.21.15.86:0/2685654706 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe714034610 con 0x7fe71c072340 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe722000640 1 -- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c072340 msgr2=0x7fe71c06ea30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe722000640 1 --2- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c072340 0x7fe71c06ea30 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fe714009e50 tx=0x7fe71402bf60 comp rx=0 tx=0).stop 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe722000640 1 -- 172.21.15.86:0/2685654706 shutdown_connections 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe722000640 1 --2- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c072340 0x7fe71c06ea30 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe722000640 1 --2- 172.21.15.86:0/2685654706 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c0b4680 0x7fe71c06e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.954+0000 7fe722000640 1 -- 172.21.15.86:0/2685654706 >> 172.21.15.86:0/2685654706 conn(0x7fe71c0b1730 msgr2=0x7fe71c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.955+0000 7fe722000640 1 -- 172.21.15.86:0/2685654706 shutdown_connections 2024-03-06T07:30:36.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.955+0000 7fe722000640 1 -- 172.21.15.86:0/2685654706 wait complete. 2024-03-06T07:30:36.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.956+0000 7fe722000640 1 Processor -- start 2024-03-06T07:30:36.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.956+0000 7fe722000640 1 -- start start 2024-03-06T07:30:36.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.957+0000 7fe722000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe71c072340 0x7fe71c1f3d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.957+0000 7fe722000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c0b4680 0x7fe71c155440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.957+0000 7fe722000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c15acb0 0x7fe71c155980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.957+0000 7fe722000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe71c0b4010 con 0x7fe71c072340 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.957+0000 7fe722000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe71c0b32b0 con 0x7fe71c0b4680 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.957+0000 7fe722000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe71c0ccfb0 con 0x7fe71c15acb0 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe71c072340 0x7fe71c1f3d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe71c072340 0x7fe71c1f3d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55658/0 (socket says 172.21.15.86:55658) 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 -- 172.21.15.86:0/1976815806 learned_addr learned my addr 172.21.15.86:0/1976815806 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe713e00640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c0b4680 0x7fe71c155440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71be00640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c15acb0 0x7fe71c155980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 -- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c15acb0 msgr2=0x7fe71c155980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c15acb0 0x7fe71c155980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 -- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c0b4680 msgr2=0x7fe71c155440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:36.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c0b4680 0x7fe71c155440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:36.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 -- 172.21.15.86:0/1976815806 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe71c0cd570 con 0x7fe71c072340 2024-03-06T07:30:36.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.958+0000 7fe71b400640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe71c072340 0x7fe71c1f3d40 secure :-1 s=READY pgs=792 cs=0 l=1 rev1=1 crypto rx=0x7fe70c00c3f0 tx=0x7fe70c00c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:36.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.959+0000 7fe713400640 1 -- 172.21.15.86:0/1976815806 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe70c0052d0 con 0x7fe71c072340 2024-03-06T07:30:36.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.959+0000 7fe713400640 1 -- 172.21.15.86:0/1976815806 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe70c005490 con 0x7fe71c072340 2024-03-06T07:30:36.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.959+0000 7fe713400640 1 -- 172.21.15.86:0/1976815806 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe70c007260 con 0x7fe71c072340 2024-03-06T07:30:36.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.959+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe71c0cd850 con 0x7fe71c072340 2024-03-06T07:30:36.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.959+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe71c0cde10 con 0x7fe71c072340 2024-03-06T07:30:36.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.960+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe71c04f330 con 0x7fe71c072340 2024-03-06T07:30:36.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.961+0000 7fe713400640 1 -- 172.21.15.86:0/1976815806 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe70c007400 con 0x7fe71c072340 2024-03-06T07:30:36.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.962+0000 7fe713400640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe6f806b4d0 0x7fe6f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:36.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.962+0000 7fe713400640 1 -- 172.21.15.86:0/1976815806 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(69..69 src has 1..69) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe70c0896e0 con 0x7fe71c072340 2024-03-06T07:30:36.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.962+0000 7fe713e00640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe6f806b4d0 0x7fe6f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:36.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.963+0000 7fe713e00640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe6f806b4d0 0x7fe6f806d990 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fe71c0b0b30 tx=0x7fe704008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:36.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:36.966+0000 7fe713400640 1 -- 172.21.15.86:0/1976815806 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe70c058560 con 0x7fe71c072340 2024-03-06T07:30:37.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.161+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe71c0b14e0 con 0x7fe71c072340 2024-03-06T07:30:37.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.164+0000 7fe713400640 1 -- 172.21.15.86:0/1976815806 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v69) v1 ==== 74+0+16693 (secure 0 0 0) 0x7fe70c057e60 con 0x7fe71c072340 2024-03-06T07:30:37.166 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:37.166 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:35.739699+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:30:33.566852+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-06T07:30:34.722690+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:37.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.166+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe6f806b4d0 msgr2=0x7fe6f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:37.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.166+0000 7fe722000640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe6f806b4d0 0x7fe6f806d990 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fe71c0b0b30 tx=0x7fe704008040 comp rx=0 tx=0).stop 2024-03-06T07:30:37.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.166+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe71c072340 msgr2=0x7fe71c1f3d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:37.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.166+0000 7fe722000640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe71c072340 0x7fe71c1f3d40 secure :-1 s=READY pgs=792 cs=0 l=1 rev1=1 crypto rx=0x7fe70c00c3f0 tx=0x7fe70c00c8b0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 shutdown_connections 2024-03-06T07:30:37.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe6f806b4d0 0x7fe6f806d990 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe71c15acb0 0x7fe71c155980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe71c0b4680 0x7fe71c155440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 --2- 172.21.15.86:0/1976815806 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe71c072340 0x7fe71c1f3d40 unknown :-1 s=CLOSED pgs=792 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 >> 172.21.15.86:0/1976815806 conn(0x7fe71c0b1730 msgr2=0x7fe71c06d3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:37.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 shutdown_connections 2024-03-06T07:30:37.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.167+0000 7fe722000640 1 -- 172.21.15.86:0/1976815806 wait complete. 2024-03-06T07:30:37.186 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:30:37.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.403+0000 7f3706e00640 1 Processor -- start 2024-03-06T07:30:37.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.403+0000 7f3706e00640 1 -- start start 2024-03-06T07:30:37.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f3706e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f37000d07f0 0x7f37000d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:37.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f3706e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000cfa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f3706e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f37000b3730 con 0x7f370015e450 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f3706e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f37000d4930 con 0x7f37000d07f0 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f3706e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3700067b20 con 0x7f37000d1bf0 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f36ffe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f37000d07f0 0x7f37000d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f36ffe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f37000d07f0 0x7f37000d0bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55674/0 (socket says 172.21.15.86:55674) 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.405+0000 7f36ffe00640 1 -- 172.21.15.86:0/2544528773 learned_addr learned my addr 172.21.15.86:0/2544528773 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.406+0000 7f36ffe00640 1 -- 172.21.15.86:0/2544528773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 msgr2=0x7f37000cfa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:37.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.406+0000 7f3704e00640 1 --2- 172.21.15.86:0/2544528773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000cfa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:37.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.406+0000 7f36ffe00640 1 --2- 172.21.15.86:0/2544528773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000cfa30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.406+0000 7f36ffe00640 1 -- 172.21.15.86:0/2544528773 >> v1:172.21.15.194:6789/0 conn(0x7f370015e450 legacy=0x7f37000b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:37.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.406+0000 7f36ffe00640 1 -- 172.21.15.86:0/2544528773 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37000d1110 con 0x7f37000d07f0 2024-03-06T07:30:37.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.408+0000 7f36ffe00640 1 --2- 172.21.15.86:0/2544528773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f37000d07f0 0x7f37000d0bd0 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7f36f0009870 tx=0x7f36f002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ff91e36987afe7a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:37.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.408+0000 7f36fea00640 1 -- 172.21.15.86:0/2544528773 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36f0035020 con 0x7f37000d07f0 2024-03-06T07:30:37.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.408+0000 7f36fea00640 1 -- 172.21.15.86:0/2544528773 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36f0031640 con 0x7f37000d07f0 2024-03-06T07:30:37.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.408+0000 7f36fea00640 1 -- 172.21.15.86:0/2544528773 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36f002d040 con 0x7f37000d07f0 2024-03-06T07:30:37.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.408+0000 7f3706e00640 1 -- 172.21.15.86:0/2544528773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f37000d07f0 msgr2=0x7f37000d0bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:37.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.408+0000 7f3706e00640 1 --2- 172.21.15.86:0/2544528773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f37000d07f0 0x7f37000d0bd0 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7f36f0009870 tx=0x7f36f002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.409+0000 7f3706e00640 1 -- 172.21.15.86:0/2544528773 shutdown_connections 2024-03-06T07:30:37.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.409+0000 7f3706e00640 1 --2- 172.21.15.86:0/2544528773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000cfa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.409+0000 7f3706e00640 1 --2- 172.21.15.86:0/2544528773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f37000d07f0 0x7f37000d0bd0 unknown :-1 s=CLOSED pgs=793 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.409+0000 7f3706e00640 1 -- 172.21.15.86:0/2544528773 >> 172.21.15.86:0/2544528773 conn(0x7f37000d3e10 msgr2=0x7f37000d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:37.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.409+0000 7f3706e00640 1 -- 172.21.15.86:0/2544528773 shutdown_connections 2024-03-06T07:30:37.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.409+0000 7f3706e00640 1 -- 172.21.15.86:0/2544528773 wait complete. 2024-03-06T07:30:37.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.410+0000 7f3706e00640 1 Processor -- start 2024-03-06T07:30:37.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.410+0000 7f3706e00640 1 -- start start 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.411+0000 7f3706e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f37000d07f0 0x7f370006bc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.411+0000 7f3706e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000b9350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.411+0000 7f3706e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f370015e450 0x7f37000b9890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.411+0000 7f3706e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37000b3c80 con 0x7f370015e450 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.411+0000 7f3706e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37000b39a0 con 0x7f37000d07f0 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.411+0000 7f3706e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f37000b3f60 con 0x7f37000d1bf0 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f370015e450 0x7f37000b9890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f370015e450 0x7f37000b9890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55676/0 (socket says 172.21.15.86:55676) 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 -- 172.21.15.86:0/194093012 learned_addr learned my addr 172.21.15.86:0/194093012 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f36ff400640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000b9350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f36ffe00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f37000d07f0 0x7f370006bc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 -- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 msgr2=0x7f37000b9350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000b9350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 -- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f37000d07f0 msgr2=0x7f370006bc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f37000d07f0 0x7f370006bc70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.412+0000 7f3704e00640 1 -- 172.21.15.86:0/194093012 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37001f3b90 con 0x7f370015e450 2024-03-06T07:30:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.413+0000 7f3704e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f370015e450 0x7f37000b9890 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7f36f800cd90 tx=0x7f36f800a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.413+0000 7f36f7e00640 1 -- 172.21.15.86:0/194093012 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36f8003b90 con 0x7f370015e450 2024-03-06T07:30:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.413+0000 7f36f7e00640 1 -- 172.21.15.86:0/194093012 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36f8003d50 con 0x7f370015e450 2024-03-06T07:30:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.413+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f37001f3e70 con 0x7f370015e450 2024-03-06T07:30:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.413+0000 7f36f7e00640 1 -- 172.21.15.86:0/194093012 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36f8004260 con 0x7f370015e450 2024-03-06T07:30:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.413+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f37001f4430 con 0x7f370015e450 2024-03-06T07:30:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.414+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f370004f330 con 0x7f370015e450 2024-03-06T07:30:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.415+0000 7f36f7e00640 1 -- 172.21.15.86:0/194093012 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f36f8004400 con 0x7f370015e450 2024-03-06T07:30:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.416+0000 7f36f7e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f36e006b4d0 0x7f36e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.416+0000 7f36f7e00640 1 -- 172.21.15.86:0/194093012 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f36f808ada0 con 0x7f370015e450 2024-03-06T07:30:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.416+0000 7f36ffe00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f36e006b4d0 0x7f36e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.417+0000 7f36ffe00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f36e006b4d0 0x7f36e006d990 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f36f0036000 tx=0x7f36f0008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.420+0000 7f36f7e00640 1 -- 172.21.15.86:0/194093012 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f36f8059b80 con 0x7f370015e450 2024-03-06T07:30:37.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:37.710+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f37000d21b0 con 0x7f370015e450 2024-03-06T07:30:38.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.301+0000 7f36f7e00640 1 -- 172.21.15.86:0/194093012 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v41) v1 ==== 118+0+0 (secure 0 0 0) 0x7f36f8059480 con 0x7f370015e450 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.304+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f36e006b4d0 msgr2=0x7f36e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f36e006b4d0 0x7f36e006d990 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f36f0036000 tx=0x7f36f0008030 comp rx=0 tx=0).stop 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f370015e450 msgr2=0x7f37000b9890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f370015e450 0x7f37000b9890 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7f36f800cd90 tx=0x7f36f800a660 comp rx=0 tx=0).stop 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 shutdown_connections 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f36e006b4d0 0x7f36e006d990 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f370015e450 0x7f37000b9890 unknown :-1 s=CLOSED pgs=794 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f37000d1bf0 0x7f37000b9350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 --2- 172.21.15.86:0/194093012 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f37000d07f0 0x7f370006bc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 >> 172.21.15.86:0/194093012 conn(0x7f37000d3e10 msgr2=0x7f37000b09e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 shutdown_connections 2024-03-06T07:30:38.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.305+0000 7f3706e00640 1 -- 172.21.15.86:0/194093012 wait complete. 2024-03-06T07:30:38.326 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:30:38.327 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:30:38.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.532+0000 7f18df200640 1 Processor -- start 2024-03-06T07:30:38.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.532+0000 7f18df200640 1 -- start start 2024-03-06T07:30:38.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18df200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e0158570 0x7f18e00ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:38.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18df200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f18e00cfde0 0x7f18e00cd7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:38.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18df200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f18e015b130 con 0x7f18e0158950 2024-03-06T07:30:38.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18df200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f18e00af170 con 0x7f18e00cfde0 2024-03-06T07:30:38.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18df200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18e015b3f0 con 0x7f18e0158570 2024-03-06T07:30:38.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18de800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f18e00cfde0 0x7f18e00cd7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:38.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18de800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f18e00cfde0 0x7f18e00cd7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55692/0 (socket says 172.21.15.86:55692) 2024-03-06T07:30:38.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18de800640 1 -- 172.21.15.86:0/1826024163 learned_addr learned my addr 172.21.15.86:0/1826024163 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:38.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18dde00640 1 --2- 172.21.15.86:0/1826024163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e0158570 0x7f18e00ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:38.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18de800640 1 -- 172.21.15.86:0/1826024163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e0158570 msgr2=0x7f18e00ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:38.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.534+0000 7f18de800640 1 --2- 172.21.15.86:0/1826024163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e0158570 0x7f18e00ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.535+0000 7f18de800640 1 -- 172.21.15.86:0/1826024163 >> v1:172.21.15.194:6789/0 conn(0x7f18e0158950 legacy=0x7f18e00cf6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:38.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.535+0000 7f18de800640 1 -- 172.21.15.86:0/1826024163 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18e015b6d0 con 0x7f18e00cfde0 2024-03-06T07:30:38.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.535+0000 7f18dde00640 1 --2- 172.21.15.86:0/1826024163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e0158570 0x7f18e00ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:38.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.536+0000 7f18de800640 1 --2- 172.21.15.86:0/1826024163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e00cfde0 0x7f18e00cd7c0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7f18d8009e50 tx=0x7f18d802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fd69da0c6b314330 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:38.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.536+0000 7f18dd400640 1 -- 172.21.15.86:0/1826024163 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d802ca80 con 0x7f18e00cfde0 2024-03-06T07:30:38.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.536+0000 7f18dd400640 1 -- 172.21.15.86:0/1826024163 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f18d802cc20 con 0x7f18e00cfde0 2024-03-06T07:30:38.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.536+0000 7f18dd400640 1 -- 172.21.15.86:0/1826024163 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d80343f0 con 0x7f18e00cfde0 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 -- 172.21.15.86:0/1826024163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e00cfde0 msgr2=0x7f18e00cd7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 --2- 172.21.15.86:0/1826024163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e00cfde0 0x7f18e00cd7c0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7f18d8009e50 tx=0x7f18d802bee0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 -- 172.21.15.86:0/1826024163 shutdown_connections 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 --2- 172.21.15.86:0/1826024163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e00cfde0 0x7f18e00cd7c0 unknown :-1 s=CLOSED pgs=795 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 --2- 172.21.15.86:0/1826024163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e0158570 0x7f18e00ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 -- 172.21.15.86:0/1826024163 >> 172.21.15.86:0/1826024163 conn(0x7f18e006afc0 msgr2=0x7f18e006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 -- 172.21.15.86:0/1826024163 shutdown_connections 2024-03-06T07:30:38.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.537+0000 7f18df200640 1 -- 172.21.15.86:0/1826024163 wait complete. 2024-03-06T07:30:38.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.539+0000 7f18df200640 1 Processor -- start 2024-03-06T07:30:38.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.539+0000 7f18df200640 1 -- start start 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18df200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00cfde0 0x7f18e015dd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18df200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e0158570 0x7f18e01e77d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18df200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e0158950 0x7f18e01e7d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18df200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18e015bed0 con 0x7f18e0158950 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18df200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18e00b4950 con 0x7f18e0158570 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18df200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18e015c050 con 0x7f18e00cfde0 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e0158950 0x7f18e01e7d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e0158950 0x7f18e01e7d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55694/0 (socket says 172.21.15.86:55694) 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 -- 172.21.15.86:0/1471994909 learned_addr learned my addr 172.21.15.86:0/1471994909 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18dde00640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00cfde0 0x7f18e015dd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18d7e00640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e0158570 0x7f18e01e77d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 -- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00cfde0 msgr2=0x7f18e015dd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00cfde0 0x7f18e015dd60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 -- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e0158570 msgr2=0x7f18e01e77d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:38.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e0158570 0x7f18e01e77d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18de800640 1 -- 172.21.15.86:0/1471994909 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18e015e2a0 con 0x7f18e0158950 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18d7e00640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e0158570 0x7f18e01e77d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.540+0000 7f18dde00640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00cfde0 0x7f18e015dd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.541+0000 7f18de800640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e0158950 0x7f18e01e7d10 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7f18d8036040 tx=0x7f18d8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.541+0000 7f18d6000640 1 -- 172.21.15.86:0/1471994909 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d8043020 con 0x7f18e0158950 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.541+0000 7f18d6000640 1 -- 172.21.15.86:0/1471994909 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f18d803e450 con 0x7f18e0158950 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.541+0000 7f18d6000640 1 -- 172.21.15.86:0/1471994909 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d8009510 con 0x7f18e0158950 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.541+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18e015c6b0 con 0x7f18e0158950 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.541+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18e015cc70 con 0x7f18e0158950 2024-03-06T07:30:38.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.542+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f18e004f330 con 0x7f18e0158950 2024-03-06T07:30:38.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.543+0000 7f18d6000640 1 -- 172.21.15.86:0/1471994909 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f18d8046020 con 0x7f18e0158950 2024-03-06T07:30:38.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.544+0000 7f18d6000640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:38.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.544+0000 7f18d6000640 1 -- 172.21.15.86:0/1471994909 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f18d80ace90 con 0x7f18e0158950 2024-03-06T07:30:38.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.545+0000 7f18dde00640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:38.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.545+0000 7f18dde00640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f18d0006fd0 tx=0x7f18d0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:38.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.551+0000 7f18d6000640 1 -- 172.21.15.86:0/1471994909 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f18d804d030 con 0x7f18e0158950 2024-03-06T07:30:38.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:38.789+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f18e00ced50 con 0x7f18e0158950 2024-03-06T07:30:39.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.313+0000 7f18d6000640 1 -- 172.21.15.86:0/1471994909 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v42) v1 ==== 122+0+0 (secure 0 0 0) 0x7f18d807bdf0 con 0x7f18e0158950 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.316+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 msgr2=0x7f18bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f18d0006fd0 tx=0x7f18d0008040 comp rx=0 tx=0).stop 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e0158950 msgr2=0x7f18e01e7d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e0158950 0x7f18e01e7d10 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7f18d8036040 tx=0x7f18d8004a40 comp rx=0 tx=0).stop 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 shutdown_connections 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18e0158950 0x7f18e01e7d10 unknown :-1 s=CLOSED pgs=796 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e0158570 0x7f18e01e77d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 --2- 172.21.15.86:0/1471994909 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00cfde0 0x7f18e015dd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 >> 172.21.15.86:0/1471994909 conn(0x7f18e006afc0 msgr2=0x7f18e0070a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 shutdown_connections 2024-03-06T07:30:39.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.317+0000 7f18df200640 1 -- 172.21.15.86:0/1471994909 wait complete. 2024-03-06T07:30:39.341 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:30:39.341 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:39.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.547+0000 7f1bae600640 1 Processor -- start 2024-03-06T07:30:39.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.547+0000 7f1bae600640 1 -- start start 2024-03-06T07:30:39.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.549+0000 7f1bae600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1ba80d24c0 0x7f1ba80d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:39.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.549+0000 7f1bae600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ba80d4c50 con 0x7f1ba80b2de0 2024-03-06T07:30:39.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.549+0000 7f1bae600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ba80cfa30 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.549+0000 7f1bae600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1ba80d4a00 con 0x7f1ba80d24c0 2024-03-06T07:30:39.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.549+0000 7f1ba7400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1ba80d24c0 0x7f1ba80d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:39.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.549+0000 7f1ba7400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1ba80d24c0 0x7f1ba80d1cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55706/0 (socket says 172.21.15.86:55706) 2024-03-06T07:30:39.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.549+0000 7f1ba7400640 1 -- 172.21.15.86:0/1990644048 learned_addr learned my addr 172.21.15.86:0/1990644048 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:39.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.550+0000 7f1ba7400640 1 -- 172.21.15.86:0/1990644048 >> v1:172.21.15.194:6789/0 conn(0x7f1ba80b0aa0 legacy=0x7f1ba80b0f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:39.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.550+0000 7f1ba7400640 1 -- 172.21.15.86:0/1990644048 >> v1:172.21.15.86:6789/0 conn(0x7f1ba80b2de0 legacy=0x7f1ba80d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.550+0000 7f1ba7400640 1 -- 172.21.15.86:0/1990644048 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ba80cfd10 con 0x7f1ba80d24c0 2024-03-06T07:30:39.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.550+0000 7f1ba6000640 1 -- 172.21.15.86:0/1990644048 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1529600420 0 0) 0x7f1ba80d4c50 con 0x7f1ba80b2de0 2024-03-06T07:30:39.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.552+0000 7f1ba7400640 1 --2- 172.21.15.86:0/1990644048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80d24c0 0x7f1ba80d1cc0 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7f1b98009870 tx=0x7f1b9802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc1c032d55e456 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:39.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.552+0000 7f1ba6000640 1 -- 172.21.15.86:0/1990644048 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b98035020 con 0x7f1ba80d24c0 2024-03-06T07:30:39.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.552+0000 7f1ba6000640 1 -- 172.21.15.86:0/1990644048 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b98031870 con 0x7f1ba80d24c0 2024-03-06T07:30:39.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.552+0000 7f1ba6000640 1 -- 172.21.15.86:0/1990644048 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b98031c40 con 0x7f1ba80d24c0 2024-03-06T07:30:39.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.553+0000 7f1bae600640 1 -- 172.21.15.86:0/1990644048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80d24c0 msgr2=0x7f1ba80d1cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.553+0000 7f1bae600640 1 --2- 172.21.15.86:0/1990644048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80d24c0 0x7f1ba80d1cc0 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7f1b98009870 tx=0x7f1b9802b910 comp rx=0 tx=0).stop 2024-03-06T07:30:39.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.553+0000 7f1bae600640 1 -- 172.21.15.86:0/1990644048 shutdown_connections 2024-03-06T07:30:39.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.553+0000 7f1bae600640 1 --2- 172.21.15.86:0/1990644048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80d24c0 0x7f1ba80d1cc0 unknown :-1 s=CLOSED pgs=798 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.553+0000 7f1bae600640 1 -- 172.21.15.86:0/1990644048 >> 172.21.15.86:0/1990644048 conn(0x7f1ba80d3ee0 msgr2=0x7f1ba80d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:39.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.553+0000 7f1bae600640 1 -- 172.21.15.86:0/1990644048 shutdown_connections 2024-03-06T07:30:39.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.553+0000 7f1bae600640 1 -- 172.21.15.86:0/1990644048 wait complete. 2024-03-06T07:30:39.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.555+0000 7f1bae600640 1 Processor -- start 2024-03-06T07:30:39.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.555+0000 7f1bae600640 1 -- start start 2024-03-06T07:30:39.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1bae600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80b0aa0 0x7f1ba814e560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1bae600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ba80b2de0 0x7f1ba80cccc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1bae600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ba80d24c0 0x7f1ba80cd200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1bae600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ba80d0080 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1bae600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ba80cf7e0 con 0x7f1ba80d24c0 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1bae600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ba80b1640 con 0x7f1ba80b2de0 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1ba7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80b0aa0 0x7f1ba814e560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1ba7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80b0aa0 0x7f1ba814e560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55718/0 (socket says 172.21.15.86:55718) 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1ba7400640 1 -- 172.21.15.86:0/2142864890 learned_addr learned my addr 172.21.15.86:0/2142864890 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1ba7400640 1 -- 172.21.15.86:0/2142864890 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ba80b2de0 msgr2=0x7f1ba80cccc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1ba7400640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ba80b2de0 0x7f1ba80cccc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1ba7400640 1 -- 172.21.15.86:0/2142864890 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ba80d24c0 msgr2=0x7f1ba80cd200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.556+0000 7f1ba7400640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ba80d24c0 0x7f1ba80cd200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.557+0000 7f1ba7400640 1 -- 172.21.15.86:0/2142864890 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ba814eaa0 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.557+0000 7f1ba7400640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80b0aa0 0x7f1ba814e560 secure :-1 s=READY pgs=799 cs=0 l=1 rev1=1 crypto rx=0x7f1b98009870 tx=0x7f1b980046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:39.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.557+0000 7f1b9f400640 1 -- 172.21.15.86:0/2142864890 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b98040070 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.557+0000 7f1b9f400640 1 -- 172.21.15.86:0/2142864890 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b9803a420 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.557+0000 7f1b9f400640 1 -- 172.21.15.86:0/2142864890 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b98035040 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.557+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ba815f3a0 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.557+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ba815f960 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.559+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1ba804f330 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.559+0000 7f1b9f400640 1 -- 172.21.15.86:0/2142864890 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1b9804b060 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.560+0000 7f1b9f400640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b8406b4d0 0x7f1b8406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:39.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.560+0000 7f1b9f400640 1 -- 172.21.15.86:0/2142864890 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1b980ac1e0 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.560+0000 7f1ba6a00640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b8406b4d0 0x7f1b8406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:39.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.561+0000 7f1ba6a00640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b8406b4d0 0x7f1b8406d990 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f1ba81f7350 tx=0x7f1b90007910 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:39.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.565+0000 7f1b9f400640 1 -- 172.21.15.86:0/2142864890 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1b9807afc0 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.862+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1ba80c96a0 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.863+0000 7f1b9f400640 1 -- 172.21.15.86:0/2142864890 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f1b9807a8c0 con 0x7f1ba80b0aa0 2024-03-06T07:30:39.865 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:39.865 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:30:39.865 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:30:39.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b8406b4d0 msgr2=0x7f1b8406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b8406b4d0 0x7f1b8406d990 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f1ba81f7350 tx=0x7f1b90007910 comp rx=0 tx=0).stop 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80b0aa0 msgr2=0x7f1ba814e560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80b0aa0 0x7f1ba814e560 secure :-1 s=READY pgs=799 cs=0 l=1 rev1=1 crypto rx=0x7f1b98009870 tx=0x7f1b980046c0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 shutdown_connections 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b8406b4d0 0x7f1b8406d990 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ba80d24c0 0x7f1ba80cd200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ba80b2de0 0x7f1ba80cccc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 --2- 172.21.15.86:0/2142864890 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ba80b0aa0 0x7f1ba814e560 unknown :-1 s=CLOSED pgs=799 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 >> 172.21.15.86:0/2142864890 conn(0x7f1ba80d3ee0 msgr2=0x7f1ba806cb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 shutdown_connections 2024-03-06T07:30:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:39.866+0000 7f1bae600640 1 -- 172.21.15.86:0/2142864890 wait complete. 2024-03-06T07:30:39.886 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:30:40.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.069+0000 7f6cf2000640 1 Processor -- start 2024-03-06T07:30:40.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.069+0000 7f6cf2000640 1 -- start start 2024-03-06T07:30:40.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.070+0000 7f6cf2000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 0x7f6cec0cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6cf2000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6cec0b2b30 con 0x7f6cec15a940 2024-03-06T07:30:40.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6cf2000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6cec0b2e10 con 0x7f6cec0cf370 2024-03-06T07:30:40.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6cf2000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6cec0b28e0 con 0x7f6cec0cc2a0 2024-03-06T07:30:40.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6cf0c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6cec15a940 0x7f6cec0b45e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54832/0 (socket says 172.21.15.86:54832) 2024-03-06T07:30:40.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2151048592 learned_addr learned my addr 172.21.15.86:0/2151048592 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2715368889 0 0) 0x7f6cec0b2b30 con 0x7f6cec15a940 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6cd0005730 con 0x7f6cec15a940 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.071+0000 7f6cf1600640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 0x7f6cec0cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3379868170 0 0) 0x7f6cd0005730 con 0x7f6cec15a940 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 msgr2=0x7f6cec0cc750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 0x7f6cec0cc750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 >> v1:172.21.15.194:6789/0 conn(0x7f6cec0cf370 legacy=0x7f6cec06d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cec06ca50 con 0x7f6cec15a940 2024-03-06T07:30:40.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6cdc001670 con 0x7f6cec15a940 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 >> v1:172.21.15.86:6789/0 conn(0x7f6cec15a940 legacy=0x7f6cec0b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cd0006010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 0x7f6cd0006a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cd0007020 0x7f6cd00074a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cec0b28e0 con 0x7f6cd0005bf0 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cd000de30 con 0x7f6cd0007020 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6cd000e030 con 0x7f6cec0cc2a0 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6cf0c00640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cd0006010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6cf1600640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cd0007020 0x7f6cd00074a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6cebe00640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 0x7f6cd0006a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 msgr2=0x7f6cd0006a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.072+0000 7f6cf0c00640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec0cc2a0 0x7f6cd0006a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.073+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cd0007020 msgr2=0x7f6cd00074a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:40.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.073+0000 7f6cf0c00640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cd0007020 0x7f6cd00074a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.073+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2151048592 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cd0005730 con 0x7f6cd0005bf0 2024-03-06T07:30:40.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.073+0000 7f6cf1600640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cd0007020 0x7f6cd00074a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:40.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.074+0000 7f6cf0c00640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cd0006010 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc00c0e0 tx=0x7f6cd0007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea1d626c35df797f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:40.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.074+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cdc007c20 con 0x7f6cd0005bf0 2024-03-06T07:30:40.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.074+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cdc030070 con 0x7f6cd0005bf0 2024-03-06T07:30:40.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.074+0000 7f6ceb400640 1 -- 172.21.15.86:0/2151048592 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cdc005d30 con 0x7f6cd0005bf0 2024-03-06T07:30:40.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.075+0000 7f6cf2000640 1 -- 172.21.15.86:0/2151048592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 msgr2=0x7f6cd0006010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.075+0000 7f6cf2000640 1 --2- 172.21.15.86:0/2151048592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cd0006010 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc00c0e0 tx=0x7f6cd0007a40 comp rx=0 tx=0).stop 2024-03-06T07:30:40.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.075+0000 7f6cf1600640 1 -- 172.21.15.86:0/2151048592 reap_dead start 2024-03-06T07:30:40.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.075+0000 7f6cf2000640 1 -- 172.21.15.86:0/2151048592 shutdown_connections 2024-03-06T07:30:40.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.075+0000 7f6cf2000640 1 -- 172.21.15.86:0/2151048592 >> 172.21.15.86:0/2151048592 conn(0x7f6cec06afc0 msgr2=0x7f6cec06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:40.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.075+0000 7f6cf2000640 1 -- 172.21.15.86:0/2151048592 shutdown_connections 2024-03-06T07:30:40.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.075+0000 7f6cf2000640 1 -- 172.21.15.86:0/2151048592 wait complete. 2024-03-06T07:30:40.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.077+0000 7f6cf2000640 1 Processor -- start 2024-03-06T07:30:40.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.077+0000 7f6cf2000640 1 -- start start 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf2000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cec151440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf2000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cec15c130 0x7f6cec15c5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf2000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec15d530 0x7f6cec15d9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf2000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cec15ad20 con 0x7f6cd0005bf0 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf2000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cec0b30f0 con 0x7f6cec15c130 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf2000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6cec15aea0 con 0x7f6cec15d530 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cec151440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cec151440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55736/0 (socket says 172.21.15.86:55736) 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2370121938 learned_addr learned my addr 172.21.15.86:0/2370121938 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:40.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2370121938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec15d530 msgr2=0x7f6cec15d9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf0c00640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec15d530 0x7f6cec15d9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2370121938 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cec15c130 msgr2=0x7f6cec15c5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.078+0000 7f6cebe00640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cec15c130 0x7f6cec15c5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6cf0c00640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cec15c130 0x7f6cec15c5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6cf0c00640 1 -- 172.21.15.86:0/2370121938 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cec151980 con 0x7f6cd0005bf0 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6cf0c00640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cec151440 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc00c0e0 tx=0x7f6cec0b2fe0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6ce8c00640 1 -- 172.21.15.86:0/2370121938 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cdc004bf0 con 0x7f6cd0005bf0 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6ce8c00640 1 -- 172.21.15.86:0/2370121938 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cdc000ed0 con 0x7f6cd0005bf0 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6ce8c00640 1 -- 172.21.15.86:0/2370121938 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cdc007070 con 0x7f6cd0005bf0 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6cec15b370 con 0x7f6cd0005bf0 2024-03-06T07:30:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.079+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6cec15b930 con 0x7f6cd0005bf0 2024-03-06T07:30:40.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.081+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6cec04f330 con 0x7f6cd0005bf0 2024-03-06T07:30:40.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.081+0000 7f6ce8c00640 1 -- 172.21.15.86:0/2370121938 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6cdc047020 con 0x7f6cd0005bf0 2024-03-06T07:30:40.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.082+0000 7f6ce8c00640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cc806b4d0 0x7f6cc806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.082+0000 7f6ce8c00640 1 -- 172.21.15.86:0/2370121938 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6cdc0ab3f0 con 0x7f6cd0005bf0 2024-03-06T07:30:40.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.082+0000 7f6cebe00640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cc806b4d0 0x7f6cc806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.083+0000 7f6cebe00640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cc806b4d0 0x7f6cc806d990 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f6cec0ae420 tx=0x7f6cd4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.086+0000 7f6ce8c00640 1 -- 172.21.15.86:0/2370121938 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6cec151980 con 0x7f6cd0005bf0 2024-03-06T07:30:40.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.274+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6cec0ccc10 con 0x7f6cd0005bf0 2024-03-06T07:30:40.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.276+0000 7f6ce8c00640 1 -- 172.21.15.86:0/2370121938 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v70) v1 ==== 74+0+16680 (secure 0 0 0) 0x7f6cdc07a270 con 0x7f6cd0005bf0 2024-03-06T07:30:40.278 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:40.278 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":70,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:37.288921+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:30:33.566852+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-06T07:30:34.722690+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cc806b4d0 msgr2=0x7f6cc806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cc806b4d0 0x7f6cc806d990 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f6cec0ae420 tx=0x7f6cd4009040 comp rx=0 tx=0).stop 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 msgr2=0x7f6cec151440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cec151440 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc00c0e0 tx=0x7f6cec0b2fe0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 shutdown_connections 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cc806b4d0 0x7f6cc806d990 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cec15d530 0x7f6cec15d9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cec15c130 0x7f6cec15c5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 --2- 172.21.15.86:0/2370121938 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cd0005bf0 0x7f6cec151440 unknown :-1 s=CLOSED pgs=802 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 >> 172.21.15.86:0/2370121938 conn(0x7f6cec06afc0 msgr2=0x7f6cec15e9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 shutdown_connections 2024-03-06T07:30:40.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.279+0000 7f6cf2000640 1 -- 172.21.15.86:0/2370121938 wait complete. 2024-03-06T07:30:40.306 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:30:40.306 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:30:40.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.493+0000 7f51b5400640 1 Processor -- start 2024-03-06T07:30:40.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.494+0000 7f51b5400640 1 -- start start 2024-03-06T07:30:40.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.495+0000 7f51b5400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f51b015fe60 0x7f51b0071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.495+0000 7f51b5400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f51b006cef0 con 0x7f51b0144c00 2024-03-06T07:30:40.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.495+0000 7f51b5400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f51b006d1d0 con 0x7f51b015f9d0 2024-03-06T07:30:40.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.495+0000 7f51b5400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f51b006d4b0 con 0x7f51b015fe60 2024-03-06T07:30:40.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ae800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f51b0144c00 0x7f51b006e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54834/0 (socket says 172.21.15.86:54834) 2024-03-06T07:30:40.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ae800640 1 -- 172.21.15.86:0/67582251 learned_addr learned my addr 172.21.15.86:0/67582251 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:40.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51af200640 1 --2- 172.21.15.86:0/67582251 >> v2:172.21.15.194:3300/0 conn(0x7f51b015fe60 0x7f51b0071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1327799307 0 0) 0x7f51b006cef0 con 0x7f51b0144c00 2024-03-06T07:30:40.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5190003520 con 0x7f51b0144c00 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1766905116 0 0) 0x7f5190003520 con 0x7f51b0144c00 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 >> v2:172.21.15.194:3300/0 conn(0x7f51b015fe60 msgr2=0x7f51b0071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ade00640 1 --2- 172.21.15.86:0/67582251 >> v2:172.21.15.194:3300/0 conn(0x7f51b015fe60 0x7f51b0071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.496+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 >> v1:172.21.15.194:6789/0 conn(0x7f51b015f9d0 legacy=0x7f51b0070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51b006d790 con 0x7f51b0144c00 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f51a0001670 con 0x7f51b0144c00 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 >> v1:172.21.15.86:6789/0 conn(0x7f51b0144c00 legacy=0x7f51b006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51900039b0 0x7f5190003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5190004800 0x7f5190004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f519000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51b006d4b0 con 0x7f51900051c0 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f519000c060 con 0x7f51900039b0 2024-03-06T07:30:40.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f519000c260 con 0x7f5190004800 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51af200640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f519000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ae800640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51900039b0 0x7f5190003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51a5e00640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5190004800 0x7f5190004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51af200640 1 -- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5190004800 msgr2=0x7f5190004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51af200640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5190004800 0x7f5190004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51af200640 1 -- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51900039b0 msgr2=0x7f5190003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51af200640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51900039b0 0x7f5190003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51af200640 1 -- 172.21.15.86:0/67582251 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5190003520 con 0x7f51900051c0 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51ae800640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51900039b0 0x7f5190003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.497+0000 7f51a5e00640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5190004800 0x7f5190004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.498+0000 7f51af200640 1 -- 172.21.15.86:0/67582251 reap_dead start 2024-03-06T07:30:40.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.498+0000 7f51af200640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f519000b9c0 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7f51a8009fc0 tx=0x7f5190005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c6628d83c6f5956 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:40.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51a8009c10 con 0x7f51900051c0 2024-03-06T07:30:40.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f51a802cc30 con 0x7f51900051c0 2024-03-06T07:30:40.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51ade00640 1 -- 172.21.15.86:0/67582251 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51a8003990 con 0x7f51900051c0 2024-03-06T07:30:40.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51b5400640 1 -- 172.21.15.86:0/67582251 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 msgr2=0x7f519000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51b5400640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f519000b9c0 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7f51a8009fc0 tx=0x7f5190005830 comp rx=0 tx=0).stop 2024-03-06T07:30:40.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51b5400640 1 -- 172.21.15.86:0/67582251 shutdown_connections 2024-03-06T07:30:40.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51b5400640 1 --2- 172.21.15.86:0/67582251 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f519000b9c0 unknown :-1 s=CLOSED pgs=804 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.499+0000 7f51b5400640 1 -- 172.21.15.86:0/67582251 >> 172.21.15.86:0/67582251 conn(0x7f51b00b1730 msgr2=0x7f51b00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:40.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.500+0000 7f51b5400640 1 -- 172.21.15.86:0/67582251 shutdown_connections 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.500+0000 7f51b5400640 1 -- 172.21.15.86:0/67582251 wait complete. 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.501+0000 7f51b5400640 1 Processor -- start 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.502+0000 7f51b5400640 1 -- start start 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51b5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f51b015b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51b5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f51b00afe10 0x7f51b00b0290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51b5400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51b01f7430 0x7f51b01f7890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51b5400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51b006cb50 con 0x7f51900051c0 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51b5400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51b0160730 con 0x7f51b01f7430 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51b5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f51b006ccd0 con 0x7f51b00afe10 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f51b015b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f51b015b760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55754/0 (socket says 172.21.15.86:55754) 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 -- 172.21.15.86:0/3501683783 learned_addr learned my addr 172.21.15.86:0/3501683783 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:40.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51a5e00640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f51b00afe10 0x7f51b00b0290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 -- 172.21.15.86:0/3501683783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f51b00afe10 msgr2=0x7f51b00b0290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f51b00afe10 0x7f51b00b0290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 -- 172.21.15.86:0/3501683783 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51b01f7430 msgr2=0x7f51b01f7890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51b01f7430 0x7f51b01f7890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.503+0000 7f51ae800640 1 -- 172.21.15.86:0/3501683783 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51b01f3dd0 con 0x7f51900051c0 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.504+0000 7f51ae800640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f51b015b760 secure :-1 s=READY pgs=805 cs=0 l=1 rev1=1 crypto rx=0x7f51a000d810 tx=0x7f51a000dcd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.504+0000 7f51a7400640 1 -- 172.21.15.86:0/3501683783 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51a0019070 con 0x7f51900051c0 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.504+0000 7f51a7400640 1 -- 172.21.15.86:0/3501683783 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f51a00074e0 con 0x7f51900051c0 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.504+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f51b01f40b0 con 0x7f51900051c0 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.504+0000 7f51a7400640 1 -- 172.21.15.86:0/3501683783 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51a00049e0 con 0x7f51900051c0 2024-03-06T07:30:40.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.504+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f51b01f3ac0 con 0x7f51900051c0 2024-03-06T07:30:40.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.506+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f51b00b07d0 con 0x7f51900051c0 2024-03-06T07:30:40.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.506+0000 7f51a7400640 1 -- 172.21.15.86:0/3501683783 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f51a001c070 con 0x7f51900051c0 2024-03-06T07:30:40.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.508+0000 7f51a7400640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f518c06b4d0 0x7f518c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:40.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.508+0000 7f51a7400640 1 -- 172.21.15.86:0/3501683783 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f51a0089da0 con 0x7f51900051c0 2024-03-06T07:30:40.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.508+0000 7f51a5e00640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f518c06b4d0 0x7f518c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:40.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.509+0000 7f51a5e00640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f518c06b4d0 0x7f518c06d990 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f5198000f60 tx=0x7f5198009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:40.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.511+0000 7f51a7400640 1 -- 172.21.15.86:0/3501683783 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f51a0058b80 con 0x7f51900051c0 2024-03-06T07:30:40.657 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:30:40.656+0000 7ff455200640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:30:40.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.702+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f51b004f330 con 0x7f518c06b4d0 2024-03-06T07:30:40.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.719+0000 7f51a7400640 1 -- 172.21.15.86:0/3501683783 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+240186 (secure 0 0 0) 0x7f51b004f330 con 0x7f518c06b4d0 2024-03-06T07:30:40.725 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:40.727 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f518c06b4d0 msgr2=0x7f518c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f518c06b4d0 0x7f518c06d990 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f5198000f60 tx=0x7f5198009040 comp rx=0 tx=0).stop 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 msgr2=0x7f51b015b760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f51b015b760 secure :-1 s=READY pgs=805 cs=0 l=1 rev1=1 crypto rx=0x7f51a000d810 tx=0x7f51a000dcd0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 shutdown_connections 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f518c06b4d0 0x7f518c06d990 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f51b01f7430 0x7f51b01f7890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f51b00afe10 0x7f51b00b0290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 --2- 172.21.15.86:0/3501683783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f51900051c0 0x7f51b015b760 unknown :-1 s=CLOSED pgs=805 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:40.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 >> 172.21.15.86:0/3501683783 conn(0x7f51b00b1730 msgr2=0x7f51b00aecc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:40.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.724+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 shutdown_connections 2024-03-06T07:30:40.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.725+0000 7f51b5400640 1 -- 172.21.15.86:0/3501683783 wait complete. 2024-03-06T07:30:40.748 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":173,"stamp":"2024-03-06T07:30:38.915608+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":66,"acting":66,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":256932,"kb_used_data":7472,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754717788,"statfs":{"total":773094113280,"available":772831014912,"internally_reserved":0,"allocated":7651328,"data_stored":3852462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.665148"},"pg_stats":[{"pgid":"11.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.3e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.31","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.25","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"11.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.1f","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.263154+0000","last_change":"2024-03-06T07:30:34.263154+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.263154+0000","last_undegraded":"2024-03-06T07:30:34.263154+0000","last_fullsized":"2024-03-06T07:30:34.263154+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.1e","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.255692+0000","last_change":"2024-03-06T07:30:34.255692+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.255692+0000","last_undegraded":"2024-03-06T07:30:34.255692+0000","last_fullsized":"2024-03-06T07:30:34.255692+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.1d","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.255593+0000","last_change":"2024-03-06T07:30:34.255593+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.255593+0000","last_undegraded":"2024-03-06T07:30:34.255593+0000","last_fullsized":"2024-03-06T07:30:34.255593+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.1c","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.263462+0000","last_change":"2024-03-06T07:30:34.263462+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.263462+0000","last_undegraded":"2024-03-06T07:30:34.263462+0000","last_fullsized":"2024-03-06T07:30:34.263462+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.1b","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.259963+0000","last_change":"2024-03-06T07:30:34.259963+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.259963+0000","last_undegraded":"2024-03-06T07:30:34.259963+0000","last_fullsized":"2024-03-06T07:30:34.259963+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.1a","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.270327+0000","last_change":"2024-03-06T07:30:34.270327+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.270327+0000","last_undegraded":"2024-03-06T07:30:34.270327+0000","last_fullsized":"2024-03-06T07:30:34.270327+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.19","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.263158+0000","last_change":"2024-03-06T07:30:34.263158+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.263158+0000","last_undegraded":"2024-03-06T07:30:34.263158+0000","last_fullsized":"2024-03-06T07:30:34.263158+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.18","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.258692+0000","last_change":"2024-03-06T07:30:34.258692+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.258692+0000","last_undegraded":"2024-03-06T07:30:34.258692+0000","last_fullsized":"2024-03-06T07:30:34.258692+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.17","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.255796+0000","last_change":"2024-03-06T07:30:34.255796+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.255796+0000","last_undegraded":"2024-03-06T07:30:34.255796+0000","last_fullsized":"2024-03-06T07:30:34.255796+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.16","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.264673+0000","last_change":"2024-03-06T07:30:34.264673+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.264673+0000","last_undegraded":"2024-03-06T07:30:34.264673+0000","last_fullsized":"2024-03-06T07:30:34.264673+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.15","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.259856+0000","last_change":"2024-03-06T07:30:34.259856+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.259856+0000","last_undegraded":"2024-03-06T07:30:34.259856+0000","last_fullsized":"2024-03-06T07:30:34.259856+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.14","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.263405+0000","last_change":"2024-03-06T07:30:34.263405+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.263405+0000","last_undegraded":"2024-03-06T07:30:34.263405+0000","last_fullsized":"2024-03-06T07:30:34.263405+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.13","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.262981+0000","last_change":"2024-03-06T07:30:34.262981+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.262981+0000","last_undegraded":"2024-03-06T07:30:34.262981+0000","last_fullsized":"2024-03-06T07:30:34.262981+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.12","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.269281+0000","last_change":"2024-03-06T07:30:34.269281+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.269281+0000","last_undegraded":"2024-03-06T07:30:34.269281+0000","last_fullsized":"2024-03-06T07:30:34.269281+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.11","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.266509+0000","last_change":"2024-03-06T07:30:34.266509+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.266509+0000","last_undegraded":"2024-03-06T07:30:34.266509+0000","last_fullsized":"2024-03-06T07:30:34.266509+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.10","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.255653+0000","last_change":"2024-03-06T07:30:34.255653+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.255653+0000","last_undegraded":"2024-03-06T07:30:34.255653+0000","last_fullsized":"2024-03-06T07:30:34.255653+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.267028+0000","last_change":"2024-03-06T07:30:34.267028+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.267028+0000","last_undegraded":"2024-03-06T07:30:34.267028+0000","last_fullsized":"2024-03-06T07:30:34.267028+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.256493+0000","last_change":"2024-03-06T07:30:34.256493+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.256493+0000","last_undegraded":"2024-03-06T07:30:34.256493+0000","last_fullsized":"2024-03-06T07:30:34.256493+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.263118+0000","last_change":"2024-03-06T07:30:34.263118+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.263118+0000","last_undegraded":"2024-03-06T07:30:34.263118+0000","last_fullsized":"2024-03-06T07:30:34.263118+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.262049+0000","last_change":"2024-03-06T07:30:34.262049+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.262049+0000","last_undegraded":"2024-03-06T07:30:34.262049+0000","last_fullsized":"2024-03-06T07:30:34.262049+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.264901+0000","last_change":"2024-03-06T07:30:34.264901+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.264901+0000","last_undegraded":"2024-03-06T07:30:34.264901+0000","last_fullsized":"2024-03-06T07:30:34.264901+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.265427+0000","last_change":"2024-03-06T07:30:34.265427+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.265427+0000","last_undegraded":"2024-03-06T07:30:34.265427+0000","last_fullsized":"2024-03-06T07:30:34.265427+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.255752+0000","last_change":"2024-03-06T07:30:34.255752+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.255752+0000","last_undegraded":"2024-03-06T07:30:34.255752+0000","last_fullsized":"2024-03-06T07:30:34.255752+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.264883+0000","last_change":"2024-03-06T07:30:34.264883+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.264883+0000","last_undegraded":"2024-03-06T07:30:34.264883+0000","last_fullsized":"2024-03-06T07:30:34.264883+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.266768+0000","last_change":"2024-03-06T07:30:34.266768+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.266768+0000","last_undegraded":"2024-03-06T07:30:34.266768+0000","last_fullsized":"2024-03-06T07:30:34.266768+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.258973+0000","last_change":"2024-03-06T07:30:34.258973+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.258973+0000","last_undegraded":"2024-03-06T07:30:34.258973+0000","last_fullsized":"2024-03-06T07:30:34.258973+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.255723+0000","last_change":"2024-03-06T07:30:34.255723+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.255723+0000","last_undegraded":"2024-03-06T07:30:34.255723+0000","last_fullsized":"2024-03-06T07:30:34.255723+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.258983+0000","last_change":"2024-03-06T07:30:34.258983+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.258983+0000","last_undegraded":"2024-03-06T07:30:34.258983+0000","last_fullsized":"2024-03-06T07:30:34.258983+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.262984+0000","last_change":"2024-03-06T07:30:34.262984+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.262984+0000","last_undegraded":"2024-03-06T07:30:34.262984+0000","last_fullsized":"2024-03-06T07:30:34.262984+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.255842+0000","last_change":"2024-03-06T07:30:34.255842+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.255842+0000","last_undegraded":"2024-03-06T07:30:34.255842+0000","last_fullsized":"2024-03-06T07:30:34.255842+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.262222+0000","last_change":"2024-03-06T07:30:34.262222+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.262222+0000","last_undegraded":"2024-03-06T07:30:34.262222+0000","last_fullsized":"2024-03-06T07:30:34.262222+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:30:35.250081+0000","last_change":"2024-03-06T07:30:35.250081+0000","last_active":"2024-03-06T07:30:35.250081+0000","last_peered":"2024-03-06T07:30:35.250081+0000","last_clean":"2024-03-06T07:30:35.250081+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:35.250081+0000","last_undegraded":"2024-03-06T07:30:35.250081+0000","last_fullsized":"2024-03-06T07:30:35.250081+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":1,"reported_epoch":67,"state":"creating+peering","last_fresh":"2024-03-06T07:30:34.262434+0000","last_change":"2024-03-06T07:30:34.262434+0000","last_active":"2024-03-06T07:30:34.235003+0000","last_peered":"2024-03-06T07:30:34.235003+0000","last_clean":"2024-03-06T07:30:34.235003+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:30:34.262434+0000","last_undegraded":"2024-03-06T07:30:34.262434+0000","last_fullsized":"2024-03-06T07:30:34.262434+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":250,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-03-06T07:30:34.267597+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:30:34.267597+0000","last_peered":"2024-03-06T07:30:34.267597+0000","last_clean":"2024-03-06T07:30:34.267597+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:30:34.267597+0000","last_undegraded":"2024-03-06T07:30:34.267597+0000","last_fullsized":"2024-03-06T07:30:34.267597+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":10,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542191,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32556,"kb_used_data":1372,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339284,"statfs":{"total":96636764160,"available":96603426816,"internally_reserved":0,"allocated":1404928,"data_stored":925788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542190,"num_pgs":10,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31972,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339868,"statfs":{"total":96636764160,"available":96604024832,"internally_reserved":0,"allocated":811008,"data_stored":335420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542191,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31972,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339868,"statfs":{"total":96636764160,"available":96604024832,"internally_reserved":0,"allocated":811008,"data_stored":335420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542190,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31972,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339868,"statfs":{"total":96636764160,"available":96604024832,"internally_reserved":0,"allocated":811008,"data_stored":335420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542190,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31960,"kb_used_data":780,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339880,"statfs":{"total":96636764160,"available":96604037120,"internally_reserved":0,"allocated":798720,"data_stored":329603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542191,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31976,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339864,"statfs":{"total":96636764160,"available":96604020736,"internally_reserved":0,"allocated":811008,"data_stored":335420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542192,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31976,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339864,"statfs":{"total":96636764160,"available":96604020736,"internally_reserved":0,"allocated":811008,"data_stored":335420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542189,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32548,"kb_used_data":1360,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339292,"statfs":{"total":96636764160,"available":96603435008,"internally_reserved":0,"allocated":1392640,"data_stored":919971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:30:40.752 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:30:40.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.993+0000 7ff59a000640 1 Processor -- start 2024-03-06T07:30:40.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.993+0000 7ff59a000640 1 -- start start 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.996+0000 7ff59a000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff5940d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.996+0000 7ff59a000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff5940cea60 con 0x7ff59415f470 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.996+0000 7ff59a000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff5940d2350 con 0x7ff5940d0fc0 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.996+0000 7ff59a000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5940d2f60 con 0x7ff59415e0d0 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.996+0000 7ff592a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff59415f470 0x7ff5940d08b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54840/0 (socket says 172.21.15.86:54840) 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.996+0000 7ff592a00640 1 -- 172.21.15.86:0/3434391154 learned_addr learned my addr 172.21.15.86:0/3434391154 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.996+0000 7ff593400640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff5940d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 501102378 0 0) 0x7ff5940cea60 con 0x7ff59415f470 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff578003520 con 0x7ff59415f470 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4031198034 0 0) 0x7ff578003520 con 0x7ff59415f470 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 msgr2=0x7ff5940d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff5940d1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 >> v1:172.21.15.194:6789/0 conn(0x7ff5940d0fc0 legacy=0x7ff594160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5940d2630 con 0x7ff59415f470 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff580001670 con 0x7ff59415f470 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 >> v1:172.21.15.86:6789/0 conn(0x7ff59415f470 legacy=0x7ff5940d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5780039e0 0x7ff578003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff578004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff578004d40 0x7ff5780051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff57800bb00 con 0x7ff5780039e0 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff57800bc80 con 0x7ff578004d40 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.997+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff57800be80 con 0x7ff59415e0d0 2024-03-06T07:30:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff593400640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5780039e0 0x7ff578003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff592a00640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff578004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff593e00640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff578004d40 0x7ff5780051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff592a00640 1 -- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff578004d40 msgr2=0x7ff5780051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff592a00640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff578004d40 0x7ff5780051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff592a00640 1 -- 172.21.15.86:0/3434391154 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5780039e0 msgr2=0x7ff578003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff592a00640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5780039e0 0x7ff578003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff592a00640 1 -- 172.21.15.86:0/3434391154 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff578003520 con 0x7ff59415e0d0 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:40.998+0000 7ff593400640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5780039e0 0x7ff578003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:30:41.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.000+0000 7ff592a00640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff578004800 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7ff580001eb0 tx=0x7ff578005830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=69b596dafab160fb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.000+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff580007d30 con 0x7ff59415e0d0 2024-03-06T07:30:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.000+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff580030070 con 0x7ff59415e0d0 2024-03-06T07:30:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.000+0000 7ff592000640 1 -- 172.21.15.86:0/3434391154 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff580005d30 con 0x7ff59415e0d0 2024-03-06T07:30:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.001+0000 7ff59a000640 1 -- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 msgr2=0x7ff578004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.001+0000 7ff59a000640 1 --2- 172.21.15.86:0/3434391154 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff578004800 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7ff580001eb0 tx=0x7ff578005830 comp rx=0 tx=0).stop 2024-03-06T07:30:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.001+0000 7ff593e00640 1 -- 172.21.15.86:0/3434391154 reap_dead start 2024-03-06T07:30:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.001+0000 7ff59a000640 1 -- 172.21.15.86:0/3434391154 shutdown_connections 2024-03-06T07:30:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.001+0000 7ff59a000640 1 -- 172.21.15.86:0/3434391154 >> 172.21.15.86:0/3434391154 conn(0x7ff5940d3ec0 msgr2=0x7ff5940d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.001+0000 7ff59a000640 1 -- 172.21.15.86:0/3434391154 shutdown_connections 2024-03-06T07:30:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.001+0000 7ff59a000640 1 -- 172.21.15.86:0/3434391154 wait complete. 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.003+0000 7ff59a000640 1 Processor -- start 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.003+0000 7ff59a000640 1 -- start start 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.004+0000 7ff59a000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff5940c99b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.004+0000 7ff59a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5941551c0 0x7ff5940b9560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.004+0000 7ff59a000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5940c9ef0 0x7ff594148090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.004+0000 7ff59a000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5940cff10 con 0x7ff5941551c0 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.004+0000 7ff59a000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5940d2bf0 con 0x7ff5940c9ef0 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.004+0000 7ff59a000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff59415fb30 con 0x7ff59415e0d0 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5941551c0 0x7ff5940b9560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5941551c0 0x7ff5940b9560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55772/0 (socket says 172.21.15.86:55772) 2024-03-06T07:30:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 -- 172.21.15.86:0/4189163651 learned_addr learned my addr 172.21.15.86:0/4189163651 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff593400640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff5940c99b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 -- 172.21.15.86:0/4189163651 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 msgr2=0x7ff5940c99b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff5940c99b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 -- 172.21.15.86:0/4189163651 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5940c9ef0 msgr2=0x7ff594148090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5940c9ef0 0x7ff594148090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 -- 172.21.15.86:0/4189163651 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5940c8910 con 0x7ff5941551c0 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.005+0000 7ff592a00640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5941551c0 0x7ff5940b9560 secure :-1 s=READY pgs=807 cs=0 l=1 rev1=1 crypto rx=0x7ff580001970 tx=0x7ff580001a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.006+0000 7ff58b400640 1 -- 172.21.15.86:0/4189163651 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff580045070 con 0x7ff5941551c0 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.006+0000 7ff58b400640 1 -- 172.21.15.86:0/4189163651 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff580004bf0 con 0x7ff5941551c0 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.006+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff5940c8bf0 con 0x7ff5941551c0 2024-03-06T07:30:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.006+0000 7ff58b400640 1 -- 172.21.15.86:0/4189163651 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff580007d10 con 0x7ff5941551c0 2024-03-06T07:30:41.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.006+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff5940c91b0 con 0x7ff5941551c0 2024-03-06T07:30:41.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.007+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff59404f330 con 0x7ff5941551c0 2024-03-06T07:30:41.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.008+0000 7ff58b400640 1 -- 172.21.15.86:0/4189163651 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff580007070 con 0x7ff5941551c0 2024-03-06T07:30:41.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.009+0000 7ff58b400640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff57406b4d0 0x7ff57406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.009+0000 7ff58b400640 1 -- 172.21.15.86:0/4189163651 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff5800ab580 con 0x7ff5941551c0 2024-03-06T07:30:41.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.009+0000 7ff593400640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff57406b4d0 0x7ff57406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.010+0000 7ff593400640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff57406b4d0 0x7ff57406d990 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7ff584009ca0 tx=0x7ff584008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:41.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.013+0000 7ff58b400640 1 -- 172.21.15.86:0/4189163651 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff58007a3e0 con 0x7ff5941551c0 2024-03-06T07:30:41.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.283+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7ff594068500 con 0x7ff57406b4d0 2024-03-06T07:30:41.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.293+0000 7ff58b400640 1 -- 172.21.15.86:0/4189163651 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252069 (secure 0 0 0) 0x7ff584009cd0 con 0x7ff57406b4d0 2024-03-06T07:30:41.297 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:41.298 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:30:41.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff57406b4d0 msgr2=0x7ff57406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff57406b4d0 0x7ff57406d990 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7ff584009ca0 tx=0x7ff584008040 comp rx=0 tx=0).stop 2024-03-06T07:30:41.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5941551c0 msgr2=0x7ff5940b9560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5941551c0 0x7ff5940b9560 secure :-1 s=READY pgs=807 cs=0 l=1 rev1=1 crypto rx=0x7ff580001970 tx=0x7ff580001a40 comp rx=0 tx=0).stop 2024-03-06T07:30:41.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 shutdown_connections 2024-03-06T07:30:41.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff57406b4d0 0x7ff57406d990 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5940c9ef0 0x7ff594148090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5941551c0 0x7ff5940b9560 unknown :-1 s=CLOSED pgs=807 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 --2- 172.21.15.86:0/4189163651 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff59415e0d0 0x7ff5940c99b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 >> 172.21.15.86:0/4189163651 conn(0x7ff5940d3ec0 msgr2=0x7ff5940cf630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:41.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 shutdown_connections 2024-03-06T07:30:41.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.297+0000 7ff59a000640 1 -- 172.21.15.86:0/4189163651 wait complete. 2024-03-06T07:30:41.321 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":174,"stamp":"2024-03-06T07:30:40.916091+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":180,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":257312,"kb_used_data":7848,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754717408,"statfs":{"total":773094113280,"available":772830625792,"internally_reserved":0,"allocated":8036352,"data_stored":4023716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":32,"apply_latency_ms":32,"commit_latency_ns":32000000,"apply_latency_ns":32000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001646"},"pg_stats":[{"pgid":"11.3f","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536646+0000","last_change":"2024-03-06T07:30:36.309273+0000","last_active":"2024-03-06T07:30:37.536646+0000","last_peered":"2024-03-06T07:30:37.536646+0000","last_clean":"2024-03-06T07:30:37.536646+0000","last_became_active":"2024-03-06T07:30:36.308931+0000","last_became_peered":"2024-03-06T07:30:36.308931+0000","last_unstale":"2024-03-06T07:30:37.536646+0000","last_undegraded":"2024-03-06T07:30:37.536646+0000","last_fullsized":"2024-03-06T07:30:37.536646+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:26:09.252177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.3e","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.537283+0000","last_change":"2024-03-06T07:30:36.315631+0000","last_active":"2024-03-06T07:30:37.537283+0000","last_peered":"2024-03-06T07:30:37.537283+0000","last_clean":"2024-03-06T07:30:37.537283+0000","last_became_active":"2024-03-06T07:30:36.315333+0000","last_became_peered":"2024-03-06T07:30:36.315333+0000","last_unstale":"2024-03-06T07:30:37.537283+0000","last_undegraded":"2024-03-06T07:30:37.537283+0000","last_fullsized":"2024-03-06T07:30:37.537283+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:51.389968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.3d","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533443+0000","last_change":"2024-03-06T07:30:36.303296+0000","last_active":"2024-03-06T07:30:37.533443+0000","last_peered":"2024-03-06T07:30:37.533443+0000","last_clean":"2024-03-06T07:30:37.533443+0000","last_became_active":"2024-03-06T07:30:36.303008+0000","last_became_peered":"2024-03-06T07:30:36.303008+0000","last_unstale":"2024-03-06T07:30:37.533443+0000","last_undegraded":"2024-03-06T07:30:37.533443+0000","last_fullsized":"2024-03-06T07:30:37.533443+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:13.173259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.3c","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533321+0000","last_change":"2024-03-06T07:30:36.311037+0000","last_active":"2024-03-06T07:30:37.533321+0000","last_peered":"2024-03-06T07:30:37.533321+0000","last_clean":"2024-03-06T07:30:37.533321+0000","last_became_active":"2024-03-06T07:30:36.310864+0000","last_became_peered":"2024-03-06T07:30:36.310864+0000","last_unstale":"2024-03-06T07:30:37.533321+0000","last_undegraded":"2024-03-06T07:30:37.533321+0000","last_fullsized":"2024-03-06T07:30:37.533321+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:24:04.326478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3b","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301412+0000","last_change":"2024-03-06T07:30:36.292441+0000","last_active":"2024-03-06T07:30:37.301412+0000","last_peered":"2024-03-06T07:30:37.301412+0000","last_clean":"2024-03-06T07:30:37.301412+0000","last_became_active":"2024-03-06T07:30:36.292115+0000","last_became_peered":"2024-03-06T07:30:36.292115+0000","last_unstale":"2024-03-06T07:30:37.301412+0000","last_undegraded":"2024-03-06T07:30:37.301412+0000","last_fullsized":"2024-03-06T07:30:37.301412+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:29:56.357683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3a","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535710+0000","last_change":"2024-03-06T07:30:36.315808+0000","last_active":"2024-03-06T07:30:37.535710+0000","last_peered":"2024-03-06T07:30:37.535710+0000","last_clean":"2024-03-06T07:30:37.535710+0000","last_became_active":"2024-03-06T07:30:36.315501+0000","last_became_peered":"2024-03-06T07:30:36.315501+0000","last_unstale":"2024-03-06T07:30:37.535710+0000","last_undegraded":"2024-03-06T07:30:37.535710+0000","last_fullsized":"2024-03-06T07:30:37.535710+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:22:40.167569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.39","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327136+0000","last_change":"2024-03-06T07:30:36.302808+0000","last_active":"2024-03-06T07:30:37.327136+0000","last_peered":"2024-03-06T07:30:37.327136+0000","last_clean":"2024-03-06T07:30:37.327136+0000","last_became_active":"2024-03-06T07:30:36.302421+0000","last_became_peered":"2024-03-06T07:30:36.302421+0000","last_unstale":"2024-03-06T07:30:37.327136+0000","last_undegraded":"2024-03-06T07:30:37.327136+0000","last_fullsized":"2024-03-06T07:30:37.327136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:37:03.954925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.38","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535626+0000","last_change":"2024-03-06T07:30:36.304241+0000","last_active":"2024-03-06T07:30:37.535626+0000","last_peered":"2024-03-06T07:30:37.535626+0000","last_clean":"2024-03-06T07:30:37.535626+0000","last_became_active":"2024-03-06T07:30:36.303958+0000","last_became_peered":"2024-03-06T07:30:36.303958+0000","last_unstale":"2024-03-06T07:30:37.535626+0000","last_undegraded":"2024-03-06T07:30:37.535626+0000","last_fullsized":"2024-03-06T07:30:37.535626+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:09:52.081168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.37","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301807+0000","last_change":"2024-03-06T07:30:36.312233+0000","last_active":"2024-03-06T07:30:37.301807+0000","last_peered":"2024-03-06T07:30:37.301807+0000","last_clean":"2024-03-06T07:30:37.301807+0000","last_became_active":"2024-03-06T07:30:36.311334+0000","last_became_peered":"2024-03-06T07:30:36.311334+0000","last_unstale":"2024-03-06T07:30:37.301807+0000","last_undegraded":"2024-03-06T07:30:37.301807+0000","last_fullsized":"2024-03-06T07:30:37.301807+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:11.776138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.36","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305524+0000","last_change":"2024-03-06T07:30:36.315248+0000","last_active":"2024-03-06T07:30:37.305524+0000","last_peered":"2024-03-06T07:30:37.305524+0000","last_clean":"2024-03-06T07:30:37.305524+0000","last_became_active":"2024-03-06T07:30:36.314931+0000","last_became_peered":"2024-03-06T07:30:36.314931+0000","last_unstale":"2024-03-06T07:30:37.305524+0000","last_undegraded":"2024-03-06T07:30:37.305524+0000","last_fullsized":"2024-03-06T07:30:37.305524+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:59:24.449988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.35","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534608+0000","last_change":"2024-03-06T07:30:36.312801+0000","last_active":"2024-03-06T07:30:37.534608+0000","last_peered":"2024-03-06T07:30:37.534608+0000","last_clean":"2024-03-06T07:30:37.534608+0000","last_became_active":"2024-03-06T07:30:36.311617+0000","last_became_peered":"2024-03-06T07:30:36.311617+0000","last_unstale":"2024-03-06T07:30:37.534608+0000","last_undegraded":"2024-03-06T07:30:37.534608+0000","last_fullsized":"2024-03-06T07:30:37.534608+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:54:29.708448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.34","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533676+0000","last_change":"2024-03-06T07:30:36.306367+0000","last_active":"2024-03-06T07:30:37.533676+0000","last_peered":"2024-03-06T07:30:37.533676+0000","last_clean":"2024-03-06T07:30:37.533676+0000","last_became_active":"2024-03-06T07:30:36.305650+0000","last_became_peered":"2024-03-06T07:30:36.305650+0000","last_unstale":"2024-03-06T07:30:37.533676+0000","last_undegraded":"2024-03-06T07:30:37.533676+0000","last_fullsized":"2024-03-06T07:30:37.533676+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:38:26.938141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.33","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.331034+0000","last_change":"2024-03-06T07:30:36.309845+0000","last_active":"2024-03-06T07:30:37.331034+0000","last_peered":"2024-03-06T07:30:37.331034+0000","last_clean":"2024-03-06T07:30:37.331034+0000","last_became_active":"2024-03-06T07:30:36.309528+0000","last_became_peered":"2024-03-06T07:30:36.309528+0000","last_unstale":"2024-03-06T07:30:37.331034+0000","last_undegraded":"2024-03-06T07:30:37.331034+0000","last_fullsized":"2024-03-06T07:30:37.331034+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:26:09.252177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.32","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301151+0000","last_change":"2024-03-06T07:30:36.303890+0000","last_active":"2024-03-06T07:30:37.301151+0000","last_peered":"2024-03-06T07:30:37.301151+0000","last_clean":"2024-03-06T07:30:37.301151+0000","last_became_active":"2024-03-06T07:30:36.302188+0000","last_became_peered":"2024-03-06T07:30:36.302188+0000","last_unstale":"2024-03-06T07:30:37.301151+0000","last_undegraded":"2024-03-06T07:30:37.301151+0000","last_fullsized":"2024-03-06T07:30:37.301151+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:02:53.824534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.31","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.330831+0000","last_change":"2024-03-06T07:30:36.304464+0000","last_active":"2024-03-06T07:30:37.330831+0000","last_peered":"2024-03-06T07:30:37.330831+0000","last_clean":"2024-03-06T07:30:37.330831+0000","last_became_active":"2024-03-06T07:30:36.304192+0000","last_became_peered":"2024-03-06T07:30:36.304192+0000","last_unstale":"2024-03-06T07:30:37.330831+0000","last_undegraded":"2024-03-06T07:30:37.330831+0000","last_fullsized":"2024-03-06T07:30:37.330831+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:40:18.218057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.30","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.304844+0000","last_change":"2024-03-06T07:30:36.318361+0000","last_active":"2024-03-06T07:30:37.304844+0000","last_peered":"2024-03-06T07:30:37.304844+0000","last_clean":"2024-03-06T07:30:37.304844+0000","last_became_active":"2024-03-06T07:30:36.315322+0000","last_became_peered":"2024-03-06T07:30:36.315322+0000","last_unstale":"2024-03-06T07:30:37.304844+0000","last_undegraded":"2024-03-06T07:30:37.304844+0000","last_fullsized":"2024-03-06T07:30:37.304844+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:31:07.856618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2f","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.302055+0000","last_change":"2024-03-06T07:30:36.311848+0000","last_active":"2024-03-06T07:30:37.302055+0000","last_peered":"2024-03-06T07:30:37.302055+0000","last_clean":"2024-03-06T07:30:37.302055+0000","last_became_active":"2024-03-06T07:30:36.310998+0000","last_became_peered":"2024-03-06T07:30:36.310998+0000","last_unstale":"2024-03-06T07:30:37.302055+0000","last_undegraded":"2024-03-06T07:30:37.302055+0000","last_fullsized":"2024-03-06T07:30:37.302055+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:26:01.712821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.2e","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534890+0000","last_change":"2024-03-06T07:30:36.312388+0000","last_active":"2024-03-06T07:30:37.534890+0000","last_peered":"2024-03-06T07:30:37.534890+0000","last_clean":"2024-03-06T07:30:37.534890+0000","last_became_active":"2024-03-06T07:30:36.312068+0000","last_became_peered":"2024-03-06T07:30:36.312068+0000","last_unstale":"2024-03-06T07:30:37.534890+0000","last_undegraded":"2024-03-06T07:30:37.534890+0000","last_fullsized":"2024-03-06T07:30:37.534890+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:17:29.834576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.2d","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536600+0000","last_change":"2024-03-06T07:30:36.299434+0000","last_active":"2024-03-06T07:30:37.536600+0000","last_peered":"2024-03-06T07:30:37.536600+0000","last_clean":"2024-03-06T07:30:37.536600+0000","last_became_active":"2024-03-06T07:30:36.299205+0000","last_became_peered":"2024-03-06T07:30:36.299205+0000","last_unstale":"2024-03-06T07:30:37.536600+0000","last_undegraded":"2024-03-06T07:30:37.536600+0000","last_fullsized":"2024-03-06T07:30:37.536600+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:35:58.592554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.2c","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.331142+0000","last_change":"2024-03-06T07:30:36.313378+0000","last_active":"2024-03-06T07:30:37.331142+0000","last_peered":"2024-03-06T07:30:37.331142+0000","last_clean":"2024-03-06T07:30:37.331142+0000","last_became_active":"2024-03-06T07:30:36.305962+0000","last_became_peered":"2024-03-06T07:30:36.305962+0000","last_unstale":"2024-03-06T07:30:37.331142+0000","last_undegraded":"2024-03-06T07:30:37.331142+0000","last_fullsized":"2024-03-06T07:30:37.331142+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:33:47.294454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2b","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305128+0000","last_change":"2024-03-06T07:30:36.313844+0000","last_active":"2024-03-06T07:30:37.305128+0000","last_peered":"2024-03-06T07:30:37.305128+0000","last_clean":"2024-03-06T07:30:37.305128+0000","last_became_active":"2024-03-06T07:30:36.309318+0000","last_became_peered":"2024-03-06T07:30:36.309318+0000","last_unstale":"2024-03-06T07:30:37.305128+0000","last_undegraded":"2024-03-06T07:30:37.305128+0000","last_fullsized":"2024-03-06T07:30:37.305128+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:00:38.162458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2a","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.304820+0000","last_change":"2024-03-06T07:30:36.305012+0000","last_active":"2024-03-06T07:30:37.304820+0000","last_peered":"2024-03-06T07:30:37.304820+0000","last_clean":"2024-03-06T07:30:37.304820+0000","last_became_active":"2024-03-06T07:30:36.304711+0000","last_became_peered":"2024-03-06T07:30:36.304711+0000","last_unstale":"2024-03-06T07:30:37.304820+0000","last_undegraded":"2024-03-06T07:30:37.304820+0000","last_fullsized":"2024-03-06T07:30:37.304820+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:17:53.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.29","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534033+0000","last_change":"2024-03-06T07:30:36.308067+0000","last_active":"2024-03-06T07:30:37.534033+0000","last_peered":"2024-03-06T07:30:37.534033+0000","last_clean":"2024-03-06T07:30:37.534033+0000","last_became_active":"2024-03-06T07:30:36.306566+0000","last_became_peered":"2024-03-06T07:30:36.306566+0000","last_unstale":"2024-03-06T07:30:37.534033+0000","last_undegraded":"2024-03-06T07:30:37.534033+0000","last_fullsized":"2024-03-06T07:30:37.534033+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:49:31.875214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.28","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533808+0000","last_change":"2024-03-06T07:30:36.308852+0000","last_active":"2024-03-06T07:30:37.533808+0000","last_peered":"2024-03-06T07:30:37.533808+0000","last_clean":"2024-03-06T07:30:37.533808+0000","last_became_active":"2024-03-06T07:30:36.308533+0000","last_became_peered":"2024-03-06T07:30:36.308533+0000","last_unstale":"2024-03-06T07:30:37.533808+0000","last_undegraded":"2024-03-06T07:30:37.533808+0000","last_fullsized":"2024-03-06T07:30:37.533808+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:59:24.449988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.27","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305358+0000","last_change":"2024-03-06T07:30:36.309729+0000","last_active":"2024-03-06T07:30:37.305358+0000","last_peered":"2024-03-06T07:30:37.305358+0000","last_clean":"2024-03-06T07:30:37.305358+0000","last_became_active":"2024-03-06T07:30:36.307631+0000","last_became_peered":"2024-03-06T07:30:36.307631+0000","last_unstale":"2024-03-06T07:30:37.305358+0000","last_undegraded":"2024-03-06T07:30:37.305358+0000","last_fullsized":"2024-03-06T07:30:37.305358+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:17:46.651594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.26","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327163+0000","last_change":"2024-03-06T07:30:36.308703+0000","last_active":"2024-03-06T07:30:37.327163+0000","last_peered":"2024-03-06T07:30:37.327163+0000","last_clean":"2024-03-06T07:30:37.327163+0000","last_became_active":"2024-03-06T07:30:36.308381+0000","last_became_peered":"2024-03-06T07:30:36.308381+0000","last_unstale":"2024-03-06T07:30:37.327163+0000","last_undegraded":"2024-03-06T07:30:37.327163+0000","last_fullsized":"2024-03-06T07:30:37.327163+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:55:50.843334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.25","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533830+0000","last_change":"2024-03-06T07:30:36.315360+0000","last_active":"2024-03-06T07:30:37.533830+0000","last_peered":"2024-03-06T07:30:37.533830+0000","last_clean":"2024-03-06T07:30:37.533830+0000","last_became_active":"2024-03-06T07:30:36.314898+0000","last_became_peered":"2024-03-06T07:30:36.314898+0000","last_unstale":"2024-03-06T07:30:37.533830+0000","last_undegraded":"2024-03-06T07:30:37.533830+0000","last_fullsized":"2024-03-06T07:30:37.533830+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:11:46.759638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.24","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305798+0000","last_change":"2024-03-06T07:30:36.311845+0000","last_active":"2024-03-06T07:30:37.305798+0000","last_peered":"2024-03-06T07:30:37.305798+0000","last_clean":"2024-03-06T07:30:37.305798+0000","last_became_active":"2024-03-06T07:30:36.311579+0000","last_became_peered":"2024-03-06T07:30:36.311579+0000","last_unstale":"2024-03-06T07:30:37.305798+0000","last_undegraded":"2024-03-06T07:30:37.305798+0000","last_fullsized":"2024-03-06T07:30:37.305798+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:10:57.235113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.23","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534902+0000","last_change":"2024-03-06T07:30:36.310688+0000","last_active":"2024-03-06T07:30:37.534902+0000","last_peered":"2024-03-06T07:30:37.534902+0000","last_clean":"2024-03-06T07:30:37.534902+0000","last_became_active":"2024-03-06T07:30:36.307597+0000","last_became_peered":"2024-03-06T07:30:36.307597+0000","last_unstale":"2024-03-06T07:30:37.534902+0000","last_undegraded":"2024-03-06T07:30:37.534902+0000","last_fullsized":"2024-03-06T07:30:37.534902+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:52:59.855178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.22","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.326786+0000","last_change":"2024-03-06T07:30:36.307982+0000","last_active":"2024-03-06T07:30:37.326786+0000","last_peered":"2024-03-06T07:30:37.326786+0000","last_clean":"2024-03-06T07:30:37.326786+0000","last_became_active":"2024-03-06T07:30:36.307675+0000","last_became_peered":"2024-03-06T07:30:36.307675+0000","last_unstale":"2024-03-06T07:30:37.326786+0000","last_undegraded":"2024-03-06T07:30:37.326786+0000","last_fullsized":"2024-03-06T07:30:37.326786+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:28:44.138239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.21","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327358+0000","last_change":"2024-03-06T07:30:36.306375+0000","last_active":"2024-03-06T07:30:37.327358+0000","last_peered":"2024-03-06T07:30:37.327358+0000","last_clean":"2024-03-06T07:30:37.327358+0000","last_became_active":"2024-03-06T07:30:36.305966+0000","last_became_peered":"2024-03-06T07:30:36.305966+0000","last_unstale":"2024-03-06T07:30:37.327358+0000","last_undegraded":"2024-03-06T07:30:37.327358+0000","last_fullsized":"2024-03-06T07:30:37.327358+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:25:56.484931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.20","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535699+0000","last_change":"2024-03-06T07:30:36.298464+0000","last_active":"2024-03-06T07:30:37.535699+0000","last_peered":"2024-03-06T07:30:37.535699+0000","last_clean":"2024-03-06T07:30:37.535699+0000","last_became_active":"2024-03-06T07:30:36.298088+0000","last_became_peered":"2024-03-06T07:30:36.298088+0000","last_unstale":"2024-03-06T07:30:37.535699+0000","last_undegraded":"2024-03-06T07:30:37.535699+0000","last_fullsized":"2024-03-06T07:30:37.535699+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:08.683945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.1e","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534921+0000","last_change":"2024-03-06T07:30:36.311080+0000","last_active":"2024-03-06T07:30:37.534921+0000","last_peered":"2024-03-06T07:30:37.534921+0000","last_clean":"2024-03-06T07:30:37.534921+0000","last_became_active":"2024-03-06T07:30:36.310745+0000","last_became_peered":"2024-03-06T07:30:36.310745+0000","last_unstale":"2024-03-06T07:30:37.534921+0000","last_undegraded":"2024-03-06T07:30:37.534921+0000","last_fullsized":"2024-03-06T07:30:37.534921+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:39:48.666363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536249+0000","last_change":"2024-03-06T07:30:35.294455+0000","last_active":"2024-03-06T07:30:37.536249+0000","last_peered":"2024-03-06T07:30:37.536249+0000","last_clean":"2024-03-06T07:30:37.536249+0000","last_became_active":"2024-03-06T07:30:35.294176+0000","last_became_peered":"2024-03-06T07:30:35.294176+0000","last_unstale":"2024-03-06T07:30:37.536249+0000","last_undegraded":"2024-03-06T07:30:37.536249+0000","last_fullsized":"2024-03-06T07:30:37.536249+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:36:20.939266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.1f","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.302060+0000","last_change":"2024-03-06T07:30:36.311627+0000","last_active":"2024-03-06T07:30:37.302060+0000","last_peered":"2024-03-06T07:30:37.302060+0000","last_clean":"2024-03-06T07:30:37.302060+0000","last_became_active":"2024-03-06T07:30:36.310669+0000","last_became_peered":"2024-03-06T07:30:36.310669+0000","last_unstale":"2024-03-06T07:30:37.302060+0000","last_undegraded":"2024-03-06T07:30:37.302060+0000","last_fullsized":"2024-03-06T07:30:37.302060+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:17:14.045037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.331152+0000","last_change":"2024-03-06T07:30:35.285682+0000","last_active":"2024-03-06T07:30:37.331152+0000","last_peered":"2024-03-06T07:30:37.331152+0000","last_clean":"2024-03-06T07:30:37.331152+0000","last_became_active":"2024-03-06T07:30:35.285372+0000","last_became_peered":"2024-03-06T07:30:35.285372+0000","last_unstale":"2024-03-06T07:30:37.331152+0000","last_undegraded":"2024-03-06T07:30:37.331152+0000","last_fullsized":"2024-03-06T07:30:37.331152+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:59:23.434910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.1c","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536464+0000","last_change":"2024-03-06T07:30:36.303578+0000","last_active":"2024-03-06T07:30:37.536464+0000","last_peered":"2024-03-06T07:30:37.536464+0000","last_clean":"2024-03-06T07:30:37.536464+0000","last_became_active":"2024-03-06T07:30:36.301808+0000","last_became_peered":"2024-03-06T07:30:36.301808+0000","last_unstale":"2024-03-06T07:30:37.536464+0000","last_undegraded":"2024-03-06T07:30:37.536464+0000","last_fullsized":"2024-03-06T07:30:37.536464+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:08:37.785351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.304960+0000","last_change":"2024-03-06T07:30:35.300821+0000","last_active":"2024-03-06T07:30:37.304960+0000","last_peered":"2024-03-06T07:30:37.304960+0000","last_clean":"2024-03-06T07:30:37.304960+0000","last_became_active":"2024-03-06T07:30:35.300540+0000","last_became_peered":"2024-03-06T07:30:35.300540+0000","last_unstale":"2024-03-06T07:30:37.304960+0000","last_undegraded":"2024-03-06T07:30:37.304960+0000","last_fullsized":"2024-03-06T07:30:37.304960+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:13:35.114408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1d","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534986+0000","last_change":"2024-03-06T07:30:36.315302+0000","last_active":"2024-03-06T07:30:37.534986+0000","last_peered":"2024-03-06T07:30:37.534986+0000","last_clean":"2024-03-06T07:30:37.534986+0000","last_became_active":"2024-03-06T07:30:36.314526+0000","last_became_peered":"2024-03-06T07:30:36.314526+0000","last_unstale":"2024-03-06T07:30:37.534986+0000","last_undegraded":"2024-03-06T07:30:37.534986+0000","last_fullsized":"2024-03-06T07:30:37.534986+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:10.295044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536670+0000","last_change":"2024-03-06T07:30:35.298895+0000","last_active":"2024-03-06T07:30:37.536670+0000","last_peered":"2024-03-06T07:30:37.536670+0000","last_clean":"2024-03-06T07:30:37.536670+0000","last_became_active":"2024-03-06T07:30:35.298649+0000","last_became_peered":"2024-03-06T07:30:35.298649+0000","last_unstale":"2024-03-06T07:30:37.536670+0000","last_undegraded":"2024-03-06T07:30:37.536670+0000","last_fullsized":"2024-03-06T07:30:37.536670+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:50:29.103301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.1a","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327337+0000","last_change":"2024-03-06T07:30:36.303378+0000","last_active":"2024-03-06T07:30:37.327337+0000","last_peered":"2024-03-06T07:30:37.327337+0000","last_clean":"2024-03-06T07:30:37.327337+0000","last_became_active":"2024-03-06T07:30:36.302446+0000","last_became_peered":"2024-03-06T07:30:36.302446+0000","last_unstale":"2024-03-06T07:30:37.327337+0000","last_undegraded":"2024-03-06T07:30:37.327337+0000","last_fullsized":"2024-03-06T07:30:37.327337+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:58.823203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534346+0000","last_change":"2024-03-06T07:30:35.297923+0000","last_active":"2024-03-06T07:30:37.534346+0000","last_peered":"2024-03-06T07:30:37.534346+0000","last_clean":"2024-03-06T07:30:37.534346+0000","last_became_active":"2024-03-06T07:30:35.296655+0000","last_became_peered":"2024-03-06T07:30:35.296655+0000","last_unstale":"2024-03-06T07:30:37.534346+0000","last_undegraded":"2024-03-06T07:30:37.534346+0000","last_fullsized":"2024-03-06T07:30:37.534346+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:09:45.028325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1b","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305255+0000","last_change":"2024-03-06T07:30:36.307765+0000","last_active":"2024-03-06T07:30:37.305255+0000","last_peered":"2024-03-06T07:30:37.305255+0000","last_clean":"2024-03-06T07:30:37.305255+0000","last_became_active":"2024-03-06T07:30:36.307435+0000","last_became_peered":"2024-03-06T07:30:36.307435+0000","last_unstale":"2024-03-06T07:30:37.305255+0000","last_undegraded":"2024-03-06T07:30:37.305255+0000","last_fullsized":"2024-03-06T07:30:37.305255+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:13:20.274813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535485+0000","last_change":"2024-03-06T07:30:35.303354+0000","last_active":"2024-03-06T07:30:37.535485+0000","last_peered":"2024-03-06T07:30:37.535485+0000","last_clean":"2024-03-06T07:30:37.535485+0000","last_became_active":"2024-03-06T07:30:35.301360+0000","last_became_peered":"2024-03-06T07:30:35.301360+0000","last_unstale":"2024-03-06T07:30:37.535485+0000","last_undegraded":"2024-03-06T07:30:37.535485+0000","last_fullsized":"2024-03-06T07:30:37.535485+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:29:05.156313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.18","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.300954+0000","last_change":"2024-03-06T07:30:36.317776+0000","last_active":"2024-03-06T07:30:37.300954+0000","last_peered":"2024-03-06T07:30:37.300954+0000","last_clean":"2024-03-06T07:30:37.300954+0000","last_became_active":"2024-03-06T07:30:36.316727+0000","last_became_peered":"2024-03-06T07:30:36.316727+0000","last_unstale":"2024-03-06T07:30:37.300954+0000","last_undegraded":"2024-03-06T07:30:37.300954+0000","last_fullsized":"2024-03-06T07:30:37.300954+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:21:59.210531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.532422+0000","last_change":"2024-03-06T07:30:35.297639+0000","last_active":"2024-03-06T07:30:37.532422+0000","last_peered":"2024-03-06T07:30:37.532422+0000","last_clean":"2024-03-06T07:30:37.532422+0000","last_became_active":"2024-03-06T07:30:35.297334+0000","last_became_peered":"2024-03-06T07:30:35.297334+0000","last_unstale":"2024-03-06T07:30:37.532422+0000","last_undegraded":"2024-03-06T07:30:37.532422+0000","last_fullsized":"2024-03-06T07:30:37.532422+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:06:45.835404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.19","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.330943+0000","last_change":"2024-03-06T07:30:36.311669+0000","last_active":"2024-03-06T07:30:37.330943+0000","last_peered":"2024-03-06T07:30:37.330943+0000","last_clean":"2024-03-06T07:30:37.330943+0000","last_became_active":"2024-03-06T07:30:36.305082+0000","last_became_peered":"2024-03-06T07:30:36.305082+0000","last_unstale":"2024-03-06T07:30:37.330943+0000","last_undegraded":"2024-03-06T07:30:37.330943+0000","last_fullsized":"2024-03-06T07:30:37.330943+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:18:53.475957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.300907+0000","last_change":"2024-03-06T07:30:35.283610+0000","last_active":"2024-03-06T07:30:37.300907+0000","last_peered":"2024-03-06T07:30:37.300907+0000","last_clean":"2024-03-06T07:30:37.300907+0000","last_became_active":"2024-03-06T07:30:35.281958+0000","last_became_peered":"2024-03-06T07:30:35.281958+0000","last_unstale":"2024-03-06T07:30:37.300907+0000","last_undegraded":"2024-03-06T07:30:37.300907+0000","last_fullsized":"2024-03-06T07:30:37.300907+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:41:09.563158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.16","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327640+0000","last_change":"2024-03-06T07:30:36.305158+0000","last_active":"2024-03-06T07:30:37.327640+0000","last_peered":"2024-03-06T07:30:37.327640+0000","last_clean":"2024-03-06T07:30:37.327640+0000","last_became_active":"2024-03-06T07:30:36.304853+0000","last_became_peered":"2024-03-06T07:30:36.304853+0000","last_unstale":"2024-03-06T07:30:37.327640+0000","last_undegraded":"2024-03-06T07:30:37.327640+0000","last_fullsized":"2024-03-06T07:30:37.327640+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:17:04.344793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305457+0000","last_change":"2024-03-06T07:30:35.295896+0000","last_active":"2024-03-06T07:30:37.305457+0000","last_peered":"2024-03-06T07:30:37.305457+0000","last_clean":"2024-03-06T07:30:37.305457+0000","last_became_active":"2024-03-06T07:30:35.295592+0000","last_became_peered":"2024-03-06T07:30:35.295592+0000","last_unstale":"2024-03-06T07:30:37.305457+0000","last_undegraded":"2024-03-06T07:30:37.305457+0000","last_fullsized":"2024-03-06T07:30:37.305457+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:48.765755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.17","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535732+0000","last_change":"2024-03-06T07:30:36.309624+0000","last_active":"2024-03-06T07:30:37.535732+0000","last_peered":"2024-03-06T07:30:37.535732+0000","last_clean":"2024-03-06T07:30:37.535732+0000","last_became_active":"2024-03-06T07:30:36.309302+0000","last_became_peered":"2024-03-06T07:30:36.309302+0000","last_unstale":"2024-03-06T07:30:37.535732+0000","last_undegraded":"2024-03-06T07:30:37.535732+0000","last_fullsized":"2024-03-06T07:30:37.535732+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:40:18.218057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.537151+0000","last_change":"2024-03-06T07:30:35.302820+0000","last_active":"2024-03-06T07:30:37.537151+0000","last_peered":"2024-03-06T07:30:37.537151+0000","last_clean":"2024-03-06T07:30:37.537151+0000","last_became_active":"2024-03-06T07:30:35.302436+0000","last_became_peered":"2024-03-06T07:30:35.302436+0000","last_unstale":"2024-03-06T07:30:37.537151+0000","last_undegraded":"2024-03-06T07:30:37.537151+0000","last_fullsized":"2024-03-06T07:30:37.537151+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:51:47.736461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.14","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536623+0000","last_change":"2024-03-06T07:30:36.316564+0000","last_active":"2024-03-06T07:30:37.536623+0000","last_peered":"2024-03-06T07:30:37.536623+0000","last_clean":"2024-03-06T07:30:37.536623+0000","last_became_active":"2024-03-06T07:30:36.316205+0000","last_became_peered":"2024-03-06T07:30:36.316205+0000","last_unstale":"2024-03-06T07:30:37.536623+0000","last_undegraded":"2024-03-06T07:30:37.536623+0000","last_fullsized":"2024-03-06T07:30:37.536623+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:50:15.983154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533982+0000","last_change":"2024-03-06T07:30:35.304481+0000","last_active":"2024-03-06T07:30:37.533982+0000","last_peered":"2024-03-06T07:30:37.533982+0000","last_clean":"2024-03-06T07:30:37.533982+0000","last_became_active":"2024-03-06T07:30:35.304237+0000","last_became_peered":"2024-03-06T07:30:35.304237+0000","last_unstale":"2024-03-06T07:30:37.533982+0000","last_undegraded":"2024-03-06T07:30:37.533982+0000","last_fullsized":"2024-03-06T07:30:37.533982+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:20:24.462902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301708+0000","last_change":"2024-03-06T07:30:36.295784+0000","last_active":"2024-03-06T07:30:37.301708+0000","last_peered":"2024-03-06T07:30:37.301708+0000","last_clean":"2024-03-06T07:30:37.301708+0000","last_became_active":"2024-03-06T07:30:36.295393+0000","last_became_peered":"2024-03-06T07:30:36.295393+0000","last_unstale":"2024-03-06T07:30:37.301708+0000","last_undegraded":"2024-03-06T07:30:37.301708+0000","last_fullsized":"2024-03-06T07:30:37.301708+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:08.713467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533849+0000","last_change":"2024-03-06T07:30:35.285553+0000","last_active":"2024-03-06T07:30:37.533849+0000","last_peered":"2024-03-06T07:30:37.533849+0000","last_clean":"2024-03-06T07:30:37.533849+0000","last_became_active":"2024-03-06T07:30:35.285155+0000","last_became_peered":"2024-03-06T07:30:35.285155+0000","last_unstale":"2024-03-06T07:30:37.533849+0000","last_undegraded":"2024-03-06T07:30:37.533849+0000","last_fullsized":"2024-03-06T07:30:37.533849+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:09:30.736818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534812+0000","last_change":"2024-03-06T07:30:36.310066+0000","last_active":"2024-03-06T07:30:37.534812+0000","last_peered":"2024-03-06T07:30:37.534812+0000","last_clean":"2024-03-06T07:30:37.534812+0000","last_became_active":"2024-03-06T07:30:36.307955+0000","last_became_peered":"2024-03-06T07:30:36.307955+0000","last_unstale":"2024-03-06T07:30:37.534812+0000","last_undegraded":"2024-03-06T07:30:37.534812+0000","last_fullsized":"2024-03-06T07:30:37.534812+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:28:28.377029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.532754+0000","last_change":"2024-03-06T07:30:35.291567+0000","last_active":"2024-03-06T07:30:37.532754+0000","last_peered":"2024-03-06T07:30:37.532754+0000","last_clean":"2024-03-06T07:30:37.532754+0000","last_became_active":"2024-03-06T07:30:35.291200+0000","last_became_peered":"2024-03-06T07:30:35.291200+0000","last_unstale":"2024-03-06T07:30:37.532754+0000","last_undegraded":"2024-03-06T07:30:37.532754+0000","last_fullsized":"2024-03-06T07:30:37.532754+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:23:44.818541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.13","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.326895+0000","last_change":"2024-03-06T07:30:36.313583+0000","last_active":"2024-03-06T07:30:37.326895+0000","last_peered":"2024-03-06T07:30:37.326895+0000","last_clean":"2024-03-06T07:30:37.326895+0000","last_became_active":"2024-03-06T07:30:36.313230+0000","last_became_peered":"2024-03-06T07:30:36.313230+0000","last_unstale":"2024-03-06T07:30:37.326895+0000","last_undegraded":"2024-03-06T07:30:37.326895+0000","last_fullsized":"2024-03-06T07:30:37.326895+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:59:57.807582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536115+0000","last_change":"2024-03-06T07:30:35.302160+0000","last_active":"2024-03-06T07:30:37.536115+0000","last_peered":"2024-03-06T07:30:37.536115+0000","last_clean":"2024-03-06T07:30:37.536115+0000","last_became_active":"2024-03-06T07:30:35.301861+0000","last_became_peered":"2024-03-06T07:30:35.301861+0000","last_unstale":"2024-03-06T07:30:37.536115+0000","last_undegraded":"2024-03-06T07:30:37.536115+0000","last_fullsized":"2024-03-06T07:30:37.536115+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:24:51.902990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.10","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.330767+0000","last_change":"2024-03-06T07:30:36.302741+0000","last_active":"2024-03-06T07:30:37.330767+0000","last_peered":"2024-03-06T07:30:37.330767+0000","last_clean":"2024-03-06T07:30:37.330767+0000","last_became_active":"2024-03-06T07:30:36.302408+0000","last_became_peered":"2024-03-06T07:30:36.302408+0000","last_unstale":"2024-03-06T07:30:37.330767+0000","last_undegraded":"2024-03-06T07:30:37.330767+0000","last_fullsized":"2024-03-06T07:30:37.330767+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:52:11.419818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.326747+0000","last_change":"2024-03-06T07:30:35.301741+0000","last_active":"2024-03-06T07:30:37.326747+0000","last_peered":"2024-03-06T07:30:37.326747+0000","last_clean":"2024-03-06T07:30:37.326747+0000","last_became_active":"2024-03-06T07:30:35.301450+0000","last_became_peered":"2024-03-06T07:30:35.301450+0000","last_unstale":"2024-03-06T07:30:37.326747+0000","last_undegraded":"2024-03-06T07:30:37.326747+0000","last_fullsized":"2024-03-06T07:30:37.326747+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:09:51.066090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.11","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301170+0000","last_change":"2024-03-06T07:30:36.300163+0000","last_active":"2024-03-06T07:30:37.301170+0000","last_peered":"2024-03-06T07:30:37.301170+0000","last_clean":"2024-03-06T07:30:37.301170+0000","last_became_active":"2024-03-06T07:30:36.299750+0000","last_became_peered":"2024-03-06T07:30:36.299750+0000","last_unstale":"2024-03-06T07:30:37.301170+0000","last_undegraded":"2024-03-06T07:30:37.301170+0000","last_fullsized":"2024-03-06T07:30:37.301170+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:11:23.538230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.304714+0000","last_change":"2024-03-06T07:30:35.293678+0000","last_active":"2024-03-06T07:30:37.304714+0000","last_peered":"2024-03-06T07:30:37.304714+0000","last_clean":"2024-03-06T07:30:37.304714+0000","last_became_active":"2024-03-06T07:30:35.292074+0000","last_became_peered":"2024-03-06T07:30:35.292074+0000","last_unstale":"2024-03-06T07:30:37.304714+0000","last_undegraded":"2024-03-06T07:30:37.304714+0000","last_fullsized":"2024-03-06T07:30:37.304714+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:38:25.923063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305577+0000","last_change":"2024-03-06T07:30:36.302879+0000","last_active":"2024-03-06T07:30:37.305577+0000","last_peered":"2024-03-06T07:30:37.305577+0000","last_clean":"2024-03-06T07:30:37.305577+0000","last_became_active":"2024-03-06T07:30:36.302491+0000","last_became_peered":"2024-03-06T07:30:36.302491+0000","last_unstale":"2024-03-06T07:30:37.305577+0000","last_undegraded":"2024-03-06T07:30:37.305577+0000","last_fullsized":"2024-03-06T07:30:37.305577+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:49:31.875214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.537002+0000","last_change":"2024-03-06T07:30:35.297456+0000","last_active":"2024-03-06T07:30:37.537002+0000","last_peered":"2024-03-06T07:30:37.537002+0000","last_clean":"2024-03-06T07:30:37.537002+0000","last_became_active":"2024-03-06T07:30:35.297172+0000","last_became_peered":"2024-03-06T07:30:35.297172+0000","last_unstale":"2024-03-06T07:30:37.537002+0000","last_undegraded":"2024-03-06T07:30:37.537002+0000","last_fullsized":"2024-03-06T07:30:37.537002+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:27:25.356385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536108+0000","last_change":"2024-03-06T07:30:36.303578+0000","last_active":"2024-03-06T07:30:37.536108+0000","last_peered":"2024-03-06T07:30:37.536108+0000","last_clean":"2024-03-06T07:30:37.536108+0000","last_became_active":"2024-03-06T07:30:36.303235+0000","last_became_peered":"2024-03-06T07:30:36.303235+0000","last_unstale":"2024-03-06T07:30:37.536108+0000","last_undegraded":"2024-03-06T07:30:37.536108+0000","last_fullsized":"2024-03-06T07:30:37.536108+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:18:53.475957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.305699+0000","last_change":"2024-03-06T07:30:35.301600+0000","last_active":"2024-03-06T07:30:37.305699+0000","last_peered":"2024-03-06T07:30:37.305699+0000","last_clean":"2024-03-06T07:30:37.305699+0000","last_became_active":"2024-03-06T07:30:35.301321+0000","last_became_peered":"2024-03-06T07:30:35.301321+0000","last_unstale":"2024-03-06T07:30:37.305699+0000","last_undegraded":"2024-03-06T07:30:37.305699+0000","last_fullsized":"2024-03-06T07:30:37.305699+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:36:20.939266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533688+0000","last_change":"2024-03-06T07:30:36.306118+0000","last_active":"2024-03-06T07:30:37.533688+0000","last_peered":"2024-03-06T07:30:37.533688+0000","last_clean":"2024-03-06T07:30:37.533688+0000","last_became_active":"2024-03-06T07:30:36.304914+0000","last_became_peered":"2024-03-06T07:30:36.304914+0000","last_unstale":"2024-03-06T07:30:37.533688+0000","last_undegraded":"2024-03-06T07:30:37.533688+0000","last_fullsized":"2024-03-06T07:30:37.533688+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:00:21.475493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536617+0000","last_change":"2024-03-06T07:30:35.303492+0000","last_active":"2024-03-06T07:30:37.536617+0000","last_peered":"2024-03-06T07:30:37.536617+0000","last_clean":"2024-03-06T07:30:37.536617+0000","last_became_active":"2024-03-06T07:30:35.302869+0000","last_became_peered":"2024-03-06T07:30:35.302869+0000","last_unstale":"2024-03-06T07:30:37.536617+0000","last_undegraded":"2024-03-06T07:30:37.536617+0000","last_fullsized":"2024-03-06T07:30:37.536617+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:38:02.065932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301801+0000","last_change":"2024-03-06T07:30:36.305147+0000","last_active":"2024-03-06T07:30:37.301801+0000","last_peered":"2024-03-06T07:30:37.301801+0000","last_clean":"2024-03-06T07:30:37.301801+0000","last_became_active":"2024-03-06T07:30:36.304856+0000","last_became_peered":"2024-03-06T07:30:36.304856+0000","last_unstale":"2024-03-06T07:30:37.301801+0000","last_undegraded":"2024-03-06T07:30:37.301801+0000","last_fullsized":"2024-03-06T07:30:37.301801+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:41:31.074788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534195+0000","last_change":"2024-03-06T07:30:35.302283+0000","last_active":"2024-03-06T07:30:37.534195+0000","last_peered":"2024-03-06T07:30:37.534195+0000","last_clean":"2024-03-06T07:30:37.534195+0000","last_became_active":"2024-03-06T07:30:35.300747+0000","last_became_peered":"2024-03-06T07:30:35.300747+0000","last_unstale":"2024-03-06T07:30:37.534195+0000","last_undegraded":"2024-03-06T07:30:37.534195+0000","last_fullsized":"2024-03-06T07:30:37.534195+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:07:48.765755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534206+0000","last_change":"2024-03-06T07:30:36.309001+0000","last_active":"2024-03-06T07:30:37.534206+0000","last_peered":"2024-03-06T07:30:37.534206+0000","last_clean":"2024-03-06T07:30:37.534206+0000","last_became_active":"2024-03-06T07:30:36.304077+0000","last_became_peered":"2024-03-06T07:30:36.304077+0000","last_unstale":"2024-03-06T07:30:37.534206+0000","last_undegraded":"2024-03-06T07:30:37.534206+0000","last_fullsized":"2024-03-06T07:30:37.534206+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:02:58.197992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535858+0000","last_change":"2024-03-06T07:30:35.299498+0000","last_active":"2024-03-06T07:30:37.535858+0000","last_peered":"2024-03-06T07:30:37.535858+0000","last_clean":"2024-03-06T07:30:37.535858+0000","last_became_active":"2024-03-06T07:30:35.294814+0000","last_became_peered":"2024-03-06T07:30:35.294814+0000","last_unstale":"2024-03-06T07:30:37.535858+0000","last_undegraded":"2024-03-06T07:30:37.535858+0000","last_fullsized":"2024-03-06T07:30:37.535858+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:26:27.270755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533205+0000","last_change":"2024-03-06T07:30:36.306708+0000","last_active":"2024-03-06T07:30:37.533205+0000","last_peered":"2024-03-06T07:30:37.533205+0000","last_clean":"2024-03-06T07:30:37.533205+0000","last_became_active":"2024-03-06T07:30:36.306426+0000","last_became_peered":"2024-03-06T07:30:36.306426+0000","last_unstale":"2024-03-06T07:30:37.533205+0000","last_undegraded":"2024-03-06T07:30:37.533205+0000","last_fullsized":"2024-03-06T07:30:37.533205+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:05:15.847168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327284+0000","last_change":"2024-03-06T07:30:35.301021+0000","last_active":"2024-03-06T07:30:37.327284+0000","last_peered":"2024-03-06T07:30:37.327284+0000","last_clean":"2024-03-06T07:30:37.327284+0000","last_became_active":"2024-03-06T07:30:35.300732+0000","last_became_peered":"2024-03-06T07:30:35.300732+0000","last_unstale":"2024-03-06T07:30:37.327284+0000","last_undegraded":"2024-03-06T07:30:37.327284+0000","last_fullsized":"2024-03-06T07:30:37.327284+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:07.668867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533417+0000","last_change":"2024-03-06T07:30:36.296841+0000","last_active":"2024-03-06T07:30:37.533417+0000","last_peered":"2024-03-06T07:30:37.533417+0000","last_clean":"2024-03-06T07:30:37.533417+0000","last_became_active":"2024-03-06T07:30:36.296491+0000","last_became_peered":"2024-03-06T07:30:36.296491+0000","last_unstale":"2024-03-06T07:30:37.533417+0000","last_undegraded":"2024-03-06T07:30:37.533417+0000","last_fullsized":"2024-03-06T07:30:37.533417+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:41:43.847281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.330498+0000","last_change":"2024-03-06T07:30:35.288328+0000","last_active":"2024-03-06T07:30:37.330498+0000","last_peered":"2024-03-06T07:30:37.330498+0000","last_clean":"2024-03-06T07:30:37.330498+0000","last_became_active":"2024-03-06T07:30:35.287547+0000","last_became_peered":"2024-03-06T07:30:35.287547+0000","last_unstale":"2024-03-06T07:30:37.330498+0000","last_undegraded":"2024-03-06T07:30:37.330498+0000","last_fullsized":"2024-03-06T07:30:37.330498+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:13:19.259735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.331246+0000","last_change":"2024-03-06T07:30:36.310127+0000","last_active":"2024-03-06T07:30:37.331246+0000","last_peered":"2024-03-06T07:30:37.331246+0000","last_clean":"2024-03-06T07:30:37.331246+0000","last_became_active":"2024-03-06T07:30:36.309859+0000","last_became_peered":"2024-03-06T07:30:36.309859+0000","last_unstale":"2024-03-06T07:30:37.331246+0000","last_undegraded":"2024-03-06T07:30:37.331246+0000","last_fullsized":"2024-03-06T07:30:37.331246+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:26:45.164017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.537123+0000","last_change":"2024-03-06T07:30:35.297451+0000","last_active":"2024-03-06T07:30:37.537123+0000","last_peered":"2024-03-06T07:30:37.537123+0000","last_clean":"2024-03-06T07:30:37.537123+0000","last_became_active":"2024-03-06T07:30:35.297127+0000","last_became_peered":"2024-03-06T07:30:35.297127+0000","last_unstale":"2024-03-06T07:30:37.537123+0000","last_undegraded":"2024-03-06T07:30:37.537123+0000","last_fullsized":"2024-03-06T07:30:37.537123+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:04:48.098241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327156+0000","last_change":"2024-03-06T07:30:36.301511+0000","last_active":"2024-03-06T07:30:37.327156+0000","last_peered":"2024-03-06T07:30:37.327156+0000","last_clean":"2024-03-06T07:30:37.327156+0000","last_became_active":"2024-03-06T07:30:36.301180+0000","last_became_peered":"2024-03-06T07:30:36.301180+0000","last_unstale":"2024-03-06T07:30:37.327156+0000","last_undegraded":"2024-03-06T07:30:37.327156+0000","last_fullsized":"2024-03-06T07:30:37.327156+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:36:11.969189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.6","version":"70'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.327604+0000","last_change":"2024-03-06T07:30:35.299663+0000","last_active":"2024-03-06T07:30:37.327604+0000","last_peered":"2024-03-06T07:30:37.327604+0000","last_clean":"2024-03-06T07:30:37.327604+0000","last_became_active":"2024-03-06T07:30:35.299289+0000","last_became_peered":"2024-03-06T07:30:35.299289+0000","last_unstale":"2024-03-06T07:30:37.327604+0000","last_undegraded":"2024-03-06T07:30:37.327604+0000","last_fullsized":"2024-03-06T07:30:37.327604+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:22:39.152491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.534416+0000","last_change":"2024-03-06T07:30:36.308843+0000","last_active":"2024-03-06T07:30:37.534416+0000","last_peered":"2024-03-06T07:30:37.534416+0000","last_clean":"2024-03-06T07:30:37.534416+0000","last_became_active":"2024-03-06T07:30:36.308516+0000","last_became_peered":"2024-03-06T07:30:36.308516+0000","last_unstale":"2024-03-06T07:30:37.534416+0000","last_undegraded":"2024-03-06T07:30:37.534416+0000","last_fullsized":"2024-03-06T07:30:37.534416+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:10:57.235113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301764+0000","last_change":"2024-03-06T07:30:35.287090+0000","last_active":"2024-03-06T07:30:37.301764+0000","last_peered":"2024-03-06T07:30:37.301764+0000","last_clean":"2024-03-06T07:30:37.301764+0000","last_became_active":"2024-03-06T07:30:35.286066+0000","last_became_peered":"2024-03-06T07:30:35.286066+0000","last_unstale":"2024-03-06T07:30:37.301764+0000","last_undegraded":"2024-03-06T07:30:37.301764+0000","last_fullsized":"2024-03-06T07:30:37.301764+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:02:12.355466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535285+0000","last_change":"2024-03-06T07:30:36.306644+0000","last_active":"2024-03-06T07:30:37.535285+0000","last_peered":"2024-03-06T07:30:37.535285+0000","last_clean":"2024-03-06T07:30:37.535285+0000","last_became_active":"2024-03-06T07:30:36.306299+0000","last_became_peered":"2024-03-06T07:30:36.306299+0000","last_unstale":"2024-03-06T07:30:37.535285+0000","last_undegraded":"2024-03-06T07:30:37.535285+0000","last_fullsized":"2024-03-06T07:30:37.535285+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:52:11.419818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.330995+0000","last_change":"2024-03-06T07:30:35.296463+0000","last_active":"2024-03-06T07:30:37.330995+0000","last_peered":"2024-03-06T07:30:37.330995+0000","last_clean":"2024-03-06T07:30:37.330995+0000","last_became_active":"2024-03-06T07:30:35.296141+0000","last_became_peered":"2024-03-06T07:30:35.296141+0000","last_unstale":"2024-03-06T07:30:37.330995+0000","last_undegraded":"2024-03-06T07:30:37.330995+0000","last_fullsized":"2024-03-06T07:30:37.330995+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:17:45.636516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301245+0000","last_change":"2024-03-06T07:30:36.310428+0000","last_active":"2024-03-06T07:30:37.301245+0000","last_peered":"2024-03-06T07:30:37.301245+0000","last_clean":"2024-03-06T07:30:37.301245+0000","last_became_active":"2024-03-06T07:30:36.310143+0000","last_became_peered":"2024-03-06T07:30:36.310143+0000","last_unstale":"2024-03-06T07:30:37.301245+0000","last_undegraded":"2024-03-06T07:30:37.301245+0000","last_fullsized":"2024-03-06T07:30:37.301245+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:51:33.540728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.301340+0000","last_change":"2024-03-06T07:30:35.299983+0000","last_active":"2024-03-06T07:30:37.301340+0000","last_peered":"2024-03-06T07:30:37.301340+0000","last_clean":"2024-03-06T07:30:37.301340+0000","last_became_active":"2024-03-06T07:30:35.299497+0000","last_became_peered":"2024-03-06T07:30:35.299497+0000","last_unstale":"2024-03-06T07:30:37.301340+0000","last_undegraded":"2024-03-06T07:30:37.301340+0000","last_fullsized":"2024-03-06T07:30:37.301340+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:15:20.775976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536172+0000","last_change":"2024-03-06T07:30:36.302922+0000","last_active":"2024-03-06T07:30:37.536172+0000","last_peered":"2024-03-06T07:30:37.536172+0000","last_clean":"2024-03-06T07:30:37.536172+0000","last_became_active":"2024-03-06T07:30:36.301723+0000","last_became_peered":"2024-03-06T07:30:36.301723+0000","last_unstale":"2024-03-06T07:30:37.536172+0000","last_undegraded":"2024-03-06T07:30:37.536172+0000","last_fullsized":"2024-03-06T07:30:37.536172+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:26:50.384334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.532588+0000","last_change":"2024-03-06T07:30:35.293888+0000","last_active":"2024-03-06T07:30:37.532588+0000","last_peered":"2024-03-06T07:30:37.532588+0000","last_clean":"2024-03-06T07:30:37.532588+0000","last_became_active":"2024-03-06T07:30:35.293603+0000","last_became_peered":"2024-03-06T07:30:35.293603+0000","last_unstale":"2024-03-06T07:30:37.532588+0000","last_undegraded":"2024-03-06T07:30:37.532588+0000","last_fullsized":"2024-03-06T07:30:37.532588+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:25:39.493483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.532497+0000","last_change":"2024-03-06T07:30:36.308280+0000","last_active":"2024-03-06T07:30:37.532497+0000","last_peered":"2024-03-06T07:30:37.532497+0000","last_clean":"2024-03-06T07:30:37.532497+0000","last_became_active":"2024-03-06T07:30:36.308009+0000","last_became_peered":"2024-03-06T07:30:36.308009+0000","last_unstale":"2024-03-06T07:30:37.532497+0000","last_undegraded":"2024-03-06T07:30:37.532497+0000","last_fullsized":"2024-03-06T07:30:37.532497+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:04:33.082440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.304445+0000","last_change":"2024-03-06T07:30:35.296475+0000","last_active":"2024-03-06T07:30:37.304445+0000","last_peered":"2024-03-06T07:30:37.304445+0000","last_clean":"2024-03-06T07:30:37.304445+0000","last_became_active":"2024-03-06T07:30:35.296145+0000","last_became_peered":"2024-03-06T07:30:35.296145+0000","last_unstale":"2024-03-06T07:30:37.304445+0000","last_undegraded":"2024-03-06T07:30:37.304445+0000","last_fullsized":"2024-03-06T07:30:37.304445+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:00:20.460415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535010+0000","last_change":"2024-03-06T07:30:36.314179+0000","last_active":"2024-03-06T07:30:37.535010+0000","last_peered":"2024-03-06T07:30:37.535010+0000","last_clean":"2024-03-06T07:30:37.535010+0000","last_became_active":"2024-03-06T07:30:36.313817+0000","last_became_peered":"2024-03-06T07:30:36.313817+0000","last_unstale":"2024-03-06T07:30:37.535010+0000","last_undegraded":"2024-03-06T07:30:37.535010+0000","last_fullsized":"2024-03-06T07:30:37.535010+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:47:11.267856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.535793+0000","last_change":"2024-03-06T07:30:35.295849+0000","last_active":"2024-03-06T07:30:37.535793+0000","last_peered":"2024-03-06T07:30:37.535793+0000","last_clean":"2024-03-06T07:30:37.535793+0000","last_became_active":"2024-03-06T07:30:35.295580+0000","last_became_peered":"2024-03-06T07:30:35.295580+0000","last_unstale":"2024-03-06T07:30:37.535793+0000","last_undegraded":"2024-03-06T07:30:37.535793+0000","last_fullsized":"2024-03-06T07:30:37.535793+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:42:13.715994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":12,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533757+0000","last_change":"2024-03-06T07:30:36.311167+0000","last_active":"2024-03-06T07:30:37.533757+0000","last_peered":"2024-03-06T07:30:37.533757+0000","last_clean":"2024-03-06T07:30:37.533757+0000","last_became_active":"2024-03-06T07:30:36.310875+0000","last_became_peered":"2024-03-06T07:30:36.310875+0000","last_unstale":"2024-03-06T07:30:37.533757+0000","last_undegraded":"2024-03-06T07:30:37.533757+0000","last_fullsized":"2024-03-06T07:30:37.533757+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:35.250081+0000","last_clean_scrub_stamp":"2024-03-06T07:30:35.250081+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:05:48.453685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.536320+0000","last_change":"2024-03-06T07:30:35.298086+0000","last_active":"2024-03-06T07:30:37.536320+0000","last_peered":"2024-03-06T07:30:37.536320+0000","last_clean":"2024-03-06T07:30:37.536320+0000","last_became_active":"2024-03-06T07:30:35.297849+0000","last_became_peered":"2024-03-06T07:30:35.297849+0000","last_unstale":"2024-03-06T07:30:37.536320+0000","last_undegraded":"2024-03-06T07:30:37.536320+0000","last_fullsized":"2024-03-06T07:30:37.536320+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:30:34.235003+0000","last_clean_scrub_stamp":"2024-03-06T07:30:34.235003+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:20:18.750764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":256,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-06T07:30:37.533617+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:30:37.533617+0000","last_peered":"2024-03-06T07:30:37.533617+0000","last_clean":"2024-03-06T07:30:37.533617+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:30:37.533617+0000","last_undegraded":"2024-03-06T07:30:37.533617+0000","last_fullsized":"2024-03-06T07:30:37.533617+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":10,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542192,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32596,"kb_used_data":1416,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339244,"statfs":{"total":96636764160,"available":96603385856,"internally_reserved":0,"allocated":1449984,"data_stored":945754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542191,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32020,"kb_used_data":836,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339820,"statfs":{"total":96636764160,"available":96603975680,"internally_reserved":0,"allocated":856064,"data_stored":354862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542192,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32016,"kb_used_data":832,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339824,"statfs":{"total":96636764160,"available":96603979776,"internally_reserved":0,"allocated":851968,"data_stored":354860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542191,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32020,"kb_used_data":832,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339820,"statfs":{"total":96636764160,"available":96603975680,"internally_reserved":0,"allocated":851968,"data_stored":354840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542191,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32016,"kb_used_data":836,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339824,"statfs":{"total":96636764160,"available":96603979776,"internally_reserved":0,"allocated":856064,"data_stored":354882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542192,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32028,"kb_used_data":840,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339812,"statfs":{"total":96636764160,"available":96603967488,"internally_reserved":0,"allocated":860160,"data_stored":356382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542193,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32028,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339812,"statfs":{"total":96636764160,"available":96603967488,"internally_reserved":0,"allocated":868352,"data_stored":356998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542190,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32588,"kb_used_data":1408,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339252,"statfs":{"total":96636764160,"available":96603394048,"internally_reserved":0,"allocated":1441792,"data_stored":945138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":92,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:30:41.325 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:30:41.326 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:30:41.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.567+0000 7f38e1600640 1 Processor -- start 2024-03-06T07:30:41.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.567+0000 7f38e1600640 1 -- start start 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38e1600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38dc0b3df0 0x7f38dc0727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38e1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 0x7f38dc06c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38e1600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f38dc0719e0 con 0x7f38dc0b3650 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38e1600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38dc072310 con 0x7f38dc0b3df0 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38e1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38dc071cc0 con 0x7f38dc0cda90 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38daa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38dc0b3df0 0x7f38dc0727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38daa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38dc0b3df0 0x7f38dc0727a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55782/0 (socket says 172.21.15.86:55782) 2024-03-06T07:30:41.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.569+0000 7f38daa00640 1 -- 172.21.15.86:0/266916590 learned_addr learned my addr 172.21.15.86:0/266916590 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:41.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.570+0000 7f38daa00640 1 -- 172.21.15.86:0/266916590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 msgr2=0x7f38dc06c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:41.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.570+0000 7f38daa00640 1 --2- 172.21.15.86:0/266916590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 0x7f38dc06c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.570+0000 7f38daa00640 1 -- 172.21.15.86:0/266916590 >> v1:172.21.15.194:6789/0 conn(0x7f38dc0b3650 legacy=0x7f38dc0b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:41.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.570+0000 7f38daa00640 1 -- 172.21.15.86:0/266916590 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38dc071fa0 con 0x7f38dc0b3df0 2024-03-06T07:30:41.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.571+0000 7f38daa00640 1 --2- 172.21.15.86:0/266916590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc0727a0 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7f38cc009870 tx=0x7f38cc02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f2cd72ebd135c769 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:41.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.571+0000 7f38da000640 1 -- 172.21.15.86:0/266916590 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38cc035020 con 0x7f38dc0b3df0 2024-03-06T07:30:41.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.571+0000 7f38da000640 1 -- 172.21.15.86:0/266916590 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f38cc031640 con 0x7f38dc0b3df0 2024-03-06T07:30:41.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.571+0000 7f38da000640 1 -- 172.21.15.86:0/266916590 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38cc02d040 con 0x7f38dc0b3df0 2024-03-06T07:30:41.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.571+0000 7f38e1600640 1 -- 172.21.15.86:0/266916590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 msgr2=0x7f38dc0727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.571+0000 7f38e1600640 1 --2- 172.21.15.86:0/266916590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc0727a0 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7f38cc009870 tx=0x7f38cc02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.572+0000 7f38e1600640 1 -- 172.21.15.86:0/266916590 shutdown_connections 2024-03-06T07:30:41.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.572+0000 7f38e1600640 1 --2- 172.21.15.86:0/266916590 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 0x7f38dc06c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.572+0000 7f38e1600640 1 --2- 172.21.15.86:0/266916590 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc0727a0 unknown :-1 s=CLOSED pgs=808 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.572+0000 7f38e1600640 1 -- 172.21.15.86:0/266916590 >> 172.21.15.86:0/266916590 conn(0x7f38dc0b1940 msgr2=0x7f38dc0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:41.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.572+0000 7f38e1600640 1 -- 172.21.15.86:0/266916590 shutdown_connections 2024-03-06T07:30:41.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.572+0000 7f38e1600640 1 -- 172.21.15.86:0/266916590 wait complete. 2024-03-06T07:30:41.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.573+0000 7f38e1600640 1 Processor -- start 2024-03-06T07:30:41.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.573+0000 7f38e1600640 1 -- start start 2024-03-06T07:30:41.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.574+0000 7f38e1600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38dc0b3650 0x7f38dc14e390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.574+0000 7f38e1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc155a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.574+0000 7f38e1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 0x7f38dc155f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.574+0000 7f38e1600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38dc15b3c0 con 0x7f38dc0b3df0 2024-03-06T07:30:41.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.574+0000 7f38e1600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38dc15b240 con 0x7f38dc0b3650 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.574+0000 7f38e1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38dc0cf510 con 0x7f38dc0cda90 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.574+0000 7f38d3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc155a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc155a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55788/0 (socket says 172.21.15.86:55788) 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 -- 172.21.15.86:0/2642573693 learned_addr learned my addr 172.21.15.86:0/2642573693 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38daa00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38dc0b3650 0x7f38dc14e390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 -- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 msgr2=0x7f38dc155f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38db400640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 0x7f38dc155f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 0x7f38dc155f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 -- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38dc0b3650 msgr2=0x7f38dc14e390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38dc0b3650 0x7f38dc14e390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 -- 172.21.15.86:0/2642573693 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38dc1512a0 con 0x7f38dc0b3df0 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38daa00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38dc0b3650 0x7f38dc14e390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.575+0000 7f38d3e00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc155a00 secure :-1 s=READY pgs=809 cs=0 l=1 rev1=1 crypto rx=0x7f38c400c440 tx=0x7f38c400c900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:41.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.576+0000 7f38d2a00640 1 -- 172.21.15.86:0/2642573693 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38c4017070 con 0x7f38dc0b3df0 2024-03-06T07:30:41.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.576+0000 7f38d2a00640 1 -- 172.21.15.86:0/2642573693 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f38c40035f0 con 0x7f38dc0b3df0 2024-03-06T07:30:41.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.576+0000 7f38d2a00640 1 -- 172.21.15.86:0/2642573693 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38c40039e0 con 0x7f38dc0b3df0 2024-03-06T07:30:41.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.576+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f38dc151540 con 0x7f38dc0b3df0 2024-03-06T07:30:41.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.576+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f38dc151b00 con 0x7f38dc0b3df0 2024-03-06T07:30:41.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.577+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f38dc06e800 con 0x7f38dc0b3df0 2024-03-06T07:30:41.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.578+0000 7f38d2a00640 1 -- 172.21.15.86:0/2642573693 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f38c4003b80 con 0x7f38dc0b3df0 2024-03-06T07:30:41.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.578+0000 7f38d2a00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f38b806b4d0 0x7f38b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:41.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.578+0000 7f38d2a00640 1 -- 172.21.15.86:0/2642573693 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f38c4089420 con 0x7f38dc0b3df0 2024-03-06T07:30:41.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.579+0000 7f38daa00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f38b806b4d0 0x7f38b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:41.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.579+0000 7f38daa00640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f38b806b4d0 0x7f38b806d990 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f38cc036000 tx=0x7f38cc03c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:41.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.583+0000 7f38d2a00640 1 -- 172.21.15.86:0/2642573693 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f38dc06e800 con 0x7f38dc0b3df0 2024-03-06T07:30:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.849+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f38dc15ad80 con 0x7f38dc0b3df0 2024-03-06T07:30:41.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.852+0000 7f38d2a00640 1 -- 172.21.15.86:0/2642573693 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v16) v1 ==== 190+0+147 (secure 0 0 0) 0x7f38c4058330 con 0x7f38dc0b3df0 2024-03-06T07:30:41.854 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:30:41.854 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:30:41.854 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:30:41.854 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:30:41.855 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:30:41.855 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:30:41.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f38b806b4d0 msgr2=0x7f38b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f38b806b4d0 0x7f38b806d990 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f38cc036000 tx=0x7f38cc03c040 comp rx=0 tx=0).stop 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 msgr2=0x7f38dc155a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc155a00 secure :-1 s=READY pgs=809 cs=0 l=1 rev1=1 crypto rx=0x7f38c400c440 tx=0x7f38c400c900 comp rx=0 tx=0).stop 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 shutdown_connections 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f38b806b4d0 0x7f38b806d990 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38dc0cda90 0x7f38dc155f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38dc0b3df0 0x7f38dc155a00 unknown :-1 s=CLOSED pgs=809 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 --2- 172.21.15.86:0/2642573693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38dc0b3650 0x7f38dc14e390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 >> 172.21.15.86:0/2642573693 conn(0x7f38dc0b1940 msgr2=0x7f38dc06d2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 shutdown_connections 2024-03-06T07:30:41.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:41.855+0000 7f38e1600640 1 -- 172.21.15.86:0/2642573693 wait complete. 2024-03-06T07:30:41.874 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:42.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.071+0000 7f960e600640 1 Processor -- start 2024-03-06T07:30:42.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.071+0000 7f960e600640 1 -- start start 2024-03-06T07:30:42.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f960e600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f96080d0fc0 0x7f9608160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f960e600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f96080cea60 con 0x7f960815f470 2024-03-06T07:30:42.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f960e600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f96080d2350 con 0x7f960815e0d0 2024-03-06T07:30:42.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f960e600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96080d2f60 con 0x7f96080d0fc0 2024-03-06T07:30:42.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f9606a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f960815f470 0x7f96080d08b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54842/0 (socket says 172.21.15.86:54842) 2024-03-06T07:30:42.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f9606a00640 1 -- 172.21.15.86:0/3702730051 learned_addr learned my addr 172.21.15.86:0/3702730051 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 344451644 0 0) 0x7f96080cea60 con 0x7f960815f470 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f95ec003520 con 0x7f960815f470 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.073+0000 7f9607e00640 1 --2- 172.21.15.86:0/3702730051 >> v2:172.21.15.194:3300/0 conn(0x7f96080d0fc0 0x7f9608160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1734745584 0 0) 0x7f95ec003520 con 0x7f960815f470 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 >> v2:172.21.15.194:3300/0 conn(0x7f96080d0fc0 msgr2=0x7f9608160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 --2- 172.21.15.86:0/3702730051 >> v2:172.21.15.194:3300/0 conn(0x7f96080d0fc0 0x7f9608160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 >> v1:172.21.15.194:6789/0 conn(0x7f960815e0d0 legacy=0x7f96080d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96080d2630 con 0x7f960815f470 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f95f4001670 con 0x7f960815f470 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 >> v1:172.21.15.86:6789/0 conn(0x7f960815f470 legacy=0x7f96080d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95ec003a00 0x7f95ec003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f95ec004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95ec005210 0x7f95ec00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95ec00c120 con 0x7f95ec004800 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95ec00c2d0 con 0x7f95ec003a00 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95ec00c4d0 con 0x7f95ec005210 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606a00640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f95ec004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606a00640 1 -- 172.21.15.86:0/3702730051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95ec005210 msgr2=0x7f95ec00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606a00640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f95ec005210 0x7f95ec00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9606a00640 1 -- 172.21.15.86:0/3702730051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95ec003a00 msgr2=0x7f95ec003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.074+0000 7f9607400640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95ec003a00 0x7f95ec003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.075+0000 7f9606a00640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f95ec003a00 0x7f95ec003e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.075+0000 7f9606a00640 1 -- 172.21.15.86:0/3702730051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95ec003520 con 0x7f95ec004800 2024-03-06T07:30:42.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.075+0000 7f9607e00640 1 -- 172.21.15.86:0/3702730051 reap_dead start 2024-03-06T07:30:42.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f9606a00640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f95ec004c80 secure :-1 s=READY pgs=811 cs=0 l=1 rev1=1 crypto rx=0x7f95f400c190 tx=0x7f95ec005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d2d6a5d548eb98f9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95f4007c60 con 0x7f95ec004800 2024-03-06T07:30:42.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95f4030070 con 0x7f95ec004800 2024-03-06T07:30:42.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f9606000640 1 -- 172.21.15.86:0/3702730051 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95f4005d90 con 0x7f95ec004800 2024-03-06T07:30:42.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f960e600640 1 -- 172.21.15.86:0/3702730051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 msgr2=0x7f95ec004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f960e600640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f95ec004c80 secure :-1 s=READY pgs=811 cs=0 l=1 rev1=1 crypto rx=0x7f95f400c190 tx=0x7f95ec005830 comp rx=0 tx=0).stop 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f960e600640 1 -- 172.21.15.86:0/3702730051 shutdown_connections 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f960e600640 1 --2- 172.21.15.86:0/3702730051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f95ec004c80 unknown :-1 s=CLOSED pgs=811 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f960e600640 1 -- 172.21.15.86:0/3702730051 >> 172.21.15.86:0/3702730051 conn(0x7f96080d3ec0 msgr2=0x7f96080d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.076+0000 7f960e600640 1 -- 172.21.15.86:0/3702730051 shutdown_connections 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.077+0000 7f960e600640 1 -- 172.21.15.86:0/3702730051 wait complete. 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.078+0000 7f960e600640 1 Processor -- start 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.078+0000 7f960e600640 1 -- start start 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f960e600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f96080c8a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f960e600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96080ca140 0x7f96081610d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f960e600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96081522d0 0x7f96080b1ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f960e600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f960815fb30 con 0x7f95ec004800 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f960e600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96080d2bf0 con 0x7f96081522d0 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f960e600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f960815e4b0 con 0x7f96080ca140 2024-03-06T07:30:42.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f9607400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f96080c8a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f9607400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f96080c8a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55812/0 (socket says 172.21.15.86:55812) 2024-03-06T07:30:42.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.079+0000 7f9607400640 1 -- 172.21.15.86:0/3454468629 learned_addr learned my addr 172.21.15.86:0/3454468629 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f9607400640 1 -- 172.21.15.86:0/3454468629 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96080ca140 msgr2=0x7f96081610d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f9607400640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96080ca140 0x7f96081610d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f9607400640 1 -- 172.21.15.86:0/3454468629 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96081522d0 msgr2=0x7f96080b1ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f9607400640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96081522d0 0x7f96080b1ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f9607400640 1 -- 172.21.15.86:0/3454468629 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9608152780 con 0x7f95ec004800 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f9607400640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f96080c8a10 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7f95f800d430 tx=0x7f95f800d8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f95ff400640 1 -- 172.21.15.86:0/3454468629 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95f8016020 con 0x7f95ec004800 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f95ff400640 1 -- 172.21.15.86:0/3454468629 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95f8012800 con 0x7f95ec004800 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f95ff400640 1 -- 172.21.15.86:0/3454468629 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95f8012c10 con 0x7f95ec004800 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96080c90a0 con 0x7f95ec004800 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.080+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96080c9660 con 0x7f95ec004800 2024-03-06T07:30:42.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.081+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f960804f330 con 0x7f95ec004800 2024-03-06T07:30:42.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.083+0000 7f95ff400640 1 -- 172.21.15.86:0/3454468629 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f95f8019020 con 0x7f95ec004800 2024-03-06T07:30:42.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.084+0000 7f95ff400640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95e806b4d0 0x7f95e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.085+0000 7f95ff400640 1 -- 172.21.15.86:0/3454468629 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f95f808a160 con 0x7f95ec004800 2024-03-06T07:30:42.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.085+0000 7f9606a00640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95e806b4d0 0x7f95e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.086+0000 7f9606a00640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95e806b4d0 0x7f95e806d990 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f95f400c160 tx=0x7f95f4007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.092+0000 7f95ff400640 1 -- 172.21.15.86:0/3454468629 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f95f8020020 con 0x7f95ec004800 2024-03-06T07:30:42.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.316+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f960808d920 con 0x7f95ec004800 2024-03-06T07:30:42.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.317+0000 7f95ff400640 1 -- 172.21.15.86:0/3454468629 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f95f8059070 con 0x7f95ec004800 2024-03-06T07:30:42.319 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:42.319 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:30:42.319 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95e806b4d0 msgr2=0x7f95e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95e806b4d0 0x7f95e806d990 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f95f400c160 tx=0x7f95f4007000 comp rx=0 tx=0).stop 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 msgr2=0x7f96080c8a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f96080c8a10 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7f95f800d430 tx=0x7f95f800d8f0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 shutdown_connections 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f95e806b4d0 0x7f95e806d990 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f96081522d0 0x7f96080b1ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f96080ca140 0x7f96081610d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 --2- 172.21.15.86:0/3454468629 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f95ec004800 0x7f96080c8a10 unknown :-1 s=CLOSED pgs=812 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.319+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 >> 172.21.15.86:0/3454468629 conn(0x7f96080d3ec0 msgr2=0x7f960815c090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:42.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.320+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 shutdown_connections 2024-03-06T07:30:42.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.320+0000 7f960e600640 1 -- 172.21.15.86:0/3454468629 wait complete. 2024-03-06T07:30:42.748 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 42, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:30:35.740469+0000', 'modified': '2024-03-06T07:30:38.790632+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 6204}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6204': {'gid': 6204, 'name': 'c', 'rank': 0, 'incarnation': 38, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/195049401', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 195049401}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 195049401}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [11], 'metadata_pool': 10, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 6204, 'qdb_cluster': [6204]} 2024-03-06T07:30:42.748 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:30:42.748 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-06T07:30:42.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.962+0000 7f76bcc00640 1 Processor -- start 2024-03-06T07:30:42.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.962+0000 7f76bcc00640 1 -- start start 2024-03-06T07:30:42.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76bcc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f76b80d2360 0x7f76b80d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76bcc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f76b80b3ec0 0x7f76b80b4310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76bcc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f76b8161960 con 0x7f76b8161580 2024-03-06T07:30:42.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76bcc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f76b80d2ce0 con 0x7f76b80b3ec0 2024-03-06T07:30:42.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76bcc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f76b80cf6f0 con 0x7f76b80d2360 2024-03-06T07:30:42.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76b7e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f76b80b3ec0 0x7f76b80b4310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76b7e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f76b80b3ec0 0x7f76b80b4310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55824/0 (socket says 172.21.15.86:55824) 2024-03-06T07:30:42.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76b7e00640 1 -- 172.21.15.86:0/624347510 learned_addr learned my addr 172.21.15.86:0/624347510 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:42.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.964+0000 7f76afe00640 1 --2- 172.21.15.86:0/624347510 >> v2:172.21.15.194:3300/0 conn(0x7f76b80d2360 0x7f76b80d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.965+0000 7f76b7e00640 1 -- 172.21.15.86:0/624347510 >> v2:172.21.15.194:3300/0 conn(0x7f76b80d2360 msgr2=0x7f76b80d27a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.965+0000 7f76b7e00640 1 --2- 172.21.15.86:0/624347510 >> v2:172.21.15.194:3300/0 conn(0x7f76b80d2360 0x7f76b80d27a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.965+0000 7f76b7e00640 1 -- 172.21.15.86:0/624347510 >> v1:172.21.15.194:6789/0 conn(0x7f76b8161580 legacy=0x7f76b80d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.965+0000 7f76b7e00640 1 -- 172.21.15.86:0/624347510 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76b80cf9d0 con 0x7f76b80b3ec0 2024-03-06T07:30:42.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.966+0000 7f76b7e00640 1 --2- 172.21.15.86:0/624347510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b4310 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7f76b80d2e60 tx=0x7f76b002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c03514dab928410 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.966+0000 7f76b6a00640 1 -- 172.21.15.86:0/624347510 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b002ca70 con 0x7f76b80b3ec0 2024-03-06T07:30:42.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.966+0000 7f76b6a00640 1 -- 172.21.15.86:0/624347510 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76b002cc10 con 0x7f76b80b3ec0 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.966+0000 7f76b6a00640 1 -- 172.21.15.86:0/624347510 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b00343f0 con 0x7f76b80b3ec0 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.966+0000 7f76bcc00640 1 -- 172.21.15.86:0/624347510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80b3ec0 msgr2=0x7f76b80b4310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.966+0000 7f76bcc00640 1 --2- 172.21.15.86:0/624347510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b4310 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7f76b80d2e60 tx=0x7f76b002bed0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.967+0000 7f76bcc00640 1 -- 172.21.15.86:0/624347510 shutdown_connections 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.967+0000 7f76bcc00640 1 --2- 172.21.15.86:0/624347510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b4310 unknown :-1 s=CLOSED pgs=813 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.967+0000 7f76bcc00640 1 --2- 172.21.15.86:0/624347510 >> v2:172.21.15.194:3300/0 conn(0x7f76b80d2360 0x7f76b80d27a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.967+0000 7f76bcc00640 1 -- 172.21.15.86:0/624347510 >> 172.21.15.86:0/624347510 conn(0x7f76b80d3d80 msgr2=0x7f76b80d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.967+0000 7f76bcc00640 1 -- 172.21.15.86:0/624347510 shutdown_connections 2024-03-06T07:30:42.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.967+0000 7f76bcc00640 1 -- 172.21.15.86:0/624347510 wait complete. 2024-03-06T07:30:42.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.968+0000 7f76bcc00640 1 Processor -- start 2024-03-06T07:30:42.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.969+0000 7f76bcc00640 1 -- start start 2024-03-06T07:30:42.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.969+0000 7f76bcc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b6940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.969+0000 7f76bcc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80d2360 0x7f76b815ae40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76bcc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76b8161580 0x7f76b80c7410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76bcc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76b80b39a0 con 0x7f76b80d2360 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76bcc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76b80b36c0 con 0x7f76b80b3ec0 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76bcc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76b80cf3a0 con 0x7f76b8161580 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80d2360 0x7f76b815ae40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80d2360 0x7f76b815ae40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55828/0 (socket says 172.21.15.86:55828) 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 -- 172.21.15.86:0/170967429 learned_addr learned my addr 172.21.15.86:0/170967429 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76b7400640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b6940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 -- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76b8161580 msgr2=0x7f76b80c7410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76b8161580 0x7f76b80c7410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 -- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76b80b3ec0 msgr2=0x7f76b80b6940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b6940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76afe00640 1 -- 172.21.15.86:0/170967429 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76b815b380 con 0x7f76b80d2360 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.970+0000 7f76b7400640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b6940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.971+0000 7f76afe00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80d2360 0x7f76b815ae40 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f76a000d5f0 tx=0x7f76a000dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.971+0000 7f76af400640 1 -- 172.21.15.86:0/170967429 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76a00125f0 con 0x7f76b80d2360 2024-03-06T07:30:42.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.971+0000 7f76af400640 1 -- 172.21.15.86:0/170967429 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76a0019070 con 0x7f76b80d2360 2024-03-06T07:30:42.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.971+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f76b80cbe30 con 0x7f76b80d2360 2024-03-06T07:30:42.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.971+0000 7f76af400640 1 -- 172.21.15.86:0/170967429 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76a00129d0 con 0x7f76b80d2360 2024-03-06T07:30:42.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.971+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f76b80cc3f0 con 0x7f76b80d2360 2024-03-06T07:30:42.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.973+0000 7f76af400640 1 -- 172.21.15.86:0/170967429 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f76a0012b70 con 0x7f76b80d2360 2024-03-06T07:30:42.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.974+0000 7f76af400640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f769406b4d0 0x7f769406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.974+0000 7f76af400640 1 -- 172.21.15.86:0/170967429 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f76a008a660 con 0x7f76b80d2360 2024-03-06T07:30:42.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.974+0000 7f76b7400640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f769406b4d0 0x7f769406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.975+0000 7f76c0f9f740 1 Processor -- start 2024-03-06T07:30:42.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.975+0000 7f76c0f9f740 1 -- start start 2024-03-06T07:30:42.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.975+0000 7f76b7400640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f769406b4d0 0x7f769406d990 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f76a800a830 tx=0x7f76a8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b48e990 0x560b2b461930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b491720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b461e70 0x560b2b491c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560b2b3cb600 con 0x560b2b48e990 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560b2b374930 con 0x560b2b48ed70 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560b2b3c76a0 con 0x560b2b461e70 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76b7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b48e990 0x560b2b461930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76b7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b48e990 0x560b2b461930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55840/0 (socket says 172.21.15.86:55840) 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.976+0000 7f76b7400640 1 -- 172.21.15.86:0/2132446775 learned_addr learned my addr 172.21.15.86:0/2132446775 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76b7400640 1 -- 172.21.15.86:0/2132446775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b461e70 msgr2=0x560b2b491c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76b7400640 1 --2- 172.21.15.86:0/2132446775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b461e70 0x560b2b491c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76b7400640 1 -- 172.21.15.86:0/2132446775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 msgr2=0x560b2b491720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76b7400640 1 --2- 172.21.15.86:0/2132446775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b491720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76b7400640 1 -- 172.21.15.86:0/2132446775 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560b2b0e02d0 con 0x560b2b48e990 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76b7400640 1 --2- 172.21.15.86:0/2132446775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b48e990 0x560b2b461930 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f76a800ca90 tx=0x7f76a800f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a874cfbff816077c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76a7400640 1 -- 172.21.15.86:0/2132446775 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76a800ae60 con 0x560b2b48e990 2024-03-06T07:30:42.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76a7400640 1 -- 172.21.15.86:0/2132446775 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76a8013d50 con 0x560b2b48e990 2024-03-06T07:30:42.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.977+0000 7f76a7400640 1 -- 172.21.15.86:0/2132446775 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76a8004260 con 0x560b2b48e990 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2132446775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b48e990 msgr2=0x560b2b461930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2132446775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b48e990 0x560b2b461930 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f76a800ca90 tx=0x7f76a800f000 comp rx=0 tx=0).stop 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2132446775 shutdown_connections 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2132446775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b461e70 0x560b2b491c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2132446775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b491720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2132446775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b48e990 0x560b2b461930 unknown :-1 s=CLOSED pgs=815 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2132446775 >> 172.21.15.86:0/2132446775 conn(0x560b2b463ba0 msgr2=0x560b2b4ac720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2132446775 shutdown_connections 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.978+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2132446775 wait complete. 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.979+0000 7f76c0f9f740 1 -- start start 2024-03-06T07:30:42.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.979+0000 7f76c0f9f740 1 Processor -- start 2024-03-06T07:30:42.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48e990 0x560b2b4941b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b4a7060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b47c4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560b2b466140 con 0x560b2b4a75a0 2024-03-06T07:30:42.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560b2b387d90 con 0x560b2b48ed70 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560b2b465530 con 0x560b2b48e990 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76c0f9f740 10 client.0 _ll_register_callbacks cb 0x560b2b3c3350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b47c4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b47c4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55842/0 (socket says 172.21.15.86:55842) 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.980+0000 7f76b7e00640 1 -- 172.21.15.86:0/1952406276 learned_addr learned my addr 172.21.15.86:0/1952406276 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76afe00640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b4a7060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76b7400640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48e990 0x560b2b4941b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76b7e00640 1 -- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48e990 msgr2=0x560b2b4941b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76b7e00640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48e990 0x560b2b4941b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76b7e00640 1 -- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 msgr2=0x560b2b4a7060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76b7e00640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b4a7060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76b7e00640 1 -- 172.21.15.86:0/1952406276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560b2b1aa6b0 con 0x560b2b4a75a0 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76afe00640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b4a7060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76b7e00640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b47c4d0 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f76b0009f10 tx=0x7f76b00071f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5864ba879892a2ef server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:42.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f769be00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f769be00640 1 -- 172.21.15.86:0/1952406276 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b0043020 con 0x560b2b4a75a0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f76a8013d50 con 0x560b2b4a75a0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.981+0000 7f769be00640 1 -- 172.21.15.86:0/1952406276 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f76b003e660 con 0x560b2b4a75a0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f769be00640 1 -- 172.21.15.86:0/1952406276 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=42) v2 ==== 24+0+0 (secure 0 0 0) 0x7f76b004d050 con 0x560b2b4a75a0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 10 client.6348 fetch_fsmap learned FSMap version 42 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=42,monmap=2+}) v3 -- 0x560b2b4a3770 con 0x560b2b4a75a0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f769be00640 1 -- 172.21.15.86:0/1952406276 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76a8013d50 con 0x560b2b4a75a0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f769be00640 1 -- 172.21.15.86:0/1952406276 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 42) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f76b0050020 con 0x560b2b4a75a0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 10 client.6348 fetch_fsmap finished waiting for FSMap version 42 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 10 client.6348 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 20 client.6348 populate_metadata read hostname 'smithi086' 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b47d6f0 0x560b2b4c0560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 4 client.6348 mds_command: new command op to 6204 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 --> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560b2b3d8b40 con 0x560b2b47d6f0 2024-03-06T07:30:42.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.982+0000 7f76b7400640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b47d6f0 0x560b2b4c0560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:42.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.983+0000 7f76b7400640 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b47d6f0 0x560b2b4c0560 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4b01237143786532 server_cookie=dd617b5c605171cf in_seq=0 out_seq=0 2024-03-06T07:30:42.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.983+0000 7f769be00640 10 client.6348 ms_handle_connect on v2:172.21.15.86:6834/195049401 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f769be00640 1 -- 172.21.15.86:0/1952406276 <== mds.0 v2:172.21.15.86:6834/195049401 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x560b2b3d8b40 con 0x560b2b47d6f0 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f769be00640 10 client.6348 handle_command_reply: tid=0 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 client.6348 shutdown 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 msgr2=0x560b2b47c4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b47c4d0 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f76b0009f10 tx=0x7f76b00071f0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 shutdown_connections 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b47d6f0 0x560b2b4c0560 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b47c4d0 unknown :-1 s=CLOSED pgs=816 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48ed70 0x560b2b4a7060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/1952406276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48e990 0x560b2b4941b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:42.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 >> 172.21.15.86:0/1952406276 conn(0x560b2b461e70 msgr2=0x560b2b4afc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:42.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f769be00640 0 client.6348 ms_handle_reset on v2:172.21.15.86:6834/195049401 2024-03-06T07:30:42.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 shutdown_connections 2024-03-06T07:30:42.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 1 -- 172.21.15.86:0/1952406276 wait complete. 2024-03-06T07:30:42.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:42.984+0000 7f76c0f9f740 20 client.6348 trim_cache size 0 max 16384 2024-03-06T07:30:43.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.003+0000 7f76c0f9f740 1 Processor -- start 2024-03-06T07:30:43.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.003+0000 7f76c0f9f740 1 -- start start 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.004+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 0x560b2b4bc0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.004+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4607c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.004+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b460d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.004+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76b81ff430 con 0x560b2b4a75a0 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.004+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76b81fe830 con 0x560b2b48e990 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.004+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76b81fee30 con 0x560b2b48ed70 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b460d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b460d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55848/0 (socket says 172.21.15.86:55848) 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 -- 172.21.15.86:0/3600324512 learned_addr learned my addr 172.21.15.86:0/3600324512 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 -- 172.21.15.86:0/3600324512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 msgr2=0x560b2b4607c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 --2- 172.21.15.86:0/3600324512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4607c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 -- 172.21.15.86:0/3600324512 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 msgr2=0x560b2b4bc0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:43.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 --2- 172.21.15.86:0/3600324512 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 0x560b2b4bc0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 -- 172.21.15.86:0/3600324512 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560b2b48f500 con 0x560b2b4a75a0 2024-03-06T07:30:43.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76b7e00640 1 --2- 172.21.15.86:0/3600324512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b460d00 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7f76b0009c10 tx=0x7f76b003f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=521e9705e4211491 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:43.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76a6a00640 1 -- 172.21.15.86:0/3600324512 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b0043070 con 0x560b2b4a75a0 2024-03-06T07:30:43.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76a6a00640 1 -- 172.21.15.86:0/3600324512 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76b0003b90 con 0x560b2b4a75a0 2024-03-06T07:30:43.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.005+0000 7f76a6a00640 1 -- 172.21.15.86:0/3600324512 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b003e660 con 0x560b2b4a75a0 2024-03-06T07:30:43.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 -- 172.21.15.86:0/3600324512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 msgr2=0x560b2b460d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/3600324512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b460d00 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7f76b0009c10 tx=0x7f76b003f000 comp rx=0 tx=0).stop 2024-03-06T07:30:43.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 -- 172.21.15.86:0/3600324512 shutdown_connections 2024-03-06T07:30:43.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/3600324512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b460d00 unknown :-1 s=CLOSED pgs=817 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/3600324512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4607c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/3600324512 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 0x560b2b4bc0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 -- 172.21.15.86:0/3600324512 >> 172.21.15.86:0/3600324512 conn(0x560b2b461e70 msgr2=0x560b2b493fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:43.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 -- 172.21.15.86:0/3600324512 shutdown_connections 2024-03-06T07:30:43.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.006+0000 7f76c0f9f740 1 -- 172.21.15.86:0/3600324512 wait complete. 2024-03-06T07:30:43.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.007+0000 7f76c0f9f740 1 -- start start 2024-03-06T07:30:43.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.007+0000 7f76c0f9f740 1 Processor -- start 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.008+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 0x560b2b51ba60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.008+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4aff20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.008+0000 7f76c0f9f740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b4bf620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.008+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560b2b47e160 con 0x560b2b4a75a0 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.008+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x560b2b4a3b10 con 0x560b2b48e990 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.008+0000 7f76c0f9f740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560b2b47dad0 con 0x560b2b48ed70 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.008+0000 7f76c0f9f740 10 client.0 _ll_register_callbacks cb 0x7f76b0009d50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b4bf620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b4bf620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55862/0 (socket says 172.21.15.86:55862) 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 -- 172.21.15.86:0/2974748317 learned_addr learned my addr 172.21.15.86:0/2974748317 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7400640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 0x560b2b51ba60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76afe00640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4aff20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 -- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 msgr2=0x560b2b4aff20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4aff20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 -- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 msgr2=0x560b2b51ba60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 0x560b2b51ba60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 -- 172.21.15.86:0/2974748317 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560b2b47f3e0 con 0x560b2b4a75a0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76afe00640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4aff20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f76b7e00640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b4bf620 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7f76b00071c0 tx=0x7f76b003c000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=30ade5140e7e2903 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.009+0000 7f769be00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f769be00640 1 -- 172.21.15.86:0/2974748317 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b0057040 con 0x560b2b4a75a0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f769be00640 1 -- 172.21.15.86:0/2974748317 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f76b0003990 con 0x560b2b4a75a0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f76b0003b90 con 0x560b2b4a75a0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f769be00640 1 -- 172.21.15.86:0/2974748317 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=42) v2 ==== 24+0+0 (secure 0 0 0) 0x7f76b0067050 con 0x560b2b4a75a0 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 10 client.6354 fetch_fsmap learned FSMap version 42 2024-03-06T07:30:43.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=42,monmap=2+}) v3 -- 0x560b2b47c440 con 0x560b2b4a75a0 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f769be00640 1 -- 172.21.15.86:0/2974748317 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76b0003b90 con 0x560b2b4a75a0 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f769be00640 1 -- 172.21.15.86:0/2974748317 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 42) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f76b006a020 con 0x560b2b4a75a0 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 10 client.6354 fetch_fsmap finished waiting for FSMap version 42 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 10 client.6354 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 20 client.6354 populate_metadata read hostname 'smithi086' 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b460e30 0x560b2b466630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 4 client.6354 mds_command: new command op to 6204 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.010+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 --> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] -- command(tid 0: {"prefix": "status"}) v1 -- 0x560b2b494c50 con 0x560b2b460e30 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.011+0000 7f76b7400640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b460e30 0x560b2b466630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.011+0000 7f76b7400640 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b460e30 0x560b2b466630 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a3e3f22d84d3fef4 server_cookie=5080607df67db798 in_seq=0 out_seq=0 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.011+0000 7f769be00640 10 client.6354 ms_handle_connect on v2:172.21.15.86:6834/195049401 2024-03-06T07:30:43.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.011+0000 7f769be00640 1 -- 172.21.15.86:0/2974748317 <== mds.0 v2:172.21.15.86:6834/195049401 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+317 (crc 0 0 0) 0x560b2b494c50 con 0x560b2b460e30 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.011+0000 7f769be00640 10 client.6354 handle_command_reply: tid=0 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.011+0000 7f76c0f9f740 1 client.6354 shutdown 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 msgr2=0x560b2b4bf620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b4bf620 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7f76b00071c0 tx=0x7f76b003c000 comp rx=0 tx=0).stop 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 shutdown_connections 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:6834/195049401,v1:172.21.15.86:6837/195049401] conn(0x560b2b460e30 0x560b2b466630 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x560b2b4a75a0 0x560b2b4bf620 unknown :-1 s=CLOSED pgs=818 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560b2b48ed70 0x560b2b4aff20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 --2- 172.21.15.86:0/2974748317 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560b2b48e990 0x560b2b51ba60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f769be00640 0 client.6354 ms_handle_reset on v2:172.21.15.86:6834/195049401 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 >> 172.21.15.86:0/2974748317 conn(0x560b2b461e70 msgr2=0x560b2b4657b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 shutdown_connections 2024-03-06T07:30:43.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 1 -- 172.21.15.86:0/2974748317 wait complete. 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.012+0000 7f76c0f9f740 20 client.6354 trim_cache size 0 max 16384 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 6204, 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 6.717417889, 2024-03-06T07:30:43.018 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 42, 2024-03-06T07:30:43.019 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 70, 2024-03-06T07:30:43.019 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 70, 2024-03-06T07:30:43.019 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 13.16799529 2024-03-06T07:30:43.019 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f769406b4d0 msgr2=0x7f769406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f769406b4d0 0x7f769406d990 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f76a800a830 tx=0x7f76a8008040 comp rx=0 tx=0).stop 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80d2360 msgr2=0x7f76b815ae40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80d2360 0x7f76b815ae40 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f76a000d5f0 tx=0x7f76a000dab0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 shutdown_connections 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f769406b4d0 0x7f769406d990 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f76b8161580 0x7f76b80c7410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76b80d2360 0x7f76b815ae40 unknown :-1 s=CLOSED pgs=814 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 --2- 172.21.15.86:0/170967429 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f76b80b3ec0 0x7f76b80b6940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 >> 172.21.15.86:0/170967429 conn(0x7f76b80d3d80 msgr2=0x7f76b8160360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 shutdown_connections 2024-03-06T07:30:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.013+0000 7f76bcc00640 1 -- 172.21.15.86:0/170967429 wait complete. 2024-03-06T07:30:43.033 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:30:43.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.236+0000 7f3cb6a00640 1 Processor -- start 2024-03-06T07:30:43.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.236+0000 7f3cb6a00640 1 -- start start 2024-03-06T07:30:43.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cb6a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3cb015f4f0 0x7f3cb00706d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cb6a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3cb0070c10 0x7f3cb00cc700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cb6a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3cb0160a20 con 0x7f3cb015f8d0 2024-03-06T07:30:43.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cb6a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3cb006ef50 con 0x7f3cb0070c10 2024-03-06T07:30:43.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cb6a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3cb006f230 con 0x7f3cb015f4f0 2024-03-06T07:30:43.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3ca7400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3cb0070c10 0x7f3cb00cc700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cb4a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3cb015f8d0 0x7f3cb015fd50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54852/0 (socket says 172.21.15.86:54852) 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cb4a00640 1 -- 172.21.15.86:0/1044808325 learned_addr learned my addr 172.21.15.86:0/1044808325 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.238+0000 7f3cafe00640 1 --2- 172.21.15.86:0/1044808325 >> v2:172.21.15.194:3300/0 conn(0x7f3cb015f4f0 0x7f3cb00706d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.239+0000 7f3ca7400640 1 -- 172.21.15.86:0/1044808325 >> v2:172.21.15.194:3300/0 conn(0x7f3cb015f4f0 msgr2=0x7f3cb00706d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.239+0000 7f3ca7400640 1 --2- 172.21.15.86:0/1044808325 >> v2:172.21.15.194:3300/0 conn(0x7f3cb015f4f0 0x7f3cb00706d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.239+0000 7f3ca7400640 1 -- 172.21.15.86:0/1044808325 >> v1:172.21.15.86:6789/0 conn(0x7f3cb015f8d0 legacy=0x7f3cb015fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.239+0000 7f3ca7400640 1 -- 172.21.15.86:0/1044808325 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3cb0160460 con 0x7f3cb0070c10 2024-03-06T07:30:43.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.239+0000 7f3caf400640 1 -- 172.21.15.86:0/1044808325 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1990165605 0 0) 0x7f3cb0160a20 con 0x7f3cb015f8d0 2024-03-06T07:30:43.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.240+0000 7f3ca7400640 1 --2- 172.21.15.86:0/1044808325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb0070c10 0x7f3cb00cc700 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7f3cb006bf30 tx=0x7f3c9802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c365ace082de3350 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:43.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.240+0000 7f3caf400640 1 -- 172.21.15.86:0/1044808325 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c98035020 con 0x7f3cb0070c10 2024-03-06T07:30:43.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.240+0000 7f3caf400640 1 -- 172.21.15.86:0/1044808325 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c98031830 con 0x7f3cb0070c10 2024-03-06T07:30:43.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.240+0000 7f3caf400640 1 -- 172.21.15.86:0/1044808325 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c98031c00 con 0x7f3cb0070c10 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 -- 172.21.15.86:0/1044808325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb0070c10 msgr2=0x7f3cb00cc700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/1044808325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb0070c10 0x7f3cb00cc700 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7f3cb006bf30 tx=0x7f3c9802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 -- 172.21.15.86:0/1044808325 shutdown_connections 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/1044808325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb0070c10 0x7f3cb00cc700 unknown :-1 s=CLOSED pgs=820 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/1044808325 >> v2:172.21.15.194:3300/0 conn(0x7f3cb015f4f0 0x7f3cb00706d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 -- 172.21.15.86:0/1044808325 >> 172.21.15.86:0/1044808325 conn(0x7f3cb00b3800 msgr2=0x7f3cb00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 -- 172.21.15.86:0/1044808325 shutdown_connections 2024-03-06T07:30:43.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.241+0000 7f3cb6a00640 1 -- 172.21.15.86:0/1044808325 wait complete. 2024-03-06T07:30:43.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.243+0000 7f3cb6a00640 1 Processor -- start 2024-03-06T07:30:43.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.243+0000 7f3cb6a00640 1 -- start start 2024-03-06T07:30:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cb6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb015f4f0 0x7f3cb01f42f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cb6a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cb015f8d0 0x7f3cb01f78a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cb6a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cb01f7de0 0x7f3cb00b6410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cb6a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3cb00ccdc0 con 0x7f3cb015f4f0 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cb6a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3cb00ccc40 con 0x7f3cb015f8d0 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cb6a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3cb00ccf40 con 0x7f3cb01f7de0 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cafe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb015f4f0 0x7f3cb01f42f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cafe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb015f4f0 0x7f3cb01f42f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55874/0 (socket says 172.21.15.86:55874) 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3cafe00640 1 -- 172.21.15.86:0/3194649873 learned_addr learned my addr 172.21.15.86:0/3194649873 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.244+0000 7f3ca7400640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cb015f8d0 0x7f3cb01f78a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.245+0000 7f3cb4a00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cb01f7de0 0x7f3cb00b6410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.245+0000 7f3cafe00640 1 -- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cb01f7de0 msgr2=0x7f3cb00b6410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.245+0000 7f3cafe00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cb01f7de0 0x7f3cb00b6410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.245+0000 7f3cafe00640 1 -- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cb015f8d0 msgr2=0x7f3cb01f78a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.245+0000 7f3cafe00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cb015f8d0 0x7f3cb01f78a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.245+0000 7f3cafe00640 1 -- 172.21.15.86:0/3194649873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3cb01f7660 con 0x7f3cb015f4f0 2024-03-06T07:30:43.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.245+0000 7f3cafe00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb015f4f0 0x7f3cb01f42f0 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7f3ca000d5f0 tx=0x7f3ca000dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:43.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.246+0000 7f3cacc00640 1 -- 172.21.15.86:0/3194649873 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ca00125f0 con 0x7f3cb015f4f0 2024-03-06T07:30:43.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.246+0000 7f3cacc00640 1 -- 172.21.15.86:0/3194649873 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ca0019070 con 0x7f3cb015f4f0 2024-03-06T07:30:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.246+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3cb01e5bb0 con 0x7f3cb015f4f0 2024-03-06T07:30:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.246+0000 7f3cacc00640 1 -- 172.21.15.86:0/3194649873 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ca00129d0 con 0x7f3cb015f4f0 2024-03-06T07:30:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.246+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3cb01e6170 con 0x7f3cb015f4f0 2024-03-06T07:30:43.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.247+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3cb01f81f0 con 0x7f3cb015f4f0 2024-03-06T07:30:43.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.250+0000 7f3cacc00640 1 -- 172.21.15.86:0/3194649873 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3ca0012b70 con 0x7f3cb015f4f0 2024-03-06T07:30:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.250+0000 7f3cacc00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c8c06b500 0x7f3c8c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.251+0000 7f3cacc00640 1 -- 172.21.15.86:0/3194649873 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3ca008a550 con 0x7f3cb015f4f0 2024-03-06T07:30:43.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.251+0000 7f3ca7400640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c8c06b500 0x7f3c8c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:43.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.251+0000 7f3ca7400640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c8c06b500 0x7f3c8c06d9c0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f3c98036000 tx=0x7f3c9803c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:43.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.255+0000 7f3cacc00640 1 -- 172.21.15.86:0/3194649873 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3cb01f81f0 con 0x7f3cb015f4f0 2024-03-06T07:30:43.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.614+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f3cb004f330 con 0x7f3cb015f4f0 2024-03-06T07:30:43.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.615+0000 7f3cacc00640 1 -- 172.21.15.86:0/3194649873 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v42) v1 ==== 53+0+73 (secure 0 0 0) 0x7f3ca00593d0 con 0x7f3cb015f4f0 2024-03-06T07:30:43.617 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c8c06b500 msgr2=0x7f3c8c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c8c06b500 0x7f3c8c06d9c0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f3c98036000 tx=0x7f3c9803c040 comp rx=0 tx=0).stop 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb015f4f0 msgr2=0x7f3cb01f42f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb015f4f0 0x7f3cb01f42f0 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7f3ca000d5f0 tx=0x7f3ca000dab0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 shutdown_connections 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c8c06b500 0x7f3c8c06d9c0 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cb01f7de0 0x7f3cb00b6410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cb015f8d0 0x7f3cb01f78a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 --2- 172.21.15.86:0/3194649873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cb015f4f0 0x7f3cb01f42f0 unknown :-1 s=CLOSED pgs=821 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:43.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 >> 172.21.15.86:0/3194649873 conn(0x7f3cb00b3800 msgr2=0x7f3cb00cd980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:43.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 shutdown_connections 2024-03-06T07:30:43.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:43.618+0000 7f3cb6a00640 1 -- 172.21.15.86:0/3194649873 wait complete. 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:30:43.639 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:30:43.639 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:43.639 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:30:43.684 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42223sec preferred_lft 42223sec 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:43.685 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:30:43.686 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:30:43.687 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:43.687 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:30:43.812 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:30:43.816 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:43.816 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:30:43.864 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:30:43.864 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:30:43.864 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:30:43.864 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:30:43.864 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:30:43.864 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:43.864 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:30:43.865 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:30:43.865 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:30:43.865 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:30:43.865 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:30:43.865 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:30:44.025 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:44.025 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:30:44.073 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:30:44.073 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:30:44.074 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:30:44.074 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:44.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:30:44.160 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:30:44.160 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:44.208 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:30:44.209 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:30:44.209 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:44.299 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:30:44.300 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:44.300 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:30:44.428 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:30:44.429 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:30:44.461 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:30:44.461 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:30:44.461 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:30:44.461 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:30:44.461 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:30:44.467 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:30:44.467 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:30:44.467 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:30:44.467 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:30:44.467 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:30:44.471 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:30:44.471 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:44.471 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:30:44.552 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:30:44.552 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:30:44.611 INFO:teuthology.orchestra.run.smithi086.stdout:client5566 2024-03-06T07:30:44.611 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:44.611 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:44.657 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_fragment_size_max 10 2024-03-06T07:30:44.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.909+0000 7fcc2ae00640 1 Processor -- start 2024-03-06T07:30:44.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.909+0000 7fcc2ae00640 1 -- start start 2024-03-06T07:30:44.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.911+0000 7fcc2ae00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fcc240b3f70 0x7fcc24160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.911+0000 7fcc2ae00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fcc24070ff0 0x7fcc240ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.911+0000 7fcc2ae00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc24071440 con 0x7fcc2415fc10 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.911+0000 7fcc2ae00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc240b46c0 con 0x7fcc24070ff0 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.911+0000 7fcc2ae00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc240b49a0 con 0x7fcc240b3f70 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.911+0000 7fcc28e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fcc24070ff0 0x7fcc240ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc28e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fcc24070ff0 0x7fcc240ce4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55884/0 (socket says 172.21.15.86:55884) 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc28e00640 1 -- 172.21.15.86:0/3841807131 learned_addr learned my addr 172.21.15.86:0/3841807131 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc23400640 1 --2- 172.21.15.86:0/3841807131 >> v2:172.21.15.194:3300/0 conn(0x7fcc240b3f70 0x7fcc24160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc28e00640 1 -- 172.21.15.86:0/3841807131 >> v2:172.21.15.194:3300/0 conn(0x7fcc240b3f70 msgr2=0x7fcc24160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc28e00640 1 --2- 172.21.15.86:0/3841807131 >> v2:172.21.15.194:3300/0 conn(0x7fcc240b3f70 0x7fcc24160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc28e00640 1 -- 172.21.15.86:0/3841807131 >> v1:172.21.15.86:6789/0 conn(0x7fcc2415fc10 legacy=0x7fcc2415fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc22a00640 1 -- 172.21.15.86:0/3841807131 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 252345860 0 0) 0x7fcc24071440 con 0x7fcc2415fc10 2024-03-06T07:30:44.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.912+0000 7fcc28e00640 1 -- 172.21.15.86:0/3841807131 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc24072c90 con 0x7fcc24070ff0 2024-03-06T07:30:44.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.914+0000 7fcc28e00640 1 --2- 172.21.15.86:0/3841807131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc24070ff0 0x7fcc240ce4a0 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7fcc24074a50 tx=0x7fcc1c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f34252f1c1a19b9d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:44.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.914+0000 7fcc22a00640 1 -- 172.21.15.86:0/3841807131 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1c034070 con 0x7fcc24070ff0 2024-03-06T07:30:44.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.914+0000 7fcc22a00640 1 -- 172.21.15.86:0/3841807131 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc1c030860 con 0x7fcc24070ff0 2024-03-06T07:30:44.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.914+0000 7fcc22a00640 1 -- 172.21.15.86:0/3841807131 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1c030c30 con 0x7fcc24070ff0 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 -- 172.21.15.86:0/3841807131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc24070ff0 msgr2=0x7fcc240ce4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/3841807131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc24070ff0 0x7fcc240ce4a0 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7fcc24074a50 tx=0x7fcc1c02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 -- 172.21.15.86:0/3841807131 shutdown_connections 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/3841807131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc24070ff0 0x7fcc240ce4a0 unknown :-1 s=CLOSED pgs=823 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/3841807131 >> v2:172.21.15.194:3300/0 conn(0x7fcc240b3f70 0x7fcc24160700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 -- 172.21.15.86:0/3841807131 >> 172.21.15.86:0/3841807131 conn(0x7fcc240b1940 msgr2=0x7fcc240b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 -- 172.21.15.86:0/3841807131 shutdown_connections 2024-03-06T07:30:44.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.915+0000 7fcc2ae00640 1 -- 172.21.15.86:0/3841807131 wait complete. 2024-03-06T07:30:44.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.917+0000 7fcc2ae00640 1 Processor -- start 2024-03-06T07:30:44.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.917+0000 7fcc2ae00640 1 -- start start 2024-03-06T07:30:44.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc2ae00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcc24070ff0 0x7fcc241f2760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc2ae00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc240b3f70 0x7fcc2409bfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc2ae00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcc2415fc10 0x7fcc2409c520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc2ae00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc24072a30 con 0x7fcc240b3f70 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc2ae00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc240728b0 con 0x7fcc2415fc10 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc2ae00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc2415c2e0 con 0x7fcc24070ff0 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc23400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc240b3f70 0x7fcc2409bfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc23400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc240b3f70 0x7fcc2409bfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55890/0 (socket says 172.21.15.86:55890) 2024-03-06T07:30:44.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc23400640 1 -- 172.21.15.86:0/2164684865 learned_addr learned my addr 172.21.15.86:0/2164684865 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:44.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.918+0000 7fcc23e00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcc24070ff0 0x7fcc241f2760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:44.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc28e00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcc2415fc10 0x7fcc2409c520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:44.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc23400640 1 -- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcc24070ff0 msgr2=0x7fcc241f2760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc23400640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcc24070ff0 0x7fcc241f2760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc23400640 1 -- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcc2415fc10 msgr2=0x7fcc2409c520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc23400640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcc2415fc10 0x7fcc2409c520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc23400640 1 -- 172.21.15.86:0/2164684865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc2415f530 con 0x7fcc240b3f70 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc23400640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc240b3f70 0x7fcc2409bfe0 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7fcc1000d7d0 tx=0x7fcc1000dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc1be00640 1 -- 172.21.15.86:0/2164684865 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1000b6c0 con 0x7fcc240b3f70 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc1be00640 1 -- 172.21.15.86:0/2164684865 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc10019070 con 0x7fcc240b3f70 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc241f2ca0 con 0x7fcc240b3f70 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc1be00640 1 -- 172.21.15.86:0/2164684865 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1000ba80 con 0x7fcc240b3f70 2024-03-06T07:30:44.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.919+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc2415ef70 con 0x7fcc240b3f70 2024-03-06T07:30:44.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.921+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc240b27d0 con 0x7fcc240b3f70 2024-03-06T07:30:44.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.921+0000 7fcc1be00640 1 -- 172.21.15.86:0/2164684865 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcc1000bc20 con 0x7fcc240b3f70 2024-03-06T07:30:44.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.922+0000 7fcc1be00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcc0406b500 0x7fcc0406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:44.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.922+0000 7fcc1be00640 1 -- 172.21.15.86:0/2164684865 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcc1008a630 con 0x7fcc240b3f70 2024-03-06T07:30:44.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.923+0000 7fcc23e00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcc0406b500 0x7fcc0406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:44.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.923+0000 7fcc23e00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcc0406b500 0x7fcc0406d9c0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fcc14007fd0 tx=0x7fcc14009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:44.928 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:44.927+0000 7fcc1be00640 1 -- 172.21.15.86:0/2164684865 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcc240b27d0 con 0x7fcc240b3f70 2024-03-06T07:30:45.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.130+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mds_bal_fragment_size_max}] v 0) v1 -- 0x7fcc2406ef30 con 0x7fcc240b3f70 2024-03-06T07:30:45.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.136+0000 7fcc1be00640 1 -- 172.21.15.86:0/2164684865 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_fragment_size_max}]=0 v23)=0 v23) v1 ==== 124+0+0 (secure 0 0 0) 0x7fcc241f2ca0 con 0x7fcc240b3f70 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcc0406b500 msgr2=0x7fcc0406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcc0406b500 0x7fcc0406d9c0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fcc14007fd0 tx=0x7fcc14009040 comp rx=0 tx=0).stop 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc240b3f70 msgr2=0x7fcc2409bfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc240b3f70 0x7fcc2409bfe0 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7fcc1000d7d0 tx=0x7fcc1000dc90 comp rx=0 tx=0).stop 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 shutdown_connections 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcc0406b500 0x7fcc0406d9c0 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcc2415fc10 0x7fcc2409c520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcc240b3f70 0x7fcc2409bfe0 unknown :-1 s=CLOSED pgs=824 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 --2- 172.21.15.86:0/2164684865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcc24070ff0 0x7fcc241f2760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:45.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 >> 172.21.15.86:0/2164684865 conn(0x7fcc240b1940 msgr2=0x7fcc240cc690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:45.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 shutdown_connections 2024-03-06T07:30:45.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:45.139+0000 7fcc2ae00640 1 -- 172.21.15.86:0/2164684865 wait complete. 2024-03-06T07:30:53.391 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:30:53.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:30:55.161 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:55.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.373+0000 7fde2d400640 1 Processor -- start 2024-03-06T07:30:55.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.373+0000 7fde2d400640 1 -- start start 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde2d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde280aedc0 0x7fde280cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde2d400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde2815c240 con 0x7fde280cc800 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde2d400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde2815c500 con 0x7fde281561f0 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde2d400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fde2815ce30 con 0x7fde280aedc0 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde27400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde280aedc0 0x7fde280cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde27400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde280aedc0 0x7fde280cc2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45420/0 (socket says 172.21.15.86:45420) 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde27400640 1 -- 172.21.15.86:0/634670697 learned_addr learned my addr 172.21.15.86:0/634670697 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:55.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.375+0000 7fde2ca00640 1 --1- 172.21.15.86:0/634670697 >> v1:172.21.15.86:6789/0 conn(0x7fde280cc800 0x7fde2815f830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39882/0 (socket says 172.21.15.86:39882) 2024-03-06T07:30:55.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.376+0000 7fde27400640 1 -- 172.21.15.86:0/634670697 >> v1:172.21.15.194:6789/0 conn(0x7fde281561f0 legacy=0x7fde280ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:55.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.376+0000 7fde27400640 1 -- 172.21.15.86:0/634670697 >> v1:172.21.15.86:6789/0 conn(0x7fde280cc800 legacy=0x7fde2815f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:55.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.376+0000 7fde27400640 1 -- 172.21.15.86:0/634670697 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde2815c7e0 con 0x7fde280aedc0 2024-03-06T07:30:55.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.376+0000 7fde26a00640 1 -- 172.21.15.86:0/634670697 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1377725935 0 0) 0x7fde2815c240 con 0x7fde280cc800 2024-03-06T07:30:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.377+0000 7fde27400640 1 --2- 172.21.15.86:0/634670697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde280aedc0 0x7fde280cc2c0 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fde2815cfb0 tx=0x7fde1002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=97c5adee07277805 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde26a00640 1 -- 172.21.15.86:0/634670697 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde10034070 con 0x7fde280aedc0 2024-03-06T07:30:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde26a00640 1 -- 172.21.15.86:0/634670697 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde10030840 con 0x7fde280aedc0 2024-03-06T07:30:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde26a00640 1 -- 172.21.15.86:0/634670697 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde10030c10 con 0x7fde280aedc0 2024-03-06T07:30:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde2d400640 1 -- 172.21.15.86:0/634670697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde280aedc0 msgr2=0x7fde280cc2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde2d400640 1 --2- 172.21.15.86:0/634670697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde280aedc0 0x7fde280cc2c0 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fde2815cfb0 tx=0x7fde1002b980 comp rx=0 tx=0).stop 2024-03-06T07:30:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde2d400640 1 -- 172.21.15.86:0/634670697 shutdown_connections 2024-03-06T07:30:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde2d400640 1 --2- 172.21.15.86:0/634670697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde280aedc0 0x7fde280cc2c0 unknown :-1 s=CLOSED pgs=826 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde2d400640 1 -- 172.21.15.86:0/634670697 >> 172.21.15.86:0/634670697 conn(0x7fde2806afc0 msgr2=0x7fde2806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:55.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.378+0000 7fde2d400640 1 -- 172.21.15.86:0/634670697 shutdown_connections 2024-03-06T07:30:55.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.379+0000 7fde2d400640 1 -- 172.21.15.86:0/634670697 wait complete. 2024-03-06T07:30:55.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.380+0000 7fde2d400640 1 Processor -- start 2024-03-06T07:30:55.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.380+0000 7fde2d400640 1 -- start start 2024-03-06T07:30:55.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.381+0000 7fde2d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde280aedc0 0x7fde280ae6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:55.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.381+0000 7fde2d400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde280cc800 0x7fde2815b250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:55.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.381+0000 7fde2d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde281561f0 0x7fde28145110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:55.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.381+0000 7fde2d400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde280ccc30 con 0x7fde281561f0 2024-03-06T07:30:55.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.381+0000 7fde2d400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde2806e360 con 0x7fde280cc800 2024-03-06T07:30:55.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.381+0000 7fde2d400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde281565d0 con 0x7fde280aedc0 2024-03-06T07:30:55.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde281561f0 0x7fde28145110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:55.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde281561f0 0x7fde28145110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45422/0 (socket says 172.21.15.86:45422) 2024-03-06T07:30:55.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 -- 172.21.15.86:0/1538467373 learned_addr learned my addr 172.21.15.86:0/1538467373 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde27400640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde280cc800 0x7fde2815b250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde27e00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde280aedc0 0x7fde280ae6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 -- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde280aedc0 msgr2=0x7fde280ae6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde280aedc0 0x7fde280ae6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 -- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde280cc800 msgr2=0x7fde2815b250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde280cc800 0x7fde2815b250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 -- 172.21.15.86:0/1538467373 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde280ad9b0 con 0x7fde281561f0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde27e00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde280aedc0 0x7fde280ae6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.382+0000 7fde2ca00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde281561f0 0x7fde28145110 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7fde2000c7a0 tx=0x7fde2000cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.383+0000 7fde1fe00640 1 -- 172.21.15.86:0/1538467373 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde20018070 con 0x7fde281561f0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.383+0000 7fde1fe00640 1 -- 172.21.15.86:0/1538467373 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde2000a640 con 0x7fde281561f0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.383+0000 7fde1fe00640 1 -- 172.21.15.86:0/1538467373 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde2000aab0 con 0x7fde281561f0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.383+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde280adc30 con 0x7fde281561f0 2024-03-06T07:30:55.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.383+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde280aded0 con 0x7fde281561f0 2024-03-06T07:30:55.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.384+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde2804f330 con 0x7fde281561f0 2024-03-06T07:30:55.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.385+0000 7fde1fe00640 1 -- 172.21.15.86:0/1538467373 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fde2000ac50 con 0x7fde281561f0 2024-03-06T07:30:55.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.386+0000 7fde1fe00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde0406b4d0 0x7fde0406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:55.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.386+0000 7fde1fe00640 1 -- 172.21.15.86:0/1538467373 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fde20089960 con 0x7fde281561f0 2024-03-06T07:30:55.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.386+0000 7fde27e00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde0406b4d0 0x7fde0406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:55.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.387+0000 7fde27e00640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde0406b4d0 0x7fde0406d990 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fde1800a830 tx=0x7fde18008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:55.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.391+0000 7fde1fe00640 1 -- 172.21.15.86:0/1538467373 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fde280ad9b0 con 0x7fde281561f0 2024-03-06T07:30:55.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.621+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fde2806ce70 con 0x7fde281561f0 2024-03-06T07:30:55.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.622+0000 7fde1fe00640 1 -- 172.21.15.86:0/1538467373 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fde20058700 con 0x7fde281561f0 2024-03-06T07:30:55.624 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:55.624 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:30:55.624 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:30:55.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde0406b4d0 msgr2=0x7fde0406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:55.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde0406b4d0 0x7fde0406d990 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fde1800a830 tx=0x7fde18008040 comp rx=0 tx=0).stop 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde281561f0 msgr2=0x7fde28145110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde281561f0 0x7fde28145110 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7fde2000c7a0 tx=0x7fde2000cc60 comp rx=0 tx=0).stop 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 shutdown_connections 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde0406b4d0 0x7fde0406d990 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde281561f0 0x7fde28145110 unknown :-1 s=CLOSED pgs=827 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde280cc800 0x7fde2815b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 --2- 172.21.15.86:0/1538467373 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde280aedc0 0x7fde280ae6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 >> 172.21.15.86:0/1538467373 conn(0x7fde2806afc0 msgr2=0x7fde281550e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 shutdown_connections 2024-03-06T07:30:55.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:55.625+0000 7fde2d400640 1 -- 172.21.15.86:0/1538467373 wait complete. 2024-03-06T07:30:55.644 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_created 2024-03-06T07:30:55.862 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_created":0}} 2024-03-06T07:30:55.862 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 0 } } 2024-03-06T07:30:55.863 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:30:55.863 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:30:55.910 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:30:55.910 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> import uuid 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> n = 100 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = False 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> if 0 > 0: 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = True 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> dpath = os.path.dirname(path) 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> fnameprefix = os.path.basename(path) 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> os.makedirs(dpath, exist_ok=True) 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> try: 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:30:55.911 DEBUG:teuthology.orchestra.run.smithi086:> for i in range(n): 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> with open(fpath, '"'"'w'"'"') as f: 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> f.write(f"{i}") 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> f.flush() 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(f.fileno()) 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> if True: 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> os.unlink(fpath) 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> if create_hard_links: 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> for j in range(0): 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> if True: 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:30:55.912 DEBUG:teuthology.orchestra.run.smithi086:> finally: 2024-03-06T07:30:55.913 DEBUG:teuthology.orchestra.run.smithi086:> os.close(dirfd) 2024-03-06T07:30:55.913 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:30:56.163 INFO:teuthology.orchestra.run.smithi086.stderr:Traceback (most recent call last): 2024-03-06T07:30:56.163 INFO:teuthology.orchestra.run.smithi086.stderr: File "", line 26, in 2024-03-06T07:30:56.163 INFO:teuthology.orchestra.run.smithi086.stderr:OSError: [Errno 28] No space left on device: '/home/ubuntu/cephtest/mnt.0/subdir/file_10' 2024-03-06T07:30:56.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56 stdin-killer INFO: command exited with status 1: exiting normally with same code! 2024-03-06T07:30:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:30:56.185 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:56.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.423+0000 7f8409600640 1 Processor -- start 2024-03-06T07:30:56.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.423+0000 7f8409600640 1 -- start start 2024-03-06T07:30:56.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8409600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 0x7f840406d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8409600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f840415c240 con 0x7f84041561f0 2024-03-06T07:30:56.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8409600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f840415c500 con 0x7f84040aedc0 2024-03-06T07:30:56.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8409600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f840415ce30 con 0x7f84040cc9d0 2024-03-06T07:30:56.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8403e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f84041561f0 0x7f84040ae6b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37192/0 (socket says 172.21.15.86:37192) 2024-03-06T07:30:56.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8403e00640 1 -- 172.21.15.86:0/3896131332 learned_addr learned my addr 172.21.15.86:0/3896131332 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8408c00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 0x7f840406d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.425+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1928555854 0 0) 0x7f840415c240 con 0x7f84041561f0 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f83e8003520 con 0x7f84041561f0 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 73333025 0 0) 0x7f83e8003520 con 0x7f84041561f0 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 msgr2=0x7f840406d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 0x7f840406d350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 >> v1:172.21.15.194:6789/0 conn(0x7f84040aedc0 legacy=0x7f84040cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f840415c7e0 con 0x7f84041561f0 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f83f4001670 con 0x7f84041561f0 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 >> v1:172.21.15.86:6789/0 conn(0x7f84041561f0 legacy=0x7f84040ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 0x7f83e8003a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8003f80 0x7f83e8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83e8004d40 0x7f83e80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f840415ce30 con 0x7f83e8004d40 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83e800bad0 con 0x7f83e8003f80 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f83e800bcd0 con 0x7f84040cc9d0 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.426+0000 7f8408c00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83e8004d40 0x7f83e80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8403e00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 0x7f83e8003a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8403400640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8003f80 0x7f83e8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8408c00640 1 -- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 msgr2=0x7f83e8003a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8408c00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84040cc9d0 0x7f83e8003a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8408c00640 1 -- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8003f80 msgr2=0x7f83e8004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8408c00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8003f80 0x7f83e8004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8408c00640 1 -- 172.21.15.86:0/3896131332 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83e8003520 con 0x7f83e8004d40 2024-03-06T07:30:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.427+0000 7f8403400640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8003f80 0x7f83e8004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.428+0000 7f8408c00640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83e8004d40 0x7f83e80051c0 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7f83fc009f90 tx=0x7f83e8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6811520871fdc2f3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.428+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83fc039070 con 0x7f83e8004d40 2024-03-06T07:30:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.428+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f83fc02cbd0 con 0x7f83e8004d40 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.428+0000 7f8402a00640 1 -- 172.21.15.86:0/3896131332 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83fc009c10 con 0x7f83e8004d40 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.429+0000 7f8409600640 1 -- 172.21.15.86:0/3896131332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83e8004d40 msgr2=0x7f83e80051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.429+0000 7f8409600640 1 --2- 172.21.15.86:0/3896131332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83e8004d40 0x7f83e80051c0 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7f83fc009f90 tx=0x7f83e8005830 comp rx=0 tx=0).stop 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.429+0000 7f8408c00640 1 -- 172.21.15.86:0/3896131332 reap_dead start 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.429+0000 7f8409600640 1 -- 172.21.15.86:0/3896131332 shutdown_connections 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.429+0000 7f8409600640 1 -- 172.21.15.86:0/3896131332 >> 172.21.15.86:0/3896131332 conn(0x7f840406afc0 msgr2=0x7f840406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.429+0000 7f8409600640 1 -- 172.21.15.86:0/3896131332 shutdown_connections 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.429+0000 7f8409600640 1 -- 172.21.15.86:0/3896131332 wait complete. 2024-03-06T07:30:56.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.431+0000 7f8409600640 1 Processor -- start 2024-03-06T07:30:56.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.431+0000 7f8409600640 1 -- start start 2024-03-06T07:30:56.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8409600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8004d40 0x7f8404144270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8409600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f840415b2c0 0x7f84040cd440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8409600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84040cddd0 0x7f84040ce260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8409600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8404156750 con 0x7f84040cddd0 2024-03-06T07:30:56.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8409600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84041565d0 con 0x7f83e8004d40 2024-03-06T07:30:56.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8409600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f840406d350 con 0x7f840415b2c0 2024-03-06T07:30:56.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8408c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84040cddd0 0x7f84040ce260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:56.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8408c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84040cddd0 0x7f84040ce260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48786/0 (socket says 172.21.15.86:48786) 2024-03-06T07:30:56.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.432+0000 7f8408c00640 1 -- 172.21.15.86:0/1828688547 learned_addr learned my addr 172.21.15.86:0/1828688547 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:56.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8403e00640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8004d40 0x7f8404144270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:56.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8408c00640 1 -- 172.21.15.86:0/1828688547 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f840415b2c0 msgr2=0x7f84040cd440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:56.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8408c00640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f840415b2c0 0x7f84040cd440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8408c00640 1 -- 172.21.15.86:0/1828688547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8004d40 msgr2=0x7f8404144270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8408c00640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8004d40 0x7f8404144270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8408c00640 1 -- 172.21.15.86:0/1828688547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f840415ddf0 con 0x7f84040cddd0 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8408c00640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84040cddd0 0x7f84040ce260 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f83fc00e230 tx=0x7f83fc009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f83fbe00640 1 -- 172.21.15.86:0/1828688547 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83fc043020 con 0x7f84040cddd0 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f83fbe00640 1 -- 172.21.15.86:0/1828688547 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f83fc03a040 con 0x7f84040cddd0 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f83fbe00640 1 -- 172.21.15.86:0/1828688547 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83fc039070 con 0x7f84040cddd0 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f840415e0b0 con 0x7f84040cddd0 2024-03-06T07:30:56.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.433+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f840415e670 con 0x7f84040cddd0 2024-03-06T07:30:56.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.435+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f840404f330 con 0x7f84040cddd0 2024-03-06T07:30:56.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.435+0000 7f83fbe00640 1 -- 172.21.15.86:0/1828688547 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f83fc046070 con 0x7f84040cddd0 2024-03-06T07:30:56.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.436+0000 7f83fbe00640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83e406b500 0x7f83e406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:56.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.436+0000 7f83fbe00640 1 -- 172.21.15.86:0/1828688547 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f83fc0abd10 con 0x7f84040cddd0 2024-03-06T07:30:56.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.437+0000 7f8403e00640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83e406b500 0x7f83e406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:56.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.437+0000 7f8403e00640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83e406b500 0x7f83e406d9c0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f83f4001cc0 tx=0x7f83f4001d70 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:56.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.441+0000 7f83fbe00640 1 -- 172.21.15.86:0/1828688547 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f83fc07a4e0 con 0x7f84040cddd0 2024-03-06T07:30:56.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.695+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f84040b2b10 con 0x7f84040cddd0 2024-03-06T07:30:56.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.696+0000 7f83fbe00640 1 -- 172.21.15.86:0/1828688547 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f83fc07e580 con 0x7f84040cddd0 2024-03-06T07:30:56.698 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:56.698 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:30:56.698 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:30:56.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83e406b500 msgr2=0x7f83e406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:56.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83e406b500 0x7f83e406d9c0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f83f4001cc0 tx=0x7f83f4001d70 comp rx=0 tx=0).stop 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84040cddd0 msgr2=0x7f84040ce260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84040cddd0 0x7f84040ce260 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f83fc00e230 tx=0x7f83fc009750 comp rx=0 tx=0).stop 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 shutdown_connections 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83e406b500 0x7f83e406d9c0 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84040cddd0 0x7f84040ce260 unknown :-1 s=CLOSED pgs=830 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f840415b2c0 0x7f84040cd440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 --2- 172.21.15.86:0/1828688547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83e8004d40 0x7f8404144270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 >> 172.21.15.86:0/1828688547 conn(0x7f840406afc0 msgr2=0x7f840406d6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 shutdown_connections 2024-03-06T07:30:56.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:56.699+0000 7f8409600640 1 -- 172.21.15.86:0/1828688547 wait complete. 2024-03-06T07:30:56.720 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_created 2024-03-06T07:30:56.937 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_created":10}} 2024-03-06T07:30:56.937 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 10 } } 2024-03-06T07:30:56.938 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:30:57.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.148+0000 7f9a57e00640 1 Processor -- start 2024-03-06T07:30:57.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.148+0000 7f9a57e00640 1 -- start start 2024-03-06T07:30:57.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.149+0000 7f9a57e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a5006ec00 0x7f9a500ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.149+0000 7f9a57e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9a5015b090 con 0x7f9a5015acb0 2024-03-06T07:30:57.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.149+0000 7f9a57e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9a500b2a30 con 0x7f9a500b4680 2024-03-06T07:30:57.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.149+0000 7f9a57e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a500b4360 con 0x7f9a5006ec00 2024-03-06T07:30:57.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.150+0000 7f9a55400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9a5015acb0 0x7f9a500b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37198/0 (socket says 172.21.15.86:37198) 2024-03-06T07:30:57.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.150+0000 7f9a55400640 1 -- 172.21.15.86:0/29585318 learned_addr learned my addr 172.21.15.86:0/29585318 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:30:57.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.150+0000 7f9a55e00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a5006ec00 0x7f9a500ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:57.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.150+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3792771812 0 0) 0x7f9a5015b090 con 0x7f9a5015acb0 2024-03-06T07:30:57.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.150+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9a38005730 con 0x7f9a5015acb0 2024-03-06T07:30:57.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.150+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3469966732 0 0) 0x7f9a38005730 con 0x7f9a5015acb0 2024-03-06T07:30:57.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a5006ec00 msgr2=0x7f9a500ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a5006ec00 0x7f9a500ccfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 >> v1:172.21.15.194:6789/0 conn(0x7f9a500b4680 legacy=0x7f9a5006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a500b2cf0 con 0x7f9a5015acb0 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9a44001670 con 0x7f9a5015acb0 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 >> v1:172.21.15.86:6789/0 conn(0x7f9a5015acb0 legacy=0x7f9a500b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a38005bf0 0x7f9a38006010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a38006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a5006ec00 0x7f9a380073d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a500b4360 con 0x7f9a38006a10 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a3800dcb0 con 0x7f9a38005bf0 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a3800deb0 con 0x7f9a5006ec00 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a54a00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a38006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:57.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a55400640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a38005bf0 0x7f9a38006010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:57.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a54a00640 1 -- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a5006ec00 msgr2=0x7f9a380073d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:57.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a54a00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a5006ec00 0x7f9a380073d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a54a00640 1 -- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a38005bf0 msgr2=0x7f9a38006010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:30:57.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a54a00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a38005bf0 0x7f9a38006010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a54a00640 1 -- 172.21.15.86:0/29585318 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a38005730 con 0x7f9a38006a10 2024-03-06T07:30:57.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.151+0000 7f9a55400640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a38005bf0 0x7f9a38006010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:30:57.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.152+0000 7f9a54a00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a38006e90 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f9a3c001720 tx=0x7f9a38007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=580275e9330df2df server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:57.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a3c001370 con 0x7f9a38006a10 2024-03-06T07:30:57.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9a3c0052d0 con 0x7f9a38006a10 2024-03-06T07:30:57.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a4fe00640 1 -- 172.21.15.86:0/29585318 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a3c003830 con 0x7f9a38006a10 2024-03-06T07:30:57.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a57e00640 1 -- 172.21.15.86:0/29585318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 msgr2=0x7f9a38006e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:57.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a57e00640 1 --2- 172.21.15.86:0/29585318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a38006e90 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f9a3c001720 tx=0x7f9a38007a40 comp rx=0 tx=0).stop 2024-03-06T07:30:57.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a55e00640 1 -- 172.21.15.86:0/29585318 reap_dead start 2024-03-06T07:30:57.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a57e00640 1 -- 172.21.15.86:0/29585318 shutdown_connections 2024-03-06T07:30:57.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a57e00640 1 -- 172.21.15.86:0/29585318 >> 172.21.15.86:0/29585318 conn(0x7f9a500b1730 msgr2=0x7f9a500b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:57.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a57e00640 1 -- 172.21.15.86:0/29585318 shutdown_connections 2024-03-06T07:30:57.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.153+0000 7f9a57e00640 1 -- 172.21.15.86:0/29585318 wait complete. 2024-03-06T07:30:57.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.155+0000 7f9a57e00640 1 Processor -- start 2024-03-06T07:30:57.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.155+0000 7f9a57e00640 1 -- start start 2024-03-06T07:30:57.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a57e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a501544b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a57e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a501446b0 0x7f9a501549f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a57e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a50152570 0x7f9a501529f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a57e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a500b4010 con 0x7f9a38006a10 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a57e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a500b32b0 con 0x7f9a50152570 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a57e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a50072340 con 0x7f9a501446b0 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a55400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a501544b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a55400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a501544b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48792/0 (socket says 172.21.15.86:48792) 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.156+0000 7f9a55400640 1 -- 172.21.15.86:0/442382888 learned_addr learned my addr 172.21.15.86:0/442382888 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a55400640 1 -- 172.21.15.86:0/442382888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a501446b0 msgr2=0x7f9a501549f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a55400640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a501446b0 0x7f9a501549f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a55400640 1 -- 172.21.15.86:0/442382888 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a50152570 msgr2=0x7f9a501529f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a55400640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a50152570 0x7f9a501529f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a55400640 1 -- 172.21.15.86:0/442382888 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a50153260 con 0x7f9a38006a10 2024-03-06T07:30:57.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a55400640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a501544b0 secure :-1 s=READY pgs=833 cs=0 l=1 rev1=1 crypto rx=0x7f9a44000ff0 tx=0x7f9a4400dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:57.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a4d600640 1 -- 172.21.15.86:0/442382888 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a44019070 con 0x7f9a38006a10 2024-03-06T07:30:57.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a4d600640 1 -- 172.21.15.86:0/442382888 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9a440074e0 con 0x7f9a38006a10 2024-03-06T07:30:57.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a4d600640 1 -- 172.21.15.86:0/442382888 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a440049e0 con 0x7f9a38006a10 2024-03-06T07:30:57.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9a50153540 con 0x7f9a38006a10 2024-03-06T07:30:57.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.157+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9a50153b00 con 0x7f9a38006a10 2024-03-06T07:30:57.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.159+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9a5004f330 con 0x7f9a38006a10 2024-03-06T07:30:57.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.159+0000 7f9a4d600640 1 -- 172.21.15.86:0/442382888 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9a4401c070 con 0x7f9a38006a10 2024-03-06T07:30:57.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.160+0000 7f9a4d600640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a3006b4d0 0x7f9a3006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:30:57.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.160+0000 7f9a4d600640 1 -- 172.21.15.86:0/442382888 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9a44089bc0 con 0x7f9a38006a10 2024-03-06T07:30:57.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.161+0000 7f9a54a00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a3006b4d0 0x7f9a3006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:30:57.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.161+0000 7f9a54a00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a3006b4d0 0x7f9a3006d990 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f9a3c033000 tx=0x7f9a3c02f000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:30:57.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.165+0000 7f9a4d600640 1 -- 172.21.15.86:0/442382888 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9a44058310 con 0x7f9a38006a10 2024-03-06T07:30:57.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.399+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9a500d0850 con 0x7f9a38006a10 2024-03-06T07:30:57.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.399+0000 7f9a4d600640 1 -- 172.21.15.86:0/442382888 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f9a4405c3b0 con 0x7f9a38006a10 2024-03-06T07:30:57.402 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:30:57.402 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:30:57.402 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:30:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a3006b4d0 msgr2=0x7f9a3006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a3006b4d0 0x7f9a3006d990 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f9a3c033000 tx=0x7f9a3c02f000 comp rx=0 tx=0).stop 2024-03-06T07:30:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 msgr2=0x7f9a501544b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:30:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a501544b0 secure :-1 s=READY pgs=833 cs=0 l=1 rev1=1 crypto rx=0x7f9a44000ff0 tx=0x7f9a4400dd40 comp rx=0 tx=0).stop 2024-03-06T07:30:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 shutdown_connections 2024-03-06T07:30:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a3006b4d0 0x7f9a3006d990 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a50152570 0x7f9a501529f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a501446b0 0x7f9a501549f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 --2- 172.21.15.86:0/442382888 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a38006a10 0x7f9a501544b0 unknown :-1 s=CLOSED pgs=833 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:30:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.403+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 >> 172.21.15.86:0/442382888 conn(0x7f9a500b1730 msgr2=0x7f9a500b0e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:30:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.404+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 shutdown_connections 2024-03-06T07:30:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:30:57.404+0000 7f9a57e00640 1 -- 172.21.15.86:0/442382888 wait complete. 2024-03-06T07:30:57.427 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:30:57.647 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-06T07:30:57.647 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-06T07:30:57.647 DEBUG:tasks.ceph_test_case:wait_until_equal: 0 != 10, waiting (timeout=60)... 2024-03-06T07:31:02.648 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:02.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.857+0000 7f8a5ce00640 1 Processor -- start 2024-03-06T07:31:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.857+0000 7f8a5ce00640 1 -- start start 2024-03-06T07:31:02.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a5ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5806d2e0 0x7f8a580cc2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:02.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a5ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8a580cc7e0 0x7f8a580cd460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:02.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a5ce00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8a5815ae90 con 0x7f8a5815aab0 2024-03-06T07:31:02.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a5ce00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8a5806ce50 con 0x7f8a580cc7e0 2024-03-06T07:31:02.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a5ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8a580b3160 con 0x7f8a5806d2e0 2024-03-06T07:31:02.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a57e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8a580cc7e0 0x7f8a580cd460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:02.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a57e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8a580cc7e0 0x7f8a580cd460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48804/0 (socket says 172.21.15.86:48804) 2024-03-06T07:31:02.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a57e00640 1 -- 172.21.15.86:0/630236185 learned_addr learned my addr 172.21.15.86:0/630236185 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:02.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.859+0000 7f8a56a00640 1 --2- 172.21.15.86:0/630236185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5806d2e0 0x7f8a580cc2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:02.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.860+0000 7f8a57e00640 1 -- 172.21.15.86:0/630236185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5806d2e0 msgr2=0x7f8a580cc2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:02.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.860+0000 7f8a57e00640 1 --2- 172.21.15.86:0/630236185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5806d2e0 0x7f8a580cc2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:02.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.860+0000 7f8a57e00640 1 -- 172.21.15.86:0/630236185 >> v1:172.21.15.194:6789/0 conn(0x7f8a5815aab0 legacy=0x7f8a580b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:02.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.860+0000 7f8a57e00640 1 -- 172.21.15.86:0/630236185 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a580b28e0 con 0x7f8a580cc7e0 2024-03-06T07:31:02.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.861+0000 7f8a57e00640 1 --2- 172.21.15.86:0/630236185 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a580cc7e0 0x7f8a580cd460 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f8a50009e50 tx=0x7f8a5002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b8b94aaeb6c9ee42 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:02.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.861+0000 7f8a56000640 1 -- 172.21.15.86:0/630236185 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a5002ca70 con 0x7f8a580cc7e0 2024-03-06T07:31:02.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.861+0000 7f8a56000640 1 -- 172.21.15.86:0/630236185 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8a5002cc10 con 0x7f8a580cc7e0 2024-03-06T07:31:02.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.861+0000 7f8a56000640 1 -- 172.21.15.86:0/630236185 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a500343f0 con 0x7f8a580cc7e0 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.861+0000 7f8a5ce00640 1 -- 172.21.15.86:0/630236185 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a580cc7e0 msgr2=0x7f8a580cd460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.861+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/630236185 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a580cc7e0 0x7f8a580cd460 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f8a50009e50 tx=0x7f8a5002bed0 comp rx=0 tx=0).stop 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.862+0000 7f8a5ce00640 1 -- 172.21.15.86:0/630236185 shutdown_connections 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.862+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/630236185 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a580cc7e0 0x7f8a580cd460 unknown :-1 s=CLOSED pgs=834 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.862+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/630236185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5806d2e0 0x7f8a580cc2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.862+0000 7f8a5ce00640 1 -- 172.21.15.86:0/630236185 >> 172.21.15.86:0/630236185 conn(0x7f8a5806afc0 msgr2=0x7f8a5806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.862+0000 7f8a5ce00640 1 -- 172.21.15.86:0/630236185 shutdown_connections 2024-03-06T07:31:02.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.862+0000 7f8a5ce00640 1 -- 172.21.15.86:0/630236185 wait complete. 2024-03-06T07:31:02.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.863+0000 7f8a5ce00640 1 Processor -- start 2024-03-06T07:31:02.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.864+0000 7f8a5ce00640 1 -- start start 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.864+0000 7f8a5ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a5806d2e0 0x7f8a58152e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.864+0000 7f8a5ce00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a580cc7e0 0x7f8a581f7920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.864+0000 7f8a5ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5815aab0 0x7f8a58144ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.864+0000 7f8a5ce00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a5806cae0 con 0x7f8a5806d2e0 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.864+0000 7f8a5ce00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a580b2e80 con 0x7f8a580cc7e0 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.864+0000 7f8a5ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8a580cf3e0 con 0x7f8a5815aab0 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a5806d2e0 0x7f8a58152e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a5806d2e0 0x7f8a58152e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48818/0 (socket says 172.21.15.86:48818) 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 -- 172.21.15.86:0/3871965370 learned_addr learned my addr 172.21.15.86:0/3871965370 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57e00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5815aab0 0x7f8a58144ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a56a00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a580cc7e0 0x7f8a581f7920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:02.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 -- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5815aab0 msgr2=0x7f8a58144ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:02.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5815aab0 0x7f8a58144ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:02.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 -- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a580cc7e0 msgr2=0x7f8a581f7920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:02.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a580cc7e0 0x7f8a581f7920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:02.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 -- 172.21.15.86:0/3871965370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a58155780 con 0x7f8a5806d2e0 2024-03-06T07:31:02.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.865+0000 7f8a57400640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a5806d2e0 0x7f8a58152e20 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f8a4800b3f0 tx=0x7f8a4800b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:02.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.866+0000 7f8a4f400640 1 -- 172.21.15.86:0/3871965370 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a48015020 con 0x7f8a5806d2e0 2024-03-06T07:31:02.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.866+0000 7f8a4f400640 1 -- 172.21.15.86:0/3871965370 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8a48011830 con 0x7f8a5806d2e0 2024-03-06T07:31:02.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.866+0000 7f8a4f400640 1 -- 172.21.15.86:0/3871965370 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a48011c20 con 0x7f8a5806d2e0 2024-03-06T07:31:02.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.866+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8a58155a60 con 0x7f8a5806d2e0 2024-03-06T07:31:02.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.866+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a58153670 con 0x7f8a5806d2e0 2024-03-06T07:31:02.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.867+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8a581533c0 con 0x7f8a5806d2e0 2024-03-06T07:31:02.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.868+0000 7f8a4f400640 1 -- 172.21.15.86:0/3871965370 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8a48018020 con 0x7f8a5806d2e0 2024-03-06T07:31:02.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.869+0000 7f8a4f400640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8a3806b4d0 0x7f8a3806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:02.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.869+0000 7f8a4f400640 1 -- 172.21.15.86:0/3871965370 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f8a48089e90 con 0x7f8a5806d2e0 2024-03-06T07:31:02.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.869+0000 7f8a56a00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8a3806b4d0 0x7f8a3806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:02.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.870+0000 7f8a56a00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8a3806b4d0 0x7f8a3806d990 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f8a580cd950 tx=0x7f8a44008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:02.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:02.873+0000 7f8a4f400640 1 -- 172.21.15.86:0/3871965370 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8a4801f030 con 0x7f8a5806d2e0 2024-03-06T07:31:03.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.224+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8a5806e180 con 0x7f8a5806d2e0 2024-03-06T07:31:03.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.225+0000 7f8a4f400640 1 -- 172.21.15.86:0/3871965370 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f8a4808b090 con 0x7f8a5806d2e0 2024-03-06T07:31:03.228 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:03.228 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:31:03.228 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:31:03.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8a3806b4d0 msgr2=0x7f8a3806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:03.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8a3806b4d0 0x7f8a3806d990 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f8a580cd950 tx=0x7f8a44008040 comp rx=0 tx=0).stop 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a5806d2e0 msgr2=0x7f8a58152e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a5806d2e0 0x7f8a58152e20 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f8a4800b3f0 tx=0x7f8a4800b8b0 comp rx=0 tx=0).stop 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 shutdown_connections 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8a3806b4d0 0x7f8a3806d990 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8a5815aab0 0x7f8a58144ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8a580cc7e0 0x7f8a581f7920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 --2- 172.21.15.86:0/3871965370 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8a5806d2e0 0x7f8a58152e20 unknown :-1 s=CLOSED pgs=835 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 >> 172.21.15.86:0/3871965370 conn(0x7f8a5806afc0 msgr2=0x7f8a580cf890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:03.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 shutdown_connections 2024-03-06T07:31:03.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:03.228+0000 7f8a5ce00640 1 -- 172.21.15.86:0/3871965370 wait complete. 2024-03-06T07:31:03.256 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:31:03.531 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-06T07:31:03.532 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-06T07:31:03.532 DEBUG:tasks.ceph_test_case:wait_until_equal: 0 != 10, waiting (timeout=60)... 2024-03-06T07:31:08.533 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:08.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.752+0000 7f0529400640 1 Processor -- start 2024-03-06T07:31:08.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.752+0000 7f0529400640 1 -- start start 2024-03-06T07:31:08.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.753+0000 7f0529400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 0x7f052415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.753+0000 7f0529400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f052406cef0 con 0x7f0524070b10 2024-03-06T07:31:08.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.753+0000 7f0529400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f052406d1d0 con 0x7f0524144c00 2024-03-06T07:31:08.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.753+0000 7f0529400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0524160370 con 0x7f052415f9d0 2024-03-06T07:31:08.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0523200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0524070b10 0x7f0524070f60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46056/0 (socket says 172.21.15.86:46056) 2024-03-06T07:31:08.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0523200640 1 -- 172.21.15.86:0/4172901581 learned_addr learned my addr 172.21.15.86:0/4172901581 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:08.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521e00640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 0x7f052415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2596636528 0 0) 0x7f052406cef0 con 0x7f0524070b10 2024-03-06T07:31:08.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0504003520 con 0x7f0524070b10 2024-03-06T07:31:08.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4041037686 0 0) 0x7f0504003520 con 0x7f0524070b10 2024-03-06T07:31:08.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 msgr2=0x7f052415fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:08.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 0x7f052415fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 >> v1:172.21.15.194:6789/0 conn(0x7f0524144c00 legacy=0x7f052406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f052406d4b0 con 0x7f0524070b10 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f051c001670 con 0x7f0524070b10 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 >> v1:172.21.15.86:6789/0 conn(0x7f0524070b10 legacy=0x7f0524070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 0x7f05040039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0504004010 0x7f0504004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f05040051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.754+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f050400bb50 con 0x7f0504004d40 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f050400bd00 con 0x7f0504004010 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f050400bf00 con 0x7f052415f9d0 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0523200640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f05040051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0522800640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 0x7f05040039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0521e00640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0504004010 0x7f0504004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0523200640 1 -- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 msgr2=0x7f05040039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:08.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0523200640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f052415f9d0 0x7f05040039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:08.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0523200640 1 -- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0504004010 msgr2=0x7f0504004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:08.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0523200640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0504004010 0x7f0504004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:08.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0523200640 1 -- 172.21.15.86:0/4172901581 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0504003520 con 0x7f0504004d40 2024-03-06T07:31:08.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.755+0000 7f0521e00640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0504004010 0x7f0504004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.756+0000 7f0523200640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f05040051c0 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7f051c00c210 tx=0x7f0504005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=64b341ea568d7b2d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.756+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f051c007d30 con 0x7f0504004d40 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f051c030070 con 0x7f0504004d40 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0521400640 1 -- 172.21.15.86:0/4172901581 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f051c02cda0 con 0x7f0504004d40 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0529400640 1 -- 172.21.15.86:0/4172901581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 msgr2=0x7f05040051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0529400640 1 --2- 172.21.15.86:0/4172901581 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f05040051c0 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7f051c00c210 tx=0x7f0504005830 comp rx=0 tx=0).stop 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0523200640 1 -- 172.21.15.86:0/4172901581 reap_dead start 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0529400640 1 -- 172.21.15.86:0/4172901581 shutdown_connections 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0529400640 1 -- 172.21.15.86:0/4172901581 >> 172.21.15.86:0/4172901581 conn(0x7f05240b1730 msgr2=0x7f05240b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0529400640 1 -- 172.21.15.86:0/4172901581 shutdown_connections 2024-03-06T07:31:08.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.757+0000 7f0529400640 1 -- 172.21.15.86:0/4172901581 wait complete. 2024-03-06T07:31:08.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.759+0000 7f0529400640 1 Processor -- start 2024-03-06T07:31:08.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.759+0000 7f0529400640 1 -- start start 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0529400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f052413c7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0529400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05240aeec0 0x7f05240b2cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0529400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05240ad860 0x7f05240b35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0529400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f052406cb50 con 0x7f0504004d40 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0529400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0524160730 con 0x7f05240aeec0 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0529400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f052406ccd0 con 0x7f05240ad860 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0522800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f052413c7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0522800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f052413c7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60000/0 (socket says 172.21.15.86:60000) 2024-03-06T07:31:08.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.760+0000 7f0522800640 1 -- 172.21.15.86:0/3290814002 learned_addr learned my addr 172.21.15.86:0/3290814002 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:08.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0523200640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05240ad860 0x7f05240b35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0522800640 1 -- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05240ad860 msgr2=0x7f05240b35e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0521e00640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05240aeec0 0x7f05240b2cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0522800640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05240ad860 0x7f05240b35e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0522800640 1 -- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05240aeec0 msgr2=0x7f05240b2cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0522800640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05240aeec0 0x7f05240b2cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0522800640 1 -- 172.21.15.86:0/3290814002 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f052413cce0 con 0x7f0504004d40 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f0522800640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f052413c7a0 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7f051400e550 tx=0x7f051400ea10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f051aa00640 1 -- 172.21.15.86:0/3290814002 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05140052e0 con 0x7f0504004d40 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.761+0000 7f051aa00640 1 -- 172.21.15.86:0/3290814002 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0514018070 con 0x7f0504004d40 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.762+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f05240b3230 con 0x7f0504004d40 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.762+0000 7f051aa00640 1 -- 172.21.15.86:0/3290814002 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0514003810 con 0x7f0504004d40 2024-03-06T07:31:08.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.762+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f05240ae100 con 0x7f0504004d40 2024-03-06T07:31:08.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.763+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f052404f330 con 0x7f0504004d40 2024-03-06T07:31:08.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.764+0000 7f051aa00640 1 -- 172.21.15.86:0/3290814002 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f05140039b0 con 0x7f0504004d40 2024-03-06T07:31:08.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.765+0000 7f051aa00640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050006b500 0x7f050006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:08.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.765+0000 7f051aa00640 1 -- 172.21.15.86:0/3290814002 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0514089eb0 con 0x7f0504004d40 2024-03-06T07:31:08.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.765+0000 7f0521e00640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050006b500 0x7f050006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:08.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.765+0000 7f0521e00640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050006b500 0x7f050006d9c0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f05241f6e80 tx=0x7f050c00b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:08.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:08.774+0000 7f051aa00640 1 -- 172.21.15.86:0/3290814002 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0514058cf0 con 0x7f0504004d40 2024-03-06T07:31:09.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.009+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f05241442b0 con 0x7f0504004d40 2024-03-06T07:31:09.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.010+0000 7f051aa00640 1 -- 172.21.15.86:0/3290814002 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f05140585f0 con 0x7f0504004d40 2024-03-06T07:31:09.012 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:09.013 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:31:09.013 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:31:09.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.012+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050006b500 msgr2=0x7f050006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:09.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.012+0000 7f0529400640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050006b500 0x7f050006d9c0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f05241f6e80 tx=0x7f050c00b040 comp rx=0 tx=0).stop 2024-03-06T07:31:09.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.012+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 msgr2=0x7f052413c7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:09.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.012+0000 7f0529400640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f052413c7a0 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7f051400e550 tx=0x7f051400ea10 comp rx=0 tx=0).stop 2024-03-06T07:31:09.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 shutdown_connections 2024-03-06T07:31:09.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050006b500 0x7f050006d9c0 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05240ad860 0x7f05240b35e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05240aeec0 0x7f05240b2cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 --2- 172.21.15.86:0/3290814002 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0504004d40 0x7f052413c7a0 unknown :-1 s=CLOSED pgs=838 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 >> 172.21.15.86:0/3290814002 conn(0x7f05240b1730 msgr2=0x7f052406e590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:09.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 shutdown_connections 2024-03-06T07:31:09.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.013+0000 7f0529400640 1 -- 172.21.15.86:0/3290814002 wait complete. 2024-03-06T07:31:09.034 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:31:09.246 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":10}} 2024-03-06T07:31:09.246 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 10 } } 2024-03-06T07:31:09.247 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:09.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.449+0000 7f9f31800640 1 Processor -- start 2024-03-06T07:31:09.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.449+0000 7f9f31800640 1 -- start start 2024-03-06T07:31:09.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.450+0000 7f9f31800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c0aedc0 0x7f9f2c0cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:09.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.450+0000 7f9f31800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9f2c0cc800 0x7f9f2c15f830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:09.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.450+0000 7f9f31800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f2c15c240 con 0x7f9f2c1561f0 2024-03-06T07:31:09.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.450+0000 7f9f31800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f2c15ce30 con 0x7f9f2c0cc800 2024-03-06T07:31:09.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.450+0000 7f9f31800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f2c15c500 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.451+0000 7f9f30e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9f2c0cc800 0x7f9f2c15f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:09.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.451+0000 7f9f30e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9f2c0cc800 0x7f9f2c15f830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60006/0 (socket says 172.21.15.86:60006) 2024-03-06T07:31:09.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.451+0000 7f9f30e00640 1 -- 172.21.15.86:0/1155564103 learned_addr learned my addr 172.21.15.86:0/1155564103 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:09.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.451+0000 7f9f30e00640 1 -- 172.21.15.86:0/1155564103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c0aedc0 msgr2=0x7f9f2c0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:09.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.451+0000 7f9f30e00640 1 --2- 172.21.15.86:0/1155564103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c0aedc0 0x7f9f2c0cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.451+0000 7f9f30e00640 1 -- 172.21.15.86:0/1155564103 >> v1:172.21.15.194:6789/0 conn(0x7f9f2c1561f0 legacy=0x7f9f2c0ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:09.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.451+0000 7f9f30e00640 1 -- 172.21.15.86:0/1155564103 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f2c15c7e0 con 0x7f9f2c0cc800 2024-03-06T07:31:09.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.452+0000 7f9f30e00640 1 --2- 172.21.15.86:0/1155564103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c15f830 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7f9f24009e50 tx=0x7f9f2402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1626fc1f5a4957fb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.452+0000 7f9f2aa00640 1 -- 172.21.15.86:0/1155564103 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f2402ca70 con 0x7f9f2c0cc800 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f2aa00640 1 -- 172.21.15.86:0/1155564103 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f2402cc10 con 0x7f9f2c0cc800 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f2aa00640 1 -- 172.21.15.86:0/1155564103 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f240343f0 con 0x7f9f2c0cc800 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 -- 172.21.15.86:0/1155564103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0cc800 msgr2=0x7f9f2c15f830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 --2- 172.21.15.86:0/1155564103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c15f830 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7f9f24009e50 tx=0x7f9f2402bed0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 -- 172.21.15.86:0/1155564103 shutdown_connections 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 --2- 172.21.15.86:0/1155564103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c15f830 unknown :-1 s=CLOSED pgs=839 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 --2- 172.21.15.86:0/1155564103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c0aedc0 0x7f9f2c0cc2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 -- 172.21.15.86:0/1155564103 >> 172.21.15.86:0/1155564103 conn(0x7f9f2c06afc0 msgr2=0x7f9f2c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 -- 172.21.15.86:0/1155564103 shutdown_connections 2024-03-06T07:31:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.453+0000 7f9f31800640 1 -- 172.21.15.86:0/1155564103 wait complete. 2024-03-06T07:31:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.455+0000 7f9f31800640 1 Processor -- start 2024-03-06T07:31:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.455+0000 7f9f31800640 1 -- start start 2024-03-06T07:31:09.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f31800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0aedc0 0x7f9f2c1f3c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:09.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f31800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c1e64e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:09.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f31800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c1561f0 0x7f9f2c1e6a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:09.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f31800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f2c1565d0 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f31800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f2c0ccc90 con 0x7f9f2c0cc800 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f31800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f2c156750 con 0x7f9f2c1561f0 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f2be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0aedc0 0x7f9f2c1f3c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f2be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0aedc0 0x7f9f2c1f3c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60010/0 (socket says 172.21.15.86:60010) 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f2be00640 1 -- 172.21.15.86:0/1031455710 learned_addr learned my addr 172.21.15.86:0/1031455710 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f30e00640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c1561f0 0x7f9f2c1e6a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.457+0000 7f9f2b400640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c1e64e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f2be00640 1 -- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c1561f0 msgr2=0x7f9f2c1e6a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f2be00640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c1561f0 0x7f9f2c1e6a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f2be00640 1 -- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f2c0cc800 msgr2=0x7f9f2c1e64e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f2be00640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c1e64e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f2be00640 1 -- 172.21.15.86:0/1031455710 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f2c147050 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f2b400640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c1e64e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:09.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f2be00640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0aedc0 0x7f9f2c1f3c00 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7f9f1c00b3e0 tx=0x7f9f1c00b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:09.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f23e00640 1 -- 172.21.15.86:0/1031455710 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f1c015020 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f23e00640 1 -- 172.21.15.86:0/1031455710 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f1c011850 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f23e00640 1 -- 172.21.15.86:0/1031455710 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f1c011c40 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.458+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f2c147330 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.459+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f2c1478f0 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.460+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f2c04f330 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.460+0000 7f9f23e00640 1 -- 172.21.15.86:0/1031455710 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9f1c018020 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.461+0000 7f9f23e00640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f0c06b4d0 0x7f9f0c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:09.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.462+0000 7f9f23e00640 1 -- 172.21.15.86:0/1031455710 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9f1c08a200 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.462+0000 7f9f2b400640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f0c06b4d0 0x7f9f0c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:09.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.462+0000 7f9f2b400640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f0c06b4d0 0x7f9f0c06d990 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f9f18006fd0 tx=0x7f9f18008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:09.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.466+0000 7f9f23e00640 1 -- 172.21.15.86:0/1031455710 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9f1c01f020 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.734+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9f2c06b680 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.735+0000 7f9f23e00640 1 -- 172.21.15.86:0/1031455710 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f9f1c0172c0 con 0x7f9f2c0aedc0 2024-03-06T07:31:09.737 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:09.738 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:31:09.738 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:31:09.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f0c06b4d0 msgr2=0x7f9f0c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:09.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f0c06b4d0 0x7f9f0c06d990 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f9f18006fd0 tx=0x7f9f18008040 comp rx=0 tx=0).stop 2024-03-06T07:31:09.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0aedc0 msgr2=0x7f9f2c1f3c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:09.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0aedc0 0x7f9f2c1f3c00 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7f9f1c00b3e0 tx=0x7f9f1c00b8a0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 shutdown_connections 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f0c06b4d0 0x7f9f0c06d990 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f2c1561f0 0x7f9f2c1e6a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f2c0cc800 0x7f9f2c1e64e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 --2- 172.21.15.86:0/1031455710 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f2c0aedc0 0x7f9f2c1f3c00 unknown :-1 s=CLOSED pgs=840 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.737+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 >> 172.21.15.86:0/1031455710 conn(0x7f9f2c06afc0 msgr2=0x7f9f2c155100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.738+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 shutdown_connections 2024-03-06T07:31:09.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:09.738+0000 7f9f31800640 1 -- 172.21.15.86:0/1031455710 wait complete. 2024-03-06T07:31:09.760 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump purge_queue pq_executed 2024-03-06T07:31:09.986 INFO:teuthology.orchestra.run.smithi086.stdout:{"purge_queue":{"pq_executed":10}} 2024-03-06T07:31:09.987 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 10 } } 2024-03-06T07:31:09.987 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:09.987 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> import uuid 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> n = 10 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = False 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> if 0 > 0: 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> create_hard_links = True 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> dpath = os.path.dirname(path) 2024-03-06T07:31:10.035 DEBUG:teuthology.orchestra.run.smithi086:> fnameprefix = os.path.basename(path) 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> os.makedirs(dpath, exist_ok=True) 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> try: 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> for i in range(n): 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> with open(fpath, '"'"'w'"'"') as f: 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> f.write(f"{i}") 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> f.flush() 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(f.fileno()) 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> if True: 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> os.unlink(fpath) 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> if True: 2024-03-06T07:31:10.036 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> if create_hard_links: 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> for j in range(0): 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> if False: 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> os.fsync(dirfd) 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> finally: 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> os.close(dirfd) 2024-03-06T07:31:10.037 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:31:10.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:31:10.338 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:31:10.339 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:10.483 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:10.598 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:31:10.598 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:10.599 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:10.651 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:10.651 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:10.694 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:10.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.935+0000 7fe0ee600640 1 Processor -- start 2024-03-06T07:31:10.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.935+0000 7fe0ee600640 1 -- start start 2024-03-06T07:31:10.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.936+0000 7fe0ee600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0e806b6d0 0x7fe0e80b5e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.936+0000 7fe0ee600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe0e8066d00 0x7fe0e80d0a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.936+0000 7fe0ee600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe0e80afc20 con 0x7fe0e815c120 2024-03-06T07:31:10.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.936+0000 7fe0ee600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe0e80d1130 con 0x7fe0e8066d00 2024-03-06T07:31:10.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.936+0000 7fe0ee600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe0e80b59b0 con 0x7fe0e806b6d0 2024-03-06T07:31:10.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.937+0000 7fe0e7400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe0e815c120 0x7fe0e806afc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46064/0 (socket says 172.21.15.86:46064) 2024-03-06T07:31:10.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.937+0000 7fe0e7400640 1 -- 172.21.15.86:0/2940678645 learned_addr learned my addr 172.21.15.86:0/2940678645 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:10.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.937+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2213282030 0 0) 0x7fe0e80afc20 con 0x7fe0e815c120 2024-03-06T07:31:10.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.937+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe0cc003520 con 0x7fe0e815c120 2024-03-06T07:31:10.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.937+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0e806b6d0 0x7fe0e80b5e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.937+0000 7fe0e7e00640 1 --2- 172.21.15.86:0/2940678645 >> v2:172.21.15.194:3300/0 conn(0x7fe0e8066d00 0x7fe0e80d0a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4057010710 0 0) 0x7fe0cc003520 con 0x7fe0e815c120 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0e806b6d0 msgr2=0x7fe0e80b5e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0e806b6d0 0x7fe0e80b5e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 >> v2:172.21.15.194:3300/0 conn(0x7fe0e8066d00 msgr2=0x7fe0e80d0a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 --2- 172.21.15.86:0/2940678645 >> v2:172.21.15.194:3300/0 conn(0x7fe0e8066d00 0x7fe0e80d0a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe0e80b5080 con 0x7fe0e815c120 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fe0d8001670 con 0x7fe0e815c120 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 >> v1:172.21.15.86:6789/0 conn(0x7fe0e815c120 legacy=0x7fe0e806afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0cc0039b0 0x7fe0cc003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0cc004800 0x7fe0cc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0e806b6d0 0x7fe0cc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0e80d1130 con 0x7fe0cc004800 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0e80b59b0 con 0x7fe0cc0039b0 2024-03-06T07:31:10.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe0cc00ba70 con 0x7fe0e806b6d0 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0cc004800 0x7fe0cc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0dfe00640 1 -- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0e806b6d0 msgr2=0x7fe0cc0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0e7400640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0cc0039b0 0x7fe0cc003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0e806b6d0 0x7fe0cc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0dfe00640 1 -- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0cc0039b0 msgr2=0x7fe0cc003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0cc0039b0 0x7fe0cc003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.938+0000 7fe0dfe00640 1 -- 172.21.15.86:0/2940678645 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe0cc003520 con 0x7fe0cc004800 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.939+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0cc004800 0x7fe0cc004c80 secure :-1 s=READY pgs=842 cs=0 l=1 rev1=1 crypto rx=0x7fe0d000ba00 tx=0x7fe0cc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=501e56e0af3b572c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d0036020 con 0x7fe0cc004800 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe0d0032630 con 0x7fe0cc004800 2024-03-06T07:31:10.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0e6a00640 1 -- 172.21.15.86:0/2940678645 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d000b600 con 0x7fe0cc004800 2024-03-06T07:31:10.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0ee600640 1 -- 172.21.15.86:0/2940678645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0cc004800 msgr2=0x7fe0cc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:10.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0ee600640 1 --2- 172.21.15.86:0/2940678645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0cc004800 0x7fe0cc004c80 secure :-1 s=READY pgs=842 cs=0 l=1 rev1=1 crypto rx=0x7fe0d000ba00 tx=0x7fe0cc005830 comp rx=0 tx=0).stop 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0e7e00640 1 -- 172.21.15.86:0/2940678645 reap_dead start 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0ee600640 1 -- 172.21.15.86:0/2940678645 shutdown_connections 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0ee600640 1 -- 172.21.15.86:0/2940678645 >> 172.21.15.86:0/2940678645 conn(0x7fe0e80b1e80 msgr2=0x7fe0e80b2290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0ee600640 1 -- 172.21.15.86:0/2940678645 shutdown_connections 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.940+0000 7fe0ee600640 1 -- 172.21.15.86:0/2940678645 wait complete. 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.942+0000 7fe0ee600640 1 Processor -- start 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.942+0000 7fe0ee600640 1 -- start start 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.943+0000 7fe0ee600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0cc004800 0x7fe0e81f8260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.943+0000 7fe0ee600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0e81f4180 0x7fe0e80aeaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.943+0000 7fe0ee600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0e80c5320 0x7fe0e80c57b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.943+0000 7fe0ee600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0e815c500 con 0x7fe0e81f4180 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.943+0000 7fe0ee600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0e80b5640 con 0x7fe0e80c5320 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.943+0000 7fe0ee600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe0e80b5e40 con 0x7fe0cc004800 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0e81f4180 0x7fe0e80aeaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0e81f4180 0x7fe0e80aeaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60016/0 (socket says 172.21.15.86:60016) 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 -- 172.21.15.86:0/762077026 learned_addr learned my addr 172.21.15.86:0/762077026 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0e7400640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0cc004800 0x7fe0e81f8260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0e7e00640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0e80c5320 0x7fe0e80c57b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 -- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0cc004800 msgr2=0x7fe0e81f8260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0cc004800 0x7fe0e81f8260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 -- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0e80c5320 msgr2=0x7fe0e80c57b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:10.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0e80c5320 0x7fe0e80c57b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:10.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 -- 172.21.15.86:0/762077026 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe0e81f48f0 con 0x7fe0e81f4180 2024-03-06T07:31:10.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.944+0000 7fe0dfe00640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0e81f4180 0x7fe0e80aeaf0 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7fe0d000bb20 tx=0x7fe0d0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:10.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.945+0000 7fe0df400640 1 -- 172.21.15.86:0/762077026 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d0041070 con 0x7fe0e81f4180 2024-03-06T07:31:10.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.945+0000 7fe0df400640 1 -- 172.21.15.86:0/762077026 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe0d0030a40 con 0x7fe0e81f4180 2024-03-06T07:31:10.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.945+0000 7fe0df400640 1 -- 172.21.15.86:0/762077026 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d0036040 con 0x7fe0e81f4180 2024-03-06T07:31:10.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.945+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe0e81f4bd0 con 0x7fe0e81f4180 2024-03-06T07:31:10.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.945+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe0e80c4e00 con 0x7fe0e81f4180 2024-03-06T07:31:10.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.946+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe0e81f50f0 con 0x7fe0e81f4180 2024-03-06T07:31:10.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.947+0000 7fe0df400640 1 -- 172.21.15.86:0/762077026 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe0d0039020 con 0x7fe0e81f4180 2024-03-06T07:31:10.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.948+0000 7fe0df400640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0c406b4d0 0x7fe0c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:10.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.948+0000 7fe0df400640 1 -- 172.21.15.86:0/762077026 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fe0d00ac480 con 0x7fe0e81f4180 2024-03-06T07:31:10.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.948+0000 7fe0e7400640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0c406b4d0 0x7fe0c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:10.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.949+0000 7fe0e7400640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0c406b4d0 0x7fe0c406d990 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7fe0e81f52f0 tx=0x7fe0d8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:10.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:10.953+0000 7fe0df400640 1 -- 172.21.15.86:0/762077026 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe0e81f50f0 con 0x7fe0e81f4180 2024-03-06T07:31:11.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.149+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe0e804f330 con 0x7fe0e81f4180 2024-03-06T07:31:11.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.151+0000 7fe0df400640 1 -- 172.21.15.86:0/762077026 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v72) v1 ==== 74+0+16778 (secure 0 0 0) 0x7fe0d007b190 con 0x7fe0e81f4180 2024-03-06T07:31:11.154 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:11.154 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":72,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:54.997378+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:30:33.566852+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-06T07:30:34.722690+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"11.14","mappings":[{"from":6,"to":0}]},{"pgid":"11.28","mappings":[{"from":6,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0c406b4d0 msgr2=0x7fe0c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0c406b4d0 0x7fe0c406d990 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7fe0e81f52f0 tx=0x7fe0d8009040 comp rx=0 tx=0).stop 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0e81f4180 msgr2=0x7fe0e80aeaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0e81f4180 0x7fe0e80aeaf0 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7fe0d000bb20 tx=0x7fe0d0004730 comp rx=0 tx=0).stop 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 shutdown_connections 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe0c406b4d0 0x7fe0c406d990 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0e80c5320 0x7fe0e80c57b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe0e81f4180 0x7fe0e80aeaf0 unknown :-1 s=CLOSED pgs=843 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 --2- 172.21.15.86:0/762077026 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0cc004800 0x7fe0e81f8260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 >> 172.21.15.86:0/762077026 conn(0x7fe0e80b1e80 msgr2=0x7fe0e80afdc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:11.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 shutdown_connections 2024-03-06T07:31:11.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.154+0000 7fe0ee600640 1 -- 172.21.15.86:0/762077026 wait complete. 2024-03-06T07:31:11.174 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 10, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:30:33.566852+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '69', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 11, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:30:34.722690+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '70', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:31:11.175 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:31:11.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.376+0000 7fb5d4000640 1 Processor -- start 2024-03-06T07:31:11.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.376+0000 7fb5d4000640 1 -- start start 2024-03-06T07:31:11.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d4000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc06dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d4000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb5cc06afc0 0x7fb5cc06b450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d4000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5cc071ea0 con 0x7fb5cc15efa0 2024-03-06T07:31:11.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d4000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5cc15bf70 con 0x7fb5cc06afc0 2024-03-06T07:31:11.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d4000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5cc06e380 con 0x7fb5cc15bb90 2024-03-06T07:31:11.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d0c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb5cc15efa0 0x7fb5cc0cec20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46072/0 (socket says 172.21.15.86:46072) 2024-03-06T07:31:11.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d0c00640 1 -- 172.21.15.86:0/3409946652 learned_addr learned my addr 172.21.15.86:0/3409946652 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d2000640 1 --2- 172.21.15.86:0/3409946652 >> v2:172.21.15.86:3300/0 conn(0x7fb5cc06afc0 0x7fb5cc06b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.378+0000 7fb5d1600640 1 --2- 172.21.15.86:0/3409946652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc06dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.379+0000 7fb5cbe00640 1 -- 172.21.15.86:0/3409946652 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3790799824 0 0) 0x7fb5cc071ea0 con 0x7fb5cc15efa0 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.379+0000 7fb5cbe00640 1 -- 172.21.15.86:0/3409946652 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb5b4003520 con 0x7fb5cc15efa0 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.379+0000 7fb5d2000640 1 -- 172.21.15.86:0/3409946652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 msgr2=0x7fb5cc06dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.379+0000 7fb5d2000640 1 --2- 172.21.15.86:0/3409946652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc06dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.379+0000 7fb5d2000640 1 -- 172.21.15.86:0/3409946652 >> v1:172.21.15.86:6789/0 conn(0x7fb5cc15efa0 legacy=0x7fb5cc0cec20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.379+0000 7fb5d2000640 1 -- 172.21.15.86:0/3409946652 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5cc06e660 con 0x7fb5cc06afc0 2024-03-06T07:31:11.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.380+0000 7fb5d2000640 1 --2- 172.21.15.86:0/3409946652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc06b450 secure :-1 s=READY pgs=845 cs=0 l=1 rev1=1 crypto rx=0x7fb5c4009d90 tx=0x7fb5c402beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d4ce0e4b74e42f19 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:11.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.380+0000 7fb5cbe00640 1 -- 172.21.15.86:0/3409946652 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5c4039070 con 0x7fb5cc06afc0 2024-03-06T07:31:11.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.380+0000 7fb5cbe00640 1 -- 172.21.15.86:0/3409946652 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5c402cc60 con 0x7fb5cc06afc0 2024-03-06T07:31:11.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.380+0000 7fb5cbe00640 1 -- 172.21.15.86:0/3409946652 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5c4034660 con 0x7fb5cc06afc0 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 -- 172.21.15.86:0/3409946652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 msgr2=0x7fb5cc06b450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3409946652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc06b450 secure :-1 s=READY pgs=845 cs=0 l=1 rev1=1 crypto rx=0x7fb5c4009d90 tx=0x7fb5c402beb0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 -- 172.21.15.86:0/3409946652 shutdown_connections 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3409946652 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc06b450 unknown :-1 s=CLOSED pgs=845 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3409946652 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc06dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 -- 172.21.15.86:0/3409946652 >> 172.21.15.86:0/3409946652 conn(0x7fb5cc0b27d0 msgr2=0x7fb5cc0b2bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 -- 172.21.15.86:0/3409946652 shutdown_connections 2024-03-06T07:31:11.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.381+0000 7fb5d4000640 1 -- 172.21.15.86:0/3409946652 wait complete. 2024-03-06T07:31:11.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.383+0000 7fb5d4000640 1 Processor -- start 2024-03-06T07:31:11.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.383+0000 7fb5d4000640 1 -- start start 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d4000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc0cc860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d4000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc14b5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d4000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5cc15efa0 0x7fb5cc0b52c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d4000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5cc15f400 con 0x7fb5cc06afc0 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d4000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5cc06ec20 con 0x7fb5cc15efa0 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d4000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5cc0723a0 con 0x7fb5cc15bb90 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc0cc860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc0cc860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60038/0 (socket says 172.21.15.86:60038) 2024-03-06T07:31:11.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.384+0000 7fb5d1600640 1 -- 172.21.15.86:0/3403914011 learned_addr learned my addr 172.21.15.86:0/3403914011 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d0c00640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc14b5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d1600640 1 -- 172.21.15.86:0/3403914011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 msgr2=0x7fb5cc14b5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d1600640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc14b5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d1600640 1 -- 172.21.15.86:0/3403914011 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5cc15efa0 msgr2=0x7fb5cc0b52c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d1600640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5cc15efa0 0x7fb5cc0b52c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d1600640 1 -- 172.21.15.86:0/3403914011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5cc14bb10 con 0x7fb5cc06afc0 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d1600640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc0cc860 secure :-1 s=READY pgs=846 cs=0 l=1 rev1=1 crypto rx=0x7fb5c000d800 tx=0x7fb5c000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5c9600640 1 -- 172.21.15.86:0/3403914011 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5c000b6c0 con 0x7fb5cc06afc0 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5c9600640 1 -- 172.21.15.86:0/3403914011 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5c0019070 con 0x7fb5cc06afc0 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5c9600640 1 -- 172.21.15.86:0/3403914011 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5c000ba80 con 0x7fb5cc06afc0 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5cc0cb070 con 0x7fb5cc06afc0 2024-03-06T07:31:11.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.385+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5cc0cb5f0 con 0x7fb5cc06afc0 2024-03-06T07:31:11.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.387+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5cc04f330 con 0x7fb5cc06afc0 2024-03-06T07:31:11.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.387+0000 7fb5c9600640 1 -- 172.21.15.86:0/3403914011 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb5c000bc20 con 0x7fb5cc06afc0 2024-03-06T07:31:11.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.388+0000 7fb5c9600640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5ac06b4d0 0x7fb5ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.388+0000 7fb5c9600640 1 -- 172.21.15.86:0/3403914011 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb5c008a790 con 0x7fb5cc06afc0 2024-03-06T07:31:11.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.389+0000 7fb5d0c00640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5ac06b4d0 0x7fb5ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.389+0000 7fb5d0c00640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5ac06b4d0 0x7fb5ac06d990 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7fb5b8005880 tx=0x7fb5b8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:11.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.393+0000 7fb5c9600640 1 -- 172.21.15.86:0/3403914011 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb5cc14bb10 con 0x7fb5cc06afc0 2024-03-06T07:31:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.727+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fb5cc0ca2a0 con 0x7fb5cc06afc0 2024-03-06T07:31:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.728+0000 7fb5c9600640 1 -- 172.21.15.86:0/3403914011 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v72) v1 ==== 93+0+10 (secure 0 0 0) 0x7fb5c0059530 con 0x7fb5cc06afc0 2024-03-06T07:31:11.730 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:31:11.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5ac06b4d0 msgr2=0x7fb5ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5ac06b4d0 0x7fb5ac06d990 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7fb5b8005880 tx=0x7fb5b8009040 comp rx=0 tx=0).stop 2024-03-06T07:31:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 msgr2=0x7fb5cc0cc860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc0cc860 secure :-1 s=READY pgs=846 cs=0 l=1 rev1=1 crypto rx=0x7fb5c000d800 tx=0x7fb5c000dcc0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 shutdown_connections 2024-03-06T07:31:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb5ac06b4d0 0x7fb5ac06d990 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb5cc15efa0 0x7fb5cc0b52c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5cc15bb90 0x7fb5cc14b5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 --2- 172.21.15.86:0/3403914011 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5cc06afc0 0x7fb5cc0cc860 unknown :-1 s=CLOSED pgs=846 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.730+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 >> 172.21.15.86:0/3403914011 conn(0x7fb5cc0b27d0 msgr2=0x7fb5cc0b2bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:11.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.731+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 shutdown_connections 2024-03-06T07:31:11.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.731+0000 7fb5d4000640 1 -- 172.21.15.86:0/3403914011 wait complete. 2024-03-06T07:31:11.752 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:31:11.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.957+0000 7f6347a00640 1 Processor -- start 2024-03-06T07:31:11.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.958+0000 7f6347a00640 1 -- start start 2024-03-06T07:31:11.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.959+0000 7f6347a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f63400b49c0 0x7f634015fe20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.959+0000 7f6347a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63400d49e0 con 0x7f63400b4f10 2024-03-06T07:31:11.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.959+0000 7f6347a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63400d1ca0 con 0x7f63400d24a0 2024-03-06T07:31:11.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.959+0000 7f6347a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63400d1f80 con 0x7f63400b49c0 2024-03-06T07:31:11.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.959+0000 7f633fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f63400b49c0 0x7f634015fe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.959+0000 7f6345a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f63400b4f10 0x7f63400cf340 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46084/0 (socket says 172.21.15.86:46084) 2024-03-06T07:31:11.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.960+0000 7f6345a00640 1 -- 172.21.15.86:0/3349507090 learned_addr learned my addr 172.21.15.86:0/3349507090 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:11.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.960+0000 7f633fe00640 1 -- 172.21.15.86:0/3349507090 >> v1:172.21.15.194:6789/0 conn(0x7f63400d24a0 legacy=0x7f63400d2880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:11.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.960+0000 7f633fe00640 1 -- 172.21.15.86:0/3349507090 >> v1:172.21.15.86:6789/0 conn(0x7f63400b4f10 legacy=0x7f63400cf340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.960+0000 7f633fe00640 1 -- 172.21.15.86:0/3349507090 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63400d4ca0 con 0x7f63400b49c0 2024-03-06T07:31:11.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.960+0000 7f633f400640 1 -- 172.21.15.86:0/3349507090 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2775930860 0 0) 0x7f63400d49e0 con 0x7f63400b4f10 2024-03-06T07:31:11.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.961+0000 7f633fe00640 1 --2- 172.21.15.86:0/3349507090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b49c0 0x7f634015fe20 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7f632c0097a0 tx=0x7f632c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=686c485918542815 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:11.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.961+0000 7f633f400640 1 -- 172.21.15.86:0/3349507090 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f632c035020 con 0x7f63400b49c0 2024-03-06T07:31:11.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.961+0000 7f633f400640 1 -- 172.21.15.86:0/3349507090 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f632c031830 con 0x7f63400b49c0 2024-03-06T07:31:11.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f633f400640 1 -- 172.21.15.86:0/3349507090 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f632c031c00 con 0x7f63400b49c0 2024-03-06T07:31:11.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f6347a00640 1 -- 172.21.15.86:0/3349507090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b49c0 msgr2=0x7f634015fe20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f6347a00640 1 --2- 172.21.15.86:0/3349507090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b49c0 0x7f634015fe20 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7f632c0097a0 tx=0x7f632c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f6347a00640 1 -- 172.21.15.86:0/3349507090 shutdown_connections 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f6347a00640 1 --2- 172.21.15.86:0/3349507090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b49c0 0x7f634015fe20 unknown :-1 s=CLOSED pgs=848 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f6347a00640 1 -- 172.21.15.86:0/3349507090 >> 172.21.15.86:0/3349507090 conn(0x7f63400d3ec0 msgr2=0x7f63400d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f6347a00640 1 -- 172.21.15.86:0/3349507090 shutdown_connections 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.962+0000 7f6347a00640 1 -- 172.21.15.86:0/3349507090 wait complete. 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.964+0000 7f6347a00640 1 Processor -- start 2024-03-06T07:31:11.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.965+0000 7f6347a00640 1 -- start start 2024-03-06T07:31:11.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f6347a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63400b49c0 0x7f63400b94d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f6347a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b4f10 0x7f63400c8400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f6347a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63400d24a0 0x7f6340158ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f6347a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63400cead0 con 0x7f63400b4f10 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f6347a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6340160360 con 0x7f63400b49c0 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f6347a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63400b4630 con 0x7f63400d24a0 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f633fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b4f10 0x7f63400c8400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f633fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b4f10 0x7f63400c8400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60060/0 (socket says 172.21.15.86:60060) 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.966+0000 7f633fe00640 1 -- 172.21.15.86:0/930874778 learned_addr learned my addr 172.21.15.86:0/930874778 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f6345a00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63400d24a0 0x7f6340158ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f6345000640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63400b49c0 0x7f63400b94d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633fe00640 1 -- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63400d24a0 msgr2=0x7f6340158ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633fe00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63400d24a0 0x7f6340158ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633fe00640 1 -- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63400b49c0 msgr2=0x7f63400b94d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633fe00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63400b49c0 0x7f63400b94d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633fe00640 1 -- 172.21.15.86:0/930874778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63400c8b30 con 0x7f63400b4f10 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633fe00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b4f10 0x7f63400c8400 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7f632c000c00 tx=0x7f632c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:11.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633cc00640 1 -- 172.21.15.86:0/930874778 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f632c041070 con 0x7f63400b4f10 2024-03-06T07:31:11.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633cc00640 1 -- 172.21.15.86:0/930874778 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f632c004230 con 0x7f63400b4f10 2024-03-06T07:31:11.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f633cc00640 1 -- 172.21.15.86:0/930874778 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f632c035040 con 0x7f63400b4f10 2024-03-06T07:31:11.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.967+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63400c9660 con 0x7f63400b4f10 2024-03-06T07:31:11.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.968+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6340159000 con 0x7f63400b4f10 2024-03-06T07:31:11.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.969+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f634004f330 con 0x7f63400b4f10 2024-03-06T07:31:11.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.970+0000 7f633cc00640 1 -- 172.21.15.86:0/930874778 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f632c038020 con 0x7f63400b4f10 2024-03-06T07:31:11.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.971+0000 7f633cc00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f632006b4d0 0x7f632006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:11.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.971+0000 7f633cc00640 1 -- 172.21.15.86:0/930874778 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f632c0abd50 con 0x7f63400b4f10 2024-03-06T07:31:11.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.972+0000 7f6345000640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f632006b4d0 0x7f632006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:11.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.972+0000 7f6345000640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f632006b4d0 0x7f632006d990 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f6334003d90 tx=0x7f6334009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:11.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:11.977+0000 7f633cc00640 1 -- 172.21.15.86:0/930874778 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f632c07ab80 con 0x7f63400b4f10 2024-03-06T07:31:12.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.175+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f63400cfbd0 con 0x7f63400b4f10 2024-03-06T07:31:12.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.176+0000 7f633cc00640 1 -- 172.21.15.86:0/930874778 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v72) v1 ==== 104+0+11 (secure 0 0 0) 0x7f632c07a480 con 0x7f63400b4f10 2024-03-06T07:31:12.179 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.179+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f632006b4d0 msgr2=0x7f632006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.179+0000 7f6347a00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f632006b4d0 0x7f632006d990 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f6334003d90 tx=0x7f6334009040 comp rx=0 tx=0).stop 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.179+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b4f10 msgr2=0x7f63400c8400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.179+0000 7f6347a00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b4f10 0x7f63400c8400 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7f632c000c00 tx=0x7f632c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 shutdown_connections 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f632006b4d0 0x7f632006d990 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63400d24a0 0x7f6340158ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63400b4f10 0x7f63400c8400 unknown :-1 s=CLOSED pgs=849 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 --2- 172.21.15.86:0/930874778 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63400b49c0 0x7f63400b94d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 >> 172.21.15.86:0/930874778 conn(0x7f63400d3ec0 msgr2=0x7f63400cd470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 shutdown_connections 2024-03-06T07:31:12.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.180+0000 7f6347a00640 1 -- 172.21.15.86:0/930874778 wait complete. 2024-03-06T07:31:12.203 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:31:12.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.401+0000 7f0daaa00640 1 Processor -- start 2024-03-06T07:31:12.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.401+0000 7f0daaa00640 1 -- start start 2024-03-06T07:31:12.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0daaa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0da415e130 0x7f0da40cf5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0daaa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0da40cfaf0 0x7f0da40d1ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0daaa00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0da40d13e0 con 0x7f0da40d23e0 2024-03-06T07:31:12.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0daaa00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0da40cf120 con 0x7f0da40cfaf0 2024-03-06T07:31:12.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0daaa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0da40cead0 con 0x7f0da415e130 2024-03-06T07:31:12.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0da8c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0da40cfaf0 0x7f0da40d1ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0da8c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0da40cfaf0 0x7f0da40d1ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60068/0 (socket says 172.21.15.86:60068) 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0da8c00640 1 -- 172.21.15.86:0/3836166951 learned_addr learned my addr 172.21.15.86:0/3836166951 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0daa000640 1 --1- 172.21.15.86:0/3836166951 >> v1:172.21.15.86:6789/0 conn(0x7f0da40d23e0 0x7f0da40c7970 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46100/0 (socket says 172.21.15.86:46100) 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0da9600640 1 --2- 172.21.15.86:0/3836166951 >> v2:172.21.15.194:3300/0 conn(0x7f0da415e130 0x7f0da40cf5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0da8c00640 1 -- 172.21.15.86:0/3836166951 >> v2:172.21.15.194:3300/0 conn(0x7f0da415e130 msgr2=0x7f0da40cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.403+0000 7f0da8c00640 1 --2- 172.21.15.86:0/3836166951 >> v2:172.21.15.194:3300/0 conn(0x7f0da415e130 0x7f0da40cf5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.404+0000 7f0da8c00640 1 -- 172.21.15.86:0/3836166951 >> v1:172.21.15.86:6789/0 conn(0x7f0da40d23e0 legacy=0x7f0da40c7970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.404+0000 7f0da3e00640 1 -- 172.21.15.86:0/3836166951 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1431597553 0 0) 0x7f0da40d13e0 con 0x7f0da40d23e0 2024-03-06T07:31:12.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.404+0000 7f0da8c00640 1 -- 172.21.15.86:0/3836166951 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0da40cedb0 con 0x7f0da40cfaf0 2024-03-06T07:31:12.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.406+0000 7f0da8c00640 1 --2- 172.21.15.86:0/3836166951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40d1ea0 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f0da4067ce0 tx=0x7f0d9002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e57f2af8e9c352df server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:12.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.406+0000 7f0da3e00640 1 -- 172.21.15.86:0/3836166951 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d90034070 con 0x7f0da40cfaf0 2024-03-06T07:31:12.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.406+0000 7f0da3e00640 1 -- 172.21.15.86:0/3836166951 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d90030860 con 0x7f0da40cfaf0 2024-03-06T07:31:12.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.406+0000 7f0da3e00640 1 -- 172.21.15.86:0/3836166951 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d90030c30 con 0x7f0da40cfaf0 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 -- 172.21.15.86:0/3836166951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 msgr2=0x7f0da40d1ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 --2- 172.21.15.86:0/3836166951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40d1ea0 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f0da4067ce0 tx=0x7f0d9002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 -- 172.21.15.86:0/3836166951 shutdown_connections 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 --2- 172.21.15.86:0/3836166951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40d1ea0 unknown :-1 s=CLOSED pgs=851 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 --2- 172.21.15.86:0/3836166951 >> v2:172.21.15.194:3300/0 conn(0x7f0da415e130 0x7f0da40cf5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 -- 172.21.15.86:0/3836166951 >> 172.21.15.86:0/3836166951 conn(0x7f0da409d070 msgr2=0x7f0da40d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 -- 172.21.15.86:0/3836166951 shutdown_connections 2024-03-06T07:31:12.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.407+0000 7f0daaa00640 1 -- 172.21.15.86:0/3836166951 wait complete. 2024-03-06T07:31:12.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.409+0000 7f0daaa00640 1 Processor -- start 2024-03-06T07:31:12.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.409+0000 7f0daaa00640 1 -- start start 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0daaa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40b6e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0daaa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0da40d23e0 0x7f0da41603c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0daaa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0da415e130 0x7f0da41553d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0daaa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0da40c8200 con 0x7f0da40cfaf0 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0daaa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0da40c8080 con 0x7f0da40d23e0 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0daaa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0da40d06b0 con 0x7f0da415e130 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40b6e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40b6e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60082/0 (socket says 172.21.15.86:60082) 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 -- 172.21.15.86:0/4256037622 learned_addr learned my addr 172.21.15.86:0/4256037622 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:12.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 -- 172.21.15.86:0/4256037622 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0da415e130 msgr2=0x7f0da41553d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0da415e130 0x7f0da41553d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 -- 172.21.15.86:0/4256037622 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0da40d23e0 msgr2=0x7f0da41603c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0da40d23e0 0x7f0da41603c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.411+0000 7f0da9600640 1 -- 172.21.15.86:0/4256037622 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0da415e720 con 0x7f0da40cfaf0 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.412+0000 7f0da9600640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40b6e60 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7f0d9800d5f0 tx=0x7f0d9800dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.412+0000 7f0da1600640 1 -- 172.21.15.86:0/4256037622 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d980125f0 con 0x7f0da40cfaf0 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.412+0000 7f0da1600640 1 -- 172.21.15.86:0/4256037622 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d98019070 con 0x7f0da40cfaf0 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.412+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0da415ea00 con 0x7f0da40cfaf0 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.412+0000 7f0da1600640 1 -- 172.21.15.86:0/4256037622 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d980129d0 con 0x7f0da40cfaf0 2024-03-06T07:31:12.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.412+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0da4160900 con 0x7f0da40cfaf0 2024-03-06T07:31:12.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.413+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0da404f330 con 0x7f0da40cfaf0 2024-03-06T07:31:12.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.414+0000 7f0da1600640 1 -- 172.21.15.86:0/4256037622 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0d98012b70 con 0x7f0da40cfaf0 2024-03-06T07:31:12.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.415+0000 7f0da1600640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d8406b4d0 0x7f0d8406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.415+0000 7f0da1600640 1 -- 172.21.15.86:0/4256037622 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0d9808a640 con 0x7f0da40cfaf0 2024-03-06T07:31:12.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.416+0000 7f0da8c00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d8406b4d0 0x7f0d8406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.416+0000 7f0da8c00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d8406b4d0 0x7f0d8406d990 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f0d90036000 tx=0x7f0d9003b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:12.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.420+0000 7f0da1600640 1 -- 172.21.15.86:0/4256037622 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0d98059480 con 0x7f0da40cfaf0 2024-03-06T07:31:12.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.620+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f0da409d690 con 0x7f0da40cfaf0 2024-03-06T07:31:12.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.620+0000 7f0da1600640 1 -- 172.21.15.86:0/4256037622 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v72) v1 ==== 100+0+11 (secure 0 0 0) 0x7f0d98058d80 con 0x7f0da40cfaf0 2024-03-06T07:31:12.623 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:31:12.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d8406b4d0 msgr2=0x7f0d8406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d8406b4d0 0x7f0d8406d990 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f0d90036000 tx=0x7f0d9003b040 comp rx=0 tx=0).stop 2024-03-06T07:31:12.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 msgr2=0x7f0da40b6e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40b6e60 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7f0d9800d5f0 tx=0x7f0d9800dab0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 shutdown_connections 2024-03-06T07:31:12.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d8406b4d0 0x7f0d8406d990 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0da415e130 0x7f0da41553d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0da40d23e0 0x7f0da41603c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 --2- 172.21.15.86:0/4256037622 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0da40cfaf0 0x7f0da40b6e60 unknown :-1 s=CLOSED pgs=852 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 >> 172.21.15.86:0/4256037622 conn(0x7f0da409d070 msgr2=0x7f0da40b5c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:12.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 shutdown_connections 2024-03-06T07:31:12.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.623+0000 7f0daaa00640 1 -- 172.21.15.86:0/4256037622 wait complete. 2024-03-06T07:31:12.644 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:31:12.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.841+0000 7f7e3ca00640 1 Processor -- start 2024-03-06T07:31:12.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.842+0000 7f7e3ca00640 1 -- start start 2024-03-06T07:31:12.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.843+0000 7f7e3ca00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7e3806b6d0 0x7f7e38162700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.843+0000 7f7e3ca00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e380cef70 con 0x7f7e3815dfd0 2024-03-06T07:31:12.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.843+0000 7f7e3ca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e3815e3b0 con 0x7f7e380affe0 2024-03-06T07:31:12.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.843+0000 7f7e3ca00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7e380b0400 con 0x7f7e3806b6d0 2024-03-06T07:31:12.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.843+0000 7f7e36800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7e3815dfd0 0x7f7e3806afc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46110/0 (socket says 172.21.15.86:46110) 2024-03-06T07:31:12.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e36800640 1 -- 172.21.15.86:0/4065046751 learned_addr learned my addr 172.21.15.86:0/4065046751 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:12.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35e00640 1 --2- 172.21.15.86:0/4065046751 >> v2:172.21.15.194:3300/0 conn(0x7f7e3806b6d0 0x7f7e38162700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1427207446 0 0) 0x7f7e380cef70 con 0x7f7e3815dfd0 2024-03-06T07:31:12.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7e1c003520 con 0x7f7e3815dfd0 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2123426228 0 0) 0x7f7e1c003520 con 0x7f7e3815dfd0 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 >> v2:172.21.15.194:3300/0 conn(0x7f7e3806b6d0 msgr2=0x7f7e38162700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 --2- 172.21.15.86:0/4065046751 >> v2:172.21.15.194:3300/0 conn(0x7f7e3806b6d0 0x7f7e38162700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 >> v1:172.21.15.194:6789/0 conn(0x7f7e380affe0 legacy=0x7f7e380d06f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e38161c60 con 0x7f7e3815dfd0 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7e28001670 con 0x7f7e3815dfd0 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 >> v1:172.21.15.86:6789/0 conn(0x7f7e3815dfd0 legacy=0x7f7e3806afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e1c0039b0 0x7f7e1c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.844+0000 7f7e35400640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e1c004800 0x7f7e1c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e35400640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0052e0 0x7f7e1c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e380b0400 con 0x7f7e1c0039b0 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e1c00c1b0 con 0x7f7e1c0052e0 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e1c00c3b0 con 0x7f7e1c004800 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e36800640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e1c0039b0 0x7f7e1c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e35e00640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e1c004800 0x7f7e1c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e36800640 1 -- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e1c004800 msgr2=0x7f7e1c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e36800640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e1c004800 0x7f7e1c004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e37200640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0052e0 0x7f7e1c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e36800640 1 -- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0052e0 msgr2=0x7f7e1c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e36800640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0052e0 0x7f7e1c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e36800640 1 -- 172.21.15.86:0/4065046751 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e1c003520 con 0x7f7e1c0039b0 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e37200640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0052e0 0x7f7e1c00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.845+0000 7f7e37200640 1 -- 172.21.15.86:0/4065046751 reap_dead start 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.846+0000 7f7e36800640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e1c0039b0 0x7f7e1c003dd0 secure :-1 s=READY pgs=854 cs=0 l=1 rev1=1 crypto rx=0x7f7e28001eb0 tx=0x7f7e1c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ac0301efc2dedb5c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:12.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.846+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e28007d30 con 0x7f7e1c0039b0 2024-03-06T07:31:12.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.846+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e28030070 con 0x7f7e1c0039b0 2024-03-06T07:31:12.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e35400640 1 -- 172.21.15.86:0/4065046751 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e28005d70 con 0x7f7e1c0039b0 2024-03-06T07:31:12.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e3ca00640 1 -- 172.21.15.86:0/4065046751 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e1c0039b0 msgr2=0x7f7e1c003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e1c0039b0 0x7f7e1c003dd0 secure :-1 s=READY pgs=854 cs=0 l=1 rev1=1 crypto rx=0x7f7e28001eb0 tx=0x7f7e1c005830 comp rx=0 tx=0).stop 2024-03-06T07:31:12.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e3ca00640 1 -- 172.21.15.86:0/4065046751 shutdown_connections 2024-03-06T07:31:12.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/4065046751 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e1c0039b0 0x7f7e1c003dd0 unknown :-1 s=CLOSED pgs=854 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e3ca00640 1 -- 172.21.15.86:0/4065046751 >> 172.21.15.86:0/4065046751 conn(0x7f7e3806c0d0 msgr2=0x7f7e3806c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:12.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e3ca00640 1 -- 172.21.15.86:0/4065046751 shutdown_connections 2024-03-06T07:31:12.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.847+0000 7f7e3ca00640 1 -- 172.21.15.86:0/4065046751 wait complete. 2024-03-06T07:31:12.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.849+0000 7f7e3ca00640 1 Processor -- start 2024-03-06T07:31:12.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.850+0000 7f7e3ca00640 1 -- start start 2024-03-06T07:31:12.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.850+0000 7f7e3ca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0039b0 0x7f7e381432f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.850+0000 7f7e3ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e38161130 0x7f7e3815f720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.850+0000 7f7e3ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e380b50f0 0x7f7e381e7990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e3ca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e38161f00 con 0x7f7e38161130 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e3ca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e38162270 con 0x7f7e1c0039b0 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e3ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e380ce8d0 con 0x7f7e380b50f0 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e38161130 0x7f7e3815f720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e38161130 0x7f7e3815f720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60106/0 (socket says 172.21.15.86:60106) 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 -- 172.21.15.86:0/3780456087 learned_addr learned my addr 172.21.15.86:0/3780456087 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 -- 172.21.15.86:0/3780456087 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e380b50f0 msgr2=0x7f7e381e7990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e380b50f0 0x7f7e381e7990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 -- 172.21.15.86:0/3780456087 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0039b0 msgr2=0x7f7e381432f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e36800640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0039b0 0x7f7e381432f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0039b0 0x7f7e381432f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 -- 172.21.15.86:0/3780456087 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e38160090 con 0x7f7e38161130 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.851+0000 7f7e35e00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e38161130 0x7f7e3815f720 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7f7e240108e0 tx=0x7f7e24010da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:12.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.852+0000 7f7e2ea00640 1 -- 172.21.15.86:0/3780456087 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e2400f040 con 0x7f7e38161130 2024-03-06T07:31:12.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.852+0000 7f7e2ea00640 1 -- 172.21.15.86:0/3780456087 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e2400eb60 con 0x7f7e38161130 2024-03-06T07:31:12.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.852+0000 7f7e2ea00640 1 -- 172.21.15.86:0/3780456087 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e2401c400 con 0x7f7e38161130 2024-03-06T07:31:12.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.852+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7e38160370 con 0x7f7e38161130 2024-03-06T07:31:12.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.852+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e38160930 con 0x7f7e38161130 2024-03-06T07:31:12.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.853+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7e3804f330 con 0x7f7e38161130 2024-03-06T07:31:12.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.854+0000 7f7e2ea00640 1 -- 172.21.15.86:0/3780456087 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7e2401c5a0 con 0x7f7e38161130 2024-03-06T07:31:12.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.855+0000 7f7e2ea00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e1806b4d0 0x7f7e1806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:12.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.855+0000 7f7e2ea00640 1 -- 172.21.15.86:0/3780456087 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f7e2408aa20 con 0x7f7e38161130 2024-03-06T07:31:12.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.855+0000 7f7e36800640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e1806b4d0 0x7f7e1806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:12.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.856+0000 7f7e36800640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e1806b4d0 0x7f7e1806d990 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f7e28034000 tx=0x7f7e28039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:12.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:12.860+0000 7f7e2ea00640 1 -- 172.21.15.86:0/3780456087 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7e24018030 con 0x7f7e38161130 2024-03-06T07:31:13.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.069+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f7e380cdea0 con 0x7f7e38161130 2024-03-06T07:31:13.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.070+0000 7f7e2ea00640 1 -- 172.21.15.86:0/3780456087 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v72) v1 ==== 80+0+0 (secure 0 0 0) 0x7f7e24059170 con 0x7f7e38161130 2024-03-06T07:31:13.073 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:31:13.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e1806b4d0 msgr2=0x7f7e1806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e1806b4d0 0x7f7e1806d990 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f7e28034000 tx=0x7f7e28039040 comp rx=0 tx=0).stop 2024-03-06T07:31:13.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e38161130 msgr2=0x7f7e3815f720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e38161130 0x7f7e3815f720 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7f7e240108e0 tx=0x7f7e24010da0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 shutdown_connections 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7e1806b4d0 0x7f7e1806d990 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7e380b50f0 0x7f7e381e7990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7e38161130 0x7f7e3815f720 unknown :-1 s=CLOSED pgs=855 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 --2- 172.21.15.86:0/3780456087 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7e1c0039b0 0x7f7e381432f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.072+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 >> 172.21.15.86:0/3780456087 conn(0x7f7e3806c0d0 msgr2=0x7f7e3806f230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.073+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 shutdown_connections 2024-03-06T07:31:13.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.073+0000 7f7e3ca00640 1 -- 172.21.15.86:0/3780456087 wait complete. 2024-03-06T07:31:13.093 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:31:13.094 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:13.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:31:13.219 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:31:13.219 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:13.219 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:31:13.272 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:13.272 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:13.311 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:13.311 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:13.311 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:13.348 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:31:13.348 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:13.348 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:13.396 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:13.397 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:13.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.639+0000 7fe7db200640 1 Processor -- start 2024-03-06T07:31:13.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.639+0000 7fe7db200640 1 -- start start 2024-03-06T07:31:13.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.640+0000 7fe7db200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe7dc1561f0 0x7fe7dc0ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7db200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe7dc15c240 con 0x7fe7dc0cc9d0 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7db200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe7dc15c500 con 0x7fe7dc0aebf0 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7db200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe7dc15ce30 con 0x7fe7dc1561f0 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7d9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe7dc1561f0 0x7fe7dc0ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7da800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe7dc0cc9d0 0x7fe7dc15f830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46124/0 (socket says 172.21.15.86:46124) 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7d9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe7dc1561f0 0x7fe7dc0ae6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60110/0 (socket says 172.21.15.86:60110) 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7da800640 1 -- 172.21.15.86:0/587834319 learned_addr learned my addr 172.21.15.86:0/587834319 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7d9e00640 1 -- 172.21.15.86:0/587834319 >> v1:172.21.15.194:6789/0 conn(0x7fe7dc0aebf0 legacy=0x7fe7dc0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7d9e00640 1 -- 172.21.15.86:0/587834319 >> v1:172.21.15.86:6789/0 conn(0x7fe7dc0cc9d0 legacy=0x7fe7dc15f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7d9e00640 1 -- 172.21.15.86:0/587834319 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe7dc15c7e0 con 0x7fe7dc1561f0 2024-03-06T07:31:13.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.641+0000 7fe7d8a00640 1 -- 172.21.15.86:0/587834319 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3344451786 0 0) 0x7fe7dc15c240 con 0x7fe7dc0cc9d0 2024-03-06T07:31:13.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.642+0000 7fe7d9e00640 1 --2- 172.21.15.86:0/587834319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0ae6b0 secure :-1 s=READY pgs=857 cs=0 l=1 rev1=1 crypto rx=0x7fe7c40097a0 tx=0x7fe7c402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=688a55ab01d4f7e8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:13.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7d8a00640 1 -- 172.21.15.86:0/587834319 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe7c4035020 con 0x7fe7dc1561f0 2024-03-06T07:31:13.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7d8a00640 1 -- 172.21.15.86:0/587834319 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe7c4031830 con 0x7fe7dc1561f0 2024-03-06T07:31:13.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7d8a00640 1 -- 172.21.15.86:0/587834319 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe7c4031c00 con 0x7fe7dc1561f0 2024-03-06T07:31:13.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7db200640 1 -- 172.21.15.86:0/587834319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 msgr2=0x7fe7dc0ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7db200640 1 --2- 172.21.15.86:0/587834319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0ae6b0 secure :-1 s=READY pgs=857 cs=0 l=1 rev1=1 crypto rx=0x7fe7c40097a0 tx=0x7fe7c402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7db200640 1 -- 172.21.15.86:0/587834319 shutdown_connections 2024-03-06T07:31:13.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7db200640 1 --2- 172.21.15.86:0/587834319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0ae6b0 unknown :-1 s=CLOSED pgs=857 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7db200640 1 -- 172.21.15.86:0/587834319 >> 172.21.15.86:0/587834319 conn(0x7fe7dc06afc0 msgr2=0x7fe7dc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:13.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7db200640 1 -- 172.21.15.86:0/587834319 shutdown_connections 2024-03-06T07:31:13.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.643+0000 7fe7db200640 1 -- 172.21.15.86:0/587834319 wait complete. 2024-03-06T07:31:13.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.645+0000 7fe7db200640 1 Processor -- start 2024-03-06T07:31:13.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.645+0000 7fe7db200640 1 -- start start 2024-03-06T07:31:13.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7db200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe7dc0aebf0 0x7fe7dc156900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:13.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7db200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe7dc0cc9d0 0x7fe7dc158c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7db200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0cbb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7db200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe7dc1565d0 con 0x7fe7dc1561f0 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7db200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe7dc06e360 con 0x7fe7dc0cc9d0 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7db200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe7dc156750 con 0x7fe7dc0aebf0 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7da800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0cbb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7da800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0cbb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60126/0 (socket says 172.21.15.86:60126) 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.646+0000 7fe7da800640 1 -- 172.21.15.86:0/964734966 learned_addr learned my addr 172.21.15.86:0/964734966 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7d9e00640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe7dc0aebf0 0x7fe7dc156900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7d9400640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe7dc0cc9d0 0x7fe7dc158c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7da800640 1 -- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe7dc0aebf0 msgr2=0x7fe7dc156900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7da800640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe7dc0aebf0 0x7fe7dc156900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7da800640 1 -- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe7dc0cc9d0 msgr2=0x7fe7dc158c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7da800640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe7dc0cc9d0 0x7fe7dc158c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7da800640 1 -- 172.21.15.86:0/964734966 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe7dc157000 con 0x7fe7dc1561f0 2024-03-06T07:31:13.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7da800640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0cbb50 secure :-1 s=READY pgs=858 cs=0 l=1 rev1=1 crypto rx=0x7fe7d400c7d0 tx=0x7fe7d400cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:13.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7d2000640 1 -- 172.21.15.86:0/964734966 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe7d4018070 con 0x7fe7dc1561f0 2024-03-06T07:31:13.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7d2000640 1 -- 172.21.15.86:0/964734966 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe7d400a640 con 0x7fe7dc1561f0 2024-03-06T07:31:13.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.647+0000 7fe7d2000640 1 -- 172.21.15.86:0/964734966 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe7d400aa70 con 0x7fe7dc1561f0 2024-03-06T07:31:13.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.648+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe7dc1572e0 con 0x7fe7dc1561f0 2024-03-06T07:31:13.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.648+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe7dc1578a0 con 0x7fe7dc1561f0 2024-03-06T07:31:13.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.649+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe7dc04f330 con 0x7fe7dc1561f0 2024-03-06T07:31:13.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.650+0000 7fe7d2000640 1 -- 172.21.15.86:0/964734966 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe7d400ac10 con 0x7fe7dc1561f0 2024-03-06T07:31:13.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.651+0000 7fe7d2000640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7b806b4d0 0x7fe7b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:13.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.651+0000 7fe7d2000640 1 -- 172.21.15.86:0/964734966 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fe7d4089880 con 0x7fe7dc1561f0 2024-03-06T07:31:13.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.651+0000 7fe7d9e00640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7b806b4d0 0x7fe7b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:13.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.652+0000 7fe7d9e00640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7b806b4d0 0x7fe7b806d990 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fe7c4036000 tx=0x7fe7c4008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:13.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.655+0000 7fe7d2000640 1 -- 172.21.15.86:0/964734966 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe7d40586b0 con 0x7fe7dc1561f0 2024-03-06T07:31:13.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.894+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe7dc0b2b10 con 0x7fe7dc1561f0 2024-03-06T07:31:13.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.895+0000 7fe7d2000640 1 -- 172.21.15.86:0/964734966 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fe7d4057fb0 con 0x7fe7dc1561f0 2024-03-06T07:31:13.898 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:13.899 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:31:13.899 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:31:13.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7b806b4d0 msgr2=0x7fe7b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7b806b4d0 0x7fe7b806d990 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fe7c4036000 tx=0x7fe7c4008030 comp rx=0 tx=0).stop 2024-03-06T07:31:13.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 msgr2=0x7fe7dc0cbb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:13.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0cbb50 secure :-1 s=READY pgs=858 cs=0 l=1 rev1=1 crypto rx=0x7fe7d400c7d0 tx=0x7fe7d400cc90 comp rx=0 tx=0).stop 2024-03-06T07:31:13.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 shutdown_connections 2024-03-06T07:31:13.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7b806b4d0 0x7fe7b806d990 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe7dc1561f0 0x7fe7dc0cbb50 unknown :-1 s=CLOSED pgs=858 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe7dc0cc9d0 0x7fe7dc158c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 --2- 172.21.15.86:0/964734966 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe7dc0aebf0 0x7fe7dc156900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:13.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 >> 172.21.15.86:0/964734966 conn(0x7fe7dc06afc0 msgr2=0x7fe7dc0cc2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:13.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 shutdown_connections 2024-03-06T07:31:13.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:13.898+0000 7fe7db200640 1 -- 172.21.15.86:0/964734966 wait complete. 2024-03-06T07:31:13.919 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:14.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.119+0000 7fb8a6600640 1 Processor -- start 2024-03-06T07:31:14.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.119+0000 7fb8a6600640 1 -- start start 2024-03-06T07:31:14.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.121+0000 7fb8a6600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb8a0144c00 0x7fb8a006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.121+0000 7fb8a6600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a015f9d0 0x7fb8a015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.121+0000 7fb8a6600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb8a006ee70 con 0x7fb8a00ce0a0 2024-03-06T07:31:14.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.121+0000 7fb8a6600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb8a0160370 con 0x7fb8a0144c00 2024-03-06T07:31:14.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.121+0000 7fb8a6600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb8a006cef0 con 0x7fb8a015f9d0 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.121+0000 7fb897e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a015f9d0 0x7fb8a015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.122+0000 7fb897e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a015f9d0 0x7fb8a015fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:37788/0 (socket says 172.21.15.86:37788) 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.122+0000 7fb897e00640 1 -- 172.21.15.86:0/588350180 learned_addr learned my addr 172.21.15.86:0/588350180 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.122+0000 7fb89f400640 1 --2- 172.21.15.86:0/588350180 >> v2:172.21.15.194:3300/0 conn(0x7fb8a0144c00 0x7fb8a006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.122+0000 7fb897e00640 1 -- 172.21.15.86:0/588350180 >> v2:172.21.15.194:3300/0 conn(0x7fb8a0144c00 msgr2=0x7fb8a006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.122+0000 7fb897e00640 1 --2- 172.21.15.86:0/588350180 >> v2:172.21.15.194:3300/0 conn(0x7fb8a0144c00 0x7fb8a006e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.122+0000 7fb897e00640 1 -- 172.21.15.86:0/588350180 >> v1:172.21.15.194:6789/0 conn(0x7fb8a00ce0a0 legacy=0x7fb8a00ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.122+0000 7fb897e00640 1 -- 172.21.15.86:0/588350180 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8a006d1d0 con 0x7fb8a015f9d0 2024-03-06T07:31:14.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.123+0000 7fb897e00640 1 --2- 172.21.15.86:0/588350180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a015f9d0 0x7fb8a015fe30 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fb888009870 tx=0x7fb88802b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=67f8ee606ff1fc4a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:14.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.124+0000 7fb89ea00640 1 -- 172.21.15.86:0/588350180 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb888034070 con 0x7fb8a015f9d0 2024-03-06T07:31:14.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.124+0000 7fb89ea00640 1 -- 172.21.15.86:0/588350180 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb888030840 con 0x7fb8a015f9d0 2024-03-06T07:31:14.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.124+0000 7fb89ea00640 1 -- 172.21.15.86:0/588350180 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb888030c10 con 0x7fb8a015f9d0 2024-03-06T07:31:14.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 -- 172.21.15.86:0/588350180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a015f9d0 msgr2=0x7fb8a015fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 --2- 172.21.15.86:0/588350180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a015f9d0 0x7fb8a015fe30 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fb888009870 tx=0x7fb88802b980 comp rx=0 tx=0).stop 2024-03-06T07:31:14.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 -- 172.21.15.86:0/588350180 shutdown_connections 2024-03-06T07:31:14.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 --2- 172.21.15.86:0/588350180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a015f9d0 0x7fb8a015fe30 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 --2- 172.21.15.86:0/588350180 >> v2:172.21.15.194:3300/0 conn(0x7fb8a0144c00 0x7fb8a006e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 -- 172.21.15.86:0/588350180 >> 172.21.15.86:0/588350180 conn(0x7fb8a00b1730 msgr2=0x7fb8a00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:14.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 -- 172.21.15.86:0/588350180 shutdown_connections 2024-03-06T07:31:14.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.125+0000 7fb8a6600640 1 -- 172.21.15.86:0/588350180 wait complete. 2024-03-06T07:31:14.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.127+0000 7fb8a6600640 1 Processor -- start 2024-03-06T07:31:14.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.127+0000 7fb8a6600640 1 -- start start 2024-03-06T07:31:14.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb8a6600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a00ce0a0 0x7fb8a0138f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb8a6600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb8a0144c00 0x7fb8a015ad20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb8a6600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb8a015f9d0 0x7fb8a015b260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb8a6600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8a0160730 con 0x7fb8a015f9d0 2024-03-06T07:31:14.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb8a6600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8a006d790 con 0x7fb8a0144c00 2024-03-06T07:31:14.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb8a6600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb8a006cb50 con 0x7fb8a00ce0a0 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb89fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb8a015f9d0 0x7fb8a015b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb89fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb8a015f9d0 0x7fb8a015b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60138/0 (socket says 172.21.15.86:60138) 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb89fe00640 1 -- 172.21.15.86:0/3364678407 learned_addr learned my addr 172.21.15.86:0/3364678407 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb89f400640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a00ce0a0 0x7fb8a0138f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.128+0000 7fb897e00640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb8a0144c00 0x7fb8a015ad20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb89fe00640 1 -- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a00ce0a0 msgr2=0x7fb8a0138f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb89fe00640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a00ce0a0 0x7fb8a0138f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb89fe00640 1 -- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb8a0144c00 msgr2=0x7fb8a015ad20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb89fe00640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb8a0144c00 0x7fb8a015ad20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb89fe00640 1 -- 172.21.15.86:0/3364678407 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8a01e55c0 con 0x7fb8a015f9d0 2024-03-06T07:31:14.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb89fe00640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb8a015f9d0 0x7fb8a015b260 secure :-1 s=READY pgs=859 cs=0 l=1 rev1=1 crypto rx=0x7fb89800c6b0 tx=0x7fb89800cb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:14.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb897400640 1 -- 172.21.15.86:0/3364678407 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb898003b70 con 0x7fb8a015f9d0 2024-03-06T07:31:14.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb897400640 1 -- 172.21.15.86:0/3364678407 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb898003d30 con 0x7fb8a015f9d0 2024-03-06T07:31:14.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb897400640 1 -- 172.21.15.86:0/3364678407 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb898005310 con 0x7fb8a015f9d0 2024-03-06T07:31:14.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb8a01e58a0 con 0x7fb8a015f9d0 2024-03-06T07:31:14.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.129+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb8a01e5e60 con 0x7fb8a015f9d0 2024-03-06T07:31:14.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.131+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb8a004f330 con 0x7fb8a015f9d0 2024-03-06T07:31:14.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.131+0000 7fb897400640 1 -- 172.21.15.86:0/3364678407 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb8980054b0 con 0x7fb8a015f9d0 2024-03-06T07:31:14.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.132+0000 7fb897400640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb87c06b4d0 0x7fb87c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.132+0000 7fb897400640 1 -- 172.21.15.86:0/3364678407 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb89808a1c0 con 0x7fb8a015f9d0 2024-03-06T07:31:14.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.133+0000 7fb89f400640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb87c06b4d0 0x7fb87c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.133+0000 7fb89f400640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb87c06b4d0 0x7fb87c06d990 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fb8a013c6e0 tx=0x7fb890008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:14.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.137+0000 7fb897400640 1 -- 172.21.15.86:0/3364678407 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb8a006ee70 con 0x7fb8a015f9d0 2024-03-06T07:31:14.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.341+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fb8a0072890 con 0x7fb8a015f9d0 2024-03-06T07:31:14.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.343+0000 7fb897400640 1 -- 172.21.15.86:0/3364678407 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v72) v1 ==== 74+0+16778 (secure 0 0 0) 0x7fb898058ee0 con 0x7fb8a015f9d0 2024-03-06T07:31:14.346 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:14.346 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":72,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:54.997378+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:30:33.566852+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-06T07:30:34.722690+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"11.14","mappings":[{"from":6,"to":0}]},{"pgid":"11.28","mappings":[{"from":6,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.345+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb87c06b4d0 msgr2=0x7fb87c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.345+0000 7fb8a6600640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb87c06b4d0 0x7fb87c06d990 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fb8a013c6e0 tx=0x7fb890008040 comp rx=0 tx=0).stop 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb8a015f9d0 msgr2=0x7fb8a015b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb8a015f9d0 0x7fb8a015b260 secure :-1 s=READY pgs=859 cs=0 l=1 rev1=1 crypto rx=0x7fb89800c6b0 tx=0x7fb89800cb70 comp rx=0 tx=0).stop 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 shutdown_connections 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb87c06b4d0 0x7fb87c06d990 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb8a015f9d0 0x7fb8a015b260 unknown :-1 s=CLOSED pgs=859 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb8a0144c00 0x7fb8a015ad20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 --2- 172.21.15.86:0/3364678407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb8a00ce0a0 0x7fb8a0138f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 >> 172.21.15.86:0/3364678407 conn(0x7fb8a00b1730 msgr2=0x7fb8a00cd060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 shutdown_connections 2024-03-06T07:31:14.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.346+0000 7fb8a6600640 1 -- 172.21.15.86:0/3364678407 wait complete. 2024-03-06T07:31:14.367 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 10, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:30:33.566852+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '69', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 11, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:30:34.722690+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '70', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:31:14.367 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:31:14.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.567+0000 7fb036200640 1 Processor -- start 2024-03-06T07:31:14.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.567+0000 7fb036200640 1 -- start start 2024-03-06T07:31:14.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.569+0000 7fb036200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb03015e130 0x7fb0300cf5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.569+0000 7fb036200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb0300cfaf0 0x7fb0300d1ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb036200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb0300d13e0 con 0x7fb0300d23e0 2024-03-06T07:31:14.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb036200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0300cf120 con 0x7fb03015e130 2024-03-06T07:31:14.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb036200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0300cead0 con 0x7fb0300cfaf0 2024-03-06T07:31:14.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb034e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb03015e130 0x7fb0300cf5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb034e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb03015e130 0x7fb0300cf5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60148/0 (socket says 172.21.15.86:60148) 2024-03-06T07:31:14.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb034e00640 1 -- 172.21.15.86:0/1628377759 learned_addr learned my addr 172.21.15.86:0/1628377759 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:14.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb034e00640 1 -- 172.21.15.86:0/1628377759 >> v2:172.21.15.194:3300/0 conn(0x7fb0300cfaf0 msgr2=0x7fb0300d1ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:14.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb034e00640 1 --2- 172.21.15.86:0/1628377759 >> v2:172.21.15.194:3300/0 conn(0x7fb0300cfaf0 0x7fb0300d1ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb034e00640 1 -- 172.21.15.86:0/1628377759 >> v1:172.21.15.194:6789/0 conn(0x7fb0300d23e0 legacy=0x7fb0300c7970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:14.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.570+0000 7fb034e00640 1 -- 172.21.15.86:0/1628377759 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0300cedb0 con 0x7fb03015e130 2024-03-06T07:31:14.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.572+0000 7fb034e00640 1 --2- 172.21.15.86:0/1628377759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb03015e130 0x7fb0300cf5b0 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7fb0300b9ed0 tx=0x7fb02402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e3ebbb0482db18ba server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:14.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.572+0000 7fb02f400640 1 -- 172.21.15.86:0/1628377759 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb024035020 con 0x7fb03015e130 2024-03-06T07:31:14.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.572+0000 7fb02f400640 1 -- 172.21.15.86:0/1628377759 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb024031870 con 0x7fb03015e130 2024-03-06T07:31:14.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.572+0000 7fb02f400640 1 -- 172.21.15.86:0/1628377759 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb024031c40 con 0x7fb03015e130 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 -- 172.21.15.86:0/1628377759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb03015e130 msgr2=0x7fb0300cf5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 --2- 172.21.15.86:0/1628377759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb03015e130 0x7fb0300cf5b0 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7fb0300b9ed0 tx=0x7fb02402b910 comp rx=0 tx=0).stop 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 -- 172.21.15.86:0/1628377759 shutdown_connections 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 --2- 172.21.15.86:0/1628377759 >> v2:172.21.15.194:3300/0 conn(0x7fb0300cfaf0 0x7fb0300d1ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 --2- 172.21.15.86:0/1628377759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb03015e130 0x7fb0300cf5b0 unknown :-1 s=CLOSED pgs=860 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 -- 172.21.15.86:0/1628377759 >> 172.21.15.86:0/1628377759 conn(0x7fb03009d070 msgr2=0x7fb0300d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 -- 172.21.15.86:0/1628377759 shutdown_connections 2024-03-06T07:31:14.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.573+0000 7fb036200640 1 -- 172.21.15.86:0/1628377759 wait complete. 2024-03-06T07:31:14.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.576+0000 7fb036200640 1 Processor -- start 2024-03-06T07:31:14.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.576+0000 7fb036200640 1 -- start start 2024-03-06T07:31:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb036200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0300cfaf0 0x7fb03009e5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb036200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0300d23e0 0x7fb0300cc990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb036200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb03015e130 0x7fb03014e6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb036200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0300c8200 con 0x7fb0300cfaf0 2024-03-06T07:31:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb036200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0300c8080 con 0x7fb0300d23e0 2024-03-06T07:31:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb036200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0300d06b0 con 0x7fb03015e130 2024-03-06T07:31:14.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0300cfaf0 0x7fb03009e5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0300cfaf0 0x7fb03009e5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60162/0 (socket says 172.21.15.86:60162) 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 -- 172.21.15.86:0/1574557614 learned_addr learned my addr 172.21.15.86:0/1574557614 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 -- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb03015e130 msgr2=0x7fb03014e6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb03015e130 0x7fb03014e6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 -- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0300d23e0 msgr2=0x7fb0300cc990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb02fe00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0300d23e0 0x7fb0300cc990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0300d23e0 0x7fb0300cc990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb034e00640 1 -- 172.21.15.86:0/1574557614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb03009eb10 con 0x7fb0300cfaf0 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.578+0000 7fb02fe00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0300d23e0 0x7fb0300cc990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:14.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.579+0000 7fb034e00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0300cfaf0 0x7fb03009e5d0 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7fb024000c00 tx=0x7fb0240046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.579+0000 7fb02cc00640 1 -- 172.21.15.86:0/1574557614 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb02403ad60 con 0x7fb0300cfaf0 2024-03-06T07:31:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.579+0000 7fb02cc00640 1 -- 172.21.15.86:0/1574557614 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0240443f0 con 0x7fb0300cfaf0 2024-03-06T07:31:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.579+0000 7fb02cc00640 1 -- 172.21.15.86:0/1574557614 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0240447e0 con 0x7fb0300cfaf0 2024-03-06T07:31:14.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.579+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb03014d630 con 0x7fb0300cfaf0 2024-03-06T07:31:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.579+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb03014dbf0 con 0x7fb0300cfaf0 2024-03-06T07:31:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.581+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb03004f330 con 0x7fb0300cfaf0 2024-03-06T07:31:14.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.581+0000 7fb02cc00640 1 -- 172.21.15.86:0/1574557614 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb02404b060 con 0x7fb0300cfaf0 2024-03-06T07:31:14.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.583+0000 7fb02cc00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb01006b4d0 0x7fb01006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:14.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.583+0000 7fb02cc00640 1 -- 172.21.15.86:0/1574557614 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb0240ac3e0 con 0x7fb0300cfaf0 2024-03-06T07:31:14.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.583+0000 7fb02fe00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb01006b4d0 0x7fb01006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:14.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.584+0000 7fb02fe00640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb01006b4d0 0x7fb01006d990 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fb01c006fd0 tx=0x7fb01c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:14.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.587+0000 7fb02cc00640 1 -- 172.21.15.86:0/1574557614 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb02403f070 con 0x7fb0300cfaf0 2024-03-06T07:31:14.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.781+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fb0300c7090 con 0x7fb0300cfaf0 2024-03-06T07:31:14.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.782+0000 7fb02cc00640 1 -- 172.21.15.86:0/1574557614 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v72) v1 ==== 93+0+10 (secure 0 0 0) 0x7fb02407b220 con 0x7fb0300cfaf0 2024-03-06T07:31:14.785 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:31:14.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb01006b4d0 msgr2=0x7fb01006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb01006b4d0 0x7fb01006d990 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fb01c006fd0 tx=0x7fb01c008040 comp rx=0 tx=0).stop 2024-03-06T07:31:14.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0300cfaf0 msgr2=0x7fb03009e5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:14.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0300cfaf0 0x7fb03009e5d0 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7fb024000c00 tx=0x7fb0240046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 shutdown_connections 2024-03-06T07:31:14.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb01006b4d0 0x7fb01006d990 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb03015e130 0x7fb03014e6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0300d23e0 0x7fb0300cc990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 --2- 172.21.15.86:0/1574557614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0300cfaf0 0x7fb03009e5d0 unknown :-1 s=CLOSED pgs=861 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 >> 172.21.15.86:0/1574557614 conn(0x7fb03009d070 msgr2=0x7fb03006c4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 shutdown_connections 2024-03-06T07:31:14.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:14.785+0000 7fb036200640 1 -- 172.21.15.86:0/1574557614 wait complete. 2024-03-06T07:31:14.807 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:31:15.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.006+0000 7f7c7d200640 1 Processor -- start 2024-03-06T07:31:15.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.006+0000 7f7c7d200640 1 -- start start 2024-03-06T07:31:15.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c7d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780d1ea0 0x7f7c780d2300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c7d200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7c780c7970 0x7f7c780c7e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c7d200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7c780d13e0 con 0x7f7c7815e130 2024-03-06T07:31:15.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c7d200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7c780d2840 con 0x7f7c780c7970 2024-03-06T07:31:15.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c7d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7c780cf120 con 0x7f7c780d1ea0 2024-03-06T07:31:15.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c77e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7c780c7970 0x7f7c780c7e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c77e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7c780c7970 0x7f7c780c7e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60166/0 (socket says 172.21.15.86:60166) 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c77e00640 1 -- 172.21.15.86:0/3511967321 learned_addr learned my addr 172.21.15.86:0/3511967321 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c77400640 1 --1- 172.21.15.86:0/3511967321 >> v1:172.21.15.86:6789/0 conn(0x7f7c7815e130 0x7f7c780cf5b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46140/0 (socket says 172.21.15.86:46140) 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.008+0000 7f7c6fe00640 1 --2- 172.21.15.86:0/3511967321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780d1ea0 0x7f7c780d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.009+0000 7f7c77e00640 1 -- 172.21.15.86:0/3511967321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780d1ea0 msgr2=0x7f7c780d2300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.009+0000 7f7c77e00640 1 --2- 172.21.15.86:0/3511967321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780d1ea0 0x7f7c780d2300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.009+0000 7f7c77e00640 1 -- 172.21.15.86:0/3511967321 >> v1:172.21.15.86:6789/0 conn(0x7f7c7815e130 legacy=0x7f7c780cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.009+0000 7f7c77e00640 1 -- 172.21.15.86:0/3511967321 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c780cead0 con 0x7f7c780c7970 2024-03-06T07:31:15.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.009+0000 7f7c76a00640 1 -- 172.21.15.86:0/3511967321 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2801443509 0 0) 0x7f7c780d13e0 con 0x7f7c7815e130 2024-03-06T07:31:15.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.010+0000 7f7c77e00640 1 --2- 172.21.15.86:0/3511967321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c780c7970 0x7f7c780c7e00 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7f7c70009e50 tx=0x7f7c7002bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f8f9e0e4d65402b8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.010+0000 7f7c76a00640 1 -- 172.21.15.86:0/3511967321 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c7002cb00 con 0x7f7c780c7970 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.010+0000 7f7c76a00640 1 -- 172.21.15.86:0/3511967321 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7c7002cca0 con 0x7f7c780c7970 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.010+0000 7f7c76a00640 1 -- 172.21.15.86:0/3511967321 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c70034610 con 0x7f7c780c7970 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 -- 172.21.15.86:0/3511967321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c780c7970 msgr2=0x7f7c780c7e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 --2- 172.21.15.86:0/3511967321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c780c7970 0x7f7c780c7e00 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7f7c70009e50 tx=0x7f7c7002bf60 comp rx=0 tx=0).stop 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 -- 172.21.15.86:0/3511967321 shutdown_connections 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 --2- 172.21.15.86:0/3511967321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c780c7970 0x7f7c780c7e00 unknown :-1 s=CLOSED pgs=863 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 --2- 172.21.15.86:0/3511967321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780d1ea0 0x7f7c780d2300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 -- 172.21.15.86:0/3511967321 >> 172.21.15.86:0/3511967321 conn(0x7f7c7809d070 msgr2=0x7f7c780d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 -- 172.21.15.86:0/3511967321 shutdown_connections 2024-03-06T07:31:15.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.011+0000 7f7c7d200640 1 -- 172.21.15.86:0/3511967321 wait complete. 2024-03-06T07:31:15.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.013+0000 7f7c7d200640 1 Processor -- start 2024-03-06T07:31:15.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.013+0000 7f7c7d200640 1 -- start start 2024-03-06T07:31:15.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c7d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780c7970 0x7f7c780b8000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c7d200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7c780d1ea0 0x7f7c780b97f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c7d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c7815e130 0x7f7c7814a900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c7d200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c780cfcc0 con 0x7f7c7815e130 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c7d200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c780c8340 con 0x7f7c780d1ea0 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c7d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7c780d16c0 con 0x7f7c780c7970 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c77e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c7815e130 0x7f7c7814a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c77e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c7815e130 0x7f7c7814a900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60174/0 (socket says 172.21.15.86:60174) 2024-03-06T07:31:15.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c77e00640 1 -- 172.21.15.86:0/1072621593 learned_addr learned my addr 172.21.15.86:0/1072621593 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c6fe00640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7c780d1ea0 0x7f7c780b97f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.014+0000 7f7c77400640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780c7970 0x7f7c780b8000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.015+0000 7f7c77e00640 1 -- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780c7970 msgr2=0x7f7c780b8000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.015+0000 7f7c77e00640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780c7970 0x7f7c780b8000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.015+0000 7f7c77e00640 1 -- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7c780d1ea0 msgr2=0x7f7c780b97f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.015+0000 7f7c77e00640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7c780d1ea0 0x7f7c780b97f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.015+0000 7f7c77e00640 1 -- 172.21.15.86:0/1072621593 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c780b9d30 con 0x7f7c7815e130 2024-03-06T07:31:15.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.015+0000 7f7c77e00640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c7815e130 0x7f7c7814a900 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f7c70036040 tx=0x7f7c70004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.018+0000 7f7c6f400640 1 -- 172.21.15.86:0/1072621593 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c70043020 con 0x7f7c7815e130 2024-03-06T07:31:15.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.018+0000 7f7c6f400640 1 -- 172.21.15.86:0/1072621593 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7c7002ce50 con 0x7f7c7815e130 2024-03-06T07:31:15.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.018+0000 7f7c6f400640 1 -- 172.21.15.86:0/1072621593 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c70009510 con 0x7f7c7815e130 2024-03-06T07:31:15.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.018+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7c780b6f60 con 0x7f7c7815e130 2024-03-06T07:31:15.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.018+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c780b7520 con 0x7f7c7815e130 2024-03-06T07:31:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.019+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7c7804f330 con 0x7f7c7815e130 2024-03-06T07:31:15.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.020+0000 7f7c6f400640 1 -- 172.21.15.86:0/1072621593 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7c70046020 con 0x7f7c7815e130 2024-03-06T07:31:15.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.021+0000 7f7c6f400640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7c5406b500 0x7f7c5406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.021+0000 7f7c6f400640 1 -- 172.21.15.86:0/1072621593 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f7c700ac820 con 0x7f7c7815e130 2024-03-06T07:31:15.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.021+0000 7f7c77400640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7c5406b500 0x7f7c5406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.022+0000 7f7c77400640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7c5406b500 0x7f7c5406d9c0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f7c68007fd0 tx=0x7f7c68009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.026+0000 7f7c6f400640 1 -- 172.21.15.86:0/1072621593 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7c7004d030 con 0x7f7c7815e130 2024-03-06T07:31:15.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.222+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f7c7809d690 con 0x7f7c7815e130 2024-03-06T07:31:15.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.223+0000 7f7c6f400640 1 -- 172.21.15.86:0/1072621593 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v72) v1 ==== 104+0+11 (secure 0 0 0) 0x7f7c7007b650 con 0x7f7c7815e130 2024-03-06T07:31:15.225 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:31:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7c5406b500 msgr2=0x7f7c5406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7c5406b500 0x7f7c5406d9c0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f7c68007fd0 tx=0x7f7c68009040 comp rx=0 tx=0).stop 2024-03-06T07:31:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c7815e130 msgr2=0x7f7c7814a900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c7815e130 0x7f7c7814a900 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f7c70036040 tx=0x7f7c70004a40 comp rx=0 tx=0).stop 2024-03-06T07:31:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 shutdown_connections 2024-03-06T07:31:15.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7c5406b500 0x7f7c5406d9c0 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7c7815e130 0x7f7c7814a900 unknown :-1 s=CLOSED pgs=864 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7c780d1ea0 0x7f7c780b97f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 --2- 172.21.15.86:0/1072621593 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7c780c7970 0x7f7c780b8000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.225+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 >> 172.21.15.86:0/1072621593 conn(0x7f7c7809d070 msgr2=0x7f7c780c8f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:15.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.226+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 shutdown_connections 2024-03-06T07:31:15.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.226+0000 7f7c7d200640 1 -- 172.21.15.86:0/1072621593 wait complete. 2024-03-06T07:31:15.246 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:31:15.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.446+0000 7fd3f1400640 1 Processor -- start 2024-03-06T07:31:15.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.446+0000 7fd3f1400640 1 -- start start 2024-03-06T07:31:15.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.448+0000 7fd3f1400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3ec0b3530 0x7fd3ec15b780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3f1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0ad670 0x7fd3ec0adb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3f1400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3ec15bcc0 con 0x7fd3ec0b3910 2024-03-06T07:31:15.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3f1400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3ec0ce690 con 0x7fd3ec0b3530 2024-03-06T07:31:15.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3f1400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3ec0ce970 con 0x7fd3ec0ad670 2024-03-06T07:31:15.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3ea800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3ec0b3530 0x7fd3ec15b780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3ea800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3ec0b3530 0x7fd3ec15b780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60182/0 (socket says 172.21.15.86:60182) 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3ea800640 1 -- 172.21.15.86:0/1169080293 learned_addr learned my addr 172.21.15.86:0/1169080293 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3eb200640 1 --2- 172.21.15.86:0/1169080293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0ad670 0x7fd3ec0adb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3ea800640 1 -- 172.21.15.86:0/1169080293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0ad670 msgr2=0x7fd3ec0adb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3ea800640 1 --2- 172.21.15.86:0/1169080293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0ad670 0x7fd3ec0adb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3ea800640 1 -- 172.21.15.86:0/1169080293 >> v1:172.21.15.194:6789/0 conn(0x7fd3ec0b3910 legacy=0x7fd3ec0acf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3ea800640 1 -- 172.21.15.86:0/1169080293 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3ec0cec50 con 0x7fd3ec0b3530 2024-03-06T07:31:15.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.449+0000 7fd3eb200640 1 --2- 172.21.15.86:0/1169080293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0ad670 0x7fd3ec0adb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:15.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.451+0000 7fd3ea800640 1 --2- 172.21.15.86:0/1169080293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3530 0x7fd3ec15b780 secure :-1 s=READY pgs=865 cs=0 l=1 rev1=1 crypto rx=0x7fd3dc009870 tx=0x7fd3dc02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=510fcc98044445af server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.451+0000 7fd3e9e00640 1 -- 172.21.15.86:0/1169080293 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3dc035020 con 0x7fd3ec0b3530 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.451+0000 7fd3e9e00640 1 -- 172.21.15.86:0/1169080293 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3dc031640 con 0x7fd3ec0b3530 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.451+0000 7fd3e9e00640 1 -- 172.21.15.86:0/1169080293 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3dc02d040 con 0x7fd3ec0b3530 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 -- 172.21.15.86:0/1169080293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3530 msgr2=0x7fd3ec15b780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 --2- 172.21.15.86:0/1169080293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3530 0x7fd3ec15b780 secure :-1 s=READY pgs=865 cs=0 l=1 rev1=1 crypto rx=0x7fd3dc009870 tx=0x7fd3dc02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 -- 172.21.15.86:0/1169080293 shutdown_connections 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 --2- 172.21.15.86:0/1169080293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0ad670 0x7fd3ec0adb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 --2- 172.21.15.86:0/1169080293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3530 0x7fd3ec15b780 unknown :-1 s=CLOSED pgs=865 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 -- 172.21.15.86:0/1169080293 >> 172.21.15.86:0/1169080293 conn(0x7fd3ec0b1940 msgr2=0x7fd3ec0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 -- 172.21.15.86:0/1169080293 shutdown_connections 2024-03-06T07:31:15.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.452+0000 7fd3f1400640 1 -- 172.21.15.86:0/1169080293 wait complete. 2024-03-06T07:31:15.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.453+0000 7fd3f1400640 1 Processor -- start 2024-03-06T07:31:15.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.454+0000 7fd3f1400640 1 -- start start 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.454+0000 7fd3f1400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3ec0ad670 0x7fd3ec06cd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.454+0000 7fd3f1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0b3530 0x7fd3ec072170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.454+0000 7fd3f1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3910 0x7fd3ec15cef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.454+0000 7fd3f1400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3ec0b3d90 con 0x7fd3ec0b3910 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3f1400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3ec0cf210 con 0x7fd3ec0ad670 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3f1400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3ec15aa30 con 0x7fd3ec0b3530 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3910 0x7fd3ec15cef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3910 0x7fd3ec15cef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60196/0 (socket says 172.21.15.86:60196) 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 -- 172.21.15.86:0/4145710571 learned_addr learned my addr 172.21.15.86:0/4145710571 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3ea800640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3ec0ad670 0x7fd3ec06cd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3e3e00640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0b3530 0x7fd3ec072170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 -- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0b3530 msgr2=0x7fd3ec072170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0b3530 0x7fd3ec072170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 -- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3ec0ad670 msgr2=0x7fd3ec06cd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3ec0ad670 0x7fd3ec06cd30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.455+0000 7fd3eb200640 1 -- 172.21.15.86:0/4145710571 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3ec1484f0 con 0x7fd3ec0b3910 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.456+0000 7fd3eb200640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3910 0x7fd3ec15cef0 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7fd3e400cf20 tx=0x7fd3e400a7d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.456+0000 7fd3e2a00640 1 -- 172.21.15.86:0/4145710571 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e401a070 con 0x7fd3ec0b3910 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.456+0000 7fd3e2a00640 1 -- 172.21.15.86:0/4145710571 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3e400d840 con 0x7fd3ec0b3910 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.456+0000 7fd3e2a00640 1 -- 172.21.15.86:0/4145710571 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e400dc30 con 0x7fd3ec0b3910 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.456+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3ec06d440 con 0x7fd3ec0b3910 2024-03-06T07:31:15.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.456+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3ec06da00 con 0x7fd3ec0b3910 2024-03-06T07:31:15.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.457+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd3ec04f330 con 0x7fd3ec0b3910 2024-03-06T07:31:15.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.458+0000 7fd3e2a00640 1 -- 172.21.15.86:0/4145710571 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd3e401d070 con 0x7fd3ec0b3910 2024-03-06T07:31:15.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.459+0000 7fd3e2a00640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3c806b4d0 0x7fd3c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.459+0000 7fd3e2a00640 1 -- 172.21.15.86:0/4145710571 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd3e408aba0 con 0x7fd3ec0b3910 2024-03-06T07:31:15.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.459+0000 7fd3ea800640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3c806b4d0 0x7fd3c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.460+0000 7fd3ea800640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3c806b4d0 0x7fd3c806d990 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fd3dc036000 tx=0x7fd3dc03c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.464+0000 7fd3e2a00640 1 -- 172.21.15.86:0/4145710571 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd3ec1484f0 con 0x7fd3ec0b3910 2024-03-06T07:31:15.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.658+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fd3ec0b0040 con 0x7fd3ec0b3910 2024-03-06T07:31:15.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.659+0000 7fd3e2a00640 1 -- 172.21.15.86:0/4145710571 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v72) v1 ==== 100+0+11 (secure 0 0 0) 0x7fd3e4059940 con 0x7fd3ec0b3910 2024-03-06T07:31:15.662 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.661+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3c806b4d0 msgr2=0x7fd3c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.661+0000 7fd3f1400640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3c806b4d0 0x7fd3c806d990 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fd3dc036000 tx=0x7fd3dc03c040 comp rx=0 tx=0).stop 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.661+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3910 msgr2=0x7fd3ec15cef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.661+0000 7fd3f1400640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3910 0x7fd3ec15cef0 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7fd3e400cf20 tx=0x7fd3e400a7d0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 shutdown_connections 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3c806b4d0 0x7fd3c806d990 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3ec0b3910 0x7fd3ec15cef0 unknown :-1 s=CLOSED pgs=866 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3ec0b3530 0x7fd3ec072170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 --2- 172.21.15.86:0/4145710571 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3ec0ad670 0x7fd3ec06cd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 >> 172.21.15.86:0/4145710571 conn(0x7fd3ec0b1940 msgr2=0x7fd3ec0b02c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 shutdown_connections 2024-03-06T07:31:15.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.662+0000 7fd3f1400640 1 -- 172.21.15.86:0/4145710571 wait complete. 2024-03-06T07:31:15.683 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:15.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.878+0000 7f3b4cc00640 1 Processor -- start 2024-03-06T07:31:15.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.879+0000 7f3b4cc00640 1 -- start start 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.880+0000 7f3b4cc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b4815acb0 0x7f3b480b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.880+0000 7f3b4cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3b480b4680 0x7f3b4806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.880+0000 7f3b4cc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3b4815b090 con 0x7f3b48072340 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.880+0000 7f3b4cc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3b480b2a30 con 0x7f3b480b4680 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.880+0000 7f3b4cc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b480b4360 con 0x7f3b4815acb0 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.880+0000 7f3b3fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3b480b4680 0x7f3b4806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.881+0000 7f3b46800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3b48072340 0x7f3b480ccfb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34154/0 (socket says 172.21.15.86:34154) 2024-03-06T07:31:15.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.881+0000 7f3b46800640 1 -- 172.21.15.86:0/1067972791 learned_addr learned my addr 172.21.15.86:0/1067972791 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:15.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.881+0000 7f3b45e00640 1 --2- 172.21.15.86:0/1067972791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b4815acb0 0x7f3b480b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.881+0000 7f3b3fe00640 1 -- 172.21.15.86:0/1067972791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b4815acb0 msgr2=0x7f3b480b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:15.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.881+0000 7f3b3fe00640 1 --2- 172.21.15.86:0/1067972791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b4815acb0 0x7f3b480b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.881+0000 7f3b3fe00640 1 -- 172.21.15.86:0/1067972791 >> v1:172.21.15.86:6789/0 conn(0x7f3b48072340 legacy=0x7f3b480ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.881+0000 7f3b3fe00640 1 -- 172.21.15.86:0/1067972791 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b480b2cf0 con 0x7f3b480b4680 2024-03-06T07:31:15.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.882+0000 7f3b3fe00640 1 --2- 172.21.15.86:0/1067972791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4806e4f0 secure :-1 s=READY pgs=868 cs=0 l=1 rev1=1 crypto rx=0x7f3b300097a0 tx=0x7f3b3002b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bbdfed11e50e91c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.882+0000 7f3b45400640 1 -- 172.21.15.86:0/1067972791 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b30035020 con 0x7f3b480b4680 2024-03-06T07:31:15.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b45400640 1 -- 172.21.15.86:0/1067972791 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b30031810 con 0x7f3b480b4680 2024-03-06T07:31:15.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b45400640 1 -- 172.21.15.86:0/1067972791 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b30031be0 con 0x7f3b480b4680 2024-03-06T07:31:15.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 -- 172.21.15.86:0/1067972791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 msgr2=0x7f3b4806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/1067972791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4806e4f0 secure :-1 s=READY pgs=868 cs=0 l=1 rev1=1 crypto rx=0x7f3b300097a0 tx=0x7f3b3002b8b0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 -- 172.21.15.86:0/1067972791 shutdown_connections 2024-03-06T07:31:15.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/1067972791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4806e4f0 unknown :-1 s=CLOSED pgs=868 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/1067972791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b4815acb0 0x7f3b480b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 -- 172.21.15.86:0/1067972791 >> 172.21.15.86:0/1067972791 conn(0x7f3b480b1730 msgr2=0x7f3b480b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:15.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 -- 172.21.15.86:0/1067972791 shutdown_connections 2024-03-06T07:31:15.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.883+0000 7f3b4cc00640 1 -- 172.21.15.86:0/1067972791 wait complete. 2024-03-06T07:31:15.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.885+0000 7f3b4cc00640 1 Processor -- start 2024-03-06T07:31:15.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.885+0000 7f3b4cc00640 1 -- start start 2024-03-06T07:31:15.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.886+0000 7f3b4cc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b48072340 0x7f3b481e5230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.886+0000 7f3b4cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4815bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.886+0000 7f3b4cc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b4815acb0 0x7f3b481f3410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.886+0000 7f3b4cc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b480b3d10 con 0x7f3b480b4680 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.886+0000 7f3b4cc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b480b32b0 con 0x7f3b4815acb0 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.886+0000 7f3b4cc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b480b4010 con 0x7f3b48072340 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.886+0000 7f3b3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4815bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4815bdf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32902/0 (socket says 172.21.15.86:32902) 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 -- 172.21.15.86:0/2851586079 learned_addr learned my addr 172.21.15.86:0/2851586079 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b45e00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b48072340 0x7f3b481e5230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 -- 172.21.15.86:0/2851586079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b48072340 msgr2=0x7f3b481e5230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b48072340 0x7f3b481e5230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 -- 172.21.15.86:0/2851586079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b4815acb0 msgr2=0x7f3b481f3410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b4815acb0 0x7f3b481f3410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 -- 172.21.15.86:0/2851586079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b4815c330 con 0x7f3b480b4680 2024-03-06T07:31:15.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3fe00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4815bdf0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7f3b30009890 tx=0x7f3b300046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.887+0000 7f3b3e000640 1 -- 172.21.15.86:0/2851586079 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b30041070 con 0x7f3b480b4680 2024-03-06T07:31:15.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.888+0000 7f3b3e000640 1 -- 172.21.15.86:0/2851586079 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b3003f070 con 0x7f3b480b4680 2024-03-06T07:31:15.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.888+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b4815c5d0 con 0x7f3b480b4680 2024-03-06T07:31:15.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.888+0000 7f3b3e000640 1 -- 172.21.15.86:0/2851586079 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b30035040 con 0x7f3b480b4680 2024-03-06T07:31:15.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.888+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b4815cb90 con 0x7f3b480b4680 2024-03-06T07:31:15.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.889+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3b4804f330 con 0x7f3b480b4680 2024-03-06T07:31:15.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.890+0000 7f3b3e000640 1 -- 172.21.15.86:0/2851586079 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3b30038020 con 0x7f3b480b4680 2024-03-06T07:31:15.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.891+0000 7f3b3e000640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b2406b4d0 0x7f3b2406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:15.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.891+0000 7f3b3e000640 1 -- 172.21.15.86:0/2851586079 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3b300ac0e0 con 0x7f3b480b4680 2024-03-06T07:31:15.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.891+0000 7f3b45e00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b2406b4d0 0x7f3b2406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:15.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.891+0000 7f3b45e00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b2406b4d0 0x7f3b2406d990 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f3b38006fd0 tx=0x7f3b38008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:15.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:15.897+0000 7f3b3e000640 1 -- 172.21.15.86:0/2851586079 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3b3007ae80 con 0x7f3b480b4680 2024-03-06T07:31:16.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.146+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3b480b3970 con 0x7f3b480b4680 2024-03-06T07:31:16.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.147+0000 7f3b3e000640 1 -- 172.21.15.86:0/2851586079 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f3b3007a780 con 0x7f3b480b4680 2024-03-06T07:31:16.150 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:16.150 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:31:16.150 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:31:16.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b2406b4d0 msgr2=0x7f3b2406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b2406b4d0 0x7f3b2406d990 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f3b38006fd0 tx=0x7f3b38008040 comp rx=0 tx=0).stop 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 msgr2=0x7f3b4815bdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4815bdf0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7f3b30009890 tx=0x7f3b300046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 shutdown_connections 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3b2406b4d0 0x7f3b2406d990 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3b4815acb0 0x7f3b481f3410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3b480b4680 0x7f3b4815bdf0 unknown :-1 s=CLOSED pgs=869 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 --2- 172.21.15.86:0/2851586079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3b48072340 0x7f3b481e5230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 >> 172.21.15.86:0/2851586079 conn(0x7f3b480b1730 msgr2=0x7f3b48071180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 shutdown_connections 2024-03-06T07:31:16.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.150+0000 7f3b4cc00640 1 -- 172.21.15.86:0/2851586079 wait complete. 2024-03-06T07:31:16.169 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:16.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.377+0000 7f448aa00640 1 Processor -- start 2024-03-06T07:31:16.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.377+0000 7f448aa00640 1 -- start start 2024-03-06T07:31:16.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448aa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f448415e0d0 0x7f448406afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44840af950 0x7f44840ad160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448aa00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f44840b1440 con 0x7f448415e4b0 2024-03-06T07:31:16.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448aa00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f448406b500 con 0x7f44840af950 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448aa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f448406c0d0 con 0x7f448415e0d0 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448a000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44840af950 0x7f44840ad160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448a000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44840af950 0x7f44840ad160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32918/0 (socket says 172.21.15.86:32918) 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f4488c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f448415e4b0 0x7f44840af240 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34168/0 (socket says 172.21.15.86:34168) 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f448a000640 1 -- 172.21.15.86:0/2945086435 learned_addr learned my addr 172.21.15.86:0/2945086435 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.379+0000 7f4489600640 1 --2- 172.21.15.86:0/2945086435 >> v2:172.21.15.194:3300/0 conn(0x7f448415e0d0 0x7f448406afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.380+0000 7f448a000640 1 -- 172.21.15.86:0/2945086435 >> v2:172.21.15.194:3300/0 conn(0x7f448415e0d0 msgr2=0x7f448406afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.380+0000 7f448a000640 1 --2- 172.21.15.86:0/2945086435 >> v2:172.21.15.194:3300/0 conn(0x7f448415e0d0 0x7f448406afc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.380+0000 7f448a000640 1 -- 172.21.15.86:0/2945086435 >> v1:172.21.15.86:6789/0 conn(0x7f448415e4b0 legacy=0x7f44840af240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.380+0000 7f448a000640 1 -- 172.21.15.86:0/2945086435 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44840b16e0 con 0x7f44840af950 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.380+0000 7f4483e00640 1 -- 172.21.15.86:0/2945086435 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2960303164 0 0) 0x7f44840b1440 con 0x7f448415e4b0 2024-03-06T07:31:16.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.380+0000 7f4489600640 1 --2- 172.21.15.86:0/2945086435 >> v2:172.21.15.194:3300/0 conn(0x7f448415e0d0 0x7f448406afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:31:16.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.381+0000 7f448a000640 1 --2- 172.21.15.86:0/2945086435 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44840af950 0x7f44840ad160 secure :-1 s=READY pgs=871 cs=0 l=1 rev1=1 crypto rx=0x7f44840724d0 tx=0x7f447c02c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=417478e58589ac94 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.381+0000 7f4483e00640 1 -- 172.21.15.86:0/2945086435 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f447c02cbb0 con 0x7f44840af950 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.381+0000 7f4483e00640 1 -- 172.21.15.86:0/2945086435 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f447c02cd50 con 0x7f44840af950 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.381+0000 7f4483e00640 1 -- 172.21.15.86:0/2945086435 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f447c034610 con 0x7f44840af950 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 -- 172.21.15.86:0/2945086435 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44840af950 msgr2=0x7f44840ad160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 --2- 172.21.15.86:0/2945086435 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44840af950 0x7f44840ad160 secure :-1 s=READY pgs=871 cs=0 l=1 rev1=1 crypto rx=0x7f44840724d0 tx=0x7f447c02c010 comp rx=0 tx=0).stop 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 -- 172.21.15.86:0/2945086435 shutdown_connections 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 --2- 172.21.15.86:0/2945086435 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44840af950 0x7f44840ad160 unknown :-1 s=CLOSED pgs=871 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 --2- 172.21.15.86:0/2945086435 >> v2:172.21.15.194:3300/0 conn(0x7f448415e0d0 0x7f448406afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 -- 172.21.15.86:0/2945086435 >> 172.21.15.86:0/2945086435 conn(0x7f44840b4fe0 msgr2=0x7f44840b56a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 -- 172.21.15.86:0/2945086435 shutdown_connections 2024-03-06T07:31:16.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.382+0000 7f448aa00640 1 -- 172.21.15.86:0/2945086435 wait complete. 2024-03-06T07:31:16.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.383+0000 7f448aa00640 1 Processor -- start 2024-03-06T07:31:16.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.384+0000 7f448aa00640 1 -- start start 2024-03-06T07:31:16.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.384+0000 7f448aa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44840af950 0x7f448415ae50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.384+0000 7f448aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448415e0d0 0x7f44841f3300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.384+0000 7f448aa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f448415e4b0 0x7f4484156fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.384+0000 7f448aa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f448406c320 con 0x7f448415e0d0 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.384+0000 7f448aa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f448406c690 con 0x7f44840af950 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.384+0000 7f448aa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f44840b5db0 con 0x7f448415e4b0 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448415e0d0 0x7f44841f3300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448415e0d0 0x7f44841f3300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32928/0 (socket says 172.21.15.86:32928) 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 -- 172.21.15.86:0/213000614 learned_addr learned my addr 172.21.15.86:0/213000614 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4489600640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44840af950 0x7f448415ae50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f448a000640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f448415e4b0 0x7f4484156fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 -- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f448415e4b0 msgr2=0x7f4484156fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f448415e4b0 0x7f4484156fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 -- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44840af950 msgr2=0x7f448415ae50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44840af950 0x7f448415ae50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 -- 172.21.15.86:0/213000614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44841f3840 con 0x7f448415e0d0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.385+0000 7f4488c00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448415e0d0 0x7f44841f3300 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7f447000c430 tx=0x7f447000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.386+0000 7f4481600640 1 -- 172.21.15.86:0/213000614 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44700074e0 con 0x7f448415e0d0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.386+0000 7f4481600640 1 -- 172.21.15.86:0/213000614 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f44700047e0 con 0x7f448415e0d0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.386+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44841f3ab0 con 0x7f448415e0d0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.386+0000 7f4481600640 1 -- 172.21.15.86:0/213000614 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4470004bd0 con 0x7f448415e0d0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.386+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44841f4070 con 0x7f448415e0d0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.387+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f448404f330 con 0x7f448415e0d0 2024-03-06T07:31:16.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.388+0000 7f4481600640 1 -- 172.21.15.86:0/213000614 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4470011070 con 0x7f448415e0d0 2024-03-06T07:31:16.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.389+0000 7f4481600640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446406b4d0 0x7f446406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.389+0000 7f4481600640 1 -- 172.21.15.86:0/213000614 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4470089eb0 con 0x7f448415e0d0 2024-03-06T07:31:16.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.389+0000 7f4489600640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446406b4d0 0x7f446406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.390+0000 7f4489600640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446406b4d0 0x7f446406d990 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f4478006fd0 tx=0x7f4478008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:16.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.398+0000 7f4481600640 1 -- 172.21.15.86:0/213000614 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4470058cf0 con 0x7f448415e0d0 2024-03-06T07:31:16.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.586+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4484067590 con 0x7f448415e0d0 2024-03-06T07:31:16.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.589+0000 7f4481600640 1 -- 172.21.15.86:0/213000614 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v72) v1 ==== 74+0+16778 (secure 0 0 0) 0x7f44700585f0 con 0x7f448415e0d0 2024-03-06T07:31:16.592 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:16.592 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":72,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:54.997378+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:30:33.566852+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-06T07:30:34.722690+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"11.14","mappings":[{"from":6,"to":0}]},{"pgid":"11.28","mappings":[{"from":6,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:16.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.591+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446406b4d0 msgr2=0x7f446406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.591+0000 7f448aa00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446406b4d0 0x7f446406d990 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f4478006fd0 tx=0x7f4478008040 comp rx=0 tx=0).stop 2024-03-06T07:31:16.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.591+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448415e0d0 msgr2=0x7f44841f3300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448415e0d0 0x7f44841f3300 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7f447000c430 tx=0x7f447000c8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 shutdown_connections 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446406b4d0 0x7f446406d990 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f448415e4b0 0x7f4484156fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448415e0d0 0x7f44841f3300 unknown :-1 s=CLOSED pgs=872 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 --2- 172.21.15.86:0/213000614 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44840af950 0x7f448415ae50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 >> 172.21.15.86:0/213000614 conn(0x7f44840b4fe0 msgr2=0x7f44840ad2d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 shutdown_connections 2024-03-06T07:31:16.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.592+0000 7f448aa00640 1 -- 172.21.15.86:0/213000614 wait complete. 2024-03-06T07:31:16.612 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:31:16.612 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:16.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.809+0000 7f9f11a00640 1 Processor -- start 2024-03-06T07:31:16.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.809+0000 7f9f11a00640 1 -- start start 2024-03-06T07:31:16.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.810+0000 7f9f11a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9f0c15f470 0x7f9f0c0d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.810+0000 7f9f11a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9f0c0d0df0 0x7f9f0c160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.810+0000 7f9f11a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f0c0d1200 con 0x7f9f0c15e0d0 2024-03-06T07:31:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f11a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f0c0d2f60 con 0x7f9f0c15f470 2024-03-06T07:31:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f11a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f0c0cea60 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f03e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9f0c15f470 0x7f9f0c0d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f03e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9f0c15f470 0x7f9f0c0d08b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32930/0 (socket says 172.21.15.86:32930) 2024-03-06T07:31:16.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f03e00640 1 -- 172.21.15.86:0/471237930 learned_addr learned my addr 172.21.15.86:0/471237930 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:16.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f03e00640 1 -- 172.21.15.86:0/471237930 >> v2:172.21.15.194:3300/0 conn(0x7f9f0c0d0df0 msgr2=0x7f9f0c160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:16.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f03e00640 1 --2- 172.21.15.86:0/471237930 >> v2:172.21.15.194:3300/0 conn(0x7f9f0c0d0df0 0x7f9f0c160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f03e00640 1 -- 172.21.15.86:0/471237930 >> v1:172.21.15.194:6789/0 conn(0x7f9f0c15e0d0 legacy=0x7f9f0c0d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:16.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.811+0000 7f9f03e00640 1 -- 172.21.15.86:0/471237930 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f0c0d2350 con 0x7f9f0c15f470 2024-03-06T07:31:16.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.812+0000 7f9f03e00640 1 --2- 172.21.15.86:0/471237930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c15f470 0x7f9f0c0d08b0 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7f9f0c0caf20 tx=0x7f9ef402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cfd74ceac09ba0f7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:16.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.813+0000 7f9f0a000640 1 -- 172.21.15.86:0/471237930 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ef4034070 con 0x7f9f0c15f470 2024-03-06T07:31:16.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.813+0000 7f9f0a000640 1 -- 172.21.15.86:0/471237930 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9ef4030860 con 0x7f9f0c15f470 2024-03-06T07:31:16.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.813+0000 7f9f0a000640 1 -- 172.21.15.86:0/471237930 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ef4030c30 con 0x7f9f0c15f470 2024-03-06T07:31:16.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.813+0000 7f9f11a00640 1 -- 172.21.15.86:0/471237930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c15f470 msgr2=0x7f9f0c0d08b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.813+0000 7f9f11a00640 1 --2- 172.21.15.86:0/471237930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c15f470 0x7f9f0c0d08b0 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7f9f0c0caf20 tx=0x7f9ef402b9a0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.814+0000 7f9f11a00640 1 -- 172.21.15.86:0/471237930 shutdown_connections 2024-03-06T07:31:16.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.814+0000 7f9f11a00640 1 --2- 172.21.15.86:0/471237930 >> v2:172.21.15.194:3300/0 conn(0x7f9f0c0d0df0 0x7f9f0c160dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.814+0000 7f9f11a00640 1 --2- 172.21.15.86:0/471237930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c15f470 0x7f9f0c0d08b0 unknown :-1 s=CLOSED pgs=873 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.814+0000 7f9f11a00640 1 -- 172.21.15.86:0/471237930 >> 172.21.15.86:0/471237930 conn(0x7f9f0c0d3ec0 msgr2=0x7f9f0c0d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:16.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.814+0000 7f9f11a00640 1 -- 172.21.15.86:0/471237930 shutdown_connections 2024-03-06T07:31:16.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.814+0000 7f9f11a00640 1 -- 172.21.15.86:0/471237930 wait complete. 2024-03-06T07:31:16.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.815+0000 7f9f11a00640 1 Processor -- start 2024-03-06T07:31:16.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.815+0000 7f9f11a00640 1 -- start start 2024-03-06T07:31:16.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f11a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c0d0df0 0x7f9f0c0c8160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f11a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f0c15e0d0 0x7f9f0c0c9db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f11a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f0c15f470 0x7f9f0c1550e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f11a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f0c0d2bf0 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f11a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f0c0d2910 con 0x7f9f0c15e0d0 2024-03-06T07:31:16.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f11a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f0c15fb30 con 0x7f9f0c15f470 2024-03-06T07:31:16.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f0aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c0d0df0 0x7f9f0c0c8160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f0aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c0d0df0 0x7f9f0c0c8160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32942/0 (socket says 172.21.15.86:32942) 2024-03-06T07:31:16.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.816+0000 7f9f0aa00640 1 -- 172.21.15.86:0/3042898160 learned_addr learned my addr 172.21.15.86:0/3042898160 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:16.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f03e00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f0c15e0d0 0x7f9f0c0c9db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f0aa00640 1 -- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f0c15f470 msgr2=0x7f9f0c1550e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f0b400640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f0c15f470 0x7f9f0c1550e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f0aa00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f0c15f470 0x7f9f0c1550e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f0aa00640 1 -- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f0c15e0d0 msgr2=0x7f9f0c0c9db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:16.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f0aa00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f0c15e0d0 0x7f9f0c0c9db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:16.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f0aa00640 1 -- 172.21.15.86:0/3042898160 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f0c15c880 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f03e00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f0c15e0d0 0x7f9f0c0c9db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:16.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f0aa00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c0d0df0 0x7f9f0c0c8160 secure :-1 s=READY pgs=874 cs=0 l=1 rev1=1 crypto rx=0x7f9efc00c410 tx=0x7f9efc00c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:16.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f02a00640 1 -- 172.21.15.86:0/3042898160 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9efc017070 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.817+0000 7f9f02a00640 1 -- 172.21.15.86:0/3042898160 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9efc0035f0 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.818+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f0c1f48c0 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.818+0000 7f9f02a00640 1 -- 172.21.15.86:0/3042898160 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9efc003a20 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.818+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f0c1f4e80 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.819+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f0c04f330 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.821+0000 7f9f02a00640 1 -- 172.21.15.86:0/3042898160 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9efc01a070 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.821+0000 7f9f02a00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ee806b4d0 0x7f9ee806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:16.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.821+0000 7f9f02a00640 1 -- 172.21.15.86:0/3042898160 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9efc0896b0 con 0x7f9f0c0d0df0 2024-03-06T07:31:16.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.822+0000 7f9f03e00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ee806b4d0 0x7f9ee806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:16.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.822+0000 7f9f03e00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ee806b4d0 0x7f9ee806d990 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f9ef4036000 tx=0x7f9ef403b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:16.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:16.829+0000 7f9f02a00640 1 -- 172.21.15.86:0/3042898160 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9efc058560 con 0x7f9f0c0d0df0 2024-03-06T07:31:17.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.095+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9f0c08d920 con 0x7f9f0c0d0df0 2024-03-06T07:31:17.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.096+0000 7f9f02a00640 1 -- 172.21.15.86:0/3042898160 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f9efc0010c0 con 0x7f9f0c0d0df0 2024-03-06T07:31:17.098 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:17.098 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:31:17.099 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ee806b4d0 msgr2=0x7f9ee806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ee806b4d0 0x7f9ee806d990 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f9ef4036000 tx=0x7f9ef403b040 comp rx=0 tx=0).stop 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c0d0df0 msgr2=0x7f9f0c0c8160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c0d0df0 0x7f9f0c0c8160 secure :-1 s=READY pgs=874 cs=0 l=1 rev1=1 crypto rx=0x7f9efc00c410 tx=0x7f9efc00c8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 shutdown_connections 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ee806b4d0 0x7f9ee806d990 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f0c15f470 0x7f9f0c1550e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f0c15e0d0 0x7f9f0c0c9db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 --2- 172.21.15.86:0/3042898160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f0c0d0df0 0x7f9f0c0c8160 unknown :-1 s=CLOSED pgs=874 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 >> 172.21.15.86:0/3042898160 conn(0x7f9f0c0d3ec0 msgr2=0x7f9f0c0b1f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 shutdown_connections 2024-03-06T07:31:17.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.098+0000 7f9f11a00640 1 -- 172.21.15.86:0/3042898160 wait complete. 2024-03-06T07:31:17.118 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:17.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.306+0000 7fc5b7800640 1 Processor -- start 2024-03-06T07:31:17.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.306+0000 7fc5b7800640 1 -- start start 2024-03-06T07:31:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b7800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc5b015e790 0x7fc5b00b33b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b7800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc5b006eff0 0x7fc5b006e1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b7800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5b00b3ef0 con 0x7fc5b0157fe0 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b7800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5b006eb60 con 0x7fc5b006eff0 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b7800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5b00b41d0 con 0x7fc5b015e790 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b5800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc5b006eff0 0x7fc5b006e1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b5800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc5b006eff0 0x7fc5b006e1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32946/0 (socket says 172.21.15.86:32946) 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.308+0000 7fc5b5800640 1 -- 172.21.15.86:0/3510619507 learned_addr learned my addr 172.21.15.86:0/3510619507 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.309+0000 7fc5b5800640 1 -- 172.21.15.86:0/3510619507 >> v2:172.21.15.194:3300/0 conn(0x7fc5b015e790 msgr2=0x7fc5b00b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.309+0000 7fc5afe00640 1 --2- 172.21.15.86:0/3510619507 >> v2:172.21.15.194:3300/0 conn(0x7fc5b015e790 0x7fc5b00b33b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.309+0000 7fc5b5800640 1 --2- 172.21.15.86:0/3510619507 >> v2:172.21.15.194:3300/0 conn(0x7fc5b015e790 0x7fc5b00b33b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.309+0000 7fc5b5800640 1 -- 172.21.15.86:0/3510619507 >> v1:172.21.15.194:6789/0 conn(0x7fc5b0157fe0 legacy=0x7fc5b015e080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:17.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.309+0000 7fc5b5800640 1 -- 172.21.15.86:0/3510619507 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b00b44b0 con 0x7fc5b006eff0 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.310+0000 7fc5b5800640 1 --2- 172.21.15.86:0/3510619507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b006eff0 0x7fc5b006e1f0 secure :-1 s=READY pgs=875 cs=0 l=1 rev1=1 crypto rx=0x7fc5b00d11b0 tx=0x7fc5a802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=74fa4638e24937b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.310+0000 7fc5af400640 1 -- 172.21.15.86:0/3510619507 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a802ca80 con 0x7fc5b006eff0 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.310+0000 7fc5af400640 1 -- 172.21.15.86:0/3510619507 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5a802cc20 con 0x7fc5b006eff0 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.310+0000 7fc5af400640 1 -- 172.21.15.86:0/3510619507 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a80343f0 con 0x7fc5b006eff0 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 -- 172.21.15.86:0/3510619507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b006eff0 msgr2=0x7fc5b006e1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3510619507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b006eff0 0x7fc5b006e1f0 secure :-1 s=READY pgs=875 cs=0 l=1 rev1=1 crypto rx=0x7fc5b00d11b0 tx=0x7fc5a802bee0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 -- 172.21.15.86:0/3510619507 shutdown_connections 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3510619507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b006eff0 0x7fc5b006e1f0 unknown :-1 s=CLOSED pgs=875 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3510619507 >> v2:172.21.15.194:3300/0 conn(0x7fc5b015e790 0x7fc5b00b33b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 -- 172.21.15.86:0/3510619507 >> 172.21.15.86:0/3510619507 conn(0x7fc5b00d0340 msgr2=0x7fc5b00d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:17.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 -- 172.21.15.86:0/3510619507 shutdown_connections 2024-03-06T07:31:17.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.311+0000 7fc5b7800640 1 -- 172.21.15.86:0/3510619507 wait complete. 2024-03-06T07:31:17.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.313+0000 7fc5b7800640 1 Processor -- start 2024-03-06T07:31:17.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.313+0000 7fc5b7800640 1 -- start start 2024-03-06T07:31:17.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b7800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b006eff0 0x7fc5b00cf270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b7800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b0157fe0 0x7fc5b014ce80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b7800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b015e790 0x7fc5b014b1a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b7800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b0066d00 con 0x7fc5b015e790 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b7800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b01583c0 con 0x7fc5b006eff0 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b7800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b0066e80 con 0x7fc5b0157fe0 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b5800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b015e790 0x7fc5b014b1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b5800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b015e790 0x7fc5b014b1a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32956/0 (socket says 172.21.15.86:32956) 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.314+0000 7fc5b5800640 1 -- 172.21.15.86:0/3058205146 learned_addr learned my addr 172.21.15.86:0/3058205146 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5b4e00640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b006eff0 0x7fc5b00cf270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5afe00640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b0157fe0 0x7fc5b014ce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5b5800640 1 -- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b0157fe0 msgr2=0x7fc5b014ce80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5b5800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b0157fe0 0x7fc5b014ce80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5b5800640 1 -- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b006eff0 msgr2=0x7fc5b00cf270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5b5800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b006eff0 0x7fc5b00cf270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5b5800640 1 -- 172.21.15.86:0/3058205146 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b0157580 con 0x7fc5b015e790 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5b5800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b015e790 0x7fc5b014b1a0 secure :-1 s=READY pgs=876 cs=0 l=1 rev1=1 crypto rx=0x7fc5a802c4f0 tx=0x7fc5a8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.315+0000 7fc5acc00640 1 -- 172.21.15.86:0/3058205146 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a8043020 con 0x7fc5b015e790 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.316+0000 7fc5acc00640 1 -- 172.21.15.86:0/3058205146 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5a803e450 con 0x7fc5b015e790 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.316+0000 7fc5acc00640 1 -- 172.21.15.86:0/3058205146 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a8009510 con 0x7fc5b015e790 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.316+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5b0154c20 con 0x7fc5b015e790 2024-03-06T07:31:17.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.316+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5b01551e0 con 0x7fc5b015e790 2024-03-06T07:31:17.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.317+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5b0074ae0 con 0x7fc5b015e790 2024-03-06T07:31:17.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.318+0000 7fc5acc00640 1 -- 172.21.15.86:0/3058205146 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc5a8046020 con 0x7fc5b015e790 2024-03-06T07:31:17.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.319+0000 7fc5acc00640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59006b4d0 0x7fc59006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.319+0000 7fc5acc00640 1 -- 172.21.15.86:0/3058205146 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc5a80acb90 con 0x7fc5b015e790 2024-03-06T07:31:17.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.319+0000 7fc5b4e00640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59006b4d0 0x7fc59006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.320+0000 7fc5b4e00640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59006b4d0 0x7fc59006d990 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fc5b0155b70 tx=0x7fc5a4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:17.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.323+0000 7fc5acc00640 1 -- 172.21.15.86:0/3058205146 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc5a804d030 con 0x7fc5b015e790 2024-03-06T07:31:17.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.546+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc5b004f330 con 0x7fc5b015e790 2024-03-06T07:31:17.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.547+0000 7fc5acc00640 1 -- 172.21.15.86:0/3058205146 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 42 v42) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fc5a807b9c0 con 0x7fc5b015e790 2024-03-06T07:31:17.550 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:17.550 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":42,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37}],"filesystems":[{"mdsmap":{"epoch":42,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:30:35.740469+0000","modified":"2024-03-06T07:30:38.790632+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6204},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6204":{"gid":6204,"name":"c","rank":0,"incarnation":38,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/195049401","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":195049401},{"type":"v1","addr":"172.21.15.86:6837","nonce":195049401}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6204,"qdb_cluster":[6204]},"id":5}]} 2024-03-06T07:31:17.550 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 42 2024-03-06T07:31:17.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.549+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59006b4d0 msgr2=0x7fc59006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.549+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59006b4d0 0x7fc59006d990 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fc5b0155b70 tx=0x7fc5a4009040 comp rx=0 tx=0).stop 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.549+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b015e790 msgr2=0x7fc5b014b1a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.549+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b015e790 0x7fc5b014b1a0 secure :-1 s=READY pgs=876 cs=0 l=1 rev1=1 crypto rx=0x7fc5a802c4f0 tx=0x7fc5a8004a40 comp rx=0 tx=0).stop 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 shutdown_connections 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc59006b4d0 0x7fc59006d990 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5b015e790 0x7fc5b014b1a0 unknown :-1 s=CLOSED pgs=876 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5b0157fe0 0x7fc5b014ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 --2- 172.21.15.86:0/3058205146 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5b006eff0 0x7fc5b00cf270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 >> 172.21.15.86:0/3058205146 conn(0x7fc5b00d0340 msgr2=0x7fc5b00afe70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 shutdown_connections 2024-03-06T07:31:17.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.550+0000 7fc5b7800640 1 -- 172.21.15.86:0/3058205146 wait complete. 2024-03-06T07:31:17.570 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:17.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.770+0000 7fca06400640 1 Processor -- start 2024-03-06T07:31:17.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.770+0000 7fca06400640 1 -- start start 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.771+0000 7fca06400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fca000b4680 0x7fca0006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.771+0000 7fca06400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 0x7fca0006ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fca06400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fca0015b090 con 0x7fca0015acb0 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fca06400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fca000b2a30 con 0x7fca000b4680 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fca06400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca000b4360 con 0x7fca00072340 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fc9ff400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fca0015acb0 0x7fca000b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34176/0 (socket says 172.21.15.86:34176) 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fc9ff400640 1 -- 172.21.15.86:0/991131469 learned_addr learned my addr 172.21.15.86:0/991131469 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fc9fea00640 1 --2- 172.21.15.86:0/991131469 >> v2:172.21.15.194:3300/0 conn(0x7fca000b4680 0x7fca0006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fc9ffe00640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 0x7fca0006ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3035514341 0 0) 0x7fca0015b090 con 0x7fca0015acb0 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.772+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc9e4003520 con 0x7fca0015acb0 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3414808088 0 0) 0x7fc9e4003520 con 0x7fca0015acb0 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 msgr2=0x7fca0006ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 0x7fca0006ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 >> v2:172.21.15.194:3300/0 conn(0x7fca000b4680 msgr2=0x7fca0006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 --2- 172.21.15.86:0/991131469 >> v2:172.21.15.194:3300/0 conn(0x7fca000b4680 0x7fca0006e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca000b2cf0 con 0x7fca0015acb0 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc9f0001670 con 0x7fca0015acb0 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 >> v1:172.21.15.86:6789/0 conn(0x7fca0015acb0 legacy=0x7fca000b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e40039b0 0x7fc9e4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9e4004800 0x7fc9e4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 0x7fc9e40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca000b2a30 con 0x7fc9e4004800 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca000b4360 con 0x7fc9e40039b0 2024-03-06T07:31:17.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9e400b9c0 con 0x7fca00072340 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9fea00640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9e4004800 0x7fc9e4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9ffe00640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 0x7fc9e40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.773+0000 7fc9ff400640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e40039b0 0x7fc9e4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.774+0000 7fc9fea00640 1 -- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 msgr2=0x7fc9e40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.774+0000 7fc9fea00640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 0x7fc9e40051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.774+0000 7fc9fea00640 1 -- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e40039b0 msgr2=0x7fc9e4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.774+0000 7fc9fea00640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e40039b0 0x7fc9e4003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.774+0000 7fc9fea00640 1 -- 172.21.15.86:0/991131469 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9e4003520 con 0x7fc9e4004800 2024-03-06T07:31:17.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.774+0000 7fc9ffe00640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00072340 0x7fc9e40051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fc9fea00640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9e4004800 0x7fc9e4004c80 secure :-1 s=READY pgs=878 cs=0 l=1 rev1=1 crypto rx=0x7fc9ec009650 tx=0x7fc9e4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=66727101acea2043 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9ec03b070 con 0x7fc9e4004800 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc9ec02e8e0 con 0x7fc9e4004800 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fc9fe000640 1 -- 172.21.15.86:0/991131469 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9ec009690 con 0x7fc9e4004800 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fca06400640 1 -- 172.21.15.86:0/991131469 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9e4004800 msgr2=0x7fc9e4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fca06400640 1 --2- 172.21.15.86:0/991131469 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9e4004800 0x7fc9e4004c80 secure :-1 s=READY pgs=878 cs=0 l=1 rev1=1 crypto rx=0x7fc9ec009650 tx=0x7fc9e4005830 comp rx=0 tx=0).stop 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fc9ffe00640 1 -- 172.21.15.86:0/991131469 reap_dead start 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.775+0000 7fca06400640 1 -- 172.21.15.86:0/991131469 shutdown_connections 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.776+0000 7fca06400640 1 -- 172.21.15.86:0/991131469 >> 172.21.15.86:0/991131469 conn(0x7fca000b1730 msgr2=0x7fca000b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.776+0000 7fca06400640 1 -- 172.21.15.86:0/991131469 shutdown_connections 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.776+0000 7fca06400640 1 -- 172.21.15.86:0/991131469 wait complete. 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.777+0000 7fca06400640 1 Processor -- start 2024-03-06T07:31:17.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.777+0000 7fca06400640 1 -- start start 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.778+0000 7fca06400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e4004800 0x7fca000ce030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.778+0000 7fca06400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca000adac0 0x7fca000adf40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.778+0000 7fca06400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00142ec0 0x7fca00143370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.778+0000 7fca06400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca000b4010 con 0x7fca000adac0 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.778+0000 7fca06400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca000b32b0 con 0x7fc9e4004800 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.778+0000 7fca06400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca0006ea30 con 0x7fca00142ec0 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca000adac0 0x7fca000adf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca000adac0 0x7fca000adf40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:32982/0 (socket says 172.21.15.86:32982) 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 -- 172.21.15.86:0/3039394526 learned_addr learned my addr 172.21.15.86:0/3039394526 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9ff400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e4004800 0x7fca000ce030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9ffe00640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00142ec0 0x7fca00143370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 -- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00142ec0 msgr2=0x7fca00143370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00142ec0 0x7fca00143370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 -- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e4004800 msgr2=0x7fca000ce030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e4004800 0x7fca000ce030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 -- 172.21.15.86:0/3039394526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca000ce780 con 0x7fca000adac0 2024-03-06T07:31:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.779+0000 7fc9fea00640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca000adac0 0x7fca000adf40 secure :-1 s=READY pgs=879 cs=0 l=1 rev1=1 crypto rx=0x7fc9ec0091d0 tx=0x7fc9ec004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.780+0000 7fc9f7400640 1 -- 172.21.15.86:0/3039394526 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9ec03b040 con 0x7fca000adac0 2024-03-06T07:31:17.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.780+0000 7fc9f7400640 1 -- 172.21.15.86:0/3039394526 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc9ec044070 con 0x7fca000adac0 2024-03-06T07:31:17.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.780+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca000cea60 con 0x7fca000adac0 2024-03-06T07:31:17.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.780+0000 7fc9f7400640 1 -- 172.21.15.86:0/3039394526 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9ec003850 con 0x7fca000adac0 2024-03-06T07:31:17.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.780+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca000cf020 con 0x7fca000adac0 2024-03-06T07:31:17.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.781+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca0004f330 con 0x7fca000adac0 2024-03-06T07:31:17.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.782+0000 7fc9f7400640 1 -- 172.21.15.86:0/3039394526 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc9ec0039f0 con 0x7fca000adac0 2024-03-06T07:31:17.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.783+0000 7fc9f7400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9e006b4d0 0x7fc9e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:17.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.783+0000 7fc9f7400640 1 -- 172.21.15.86:0/3039394526 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc9ec0abc20 con 0x7fca000adac0 2024-03-06T07:31:17.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.784+0000 7fc9ff400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9e006b4d0 0x7fc9e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:17.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.784+0000 7fc9ff400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9e006b4d0 0x7fc9e006d990 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fc9f0000c00 tx=0x7fc9f0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:17.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.788+0000 7fc9f7400640 1 -- 172.21.15.86:0/3039394526 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc9ec07a9c0 con 0x7fca000adac0 2024-03-06T07:31:17.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.986+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fca0006e190 con 0x7fca000adac0 2024-03-06T07:31:17.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.988+0000 7fc9f7400640 1 -- 172.21.15.86:0/3039394526 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v72) v1 ==== 74+0+16778 (secure 0 0 0) 0x7fc9ec07a2c0 con 0x7fca000adac0 2024-03-06T07:31:17.991 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:17.991 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":72,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:30:54.997378+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:30:33.566852+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-06T07:30:34.722690+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"11.14","mappings":[{"from":6,"to":0}]},{"pgid":"11.28","mappings":[{"from":6,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9e006b4d0 msgr2=0x7fc9e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9e006b4d0 0x7fc9e006d990 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fc9f0000c00 tx=0x7fc9f0009040 comp rx=0 tx=0).stop 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca000adac0 msgr2=0x7fca000adf40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca000adac0 0x7fca000adf40 secure :-1 s=READY pgs=879 cs=0 l=1 rev1=1 crypto rx=0x7fc9ec0091d0 tx=0x7fc9ec004730 comp rx=0 tx=0).stop 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 shutdown_connections 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9e006b4d0 0x7fc9e006d990 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fca00142ec0 0x7fca00143370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca000adac0 0x7fca000adf40 unknown :-1 s=CLOSED pgs=879 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 --2- 172.21.15.86:0/3039394526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9e4004800 0x7fca000ce030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 >> 172.21.15.86:0/3039394526 conn(0x7fca000b1730 msgr2=0x7fca00156110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 shutdown_connections 2024-03-06T07:31:17.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:17.990+0000 7fca06400640 1 -- 172.21.15.86:0/3039394526 wait complete. 2024-03-06T07:31:18.010 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:31:18.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.210+0000 7f1ae8000640 1 Processor -- start 2024-03-06T07:31:18.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.210+0000 7f1ae8000640 1 -- start start 2024-03-06T07:31:18.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.213+0000 7f1ae8000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1ae015bfc0 0x7f1ae00b1670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.213+0000 7f1ae8000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ae015c3a0 con 0x7f1ae00b4390 2024-03-06T07:31:18.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.213+0000 7f1ae8000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ae00cd660 con 0x7f1ae00b1bb0 2024-03-06T07:31:18.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.213+0000 7f1ae8000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1ae00cdc20 con 0x7f1ae015bfc0 2024-03-06T07:31:18.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.213+0000 7f1ae6000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1ae00b4390 0x7f1ae015fab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34190/0 (socket says 172.21.15.86:34190) 2024-03-06T07:31:18.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.213+0000 7f1ae6000640 1 -- 172.21.15.86:0/1452297206 learned_addr learned my addr 172.21.15.86:0/1452297206 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:18.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1452297206 >> v2:172.21.15.194:3300/0 conn(0x7f1ae015bfc0 0x7f1ae00b1670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3773949710 0 0) 0x7f1ae015c3a0 con 0x7f1ae00b4390 2024-03-06T07:31:18.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1ac8003520 con 0x7f1ae00b4390 2024-03-06T07:31:18.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4075287697 0 0) 0x7f1ac8003520 con 0x7f1ae00b4390 2024-03-06T07:31:18.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 >> v2:172.21.15.194:3300/0 conn(0x7f1ae015bfc0 msgr2=0x7f1ae00b1670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:18.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 --2- 172.21.15.86:0/1452297206 >> v2:172.21.15.194:3300/0 conn(0x7f1ae015bfc0 0x7f1ae00b1670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 >> v1:172.21.15.194:6789/0 conn(0x7f1ae00b1bb0 legacy=0x7f1ae00d0240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ae00cd940 con 0x7f1ae00b4390 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1ad8001670 con 0x7f1ae00b4390 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 >> v1:172.21.15.86:6789/0 conn(0x7f1ae00b4390 legacy=0x7f1ae015fab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ac8003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ac8004800 0x7f1ac8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ac80051c0 0x7f1ac800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ac800c090 con 0x7f1ac8003a00 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ac800c240 con 0x7f1ac80051c0 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.214+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ac800c440 con 0x7f1ac8004800 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ac8003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae4c00640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ac8004800 0x7f1ac8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae6000640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ac80051c0 0x7f1ac800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae5600640 1 -- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ac8004800 msgr2=0x7f1ac8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:18.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ac8004800 0x7f1ac8004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae5600640 1 -- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ac80051c0 msgr2=0x7f1ac800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ac80051c0 0x7f1ac800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae5600640 1 -- 172.21.15.86:0/1452297206 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ac8003520 con 0x7f1ac8003a00 2024-03-06T07:31:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae6000640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ac80051c0 0x7f1ac800b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:18.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.215+0000 7f1ae6000640 1 -- 172.21.15.86:0/1452297206 reap_dead start 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.216+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ac8003e20 secure :-1 s=READY pgs=881 cs=0 l=1 rev1=1 crypto rx=0x7f1ad4009fc0 tx=0x7f1ac8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b0341359950effa server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4039070 con 0x7f1ac8003a00 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ad402cc20 con 0x7f1ac8003a00 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1adfe00640 1 -- 172.21.15.86:0/1452297206 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4009c10 con 0x7f1ac8003a00 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1ae8000640 1 -- 172.21.15.86:0/1452297206 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 msgr2=0x7f1ac8003e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ac8003e20 secure :-1 s=READY pgs=881 cs=0 l=1 rev1=1 crypto rx=0x7f1ad4009fc0 tx=0x7f1ac8005830 comp rx=0 tx=0).stop 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1ae8000640 1 -- 172.21.15.86:0/1452297206 shutdown_connections 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1452297206 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ac8003e20 unknown :-1 s=CLOSED pgs=881 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1ae8000640 1 -- 172.21.15.86:0/1452297206 >> 172.21.15.86:0/1452297206 conn(0x7f1ae00b3800 msgr2=0x7f1ae00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:18.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1ae8000640 1 -- 172.21.15.86:0/1452297206 shutdown_connections 2024-03-06T07:31:18.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.217+0000 7f1ae8000640 1 -- 172.21.15.86:0/1452297206 wait complete. 2024-03-06T07:31:18.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.219+0000 7f1ae8000640 1 Processor -- start 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.219+0000 7f1ae8000640 1 -- start start 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae8000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ae01512f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae8000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae00b6050 0x7f1ae00b64d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae8000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae00b0710 0x7f1ae00b0b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae8000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae00cd2f0 con 0x7f1ac8003a00 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae8000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae00cfd20 con 0x7f1ae00b0710 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae8000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae00b1670 con 0x7f1ae00b6050 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ae01512f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ae01512f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33002/0 (socket says 172.21.15.86:33002) 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 -- 172.21.15.86:0/1517247292 learned_addr learned my addr 172.21.15.86:0/1517247292 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 -- 172.21.15.86:0/1517247292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae00b6050 msgr2=0x7f1ae00b64d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae00b6050 0x7f1ae00b64d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 -- 172.21.15.86:0/1517247292 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae00b0710 msgr2=0x7f1ae00b0b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae00b0710 0x7f1ae00b0b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.220+0000 7f1ae5600640 1 -- 172.21.15.86:0/1517247292 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ae00b6c10 con 0x7f1ac8003a00 2024-03-06T07:31:18.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.221+0000 7f1ae5600640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ae01512f0 secure :-1 s=READY pgs=882 cs=0 l=1 rev1=1 crypto rx=0x7f1ad400e200 tx=0x7f1ad4009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:18.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.221+0000 7f1add600640 1 -- 172.21.15.86:0/1517247292 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4039050 con 0x7f1ac8003a00 2024-03-06T07:31:18.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.221+0000 7f1add600640 1 -- 172.21.15.86:0/1517247292 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ad403a040 con 0x7f1ac8003a00 2024-03-06T07:31:18.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.221+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ae00b6ef0 con 0x7f1ac8003a00 2024-03-06T07:31:18.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.221+0000 7f1add600640 1 -- 172.21.15.86:0/1517247292 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4009c10 con 0x7f1ac8003a00 2024-03-06T07:31:18.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.221+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ae00b74b0 con 0x7f1ac8003a00 2024-03-06T07:31:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.222+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1ae00b10d0 con 0x7f1ac8003a00 2024-03-06T07:31:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.223+0000 7f1add600640 1 -- 172.21.15.86:0/1517247292 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1ad4003bb0 con 0x7f1ac8003a00 2024-03-06T07:31:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.224+0000 7f1add600640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.225+0000 7f1add600640 1 -- 172.21.15.86:0/1517247292 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1ad40aca90 con 0x7f1ac8003a00 2024-03-06T07:31:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.225+0000 7f1ae4c00640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.225+0000 7f1ae4c00640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00ce1a0 tx=0x7f1acc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:18.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.229+0000 7f1add600640 1 -- 172.21.15.86:0/1517247292 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1ad407b9e0 con 0x7f1ac8003a00 2024-03-06T07:31:18.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.496+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f1ae004f330 con 0x7f1ac8003a00 2024-03-06T07:31:18.500 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:31:18.497+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:31:18.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.500+0000 7f1add600640 1 -- 172.21.15.86:0/1517247292 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v43) v1 ==== 161+0+0 (secure 0 0 0) 0x7f1ad407b2e0 con 0x7f1ac8003a00 2024-03-06T07:31:18.503 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:31:18.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 msgr2=0x7f1ac006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00ce1a0 tx=0x7f1acc009040 comp rx=0 tx=0).stop 2024-03-06T07:31:18.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 msgr2=0x7f1ae01512f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ae01512f0 secure :-1 s=READY pgs=882 cs=0 l=1 rev1=1 crypto rx=0x7f1ad400e200 tx=0x7f1ad4009750 comp rx=0 tx=0).stop 2024-03-06T07:31:18.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 shutdown_connections 2024-03-06T07:31:18.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ac006b4d0 0x7f1ac006d990 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ae00b0710 0x7f1ae00b0b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ae00b6050 0x7f1ae00b64d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 --2- 172.21.15.86:0/1517247292 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ac8003a00 0x7f1ae01512f0 unknown :-1 s=CLOSED pgs=882 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.503+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 >> 172.21.15.86:0/1517247292 conn(0x7f1ae00b3800 msgr2=0x7f1ae015b0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:18.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.504+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 shutdown_connections 2024-03-06T07:31:18.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.504+0000 7f1ae8000640 1 -- 172.21.15.86:0/1517247292 wait complete. 2024-03-06T07:31:18.524 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:31:18.598 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:31:18.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.704+0000 7f713be00640 1 Processor -- start 2024-03-06T07:31:18.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.704+0000 7f713be00640 1 -- start start 2024-03-06T07:31:18.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.705+0000 7f713be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fc10 0x7f7134070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.705+0000 7f713be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f713415fff0 0x7f7134160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.706+0000 7f713be00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71340b46c0 con 0x7f71340ce4a0 2024-03-06T07:31:18.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.706+0000 7f713be00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f71340b49a0 con 0x7f713415fff0 2024-03-06T07:31:18.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.706+0000 7f713be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7134072c90 con 0x7f713415fc10 2024-03-06T07:31:18.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.706+0000 7f7139400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fc10 0x7f7134070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.706+0000 7f7139400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fc10 0x7f7134070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:48806/0 (socket says 172.21.15.86:48806) 2024-03-06T07:31:18.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.706+0000 7f7139e00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f71340ce4a0 0x7f71340ce930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:34012/0 (socket says 172.21.15.86:34012) 2024-03-06T07:31:18.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.707+0000 7f7139400640 1 -- 172.21.15.86:0/3162421961 learned_addr learned my addr 172.21.15.86:0/3162421961 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:18.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.706+0000 7f7138a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f713415fff0 0x7f7134160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.707+0000 7f7139400640 1 -- 172.21.15.86:0/3162421961 >> v2:172.21.15.194:3300/0 conn(0x7f713415fff0 msgr2=0x7f7134160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:18.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.707+0000 7f7139400640 1 --2- 172.21.15.86:0/3162421961 >> v2:172.21.15.194:3300/0 conn(0x7f713415fff0 0x7f7134160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.707+0000 7f7139400640 1 -- 172.21.15.86:0/3162421961 >> v1:172.21.15.194:6789/0 conn(0x7f71340ce4a0 legacy=0x7f71340ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.707+0000 7f7139400640 1 -- 172.21.15.86:0/3162421961 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7134160990 con 0x7f713415fc10 2024-03-06T07:31:18.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.707+0000 7f7133e00640 1 -- 172.21.15.86:0/3162421961 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 397794169 0 0) 0x7f71340b46c0 con 0x7f71340ce4a0 2024-03-06T07:31:18.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f7139400640 1 --2- 172.21.15.86:0/3162421961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fc10 0x7f7134070ff0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f7128009870 tx=0x7f712802b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b038bd1df94a721 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:18.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f7133e00640 1 -- 172.21.15.86:0/3162421961 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7128034070 con 0x7f713415fc10 2024-03-06T07:31:18.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f7133e00640 1 -- 172.21.15.86:0/3162421961 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7128030840 con 0x7f713415fc10 2024-03-06T07:31:18.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f7133e00640 1 -- 172.21.15.86:0/3162421961 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7128030c10 con 0x7f713415fc10 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 -- 172.21.15.86:0/3162421961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fc10 msgr2=0x7f7134070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 --2- 172.21.15.86:0/3162421961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fc10 0x7f7134070ff0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f7128009870 tx=0x7f712802b980 comp rx=0 tx=0).stop 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 -- 172.21.15.86:0/3162421961 shutdown_connections 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 --2- 172.21.15.86:0/3162421961 >> v2:172.21.15.194:3300/0 conn(0x7f713415fff0 0x7f7134160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 --2- 172.21.15.86:0/3162421961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fc10 0x7f7134070ff0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 -- 172.21.15.86:0/3162421961 >> 172.21.15.86:0/3162421961 conn(0x7f71340b1940 msgr2=0x7f71340b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 -- 172.21.15.86:0/3162421961 shutdown_connections 2024-03-06T07:31:18.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.710+0000 7f713be00640 1 -- 172.21.15.86:0/3162421961 wait complete. 2024-03-06T07:31:18.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 Processor -- start 2024-03-06T07:31:18.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 -- start start 2024-03-06T07:31:18.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71340ce4a0 0x7f713414e650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f713415fc10 0x7f713415e730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fff0 0x7f713409ce30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7134072a30 con 0x7f71340ce4a0 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71340728b0 con 0x7f713415fc10 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f713be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f713415c2e0 con 0x7f713415fff0 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f7139400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71340ce4a0 0x7f713414e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f7139400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71340ce4a0 0x7f713414e650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33040/0 (socket says 172.21.15.86:33040) 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.713+0000 7f7139400640 1 -- 172.21.15.86:0/3355391240 learned_addr learned my addr 172.21.15.86:0/3355391240 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7139400640 1 -- 172.21.15.86:0/3355391240 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fff0 msgr2=0x7f713409ce30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7139400640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fff0 0x7f713409ce30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7139400640 1 -- 172.21.15.86:0/3355391240 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f713415fc10 msgr2=0x7f713415e730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7139400640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f713415fc10 0x7f713415e730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7139400640 1 -- 172.21.15.86:0/3355391240 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f713415c5c0 con 0x7f71340ce4a0 2024-03-06T07:31:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7139400640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71340ce4a0 0x7f713414e650 secure :-1 s=READY pgs=886 cs=0 l=1 rev1=1 crypto rx=0x7f71280097c0 tx=0x7f7128004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:18.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7131600640 1 -- 172.21.15.86:0/3355391240 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7128048070 con 0x7f71340ce4a0 2024-03-06T07:31:18.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7131600640 1 -- 172.21.15.86:0/3355391240 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7128030640 con 0x7f71340ce4a0 2024-03-06T07:31:18.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f713415c8a0 con 0x7f71340ce4a0 2024-03-06T07:31:18.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f7131600640 1 -- 172.21.15.86:0/3355391240 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7128034040 con 0x7f71340ce4a0 2024-03-06T07:31:18.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.714+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f713415ce60 con 0x7f71340ce4a0 2024-03-06T07:31:18.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.715+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f713404f330 con 0x7f71340ce4a0 2024-03-06T07:31:18.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.716+0000 7f7131600640 1 -- 172.21.15.86:0/3355391240 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f712804b070 con 0x7f71340ce4a0 2024-03-06T07:31:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.717+0000 7f7131600640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f711406b500 0x7f711406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.717+0000 7f7131600640 1 -- 172.21.15.86:0/3355391240 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(73..73 src has 1..73) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f71280ac130 con 0x7f71340ce4a0 2024-03-06T07:31:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.718+0000 7f7138a00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f711406b500 0x7f711406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.718+0000 7f7138a00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f711406b500 0x7f711406d9c0 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f7120006a90 tx=0x7f7120006a20 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:18.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.722+0000 7f7131600640 1 -- 172.21.15.86:0/3355391240 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f712807afa0 con 0x7f71340ce4a0 2024-03-06T07:31:18.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:18.952+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f713406ef30 con 0x7f71340ce4a0 2024-03-06T07:31:19.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.505+0000 7f7131600640 1 -- 172.21.15.86:0/3355391240 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v44) v1 ==== 104+0+0 (secure 0 0 0) 0x7f712807a8a0 con 0x7f71340ce4a0 2024-03-06T07:31:19.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.508+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f711406b500 msgr2=0x7f711406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.508+0000 7f713be00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f711406b500 0x7f711406d9c0 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f7120006a90 tx=0x7f7120006a20 comp rx=0 tx=0).stop 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.508+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71340ce4a0 msgr2=0x7f713414e650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71340ce4a0 0x7f713414e650 secure :-1 s=READY pgs=886 cs=0 l=1 rev1=1 crypto rx=0x7f71280097c0 tx=0x7f7128004730 comp rx=0 tx=0).stop 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 shutdown_connections 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f711406b500 0x7f711406d9c0 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f713415fff0 0x7f713409ce30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f713415fc10 0x7f713415e730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 --2- 172.21.15.86:0/3355391240 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71340ce4a0 0x7f713414e650 unknown :-1 s=CLOSED pgs=886 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 >> 172.21.15.86:0/3355391240 conn(0x7f71340b1940 msgr2=0x7f713415b310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 shutdown_connections 2024-03-06T07:31:19.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.509+0000 7f713be00640 1 -- 172.21.15.86:0/3355391240 wait complete. 2024-03-06T07:31:19.531 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:31:19.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.733+0000 7f61a5600640 1 Processor -- start 2024-03-06T07:31:19.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.734+0000 7f61a5600640 1 -- start start 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f61a5600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f61a00b2240 0x7f61a0161eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f61a5600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f61a006d1c0 0x7f61a01623f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f61a5600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a00b2b10 0x7f61a006c0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f61a5600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f61a00d0fb0 con 0x7f61a006d1c0 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f61a5600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f61a00d1440 con 0x7f61a00b2240 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f61a5600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f61a0162cc0 con 0x7f61a00b2b10 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f6196000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f61a006d1c0 0x7f61a01623f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.735+0000 7f6196000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f61a006d1c0 0x7f61a01623f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33046/0 (socket says 172.21.15.86:33046) 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f6196000640 1 -- 172.21.15.86:0/363352341 learned_addr learned my addr 172.21.15.86:0/363352341 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f619f400640 1 --2- 172.21.15.86:0/363352341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a00b2b10 0x7f61a006c0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:19.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f6196000640 1 -- 172.21.15.86:0/363352341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a00b2b10 msgr2=0x7f61a006c0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:19.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f6196000640 1 --2- 172.21.15.86:0/363352341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a00b2b10 0x7f61a006c0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f619ea00640 1 --2- 172.21.15.86:0/363352341 >> v2:172.21.15.194:3300/0 conn(0x7f61a00b2240 0x7f61a0161eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:19.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f6196000640 1 -- 172.21.15.86:0/363352341 >> v2:172.21.15.194:3300/0 conn(0x7f61a00b2240 msgr2=0x7f61a0161eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:19.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f6196000640 1 --2- 172.21.15.86:0/363352341 >> v2:172.21.15.194:3300/0 conn(0x7f61a00b2240 0x7f61a0161eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f6196000640 1 -- 172.21.15.86:0/363352341 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61a00d0c40 con 0x7f61a006d1c0 2024-03-06T07:31:19.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f619f400640 1 --2- 172.21.15.86:0/363352341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a00b2b10 0x7f61a006c0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:19.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.736+0000 7f619ea00640 1 --2- 172.21.15.86:0/363352341 >> v2:172.21.15.194:3300/0 conn(0x7f61a00b2240 0x7f61a0161eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:19.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.737+0000 7f6196000640 1 --2- 172.21.15.86:0/363352341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a006d1c0 0x7f61a01623f0 secure :-1 s=READY pgs=887 cs=0 l=1 rev1=1 crypto rx=0x7f61a006e2b0 tx=0x7f618802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=adacdd35292ccfa8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:19.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.737+0000 7f619e000640 1 -- 172.21.15.86:0/363352341 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6188035020 con 0x7f61a006d1c0 2024-03-06T07:31:19.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.737+0000 7f619e000640 1 -- 172.21.15.86:0/363352341 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6188031870 con 0x7f61a006d1c0 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.737+0000 7f619e000640 1 -- 172.21.15.86:0/363352341 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6188031c40 con 0x7f61a006d1c0 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 -- 172.21.15.86:0/363352341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a006d1c0 msgr2=0x7f61a01623f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 --2- 172.21.15.86:0/363352341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a006d1c0 0x7f61a01623f0 secure :-1 s=READY pgs=887 cs=0 l=1 rev1=1 crypto rx=0x7f61a006e2b0 tx=0x7f618802b910 comp rx=0 tx=0).stop 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 -- 172.21.15.86:0/363352341 shutdown_connections 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 --2- 172.21.15.86:0/363352341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a00b2b10 0x7f61a006c0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 --2- 172.21.15.86:0/363352341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a006d1c0 0x7f61a01623f0 unknown :-1 s=CLOSED pgs=887 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 --2- 172.21.15.86:0/363352341 >> v2:172.21.15.194:3300/0 conn(0x7f61a00b2240 0x7f61a0161eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 -- 172.21.15.86:0/363352341 >> 172.21.15.86:0/363352341 conn(0x7f61a00b3870 msgr2=0x7f61a00b3c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 -- 172.21.15.86:0/363352341 shutdown_connections 2024-03-06T07:31:19.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.738+0000 7f61a5600640 1 -- 172.21.15.86:0/363352341 wait complete. 2024-03-06T07:31:19.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.740+0000 7f61a5600640 1 Processor -- start 2024-03-06T07:31:19.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.740+0000 7f61a5600640 1 -- start start 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.741+0000 7f61a5600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a006d1c0 0x7f61a009bc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.741+0000 7f61a5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a00b2240 0x7f61a01500b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.741+0000 7f61a5600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61a00b2b10 0x7f61a009d630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.741+0000 7f61a5600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61a00d1850 con 0x7f61a00b2240 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.741+0000 7f61a5600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61a0162930 con 0x7f61a00b2b10 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.741+0000 7f61a5600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f61a00d19d0 con 0x7f61a006d1c0 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.741+0000 7f6196000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a00b2240 0x7f61a01500b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:19.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a00b2240 0x7f61a01500b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33048/0 (socket says 172.21.15.86:33048) 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 -- 172.21.15.86:0/2438620932 learned_addr learned my addr 172.21.15.86:0/2438620932 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f619ea00640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a006d1c0 0x7f61a009bc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 -- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a006d1c0 msgr2=0x7f61a009bc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a006d1c0 0x7f61a009bc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 -- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61a00b2b10 msgr2=0x7f61a009d630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f619f400640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61a00b2b10 0x7f61a009d630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61a00b2b10 0x7f61a009d630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 -- 172.21.15.86:0/2438620932 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61a009c1a0 con 0x7f61a00b2240 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.742+0000 7f6196000640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a00b2240 0x7f61a01500b0 secure :-1 s=READY pgs=888 cs=0 l=1 rev1=1 crypto rx=0x7f618802b950 tx=0x7f61880046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.743+0000 7f6197400640 1 -- 172.21.15.86:0/2438620932 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6188035040 con 0x7f61a00b2240 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.743+0000 7f6197400640 1 -- 172.21.15.86:0/2438620932 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6188045480 con 0x7f61a00b2240 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.743+0000 7f6197400640 1 -- 172.21.15.86:0/2438620932 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6188045870 con 0x7f61a00b2240 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.743+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61a00b7830 con 0x7f61a00b2240 2024-03-06T07:31:19.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.743+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61a00b5b50 con 0x7f61a00b2240 2024-03-06T07:31:19.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.744+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f61a004f3a0 con 0x7f61a00b2240 2024-03-06T07:31:19.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.745+0000 7f6197400640 1 -- 172.21.15.86:0/2438620932 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6188038020 con 0x7f61a00b2240 2024-03-06T07:31:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.745+0000 7f6197400640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b4d0 0x7f617c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.746+0000 7f6197400640 1 -- 172.21.15.86:0/2438620932 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(73..73 src has 1..73) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f61880ac710 con 0x7f61a00b2240 2024-03-06T07:31:19.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.746+0000 7f619ea00640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b4d0 0x7f617c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:19.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.746+0000 7f619ea00640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b4d0 0x7f617c06d990 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f6190008f10 tx=0x7f6190008460 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:19.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.750+0000 7f6197400640 1 -- 172.21.15.86:0/2438620932 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f618807b4e0 con 0x7f61a00b2240 2024-03-06T07:31:19.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:19.945+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f61a00714b0 con 0x7f61a00b2240 2024-03-06T07:31:20.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.518+0000 7f6197400640 1 -- 172.21.15.86:0/2438620932 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v74) v1 ==== 181+0+0 (secure 0 0 0) 0x7f618807ade0 con 0x7f61a00b2240 2024-03-06T07:31:20.520 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:31:20.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.522+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b4d0 msgr2=0x7f617c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:20.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.522+0000 7f61a5600640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b4d0 0x7f617c06d990 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f6190008f10 tx=0x7f6190008460 comp rx=0 tx=0).stop 2024-03-06T07:31:20.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.522+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a00b2240 msgr2=0x7f61a01500b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:20.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.522+0000 7f61a5600640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a00b2240 0x7f61a01500b0 secure :-1 s=READY pgs=888 cs=0 l=1 rev1=1 crypto rx=0x7f618802b950 tx=0x7f61880046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.523+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 shutdown_connections 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.523+0000 7f61a5600640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f617c06b4d0 0x7f617c06d990 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.523+0000 7f61a5600640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f61a00b2b10 0x7f61a009d630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.523+0000 7f61a5600640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61a00b2240 0x7f61a01500b0 unknown :-1 s=CLOSED pgs=888 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.523+0000 7f61a5600640 1 --2- 172.21.15.86:0/2438620932 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61a006d1c0 0x7f61a009bc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.523+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 >> 172.21.15.86:0/2438620932 conn(0x7f61a00b3870 msgr2=0x7f61a00af940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.524+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 shutdown_connections 2024-03-06T07:31:20.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.524+0000 7f61a5600640 1 -- 172.21.15.86:0/2438620932 wait complete. 2024-03-06T07:31:20.549 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:31:20.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.740+0000 7fb6e0a00640 1 Processor -- start 2024-03-06T07:31:20.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.740+0000 7fb6e0a00640 1 -- start start 2024-03-06T07:31:20.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.741+0000 7fb6e0a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc06e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:20.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.741+0000 7fb6e0a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb6dc15f1f0 0x7fb6dc0cd090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:20.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6e0a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6dc15b1d0 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6e0a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb6dc06ed40 con 0x7fb6dc15f1f0 2024-03-06T07:31:20.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6e0a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6dc0b2bd0 con 0x7fb6dc15adf0 2024-03-06T07:31:20.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6d1400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb6dc06e8e0 0x7fb6dc15eae0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34200/0 (socket says 172.21.15.86:34200) 2024-03-06T07:31:20.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6d1400640 1 -- 172.21.15.86:0/4205239439 learned_addr learned my addr 172.21.15.86:0/4205239439 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:20.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6d9e00640 1 --2- 172.21.15.86:0/4205239439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc06e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:20.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6da800640 1 --2- 172.21.15.86:0/4205239439 >> v2:172.21.15.194:3300/0 conn(0x7fb6dc15f1f0 0x7fb6dc0cd090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:20.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.742+0000 7fb6d9400640 1 -- 172.21.15.86:0/4205239439 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1061055744 0 0) 0x7fb6dc15b1d0 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.743+0000 7fb6d9400640 1 -- 172.21.15.86:0/4205239439 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb6c0003520 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.743+0000 7fb6d9e00640 1 -- 172.21.15.86:0/4205239439 >> v2:172.21.15.194:3300/0 conn(0x7fb6dc15f1f0 msgr2=0x7fb6dc0cd090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:20.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.743+0000 7fb6d9e00640 1 --2- 172.21.15.86:0/4205239439 >> v2:172.21.15.194:3300/0 conn(0x7fb6dc15f1f0 0x7fb6dc0cd090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.743+0000 7fb6d9400640 1 -- 172.21.15.86:0/4205239439 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 316586293 0 0) 0x7fb6c0003520 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.743+0000 7fb6d9e00640 1 -- 172.21.15.86:0/4205239439 >> v1:172.21.15.86:6789/0 conn(0x7fb6dc06e8e0 legacy=0x7fb6dc15eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:20.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.743+0000 7fb6d9e00640 1 -- 172.21.15.86:0/4205239439 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6dc0b2eb0 con 0x7fb6dc15adf0 2024-03-06T07:31:20.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.744+0000 7fb6d9e00640 1 --2- 172.21.15.86:0/4205239439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc06e3a0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fb6cc0097a0 tx=0x7fb6cc02b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3eab401d3f50649c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:20.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.744+0000 7fb6d9400640 1 -- 172.21.15.86:0/4205239439 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6cc037070 con 0x7fb6dc15adf0 2024-03-06T07:31:20.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6d9400640 1 -- 172.21.15.86:0/4205239439 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6cc030510 con 0x7fb6dc15adf0 2024-03-06T07:31:20.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6d9400640 1 -- 172.21.15.86:0/4205239439 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6c4001670 con 0x7fb6dc15adf0 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 -- 172.21.15.86:0/4205239439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 msgr2=0x7fb6dc06e3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/4205239439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc06e3a0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fb6cc0097a0 tx=0x7fb6cc02b790 comp rx=0 tx=0).stop 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 -- 172.21.15.86:0/4205239439 shutdown_connections 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/4205239439 >> v2:172.21.15.194:3300/0 conn(0x7fb6dc15f1f0 0x7fb6dc0cd090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/4205239439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc06e3a0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 -- 172.21.15.86:0/4205239439 >> 172.21.15.86:0/4205239439 conn(0x7fb6dc0cfa80 msgr2=0x7fb6dc0cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 -- 172.21.15.86:0/4205239439 shutdown_connections 2024-03-06T07:31:20.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.745+0000 7fb6e0a00640 1 -- 172.21.15.86:0/4205239439 wait complete. 2024-03-06T07:31:20.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.747+0000 7fb6e0a00640 1 Processor -- start 2024-03-06T07:31:20.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.747+0000 7fb6e0a00640 1 -- start start 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6e0a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6dc06e8e0 0x7fb6dc153aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6e0a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc1555e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6e0a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6dc15f1f0 0x7fb6dc140400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6e0a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6dc15f850 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6e0a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6dc0b3470 con 0x7fb6dc15f1f0 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6e0a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6dc0b3990 con 0x7fb6dc15adf0 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6d9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6dc06e8e0 0x7fb6dc153aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6d9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6dc06e8e0 0x7fb6dc153aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33060/0 (socket says 172.21.15.86:33060) 2024-03-06T07:31:20.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.748+0000 7fb6d9e00640 1 -- 172.21.15.86:0/1887156984 learned_addr learned my addr 172.21.15.86:0/1887156984 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:20.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d1400640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc1555e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6da800640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6dc15f1f0 0x7fb6dc140400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d9e00640 1 -- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 msgr2=0x7fb6dc1555e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d9e00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc1555e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d9e00640 1 -- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6dc15f1f0 msgr2=0x7fb6dc140400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d9e00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6dc15f1f0 0x7fb6dc140400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d9e00640 1 -- 172.21.15.86:0/1887156984 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6dc151460 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6da800640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6dc15f1f0 0x7fb6dc140400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d9e00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6dc06e8e0 0x7fb6dc153aa0 secure :-1 s=READY pgs=890 cs=0 l=1 rev1=1 crypto rx=0x7fb6cc0098c0 tx=0x7fb6cc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d2a00640 1 -- 172.21.15.86:0/1887156984 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6cc041020 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d2a00640 1 -- 172.21.15.86:0/1887156984 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6cc03c870 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6dc155b20 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.749+0000 7fb6d2a00640 1 -- 172.21.15.86:0/1887156984 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6cc037040 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.750+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6dc150ea0 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.751+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb6dc04f330 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.752+0000 7fb6d2a00640 1 -- 172.21.15.86:0/1887156984 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb6cc044020 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.752+0000 7fb6d2a00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6b806b4d0 0x7fb6b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:20.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.752+0000 7fb6d2a00640 1 -- 172.21.15.86:0/1887156984 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(74..74 src has 1..74) v4 ==== 5415+0+0 (secure 0 0 0) 0x7fb6cc0abaa0 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.753+0000 7fb6d1400640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6b806b4d0 0x7fb6b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:20.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.753+0000 7fb6d1400640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6b806b4d0 0x7fb6b806d990 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7fb6c4001eb0 tx=0x7fb6c4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:20.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.757+0000 7fb6d2a00640 1 -- 172.21.15.86:0/1887156984 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb6cc04b030 con 0x7fb6dc06e8e0 2024-03-06T07:31:20.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:20.951+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fb6dc15e700 con 0x7fb6dc06e8e0 2024-03-06T07:31:21.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.521+0000 7fb6d2a00640 1 -- 172.21.15.86:0/1887156984 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v75) v1 ==== 169+0+0 (secure 0 0 0) 0x7fb6cc07a9c0 con 0x7fb6dc06e8e0 2024-03-06T07:31:21.523 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:31:21.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.525+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6b806b4d0 msgr2=0x7fb6b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:21.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.525+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6b806b4d0 0x7fb6b806d990 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7fb6c4001eb0 tx=0x7fb6c4009040 comp rx=0 tx=0).stop 2024-03-06T07:31:21.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.525+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6dc06e8e0 msgr2=0x7fb6dc153aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:21.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.525+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6dc06e8e0 0x7fb6dc153aa0 secure :-1 s=READY pgs=890 cs=0 l=1 rev1=1 crypto rx=0x7fb6cc0098c0 tx=0x7fb6cc0046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 shutdown_connections 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb6b806b4d0 0x7fb6b806d990 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb6dc15f1f0 0x7fb6dc140400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb6dc15adf0 0x7fb6dc1555e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 --2- 172.21.15.86:0/1887156984 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb6dc06e8e0 0x7fb6dc153aa0 unknown :-1 s=CLOSED pgs=890 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 >> 172.21.15.86:0/1887156984 conn(0x7fb6dc0cfa80 msgr2=0x7fb6dc14cce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 shutdown_connections 2024-03-06T07:31:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.526+0000 7fb6e0a00640 1 -- 172.21.15.86:0/1887156984 wait complete. 2024-03-06T07:31:21.553 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-06T07:31:21.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.765+0000 7fb0c1600640 1 Processor -- start 2024-03-06T07:31:21.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.765+0000 7fb0c1600640 1 -- start start 2024-03-06T07:31:21.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.766+0000 7fb0c1600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb0bc15fc10 0x7fb0bc070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:21.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.766+0000 7fb0c1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc15fff0 0x7fb0bc160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:21.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.766+0000 7fb0c1600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb0bc0b46c0 con 0x7fb0bc0ce4a0 2024-03-06T07:31:21.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.766+0000 7fb0c1600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0bc0b49a0 con 0x7fb0bc15fc10 2024-03-06T07:31:21.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.766+0000 7fb0c1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0bc072c90 con 0x7fb0bc15fff0 2024-03-06T07:31:21.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.767+0000 7fb0baa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb0bc15fc10 0x7fb0bc070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:21.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.767+0000 7fb0baa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb0bc15fc10 0x7fb0bc070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33072/0 (socket says 172.21.15.86:33072) 2024-03-06T07:31:21.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.767+0000 7fb0baa00640 1 -- 172.21.15.86:0/705079301 learned_addr learned my addr 172.21.15.86:0/705079301 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:21.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.767+0000 7fb0baa00640 1 -- 172.21.15.86:0/705079301 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc15fff0 msgr2=0x7fb0bc160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:21.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.767+0000 7fb0baa00640 1 --2- 172.21.15.86:0/705079301 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc15fff0 0x7fb0bc160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.767+0000 7fb0baa00640 1 -- 172.21.15.86:0/705079301 >> v1:172.21.15.194:6789/0 conn(0x7fb0bc0ce4a0 legacy=0x7fb0bc0ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:21.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.767+0000 7fb0baa00640 1 -- 172.21.15.86:0/705079301 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0bc160990 con 0x7fb0bc15fc10 2024-03-06T07:31:21.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.768+0000 7fb0baa00640 1 --2- 172.21.15.86:0/705079301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc070ff0 secure :-1 s=READY pgs=891 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac009870 tx=0x7fb0ac02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e54d90e8f9b873b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:21.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.768+0000 7fb0b9600640 1 -- 172.21.15.86:0/705079301 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac035020 con 0x7fb0bc15fc10 2024-03-06T07:31:21.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.768+0000 7fb0b9600640 1 -- 172.21.15.86:0/705079301 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0ac031640 con 0x7fb0bc15fc10 2024-03-06T07:31:21.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.768+0000 7fb0b9600640 1 -- 172.21.15.86:0/705079301 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac02d040 con 0x7fb0bc15fc10 2024-03-06T07:31:21.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 -- 172.21.15.86:0/705079301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 msgr2=0x7fb0bc070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:21.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 --2- 172.21.15.86:0/705079301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc070ff0 secure :-1 s=READY pgs=891 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac009870 tx=0x7fb0ac02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 -- 172.21.15.86:0/705079301 shutdown_connections 2024-03-06T07:31:21.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 --2- 172.21.15.86:0/705079301 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc15fff0 0x7fb0bc160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 --2- 172.21.15.86:0/705079301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc070ff0 unknown :-1 s=CLOSED pgs=891 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 -- 172.21.15.86:0/705079301 >> 172.21.15.86:0/705079301 conn(0x7fb0bc0b1940 msgr2=0x7fb0bc0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:21.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 -- 172.21.15.86:0/705079301 shutdown_connections 2024-03-06T07:31:21.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.769+0000 7fb0c1600640 1 -- 172.21.15.86:0/705079301 wait complete. 2024-03-06T07:31:21.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.771+0000 7fb0c1600640 1 Processor -- start 2024-03-06T07:31:21.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.771+0000 7fb0c1600640 1 -- start start 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0c1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc0ce4a0 0x7fb0bc06e3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0c1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc150340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0c1600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0bc15fff0 0x7fb0bc09c630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0c1600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0bc072a30 con 0x7fb0bc15fc10 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0c1600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0bc0728b0 con 0x7fb0bc15fff0 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0c1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0bc15c2e0 con 0x7fb0bc0ce4a0 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0ba000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc150340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0ba000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc150340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33086/0 (socket says 172.21.15.86:33086) 2024-03-06T07:31:21.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.772+0000 7fb0ba000640 1 -- 172.21.15.86:0/3199393535 learned_addr learned my addr 172.21.15.86:0/3199393535 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0ba000640 1 -- 172.21.15.86:0/3199393535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc0ce4a0 msgr2=0x7fb0bc06e3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0ba000640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc0ce4a0 0x7fb0bc06e3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0ba000640 1 -- 172.21.15.86:0/3199393535 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0bc15fff0 msgr2=0x7fb0bc09c630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0ba000640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0bc15fff0 0x7fb0bc09c630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0ba000640 1 -- 172.21.15.86:0/3199393535 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0bc09d110 con 0x7fb0bc15fc10 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0ba000640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc150340 secure :-1 s=READY pgs=892 cs=0 l=1 rev1=1 crypto rx=0x7fb0a800b440 tx=0x7fb0a800b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0b2a00640 1 -- 172.21.15.86:0/3199393535 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0a8015020 con 0x7fb0bc15fc10 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0b2a00640 1 -- 172.21.15.86:0/3199393535 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0a8011890 con 0x7fb0bc15fc10 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0b2a00640 1 -- 172.21.15.86:0/3199393535 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0a8011c80 con 0x7fb0bc15fc10 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0bc09cb70 con 0x7fb0bc15fc10 2024-03-06T07:31:21.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.773+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0bc09ad70 con 0x7fb0bc15fc10 2024-03-06T07:31:21.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.774+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb0bc04f330 con 0x7fb0bc15fc10 2024-03-06T07:31:21.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.775+0000 7fb0b2a00640 1 -- 172.21.15.86:0/3199393535 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb0a8018020 con 0x7fb0bc15fc10 2024-03-06T07:31:21.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.776+0000 7fb0b2a00640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09c06b4d0 0x7fb09c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:21.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.776+0000 7fb0b2a00640 1 -- 172.21.15.86:0/3199393535 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fb0a808a0d0 con 0x7fb0bc15fc10 2024-03-06T07:31:21.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.776+0000 7fb0baa00640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09c06b4d0 0x7fb09c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:21.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.777+0000 7fb0baa00640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09c06b4d0 0x7fb09c06d990 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac036000 tx=0x7fb0ac030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:21.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.781+0000 7fb0b2a00640 1 -- 172.21.15.86:0/3199393535 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb0a801f020 con 0x7fb0bc15fc10 2024-03-06T07:31:21.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.990+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7fb0bc06e980 con 0x7fb0bc15fc10 2024-03-06T07:31:21.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.995+0000 7fb0b2a00640 1 -- 172.21.15.86:0/3199393535 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v24) v1 ==== 104+0+0 (secure 0 0 0) 0x7fb0a800d040 con 0x7fb0bc15fc10 2024-03-06T07:31:21.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09c06b4d0 msgr2=0x7fb09c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:21.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09c06b4d0 0x7fb09c06d990 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac036000 tx=0x7fb0ac030000 comp rx=0 tx=0).stop 2024-03-06T07:31:21.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 msgr2=0x7fb0bc150340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:21.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc150340 secure :-1 s=READY pgs=892 cs=0 l=1 rev1=1 crypto rx=0x7fb0a800b440 tx=0x7fb0a800b900 comp rx=0 tx=0).stop 2024-03-06T07:31:21.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 shutdown_connections 2024-03-06T07:31:21.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09c06b4d0 0x7fb09c06d990 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:21.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0bc15fff0 0x7fb0bc09c630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0bc15fc10 0x7fb0bc150340 unknown :-1 s=CLOSED pgs=892 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 --2- 172.21.15.86:0/3199393535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0bc0ce4a0 0x7fb0bc06e3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.997+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 >> 172.21.15.86:0/3199393535 conn(0x7fb0bc0b1940 msgr2=0x7fb0bc0b3d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:22.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.998+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 shutdown_connections 2024-03-06T07:31:22.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:21.998+0000 7fb0c1600640 1 -- 172.21.15.86:0/3199393535 wait complete. 2024-03-06T07:31:22.016 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_fragment_size_max 2024-03-06T07:31:22.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.205+0000 7f070d400640 1 Processor -- start 2024-03-06T07:31:22.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.205+0000 7f070d400640 1 -- start start 2024-03-06T07:31:22.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.206+0000 7f070d400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f07080b3df0 0x7f07080727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.206+0000 7f070d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070806c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.206+0000 7f070d400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f07080719e0 con 0x7f07080b3650 2024-03-06T07:31:22.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.206+0000 7f070d400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0708072310 con 0x7f07080b3df0 2024-03-06T07:31:22.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.206+0000 7f070d400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0708071cc0 con 0x7f07080cda90 2024-03-06T07:31:22.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.207+0000 7f0707200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070806c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.207+0000 7f0707200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070806c990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:48880/0 (socket says 172.21.15.86:48880) 2024-03-06T07:31:22.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.207+0000 7f0706800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f07080b3df0 0x7f07080727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.207+0000 7f0707200640 1 -- 172.21.15.86:0/3282483837 learned_addr learned my addr 172.21.15.86:0/3282483837 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:22.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.208+0000 7f0706800640 1 -- 172.21.15.86:0/3282483837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 msgr2=0x7f070806c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:22.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.208+0000 7f0706800640 1 --2- 172.21.15.86:0/3282483837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070806c990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.208+0000 7f0706800640 1 -- 172.21.15.86:0/3282483837 >> v1:172.21.15.194:6789/0 conn(0x7f07080b3650 legacy=0x7f07080b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.208+0000 7f0706800640 1 -- 172.21.15.86:0/3282483837 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0708071fa0 con 0x7f07080b3df0 2024-03-06T07:31:22.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.209+0000 7f0706800640 1 --2- 172.21.15.86:0/3282483837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 0x7f07080727a0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f06f80097a0 tx=0x7f06f802dae0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e7499c8006607c61 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:22.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.210+0000 7f0705400640 1 -- 172.21.15.86:0/3282483837 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06f802f8a0 con 0x7f07080b3df0 2024-03-06T07:31:22.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.210+0000 7f0705400640 1 -- 172.21.15.86:0/3282483837 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f06f802fa40 con 0x7f07080b3df0 2024-03-06T07:31:22.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.210+0000 7f0705400640 1 -- 172.21.15.86:0/3282483837 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06f802fe10 con 0x7f07080b3df0 2024-03-06T07:31:22.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.210+0000 7f070d400640 1 -- 172.21.15.86:0/3282483837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 msgr2=0x7f07080727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.210+0000 7f070d400640 1 --2- 172.21.15.86:0/3282483837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 0x7f07080727a0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f06f80097a0 tx=0x7f06f802dae0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.211+0000 7f070d400640 1 -- 172.21.15.86:0/3282483837 shutdown_connections 2024-03-06T07:31:22.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.211+0000 7f070d400640 1 --2- 172.21.15.86:0/3282483837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070806c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.211+0000 7f070d400640 1 --2- 172.21.15.86:0/3282483837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 0x7f07080727a0 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.211+0000 7f070d400640 1 -- 172.21.15.86:0/3282483837 >> 172.21.15.86:0/3282483837 conn(0x7f07080b1940 msgr2=0x7f07080b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:22.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.211+0000 7f070d400640 1 -- 172.21.15.86:0/3282483837 shutdown_connections 2024-03-06T07:31:22.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.211+0000 7f070d400640 1 -- 172.21.15.86:0/3282483837 wait complete. 2024-03-06T07:31:22.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.212+0000 7f070d400640 1 Processor -- start 2024-03-06T07:31:22.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.213+0000 7f070d400640 1 -- start start 2024-03-06T07:31:22.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.213+0000 7f070d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07080b3650 0x7f0708151000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f070d400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 0x7f070814d810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f070d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070814c4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f070d400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f070815b3c0 con 0x7f07080b3650 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f070d400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f070815b240 con 0x7f07080b3df0 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f070d400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f07080cf510 con 0x7f07080cda90 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07080b3650 0x7f0708151000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07080b3650 0x7f0708151000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33098/0 (socket says 172.21.15.86:33098) 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 -- 172.21.15.86:0/4224393847 learned_addr learned my addr 172.21.15.86:0/4224393847 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0707200640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070814c4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0705e00640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 0x7f070814d810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 -- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 msgr2=0x7f070814c4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070814c4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 -- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 msgr2=0x7f070814d810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 0x7f070814d810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0706800640 1 -- 172.21.15.86:0/4224393847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f070814e040 con 0x7f07080b3650 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.214+0000 7f0707200640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070814c4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.215+0000 7f0706800640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07080b3650 0x7f0708151000 secure :-1 s=READY pgs=893 cs=0 l=1 rev1=1 crypto rx=0x7f06f80098c0 tx=0x7f06f8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.215+0000 7f06fea00640 1 -- 172.21.15.86:0/4224393847 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06f8049070 con 0x7f07080b3650 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.215+0000 7f06fea00640 1 -- 172.21.15.86:0/4224393847 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f06f8030d90 con 0x7f07080b3650 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.215+0000 7f06fea00640 1 -- 172.21.15.86:0/4224393847 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06f8039640 con 0x7f07080b3650 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.215+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f070815b5a0 con 0x7f07080b3650 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.215+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f070815bb60 con 0x7f07080b3650 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.216+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f070804f330 con 0x7f07080b3650 2024-03-06T07:31:22.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.217+0000 7f06fea00640 1 -- 172.21.15.86:0/4224393847 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f06f80397e0 con 0x7f07080b3650 2024-03-06T07:31:22.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.218+0000 7f06fea00640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f06e806b4d0 0x7f06e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.218+0000 7f06fea00640 1 -- 172.21.15.86:0/4224393847 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f06f80adca0 con 0x7f07080b3650 2024-03-06T07:31:22.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.218+0000 7f0705e00640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f06e806b4d0 0x7f06e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.219+0000 7f0705e00640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f06e806b4d0 0x7f06e806d990 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f070809e070 tx=0x7f06f4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:22.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.223+0000 7f06fea00640 1 -- 172.21.15.86:0/4224393847 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f06f8035030 con 0x7f07080b3650 2024-03-06T07:31:22.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.421+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"} v 0) v1 -- 0x7f07080cc0e0 con 0x7f07080b3650 2024-03-06T07:31:22.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.427+0000 7f06fea00640 1 -- 172.21.15.86:0/4224393847 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"}]=0 v25) v1 ==== 108+0+0 (secure 0 0 0) 0x7f06f807cee0 con 0x7f07080b3650 2024-03-06T07:31:22.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f06e806b4d0 msgr2=0x7f06e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f06e806b4d0 0x7f06e806d990 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f070809e070 tx=0x7f06f4009040 comp rx=0 tx=0).stop 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07080b3650 msgr2=0x7f0708151000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07080b3650 0x7f0708151000 secure :-1 s=READY pgs=893 cs=0 l=1 rev1=1 crypto rx=0x7f06f80098c0 tx=0x7f06f8004730 comp rx=0 tx=0).stop 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 shutdown_connections 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f06e806b4d0 0x7f06e806d990 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07080cda90 0x7f070814c4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07080b3df0 0x7f070814d810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 --2- 172.21.15.86:0/4224393847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07080b3650 0x7f0708151000 unknown :-1 s=CLOSED pgs=893 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.430+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 >> 172.21.15.86:0/4224393847 conn(0x7f07080b1940 msgr2=0x7f07080ce1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.431+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 shutdown_connections 2024-03-06T07:31:22.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.431+0000 7f070d400640 1 -- 172.21.15.86:0/4224393847 wait complete. 2024-03-06T07:31:22.450 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays' 2024-03-06T07:31:22.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.655+0000 7fbf6ae00640 1 Processor -- start 2024-03-06T07:31:22.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.655+0000 7fbf6ae00640 1 -- start start 2024-03-06T07:31:22.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.657+0000 7fbf6ae00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbf640af1c0 0x7fbf6415eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.657+0000 7fbf6ae00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbf64071d30 con 0x7fbf64070d20 2024-03-06T07:31:22.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.657+0000 7fbf6ae00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbf640711d0 con 0x7fbf6406e4d0 2024-03-06T07:31:22.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.657+0000 7fbf6ae00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbf64071490 con 0x7fbf640af1c0 2024-03-06T07:31:22.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.657+0000 7fbf5b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fbf64070d20 0x7fbf6406cc60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34212/0 (socket says 172.21.15.86:34212) 2024-03-06T07:31:22.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.657+0000 7fbf5b400640 1 -- 172.21.15.86:0/1933900464 learned_addr learned my addr 172.21.15.86:0/1933900464 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1351458178 0 0) 0x7fbf4c001110 con 0x7fbf64070d20 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbf48003520 con 0x7fbf64070d20 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63e00640 1 --2- 172.21.15.86:0/1933900464 >> v2:172.21.15.194:3300/0 conn(0x7fbf640af1c0 0x7fbf6415eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1297031285 0 0) 0x7fbf48003520 con 0x7fbf64070d20 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 >> v2:172.21.15.194:3300/0 conn(0x7fbf640af1c0 msgr2=0x7fbf6415eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 --2- 172.21.15.86:0/1933900464 >> v2:172.21.15.194:3300/0 conn(0x7fbf640af1c0 0x7fbf6415eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 >> v1:172.21.15.194:6789/0 conn(0x7fbf6406e4d0 legacy=0x7fbf640b35d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf64071770 con 0x7fbf64070d20 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fbf4c001840 con 0x7fbf64070d20 2024-03-06T07:31:22.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 >> v1:172.21.15.86:6789/0 conn(0x7fbf64070d20 legacy=0x7fbf6406cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf480039b0 0x7fbf48003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf48004800 0x7fbf48004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf4800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf64071490 con 0x7fbf48005210 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf4800c140 con 0x7fbf480039b0 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.658+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbf4800c340 con 0x7fbf48004800 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf68e00640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf4800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf63e00640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf480039b0 0x7fbf48003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf5b400640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf48004800 0x7fbf48004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf68e00640 1 -- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf48004800 msgr2=0x7fbf48004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf68e00640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf48004800 0x7fbf48004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf68e00640 1 -- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf480039b0 msgr2=0x7fbf48003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf68e00640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf480039b0 0x7fbf48003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.659+0000 7fbf68e00640 1 -- 172.21.15.86:0/1933900464 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf48003520 con 0x7fbf48005210 2024-03-06T07:31:22.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.660+0000 7fbf68e00640 1 -- 172.21.15.86:0/1933900464 reap_dead start 2024-03-06T07:31:22.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.660+0000 7fbf68e00640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf4800b9c0 secure :-1 s=READY pgs=895 cs=0 l=1 rev1=1 crypto rx=0x7fbf5c001e20 tx=0x7fbf48005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44e735013af62290 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:22.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.660+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf5c001a70 con 0x7fbf48005210 2024-03-06T07:31:22.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.660+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf5c003b70 con 0x7fbf48005210 2024-03-06T07:31:22.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.660+0000 7fbf63400640 1 -- 172.21.15.86:0/1933900464 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf5c0050d0 con 0x7fbf48005210 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.661+0000 7fbf6ae00640 1 -- 172.21.15.86:0/1933900464 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 msgr2=0x7fbf4800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.661+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf4800b9c0 secure :-1 s=READY pgs=895 cs=0 l=1 rev1=1 crypto rx=0x7fbf5c001e20 tx=0x7fbf48005830 comp rx=0 tx=0).stop 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.661+0000 7fbf6ae00640 1 -- 172.21.15.86:0/1933900464 shutdown_connections 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.661+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/1933900464 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf4800b9c0 unknown :-1 s=CLOSED pgs=895 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.661+0000 7fbf6ae00640 1 -- 172.21.15.86:0/1933900464 >> 172.21.15.86:0/1933900464 conn(0x7fbf640b1730 msgr2=0x7fbf640b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.661+0000 7fbf6ae00640 1 -- 172.21.15.86:0/1933900464 shutdown_connections 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.661+0000 7fbf6ae00640 1 -- 172.21.15.86:0/1933900464 wait complete. 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.663+0000 7fbf6ae00640 1 Processor -- start 2024-03-06T07:31:22.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.663+0000 7fbf6ae00640 1 -- start start 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf6ae00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf641468d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf6ae00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf641578e0 0x7fbf6415c930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf6ae00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf6415ce70 0x7fbf640cc5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf6ae00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf6415eff0 con 0x7fbf48005210 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf6ae00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf640b30b0 con 0x7fbf6415ce70 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf6ae00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbf6415f170 con 0x7fbf641578e0 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf63e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf641468d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf63e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf641468d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33116/0 (socket says 172.21.15.86:33116) 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf63e00640 1 -- 172.21.15.86:0/3613428169 learned_addr learned my addr 172.21.15.86:0/3613428169 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.664+0000 7fbf5b400640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf641578e0 0x7fbf6415c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf63e00640 1 -- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf641578e0 msgr2=0x7fbf6415c930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf63e00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf641578e0 0x7fbf6415c930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf63e00640 1 -- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf6415ce70 msgr2=0x7fbf640cc5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:22.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf68e00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf6415ce70 0x7fbf640cc5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf63e00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf6415ce70 0x7fbf640cc5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:22.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf63e00640 1 -- 172.21.15.86:0/3613428169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf640cd320 con 0x7fbf48005210 2024-03-06T07:31:22.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf68e00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf6415ce70 0x7fbf640cc5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:22.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf63e00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf641468d0 secure :-1 s=READY pgs=896 cs=0 l=1 rev1=1 crypto rx=0x7fbf5400f580 tx=0x7fbf5400fa40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:22.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf60c00640 1 -- 172.21.15.86:0/3613428169 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf54013810 con 0x7fbf48005210 2024-03-06T07:31:22.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf60c00640 1 -- 172.21.15.86:0/3613428169 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf5401a070 con 0x7fbf48005210 2024-03-06T07:31:22.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf60c00640 1 -- 172.21.15.86:0/3613428169 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf54013bb0 con 0x7fbf48005210 2024-03-06T07:31:22.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.665+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbf64156b20 con 0x7fbf48005210 2024-03-06T07:31:22.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.666+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbf641570e0 con 0x7fbf48005210 2024-03-06T07:31:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.667+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbf6415d280 con 0x7fbf48005210 2024-03-06T07:31:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.667+0000 7fbf60c00640 1 -- 172.21.15.86:0/3613428169 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbf54012070 con 0x7fbf48005210 2024-03-06T07:31:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.669+0000 7fbf60c00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf4006b500 0x7fbf4006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.669+0000 7fbf60c00640 1 -- 172.21.15.86:0/3613428169 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fbf54089e70 con 0x7fbf48005210 2024-03-06T07:31:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.669+0000 7fbf5b400640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf4006b500 0x7fbf4006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.670+0000 7fbf5b400640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf4006b500 0x7fbf4006d9c0 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fbf4c0012f0 tx=0x7fbf4c001220 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:22.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.673+0000 7fbf60c00640 1 -- 172.21.15.86:0/3613428169 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbf6415d280 con 0x7fbf48005210 2024-03-06T07:31:22.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:22.943+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]} v 0) v1 -- 0x7fbf64157d70 con 0x7fbf48005210 2024-03-06T07:31:23.546 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:31:23.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.545+0000 7fbf60c00640 1 -- 172.21.15.86:0/3613428169 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]}]=0 v250) v1 ==== 140+0+0 (secure 0 0 0) 0x7fbf5401c310 con 0x7fbf48005210 2024-03-06T07:31:23.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:31:23.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf4006b500 msgr2=0x7fbf4006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:23.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf4006b500 0x7fbf4006d9c0 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fbf4c0012f0 tx=0x7fbf4c001220 comp rx=0 tx=0).stop 2024-03-06T07:31:23.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 msgr2=0x7fbf641468d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:23.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf641468d0 secure :-1 s=READY pgs=896 cs=0 l=1 rev1=1 crypto rx=0x7fbf5400f580 tx=0x7fbf5400fa40 comp rx=0 tx=0).stop 2024-03-06T07:31:23.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 shutdown_connections 2024-03-06T07:31:23.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf4006b500 0x7fbf4006d9c0 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbf6415ce70 0x7fbf640cc5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbf641578e0 0x7fbf6415c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 --2- 172.21.15.86:0/3613428169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbf48005210 0x7fbf641468d0 unknown :-1 s=CLOSED pgs=896 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 >> 172.21.15.86:0/3613428169 conn(0x7fbf640b1730 msgr2=0x7fbf6406db50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:23.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 shutdown_connections 2024-03-06T07:31:23.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.550+0000 7fbf6ae00640 1 -- 172.21.15.86:0/3613428169 wait complete. 2024-03-06T07:31:23.572 INFO:tasks.cephfs_test_runner:That unlinking fails when the stray directory fragment becomes too ... ok 2024-03-06T07:31:23.573 INFO:tasks.cephfs_test_runner:Starting test: test_fancy_layout (tasks.cephfs.test_strays.TestStrays) purge stray file with fancy layout 2024-03-06T07:31:23.573 INFO:tasks.cephfs_test_runner:test_fancy_layout (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:31:23.573 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:23.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.817+0000 7f535a800640 1 Processor -- start 2024-03-06T07:31:23.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.818+0000 7f535a800640 1 -- start start 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f535a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53540cf430 0x7f535406d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f535a800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f535406d920 0x7f535415dff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f535a800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f53540b35b0 con 0x7f5354151660 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f535a800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f53540b3ee0 con 0x7f535406d920 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f535a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f53540b3890 con 0x7f53540cf430 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f5353e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f535406d920 0x7f535415dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f5353e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f535406d920 0x7f535415dff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33122/0 (socket says 172.21.15.86:33122) 2024-03-06T07:31:23.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.819+0000 7f5353e00640 1 -- 172.21.15.86:0/665647199 learned_addr learned my addr 172.21.15.86:0/665647199 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:23.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.820+0000 7f534aa00640 1 --2- 172.21.15.86:0/665647199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53540cf430 0x7f535406d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:23.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.820+0000 7f5353e00640 1 -- 172.21.15.86:0/665647199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53540cf430 msgr2=0x7f535406d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:23.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.820+0000 7f5353e00640 1 --2- 172.21.15.86:0/665647199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53540cf430 0x7f535406d3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.820+0000 7f5353e00640 1 -- 172.21.15.86:0/665647199 >> v1:172.21.15.194:6789/0 conn(0x7f5354151660 legacy=0x7f5354071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:23.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.820+0000 7f5353e00640 1 -- 172.21.15.86:0/665647199 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f53540b3b70 con 0x7f535406d920 2024-03-06T07:31:23.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.821+0000 7f5353e00640 1 --2- 172.21.15.86:0/665647199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f535415dff0 secure :-1 s=READY pgs=897 cs=0 l=1 rev1=1 crypto rx=0x7f534c009e50 tx=0x7f534c02bdb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c65e7ceb313fb8e5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.821+0000 7f5352a00640 1 -- 172.21.15.86:0/665647199 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f534c02c950 con 0x7f535406d920 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.821+0000 7f5352a00640 1 -- 172.21.15.86:0/665647199 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f534c02caf0 con 0x7f535406d920 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.821+0000 7f5352a00640 1 -- 172.21.15.86:0/665647199 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f534c0343f0 con 0x7f535406d920 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 -- 172.21.15.86:0/665647199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 msgr2=0x7f535415dff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 --2- 172.21.15.86:0/665647199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f535415dff0 secure :-1 s=READY pgs=897 cs=0 l=1 rev1=1 crypto rx=0x7f534c009e50 tx=0x7f534c02bdb0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 -- 172.21.15.86:0/665647199 shutdown_connections 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 --2- 172.21.15.86:0/665647199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f535415dff0 unknown :-1 s=CLOSED pgs=897 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 --2- 172.21.15.86:0/665647199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53540cf430 0x7f535406d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 -- 172.21.15.86:0/665647199 >> 172.21.15.86:0/665647199 conn(0x7f53540b1730 msgr2=0x7f53540b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 -- 172.21.15.86:0/665647199 shutdown_connections 2024-03-06T07:31:23.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.822+0000 7f535a800640 1 -- 172.21.15.86:0/665647199 wait complete. 2024-03-06T07:31:23.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.824+0000 7f535a800640 1 Processor -- start 2024-03-06T07:31:23.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.824+0000 7f535a800640 1 -- start start 2024-03-06T07:31:23.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f535a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f5354148bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:23.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f535a800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f53540cf430 0x7f53541593d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:23.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f535a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5354151660 0x7f5354159910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:23.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f535a800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f535415e6b0 con 0x7f535406d920 2024-03-06T07:31:23.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f535a800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f535415e530 con 0x7f53540cf430 2024-03-06T07:31:23.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f535a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f535415e830 con 0x7f5354151660 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f5354148bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f5354148bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33134/0 (socket says 172.21.15.86:33134) 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 -- 172.21.15.86:0/1811696943 learned_addr learned my addr 172.21.15.86:0/1811696943 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353e00640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5354151660 0x7f5354159910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f534aa00640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f53540cf430 0x7f53541593d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 -- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5354151660 msgr2=0x7f5354159910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5354151660 0x7f5354159910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 -- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f53540cf430 msgr2=0x7f53541593d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:23.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f53540cf430 0x7f53541593d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353400640 1 -- 172.21.15.86:0/1811696943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f53541490f0 con 0x7f535406d920 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f5353e00640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5354151660 0x7f5354159910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.825+0000 7f534aa00640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f53540cf430 0x7f53541593d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.826+0000 7f5353400640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f5354148bb0 secure :-1 s=READY pgs=898 cs=0 l=1 rev1=1 crypto rx=0x7f534400b3f0 tx=0x7f534400b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.826+0000 7f534be00640 1 -- 172.21.15.86:0/1811696943 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5344015020 con 0x7f535406d920 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.826+0000 7f534be00640 1 -- 172.21.15.86:0/1811696943 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5344011830 con 0x7f535406d920 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.826+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5354150920 con 0x7f535406d920 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.826+0000 7f534be00640 1 -- 172.21.15.86:0/1811696943 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5344011c20 con 0x7f535406d920 2024-03-06T07:31:23.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.826+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f53541532f0 con 0x7f535406d920 2024-03-06T07:31:23.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.827+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f535404f330 con 0x7f535406d920 2024-03-06T07:31:23.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.828+0000 7f534be00640 1 -- 172.21.15.86:0/1811696943 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5344018020 con 0x7f535406d920 2024-03-06T07:31:23.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.829+0000 7f534be00640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f533006b4d0 0x7f533006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:23.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.830+0000 7f534be00640 1 -- 172.21.15.86:0/1811696943 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f5344089b40 con 0x7f535406d920 2024-03-06T07:31:23.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.830+0000 7f534aa00640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f533006b4d0 0x7f533006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:23.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.830+0000 7f534aa00640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f533006b4d0 0x7f533006d990 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f535414bd60 tx=0x7f533c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:23.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:23.835+0000 7f534be00640 1 -- 172.21.15.86:0/1811696943 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f534401f030 con 0x7f535406d920 2024-03-06T07:31:24.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.031+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f535415beb0 con 0x7f535406d920 2024-03-06T07:31:24.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.033+0000 7f534be00640 1 -- 172.21.15.86:0/1811696943 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v75) v1 ==== 74+0+13031 (secure 0 0 0) 0x7f53541490f0 con 0x7f535406d920 2024-03-06T07:31:24.035 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:24.035 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":75,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:21.518171+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/195049401":"2024-03-07T07:31:18.498344+0000","172.21.15.86:6834/195049401":"2024-03-07T07:31:18.498344+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:24.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f533006b4d0 msgr2=0x7f533006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f533006b4d0 0x7f533006d990 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f535414bd60 tx=0x7f533c008040 comp rx=0 tx=0).stop 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 msgr2=0x7f5354148bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f5354148bb0 secure :-1 s=READY pgs=898 cs=0 l=1 rev1=1 crypto rx=0x7f534400b3f0 tx=0x7f534400b8b0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 shutdown_connections 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f533006b4d0 0x7f533006d990 secure :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f535414bd60 tx=0x7f533c008040 comp rx=0 tx=0).stop 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5354151660 0x7f5354159910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f53540cf430 0x7f53541593d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 --2- 172.21.15.86:0/1811696943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f535406d920 0x7f5354148bb0 unknown :-1 s=CLOSED pgs=898 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 >> 172.21.15.86:0/1811696943 conn(0x7f53540b1730 msgr2=0x7f53540b4370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:24.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 shutdown_connections 2024-03-06T07:31:24.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.036+0000 7f535a800640 1 -- 172.21.15.86:0/1811696943 wait complete. 2024-03-06T07:31:24.057 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:31:24.057 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:31:24.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.257+0000 7f17ee600640 1 Processor -- start 2024-03-06T07:31:24.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.257+0000 7f17ee600640 1 -- start start 2024-03-06T07:31:24.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.258+0000 7f17ee600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f17e815fc10 0x7f17e8070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.258+0000 7f17ee600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e815fff0 0x7f17e8160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.258+0000 7f17ee600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f17e80ce4a0 0x7f17e80ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.258+0000 7f17ee600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17e80b46c0 con 0x7f17e815fc10 2024-03-06T07:31:24.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.258+0000 7f17ee600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17e80b49a0 con 0x7f17e80ce4a0 2024-03-06T07:31:24.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.258+0000 7f17ee600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17e8072c90 con 0x7f17e815fff0 2024-03-06T07:31:24.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f17e815fc10 0x7f17e8070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f17e815fc10 0x7f17e8070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33142/0 (socket says 172.21.15.86:33142) 2024-03-06T07:31:24.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 -- 172.21.15.86:0/126463582 learned_addr learned my addr 172.21.15.86:0/126463582 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:24.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17dea00640 1 --2- 172.21.15.86:0/126463582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e815fff0 0x7f17e8160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7e00640 1 --2- 172.21.15.86:0/126463582 >> v2:172.21.15.194:3300/0 conn(0x7f17e80ce4a0 0x7f17e80ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 -- 172.21.15.86:0/126463582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e815fff0 msgr2=0x7f17e8160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:24.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 --2- 172.21.15.86:0/126463582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e815fff0 0x7f17e8160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 -- 172.21.15.86:0/126463582 >> v2:172.21.15.194:3300/0 conn(0x7f17e80ce4a0 msgr2=0x7f17e80ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:24.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 --2- 172.21.15.86:0/126463582 >> v2:172.21.15.194:3300/0 conn(0x7f17e80ce4a0 0x7f17e80ce930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.259+0000 7f17e7400640 1 -- 172.21.15.86:0/126463582 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17e8160990 con 0x7f17e815fc10 2024-03-06T07:31:24.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.261+0000 7f17e7400640 1 --2- 172.21.15.86:0/126463582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fc10 0x7f17e8070ff0 secure :-1 s=READY pgs=899 cs=0 l=1 rev1=1 crypto rx=0x7f17e815fb70 tx=0x7f17d802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59e449334d73e631 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:24.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.261+0000 7f17e6a00640 1 -- 172.21.15.86:0/126463582 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17d8035020 con 0x7f17e815fc10 2024-03-06T07:31:24.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.261+0000 7f17e6a00640 1 -- 172.21.15.86:0/126463582 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17d8031870 con 0x7f17e815fc10 2024-03-06T07:31:24.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.261+0000 7f17e6a00640 1 -- 172.21.15.86:0/126463582 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17d8031c40 con 0x7f17e815fc10 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 -- 172.21.15.86:0/126463582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fc10 msgr2=0x7f17e8070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 --2- 172.21.15.86:0/126463582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fc10 0x7f17e8070ff0 secure :-1 s=READY pgs=899 cs=0 l=1 rev1=1 crypto rx=0x7f17e815fb70 tx=0x7f17d802b910 comp rx=0 tx=0).stop 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 -- 172.21.15.86:0/126463582 shutdown_connections 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 --2- 172.21.15.86:0/126463582 >> v2:172.21.15.194:3300/0 conn(0x7f17e80ce4a0 0x7f17e80ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 --2- 172.21.15.86:0/126463582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e815fff0 0x7f17e8160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 --2- 172.21.15.86:0/126463582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fc10 0x7f17e8070ff0 unknown :-1 s=CLOSED pgs=899 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 -- 172.21.15.86:0/126463582 >> 172.21.15.86:0/126463582 conn(0x7f17e80b1940 msgr2=0x7f17e80b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 -- 172.21.15.86:0/126463582 shutdown_connections 2024-03-06T07:31:24.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.262+0000 7f17ee600640 1 -- 172.21.15.86:0/126463582 wait complete. 2024-03-06T07:31:24.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.264+0000 7f17ee600640 1 Processor -- start 2024-03-06T07:31:24.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.265+0000 7f17ee600640 1 -- start start 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17ee600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e80ce4a0 0x7f17e8150670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17ee600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fff0 0x7f17e81e77f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17ee600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17e81e7d30 0x7f17e809ca00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17ee600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17e8072a30 con 0x7f17e815fff0 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17ee600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17e80728b0 con 0x7f17e81e7d30 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17ee600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17e815c2e0 con 0x7f17e80ce4a0 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fff0 0x7f17e81e77f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fff0 0x7f17e81e77f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33152/0 (socket says 172.21.15.86:33152) 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17dea00640 1 -- 172.21.15.86:0/1718370929 learned_addr learned my addr 172.21.15.86:0/1718370929 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.266+0000 7f17e7400640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e80ce4a0 0x7f17e8150670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dea00640 1 -- 172.21.15.86:0/1718370929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e80ce4a0 msgr2=0x7f17e8150670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dea00640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e80ce4a0 0x7f17e8150670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dea00640 1 -- 172.21.15.86:0/1718370929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17e81e7d30 msgr2=0x7f17e809ca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dea00640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17e81e7d30 0x7f17e809ca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dea00640 1 -- 172.21.15.86:0/1718370929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17e81e8290 con 0x7f17e815fff0 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dea00640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fff0 0x7f17e81e77f0 secure :-1 s=READY pgs=900 cs=0 l=1 rev1=1 crypto rx=0x7f17d000d620 tx=0x7f17d000dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dfe00640 1 -- 172.21.15.86:0/1718370929 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17d00125f0 con 0x7f17e815fff0 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dfe00640 1 -- 172.21.15.86:0/1718370929 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17d0019070 con 0x7f17e815fff0 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17dfe00640 1 -- 172.21.15.86:0/1718370929 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17d00129d0 con 0x7f17e815fff0 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17e806ca30 con 0x7f17e815fff0 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.267+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17e806cff0 con 0x7f17e815fff0 2024-03-06T07:31:24.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.269+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f17e804f330 con 0x7f17e815fff0 2024-03-06T07:31:24.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.269+0000 7f17dfe00640 1 -- 172.21.15.86:0/1718370929 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f17d0012b70 con 0x7f17e815fff0 2024-03-06T07:31:24.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.270+0000 7f17dfe00640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17c406b4d0 0x7f17c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.270+0000 7f17dfe00640 1 -- 172.21.15.86:0/1718370929 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f17d008a2a0 con 0x7f17e815fff0 2024-03-06T07:31:24.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.271+0000 7f17e7400640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17c406b4d0 0x7f17c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.271+0000 7f17e7400640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17c406b4d0 0x7f17c406d990 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f17d8036000 tx=0x7f17d803c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:24.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.275+0000 7f17dfe00640 1 -- 172.21.15.86:0/1718370929 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f17d0059410 con 0x7f17e815fff0 2024-03-06T07:31:24.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.469+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f17e80b2d40 con 0x7f17e815fff0 2024-03-06T07:31:24.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.470+0000 7f17dfe00640 1 -- 172.21.15.86:0/1718370929 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v75) v1 ==== 93+0+10 (secure 0 0 0) 0x7f17d0058d10 con 0x7f17e815fff0 2024-03-06T07:31:24.472 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:31:24.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17c406b4d0 msgr2=0x7f17c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17c406b4d0 0x7f17c406d990 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f17d8036000 tx=0x7f17d803c040 comp rx=0 tx=0).stop 2024-03-06T07:31:24.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fff0 msgr2=0x7f17e81e77f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fff0 0x7f17e81e77f0 secure :-1 s=READY pgs=900 cs=0 l=1 rev1=1 crypto rx=0x7f17d000d620 tx=0x7f17d000dae0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 shutdown_connections 2024-03-06T07:31:24.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17c406b4d0 0x7f17c406d990 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17e81e7d30 0x7f17e809ca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17e815fff0 0x7f17e81e77f0 unknown :-1 s=CLOSED pgs=900 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 --2- 172.21.15.86:0/1718370929 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17e80ce4a0 0x7f17e8150670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 >> 172.21.15.86:0/1718370929 conn(0x7f17e80b1940 msgr2=0x7f17e81513e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:24.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 shutdown_connections 2024-03-06T07:31:24.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.473+0000 7f17ee600640 1 -- 172.21.15.86:0/1718370929 wait complete. 2024-03-06T07:31:24.493 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_fancy_layout' 2024-03-06T07:31:24.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.700+0000 7f18e5800640 1 Processor -- start 2024-03-06T07:31:24.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.700+0000 7f18e5800640 1 -- start start 2024-03-06T07:31:24.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.701+0000 7f18e5800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f18e006edb0 0x7f18e00b4320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.701+0000 7f18e5800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 0x7f18e00ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.701+0000 7f18e5800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f18e00cf340 con 0x7f18e015e230 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.701+0000 7f18e5800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f18e015e610 con 0x7f18e006edb0 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.701+0000 7f18e5800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18e00cfe60 con 0x7f18e00ce550 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.702+0000 7f18dea00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f18e015e230 0x7f18e00d0be0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34224/0 (socket says 172.21.15.86:34224) 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.702+0000 7f18dea00640 1 -- 172.21.15.86:0/2305070747 learned_addr learned my addr 172.21.15.86:0/2305070747 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.702+0000 7f18d6000640 1 --2- 172.21.15.86:0/2305070747 >> v2:172.21.15.194:3300/0 conn(0x7f18e006edb0 0x7f18e00b4320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.702+0000 7f18df400640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 0x7f18e00ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.702+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4293629926 0 0) 0x7f18e00cf340 con 0x7f18e015e230 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.702+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f18c4003520 con 0x7f18e015e230 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.702+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 49230826 0 0) 0x7f18c4003520 con 0x7f18e015e230 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 msgr2=0x7f18e00ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:24.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 0x7f18e00ce9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 >> v2:172.21.15.194:3300/0 conn(0x7f18e006edb0 msgr2=0x7f18e00b4320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 --2- 172.21.15.86:0/2305070747 >> v2:172.21.15.194:3300/0 conn(0x7f18e006edb0 0x7f18e00b4320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18e006f390 con 0x7f18e015e230 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f18d0001670 con 0x7f18e015e230 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 >> v1:172.21.15.86:6789/0 conn(0x7f18e015e230 legacy=0x7f18e00d0be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 0x7f18c40039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18c4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18c4004d40 0x7f18c40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18e015e610 con 0x7f18c4003f30 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18e00cfe60 con 0x7f18c4004d40 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18c400ba10 con 0x7f18e00ce550 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18d6000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18c4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18dea00640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 0x7f18c40039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.703+0000 7f18d6000640 1 -- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 msgr2=0x7f18c40039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:24.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.704+0000 7f18d6000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e00ce550 0x7f18c40039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.704+0000 7f18d6000640 1 -- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18c4004d40 msgr2=0x7f18c40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:24.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.704+0000 7f18d6000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18c4004d40 0x7f18c40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.704+0000 7f18d6000640 1 -- 172.21.15.86:0/2305070747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18c4003520 con 0x7f18c4003f30 2024-03-06T07:31:24.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18d6000640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18c4004800 secure :-1 s=READY pgs=902 cs=0 l=1 rev1=1 crypto rx=0x7f18c8009af0 tx=0x7f18c4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=390b7a50dce98c3a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:24.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18c8039070 con 0x7f18c4003f30 2024-03-06T07:31:24.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f18c802c710 con 0x7f18c4003f30 2024-03-06T07:31:24.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18de000640 1 -- 172.21.15.86:0/2305070747 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18c8009690 con 0x7f18c4003f30 2024-03-06T07:31:24.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18e5800640 1 -- 172.21.15.86:0/2305070747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 msgr2=0x7f18c4004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18e5800640 1 --2- 172.21.15.86:0/2305070747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18c4004800 secure :-1 s=READY pgs=902 cs=0 l=1 rev1=1 crypto rx=0x7f18c8009af0 tx=0x7f18c4005830 comp rx=0 tx=0).stop 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18df400640 1 -- 172.21.15.86:0/2305070747 reap_dead start 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18e5800640 1 -- 172.21.15.86:0/2305070747 shutdown_connections 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18e5800640 1 -- 172.21.15.86:0/2305070747 >> 172.21.15.86:0/2305070747 conn(0x7f18e00b3800 msgr2=0x7f18e00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.705+0000 7f18e5800640 1 -- 172.21.15.86:0/2305070747 shutdown_connections 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.706+0000 7f18e5800640 1 -- 172.21.15.86:0/2305070747 wait complete. 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.707+0000 7f18e5800640 1 Processor -- start 2024-03-06T07:31:24.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.707+0000 7f18e5800640 1 -- start start 2024-03-06T07:31:24.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18e5800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18e01f1a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18e5800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e00d0ae0 0x7f18e00d0f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18e5800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e01f7ec0 0x7f18e01f8340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18e5800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18e00d06c0 con 0x7f18c4003f30 2024-03-06T07:31:24.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18e5800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18e00d03e0 con 0x7f18e00d0ae0 2024-03-06T07:31:24.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18e5800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18e00ce9e0 con 0x7f18e01f7ec0 2024-03-06T07:31:24.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18e01f1a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.708+0000 7f18dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18e01f1a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33162/0 (socket says 172.21.15.86:33162) 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18dea00640 1 -- 172.21.15.86:0/2009290902 learned_addr learned my addr 172.21.15.86:0/2009290902 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18dea00640 1 -- 172.21.15.86:0/2009290902 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e01f7ec0 msgr2=0x7f18e01f8340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18dea00640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e01f7ec0 0x7f18e01f8340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18dea00640 1 -- 172.21.15.86:0/2009290902 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e00d0ae0 msgr2=0x7f18e00d0f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18dea00640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e00d0ae0 0x7f18e00d0f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18dea00640 1 -- 172.21.15.86:0/2009290902 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18e01f8bf0 con 0x7f18c4003f30 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18dea00640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18e01f1a40 secure :-1 s=READY pgs=903 cs=0 l=1 rev1=1 crypto rx=0x7f18d000e850 tx=0x7f18d000ed10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18d7400640 1 -- 172.21.15.86:0/2009290902 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d000c7f0 con 0x7f18c4003f30 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18d7400640 1 -- 172.21.15.86:0/2009290902 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f18d0018070 con 0x7f18c4003f30 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18e014dbc0 con 0x7f18c4003f30 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.709+0000 7f18d7400640 1 -- 172.21.15.86:0/2009290902 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d000cbd0 con 0x7f18c4003f30 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.710+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18e014d0e0 con 0x7f18c4003f30 2024-03-06T07:31:24.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.711+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f18e004f330 con 0x7f18c4003f30 2024-03-06T07:31:24.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.712+0000 7f18d7400640 1 -- 172.21.15.86:0/2009290902 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f18d001d460 con 0x7f18c4003f30 2024-03-06T07:31:24.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.712+0000 7f18d7400640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:24.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.713+0000 7f18d7400640 1 -- 172.21.15.86:0/2009290902 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f18d0089de0 con 0x7f18c4003f30 2024-03-06T07:31:24.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.713+0000 7f18d6000640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:24.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.713+0000 7f18d6000640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f18c8036000 tx=0x7f18c802e000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:24.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.717+0000 7f18d7400640 1 -- 172.21.15.86:0/2009290902 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f18d0058f60 con 0x7f18c4003f30 2024-03-06T07:31:24.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:24.986+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]} v 0) v1 -- 0x7f18e00d1740 con 0x7f18c4003f30 2024-03-06T07:31:25.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.557+0000 7f18d7400640 1 -- 172.21.15.86:0/2009290902 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]}]=0 v252) v1 ==== 135+0+0 (secure 0 0 0) 0x7f18d0058860 con 0x7f18c4003f30 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 msgr2=0x7f18bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f18c8036000 tx=0x7f18c802e000 comp rx=0 tx=0).stop 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 msgr2=0x7f18e01f1a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18e01f1a40 secure :-1 s=READY pgs=903 cs=0 l=1 rev1=1 crypto rx=0x7f18d000e850 tx=0x7f18d000ed10 comp rx=0 tx=0).stop 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 shutdown_connections 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f18bc06b4d0 0x7f18bc06d990 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f18e01f7ec0 0x7f18e01f8340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f18e00d0ae0 0x7f18e00d0f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 --2- 172.21.15.86:0/2009290902 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18c4003f30 0x7f18e01f1a40 unknown :-1 s=CLOSED pgs=903 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.561+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 >> 172.21.15.86:0/2009290902 conn(0x7f18e00b3800 msgr2=0x7f18e006d230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:25.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.562+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 shutdown_connections 2024-03-06T07:31:25.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.562+0000 7f18e5800640 1 -- 172.21.15.86:0/2009290902 wait complete. 2024-03-06T07:31:25.583 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:31:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.793+0000 7f8016000640 1 Processor -- start 2024-03-06T07:31:25.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.794+0000 7f8016000640 1 -- start start 2024-03-06T07:31:25.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f8016000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f801015fc10 0x7f8010070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f8016000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f801015fff0 0x7f8010160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f8016000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f80100b46c0 con 0x7f80100ce4a0 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f8016000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f80100b49a0 con 0x7f801015fc10 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f8016000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8010072c90 con 0x7f801015fff0 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f800f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f801015fc10 0x7f8010070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f800f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f801015fc10 0x7f8010070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33170/0 (socket says 172.21.15.86:33170) 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.795+0000 7f800f400640 1 -- 172.21.15.86:0/3437305937 learned_addr learned my addr 172.21.15.86:0/3437305937 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:25.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.796+0000 7f800f400640 1 -- 172.21.15.86:0/3437305937 >> v2:172.21.15.194:3300/0 conn(0x7f801015fff0 msgr2=0x7f8010160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.796+0000 7f800f400640 1 --2- 172.21.15.86:0/3437305937 >> v2:172.21.15.194:3300/0 conn(0x7f801015fff0 0x7f8010160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.796+0000 7f800f400640 1 -- 172.21.15.86:0/3437305937 >> v1:172.21.15.194:6789/0 conn(0x7f80100ce4a0 legacy=0x7f80100ce8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.796+0000 7f800f400640 1 -- 172.21.15.86:0/3437305937 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8010160990 con 0x7f801015fc10 2024-03-06T07:31:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.797+0000 7f800f400640 1 --2- 172.21.15.86:0/3437305937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f801015fc10 0x7f8010070ff0 secure :-1 s=READY pgs=904 cs=0 l=1 rev1=1 crypto rx=0x7f8010074a50 tx=0x7f800002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d134fb2f95dd5150 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.797+0000 7f800ea00640 1 -- 172.21.15.86:0/3437305937 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8000035020 con 0x7f801015fc10 2024-03-06T07:31:25.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.797+0000 7f800ea00640 1 -- 172.21.15.86:0/3437305937 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8000031870 con 0x7f801015fc10 2024-03-06T07:31:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.797+0000 7f800ea00640 1 -- 172.21.15.86:0/3437305937 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8000031c40 con 0x7f801015fc10 2024-03-06T07:31:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 -- 172.21.15.86:0/3437305937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f801015fc10 msgr2=0x7f8010070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 --2- 172.21.15.86:0/3437305937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f801015fc10 0x7f8010070ff0 secure :-1 s=READY pgs=904 cs=0 l=1 rev1=1 crypto rx=0x7f8010074a50 tx=0x7f800002b910 comp rx=0 tx=0).stop 2024-03-06T07:31:25.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 -- 172.21.15.86:0/3437305937 shutdown_connections 2024-03-06T07:31:25.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 --2- 172.21.15.86:0/3437305937 >> v2:172.21.15.194:3300/0 conn(0x7f801015fff0 0x7f8010160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 --2- 172.21.15.86:0/3437305937 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f801015fc10 0x7f8010070ff0 unknown :-1 s=CLOSED pgs=904 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 -- 172.21.15.86:0/3437305937 >> 172.21.15.86:0/3437305937 conn(0x7f80100b1940 msgr2=0x7f80100b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:25.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 -- 172.21.15.86:0/3437305937 shutdown_connections 2024-03-06T07:31:25.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.798+0000 7f8016000640 1 -- 172.21.15.86:0/3437305937 wait complete. 2024-03-06T07:31:25.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.800+0000 7f8016000640 1 Processor -- start 2024-03-06T07:31:25.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.800+0000 7f8016000640 1 -- start start 2024-03-06T07:31:25.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f8016000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80100ce4a0 0x7f8010141480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:25.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f8016000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f801015fc10 0x7f80100add20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:25.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f8016000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f801015fff0 0x7f80100b08d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:25.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f8016000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8010072a30 con 0x7f80100ce4a0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f8016000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80100728b0 con 0x7f801015fc10 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f8016000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f801015c2e0 con 0x7f801015fff0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80100ce4a0 0x7f8010141480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80100ce4a0 0x7f8010141480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33174/0 (socket says 172.21.15.86:33174) 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 -- 172.21.15.86:0/3646700055 learned_addr learned my addr 172.21.15.86:0/3646700055 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 -- 172.21.15.86:0/3646700055 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f801015fff0 msgr2=0x7f80100b08d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f801015fff0 0x7f80100b08d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 -- 172.21.15.86:0/3646700055 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f801015fc10 msgr2=0x7f80100add20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f801015fc10 0x7f80100add20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.801+0000 7f800f400640 1 -- 172.21.15.86:0/3646700055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80101419c0 con 0x7f80100ce4a0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.802+0000 7f800f400640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80100ce4a0 0x7f8010141480 secure :-1 s=READY pgs=905 cs=0 l=1 rev1=1 crypto rx=0x7f8000000c00 tx=0x7f80000046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.802+0000 7f8007e00640 1 -- 172.21.15.86:0/3646700055 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f800003ad60 con 0x7f80100ce4a0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.802+0000 7f8007e00640 1 -- 172.21.15.86:0/3646700055 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f80000443f0 con 0x7f80100ce4a0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.802+0000 7f8007e00640 1 -- 172.21.15.86:0/3646700055 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80000447e0 con 0x7f80100ce4a0 2024-03-06T07:31:25.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.802+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f801015d1c0 con 0x7f80100ce4a0 2024-03-06T07:31:25.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.802+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f801015d780 con 0x7f80100ce4a0 2024-03-06T07:31:25.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.803+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f801004f330 con 0x7f80100ce4a0 2024-03-06T07:31:25.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.804+0000 7f8007e00640 1 -- 172.21.15.86:0/3646700055 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8000038020 con 0x7f80100ce4a0 2024-03-06T07:31:25.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.805+0000 7f8007e00640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fec06b4d0 0x7f7fec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:25.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.805+0000 7f8007e00640 1 -- 172.21.15.86:0/3646700055 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f80000ac250 con 0x7f80100ce4a0 2024-03-06T07:31:25.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.805+0000 7f8006a00640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fec06b4d0 0x7f7fec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:25.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.806+0000 7f8006a00640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fec06b4d0 0x7f7fec06d990 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f80100b1420 tx=0x7f7ff8006bf0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:25.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:25.810+0000 7f8007e00640 1 -- 172.21.15.86:0/3646700055 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f80101419c0 con 0x7f80100ce4a0 2024-03-06T07:31:26.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.007+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f801006ef30 con 0x7f80100ce4a0 2024-03-06T07:31:26.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.012+0000 7f8007e00640 1 -- 172.21.15.86:0/3646700055 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v26)=0 v26) v1 ==== 122+0+0 (secure 0 0 0) 0x7f800003f070 con 0x7f80100ce4a0 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fec06b4d0 msgr2=0x7f7fec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fec06b4d0 0x7f7fec06d990 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f80100b1420 tx=0x7f7ff8006bf0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80100ce4a0 msgr2=0x7f8010141480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80100ce4a0 0x7f8010141480 secure :-1 s=READY pgs=905 cs=0 l=1 rev1=1 crypto rx=0x7f8000000c00 tx=0x7f80000046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 shutdown_connections 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7fec06b4d0 0x7f7fec06d990 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f801015fff0 0x7f80100b08d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f801015fc10 0x7f80100add20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 --2- 172.21.15.86:0/3646700055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f80100ce4a0 0x7f8010141480 unknown :-1 s=CLOSED pgs=905 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 >> 172.21.15.86:0/3646700055 conn(0x7f80100b1940 msgr2=0x7f8010151370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:26.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 shutdown_connections 2024-03-06T07:31:26.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.015+0000 7f8016000640 1 -- 172.21.15.86:0/3646700055 wait complete. 2024-03-06T07:31:26.035 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:31:26.035 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:26.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:26.279 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:31:26.279 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:26.279 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:26.335 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:31:26.335 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:26.335 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:26.391 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:26.392 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:26.392 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:26.450 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:31:26.450 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:26.450 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:26.506 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:26.506 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:31:26.506 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:26.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.717+0000 7f8e2f200640 1 Processor -- start 2024-03-06T07:31:26.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.717+0000 7f8e2f200640 1 -- start start 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.718+0000 7f8e2f200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8e2815a940 0x7f8e280b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.718+0000 7f8e2f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8e280cf370 0x7f8e2806d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.718+0000 7f8e2f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cc2a0 0x7f8e280cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2f200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e280b28e0 con 0x7f8e280cf370 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2f200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e280b2b30 con 0x7f8e2815a940 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e280b2e10 con 0x7f8e280cc2a0 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8e280cf370 0x7f8e2806d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8e280cf370 0x7f8e2806d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42080/0 (socket says 172.21.15.86:42080) 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 -- 172.21.15.86:0/875647177 learned_addr learned my addr 172.21.15.86:0/875647177 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 -- 172.21.15.86:0/875647177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cc2a0 msgr2=0x7f8e280cc750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 --2- 172.21.15.86:0/875647177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cc2a0 0x7f8e280cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 -- 172.21.15.86:0/875647177 >> v2:172.21.15.194:3300/0 conn(0x7f8e2815a940 msgr2=0x7f8e280b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 --2- 172.21.15.86:0/875647177 >> v2:172.21.15.194:3300/0 conn(0x7f8e2815a940 0x7f8e280b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.719+0000 7f8e2d400640 1 -- 172.21.15.86:0/875647177 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e2806ca50 con 0x7f8e280cf370 2024-03-06T07:31:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.720+0000 7f8e2d400640 1 --2- 172.21.15.86:0/875647177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cf370 0x7f8e2806d250 secure :-1 s=READY pgs=906 cs=0 l=1 rev1=1 crypto rx=0x7f8e28068510 tx=0x7f8e1802db10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6da8744fe63847da server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2ca00640 1 -- 172.21.15.86:0/875647177 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e1802f8d0 con 0x7f8e280cf370 2024-03-06T07:31:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2ca00640 1 -- 172.21.15.86:0/875647177 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e1802fa70 con 0x7f8e280cf370 2024-03-06T07:31:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2ca00640 1 -- 172.21.15.86:0/875647177 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e1802fe40 con 0x7f8e280cf370 2024-03-06T07:31:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2f200640 1 -- 172.21.15.86:0/875647177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cf370 msgr2=0x7f8e2806d250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2f200640 1 --2- 172.21.15.86:0/875647177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cf370 0x7f8e2806d250 secure :-1 s=READY pgs=906 cs=0 l=1 rev1=1 crypto rx=0x7f8e28068510 tx=0x7f8e1802db10 comp rx=0 tx=0).stop 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2f200640 1 -- 172.21.15.86:0/875647177 shutdown_connections 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2f200640 1 --2- 172.21.15.86:0/875647177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cc2a0 0x7f8e280cc750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2f200640 1 --2- 172.21.15.86:0/875647177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cf370 0x7f8e2806d250 unknown :-1 s=CLOSED pgs=906 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2f200640 1 --2- 172.21.15.86:0/875647177 >> v2:172.21.15.194:3300/0 conn(0x7f8e2815a940 0x7f8e280b45e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.721+0000 7f8e2f200640 1 -- 172.21.15.86:0/875647177 >> 172.21.15.86:0/875647177 conn(0x7f8e2806afc0 msgr2=0x7f8e2806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.722+0000 7f8e2f200640 1 -- 172.21.15.86:0/875647177 shutdown_connections 2024-03-06T07:31:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.722+0000 7f8e2f200640 1 -- 172.21.15.86:0/875647177 wait complete. 2024-03-06T07:31:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.723+0000 7f8e2f200640 1 Processor -- start 2024-03-06T07:31:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.723+0000 7f8e2f200640 1 -- start start 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.724+0000 7f8e2f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cc2a0 0x7f8e28155c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.724+0000 7f8e2f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cf370 0x7f8e28144a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.724+0000 7f8e2f200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e2815a940 0x7f8e28156cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.724+0000 7f8e2f200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e280b4b20 con 0x7f8e280cc2a0 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.724+0000 7f8e2f200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e280b30f0 con 0x7f8e2815a940 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.724+0000 7f8e2f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e2806d790 con 0x7f8e280cf370 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cc2a0 0x7f8e28155c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cc2a0 0x7f8e28155c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42092/0 (socket says 172.21.15.86:42092) 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 -- 172.21.15.86:0/3281848253 learned_addr learned my addr 172.21.15.86:0/3281848253 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2d400640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cf370 0x7f8e28144a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2e800640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e2815a940 0x7f8e28156cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 -- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cf370 msgr2=0x7f8e28144a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:26.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cf370 0x7f8e28144a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 -- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e2815a940 msgr2=0x7f8e28156cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:26.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e2815a940 0x7f8e28156cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 -- 172.21.15.86:0/3281848253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e28154c30 con 0x7f8e280cc2a0 2024-03-06T07:31:26.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2e800640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e2815a940 0x7f8e28156cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:26.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.725+0000 7f8e2de00640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cc2a0 0x7f8e28155c90 secure :-1 s=READY pgs=907 cs=0 l=1 rev1=1 crypto rx=0x7f8e1c00b440 tx=0x7f8e1c00b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:26.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.726+0000 7f8e22000640 1 -- 172.21.15.86:0/3281848253 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e1c015020 con 0x7f8e280cc2a0 2024-03-06T07:31:26.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.726+0000 7f8e22000640 1 -- 172.21.15.86:0/3281848253 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e1c011890 con 0x7f8e280cc2a0 2024-03-06T07:31:26.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.726+0000 7f8e22000640 1 -- 172.21.15.86:0/3281848253 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e1c011c80 con 0x7f8e280cc2a0 2024-03-06T07:31:26.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.726+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e28146060 con 0x7f8e280cc2a0 2024-03-06T07:31:26.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.726+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e281551b0 con 0x7f8e280cc2a0 2024-03-06T07:31:26.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.727+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e2804f330 con 0x7f8e280cc2a0 2024-03-06T07:31:26.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.728+0000 7f8e22000640 1 -- 172.21.15.86:0/3281848253 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8e1c018020 con 0x7f8e280cc2a0 2024-03-06T07:31:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.729+0000 7f8e22000640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8e0c06b500 0x7f8e0c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.729+0000 7f8e22000640 1 -- 172.21.15.86:0/3281848253 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f8e1c089f30 con 0x7f8e280cc2a0 2024-03-06T07:31:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.729+0000 7f8e2d400640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8e0c06b500 0x7f8e0c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.730+0000 7f8e2d400640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8e0c06b500 0x7f8e0c06d9c0 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f8e18038000 tx=0x7f8e18008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:26.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.733+0000 7f8e22000640 1 -- 172.21.15.86:0/3281848253 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8e1c01f020 con 0x7f8e280cc2a0 2024-03-06T07:31:26.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.971+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8e2806e180 con 0x7f8e280cc2a0 2024-03-06T07:31:26.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.972+0000 7f8e22000640 1 -- 172.21.15.86:0/3281848253 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 44 v44) v1 ==== 94+0+3699 (secure 0 0 0) 0x7f8e1c00d040 con 0x7f8e280cc2a0 2024-03-06T07:31:26.974 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:26.974 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":44,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5509,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/569595347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":569595347},{"type":"v1","addr":"172.21.15.194:6834","nonce":569595347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":5512,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/890462762","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":890462762},{"type":"v1","addr":"172.21.15.194:6835","nonce":890462762}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6201,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/1093854788","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1093854788},{"type":"v1","addr":"172.21.15.86:6836","nonce":1093854788}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":37},{"gid":6531,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/2465161951","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2465161951},{"type":"v1","addr":"172.21.15.86:6837","nonce":2465161951}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":44}],"filesystems":[]} 2024-03-06T07:31:26.974 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 44 2024-03-06T07:31:26.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8e0c06b500 msgr2=0x7f8e0c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:26.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8e0c06b500 0x7f8e0c06d9c0 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f8e18038000 tx=0x7f8e18008030 comp rx=0 tx=0).stop 2024-03-06T07:31:26.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cc2a0 msgr2=0x7f8e28155c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:26.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cc2a0 0x7f8e28155c90 secure :-1 s=READY pgs=907 cs=0 l=1 rev1=1 crypto rx=0x7f8e1c00b440 tx=0x7f8e1c00b900 comp rx=0 tx=0).stop 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 shutdown_connections 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8e0c06b500 0x7f8e0c06d9c0 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8e2815a940 0x7f8e28156cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8e280cf370 0x7f8e28144a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.974+0000 7f8e2f200640 1 --2- 172.21.15.86:0/3281848253 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8e280cc2a0 0x7f8e28155c90 unknown :-1 s=CLOSED pgs=907 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.975+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 >> 172.21.15.86:0/3281848253 conn(0x7f8e2806afc0 msgr2=0x7f8e280ae170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.975+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 shutdown_connections 2024-03-06T07:31:26.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:26.975+0000 7f8e2f200640 1 -- 172.21.15.86:0/3281848253 wait complete. 2024-03-06T07:31:26.995 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:31:26.995 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:31:26.996 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:31:26.996 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:26.996 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:31:26.996 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:31:26.996 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:31:26.996 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:26.997 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:31:26.997 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:31:26.997 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:31:26.997 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:26.997 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:31:26.997 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:31:26.997 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:31:26.997 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:27.014 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:31:27.014 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:31:27.019 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:31:27.019 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:31:27.020 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:31:27.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:31:27.021 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:31:27.021 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:31:27.066 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:31:27.066 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:27.068 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:31:27.068 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:27.072 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:31:27.072 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:27.073 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:31:27.073 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:27.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:31:27.215 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:31:27.216 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:31:27.225 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:31:27.226 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:31:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.247+0000 7f4432a00640 1 Processor -- start 2024-03-06T07:31:27.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.247+0000 7f4432a00640 1 -- start start 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.248+0000 7f4432a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f442c0d1cf0 0x7f442c0d4a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.248+0000 7f4432a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f442c0cf940 con 0x7f442c161ad0 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.248+0000 7f4432a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f442c0cfc20 con 0x7f442c0d2c00 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.248+0000 7f4432a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f442c0cff00 con 0x7f442c0d1cf0 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.248+0000 7f442be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f442c161ad0 0x7f442c0d24f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37930/0 (socket says 172.21.15.86:37930) 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.248+0000 7f442be00640 1 -- 172.21.15.86:0/2834763552 learned_addr learned my addr 172.21.15.86:0/2834763552 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f4430a00640 1 --2- 172.21.15.86:0/2834763552 >> v2:172.21.15.194:3300/0 conn(0x7f442c0d1cf0 0x7f442c0d4a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1902990658 0 0) 0x7f442c0cf940 con 0x7f442c161ad0 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4410003520 con 0x7f442c161ad0 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4138901965 0 0) 0x7f442c0cfc20 con 0x7f442c0d2c00 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f442c0cf940 con 0x7f442c0d2c00 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2714889792 0 0) 0x7f4410003520 con 0x7f442c161ad0 2024-03-06T07:31:27.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f4430a00640 1 -- 172.21.15.86:0/2834763552 >> v1:172.21.15.194:6789/0 conn(0x7f442c0d2c00 legacy=0x7f442c0b0db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f4430a00640 1 -- 172.21.15.86:0/2834763552 >> v1:172.21.15.86:6789/0 conn(0x7f442c161ad0 legacy=0x7f442c0d24f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.249+0000 7f4430a00640 1 -- 172.21.15.86:0/2834763552 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f442c0d01e0 con 0x7f442c0d1cf0 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.250+0000 7f4430a00640 1 --2- 172.21.15.86:0/2834763552 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c0d1cf0 0x7f442c0d4a30 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f4424009d90 tx=0x7f442402bd80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=94ec837613a5c669 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4424039070 con 0x7f442c0d1cf0 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f442402c8b0 con 0x7f442c0d1cf0 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f442aa00640 1 -- 172.21.15.86:0/2834763552 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f441c001670 con 0x7f442c0d1cf0 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f4432a00640 1 -- 172.21.15.86:0/2834763552 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c0d1cf0 msgr2=0x7f442c0d4a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f4432a00640 1 --2- 172.21.15.86:0/2834763552 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c0d1cf0 0x7f442c0d4a30 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f4424009d90 tx=0x7f442402bd80 comp rx=0 tx=0).stop 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f4432a00640 1 -- 172.21.15.86:0/2834763552 shutdown_connections 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f4432a00640 1 --2- 172.21.15.86:0/2834763552 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c0d1cf0 0x7f442c0d4a30 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f4432a00640 1 -- 172.21.15.86:0/2834763552 >> 172.21.15.86:0/2834763552 conn(0x7f442c0d3f10 msgr2=0x7f442c0d4320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:27.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f4432a00640 1 -- 172.21.15.86:0/2834763552 shutdown_connections 2024-03-06T07:31:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.251+0000 7f4432a00640 1 -- 172.21.15.86:0/2834763552 wait complete. 2024-03-06T07:31:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.253+0000 7f4432a00640 1 Processor -- start 2024-03-06T07:31:27.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.253+0000 7f4432a00640 1 -- start start 2024-03-06T07:31:27.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f4432a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f442c0d1cf0 0x7f442c0ccf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f4432a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f442c0d2c00 0x7f442c1f35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f4432a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c161ad0 0x7f442c0b83c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f4432a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f442c0b1850 con 0x7f442c0d2c00 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f4432a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f442c0b16d0 con 0x7f442c161ad0 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f4432a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f442c0b19d0 con 0x7f442c0d1cf0 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f442c0d2c00 0x7f442c1f35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f442c0d2c00 0x7f442c1f35e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42122/0 (socket says 172.21.15.86:42122) 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442b400640 1 -- 172.21.15.86:0/3745293697 learned_addr learned my addr 172.21.15.86:0/3745293697 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442be00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f442c0d1cf0 0x7f442c0ccf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f4430a00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c161ad0 0x7f442c0b83c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442b400640 1 -- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f442c0d1cf0 msgr2=0x7f442c0ccf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442b400640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f442c0d1cf0 0x7f442c0ccf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442b400640 1 -- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c161ad0 msgr2=0x7f442c0b83c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.254+0000 7f442b400640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c161ad0 0x7f442c0b83c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.255+0000 7f442b400640 1 -- 172.21.15.86:0/3745293697 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f442c0cc160 con 0x7f442c0d2c00 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.255+0000 7f442b400640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f442c0d2c00 0x7f442c1f35e0 secure :-1 s=READY pgs=917 cs=0 l=1 rev1=1 crypto rx=0x7f441800c530 tx=0x7f441800c9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.255+0000 7f4423e00640 1 -- 172.21.15.86:0/3745293697 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44180052d0 con 0x7f442c0d2c00 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.255+0000 7f4423e00640 1 -- 172.21.15.86:0/3745293697 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4418005490 con 0x7f442c0d2c00 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.255+0000 7f4423e00640 1 -- 172.21.15.86:0/3745293697 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4418012870 con 0x7f442c0d2c00 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.256+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f442c0cc440 con 0x7f442c0d2c00 2024-03-06T07:31:27.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.256+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f442c0cca00 con 0x7f442c0d2c00 2024-03-06T07:31:27.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.257+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f442c04f330 con 0x7f442c0d2c00 2024-03-06T07:31:27.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.257+0000 7f4423e00640 1 -- 172.21.15.86:0/3745293697 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4418012a10 con 0x7f442c0d2c00 2024-03-06T07:31:27.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.258+0000 7f4423e00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f440c06b4d0 0x7f440c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.258+0000 7f4423e00640 1 -- 172.21.15.86:0/3745293697 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f4418089b10 con 0x7f442c0d2c00 2024-03-06T07:31:27.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.258+0000 7f442be00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f440c06b4d0 0x7f440c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.259+0000 7f442be00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f440c06b4d0 0x7f440c06d990 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f441c0011e0 tx=0x7f441c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:27.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.263+0000 7f4423e00640 1 -- 172.21.15.86:0/3745293697 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4418058c80 con 0x7f442c0d2c00 2024-03-06T07:31:27.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.468+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f442c0d22d0 con 0x7f442c0d2c00 2024-03-06T07:31:27.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.581+0000 7f4423e00640 1 -- 172.21.15.86:0/3745293697 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v77) v1 ==== 97+0+0 (secure 0 0 0) 0x7f4418058580 con 0x7f442c0d2c00 2024-03-06T07:31:27.583 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:31:27.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f440c06b4d0 msgr2=0x7f440c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f440c06b4d0 0x7f440c06d990 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f441c0011e0 tx=0x7f441c009040 comp rx=0 tx=0).stop 2024-03-06T07:31:27.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f442c0d2c00 msgr2=0x7f442c1f35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f442c0d2c00 0x7f442c1f35e0 secure :-1 s=READY pgs=917 cs=0 l=1 rev1=1 crypto rx=0x7f441800c530 tx=0x7f441800c9f0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 shutdown_connections 2024-03-06T07:31:27.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f440c06b4d0 0x7f440c06d990 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f442c161ad0 0x7f442c0b83c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f442c0d2c00 0x7f442c1f35e0 unknown :-1 s=CLOSED pgs=917 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 --2- 172.21.15.86:0/3745293697 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f442c0d1cf0 0x7f442c0ccf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 >> 172.21.15.86:0/3745293697 conn(0x7f442c0d3f10 msgr2=0x7f442c0b0db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:27.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.586+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 shutdown_connections 2024-03-06T07:31:27.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.587+0000 7f4432a00640 1 -- 172.21.15.86:0/3745293697 wait complete. 2024-03-06T07:31:27.611 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:31:27.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.807+0000 7f4f9b000640 1 Processor -- start 2024-03-06T07:31:27.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.807+0000 7f4f9b000640 1 -- start start 2024-03-06T07:31:27.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f9b000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4f9415f9d0 0x7f4f9415fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f9b000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4f9406cef0 con 0x7f4f94144c00 2024-03-06T07:31:27.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f9b000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4f9406d1d0 con 0x7f4f94070b10 2024-03-06T07:31:27.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f9b000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f9406d4b0 con 0x7f4f9415f9d0 2024-03-06T07:31:27.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f8b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4f9415f9d0 0x7f4f9415fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f8b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4f9415f9d0 0x7f4f9415fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42126/0 (socket says 172.21.15.86:42126) 2024-03-06T07:31:27.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f8b400640 1 -- 172.21.15.86:0/3537388969 learned_addr learned my addr 172.21.15.86:0/3537388969 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:27.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f93e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4f94144c00 0x7f4f9406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37938/0 (socket says 172.21.15.86:37938) 2024-03-06T07:31:27.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.809+0000 7f4f8b400640 1 -- 172.21.15.86:0/3537388969 >> v1:172.21.15.194:6789/0 conn(0x7f4f94070b10 legacy=0x7f4f94070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:27.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.810+0000 7f4f8b400640 1 -- 172.21.15.86:0/3537388969 >> v1:172.21.15.86:6789/0 conn(0x7f4f94144c00 legacy=0x7f4f9406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.810+0000 7f4f8b400640 1 -- 172.21.15.86:0/3537388969 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f9406d790 con 0x7f4f9415f9d0 2024-03-06T07:31:27.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.810+0000 7f4f93400640 1 -- 172.21.15.86:0/3537388969 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 451277723 0 0) 0x7f4f9406cef0 con 0x7f4f94144c00 2024-03-06T07:31:27.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.811+0000 7f4f8b400640 1 --2- 172.21.15.86:0/3537388969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415fe60 secure :-1 s=READY pgs=919 cs=0 l=1 rev1=1 crypto rx=0x7f4f7c009870 tx=0x7f4f7c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da2927fd1f5fb80c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:27.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.811+0000 7f4f93400640 1 -- 172.21.15.86:0/3537388969 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f7c034070 con 0x7f4f9415f9d0 2024-03-06T07:31:27.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.811+0000 7f4f93400640 1 -- 172.21.15.86:0/3537388969 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f7c030860 con 0x7f4f9415f9d0 2024-03-06T07:31:27.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.811+0000 7f4f93400640 1 -- 172.21.15.86:0/3537388969 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f7c030c30 con 0x7f4f9415f9d0 2024-03-06T07:31:27.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.812+0000 7f4f9b000640 1 -- 172.21.15.86:0/3537388969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 msgr2=0x7f4f9415fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.812+0000 7f4f9b000640 1 --2- 172.21.15.86:0/3537388969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415fe60 secure :-1 s=READY pgs=919 cs=0 l=1 rev1=1 crypto rx=0x7f4f7c009870 tx=0x7f4f7c02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.812+0000 7f4f9b000640 1 -- 172.21.15.86:0/3537388969 shutdown_connections 2024-03-06T07:31:27.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.812+0000 7f4f9b000640 1 --2- 172.21.15.86:0/3537388969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415fe60 unknown :-1 s=CLOSED pgs=919 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.812+0000 7f4f9b000640 1 -- 172.21.15.86:0/3537388969 >> 172.21.15.86:0/3537388969 conn(0x7f4f940b1730 msgr2=0x7f4f940b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:27.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.812+0000 7f4f9b000640 1 -- 172.21.15.86:0/3537388969 shutdown_connections 2024-03-06T07:31:27.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.812+0000 7f4f9b000640 1 -- 172.21.15.86:0/3537388969 wait complete. 2024-03-06T07:31:27.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.814+0000 7f4f9b000640 1 Processor -- start 2024-03-06T07:31:27.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.814+0000 7f4f9b000640 1 -- start start 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f9b000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f94070b10 0x7f4f9413c410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f9b000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f94144c00 0x7f4f941e7ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f9b000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415f040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f9b000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f9406ccd0 con 0x7f4f9415f9d0 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f9b000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f9406cb50 con 0x7f4f94070b10 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f9b000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f9406df90 con 0x7f4f94144c00 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f99000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415f040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f99000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415f040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42138/0 (socket says 172.21.15.86:42138) 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f99000640 1 -- 172.21.15.86:0/2874928403 learned_addr learned my addr 172.21.15.86:0/2874928403 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:27.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.815+0000 7f4f93e00640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f94070b10 0x7f4f9413c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f99000640 1 -- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f94144c00 msgr2=0x7f4f941e7ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f8b400640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f94144c00 0x7f4f941e7ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f99000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f94144c00 0x7f4f941e7ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f99000640 1 -- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f94070b10 msgr2=0x7f4f9413c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f99000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f94070b10 0x7f4f9413c410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f99000640 1 -- 172.21.15.86:0/2874928403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f9413c950 con 0x7f4f9415f9d0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f93e00640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f94070b10 0x7f4f9413c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f99000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415f040 secure :-1 s=READY pgs=920 cs=0 l=1 rev1=1 crypto rx=0x7f4f8c00c780 tx=0x7f4f8c00cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f90c00640 1 -- 172.21.15.86:0/2874928403 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f8c004b60 con 0x7f4f9415f9d0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f90c00640 1 -- 172.21.15.86:0/2874928403 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f8c004d00 con 0x7f4f9415f9d0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f90c00640 1 -- 172.21.15.86:0/2874928403 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f8c00aa90 con 0x7f4f9415f9d0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f941e6770 con 0x7f4f9415f9d0 2024-03-06T07:31:27.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.816+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f941e6cf0 con 0x7f4f9415f9d0 2024-03-06T07:31:27.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.818+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f9404f330 con 0x7f4f9415f9d0 2024-03-06T07:31:27.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.818+0000 7f4f90c00640 1 -- 172.21.15.86:0/2874928403 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4f8c00ac30 con 0x7f4f9415f9d0 2024-03-06T07:31:27.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.819+0000 7f4f90c00640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f7006b4d0 0x7f4f7006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:27.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.819+0000 7f4f90c00640 1 -- 172.21.15.86:0/2874928403 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f4f8c08a1b0 con 0x7f4f9415f9d0 2024-03-06T07:31:27.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.820+0000 7f4f93e00640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f7006b4d0 0x7f4f7006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:27.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.820+0000 7f4f93e00640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f7006b4d0 0x7f4f7006d990 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f4f84000f60 tx=0x7f4f84009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:27.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:27.824+0000 7f4f90c00640 1 -- 172.21.15.86:0/2874928403 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4f8c020070 con 0x7f4f9415f9d0 2024-03-06T07:31:28.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.089+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f4f94072890 con 0x7f4f9415f9d0 2024-03-06T07:31:28.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.090+0000 7f4f90c00640 1 -- 172.21.15.86:0/2874928403 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v16) v1 ==== 80+0+5304 (secure 0 0 0) 0x7f4f9413c950 con 0x7f4f9415f9d0 2024-03-06T07:31:28.092 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:28.092 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:31:28.092 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:31:28.092 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.093 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.094 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.095 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.096 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.097 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:31:28.098 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.099 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:31:28.100 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow r", 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.101 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:28.102 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.103 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:31:28.104 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:28.105 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.105 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:28.105 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:31:28.105 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:31:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.092+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f7006b4d0 msgr2=0x7f4f7006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.092+0000 7f4f9b000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f7006b4d0 0x7f4f7006d990 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f4f84000f60 tx=0x7f4f84009040 comp rx=0 tx=0).stop 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.092+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 msgr2=0x7f4f9415f040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.092+0000 7f4f9b000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415f040 secure :-1 s=READY pgs=920 cs=0 l=1 rev1=1 crypto rx=0x7f4f8c00c780 tx=0x7f4f8c00cc40 comp rx=0 tx=0).stop 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 shutdown_connections 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f7006b4d0 0x7f4f7006d990 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4f9415f9d0 0x7f4f9415f040 unknown :-1 s=CLOSED pgs=920 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4f94144c00 0x7f4f941e7ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 --2- 172.21.15.86:0/2874928403 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4f94070b10 0x7f4f9413c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 >> 172.21.15.86:0/2874928403 conn(0x7f4f940b1730 msgr2=0x7f4f94145040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 shutdown_connections 2024-03-06T07:31:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.093+0000 7f4f9b000640 1 -- 172.21.15.86:0/2874928403 wait complete. 2024-03-06T07:31:28.111 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:28.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.315+0000 7f5996600640 1 Processor -- start 2024-03-06T07:31:28.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.315+0000 7f5996600640 1 -- start start 2024-03-06T07:31:28.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.316+0000 7f5996600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f59900b4680 0x7f599006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.316+0000 7f5996600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5990072340 0x7f599006ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5996600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f599015b090 con 0x7f599015acb0 2024-03-06T07:31:28.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5996600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f59900b2a30 con 0x7f59900b4680 2024-03-06T07:31:28.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5996600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f59900b4360 con 0x7f5990072340 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5986a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f59900b4680 0x7f599006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5986a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f59900b4680 0x7f599006e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42144/0 (socket says 172.21.15.86:42144) 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5986a00640 1 -- 172.21.15.86:0/731301184 learned_addr learned my addr 172.21.15.86:0/731301184 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f598f400640 1 --1- 172.21.15.86:0/731301184 >> v1:172.21.15.86:6789/0 conn(0x7f599015acb0 0x7f59900b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37950/0 (socket says 172.21.15.86:37950) 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5986a00640 1 -- 172.21.15.86:0/731301184 >> v2:172.21.15.194:3300/0 conn(0x7f5990072340 msgr2=0x7f599006ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5986a00640 1 --2- 172.21.15.86:0/731301184 >> v2:172.21.15.194:3300/0 conn(0x7f5990072340 0x7f599006ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5986a00640 1 -- 172.21.15.86:0/731301184 >> v1:172.21.15.86:6789/0 conn(0x7f599015acb0 legacy=0x7f59900b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f5986a00640 1 -- 172.21.15.86:0/731301184 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59900b2cf0 con 0x7f59900b4680 2024-03-06T07:31:28.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.317+0000 7f598ea00640 1 -- 172.21.15.86:0/731301184 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1630246064 0 0) 0x7f599015b090 con 0x7f599015acb0 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.318+0000 7f5986a00640 1 --2- 172.21.15.86:0/731301184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f59900b4680 0x7f599006e4f0 secure :-1 s=READY pgs=922 cs=0 l=1 rev1=1 crypto rx=0x7f59900b4e60 tx=0x7f597802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0c30262229a3cf4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f598ea00640 1 -- 172.21.15.86:0/731301184 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5978034070 con 0x7f59900b4680 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f598ea00640 1 -- 172.21.15.86:0/731301184 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5978030860 con 0x7f59900b4680 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f598ea00640 1 -- 172.21.15.86:0/731301184 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5978030c30 con 0x7f59900b4680 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 -- 172.21.15.86:0/731301184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f59900b4680 msgr2=0x7f599006e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 --2- 172.21.15.86:0/731301184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f59900b4680 0x7f599006e4f0 secure :-1 s=READY pgs=922 cs=0 l=1 rev1=1 crypto rx=0x7f59900b4e60 tx=0x7f597802b9a0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 -- 172.21.15.86:0/731301184 shutdown_connections 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 --2- 172.21.15.86:0/731301184 >> v2:172.21.15.194:3300/0 conn(0x7f5990072340 0x7f599006ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 --2- 172.21.15.86:0/731301184 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f59900b4680 0x7f599006e4f0 unknown :-1 s=CLOSED pgs=922 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 -- 172.21.15.86:0/731301184 >> 172.21.15.86:0/731301184 conn(0x7f59900b1730 msgr2=0x7f59900b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:28.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 -- 172.21.15.86:0/731301184 shutdown_connections 2024-03-06T07:31:28.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.319+0000 7f5996600640 1 -- 172.21.15.86:0/731301184 wait complete. 2024-03-06T07:31:28.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.321+0000 7f5996600640 1 Processor -- start 2024-03-06T07:31:28.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.321+0000 7f5996600640 1 -- start start 2024-03-06T07:31:28.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f5996600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990072340 0x7f5990158430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f5996600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f59900b4680 0x7f59901475c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f5996600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f599015acb0 0x7f59900cd8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f5996600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59900b4010 con 0x7f5990072340 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f5996600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59900b32b0 con 0x7f599015acb0 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f5996600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f59900ccfb0 con 0x7f59900b4680 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f598f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990072340 0x7f5990158430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f598f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990072340 0x7f5990158430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42150/0 (socket says 172.21.15.86:42150) 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.322+0000 7f598f400640 1 -- 172.21.15.86:0/2532859840 learned_addr learned my addr 172.21.15.86:0/2532859840 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f598f400640 1 -- 172.21.15.86:0/2532859840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f59900b4680 msgr2=0x7f59901475c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f598f400640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f59900b4680 0x7f59901475c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f598fe00640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f599015acb0 0x7f59900cd8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f598f400640 1 -- 172.21.15.86:0/2532859840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f599015acb0 msgr2=0x7f59900cd8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f598f400640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f599015acb0 0x7f59900cd8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f598f400640 1 -- 172.21.15.86:0/2532859840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59900ce040 con 0x7f5990072340 2024-03-06T07:31:28.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f598f400640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990072340 0x7f5990158430 secure :-1 s=READY pgs=923 cs=0 l=1 rev1=1 crypto rx=0x7f598000c430 tx=0x7f598000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:28.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f5987e00640 1 -- 172.21.15.86:0/2532859840 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f59800074e0 con 0x7f5990072340 2024-03-06T07:31:28.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f5987e00640 1 -- 172.21.15.86:0/2532859840 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f59800047e0 con 0x7f5990072340 2024-03-06T07:31:28.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f5987e00640 1 -- 172.21.15.86:0/2532859840 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5980004bd0 con 0x7f5990072340 2024-03-06T07:31:28.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f59900ce320 con 0x7f5990072340 2024-03-06T07:31:28.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.323+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f59900ce8e0 con 0x7f5990072340 2024-03-06T07:31:28.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.324+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f599004f330 con 0x7f5990072340 2024-03-06T07:31:28.326 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.325+0000 7f5987e00640 1 -- 172.21.15.86:0/2532859840 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5980011070 con 0x7f5990072340 2024-03-06T07:31:28.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.326+0000 7f5987e00640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f596c06b4d0 0x7f596c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.326+0000 7f5987e00640 1 -- 172.21.15.86:0/2532859840 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f5980089b40 con 0x7f5990072340 2024-03-06T07:31:28.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.326+0000 7f5986a00640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f596c06b4d0 0x7f596c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:28.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.327+0000 7f5986a00640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f596c06b4d0 0x7f596c06d990 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f5978036000 tx=0x7f597803b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:28.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.331+0000 7f5987e00640 1 -- 172.21.15.86:0/2532859840 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5980058d00 con 0x7f5990072340 2024-03-06T07:31:28.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.525+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f599006ce60 con 0x7f5990072340 2024-03-06T07:31:28.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.527+0000 7f5987e00640 1 -- 172.21.15.86:0/2532859840 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v79) v1 ==== 74+0+12904 (secure 0 0 0) 0x7f5980058600 con 0x7f5990072340 2024-03-06T07:31:28.529 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:28.529 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":79,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:27.592152+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.529+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f596c06b4d0 msgr2=0x7f596c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.529+0000 7f5996600640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f596c06b4d0 0x7f596c06d990 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f5978036000 tx=0x7f597803b040 comp rx=0 tx=0).stop 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990072340 msgr2=0x7f5990158430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990072340 0x7f5990158430 secure :-1 s=READY pgs=923 cs=0 l=1 rev1=1 crypto rx=0x7f598000c430 tx=0x7f598000c8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 shutdown_connections 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f596c06b4d0 0x7f596c06d990 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f599015acb0 0x7f59900cd8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f59900b4680 0x7f59901475c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 --2- 172.21.15.86:0/2532859840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990072340 0x7f5990158430 unknown :-1 s=CLOSED pgs=923 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 >> 172.21.15.86:0/2532859840 conn(0x7f59900b1730 msgr2=0x7f59901554b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 shutdown_connections 2024-03-06T07:31:28.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.530+0000 7f5996600640 1 -- 172.21.15.86:0/2532859840 wait complete. 2024-03-06T07:31:28.549 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:31:28.549 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:31:28.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.742+0000 7f70c4a00640 1 Processor -- start 2024-03-06T07:31:28.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.742+0000 7f70c4a00640 1 -- start start 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.743+0000 7f70c4a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f70c015e000 0x7f70c006f130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70c4a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e3e0 0x7f70c00d0ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70c4a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f70c00d1260 con 0x7f70c006f7e0 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70c4a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f70c00d1020 con 0x7f70c015e000 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70c4a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70c006c0d0 con 0x7f70c015e3e0 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70bde00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f70c015e000 0x7f70c006f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70bde00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f70c015e000 0x7f70c006f130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42160/0 (socket says 172.21.15.86:42160) 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70bde00640 1 -- 172.21.15.86:0/4046702466 learned_addr learned my addr 172.21.15.86:0/4046702466 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70bde00640 1 -- 172.21.15.86:0/4046702466 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e3e0 msgr2=0x7f70c00d0ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70bde00640 1 --2- 172.21.15.86:0/4046702466 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e3e0 0x7f70c00d0ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70bde00640 1 -- 172.21.15.86:0/4046702466 >> v1:172.21.15.194:6789/0 conn(0x7f70c006f7e0 legacy=0x7f70c00ce340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:28.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.744+0000 7f70bde00640 1 -- 172.21.15.86:0/4046702466 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70c006c320 con 0x7f70c015e000 2024-03-06T07:31:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.745+0000 7f70bde00640 1 --2- 172.21.15.86:0/4046702466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e000 0x7f70c006f130 secure :-1 s=READY pgs=924 cs=0 l=1 rev1=1 crypto rx=0x7f70b0009870 tx=0x7f70b002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be989dc1b0dab0c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70bca00640 1 -- 172.21.15.86:0/4046702466 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70b0035020 con 0x7f70c015e000 2024-03-06T07:31:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70bca00640 1 -- 172.21.15.86:0/4046702466 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f70b0031640 con 0x7f70c015e000 2024-03-06T07:31:28.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70bca00640 1 -- 172.21.15.86:0/4046702466 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70b002d040 con 0x7f70c015e000 2024-03-06T07:31:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 -- 172.21.15.86:0/4046702466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e000 msgr2=0x7f70c006f130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 --2- 172.21.15.86:0/4046702466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e000 0x7f70c006f130 secure :-1 s=READY pgs=924 cs=0 l=1 rev1=1 crypto rx=0x7f70b0009870 tx=0x7f70b002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 -- 172.21.15.86:0/4046702466 shutdown_connections 2024-03-06T07:31:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 --2- 172.21.15.86:0/4046702466 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e3e0 0x7f70c00d0ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 --2- 172.21.15.86:0/4046702466 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e000 0x7f70c006f130 unknown :-1 s=CLOSED pgs=924 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 -- 172.21.15.86:0/4046702466 >> 172.21.15.86:0/4046702466 conn(0x7f70c00b3800 msgr2=0x7f70c00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:28.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 -- 172.21.15.86:0/4046702466 shutdown_connections 2024-03-06T07:31:28.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.746+0000 7f70c4a00640 1 -- 172.21.15.86:0/4046702466 wait complete. 2024-03-06T07:31:28.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.748+0000 7f70c4a00640 1 Processor -- start 2024-03-06T07:31:28.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.748+0000 7f70c4a00640 1 -- start start 2024-03-06T07:31:28.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70c4a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70c006f7e0 0x7f70c015f7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70c4a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e000 0x7f70c0152270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70c4a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e3e0 0x7f70c00b2b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70c4a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70c006fdf0 con 0x7f70c015e3e0 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70c4a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70c006fc70 con 0x7f70c006f7e0 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70c4a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70c006ff70 con 0x7f70c015e000 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70be800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e3e0 0x7f70c00b2b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70be800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e3e0 0x7f70c00b2b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42172/0 (socket says 172.21.15.86:42172) 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.749+0000 7f70be800640 1 -- 172.21.15.86:0/773369508 learned_addr learned my addr 172.21.15.86:0/773369508 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70be800640 1 -- 172.21.15.86:0/773369508 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e000 msgr2=0x7f70c0152270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70be800640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e000 0x7f70c0152270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70be800640 1 -- 172.21.15.86:0/773369508 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70c006f7e0 msgr2=0x7f70c015f7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70be800640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70c006f7e0 0x7f70c015f7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70be800640 1 -- 172.21.15.86:0/773369508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70c00c8270 con 0x7f70c015e3e0 2024-03-06T07:31:28.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70be800640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e3e0 0x7f70c00b2b20 secure :-1 s=READY pgs=925 cs=0 l=1 rev1=1 crypto rx=0x7f70b800c390 tx=0x7f70b800c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:28.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70b6000640 1 -- 172.21.15.86:0/773369508 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70b8017070 con 0x7f70c015e3e0 2024-03-06T07:31:28.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70b6000640 1 -- 172.21.15.86:0/773369508 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f70b80035f0 con 0x7f70c015e3e0 2024-03-06T07:31:28.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70b6000640 1 -- 172.21.15.86:0/773369508 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70b80039e0 con 0x7f70c015e3e0 2024-03-06T07:31:28.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70c015e9f0 con 0x7f70c015e3e0 2024-03-06T07:31:28.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.750+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70c015efb0 con 0x7f70c015e3e0 2024-03-06T07:31:28.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.751+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f70c004f330 con 0x7f70c015e3e0 2024-03-06T07:31:28.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.752+0000 7f70b6000640 1 -- 172.21.15.86:0/773369508 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f70b8003b80 con 0x7f70c015e3e0 2024-03-06T07:31:28.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.753+0000 7f70b6000640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f709c06b4d0 0x7f709c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:28.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.753+0000 7f70b6000640 1 -- 172.21.15.86:0/773369508 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f70b80891c0 con 0x7f70c015e3e0 2024-03-06T07:31:28.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.753+0000 7f70bde00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f709c06b4d0 0x7f709c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:28.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.754+0000 7f70bde00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f709c06b4d0 0x7f709c06d990 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f70b0036000 tx=0x7f70b0030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:28.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.757+0000 7f70b6000640 1 -- 172.21.15.86:0/773369508 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f70c00c8270 con 0x7f70c015e3e0 2024-03-06T07:31:28.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.951+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f70c00b7c50 con 0x7f70c015e3e0 2024-03-06T07:31:28.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.952+0000 7f70b6000640 1 -- 172.21.15.86:0/773369508 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v79) v1 ==== 93+0+10 (secure 0 0 0) 0x7f70b80582f0 con 0x7f70c015e3e0 2024-03-06T07:31:28.954 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:31:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f709c06b4d0 msgr2=0x7f709c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f709c06b4d0 0x7f709c06d990 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f70b0036000 tx=0x7f70b0030000 comp rx=0 tx=0).stop 2024-03-06T07:31:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e3e0 msgr2=0x7f70c00b2b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e3e0 0x7f70c00b2b20 secure :-1 s=READY pgs=925 cs=0 l=1 rev1=1 crypto rx=0x7f70b800c390 tx=0x7f70b800c850 comp rx=0 tx=0).stop 2024-03-06T07:31:28.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 shutdown_connections 2024-03-06T07:31:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f709c06b4d0 0x7f709c06d990 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f70c015e3e0 0x7f70c00b2b20 unknown :-1 s=CLOSED pgs=925 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f70c015e000 0x7f70c0152270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 --2- 172.21.15.86:0/773369508 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f70c006f7e0 0x7f70c015f7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 >> 172.21.15.86:0/773369508 conn(0x7f70c00b3800 msgr2=0x7f70c00d0100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 shutdown_connections 2024-03-06T07:31:28.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:28.955+0000 7f70c4a00640 1 -- 172.21.15.86:0/773369508 wait complete. 2024-03-06T07:31:28.975 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:31:29.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.178+0000 7f9e2a600640 1 Processor -- start 2024-03-06T07:31:29.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.178+0000 7f9e2a600640 1 -- start start 2024-03-06T07:31:29.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.179+0000 7f9e2a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e2406e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.179+0000 7f9e2a600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9e24072340 0x7f9e2406ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.179+0000 7f9e2a600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9e2415b090 con 0x7f9e2415acb0 2024-03-06T07:31:29.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.179+0000 7f9e2a600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9e240b2a30 con 0x7f9e24072340 2024-03-06T07:31:29.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.179+0000 7f9e2a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e240b4360 con 0x7f9e240b4680 2024-03-06T07:31:29.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.180+0000 7f9e23e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9e24072340 0x7f9e2406ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.180+0000 7f9e1be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e2406e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.180+0000 7f9e23e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9e24072340 0x7f9e2406ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:59766/0 (socket says 172.21.15.86:59766) 2024-03-06T07:31:29.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.180+0000 7f9e23e00640 1 -- 172.21.15.86:0/2455200592 learned_addr learned my addr 172.21.15.86:0/2455200592 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:29.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.181+0000 7f9e1be00640 1 -- 172.21.15.86:0/2455200592 >> v2:172.21.15.194:3300/0 conn(0x7f9e24072340 msgr2=0x7f9e2406ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:29.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.181+0000 7f9e1be00640 1 --2- 172.21.15.86:0/2455200592 >> v2:172.21.15.194:3300/0 conn(0x7f9e24072340 0x7f9e2406ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.181+0000 7f9e1be00640 1 -- 172.21.15.86:0/2455200592 >> v1:172.21.15.194:6789/0 conn(0x7f9e2415acb0 legacy=0x7f9e240b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.181+0000 7f9e1be00640 1 -- 172.21.15.86:0/2455200592 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e240b2cf0 con 0x7f9e240b4680 2024-03-06T07:31:29.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.181+0000 7f9e23e00640 1 --2- 172.21.15.86:0/2455200592 >> v2:172.21.15.194:3300/0 conn(0x7f9e24072340 0x7f9e2406ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:31:29.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.182+0000 7f9e1be00640 1 --2- 172.21.15.86:0/2455200592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e2406e4f0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f9e0c0097a0 tx=0x7f9e0c02b8b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=604744c3cccb9603 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:29.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e22a00640 1 -- 172.21.15.86:0/2455200592 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e0c035020 con 0x7f9e240b4680 2024-03-06T07:31:29.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e22a00640 1 -- 172.21.15.86:0/2455200592 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e0c031810 con 0x7f9e240b4680 2024-03-06T07:31:29.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e22a00640 1 -- 172.21.15.86:0/2455200592 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e0c031be0 con 0x7f9e240b4680 2024-03-06T07:31:29.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 -- 172.21.15.86:0/2455200592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 msgr2=0x7f9e2406e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 --2- 172.21.15.86:0/2455200592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e2406e4f0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f9e0c0097a0 tx=0x7f9e0c02b8b0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 -- 172.21.15.86:0/2455200592 shutdown_connections 2024-03-06T07:31:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 --2- 172.21.15.86:0/2455200592 >> v2:172.21.15.194:3300/0 conn(0x7f9e24072340 0x7f9e2406ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 --2- 172.21.15.86:0/2455200592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e2406e4f0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 -- 172.21.15.86:0/2455200592 >> 172.21.15.86:0/2455200592 conn(0x7f9e240b1730 msgr2=0x7f9e240b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 -- 172.21.15.86:0/2455200592 shutdown_connections 2024-03-06T07:31:29.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.183+0000 7f9e2a600640 1 -- 172.21.15.86:0/2455200592 wait complete. 2024-03-06T07:31:29.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.185+0000 7f9e2a600640 1 Processor -- start 2024-03-06T07:31:29.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.185+0000 7f9e2a600640 1 -- start start 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e2a600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e24072340 0x7f9e24149120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e2a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e240aecd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e2a600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e2415acb0 0x7f9e24155590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e2a600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e240b4010 con 0x7f9e24072340 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e2a600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e240b32b0 con 0x7f9e2415acb0 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e2a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e240ccfb0 con 0x7f9e240b4680 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e23400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e24072340 0x7f9e24149120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e23400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e24072340 0x7f9e24149120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42182/0 (socket says 172.21.15.86:42182) 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.186+0000 7f9e23400640 1 -- 172.21.15.86:0/3850699763 learned_addr learned my addr 172.21.15.86:0/3850699763 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e23400640 1 -- 172.21.15.86:0/3850699763 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 msgr2=0x7f9e240aecd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e23400640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e240aecd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e23400640 1 -- 172.21.15.86:0/3850699763 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e2415acb0 msgr2=0x7f9e24155590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e23400640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e2415acb0 0x7f9e24155590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e23400640 1 -- 172.21.15.86:0/3850699763 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e240adab0 con 0x7f9e24072340 2024-03-06T07:31:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e23400640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e24072340 0x7f9e24149120 secure :-1 s=READY pgs=926 cs=0 l=1 rev1=1 crypto rx=0x7f9e1400c3c0 tx=0x7f9e1400c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e1b400640 1 -- 172.21.15.86:0/3850699763 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e140052d0 con 0x7f9e24072340 2024-03-06T07:31:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e1b400640 1 -- 172.21.15.86:0/3850699763 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e14005470 con 0x7f9e24072340 2024-03-06T07:31:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e1b400640 1 -- 172.21.15.86:0/3850699763 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e14007260 con 0x7f9e24072340 2024-03-06T07:31:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e240add90 con 0x7f9e24072340 2024-03-06T07:31:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.187+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e240ae350 con 0x7f9e24072340 2024-03-06T07:31:29.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.189+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9e2404f330 con 0x7f9e24072340 2024-03-06T07:31:29.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.189+0000 7f9e1b400640 1 -- 172.21.15.86:0/3850699763 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9e14007400 con 0x7f9e24072340 2024-03-06T07:31:29.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.190+0000 7f9e1b400640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e0006b4d0 0x7f9e0006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.190+0000 7f9e1b400640 1 -- 172.21.15.86:0/3850699763 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f9e140893a0 con 0x7f9e24072340 2024-03-06T07:31:29.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.191+0000 7f9e1be00640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e0006b4d0 0x7f9e0006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.191+0000 7f9e1be00640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e0006b4d0 0x7f9e0006d990 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f9e0c036000 tx=0x7f9e0c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:29.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.195+0000 7f9e1b400640 1 -- 172.21.15.86:0/3850699763 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9e140584d0 con 0x7f9e24072340 2024-03-06T07:31:29.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.391+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f9e2406ce60 con 0x7f9e24072340 2024-03-06T07:31:29.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.392+0000 7f9e1b400640 1 -- 172.21.15.86:0/3850699763 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v79) v1 ==== 84+0+65 (secure 0 0 0) 0x7f9e14057dd0 con 0x7f9e24072340 2024-03-06T07:31:29.394 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:29.394 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:31:29.394 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:29.394 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:31:29.394 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:31:29.394 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:29.394 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:31:29.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.394+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e0006b4d0 msgr2=0x7f9e0006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.394+0000 7f9e2a600640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e0006b4d0 0x7f9e0006d990 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f9e0c036000 tx=0x7f9e0c030000 comp rx=0 tx=0).stop 2024-03-06T07:31:29.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.394+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e24072340 msgr2=0x7f9e24149120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.394+0000 7f9e2a600640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e24072340 0x7f9e24149120 secure :-1 s=READY pgs=926 cs=0 l=1 rev1=1 crypto rx=0x7f9e1400c3c0 tx=0x7f9e1400c880 comp rx=0 tx=0).stop 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.394+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 shutdown_connections 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.395+0000 7f9e2a600640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9e0006b4d0 0x7f9e0006d990 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.395+0000 7f9e2a600640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9e2415acb0 0x7f9e24155590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.395+0000 7f9e2a600640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9e240b4680 0x7f9e240aecd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.395+0000 7f9e2a600640 1 --2- 172.21.15.86:0/3850699763 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9e24072340 0x7f9e24149120 unknown :-1 s=CLOSED pgs=926 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.395+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 >> 172.21.15.86:0/3850699763 conn(0x7f9e240b1730 msgr2=0x7f9e240b0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.395+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 shutdown_connections 2024-03-06T07:31:29.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.395+0000 7f9e2a600640 1 -- 172.21.15.86:0/3850699763 wait complete. 2024-03-06T07:31:29.415 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:31:29.415 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:31:29.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.629+0000 7ff0ed200640 1 Processor -- start 2024-03-06T07:31:29.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.629+0000 7ff0ed200640 1 -- start start 2024-03-06T07:31:29.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0ed200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff0e806e570 0x7ff0e80b4450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0ed200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0e80adf70 con 0x7ff0e815e170 2024-03-06T07:31:29.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0ed200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0e80b3690 con 0x7ff0e80b4990 2024-03-06T07:31:29.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0ed200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0e806ea70 con 0x7ff0e806e570 2024-03-06T07:31:29.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e5400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff0e806e570 0x7ff0e80b4450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e5e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff0e815e170 0x7ff0e80ad860 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37966/0 (socket says 172.21.15.86:37966) 2024-03-06T07:31:29.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e5e00640 1 -- 172.21.15.86:0/1892133186 learned_addr learned my addr 172.21.15.86:0/1892133186 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:29.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e4a00640 1 -- 172.21.15.86:0/1892133186 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2700963962 0 0) 0x7ff0e80adf70 con 0x7ff0e815e170 2024-03-06T07:31:29.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e4a00640 1 -- 172.21.15.86:0/1892133186 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff0cc003520 con 0x7ff0e815e170 2024-03-06T07:31:29.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e5400640 1 -- 172.21.15.86:0/1892133186 >> v1:172.21.15.194:6789/0 conn(0x7ff0e80b4990 legacy=0x7ff0e8161b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:29.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e5400640 1 -- 172.21.15.86:0/1892133186 >> v1:172.21.15.86:6789/0 conn(0x7ff0e815e170 legacy=0x7ff0e80ad860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.631+0000 7ff0e5400640 1 -- 172.21.15.86:0/1892133186 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0e80b3970 con 0x7ff0e806e570 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0e5400640 1 --2- 172.21.15.86:0/1892133186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e80b4450 secure :-1 s=READY pgs=928 cs=0 l=1 rev1=1 crypto rx=0x7ff0d40097a0 tx=0x7ff0d402b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f50a292408011de4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0e4a00640 1 -- 172.21.15.86:0/1892133186 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0d40315a0 con 0x7ff0e806e570 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0e4a00640 1 -- 172.21.15.86:0/1892133186 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0d4037070 con 0x7ff0e806e570 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0e4a00640 1 -- 172.21.15.86:0/1892133186 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0d4031940 con 0x7ff0e806e570 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0ed200640 1 -- 172.21.15.86:0/1892133186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 msgr2=0x7ff0e80b4450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1892133186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e80b4450 secure :-1 s=READY pgs=928 cs=0 l=1 rev1=1 crypto rx=0x7ff0d40097a0 tx=0x7ff0d402b840 comp rx=0 tx=0).stop 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0ed200640 1 -- 172.21.15.86:0/1892133186 shutdown_connections 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1892133186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e80b4450 unknown :-1 s=CLOSED pgs=928 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.633+0000 7ff0ed200640 1 -- 172.21.15.86:0/1892133186 >> 172.21.15.86:0/1892133186 conn(0x7ff0e80cfa80 msgr2=0x7ff0e80cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:29.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.634+0000 7ff0ed200640 1 -- 172.21.15.86:0/1892133186 shutdown_connections 2024-03-06T07:31:29.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.634+0000 7ff0ed200640 1 -- 172.21.15.86:0/1892133186 wait complete. 2024-03-06T07:31:29.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.635+0000 7ff0ed200640 1 Processor -- start 2024-03-06T07:31:29.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.635+0000 7ff0ed200640 1 -- start start 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0ed200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e813e620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0ed200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80b4990 0x7ff0e813db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0ed200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e815e170 0x7ff0e8151ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0ed200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0e80b4dd0 con 0x7ff0e806e570 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0ed200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0e80b3f30 con 0x7ff0e80b4990 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0ed200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0e806df80 con 0x7ff0e815e170 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0e5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e813e620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0e5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e813e620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42200/0 (socket says 172.21.15.86:42200) 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.636+0000 7ff0e5e00640 1 -- 172.21.15.86:0/1762172570 learned_addr learned my addr 172.21.15.86:0/1762172570 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:29.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0e5e00640 1 -- 172.21.15.86:0/1762172570 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e815e170 msgr2=0x7ff0e8151ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0e5e00640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e815e170 0x7ff0e8151ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0e5400640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80b4990 0x7ff0e813db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0e5e00640 1 -- 172.21.15.86:0/1762172570 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80b4990 msgr2=0x7ff0e813db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0e5e00640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80b4990 0x7ff0e813db60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0e5e00640 1 -- 172.21.15.86:0/1762172570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0e8154fb0 con 0x7ff0e806e570 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0e5e00640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e813e620 secure :-1 s=READY pgs=929 cs=0 l=1 rev1=1 crypto rx=0x7ff0d800c390 tx=0x7ff0d800c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0de000640 1 -- 172.21.15.86:0/1762172570 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0d8017070 con 0x7ff0e806e570 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0de000640 1 -- 172.21.15.86:0/1762172570 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0d8005ce0 con 0x7ff0e806e570 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0de000640 1 -- 172.21.15.86:0/1762172570 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0d80072f0 con 0x7ff0e806e570 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff0e81e8940 con 0x7ff0e806e570 2024-03-06T07:31:29.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.637+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff0e81e8f00 con 0x7ff0e806e570 2024-03-06T07:31:29.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.638+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff0e815be70 con 0x7ff0e806e570 2024-03-06T07:31:29.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.640+0000 7ff0de000640 1 -- 172.21.15.86:0/1762172570 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff0d801a070 con 0x7ff0e806e570 2024-03-06T07:31:29.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.640+0000 7ff0de000640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c806b4d0 0x7ff0c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:29.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.640+0000 7ff0de000640 1 -- 172.21.15.86:0/1762172570 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 4857+0+0 (secure 0 0 0) 0x7ff0d8089250 con 0x7ff0e806e570 2024-03-06T07:31:29.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.641+0000 7ff0e5400640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c806b4d0 0x7ff0c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:29.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.641+0000 7ff0e5400640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c806b4d0 0x7ff0c806d990 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7ff0d4036000 tx=0x7ff0d4008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:29.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.645+0000 7ff0de000640 1 -- 172.21.15.86:0/1762172570 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff0e815be70 con 0x7ff0e806e570 2024-03-06T07:31:29.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:29.826+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7ff0e804f330 con 0x7ff0e806e570 2024-03-06T07:31:30.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.613+0000 7ff0de000640 1 -- 172.21.15.86:0/1762172570 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v80) v1 ==== 138+0+0 (secure 0 0 0) 0x7ff0d8058420 con 0x7ff0e806e570 2024-03-06T07:31:30.615 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:31:30.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.617+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c806b4d0 msgr2=0x7ff0c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:30.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.617+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c806b4d0 0x7ff0c806d990 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7ff0d4036000 tx=0x7ff0d4008030 comp rx=0 tx=0).stop 2024-03-06T07:31:30.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.617+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 msgr2=0x7ff0e813e620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:30.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.617+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e813e620 secure :-1 s=READY pgs=929 cs=0 l=1 rev1=1 crypto rx=0x7ff0d800c390 tx=0x7ff0d800c850 comp rx=0 tx=0).stop 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 shutdown_connections 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c806b4d0 0x7ff0c806d990 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e815e170 0x7ff0e8151ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80b4990 0x7ff0e813db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 --2- 172.21.15.86:0/1762172570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e806e570 0x7ff0e813e620 unknown :-1 s=CLOSED pgs=929 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 >> 172.21.15.86:0/1762172570 conn(0x7ff0e80cfa80 msgr2=0x7ff0e815b760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 shutdown_connections 2024-03-06T07:31:30.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.618+0000 7ff0ed200640 1 -- 172.21.15.86:0/1762172570 wait complete. 2024-03-06T07:31:30.644 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:31:30.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.847+0000 7f7357c00640 1 Processor -- start 2024-03-06T07:31:30.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.848+0000 7f7357c00640 1 -- start start 2024-03-06T07:31:30.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.849+0000 7f7357c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7350160700 0x7f7350070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:30.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.849+0000 7f7357c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f73500b46c0 con 0x7f73500b3f70 2024-03-06T07:31:30.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.849+0000 7f7357c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f73500b49a0 con 0x7f735015fc10 2024-03-06T07:31:30.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.849+0000 7f7357c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7350072c90 con 0x7f7350160700 2024-03-06T07:31:30.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.849+0000 7f7355c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7350160700 0x7f7350070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:30.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.849+0000 7f7355c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7350160700 0x7f7350070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42210/0 (socket says 172.21.15.86:42210) 2024-03-06T07:31:30.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.849+0000 7f7355c00640 1 -- 172.21.15.86:0/2521673770 learned_addr learned my addr 172.21.15.86:0/2521673770 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:30.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.850+0000 7f7355c00640 1 -- 172.21.15.86:0/2521673770 >> v1:172.21.15.194:6789/0 conn(0x7f735015fc10 legacy=0x7f735015fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:30.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.850+0000 7f7355c00640 1 -- 172.21.15.86:0/2521673770 >> v1:172.21.15.86:6789/0 conn(0x7f73500b3f70 legacy=0x7f73500ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:30.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.850+0000 7f7355c00640 1 -- 172.21.15.86:0/2521673770 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f735009dd80 con 0x7f7350160700 2024-03-06T07:31:30.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.851+0000 7f7355c00640 1 --2- 172.21.15.86:0/2521673770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7350160700 0x7f7350070ff0 secure :-1 s=READY pgs=931 cs=0 l=1 rev1=1 crypto rx=0x7f7348009870 tx=0x7f734802b990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cfce6d58b3a34aba server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:30.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.851+0000 7f734f400640 1 -- 172.21.15.86:0/2521673770 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7348034070 con 0x7f7350160700 2024-03-06T07:31:30.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.851+0000 7f734f400640 1 -- 172.21.15.86:0/2521673770 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7348030850 con 0x7f7350160700 2024-03-06T07:31:30.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.851+0000 7f734f400640 1 -- 172.21.15.86:0/2521673770 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7348030c20 con 0x7f7350160700 2024-03-06T07:31:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.852+0000 7f7357c00640 1 -- 172.21.15.86:0/2521673770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7350160700 msgr2=0x7f7350070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.852+0000 7f7357c00640 1 --2- 172.21.15.86:0/2521673770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7350160700 0x7f7350070ff0 secure :-1 s=READY pgs=931 cs=0 l=1 rev1=1 crypto rx=0x7f7348009870 tx=0x7f734802b990 comp rx=0 tx=0).stop 2024-03-06T07:31:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.852+0000 7f7357c00640 1 -- 172.21.15.86:0/2521673770 shutdown_connections 2024-03-06T07:31:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.852+0000 7f7357c00640 1 --2- 172.21.15.86:0/2521673770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7350160700 0x7f7350070ff0 unknown :-1 s=CLOSED pgs=931 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.852+0000 7f7357c00640 1 -- 172.21.15.86:0/2521673770 >> 172.21.15.86:0/2521673770 conn(0x7f73500b1940 msgr2=0x7f73500b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.852+0000 7f7357c00640 1 -- 172.21.15.86:0/2521673770 shutdown_connections 2024-03-06T07:31:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.852+0000 7f7357c00640 1 -- 172.21.15.86:0/2521673770 wait complete. 2024-03-06T07:31:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.854+0000 7f7357c00640 1 Processor -- start 2024-03-06T07:31:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.854+0000 7f7357c00640 1 -- start start 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7357c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73500b3f70 0x7f735015b430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7357c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f735015fc10 0x7f73500acf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7357c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7350160700 0x7f73501598d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7357c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f735015c460 con 0x7f73500b3f70 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7357c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f735015c2e0 con 0x7f735015fc10 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7357c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f73500cebb0 con 0x7f7350160700 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7355200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73500b3f70 0x7f735015b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7355200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73500b3f70 0x7f735015b430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42212/0 (socket says 172.21.15.86:42212) 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7355200640 1 -- 172.21.15.86:0/2496185296 learned_addr learned my addr 172.21.15.86:0/2496185296 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f734fe00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f735015fc10 0x7f73500acf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.855+0000 7f7355c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7350160700 0x7f73501598d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7355200640 1 -- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7350160700 msgr2=0x7f73501598d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7355200640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7350160700 0x7f73501598d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7355200640 1 -- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f735015fc10 msgr2=0x7f73500acf90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7355200640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f735015fc10 0x7f73500acf90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7355200640 1 -- 172.21.15.86:0/2496185296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f735015a390 con 0x7f73500b3f70 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7355c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7350160700 0x7f73501598d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7355200640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73500b3f70 0x7f735015b430 secure :-1 s=READY pgs=932 cs=0 l=1 rev1=1 crypto rx=0x7f734000c7c0 tx=0x7f734000cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f734cc00640 1 -- 172.21.15.86:0/2496185296 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7340016020 con 0x7f73500b3f70 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f734cc00640 1 -- 172.21.15.86:0/2496185296 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f734000aa50 con 0x7f73500b3f70 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f735015a670 con 0x7f73500b3f70 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f734cc00640 1 -- 172.21.15.86:0/2496185296 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7340012420 con 0x7f73500b3f70 2024-03-06T07:31:30.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.856+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f735015ac30 con 0x7f73500b3f70 2024-03-06T07:31:30.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.857+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f735004f330 con 0x7f73500b3f70 2024-03-06T07:31:30.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.859+0000 7f734cc00640 1 -- 172.21.15.86:0/2496185296 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7340019020 con 0x7f73500b3f70 2024-03-06T07:31:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.860+0000 7f734cc00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f732c06b4d0 0x7f732c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.860+0000 7f734cc00640 1 -- 172.21.15.86:0/2496185296 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f734008a590 con 0x7f73500b3f70 2024-03-06T07:31:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.860+0000 7f734fe00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f732c06b4d0 0x7f732c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.861+0000 7f734fe00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f732c06b4d0 0x7f732c06d990 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f73500ad8e0 tx=0x7f7338009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:30.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:30.864+0000 7f734cc00640 1 -- 172.21.15.86:0/2496185296 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7340020070 con 0x7f73500b3f70 2024-03-06T07:31:31.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.065+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f735006ef30 con 0x7f73500b3f70 2024-03-06T07:31:31.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.620+0000 7f734cc00640 1 -- 172.21.15.86:0/2496185296 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v81) v1 ==== 170+0+0 (secure 0 0 0) 0x7f7340022330 con 0x7f73500b3f70 2024-03-06T07:31:31.622 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:31:31.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f732c06b4d0 msgr2=0x7f732c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:31.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f732c06b4d0 0x7f732c06d990 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f73500ad8e0 tx=0x7f7338009040 comp rx=0 tx=0).stop 2024-03-06T07:31:31.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73500b3f70 msgr2=0x7f735015b430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:31.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73500b3f70 0x7f735015b430 secure :-1 s=READY pgs=932 cs=0 l=1 rev1=1 crypto rx=0x7f734000c7c0 tx=0x7f734000cc80 comp rx=0 tx=0).stop 2024-03-06T07:31:31.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 shutdown_connections 2024-03-06T07:31:31.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f732c06b4d0 0x7f732c06d990 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7350160700 0x7f73501598d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f735015fc10 0x7f73500acf90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 --2- 172.21.15.86:0/2496185296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73500b3f70 0x7f735015b430 unknown :-1 s=CLOSED pgs=932 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.624+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 >> 172.21.15.86:0/2496185296 conn(0x7f73500b1940 msgr2=0x7f735015fff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:31.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.625+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 shutdown_connections 2024-03-06T07:31:31.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.625+0000 7f7357c00640 1 -- 172.21.15.86:0/2496185296 wait complete. 2024-03-06T07:31:31.658 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:31:31.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.840+0000 7f20a1600640 1 Processor -- start 2024-03-06T07:31:31.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.840+0000 7f20a1600640 1 -- start start 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f20a1600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f209c15be10 0x7f209c0ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f20a1600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f209c06da40 0x7f209c0ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f20a1600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f209c15c1f0 con 0x7f209c0aef20 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f20a1600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f209c0cd200 con 0x7f209c15be10 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f20a1600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f209c06cac0 con 0x7f209c06da40 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f209be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f209c15be10 0x7f209c0ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f209be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f209c15be10 0x7f209c0ccc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42222/0 (socket says 172.21.15.86:42222) 2024-03-06T07:31:31.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f209be00640 1 -- 172.21.15.86:0/501591163 learned_addr learned my addr 172.21.15.86:0/501591163 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:31.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.842+0000 7f20a0c00640 1 --2- 172.21.15.86:0/501591163 >> v2:172.21.15.194:3300/0 conn(0x7f209c06da40 0x7f209c0ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:31.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.843+0000 7f209be00640 1 -- 172.21.15.86:0/501591163 >> v2:172.21.15.194:3300/0 conn(0x7f209c06da40 msgr2=0x7f209c0ce2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:31.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.843+0000 7f209be00640 1 --2- 172.21.15.86:0/501591163 >> v2:172.21.15.194:3300/0 conn(0x7f209c06da40 0x7f209c0ce2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.843+0000 7f209be00640 1 -- 172.21.15.86:0/501591163 >> v1:172.21.15.194:6789/0 conn(0x7f209c0aef20 legacy=0x7f209c06d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:31.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.843+0000 7f209be00640 1 -- 172.21.15.86:0/501591163 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f209c06cda0 con 0x7f209c15be10 2024-03-06T07:31:31.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.844+0000 7f209be00640 1 --2- 172.21.15.86:0/501591163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c0ccc20 secure :-1 s=READY pgs=933 cs=0 l=1 rev1=1 crypto rx=0x7f209c0b1500 tx=0x7f208c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e9dede81a64c57f6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:31.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f209aa00640 1 -- 172.21.15.86:0/501591163 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f208c035020 con 0x7f209c15be10 2024-03-06T07:31:31.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f209aa00640 1 -- 172.21.15.86:0/501591163 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f208c031870 con 0x7f209c15be10 2024-03-06T07:31:31.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f209aa00640 1 -- 172.21.15.86:0/501591163 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f208c031c40 con 0x7f209c15be10 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f20a1600640 1 -- 172.21.15.86:0/501591163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 msgr2=0x7f209c0ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f20a1600640 1 --2- 172.21.15.86:0/501591163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c0ccc20 secure :-1 s=READY pgs=933 cs=0 l=1 rev1=1 crypto rx=0x7f209c0b1500 tx=0x7f208c02b910 comp rx=0 tx=0).stop 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f20a1600640 1 -- 172.21.15.86:0/501591163 shutdown_connections 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f20a1600640 1 --2- 172.21.15.86:0/501591163 >> v2:172.21.15.194:3300/0 conn(0x7f209c06da40 0x7f209c0ce2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f20a1600640 1 --2- 172.21.15.86:0/501591163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c0ccc20 unknown :-1 s=CLOSED pgs=933 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.845+0000 7f20a1600640 1 -- 172.21.15.86:0/501591163 >> 172.21.15.86:0/501591163 conn(0x7f209c06afc0 msgr2=0x7f209c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.846+0000 7f20a1600640 1 -- 172.21.15.86:0/501591163 shutdown_connections 2024-03-06T07:31:31.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.846+0000 7f20a1600640 1 -- 172.21.15.86:0/501591163 wait complete. 2024-03-06T07:31:31.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.847+0000 7f20a1600640 1 Processor -- start 2024-03-06T07:31:31.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.848+0000 7f20a1600640 1 -- start start 2024-03-06T07:31:31.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a1600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f209c06da40 0x7f209c0adc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:31.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f209c0aef20 0x7f209c1f4110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:31.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c15c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:31.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a1600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f209c0cda90 con 0x7f209c15be10 2024-03-06T07:31:31.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a1600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f209c06d080 con 0x7f209c06da40 2024-03-06T07:31:31.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f209c071ab0 con 0x7f209c0aef20 2024-03-06T07:31:31.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c15c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c15c430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42232/0 (socket says 172.21.15.86:42232) 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f20a0c00640 1 -- 172.21.15.86:0/3233954127 learned_addr learned my addr 172.21.15.86:0/3233954127 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f209be00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f209c06da40 0x7f209c0adc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.849+0000 7f209b400640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f209c0aef20 0x7f209c1f4110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a0c00640 1 -- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f209c0aef20 msgr2=0x7f209c1f4110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a0c00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f209c0aef20 0x7f209c1f4110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a0c00640 1 -- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f209c06da40 msgr2=0x7f209c0adc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a0c00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f209c06da40 0x7f209c0adc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a0c00640 1 -- 172.21.15.86:0/3233954127 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f209c0ae1a0 con 0x7f209c15be10 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f209be00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f209c06da40 0x7f209c0adc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:31.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a0c00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c15c430 secure :-1 s=READY pgs=934 cs=0 l=1 rev1=1 crypto rx=0x7f2094007de0 tx=0x7f209400e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:31.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f2093e00640 1 -- 172.21.15.86:0/3233954127 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2094014620 con 0x7f209c15be10 2024-03-06T07:31:31.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f2093e00640 1 -- 172.21.15.86:0/3233954127 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2094019070 con 0x7f209c15be10 2024-03-06T07:31:31.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f209c160830 con 0x7f209c15be10 2024-03-06T07:31:31.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f2093e00640 1 -- 172.21.15.86:0/3233954127 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2094014a20 con 0x7f209c15be10 2024-03-06T07:31:31.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.850+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f209c15d280 con 0x7f209c15be10 2024-03-06T07:31:31.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.851+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f209c04f330 con 0x7f209c15be10 2024-03-06T07:31:31.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.852+0000 7f2093e00640 1 -- 172.21.15.86:0/3233954127 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2094014bc0 con 0x7f209c15be10 2024-03-06T07:31:31.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.853+0000 7f2093e00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f207806b4d0 0x7f207806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:31.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.853+0000 7f2093e00640 1 -- 172.21.15.86:0/3233954127 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f209408aaf0 con 0x7f209c15be10 2024-03-06T07:31:31.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.854+0000 7f209be00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f207806b4d0 0x7f207806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:31.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.854+0000 7f209be00640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f207806b4d0 0x7f207806d990 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f208c036000 tx=0x7f208c008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:31.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:31.859+0000 7f2093e00640 1 -- 172.21.15.86:0/3233954127 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f20940599c0 con 0x7f209c15be10 2024-03-06T07:31:32.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.093+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f209c0b2040 con 0x7f209c15be10 2024-03-06T07:31:32.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.641+0000 7f2093e00640 1 -- 172.21.15.86:0/3233954127 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 12 and data pool 13 v47) v1 ==== 174+0+0 (secure 0 0 0) 0x7f20940592c0 con 0x7f209c15be10 2024-03-06T07:31:32.643 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 12 and data pool 13 2024-03-06T07:31:32.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f207806b4d0 msgr2=0x7f207806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:32.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f207806b4d0 0x7f207806d990 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f208c036000 tx=0x7f208c008030 comp rx=0 tx=0).stop 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 msgr2=0x7f209c15c430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c15c430 secure :-1 s=READY pgs=934 cs=0 l=1 rev1=1 crypto rx=0x7f2094007de0 tx=0x7f209400e4b0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 shutdown_connections 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f207806b4d0 0x7f207806d990 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f209c15be10 0x7f209c15c430 unknown :-1 s=CLOSED pgs=934 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f209c0aef20 0x7f209c1f4110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 --2- 172.21.15.86:0/3233954127 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f209c06da40 0x7f209c0adc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 >> 172.21.15.86:0/3233954127 conn(0x7f209c06afc0 msgr2=0x7f209c0ce4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 shutdown_connections 2024-03-06T07:31:32.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.645+0000 7f20a1600640 1 -- 172.21.15.86:0/3233954127 wait complete. 2024-03-06T07:31:32.685 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:32.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.863+0000 7f75cfa00640 1 Processor -- start 2024-03-06T07:31:32.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.864+0000 7f75cfa00640 1 -- start start 2024-03-06T07:31:32.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cfa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75c80d0ca0 0x7f75c80cfe50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:32.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cfa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f75c806b6d0 0x7f75c80b5530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:32.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cfa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75c815d4f0 con 0x7f75c8160530 2024-03-06T07:31:32.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cfa00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75c80b50a0 con 0x7f75c80d0ca0 2024-03-06T07:31:32.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cfa00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75c815cc50 con 0x7f75c806b6d0 2024-03-06T07:31:32.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cd000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75c80d0ca0 0x7f75c80cfe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:32.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cd000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75c80d0ca0 0x7f75c80cfe50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42238/0 (socket says 172.21.15.86:42238) 2024-03-06T07:31:32.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.865+0000 7f75cd000640 1 -- 172.21.15.86:0/1744293099 learned_addr learned my addr 172.21.15.86:0/1744293099 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:32.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.866+0000 7f75cda00640 1 --2- 172.21.15.86:0/1744293099 >> v2:172.21.15.194:3300/0 conn(0x7f75c806b6d0 0x7f75c80b5530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:32.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.866+0000 7f75cd000640 1 -- 172.21.15.86:0/1744293099 >> v2:172.21.15.194:3300/0 conn(0x7f75c806b6d0 msgr2=0x7f75c80b5530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:32.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.866+0000 7f75cd000640 1 --2- 172.21.15.86:0/1744293099 >> v2:172.21.15.194:3300/0 conn(0x7f75c806b6d0 0x7f75c80b5530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.866+0000 7f75cd000640 1 -- 172.21.15.86:0/1744293099 >> v1:172.21.15.194:6789/0 conn(0x7f75c8160530 legacy=0x7f75c806afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:32.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.866+0000 7f75cd000640 1 -- 172.21.15.86:0/1744293099 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75c815cf30 con 0x7f75c80d0ca0 2024-03-06T07:31:32.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.867+0000 7f75cd000640 1 --2- 172.21.15.86:0/1744293099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c80d0ca0 0x7f75c80cfe50 secure :-1 s=READY pgs=935 cs=0 l=1 rev1=1 crypto rx=0x7f75c80ccbb0 tx=0x7f75bc02b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=183f4afae5aa7814 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:32.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.868+0000 7f75c7400640 1 -- 172.21.15.86:0/1744293099 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75bc02e070 con 0x7f75c80d0ca0 2024-03-06T07:31:32.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.868+0000 7f75c7400640 1 -- 172.21.15.86:0/1744293099 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75bc037070 con 0x7f75c80d0ca0 2024-03-06T07:31:32.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.868+0000 7f75c7400640 1 -- 172.21.15.86:0/1744293099 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75bc031a00 con 0x7f75c80d0ca0 2024-03-06T07:31:32.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 -- 172.21.15.86:0/1744293099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c80d0ca0 msgr2=0x7f75c80cfe50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:32.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 --2- 172.21.15.86:0/1744293099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c80d0ca0 0x7f75c80cfe50 secure :-1 s=READY pgs=935 cs=0 l=1 rev1=1 crypto rx=0x7f75c80ccbb0 tx=0x7f75bc02b880 comp rx=0 tx=0).stop 2024-03-06T07:31:32.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 -- 172.21.15.86:0/1744293099 shutdown_connections 2024-03-06T07:31:32.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 --2- 172.21.15.86:0/1744293099 >> v2:172.21.15.194:3300/0 conn(0x7f75c806b6d0 0x7f75c80b5530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 --2- 172.21.15.86:0/1744293099 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c80d0ca0 0x7f75c80cfe50 unknown :-1 s=CLOSED pgs=935 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 -- 172.21.15.86:0/1744293099 >> 172.21.15.86:0/1744293099 conn(0x7f75c80b1e10 msgr2=0x7f75c80b2220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:32.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 -- 172.21.15.86:0/1744293099 shutdown_connections 2024-03-06T07:31:32.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.869+0000 7f75cfa00640 1 -- 172.21.15.86:0/1744293099 wait complete. 2024-03-06T07:31:32.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.870+0000 7f75cfa00640 1 Processor -- start 2024-03-06T07:31:32.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.871+0000 7f75cfa00640 1 -- start start 2024-03-06T07:31:32.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.871+0000 7f75cfa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75c806b6d0 0x7f75c81f6b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:32.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.871+0000 7f75cfa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75c80d0ca0 0x7f75c80c5d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:32.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cfa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c8160530 0x7f75c80cb780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:32.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cfa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75c80d1080 con 0x7f75c8160530 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cfa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75c8160990 con 0x7f75c80d0ca0 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cfa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75c80b5a70 con 0x7f75c806b6d0 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c8160530 0x7f75c80cb780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c8160530 0x7f75c80cb780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42246/0 (socket says 172.21.15.86:42246) 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 -- 172.21.15.86:0/3304625276 learned_addr learned my addr 172.21.15.86:0/3304625276 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cd000640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75c806b6d0 0x7f75c81f6b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75c7e00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75c80d0ca0 0x7f75c80c5d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 -- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75c806b6d0 msgr2=0x7f75c81f6b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75c806b6d0 0x7f75c81f6b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 -- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75c80d0ca0 msgr2=0x7f75c80c5d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75c80d0ca0 0x7f75c80c5d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.872+0000 7f75cda00640 1 -- 172.21.15.86:0/3304625276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75c806d450 con 0x7f75c8160530 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.873+0000 7f75cda00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c8160530 0x7f75c80cb780 secure :-1 s=READY pgs=936 cs=0 l=1 rev1=1 crypto rx=0x7f75c000cd40 tx=0x7f75c000a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.873+0000 7f75c4c00640 1 -- 172.21.15.86:0/3304625276 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75c0003b90 con 0x7f75c8160530 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.873+0000 7f75c4c00640 1 -- 172.21.15.86:0/3304625276 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75c0003d30 con 0x7f75c8160530 2024-03-06T07:31:32.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.873+0000 7f75c4c00640 1 -- 172.21.15.86:0/3304625276 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75c0004240 con 0x7f75c8160530 2024-03-06T07:31:32.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.873+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f75c806d6f0 con 0x7f75c8160530 2024-03-06T07:31:32.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.873+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f75c806dcb0 con 0x7f75c8160530 2024-03-06T07:31:32.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.874+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f75c806deb0 con 0x7f75c8160530 2024-03-06T07:31:32.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.876+0000 7f75c4c00640 1 -- 172.21.15.86:0/3304625276 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f75c00043e0 con 0x7f75c8160530 2024-03-06T07:31:32.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.877+0000 7f75c4c00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75a806b500 0x7f75a806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:32.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.877+0000 7f75c4c00640 1 -- 172.21.15.86:0/3304625276 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f75c008ada0 con 0x7f75c8160530 2024-03-06T07:31:32.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.877+0000 7f75cd000640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75a806b500 0x7f75a806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:32.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.878+0000 7f75cd000640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75a806b500 0x7f75a806d9c0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f75bc036000 tx=0x7f75bc008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:32.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:32.882+0000 7f75c4c00640 1 -- 172.21.15.86:0/3304625276 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f75c806deb0 con 0x7f75c8160530 2024-03-06T07:31:33.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.073+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f75c80b2ae0 con 0x7f75c8160530 2024-03-06T07:31:33.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.076+0000 7f75c4c00640 1 -- 172.21.15.86:0/3304625276 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v82) v1 ==== 74+0+16685 (secure 0 0 0) 0x7f75c0059b80 con 0x7f75c8160530 2024-03-06T07:31:33.078 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:33.078 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":82,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:32.095056+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:31:29.878719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-06T07:31:31.115332+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75a806b500 msgr2=0x7f75a806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75a806b500 0x7f75a806d9c0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f75bc036000 tx=0x7f75bc008030 comp rx=0 tx=0).stop 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c8160530 msgr2=0x7f75c80cb780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c8160530 0x7f75c80cb780 secure :-1 s=READY pgs=936 cs=0 l=1 rev1=1 crypto rx=0x7f75c000cd40 tx=0x7f75c000a660 comp rx=0 tx=0).stop 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 shutdown_connections 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75a806b500 0x7f75a806d9c0 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75c8160530 0x7f75c80cb780 unknown :-1 s=CLOSED pgs=936 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75c80d0ca0 0x7f75c80c5d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 --2- 172.21.15.86:0/3304625276 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75c806b6d0 0x7f75c81f6b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 >> 172.21.15.86:0/3304625276 conn(0x7f75c80b1e10 msgr2=0x7f75c80b0990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 shutdown_connections 2024-03-06T07:31:33.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.079+0000 7f75cfa00640 1 -- 172.21.15.86:0/3304625276 wait complete. 2024-03-06T07:31:33.098 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:33.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.297+0000 7fe5f5200640 1 Processor -- start 2024-03-06T07:31:33.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.298+0000 7fe5f5200640 1 -- start start 2024-03-06T07:31:33.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.300+0000 7fe5f5200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f015e230 0x7fe5f006edb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.300+0000 7fe5f5200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe5f00cf340 con 0x7fe5f00ce550 2024-03-06T07:31:33.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.300+0000 7fe5f5200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe5f00cfe60 con 0x7fe5f00b4320 2024-03-06T07:31:33.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.300+0000 7fe5f5200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f015e610 con 0x7fe5f015e230 2024-03-06T07:31:33.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ee800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe5f00ce550 0x7fe5f00cc320 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37982/0 (socket says 172.21.15.86:37982) 2024-03-06T07:31:33.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ee800640 1 -- 172.21.15.86:0/498810146 learned_addr learned my addr 172.21.15.86:0/498810146 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ede00640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f015e230 0x7fe5f006edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4188869167 0 0) 0x7fe5f00cf340 con 0x7fe5f00ce550 2024-03-06T07:31:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe5d4003520 con 0x7fe5f00ce550 2024-03-06T07:31:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3651128493 0 0) 0x7fe5d4003520 con 0x7fe5f00ce550 2024-03-06T07:31:33.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f015e230 msgr2=0x7fe5f006edb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f015e230 0x7fe5f006edb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 >> v1:172.21.15.194:6789/0 conn(0x7fe5f00b4320 legacy=0x7fe5f00d0be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5f006f390 con 0x7fe5f00ce550 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fe5e80019f0 con 0x7fe5f00ce550 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.301+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 >> v1:172.21.15.86:6789/0 conn(0x7fe5f00ce550 legacy=0x7fe5f00cc320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5ed400640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5d40039e0 0x7fe5d4003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5ed400640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5d4004800 0x7fe5d4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5ed400640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f015e230 0x7fe5d4005200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f015e610 con 0x7fe5d4004800 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5d400baf0 con 0x7fe5d40039e0 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5d400bcf0 con 0x7fe5f015e230 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5e7e00640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5d4004800 0x7fe5d4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5e7e00640 1 -- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f015e230 msgr2=0x7fe5d4005200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:33.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5e7e00640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f015e230 0x7fe5d4005200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5e7e00640 1 -- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5d40039e0 msgr2=0x7fe5d4003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:33.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5e7e00640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5d40039e0 0x7fe5d4003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.302+0000 7fe5e7e00640 1 -- 172.21.15.86:0/498810146 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5d4003520 con 0x7fe5d4004800 2024-03-06T07:31:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.303+0000 7fe5e7e00640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5d4004800 0x7fe5d4004c80 secure :-1 s=READY pgs=938 cs=0 l=1 rev1=1 crypto rx=0x7fe5d8001750 tx=0x7fe5d4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1fa2e0f27e317fb3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5d8001370 con 0x7fe5d4004800 2024-03-06T07:31:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5d80052d0 con 0x7fe5d4004800 2024-03-06T07:31:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5ed400640 1 -- 172.21.15.86:0/498810146 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5d8003830 con 0x7fe5d4004800 2024-03-06T07:31:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5f5200640 1 -- 172.21.15.86:0/498810146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5d4004800 msgr2=0x7fe5d4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5f5200640 1 --2- 172.21.15.86:0/498810146 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5d4004800 0x7fe5d4004c80 secure :-1 s=READY pgs=938 cs=0 l=1 rev1=1 crypto rx=0x7fe5d8001750 tx=0x7fe5d4005830 comp rx=0 tx=0).stop 2024-03-06T07:31:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5ee800640 1 -- 172.21.15.86:0/498810146 reap_dead start 2024-03-06T07:31:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5f5200640 1 -- 172.21.15.86:0/498810146 shutdown_connections 2024-03-06T07:31:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5f5200640 1 -- 172.21.15.86:0/498810146 >> 172.21.15.86:0/498810146 conn(0x7fe5f00b3800 msgr2=0x7fe5f00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5f5200640 1 -- 172.21.15.86:0/498810146 shutdown_connections 2024-03-06T07:31:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.304+0000 7fe5f5200640 1 -- 172.21.15.86:0/498810146 wait complete. 2024-03-06T07:31:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.306+0000 7fe5f5200640 1 Processor -- start 2024-03-06T07:31:33.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.306+0000 7fe5f5200640 1 -- start start 2024-03-06T07:31:33.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5f5200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f014d0b0 0x7fe5f00b65b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5f5200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f00b6af0 0x7fe5f00c7b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5f5200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5f00c8070 0x7fe5f00b48f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5f5200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f00d06c0 con 0x7fe5f00c8070 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5f5200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f00d03e0 con 0x7fe5f00b6af0 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5f5200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f006edb0 con 0x7fe5f014d0b0 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5ee800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5f00c8070 0x7fe5f00b48f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5ee800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5f00c8070 0x7fe5f00b48f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42268/0 (socket says 172.21.15.86:42268) 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.308+0000 7fe5ee800640 1 -- 172.21.15.86:0/653812047 learned_addr learned my addr 172.21.15.86:0/653812047 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5e7e00640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f00b6af0 0x7fe5f00c7b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5ee800640 1 -- 172.21.15.86:0/653812047 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f014d0b0 msgr2=0x7fe5f00b65b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5ee800640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f014d0b0 0x7fe5f00b65b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5ee800640 1 -- 172.21.15.86:0/653812047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f00b6af0 msgr2=0x7fe5f00c7b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5ee800640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f00b6af0 0x7fe5f00c7b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5ee800640 1 -- 172.21.15.86:0/653812047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5f00c62b0 con 0x7fe5f00c8070 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5ee800640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5f00c8070 0x7fe5f00b48f0 secure :-1 s=READY pgs=939 cs=0 l=1 rev1=1 crypto rx=0x7fe5e8001350 tx=0x7fe5e800cf00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5e6000640 1 -- 172.21.15.86:0/653812047 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5e800a9c0 con 0x7fe5f00c8070 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5e6000640 1 -- 172.21.15.86:0/653812047 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5e800ab60 con 0x7fe5f00c8070 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5e6000640 1 -- 172.21.15.86:0/653812047 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5e8014420 con 0x7fe5f00c8070 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe5f00c6590 con 0x7fe5f00c8070 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.309+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5f00c6b50 con 0x7fe5f00c8070 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.310+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe5f004f330 con 0x7fe5f00c8070 2024-03-06T07:31:33.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.311+0000 7fe5e6000640 1 -- 172.21.15.86:0/653812047 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe5e8011070 con 0x7fe5f00c8070 2024-03-06T07:31:33.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.312+0000 7fe5e6000640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5cc06b4d0 0x7fe5cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.312+0000 7fe5e6000640 1 -- 172.21.15.86:0/653812047 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe5e8089cf0 con 0x7fe5f00c8070 2024-03-06T07:31:33.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.312+0000 7fe5ede00640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5cc06b4d0 0x7fe5cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.313+0000 7fe5ede00640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5cc06b4d0 0x7fe5cc06d990 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fe5e0009a20 tx=0x7fe5e00094e0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:33.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.317+0000 7fe5e6000640 1 -- 172.21.15.86:0/653812047 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe5e8058ad0 con 0x7fe5f00c8070 2024-03-06T07:31:33.594 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:31:33.592+0000 7fc3dfa00640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:31:33.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.648+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe5f00b6ed0 con 0x7fe5f00c8070 2024-03-06T07:31:33.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.651+0000 7fe5e6000640 1 -- 172.21.15.86:0/653812047 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v82) v1 ==== 74+0+16685 (secure 0 0 0) 0x7fe5e80583d0 con 0x7fe5f00c8070 2024-03-06T07:31:33.653 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:33.653 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":82,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:32.095056+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:31:29.878719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-06T07:31:31.115332+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5cc06b4d0 msgr2=0x7fe5cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5cc06b4d0 0x7fe5cc06d990 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fe5e0009a20 tx=0x7fe5e00094e0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5f00c8070 msgr2=0x7fe5f00b48f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5f00c8070 0x7fe5f00b48f0 secure :-1 s=READY pgs=939 cs=0 l=1 rev1=1 crypto rx=0x7fe5e8001350 tx=0x7fe5e800cf00 comp rx=0 tx=0).stop 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 shutdown_connections 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe5cc06b4d0 0x7fe5cc06d990 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5f00c8070 0x7fe5f00b48f0 unknown :-1 s=CLOSED pgs=939 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5f00b6af0 0x7fe5f00c7b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 --2- 172.21.15.86:0/653812047 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5f014d0b0 0x7fe5f00b65b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 >> 172.21.15.86:0/653812047 conn(0x7fe5f00b3800 msgr2=0x7fe5f006d8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:33.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.655+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 shutdown_connections 2024-03-06T07:31:33.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.656+0000 7fe5f5200640 1 -- 172.21.15.86:0/653812047 wait complete. 2024-03-06T07:31:33.677 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:31:33.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.880+0000 7fe002c00640 1 Processor -- start 2024-03-06T07:31:33.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.880+0000 7fe002c00640 1 -- start start 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.881+0000 7fe002c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fdffc15be10 0x7fdffc0ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe002c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdffc0aef20 0x7fdffc0ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe002c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc06d330 0x7fdffc06d7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe002c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdffc0cd200 con 0x7fdffc0aef20 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe002c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdffc15c1f0 con 0x7fdffc15be10 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe002c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdffc06cac0 con 0x7fdffc06d330 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdffc0aef20 0x7fdffc0ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdffc0aef20 0x7fdffc0ce2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42282/0 (socket says 172.21.15.86:42282) 2024-03-06T07:31:33.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 -- 172.21.15.86:0/2994992352 learned_addr learned my addr 172.21.15.86:0/2994992352 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:33.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe001800640 1 --2- 172.21.15.86:0/2994992352 >> v2:172.21.15.194:3300/0 conn(0x7fdffc15be10 0x7fdffc0ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 -- 172.21.15.86:0/2994992352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc06d330 msgr2=0x7fdffc06d7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:33.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 --2- 172.21.15.86:0/2994992352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc06d330 0x7fdffc06d7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 -- 172.21.15.86:0/2994992352 >> v2:172.21.15.194:3300/0 conn(0x7fdffc15be10 msgr2=0x7fdffc0ccc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:33.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 --2- 172.21.15.86:0/2994992352 >> v2:172.21.15.194:3300/0 conn(0x7fdffc15be10 0x7fdffc0ccc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.882+0000 7fe000e00640 1 -- 172.21.15.86:0/2994992352 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdffc06cda0 con 0x7fdffc0aef20 2024-03-06T07:31:33.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.883+0000 7fe000e00640 1 --2- 172.21.15.86:0/2994992352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0ce2b0 secure :-1 s=READY pgs=940 cs=0 l=1 rev1=1 crypto rx=0x7fdffc0d09c0 tx=0x7fdfec02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2eaa9caccbaba65 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:33.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.884+0000 7fdffbe00640 1 -- 172.21.15.86:0/2994992352 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfec035020 con 0x7fdffc0aef20 2024-03-06T07:31:33.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.884+0000 7fdffbe00640 1 -- 172.21.15.86:0/2994992352 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdfec031870 con 0x7fdffc0aef20 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.884+0000 7fdffbe00640 1 -- 172.21.15.86:0/2994992352 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfec031c40 con 0x7fdffc0aef20 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.884+0000 7fe002c00640 1 -- 172.21.15.86:0/2994992352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 msgr2=0x7fdffc0ce2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.884+0000 7fe002c00640 1 --2- 172.21.15.86:0/2994992352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0ce2b0 secure :-1 s=READY pgs=940 cs=0 l=1 rev1=1 crypto rx=0x7fdffc0d09c0 tx=0x7fdfec02b910 comp rx=0 tx=0).stop 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.885+0000 7fe002c00640 1 -- 172.21.15.86:0/2994992352 shutdown_connections 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.885+0000 7fe002c00640 1 --2- 172.21.15.86:0/2994992352 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc06d330 0x7fdffc06d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.885+0000 7fe002c00640 1 --2- 172.21.15.86:0/2994992352 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0ce2b0 unknown :-1 s=CLOSED pgs=940 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.885+0000 7fe002c00640 1 --2- 172.21.15.86:0/2994992352 >> v2:172.21.15.194:3300/0 conn(0x7fdffc15be10 0x7fdffc0ccc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.885+0000 7fe002c00640 1 -- 172.21.15.86:0/2994992352 >> 172.21.15.86:0/2994992352 conn(0x7fdffc06afc0 msgr2=0x7fdffc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.885+0000 7fe002c00640 1 -- 172.21.15.86:0/2994992352 shutdown_connections 2024-03-06T07:31:33.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.885+0000 7fe002c00640 1 -- 172.21.15.86:0/2994992352 wait complete. 2024-03-06T07:31:33.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.886+0000 7fe002c00640 1 Processor -- start 2024-03-06T07:31:33.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.886+0000 7fe002c00640 1 -- start start 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.887+0000 7fe002c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdffc06d330 0x7fdffc0b09d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.887+0000 7fe002c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0addd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.887+0000 7fe002c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc15be10 0x7fdffc0cad40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.887+0000 7fe002c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdffc0cda90 con 0x7fdffc0aef20 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.887+0000 7fe002c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdffc06d080 con 0x7fdffc06d330 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.887+0000 7fe002c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdffc071ab0 con 0x7fdffc15be10 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0addd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0addd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42294/0 (socket says 172.21.15.86:42294) 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 -- 172.21.15.86:0/3802206859 learned_addr learned my addr 172.21.15.86:0/3802206859 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe001800640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdffc06d330 0x7fdffc0b09d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 -- 172.21.15.86:0/3802206859 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc15be10 msgr2=0x7fdffc0cad40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc15be10 0x7fdffc0cad40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 -- 172.21.15.86:0/3802206859 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdffc06d330 msgr2=0x7fdffc0b09d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:33.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdffc06d330 0x7fdffc0b09d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:33.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 -- 172.21.15.86:0/3802206859 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdffc0b3630 con 0x7fdffc0aef20 2024-03-06T07:31:33.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.888+0000 7fe000e00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0addd0 secure :-1 s=READY pgs=941 cs=0 l=1 rev1=1 crypto rx=0x7fdfec02b950 tx=0x7fdfec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:33.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.889+0000 7fdff9600640 1 -- 172.21.15.86:0/3802206859 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfec03ad60 con 0x7fdffc0aef20 2024-03-06T07:31:33.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.889+0000 7fdff9600640 1 -- 172.21.15.86:0/3802206859 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdfec0443f0 con 0x7fdffc0aef20 2024-03-06T07:31:33.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.889+0000 7fdff9600640 1 -- 172.21.15.86:0/3802206859 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfec0447e0 con 0x7fdffc0aef20 2024-03-06T07:31:33.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.889+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdffc15a8a0 con 0x7fdffc0aef20 2024-03-06T07:31:33.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.889+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdffc15ae60 con 0x7fdffc0aef20 2024-03-06T07:31:33.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.890+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdffc04f330 con 0x7fdffc0aef20 2024-03-06T07:31:33.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.891+0000 7fdff9600640 1 -- 172.21.15.86:0/3802206859 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdfec038020 con 0x7fdffc0aef20 2024-03-06T07:31:33.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.892+0000 7fdff9600640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdfe006b4d0 0x7fdfe006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:33.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.892+0000 7fdff9600640 1 -- 172.21.15.86:0/3802206859 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fdfec0ac5a0 con 0x7fdffc0aef20 2024-03-06T07:31:33.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.892+0000 7fe001800640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdfe006b4d0 0x7fdfe006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:33.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.892+0000 7fe001800640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdfe006b4d0 0x7fdfe006d990 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fdffc0cf2a0 tx=0x7fdff0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:33.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:33.900+0000 7fdff9600640 1 -- 172.21.15.86:0/3802206859 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdffc0b3630 con 0x7fdffc0aef20 2024-03-06T07:31:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.140+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7fdffc0cee60 con 0x7fdffc0aef20 2024-03-06T07:31:34.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.856+0000 7fdff9600640 1 -- 172.21.15.86:0/3802206859 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v49) v1 ==== 118+0+0 (secure 0 0 0) 0x7fdfec03f070 con 0x7fdffc0aef20 2024-03-06T07:31:34.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdfe006b4d0 msgr2=0x7fdfe006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:34.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdfe006b4d0 0x7fdfe006d990 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fdffc0cf2a0 tx=0x7fdff0008040 comp rx=0 tx=0).stop 2024-03-06T07:31:34.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 msgr2=0x7fdffc0addd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:34.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0addd0 secure :-1 s=READY pgs=941 cs=0 l=1 rev1=1 crypto rx=0x7fdfec02b950 tx=0x7fdfec0046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 shutdown_connections 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdfe006b4d0 0x7fdfe006d990 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdffc15be10 0x7fdffc0cad40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdffc0aef20 0x7fdffc0addd0 unknown :-1 s=CLOSED pgs=941 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 --2- 172.21.15.86:0/3802206859 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdffc06d330 0x7fdffc0b09d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.860+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 >> 172.21.15.86:0/3802206859 conn(0x7fdffc06afc0 msgr2=0x7fdffc157120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.861+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 shutdown_connections 2024-03-06T07:31:34.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:34.861+0000 7fe002c00640 1 -- 172.21.15.86:0/3802206859 wait complete. 2024-03-06T07:31:34.881 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:31:34.882 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:31:35.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.079+0000 7f8517e00640 1 Processor -- start 2024-03-06T07:31:35.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.079+0000 7f8517e00640 1 -- start start 2024-03-06T07:31:35.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.082+0000 7f8517e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f851006b480 0x7f851015edd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.082+0000 7f8517e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f85100b2a40 con 0x7f851006c4a0 2024-03-06T07:31:35.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.082+0000 7f8517e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f851015eb80 con 0x7f851015e7a0 2024-03-06T07:31:35.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.082+0000 7f8517e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f85100b14d0 con 0x7f851006b480 2024-03-06T07:31:35.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.082+0000 7f8515e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f851006c4a0 0x7f851015f310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37998/0 (socket says 172.21.15.86:37998) 2024-03-06T07:31:35.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.082+0000 7f8515e00640 1 -- 172.21.15.86:0/796894680 learned_addr learned my addr 172.21.15.86:0/796894680 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.082+0000 7f8514a00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f851006b480 0x7f851015edd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 252265879 0 0) 0x7f85100b2a40 con 0x7f851006c4a0 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f84f8005730 con 0x7f851006c4a0 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1753114285 0 0) 0x7f84f8005730 con 0x7f851006c4a0 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f851006b480 msgr2=0x7f851015edd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f851006b480 0x7f851015edd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 >> v1:172.21.15.194:6789/0 conn(0x7f851015e7a0 legacy=0x7f851006bd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8510073050 con 0x7f851006c4a0 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f85080019f0 con 0x7f851006c4a0 2024-03-06T07:31:35.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 >> v1:172.21.15.86:6789/0 conn(0x7f851006c4a0 legacy=0x7f851015f310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84f8005bf0 0x7f84f8006010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84f8006a10 0x7f84f8006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f851006b480 0x7f84f8007410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85100b14d0 con 0x7f84f8006a10 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84f800dd30 con 0x7f84f8005bf0 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.083+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f84f800df30 con 0x7f851006b480 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.084+0000 7f8514a00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84f8006a10 0x7f84f8006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.084+0000 7f8515400640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84f8005bf0 0x7f84f8006010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.084+0000 7f8514a00640 1 -- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f851006b480 msgr2=0x7f84f8007410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.084+0000 7f8514a00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f851006b480 0x7f84f8007410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.084+0000 7f8514a00640 1 -- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84f8005bf0 msgr2=0x7f84f8006010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.084+0000 7f8514a00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84f8005bf0 0x7f84f8006010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.084+0000 7f8514a00640 1 -- 172.21.15.86:0/796894680 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84f8005730 con 0x7f84f8006a10 2024-03-06T07:31:35.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.085+0000 7f8514a00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84f8006a10 0x7f84f8006e90 secure :-1 s=READY pgs=943 cs=0 l=1 rev1=1 crypto rx=0x7f84fc009a90 tx=0x7f84f8007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bd23da6d0525b9b5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:35.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.085+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84fc039070 con 0x7f84f8006a10 2024-03-06T07:31:35.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.085+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f84fc02c6b0 con 0x7f84f8006a10 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f850fe00640 1 -- 172.21.15.86:0/796894680 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84fc009690 con 0x7f84f8006a10 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f8517e00640 1 -- 172.21.15.86:0/796894680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84f8006a10 msgr2=0x7f84f8006e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f8517e00640 1 --2- 172.21.15.86:0/796894680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84f8006a10 0x7f84f8006e90 secure :-1 s=READY pgs=943 cs=0 l=1 rev1=1 crypto rx=0x7f84fc009a90 tx=0x7f84f8007a40 comp rx=0 tx=0).stop 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f8515e00640 1 -- 172.21.15.86:0/796894680 reap_dead start 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f8517e00640 1 -- 172.21.15.86:0/796894680 shutdown_connections 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f8517e00640 1 -- 172.21.15.86:0/796894680 >> 172.21.15.86:0/796894680 conn(0x7f85100b1e80 msgr2=0x7f85100b2290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f8517e00640 1 -- 172.21.15.86:0/796894680 shutdown_connections 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.086+0000 7f8517e00640 1 -- 172.21.15.86:0/796894680 wait complete. 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.088+0000 7f8517e00640 1 Processor -- start 2024-03-06T07:31:35.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.088+0000 7f8517e00640 1 -- start start 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8517e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84f8006a10 0x7f85100998d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8517e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85100c97a0 0x7f85100c9c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8517e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f85100c4ea0 0x7f85100c5320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8517e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85100b19f0 con 0x7f85100c97a0 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8517e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8510073360 con 0x7f85100c4ea0 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8517e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f85100b1b70 con 0x7f84f8006a10 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8514a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85100c97a0 0x7f85100c9c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8514a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85100c97a0 0x7f85100c9c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42312/0 (socket says 172.21.15.86:42312) 2024-03-06T07:31:35.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.089+0000 7f8514a00640 1 -- 172.21.15.86:0/1318638108 learned_addr learned my addr 172.21.15.86:0/1318638108 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:35.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f8514a00640 1 -- 172.21.15.86:0/1318638108 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84f8006a10 msgr2=0x7f85100998d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:35.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f8514a00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84f8006a10 0x7f85100998d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f8514a00640 1 -- 172.21.15.86:0/1318638108 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f85100c4ea0 msgr2=0x7f85100c5320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f8514a00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f85100c4ea0 0x7f85100c5320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f8514a00640 1 -- 172.21.15.86:0/1318638108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f851015b9a0 con 0x7f85100c97a0 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f8514a00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85100c97a0 0x7f85100c9c20 secure :-1 s=READY pgs=944 cs=0 l=1 rev1=1 crypto rx=0x7f84fc000c00 tx=0x7f84fc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f850d600640 1 -- 172.21.15.86:0/1318638108 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84fc039050 con 0x7f85100c97a0 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f850d600640 1 -- 172.21.15.86:0/1318638108 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f84fc045070 con 0x7f85100c97a0 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.090+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f851015bc80 con 0x7f85100c97a0 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.091+0000 7f850d600640 1 -- 172.21.15.86:0/1318638108 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84fc0044e0 con 0x7f85100c97a0 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.091+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f851015c240 con 0x7f85100c97a0 2024-03-06T07:31:35.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.092+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f851015c620 con 0x7f85100c97a0 2024-03-06T07:31:35.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.093+0000 7f850d600640 1 -- 172.21.15.86:0/1318638108 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f84fc003a50 con 0x7f85100c97a0 2024-03-06T07:31:35.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.094+0000 7f850d600640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84f006b500 0x7f84f006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:35.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.094+0000 7f850d600640 1 -- 172.21.15.86:0/1318638108 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f84fc0ac3f0 con 0x7f85100c97a0 2024-03-06T07:31:35.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.094+0000 7f8515400640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84f006b500 0x7f84f006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:35.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.095+0000 7f8515400640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84f006b500 0x7f84f006d9c0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f8504007fd0 tx=0x7f8504009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:35.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.102+0000 7f850d600640 1 -- 172.21.15.86:0/1318638108 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f851015c620 con 0x7f85100c97a0 2024-03-06T07:31:35.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.342+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f8510068220 con 0x7f85100c97a0 2024-03-06T07:31:35.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.869+0000 7f850d600640 1 -- 172.21.15.86:0/1318638108 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v50) v1 ==== 122+0+0 (secure 0 0 0) 0x7f8510068220 con 0x7f85100c97a0 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84f006b500 msgr2=0x7f84f006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84f006b500 0x7f84f006d9c0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f8504007fd0 tx=0x7f8504009040 comp rx=0 tx=0).stop 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85100c97a0 msgr2=0x7f85100c9c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85100c97a0 0x7f85100c9c20 secure :-1 s=READY pgs=944 cs=0 l=1 rev1=1 crypto rx=0x7f84fc000c00 tx=0x7f84fc004730 comp rx=0 tx=0).stop 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 shutdown_connections 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84f006b500 0x7f84f006d9c0 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f85100c4ea0 0x7f85100c5320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f85100c97a0 0x7f85100c9c20 unknown :-1 s=CLOSED pgs=944 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 --2- 172.21.15.86:0/1318638108 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84f8006a10 0x7f85100998d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 >> 172.21.15.86:0/1318638108 conn(0x7f85100b1e80 msgr2=0x7f85100cab30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:35.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 shutdown_connections 2024-03-06T07:31:35.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:35.872+0000 7f8517e00640 1 -- 172.21.15.86:0/1318638108 wait complete. 2024-03-06T07:31:35.897 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:31:35.898 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.093+0000 7f3cdb200640 1 Processor -- start 2024-03-06T07:31:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.093+0000 7f3cdb200640 1 -- start start 2024-03-06T07:31:36.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cdb200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc161580 0x7f3cdc0d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cdb200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3cdc0d2a70 0x7f3cdc0b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cdb200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3cdc161960 con 0x7f3cdc0d1e90 2024-03-06T07:31:36.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cdb200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3cdc0cf6f0 con 0x7f3cdc0d2a70 2024-03-06T07:31:36.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cdb200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3cdc0cf9d0 con 0x7f3cdc161580 2024-03-06T07:31:36.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cda800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3cdc0d2a70 0x7f3cdc0b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cda800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3cdc0d2a70 0x7f3cdc0b3ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53224/0 (socket says 172.21.15.86:53224) 2024-03-06T07:31:36.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cda800640 1 -- 172.21.15.86:0/290393320 learned_addr learned my addr 172.21.15.86:0/290393320 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:36.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.095+0000 7f3cd9e00640 1 --2- 172.21.15.86:0/290393320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc161580 0x7f3cdc0d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.096+0000 7f3cda800640 1 -- 172.21.15.86:0/290393320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc161580 msgr2=0x7f3cdc0d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:36.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.096+0000 7f3cda800640 1 --2- 172.21.15.86:0/290393320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc161580 0x7f3cdc0d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.096+0000 7f3cda800640 1 -- 172.21.15.86:0/290393320 >> v1:172.21.15.86:6789/0 conn(0x7f3cdc0d1e90 legacy=0x7f3cdc0d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.096+0000 7f3cd8a00640 1 -- 172.21.15.86:0/290393320 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4004833549 0 0) 0x7f3cdc161960 con 0x7f3cdc0d1e90 2024-03-06T07:31:36.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.096+0000 7f3cda800640 1 -- 172.21.15.86:0/290393320 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3cdc0cfcb0 con 0x7f3cdc0d2a70 2024-03-06T07:31:36.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.096+0000 7f3cd9e00640 1 --2- 172.21.15.86:0/290393320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc161580 0x7f3cdc0d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:36.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.097+0000 7f3cda800640 1 --2- 172.21.15.86:0/290393320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc0d2a70 0x7f3cdc0b3ec0 secure :-1 s=READY pgs=946 cs=0 l=1 rev1=1 crypto rx=0x7f3cd4009e50 tx=0x7f3cd402bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=34de2d7a797f8e70 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.097+0000 7f3cd8a00640 1 -- 172.21.15.86:0/290393320 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cd402cb00 con 0x7f3cdc0d2a70 2024-03-06T07:31:36.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.097+0000 7f3cd8a00640 1 -- 172.21.15.86:0/290393320 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3cd402cca0 con 0x7f3cdc0d2a70 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.097+0000 7f3cd8a00640 1 -- 172.21.15.86:0/290393320 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cd4034610 con 0x7f3cdc0d2a70 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 -- 172.21.15.86:0/290393320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc0d2a70 msgr2=0x7f3cdc0b3ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 --2- 172.21.15.86:0/290393320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc0d2a70 0x7f3cdc0b3ec0 secure :-1 s=READY pgs=946 cs=0 l=1 rev1=1 crypto rx=0x7f3cd4009e50 tx=0x7f3cd402bf60 comp rx=0 tx=0).stop 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 -- 172.21.15.86:0/290393320 shutdown_connections 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 --2- 172.21.15.86:0/290393320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc0d2a70 0x7f3cdc0b3ec0 unknown :-1 s=CLOSED pgs=946 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 --2- 172.21.15.86:0/290393320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc161580 0x7f3cdc0d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 -- 172.21.15.86:0/290393320 >> 172.21.15.86:0/290393320 conn(0x7f3cdc0d3d80 msgr2=0x7f3cdc0d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 -- 172.21.15.86:0/290393320 shutdown_connections 2024-03-06T07:31:36.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.098+0000 7f3cdb200640 1 -- 172.21.15.86:0/290393320 wait complete. 2024-03-06T07:31:36.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.100+0000 7f3cdb200640 1 Processor -- start 2024-03-06T07:31:36.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.100+0000 7f3cdb200640 1 -- start start 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cdb200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cdc0d1e90 0x7f3cdc158bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cdb200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc0d2a70 0x7f3cdc0b75a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cdb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc161580 0x7f3cdc0b90e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cdb200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3cdc0cf3a0 con 0x7f3cdc161580 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cdb200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3cdc0b39a0 con 0x7f3cdc0d1e90 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cdb200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3cdc0b4400 con 0x7f3cdc0d2a70 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc161580 0x7f3cdc0b90e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc161580 0x7f3cdc0b90e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53226/0 (socket says 172.21.15.86:53226) 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 -- 172.21.15.86:0/1311414033 learned_addr learned my addr 172.21.15.86:0/1311414033 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cd9e00640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cdc0d1e90 0x7f3cdc158bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 -- 172.21.15.86:0/1311414033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc0d2a70 msgr2=0x7f3cdc0b75a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc0d2a70 0x7f3cdc0b75a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 -- 172.21.15.86:0/1311414033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cdc0d1e90 msgr2=0x7f3cdc158bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cdc0d1e90 0x7f3cdc158bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.101+0000 7f3cda800640 1 -- 172.21.15.86:0/1311414033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3cdc0b7dd0 con 0x7f3cdc161580 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.102+0000 7f3cda800640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc161580 0x7f3cdc0b90e0 secure :-1 s=READY pgs=947 cs=0 l=1 rev1=1 crypto rx=0x7f3cd4036040 tx=0x7f3cd4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.102+0000 7f3cd2000640 1 -- 172.21.15.86:0/1311414033 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cd4043030 con 0x7f3cdc161580 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.102+0000 7f3cd2000640 1 -- 172.21.15.86:0/1311414033 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3cd402ce50 con 0x7f3cdc161580 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.102+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3cdc0b8320 con 0x7f3cdc161580 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.102+0000 7f3cd2000640 1 -- 172.21.15.86:0/1311414033 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cd403e450 con 0x7f3cdc161580 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.102+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3cdc0b88e0 con 0x7f3cdc161580 2024-03-06T07:31:36.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.103+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3cdc04f330 con 0x7f3cdc161580 2024-03-06T07:31:36.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.104+0000 7f3cd2000640 1 -- 172.21.15.86:0/1311414033 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3cd4046070 con 0x7f3cdc161580 2024-03-06T07:31:36.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.105+0000 7f3cd2000640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3cbc06b4d0 0x7f3cbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.105+0000 7f3cd2000640 1 -- 172.21.15.86:0/1311414033 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3cd40ac5e0 con 0x7f3cdc161580 2024-03-06T07:31:36.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.105+0000 7f3cd9e00640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3cbc06b4d0 0x7f3cbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.106+0000 7f3cd9e00640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3cbc06b4d0 0x7f3cbc06d990 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f3ccc00b840 tx=0x7f3ccc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.110+0000 7f3cd2000640 1 -- 172.21.15.86:0/1311414033 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3cd404d070 con 0x7f3cdc161580 2024-03-06T07:31:36.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.341+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3cdc0d3b60 con 0x7f3cdc161580 2024-03-06T07:31:36.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.342+0000 7f3cd2000640 1 -- 172.21.15.86:0/1311414033 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f3cd40333a0 con 0x7f3cdc161580 2024-03-06T07:31:36.343 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:36.344 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:31:32.095887+0000","modified":"2024-03-06T07:31:35.343932+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6609},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6609":{"gid":6609,"name":"c","rank":0,"incarnation":47,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/255958031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":255958031},{"type":"v1","addr":"172.21.15.86:6837","nonce":255958031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6609,"qdb_cluster":[6609]},"id":6}]} 2024-03-06T07:31:36.344 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 50 2024-03-06T07:31:36.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3cbc06b4d0 msgr2=0x7f3cbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3cbc06b4d0 0x7f3cbc06d990 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f3ccc00b840 tx=0x7f3ccc008040 comp rx=0 tx=0).stop 2024-03-06T07:31:36.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc161580 msgr2=0x7f3cdc0b90e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc161580 0x7f3cdc0b90e0 secure :-1 s=READY pgs=947 cs=0 l=1 rev1=1 crypto rx=0x7f3cd4036040 tx=0x7f3cd4004a40 comp rx=0 tx=0).stop 2024-03-06T07:31:36.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 shutdown_connections 2024-03-06T07:31:36.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3cbc06b4d0 0x7f3cbc06d990 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3cdc161580 0x7f3cdc0b90e0 unknown :-1 s=CLOSED pgs=947 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3cdc0d2a70 0x7f3cdc0b75a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 --2- 172.21.15.86:0/1311414033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3cdc0d1e90 0x7f3cdc158bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 >> 172.21.15.86:0/1311414033 conn(0x7f3cdc0d3d80 msgr2=0x7f3cdc0cff10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:36.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 shutdown_connections 2024-03-06T07:31:36.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.344+0000 7f3cdb200640 1 -- 172.21.15.86:0/1311414033 wait complete. 2024-03-06T07:31:36.363 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:36.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.555+0000 7f0d14a00640 1 Processor -- start 2024-03-06T07:31:36.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.555+0000 7f0d14a00640 1 -- start start 2024-03-06T07:31:36.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.556+0000 7f0d14a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0d100cf370 0x7f0d1006d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.556+0000 7f0d14a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d100cc2a0 0x7f0d100cc720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.556+0000 7f0d14a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0d100b2b30 con 0x7f0d1015a940 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.556+0000 7f0d14a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d100b28e0 con 0x7f0d100cc2a0 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.556+0000 7f0d14a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d100b2e10 con 0x7f0d100cf370 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d100cc2a0 0x7f0d100cc720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d100cc2a0 0x7f0d100cc720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53240/0 (socket says 172.21.15.86:53240) 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0f200640 1 -- 172.21.15.86:0/2540830165 learned_addr learned my addr 172.21.15.86:0/2540830165 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0de00640 1 --2- 172.21.15.86:0/2540830165 >> v2:172.21.15.194:3300/0 conn(0x7f0d100cf370 0x7f0d1006d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0f200640 1 -- 172.21.15.86:0/2540830165 >> v2:172.21.15.194:3300/0 conn(0x7f0d100cf370 msgr2=0x7f0d1006d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:36.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0f200640 1 --2- 172.21.15.86:0/2540830165 >> v2:172.21.15.194:3300/0 conn(0x7f0d100cf370 0x7f0d1006d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0f200640 1 -- 172.21.15.86:0/2540830165 >> v1:172.21.15.194:6789/0 conn(0x7f0d1015a940 legacy=0x7f0d100b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:36.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.557+0000 7f0d0f200640 1 -- 172.21.15.86:0/2540830165 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d1006ca50 con 0x7f0d100cc2a0 2024-03-06T07:31:36.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.558+0000 7f0d0f200640 1 --2- 172.21.15.86:0/2540830165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cc2a0 0x7f0d100cc720 secure :-1 s=READY pgs=948 cs=0 l=1 rev1=1 crypto rx=0x7f0d10068400 tx=0x7f0d0802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4acd4042a737ee0c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d0d400640 1 -- 172.21.15.86:0/2540830165 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d0802ca80 con 0x7f0d100cc2a0 2024-03-06T07:31:36.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d0d400640 1 -- 172.21.15.86:0/2540830165 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d0802cc20 con 0x7f0d100cc2a0 2024-03-06T07:31:36.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d0d400640 1 -- 172.21.15.86:0/2540830165 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d080343f0 con 0x7f0d100cc2a0 2024-03-06T07:31:36.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d14a00640 1 -- 172.21.15.86:0/2540830165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cc2a0 msgr2=0x7f0d100cc720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2540830165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cc2a0 0x7f0d100cc720 secure :-1 s=READY pgs=948 cs=0 l=1 rev1=1 crypto rx=0x7f0d10068400 tx=0x7f0d0802bee0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d14a00640 1 -- 172.21.15.86:0/2540830165 shutdown_connections 2024-03-06T07:31:36.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2540830165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cc2a0 0x7f0d100cc720 unknown :-1 s=CLOSED pgs=948 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.559+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2540830165 >> v2:172.21.15.194:3300/0 conn(0x7f0d100cf370 0x7f0d1006d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.560+0000 7f0d14a00640 1 -- 172.21.15.86:0/2540830165 >> 172.21.15.86:0/2540830165 conn(0x7f0d1006afc0 msgr2=0x7f0d1006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:36.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.560+0000 7f0d14a00640 1 -- 172.21.15.86:0/2540830165 shutdown_connections 2024-03-06T07:31:36.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.560+0000 7f0d14a00640 1 -- 172.21.15.86:0/2540830165 wait complete. 2024-03-06T07:31:36.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.561+0000 7f0d14a00640 1 Processor -- start 2024-03-06T07:31:36.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.561+0000 7f0d14a00640 1 -- start start 2024-03-06T07:31:36.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d14a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d100cc2a0 0x7f0d100b0ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d14a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cf370 0x7f0d100cdc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d14a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d1015a940 0x7f0d100ce1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d14a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d1006d790 con 0x7f0d100cf370 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d14a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d100b30f0 con 0x7f0d1015a940 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d14a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d10147c70 con 0x7f0d100cc2a0 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d0de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cf370 0x7f0d100cdc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d0de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cf370 0x7f0d100cdc90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53244/0 (socket says 172.21.15.86:53244) 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.562+0000 7f0d0de00640 1 -- 172.21.15.86:0/2482546714 learned_addr learned my addr 172.21.15.86:0/2482546714 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d0f200640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d1015a940 0x7f0d100ce1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d0de00640 1 -- 172.21.15.86:0/2482546714 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d100cc2a0 msgr2=0x7f0d100b0ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d0de00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d100cc2a0 0x7f0d100b0ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d0de00640 1 -- 172.21.15.86:0/2482546714 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d1015a940 msgr2=0x7f0d100ce1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d0de00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d1015a940 0x7f0d100ce1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d0de00640 1 -- 172.21.15.86:0/2482546714 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d10071b40 con 0x7f0d100cf370 2024-03-06T07:31:36.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d0de00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cf370 0x7f0d100cdc90 secure :-1 s=READY pgs=949 cs=0 l=1 rev1=1 crypto rx=0x7f0cfc00d830 tx=0x7f0cfc00dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d06a00640 1 -- 172.21.15.86:0/2482546714 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cfc00b6c0 con 0x7f0d100cf370 2024-03-06T07:31:36.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d06a00640 1 -- 172.21.15.86:0/2482546714 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0cfc019070 con 0x7f0d100cf370 2024-03-06T07:31:36.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.563+0000 7f0d06a00640 1 -- 172.21.15.86:0/2482546714 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cfc00ba80 con 0x7f0d100cf370 2024-03-06T07:31:36.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.564+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d10071e20 con 0x7f0d100cf370 2024-03-06T07:31:36.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.564+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d100723e0 con 0x7f0d100cf370 2024-03-06T07:31:36.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.565+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d1004f330 con 0x7f0d100cf370 2024-03-06T07:31:36.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.566+0000 7f0d06a00640 1 -- 172.21.15.86:0/2482546714 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0cfc01e460 con 0x7f0d100cf370 2024-03-06T07:31:36.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.567+0000 7f0d06a00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cf006b4d0 0x7f0cf006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.567+0000 7f0d06a00640 1 -- 172.21.15.86:0/2482546714 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0cfc08a950 con 0x7f0d100cf370 2024-03-06T07:31:36.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.567+0000 7f0d0e800640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cf006b4d0 0x7f0cf006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.568+0000 7f0d0e800640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cf006b4d0 0x7f0cf006d990 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f0d00006fd0 tx=0x7f0d00006bc0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.571+0000 7f0d06a00640 1 -- 172.21.15.86:0/2482546714 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0cfc059730 con 0x7f0d100cf370 2024-03-06T07:31:36.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.767+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0d100ae640 con 0x7f0d100cf370 2024-03-06T07:31:36.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.769+0000 7f0d06a00640 1 -- 172.21.15.86:0/2482546714 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+16672 (secure 0 0 0) 0x7f0cfc059030 con 0x7f0d100cf370 2024-03-06T07:31:36.771 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:36.771 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":83,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:33.657039+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:31:29.878719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-06T07:31:31.115332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cf006b4d0 msgr2=0x7f0cf006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cf006b4d0 0x7f0cf006d990 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f0d00006fd0 tx=0x7f0d00006bc0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cf370 msgr2=0x7f0d100cdc90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cf370 0x7f0d100cdc90 secure :-1 s=READY pgs=949 cs=0 l=1 rev1=1 crypto rx=0x7f0cfc00d830 tx=0x7f0cfc00dcf0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 shutdown_connections 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0cf006b4d0 0x7f0cf006d990 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d1015a940 0x7f0d100ce1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d100cf370 0x7f0d100cdc90 unknown :-1 s=CLOSED pgs=949 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 --2- 172.21.15.86:0/2482546714 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d100cc2a0 0x7f0d100b0ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 >> 172.21.15.86:0/2482546714 conn(0x7f0d1006afc0 msgr2=0x7f0d100d02a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 shutdown_connections 2024-03-06T07:31:36.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.772+0000 7f0d14a00640 1 -- 172.21.15.86:0/2482546714 wait complete. 2024-03-06T07:31:36.792 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:31:36.792 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:31:36.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.980+0000 7f3303000640 1 Processor -- start 2024-03-06T07:31:36.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.980+0000 7f3303000640 1 -- start start 2024-03-06T07:31:36.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.982+0000 7f3303000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f32fc144c00 0x7f32fc06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.982+0000 7f3303000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f32fc06ee70 con 0x7f32fc15fe60 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.982+0000 7f3303000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f32fc06cef0 con 0x7f32fc15f9d0 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.982+0000 7f3303000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f32fc06d1d0 con 0x7f32fc144c00 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.982+0000 7f3301000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f32fc15fe60 0x7f32fc0ce7b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55872/0 (socket says 172.21.15.86:55872) 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.982+0000 7f3301000640 1 -- 172.21.15.86:0/2201838101 learned_addr learned my addr 172.21.15.86:0/2201838101 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1648874244 0 0) 0x7f32f4001490 con 0x7f32fc15fe60 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fbe00640 1 --2- 172.21.15.86:0/2201838101 >> v2:172.21.15.194:3300/0 conn(0x7f32fc144c00 0x7f32fc06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f32e0003520 con 0x7f32fc15fe60 2024-03-06T07:31:36.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 615569387 0 0) 0x7f32e0003520 con 0x7f32fc15fe60 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 >> v2:172.21.15.194:3300/0 conn(0x7f32fc144c00 msgr2=0x7f32fc06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 --2- 172.21.15.86:0/2201838101 >> v2:172.21.15.194:3300/0 conn(0x7f32fc144c00 0x7f32fc06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 >> v1:172.21.15.194:6789/0 conn(0x7f32fc15f9d0 legacy=0x7f32fc0ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32fc06d4b0 con 0x7f32fc15fe60 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f32f4001bc0 con 0x7f32fc15fe60 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 >> v1:172.21.15.86:6789/0 conn(0x7f32fc15fe60 legacy=0x7f32fc0ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32e0003a00 0x7f32e0003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32e0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32e0005210 0x7f32e000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32fc06d1d0 con 0x7f32e0004800 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32e000c140 con 0x7f32e0003a00 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.983+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32e000c340 con 0x7f32e0005210 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32f3400640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32e0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32fbe00640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32e0003a00 0x7f32e0003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32f3400640 1 -- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32e0005210 msgr2=0x7f32e000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32f3400640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32e0005210 0x7f32e000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32f3400640 1 -- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32e0003a00 msgr2=0x7f32e0003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32f3400640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32e0003a00 0x7f32e0003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32f3400640 1 -- 172.21.15.86:0/2201838101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32e0003520 con 0x7f32e0004800 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f32fbe00640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32e0003a00 0x7f32e0003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.984+0000 7f3301000640 1 -- 172.21.15.86:0/2201838101 reap_dead start 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.985+0000 7f32f3400640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32e0004c80 secure :-1 s=READY pgs=951 cs=0 l=1 rev1=1 crypto rx=0x7f32e4003d80 tx=0x7f32e0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a62263cf9f06c015 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.985+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32e4037070 con 0x7f32e0004800 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.985+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f32e4032600 con 0x7f32e0004800 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.985+0000 7f32fb400640 1 -- 172.21.15.86:0/2201838101 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32e4003980 con 0x7f32e0004800 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.986+0000 7f3303000640 1 -- 172.21.15.86:0/2201838101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 msgr2=0x7f32e0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.986+0000 7f3303000640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32e0004c80 secure :-1 s=READY pgs=951 cs=0 l=1 rev1=1 crypto rx=0x7f32e4003d80 tx=0x7f32e0005830 comp rx=0 tx=0).stop 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.986+0000 7f3303000640 1 -- 172.21.15.86:0/2201838101 shutdown_connections 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.986+0000 7f3303000640 1 --2- 172.21.15.86:0/2201838101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32e0004c80 unknown :-1 s=CLOSED pgs=951 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.986+0000 7f3303000640 1 -- 172.21.15.86:0/2201838101 >> 172.21.15.86:0/2201838101 conn(0x7f32fc0b1730 msgr2=0x7f32fc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:36.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.986+0000 7f3303000640 1 -- 172.21.15.86:0/2201838101 shutdown_connections 2024-03-06T07:31:36.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.986+0000 7f3303000640 1 -- 172.21.15.86:0/2201838101 wait complete. 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.988+0000 7f3303000640 1 Processor -- start 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.988+0000 7f3303000640 1 -- start start 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f3303000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32fc15b890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f3303000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32fc15ed40 0x7f32fc15f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f3303000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32fc141890 0x7f32fc141d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f3303000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32fc160730 con 0x7f32e0004800 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f3303000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32fc160270 con 0x7f32fc141890 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f3303000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32fc06cb50 con 0x7f32fc15ed40 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f32fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32fc15b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f32fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32fc15b890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53262/0 (socket says 172.21.15.86:53262) 2024-03-06T07:31:36.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.989+0000 7f32fbe00640 1 -- 172.21.15.86:0/940958313 learned_addr learned my addr 172.21.15.86:0/940958313 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32fbe00640 1 -- 172.21.15.86:0/940958313 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32fc15ed40 msgr2=0x7f32fc15f1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32fbe00640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32fc15ed40 0x7f32fc15f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f3301000640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32fc141890 0x7f32fc141d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32fbe00640 1 -- 172.21.15.86:0/940958313 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32fc141890 msgr2=0x7f32fc141d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32fbe00640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32fc141890 0x7f32fc141d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32fbe00640 1 -- 172.21.15.86:0/940958313 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32fc142640 con 0x7f32e0004800 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32fbe00640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32fc15b890 secure :-1 s=READY pgs=952 cs=0 l=1 rev1=1 crypto rx=0x7f32ec00e840 tx=0x7f32ec00ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32f8c00640 1 -- 172.21.15.86:0/940958313 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32ec017070 con 0x7f32e0004800 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32f8c00640 1 -- 172.21.15.86:0/940958313 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f32ec00c8d0 con 0x7f32e0004800 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f32f8c00640 1 -- 172.21.15.86:0/940958313 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32ec00f040 con 0x7f32e0004800 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.990+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32fc15c6f0 con 0x7f32e0004800 2024-03-06T07:31:36.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.991+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f32fc15bdd0 con 0x7f32e0004800 2024-03-06T07:31:36.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.992+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f32fc06da70 con 0x7f32e0004800 2024-03-06T07:31:36.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.993+0000 7f32f8c00640 1 -- 172.21.15.86:0/940958313 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f32ec01a070 con 0x7f32e0004800 2024-03-06T07:31:36.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.993+0000 7f32f8c00640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f32d806b500 0x7f32d806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:36.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.994+0000 7f32f8c00640 1 -- 172.21.15.86:0/940958313 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f32ec089e90 con 0x7f32e0004800 2024-03-06T07:31:36.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.994+0000 7f32f3400640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f32d806b500 0x7f32d806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:36.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.994+0000 7f32f3400640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f32d806b500 0x7f32d806d9c0 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f32e4034000 tx=0x7f32e402f000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:36.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:36.998+0000 7f32f8c00640 1 -- 172.21.15.86:0/940958313 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f32fc06da70 con 0x7f32e0004800 2024-03-06T07:31:37.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.192+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f32fc04f330 con 0x7f32d806b500 2024-03-06T07:31:37.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.203+0000 7f32f8c00640 1 -- 172.21.15.86:0/940958313 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252048 (secure 0 0 0) 0x7f32fc04f330 con 0x7f32d806b500 2024-03-06T07:31:37.207 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:37.210 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:31:37.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f32d806b500 msgr2=0x7f32d806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f32d806b500 0x7f32d806d9c0 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f32e4034000 tx=0x7f32e402f000 comp rx=0 tx=0).stop 2024-03-06T07:31:37.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 msgr2=0x7f32fc15b890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32fc15b890 secure :-1 s=READY pgs=952 cs=0 l=1 rev1=1 crypto rx=0x7f32ec00e840 tx=0x7f32ec00ed00 comp rx=0 tx=0).stop 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 shutdown_connections 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f32d806b500 0x7f32d806d9c0 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32fc141890 0x7f32fc141d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32fc15ed40 0x7f32fc15f1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.209+0000 7f3303000640 1 --2- 172.21.15.86:0/940958313 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f32e0004800 0x7f32fc15b890 unknown :-1 s=CLOSED pgs=952 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.210+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 >> 172.21.15.86:0/940958313 conn(0x7f32fc0b1730 msgr2=0x7f32fc0ce0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.210+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 shutdown_connections 2024-03-06T07:31:37.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.210+0000 7f3303000640 1 -- 172.21.15.86:0/940958313 wait complete. 2024-03-06T07:31:37.230 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":215,"stamp":"2024-03-06T07:31:36.934215+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":164,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":258596,"kb_used_data":9100,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754716124,"statfs":{"total":773094113280,"available":772829310976,"internally_reserved":0,"allocated":9318400,"data_stored":4595080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001549"},"pg_stats":[{"pgid":"13.3f","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687923+0000","last_change":"2024-03-06T07:31:32.681418+0000","last_active":"2024-03-06T07:31:32.687923+0000","last_peered":"2024-03-06T07:31:32.687923+0000","last_clean":"2024-03-06T07:31:32.687923+0000","last_became_active":"2024-03-06T07:31:32.681101+0000","last_became_peered":"2024-03-06T07:31:32.681101+0000","last_unstale":"2024-03-06T07:31:32.687923+0000","last_undegraded":"2024-03-06T07:31:32.687923+0000","last_fullsized":"2024-03-06T07:31:32.687923+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:52.308934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3e","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691654+0000","last_change":"2024-03-06T07:31:32.685120+0000","last_active":"2024-03-06T07:31:32.691654+0000","last_peered":"2024-03-06T07:31:32.691654+0000","last_clean":"2024-03-06T07:31:32.691654+0000","last_became_active":"2024-03-06T07:31:32.684664+0000","last_became_peered":"2024-03-06T07:31:32.684664+0000","last_unstale":"2024-03-06T07:31:32.691654+0000","last_undegraded":"2024-03-06T07:31:32.691654+0000","last_fullsized":"2024-03-06T07:31:32.691654+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:16:35.428940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.3d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868044+0000","last_change":"2024-03-06T07:31:32.685318+0000","last_active":"2024-03-06T07:31:34.868044+0000","last_peered":"2024-03-06T07:31:34.868044+0000","last_clean":"2024-03-06T07:31:34.868044+0000","last_became_active":"2024-03-06T07:31:32.684802+0000","last_became_peered":"2024-03-06T07:31:32.684802+0000","last_unstale":"2024-03-06T07:31:34.868044+0000","last_undegraded":"2024-03-06T07:31:34.868044+0000","last_fullsized":"2024-03-06T07:31:34.868044+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:46:33.261291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.3c","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108061+0000","last_change":"2024-03-06T07:31:32.674062+0000","last_active":"2024-03-06T07:31:34.108061+0000","last_peered":"2024-03-06T07:31:34.108061+0000","last_clean":"2024-03-06T07:31:34.108061+0000","last_became_active":"2024-03-06T07:31:32.670203+0000","last_became_peered":"2024-03-06T07:31:32.670203+0000","last_unstale":"2024-03-06T07:31:34.108061+0000","last_undegraded":"2024-03-06T07:31:34.108061+0000","last_fullsized":"2024-03-06T07:31:34.108061+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:53:07.783439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.3b","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672549+0000","last_change":"2024-03-06T07:31:32.664938+0000","last_active":"2024-03-06T07:31:33.672549+0000","last_peered":"2024-03-06T07:31:33.672549+0000","last_clean":"2024-03-06T07:31:33.672549+0000","last_became_active":"2024-03-06T07:31:32.655854+0000","last_became_peered":"2024-03-06T07:31:32.655854+0000","last_unstale":"2024-03-06T07:31:33.672549+0000","last_undegraded":"2024-03-06T07:31:33.672549+0000","last_fullsized":"2024-03-06T07:31:33.672549+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:56:26.632423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3a","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687093+0000","last_change":"2024-03-06T07:31:32.681677+0000","last_active":"2024-03-06T07:31:32.687093+0000","last_peered":"2024-03-06T07:31:32.687093+0000","last_clean":"2024-03-06T07:31:32.687093+0000","last_became_active":"2024-03-06T07:31:32.681345+0000","last_became_peered":"2024-03-06T07:31:32.681345+0000","last_unstale":"2024-03-06T07:31:32.687093+0000","last_undegraded":"2024-03-06T07:31:32.687093+0000","last_fullsized":"2024-03-06T07:31:32.687093+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:43.487650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.39","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107357+0000","last_change":"2024-03-06T07:31:32.677326+0000","last_active":"2024-03-06T07:31:34.107357+0000","last_peered":"2024-03-06T07:31:34.107357+0000","last_clean":"2024-03-06T07:31:34.107357+0000","last_became_active":"2024-03-06T07:31:32.676744+0000","last_became_peered":"2024-03-06T07:31:32.676744+0000","last_unstale":"2024-03-06T07:31:34.107357+0000","last_undegraded":"2024-03-06T07:31:34.107357+0000","last_fullsized":"2024-03-06T07:31:34.107357+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:55:26.072069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.38","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688837+0000","last_change":"2024-03-06T07:31:32.687329+0000","last_active":"2024-03-06T07:31:32.688837+0000","last_peered":"2024-03-06T07:31:32.688837+0000","last_clean":"2024-03-06T07:31:32.688837+0000","last_became_active":"2024-03-06T07:31:32.686450+0000","last_became_peered":"2024-03-06T07:31:32.686450+0000","last_unstale":"2024-03-06T07:31:32.688837+0000","last_undegraded":"2024-03-06T07:31:32.688837+0000","last_fullsized":"2024-03-06T07:31:32.688837+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:32:28.951977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.37","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.689637+0000","last_change":"2024-03-06T07:31:32.688585+0000","last_active":"2024-03-06T07:31:32.689637+0000","last_peered":"2024-03-06T07:31:32.689637+0000","last_clean":"2024-03-06T07:31:32.689637+0000","last_became_active":"2024-03-06T07:31:32.687930+0000","last_became_peered":"2024-03-06T07:31:32.687930+0000","last_unstale":"2024-03-06T07:31:32.689637+0000","last_undegraded":"2024-03-06T07:31:32.689637+0000","last_fullsized":"2024-03-06T07:31:32.689637+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:52:47.641014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.36","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686420+0000","last_change":"2024-03-06T07:31:32.681534+0000","last_active":"2024-03-06T07:31:32.686420+0000","last_peered":"2024-03-06T07:31:32.686420+0000","last_clean":"2024-03-06T07:31:32.686420+0000","last_became_active":"2024-03-06T07:31:32.679350+0000","last_became_peered":"2024-03-06T07:31:32.679350+0000","last_unstale":"2024-03-06T07:31:32.686420+0000","last_undegraded":"2024-03-06T07:31:32.686420+0000","last_fullsized":"2024-03-06T07:31:32.686420+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:31:30.998038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.35","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668230+0000","last_change":"2024-03-06T07:31:32.667038+0000","last_active":"2024-03-06T07:31:32.668230+0000","last_peered":"2024-03-06T07:31:32.668230+0000","last_clean":"2024-03-06T07:31:32.668230+0000","last_became_active":"2024-03-06T07:31:32.666717+0000","last_became_peered":"2024-03-06T07:31:32.666717+0000","last_unstale":"2024-03-06T07:31:32.668230+0000","last_undegraded":"2024-03-06T07:31:32.668230+0000","last_fullsized":"2024-03-06T07:31:32.668230+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:21.841602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.34","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686142+0000","last_change":"2024-03-06T07:31:32.681917+0000","last_active":"2024-03-06T07:31:32.686142+0000","last_peered":"2024-03-06T07:31:32.686142+0000","last_clean":"2024-03-06T07:31:32.686142+0000","last_became_active":"2024-03-06T07:31:32.679930+0000","last_became_peered":"2024-03-06T07:31:32.679930+0000","last_unstale":"2024-03-06T07:31:32.686142+0000","last_undegraded":"2024-03-06T07:31:32.686142+0000","last_fullsized":"2024-03-06T07:31:32.686142+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:27.894397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.33","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867773+0000","last_change":"2024-03-06T07:31:32.684603+0000","last_active":"2024-03-06T07:31:34.867773+0000","last_peered":"2024-03-06T07:31:34.867773+0000","last_clean":"2024-03-06T07:31:34.867773+0000","last_became_active":"2024-03-06T07:31:32.683801+0000","last_became_peered":"2024-03-06T07:31:32.683801+0000","last_unstale":"2024-03-06T07:31:34.867773+0000","last_undegraded":"2024-03-06T07:31:34.867773+0000","last_fullsized":"2024-03-06T07:31:34.867773+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:03:14.713038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.32","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107385+0000","last_change":"2024-03-06T07:31:32.672174+0000","last_active":"2024-03-06T07:31:34.107385+0000","last_peered":"2024-03-06T07:31:34.107385+0000","last_clean":"2024-03-06T07:31:34.107385+0000","last_became_active":"2024-03-06T07:31:32.671226+0000","last_became_peered":"2024-03-06T07:31:32.671226+0000","last_unstale":"2024-03-06T07:31:34.107385+0000","last_undegraded":"2024-03-06T07:31:34.107385+0000","last_fullsized":"2024-03-06T07:31:34.107385+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:11:53.598734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.31","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.676802+0000","last_change":"2024-03-06T07:31:32.670578+0000","last_active":"2024-03-06T07:31:32.676802+0000","last_peered":"2024-03-06T07:31:32.676802+0000","last_clean":"2024-03-06T07:31:32.676802+0000","last_became_active":"2024-03-06T07:31:32.670295+0000","last_became_peered":"2024-03-06T07:31:32.670295+0000","last_unstale":"2024-03-06T07:31:32.676802+0000","last_undegraded":"2024-03-06T07:31:32.676802+0000","last_fullsized":"2024-03-06T07:31:32.676802+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:34:50.495990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.30","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672235+0000","last_change":"2024-03-06T07:31:32.675340+0000","last_active":"2024-03-06T07:31:33.672235+0000","last_peered":"2024-03-06T07:31:33.672235+0000","last_clean":"2024-03-06T07:31:33.672235+0000","last_became_active":"2024-03-06T07:31:32.675058+0000","last_became_peered":"2024-03-06T07:31:32.675058+0000","last_unstale":"2024-03-06T07:31:33.672235+0000","last_undegraded":"2024-03-06T07:31:33.672235+0000","last_fullsized":"2024-03-06T07:31:33.672235+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:20.052352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2f","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673599+0000","last_change":"2024-03-06T07:31:32.670892+0000","last_active":"2024-03-06T07:31:33.673599+0000","last_peered":"2024-03-06T07:31:33.673599+0000","last_clean":"2024-03-06T07:31:33.673599+0000","last_became_active":"2024-03-06T07:31:32.669520+0000","last_became_peered":"2024-03-06T07:31:32.669520+0000","last_unstale":"2024-03-06T07:31:33.673599+0000","last_undegraded":"2024-03-06T07:31:33.673599+0000","last_fullsized":"2024-03-06T07:31:33.673599+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:53:04.940938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2e","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108036+0000","last_change":"2024-03-06T07:31:32.684321+0000","last_active":"2024-03-06T07:31:34.108036+0000","last_peered":"2024-03-06T07:31:34.108036+0000","last_clean":"2024-03-06T07:31:34.108036+0000","last_became_active":"2024-03-06T07:31:32.682769+0000","last_became_peered":"2024-03-06T07:31:32.682769+0000","last_unstale":"2024-03-06T07:31:34.108036+0000","last_undegraded":"2024-03-06T07:31:34.108036+0000","last_fullsized":"2024-03-06T07:31:34.108036+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:01:06.658665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670477+0000","last_change":"2024-03-06T07:31:32.668423+0000","last_active":"2024-03-06T07:31:33.670477+0000","last_peered":"2024-03-06T07:31:33.670477+0000","last_clean":"2024-03-06T07:31:33.670477+0000","last_became_active":"2024-03-06T07:31:32.664979+0000","last_became_peered":"2024-03-06T07:31:32.664979+0000","last_unstale":"2024-03-06T07:31:33.670477+0000","last_undegraded":"2024-03-06T07:31:33.670477+0000","last_fullsized":"2024-03-06T07:31:33.670477+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:09.898923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.2c","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108007+0000","last_change":"2024-03-06T07:31:32.676384+0000","last_active":"2024-03-06T07:31:34.108007+0000","last_peered":"2024-03-06T07:31:34.108007+0000","last_clean":"2024-03-06T07:31:34.108007+0000","last_became_active":"2024-03-06T07:31:32.675597+0000","last_became_peered":"2024-03-06T07:31:32.675597+0000","last_unstale":"2024-03-06T07:31:34.108007+0000","last_undegraded":"2024-03-06T07:31:34.108007+0000","last_fullsized":"2024-03-06T07:31:34.108007+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:40:45.029984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2b","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107762+0000","last_change":"2024-03-06T07:31:32.682026+0000","last_active":"2024-03-06T07:31:34.107762+0000","last_peered":"2024-03-06T07:31:34.107762+0000","last_clean":"2024-03-06T07:31:34.107762+0000","last_became_active":"2024-03-06T07:31:32.676359+0000","last_became_peered":"2024-03-06T07:31:32.676359+0000","last_unstale":"2024-03-06T07:31:34.107762+0000","last_undegraded":"2024-03-06T07:31:34.107762+0000","last_fullsized":"2024-03-06T07:31:34.107762+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:29:24.740651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2a","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668410+0000","last_change":"2024-03-06T07:31:32.665825+0000","last_active":"2024-03-06T07:31:32.668410+0000","last_peered":"2024-03-06T07:31:32.668410+0000","last_clean":"2024-03-06T07:31:32.668410+0000","last_became_active":"2024-03-06T07:31:32.664824+0000","last_became_peered":"2024-03-06T07:31:32.664824+0000","last_unstale":"2024-03-06T07:31:32.668410+0000","last_undegraded":"2024-03-06T07:31:32.668410+0000","last_fullsized":"2024-03-06T07:31:32.668410+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:56:51.841231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.29","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691614+0000","last_change":"2024-03-06T07:31:32.687227+0000","last_active":"2024-03-06T07:31:32.691614+0000","last_peered":"2024-03-06T07:31:32.691614+0000","last_clean":"2024-03-06T07:31:32.691614+0000","last_became_active":"2024-03-06T07:31:32.686932+0000","last_became_peered":"2024-03-06T07:31:32.686932+0000","last_unstale":"2024-03-06T07:31:32.691614+0000","last_undegraded":"2024-03-06T07:31:32.691614+0000","last_fullsized":"2024-03-06T07:31:32.691614+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:14:44.586900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.28","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.106978+0000","last_change":"2024-03-06T07:31:32.678144+0000","last_active":"2024-03-06T07:31:34.106978+0000","last_peered":"2024-03-06T07:31:34.106978+0000","last_clean":"2024-03-06T07:31:34.106978+0000","last_became_active":"2024-03-06T07:31:32.677856+0000","last_became_peered":"2024-03-06T07:31:32.677856+0000","last_unstale":"2024-03-06T07:31:34.106978+0000","last_undegraded":"2024-03-06T07:31:34.106978+0000","last_fullsized":"2024-03-06T07:31:34.106978+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:20.813609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.27","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687651+0000","last_change":"2024-03-06T07:31:32.682633+0000","last_active":"2024-03-06T07:31:32.687651+0000","last_peered":"2024-03-06T07:31:32.687651+0000","last_clean":"2024-03-06T07:31:32.687651+0000","last_became_active":"2024-03-06T07:31:32.682339+0000","last_became_peered":"2024-03-06T07:31:32.682339+0000","last_unstale":"2024-03-06T07:31:32.687651+0000","last_undegraded":"2024-03-06T07:31:32.687651+0000","last_fullsized":"2024-03-06T07:31:32.687651+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:27:08.545866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.26","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868589+0000","last_change":"2024-03-06T07:31:32.672517+0000","last_active":"2024-03-06T07:31:34.868589+0000","last_peered":"2024-03-06T07:31:34.868589+0000","last_clean":"2024-03-06T07:31:34.868589+0000","last_became_active":"2024-03-06T07:31:32.672224+0000","last_became_peered":"2024-03-06T07:31:32.672224+0000","last_unstale":"2024-03-06T07:31:34.868589+0000","last_undegraded":"2024-03-06T07:31:34.868589+0000","last_fullsized":"2024-03-06T07:31:34.868589+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:04:10.713397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.25","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673154+0000","last_change":"2024-03-06T07:31:32.664377+0000","last_active":"2024-03-06T07:31:33.673154+0000","last_peered":"2024-03-06T07:31:33.673154+0000","last_clean":"2024-03-06T07:31:33.673154+0000","last_became_active":"2024-03-06T07:31:32.663658+0000","last_became_peered":"2024-03-06T07:31:32.663658+0000","last_unstale":"2024-03-06T07:31:33.673154+0000","last_undegraded":"2024-03-06T07:31:33.673154+0000","last_fullsized":"2024-03-06T07:31:33.673154+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:03:14.713038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.24","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.682194+0000","last_change":"2024-03-06T07:31:32.675255+0000","last_active":"2024-03-06T07:31:32.682194+0000","last_peered":"2024-03-06T07:31:32.682194+0000","last_clean":"2024-03-06T07:31:32.682194+0000","last_became_active":"2024-03-06T07:31:32.674713+0000","last_became_peered":"2024-03-06T07:31:32.674713+0000","last_unstale":"2024-03-06T07:31:32.682194+0000","last_undegraded":"2024-03-06T07:31:32.682194+0000","last_fullsized":"2024-03-06T07:31:32.682194+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:22.109879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.23","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668345+0000","last_change":"2024-03-06T07:31:32.665334+0000","last_active":"2024-03-06T07:31:32.668345+0000","last_peered":"2024-03-06T07:31:32.668345+0000","last_clean":"2024-03-06T07:31:32.668345+0000","last_became_active":"2024-03-06T07:31:32.665040+0000","last_became_peered":"2024-03-06T07:31:32.665040+0000","last_unstale":"2024-03-06T07:31:32.668345+0000","last_undegraded":"2024-03-06T07:31:32.668345+0000","last_fullsized":"2024-03-06T07:31:32.668345+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:27:41.527639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.22","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668393+0000","last_change":"2024-03-06T07:31:32.666617+0000","last_active":"2024-03-06T07:31:32.668393+0000","last_peered":"2024-03-06T07:31:32.668393+0000","last_clean":"2024-03-06T07:31:32.668393+0000","last_became_active":"2024-03-06T07:31:32.665570+0000","last_became_peered":"2024-03-06T07:31:32.665570+0000","last_unstale":"2024-03-06T07:31:32.668393+0000","last_undegraded":"2024-03-06T07:31:32.668393+0000","last_fullsized":"2024-03-06T07:31:32.668393+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:10:42.407024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.21","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672307+0000","last_change":"2024-03-06T07:31:32.659145+0000","last_active":"2024-03-06T07:31:33.672307+0000","last_peered":"2024-03-06T07:31:33.672307+0000","last_clean":"2024-03-06T07:31:33.672307+0000","last_became_active":"2024-03-06T07:31:32.658411+0000","last_became_peered":"2024-03-06T07:31:32.658411+0000","last_unstale":"2024-03-06T07:31:33.672307+0000","last_undegraded":"2024-03-06T07:31:33.672307+0000","last_fullsized":"2024-03-06T07:31:33.672307+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:37:54.485529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.20","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.694244+0000","last_change":"2024-03-06T07:31:32.689906+0000","last_active":"2024-03-06T07:31:32.694244+0000","last_peered":"2024-03-06T07:31:32.694244+0000","last_clean":"2024-03-06T07:31:32.694244+0000","last_became_active":"2024-03-06T07:31:32.689401+0000","last_became_peered":"2024-03-06T07:31:32.689401+0000","last_unstale":"2024-03-06T07:31:32.694244+0000","last_undegraded":"2024-03-06T07:31:32.694244+0000","last_fullsized":"2024-03-06T07:31:32.694244+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:32:28.951977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1e","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673530+0000","last_change":"2024-03-06T07:31:32.683082+0000","last_active":"2024-03-06T07:31:33.673530+0000","last_peered":"2024-03-06T07:31:33.673530+0000","last_clean":"2024-03-06T07:31:33.673530+0000","last_became_active":"2024-03-06T07:31:32.682299+0000","last_became_peered":"2024-03-06T07:31:32.682299+0000","last_unstale":"2024-03-06T07:31:33.673530+0000","last_undegraded":"2024-03-06T07:31:33.673530+0000","last_fullsized":"2024-03-06T07:31:33.673530+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:58:40.759611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1f","version":"82'4","reported_seq":16,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.689127+0000","last_change":"2024-03-06T07:31:31.659131+0000","last_active":"2024-03-06T07:31:32.689127+0000","last_peered":"2024-03-06T07:31:32.689127+0000","last_clean":"2024-03-06T07:31:32.689127+0000","last_became_active":"2024-03-06T07:31:31.658820+0000","last_became_peered":"2024-03-06T07:31:31.658820+0000","last_unstale":"2024-03-06T07:31:32.689127+0000","last_undegraded":"2024-03-06T07:31:32.689127+0000","last_fullsized":"2024-03-06T07:31:32.689127+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:56.545347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1f","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691578+0000","last_change":"2024-03-06T07:31:32.687031+0000","last_active":"2024-03-06T07:31:32.691578+0000","last_peered":"2024-03-06T07:31:32.691578+0000","last_clean":"2024-03-06T07:31:32.691578+0000","last_became_active":"2024-03-06T07:31:32.686723+0000","last_became_peered":"2024-03-06T07:31:32.686723+0000","last_unstale":"2024-03-06T07:31:32.691578+0000","last_undegraded":"2024-03-06T07:31:32.691578+0000","last_fullsized":"2024-03-06T07:31:32.691578+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:43.487650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1e","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673835+0000","last_change":"2024-03-06T07:31:31.647876+0000","last_active":"2024-03-06T07:31:33.673835+0000","last_peered":"2024-03-06T07:31:33.673835+0000","last_clean":"2024-03-06T07:31:33.673835+0000","last_became_active":"2024-03-06T07:31:31.647139+0000","last_became_peered":"2024-03-06T07:31:31.647139+0000","last_unstale":"2024-03-06T07:31:33.673835+0000","last_undegraded":"2024-03-06T07:31:33.673835+0000","last_fullsized":"2024-03-06T07:31:33.673835+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:08.521391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1c","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.694261+0000","last_change":"2024-03-06T07:31:32.690207+0000","last_active":"2024-03-06T07:31:32.694261+0000","last_peered":"2024-03-06T07:31:32.694261+0000","last_clean":"2024-03-06T07:31:32.694261+0000","last_became_active":"2024-03-06T07:31:32.689838+0000","last_became_peered":"2024-03-06T07:31:32.689838+0000","last_unstale":"2024-03-06T07:31:32.694261+0000","last_undegraded":"2024-03-06T07:31:32.694261+0000","last_fullsized":"2024-03-06T07:31:32.694261+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:36:12.582342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1d","version":"82'2","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670506+0000","last_change":"2024-03-06T07:31:31.651404+0000","last_active":"2024-03-06T07:31:33.670506+0000","last_peered":"2024-03-06T07:31:33.670506+0000","last_clean":"2024-03-06T07:31:33.670506+0000","last_became_active":"2024-03-06T07:31:31.650775+0000","last_became_peered":"2024-03-06T07:31:31.650775+0000","last_unstale":"2024-03-06T07:31:33.670506+0000","last_undegraded":"2024-03-06T07:31:33.670506+0000","last_fullsized":"2024-03-06T07:31:33.670506+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:19.036862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868493+0000","last_change":"2024-03-06T07:31:32.676577+0000","last_active":"2024-03-06T07:31:34.868493+0000","last_peered":"2024-03-06T07:31:34.868493+0000","last_clean":"2024-03-06T07:31:34.868493+0000","last_became_active":"2024-03-06T07:31:32.675322+0000","last_became_peered":"2024-03-06T07:31:32.675322+0000","last_unstale":"2024-03-06T07:31:34.868493+0000","last_undegraded":"2024-03-06T07:31:34.868493+0000","last_fullsized":"2024-03-06T07:31:34.868493+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:58:40.759611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1c","version":"82'2","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670256+0000","last_change":"2024-03-06T07:31:31.649485+0000","last_active":"2024-03-06T07:31:33.670256+0000","last_peered":"2024-03-06T07:31:33.670256+0000","last_clean":"2024-03-06T07:31:33.670256+0000","last_became_active":"2024-03-06T07:31:31.648894+0000","last_became_peered":"2024-03-06T07:31:31.648894+0000","last_unstale":"2024-03-06T07:31:33.670256+0000","last_undegraded":"2024-03-06T07:31:33.670256+0000","last_fullsized":"2024-03-06T07:31:33.670256+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:56:25.616933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1a","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668376+0000","last_change":"2024-03-06T07:31:32.666076+0000","last_active":"2024-03-06T07:31:32.668376+0000","last_peered":"2024-03-06T07:31:32.668376+0000","last_clean":"2024-03-06T07:31:32.668376+0000","last_became_active":"2024-03-06T07:31:32.665182+0000","last_became_peered":"2024-03-06T07:31:32.665182+0000","last_unstale":"2024-03-06T07:31:32.668376+0000","last_undegraded":"2024-03-06T07:31:32.668376+0000","last_fullsized":"2024-03-06T07:31:32.668376+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:21:38.242551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.1b","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672859+0000","last_change":"2024-03-06T07:31:31.643318+0000","last_active":"2024-03-06T07:31:33.672859+0000","last_peered":"2024-03-06T07:31:33.672859+0000","last_clean":"2024-03-06T07:31:33.672859+0000","last_became_active":"2024-03-06T07:31:31.640903+0000","last_became_peered":"2024-03-06T07:31:31.640903+0000","last_unstale":"2024-03-06T07:31:33.672859+0000","last_undegraded":"2024-03-06T07:31:33.672859+0000","last_fullsized":"2024-03-06T07:31:33.672859+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:12:42.107769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1b","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672668+0000","last_change":"2024-03-06T07:31:32.668424+0000","last_active":"2024-03-06T07:31:33.672668+0000","last_peered":"2024-03-06T07:31:33.672668+0000","last_clean":"2024-03-06T07:31:33.672668+0000","last_became_active":"2024-03-06T07:31:32.668097+0000","last_became_peered":"2024-03-06T07:31:32.668097+0000","last_unstale":"2024-03-06T07:31:33.672668+0000","last_undegraded":"2024-03-06T07:31:33.672668+0000","last_fullsized":"2024-03-06T07:31:33.672668+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:19:29.966268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1a","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672275+0000","last_change":"2024-03-06T07:31:31.651337+0000","last_active":"2024-03-06T07:31:33.672275+0000","last_peered":"2024-03-06T07:31:33.672275+0000","last_clean":"2024-03-06T07:31:33.672275+0000","last_became_active":"2024-03-06T07:31:31.651061+0000","last_became_peered":"2024-03-06T07:31:31.651061+0000","last_unstale":"2024-03-06T07:31:33.672275+0000","last_undegraded":"2024-03-06T07:31:33.672275+0000","last_fullsized":"2024-03-06T07:31:33.672275+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:06:11.195300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.18","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867476+0000","last_change":"2024-03-06T07:31:32.670874+0000","last_active":"2024-03-06T07:31:34.867476+0000","last_peered":"2024-03-06T07:31:34.867476+0000","last_clean":"2024-03-06T07:31:34.867476+0000","last_became_active":"2024-03-06T07:31:32.670542+0000","last_became_peered":"2024-03-06T07:31:32.670542+0000","last_unstale":"2024-03-06T07:31:34.867476+0000","last_undegraded":"2024-03-06T07:31:34.867476+0000","last_fullsized":"2024-03-06T07:31:34.867476+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:37:54.485529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669632+0000","last_change":"2024-03-06T07:31:31.657497+0000","last_active":"2024-03-06T07:31:33.669632+0000","last_peered":"2024-03-06T07:31:33.669632+0000","last_clean":"2024-03-06T07:31:33.669632+0000","last_became_active":"2024-03-06T07:31:31.657120+0000","last_became_peered":"2024-03-06T07:31:31.657120+0000","last_unstale":"2024-03-06T07:31:33.669632+0000","last_undegraded":"2024-03-06T07:31:33.669632+0000","last_fullsized":"2024-03-06T07:31:33.669632+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:42:19.467943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.19","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.677287+0000","last_change":"2024-03-06T07:31:32.673212+0000","last_active":"2024-03-06T07:31:32.677287+0000","last_peered":"2024-03-06T07:31:32.677287+0000","last_clean":"2024-03-06T07:31:32.677287+0000","last_became_active":"2024-03-06T07:31:32.672905+0000","last_became_peered":"2024-03-06T07:31:32.672905+0000","last_unstale":"2024-03-06T07:31:32.677287+0000","last_undegraded":"2024-03-06T07:31:32.677287+0000","last_fullsized":"2024-03-06T07:31:32.677287+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:33:32.681162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.646886+0000","last_change":"2024-03-06T07:31:31.645464+0000","last_active":"2024-03-06T07:31:32.646886+0000","last_peered":"2024-03-06T07:31:32.646886+0000","last_clean":"2024-03-06T07:31:32.646886+0000","last_became_active":"2024-03-06T07:31:31.645201+0000","last_became_peered":"2024-03-06T07:31:31.645201+0000","last_unstale":"2024-03-06T07:31:32.646886+0000","last_undegraded":"2024-03-06T07:31:32.646886+0000","last_fullsized":"2024-03-06T07:31:32.646886+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:13.566188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.16","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.664798+0000","last_change":"2024-03-06T07:31:32.662604+0000","last_active":"2024-03-06T07:31:32.664798+0000","last_peered":"2024-03-06T07:31:32.664798+0000","last_clean":"2024-03-06T07:31:32.664798+0000","last_became_active":"2024-03-06T07:31:32.662316+0000","last_became_peered":"2024-03-06T07:31:32.662316+0000","last_unstale":"2024-03-06T07:31:32.664798+0000","last_undegraded":"2024-03-06T07:31:32.664798+0000","last_fullsized":"2024-03-06T07:31:32.664798+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:26:04.337185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.667170+0000","last_change":"2024-03-06T07:31:31.656325+0000","last_active":"2024-03-06T07:31:32.667170+0000","last_peered":"2024-03-06T07:31:32.667170+0000","last_clean":"2024-03-06T07:31:32.667170+0000","last_became_active":"2024-03-06T07:31:31.656045+0000","last_became_peered":"2024-03-06T07:31:31.656045+0000","last_unstale":"2024-03-06T07:31:32.667170+0000","last_undegraded":"2024-03-06T07:31:32.667170+0000","last_fullsized":"2024-03-06T07:31:32.667170+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:14:20.341113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688275+0000","last_change":"2024-03-06T07:31:32.683458+0000","last_active":"2024-03-06T07:31:32.688275+0000","last_peered":"2024-03-06T07:31:32.688275+0000","last_clean":"2024-03-06T07:31:32.688275+0000","last_became_active":"2024-03-06T07:31:32.683186+0000","last_became_peered":"2024-03-06T07:31:32.683186+0000","last_unstale":"2024-03-06T07:31:32.688275+0000","last_undegraded":"2024-03-06T07:31:32.688275+0000","last_fullsized":"2024-03-06T07:31:32.688275+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:14:44.586900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.669694+0000","last_change":"2024-03-06T07:31:31.654941+0000","last_active":"2024-03-06T07:31:32.669694+0000","last_peered":"2024-03-06T07:31:32.669694+0000","last_clean":"2024-03-06T07:31:32.669694+0000","last_became_active":"2024-03-06T07:31:31.654185+0000","last_became_peered":"2024-03-06T07:31:31.654185+0000","last_unstale":"2024-03-06T07:31:32.669694+0000","last_undegraded":"2024-03-06T07:31:32.669694+0000","last_fullsized":"2024-03-06T07:31:32.669694+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:24:12.685215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.14","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670163+0000","last_change":"2024-03-06T07:31:32.685450+0000","last_active":"2024-03-06T07:31:33.670163+0000","last_peered":"2024-03-06T07:31:33.670163+0000","last_clean":"2024-03-06T07:31:33.670163+0000","last_became_active":"2024-03-06T07:31:32.685174+0000","last_became_peered":"2024-03-06T07:31:32.685174+0000","last_unstale":"2024-03-06T07:31:33.670163+0000","last_undegraded":"2024-03-06T07:31:33.670163+0000","last_fullsized":"2024-03-06T07:31:33.670163+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:23:44.975998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.15","version":"82'3","reported_seq":15,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.692069+0000","last_change":"2024-03-06T07:31:31.658088+0000","last_active":"2024-03-06T07:31:32.692069+0000","last_peered":"2024-03-06T07:31:32.692069+0000","last_clean":"2024-03-06T07:31:32.692069+0000","last_became_active":"2024-03-06T07:31:31.657484+0000","last_became_peered":"2024-03-06T07:31:31.657484+0000","last_unstale":"2024-03-06T07:31:32.692069+0000","last_undegraded":"2024-03-06T07:31:32.692069+0000","last_fullsized":"2024-03-06T07:31:32.692069+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:01:54.010077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.15","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673253+0000","last_change":"2024-03-06T07:31:32.664158+0000","last_active":"2024-03-06T07:31:33.673253+0000","last_peered":"2024-03-06T07:31:33.673253+0000","last_clean":"2024-03-06T07:31:33.673253+0000","last_became_active":"2024-03-06T07:31:32.663298+0000","last_became_peered":"2024-03-06T07:31:32.663298+0000","last_unstale":"2024-03-06T07:31:33.673253+0000","last_undegraded":"2024-03-06T07:31:33.673253+0000","last_fullsized":"2024-03-06T07:31:33.673253+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:46:33.261291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"82'6","reported_seq":18,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688160+0000","last_change":"2024-03-06T07:31:31.657354+0000","last_active":"2024-03-06T07:31:32.688160+0000","last_peered":"2024-03-06T07:31:32.688160+0000","last_clean":"2024-03-06T07:31:32.688160+0000","last_became_active":"2024-03-06T07:31:31.657065+0000","last_became_peered":"2024-03-06T07:31:31.657065+0000","last_unstale":"2024-03-06T07:31:32.688160+0000","last_undegraded":"2024-03-06T07:31:32.688160+0000","last_fullsized":"2024-03-06T07:31:32.688160+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:31:29.982548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.12","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669913+0000","last_change":"2024-03-06T07:31:32.665264+0000","last_active":"2024-03-06T07:31:33.669913+0000","last_peered":"2024-03-06T07:31:33.669913+0000","last_clean":"2024-03-06T07:31:33.669913+0000","last_became_active":"2024-03-06T07:31:32.664967+0000","last_became_peered":"2024-03-06T07:31:32.664967+0000","last_unstale":"2024-03-06T07:31:33.669913+0000","last_undegraded":"2024-03-06T07:31:33.669913+0000","last_fullsized":"2024-03-06T07:31:33.669913+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:06.083666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.13","version":"82'2","reported_seq":14,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.690741+0000","last_change":"2024-03-06T07:31:31.647387+0000","last_active":"2024-03-06T07:31:32.690741+0000","last_peered":"2024-03-06T07:31:32.690741+0000","last_clean":"2024-03-06T07:31:32.690741+0000","last_became_active":"2024-03-06T07:31:31.647084+0000","last_became_peered":"2024-03-06T07:31:31.647084+0000","last_unstale":"2024-03-06T07:31:32.690741+0000","last_undegraded":"2024-03-06T07:31:32.690741+0000","last_fullsized":"2024-03-06T07:31:32.690741+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:34:42.642586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.13","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867739+0000","last_change":"2024-03-06T07:31:32.681572+0000","last_active":"2024-03-06T07:31:34.867739+0000","last_peered":"2024-03-06T07:31:34.867739+0000","last_clean":"2024-03-06T07:31:34.867739+0000","last_became_active":"2024-03-06T07:31:32.681291+0000","last_became_peered":"2024-03-06T07:31:32.681291+0000","last_unstale":"2024-03-06T07:31:34.867739+0000","last_undegraded":"2024-03-06T07:31:34.867739+0000","last_fullsized":"2024-03-06T07:31:34.867739+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:19:29.966268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.654790+0000","last_change":"2024-03-06T07:31:31.650179+0000","last_active":"2024-03-06T07:31:32.654790+0000","last_peered":"2024-03-06T07:31:32.654790+0000","last_clean":"2024-03-06T07:31:32.654790+0000","last_became_active":"2024-03-06T07:31:31.648291+0000","last_became_peered":"2024-03-06T07:31:31.648291+0000","last_unstale":"2024-03-06T07:31:32.654790+0000","last_undegraded":"2024-03-06T07:31:32.654790+0000","last_fullsized":"2024-03-06T07:31:32.654790+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:08.883433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.10","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.678151+0000","last_change":"2024-03-06T07:31:32.669793+0000","last_active":"2024-03-06T07:31:32.678151+0000","last_peered":"2024-03-06T07:31:32.678151+0000","last_clean":"2024-03-06T07:31:32.678151+0000","last_became_active":"2024-03-06T07:31:32.668693+0000","last_became_peered":"2024-03-06T07:31:32.668693+0000","last_unstale":"2024-03-06T07:31:32.678151+0000","last_undegraded":"2024-03-06T07:31:32.678151+0000","last_fullsized":"2024-03-06T07:31:32.678151+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:49:09.065828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.669837+0000","last_change":"2024-03-06T07:31:31.659907+0000","last_active":"2024-03-06T07:31:32.669837+0000","last_peered":"2024-03-06T07:31:32.669837+0000","last_clean":"2024-03-06T07:31:32.669837+0000","last_became_active":"2024-03-06T07:31:31.659582+0000","last_became_peered":"2024-03-06T07:31:31.659582+0000","last_unstale":"2024-03-06T07:31:32.669837+0000","last_undegraded":"2024-03-06T07:31:32.669837+0000","last_fullsized":"2024-03-06T07:31:32.669837+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:26.878907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.11","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691107+0000","last_change":"2024-03-06T07:31:32.685033+0000","last_active":"2024-03-06T07:31:32.691107+0000","last_peered":"2024-03-06T07:31:32.691107+0000","last_clean":"2024-03-06T07:31:32.691107+0000","last_became_active":"2024-03-06T07:31:32.684557+0000","last_became_peered":"2024-03-06T07:31:32.684557+0000","last_unstale":"2024-03-06T07:31:32.691107+0000","last_undegraded":"2024-03-06T07:31:32.691107+0000","last_fullsized":"2024-03-06T07:31:32.691107+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:51:05.876283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.10","version":"82'8","reported_seq":22,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669589+0000","last_change":"2024-03-06T07:31:31.659643+0000","last_active":"2024-03-06T07:31:33.669589+0000","last_peered":"2024-03-06T07:31:33.669589+0000","last_clean":"2024-03-06T07:31:33.669589+0000","last_became_active":"2024-03-06T07:31:31.659411+0000","last_became_peered":"2024-03-06T07:31:31.659411+0000","last_unstale":"2024-03-06T07:31:33.669589+0000","last_undegraded":"2024-03-06T07:31:33.669589+0000","last_fullsized":"2024-03-06T07:31:33.669589+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:00.810966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868641+0000","last_change":"2024-03-06T07:31:32.679538+0000","last_active":"2024-03-06T07:31:34.868641+0000","last_peered":"2024-03-06T07:31:34.868641+0000","last_clean":"2024-03-06T07:31:34.868641+0000","last_became_active":"2024-03-06T07:31:32.679229+0000","last_became_peered":"2024-03-06T07:31:32.679229+0000","last_unstale":"2024-03-06T07:31:34.868641+0000","last_undegraded":"2024-03-06T07:31:34.868641+0000","last_fullsized":"2024-03-06T07:31:34.868641+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:06.083666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.f","version":"82'1","reported_seq":13,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687864+0000","last_change":"2024-03-06T07:31:31.661061+0000","last_active":"2024-03-06T07:31:32.687864+0000","last_peered":"2024-03-06T07:31:32.687864+0000","last_clean":"2024-03-06T07:31:32.687864+0000","last_became_active":"2024-03-06T07:31:31.660259+0000","last_became_peered":"2024-03-06T07:31:31.660259+0000","last_unstale":"2024-03-06T07:31:32.687864+0000","last_undegraded":"2024-03-06T07:31:32.687864+0000","last_fullsized":"2024-03-06T07:31:32.687864+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:09:26.069429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.680984+0000","last_change":"2024-03-06T07:31:32.676087+0000","last_active":"2024-03-06T07:31:32.680984+0000","last_peered":"2024-03-06T07:31:32.680984+0000","last_clean":"2024-03-06T07:31:32.680984+0000","last_became_active":"2024-03-06T07:31:32.675819+0000","last_became_peered":"2024-03-06T07:31:32.675819+0000","last_unstale":"2024-03-06T07:31:32.680984+0000","last_undegraded":"2024-03-06T07:31:32.680984+0000","last_fullsized":"2024-03-06T07:31:32.680984+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:55:05.336895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"82'2","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670014+0000","last_change":"2024-03-06T07:31:31.646735+0000","last_active":"2024-03-06T07:31:33.670014+0000","last_peered":"2024-03-06T07:31:33.670014+0000","last_clean":"2024-03-06T07:31:33.670014+0000","last_became_active":"2024-03-06T07:31:31.646124+0000","last_became_peered":"2024-03-06T07:31:31.646124+0000","last_unstale":"2024-03-06T07:31:33.670014+0000","last_undegraded":"2024-03-06T07:31:33.670014+0000","last_fullsized":"2024-03-06T07:31:33.670014+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:37:53.470039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673757+0000","last_change":"2024-03-06T07:31:32.661002+0000","last_active":"2024-03-06T07:31:33.673757+0000","last_peered":"2024-03-06T07:31:33.673757+0000","last_clean":"2024-03-06T07:31:33.673757+0000","last_became_active":"2024-03-06T07:31:32.660602+0000","last_became_peered":"2024-03-06T07:31:32.660602+0000","last_unstale":"2024-03-06T07:31:33.673757+0000","last_undegraded":"2024-03-06T07:31:33.673757+0000","last_fullsized":"2024-03-06T07:31:33.673757+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:48:41.873102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670337+0000","last_change":"2024-03-06T07:31:31.661640+0000","last_active":"2024-03-06T07:31:33.670337+0000","last_peered":"2024-03-06T07:31:33.670337+0000","last_clean":"2024-03-06T07:31:33.670337+0000","last_became_active":"2024-03-06T07:31:31.661318+0000","last_became_peered":"2024-03-06T07:31:31.661318+0000","last_unstale":"2024-03-06T07:31:33.670337+0000","last_undegraded":"2024-03-06T07:31:33.670337+0000","last_fullsized":"2024-03-06T07:31:33.670337+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:19:28.950778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108172+0000","last_change":"2024-03-06T07:31:32.674307+0000","last_active":"2024-03-06T07:31:34.108172+0000","last_peered":"2024-03-06T07:31:34.108172+0000","last_clean":"2024-03-06T07:31:34.108172+0000","last_became_active":"2024-03-06T07:31:32.669944+0000","last_became_peered":"2024-03-06T07:31:32.669944+0000","last_unstale":"2024-03-06T07:31:34.108172+0000","last_undegraded":"2024-03-06T07:31:34.108172+0000","last_fullsized":"2024-03-06T07:31:34.108172+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:14.581678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"82'2","reported_seq":14,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687842+0000","last_change":"2024-03-06T07:31:31.651064+0000","last_active":"2024-03-06T07:31:32.687842+0000","last_peered":"2024-03-06T07:31:32.687842+0000","last_clean":"2024-03-06T07:31:32.687842+0000","last_became_active":"2024-03-06T07:31:31.650778+0000","last_became_peered":"2024-03-06T07:31:31.650778+0000","last_unstale":"2024-03-06T07:31:32.687842+0000","last_undegraded":"2024-03-06T07:31:32.687842+0000","last_fullsized":"2024-03-06T07:31:32.687842+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:22:23.068492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.681840+0000","last_change":"2024-03-06T07:31:32.676297+0000","last_active":"2024-03-06T07:31:32.681840+0000","last_peered":"2024-03-06T07:31:32.681840+0000","last_clean":"2024-03-06T07:31:32.681840+0000","last_became_active":"2024-03-06T07:31:32.675984+0000","last_became_peered":"2024-03-06T07:31:32.675984+0000","last_unstale":"2024-03-06T07:31:32.681840+0000","last_undegraded":"2024-03-06T07:31:32.681840+0000","last_fullsized":"2024-03-06T07:31:32.681840+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:07.717126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107854+0000","last_change":"2024-03-06T07:31:31.648998+0000","last_active":"2024-03-06T07:31:34.107854+0000","last_peered":"2024-03-06T07:31:34.107854+0000","last_clean":"2024-03-06T07:31:34.107854+0000","last_became_active":"2024-03-06T07:31:31.647766+0000","last_became_peered":"2024-03-06T07:31:31.647766+0000","last_unstale":"2024-03-06T07:31:34.107854+0000","last_undegraded":"2024-03-06T07:31:34.107854+0000","last_fullsized":"2024-03-06T07:31:34.107854+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:37:17.302476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107070+0000","last_change":"2024-03-06T07:31:32.678203+0000","last_active":"2024-03-06T07:31:34.107070+0000","last_peered":"2024-03-06T07:31:34.107070+0000","last_clean":"2024-03-06T07:31:34.107070+0000","last_became_active":"2024-03-06T07:31:32.677921+0000","last_became_peered":"2024-03-06T07:31:32.677921+0000","last_unstale":"2024-03-06T07:31:34.107070+0000","last_undegraded":"2024-03-06T07:31:34.107070+0000","last_fullsized":"2024-03-06T07:31:34.107070+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:50:28.238836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107347+0000","last_change":"2024-03-06T07:31:31.662031+0000","last_active":"2024-03-06T07:31:34.107347+0000","last_peered":"2024-03-06T07:31:34.107347+0000","last_clean":"2024-03-06T07:31:34.107347+0000","last_became_active":"2024-03-06T07:31:31.661697+0000","last_became_peered":"2024-03-06T07:31:31.661697+0000","last_unstale":"2024-03-06T07:31:34.107347+0000","last_undegraded":"2024-03-06T07:31:34.107347+0000","last_fullsized":"2024-03-06T07:31:34.107347+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:08:45.128965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.9","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688854+0000","last_change":"2024-03-06T07:31:32.687887+0000","last_active":"2024-03-06T07:31:32.688854+0000","last_peered":"2024-03-06T07:31:32.688854+0000","last_clean":"2024-03-06T07:31:32.688854+0000","last_became_active":"2024-03-06T07:31:32.687573+0000","last_became_peered":"2024-03-06T07:31:32.687573+0000","last_unstale":"2024-03-06T07:31:32.688854+0000","last_undegraded":"2024-03-06T07:31:32.688854+0000","last_fullsized":"2024-03-06T07:31:32.688854+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:36:12.582342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.654550+0000","last_change":"2024-03-06T07:31:31.649164+0000","last_active":"2024-03-06T07:31:32.654550+0000","last_peered":"2024-03-06T07:31:32.654550+0000","last_clean":"2024-03-06T07:31:32.654550+0000","last_became_active":"2024-03-06T07:31:31.648895+0000","last_became_peered":"2024-03-06T07:31:31.648895+0000","last_unstale":"2024-03-06T07:31:32.654550+0000","last_undegraded":"2024-03-06T07:31:32.654550+0000","last_fullsized":"2024-03-06T07:31:32.654550+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:23:43.960508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673751+0000","last_change":"2024-03-06T07:31:32.676758+0000","last_active":"2024-03-06T07:31:33.673751+0000","last_peered":"2024-03-06T07:31:33.673751+0000","last_clean":"2024-03-06T07:31:33.673751+0000","last_became_active":"2024-03-06T07:31:32.676479+0000","last_became_peered":"2024-03-06T07:31:32.676479+0000","last_unstale":"2024-03-06T07:31:33.673751+0000","last_undegraded":"2024-03-06T07:31:33.673751+0000","last_fullsized":"2024-03-06T07:31:33.673751+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:06.083666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"82'2","reported_seq":14,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.681646+0000","last_change":"2024-03-06T07:31:31.659130+0000","last_active":"2024-03-06T07:31:32.681646+0000","last_peered":"2024-03-06T07:31:32.681646+0000","last_clean":"2024-03-06T07:31:32.681646+0000","last_became_active":"2024-03-06T07:31:31.658936+0000","last_became_peered":"2024-03-06T07:31:31.658936+0000","last_unstale":"2024-03-06T07:31:32.681646+0000","last_undegraded":"2024-03-06T07:31:32.681646+0000","last_fullsized":"2024-03-06T07:31:32.681646+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:19:48.824088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.7","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868533+0000","last_change":"2024-03-06T07:31:32.683857+0000","last_active":"2024-03-06T07:31:34.868533+0000","last_peered":"2024-03-06T07:31:34.868533+0000","last_clean":"2024-03-06T07:31:34.868533+0000","last_became_active":"2024-03-06T07:31:32.683443+0000","last_became_peered":"2024-03-06T07:31:32.683443+0000","last_unstale":"2024-03-06T07:31:34.868533+0000","last_undegraded":"2024-03-06T07:31:34.868533+0000","last_fullsized":"2024-03-06T07:31:34.868533+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:53:04.940938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.6","version":"82'1","reported_seq":13,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.656440+0000","last_change":"2024-03-06T07:31:31.643000+0000","last_active":"2024-03-06T07:31:33.656440+0000","last_peered":"2024-03-06T07:31:33.656440+0000","last_clean":"2024-03-06T07:31:33.656440+0000","last_became_active":"2024-03-06T07:31:31.642724+0000","last_became_peered":"2024-03-06T07:31:31.642724+0000","last_unstale":"2024-03-06T07:31:33.656440+0000","last_undegraded":"2024-03-06T07:31:33.656440+0000","last_fullsized":"2024-03-06T07:31:33.656440+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:27:04.600309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.4","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686217+0000","last_change":"2024-03-06T07:31:32.683434+0000","last_active":"2024-03-06T07:31:32.686217+0000","last_peered":"2024-03-06T07:31:32.686217+0000","last_clean":"2024-03-06T07:31:32.686217+0000","last_became_active":"2024-03-06T07:31:32.681959+0000","last_became_peered":"2024-03-06T07:31:32.681959+0000","last_unstale":"2024-03-06T07:31:32.686217+0000","last_undegraded":"2024-03-06T07:31:32.686217+0000","last_fullsized":"2024-03-06T07:31:32.686217+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:07.717126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.662612+0000","last_change":"2024-03-06T07:31:31.658223+0000","last_active":"2024-03-06T07:31:32.662612+0000","last_peered":"2024-03-06T07:31:32.662612+0000","last_clean":"2024-03-06T07:31:32.662612+0000","last_became_active":"2024-03-06T07:31:31.656082+0000","last_became_peered":"2024-03-06T07:31:31.656082+0000","last_unstale":"2024-03-06T07:31:32.662612+0000","last_undegraded":"2024-03-06T07:31:32.662612+0000","last_fullsized":"2024-03-06T07:31:32.662612+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:08:56.998640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673748+0000","last_change":"2024-03-06T07:31:32.663935+0000","last_active":"2024-03-06T07:31:33.673748+0000","last_peered":"2024-03-06T07:31:33.673748+0000","last_clean":"2024-03-06T07:31:33.673748+0000","last_became_active":"2024-03-06T07:31:32.662929+0000","last_became_peered":"2024-03-06T07:31:32.662929+0000","last_unstale":"2024-03-06T07:31:33.673748+0000","last_undegraded":"2024-03-06T07:31:33.673748+0000","last_fullsized":"2024-03-06T07:31:33.673748+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:04:10.713397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.666602+0000","last_change":"2024-03-06T07:31:31.658449+0000","last_active":"2024-03-06T07:31:32.666602+0000","last_peered":"2024-03-06T07:31:32.666602+0000","last_clean":"2024-03-06T07:31:32.666602+0000","last_became_active":"2024-03-06T07:31:31.656269+0000","last_became_peered":"2024-03-06T07:31:31.656269+0000","last_unstale":"2024-03-06T07:31:32.666602+0000","last_undegraded":"2024-03-06T07:31:32.666602+0000","last_fullsized":"2024-03-06T07:31:32.666602+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:49:45.336355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.677448+0000","last_change":"2024-03-06T07:31:32.674786+0000","last_active":"2024-03-06T07:31:32.677448+0000","last_peered":"2024-03-06T07:31:32.677448+0000","last_clean":"2024-03-06T07:31:32.677448+0000","last_became_active":"2024-03-06T07:31:32.674501+0000","last_became_peered":"2024-03-06T07:31:32.674501+0000","last_unstale":"2024-03-06T07:31:32.677448+0000","last_undegraded":"2024-03-06T07:31:32.677448+0000","last_fullsized":"2024-03-06T07:31:32.677448+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:48:21.013778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669966+0000","last_change":"2024-03-06T07:31:31.658000+0000","last_active":"2024-03-06T07:31:33.669966+0000","last_peered":"2024-03-06T07:31:33.669966+0000","last_clean":"2024-03-06T07:31:33.669966+0000","last_became_active":"2024-03-06T07:31:31.657669+0000","last_became_peered":"2024-03-06T07:31:31.657669+0000","last_unstale":"2024-03-06T07:31:33.669966+0000","last_undegraded":"2024-03-06T07:31:33.669966+0000","last_fullsized":"2024-03-06T07:31:33.669966+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:53:13.572579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.3","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669942+0000","last_change":"2024-03-06T07:31:32.684067+0000","last_active":"2024-03-06T07:31:33.669942+0000","last_peered":"2024-03-06T07:31:33.669942+0000","last_clean":"2024-03-06T07:31:33.669942+0000","last_became_active":"2024-03-06T07:31:32.683493+0000","last_became_peered":"2024-03-06T07:31:32.683493+0000","last_unstale":"2024-03-06T07:31:33.669942+0000","last_undegraded":"2024-03-06T07:31:33.669942+0000","last_fullsized":"2024-03-06T07:31:33.669942+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:48:41.873102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672527+0000","last_change":"2024-03-06T07:31:31.651943+0000","last_active":"2024-03-06T07:31:33.672527+0000","last_peered":"2024-03-06T07:31:33.672527+0000","last_clean":"2024-03-06T07:31:33.672527+0000","last_became_active":"2024-03-06T07:31:31.649489+0000","last_became_peered":"2024-03-06T07:31:31.649489+0000","last_unstale":"2024-03-06T07:31:33.672527+0000","last_undegraded":"2024-03-06T07:31:33.672527+0000","last_fullsized":"2024-03-06T07:31:33.672527+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:42:39.195412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.0","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867710+0000","last_change":"2024-03-06T07:31:32.666310+0000","last_active":"2024-03-06T07:31:34.867710+0000","last_peered":"2024-03-06T07:31:34.867710+0000","last_clean":"2024-03-06T07:31:34.867710+0000","last_became_active":"2024-03-06T07:31:32.665981+0000","last_became_peered":"2024-03-06T07:31:32.665981+0000","last_unstale":"2024-03-06T07:31:34.867710+0000","last_undegraded":"2024-03-06T07:31:34.867710+0000","last_fullsized":"2024-03-06T07:31:34.867710+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:56:26.632423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.670354+0000","last_change":"2024-03-06T07:31:31.659996+0000","last_active":"2024-03-06T07:31:32.670354+0000","last_peered":"2024-03-06T07:31:32.670354+0000","last_clean":"2024-03-06T07:31:32.670354+0000","last_became_active":"2024-03-06T07:31:31.658002+0000","last_became_peered":"2024-03-06T07:31:31.658002+0000","last_unstale":"2024-03-06T07:31:32.670354+0000","last_undegraded":"2024-03-06T07:31:32.670354+0000","last_fullsized":"2024-03-06T07:31:32.670354+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:29:40.256587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686198+0000","last_change":"2024-03-06T07:31:32.682870+0000","last_active":"2024-03-06T07:31:32.686198+0000","last_peered":"2024-03-06T07:31:32.686198+0000","last_clean":"2024-03-06T07:31:32.686198+0000","last_became_active":"2024-03-06T07:31:32.682432+0000","last_became_peered":"2024-03-06T07:31:32.682432+0000","last_unstale":"2024-03-06T07:31:32.686198+0000","last_undegraded":"2024-03-06T07:31:32.686198+0000","last_fullsized":"2024-03-06T07:31:32.686198+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:48:21.013778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672187+0000","last_change":"2024-03-06T07:31:31.655903+0000","last_active":"2024-03-06T07:31:33.672187+0000","last_peered":"2024-03-06T07:31:33.672187+0000","last_clean":"2024-03-06T07:31:33.672187+0000","last_became_active":"2024-03-06T07:31:31.654356+0000","last_became_peered":"2024-03-06T07:31:31.654356+0000","last_unstale":"2024-03-06T07:31:33.672187+0000","last_undegraded":"2024-03-06T07:31:33.672187+0000","last_fullsized":"2024-03-06T07:31:33.672187+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:06:43.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.c","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.681715+0000","last_change":"2024-03-06T07:31:32.675848+0000","last_active":"2024-03-06T07:31:32.681715+0000","last_peered":"2024-03-06T07:31:32.681715+0000","last_clean":"2024-03-06T07:31:32.681715+0000","last_became_active":"2024-03-06T07:31:32.675571+0000","last_became_peered":"2024-03-06T07:31:32.675571+0000","last_unstale":"2024-03-06T07:31:32.681715+0000","last_undegraded":"2024-03-06T07:31:32.681715+0000","last_fullsized":"2024-03-06T07:31:32.681715+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:47:56.937757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868447+0000","last_change":"2024-03-06T07:31:31.662414+0000","last_active":"2024-03-06T07:31:34.868447+0000","last_peered":"2024-03-06T07:31:34.868447+0000","last_clean":"2024-03-06T07:31:34.868447+0000","last_became_active":"2024-03-06T07:31:31.662126+0000","last_became_peered":"2024-03-06T07:31:31.662126+0000","last_unstale":"2024-03-06T07:31:34.868447+0000","last_undegraded":"2024-03-06T07:31:34.868447+0000","last_fullsized":"2024-03-06T07:31:34.868447+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:53:55.203310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":283,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867083+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:31:34.867083+0000","last_peered":"2024-03-06T07:31:34.867083+0000","last_clean":"2024-03-06T07:31:34.867083+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:31:34.867083+0000","last_undegraded":"2024-03-06T07:31:34.867083+0000","last_fullsized":"2024-03-06T07:31:34.867083+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542203,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32748,"kb_used_data":1564,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339092,"statfs":{"total":96636764160,"available":96603230208,"internally_reserved":0,"allocated":1601536,"data_stored":1014266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542202,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32180,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339660,"statfs":{"total":96636764160,"available":96603811840,"internally_reserved":0,"allocated":1015808,"data_stored":424522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542203,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32188,"kb_used_data":996,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339652,"statfs":{"total":96636764160,"available":96603803648,"internally_reserved":0,"allocated":1019904,"data_stored":425462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542202,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32180,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339660,"statfs":{"total":96636764160,"available":96603811840,"internally_reserved":0,"allocated":1015808,"data_stored":430128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542202,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32176,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339664,"statfs":{"total":96636764160,"available":96603815936,"internally_reserved":0,"allocated":1015808,"data_stored":425440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542203,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32184,"kb_used_data":996,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339656,"statfs":{"total":96636764160,"available":96603807744,"internally_reserved":0,"allocated":1019904,"data_stored":430710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542204,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32176,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339664,"statfs":{"total":96636764160,"available":96603815936,"internally_reserved":0,"allocated":1015808,"data_stored":424022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542201,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32764,"kb_used_data":1576,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339076,"statfs":{"total":96636764160,"available":96603213824,"internally_reserved":0,"allocated":1613824,"data_stored":1020530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:31:37.235 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:31:37.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.477+0000 7f74d5600640 1 Processor -- start 2024-03-06T07:31:37.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.477+0000 7f74d5600640 1 -- start start 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.478+0000 7f74d5600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f74d015acb0 0x7f74d00b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.478+0000 7f74d5600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d00b4680 0x7f74d006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.478+0000 7f74d5600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f74d015b090 con 0x7f74d0072340 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.478+0000 7f74d5600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f74d00b2a30 con 0x7f74d015acb0 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.478+0000 7f74d5600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f74d00b4360 con 0x7f74d00b4680 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f74d015acb0 0x7f74d00b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f74d015acb0 0x7f74d00b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53266/0 (socket says 172.21.15.86:53266) 2024-03-06T07:31:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cea00640 1 -- 172.21.15.86:0/1341662630 learned_addr learned my addr 172.21.15.86:0/1341662630 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:37.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74ce000640 1 --2- 172.21.15.86:0/1341662630 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d00b4680 0x7f74d006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cea00640 1 -- 172.21.15.86:0/1341662630 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d00b4680 msgr2=0x7f74d006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:37.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cea00640 1 --2- 172.21.15.86:0/1341662630 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d00b4680 0x7f74d006e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cea00640 1 -- 172.21.15.86:0/1341662630 >> v1:172.21.15.86:6789/0 conn(0x7f74d0072340 legacy=0x7f74d00ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cea00640 1 -- 172.21.15.86:0/1341662630 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74d00b2cf0 con 0x7f74d015acb0 2024-03-06T07:31:37.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.479+0000 7f74cd600640 1 -- 172.21.15.86:0/1341662630 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3986145681 0 0) 0x7f74d015b090 con 0x7f74d0072340 2024-03-06T07:31:37.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.480+0000 7f74cea00640 1 --2- 172.21.15.86:0/1341662630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d015acb0 0x7f74d00b37d0 secure :-1 s=READY pgs=954 cs=0 l=1 rev1=1 crypto rx=0x7f74c0009870 tx=0x7f74c002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=605e9af244a44b0c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:37.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74cd600640 1 -- 172.21.15.86:0/1341662630 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74c0034070 con 0x7f74d015acb0 2024-03-06T07:31:37.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74cd600640 1 -- 172.21.15.86:0/1341662630 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f74c0030840 con 0x7f74d015acb0 2024-03-06T07:31:37.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74cd600640 1 -- 172.21.15.86:0/1341662630 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74c0030c10 con 0x7f74d015acb0 2024-03-06T07:31:37.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74d5600640 1 -- 172.21.15.86:0/1341662630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d015acb0 msgr2=0x7f74d00b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74d5600640 1 --2- 172.21.15.86:0/1341662630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d015acb0 0x7f74d00b37d0 secure :-1 s=READY pgs=954 cs=0 l=1 rev1=1 crypto rx=0x7f74c0009870 tx=0x7f74c002b980 comp rx=0 tx=0).stop 2024-03-06T07:31:37.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74d5600640 1 -- 172.21.15.86:0/1341662630 shutdown_connections 2024-03-06T07:31:37.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74d5600640 1 --2- 172.21.15.86:0/1341662630 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d00b4680 0x7f74d006e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74d5600640 1 --2- 172.21.15.86:0/1341662630 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d015acb0 0x7f74d00b37d0 unknown :-1 s=CLOSED pgs=954 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.481+0000 7f74d5600640 1 -- 172.21.15.86:0/1341662630 >> 172.21.15.86:0/1341662630 conn(0x7f74d00b1730 msgr2=0x7f74d00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:37.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.482+0000 7f74d5600640 1 -- 172.21.15.86:0/1341662630 shutdown_connections 2024-03-06T07:31:37.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.482+0000 7f74d5600640 1 -- 172.21.15.86:0/1341662630 wait complete. 2024-03-06T07:31:37.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.483+0000 7f74d5600640 1 Processor -- start 2024-03-06T07:31:37.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.483+0000 7f74d5600640 1 -- start start 2024-03-06T07:31:37.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.484+0000 7f74d5600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74d0072340 0x7f74d0144500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.484+0000 7f74d5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d00b4680 0x7f74d01480a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.484+0000 7f74d5600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d015acb0 0x7f74d015d700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.484+0000 7f74d5600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74d00b3d10 con 0x7f74d00b4680 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.484+0000 7f74d5600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74d00b32b0 con 0x7f74d0072340 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.484+0000 7f74d5600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f74d00b4010 con 0x7f74d015acb0 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d00b4680 0x7f74d01480a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d00b4680 0x7f74d01480a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53276/0 (socket says 172.21.15.86:53276) 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 -- 172.21.15.86:0/2284505438 learned_addr learned my addr 172.21.15.86:0/2284505438 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74cea00640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74d0072340 0x7f74d0144500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 -- 172.21.15.86:0/2284505438 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d015acb0 msgr2=0x7f74d015d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d015acb0 0x7f74d015d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 -- 172.21.15.86:0/2284505438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74d0072340 msgr2=0x7f74d0144500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74d0072340 0x7f74d0144500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 -- 172.21.15.86:0/2284505438 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74d015c6a0 con 0x7f74d00b4680 2024-03-06T07:31:37.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.485+0000 7f74ce000640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d00b4680 0x7f74d01480a0 secure :-1 s=READY pgs=955 cs=0 l=1 rev1=1 crypto rx=0x7f74bc00d5f0 tx=0x7f74bc00dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:37.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.486+0000 7f74c6a00640 1 -- 172.21.15.86:0/2284505438 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74bc0125f0 con 0x7f74d00b4680 2024-03-06T07:31:37.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.486+0000 7f74c6a00640 1 -- 172.21.15.86:0/2284505438 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f74bc019070 con 0x7f74d00b4680 2024-03-06T07:31:37.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.486+0000 7f74c6a00640 1 -- 172.21.15.86:0/2284505438 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74bc0129d0 con 0x7f74d00b4680 2024-03-06T07:31:37.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.486+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f74d015c940 con 0x7f74d00b4680 2024-03-06T07:31:37.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.486+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f74d015cf00 con 0x7f74d00b4680 2024-03-06T07:31:37.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.487+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f74d004f330 con 0x7f74d00b4680 2024-03-06T07:31:37.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.488+0000 7f74c6a00640 1 -- 172.21.15.86:0/2284505438 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f74bc012b70 con 0x7f74d00b4680 2024-03-06T07:31:37.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.489+0000 7f74c6a00640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f74b006b4d0 0x7f74b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.489+0000 7f74c6a00640 1 -- 172.21.15.86:0/2284505438 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f74bc08a620 con 0x7f74d00b4680 2024-03-06T07:31:37.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.489+0000 7f74cea00640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f74b006b4d0 0x7f74b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.490+0000 7f74cea00640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f74b006b4d0 0x7f74b006d990 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f74c0036000 tx=0x7f74c003b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:37.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.493+0000 7f74c6a00640 1 -- 172.21.15.86:0/2284505438 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f74bc059490 con 0x7f74d00b4680 2024-03-06T07:31:37.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.685+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f74d006ce60 con 0x7f74b006b4d0 2024-03-06T07:31:37.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.694+0000 7f74c6a00640 1 -- 172.21.15.86:0/2284505438 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252048 (secure 0 0 0) 0x7f74d006ce60 con 0x7f74b006b4d0 2024-03-06T07:31:37.699 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:31:37.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f74b006b4d0 msgr2=0x7f74b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f74b006b4d0 0x7f74b006d990 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f74c0036000 tx=0x7f74c003b040 comp rx=0 tx=0).stop 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d00b4680 msgr2=0x7f74d01480a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d00b4680 0x7f74d01480a0 secure :-1 s=READY pgs=955 cs=0 l=1 rev1=1 crypto rx=0x7f74bc00d5f0 tx=0x7f74bc00dab0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 shutdown_connections 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f74b006b4d0 0x7f74b006d990 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f74d015acb0 0x7f74d015d700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f74d00b4680 0x7f74d01480a0 unknown :-1 s=CLOSED pgs=955 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 --2- 172.21.15.86:0/2284505438 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74d0072340 0x7f74d0144500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 >> 172.21.15.86:0/2284505438 conn(0x7f74d00b1730 msgr2=0x7f74d015e4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 shutdown_connections 2024-03-06T07:31:37.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.700+0000 7f74d5600640 1 -- 172.21.15.86:0/2284505438 wait complete. 2024-03-06T07:31:37.720 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":215,"stamp":"2024-03-06T07:31:36.934215+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":164,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":258596,"kb_used_data":9100,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754716124,"statfs":{"total":773094113280,"available":772829310976,"internally_reserved":0,"allocated":9318400,"data_stored":4595080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001549"},"pg_stats":[{"pgid":"13.3f","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687923+0000","last_change":"2024-03-06T07:31:32.681418+0000","last_active":"2024-03-06T07:31:32.687923+0000","last_peered":"2024-03-06T07:31:32.687923+0000","last_clean":"2024-03-06T07:31:32.687923+0000","last_became_active":"2024-03-06T07:31:32.681101+0000","last_became_peered":"2024-03-06T07:31:32.681101+0000","last_unstale":"2024-03-06T07:31:32.687923+0000","last_undegraded":"2024-03-06T07:31:32.687923+0000","last_fullsized":"2024-03-06T07:31:32.687923+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:52.308934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3e","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691654+0000","last_change":"2024-03-06T07:31:32.685120+0000","last_active":"2024-03-06T07:31:32.691654+0000","last_peered":"2024-03-06T07:31:32.691654+0000","last_clean":"2024-03-06T07:31:32.691654+0000","last_became_active":"2024-03-06T07:31:32.684664+0000","last_became_peered":"2024-03-06T07:31:32.684664+0000","last_unstale":"2024-03-06T07:31:32.691654+0000","last_undegraded":"2024-03-06T07:31:32.691654+0000","last_fullsized":"2024-03-06T07:31:32.691654+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:16:35.428940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.3d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868044+0000","last_change":"2024-03-06T07:31:32.685318+0000","last_active":"2024-03-06T07:31:34.868044+0000","last_peered":"2024-03-06T07:31:34.868044+0000","last_clean":"2024-03-06T07:31:34.868044+0000","last_became_active":"2024-03-06T07:31:32.684802+0000","last_became_peered":"2024-03-06T07:31:32.684802+0000","last_unstale":"2024-03-06T07:31:34.868044+0000","last_undegraded":"2024-03-06T07:31:34.868044+0000","last_fullsized":"2024-03-06T07:31:34.868044+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:46:33.261291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.3c","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108061+0000","last_change":"2024-03-06T07:31:32.674062+0000","last_active":"2024-03-06T07:31:34.108061+0000","last_peered":"2024-03-06T07:31:34.108061+0000","last_clean":"2024-03-06T07:31:34.108061+0000","last_became_active":"2024-03-06T07:31:32.670203+0000","last_became_peered":"2024-03-06T07:31:32.670203+0000","last_unstale":"2024-03-06T07:31:34.108061+0000","last_undegraded":"2024-03-06T07:31:34.108061+0000","last_fullsized":"2024-03-06T07:31:34.108061+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:53:07.783439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.3b","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672549+0000","last_change":"2024-03-06T07:31:32.664938+0000","last_active":"2024-03-06T07:31:33.672549+0000","last_peered":"2024-03-06T07:31:33.672549+0000","last_clean":"2024-03-06T07:31:33.672549+0000","last_became_active":"2024-03-06T07:31:32.655854+0000","last_became_peered":"2024-03-06T07:31:32.655854+0000","last_unstale":"2024-03-06T07:31:33.672549+0000","last_undegraded":"2024-03-06T07:31:33.672549+0000","last_fullsized":"2024-03-06T07:31:33.672549+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:56:26.632423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3a","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687093+0000","last_change":"2024-03-06T07:31:32.681677+0000","last_active":"2024-03-06T07:31:32.687093+0000","last_peered":"2024-03-06T07:31:32.687093+0000","last_clean":"2024-03-06T07:31:32.687093+0000","last_became_active":"2024-03-06T07:31:32.681345+0000","last_became_peered":"2024-03-06T07:31:32.681345+0000","last_unstale":"2024-03-06T07:31:32.687093+0000","last_undegraded":"2024-03-06T07:31:32.687093+0000","last_fullsized":"2024-03-06T07:31:32.687093+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:43.487650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.39","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107357+0000","last_change":"2024-03-06T07:31:32.677326+0000","last_active":"2024-03-06T07:31:34.107357+0000","last_peered":"2024-03-06T07:31:34.107357+0000","last_clean":"2024-03-06T07:31:34.107357+0000","last_became_active":"2024-03-06T07:31:32.676744+0000","last_became_peered":"2024-03-06T07:31:32.676744+0000","last_unstale":"2024-03-06T07:31:34.107357+0000","last_undegraded":"2024-03-06T07:31:34.107357+0000","last_fullsized":"2024-03-06T07:31:34.107357+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:55:26.072069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.38","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688837+0000","last_change":"2024-03-06T07:31:32.687329+0000","last_active":"2024-03-06T07:31:32.688837+0000","last_peered":"2024-03-06T07:31:32.688837+0000","last_clean":"2024-03-06T07:31:32.688837+0000","last_became_active":"2024-03-06T07:31:32.686450+0000","last_became_peered":"2024-03-06T07:31:32.686450+0000","last_unstale":"2024-03-06T07:31:32.688837+0000","last_undegraded":"2024-03-06T07:31:32.688837+0000","last_fullsized":"2024-03-06T07:31:32.688837+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:32:28.951977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.37","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.689637+0000","last_change":"2024-03-06T07:31:32.688585+0000","last_active":"2024-03-06T07:31:32.689637+0000","last_peered":"2024-03-06T07:31:32.689637+0000","last_clean":"2024-03-06T07:31:32.689637+0000","last_became_active":"2024-03-06T07:31:32.687930+0000","last_became_peered":"2024-03-06T07:31:32.687930+0000","last_unstale":"2024-03-06T07:31:32.689637+0000","last_undegraded":"2024-03-06T07:31:32.689637+0000","last_fullsized":"2024-03-06T07:31:32.689637+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:52:47.641014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.36","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686420+0000","last_change":"2024-03-06T07:31:32.681534+0000","last_active":"2024-03-06T07:31:32.686420+0000","last_peered":"2024-03-06T07:31:32.686420+0000","last_clean":"2024-03-06T07:31:32.686420+0000","last_became_active":"2024-03-06T07:31:32.679350+0000","last_became_peered":"2024-03-06T07:31:32.679350+0000","last_unstale":"2024-03-06T07:31:32.686420+0000","last_undegraded":"2024-03-06T07:31:32.686420+0000","last_fullsized":"2024-03-06T07:31:32.686420+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:31:30.998038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.35","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668230+0000","last_change":"2024-03-06T07:31:32.667038+0000","last_active":"2024-03-06T07:31:32.668230+0000","last_peered":"2024-03-06T07:31:32.668230+0000","last_clean":"2024-03-06T07:31:32.668230+0000","last_became_active":"2024-03-06T07:31:32.666717+0000","last_became_peered":"2024-03-06T07:31:32.666717+0000","last_unstale":"2024-03-06T07:31:32.668230+0000","last_undegraded":"2024-03-06T07:31:32.668230+0000","last_fullsized":"2024-03-06T07:31:32.668230+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:21.841602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.34","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686142+0000","last_change":"2024-03-06T07:31:32.681917+0000","last_active":"2024-03-06T07:31:32.686142+0000","last_peered":"2024-03-06T07:31:32.686142+0000","last_clean":"2024-03-06T07:31:32.686142+0000","last_became_active":"2024-03-06T07:31:32.679930+0000","last_became_peered":"2024-03-06T07:31:32.679930+0000","last_unstale":"2024-03-06T07:31:32.686142+0000","last_undegraded":"2024-03-06T07:31:32.686142+0000","last_fullsized":"2024-03-06T07:31:32.686142+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:27.894397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.33","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867773+0000","last_change":"2024-03-06T07:31:32.684603+0000","last_active":"2024-03-06T07:31:34.867773+0000","last_peered":"2024-03-06T07:31:34.867773+0000","last_clean":"2024-03-06T07:31:34.867773+0000","last_became_active":"2024-03-06T07:31:32.683801+0000","last_became_peered":"2024-03-06T07:31:32.683801+0000","last_unstale":"2024-03-06T07:31:34.867773+0000","last_undegraded":"2024-03-06T07:31:34.867773+0000","last_fullsized":"2024-03-06T07:31:34.867773+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:03:14.713038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.32","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107385+0000","last_change":"2024-03-06T07:31:32.672174+0000","last_active":"2024-03-06T07:31:34.107385+0000","last_peered":"2024-03-06T07:31:34.107385+0000","last_clean":"2024-03-06T07:31:34.107385+0000","last_became_active":"2024-03-06T07:31:32.671226+0000","last_became_peered":"2024-03-06T07:31:32.671226+0000","last_unstale":"2024-03-06T07:31:34.107385+0000","last_undegraded":"2024-03-06T07:31:34.107385+0000","last_fullsized":"2024-03-06T07:31:34.107385+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:11:53.598734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.31","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.676802+0000","last_change":"2024-03-06T07:31:32.670578+0000","last_active":"2024-03-06T07:31:32.676802+0000","last_peered":"2024-03-06T07:31:32.676802+0000","last_clean":"2024-03-06T07:31:32.676802+0000","last_became_active":"2024-03-06T07:31:32.670295+0000","last_became_peered":"2024-03-06T07:31:32.670295+0000","last_unstale":"2024-03-06T07:31:32.676802+0000","last_undegraded":"2024-03-06T07:31:32.676802+0000","last_fullsized":"2024-03-06T07:31:32.676802+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:34:50.495990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.30","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672235+0000","last_change":"2024-03-06T07:31:32.675340+0000","last_active":"2024-03-06T07:31:33.672235+0000","last_peered":"2024-03-06T07:31:33.672235+0000","last_clean":"2024-03-06T07:31:33.672235+0000","last_became_active":"2024-03-06T07:31:32.675058+0000","last_became_peered":"2024-03-06T07:31:32.675058+0000","last_unstale":"2024-03-06T07:31:33.672235+0000","last_undegraded":"2024-03-06T07:31:33.672235+0000","last_fullsized":"2024-03-06T07:31:33.672235+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:20.052352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2f","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673599+0000","last_change":"2024-03-06T07:31:32.670892+0000","last_active":"2024-03-06T07:31:33.673599+0000","last_peered":"2024-03-06T07:31:33.673599+0000","last_clean":"2024-03-06T07:31:33.673599+0000","last_became_active":"2024-03-06T07:31:32.669520+0000","last_became_peered":"2024-03-06T07:31:32.669520+0000","last_unstale":"2024-03-06T07:31:33.673599+0000","last_undegraded":"2024-03-06T07:31:33.673599+0000","last_fullsized":"2024-03-06T07:31:33.673599+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:53:04.940938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2e","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108036+0000","last_change":"2024-03-06T07:31:32.684321+0000","last_active":"2024-03-06T07:31:34.108036+0000","last_peered":"2024-03-06T07:31:34.108036+0000","last_clean":"2024-03-06T07:31:34.108036+0000","last_became_active":"2024-03-06T07:31:32.682769+0000","last_became_peered":"2024-03-06T07:31:32.682769+0000","last_unstale":"2024-03-06T07:31:34.108036+0000","last_undegraded":"2024-03-06T07:31:34.108036+0000","last_fullsized":"2024-03-06T07:31:34.108036+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:01:06.658665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670477+0000","last_change":"2024-03-06T07:31:32.668423+0000","last_active":"2024-03-06T07:31:33.670477+0000","last_peered":"2024-03-06T07:31:33.670477+0000","last_clean":"2024-03-06T07:31:33.670477+0000","last_became_active":"2024-03-06T07:31:32.664979+0000","last_became_peered":"2024-03-06T07:31:32.664979+0000","last_unstale":"2024-03-06T07:31:33.670477+0000","last_undegraded":"2024-03-06T07:31:33.670477+0000","last_fullsized":"2024-03-06T07:31:33.670477+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:09.898923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.2c","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108007+0000","last_change":"2024-03-06T07:31:32.676384+0000","last_active":"2024-03-06T07:31:34.108007+0000","last_peered":"2024-03-06T07:31:34.108007+0000","last_clean":"2024-03-06T07:31:34.108007+0000","last_became_active":"2024-03-06T07:31:32.675597+0000","last_became_peered":"2024-03-06T07:31:32.675597+0000","last_unstale":"2024-03-06T07:31:34.108007+0000","last_undegraded":"2024-03-06T07:31:34.108007+0000","last_fullsized":"2024-03-06T07:31:34.108007+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:40:45.029984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2b","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107762+0000","last_change":"2024-03-06T07:31:32.682026+0000","last_active":"2024-03-06T07:31:34.107762+0000","last_peered":"2024-03-06T07:31:34.107762+0000","last_clean":"2024-03-06T07:31:34.107762+0000","last_became_active":"2024-03-06T07:31:32.676359+0000","last_became_peered":"2024-03-06T07:31:32.676359+0000","last_unstale":"2024-03-06T07:31:34.107762+0000","last_undegraded":"2024-03-06T07:31:34.107762+0000","last_fullsized":"2024-03-06T07:31:34.107762+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:29:24.740651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2a","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668410+0000","last_change":"2024-03-06T07:31:32.665825+0000","last_active":"2024-03-06T07:31:32.668410+0000","last_peered":"2024-03-06T07:31:32.668410+0000","last_clean":"2024-03-06T07:31:32.668410+0000","last_became_active":"2024-03-06T07:31:32.664824+0000","last_became_peered":"2024-03-06T07:31:32.664824+0000","last_unstale":"2024-03-06T07:31:32.668410+0000","last_undegraded":"2024-03-06T07:31:32.668410+0000","last_fullsized":"2024-03-06T07:31:32.668410+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:56:51.841231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.29","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691614+0000","last_change":"2024-03-06T07:31:32.687227+0000","last_active":"2024-03-06T07:31:32.691614+0000","last_peered":"2024-03-06T07:31:32.691614+0000","last_clean":"2024-03-06T07:31:32.691614+0000","last_became_active":"2024-03-06T07:31:32.686932+0000","last_became_peered":"2024-03-06T07:31:32.686932+0000","last_unstale":"2024-03-06T07:31:32.691614+0000","last_undegraded":"2024-03-06T07:31:32.691614+0000","last_fullsized":"2024-03-06T07:31:32.691614+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:14:44.586900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.28","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.106978+0000","last_change":"2024-03-06T07:31:32.678144+0000","last_active":"2024-03-06T07:31:34.106978+0000","last_peered":"2024-03-06T07:31:34.106978+0000","last_clean":"2024-03-06T07:31:34.106978+0000","last_became_active":"2024-03-06T07:31:32.677856+0000","last_became_peered":"2024-03-06T07:31:32.677856+0000","last_unstale":"2024-03-06T07:31:34.106978+0000","last_undegraded":"2024-03-06T07:31:34.106978+0000","last_fullsized":"2024-03-06T07:31:34.106978+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:20.813609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.27","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687651+0000","last_change":"2024-03-06T07:31:32.682633+0000","last_active":"2024-03-06T07:31:32.687651+0000","last_peered":"2024-03-06T07:31:32.687651+0000","last_clean":"2024-03-06T07:31:32.687651+0000","last_became_active":"2024-03-06T07:31:32.682339+0000","last_became_peered":"2024-03-06T07:31:32.682339+0000","last_unstale":"2024-03-06T07:31:32.687651+0000","last_undegraded":"2024-03-06T07:31:32.687651+0000","last_fullsized":"2024-03-06T07:31:32.687651+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:27:08.545866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.26","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868589+0000","last_change":"2024-03-06T07:31:32.672517+0000","last_active":"2024-03-06T07:31:34.868589+0000","last_peered":"2024-03-06T07:31:34.868589+0000","last_clean":"2024-03-06T07:31:34.868589+0000","last_became_active":"2024-03-06T07:31:32.672224+0000","last_became_peered":"2024-03-06T07:31:32.672224+0000","last_unstale":"2024-03-06T07:31:34.868589+0000","last_undegraded":"2024-03-06T07:31:34.868589+0000","last_fullsized":"2024-03-06T07:31:34.868589+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:04:10.713397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.25","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673154+0000","last_change":"2024-03-06T07:31:32.664377+0000","last_active":"2024-03-06T07:31:33.673154+0000","last_peered":"2024-03-06T07:31:33.673154+0000","last_clean":"2024-03-06T07:31:33.673154+0000","last_became_active":"2024-03-06T07:31:32.663658+0000","last_became_peered":"2024-03-06T07:31:32.663658+0000","last_unstale":"2024-03-06T07:31:33.673154+0000","last_undegraded":"2024-03-06T07:31:33.673154+0000","last_fullsized":"2024-03-06T07:31:33.673154+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:03:14.713038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.24","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.682194+0000","last_change":"2024-03-06T07:31:32.675255+0000","last_active":"2024-03-06T07:31:32.682194+0000","last_peered":"2024-03-06T07:31:32.682194+0000","last_clean":"2024-03-06T07:31:32.682194+0000","last_became_active":"2024-03-06T07:31:32.674713+0000","last_became_peered":"2024-03-06T07:31:32.674713+0000","last_unstale":"2024-03-06T07:31:32.682194+0000","last_undegraded":"2024-03-06T07:31:32.682194+0000","last_fullsized":"2024-03-06T07:31:32.682194+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:33:22.109879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.23","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668345+0000","last_change":"2024-03-06T07:31:32.665334+0000","last_active":"2024-03-06T07:31:32.668345+0000","last_peered":"2024-03-06T07:31:32.668345+0000","last_clean":"2024-03-06T07:31:32.668345+0000","last_became_active":"2024-03-06T07:31:32.665040+0000","last_became_peered":"2024-03-06T07:31:32.665040+0000","last_unstale":"2024-03-06T07:31:32.668345+0000","last_undegraded":"2024-03-06T07:31:32.668345+0000","last_fullsized":"2024-03-06T07:31:32.668345+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:27:41.527639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.22","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668393+0000","last_change":"2024-03-06T07:31:32.666617+0000","last_active":"2024-03-06T07:31:32.668393+0000","last_peered":"2024-03-06T07:31:32.668393+0000","last_clean":"2024-03-06T07:31:32.668393+0000","last_became_active":"2024-03-06T07:31:32.665570+0000","last_became_peered":"2024-03-06T07:31:32.665570+0000","last_unstale":"2024-03-06T07:31:32.668393+0000","last_undegraded":"2024-03-06T07:31:32.668393+0000","last_fullsized":"2024-03-06T07:31:32.668393+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:10:42.407024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.21","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672307+0000","last_change":"2024-03-06T07:31:32.659145+0000","last_active":"2024-03-06T07:31:33.672307+0000","last_peered":"2024-03-06T07:31:33.672307+0000","last_clean":"2024-03-06T07:31:33.672307+0000","last_became_active":"2024-03-06T07:31:32.658411+0000","last_became_peered":"2024-03-06T07:31:32.658411+0000","last_unstale":"2024-03-06T07:31:33.672307+0000","last_undegraded":"2024-03-06T07:31:33.672307+0000","last_fullsized":"2024-03-06T07:31:33.672307+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:37:54.485529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.20","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.694244+0000","last_change":"2024-03-06T07:31:32.689906+0000","last_active":"2024-03-06T07:31:32.694244+0000","last_peered":"2024-03-06T07:31:32.694244+0000","last_clean":"2024-03-06T07:31:32.694244+0000","last_became_active":"2024-03-06T07:31:32.689401+0000","last_became_peered":"2024-03-06T07:31:32.689401+0000","last_unstale":"2024-03-06T07:31:32.694244+0000","last_undegraded":"2024-03-06T07:31:32.694244+0000","last_fullsized":"2024-03-06T07:31:32.694244+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:32:28.951977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1e","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673530+0000","last_change":"2024-03-06T07:31:32.683082+0000","last_active":"2024-03-06T07:31:33.673530+0000","last_peered":"2024-03-06T07:31:33.673530+0000","last_clean":"2024-03-06T07:31:33.673530+0000","last_became_active":"2024-03-06T07:31:32.682299+0000","last_became_peered":"2024-03-06T07:31:32.682299+0000","last_unstale":"2024-03-06T07:31:33.673530+0000","last_undegraded":"2024-03-06T07:31:33.673530+0000","last_fullsized":"2024-03-06T07:31:33.673530+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:58:40.759611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1f","version":"82'4","reported_seq":16,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.689127+0000","last_change":"2024-03-06T07:31:31.659131+0000","last_active":"2024-03-06T07:31:32.689127+0000","last_peered":"2024-03-06T07:31:32.689127+0000","last_clean":"2024-03-06T07:31:32.689127+0000","last_became_active":"2024-03-06T07:31:31.658820+0000","last_became_peered":"2024-03-06T07:31:31.658820+0000","last_unstale":"2024-03-06T07:31:32.689127+0000","last_undegraded":"2024-03-06T07:31:32.689127+0000","last_fullsized":"2024-03-06T07:31:32.689127+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:39:56.545347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1f","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691578+0000","last_change":"2024-03-06T07:31:32.687031+0000","last_active":"2024-03-06T07:31:32.691578+0000","last_peered":"2024-03-06T07:31:32.691578+0000","last_clean":"2024-03-06T07:31:32.691578+0000","last_became_active":"2024-03-06T07:31:32.686723+0000","last_became_peered":"2024-03-06T07:31:32.686723+0000","last_unstale":"2024-03-06T07:31:32.691578+0000","last_undegraded":"2024-03-06T07:31:32.691578+0000","last_fullsized":"2024-03-06T07:31:32.691578+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:49:43.487650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1e","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673835+0000","last_change":"2024-03-06T07:31:31.647876+0000","last_active":"2024-03-06T07:31:33.673835+0000","last_peered":"2024-03-06T07:31:33.673835+0000","last_clean":"2024-03-06T07:31:33.673835+0000","last_became_active":"2024-03-06T07:31:31.647139+0000","last_became_peered":"2024-03-06T07:31:31.647139+0000","last_unstale":"2024-03-06T07:31:33.673835+0000","last_undegraded":"2024-03-06T07:31:33.673835+0000","last_fullsized":"2024-03-06T07:31:33.673835+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:08.521391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1c","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.694261+0000","last_change":"2024-03-06T07:31:32.690207+0000","last_active":"2024-03-06T07:31:32.694261+0000","last_peered":"2024-03-06T07:31:32.694261+0000","last_clean":"2024-03-06T07:31:32.694261+0000","last_became_active":"2024-03-06T07:31:32.689838+0000","last_became_peered":"2024-03-06T07:31:32.689838+0000","last_unstale":"2024-03-06T07:31:32.694261+0000","last_undegraded":"2024-03-06T07:31:32.694261+0000","last_fullsized":"2024-03-06T07:31:32.694261+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:36:12.582342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1d","version":"82'2","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670506+0000","last_change":"2024-03-06T07:31:31.651404+0000","last_active":"2024-03-06T07:31:33.670506+0000","last_peered":"2024-03-06T07:31:33.670506+0000","last_clean":"2024-03-06T07:31:33.670506+0000","last_became_active":"2024-03-06T07:31:31.650775+0000","last_became_peered":"2024-03-06T07:31:31.650775+0000","last_unstale":"2024-03-06T07:31:33.670506+0000","last_undegraded":"2024-03-06T07:31:33.670506+0000","last_fullsized":"2024-03-06T07:31:33.670506+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:19.036862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868493+0000","last_change":"2024-03-06T07:31:32.676577+0000","last_active":"2024-03-06T07:31:34.868493+0000","last_peered":"2024-03-06T07:31:34.868493+0000","last_clean":"2024-03-06T07:31:34.868493+0000","last_became_active":"2024-03-06T07:31:32.675322+0000","last_became_peered":"2024-03-06T07:31:32.675322+0000","last_unstale":"2024-03-06T07:31:34.868493+0000","last_undegraded":"2024-03-06T07:31:34.868493+0000","last_fullsized":"2024-03-06T07:31:34.868493+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:58:40.759611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1c","version":"82'2","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670256+0000","last_change":"2024-03-06T07:31:31.649485+0000","last_active":"2024-03-06T07:31:33.670256+0000","last_peered":"2024-03-06T07:31:33.670256+0000","last_clean":"2024-03-06T07:31:33.670256+0000","last_became_active":"2024-03-06T07:31:31.648894+0000","last_became_peered":"2024-03-06T07:31:31.648894+0000","last_unstale":"2024-03-06T07:31:33.670256+0000","last_undegraded":"2024-03-06T07:31:33.670256+0000","last_fullsized":"2024-03-06T07:31:33.670256+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:56:25.616933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1a","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.668376+0000","last_change":"2024-03-06T07:31:32.666076+0000","last_active":"2024-03-06T07:31:32.668376+0000","last_peered":"2024-03-06T07:31:32.668376+0000","last_clean":"2024-03-06T07:31:32.668376+0000","last_became_active":"2024-03-06T07:31:32.665182+0000","last_became_peered":"2024-03-06T07:31:32.665182+0000","last_unstale":"2024-03-06T07:31:32.668376+0000","last_undegraded":"2024-03-06T07:31:32.668376+0000","last_fullsized":"2024-03-06T07:31:32.668376+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:21:38.242551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.1b","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672859+0000","last_change":"2024-03-06T07:31:31.643318+0000","last_active":"2024-03-06T07:31:33.672859+0000","last_peered":"2024-03-06T07:31:33.672859+0000","last_clean":"2024-03-06T07:31:33.672859+0000","last_became_active":"2024-03-06T07:31:31.640903+0000","last_became_peered":"2024-03-06T07:31:31.640903+0000","last_unstale":"2024-03-06T07:31:33.672859+0000","last_undegraded":"2024-03-06T07:31:33.672859+0000","last_fullsized":"2024-03-06T07:31:33.672859+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:12:42.107769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1b","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672668+0000","last_change":"2024-03-06T07:31:32.668424+0000","last_active":"2024-03-06T07:31:33.672668+0000","last_peered":"2024-03-06T07:31:33.672668+0000","last_clean":"2024-03-06T07:31:33.672668+0000","last_became_active":"2024-03-06T07:31:32.668097+0000","last_became_peered":"2024-03-06T07:31:32.668097+0000","last_unstale":"2024-03-06T07:31:33.672668+0000","last_undegraded":"2024-03-06T07:31:33.672668+0000","last_fullsized":"2024-03-06T07:31:33.672668+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:19:29.966268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1a","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672275+0000","last_change":"2024-03-06T07:31:31.651337+0000","last_active":"2024-03-06T07:31:33.672275+0000","last_peered":"2024-03-06T07:31:33.672275+0000","last_clean":"2024-03-06T07:31:33.672275+0000","last_became_active":"2024-03-06T07:31:31.651061+0000","last_became_peered":"2024-03-06T07:31:31.651061+0000","last_unstale":"2024-03-06T07:31:33.672275+0000","last_undegraded":"2024-03-06T07:31:33.672275+0000","last_fullsized":"2024-03-06T07:31:33.672275+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:06:11.195300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.18","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867476+0000","last_change":"2024-03-06T07:31:32.670874+0000","last_active":"2024-03-06T07:31:34.867476+0000","last_peered":"2024-03-06T07:31:34.867476+0000","last_clean":"2024-03-06T07:31:34.867476+0000","last_became_active":"2024-03-06T07:31:32.670542+0000","last_became_peered":"2024-03-06T07:31:32.670542+0000","last_unstale":"2024-03-06T07:31:34.867476+0000","last_undegraded":"2024-03-06T07:31:34.867476+0000","last_fullsized":"2024-03-06T07:31:34.867476+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:37:54.485529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669632+0000","last_change":"2024-03-06T07:31:31.657497+0000","last_active":"2024-03-06T07:31:33.669632+0000","last_peered":"2024-03-06T07:31:33.669632+0000","last_clean":"2024-03-06T07:31:33.669632+0000","last_became_active":"2024-03-06T07:31:31.657120+0000","last_became_peered":"2024-03-06T07:31:31.657120+0000","last_unstale":"2024-03-06T07:31:33.669632+0000","last_undegraded":"2024-03-06T07:31:33.669632+0000","last_fullsized":"2024-03-06T07:31:33.669632+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:42:19.467943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.19","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.677287+0000","last_change":"2024-03-06T07:31:32.673212+0000","last_active":"2024-03-06T07:31:32.677287+0000","last_peered":"2024-03-06T07:31:32.677287+0000","last_clean":"2024-03-06T07:31:32.677287+0000","last_became_active":"2024-03-06T07:31:32.672905+0000","last_became_peered":"2024-03-06T07:31:32.672905+0000","last_unstale":"2024-03-06T07:31:32.677287+0000","last_undegraded":"2024-03-06T07:31:32.677287+0000","last_fullsized":"2024-03-06T07:31:32.677287+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:33:32.681162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.646886+0000","last_change":"2024-03-06T07:31:31.645464+0000","last_active":"2024-03-06T07:31:32.646886+0000","last_peered":"2024-03-06T07:31:32.646886+0000","last_clean":"2024-03-06T07:31:32.646886+0000","last_became_active":"2024-03-06T07:31:31.645201+0000","last_became_peered":"2024-03-06T07:31:31.645201+0000","last_unstale":"2024-03-06T07:31:32.646886+0000","last_undegraded":"2024-03-06T07:31:32.646886+0000","last_fullsized":"2024-03-06T07:31:32.646886+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:13.566188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.16","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.664798+0000","last_change":"2024-03-06T07:31:32.662604+0000","last_active":"2024-03-06T07:31:32.664798+0000","last_peered":"2024-03-06T07:31:32.664798+0000","last_clean":"2024-03-06T07:31:32.664798+0000","last_became_active":"2024-03-06T07:31:32.662316+0000","last_became_peered":"2024-03-06T07:31:32.662316+0000","last_unstale":"2024-03-06T07:31:32.664798+0000","last_undegraded":"2024-03-06T07:31:32.664798+0000","last_fullsized":"2024-03-06T07:31:32.664798+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:26:04.337185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.667170+0000","last_change":"2024-03-06T07:31:31.656325+0000","last_active":"2024-03-06T07:31:32.667170+0000","last_peered":"2024-03-06T07:31:32.667170+0000","last_clean":"2024-03-06T07:31:32.667170+0000","last_became_active":"2024-03-06T07:31:31.656045+0000","last_became_peered":"2024-03-06T07:31:31.656045+0000","last_unstale":"2024-03-06T07:31:32.667170+0000","last_undegraded":"2024-03-06T07:31:32.667170+0000","last_fullsized":"2024-03-06T07:31:32.667170+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:14:20.341113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688275+0000","last_change":"2024-03-06T07:31:32.683458+0000","last_active":"2024-03-06T07:31:32.688275+0000","last_peered":"2024-03-06T07:31:32.688275+0000","last_clean":"2024-03-06T07:31:32.688275+0000","last_became_active":"2024-03-06T07:31:32.683186+0000","last_became_peered":"2024-03-06T07:31:32.683186+0000","last_unstale":"2024-03-06T07:31:32.688275+0000","last_undegraded":"2024-03-06T07:31:32.688275+0000","last_fullsized":"2024-03-06T07:31:32.688275+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:14:44.586900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.669694+0000","last_change":"2024-03-06T07:31:31.654941+0000","last_active":"2024-03-06T07:31:32.669694+0000","last_peered":"2024-03-06T07:31:32.669694+0000","last_clean":"2024-03-06T07:31:32.669694+0000","last_became_active":"2024-03-06T07:31:31.654185+0000","last_became_peered":"2024-03-06T07:31:31.654185+0000","last_unstale":"2024-03-06T07:31:32.669694+0000","last_undegraded":"2024-03-06T07:31:32.669694+0000","last_fullsized":"2024-03-06T07:31:32.669694+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:24:12.685215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.14","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670163+0000","last_change":"2024-03-06T07:31:32.685450+0000","last_active":"2024-03-06T07:31:33.670163+0000","last_peered":"2024-03-06T07:31:33.670163+0000","last_clean":"2024-03-06T07:31:33.670163+0000","last_became_active":"2024-03-06T07:31:32.685174+0000","last_became_peered":"2024-03-06T07:31:32.685174+0000","last_unstale":"2024-03-06T07:31:33.670163+0000","last_undegraded":"2024-03-06T07:31:33.670163+0000","last_fullsized":"2024-03-06T07:31:33.670163+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:23:44.975998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.15","version":"82'3","reported_seq":15,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.692069+0000","last_change":"2024-03-06T07:31:31.658088+0000","last_active":"2024-03-06T07:31:32.692069+0000","last_peered":"2024-03-06T07:31:32.692069+0000","last_clean":"2024-03-06T07:31:32.692069+0000","last_became_active":"2024-03-06T07:31:31.657484+0000","last_became_peered":"2024-03-06T07:31:31.657484+0000","last_unstale":"2024-03-06T07:31:32.692069+0000","last_undegraded":"2024-03-06T07:31:32.692069+0000","last_fullsized":"2024-03-06T07:31:32.692069+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:01:54.010077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.15","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673253+0000","last_change":"2024-03-06T07:31:32.664158+0000","last_active":"2024-03-06T07:31:33.673253+0000","last_peered":"2024-03-06T07:31:33.673253+0000","last_clean":"2024-03-06T07:31:33.673253+0000","last_became_active":"2024-03-06T07:31:32.663298+0000","last_became_peered":"2024-03-06T07:31:32.663298+0000","last_unstale":"2024-03-06T07:31:33.673253+0000","last_undegraded":"2024-03-06T07:31:33.673253+0000","last_fullsized":"2024-03-06T07:31:33.673253+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:46:33.261291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"82'6","reported_seq":18,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688160+0000","last_change":"2024-03-06T07:31:31.657354+0000","last_active":"2024-03-06T07:31:32.688160+0000","last_peered":"2024-03-06T07:31:32.688160+0000","last_clean":"2024-03-06T07:31:32.688160+0000","last_became_active":"2024-03-06T07:31:31.657065+0000","last_became_peered":"2024-03-06T07:31:31.657065+0000","last_unstale":"2024-03-06T07:31:32.688160+0000","last_undegraded":"2024-03-06T07:31:32.688160+0000","last_fullsized":"2024-03-06T07:31:32.688160+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:31:29.982548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.12","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669913+0000","last_change":"2024-03-06T07:31:32.665264+0000","last_active":"2024-03-06T07:31:33.669913+0000","last_peered":"2024-03-06T07:31:33.669913+0000","last_clean":"2024-03-06T07:31:33.669913+0000","last_became_active":"2024-03-06T07:31:32.664967+0000","last_became_peered":"2024-03-06T07:31:32.664967+0000","last_unstale":"2024-03-06T07:31:33.669913+0000","last_undegraded":"2024-03-06T07:31:33.669913+0000","last_fullsized":"2024-03-06T07:31:33.669913+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:06.083666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.13","version":"82'2","reported_seq":14,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.690741+0000","last_change":"2024-03-06T07:31:31.647387+0000","last_active":"2024-03-06T07:31:32.690741+0000","last_peered":"2024-03-06T07:31:32.690741+0000","last_clean":"2024-03-06T07:31:32.690741+0000","last_became_active":"2024-03-06T07:31:31.647084+0000","last_became_peered":"2024-03-06T07:31:31.647084+0000","last_unstale":"2024-03-06T07:31:32.690741+0000","last_undegraded":"2024-03-06T07:31:32.690741+0000","last_fullsized":"2024-03-06T07:31:32.690741+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:34:42.642586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.13","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867739+0000","last_change":"2024-03-06T07:31:32.681572+0000","last_active":"2024-03-06T07:31:34.867739+0000","last_peered":"2024-03-06T07:31:34.867739+0000","last_clean":"2024-03-06T07:31:34.867739+0000","last_became_active":"2024-03-06T07:31:32.681291+0000","last_became_peered":"2024-03-06T07:31:32.681291+0000","last_unstale":"2024-03-06T07:31:34.867739+0000","last_undegraded":"2024-03-06T07:31:34.867739+0000","last_fullsized":"2024-03-06T07:31:34.867739+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:19:29.966268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.654790+0000","last_change":"2024-03-06T07:31:31.650179+0000","last_active":"2024-03-06T07:31:32.654790+0000","last_peered":"2024-03-06T07:31:32.654790+0000","last_clean":"2024-03-06T07:31:32.654790+0000","last_became_active":"2024-03-06T07:31:31.648291+0000","last_became_peered":"2024-03-06T07:31:31.648291+0000","last_unstale":"2024-03-06T07:31:32.654790+0000","last_undegraded":"2024-03-06T07:31:32.654790+0000","last_fullsized":"2024-03-06T07:31:32.654790+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:08.883433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.10","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.678151+0000","last_change":"2024-03-06T07:31:32.669793+0000","last_active":"2024-03-06T07:31:32.678151+0000","last_peered":"2024-03-06T07:31:32.678151+0000","last_clean":"2024-03-06T07:31:32.678151+0000","last_became_active":"2024-03-06T07:31:32.668693+0000","last_became_peered":"2024-03-06T07:31:32.668693+0000","last_unstale":"2024-03-06T07:31:32.678151+0000","last_undegraded":"2024-03-06T07:31:32.678151+0000","last_fullsized":"2024-03-06T07:31:32.678151+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:49:09.065828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.669837+0000","last_change":"2024-03-06T07:31:31.659907+0000","last_active":"2024-03-06T07:31:32.669837+0000","last_peered":"2024-03-06T07:31:32.669837+0000","last_clean":"2024-03-06T07:31:32.669837+0000","last_became_active":"2024-03-06T07:31:31.659582+0000","last_became_peered":"2024-03-06T07:31:31.659582+0000","last_unstale":"2024-03-06T07:31:32.669837+0000","last_undegraded":"2024-03-06T07:31:32.669837+0000","last_fullsized":"2024-03-06T07:31:32.669837+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:24:26.878907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.11","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.691107+0000","last_change":"2024-03-06T07:31:32.685033+0000","last_active":"2024-03-06T07:31:32.691107+0000","last_peered":"2024-03-06T07:31:32.691107+0000","last_clean":"2024-03-06T07:31:32.691107+0000","last_became_active":"2024-03-06T07:31:32.684557+0000","last_became_peered":"2024-03-06T07:31:32.684557+0000","last_unstale":"2024-03-06T07:31:32.691107+0000","last_undegraded":"2024-03-06T07:31:32.691107+0000","last_fullsized":"2024-03-06T07:31:32.691107+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:51:05.876283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.10","version":"82'8","reported_seq":22,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669589+0000","last_change":"2024-03-06T07:31:31.659643+0000","last_active":"2024-03-06T07:31:33.669589+0000","last_peered":"2024-03-06T07:31:33.669589+0000","last_clean":"2024-03-06T07:31:33.669589+0000","last_became_active":"2024-03-06T07:31:31.659411+0000","last_became_peered":"2024-03-06T07:31:31.659411+0000","last_unstale":"2024-03-06T07:31:33.669589+0000","last_undegraded":"2024-03-06T07:31:33.669589+0000","last_fullsized":"2024-03-06T07:31:33.669589+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:00.810966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868641+0000","last_change":"2024-03-06T07:31:32.679538+0000","last_active":"2024-03-06T07:31:34.868641+0000","last_peered":"2024-03-06T07:31:34.868641+0000","last_clean":"2024-03-06T07:31:34.868641+0000","last_became_active":"2024-03-06T07:31:32.679229+0000","last_became_peered":"2024-03-06T07:31:32.679229+0000","last_unstale":"2024-03-06T07:31:34.868641+0000","last_undegraded":"2024-03-06T07:31:34.868641+0000","last_fullsized":"2024-03-06T07:31:34.868641+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:06.083666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.f","version":"82'1","reported_seq":13,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687864+0000","last_change":"2024-03-06T07:31:31.661061+0000","last_active":"2024-03-06T07:31:32.687864+0000","last_peered":"2024-03-06T07:31:32.687864+0000","last_clean":"2024-03-06T07:31:32.687864+0000","last_became_active":"2024-03-06T07:31:31.660259+0000","last_became_peered":"2024-03-06T07:31:31.660259+0000","last_unstale":"2024-03-06T07:31:32.687864+0000","last_undegraded":"2024-03-06T07:31:32.687864+0000","last_fullsized":"2024-03-06T07:31:32.687864+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:09:26.069429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.680984+0000","last_change":"2024-03-06T07:31:32.676087+0000","last_active":"2024-03-06T07:31:32.680984+0000","last_peered":"2024-03-06T07:31:32.680984+0000","last_clean":"2024-03-06T07:31:32.680984+0000","last_became_active":"2024-03-06T07:31:32.675819+0000","last_became_peered":"2024-03-06T07:31:32.675819+0000","last_unstale":"2024-03-06T07:31:32.680984+0000","last_undegraded":"2024-03-06T07:31:32.680984+0000","last_fullsized":"2024-03-06T07:31:32.680984+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:55:05.336895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"82'2","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670014+0000","last_change":"2024-03-06T07:31:31.646735+0000","last_active":"2024-03-06T07:31:33.670014+0000","last_peered":"2024-03-06T07:31:33.670014+0000","last_clean":"2024-03-06T07:31:33.670014+0000","last_became_active":"2024-03-06T07:31:31.646124+0000","last_became_peered":"2024-03-06T07:31:31.646124+0000","last_unstale":"2024-03-06T07:31:33.670014+0000","last_undegraded":"2024-03-06T07:31:33.670014+0000","last_fullsized":"2024-03-06T07:31:33.670014+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:37:53.470039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.d","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673757+0000","last_change":"2024-03-06T07:31:32.661002+0000","last_active":"2024-03-06T07:31:33.673757+0000","last_peered":"2024-03-06T07:31:33.673757+0000","last_clean":"2024-03-06T07:31:33.673757+0000","last_became_active":"2024-03-06T07:31:32.660602+0000","last_became_peered":"2024-03-06T07:31:32.660602+0000","last_unstale":"2024-03-06T07:31:33.673757+0000","last_undegraded":"2024-03-06T07:31:33.673757+0000","last_fullsized":"2024-03-06T07:31:33.673757+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:48:41.873102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.670337+0000","last_change":"2024-03-06T07:31:31.661640+0000","last_active":"2024-03-06T07:31:33.670337+0000","last_peered":"2024-03-06T07:31:33.670337+0000","last_clean":"2024-03-06T07:31:33.670337+0000","last_became_active":"2024-03-06T07:31:31.661318+0000","last_became_peered":"2024-03-06T07:31:31.661318+0000","last_unstale":"2024-03-06T07:31:33.670337+0000","last_undegraded":"2024-03-06T07:31:33.670337+0000","last_fullsized":"2024-03-06T07:31:33.670337+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:19:28.950778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.108172+0000","last_change":"2024-03-06T07:31:32.674307+0000","last_active":"2024-03-06T07:31:34.108172+0000","last_peered":"2024-03-06T07:31:34.108172+0000","last_clean":"2024-03-06T07:31:34.108172+0000","last_became_active":"2024-03-06T07:31:32.669944+0000","last_became_peered":"2024-03-06T07:31:32.669944+0000","last_unstale":"2024-03-06T07:31:34.108172+0000","last_undegraded":"2024-03-06T07:31:34.108172+0000","last_fullsized":"2024-03-06T07:31:34.108172+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:14.581678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"82'2","reported_seq":14,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.687842+0000","last_change":"2024-03-06T07:31:31.651064+0000","last_active":"2024-03-06T07:31:32.687842+0000","last_peered":"2024-03-06T07:31:32.687842+0000","last_clean":"2024-03-06T07:31:32.687842+0000","last_became_active":"2024-03-06T07:31:31.650778+0000","last_became_peered":"2024-03-06T07:31:31.650778+0000","last_unstale":"2024-03-06T07:31:32.687842+0000","last_undegraded":"2024-03-06T07:31:32.687842+0000","last_fullsized":"2024-03-06T07:31:32.687842+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:22:23.068492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.681840+0000","last_change":"2024-03-06T07:31:32.676297+0000","last_active":"2024-03-06T07:31:32.681840+0000","last_peered":"2024-03-06T07:31:32.681840+0000","last_clean":"2024-03-06T07:31:32.681840+0000","last_became_active":"2024-03-06T07:31:32.675984+0000","last_became_peered":"2024-03-06T07:31:32.675984+0000","last_unstale":"2024-03-06T07:31:32.681840+0000","last_undegraded":"2024-03-06T07:31:32.681840+0000","last_fullsized":"2024-03-06T07:31:32.681840+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:07.717126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107854+0000","last_change":"2024-03-06T07:31:31.648998+0000","last_active":"2024-03-06T07:31:34.107854+0000","last_peered":"2024-03-06T07:31:34.107854+0000","last_clean":"2024-03-06T07:31:34.107854+0000","last_became_active":"2024-03-06T07:31:31.647766+0000","last_became_peered":"2024-03-06T07:31:31.647766+0000","last_unstale":"2024-03-06T07:31:34.107854+0000","last_undegraded":"2024-03-06T07:31:34.107854+0000","last_fullsized":"2024-03-06T07:31:34.107854+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:37:17.302476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107070+0000","last_change":"2024-03-06T07:31:32.678203+0000","last_active":"2024-03-06T07:31:34.107070+0000","last_peered":"2024-03-06T07:31:34.107070+0000","last_clean":"2024-03-06T07:31:34.107070+0000","last_became_active":"2024-03-06T07:31:32.677921+0000","last_became_peered":"2024-03-06T07:31:32.677921+0000","last_unstale":"2024-03-06T07:31:34.107070+0000","last_undegraded":"2024-03-06T07:31:34.107070+0000","last_fullsized":"2024-03-06T07:31:34.107070+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:50:28.238836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.107347+0000","last_change":"2024-03-06T07:31:31.662031+0000","last_active":"2024-03-06T07:31:34.107347+0000","last_peered":"2024-03-06T07:31:34.107347+0000","last_clean":"2024-03-06T07:31:34.107347+0000","last_became_active":"2024-03-06T07:31:31.661697+0000","last_became_peered":"2024-03-06T07:31:31.661697+0000","last_unstale":"2024-03-06T07:31:34.107347+0000","last_undegraded":"2024-03-06T07:31:34.107347+0000","last_fullsized":"2024-03-06T07:31:34.107347+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:08:45.128965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.9","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.688854+0000","last_change":"2024-03-06T07:31:32.687887+0000","last_active":"2024-03-06T07:31:32.688854+0000","last_peered":"2024-03-06T07:31:32.688854+0000","last_clean":"2024-03-06T07:31:32.688854+0000","last_became_active":"2024-03-06T07:31:32.687573+0000","last_became_peered":"2024-03-06T07:31:32.687573+0000","last_unstale":"2024-03-06T07:31:32.688854+0000","last_undegraded":"2024-03-06T07:31:32.688854+0000","last_fullsized":"2024-03-06T07:31:32.688854+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:36:12.582342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.654550+0000","last_change":"2024-03-06T07:31:31.649164+0000","last_active":"2024-03-06T07:31:32.654550+0000","last_peered":"2024-03-06T07:31:32.654550+0000","last_clean":"2024-03-06T07:31:32.654550+0000","last_became_active":"2024-03-06T07:31:31.648895+0000","last_became_peered":"2024-03-06T07:31:31.648895+0000","last_unstale":"2024-03-06T07:31:32.654550+0000","last_undegraded":"2024-03-06T07:31:32.654550+0000","last_fullsized":"2024-03-06T07:31:32.654550+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:23:43.960508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673751+0000","last_change":"2024-03-06T07:31:32.676758+0000","last_active":"2024-03-06T07:31:33.673751+0000","last_peered":"2024-03-06T07:31:33.673751+0000","last_clean":"2024-03-06T07:31:33.673751+0000","last_became_active":"2024-03-06T07:31:32.676479+0000","last_became_peered":"2024-03-06T07:31:32.676479+0000","last_unstale":"2024-03-06T07:31:33.673751+0000","last_undegraded":"2024-03-06T07:31:33.673751+0000","last_fullsized":"2024-03-06T07:31:33.673751+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:06.083666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"82'2","reported_seq":14,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.681646+0000","last_change":"2024-03-06T07:31:31.659130+0000","last_active":"2024-03-06T07:31:32.681646+0000","last_peered":"2024-03-06T07:31:32.681646+0000","last_clean":"2024-03-06T07:31:32.681646+0000","last_became_active":"2024-03-06T07:31:31.658936+0000","last_became_peered":"2024-03-06T07:31:31.658936+0000","last_unstale":"2024-03-06T07:31:32.681646+0000","last_undegraded":"2024-03-06T07:31:32.681646+0000","last_fullsized":"2024-03-06T07:31:32.681646+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:19:48.824088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.7","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868533+0000","last_change":"2024-03-06T07:31:32.683857+0000","last_active":"2024-03-06T07:31:34.868533+0000","last_peered":"2024-03-06T07:31:34.868533+0000","last_clean":"2024-03-06T07:31:34.868533+0000","last_became_active":"2024-03-06T07:31:32.683443+0000","last_became_peered":"2024-03-06T07:31:32.683443+0000","last_unstale":"2024-03-06T07:31:34.868533+0000","last_undegraded":"2024-03-06T07:31:34.868533+0000","last_fullsized":"2024-03-06T07:31:34.868533+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:53:04.940938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.6","version":"82'1","reported_seq":13,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.656440+0000","last_change":"2024-03-06T07:31:31.643000+0000","last_active":"2024-03-06T07:31:33.656440+0000","last_peered":"2024-03-06T07:31:33.656440+0000","last_clean":"2024-03-06T07:31:33.656440+0000","last_became_active":"2024-03-06T07:31:31.642724+0000","last_became_peered":"2024-03-06T07:31:31.642724+0000","last_unstale":"2024-03-06T07:31:33.656440+0000","last_undegraded":"2024-03-06T07:31:33.656440+0000","last_fullsized":"2024-03-06T07:31:33.656440+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:27:04.600309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.4","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686217+0000","last_change":"2024-03-06T07:31:32.683434+0000","last_active":"2024-03-06T07:31:32.686217+0000","last_peered":"2024-03-06T07:31:32.686217+0000","last_clean":"2024-03-06T07:31:32.686217+0000","last_became_active":"2024-03-06T07:31:32.681959+0000","last_became_peered":"2024-03-06T07:31:32.681959+0000","last_unstale":"2024-03-06T07:31:32.686217+0000","last_undegraded":"2024-03-06T07:31:32.686217+0000","last_fullsized":"2024-03-06T07:31:32.686217+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:37:07.717126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.662612+0000","last_change":"2024-03-06T07:31:31.658223+0000","last_active":"2024-03-06T07:31:32.662612+0000","last_peered":"2024-03-06T07:31:32.662612+0000","last_clean":"2024-03-06T07:31:32.662612+0000","last_became_active":"2024-03-06T07:31:31.656082+0000","last_became_peered":"2024-03-06T07:31:31.656082+0000","last_unstale":"2024-03-06T07:31:32.662612+0000","last_undegraded":"2024-03-06T07:31:32.662612+0000","last_fullsized":"2024-03-06T07:31:32.662612+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:08:56.998640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.673748+0000","last_change":"2024-03-06T07:31:32.663935+0000","last_active":"2024-03-06T07:31:33.673748+0000","last_peered":"2024-03-06T07:31:33.673748+0000","last_clean":"2024-03-06T07:31:33.673748+0000","last_became_active":"2024-03-06T07:31:32.662929+0000","last_became_peered":"2024-03-06T07:31:32.662929+0000","last_unstale":"2024-03-06T07:31:33.673748+0000","last_undegraded":"2024-03-06T07:31:33.673748+0000","last_fullsized":"2024-03-06T07:31:33.673748+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:04:10.713397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.666602+0000","last_change":"2024-03-06T07:31:31.658449+0000","last_active":"2024-03-06T07:31:32.666602+0000","last_peered":"2024-03-06T07:31:32.666602+0000","last_clean":"2024-03-06T07:31:32.666602+0000","last_became_active":"2024-03-06T07:31:31.656269+0000","last_became_peered":"2024-03-06T07:31:31.656269+0000","last_unstale":"2024-03-06T07:31:32.666602+0000","last_undegraded":"2024-03-06T07:31:32.666602+0000","last_fullsized":"2024-03-06T07:31:32.666602+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:49:45.336355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.677448+0000","last_change":"2024-03-06T07:31:32.674786+0000","last_active":"2024-03-06T07:31:32.677448+0000","last_peered":"2024-03-06T07:31:32.677448+0000","last_clean":"2024-03-06T07:31:32.677448+0000","last_became_active":"2024-03-06T07:31:32.674501+0000","last_became_peered":"2024-03-06T07:31:32.674501+0000","last_unstale":"2024-03-06T07:31:32.677448+0000","last_undegraded":"2024-03-06T07:31:32.677448+0000","last_fullsized":"2024-03-06T07:31:32.677448+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:48:21.013778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669966+0000","last_change":"2024-03-06T07:31:31.658000+0000","last_active":"2024-03-06T07:31:33.669966+0000","last_peered":"2024-03-06T07:31:33.669966+0000","last_clean":"2024-03-06T07:31:33.669966+0000","last_became_active":"2024-03-06T07:31:31.657669+0000","last_became_peered":"2024-03-06T07:31:31.657669+0000","last_unstale":"2024-03-06T07:31:33.669966+0000","last_undegraded":"2024-03-06T07:31:33.669966+0000","last_fullsized":"2024-03-06T07:31:33.669966+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:53:13.572579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.3","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.669942+0000","last_change":"2024-03-06T07:31:32.684067+0000","last_active":"2024-03-06T07:31:33.669942+0000","last_peered":"2024-03-06T07:31:33.669942+0000","last_clean":"2024-03-06T07:31:33.669942+0000","last_became_active":"2024-03-06T07:31:32.683493+0000","last_became_peered":"2024-03-06T07:31:32.683493+0000","last_unstale":"2024-03-06T07:31:33.669942+0000","last_undegraded":"2024-03-06T07:31:33.669942+0000","last_fullsized":"2024-03-06T07:31:33.669942+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:48:41.873102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672527+0000","last_change":"2024-03-06T07:31:31.651943+0000","last_active":"2024-03-06T07:31:33.672527+0000","last_peered":"2024-03-06T07:31:33.672527+0000","last_clean":"2024-03-06T07:31:33.672527+0000","last_became_active":"2024-03-06T07:31:31.649489+0000","last_became_peered":"2024-03-06T07:31:31.649489+0000","last_unstale":"2024-03-06T07:31:33.672527+0000","last_undegraded":"2024-03-06T07:31:33.672527+0000","last_fullsized":"2024-03-06T07:31:33.672527+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:42:39.195412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.0","version":"0'0","reported_seq":12,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867710+0000","last_change":"2024-03-06T07:31:32.666310+0000","last_active":"2024-03-06T07:31:34.867710+0000","last_peered":"2024-03-06T07:31:34.867710+0000","last_clean":"2024-03-06T07:31:34.867710+0000","last_became_active":"2024-03-06T07:31:32.665981+0000","last_became_peered":"2024-03-06T07:31:32.665981+0000","last_unstale":"2024-03-06T07:31:34.867710+0000","last_undegraded":"2024-03-06T07:31:34.867710+0000","last_fullsized":"2024-03-06T07:31:34.867710+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:56:26.632423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":12,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.670354+0000","last_change":"2024-03-06T07:31:31.659996+0000","last_active":"2024-03-06T07:31:32.670354+0000","last_peered":"2024-03-06T07:31:32.670354+0000","last_clean":"2024-03-06T07:31:32.670354+0000","last_became_active":"2024-03-06T07:31:31.658002+0000","last_became_peered":"2024-03-06T07:31:31.658002+0000","last_unstale":"2024-03-06T07:31:32.670354+0000","last_undegraded":"2024-03-06T07:31:32.670354+0000","last_fullsized":"2024-03-06T07:31:32.670354+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:29:40.256587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.686198+0000","last_change":"2024-03-06T07:31:32.682870+0000","last_active":"2024-03-06T07:31:32.686198+0000","last_peered":"2024-03-06T07:31:32.686198+0000","last_clean":"2024-03-06T07:31:32.686198+0000","last_became_active":"2024-03-06T07:31:32.682432+0000","last_became_peered":"2024-03-06T07:31:32.682432+0000","last_unstale":"2024-03-06T07:31:32.686198+0000","last_undegraded":"2024-03-06T07:31:32.686198+0000","last_fullsized":"2024-03-06T07:31:32.686198+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:48:21.013778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":14,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:33.672187+0000","last_change":"2024-03-06T07:31:31.655903+0000","last_active":"2024-03-06T07:31:33.672187+0000","last_peered":"2024-03-06T07:31:33.672187+0000","last_clean":"2024-03-06T07:31:33.672187+0000","last_became_active":"2024-03-06T07:31:31.654356+0000","last_became_peered":"2024-03-06T07:31:31.654356+0000","last_unstale":"2024-03-06T07:31:33.672187+0000","last_undegraded":"2024-03-06T07:31:33.672187+0000","last_fullsized":"2024-03-06T07:31:33.672187+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:06:43.801817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.c","version":"0'0","reported_seq":9,"reported_epoch":82,"state":"active+clean","last_fresh":"2024-03-06T07:31:32.681715+0000","last_change":"2024-03-06T07:31:32.675848+0000","last_active":"2024-03-06T07:31:32.681715+0000","last_peered":"2024-03-06T07:31:32.681715+0000","last_clean":"2024-03-06T07:31:32.681715+0000","last_became_active":"2024-03-06T07:31:32.675571+0000","last_became_peered":"2024-03-06T07:31:32.675571+0000","last_unstale":"2024-03-06T07:31:32.681715+0000","last_undegraded":"2024-03-06T07:31:32.681715+0000","last_fullsized":"2024-03-06T07:31:32.681715+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:31.613702+0000","last_clean_scrub_stamp":"2024-03-06T07:31:31.613702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:47:56.937757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"82'1","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.868447+0000","last_change":"2024-03-06T07:31:31.662414+0000","last_active":"2024-03-06T07:31:34.868447+0000","last_peered":"2024-03-06T07:31:34.868447+0000","last_clean":"2024-03-06T07:31:34.868447+0000","last_became_active":"2024-03-06T07:31:31.662126+0000","last_became_peered":"2024-03-06T07:31:31.662126+0000","last_unstale":"2024-03-06T07:31:34.868447+0000","last_undegraded":"2024-03-06T07:31:34.868447+0000","last_fullsized":"2024-03-06T07:31:34.868447+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":80,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:31:30.598212+0000","last_clean_scrub_stamp":"2024-03-06T07:31:30.598212+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:53:55.203310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":283,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-03-06T07:31:34.867083+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:31:34.867083+0000","last_peered":"2024-03-06T07:31:34.867083+0000","last_clean":"2024-03-06T07:31:34.867083+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:31:34.867083+0000","last_undegraded":"2024-03-06T07:31:34.867083+0000","last_fullsized":"2024-03-06T07:31:34.867083+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542203,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32748,"kb_used_data":1564,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339092,"statfs":{"total":96636764160,"available":96603230208,"internally_reserved":0,"allocated":1601536,"data_stored":1014266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542202,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32180,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339660,"statfs":{"total":96636764160,"available":96603811840,"internally_reserved":0,"allocated":1015808,"data_stored":424522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542203,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32188,"kb_used_data":996,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339652,"statfs":{"total":96636764160,"available":96603803648,"internally_reserved":0,"allocated":1019904,"data_stored":425462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542202,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32180,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339660,"statfs":{"total":96636764160,"available":96603811840,"internally_reserved":0,"allocated":1015808,"data_stored":430128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542202,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32176,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339664,"statfs":{"total":96636764160,"available":96603815936,"internally_reserved":0,"allocated":1015808,"data_stored":425440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542203,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32184,"kb_used_data":996,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339656,"statfs":{"total":96636764160,"available":96603807744,"internally_reserved":0,"allocated":1019904,"data_stored":430710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542204,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32176,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339664,"statfs":{"total":96636764160,"available":96603815936,"internally_reserved":0,"allocated":1015808,"data_stored":424022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542201,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32764,"kb_used_data":1576,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339076,"statfs":{"total":96636764160,"available":96603213824,"internally_reserved":0,"allocated":1613824,"data_stored":1020530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:31:37.724 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:31:37.724 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:31:37.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.979+0000 7f38a6e00640 1 Processor -- start 2024-03-06T07:31:37.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.980+0000 7f38a6e00640 1 -- start start 2024-03-06T07:31:37.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a6e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f38a0161b50 0x7f38a00d05a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a6e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a00d4a00 0x7f38a00d0ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a6e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38a00b5490 0x7f38a00d1d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a6e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38a00cef40 con 0x7f38a00b5490 2024-03-06T07:31:37.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a6e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38a00cecd0 con 0x7f38a0161b50 2024-03-06T07:31:37.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a6e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38a00cfc80 con 0x7f38a00d4a00 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a4e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38a00b5490 0x7f38a00d1d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a4e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f38a00b5490 0x7f38a00d1d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53282/0 (socket says 172.21.15.86:53282) 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.981+0000 7f38a4e00640 1 -- 172.21.15.86:0/803402634 learned_addr learned my addr 172.21.15.86:0/803402634 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f38a4e00640 1 -- 172.21.15.86:0/803402634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a00d4a00 msgr2=0x7f38a00d0ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f3897400640 1 --2- 172.21.15.86:0/803402634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a00d4a00 0x7f38a00d0ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f389fe00640 1 --2- 172.21.15.86:0/803402634 >> v2:172.21.15.194:3300/0 conn(0x7f38a0161b50 0x7f38a00d05a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f38a4e00640 1 --2- 172.21.15.86:0/803402634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a00d4a00 0x7f38a00d0ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f38a4e00640 1 -- 172.21.15.86:0/803402634 >> v2:172.21.15.194:3300/0 conn(0x7f38a0161b50 msgr2=0x7f38a00d05a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f38a4e00640 1 --2- 172.21.15.86:0/803402634 >> v2:172.21.15.194:3300/0 conn(0x7f38a0161b50 0x7f38a00d05a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f38a4e00640 1 -- 172.21.15.86:0/803402634 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38a00c7b60 con 0x7f38a00b5490 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f3897400640 1 --2- 172.21.15.86:0/803402634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a00d4a00 0x7f38a00d0ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:37.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.982+0000 7f389fe00640 1 --2- 172.21.15.86:0/803402634 >> v2:172.21.15.194:3300/0 conn(0x7f38a0161b50 0x7f38a00d05a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:37.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.983+0000 7f38a4e00640 1 --2- 172.21.15.86:0/803402634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00d1d70 secure :-1 s=READY pgs=956 cs=0 l=1 rev1=1 crypto rx=0x7f3898009e50 tx=0x7f389802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cac520867ca39adc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:37.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.983+0000 7f389f400640 1 -- 172.21.15.86:0/803402634 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f389802ca80 con 0x7f38a00b5490 2024-03-06T07:31:37.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.983+0000 7f389f400640 1 -- 172.21.15.86:0/803402634 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f389802cc20 con 0x7f38a00b5490 2024-03-06T07:31:37.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.983+0000 7f389f400640 1 -- 172.21.15.86:0/803402634 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38980343f0 con 0x7f38a00b5490 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 -- 172.21.15.86:0/803402634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 msgr2=0x7f38a00d1d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 --2- 172.21.15.86:0/803402634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00d1d70 secure :-1 s=READY pgs=956 cs=0 l=1 rev1=1 crypto rx=0x7f3898009e50 tx=0x7f389802bee0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 -- 172.21.15.86:0/803402634 shutdown_connections 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 --2- 172.21.15.86:0/803402634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00d1d70 unknown :-1 s=CLOSED pgs=956 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 --2- 172.21.15.86:0/803402634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a00d4a00 0x7f38a00d0ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 --2- 172.21.15.86:0/803402634 >> v2:172.21.15.194:3300/0 conn(0x7f38a0161b50 0x7f38a00d05a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 -- 172.21.15.86:0/803402634 >> 172.21.15.86:0/803402634 conn(0x7f38a00d3e70 msgr2=0x7f38a00d4280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 -- 172.21.15.86:0/803402634 shutdown_connections 2024-03-06T07:31:37.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.984+0000 7f38a6e00640 1 -- 172.21.15.86:0/803402634 wait complete. 2024-03-06T07:31:37.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.986+0000 7f38a6e00640 1 Processor -- start 2024-03-06T07:31:37.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.986+0000 7f38a6e00640 1 -- start start 2024-03-06T07:31:37.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.987+0000 7f38a6e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00b7c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.987+0000 7f38a6e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38a00d4a00 0x7f38a00b8930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.987+0000 7f38a6e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a0161b50 0x7f38a00cd710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.987+0000 7f38a6e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38a00d1230 con 0x7f38a00b5490 2024-03-06T07:31:37.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.987+0000 7f38a6e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38a00d22b0 con 0x7f38a00d4a00 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.987+0000 7f38a6e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38a00b6b90 con 0x7f38a0161b50 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.987+0000 7f389fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00b7c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00b7c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53290/0 (socket says 172.21.15.86:53290) 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 -- 172.21.15.86:0/2978771941 learned_addr learned my addr 172.21.15.86:0/2978771941 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f3897400640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38a00d4a00 0x7f38a00b8930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 -- 172.21.15.86:0/2978771941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a0161b50 msgr2=0x7f38a00cd710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a0161b50 0x7f38a00cd710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 -- 172.21.15.86:0/2978771941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38a00d4a00 msgr2=0x7f38a00b8930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38a00d4a00 0x7f38a00b8930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 -- 172.21.15.86:0/2978771941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38a00b6e70 con 0x7f38a00b5490 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389fe00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00b7c30 secure :-1 s=READY pgs=957 cs=0 l=1 rev1=1 crypto rx=0x7f389000c7f0 tx=0x7f389000ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389cc00640 1 -- 172.21.15.86:0/2978771941 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3890015070 con 0x7f38a00b5490 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389cc00640 1 -- 172.21.15.86:0/2978771941 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3890007820 con 0x7f38a00b5490 2024-03-06T07:31:37.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.988+0000 7f389cc00640 1 -- 172.21.15.86:0/2978771941 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3890007bd0 con 0x7f38a00b5490 2024-03-06T07:31:37.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.989+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f38a00b7150 con 0x7f38a00b5490 2024-03-06T07:31:37.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.989+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f38a00b7710 con 0x7f38a00b5490 2024-03-06T07:31:37.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.990+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f38a004f330 con 0x7f38a00b5490 2024-03-06T07:31:37.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.991+0000 7f389cc00640 1 -- 172.21.15.86:0/2978771941 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3890018070 con 0x7f38a00b5490 2024-03-06T07:31:37.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.992+0000 7f389cc00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f387c06b4d0 0x7f387c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:37.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.992+0000 7f389cc00640 1 -- 172.21.15.86:0/2978771941 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f389008a140 con 0x7f38a00b5490 2024-03-06T07:31:37.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.992+0000 7f3897400640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f387c06b4d0 0x7f387c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:37.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.993+0000 7f3897400640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f387c06b4d0 0x7f387c06d990 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f3888006fd0 tx=0x7f3888008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:37.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:37.997+0000 7f389cc00640 1 -- 172.21.15.86:0/2978771941 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3890058fc0 con 0x7f38a00b5490 2024-03-06T07:31:38.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.279+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f38a00d16d0 con 0x7f38a00b5490 2024-03-06T07:31:38.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.282+0000 7f389cc00640 1 -- 172.21.15.86:0/2978771941 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v17) v1 ==== 190+0+147 (secure 0 0 0) 0x7f38900588c0 con 0x7f38a00b5490 2024-03-06T07:31:38.284 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:31:38.284 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:31:38.284 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:31:38.284 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:31:38.284 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:31:38.285 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f387c06b4d0 msgr2=0x7f387c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f387c06b4d0 0x7f387c06d990 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f3888006fd0 tx=0x7f3888008040 comp rx=0 tx=0).stop 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 msgr2=0x7f38a00b7c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00b7c30 secure :-1 s=READY pgs=957 cs=0 l=1 rev1=1 crypto rx=0x7f389000c7f0 tx=0x7f389000ccb0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 shutdown_connections 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f387c06b4d0 0x7f387c06d990 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f38a0161b50 0x7f38a00cd710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f38a00d4a00 0x7f38a00b8930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 --2- 172.21.15.86:0/2978771941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f38a00b5490 0x7f38a00b7c30 unknown :-1 s=CLOSED pgs=957 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 >> 172.21.15.86:0/2978771941 conn(0x7f38a00d3e70 msgr2=0x7f38a00d31e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 shutdown_connections 2024-03-06T07:31:38.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.285+0000 7f38a6e00640 1 -- 172.21.15.86:0/2978771941 wait complete. 2024-03-06T07:31:38.304 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:38.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.493+0000 7f1962800640 1 Processor -- start 2024-03-06T07:31:38.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.493+0000 7f1962800640 1 -- start start 2024-03-06T07:31:38.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f1962800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f195c0cf430 0x7f195c070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f1962800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c06d260 0x7f195c0b2d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f1962800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f195c0717c0 con 0x7f195c071260 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f1962800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f195c071aa0 con 0x7f195c0cf430 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f1962800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f195c071d80 con 0x7f195c06d260 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f195b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f195c0cf430 0x7f195c070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f195b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f195c0cf430 0x7f195c070d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53304/0 (socket says 172.21.15.86:53304) 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.495+0000 7f195b400640 1 -- 172.21.15.86:0/3780218995 learned_addr learned my addr 172.21.15.86:0/3780218995 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.496+0000 7f195be00640 1 --2- 172.21.15.86:0/3780218995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c06d260 0x7f195c0b2d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.496+0000 7f195b400640 1 -- 172.21.15.86:0/3780218995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c06d260 msgr2=0x7f195c0b2d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.496+0000 7f195b400640 1 --2- 172.21.15.86:0/3780218995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c06d260 0x7f195c0b2d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.496+0000 7f195b400640 1 -- 172.21.15.86:0/3780218995 >> v1:172.21.15.194:6789/0 conn(0x7f195c071260 legacy=0x7f195c06cb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.496+0000 7f195b400640 1 -- 172.21.15.86:0/3780218995 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f195c072060 con 0x7f195c0cf430 2024-03-06T07:31:38.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.496+0000 7f195be00640 1 --2- 172.21.15.86:0/3780218995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c06d260 0x7f195c0b2d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:38.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.497+0000 7f195b400640 1 --2- 172.21.15.86:0/3780218995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c0cf430 0x7f195c070d20 secure :-1 s=READY pgs=958 cs=0 l=1 rev1=1 crypto rx=0x7f194c009870 tx=0x7f194c02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4879a2ce3179ca7c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:38.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.497+0000 7f195aa00640 1 -- 172.21.15.86:0/3780218995 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f194c035020 con 0x7f195c0cf430 2024-03-06T07:31:38.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.497+0000 7f195aa00640 1 -- 172.21.15.86:0/3780218995 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f194c031640 con 0x7f195c0cf430 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.497+0000 7f195aa00640 1 -- 172.21.15.86:0/3780218995 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f194c02d040 con 0x7f195c0cf430 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 -- 172.21.15.86:0/3780218995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c0cf430 msgr2=0x7f195c070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 --2- 172.21.15.86:0/3780218995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c0cf430 0x7f195c070d20 secure :-1 s=READY pgs=958 cs=0 l=1 rev1=1 crypto rx=0x7f194c009870 tx=0x7f194c02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 -- 172.21.15.86:0/3780218995 shutdown_connections 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 --2- 172.21.15.86:0/3780218995 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c06d260 0x7f195c0b2d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 --2- 172.21.15.86:0/3780218995 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c0cf430 0x7f195c070d20 unknown :-1 s=CLOSED pgs=958 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 -- 172.21.15.86:0/3780218995 >> 172.21.15.86:0/3780218995 conn(0x7f195c0b1730 msgr2=0x7f195c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 -- 172.21.15.86:0/3780218995 shutdown_connections 2024-03-06T07:31:38.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.498+0000 7f1962800640 1 -- 172.21.15.86:0/3780218995 wait complete. 2024-03-06T07:31:38.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.500+0000 7f1962800640 1 Processor -- start 2024-03-06T07:31:38.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.500+0000 7f1962800640 1 -- start start 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1962800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c06d260 0x7f195c1f25e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1962800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f195c071260 0x7f195c1f8e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1962800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c0cf430 0x7f195c165050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1962800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f195c162cf0 con 0x7f195c06d260 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1962800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f195c162b70 con 0x7f195c071260 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1962800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f195c0b3250 con 0x7f195c0cf430 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c06d260 0x7f195c1f25e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:38.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c06d260 0x7f195c1f25e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53320/0 (socket says 172.21.15.86:53320) 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 -- 172.21.15.86:0/753079290 learned_addr learned my addr 172.21.15.86:0/753079290 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195be00640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c0cf430 0x7f195c165050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1953e00640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f195c071260 0x7f195c1f8e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 -- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c0cf430 msgr2=0x7f195c165050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c0cf430 0x7f195c165050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 -- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f195c071260 msgr2=0x7f195c1f8e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f195c071260 0x7f195c1f8e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f195b400640 1 -- 172.21.15.86:0/753079290 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f195c1468b0 con 0x7f195c06d260 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.501+0000 7f1953e00640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f195c071260 0x7f195c1f8e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:38.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.502+0000 7f195b400640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c06d260 0x7f195c1f25e0 secure :-1 s=READY pgs=959 cs=0 l=1 rev1=1 crypto rx=0x7f194c02b930 tx=0x7f194c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:38.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.502+0000 7f1953400640 1 -- 172.21.15.86:0/753079290 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f194c03ac60 con 0x7f195c06d260 2024-03-06T07:31:38.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.502+0000 7f1953400640 1 -- 172.21.15.86:0/753079290 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f194c03ae20 con 0x7f195c06d260 2024-03-06T07:31:38.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.502+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f195c146b90 con 0x7f195c06d260 2024-03-06T07:31:38.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.502+0000 7f1953400640 1 -- 172.21.15.86:0/753079290 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f194c044630 con 0x7f195c06d260 2024-03-06T07:31:38.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.502+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f195c147150 con 0x7f195c06d260 2024-03-06T07:31:38.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.503+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f195c04f330 con 0x7f195c06d260 2024-03-06T07:31:38.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.505+0000 7f1953400640 1 -- 172.21.15.86:0/753079290 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f194c04b060 con 0x7f195c06d260 2024-03-06T07:31:38.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.507+0000 7f1953400640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f193806b4d0 0x7f193806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:38.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.507+0000 7f1953400640 1 -- 172.21.15.86:0/753079290 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f194c0ac010 con 0x7f195c06d260 2024-03-06T07:31:38.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.507+0000 7f1953e00640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f193806b4d0 0x7f193806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:38.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.508+0000 7f1953e00640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f193806b4d0 0x7f193806d990 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f195c1578b0 tx=0x7f1944009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:38.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.511+0000 7f1953400640 1 -- 172.21.15.86:0/753079290 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f194c03f070 con 0x7f195c06d260 2024-03-06T07:31:38.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.747+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f195c0b4a40 con 0x7f195c06d260 2024-03-06T07:31:38.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.748+0000 7f1953400640 1 -- 172.21.15.86:0/753079290 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f194c02f300 con 0x7f195c06d260 2024-03-06T07:31:38.750 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:38.750 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:31:32.095887+0000","modified":"2024-03-06T07:31:35.343932+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6609},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6609":{"gid":6609,"name":"c","rank":0,"incarnation":47,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/255958031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":255958031},{"type":"v1","addr":"172.21.15.86:6837","nonce":255958031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6609,"qdb_cluster":[6609]},"id":6}]} 2024-03-06T07:31:38.750 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 50 2024-03-06T07:31:38.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.750+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f193806b4d0 msgr2=0x7f193806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:38.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.750+0000 7f1962800640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f193806b4d0 0x7f193806d990 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f195c1578b0 tx=0x7f1944009040 comp rx=0 tx=0).stop 2024-03-06T07:31:38.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.750+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c06d260 msgr2=0x7f195c1f25e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:38.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.750+0000 7f1962800640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c06d260 0x7f195c1f25e0 secure :-1 s=READY pgs=959 cs=0 l=1 rev1=1 crypto rx=0x7f194c02b930 tx=0x7f194c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 shutdown_connections 2024-03-06T07:31:38.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f193806b4d0 0x7f193806d990 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f195c0cf430 0x7f195c165050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f195c071260 0x7f195c1f8e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 --2- 172.21.15.86:0/753079290 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f195c06d260 0x7f195c1f25e0 unknown :-1 s=CLOSED pgs=959 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:38.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 >> 172.21.15.86:0/753079290 conn(0x7f195c0b1730 msgr2=0x7f195c165e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:38.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 shutdown_connections 2024-03-06T07:31:38.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:38.751+0000 7f1962800640 1 -- 172.21.15.86:0/753079290 wait complete. 2024-03-06T07:31:39.172 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 50, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:31:32.095887+0000', 'modified': '2024-03-06T07:31:35.343932+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 6609}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6609': {'gid': 6609, 'name': 'c', 'rank': 0, 'incarnation': 47, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/255958031', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 255958031}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 255958031}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [13], 'metadata_pool': 12, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 6609, 'qdb_cluster': [6609]} 2024-03-06T07:31:39.172 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:31:39.172 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-06T07:31:39.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.382+0000 7fa919200640 1 Processor -- start 2024-03-06T07:31:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.382+0000 7fa919200640 1 -- start start 2024-03-06T07:31:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.383+0000 7fa919200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa9140c5420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.383+0000 7fa919200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa9140c5960 0x7fa914165b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.383+0000 7fa919200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa914145b30 con 0x7fa9140c4930 2024-03-06T07:31:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.383+0000 7fa919200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa9140c68c0 con 0x7fa9140c5960 2024-03-06T07:31:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.383+0000 7fa919200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa914142ce0 con 0x7fa9140c6460 2024-03-06T07:31:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.384+0000 7fa90be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa9140c5420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.384+0000 7fa912800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa9140c5960 0x7fa914165b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.384+0000 7fa90be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa9140c5420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:48060/0 (socket says 172.21.15.86:48060) 2024-03-06T07:31:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.384+0000 7fa90be00640 1 -- 172.21.15.86:0/275811383 learned_addr learned my addr 172.21.15.86:0/275811383 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.385+0000 7fa90be00640 1 -- 172.21.15.86:0/275811383 >> v2:172.21.15.194:3300/0 conn(0x7fa9140c5960 msgr2=0x7fa914165b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.385+0000 7fa90be00640 1 --2- 172.21.15.86:0/275811383 >> v2:172.21.15.194:3300/0 conn(0x7fa9140c5960 0x7fa914165b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.385+0000 7fa90be00640 1 -- 172.21.15.86:0/275811383 >> v1:172.21.15.194:6789/0 conn(0x7fa9140c4930 legacy=0x7fa9140c4d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.385+0000 7fa90be00640 1 -- 172.21.15.86:0/275811383 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9140cbca0 con 0x7fa9140c6460 2024-03-06T07:31:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.386+0000 7fa90be00640 1 --2- 172.21.15.86:0/275811383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa9140c5420 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fa8fc009870 tx=0x7fa8fc02b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f070b644addd8611 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.387+0000 7fa911400640 1 -- 172.21.15.86:0/275811383 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8fc035020 con 0x7fa9140c6460 2024-03-06T07:31:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.387+0000 7fa911400640 1 -- 172.21.15.86:0/275811383 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8fc031640 con 0x7fa9140c6460 2024-03-06T07:31:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.387+0000 7fa911400640 1 -- 172.21.15.86:0/275811383 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8fc02d040 con 0x7fa9140c6460 2024-03-06T07:31:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.387+0000 7fa919200640 1 -- 172.21.15.86:0/275811383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 msgr2=0x7fa9140c5420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.387+0000 7fa919200640 1 --2- 172.21.15.86:0/275811383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa9140c5420 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fa8fc009870 tx=0x7fa8fc02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.388+0000 7fa919200640 1 -- 172.21.15.86:0/275811383 shutdown_connections 2024-03-06T07:31:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.388+0000 7fa919200640 1 --2- 172.21.15.86:0/275811383 >> v2:172.21.15.194:3300/0 conn(0x7fa9140c5960 0x7fa914165b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.388+0000 7fa919200640 1 --2- 172.21.15.86:0/275811383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa9140c5420 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.388+0000 7fa919200640 1 -- 172.21.15.86:0/275811383 >> 172.21.15.86:0/275811383 conn(0x7fa9140cc4f0 msgr2=0x7fa9140cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.388+0000 7fa919200640 1 -- 172.21.15.86:0/275811383 shutdown_connections 2024-03-06T07:31:39.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.388+0000 7fa919200640 1 -- 172.21.15.86:0/275811383 wait complete. 2024-03-06T07:31:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.389+0000 7fa919200640 1 Processor -- start 2024-03-06T07:31:39.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.389+0000 7fa919200640 1 -- start start 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.390+0000 7fa919200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9140c4930 0x7fa9140aecc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.390+0000 7fa919200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9140c5960 0x7fa91413bb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.390+0000 7fa919200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa91415db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.390+0000 7fa919200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9140ca3b0 con 0x7fa9140c4930 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.390+0000 7fa919200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9140ca0d0 con 0x7fa9140c5960 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.390+0000 7fa919200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa91406e600 con 0x7fa9140c6460 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9140c4930 0x7fa9140aecc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9140c4930 0x7fa9140aecc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53330/0 (socket says 172.21.15.86:53330) 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 -- 172.21.15.86:0/3525664883 learned_addr learned my addr 172.21.15.86:0/3525664883 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa90be00640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9140c5960 0x7fa91413bb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa912800640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa91415db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 -- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 msgr2=0x7fa91415db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa91415db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 -- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9140c5960 msgr2=0x7fa91413bb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9140c5960 0x7fa91413bb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa911e00640 1 -- 172.21.15.86:0/3525664883 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa91413c0b0 con 0x7fa9140c4930 2024-03-06T07:31:39.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.391+0000 7fa912800640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa91415db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:39.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.392+0000 7fa911e00640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9140c4930 0x7fa9140aecc0 secure :-1 s=READY pgs=960 cs=0 l=1 rev1=1 crypto rx=0x7fa90400c3c0 tx=0x7fa90400c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.392+0000 7fa90a000640 1 -- 172.21.15.86:0/3525664883 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9040052d0 con 0x7fa9140c4930 2024-03-06T07:31:39.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.392+0000 7fa90a000640 1 -- 172.21.15.86:0/3525664883 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa904005490 con 0x7fa9140c4930 2024-03-06T07:31:39.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.392+0000 7fa90a000640 1 -- 172.21.15.86:0/3525664883 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa904007260 con 0x7fa9140c4930 2024-03-06T07:31:39.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.392+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa91415e5b0 con 0x7fa9140c4930 2024-03-06T07:31:39.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.392+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9140ae1e0 con 0x7fa9140c4930 2024-03-06T07:31:39.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.394+0000 7fa90a000640 1 -- 172.21.15.86:0/3525664883 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa904007400 con 0x7fa9140c4930 2024-03-06T07:31:39.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.396+0000 7fa91be3a740 1 Processor -- start 2024-03-06T07:31:39.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.396+0000 7fa90a000640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f006b4d0 0x7fa8f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.396+0000 7fa91be3a740 1 -- start start 2024-03-06T07:31:39.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.396+0000 7fa90a000640 1 -- 172.21.15.86:0/3525664883 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa9040896f0 con 0x7fa9140c4930 2024-03-06T07:31:39.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.396+0000 7fa912800640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f006b4d0 0x7fa8f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159c7b0 0x55c11156f750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159cb90 0x55c11159f540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa912800640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f006b4d0 0x7fa8f006d990 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7fa90c007560 tx=0x7fa90c00b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11156fc90 0x55c11159fa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa91be3a740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c1114d9600 con 0x55c11159cb90 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c111482930 con 0x55c11156fc90 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55c1114d56a0 con 0x55c11159c7b0 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa90be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159cb90 0x55c11159f540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa90be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159cb90 0x55c11159f540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53332/0 (socket says 172.21.15.86:53332) 2024-03-06T07:31:39.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.397+0000 7fa90be00640 1 -- 172.21.15.86:0/1230899661 learned_addr learned my addr 172.21.15.86:0/1230899661 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa911e00640 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159c7b0 0x55c11156f750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa912800640 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11156fc90 0x55c11159fa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa90be00640 1 -- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159c7b0 msgr2=0x55c11156f750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa90be00640 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159c7b0 0x55c11156f750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa90be00640 1 -- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11156fc90 msgr2=0x55c11159fa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa90be00640 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11156fc90 0x55c11159fa80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa90be00640 1 -- 172.21.15.86:0/1230899661 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c1111ee2d0 con 0x55c11159cb90 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa911e00640 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159c7b0 0x55c11156f750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.398+0000 7fa90be00640 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159cb90 0x55c11159f540 secure :-1 s=READY pgs=961 cs=0 l=1 rev1=1 crypto rx=0x7fa8fc000c00 tx=0x7fa8fc008030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2e4de3333e00e9d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa902a00640 1 -- 172.21.15.86:0/1230899661 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8fc035040 con 0x55c11159cb90 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa902a00640 1 -- 172.21.15.86:0/1230899661 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8fc0317d0 con 0x55c11159cb90 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa902a00640 1 -- 172.21.15.86:0/1230899661 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8fc031b80 con 0x55c11159cb90 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 -- 172.21.15.86:0/1230899661 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159cb90 msgr2=0x55c11159f540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159cb90 0x55c11159f540 secure :-1 s=READY pgs=961 cs=0 l=1 rev1=1 crypto rx=0x7fa8fc000c00 tx=0x7fa8fc008030 comp rx=0 tx=0).stop 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 -- 172.21.15.86:0/1230899661 shutdown_connections 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11156fc90 0x55c11159fa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159cb90 0x55c11159f540 unknown :-1 s=CLOSED pgs=961 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 --2- 172.21.15.86:0/1230899661 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159c7b0 0x55c11156f750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 -- 172.21.15.86:0/1230899661 >> 172.21.15.86:0/1230899661 conn(0x55c1115719c0 msgr2=0x55c1115ba540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 -- 172.21.15.86:0/1230899661 shutdown_connections 2024-03-06T07:31:39.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.399+0000 7fa91be3a740 1 -- 172.21.15.86:0/1230899661 wait complete. 2024-03-06T07:31:39.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.401+0000 7fa91be3a740 1 -- start start 2024-03-06T07:31:39.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.401+0000 7fa91be3a740 1 Processor -- start 2024-03-06T07:31:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c1115a2eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c111585ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 0x55c11159e9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa91be3a740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c111573f60 con 0x55c11159c7b0 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c111495d90 con 0x55c11159cb90 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55c111573350 con 0x55c11159e570 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa91be3a740 10 client.0 _ll_register_callbacks cb 0x55c1114d1350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c1115a2eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c1115a2eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53342/0 (socket says 172.21.15.86:53342) 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 -- 172.21.15.86:0/871943306 learned_addr learned my addr 172.21.15.86:0/871943306 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa90be00640 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c111585ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 -- 172.21.15.86:0/871943306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 msgr2=0x55c11159e9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 0x55c11159e9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 -- 172.21.15.86:0/871943306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 msgr2=0x55c111585ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c111585ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.403+0000 7fa911e00640 1 -- 172.21.15.86:0/871943306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c1112b86b0 con 0x55c11159c7b0 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa911e00640 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c1115a2eb0 secure :-1 s=READY pgs=962 cs=0 l=1 rev1=1 crypto rx=0x7fa90400cd80 tx=0x7fa904057f20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52ea6b75ecc1cdc2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa8f7400640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa8f7400640 1 -- 172.21.15.86:0/871943306 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa90405e070 con 0x55c11159c7b0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa8f7400640 1 -- 172.21.15.86:0/871943306 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa90405e210 con 0x55c11159c7b0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa8fc0317d0 con 0x55c11159c7b0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa8f7400640 1 -- 172.21.15.86:0/871943306 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=50) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa9040572b0 con 0x55c11159c7b0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 10 client.6750 fetch_fsmap learned FSMap version 50 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=50,monmap=2+}) v3 -- 0x55c1115842e0 con 0x55c11159c7b0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa8f7400640 1 -- 172.21.15.86:0/871943306 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8fc0317d0 con 0x55c11159c7b0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa8f7400640 1 -- 172.21.15.86:0/871943306 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 50) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fa904057450 con 0x55c11159c7b0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 10 client.6750 fetch_fsmap finished waiting for FSMap version 50 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 10 client.6750 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 20 client.6750 populate_metadata read hostname 'smithi086' 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c111598030 0x55c1115851a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 4 client.6750 mds_command: new command op to 6609 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.404+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 --> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c1114e6b40 con 0x55c111598030 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.405+0000 7fa90be00640 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c111598030 0x55c1115851a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.405+0000 7fa90be00640 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c111598030 0x55c1115851a0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8f231025516839b5 server_cookie=ceb41745e7faf0c0 in_seq=0 out_seq=0 2024-03-06T07:31:39.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.405+0000 7fa8f7400640 10 client.6750 ms_handle_connect on v2:172.21.15.86:6835/255958031 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa8f7400640 1 -- 172.21.15.86:0/871943306 <== mds.0 v2:172.21.15.86:6835/255958031 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x55c1114e6b40 con 0x55c111598030 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa8f7400640 10 client.6750 handle_command_reply: tid=0 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 client.6750 shutdown 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 msgr2=0x55c1115a2eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c1115a2eb0 secure :-1 s=READY pgs=962 cs=0 l=1 rev1=1 crypto rx=0x7fa90400cd80 tx=0x7fa904057f20 comp rx=0 tx=0).stop 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 shutdown_connections 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c111598030 0x55c1115851a0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 0x55c11159e9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c111585ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 --2- 172.21.15.86:0/871943306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c1115a2eb0 unknown :-1 s=CLOSED pgs=962 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa8f7400640 0 client.6750 ms_handle_reset on v2:172.21.15.86:6835/255958031 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.406+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 >> 172.21.15.86:0/871943306 conn(0x55c11156fc90 msgr2=0x55c11156eeb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.407+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 shutdown_connections 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.407+0000 7fa91be3a740 1 -- 172.21.15.86:0/871943306 wait complete. 2024-03-06T07:31:39.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.407+0000 7fa91be3a740 20 client.6750 trim_cache size 0 max 16384 2024-03-06T07:31:39.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.425+0000 7fa91be3a740 1 Processor -- start 2024-03-06T07:31:39.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.425+0000 7fa91be3a740 1 -- start start 2024-03-06T07:31:39.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111596b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159cb90 0x55c1115cd080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159e570 0x55c1115bdcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa91be3a740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9140ce190 con 0x55c11159c7b0 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9140cd590 con 0x55c11159e570 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9140cdb90 con 0x55c11159cb90 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111596b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111596b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53344/0 (socket says 172.21.15.86:53344) 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa911e00640 1 -- 172.21.15.86:0/4102396272 learned_addr learned my addr 172.21.15.86:0/4102396272 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa912800640 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159e570 0x55c1115bdcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa90be00640 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159cb90 0x55c1115cd080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa911e00640 1 -- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159cb90 msgr2=0x55c1115cd080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.426+0000 7fa911e00640 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159cb90 0x55c1115cd080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.427+0000 7fa911e00640 1 -- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159e570 msgr2=0x55c1115bdcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.427+0000 7fa911e00640 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159e570 0x55c1115bdcf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.427+0000 7fa911e00640 1 -- 172.21.15.86:0/4102396272 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c1114ed1a0 con 0x55c11159c7b0 2024-03-06T07:31:39.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.427+0000 7fa911e00640 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111596b50 secure :-1 s=READY pgs=963 cs=0 l=1 rev1=1 crypto rx=0x7fa904057d80 tx=0x7fa90405e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d9fddf75ecc3015 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.427+0000 7fa902000640 1 -- 172.21.15.86:0/4102396272 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa904076030 con 0x55c11159c7b0 2024-03-06T07:31:39.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.427+0000 7fa902000640 1 -- 172.21.15.86:0/4102396272 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa90405e070 con 0x55c11159c7b0 2024-03-06T07:31:39.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.427+0000 7fa902000640 1 -- 172.21.15.86:0/4102396272 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa904053bb0 con 0x55c11159c7b0 2024-03-06T07:31:39.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 -- 172.21.15.86:0/4102396272 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 msgr2=0x55c111596b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111596b50 secure :-1 s=READY pgs=963 cs=0 l=1 rev1=1 crypto rx=0x7fa904057d80 tx=0x7fa90405e000 comp rx=0 tx=0).stop 2024-03-06T07:31:39.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 -- 172.21.15.86:0/4102396272 shutdown_connections 2024-03-06T07:31:39.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159e570 0x55c1115bdcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159cb90 0x55c1115cd080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 --2- 172.21.15.86:0/4102396272 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111596b50 unknown :-1 s=CLOSED pgs=963 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 -- 172.21.15.86:0/4102396272 >> 172.21.15.86:0/4102396272 conn(0x55c11156fc90 msgr2=0x55c1115bce40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 -- 172.21.15.86:0/4102396272 shutdown_connections 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 -- 172.21.15.86:0/4102396272 wait complete. 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.428+0000 7fa91be3a740 1 -- start start 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.429+0000 7fa91be3a740 1 Processor -- start 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.429+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111623eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.429+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c1115cccc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa91be3a740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 0x55c1115833b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa91be3a740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c111597eb0 con 0x55c11159c7b0 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c1115845c0 con 0x55c11159cb90 2024-03-06T07:31:39.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa91be3a740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55c111598480 con 0x55c11159e570 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa91be3a740 10 client.0 _ll_register_callbacks cb 0x7fa904057c30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111623eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111623eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53352/0 (socket says 172.21.15.86:53352) 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 -- 172.21.15.86:0/3408112210 learned_addr learned my addr 172.21.15.86:0/3408112210 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 -- 172.21.15.86:0/3408112210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 msgr2=0x55c1115833b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 0x55c1115833b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa90be00640 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c1115cccc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 -- 172.21.15.86:0/3408112210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 msgr2=0x55c1115cccc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c1115cccc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.430+0000 7fa911e00640 1 -- 172.21.15.86:0/3408112210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c111629dc0 con 0x55c11159c7b0 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa911e00640 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111623eb0 secure :-1 s=READY pgs=964 cs=0 l=1 rev1=1 crypto rx=0x7fa904087c10 tx=0x7fa904087ae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=76605f8288bd24e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa8f7400640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa8f7400640 1 -- 172.21.15.86:0/3408112210 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa90407a070 con 0x55c11159c7b0 2024-03-06T07:31:39.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa8f7400640 1 -- 172.21.15.86:0/3408112210 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa904056d20 con 0x55c11159c7b0 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa90405e070 con 0x55c11159c7b0 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa8f7400640 1 -- 172.21.15.86:0/3408112210 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=50) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa904076040 con 0x55c11159c7b0 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 10 client.6756 fetch_fsmap learned FSMap version 50 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=50,monmap=2+}) v3 -- 0x55c1115b9fe0 con 0x55c11159c7b0 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa8f7400640 1 -- 172.21.15.86:0/3408112210 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa90405e070 con 0x55c11159c7b0 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa8f7400640 1 -- 172.21.15.86:0/3408112210 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 50) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fa90407d030 con 0x55c11159c7b0 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 10 client.6756 fetch_fsmap finished waiting for FSMap version 50 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 10 client.6756 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 20 client.6756 populate_metadata read hostname 'smithi086' 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c1115c8fd0 0x55c1115c93b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 4 client.6756 mds_command: new command op to 6609 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.431+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 --> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55c1114e0e90 con 0x55c1115c8fd0 2024-03-06T07:31:39.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.432+0000 7fa90be00640 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c1115c8fd0 0x55c1115c93b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.432+0000 7fa90be00640 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c1115c8fd0 0x55c1115c93b0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7d0af047be201552 server_cookie=d91ad09f7a855bca in_seq=0 out_seq=0 2024-03-06T07:31:39.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.432+0000 7fa8f7400640 10 client.6756 ms_handle_connect on v2:172.21.15.86:6835/255958031 2024-03-06T07:31:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.432+0000 7fa8f7400640 1 -- 172.21.15.86:0/3408112210 <== mds.0 v2:172.21.15.86:6835/255958031 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x55c1114e0e90 con 0x55c1115c8fd0 2024-03-06T07:31:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.432+0000 7fa8f7400640 10 client.6756 handle_command_reply: tid=0 2024-03-06T07:31:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.432+0000 7fa91be3a740 1 client.6756 shutdown 2024-03-06T07:31:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 msgr2=0x55c111623eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111623eb0 secure :-1 s=READY pgs=964 cs=0 l=1 rev1=1 crypto rx=0x7fa904087c10 tx=0x7fa904087ae0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 shutdown_connections 2024-03-06T07:31:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:6835/255958031,v1:172.21.15.86:6837/255958031] conn(0x55c1115c8fd0 0x55c1115c93b0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55c11159e570 0x55c1115833b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55c11159cb90 0x55c1115cccc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 --2- 172.21.15.86:0/3408112210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55c11159c7b0 0x55c111623eb0 unknown :-1 s=CLOSED pgs=964 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa8f7400640 0 client.6756 ms_handle_reset on v2:172.21.15.86:6835/255958031 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 >> 172.21.15.86:0/3408112210 conn(0x55c11156fc90 msgr2=0x55c1115bdf30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 shutdown_connections 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 1 -- 172.21.15.86:0/3408112210 wait complete. 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.433+0000 7fa91be3a740 20 client.6756 trim_cache size 0 max 16384 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:31:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 6609, 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 6.7896736369999999, 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 50, 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 83, 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 83, 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 12.214151577999999 2024-03-06T07:31:39.440 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:31:39.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f006b4d0 msgr2=0x7fa8f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f006b4d0 0x7fa8f006d990 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7fa90c007560 tx=0x7fa90c00b040 comp rx=0 tx=0).stop 2024-03-06T07:31:39.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9140c4930 msgr2=0x7fa9140aecc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9140c4930 0x7fa9140aecc0 secure :-1 s=READY pgs=960 cs=0 l=1 rev1=1 crypto rx=0x7fa90400c3c0 tx=0x7fa90400c880 comp rx=0 tx=0).stop 2024-03-06T07:31:39.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 shutdown_connections 2024-03-06T07:31:39.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa8f006b4d0 0x7fa8f006d990 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9140c6460 0x7fa91415db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9140c5960 0x7fa91413bb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 --2- 172.21.15.86:0/3525664883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9140c4930 0x7fa9140aecc0 unknown :-1 s=CLOSED pgs=960 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 >> 172.21.15.86:0/3525664883 conn(0x7fa9140cc4f0 msgr2=0x7fa914142e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 shutdown_connections 2024-03-06T07:31:39.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.434+0000 7fa919200640 1 -- 172.21.15.86:0/3525664883 wait complete. 2024-03-06T07:31:39.451 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:31:39.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.646+0000 7f1b2cc00640 1 Processor -- start 2024-03-06T07:31:39.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.647+0000 7f1b2cc00640 1 -- start start 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b2cc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b2cc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1b280cf3e0 0x7f1b280724b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b2cc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b280cabf0 con 0x7f1b280c9ec0 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b2cc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1b280ce900 con 0x7f1b280cf3e0 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b2cc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1b280cebe0 con 0x7f1b28144660 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b25e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b25400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1b280cf3e0 0x7f1b280724b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b25e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28071f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:48126/0 (socket says 172.21.15.86:48126) 2024-03-06T07:31:39.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.649+0000 7f1b25e00640 1 -- 172.21.15.86:0/2942704421 learned_addr learned my addr 172.21.15.86:0/2942704421 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.650+0000 7f1b25e00640 1 -- 172.21.15.86:0/2942704421 >> v2:172.21.15.194:3300/0 conn(0x7f1b280cf3e0 msgr2=0x7f1b280724b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:39.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.650+0000 7f1b25e00640 1 --2- 172.21.15.86:0/2942704421 >> v2:172.21.15.194:3300/0 conn(0x7f1b280cf3e0 0x7f1b280724b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.650+0000 7f1b25e00640 1 -- 172.21.15.86:0/2942704421 >> v1:172.21.15.194:6789/0 conn(0x7f1b280c9ec0 legacy=0x7f1b280ca310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.650+0000 7f1b25e00640 1 -- 172.21.15.86:0/2942704421 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b280ceec0 con 0x7f1b28144660 2024-03-06T07:31:39.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.652+0000 7f1b25e00640 1 --2- 172.21.15.86:0/2942704421 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28071f70 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f1b18009870 tx=0x7f1b1802b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=324fd65865c3f5fd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.652+0000 7f1b24a00640 1 -- 172.21.15.86:0/2942704421 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b18034070 con 0x7f1b28144660 2024-03-06T07:31:39.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.652+0000 7f1b24a00640 1 -- 172.21.15.86:0/2942704421 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b18030840 con 0x7f1b28144660 2024-03-06T07:31:39.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.652+0000 7f1b24a00640 1 -- 172.21.15.86:0/2942704421 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b18030c10 con 0x7f1b28144660 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 -- 172.21.15.86:0/2942704421 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 msgr2=0x7f1b28071f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/2942704421 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28071f70 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f1b18009870 tx=0x7f1b1802b980 comp rx=0 tx=0).stop 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 -- 172.21.15.86:0/2942704421 shutdown_connections 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/2942704421 >> v2:172.21.15.194:3300/0 conn(0x7f1b280cf3e0 0x7f1b280724b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/2942704421 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28071f70 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 -- 172.21.15.86:0/2942704421 >> 172.21.15.86:0/2942704421 conn(0x7f1b280d0b90 msgr2=0x7f1b280d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 -- 172.21.15.86:0/2942704421 shutdown_connections 2024-03-06T07:31:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.653+0000 7f1b2cc00640 1 -- 172.21.15.86:0/2942704421 wait complete. 2024-03-06T07:31:39.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.655+0000 7f1b2cc00640 1 Processor -- start 2024-03-06T07:31:39.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.656+0000 7f1b2cc00640 1 -- start start 2024-03-06T07:31:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.657+0000 7f1b2cc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b280c9ec0 0x7f1b2815d300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.657+0000 7f1b2cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b280cf3e0 0x7f1b2815e530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.657+0000 7f1b2cc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28160690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.657+0000 7f1b2cc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b280cc270 con 0x7f1b280cf3e0 2024-03-06T07:31:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.657+0000 7f1b2cc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b280cc0f0 con 0x7f1b280c9ec0 2024-03-06T07:31:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.657+0000 7f1b2cc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1b280cc3f0 con 0x7f1b28144660 2024-03-06T07:31:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b280cf3e0 0x7f1b2815e530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b280cf3e0 0x7f1b2815e530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53356/0 (socket says 172.21.15.86:53356) 2024-03-06T07:31:39.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 -- 172.21.15.86:0/1381353954 learned_addr learned my addr 172.21.15.86:0/1381353954 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25e00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b280c9ec0 0x7f1b2815d300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b26800640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28160690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 -- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 msgr2=0x7f1b28160690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28160690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 -- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b280c9ec0 msgr2=0x7f1b2815d300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b280c9ec0 0x7f1b2815d300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b25400640 1 -- 172.21.15.86:0/1381353954 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b2815ef80 con 0x7f1b280cf3e0 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.658+0000 7f1b26800640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28160690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.659+0000 7f1b25400640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b280cf3e0 0x7f1b2815e530 secure :-1 s=READY pgs=965 cs=0 l=1 rev1=1 crypto rx=0x7f1b1400e9b0 tx=0x7f1b1400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.659+0000 7f1b1e000640 1 -- 172.21.15.86:0/1381353954 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b14017070 con 0x7f1b280cf3e0 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.659+0000 7f1b1e000640 1 -- 172.21.15.86:0/1381353954 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b1400c9a0 con 0x7f1b280cf3e0 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.659+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b2815f260 con 0x7f1b280cf3e0 2024-03-06T07:31:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.659+0000 7f1b1e000640 1 -- 172.21.15.86:0/1381353954 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b1400cd90 con 0x7f1b280cf3e0 2024-03-06T07:31:39.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.659+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b2815f820 con 0x7f1b280cf3e0 2024-03-06T07:31:39.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.660+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1b2804f330 con 0x7f1b280cf3e0 2024-03-06T07:31:39.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.661+0000 7f1b1e000640 1 -- 172.21.15.86:0/1381353954 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1b1401a070 con 0x7f1b280cf3e0 2024-03-06T07:31:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.663+0000 7f1b1e000640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b0806b4d0 0x7f1b0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.664+0000 7f1b1e000640 1 -- 172.21.15.86:0/1381353954 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1b1408a5e0 con 0x7f1b280cf3e0 2024-03-06T07:31:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.664+0000 7f1b25e00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b0806b4d0 0x7f1b0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.664+0000 7f1b25e00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b0806b4d0 0x7f1b0806d990 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f1b18036000 tx=0x7f1b18008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:39.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.667+0000 7f1b1e000640 1 -- 172.21.15.86:0/1381353954 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1b14059450 con 0x7f1b280cf3e0 2024-03-06T07:31:39.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.904+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f1b280d1880 con 0x7f1b280cf3e0 2024-03-06T07:31:39.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.905+0000 7f1b1e000640 1 -- 172.21.15.86:0/1381353954 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v50) v1 ==== 53+0+73 (secure 0 0 0) 0x7f1b14058d50 con 0x7f1b280cf3e0 2024-03-06T07:31:39.906 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b0806b4d0 msgr2=0x7f1b0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b0806b4d0 0x7f1b0806d990 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f1b18036000 tx=0x7f1b18008030 comp rx=0 tx=0).stop 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b280cf3e0 msgr2=0x7f1b2815e530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b280cf3e0 0x7f1b2815e530 secure :-1 s=READY pgs=965 cs=0 l=1 rev1=1 crypto rx=0x7f1b1400e9b0 tx=0x7f1b1400ee70 comp rx=0 tx=0).stop 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 shutdown_connections 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1b0806b4d0 0x7f1b0806d990 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1b28144660 0x7f1b28160690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1b280cf3e0 0x7f1b2815e530 unknown :-1 s=CLOSED pgs=965 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 --2- 172.21.15.86:0/1381353954 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1b280c9ec0 0x7f1b2815d300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 >> 172.21.15.86:0/1381353954 conn(0x7f1b280d0b90 msgr2=0x7f1b280b1a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:39.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 shutdown_connections 2024-03-06T07:31:39.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:39.907+0000 7f1b2cc00640 1 -- 172.21.15.86:0/1381353954 wait complete. 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:31:39.927 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:31:39.928 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:31:39.928 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:39.928 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:31:39.972 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42167sec preferred_lft 42167sec 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:31:39.973 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:31:39.974 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:31:39.974 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:31:39.974 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:31:39.974 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:31:39.974 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:31:39.974 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:31:39.974 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:39.974 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:31:40.096 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:31:40.100 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.101 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:31:40.148 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:31:40.148 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:31:40.148 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:31:40.148 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:31:40.149 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:31:40.149 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.149 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:31:40.149 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:31:40.149 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:31:40.149 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:31:40.149 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:31:40.149 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:31:40.306 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.307 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:31:40.353 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:31:40.354 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:31:40.354 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:31:40.354 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:31:40.437 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:31:40.437 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:31:40.485 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:31:40.485 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:31:40.485 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:31:40.530 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:31:40.530 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.531 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:31:40.707 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:31:40.707 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:31:40.739 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:31:40.739 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:31:40.739 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:31:40.739 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:31:40.739 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:31:40.741 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:31:40.741 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:31:40.741 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:31:40.742 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:31:40.742 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:31:40.748 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:31:40.748 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.748 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:31:40.825 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.825 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:31:40.884 INFO:teuthology.orchestra.run.smithi086.stdout:client6762 2024-03-06T07:31:40.884 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:40.885 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:40.938 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.938 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec touch fancy_layout_file) 2024-03-06T07:31:40.996 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:40.996 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec setfattr -n ceph.file.layout -v 'stripe_unit=1048576 stripe_count=4 object_size=8388608' fancy_layout_file) 2024-03-06T07:31:41.074 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:41.074 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:41.121 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:41.121 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec dd if=/dev/urandom of=fancy_layout_file bs=1M conv=fdatasync count=35 seek=0) 2024-03-06T07:31:41.563 INFO:teuthology.orchestra.run.smithi086.stderr:35+0 records in 2024-03-06T07:31:41.563 INFO:teuthology.orchestra.run.smithi086.stderr:35+0 records out 2024-03-06T07:31:41.563 INFO:teuthology.orchestra.run.smithi086.stderr:36700160 bytes (37 MB, 35 MiB) copied, 0.34543 s, 106 MB/s 2024-03-06T07:31:41.565 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:41.565 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f fancy_layout_file) 2024-03-06T07:31:41.605 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:41.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.839+0000 7f3c92000640 1 Processor -- start 2024-03-06T07:31:41.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.839+0000 7f3c92000640 1 -- start start 2024-03-06T07:31:41.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c92000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3c8c15dfd0 0x7f3c8c162700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:41.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c92000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3c8c06b6d0 0x7f3c8c0d06f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:41.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c92000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c8c0cef70 con 0x7f3c8c0affe0 2024-03-06T07:31:41.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c92000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c8c162270 con 0x7f3c8c15dfd0 2024-03-06T07:31:41.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c92000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c8c15e3b0 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c90c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3c8c15dfd0 0x7f3c8c162700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:41.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c90c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3c8c15dfd0 0x7f3c8c162700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53362/0 (socket says 172.21.15.86:53362) 2024-03-06T07:31:41.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c90c00640 1 -- 172.21.15.86:0/3474797287 learned_addr learned my addr 172.21.15.86:0/3474797287 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:41.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c90c00640 1 -- 172.21.15.86:0/3474797287 >> v2:172.21.15.194:3300/0 conn(0x7f3c8c06b6d0 msgr2=0x7f3c8c0d06f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:41.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c90c00640 1 --2- 172.21.15.86:0/3474797287 >> v2:172.21.15.194:3300/0 conn(0x7f3c8c06b6d0 0x7f3c8c0d06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:41.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c90c00640 1 -- 172.21.15.86:0/3474797287 >> v1:172.21.15.194:6789/0 conn(0x7f3c8c0affe0 legacy=0x7f3c8c06afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:41.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.841+0000 7f3c90c00640 1 -- 172.21.15.86:0/3474797287 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c8c161c60 con 0x7f3c8c15dfd0 2024-03-06T07:31:41.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.842+0000 7f3c90c00640 1 --2- 172.21.15.86:0/3474797287 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c162700 secure :-1 s=READY pgs=967 cs=0 l=1 rev1=1 crypto rx=0x7f3c8c0d1990 tx=0x7f3c8002b7f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb714e8b89f5da77 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:41.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c8b400640 1 -- 172.21.15.86:0/3474797287 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c80038070 con 0x7f3c8c15dfd0 2024-03-06T07:31:41.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c8b400640 1 -- 172.21.15.86:0/3474797287 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c8002f050 con 0x7f3c8c15dfd0 2024-03-06T07:31:41.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c8b400640 1 -- 172.21.15.86:0/3474797287 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c80031970 con 0x7f3c8c15dfd0 2024-03-06T07:31:41.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c92000640 1 -- 172.21.15.86:0/3474797287 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c15dfd0 msgr2=0x7f3c8c162700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:41.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c92000640 1 --2- 172.21.15.86:0/3474797287 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c162700 secure :-1 s=READY pgs=967 cs=0 l=1 rev1=1 crypto rx=0x7f3c8c0d1990 tx=0x7f3c8002b7f0 comp rx=0 tx=0).stop 2024-03-06T07:31:41.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c92000640 1 -- 172.21.15.86:0/3474797287 shutdown_connections 2024-03-06T07:31:41.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c92000640 1 --2- 172.21.15.86:0/3474797287 >> v2:172.21.15.194:3300/0 conn(0x7f3c8c06b6d0 0x7f3c8c0d06f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:41.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c92000640 1 --2- 172.21.15.86:0/3474797287 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c162700 unknown :-1 s=CLOSED pgs=967 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:41.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.843+0000 7f3c92000640 1 -- 172.21.15.86:0/3474797287 >> 172.21.15.86:0/3474797287 conn(0x7f3c8c06c0d0 msgr2=0x7f3c8c06c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:41.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.844+0000 7f3c92000640 1 -- 172.21.15.86:0/3474797287 shutdown_connections 2024-03-06T07:31:41.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.844+0000 7f3c92000640 1 -- 172.21.15.86:0/3474797287 wait complete. 2024-03-06T07:31:41.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.845+0000 7f3c92000640 1 Processor -- start 2024-03-06T07:31:41.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.845+0000 7f3c92000640 1 -- start start 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.846+0000 7f3c92000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c06b6d0 0x7f3c8c142f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.846+0000 7f3c92000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8c0affe0 0x7f3c8c15eaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.846+0000 7f3c92000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c13fb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.846+0000 7f3c92000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8c0ce8d0 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.846+0000 7f3c92000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8c161f00 con 0x7f3c8c15dfd0 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.846+0000 7f3c92000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8c0cea50 con 0x7f3c8c0affe0 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c06b6d0 0x7f3c8c142f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c06b6d0 0x7f3c8c142f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53376/0 (socket says 172.21.15.86:53376) 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 -- 172.21.15.86:0/203692480 learned_addr learned my addr 172.21.15.86:0/203692480 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c91600640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c13fb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 -- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8c0affe0 msgr2=0x7f3c8c15eaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:41.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c8be00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8c0affe0 0x7f3c8c15eaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8c0affe0 0x7f3c8c15eaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 -- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c8c15dfd0 msgr2=0x7f3c8c13fb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c13fb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c90c00640 1 -- 172.21.15.86:0/203692480 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c8c143450 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c91600640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c13fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.847+0000 7f3c8be00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8c0affe0 0x7f3c8c15eaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:41.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.848+0000 7f3c90c00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c06b6d0 0x7f3c8c142f10 secure :-1 s=READY pgs=968 cs=0 l=1 rev1=1 crypto rx=0x7f3c80000c00 tx=0x7f3c800046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:41.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.848+0000 7f3c88c00640 1 -- 172.21.15.86:0/203692480 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c80043070 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.848+0000 7f3c88c00640 1 -- 172.21.15.86:0/203692480 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c8003f870 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.848+0000 7f3c88c00640 1 -- 172.21.15.86:0/203692480 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c80038040 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.848+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c8c1406f0 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.848+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c8c140cb0 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.849+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c8c04f330 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.851+0000 7f3c88c00640 1 -- 172.21.15.86:0/203692480 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3c80046070 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.852+0000 7f3c88c00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b4d0 0x7f3c6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:41.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.853+0000 7f3c88c00640 1 -- 172.21.15.86:0/203692480 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3c800ac220 con 0x7f3c8c06b6d0 2024-03-06T07:31:41.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.853+0000 7f3c8be00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b4d0 0x7f3c6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:41.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.853+0000 7f3c8be00640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b4d0 0x7f3c6c06d990 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f3c78007fd0 tx=0x7f3c78009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:41.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:41.857+0000 7f3c88c00640 1 -- 172.21.15.86:0/203692480 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3c8007b1c0 con 0x7f3c8c06b6d0 2024-03-06T07:31:42.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.092+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3c8c0b5e60 con 0x7f3c8c06b6d0 2024-03-06T07:31:42.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.093+0000 7f3c88c00640 1 -- 172.21.15.86:0/203692480 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f3c8007aac0 con 0x7f3c8c06b6d0 2024-03-06T07:31:42.095 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:42.095 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:31:32.095887+0000","modified":"2024-03-06T07:31:35.343932+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6609},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6609":{"gid":6609,"name":"c","rank":0,"incarnation":47,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/255958031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":255958031},{"type":"v1","addr":"172.21.15.86:6837","nonce":255958031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6609,"qdb_cluster":[6609]},"id":6}]} 2024-03-06T07:31:42.095 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 50 2024-03-06T07:31:42.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.095+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b4d0 msgr2=0x7f3c6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:42.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.095+0000 7f3c92000640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b4d0 0x7f3c6c06d990 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f3c78007fd0 tx=0x7f3c78009040 comp rx=0 tx=0).stop 2024-03-06T07:31:42.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.095+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c06b6d0 msgr2=0x7f3c8c142f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:42.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.095+0000 7f3c92000640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c06b6d0 0x7f3c8c142f10 secure :-1 s=READY pgs=968 cs=0 l=1 rev1=1 crypto rx=0x7f3c80000c00 tx=0x7f3c800046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 shutdown_connections 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b4d0 0x7f3c6c06d990 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c8c15dfd0 0x7f3c8c13fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8c0affe0 0x7f3c8c15eaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 --2- 172.21.15.86:0/203692480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8c06b6d0 0x7f3c8c142f10 unknown :-1 s=CLOSED pgs=968 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 >> 172.21.15.86:0/203692480 conn(0x7f3c8c06c0d0 msgr2=0x7f3c8c06afc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 shutdown_connections 2024-03-06T07:31:42.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.096+0000 7f3c92000640 1 -- 172.21.15.86:0/203692480 wait complete. 2024-03-06T07:31:42.117 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:31:42.359 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:31:42.360 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:31:42.360 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph df --format=json-pretty 2024-03-06T07:31:42.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.561+0000 7f62ec400640 1 Processor -- start 2024-03-06T07:31:42.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.561+0000 7f62ec400640 1 -- start start 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62ec400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f62e41566a0 0x7f62e415ce60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62ec400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f62e4071d20 0x7f62e40b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62ec400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f62e415d3a0 con 0x7f62e40b5310 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62ec400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62e40b5740 con 0x7f62e4071d20 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62ec400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62e4071280 con 0x7f62e41566a0 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62e9000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f62e4071d20 0x7f62e40b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62e9000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f62e4071d20 0x7f62e40b4dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53384/0 (socket says 172.21.15.86:53384) 2024-03-06T07:31:42.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.562+0000 7f62e9000640 1 -- 172.21.15.86:0/712793004 learned_addr learned my addr 172.21.15.86:0/712793004 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:42.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.563+0000 7f62e9000640 1 -- 172.21.15.86:0/712793004 >> v2:172.21.15.194:3300/0 conn(0x7f62e41566a0 msgr2=0x7f62e415ce60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:42.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.563+0000 7f62e9a00640 1 --2- 172.21.15.86:0/712793004 >> v2:172.21.15.194:3300/0 conn(0x7f62e41566a0 0x7f62e415ce60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:42.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.563+0000 7f62e9000640 1 --2- 172.21.15.86:0/712793004 >> v2:172.21.15.194:3300/0 conn(0x7f62e41566a0 0x7f62e415ce60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.563+0000 7f62e9000640 1 -- 172.21.15.86:0/712793004 >> v1:172.21.15.194:6789/0 conn(0x7f62e40b5310 legacy=0x7f62e40ce070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:42.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.563+0000 7f62e9000640 1 -- 172.21.15.86:0/712793004 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62e4072df0 con 0x7f62e4071d20 2024-03-06T07:31:42.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.564+0000 7f62e9000640 1 --2- 172.21.15.86:0/712793004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e4071d20 0x7f62e40b4dd0 secure :-1 s=READY pgs=969 cs=0 l=1 rev1=1 crypto rx=0x7f62e40734e0 tx=0x7f62d002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ba4eb435a0fa85e8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:42.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.564+0000 7f62e3e00640 1 -- 172.21.15.86:0/712793004 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62d0035020 con 0x7f62e4071d20 2024-03-06T07:31:42.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.564+0000 7f62e3e00640 1 -- 172.21.15.86:0/712793004 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62d0031870 con 0x7f62e4071d20 2024-03-06T07:31:42.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.564+0000 7f62e3e00640 1 -- 172.21.15.86:0/712793004 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62d0031c40 con 0x7f62e4071d20 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 -- 172.21.15.86:0/712793004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e4071d20 msgr2=0x7f62e40b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 --2- 172.21.15.86:0/712793004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e4071d20 0x7f62e40b4dd0 secure :-1 s=READY pgs=969 cs=0 l=1 rev1=1 crypto rx=0x7f62e40734e0 tx=0x7f62d002b910 comp rx=0 tx=0).stop 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 -- 172.21.15.86:0/712793004 shutdown_connections 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 --2- 172.21.15.86:0/712793004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e4071d20 0x7f62e40b4dd0 unknown :-1 s=CLOSED pgs=969 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 --2- 172.21.15.86:0/712793004 >> v2:172.21.15.194:3300/0 conn(0x7f62e41566a0 0x7f62e415ce60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 -- 172.21.15.86:0/712793004 >> 172.21.15.86:0/712793004 conn(0x7f62e40cfa30 msgr2=0x7f62e40cfe40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 -- 172.21.15.86:0/712793004 shutdown_connections 2024-03-06T07:31:42.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.565+0000 7f62ec400640 1 -- 172.21.15.86:0/712793004 wait complete. 2024-03-06T07:31:42.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.567+0000 7f62ec400640 1 Processor -- start 2024-03-06T07:31:42.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.567+0000 7f62ec400640 1 -- start start 2024-03-06T07:31:42.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.568+0000 7f62ec400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62e4071d20 0x7f62e4151350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:42.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.568+0000 7f62ec400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62e40b5310 0x7f62e415e5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:42.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.568+0000 7f62ec400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e41566a0 0x7f62e415eb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.568+0000 7f62ec400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62e4071520 con 0x7f62e41566a0 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.568+0000 7f62ec400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62e40cc8f0 con 0x7f62e4071d20 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.568+0000 7f62ec400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62e4071800 con 0x7f62e40b5310 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.568+0000 7f62ea400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e41566a0 0x7f62e415eb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e41566a0 0x7f62e415eb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53398/0 (socket says 172.21.15.86:53398) 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 -- 172.21.15.86:0/4154468101 learned_addr learned my addr 172.21.15.86:0/4154468101 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62e9000640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62e40b5310 0x7f62e415e5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 -- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62e40b5310 msgr2=0x7f62e415e5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62e9a00640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62e4071d20 0x7f62e4151350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62e40b5310 0x7f62e415e5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 -- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62e4071d20 msgr2=0x7f62e4151350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62e4071d20 0x7f62e4151350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 -- 172.21.15.86:0/4154468101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62e4151890 con 0x7f62e41566a0 2024-03-06T07:31:42.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.569+0000 7f62ea400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e41566a0 0x7f62e415eb10 secure :-1 s=READY pgs=970 cs=0 l=1 rev1=1 crypto rx=0x7f62dc00c780 tx=0x7f62dc00cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.570+0000 7f62e1600640 1 -- 172.21.15.86:0/4154468101 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62dc017070 con 0x7f62e41566a0 2024-03-06T07:31:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.570+0000 7f62e1600640 1 -- 172.21.15.86:0/4154468101 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62dc003da0 con 0x7f62e41566a0 2024-03-06T07:31:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.570+0000 7f62e1600640 1 -- 172.21.15.86:0/4154468101 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62dc0052b0 con 0x7f62e41566a0 2024-03-06T07:31:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.570+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f62e41f4a10 con 0x7f62e41566a0 2024-03-06T07:31:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.570+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62e41524f0 con 0x7f62e41566a0 2024-03-06T07:31:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.571+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f62e404f330 con 0x7f62e41566a0 2024-03-06T07:31:42.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.572+0000 7f62e1600640 1 -- 172.21.15.86:0/4154468101 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f62dc01a070 con 0x7f62e41566a0 2024-03-06T07:31:42.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.573+0000 7f62e1600640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62c406b4d0 0x7f62c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:42.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.573+0000 7f62e1600640 1 -- 172.21.15.86:0/4154468101 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f62dc089890 con 0x7f62e41566a0 2024-03-06T07:31:42.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.573+0000 7f62e9a00640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62c406b4d0 0x7f62c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:42.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.574+0000 7f62e9a00640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62c406b4d0 0x7f62c406d990 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f62d800b7c0 tx=0x7f62d8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:42.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.577+0000 7f62e1600640 1 -- 172.21.15.86:0/4154468101 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f62e4151890 con 0x7f62e41566a0 2024-03-06T07:31:42.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.845+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "df", "format": "json-pretty"} v 0) v1 -- 0x7f62e40cf1c0 con 0x7f62e41566a0 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.846+0000 7f62e1600640 1 -- 172.21.15.86:0/4154468101 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "df", "format": "json-pretty"}]=0 v0) v1 ==== 75+0+1651 (secure 0 0 0) 0x7f62dc058700 con 0x7f62e41566a0 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stdout: "total_bytes": 773094113280, 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stdout: "total_avail_bytes": 772829282304, 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_bytes": 264830976, 2024-03-06T07:31:42.848 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_bytes": 264830976, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_ratio": 0.00034255982609465718, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "num_osds": 8, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "num_per_pool_osds": 8, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "num_per_pool_omap_osds": 8 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "stats_by_class": { 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "ssd": { 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "total_bytes": 773094113280, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "total_avail_bytes": 772829282304, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_bytes": 264830976, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_bytes": 264830976, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "total_used_raw_ratio": 0.00034255982609465718 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "pools": [ 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "name": ".mgr", 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 1, 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "stored": 590368, 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "objects": 2, 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "kb_used": 1160, 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "bytes_used": 1187840, 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "percent_used": 1.61793116149056e-06, 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "max_avail": 367085486080 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "cephfs_metadata", 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 12, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "stored": 2410, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "objects": 22, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "kb_used": 64, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "bytes_used": 65536, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "percent_used": 8.9265306257857446e-08, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "max_avail": 367085486080 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "cephfs_data", 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 13, 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "stats": { 2024-03-06T07:31:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: "stored": 0, 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: "objects": 0, 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: "kb_used": 0, 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: "bytes_used": 0, 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: "percent_used": 0, 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_avail": 367085486080 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:31:42.852 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:31:42.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62c406b4d0 msgr2=0x7f62c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:42.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62c406b4d0 0x7f62c406d990 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f62d800b7c0 tx=0x7f62d8008040 comp rx=0 tx=0).stop 2024-03-06T07:31:42.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e41566a0 msgr2=0x7f62e415eb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:42.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e41566a0 0x7f62e415eb10 secure :-1 s=READY pgs=970 cs=0 l=1 rev1=1 crypto rx=0x7f62dc00c780 tx=0x7f62dc00cc40 comp rx=0 tx=0).stop 2024-03-06T07:31:42.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 shutdown_connections 2024-03-06T07:31:42.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f62c406b4d0 0x7f62c406d990 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62e41566a0 0x7f62e415eb10 unknown :-1 s=CLOSED pgs=970 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f62e40b5310 0x7f62e415e5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 --2- 172.21.15.86:0/4154468101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62e4071d20 0x7f62e4151350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:42.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 >> 172.21.15.86:0/4154468101 conn(0x7f62e40cfa30 msgr2=0x7f62e415bd80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:42.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 shutdown_connections 2024-03-06T07:31:42.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:42.849+0000 7f62ec400640 1 -- 172.21.15.86:0/4154468101 wait complete. 2024-03-06T07:31:42.869 DEBUG:tasks.ceph_test_case:wait_until_true: success in 0s and 0 retries 2024-03-06T07:31:42.869 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:31:42.869 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:42.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:43.100 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:31:43.101 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:43.101 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:43.157 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:43.157 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:43.205 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:43.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.448+0000 7fa113800640 1 Processor -- start 2024-03-06T07:31:43.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.448+0000 7fa113800640 1 -- start start 2024-03-06T07:31:43.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.449+0000 7fa113800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa10c06ec00 0x7fa10c0ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.449+0000 7fa113800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa10c15b090 con 0x7fa10c15acb0 2024-03-06T07:31:43.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.449+0000 7fa113800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa10c0b2a30 con 0x7fa10c0b4680 2024-03-06T07:31:43.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.449+0000 7fa113800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa10c0b4360 con 0x7fa10c06ec00 2024-03-06T07:31:43.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.449+0000 7fa110e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa10c15acb0 0x7fa10c0b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55890/0 (socket says 172.21.15.86:55890) 2024-03-06T07:31:43.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.449+0000 7fa110e00640 1 -- 172.21.15.86:0/1424058877 learned_addr learned my addr 172.21.15.86:0/1424058877 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:43.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.450+0000 7fa111800640 1 --2- 172.21.15.86:0/1424058877 >> v2:172.21.15.86:3300/0 conn(0x7fa10c06ec00 0x7fa10c0ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.450+0000 7fa10b400640 1 -- 172.21.15.86:0/1424058877 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1300053852 0 0) 0x7fa10c15b090 con 0x7fa10c15acb0 2024-03-06T07:31:43.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.450+0000 7fa10b400640 1 -- 172.21.15.86:0/1424058877 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa0f4005730 con 0x7fa10c15acb0 2024-03-06T07:31:43.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.450+0000 7fa111800640 1 -- 172.21.15.86:0/1424058877 >> v1:172.21.15.194:6789/0 conn(0x7fa10c0b4680 legacy=0x7fa10c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.450+0000 7fa111800640 1 -- 172.21.15.86:0/1424058877 >> v1:172.21.15.86:6789/0 conn(0x7fa10c15acb0 legacy=0x7fa10c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.450+0000 7fa111800640 1 -- 172.21.15.86:0/1424058877 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa10c0b2cf0 con 0x7fa10c06ec00 2024-03-06T07:31:43.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.451+0000 7fa111800640 1 --2- 172.21.15.86:0/1424058877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c06ec00 0x7fa10c0ccfb0 secure :-1 s=READY pgs=972 cs=0 l=1 rev1=1 crypto rx=0x7fa104009d90 tx=0x7fa10402bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d40eaeb64fa94e43 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.451+0000 7fa10b400640 1 -- 172.21.15.86:0/1424058877 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa104039070 con 0x7fa10c06ec00 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.451+0000 7fa10b400640 1 -- 172.21.15.86:0/1424058877 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa10402cd00 con 0x7fa10c06ec00 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.451+0000 7fa10b400640 1 -- 172.21.15.86:0/1424058877 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa104034660 con 0x7fa10c06ec00 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.452+0000 7fa113800640 1 -- 172.21.15.86:0/1424058877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c06ec00 msgr2=0x7fa10c0ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.452+0000 7fa113800640 1 --2- 172.21.15.86:0/1424058877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c06ec00 0x7fa10c0ccfb0 secure :-1 s=READY pgs=972 cs=0 l=1 rev1=1 crypto rx=0x7fa104009d90 tx=0x7fa10402bf50 comp rx=0 tx=0).stop 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.452+0000 7fa113800640 1 -- 172.21.15.86:0/1424058877 shutdown_connections 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.452+0000 7fa113800640 1 --2- 172.21.15.86:0/1424058877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c06ec00 0x7fa10c0ccfb0 unknown :-1 s=CLOSED pgs=972 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.452+0000 7fa113800640 1 -- 172.21.15.86:0/1424058877 >> 172.21.15.86:0/1424058877 conn(0x7fa10c0b1730 msgr2=0x7fa10c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.452+0000 7fa113800640 1 -- 172.21.15.86:0/1424058877 shutdown_connections 2024-03-06T07:31:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.452+0000 7fa113800640 1 -- 172.21.15.86:0/1424058877 wait complete. 2024-03-06T07:31:43.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.454+0000 7fa113800640 1 Processor -- start 2024-03-06T07:31:43.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.454+0000 7fa113800640 1 -- start start 2024-03-06T07:31:43.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa113800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa10c06ec00 0x7fa10c0ae3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa113800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa10c0b4680 0x7fa10c0ceab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa113800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c15acb0 0x7fa10c15d2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa113800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa10c0b4010 con 0x7fa10c15acb0 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa113800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa10c0b32b0 con 0x7fa10c06ec00 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa113800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa10c072340 con 0x7fa10c0b4680 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa111800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c15acb0 0x7fa10c15d2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa111800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c15acb0 0x7fa10c15d2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53422/0 (socket says 172.21.15.86:53422) 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.455+0000 7fa111800640 1 -- 172.21.15.86:0/3186637491 learned_addr learned my addr 172.21.15.86:0/3186637491 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa110e00640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa10c06ec00 0x7fa10c0ae3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa111800640 1 -- 172.21.15.86:0/3186637491 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa10c0b4680 msgr2=0x7fa10c0ceab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa111800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa10c0b4680 0x7fa10c0ceab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa111800640 1 -- 172.21.15.86:0/3186637491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa10c06ec00 msgr2=0x7fa10c0ae3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa111800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa10c06ec00 0x7fa10c0ae3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa111800640 1 -- 172.21.15.86:0/3186637491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa10c15e070 con 0x7fa10c15acb0 2024-03-06T07:31:43.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa111800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c15acb0 0x7fa10c15d2b0 secure :-1 s=READY pgs=973 cs=0 l=1 rev1=1 crypto rx=0x7fa104009d90 tx=0x7fa104004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:43.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa108c00640 1 -- 172.21.15.86:0/3186637491 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa104039050 con 0x7fa10c15acb0 2024-03-06T07:31:43.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa108c00640 1 -- 172.21.15.86:0/3186637491 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa104045070 con 0x7fa10c15acb0 2024-03-06T07:31:43.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa10c15dd90 con 0x7fa10c15acb0 2024-03-06T07:31:43.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.456+0000 7fa108c00640 1 -- 172.21.15.86:0/3186637491 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa104009450 con 0x7fa10c15acb0 2024-03-06T07:31:43.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.457+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa10c0adbe0 con 0x7fa10c15acb0 2024-03-06T07:31:43.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.458+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa10c04f330 con 0x7fa10c15acb0 2024-03-06T07:31:43.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.459+0000 7fa108c00640 1 -- 172.21.15.86:0/3186637491 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa10403e830 con 0x7fa10c15acb0 2024-03-06T07:31:43.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.459+0000 7fa108c00640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa0ec06b4d0 0x7fa0ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.460+0000 7fa108c00640 1 -- 172.21.15.86:0/3186637491 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa1040acf80 con 0x7fa10c15acb0 2024-03-06T07:31:43.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.460+0000 7fa110e00640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa0ec06b4d0 0x7fa0ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.460+0000 7fa110e00640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa0ec06b4d0 0x7fa0ec06d990 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7fa1000011e0 tx=0x7fa100009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:43.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.469+0000 7fa108c00640 1 -- 172.21.15.86:0/3186637491 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa104032e20 con 0x7fa10c15acb0 2024-03-06T07:31:43.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.665+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa10c06d760 con 0x7fa10c15acb0 2024-03-06T07:31:43.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.667+0000 7fa108c00640 1 -- 172.21.15.86:0/3186637491 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+16672 (secure 0 0 0) 0x7fa10407be80 con 0x7fa10c15acb0 2024-03-06T07:31:43.669 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:43.669 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":83,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:33.657039+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:31:29.878719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-06T07:31:31.115332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:43.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.669+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa0ec06b4d0 msgr2=0x7fa0ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.669+0000 7fa113800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa0ec06b4d0 0x7fa0ec06d990 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7fa1000011e0 tx=0x7fa100009040 comp rx=0 tx=0).stop 2024-03-06T07:31:43.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.669+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c15acb0 msgr2=0x7fa10c15d2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.669+0000 7fa113800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c15acb0 0x7fa10c15d2b0 secure :-1 s=READY pgs=973 cs=0 l=1 rev1=1 crypto rx=0x7fa104009d90 tx=0x7fa104004a40 comp rx=0 tx=0).stop 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 shutdown_connections 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa0ec06b4d0 0x7fa0ec06d990 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa10c15acb0 0x7fa10c15d2b0 unknown :-1 s=CLOSED pgs=973 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa10c0b4680 0x7fa10c0ceab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 --2- 172.21.15.86:0/3186637491 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa10c06ec00 0x7fa10c0ae3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 >> 172.21.15.86:0/3186637491 conn(0x7fa10c0b1730 msgr2=0x7fa10c0af730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 shutdown_connections 2024-03-06T07:31:43.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.670+0000 7fa113800640 1 -- 172.21.15.86:0/3186637491 wait complete. 2024-03-06T07:31:43.697 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 12, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:31:29.878719+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '82', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2, 'score_stable': 2, 'optimal_score': 1, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 13, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:31:31.115332+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:31:43.698 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:31:43.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.898+0000 7f4b5be00640 1 Processor -- start 2024-03-06T07:31:43.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.898+0000 7f4b5be00640 1 -- start start 2024-03-06T07:31:43.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.899+0000 7f4b5be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c06da40 0x7f4b5c0ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.899+0000 7f4b5be00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b5c15c1f0 con 0x7f4b5c0aef20 2024-03-06T07:31:43.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.899+0000 7f4b5be00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b5c06cac0 con 0x7f4b5c15be10 2024-03-06T07:31:43.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.900+0000 7f4b5be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c0cd200 con 0x7f4b5c06da40 2024-03-06T07:31:43.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.900+0000 7f4b5b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4b5c0aef20 0x7f4b5c071ab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55906/0 (socket says 172.21.15.86:55906) 2024-03-06T07:31:43.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.900+0000 7f4b5b400640 1 -- 172.21.15.86:0/2603282512 learned_addr learned my addr 172.21.15.86:0/2603282512 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:43.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.900+0000 7f4b53e00640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c06da40 0x7f4b5c0ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.900+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4053606096 0 0) 0x7f4b5c15c1f0 con 0x7f4b5c0aef20 2024-03-06T07:31:43.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.900+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4b40003520 con 0x7f4b5c0aef20 2024-03-06T07:31:43.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3608800114 0 0) 0x7f4b40003520 con 0x7f4b5c0aef20 2024-03-06T07:31:43.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c06da40 msgr2=0x7f4b5c0ccc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:43.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c06da40 0x7f4b5c0ccc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 >> v1:172.21.15.194:6789/0 conn(0x7f4b5c15be10 legacy=0x7f4b5c06d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b5c06cda0 con 0x7f4b5c0aef20 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f4b540019f0 con 0x7f4b5c0aef20 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 >> v1:172.21.15.86:6789/0 conn(0x7f4b5c0aef20 legacy=0x7f4b5c071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b40003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b40004800 0x7f4b40004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c06da40 0x7f4b400051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c0cd200 con 0x7f4b400039e0 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b4000ba70 con 0x7f4b40004800 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b4000bc70 con 0x7f4b5c06da40 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.901+0000 7f4b5aa00640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b40003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.902+0000 7f4b5aa00640 1 -- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c06da40 msgr2=0x7f4b400051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:43.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.902+0000 7f4b5aa00640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c06da40 0x7f4b400051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.902+0000 7f4b53e00640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b40004800 0x7f4b40004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.902+0000 7f4b5aa00640 1 -- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b40004800 msgr2=0x7f4b40004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:43.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.902+0000 7f4b5aa00640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b40004800 0x7f4b40004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.902+0000 7f4b5aa00640 1 -- 172.21.15.86:0/2603282512 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b40003520 con 0x7f4b400039e0 2024-03-06T07:31:43.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.903+0000 7f4b5aa00640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b40003e00 secure :-1 s=READY pgs=975 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c001720 tx=0x7f4b40005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=964c476c7a9b9093 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.903+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4c001370 con 0x7f4b400039e0 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.903+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b4c0052d0 con 0x7f4b400039e0 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.903+0000 7f4b5a000640 1 -- 172.21.15.86:0/2603282512 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4c003830 con 0x7f4b400039e0 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.903+0000 7f4b5be00640 1 -- 172.21.15.86:0/2603282512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 msgr2=0x7f4b40003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.903+0000 7f4b5be00640 1 --2- 172.21.15.86:0/2603282512 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b40003e00 secure :-1 s=READY pgs=975 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c001720 tx=0x7f4b40005830 comp rx=0 tx=0).stop 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.903+0000 7f4b5b400640 1 -- 172.21.15.86:0/2603282512 reap_dead start 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.904+0000 7f4b5be00640 1 -- 172.21.15.86:0/2603282512 shutdown_connections 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.904+0000 7f4b5be00640 1 -- 172.21.15.86:0/2603282512 >> 172.21.15.86:0/2603282512 conn(0x7f4b5c06afc0 msgr2=0x7f4b5c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.904+0000 7f4b5be00640 1 -- 172.21.15.86:0/2603282512 shutdown_connections 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.904+0000 7f4b5be00640 1 -- 172.21.15.86:0/2603282512 wait complete. 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.906+0000 7f4b5be00640 1 Processor -- start 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.907+0000 7f4b5be00640 1 -- start start 2024-03-06T07:31:43.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b5c151dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b5c151950 0x7f4b5c0b3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c0b3ba0 0x7f4b5c1f3060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c0cda90 con 0x7f4b400039e0 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c06d080 con 0x7f4b5c151950 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c0ccc20 con 0x7f4b5c0b3ba0 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b5c151dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.908+0000 7f4b5aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b5c151dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53452/0 (socket says 172.21.15.86:53452) 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b5aa00640 1 -- 172.21.15.86:0/2426965140 learned_addr learned my addr 172.21.15.86:0/2426965140 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b53e00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b5c151950 0x7f4b5c0b3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b5aa00640 1 -- 172.21.15.86:0/2426965140 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c0b3ba0 msgr2=0x7f4b5c1f3060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b5aa00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c0b3ba0 0x7f4b5c1f3060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b5aa00640 1 -- 172.21.15.86:0/2426965140 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b5c151950 msgr2=0x7f4b5c0b3660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b5aa00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b5c151950 0x7f4b5c0b3660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:43.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b5aa00640 1 -- 172.21.15.86:0/2426965140 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b5c152710 con 0x7f4b400039e0 2024-03-06T07:31:43.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b5aa00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b5c151dc0 secure :-1 s=READY pgs=976 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c034040 tx=0x7f4b4c001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:43.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.909+0000 7f4b52a00640 1 -- 172.21.15.86:0/2426965140 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4c042070 con 0x7f4b400039e0 2024-03-06T07:31:43.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.910+0000 7f4b52a00640 1 -- 172.21.15.86:0/2426965140 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b4c005480 con 0x7f4b400039e0 2024-03-06T07:31:43.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.910+0000 7f4b52a00640 1 -- 172.21.15.86:0/2426965140 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4c007a30 con 0x7f4b400039e0 2024-03-06T07:31:43.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.910+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b5c14eb10 con 0x7f4b400039e0 2024-03-06T07:31:43.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.910+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b5c14f0d0 con 0x7f4b400039e0 2024-03-06T07:31:43.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.911+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b5c04f330 con 0x7f4b400039e0 2024-03-06T07:31:43.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.912+0000 7f4b52a00640 1 -- 172.21.15.86:0/2426965140 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b4c045070 con 0x7f4b400039e0 2024-03-06T07:31:43.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.913+0000 7f4b52a00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b3806b4d0 0x7f4b3806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:43.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.913+0000 7f4b52a00640 1 -- 172.21.15.86:0/2426965140 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4b4c0abd90 con 0x7f4b400039e0 2024-03-06T07:31:43.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.913+0000 7f4b53e00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b3806b4d0 0x7f4b3806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:43.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.914+0000 7f4b53e00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b3806b4d0 0x7f4b3806d990 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f4b5c0b35b0 tx=0x7f4b44008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:43.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:43.922+0000 7f4b52a00640 1 -- 172.21.15.86:0/2426965140 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b4c07aca0 con 0x7f4b400039e0 2024-03-06T07:31:44.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.117+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f4b5c0b1190 con 0x7f4b400039e0 2024-03-06T07:31:44.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.118+0000 7f4b52a00640 1 -- 172.21.15.86:0/2426965140 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v83) v1 ==== 93+0+10 (secure 0 0 0) 0x7f4b4c07a5a0 con 0x7f4b400039e0 2024-03-06T07:31:44.120 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:31:44.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b3806b4d0 msgr2=0x7f4b3806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b3806b4d0 0x7f4b3806d990 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f4b5c0b35b0 tx=0x7f4b44008040 comp rx=0 tx=0).stop 2024-03-06T07:31:44.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 msgr2=0x7f4b5c151dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b5c151dc0 secure :-1 s=READY pgs=976 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c034040 tx=0x7f4b4c001120 comp rx=0 tx=0).stop 2024-03-06T07:31:44.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 shutdown_connections 2024-03-06T07:31:44.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b3806b4d0 0x7f4b3806d990 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b5c0b3ba0 0x7f4b5c1f3060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b5c151950 0x7f4b5c0b3660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 --2- 172.21.15.86:0/2426965140 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b400039e0 0x7f4b5c151dc0 unknown :-1 s=CLOSED pgs=976 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 >> 172.21.15.86:0/2426965140 conn(0x7f4b5c06afc0 msgr2=0x7f4b5c0b0150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:44.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 shutdown_connections 2024-03-06T07:31:44.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.121+0000 7f4b5be00640 1 -- 172.21.15.86:0/2426965140 wait complete. 2024-03-06T07:31:44.140 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:31:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.329+0000 7fd0b6800640 1 Processor -- start 2024-03-06T07:31:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.329+0000 7fd0b6800640 1 -- start start 2024-03-06T07:31:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.330+0000 7fd0b6800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00d06d0 0x7fd0b00d0ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.330+0000 7fd0b6800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd0b00b4b90 0x7fd0b00cea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.330+0000 7fd0b6800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd0b00d0ff0 con 0x7fd0b00b3760 2024-03-06T07:31:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.330+0000 7fd0b6800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd0b00cf2f0 con 0x7fd0b00b4b90 2024-03-06T07:31:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.330+0000 7fd0b6800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0b00cf5d0 con 0x7fd0b00d06d0 2024-03-06T07:31:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.331+0000 7fd0b4a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd0b00b4b90 0x7fd0b00cea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.331+0000 7fd0b4a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd0b00b4b90 0x7fd0b00cea10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:47030/0 (socket says 172.21.15.86:47030) 2024-03-06T07:31:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.331+0000 7fd0b4a00640 1 -- 172.21.15.86:0/3868191937 learned_addr learned my addr 172.21.15.86:0/3868191937 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.331+0000 7fd0b5400640 1 --2- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00d06d0 0x7fd0b00d0ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.332+0000 7fd0b4a00640 1 -- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00d06d0 msgr2=0x7fd0b00d0ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.332+0000 7fd0b4a00640 1 --2- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00d06d0 0x7fd0b00d0ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.332+0000 7fd0b4a00640 1 -- 172.21.15.86:0/3868191937 >> v1:172.21.15.194:6789/0 conn(0x7fd0b00b3760 legacy=0x7fd0b00b3bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.332+0000 7fd0b4a00640 1 -- 172.21.15.86:0/3868191937 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0b00cf8b0 con 0x7fd0b00b4b90 2024-03-06T07:31:44.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.334+0000 7fd0b4a00640 1 --2- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00b4b90 0x7fd0b00cea10 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7fd09c009870 tx=0x7fd09c02b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ec1a02218e9f8680 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:44.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0afe00640 1 -- 172.21.15.86:0/3868191937 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd09c034070 con 0x7fd0b00b4b90 2024-03-06T07:31:44.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0afe00640 1 -- 172.21.15.86:0/3868191937 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd09c030840 con 0x7fd0b00b4b90 2024-03-06T07:31:44.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0afe00640 1 -- 172.21.15.86:0/3868191937 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd09c030c10 con 0x7fd0b00b4b90 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0b6800640 1 -- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00b4b90 msgr2=0x7fd0b00cea10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0b6800640 1 --2- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00b4b90 0x7fd0b00cea10 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7fd09c009870 tx=0x7fd09c02b980 comp rx=0 tx=0).stop 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0b6800640 1 -- 172.21.15.86:0/3868191937 shutdown_connections 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0b6800640 1 --2- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00b4b90 0x7fd0b00cea10 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0b6800640 1 --2- 172.21.15.86:0/3868191937 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00d06d0 0x7fd0b00d0ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.335+0000 7fd0b6800640 1 -- 172.21.15.86:0/3868191937 >> 172.21.15.86:0/3868191937 conn(0x7fd0b009d070 msgr2=0x7fd0b00d3e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.336+0000 7fd0b6800640 1 -- 172.21.15.86:0/3868191937 shutdown_connections 2024-03-06T07:31:44.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.336+0000 7fd0b6800640 1 -- 172.21.15.86:0/3868191937 wait complete. 2024-03-06T07:31:44.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.338+0000 7fd0b6800640 1 Processor -- start 2024-03-06T07:31:44.339 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.338+0000 7fd0b6800640 1 -- start start 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.339+0000 7fd0b6800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00b3760 0x7fd0b01574f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.339+0000 7fd0b6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0b00b4b90 0x7fd0b015d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.339+0000 7fd0b6800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00d06d0 0x7fd0b00b1b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.339+0000 7fd0b6800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0b00cef50 con 0x7fd0b00b4b90 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.339+0000 7fd0b6800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0b00b0960 con 0x7fd0b00d06d0 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.339+0000 7fd0b6800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0b00cf0d0 con 0x7fd0b00b3760 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0b00b4b90 0x7fd0b015d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0b00b4b90 0x7fd0b015d930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53462/0 (socket says 172.21.15.86:53462) 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 -- 172.21.15.86:0/1818260194 learned_addr learned my addr 172.21.15.86:0/1818260194 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b5400640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00b3760 0x7fd0b01574f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b5e00640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00d06d0 0x7fd0b00b1b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 -- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00b3760 msgr2=0x7fd0b01574f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00b3760 0x7fd0b01574f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 -- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00d06d0 msgr2=0x7fd0b00b1b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00d06d0 0x7fd0b00b1b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 -- 172.21.15.86:0/1818260194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0b015e2a0 con 0x7fd0b00b4b90 2024-03-06T07:31:44.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b5e00640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00d06d0 0x7fd0b00b1b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:44.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.340+0000 7fd0b4a00640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0b00b4b90 0x7fd0b015d930 secure :-1 s=READY pgs=977 cs=0 l=1 rev1=1 crypto rx=0x7fd09c02b9c0 tx=0x7fd09c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:44.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.341+0000 7fd0ad600640 1 -- 172.21.15.86:0/1818260194 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd09c048070 con 0x7fd0b00b4b90 2024-03-06T07:31:44.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.341+0000 7fd0ad600640 1 -- 172.21.15.86:0/1818260194 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd09c030640 con 0x7fd0b00b4b90 2024-03-06T07:31:44.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.341+0000 7fd0ad600640 1 -- 172.21.15.86:0/1818260194 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd09c034040 con 0x7fd0b00b4b90 2024-03-06T07:31:44.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.341+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd0b015e580 con 0x7fd0b00b4b90 2024-03-06T07:31:44.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.341+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0b015eb40 con 0x7fd0b00b4b90 2024-03-06T07:31:44.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.342+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd0b00c8280 con 0x7fd0b00b4b90 2024-03-06T07:31:44.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.343+0000 7fd0ad600640 1 -- 172.21.15.86:0/1818260194 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd09c04b070 con 0x7fd0b00b4b90 2024-03-06T07:31:44.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.344+0000 7fd0ad600640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd09006b4d0 0x7fd09006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.344+0000 7fd0ad600640 1 -- 172.21.15.86:0/1818260194 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd09c0ac290 con 0x7fd0b00b4b90 2024-03-06T07:31:44.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.344+0000 7fd0b5400640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd09006b4d0 0x7fd09006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.345+0000 7fd0b5400640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd09006b4d0 0x7fd09006d990 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7fd0a4009a30 tx=0x7fd0a4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:44.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.353+0000 7fd0ad600640 1 -- 172.21.15.86:0/1818260194 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd0b00c8280 con 0x7fd0b00b4b90 2024-03-06T07:31:44.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.560+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fd0b00ca330 con 0x7fd0b00b4b90 2024-03-06T07:31:44.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.561+0000 7fd0ad600640 1 -- 172.21.15.86:0/1818260194 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v83) v1 ==== 104+0+11 (secure 0 0 0) 0x7fd09c07b230 con 0x7fd0b00b4b90 2024-03-06T07:31:44.563 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:31:44.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.563+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd09006b4d0 msgr2=0x7fd09006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.563+0000 7fd0b6800640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd09006b4d0 0x7fd09006d990 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7fd0a4009a30 tx=0x7fd0a4008040 comp rx=0 tx=0).stop 2024-03-06T07:31:44.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.563+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0b00b4b90 msgr2=0x7fd0b015d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.563+0000 7fd0b6800640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0b00b4b90 0x7fd0b015d930 secure :-1 s=READY pgs=977 cs=0 l=1 rev1=1 crypto rx=0x7fd09c02b9c0 tx=0x7fd09c004730 comp rx=0 tx=0).stop 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 shutdown_connections 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd09006b4d0 0x7fd09006d990 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0b00d06d0 0x7fd0b00b1b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0b00b4b90 0x7fd0b015d930 unknown :-1 s=CLOSED pgs=977 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 --2- 172.21.15.86:0/1818260194 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0b00b3760 0x7fd0b01574f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 >> 172.21.15.86:0/1818260194 conn(0x7fd0b009d070 msgr2=0x7fd0b00d2710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 shutdown_connections 2024-03-06T07:31:44.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.564+0000 7fd0b6800640 1 -- 172.21.15.86:0/1818260194 wait complete. 2024-03-06T07:31:44.585 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:31:44.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.785+0000 7f746f600640 1 Processor -- start 2024-03-06T07:31:44.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.786+0000 7f746f600640 1 -- start start 2024-03-06T07:31:44.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.787+0000 7f746f600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f74680b3f70 0x7f7468160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.787+0000 7f746f600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f74680ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.787+0000 7f746f600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f74680b46c0 con 0x7f746815fc10 2024-03-06T07:31:44.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.787+0000 7f746f600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7468071480 con 0x7f74680b3f70 2024-03-06T07:31:44.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.787+0000 7f746f600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f74680b49a0 con 0x7f7468070ff0 2024-03-06T07:31:44.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f746cc00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f746815fc10 0x7f746815fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55916/0 (socket says 172.21.15.86:55916) 2024-03-06T07:31:44.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f746cc00640 1 -- 172.21.15.86:0/995063447 learned_addr learned my addr 172.21.15.86:0/995063447 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:44.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f745fe00640 1 --2- 172.21.15.86:0/995063447 >> v2:172.21.15.194:3300/0 conn(0x7f74680b3f70 0x7f7468160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f746d600640 1 --2- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f74680ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f7467e00640 1 -- 172.21.15.86:0/995063447 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2002463619 0 0) 0x7f74680b46c0 con 0x7f746815fc10 2024-03-06T07:31:44.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f7467e00640 1 -- 172.21.15.86:0/995063447 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f744c003520 con 0x7f746815fc10 2024-03-06T07:31:44.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f745fe00640 1 -- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 msgr2=0x7f74680ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:44.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.788+0000 7f745fe00640 1 --2- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f74680ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.789+0000 7f745fe00640 1 -- 172.21.15.86:0/995063447 >> v1:172.21.15.86:6789/0 conn(0x7f746815fc10 legacy=0x7f746815fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.789+0000 7f7467e00640 1 -- 172.21.15.86:0/995063447 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1146686396 0 0) 0x7f744c003520 con 0x7f746815fc10 2024-03-06T07:31:44.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.789+0000 7f745fe00640 1 -- 172.21.15.86:0/995063447 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7468072c90 con 0x7f74680b3f70 2024-03-06T07:31:44.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.790+0000 7f745fe00640 1 --2- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f7468160700 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f74500097a0 tx=0x7f745002b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=127cec4f45095219 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:44.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.790+0000 7f7467e00640 1 -- 172.21.15.86:0/995063447 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7450037070 con 0x7f74680b3f70 2024-03-06T07:31:44.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.790+0000 7f7467e00640 1 -- 172.21.15.86:0/995063447 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7450030510 con 0x7f74680b3f70 2024-03-06T07:31:44.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.790+0000 7f7467e00640 1 -- 172.21.15.86:0/995063447 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7458001670 con 0x7f74680b3f70 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 -- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 msgr2=0x7f7468160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 --2- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f7468160700 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f74500097a0 tx=0x7f745002b790 comp rx=0 tx=0).stop 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 -- 172.21.15.86:0/995063447 shutdown_connections 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 --2- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f74680ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 --2- 172.21.15.86:0/995063447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f7468160700 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 -- 172.21.15.86:0/995063447 >> 172.21.15.86:0/995063447 conn(0x7f74680b1940 msgr2=0x7f74680b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 -- 172.21.15.86:0/995063447 shutdown_connections 2024-03-06T07:31:44.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.791+0000 7f746f600640 1 -- 172.21.15.86:0/995063447 wait complete. 2024-03-06T07:31:44.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.793+0000 7f746f600640 1 Processor -- start 2024-03-06T07:31:44.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.793+0000 7f746f600640 1 -- start start 2024-03-06T07:31:44.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746f600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f746809d340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746f600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f746806cab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746f600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f746815fc10 0x7f746809b570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746f600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7468072a30 con 0x7f746815fc10 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746f600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74680728b0 con 0x7f74680b3f70 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746f600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f746815c2e0 con 0x7f7468070ff0 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f746815fc10 0x7f746809b570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f746815fc10 0x7f746809b570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53478/0 (socket says 172.21.15.86:53478) 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 -- 172.21.15.86:0/302484962 learned_addr learned my addr 172.21.15.86:0/302484962 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f745fe00640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f746806cab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746cc00640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f746809d340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 -- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 msgr2=0x7f746809d340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f746809d340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 -- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 msgr2=0x7f746806cab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f746806cab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f746d600640 1 -- 172.21.15.86:0/302484962 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f746806d220 con 0x7f746815fc10 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.794+0000 7f745fe00640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f746806cab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.795+0000 7f746d600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f746815fc10 0x7f746809b570 secure :-1 s=READY pgs=979 cs=0 l=1 rev1=1 crypto rx=0x7f746000efc0 tx=0x7f746000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.795+0000 7f7465600640 1 -- 172.21.15.86:0/302484962 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7460018020 con 0x7f746815fc10 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.795+0000 7f7465600640 1 -- 172.21.15.86:0/302484962 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7460014480 con 0x7f746815fc10 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.795+0000 7f7465600640 1 -- 172.21.15.86:0/302484962 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7460014870 con 0x7f746815fc10 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.795+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f746806d500 con 0x7f746815fc10 2024-03-06T07:31:44.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.795+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f746806dac0 con 0x7f746815fc10 2024-03-06T07:31:44.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.797+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f746804f330 con 0x7f746815fc10 2024-03-06T07:31:44.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.797+0000 7f7465600640 1 -- 172.21.15.86:0/302484962 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7460005d10 con 0x7f746815fc10 2024-03-06T07:31:44.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.798+0000 7f7465600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f744406b4d0 0x7f744406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:44.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.798+0000 7f7465600640 1 -- 172.21.15.86:0/302484962 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f746008b0d0 con 0x7f746815fc10 2024-03-06T07:31:44.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.798+0000 7f746cc00640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f744406b4d0 0x7f744406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:44.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.799+0000 7f746cc00640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f744406b4d0 0x7f744406d990 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f74580011e0 tx=0x7f7458009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:44.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:44.803+0000 7f7465600640 1 -- 172.21.15.86:0/302484962 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7460022030 con 0x7f746815fc10 2024-03-06T07:31:45.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.022+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f746806ef30 con 0x7f746815fc10 2024-03-06T07:31:45.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.023+0000 7f7465600640 1 -- 172.21.15.86:0/302484962 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v83) v1 ==== 100+0+11 (secure 0 0 0) 0x7f7460010040 con 0x7f746815fc10 2024-03-06T07:31:45.025 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:31:45.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.025+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f744406b4d0 msgr2=0x7f744406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.025+0000 7f746f600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f744406b4d0 0x7f744406d990 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f74580011e0 tx=0x7f7458009040 comp rx=0 tx=0).stop 2024-03-06T07:31:45.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.025+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f746815fc10 msgr2=0x7f746809b570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.025+0000 7f746f600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f746815fc10 0x7f746809b570 secure :-1 s=READY pgs=979 cs=0 l=1 rev1=1 crypto rx=0x7f746000efc0 tx=0x7f746000c490 comp rx=0 tx=0).stop 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 shutdown_connections 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f744406b4d0 0x7f744406d990 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f746815fc10 0x7f746809b570 unknown :-1 s=CLOSED pgs=979 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f74680b3f70 0x7f746806cab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 --2- 172.21.15.86:0/302484962 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7468070ff0 0x7f746809d340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 >> 172.21.15.86:0/302484962 conn(0x7f74680b1940 msgr2=0x7f74681511e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 shutdown_connections 2024-03-06T07:31:45.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.026+0000 7f746f600640 1 -- 172.21.15.86:0/302484962 wait complete. 2024-03-06T07:31:45.046 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:31:45.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.229+0000 7f43ebc00640 1 Processor -- start 2024-03-06T07:31:45.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.229+0000 7f43ebc00640 1 -- start start 2024-03-06T07:31:45.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.231+0000 7f43ebc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f43e40c4930 0x7f43e40c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:45.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.231+0000 7f43ebc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f43e4145b30 con 0x7f43e40c5880 2024-03-06T07:31:45.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.231+0000 7f43ebc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f43e4142ce0 con 0x7f43e40c4d10 2024-03-06T07:31:45.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.231+0000 7f43ebc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43e40cbca0 con 0x7f43e40c4930 2024-03-06T07:31:45.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.231+0000 7f43e9200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f43e40c4930 0x7f43e40c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:45.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.231+0000 7f43e9200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f43e40c4930 0x7f43e40c6460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53494/0 (socket says 172.21.15.86:53494) 2024-03-06T07:31:45.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.231+0000 7f43e9200640 1 -- 172.21.15.86:0/1306035698 learned_addr learned my addr 172.21.15.86:0/1306035698 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:45.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.232+0000 7f43e9200640 1 -- 172.21.15.86:0/1306035698 >> v1:172.21.15.194:6789/0 conn(0x7f43e40c4d10 legacy=0x7f43e40c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:45.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.232+0000 7f43e9200640 1 -- 172.21.15.86:0/1306035698 >> v1:172.21.15.86:6789/0 conn(0x7f43e40c5880 legacy=0x7f43e4165b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.232+0000 7f43e9200640 1 -- 172.21.15.86:0/1306035698 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43e40cbf80 con 0x7f43e40c4930 2024-03-06T07:31:45.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.232+0000 7f43e3e00640 1 -- 172.21.15.86:0/1306035698 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 929713741 0 0) 0x7f43dc001490 con 0x7f43e40c5880 2024-03-06T07:31:45.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.233+0000 7f43e9200640 1 --2- 172.21.15.86:0/1306035698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e40c6460 secure :-1 s=READY pgs=981 cs=0 l=1 rev1=1 crypto rx=0x7f43d4009870 tx=0x7f43d402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3e14497a5377afe3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:45.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.233+0000 7f43e3e00640 1 -- 172.21.15.86:0/1306035698 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43d4035020 con 0x7f43e40c4930 2024-03-06T07:31:45.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.233+0000 7f43e3e00640 1 -- 172.21.15.86:0/1306035698 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43d4031870 con 0x7f43e40c4930 2024-03-06T07:31:45.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.233+0000 7f43e3e00640 1 -- 172.21.15.86:0/1306035698 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43d4031c40 con 0x7f43e40c4930 2024-03-06T07:31:45.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.234+0000 7f43ebc00640 1 -- 172.21.15.86:0/1306035698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 msgr2=0x7f43e40c6460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.234+0000 7f43ebc00640 1 --2- 172.21.15.86:0/1306035698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e40c6460 secure :-1 s=READY pgs=981 cs=0 l=1 rev1=1 crypto rx=0x7f43d4009870 tx=0x7f43d402b910 comp rx=0 tx=0).stop 2024-03-06T07:31:45.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.234+0000 7f43ebc00640 1 -- 172.21.15.86:0/1306035698 shutdown_connections 2024-03-06T07:31:45.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.234+0000 7f43ebc00640 1 --2- 172.21.15.86:0/1306035698 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e40c6460 unknown :-1 s=CLOSED pgs=981 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.234+0000 7f43ebc00640 1 -- 172.21.15.86:0/1306035698 >> 172.21.15.86:0/1306035698 conn(0x7f43e40cc4f0 msgr2=0x7f43e40cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:45.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.234+0000 7f43ebc00640 1 -- 172.21.15.86:0/1306035698 shutdown_connections 2024-03-06T07:31:45.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.234+0000 7f43ebc00640 1 -- 172.21.15.86:0/1306035698 wait complete. 2024-03-06T07:31:45.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.236+0000 7f43ebc00640 1 Processor -- start 2024-03-06T07:31:45.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.236+0000 7f43ebc00640 1 -- start start 2024-03-06T07:31:45.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43ebc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e41f3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:45.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43ebc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43e40c4d10 0x7f43e40b0630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:45.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43ebc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43e40c5880 0x7f43e4143ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:45.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43ebc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43e406e600 con 0x7f43e40c4930 2024-03-06T07:31:45.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43ebc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43e40ca3b0 con 0x7f43e40c4d10 2024-03-06T07:31:45.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43ebc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43e40cc260 con 0x7f43e40c5880 2024-03-06T07:31:45.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43e9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e41f3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43e9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e41f3ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53498/0 (socket says 172.21.15.86:53498) 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.237+0000 7f43e9200640 1 -- 172.21.15.86:0/3976867378 learned_addr learned my addr 172.21.15.86:0/3976867378 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43dbe00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43e40c4d10 0x7f43e40b0630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43e9200640 1 -- 172.21.15.86:0/3976867378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43e40c5880 msgr2=0x7f43e4143ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43e9200640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43e40c5880 0x7f43e4143ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43e9200640 1 -- 172.21.15.86:0/3976867378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43e40c4d10 msgr2=0x7f43e40b0630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43e9200640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43e40c4d10 0x7f43e40b0630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43e9200640 1 -- 172.21.15.86:0/3976867378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43e40b0f10 con 0x7f43e40c4930 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43e9200640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e41f3ec0 secure :-1 s=READY pgs=982 cs=0 l=1 rev1=1 crypto rx=0x7f43d4009870 tx=0x7f43d40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.238+0000 7f43e1600640 1 -- 172.21.15.86:0/3976867378 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43d403ab50 con 0x7f43e40c4930 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.239+0000 7f43e1600640 1 -- 172.21.15.86:0/3976867378 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43d403ad10 con 0x7f43e40c4930 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.239+0000 7f43e1600640 1 -- 172.21.15.86:0/3976867378 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43d4004270 con 0x7f43e40c4930 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.239+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f43e40b11f0 con 0x7f43e40c4930 2024-03-06T07:31:45.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.239+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f43e40b17b0 con 0x7f43e40c4930 2024-03-06T07:31:45.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.240+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f43e404f330 con 0x7f43e40c4930 2024-03-06T07:31:45.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.241+0000 7f43e1600640 1 -- 172.21.15.86:0/3976867378 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f43d404b060 con 0x7f43e40c4930 2024-03-06T07:31:45.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.241+0000 7f43e1600640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43c006b4d0 0x7f43c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:45.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.242+0000 7f43e1600640 1 -- 172.21.15.86:0/3976867378 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f43d40ac170 con 0x7f43e40c4930 2024-03-06T07:31:45.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.242+0000 7f43dbe00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43c006b4d0 0x7f43c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:45.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.242+0000 7f43dbe00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43c006b4d0 0x7f43c006d990 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f43e40c84f0 tx=0x7f43cc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:45.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.250+0000 7f43e1600640 1 -- 172.21.15.86:0/3976867378 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f43d403f070 con 0x7f43e40c4930 2024-03-06T07:31:45.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.458+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f43e40c8f00 con 0x7f43e40c4930 2024-03-06T07:31:45.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.459+0000 7f43e1600640 1 -- 172.21.15.86:0/3976867378 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v83) v1 ==== 80+0+0 (secure 0 0 0) 0x7f43d404b350 con 0x7f43e40c4930 2024-03-06T07:31:45.462 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:31:45.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.461+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43c006b4d0 msgr2=0x7f43c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.461+0000 7f43ebc00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43c006b4d0 0x7f43c006d990 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f43e40c84f0 tx=0x7f43cc008040 comp rx=0 tx=0).stop 2024-03-06T07:31:45.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.461+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 msgr2=0x7f43e41f3ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.461+0000 7f43ebc00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e41f3ec0 secure :-1 s=READY pgs=982 cs=0 l=1 rev1=1 crypto rx=0x7f43d4009870 tx=0x7f43d40046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 shutdown_connections 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43c006b4d0 0x7f43c006d990 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43e40c5880 0x7f43e4143ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43e40c4d10 0x7f43e40b0630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 --2- 172.21.15.86:0/3976867378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43e40c4930 0x7f43e41f3ec0 unknown :-1 s=CLOSED pgs=982 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 >> 172.21.15.86:0/3976867378 conn(0x7f43e40cc4f0 msgr2=0x7f43e40c5170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 shutdown_connections 2024-03-06T07:31:45.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:45.462+0000 7f43ebc00640 1 -- 172.21.15.86:0/3976867378 wait complete. 2024-03-06T07:31:45.481 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:31:45.481 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:45.481 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:31:45.573 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:31:45.573 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:31:45.574 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:31:45.616 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:45.616 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:45.676 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:45.676 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:45.676 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:45.726 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:31:45.726 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:45.726 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:45.782 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:45.783 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:46.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.016+0000 7f5c35000640 1 Processor -- start 2024-03-06T07:31:46.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.016+0000 7f5c35000640 1 -- start start 2024-03-06T07:31:46.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.017+0000 7f5c35000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5c3015f9d0 0x7f5c3015fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c35000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c3006cef0 con 0x7f5c30070b10 2024-03-06T07:31:46.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c35000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c3006d1d0 con 0x7f5c30144c00 2024-03-06T07:31:46.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c35000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5c3006d4b0 con 0x7f5c3015f9d0 2024-03-06T07:31:46.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c27e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5c3015f9d0 0x7f5c3015fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c27e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5c3015f9d0 0x7f5c3015fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40980/0 (socket says 172.21.15.86:40980) 2024-03-06T07:31:46.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c2e800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f5c30070b10 0x7f5c30070f20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44278/0 (socket says 172.21.15.86:44278) 2024-03-06T07:31:46.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c27e00640 1 -- 172.21.15.86:0/4206697963 learned_addr learned my addr 172.21.15.86:0/4206697963 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:46.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c27e00640 1 -- 172.21.15.86:0/4206697963 >> v1:172.21.15.194:6789/0 conn(0x7f5c30144c00 legacy=0x7f5c3006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:46.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c27e00640 1 -- 172.21.15.86:0/4206697963 >> v1:172.21.15.86:6789/0 conn(0x7f5c30070b10 legacy=0x7f5c30070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c27e00640 1 -- 172.21.15.86:0/4206697963 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c3006d790 con 0x7f5c3015f9d0 2024-03-06T07:31:46.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.018+0000 7f5c2d400640 1 -- 172.21.15.86:0/4206697963 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1073782120 0 0) 0x7f5c3006cef0 con 0x7f5c30070b10 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.019+0000 7f5c27e00640 1 --2- 172.21.15.86:0/4206697963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c3015f9d0 0x7f5c3015fe60 secure :-1 s=READY pgs=984 cs=0 l=1 rev1=1 crypto rx=0x7f5c18009870 tx=0x7f5c1802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cca2c6c1f2c61c79 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c2d400640 1 -- 172.21.15.86:0/4206697963 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c18034070 con 0x7f5c3015f9d0 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c2d400640 1 -- 172.21.15.86:0/4206697963 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c18030860 con 0x7f5c3015f9d0 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c2d400640 1 -- 172.21.15.86:0/4206697963 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c18030c30 con 0x7f5c3015f9d0 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c35000640 1 -- 172.21.15.86:0/4206697963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c3015f9d0 msgr2=0x7f5c3015fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c35000640 1 --2- 172.21.15.86:0/4206697963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c3015f9d0 0x7f5c3015fe60 secure :-1 s=READY pgs=984 cs=0 l=1 rev1=1 crypto rx=0x7f5c18009870 tx=0x7f5c1802b9a0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c35000640 1 -- 172.21.15.86:0/4206697963 shutdown_connections 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c35000640 1 --2- 172.21.15.86:0/4206697963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c3015f9d0 0x7f5c3015fe60 unknown :-1 s=CLOSED pgs=984 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c35000640 1 -- 172.21.15.86:0/4206697963 >> 172.21.15.86:0/4206697963 conn(0x7f5c300b1730 msgr2=0x7f5c300b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c35000640 1 -- 172.21.15.86:0/4206697963 shutdown_connections 2024-03-06T07:31:46.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.020+0000 7f5c35000640 1 -- 172.21.15.86:0/4206697963 wait complete. 2024-03-06T07:31:46.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.022+0000 7f5c35000640 1 Processor -- start 2024-03-06T07:31:46.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.022+0000 7f5c35000640 1 -- start start 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c35000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c30070b10 0x7f5c30155fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c35000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c30144c00 0x7f5c300a9450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c35000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c3015f9d0 0x7f5c3015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c35000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c3006ccd0 con 0x7f5c30070b10 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c35000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c3006cb50 con 0x7f5c30144c00 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c35000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c3006df90 con 0x7f5c3015f9d0 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c2de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c30070b10 0x7f5c30155fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c2de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c30070b10 0x7f5c30155fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40984/0 (socket says 172.21.15.86:40984) 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.023+0000 7f5c2de00640 1 -- 172.21.15.86:0/1476145214 learned_addr learned my addr 172.21.15.86:0/1476145214 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c2de00640 1 -- 172.21.15.86:0/1476145214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c3015f9d0 msgr2=0x7f5c3015ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c2de00640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c3015f9d0 0x7f5c3015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c2de00640 1 -- 172.21.15.86:0/1476145214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c30144c00 msgr2=0x7f5c300a9450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c2de00640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c30144c00 0x7f5c300a9450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c2de00640 1 -- 172.21.15.86:0/1476145214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c300a9b10 con 0x7f5c30070b10 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c2de00640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c30070b10 0x7f5c30155fa0 secure :-1 s=READY pgs=985 cs=0 l=1 rev1=1 crypto rx=0x7f5c2000c460 tx=0x7f5c2000c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c26000640 1 -- 172.21.15.86:0/1476145214 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c20016020 con 0x7f5c30070b10 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c26000640 1 -- 172.21.15.86:0/1476145214 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c20012870 con 0x7f5c30070b10 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c300a9df0 con 0x7f5c30070b10 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c26000640 1 -- 172.21.15.86:0/1476145214 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c20012c60 con 0x7f5c30070b10 2024-03-06T07:31:46.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.024+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c300aa3b0 con 0x7f5c30070b10 2024-03-06T07:31:46.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.025+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c3004f330 con 0x7f5c30070b10 2024-03-06T07:31:46.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.026+0000 7f5c26000640 1 -- 172.21.15.86:0/1476145214 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5c20019020 con 0x7f5c30070b10 2024-03-06T07:31:46.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.027+0000 7f5c26000640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0c06b4d0 0x7f5c0c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.027+0000 7f5c26000640 1 -- 172.21.15.86:0/1476145214 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5c20089ed0 con 0x7f5c30070b10 2024-03-06T07:31:46.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.027+0000 7f5c27e00640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0c06b4d0 0x7f5c0c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.028+0000 7f5c27e00640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0c06b4d0 0x7f5c0c06d990 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f5c18036000 tx=0x7f5c18030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:46.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.035+0000 7f5c26000640 1 -- 172.21.15.86:0/1476145214 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5c20020030 con 0x7f5c30070b10 2024-03-06T07:31:46.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.489+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5c30072890 con 0x7f5c30070b10 2024-03-06T07:31:46.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.490+0000 7f5c26000640 1 -- 172.21.15.86:0/1476145214 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f5c20058d40 con 0x7f5c30070b10 2024-03-06T07:31:46.493 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:46.493 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:31:32.095887+0000","modified":"2024-03-06T07:31:35.343932+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6609},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6609":{"gid":6609,"name":"c","rank":0,"incarnation":47,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/255958031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":255958031},{"type":"v1","addr":"172.21.15.86:6837","nonce":255958031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6609,"qdb_cluster":[6609]},"id":6}]} 2024-03-06T07:31:46.493 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 50 2024-03-06T07:31:46.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.492+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0c06b4d0 msgr2=0x7f5c0c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.492+0000 7f5c35000640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0c06b4d0 0x7f5c0c06d990 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f5c18036000 tx=0x7f5c18030000 comp rx=0 tx=0).stop 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c30070b10 msgr2=0x7f5c30155fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c30070b10 0x7f5c30155fa0 secure :-1 s=READY pgs=985 cs=0 l=1 rev1=1 crypto rx=0x7f5c2000c460 tx=0x7f5c2000c920 comp rx=0 tx=0).stop 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 shutdown_connections 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0c06b4d0 0x7f5c0c06d990 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c3015f9d0 0x7f5c3015ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c30144c00 0x7f5c300a9450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 --2- 172.21.15.86:0/1476145214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c30070b10 0x7f5c30155fa0 unknown :-1 s=CLOSED pgs=985 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 >> 172.21.15.86:0/1476145214 conn(0x7f5c300b1730 msgr2=0x7f5c3006e530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 shutdown_connections 2024-03-06T07:31:46.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.493+0000 7f5c35000640 1 -- 172.21.15.86:0/1476145214 wait complete. 2024-03-06T07:31:46.514 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:46.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.703+0000 7f2264a00640 1 Processor -- start 2024-03-06T07:31:46.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.703+0000 7f2264a00640 1 -- start start 2024-03-06T07:31:46.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.705+0000 7f2264a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 0x7f2260071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.705+0000 7f2264a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f22600ca5d0 0x7f22600724b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.705+0000 7f2264a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f22600cabf0 con 0x7f22600cf3e0 2024-03-06T07:31:46.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.705+0000 7f2264a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f22600ce900 con 0x7f22600ca5d0 2024-03-06T07:31:46.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.705+0000 7f2264a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f22600cebe0 con 0x7f2260144660 2024-03-06T07:31:46.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.705+0000 7f2257e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f22600cf3e0 0x7f22600c9ec0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44286/0 (socket says 172.21.15.86:44286) 2024-03-06T07:31:46.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.705+0000 7f2257e00640 1 -- 172.21.15.86:0/2337291211 learned_addr learned my addr 172.21.15.86:0/2337291211 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:46.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225e800640 1 --2- 172.21.15.86:0/2337291211 >> v2:172.21.15.194:3300/0 conn(0x7f22600ca5d0 0x7f22600724b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225de00640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 0x7f2260071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3029286810 0 0) 0x7f22600cabf0 con 0x7f22600cf3e0 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2244003520 con 0x7f22600cf3e0 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3096562130 0 0) 0x7f2244003520 con 0x7f22600cf3e0 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 msgr2=0x7f2260071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 0x7f2260071f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 >> v2:172.21.15.194:3300/0 conn(0x7f22600ca5d0 msgr2=0x7f22600724b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 --2- 172.21.15.86:0/2337291211 >> v2:172.21.15.194:3300/0 conn(0x7f22600ca5d0 0x7f22600724b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22600ceec0 con 0x7f22600cf3e0 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f2248001670 con 0x7f22600cf3e0 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 >> v1:172.21.15.86:6789/0 conn(0x7f22600cf3e0 legacy=0x7f22600c9ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 0x7f22440039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2244003f80 0x7f2244004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2244004d40 0x7f22440051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22600ce900 con 0x7f2244004d40 2024-03-06T07:31:46.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22600cebe0 con 0x7f2244003f80 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.706+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f224400b9c0 con 0x7f2260144660 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225e800640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2244004d40 0x7f22440051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f2257e00640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2244003f80 0x7f2244004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225de00640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 0x7f22440039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225e800640 1 -- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 msgr2=0x7f22440039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225e800640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 0x7f22440039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225e800640 1 -- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2244003f80 msgr2=0x7f2244004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225e800640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2244003f80 0x7f2244004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225e800640 1 -- 172.21.15.86:0/2337291211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2244003520 con 0x7f2244004d40 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.707+0000 7f225de00640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260144660 0x7f22440039b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:46.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.708+0000 7f225e800640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2244004d40 0x7f22440051c0 secure :-1 s=READY pgs=987 cs=0 l=1 rev1=1 crypto rx=0x7f225800e170 tx=0x7f2244005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2097950bfd169c8d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:46.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2258039070 con 0x7f2244004d40 2024-03-06T07:31:46.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f225802cc20 con 0x7f2244004d40 2024-03-06T07:31:46.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f225d400640 1 -- 172.21.15.86:0/2337291211 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2258009c10 con 0x7f2244004d40 2024-03-06T07:31:46.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f2264a00640 1 -- 172.21.15.86:0/2337291211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2244004d40 msgr2=0x7f22440051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f2264a00640 1 --2- 172.21.15.86:0/2337291211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2244004d40 0x7f22440051c0 secure :-1 s=READY pgs=987 cs=0 l=1 rev1=1 crypto rx=0x7f225800e170 tx=0x7f2244005830 comp rx=0 tx=0).stop 2024-03-06T07:31:46.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f225e800640 1 -- 172.21.15.86:0/2337291211 reap_dead start 2024-03-06T07:31:46.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f2264a00640 1 -- 172.21.15.86:0/2337291211 shutdown_connections 2024-03-06T07:31:46.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f2264a00640 1 -- 172.21.15.86:0/2337291211 >> 172.21.15.86:0/2337291211 conn(0x7f22600d0b90 msgr2=0x7f22600d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:46.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f2264a00640 1 -- 172.21.15.86:0/2337291211 shutdown_connections 2024-03-06T07:31:46.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.709+0000 7f2264a00640 1 -- 172.21.15.86:0/2337291211 wait complete. 2024-03-06T07:31:46.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.711+0000 7f2264a00640 1 Processor -- start 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.711+0000 7f2264a00640 1 -- start start 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.712+0000 7f2264a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f226015d1e0 0x7f2260139a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.712+0000 7f2264a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f226015d900 0x7f22601424f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.712+0000 7f2264a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260142a30 0x7f226013c690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.712+0000 7f2264a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22600720f0 con 0x7f226015d900 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.712+0000 7f2264a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2260071f70 con 0x7f226015d1e0 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.712+0000 7f2264a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2260072270 con 0x7f2260142a30 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.712+0000 7f2257e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f226015d900 0x7f22601424f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f226015d900 0x7f22601424f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40992/0 (socket says 172.21.15.86:40992) 2024-03-06T07:31:46.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 -- 172.21.15.86:0/4236555495 learned_addr learned my addr 172.21.15.86:0/4236555495 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 -- 172.21.15.86:0/4236555495 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260142a30 msgr2=0x7f226013c690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260142a30 0x7f226013c690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 -- 172.21.15.86:0/4236555495 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f226015d1e0 msgr2=0x7f2260139a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f225de00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f226015d1e0 0x7f2260139a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f226015d1e0 0x7f2260139a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 -- 172.21.15.86:0/4236555495 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f226013b960 con 0x7f226015d900 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.713+0000 7f2257e00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f226015d900 0x7f22601424f0 secure :-1 s=READY pgs=988 cs=0 l=1 rev1=1 crypto rx=0x7f224800ea70 tx=0x7f224800ef30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.714+0000 7f2256000640 1 -- 172.21.15.86:0/4236555495 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f224800c870 con 0x7f226015d900 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.714+0000 7f2256000640 1 -- 172.21.15.86:0/4236555495 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2248018070 con 0x7f226015d900 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.714+0000 7f2256000640 1 -- 172.21.15.86:0/4236555495 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f224800cc50 con 0x7f226015d900 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.714+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f226013bc40 con 0x7f226015d900 2024-03-06T07:31:46.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.714+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f226013c200 con 0x7f226015d900 2024-03-06T07:31:46.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.715+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2260142e50 con 0x7f226015d900 2024-03-06T07:31:46.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.716+0000 7f2256000640 1 -- 172.21.15.86:0/4236555495 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2248013420 con 0x7f226015d900 2024-03-06T07:31:46.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.717+0000 7f2256000640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f223c06b500 0x7f223c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:46.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.717+0000 7f2256000640 1 -- 172.21.15.86:0/4236555495 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2248089de0 con 0x7f226015d900 2024-03-06T07:31:46.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.718+0000 7f225de00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f223c06b500 0x7f223c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:46.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.718+0000 7f225de00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f223c06b500 0x7f223c06d9c0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f22600cc780 tx=0x7f2250008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:46.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.721+0000 7f2256000640 1 -- 172.21.15.86:0/4236555495 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2248058bc0 con 0x7f226015d900 2024-03-06T07:31:46.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.917+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f226004f330 con 0x7f226015d900 2024-03-06T07:31:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.919+0000 7f2256000640 1 -- 172.21.15.86:0/4236555495 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+16672 (secure 0 0 0) 0x7f22480584c0 con 0x7f226015d900 2024-03-06T07:31:46.922 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:46.922 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":83,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:33.657039+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:31:29.878719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-06T07:31:31.115332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f223c06b500 msgr2=0x7f223c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f223c06b500 0x7f223c06d9c0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f22600cc780 tx=0x7f2250008040 comp rx=0 tx=0).stop 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f226015d900 msgr2=0x7f22601424f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f226015d900 0x7f22601424f0 secure :-1 s=READY pgs=988 cs=0 l=1 rev1=1 crypto rx=0x7f224800ea70 tx=0x7f224800ef30 comp rx=0 tx=0).stop 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 shutdown_connections 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f223c06b500 0x7f223c06d9c0 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2260142a30 0x7f226013c690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f226015d900 0x7f22601424f0 unknown :-1 s=CLOSED pgs=988 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 --2- 172.21.15.86:0/4236555495 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f226015d1e0 0x7f2260139a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 >> 172.21.15.86:0/4236555495 conn(0x7f22600d0b90 msgr2=0x7f226006dbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 shutdown_connections 2024-03-06T07:31:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:46.922+0000 7f2264a00640 1 -- 172.21.15.86:0/4236555495 wait complete. 2024-03-06T07:31:46.943 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 12, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:31:29.878719+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '82', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2, 'score_stable': 2, 'optimal_score': 1, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 13, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:31:31.115332+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:31:46.943 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:31:47.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.135+0000 7fd959600640 1 Processor -- start 2024-03-06T07:31:47.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.136+0000 7fd959600640 1 -- start start 2024-03-06T07:31:47.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd959600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd9540d07f0 0x7fd9540d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd959600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd95415e450 0x7fd9540b4480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd959600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd9540b3730 con 0x7fd9540d1bf0 2024-03-06T07:31:47.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd959600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd9540d4930 con 0x7fd9540d07f0 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd959600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd954067b20 con 0x7fd95415e450 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd952a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd9540d07f0 0x7fd9540d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd952a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd9540d07f0 0x7fd9540d0bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41002/0 (socket says 172.21.15.86:41002) 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.137+0000 7fd952a00640 1 -- 172.21.15.86:0/576757832 learned_addr learned my addr 172.21.15.86:0/576757832 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.138+0000 7fd952a00640 1 -- 172.21.15.86:0/576757832 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd95415e450 msgr2=0x7fd9540b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.138+0000 7fd952a00640 1 --2- 172.21.15.86:0/576757832 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd95415e450 0x7fd9540b4480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.138+0000 7fd952a00640 1 -- 172.21.15.86:0/576757832 >> v1:172.21.15.194:6789/0 conn(0x7fd9540d1bf0 legacy=0x7fd9540b49c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:47.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.138+0000 7fd952a00640 1 -- 172.21.15.86:0/576757832 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9540d1110 con 0x7fd9540d07f0 2024-03-06T07:31:47.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.139+0000 7fd952a00640 1 --2- 172.21.15.86:0/576757832 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d07f0 0x7fd9540d0bd0 secure :-1 s=READY pgs=989 cs=0 l=1 rev1=1 crypto rx=0x7fd944009e50 tx=0x7fd94402be70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=882f7c15aa3b35f3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:47.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.139+0000 7fd952000640 1 -- 172.21.15.86:0/576757832 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd94402ca10 con 0x7fd9540d07f0 2024-03-06T07:31:47.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.139+0000 7fd952000640 1 -- 172.21.15.86:0/576757832 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd94402cbb0 con 0x7fd9540d07f0 2024-03-06T07:31:47.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.139+0000 7fd952000640 1 -- 172.21.15.86:0/576757832 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9440343f0 con 0x7fd9540d07f0 2024-03-06T07:31:47.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 -- 172.21.15.86:0/576757832 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d07f0 msgr2=0x7fd9540d0bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 --2- 172.21.15.86:0/576757832 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d07f0 0x7fd9540d0bd0 secure :-1 s=READY pgs=989 cs=0 l=1 rev1=1 crypto rx=0x7fd944009e50 tx=0x7fd94402be70 comp rx=0 tx=0).stop 2024-03-06T07:31:47.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 -- 172.21.15.86:0/576757832 shutdown_connections 2024-03-06T07:31:47.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 --2- 172.21.15.86:0/576757832 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd95415e450 0x7fd9540b4480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 --2- 172.21.15.86:0/576757832 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d07f0 0x7fd9540d0bd0 unknown :-1 s=CLOSED pgs=989 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 -- 172.21.15.86:0/576757832 >> 172.21.15.86:0/576757832 conn(0x7fd9540d3e10 msgr2=0x7fd9540d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:47.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 -- 172.21.15.86:0/576757832 shutdown_connections 2024-03-06T07:31:47.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.140+0000 7fd959600640 1 -- 172.21.15.86:0/576757832 wait complete. 2024-03-06T07:31:47.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.143+0000 7fd959600640 1 Processor -- start 2024-03-06T07:31:47.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.143+0000 7fd959600640 1 -- start start 2024-03-06T07:31:47.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.144+0000 7fd959600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9540d07f0 0x7fd9540b9220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.144+0000 7fd959600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d1bf0 0x7fd95414f2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.144+0000 7fd959600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd95415e450 0x7fd9540c7520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.144+0000 7fd959600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9540b3c80 con 0x7fd9540d1bf0 2024-03-06T07:31:47.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.144+0000 7fd959600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9540b39a0 con 0x7fd95415e450 2024-03-06T07:31:47.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.144+0000 7fd959600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd9540b3f60 con 0x7fd9540d07f0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d1bf0 0x7fd95414f2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d1bf0 0x7fd95414f2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41014/0 (socket says 172.21.15.86:41014) 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 -- 172.21.15.86:0/1548901626 learned_addr learned my addr 172.21.15.86:0/1548901626 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd952a00640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9540d07f0 0x7fd9540b9220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd953400640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd95415e450 0x7fd9540c7520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 -- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9540d07f0 msgr2=0x7fd9540b9220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9540d07f0 0x7fd9540b9220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 -- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd95415e450 msgr2=0x7fd9540c7520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd95415e450 0x7fd9540c7520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 -- 172.21.15.86:0/1548901626 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9540c7a60 con 0x7fd9540d1bf0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd952a00640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9540d07f0 0x7fd9540b9220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.145+0000 7fd94a000640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d1bf0 0x7fd95414f2f0 secure :-1 s=READY pgs=990 cs=0 l=1 rev1=1 crypto rx=0x7fd93c00b410 tx=0x7fd93c009f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.146+0000 7fd94b400640 1 -- 172.21.15.86:0/1548901626 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd93c015020 con 0x7fd9540d1bf0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.146+0000 7fd94b400640 1 -- 172.21.15.86:0/1548901626 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd93c011830 con 0x7fd9540d1bf0 2024-03-06T07:31:47.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.146+0000 7fd94b400640 1 -- 172.21.15.86:0/1548901626 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd93c011c20 con 0x7fd9540d1bf0 2024-03-06T07:31:47.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.146+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd9540c7d30 con 0x7fd9540d1bf0 2024-03-06T07:31:47.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.146+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd9540c82f0 con 0x7fd9540d1bf0 2024-03-06T07:31:47.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.147+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd95404f330 con 0x7fd9540d1bf0 2024-03-06T07:31:47.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.148+0000 7fd94b400640 1 -- 172.21.15.86:0/1548901626 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd93c018020 con 0x7fd9540d1bf0 2024-03-06T07:31:47.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.148+0000 7fd94b400640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd93006b4d0 0x7fd93006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.148+0000 7fd94b400640 1 -- 172.21.15.86:0/1548901626 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd93c08a160 con 0x7fd9540d1bf0 2024-03-06T07:31:47.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.149+0000 7fd952a00640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd93006b4d0 0x7fd93006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.149+0000 7fd952a00640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd93006b4d0 0x7fd93006d990 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fd944035000 tx=0x7fd944030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:47.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.153+0000 7fd94b400640 1 -- 172.21.15.86:0/1548901626 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd93c01f020 con 0x7fd9540d1bf0 2024-03-06T07:31:47.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.347+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd9540d21d0 con 0x7fd9540d1bf0 2024-03-06T07:31:47.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.347+0000 7fd94b400640 1 -- 172.21.15.86:0/1548901626 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v83) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd93c0172c0 con 0x7fd9540d1bf0 2024-03-06T07:31:47.350 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd93006b4d0 msgr2=0x7fd93006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd93006b4d0 0x7fd93006d990 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fd944035000 tx=0x7fd944030000 comp rx=0 tx=0).stop 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d1bf0 msgr2=0x7fd95414f2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d1bf0 0x7fd95414f2f0 secure :-1 s=READY pgs=990 cs=0 l=1 rev1=1 crypto rx=0x7fd93c00b410 tx=0x7fd93c009f90 comp rx=0 tx=0).stop 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 shutdown_connections 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd93006b4d0 0x7fd93006d990 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd95415e450 0x7fd9540c7520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9540d1bf0 0x7fd95414f2f0 unknown :-1 s=CLOSED pgs=990 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 --2- 172.21.15.86:0/1548901626 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9540d07f0 0x7fd9540b9220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 >> 172.21.15.86:0/1548901626 conn(0x7fd9540d3e10 msgr2=0x7fd9540b09e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 shutdown_connections 2024-03-06T07:31:47.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.350+0000 7fd959600640 1 -- 172.21.15.86:0/1548901626 wait complete. 2024-03-06T07:31:47.369 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:31:47.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.546+0000 7f3031a00640 1 Processor -- start 2024-03-06T07:31:47.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.546+0000 7f3031a00640 1 -- start start 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.547+0000 7f3031a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f302c0b3f70 0x7f302c160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.547+0000 7f3031a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 0x7f302c0ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.547+0000 7f3031a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f302c0b46c0 con 0x7f302c15fc10 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.547+0000 7f3031a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f302c071480 con 0x7f302c0b3f70 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.547+0000 7f3031a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f302c0b49a0 con 0x7f302c070ff0 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f3022000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f302c0b3f70 0x7f302c160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f3022000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f302c0b3f70 0x7f302c160700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41020/0 (socket says 172.21.15.86:41020) 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f3022000640 1 -- 172.21.15.86:0/815056775 learned_addr learned my addr 172.21.15.86:0/815056775 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f302aa00640 1 --1- 172.21.15.86:0/815056775 >> v1:172.21.15.86:6789/0 conn(0x7f302c15fc10 0x7f302c15fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44298/0 (socket says 172.21.15.86:44298) 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f302b400640 1 --2- 172.21.15.86:0/815056775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 0x7f302c0ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f3022000640 1 -- 172.21.15.86:0/815056775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 msgr2=0x7f302c0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:47.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f3022000640 1 --2- 172.21.15.86:0/815056775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 0x7f302c0ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f3022000640 1 -- 172.21.15.86:0/815056775 >> v1:172.21.15.86:6789/0 conn(0x7f302c15fc10 legacy=0x7f302c15fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.548+0000 7f3022000640 1 -- 172.21.15.86:0/815056775 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f302c072c90 con 0x7f302c0b3f70 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.549+0000 7f3022000640 1 --2- 172.21.15.86:0/815056775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c0b3f70 0x7f302c160700 secure :-1 s=READY pgs=992 cs=0 l=1 rev1=1 crypto rx=0x7f3014009870 tx=0x7f301402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1f077f8d4e70a8e2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f302a000640 1 -- 172.21.15.86:0/815056775 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3014034070 con 0x7f302c0b3f70 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f302a000640 1 -- 172.21.15.86:0/815056775 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3014030840 con 0x7f302c0b3f70 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f302a000640 1 -- 172.21.15.86:0/815056775 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3014030c10 con 0x7f302c0b3f70 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 -- 172.21.15.86:0/815056775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c0b3f70 msgr2=0x7f302c160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 --2- 172.21.15.86:0/815056775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c0b3f70 0x7f302c160700 secure :-1 s=READY pgs=992 cs=0 l=1 rev1=1 crypto rx=0x7f3014009870 tx=0x7f301402b980 comp rx=0 tx=0).stop 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 -- 172.21.15.86:0/815056775 shutdown_connections 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 --2- 172.21.15.86:0/815056775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 0x7f302c0ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 --2- 172.21.15.86:0/815056775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c0b3f70 0x7f302c160700 unknown :-1 s=CLOSED pgs=992 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 -- 172.21.15.86:0/815056775 >> 172.21.15.86:0/815056775 conn(0x7f302c0b1940 msgr2=0x7f302c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:47.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 -- 172.21.15.86:0/815056775 shutdown_connections 2024-03-06T07:31:47.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.550+0000 7f3031a00640 1 -- 172.21.15.86:0/815056775 wait complete. 2024-03-06T07:31:47.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.552+0000 7f3031a00640 1 Processor -- start 2024-03-06T07:31:47.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.552+0000 7f3031a00640 1 -- start start 2024-03-06T07:31:47.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f3031a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 0x7f302c09c280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f3031a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f302c0b3f70 0x7f302c14d3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f3031a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c15fc10 0x7f302c09cb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f3031a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f302c072a30 con 0x7f302c15fc10 2024-03-06T07:31:47.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f3031a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f302c0728b0 con 0x7f302c0b3f70 2024-03-06T07:31:47.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f3031a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f302c15c2e0 con 0x7f302c070ff0 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f302b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c15fc10 0x7f302c09cb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f302b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c15fc10 0x7f302c09cb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41030/0 (socket says 172.21.15.86:41030) 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.553+0000 7f302b400640 1 -- 172.21.15.86:0/3323204686 learned_addr learned my addr 172.21.15.86:0/3323204686 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f3022000640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f302c0b3f70 0x7f302c14d3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f302b400640 1 -- 172.21.15.86:0/3323204686 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 msgr2=0x7f302c09c280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f302b400640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 0x7f302c09c280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f302b400640 1 -- 172.21.15.86:0/3323204686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f302c0b3f70 msgr2=0x7f302c14d3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f302b400640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f302c0b3f70 0x7f302c14d3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f302b400640 1 -- 172.21.15.86:0/3323204686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f302c14d8e0 con 0x7f302c15fc10 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f302b400640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c15fc10 0x7f302c09cb80 secure :-1 s=READY pgs=993 cs=0 l=1 rev1=1 crypto rx=0x7f3024007e40 tx=0x7f302400e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f3023400640 1 -- 172.21.15.86:0/3323204686 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3024014620 con 0x7f302c15fc10 2024-03-06T07:31:47.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f3023400640 1 -- 172.21.15.86:0/3323204686 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3024019070 con 0x7f302c15fc10 2024-03-06T07:31:47.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f302c09d6f0 con 0x7f302c15fc10 2024-03-06T07:31:47.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f3023400640 1 -- 172.21.15.86:0/3323204686 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3024014a20 con 0x7f302c15fc10 2024-03-06T07:31:47.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.554+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f302c09b7a0 con 0x7f302c15fc10 2024-03-06T07:31:47.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.555+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f302c04f330 con 0x7f302c15fc10 2024-03-06T07:31:47.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.556+0000 7f3023400640 1 -- 172.21.15.86:0/3323204686 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3024014bc0 con 0x7f302c15fc10 2024-03-06T07:31:47.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.557+0000 7f3023400640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f300806b4d0 0x7f300806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:47.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.557+0000 7f3023400640 1 -- 172.21.15.86:0/3323204686 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f302408aa90 con 0x7f302c15fc10 2024-03-06T07:31:47.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.557+0000 7f302aa00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f300806b4d0 0x7f300806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:47.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.558+0000 7f302aa00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f300806b4d0 0x7f300806d990 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f301c000f60 tx=0x7f301c007a00 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:47.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.561+0000 7f3023400640 1 -- 172.21.15.86:0/3323204686 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3024059900 con 0x7f302c15fc10 2024-03-06T07:31:47.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.798+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f302c06ef30 con 0x7f302c15fc10 2024-03-06T07:31:47.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.799+0000 7f3023400640 1 -- 172.21.15.86:0/3323204686 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v83) v1 ==== 104+0+11 (secure 0 0 0) 0x7f3024059200 con 0x7f302c15fc10 2024-03-06T07:31:47.802 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f300806b4d0 msgr2=0x7f300806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f300806b4d0 0x7f300806d990 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f301c000f60 tx=0x7f301c007a00 comp rx=0 tx=0).stop 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c15fc10 msgr2=0x7f302c09cb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c15fc10 0x7f302c09cb80 secure :-1 s=READY pgs=993 cs=0 l=1 rev1=1 crypto rx=0x7f3024007e40 tx=0x7f302400e4b0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 shutdown_connections 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f300806b4d0 0x7f300806d990 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f302c15fc10 0x7f302c09cb80 unknown :-1 s=CLOSED pgs=993 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f302c0b3f70 0x7f302c14d3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 --2- 172.21.15.86:0/3323204686 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f302c070ff0 0x7f302c09c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:47.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.802+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 >> 172.21.15.86:0/3323204686 conn(0x7f302c0b1940 msgr2=0x7f302c0b1d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:47.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.803+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 shutdown_connections 2024-03-06T07:31:47.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:47.803+0000 7f3031a00640 1 -- 172.21.15.86:0/3323204686 wait complete. 2024-03-06T07:31:47.824 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:31:48.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.021+0000 7f8794e00640 1 Processor -- start 2024-03-06T07:31:48.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.021+0000 7f8794e00640 1 -- start start 2024-03-06T07:31:48.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.023+0000 7f8794e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f879015fc10 0x7f8790070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.023+0000 7f8794e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87900b3f70 0x7f87900ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.023+0000 7f8794e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f87900b46c0 con 0x7f879015fff0 2024-03-06T07:31:48.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.023+0000 7f8794e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f87900b49a0 con 0x7f879015fc10 2024-03-06T07:31:48.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.023+0000 7f8794e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8790072c90 con 0x7f87900b3f70 2024-03-06T07:31:48.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878de00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f879015fc10 0x7f8790070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878e800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87900b3f70 0x7f87900ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878de00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f879015fc10 0x7f8790070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:53100/0 (socket says 172.21.15.86:53100) 2024-03-06T07:31:48.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878de00640 1 -- 172.21.15.86:0/1890218523 learned_addr learned my addr 172.21.15.86:0/1890218523 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:48.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878de00640 1 -- 172.21.15.86:0/1890218523 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87900b3f70 msgr2=0x7f87900ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:48.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878de00640 1 --2- 172.21.15.86:0/1890218523 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87900b3f70 0x7f87900ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878de00640 1 -- 172.21.15.86:0/1890218523 >> v1:172.21.15.194:6789/0 conn(0x7f879015fff0 legacy=0x7f8790160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.024+0000 7f878de00640 1 -- 172.21.15.86:0/1890218523 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f879009dd80 con 0x7f879015fc10 2024-03-06T07:31:48.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.026+0000 7f878de00640 1 --2- 172.21.15.86:0/1890218523 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fc10 0x7f8790070ff0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f8780009870 tx=0x7f878002b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=becfb6acf28825ec server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.026+0000 7f878d400640 1 -- 172.21.15.86:0/1890218523 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8780034070 con 0x7f879015fc10 2024-03-06T07:31:48.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.026+0000 7f878d400640 1 -- 172.21.15.86:0/1890218523 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8780030840 con 0x7f879015fc10 2024-03-06T07:31:48.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.026+0000 7f878d400640 1 -- 172.21.15.86:0/1890218523 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8780030c10 con 0x7f879015fc10 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 -- 172.21.15.86:0/1890218523 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fc10 msgr2=0x7f8790070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 --2- 172.21.15.86:0/1890218523 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fc10 0x7f8790070ff0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f8780009870 tx=0x7f878002b980 comp rx=0 tx=0).stop 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 -- 172.21.15.86:0/1890218523 shutdown_connections 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 --2- 172.21.15.86:0/1890218523 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f87900b3f70 0x7f87900ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 --2- 172.21.15.86:0/1890218523 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fc10 0x7f8790070ff0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 -- 172.21.15.86:0/1890218523 >> 172.21.15.86:0/1890218523 conn(0x7f87900b1940 msgr2=0x7f87900b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 -- 172.21.15.86:0/1890218523 shutdown_connections 2024-03-06T07:31:48.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.027+0000 7f8794e00640 1 -- 172.21.15.86:0/1890218523 wait complete. 2024-03-06T07:31:48.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.029+0000 7f8794e00640 1 Processor -- start 2024-03-06T07:31:48.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.029+0000 7f8794e00640 1 -- start start 2024-03-06T07:31:48.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f8794e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87900b3f70 0x7f8790151fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f8794e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f879015fc10 0x7f879009c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f8794e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fff0 0x7f879009c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f8794e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f879015c460 con 0x7f87900b3f70 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f8794e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f879015c2e0 con 0x7f879015fff0 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f8794e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f87900ce9e0 con 0x7f879015fc10 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f878de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87900b3f70 0x7f8790151fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f878de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87900b3f70 0x7f8790151fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41044/0 (socket says 172.21.15.86:41044) 2024-03-06T07:31:48.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.030+0000 7f878de00640 1 -- 172.21.15.86:0/3916191692 learned_addr learned my addr 172.21.15.86:0/3916191692 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:48.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f878de00640 1 -- 172.21.15.86:0/3916191692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f879015fc10 msgr2=0x7f879009c380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f878de00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f879015fc10 0x7f879009c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f878de00640 1 -- 172.21.15.86:0/3916191692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fff0 msgr2=0x7f879009c8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f878de00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fff0 0x7f879009c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f878de00640 1 -- 172.21.15.86:0/3916191692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87901524f0 con 0x7f87900b3f70 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f878de00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87900b3f70 0x7f8790151fb0 secure :-1 s=READY pgs=994 cs=0 l=1 rev1=1 crypto rx=0x7f8780000c00 tx=0x7f8780004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f8786000640 1 -- 172.21.15.86:0/3916191692 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8780048070 con 0x7f87900b3f70 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f8786000640 1 -- 172.21.15.86:0/3916191692 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8780030640 con 0x7f87900b3f70 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f879015c780 con 0x7f87900b3f70 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.031+0000 7f8786000640 1 -- 172.21.15.86:0/3916191692 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8780034040 con 0x7f87900b3f70 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.032+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f879015cd40 con 0x7f87900b3f70 2024-03-06T07:31:48.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.033+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f879015be20 con 0x7f87900b3f70 2024-03-06T07:31:48.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.034+0000 7f8786000640 1 -- 172.21.15.86:0/3916191692 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f878004b070 con 0x7f87900b3f70 2024-03-06T07:31:48.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.034+0000 7f8786000640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f876c06b500 0x7f876c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.035+0000 7f8786000640 1 -- 172.21.15.86:0/3916191692 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f87800ac270 con 0x7f87900b3f70 2024-03-06T07:31:48.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.035+0000 7f8787e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f876c06b500 0x7f876c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.036+0000 7f8787e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f876c06b500 0x7f876c06d9c0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f8778000f60 tx=0x7f87780079d0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.039+0000 7f8786000640 1 -- 172.21.15.86:0/3916191692 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f878007b0e0 con 0x7f87900b3f70 2024-03-06T07:31:48.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.228+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f879004f330 con 0x7f87900b3f70 2024-03-06T07:31:48.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.229+0000 7f8786000640 1 -- 172.21.15.86:0/3916191692 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v83) v1 ==== 100+0+11 (secure 0 0 0) 0x7f8780030810 con 0x7f87900b3f70 2024-03-06T07:31:48.231 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:31:48.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f876c06b500 msgr2=0x7f876c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f876c06b500 0x7f876c06d9c0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f8778000f60 tx=0x7f87780079d0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87900b3f70 msgr2=0x7f8790151fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87900b3f70 0x7f8790151fb0 secure :-1 s=READY pgs=994 cs=0 l=1 rev1=1 crypto rx=0x7f8780000c00 tx=0x7f8780004730 comp rx=0 tx=0).stop 2024-03-06T07:31:48.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 shutdown_connections 2024-03-06T07:31:48.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f876c06b500 0x7f876c06d9c0 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f879015fff0 0x7f879009c8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f879015fc10 0x7f879009c380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 --2- 172.21.15.86:0/3916191692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f87900b3f70 0x7f8790151fb0 unknown :-1 s=CLOSED pgs=994 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 >> 172.21.15.86:0/3916191692 conn(0x7f87900b1940 msgr2=0x7f87900b1d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:48.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 shutdown_connections 2024-03-06T07:31:48.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.231+0000 7f8794e00640 1 -- 172.21.15.86:0/3916191692 wait complete. 2024-03-06T07:31:48.250 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:48.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.448+0000 7f2149000640 1 Processor -- start 2024-03-06T07:31:48.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.448+0000 7f2149000640 1 -- start start 2024-03-06T07:31:48.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.450+0000 7f2149000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441561f0 0x7f21440ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.450+0000 7f2149000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f214415c240 con 0x7f21440cc9d0 2024-03-06T07:31:48.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.450+0000 7f2149000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f214415c500 con 0x7f21440aebf0 2024-03-06T07:31:48.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.450+0000 7f2149000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f214415ce30 con 0x7f21441561f0 2024-03-06T07:31:48.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.450+0000 7f2143e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f21440cc9d0 0x7f214415f830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44308/0 (socket says 172.21.15.86:44308) 2024-03-06T07:31:48.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.450+0000 7f2143e00640 1 -- 172.21.15.86:0/1031189316 learned_addr learned my addr 172.21.15.86:0/1031189316 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:48.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.450+0000 7f2143400640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441561f0 0x7f21440ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4148055837 0 0) 0x7f214415c240 con 0x7f21440cc9d0 2024-03-06T07:31:48.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2128003520 con 0x7f21440cc9d0 2024-03-06T07:31:48.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1557075099 0 0) 0x7f2128003520 con 0x7f21440cc9d0 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441561f0 msgr2=0x7f21440ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441561f0 0x7f21440ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 >> v1:172.21.15.194:6789/0 conn(0x7f21440aebf0 legacy=0x7f21440cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f214415c7e0 con 0x7f21440cc9d0 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f213c0019f0 con 0x7f21440cc9d0 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 >> v1:172.21.15.86:6789/0 conn(0x7f21440cc9d0 legacy=0x7f214415f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21280039e0 0x7f2128003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2128004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441561f0 0x7f21280051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f214415ce30 con 0x7f2128004800 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f212800ba70 con 0x7f21280039e0 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f212800bc70 con 0x7f21441561f0 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.451+0000 7f213aa00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2128004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.452+0000 7f213aa00640 1 -- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441561f0 msgr2=0x7f21280051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:48.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.452+0000 7f213aa00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21441561f0 0x7f21280051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.452+0000 7f213aa00640 1 -- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21280039e0 msgr2=0x7f2128003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:48.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.452+0000 7f213aa00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21280039e0 0x7f2128003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.452+0000 7f213aa00640 1 -- 172.21.15.86:0/1031189316 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2128003520 con 0x7f2128004800 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.453+0000 7f213aa00640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2128004c80 secure :-1 s=READY pgs=996 cs=0 l=1 rev1=1 crypto rx=0x7f212c001720 tx=0x7f2128005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d854eacdfccfd98 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.453+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f212c001370 con 0x7f2128004800 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.453+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f212c0052d0 con 0x7f2128004800 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.453+0000 7f2142a00640 1 -- 172.21.15.86:0/1031189316 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f212c003830 con 0x7f2128004800 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.454+0000 7f2149000640 1 -- 172.21.15.86:0/1031189316 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 msgr2=0x7f2128004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.454+0000 7f2149000640 1 --2- 172.21.15.86:0/1031189316 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2128004c80 secure :-1 s=READY pgs=996 cs=0 l=1 rev1=1 crypto rx=0x7f212c001720 tx=0x7f2128005830 comp rx=0 tx=0).stop 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.454+0000 7f2143e00640 1 -- 172.21.15.86:0/1031189316 reap_dead start 2024-03-06T07:31:48.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.454+0000 7f2149000640 1 -- 172.21.15.86:0/1031189316 shutdown_connections 2024-03-06T07:31:48.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.454+0000 7f2149000640 1 -- 172.21.15.86:0/1031189316 >> 172.21.15.86:0/1031189316 conn(0x7f214406afc0 msgr2=0x7f214406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:48.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.454+0000 7f2149000640 1 -- 172.21.15.86:0/1031189316 shutdown_connections 2024-03-06T07:31:48.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.454+0000 7f2149000640 1 -- 172.21.15.86:0/1031189316 wait complete. 2024-03-06T07:31:48.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.456+0000 7f2149000640 1 Processor -- start 2024-03-06T07:31:48.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.457+0000 7f2149000640 1 -- start start 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2149000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2144157f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2149000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154db0 0x7f214415b380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2149000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21441e44d0 0x7f2144154200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2149000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21441565d0 con 0x7f2128004800 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2149000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f214406e360 con 0x7f21441e44d0 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2149000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2144156750 con 0x7f2144154db0 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2143400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2144157f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2143400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2144157f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41066/0 (socket says 172.21.15.86:41066) 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.458+0000 7f2143400640 1 -- 172.21.15.86:0/1450674745 learned_addr learned my addr 172.21.15.86:0/1450674745 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2143400640 1 -- 172.21.15.86:0/1450674745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154db0 msgr2=0x7f214415b380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2143400640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154db0 0x7f214415b380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2143400640 1 -- 172.21.15.86:0/1450674745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21441e44d0 msgr2=0x7f2144154200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2143400640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21441e44d0 0x7f2144154200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2143400640 1 -- 172.21.15.86:0/1450674745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21441e7320 con 0x7f2128004800 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2143400640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2144157f50 secure :-1 s=READY pgs=997 cs=0 l=1 rev1=1 crypto rx=0x7f213400e7a0 tx=0x7f213400ec60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f213be00640 1 -- 172.21.15.86:0/1450674745 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2134017070 con 0x7f2128004800 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f213be00640 1 -- 172.21.15.86:0/1450674745 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f213400c940 con 0x7f2128004800 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21441f76c0 con 0x7f2128004800 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f213be00640 1 -- 172.21.15.86:0/1450674745 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f213400cd10 con 0x7f2128004800 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.459+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21441f7c80 con 0x7f2128004800 2024-03-06T07:31:48.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.460+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f214404f330 con 0x7f2128004800 2024-03-06T07:31:48.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.462+0000 7f213be00640 1 -- 172.21.15.86:0/1450674745 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f213401a070 con 0x7f2128004800 2024-03-06T07:31:48.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.463+0000 7f213be00640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.463+0000 7f213be00640 1 -- 172.21.15.86:0/1450674745 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f213408a540 con 0x7f2128004800 2024-03-06T07:31:48.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.464+0000 7f213aa00640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.464+0000 7f213aa00640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f212c033000 tx=0x7f212c02f000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.471+0000 7f213be00640 1 -- 172.21.15.86:0/1450674745 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f21441e7320 con 0x7f2128004800 2024-03-06T07:31:48.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.720+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f21440b2b10 con 0x7f2128004800 2024-03-06T07:31:48.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.721+0000 7f213be00640 1 -- 172.21.15.86:0/1450674745 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f2134059320 con 0x7f2128004800 2024-03-06T07:31:48.724 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:48.724 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:31:32.095887+0000","modified":"2024-03-06T07:31:35.343932+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6609},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6609":{"gid":6609,"name":"c","rank":0,"incarnation":47,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/255958031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":255958031},{"type":"v1","addr":"172.21.15.86:6837","nonce":255958031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6609,"qdb_cluster":[6609]},"id":6}]} 2024-03-06T07:31:48.724 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 50 2024-03-06T07:31:48.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 msgr2=0x7f212006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f212c033000 tx=0x7f212c02f000 comp rx=0 tx=0).stop 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 msgr2=0x7f2144157f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2144157f50 secure :-1 s=READY pgs=997 cs=0 l=1 rev1=1 crypto rx=0x7f213400e7a0 tx=0x7f213400ec60 comp rx=0 tx=0).stop 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 shutdown_connections 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f212006b4d0 0x7f212006d990 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21441e44d0 0x7f2144154200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2144154db0 0x7f214415b380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 --2- 172.21.15.86:0/1450674745 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2128004800 0x7f2144157f50 unknown :-1 s=CLOSED pgs=997 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 >> 172.21.15.86:0/1450674745 conn(0x7f214406afc0 msgr2=0x7f2144155390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 shutdown_connections 2024-03-06T07:31:48.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.725+0000 7f2149000640 1 -- 172.21.15.86:0/1450674745 wait complete. 2024-03-06T07:31:48.753 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:48.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.945+0000 7f4e6a800640 1 Processor -- start 2024-03-06T07:31:48.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.945+0000 7f4e6a800640 1 -- start start 2024-03-06T07:31:48.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.946+0000 7f4e6a800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4e641561f0 0x7f4e640ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.946+0000 7f4e6a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e640cc9d0 0x7f4e6415f830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.946+0000 7f4e6a800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4e6415c240 con 0x7f4e640aebf0 2024-03-06T07:31:48.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.946+0000 7f4e6a800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4e6415ce30 con 0x7f4e641561f0 2024-03-06T07:31:48.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.946+0000 7f4e6a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4e6415c500 con 0x7f4e640cc9d0 2024-03-06T07:31:48.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e69400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4e641561f0 0x7f4e640ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e68a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4e640aebf0 0x7f4e640cc2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44312/0 (socket says 172.21.15.86:44312) 2024-03-06T07:31:48.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e68a00640 1 -- 172.21.15.86:0/1182972129 learned_addr learned my addr 172.21.15.86:0/1182972129 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:48.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e69400640 1 -- 172.21.15.86:0/1182972129 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e640cc9d0 msgr2=0x7f4e6415f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:48.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e69e00640 1 --2- 172.21.15.86:0/1182972129 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e640cc9d0 0x7f4e6415f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e69400640 1 --2- 172.21.15.86:0/1182972129 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e640cc9d0 0x7f4e6415f830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e69400640 1 -- 172.21.15.86:0/1182972129 >> v1:172.21.15.86:6789/0 conn(0x7f4e640aebf0 legacy=0x7f4e640cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e63e00640 1 -- 172.21.15.86:0/1182972129 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1735437835 0 0) 0x7f4e6415c240 con 0x7f4e640aebf0 2024-03-06T07:31:48.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.947+0000 7f4e69400640 1 -- 172.21.15.86:0/1182972129 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e6415c7e0 con 0x7f4e641561f0 2024-03-06T07:31:48.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.948+0000 7f4e69400640 1 --2- 172.21.15.86:0/1182972129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e641561f0 0x7f4e640ae6b0 secure :-1 s=READY pgs=999 cs=0 l=1 rev1=1 crypto rx=0x7f4e580097a0 tx=0x7f4e5802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4762f16ac424981b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.948+0000 7f4e63e00640 1 -- 172.21.15.86:0/1182972129 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e58035020 con 0x7f4e641561f0 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.948+0000 7f4e63e00640 1 -- 172.21.15.86:0/1182972129 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4e58031830 con 0x7f4e641561f0 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e63e00640 1 -- 172.21.15.86:0/1182972129 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e58031c00 con 0x7f4e641561f0 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 -- 172.21.15.86:0/1182972129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e641561f0 msgr2=0x7f4e640ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1182972129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e641561f0 0x7f4e640ae6b0 secure :-1 s=READY pgs=999 cs=0 l=1 rev1=1 crypto rx=0x7f4e580097a0 tx=0x7f4e5802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 -- 172.21.15.86:0/1182972129 shutdown_connections 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1182972129 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e640cc9d0 0x7f4e6415f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1182972129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e641561f0 0x7f4e640ae6b0 unknown :-1 s=CLOSED pgs=999 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 -- 172.21.15.86:0/1182972129 >> 172.21.15.86:0/1182972129 conn(0x7f4e6406afc0 msgr2=0x7f4e6406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:48.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 -- 172.21.15.86:0/1182972129 shutdown_connections 2024-03-06T07:31:48.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.949+0000 7f4e6a800640 1 -- 172.21.15.86:0/1182972129 wait complete. 2024-03-06T07:31:48.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.951+0000 7f4e6a800640 1 Processor -- start 2024-03-06T07:31:48.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.951+0000 7f4e6a800640 1 -- start start 2024-03-06T07:31:48.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e6a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e640aebf0 0x7f4e640b0f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e6a800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e640cc9d0 0x7f4e640cd440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e6a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e641561f0 0x7f4e640cee20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e6a800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e64156750 con 0x7f4e640aebf0 2024-03-06T07:31:48.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e6a800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e641565d0 con 0x7f4e640cc9d0 2024-03-06T07:31:48.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e6a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4e6415fd70 con 0x7f4e641561f0 2024-03-06T07:31:48.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e640aebf0 0x7f4e640b0f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e640aebf0 0x7f4e640b0f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41074/0 (socket says 172.21.15.86:41074) 2024-03-06T07:31:48.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 -- 172.21.15.86:0/1664836496 learned_addr learned my addr 172.21.15.86:0/1664836496 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:48.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 -- 172.21.15.86:0/1664836496 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e641561f0 msgr2=0x7f4e640cee20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e641561f0 0x7f4e640cee20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 -- 172.21.15.86:0/1664836496 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e640cc9d0 msgr2=0x7f4e640cd440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:48.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e640cc9d0 0x7f4e640cd440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:48.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.952+0000 7f4e69400640 1 -- 172.21.15.86:0/1664836496 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e640ce370 con 0x7f4e640aebf0 2024-03-06T07:31:48.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.953+0000 7f4e69400640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e640aebf0 0x7f4e640b0f60 secure :-1 s=READY pgs=1000 cs=0 l=1 rev1=1 crypto rx=0x7f4e58034040 tx=0x7f4e580046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.953+0000 7f4e61600640 1 -- 172.21.15.86:0/1664836496 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e58040070 con 0x7f4e640aebf0 2024-03-06T07:31:48.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.953+0000 7f4e61600640 1 -- 172.21.15.86:0/1664836496 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4e58039a00 con 0x7f4e640aebf0 2024-03-06T07:31:48.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.953+0000 7f4e61600640 1 -- 172.21.15.86:0/1664836496 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e58035040 con 0x7f4e640aebf0 2024-03-06T07:31:48.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.953+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4e640cdad0 con 0x7f4e640aebf0 2024-03-06T07:31:48.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.953+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4e640ce090 con 0x7f4e640aebf0 2024-03-06T07:31:48.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.955+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4e6404f330 con 0x7f4e640aebf0 2024-03-06T07:31:48.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.956+0000 7f4e61600640 1 -- 172.21.15.86:0/1664836496 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4e58038020 con 0x7f4e640aebf0 2024-03-06T07:31:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.956+0000 7f4e61600640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4e4406b4d0 0x7f4e4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.957+0000 7f4e61600640 1 -- 172.21.15.86:0/1664836496 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4e580abbb0 con 0x7f4e640aebf0 2024-03-06T07:31:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.957+0000 7f4e68a00640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4e4406b4d0 0x7f4e4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:48.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.957+0000 7f4e68a00640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4e4406b4d0 0x7f4e4406d990 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f4e50007fd0 tx=0x7f4e50007920 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:48.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:48.962+0000 7f4e61600640 1 -- 172.21.15.86:0/1664836496 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4e5807aac0 con 0x7f4e640aebf0 2024-03-06T07:31:49.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.193+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4e640b2b10 con 0x7f4e640aebf0 2024-03-06T07:31:49.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.195+0000 7f4e61600640 1 -- 172.21.15.86:0/1664836496 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+16672 (secure 0 0 0) 0x7f4e5807a3c0 con 0x7f4e640aebf0 2024-03-06T07:31:49.198 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:49.198 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":83,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:33.657039+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:31:29.878719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-06T07:31:31.115332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4e4406b4d0 msgr2=0x7f4e4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4e4406b4d0 0x7f4e4406d990 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f4e50007fd0 tx=0x7f4e50007920 comp rx=0 tx=0).stop 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e640aebf0 msgr2=0x7f4e640b0f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e640aebf0 0x7f4e640b0f60 secure :-1 s=READY pgs=1000 cs=0 l=1 rev1=1 crypto rx=0x7f4e58034040 tx=0x7f4e580046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 shutdown_connections 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4e4406b4d0 0x7f4e4406d990 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4e641561f0 0x7f4e640cee20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4e640cc9d0 0x7f4e640cd440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 --2- 172.21.15.86:0/1664836496 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4e640aebf0 0x7f4e640b0f60 unknown :-1 s=CLOSED pgs=1000 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 >> 172.21.15.86:0/1664836496 conn(0x7f4e6406afc0 msgr2=0x7f4e64071350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:49.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 shutdown_connections 2024-03-06T07:31:49.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.198+0000 7f4e6a800640 1 -- 172.21.15.86:0/1664836496 wait complete. 2024-03-06T07:31:49.219 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:31:49.220 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:49.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.411+0000 7f178b200640 1 Processor -- start 2024-03-06T07:31:49.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.411+0000 7f178b200640 1 -- start start 2024-03-06T07:31:49.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f178b200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f178c161580 0x7f178c0d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f178b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 0x7f178c0b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f178b200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f178c161960 con 0x7f178c0d1e90 2024-03-06T07:31:49.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f178b200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f178c0cf6f0 con 0x7f178c161580 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f178b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f178c0cf9d0 con 0x7f178c0d2a70 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f1789400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f178c0d1e90 0x7f178c0d2360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44322/0 (socket says 172.21.15.86:44322) 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f1789400640 1 -- 172.21.15.86:0/684103210 learned_addr learned my addr 172.21.15.86:0/684103210 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f1789e00640 1 --2- 172.21.15.86:0/684103210 >> v2:172.21.15.194:3300/0 conn(0x7f178c161580 0x7f178c0d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f178a800640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 0x7f178c0b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.413+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1637350369 0 0) 0x7f178c161960 con 0x7f178c0d1e90 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1770003520 con 0x7f178c0d1e90 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3812314109 0 0) 0x7f1770003520 con 0x7f178c0d1e90 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 msgr2=0x7f178c0b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 0x7f178c0b3ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 >> v2:172.21.15.194:3300/0 conn(0x7f178c161580 msgr2=0x7f178c0d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 --2- 172.21.15.86:0/684103210 >> v2:172.21.15.194:3300/0 conn(0x7f178c161580 0x7f178c0d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f178c0cfcb0 con 0x7f178c0d1e90 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1778001690 con 0x7f178c0d1e90 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 >> v1:172.21.15.86:6789/0 conn(0x7f178c0d1e90 legacy=0x7f178c0d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 0x7f17700039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770003f80 0x7f1770004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1770004d40 0x7f17700051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f178c0cf6f0 con 0x7f1770004d40 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f178c0cf9d0 con 0x7f1770003f80 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f177000b9c0 con 0x7f178c0d2a70 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.414+0000 7f178a800640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1770004d40 0x7f17700051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f1789400640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770003f80 0x7f1770004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f1789e00640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 0x7f17700039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f178a800640 1 -- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 msgr2=0x7f17700039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f178a800640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0d2a70 0x7f17700039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f178a800640 1 -- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770003f80 msgr2=0x7f1770004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:49.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f178a800640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770003f80 0x7f1770004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f178a800640 1 -- 172.21.15.86:0/684103210 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1770003520 con 0x7f1770004d40 2024-03-06T07:31:49.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.415+0000 7f1789400640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770003f80 0x7f1770004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:49.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.416+0000 7f178a800640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1770004d40 0x7f17700051c0 secure :-1 s=READY pgs=1002 cs=0 l=1 rev1=1 crypto rx=0x7f1784009fc0 tx=0x7f1770005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f7617d45e54cd81f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:49.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.416+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1784039070 con 0x7f1770004d40 2024-03-06T07:31:49.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.416+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f178402cbd0 con 0x7f1770004d40 2024-03-06T07:31:49.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.416+0000 7f1788a00640 1 -- 172.21.15.86:0/684103210 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1784009c10 con 0x7f1770004d40 2024-03-06T07:31:49.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.417+0000 7f178b200640 1 -- 172.21.15.86:0/684103210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1770004d40 msgr2=0x7f17700051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.417+0000 7f178b200640 1 --2- 172.21.15.86:0/684103210 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1770004d40 0x7f17700051c0 secure :-1 s=READY pgs=1002 cs=0 l=1 rev1=1 crypto rx=0x7f1784009fc0 tx=0x7f1770005830 comp rx=0 tx=0).stop 2024-03-06T07:31:49.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.417+0000 7f178a800640 1 -- 172.21.15.86:0/684103210 reap_dead start 2024-03-06T07:31:49.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.417+0000 7f178b200640 1 -- 172.21.15.86:0/684103210 shutdown_connections 2024-03-06T07:31:49.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.417+0000 7f178b200640 1 -- 172.21.15.86:0/684103210 >> 172.21.15.86:0/684103210 conn(0x7f178c0d3d80 msgr2=0x7f178c0d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:49.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.417+0000 7f178b200640 1 -- 172.21.15.86:0/684103210 shutdown_connections 2024-03-06T07:31:49.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.417+0000 7f178b200640 1 -- 172.21.15.86:0/684103210 wait complete. 2024-03-06T07:31:49.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.418+0000 7f178b200640 1 Processor -- start 2024-03-06T07:31:49.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.419+0000 7f178b200640 1 -- start start 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f178b200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770004d40 0x7f178c0b9060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f178b200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f178c1e99d0 0x7f178c1e9e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f178b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0cb850 0x7f178c0cbd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f178b200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f178c0cf3a0 con 0x7f178c1e99d0 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f178b200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f178c0b39a0 con 0x7f1770004d40 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f178b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f178c0b3ec0 con 0x7f178c0cb850 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f178c1e99d0 0x7f178c1e9e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f178c1e99d0 0x7f178c1e9e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41096/0 (socket says 172.21.15.86:41096) 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 -- 172.21.15.86:0/2303448179 learned_addr learned my addr 172.21.15.86:0/2303448179 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 -- 172.21.15.86:0/2303448179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0cb850 msgr2=0x7f178c0cbd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0cb850 0x7f178c0cbd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 -- 172.21.15.86:0/2303448179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770004d40 msgr2=0x7f178c0b9060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789e00640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770004d40 0x7f178c0b9060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770004d40 0x7f178c0b9060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 -- 172.21.15.86:0/2303448179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f178c0cc640 con 0x7f178c1e99d0 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.420+0000 7f1789400640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f178c1e99d0 0x7f178c1e9e50 secure :-1 s=READY pgs=1003 cs=0 l=1 rev1=1 crypto rx=0x7f1778001010 tx=0x7f177800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.421+0000 7f1782000640 1 -- 172.21.15.86:0/2303448179 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1778019070 con 0x7f178c1e99d0 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.421+0000 7f1782000640 1 -- 172.21.15.86:0/2303448179 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1778007500 con 0x7f178c1e99d0 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.421+0000 7f1782000640 1 -- 172.21.15.86:0/2303448179 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1778004a20 con 0x7f178c1e99d0 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.421+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f178c0cc920 con 0x7f178c1e99d0 2024-03-06T07:31:49.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.421+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f178c0ccee0 con 0x7f178c1e99d0 2024-03-06T07:31:49.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.422+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f178c1ea390 con 0x7f178c1e99d0 2024-03-06T07:31:49.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.423+0000 7f1782000640 1 -- 172.21.15.86:0/2303448179 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1778004bc0 con 0x7f178c1e99d0 2024-03-06T07:31:49.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.424+0000 7f1782000640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f176c06b500 0x7f176c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.424+0000 7f1782000640 1 -- 172.21.15.86:0/2303448179 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1778089b00 con 0x7f178c1e99d0 2024-03-06T07:31:49.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.424+0000 7f1789e00640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f176c06b500 0x7f176c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.425+0000 7f1789e00640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f176c06b500 0x7f176c06d9c0 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f178c0cde50 tx=0x7f177c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:49.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.428+0000 7f1782000640 1 -- 172.21.15.86:0/2303448179 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1778058a10 con 0x7f178c1e99d0 2024-03-06T07:31:49.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.668+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f178c04f330 con 0x7f178c1e99d0 2024-03-06T07:31:49.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.669+0000 7f1782000640 1 -- 172.21.15.86:0/2303448179 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f1778058310 con 0x7f178c1e99d0 2024-03-06T07:31:49.672 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:49.672 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:31:32.095887+0000","modified":"2024-03-06T07:31:35.343932+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6609},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6609":{"gid":6609,"name":"c","rank":0,"incarnation":47,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/255958031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":255958031},{"type":"v1","addr":"172.21.15.86:6837","nonce":255958031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6609,"qdb_cluster":[6609]},"id":6}]} 2024-03-06T07:31:49.672 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 50 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.671+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f176c06b500 msgr2=0x7f176c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.671+0000 7f178b200640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f176c06b500 0x7f176c06d9c0 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f178c0cde50 tx=0x7f177c008040 comp rx=0 tx=0).stop 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.671+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f178c1e99d0 msgr2=0x7f178c1e9e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.671+0000 7f178b200640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f178c1e99d0 0x7f178c1e9e50 secure :-1 s=READY pgs=1003 cs=0 l=1 rev1=1 crypto rx=0x7f1778001010 tx=0x7f177800dcc0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 shutdown_connections 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f176c06b500 0x7f176c06d9c0 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f178c0cb850 0x7f178c0cbd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f178c1e99d0 0x7f178c1e9e50 unknown :-1 s=CLOSED pgs=1003 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 --2- 172.21.15.86:0/2303448179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1770004d40 0x7f178c0b9060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 >> 172.21.15.86:0/2303448179 conn(0x7f178c0d3d80 msgr2=0x7f178c0ca650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 shutdown_connections 2024-03-06T07:31:49.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.672+0000 7f178b200640 1 -- 172.21.15.86:0/2303448179 wait complete. 2024-03-06T07:31:49.692 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:49.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.889+0000 7fa609a00640 1 Processor -- start 2024-03-06T07:31:49.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.889+0000 7fa609a00640 1 -- start start 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa609a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa604144c00 0x7fa60406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa609a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 0x7fa6040ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa609a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa60406ee70 con 0x7fa60415f9d0 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa609a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa60406cef0 con 0x7fa604144c00 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa609a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa60406d1d0 con 0x7fa60415fe60 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa602a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa604144c00 0x7fa60406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa602000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa60415f9d0 0x7fa6040ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44332/0 (socket says 172.21.15.86:44332) 2024-03-06T07:31:49.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.890+0000 7fa602000640 1 -- 172.21.15.86:0/1087969030 learned_addr learned my addr 172.21.15.86:0/1087969030 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:49.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.891+0000 7fa603400640 1 --2- 172.21.15.86:0/1087969030 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 0x7fa6040ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.891+0000 7fa601600640 1 -- 172.21.15.86:0/1087969030 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2938521992 0 0) 0x7fa5ec001490 con 0x7fa60415f9d0 2024-03-06T07:31:49.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.891+0000 7fa601600640 1 -- 172.21.15.86:0/1087969030 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa5e4003520 con 0x7fa60415f9d0 2024-03-06T07:31:49.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.891+0000 7fa602a00640 1 -- 172.21.15.86:0/1087969030 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 msgr2=0x7fa6040ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:49.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.891+0000 7fa602a00640 1 --2- 172.21.15.86:0/1087969030 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 0x7fa6040ce7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.891+0000 7fa602a00640 1 -- 172.21.15.86:0/1087969030 >> v1:172.21.15.86:6789/0 conn(0x7fa60415f9d0 legacy=0x7fa6040ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.891+0000 7fa602a00640 1 -- 172.21.15.86:0/1087969030 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa60406d4b0 con 0x7fa604144c00 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.892+0000 7fa602a00640 1 --2- 172.21.15.86:0/1087969030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60406e930 secure :-1 s=READY pgs=1005 cs=0 l=1 rev1=1 crypto rx=0x7fa5f40097a0 tx=0x7fa5f402b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2c8b076c0536083 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa601600640 1 -- 172.21.15.86:0/1087969030 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f4035020 con 0x7fa604144c00 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa601600640 1 -- 172.21.15.86:0/1087969030 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa5f4031810 con 0x7fa604144c00 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa601600640 1 -- 172.21.15.86:0/1087969030 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f4031be0 con 0x7fa604144c00 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 -- 172.21.15.86:0/1087969030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 msgr2=0x7fa60406e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 --2- 172.21.15.86:0/1087969030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60406e930 secure :-1 s=READY pgs=1005 cs=0 l=1 rev1=1 crypto rx=0x7fa5f40097a0 tx=0x7fa5f402b8b0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 -- 172.21.15.86:0/1087969030 shutdown_connections 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 --2- 172.21.15.86:0/1087969030 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 0x7fa6040ce7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 --2- 172.21.15.86:0/1087969030 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60406e930 unknown :-1 s=CLOSED pgs=1005 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 -- 172.21.15.86:0/1087969030 >> 172.21.15.86:0/1087969030 conn(0x7fa6040b1730 msgr2=0x7fa6040b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 -- 172.21.15.86:0/1087969030 shutdown_connections 2024-03-06T07:31:49.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.893+0000 7fa609a00640 1 -- 172.21.15.86:0/1087969030 wait complete. 2024-03-06T07:31:49.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.895+0000 7fa609a00640 1 Processor -- start 2024-03-06T07:31:49.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.895+0000 7fa609a00640 1 -- start start 2024-03-06T07:31:49.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa609a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60413c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa609a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60415f9d0 0x7fa60415c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa609a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 0x7fa60415c880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa609a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa604160730 con 0x7fa604144c00 2024-03-06T07:31:49.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa609a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa604160290 con 0x7fa60415f9d0 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa609a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa60406cb50 con 0x7fa60415fe60 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa602a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60413c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa602a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60413c380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41110/0 (socket says 172.21.15.86:41110) 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.896+0000 7fa602a00640 1 -- 172.21.15.86:0/2549769719 learned_addr learned my addr 172.21.15.86:0/2549769719 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa602a00640 1 -- 172.21.15.86:0/2549769719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 msgr2=0x7fa60415c880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa602a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 0x7fa60415c880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa602a00640 1 -- 172.21.15.86:0/2549769719 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60415f9d0 msgr2=0x7fa60415c340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa602a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60415f9d0 0x7fa60415c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa602a00640 1 -- 172.21.15.86:0/2549769719 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa60413da30 con 0x7fa604144c00 2024-03-06T07:31:49.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa602a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60413c380 secure :-1 s=READY pgs=1006 cs=0 l=1 rev1=1 crypto rx=0x7fa5f4034040 tx=0x7fa5f40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:49.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa5faa00640 1 -- 172.21.15.86:0/2549769719 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f4035040 con 0x7fa604144c00 2024-03-06T07:31:49.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa5faa00640 1 -- 172.21.15.86:0/2549769719 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa5f4039cf0 con 0x7fa604144c00 2024-03-06T07:31:49.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa5faa00640 1 -- 172.21.15.86:0/2549769719 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f4039ea0 con 0x7fa604144c00 2024-03-06T07:31:49.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.897+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa60413dd10 con 0x7fa604144c00 2024-03-06T07:31:49.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.898+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa60413c8c0 con 0x7fa604144c00 2024-03-06T07:31:49.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.899+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa60404f330 con 0x7fa604144c00 2024-03-06T07:31:49.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.899+0000 7fa5faa00640 1 -- 172.21.15.86:0/2549769719 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa5f4038020 con 0x7fa604144c00 2024-03-06T07:31:49.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.900+0000 7fa5faa00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa5e006b4d0 0x7fa5e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:49.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.900+0000 7fa5faa00640 1 -- 172.21.15.86:0/2549769719 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa5f40ac240 con 0x7fa604144c00 2024-03-06T07:31:49.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.900+0000 7fa602000640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa5e006b4d0 0x7fa5e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:49.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.901+0000 7fa602000640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa5e006b4d0 0x7fa5e006d990 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fa5ec0012e0 tx=0x7fa5ec001490 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:49.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:49.905+0000 7fa5faa00640 1 -- 172.21.15.86:0/2549769719 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa5f407b150 con 0x7fa604144c00 2024-03-06T07:31:50.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.143+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa604072890 con 0x7fa604144c00 2024-03-06T07:31:50.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.144+0000 7fa5faa00640 1 -- 172.21.15.86:0/2549769719 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4982 (secure 0 0 0) 0x7fa5f407aa50 con 0x7fa604144c00 2024-03-06T07:31:50.147 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:50.147 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":50,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:31:32.095887+0000","modified":"2024-03-06T07:31:35.343932+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6609},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6609":{"gid":6609,"name":"c","rank":0,"incarnation":47,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/255958031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":255958031},{"type":"v1","addr":"172.21.15.86:6837","nonce":255958031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6609,"qdb_cluster":[6609]},"id":6}]} 2024-03-06T07:31:50.147 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 50 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa5e006b4d0 msgr2=0x7fa5e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa5e006b4d0 0x7fa5e006d990 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fa5ec0012e0 tx=0x7fa5ec001490 comp rx=0 tx=0).stop 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 msgr2=0x7fa60413c380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60413c380 secure :-1 s=READY pgs=1006 cs=0 l=1 rev1=1 crypto rx=0x7fa5f4034040 tx=0x7fa5f40046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 shutdown_connections 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa5e006b4d0 0x7fa5e006d990 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60415fe60 0x7fa60415c880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60415f9d0 0x7fa60415c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 --2- 172.21.15.86:0/2549769719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa604144c00 0x7fa60413c380 unknown :-1 s=CLOSED pgs=1006 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 >> 172.21.15.86:0/2549769719 conn(0x7fa6040b1730 msgr2=0x7fa6040af880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:50.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 shutdown_connections 2024-03-06T07:31:50.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.147+0000 7fa609a00640 1 -- 172.21.15.86:0/2549769719 wait complete. 2024-03-06T07:31:50.172 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:50.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.367+0000 7f2488e00640 1 Processor -- start 2024-03-06T07:31:50.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.367+0000 7f2488e00640 1 -- start start 2024-03-06T07:31:50.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.369+0000 7f2488e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2484144c00 0x7f248406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.369+0000 7f2488e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f248415f9d0 0x7f248415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.369+0000 7f2488e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f248406ee70 con 0x7f24840ce0a0 2024-03-06T07:31:50.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.369+0000 7f2488e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2484160370 con 0x7f248415f9d0 2024-03-06T07:31:50.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.369+0000 7f2488e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f248406cef0 con 0x7f2484144c00 2024-03-06T07:31:50.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.369+0000 7f2481e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2484144c00 0x7f248406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.369+0000 7f247be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f248415f9d0 0x7f248415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.370+0000 7f2481e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2484144c00 0x7f248406e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:48380/0 (socket says 172.21.15.86:48380) 2024-03-06T07:31:50.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.370+0000 7f2481e00640 1 -- 172.21.15.86:0/2245564642 learned_addr learned my addr 172.21.15.86:0/2245564642 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:50.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.370+0000 7f247be00640 1 -- 172.21.15.86:0/2245564642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2484144c00 msgr2=0x7f248406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:50.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.370+0000 7f247be00640 1 --2- 172.21.15.86:0/2245564642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2484144c00 0x7f248406e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.370+0000 7f247be00640 1 -- 172.21.15.86:0/2245564642 >> v1:172.21.15.194:6789/0 conn(0x7f24840ce0a0 legacy=0x7f24840ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.370+0000 7f247be00640 1 -- 172.21.15.86:0/2245564642 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f248406d1d0 con 0x7f248415f9d0 2024-03-06T07:31:50.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.372+0000 7f247be00640 1 --2- 172.21.15.86:0/2245564642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f248415f9d0 0x7f248415fe30 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f246c0097a0 tx=0x7f246c02b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2cca1f59e078cf42 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:50.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.372+0000 7f2481400640 1 -- 172.21.15.86:0/2245564642 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f246c035020 con 0x7f248415f9d0 2024-03-06T07:31:50.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.372+0000 7f2481400640 1 -- 172.21.15.86:0/2245564642 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f246c031830 con 0x7f248415f9d0 2024-03-06T07:31:50.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.372+0000 7f2481400640 1 -- 172.21.15.86:0/2245564642 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f246c031c00 con 0x7f248415f9d0 2024-03-06T07:31:50.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 -- 172.21.15.86:0/2245564642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f248415f9d0 msgr2=0x7f248415fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 --2- 172.21.15.86:0/2245564642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f248415f9d0 0x7f248415fe30 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f246c0097a0 tx=0x7f246c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 -- 172.21.15.86:0/2245564642 shutdown_connections 2024-03-06T07:31:50.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 --2- 172.21.15.86:0/2245564642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f248415f9d0 0x7f248415fe30 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 --2- 172.21.15.86:0/2245564642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2484144c00 0x7f248406e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 -- 172.21.15.86:0/2245564642 >> 172.21.15.86:0/2245564642 conn(0x7f24840b1730 msgr2=0x7f24840b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:50.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 -- 172.21.15.86:0/2245564642 shutdown_connections 2024-03-06T07:31:50.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.373+0000 7f2488e00640 1 -- 172.21.15.86:0/2245564642 wait complete. 2024-03-06T07:31:50.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.376+0000 7f2488e00640 1 Processor -- start 2024-03-06T07:31:50.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.376+0000 7f2488e00640 1 -- start start 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.377+0000 7f2488e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f24840ce0a0 0x7f248415c500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.377+0000 7f2488e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2484144c00 0x7f248413bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2488e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f248415f9d0 0x7f248413c320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2488e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2484160730 con 0x7f248415f9d0 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2488e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f248406d790 con 0x7f2484144c00 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2488e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f248406cb50 con 0x7f24840ce0a0 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f248415f9d0 0x7f248413c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f248415f9d0 0x7f248413c320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41114/0 (socket says 172.21.15.86:41114) 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 -- 172.21.15.86:0/3353013284 learned_addr learned my addr 172.21.15.86:0/3353013284 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:50.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2481e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f24840ce0a0 0x7f248415c500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f247be00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2484144c00 0x7f248413bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 -- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f24840ce0a0 msgr2=0x7f248415c500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f24840ce0a0 0x7f248415c500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 -- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2484144c00 msgr2=0x7f248413bde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2484144c00 0x7f248413bde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f2482800640 1 -- 172.21.15.86:0/3353013284 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f248415f1c0 con 0x7f248415f9d0 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.378+0000 7f247be00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2484144c00 0x7f248413bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:50.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.379+0000 7f2482800640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f248415f9d0 0x7f248413c320 secure :-1 s=READY pgs=1007 cs=0 l=1 rev1=1 crypto rx=0x7f247c00c6b0 tx=0x7f247c00cb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:50.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.379+0000 7f247a000640 1 -- 172.21.15.86:0/3353013284 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f247c003b70 con 0x7f248415f9d0 2024-03-06T07:31:50.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.379+0000 7f247a000640 1 -- 172.21.15.86:0/3353013284 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f247c003d30 con 0x7f248415f9d0 2024-03-06T07:31:50.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.379+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f248415f4a0 con 0x7f248415f9d0 2024-03-06T07:31:50.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.379+0000 7f247a000640 1 -- 172.21.15.86:0/3353013284 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f247c005310 con 0x7f248415f9d0 2024-03-06T07:31:50.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.379+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f248415ca40 con 0x7f248415f9d0 2024-03-06T07:31:50.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.380+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f248404f330 con 0x7f248415f9d0 2024-03-06T07:31:50.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.382+0000 7f247a000640 1 -- 172.21.15.86:0/3353013284 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f247c0054b0 con 0x7f248415f9d0 2024-03-06T07:31:50.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.383+0000 7f247a000640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f246006b4d0 0x7f246006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.383+0000 7f247a000640 1 -- 172.21.15.86:0/3353013284 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f247c08a100 con 0x7f248415f9d0 2024-03-06T07:31:50.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.383+0000 7f2481e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f246006b4d0 0x7f246006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.384+0000 7f2481e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f246006b4d0 0x7f246006d990 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7f2474009b60 tx=0x7f2474008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:50.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.387+0000 7f247a000640 1 -- 172.21.15.86:0/3353013284 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f248406ee70 con 0x7f248415f9d0 2024-03-06T07:31:50.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.585+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2484072890 con 0x7f248415f9d0 2024-03-06T07:31:50.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.587+0000 7f247a000640 1 -- 172.21.15.86:0/3353013284 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+16672 (secure 0 0 0) 0x7f247c058ee0 con 0x7f248415f9d0 2024-03-06T07:31:50.589 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:50.590 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":83,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:33.657039+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:31:29.878719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-06T07:31:31.115332+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f246006b4d0 msgr2=0x7f246006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f246006b4d0 0x7f246006d990 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7f2474009b60 tx=0x7f2474008040 comp rx=0 tx=0).stop 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f248415f9d0 msgr2=0x7f248413c320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f248415f9d0 0x7f248413c320 secure :-1 s=READY pgs=1007 cs=0 l=1 rev1=1 crypto rx=0x7f247c00c6b0 tx=0x7f247c00cb70 comp rx=0 tx=0).stop 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 shutdown_connections 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f246006b4d0 0x7f246006d990 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f248415f9d0 0x7f248413c320 unknown :-1 s=CLOSED pgs=1007 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2484144c00 0x7f248413bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 --2- 172.21.15.86:0/3353013284 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f24840ce0a0 0x7f248415c500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 >> 172.21.15.86:0/3353013284 conn(0x7f24840b1730 msgr2=0x7f24840b0280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:50.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 shutdown_connections 2024-03-06T07:31:50.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.589+0000 7f2488e00640 1 -- 172.21.15.86:0/3353013284 wait complete. 2024-03-06T07:31:50.612 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:31:50.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.804+0000 7ff3a1000640 1 Processor -- start 2024-03-06T07:31:50.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.804+0000 7ff3a1000640 1 -- start start 2024-03-06T07:31:50.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff3a1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 0x7ff39c0cbab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff3a1000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff39c06db80 0x7ff39c0ca240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff3a1000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff39c15b9e0 con 0x7ff39c06e470 2024-03-06T07:31:50.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff3a1000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff39c0cbff0 con 0x7ff39c06db80 2024-03-06T07:31:50.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff3a1000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff39c0cc220 con 0x7ff39c1581d0 2024-03-06T07:31:50.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff399400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff39c06e470 0x7ff39c06d470 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44342/0 (socket says 172.21.15.86:44342) 2024-03-06T07:31:50.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff399400640 1 -- 172.21.15.86:0/1164095160 learned_addr learned my addr 172.21.15.86:0/1164095160 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.807+0000 7ff399e00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 0x7ff39c0cbab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1052790334 0 0) 0x7ff39c15b9e0 con 0x7ff39c06e470 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff39a800640 1 --2- 172.21.15.86:0/1164095160 >> v2:172.21.15.194:3300/0 conn(0x7ff39c06db80 0x7ff39c0ca240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff380003520 con 0x7ff39c06e470 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1023201116 0 0) 0x7ff380003520 con 0x7ff39c06e470 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 msgr2=0x7ff39c0cbab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 0x7ff39c0cbab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 >> v2:172.21.15.194:3300/0 conn(0x7ff39c06db80 msgr2=0x7ff39c0ca240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:50.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 --2- 172.21.15.86:0/1164095160 >> v2:172.21.15.194:3300/0 conn(0x7ff39c06db80 0x7ff39c0ca240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff39c15ac20 con 0x7ff39c06e470 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ff388001670 con 0x7ff39c06e470 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 >> v1:172.21.15.86:6789/0 conn(0x7ff39c06e470 legacy=0x7ff39c06d470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff380003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 0x7ff380004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff380004d40 0x7ff3800051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff39c0cc220 con 0x7ff3800039b0 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff38000b9c0 con 0x7ff380004d40 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff38000ddd0 con 0x7ff39c1581d0 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.808+0000 7ff399e00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff380003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff399400640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 0x7ff380004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff39a800640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff380004d40 0x7ff3800051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff399e00640 1 -- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 msgr2=0x7ff380004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff399e00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 0x7ff380004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff399e00640 1 -- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff380004d40 msgr2=0x7ff3800051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:50.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff399e00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff380004d40 0x7ff3800051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff399e00640 1 -- 172.21.15.86:0/1164095160 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff380003520 con 0x7ff3800039b0 2024-03-06T07:31:50.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.809+0000 7ff399400640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c1581d0 0x7ff380004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.810+0000 7ff399e00640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff380003dd0 secure :-1 s=READY pgs=1009 cs=0 l=1 rev1=1 crypto rx=0x7ff38c009ac0 tx=0x7ff380005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8bee1f5a4ce27917 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.810+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff38c039070 con 0x7ff3800039b0 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.810+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff38c02c6b0 con 0x7ff3800039b0 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.810+0000 7ff398a00640 1 -- 172.21.15.86:0/1164095160 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff38c009690 con 0x7ff3800039b0 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.811+0000 7ff3a1000640 1 -- 172.21.15.86:0/1164095160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 msgr2=0x7ff380003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.811+0000 7ff3a1000640 1 --2- 172.21.15.86:0/1164095160 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff380003dd0 secure :-1 s=READY pgs=1009 cs=0 l=1 rev1=1 crypto rx=0x7ff38c009ac0 tx=0x7ff380005830 comp rx=0 tx=0).stop 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.811+0000 7ff39a800640 1 -- 172.21.15.86:0/1164095160 reap_dead start 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.811+0000 7ff3a1000640 1 -- 172.21.15.86:0/1164095160 shutdown_connections 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.811+0000 7ff3a1000640 1 -- 172.21.15.86:0/1164095160 >> 172.21.15.86:0/1164095160 conn(0x7ff39c0cc4f0 msgr2=0x7ff39c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.811+0000 7ff3a1000640 1 -- 172.21.15.86:0/1164095160 shutdown_connections 2024-03-06T07:31:50.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.811+0000 7ff3a1000640 1 -- 172.21.15.86:0/1164095160 wait complete. 2024-03-06T07:31:50.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.814+0000 7ff3a1000640 1 Processor -- start 2024-03-06T07:31:50.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.814+0000 7ff3a1000640 1 -- start start 2024-03-06T07:31:50.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.815+0000 7ff3a1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff39c0c9570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.815+0000 7ff3a1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c14bc70 0x7ff39c14c0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.815+0000 7ff3a1000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff39c14d0b0 0x7ff39c14d530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.815+0000 7ff3a1000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff39c15b4c0 con 0x7ff3800039b0 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.815+0000 7ff3a1000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff39c15b1e0 con 0x7ff39c14d0b0 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.815+0000 7ff3a1000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff39c0cbab0 con 0x7ff39c14bc70 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff39c0c9570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff39c0c9570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41128/0 (socket says 172.21.15.86:41128) 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 -- 172.21.15.86:0/4040330570 learned_addr learned my addr 172.21.15.86:0/4040330570 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 -- 172.21.15.86:0/4040330570 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c14bc70 msgr2=0x7ff39c14c0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399400640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c14bc70 0x7ff39c14c0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c14bc70 0x7ff39c14c0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 -- 172.21.15.86:0/4040330570 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff39c14d0b0 msgr2=0x7ff39c14d530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:50.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff39c14d0b0 0x7ff39c14d530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:50.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 -- 172.21.15.86:0/4040330570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff39c0c84d0 con 0x7ff3800039b0 2024-03-06T07:31:50.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.816+0000 7ff399e00640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff39c0c9570 secure :-1 s=READY pgs=1010 cs=0 l=1 rev1=1 crypto rx=0x7ff38c009ac0 tx=0x7ff38c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:50.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.817+0000 7ff392000640 1 -- 172.21.15.86:0/4040330570 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff38c039050 con 0x7ff3800039b0 2024-03-06T07:31:50.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.817+0000 7ff392000640 1 -- 172.21.15.86:0/4040330570 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff38c045070 con 0x7ff3800039b0 2024-03-06T07:31:50.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.817+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff39c0c87b0 con 0x7ff3800039b0 2024-03-06T07:31:50.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.817+0000 7ff392000640 1 -- 172.21.15.86:0/4040330570 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff38c003a50 con 0x7ff3800039b0 2024-03-06T07:31:50.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.817+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff39c0c8d70 con 0x7ff3800039b0 2024-03-06T07:31:50.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.818+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff39c0c91b0 con 0x7ff3800039b0 2024-03-06T07:31:50.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.819+0000 7ff392000640 1 -- 172.21.15.86:0/4040330570 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff38c003bf0 con 0x7ff3800039b0 2024-03-06T07:31:50.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.819+0000 7ff392000640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff37c06b500 0x7ff37c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:50.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.820+0000 7ff392000640 1 -- 172.21.15.86:0/4040330570 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff38c0ac310 con 0x7ff3800039b0 2024-03-06T07:31:50.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.820+0000 7ff399400640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff37c06b500 0x7ff37c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:50.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.820+0000 7ff399400640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff37c06b500 0x7ff37c06d9c0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7ff388000c00 tx=0x7ff388009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:50.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:50.824+0000 7ff392000640 1 -- 172.21.15.86:0/4040330570 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff38c04f020 con 0x7ff3800039b0 2024-03-06T07:31:51.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.168+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7ff39c04f330 con 0x7ff3800039b0 2024-03-06T07:31:51.172 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:31:51.169+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:31:51.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.172+0000 7ff392000640 1 -- 172.21.15.86:0/4040330570 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v51) v1 ==== 161+0+0 (secure 0 0 0) 0x7ff38c07b360 con 0x7ff3800039b0 2024-03-06T07:31:51.175 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:31:51.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.175+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff37c06b500 msgr2=0x7ff37c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:51.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.175+0000 7ff3a1000640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff37c06b500 0x7ff37c06d9c0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7ff388000c00 tx=0x7ff388009040 comp rx=0 tx=0).stop 2024-03-06T07:31:51.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.175+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 msgr2=0x7ff39c0c9570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:51.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.175+0000 7ff3a1000640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff39c0c9570 secure :-1 s=READY pgs=1010 cs=0 l=1 rev1=1 crypto rx=0x7ff38c009ac0 tx=0x7ff38c004730 comp rx=0 tx=0).stop 2024-03-06T07:31:51.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.176+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 shutdown_connections 2024-03-06T07:31:51.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.176+0000 7ff3a1000640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff37c06b500 0x7ff37c06d9c0 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.176+0000 7ff3a1000640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff39c14d0b0 0x7ff39c14d530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.176+0000 7ff3a1000640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff39c14bc70 0x7ff39c14c0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.176+0000 7ff3a1000640 1 --2- 172.21.15.86:0/4040330570 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff3800039b0 0x7ff39c0c9570 unknown :-1 s=CLOSED pgs=1010 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.176+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 >> 172.21.15.86:0/4040330570 conn(0x7ff39c0cc4f0 msgr2=0x7ff39c15a250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:51.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.176+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 shutdown_connections 2024-03-06T07:31:51.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.177+0000 7ff3a1000640 1 -- 172.21.15.86:0/4040330570 wait complete. 2024-03-06T07:31:51.190 INFO:tasks.ceph.mds.c.smithi086.stderr: -4146> 2024-03-06T07:31:33.592+0000 7fc3dfa00640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:31:51.196 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:31:51.247 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:31:51.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.378+0000 7f392fa00640 1 Processor -- start 2024-03-06T07:31:51.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.378+0000 7f392fa00640 1 -- start start 2024-03-06T07:31:51.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.379+0000 7f392fa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 0x7f392806d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.379+0000 7f392fa00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f392815cf70 con 0x7f392806e060 2024-03-06T07:31:51.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.379+0000 7f392fa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39280b35b0 con 0x7f3928163800 2024-03-06T07:31:51.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.379+0000 7f392fa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39280b3890 con 0x7f392815cb90 2024-03-06T07:31:51.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.380+0000 7f392da00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f392806e060 0x7f39280cda90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44346/0 (socket says 172.21.15.86:44346) 2024-03-06T07:31:51.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.380+0000 7f392da00640 1 -- 172.21.15.86:0/2794270319 learned_addr learned my addr 172.21.15.86:0/2794270319 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:51.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.380+0000 7f392d000640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 0x7f392806d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:51.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.380+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3439653992 0 0) 0x7f392815cf70 con 0x7f392806e060 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.380+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3910003520 con 0x7f392806e060 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2575137994 0 0) 0x7f3910003520 con 0x7f392806e060 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 msgr2=0x7f392806d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 0x7f392806d6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 >> v1:172.21.15.194:6789/0 conn(0x7f3928163800 legacy=0x7f39280b2c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f392806cbe0 con 0x7f392806e060 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f39200019f0 con 0x7f392806e060 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 >> v1:172.21.15.86:6789/0 conn(0x7f392806e060 legacy=0x7f39280cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 0x7f39100039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3910003fb0 0x7f3910004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910004d40 0x7f39100051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39280b3890 con 0x7f3910003fb0 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f391000bb00 con 0x7f3910004d40 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f391000bd00 con 0x7f392815cb90 2024-03-06T07:31:51.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.381+0000 7f3927e00640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3910003fb0 0x7f3910004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f392d000640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 0x7f39100039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f392da00640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910004d40 0x7f39100051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f3927e00640 1 -- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 msgr2=0x7f39100039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f3927e00640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 0x7f39100039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f3927e00640 1 -- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910004d40 msgr2=0x7f39100051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f3927e00640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910004d40 0x7f39100051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f3927e00640 1 -- 172.21.15.86:0/2794270319 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3910003520 con 0x7f3910003fb0 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f392da00640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910004d40 0x7f39100051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:51.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.382+0000 7f392d000640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392815cb90 0x7f39100039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:51.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.383+0000 7f3927e00640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3910003fb0 0x7f3910004800 secure :-1 s=READY pgs=1014 cs=0 l=1 rev1=1 crypto rx=0x7f3914001750 tx=0x7f3910005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fe5583df4738de1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:51.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.383+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3914001370 con 0x7f3910003fb0 2024-03-06T07:31:51.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.383+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f39140052d0 con 0x7f3910003fb0 2024-03-06T07:31:51.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.383+0000 7f3927400640 1 -- 172.21.15.86:0/2794270319 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3914003830 con 0x7f3910003fb0 2024-03-06T07:31:51.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.384+0000 7f392fa00640 1 -- 172.21.15.86:0/2794270319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3910003fb0 msgr2=0x7f3910004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:51.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.384+0000 7f392fa00640 1 --2- 172.21.15.86:0/2794270319 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3910003fb0 0x7f3910004800 secure :-1 s=READY pgs=1014 cs=0 l=1 rev1=1 crypto rx=0x7f3914001750 tx=0x7f3910005830 comp rx=0 tx=0).stop 2024-03-06T07:31:51.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.384+0000 7f392da00640 1 -- 172.21.15.86:0/2794270319 reap_dead start 2024-03-06T07:31:51.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.384+0000 7f392fa00640 1 -- 172.21.15.86:0/2794270319 shutdown_connections 2024-03-06T07:31:51.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.384+0000 7f392fa00640 1 -- 172.21.15.86:0/2794270319 >> 172.21.15.86:0/2794270319 conn(0x7f39280b1940 msgr2=0x7f39280b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:51.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.384+0000 7f392fa00640 1 -- 172.21.15.86:0/2794270319 shutdown_connections 2024-03-06T07:31:51.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.384+0000 7f392fa00640 1 -- 172.21.15.86:0/2794270319 wait complete. 2024-03-06T07:31:51.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.385+0000 7f392fa00640 1 Processor -- start 2024-03-06T07:31:51.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.386+0000 7f392fa00640 1 -- start start 2024-03-06T07:31:51.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.386+0000 7f392fa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910003fb0 0x7f39281621e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.386+0000 7f392fa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392809a9f0 0x7f392809c580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.386+0000 7f392fa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f392815ae10 0x7f39280add70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.386+0000 7f392fa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f392806d230 con 0x7f392815ae10 2024-03-06T07:31:51.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.386+0000 7f392fa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f392806c990 con 0x7f3910003fb0 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.386+0000 7f392fa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39280b3350 con 0x7f392809a9f0 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f392815ae10 0x7f39280add70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f392815ae10 0x7f39280add70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41170/0 (socket says 172.21.15.86:41170) 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 -- 172.21.15.86:0/760066291 learned_addr learned my addr 172.21.15.86:0/760066291 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392d000640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910003fb0 0x7f39281621e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 -- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392809a9f0 msgr2=0x7f392809c580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392809a9f0 0x7f392809c580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 -- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910003fb0 msgr2=0x7f39281621e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910003fb0 0x7f39281621e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 -- 172.21.15.86:0/760066291 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f392809af60 con 0x7f392815ae10 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392d000640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910003fb0 0x7f39281621e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.387+0000 7f392da00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f392815ae10 0x7f39280add70 secure :-1 s=READY pgs=1015 cs=0 l=1 rev1=1 crypto rx=0x7f392000dc90 tx=0x7f392000b4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.388+0000 7f3924c00640 1 -- 172.21.15.86:0/760066291 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f392000dea0 con 0x7f392815ae10 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.388+0000 7f3924c00640 1 -- 172.21.15.86:0/760066291 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3920004b60 con 0x7f392815ae10 2024-03-06T07:31:51.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.388+0000 7f3924c00640 1 -- 172.21.15.86:0/760066291 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3920007110 con 0x7f392815ae10 2024-03-06T07:31:51.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.388+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f392809b240 con 0x7f392815ae10 2024-03-06T07:31:51.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.388+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f392809b800 con 0x7f392815ae10 2024-03-06T07:31:51.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.389+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f392804f330 con 0x7f392815ae10 2024-03-06T07:31:51.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.390+0000 7f3924c00640 1 -- 172.21.15.86:0/760066291 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f392001b020 con 0x7f392815ae10 2024-03-06T07:31:51.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.391+0000 7f3924c00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:51.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.391+0000 7f3924c00640 1 -- 172.21.15.86:0/760066291 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f392008a440 con 0x7f392815ae10 2024-03-06T07:31:51.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.391+0000 7f392d000640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:51.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.392+0000 7f392d000640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f39281fbac0 tx=0x7f391c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:51.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.395+0000 7f3924c00640 1 -- 172.21.15.86:0/760066291 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f39200593b0 con 0x7f392815ae10 2024-03-06T07:31:51.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:51.634+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f39280ced70 con 0x7f392815ae10 2024-03-06T07:31:52.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.175+0000 7f3924c00640 1 -- 172.21.15.86:0/760066291 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v52) v1 ==== 104+0+0 (secure 0 0 0) 0x7f3920058cb0 con 0x7f392815ae10 2024-03-06T07:31:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.179+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 msgr2=0x7f390806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.179+0000 7f392fa00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f39281fbac0 tx=0x7f391c008040 comp rx=0 tx=0).stop 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.179+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f392815ae10 msgr2=0x7f39280add70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.179+0000 7f392fa00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f392815ae10 0x7f39280add70 secure :-1 s=READY pgs=1015 cs=0 l=1 rev1=1 crypto rx=0x7f392000dc90 tx=0x7f392000b4a0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 shutdown_connections 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f390806b4d0 0x7f390806d990 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f392815ae10 0x7f39280add70 unknown :-1 s=CLOSED pgs=1015 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f392809a9f0 0x7f392809c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 --2- 172.21.15.86:0/760066291 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3910003fb0 0x7f39281621e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 >> 172.21.15.86:0/760066291 conn(0x7f39280b1940 msgr2=0x7f392806d5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 shutdown_connections 2024-03-06T07:31:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.180+0000 7f392fa00640 1 -- 172.21.15.86:0/760066291 wait complete. 2024-03-06T07:31:52.202 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:31:52.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.407+0000 7f5f63e00640 1 Processor -- start 2024-03-06T07:31:52.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.407+0000 7f5f63e00640 1 -- start start 2024-03-06T07:31:52.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.408+0000 7f5f63e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f5f6415df30 0x7f5f640cf2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:52.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.408+0000 7f5f63e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5f640d06a0 0x7f5f640d0ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:52.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5f640d0ff0 con 0x7f5f640d47b0 2024-03-06T07:31:52.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5f640d1860 con 0x7f5f640d06a0 2024-03-06T07:31:52.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5f640d2ca0 con 0x7f5f6415df30 2024-03-06T07:31:52.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5f640d06a0 0x7f5f640d0ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:52.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5f640d06a0 0x7f5f640d0ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41182/0 (socket says 172.21.15.86:41182) 2024-03-06T07:31:52.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63400640 1 -- 172.21.15.86:0/176628721 learned_addr learned my addr 172.21.15.86:0/176628721 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:52.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63400640 1 -- 172.21.15.86:0/176628721 >> v2:172.21.15.194:3300/0 conn(0x7f5f6415df30 msgr2=0x7f5f640cf2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:52.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63400640 1 --2- 172.21.15.86:0/176628721 >> v2:172.21.15.194:3300/0 conn(0x7f5f6415df30 0x7f5f640cf2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63400640 1 -- 172.21.15.86:0/176628721 >> v1:172.21.15.194:6789/0 conn(0x7f5f640d47b0 legacy=0x7f5f640d2590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:52.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.409+0000 7f5f63400640 1 -- 172.21.15.86:0/176628721 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f640cfa00 con 0x7f5f640d06a0 2024-03-06T07:31:52.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.410+0000 7f5f63400640 1 --2- 172.21.15.86:0/176628721 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d06a0 0x7f5f640d0ab0 secure :-1 s=READY pgs=1016 cs=0 l=1 rev1=1 crypto rx=0x7f5f640b6850 tx=0x7f5f5c02be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f891052e8f961c0b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:52.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f61600640 1 -- 172.21.15.86:0/176628721 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f5c02c9f0 con 0x7f5f640d06a0 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f61600640 1 -- 172.21.15.86:0/176628721 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f5c02cb90 con 0x7f5f640d06a0 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f61600640 1 -- 172.21.15.86:0/176628721 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f5c0343f0 con 0x7f5f640d06a0 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 -- 172.21.15.86:0/176628721 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d06a0 msgr2=0x7f5f640d0ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 --2- 172.21.15.86:0/176628721 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d06a0 0x7f5f640d0ab0 secure :-1 s=READY pgs=1016 cs=0 l=1 rev1=1 crypto rx=0x7f5f640b6850 tx=0x7f5f5c02be50 comp rx=0 tx=0).stop 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 -- 172.21.15.86:0/176628721 shutdown_connections 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 --2- 172.21.15.86:0/176628721 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d06a0 0x7f5f640d0ab0 unknown :-1 s=CLOSED pgs=1016 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 --2- 172.21.15.86:0/176628721 >> v2:172.21.15.194:3300/0 conn(0x7f5f6415df30 0x7f5f640cf2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 -- 172.21.15.86:0/176628721 >> 172.21.15.86:0/176628721 conn(0x7f5f640d3c90 msgr2=0x7f5f640d40a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:52.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 -- 172.21.15.86:0/176628721 shutdown_connections 2024-03-06T07:31:52.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.411+0000 7f5f63e00640 1 -- 172.21.15.86:0/176628721 wait complete. 2024-03-06T07:31:52.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.413+0000 7f5f63e00640 1 Processor -- start 2024-03-06T07:31:52.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.413+0000 7f5f63e00640 1 -- start start 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f63e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f640d06a0 0x7f5f640b2400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f63e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d47b0 0x7f5f64152f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f63e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f6415df30 0x7f5f640b0980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f63e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f640d2070 con 0x7f5f640d47b0 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f63e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f640d1d90 con 0x7f5f640d06a0 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f63e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f6415e310 con 0x7f5f6415df30 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f62000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d47b0 0x7f5f64152f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f62000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d47b0 0x7f5f64152f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41184/0 (socket says 172.21.15.86:41184) 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.414+0000 7f5f62000640 1 -- 172.21.15.86:0/276791296 learned_addr learned my addr 172.21.15.86:0/276791296 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f62a00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f640d06a0 0x7f5f640b2400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f63400640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f6415df30 0x7f5f640b0980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:52.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f62000640 1 -- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f6415df30 msgr2=0x7f5f640b0980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f62000640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f6415df30 0x7f5f640b0980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f62000640 1 -- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f640d06a0 msgr2=0x7f5f640b2400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f62000640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f640d06a0 0x7f5f640b2400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f62000640 1 -- 172.21.15.86:0/276791296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f640b1360 con 0x7f5f640d47b0 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f62000640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d47b0 0x7f5f64152f60 secure :-1 s=READY pgs=1017 cs=0 l=1 rev1=1 crypto rx=0x7f5f5000c440 tx=0x7f5f5000c900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f5aa00640 1 -- 172.21.15.86:0/276791296 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f50017070 con 0x7f5f640d47b0 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f5aa00640 1 -- 172.21.15.86:0/276791296 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f500035f0 con 0x7f5f640d47b0 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f5aa00640 1 -- 172.21.15.86:0/276791296 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f500039e0 con 0x7f5f640d47b0 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.415+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5f640b1640 con 0x7f5f640d47b0 2024-03-06T07:31:52.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.416+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5f640b1c00 con 0x7f5f640d47b0 2024-03-06T07:31:52.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.416+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5f6404f330 con 0x7f5f640d47b0 2024-03-06T07:31:52.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.417+0000 7f5f5aa00640 1 -- 172.21.15.86:0/276791296 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5f50003b80 con 0x7f5f640d47b0 2024-03-06T07:31:52.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.418+0000 7f5f5aa00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f4406b4d0 0x7f5f4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:52.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.418+0000 7f5f5aa00640 1 -- 172.21.15.86:0/276791296 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f5f500895c0 con 0x7f5f640d47b0 2024-03-06T07:31:52.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.419+0000 7f5f62a00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f4406b4d0 0x7f5f4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:52.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.419+0000 7f5f62a00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f4406b4d0 0x7f5f4406d990 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f5f5400b810 tx=0x7f5f54008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:52.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.423+0000 7f5f5aa00640 1 -- 172.21.15.86:0/276791296 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5f50058340 con 0x7f5f640d47b0 2024-03-06T07:31:52.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:52.617+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f5f640cea90 con 0x7f5f640d47b0 2024-03-06T07:31:53.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.179+0000 7f5f5aa00640 1 -- 172.21.15.86:0/276791296 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v85) v1 ==== 181+0+0 (secure 0 0 0) 0x7f5f500010c0 con 0x7f5f640d47b0 2024-03-06T07:31:53.182 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:31:53.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f4406b4d0 msgr2=0x7f5f4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:53.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f4406b4d0 0x7f5f4406d990 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f5f5400b810 tx=0x7f5f54008040 comp rx=0 tx=0).stop 2024-03-06T07:31:53.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d47b0 msgr2=0x7f5f64152f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:53.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d47b0 0x7f5f64152f60 secure :-1 s=READY pgs=1017 cs=0 l=1 rev1=1 crypto rx=0x7f5f5000c440 tx=0x7f5f5000c900 comp rx=0 tx=0).stop 2024-03-06T07:31:53.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 shutdown_connections 2024-03-06T07:31:53.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5f4406b4d0 0x7f5f4406d990 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5f6415df30 0x7f5f640b0980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.183+0000 7f5f63e00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5f640d47b0 0x7f5f64152f60 unknown :-1 s=CLOSED pgs=1017 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.184+0000 7f5f63e00640 1 --2- 172.21.15.86:0/276791296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5f640d06a0 0x7f5f640b2400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.184+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 >> 172.21.15.86:0/276791296 conn(0x7f5f640d3c90 msgr2=0x7f5f640d4070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:53.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.185+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 shutdown_connections 2024-03-06T07:31:53.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.185+0000 7f5f63e00640 1 -- 172.21.15.86:0/276791296 wait complete. 2024-03-06T07:31:53.212 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:31:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.419+0000 7f8469000640 1 Processor -- start 2024-03-06T07:31:53.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.419+0000 7f8469000640 1 -- start start 2024-03-06T07:31:53.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8469000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f846415adf0 0x7f846406e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:53.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8469000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f846415f1f0 0x7f84640cd090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:53.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8469000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f846415b1d0 con 0x7f846406e8e0 2024-03-06T07:31:53.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8469000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f84640b2bd0 con 0x7f846415f1f0 2024-03-06T07:31:53.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8469000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f84640b2eb0 con 0x7f846415adf0 2024-03-06T07:31:53.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8461400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f846406e8e0 0x7f846415eae0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44356/0 (socket says 172.21.15.86:44356) 2024-03-06T07:31:53.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8461400640 1 -- 172.21.15.86:0/1352033228 learned_addr learned my addr 172.21.15.86:0/1352033228 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:53.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8462800640 1 --2- 172.21.15.86:0/1352033228 >> v2:172.21.15.86:3300/0 conn(0x7f846415f1f0 0x7f84640cd090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:53.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8461e00640 1 --2- 172.21.15.86:0/1352033228 >> v2:172.21.15.194:3300/0 conn(0x7f846415adf0 0x7f846406e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:53.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.421+0000 7f8462800640 1 -- 172.21.15.86:0/1352033228 >> v2:172.21.15.194:3300/0 conn(0x7f846415adf0 msgr2=0x7f846406e3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:53.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.422+0000 7f8462800640 1 --2- 172.21.15.86:0/1352033228 >> v2:172.21.15.194:3300/0 conn(0x7f846415adf0 0x7f846406e3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.422+0000 7f8462800640 1 -- 172.21.15.86:0/1352033228 >> v1:172.21.15.86:6789/0 conn(0x7f846406e8e0 legacy=0x7f846415eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:53.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.422+0000 7f8462800640 1 -- 172.21.15.86:0/1352033228 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84640b3190 con 0x7f846415f1f0 2024-03-06T07:31:53.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.422+0000 7f8460a00640 1 -- 172.21.15.86:0/1352033228 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2150070671 0 0) 0x7f846415b1d0 con 0x7f846406e8e0 2024-03-06T07:31:53.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.422+0000 7f8461e00640 1 --2- 172.21.15.86:0/1352033228 >> v2:172.21.15.194:3300/0 conn(0x7f846415adf0 0x7f846406e3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.423+0000 7f8462800640 1 --2- 172.21.15.86:0/1352033228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846415f1f0 0x7f84640cd090 secure :-1 s=READY pgs=1019 cs=0 l=1 rev1=1 crypto rx=0x7f8464075000 tx=0x7f845c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=180ccf0581a40269 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.423+0000 7f8460a00640 1 -- 172.21.15.86:0/1352033228 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f845c035020 con 0x7f846415f1f0 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.423+0000 7f8460a00640 1 -- 172.21.15.86:0/1352033228 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f845c031830 con 0x7f846415f1f0 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.423+0000 7f8460a00640 1 -- 172.21.15.86:0/1352033228 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f845c031c00 con 0x7f846415f1f0 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 -- 172.21.15.86:0/1352033228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846415f1f0 msgr2=0x7f84640cd090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 --2- 172.21.15.86:0/1352033228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846415f1f0 0x7f84640cd090 secure :-1 s=READY pgs=1019 cs=0 l=1 rev1=1 crypto rx=0x7f8464075000 tx=0x7f845c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 -- 172.21.15.86:0/1352033228 shutdown_connections 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 --2- 172.21.15.86:0/1352033228 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846415f1f0 0x7f84640cd090 unknown :-1 s=CLOSED pgs=1019 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 --2- 172.21.15.86:0/1352033228 >> v2:172.21.15.194:3300/0 conn(0x7f846415adf0 0x7f846406e3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 -- 172.21.15.86:0/1352033228 >> 172.21.15.86:0/1352033228 conn(0x7f84640cfa80 msgr2=0x7f84640cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:53.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 -- 172.21.15.86:0/1352033228 shutdown_connections 2024-03-06T07:31:53.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.424+0000 7f8469000640 1 -- 172.21.15.86:0/1352033228 wait complete. 2024-03-06T07:31:53.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.425+0000 7f8469000640 1 Processor -- start 2024-03-06T07:31:53.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.426+0000 7f8469000640 1 -- start start 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.426+0000 7f8469000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846406e8e0 0x7f84640c57a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.426+0000 7f8469000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f846415adf0 0x7f846415cbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8469000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f846415f1f0 0x7f846406db90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8469000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84640b3990 con 0x7f846406e8e0 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8469000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f846415f850 con 0x7f846415f1f0 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8469000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f84640cd5d0 con 0x7f846415adf0 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846406e8e0 0x7f84640c57a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846406e8e0 0x7f84640c57a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41200/0 (socket says 172.21.15.86:41200) 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 -- 172.21.15.86:0/3327895882 learned_addr learned my addr 172.21.15.86:0/3327895882 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 -- 172.21.15.86:0/3327895882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f846415adf0 msgr2=0x7f846415cbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f846415adf0 0x7f846415cbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 -- 172.21.15.86:0/3327895882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f846415f1f0 msgr2=0x7f846406db90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:53.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8462800640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f846415f1f0 0x7f846406db90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:53.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f846415f1f0 0x7f846406db90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:53.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.427+0000 7f8461e00640 1 -- 172.21.15.86:0/3327895882 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f846415d440 con 0x7f846406e8e0 2024-03-06T07:31:53.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.428+0000 7f8461e00640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846406e8e0 0x7f84640c57a0 secure :-1 s=READY pgs=1020 cs=0 l=1 rev1=1 crypto rx=0x7f845400c820 tx=0x7f845400cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:53.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.428+0000 7f845a000640 1 -- 172.21.15.86:0/3327895882 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8454015070 con 0x7f846406e8e0 2024-03-06T07:31:53.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.428+0000 7f845a000640 1 -- 172.21.15.86:0/3327895882 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8454007a30 con 0x7f846406e8e0 2024-03-06T07:31:53.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.428+0000 7f845a000640 1 -- 172.21.15.86:0/3327895882 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8454007e00 con 0x7f846406e8e0 2024-03-06T07:31:53.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.428+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f846415d720 con 0x7f846406e8e0 2024-03-06T07:31:53.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.428+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f846415dce0 con 0x7f846406e8e0 2024-03-06T07:31:53.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.429+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f846404f330 con 0x7f846406e8e0 2024-03-06T07:31:53.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.430+0000 7f845a000640 1 -- 172.21.15.86:0/3327895882 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8454018070 con 0x7f846406e8e0 2024-03-06T07:31:53.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.431+0000 7f845a000640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f844006b4d0 0x7f844006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:53.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.431+0000 7f845a000640 1 -- 172.21.15.86:0/3327895882 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 5357+0+0 (secure 0 0 0) 0x7f845408a090 con 0x7f846406e8e0 2024-03-06T07:31:53.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.432+0000 7f8461400640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f844006b4d0 0x7f844006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:53.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.432+0000 7f8461400640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f844006b4d0 0x7f844006d990 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f844c001560 tx=0x7f844c007ca0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:53.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.436+0000 7f845a000640 1 -- 172.21.15.86:0/3327895882 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8454059060 con 0x7f846406e8e0 2024-03-06T07:31:53.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:53.634+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f846406cd10 con 0x7f846406e8e0 2024-03-06T07:31:53.684 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:31:53.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:31:54.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.193+0000 7f845a000640 1 -- 172.21.15.86:0/3327895882 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v86) v1 ==== 169+0+0 (secure 0 0 0) 0x7f8454058960 con 0x7f846406e8e0 2024-03-06T07:31:54.196 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f844006b4d0 msgr2=0x7f844006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f844006b4d0 0x7f844006d990 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f844c001560 tx=0x7f844c007ca0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846406e8e0 msgr2=0x7f84640c57a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846406e8e0 0x7f84640c57a0 secure :-1 s=READY pgs=1020 cs=0 l=1 rev1=1 crypto rx=0x7f845400c820 tx=0x7f845400cc20 comp rx=0 tx=0).stop 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 shutdown_connections 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f844006b4d0 0x7f844006d990 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f846415f1f0 0x7f846406db90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f846415adf0 0x7f846415cbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 --2- 172.21.15.86:0/3327895882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f846406e8e0 0x7f84640c57a0 unknown :-1 s=CLOSED pgs=1020 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 >> 172.21.15.86:0/3327895882 conn(0x7f84640cfa80 msgr2=0x7f84640cef30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 shutdown_connections 2024-03-06T07:31:54.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.197+0000 7f8469000640 1 -- 172.21.15.86:0/3327895882 wait complete. 2024-03-06T07:31:54.226 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-06T07:31:54.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.415+0000 7f48ae600640 1 Processor -- start 2024-03-06T07:31:54.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.415+0000 7f48ae600640 1 -- start start 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48ae600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f48a80b3f70 0x7f48a8160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48ae600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f48a8070ff0 0x7f48a80ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48ae600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f48a8071440 con 0x7f48a815fc10 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48ae600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f48a80b46c0 con 0x7f48a8070ff0 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48ae600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f48a80b49a0 con 0x7f48a80b3f70 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48a7e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f48a8070ff0 0x7f48a80ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48a7400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f48a815fc10 0x7f48a815fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44358/0 (socket says 172.21.15.86:44358) 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.417+0000 7f48a7400640 1 -- 172.21.15.86:0/608902330 learned_addr learned my addr 172.21.15.86:0/608902330 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.418+0000 7f48a6a00640 1 --2- 172.21.15.86:0/608902330 >> v2:172.21.15.194:3300/0 conn(0x7f48a80b3f70 0x7f48a8160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.418+0000 7f48a7e00640 1 -- 172.21.15.86:0/608902330 >> v2:172.21.15.194:3300/0 conn(0x7f48a80b3f70 msgr2=0x7f48a8160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:54.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.418+0000 7f48a7e00640 1 --2- 172.21.15.86:0/608902330 >> v2:172.21.15.194:3300/0 conn(0x7f48a80b3f70 0x7f48a8160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.418+0000 7f48a7e00640 1 -- 172.21.15.86:0/608902330 >> v1:172.21.15.86:6789/0 conn(0x7f48a815fc10 legacy=0x7f48a815fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.418+0000 7f48a7e00640 1 -- 172.21.15.86:0/608902330 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48a8072c90 con 0x7f48a8070ff0 2024-03-06T07:31:54.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.418+0000 7f48a6000640 1 -- 172.21.15.86:0/608902330 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2589809883 0 0) 0x7f48a8071440 con 0x7f48a815fc10 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.419+0000 7f48a7e00640 1 --2- 172.21.15.86:0/608902330 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a8070ff0 0x7f48a80ce4a0 secure :-1 s=READY pgs=1022 cs=0 l=1 rev1=1 crypto rx=0x7f48a8074a50 tx=0x7f48a002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=318cd12ae1be2d8e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.419+0000 7f48a6000640 1 -- 172.21.15.86:0/608902330 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48a0039070 con 0x7f48a8070ff0 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.419+0000 7f48a6000640 1 -- 172.21.15.86:0/608902330 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f48a002cc80 con 0x7f48a8070ff0 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.419+0000 7f48a6000640 1 -- 172.21.15.86:0/608902330 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48a0034660 con 0x7f48a8070ff0 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 -- 172.21.15.86:0/608902330 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a8070ff0 msgr2=0x7f48a80ce4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 --2- 172.21.15.86:0/608902330 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a8070ff0 0x7f48a80ce4a0 secure :-1 s=READY pgs=1022 cs=0 l=1 rev1=1 crypto rx=0x7f48a8074a50 tx=0x7f48a002bed0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 -- 172.21.15.86:0/608902330 shutdown_connections 2024-03-06T07:31:54.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 --2- 172.21.15.86:0/608902330 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a8070ff0 0x7f48a80ce4a0 unknown :-1 s=CLOSED pgs=1022 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 --2- 172.21.15.86:0/608902330 >> v2:172.21.15.194:3300/0 conn(0x7f48a80b3f70 0x7f48a8160700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 -- 172.21.15.86:0/608902330 >> 172.21.15.86:0/608902330 conn(0x7f48a80b1940 msgr2=0x7f48a80b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:54.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 -- 172.21.15.86:0/608902330 shutdown_connections 2024-03-06T07:31:54.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.420+0000 7f48ae600640 1 -- 172.21.15.86:0/608902330 wait complete. 2024-03-06T07:31:54.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.422+0000 7f48ae600640 1 Processor -- start 2024-03-06T07:31:54.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.422+0000 7f48ae600640 1 -- start start 2024-03-06T07:31:54.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48ae600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48a8070ff0 0x7f48a814de90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48ae600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48a80b3f70 0x7f48a815c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48ae600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a815fc10 0x7f48a815a920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48ae600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48a8072a30 con 0x7f48a815fc10 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48ae600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48a80728b0 con 0x7f48a8070ff0 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48ae600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f48a815c2e0 con 0x7f48a80b3f70 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48a7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a815fc10 0x7f48a815a920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48a7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a815fc10 0x7f48a815a920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41210/0 (socket says 172.21.15.86:41210) 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.423+0000 7f48a7e00640 1 -- 172.21.15.86:0/3675041414 learned_addr learned my addr 172.21.15.86:0/3675041414 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f48a7e00640 1 -- 172.21.15.86:0/3675041414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48a80b3f70 msgr2=0x7f48a815c5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f48a7e00640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48a80b3f70 0x7f48a815c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f48a7e00640 1 -- 172.21.15.86:0/3675041414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48a8070ff0 msgr2=0x7f48a814de90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f48a7e00640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48a8070ff0 0x7f48a814de90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f48a7e00640 1 -- 172.21.15.86:0/3675041414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48a81544d0 con 0x7f48a815fc10 2024-03-06T07:31:54.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f48a7e00640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a815fc10 0x7f48a815a920 secure :-1 s=READY pgs=1023 cs=0 l=1 rev1=1 crypto rx=0x7f48a0037040 tx=0x7f48a0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:54.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f489f400640 1 -- 172.21.15.86:0/3675041414 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48a0044020 con 0x7f48a815fc10 2024-03-06T07:31:54.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f489f400640 1 -- 172.21.15.86:0/3675041414 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f48a003a040 con 0x7f48a815fc10 2024-03-06T07:31:54.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f489f400640 1 -- 172.21.15.86:0/3675041414 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48a0039050 con 0x7f48a815fc10 2024-03-06T07:31:54.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.424+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f48a81547b0 con 0x7f48a815fc10 2024-03-06T07:31:54.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.425+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48a8154d70 con 0x7f48a815fc10 2024-03-06T07:31:54.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.426+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f48a804f330 con 0x7f48a815fc10 2024-03-06T07:31:54.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.427+0000 7f489f400640 1 -- 172.21.15.86:0/3675041414 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f48a0003b90 con 0x7f48a815fc10 2024-03-06T07:31:54.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.427+0000 7f489f400640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f488806b500 0x7f488806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.428+0000 7f489f400640 1 -- 172.21.15.86:0/3675041414 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f48a00ac7a0 con 0x7f48a815fc10 2024-03-06T07:31:54.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.428+0000 7f48a7400640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f488806b500 0x7f488806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.429+0000 7f48a7400640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f488806b500 0x7f488806d9c0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f4898005880 tx=0x7f4898007a00 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:54.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.432+0000 7f489f400640 1 -- 172.21.15.86:0/3675041414 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f48a004e020 con 0x7f48a815fc10 2024-03-06T07:31:54.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.626+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f48a8154f40 con 0x7f48a815fc10 2024-03-06T07:31:54.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.633+0000 7f489f400640 1 -- 172.21.15.86:0/3675041414 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v27) v1 ==== 104+0+0 (secure 0 0 0) 0x7f48a007b910 con 0x7f48a815fc10 2024-03-06T07:31:54.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f488806b500 msgr2=0x7f488806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f488806b500 0x7f488806d9c0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f4898005880 tx=0x7f4898007a00 comp rx=0 tx=0).stop 2024-03-06T07:31:54.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a815fc10 msgr2=0x7f48a815a920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a815fc10 0x7f48a815a920 secure :-1 s=READY pgs=1023 cs=0 l=1 rev1=1 crypto rx=0x7f48a0037040 tx=0x7f48a0004a40 comp rx=0 tx=0).stop 2024-03-06T07:31:54.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 shutdown_connections 2024-03-06T07:31:54.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f488806b500 0x7f488806d9c0 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f48a815fc10 0x7f48a815a920 unknown :-1 s=CLOSED pgs=1023 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f48a80b3f70 0x7f48a815c5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 --2- 172.21.15.86:0/3675041414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f48a8070ff0 0x7f48a814de90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.636+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 >> 172.21.15.86:0/3675041414 conn(0x7f48a80b1940 msgr2=0x7f48a80cda90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:54.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.637+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 shutdown_connections 2024-03-06T07:31:54.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.637+0000 7f48ae600640 1 -- 172.21.15.86:0/3675041414 wait complete. 2024-03-06T07:31:54.658 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_fancy_layout' 2024-03-06T07:31:54.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.857+0000 7f0bc1400640 1 Processor -- start 2024-03-06T07:31:54.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.857+0000 7f0bc1400640 1 -- start start 2024-03-06T07:31:54.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.859+0000 7f0bc1400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0bbc06ebe0 0x7f0bbc070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.859+0000 7f0bc1400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0bbc071d30 con 0x7f0bbc15eff0 2024-03-06T07:31:54.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.859+0000 7f0bc1400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0bbc15f4a0 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.859+0000 7f0bc1400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0bbc071260 con 0x7f0bbc06ebe0 2024-03-06T07:31:54.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.859+0000 7f0bbb200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0bbc06ebe0 0x7f0bbc070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.859+0000 7f0bb9e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0bbc15eff0 0x7f0bbc06e4d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44374/0 (socket says 172.21.15.86:44374) 2024-03-06T07:31:54.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.859+0000 7f0bb9e00640 1 -- 172.21.15.86:0/1424883889 learned_addr learned my addr 172.21.15.86:0/1424883889 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:54.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.860+0000 7f0bbb200640 1 -- 172.21.15.86:0/1424883889 >> v1:172.21.15.194:6789/0 conn(0x7f0bbc0af1c0 legacy=0x7f0bbc06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:54.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.860+0000 7f0bbb200640 1 -- 172.21.15.86:0/1424883889 >> v1:172.21.15.86:6789/0 conn(0x7f0bbc15eff0 legacy=0x7f0bbc06e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.860+0000 7f0bbb200640 1 -- 172.21.15.86:0/1424883889 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bbc071490 con 0x7f0bbc06ebe0 2024-03-06T07:31:54.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.860+0000 7f0bb9400640 1 -- 172.21.15.86:0/1424883889 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1651142991 0 0) 0x7f0ba8001110 con 0x7f0bbc15eff0 2024-03-06T07:31:54.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.861+0000 7f0bbb200640 1 --2- 172.21.15.86:0/1424883889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc070d20 secure :-1 s=READY pgs=1025 cs=0 l=1 rev1=1 crypto rx=0x7f0bb40097d0 tx=0x7f0bb402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2c631ace0b5dda60 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:54.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.861+0000 7f0bb9400640 1 -- 172.21.15.86:0/1424883889 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bb4035020 con 0x7f0bbc06ebe0 2024-03-06T07:31:54.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.861+0000 7f0bb9400640 1 -- 172.21.15.86:0/1424883889 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0bb4031830 con 0x7f0bbc06ebe0 2024-03-06T07:31:54.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.861+0000 7f0bb9400640 1 -- 172.21.15.86:0/1424883889 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bb4031c00 con 0x7f0bbc06ebe0 2024-03-06T07:31:54.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.862+0000 7f0bc1400640 1 -- 172.21.15.86:0/1424883889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc06ebe0 msgr2=0x7f0bbc070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.862+0000 7f0bc1400640 1 --2- 172.21.15.86:0/1424883889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc070d20 secure :-1 s=READY pgs=1025 cs=0 l=1 rev1=1 crypto rx=0x7f0bb40097d0 tx=0x7f0bb402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.862+0000 7f0bc1400640 1 -- 172.21.15.86:0/1424883889 shutdown_connections 2024-03-06T07:31:54.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.862+0000 7f0bc1400640 1 --2- 172.21.15.86:0/1424883889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc070d20 unknown :-1 s=CLOSED pgs=1025 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.862+0000 7f0bc1400640 1 -- 172.21.15.86:0/1424883889 >> 172.21.15.86:0/1424883889 conn(0x7f0bbc0b1730 msgr2=0x7f0bbc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:54.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.862+0000 7f0bc1400640 1 -- 172.21.15.86:0/1424883889 shutdown_connections 2024-03-06T07:31:54.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.862+0000 7f0bc1400640 1 -- 172.21.15.86:0/1424883889 wait complete. 2024-03-06T07:31:54.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.864+0000 7f0bc1400640 1 Processor -- start 2024-03-06T07:31:54.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.864+0000 7f0bc1400640 1 -- start start 2024-03-06T07:31:54.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bc1400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc0ce2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bc1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc0af1c0 0x7f0bbc1e7ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bc1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0bbc15eff0 0x7f0bbc1e8420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bc1400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bbc0b30b0 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bc1400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bbc071a50 con 0x7f0bbc06ebe0 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bc1400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0bbc06d370 con 0x7f0bbc15eff0 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc0af1c0 0x7f0bbc1e7ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc0af1c0 0x7f0bbc1e7ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41224/0 (socket says 172.21.15.86:41224) 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 -- 172.21.15.86:0/3183223051 learned_addr learned my addr 172.21.15.86:0/3183223051 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bba800640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc0ce2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 -- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0bbc15eff0 msgr2=0x7f0bbc1e8420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0bbc15eff0 0x7f0bbc1e8420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 -- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0bbc06ebe0 msgr2=0x7f0bbc0ce2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc0ce2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bb9e00640 1 -- 172.21.15.86:0/3183223051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bbc1f42b0 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.865+0000 7f0bba800640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc0ce2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.866+0000 7f0bb9e00640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc0af1c0 0x7f0bbc1e7ee0 secure :-1 s=READY pgs=1026 cs=0 l=1 rev1=1 crypto rx=0x7f0ba800c430 tx=0x7f0ba800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.866+0000 7f0bb2a00640 1 -- 172.21.15.86:0/3183223051 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ba80074e0 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.866+0000 7f0bb2a00640 1 -- 172.21.15.86:0/3183223051 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ba80047e0 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.866+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0bbc15b340 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.866+0000 7f0bb2a00640 1 -- 172.21.15.86:0/3183223051 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ba8004bd0 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.866+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0bbc0cec40 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.867+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0bbc15fd90 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.868+0000 7f0bb2a00640 1 -- 172.21.15.86:0/3183223051 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0ba8011070 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.869+0000 7f0bb2a00640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b9c06b4d0 0x7f0b9c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:54.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.869+0000 7f0bb2a00640 1 -- 172.21.15.86:0/3183223051 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f0ba8089b70 con 0x7f0bbc0af1c0 2024-03-06T07:31:54.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.869+0000 7f0bba800640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b9c06b4d0 0x7f0b9c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:54.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.870+0000 7f0bba800640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b9c06b4d0 0x7f0b9c06d990 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7f0bbc0cf660 tx=0x7f0bac009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:54.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:54.873+0000 7f0bb2a00640 1 -- 172.21.15.86:0/3183223051 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0bbc15fd90 con 0x7f0bbc0af1c0 2024-03-06T07:31:55.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.141+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]} v 0) v1 -- 0x7f0bbc04f330 con 0x7f0bbc0af1c0 2024-03-06T07:31:55.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.196+0000 7f0bb2a00640 1 -- 172.21.15.86:0/3183223051 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]}]=0 v281) v1 ==== 132+0+0 (secure 0 0 0) 0x7f0bbc071d30 con 0x7f0bbc0af1c0 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b9c06b4d0 msgr2=0x7f0b9c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b9c06b4d0 0x7f0b9c06d990 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7f0bbc0cf660 tx=0x7f0bac009040 comp rx=0 tx=0).stop 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc0af1c0 msgr2=0x7f0bbc1e7ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc0af1c0 0x7f0bbc1e7ee0 secure :-1 s=READY pgs=1026 cs=0 l=1 rev1=1 crypto rx=0x7f0ba800c430 tx=0x7f0ba800c8f0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 shutdown_connections 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b9c06b4d0 0x7f0b9c06d990 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0bbc15eff0 0x7f0bbc1e8420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0bbc0af1c0 0x7f0bbc1e7ee0 unknown :-1 s=CLOSED pgs=1026 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 --2- 172.21.15.86:0/3183223051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0bbc06ebe0 0x7f0bbc0ce2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 >> 172.21.15.86:0/3183223051 conn(0x7f0bbc0b1730 msgr2=0x7f0bbc15f680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:55.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 shutdown_connections 2024-03-06T07:31:55.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.199+0000 7f0bc1400640 1 -- 172.21.15.86:0/3183223051 wait complete. 2024-03-06T07:31:55.220 INFO:tasks.cephfs_test_runner:purge stray file with fancy layout ... ok 2024-03-06T07:31:55.221 INFO:tasks.cephfs_test_runner:Starting test: test_files_throttle (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:31:55.221 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:55.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.423+0000 7fb94aa00640 1 Processor -- start 2024-03-06T07:31:55.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.423+0000 7fb94aa00640 1 -- start start 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb94aa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb94415acb0 0x7fb9440b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb94aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb94406ec00 0x7fb9440ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb94aa00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb94415b090 con 0x7fb9440b4680 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb94aa00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb9440b2a30 con 0x7fb94406ec00 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb94aa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb9440b4360 con 0x7fb94415acb0 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb948a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb94406ec00 0x7fb9440ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb948a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb94406ec00 0x7fb9440ccfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41238/0 (socket says 172.21.15.86:41238) 2024-03-06T07:31:55.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.425+0000 7fb948a00640 1 -- 172.21.15.86:0/341834176 learned_addr learned my addr 172.21.15.86:0/341834176 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:55.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.426+0000 7fb943e00640 1 --2- 172.21.15.86:0/341834176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb94415acb0 0x7fb9440b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.426+0000 7fb948a00640 1 -- 172.21.15.86:0/341834176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb94415acb0 msgr2=0x7fb9440b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:55.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.426+0000 7fb948a00640 1 --2- 172.21.15.86:0/341834176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb94415acb0 0x7fb9440b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.426+0000 7fb948a00640 1 -- 172.21.15.86:0/341834176 >> v1:172.21.15.194:6789/0 conn(0x7fb9440b4680 legacy=0x7fb94406e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:55.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.426+0000 7fb948a00640 1 -- 172.21.15.86:0/341834176 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9440b2cf0 con 0x7fb94406ec00 2024-03-06T07:31:55.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.427+0000 7fb948a00640 1 --2- 172.21.15.86:0/341834176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94406ec00 0x7fb9440ccfb0 secure :-1 s=READY pgs=1027 cs=0 l=1 rev1=1 crypto rx=0x7fb93c009870 tx=0x7fb93c02db00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c338f8d595d086f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:55.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.427+0000 7fb942a00640 1 -- 172.21.15.86:0/341834176 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb93c02f8c0 con 0x7fb94406ec00 2024-03-06T07:31:55.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.427+0000 7fb942a00640 1 -- 172.21.15.86:0/341834176 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb93c02fa60 con 0x7fb94406ec00 2024-03-06T07:31:55.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.427+0000 7fb942a00640 1 -- 172.21.15.86:0/341834176 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb93c02fe30 con 0x7fb94406ec00 2024-03-06T07:31:55.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 -- 172.21.15.86:0/341834176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94406ec00 msgr2=0x7fb9440ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 --2- 172.21.15.86:0/341834176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94406ec00 0x7fb9440ccfb0 secure :-1 s=READY pgs=1027 cs=0 l=1 rev1=1 crypto rx=0x7fb93c009870 tx=0x7fb93c02db00 comp rx=0 tx=0).stop 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 -- 172.21.15.86:0/341834176 shutdown_connections 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 --2- 172.21.15.86:0/341834176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94406ec00 0x7fb9440ccfb0 unknown :-1 s=CLOSED pgs=1027 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 --2- 172.21.15.86:0/341834176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb94415acb0 0x7fb9440b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 -- 172.21.15.86:0/341834176 >> 172.21.15.86:0/341834176 conn(0x7fb9440b1730 msgr2=0x7fb9440b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 -- 172.21.15.86:0/341834176 shutdown_connections 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.428+0000 7fb94aa00640 1 -- 172.21.15.86:0/341834176 wait complete. 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.430+0000 7fb94aa00640 1 Processor -- start 2024-03-06T07:31:55.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.430+0000 7fb94aa00640 1 -- start start 2024-03-06T07:31:55.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb94aa00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb94406ec00 0x7fb944146760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb94aa00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9440b4680 0x7fb9440ce5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb94aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94415acb0 0x7fb9440ceb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb94aa00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9440b3d10 con 0x7fb94415acb0 2024-03-06T07:31:55.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb94aa00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9440b32b0 con 0x7fb94406ec00 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb94aa00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb9440b4010 con 0x7fb9440b4680 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb948a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94415acb0 0x7fb9440ceb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb948a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94415acb0 0x7fb9440ceb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41242/0 (socket says 172.21.15.86:41242) 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb948a00640 1 -- 172.21.15.86:0/2395043819 learned_addr learned my addr 172.21.15.86:0/2395043819 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb943e00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb94406ec00 0x7fb944146760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.431+0000 7fb943400640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9440b4680 0x7fb9440ce5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb948a00640 1 -- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9440b4680 msgr2=0x7fb9440ce5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb948a00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9440b4680 0x7fb9440ce5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb948a00640 1 -- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb94406ec00 msgr2=0x7fb944146760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb948a00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb94406ec00 0x7fb944146760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb948a00640 1 -- 172.21.15.86:0/2395043819 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9440ae590 con 0x7fb94415acb0 2024-03-06T07:31:55.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb943400640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9440b4680 0x7fb9440ce5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb948a00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94415acb0 0x7fb9440ceb00 secure :-1 s=READY pgs=1028 cs=0 l=1 rev1=1 crypto rx=0x7fb93c02db40 tx=0x7fb93c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb93be00640 1 -- 172.21.15.86:0/2395043819 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb93c046020 con 0x7fb94415acb0 2024-03-06T07:31:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb93be00640 1 -- 172.21.15.86:0/2395043819 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb93c042880 con 0x7fb94415acb0 2024-03-06T07:31:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb9440ae870 con 0x7fb94415acb0 2024-03-06T07:31:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.432+0000 7fb93be00640 1 -- 172.21.15.86:0/2395043819 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb93c042c70 con 0x7fb94415acb0 2024-03-06T07:31:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.433+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9440aee30 con 0x7fb94415acb0 2024-03-06T07:31:55.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.434+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb94404f330 con 0x7fb94415acb0 2024-03-06T07:31:55.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.435+0000 7fb93be00640 1 -- 172.21.15.86:0/2395043819 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb93c049020 con 0x7fb94415acb0 2024-03-06T07:31:55.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.435+0000 7fb93be00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb92406b4d0 0x7fb92406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.436+0000 7fb93be00640 1 -- 172.21.15.86:0/2395043819 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fb93c0ac670 con 0x7fb94415acb0 2024-03-06T07:31:55.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.436+0000 7fb943e00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb92406b4d0 0x7fb92406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.436+0000 7fb943e00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb92406b4d0 0x7fb92406d990 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fb9441f7ab0 tx=0x7fb9340039b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:55.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.440+0000 7fb93be00640 1 -- 172.21.15.86:0/2395043819 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb93c050030 con 0x7fb94415acb0 2024-03-06T07:31:55.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.636+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fb94406ce60 con 0x7fb94415acb0 2024-03-06T07:31:55.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.637+0000 7fb93be00640 1 -- 172.21.15.86:0/2395043819 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v86) v1 ==== 74+0+13031 (secure 0 0 0) 0x7fb93c07b7e0 con 0x7fb94415acb0 2024-03-06T07:31:55.640 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:55.641 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":86,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:54.191153+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/255958031":"2024-03-07T07:31:51.170944+0000","172.21.15.86:6835/255958031":"2024-03-07T07:31:51.170944+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.640+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb92406b4d0 msgr2=0x7fb92406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.640+0000 7fb94aa00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb92406b4d0 0x7fb92406d990 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fb9441f7ab0 tx=0x7fb9340039b0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94415acb0 msgr2=0x7fb9440ceb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94415acb0 0x7fb9440ceb00 secure :-1 s=READY pgs=1028 cs=0 l=1 rev1=1 crypto rx=0x7fb93c02db40 tx=0x7fb93c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 shutdown_connections 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb92406b4d0 0x7fb92406d990 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb94415acb0 0x7fb9440ceb00 unknown :-1 s=CLOSED pgs=1028 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9440b4680 0x7fb9440ce5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 --2- 172.21.15.86:0/2395043819 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb94406ec00 0x7fb944146760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 >> 172.21.15.86:0/2395043819 conn(0x7fb9440b1730 msgr2=0x7fb9440af8e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:55.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 shutdown_connections 2024-03-06T07:31:55.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.641+0000 7fb94aa00640 1 -- 172.21.15.86:0/2395043819 wait complete. 2024-03-06T07:31:55.662 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:31:55.662 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:31:55.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.850+0000 7f9a08e00640 1 Processor -- start 2024-03-06T07:31:55.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.850+0000 7f9a08e00640 1 -- start start 2024-03-06T07:31:55.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.852+0000 7f9a08e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a040b2c40 0x7f9a0415f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.852+0000 7f9a08e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9a0406e2f0 0x7f9a040cda90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.852+0000 7f9a08e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9a0406cba0 con 0x7f9a0406d960 2024-03-06T07:31:55.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.852+0000 7f9a08e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9a0415f700 con 0x7f9a0406e2f0 2024-03-06T07:31:55.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.852+0000 7f9a08e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a0406d4d0 con 0x7f9a040b2c40 2024-03-06T07:31:55.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.852+0000 7f99fbe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9a0406d960 0x7f9a040b3020 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44380/0 (socket says 172.21.15.86:44380) 2024-03-06T07:31:55.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.852+0000 7f99fbe00640 1 -- 172.21.15.86:0/717022407 learned_addr learned my addr 172.21.15.86:0/717022407 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:55.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01e00640 1 --2- 172.21.15.86:0/717022407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a040b2c40 0x7f9a0415f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a02800640 1 --2- 172.21.15.86:0/717022407 >> v2:172.21.15.194:3300/0 conn(0x7f9a0406e2f0 0x7f9a040cda90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01400640 1 -- 172.21.15.86:0/717022407 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2835050728 0 0) 0x7f9a0406cba0 con 0x7f9a0406d960 2024-03-06T07:31:55.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01400640 1 -- 172.21.15.86:0/717022407 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f99e8003520 con 0x7f9a0406d960 2024-03-06T07:31:55.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01400640 1 -- 172.21.15.86:0/717022407 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4041130596 0 0) 0x7f99e8003520 con 0x7f9a0406d960 2024-03-06T07:31:55.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01e00640 1 -- 172.21.15.86:0/717022407 >> v2:172.21.15.194:3300/0 conn(0x7f9a0406e2f0 msgr2=0x7f9a040cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:55.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01e00640 1 --2- 172.21.15.86:0/717022407 >> v2:172.21.15.194:3300/0 conn(0x7f9a0406e2f0 0x7f9a040cda90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01e00640 1 -- 172.21.15.86:0/717022407 >> v1:172.21.15.86:6789/0 conn(0x7f9a0406d960 legacy=0x7f9a040b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.853+0000 7f9a01e00640 1 -- 172.21.15.86:0/717022407 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a0406ce80 con 0x7f9a040b2c40 2024-03-06T07:31:55.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.855+0000 7f9a01e00640 1 --2- 172.21.15.86:0/717022407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a040b2c40 0x7f9a0415f1c0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f99f40097a0 tx=0x7f99f402b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=668d3a18e98f47aa server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:55.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.855+0000 7f9a01400640 1 -- 172.21.15.86:0/717022407 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99f4037070 con 0x7f9a040b2c40 2024-03-06T07:31:55.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a01400640 1 -- 172.21.15.86:0/717022407 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99f4030510 con 0x7f9a040b2c40 2024-03-06T07:31:55.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a01400640 1 -- 172.21.15.86:0/717022407 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99ec001670 con 0x7f9a040b2c40 2024-03-06T07:31:55.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 -- 172.21.15.86:0/717022407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a040b2c40 msgr2=0x7f9a0415f1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 --2- 172.21.15.86:0/717022407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a040b2c40 0x7f9a0415f1c0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f99f40097a0 tx=0x7f99f402b790 comp rx=0 tx=0).stop 2024-03-06T07:31:55.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 -- 172.21.15.86:0/717022407 shutdown_connections 2024-03-06T07:31:55.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 --2- 172.21.15.86:0/717022407 >> v2:172.21.15.194:3300/0 conn(0x7f9a0406e2f0 0x7f9a040cda90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 --2- 172.21.15.86:0/717022407 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a040b2c40 0x7f9a0415f1c0 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 -- 172.21.15.86:0/717022407 >> 172.21.15.86:0/717022407 conn(0x7f9a040b1940 msgr2=0x7f9a040b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:55.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 -- 172.21.15.86:0/717022407 shutdown_connections 2024-03-06T07:31:55.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.856+0000 7f9a08e00640 1 -- 172.21.15.86:0/717022407 wait complete. 2024-03-06T07:31:55.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.858+0000 7f9a08e00640 1 Processor -- start 2024-03-06T07:31:55.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.858+0000 7f9a08e00640 1 -- start start 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a08e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a0406d960 0x7f9a041e7d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a08e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a0406e2f0 0x7f9a0409a9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a08e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a040b2c40 0x7f9a041e9040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a08e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a0406e8d0 con 0x7f9a0406d960 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a08e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a0406e750 con 0x7f9a040b2c40 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a08e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a040cdfd0 con 0x7f9a0406e2f0 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a01e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a0406d960 0x7f9a041e7d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a01e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a0406d960 0x7f9a041e7d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:41244/0 (socket says 172.21.15.86:41244) 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a01e00640 1 -- 172.21.15.86:0/2756979759 learned_addr learned my addr 172.21.15.86:0/2756979759 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a01e00640 1 -- 172.21.15.86:0/2756979759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a0406e2f0 msgr2=0x7f9a0409a9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a01e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a0406e2f0 0x7f9a0409a9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.859+0000 7f9a02800640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a040b2c40 0x7f9a041e9040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f9a01e00640 1 -- 172.21.15.86:0/2756979759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a040b2c40 msgr2=0x7f9a041e9040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f9a01e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a040b2c40 0x7f9a041e9040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f9a01e00640 1 -- 172.21.15.86:0/2756979759 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a0409b490 con 0x7f9a0406d960 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f9a01e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a0406d960 0x7f9a041e7d70 secure :-1 s=READY pgs=1030 cs=0 l=1 rev1=1 crypto rx=0x7f99f4035040 tx=0x7f99f40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f99fa000640 1 -- 172.21.15.86:0/2756979759 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99f4041020 con 0x7f9a0406d960 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f99fa000640 1 -- 172.21.15.86:0/2756979759 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99f40044c0 con 0x7f9a0406d960 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f99fa000640 1 -- 172.21.15.86:0/2756979759 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99f4037040 con 0x7f9a0406d960 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9a0409b770 con 0x7f9a0406d960 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.860+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9a0409bd30 con 0x7f9a0406d960 2024-03-06T07:31:55.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.861+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9a0404f330 con 0x7f9a0406d960 2024-03-06T07:31:55.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.862+0000 7f99fa000640 1 -- 172.21.15.86:0/2756979759 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f99f4044020 con 0x7f9a0406d960 2024-03-06T07:31:55.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.863+0000 7f99fa000640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99e006b4d0 0x7f99e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:55.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.863+0000 7f99fa000640 1 -- 172.21.15.86:0/2756979759 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f99f40ab8a0 con 0x7f9a0406d960 2024-03-06T07:31:55.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.864+0000 7f99fbe00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99e006b4d0 0x7f99e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:55.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.864+0000 7f99fbe00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99e006b4d0 0x7f99e006d990 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7f99ec001eb0 tx=0x7f99ec001a40 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:55.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:55.868+0000 7f99fa000640 1 -- 172.21.15.86:0/2756979759 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f99f404b030 con 0x7f9a0406d960 2024-03-06T07:31:56.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.048+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f9a040b2530 con 0x7f9a0406d960 2024-03-06T07:31:56.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.050+0000 7f99fa000640 1 -- 172.21.15.86:0/2756979759 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v86) v1 ==== 93+0+10 (secure 0 0 0) 0x7f99f403be20 con 0x7f9a0406d960 2024-03-06T07:31:56.053 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:31:56.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99e006b4d0 msgr2=0x7f99e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:56.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99e006b4d0 0x7f99e006d990 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7f99ec001eb0 tx=0x7f99ec001a40 comp rx=0 tx=0).stop 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a0406d960 msgr2=0x7f9a041e7d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a0406d960 0x7f9a041e7d70 secure :-1 s=READY pgs=1030 cs=0 l=1 rev1=1 crypto rx=0x7f99f4035040 tx=0x7f99f40046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 shutdown_connections 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f99e006b4d0 0x7f99e006d990 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a040b2c40 0x7f9a041e9040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a0406e2f0 0x7f9a0409a9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 --2- 172.21.15.86:0/2756979759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a0406d960 0x7f9a041e7d70 unknown :-1 s=CLOSED pgs=1030 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.052+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 >> 172.21.15.86:0/2756979759 conn(0x7f9a040b1940 msgr2=0x7f9a0415c7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.053+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 shutdown_connections 2024-03-06T07:31:56.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.053+0000 7f9a08e00640 1 -- 172.21.15.86:0/2756979759 wait complete. 2024-03-06T07:31:56.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_files_throttle' 2024-03-06T07:31:56.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.278+0000 7fde6a600640 1 Processor -- start 2024-03-06T07:31:56.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.279+0000 7fde6a600640 1 -- start start 2024-03-06T07:31:56.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.280+0000 7fde6a600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde640cf430 0x7fde64070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:56.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde6a600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde640717c0 con 0x7fde6406d260 2024-03-06T07:31:56.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde6a600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde64071aa0 con 0x7fde64071260 2024-03-06T07:31:56.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde6a600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fde64071d80 con 0x7fde640cf430 2024-03-06T07:31:56.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde63400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde640cf430 0x7fde64070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:56.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde63e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fde6406d260 0x7fde640b2d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36304/0 (socket says 172.21.15.86:36304) 2024-03-06T07:31:56.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde63e00640 1 -- 172.21.15.86:0/1850871853 learned_addr learned my addr 172.21.15.86:0/1850871853 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:56.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde63400640 1 -- 172.21.15.86:0/1850871853 >> v1:172.21.15.194:6789/0 conn(0x7fde64071260 legacy=0x7fde6406cb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:56.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde63400640 1 -- 172.21.15.86:0/1850871853 >> v1:172.21.15.86:6789/0 conn(0x7fde6406d260 legacy=0x7fde640b2d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:56.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.281+0000 7fde63400640 1 -- 172.21.15.86:0/1850871853 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde64072060 con 0x7fde640cf430 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.282+0000 7fde63400640 1 --2- 172.21.15.86:0/1850871853 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde640cf430 0x7fde64070d20 secure :-1 s=READY pgs=1032 cs=0 l=1 rev1=1 crypto rx=0x7fde540097a0 tx=0x7fde5402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7ef4c53e73b1ca09 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde62000640 1 -- 172.21.15.86:0/1850871853 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde54035020 con 0x7fde640cf430 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde62000640 1 -- 172.21.15.86:0/1850871853 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde54031830 con 0x7fde640cf430 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde62000640 1 -- 172.21.15.86:0/1850871853 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde54031c00 con 0x7fde640cf430 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde6a600640 1 -- 172.21.15.86:0/1850871853 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde640cf430 msgr2=0x7fde64070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde6a600640 1 --2- 172.21.15.86:0/1850871853 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde640cf430 0x7fde64070d20 secure :-1 s=READY pgs=1032 cs=0 l=1 rev1=1 crypto rx=0x7fde540097a0 tx=0x7fde5402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde6a600640 1 -- 172.21.15.86:0/1850871853 shutdown_connections 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde6a600640 1 --2- 172.21.15.86:0/1850871853 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde640cf430 0x7fde64070d20 unknown :-1 s=CLOSED pgs=1032 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.283+0000 7fde6a600640 1 -- 172.21.15.86:0/1850871853 >> 172.21.15.86:0/1850871853 conn(0x7fde640b1730 msgr2=0x7fde640b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:56.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.284+0000 7fde6a600640 1 -- 172.21.15.86:0/1850871853 shutdown_connections 2024-03-06T07:31:56.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.284+0000 7fde6a600640 1 -- 172.21.15.86:0/1850871853 wait complete. 2024-03-06T07:31:56.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.285+0000 7fde6a600640 1 Processor -- start 2024-03-06T07:31:56.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.285+0000 7fde6a600640 1 -- start start 2024-03-06T07:31:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.286+0000 7fde6a600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6406d260 0x7fde64146b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.286+0000 7fde6a600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde64071260 0x7fde641ed310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.286+0000 7fde6a600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde640cf430 0x7fde641ed850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.286+0000 7fde6a600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde64162cf0 con 0x7fde6406d260 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.286+0000 7fde6a600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde64162b70 con 0x7fde640cf430 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.286+0000 7fde6a600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde640b3420 con 0x7fde64071260 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6406d260 0x7fde64146b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6406d260 0x7fde64146b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36602/0 (socket says 172.21.15.86:36602) 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 -- 172.21.15.86:0/583535648 learned_addr learned my addr 172.21.15.86:0/583535648 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63e00640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde640cf430 0x7fde641ed850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde62a00640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde64071260 0x7fde641ed310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 -- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde64071260 msgr2=0x7fde641ed310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde64071260 0x7fde641ed310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 -- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde640cf430 msgr2=0x7fde641ed850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde640cf430 0x7fde641ed850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 -- 172.21.15.86:0/583535648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde64159b60 con 0x7fde6406d260 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.287+0000 7fde63400640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6406d260 0x7fde64146b10 secure :-1 s=READY pgs=1033 cs=0 l=1 rev1=1 crypto rx=0x7fde54000c00 tx=0x7fde540046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.288+0000 7fde5b400640 1 -- 172.21.15.86:0/583535648 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde5403aa70 con 0x7fde6406d260 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.288+0000 7fde5b400640 1 -- 172.21.15.86:0/583535648 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde5403ac10 con 0x7fde6406d260 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.288+0000 7fde5b400640 1 -- 172.21.15.86:0/583535648 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde540443f0 con 0x7fde6406d260 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.288+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde64159e40 con 0x7fde6406d260 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.288+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde6415a400 con 0x7fde6406d260 2024-03-06T07:31:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.289+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde6404f330 con 0x7fde6406d260 2024-03-06T07:31:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.290+0000 7fde5b400640 1 -- 172.21.15.86:0/583535648 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fde5404b060 con 0x7fde6406d260 2024-03-06T07:31:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.291+0000 7fde5b400640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b4d0 0x7fde4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.291+0000 7fde5b400640 1 -- 172.21.15.86:0/583535648 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fde540abc40 con 0x7fde6406d260 2024-03-06T07:31:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.291+0000 7fde62a00640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b4d0 0x7fde4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:56.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.292+0000 7fde62a00640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b4d0 0x7fde4406d990 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fde6406cb00 tx=0x7fde50009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.295+0000 7fde5b400640 1 -- 172.21.15.86:0/583535648 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fde5403f070 con 0x7fde6406d260 2024-03-06T07:31:56.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:56.564+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]} v 0) v1 -- 0x7fde640b4a40 con 0x7fde6406d260 2024-03-06T07:31:57.205 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.202+0000 7fde5b400640 1 -- 172.21.15.86:0/583535648 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]}]=0 v283) v1 ==== 137+0+0 (secure 0 0 0) 0x7fde5407adc0 con 0x7fde6406d260 2024-03-06T07:31:57.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.205+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b4d0 msgr2=0x7fde4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.205+0000 7fde6a600640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b4d0 0x7fde4406d990 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fde6406cb00 tx=0x7fde50009040 comp rx=0 tx=0).stop 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.205+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6406d260 msgr2=0x7fde64146b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.205+0000 7fde6a600640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6406d260 0x7fde64146b10 secure :-1 s=READY pgs=1033 cs=0 l=1 rev1=1 crypto rx=0x7fde54000c00 tx=0x7fde540046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 shutdown_connections 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b4d0 0x7fde4406d990 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde640cf430 0x7fde641ed850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde64071260 0x7fde641ed310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 --2- 172.21.15.86:0/583535648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6406d260 0x7fde64146b10 unknown :-1 s=CLOSED pgs=1033 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 >> 172.21.15.86:0/583535648 conn(0x7fde640b1730 msgr2=0x7fde6406cb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 shutdown_connections 2024-03-06T07:31:57.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.206+0000 7fde6a600640 1 -- 172.21.15.86:0/583535648 wait complete. 2024-03-06T07:31:57.232 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:31:57.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.438+0000 7f6728000640 1 Processor -- start 2024-03-06T07:31:57.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.438+0000 7f6728000640 1 -- start start 2024-03-06T07:31:57.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.440+0000 7f6728000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6720160700 0x7f6720070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.440+0000 7f6728000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67200b46c0 con 0x7f672015fc10 2024-03-06T07:31:57.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.440+0000 7f6728000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67200b49a0 con 0x7f67200b3f70 2024-03-06T07:31:57.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.440+0000 7f6728000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6720072c90 con 0x7f6720160700 2024-03-06T07:31:57.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.440+0000 7f6725600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f672015fc10 0x7f672015fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36310/0 (socket says 172.21.15.86:36310) 2024-03-06T07:31:57.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.440+0000 7f6725600640 1 -- 172.21.15.86:0/735288798 learned_addr learned my addr 172.21.15.86:0/735288798 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:57.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.440+0000 7f6726000640 1 --2- 172.21.15.86:0/735288798 >> v2:172.21.15.194:3300/0 conn(0x7f6720160700 0x7f6720070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:57.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3235893336 0 0) 0x7f67200b46c0 con 0x7f672015fc10 2024-03-06T07:31:57.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6708003520 con 0x7f672015fc10 2024-03-06T07:31:57.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 37695069 0 0) 0x7f6708003520 con 0x7f672015fc10 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 >> v2:172.21.15.194:3300/0 conn(0x7f6720160700 msgr2=0x7f6720070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 --2- 172.21.15.86:0/735288798 >> v2:172.21.15.194:3300/0 conn(0x7f6720160700 0x7f6720070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 >> v1:172.21.15.194:6789/0 conn(0x7f67200b3f70 legacy=0x7f67200ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f672009dd80 con 0x7f672015fc10 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6714001670 con 0x7f672015fc10 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 >> v1:172.21.15.86:6789/0 conn(0x7f672015fc10 legacy=0x7f672015fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f6708003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6708004800 0x7f6708004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67080051c0 0x7f670800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6720072c90 con 0x7f67080039b0 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f670800c060 con 0x7f67080051c0 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.441+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f670800c260 con 0x7f6708004800 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6725600640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f6708003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:57.446 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6726000640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67080051c0 0x7f670800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6725600640 1 -- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6708004800 msgr2=0x7f6708004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6725600640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6708004800 0x7f6708004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6725600640 1 -- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67080051c0 msgr2=0x7f670800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6725600640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67080051c0 0x7f670800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6725600640 1 -- 172.21.15.86:0/735288798 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6708003520 con 0x7f67080039b0 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6726000640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67080051c0 0x7f670800b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.442+0000 7f6726000640 1 -- 172.21.15.86:0/735288798 reap_dead start 2024-03-06T07:31:57.447 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.443+0000 7f6725600640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f6708003dd0 secure :-1 s=READY pgs=1035 cs=0 l=1 rev1=1 crypto rx=0x7f671400c210 tx=0x7f6708005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=634493b34968f8cf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:57.448 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.443+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6714007d30 con 0x7f67080039b0 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.443+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6714030070 con 0x7f67080039b0 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.443+0000 7f671fe00640 1 -- 172.21.15.86:0/735288798 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6714005d70 con 0x7f67080039b0 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.444+0000 7f6728000640 1 -- 172.21.15.86:0/735288798 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 msgr2=0x7f6708003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.444+0000 7f6728000640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f6708003dd0 secure :-1 s=READY pgs=1035 cs=0 l=1 rev1=1 crypto rx=0x7f671400c210 tx=0x7f6708005830 comp rx=0 tx=0).stop 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.444+0000 7f6728000640 1 -- 172.21.15.86:0/735288798 shutdown_connections 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.444+0000 7f6728000640 1 --2- 172.21.15.86:0/735288798 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f6708003dd0 unknown :-1 s=CLOSED pgs=1035 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.444+0000 7f6728000640 1 -- 172.21.15.86:0/735288798 >> 172.21.15.86:0/735288798 conn(0x7f67200b1940 msgr2=0x7f67200b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.444+0000 7f6728000640 1 -- 172.21.15.86:0/735288798 shutdown_connections 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.444+0000 7f6728000640 1 -- 172.21.15.86:0/735288798 wait complete. 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.446+0000 7f6728000640 1 Processor -- start 2024-03-06T07:31:57.449 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.446+0000 7f6728000640 1 -- start start 2024-03-06T07:31:57.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6728000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f672015c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6728000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67201586d0 0x7f672014a4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6728000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f672006d9e0 0x7f672006de60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6728000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f672015c460 con 0x7f67080039b0 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6728000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f672015c2e0 con 0x7f672006d9e0 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6728000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6720070ff0 con 0x7f67201586d0 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6725600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f672015c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6725600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f672015c5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36616/0 (socket says 172.21.15.86:36616) 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.447+0000 7f6725600640 1 -- 172.21.15.86:0/1209324248 learned_addr learned my addr 172.21.15.86:0/1209324248 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6725600640 1 -- 172.21.15.86:0/1209324248 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67201586d0 msgr2=0x7f672014a4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6725600640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67201586d0 0x7f672014a4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6725600640 1 -- 172.21.15.86:0/1209324248 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f672006d9e0 msgr2=0x7f672006de60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6725600640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f672006d9e0 0x7f672006de60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6725600640 1 -- 172.21.15.86:0/1209324248 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f672014a9e0 con 0x7f67080039b0 2024-03-06T07:31:57.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6725600640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f672015c5e0 secure :-1 s=READY pgs=1036 cs=0 l=1 rev1=1 crypto rx=0x7f671400c210 tx=0x7f671402e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:57.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f671d600640 1 -- 172.21.15.86:0/1209324248 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6714004bf0 con 0x7f67080039b0 2024-03-06T07:31:57.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f671d600640 1 -- 172.21.15.86:0/1209324248 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6714000ed0 con 0x7f67080039b0 2024-03-06T07:31:57.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f671d600640 1 -- 172.21.15.86:0/1209324248 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6714007d10 con 0x7f67080039b0 2024-03-06T07:31:57.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f672015d5d0 con 0x7f67080039b0 2024-03-06T07:31:57.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.448+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f672015d010 con 0x7f67080039b0 2024-03-06T07:31:57.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.449+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f672006e3a0 con 0x7f67080039b0 2024-03-06T07:31:57.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.450+0000 7f671d600640 1 -- 172.21.15.86:0/1209324248 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6714048020 con 0x7f67080039b0 2024-03-06T07:31:57.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.451+0000 7f671d600640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f670006b500 0x7f670006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:57.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.451+0000 7f671d600640 1 -- 172.21.15.86:0/1209324248 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f67140ab170 con 0x7f67080039b0 2024-03-06T07:31:57.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.452+0000 7f6724c00640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f670006b500 0x7f670006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:57.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.452+0000 7f6724c00640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f670006b500 0x7f670006d9c0 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f672006e710 tx=0x7f670c0079b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:57.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.456+0000 7f671d600640 1 -- 172.21.15.86:0/1209324248 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f671407a2e0 con 0x7f67080039b0 2024-03-06T07:31:57.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.652+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f672006ef30 con 0x7f67080039b0 2024-03-06T07:31:57.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.657+0000 7f671d600640 1 -- 172.21.15.86:0/1209324248 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v28)=0 v28) v1 ==== 122+0+0 (secure 0 0 0) 0x7f6714079be0 con 0x7f67080039b0 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f670006b500 msgr2=0x7f670006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f670006b500 0x7f670006d9c0 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f672006e710 tx=0x7f670c0079b0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 msgr2=0x7f672015c5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f672015c5e0 secure :-1 s=READY pgs=1036 cs=0 l=1 rev1=1 crypto rx=0x7f671400c210 tx=0x7f671402e070 comp rx=0 tx=0).stop 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 shutdown_connections 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f670006b500 0x7f670006d9c0 unknown :-1 s=CLOSED pgs=454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f672006d9e0 0x7f672006de60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67201586d0 0x7f672014a4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 --2- 172.21.15.86:0/1209324248 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67080039b0 0x7f672015c5e0 unknown :-1 s=CLOSED pgs=1036 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:57.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 >> 172.21.15.86:0/1209324248 conn(0x7f67200b1940 msgr2=0x7f67200acd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:57.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 shutdown_connections 2024-03-06T07:31:57.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:57.660+0000 7f6728000640 1 -- 172.21.15.86:0/1209324248 wait complete. 2024-03-06T07:31:57.681 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:31:57.682 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:57.781 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:31:57.899 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:31:57.899 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:57.899 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:57.955 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:31:57.956 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:57.956 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:58.012 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:58.012 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:31:58.012 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:58.068 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:31:58.068 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:31:58.069 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:31:58.126 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:31:58.126 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:31:58.127 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:31:58.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.339+0000 7ff889200640 1 Processor -- start 2024-03-06T07:31:58.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.339+0000 7ff889200640 1 -- start start 2024-03-06T07:31:58.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff889200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff8840b40c0 0x7ff8840b44a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff889200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff8840ae670 0x7ff88415e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff889200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8841620c0 0x7ff8840b28e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff889200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff884161900 con 0x7ff8840ae670 2024-03-06T07:31:58.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff889200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff8840b49e0 con 0x7ff8840b40c0 2024-03-06T07:31:58.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff889200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff88415ebd0 con 0x7ff8841620c0 2024-03-06T07:31:58.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff882a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff8840ae670 0x7ff88415e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff882a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff8840ae670 0x7ff88415e2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36624/0 (socket says 172.21.15.86:36624) 2024-03-06T07:31:58.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff882a00640 1 -- 172.21.15.86:0/3036122337 learned_addr learned my addr 172.21.15.86:0/3036122337 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff883e00640 1 --2- 172.21.15.86:0/3036122337 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8841620c0 0x7ff8840b28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.341+0000 7ff883400640 1 --2- 172.21.15.86:0/3036122337 >> v2:172.21.15.194:3300/0 conn(0x7ff8840b40c0 0x7ff8840b44a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.342+0000 7ff882a00640 1 -- 172.21.15.86:0/3036122337 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8841620c0 msgr2=0x7ff8840b28e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.342+0000 7ff882a00640 1 --2- 172.21.15.86:0/3036122337 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8841620c0 0x7ff8840b28e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.342+0000 7ff882a00640 1 -- 172.21.15.86:0/3036122337 >> v2:172.21.15.194:3300/0 conn(0x7ff8840b40c0 msgr2=0x7ff8840b44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.342+0000 7ff882a00640 1 --2- 172.21.15.86:0/3036122337 >> v2:172.21.15.194:3300/0 conn(0x7ff8840b40c0 0x7ff8840b44a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.342+0000 7ff882a00640 1 -- 172.21.15.86:0/3036122337 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff88415eeb0 con 0x7ff8840ae670 2024-03-06T07:31:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.343+0000 7ff882a00640 1 --2- 172.21.15.86:0/3036122337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840ae670 0x7ff88415e2c0 secure :-1 s=READY pgs=1037 cs=0 l=1 rev1=1 crypto rx=0x7ff8840ad980 tx=0x7ff87002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a2c16052003281d7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:58.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.343+0000 7ff882000640 1 -- 172.21.15.86:0/3036122337 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff870035020 con 0x7ff8840ae670 2024-03-06T07:31:58.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.343+0000 7ff882000640 1 -- 172.21.15.86:0/3036122337 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff870031870 con 0x7ff8840ae670 2024-03-06T07:31:58.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.343+0000 7ff882000640 1 -- 172.21.15.86:0/3036122337 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff870031c40 con 0x7ff8840ae670 2024-03-06T07:31:58.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 -- 172.21.15.86:0/3036122337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840ae670 msgr2=0x7ff88415e2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 --2- 172.21.15.86:0/3036122337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840ae670 0x7ff88415e2c0 secure :-1 s=READY pgs=1037 cs=0 l=1 rev1=1 crypto rx=0x7ff8840ad980 tx=0x7ff87002b910 comp rx=0 tx=0).stop 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 -- 172.21.15.86:0/3036122337 shutdown_connections 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 --2- 172.21.15.86:0/3036122337 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8841620c0 0x7ff8840b28e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 --2- 172.21.15.86:0/3036122337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840ae670 0x7ff88415e2c0 unknown :-1 s=CLOSED pgs=1037 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 --2- 172.21.15.86:0/3036122337 >> v2:172.21.15.194:3300/0 conn(0x7ff8840b40c0 0x7ff8840b44a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 -- 172.21.15.86:0/3036122337 >> 172.21.15.86:0/3036122337 conn(0x7ff88406afc0 msgr2=0x7ff88406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 -- 172.21.15.86:0/3036122337 shutdown_connections 2024-03-06T07:31:58.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.344+0000 7ff889200640 1 -- 172.21.15.86:0/3036122337 wait complete. 2024-03-06T07:31:58.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.345+0000 7ff889200640 1 Processor -- start 2024-03-06T07:31:58.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.346+0000 7ff889200640 1 -- start start 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.346+0000 7ff889200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8840ae670 0x7ff884148440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.346+0000 7ff889200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840b40c0 0x7ff8840b2f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.346+0000 7ff889200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8841620c0 0x7ff88415c950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.346+0000 7ff889200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff88415e800 con 0x7ff8840b40c0 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.346+0000 7ff889200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff884161ba0 con 0x7ff8841620c0 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.346+0000 7ff889200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff88415e980 con 0x7ff8840ae670 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840b40c0 0x7ff8840b2f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840b40c0 0x7ff8840b2f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36628/0 (socket says 172.21.15.86:36628) 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 -- 172.21.15.86:0/3860746315 learned_addr learned my addr 172.21.15.86:0/3860746315 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff883e00640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8841620c0 0x7ff88415c950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff883400640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8840ae670 0x7ff884148440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 -- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8840ae670 msgr2=0x7ff884148440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8840ae670 0x7ff884148440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 -- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8841620c0 msgr2=0x7ff88415c950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8841620c0 0x7ff88415c950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.347+0000 7ff882a00640 1 -- 172.21.15.86:0/3860746315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8841f81b0 con 0x7ff8840b40c0 2024-03-06T07:31:58.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.348+0000 7ff882a00640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840b40c0 0x7ff8840b2f40 secure :-1 s=READY pgs=1038 cs=0 l=1 rev1=1 crypto rx=0x7ff870000c00 tx=0x7ff8700046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:58.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.348+0000 7ff87b400640 1 -- 172.21.15.86:0/3860746315 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff870035040 con 0x7ff8840b40c0 2024-03-06T07:31:58.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.348+0000 7ff87b400640 1 -- 172.21.15.86:0/3860746315 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff870030d00 con 0x7ff8840b40c0 2024-03-06T07:31:58.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.348+0000 7ff87b400640 1 -- 172.21.15.86:0/3860746315 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8700456a0 con 0x7ff8840b40c0 2024-03-06T07:31:58.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.348+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8840b2460 con 0x7ff8840b40c0 2024-03-06T07:31:58.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.348+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff884148980 con 0x7ff8840b40c0 2024-03-06T07:31:58.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.349+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff88414be10 con 0x7ff8840b40c0 2024-03-06T07:31:58.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.350+0000 7ff87b400640 1 -- 172.21.15.86:0/3860746315 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff870038020 con 0x7ff8840b40c0 2024-03-06T07:31:58.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.352+0000 7ff87b400640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff86406b4d0 0x7ff86406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.352+0000 7ff87b400640 1 -- 172.21.15.86:0/3860746315 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7ff8700ac1b0 con 0x7ff8840b40c0 2024-03-06T07:31:58.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.352+0000 7ff883400640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff86406b4d0 0x7ff86406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.353+0000 7ff883400640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff86406b4d0 0x7ff86406d990 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7ff874006fd0 tx=0x7ff874008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:58.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.357+0000 7ff87b400640 1 -- 172.21.15.86:0/3860746315 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff87007b3a0 con 0x7ff8840b40c0 2024-03-06T07:31:58.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.594+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff88404f330 con 0x7ff8840b40c0 2024-03-06T07:31:58.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.595+0000 7ff87b400640 1 -- 172.21.15.86:0/3860746315 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 52 v52) v1 ==== 94+0+3705 (secure 0 0 0) 0x7ff87007aca0 con 0x7ff8840b40c0 2024-03-06T07:31:58.597 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:58.597 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":52,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5060,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/4227315550","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":4227315550},{"type":"v1","addr":"172.21.15.194:6835","nonce":4227315550}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":5683,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2428515543","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2428515543},{"type":"v1","addr":"172.21.15.194:6834","nonce":2428515543}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6603,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/2877001314","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2877001314},{"type":"v1","addr":"172.21.15.86:6836","nonce":2877001314}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45},{"gid":6885,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/2626343150","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":2626343150},{"type":"v1","addr":"172.21.15.86:6837","nonce":2626343150}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":52}],"filesystems":[]} 2024-03-06T07:31:58.598 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 52 2024-03-06T07:31:58.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff86406b4d0 msgr2=0x7ff86406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff86406b4d0 0x7ff86406d990 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7ff874006fd0 tx=0x7ff874008040 comp rx=0 tx=0).stop 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840b40c0 msgr2=0x7ff8840b2f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840b40c0 0x7ff8840b2f40 secure :-1 s=READY pgs=1038 cs=0 l=1 rev1=1 crypto rx=0x7ff870000c00 tx=0x7ff8700046c0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 shutdown_connections 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff86406b4d0 0x7ff86406d990 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8841620c0 0x7ff88415c950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8840b40c0 0x7ff8840b2f40 unknown :-1 s=CLOSED pgs=1038 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 --2- 172.21.15.86:0/3860746315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8840ae670 0x7ff884148440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.597+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 >> 172.21.15.86:0/3860746315 conn(0x7ff88406afc0 msgr2=0x7ff8840af550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.598+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 shutdown_connections 2024-03-06T07:31:58.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.598+0000 7ff889200640 1 -- 172.21.15.86:0/3860746315 wait complete. 2024-03-06T07:31:58.617 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:31:58.617 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:31:58.618 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:31:58.618 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:58.618 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:31:58.618 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:31:58.618 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:31:58.618 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:58.619 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:31:58.619 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:31:58.619 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:31:58.619 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:58.619 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:31:58.619 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:31:58.619 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:31:58.620 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:31:58.635 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:31:58.635 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:31:58.636 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:31:58.636 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:31:58.639 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:31:58.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:31:58.642 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:31:58.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:31:58.683 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:31:58.683 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:58.689 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:31:58.689 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:58.689 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:31:58.690 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:58.695 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:31:58.695 DEBUG:teuthology.parallel:result is None 2024-03-06T07:31:58.695 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:31:58.835 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:31:58.841 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:31:58.854 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:31:58.855 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:31:58.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.878+0000 7f6c49c00640 1 Processor -- start 2024-03-06T07:31:58.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.878+0000 7f6c49c00640 1 -- start start 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c49c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6c440d2360 0x7f6c440d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c49c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6c440b3ec0 0x7f6c440b4310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c49c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6c44161960 con 0x7f6c44161580 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c49c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c440d2ce0 con 0x7f6c440b3ec0 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c49c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c440cf6f0 con 0x7f6c440d2360 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c49200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6c440b3ec0 0x7f6c440b4310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c43e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6c44161580 0x7f6c440d1950 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36344/0 (socket says 172.21.15.86:36344) 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c43e00640 1 -- 172.21.15.86:0/125458896 learned_addr learned my addr 172.21.15.86:0/125458896 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:31:58.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.879+0000 7f6c3b400640 1 --2- 172.21.15.86:0/125458896 >> v2:172.21.15.194:3300/0 conn(0x7f6c440d2360 0x7f6c440d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.880+0000 7f6c49200640 1 -- 172.21.15.86:0/125458896 >> v2:172.21.15.194:3300/0 conn(0x7f6c440d2360 msgr2=0x7f6c440d27a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:58.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.880+0000 7f6c49200640 1 --2- 172.21.15.86:0/125458896 >> v2:172.21.15.194:3300/0 conn(0x7f6c440d2360 0x7f6c440d27a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.880+0000 7f6c49200640 1 -- 172.21.15.86:0/125458896 >> v1:172.21.15.86:6789/0 conn(0x7f6c44161580 legacy=0x7f6c440d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.880+0000 7f6c43400640 1 -- 172.21.15.86:0/125458896 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2142127211 0 0) 0x7f6c44161960 con 0x7f6c44161580 2024-03-06T07:31:58.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.880+0000 7f6c49200640 1 -- 172.21.15.86:0/125458896 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c440cf9d0 con 0x7f6c440b3ec0 2024-03-06T07:31:58.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.881+0000 7f6c49200640 1 --2- 172.21.15.86:0/125458896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c440b4310 secure :-1 s=READY pgs=1047 cs=0 l=1 rev1=1 crypto rx=0x7f6c440d2e60 tx=0x7f6c3c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a90b0322ffcc8bb4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:58.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.881+0000 7f6c43400640 1 -- 172.21.15.86:0/125458896 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3c035020 con 0x7f6c440b3ec0 2024-03-06T07:31:58.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.881+0000 7f6c43400640 1 -- 172.21.15.86:0/125458896 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c3c031830 con 0x7f6c440b3ec0 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.881+0000 7f6c43400640 1 -- 172.21.15.86:0/125458896 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3c031c00 con 0x7f6c440b3ec0 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 -- 172.21.15.86:0/125458896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 msgr2=0x7f6c440b4310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 --2- 172.21.15.86:0/125458896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c440b4310 secure :-1 s=READY pgs=1047 cs=0 l=1 rev1=1 crypto rx=0x7f6c440d2e60 tx=0x7f6c3c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 -- 172.21.15.86:0/125458896 shutdown_connections 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 --2- 172.21.15.86:0/125458896 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c440b4310 unknown :-1 s=CLOSED pgs=1047 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 --2- 172.21.15.86:0/125458896 >> v2:172.21.15.194:3300/0 conn(0x7f6c440d2360 0x7f6c440d27a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 -- 172.21.15.86:0/125458896 >> 172.21.15.86:0/125458896 conn(0x7f6c440d3d80 msgr2=0x7f6c440d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 -- 172.21.15.86:0/125458896 shutdown_connections 2024-03-06T07:31:58.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.882+0000 7f6c49c00640 1 -- 172.21.15.86:0/125458896 wait complete. 2024-03-06T07:31:58.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.883+0000 7f6c49c00640 1 Processor -- start 2024-03-06T07:31:58.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.884+0000 7f6c49c00640 1 -- start start 2024-03-06T07:31:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.884+0000 7f6c49c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c441578d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.884+0000 7f6c49c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c440d2360 0x7f6c440b6b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.884+0000 7f6c49c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c44161580 0x7f6c440b55a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.884+0000 7f6c49c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c440b39a0 con 0x7f6c440b3ec0 2024-03-06T07:31:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.884+0000 7f6c49c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c440b36c0 con 0x7f6c44161580 2024-03-06T07:31:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.884+0000 7f6c49c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c440cf3a0 con 0x7f6c440d2360 2024-03-06T07:31:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c441578d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c441578d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36680/0 (socket says 172.21.15.86:36680) 2024-03-06T07:31:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 -- 172.21.15.86:0/2197095943 learned_addr learned my addr 172.21.15.86:0/2197095943 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c3b400640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c440d2360 0x7f6c440b6b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c49200640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c44161580 0x7f6c440b55a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 -- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c440d2360 msgr2=0x7f6c440b6b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c440d2360 0x7f6c440b6b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 -- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c44161580 msgr2=0x7f6c440b55a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c44161580 0x7f6c440b55a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c43e00640 1 -- 172.21.15.86:0/2197095943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c440c75b0 con 0x7f6c440b3ec0 2024-03-06T07:31:58.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.885+0000 7f6c3b400640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c440d2360 0x7f6c440b6b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:31:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.886+0000 7f6c43e00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c441578d0 secure :-1 s=READY pgs=1048 cs=0 l=1 rev1=1 crypto rx=0x7f6c3400c7c0 tx=0x7f6c3400cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.886+0000 7f6c40c00640 1 -- 172.21.15.86:0/2197095943 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c34018070 con 0x7f6c440b3ec0 2024-03-06T07:31:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.886+0000 7f6c40c00640 1 -- 172.21.15.86:0/2197095943 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c3400a640 con 0x7f6c440b3ec0 2024-03-06T07:31:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.886+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6c440c7890 con 0x7f6c440b3ec0 2024-03-06T07:31:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.886+0000 7f6c40c00640 1 -- 172.21.15.86:0/2197095943 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3400aa30 con 0x7f6c440b3ec0 2024-03-06T07:31:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.886+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c440c7e50 con 0x7f6c440b3ec0 2024-03-06T07:31:58.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.887+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6c4404f330 con 0x7f6c440b3ec0 2024-03-06T07:31:58.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.888+0000 7f6c40c00640 1 -- 172.21.15.86:0/2197095943 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6c3401b070 con 0x7f6c440b3ec0 2024-03-06T07:31:58.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.889+0000 7f6c40c00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2006b500 0x7f6c2006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:58.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.889+0000 7f6c40c00640 1 -- 172.21.15.86:0/2197095943 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f6c34089520 con 0x7f6c440b3ec0 2024-03-06T07:31:58.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.889+0000 7f6c3b400640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2006b500 0x7f6c2006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:58.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.890+0000 7f6c3b400640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2006b500 0x7f6c2006d9c0 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7f6c44069330 tx=0x7f6c2c009190 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:58.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:58.894+0000 7f6c40c00640 1 -- 172.21.15.86:0/2197095943 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6c34057fb0 con 0x7f6c440b3ec0 2024-03-06T07:31:59.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.104+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f6c440cebb0 con 0x7f6c440b3ec0 2024-03-06T07:31:59.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.227+0000 7f6c40c00640 1 -- 172.21.15.86:0/2197095943 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v88) v1 ==== 97+0+0 (secure 0 0 0) 0x7f6c3405c050 con 0x7f6c440b3ec0 2024-03-06T07:31:59.230 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:31:59.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.231+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2006b500 msgr2=0x7f6c2006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.231+0000 7f6c49c00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2006b500 0x7f6c2006d9c0 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7f6c44069330 tx=0x7f6c2c009190 comp rx=0 tx=0).stop 2024-03-06T07:31:59.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 msgr2=0x7f6c441578d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c441578d0 secure :-1 s=READY pgs=1048 cs=0 l=1 rev1=1 crypto rx=0x7f6c3400c7c0 tx=0x7f6c3400cc80 comp rx=0 tx=0).stop 2024-03-06T07:31:59.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 shutdown_connections 2024-03-06T07:31:59.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c2006b500 0x7f6c2006d9c0 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c44161580 0x7f6c440b55a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c440d2360 0x7f6c440b6b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 --2- 172.21.15.86:0/2197095943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c440b3ec0 0x7f6c441578d0 unknown :-1 s=CLOSED pgs=1048 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 >> 172.21.15.86:0/2197095943 conn(0x7f6c440d3d80 msgr2=0x7f6c4406b950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:59.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.232+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 shutdown_connections 2024-03-06T07:31:59.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.233+0000 7f6c49c00640 1 -- 172.21.15.86:0/2197095943 wait complete. 2024-03-06T07:31:59.256 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:31:59.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.454+0000 7fa7a8c00640 1 Processor -- start 2024-03-06T07:31:59.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.454+0000 7fa7a8c00640 1 -- start start 2024-03-06T07:31:59.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.455+0000 7fa7a8c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa7a415f100 0x7fa7a40ce4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.455+0000 7fa7a8c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa7a40b4140 0x7fa7a40cea00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa7a8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 0x7fa7a40cef40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa7a8c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa7a40b1f00 con 0x7fa7a40b4140 2024-03-06T07:31:59.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa7a8c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa7a40b21a0 con 0x7fa7a415f100 2024-03-06T07:31:59.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa7a8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a40b2480 con 0x7fa7a415f4e0 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa7a40b4140 0x7fa7a40cea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa7a40b4140 0x7fa7a40cea00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36682/0 (socket says 172.21.15.86:36682) 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 -- 172.21.15.86:0/2982675717 learned_addr learned my addr 172.21.15.86:0/2982675717 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa7a3e00640 1 --2- 172.21.15.86:0/2982675717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 0x7fa7a40cef40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 -- 172.21.15.86:0/2982675717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 msgr2=0x7fa7a40cef40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 --2- 172.21.15.86:0/2982675717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 0x7fa7a40cef40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 -- 172.21.15.86:0/2982675717 >> v2:172.21.15.194:3300/0 conn(0x7fa7a415f100 msgr2=0x7fa7a40ce4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa7a3400640 1 --2- 172.21.15.86:0/2982675717 >> v2:172.21.15.194:3300/0 conn(0x7fa7a415f100 0x7fa7a40ce4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 --2- 172.21.15.86:0/2982675717 >> v2:172.21.15.194:3300/0 conn(0x7fa7a415f100 0x7fa7a40ce4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa79be00640 1 -- 172.21.15.86:0/2982675717 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7a40b2760 con 0x7fa7a40b4140 2024-03-06T07:31:59.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.456+0000 7fa7a3400640 1 --2- 172.21.15.86:0/2982675717 >> v2:172.21.15.194:3300/0 conn(0x7fa7a415f100 0x7fa7a40ce4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:59.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.457+0000 7fa79be00640 1 --2- 172.21.15.86:0/2982675717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a40cea00 secure :-1 s=READY pgs=1049 cs=0 l=1 rev1=1 crypto rx=0x7fa7a40b45e0 tx=0x7fa78c02b870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f00004b4626d81fe server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:59.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a2a00640 1 -- 172.21.15.86:0/2982675717 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa78c0317d0 con 0x7fa7a40b4140 2024-03-06T07:31:59.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a2a00640 1 -- 172.21.15.86:0/2982675717 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa78c037070 con 0x7fa7a40b4140 2024-03-06T07:31:59.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a2a00640 1 -- 172.21.15.86:0/2982675717 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa78c031b90 con 0x7fa7a40b4140 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a8c00640 1 -- 172.21.15.86:0/2982675717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 msgr2=0x7fa7a40cea00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/2982675717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a40cea00 secure :-1 s=READY pgs=1049 cs=0 l=1 rev1=1 crypto rx=0x7fa7a40b45e0 tx=0x7fa78c02b870 comp rx=0 tx=0).stop 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a8c00640 1 -- 172.21.15.86:0/2982675717 shutdown_connections 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/2982675717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 0x7fa7a40cef40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/2982675717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a40cea00 unknown :-1 s=CLOSED pgs=1049 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.458+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/2982675717 >> v2:172.21.15.194:3300/0 conn(0x7fa7a415f100 0x7fa7a40ce4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.459+0000 7fa7a8c00640 1 -- 172.21.15.86:0/2982675717 >> 172.21.15.86:0/2982675717 conn(0x7fa7a40b3620 msgr2=0x7fa7a40b3a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.459+0000 7fa7a8c00640 1 -- 172.21.15.86:0/2982675717 shutdown_connections 2024-03-06T07:31:59.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.459+0000 7fa7a8c00640 1 -- 172.21.15.86:0/2982675717 wait complete. 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.460+0000 7fa7a8c00640 1 Processor -- start 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.460+0000 7fa7a8c00640 1 -- start start 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.461+0000 7fa7a8c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a4156090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.461+0000 7fa7a8c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7a415f100 0x7fa7a40cb210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.461+0000 7fa7a8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 0x7fa7a406fc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.461+0000 7fa7a8c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a40b33b0 con 0x7fa7a40b4140 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.461+0000 7fa7a8c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a40b3230 con 0x7fa7a415f100 2024-03-06T07:31:59.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.461+0000 7fa7a8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a40cfa20 con 0x7fa7a415f4e0 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.461+0000 7fa7a3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a4156090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a4156090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36696/0 (socket says 172.21.15.86:36696) 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 -- 172.21.15.86:0/1842217753 learned_addr learned my addr 172.21.15.86:0/1842217753 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 -- 172.21.15.86:0/1842217753 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 msgr2=0x7fa7a406fc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 0x7fa7a406fc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 -- 172.21.15.86:0/1842217753 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7a415f100 msgr2=0x7fa7a40cb210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7a415f100 0x7fa7a40cb210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 -- 172.21.15.86:0/1842217753 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7a40cb750 con 0x7fa7a40b4140 2024-03-06T07:31:59.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a3400640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a4156090 secure :-1 s=READY pgs=1050 cs=0 l=1 rev1=1 crypto rx=0x7fa79400c7c0 tx=0x7fa79400cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:59.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa79b400640 1 -- 172.21.15.86:0/1842217753 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa794015070 con 0x7fa7a40b4140 2024-03-06T07:31:59.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa79b400640 1 -- 172.21.15.86:0/1842217753 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa794007a30 con 0x7fa7a40b4140 2024-03-06T07:31:59.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa79b400640 1 -- 172.21.15.86:0/1842217753 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa794007e00 con 0x7fa7a40b4140 2024-03-06T07:31:59.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.462+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7a40b5ab0 con 0x7fa7a40b4140 2024-03-06T07:31:59.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.463+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7a40b6070 con 0x7fa7a40b4140 2024-03-06T07:31:59.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.464+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa7a404f330 con 0x7fa7a40b4140 2024-03-06T07:31:59.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.464+0000 7fa79b400640 1 -- 172.21.15.86:0/1842217753 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa794018070 con 0x7fa7a40b4140 2024-03-06T07:31:59.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.465+0000 7fa79b400640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa78006b4d0 0x7fa78006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.465+0000 7fa79b400640 1 -- 172.21.15.86:0/1842217753 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fa794089df0 con 0x7fa7a40b4140 2024-03-06T07:31:59.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.466+0000 7fa79be00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa78006b4d0 0x7fa78006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.466+0000 7fa79be00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa78006b4d0 0x7fa78006d990 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fa78c036000 tx=0x7fa78c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:59.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.470+0000 7fa79b400640 1 -- 172.21.15.86:0/1842217753 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa794058fb0 con 0x7fa7a40b4140 2024-03-06T07:31:59.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.734+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7fa7a40cc000 con 0x7fa7a40b4140 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.735+0000 7fa79b400640 1 -- 172.21.15.86:0/1842217753 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v17) v1 ==== 80+0+5304 (secure 0 0 0) 0x7fa7940588b0 con 0x7fa7a40b4140 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:31:59.738 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.739 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.740 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:31:59.741 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.742 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.743 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:31:59.744 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.745 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow r", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:31:59.746 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.747 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:31:59.748 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:31:59.749 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:31:59.750 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa78006b4d0 msgr2=0x7fa78006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa78006b4d0 0x7fa78006d990 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fa78c036000 tx=0x7fa78c030000 comp rx=0 tx=0).stop 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 msgr2=0x7fa7a4156090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a4156090 secure :-1 s=READY pgs=1050 cs=0 l=1 rev1=1 crypto rx=0x7fa79400c7c0 tx=0x7fa79400cc80 comp rx=0 tx=0).stop 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 shutdown_connections 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa78006b4d0 0x7fa78006d990 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa7a415f4e0 0x7fa7a406fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa7a415f100 0x7fa7a40cb210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 --2- 172.21.15.86:0/1842217753 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa7a40b4140 0x7fa7a4156090 unknown :-1 s=CLOSED pgs=1050 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 >> 172.21.15.86:0/1842217753 conn(0x7fa7a40b3620 msgr2=0x7fa7a40cfda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 shutdown_connections 2024-03-06T07:31:59.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.738+0000 7fa7a8c00640 1 -- 172.21.15.86:0/1842217753 wait complete. 2024-03-06T07:31:59.760 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:31:59.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.959+0000 7f641fc00640 1 Processor -- start 2024-03-06T07:31:59.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.959+0000 7f641fc00640 1 -- start start 2024-03-06T07:31:59.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641fc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6418144c00 0x7f641806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641fc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f641815f9d0 0x7f641815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641fc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f64180ce0a0 0x7f64180ce4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641fc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6418160370 con 0x7f64180ce0a0 2024-03-06T07:31:59.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641fc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f641806ee70 con 0x7f641815f9d0 2024-03-06T07:31:59.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641fc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f641806cef0 con 0x7f6418144c00 2024-03-06T07:31:59.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f64180ce0a0 0x7f64180ce4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f64180ce0a0 0x7f64180ce4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36700/0 (socket says 172.21.15.86:36700) 2024-03-06T07:31:59.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 -- 172.21.15.86:0/1879189364 learned_addr learned my addr 172.21.15.86:0/1879189364 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:59.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 -- 172.21.15.86:0/1879189364 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6418144c00 msgr2=0x7f641806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:59.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 --2- 172.21.15.86:0/1879189364 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6418144c00 0x7f641806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 -- 172.21.15.86:0/1879189364 >> v2:172.21.15.194:3300/0 conn(0x7f641815f9d0 msgr2=0x7f641815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:31:59.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 --2- 172.21.15.86:0/1879189364 >> v2:172.21.15.194:3300/0 conn(0x7f641815f9d0 0x7f641815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.961+0000 7f641dc00640 1 -- 172.21.15.86:0/1879189364 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f641806d1d0 con 0x7f64180ce0a0 2024-03-06T07:31:59.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.962+0000 7f641dc00640 1 --2- 172.21.15.86:0/1879189364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64180ce0a0 0x7f64180ce4f0 secure :-1 s=READY pgs=1051 cs=0 l=1 rev1=1 crypto rx=0x7f64180b27f0 tx=0x7f640802b920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=871eb010c2f9ac69 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:59.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f6417400640 1 -- 172.21.15.86:0/1879189364 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6408035020 con 0x7f64180ce0a0 2024-03-06T07:31:59.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f6417400640 1 -- 172.21.15.86:0/1879189364 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6408031880 con 0x7f64180ce0a0 2024-03-06T07:31:59.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f6417400640 1 -- 172.21.15.86:0/1879189364 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6408031c50 con 0x7f64180ce0a0 2024-03-06T07:31:59.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f641fc00640 1 -- 172.21.15.86:0/1879189364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64180ce0a0 msgr2=0x7f64180ce4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f641fc00640 1 --2- 172.21.15.86:0/1879189364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64180ce0a0 0x7f64180ce4f0 secure :-1 s=READY pgs=1051 cs=0 l=1 rev1=1 crypto rx=0x7f64180b27f0 tx=0x7f640802b920 comp rx=0 tx=0).stop 2024-03-06T07:31:59.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f641fc00640 1 -- 172.21.15.86:0/1879189364 shutdown_connections 2024-03-06T07:31:59.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f641fc00640 1 --2- 172.21.15.86:0/1879189364 >> v2:172.21.15.194:3300/0 conn(0x7f641815f9d0 0x7f641815fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f641fc00640 1 --2- 172.21.15.86:0/1879189364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64180ce0a0 0x7f64180ce4f0 unknown :-1 s=CLOSED pgs=1051 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f641fc00640 1 --2- 172.21.15.86:0/1879189364 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6418144c00 0x7f641806e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.963+0000 7f641fc00640 1 -- 172.21.15.86:0/1879189364 >> 172.21.15.86:0/1879189364 conn(0x7f64180b1730 msgr2=0x7f64180b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:31:59.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.964+0000 7f641fc00640 1 -- 172.21.15.86:0/1879189364 shutdown_connections 2024-03-06T07:31:59.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.964+0000 7f641fc00640 1 -- 172.21.15.86:0/1879189364 wait complete. 2024-03-06T07:31:59.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.965+0000 7f641fc00640 1 Processor -- start 2024-03-06T07:31:59.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.965+0000 7f641fc00640 1 -- start start 2024-03-06T07:31:59.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.966+0000 7f641fc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64180ce0a0 0x7f64181f5440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.966+0000 7f641fc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6418144c00 0x7f641813c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.966+0000 7f641fc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641815f9d0 0x7f641813cf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.966+0000 7f641fc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6418160730 con 0x7f6418144c00 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.966+0000 7f641fc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f641806d790 con 0x7f64180ce0a0 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.966+0000 7f641fc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f641806cb50 con 0x7f641815f9d0 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6418144c00 0x7f641813c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6418144c00 0x7f641813c3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36708/0 (socket says 172.21.15.86:36708) 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 -- 172.21.15.86:0/2780231873 learned_addr learned my addr 172.21.15.86:0/2780231873 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f641d200640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64180ce0a0 0x7f64181f5440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 -- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641815f9d0 msgr2=0x7f641813cf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641815f9d0 0x7f641813cf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 -- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64180ce0a0 msgr2=0x7f64181f5440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64180ce0a0 0x7f64181f5440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 -- 172.21.15.86:0/2780231873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64181400c0 con 0x7f6418144c00 2024-03-06T07:31:59.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f641d200640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64180ce0a0 0x7f64181f5440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:31:59.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.967+0000 7f6417e00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6418144c00 0x7f641813c3f0 secure :-1 s=READY pgs=1052 cs=0 l=1 rev1=1 crypto rx=0x7f640000b4a0 tx=0x7f640000b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:59.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.968+0000 7f6414c00640 1 -- 172.21.15.86:0/2780231873 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6400014070 con 0x7f6418144c00 2024-03-06T07:31:59.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.968+0000 7f6414c00640 1 -- 172.21.15.86:0/2780231873 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6400010610 con 0x7f6418144c00 2024-03-06T07:31:59.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.968+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f641815d5a0 con 0x7f6418144c00 2024-03-06T07:31:59.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.968+0000 7f6414c00640 1 -- 172.21.15.86:0/2780231873 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6400010a00 con 0x7f6418144c00 2024-03-06T07:31:59.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.968+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f641815db60 con 0x7f6418144c00 2024-03-06T07:31:59.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.969+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f64180ae870 con 0x7f6418144c00 2024-03-06T07:31:59.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.970+0000 7f6414c00640 1 -- 172.21.15.86:0/2780231873 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6400017070 con 0x7f6418144c00 2024-03-06T07:31:59.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.971+0000 7f6414c00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f406b500 0x7f63f406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:31:59.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.971+0000 7f6414c00640 1 -- 172.21.15.86:0/2780231873 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f64000898e0 con 0x7f6418144c00 2024-03-06T07:31:59.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.971+0000 7f641d200640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f406b500 0x7f63f406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:31:59.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.972+0000 7f641d200640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f406b500 0x7f63f406d9c0 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f64180cf5d0 tx=0x7f64100073e0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:31:59.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:31:59.976+0000 7f6414c00640 1 -- 172.21.15.86:0/2780231873 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f64180ae870 con 0x7f6418144c00 2024-03-06T07:32:00.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.169+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f64180cd9a0 con 0x7f6418144c00 2024-03-06T07:32:00.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.172+0000 7f6414c00640 1 -- 172.21.15.86:0/2780231873 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v90) v1 ==== 74+0+12904 (secure 0 0 0) 0x7f64180cd9a0 con 0x7f6418144c00 2024-03-06T07:32:00.174 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:00.175 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":90,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:31:59.245225+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":81,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:00.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.174+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f406b500 msgr2=0x7f63f406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.174+0000 7f641fc00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f406b500 0x7f63f406d9c0 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f64180cf5d0 tx=0x7f64100073e0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.174+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6418144c00 msgr2=0x7f641813c3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.174+0000 7f641fc00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6418144c00 0x7f641813c3f0 secure :-1 s=READY pgs=1052 cs=0 l=1 rev1=1 crypto rx=0x7f640000b4a0 tx=0x7f640000b960 comp rx=0 tx=0).stop 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 shutdown_connections 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63f406b500 0x7f63f406d9c0 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f641815f9d0 0x7f641813cf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6418144c00 0x7f641813c3f0 unknown :-1 s=CLOSED pgs=1052 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 --2- 172.21.15.86:0/2780231873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f64180ce0a0 0x7f64181f5440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 >> 172.21.15.86:0/2780231873 conn(0x7f64180b1730 msgr2=0x7f6418070b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 shutdown_connections 2024-03-06T07:32:00.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.175+0000 7f641fc00640 1 -- 172.21.15.86:0/2780231873 wait complete. 2024-03-06T07:32:00.195 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:32:00.196 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:32:00.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.387+0000 7f86eba00640 1 Processor -- start 2024-03-06T07:32:00.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.387+0000 7f86eba00640 1 -- start start 2024-03-06T07:32:00.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86eba00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f86e40b3df0 0x7f86e40727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86eba00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f86e40cda90 0x7f86e406c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86eba00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f86e40719e0 con 0x7f86e40b3650 2024-03-06T07:32:00.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86eba00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f86e4072310 con 0x7f86e40cda90 2024-03-06T07:32:00.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86eba00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f86e4071cc0 con 0x7f86e40b3df0 2024-03-06T07:32:00.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f86e40cda90 0x7f86e406c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f86e40cda90 0x7f86e406c990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36716/0 (socket says 172.21.15.86:36716) 2024-03-06T07:32:00.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9a00640 1 -- 172.21.15.86:0/3874038765 learned_addr learned my addr 172.21.15.86:0/3874038765 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:00.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9000640 1 --2- 172.21.15.86:0/3874038765 >> v2:172.21.15.194:3300/0 conn(0x7f86e40b3df0 0x7f86e40727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9a00640 1 -- 172.21.15.86:0/3874038765 >> v2:172.21.15.194:3300/0 conn(0x7f86e40b3df0 msgr2=0x7f86e40727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:00.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9a00640 1 --2- 172.21.15.86:0/3874038765 >> v2:172.21.15.194:3300/0 conn(0x7f86e40b3df0 0x7f86e40727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9a00640 1 -- 172.21.15.86:0/3874038765 >> v1:172.21.15.194:6789/0 conn(0x7f86e40b3650 legacy=0x7f86e40b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:00.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.389+0000 7f86e9a00640 1 -- 172.21.15.86:0/3874038765 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f86e4071fa0 con 0x7f86e40cda90 2024-03-06T07:32:00.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.390+0000 7f86e9a00640 1 --2- 172.21.15.86:0/3874038765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40cda90 0x7f86e406c990 secure :-1 s=READY pgs=1053 cs=0 l=1 rev1=1 crypto rx=0x7f86e40b3dc0 tx=0x7f86dc02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c2baa6facd69cc24 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:00.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.391+0000 7f86e3e00640 1 -- 172.21.15.86:0/3874038765 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86dc035020 con 0x7f86e40cda90 2024-03-06T07:32:00.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.391+0000 7f86e3e00640 1 -- 172.21.15.86:0/3874038765 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f86dc031870 con 0x7f86e40cda90 2024-03-06T07:32:00.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.391+0000 7f86e3e00640 1 -- 172.21.15.86:0/3874038765 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86dc031c40 con 0x7f86e40cda90 2024-03-06T07:32:00.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.391+0000 7f86eba00640 1 -- 172.21.15.86:0/3874038765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40cda90 msgr2=0x7f86e406c990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.391+0000 7f86eba00640 1 --2- 172.21.15.86:0/3874038765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40cda90 0x7f86e406c990 secure :-1 s=READY pgs=1053 cs=0 l=1 rev1=1 crypto rx=0x7f86e40b3dc0 tx=0x7f86dc02b910 comp rx=0 tx=0).stop 2024-03-06T07:32:00.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.391+0000 7f86eba00640 1 -- 172.21.15.86:0/3874038765 shutdown_connections 2024-03-06T07:32:00.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.391+0000 7f86eba00640 1 --2- 172.21.15.86:0/3874038765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40cda90 0x7f86e406c990 unknown :-1 s=CLOSED pgs=1053 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.392+0000 7f86eba00640 1 --2- 172.21.15.86:0/3874038765 >> v2:172.21.15.194:3300/0 conn(0x7f86e40b3df0 0x7f86e40727a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.392+0000 7f86eba00640 1 -- 172.21.15.86:0/3874038765 >> 172.21.15.86:0/3874038765 conn(0x7f86e40b1940 msgr2=0x7f86e40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:00.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.392+0000 7f86eba00640 1 -- 172.21.15.86:0/3874038765 shutdown_connections 2024-03-06T07:32:00.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.392+0000 7f86eba00640 1 -- 172.21.15.86:0/3874038765 wait complete. 2024-03-06T07:32:00.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.393+0000 7f86eba00640 1 Processor -- start 2024-03-06T07:32:00.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.393+0000 7f86eba00640 1 -- start start 2024-03-06T07:32:00.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.394+0000 7f86eba00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40b3650 0x7f86e40ca490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.394+0000 7f86eba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f86e40b3df0 0x7f86e40ae9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.394+0000 7f86eba00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f86e40cda90 0x7f86e41f6d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.394+0000 7f86eba00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86e415b3c0 con 0x7f86e40b3650 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.394+0000 7f86eba00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86e415b240 con 0x7f86e40cda90 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.394+0000 7f86eba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f86e40cf510 con 0x7f86e40b3df0 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.394+0000 7f86e9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40b3650 0x7f86e40ca490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40b3650 0x7f86e40ca490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36732/0 (socket says 172.21.15.86:36732) 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 -- 172.21.15.86:0/2007116296 learned_addr learned my addr 172.21.15.86:0/2007116296 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9a00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f86e40cda90 0x7f86e41f6d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 -- 172.21.15.86:0/2007116296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f86e40b3df0 msgr2=0x7f86e40ae9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f86e40b3df0 0x7f86e40ae9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 -- 172.21.15.86:0/2007116296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f86e40cda90 msgr2=0x7f86e41f6d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f86e40cda90 0x7f86e41f6d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 -- 172.21.15.86:0/2007116296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f86e40ca9d0 con 0x7f86e40b3650 2024-03-06T07:32:00.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e9000640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40b3650 0x7f86e40ca490 secure :-1 s=READY pgs=1054 cs=0 l=1 rev1=1 crypto rx=0x7f86d4007e70 tx=0x7f86d400e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:00.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.395+0000 7f86e1600640 1 -- 172.21.15.86:0/2007116296 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86d4014620 con 0x7f86e40b3650 2024-03-06T07:32:00.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.396+0000 7f86e1600640 1 -- 172.21.15.86:0/2007116296 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f86d4019070 con 0x7f86e40b3650 2024-03-06T07:32:00.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.396+0000 7f86e1600640 1 -- 172.21.15.86:0/2007116296 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86d4014a00 con 0x7f86e40b3650 2024-03-06T07:32:00.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.396+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f86e40caca0 con 0x7f86e40b3650 2024-03-06T07:32:00.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.396+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f86e40cb540 con 0x7f86e40b3650 2024-03-06T07:32:00.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.397+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f86e404f330 con 0x7f86e40b3650 2024-03-06T07:32:00.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.398+0000 7f86e1600640 1 -- 172.21.15.86:0/2007116296 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f86d4014ba0 con 0x7f86e40b3650 2024-03-06T07:32:00.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.399+0000 7f86e1600640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f86c006b4d0 0x7f86c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.399+0000 7f86e1600640 1 -- 172.21.15.86:0/2007116296 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f86d408a690 con 0x7f86e40b3650 2024-03-06T07:32:00.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.399+0000 7f86dbe00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f86c006b4d0 0x7f86c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.400+0000 7f86dbe00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f86c006b4d0 0x7f86c006d990 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f86e40ae650 tx=0x7f86cc006ba0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:00.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.404+0000 7f86e1600640 1 -- 172.21.15.86:0/2007116296 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f86d40598f0 con 0x7f86e40b3650 2024-03-06T07:32:00.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.599+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f86e40cc0e0 con 0x7f86e40b3650 2024-03-06T07:32:00.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.600+0000 7f86e1600640 1 -- 172.21.15.86:0/2007116296 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v90) v1 ==== 93+0+10 (secure 0 0 0) 0x7f86d40591f0 con 0x7f86e40b3650 2024-03-06T07:32:00.602 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:32:00.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.602+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f86c006b4d0 msgr2=0x7f86c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.602+0000 7f86eba00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f86c006b4d0 0x7f86c006d990 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f86e40ae650 tx=0x7f86cc006ba0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.602+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40b3650 msgr2=0x7f86e40ca490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.602+0000 7f86eba00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40b3650 0x7f86e40ca490 secure :-1 s=READY pgs=1054 cs=0 l=1 rev1=1 crypto rx=0x7f86d4007e70 tx=0x7f86d400e4b0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.602+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 shutdown_connections 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.603+0000 7f86eba00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f86c006b4d0 0x7f86c006d990 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.603+0000 7f86eba00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f86e40cda90 0x7f86e41f6d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.603+0000 7f86eba00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f86e40b3df0 0x7f86e40ae9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.603+0000 7f86eba00640 1 --2- 172.21.15.86:0/2007116296 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f86e40b3650 0x7f86e40ca490 unknown :-1 s=CLOSED pgs=1054 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.603+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 >> 172.21.15.86:0/2007116296 conn(0x7f86e40b1940 msgr2=0x7f86e414fd60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.603+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 shutdown_connections 2024-03-06T07:32:00.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.603+0000 7f86eba00640 1 -- 172.21.15.86:0/2007116296 wait complete. 2024-03-06T07:32:00.622 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:32:00.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.821+0000 7efec9600640 1 Processor -- start 2024-03-06T07:32:00.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.821+0000 7efec9600640 1 -- start start 2024-03-06T07:32:00.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec9600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec9600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efec415f9d0 0x7efec415fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec9600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec40ce0a0 0x7efec40ce4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec9600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efec406ee70 con 0x7efec415f9d0 2024-03-06T07:32:00.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec9600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efec406cef0 con 0x7efec4144c00 2024-03-06T07:32:00.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec9600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efec406d1d0 con 0x7efec40ce0a0 2024-03-06T07:32:00.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec2000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efec415f9d0 0x7efec415fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec2000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7efec415f9d0 0x7efec415fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36736/0 (socket says 172.21.15.86:36736) 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec2000640 1 -- 172.21.15.86:0/1283909726 learned_addr learned my addr 172.21.15.86:0/1283909726 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec3400640 1 --2- 172.21.15.86:0/1283909726 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec40ce0a0 0x7efec40ce4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.823+0000 7efec2a00640 1 --2- 172.21.15.86:0/1283909726 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.824+0000 7efec2000640 1 -- 172.21.15.86:0/1283909726 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec40ce0a0 msgr2=0x7efec40ce4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.824+0000 7efec2000640 1 --2- 172.21.15.86:0/1283909726 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec40ce0a0 0x7efec40ce4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.824+0000 7efec2000640 1 -- 172.21.15.86:0/1283909726 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 msgr2=0x7efec406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.824+0000 7efec2000640 1 --2- 172.21.15.86:0/1283909726 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.824+0000 7efec2000640 1 -- 172.21.15.86:0/1283909726 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efec406d4b0 con 0x7efec415f9d0 2024-03-06T07:32:00.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.825+0000 7efec2000640 1 --2- 172.21.15.86:0/1283909726 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec415fe60 secure :-1 s=READY pgs=1055 cs=0 l=1 rev1=1 crypto rx=0x7efec40b25f0 tx=0x7efeb002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e12728b41f2efbc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:00.828 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.825+0000 7efec1600640 1 -- 172.21.15.86:0/1283909726 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb0035020 con 0x7efec415f9d0 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.825+0000 7efec1600640 1 -- 172.21.15.86:0/1283909726 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efeb0031870 con 0x7efec415f9d0 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.825+0000 7efec1600640 1 -- 172.21.15.86:0/1283909726 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb0031c40 con 0x7efec415f9d0 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 -- 172.21.15.86:0/1283909726 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 msgr2=0x7efec415fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 --2- 172.21.15.86:0/1283909726 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec415fe60 secure :-1 s=READY pgs=1055 cs=0 l=1 rev1=1 crypto rx=0x7efec40b25f0 tx=0x7efeb002b910 comp rx=0 tx=0).stop 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 -- 172.21.15.86:0/1283909726 shutdown_connections 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 --2- 172.21.15.86:0/1283909726 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec40ce0a0 0x7efec40ce4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 --2- 172.21.15.86:0/1283909726 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec415f9d0 0x7efec415fe60 unknown :-1 s=CLOSED pgs=1055 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 --2- 172.21.15.86:0/1283909726 >> v2:172.21.15.194:3300/0 conn(0x7efec4144c00 0x7efec406e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 -- 172.21.15.86:0/1283909726 >> 172.21.15.86:0/1283909726 conn(0x7efec40b1730 msgr2=0x7efec40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 -- 172.21.15.86:0/1283909726 shutdown_connections 2024-03-06T07:32:00.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.826+0000 7efec9600640 1 -- 172.21.15.86:0/1283909726 wait complete. 2024-03-06T07:32:00.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.828+0000 7efec9600640 1 Processor -- start 2024-03-06T07:32:00.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.828+0000 7efec9600640 1 -- start start 2024-03-06T07:32:00.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec9600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec415eb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec9600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec4144c00 0x7efec41f7d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec9600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 0x7efec415c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec9600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec406cb50 con 0x7efec40ce0a0 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec9600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec4160730 con 0x7efec4144c00 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec9600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efec406ccd0 con 0x7efec415f9d0 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec415eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec415eb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36752/0 (socket says 172.21.15.86:36752) 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 -- 172.21.15.86:0/3543208212 learned_addr learned my addr 172.21.15.86:0/3543208212 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 -- 172.21.15.86:0/3543208212 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 msgr2=0x7efec415c340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 0x7efec415c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 -- 172.21.15.86:0/3543208212 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec4144c00 msgr2=0x7efec41f7d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec4144c00 0x7efec41f7d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.829+0000 7efec2a00640 1 -- 172.21.15.86:0/3543208212 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efec415cc40 con 0x7efec40ce0a0 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.830+0000 7efec2a00640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec415eb90 secure :-1 s=READY pgs=1056 cs=0 l=1 rev1=1 crypto rx=0x7efeb400c820 tx=0x7efeb400cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:00.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.830+0000 7efebaa00640 1 -- 172.21.15.86:0/3543208212 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb4015070 con 0x7efec40ce0a0 2024-03-06T07:32:00.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.830+0000 7efebaa00640 1 -- 172.21.15.86:0/3543208212 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efeb4004240 con 0x7efec40ce0a0 2024-03-06T07:32:00.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.830+0000 7efebaa00640 1 -- 172.21.15.86:0/3543208212 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb400ce10 con 0x7efec40ce0a0 2024-03-06T07:32:00.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.830+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efec415cf20 con 0x7efec40ce0a0 2024-03-06T07:32:00.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.830+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efec41f8250 con 0x7efec40ce0a0 2024-03-06T07:32:00.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.831+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efec40b29e0 con 0x7efec40ce0a0 2024-03-06T07:32:00.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.832+0000 7efebaa00640 1 -- 172.21.15.86:0/3543208212 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efeb4007830 con 0x7efec40ce0a0 2024-03-06T07:32:00.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.833+0000 7efebaa00640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea406b4d0 0x7efea406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:00.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.833+0000 7efebaa00640 1 -- 172.21.15.86:0/3543208212 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 4857+0+0 (secure 0 0 0) 0x7efeb4089e00 con 0x7efec40ce0a0 2024-03-06T07:32:00.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.833+0000 7efec2000640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea406b4d0 0x7efea406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:00.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.834+0000 7efec2000640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea406b4d0 0x7efea406d990 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7efeb0036000 tx=0x7efeb0030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:00.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:00.838+0000 7efebaa00640 1 -- 172.21.15.86:0/3543208212 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efeb4058fd0 con 0x7efec40ce0a0 2024-03-06T07:32:01.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.021+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7efec40b4cb0 con 0x7efec40ce0a0 2024-03-06T07:32:01.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.022+0000 7efebaa00640 1 -- 172.21.15.86:0/3543208212 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v90) v1 ==== 84+0+65 (secure 0 0 0) 0x7efeb40588d0 con 0x7efec40ce0a0 2024-03-06T07:32:01.025 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:01.025 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:32:01.025 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:32:01.025 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:32:01.025 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:32:01.025 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:32:01.025 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea406b4d0 msgr2=0x7efea406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea406b4d0 0x7efea406d990 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7efeb0036000 tx=0x7efeb0030000 comp rx=0 tx=0).stop 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 msgr2=0x7efec415eb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec415eb90 secure :-1 s=READY pgs=1056 cs=0 l=1 rev1=1 crypto rx=0x7efeb400c820 tx=0x7efeb400cce0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 shutdown_connections 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efea406b4d0 0x7efea406d990 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efec415f9d0 0x7efec415c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efec4144c00 0x7efec41f7d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 --2- 172.21.15.86:0/3543208212 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efec40ce0a0 0x7efec415eb90 unknown :-1 s=CLOSED pgs=1056 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 >> 172.21.15.86:0/3543208212 conn(0x7efec40b1730 msgr2=0x7efec40ce4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 shutdown_connections 2024-03-06T07:32:01.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.024+0000 7efec9600640 1 -- 172.21.15.86:0/3543208212 wait complete. 2024-03-06T07:32:01.044 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:32:01.044 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:32:01.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.246+0000 7ff8df200640 1 Processor -- start 2024-03-06T07:32:01.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.246+0000 7ff8df200640 1 -- start start 2024-03-06T07:32:01.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8df200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 0x7ff8e00add70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:01.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8df200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff8e00cf7d0 0x7ff8e006bc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8df200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8e006b140 con 0x7ff8e00acf50 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8df200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff8e006b770 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8df200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8e00cfd20 con 0x7ff8e015e020 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8dd400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff8e00cf7d0 0x7ff8e006bc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8dd400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff8e00cf7d0 0x7ff8e006bc00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36768/0 (socket says 172.21.15.86:36768) 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8dd400640 1 -- 172.21.15.86:0/2700090920 learned_addr learned my addr 172.21.15.86:0/2700090920 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8dd400640 1 -- 172.21.15.86:0/2700090920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 msgr2=0x7ff8e00add70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8dd400640 1 --2- 172.21.15.86:0/2700090920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 0x7ff8e00add70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.248+0000 7ff8dd400640 1 -- 172.21.15.86:0/2700090920 >> v1:172.21.15.194:6789/0 conn(0x7ff8e00acf50 legacy=0x7ff8e006d210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:01.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.249+0000 7ff8dd400640 1 -- 172.21.15.86:0/2700090920 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8e006b400 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.250+0000 7ff8dd400640 1 --2- 172.21.15.86:0/2700090920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e006bc00 secure :-1 s=READY pgs=1057 cs=0 l=1 rev1=1 crypto rx=0x7ff8c8009870 tx=0x7ff8c802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f7a65a3536434421 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:01.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.250+0000 7ff8dca00640 1 -- 172.21.15.86:0/2700090920 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8c8035020 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.250+0000 7ff8dca00640 1 -- 172.21.15.86:0/2700090920 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff8c8031640 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.250+0000 7ff8dca00640 1 -- 172.21.15.86:0/2700090920 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8c802d040 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 -- 172.21.15.86:0/2700090920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 msgr2=0x7ff8e006bc00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:01.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 --2- 172.21.15.86:0/2700090920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e006bc00 secure :-1 s=READY pgs=1057 cs=0 l=1 rev1=1 crypto rx=0x7ff8c8009870 tx=0x7ff8c802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 -- 172.21.15.86:0/2700090920 shutdown_connections 2024-03-06T07:32:01.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 --2- 172.21.15.86:0/2700090920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e006bc00 unknown :-1 s=CLOSED pgs=1057 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 --2- 172.21.15.86:0/2700090920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 0x7ff8e00add70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 -- 172.21.15.86:0/2700090920 >> 172.21.15.86:0/2700090920 conn(0x7ff8e00d0120 msgr2=0x7ff8e00d0500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:01.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 -- 172.21.15.86:0/2700090920 shutdown_connections 2024-03-06T07:32:01.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.251+0000 7ff8df200640 1 -- 172.21.15.86:0/2700090920 wait complete. 2024-03-06T07:32:01.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.253+0000 7ff8df200640 1 Processor -- start 2024-03-06T07:32:01.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.253+0000 7ff8df200640 1 -- start start 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.254+0000 7ff8df200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8e00acf50 0x7ff8e00c4640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.254+0000 7ff8df200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e00b5280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.254+0000 7ff8df200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 0x7ff8e00b0c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.254+0000 7ff8df200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8e006c140 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.254+0000 7ff8df200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8e015e400 con 0x7ff8e00acf50 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.254+0000 7ff8df200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8e006d920 con 0x7ff8e015e020 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.254+0000 7ff8dd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e00b5280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e00b5280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36784/0 (socket says 172.21.15.86:36784) 2024-03-06T07:32:01.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 -- 172.21.15.86:0/13809325 learned_addr learned my addr 172.21.15.86:0/13809325 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8de800640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 0x7ff8e00b0c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 -- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 msgr2=0x7ff8e00b0c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 0x7ff8e00b0c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 -- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8e00acf50 msgr2=0x7ff8e00c4640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dde00640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8e00acf50 0x7ff8e00c4640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8e00acf50 0x7ff8e00c4640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 -- 172.21.15.86:0/13809325 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8e00c6210 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.255+0000 7ff8dd400640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e00b5280 secure :-1 s=READY pgs=1058 cs=0 l=1 rev1=1 crypto rx=0x7ff8c802b930 tx=0x7ff8c80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.256+0000 7ff8d6000640 1 -- 172.21.15.86:0/13809325 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8c803ac60 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.256+0000 7ff8d6000640 1 -- 172.21.15.86:0/13809325 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff8c803ae20 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.256+0000 7ff8d6000640 1 -- 172.21.15.86:0/13809325 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8c8044630 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.256+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8e00c64f0 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.256+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8e00c6ab0 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.257+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff8e004f330 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.258+0000 7ff8d6000640 1 -- 172.21.15.86:0/13809325 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff8c8038020 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.258+0000 7ff8d6000640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff8bc06b4d0 0x7ff8bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:01.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.259+0000 7ff8d6000640 1 -- 172.21.15.86:0/13809325 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 4857+0+0 (secure 0 0 0) 0x7ff8c80abcb0 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.259+0000 7ff8dde00640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff8bc06b4d0 0x7ff8bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:01.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.259+0000 7ff8dde00640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff8bc06b4d0 0x7ff8bc06d990 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7ff8d0007dc0 tx=0x7ff8d00072f0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:01.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.263+0000 7ff8d6000640 1 -- 172.21.15.86:0/13809325 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff8c803f070 con 0x7ff8e00cf7d0 2024-03-06T07:32:01.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:01.473+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7ff8e006bd20 con 0x7ff8e00cf7d0 2024-03-06T07:32:02.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.258+0000 7ff8d6000640 1 -- 172.21.15.86:0/13809325 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v91) v1 ==== 138+0+0 (secure 0 0 0) 0x7ff8c807af80 con 0x7ff8e00cf7d0 2024-03-06T07:32:02.261 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff8bc06b4d0 msgr2=0x7ff8bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff8bc06b4d0 0x7ff8bc06d990 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7ff8d0007dc0 tx=0x7ff8d00072f0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 msgr2=0x7ff8e00b5280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e00b5280 secure :-1 s=READY pgs=1058 cs=0 l=1 rev1=1 crypto rx=0x7ff8c802b930 tx=0x7ff8c80046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 shutdown_connections 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff8bc06b4d0 0x7ff8bc06d990 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff8e015e020 0x7ff8e00b0c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff8e00cf7d0 0x7ff8e00b5280 unknown :-1 s=CLOSED pgs=1058 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 --2- 172.21.15.86:0/13809325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff8e00acf50 0x7ff8e00c4640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 >> 172.21.15.86:0/13809325 conn(0x7ff8e00d0120 msgr2=0x7ff8e00cc170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 shutdown_connections 2024-03-06T07:32:02.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.262+0000 7ff8df200640 1 -- 172.21.15.86:0/13809325 wait complete. 2024-03-06T07:32:02.292 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:32:02.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.489+0000 7f3c5de00640 1 Processor -- start 2024-03-06T07:32:02.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.490+0000 7f3c5de00640 1 -- start start 2024-03-06T07:32:02.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.491+0000 7f3c5de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 0x7f3c580afa10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.491+0000 7f3c5de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3c580b4250 0x7f3c5806dda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.491+0000 7f3c5de00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c580b04b0 con 0x7f3c580aff50 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.491+0000 7f3c5de00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c5815ef20 con 0x7f3c580b4250 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.491+0000 7f3c5de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c5815e8d0 con 0x7f3c58158770 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.492+0000 7f3c57400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3c580b4250 0x7f3c5806dda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.492+0000 7f3c57400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3c580b4250 0x7f3c5806dda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36800/0 (socket says 172.21.15.86:36800) 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.492+0000 7f3c57400640 1 -- 172.21.15.86:0/1412549567 learned_addr learned my addr 172.21.15.86:0/1412549567 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.492+0000 7f3c57400640 1 -- 172.21.15.86:0/1412549567 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 msgr2=0x7f3c580afa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.492+0000 7f3c57400640 1 --2- 172.21.15.86:0/1412549567 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 0x7f3c580afa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.492+0000 7f3c57400640 1 -- 172.21.15.86:0/1412549567 >> v1:172.21.15.86:6789/0 conn(0x7f3c580aff50 legacy=0x7f3c580b3b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:02.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.492+0000 7f3c57400640 1 -- 172.21.15.86:0/1412549567 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c5815ebb0 con 0x7f3c580b4250 2024-03-06T07:32:02.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.493+0000 7f3c57400640 1 --2- 172.21.15.86:0/1412549567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5806dda0 secure :-1 s=READY pgs=1060 cs=0 l=1 rev1=1 crypto rx=0x7f3c50009870 tx=0x7f3c5002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f20602fb89475997 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:02.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c55600640 1 -- 172.21.15.86:0/1412549567 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c50034070 con 0x7f3c580b4250 2024-03-06T07:32:02.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c55600640 1 -- 172.21.15.86:0/1412549567 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c50030840 con 0x7f3c580b4250 2024-03-06T07:32:02.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c55600640 1 -- 172.21.15.86:0/1412549567 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c50030c10 con 0x7f3c580b4250 2024-03-06T07:32:02.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 -- 172.21.15.86:0/1412549567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 msgr2=0x7f3c5806dda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:02.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 --2- 172.21.15.86:0/1412549567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5806dda0 secure :-1 s=READY pgs=1060 cs=0 l=1 rev1=1 crypto rx=0x7f3c50009870 tx=0x7f3c5002b980 comp rx=0 tx=0).stop 2024-03-06T07:32:02.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 -- 172.21.15.86:0/1412549567 shutdown_connections 2024-03-06T07:32:02.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 --2- 172.21.15.86:0/1412549567 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5806dda0 unknown :-1 s=CLOSED pgs=1060 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 --2- 172.21.15.86:0/1412549567 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 0x7f3c580afa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 -- 172.21.15.86:0/1412549567 >> 172.21.15.86:0/1412549567 conn(0x7f3c580b2840 msgr2=0x7f3c580b2c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:02.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 -- 172.21.15.86:0/1412549567 shutdown_connections 2024-03-06T07:32:02.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.494+0000 7f3c5de00640 1 -- 172.21.15.86:0/1412549567 wait complete. 2024-03-06T07:32:02.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.496+0000 7f3c5de00640 1 Processor -- start 2024-03-06T07:32:02.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.496+0000 7f3c5de00640 1 -- start start 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c5de00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c580aff50 0x7f3c5813c720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c5de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5814fcb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c5de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 0x7f3c58160790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c5de00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c5815f3b0 con 0x7f3c580b4250 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c5de00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c580b5d70 con 0x7f3c580aff50 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c5de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c5813ef90 con 0x7f3c58158770 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c56000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5814fcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c56000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5814fcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36806/0 (socket says 172.21.15.86:36806) 2024-03-06T07:32:02.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.497+0000 7f3c56000640 1 -- 172.21.15.86:0/334701961 learned_addr learned my addr 172.21.15.86:0/334701961 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:02.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c56000640 1 -- 172.21.15.86:0/334701961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 msgr2=0x7f3c58160790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:02.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c56000640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 0x7f3c58160790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c56000640 1 -- 172.21.15.86:0/334701961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c580aff50 msgr2=0x7f3c5813c720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c56000640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c580aff50 0x7f3c5813c720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c56000640 1 -- 172.21.15.86:0/334701961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c5813f270 con 0x7f3c580b4250 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c56000640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5814fcb0 secure :-1 s=READY pgs=1061 cs=0 l=1 rev1=1 crypto rx=0x7f3c4400c7c0 tx=0x7f3c4400cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c4f400640 1 -- 172.21.15.86:0/334701961 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c44016020 con 0x7f3c580b4250 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c4f400640 1 -- 172.21.15.86:0/334701961 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c44006d50 con 0x7f3c580b4250 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c4f400640 1 -- 172.21.15.86:0/334701961 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c44003bb0 con 0x7f3c580b4250 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c5813f550 con 0x7f3c580b4250 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.498+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c581501f0 con 0x7f3c580b4250 2024-03-06T07:32:02.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.500+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c5804f330 con 0x7f3c580b4250 2024-03-06T07:32:02.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.501+0000 7f3c4f400640 1 -- 172.21.15.86:0/334701961 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3c4400a850 con 0x7f3c580b4250 2024-03-06T07:32:02.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.502+0000 7f3c4f400640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c3806b4d0 0x7f3c3806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:02.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.502+0000 7f3c4f400640 1 -- 172.21.15.86:0/334701961 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(91..91 src has 1..91) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f3c44089cc0 con 0x7f3c580b4250 2024-03-06T07:32:02.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.502+0000 7f3c56a00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c3806b4d0 0x7f3c3806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:02.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.503+0000 7f3c56a00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c3806b4d0 0x7f3c3806d990 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7f3c48009670 tx=0x7f3c48006bf0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.507+0000 7f3c4f400640 1 -- 172.21.15.86:0/334701961 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3c44020020 con 0x7f3c580b4250 2024-03-06T07:32:02.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:02.704+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f3c5806e590 con 0x7f3c580b4250 2024-03-06T07:32:03.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.276+0000 7f3c4f400640 1 -- 172.21.15.86:0/334701961 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v92) v1 ==== 170+0+0 (secure 0 0 0) 0x7f3c44019310 con 0x7f3c580b4250 2024-03-06T07:32:03.279 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:32:03.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.279+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c3806b4d0 msgr2=0x7f3c3806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:03.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.279+0000 7f3c5de00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c3806b4d0 0x7f3c3806d990 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7f3c48009670 tx=0x7f3c48006bf0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.279+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 msgr2=0x7f3c5814fcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.279+0000 7f3c5de00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5814fcb0 secure :-1 s=READY pgs=1061 cs=0 l=1 rev1=1 crypto rx=0x7f3c4400c7c0 tx=0x7f3c4400cc80 comp rx=0 tx=0).stop 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.279+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 shutdown_connections 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.280+0000 7f3c5de00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c3806b4d0 0x7f3c3806d990 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.280+0000 7f3c5de00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c58158770 0x7f3c58160790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.280+0000 7f3c5de00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c580b4250 0x7f3c5814fcb0 unknown :-1 s=CLOSED pgs=1061 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.280+0000 7f3c5de00640 1 --2- 172.21.15.86:0/334701961 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c580aff50 0x7f3c5813c720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.280+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 >> 172.21.15.86:0/334701961 conn(0x7f3c580b2840 msgr2=0x7f3c5806e7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.280+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 shutdown_connections 2024-03-06T07:32:03.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.280+0000 7f3c5de00640 1 -- 172.21.15.86:0/334701961 wait complete. 2024-03-06T07:32:03.318 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:32:03.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.499+0000 7fd98d200640 1 Processor -- start 2024-03-06T07:32:03.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.499+0000 7fd98d200640 1 -- start start 2024-03-06T07:32:03.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.501+0000 7fd98d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 0x7fd98815f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.501+0000 7fd98d200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd98806e2f0 0x7fd9880cda90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.501+0000 7fd98d200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd98806cba0 con 0x7fd98806d960 2024-03-06T07:32:03.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.501+0000 7fd98d200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd98815f700 con 0x7fd98806e2f0 2024-03-06T07:32:03.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.501+0000 7fd98d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd98806d4d0 con 0x7fd9880b2c40 2024-03-06T07:32:03.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.501+0000 7fd985400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd98806d960 0x7fd9880b3020 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36356/0 (socket says 172.21.15.86:36356) 2024-03-06T07:32:03.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.501+0000 7fd985400640 1 -- 172.21.15.86:0/216580577 learned_addr learned my addr 172.21.15.86:0/216580577 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd986800640 1 --2- 172.21.15.86:0/216580577 >> v2:172.21.15.194:3300/0 conn(0x7fd98806e2f0 0x7fd9880cda90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd985e00640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 0x7fd98815f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1375988258 0 0) 0x7fd98806cba0 con 0x7fd98806d960 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd96c003520 con 0x7fd98806d960 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1997528883 0 0) 0x7fd96c003520 con 0x7fd98806d960 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 msgr2=0x7fd98815f1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 0x7fd98815f1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 >> v2:172.21.15.194:3300/0 conn(0x7fd98806e2f0 msgr2=0x7fd9880cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 --2- 172.21.15.86:0/216580577 >> v2:172.21.15.194:3300/0 conn(0x7fd98806e2f0 0x7fd9880cda90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd98806ce80 con 0x7fd98806d960 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd970001670 con 0x7fd98806d960 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 >> v1:172.21.15.86:6789/0 conn(0x7fd98806d960 legacy=0x7fd9880b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.502+0000 7fd984a00640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 0x7fd96c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd984a00640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd96c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd984a00640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd96c004d40 0x7fd96c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd98806d4d0 con 0x7fd96c003f30 2024-03-06T07:32:03.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd96c00b9c0 con 0x7fd96c004d40 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd96c00bb40 con 0x7fd9880b2c40 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd985400640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd96c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd985e00640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 0x7fd96c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd985400640 1 -- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 msgr2=0x7fd96c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd985400640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd9880b2c40 0x7fd96c0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd986800640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd96c004d40 0x7fd96c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd985400640 1 -- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd96c004d40 msgr2=0x7fd96c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd985400640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd96c004d40 0x7fd96c0051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.503+0000 7fd985400640 1 -- 172.21.15.86:0/216580577 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd96c003520 con 0x7fd96c003f30 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.504+0000 7fd985400640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd96c004800 secure :-1 s=READY pgs=1063 cs=0 l=1 rev1=1 crypto rx=0x7fd97000c210 tx=0x7fd96c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=664a06fd72fb6ef2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.504+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd970007d50 con 0x7fd96c003f30 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd970030070 con 0x7fd96c003f30 2024-03-06T07:32:03.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd984a00640 1 -- 172.21.15.86:0/216580577 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd97002cdb0 con 0x7fd96c003f30 2024-03-06T07:32:03.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd98d200640 1 -- 172.21.15.86:0/216580577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 msgr2=0x7fd96c004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:03.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd98d200640 1 --2- 172.21.15.86:0/216580577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd96c004800 secure :-1 s=READY pgs=1063 cs=0 l=1 rev1=1 crypto rx=0x7fd97000c210 tx=0x7fd96c005830 comp rx=0 tx=0).stop 2024-03-06T07:32:03.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd986800640 1 -- 172.21.15.86:0/216580577 reap_dead start 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd98d200640 1 -- 172.21.15.86:0/216580577 shutdown_connections 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd98d200640 1 -- 172.21.15.86:0/216580577 >> 172.21.15.86:0/216580577 conn(0x7fd9880b1940 msgr2=0x7fd9880b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd98d200640 1 -- 172.21.15.86:0/216580577 shutdown_connections 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.505+0000 7fd98d200640 1 -- 172.21.15.86:0/216580577 wait complete. 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.507+0000 7fd98d200640 1 Processor -- start 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.507+0000 7fd98d200640 1 -- start start 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.508+0000 7fd98d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd988159260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.508+0000 7fd98d200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9881597a0 0x7fd98809cbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.508+0000 7fd98d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd98809d130 0x7fd9881f7800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.508+0000 7fd98d200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd98806e8d0 con 0x7fd96c003f30 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.508+0000 7fd98d200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd98806e750 con 0x7fd9881597a0 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.508+0000 7fd98d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd98815f1c0 con 0x7fd98809d130 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.508+0000 7fd985e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd988159260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd988159260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36820/0 (socket says 172.21.15.86:36820) 2024-03-06T07:32:03.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 -- 172.21.15.86:0/768264910 learned_addr learned my addr 172.21.15.86:0/768264910 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9881597a0 0x7fd98809cbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 -- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd98809d130 msgr2=0x7fd9881f7800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd98809d130 0x7fd9881f7800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 -- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9881597a0 msgr2=0x7fd98809cbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9881597a0 0x7fd98809cbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 -- 172.21.15.86:0/768264910 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd988159ff0 con 0x7fd96c003f30 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9881597a0 0x7fd98809cbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd985e00640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd988159260 secure :-1 s=READY pgs=1064 cs=0 l=1 rev1=1 crypto rx=0x7fd978011660 tx=0x7fd978011b20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.509+0000 7fd97e000640 1 -- 172.21.15.86:0/768264910 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd97800f940 con 0x7fd96c003f30 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.510+0000 7fd97e000640 1 -- 172.21.15.86:0/768264910 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd97800fae0 con 0x7fd96c003f30 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.510+0000 7fd97e000640 1 -- 172.21.15.86:0/768264910 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd97800fe90 con 0x7fd96c003f30 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.510+0000 7fd98d200640 1 -- 172.21.15.86:0/768264910 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd98815a2d0 con 0x7fd96c003f30 2024-03-06T07:32:03.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.510+0000 7fd98d200640 1 -- 172.21.15.86:0/768264910 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd98815a890 con 0x7fd96c003f30 2024-03-06T07:32:03.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.511+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd98804f330 con 0x7fd96c003f30 2024-03-06T07:32:03.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.512+0000 7fd97e000640 1 -- 172.21.15.86:0/768264910 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd978015460 con 0x7fd96c003f30 2024-03-06T07:32:03.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.513+0000 7fd97e000640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd96406b670 0x7fd96406db30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:03.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.513+0000 7fd97e000640 1 -- 172.21.15.86:0/768264910 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 5609+0+0 (secure 0 0 0) 0x7fd97808a620 con 0x7fd96c003f30 2024-03-06T07:32:03.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.513+0000 7fd985400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd96406b670 0x7fd96406db30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:03.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.514+0000 7fd985400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd96406b670 0x7fd96406db30 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fd970034000 tx=0x7fd970007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:03.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.518+0000 7fd97e000640 1 -- 172.21.15.86:0/768264910 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd978019030 con 0x7fd96c003f30 2024-03-06T07:32:03.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:03.744+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7fd9880b2530 con 0x7fd96c003f30 2024-03-06T07:32:04.295 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 14 and data pool 15 2024-03-06T07:32:04.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.289+0000 7fd97e000640 1 -- 172.21.15.86:0/768264910 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 14 and data pool 15 v55) v1 ==== 174+0+0 (secure 0 0 0) 0x7fd9780594f0 con 0x7fd96c003f30 2024-03-06T07:32:04.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd96406b670 msgr2=0x7fd96406db30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd96406b670 0x7fd96406db30 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fd970034000 tx=0x7fd970007000 comp rx=0 tx=0).stop 2024-03-06T07:32:04.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 msgr2=0x7fd988159260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd988159260 secure :-1 s=READY pgs=1064 cs=0 l=1 rev1=1 crypto rx=0x7fd978011660 tx=0x7fd978011b20 comp rx=0 tx=0).stop 2024-03-06T07:32:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 shutdown_connections 2024-03-06T07:32:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd96406b670 0x7fd96406db30 unknown :-1 s=CLOSED pgs=467 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd98809d130 0x7fd9881f7800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9881597a0 0x7fd98809cbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 --2- 172.21.15.86:0/768264910 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd96c003f30 0x7fd988159260 secure :-1 s=CLOSED pgs=1064 cs=0 l=1 rev1=1 crypto rx=0x7fd978011660 tx=0x7fd978011b20 comp rx=0 tx=0).stop 2024-03-06T07:32:04.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.296+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 >> 172.21.15.86:0/768264910 conn(0x7fd9880b1940 msgr2=0x7fd9880711f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.297+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 shutdown_connections 2024-03-06T07:32:04.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.297+0000 7fd977400640 1 -- 172.21.15.86:0/768264910 wait complete. 2024-03-06T07:32:04.335 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.508+0000 7f0953e00640 1 Processor -- start 2024-03-06T07:32:04.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.509+0000 7f0953e00640 1 -- start start 2024-03-06T07:32:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.510+0000 7f0953e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f09541561f0 0x7f09540ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.510+0000 7f0953e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f095415c240 con 0x7f09540cc9d0 2024-03-06T07:32:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.510+0000 7f0953e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f095415c500 con 0x7f09540aebf0 2024-03-06T07:32:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.510+0000 7f0953e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f095415ce30 con 0x7f09541561f0 2024-03-06T07:32:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.510+0000 7f0953400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f09540cc9d0 0x7f095415f830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36368/0 (socket says 172.21.15.86:36368) 2024-03-06T07:32:04.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.510+0000 7f0953400640 1 -- 172.21.15.86:0/2130669477 learned_addr learned my addr 172.21.15.86:0/2130669477 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2407032588 0 0) 0x7f095415c240 con 0x7f09540cc9d0 2024-03-06T07:32:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952a00640 1 --2- 172.21.15.86:0/2130669477 >> v2:172.21.15.194:3300/0 conn(0x7f09541561f0 0x7f09540ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0938003520 con 0x7f09540cc9d0 2024-03-06T07:32:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1095068254 0 0) 0x7f0938003520 con 0x7f09540cc9d0 2024-03-06T07:32:04.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 >> v2:172.21.15.194:3300/0 conn(0x7f09541561f0 msgr2=0x7f09540ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 --2- 172.21.15.86:0/2130669477 >> v2:172.21.15.194:3300/0 conn(0x7f09541561f0 0x7f09540ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 >> v1:172.21.15.194:6789/0 conn(0x7f09540aebf0 legacy=0x7f09540cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f095415c7e0 con 0x7f09540cc9d0 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f094c0019f0 con 0x7f09540cc9d0 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 >> v1:172.21.15.86:6789/0 conn(0x7f09540cc9d0 legacy=0x7f095415f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09380039b0 0x7f0938003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0938004800 0x7f0938004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.511+0000 7f0952000640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f093800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f095415ce30 con 0x7f09380051c0 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f093800c040 con 0x7f0938004800 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f093800c240 con 0x7f09380039b0 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0953400640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f093800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f094a000640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0938004800 0x7f0938004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0952a00640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09380039b0 0x7f0938003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0953400640 1 -- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09380039b0 msgr2=0x7f0938003db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:04.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0953400640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09380039b0 0x7f0938003db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0953400640 1 -- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0938004800 msgr2=0x7f0938004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:04.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0953400640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0938004800 0x7f0938004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.512+0000 7f0953400640 1 -- 172.21.15.86:0/2130669477 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0938003520 con 0x7f09380051c0 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.513+0000 7f0953400640 1 -- 172.21.15.86:0/2130669477 reap_dead start 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.513+0000 7f0953400640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f093800b9c0 secure :-1 s=READY pgs=1066 cs=0 l=1 rev1=1 crypto rx=0x7f094c00a880 tx=0x7f0938005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad67a5a529bc0fc1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f094c00a4a0 con 0x7f09380051c0 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f094c034650 con 0x7f09380051c0 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0952000640 1 -- 172.21.15.86:0/2130669477 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f094c034a40 con 0x7f09380051c0 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0953e00640 1 -- 172.21.15.86:0/2130669477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 msgr2=0x7f093800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0953e00640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f093800b9c0 secure :-1 s=READY pgs=1066 cs=0 l=1 rev1=1 crypto rx=0x7f094c00a880 tx=0x7f0938005830 comp rx=0 tx=0).stop 2024-03-06T07:32:04.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0953e00640 1 -- 172.21.15.86:0/2130669477 shutdown_connections 2024-03-06T07:32:04.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0953e00640 1 --2- 172.21.15.86:0/2130669477 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f093800b9c0 unknown :-1 s=CLOSED pgs=1066 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0953e00640 1 -- 172.21.15.86:0/2130669477 >> 172.21.15.86:0/2130669477 conn(0x7f095406afc0 msgr2=0x7f095406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:04.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0953e00640 1 -- 172.21.15.86:0/2130669477 shutdown_connections 2024-03-06T07:32:04.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.514+0000 7f0953e00640 1 -- 172.21.15.86:0/2130669477 wait complete. 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.516+0000 7f0953e00640 1 Processor -- start 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.516+0000 7f0953e00640 1 -- start start 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0953e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f09541f1b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0953e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0954159130 0x7f0954071930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0953e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0954071e70 0x7f095415e430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0953e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0954156750 con 0x7f09380051c0 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0953e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09541565d0 con 0x7f0954071e70 2024-03-06T07:32:04.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0953e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09540ae6b0 con 0x7f0954159130 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0952a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f09541f1b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0952a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f09541f1b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36844/0 (socket says 172.21.15.86:36844) 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.517+0000 7f0952a00640 1 -- 172.21.15.86:0/2370080470 learned_addr learned my addr 172.21.15.86:0/2370080470 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f0952a00640 1 -- 172.21.15.86:0/2370080470 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0954159130 msgr2=0x7f0954071930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f0952a00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0954159130 0x7f0954071930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f0952a00640 1 -- 172.21.15.86:0/2370080470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0954071e70 msgr2=0x7f095415e430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f0953400640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0954071e70 0x7f095415e430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f0952a00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0954071e70 0x7f095415e430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f0952a00640 1 -- 172.21.15.86:0/2370080470 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09541523c0 con 0x7f09380051c0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f0952a00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f09541f1b70 secure :-1 s=READY pgs=1067 cs=0 l=1 rev1=1 crypto rx=0x7f0944011570 tx=0x7f0944011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f094b400640 1 -- 172.21.15.86:0/2370080470 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0944015930 con 0x7f09380051c0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f094b400640 1 -- 172.21.15.86:0/2370080470 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0944015ad0 con 0x7f09380051c0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.518+0000 7f094b400640 1 -- 172.21.15.86:0/2370080470 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f094401d400 con 0x7f09380051c0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.519+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f09541526a0 con 0x7f09380051c0 2024-03-06T07:32:04.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.519+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0954152c60 con 0x7f09380051c0 2024-03-06T07:32:04.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.520+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f095404f330 con 0x7f09380051c0 2024-03-06T07:32:04.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.521+0000 7f094b400640 1 -- 172.21.15.86:0/2370080470 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f094401d5a0 con 0x7f09380051c0 2024-03-06T07:32:04.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.522+0000 7f094b400640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f093006b4d0 0x7f093006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.522+0000 7f094b400640 1 -- 172.21.15.86:0/2370080470 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f094408aa00 con 0x7f09380051c0 2024-03-06T07:32:04.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.522+0000 7f094a000640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f093006b4d0 0x7f093006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.523+0000 7f094a000640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f093006b4d0 0x7f093006d990 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f09541462a0 tx=0x7f093c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:04.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.526+0000 7f094b400640 1 -- 172.21.15.86:0/2370080470 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0944019030 con 0x7f09380051c0 2024-03-06T07:32:04.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.722+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0954072250 con 0x7f09380051c0 2024-03-06T07:32:04.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.725+0000 7f094b400640 1 -- 172.21.15.86:0/2370080470 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v93) v1 ==== 74+0+16633 (secure 0 0 0) 0x7f0944059880 con 0x7f09380051c0 2024-03-06T07:32:04.728 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:04.728 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":93,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:03.746329+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:32:01.521762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-06T07:32:02.759692+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:04.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.727+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f093006b4d0 msgr2=0x7f093006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.727+0000 7f0953e00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f093006b4d0 0x7f093006d990 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f09541462a0 tx=0x7f093c009040 comp rx=0 tx=0).stop 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.727+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 msgr2=0x7f09541f1b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.727+0000 7f0953e00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f09541f1b70 secure :-1 s=READY pgs=1067 cs=0 l=1 rev1=1 crypto rx=0x7f0944011570 tx=0x7f0944011a30 comp rx=0 tx=0).stop 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 shutdown_connections 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f093006b4d0 0x7f093006d990 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0954071e70 0x7f095415e430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0954159130 0x7f0954071930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 --2- 172.21.15.86:0/2370080470 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09380051c0 0x7f09541f1b70 unknown :-1 s=CLOSED pgs=1067 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 >> 172.21.15.86:0/2370080470 conn(0x7f095406afc0 msgr2=0x7f095414f460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 shutdown_connections 2024-03-06T07:32:04.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.728+0000 7f0953e00640 1 -- 172.21.15.86:0/2370080470 wait complete. 2024-03-06T07:32:04.749 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:04.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.944+0000 7f4fe9e00640 1 Processor -- start 2024-03-06T07:32:04.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.945+0000 7f4fe9e00640 1 -- start start 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.946+0000 7f4fe9e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe415f9d0 0x7f4fe415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.946+0000 7f4fe9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4fe4070b10 0x7f4fe4070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.946+0000 7f4fe9e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4fe406cef0 con 0x7f4fe4144c00 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.946+0000 7f4fe9e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4fe4160370 con 0x7f4fe4070b10 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.946+0000 7f4fe9e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4fe406d1d0 con 0x7f4fe415f9d0 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.946+0000 7f4fe3400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4fe4070b10 0x7f4fe4070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fe2a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4fe4144c00 0x7f4fe406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36370/0 (socket says 172.21.15.86:36370) 2024-03-06T07:32:04.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fe2a00640 1 -- 172.21.15.86:0/1090803719 learned_addr learned my addr 172.21.15.86:0/1090803719 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:04.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fda000640 1 --2- 172.21.15.86:0/1090803719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe415f9d0 0x7f4fe415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fe2000640 1 -- 172.21.15.86:0/1090803719 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1573156884 0 0) 0x7f4fe406cef0 con 0x7f4fe4144c00 2024-03-06T07:32:04.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fe3400640 1 -- 172.21.15.86:0/1090803719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe415f9d0 msgr2=0x7f4fe415fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:04.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fe3400640 1 --2- 172.21.15.86:0/1090803719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe415f9d0 0x7f4fe415fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fe3400640 1 -- 172.21.15.86:0/1090803719 >> v1:172.21.15.86:6789/0 conn(0x7f4fe4144c00 legacy=0x7f4fe406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.947+0000 7f4fe3400640 1 -- 172.21.15.86:0/1090803719 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4fe406d4b0 con 0x7f4fe4070b10 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.948+0000 7f4fe3400640 1 --2- 172.21.15.86:0/1090803719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe4070b10 0x7f4fe4070f60 secure :-1 s=READY pgs=1069 cs=0 l=1 rev1=1 crypto rx=0x7f4fdc009d90 tx=0x7f4fdc02beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a69c71163722e5d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.948+0000 7f4fe2000640 1 -- 172.21.15.86:0/1090803719 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc039070 con 0x7f4fe4070b10 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.948+0000 7f4fe2000640 1 -- 172.21.15.86:0/1090803719 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4fdc02cc60 con 0x7f4fe4070b10 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.948+0000 7f4fe2000640 1 -- 172.21.15.86:0/1090803719 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc034660 con 0x7f4fe4070b10 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 -- 172.21.15.86:0/1090803719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe4070b10 msgr2=0x7f4fe4070f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/1090803719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe4070b10 0x7f4fe4070f60 secure :-1 s=READY pgs=1069 cs=0 l=1 rev1=1 crypto rx=0x7f4fdc009d90 tx=0x7f4fdc02beb0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 -- 172.21.15.86:0/1090803719 shutdown_connections 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/1090803719 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe4070b10 0x7f4fe4070f60 unknown :-1 s=CLOSED pgs=1069 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/1090803719 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe415f9d0 0x7f4fe415fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 -- 172.21.15.86:0/1090803719 >> 172.21.15.86:0/1090803719 conn(0x7f4fe40b1730 msgr2=0x7f4fe40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 -- 172.21.15.86:0/1090803719 shutdown_connections 2024-03-06T07:32:04.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.949+0000 7f4fe9e00640 1 -- 172.21.15.86:0/1090803719 wait complete. 2024-03-06T07:32:04.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.951+0000 7f4fe9e00640 1 Processor -- start 2024-03-06T07:32:04.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.951+0000 7f4fe9e00640 1 -- start start 2024-03-06T07:32:04.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe9e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fe4070b10 0x7f4fe415e840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe9e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe4144c00 0x7f4fe41e65a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe415f9d0 0x7f4fe41e8400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe9e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4fe406cb50 con 0x7f4fe415f9d0 2024-03-06T07:32:04.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe9e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4fe4160730 con 0x7f4fe4070b10 2024-03-06T07:32:04.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe9e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4fe406ccd0 con 0x7f4fe4144c00 2024-03-06T07:32:04.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe415f9d0 0x7f4fe41e8400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe415f9d0 0x7f4fe41e8400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36858/0 (socket says 172.21.15.86:36858) 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe3400640 1 -- 172.21.15.86:0/2078895120 learned_addr learned my addr 172.21.15.86:0/2078895120 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fe2a00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fe4070b10 0x7f4fe415e840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.952+0000 7f4fda000640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe4144c00 0x7f4fe41e65a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.953+0000 7f4fe3400640 1 -- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe4144c00 msgr2=0x7f4fe41e65a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.953+0000 7f4fe3400640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe4144c00 0x7f4fe41e65a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.953+0000 7f4fe3400640 1 -- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fe4070b10 msgr2=0x7f4fe415e840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.953+0000 7f4fe3400640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fe4070b10 0x7f4fe415e840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:04.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.953+0000 7f4fe3400640 1 -- 172.21.15.86:0/2078895120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4fe41e6ae0 con 0x7f4fe415f9d0 2024-03-06T07:32:04.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.953+0000 7f4fe3400640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe415f9d0 0x7f4fe41e8400 secure :-1 s=READY pgs=1070 cs=0 l=1 rev1=1 crypto rx=0x7f4fdc009d90 tx=0x7f4fdc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:04.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.953+0000 7f4fdbe00640 1 -- 172.21.15.86:0/2078895120 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc039050 con 0x7f4fe415f9d0 2024-03-06T07:32:04.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.954+0000 7f4fdbe00640 1 -- 172.21.15.86:0/2078895120 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4fdc045070 con 0x7f4fe415f9d0 2024-03-06T07:32:04.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.954+0000 7f4fdbe00640 1 -- 172.21.15.86:0/2078895120 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc03e830 con 0x7f4fe415f9d0 2024-03-06T07:32:04.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.954+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4fe41e6db0 con 0x7f4fe415f9d0 2024-03-06T07:32:04.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.954+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4fe41e7370 con 0x7f4fe415f9d0 2024-03-06T07:32:04.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.955+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4fe404f330 con 0x7f4fe415f9d0 2024-03-06T07:32:04.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.956+0000 7f4fdbe00640 1 -- 172.21.15.86:0/2078895120 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4fdc03e9d0 con 0x7f4fe415f9d0 2024-03-06T07:32:04.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.957+0000 7f4fdbe00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc006b4d0 0x7f4fc006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:04.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.957+0000 7f4fdbe00640 1 -- 172.21.15.86:0/2078895120 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4fdc0ad0b0 con 0x7f4fe415f9d0 2024-03-06T07:32:04.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.957+0000 7f4fe2a00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc006b4d0 0x7f4fc006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:04.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.958+0000 7f4fe2a00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc006b4d0 0x7f4fc006d990 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f4fd4000f60 tx=0x7f4fd4007b50 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:04.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:04.962+0000 7f4fdbe00640 1 -- 172.21.15.86:0/2078895120 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4fdc032e20 con 0x7f4fe415f9d0 2024-03-06T07:32:05.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.156+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4fe40cf2c0 con 0x7f4fe415f9d0 2024-03-06T07:32:05.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.159+0000 7f4fdbe00640 1 -- 172.21.15.86:0/2078895120 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v93) v1 ==== 74+0+16633 (secure 0 0 0) 0x7f4fdc07bfb0 con 0x7f4fe415f9d0 2024-03-06T07:32:05.162 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:05.162 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":93,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:03.746329+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:32:01.521762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-06T07:32:02.759692+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:05.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.161+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc006b4d0 msgr2=0x7f4fc006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:05.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.161+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc006b4d0 0x7f4fc006d990 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f4fd4000f60 tx=0x7f4fd4007b50 comp rx=0 tx=0).stop 2024-03-06T07:32:05.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.161+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe415f9d0 msgr2=0x7f4fe41e8400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.161+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe415f9d0 0x7f4fe41e8400 secure :-1 s=READY pgs=1070 cs=0 l=1 rev1=1 crypto rx=0x7f4fdc009d90 tx=0x7f4fdc004a40 comp rx=0 tx=0).stop 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 shutdown_connections 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc006b4d0 0x7f4fc006d990 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fe415f9d0 0x7f4fe41e8400 unknown :-1 s=CLOSED pgs=1070 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fe4144c00 0x7f4fe41e65a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 --2- 172.21.15.86:0/2078895120 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fe4070b10 0x7f4fe415e840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 >> 172.21.15.86:0/2078895120 conn(0x7f4fe40b1730 msgr2=0x7f4fe40ae7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 shutdown_connections 2024-03-06T07:32:05.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.162+0000 7f4fe9e00640 1 -- 172.21.15.86:0/2078895120 wait complete. 2024-03-06T07:32:05.183 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:32:05.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.386+0000 7f7f4a800640 1 Processor -- start 2024-03-06T07:32:05.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.386+0000 7f7f4a800640 1 -- start start 2024-03-06T07:32:05.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.388+0000 7f7f4a800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7f4415f1c0 0x7f7f441690e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.388+0000 7f7f4a800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7f4406cba0 con 0x7f7f440b2c40 2024-03-06T07:32:05.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.388+0000 7f7f4a800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7f4406ce80 con 0x7f7f4406d960 2024-03-06T07:32:05.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.388+0000 7f7f4a800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7f4406ea00 con 0x7f7f4415f1c0 2024-03-06T07:32:05.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.388+0000 7f7f43400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7f440b2c40 0x7f7f440b3020 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36378/0 (socket says 172.21.15.86:36378) 2024-03-06T07:32:05.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.388+0000 7f7f43400640 1 -- 172.21.15.86:0/838284416 learned_addr learned my addr 172.21.15.86:0/838284416 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:05.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.388+0000 7f7f43e00640 1 --2- 172.21.15.86:0/838284416 >> v2:172.21.15.194:3300/0 conn(0x7f7f4415f1c0 0x7f7f441690e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1318217593 0 0) 0x7f7f4406cba0 con 0x7f7f440b2c40 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7f28003520 con 0x7f7f440b2c40 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1074670931 0 0) 0x7f7f28003520 con 0x7f7f440b2c40 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 >> v2:172.21.15.194:3300/0 conn(0x7f7f4415f1c0 msgr2=0x7f7f441690e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 --2- 172.21.15.86:0/838284416 >> v2:172.21.15.194:3300/0 conn(0x7f7f4415f1c0 0x7f7f441690e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 >> v1:172.21.15.194:6789/0 conn(0x7f7f4406d960 legacy=0x7f7f4406e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f4406d160 con 0x7f7f440b2c40 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7f34001670 con 0x7f7f440b2c40 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 >> v1:172.21.15.86:6789/0 conn(0x7f7f440b2c40 legacy=0x7f7f440b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f28003a00 0x7f7f28003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f28004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f280051c0 0x7f7f2800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f4406ea00 con 0x7f7f28004800 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f2800c0c0 con 0x7f7f280051c0 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7f2800c2c0 con 0x7f7f28003a00 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.389+0000 7f7f42a00640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f28004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.390+0000 7f7f42a00640 1 -- 172.21.15.86:0/838284416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f28003a00 msgr2=0x7f7f28003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.390+0000 7f7f42a00640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f28003a00 0x7f7f28003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.390+0000 7f7f42a00640 1 -- 172.21.15.86:0/838284416 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f280051c0 msgr2=0x7f7f2800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.390+0000 7f7f42a00640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f280051c0 0x7f7f2800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.390+0000 7f7f42a00640 1 -- 172.21.15.86:0/838284416 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f28003520 con 0x7f7f28004800 2024-03-06T07:32:05.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.390+0000 7f7f43e00640 1 -- 172.21.15.86:0/838284416 reap_dead start 2024-03-06T07:32:05.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.391+0000 7f7f42a00640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f28004c80 secure :-1 s=READY pgs=1072 cs=0 l=1 rev1=1 crypto rx=0x7f7f30001750 tx=0x7f7f28005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b6247b17d18ba4f6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:05.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.391+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f30001370 con 0x7f7f28004800 2024-03-06T07:32:05.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.391+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f300052d0 con 0x7f7f28004800 2024-03-06T07:32:05.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.391+0000 7f7f42000640 1 -- 172.21.15.86:0/838284416 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f30003830 con 0x7f7f28004800 2024-03-06T07:32:05.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.392+0000 7f7f4a800640 1 -- 172.21.15.86:0/838284416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 msgr2=0x7f7f28004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:05.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.392+0000 7f7f4a800640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f28004c80 secure :-1 s=READY pgs=1072 cs=0 l=1 rev1=1 crypto rx=0x7f7f30001750 tx=0x7f7f28005830 comp rx=0 tx=0).stop 2024-03-06T07:32:05.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.392+0000 7f7f4a800640 1 -- 172.21.15.86:0/838284416 shutdown_connections 2024-03-06T07:32:05.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.392+0000 7f7f4a800640 1 --2- 172.21.15.86:0/838284416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f28004c80 unknown :-1 s=CLOSED pgs=1072 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.392+0000 7f7f4a800640 1 -- 172.21.15.86:0/838284416 >> 172.21.15.86:0/838284416 conn(0x7f7f440b1940 msgr2=0x7f7f440b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:05.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.392+0000 7f7f4a800640 1 -- 172.21.15.86:0/838284416 shutdown_connections 2024-03-06T07:32:05.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.392+0000 7f7f4a800640 1 -- 172.21.15.86:0/838284416 wait complete. 2024-03-06T07:32:05.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.394+0000 7f7f4a800640 1 Processor -- start 2024-03-06T07:32:05.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.394+0000 7f7f4a800640 1 -- start start 2024-03-06T07:32:05.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f4a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f4409c050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f4a800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f440ca950 0x7f7f44152240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f4a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f440ae440 0x7f7f440ae8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f4a800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f4406e2f0 con 0x7f7f28004800 2024-03-06T07:32:05.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f4a800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f4406d4d0 con 0x7f7f440ca950 2024-03-06T07:32:05.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f4a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7f4406e470 con 0x7f7f440ae440 2024-03-06T07:32:05.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f4409c050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:05.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f4409c050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36878/0 (socket says 172.21.15.86:36878) 2024-03-06T07:32:05.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 -- 172.21.15.86:0/1522830773 learned_addr learned my addr 172.21.15.86:0/1522830773 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 -- 172.21.15.86:0/1522830773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f440ae440 msgr2=0x7f7f440ae8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f440ae440 0x7f7f440ae8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 -- 172.21.15.86:0/1522830773 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f440ca950 msgr2=0x7f7f44152240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f42a00640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f440ca950 0x7f7f44152240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f440ca950 0x7f7f44152240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.395+0000 7f7f43400640 1 -- 172.21.15.86:0/1522830773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f44152db0 con 0x7f7f28004800 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.396+0000 7f7f43400640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f4409c050 secure :-1 s=READY pgs=1073 cs=0 l=1 rev1=1 crypto rx=0x7f7f34000fd0 tx=0x7f7f3400caa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:05.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.396+0000 7f7f3b400640 1 -- 172.21.15.86:0/1522830773 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f34018070 con 0x7f7f28004800 2024-03-06T07:32:05.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.396+0000 7f7f3b400640 1 -- 172.21.15.86:0/1522830773 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f340074e0 con 0x7f7f28004800 2024-03-06T07:32:05.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.396+0000 7f7f3b400640 1 -- 172.21.15.86:0/1522830773 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f34004a00 con 0x7f7f28004800 2024-03-06T07:32:05.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.396+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7f4409c590 con 0x7f7f28004800 2024-03-06T07:32:05.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.396+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7f440cb160 con 0x7f7f28004800 2024-03-06T07:32:05.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.397+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7f4404f330 con 0x7f7f28004800 2024-03-06T07:32:05.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.398+0000 7f7f3b400640 1 -- 172.21.15.86:0/1522830773 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7f34004ba0 con 0x7f7f28004800 2024-03-06T07:32:05.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.399+0000 7f7f3b400640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f2406b4d0 0x7f7f2406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:05.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.400+0000 7f7f3b400640 1 -- 172.21.15.86:0/1522830773 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7f34089710 con 0x7f7f28004800 2024-03-06T07:32:05.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.400+0000 7f7f42a00640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f2406b4d0 0x7f7f2406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:05.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.400+0000 7f7f42a00640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f2406b4d0 0x7f7f2406d990 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f7f30033000 tx=0x7f7f30009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:05.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.404+0000 7f7f3b400640 1 -- 172.21.15.86:0/1522830773 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7f44152db0 con 0x7f7f28004800 2024-03-06T07:32:05.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:05.629+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f7f440b2530 con 0x7f7f28004800 2024-03-06T07:32:06.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.307+0000 7f7f3b400640 1 -- 172.21.15.86:0/1522830773 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v58) v1 ==== 118+0+0 (secure 0 0 0) 0x7f7f34058580 con 0x7f7f28004800 2024-03-06T07:32:06.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.311+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f2406b4d0 msgr2=0x7f7f2406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.311+0000 7f7f4a800640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f2406b4d0 0x7f7f2406d990 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f7f30033000 tx=0x7f7f30009030 comp rx=0 tx=0).stop 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.311+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 msgr2=0x7f7f4409c050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.311+0000 7f7f4a800640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f4409c050 secure :-1 s=READY pgs=1073 cs=0 l=1 rev1=1 crypto rx=0x7f7f34000fd0 tx=0x7f7f3400caa0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 shutdown_connections 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f2406b4d0 0x7f7f2406d990 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f440ae440 0x7f7f440ae8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f440ca950 0x7f7f44152240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 --2- 172.21.15.86:0/1522830773 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f28004800 0x7f7f4409c050 unknown :-1 s=CLOSED pgs=1073 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 >> 172.21.15.86:0/1522830773 conn(0x7f7f440b1940 msgr2=0x7f7f44070c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 shutdown_connections 2024-03-06T07:32:06.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.312+0000 7f7f4a800640 1 -- 172.21.15.86:0/1522830773 wait complete. 2024-03-06T07:32:06.334 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:32:06.334 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:32:06.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.539+0000 7f0527200640 1 Processor -- start 2024-03-06T07:32:06.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.539+0000 7f0527200640 1 -- start start 2024-03-06T07:32:06.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.541+0000 7f0527200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0528158950 0x7f05280cf6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.541+0000 7f0527200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f052815b130 con 0x7f05280cfc10 2024-03-06T07:32:06.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.541+0000 7f0527200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f05280af170 con 0x7f0528158570 2024-03-06T07:32:06.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.541+0000 7f0527200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05280d0080 con 0x7f0528158950 2024-03-06T07:32:06.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.541+0000 7f0526800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f05280cfc10 0x7f05280cd7c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55926/0 (socket says 172.21.15.86:55926) 2024-03-06T07:32:06.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.541+0000 7f0526800640 1 -- 172.21.15.86:0/1119826465 learned_addr learned my addr 172.21.15.86:0/1119826465 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.541+0000 7f051d400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0528158950 0x7f05280cf6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3967818537 0 0) 0x7f052815b130 con 0x7f05280cfc10 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f050c003520 con 0x7f05280cfc10 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1745793535 0 0) 0x7f050c003520 con 0x7f05280cfc10 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0528158950 msgr2=0x7f05280cf6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0528158950 0x7f05280cf6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 >> v1:172.21.15.194:6789/0 conn(0x7f0528158570 legacy=0x7f05280ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f052815b3f0 con 0x7f05280cfc10 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f05200019f0 con 0x7f05280cfc10 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 >> v1:172.21.15.86:6789/0 conn(0x7f05280cfc10 legacy=0x7f05280cd7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f050c0039e0 0x7f050c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f050c004800 0x7f050c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0528158950 0x7f050c005200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05280d0080 con 0x7f050c004800 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f050c00baf0 con 0x7f050c0039e0 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f050c00df00 con 0x7f0528158950 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.542+0000 7f051d400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f050c004800 0x7f050c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.543+0000 7f0525e00640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f050c0039e0 0x7f050c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.543+0000 7f051d400640 1 -- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0528158950 msgr2=0x7f050c005200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.543+0000 7f051d400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0528158950 0x7f050c005200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.543+0000 7f051d400640 1 -- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f050c0039e0 msgr2=0x7f050c003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.543+0000 7f051d400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f050c0039e0 0x7f050c003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.543+0000 7f051d400640 1 -- 172.21.15.86:0/1119826465 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f050c003520 con 0x7f050c004800 2024-03-06T07:32:06.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.544+0000 7f051d400640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f050c004800 0x7f050c004c80 secure :-1 s=READY pgs=1075 cs=0 l=1 rev1=1 crypto rx=0x7f0510009ac0 tx=0x7f050c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4b56eeab4343d24e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:06.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.544+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0510039070 con 0x7f050c004800 2024-03-06T07:32:06.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.544+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f051002c710 con 0x7f050c004800 2024-03-06T07:32:06.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.544+0000 7f0525400640 1 -- 172.21.15.86:0/1119826465 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0510009690 con 0x7f050c004800 2024-03-06T07:32:06.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.545+0000 7f0527200640 1 -- 172.21.15.86:0/1119826465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f050c004800 msgr2=0x7f050c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.545+0000 7f0527200640 1 --2- 172.21.15.86:0/1119826465 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f050c004800 0x7f050c004c80 secure :-1 s=READY pgs=1075 cs=0 l=1 rev1=1 crypto rx=0x7f0510009ac0 tx=0x7f050c005830 comp rx=0 tx=0).stop 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.545+0000 7f0526800640 1 -- 172.21.15.86:0/1119826465 reap_dead start 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.545+0000 7f0527200640 1 -- 172.21.15.86:0/1119826465 shutdown_connections 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.545+0000 7f0527200640 1 -- 172.21.15.86:0/1119826465 >> 172.21.15.86:0/1119826465 conn(0x7f052806afc0 msgr2=0x7f052806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.545+0000 7f0527200640 1 -- 172.21.15.86:0/1119826465 shutdown_connections 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.545+0000 7f0527200640 1 -- 172.21.15.86:0/1119826465 wait complete. 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.547+0000 7f0527200640 1 Processor -- start 2024-03-06T07:32:06.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.547+0000 7f0527200640 1 -- start start 2024-03-06T07:32:06.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0527200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f050c004800 0x7f0528159c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0527200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f052815a150 0x7f0528155da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0527200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05281562e0 0x7f0528156760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0527200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05280b4950 con 0x7f05281562e0 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0527200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f052815b9b0 con 0x7f052815a150 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0527200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05280aedc0 con 0x7f050c004800 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05281562e0 0x7f0528156760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05281562e0 0x7f0528156760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45606/0 (socket says 172.21.15.86:45606) 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 -- 172.21.15.86:0/1287658791 learned_addr learned my addr 172.21.15.86:0/1287658791 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f051d400640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f052815a150 0x7f0528155da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0525e00640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f050c004800 0x7f0528159c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 -- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f050c004800 msgr2=0x7f0528159c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f050c004800 0x7f0528159c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 -- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f052815a150 msgr2=0x7f0528155da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f052815a150 0x7f0528155da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 -- 172.21.15.86:0/1287658791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f052815a940 con 0x7f05281562e0 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.548+0000 7f0526800640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05281562e0 0x7f0528156760 secure :-1 s=READY pgs=1076 cs=0 l=1 rev1=1 crypto rx=0x7f05280b3360 tx=0x7f052000cf00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:06.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.549+0000 7f051ea00640 1 -- 172.21.15.86:0/1287658791 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f052000a5b0 con 0x7f05281562e0 2024-03-06T07:32:06.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.549+0000 7f051ea00640 1 -- 172.21.15.86:0/1287658791 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f052000a750 con 0x7f05281562e0 2024-03-06T07:32:06.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.549+0000 7f051ea00640 1 -- 172.21.15.86:0/1287658791 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f052000ab20 con 0x7f05281562e0 2024-03-06T07:32:06.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.549+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f052815c2d0 con 0x7f05281562e0 2024-03-06T07:32:06.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.549+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f052815c890 con 0x7f05281562e0 2024-03-06T07:32:06.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.550+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f052804f330 con 0x7f05281562e0 2024-03-06T07:32:06.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.551+0000 7f051ea00640 1 -- 172.21.15.86:0/1287658791 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f052000acc0 con 0x7f05281562e0 2024-03-06T07:32:06.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.552+0000 7f051ea00640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050406b4d0 0x7f050406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:06.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.552+0000 7f051ea00640 1 -- 172.21.15.86:0/1287658791 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0520089a30 con 0x7f05281562e0 2024-03-06T07:32:06.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.552+0000 7f0525e00640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050406b4d0 0x7f050406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:06.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.553+0000 7f0525e00640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050406b4d0 0x7f050406d990 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f0518007fd0 tx=0x7f0518009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:06.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.556+0000 7f051ea00640 1 -- 172.21.15.86:0/1287658791 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f052001a020 con 0x7f05281562e0 2024-03-06T07:32:06.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:06.774+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f05280cd060 con 0x7f05281562e0 2024-03-06T07:32:07.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.312+0000 7f051ea00640 1 -- 172.21.15.86:0/1287658791 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v59) v1 ==== 122+0+0 (secure 0 0 0) 0x7f05200588e0 con 0x7f05281562e0 2024-03-06T07:32:07.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.316+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050406b4d0 msgr2=0x7f050406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.316+0000 7f0527200640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050406b4d0 0x7f050406d990 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f0518007fd0 tx=0x7f0518009040 comp rx=0 tx=0).stop 2024-03-06T07:32:07.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.316+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05281562e0 msgr2=0x7f0528156760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.316+0000 7f0527200640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05281562e0 0x7f0528156760 secure :-1 s=READY pgs=1076 cs=0 l=1 rev1=1 crypto rx=0x7f05280b3360 tx=0x7f052000cf00 comp rx=0 tx=0).stop 2024-03-06T07:32:07.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 shutdown_connections 2024-03-06T07:32:07.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f050406b4d0 0x7f050406d990 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05281562e0 0x7f0528156760 unknown :-1 s=CLOSED pgs=1076 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f052815a150 0x7f0528155da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 --2- 172.21.15.86:0/1287658791 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f050c004800 0x7f0528159c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 >> 172.21.15.86:0/1287658791 conn(0x7f052806afc0 msgr2=0x7f05280ae6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:07.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 shutdown_connections 2024-03-06T07:32:07.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.317+0000 7f0527200640 1 -- 172.21.15.86:0/1287658791 wait complete. 2024-03-06T07:32:07.339 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:32:07.339 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:07.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.547+0000 7f81c6200640 1 Processor -- start 2024-03-06T07:32:07.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.547+0000 7f81c6200640 1 -- start start 2024-03-06T07:32:07.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.548+0000 7f81c6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 0x7f81c006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.548+0000 7f81c6200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81c006ee70 con 0x7f81c015f9d0 2024-03-06T07:32:07.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.548+0000 7f81c6200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81c006cef0 con 0x7f81c015fe30 2024-03-06T07:32:07.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.548+0000 7f81c6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81c006d1d0 con 0x7f81c0144c00 2024-03-06T07:32:07.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.549+0000 7f81b7e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f81c015f9d0 0x7f81c00ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55928/0 (socket says 172.21.15.86:55928) 2024-03-06T07:32:07.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.549+0000 7f81b7e00640 1 -- 172.21.15.86:0/1953646238 learned_addr learned my addr 172.21.15.86:0/1953646238 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:07.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.549+0000 7f81bf400640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 0x7f81c006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.549+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2928017475 0 0) 0x7f81a8001110 con 0x7f81c015f9d0 2024-03-06T07:32:07.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.549+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f81a4003520 con 0x7f81c015f9d0 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3918859066 0 0) 0x7f81a4003520 con 0x7f81c015f9d0 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 msgr2=0x7f81c006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 0x7f81c006e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 >> v1:172.21.15.194:6789/0 conn(0x7f81c015fe30 legacy=0x7f81c00ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81c006d4b0 con 0x7f81c015f9d0 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f81a8001840 con 0x7f81c015f9d0 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 >> v1:172.21.15.86:6789/0 conn(0x7f81c015f9d0 legacy=0x7f81c00ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 0x7f81a40039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a4003f60 0x7f81a4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a4004d40 0x7f81a40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81c006d1d0 con 0x7f81a4004d40 2024-03-06T07:32:07.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81a400baa0 con 0x7f81a4003f60 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81a400bca0 con 0x7f81c0144c00 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.550+0000 7f81bfe00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a4004d40 0x7f81a40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81b7e00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a4003f60 0x7f81a4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81bf400640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 0x7f81a40039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81bfe00640 1 -- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 msgr2=0x7f81a40039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81bfe00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 0x7f81a40039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81bfe00640 1 -- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a4003f60 msgr2=0x7f81a4004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81bfe00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a4003f60 0x7f81a4004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81bfe00640 1 -- 172.21.15.86:0/1953646238 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81a4003520 con 0x7f81a4004d40 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81b7e00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81a4003f60 0x7f81a4004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:07.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.551+0000 7f81bf400640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81c0144c00 0x7f81a40039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:07.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.552+0000 7f81bfe00640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a4004d40 0x7f81a40051c0 secure :-1 s=READY pgs=1078 cs=0 l=1 rev1=1 crypto rx=0x7f81b8001e50 tx=0x7f81a4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2220c0a3200bbe6c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:07.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.552+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81b8001a70 con 0x7f81a4004d40 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.552+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81b8003b70 con 0x7f81a4004d40 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.552+0000 7f81bea00640 1 -- 172.21.15.86:0/1953646238 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81b80050d0 con 0x7f81a4004d40 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.553+0000 7f81c6200640 1 -- 172.21.15.86:0/1953646238 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a4004d40 msgr2=0x7f81a40051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.553+0000 7f81c6200640 1 --2- 172.21.15.86:0/1953646238 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81a4004d40 0x7f81a40051c0 secure :-1 s=READY pgs=1078 cs=0 l=1 rev1=1 crypto rx=0x7f81b8001e50 tx=0x7f81a4005830 comp rx=0 tx=0).stop 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.553+0000 7f81bfe00640 1 -- 172.21.15.86:0/1953646238 reap_dead start 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.553+0000 7f81c6200640 1 -- 172.21.15.86:0/1953646238 shutdown_connections 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.553+0000 7f81c6200640 1 -- 172.21.15.86:0/1953646238 >> 172.21.15.86:0/1953646238 conn(0x7f81c00b1730 msgr2=0x7f81c00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.553+0000 7f81c6200640 1 -- 172.21.15.86:0/1953646238 shutdown_connections 2024-03-06T07:32:07.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.553+0000 7f81c6200640 1 -- 172.21.15.86:0/1953646238 wait complete. 2024-03-06T07:32:07.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.555+0000 7f81c6200640 1 Processor -- start 2024-03-06T07:32:07.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.555+0000 7f81c6200640 1 -- start start 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81c6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a4004d40 0x7f81c015af60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81c6200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c013ce80 0x7f81c015db10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81c6200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c015a360 0x7f81c015a7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81c6200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81c0160730 con 0x7f81c015a360 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81c6200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81c0160280 con 0x7f81c013ce80 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81c6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81c00ceec0 con 0x7f81a4004d40 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81bfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c015a360 0x7f81c015a7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81bfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c015a360 0x7f81c015a7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45630/0 (socket says 172.21.15.86:45630) 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81bfe00640 1 -- 172.21.15.86:0/2661800919 learned_addr learned my addr 172.21.15.86:0/2661800919 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.556+0000 7f81bf400640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a4004d40 0x7f81c015af60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81b7e00640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c013ce80 0x7f81c015db10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81bfe00640 1 -- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a4004d40 msgr2=0x7f81c015af60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81bfe00640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a4004d40 0x7f81c015af60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81bfe00640 1 -- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c013ce80 msgr2=0x7f81c015db10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81bfe00640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c013ce80 0x7f81c015db10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81bfe00640 1 -- 172.21.15.86:0/2661800919 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81c013a870 con 0x7f81c015a360 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81bfe00640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c015a360 0x7f81c015a7c0 secure :-1 s=READY pgs=1079 cs=0 l=1 rev1=1 crypto rx=0x7f81b8001f70 tx=0x7f81b80014a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81b7400640 1 -- 172.21.15.86:0/2661800919 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81b8007db0 con 0x7f81c015a360 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81b7400640 1 -- 172.21.15.86:0/2661800919 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81b8003970 con 0x7f81c015a360 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81c013aae0 con 0x7f81c015a360 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81b7400640 1 -- 172.21.15.86:0/2661800919 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81b8006d70 con 0x7f81c015a360 2024-03-06T07:32:07.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.557+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81c0139ec0 con 0x7f81c015a360 2024-03-06T07:32:07.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.559+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81c004f330 con 0x7f81c015a360 2024-03-06T07:32:07.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.559+0000 7f81b7400640 1 -- 172.21.15.86:0/2661800919 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f81b8041030 con 0x7f81c015a360 2024-03-06T07:32:07.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.560+0000 7f81b7400640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f819c06b4d0 0x7f819c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:07.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.560+0000 7f81b7400640 1 -- 172.21.15.86:0/2661800919 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f81b80ab1e0 con 0x7f81c015a360 2024-03-06T07:32:07.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.561+0000 7f81bf400640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f819c06b4d0 0x7f819c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:07.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.561+0000 7f81bf400640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f819c06b4d0 0x7f819c06d990 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f81b0009aa0 tx=0x7f81b0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:07.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.565+0000 7f81b7400640 1 -- 172.21.15.86:0/2661800919 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f81b8033020 con 0x7f81c015a360 2024-03-06T07:32:07.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.798+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f81c0072890 con 0x7f81c015a360 2024-03-06T07:32:07.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.799+0000 7f81b7400640 1 -- 172.21.15.86:0/2661800919 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f81b807a050 con 0x7f81c015a360 2024-03-06T07:32:07.801 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:07.801 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5824,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1969090720","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1969090720},{"type":"v1","addr":"172.21.15.194:6835","nonce":1969090720}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6969,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/883748988","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":883748988},{"type":"v1","addr":"172.21.15.86:6835","nonce":883748988}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":53},{"gid":6972,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/2563126484","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2563126484},{"type":"v1","addr":"172.21.15.86:6837","nonce":2563126484}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:06.775964+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6978},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6978":{"gid":6978,"name":"b","rank":0,"incarnation":55,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/2712222804","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2712222804},{"type":"v1","addr":"172.21.15.194:6834","nonce":2712222804}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6978,"qdb_cluster":[6978]},"id":7}]} 2024-03-06T07:32:07.801 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 59 2024-03-06T07:32:07.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f819c06b4d0 msgr2=0x7f819c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f819c06b4d0 0x7f819c06d990 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f81b0009aa0 tx=0x7f81b0008040 comp rx=0 tx=0).stop 2024-03-06T07:32:07.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c015a360 msgr2=0x7f81c015a7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:07.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c015a360 0x7f81c015a7c0 secure :-1 s=READY pgs=1079 cs=0 l=1 rev1=1 crypto rx=0x7f81b8001f70 tx=0x7f81b80014a0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 shutdown_connections 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f819c06b4d0 0x7f819c06d990 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f81c015a360 0x7f81c015a7c0 unknown :-1 s=CLOSED pgs=1079 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f81c013ce80 0x7f81c015db10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 --2- 172.21.15.86:0/2661800919 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f81a4004d40 0x7f81c015af60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 >> 172.21.15.86:0/2661800919 conn(0x7f81c00b1730 msgr2=0x7f81c00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 shutdown_connections 2024-03-06T07:32:07.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:07.802+0000 7f81c6200640 1 -- 172.21.15.86:0/2661800919 wait complete. 2024-03-06T07:32:07.822 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.022+0000 7f4cfb800640 1 Processor -- start 2024-03-06T07:32:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.023+0000 7f4cfb800640 1 -- start start 2024-03-06T07:32:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.024+0000 7f4cfb800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4cf406ec00 0x7f4cf40ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.024+0000 7f4cfb800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4cf415b090 con 0x7f4cf415acb0 2024-03-06T07:32:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.024+0000 7f4cfb800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4cf40b2a30 con 0x7f4cf40b4680 2024-03-06T07:32:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.024+0000 7f4cfb800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4cf40b4360 con 0x7f4cf406ec00 2024-03-06T07:32:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf8e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4cf415acb0 0x7f4cf40b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55934/0 (socket says 172.21.15.86:55934) 2024-03-06T07:32:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf8e00640 1 -- 172.21.15.86:0/1620800129 learned_addr learned my addr 172.21.15.86:0/1620800129 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf9800640 1 --2- 172.21.15.86:0/1620800129 >> v2:172.21.15.86:3300/0 conn(0x7f4cf406ec00 0x7f4cf40ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf3e00640 1 -- 172.21.15.86:0/1620800129 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 362494331 0 0) 0x7f4cf415b090 con 0x7f4cf415acb0 2024-03-06T07:32:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf3e00640 1 -- 172.21.15.86:0/1620800129 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4cd8005730 con 0x7f4cf415acb0 2024-03-06T07:32:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf9800640 1 -- 172.21.15.86:0/1620800129 >> v1:172.21.15.194:6789/0 conn(0x7f4cf40b4680 legacy=0x7f4cf406e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf9800640 1 -- 172.21.15.86:0/1620800129 >> v1:172.21.15.86:6789/0 conn(0x7f4cf415acb0 legacy=0x7f4cf40b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.025+0000 7f4cf9800640 1 -- 172.21.15.86:0/1620800129 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cf40b2cf0 con 0x7f4cf406ec00 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.026+0000 7f4cf9800640 1 --2- 172.21.15.86:0/1620800129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf406ec00 0x7f4cf40ccfb0 secure :-1 s=READY pgs=1081 cs=0 l=1 rev1=1 crypto rx=0x7f4cec009d90 tx=0x7f4cec02beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d480e2cd616864a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cf3e00640 1 -- 172.21.15.86:0/1620800129 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cec039070 con 0x7f4cf406ec00 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cf3e00640 1 -- 172.21.15.86:0/1620800129 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cec02cc60 con 0x7f4cf406ec00 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cf3e00640 1 -- 172.21.15.86:0/1620800129 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cec034640 con 0x7f4cf406ec00 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cfb800640 1 -- 172.21.15.86:0/1620800129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf406ec00 msgr2=0x7f4cf40ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cfb800640 1 --2- 172.21.15.86:0/1620800129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf406ec00 0x7f4cf40ccfb0 secure :-1 s=READY pgs=1081 cs=0 l=1 rev1=1 crypto rx=0x7f4cec009d90 tx=0x7f4cec02beb0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cfb800640 1 -- 172.21.15.86:0/1620800129 shutdown_connections 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cfb800640 1 --2- 172.21.15.86:0/1620800129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf406ec00 0x7f4cf40ccfb0 unknown :-1 s=CLOSED pgs=1081 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cfb800640 1 -- 172.21.15.86:0/1620800129 >> 172.21.15.86:0/1620800129 conn(0x7f4cf40b1730 msgr2=0x7f4cf40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:08.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cfb800640 1 -- 172.21.15.86:0/1620800129 shutdown_connections 2024-03-06T07:32:08.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.027+0000 7f4cfb800640 1 -- 172.21.15.86:0/1620800129 wait complete. 2024-03-06T07:32:08.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.029+0000 7f4cfb800640 1 Processor -- start 2024-03-06T07:32:08.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.029+0000 7f4cfb800640 1 -- start start 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cfb800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4cf406ec00 0x7f4cf41455a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cfb800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4cf40b4680 0x7f4cf40ae620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cfb800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf415acb0 0x7f4cf40aeb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cfb800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf40b4010 con 0x7f4cf415acb0 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cfb800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf40b32b0 con 0x7f4cf40b4680 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cfb800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf4072340 con 0x7f4cf406ec00 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cf9800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf415acb0 0x7f4cf40aeb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cf9800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf415acb0 0x7f4cf40aeb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45644/0 (socket says 172.21.15.86:45644) 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.030+0000 7f4cf9800640 1 -- 172.21.15.86:0/4147201312 learned_addr learned my addr 172.21.15.86:0/4147201312 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cebe00640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4cf40b4680 0x7f4cf40ae620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf8e00640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4cf406ec00 0x7f4cf41455a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf9800640 1 -- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4cf406ec00 msgr2=0x7f4cf41455a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf9800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4cf406ec00 0x7f4cf41455a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf9800640 1 -- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4cf40b4680 msgr2=0x7f4cf40ae620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf9800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4cf40b4680 0x7f4cf40ae620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf9800640 1 -- 172.21.15.86:0/4147201312 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cf4148950 con 0x7f4cf415acb0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf9800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf415acb0 0x7f4cf40aeb60 secure :-1 s=READY pgs=1082 cs=0 l=1 rev1=1 crypto rx=0x7f4cec009eb0 tx=0x7f4cec004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf1600640 1 -- 172.21.15.86:0/4147201312 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cec039050 con 0x7f4cf415acb0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf1600640 1 -- 172.21.15.86:0/4147201312 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cec045070 con 0x7f4cf415acb0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cf1600640 1 -- 172.21.15.86:0/4147201312 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cec009450 con 0x7f4cf415acb0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4cf4148c30 con 0x7f4cf415acb0 2024-03-06T07:32:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.031+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4cf4146060 con 0x7f4cf415acb0 2024-03-06T07:32:08.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.033+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4cf404f330 con 0x7f4cf415acb0 2024-03-06T07:32:08.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.033+0000 7f4cf1600640 1 -- 172.21.15.86:0/4147201312 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4cec03e830 con 0x7f4cf415acb0 2024-03-06T07:32:08.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.034+0000 7f4cf1600640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cd006b4d0 0x7f4cd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.034+0000 7f4cf1600640 1 -- 172.21.15.86:0/4147201312 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4cec0acc90 con 0x7f4cf415acb0 2024-03-06T07:32:08.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.034+0000 7f4cf8e00640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cd006b4d0 0x7f4cd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.035+0000 7f4cf8e00640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cd006b4d0 0x7f4cd006d990 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f4ce4000f60 tx=0x7f4ce4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.039+0000 7f4cf1600640 1 -- 172.21.15.86:0/4147201312 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4cec04f020 con 0x7f4cf415acb0 2024-03-06T07:32:08.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.235+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4cf414bae0 con 0x7f4cf415acb0 2024-03-06T07:32:08.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.236+0000 7f4cf1600640 1 -- 172.21.15.86:0/4147201312 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) v1 ==== 74+0+16620 (secure 0 0 0) 0x7f4cec032e20 con 0x7f4cf415acb0 2024-03-06T07:32:08.239 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:08.239 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":94,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:05.268719+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:32:01.521762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-06T07:32:02.759692+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"94","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:08.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cd006b4d0 msgr2=0x7f4cd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cd006b4d0 0x7f4cd006d990 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f4ce4000f60 tx=0x7f4ce4009040 comp rx=0 tx=0).stop 2024-03-06T07:32:08.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf415acb0 msgr2=0x7f4cf40aeb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf415acb0 0x7f4cf40aeb60 secure :-1 s=READY pgs=1082 cs=0 l=1 rev1=1 crypto rx=0x7f4cec009eb0 tx=0x7f4cec004a40 comp rx=0 tx=0).stop 2024-03-06T07:32:08.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 shutdown_connections 2024-03-06T07:32:08.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4cd006b4d0 0x7f4cd006d990 unknown :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4cf415acb0 0x7f4cf40aeb60 unknown :-1 s=CLOSED pgs=1082 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4cf40b4680 0x7f4cf40ae620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 --2- 172.21.15.86:0/4147201312 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4cf406ec00 0x7f4cf41455a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 >> 172.21.15.86:0/4147201312 conn(0x7f4cf40b1730 msgr2=0x7f4cf406d3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:08.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.239+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 shutdown_connections 2024-03-06T07:32:08.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.240+0000 7f4cfb800640 1 -- 172.21.15.86:0/4147201312 wait complete. 2024-03-06T07:32:08.260 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:32:08.261 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:32:08.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.472+0000 7f4121200640 1 Processor -- start 2024-03-06T07:32:08.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.472+0000 7f4121200640 1 -- start start 2024-03-06T07:32:08.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4121200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f411c144c00 0x7f411c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4121200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f411c15f9d0 0x7f411c15fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4121200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c0ce0a0 0x7f411c0ce4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4121200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f411c06ee70 con 0x7f411c15f9d0 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4121200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f411c06cef0 con 0x7f411c144c00 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4121200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f411c06d1d0 con 0x7f411c0ce0a0 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4119400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f411c15f9d0 0x7f411c15fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4119400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f411c15f9d0 0x7f411c15fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45658/0 (socket says 172.21.15.86:45658) 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4119400640 1 -- 172.21.15.86:0/3299586618 learned_addr learned my addr 172.21.15.86:0/3299586618 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4119e00640 1 --2- 172.21.15.86:0/3299586618 >> v2:172.21.15.194:3300/0 conn(0x7f411c144c00 0x7f411c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f411a800640 1 --2- 172.21.15.86:0/3299586618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c0ce0a0 0x7f411c0ce4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.474+0000 7f4119400640 1 -- 172.21.15.86:0/3299586618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c0ce0a0 msgr2=0x7f411c0ce4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.475+0000 7f4119400640 1 --2- 172.21.15.86:0/3299586618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c0ce0a0 0x7f411c0ce4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.475+0000 7f4119400640 1 -- 172.21.15.86:0/3299586618 >> v2:172.21.15.194:3300/0 conn(0x7f411c144c00 msgr2=0x7f411c06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.475+0000 7f4119400640 1 --2- 172.21.15.86:0/3299586618 >> v2:172.21.15.194:3300/0 conn(0x7f411c144c00 0x7f411c06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.475+0000 7f4119400640 1 -- 172.21.15.86:0/3299586618 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f411c06d4b0 con 0x7f411c15f9d0 2024-03-06T07:32:08.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.475+0000 7f4119e00640 1 --2- 172.21.15.86:0/3299586618 >> v2:172.21.15.194:3300/0 conn(0x7f411c144c00 0x7f411c06e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.476+0000 7f4119400640 1 --2- 172.21.15.86:0/3299586618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c15f9d0 0x7f411c15fe60 secure :-1 s=READY pgs=1083 cs=0 l=1 rev1=1 crypto rx=0x7f411c0b25f0 tx=0x7f410402b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6287aea0e9964934 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.476+0000 7f4118a00640 1 -- 172.21.15.86:0/3299586618 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4104035020 con 0x7f411c15f9d0 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.476+0000 7f4118a00640 1 -- 172.21.15.86:0/3299586618 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f410402d040 con 0x7f411c15f9d0 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.476+0000 7f4118a00640 1 -- 172.21.15.86:0/3299586618 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4104031a80 con 0x7f411c15f9d0 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 -- 172.21.15.86:0/3299586618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c15f9d0 msgr2=0x7f411c15fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 --2- 172.21.15.86:0/3299586618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c15f9d0 0x7f411c15fe60 secure :-1 s=READY pgs=1083 cs=0 l=1 rev1=1 crypto rx=0x7f411c0b25f0 tx=0x7f410402b900 comp rx=0 tx=0).stop 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 -- 172.21.15.86:0/3299586618 shutdown_connections 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 --2- 172.21.15.86:0/3299586618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c0ce0a0 0x7f411c0ce4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 --2- 172.21.15.86:0/3299586618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c15f9d0 0x7f411c15fe60 unknown :-1 s=CLOSED pgs=1083 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 --2- 172.21.15.86:0/3299586618 >> v2:172.21.15.194:3300/0 conn(0x7f411c144c00 0x7f411c06e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 -- 172.21.15.86:0/3299586618 >> 172.21.15.86:0/3299586618 conn(0x7f411c0b1730 msgr2=0x7f411c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 -- 172.21.15.86:0/3299586618 shutdown_connections 2024-03-06T07:32:08.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.477+0000 7f4121200640 1 -- 172.21.15.86:0/3299586618 wait complete. 2024-03-06T07:32:08.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.479+0000 7f4121200640 1 Processor -- start 2024-03-06T07:32:08.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.479+0000 7f4121200640 1 -- start start 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4121200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f411c0ce0a0 0x7f411c15c900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4121200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c144c00 0x7f411c13c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4121200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c15f9d0 0x7f411c15e140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4121200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f411c06cb50 con 0x7f411c144c00 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4121200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f411c160730 con 0x7f411c0ce0a0 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4121200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f411c06ccd0 con 0x7f411c15f9d0 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c144c00 0x7f411c13c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c144c00 0x7f411c13c380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45672/0 (socket says 172.21.15.86:45672) 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 -- 172.21.15.86:0/177197372 learned_addr learned my addr 172.21.15.86:0/177197372 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:08.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 -- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c15f9d0 msgr2=0x7f411c15e140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f411a800640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c15f9d0 0x7f411c15e140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119e00640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f411c0ce0a0 0x7f411c15c900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c15f9d0 0x7f411c15e140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 -- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f411c0ce0a0 msgr2=0x7f411c15c900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f411c0ce0a0 0x7f411c15c900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.480+0000 7f4119400640 1 -- 172.21.15.86:0/177197372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f411c13d790 con 0x7f411c144c00 2024-03-06T07:32:08.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.481+0000 7f4119400640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c144c00 0x7f411c13c380 secure :-1 s=READY pgs=1084 cs=0 l=1 rev1=1 crypto rx=0x7f4104009960 tx=0x7f41040046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.481+0000 7f4112000640 1 -- 172.21.15.86:0/177197372 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4104035040 con 0x7f411c144c00 2024-03-06T07:32:08.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.481+0000 7f4112000640 1 -- 172.21.15.86:0/177197372 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4104039cb0 con 0x7f411c144c00 2024-03-06T07:32:08.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.481+0000 7f4112000640 1 -- 172.21.15.86:0/177197372 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41040456c0 con 0x7f411c144c00 2024-03-06T07:32:08.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.481+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f411c13c8c0 con 0x7f411c144c00 2024-03-06T07:32:08.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.481+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f411c13da00 con 0x7f411c144c00 2024-03-06T07:32:08.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.482+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f411c04f330 con 0x7f411c144c00 2024-03-06T07:32:08.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.483+0000 7f4112000640 1 -- 172.21.15.86:0/177197372 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4104038020 con 0x7f411c144c00 2024-03-06T07:32:08.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.484+0000 7f4112000640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40f806b500 0x7f40f806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.484+0000 7f4112000640 1 -- 172.21.15.86:0/177197372 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f41040ac450 con 0x7f411c144c00 2024-03-06T07:32:08.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.484+0000 7f4119e00640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40f806b500 0x7f40f806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.485+0000 7f4119e00640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40f806b500 0x7f40f806d9c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f410c008f80 tx=0x7f410c0084d0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.488+0000 7f4112000640 1 -- 172.21.15.86:0/177197372 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4104039e60 con 0x7f411c144c00 2024-03-06T07:32:08.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.679+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f411c0afa90 con 0x7f40f806b500 2024-03-06T07:32:08.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.692+0000 7f4112000640 1 -- 172.21.15.86:0/177197372 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252020 (secure 0 0 0) 0x7f411c0afa90 con 0x7f40f806b500 2024-03-06T07:32:08.696 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:08.698 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:32:08.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.696+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40f806b500 msgr2=0x7f40f806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.696+0000 7f4121200640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40f806b500 0x7f40f806d9c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f410c008f80 tx=0x7f410c0084d0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.696+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c144c00 msgr2=0x7f411c13c380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.696+0000 7f4121200640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c144c00 0x7f411c13c380 secure :-1 s=READY pgs=1084 cs=0 l=1 rev1=1 crypto rx=0x7f4104009960 tx=0x7f41040046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 shutdown_connections 2024-03-06T07:32:08.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f40f806b500 0x7f40f806d9c0 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f411c15f9d0 0x7f411c15e140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f411c144c00 0x7f411c13c380 unknown :-1 s=CLOSED pgs=1084 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 --2- 172.21.15.86:0/177197372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f411c0ce0a0 0x7f411c15c900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 >> 172.21.15.86:0/177197372 conn(0x7f411c0b1730 msgr2=0x7f411c145040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:08.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 shutdown_connections 2024-03-06T07:32:08.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.697+0000 7f4121200640 1 -- 172.21.15.86:0/177197372 wait complete. 2024-03-06T07:32:08.718 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2024-03-06T07:32:06.943235+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":126,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":259588,"kb_used_data":10108,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754715132,"statfs":{"total":773094113280,"available":772828295168,"internally_reserved":0,"allocated":10350592,"data_stored":5051812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":33,"apply_latency_ms":33,"commit_latency_ns":33000000,"apply_latency_ns":33000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.681575"},"pg_stats":[{"pgid":"15.3f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317906+0000","last_change":"2024-03-06T07:32:04.314244+0000","last_active":"2024-03-06T07:32:04.317906+0000","last_peered":"2024-03-06T07:32:04.317906+0000","last_clean":"2024-03-06T07:32:04.317906+0000","last_became_active":"2024-03-06T07:32:04.313906+0000","last_became_peered":"2024-03-06T07:32:04.313906+0000","last_unstale":"2024-03-06T07:32:04.317906+0000","last_undegraded":"2024-03-06T07:32:04.317906+0000","last_fullsized":"2024-03-06T07:32:04.317906+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:35:22.142943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.320921+0000","last_change":"2024-03-06T07:32:04.314655+0000","last_active":"2024-03-06T07:32:04.320921+0000","last_peered":"2024-03-06T07:32:04.320921+0000","last_clean":"2024-03-06T07:32:04.320921+0000","last_became_active":"2024-03-06T07:32:04.313184+0000","last_became_peered":"2024-03-06T07:32:04.313184+0000","last_unstale":"2024-03-06T07:32:04.320921+0000","last_undegraded":"2024-03-06T07:32:04.320921+0000","last_fullsized":"2024-03-06T07:32:04.320921+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:49:41.093474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.323215+0000","last_change":"2024-03-06T07:32:04.316023+0000","last_active":"2024-03-06T07:32:04.323215+0000","last_peered":"2024-03-06T07:32:04.323215+0000","last_clean":"2024-03-06T07:32:04.323215+0000","last_became_active":"2024-03-06T07:32:04.315681+0000","last_became_peered":"2024-03-06T07:32:04.315681+0000","last_unstale":"2024-03-06T07:32:04.323215+0000","last_undegraded":"2024-03-06T07:32:04.323215+0000","last_fullsized":"2024-03-06T07:32:04.323215+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:34:04.328114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312041+0000","last_change":"2024-03-06T07:32:04.303947+0000","last_active":"2024-03-06T07:32:04.312041+0000","last_peered":"2024-03-06T07:32:04.312041+0000","last_clean":"2024-03-06T07:32:04.312041+0000","last_became_active":"2024-03-06T07:32:04.303659+0000","last_became_peered":"2024-03-06T07:32:04.303659+0000","last_unstale":"2024-03-06T07:32:04.312041+0000","last_undegraded":"2024-03-06T07:32:04.312041+0000","last_fullsized":"2024-03-06T07:32:04.312041+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:53:12.334707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.3b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311609+0000","last_change":"2024-03-06T07:32:04.307511+0000","last_active":"2024-03-06T07:32:04.311609+0000","last_peered":"2024-03-06T07:32:04.311609+0000","last_clean":"2024-03-06T07:32:04.311609+0000","last_became_active":"2024-03-06T07:32:04.307258+0000","last_became_peered":"2024-03-06T07:32:04.307258+0000","last_unstale":"2024-03-06T07:32:04.311609+0000","last_undegraded":"2024-03-06T07:32:04.311609+0000","last_fullsized":"2024-03-06T07:32:04.311609+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:04:19.985344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.3a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.313026+0000","last_change":"2024-03-06T07:32:04.306619+0000","last_active":"2024-03-06T07:32:04.313026+0000","last_peered":"2024-03-06T07:32:04.313026+0000","last_clean":"2024-03-06T07:32:04.313026+0000","last_became_active":"2024-03-06T07:32:04.306324+0000","last_became_peered":"2024-03-06T07:32:04.306324+0000","last_unstale":"2024-03-06T07:32:04.313026+0000","last_undegraded":"2024-03-06T07:32:04.313026+0000","last_fullsized":"2024-03-06T07:32:04.313026+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:02:41.927660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.39","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312996+0000","last_change":"2024-03-06T07:32:04.307198+0000","last_active":"2024-03-06T07:32:04.312996+0000","last_peered":"2024-03-06T07:32:04.312996+0000","last_clean":"2024-03-06T07:32:04.312996+0000","last_became_active":"2024-03-06T07:32:04.306753+0000","last_became_peered":"2024-03-06T07:32:04.306753+0000","last_unstale":"2024-03-06T07:32:04.312996+0000","last_undegraded":"2024-03-06T07:32:04.312996+0000","last_fullsized":"2024-03-06T07:32:04.312996+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:39:57.997996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.38","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317991+0000","last_change":"2024-03-06T07:32:04.308105+0000","last_active":"2024-03-06T07:32:04.317991+0000","last_peered":"2024-03-06T07:32:04.317991+0000","last_clean":"2024-03-06T07:32:04.317991+0000","last_became_active":"2024-03-06T07:32:04.307834+0000","last_became_peered":"2024-03-06T07:32:04.307834+0000","last_unstale":"2024-03-06T07:32:04.317991+0000","last_undegraded":"2024-03-06T07:32:04.317991+0000","last_fullsized":"2024-03-06T07:32:04.317991+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:14:18.650793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.37","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315140+0000","last_change":"2024-03-06T07:32:04.306823+0000","last_active":"2024-03-06T07:32:04.315140+0000","last_peered":"2024-03-06T07:32:04.315140+0000","last_clean":"2024-03-06T07:32:04.315140+0000","last_became_active":"2024-03-06T07:32:04.306533+0000","last_became_peered":"2024-03-06T07:32:04.306533+0000","last_unstale":"2024-03-06T07:32:04.315140+0000","last_undegraded":"2024-03-06T07:32:04.315140+0000","last_fullsized":"2024-03-06T07:32:04.315140+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:46.541754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.36","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316494+0000","last_change":"2024-03-06T07:32:04.312947+0000","last_active":"2024-03-06T07:32:04.316494+0000","last_peered":"2024-03-06T07:32:04.316494+0000","last_clean":"2024-03-06T07:32:04.316494+0000","last_became_active":"2024-03-06T07:32:04.312619+0000","last_became_peered":"2024-03-06T07:32:04.312619+0000","last_unstale":"2024-03-06T07:32:04.316494+0000","last_undegraded":"2024-03-06T07:32:04.316494+0000","last_fullsized":"2024-03-06T07:32:04.316494+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:13:55.263699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.35","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.313857+0000","last_change":"2024-03-06T07:32:04.307949+0000","last_active":"2024-03-06T07:32:04.313857+0000","last_peered":"2024-03-06T07:32:04.313857+0000","last_clean":"2024-03-06T07:32:04.313857+0000","last_became_active":"2024-03-06T07:32:04.307208+0000","last_became_peered":"2024-03-06T07:32:04.307208+0000","last_unstale":"2024-03-06T07:32:04.313857+0000","last_undegraded":"2024-03-06T07:32:04.313857+0000","last_fullsized":"2024-03-06T07:32:04.313857+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:09:29.522513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.34","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308463+0000","last_change":"2024-03-06T07:32:04.301664+0000","last_active":"2024-03-06T07:32:04.308463+0000","last_peered":"2024-03-06T07:32:04.308463+0000","last_clean":"2024-03-06T07:32:04.308463+0000","last_became_active":"2024-03-06T07:32:04.301166+0000","last_became_peered":"2024-03-06T07:32:04.301166+0000","last_unstale":"2024-03-06T07:32:04.308463+0000","last_undegraded":"2024-03-06T07:32:04.308463+0000","last_fullsized":"2024-03-06T07:32:04.308463+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:25:16.547115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.33","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311672+0000","last_change":"2024-03-06T07:32:04.304723+0000","last_active":"2024-03-06T07:32:04.311672+0000","last_peered":"2024-03-06T07:32:04.311672+0000","last_clean":"2024-03-06T07:32:04.311672+0000","last_became_active":"2024-03-06T07:32:04.304389+0000","last_became_peered":"2024-03-06T07:32:04.304389+0000","last_unstale":"2024-03-06T07:32:04.311672+0000","last_undegraded":"2024-03-06T07:32:04.311672+0000","last_fullsized":"2024-03-06T07:32:04.311672+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:22:05.288497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.32","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306910+0000","last_change":"2024-03-06T07:32:04.302128+0000","last_active":"2024-03-06T07:32:04.306910+0000","last_peered":"2024-03-06T07:32:04.306910+0000","last_clean":"2024-03-06T07:32:04.306910+0000","last_became_active":"2024-03-06T07:32:04.301306+0000","last_became_peered":"2024-03-06T07:32:04.301306+0000","last_unstale":"2024-03-06T07:32:04.306910+0000","last_undegraded":"2024-03-06T07:32:04.306910+0000","last_fullsized":"2024-03-06T07:32:04.306910+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:34:22.621861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.31","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306474+0000","last_change":"2024-03-06T07:32:04.301038+0000","last_active":"2024-03-06T07:32:04.306474+0000","last_peered":"2024-03-06T07:32:04.306474+0000","last_clean":"2024-03-06T07:32:04.306474+0000","last_became_active":"2024-03-06T07:32:04.299831+0000","last_became_peered":"2024-03-06T07:32:04.299831+0000","last_unstale":"2024-03-06T07:32:04.306474+0000","last_undegraded":"2024-03-06T07:32:04.306474+0000","last_fullsized":"2024-03-06T07:32:04.306474+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:18:03.969159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.30","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308531+0000","last_change":"2024-03-06T07:32:04.303566+0000","last_active":"2024-03-06T07:32:04.308531+0000","last_peered":"2024-03-06T07:32:04.308531+0000","last_clean":"2024-03-06T07:32:04.308531+0000","last_became_active":"2024-03-06T07:32:04.303326+0000","last_became_peered":"2024-03-06T07:32:04.303326+0000","last_unstale":"2024-03-06T07:32:04.308531+0000","last_undegraded":"2024-03-06T07:32:04.308531+0000","last_fullsized":"2024-03-06T07:32:04.308531+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:29:09.128286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.2f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317983+0000","last_change":"2024-03-06T07:32:04.308364+0000","last_active":"2024-03-06T07:32:04.317983+0000","last_peered":"2024-03-06T07:32:04.317983+0000","last_clean":"2024-03-06T07:32:04.317983+0000","last_became_active":"2024-03-06T07:32:04.307768+0000","last_became_peered":"2024-03-06T07:32:04.307768+0000","last_unstale":"2024-03-06T07:32:04.317983+0000","last_undegraded":"2024-03-06T07:32:04.317983+0000","last_fullsized":"2024-03-06T07:32:04.317983+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:56:47.555774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.2e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311707+0000","last_change":"2024-03-06T07:32:04.304215+0000","last_active":"2024-03-06T07:32:04.311707+0000","last_peered":"2024-03-06T07:32:04.311707+0000","last_clean":"2024-03-06T07:32:04.311707+0000","last_became_active":"2024-03-06T07:32:04.303811+0000","last_became_peered":"2024-03-06T07:32:04.303811+0000","last_unstale":"2024-03-06T07:32:04.311707+0000","last_undegraded":"2024-03-06T07:32:04.311707+0000","last_fullsized":"2024-03-06T07:32:04.311707+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:37.296375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.2d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311548+0000","last_change":"2024-03-06T07:32:04.306389+0000","last_active":"2024-03-06T07:32:04.311548+0000","last_peered":"2024-03-06T07:32:04.311548+0000","last_clean":"2024-03-06T07:32:04.311548+0000","last_became_active":"2024-03-06T07:32:04.306129+0000","last_became_peered":"2024-03-06T07:32:04.306129+0000","last_unstale":"2024-03-06T07:32:04.311548+0000","last_undegraded":"2024-03-06T07:32:04.311548+0000","last_fullsized":"2024-03-06T07:32:04.311548+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:13:14.770211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315162+0000","last_change":"2024-03-06T07:32:04.308328+0000","last_active":"2024-03-06T07:32:04.315162+0000","last_peered":"2024-03-06T07:32:04.315162+0000","last_clean":"2024-03-06T07:32:04.315162+0000","last_became_active":"2024-03-06T07:32:04.307795+0000","last_became_peered":"2024-03-06T07:32:04.307795+0000","last_unstale":"2024-03-06T07:32:04.315162+0000","last_undegraded":"2024-03-06T07:32:04.315162+0000","last_fullsized":"2024-03-06T07:32:04.315162+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:59:11.714906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.2b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.307080+0000","last_change":"2024-03-06T07:32:04.303509+0000","last_active":"2024-03-06T07:32:04.307080+0000","last_peered":"2024-03-06T07:32:04.307080+0000","last_clean":"2024-03-06T07:32:04.307080+0000","last_became_active":"2024-03-06T07:32:04.303246+0000","last_became_peered":"2024-03-06T07:32:04.303246+0000","last_unstale":"2024-03-06T07:32:04.307080+0000","last_undegraded":"2024-03-06T07:32:04.307080+0000","last_fullsized":"2024-03-06T07:32:04.307080+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:11.857855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315151+0000","last_change":"2024-03-06T07:32:04.305219+0000","last_active":"2024-03-06T07:32:04.315151+0000","last_peered":"2024-03-06T07:32:04.315151+0000","last_clean":"2024-03-06T07:32:04.315151+0000","last_became_active":"2024-03-06T07:32:04.304957+0000","last_became_peered":"2024-03-06T07:32:04.304957+0000","last_unstale":"2024-03-06T07:32:04.315151+0000","last_undegraded":"2024-03-06T07:32:04.315151+0000","last_fullsized":"2024-03-06T07:32:04.315151+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:22:05.288497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.29","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317761+0000","last_change":"2024-03-06T07:32:04.308102+0000","last_active":"2024-03-06T07:32:04.317761+0000","last_peered":"2024-03-06T07:32:04.317761+0000","last_clean":"2024-03-06T07:32:04.317761+0000","last_became_active":"2024-03-06T07:32:04.307756+0000","last_became_peered":"2024-03-06T07:32:04.307756+0000","last_unstale":"2024-03-06T07:32:04.317761+0000","last_undegraded":"2024-03-06T07:32:04.317761+0000","last_fullsized":"2024-03-06T07:32:04.317761+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:53:12.334707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.28","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308504+0000","last_change":"2024-03-06T07:32:04.303021+0000","last_active":"2024-03-06T07:32:04.308504+0000","last_peered":"2024-03-06T07:32:04.308504+0000","last_clean":"2024-03-06T07:32:04.308504+0000","last_became_active":"2024-03-06T07:32:04.302729+0000","last_became_peered":"2024-03-06T07:32:04.302729+0000","last_unstale":"2024-03-06T07:32:04.308504+0000","last_undegraded":"2024-03-06T07:32:04.308504+0000","last_fullsized":"2024-03-06T07:32:04.308504+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:59:11.714906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.27","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316031+0000","last_change":"2024-03-06T07:32:04.312714+0000","last_active":"2024-03-06T07:32:04.316031+0000","last_peered":"2024-03-06T07:32:04.316031+0000","last_clean":"2024-03-06T07:32:04.316031+0000","last_became_active":"2024-03-06T07:32:04.312421+0000","last_became_peered":"2024-03-06T07:32:04.312421+0000","last_unstale":"2024-03-06T07:32:04.316031+0000","last_undegraded":"2024-03-06T07:32:04.316031+0000","last_fullsized":"2024-03-06T07:32:04.316031+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:00:49.753237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.26","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318243+0000","last_change":"2024-03-06T07:32:04.313426+0000","last_active":"2024-03-06T07:32:04.318243+0000","last_peered":"2024-03-06T07:32:04.318243+0000","last_clean":"2024-03-06T07:32:04.318243+0000","last_became_active":"2024-03-06T07:32:04.312661+0000","last_became_peered":"2024-03-06T07:32:04.312661+0000","last_unstale":"2024-03-06T07:32:04.318243+0000","last_undegraded":"2024-03-06T07:32:04.318243+0000","last_fullsized":"2024-03-06T07:32:04.318243+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:25:09.502176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.25","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.302023+0000","last_change":"2024-03-06T07:32:04.297909+0000","last_active":"2024-03-06T07:32:04.302023+0000","last_peered":"2024-03-06T07:32:04.302023+0000","last_clean":"2024-03-06T07:32:04.302023+0000","last_became_active":"2024-03-06T07:32:04.297610+0000","last_became_peered":"2024-03-06T07:32:04.297610+0000","last_unstale":"2024-03-06T07:32:04.302023+0000","last_undegraded":"2024-03-06T07:32:04.302023+0000","last_fullsized":"2024-03-06T07:32:04.302023+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:39:56.428272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.24","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317267+0000","last_change":"2024-03-06T07:32:04.311830+0000","last_active":"2024-03-06T07:32:04.317267+0000","last_peered":"2024-03-06T07:32:04.317267+0000","last_clean":"2024-03-06T07:32:04.317267+0000","last_became_active":"2024-03-06T07:32:04.310826+0000","last_became_peered":"2024-03-06T07:32:04.310826+0000","last_unstale":"2024-03-06T07:32:04.317267+0000","last_undegraded":"2024-03-06T07:32:04.317267+0000","last_fullsized":"2024-03-06T07:32:04.317267+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:09:47.827094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.23","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317959+0000","last_change":"2024-03-06T07:32:04.312560+0000","last_active":"2024-03-06T07:32:04.317959+0000","last_peered":"2024-03-06T07:32:04.317959+0000","last_clean":"2024-03-06T07:32:04.317959+0000","last_became_active":"2024-03-06T07:32:04.312218+0000","last_became_peered":"2024-03-06T07:32:04.312218+0000","last_unstale":"2024-03-06T07:32:04.317959+0000","last_undegraded":"2024-03-06T07:32:04.317959+0000","last_fullsized":"2024-03-06T07:32:04.317959+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:47:40.026034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.22","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311575+0000","last_change":"2024-03-06T07:32:04.308606+0000","last_active":"2024-03-06T07:32:04.311575+0000","last_peered":"2024-03-06T07:32:04.311575+0000","last_clean":"2024-03-06T07:32:04.311575+0000","last_became_active":"2024-03-06T07:32:04.308278+0000","last_became_peered":"2024-03-06T07:32:04.308278+0000","last_unstale":"2024-03-06T07:32:04.311575+0000","last_undegraded":"2024-03-06T07:32:04.311575+0000","last_fullsized":"2024-03-06T07:32:04.311575+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:54:27.865752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.21","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312370+0000","last_change":"2024-03-06T07:32:04.306610+0000","last_active":"2024-03-06T07:32:04.312370+0000","last_peered":"2024-03-06T07:32:04.312370+0000","last_clean":"2024-03-06T07:32:04.312370+0000","last_became_active":"2024-03-06T07:32:04.305576+0000","last_became_peered":"2024-03-06T07:32:04.305576+0000","last_unstale":"2024-03-06T07:32:04.312370+0000","last_undegraded":"2024-03-06T07:32:04.312370+0000","last_fullsized":"2024-03-06T07:32:04.312370+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:56:47.555774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.20","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.324677+0000","last_change":"2024-03-06T07:32:04.316454+0000","last_active":"2024-03-06T07:32:04.324677+0000","last_peered":"2024-03-06T07:32:04.324677+0000","last_clean":"2024-03-06T07:32:04.324677+0000","last_became_active":"2024-03-06T07:32:04.316121+0000","last_became_peered":"2024-03-06T07:32:04.316121+0000","last_unstale":"2024-03-06T07:32:04.324677+0000","last_undegraded":"2024-03-06T07:32:04.324677+0000","last_fullsized":"2024-03-06T07:32:04.324677+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:16:13.907316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306996+0000","last_change":"2024-03-06T07:32:04.302971+0000","last_active":"2024-03-06T07:32:04.306996+0000","last_peered":"2024-03-06T07:32:04.306996+0000","last_clean":"2024-03-06T07:32:04.306996+0000","last_became_active":"2024-03-06T07:32:04.302656+0000","last_became_peered":"2024-03-06T07:32:04.302656+0000","last_unstale":"2024-03-06T07:32:04.306996+0000","last_undegraded":"2024-03-06T07:32:04.306996+0000","last_fullsized":"2024-03-06T07:32:04.306996+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:06:43.857742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1f","version":"93'4","reported_seq":16,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.328261+0000","last_change":"2024-03-06T07:32:03.305143+0000","last_active":"2024-03-06T07:32:04.328261+0000","last_peered":"2024-03-06T07:32:04.328261+0000","last_clean":"2024-03-06T07:32:04.328261+0000","last_became_active":"2024-03-06T07:32:03.304833+0000","last_became_peered":"2024-03-06T07:32:03.304833+0000","last_unstale":"2024-03-06T07:32:04.328261+0000","last_undegraded":"2024-03-06T07:32:04.328261+0000","last_fullsized":"2024-03-06T07:32:04.328261+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:18:02.952948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306790+0000","last_change":"2024-03-06T07:32:04.302685+0000","last_active":"2024-03-06T07:32:04.306790+0000","last_peered":"2024-03-06T07:32:04.306790+0000","last_clean":"2024-03-06T07:32:04.306790+0000","last_became_active":"2024-03-06T07:32:04.302377+0000","last_became_peered":"2024-03-06T07:32:04.302377+0000","last_unstale":"2024-03-06T07:32:04.306790+0000","last_undegraded":"2024-03-06T07:32:04.306790+0000","last_fullsized":"2024-03-06T07:32:04.306790+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:49:26.188782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1e","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.325163+0000","last_change":"2024-03-06T07:32:03.316590+0000","last_active":"2024-03-06T07:32:04.325163+0000","last_peered":"2024-03-06T07:32:04.325163+0000","last_clean":"2024-03-06T07:32:04.325163+0000","last_became_active":"2024-03-06T07:32:03.316232+0000","last_became_peered":"2024-03-06T07:32:03.316232+0000","last_unstale":"2024-03-06T07:32:04.325163+0000","last_undegraded":"2024-03-06T07:32:04.325163+0000","last_fullsized":"2024-03-06T07:32:04.325163+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:14.821460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314170+0000","last_change":"2024-03-06T07:32:04.310944+0000","last_active":"2024-03-06T07:32:04.314170+0000","last_peered":"2024-03-06T07:32:04.314170+0000","last_clean":"2024-03-06T07:32:04.314170+0000","last_became_active":"2024-03-06T07:32:04.310627+0000","last_became_peered":"2024-03-06T07:32:04.310627+0000","last_unstale":"2024-03-06T07:32:04.314170+0000","last_undegraded":"2024-03-06T07:32:04.314170+0000","last_fullsized":"2024-03-06T07:32:04.314170+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:26:35.984137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.1d","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.327979+0000","last_change":"2024-03-06T07:32:03.311842+0000","last_active":"2024-03-06T07:32:04.327979+0000","last_peered":"2024-03-06T07:32:04.327979+0000","last_clean":"2024-03-06T07:32:04.327979+0000","last_became_active":"2024-03-06T07:32:03.311519+0000","last_became_peered":"2024-03-06T07:32:03.311519+0000","last_unstale":"2024-03-06T07:32:04.327979+0000","last_undegraded":"2024-03-06T07:32:04.327979+0000","last_fullsized":"2024-03-06T07:32:04.327979+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:36.280164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318515+0000","last_change":"2024-03-06T07:32:04.311491+0000","last_active":"2024-03-06T07:32:04.318515+0000","last_peered":"2024-03-06T07:32:04.318515+0000","last_clean":"2024-03-06T07:32:04.318515+0000","last_became_active":"2024-03-06T07:32:04.311173+0000","last_became_peered":"2024-03-06T07:32:04.311173+0000","last_unstale":"2024-03-06T07:32:04.318515+0000","last_undegraded":"2024-03-06T07:32:04.318515+0000","last_fullsized":"2024-03-06T07:32:04.318515+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:29:09.128286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1c","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.327967+0000","last_change":"2024-03-06T07:32:03.314678+0000","last_active":"2024-03-06T07:32:04.327967+0000","last_peered":"2024-03-06T07:32:04.327967+0000","last_clean":"2024-03-06T07:32:04.327967+0000","last_became_active":"2024-03-06T07:32:03.314429+0000","last_became_peered":"2024-03-06T07:32:03.314429+0000","last_unstale":"2024-03-06T07:32:04.327967+0000","last_undegraded":"2024-03-06T07:32:04.327967+0000","last_fullsized":"2024-03-06T07:32:04.327967+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:39:56.981785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311056+0000","last_change":"2024-03-06T07:32:04.302114+0000","last_active":"2024-03-06T07:32:04.311056+0000","last_peered":"2024-03-06T07:32:04.311056+0000","last_clean":"2024-03-06T07:32:04.311056+0000","last_became_active":"2024-03-06T07:32:04.301573+0000","last_became_peered":"2024-03-06T07:32:04.301573+0000","last_unstale":"2024-03-06T07:32:04.311056+0000","last_undegraded":"2024-03-06T07:32:04.311056+0000","last_fullsized":"2024-03-06T07:32:04.311056+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:53:54.108260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1b","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.325542+0000","last_change":"2024-03-06T07:32:03.310518+0000","last_active":"2024-03-06T07:32:04.325542+0000","last_peered":"2024-03-06T07:32:04.325542+0000","last_clean":"2024-03-06T07:32:04.325542+0000","last_became_active":"2024-03-06T07:32:03.309440+0000","last_became_peered":"2024-03-06T07:32:03.309440+0000","last_unstale":"2024-03-06T07:32:04.325542+0000","last_undegraded":"2024-03-06T07:32:04.325542+0000","last_fullsized":"2024-03-06T07:32:04.325542+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:04:41.344138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.300614+0000","last_change":"2024-03-06T07:32:04.294490+0000","last_active":"2024-03-06T07:32:04.300614+0000","last_peered":"2024-03-06T07:32:04.300614+0000","last_clean":"2024-03-06T07:32:04.300614+0000","last_became_active":"2024-03-06T07:32:04.294138+0000","last_became_peered":"2024-03-06T07:32:04.294138+0000","last_unstale":"2024-03-06T07:32:04.300614+0000","last_undegraded":"2024-03-06T07:32:04.300614+0000","last_fullsized":"2024-03-06T07:32:04.300614+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:25:16.547115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1a","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.327986+0000","last_change":"2024-03-06T07:32:03.306891+0000","last_active":"2024-03-06T07:32:04.327986+0000","last_peered":"2024-03-06T07:32:04.327986+0000","last_clean":"2024-03-06T07:32:04.327986+0000","last_became_active":"2024-03-06T07:32:03.306619+0000","last_became_peered":"2024-03-06T07:32:03.306619+0000","last_unstale":"2024-03-06T07:32:04.327986+0000","last_undegraded":"2024-03-06T07:32:04.327986+0000","last_fullsized":"2024-03-06T07:32:04.327986+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:09:28.506301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.18","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312100+0000","last_change":"2024-03-06T07:32:04.301926+0000","last_active":"2024-03-06T07:32:04.312100+0000","last_peered":"2024-03-06T07:32:04.312100+0000","last_clean":"2024-03-06T07:32:04.312100+0000","last_became_active":"2024-03-06T07:32:04.301648+0000","last_became_peered":"2024-03-06T07:32:04.301648+0000","last_unstale":"2024-03-06T07:32:04.312100+0000","last_undegraded":"2024-03-06T07:32:04.312100+0000","last_fullsized":"2024-03-06T07:32:04.312100+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:09:37.864764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.19","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.298443+0000","last_change":"2024-03-06T07:32:03.306724+0000","last_active":"2024-03-06T07:32:04.298443+0000","last_peered":"2024-03-06T07:32:04.298443+0000","last_clean":"2024-03-06T07:32:04.298443+0000","last_became_active":"2024-03-06T07:32:03.306373+0000","last_became_peered":"2024-03-06T07:32:03.306373+0000","last_unstale":"2024-03-06T07:32:04.298443+0000","last_undegraded":"2024-03-06T07:32:04.298443+0000","last_fullsized":"2024-03-06T07:32:04.298443+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:59:38.000494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.19","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318914+0000","last_change":"2024-03-06T07:32:04.312286+0000","last_active":"2024-03-06T07:32:04.318914+0000","last_peered":"2024-03-06T07:32:04.318914+0000","last_clean":"2024-03-06T07:32:04.318914+0000","last_became_active":"2024-03-06T07:32:04.312028+0000","last_became_peered":"2024-03-06T07:32:04.312028+0000","last_unstale":"2024-03-06T07:32:04.318914+0000","last_undegraded":"2024-03-06T07:32:04.318914+0000","last_fullsized":"2024-03-06T07:32:04.318914+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:47:46.548145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.18","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.294816+0000","last_change":"2024-03-06T07:32:03.301269+0000","last_active":"2024-03-06T07:32:04.294816+0000","last_peered":"2024-03-06T07:32:04.294816+0000","last_clean":"2024-03-06T07:32:04.294816+0000","last_became_active":"2024-03-06T07:32:03.300997+0000","last_became_peered":"2024-03-06T07:32:03.300997+0000","last_unstale":"2024-03-06T07:32:04.294816+0000","last_undegraded":"2024-03-06T07:32:04.294816+0000","last_fullsized":"2024-03-06T07:32:04.294816+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:36.714406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.16","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317909+0000","last_change":"2024-03-06T07:32:04.310610+0000","last_active":"2024-03-06T07:32:04.317909+0000","last_peered":"2024-03-06T07:32:04.317909+0000","last_clean":"2024-03-06T07:32:04.317909+0000","last_became_active":"2024-03-06T07:32:04.306596+0000","last_became_peered":"2024-03-06T07:32:04.306596+0000","last_unstale":"2024-03-06T07:32:04.317909+0000","last_undegraded":"2024-03-06T07:32:04.317909+0000","last_fullsized":"2024-03-06T07:32:04.317909+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:21:50.619908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.17","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.289307+0000","last_change":"2024-03-06T07:32:03.307878+0000","last_active":"2024-03-06T07:32:04.289307+0000","last_peered":"2024-03-06T07:32:04.289307+0000","last_clean":"2024-03-06T07:32:04.289307+0000","last_became_active":"2024-03-06T07:32:03.307614+0000","last_became_peered":"2024-03-06T07:32:03.307614+0000","last_unstale":"2024-03-06T07:32:04.289307+0000","last_undegraded":"2024-03-06T07:32:04.289307+0000","last_fullsized":"2024-03-06T07:32:04.289307+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:53:35.571679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.17","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.319114+0000","last_change":"2024-03-06T07:32:04.312709+0000","last_active":"2024-03-06T07:32:04.319114+0000","last_peered":"2024-03-06T07:32:04.319114+0000","last_clean":"2024-03-06T07:32:04.319114+0000","last_became_active":"2024-03-06T07:32:04.312441+0000","last_became_peered":"2024-03-06T07:32:04.312441+0000","last_unstale":"2024-03-06T07:32:04.319114+0000","last_undegraded":"2024-03-06T07:32:04.319114+0000","last_fullsized":"2024-03-06T07:32:04.319114+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:02:41.927660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.276176+0000","last_change":"2024-03-06T07:32:03.306977+0000","last_active":"2024-03-06T07:32:04.276176+0000","last_peered":"2024-03-06T07:32:04.276176+0000","last_clean":"2024-03-06T07:32:04.276176+0000","last_became_active":"2024-03-06T07:32:03.306623+0000","last_became_peered":"2024-03-06T07:32:03.306623+0000","last_unstale":"2024-03-06T07:32:04.276176+0000","last_undegraded":"2024-03-06T07:32:04.276176+0000","last_fullsized":"2024-03-06T07:32:04.276176+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:19.853592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.14","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311797+0000","last_change":"2024-03-06T07:32:04.306817+0000","last_active":"2024-03-06T07:32:04.311797+0000","last_peered":"2024-03-06T07:32:04.311797+0000","last_clean":"2024-03-06T07:32:04.311797+0000","last_became_active":"2024-03-06T07:32:04.306536+0000","last_became_peered":"2024-03-06T07:32:04.306536+0000","last_unstale":"2024-03-06T07:32:04.311797+0000","last_undegraded":"2024-03-06T07:32:04.311797+0000","last_fullsized":"2024-03-06T07:32:04.311797+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:00:27.549959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.15","version":"93'3","reported_seq":15,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.324956+0000","last_change":"2024-03-06T07:32:03.316615+0000","last_active":"2024-03-06T07:32:04.324956+0000","last_peered":"2024-03-06T07:32:04.324956+0000","last_clean":"2024-03-06T07:32:04.324956+0000","last_became_active":"2024-03-06T07:32:03.316282+0000","last_became_peered":"2024-03-06T07:32:03.316282+0000","last_unstale":"2024-03-06T07:32:04.324956+0000","last_undegraded":"2024-03-06T07:32:04.324956+0000","last_fullsized":"2024-03-06T07:32:04.324956+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:48:35.839016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.15","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308715+0000","last_change":"2024-03-06T07:32:04.304462+0000","last_active":"2024-03-06T07:32:04.308715+0000","last_peered":"2024-03-06T07:32:04.308715+0000","last_clean":"2024-03-06T07:32:04.308715+0000","last_became_active":"2024-03-06T07:32:04.304220+0000","last_became_peered":"2024-03-06T07:32:04.304220+0000","last_unstale":"2024-03-06T07:32:04.308715+0000","last_undegraded":"2024-03-06T07:32:04.308715+0000","last_fullsized":"2024-03-06T07:32:04.308715+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:47:46.548145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.14","version":"93'6","reported_seq":18,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.331045+0000","last_change":"2024-03-06T07:32:03.309787+0000","last_active":"2024-03-06T07:32:04.331045+0000","last_peered":"2024-03-06T07:32:04.331045+0000","last_clean":"2024-03-06T07:32:04.331045+0000","last_became_active":"2024-03-06T07:32:03.309406+0000","last_became_peered":"2024-03-06T07:32:03.309406+0000","last_unstale":"2024-03-06T07:32:04.331045+0000","last_undegraded":"2024-03-06T07:32:04.331045+0000","last_fullsized":"2024-03-06T07:32:04.331045+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:48:35.839016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.12","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.290045+0000","last_change":"2024-03-06T07:32:04.285596+0000","last_active":"2024-03-06T07:32:04.290045+0000","last_peered":"2024-03-06T07:32:04.290045+0000","last_clean":"2024-03-06T07:32:04.290045+0000","last_became_active":"2024-03-06T07:32:04.284787+0000","last_became_peered":"2024-03-06T07:32:04.284787+0000","last_unstale":"2024-03-06T07:32:04.290045+0000","last_undegraded":"2024-03-06T07:32:04.290045+0000","last_fullsized":"2024-03-06T07:32:04.290045+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:42:28.718856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.325659+0000","last_change":"2024-03-06T07:32:03.308806+0000","last_active":"2024-03-06T07:32:04.325659+0000","last_peered":"2024-03-06T07:32:04.325659+0000","last_clean":"2024-03-06T07:32:04.325659+0000","last_became_active":"2024-03-06T07:32:03.308522+0000","last_became_peered":"2024-03-06T07:32:03.308522+0000","last_unstale":"2024-03-06T07:32:04.325659+0000","last_undegraded":"2024-03-06T07:32:04.325659+0000","last_fullsized":"2024-03-06T07:32:04.325659+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:17:06.059681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.13","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311423+0000","last_change":"2024-03-06T07:32:04.307370+0000","last_active":"2024-03-06T07:32:04.311423+0000","last_peered":"2024-03-06T07:32:04.311423+0000","last_clean":"2024-03-06T07:32:04.311423+0000","last_became_active":"2024-03-06T07:32:04.306926+0000","last_became_peered":"2024-03-06T07:32:04.306926+0000","last_unstale":"2024-03-06T07:32:04.311423+0000","last_undegraded":"2024-03-06T07:32:04.311423+0000","last_fullsized":"2024-03-06T07:32:04.311423+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:22:55.730934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.12","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296128+0000","last_change":"2024-03-06T07:32:03.304837+0000","last_active":"2024-03-06T07:32:04.296128+0000","last_peered":"2024-03-06T07:32:04.296128+0000","last_clean":"2024-03-06T07:32:04.296128+0000","last_became_active":"2024-03-06T07:32:03.302826+0000","last_became_peered":"2024-03-06T07:32:03.302826+0000","last_unstale":"2024-03-06T07:32:04.296128+0000","last_undegraded":"2024-03-06T07:32:04.296128+0000","last_fullsized":"2024-03-06T07:32:04.296128+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:05:44.455324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.10","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314705+0000","last_change":"2024-03-06T07:32:04.306320+0000","last_active":"2024-03-06T07:32:04.314705+0000","last_peered":"2024-03-06T07:32:04.314705+0000","last_clean":"2024-03-06T07:32:04.314705+0000","last_became_active":"2024-03-06T07:32:04.306031+0000","last_became_peered":"2024-03-06T07:32:04.306031+0000","last_unstale":"2024-03-06T07:32:04.314705+0000","last_undegraded":"2024-03-06T07:32:04.314705+0000","last_fullsized":"2024-03-06T07:32:04.314705+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:39:56.428272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.11","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.298857+0000","last_change":"2024-03-06T07:32:03.312323+0000","last_active":"2024-03-06T07:32:04.298857+0000","last_peered":"2024-03-06T07:32:04.298857+0000","last_clean":"2024-03-06T07:32:04.298857+0000","last_became_active":"2024-03-06T07:32:03.312017+0000","last_became_peered":"2024-03-06T07:32:03.312017+0000","last_unstale":"2024-03-06T07:32:04.298857+0000","last_undegraded":"2024-03-06T07:32:04.298857+0000","last_fullsized":"2024-03-06T07:32:04.298857+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:58.935413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.11","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314523+0000","last_change":"2024-03-06T07:32:04.309159+0000","last_active":"2024-03-06T07:32:04.314523+0000","last_peered":"2024-03-06T07:32:04.314523+0000","last_clean":"2024-03-06T07:32:04.314523+0000","last_became_active":"2024-03-06T07:32:04.308874+0000","last_became_peered":"2024-03-06T07:32:04.308874+0000","last_unstale":"2024-03-06T07:32:04.314523+0000","last_undegraded":"2024-03-06T07:32:04.314523+0000","last_fullsized":"2024-03-06T07:32:04.314523+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:03.970098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.10","version":"93'8","reported_seq":20,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.328408+0000","last_change":"2024-03-06T07:32:03.313608+0000","last_active":"2024-03-06T07:32:04.328408+0000","last_peered":"2024-03-06T07:32:04.328408+0000","last_clean":"2024-03-06T07:32:04.328408+0000","last_became_active":"2024-03-06T07:32:03.313143+0000","last_became_peered":"2024-03-06T07:32:03.313143+0000","last_unstale":"2024-03-06T07:32:04.328408+0000","last_undegraded":"2024-03-06T07:32:04.328408+0000","last_fullsized":"2024-03-06T07:32:04.328408+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:49:12.503842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316009+0000","last_change":"2024-03-06T07:32:04.310548+0000","last_active":"2024-03-06T07:32:04.316009+0000","last_peered":"2024-03-06T07:32:04.316009+0000","last_clean":"2024-03-06T07:32:04.316009+0000","last_became_active":"2024-03-06T07:32:04.309925+0000","last_became_peered":"2024-03-06T07:32:04.309925+0000","last_unstale":"2024-03-06T07:32:04.316009+0000","last_undegraded":"2024-03-06T07:32:04.316009+0000","last_fullsized":"2024-03-06T07:32:04.316009+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:58:44.739330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.326492+0000","last_change":"2024-03-06T07:32:03.312422+0000","last_active":"2024-03-06T07:32:04.326492+0000","last_peered":"2024-03-06T07:32:04.326492+0000","last_clean":"2024-03-06T07:32:04.326492+0000","last_became_active":"2024-03-06T07:32:03.312114+0000","last_became_peered":"2024-03-06T07:32:03.312114+0000","last_unstale":"2024-03-06T07:32:04.326492+0000","last_undegraded":"2024-03-06T07:32:04.326492+0000","last_fullsized":"2024-03-06T07:32:04.326492+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:14.821460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311206+0000","last_change":"2024-03-06T07:32:04.302878+0000","last_active":"2024-03-06T07:32:04.311206+0000","last_peered":"2024-03-06T07:32:04.311206+0000","last_clean":"2024-03-06T07:32:04.311206+0000","last_became_active":"2024-03-06T07:32:04.302034+0000","last_became_peered":"2024-03-06T07:32:04.302034+0000","last_unstale":"2024-03-06T07:32:04.311206+0000","last_undegraded":"2024-03-06T07:32:04.311206+0000","last_fullsized":"2024-03-06T07:32:04.311206+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:27.142777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.d","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.322183+0000","last_change":"2024-03-06T07:32:03.302127+0000","last_active":"2024-03-06T07:32:04.322183+0000","last_peered":"2024-03-06T07:32:04.322183+0000","last_clean":"2024-03-06T07:32:04.322183+0000","last_became_active":"2024-03-06T07:32:03.301282+0000","last_became_peered":"2024-03-06T07:32:03.301282+0000","last_unstale":"2024-03-06T07:32:04.322183+0000","last_undegraded":"2024-03-06T07:32:04.322183+0000","last_fullsized":"2024-03-06T07:32:04.322183+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:59:29.195494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308694+0000","last_change":"2024-03-06T07:32:04.300889+0000","last_active":"2024-03-06T07:32:04.308694+0000","last_peered":"2024-03-06T07:32:04.308694+0000","last_clean":"2024-03-06T07:32:04.308694+0000","last_became_active":"2024-03-06T07:32:04.299489+0000","last_became_peered":"2024-03-06T07:32:04.299489+0000","last_unstale":"2024-03-06T07:32:04.308694+0000","last_undegraded":"2024-03-06T07:32:04.308694+0000","last_fullsized":"2024-03-06T07:32:04.308694+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:46.541754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.288447+0000","last_change":"2024-03-06T07:32:03.314945+0000","last_active":"2024-03-06T07:32:04.288447+0000","last_peered":"2024-03-06T07:32:04.288447+0000","last_clean":"2024-03-06T07:32:04.288447+0000","last_became_active":"2024-03-06T07:32:03.314467+0000","last_became_peered":"2024-03-06T07:32:03.314467+0000","last_unstale":"2024-03-06T07:32:04.288447+0000","last_undegraded":"2024-03-06T07:32:04.288447+0000","last_fullsized":"2024-03-06T07:32:04.288447+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:54:29.350079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311525+0000","last_change":"2024-03-06T07:32:04.306751+0000","last_active":"2024-03-06T07:32:04.311525+0000","last_peered":"2024-03-06T07:32:04.311525+0000","last_clean":"2024-03-06T07:32:04.311525+0000","last_became_active":"2024-03-06T07:32:04.306490+0000","last_became_peered":"2024-03-06T07:32:04.306490+0000","last_unstale":"2024-03-06T07:32:04.311525+0000","last_undegraded":"2024-03-06T07:32:04.311525+0000","last_fullsized":"2024-03-06T07:32:04.311525+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:04:26.208566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.328370+0000","last_change":"2024-03-06T07:32:03.304943+0000","last_active":"2024-03-06T07:32:04.328370+0000","last_peered":"2024-03-06T07:32:04.328370+0000","last_clean":"2024-03-06T07:32:04.328370+0000","last_became_active":"2024-03-06T07:32:03.304483+0000","last_became_peered":"2024-03-06T07:32:03.304483+0000","last_unstale":"2024-03-06T07:32:04.328370+0000","last_undegraded":"2024-03-06T07:32:04.328370+0000","last_fullsized":"2024-03-06T07:32:04.328370+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:19.853592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.322954+0000","last_change":"2024-03-06T07:32:04.314919+0000","last_active":"2024-03-06T07:32:04.322954+0000","last_peered":"2024-03-06T07:32:04.322954+0000","last_clean":"2024-03-06T07:32:04.322954+0000","last_became_active":"2024-03-06T07:32:04.313624+0000","last_became_peered":"2024-03-06T07:32:04.313624+0000","last_unstale":"2024-03-06T07:32:04.322954+0000","last_undegraded":"2024-03-06T07:32:04.322954+0000","last_fullsized":"2024-03-06T07:32:04.322954+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:49.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.290470+0000","last_change":"2024-03-06T07:32:03.313220+0000","last_active":"2024-03-06T07:32:04.290470+0000","last_peered":"2024-03-06T07:32:04.290470+0000","last_clean":"2024-03-06T07:32:04.290470+0000","last_became_active":"2024-03-06T07:32:03.311795+0000","last_became_peered":"2024-03-06T07:32:03.311795+0000","last_unstale":"2024-03-06T07:32:04.290470+0000","last_undegraded":"2024-03-06T07:32:04.290470+0000","last_fullsized":"2024-03-06T07:32:04.290470+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:37:25.322534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.8","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314901+0000","last_change":"2024-03-06T07:32:04.306137+0000","last_active":"2024-03-06T07:32:04.314901+0000","last_peered":"2024-03-06T07:32:04.314901+0000","last_clean":"2024-03-06T07:32:04.314901+0000","last_became_active":"2024-03-06T07:32:04.305821+0000","last_became_peered":"2024-03-06T07:32:04.305821+0000","last_unstale":"2024-03-06T07:32:04.314901+0000","last_undegraded":"2024-03-06T07:32:04.314901+0000","last_fullsized":"2024-03-06T07:32:04.314901+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:07:16.464259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.288681+0000","last_change":"2024-03-06T07:32:03.301502+0000","last_active":"2024-03-06T07:32:04.288681+0000","last_peered":"2024-03-06T07:32:04.288681+0000","last_clean":"2024-03-06T07:32:04.288681+0000","last_became_active":"2024-03-06T07:32:03.301158+0000","last_became_peered":"2024-03-06T07:32:03.301158+0000","last_unstale":"2024-03-06T07:32:04.288681+0000","last_undegraded":"2024-03-06T07:32:04.288681+0000","last_fullsized":"2024-03-06T07:32:04.288681+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:48:29.845871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.9","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.319454+0000","last_change":"2024-03-06T07:32:04.313896+0000","last_active":"2024-03-06T07:32:04.319454+0000","last_peered":"2024-03-06T07:32:04.319454+0000","last_clean":"2024-03-06T07:32:04.319454+0000","last_became_active":"2024-03-06T07:32:04.313208+0000","last_became_peered":"2024-03-06T07:32:04.313208+0000","last_unstale":"2024-03-06T07:32:04.319454+0000","last_undegraded":"2024-03-06T07:32:04.319454+0000","last_fullsized":"2024-03-06T07:32:04.319454+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:28:56.208174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296146+0000","last_change":"2024-03-06T07:32:03.306802+0000","last_active":"2024-03-06T07:32:04.296146+0000","last_peered":"2024-03-06T07:32:04.296146+0000","last_clean":"2024-03-06T07:32:04.296146+0000","last_became_active":"2024-03-06T07:32:03.306462+0000","last_became_peered":"2024-03-06T07:32:03.306462+0000","last_unstale":"2024-03-06T07:32:04.296146+0000","last_undegraded":"2024-03-06T07:32:04.296146+0000","last_fullsized":"2024-03-06T07:32:04.296146+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:42:27.702645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.6","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318084+0000","last_change":"2024-03-06T07:32:04.312917+0000","last_active":"2024-03-06T07:32:04.318084+0000","last_peered":"2024-03-06T07:32:04.318084+0000","last_clean":"2024-03-06T07:32:04.318084+0000","last_became_active":"2024-03-06T07:32:04.312110+0000","last_became_peered":"2024-03-06T07:32:04.312110+0000","last_unstale":"2024-03-06T07:32:04.318084+0000","last_undegraded":"2024-03-06T07:32:04.318084+0000","last_fullsized":"2024-03-06T07:32:04.318084+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:08:51.374255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.7","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.324963+0000","last_change":"2024-03-06T07:32:03.309874+0000","last_active":"2024-03-06T07:32:04.324963+0000","last_peered":"2024-03-06T07:32:04.324963+0000","last_clean":"2024-03-06T07:32:04.324963+0000","last_became_active":"2024-03-06T07:32:03.309270+0000","last_became_peered":"2024-03-06T07:32:03.309270+0000","last_unstale":"2024-03-06T07:32:04.324963+0000","last_undegraded":"2024-03-06T07:32:04.324963+0000","last_fullsized":"2024-03-06T07:32:04.324963+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:54:14.722610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.7","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308885+0000","last_change":"2024-03-06T07:32:04.305953+0000","last_active":"2024-03-06T07:32:04.308885+0000","last_peered":"2024-03-06T07:32:04.308885+0000","last_clean":"2024-03-06T07:32:04.308885+0000","last_became_active":"2024-03-06T07:32:04.305660+0000","last_became_peered":"2024-03-06T07:32:04.305660+0000","last_unstale":"2024-03-06T07:32:04.308885+0000","last_undegraded":"2024-03-06T07:32:04.308885+0000","last_fullsized":"2024-03-06T07:32:04.308885+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:41.545875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.301695+0000","last_change":"2024-03-06T07:32:03.303397+0000","last_active":"2024-03-06T07:32:04.301695+0000","last_peered":"2024-03-06T07:32:04.301695+0000","last_clean":"2024-03-06T07:32:04.301695+0000","last_became_active":"2024-03-06T07:32:03.303117+0000","last_became_peered":"2024-03-06T07:32:03.303117+0000","last_unstale":"2024-03-06T07:32:04.301695+0000","last_undegraded":"2024-03-06T07:32:04.301695+0000","last_fullsized":"2024-03-06T07:32:04.301695+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:09:36.848553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.4","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.309930+0000","last_change":"2024-03-06T07:32:04.303078+0000","last_active":"2024-03-06T07:32:04.309930+0000","last_peered":"2024-03-06T07:32:04.309930+0000","last_clean":"2024-03-06T07:32:04.309930+0000","last_became_active":"2024-03-06T07:32:04.302771+0000","last_became_peered":"2024-03-06T07:32:04.302771+0000","last_unstale":"2024-03-06T07:32:04.309930+0000","last_undegraded":"2024-03-06T07:32:04.309930+0000","last_fullsized":"2024-03-06T07:32:04.309930+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:24:16.622950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296635+0000","last_change":"2024-03-06T07:32:03.310369+0000","last_active":"2024-03-06T07:32:04.296635+0000","last_peered":"2024-03-06T07:32:04.296635+0000","last_clean":"2024-03-06T07:32:04.296635+0000","last_became_active":"2024-03-06T07:32:03.309892+0000","last_became_peered":"2024-03-06T07:32:03.309892+0000","last_unstale":"2024-03-06T07:32:04.296635+0000","last_undegraded":"2024-03-06T07:32:04.296635+0000","last_fullsized":"2024-03-06T07:32:04.296635+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:49:25.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.5","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316257+0000","last_change":"2024-03-06T07:32:04.311455+0000","last_active":"2024-03-06T07:32:04.316257+0000","last_peered":"2024-03-06T07:32:04.316257+0000","last_clean":"2024-03-06T07:32:04.316257+0000","last_became_active":"2024-03-06T07:32:04.311106+0000","last_became_peered":"2024-03-06T07:32:04.311106+0000","last_unstale":"2024-03-06T07:32:04.316257+0000","last_undegraded":"2024-03-06T07:32:04.316257+0000","last_fullsized":"2024-03-06T07:32:04.316257+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:51:25.630627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296510+0000","last_change":"2024-03-06T07:32:03.311940+0000","last_active":"2024-03-06T07:32:04.296510+0000","last_peered":"2024-03-06T07:32:04.296510+0000","last_clean":"2024-03-06T07:32:04.296510+0000","last_became_active":"2024-03-06T07:32:03.311270+0000","last_became_peered":"2024-03-06T07:32:03.311270+0000","last_unstale":"2024-03-06T07:32:04.296510+0000","last_undegraded":"2024-03-06T07:32:04.296510+0000","last_fullsized":"2024-03-06T07:32:04.296510+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:54:14.722610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.2","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315087+0000","last_change":"2024-03-06T07:32:04.309953+0000","last_active":"2024-03-06T07:32:04.315087+0000","last_peered":"2024-03-06T07:32:04.315087+0000","last_clean":"2024-03-06T07:32:04.315087+0000","last_became_active":"2024-03-06T07:32:04.309671+0000","last_became_peered":"2024-03-06T07:32:04.309671+0000","last_unstale":"2024-03-06T07:32:04.315087+0000","last_undegraded":"2024-03-06T07:32:04.315087+0000","last_fullsized":"2024-03-06T07:32:04.315087+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:22.833719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.292791+0000","last_change":"2024-03-06T07:32:03.302900+0000","last_active":"2024-03-06T07:32:04.292791+0000","last_peered":"2024-03-06T07:32:04.292791+0000","last_clean":"2024-03-06T07:32:04.292791+0000","last_became_active":"2024-03-06T07:32:03.301642+0000","last_became_peered":"2024-03-06T07:32:03.301642+0000","last_unstale":"2024-03-06T07:32:04.292791+0000","last_undegraded":"2024-03-06T07:32:04.292791+0000","last_fullsized":"2024-03-06T07:32:04.292791+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:34:23.525002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.319171+0000","last_change":"2024-03-06T07:32:04.315091+0000","last_active":"2024-03-06T07:32:04.319171+0000","last_peered":"2024-03-06T07:32:04.319171+0000","last_clean":"2024-03-06T07:32:04.319171+0000","last_became_active":"2024-03-06T07:32:04.314821+0000","last_became_peered":"2024-03-06T07:32:04.314821+0000","last_unstale":"2024-03-06T07:32:04.319171+0000","last_undegraded":"2024-03-06T07:32:04.319171+0000","last_fullsized":"2024-03-06T07:32:04.319171+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:39:57.997996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.295853+0000","last_change":"2024-03-06T07:32:03.308759+0000","last_active":"2024-03-06T07:32:04.295853+0000","last_peered":"2024-03-06T07:32:04.295853+0000","last_clean":"2024-03-06T07:32:04.295853+0000","last_became_active":"2024-03-06T07:32:03.308147+0000","last_became_peered":"2024-03-06T07:32:03.308147+0000","last_unstale":"2024-03-06T07:32:04.295853+0000","last_undegraded":"2024-03-06T07:32:04.295853+0000","last_fullsized":"2024-03-06T07:32:04.295853+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:26.126566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.0","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.323014+0000","last_change":"2024-03-06T07:32:04.314992+0000","last_active":"2024-03-06T07:32:04.323014+0000","last_peered":"2024-03-06T07:32:04.323014+0000","last_clean":"2024-03-06T07:32:04.323014+0000","last_became_active":"2024-03-06T07:32:04.314714+0000","last_became_peered":"2024-03-06T07:32:04.314714+0000","last_unstale":"2024-03-06T07:32:04.323014+0000","last_undegraded":"2024-03-06T07:32:04.323014+0000","last_fullsized":"2024-03-06T07:32:04.323014+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:05:58.428465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.276313+0000","last_change":"2024-03-06T07:32:03.302302+0000","last_active":"2024-03-06T07:32:04.276313+0000","last_peered":"2024-03-06T07:32:04.276313+0000","last_clean":"2024-03-06T07:32:04.276313+0000","last_became_active":"2024-03-06T07:32:03.301798+0000","last_became_peered":"2024-03-06T07:32:03.301798+0000","last_unstale":"2024-03-06T07:32:04.276313+0000","last_undegraded":"2024-03-06T07:32:04.276313+0000","last_fullsized":"2024-03-06T07:32:04.276313+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:34:23.525002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.1","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311803+0000","last_change":"2024-03-06T07:32:04.304607+0000","last_active":"2024-03-06T07:32:04.311803+0000","last_peered":"2024-03-06T07:32:04.311803+0000","last_clean":"2024-03-06T07:32:04.311803+0000","last_became_active":"2024-03-06T07:32:04.304347+0000","last_became_peered":"2024-03-06T07:32:04.304347+0000","last_unstale":"2024-03-06T07:32:04.311803+0000","last_undegraded":"2024-03-06T07:32:04.311803+0000","last_fullsized":"2024-03-06T07:32:04.311803+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:37:28.711999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.292436+0000","last_change":"2024-03-06T07:32:03.307185+0000","last_active":"2024-03-06T07:32:04.292436+0000","last_peered":"2024-03-06T07:32:04.292436+0000","last_clean":"2024-03-06T07:32:04.292436+0000","last_became_active":"2024-03-06T07:32:03.305504+0000","last_became_peered":"2024-03-06T07:32:03.305504+0000","last_unstale":"2024-03-06T07:32:04.292436+0000","last_undegraded":"2024-03-06T07:32:04.292436+0000","last_fullsized":"2024-03-06T07:32:04.292436+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:51:36.507024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.309140+0000","last_change":"2024-03-06T07:32:04.305332+0000","last_active":"2024-03-06T07:32:04.309140+0000","last_peered":"2024-03-06T07:32:04.309140+0000","last_clean":"2024-03-06T07:32:04.309140+0000","last_became_active":"2024-03-06T07:32:04.305068+0000","last_became_peered":"2024-03-06T07:32:04.305068+0000","last_unstale":"2024-03-06T07:32:04.309140+0000","last_undegraded":"2024-03-06T07:32:04.309140+0000","last_fullsized":"2024-03-06T07:32:04.309140+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:41.183833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.f","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.323034+0000","last_change":"2024-03-06T07:32:03.319022+0000","last_active":"2024-03-06T07:32:04.323034+0000","last_peered":"2024-03-06T07:32:04.323034+0000","last_clean":"2024-03-06T07:32:04.323034+0000","last_became_active":"2024-03-06T07:32:03.318753+0000","last_became_peered":"2024-03-06T07:32:03.318753+0000","last_unstale":"2024-03-06T07:32:04.323034+0000","last_undegraded":"2024-03-06T07:32:04.323034+0000","last_fullsized":"2024-03-06T07:32:04.323034+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:58.935413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":301,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.276225+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:32:04.276225+0000","last_peered":"2024-03-06T07:32:04.276225+0000","last_clean":"2024-03-06T07:32:04.276225+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:32:04.276225+0000","last_undegraded":"2024-03-06T07:32:04.276225+0000","last_fullsized":"2024-03-06T07:32:04.276225+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":15,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":14,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542209,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32896,"kb_used_data":1708,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338944,"statfs":{"total":96636764160,"available":96603078656,"internally_reserved":0,"allocated":1748992,"data_stored":1077508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542208,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32288,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339552,"statfs":{"total":96636764160,"available":96603701248,"internally_reserved":0,"allocated":1134592,"data_stored":480253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542209,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32308,"kb_used_data":1124,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339532,"statfs":{"total":96636764160,"available":96603680768,"internally_reserved":0,"allocated":1150976,"data_stored":487098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542208,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32328,"kb_used_data":1140,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339512,"statfs":{"total":96636764160,"available":96603660288,"internally_reserved":0,"allocated":1167360,"data_stored":488764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542208,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32292,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339548,"statfs":{"total":96636764160,"available":96603697152,"internally_reserved":0,"allocated":1134592,"data_stored":480253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542209,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32312,"kb_used_data":1124,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339528,"statfs":{"total":96636764160,"available":96603676672,"internally_reserved":0,"allocated":1150976,"data_stored":487062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542210,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32292,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339548,"statfs":{"total":96636764160,"available":96603697152,"internally_reserved":0,"allocated":1134592,"data_stored":480253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542207,"num_pgs":11,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32872,"kb_used_data":1688,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338968,"statfs":{"total":96636764160,"available":96603103232,"internally_reserved":0,"allocated":1728512,"data_stored":1070621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:32:08.722 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:32:08.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.973+0000 7fe04d800640 1 Processor -- start 2024-03-06T07:32:08.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.973+0000 7fe04d800640 1 -- start start 2024-03-06T07:32:08.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.975+0000 7fe04d800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 0x7fe04815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.975+0000 7fe04d800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe04806cef0 con 0x7fe048070b10 2024-03-06T07:32:08.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.975+0000 7fe04d800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe04806d1d0 con 0x7fe048144c00 2024-03-06T07:32:08.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.975+0000 7fe04d800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe048160370 con 0x7fe04815f9d0 2024-03-06T07:32:08.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.975+0000 7fe047400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe048070b10 0x7fe048070f60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55938/0 (socket says 172.21.15.86:55938) 2024-03-06T07:32:08.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.975+0000 7fe047400640 1 -- 172.21.15.86:0/633271940 learned_addr learned my addr 172.21.15.86:0/633271940 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:08.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe046000640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 0x7fe04815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 285351203 0 0) 0x7fe04806cef0 con 0x7fe048070b10 2024-03-06T07:32:08.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe02c003520 con 0x7fe048070b10 2024-03-06T07:32:08.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1130211476 0 0) 0x7fe02c003520 con 0x7fe048070b10 2024-03-06T07:32:08.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 msgr2=0x7fe04815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:08.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 0x7fe04815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 >> v1:172.21.15.194:6789/0 conn(0x7fe048144c00 legacy=0x7fe04806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe04806d4b0 con 0x7fe048070b10 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fe0400019f0 con 0x7fe048070b10 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 >> v1:172.21.15.86:6789/0 conn(0x7fe048070b10 legacy=0x7fe048070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 0x7fe02c0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe02c003f60 0x7fe02c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe02c004d40 0x7fe02c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe02c00ba70 con 0x7fe02c004d40 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe02c00bc20 con 0x7fe02c003f60 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.976+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe02c00be20 con 0x7fe04815f9d0 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe047400640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe02c004d40 0x7fe02c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe046a00640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 0x7fe02c0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe046000640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe02c003f60 0x7fe02c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe047400640 1 -- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 msgr2=0x7fe02c0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:08.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe047400640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe04815f9d0 0x7fe02c0039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe047400640 1 -- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe02c003f60 msgr2=0x7fe02c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:08.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe047400640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe02c003f60 0x7fe02c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.977+0000 7fe047400640 1 -- 172.21.15.86:0/633271940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe02c003520 con 0x7fe02c004d40 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.978+0000 7fe047400640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe02c004d40 0x7fe02c0051c0 secure :-1 s=READY pgs=1086 cs=0 l=1 rev1=1 crypto rx=0x7fe04000a850 tx=0x7fe02c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d74bf3041957563 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe04000a4a0 con 0x7fe02c004d40 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe040034650 con 0x7fe02c004d40 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe045600640 1 -- 172.21.15.86:0/633271940 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe040034a80 con 0x7fe02c004d40 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe04d800640 1 -- 172.21.15.86:0/633271940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe02c004d40 msgr2=0x7fe02c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe04d800640 1 --2- 172.21.15.86:0/633271940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe02c004d40 0x7fe02c0051c0 secure :-1 s=READY pgs=1086 cs=0 l=1 rev1=1 crypto rx=0x7fe04000a850 tx=0x7fe02c005830 comp rx=0 tx=0).stop 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe047400640 1 -- 172.21.15.86:0/633271940 reap_dead start 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe04d800640 1 -- 172.21.15.86:0/633271940 shutdown_connections 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.979+0000 7fe04d800640 1 -- 172.21.15.86:0/633271940 >> 172.21.15.86:0/633271940 conn(0x7fe0480b1730 msgr2=0x7fe0480b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.980+0000 7fe04d800640 1 -- 172.21.15.86:0/633271940 shutdown_connections 2024-03-06T07:32:08.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.980+0000 7fe04d800640 1 -- 172.21.15.86:0/633271940 wait complete. 2024-03-06T07:32:08.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.982+0000 7fe04d800640 1 Processor -- start 2024-03-06T07:32:08.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.982+0000 7fe04d800640 1 -- start start 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe04d800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0481393e0 0x7fe04815dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe04d800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0481397c0 0x7fe0480b3cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe04d800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe04815b710 0x7fe0480b4230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe04d800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe04806cb50 con 0x7fe04815b710 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe04d800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe048160730 con 0x7fe0481397c0 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe04d800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe04806ccd0 con 0x7fe0481393e0 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe047400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe04815b710 0x7fe0480b4230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe047400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe04815b710 0x7fe0480b4230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45696/0 (socket says 172.21.15.86:45696) 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe047400640 1 -- 172.21.15.86:0/4233842390 learned_addr learned my addr 172.21.15.86:0/4233842390 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.983+0000 7fe046a00640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0481393e0 0x7fe04815dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe046000640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0481397c0 0x7fe0480b3cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe047400640 1 -- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0481393e0 msgr2=0x7fe04815dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe047400640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0481393e0 0x7fe04815dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe047400640 1 -- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0481397c0 msgr2=0x7fe0480b3cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe047400640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0481397c0 0x7fe0480b3cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe047400640 1 -- 172.21.15.86:0/4233842390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe04815e230 con 0x7fe04815b710 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe046a00640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0481393e0 0x7fe04815dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe046000640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0481397c0 0x7fe0480b3cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe047400640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe04815b710 0x7fe0480b4230 secure :-1 s=READY pgs=1087 cs=0 l=1 rev1=1 crypto rx=0x7fe04002cc70 tx=0x7fe040001dc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe03ea00640 1 -- 172.21.15.86:0/4233842390 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe040045070 con 0x7fe04815b710 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe03ea00640 1 -- 172.21.15.86:0/4233842390 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe04003a040 con 0x7fe04815b710 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe04813bfc0 con 0x7fe04815b710 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe03ea00640 1 -- 172.21.15.86:0/4233842390 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe040034a40 con 0x7fe04815b710 2024-03-06T07:32:08.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.984+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe04813c580 con 0x7fe04815b710 2024-03-06T07:32:08.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.986+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe0480b2920 con 0x7fe04815b710 2024-03-06T07:32:08.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.987+0000 7fe03ea00640 1 -- 172.21.15.86:0/4233842390 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe040048070 con 0x7fe04815b710 2024-03-06T07:32:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.988+0000 7fe03ea00640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe02806b4d0 0x7fe02806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:08.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.988+0000 7fe03ea00640 1 -- 172.21.15.86:0/4233842390 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe0400abd10 con 0x7fe04815b710 2024-03-06T07:32:08.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.988+0000 7fe046a00640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe02806b4d0 0x7fe02806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:08.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.989+0000 7fe046a00640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe02806b4d0 0x7fe02806d990 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7fe0380016b0 tx=0x7fe038009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:08.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:08.993+0000 7fe03ea00640 1 -- 172.21.15.86:0/4233842390 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe0480b2920 con 0x7fe04815b710 2024-03-06T07:32:09.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.182+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fe04804f330 con 0x7fe02806b4d0 2024-03-06T07:32:09.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.189+0000 7fe03ea00640 1 -- 172.21.15.86:0/4233842390 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252020 (secure 0 0 0) 0x7fe04804f330 con 0x7fe02806b4d0 2024-03-06T07:32:09.193 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:09.196 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe02806b4d0 msgr2=0x7fe02806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe02806b4d0 0x7fe02806d990 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7fe0380016b0 tx=0x7fe038009040 comp rx=0 tx=0).stop 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe04815b710 msgr2=0x7fe0480b4230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe04815b710 0x7fe0480b4230 secure :-1 s=READY pgs=1087 cs=0 l=1 rev1=1 crypto rx=0x7fe04002cc70 tx=0x7fe040001dc0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 shutdown_connections 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe02806b4d0 0x7fe02806d990 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe04815b710 0x7fe0480b4230 unknown :-1 s=CLOSED pgs=1087 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe0481397c0 0x7fe0480b3cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 --2- 172.21.15.86:0/4233842390 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe0481393e0 0x7fe04815dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 >> 172.21.15.86:0/4233842390 conn(0x7fe0480b1730 msgr2=0x7fe0480b0ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 shutdown_connections 2024-03-06T07:32:09.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.196+0000 7fe04d800640 1 -- 172.21.15.86:0/4233842390 wait complete. 2024-03-06T07:32:09.216 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":242,"stamp":"2024-03-06T07:32:08.943803+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":126,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":259588,"kb_used_data":10108,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754715132,"statfs":{"total":773094113280,"available":772828295168,"internally_reserved":0,"allocated":10350592,"data_stored":5051812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":255315507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":33,"apply_latency_ms":33,"commit_latency_ns":33000000,"apply_latency_ns":33000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001692"},"pg_stats":[{"pgid":"15.3f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317906+0000","last_change":"2024-03-06T07:32:04.314244+0000","last_active":"2024-03-06T07:32:04.317906+0000","last_peered":"2024-03-06T07:32:04.317906+0000","last_clean":"2024-03-06T07:32:04.317906+0000","last_became_active":"2024-03-06T07:32:04.313906+0000","last_became_peered":"2024-03-06T07:32:04.313906+0000","last_unstale":"2024-03-06T07:32:04.317906+0000","last_undegraded":"2024-03-06T07:32:04.317906+0000","last_fullsized":"2024-03-06T07:32:04.317906+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:35:22.142943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.320921+0000","last_change":"2024-03-06T07:32:04.314655+0000","last_active":"2024-03-06T07:32:04.320921+0000","last_peered":"2024-03-06T07:32:04.320921+0000","last_clean":"2024-03-06T07:32:04.320921+0000","last_became_active":"2024-03-06T07:32:04.313184+0000","last_became_peered":"2024-03-06T07:32:04.313184+0000","last_unstale":"2024-03-06T07:32:04.320921+0000","last_undegraded":"2024-03-06T07:32:04.320921+0000","last_fullsized":"2024-03-06T07:32:04.320921+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:49:41.093474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.323215+0000","last_change":"2024-03-06T07:32:04.316023+0000","last_active":"2024-03-06T07:32:04.323215+0000","last_peered":"2024-03-06T07:32:04.323215+0000","last_clean":"2024-03-06T07:32:04.323215+0000","last_became_active":"2024-03-06T07:32:04.315681+0000","last_became_peered":"2024-03-06T07:32:04.315681+0000","last_unstale":"2024-03-06T07:32:04.323215+0000","last_undegraded":"2024-03-06T07:32:04.323215+0000","last_fullsized":"2024-03-06T07:32:04.323215+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:34:04.328114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312041+0000","last_change":"2024-03-06T07:32:04.303947+0000","last_active":"2024-03-06T07:32:04.312041+0000","last_peered":"2024-03-06T07:32:04.312041+0000","last_clean":"2024-03-06T07:32:04.312041+0000","last_became_active":"2024-03-06T07:32:04.303659+0000","last_became_peered":"2024-03-06T07:32:04.303659+0000","last_unstale":"2024-03-06T07:32:04.312041+0000","last_undegraded":"2024-03-06T07:32:04.312041+0000","last_fullsized":"2024-03-06T07:32:04.312041+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:53:12.334707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.3b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311609+0000","last_change":"2024-03-06T07:32:04.307511+0000","last_active":"2024-03-06T07:32:04.311609+0000","last_peered":"2024-03-06T07:32:04.311609+0000","last_clean":"2024-03-06T07:32:04.311609+0000","last_became_active":"2024-03-06T07:32:04.307258+0000","last_became_peered":"2024-03-06T07:32:04.307258+0000","last_unstale":"2024-03-06T07:32:04.311609+0000","last_undegraded":"2024-03-06T07:32:04.311609+0000","last_fullsized":"2024-03-06T07:32:04.311609+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:04:19.985344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.3a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.313026+0000","last_change":"2024-03-06T07:32:04.306619+0000","last_active":"2024-03-06T07:32:04.313026+0000","last_peered":"2024-03-06T07:32:04.313026+0000","last_clean":"2024-03-06T07:32:04.313026+0000","last_became_active":"2024-03-06T07:32:04.306324+0000","last_became_peered":"2024-03-06T07:32:04.306324+0000","last_unstale":"2024-03-06T07:32:04.313026+0000","last_undegraded":"2024-03-06T07:32:04.313026+0000","last_fullsized":"2024-03-06T07:32:04.313026+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:02:41.927660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.39","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312996+0000","last_change":"2024-03-06T07:32:04.307198+0000","last_active":"2024-03-06T07:32:04.312996+0000","last_peered":"2024-03-06T07:32:04.312996+0000","last_clean":"2024-03-06T07:32:04.312996+0000","last_became_active":"2024-03-06T07:32:04.306753+0000","last_became_peered":"2024-03-06T07:32:04.306753+0000","last_unstale":"2024-03-06T07:32:04.312996+0000","last_undegraded":"2024-03-06T07:32:04.312996+0000","last_fullsized":"2024-03-06T07:32:04.312996+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:39:57.997996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.38","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317991+0000","last_change":"2024-03-06T07:32:04.308105+0000","last_active":"2024-03-06T07:32:04.317991+0000","last_peered":"2024-03-06T07:32:04.317991+0000","last_clean":"2024-03-06T07:32:04.317991+0000","last_became_active":"2024-03-06T07:32:04.307834+0000","last_became_peered":"2024-03-06T07:32:04.307834+0000","last_unstale":"2024-03-06T07:32:04.317991+0000","last_undegraded":"2024-03-06T07:32:04.317991+0000","last_fullsized":"2024-03-06T07:32:04.317991+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:14:18.650793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.37","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315140+0000","last_change":"2024-03-06T07:32:04.306823+0000","last_active":"2024-03-06T07:32:04.315140+0000","last_peered":"2024-03-06T07:32:04.315140+0000","last_clean":"2024-03-06T07:32:04.315140+0000","last_became_active":"2024-03-06T07:32:04.306533+0000","last_became_peered":"2024-03-06T07:32:04.306533+0000","last_unstale":"2024-03-06T07:32:04.315140+0000","last_undegraded":"2024-03-06T07:32:04.315140+0000","last_fullsized":"2024-03-06T07:32:04.315140+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:46.541754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.36","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316494+0000","last_change":"2024-03-06T07:32:04.312947+0000","last_active":"2024-03-06T07:32:04.316494+0000","last_peered":"2024-03-06T07:32:04.316494+0000","last_clean":"2024-03-06T07:32:04.316494+0000","last_became_active":"2024-03-06T07:32:04.312619+0000","last_became_peered":"2024-03-06T07:32:04.312619+0000","last_unstale":"2024-03-06T07:32:04.316494+0000","last_undegraded":"2024-03-06T07:32:04.316494+0000","last_fullsized":"2024-03-06T07:32:04.316494+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:13:55.263699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.35","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.313857+0000","last_change":"2024-03-06T07:32:04.307949+0000","last_active":"2024-03-06T07:32:04.313857+0000","last_peered":"2024-03-06T07:32:04.313857+0000","last_clean":"2024-03-06T07:32:04.313857+0000","last_became_active":"2024-03-06T07:32:04.307208+0000","last_became_peered":"2024-03-06T07:32:04.307208+0000","last_unstale":"2024-03-06T07:32:04.313857+0000","last_undegraded":"2024-03-06T07:32:04.313857+0000","last_fullsized":"2024-03-06T07:32:04.313857+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:09:29.522513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.34","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308463+0000","last_change":"2024-03-06T07:32:04.301664+0000","last_active":"2024-03-06T07:32:04.308463+0000","last_peered":"2024-03-06T07:32:04.308463+0000","last_clean":"2024-03-06T07:32:04.308463+0000","last_became_active":"2024-03-06T07:32:04.301166+0000","last_became_peered":"2024-03-06T07:32:04.301166+0000","last_unstale":"2024-03-06T07:32:04.308463+0000","last_undegraded":"2024-03-06T07:32:04.308463+0000","last_fullsized":"2024-03-06T07:32:04.308463+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:25:16.547115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.33","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311672+0000","last_change":"2024-03-06T07:32:04.304723+0000","last_active":"2024-03-06T07:32:04.311672+0000","last_peered":"2024-03-06T07:32:04.311672+0000","last_clean":"2024-03-06T07:32:04.311672+0000","last_became_active":"2024-03-06T07:32:04.304389+0000","last_became_peered":"2024-03-06T07:32:04.304389+0000","last_unstale":"2024-03-06T07:32:04.311672+0000","last_undegraded":"2024-03-06T07:32:04.311672+0000","last_fullsized":"2024-03-06T07:32:04.311672+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:22:05.288497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.32","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306910+0000","last_change":"2024-03-06T07:32:04.302128+0000","last_active":"2024-03-06T07:32:04.306910+0000","last_peered":"2024-03-06T07:32:04.306910+0000","last_clean":"2024-03-06T07:32:04.306910+0000","last_became_active":"2024-03-06T07:32:04.301306+0000","last_became_peered":"2024-03-06T07:32:04.301306+0000","last_unstale":"2024-03-06T07:32:04.306910+0000","last_undegraded":"2024-03-06T07:32:04.306910+0000","last_fullsized":"2024-03-06T07:32:04.306910+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:34:22.621861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.31","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306474+0000","last_change":"2024-03-06T07:32:04.301038+0000","last_active":"2024-03-06T07:32:04.306474+0000","last_peered":"2024-03-06T07:32:04.306474+0000","last_clean":"2024-03-06T07:32:04.306474+0000","last_became_active":"2024-03-06T07:32:04.299831+0000","last_became_peered":"2024-03-06T07:32:04.299831+0000","last_unstale":"2024-03-06T07:32:04.306474+0000","last_undegraded":"2024-03-06T07:32:04.306474+0000","last_fullsized":"2024-03-06T07:32:04.306474+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:18:03.969159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.30","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308531+0000","last_change":"2024-03-06T07:32:04.303566+0000","last_active":"2024-03-06T07:32:04.308531+0000","last_peered":"2024-03-06T07:32:04.308531+0000","last_clean":"2024-03-06T07:32:04.308531+0000","last_became_active":"2024-03-06T07:32:04.303326+0000","last_became_peered":"2024-03-06T07:32:04.303326+0000","last_unstale":"2024-03-06T07:32:04.308531+0000","last_undegraded":"2024-03-06T07:32:04.308531+0000","last_fullsized":"2024-03-06T07:32:04.308531+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:29:09.128286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.2f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317983+0000","last_change":"2024-03-06T07:32:04.308364+0000","last_active":"2024-03-06T07:32:04.317983+0000","last_peered":"2024-03-06T07:32:04.317983+0000","last_clean":"2024-03-06T07:32:04.317983+0000","last_became_active":"2024-03-06T07:32:04.307768+0000","last_became_peered":"2024-03-06T07:32:04.307768+0000","last_unstale":"2024-03-06T07:32:04.317983+0000","last_undegraded":"2024-03-06T07:32:04.317983+0000","last_fullsized":"2024-03-06T07:32:04.317983+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:56:47.555774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.2e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311707+0000","last_change":"2024-03-06T07:32:04.304215+0000","last_active":"2024-03-06T07:32:04.311707+0000","last_peered":"2024-03-06T07:32:04.311707+0000","last_clean":"2024-03-06T07:32:04.311707+0000","last_became_active":"2024-03-06T07:32:04.303811+0000","last_became_peered":"2024-03-06T07:32:04.303811+0000","last_unstale":"2024-03-06T07:32:04.311707+0000","last_undegraded":"2024-03-06T07:32:04.311707+0000","last_fullsized":"2024-03-06T07:32:04.311707+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:37.296375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.2d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311548+0000","last_change":"2024-03-06T07:32:04.306389+0000","last_active":"2024-03-06T07:32:04.311548+0000","last_peered":"2024-03-06T07:32:04.311548+0000","last_clean":"2024-03-06T07:32:04.311548+0000","last_became_active":"2024-03-06T07:32:04.306129+0000","last_became_peered":"2024-03-06T07:32:04.306129+0000","last_unstale":"2024-03-06T07:32:04.311548+0000","last_undegraded":"2024-03-06T07:32:04.311548+0000","last_fullsized":"2024-03-06T07:32:04.311548+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:13:14.770211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315162+0000","last_change":"2024-03-06T07:32:04.308328+0000","last_active":"2024-03-06T07:32:04.315162+0000","last_peered":"2024-03-06T07:32:04.315162+0000","last_clean":"2024-03-06T07:32:04.315162+0000","last_became_active":"2024-03-06T07:32:04.307795+0000","last_became_peered":"2024-03-06T07:32:04.307795+0000","last_unstale":"2024-03-06T07:32:04.315162+0000","last_undegraded":"2024-03-06T07:32:04.315162+0000","last_fullsized":"2024-03-06T07:32:04.315162+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:59:11.714906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.2b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.307080+0000","last_change":"2024-03-06T07:32:04.303509+0000","last_active":"2024-03-06T07:32:04.307080+0000","last_peered":"2024-03-06T07:32:04.307080+0000","last_clean":"2024-03-06T07:32:04.307080+0000","last_became_active":"2024-03-06T07:32:04.303246+0000","last_became_peered":"2024-03-06T07:32:04.303246+0000","last_unstale":"2024-03-06T07:32:04.307080+0000","last_undegraded":"2024-03-06T07:32:04.307080+0000","last_fullsized":"2024-03-06T07:32:04.307080+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:43:11.857855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315151+0000","last_change":"2024-03-06T07:32:04.305219+0000","last_active":"2024-03-06T07:32:04.315151+0000","last_peered":"2024-03-06T07:32:04.315151+0000","last_clean":"2024-03-06T07:32:04.315151+0000","last_became_active":"2024-03-06T07:32:04.304957+0000","last_became_peered":"2024-03-06T07:32:04.304957+0000","last_unstale":"2024-03-06T07:32:04.315151+0000","last_undegraded":"2024-03-06T07:32:04.315151+0000","last_fullsized":"2024-03-06T07:32:04.315151+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:22:05.288497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.29","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317761+0000","last_change":"2024-03-06T07:32:04.308102+0000","last_active":"2024-03-06T07:32:04.317761+0000","last_peered":"2024-03-06T07:32:04.317761+0000","last_clean":"2024-03-06T07:32:04.317761+0000","last_became_active":"2024-03-06T07:32:04.307756+0000","last_became_peered":"2024-03-06T07:32:04.307756+0000","last_unstale":"2024-03-06T07:32:04.317761+0000","last_undegraded":"2024-03-06T07:32:04.317761+0000","last_fullsized":"2024-03-06T07:32:04.317761+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:53:12.334707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.28","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308504+0000","last_change":"2024-03-06T07:32:04.303021+0000","last_active":"2024-03-06T07:32:04.308504+0000","last_peered":"2024-03-06T07:32:04.308504+0000","last_clean":"2024-03-06T07:32:04.308504+0000","last_became_active":"2024-03-06T07:32:04.302729+0000","last_became_peered":"2024-03-06T07:32:04.302729+0000","last_unstale":"2024-03-06T07:32:04.308504+0000","last_undegraded":"2024-03-06T07:32:04.308504+0000","last_fullsized":"2024-03-06T07:32:04.308504+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:59:11.714906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.27","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316031+0000","last_change":"2024-03-06T07:32:04.312714+0000","last_active":"2024-03-06T07:32:04.316031+0000","last_peered":"2024-03-06T07:32:04.316031+0000","last_clean":"2024-03-06T07:32:04.316031+0000","last_became_active":"2024-03-06T07:32:04.312421+0000","last_became_peered":"2024-03-06T07:32:04.312421+0000","last_unstale":"2024-03-06T07:32:04.316031+0000","last_undegraded":"2024-03-06T07:32:04.316031+0000","last_fullsized":"2024-03-06T07:32:04.316031+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:00:49.753237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.26","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318243+0000","last_change":"2024-03-06T07:32:04.313426+0000","last_active":"2024-03-06T07:32:04.318243+0000","last_peered":"2024-03-06T07:32:04.318243+0000","last_clean":"2024-03-06T07:32:04.318243+0000","last_became_active":"2024-03-06T07:32:04.312661+0000","last_became_peered":"2024-03-06T07:32:04.312661+0000","last_unstale":"2024-03-06T07:32:04.318243+0000","last_undegraded":"2024-03-06T07:32:04.318243+0000","last_fullsized":"2024-03-06T07:32:04.318243+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:25:09.502176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.25","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.302023+0000","last_change":"2024-03-06T07:32:04.297909+0000","last_active":"2024-03-06T07:32:04.302023+0000","last_peered":"2024-03-06T07:32:04.302023+0000","last_clean":"2024-03-06T07:32:04.302023+0000","last_became_active":"2024-03-06T07:32:04.297610+0000","last_became_peered":"2024-03-06T07:32:04.297610+0000","last_unstale":"2024-03-06T07:32:04.302023+0000","last_undegraded":"2024-03-06T07:32:04.302023+0000","last_fullsized":"2024-03-06T07:32:04.302023+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:39:56.428272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.24","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317267+0000","last_change":"2024-03-06T07:32:04.311830+0000","last_active":"2024-03-06T07:32:04.317267+0000","last_peered":"2024-03-06T07:32:04.317267+0000","last_clean":"2024-03-06T07:32:04.317267+0000","last_became_active":"2024-03-06T07:32:04.310826+0000","last_became_peered":"2024-03-06T07:32:04.310826+0000","last_unstale":"2024-03-06T07:32:04.317267+0000","last_undegraded":"2024-03-06T07:32:04.317267+0000","last_fullsized":"2024-03-06T07:32:04.317267+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:09:47.827094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.23","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317959+0000","last_change":"2024-03-06T07:32:04.312560+0000","last_active":"2024-03-06T07:32:04.317959+0000","last_peered":"2024-03-06T07:32:04.317959+0000","last_clean":"2024-03-06T07:32:04.317959+0000","last_became_active":"2024-03-06T07:32:04.312218+0000","last_became_peered":"2024-03-06T07:32:04.312218+0000","last_unstale":"2024-03-06T07:32:04.317959+0000","last_undegraded":"2024-03-06T07:32:04.317959+0000","last_fullsized":"2024-03-06T07:32:04.317959+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:47:40.026034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.22","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311575+0000","last_change":"2024-03-06T07:32:04.308606+0000","last_active":"2024-03-06T07:32:04.311575+0000","last_peered":"2024-03-06T07:32:04.311575+0000","last_clean":"2024-03-06T07:32:04.311575+0000","last_became_active":"2024-03-06T07:32:04.308278+0000","last_became_peered":"2024-03-06T07:32:04.308278+0000","last_unstale":"2024-03-06T07:32:04.311575+0000","last_undegraded":"2024-03-06T07:32:04.311575+0000","last_fullsized":"2024-03-06T07:32:04.311575+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:54:27.865752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.21","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312370+0000","last_change":"2024-03-06T07:32:04.306610+0000","last_active":"2024-03-06T07:32:04.312370+0000","last_peered":"2024-03-06T07:32:04.312370+0000","last_clean":"2024-03-06T07:32:04.312370+0000","last_became_active":"2024-03-06T07:32:04.305576+0000","last_became_peered":"2024-03-06T07:32:04.305576+0000","last_unstale":"2024-03-06T07:32:04.312370+0000","last_undegraded":"2024-03-06T07:32:04.312370+0000","last_fullsized":"2024-03-06T07:32:04.312370+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:56:47.555774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.20","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.324677+0000","last_change":"2024-03-06T07:32:04.316454+0000","last_active":"2024-03-06T07:32:04.324677+0000","last_peered":"2024-03-06T07:32:04.324677+0000","last_clean":"2024-03-06T07:32:04.324677+0000","last_became_active":"2024-03-06T07:32:04.316121+0000","last_became_peered":"2024-03-06T07:32:04.316121+0000","last_unstale":"2024-03-06T07:32:04.324677+0000","last_undegraded":"2024-03-06T07:32:04.324677+0000","last_fullsized":"2024-03-06T07:32:04.324677+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:16:13.907316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306996+0000","last_change":"2024-03-06T07:32:04.302971+0000","last_active":"2024-03-06T07:32:04.306996+0000","last_peered":"2024-03-06T07:32:04.306996+0000","last_clean":"2024-03-06T07:32:04.306996+0000","last_became_active":"2024-03-06T07:32:04.302656+0000","last_became_peered":"2024-03-06T07:32:04.302656+0000","last_unstale":"2024-03-06T07:32:04.306996+0000","last_undegraded":"2024-03-06T07:32:04.306996+0000","last_fullsized":"2024-03-06T07:32:04.306996+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:06:43.857742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1f","version":"93'4","reported_seq":16,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.328261+0000","last_change":"2024-03-06T07:32:03.305143+0000","last_active":"2024-03-06T07:32:04.328261+0000","last_peered":"2024-03-06T07:32:04.328261+0000","last_clean":"2024-03-06T07:32:04.328261+0000","last_became_active":"2024-03-06T07:32:03.304833+0000","last_became_peered":"2024-03-06T07:32:03.304833+0000","last_unstale":"2024-03-06T07:32:04.328261+0000","last_undegraded":"2024-03-06T07:32:04.328261+0000","last_fullsized":"2024-03-06T07:32:04.328261+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:18:02.952948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.306790+0000","last_change":"2024-03-06T07:32:04.302685+0000","last_active":"2024-03-06T07:32:04.306790+0000","last_peered":"2024-03-06T07:32:04.306790+0000","last_clean":"2024-03-06T07:32:04.306790+0000","last_became_active":"2024-03-06T07:32:04.302377+0000","last_became_peered":"2024-03-06T07:32:04.302377+0000","last_unstale":"2024-03-06T07:32:04.306790+0000","last_undegraded":"2024-03-06T07:32:04.306790+0000","last_fullsized":"2024-03-06T07:32:04.306790+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:49:26.188782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1e","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.325163+0000","last_change":"2024-03-06T07:32:03.316590+0000","last_active":"2024-03-06T07:32:04.325163+0000","last_peered":"2024-03-06T07:32:04.325163+0000","last_clean":"2024-03-06T07:32:04.325163+0000","last_became_active":"2024-03-06T07:32:03.316232+0000","last_became_peered":"2024-03-06T07:32:03.316232+0000","last_unstale":"2024-03-06T07:32:04.325163+0000","last_undegraded":"2024-03-06T07:32:04.325163+0000","last_fullsized":"2024-03-06T07:32:04.325163+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:14.821460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314170+0000","last_change":"2024-03-06T07:32:04.310944+0000","last_active":"2024-03-06T07:32:04.314170+0000","last_peered":"2024-03-06T07:32:04.314170+0000","last_clean":"2024-03-06T07:32:04.314170+0000","last_became_active":"2024-03-06T07:32:04.310627+0000","last_became_peered":"2024-03-06T07:32:04.310627+0000","last_unstale":"2024-03-06T07:32:04.314170+0000","last_undegraded":"2024-03-06T07:32:04.314170+0000","last_fullsized":"2024-03-06T07:32:04.314170+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:26:35.984137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.1d","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.327979+0000","last_change":"2024-03-06T07:32:03.311842+0000","last_active":"2024-03-06T07:32:04.327979+0000","last_peered":"2024-03-06T07:32:04.327979+0000","last_clean":"2024-03-06T07:32:04.327979+0000","last_became_active":"2024-03-06T07:32:03.311519+0000","last_became_peered":"2024-03-06T07:32:03.311519+0000","last_unstale":"2024-03-06T07:32:04.327979+0000","last_undegraded":"2024-03-06T07:32:04.327979+0000","last_fullsized":"2024-03-06T07:32:04.327979+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:39:36.280164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318515+0000","last_change":"2024-03-06T07:32:04.311491+0000","last_active":"2024-03-06T07:32:04.318515+0000","last_peered":"2024-03-06T07:32:04.318515+0000","last_clean":"2024-03-06T07:32:04.318515+0000","last_became_active":"2024-03-06T07:32:04.311173+0000","last_became_peered":"2024-03-06T07:32:04.311173+0000","last_unstale":"2024-03-06T07:32:04.318515+0000","last_undegraded":"2024-03-06T07:32:04.318515+0000","last_fullsized":"2024-03-06T07:32:04.318515+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:29:09.128286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1c","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.327967+0000","last_change":"2024-03-06T07:32:03.314678+0000","last_active":"2024-03-06T07:32:04.327967+0000","last_peered":"2024-03-06T07:32:04.327967+0000","last_clean":"2024-03-06T07:32:04.327967+0000","last_became_active":"2024-03-06T07:32:03.314429+0000","last_became_peered":"2024-03-06T07:32:03.314429+0000","last_unstale":"2024-03-06T07:32:04.327967+0000","last_undegraded":"2024-03-06T07:32:04.327967+0000","last_fullsized":"2024-03-06T07:32:04.327967+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:39:56.981785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311056+0000","last_change":"2024-03-06T07:32:04.302114+0000","last_active":"2024-03-06T07:32:04.311056+0000","last_peered":"2024-03-06T07:32:04.311056+0000","last_clean":"2024-03-06T07:32:04.311056+0000","last_became_active":"2024-03-06T07:32:04.301573+0000","last_became_peered":"2024-03-06T07:32:04.301573+0000","last_unstale":"2024-03-06T07:32:04.311056+0000","last_undegraded":"2024-03-06T07:32:04.311056+0000","last_fullsized":"2024-03-06T07:32:04.311056+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:53:54.108260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1b","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.325542+0000","last_change":"2024-03-06T07:32:03.310518+0000","last_active":"2024-03-06T07:32:04.325542+0000","last_peered":"2024-03-06T07:32:04.325542+0000","last_clean":"2024-03-06T07:32:04.325542+0000","last_became_active":"2024-03-06T07:32:03.309440+0000","last_became_peered":"2024-03-06T07:32:03.309440+0000","last_unstale":"2024-03-06T07:32:04.325542+0000","last_undegraded":"2024-03-06T07:32:04.325542+0000","last_fullsized":"2024-03-06T07:32:04.325542+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:04:41.344138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.300614+0000","last_change":"2024-03-06T07:32:04.294490+0000","last_active":"2024-03-06T07:32:04.300614+0000","last_peered":"2024-03-06T07:32:04.300614+0000","last_clean":"2024-03-06T07:32:04.300614+0000","last_became_active":"2024-03-06T07:32:04.294138+0000","last_became_peered":"2024-03-06T07:32:04.294138+0000","last_unstale":"2024-03-06T07:32:04.300614+0000","last_undegraded":"2024-03-06T07:32:04.300614+0000","last_fullsized":"2024-03-06T07:32:04.300614+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:25:16.547115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1a","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.327986+0000","last_change":"2024-03-06T07:32:03.306891+0000","last_active":"2024-03-06T07:32:04.327986+0000","last_peered":"2024-03-06T07:32:04.327986+0000","last_clean":"2024-03-06T07:32:04.327986+0000","last_became_active":"2024-03-06T07:32:03.306619+0000","last_became_peered":"2024-03-06T07:32:03.306619+0000","last_unstale":"2024-03-06T07:32:04.327986+0000","last_undegraded":"2024-03-06T07:32:04.327986+0000","last_fullsized":"2024-03-06T07:32:04.327986+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:09:28.506301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.18","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.312100+0000","last_change":"2024-03-06T07:32:04.301926+0000","last_active":"2024-03-06T07:32:04.312100+0000","last_peered":"2024-03-06T07:32:04.312100+0000","last_clean":"2024-03-06T07:32:04.312100+0000","last_became_active":"2024-03-06T07:32:04.301648+0000","last_became_peered":"2024-03-06T07:32:04.301648+0000","last_unstale":"2024-03-06T07:32:04.312100+0000","last_undegraded":"2024-03-06T07:32:04.312100+0000","last_fullsized":"2024-03-06T07:32:04.312100+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:09:37.864764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.19","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.298443+0000","last_change":"2024-03-06T07:32:03.306724+0000","last_active":"2024-03-06T07:32:04.298443+0000","last_peered":"2024-03-06T07:32:04.298443+0000","last_clean":"2024-03-06T07:32:04.298443+0000","last_became_active":"2024-03-06T07:32:03.306373+0000","last_became_peered":"2024-03-06T07:32:03.306373+0000","last_unstale":"2024-03-06T07:32:04.298443+0000","last_undegraded":"2024-03-06T07:32:04.298443+0000","last_fullsized":"2024-03-06T07:32:04.298443+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:59:38.000494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.19","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318914+0000","last_change":"2024-03-06T07:32:04.312286+0000","last_active":"2024-03-06T07:32:04.318914+0000","last_peered":"2024-03-06T07:32:04.318914+0000","last_clean":"2024-03-06T07:32:04.318914+0000","last_became_active":"2024-03-06T07:32:04.312028+0000","last_became_peered":"2024-03-06T07:32:04.312028+0000","last_unstale":"2024-03-06T07:32:04.318914+0000","last_undegraded":"2024-03-06T07:32:04.318914+0000","last_fullsized":"2024-03-06T07:32:04.318914+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:47:46.548145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.18","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.294816+0000","last_change":"2024-03-06T07:32:03.301269+0000","last_active":"2024-03-06T07:32:04.294816+0000","last_peered":"2024-03-06T07:32:04.294816+0000","last_clean":"2024-03-06T07:32:04.294816+0000","last_became_active":"2024-03-06T07:32:03.300997+0000","last_became_peered":"2024-03-06T07:32:03.300997+0000","last_unstale":"2024-03-06T07:32:04.294816+0000","last_undegraded":"2024-03-06T07:32:04.294816+0000","last_fullsized":"2024-03-06T07:32:04.294816+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:15:36.714406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.16","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.317909+0000","last_change":"2024-03-06T07:32:04.310610+0000","last_active":"2024-03-06T07:32:04.317909+0000","last_peered":"2024-03-06T07:32:04.317909+0000","last_clean":"2024-03-06T07:32:04.317909+0000","last_became_active":"2024-03-06T07:32:04.306596+0000","last_became_peered":"2024-03-06T07:32:04.306596+0000","last_unstale":"2024-03-06T07:32:04.317909+0000","last_undegraded":"2024-03-06T07:32:04.317909+0000","last_fullsized":"2024-03-06T07:32:04.317909+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:21:50.619908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.17","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.289307+0000","last_change":"2024-03-06T07:32:03.307878+0000","last_active":"2024-03-06T07:32:04.289307+0000","last_peered":"2024-03-06T07:32:04.289307+0000","last_clean":"2024-03-06T07:32:04.289307+0000","last_became_active":"2024-03-06T07:32:03.307614+0000","last_became_peered":"2024-03-06T07:32:03.307614+0000","last_unstale":"2024-03-06T07:32:04.289307+0000","last_undegraded":"2024-03-06T07:32:04.289307+0000","last_fullsized":"2024-03-06T07:32:04.289307+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:53:35.571679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.17","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.319114+0000","last_change":"2024-03-06T07:32:04.312709+0000","last_active":"2024-03-06T07:32:04.319114+0000","last_peered":"2024-03-06T07:32:04.319114+0000","last_clean":"2024-03-06T07:32:04.319114+0000","last_became_active":"2024-03-06T07:32:04.312441+0000","last_became_peered":"2024-03-06T07:32:04.312441+0000","last_unstale":"2024-03-06T07:32:04.319114+0000","last_undegraded":"2024-03-06T07:32:04.319114+0000","last_fullsized":"2024-03-06T07:32:04.319114+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:02:41.927660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.276176+0000","last_change":"2024-03-06T07:32:03.306977+0000","last_active":"2024-03-06T07:32:04.276176+0000","last_peered":"2024-03-06T07:32:04.276176+0000","last_clean":"2024-03-06T07:32:04.276176+0000","last_became_active":"2024-03-06T07:32:03.306623+0000","last_became_peered":"2024-03-06T07:32:03.306623+0000","last_unstale":"2024-03-06T07:32:04.276176+0000","last_undegraded":"2024-03-06T07:32:04.276176+0000","last_fullsized":"2024-03-06T07:32:04.276176+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:19.853592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.14","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311797+0000","last_change":"2024-03-06T07:32:04.306817+0000","last_active":"2024-03-06T07:32:04.311797+0000","last_peered":"2024-03-06T07:32:04.311797+0000","last_clean":"2024-03-06T07:32:04.311797+0000","last_became_active":"2024-03-06T07:32:04.306536+0000","last_became_peered":"2024-03-06T07:32:04.306536+0000","last_unstale":"2024-03-06T07:32:04.311797+0000","last_undegraded":"2024-03-06T07:32:04.311797+0000","last_fullsized":"2024-03-06T07:32:04.311797+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:00:27.549959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.15","version":"93'3","reported_seq":15,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.324956+0000","last_change":"2024-03-06T07:32:03.316615+0000","last_active":"2024-03-06T07:32:04.324956+0000","last_peered":"2024-03-06T07:32:04.324956+0000","last_clean":"2024-03-06T07:32:04.324956+0000","last_became_active":"2024-03-06T07:32:03.316282+0000","last_became_peered":"2024-03-06T07:32:03.316282+0000","last_unstale":"2024-03-06T07:32:04.324956+0000","last_undegraded":"2024-03-06T07:32:04.324956+0000","last_fullsized":"2024-03-06T07:32:04.324956+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:48:35.839016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.15","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308715+0000","last_change":"2024-03-06T07:32:04.304462+0000","last_active":"2024-03-06T07:32:04.308715+0000","last_peered":"2024-03-06T07:32:04.308715+0000","last_clean":"2024-03-06T07:32:04.308715+0000","last_became_active":"2024-03-06T07:32:04.304220+0000","last_became_peered":"2024-03-06T07:32:04.304220+0000","last_unstale":"2024-03-06T07:32:04.308715+0000","last_undegraded":"2024-03-06T07:32:04.308715+0000","last_fullsized":"2024-03-06T07:32:04.308715+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:47:46.548145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.14","version":"93'6","reported_seq":18,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.331045+0000","last_change":"2024-03-06T07:32:03.309787+0000","last_active":"2024-03-06T07:32:04.331045+0000","last_peered":"2024-03-06T07:32:04.331045+0000","last_clean":"2024-03-06T07:32:04.331045+0000","last_became_active":"2024-03-06T07:32:03.309406+0000","last_became_peered":"2024-03-06T07:32:03.309406+0000","last_unstale":"2024-03-06T07:32:04.331045+0000","last_undegraded":"2024-03-06T07:32:04.331045+0000","last_fullsized":"2024-03-06T07:32:04.331045+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:48:35.839016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.12","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.290045+0000","last_change":"2024-03-06T07:32:04.285596+0000","last_active":"2024-03-06T07:32:04.290045+0000","last_peered":"2024-03-06T07:32:04.290045+0000","last_clean":"2024-03-06T07:32:04.290045+0000","last_became_active":"2024-03-06T07:32:04.284787+0000","last_became_peered":"2024-03-06T07:32:04.284787+0000","last_unstale":"2024-03-06T07:32:04.290045+0000","last_undegraded":"2024-03-06T07:32:04.290045+0000","last_fullsized":"2024-03-06T07:32:04.290045+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:42:28.718856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.325659+0000","last_change":"2024-03-06T07:32:03.308806+0000","last_active":"2024-03-06T07:32:04.325659+0000","last_peered":"2024-03-06T07:32:04.325659+0000","last_clean":"2024-03-06T07:32:04.325659+0000","last_became_active":"2024-03-06T07:32:03.308522+0000","last_became_peered":"2024-03-06T07:32:03.308522+0000","last_unstale":"2024-03-06T07:32:04.325659+0000","last_undegraded":"2024-03-06T07:32:04.325659+0000","last_fullsized":"2024-03-06T07:32:04.325659+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:17:06.059681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.13","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311423+0000","last_change":"2024-03-06T07:32:04.307370+0000","last_active":"2024-03-06T07:32:04.311423+0000","last_peered":"2024-03-06T07:32:04.311423+0000","last_clean":"2024-03-06T07:32:04.311423+0000","last_became_active":"2024-03-06T07:32:04.306926+0000","last_became_peered":"2024-03-06T07:32:04.306926+0000","last_unstale":"2024-03-06T07:32:04.311423+0000","last_undegraded":"2024-03-06T07:32:04.311423+0000","last_fullsized":"2024-03-06T07:32:04.311423+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:22:55.730934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.12","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296128+0000","last_change":"2024-03-06T07:32:03.304837+0000","last_active":"2024-03-06T07:32:04.296128+0000","last_peered":"2024-03-06T07:32:04.296128+0000","last_clean":"2024-03-06T07:32:04.296128+0000","last_became_active":"2024-03-06T07:32:03.302826+0000","last_became_peered":"2024-03-06T07:32:03.302826+0000","last_unstale":"2024-03-06T07:32:04.296128+0000","last_undegraded":"2024-03-06T07:32:04.296128+0000","last_fullsized":"2024-03-06T07:32:04.296128+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:05:44.455324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.10","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314705+0000","last_change":"2024-03-06T07:32:04.306320+0000","last_active":"2024-03-06T07:32:04.314705+0000","last_peered":"2024-03-06T07:32:04.314705+0000","last_clean":"2024-03-06T07:32:04.314705+0000","last_became_active":"2024-03-06T07:32:04.306031+0000","last_became_peered":"2024-03-06T07:32:04.306031+0000","last_unstale":"2024-03-06T07:32:04.314705+0000","last_undegraded":"2024-03-06T07:32:04.314705+0000","last_fullsized":"2024-03-06T07:32:04.314705+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:39:56.428272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.11","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.298857+0000","last_change":"2024-03-06T07:32:03.312323+0000","last_active":"2024-03-06T07:32:04.298857+0000","last_peered":"2024-03-06T07:32:04.298857+0000","last_clean":"2024-03-06T07:32:04.298857+0000","last_became_active":"2024-03-06T07:32:03.312017+0000","last_became_peered":"2024-03-06T07:32:03.312017+0000","last_unstale":"2024-03-06T07:32:04.298857+0000","last_undegraded":"2024-03-06T07:32:04.298857+0000","last_fullsized":"2024-03-06T07:32:04.298857+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:58.935413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.11","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314523+0000","last_change":"2024-03-06T07:32:04.309159+0000","last_active":"2024-03-06T07:32:04.314523+0000","last_peered":"2024-03-06T07:32:04.314523+0000","last_clean":"2024-03-06T07:32:04.314523+0000","last_became_active":"2024-03-06T07:32:04.308874+0000","last_became_peered":"2024-03-06T07:32:04.308874+0000","last_unstale":"2024-03-06T07:32:04.314523+0000","last_undegraded":"2024-03-06T07:32:04.314523+0000","last_fullsized":"2024-03-06T07:32:04.314523+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:03.970098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.10","version":"93'8","reported_seq":20,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.328408+0000","last_change":"2024-03-06T07:32:03.313608+0000","last_active":"2024-03-06T07:32:04.328408+0000","last_peered":"2024-03-06T07:32:04.328408+0000","last_clean":"2024-03-06T07:32:04.328408+0000","last_became_active":"2024-03-06T07:32:03.313143+0000","last_became_peered":"2024-03-06T07:32:03.313143+0000","last_unstale":"2024-03-06T07:32:04.328408+0000","last_undegraded":"2024-03-06T07:32:04.328408+0000","last_fullsized":"2024-03-06T07:32:04.328408+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:49:12.503842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.f","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316009+0000","last_change":"2024-03-06T07:32:04.310548+0000","last_active":"2024-03-06T07:32:04.316009+0000","last_peered":"2024-03-06T07:32:04.316009+0000","last_clean":"2024-03-06T07:32:04.316009+0000","last_became_active":"2024-03-06T07:32:04.309925+0000","last_became_peered":"2024-03-06T07:32:04.309925+0000","last_unstale":"2024-03-06T07:32:04.316009+0000","last_undegraded":"2024-03-06T07:32:04.316009+0000","last_fullsized":"2024-03-06T07:32:04.316009+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:58:44.739330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.326492+0000","last_change":"2024-03-06T07:32:03.312422+0000","last_active":"2024-03-06T07:32:04.326492+0000","last_peered":"2024-03-06T07:32:04.326492+0000","last_clean":"2024-03-06T07:32:04.326492+0000","last_became_active":"2024-03-06T07:32:03.312114+0000","last_became_peered":"2024-03-06T07:32:03.312114+0000","last_unstale":"2024-03-06T07:32:04.326492+0000","last_undegraded":"2024-03-06T07:32:04.326492+0000","last_fullsized":"2024-03-06T07:32:04.326492+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:15:14.821460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.c","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311206+0000","last_change":"2024-03-06T07:32:04.302878+0000","last_active":"2024-03-06T07:32:04.311206+0000","last_peered":"2024-03-06T07:32:04.311206+0000","last_clean":"2024-03-06T07:32:04.311206+0000","last_became_active":"2024-03-06T07:32:04.302034+0000","last_became_peered":"2024-03-06T07:32:04.302034+0000","last_unstale":"2024-03-06T07:32:04.311206+0000","last_undegraded":"2024-03-06T07:32:04.311206+0000","last_fullsized":"2024-03-06T07:32:04.311206+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:27.142777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.d","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.322183+0000","last_change":"2024-03-06T07:32:03.302127+0000","last_active":"2024-03-06T07:32:04.322183+0000","last_peered":"2024-03-06T07:32:04.322183+0000","last_clean":"2024-03-06T07:32:04.322183+0000","last_became_active":"2024-03-06T07:32:03.301282+0000","last_became_peered":"2024-03-06T07:32:03.301282+0000","last_unstale":"2024-03-06T07:32:04.322183+0000","last_undegraded":"2024-03-06T07:32:04.322183+0000","last_fullsized":"2024-03-06T07:32:04.322183+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:59:29.195494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.d","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308694+0000","last_change":"2024-03-06T07:32:04.300889+0000","last_active":"2024-03-06T07:32:04.308694+0000","last_peered":"2024-03-06T07:32:04.308694+0000","last_clean":"2024-03-06T07:32:04.308694+0000","last_became_active":"2024-03-06T07:32:04.299489+0000","last_became_peered":"2024-03-06T07:32:04.299489+0000","last_unstale":"2024-03-06T07:32:04.308694+0000","last_undegraded":"2024-03-06T07:32:04.308694+0000","last_fullsized":"2024-03-06T07:32:04.308694+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:46.541754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.288447+0000","last_change":"2024-03-06T07:32:03.314945+0000","last_active":"2024-03-06T07:32:04.288447+0000","last_peered":"2024-03-06T07:32:04.288447+0000","last_clean":"2024-03-06T07:32:04.288447+0000","last_became_active":"2024-03-06T07:32:03.314467+0000","last_became_peered":"2024-03-06T07:32:03.314467+0000","last_unstale":"2024-03-06T07:32:04.288447+0000","last_undegraded":"2024-03-06T07:32:04.288447+0000","last_fullsized":"2024-03-06T07:32:04.288447+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:54:29.350079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.a","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311525+0000","last_change":"2024-03-06T07:32:04.306751+0000","last_active":"2024-03-06T07:32:04.311525+0000","last_peered":"2024-03-06T07:32:04.311525+0000","last_clean":"2024-03-06T07:32:04.311525+0000","last_became_active":"2024-03-06T07:32:04.306490+0000","last_became_peered":"2024-03-06T07:32:04.306490+0000","last_unstale":"2024-03-06T07:32:04.311525+0000","last_undegraded":"2024-03-06T07:32:04.311525+0000","last_fullsized":"2024-03-06T07:32:04.311525+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:04:26.208566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.328370+0000","last_change":"2024-03-06T07:32:03.304943+0000","last_active":"2024-03-06T07:32:04.328370+0000","last_peered":"2024-03-06T07:32:04.328370+0000","last_clean":"2024-03-06T07:32:04.328370+0000","last_became_active":"2024-03-06T07:32:03.304483+0000","last_became_peered":"2024-03-06T07:32:03.304483+0000","last_unstale":"2024-03-06T07:32:04.328370+0000","last_undegraded":"2024-03-06T07:32:04.328370+0000","last_fullsized":"2024-03-06T07:32:04.328370+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:19.853592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.b","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.322954+0000","last_change":"2024-03-06T07:32:04.314919+0000","last_active":"2024-03-06T07:32:04.322954+0000","last_peered":"2024-03-06T07:32:04.322954+0000","last_clean":"2024-03-06T07:32:04.322954+0000","last_became_active":"2024-03-06T07:32:04.313624+0000","last_became_peered":"2024-03-06T07:32:04.313624+0000","last_unstale":"2024-03-06T07:32:04.322954+0000","last_undegraded":"2024-03-06T07:32:04.322954+0000","last_fullsized":"2024-03-06T07:32:04.322954+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:34:49.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.290470+0000","last_change":"2024-03-06T07:32:03.313220+0000","last_active":"2024-03-06T07:32:04.290470+0000","last_peered":"2024-03-06T07:32:04.290470+0000","last_clean":"2024-03-06T07:32:04.290470+0000","last_became_active":"2024-03-06T07:32:03.311795+0000","last_became_peered":"2024-03-06T07:32:03.311795+0000","last_unstale":"2024-03-06T07:32:04.290470+0000","last_undegraded":"2024-03-06T07:32:04.290470+0000","last_fullsized":"2024-03-06T07:32:04.290470+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:37:25.322534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.8","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.314901+0000","last_change":"2024-03-06T07:32:04.306137+0000","last_active":"2024-03-06T07:32:04.314901+0000","last_peered":"2024-03-06T07:32:04.314901+0000","last_clean":"2024-03-06T07:32:04.314901+0000","last_became_active":"2024-03-06T07:32:04.305821+0000","last_became_peered":"2024-03-06T07:32:04.305821+0000","last_unstale":"2024-03-06T07:32:04.314901+0000","last_undegraded":"2024-03-06T07:32:04.314901+0000","last_fullsized":"2024-03-06T07:32:04.314901+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:07:16.464259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.288681+0000","last_change":"2024-03-06T07:32:03.301502+0000","last_active":"2024-03-06T07:32:04.288681+0000","last_peered":"2024-03-06T07:32:04.288681+0000","last_clean":"2024-03-06T07:32:04.288681+0000","last_became_active":"2024-03-06T07:32:03.301158+0000","last_became_peered":"2024-03-06T07:32:03.301158+0000","last_unstale":"2024-03-06T07:32:04.288681+0000","last_undegraded":"2024-03-06T07:32:04.288681+0000","last_fullsized":"2024-03-06T07:32:04.288681+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:48:29.845871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.9","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.319454+0000","last_change":"2024-03-06T07:32:04.313896+0000","last_active":"2024-03-06T07:32:04.319454+0000","last_peered":"2024-03-06T07:32:04.319454+0000","last_clean":"2024-03-06T07:32:04.319454+0000","last_became_active":"2024-03-06T07:32:04.313208+0000","last_became_peered":"2024-03-06T07:32:04.313208+0000","last_unstale":"2024-03-06T07:32:04.319454+0000","last_undegraded":"2024-03-06T07:32:04.319454+0000","last_fullsized":"2024-03-06T07:32:04.319454+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:28:56.208174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296146+0000","last_change":"2024-03-06T07:32:03.306802+0000","last_active":"2024-03-06T07:32:04.296146+0000","last_peered":"2024-03-06T07:32:04.296146+0000","last_clean":"2024-03-06T07:32:04.296146+0000","last_became_active":"2024-03-06T07:32:03.306462+0000","last_became_peered":"2024-03-06T07:32:03.306462+0000","last_unstale":"2024-03-06T07:32:04.296146+0000","last_undegraded":"2024-03-06T07:32:04.296146+0000","last_fullsized":"2024-03-06T07:32:04.296146+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:42:27.702645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.6","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.318084+0000","last_change":"2024-03-06T07:32:04.312917+0000","last_active":"2024-03-06T07:32:04.318084+0000","last_peered":"2024-03-06T07:32:04.318084+0000","last_clean":"2024-03-06T07:32:04.318084+0000","last_became_active":"2024-03-06T07:32:04.312110+0000","last_became_peered":"2024-03-06T07:32:04.312110+0000","last_unstale":"2024-03-06T07:32:04.318084+0000","last_undegraded":"2024-03-06T07:32:04.318084+0000","last_fullsized":"2024-03-06T07:32:04.318084+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:08:51.374255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.7","version":"93'2","reported_seq":14,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.324963+0000","last_change":"2024-03-06T07:32:03.309874+0000","last_active":"2024-03-06T07:32:04.324963+0000","last_peered":"2024-03-06T07:32:04.324963+0000","last_clean":"2024-03-06T07:32:04.324963+0000","last_became_active":"2024-03-06T07:32:03.309270+0000","last_became_peered":"2024-03-06T07:32:03.309270+0000","last_unstale":"2024-03-06T07:32:04.324963+0000","last_undegraded":"2024-03-06T07:32:04.324963+0000","last_fullsized":"2024-03-06T07:32:04.324963+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:54:14.722610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.7","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.308885+0000","last_change":"2024-03-06T07:32:04.305953+0000","last_active":"2024-03-06T07:32:04.308885+0000","last_peered":"2024-03-06T07:32:04.308885+0000","last_clean":"2024-03-06T07:32:04.308885+0000","last_became_active":"2024-03-06T07:32:04.305660+0000","last_became_peered":"2024-03-06T07:32:04.305660+0000","last_unstale":"2024-03-06T07:32:04.308885+0000","last_undegraded":"2024-03-06T07:32:04.308885+0000","last_fullsized":"2024-03-06T07:32:04.308885+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:00:41.545875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.301695+0000","last_change":"2024-03-06T07:32:03.303397+0000","last_active":"2024-03-06T07:32:04.301695+0000","last_peered":"2024-03-06T07:32:04.301695+0000","last_clean":"2024-03-06T07:32:04.301695+0000","last_became_active":"2024-03-06T07:32:03.303117+0000","last_became_peered":"2024-03-06T07:32:03.303117+0000","last_unstale":"2024-03-06T07:32:04.301695+0000","last_undegraded":"2024-03-06T07:32:04.301695+0000","last_fullsized":"2024-03-06T07:32:04.301695+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:09:36.848553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.4","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.309930+0000","last_change":"2024-03-06T07:32:04.303078+0000","last_active":"2024-03-06T07:32:04.309930+0000","last_peered":"2024-03-06T07:32:04.309930+0000","last_clean":"2024-03-06T07:32:04.309930+0000","last_became_active":"2024-03-06T07:32:04.302771+0000","last_became_peered":"2024-03-06T07:32:04.302771+0000","last_unstale":"2024-03-06T07:32:04.309930+0000","last_undegraded":"2024-03-06T07:32:04.309930+0000","last_fullsized":"2024-03-06T07:32:04.309930+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:24:16.622950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296635+0000","last_change":"2024-03-06T07:32:03.310369+0000","last_active":"2024-03-06T07:32:04.296635+0000","last_peered":"2024-03-06T07:32:04.296635+0000","last_clean":"2024-03-06T07:32:04.296635+0000","last_became_active":"2024-03-06T07:32:03.309892+0000","last_became_peered":"2024-03-06T07:32:03.309892+0000","last_unstale":"2024-03-06T07:32:04.296635+0000","last_undegraded":"2024-03-06T07:32:04.296635+0000","last_fullsized":"2024-03-06T07:32:04.296635+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:49:25.172571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.5","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.316257+0000","last_change":"2024-03-06T07:32:04.311455+0000","last_active":"2024-03-06T07:32:04.316257+0000","last_peered":"2024-03-06T07:32:04.316257+0000","last_clean":"2024-03-06T07:32:04.316257+0000","last_became_active":"2024-03-06T07:32:04.311106+0000","last_became_peered":"2024-03-06T07:32:04.311106+0000","last_unstale":"2024-03-06T07:32:04.316257+0000","last_undegraded":"2024-03-06T07:32:04.316257+0000","last_fullsized":"2024-03-06T07:32:04.316257+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:51:25.630627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.296510+0000","last_change":"2024-03-06T07:32:03.311940+0000","last_active":"2024-03-06T07:32:04.296510+0000","last_peered":"2024-03-06T07:32:04.296510+0000","last_clean":"2024-03-06T07:32:04.296510+0000","last_became_active":"2024-03-06T07:32:03.311270+0000","last_became_peered":"2024-03-06T07:32:03.311270+0000","last_unstale":"2024-03-06T07:32:04.296510+0000","last_undegraded":"2024-03-06T07:32:04.296510+0000","last_fullsized":"2024-03-06T07:32:04.296510+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:54:14.722610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.2","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.315087+0000","last_change":"2024-03-06T07:32:04.309953+0000","last_active":"2024-03-06T07:32:04.315087+0000","last_peered":"2024-03-06T07:32:04.315087+0000","last_clean":"2024-03-06T07:32:04.315087+0000","last_became_active":"2024-03-06T07:32:04.309671+0000","last_became_peered":"2024-03-06T07:32:04.309671+0000","last_unstale":"2024-03-06T07:32:04.315087+0000","last_undegraded":"2024-03-06T07:32:04.315087+0000","last_fullsized":"2024-03-06T07:32:04.315087+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:22.833719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.292791+0000","last_change":"2024-03-06T07:32:03.302900+0000","last_active":"2024-03-06T07:32:04.292791+0000","last_peered":"2024-03-06T07:32:04.292791+0000","last_clean":"2024-03-06T07:32:04.292791+0000","last_became_active":"2024-03-06T07:32:03.301642+0000","last_became_peered":"2024-03-06T07:32:03.301642+0000","last_unstale":"2024-03-06T07:32:04.292791+0000","last_undegraded":"2024-03-06T07:32:04.292791+0000","last_fullsized":"2024-03-06T07:32:04.292791+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:34:23.525002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.319171+0000","last_change":"2024-03-06T07:32:04.315091+0000","last_active":"2024-03-06T07:32:04.319171+0000","last_peered":"2024-03-06T07:32:04.319171+0000","last_clean":"2024-03-06T07:32:04.319171+0000","last_became_active":"2024-03-06T07:32:04.314821+0000","last_became_peered":"2024-03-06T07:32:04.314821+0000","last_unstale":"2024-03-06T07:32:04.319171+0000","last_undegraded":"2024-03-06T07:32:04.319171+0000","last_fullsized":"2024-03-06T07:32:04.319171+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:39:57.997996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.295853+0000","last_change":"2024-03-06T07:32:03.308759+0000","last_active":"2024-03-06T07:32:04.295853+0000","last_peered":"2024-03-06T07:32:04.295853+0000","last_clean":"2024-03-06T07:32:04.295853+0000","last_became_active":"2024-03-06T07:32:03.308147+0000","last_became_peered":"2024-03-06T07:32:03.308147+0000","last_unstale":"2024-03-06T07:32:04.295853+0000","last_undegraded":"2024-03-06T07:32:04.295853+0000","last_fullsized":"2024-03-06T07:32:04.295853+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:26.126566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.0","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.323014+0000","last_change":"2024-03-06T07:32:04.314992+0000","last_active":"2024-03-06T07:32:04.323014+0000","last_peered":"2024-03-06T07:32:04.323014+0000","last_clean":"2024-03-06T07:32:04.323014+0000","last_became_active":"2024-03-06T07:32:04.314714+0000","last_became_peered":"2024-03-06T07:32:04.314714+0000","last_unstale":"2024-03-06T07:32:04.323014+0000","last_undegraded":"2024-03-06T07:32:04.323014+0000","last_fullsized":"2024-03-06T07:32:04.323014+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:05:58.428465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.276313+0000","last_change":"2024-03-06T07:32:03.302302+0000","last_active":"2024-03-06T07:32:04.276313+0000","last_peered":"2024-03-06T07:32:04.276313+0000","last_clean":"2024-03-06T07:32:04.276313+0000","last_became_active":"2024-03-06T07:32:03.301798+0000","last_became_peered":"2024-03-06T07:32:03.301798+0000","last_unstale":"2024-03-06T07:32:04.276313+0000","last_undegraded":"2024-03-06T07:32:04.276313+0000","last_fullsized":"2024-03-06T07:32:04.276313+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:34:23.525002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.1","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.311803+0000","last_change":"2024-03-06T07:32:04.304607+0000","last_active":"2024-03-06T07:32:04.311803+0000","last_peered":"2024-03-06T07:32:04.311803+0000","last_clean":"2024-03-06T07:32:04.311803+0000","last_became_active":"2024-03-06T07:32:04.304347+0000","last_became_peered":"2024-03-06T07:32:04.304347+0000","last_unstale":"2024-03-06T07:32:04.311803+0000","last_undegraded":"2024-03-06T07:32:04.311803+0000","last_fullsized":"2024-03-06T07:32:04.311803+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:37:28.711999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":12,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.292436+0000","last_change":"2024-03-06T07:32:03.307185+0000","last_active":"2024-03-06T07:32:04.292436+0000","last_peered":"2024-03-06T07:32:04.292436+0000","last_clean":"2024-03-06T07:32:04.292436+0000","last_became_active":"2024-03-06T07:32:03.305504+0000","last_became_peered":"2024-03-06T07:32:03.305504+0000","last_unstale":"2024-03-06T07:32:04.292436+0000","last_undegraded":"2024-03-06T07:32:04.292436+0000","last_fullsized":"2024-03-06T07:32:04.292436+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:51:36.507024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.e","version":"0'0","reported_seq":9,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.309140+0000","last_change":"2024-03-06T07:32:04.305332+0000","last_active":"2024-03-06T07:32:04.309140+0000","last_peered":"2024-03-06T07:32:04.309140+0000","last_clean":"2024-03-06T07:32:04.309140+0000","last_became_active":"2024-03-06T07:32:04.305068+0000","last_became_peered":"2024-03-06T07:32:04.305068+0000","last_unstale":"2024-03-06T07:32:04.309140+0000","last_undegraded":"2024-03-06T07:32:04.309140+0000","last_fullsized":"2024-03-06T07:32:04.309140+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":92,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:03.260654+0000","last_clean_scrub_stamp":"2024-03-06T07:32:03.260654+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:41.183833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.f","version":"93'1","reported_seq":13,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.323034+0000","last_change":"2024-03-06T07:32:03.319022+0000","last_active":"2024-03-06T07:32:04.323034+0000","last_peered":"2024-03-06T07:32:04.323034+0000","last_clean":"2024-03-06T07:32:04.323034+0000","last_became_active":"2024-03-06T07:32:03.318753+0000","last_became_peered":"2024-03-06T07:32:03.318753+0000","last_unstale":"2024-03-06T07:32:04.323034+0000","last_undegraded":"2024-03-06T07:32:04.323034+0000","last_fullsized":"2024-03-06T07:32:04.323034+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":91,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:32:02.244443+0000","last_clean_scrub_stamp":"2024-03-06T07:32:02.244443+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:32:58.935413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":301,"reported_epoch":93,"state":"active+clean","last_fresh":"2024-03-06T07:32:04.276225+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:32:04.276225+0000","last_peered":"2024-03-06T07:32:04.276225+0000","last_clean":"2024-03-06T07:32:04.276225+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:32:04.276225+0000","last_undegraded":"2024-03-06T07:32:04.276225+0000","last_fullsized":"2024-03-06T07:32:04.276225+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":15,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":14,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542209,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32896,"kb_used_data":1708,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338944,"statfs":{"total":96636764160,"available":96603078656,"internally_reserved":0,"allocated":1748992,"data_stored":1077508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542208,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32288,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339552,"statfs":{"total":96636764160,"available":96603701248,"internally_reserved":0,"allocated":1134592,"data_stored":480253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542209,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32308,"kb_used_data":1124,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339532,"statfs":{"total":96636764160,"available":96603680768,"internally_reserved":0,"allocated":1150976,"data_stored":487098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542208,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32328,"kb_used_data":1140,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339512,"statfs":{"total":96636764160,"available":96603660288,"internally_reserved":0,"allocated":1167360,"data_stored":488764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542208,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32292,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339548,"statfs":{"total":96636764160,"available":96603697152,"internally_reserved":0,"allocated":1134592,"data_stored":480253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542209,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32312,"kb_used_data":1124,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339528,"statfs":{"total":96636764160,"available":96603676672,"internally_reserved":0,"allocated":1150976,"data_stored":487062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542210,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32292,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339548,"statfs":{"total":96636764160,"available":96603697152,"internally_reserved":0,"allocated":1134592,"data_stored":480253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542207,"num_pgs":11,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32872,"kb_used_data":1688,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338968,"statfs":{"total":96636764160,"available":96603103232,"internally_reserved":0,"allocated":1728512,"data_stored":1070621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:32:09.220 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:32:09.220 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:32:09.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.466+0000 7f578b600640 1 Processor -- start 2024-03-06T07:32:09.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.467+0000 7f578b600640 1 -- start start 2024-03-06T07:32:09.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f578b600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f578415fc10 0x7f5784070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f578b600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f578415fff0 0x7f5784160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f578b600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f57840b46c0 con 0x7f57840ce4a0 2024-03-06T07:32:09.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f578b600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f57840b49a0 con 0x7f578415fff0 2024-03-06T07:32:09.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f578b600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5784072c90 con 0x7f578415fc10 2024-03-06T07:32:09.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f5783e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f578415fff0 0x7f5784160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f5783e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f578415fff0 0x7f5784160450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45702/0 (socket says 172.21.15.86:45702) 2024-03-06T07:32:09.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.468+0000 7f5783e00640 1 -- 172.21.15.86:0/1417294051 learned_addr learned my addr 172.21.15.86:0/1417294051 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:09.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.469+0000 7f5783e00640 1 -- 172.21.15.86:0/1417294051 >> v2:172.21.15.194:3300/0 conn(0x7f578415fc10 msgr2=0x7f5784070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:09.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.469+0000 7f5783e00640 1 --2- 172.21.15.86:0/1417294051 >> v2:172.21.15.194:3300/0 conn(0x7f578415fc10 0x7f5784070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.469+0000 7f5783e00640 1 -- 172.21.15.86:0/1417294051 >> v1:172.21.15.86:6789/0 conn(0x7f57840ce4a0 legacy=0x7f57840ce8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.469+0000 7f5783e00640 1 -- 172.21.15.86:0/1417294051 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5784160990 con 0x7f578415fff0 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f5783e00640 1 --2- 172.21.15.86:0/1417294051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f578415fff0 0x7f5784160450 secure :-1 s=READY pgs=1089 cs=0 l=1 rev1=1 crypto rx=0x7f57840cf770 tx=0x7f576c02b9b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=54cf4ab1172587c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f5783400640 1 -- 172.21.15.86:0/1417294051 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f576c034070 con 0x7f578415fff0 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f5783400640 1 -- 172.21.15.86:0/1417294051 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f576c030870 con 0x7f578415fff0 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f5783400640 1 -- 172.21.15.86:0/1417294051 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f576c030c40 con 0x7f578415fff0 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f578b600640 1 -- 172.21.15.86:0/1417294051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f578415fff0 msgr2=0x7f5784160450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f578b600640 1 --2- 172.21.15.86:0/1417294051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f578415fff0 0x7f5784160450 secure :-1 s=READY pgs=1089 cs=0 l=1 rev1=1 crypto rx=0x7f57840cf770 tx=0x7f576c02b9b0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f578b600640 1 -- 172.21.15.86:0/1417294051 shutdown_connections 2024-03-06T07:32:09.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f578b600640 1 --2- 172.21.15.86:0/1417294051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f578415fff0 0x7f5784160450 unknown :-1 s=CLOSED pgs=1089 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f578b600640 1 --2- 172.21.15.86:0/1417294051 >> v2:172.21.15.194:3300/0 conn(0x7f578415fc10 0x7f5784070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.470+0000 7f578b600640 1 -- 172.21.15.86:0/1417294051 >> 172.21.15.86:0/1417294051 conn(0x7f57840b1940 msgr2=0x7f57840b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:09.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.471+0000 7f578b600640 1 -- 172.21.15.86:0/1417294051 shutdown_connections 2024-03-06T07:32:09.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.471+0000 7f578b600640 1 -- 172.21.15.86:0/1417294051 wait complete. 2024-03-06T07:32:09.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.472+0000 7f578b600640 1 Processor -- start 2024-03-06T07:32:09.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.472+0000 7f578b600640 1 -- start start 2024-03-06T07:32:09.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.473+0000 7f578b600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57840ce4a0 0x7f578414ef00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.473+0000 7f578b600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f578415fc10 0x7f578415a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.473+0000 7f578b600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f578415fff0 0x7f57840ccb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.473+0000 7f578b600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5784072a30 con 0x7f57840ce4a0 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.473+0000 7f578b600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57840728b0 con 0x7f578415fff0 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.473+0000 7f578b600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f578415c2e0 con 0x7f578415fc10 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.473+0000 7f5788c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57840ce4a0 0x7f578414ef00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57840ce4a0 0x7f578414ef00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45716/0 (socket says 172.21.15.86:45716) 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 -- 172.21.15.86:0/3162136061 learned_addr learned my addr 172.21.15.86:0/3162136061 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 -- 172.21.15.86:0/3162136061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f578415fc10 msgr2=0x7f578415a2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5783e00640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f578415fc10 0x7f578415a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f578415fc10 0x7f578415a2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 -- 172.21.15.86:0/3162136061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f578415fff0 msgr2=0x7f57840ccb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f578415fff0 0x7f57840ccb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 -- 172.21.15.86:0/3162136061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5784150630 con 0x7f57840ce4a0 2024-03-06T07:32:09.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5788c00640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57840ce4a0 0x7f578414ef00 secure :-1 s=READY pgs=1090 cs=0 l=1 rev1=1 crypto rx=0x7f577400b4a0 tx=0x7f577400b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:09.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5780c00640 1 -- 172.21.15.86:0/3162136061 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5774014070 con 0x7f57840ce4a0 2024-03-06T07:32:09.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5780c00640 1 -- 172.21.15.86:0/3162136061 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5774010610 con 0x7f57840ce4a0 2024-03-06T07:32:09.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.474+0000 7f5780c00640 1 -- 172.21.15.86:0/3162136061 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5774010a00 con 0x7f57840ce4a0 2024-03-06T07:32:09.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.475+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f578414f590 con 0x7f57840ce4a0 2024-03-06T07:32:09.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.475+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f578414fb50 con 0x7f57840ce4a0 2024-03-06T07:32:09.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.476+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f578404f330 con 0x7f57840ce4a0 2024-03-06T07:32:09.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.477+0000 7f5780c00640 1 -- 172.21.15.86:0/3162136061 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5774017070 con 0x7f57840ce4a0 2024-03-06T07:32:09.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.477+0000 7f5780c00640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f576006b4d0 0x7f576006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.478+0000 7f5780c00640 1 -- 172.21.15.86:0/3162136061 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5774089ca0 con 0x7f57840ce4a0 2024-03-06T07:32:09.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.478+0000 7f5783e00640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f576006b4d0 0x7f576006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.478+0000 7f5783e00640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f576006b4d0 0x7f576006d990 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f576c036000 tx=0x7f576c03b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:09.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.483+0000 7f5780c00640 1 -- 172.21.15.86:0/3162136061 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5774058a80 con 0x7f57840ce4a0 2024-03-06T07:32:09.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.738+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f578406ef30 con 0x7f57840ce4a0 2024-03-06T07:32:09.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.743+0000 7f5780c00640 1 -- 172.21.15.86:0/3162136061 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v18) v1 ==== 190+0+147 (secure 0 0 0) 0x7f5774058380 con 0x7f57840ce4a0 2024-03-06T07:32:09.744 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:32:09.745 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:32:09.745 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:32:09.745 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:32:09.745 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:32:09.745 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f576006b4d0 msgr2=0x7f576006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f576006b4d0 0x7f576006d990 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f576c036000 tx=0x7f576c03b040 comp rx=0 tx=0).stop 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57840ce4a0 msgr2=0x7f578414ef00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57840ce4a0 0x7f578414ef00 secure :-1 s=READY pgs=1090 cs=0 l=1 rev1=1 crypto rx=0x7f577400b4a0 tx=0x7f577400b960 comp rx=0 tx=0).stop 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 shutdown_connections 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f576006b4d0 0x7f576006d990 unknown :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f578415fff0 0x7f57840ccb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f578415fc10 0x7f578415a2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 --2- 172.21.15.86:0/3162136061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57840ce4a0 0x7f578414ef00 unknown :-1 s=CLOSED pgs=1090 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 >> 172.21.15.86:0/3162136061 conn(0x7f57840b1940 msgr2=0x7f57840ae0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:09.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 shutdown_connections 2024-03-06T07:32:09.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.745+0000 7f578b600640 1 -- 172.21.15.86:0/3162136061 wait complete. 2024-03-06T07:32:09.764 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:09.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.959+0000 7f6b85c00640 1 Processor -- start 2024-03-06T07:32:09.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.960+0000 7f6b85c00640 1 -- start start 2024-03-06T07:32:09.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b85c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b8015e0d0 0x7f6b800d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b85c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6b800d0fc0 0x7f6b80160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b85c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b800cea60 con 0x7f6b8015f470 2024-03-06T07:32:09.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b85c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b800d2f60 con 0x7f6b8015e0d0 2024-03-06T07:32:09.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b85c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b800d2350 con 0x7f6b800d0fc0 2024-03-06T07:32:09.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b7ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b8015e0d0 0x7f6b800d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b76000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6b8015f470 0x7f6b800d08b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55950/0 (socket says 172.21.15.86:55950) 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.961+0000 7f6b76000640 1 -- 172.21.15.86:0/1485832566 learned_addr learned my addr 172.21.15.86:0/1485832566 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.962+0000 7f6b7f400640 1 --2- 172.21.15.86:0/1485832566 >> v2:172.21.15.194:3300/0 conn(0x7f6b800d0fc0 0x7f6b80160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.962+0000 7f6b7e000640 1 -- 172.21.15.86:0/1485832566 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 760182471 0 0) 0x7f6b800cea60 con 0x7f6b8015f470 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.962+0000 7f6b7ea00640 1 -- 172.21.15.86:0/1485832566 >> v2:172.21.15.194:3300/0 conn(0x7f6b800d0fc0 msgr2=0x7f6b80160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.962+0000 7f6b7ea00640 1 --2- 172.21.15.86:0/1485832566 >> v2:172.21.15.194:3300/0 conn(0x7f6b800d0fc0 0x7f6b80160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.962+0000 7f6b7ea00640 1 -- 172.21.15.86:0/1485832566 >> v1:172.21.15.86:6789/0 conn(0x7f6b8015f470 legacy=0x7f6b800d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.962+0000 7f6b7ea00640 1 -- 172.21.15.86:0/1485832566 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b800d2630 con 0x7f6b8015e0d0 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.963+0000 7f6b7ea00640 1 --2- 172.21.15.86:0/1485832566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015e0d0 0x7f6b800d1a90 secure :-1 s=READY pgs=1092 cs=0 l=1 rev1=1 crypto rx=0x7f6b800caf20 tx=0x7f6b7002bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d32f07cf1c9a97f2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.963+0000 7f6b7e000640 1 -- 172.21.15.86:0/1485832566 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b70039070 con 0x7f6b8015e0d0 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.963+0000 7f6b7e000640 1 -- 172.21.15.86:0/1485832566 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b7002cd00 con 0x7f6b8015e0d0 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.963+0000 7f6b7e000640 1 -- 172.21.15.86:0/1485832566 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b70034660 con 0x7f6b8015e0d0 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 -- 172.21.15.86:0/1485832566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015e0d0 msgr2=0x7f6b800d1a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 --2- 172.21.15.86:0/1485832566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015e0d0 0x7f6b800d1a90 secure :-1 s=READY pgs=1092 cs=0 l=1 rev1=1 crypto rx=0x7f6b800caf20 tx=0x7f6b7002bf50 comp rx=0 tx=0).stop 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 -- 172.21.15.86:0/1485832566 shutdown_connections 2024-03-06T07:32:09.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 --2- 172.21.15.86:0/1485832566 >> v2:172.21.15.194:3300/0 conn(0x7f6b800d0fc0 0x7f6b80160dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 --2- 172.21.15.86:0/1485832566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015e0d0 0x7f6b800d1a90 unknown :-1 s=CLOSED pgs=1092 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 -- 172.21.15.86:0/1485832566 >> 172.21.15.86:0/1485832566 conn(0x7f6b800d3ec0 msgr2=0x7f6b800d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 -- 172.21.15.86:0/1485832566 shutdown_connections 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.964+0000 7f6b85c00640 1 -- 172.21.15.86:0/1485832566 wait complete. 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.966+0000 7f6b85c00640 1 Processor -- start 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.966+0000 7f6b85c00640 1 -- start start 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b85c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b800d0fc0 0x7f6b80155590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b85c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8015e0d0 0x7f6b800cd470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b85c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015f470 0x7f6b800cd9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b85c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b8015fb30 con 0x7f6b8015f470 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b85c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b800d2bf0 con 0x7f6b800d0fc0 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b85c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b800d1fd0 con 0x7f6b8015e0d0 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b7f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015f470 0x7f6b800cd9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b7f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015f470 0x7f6b800cd9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45722/0 (socket says 172.21.15.86:45722) 2024-03-06T07:32:09.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.967+0000 7f6b7f400640 1 -- 172.21.15.86:0/486772365 learned_addr learned my addr 172.21.15.86:0/486772365 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b76000640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8015e0d0 0x7f6b800cd470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b7ea00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b800d0fc0 0x7f6b80155590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b7f400640 1 -- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8015e0d0 msgr2=0x7f6b800cd470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b7f400640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8015e0d0 0x7f6b800cd470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b7f400640 1 -- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b800d0fc0 msgr2=0x7f6b80155590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b7f400640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b800d0fc0 0x7f6b80155590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b7f400640 1 -- 172.21.15.86:0/486772365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b801f4db0 con 0x7f6b8015f470 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b7f400640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015f470 0x7f6b800cd9b0 secure :-1 s=READY pgs=1093 cs=0 l=1 rev1=1 crypto rx=0x7f6b7800d830 tx=0x7f6b7800dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b77400640 1 -- 172.21.15.86:0/486772365 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b7800b6c0 con 0x7f6b8015f470 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b77400640 1 -- 172.21.15.86:0/486772365 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b78019070 con 0x7f6b8015f470 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b77400640 1 -- 172.21.15.86:0/486772365 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b7800bac0 con 0x7f6b8015f470 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.968+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b800cb8a0 con 0x7f6b8015f470 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.969+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b800cbe60 con 0x7f6b8015f470 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.970+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b8004f330 con 0x7f6b8015f470 2024-03-06T07:32:09.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.970+0000 7f6b77400640 1 -- 172.21.15.86:0/486772365 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b7801e460 con 0x7f6b8015f470 2024-03-06T07:32:09.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.971+0000 7f6b77400640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b5c06b4d0 0x7f6b5c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:09.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.971+0000 7f6b77400640 1 -- 172.21.15.86:0/486772365 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6b7808a880 con 0x7f6b8015f470 2024-03-06T07:32:09.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.972+0000 7f6b7ea00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b5c06b4d0 0x7f6b5c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:09.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.972+0000 7f6b7ea00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b5c06b4d0 0x7f6b5c06d990 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f6b70036000 tx=0x7f6b70030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:09.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:09.976+0000 7f6b77400640 1 -- 172.21.15.86:0/486772365 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6b78059700 con 0x7f6b8015f470 2024-03-06T07:32:10.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.203+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6b8008d920 con 0x7f6b8015f470 2024-03-06T07:32:10.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.204+0000 7f6b77400640 1 -- 172.21.15.86:0/486772365 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f6b78059000 con 0x7f6b8015f470 2024-03-06T07:32:10.206 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:10.206 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5824,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1969090720","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1969090720},{"type":"v1","addr":"172.21.15.194:6835","nonce":1969090720}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6969,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/883748988","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":883748988},{"type":"v1","addr":"172.21.15.86:6835","nonce":883748988}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":53},{"gid":6972,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/2563126484","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2563126484},{"type":"v1","addr":"172.21.15.86:6837","nonce":2563126484}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:06.775964+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6978},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6978":{"gid":6978,"name":"b","rank":0,"incarnation":55,"state":"up:active","state_seq":3,"addr":"172.21.15.194:6834/2712222804","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2712222804},{"type":"v1","addr":"172.21.15.194:6834","nonce":2712222804}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6978,"qdb_cluster":[6978]},"id":7}]} 2024-03-06T07:32:10.206 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 59 2024-03-06T07:32:10.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b5c06b4d0 msgr2=0x7f6b5c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b5c06b4d0 0x7f6b5c06d990 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f6b70036000 tx=0x7f6b70030000 comp rx=0 tx=0).stop 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015f470 msgr2=0x7f6b800cd9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015f470 0x7f6b800cd9b0 secure :-1 s=READY pgs=1093 cs=0 l=1 rev1=1 crypto rx=0x7f6b7800d830 tx=0x7f6b7800dcf0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 shutdown_connections 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b5c06b4d0 0x7f6b5c06d990 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8015f470 0x7f6b800cd9b0 unknown :-1 s=CLOSED pgs=1093 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8015e0d0 0x7f6b800cd470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 --2- 172.21.15.86:0/486772365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b800d0fc0 0x7f6b80155590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 >> 172.21.15.86:0/486772365 conn(0x7f6b800d3ec0 msgr2=0x7f6b800b2a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 shutdown_connections 2024-03-06T07:32:10.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.207+0000 7f6b85c00640 1 -- 172.21.15.86:0/486772365 wait complete. 2024-03-06T07:32:10.240 INFO:tasks.ceph.mds.b.smithi194.stderr:2024-03-06T07:32:10.238+0000 7fc985800640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:32:10.628 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 59, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:32:03.747182+0000', 'modified': '2024-03-06T07:32:06.775964+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 6978}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6978': {'gid': 6978, 'name': 'b', 'rank': 0, 'incarnation': 55, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.194:6834/2712222804', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.194:6832', 'nonce': 2712222804}, {'type': 'v1', 'addr': '172.21.15.194:6834', 'nonce': 2712222804}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 6978, 'qdb_cluster': [6978]} 2024-03-06T07:32:10.628 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:32:10.628 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-03-06T07:32:10.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.839+0000 7f0d3b200640 1 Processor -- start 2024-03-06T07:32:10.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.839+0000 7f0d3b200640 1 -- start start 2024-03-06T07:32:10.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.840+0000 7f0d3b200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d341561f0 0x7f0d340ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.840+0000 7f0d3b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 0x7f0d340cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.840+0000 7f0d3b200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0d3415c240 con 0x7f0d340cc800 2024-03-06T07:32:10.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.840+0000 7f0d3b200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d3415ce30 con 0x7f0d341561f0 2024-03-06T07:32:10.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.840+0000 7f0d3b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d3415c500 con 0x7f0d340aebf0 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d341561f0 0x7f0d340ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d341561f0 0x7f0d340ae6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45728/0 (socket says 172.21.15.86:45728) 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39e00640 1 -- 172.21.15.86:0/3024149120 learned_addr learned my addr 172.21.15.86:0/3024149120 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39400640 1 --2- 172.21.15.86:0/3024149120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 0x7f0d340cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39e00640 1 -- 172.21.15.86:0/3024149120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 msgr2=0x7f0d340cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39e00640 1 --2- 172.21.15.86:0/3024149120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 0x7f0d340cc2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39e00640 1 -- 172.21.15.86:0/3024149120 >> v1:172.21.15.194:6789/0 conn(0x7f0d340cc800 legacy=0x7f0d340cf800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.841+0000 7f0d39e00640 1 -- 172.21.15.86:0/3024149120 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d3415c7e0 con 0x7f0d341561f0 2024-03-06T07:32:10.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.842+0000 7f0d39e00640 1 --2- 172.21.15.86:0/3024149120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d341561f0 0x7f0d340ae6b0 secure :-1 s=READY pgs=1094 cs=0 l=1 rev1=1 crypto rx=0x7f0d28009870 tx=0x7f0d2802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=584062a0f51646a9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:10.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d38a00640 1 -- 172.21.15.86:0/3024149120 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d28035020 con 0x7f0d341561f0 2024-03-06T07:32:10.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d38a00640 1 -- 172.21.15.86:0/3024149120 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d28031640 con 0x7f0d341561f0 2024-03-06T07:32:10.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d38a00640 1 -- 172.21.15.86:0/3024149120 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2802d040 con 0x7f0d341561f0 2024-03-06T07:32:10.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d3b200640 1 -- 172.21.15.86:0/3024149120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d341561f0 msgr2=0x7f0d340ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d3b200640 1 --2- 172.21.15.86:0/3024149120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d341561f0 0x7f0d340ae6b0 secure :-1 s=READY pgs=1094 cs=0 l=1 rev1=1 crypto rx=0x7f0d28009870 tx=0x7f0d2802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d3b200640 1 -- 172.21.15.86:0/3024149120 shutdown_connections 2024-03-06T07:32:10.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d3b200640 1 --2- 172.21.15.86:0/3024149120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 0x7f0d340cc2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.843+0000 7f0d3b200640 1 --2- 172.21.15.86:0/3024149120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d341561f0 0x7f0d340ae6b0 unknown :-1 s=CLOSED pgs=1094 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.844+0000 7f0d3b200640 1 -- 172.21.15.86:0/3024149120 >> 172.21.15.86:0/3024149120 conn(0x7f0d3406afc0 msgr2=0x7f0d3406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:10.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.844+0000 7f0d3b200640 1 -- 172.21.15.86:0/3024149120 shutdown_connections 2024-03-06T07:32:10.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.844+0000 7f0d3b200640 1 -- 172.21.15.86:0/3024149120 wait complete. 2024-03-06T07:32:10.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.845+0000 7f0d3b200640 1 Processor -- start 2024-03-06T07:32:10.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.845+0000 7f0d3b200640 1 -- start start 2024-03-06T07:32:10.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.846+0000 7f0d3b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 0x7f0d341579d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.846+0000 7f0d3b200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d340cc800 0x7f0d3415d650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.846+0000 7f0d3b200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d341561f0 0x7f0d34148f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.846+0000 7f0d3b200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d341565d0 con 0x7f0d340cc800 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.846+0000 7f0d3b200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d340ccc90 con 0x7f0d341561f0 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.846+0000 7f0d3b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d34156750 con 0x7f0d340aebf0 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.846+0000 7f0d39400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d340cc800 0x7f0d3415d650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d340cc800 0x7f0d3415d650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45736/0 (socket says 172.21.15.86:45736) 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 -- 172.21.15.86:0/1968405506 learned_addr learned my addr 172.21.15.86:0/1968405506 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 -- 172.21.15.86:0/1968405506 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 msgr2=0x7f0d341579d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 0x7f0d341579d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 -- 172.21.15.86:0/1968405506 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d341561f0 msgr2=0x7f0d34148f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d341561f0 0x7f0d34148f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 -- 172.21.15.86:0/1968405506 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d34150950 con 0x7f0d340cc800 2024-03-06T07:32:10.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d39400640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d340cc800 0x7f0d3415d650 secure :-1 s=READY pgs=1095 cs=0 l=1 rev1=1 crypto rx=0x7f0d2400d830 tx=0x7f0d2400dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:10.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d2e000640 1 -- 172.21.15.86:0/1968405506 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2400b6c0 con 0x7f0d340cc800 2024-03-06T07:32:10.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.847+0000 7f0d2e000640 1 -- 172.21.15.86:0/1968405506 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d24019070 con 0x7f0d340cc800 2024-03-06T07:32:10.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.848+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d34150c30 con 0x7f0d340cc800 2024-03-06T07:32:10.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.848+0000 7f0d2e000640 1 -- 172.21.15.86:0/1968405506 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2400ba80 con 0x7f0d340cc800 2024-03-06T07:32:10.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.848+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d341511f0 con 0x7f0d340cc800 2024-03-06T07:32:10.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.850+0000 7f0d2e000640 1 -- 172.21.15.86:0/1968405506 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0d2401e460 con 0x7f0d340cc800 2024-03-06T07:32:10.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.851+0000 7f0d2e000640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d1806b4d0 0x7f0d1806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.851+0000 7f0d2e000640 1 -- 172.21.15.86:0/1968405506 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0d2408a950 con 0x7f0d340cc800 2024-03-06T07:32:10.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.851+0000 7f0d39e00640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d1806b4d0 0x7f0d1806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.851+0000 7f0d3f573740 1 Processor -- start 2024-03-06T07:32:10.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d3f573740 1 -- start start 2024-03-06T07:32:10.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d39e00640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d1806b4d0 0x7f0d1806d990 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f0d28036000 tx=0x7f0d28030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:10.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e079990 0x55b42e04c930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e07c720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e04ce70 0x55b42e07cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d3f573740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42dfb6600 con 0x55b42e04ce70 2024-03-06T07:32:10.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42df5f930 con 0x55b42e079d70 2024-03-06T07:32:10.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.852+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55b42dfb26a0 con 0x55b42e079990 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e04ce70 0x55b42e07cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e04ce70 0x55b42e07cc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45742/0 (socket says 172.21.15.86:45742) 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 -- 172.21.15.86:0/3267988598 learned_addr learned my addr 172.21.15.86:0/3267988598 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d39400640 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e07c720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d39e00640 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e079990 0x55b42e04c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 -- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e079990 msgr2=0x55b42e04c930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e079990 0x55b42e04c930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 -- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 msgr2=0x55b42e07c720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e07c720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 -- 172.21.15.86:0/3267988598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b42dccb2d0 con 0x55b42e04ce70 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.853+0000 7f0d3a800640 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e04ce70 0x55b42e07cc60 secure :-1 s=READY pgs=1096 cs=0 l=1 rev1=1 crypto rx=0x7f0d30004440 tx=0x7f0d30004900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=25cc1d395ff5478b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d22600640 1 -- 172.21.15.86:0/3267988598 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d3000cc80 con 0x55b42e04ce70 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d22600640 1 -- 172.21.15.86:0/3267988598 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d30017070 con 0x55b42e04ce70 2024-03-06T07:32:10.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d22600640 1 -- 172.21.15.86:0/3267988598 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d30012640 con 0x55b42e04ce70 2024-03-06T07:32:10.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d3f573740 1 -- 172.21.15.86:0/3267988598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e04ce70 msgr2=0x55b42e07cc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e04ce70 0x55b42e07cc60 secure :-1 s=READY pgs=1096 cs=0 l=1 rev1=1 crypto rx=0x7f0d30004440 tx=0x7f0d30004900 comp rx=0 tx=0).stop 2024-03-06T07:32:10.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d3f573740 1 -- 172.21.15.86:0/3267988598 shutdown_connections 2024-03-06T07:32:10.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e04ce70 0x55b42e07cc60 unknown :-1 s=CLOSED pgs=1096 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e07c720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3267988598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e079990 0x55b42e04c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.854+0000 7f0d3f573740 1 -- 172.21.15.86:0/3267988598 >> 172.21.15.86:0/3267988598 conn(0x55b42e04eba0 msgr2=0x55b42e097720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.855+0000 7f0d3f573740 1 -- 172.21.15.86:0/3267988598 shutdown_connections 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.855+0000 7f0d3f573740 1 -- 172.21.15.86:0/3267988598 wait complete. 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.855+0000 7f0d3f573740 1 -- start start 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.855+0000 7f0d3f573740 1 Processor -- start 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.856+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079990 0x55b42e07f650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.856+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e095f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.856+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e070eb0 0x55b42e0964a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.856+0000 7f0d3f573740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42e051140 con 0x55b42e079990 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42df72d90 con 0x55b42e079d70 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55b42e050530 con 0x55b42e070eb0 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d3f573740 10 client.0 _ll_register_callbacks cb 0x55b42dfae350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079990 0x55b42e07f650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079990 0x55b42e07f650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45748/0 (socket says 172.21.15.86:45748) 2024-03-06T07:32:10.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 -- 172.21.15.86:0/2025099432 learned_addr learned my addr 172.21.15.86:0/2025099432 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 -- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e070eb0 msgr2=0x55b42e0964a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e070eb0 0x55b42e0964a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 -- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 msgr2=0x55b42e095f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e095f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 -- 172.21.15.86:0/2025099432 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55b42dd956b0 con 0x55b42e079990 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.857+0000 7f0d39e00640 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079990 0x55b42e07f650 secure :-1 s=READY pgs=1097 cs=0 l=1 rev1=1 crypto rx=0x7f0d280090b0 tx=0x7f0d28004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cd11c50d5efc413e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d16a00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d16a00640 1 -- 172.21.15.86:0/2025099432 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2804a070 con 0x55b42e079990 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d16a00640 1 -- 172.21.15.86:0/2025099432 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0d2803ad80 con 0x55b42e079990 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0d30017070 con 0x55b42e079990 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d16a00640 1 -- 172.21.15.86:0/2025099432 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=59) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0d28040070 con 0x55b42e079990 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d3f573740 10 client.7119 fetch_fsmap learned FSMap version 59 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=59,monmap=2+}) v3 -- 0x55b42e093ff0 con 0x55b42e079990 2024-03-06T07:32:10.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d16a00640 1 -- 172.21.15.86:0/2025099432 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d30017070 con 0x55b42e079990 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d16a00640 1 -- 172.21.15.86:0/2025099432 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 59) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f0d28058480 con 0x55b42e079990 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d3f573740 10 client.7119 fetch_fsmap finished waiting for FSMap version 59 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d3f573740 10 client.7119 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.858+0000 7f0d3f573740 20 client.7119 populate_metadata read hostname 'smithi086' 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.859+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0afbe0 0x55b42e094fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.859+0000 7f0d3f573740 4 client.7119 mds_command: new command op to 6978 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.859+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 --> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b42dfc3b40 con 0x55b42e0afbe0 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.859+0000 7f0d39400640 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0afbe0 0x55b42e094fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.860+0000 7f0d39400640 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0afbe0 0x55b42e094fa0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=86ffc96d6a05982c server_cookie=69638a02e6672237 in_seq=0 out_seq=0 2024-03-06T07:32:10.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.860+0000 7f0d16a00640 10 client.7119 ms_handle_connect on v2:172.21.15.194:6832/2712222804 2024-03-06T07:32:10.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.862+0000 7f0d16a00640 1 -- 172.21.15.86:0/2025099432 <== mds.0 v2:172.21.15.194:6832/2712222804 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x55b42dfc3b40 con 0x55b42e0afbe0 2024-03-06T07:32:10.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.862+0000 7f0d16a00640 10 client.7119 handle_command_reply: tid=0 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.862+0000 7f0d3f573740 1 client.7119 shutdown 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079990 msgr2=0x55b42e07f650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079990 0x55b42e07f650 secure :-1 s=READY pgs=1097 cs=0 l=1 rev1=1 crypto rx=0x7f0d280090b0 tx=0x7f0d28004730 comp rx=0 tx=0).stop 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 shutdown_connections 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0afbe0 0x55b42e094fa0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e070eb0 0x55b42e0964a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e095f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2025099432 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079990 0x55b42e07f650 unknown :-1 s=CLOSED pgs=1097 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 >> 172.21.15.86:0/2025099432 conn(0x55b42e04ce70 msgr2=0x55b42e04c050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d16a00640 0 client.7119 ms_handle_reset on v2:172.21.15.194:6832/2712222804 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 shutdown_connections 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 1 -- 172.21.15.86:0/2025099432 wait complete. 2024-03-06T07:32:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.863+0000 7f0d3f573740 20 client.7119 trim_cache size 0 max 16384 2024-03-06T07:32:10.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.881+0000 7f0d3f573740 1 Processor -- start 2024-03-06T07:32:10.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.881+0000 7f0d3f573740 1 -- start start 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.882+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e070eb0 0x55b42e047c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.882+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079d70 0x55b42e050680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.882+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 0x55b42e07ecd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.882+0000 7f0d3f573740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d340cdd90 con 0x55b42e079d70 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.882+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d341f6460 con 0x55b42e070eb0 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.882+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d340cd790 con 0x55b42e050bc0 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.882+0000 7f0d39400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079d70 0x55b42e050680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079d70 0x55b42e050680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45754/0 (socket says 172.21.15.86:45754) 2024-03-06T07:32:10.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 -- 172.21.15.86:0/2753349211 learned_addr learned my addr 172.21.15.86:0/2753349211 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 -- 172.21.15.86:0/2753349211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 msgr2=0x55b42e07ecd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39e00640 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e070eb0 0x55b42e047c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 0x55b42e07ecd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 -- 172.21.15.86:0/2753349211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e070eb0 msgr2=0x55b42e047c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e070eb0 0x55b42e047c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 -- 172.21.15.86:0/2753349211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b42df7b720 con 0x55b42e079d70 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d39400640 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079d70 0x55b42e050680 secure :-1 s=READY pgs=1098 cs=0 l=1 rev1=1 crypto rx=0x7f0d24058d30 tx=0x7f0d24059850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3b94c9135be7da8d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d21c00640 1 -- 172.21.15.86:0/2753349211 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d24075070 con 0x55b42e079d70 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.883+0000 7f0d21c00640 1 -- 172.21.15.86:0/2753349211 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d2405d180 con 0x55b42e079d70 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d21c00640 1 -- 172.21.15.86:0/2753349211 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d24061740 con 0x55b42e079d70 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 -- 172.21.15.86:0/2753349211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079d70 msgr2=0x55b42e050680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079d70 0x55b42e050680 secure :-1 s=READY pgs=1098 cs=0 l=1 rev1=1 crypto rx=0x7f0d24058d30 tx=0x7f0d24059850 comp rx=0 tx=0).stop 2024-03-06T07:32:10.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 -- 172.21.15.86:0/2753349211 shutdown_connections 2024-03-06T07:32:10.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 0x55b42e07ecd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e079d70 0x55b42e050680 unknown :-1 s=CLOSED pgs=1098 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 --2- 172.21.15.86:0/2753349211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e070eb0 0x55b42e047c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 -- 172.21.15.86:0/2753349211 >> 172.21.15.86:0/2753349211 conn(0x55b42e04ce70 msgr2=0x55b42e0806c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:10.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 -- 172.21.15.86:0/2753349211 shutdown_connections 2024-03-06T07:32:10.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.884+0000 7f0d3f573740 1 -- 172.21.15.86:0/2753349211 wait complete. 2024-03-06T07:32:10.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.885+0000 7f0d3f573740 1 -- start start 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.885+0000 7f0d3f573740 1 Processor -- start 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 0x55b42e101610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e070eb0 0x55b42e104910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d3f573740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e104e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d3f573740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42e0afa60 con 0x55b42e070eb0 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42e0abed0 con 0x55b42e079d70 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d3f573740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55b42e0b0030 con 0x55b42e050bc0 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d3f573740 10 client.0 _ll_register_callbacks cb 0x7f0d280096f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d39400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e070eb0 0x55b42e104910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d39400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e070eb0 0x55b42e104910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45760/0 (socket says 172.21.15.86:45760) 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.886+0000 7f0d39400640 1 -- 172.21.15.86:0/3062912040 learned_addr learned my addr 172.21.15.86:0/3062912040 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d39400640 1 -- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 msgr2=0x55b42e101610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d3a800640 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e104e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d39e00640 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 0x55b42e101610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d39400640 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 0x55b42e101610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d39400640 1 -- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 msgr2=0x55b42e104e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d39400640 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e104e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d39400640 1 -- 172.21.15.86:0/3062912040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55b42e101d40 con 0x55b42e070eb0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d39400640 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e070eb0 0x55b42e104910 secure :-1 s=READY pgs=1099 cs=0 l=1 rev1=1 crypto rx=0x7f0d24055440 tx=0x7f0d24088cb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4f3778bb48d2308e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d16a00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d16a00640 1 -- 172.21.15.86:0/3062912040 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2407c060 con 0x55b42e070eb0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d16a00640 1 -- 172.21.15.86:0/3062912040 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0d24088ef0 con 0x55b42e070eb0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0d2405d180 con 0x55b42e070eb0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d16a00640 1 -- 172.21.15.86:0/3062912040 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=59) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0d24075030 con 0x55b42e070eb0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d3f573740 10 client.7125 fetch_fsmap learned FSMap version 59 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=59,monmap=2+}) v3 -- 0x55b42e1022d0 con 0x55b42e070eb0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.887+0000 7f0d16a00640 1 -- 172.21.15.86:0/3062912040 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d2405d180 con 0x55b42e070eb0 2024-03-06T07:32:10.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d16a00640 1 -- 172.21.15.86:0/3062912040 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 59) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f0d24013070 con 0x55b42e070eb0 2024-03-06T07:32:10.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d3f573740 10 client.7125 fetch_fsmap finished waiting for FSMap version 59 2024-03-06T07:32:10.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d3f573740 10 client.7125 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-06T07:32:10.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d3f573740 20 client.7125 populate_metadata read hostname 'smithi086' 2024-03-06T07:32:10.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0ae880 0x55b42e0aec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:10.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d3f573740 4 client.7125 mds_command: new command op to 6978 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:32:10.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 --> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55b42dfbde90 con 0x55b42e0ae880 2024-03-06T07:32:10.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.888+0000 7f0d39e00640 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0ae880 0x55b42e0aec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:10.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d39e00640 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0ae880 0x55b42e0aec60 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6fa592cd7ffd52ab server_cookie=3fac12cc755cd939 in_seq=0 out_seq=0 2024-03-06T07:32:10.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d16a00640 10 client.7125 ms_handle_connect on v2:172.21.15.194:6832/2712222804 2024-03-06T07:32:10.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d16a00640 1 -- 172.21.15.86:0/3062912040 <== mds.0 v2:172.21.15.194:6832/2712222804 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+318 (crc 0 0 0) 0x55b42dfbde90 con 0x55b42e0ae880 2024-03-06T07:32:10.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d16a00640 10 client.7125 handle_command_reply: tid=0 2024-03-06T07:32:10.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 client.7125 shutdown 2024-03-06T07:32:10.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e070eb0 msgr2=0x55b42e104910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e070eb0 0x55b42e104910 secure :-1 s=READY pgs=1099 cs=0 l=1 rev1=1 crypto rx=0x7f0d24055440 tx=0x7f0d24088cb0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 shutdown_connections 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:6832/2712222804,v1:172.21.15.194:6834/2712222804] conn(0x55b42e0ae880 0x55b42e0aec60 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55b42e079d70 0x55b42e104e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55b42e070eb0 0x55b42e104910 unknown :-1 s=CLOSED pgs=1099 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 --2- 172.21.15.86:0/3062912040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55b42e050bc0 0x55b42e101610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d16a00640 0 client.7125 ms_handle_reset on v2:172.21.15.194:6832/2712222804 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.890+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 >> 172.21.15.86:0/3062912040 conn(0x55b42e04ce70 msgr2=0x55b42e0ab4e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 shutdown_connections 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3f573740 1 -- 172.21.15.86:0/3062912040 wait complete. 2024-03-06T07:32:10.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3f573740 20 client.7125 trim_cache size 0 max 16384 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 6978, 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 6.596407546, 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 59, 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 94, 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 94, 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 12.032972438 2024-03-06T07:32:10.895 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:32:10.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d1806b4d0 msgr2=0x7f0d1806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d1806b4d0 0x7f0d1806d990 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f0d28036000 tx=0x7f0d28030000 comp rx=0 tx=0).stop 2024-03-06T07:32:10.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d340cc800 msgr2=0x7f0d3415d650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:10.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d340cc800 0x7f0d3415d650 secure :-1 s=READY pgs=1095 cs=0 l=1 rev1=1 crypto rx=0x7f0d2400d830 tx=0x7f0d2400dcf0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 shutdown_connections 2024-03-06T07:32:10.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d1806b4d0 0x7f0d1806d990 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d341561f0 0x7f0d34148f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d340cc800 0x7f0d3415d650 unknown :-1 s=CLOSED pgs=1095 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 --2- 172.21.15.86:0/1968405506 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d340aebf0 0x7f0d341579d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:10.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.891+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 >> 172.21.15.86:0/1968405506 conn(0x7f0d3406afc0 msgr2=0x7f0d3415ae70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:10.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.892+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 shutdown_connections 2024-03-06T07:32:10.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:10.892+0000 7f0d3b200640 1 -- 172.21.15.86:0/1968405506 wait complete. 2024-03-06T07:32:10.909 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:32:11.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.095+0000 7f5c98000640 1 Processor -- start 2024-03-06T07:32:11.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.095+0000 7f5c98000640 1 -- start start 2024-03-06T07:32:11.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.097+0000 7f5c98000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 0x7f5c90071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.097+0000 7f5c98000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c9006cef0 con 0x7f5c9015f9d0 2024-03-06T07:32:11.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.097+0000 7f5c98000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c9006d1d0 con 0x7f5c90144c00 2024-03-06T07:32:11.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.097+0000 7f5c98000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c9006d4b0 con 0x7f5c9015fe60 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.097+0000 7f5c94c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f5c9015f9d0 0x7f5c90070b10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55952/0 (socket says 172.21.15.86:55952) 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.097+0000 7f5c94c00640 1 -- 172.21.15.86:0/4070691121 learned_addr learned my addr 172.21.15.86:0/4070691121 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c96000640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 0x7f5c90071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 151455478 0 0) 0x7f5c9006cef0 con 0x7f5c9015f9d0 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5c78003520 con 0x7f5c9015f9d0 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3738305006 0 0) 0x7f5c78003520 con 0x7f5c9015f9d0 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 msgr2=0x7f5c90071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 0x7f5c90071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 >> v1:172.21.15.194:6789/0 conn(0x7f5c90144c00 legacy=0x7f5c9006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:11.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c9006d790 con 0x7f5c9015f9d0 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f5c7c001670 con 0x7f5c9015f9d0 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 >> v1:172.21.15.86:6789/0 conn(0x7f5c9015f9d0 legacy=0x7f5c90070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c780039e0 0x7f5c78003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 0x7f5c78004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c78004d40 0x7f5c780051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c9006d4b0 con 0x7f5c78004d40 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c7800ba70 con 0x7f5c780039e0 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.098+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c7800bc70 con 0x7f5c9015fe60 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c96000640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c78004d40 0x7f5c780051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c94c00640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 0x7f5c78004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c95600640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c780039e0 0x7f5c78003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c96000640 1 -- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 msgr2=0x7f5c78004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c96000640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c9015fe60 0x7f5c78004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c96000640 1 -- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c780039e0 msgr2=0x7f5c78003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:11.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c96000640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c780039e0 0x7f5c78003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c96000640 1 -- 172.21.15.86:0/4070691121 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c78003520 con 0x7f5c78004d40 2024-03-06T07:32:11.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.099+0000 7f5c95600640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c780039e0 0x7f5c78003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:32:11.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.100+0000 7f5c96000640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c78004d40 0x7f5c780051c0 secure :-1 s=READY pgs=1101 cs=0 l=1 rev1=1 crypto rx=0x7f5c88009f90 tx=0x7f5c78005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4791247cf5a1d324 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:11.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.100+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c88039070 con 0x7f5c78004d40 2024-03-06T07:32:11.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.100+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c8802cbd0 con 0x7f5c78004d40 2024-03-06T07:32:11.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.100+0000 7f5c8fe00640 1 -- 172.21.15.86:0/4070691121 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c88009c10 con 0x7f5c78004d40 2024-03-06T07:32:11.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.101+0000 7f5c98000640 1 -- 172.21.15.86:0/4070691121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c78004d40 msgr2=0x7f5c780051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:11.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.101+0000 7f5c98000640 1 --2- 172.21.15.86:0/4070691121 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c78004d40 0x7f5c780051c0 secure :-1 s=READY pgs=1101 cs=0 l=1 rev1=1 crypto rx=0x7f5c88009f90 tx=0x7f5c78005830 comp rx=0 tx=0).stop 2024-03-06T07:32:11.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.101+0000 7f5c96000640 1 -- 172.21.15.86:0/4070691121 reap_dead start 2024-03-06T07:32:11.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.101+0000 7f5c98000640 1 -- 172.21.15.86:0/4070691121 shutdown_connections 2024-03-06T07:32:11.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.101+0000 7f5c98000640 1 -- 172.21.15.86:0/4070691121 >> 172.21.15.86:0/4070691121 conn(0x7f5c900b1730 msgr2=0x7f5c900b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:11.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.101+0000 7f5c98000640 1 -- 172.21.15.86:0/4070691121 shutdown_connections 2024-03-06T07:32:11.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.101+0000 7f5c98000640 1 -- 172.21.15.86:0/4070691121 wait complete. 2024-03-06T07:32:11.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.103+0000 7f5c98000640 1 Processor -- start 2024-03-06T07:32:11.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.103+0000 7f5c98000640 1 -- start start 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c98000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c78004d40 0x7f5c90138f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c98000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c9013bea0 0x7f5c9015ac00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c98000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c901e55c0 0x7f5c9015b140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c98000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c9006cb50 con 0x7f5c9013bea0 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c98000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c90160730 con 0x7f5c901e55c0 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c98000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c9006ccd0 con 0x7f5c78004d40 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c94c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c9013bea0 0x7f5c9015ac00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c94c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c9013bea0 0x7f5c9015ac00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45778/0 (socket says 172.21.15.86:45778) 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c94c00640 1 -- 172.21.15.86:0/3807166442 learned_addr learned my addr 172.21.15.86:0/3807166442 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:11.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c95600640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c78004d40 0x7f5c90138f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c96000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c901e55c0 0x7f5c9015b140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c94c00640 1 -- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c78004d40 msgr2=0x7f5c90138f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.104+0000 7f5c94c00640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c78004d40 0x7f5c90138f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c94c00640 1 -- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c901e55c0 msgr2=0x7f5c9015b140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c94c00640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c901e55c0 0x7f5c9015b140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c94c00640 1 -- 172.21.15.86:0/3807166442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c901e5b60 con 0x7f5c9013bea0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c96000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c901e55c0 0x7f5c9015b140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c94c00640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c9013bea0 0x7f5c9015ac00 secure :-1 s=READY pgs=1102 cs=0 l=1 rev1=1 crypto rx=0x7f5c7c007320 tx=0x7f5c7c0077e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c8d600640 1 -- 172.21.15.86:0/3807166442 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c7c01a070 con 0x7f5c9013bea0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c8d600640 1 -- 172.21.15.86:0/3807166442 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c7c00f9a0 con 0x7f5c9013bea0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c8d600640 1 -- 172.21.15.86:0/3807166442 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c7c00fd90 con 0x7f5c9013bea0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c901e5e40 con 0x7f5c9013bea0 2024-03-06T07:32:11.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.105+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c901e6400 con 0x7f5c9013bea0 2024-03-06T07:32:11.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.106+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c9004f330 con 0x7f5c9013bea0 2024-03-06T07:32:11.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.107+0000 7f5c8d600640 1 -- 172.21.15.86:0/3807166442 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5c7c01d070 con 0x7f5c9013bea0 2024-03-06T07:32:11.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.108+0000 7f5c8d600640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c7006b4d0 0x7f5c7006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:11.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.108+0000 7f5c8d600640 1 -- 172.21.15.86:0/3807166442 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5c7c089c90 con 0x7f5c9013bea0 2024-03-06T07:32:11.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.108+0000 7f5c95600640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c7006b4d0 0x7f5c7006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:11.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.109+0000 7f5c95600640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c7006b4d0 0x7f5c7006d990 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f5c840054d0 tx=0x7f5c84009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:11.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.113+0000 7f5c8d600640 1 -- 172.21.15.86:0/3807166442 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5c7c058b10 con 0x7f5c9013bea0 2024-03-06T07:32:11.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.335+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f5c90072890 con 0x7f5c9013bea0 2024-03-06T07:32:11.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.336+0000 7f5c8d600640 1 -- 172.21.15.86:0/3807166442 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v59) v1 ==== 53+0+73 (secure 0 0 0) 0x7f5c7c058410 con 0x7f5c9013bea0 2024-03-06T07:32:11.337 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c7006b4d0 msgr2=0x7f5c7006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c7006b4d0 0x7f5c7006d990 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f5c840054d0 tx=0x7f5c84009040 comp rx=0 tx=0).stop 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c9013bea0 msgr2=0x7f5c9015ac00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c9013bea0 0x7f5c9015ac00 secure :-1 s=READY pgs=1102 cs=0 l=1 rev1=1 crypto rx=0x7f5c7c007320 tx=0x7f5c7c0077e0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 shutdown_connections 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c7006b4d0 0x7f5c7006d990 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c901e55c0 0x7f5c9015b140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c9013bea0 0x7f5c9015ac00 unknown :-1 s=CLOSED pgs=1102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 --2- 172.21.15.86:0/3807166442 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c78004d40 0x7f5c90138f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.338+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 >> 172.21.15.86:0/3807166442 conn(0x7f5c900b1730 msgr2=0x7f5c90070b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:11.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.339+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 shutdown_connections 2024-03-06T07:32:11.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:11.339+0000 7f5c98000640 1 -- 172.21.15.86:0/3807166442 wait complete. 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:32:11.357 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:32:11.358 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:11.358 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:32:11.402 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:32:11.402 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:32:11.402 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:32:11.403 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42135sec preferred_lft 42135sec 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:32:11.404 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:32:11.405 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:11.405 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:32:11.524 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:32:11.528 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:11.528 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:32:11.580 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:32:11.580 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:32:11.581 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:32:11.581 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:32:11.581 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:32:11.581 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:11.581 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:32:11.581 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:32:11.581 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:32:11.581 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:32:11.581 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:32:11.581 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:32:11.759 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:11.759 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:32:11.809 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:32:11.809 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:32:11.810 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:32:11.810 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:11.810 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:32:11.901 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:32:11.901 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:32:11.954 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:32:11.955 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:32:11.955 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:32:12.004 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:32:12.004 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:12.004 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:32:12.144 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:32:12.145 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:32:12.180 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:32:12.180 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:32:12.180 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:32:12.180 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:32:12.181 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:32:12.183 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:32:12.183 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:32:12.183 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:32:12.183 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:32:12.183 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:32:12.190 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:32:12.190 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:12.190 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:32:12.270 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:12.271 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:32:12.350 INFO:teuthology.orchestra.run.smithi086.stdout:client5195 2024-03-06T07:32:12.350 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:32:12.350 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:12.402 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:12.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:12.562 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: chdir = "" 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: auth supported = cephx 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: filestore xattr use omap = true 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: auth debug = true 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.563 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on old message = true 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on bug = true 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default size = 2 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow pg remap = true 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:12.564 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on too few osds = false 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: osd default data pool replay window = 5 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon allow pool delete = true 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.565 INFO:teuthology.orchestra.run.smithi086.stdout: mon cluster log file level = debug 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: debug asserts on shutdown = true 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: mon health detail to clog = false 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on skipped message = False 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout:[osd] 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: osd journal size = 100 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub max interval = 600 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:12.566 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd recover clone overlap = true 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug shutdown = true 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug op order = true 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug trim objects = true 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd open classes on start = true 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug pg log writeout = true 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: osd map max advance = 10 2024-03-06T07:32:12.567 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: journal zero on create = true 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: bdev debug aio = true 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug misdirected ops = true 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: bdev async discard = True 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: bdev enable discard = True 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore allocator = bitmap 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore block size = 96636764160 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore fsck on mount = True 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluefs = 1/20 2024-03-06T07:32:12.568 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluestore = 1/20 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: debug osd = 20 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: debug rocksdb = 4/10 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: osd objectstore = bluestore 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr] 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: debug mgr = 20 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:12.569 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout:[mon] 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: debug paxos = 20 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mon data avail warn = 5 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd prime pg temp = true 2024-03-06T07:32:12.570 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: # 1m isn't quite enough 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: mon op complaint time = 120 2024-03-06T07:32:12.571 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout:[client] 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: rgw cache enabled = true 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable ops log = true 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable usage log = true 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: client mount timeout = 600 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: debug client = 20 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = "" 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.572 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.a] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.b] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.c] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout:[mds] 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds = 20 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds balancer = 20 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug frag = True 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug scatterstat = True 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout: mds op complaint time = 180 2024-03-06T07:32:12.573 INFO:teuthology.orchestra.run.smithi086.stdout: mds verify scatter = True 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_files = 6 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi194.stdout:[global] 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi194.stdout: chdir = "" 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi194.stdout: auth supported = cephx 2024-03-06T07:32:12.574 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: filestore xattr use omap = true 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: auth debug = true 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on old message = true 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on bug = true 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:12.575 INFO:teuthology.orchestra.run.smithi194.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default size = 2 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow pg remap = true 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:12.576 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on too few osds = false 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: osd default data pool replay window = 5 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: mon allow pool delete = true 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: mon cluster log file level = debug 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: debug asserts on shutdown = true 2024-03-06T07:32:12.577 INFO:teuthology.orchestra.run.smithi194.stdout: mon health detail to clog = false 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on skipped message = False 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout:[osd] 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: osd journal size = 100 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub max interval = 600 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.578 INFO:teuthology.orchestra.run.smithi194.stdout: osd recover clone overlap = true 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug shutdown = true 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug op order = true 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug trim objects = true 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd open classes on start = true 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug pg log writeout = true 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.579 INFO:teuthology.orchestra.run.smithi194.stdout: osd map max advance = 10 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: journal zero on create = true 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: bdev debug aio = true 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug misdirected ops = true 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: bdev async discard = True 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: bdev enable discard = True 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore allocator = bitmap 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore block size = 96636764160 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore fsck on mount = True 2024-03-06T07:32:12.580 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluefs = 1/20 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluestore = 1/20 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: debug osd = 20 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: debug rocksdb = 4/10 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: osd objectstore = bluestore 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr] 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: debug mgr = 20 2024-03-06T07:32:12.581 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout:[mon] 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: debug paxos = 20 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mon data avail warn = 5 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:12.582 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd prime pg temp = true 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: # 1m isn't quite enough 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.583 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: mon op complaint time = 120 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout:[client] 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: rgw cache enabled = true 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable ops log = true 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable usage log = true 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: client mount timeout = 600 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: debug client = 20 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = "" 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.584 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.a] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.b] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.c] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout:[mds] 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds = 20 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds balancer = 20 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug frag = True 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug scatterstat = True 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: mds op complaint time = 180 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: mds verify scatter = True 2024-03-06T07:32:12.585 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:12.586 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.586 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.586 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_files = 6 2024-03-06T07:32:12.628 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:12.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:12.786 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-03-06T07:32:12.786 INFO:teuthology.orchestra.run.smithi086.stdout: chdir = "" 2024-03-06T07:32:12.786 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:12.786 INFO:teuthology.orchestra.run.smithi086.stdout: auth supported = cephx 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: filestore xattr use omap = true 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: auth debug = true 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on old message = true 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on bug = true 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:12.787 INFO:teuthology.orchestra.run.smithi086.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default size = 2 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow pg remap = true 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on too few osds = false 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.788 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: osd default data pool replay window = 5 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: mon allow pool delete = true 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: mon cluster log file level = debug 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: debug asserts on shutdown = true 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: mon health detail to clog = false 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on skipped message = False 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout:[osd] 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: osd journal size = 100 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:12.789 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub max interval = 600 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd recover clone overlap = true 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug shutdown = true 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug op order = true 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug trim objects = true 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd open classes on start = true 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug pg log writeout = true 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.790 INFO:teuthology.orchestra.run.smithi086.stdout: osd map max advance = 10 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: journal zero on create = true 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: bdev debug aio = true 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug misdirected ops = true 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: bdev async discard = True 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: bdev enable discard = True 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore allocator = bitmap 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore block size = 96636764160 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore fsck on mount = True 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluefs = 1/20 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluestore = 1/20 2024-03-06T07:32:12.791 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: debug osd = 20 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: debug rocksdb = 4/10 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: osd objectstore = bluestore 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr] 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: debug mgr = 20 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:12.792 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout:[mon] 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: debug paxos = 20 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: mon data avail warn = 5 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:12.793 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd prime pg temp = true 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: # 1m isn't quite enough 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.794 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: mon op complaint time = 120 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout:[client] 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: rgw cache enabled = true 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable ops log = true 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable usage log = true 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: client mount timeout = 600 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: debug client = 20 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = "" 2024-03-06T07:32:12.795 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.a] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.b] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.c] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout:[mds] 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds = 20 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds balancer = 20 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:12.796 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug frag = True 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug scatterstat = True 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: mds op complaint time = 180 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: mds verify scatter = True 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_files = 6 2024-03-06T07:32:12.797 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout:[global] 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: chdir = "" 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: auth supported = cephx 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: filestore xattr use omap = true 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:12.800 INFO:teuthology.orchestra.run.smithi194.stdout: auth debug = true 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on old message = true 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on bug = true 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default size = 2 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.801 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow pg remap = true 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on too few osds = false 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: osd default data pool replay window = 5 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.802 INFO:teuthology.orchestra.run.smithi194.stdout: mon allow pool delete = true 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: mon cluster log file level = debug 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: debug asserts on shutdown = true 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: mon health detail to clog = false 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on skipped message = False 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout:[osd] 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: osd journal size = 100 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub max interval = 600 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.803 INFO:teuthology.orchestra.run.smithi194.stdout: osd recover clone overlap = true 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug shutdown = true 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug op order = true 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug trim objects = true 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd open classes on start = true 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug pg log writeout = true 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: osd map max advance = 10 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.804 INFO:teuthology.orchestra.run.smithi194.stdout: journal zero on create = true 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: bdev debug aio = true 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug misdirected ops = true 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: bdev async discard = True 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: bdev enable discard = True 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore allocator = bitmap 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore block size = 96636764160 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore fsck on mount = True 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluefs = 1/20 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluestore = 1/20 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.805 INFO:teuthology.orchestra.run.smithi194.stdout: debug osd = 20 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: debug rocksdb = 4/10 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: osd objectstore = bluestore 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr] 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: debug mgr = 20 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:12.806 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout:[mon] 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: debug paxos = 20 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mon data avail warn = 5 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:12.807 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd prime pg temp = true 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: # 1m isn't quite enough 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.808 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: mon op complaint time = 120 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout:[client] 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: rgw cache enabled = true 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable ops log = true 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable usage log = true 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: client mount timeout = 600 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: debug client = 20 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = "" 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.809 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.a] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.b] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.c] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout:[mds] 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds = 20 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds balancer = 20 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug frag = True 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug scatterstat = True 2024-03-06T07:32:12.810 INFO:teuthology.orchestra.run.smithi194.stdout: mds op complaint time = 180 2024-03-06T07:32:12.811 INFO:teuthology.orchestra.run.smithi194.stdout: mds verify scatter = True 2024-03-06T07:32:12.811 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:12.811 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:12.811 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:12.811 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_files = 6 2024-03-06T07:32:12.811 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:12.844 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:32:13.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.052+0000 7f1ee6200640 1 Processor -- start 2024-03-06T07:32:13.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.052+0000 7f1ee6200640 1 -- start start 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1ee6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee00b3bf0 0x7f1ee006cb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1ee6200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1ee006d0c0 0x7f1ee015c460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1ee6200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ee00b41b0 con 0x7f1ee0145680 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1ee6200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1ee00b2f60 con 0x7f1ee006d0c0 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1ee6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ee00ce380 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1edf400640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f1ee0145680 0x7f1ee00cf090 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:44282/0 (socket says 172.21.15.86:44282) 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1edf400640 1 -- 172.21.15.86:0/72941334 learned_addr learned my addr 172.21.15.86:0/72941334 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1edfe00640 1 --2- 172.21.15.86:0/72941334 >> v2:172.21.15.194:3300/0 conn(0x7f1ee006d0c0 0x7f1ee015c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.054+0000 7f1edea00640 1 --2- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee00b3bf0 0x7f1ee006cb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.055+0000 7f1ede000640 1 -- 172.21.15.86:0/72941334 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1575393647 0 0) 0x7f1ee00b41b0 con 0x7f1ee0145680 2024-03-06T07:32:13.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.055+0000 7f1ede000640 1 -- 172.21.15.86:0/72941334 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1ec4003520 con 0x7f1ee0145680 2024-03-06T07:32:13.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.055+0000 7f1edfe00640 1 -- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee00b3bf0 msgr2=0x7f1ee006cb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:13.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.055+0000 7f1edfe00640 1 --2- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee00b3bf0 0x7f1ee006cb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.055+0000 7f1edfe00640 1 -- 172.21.15.86:0/72941334 >> v1:172.21.15.194:6789/0 conn(0x7f1ee0145680 legacy=0x7f1ee00cf090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.055+0000 7f1edfe00640 1 -- 172.21.15.86:0/72941334 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ee00ce5b0 con 0x7f1ee006d0c0 2024-03-06T07:32:13.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.057+0000 7f1edfe00640 1 --2- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee015c460 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f1ed8009dc0 tx=0x7f1ed802be90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=94263ca2641711fb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.057+0000 7f1ede000640 1 -- 172.21.15.86:0/72941334 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ed8039070 con 0x7f1ee006d0c0 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.057+0000 7f1ede000640 1 -- 172.21.15.86:0/72941334 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ed802cc40 con 0x7f1ee006d0c0 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.057+0000 7f1ede000640 1 -- 172.21.15.86:0/72941334 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ed8034440 con 0x7f1ee006d0c0 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 -- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 msgr2=0x7f1ee015c460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 --2- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee015c460 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f1ed8009dc0 tx=0x7f1ed802be90 comp rx=0 tx=0).stop 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 -- 172.21.15.86:0/72941334 shutdown_connections 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 --2- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee015c460 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 --2- 172.21.15.86:0/72941334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee00b3bf0 0x7f1ee006cb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 -- 172.21.15.86:0/72941334 >> 172.21.15.86:0/72941334 conn(0x7f1ee00b1910 msgr2=0x7f1ee00b1cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 -- 172.21.15.86:0/72941334 shutdown_connections 2024-03-06T07:32:13.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.058+0000 7f1ee6200640 1 -- 172.21.15.86:0/72941334 wait complete. 2024-03-06T07:32:13.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.060+0000 7f1ee6200640 1 Processor -- start 2024-03-06T07:32:13.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.060+0000 7f1ee6200640 1 -- start start 2024-03-06T07:32:13.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1ee6200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee013b790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1ee6200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ee00b3bf0 0x7f1ee01407e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1ee6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee0145680 0x7f1ee0143cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1ee6200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ee00cf7a0 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1ee6200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ee00b4490 con 0x7f1ee006d0c0 2024-03-06T07:32:13.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1ee6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ee00ceb70 con 0x7f1ee0145680 2024-03-06T07:32:13.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ee00b3bf0 0x7f1ee01407e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ee00b3bf0 0x7f1ee01407e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45790/0 (socket says 172.21.15.86:45790) 2024-03-06T07:32:13.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 -- 172.21.15.86:0/769944637 learned_addr learned my addr 172.21.15.86:0/769944637 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:13.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edfe00640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee0145680 0x7f1ee0143cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edf400640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee013b790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 -- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee0145680 msgr2=0x7f1ee0143cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee0145680 0x7f1ee0143cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 -- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 msgr2=0x7f1ee013b790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee013b790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.061+0000 7f1edea00640 1 -- 172.21.15.86:0/769944637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ee013bcd0 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.062+0000 7f1edf400640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee013b790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.062+0000 7f1edea00640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ee00b3bf0 0x7f1ee01407e0 secure :-1 s=READY pgs=1103 cs=0 l=1 rev1=1 crypto rx=0x7f1ec800e900 tx=0x7f1ec800edc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.062+0000 7f1ed7400640 1 -- 172.21.15.86:0/769944637 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ec8017070 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.062+0000 7f1ed7400640 1 -- 172.21.15.86:0/769944637 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ec800ca00 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.062+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ee013d0f0 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.062+0000 7f1ed7400640 1 -- 172.21.15.86:0/769944637 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ec800cdb0 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.062+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ee013d990 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.063+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1ee0068050 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.064+0000 7f1ed7400640 1 -- 172.21.15.86:0/769944637 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1ec801a070 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.065+0000 7f1ed7400640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ebc06b4d0 0x7f1ebc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.065+0000 7f1ed7400640 1 -- 172.21.15.86:0/769944637 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1ec808a670 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.065+0000 7f1edf400640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ebc06b4d0 0x7f1ebc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.066+0000 7f1edf400640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ebc06b4d0 0x7f1ebc06d990 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f1ed00011e0 tx=0x7f1ed0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.070+0000 7f1ed7400640 1 -- 172.21.15.86:0/769944637 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1ec8059450 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.268+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f1ee013d480 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.269+0000 7f1ed7400640 1 -- 172.21.15.86:0/769944637 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v94) v1 ==== 100+0+11 (secure 0 0 0) 0x7f1ec8058d50 con 0x7f1ee00b3bf0 2024-03-06T07:32:13.270 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:32:13.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ebc06b4d0 msgr2=0x7f1ebc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ebc06b4d0 0x7f1ebc06d990 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f1ed00011e0 tx=0x7f1ed0009040 comp rx=0 tx=0).stop 2024-03-06T07:32:13.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ee00b3bf0 msgr2=0x7f1ee01407e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ee00b3bf0 0x7f1ee01407e0 secure :-1 s=READY pgs=1103 cs=0 l=1 rev1=1 crypto rx=0x7f1ec800e900 tx=0x7f1ec800edc0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 shutdown_connections 2024-03-06T07:32:13.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ebc06b4d0 0x7f1ebc06d990 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1ee0145680 0x7f1ee0143cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1ee00b3bf0 0x7f1ee01407e0 unknown :-1 s=CLOSED pgs=1103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 --2- 172.21.15.86:0/769944637 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1ee006d0c0 0x7f1ee013b790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.271+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 >> 172.21.15.86:0/769944637 conn(0x7f1ee00b1910 msgr2=0x7f1ee00b1e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:13.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.272+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 shutdown_connections 2024-03-06T07:32:13.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.272+0000 7f1ee6200640 1 -- 172.21.15.86:0/769944637 wait complete. 2024-03-06T07:32:13.291 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:13.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: chdir = "" 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: auth supported = cephx 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: filestore xattr use omap = true 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.459 INFO:teuthology.orchestra.run.smithi086.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: auth debug = true 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on old message = true 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on bug = true 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default size = 2 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.460 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow pg remap = true 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on too few osds = false 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: osd default data pool replay window = 5 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: mon allow pool delete = true 2024-03-06T07:32:13.461 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: mon cluster log file level = debug 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: debug asserts on shutdown = true 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: mon health detail to clog = false 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on skipped message = False 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout:[osd] 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: osd journal size = 100 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub max interval = 600 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: osd recover clone overlap = true 2024-03-06T07:32:13.462 INFO:teuthology.orchestra.run.smithi086.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug shutdown = true 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug op order = true 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug trim objects = true 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd open classes on start = true 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug pg log writeout = true 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: osd map max advance = 10 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: journal zero on create = true 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.463 INFO:teuthology.orchestra.run.smithi086.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: bdev debug aio = true 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug misdirected ops = true 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: bdev async discard = True 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: bdev enable discard = True 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore allocator = bitmap 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore block size = 96636764160 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore fsck on mount = True 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluefs = 1/20 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluestore = 1/20 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: debug osd = 20 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: debug rocksdb = 4/10 2024-03-06T07:32:13.464 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: osd objectstore = bluestore 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr] 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: debug mgr = 20 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:13.465 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout:[mon] 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: debug paxos = 20 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: mon data avail warn = 5 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd prime pg temp = true 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:13.466 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: # 1m isn't quite enough 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: mon op complaint time = 120 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout:[client] 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: rgw cache enabled = true 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable ops log = true 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable usage log = true 2024-03-06T07:32:13.467 INFO:teuthology.orchestra.run.smithi086.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout: client mount timeout = 600 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout: debug client = 20 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = "" 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.a] 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.b] 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.c] 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:32:13.468 INFO:teuthology.orchestra.run.smithi086.stdout:[mds] 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds = 20 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds balancer = 20 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug frag = True 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug scatterstat = True 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: mds op complaint time = 180 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: mds verify scatter = True 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_files = 6 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:13.469 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-06T07:32:13.474 INFO:teuthology.orchestra.run.smithi194.stdout:[global] 2024-03-06T07:32:13.474 INFO:teuthology.orchestra.run.smithi194.stdout: chdir = "" 2024-03-06T07:32:13.474 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:13.474 INFO:teuthology.orchestra.run.smithi194.stdout: auth supported = cephx 2024-03-06T07:32:13.474 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: filestore xattr use omap = true 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: auth debug = true 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on old message = true 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on bug = true 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:13.475 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default size = 2 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow pg remap = true 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on too few osds = false 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:13.476 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: osd default data pool replay window = 5 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: mon allow pool delete = true 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: mon cluster log file level = debug 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: debug asserts on shutdown = true 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: mon health detail to clog = false 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on skipped message = False 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout:[osd] 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: osd journal size = 100 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.477 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub max interval = 600 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd recover clone overlap = true 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug shutdown = true 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug op order = true 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug trim objects = true 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd open classes on start = true 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug pg log writeout = true 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.478 INFO:teuthology.orchestra.run.smithi194.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: osd map max advance = 10 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: journal zero on create = true 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: bdev debug aio = true 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug misdirected ops = true 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: bdev async discard = True 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: bdev enable discard = True 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore allocator = bitmap 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore block size = 96636764160 2024-03-06T07:32:13.479 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore fsck on mount = True 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluefs = 1/20 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluestore = 1/20 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: debug osd = 20 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: debug rocksdb = 4/10 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: osd objectstore = bluestore 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr] 2024-03-06T07:32:13.480 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: debug mgr = 20 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout:[mon] 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: debug paxos = 20 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: mon data avail warn = 5 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:13.481 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd prime pg temp = true 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: # 1m isn't quite enough 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:13.482 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: mon op complaint time = 120 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout:[client] 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: rgw cache enabled = true 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable ops log = true 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable usage log = true 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: client mount timeout = 600 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: debug client = 20 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = "" 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:13.483 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.a] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.b] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.c] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout:[mds] 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds = 20 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds balancer = 20 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug frag = True 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug scatterstat = True 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: mds op complaint time = 180 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: mds verify scatter = True 2024-03-06T07:32:13.484 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:13.485 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:13.485 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:13.485 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_files = 6 2024-03-06T07:32:13.485 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:13.485 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-06T07:32:13.520 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:32:13.521 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:13.521 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:32:13.521 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:13.521 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:32:13.522 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:13.522 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:32:13.522 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:13.539 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:32:13.539 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail d 2024-03-06T07:32:13.541 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:32:13.541 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail b 2024-03-06T07:32:13.545 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:32:13.546 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail c 2024-03-06T07:32:13.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.733+0000 7fa41ea00640 1 Processor -- start 2024-03-06T07:32:13.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.733+0000 7fa41ea00640 1 -- start start 2024-03-06T07:32:13.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.734+0000 7fa41ea00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41815ab60 0x7fa4180b4610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.734+0000 7fa41ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa41806e430 0x7fa41815af40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.734+0000 7fa41ea00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4180b3570 con 0x7fa41806e890 2024-03-06T07:32:13.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.735+0000 7fa41ea00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4180b3850 con 0x7fa41806e430 2024-03-06T07:32:13.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.735+0000 7fa41ea00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4180b3b30 con 0x7fa41815ab60 2024-03-06T07:32:13.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.735+0000 7fa40fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa41806e430 0x7fa41815af40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.735+0000 7fa40fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa41806e430 0x7fa41815af40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45802/0 (socket says 172.21.15.86:45802) 2024-03-06T07:32:13.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.735+0000 7fa40fe00640 1 -- 172.21.15.86:0/2675153680 learned_addr learned my addr 172.21.15.86:0/2675153680 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:13.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.735+0000 7fa41ca00640 1 --1- 172.21.15.86:0/2675153680 >> v1:172.21.15.86:6789/0 conn(0x7fa41806e890 0x7fa41806af40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55960/0 (socket says 172.21.15.86:55960) 2024-03-06T07:32:13.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.735+0000 7fa417e00640 1 --2- 172.21.15.86:0/2675153680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41815ab60 0x7fa4180b4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.736+0000 7fa40fe00640 1 -- 172.21.15.86:0/2675153680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41815ab60 msgr2=0x7fa4180b4610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:13.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.736+0000 7fa40fe00640 1 --2- 172.21.15.86:0/2675153680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41815ab60 0x7fa4180b4610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.736+0000 7fa40fe00640 1 -- 172.21.15.86:0/2675153680 >> v1:172.21.15.86:6789/0 conn(0x7fa41806e890 legacy=0x7fa41806af40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.736+0000 7fa417400640 1 -- 172.21.15.86:0/2675153680 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2381368191 0 0) 0x7fa4180b3570 con 0x7fa41806e890 2024-03-06T07:32:13.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.736+0000 7fa40fe00640 1 -- 172.21.15.86:0/2675153680 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4180b3e10 con 0x7fa41806e430 2024-03-06T07:32:13.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.737+0000 7fa40fe00640 1 --2- 172.21.15.86:0/2675153680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41806e430 0x7fa41815af40 secure :-1 s=READY pgs=1105 cs=0 l=1 rev1=1 crypto rx=0x7fa400009870 tx=0x7fa40002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea8e9ce9c7889aba server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.737+0000 7fa417400640 1 -- 172.21.15.86:0/2675153680 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa400034070 con 0x7fa41806e430 2024-03-06T07:32:13.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.737+0000 7fa417400640 1 -- 172.21.15.86:0/2675153680 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa400030840 con 0x7fa41806e430 2024-03-06T07:32:13.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.737+0000 7fa417400640 1 -- 172.21.15.86:0/2675153680 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa400030c10 con 0x7fa41806e430 2024-03-06T07:32:13.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 -- 172.21.15.86:0/2675153680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41806e430 msgr2=0x7fa41815af40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 --2- 172.21.15.86:0/2675153680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41806e430 0x7fa41815af40 secure :-1 s=READY pgs=1105 cs=0 l=1 rev1=1 crypto rx=0x7fa400009870 tx=0x7fa40002b980 comp rx=0 tx=0).stop 2024-03-06T07:32:13.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 -- 172.21.15.86:0/2675153680 shutdown_connections 2024-03-06T07:32:13.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 --2- 172.21.15.86:0/2675153680 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41806e430 0x7fa41815af40 unknown :-1 s=CLOSED pgs=1105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 --2- 172.21.15.86:0/2675153680 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41815ab60 0x7fa4180b4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 -- 172.21.15.86:0/2675153680 >> 172.21.15.86:0/2675153680 conn(0x7fa4180d0540 msgr2=0x7fa4180d0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:13.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 -- 172.21.15.86:0/2675153680 shutdown_connections 2024-03-06T07:32:13.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.738+0000 7fa41ea00640 1 -- 172.21.15.86:0/2675153680 wait complete. 2024-03-06T07:32:13.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.741+0000 7fa41ea00640 1 Processor -- start 2024-03-06T07:32:13.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.741+0000 7fa41ea00640 1 -- start start 2024-03-06T07:32:13.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ea00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa41806e430 0x7fa4180aef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ea00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41806e890 0x7fa4180cfc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41815ab60 0x7fa4180ce3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ea00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa41806f0e0 con 0x7fa41815ab60 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ea00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa418158470 con 0x7fa41806e430 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ea00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa41806f260 con 0x7fa41806e890 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41815ab60 0x7fa4180ce3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41815ab60 0x7fa4180ce3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45806/0 (socket says 172.21.15.86:45806) 2024-03-06T07:32:13.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.743+0000 7fa41ca00640 1 -- 172.21.15.86:0/18020747 learned_addr learned my addr 172.21.15.86:0/18020747 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:13.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa417e00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa41806e430 0x7fa4180aef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa40fe00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41806e890 0x7fa4180cfc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa41ca00640 1 -- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41806e890 msgr2=0x7fa4180cfc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa41ca00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41806e890 0x7fa4180cfc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa41ca00640 1 -- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa41806e430 msgr2=0x7fa4180aef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa41ca00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa41806e430 0x7fa4180aef70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa41ca00640 1 -- 172.21.15.86:0/18020747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4180ce910 con 0x7fa41815ab60 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa41ca00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41815ab60 0x7fa4180ce3d0 secure :-1 s=READY pgs=1106 cs=0 l=1 rev1=1 crypto rx=0x7fa410001560 tx=0x7fa41000cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa414c00640 1 -- 172.21.15.86:0/18020747 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa410018070 con 0x7fa41815ab60 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa414c00640 1 -- 172.21.15.86:0/18020747 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa41000a640 con 0x7fa41815ab60 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.744+0000 7fa414c00640 1 -- 172.21.15.86:0/18020747 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa41000aa10 con 0x7fa41815ab60 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.745+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4180cebd0 con 0x7fa41815ab60 2024-03-06T07:32:13.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.745+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4180cf190 con 0x7fa41815ab60 2024-03-06T07:32:13.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.746+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa418068070 con 0x7fa41815ab60 2024-03-06T07:32:13.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.747+0000 7fa414c00640 1 -- 172.21.15.86:0/18020747 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa41001b070 con 0x7fa41815ab60 2024-03-06T07:32:13.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.748+0000 7fa414c00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3f406b4d0 0x7fa3f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.748+0000 7fa414c00640 1 -- 172.21.15.86:0/18020747 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa4100898a0 con 0x7fa41815ab60 2024-03-06T07:32:13.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.748+0000 7fa417e00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3f406b4d0 0x7fa3f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.749+0000 7fa417e00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3f406b4d0 0x7fa3f406d990 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7fa408006fd0 tx=0x7fa408008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.757+0000 7fa414c00640 1 -- 172.21.15.86:0/18020747 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa410058680 con 0x7fa41815ab60 2024-03-06T07:32:13.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.769+0000 7f4769200640 1 Processor -- start 2024-03-06T07:32:13.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.769+0000 7f4769200640 1 -- start start 2024-03-06T07:32:13.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4769200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476415e980 0x7f476415ed60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4769200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f476415f2a0 0x7f47640b34d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4769200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f476406eff0 0x7f47640b3a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4769200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4764061e80 con 0x7f476406eff0 2024-03-06T07:32:13.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4769200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f476406e190 con 0x7f476415f2a0 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4769200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47640ccc30 con 0x7f476415e980 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4762800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f476406eff0 0x7f47640b3a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4762800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f476406eff0 0x7f47640b3a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45814/0 (socket says 172.21.15.86:45814) 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4762800640 1 -- 172.21.15.86:0/2476386525 learned_addr learned my addr 172.21.15.86:0/2476386525 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f475be00640 1 --2- 172.21.15.86:0/2476386525 >> v2:172.21.15.194:3300/0 conn(0x7f476415f2a0 0x7f47640b34d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.771+0000 7f4761e00640 1 --2- 172.21.15.86:0/2476386525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476415e980 0x7f476415ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.772+0000 7f4762800640 1 -- 172.21.15.86:0/2476386525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476415e980 msgr2=0x7f476415ed60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.772+0000 7f4762800640 1 --2- 172.21.15.86:0/2476386525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476415e980 0x7f476415ed60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.772+0000 7f4762800640 1 -- 172.21.15.86:0/2476386525 >> v2:172.21.15.194:3300/0 conn(0x7f476415f2a0 msgr2=0x7f47640b34d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.772+0000 7f4762800640 1 --2- 172.21.15.86:0/2476386525 >> v2:172.21.15.194:3300/0 conn(0x7f476415f2a0 0x7f47640b34d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.772+0000 7f4762800640 1 -- 172.21.15.86:0/2476386525 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f476406d5c0 con 0x7f476406eff0 2024-03-06T07:32:13.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.772+0000 7f4761e00640 1 --2- 172.21.15.86:0/2476386525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476415e980 0x7f476415ed60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:13.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.773+0000 7f4762800640 1 --2- 172.21.15.86:0/2476386525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476406eff0 0x7f47640b3a10 secure :-1 s=READY pgs=1107 cs=0 l=1 rev1=1 crypto rx=0x7f47640b31c0 tx=0x7f475c02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3a75700cfe49ca05 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.774+0000 7f4761400640 1 -- 172.21.15.86:0/2476386525 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f475c02ca80 con 0x7f476406eff0 2024-03-06T07:32:13.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.774+0000 7f4761400640 1 -- 172.21.15.86:0/2476386525 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f475c02cc20 con 0x7f476406eff0 2024-03-06T07:32:13.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.774+0000 7f4761400640 1 -- 172.21.15.86:0/2476386525 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f475c0343f0 con 0x7f476406eff0 2024-03-06T07:32:13.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 -- 172.21.15.86:0/2476386525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476406eff0 msgr2=0x7f47640b3a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 --2- 172.21.15.86:0/2476386525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476406eff0 0x7f47640b3a10 secure :-1 s=READY pgs=1107 cs=0 l=1 rev1=1 crypto rx=0x7f47640b31c0 tx=0x7f475c02bee0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 -- 172.21.15.86:0/2476386525 shutdown_connections 2024-03-06T07:32:13.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 --2- 172.21.15.86:0/2476386525 >> v2:172.21.15.194:3300/0 conn(0x7f476415f2a0 0x7f47640b34d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 --2- 172.21.15.86:0/2476386525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476406eff0 0x7f47640b3a10 unknown :-1 s=CLOSED pgs=1107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 --2- 172.21.15.86:0/2476386525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476415e980 0x7f476415ed60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 -- 172.21.15.86:0/2476386525 >> 172.21.15.86:0/2476386525 conn(0x7f47640d0540 msgr2=0x7f47640d0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:13.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 -- 172.21.15.86:0/2476386525 shutdown_connections 2024-03-06T07:32:13.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.775+0000 7f4769200640 1 -- 172.21.15.86:0/2476386525 wait complete. 2024-03-06T07:32:13.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.777+0000 7f4769200640 1 Processor -- start 2024-03-06T07:32:13.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.777+0000 7f4769200640 1 -- start start 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4769200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476406eff0 0x7f476414e3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4769200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476415e980 0x7f47641f51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4769200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f476415f2a0 0x7f47640ce0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4769200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f476406de20 con 0x7f476415e980 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4769200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f476406db40 con 0x7f476415f2a0 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4769200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47641547c0 con 0x7f476406eff0 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f475be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476415e980 0x7f47641f51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f475be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476415e980 0x7f47641f51c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45820/0 (socket says 172.21.15.86:45820) 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f475be00640 1 -- 172.21.15.86:0/3368181598 learned_addr learned my addr 172.21.15.86:0/3368181598 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:13.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f0410a00640 1 Processor -- start 2024-03-06T07:32:13.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4761e00640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476406eff0 0x7f476414e3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.778+0000 7f4762800640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f476415f2a0 0x7f47640ce0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475be00640 1 -- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476406eff0 msgr2=0x7f476414e3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475be00640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476406eff0 0x7f476414e3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475be00640 1 -- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f476415f2a0 msgr2=0x7f47640ce0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475be00640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f476415f2a0 0x7f47640ce0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475be00640 1 -- 172.21.15.86:0/3368181598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47641f5700 con 0x7f476415e980 2024-03-06T07:32:13.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f0410a00640 1 -- start start 2024-03-06T07:32:13.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475be00640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476415e980 0x7f47641f51c0 secure :-1 s=READY pgs=1108 cs=0 l=1 rev1=1 crypto rx=0x7f474c00d620 tx=0x7f474c00dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475a000640 1 -- 172.21.15.86:0/3368181598 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f474c0125f0 con 0x7f476415e980 2024-03-06T07:32:13.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475a000640 1 -- 172.21.15.86:0/3368181598 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f474c019070 con 0x7f476415e980 2024-03-06T07:32:13.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f475a000640 1 -- 172.21.15.86:0/3368181598 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f474c0129d0 con 0x7f476415e980 2024-03-06T07:32:13.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.779+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47640ccdb0 con 0x7f476415e980 2024-03-06T07:32:13.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.780+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47640cd330 con 0x7f476415e980 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.780+0000 7f0410a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 0x7f040c0d1050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.780+0000 7f0410a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f040c0d2040 con 0x7f040c0d4b30 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.780+0000 7f0410a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f040c0b3580 con 0x7f040c15e3b0 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.780+0000 7f0410a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f040c0d1df0 con 0x7f040c0d2d00 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.781+0000 7f040a800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f040c0d4b30 0x7f040c0b3d80 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55974/0 (socket says 172.21.15.86:55974) 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.781+0000 7f040a800640 1 -- 172.21.15.86:0/855565322 learned_addr learned my addr 172.21.15.86:0/855565322 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.781+0000 7f0409400640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 0x7f040c0d1050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.781+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f47640b5280 con 0x7f476415e980 2024-03-06T07:32:13.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.781+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3891921643 0 0) 0x7f040c0d2040 con 0x7f040c0d4b30 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.781+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f03f0003520 con 0x7f040c0d4b30 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2089258867 0 0) 0x7f03f0003520 con 0x7f040c0d4b30 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 msgr2=0x7f040c0d1050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 0x7f040c0d1050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 >> v1:172.21.15.194:6789/0 conn(0x7f040c15e3b0 legacy=0x7f040c0d25f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f040c0b3860 con 0x7f040c0d4b30 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f04040019f0 con 0x7f040c0d4b30 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 >> v1:172.21.15.86:6789/0 conn(0x7f040c0d4b30 legacy=0x7f040c0b3d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 0x7f03f00039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0003f20 0x7f03f0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03f0004d40 0x7f03f00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f040c0d1df0 con 0x7f03f0004d40 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03f000bb00 con 0x7f03f0003f20 2024-03-06T07:32:13.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03f000bd00 con 0x7f040c0d2d00 2024-03-06T07:32:13.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f475a000640 1 -- 172.21.15.86:0/3368181598 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f474c012b70 con 0x7f476415e980 2024-03-06T07:32:13.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.782+0000 7f040a800640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03f0004d40 0x7f03f00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f0409e00640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 0x7f03f00039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f0409400640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0003f20 0x7f03f0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f040a800640 1 -- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 msgr2=0x7f03f00039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f040a800640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0d2d00 0x7f03f00039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f040a800640 1 -- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0003f20 msgr2=0x7f03f0004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f040a800640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0003f20 0x7f03f0004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f040a800640 1 -- 172.21.15.86:0/855565322 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03f0003520 con 0x7f03f0004d40 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f475a000640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f474006b500 0x7f474006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.783+0000 7f475a000640 1 -- 172.21.15.86:0/3368181598 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f474c08a4f0 con 0x7f476415e980 2024-03-06T07:32:13.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.784+0000 7f4761e00640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f474006b500 0x7f474006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.784+0000 7f040a800640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03f0004d40 0x7f03f00051c0 secure :-1 s=READY pgs=1110 cs=0 l=1 rev1=1 crypto rx=0x7f040400a7a0 tx=0x7f03f0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ca47e2c541d8cfc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.784+0000 7f4761e00640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f474006b500 0x7f474006d9c0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f475400a990 tx=0x7f4754008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.792+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f040400a3f0 con 0x7f03f0004d40 2024-03-06T07:32:13.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.792+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0404034650 con 0x7f03f0004d40 2024-03-06T07:32:13.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.792+0000 7f0408a00640 1 -- 172.21.15.86:0/855565322 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0404034a80 con 0x7f03f0004d40 2024-03-06T07:32:13.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.792+0000 7f475a000640 1 -- 172.21.15.86:0/3368181598 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f47640b5280 con 0x7f476415e980 2024-03-06T07:32:13.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.794+0000 7f0410a00640 1 -- 172.21.15.86:0/855565322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03f0004d40 msgr2=0x7f03f00051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.794+0000 7f0410a00640 1 --2- 172.21.15.86:0/855565322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f03f0004d40 0x7f03f00051c0 secure :-1 s=READY pgs=1110 cs=0 l=1 rev1=1 crypto rx=0x7f040400a7a0 tx=0x7f03f0005830 comp rx=0 tx=0).stop 2024-03-06T07:32:13.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.794+0000 7f040a800640 1 -- 172.21.15.86:0/855565322 reap_dead start 2024-03-06T07:32:13.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.794+0000 7f0410a00640 1 -- 172.21.15.86:0/855565322 shutdown_connections 2024-03-06T07:32:13.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.794+0000 7f0410a00640 1 -- 172.21.15.86:0/855565322 >> 172.21.15.86:0/855565322 conn(0x7f040c0d4010 msgr2=0x7f040c0d4420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:13.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.794+0000 7f0410a00640 1 -- 172.21.15.86:0/855565322 shutdown_connections 2024-03-06T07:32:13.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.794+0000 7f0410a00640 1 -- 172.21.15.86:0/855565322 wait complete. 2024-03-06T07:32:13.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.796+0000 7f0410a00640 1 Processor -- start 2024-03-06T07:32:13.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.796+0000 7f0410a00640 1 -- start start 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f0410a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0004d40 0x7f040c15b600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f0410a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0b6c30 0x7f040c0cccd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f0410a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f040c1e3b80 0x7f040c1e3fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f0410a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f040c15e790 con 0x7f040c1e3b80 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f0410a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f040c0cff60 con 0x7f03f0004d40 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f0410a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f040c15e910 con 0x7f040c0b6c30 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f040a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f040c1e3b80 0x7f040c1e3fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f040a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f040c1e3b80 0x7f040c1e3fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:45838/0 (socket says 172.21.15.86:45838) 2024-03-06T07:32:13.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f040a800640 1 -- 172.21.15.86:0/4269754670 learned_addr learned my addr 172.21.15.86:0/4269754670 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.797+0000 7f0409e00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0004d40 0x7f040c15b600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f0409400640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0b6c30 0x7f040c0cccd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f040a800640 1 -- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0b6c30 msgr2=0x7f040c0cccd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f040a800640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0b6c30 0x7f040c0cccd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f040a800640 1 -- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0004d40 msgr2=0x7f040c15b600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f040a800640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0004d40 0x7f040c15b600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f040a800640 1 -- 172.21.15.86:0/4269754670 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f040c0b71a0 con 0x7f040c1e3b80 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f0409400640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0b6c30 0x7f040c0cccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f040a800640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f040c1e3b80 0x7f040c1e3fe0 secure :-1 s=READY pgs=1111 cs=0 l=1 rev1=1 crypto rx=0x7f040402cba0 tx=0x7f0404001cf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f0402000640 1 -- 172.21.15.86:0/4269754670 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f040400a3d0 con 0x7f040c1e3b80 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f0402000640 1 -- 172.21.15.86:0/4269754670 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0404044070 con 0x7f040c1e3b80 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f0402000640 1 -- 172.21.15.86:0/4269754670 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0404034a00 con 0x7f040c1e3b80 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.798+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f040c0b7480 con 0x7f040c1e3b80 2024-03-06T07:32:13.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.799+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f040c0b7a40 con 0x7f040c1e3b80 2024-03-06T07:32:13.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.800+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f040c068ea0 con 0x7f040c1e3b80 2024-03-06T07:32:13.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.801+0000 7f0402000640 1 -- 172.21.15.86:0/4269754670 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0404034ba0 con 0x7f040c1e3b80 2024-03-06T07:32:13.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.802+0000 7f0402000640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03ec06b550 0x7f03ec06da10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:13.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.802+0000 7f0402000640 1 -- 172.21.15.86:0/4269754670 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f04040ac6a0 con 0x7f040c1e3b80 2024-03-06T07:32:13.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.802+0000 7f0409e00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03ec06b550 0x7f03ec06da10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:13.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.803+0000 7f0409e00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03ec06b550 0x7f03ec06da10 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f03fc005540 tx=0x7f03fc007c50 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:13.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:13.807+0000 7f0402000640 1 -- 172.21.15.86:0/4269754670 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0404032380 con 0x7f040c1e3b80 2024-03-06T07:32:14.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.078+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "c"} v 0) v1 -- 0x7f040c0b4cf0 con 0x7f040c1e3b80 2024-03-06T07:32:14.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.098+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "d"} v 0) v1 -- 0x7fa418062550 con 0x7fa41815ab60 2024-03-06T07:32:14.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.114+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "b"} v 0) v1 -- 0x7f476415dfe0 con 0x7f476415e980 2024-03-06T07:32:14.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.403+0000 7f0402000640 1 -- 172.21.15.86:0/4269754670 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "c"}]=0 failed mds gid 6972 v60) v1 ==== 95+0+0 (secure 0 0 0) 0x7f040407b500 con 0x7f040c1e3b80 2024-03-06T07:32:14.405 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 6972 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03ec06b550 msgr2=0x7f03ec06da10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03ec06b550 0x7f03ec06da10 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f03fc005540 tx=0x7f03fc007c50 comp rx=0 tx=0).stop 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f040c1e3b80 msgr2=0x7f040c1e3fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f040c1e3b80 0x7f040c1e3fe0 secure :-1 s=READY pgs=1111 cs=0 l=1 rev1=1 crypto rx=0x7f040402cba0 tx=0x7f0404001cf0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 shutdown_connections 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f03ec06b550 0x7f03ec06da10 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f040c1e3b80 0x7f040c1e3fe0 unknown :-1 s=CLOSED pgs=1111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f040c0b6c30 0x7f040c0cccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 --2- 172.21.15.86:0/4269754670 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f03f0004d40 0x7f040c15b600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 >> 172.21.15.86:0/4269754670 conn(0x7f040c0d4010 msgr2=0x7f040c0b2cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:14.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.406+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 shutdown_connections 2024-03-06T07:32:14.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.407+0000 7f0410a00640 1 -- 172.21.15.86:0/4269754670 wait complete. 2024-03-06T07:32:14.425 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:32:14.424+0000 7f41a3400640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:32:14.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.429+0000 7fa414c00640 1 -- 172.21.15.86:0/18020747 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "d"}]=0 failed mds gid 5824 v61) v1 ==== 95+0+0 (secure 0 0 0) 0x7fa410057f80 con 0x7fa41815ab60 2024-03-06T07:32:14.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.429+0000 7f475a000640 1 -- 172.21.15.86:0/3368181598 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "b"}]=0 failed mds gid 6978 v61) v1 ==== 95+0+0 (secure 0 0 0) 0x7f47641f5700 con 0x7f476415e980 2024-03-06T07:32:14.431 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 6978 2024-03-06T07:32:14.431 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 5824 2024-03-06T07:32:14.432 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:32:14.433 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:32:14.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f474006b500 msgr2=0x7f474006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:14.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f474006b500 0x7f474006d9c0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f475400a990 tx=0x7f4754008040 comp rx=0 tx=0).stop 2024-03-06T07:32:14.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476415e980 msgr2=0x7f47641f51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:14.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476415e980 0x7f47641f51c0 secure :-1 s=READY pgs=1108 cs=0 l=1 rev1=1 crypto rx=0x7f474c00d620 tx=0x7f474c00dae0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 shutdown_connections 2024-03-06T07:32:14.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f474006b500 0x7f474006d9c0 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f476415f2a0 0x7f47640ce0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f476415e980 0x7f47641f51c0 unknown :-1 s=CLOSED pgs=1108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 --2- 172.21.15.86:0/3368181598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f476406eff0 0x7f476414e3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 >> 172.21.15.86:0/3368181598 conn(0x7f47640d0540 msgr2=0x7f476406af40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.432+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 shutdown_connections 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7f4769200640 1 -- 172.21.15.86:0/3368181598 wait complete. 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3f406b4d0 msgr2=0x7fa3f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3f406b4d0 0x7fa3f406d990 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7fa408006fd0 tx=0x7fa408008040 comp rx=0 tx=0).stop 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41815ab60 msgr2=0x7fa4180ce3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:14.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41815ab60 0x7fa4180ce3d0 secure :-1 s=READY pgs=1106 cs=0 l=1 rev1=1 crypto rx=0x7fa410001560 tx=0x7fa41000cc70 comp rx=0 tx=0).stop 2024-03-06T07:32:14.438 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:32:14.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 shutdown_connections 2024-03-06T07:32:14.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa3f406b4d0 0x7fa3f406d990 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa41815ab60 0x7fa4180ce3d0 unknown :-1 s=CLOSED pgs=1106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa41806e890 0x7fa4180cfc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 --2- 172.21.15.86:0/18020747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa41806e430 0x7fa4180aef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:14.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 >> 172.21.15.86:0/18020747 conn(0x7fa4180d0540 msgr2=0x7fa4180d0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:14.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 shutdown_connections 2024-03-06T07:32:14.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:14.433+0000 7fa41ea00640 1 -- 172.21.15.86:0/18020747 wait complete. 2024-03-06T07:32:14.439 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:14.456 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:32:14.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:32:14.457 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:32:14.457 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:32:14.464 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:32:14.464 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:14.470 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:32:14.470 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:14.583 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:32:14.619 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:32:14.620 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:32:16.474 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.a is failed for ~0s 2024-03-06T07:32:19.546 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:32:19.546 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail a 2024-03-06T07:32:19.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.761+0000 7f02cf200640 1 Processor -- start 2024-03-06T07:32:19.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.762+0000 7f02cf200640 1 -- start start 2024-03-06T07:32:19.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cf200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f02d015e230 0x7f02d00d0e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:19.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cf200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f02d00d2b10 0x7f02d00b4790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:19.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cf200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f02d00b39d0 con 0x7f02d00d13a0 2024-03-06T07:32:19.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cf200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f02d00b3780 con 0x7f02d015e230 2024-03-06T07:32:19.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cf200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f02d00b3cb0 con 0x7f02d00d2b10 2024-03-06T07:32:19.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cde00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f02d015e230 0x7f02d00d0e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:19.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cde00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f02d015e230 0x7f02d00d0e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37090/0 (socket says 172.21.15.86:37090) 2024-03-06T07:32:19.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.763+0000 7f02cde00640 1 -- 172.21.15.86:0/3129937217 learned_addr learned my addr 172.21.15.86:0/3129937217 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:19.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.764+0000 7f02cde00640 1 -- 172.21.15.86:0/3129937217 >> v2:172.21.15.194:3300/0 conn(0x7f02d00d2b10 msgr2=0x7f02d00b4790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:19.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.764+0000 7f02cde00640 1 --2- 172.21.15.86:0/3129937217 >> v2:172.21.15.194:3300/0 conn(0x7f02d00d2b10 0x7f02d00b4790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:19.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.764+0000 7f02cde00640 1 -- 172.21.15.86:0/3129937217 >> v1:172.21.15.194:6789/0 conn(0x7f02d00d13a0 legacy=0x7f02d00d2400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:19.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.764+0000 7f02cde00640 1 -- 172.21.15.86:0/3129937217 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02d00b3f90 con 0x7f02d015e230 2024-03-06T07:32:19.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.765+0000 7f02cde00640 1 --2- 172.21.15.86:0/3129937217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d015e230 0x7f02d00d0e60 secure :-1 s=READY pgs=1117 cs=0 l=1 rev1=1 crypto rx=0x7f02d00ce400 tx=0x7f02c002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4d944be84fb740e2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:19.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.765+0000 7f02cca00640 1 -- 172.21.15.86:0/3129937217 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02c0034070 con 0x7f02d015e230 2024-03-06T07:32:19.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.765+0000 7f02cca00640 1 -- 172.21.15.86:0/3129937217 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f02c0030860 con 0x7f02d015e230 2024-03-06T07:32:19.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.765+0000 7f02cca00640 1 -- 172.21.15.86:0/3129937217 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02c0030c30 con 0x7f02d015e230 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 -- 172.21.15.86:0/3129937217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d015e230 msgr2=0x7f02d00d0e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 --2- 172.21.15.86:0/3129937217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d015e230 0x7f02d00d0e60 secure :-1 s=READY pgs=1117 cs=0 l=1 rev1=1 crypto rx=0x7f02d00ce400 tx=0x7f02c002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 -- 172.21.15.86:0/3129937217 shutdown_connections 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 --2- 172.21.15.86:0/3129937217 >> v2:172.21.15.194:3300/0 conn(0x7f02d00d2b10 0x7f02d00b4790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 --2- 172.21.15.86:0/3129937217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d015e230 0x7f02d00d0e60 unknown :-1 s=CLOSED pgs=1117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 -- 172.21.15.86:0/3129937217 >> 172.21.15.86:0/3129937217 conn(0x7f02d00d3e20 msgr2=0x7f02d00d4230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 -- 172.21.15.86:0/3129937217 shutdown_connections 2024-03-06T07:32:19.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.766+0000 7f02cf200640 1 -- 172.21.15.86:0/3129937217 wait complete. 2024-03-06T07:32:19.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.768+0000 7f02cf200640 1 Processor -- start 2024-03-06T07:32:19.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.768+0000 7f02cf200640 1 -- start start 2024-03-06T07:32:19.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cf200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d00d13a0 0x7f02d00b8bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cf200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02d00d2b10 0x7f02d01f74e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02d015e230 0x7f02d01f7a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cf200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02d00d1d90 con 0x7f02d00d13a0 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cf200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02d015e610 con 0x7f02d00d2b10 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f02d00d1f10 con 0x7f02d015e230 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d00d13a0 0x7f02d00b8bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d00d13a0 0x7f02d00b8bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37092/0 (socket says 172.21.15.86:37092) 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 -- 172.21.15.86:0/1697156306 learned_addr learned my addr 172.21.15.86:0/1697156306 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 -- 172.21.15.86:0/1697156306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02d015e230 msgr2=0x7f02d01f7a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02d015e230 0x7f02d01f7a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 -- 172.21.15.86:0/1697156306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02d00d2b10 msgr2=0x7f02d01f74e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02d00d2b10 0x7f02d01f74e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:19.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.769+0000 7f02cde00640 1 -- 172.21.15.86:0/1697156306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02d00b7060 con 0x7f02d00d13a0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.770+0000 7f02cde00640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d00d13a0 0x7f02d00b8bf0 secure :-1 s=READY pgs=1118 cs=0 l=1 rev1=1 crypto rx=0x7f02c0000c00 tx=0x7f02c00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.770+0000 7f02c6000640 1 -- 172.21.15.86:0/1697156306 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02c0034040 con 0x7f02d00d13a0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.770+0000 7f02c6000640 1 -- 172.21.15.86:0/1697156306 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f02c0039d90 con 0x7f02d00d13a0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.770+0000 7f02c6000640 1 -- 172.21.15.86:0/1697156306 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02c0043630 con 0x7f02d00d13a0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.770+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f02d00b7340 con 0x7f02d00d13a0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.770+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02d00b7900 con 0x7f02d00d13a0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.771+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f02d004f330 con 0x7f02d00d13a0 2024-03-06T07:32:19.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.772+0000 7f02c6000640 1 -- 172.21.15.86:0/1697156306 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f02c0037070 con 0x7f02d00d13a0 2024-03-06T07:32:19.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.773+0000 7f02c6000640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02ac06b4d0 0x7f02ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:19.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.773+0000 7f02c6000640 1 -- 172.21.15.86:0/1697156306 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f02c00ac590 con 0x7f02d00d13a0 2024-03-06T07:32:19.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.773+0000 7f02cd400640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02ac06b4d0 0x7f02ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:19.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.774+0000 7f02cd400640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02ac06b4d0 0x7f02ac06d990 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f02d0161680 tx=0x7f02b8007d10 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:19.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:19.778+0000 7f02c6000640 1 -- 172.21.15.86:0/1697156306 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f02c003e070 con 0x7f02d00d13a0 2024-03-06T07:32:20.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.050+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "a"} v 0) v1 -- 0x7f02d00b0fa0 con 0x7f02d00d13a0 2024-03-06T07:32:20.052 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:32:20.051+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:32:20.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.054+0000 7f02c6000640 1 -- 172.21.15.86:0/1697156306 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "a"}]=0 failed mds gid 6969 v64) v1 ==== 95+0+0 (secure 0 0 0) 0x7f02c007b310 con 0x7f02d00d13a0 2024-03-06T07:32:20.055 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 6969 2024-03-06T07:32:20.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.056+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02ac06b4d0 msgr2=0x7f02ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.056+0000 7f02cf200640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02ac06b4d0 0x7f02ac06d990 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f02d0161680 tx=0x7f02b8007d10 comp rx=0 tx=0).stop 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.056+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d00d13a0 msgr2=0x7f02d00b8bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.056+0000 7f02cf200640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d00d13a0 0x7f02d00b8bf0 secure :-1 s=READY pgs=1118 cs=0 l=1 rev1=1 crypto rx=0x7f02c0000c00 tx=0x7f02c00046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.057+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 shutdown_connections 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.057+0000 7f02cf200640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02ac06b4d0 0x7f02ac06d990 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.057+0000 7f02cf200640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02d015e230 0x7f02d01f7a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.057+0000 7f02cf200640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02d00d2b10 0x7f02d01f74e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.057+0000 7f02cf200640 1 --2- 172.21.15.86:0/1697156306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02d00d13a0 0x7f02d00b8bf0 unknown :-1 s=CLOSED pgs=1118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.057+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 >> 172.21.15.86:0/1697156306 conn(0x7f02d00d3e20 msgr2=0x7f02d00ccf00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:20.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.057+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 shutdown_connections 2024-03-06T07:32:20.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.058+0000 7f02cf200640 1 -- 172.21.15.86:0/1697156306 wait complete. 2024-03-06T07:32:20.079 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:32:20.079 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:32:20.083 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:32:20.083 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:20.084 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:20.227 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:32:20.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.254+0000 7f17f5400640 1 Processor -- start 2024-03-06T07:32:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.254+0000 7f17f5400640 1 -- start start 2024-03-06T07:32:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.255+0000 7f17f5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00d0cf0 0x7f17f006c610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.255+0000 7f17f5400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f17f006cb50 0x7f17f00d1e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.255+0000 7f17f5400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17f00cf070 con 0x7f17f01617a0 2024-03-06T07:32:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.255+0000 7f17f5400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17f00b2120 con 0x7f17f006cb50 2024-03-06T07:32:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.255+0000 7f17f5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17f00cee10 con 0x7f17f00d0cf0 2024-03-06T07:32:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ee800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f17f01617a0 0x7f17f00ce700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36056/0 (socket says 172.21.15.86:36056) 2024-03-06T07:32:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ee800640 1 -- 172.21.15.86:0/2691851389 learned_addr learned my addr 172.21.15.86:0/2691851389 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3021952607 0 0) 0x7f17f00cf070 con 0x7f17f01617a0 2024-03-06T07:32:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f17d4003520 con 0x7f17f01617a0 2024-03-06T07:32:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ef200640 1 --2- 172.21.15.86:0/2691851389 >> v2:172.21.15.194:3300/0 conn(0x7f17f006cb50 0x7f17f00d1e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ede00640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00d0cf0 0x7f17f006c610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2834106591 0 0) 0x7f17d4003520 con 0x7f17f01617a0 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00d0cf0 msgr2=0x7f17f006c610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00d0cf0 0x7f17f006c610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 >> v2:172.21.15.194:3300/0 conn(0x7f17f006cb50 msgr2=0x7f17f00d1e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 --2- 172.21.15.86:0/2691851389 >> v2:172.21.15.194:3300/0 conn(0x7f17f006cb50 0x7f17f00d1e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17f00cf350 con 0x7f17f01617a0 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f17e0001670 con 0x7f17f01617a0 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 >> v1:172.21.15.86:6789/0 conn(0x7f17f01617a0 legacy=0x7f17f00ce700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17d40039b0 0x7f17d4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17d4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00d0cf0 0x7f17d40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17f00b2120 con 0x7f17d4004800 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17f00cee10 con 0x7f17d40039b0 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.256+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17d400ba70 con 0x7f17f00d0cf0 2024-03-06T07:32:20.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ede00640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17d4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:20.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ee800640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17d40039b0 0x7f17d4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:20.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ede00640 1 -- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00d0cf0 msgr2=0x7f17d40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:20.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ede00640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00d0cf0 0x7f17d40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ede00640 1 -- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17d40039b0 msgr2=0x7f17d4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:20.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ede00640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17d40039b0 0x7f17d4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ede00640 1 -- 172.21.15.86:0/2691851389 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17d4003520 con 0x7f17d4004800 2024-03-06T07:32:20.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.257+0000 7f17ee800640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17d40039b0 0x7f17d4003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:20.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.258+0000 7f17ede00640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17d4004c80 secure :-1 s=READY pgs=1122 cs=0 l=1 rev1=1 crypto rx=0x7f17dc00ba00 tx=0x7f17d4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ae623832ab28d629 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:20.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.258+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17dc036020 con 0x7f17d4004800 2024-03-06T07:32:20.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.258+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17dc032630 con 0x7f17d4004800 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.258+0000 7f17ed400640 1 -- 172.21.15.86:0/2691851389 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17dc00b600 con 0x7f17d4004800 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.259+0000 7f17f5400640 1 -- 172.21.15.86:0/2691851389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 msgr2=0x7f17d4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.259+0000 7f17f5400640 1 --2- 172.21.15.86:0/2691851389 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17d4004c80 secure :-1 s=READY pgs=1122 cs=0 l=1 rev1=1 crypto rx=0x7f17dc00ba00 tx=0x7f17d4005830 comp rx=0 tx=0).stop 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.259+0000 7f17ef200640 1 -- 172.21.15.86:0/2691851389 reap_dead start 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.259+0000 7f17f5400640 1 -- 172.21.15.86:0/2691851389 shutdown_connections 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.259+0000 7f17f5400640 1 -- 172.21.15.86:0/2691851389 >> 172.21.15.86:0/2691851389 conn(0x7f17f009aa30 msgr2=0x7f17f009ae40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.259+0000 7f17f5400640 1 -- 172.21.15.86:0/2691851389 shutdown_connections 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.259+0000 7f17f5400640 1 -- 172.21.15.86:0/2691851389 wait complete. 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.261+0000 7f17f5400640 1 Processor -- start 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.261+0000 7f17f5400640 1 -- start start 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17f5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17f0160880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17f5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00ce5d0 0x7f17f01f7560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17f5400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17f01f7aa0 0x7f17f01f3f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17f5400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17f0161b80 con 0x7f17d4004800 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17f5400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17f00b1db0 con 0x7f17f01f7aa0 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17f5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17f0161d00 con 0x7f17f00ce5d0 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17f0160880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17f0160880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37126/0 (socket says 172.21.15.86:37126) 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 -- 172.21.15.86:0/2096171694 learned_addr learned my addr 172.21.15.86:0/2096171694 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 -- 172.21.15.86:0/2096171694 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00ce5d0 msgr2=0x7f17f01f7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00ce5d0 0x7f17f01f7560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 -- 172.21.15.86:0/2096171694 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17f01f7aa0 msgr2=0x7f17f01f3f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17f01f7aa0 0x7f17f01f3f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.262+0000 7f17ee800640 1 -- 172.21.15.86:0/2096171694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17f015da40 con 0x7f17d4004800 2024-03-06T07:32:20.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.263+0000 7f17ee800640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17f0160880 secure :-1 s=READY pgs=1123 cs=0 l=1 rev1=1 crypto rx=0x7f17e000d900 tx=0x7f17e000ddc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:20.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.263+0000 7f17e6a00640 1 -- 172.21.15.86:0/2096171694 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17e0019070 con 0x7f17d4004800 2024-03-06T07:32:20.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.263+0000 7f17e6a00640 1 -- 172.21.15.86:0/2096171694 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17e00074e0 con 0x7f17d4004800 2024-03-06T07:32:20.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.263+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17f015dd20 con 0x7f17d4004800 2024-03-06T07:32:20.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.263+0000 7f17e6a00640 1 -- 172.21.15.86:0/2096171694 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17e00047e0 con 0x7f17d4004800 2024-03-06T07:32:20.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.263+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17f015e2e0 con 0x7f17d4004800 2024-03-06T07:32:20.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.264+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f17f01f7ec0 con 0x7f17d4004800 2024-03-06T07:32:20.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.265+0000 7f17e6a00640 1 -- 172.21.15.86:0/2096171694 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f17e001c070 con 0x7f17d4004800 2024-03-06T07:32:20.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.266+0000 7f17e6a00640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17d006b4d0 0x7f17d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:20.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.266+0000 7f17e6a00640 1 -- 172.21.15.86:0/2096171694 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f17e0089ea0 con 0x7f17d4004800 2024-03-06T07:32:20.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.266+0000 7f17ede00640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17d006b4d0 0x7f17d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:20.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.266+0000 7f17ede00640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17d006b4d0 0x7f17d006d990 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f17d40057e0 tx=0x7f17dc032000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:20.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.270+0000 7f17e6a00640 1 -- 172.21.15.86:0/2096171694 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f17e0058c60 con 0x7f17d4004800 2024-03-06T07:32:20.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.504+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f17f0068070 con 0x7f17d4004800 2024-03-06T07:32:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.506+0000 7f17e6a00640 1 -- 172.21.15.86:0/2096171694 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 65 v65) v1 ==== 94+0+4188 (secure 0 0 0) 0x7f17e0058560 con 0x7f17d4004800 2024-03-06T07:32:20.507 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:20.507 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":65,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63}],"filesystems":[{"mdsmap":{"epoch":65,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:20.058980+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:replay","state_seq":1,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":7}]} 2024-03-06T07:32:20.508 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 65 2024-03-06T07:32:20.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.508+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17d006b4d0 msgr2=0x7f17d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.508+0000 7f17f5400640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17d006b4d0 0x7f17d006d990 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f17d40057e0 tx=0x7f17dc032000 comp rx=0 tx=0).stop 2024-03-06T07:32:20.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.508+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 msgr2=0x7f17f0160880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:20.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.508+0000 7f17f5400640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17f0160880 secure :-1 s=READY pgs=1123 cs=0 l=1 rev1=1 crypto rx=0x7f17e000d900 tx=0x7f17e000ddc0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 shutdown_connections 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f17d006b4d0 0x7f17d006d990 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f17f01f7aa0 0x7f17f01f3f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f17f00ce5d0 0x7f17f01f7560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 --2- 172.21.15.86:0/2096171694 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f17d4004800 0x7f17f0160880 unknown :-1 s=CLOSED pgs=1123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 >> 172.21.15.86:0/2096171694 conn(0x7f17f009aa30 msgr2=0x7f17f006edc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 shutdown_connections 2024-03-06T07:32:20.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:20.509+0000 7f17f5400640 1 -- 172.21.15.86:0/2096171694 wait complete. 2024-03-06T07:32:20.929 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 65, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:32:03.747182+0000', 'modified': '2024-03-06T07:32:20.058980+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 98, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7164}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7164': {'gid': 7164, 'name': 'c', 'rank': 0, 'incarnation': 65, 'state': 'up:replay', 'state_seq': 1, 'addr': '172.21.15.86:6835/2542557324', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 2542557324}, {'type': 'v1', 'addr': '172.21.15.86:6835', 'nonce': 2542557324}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-06T07:32:20.929 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_7164:up:replay 2024-03-06T07:32:21.930 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:22.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.149+0000 7f9297e00640 1 Processor -- start 2024-03-06T07:32:22.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.149+0000 7f9297e00640 1 -- start start 2024-03-06T07:32:22.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9297e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9290160210 0x7f92900ceca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9297e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9290143de0 0x7f92900cf1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9297e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9290144f60 con 0x7f9290144b80 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9297e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f929006bc80 con 0x7f9290160210 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9297e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f929006bf60 con 0x7f9290143de0 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9294a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9290160210 0x7f92900ceca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9294a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9290160210 0x7f92900ceca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37130/0 (socket says 172.21.15.86:37130) 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9295400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9290144b80 0x7f929015fb00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36066/0 (socket says 172.21.15.86:36066) 2024-03-06T07:32:22.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.151+0000 7f9294a00640 1 -- 172.21.15.86:0/1829360055 learned_addr learned my addr 172.21.15.86:0/1829360055 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:22.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.152+0000 7f9295e00640 1 --2- 172.21.15.86:0/1829360055 >> v2:172.21.15.194:3300/0 conn(0x7f9290143de0 0x7f92900cf1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:22.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.152+0000 7f9294a00640 1 -- 172.21.15.86:0/1829360055 >> v2:172.21.15.194:3300/0 conn(0x7f9290143de0 msgr2=0x7f92900cf1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:22.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.152+0000 7f9294a00640 1 --2- 172.21.15.86:0/1829360055 >> v2:172.21.15.194:3300/0 conn(0x7f9290143de0 0x7f92900cf1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.152+0000 7f9294a00640 1 -- 172.21.15.86:0/1829360055 >> v1:172.21.15.86:6789/0 conn(0x7f9290144b80 legacy=0x7f929015fb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:22.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.152+0000 7f9294a00640 1 -- 172.21.15.86:0/1829360055 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f929006c240 con 0x7f9290160210 2024-03-06T07:32:22.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.152+0000 7f9295e00640 1 --2- 172.21.15.86:0/1829360055 >> v2:172.21.15.194:3300/0 conn(0x7f9290143de0 0x7f92900cf1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:22.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.152+0000 7f928fe00640 1 -- 172.21.15.86:0/1829360055 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3609503452 0 0) 0x7f9290144f60 con 0x7f9290144b80 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.153+0000 7f9294a00640 1 --2- 172.21.15.86:0/1829360055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290160210 0x7f92900ceca0 secure :-1 s=READY pgs=1125 cs=0 l=1 rev1=1 crypto rx=0x7f92900d15a0 tx=0x7f927c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=72e5116aee228807 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.153+0000 7f928fe00640 1 -- 172.21.15.86:0/1829360055 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f927c034070 con 0x7f9290160210 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.153+0000 7f928fe00640 1 -- 172.21.15.86:0/1829360055 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f927c030860 con 0x7f9290160210 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.153+0000 7f928fe00640 1 -- 172.21.15.86:0/1829360055 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f927c030c30 con 0x7f9290160210 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 -- 172.21.15.86:0/1829360055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290160210 msgr2=0x7f92900ceca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 --2- 172.21.15.86:0/1829360055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290160210 0x7f92900ceca0 secure :-1 s=READY pgs=1125 cs=0 l=1 rev1=1 crypto rx=0x7f92900d15a0 tx=0x7f927c02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 -- 172.21.15.86:0/1829360055 shutdown_connections 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 --2- 172.21.15.86:0/1829360055 >> v2:172.21.15.194:3300/0 conn(0x7f9290143de0 0x7f92900cf1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 --2- 172.21.15.86:0/1829360055 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290160210 0x7f92900ceca0 unknown :-1 s=CLOSED pgs=1125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 -- 172.21.15.86:0/1829360055 >> 172.21.15.86:0/1829360055 conn(0x7f92900d0540 msgr2=0x7f92900d0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 -- 172.21.15.86:0/1829360055 shutdown_connections 2024-03-06T07:32:22.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.154+0000 7f9297e00640 1 -- 172.21.15.86:0/1829360055 wait complete. 2024-03-06T07:32:22.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.156+0000 7f9297e00640 1 Processor -- start 2024-03-06T07:32:22.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.156+0000 7f9297e00640 1 -- start start 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9297e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9290143de0 0x7f9290140f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9297e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290144b80 0x7f929013e070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9297e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9290160210 0x7f929013e810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9297e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9290145970 con 0x7f9290144b80 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9297e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f929006c800 con 0x7f9290143de0 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9297e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f929006b920 con 0x7f9290160210 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290144b80 0x7f929013e070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290144b80 0x7f929013e070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37136/0 (socket says 172.21.15.86:37136) 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 -- 172.21.15.86:0/2604780597 learned_addr learned my addr 172.21.15.86:0/2604780597 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9295400640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9290143de0 0x7f9290140f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 -- 172.21.15.86:0/2604780597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9290160210 msgr2=0x7f929013e810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:22.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9290160210 0x7f929013e810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 -- 172.21.15.86:0/2604780597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9290143de0 msgr2=0x7f9290140f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9290143de0 0x7f9290140f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.157+0000 7f9294a00640 1 -- 172.21.15.86:0/2604780597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9290151290 con 0x7f9290144b80 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.158+0000 7f9294a00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290144b80 0x7f929013e070 secure :-1 s=READY pgs=1126 cs=0 l=1 rev1=1 crypto rx=0x7f927c02b9e0 tx=0x7f927c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.158+0000 7f928d600640 1 -- 172.21.15.86:0/2604780597 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f927c034040 con 0x7f9290144b80 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.158+0000 7f928d600640 1 -- 172.21.15.86:0/2604780597 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f927c039d10 con 0x7f9290144b80 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.158+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9290141570 con 0x7f9290144b80 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.158+0000 7f928d600640 1 -- 172.21.15.86:0/2604780597 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f927c042630 con 0x7f9290144b80 2024-03-06T07:32:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.158+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9290151530 con 0x7f9290144b80 2024-03-06T07:32:22.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.159+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9290068070 con 0x7f9290144b80 2024-03-06T07:32:22.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.160+0000 7f928d600640 1 -- 172.21.15.86:0/2604780597 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f927c037070 con 0x7f9290144b80 2024-03-06T07:32:22.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.161+0000 7f928d600640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f927006b4d0 0x7f927006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:22.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.161+0000 7f928d600640 1 -- 172.21.15.86:0/2604780597 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f927c0ac8d0 con 0x7f9290144b80 2024-03-06T07:32:22.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.161+0000 7f9295400640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f927006b4d0 0x7f927006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:22.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.162+0000 7f9295400640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f927006b4d0 0x7f927006d990 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f9280007fd0 tx=0x7f9280009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:22.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.165+0000 7f928d600640 1 -- 172.21.15.86:0/2604780597 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f927c07b570 con 0x7f9290144b80 2024-03-06T07:32:22.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.400+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9290061d20 con 0x7f9290144b80 2024-03-06T07:32:22.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.401+0000 7f928d600640 1 -- 172.21.15.86:0/2604780597 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 67 v67) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f927c07ae70 con 0x7f9290144b80 2024-03-06T07:32:22.403 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:22.403 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":67,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":67,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:21.078668+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:rejoin","state_seq":4,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":7}]} 2024-03-06T07:32:22.404 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 67 2024-03-06T07:32:22.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f927006b4d0 msgr2=0x7f927006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f927006b4d0 0x7f927006d990 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f9280007fd0 tx=0x7f9280009040 comp rx=0 tx=0).stop 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290144b80 msgr2=0x7f929013e070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290144b80 0x7f929013e070 secure :-1 s=READY pgs=1126 cs=0 l=1 rev1=1 crypto rx=0x7f927c02b9e0 tx=0x7f927c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 shutdown_connections 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f927006b4d0 0x7f927006d990 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9290160210 0x7f929013e810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9290144b80 0x7f929013e070 unknown :-1 s=CLOSED pgs=1126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 --2- 172.21.15.86:0/2604780597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9290143de0 0x7f9290140f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 >> 172.21.15.86:0/2604780597 conn(0x7f92900d0540 msgr2=0x7f92900b0f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 shutdown_connections 2024-03-06T07:32:22.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:22.404+0000 7f9297e00640 1 -- 172.21.15.86:0/2604780597 wait complete. 2024-03-06T07:32:22.826 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 67, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:32:03.747182+0000', 'modified': '2024-03-06T07:32:21.078668+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 98, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7164}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7164': {'gid': 7164, 'name': 'c', 'rank': 0, 'incarnation': 65, 'state': 'up:rejoin', 'state_seq': 4, 'addr': '172.21.15.86:6835/2542557324', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 2542557324}, {'type': 'v1', 'addr': '172.21.15.86:6835', 'nonce': 2542557324}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-06T07:32:22.826 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_7164:up:rejoin 2024-03-06T07:32:23.186 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.a has been restored 2024-03-06T07:32:23.815 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:32:23.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:32:23.827 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:24.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.058+0000 7f3218200640 1 Processor -- start 2024-03-06T07:32:24.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.058+0000 7f3218200640 1 -- start start 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.059+0000 7f3218200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f321015be90 0x7f32100b49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3218200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f321015fb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3218200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f32100b5160 con 0x7f32100acd60 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3218200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f32100b4f30 con 0x7f321015be90 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3218200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32100af240 con 0x7f32100d0780 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3215800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f321015be90 0x7f32100b49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3215800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f321015be90 0x7f32100b49f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37142/0 (socket says 172.21.15.86:37142) 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3215800640 1 -- 172.21.15.86:0/685431413 learned_addr learned my addr 172.21.15.86:0/685431413 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:24.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3216200640 1 --2- 172.21.15.86:0/685431413 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f321015fb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3215800640 1 -- 172.21.15.86:0/685431413 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 msgr2=0x7f321015fb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3215800640 1 --2- 172.21.15.86:0/685431413 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f321015fb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.060+0000 7f3215800640 1 -- 172.21.15.86:0/685431413 >> v1:172.21.15.194:6789/0 conn(0x7f32100acd60 legacy=0x7f32100d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.061+0000 7f3215800640 1 -- 172.21.15.86:0/685431413 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32100ae910 con 0x7f321015be90 2024-03-06T07:32:24.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.062+0000 7f3215800640 1 --2- 172.21.15.86:0/685431413 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f32100b49f0 secure :-1 s=READY pgs=1127 cs=0 l=1 rev1=1 crypto rx=0x7f3204009870 tx=0x7f320402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=32cfc5b8823d4840 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.062+0000 7f320fe00640 1 -- 172.21.15.86:0/685431413 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3204035020 con 0x7f321015be90 2024-03-06T07:32:24.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.062+0000 7f320fe00640 1 -- 172.21.15.86:0/685431413 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3204031640 con 0x7f321015be90 2024-03-06T07:32:24.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.062+0000 7f320fe00640 1 -- 172.21.15.86:0/685431413 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f320402d040 con 0x7f321015be90 2024-03-06T07:32:24.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 -- 172.21.15.86:0/685431413 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 msgr2=0x7f32100b49f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 --2- 172.21.15.86:0/685431413 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f32100b49f0 secure :-1 s=READY pgs=1127 cs=0 l=1 rev1=1 crypto rx=0x7f3204009870 tx=0x7f320402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 -- 172.21.15.86:0/685431413 shutdown_connections 2024-03-06T07:32:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 --2- 172.21.15.86:0/685431413 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f321015fb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 --2- 172.21.15.86:0/685431413 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f32100b49f0 unknown :-1 s=CLOSED pgs=1127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 -- 172.21.15.86:0/685431413 >> 172.21.15.86:0/685431413 conn(0x7f321006b360 msgr2=0x7f321006b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 -- 172.21.15.86:0/685431413 shutdown_connections 2024-03-06T07:32:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.063+0000 7f3218200640 1 -- 172.21.15.86:0/685431413 wait complete. 2024-03-06T07:32:24.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.065+0000 7f3218200640 1 Processor -- start 2024-03-06T07:32:24.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.065+0000 7f3218200640 1 -- start start 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3218200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32100acd60 0x7f32101f1940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3218200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f32100cbda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3218200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f3210156a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3218200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32100b2c80 con 0x7f321015be90 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3218200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32100aeed0 con 0x7f32100acd60 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3218200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32100b2e00 con 0x7f32100d0780 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f3210156a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f3210156a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37154/0 (socket says 172.21.15.86:37154) 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 -- 172.21.15.86:0/2894853173 learned_addr learned my addr 172.21.15.86:0/2894853173 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3215800640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32100acd60 0x7f32101f1940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3214e00640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f32100cbda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 -- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 msgr2=0x7f32100cbda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f32100cbda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 -- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32100acd60 msgr2=0x7f32101f1940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32100acd60 0x7f32101f1940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3216200640 1 -- 172.21.15.86:0/2894853173 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32100ce8f0 con 0x7f321015be90 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.066+0000 7f3214e00640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f32100cbda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.067+0000 7f3216200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f3210156a60 secure :-1 s=READY pgs=1128 cs=0 l=1 rev1=1 crypto rx=0x7f3208007d80 tx=0x7f320800e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.067+0000 7f320d600640 1 -- 172.21.15.86:0/2894853173 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3208014420 con 0x7f321015be90 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.067+0000 7f320d600640 1 -- 172.21.15.86:0/2894853173 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3208019070 con 0x7f321015be90 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.067+0000 7f320d600640 1 -- 172.21.15.86:0/2894853173 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3208014800 con 0x7f321015be90 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.067+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32100cebd0 con 0x7f321015be90 2024-03-06T07:32:24.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.067+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f32101587b0 con 0x7f321015be90 2024-03-06T07:32:24.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.068+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3210068070 con 0x7f321015be90 2024-03-06T07:32:24.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.069+0000 7f320d600640 1 -- 172.21.15.86:0/2894853173 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f32080149a0 con 0x7f321015be90 2024-03-06T07:32:24.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.070+0000 7f320d600640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31f006b4d0 0x7f31f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.070+0000 7f320d600640 1 -- 172.21.15.86:0/2894853173 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f320808aaa0 con 0x7f321015be90 2024-03-06T07:32:24.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.070+0000 7f3215800640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31f006b4d0 0x7f31f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.071+0000 7f3215800640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31f006b4d0 0x7f31f006d990 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f3204036000 tx=0x7f320403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.075+0000 7f320d600640 1 -- 172.21.15.86:0/2894853173 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f32080598e0 con 0x7f321015be90 2024-03-06T07:32:24.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.313+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3210156540 con 0x7f321015be90 2024-03-06T07:32:24.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.314+0000 7f320d600640 1 -- 172.21.15.86:0/2894853173 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f32080591e0 con 0x7f321015be90 2024-03-06T07:32:24.316 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:24.316 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:24.316 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31f006b4d0 msgr2=0x7f31f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31f006b4d0 0x7f31f006d990 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f3204036000 tx=0x7f320403c040 comp rx=0 tx=0).stop 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 msgr2=0x7f3210156a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f3210156a60 secure :-1 s=READY pgs=1128 cs=0 l=1 rev1=1 crypto rx=0x7f3208007d80 tx=0x7f320800e4b0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 shutdown_connections 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f31f006b4d0 0x7f31f006d990 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f321015be90 0x7f3210156a60 unknown :-1 s=CLOSED pgs=1128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f32100d0780 0x7f32100cbda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 --2- 172.21.15.86:0/2894853173 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f32100acd60 0x7f32101f1940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 >> 172.21.15.86:0/2894853173 conn(0x7f321006b360 msgr2=0x7f32101547f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 shutdown_connections 2024-03-06T07:32:24.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.317+0000 7f3218200640 1 -- 172.21.15.86:0/2894853173 wait complete. 2024-03-06T07:32:24.739 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 69, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:32:03.747182+0000', 'modified': '2024-03-06T07:32:24.075822+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 98, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7164}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7164': {'gid': 7164, 'name': 'c', 'rank': 0, 'incarnation': 65, 'state': 'up:active', 'state_seq': 6, 'addr': '172.21.15.86:6835/2542557324', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 2542557324}, {'type': 'v1', 'addr': '172.21.15.86:6835', 'nonce': 2542557324}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 7164, 'qdb_cluster': [7164]} 2024-03-06T07:32:24.740 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:32:24.740 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-06T07:32:24.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.957+0000 7f3aa9200640 1 Processor -- start 2024-03-06T07:32:24.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.958+0000 7f3aa9200640 1 -- start start 2024-03-06T07:32:24.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.959+0000 7f3aa9200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 0x7f3aa40b49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.959+0000 7f3aa9200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3aa40b5160 con 0x7f3aa40acd60 2024-03-06T07:32:24.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.959+0000 7f3aa9200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3aa40ae910 con 0x7f3aa40d0780 2024-03-06T07:32:24.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.959+0000 7f3aa9200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa40b4f30 con 0x7f3aa415be90 2024-03-06T07:32:24.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.959+0000 7f3aa1400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3aa40acd60 0x7f3aa40d0070 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36074/0 (socket says 172.21.15.86:36074) 2024-03-06T07:32:24.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.959+0000 7f3aa1400640 1 -- 172.21.15.86:0/1833437222 learned_addr learned my addr 172.21.15.86:0/1833437222 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:24.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 0x7f3aa40b49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3946851853 0 0) 0x7f3aa40b5160 con 0x7f3aa40acd60 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3a88003520 con 0x7f3aa40acd60 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2819167350 0 0) 0x7f3a88003520 con 0x7f3aa40acd60 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 msgr2=0x7f3aa40b49f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 0x7f3aa40b49f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 >> v1:172.21.15.194:6789/0 conn(0x7f3aa40d0780 legacy=0x7f3aa415fb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3aa40aebf0 con 0x7f3aa40acd60 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3a8c001670 con 0x7f3aa40acd60 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 >> v1:172.21.15.86:6789/0 conn(0x7f3aa40acd60 legacy=0x7f3aa40d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 0x7f3a880039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a88003f60 0x7f3a88004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a88004d40 0x7f3a880051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a8800ba70 con 0x7f3a88003f60 2024-03-06T07:32:24.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.960+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a8800bc20 con 0x7f3a88004d40 2024-03-06T07:32:24.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3a8800be20 con 0x7f3aa415be90 2024-03-06T07:32:24.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a88003f60 0x7f3a88004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 0x7f3a880039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa1400640 1 -- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 msgr2=0x7f3a880039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3aa415be90 0x7f3a880039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa1400640 1 -- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a88004d40 msgr2=0x7f3a880051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a88004d40 0x7f3a880051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.961+0000 7f3aa1400640 1 -- 172.21.15.86:0/1833437222 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a88003520 con 0x7f3a88003f60 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.963+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a88003f60 0x7f3a88004800 secure :-1 s=READY pgs=1130 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c00c240 tx=0x7f3a88005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7b84b1249b02eaa4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.963+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a8c007cf0 con 0x7f3a88003f60 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.963+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a8c030070 con 0x7f3a88003f60 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.963+0000 7f3aa0a00640 1 -- 172.21.15.86:0/1833437222 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a8c005d30 con 0x7f3a88003f60 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.964+0000 7f3aa9200640 1 -- 172.21.15.86:0/1833437222 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a88003f60 msgr2=0x7f3a88004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.964+0000 7f3aa9200640 1 --2- 172.21.15.86:0/1833437222 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a88003f60 0x7f3a88004800 secure :-1 s=READY pgs=1130 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c00c240 tx=0x7f3a88005830 comp rx=0 tx=0).stop 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.964+0000 7f3aa2800640 1 -- 172.21.15.86:0/1833437222 reap_dead start 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.964+0000 7f3aa9200640 1 -- 172.21.15.86:0/1833437222 shutdown_connections 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.964+0000 7f3aa9200640 1 -- 172.21.15.86:0/1833437222 >> 172.21.15.86:0/1833437222 conn(0x7f3aa406b360 msgr2=0x7f3aa406b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.964+0000 7f3aa9200640 1 -- 172.21.15.86:0/1833437222 shutdown_connections 2024-03-06T07:32:24.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.964+0000 7f3aa9200640 1 -- 172.21.15.86:0/1833437222 wait complete. 2024-03-06T07:32:24.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.966+0000 7f3aa9200640 1 Processor -- start 2024-03-06T07:32:24.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.966+0000 7f3aa9200640 1 -- start start 2024-03-06T07:32:24.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa9200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a88003f60 0x7f3aa40b02e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa9200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa4156990 0x7f3aa40cd180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa4152790 0x7f3aa40b18f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa9200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa40b2c80 con 0x7f3aa4152790 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa9200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa40af240 con 0x7f3aa4156990 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa9200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa40b2e00 con 0x7f3a88003f60 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa4152790 0x7f3aa40b18f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa4152790 0x7f3aa40b18f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37166/0 (socket says 172.21.15.86:37166) 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa2800640 1 -- 172.21.15.86:0/1212124410 learned_addr learned my addr 172.21.15.86:0/1212124410 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa4156990 0x7f3aa40cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a88003f60 0x7f3aa40b02e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa2800640 1 -- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a88003f60 msgr2=0x7f3aa40b02e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa2800640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a88003f60 0x7f3aa40b02e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa2800640 1 -- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa4156990 msgr2=0x7f3aa40cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.967+0000 7f3aa2800640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa4156990 0x7f3aa40cd180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3aa2800640 1 -- 172.21.15.86:0/1212124410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3aa40b0ba0 con 0x7f3aa4152790 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa4156990 0x7f3aa40cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a88003f60 0x7f3aa40b02e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3aa2800640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa4152790 0x7f3aa40b18f0 secure :-1 s=READY pgs=1131 cs=0 l=1 rev1=1 crypto rx=0x7f3aa4072f90 tx=0x7f3a9c00cb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3a9a000640 1 -- 172.21.15.86:0/1212124410 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a9c017070 con 0x7f3aa4152790 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3a9a000640 1 -- 172.21.15.86:0/1212124410 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a9c005d60 con 0x7f3aa4152790 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3a9a000640 1 -- 172.21.15.86:0/1212124410 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a9c00a850 con 0x7f3aa4152790 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3aa40b08c0 con 0x7f3aa4152790 2024-03-06T07:32:24.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.968+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3aa41542d0 con 0x7f3aa4152790 2024-03-06T07:32:24.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.972+0000 7f3a9a000640 1 -- 172.21.15.86:0/1212124410 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3a9c01a070 con 0x7f3aa4152790 2024-03-06T07:32:24.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.972+0000 7f3aabfb8740 1 Processor -- start 2024-03-06T07:32:24.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.972+0000 7f3aabfb8740 1 -- start start 2024-03-06T07:32:24.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.973+0000 7f3a9a000640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a8006b500 0x7f3a8006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.973+0000 7f3a9a000640 1 -- 172.21.15.86:0/1212124410 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f3a9c089520 con 0x7f3aa4152790 2024-03-06T07:32:24.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.973+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a8006b500 0x7f3a8006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.973+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c587750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.973+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b4b90 0x55e75c5b7540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.973+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c587c90 0x55e75c5b7a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e75c4f1600 con 0x55e75c5b4b90 2024-03-06T07:32:24.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e75c49a930 con 0x55e75c587c90 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55e75c4ed6a0 con 0x55e75c5b47b0 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b4b90 0x55e75c5b7540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b4b90 0x55e75c5b7540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37172/0 (socket says 172.21.15.86:37172) 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 -- 172.21.15.86:0/3008396758 learned_addr learned my addr 172.21.15.86:0/3008396758 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a8006b500 0x7f3a8006d9c0 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c034000 tx=0x7f3a8c007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa2800640 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c587c90 0x55e75c5b7a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c587750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 -- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 msgr2=0x55e75c587750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c587750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 -- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c587c90 msgr2=0x55e75c5b7a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c587c90 0x55e75c5b7a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.974+0000 7f3aa1400640 1 -- 172.21.15.86:0/3008396758 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e75c2062d0 con 0x55e75c5b4b90 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.975+0000 7f3aa1400640 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b4b90 0x55e75c5b7540 secure :-1 s=READY pgs=1132 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c00c330 tx=0x7f3a8c009030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6ef89b814ba53bc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.975+0000 7f3a92a00640 1 -- 172.21.15.86:0/3008396758 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a8c050070 con 0x55e75c5b4b90 2024-03-06T07:32:24.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.975+0000 7f3a92a00640 1 -- 172.21.15.86:0/3008396758 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a8c030040 con 0x55e75c5b4b90 2024-03-06T07:32:24.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.975+0000 7f3a92a00640 1 -- 172.21.15.86:0/3008396758 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a8c0072b0 con 0x55e75c5b4b90 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 -- 172.21.15.86:0/3008396758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b4b90 msgr2=0x55e75c5b7540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b4b90 0x55e75c5b7540 secure :-1 s=READY pgs=1132 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c00c330 tx=0x7f3a8c009030 comp rx=0 tx=0).stop 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 -- 172.21.15.86:0/3008396758 shutdown_connections 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c587c90 0x55e75c5b7a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b4b90 0x55e75c5b7540 unknown :-1 s=CLOSED pgs=1132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/3008396758 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c587750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 -- 172.21.15.86:0/3008396758 >> 172.21.15.86:0/3008396758 conn(0x55e75c5899c0 msgr2=0x55e75c5d2540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 -- 172.21.15.86:0/3008396758 shutdown_connections 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.976+0000 7f3aabfb8740 1 -- 172.21.15.86:0/3008396758 wait complete. 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.977+0000 7f3aabfb8740 1 -- start start 2024-03-06T07:32:24.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.977+0000 7f3aabfb8740 1 Processor -- start 2024-03-06T07:32:24.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c5b1200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5b9a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b38e0 0x55e75c5b9f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e75c58bf60 con 0x55e75c5b38e0 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e75c4add90 con 0x55e75c5b4b90 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55e75c58b350 con 0x55e75c5b47b0 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aabfb8740 10 client.0 _ll_register_callbacks cb 0x55e75c4e9350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aa2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b38e0 0x55e75c5b9f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aa2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b38e0 0x55e75c5b9f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37186/0 (socket says 172.21.15.86:37186) 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.978+0000 7f3aa2800640 1 -- 172.21.15.86:0/2379138649 learned_addr learned my addr 172.21.15.86:0/2379138649 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa1400640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5b9a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa2800640 1 -- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 msgr2=0x55e75c5b1200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c5b1200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa2800640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c5b1200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa2800640 1 -- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 msgr2=0x55e75c5b9a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa2800640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5b9a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa2800640 1 -- 172.21.15.86:0/2379138649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55e75c2d06b0 con 0x55e75c5b38e0 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa1400640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5b9a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:24.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aa2800640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b38e0 0x55e75c5b9f60 secure :-1 s=READY pgs=1133 cs=0 l=1 rev1=1 crypto rx=0x7f3a9c0578e0 tx=0x7f3a9c057600 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb6508253e630d49 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3a87400640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3a87400640 1 -- 172.21.15.86:0/2379138649 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a9c06bda0 con 0x55e75c5b38e0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3a87400640 1 -- 172.21.15.86:0/2379138649 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f3a9c089070 con 0x55e75c5b38e0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3a8c030040 con 0x55e75c5b38e0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3a87400640 1 -- 172.21.15.86:0/2379138649 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=69) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3a9c087050 con 0x55e75c5b38e0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aabfb8740 10 client.7218 fetch_fsmap learned FSMap version 69 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.979+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=69,monmap=2+}) v3 -- 0x55e75c5ae020 con 0x55e75c5b38e0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3a87400640 1 -- 172.21.15.86:0/2379138649 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a8c030040 con 0x55e75c5b38e0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3a87400640 1 -- 172.21.15.86:0/2379138649 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 69) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f3a9c082020 con 0x55e75c5b38e0 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3aabfb8740 10 client.7218 fetch_fsmap finished waiting for FSMap version 69 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3aabfb8740 10 client.7218 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3aabfb8740 20 client.7218 populate_metadata read hostname 'smithi086' 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c5aee40 0x55e75c5e30b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3aabfb8740 4 client.7218 mds_command: new command op to 7164 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:32:24.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 --> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e75c4feb40 con 0x55e75c5aee40 2024-03-06T07:32:24.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.980+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c5aee40 0x55e75c5e30b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:24.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.981+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c5aee40 0x55e75c5e30b0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f50132b9ef51a537 server_cookie=f13f2ae90f3b6bf2 in_seq=0 out_seq=0 2024-03-06T07:32:24.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.981+0000 7f3a87400640 10 client.7218 ms_handle_connect on v2:172.21.15.86:6834/2542557324 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3a87400640 1 -- 172.21.15.86:0/2379138649 <== mds.0 v2:172.21.15.86:6834/2542557324 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x55e75c4feb40 con 0x55e75c5aee40 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3a87400640 10 client.7218 handle_command_reply: tid=0 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 client.7218 shutdown 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b38e0 msgr2=0x55e75c5b9f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b38e0 0x55e75c5b9f60 secure :-1 s=READY pgs=1133 cs=0 l=1 rev1=1 crypto rx=0x7f3a9c0578e0 tx=0x7f3a9c057600 comp rx=0 tx=0).stop 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 shutdown_connections 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c5aee40 0x55e75c5e30b0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b38e0 0x55e75c5b9f60 unknown :-1 s=CLOSED pgs=1133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5b9a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2379138649 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b47b0 0x55e75c5b1200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:24.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 >> 172.21.15.86:0/2379138649 conn(0x55e75c587c90 msgr2=0x55e75c586eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:24.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3a87400640 0 client.7218 ms_handle_reset on v2:172.21.15.86:6834/2542557324 2024-03-06T07:32:24.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 shutdown_connections 2024-03-06T07:32:24.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2379138649 wait complete. 2024-03-06T07:32:24.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:24.982+0000 7f3aabfb8740 20 client.7218 trim_cache size 0 max 16384 2024-03-06T07:32:25.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.001+0000 7f3aabfb8740 1 Processor -- start 2024-03-06T07:32:25.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.001+0000 7f3aabfb8740 1 -- start start 2024-03-06T07:32:25.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 0x55e75c58e980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:25.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5b9e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:25.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5d57a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:25.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa41ffe20 con 0x55e75c5b47b0 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa41ff220 con 0x55e75c5b4b90 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3aa41ff820 con 0x55e75c5b38e0 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aa1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5b9e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aa1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5b9e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37194/0 (socket says 172.21.15.86:37194) 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.002+0000 7f3aa1400640 1 -- 172.21.15.86:0/2157429482 learned_addr learned my addr 172.21.15.86:0/2157429482 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3aa1400640 1 -- 172.21.15.86:0/2157429482 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 msgr2=0x55e75c58e980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3aa1400640 1 --2- 172.21.15.86:0/2157429482 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 0x55e75c58e980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3aa1400640 1 -- 172.21.15.86:0/2157429482 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 msgr2=0x55e75c5d57a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3aa1400640 1 --2- 172.21.15.86:0/2157429482 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5d57a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3aa1400640 1 -- 172.21.15.86:0/2157429482 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e75c5b0f10 con 0x55e75c5b47b0 2024-03-06T07:32:25.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3aa1400640 1 --2- 172.21.15.86:0/2157429482 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5b9e90 secure :-1 s=READY pgs=1134 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c044000 tx=0x7f3a8c03e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=88a037217a886bdf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3a92000640 1 -- 172.21.15.86:0/2157429482 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a8c050040 con 0x55e75c5b47b0 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3a92000640 1 -- 172.21.15.86:0/2157429482 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a8c05b070 con 0x55e75c5b47b0 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.003+0000 7f3a92000640 1 -- 172.21.15.86:0/2157429482 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a8c007290 con 0x55e75c5b47b0 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2157429482 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 msgr2=0x55e75c5b9e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2157429482 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5b9e90 secure :-1 s=READY pgs=1134 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c044000 tx=0x7f3a8c03e000 comp rx=0 tx=0).stop 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2157429482 shutdown_connections 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2157429482 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5d57a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2157429482 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5b9e90 unknown :-1 s=CLOSED pgs=1134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/2157429482 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 0x55e75c58e980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2157429482 >> 172.21.15.86:0/2157429482 conn(0x55e75c587c90 msgr2=0x55e75c58d900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2157429482 shutdown_connections 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.004+0000 7f3aabfb8740 1 -- 172.21.15.86:0/2157429482 wait complete. 2024-03-06T07:32:25.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.005+0000 7f3aabfb8740 1 -- start start 2024-03-06T07:32:25.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.005+0000 7f3aabfb8740 1 Processor -- start 2024-03-06T07:32:25.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 0x55e75c5b8c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:25.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5d57a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:25.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aabfb8740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5d5ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:25.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e75c5aecc0 con 0x55e75c5b47b0 2024-03-06T07:32:25.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e75c5af660 con 0x55e75c5b4b90 2024-03-06T07:32:25.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aabfb8740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55e75c5e2bd0 con 0x55e75c5b38e0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aabfb8740 10 client.0 _ll_register_callbacks cb 0x7f3a9c001610 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aa1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5d57a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aa1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5d57a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:37196/0 (socket says 172.21.15.86:37196) 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.006+0000 7f3aa1400640 1 -- 172.21.15.86:0/1273152170 learned_addr learned my addr 172.21.15.86:0/1273152170 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3aa1400640 1 -- 172.21.15.86:0/1273152170 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 msgr2=0x55e75c5b8c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 0x55e75c5b8c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3aa1400640 1 -- 172.21.15.86:0/1273152170 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 msgr2=0x55e75c5d5ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5d5ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3aa1400640 1 -- 172.21.15.86:0/1273152170 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55e75c58c370 con 0x55e75c5b47b0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3aa1400640 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5d57a0 secure :-1 s=READY pgs=1135 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c00c210 tx=0x7f3a8c02e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=60e8f3fe38332d12 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3a87400640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3a87400640 1 -- 172.21.15.86:0/1273152170 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a8c050040 con 0x55e75c5b47b0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3a87400640 1 -- 172.21.15.86:0/1273152170 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f3a8c007270 con 0x55e75c5b47b0 2024-03-06T07:32:25.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.007+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3a8c05b070 con 0x55e75c5b47b0 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3a87400640 1 -- 172.21.15.86:0/1273152170 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=69) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3a8c06f050 con 0x55e75c5b47b0 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 10 client.7224 fetch_fsmap learned FSMap version 69 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=69,monmap=2+}) v3 -- 0x55e75c5ba450 con 0x55e75c5b47b0 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3a87400640 1 -- 172.21.15.86:0/1273152170 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a8c05b070 con 0x55e75c5b47b0 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3a87400640 1 -- 172.21.15.86:0/1273152170 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 69) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f3a8c072020 con 0x55e75c5b47b0 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 10 client.7224 fetch_fsmap finished waiting for FSMap version 69 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 10 client.7224 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 20 client.7224 populate_metadata read hostname 'smithi086' 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c64a510 0x55e75c64a8f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 4 client.7224 mds_command: new command op to 7164 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 --> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55e75c5b25b0 con 0x55e75c64a510 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.008+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c64a510 0x55e75c64a8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:25.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3a87400640 10 client.7224 ms_handle_connect on v2:172.21.15.86:6834/2542557324 2024-03-06T07:32:25.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aa1e00640 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c64a510 0x55e75c64a8f0 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cc21a6edba292b07 server_cookie=462ab071a0a263e1 in_seq=0 out_seq=0 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3a87400640 1 -- 172.21.15.86:0/1273152170 <== mds.0 v2:172.21.15.86:6834/2542557324 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+325 (crc 0 0 0) 0x55e75c5b25b0 con 0x55e75c64a510 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3a87400640 10 client.7224 handle_command_reply: tid=0 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 client.7224 shutdown 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 msgr2=0x55e75c5d57a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5d57a0 secure :-1 s=READY pgs=1135 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c00c210 tx=0x7f3a8c02e070 comp rx=0 tx=0).stop 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 shutdown_connections 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:6834/2542557324,v1:172.21.15.86:6835/2542557324] conn(0x55e75c64a510 0x55e75c64a8f0 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e75c5b4b90 0x55e75c5d5ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e75c5b47b0 0x55e75c5d57a0 unknown :-1 s=CLOSED pgs=1135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 --2- 172.21.15.86:0/1273152170 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e75c5b38e0 0x55e75c5b8c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3a87400640 0 client.7224 ms_handle_reset on v2:172.21.15.86:6834/2542557324 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.009+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 >> 172.21.15.86:0/1273152170 conn(0x55e75c587c90 msgr2=0x55e75c5ba7a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 shutdown_connections 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aabfb8740 1 -- 172.21.15.86:0/1273152170 wait complete. 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aabfb8740 20 client.7224 trim_cache size 0 max 16384 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 7164, 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:32:25.015 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:32:25.016 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 4.9460196229999998, 2024-03-06T07:32:25.016 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 69, 2024-03-06T07:32:25.016 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 98, 2024-03-06T07:32:25.016 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 98, 2024-03-06T07:32:25.016 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 10.422806117 2024-03-06T07:32:25.016 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a8006b500 msgr2=0x7f3a8006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a8006b500 0x7f3a8006d9c0 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f3a8c034000 tx=0x7f3a8c007000 comp rx=0 tx=0).stop 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa4152790 msgr2=0x7f3aa40b18f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa4152790 0x7f3aa40b18f0 secure :-1 s=READY pgs=1131 cs=0 l=1 rev1=1 crypto rx=0x7f3aa4072f90 tx=0x7f3a9c00cb90 comp rx=0 tx=0).stop 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 shutdown_connections 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a8006b500 0x7f3a8006d9c0 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3aa4152790 0x7f3aa40b18f0 unknown :-1 s=CLOSED pgs=1131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3aa4156990 0x7f3aa40cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 --2- 172.21.15.86:0/1212124410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a88003f60 0x7f3aa40b02e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.010+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 >> 172.21.15.86:0/1212124410 conn(0x7f3aa406b360 msgr2=0x7f3aa4070b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.011+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 shutdown_connections 2024-03-06T07:32:25.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:25.011+0000 7f3aa9200640 1 -- 172.21.15.86:0/1212124410 wait complete. 2024-03-06T07:32:25.036 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:32:25.036 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:32:25.036 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:32:25.036 DEBUG:teuthology.orchestra.run.smithi086:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> subdir = "delete_me" 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> size_unit = 1024 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> file_multiplier = 200 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> os.mkdir(os.path.join(mountpoint, subdir)) 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> for i in range(0, file_multiplier): 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> for size in range(0, 16*size_unit, size_unit): 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> filename = "{0}_{1}.bin".format(i, size // size_unit) 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> with open(os.path.join(mountpoint, subdir, filename), '"'"'w'"'"') as f: 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> f.write(size * '"'"'x'"'"') 2024-03-06T07:32:25.037 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:32:25.460 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:32:25.458+0000 7f9bb7200640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:32:29.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:32:29.232 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json config get mds_max_purge_ops 2024-03-06T07:32:29.232 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:29.232 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -rf delete_me) 2024-03-06T07:32:29.484 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_max_purge_ops":"100000000"} 2024-03-06T07:32:29.485 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_max_purge_ops": "100000000" } 2024-03-06T07:32:29.485 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json config get mds_max_purge_files 2024-03-06T07:32:29.745 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_max_purge_files":"6"} 2024-03-06T07:32:29.745 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_max_purge_files": "6" } 2024-03-06T07:32:29.745 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:29.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.958+0000 7fce5e800640 1 Processor -- start 2024-03-06T07:32:29.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.959+0000 7fce5e800640 1 -- start start 2024-03-06T07:32:29.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.960+0000 7fce5e800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fce58145920 0x7fce5806c920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.960+0000 7fce5e800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce5815faa0 0x7fce580cd980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.960+0000 7fce5e800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fce580cdec0 con 0x7fce580cc490 2024-03-06T07:32:29.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.960+0000 7fce5e800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fce581601f0 con 0x7fce58145920 2024-03-06T07:32:29.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.960+0000 7fce5e800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fce581604d0 con 0x7fce5815faa0 2024-03-06T07:32:29.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.960+0000 7fce5de00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fce580cc490 0x7fce58071dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40352/0 (socket says 172.21.15.86:40352) 2024-03-06T07:32:29.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.960+0000 7fce5de00640 1 -- 172.21.15.86:0/1874353000 learned_addr learned my addr 172.21.15.86:0/1874353000 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:29.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce5ca00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce5815faa0 0x7fce580cd980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce5d400640 1 --2- 172.21.15.86:0/1874353000 >> v2:172.21.15.194:3300/0 conn(0x7fce58145920 0x7fce5806c920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 790101916 0 0) 0x7fce580cdec0 con 0x7fce580cc490 2024-03-06T07:32:29.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fce40003520 con 0x7fce580cc490 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1683007481 0 0) 0x7fce40003520 con 0x7fce580cc490 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce5815faa0 msgr2=0x7fce580cd980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce5815faa0 0x7fce580cd980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 >> v2:172.21.15.194:3300/0 conn(0x7fce58145920 msgr2=0x7fce5806c920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 --2- 172.21.15.86:0/1874353000 >> v2:172.21.15.194:3300/0 conn(0x7fce58145920 0x7fce5806c920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce581607b0 con 0x7fce580cc490 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fce500019f0 con 0x7fce580cc490 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 >> v1:172.21.15.86:6789/0 conn(0x7fce580cc490 legacy=0x7fce58071dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce40003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce40004800 0x7fce40004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.961+0000 7fce57e00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce400051c0 0x7fce4000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.962+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce581601f0 con 0x7fce400039b0 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.962+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce581604d0 con 0x7fce40004800 2024-03-06T07:32:29.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.962+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fce4000bf00 con 0x7fce400051c0 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.962+0000 7fce5d400640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce40003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.962+0000 7fce5de00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce400051c0 0x7fce4000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.962+0000 7fce5ca00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce40004800 0x7fce40004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.963+0000 7fce5d400640 1 -- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce400051c0 msgr2=0x7fce4000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.963+0000 7fce5d400640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce400051c0 0x7fce4000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.963+0000 7fce5d400640 1 -- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce40004800 msgr2=0x7fce40004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.963+0000 7fce5d400640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce40004800 0x7fce40004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.963+0000 7fce5d400640 1 -- 172.21.15.86:0/1874353000 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce581607b0 con 0x7fce400039b0 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.963+0000 7fce5ca00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce40004800 0x7fce40004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:29.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.963+0000 7fce5de00640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce400051c0 0x7fce4000b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:29.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.964+0000 7fce5d400640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce40003dd0 secure :-1 s=READY pgs=1137 cs=0 l=1 rev1=1 crypto rx=0x7fce4c009aa0 tx=0x7fce40005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=33171271931c2103 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:29.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.964+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce4c039070 con 0x7fce400039b0 2024-03-06T07:32:29.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.964+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fce4c02c6f0 con 0x7fce400039b0 2024-03-06T07:32:29.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.964+0000 7fce57e00640 1 -- 172.21.15.86:0/1874353000 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce4c009670 con 0x7fce400039b0 2024-03-06T07:32:29.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.965+0000 7fce5e800640 1 -- 172.21.15.86:0/1874353000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 msgr2=0x7fce40003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:29.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.965+0000 7fce5e800640 1 --2- 172.21.15.86:0/1874353000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce40003dd0 secure :-1 s=READY pgs=1137 cs=0 l=1 rev1=1 crypto rx=0x7fce4c009aa0 tx=0x7fce40005830 comp rx=0 tx=0).stop 2024-03-06T07:32:29.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.965+0000 7fce5de00640 1 -- 172.21.15.86:0/1874353000 reap_dead start 2024-03-06T07:32:29.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.965+0000 7fce5e800640 1 -- 172.21.15.86:0/1874353000 shutdown_connections 2024-03-06T07:32:29.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.965+0000 7fce5e800640 1 -- 172.21.15.86:0/1874353000 >> 172.21.15.86:0/1874353000 conn(0x7fce580b1770 msgr2=0x7fce580b1b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:29.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.965+0000 7fce5e800640 1 -- 172.21.15.86:0/1874353000 shutdown_connections 2024-03-06T07:32:29.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.965+0000 7fce5e800640 1 -- 172.21.15.86:0/1874353000 wait complete. 2024-03-06T07:32:29.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.967+0000 7fce5e800640 1 Processor -- start 2024-03-06T07:32:29.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.967+0000 7fce5e800640 1 -- start start 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce581633a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5e800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce58165d10 0x7fce581422d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5e800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce58166190 0x7fce58142810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5e800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce580cd980 con 0x7fce400039b0 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5e800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce5815fec0 con 0x7fce58166190 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5e800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fce580cdb00 con 0x7fce58165d10 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce581633a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce581633a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55352/0 (socket says 172.21.15.86:55352) 2024-03-06T07:32:29.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.968+0000 7fce5d400640 1 -- 172.21.15.86:0/1442634881 learned_addr learned my addr 172.21.15.86:0/1442634881 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5de00640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce58166190 0x7fce58142810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5d400640 1 -- 172.21.15.86:0/1442634881 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce58165d10 msgr2=0x7fce581422d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5d400640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce58165d10 0x7fce581422d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5d400640 1 -- 172.21.15.86:0/1442634881 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce58166190 msgr2=0x7fce58142810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5d400640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce58166190 0x7fce58142810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5d400640 1 -- 172.21.15.86:0/1442634881 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce58162520 con 0x7fce400039b0 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5d400640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce581633a0 secure :-1 s=READY pgs=1138 cs=0 l=1 rev1=1 crypto rx=0x7fce4c009bc0 tx=0x7fce4c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:29.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce55600640 1 -- 172.21.15.86:0/1442634881 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce4c042020 con 0x7fce400039b0 2024-03-06T07:32:29.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce55600640 1 -- 172.21.15.86:0/1442634881 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fce4c03a040 con 0x7fce400039b0 2024-03-06T07:32:29.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce55600640 1 -- 172.21.15.86:0/1442634881 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce4c039070 con 0x7fce400039b0 2024-03-06T07:32:29.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.969+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fce581619a0 con 0x7fce400039b0 2024-03-06T07:32:29.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.970+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fce58161f60 con 0x7fce400039b0 2024-03-06T07:32:29.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.971+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fce5804f330 con 0x7fce400039b0 2024-03-06T07:32:29.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.972+0000 7fce55600640 1 -- 172.21.15.86:0/1442634881 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fce4c045020 con 0x7fce400039b0 2024-03-06T07:32:29.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.973+0000 7fce55600640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fce3806b500 0x7fce3806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:29.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.973+0000 7fce55600640 1 -- 172.21.15.86:0/1442634881 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fce4c0ab810 con 0x7fce400039b0 2024-03-06T07:32:29.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.973+0000 7fce5ca00640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fce3806b500 0x7fce3806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:29.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.974+0000 7fce5ca00640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fce3806b500 0x7fce3806d9c0 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fce581452e0 tx=0x7fce44008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:29.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:29.978+0000 7fce55600640 1 -- 172.21.15.86:0/1442634881 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fce4c04c070 con 0x7fce400039b0 2024-03-06T07:32:30.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.406+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fce5815ddb0 con 0x7fce400039b0 2024-03-06T07:32:30.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.407+0000 7fce55600640 1 -- 172.21.15.86:0/1442634881 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7fce4c07a540 con 0x7fce400039b0 2024-03-06T07:32:30.409 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:30.409 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:30.409 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fce3806b500 msgr2=0x7fce3806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fce3806b500 0x7fce3806d9c0 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fce581452e0 tx=0x7fce44008040 comp rx=0 tx=0).stop 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 msgr2=0x7fce581633a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce581633a0 secure :-1 s=READY pgs=1138 cs=0 l=1 rev1=1 crypto rx=0x7fce4c009bc0 tx=0x7fce4c004730 comp rx=0 tx=0).stop 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 shutdown_connections 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fce3806b500 0x7fce3806d9c0 unknown :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fce58166190 0x7fce58142810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fce58165d10 0x7fce581422d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 --2- 172.21.15.86:0/1442634881 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fce400039b0 0x7fce581633a0 unknown :-1 s=CLOSED pgs=1138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 >> 172.21.15.86:0/1442634881 conn(0x7fce580b1770 msgr2=0x7fce580b45b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 shutdown_connections 2024-03-06T07:32:30.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:30.410+0000 7fce5e800640 1 -- 172.21.15.86:0/1442634881 wait complete. 2024-03-06T07:32:30.432 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:30.707 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":20,"msgr_send_messages":21,"msgr_recv_bytes":13630,"msgr_send_bytes":3645223,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.027591732,"msgr_running_send_time":0.015085207,"msgr_running_recv_time":0.004768090,"msgr_running_fast_dispatch_time":0.003577666,"msgr_send_messages_queue_lat":{"avgcount":20,"sum":0.008240399,"avgtime":0.000412019},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":6443,"msgr_send_messages":12843,"msgr_recv_bytes":2324392,"msgr_send_bytes":14235470,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.341393319,"msgr_running_send_time":0.903822264,"msgr_running_recv_time":0.957524846,"msgr_running_fast_dispatch_time":0.008332825,"msgr_send_messages_queue_lat":{"avgcount":12841,"sum":0.486314235,"avgtime":0.000037871},"msgr_handle_ack_lat":{"avgcount":14958,"sum":0.007663783,"avgtime":0.000000512},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":92448},"AsyncMessenger::Worker-2":{"msgr_recv_messages":34,"msgr_send_messages":27,"msgr_recv_bytes":112930,"msgr_send_bytes":10032,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.015318005,"msgr_running_send_time":0.001934413,"msgr_running_recv_time":0.017156832,"msgr_running_fast_dispatch_time":0.000905219,"msgr_send_messages_queue_lat":{"avgcount":27,"sum":0.016301493,"avgtime":0.000603759},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109152,"msgr_send_encrypted_bytes":7120},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":71,"sum":2.399581422,"avgtime":0.033796921}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":4,"sum":0.000245193,"avgtime":0.000061298}},"mds":{"request":3402,"reply":3400,"reply_latency":{"avgcount":3400,"sum":631.656940173,"avgtime":0.185781452},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":3211,"inodes_top":0,"inodes_bottom":3211,"inodes_pin_tail":0,"inodes_pinned":3211,"inodes_expired":0,"inodes_with_caps":3202,"caps":3202,"subtrees":2,"traverse":3403,"traverse_hit":3403,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":3011,"root_rbytes":20671488,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":1,"ceph_cap_op_grant":3204,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":2995,"handle_client_caps_dirty":2992,"handle_client_cap_release":0,"process_request_cap_release":199},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":191,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":191,"strays_enqueued":0,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":6395,"evex":0,"evtrm":0,"ev":6398,"evexg":0,"evexd":0,"segadd":6,"segex":0,"segtrm":0,"seg":8,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":17814203,"rdpos":4195244,"jlat":{"avgcount":11,"sum":0.757037104,"avgtime":0.068821554},"replayed":3},"mds_mem":{"ino":3214,"ino+":3214,"ino-":0,"dir":13,"dir+":13,"dir-":0,"dn":3402,"dn+":3402,"dn-":0,"cap":3202,"cap+":3202,"cap-":0,"rss":108652,"heap":248164},"mds_server":{"dispatch_client_request":3402,"dispatch_server_request":0,"handle_client_request":3402,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":191,"sum":0.217368242,"avgtime":0.001138053},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":3279,"average_load":3279,"avg_session_uptime":1106,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":25580167,"buffer_anon_items":11809,"buffer_meta_bytes":97328,"buffer_meta_items":1106,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":14522825,"mds_co_items":137731,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":63,"op_send_bytes":13257162,"op_resend":0,"op_reply":63,"op_latency":{"avgcount":63,"sum":0.386991840,"avgtime":0.006142727},"op_inflight":0,"oplen_avg":{"avgcount":63,"sum":114},"op":63,"op_r":36,"op_w":27,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":8,"osdop_write":11,"osdop_writefull":1,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":8,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":11,"omap_rd":48,"omap_del":0},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":0,"pq_executing":0,"pq_executing_high_water":0,"pq_executed_ops":0,"pq_executed":0,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":6492,"get_sum":1863894,"get_or_fail_fail":0,"get_or_fail_success":6492,"take":0,"take_sum":0,"put":6492,"put_sum":1863894,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":63,"take_sum":13258096,"put":63,"put_sum":13258096,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":63,"take_sum":63,"put":63,"put_sum":63,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":0,"put_sum":0,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":646199,"max":3758096384,"get_started":0,"get":6395,"get_sum":13618701,"get_or_fail_fail":0,"get_or_fail_success":6395,"take":0,"take_sum":0,"put":11,"put_sum":12972502,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:30.710 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 13630, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 20, "msgr_running_fast_dispatch_time": 0.003577666, "msgr_running_recv_time": 0.00476809, "msgr_running_send_time": 0.015085207, "msgr_running_total_time": 0.027591732, "msgr_send_bytes": 3645223, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 21, "msgr_send_messages_queue_lat": { "avgcount": 20, "avgtime": 0.000412019, "sum": 0.008240399 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 14958, "avgtime": 5.12e-07, "sum": 0.007663783 }, "msgr_recv_bytes": 2324392, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 6443, "msgr_running_fast_dispatch_time": 0.008332825, "msgr_running_recv_time": 0.957524846, "msgr_running_send_time": 0.903822264, "msgr_running_total_time": 1.341393319, "msgr_send_bytes": 14235470, "msgr_send_encrypted_bytes": 92448, "msgr_send_messages": 12843, "msgr_send_messages_queue_lat": { "avgcount": 12841, "avgtime": 3.7871e-05, "sum": 0.486314235 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 112930, "msgr_recv_encrypted_bytes": 109152, "msgr_recv_messages": 34, "msgr_running_fast_dispatch_time": 0.000905219, "msgr_running_recv_time": 0.017156832, "msgr_running_send_time": 0.001934413, "msgr_running_total_time": 0.015318005, "msgr_send_bytes": 10032, "msgr_send_encrypted_bytes": 7120, "msgr_send_messages": 27, "msgr_send_messages_queue_lat": { "avgcount": 27, "avgtime": 0.000603759, "sum": 0.016301493 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 71, "avgtime": 0.033796921, "sum": 2.399581422 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 4, "avgtime": 6.1298e-05, "sum": 0.000245193 }, "queue_len": 0 }, "mds": { "caps": 3202, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 3204, "ceph_cap_op_revoke": 1, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 0, "handle_client_caps": 2995, "handle_client_caps_dirty": 2992, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 3211, "inodes_bottom": 3211, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 3211, "inodes_top": 0, "inodes_with_caps": 3202, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 199, "q": 0, "reply": 3400, "reply_latency": { "avgcount": 3400, "avgtime": 0.185781452, "sum": 631.656940173 }, "request": 3402, "root_rbytes": 20671488, "root_rfiles": 3011, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 3403, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 3403, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 191, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 191, "strays_enqueued": 0, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 6398, "evadd": 6395, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 11, "avgtime": 0.068821554, "sum": 0.757037104 }, "rdpos": 4195244, "replayed": 3, "seg": 8, "segadd": 6, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 17814203 }, "mds_mem": { "cap": 3202, "cap+": 3202, "cap-": 0, "dir": 13, "dir+": 13, "dir-": 0, "dn": 3402, "dn+": 3402, "dn-": 0, "heap": 248164, "ino": 3214, "ino+": 3214, "ino-": 0, "rss": 108652 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 3402, "dispatch_server_request": 0, "handle_client_request": 3402, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 191, "avgtime": 0.001138053, "sum": 0.217368242 } }, "mds_sessions": { "average_load": 3279, "avg_session_uptime": 1106, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 3279 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 25580167, "buffer_anon_items": 11809, "buffer_meta_bytes": 97328, "buffer_meta_items": 1106, "mds_co_bytes": 14522825, "mds_co_items": 137731, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 0, "omap_rd": 48, "omap_wr": 11, "op": 63, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 63, "avgtime": 0.006142727, "sum": 0.38699184 }, "op_pg": 0, "op_r": 36, "op_reply": 63, "op_resend": 0, "op_rmw": 0, "op_send": 63, "op_send_bytes": 13257162, "op_w": 27, "oplen_avg": { "avgcount": 63, "sum": 114 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 8, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 8, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 11, "osdop_writefull": 1, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 0, "pq_executed_ops": 0, "pq_executing": 0, "pq_executing_high_water": 0, "pq_executing_ops": 0, "pq_executing_ops_high_water": 0, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 6492, "get_or_fail_fail": 0, "get_or_fail_success": 6492, "get_started": 0, "get_sum": 1863894, "max": 104857600, "put": 6492, "put_sum": 1863894, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 63, "put_sum": 13258096, "take": 63, "take_sum": 13258096, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 63, "put_sum": 63, "take": 63, "take_sum": 63, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 3758096384, "put": 0, "put_sum": 0, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 6395, "get_or_fail_fail": 0, "get_or_fail_success": 6395, "get_started": 0, "get_sum": 13618701, "max": 3758096384, "put": 11, "put_sum": 12972502, "take": 0, "take_sum": 0, "val": 646199, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:30.711 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/191, 0/191 2024-03-06T07:32:31.712 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:31.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.916+0000 7f1dcb200640 1 Processor -- start 2024-03-06T07:32:31.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.916+0000 7f1dcb200640 1 -- start start 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dcb200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1dcc15e410 0x7f1dcc0d1c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dcb200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc0d4980 0x7f1dcc0d0870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dcb200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1dcc0d0db0 0x7f1dcc0b4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dcb200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1dcc0d4de0 con 0x7f1dcc0d0db0 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dcb200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1dcc0b6e50 con 0x7f1dcc15e410 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dcb200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1dcc0b7130 con 0x7f1dcc0d4980 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dca800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1dcc0d0db0 0x7f1dcc0b4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dca800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1dcc0d0db0 0x7f1dcc0b4ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55356/0 (socket says 172.21.15.86:55356) 2024-03-06T07:32:31.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.918+0000 7f1dca800640 1 -- 172.21.15.86:0/45406244 learned_addr learned my addr 172.21.15.86:0/45406244 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:31.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.919+0000 7f1dc9400640 1 --2- 172.21.15.86:0/45406244 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc0d4980 0x7f1dcc0d0870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:31.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.919+0000 7f1dca800640 1 -- 172.21.15.86:0/45406244 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc0d4980 msgr2=0x7f1dcc0d0870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:31.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.919+0000 7f1dc9e00640 1 --2- 172.21.15.86:0/45406244 >> v2:172.21.15.194:3300/0 conn(0x7f1dcc15e410 0x7f1dcc0d1c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:31.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.919+0000 7f1dca800640 1 --2- 172.21.15.86:0/45406244 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc0d4980 0x7f1dcc0d0870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.919+0000 7f1dca800640 1 -- 172.21.15.86:0/45406244 >> v2:172.21.15.194:3300/0 conn(0x7f1dcc15e410 msgr2=0x7f1dcc0d1c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:31.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.919+0000 7f1dca800640 1 --2- 172.21.15.86:0/45406244 >> v2:172.21.15.194:3300/0 conn(0x7f1dcc15e410 0x7f1dcc0d1c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.919+0000 7f1dca800640 1 -- 172.21.15.86:0/45406244 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1dcc0b7410 con 0x7f1dcc0d0db0 2024-03-06T07:32:31.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.920+0000 7f1dca800640 1 --2- 172.21.15.86:0/45406244 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d0db0 0x7f1dcc0b4ad0 secure :-1 s=READY pgs=1139 cs=0 l=1 rev1=1 crypto rx=0x7f1dcc06e120 tx=0x7f1dc402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da04e6ea0797edc3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:31.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.920+0000 7f1dc8a00640 1 -- 172.21.15.86:0/45406244 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dc402ca70 con 0x7f1dcc0d0db0 2024-03-06T07:32:31.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.920+0000 7f1dc8a00640 1 -- 172.21.15.86:0/45406244 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1dc402cc10 con 0x7f1dcc0d0db0 2024-03-06T07:32:31.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.920+0000 7f1dc8a00640 1 -- 172.21.15.86:0/45406244 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dc40343f0 con 0x7f1dcc0d0db0 2024-03-06T07:32:31.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.921+0000 7f1dcb200640 1 -- 172.21.15.86:0/45406244 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d0db0 msgr2=0x7f1dcc0b4ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:31.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.921+0000 7f1dcb200640 1 --2- 172.21.15.86:0/45406244 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d0db0 0x7f1dcc0b4ad0 secure :-1 s=READY pgs=1139 cs=0 l=1 rev1=1 crypto rx=0x7f1dcc06e120 tx=0x7f1dc402bed0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.921+0000 7f1dcb200640 1 -- 172.21.15.86:0/45406244 shutdown_connections 2024-03-06T07:32:31.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.921+0000 7f1dcb200640 1 --2- 172.21.15.86:0/45406244 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d0db0 0x7f1dcc0b4ad0 unknown :-1 s=CLOSED pgs=1139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.921+0000 7f1dcb200640 1 --2- 172.21.15.86:0/45406244 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc0d4980 0x7f1dcc0d0870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.921+0000 7f1dcb200640 1 --2- 172.21.15.86:0/45406244 >> v2:172.21.15.194:3300/0 conn(0x7f1dcc15e410 0x7f1dcc0d1c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.921+0000 7f1dcb200640 1 -- 172.21.15.86:0/45406244 >> 172.21.15.86:0/45406244 conn(0x7f1dcc0d3e60 msgr2=0x7f1dcc0d4270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:31.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.922+0000 7f1dcb200640 1 -- 172.21.15.86:0/45406244 shutdown_connections 2024-03-06T07:32:31.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.922+0000 7f1dcb200640 1 -- 172.21.15.86:0/45406244 wait complete. 2024-03-06T07:32:31.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.923+0000 7f1dcb200640 1 Processor -- start 2024-03-06T07:32:31.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.924+0000 7f1dcb200640 1 -- start start 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.925+0000 7f1dcb200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dcc0d0db0 0x7f1dcc0cb940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.925+0000 7f1dcb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d4980 0x7f1dcc160e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.925+0000 7f1dcb200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc15e410 0x7f1dcc15f9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.925+0000 7f1dcb200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1dcc0b5010 con 0x7f1dcc0d4980 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.925+0000 7f1dcb200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1dcc0b79d0 con 0x7f1dcc0d0db0 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.925+0000 7f1dcb200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1dcc0b5190 con 0x7f1dcc15e410 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.925+0000 7f1dc9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d4980 0x7f1dcc160e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d4980 0x7f1dcc160e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55360/0 (socket says 172.21.15.86:55360) 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 -- 172.21.15.86:0/3923105582 learned_addr learned my addr 172.21.15.86:0/3923105582 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dca800640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc15e410 0x7f1dcc15f9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9e00640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dcc0d0db0 0x7f1dcc0cb940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:31.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 -- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc15e410 msgr2=0x7f1dcc15f9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:31.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc15e410 0x7f1dcc15f9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 -- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dcc0d0db0 msgr2=0x7f1dcc0cb940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:31.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dcc0d0db0 0x7f1dcc0cb940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:31.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 -- 172.21.15.86:0/3923105582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1dcc15e950 con 0x7f1dcc0d4980 2024-03-06T07:32:31.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.926+0000 7f1dc9400640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d4980 0x7f1dcc160e10 secure :-1 s=READY pgs=1140 cs=0 l=1 rev1=1 crypto rx=0x7f1db800c820 tx=0x7f1db800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:31.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.927+0000 7f1dc2000640 1 -- 172.21.15.86:0/3923105582 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1db8015070 con 0x7f1dcc0d4980 2024-03-06T07:32:31.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.927+0000 7f1dc2000640 1 -- 172.21.15.86:0/3923105582 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1db8004220 con 0x7f1dcc0d4980 2024-03-06T07:32:31.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.927+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1dcc15ec30 con 0x7f1dcc0d4980 2024-03-06T07:32:31.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.927+0000 7f1dc2000640 1 -- 172.21.15.86:0/3923105582 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1db800ce10 con 0x7f1dcc0d4980 2024-03-06T07:32:31.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.927+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1dcc15f1f0 con 0x7f1dcc0d4980 2024-03-06T07:32:31.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.929+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1dcc04f330 con 0x7f1dcc0d4980 2024-03-06T07:32:31.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.931+0000 7f1dc2000640 1 -- 172.21.15.86:0/3923105582 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1db8007830 con 0x7f1dcc0d4980 2024-03-06T07:32:31.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.932+0000 7f1dc2000640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dac06b4d0 0x7f1dac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:31.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.932+0000 7f1dc2000640 1 -- 172.21.15.86:0/3923105582 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f1db808a590 con 0x7f1dcc0d4980 2024-03-06T07:32:31.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.932+0000 7f1dc9e00640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dac06b4d0 0x7f1dac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:31.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.933+0000 7f1dc9e00640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dac06b4d0 0x7f1dac06d990 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7f1dbc006fd0 tx=0x7f1dbc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:31.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:31.938+0000 7f1dc2000640 1 -- 172.21.15.86:0/3923105582 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1db8059230 con 0x7f1dcc0d4980 2024-03-06T07:32:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.228+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1dcc0d2220 con 0x7f1dcc0d4980 2024-03-06T07:32:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.229+0000 7f1dc2000640 1 -- 172.21.15.86:0/3923105582 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f1db8058b30 con 0x7f1dcc0d4980 2024-03-06T07:32:32.231 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:32.231 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:32.232 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dac06b4d0 msgr2=0x7f1dac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dac06b4d0 0x7f1dac06d990 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7f1dbc006fd0 tx=0x7f1dbc008040 comp rx=0 tx=0).stop 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d4980 msgr2=0x7f1dcc160e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d4980 0x7f1dcc160e10 secure :-1 s=READY pgs=1140 cs=0 l=1 rev1=1 crypto rx=0x7f1db800c820 tx=0x7f1db800cce0 comp rx=0 tx=0).stop 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 shutdown_connections 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dac06b4d0 0x7f1dac06d990 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1dcc15e410 0x7f1dcc15f9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1dcc0d4980 0x7f1dcc160e10 unknown :-1 s=CLOSED pgs=1140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 --2- 172.21.15.86:0/3923105582 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1dcc0d0db0 0x7f1dcc0cb940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 >> 172.21.15.86:0/3923105582 conn(0x7f1dcc0d3e60 msgr2=0x7f1dcc0b96b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 shutdown_connections 2024-03-06T07:32:32.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:32.232+0000 7f1dcb200640 1 -- 172.21.15.86:0/3923105582 wait complete. 2024-03-06T07:32:32.253 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:32.500 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":39,"msgr_send_messages":40,"msgr_recv_bytes":18440,"msgr_send_bytes":3651401,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.030339492,"msgr_running_send_time":0.015951324,"msgr_running_recv_time":0.005369305,"msgr_running_fast_dispatch_time":0.004683974,"msgr_send_messages_queue_lat":{"avgcount":39,"sum":0.008997314,"avgtime":0.000230700},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":8130,"msgr_send_messages":14206,"msgr_recv_bytes":2913255,"msgr_send_bytes":18312683,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.561121353,"msgr_running_send_time":1.035538489,"msgr_running_recv_time":1.175906136,"msgr_running_fast_dispatch_time":0.010535559,"msgr_send_messages_queue_lat":{"avgcount":14204,"sum":0.553431862,"avgtime":0.000038963},"msgr_handle_ack_lat":{"avgcount":17769,"sum":0.008190720,"avgtime":0.000000460},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":92448},"AsyncMessenger::Worker-2":{"msgr_recv_messages":46,"msgr_send_messages":39,"msgr_recv_bytes":119895,"msgr_send_bytes":18321,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.017585298,"msgr_running_send_time":0.003559673,"msgr_running_recv_time":0.017423915,"msgr_running_fast_dispatch_time":0.001182285,"msgr_send_messages_queue_lat":{"avgcount":39,"sum":0.016876401,"avgtime":0.000432728},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109424,"msgr_send_encrypted_bytes":7744},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":1491,"complete_latency":{"avgcount":74,"sum":2.400676031,"avgtime":0.032441567}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":81,"sum":0.050508978,"avgtime":0.000623567}},"mds":{"request":4356,"reply":4353,"reply_latency":{"avgcount":4353,"sum":633.165323886,"avgtime":0.145454933},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":4181,"inodes_top":0,"inodes_bottom":4181,"inodes_pin_tail":0,"inodes_pinned":4181,"inodes_expired":0,"inodes_with_caps":3127,"caps":3127,"subtrees":2,"traverse":4357,"traverse_hit":4356,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":3199,"root_rbytes":24519680,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":1,"ceph_cap_op_grant":3397,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3702,"handle_client_caps_dirty":2998,"handle_client_cap_release":1,"process_request_cap_release":1153},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":1105,"num_strays_delayed":20,"num_strays_enqueuing":0,"strays_created":1144,"strays_enqueued":39,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":8304,"evex":0,"evtrm":0,"ev":8307,"evexg":0,"evexd":0,"segadd":8,"segex":0,"segtrm":0,"seg":10,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":22693301,"rdpos":4195244,"jlat":{"avgcount":12,"sum":0.825870755,"avgtime":0.068822562},"replayed":3},"mds_mem":{"ino":3214,"ino+":3214,"ino-":0,"dir":13,"dir+":13,"dir-":0,"dn":4355,"dn+":4355,"dn-":0,"cap":3127,"cap+":3202,"cap-":75,"rss":110316,"heap":248164},"mds_server":{"dispatch_client_request":4356,"dispatch_server_request":0,"handle_client_request":4356,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":1144,"sum":1.725751955,"avgtime":0.001508524},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":4155,"average_load":4155,"avg_session_uptime":1106,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":25580167,"buffer_anon_items":11809,"buffer_meta_bytes":97328,"buffer_meta_items":1106,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":14522825,"mds_co_items":137731,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":116,"op_send_bytes":17097563,"op_resend":0,"op_reply":116,"op_latency":{"avgcount":116,"sum":0.630986703,"avgtime":0.005439540},"op_inflight":0,"oplen_avg":{"avgcount":116,"sum":171},"op":116,"op_r":37,"op_w":79,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":9,"osdop_write":13,"osdop_writefull":3,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":54,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":15,"omap_rd":48,"omap_del":2},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":39,"pq_executed":39,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":253,"max":104857600,"get_started":0,"get":8210,"get_sum":2309860,"get_or_fail_fail":0,"get_or_fail_success":8210,"take":0,"take_sum":0,"put":8209,"put_sum":2309607,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":116,"take_sum":17102436,"put":116,"put_sum":17102436,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":116,"take_sum":116,"put":116,"put_sum":116,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":39,"get_sum":3939,"get_or_fail_fail":0,"get_or_fail_success":39,"take":0,"take_sum":0,"put":1,"put_sum":3939,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":1721781,"max":3758096384,"get_started":0,"get":8304,"get_sum":18497799,"get_or_fail_fail":0,"get_or_fail_success":8304,"take":0,"take_sum":0,"put":12,"put_sum":16776018,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:32.501 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 18440, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 39, "msgr_running_fast_dispatch_time": 0.004683974, "msgr_running_recv_time": 0.005369305, "msgr_running_send_time": 0.015951324, "msgr_running_total_time": 0.030339492, "msgr_send_bytes": 3651401, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 40, "msgr_send_messages_queue_lat": { "avgcount": 39, "avgtime": 0.0002307, "sum": 0.008997314 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 17769, "avgtime": 4.6e-07, "sum": 0.00819072 }, "msgr_recv_bytes": 2913255, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 8130, "msgr_running_fast_dispatch_time": 0.010535559, "msgr_running_recv_time": 1.175906136, "msgr_running_send_time": 1.035538489, "msgr_running_total_time": 1.561121353, "msgr_send_bytes": 18312683, "msgr_send_encrypted_bytes": 92448, "msgr_send_messages": 14206, "msgr_send_messages_queue_lat": { "avgcount": 14204, "avgtime": 3.8963e-05, "sum": 0.553431862 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 119895, "msgr_recv_encrypted_bytes": 109424, "msgr_recv_messages": 46, "msgr_running_fast_dispatch_time": 0.001182285, "msgr_running_recv_time": 0.017423915, "msgr_running_send_time": 0.003559673, "msgr_running_total_time": 0.017585298, "msgr_send_bytes": 18321, "msgr_send_encrypted_bytes": 7744, "msgr_send_messages": 39, "msgr_send_messages_queue_lat": { "avgcount": 39, "avgtime": 0.000432728, "sum": 0.016876401 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 74, "avgtime": 0.032441567, "sum": 2.400676031 }, "queue_len": 1491 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 81, "avgtime": 0.000623567, "sum": 0.050508978 }, "queue_len": 0 }, "mds": { "caps": 3127, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 3397, "ceph_cap_op_revoke": 1, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 1, "handle_client_caps": 3702, "handle_client_caps_dirty": 2998, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 4181, "inodes_bottom": 4181, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 4181, "inodes_top": 0, "inodes_with_caps": 3127, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 1153, "q": 0, "reply": 4353, "reply_latency": { "avgcount": 4353, "avgtime": 0.145454933, "sum": 633.165323886 }, "request": 4356, "root_rbytes": 24519680, "root_rfiles": 3199, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 4357, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 4356, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 1105, "num_strays_delayed": 20, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 1144, "strays_enqueued": 39, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 8307, "evadd": 8304, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 12, "avgtime": 0.068822562, "sum": 0.825870755 }, "rdpos": 4195244, "replayed": 3, "seg": 10, "segadd": 8, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 22693301 }, "mds_mem": { "cap": 3127, "cap+": 3202, "cap-": 75, "dir": 13, "dir+": 13, "dir-": 0, "dn": 4355, "dn+": 4355, "dn-": 0, "heap": 248164, "ino": 3214, "ino+": 3214, "ino-": 0, "rss": 110316 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 4356, "dispatch_server_request": 0, "handle_client_request": 4356, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 1144, "avgtime": 0.001508524, "sum": 1.725751955 } }, "mds_sessions": { "average_load": 4155, "avg_session_uptime": 1106, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 4155 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 25580167, "buffer_anon_items": 11809, "buffer_meta_bytes": 97328, "buffer_meta_items": 1106, "mds_co_bytes": 14522825, "mds_co_items": 137731, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 2, "omap_rd": 48, "omap_wr": 15, "op": 116, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 116, "avgtime": 0.00543954, "sum": 0.630986703 }, "op_pg": 0, "op_r": 37, "op_reply": 116, "op_resend": 0, "op_rmw": 0, "op_send": 116, "op_send_bytes": 17097563, "op_w": 79, "oplen_avg": { "avgcount": 116, "sum": 171 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 54, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 9, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 13, "osdop_writefull": 3, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 39, "pq_executed_ops": 39, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 8210, "get_or_fail_fail": 0, "get_or_fail_success": 8210, "get_started": 0, "get_sum": 2309860, "max": 104857600, "put": 8209, "put_sum": 2309607, "take": 0, "take_sum": 0, "val": 253, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 116, "put_sum": 17102436, "take": 116, "take_sum": 17102436, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 116, "put_sum": 116, "take": 116, "take_sum": 116, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 39, "get_or_fail_fail": 0, "get_or_fail_success": 39, "get_started": 0, "get_sum": 3939, "max": 3758096384, "put": 1, "put_sum": 3939, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 8304, "get_or_fail_fail": 0, "get_or_fail_success": 8304, "get_started": 0, "get_sum": 18497799, "max": 3758096384, "put": 12, "put_sum": 16776018, "take": 0, "take_sum": 0, "val": 1721781, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:32.501 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/1105, 39/1144 2024-03-06T07:32:33.503 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:33.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.725+0000 7f1f56c00640 1 Processor -- start 2024-03-06T07:32:33.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.725+0000 7f1f56c00640 1 -- start start 2024-03-06T07:32:33.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.726+0000 7f1f56c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f50155490 0x7f1f500ae630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:33.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.726+0000 7f1f56c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1f5015f320 0x7f1f500d04f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:33.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f56c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1f50155870 con 0x7f1f500d0070 2024-03-06T07:32:33.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f56c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1f500ad640 con 0x7f1f5015f320 2024-03-06T07:32:33.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f56c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f5015f980 con 0x7f1f50155490 2024-03-06T07:32:33.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f4f400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1f500d0070 0x7f1f5015ec10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40360/0 (socket says 172.21.15.86:40360) 2024-03-06T07:32:33.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f4f400640 1 -- 172.21.15.86:0/1237179001 learned_addr learned my addr 172.21.15.86:0/1237179001 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:33.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f54c00640 1 --2- 172.21.15.86:0/1237179001 >> v2:172.21.15.86:3300/0 conn(0x7f1f5015f320 0x7f1f500d04f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:33.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f4fe00640 1 --2- 172.21.15.86:0/1237179001 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f50155490 0x7f1f500ae630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:33.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f4ea00640 1 -- 172.21.15.86:0/1237179001 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 252143977 0 0) 0x7f1f50155870 con 0x7f1f500d0070 2024-03-06T07:32:33.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f4ea00640 1 -- 172.21.15.86:0/1237179001 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1f34003520 con 0x7f1f500d0070 2024-03-06T07:32:33.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.727+0000 7f1f54c00640 1 -- 172.21.15.86:0/1237179001 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f50155490 msgr2=0x7f1f500ae630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:33.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.728+0000 7f1f54c00640 1 --2- 172.21.15.86:0/1237179001 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f50155490 0x7f1f500ae630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:33.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.728+0000 7f1f54c00640 1 -- 172.21.15.86:0/1237179001 >> v1:172.21.15.86:6789/0 conn(0x7f1f500d0070 legacy=0x7f1f5015ec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:33.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.728+0000 7f1f54c00640 1 -- 172.21.15.86:0/1237179001 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f500ad870 con 0x7f1f5015f320 2024-03-06T07:32:33.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.729+0000 7f1f54c00640 1 --2- 172.21.15.86:0/1237179001 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f5015f320 0x7f1f500d04f0 secure :-1 s=READY pgs=1142 cs=0 l=1 rev1=1 crypto rx=0x7f1f48009d90 tx=0x7f1f4802beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bfa851f28899db07 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:33.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.729+0000 7f1f4ea00640 1 -- 172.21.15.86:0/1237179001 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f48039070 con 0x7f1f5015f320 2024-03-06T07:32:33.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.729+0000 7f1f4ea00640 1 -- 172.21.15.86:0/1237179001 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f4802cc60 con 0x7f1f5015f320 2024-03-06T07:32:33.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.729+0000 7f1f4ea00640 1 -- 172.21.15.86:0/1237179001 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f48034660 con 0x7f1f5015f320 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 -- 172.21.15.86:0/1237179001 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f5015f320 msgr2=0x7f1f500d04f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 --2- 172.21.15.86:0/1237179001 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f5015f320 0x7f1f500d04f0 secure :-1 s=READY pgs=1142 cs=0 l=1 rev1=1 crypto rx=0x7f1f48009d90 tx=0x7f1f4802beb0 comp rx=0 tx=0).stop 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 -- 172.21.15.86:0/1237179001 shutdown_connections 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 --2- 172.21.15.86:0/1237179001 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f5015f320 0x7f1f500d04f0 unknown :-1 s=CLOSED pgs=1142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 --2- 172.21.15.86:0/1237179001 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f50155490 0x7f1f500ae630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 -- 172.21.15.86:0/1237179001 >> 172.21.15.86:0/1237179001 conn(0x7f1f5006b360 msgr2=0x7f1f5006b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 -- 172.21.15.86:0/1237179001 shutdown_connections 2024-03-06T07:32:33.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.730+0000 7f1f56c00640 1 -- 172.21.15.86:0/1237179001 wait complete. 2024-03-06T07:32:33.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.732+0000 7f1f56c00640 1 Processor -- start 2024-03-06T07:32:33.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.732+0000 7f1f56c00640 1 -- start start 2024-03-06T07:32:33.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.733+0000 7f1f56c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f500d0070 0x7f1f500b12c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:33.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.733+0000 7f1f56c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f50155490 0x7f1f5015d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:33.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.733+0000 7f1f56c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f5015f320 0x7f1f5014c190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:33.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.733+0000 7f1f56c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f500ae110 con 0x7f1f50155490 2024-03-06T07:32:33.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.733+0000 7f1f56c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f500ade30 con 0x7f1f500d0070 2024-03-06T07:32:33.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.733+0000 7f1f56c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f50071960 con 0x7f1f5015f320 2024-03-06T07:32:33.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f50155490 0x7f1f5015d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:33.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f50155490 0x7f1f5015d850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55380/0 (socket says 172.21.15.86:55380) 2024-03-06T07:32:33.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 -- 172.21.15.86:0/2999429870 learned_addr learned my addr 172.21.15.86:0/2999429870 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:33.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4fe00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f500d0070 0x7f1f500b12c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:33.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f54c00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f5015f320 0x7f1f5014c190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:33.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 -- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f5015f320 msgr2=0x7f1f5014c190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:33.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f5015f320 0x7f1f5014c190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:33.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 -- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f500d0070 msgr2=0x7f1f500b12c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:33.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f500d0070 0x7f1f500b12c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:33.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.734+0000 7f1f4f400640 1 -- 172.21.15.86:0/2999429870 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f5009aff0 con 0x7f1f50155490 2024-03-06T07:32:33.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.735+0000 7f1f4f400640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f50155490 0x7f1f5015d850 secure :-1 s=READY pgs=1143 cs=0 l=1 rev1=1 crypto rx=0x7f1f3c00c430 tx=0x7f1f3c00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:33.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.735+0000 7f1f47e00640 1 -- 172.21.15.86:0/2999429870 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f3c017070 con 0x7f1f50155490 2024-03-06T07:32:33.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.735+0000 7f1f47e00640 1 -- 172.21.15.86:0/2999429870 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f3c005ce0 con 0x7f1f50155490 2024-03-06T07:32:33.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.735+0000 7f1f47e00640 1 -- 172.21.15.86:0/2999429870 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f3c0072d0 con 0x7f1f50155490 2024-03-06T07:32:33.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.735+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1f5009b2d0 con 0x7f1f50155490 2024-03-06T07:32:33.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.736+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1f5009b890 con 0x7f1f50155490 2024-03-06T07:32:33.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.737+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1f50153b00 con 0x7f1f50155490 2024-03-06T07:32:33.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.738+0000 7f1f47e00640 1 -- 172.21.15.86:0/2999429870 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1f3c01a070 con 0x7f1f50155490 2024-03-06T07:32:33.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.740+0000 7f1f47e00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f3006b520 0x7f1f3006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:33.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.740+0000 7f1f47e00640 1 -- 172.21.15.86:0/2999429870 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f1f3c089660 con 0x7f1f50155490 2024-03-06T07:32:33.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.740+0000 7f1f4fe00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f3006b520 0x7f1f3006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:33.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.741+0000 7f1f4fe00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f3006b520 0x7f1f3006d9e0 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f1f40006fd0 tx=0x7f1f40008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:33.747 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:33.744+0000 7f1f47e00640 1 -- 172.21.15.86:0/2999429870 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1f3c057dc0 con 0x7f1f50155490 2024-03-06T07:32:34.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.052+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1f50071740 con 0x7f1f50155490 2024-03-06T07:32:34.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.053+0000 7f1f47e00640 1 -- 172.21.15.86:0/2999429870 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f1f3c05be60 con 0x7f1f50155490 2024-03-06T07:32:34.056 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:34.056 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:34.056 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:34.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.056+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f3006b520 msgr2=0x7f1f3006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:34.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.056+0000 7f1f56c00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f3006b520 0x7f1f3006d9e0 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f1f40006fd0 tx=0x7f1f40008040 comp rx=0 tx=0).stop 2024-03-06T07:32:34.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.056+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f50155490 msgr2=0x7f1f5015d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:34.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.056+0000 7f1f56c00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f50155490 0x7f1f5015d850 secure :-1 s=READY pgs=1143 cs=0 l=1 rev1=1 crypto rx=0x7f1f3c00c430 tx=0x7f1f3c00c8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:34.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 shutdown_connections 2024-03-06T07:32:34.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f3006b520 0x7f1f3006d9e0 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:34.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f5015f320 0x7f1f5014c190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:34.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f50155490 0x7f1f5015d850 unknown :-1 s=CLOSED pgs=1143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:34.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 --2- 172.21.15.86:0/2999429870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f500d0070 0x7f1f500b12c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:34.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 >> 172.21.15.86:0/2999429870 conn(0x7f1f5006b360 msgr2=0x7f1f5006d140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:34.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 shutdown_connections 2024-03-06T07:32:34.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:34.057+0000 7f1f56c00640 1 -- 172.21.15.86:0/2999429870 wait complete. 2024-03-06T07:32:34.082 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:34.367 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":58,"msgr_send_messages":59,"msgr_recv_bytes":23258,"msgr_send_bytes":3657587,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.032718367,"msgr_running_send_time":0.017174276,"msgr_running_recv_time":0.005854737,"msgr_running_fast_dispatch_time":0.005097966,"msgr_send_messages_queue_lat":{"avgcount":58,"sum":0.010033790,"avgtime":0.000172996},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":9067,"msgr_send_messages":16534,"msgr_recv_bytes":3238998,"msgr_send_bytes":23372626,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.815744542,"msgr_running_send_time":1.194402132,"msgr_running_recv_time":1.291109980,"msgr_running_fast_dispatch_time":0.013604224,"msgr_send_messages_queue_lat":{"avgcount":16532,"sum":0.650771629,"avgtime":0.000039364},"msgr_handle_ack_lat":{"avgcount":20296,"sum":0.008889322,"avgtime":0.000000437},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":92448},"AsyncMessenger::Worker-2":{"msgr_recv_messages":64,"msgr_send_messages":57,"msgr_recv_bytes":145185,"msgr_send_bytes":44997,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.019519001,"msgr_running_send_time":0.004512827,"msgr_running_recv_time":0.017857620,"msgr_running_fast_dispatch_time":0.001606923,"msgr_send_messages_queue_lat":{"avgcount":57,"sum":0.018005224,"avgtime":0.000315881},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109424,"msgr_send_encrypted_bytes":7744},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":1477,"complete_latency":{"avgcount":78,"sum":4.574563896,"avgtime":0.058648255}},"finisher-PurgeQueue":{"queue_len":121,"complete_latency":{"avgcount":154,"sum":0.218715884,"avgtime":0.001420233}},"mds":{"request":5274,"reply":5271,"reply_latency":{"avgcount":5271,"sum":634.747383577,"avgtime":0.120422573},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":5273,"inodes_top":0,"inodes_bottom":5273,"inodes_pin_tail":0,"inodes_pinned":5273,"inodes_expired":0,"inodes_with_caps":2876,"caps":2876,"subtrees":2,"traverse":5275,"traverse_hit":5275,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":2736,"root_rbytes":21040128,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":1,"ceph_cap_op_grant":4094,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3703,"handle_client_caps_dirty":2999,"handle_client_cap_release":3,"process_request_cap_release":2071},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":1823,"num_strays_delayed":81,"num_strays_enqueuing":5,"strays_created":2063,"strays_enqueued":240,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":9627,"evex":0,"evtrm":0,"ev":9630,"evexg":0,"evexd":0,"segadd":10,"segex":0,"segtrm":0,"seg":12,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":26107362,"rdpos":4195244,"jlat":{"avgcount":13,"sum":0.869205778,"avgtime":0.066861982},"replayed":3},"mds_mem":{"ino":3214,"ino+":3214,"ino-":0,"dir":13,"dir+":13,"dir-":0,"dn":5273,"dn+":5273,"dn-":0,"cap":2876,"cap+":3202,"cap-":326,"rss":110572,"heap":248164},"mds_server":{"dispatch_client_request":5274,"dispatch_server_request":0,"handle_client_request":5274,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":2062,"sum":3.307811646,"avgtime":0.001604176},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":4974,"average_load":4974,"avg_session_uptime":1106,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":25580167,"buffer_anon_items":11809,"buffer_meta_bytes":97328,"buffer_meta_items":1106,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":14522825,"mds_co_items":137731,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":167,"op_send_bytes":21341261,"op_resend":0,"op_reply":167,"op_latency":{"avgcount":167,"sum":0.837982344,"avgtime":0.005017858},"op_inflight":0,"oplen_avg":{"avgcount":167,"sum":223},"op":167,"op_r":42,"op_w":125,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":14,"osdop_write":19,"osdop_writefull":4,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":92,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":17,"omap_rd":48,"omap_del":2},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":1,"pq_executing_ops_high_water":6,"pq_executing":1,"pq_executing_high_water":6,"pq_executed_ops":75,"pq_executed":75,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":254,"max":104857600,"get_started":0,"get":9184,"get_sum":2578051,"get_or_fail_fail":0,"get_or_fail_success":9184,"take":0,"take_sum":0,"put":9183,"put_sum":2577797,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":167,"take_sum":21366940,"put":167,"put_sum":21366940,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":167,"take_sum":167,"put":167,"put_sum":167,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":245,"get_sum":24745,"get_or_fail_fail":0,"get_or_fail_success":245,"take":0,"take_sum":0,"put":6,"put_sum":24745,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":941538,"max":3758096384,"get_started":0,"get":9627,"get_sum":21911860,"get_or_fail_fail":0,"get_or_fail_success":9627,"take":0,"take_sum":0,"put":13,"put_sum":20970322,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:34.369 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 23258, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 58, "msgr_running_fast_dispatch_time": 0.005097966, "msgr_running_recv_time": 0.005854737, "msgr_running_send_time": 0.017174276, "msgr_running_total_time": 0.032718367, "msgr_send_bytes": 3657587, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 59, "msgr_send_messages_queue_lat": { "avgcount": 58, "avgtime": 0.000172996, "sum": 0.01003379 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 20296, "avgtime": 4.37e-07, "sum": 0.008889322 }, "msgr_recv_bytes": 3238998, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 9067, "msgr_running_fast_dispatch_time": 0.013604224, "msgr_running_recv_time": 1.29110998, "msgr_running_send_time": 1.194402132, "msgr_running_total_time": 1.815744542, "msgr_send_bytes": 23372626, "msgr_send_encrypted_bytes": 92448, "msgr_send_messages": 16534, "msgr_send_messages_queue_lat": { "avgcount": 16532, "avgtime": 3.9364e-05, "sum": 0.650771629 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 145185, "msgr_recv_encrypted_bytes": 109424, "msgr_recv_messages": 64, "msgr_running_fast_dispatch_time": 0.001606923, "msgr_running_recv_time": 0.01785762, "msgr_running_send_time": 0.004512827, "msgr_running_total_time": 0.019519001, "msgr_send_bytes": 44997, "msgr_send_encrypted_bytes": 7744, "msgr_send_messages": 57, "msgr_send_messages_queue_lat": { "avgcount": 57, "avgtime": 0.000315881, "sum": 0.018005224 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 78, "avgtime": 0.058648255, "sum": 4.574563896 }, "queue_len": 1477 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 154, "avgtime": 0.001420233, "sum": 0.218715884 }, "queue_len": 121 }, "mds": { "caps": 2876, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 4094, "ceph_cap_op_revoke": 1, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 3, "handle_client_caps": 3703, "handle_client_caps_dirty": 2999, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 5273, "inodes_bottom": 5273, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 5273, "inodes_top": 0, "inodes_with_caps": 2876, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 2071, "q": 0, "reply": 5271, "reply_latency": { "avgcount": 5271, "avgtime": 0.120422573, "sum": 634.747383577 }, "request": 5274, "root_rbytes": 21040128, "root_rfiles": 2736, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 5275, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 5275, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 1823, "num_strays_delayed": 81, "num_strays_enqueuing": 5, "recovery_completed": 0, "recovery_started": 0, "strays_created": 2063, "strays_enqueued": 240, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 9630, "evadd": 9627, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 13, "avgtime": 0.066861982, "sum": 0.869205778 }, "rdpos": 4195244, "replayed": 3, "seg": 12, "segadd": 10, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 26107362 }, "mds_mem": { "cap": 2876, "cap+": 3202, "cap-": 326, "dir": 13, "dir+": 13, "dir-": 0, "dn": 5273, "dn+": 5273, "dn-": 0, "heap": 248164, "ino": 3214, "ino+": 3214, "ino-": 0, "rss": 110572 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 5274, "dispatch_server_request": 0, "handle_client_request": 5274, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 2062, "avgtime": 0.001604176, "sum": 3.307811646 } }, "mds_sessions": { "average_load": 4974, "avg_session_uptime": 1106, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 4974 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 25580167, "buffer_anon_items": 11809, "buffer_meta_bytes": 97328, "buffer_meta_items": 1106, "mds_co_bytes": 14522825, "mds_co_items": 137731, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 2, "omap_rd": 48, "omap_wr": 17, "op": 167, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 167, "avgtime": 0.005017858, "sum": 0.837982344 }, "op_pg": 0, "op_r": 42, "op_reply": 167, "op_resend": 0, "op_rmw": 0, "op_send": 167, "op_send_bytes": 21341261, "op_w": 125, "oplen_avg": { "avgcount": 167, "sum": 223 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 92, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 14, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 19, "osdop_writefull": 4, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 75, "pq_executed_ops": 75, "pq_executing": 1, "pq_executing_high_water": 6, "pq_executing_ops": 1, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 9184, "get_or_fail_fail": 0, "get_or_fail_success": 9184, "get_started": 0, "get_sum": 2578051, "max": 104857600, "put": 9183, "put_sum": 2577797, "take": 0, "take_sum": 0, "val": 254, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 167, "put_sum": 21366940, "take": 167, "take_sum": 21366940, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 167, "put_sum": 167, "take": 167, "take_sum": 167, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 245, "get_or_fail_fail": 0, "get_or_fail_success": 245, "get_started": 0, "get_sum": 24745, "max": 3758096384, "put": 6, "put_sum": 24745, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 9627, "get_or_fail_fail": 0, "get_or_fail_success": 9627, "get_started": 0, "get_sum": 21911860, "max": 3758096384, "put": 13, "put_sum": 20970322, "take": 0, "take_sum": 0, "val": 941538, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:34.369 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 1/1823, 75/2063 2024-03-06T07:32:35.371 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:35.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.592+0000 7fcdaec00640 1 Processor -- start 2024-03-06T07:32:35.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.592+0000 7fcdaec00640 1 -- start start 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.593+0000 7fcdaec00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fcda80d0780 0x7fcda80b49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.593+0000 7fcdaec00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fcda80acd60 0x7fcda815fb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.593+0000 7fcdaec00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcda80b5160 con 0x7fcda815be90 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.593+0000 7fcdaec00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcda80b4f30 con 0x7fcda80d0780 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.593+0000 7fcdaec00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcda80af240 con 0x7fcda80acd60 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.593+0000 7fcda7400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fcda80d0780 0x7fcda80b49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.593+0000 7fcda7400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fcda80d0780 0x7fcda80b49f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55382/0 (socket says 172.21.15.86:55382) 2024-03-06T07:32:35.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.594+0000 7fcda7400640 1 -- 172.21.15.86:0/3553681754 learned_addr learned my addr 172.21.15.86:0/3553681754 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.594+0000 7fcdacc00640 1 --2- 172.21.15.86:0/3553681754 >> v2:172.21.15.194:3300/0 conn(0x7fcda80acd60 0x7fcda815fb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.594+0000 7fcda7400640 1 -- 172.21.15.86:0/3553681754 >> v2:172.21.15.194:3300/0 conn(0x7fcda80acd60 msgr2=0x7fcda815fb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.594+0000 7fcda7400640 1 --2- 172.21.15.86:0/3553681754 >> v2:172.21.15.194:3300/0 conn(0x7fcda80acd60 0x7fcda815fb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.594+0000 7fcda7400640 1 -- 172.21.15.86:0/3553681754 >> v1:172.21.15.194:6789/0 conn(0x7fcda815be90 legacy=0x7fcda80d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.594+0000 7fcda7400640 1 -- 172.21.15.86:0/3553681754 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcda80ae910 con 0x7fcda80d0780 2024-03-06T07:32:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.594+0000 7fcdacc00640 1 --2- 172.21.15.86:0/3553681754 >> v2:172.21.15.194:3300/0 conn(0x7fcda80acd60 0x7fcda815fb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:35.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.595+0000 7fcda7400640 1 --2- 172.21.15.86:0/3553681754 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda80b49f0 secure :-1 s=READY pgs=1144 cs=0 l=1 rev1=1 crypto rx=0x7fcda80b4650 tx=0x7fcd9402b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b0a6ba64e1965296 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:35.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.595+0000 7fcda6a00640 1 -- 172.21.15.86:0/3553681754 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd9402e070 con 0x7fcda80d0780 2024-03-06T07:32:35.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.595+0000 7fcda6a00640 1 -- 172.21.15.86:0/3553681754 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcd94037070 con 0x7fcda80d0780 2024-03-06T07:32:35.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.595+0000 7fcda6a00640 1 -- 172.21.15.86:0/3553681754 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd94031a00 con 0x7fcda80d0780 2024-03-06T07:32:35.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.596+0000 7fcdaec00640 1 -- 172.21.15.86:0/3553681754 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 msgr2=0x7fcda80b49f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:35.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.596+0000 7fcdaec00640 1 --2- 172.21.15.86:0/3553681754 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda80b49f0 secure :-1 s=READY pgs=1144 cs=0 l=1 rev1=1 crypto rx=0x7fcda80b4650 tx=0x7fcd9402b880 comp rx=0 tx=0).stop 2024-03-06T07:32:35.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.597+0000 7fcdaec00640 1 -- 172.21.15.86:0/3553681754 shutdown_connections 2024-03-06T07:32:35.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.597+0000 7fcdaec00640 1 --2- 172.21.15.86:0/3553681754 >> v2:172.21.15.194:3300/0 conn(0x7fcda80acd60 0x7fcda815fb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:35.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.597+0000 7fcdaec00640 1 --2- 172.21.15.86:0/3553681754 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda80b49f0 unknown :-1 s=CLOSED pgs=1144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:35.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.597+0000 7fcdaec00640 1 -- 172.21.15.86:0/3553681754 >> 172.21.15.86:0/3553681754 conn(0x7fcda806b360 msgr2=0x7fcda806b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:35.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.597+0000 7fcdaec00640 1 -- 172.21.15.86:0/3553681754 shutdown_connections 2024-03-06T07:32:35.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.597+0000 7fcdaec00640 1 -- 172.21.15.86:0/3553681754 wait complete. 2024-03-06T07:32:35.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.599+0000 7fcdaec00640 1 Processor -- start 2024-03-06T07:32:35.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.599+0000 7fcdaec00640 1 -- start start 2024-03-06T07:32:35.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.600+0000 7fcdaec00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcda80acd60 0x7fcda81f4190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:35.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.600+0000 7fcdaec00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda81f46d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:35.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.600+0000 7fcdaec00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcda815be90 0x7fcda814f3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:35.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.600+0000 7fcdaec00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcda80b2c80 con 0x7fcda80d0780 2024-03-06T07:32:35.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.600+0000 7fcdaec00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcda80aeed0 con 0x7fcda815be90 2024-03-06T07:32:35.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.600+0000 7fcdaec00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcda80b2e00 con 0x7fcda80acd60 2024-03-06T07:32:35.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda81f46d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:35.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda81f46d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55392/0 (socket says 172.21.15.86:55392) 2024-03-06T07:32:35.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 -- 172.21.15.86:0/4276158090 learned_addr learned my addr 172.21.15.86:0/4276158090 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:35.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcdacc00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcda815be90 0x7fcda814f3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:35.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7e00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcda80acd60 0x7fcda81f4190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:35.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 -- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcda80acd60 msgr2=0x7fcda81f4190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:35.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcda80acd60 0x7fcda81f4190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:35.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 -- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcda815be90 msgr2=0x7fcda814f3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:35.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcda815be90 0x7fcda814f3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:35.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcda7400640 1 -- 172.21.15.86:0/4276158090 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcda815a3e0 con 0x7fcda80d0780 2024-03-06T07:32:35.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.601+0000 7fcdacc00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcda815be90 0x7fcda814f3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:32:35.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.602+0000 7fcda7400640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda81f46d0 secure :-1 s=READY pgs=1145 cs=0 l=1 rev1=1 crypto rx=0x7fcd940090b0 tx=0x7fcd940046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:35.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.603+0000 7fcd9fe00640 1 -- 172.21.15.86:0/4276158090 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd94032880 con 0x7fcda80d0780 2024-03-06T07:32:35.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.603+0000 7fcd9fe00640 1 -- 172.21.15.86:0/4276158090 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcd94032a40 con 0x7fcda80d0780 2024-03-06T07:32:35.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.603+0000 7fcd9fe00640 1 -- 172.21.15.86:0/4276158090 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd94032e30 con 0x7fcda80d0780 2024-03-06T07:32:35.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.603+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcda8158490 con 0x7fcda80d0780 2024-03-06T07:32:35.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.603+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcda8153460 con 0x7fcda80d0780 2024-03-06T07:32:35.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.604+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcda81536a0 con 0x7fcda80d0780 2024-03-06T07:32:35.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.605+0000 7fcd9fe00640 1 -- 172.21.15.86:0/4276158090 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcd9404d3e0 con 0x7fcda80d0780 2024-03-06T07:32:35.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.606+0000 7fcd9fe00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcd8806b550 0x7fcd8806da10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:35.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.606+0000 7fcd9fe00640 1 -- 172.21.15.86:0/4276158090 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fcd940ac2e0 con 0x7fcda80d0780 2024-03-06T07:32:35.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.607+0000 7fcda7e00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcd8806b550 0x7fcd8806da10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:35.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.607+0000 7fcda7e00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcd8806b550 0x7fcd8806da10 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7fcd98006fd0 tx=0x7fcd98008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:35.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:35.615+0000 7fcd9fe00640 1 -- 172.21.15.86:0/4276158090 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcd94040070 con 0x7fcda80d0780 2024-03-06T07:32:36.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.212+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fcda8152f70 con 0x7fcda80d0780 2024-03-06T07:32:36.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.215+0000 7fcd9fe00640 1 -- 172.21.15.86:0/4276158090 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7fcd9407b120 con 0x7fcda80d0780 2024-03-06T07:32:36.219 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:36.219 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:36.219 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:36.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.220+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcd8806b550 msgr2=0x7fcd8806da10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:36.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.220+0000 7fcdaec00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcd8806b550 0x7fcd8806da10 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7fcd98006fd0 tx=0x7fcd98008040 comp rx=0 tx=0).stop 2024-03-06T07:32:36.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.220+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 msgr2=0x7fcda81f46d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:36.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.220+0000 7fcdaec00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda81f46d0 secure :-1 s=READY pgs=1145 cs=0 l=1 rev1=1 crypto rx=0x7fcd940090b0 tx=0x7fcd940046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:36.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.221+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 shutdown_connections 2024-03-06T07:32:36.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.221+0000 7fcdaec00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcd8806b550 0x7fcd8806da10 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:36.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.221+0000 7fcdaec00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcda815be90 0x7fcda814f3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:36.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.221+0000 7fcdaec00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcda80d0780 0x7fcda81f46d0 unknown :-1 s=CLOSED pgs=1145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:36.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.221+0000 7fcdaec00640 1 --2- 172.21.15.86:0/4276158090 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcda80acd60 0x7fcda81f4190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:36.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.221+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 >> 172.21.15.86:0/4276158090 conn(0x7fcda806b360 msgr2=0x7fcda8157560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:36.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.222+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 shutdown_connections 2024-03-06T07:32:36.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:36.222+0000 7fcdaec00640 1 -- 172.21.15.86:0/4276158090 wait complete. 2024-03-06T07:32:36.248 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:36.501 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":307,"msgr_send_messages":308,"msgr_recv_bytes":86504,"msgr_send_bytes":3738761,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.068822869,"msgr_running_send_time":0.033734207,"msgr_running_recv_time":0.013406398,"msgr_running_fast_dispatch_time":0.015024263,"msgr_send_messages_queue_lat":{"avgcount":307,"sum":0.057725759,"avgtime":0.000188031},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":11772,"msgr_send_messages":19111,"msgr_recv_bytes":4200162,"msgr_send_bytes":24184653,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":2.155218802,"msgr_running_send_time":1.412951158,"msgr_running_recv_time":1.652964349,"msgr_running_fast_dispatch_time":0.018399864,"msgr_send_messages_queue_lat":{"avgcount":19109,"sum":0.977544114,"avgtime":0.000051156},"msgr_handle_ack_lat":{"avgcount":24406,"sum":0.009671919,"avgtime":0.000000396},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":94240},"AsyncMessenger::Worker-2":{"msgr_recv_messages":237,"msgr_send_messages":230,"msgr_recv_bytes":226007,"msgr_send_bytes":184753,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.056392530,"msgr_running_send_time":0.021653061,"msgr_running_recv_time":0.023212069,"msgr_running_fast_dispatch_time":0.014123610,"msgr_send_messages_queue_lat":{"avgcount":230,"sum":0.048637554,"avgtime":0.000211467},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109696,"msgr_send_encrypted_bytes":8368},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":1478,"complete_latency":{"avgcount":78,"sum":4.574563896,"avgtime":0.058648255}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":1076,"sum":1.116688878,"avgtime":0.001037814}},"mds":{"request":5847,"reply":5845,"reply_latency":{"avgcount":5845,"sum":636.642211145,"avgtime":0.108920823},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":5772,"inodes_top":0,"inodes_bottom":5772,"inodes_pin_tail":0,"inodes_pinned":5772,"inodes_expired":0,"inodes_with_caps":2452,"caps":2452,"subtrees":2,"traverse":5848,"traverse_hit":5848,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":1576,"root_rbytes":12168192,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":1,"ceph_cap_op_grant":5073,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":4023,"handle_client_caps_dirty":2999,"handle_client_cap_release":9,"process_request_cap_release":2644},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":2025,"num_strays_delayed":6,"num_strays_enqueuing":0,"strays_created":2636,"strays_enqueued":611,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":11001,"evex":0,"evtrm":0,"ev":11004,"evexg":0,"evexd":0,"segadd":11,"segex":0,"segtrm":0,"seg":13,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":28994026,"rdpos":4195244,"jlat":{"avgcount":13,"sum":0.869205778,"avgtime":0.066861982},"replayed":3},"mds_mem":{"ino":3139,"ino+":3214,"ino-":75,"dir":13,"dir+":13,"dir-":0,"dn":5772,"dn+":5847,"dn-":75,"cap":2452,"cap+":3202,"cap-":750,"rss":111084,"heap":248164},"mds_server":{"dispatch_client_request":5847,"dispatch_server_request":0,"handle_client_request":5847,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":2636,"sum":5.202639214,"avgtime":0.001973687},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5419,"average_load":5419,"avg_session_uptime":1111,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":9845520,"buffer_anon_items":6706,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":16556288,"mds_co_items":226538,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":715,"op_send_bytes":22773370,"op_resend":0,"op_reply":715,"op_latency":{"avgcount":715,"sum":5.471884781,"avgtime":0.007652985},"op_inflight":0,"oplen_avg":{"avgcount":715,"sum":771},"op":715,"op_r":47,"op_w":668,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":19,"osdop_write":25,"osdop_writefull":6,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":627,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":17,"omap_rd":48,"omap_del":2},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":611,"pq_executed":611,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":455238,"max":104857600,"get_started":0,"get":12311,"get_sum":3401801,"get_or_fail_fail":0,"get_or_fail_success":12311,"take":0,"take_sum":0,"put":10634,"put_sum":2946563,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":715,"take_sum":22836015,"put":715,"put_sum":22836015,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":715,"take_sum":715,"put":715,"put_sum":715,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":611,"get_sum":61711,"get_or_fail_fail":0,"get_or_fail_success":611,"take":0,"take_sum":0,"put":11,"put_sum":61711,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":2435235,"max":3758096384,"get_started":0,"get":11002,"get_sum":24800520,"get_or_fail_fail":0,"get_or_fail_success":11002,"take":0,"take_sum":0,"put":14,"put_sum":22365285,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:36.503 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 86504, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 307, "msgr_running_fast_dispatch_time": 0.015024263, "msgr_running_recv_time": 0.013406398, "msgr_running_send_time": 0.033734207, "msgr_running_total_time": 0.068822869, "msgr_send_bytes": 3738761, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 308, "msgr_send_messages_queue_lat": { "avgcount": 307, "avgtime": 0.000188031, "sum": 0.057725759 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 24406, "avgtime": 3.96e-07, "sum": 0.009671919 }, "msgr_recv_bytes": 4200162, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 11772, "msgr_running_fast_dispatch_time": 0.018399864, "msgr_running_recv_time": 1.652964349, "msgr_running_send_time": 1.412951158, "msgr_running_total_time": 2.155218802, "msgr_send_bytes": 24184653, "msgr_send_encrypted_bytes": 94240, "msgr_send_messages": 19111, "msgr_send_messages_queue_lat": { "avgcount": 19109, "avgtime": 5.1156e-05, "sum": 0.977544114 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 226007, "msgr_recv_encrypted_bytes": 109696, "msgr_recv_messages": 237, "msgr_running_fast_dispatch_time": 0.01412361, "msgr_running_recv_time": 0.023212069, "msgr_running_send_time": 0.021653061, "msgr_running_total_time": 0.05639253, "msgr_send_bytes": 184753, "msgr_send_encrypted_bytes": 8368, "msgr_send_messages": 230, "msgr_send_messages_queue_lat": { "avgcount": 230, "avgtime": 0.000211467, "sum": 0.048637554 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 78, "avgtime": 0.058648255, "sum": 4.574563896 }, "queue_len": 1478 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 1076, "avgtime": 0.001037814, "sum": 1.116688878 }, "queue_len": 0 }, "mds": { "caps": 2452, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 5073, "ceph_cap_op_revoke": 1, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 9, "handle_client_caps": 4023, "handle_client_caps_dirty": 2999, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 5772, "inodes_bottom": 5772, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 5772, "inodes_top": 0, "inodes_with_caps": 2452, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 2644, "q": 0, "reply": 5845, "reply_latency": { "avgcount": 5845, "avgtime": 0.108920823, "sum": 636.642211145 }, "request": 5847, "root_rbytes": 12168192, "root_rfiles": 1576, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 5848, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 5848, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 2025, "num_strays_delayed": 6, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 2636, "strays_enqueued": 611, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 11004, "evadd": 11001, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 13, "avgtime": 0.066861982, "sum": 0.869205778 }, "rdpos": 4195244, "replayed": 3, "seg": 13, "segadd": 11, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 28994026 }, "mds_mem": { "cap": 2452, "cap+": 3202, "cap-": 750, "dir": 13, "dir+": 13, "dir-": 0, "dn": 5772, "dn+": 5847, "dn-": 75, "heap": 248164, "ino": 3139, "ino+": 3214, "ino-": 75, "rss": 111084 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 5847, "dispatch_server_request": 0, "handle_client_request": 5847, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 2636, "avgtime": 0.001973687, "sum": 5.202639214 } }, "mds_sessions": { "average_load": 5419, "avg_session_uptime": 1111, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5419 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 9845520, "buffer_anon_items": 6706, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 16556288, "mds_co_items": 226538, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 2, "omap_rd": 48, "omap_wr": 17, "op": 715, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 715, "avgtime": 0.007652985, "sum": 5.471884781 }, "op_pg": 0, "op_r": 47, "op_reply": 715, "op_resend": 0, "op_rmw": 0, "op_send": 715, "op_send_bytes": 22773370, "op_w": 668, "oplen_avg": { "avgcount": 715, "sum": 771 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 627, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 19, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 25, "osdop_writefull": 6, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 611, "pq_executed_ops": 611, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 12311, "get_or_fail_fail": 0, "get_or_fail_success": 12311, "get_started": 0, "get_sum": 3401801, "max": 104857600, "put": 10634, "put_sum": 2946563, "take": 0, "take_sum": 0, "val": 455238, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 715, "put_sum": 22836015, "take": 715, "take_sum": 22836015, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 715, "put_sum": 715, "take": 715, "take_sum": 715, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 611, "get_or_fail_fail": 0, "get_or_fail_success": 611, "get_started": 0, "get_sum": 61711, "max": 3758096384, "put": 11, "put_sum": 61711, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 11002, "get_or_fail_fail": 0, "get_or_fail_success": 11002, "get_started": 0, "get_sum": 24800520, "max": 3758096384, "put": 14, "put_sum": 22365285, "take": 0, "take_sum": 0, "val": 2435235, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:36.503 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/2025, 611/2636 2024-03-06T07:32:37.504 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:37.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.746+0000 7f0ca3600640 1 Processor -- start 2024-03-06T07:32:37.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.746+0000 7f0ca3600640 1 -- start start 2024-03-06T07:32:37.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.748+0000 7f0ca3600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0c9c155490 0x7f0c9c0ae630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:37.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.748+0000 7f0ca3600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c0d0070 0x7f0c9c0d04f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:37.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.748+0000 7f0ca3600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0c9c155870 con 0x7f0c9c15ec10 2024-03-06T07:32:37.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.748+0000 7f0ca3600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0c9c0ad640 con 0x7f0c9c155490 2024-03-06T07:32:37.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.748+0000 7f0ca3600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c15f980 con 0x7f0c9c0d0070 2024-03-06T07:32:37.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.748+0000 7f0c9be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c0d0070 0x7f0c9c0d04f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:37.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.748+0000 7f0ca0c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0c9c155490 0x7f0c9c0ae630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:37.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.749+0000 7f0c9be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c0d0070 0x7f0c9c0d04f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:42476/0 (socket says 172.21.15.86:42476) 2024-03-06T07:32:37.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.749+0000 7f0c9be00640 1 -- 172.21.15.86:0/2525782672 learned_addr learned my addr 172.21.15.86:0/2525782672 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:37.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.749+0000 7f0ca0c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0c9c155490 0x7f0c9c0ae630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:42748/0 (socket says 172.21.15.86:42748) 2024-03-06T07:32:37.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.749+0000 7f0ca0c00640 1 -- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c0d0070 msgr2=0x7f0c9c0d04f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:37.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.749+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c0d0070 0x7f0c9c0d04f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:37.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.749+0000 7f0ca0c00640 1 -- 172.21.15.86:0/2525782672 >> v1:172.21.15.194:6789/0 conn(0x7f0c9c15ec10 legacy=0x7f0c9c15f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:37.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.749+0000 7f0ca0c00640 1 -- 172.21.15.86:0/2525782672 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c9c0ad870 con 0x7f0c9c155490 2024-03-06T07:32:37.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.750+0000 7f0c9be00640 1 --2- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c0d0070 0x7f0c9c0d04f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:32:37.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.751+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c155490 0x7f0c9c0ae630 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f0c880097a0 tx=0x7f0c8802b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=384a2ee83b7345bf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:37.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.752+0000 7f0c9b400640 1 -- 172.21.15.86:0/2525782672 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c88035020 con 0x7f0c9c155490 2024-03-06T07:32:37.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.752+0000 7f0c9b400640 1 -- 172.21.15.86:0/2525782672 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0c88031810 con 0x7f0c9c155490 2024-03-06T07:32:37.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.752+0000 7f0c9b400640 1 -- 172.21.15.86:0/2525782672 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c88031be0 con 0x7f0c9c155490 2024-03-06T07:32:37.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.752+0000 7f0ca3600640 1 -- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c155490 msgr2=0x7f0c9c0ae630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:37.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.752+0000 7f0ca3600640 1 --2- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c155490 0x7f0c9c0ae630 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f0c880097a0 tx=0x7f0c8802b8b0 comp rx=0 tx=0).stop 2024-03-06T07:32:37.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.753+0000 7f0ca3600640 1 -- 172.21.15.86:0/2525782672 shutdown_connections 2024-03-06T07:32:37.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.753+0000 7f0ca3600640 1 --2- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c0d0070 0x7f0c9c0d04f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:37.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.753+0000 7f0ca3600640 1 --2- 172.21.15.86:0/2525782672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c155490 0x7f0c9c0ae630 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:37.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.753+0000 7f0ca3600640 1 -- 172.21.15.86:0/2525782672 >> 172.21.15.86:0/2525782672 conn(0x7f0c9c06b360 msgr2=0x7f0c9c06b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:37.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.753+0000 7f0ca3600640 1 -- 172.21.15.86:0/2525782672 shutdown_connections 2024-03-06T07:32:37.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.753+0000 7f0ca3600640 1 -- 172.21.15.86:0/2525782672 wait complete. 2024-03-06T07:32:37.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.755+0000 7f0ca3600640 1 Processor -- start 2024-03-06T07:32:37.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.755+0000 7f0ca3600640 1 -- start start 2024-03-06T07:32:37.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca3600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c0d0070 0x7f0c9c151b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca3600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 0x7f0c9c14a890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca3600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c15ec10 0x7f0c9c0affe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca3600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c0ae110 con 0x7f0c9c0d0070 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca3600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c0ade30 con 0x7f0c9c15ec10 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca3600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c071960 con 0x7f0c9c155490 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c0d0070 0x7f0c9c151b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c0d0070 0x7f0c9c151b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33412/0 (socket says 172.21.15.86:33412) 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca0c00640 1 -- 172.21.15.86:0/772092080 learned_addr learned my addr 172.21.15.86:0/772092080 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0c9be00640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 0x7f0c9c14a890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.757+0000 7f0ca1600640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c15ec10 0x7f0c9c0affe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.758+0000 7f0ca0c00640 1 -- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 msgr2=0x7f0c9c14a890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.758+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 0x7f0c9c14a890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.758+0000 7f0ca0c00640 1 -- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c15ec10 msgr2=0x7f0c9c0affe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.758+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c15ec10 0x7f0c9c0affe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.758+0000 7f0ca0c00640 1 -- 172.21.15.86:0/772092080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c9c0b1a00 con 0x7f0c9c0d0070 2024-03-06T07:32:37.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.758+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c0d0070 0x7f0c9c151b50 secure :-1 s=READY pgs=1146 cs=0 l=1 rev1=1 crypto rx=0x7f0c8802bec0 tx=0x7f0c880046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:37.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.759+0000 7f0c98c00640 1 -- 172.21.15.86:0/772092080 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c88041070 con 0x7f0c9c0d0070 2024-03-06T07:32:37.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.759+0000 7f0c98c00640 1 -- 172.21.15.86:0/772092080 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0c8803ad00 con 0x7f0c9c0d0070 2024-03-06T07:32:37.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.759+0000 7f0c98c00640 1 -- 172.21.15.86:0/772092080 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c88035040 con 0x7f0c9c0d0070 2024-03-06T07:32:37.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.759+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c9c0b1ce0 con 0x7f0c9c0d0070 2024-03-06T07:32:37.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.759+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c9c0b22a0 con 0x7f0c9c0d0070 2024-03-06T07:32:37.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.760+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c9c068070 con 0x7f0c9c0d0070 2024-03-06T07:32:37.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.762+0000 7f0c98c00640 1 -- 172.21.15.86:0/772092080 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0c88038020 con 0x7f0c9c0d0070 2024-03-06T07:32:37.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.763+0000 7f0c98c00640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7c06b520 0x7f0c7c06d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:37.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.763+0000 7f0c98c00640 1 -- 172.21.15.86:0/772092080 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f0c880abf80 con 0x7f0c9c0d0070 2024-03-06T07:32:37.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.764+0000 7f0c9be00640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7c06b520 0x7f0c7c06d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:37.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.765+0000 7f0c9be00640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7c06b520 0x7f0c7c06d9e0 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c154b50 tx=0x7f0c90008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:37.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:37.770+0000 7f0c98c00640 1 -- 172.21.15.86:0/772092080 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0c8807aca0 con 0x7f0c9c0d0070 2024-03-06T07:32:38.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.056+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0c9c071740 con 0x7f0c9c0d0070 2024-03-06T07:32:38.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.056+0000 7f0c98c00640 1 -- 172.21.15.86:0/772092080 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f0c8807a5a0 con 0x7f0c9c0d0070 2024-03-06T07:32:38.059 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:38.059 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:38.059 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:38.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7c06b520 msgr2=0x7f0c7c06d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:38.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7c06b520 0x7f0c7c06d9e0 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c154b50 tx=0x7f0c90008040 comp rx=0 tx=0).stop 2024-03-06T07:32:38.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c0d0070 msgr2=0x7f0c9c151b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:38.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c0d0070 0x7f0c9c151b50 secure :-1 s=READY pgs=1146 cs=0 l=1 rev1=1 crypto rx=0x7f0c8802bec0 tx=0x7f0c880046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:38.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 shutdown_connections 2024-03-06T07:32:38.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7c06b520 0x7f0c7c06d9e0 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:38.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c15ec10 0x7f0c9c0affe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:38.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 0x7f0c9c14a890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:38.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 --2- 172.21.15.86:0/772092080 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c0d0070 0x7f0c9c151b50 secure :-1 s=CLOSED pgs=1146 cs=0 l=1 rev1=1 crypto rx=0x7f0c8802bec0 tx=0x7f0c880046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:38.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.060+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 >> 172.21.15.86:0/772092080 conn(0x7f0c9c06b360 msgr2=0x7f0c9c0b3aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:38.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.061+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 shutdown_connections 2024-03-06T07:32:38.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:38.061+0000 7f0ca3600640 1 -- 172.21.15.86:0/772092080 wait complete. 2024-03-06T07:32:38.083 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:38.332 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":310,"msgr_send_messages":311,"msgr_recv_bytes":87258,"msgr_send_bytes":3739731,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.069109032,"msgr_running_send_time":0.033865861,"msgr_running_recv_time":0.013466250,"msgr_running_fast_dispatch_time":0.015093449,"msgr_send_messages_queue_lat":{"avgcount":310,"sum":0.057825970,"avgtime":0.000186535},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":12864,"msgr_send_messages":21912,"msgr_recv_bytes":4395854,"msgr_send_bytes":27187202,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":2.400352619,"msgr_running_send_time":1.563593045,"msgr_running_recv_time":1.770878685,"msgr_running_fast_dispatch_time":0.020425839,"msgr_send_messages_queue_lat":{"avgcount":21910,"sum":1.069223419,"avgtime":0.000048800},"msgr_handle_ack_lat":{"avgcount":27429,"sum":0.010408948,"avgtime":0.000000379},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":94240},"AsyncMessenger::Worker-2":{"msgr_recv_messages":254,"msgr_send_messages":247,"msgr_recv_bytes":241549,"msgr_send_bytes":1159089,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.073268720,"msgr_running_send_time":0.034707473,"msgr_running_recv_time":0.023707789,"msgr_running_fast_dispatch_time":0.017308417,"msgr_send_messages_queue_lat":{"avgcount":247,"sum":0.051660139,"avgtime":0.000209150},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109696,"msgr_send_encrypted_bytes":8368},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":718,"complete_latency":{"avgcount":86,"sum":8.623975316,"avgtime":0.100278782}},"finisher-PurgeQueue":{"queue_len":100,"complete_latency":{"avgcount":1108,"sum":1.181012581,"avgtime":0.001065895}},"mds":{"request":6027,"reply":6024,"reply_latency":{"avgcount":6024,"sum":638.448662826,"avgtime":0.105984173},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":5237,"inodes_top":0,"inodes_bottom":5237,"inodes_pin_tail":0,"inodes_pinned":5237,"inodes_expired":0,"inodes_with_caps":1248,"caps":1248,"subtrees":2,"traverse":6028,"traverse_hit":6027,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":1123,"root_rbytes":8722432,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":69,"ceph_cap_op_grant":6761,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":5619,"handle_client_caps_dirty":3000,"handle_client_cap_release":988,"process_request_cap_release":2824},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":2101,"num_strays_delayed":76,"num_strays_enqueuing":1607,"strays_created":2815,"strays_enqueued":714,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":12651,"evex":0,"evtrm":0,"ev":12654,"evexg":0,"evexd":0,"segadd":14,"segex":0,"segtrm":0,"seg":16,"segmjr":3,"segexg":0,"segexd":0,"expos":4194304,"wrpos":32255130,"rdpos":4195244,"jlat":{"avgcount":18,"sum":3.584236429,"avgtime":0.199124246},"replayed":3},"mds_mem":{"ino":2589,"ino+":3214,"ino-":625,"dir":13,"dir+":13,"dir-":0,"dn":5401,"dn+":6026,"dn-":625,"cap":566,"cap+":3202,"cap-":2636,"rss":111340,"heap":248164},"mds_server":{"dispatch_client_request":6027,"dispatch_server_request":0,"handle_client_request":6027,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":2815,"sum":7.009090895,"avgtime":0.002489907},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5482,"average_load":5482,"avg_session_uptime":1111,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":9845520,"buffer_anon_items":6706,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":16556288,"mds_co_items":226538,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":747,"op_send_bytes":27918210,"op_resend":0,"op_reply":747,"op_latency":{"avgcount":747,"sum":5.640881216,"avgtime":0.007551380},"op_inflight":0,"oplen_avg":{"avgcount":747,"sum":807},"op":747,"op_r":51,"op_w":696,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":23,"osdop_write":34,"osdop_writefull":8,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":642,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":21,"omap_rd":48,"omap_del":4},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":625,"pq_executed":625,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":253,"max":104857600,"get_started":0,"get":13423,"get_sum":3513709,"get_or_fail_fail":0,"get_or_fail_success":13423,"take":0,"take_sum":0,"put":13422,"put_sum":3513456,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":747,"take_sum":27992167,"put":747,"put_sum":27992167,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":747,"take_sum":747,"put":747,"put_sum":747,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":2321,"get_sum":234421,"get_or_fail_fail":0,"get_or_fail_success":2321,"take":0,"take_sum":0,"put":16,"put_sum":234421,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":813347,"max":3758096384,"get_started":0,"get":12651,"get_sum":28059628,"get_or_fail_fail":0,"get_or_fail_success":12651,"take":0,"take_sum":0,"put":18,"put_sum":27246281,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:38.333 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 87258, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 310, "msgr_running_fast_dispatch_time": 0.015093449, "msgr_running_recv_time": 0.01346625, "msgr_running_send_time": 0.033865861, "msgr_running_total_time": 0.069109032, "msgr_send_bytes": 3739731, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 311, "msgr_send_messages_queue_lat": { "avgcount": 310, "avgtime": 0.000186535, "sum": 0.05782597 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 27429, "avgtime": 3.79e-07, "sum": 0.010408948 }, "msgr_recv_bytes": 4395854, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 12864, "msgr_running_fast_dispatch_time": 0.020425839, "msgr_running_recv_time": 1.770878685, "msgr_running_send_time": 1.563593045, "msgr_running_total_time": 2.400352619, "msgr_send_bytes": 27187202, "msgr_send_encrypted_bytes": 94240, "msgr_send_messages": 21912, "msgr_send_messages_queue_lat": { "avgcount": 21910, "avgtime": 4.88e-05, "sum": 1.069223419 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 241549, "msgr_recv_encrypted_bytes": 109696, "msgr_recv_messages": 254, "msgr_running_fast_dispatch_time": 0.017308417, "msgr_running_recv_time": 0.023707789, "msgr_running_send_time": 0.034707473, "msgr_running_total_time": 0.07326872, "msgr_send_bytes": 1159089, "msgr_send_encrypted_bytes": 8368, "msgr_send_messages": 247, "msgr_send_messages_queue_lat": { "avgcount": 247, "avgtime": 0.00020915, "sum": 0.051660139 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 86, "avgtime": 0.100278782, "sum": 8.623975316 }, "queue_len": 718 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 1108, "avgtime": 0.001065895, "sum": 1.181012581 }, "queue_len": 100 }, "mds": { "caps": 1248, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 6761, "ceph_cap_op_revoke": 69, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 988, "handle_client_caps": 5619, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 5237, "inodes_bottom": 5237, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 5237, "inodes_top": 0, "inodes_with_caps": 1248, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 2824, "q": 0, "reply": 6024, "reply_latency": { "avgcount": 6024, "avgtime": 0.105984173, "sum": 638.448662826 }, "request": 6027, "root_rbytes": 8722432, "root_rfiles": 1123, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6028, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6027, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 2101, "num_strays_delayed": 76, "num_strays_enqueuing": 1607, "recovery_completed": 0, "recovery_started": 0, "strays_created": 2815, "strays_enqueued": 714, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 12654, "evadd": 12651, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 18, "avgtime": 0.199124246, "sum": 3.584236429 }, "rdpos": 4195244, "replayed": 3, "seg": 16, "segadd": 14, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 3, "segtrm": 0, "wrpos": 32255130 }, "mds_mem": { "cap": 566, "cap+": 3202, "cap-": 2636, "dir": 13, "dir+": 13, "dir-": 0, "dn": 5401, "dn+": 6026, "dn-": 625, "heap": 248164, "ino": 2589, "ino+": 3214, "ino-": 625, "rss": 111340 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6027, "dispatch_server_request": 0, "handle_client_request": 6027, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 2815, "avgtime": 0.002489907, "sum": 7.009090895 } }, "mds_sessions": { "average_load": 5482, "avg_session_uptime": 1111, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5482 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 9845520, "buffer_anon_items": 6706, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 16556288, "mds_co_items": 226538, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 4, "omap_rd": 48, "omap_wr": 21, "op": 747, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 747, "avgtime": 0.00755138, "sum": 5.640881216 }, "op_pg": 0, "op_r": 51, "op_reply": 747, "op_resend": 0, "op_rmw": 0, "op_send": 747, "op_send_bytes": 27918210, "op_w": 696, "oplen_avg": { "avgcount": 747, "sum": 807 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 642, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 23, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 34, "osdop_writefull": 8, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 625, "pq_executed_ops": 625, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 13423, "get_or_fail_fail": 0, "get_or_fail_success": 13423, "get_started": 0, "get_sum": 3513709, "max": 104857600, "put": 13422, "put_sum": 3513456, "take": 0, "take_sum": 0, "val": 253, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 747, "put_sum": 27992167, "take": 747, "take_sum": 27992167, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 747, "put_sum": 747, "take": 747, "take_sum": 747, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 2321, "get_or_fail_fail": 0, "get_or_fail_success": 2321, "get_started": 0, "get_sum": 234421, "max": 3758096384, "put": 16, "put_sum": 234421, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 12651, "get_or_fail_fail": 0, "get_or_fail_success": 12651, "get_started": 0, "get_sum": 28059628, "max": 3758096384, "put": 18, "put_sum": 27246281, "take": 0, "take_sum": 0, "val": 813347, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:38.333 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/2101, 625/2815 2024-03-06T07:32:39.335 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:39.570 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:39.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.577+0000 7f1e24800640 1 Processor -- start 2024-03-06T07:32:39.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.578+0000 7f1e24800640 1 -- start start 2024-03-06T07:32:39.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.580+0000 7f1e24800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1e1c15e980 0x7f1e1c15ed60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.580+0000 7f1e24800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1e1c15f2a0 0x7f1e1c0b34d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.580+0000 7f1e24800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1e1c0ccc30 con 0x7f1e1c06eff0 2024-03-06T07:32:39.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.580+0000 7f1e24800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1e1c061e80 con 0x7f1e1c15f2a0 2024-03-06T07:32:39.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.580+0000 7f1e24800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1e1c06e190 con 0x7f1e1c15e980 2024-03-06T07:32:39.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.581+0000 7f1e21400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1e1c15f2a0 0x7f1e1c0b34d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.581+0000 7f1e21400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1e1c15f2a0 0x7f1e1c0b34d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33422/0 (socket says 172.21.15.86:33422) 2024-03-06T07:32:39.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.581+0000 7f1e21400640 1 -- 172.21.15.86:0/2217978716 learned_addr learned my addr 172.21.15.86:0/2217978716 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:39.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.582+0000 7f1e21e00640 1 --2- 172.21.15.86:0/2217978716 >> v2:172.21.15.194:3300/0 conn(0x7f1e1c15e980 0x7f1e1c15ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.582+0000 7f1e21400640 1 -- 172.21.15.86:0/2217978716 >> v2:172.21.15.194:3300/0 conn(0x7f1e1c15e980 msgr2=0x7f1e1c15ed60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:39.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.583+0000 7f1e21400640 1 --2- 172.21.15.86:0/2217978716 >> v2:172.21.15.194:3300/0 conn(0x7f1e1c15e980 0x7f1e1c15ed60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.583+0000 7f1e21400640 1 -- 172.21.15.86:0/2217978716 >> v1:172.21.15.86:6789/0 conn(0x7f1e1c06eff0 legacy=0x7f1e1c1547c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:39.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.583+0000 7f1e21400640 1 -- 172.21.15.86:0/2217978716 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1e1c06d5c0 con 0x7f1e1c15f2a0 2024-03-06T07:32:39.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.585+0000 7f1e21400640 1 --2- 172.21.15.86:0/2217978716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c15f2a0 0x7f1e1c0b34d0 secure :-1 s=READY pgs=1147 cs=0 l=1 rev1=1 crypto rx=0x7f1e1c0b3190 tx=0x7f1e0c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=19c39aae8ab2e16 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:39.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.586+0000 7f1e20a00640 1 -- 172.21.15.86:0/2217978716 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e0c035020 con 0x7f1e1c15f2a0 2024-03-06T07:32:39.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.586+0000 7f1e20a00640 1 -- 172.21.15.86:0/2217978716 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1e0c031870 con 0x7f1e1c15f2a0 2024-03-06T07:32:39.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.586+0000 7f1e20a00640 1 -- 172.21.15.86:0/2217978716 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e0c031c40 con 0x7f1e1c15f2a0 2024-03-06T07:32:39.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.588+0000 7f1e24800640 1 -- 172.21.15.86:0/2217978716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c15f2a0 msgr2=0x7f1e1c0b34d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:39.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.588+0000 7f1e24800640 1 --2- 172.21.15.86:0/2217978716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c15f2a0 0x7f1e1c0b34d0 secure :-1 s=READY pgs=1147 cs=0 l=1 rev1=1 crypto rx=0x7f1e1c0b3190 tx=0x7f1e0c02b910 comp rx=0 tx=0).stop 2024-03-06T07:32:39.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.589+0000 7f1e24800640 1 -- 172.21.15.86:0/2217978716 shutdown_connections 2024-03-06T07:32:39.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.589+0000 7f1e24800640 1 --2- 172.21.15.86:0/2217978716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c15f2a0 0x7f1e1c0b34d0 unknown :-1 s=CLOSED pgs=1147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.589+0000 7f1e24800640 1 --2- 172.21.15.86:0/2217978716 >> v2:172.21.15.194:3300/0 conn(0x7f1e1c15e980 0x7f1e1c15ed60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.589+0000 7f1e24800640 1 -- 172.21.15.86:0/2217978716 >> 172.21.15.86:0/2217978716 conn(0x7f1e1c0d0540 msgr2=0x7f1e1c0d0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:39.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.590+0000 7f1e24800640 1 -- 172.21.15.86:0/2217978716 shutdown_connections 2024-03-06T07:32:39.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.590+0000 7f1e24800640 1 -- 172.21.15.86:0/2217978716 wait complete. 2024-03-06T07:32:39.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.595+0000 7f1e24800640 1 Processor -- start 2024-03-06T07:32:39.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.595+0000 7f1e24800640 1 -- start start 2024-03-06T07:32:39.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.599+0000 7f1e24800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c06eff0 0x7f1e1c160210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.599+0000 7f1e24800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1e1c15e980 0x7f1e1c146730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.599+0000 7f1e24800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1e1c15f2a0 0x7f1e1c15d2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.599+0000 7f1e24800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1e1c06de20 con 0x7f1e1c06eff0 2024-03-06T07:32:39.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.599+0000 7f1e24800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1e1c06db40 con 0x7f1e1c15e980 2024-03-06T07:32:39.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.599+0000 7f1e24800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1e1c0b3a10 con 0x7f1e1c15f2a0 2024-03-06T07:32:39.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.600+0000 7f1e21400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1e1c15e980 0x7f1e1c146730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.600+0000 7f1e21400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1e1c15e980 0x7f1e1c146730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:42774/0 (socket says 172.21.15.86:42774) 2024-03-06T07:32:39.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.600+0000 7f1e21400640 1 -- 172.21.15.86:0/28448913 learned_addr learned my addr 172.21.15.86:0/28448913 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:39.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.600+0000 7f1e21e00640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c06eff0 0x7f1e1c160210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.601+0000 7f1e21400640 1 -- 172.21.15.86:0/28448913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1e1c15f2a0 msgr2=0x7f1e1c15d2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:39.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.601+0000 7f1e21400640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1e1c15f2a0 0x7f1e1c15d2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.601+0000 7f1e21400640 1 -- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c06eff0 msgr2=0x7f1e1c160210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:39.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.601+0000 7f1e21400640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c06eff0 0x7f1e1c160210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.601+0000 7f1e21400640 1 -- 172.21.15.86:0/28448913 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1e1c160910 con 0x7f1e1c15e980 2024-03-06T07:32:39.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.603+0000 7f1e21400640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1e1c15e980 0x7f1e1c146730 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f1e0c0090b0 tx=0x7f1e0c0046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:39.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.604+0000 7f1e16000640 1 -- 172.21.15.86:0/28448913 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e0c040070 con 0x7f1e1c15e980 2024-03-06T07:32:39.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.604+0000 7f1e16000640 1 -- 172.21.15.86:0/28448913 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1e0c030d00 con 0x7f1e1c15e980 2024-03-06T07:32:39.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.604+0000 7f1e16000640 1 -- 172.21.15.86:0/28448913 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e0c035040 con 0x7f1e1c15e980 2024-03-06T07:32:39.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.605+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1e1c160bf0 con 0x7f1e1c15e980 2024-03-06T07:32:39.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.605+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1e1c147020 con 0x7f1e1c15e980 2024-03-06T07:32:39.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.607+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1e1c147220 con 0x7f1e1c15e980 2024-03-06T07:32:39.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.611+0000 7f1e16000640 1 -- 172.21.15.86:0/28448913 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1e0c04b060 con 0x7f1e1c15e980 2024-03-06T07:32:39.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.615+0000 7f1e16000640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1e0006b5d0 0x7f1e0006da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.615+0000 7f1e16000640 1 -- 172.21.15.86:0/28448913 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f1e0c0abec0 con 0x7f1e1c15e980 2024-03-06T07:32:39.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.615+0000 7f1e21e00640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1e0006b5d0 0x7f1e0006da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.617+0000 7f1e21e00640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1e0006b5d0 0x7f1e0006da90 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f1e10009ad0 tx=0x7f1e10008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:39.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.620+0000 7f1e16000640 1 -- 172.21.15.86:0/28448913 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1e1c147220 con 0x7f1e1c15e980 2024-03-06T07:32:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.872+0000 7ffa7b600640 1 Processor -- start 2024-03-06T07:32:39.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.873+0000 7ffa7b600640 1 -- start start 2024-03-06T07:32:39.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.874+0000 7ffa7b600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ffa7415cc90 0x7ffa7406d400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa7b600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ffa740d0780 0x7ffa74162f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa7b600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffa741635a0 con 0x7ffa74162b00 2024-03-06T07:32:39.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa7b600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffa7406d940 con 0x7ffa740d0780 2024-03-06T07:32:39.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa7b600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffa74163880 con 0x7ffa7415cc90 2024-03-06T07:32:39.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa78c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ffa7415cc90 0x7ffa7406d400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa78c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ffa7415cc90 0x7ffa7406d400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:42778/0 (socket says 172.21.15.86:42778) 2024-03-06T07:32:39.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa78c00640 1 -- 172.21.15.86:0/1622527713 learned_addr learned my addr 172.21.15.86:0/1622527713 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:39.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.875+0000 7ffa79600640 1 --2- 172.21.15.86:0/1622527713 >> v2:172.21.15.86:3300/0 conn(0x7ffa740d0780 0x7ffa74162f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.876+0000 7ffa78c00640 1 -- 172.21.15.86:0/1622527713 >> v2:172.21.15.86:3300/0 conn(0x7ffa740d0780 msgr2=0x7ffa74162f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:39.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.876+0000 7ffa78c00640 1 --2- 172.21.15.86:0/1622527713 >> v2:172.21.15.86:3300/0 conn(0x7ffa740d0780 0x7ffa74162f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.876+0000 7ffa78c00640 1 -- 172.21.15.86:0/1622527713 >> v1:172.21.15.86:6789/0 conn(0x7ffa74162b00 legacy=0x7ffa740d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:39.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.876+0000 7ffa78c00640 1 -- 172.21.15.86:0/1622527713 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffa7406cc00 con 0x7ffa7415cc90 2024-03-06T07:32:39.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.876+0000 7ffa79600640 1 --2- 172.21.15.86:0/1622527713 >> v2:172.21.15.86:3300/0 conn(0x7ffa740d0780 0x7ffa74162f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:32:39.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.876+0000 7ffa73400640 1 -- 172.21.15.86:0/1622527713 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2095520296 0 0) 0x7ffa741635a0 con 0x7ffa74162b00 2024-03-06T07:32:39.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.878+0000 7ffa78c00640 1 --2- 172.21.15.86:0/1622527713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa7415cc90 0x7ffa7406d400 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7ffa7406ee40 tx=0x7ffa6802b9a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9842899e0fad5ee server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:39.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.879+0000 7ffa73400640 1 -- 172.21.15.86:0/1622527713 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa68034070 con 0x7ffa7415cc90 2024-03-06T07:32:39.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.879+0000 7ffa73400640 1 -- 172.21.15.86:0/1622527713 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffa68030860 con 0x7ffa7415cc90 2024-03-06T07:32:39.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.879+0000 7ffa73400640 1 -- 172.21.15.86:0/1622527713 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa68030c30 con 0x7ffa7415cc90 2024-03-06T07:32:39.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.880+0000 7ffa7b600640 1 -- 172.21.15.86:0/1622527713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa7415cc90 msgr2=0x7ffa7406d400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:39.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.880+0000 7ffa7b600640 1 --2- 172.21.15.86:0/1622527713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa7415cc90 0x7ffa7406d400 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7ffa7406ee40 tx=0x7ffa6802b9a0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.881+0000 7ffa7b600640 1 -- 172.21.15.86:0/1622527713 shutdown_connections 2024-03-06T07:32:39.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.881+0000 7ffa7b600640 1 --2- 172.21.15.86:0/1622527713 >> v2:172.21.15.86:3300/0 conn(0x7ffa740d0780 0x7ffa74162f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.881+0000 7ffa7b600640 1 --2- 172.21.15.86:0/1622527713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa7415cc90 0x7ffa7406d400 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.881+0000 7ffa7b600640 1 -- 172.21.15.86:0/1622527713 >> 172.21.15.86:0/1622527713 conn(0x7ffa7406b360 msgr2=0x7ffa7406b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:39.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.881+0000 7ffa7b600640 1 -- 172.21.15.86:0/1622527713 shutdown_connections 2024-03-06T07:32:39.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.882+0000 7ffa7b600640 1 -- 172.21.15.86:0/1622527713 wait complete. 2024-03-06T07:32:39.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.884+0000 7ffa7b600640 1 Processor -- start 2024-03-06T07:32:39.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.884+0000 7ffa7b600640 1 -- start start 2024-03-06T07:32:39.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.886+0000 7ffa7b600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa740d0780 0x7ffa74163ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.886+0000 7ffa7b600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffa7415cc90 0x7ffa74155bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.886+0000 7ffa7b600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffa74162b00 0x7ffa741ee2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.887+0000 7ffa7b600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffa740b4fe0 con 0x7ffa7415cc90 2024-03-06T07:32:39.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.887+0000 7ffa7b600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffa740b4e60 con 0x7ffa740d0780 2024-03-06T07:32:39.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.887+0000 7ffa78c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa740d0780 0x7ffa74163ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.887+0000 7ffa78c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa740d0780 0x7ffa74163ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:42784/0 (socket says 172.21.15.86:42784) 2024-03-06T07:32:39.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.887+0000 7ffa78c00640 1 -- 172.21.15.86:0/2044452957 learned_addr learned my addr 172.21.15.86:0/2044452957 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:39.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.887+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffa740acd60 con 0x7ffa74162b00 2024-03-06T07:32:39.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.888+0000 7ffa73e00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffa7415cc90 0x7ffa74155bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.888+0000 7ffa79600640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffa74162b00 0x7ffa741ee2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.888+0000 7ffa78c00640 1 -- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffa74162b00 msgr2=0x7ffa741ee2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:39.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.888+0000 7ffa78c00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffa74162b00 0x7ffa741ee2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.888+0000 7ffa78c00640 1 -- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffa7415cc90 msgr2=0x7ffa74155bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:39.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.889+0000 7ffa78c00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffa7415cc90 0x7ffa74155bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:39.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.889+0000 7ffa78c00640 1 -- 172.21.15.86:0/2044452957 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffa7409aa30 con 0x7ffa740d0780 2024-03-06T07:32:39.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.889+0000 7ffa73e00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffa7415cc90 0x7ffa74155bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:32:39.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.889+0000 7ffa78c00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa740d0780 0x7ffa74163ee0 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7ffa68000c00 tx=0x7ffa68004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:39.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.889+0000 7ffa70c00640 1 -- 172.21.15.86:0/2044452957 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa68048070 con 0x7ffa740d0780 2024-03-06T07:32:39.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.890+0000 7ffa70c00640 1 -- 172.21.15.86:0/2044452957 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffa68030660 con 0x7ffa740d0780 2024-03-06T07:32:39.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.890+0000 7ffa70c00640 1 -- 172.21.15.86:0/2044452957 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa68034040 con 0x7ffa740d0780 2024-03-06T07:32:39.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.890+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffa7409b290 con 0x7ffa740d0780 2024-03-06T07:32:39.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.890+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffa7409afb0 con 0x7ffa740d0780 2024-03-06T07:32:39.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.893+0000 7ffa70c00640 1 -- 172.21.15.86:0/2044452957 <== mon.1 v2:172.21.15.194:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffa6804b070 con 0x7ffa740d0780 2024-03-06T07:32:39.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.894+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffa74068070 con 0x7ffa740d0780 2024-03-06T07:32:39.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.894+0000 7ffa70c00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa5406b550 0x7ffa5406da10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:39.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.895+0000 7ffa70c00640 1 -- 172.21.15.86:0/2044452957 <== mon.1 v2:172.21.15.194:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7ffa680ac1c0 con 0x7ffa740d0780 2024-03-06T07:32:39.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.895+0000 7ffa73e00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa5406b550 0x7ffa5406da10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:39.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.896+0000 7ffa73e00640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa5406b550 0x7ffa5406da10 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7ffa741627c0 tx=0x7ffa60009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:39.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:39.902+0000 7ffa70c00640 1 -- 172.21.15.86:0/2044452957 <== mon.1 v2:172.21.15.194:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffa6807aa40 con 0x7ffa740d0780 2024-03-06T07:32:40.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.079+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1e1c068070 con 0x7f1e1c15e980 2024-03-06T07:32:40.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.081+0000 7f1e16000640 1 -- 172.21.15.86:0/28448913 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f1e0c02f320 con 0x7f1e1c15e980 2024-03-06T07:32:40.084 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:40.084 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:40.084 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:40.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1e0006b5d0 msgr2=0x7f1e0006da90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1e0006b5d0 0x7f1e0006da90 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f1e10009ad0 tx=0x7f1e10008040 comp rx=0 tx=0).stop 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1e1c15e980 msgr2=0x7f1e1c146730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1e1c15e980 0x7f1e1c146730 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f1e0c0090b0 tx=0x7f1e0c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 shutdown_connections 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1e0006b5d0 0x7f1e0006da90 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1e1c15f2a0 0x7f1e1c15d2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1e1c15e980 0x7f1e1c146730 secure :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f1e0c0090b0 tx=0x7f1e0c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 --2- 172.21.15.86:0/28448913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1e1c06eff0 0x7f1e1c160210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.085+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 >> 172.21.15.86:0/28448913 conn(0x7f1e1c0d0540 msgr2=0x7f1e1c0b0040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:40.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.086+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 shutdown_connections 2024-03-06T07:32:40.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.086+0000 7f1e24800640 1 -- 172.21.15.86:0/28448913 wait complete. 2024-03-06T07:32:40.114 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:40.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.311+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ffa74166d50 con 0x7ffa740d0780 2024-03-06T07:32:40.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.313+0000 7ffa70c00640 1 -- 172.21.15.86:0/2044452957 <== mon.1 v2:172.21.15.194:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7ffa6807eae0 con 0x7ffa740d0780 2024-03-06T07:32:40.316 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:40.316 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:40.316 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.315+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa5406b550 msgr2=0x7ffa5406da10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.315+0000 7ffa7b600640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa5406b550 0x7ffa5406da10 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7ffa741627c0 tx=0x7ffa60009040 comp rx=0 tx=0).stop 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.315+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa740d0780 msgr2=0x7ffa74163ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.315+0000 7ffa7b600640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa740d0780 0x7ffa74163ee0 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7ffa68000c00 tx=0x7ffa68004730 comp rx=0 tx=0).stop 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.316+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 shutdown_connections 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.316+0000 7ffa7b600640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa5406b550 0x7ffa5406da10 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.316+0000 7ffa7b600640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffa74162b00 0x7ffa741ee2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.316+0000 7ffa7b600640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffa7415cc90 0x7ffa74155bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.316+0000 7ffa7b600640 1 --2- 172.21.15.86:0/2044452957 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffa740d0780 0x7ffa74163ee0 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:40.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.316+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 >> 172.21.15.86:0/2044452957 conn(0x7ffa7406b360 msgr2=0x7ffa74070ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:40.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.316+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 shutdown_connections 2024-03-06T07:32:40.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:40.317+0000 7ffa7b600640 1 -- 172.21.15.86:0/2044452957 wait complete. 2024-03-06T07:32:40.343 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:32:40.432 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":1134,"msgr_send_messages":1140,"msgr_recv_bytes":296546,"msgr_send_bytes":4009977,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.172465300,"msgr_running_send_time":0.080086355,"msgr_running_recv_time":0.033988812,"msgr_running_fast_dispatch_time":0.044606675,"msgr_send_messages_queue_lat":{"avgcount":1139,"sum":0.133528663,"avgtime":0.000117233},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":13819,"msgr_send_messages":23994,"msgr_recv_bytes":4686369,"msgr_send_bytes":31572757,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":2.645658839,"msgr_running_send_time":1.716942465,"msgr_running_recv_time":1.844668284,"msgr_running_fast_dispatch_time":0.043025432,"msgr_send_messages_queue_lat":{"avgcount":23992,"sum":1.249647757,"avgtime":0.000052086},"msgr_handle_ack_lat":{"avgcount":28791,"sum":0.010795152,"avgtime":0.000000374},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":94240},"AsyncMessenger::Worker-2":{"msgr_recv_messages":875,"msgr_send_messages":868,"msgr_recv_bytes":643936,"msgr_send_bytes":1445070,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.146318280,"msgr_running_send_time":0.065456317,"msgr_running_recv_time":0.039451317,"msgr_running_fast_dispatch_time":0.036282745,"msgr_send_messages_queue_lat":{"avgcount":868,"sum":0.108165330,"avgtime":0.000124614},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109968,"msgr_send_encrypted_bytes":8992},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":99,"sum":10.310908403,"avgtime":0.104150589}},"finisher-PurgeQueue":{"queue_len":1,"complete_latency":{"avgcount":4399,"sum":2.520854228,"avgtime":0.000573051}},"mds":{"request":6412,"reply":6410,"reply_latency":{"avgcount":6410,"sum":639.562377409,"avgtime":0.099775721},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":3606,"inodes_top":0,"inodes_bottom":3606,"inodes_pin_tail":0,"inodes_pinned":3606,"inodes_expired":0,"inodes_with_caps":56,"caps":56,"subtrees":2,"traverse":6413,"traverse_hit":6413,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":0,"root_rbytes":0,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":72,"ceph_cap_op_grant":7327,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":5622,"handle_client_caps_dirty":3000,"handle_client_cap_release":992,"process_request_cap_release":3210},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":55,"num_strays_delayed":55,"num_strays_enqueuing":0,"strays_created":3201,"strays_enqueued":3146,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":15792,"evex":0,"evtrm":0,"ev":15795,"evexg":0,"evexd":0,"segadd":18,"segex":0,"segtrm":0,"seg":20,"segmjr":3,"segexg":0,"segexd":0,"expos":4194304,"wrpos":35351722,"rdpos":4195244,"jlat":{"avgcount":23,"sum":4.018299047,"avgtime":0.174708654},"replayed":3},"mds_mem":{"ino":408,"ino+":3214,"ino-":2806,"dir":13,"dir+":13,"dir-":0,"dn":406,"dn+":6412,"dn-":6006,"cap":1,"cap+":3202,"cap-":3201,"rss":105256,"heap":248164},"mds_server":{"dispatch_client_request":6415,"dispatch_server_request":0,"handle_client_request":6412,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":1,"sum":0.508672078,"avgtime":0.508672078},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":3200,"sum":7.614133400,"avgtime":0.002379416},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5786,"average_load":5786,"avg_session_uptime":1116,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":8415757,"buffer_anon_items":3441,"buffer_meta_bytes":264,"buffer_meta_items":3,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":6943834,"mds_co_items":116856,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":6,"op_laggy":0,"op_send":2760,"op_send_bytes":31723520,"op_resend":0,"op_reply":2754,"op_latency":{"avgcount":2754,"sum":11.497757908,"avgtime":0.004174930},"op_inflight":4,"oplen_avg":{"avgcount":2760,"sum":2824},"op":2760,"op_r":57,"op_w":2703,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":29,"osdop_write":44,"osdop_writefull":8,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":2637,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":25,"omap_rd":48,"omap_del":6},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":6,"pq_executing_ops_high_water":6,"pq_executing":6,"pq_executing_high_water":6,"pq_executed_ops":2613,"pq_executed":2613,"pq_item_in_journal":533},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":15825,"get_sum":4200175,"get_or_fail_fail":0,"get_or_fail_success":15825,"take":0,"take_sum":0,"put":15825,"put_sum":4200175,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":2760,"take_sum":32042200,"put":2754,"put_sum":32042200,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":6,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":2760,"take_sum":2760,"put":2754,"put_sum":2754,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":3146,"get_sum":317746,"get_or_fail_fail":0,"get_or_fail_success":3146,"take":0,"take_sum":0,"put":21,"put_sum":317746,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":203708,"max":3758096384,"get_started":0,"get":15792,"get_sum":31156220,"get_or_fail_fail":0,"get_or_fail_success":15792,"take":0,"take_sum":0,"put":23,"put_sum":30952512,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:40.434 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 296546, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 1134, "msgr_running_fast_dispatch_time": 0.044606675, "msgr_running_recv_time": 0.033988812, "msgr_running_send_time": 0.080086355, "msgr_running_total_time": 0.1724653, "msgr_send_bytes": 4009977, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 1140, "msgr_send_messages_queue_lat": { "avgcount": 1139, "avgtime": 0.000117233, "sum": 0.133528663 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 28791, "avgtime": 3.74e-07, "sum": 0.010795152 }, "msgr_recv_bytes": 4686369, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 13819, "msgr_running_fast_dispatch_time": 0.043025432, "msgr_running_recv_time": 1.844668284, "msgr_running_send_time": 1.716942465, "msgr_running_total_time": 2.645658839, "msgr_send_bytes": 31572757, "msgr_send_encrypted_bytes": 94240, "msgr_send_messages": 23994, "msgr_send_messages_queue_lat": { "avgcount": 23992, "avgtime": 5.2086e-05, "sum": 1.249647757 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 643936, "msgr_recv_encrypted_bytes": 109968, "msgr_recv_messages": 875, "msgr_running_fast_dispatch_time": 0.036282745, "msgr_running_recv_time": 0.039451317, "msgr_running_send_time": 0.065456317, "msgr_running_total_time": 0.14631828, "msgr_send_bytes": 1445070, "msgr_send_encrypted_bytes": 8992, "msgr_send_messages": 868, "msgr_send_messages_queue_lat": { "avgcount": 868, "avgtime": 0.000124614, "sum": 0.10816533 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 99, "avgtime": 0.104150589, "sum": 10.310908403 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 4399, "avgtime": 0.000573051, "sum": 2.520854228 }, "queue_len": 1 }, "mds": { "caps": 56, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 7327, "ceph_cap_op_revoke": 72, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 992, "handle_client_caps": 5622, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 3606, "inodes_bottom": 3606, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 3606, "inodes_top": 0, "inodes_with_caps": 56, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 3210, "q": 0, "reply": 6410, "reply_latency": { "avgcount": 6410, "avgtime": 0.099775721, "sum": 639.562377409 }, "request": 6412, "root_rbytes": 0, "root_rfiles": 0, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6413, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6413, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 55, "num_strays_delayed": 55, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3201, "strays_enqueued": 3146, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 15795, "evadd": 15792, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 23, "avgtime": 0.174708654, "sum": 4.018299047 }, "rdpos": 4195244, "replayed": 3, "seg": 20, "segadd": 18, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 3, "segtrm": 0, "wrpos": 35351722 }, "mds_mem": { "cap": 1, "cap+": 3202, "cap-": 3201, "dir": 13, "dir+": 13, "dir-": 0, "dn": 406, "dn+": 6412, "dn-": 6006, "heap": 248164, "ino": 408, "ino+": 3214, "ino-": 2806, "rss": 105256 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6415, "dispatch_server_request": 0, "handle_client_request": 6412, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 1, "avgtime": 0.508672078, "sum": 0.508672078 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 3200, "avgtime": 0.002379416, "sum": 7.6141334 } }, "mds_sessions": { "average_load": 5786, "avg_session_uptime": 1116, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5786 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 8415757, "buffer_anon_items": 3441, "buffer_meta_bytes": 264, "buffer_meta_items": 3, "mds_co_bytes": 6943834, "mds_co_items": 116856, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 6, "omap_rd": 48, "omap_wr": 25, "op": 2760, "op_Infinitylight": 4, "op_active": 6, "op_laggy": 0, "op_latency": { "avgcount": 2754, "avgtime": 0.00417493, "sum": 11.497757908 }, "op_pg": 0, "op_r": 57, "op_reply": 2754, "op_resend": 0, "op_rmw": 0, "op_send": 2760, "op_send_bytes": 31723520, "op_w": 2703, "oplen_avg": { "avgcount": 2760, "sum": 2824 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 2637, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 29, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 44, "osdop_writefull": 8, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 2613, "pq_executed_ops": 2613, "pq_executing": 6, "pq_executing_high_water": 6, "pq_executing_ops": 6, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 533 }, "throttle-msgr_dispatch_throttler-mds": { "get": 15825, "get_or_fail_fail": 0, "get_or_fail_success": 15825, "get_started": 0, "get_sum": 4200175, "max": 104857600, "put": 15825, "put_sum": 4200175, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 2754, "put_sum": 32042200, "take": 2760, "take_sum": 32042200, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 2754, "put_sum": 2754, "take": 2760, "take_sum": 2760, "val": 6, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 3146, "get_or_fail_fail": 0, "get_or_fail_success": 3146, "get_started": 0, "get_sum": 317746, "max": 3758096384, "put": 21, "put_sum": 317746, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 15792, "get_or_fail_fail": 0, "get_or_fail_success": 15792, "get_started": 0, "get_sum": 31156220, "max": 3758096384, "put": 23, "put_sum": 30952512, "take": 0, "take_sum": 0, "val": 203708, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:40.434 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 6/55, 2613/3201 2024-03-06T07:32:40.684 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:32:40.684 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:32:41.435 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:41.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.639+0000 7f5825a00640 1 Processor -- start 2024-03-06T07:32:41.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.639+0000 7f5825a00640 1 -- start start 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f5825a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f582015e1d0 0x7f582006b9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f5825a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f58200cf2c0 0x7f5820162780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f5825a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5820162cc0 con 0x7f58200c97e0 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f5825a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f58200d0020 con 0x7f582015e1d0 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f5825a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f58201622f0 con 0x7f58200cf2c0 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f581ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f582015e1d0 0x7f582006b9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f581ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f582015e1d0 0x7f582006b9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33474/0 (socket says 172.21.15.86:33474) 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.641+0000 7f581ea00640 1 -- 172.21.15.86:0/1869464824 learned_addr learned my addr 172.21.15.86:0/1869464824 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:41.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.642+0000 7f581ea00640 1 -- 172.21.15.86:0/1869464824 >> v2:172.21.15.194:3300/0 conn(0x7f58200cf2c0 msgr2=0x7f5820162780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:41.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.642+0000 7f581ea00640 1 --2- 172.21.15.86:0/1869464824 >> v2:172.21.15.194:3300/0 conn(0x7f58200cf2c0 0x7f5820162780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.642+0000 7f581ea00640 1 -- 172.21.15.86:0/1869464824 >> v1:172.21.15.194:6789/0 conn(0x7f58200c97e0 legacy=0x7f58200cbb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:41.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.642+0000 7f581ea00640 1 -- 172.21.15.86:0/1869464824 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58200cfa60 con 0x7f582015e1d0 2024-03-06T07:32:41.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.643+0000 7f581ea00640 1 --2- 172.21.15.86:0/1869464824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f582015e1d0 0x7f582006b9e0 secure :-1 s=READY pgs=1149 cs=0 l=1 rev1=1 crypto rx=0x7f5820072900 tx=0x7f581002b930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3dd340b7f92f9fe7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:41.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.643+0000 7f581e000640 1 -- 172.21.15.86:0/1869464824 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5810035020 con 0x7f582015e1d0 2024-03-06T07:32:41.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.643+0000 7f581e000640 1 -- 172.21.15.86:0/1869464824 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5810031890 con 0x7f582015e1d0 2024-03-06T07:32:41.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.643+0000 7f581e000640 1 -- 172.21.15.86:0/1869464824 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5810031c60 con 0x7f582015e1d0 2024-03-06T07:32:41.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.643+0000 7f5825a00640 1 -- 172.21.15.86:0/1869464824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f582015e1d0 msgr2=0x7f582006b9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:41.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.643+0000 7f5825a00640 1 --2- 172.21.15.86:0/1869464824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f582015e1d0 0x7f582006b9e0 secure :-1 s=READY pgs=1149 cs=0 l=1 rev1=1 crypto rx=0x7f5820072900 tx=0x7f581002b930 comp rx=0 tx=0).stop 2024-03-06T07:32:41.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.644+0000 7f5825a00640 1 -- 172.21.15.86:0/1869464824 shutdown_connections 2024-03-06T07:32:41.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.644+0000 7f5825a00640 1 --2- 172.21.15.86:0/1869464824 >> v2:172.21.15.194:3300/0 conn(0x7f58200cf2c0 0x7f5820162780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.644+0000 7f5825a00640 1 --2- 172.21.15.86:0/1869464824 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f582015e1d0 0x7f582006b9e0 unknown :-1 s=CLOSED pgs=1149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.644+0000 7f5825a00640 1 -- 172.21.15.86:0/1869464824 >> 172.21.15.86:0/1869464824 conn(0x7f58200ccf50 msgr2=0x7f58200cd360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:41.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.644+0000 7f5825a00640 1 -- 172.21.15.86:0/1869464824 shutdown_connections 2024-03-06T07:32:41.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.644+0000 7f5825a00640 1 -- 172.21.15.86:0/1869464824 wait complete. 2024-03-06T07:32:41.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.645+0000 7f5825a00640 1 Processor -- start 2024-03-06T07:32:41.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.645+0000 7f5825a00640 1 -- start start 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5825a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58200c97e0 0x7f58200abff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5825a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58200cf2c0 0x7f58200b42a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5825a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f582015e1d0 0x7f58200b47e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5825a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f58200cf700 con 0x7f58200cf2c0 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5825a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5820161f80 con 0x7f58200c97e0 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5825a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f582015e5b0 con 0x7f582015e1d0 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58200cf2c0 0x7f58200b42a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58200cf2c0 0x7f58200b42a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33490/0 (socket says 172.21.15.86:33490) 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 -- 172.21.15.86:0/342717873 learned_addr learned my addr 172.21.15.86:0/342717873 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:41.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 -- 172.21.15.86:0/342717873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f582015e1d0 msgr2=0x7f58200b47e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:41.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f582015e1d0 0x7f58200b47e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 -- 172.21.15.86:0/342717873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58200c97e0 msgr2=0x7f58200abff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:41.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58200c97e0 0x7f58200abff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.647+0000 7f5817e00640 1 -- 172.21.15.86:0/342717873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58200b4e80 con 0x7f58200cf2c0 2024-03-06T07:32:41.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.648+0000 7f5817e00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58200cf2c0 0x7f58200b42a0 secure :-1 s=READY pgs=1150 cs=0 l=1 rev1=1 crypto rx=0x7f580800b4a0 tx=0x7f580800b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:41.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.648+0000 7f5816a00640 1 -- 172.21.15.86:0/342717873 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5808014070 con 0x7f58200cf2c0 2024-03-06T07:32:41.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.648+0000 7f5816a00640 1 -- 172.21.15.86:0/342717873 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5808010610 con 0x7f58200cf2c0 2024-03-06T07:32:41.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.648+0000 7f5816a00640 1 -- 172.21.15.86:0/342717873 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5808010a00 con 0x7f58200cf2c0 2024-03-06T07:32:41.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.648+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f58200b5160 con 0x7f58200cf2c0 2024-03-06T07:32:41.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.648+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f58200b5720 con 0x7f58200cf2c0 2024-03-06T07:32:41.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.649+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5820068ea0 con 0x7f58200cf2c0 2024-03-06T07:32:41.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.650+0000 7f5816a00640 1 -- 172.21.15.86:0/342717873 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5808017070 con 0x7f58200cf2c0 2024-03-06T07:32:41.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.651+0000 7f5816a00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57fc06b4d0 0x7f57fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:41.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.652+0000 7f5816a00640 1 -- 172.21.15.86:0/342717873 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f5808089ca0 con 0x7f58200cf2c0 2024-03-06T07:32:41.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.652+0000 7f581ea00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57fc06b4d0 0x7f57fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:41.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.652+0000 7f581ea00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57fc06b4d0 0x7f57fc06d990 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f5810036000 tx=0x7f5810030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:41.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.656+0000 7f5816a00640 1 -- 172.21.15.86:0/342717873 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5808058a60 con 0x7f58200cf2c0 2024-03-06T07:32:41.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.885+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5820062510 con 0x7f58200cf2c0 2024-03-06T07:32:41.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.886+0000 7f5816a00640 1 -- 172.21.15.86:0/342717873 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f5808058360 con 0x7f58200cf2c0 2024-03-06T07:32:41.889 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:41.889 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:41.889 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57fc06b4d0 msgr2=0x7f57fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57fc06b4d0 0x7f57fc06d990 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f5810036000 tx=0x7f5810030000 comp rx=0 tx=0).stop 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58200cf2c0 msgr2=0x7f58200b42a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58200cf2c0 0x7f58200b42a0 secure :-1 s=READY pgs=1150 cs=0 l=1 rev1=1 crypto rx=0x7f580800b4a0 tx=0x7f580800b960 comp rx=0 tx=0).stop 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 shutdown_connections 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57fc06b4d0 0x7f57fc06d990 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f582015e1d0 0x7f58200b47e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f58200cf2c0 0x7f58200b42a0 unknown :-1 s=CLOSED pgs=1150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 --2- 172.21.15.86:0/342717873 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f58200c97e0 0x7f58200abff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:41.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 >> 172.21.15.86:0/342717873 conn(0x7f58200ccf50 msgr2=0x7f58200cd4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:41.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 shutdown_connections 2024-03-06T07:32:41.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:41.888+0000 7f5825a00640 1 -- 172.21.15.86:0/342717873 wait complete. 2024-03-06T07:32:41.908 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:42.143 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":1424,"msgr_send_messages":1425,"msgr_recv_bytes":372681,"msgr_send_bytes":4114965,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.206566985,"msgr_running_send_time":0.095817695,"msgr_running_recv_time":0.040847460,"msgr_running_fast_dispatch_time":0.054314622,"msgr_send_messages_queue_lat":{"avgcount":1424,"sum":0.231161408,"avgtime":0.000162332},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":14021,"msgr_send_messages":24195,"msgr_recv_bytes":4739884,"msgr_send_bytes":31895000,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":2.668566249,"msgr_running_send_time":1.726558499,"msgr_running_recv_time":1.849440393,"msgr_running_fast_dispatch_time":0.050119187,"msgr_send_messages_queue_lat":{"avgcount":24193,"sum":1.267226491,"avgtime":0.000052379},"msgr_handle_ack_lat":{"avgcount":28793,"sum":0.010795581,"avgtime":0.000000374},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":96032},"AsyncMessenger::Worker-2":{"msgr_recv_messages":1076,"msgr_send_messages":1069,"msgr_recv_bytes":702053,"msgr_send_bytes":1520699,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.166032767,"msgr_running_send_time":0.073183597,"msgr_running_recv_time":0.044155039,"msgr_running_fast_dispatch_time":0.042229366,"msgr_send_messages_queue_lat":{"avgcount":1069,"sum":0.137203475,"avgtime":0.000128347},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109968,"msgr_send_encrypted_bytes":8992},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":128,"sum":10.351024436,"avgtime":0.080867378}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":5419,"sum":2.560505852,"avgtime":0.000472505}},"mds":{"request":6412,"reply":6410,"reply_latency":{"avgcount":6410,"sum":639.562377409,"avgtime":0.099775721},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":12,"dir_split":0,"dir_merge":0,"inodes":3606,"inodes_top":0,"inodes_bottom":3606,"inodes_pin_tail":0,"inodes_pinned":3606,"inodes_expired":0,"inodes_with_caps":56,"caps":56,"subtrees":2,"traverse":6413,"traverse_hit":6413,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":0,"root_rbytes":0,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":72,"ceph_cap_op_grant":7327,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":5622,"handle_client_caps_dirty":3000,"handle_client_cap_release":992,"process_request_cap_release":3210},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":0,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":3201,"strays_enqueued":3201,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":15849,"evex":15795,"evtrm":15795,"ev":57,"evexg":0,"evexd":15794,"segadd":19,"segex":20,"segtrm":20,"seg":1,"segmjr":1,"segexg":0,"segexd":19,"expos":35351722,"wrpos":35387546,"rdpos":4195244,"jlat":{"avgcount":25,"sum":4.035061996,"avgtime":0.161402479},"replayed":3},"mds_mem":{"ino":68,"ino+":3214,"ino-":3146,"dir":13,"dir+":13,"dir-":0,"dn":66,"dn+":6412,"dn-":6346,"cap":1,"cap+":3202,"cap-":3201,"rss":105512,"heap":248164},"mds_server":{"dispatch_client_request":6415,"dispatch_server_request":0,"handle_client_request":6412,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":1,"sum":0.508672078,"avgtime":0.508672078},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":3200,"sum":7.614133400,"avgtime":0.002379416},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5786,"average_load":5786,"avg_session_uptime":1116,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":8415757,"buffer_anon_items":3441,"buffer_meta_bytes":264,"buffer_meta_items":3,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":6943834,"mds_co_items":116856,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":3443,"op_send_bytes":31995682,"op_resend":0,"op_reply":3443,"op_latency":{"avgcount":3443,"sum":13.951706223,"avgtime":0.004052194},"op_inflight":0,"oplen_avg":{"avgcount":3443,"sum":3655},"op":3443,"op_r":59,"op_w":3384,"op_rmw":0,"op_pg":0,"osdop_stat":12,"osdop_create":66,"osdop_read":31,"osdop_write":48,"osdop_writefull":16,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":3226,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":132,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":43,"omap_rd":48,"omap_del":8},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":3201,"pq_executed":3201,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":16516,"get_sum":4325244,"get_or_fail_fail":0,"get_or_fail_success":16516,"take":0,"take_sum":0,"put":16516,"put_sum":4325244,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":3443,"take_sum":32319917,"put":3443,"put_sum":32319917,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":3443,"take_sum":3443,"put":3443,"put_sum":3443,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":3201,"get_sum":323301,"get_or_fail_fail":0,"get_or_fail_success":3201,"take":0,"take_sum":0,"put":23,"put_sum":323301,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":33794,"max":3758096384,"get_started":0,"get":15849,"get_sum":31192044,"get_or_fail_fail":0,"get_or_fail_success":15849,"take":0,"take_sum":0,"put":25,"put_sum":31158250,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:42.145 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 372681, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 1424, "msgr_running_fast_dispatch_time": 0.054314622, "msgr_running_recv_time": 0.04084746, "msgr_running_send_time": 0.095817695, "msgr_running_total_time": 0.206566985, "msgr_send_bytes": 4114965, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 1425, "msgr_send_messages_queue_lat": { "avgcount": 1424, "avgtime": 0.000162332, "sum": 0.231161408 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 28793, "avgtime": 3.74e-07, "sum": 0.010795581 }, "msgr_recv_bytes": 4739884, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 14021, "msgr_running_fast_dispatch_time": 0.050119187, "msgr_running_recv_time": 1.849440393, "msgr_running_send_time": 1.726558499, "msgr_running_total_time": 2.668566249, "msgr_send_bytes": 31895000, "msgr_send_encrypted_bytes": 96032, "msgr_send_messages": 24195, "msgr_send_messages_queue_lat": { "avgcount": 24193, "avgtime": 5.2379e-05, "sum": 1.267226491 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 702053, "msgr_recv_encrypted_bytes": 109968, "msgr_recv_messages": 1076, "msgr_running_fast_dispatch_time": 0.042229366, "msgr_running_recv_time": 0.044155039, "msgr_running_send_time": 0.073183597, "msgr_running_total_time": 0.166032767, "msgr_send_bytes": 1520699, "msgr_send_encrypted_bytes": 8992, "msgr_send_messages": 1069, "msgr_send_messages_queue_lat": { "avgcount": 1069, "avgtime": 0.000128347, "sum": 0.137203475 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 128, "avgtime": 0.080867378, "sum": 10.351024436 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 5419, "avgtime": 0.000472505, "sum": 2.560505852 }, "queue_len": 0 }, "mds": { "caps": 56, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 7327, "ceph_cap_op_revoke": 72, "ceph_cap_op_trunc": 0, "dir_commit": 12, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 992, "handle_client_caps": 5622, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 3606, "inodes_bottom": 3606, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 3606, "inodes_top": 0, "inodes_with_caps": 56, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 3210, "q": 0, "reply": 6410, "reply_latency": { "avgcount": 6410, "avgtime": 0.099775721, "sum": 639.562377409 }, "request": 6412, "root_rbytes": 0, "root_rfiles": 0, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6413, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6413, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 0, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3201, "strays_enqueued": 3201, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 57, "evadd": 15849, "evex": 15795, "evexd": 15794, "evexg": 0, "evlrg": 0, "evtrm": 15795, "expos": 35351722, "jlat": { "avgcount": 25, "avgtime": 0.161402479, "sum": 4.035061996 }, "rdpos": 4195244, "replayed": 3, "seg": 1, "segadd": 19, "segex": 20, "segexd": 19, "segexg": 0, "segmjr": 1, "segtrm": 20, "wrpos": 35387546 }, "mds_mem": { "cap": 1, "cap+": 3202, "cap-": 3201, "dir": 13, "dir+": 13, "dir-": 0, "dn": 66, "dn+": 6412, "dn-": 6346, "heap": 248164, "ino": 68, "ino+": 3214, "ino-": 3146, "rss": 105512 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6415, "dispatch_server_request": 0, "handle_client_request": 6412, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 1, "avgtime": 0.508672078, "sum": 0.508672078 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 3200, "avgtime": 0.002379416, "sum": 7.6141334 } }, "mds_sessions": { "average_load": 5786, "avg_session_uptime": 1116, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5786 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 8415757, "buffer_anon_items": 3441, "buffer_meta_bytes": 264, "buffer_meta_items": 3, "mds_co_bytes": 6943834, "mds_co_items": 116856, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 8, "omap_rd": 48, "omap_wr": 43, "op": 3443, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 3443, "avgtime": 0.004052194, "sum": 13.951706223 }, "op_pg": 0, "op_r": 59, "op_reply": 3443, "op_resend": 0, "op_rmw": 0, "op_send": 3443, "op_send_bytes": 31995682, "op_w": 3384, "oplen_avg": { "avgcount": 3443, "sum": 3655 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 66, "osdop_delete": 3226, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 31, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 132, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 12, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 48, "osdop_writefull": 16, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 3201, "pq_executed_ops": 3201, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 16516, "get_or_fail_fail": 0, "get_or_fail_success": 16516, "get_started": 0, "get_sum": 4325244, "max": 104857600, "put": 16516, "put_sum": 4325244, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 3443, "put_sum": 32319917, "take": 3443, "take_sum": 32319917, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 3443, "put_sum": 3443, "take": 3443, "take_sum": 3443, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 3201, "get_or_fail_fail": 0, "get_or_fail_success": 3201, "get_started": 0, "get_sum": 323301, "max": 3758096384, "put": 23, "put_sum": 323301, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 15849, "get_or_fail_fail": 0, "get_or_fail_success": 15849, "get_started": 0, "get_sum": 31192044, "max": 3758096384, "put": 25, "put_sum": 31158250, "take": 0, "take_sum": 0, "val": 33794, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:42.145 INFO:tasks.cephfs.test_strays:Complete purge in 6 seconds 2024-03-06T07:32:42.145 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:42.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.346+0000 7f2321600640 1 Processor -- start 2024-03-06T07:32:42.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.346+0000 7f2321600640 1 -- start start 2024-03-06T07:32:42.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2321600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c15be90 0x7f231c0b49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:42.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2321600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f231c0acd60 0x7f231c0d0070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2321600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f231c0d05b0 0x7f231c06c9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2321600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f231c0b4f30 con 0x7f231c0acd60 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2321600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f231c0af240 con 0x7f231c0d05b0 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2321600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f231c0b5160 con 0x7f231c15be90 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2312000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f231c0acd60 0x7f231c0d0070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2312000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f231c0acd60 0x7f231c0d0070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33502/0 (socket says 172.21.15.86:33502) 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.348+0000 7f2312000640 1 -- 172.21.15.86:0/2097850765 learned_addr learned my addr 172.21.15.86:0/2097850765 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f231b400640 1 --2- 172.21.15.86:0/2097850765 >> v2:172.21.15.194:3300/0 conn(0x7f231c0d05b0 0x7f231c06c9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f231aa00640 1 --2- 172.21.15.86:0/2097850765 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c15be90 0x7f231c0b49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f2312000640 1 -- 172.21.15.86:0/2097850765 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c15be90 msgr2=0x7f231c0b49f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f2312000640 1 --2- 172.21.15.86:0/2097850765 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c15be90 0x7f231c0b49f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f2312000640 1 -- 172.21.15.86:0/2097850765 >> v2:172.21.15.194:3300/0 conn(0x7f231c0d05b0 msgr2=0x7f231c06c9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:42.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f2312000640 1 --2- 172.21.15.86:0/2097850765 >> v2:172.21.15.194:3300/0 conn(0x7f231c0d05b0 0x7f231c06c9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f2312000640 1 -- 172.21.15.86:0/2097850765 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f231c0ae910 con 0x7f231c0acd60 2024-03-06T07:32:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.349+0000 7f231aa00640 1 --2- 172.21.15.86:0/2097850765 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c15be90 0x7f231c0b49f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.350+0000 7f2312000640 1 --2- 172.21.15.86:0/2097850765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0acd60 0x7f231c0d0070 secure :-1 s=READY pgs=1151 cs=0 l=1 rev1=1 crypto rx=0x7f231c0af880 tx=0x7f230402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc8773676f38b4b4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f231a000640 1 -- 172.21.15.86:0/2097850765 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2304035020 con 0x7f231c0acd60 2024-03-06T07:32:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f231a000640 1 -- 172.21.15.86:0/2097850765 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2304031870 con 0x7f231c0acd60 2024-03-06T07:32:42.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f231a000640 1 -- 172.21.15.86:0/2097850765 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2304031c40 con 0x7f231c0acd60 2024-03-06T07:32:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f2321600640 1 -- 172.21.15.86:0/2097850765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0acd60 msgr2=0x7f231c0d0070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f2321600640 1 --2- 172.21.15.86:0/2097850765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0acd60 0x7f231c0d0070 secure :-1 s=READY pgs=1151 cs=0 l=1 rev1=1 crypto rx=0x7f231c0af880 tx=0x7f230402b910 comp rx=0 tx=0).stop 2024-03-06T07:32:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f2321600640 1 -- 172.21.15.86:0/2097850765 shutdown_connections 2024-03-06T07:32:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f2321600640 1 --2- 172.21.15.86:0/2097850765 >> v2:172.21.15.194:3300/0 conn(0x7f231c0d05b0 0x7f231c06c9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f2321600640 1 --2- 172.21.15.86:0/2097850765 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0acd60 0x7f231c0d0070 unknown :-1 s=CLOSED pgs=1151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f2321600640 1 --2- 172.21.15.86:0/2097850765 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c15be90 0x7f231c0b49f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.351+0000 7f2321600640 1 -- 172.21.15.86:0/2097850765 >> 172.21.15.86:0/2097850765 conn(0x7f231c06b360 msgr2=0x7f231c06b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.352+0000 7f2321600640 1 -- 172.21.15.86:0/2097850765 shutdown_connections 2024-03-06T07:32:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.352+0000 7f2321600640 1 -- 172.21.15.86:0/2097850765 wait complete. 2024-03-06T07:32:42.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.353+0000 7f2321600640 1 Processor -- start 2024-03-06T07:32:42.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.353+0000 7f2321600640 1 -- start start 2024-03-06T07:32:42.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.354+0000 7f2321600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c0acd60 0x7f231c0ce0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:42.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.354+0000 7f2321600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0d05b0 0x7f231c15c270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.354+0000 7f2321600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f231c15be90 0x7f231c15da80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.354+0000 7f2321600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f231c0b2c80 con 0x7f231c0d05b0 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.354+0000 7f2321600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f231c0aeed0 con 0x7f231c15be90 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.354+0000 7f2321600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f231c0b2e00 con 0x7f231c0acd60 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.354+0000 7f2312000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0d05b0 0x7f231c15c270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0d05b0 0x7f231c15c270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33518/0 (socket says 172.21.15.86:33518) 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 -- 172.21.15.86:0/3540918604 learned_addr learned my addr 172.21.15.86:0/3540918604 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f231b400640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f231c15be90 0x7f231c15da80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f231aa00640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c0acd60 0x7f231c0ce0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 -- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c0acd60 msgr2=0x7f231c0ce0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c0acd60 0x7f231c0ce0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 -- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f231c15be90 msgr2=0x7f231c15da80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f231c15be90 0x7f231c15da80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 -- 172.21.15.86:0/3540918604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f231c15dfc0 con 0x7f231c0d05b0 2024-03-06T07:32:42.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f231b400640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f231c15be90 0x7f231c15da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.355+0000 7f2312000640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0d05b0 0x7f231c15c270 secure :-1 s=READY pgs=1152 cs=0 l=1 rev1=1 crypto rx=0x7f2304000c00 tx=0x7f23040046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.356+0000 7f2313400640 1 -- 172.21.15.86:0/3540918604 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f230403ad60 con 0x7f231c0d05b0 2024-03-06T07:32:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.356+0000 7f2313400640 1 -- 172.21.15.86:0/3540918604 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f23040443f0 con 0x7f231c0d05b0 2024-03-06T07:32:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.356+0000 7f2313400640 1 -- 172.21.15.86:0/3540918604 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f23040447e0 con 0x7f231c0d05b0 2024-03-06T07:32:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.356+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f231c15c9e0 con 0x7f231c0d05b0 2024-03-06T07:32:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.356+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f231c15cfa0 con 0x7f231c0d05b0 2024-03-06T07:32:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.357+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f231c068070 con 0x7f231c0d05b0 2024-03-06T07:32:42.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.358+0000 7f2313400640 1 -- 172.21.15.86:0/3540918604 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f230404b060 con 0x7f231c0d05b0 2024-03-06T07:32:42.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.359+0000 7f2313400640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f22f806b4d0 0x7f22f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:42.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.359+0000 7f2313400640 1 -- 172.21.15.86:0/3540918604 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f23040ac450 con 0x7f231c0d05b0 2024-03-06T07:32:42.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.359+0000 7f231aa00640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f22f806b4d0 0x7f22f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:42.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.360+0000 7f231aa00640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f22f806b4d0 0x7f22f806d990 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f230c00b7c0 tx=0x7f230c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:42.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.364+0000 7f2313400640 1 -- 172.21.15.86:0/3540918604 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f230403f070 con 0x7f231c0d05b0 2024-03-06T07:32:42.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.596+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f231c09ba00 con 0x7f231c0d05b0 2024-03-06T07:32:42.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.597+0000 7f2313400640 1 -- 172.21.15.86:0/3540918604 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f230402f320 con 0x7f231c0d05b0 2024-03-06T07:32:42.599 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:42.599 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:42.600 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f22f806b4d0 msgr2=0x7f22f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f22f806b4d0 0x7f22f806d990 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f230c00b7c0 tx=0x7f230c008040 comp rx=0 tx=0).stop 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0d05b0 msgr2=0x7f231c15c270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0d05b0 0x7f231c15c270 secure :-1 s=READY pgs=1152 cs=0 l=1 rev1=1 crypto rx=0x7f2304000c00 tx=0x7f23040046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 shutdown_connections 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f22f806b4d0 0x7f22f806d990 unknown :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f231c15be90 0x7f231c15da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f231c0d05b0 0x7f231c15c270 unknown :-1 s=CLOSED pgs=1152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 --2- 172.21.15.86:0/3540918604 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f231c0acd60 0x7f231c0ce0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:42.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 >> 172.21.15.86:0/3540918604 conn(0x7f231c06b360 msgr2=0x7f231c0afd60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:42.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.599+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 shutdown_connections 2024-03-06T07:32:42.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:42.600+0000 7f2321600640 1 -- 172.21.15.86:0/3540918604 wait complete. 2024-03-06T07:32:42.620 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-06T07:32:42.833 INFO:teuthology.orchestra.run.smithi086.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":1424,"msgr_send_messages":1425,"msgr_recv_bytes":372681,"msgr_send_bytes":4114965,"msgr_created_connections":9,"msgr_active_connections":5,"msgr_running_total_time":0.206566985,"msgr_running_send_time":0.095817695,"msgr_running_recv_time":0.040847460,"msgr_running_fast_dispatch_time":0.054314622,"msgr_send_messages_queue_lat":{"avgcount":1424,"sum":0.231161408,"avgtime":0.000162332},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000174,"avgtime":0.000000087},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"AsyncMessenger::Worker-1":{"msgr_recv_messages":14021,"msgr_send_messages":24195,"msgr_recv_bytes":4739884,"msgr_send_bytes":31895000,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":2.668566249,"msgr_running_send_time":1.726558499,"msgr_running_recv_time":1.849440393,"msgr_running_fast_dispatch_time":0.050119187,"msgr_send_messages_queue_lat":{"avgcount":24193,"sum":1.267226491,"avgtime":0.000052379},"msgr_handle_ack_lat":{"avgcount":28793,"sum":0.010795581,"avgtime":0.000000374},"msgr_recv_encrypted_bytes":160,"msgr_send_encrypted_bytes":96032},"AsyncMessenger::Worker-2":{"msgr_recv_messages":1076,"msgr_send_messages":1069,"msgr_recv_bytes":702053,"msgr_send_bytes":1520699,"msgr_created_connections":5,"msgr_active_connections":4,"msgr_running_total_time":0.166032767,"msgr_running_send_time":0.073183597,"msgr_running_recv_time":0.044155039,"msgr_running_fast_dispatch_time":0.042229366,"msgr_send_messages_queue_lat":{"avgcount":1069,"sum":0.137203475,"avgtime":0.000128347},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":109968,"msgr_send_encrypted_bytes":8992},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":128,"sum":10.351024436,"avgtime":0.080867378}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":5419,"sum":2.560505852,"avgtime":0.000472505}},"mds":{"request":6412,"reply":6410,"reply_latency":{"avgcount":6410,"sum":639.562377409,"avgtime":0.099775721},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":12,"dir_split":0,"dir_merge":0,"inodes":3606,"inodes_top":0,"inodes_bottom":3606,"inodes_pin_tail":0,"inodes_pinned":3606,"inodes_expired":0,"inodes_with_caps":56,"caps":56,"subtrees":2,"traverse":6413,"traverse_hit":6413,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":0,"root_rbytes":0,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":72,"ceph_cap_op_grant":7327,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":5622,"handle_client_caps_dirty":3000,"handle_client_cap_release":992,"process_request_cap_release":3210},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":0,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":3201,"strays_enqueued":3201,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":15849,"evex":15795,"evtrm":15795,"ev":57,"evexg":0,"evexd":15794,"segadd":19,"segex":20,"segtrm":20,"seg":1,"segmjr":1,"segexg":0,"segexd":19,"expos":35351722,"wrpos":35387546,"rdpos":4195244,"jlat":{"avgcount":25,"sum":4.035061996,"avgtime":0.161402479},"replayed":3},"mds_mem":{"ino":68,"ino+":3214,"ino-":3146,"dir":12,"dir+":13,"dir-":1,"dn":66,"dn+":6412,"dn-":6346,"cap":1,"cap+":3202,"cap-":3201,"rss":105512,"heap":248164},"mds_server":{"dispatch_client_request":6415,"dispatch_server_request":0,"handle_client_request":6412,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":630.440052260,"avgtime":0.197012516},"req_getattr_latency":{"avgcount":2,"sum":0.233200914,"avgtime":0.116600457},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.005726504,"avgtime":0.005726504},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":6,"sum":0.760592253,"avgtime":0.126765375},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":1,"sum":0.508672078,"avgtime":0.508672078},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":3200,"sum":7.614133400,"avgtime":0.002379416},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5786,"average_load":5786,"avg_session_uptime":1116,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":8415757,"buffer_anon_items":3441,"buffer_meta_bytes":264,"buffer_meta_items":3,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":6943834,"mds_co_items":116856,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":3443,"op_send_bytes":31995682,"op_resend":0,"op_reply":3443,"op_latency":{"avgcount":3443,"sum":13.951706223,"avgtime":0.004052194},"op_inflight":0,"oplen_avg":{"avgcount":3443,"sum":3655},"op":3443,"op_r":59,"op_w":3384,"op_rmw":0,"op_pg":0,"osdop_stat":12,"osdop_create":66,"osdop_read":31,"osdop_write":48,"osdop_writefull":16,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":3226,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":132,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":43,"omap_rd":48,"omap_del":8},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":3201,"pq_executed":3201,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":16516,"get_sum":4325244,"get_or_fail_fail":0,"get_or_fail_success":16516,"take":0,"take_sum":0,"put":16516,"put_sum":4325244,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":3443,"take_sum":32319917,"put":3443,"put_sum":32319917,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":3443,"take_sum":3443,"put":3443,"put_sum":3443,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":3201,"get_sum":323301,"get_or_fail_fail":0,"get_or_fail_success":3201,"take":0,"take_sum":0,"put":23,"put_sum":323301,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x5624393ccaa0":{"val":33794,"max":3758096384,"get_started":0,"get":15849,"get_sum":31192044,"get_or_fail_fail":0,"get_or_fail_success":15849,"take":0,"take_sum":0,"put":25,"put_sum":31158250,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-06T07:32:42.835 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 5, "msgr_created_connections": 9, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.7e-08, "sum": 1.74e-07 }, "msgr_recv_bytes": 372681, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 1424, "msgr_running_fast_dispatch_time": 0.054314622, "msgr_running_recv_time": 0.04084746, "msgr_running_send_time": 0.095817695, "msgr_running_total_time": 0.206566985, "msgr_send_bytes": 4114965, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 1425, "msgr_send_messages_queue_lat": { "avgcount": 1424, "avgtime": 0.000162332, "sum": 0.231161408 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 28793, "avgtime": 3.74e-07, "sum": 0.010795581 }, "msgr_recv_bytes": 4739884, "msgr_recv_encrypted_bytes": 160, "msgr_recv_messages": 14021, "msgr_running_fast_dispatch_time": 0.050119187, "msgr_running_recv_time": 1.849440393, "msgr_running_send_time": 1.726558499, "msgr_running_total_time": 2.668566249, "msgr_send_bytes": 31895000, "msgr_send_encrypted_bytes": 96032, "msgr_send_messages": 24195, "msgr_send_messages_queue_lat": { "avgcount": 24193, "avgtime": 5.2379e-05, "sum": 1.267226491 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 4, "msgr_created_connections": 5, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 702053, "msgr_recv_encrypted_bytes": 109968, "msgr_recv_messages": 1076, "msgr_running_fast_dispatch_time": 0.042229366, "msgr_running_recv_time": 0.044155039, "msgr_running_send_time": 0.073183597, "msgr_running_total_time": 0.166032767, "msgr_send_bytes": 1520699, "msgr_send_encrypted_bytes": 8992, "msgr_send_messages": 1069, "msgr_send_messages_queue_lat": { "avgcount": 1069, "avgtime": 0.000128347, "sum": 0.137203475 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 128, "avgtime": 0.080867378, "sum": 10.351024436 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 5419, "avgtime": 0.000472505, "sum": 2.560505852 }, "queue_len": 0 }, "mds": { "caps": 56, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 7327, "ceph_cap_op_revoke": 72, "ceph_cap_op_trunc": 0, "dir_commit": 12, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 992, "handle_client_caps": 5622, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 3606, "inodes_bottom": 3606, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 3606, "inodes_top": 0, "inodes_with_caps": 56, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 3210, "q": 0, "reply": 6410, "reply_latency": { "avgcount": 6410, "avgtime": 0.099775721, "sum": 639.562377409 }, "request": 6412, "root_rbytes": 0, "root_rfiles": 0, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6413, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6413, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 0, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3201, "strays_enqueued": 3201, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 57, "evadd": 15849, "evex": 15795, "evexd": 15794, "evexg": 0, "evlrg": 0, "evtrm": 15795, "expos": 35351722, "jlat": { "avgcount": 25, "avgtime": 0.161402479, "sum": 4.035061996 }, "rdpos": 4195244, "replayed": 3, "seg": 1, "segadd": 19, "segex": 20, "segexd": 19, "segexg": 0, "segmjr": 1, "segtrm": 20, "wrpos": 35387546 }, "mds_mem": { "cap": 1, "cap+": 3202, "cap-": 3201, "dir": 12, "dir+": 13, "dir-": 1, "dn": 66, "dn+": 6412, "dn-": 6346, "heap": 248164, "ino": 68, "ino+": 3214, "ino-": 3146, "rss": 105512 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6415, "dispatch_server_request": 0, "handle_client_request": 6412, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.197012516, "sum": 630.44005226 }, "req_getattr_latency": { "avgcount": 2, "avgtime": 0.116600457, "sum": 0.233200914 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.005726504, "sum": 0.005726504 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 6, "avgtime": 0.126765375, "sum": 0.760592253 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 1, "avgtime": 0.508672078, "sum": 0.508672078 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 3200, "avgtime": 0.002379416, "sum": 7.6141334 } }, "mds_sessions": { "average_load": 5786, "avg_session_uptime": 1116, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5786 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 8415757, "buffer_anon_items": 3441, "buffer_meta_bytes": 264, "buffer_meta_items": 3, "mds_co_bytes": 6943834, "mds_co_items": 116856, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 8, "omap_rd": 48, "omap_wr": 43, "op": 3443, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 3443, "avgtime": 0.004052194, "sum": 13.951706223 }, "op_pg": 0, "op_r": 59, "op_reply": 3443, "op_resend": 0, "op_rmw": 0, "op_send": 3443, "op_send_bytes": 31995682, "op_w": 3384, "oplen_avg": { "avgcount": 3443, "sum": 3655 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 66, "osdop_delete": 3226, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 31, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 132, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 12, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 48, "osdop_writefull": 16, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 3201, "pq_executed_ops": 3201, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 16516, "get_or_fail_fail": 0, "get_or_fail_success": 16516, "get_started": 0, "get_sum": 4325244, "max": 104857600, "put": 16516, "put_sum": 4325244, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 3443, "put_sum": 32319917, "take": 3443, "take_sum": 32319917, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 3443, "put_sum": 3443, "take": 3443, "take_sum": 3443, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 3201, "get_or_fail_fail": 0, "get_or_fail_success": 3201, "get_started": 0, "get_sum": 323301, "max": 3758096384, "put": 23, "put_sum": 323301, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x5624393ccaa0": { "get": 15849, "get_or_fail_fail": 0, "get_or_fail_success": 15849, "get_started": 0, "get_sum": 31192044, "max": 3758096384, "put": 25, "put_sum": 31158250, "take": 0, "take_sum": 0, "val": 33794, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-06T07:32:42.835 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:32:42.835 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:32:42.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:32:43.059 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:32:43.059 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:32:43.059 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:43.115 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:32:43.116 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:43.163 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:43.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.411+0000 7fde21200640 1 Processor -- start 2024-03-06T07:32:43.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.411+0000 7fde21200640 1 -- start start 2024-03-06T07:32:43.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.412+0000 7fde21200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fde1c15be90 0x7fde1c0b49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde21200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde1c0b5160 con 0x7fde1c0d0780 2024-03-06T07:32:43.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde21200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde1c0ae910 con 0x7fde1c0acd60 2024-03-06T07:32:43.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde21200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fde1c0b4f30 con 0x7fde1c15be90 2024-03-06T07:32:43.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde1a800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fde1c0d0780 0x7fde1c15fb70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37240/0 (socket says 172.21.15.86:37240) 2024-03-06T07:32:43.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde1a800640 1 -- 172.21.15.86:0/98095971 learned_addr learned my addr 172.21.15.86:0/98095971 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:43.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde19e00640 1 --2- 172.21.15.86:0/98095971 >> v2:172.21.15.194:3300/0 conn(0x7fde1c15be90 0x7fde1c0b49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3052943640 0 0) 0x7fde1c0b5160 con 0x7fde1c0d0780 2024-03-06T07:32:43.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.413+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fde00003520 con 0x7fde1c0d0780 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 664654748 0 0) 0x7fde00003520 con 0x7fde1c0d0780 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 >> v2:172.21.15.194:3300/0 conn(0x7fde1c15be90 msgr2=0x7fde1c0b49f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 --2- 172.21.15.86:0/98095971 >> v2:172.21.15.194:3300/0 conn(0x7fde1c15be90 0x7fde1c0b49f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 >> v1:172.21.15.194:6789/0 conn(0x7fde1c0acd60 legacy=0x7fde1c0d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde1c0aebf0 con 0x7fde1c0d0780 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fde140019f0 con 0x7fde1c0d0780 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 >> v1:172.21.15.86:6789/0 conn(0x7fde1c0d0780 legacy=0x7fde1c15fb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde00003a00 0x7fde00003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde00004800 0x7fde00004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000051c0 0x7fde0000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde0000c090 con 0x7fde000051c0 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde0000c240 con 0x7fde00003a00 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde0000c440 con 0x7fde00004800 2024-03-06T07:32:43.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde1a800640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000051c0 0x7fde0000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.414+0000 7fde19e00640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde00003a00 0x7fde00003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.415+0000 7fde1a800640 1 -- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde00004800 msgr2=0x7fde00004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.415+0000 7fde19400640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde00004800 0x7fde00004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.415+0000 7fde1a800640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde00004800 0x7fde00004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.415+0000 7fde1a800640 1 -- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde00003a00 msgr2=0x7fde00003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.415+0000 7fde1a800640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde00003a00 0x7fde00003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.415+0000 7fde1a800640 1 -- 172.21.15.86:0/98095971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde00003520 con 0x7fde000051c0 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.415+0000 7fde19e00640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde00003a00 0x7fde00003e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.416+0000 7fde1a800640 1 -- 172.21.15.86:0/98095971 reap_dead start 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.416+0000 7fde1a800640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000051c0 0x7fde0000b9c0 secure :-1 s=READY pgs=1154 cs=0 l=1 rev1=1 crypto rx=0x7fde1c0af8a0 tx=0x7fde00005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1dee12bbfa67feb2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.416+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde1400a390 con 0x7fde000051c0 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.416+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde14034650 con 0x7fde000051c0 2024-03-06T07:32:43.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.416+0000 7fde18a00640 1 -- 172.21.15.86:0/98095971 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde14034a40 con 0x7fde000051c0 2024-03-06T07:32:43.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.417+0000 7fde21200640 1 -- 172.21.15.86:0/98095971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000051c0 msgr2=0x7fde0000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.417+0000 7fde21200640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000051c0 0x7fde0000b9c0 secure :-1 s=READY pgs=1154 cs=0 l=1 rev1=1 crypto rx=0x7fde1c0af8a0 tx=0x7fde00005830 comp rx=0 tx=0).stop 2024-03-06T07:32:43.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.417+0000 7fde21200640 1 -- 172.21.15.86:0/98095971 shutdown_connections 2024-03-06T07:32:43.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.417+0000 7fde21200640 1 --2- 172.21.15.86:0/98095971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000051c0 0x7fde0000b9c0 unknown :-1 s=CLOSED pgs=1154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.417+0000 7fde21200640 1 -- 172.21.15.86:0/98095971 >> 172.21.15.86:0/98095971 conn(0x7fde1c06b360 msgr2=0x7fde1c06b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:43.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.417+0000 7fde21200640 1 -- 172.21.15.86:0/98095971 shutdown_connections 2024-03-06T07:32:43.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.417+0000 7fde21200640 1 -- 172.21.15.86:0/98095971 wait complete. 2024-03-06T07:32:43.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.419+0000 7fde21200640 1 Processor -- start 2024-03-06T07:32:43.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.419+0000 7fde21200640 1 -- start start 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde21200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde000051c0 0x7fde1c157a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde21200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde1c158320 0x7fde1c1f7310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde21200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde1c0721e0 0x7fde1c156410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde21200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde1c0b2c80 con 0x7fde1c158320 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde21200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde1c0af240 con 0x7fde1c0721e0 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde21200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde1c0b2e00 con 0x7fde000051c0 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde19400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde1c158320 0x7fde1c1f7310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde19400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde1c158320 0x7fde1c1f7310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33544/0 (socket says 172.21.15.86:33544) 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde19400640 1 -- 172.21.15.86:0/466941145 learned_addr learned my addr 172.21.15.86:0/466941145 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde19e00640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde000051c0 0x7fde1c157a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde19400640 1 -- 172.21.15.86:0/466941145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde000051c0 msgr2=0x7fde1c157a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.421+0000 7fde19400640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde000051c0 0x7fde1c157a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde19400640 1 -- 172.21.15.86:0/466941145 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde1c0721e0 msgr2=0x7fde1c156410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde19400640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde1c0721e0 0x7fde1c156410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde19400640 1 -- 172.21.15.86:0/466941145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde1c158870 con 0x7fde1c158320 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde19400640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde1c158320 0x7fde1c1f7310 secure :-1 s=READY pgs=1155 cs=0 l=1 rev1=1 crypto rx=0x7fde0800d5a0 tx=0x7fde0800da60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde12000640 1 -- 172.21.15.86:0/466941145 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde08003610 con 0x7fde1c158320 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde12000640 1 -- 172.21.15.86:0/466941145 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde08017070 con 0x7fde1c158320 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde12000640 1 -- 172.21.15.86:0/466941145 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde080039d0 con 0x7fde1c158320 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde1c071700 con 0x7fde1c158320 2024-03-06T07:32:43.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.422+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde1c071cc0 con 0x7fde1c158320 2024-03-06T07:32:43.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.423+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde1c068070 con 0x7fde1c158320 2024-03-06T07:32:43.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.425+0000 7fde12000640 1 -- 172.21.15.86:0/466941145 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fde08003b70 con 0x7fde1c158320 2024-03-06T07:32:43.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.426+0000 7fde12000640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fddfc06b4d0 0x7fddfc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.426+0000 7fde12000640 1 -- 172.21.15.86:0/466941145 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fde0808a100 con 0x7fde1c158320 2024-03-06T07:32:43.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.427+0000 7fde19e00640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fddfc06b4d0 0x7fddfc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.427+0000 7fde19e00640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fddfc06b4d0 0x7fddfc06d990 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7fde0c0098b0 tx=0x7fde0c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:43.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.432+0000 7fde12000640 1 -- 172.21.15.86:0/466941145 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fde08058f80 con 0x7fde1c158320 2024-03-06T07:32:43.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.629+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fde1c15f520 con 0x7fde1c158320 2024-03-06T07:32:43.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.631+0000 7fde12000640 1 -- 172.21.15.86:0/466941145 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v98) v1 ==== 74+0+16879 (secure 0 0 0) 0x7fde08058880 con 0x7fde1c158320 2024-03-06T07:32:43.634 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:43.634 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":98,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:20.051194+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:32:01.521762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-06T07:32:02.759692+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"94","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6835/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.86:6834/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.194:6834/2712222804":"2024-03-07T07:32:14.424511+0000","172.21.15.194:6832/2712222804":"2024-03-07T07:32:14.424511+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:43.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fddfc06b4d0 msgr2=0x7fddfc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fddfc06b4d0 0x7fddfc06d990 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7fde0c0098b0 tx=0x7fde0c008040 comp rx=0 tx=0).stop 2024-03-06T07:32:43.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde1c158320 msgr2=0x7fde1c1f7310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde1c158320 0x7fde1c1f7310 secure :-1 s=READY pgs=1155 cs=0 l=1 rev1=1 crypto rx=0x7fde0800d5a0 tx=0x7fde0800da60 comp rx=0 tx=0).stop 2024-03-06T07:32:43.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 shutdown_connections 2024-03-06T07:32:43.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fddfc06b4d0 0x7fddfc06d990 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde1c0721e0 0x7fde1c156410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde1c158320 0x7fde1c1f7310 unknown :-1 s=CLOSED pgs=1155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 --2- 172.21.15.86:0/466941145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde000051c0 0x7fde1c157a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 >> 172.21.15.86:0/466941145 conn(0x7fde1c06b360 msgr2=0x7fde1c0af8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:43.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 shutdown_connections 2024-03-06T07:32:43.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.634+0000 7fde21200640 1 -- 172.21.15.86:0/466941145 wait complete. 2024-03-06T07:32:43.654 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 14, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:32:01.521762+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '93', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 15, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:32:02.759692+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '94', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:32:43.654 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:32:43.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.855+0000 7f796fe00640 1 Processor -- start 2024-03-06T07:32:43.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.855+0000 7f796fe00640 1 -- start start 2024-03-06T07:32:43.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.857+0000 7f796fe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f79680d0780 0x7f796815e720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.857+0000 7f796fe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f796815f370 con 0x7f79680b3340 2024-03-06T07:32:43.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.857+0000 7f796fe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f796806cb20 con 0x7f796806d600 2024-03-06T07:32:43.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.857+0000 7f796fe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f796815f650 con 0x7f79680d0780 2024-03-06T07:32:43.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.857+0000 7f796d400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f79680b3340 0x7f79680d0070 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37242/0 (socket says 172.21.15.86:37242) 2024-03-06T07:32:43.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.857+0000 7f796d400640 1 -- 172.21.15.86:0/2553037786 learned_addr learned my addr 172.21.15.86:0/2553037786 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:43.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f796ca00640 1 --2- 172.21.15.86:0/2553037786 >> v2:172.21.15.194:3300/0 conn(0x7f79680d0780 0x7f796815e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1317933228 0 0) 0x7f796815f370 con 0x7f79680b3340 2024-03-06T07:32:43.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7950003520 con 0x7f79680b3340 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2164081039 0 0) 0x7f7950003520 con 0x7f79680b3340 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 >> v2:172.21.15.194:3300/0 conn(0x7f79680d0780 msgr2=0x7f796815e720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 --2- 172.21.15.86:0/2553037786 >> v2:172.21.15.194:3300/0 conn(0x7f79680d0780 0x7f796815e720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 >> v1:172.21.15.194:6789/0 conn(0x7f796806d600 legacy=0x7f79680acd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f796806ce00 con 0x7f79680b3340 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f795c001670 con 0x7f79680b3340 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 >> v1:172.21.15.86:6789/0 conn(0x7f79680b3340 legacy=0x7f79680d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79500039b0 0x7f7950003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.858+0000 7f7967e00640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7950004800 0x7f7950004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f7967e00640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500052e0 0x7f795000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f796815f650 con 0x7f79500039b0 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f795000c1b0 con 0x7f79500052e0 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f795000c3b0 con 0x7f7950004800 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796d400640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79500039b0 0x7f7950003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796de00640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500052e0 0x7f795000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796d400640 1 -- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7950004800 msgr2=0x7f7950004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:43.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796d400640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7950004800 0x7f7950004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796d400640 1 -- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500052e0 msgr2=0x7f795000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:43.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796d400640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500052e0 0x7f795000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796d400640 1 -- 172.21.15.86:0/2553037786 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7950003520 con 0x7f79500039b0 2024-03-06T07:32:43.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796de00640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500052e0 0x7f795000b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:43.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.859+0000 7f796de00640 1 -- 172.21.15.86:0/2553037786 reap_dead start 2024-03-06T07:32:43.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.860+0000 7f796d400640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79500039b0 0x7f7950003dd0 secure :-1 s=READY pgs=1157 cs=0 l=1 rev1=1 crypto rx=0x7f795c00c210 tx=0x7f7950005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=665139cf24115408 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f795c007cf0 con 0x7f79500039b0 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f795c030070 con 0x7f79500039b0 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f7967e00640 1 -- 172.21.15.86:0/2553037786 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f795c005d70 con 0x7f79500039b0 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f796fe00640 1 -- 172.21.15.86:0/2553037786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79500039b0 msgr2=0x7f7950003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f796fe00640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79500039b0 0x7f7950003dd0 secure :-1 s=READY pgs=1157 cs=0 l=1 rev1=1 crypto rx=0x7f795c00c210 tx=0x7f7950005830 comp rx=0 tx=0).stop 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f796fe00640 1 -- 172.21.15.86:0/2553037786 shutdown_connections 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f796fe00640 1 --2- 172.21.15.86:0/2553037786 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79500039b0 0x7f7950003dd0 unknown :-1 s=CLOSED pgs=1157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.861+0000 7f796fe00640 1 -- 172.21.15.86:0/2553037786 >> 172.21.15.86:0/2553037786 conn(0x7f796806b360 msgr2=0x7f796806b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.862+0000 7f796fe00640 1 -- 172.21.15.86:0/2553037786 shutdown_connections 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.862+0000 7f796fe00640 1 -- 172.21.15.86:0/2553037786 wait complete. 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.863+0000 7f796fe00640 1 Processor -- start 2024-03-06T07:32:43.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.863+0000 7f796fe00640 1 -- start start 2024-03-06T07:32:43.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.864+0000 7f796fe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500039b0 0x7f79681e8c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.864+0000 7f796fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79681f7040 0x7f79681e91a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.864+0000 7f796fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79680720b0 0x7f7968072530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.864+0000 7f796fe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79680d0070 con 0x7f79680720b0 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.864+0000 7f796fe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79680b3720 con 0x7f79500039b0 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.864+0000 7f796fe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79680d01f0 con 0x7f79681f7040 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79680720b0 0x7f7968072530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79680720b0 0x7f7968072530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33564/0 (socket says 172.21.15.86:33564) 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 -- 172.21.15.86:0/3560096597 learned_addr learned my addr 172.21.15.86:0/3560096597 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796d400640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500039b0 0x7f79681e8c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796ca00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79681f7040 0x7f79681e91a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 -- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79681f7040 msgr2=0x7f79681e91a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79681f7040 0x7f79681e91a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 -- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500039b0 msgr2=0x7f79681e8c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500039b0 0x7f79681e8c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 -- 172.21.15.86:0/3560096597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7968161450 con 0x7f79680720b0 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796ca00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79681f7040 0x7f79681e91a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:43.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.865+0000 7f796de00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79680720b0 0x7f7968072530 secure :-1 s=READY pgs=1158 cs=0 l=1 rev1=1 crypto rx=0x7f796809e0b0 tx=0x7f796000d9d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:43.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.866+0000 7f7965600640 1 -- 172.21.15.86:0/3560096597 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7960018070 con 0x7f79680720b0 2024-03-06T07:32:43.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.866+0000 7f7965600640 1 -- 172.21.15.86:0/3560096597 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7960003d80 con 0x7f79680720b0 2024-03-06T07:32:43.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.866+0000 7f7965600640 1 -- 172.21.15.86:0/3560096597 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7960005270 con 0x7f79680720b0 2024-03-06T07:32:43.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.866+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f79681e97d0 con 0x7f79680720b0 2024-03-06T07:32:43.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.866+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f79681e8060 con 0x7f79680720b0 2024-03-06T07:32:43.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.867+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7968068070 con 0x7f79680720b0 2024-03-06T07:32:43.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.868+0000 7f7965600640 1 -- 172.21.15.86:0/3560096597 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f796001b070 con 0x7f79680720b0 2024-03-06T07:32:43.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.869+0000 7f7965600640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f794806b4d0 0x7f794806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:43.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.869+0000 7f7965600640 1 -- 172.21.15.86:0/3560096597 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f7960089710 con 0x7f79680720b0 2024-03-06T07:32:43.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.869+0000 7f796d400640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f794806b4d0 0x7f794806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:43.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.870+0000 7f796d400640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f794806b4d0 0x7f794806d990 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f795c034000 tx=0x7f795c007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:43.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:43.874+0000 7f7965600640 1 -- 172.21.15.86:0/3560096597 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7960057f20 con 0x7f79680720b0 2024-03-06T07:32:44.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.079+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f796806e6a0 con 0x7f79680720b0 2024-03-06T07:32:44.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.079+0000 7f7965600640 1 -- 172.21.15.86:0/3560096597 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v98) v1 ==== 93+0+10 (secure 0 0 0) 0x7f796005bfc0 con 0x7f79680720b0 2024-03-06T07:32:44.082 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:32:44.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f794806b4d0 msgr2=0x7f794806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f794806b4d0 0x7f794806d990 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f795c034000 tx=0x7f795c007000 comp rx=0 tx=0).stop 2024-03-06T07:32:44.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79680720b0 msgr2=0x7f7968072530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79680720b0 0x7f7968072530 secure :-1 s=READY pgs=1158 cs=0 l=1 rev1=1 crypto rx=0x7f796809e0b0 tx=0x7f796000d9d0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 shutdown_connections 2024-03-06T07:32:44.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f794806b4d0 0x7f794806d990 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79680720b0 0x7f7968072530 unknown :-1 s=CLOSED pgs=1158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79681f7040 0x7f79681e91a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 --2- 172.21.15.86:0/3560096597 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79500039b0 0x7f79681e8c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.082+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 >> 172.21.15.86:0/3560096597 conn(0x7f796806b360 msgr2=0x7f796815d690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:44.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.083+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 shutdown_connections 2024-03-06T07:32:44.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.083+0000 7f796fe00640 1 -- 172.21.15.86:0/3560096597 wait complete. 2024-03-06T07:32:44.110 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:32:44.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.319+0000 7fb294800640 1 Processor -- start 2024-03-06T07:32:44.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.319+0000 7fb294800640 1 -- start start 2024-03-06T07:32:44.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb294800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c06c9f0 0x7fb28c070ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb294800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb28c06cdd0 0x7fb28c06d210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb294800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb28c15a9a0 con 0x7fb28c0b3a50 2024-03-06T07:32:44.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb294800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb28c071370 con 0x7fb28c06cdd0 2024-03-06T07:32:44.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb294800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb28c071650 con 0x7fb28c06c9f0 2024-03-06T07:32:44.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb291400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb28c0b3a50 0x7fb28c0acea0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37248/0 (socket says 172.21.15.86:37248) 2024-03-06T07:32:44.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb291400640 1 -- 172.21.15.86:0/3576253837 learned_addr learned my addr 172.21.15.86:0/3576253837 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb291e00640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c06c9f0 0x7fb28c070ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb292800640 1 --2- 172.21.15.86:0/3576253837 >> v2:172.21.15.194:3300/0 conn(0x7fb28c06cdd0 0x7fb28c06d210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.322+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1813816967 0 0) 0x7fb28c15a9a0 con 0x7fb28c0b3a50 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb274005730 con 0x7fb28c0b3a50 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3835089879 0 0) 0x7fb274005730 con 0x7fb28c0b3a50 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c06c9f0 msgr2=0x7fb28c070ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c06c9f0 0x7fb28c070ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 >> v2:172.21.15.194:3300/0 conn(0x7fb28c06cdd0 msgr2=0x7fb28c06d210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 --2- 172.21.15.86:0/3576253837 >> v2:172.21.15.194:3300/0 conn(0x7fb28c06cdd0 0x7fb28c06d210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb28c071930 con 0x7fb28c0b3a50 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fb27c001690 con 0x7fb28c0b3a50 2024-03-06T07:32:44.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 >> v1:172.21.15.86:6789/0 conn(0x7fb28c0b3a50 legacy=0x7fb28c0acea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb274005bc0 0x7fb274005fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb274006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c06c9f0 0x7fb2740073d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb28c071370 con 0x7fb274006a10 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb28c071650 con 0x7fb274005bc0 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb27400dbd0 con 0x7fb28c06c9f0 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.323+0000 7fb291400640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb274006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.324+0000 7fb291e00640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb274005bc0 0x7fb274005fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.324+0000 7fb291400640 1 -- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c06c9f0 msgr2=0x7fb2740073d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.324+0000 7fb291400640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c06c9f0 0x7fb2740073d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.324+0000 7fb291400640 1 -- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb274005bc0 msgr2=0x7fb274005fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.324+0000 7fb291400640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb274005bc0 0x7fb274005fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.324+0000 7fb291400640 1 -- 172.21.15.86:0/3576253837 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb274005730 con 0x7fb274006a10 2024-03-06T07:32:44.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.325+0000 7fb291400640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb274006e90 secure :-1 s=READY pgs=1160 cs=0 l=1 rev1=1 crypto rx=0x7fb27c00c210 tx=0x7fb274007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=20203216c2f0a241 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.325+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb27c007d70 con 0x7fb274006a10 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.325+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb27c030070 con 0x7fb274006a10 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.325+0000 7fb290a00640 1 -- 172.21.15.86:0/3576253837 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb27c02cd60 con 0x7fb274006a10 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.326+0000 7fb294800640 1 -- 172.21.15.86:0/3576253837 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 msgr2=0x7fb274006e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.326+0000 7fb294800640 1 --2- 172.21.15.86:0/3576253837 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb274006e90 secure :-1 s=READY pgs=1160 cs=0 l=1 rev1=1 crypto rx=0x7fb27c00c210 tx=0x7fb274007a40 comp rx=0 tx=0).stop 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.326+0000 7fb292800640 1 -- 172.21.15.86:0/3576253837 reap_dead start 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.326+0000 7fb294800640 1 -- 172.21.15.86:0/3576253837 shutdown_connections 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.326+0000 7fb294800640 1 -- 172.21.15.86:0/3576253837 >> 172.21.15.86:0/3576253837 conn(0x7fb28c06b360 msgr2=0x7fb28c06b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.326+0000 7fb294800640 1 -- 172.21.15.86:0/3576253837 shutdown_connections 2024-03-06T07:32:44.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.326+0000 7fb294800640 1 -- 172.21.15.86:0/3576253837 wait complete. 2024-03-06T07:32:44.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.328+0000 7fb294800640 1 Processor -- start 2024-03-06T07:32:44.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.328+0000 7fb294800640 1 -- start start 2024-03-06T07:32:44.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb294800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb28c156190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb294800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c09bfc0 0x7fb28c09c420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb294800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb28c0ae1d0 0x7fb28c0ae650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb294800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb28c0ad5b0 con 0x7fb274006a10 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb294800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb28c071ef0 con 0x7fb28c0ae1d0 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb294800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb28c06d750 con 0x7fb28c09bfc0 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb28c156190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb28c156190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33586/0 (socket says 172.21.15.86:33586) 2024-03-06T07:32:44.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 -- 172.21.15.86:0/1053969679 learned_addr learned my addr 172.21.15.86:0/1053969679 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 -- 172.21.15.86:0/1053969679 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c09bfc0 msgr2=0x7fb28c09c420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c09bfc0 0x7fb28c09c420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 -- 172.21.15.86:0/1053969679 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb28c0ae1d0 msgr2=0x7fb28c0ae650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb28c0ae1d0 0x7fb28c0ae650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.329+0000 7fb291e00640 1 -- 172.21.15.86:0/1053969679 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb28c1f4580 con 0x7fb274006a10 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.330+0000 7fb291e00640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb28c156190 secure :-1 s=READY pgs=1161 cs=0 l=1 rev1=1 crypto rx=0x7fb28000f580 tx=0x7fb28000fa40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.330+0000 7fb286000640 1 -- 172.21.15.86:0/1053969679 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb280013810 con 0x7fb274006a10 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.330+0000 7fb286000640 1 -- 172.21.15.86:0/1053969679 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb28001a070 con 0x7fb274006a10 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.330+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb28c1f4860 con 0x7fb274006a10 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.330+0000 7fb286000640 1 -- 172.21.15.86:0/1053969679 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb280013bb0 con 0x7fb274006a10 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.330+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb28c09ca80 con 0x7fb274006a10 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.331+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb28c0aeb90 con 0x7fb274006a10 2024-03-06T07:32:44.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.332+0000 7fb286000640 1 -- 172.21.15.86:0/1053969679 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb280012070 con 0x7fb274006a10 2024-03-06T07:32:44.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.333+0000 7fb286000640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb27006b500 0x7fb27006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.333+0000 7fb286000640 1 -- 172.21.15.86:0/1053969679 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fb28008a0e0 con 0x7fb274006a10 2024-03-06T07:32:44.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.333+0000 7fb291400640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb27006b500 0x7fb27006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.334+0000 7fb291400640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb27006b500 0x7fb27006d9c0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7fb27c034000 tx=0x7fb27c039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:44.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.338+0000 7fb286000640 1 -- 172.21.15.86:0/1053969679 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb28001c310 con 0x7fb274006a10 2024-03-06T07:32:44.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.534+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fb28c068070 con 0x7fb274006a10 2024-03-06T07:32:44.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.535+0000 7fb286000640 1 -- 172.21.15.86:0/1053969679 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v98) v1 ==== 104+0+11 (secure 0 0 0) 0x7fb280058ec0 con 0x7fb274006a10 2024-03-06T07:32:44.538 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb27006b500 msgr2=0x7fb27006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb27006b500 0x7fb27006d9c0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7fb27c034000 tx=0x7fb27c039040 comp rx=0 tx=0).stop 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 msgr2=0x7fb28c156190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb28c156190 secure :-1 s=READY pgs=1161 cs=0 l=1 rev1=1 crypto rx=0x7fb28000f580 tx=0x7fb28000fa40 comp rx=0 tx=0).stop 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 shutdown_connections 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb27006b500 0x7fb27006d9c0 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb28c0ae1d0 0x7fb28c0ae650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb28c09bfc0 0x7fb28c09c420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 --2- 172.21.15.86:0/1053969679 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb274006a10 0x7fb28c156190 unknown :-1 s=CLOSED pgs=1161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 >> 172.21.15.86:0/1053969679 conn(0x7fb28c06b360 msgr2=0x7fb28c152c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 shutdown_connections 2024-03-06T07:32:44.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.538+0000 7fb294800640 1 -- 172.21.15.86:0/1053969679 wait complete. 2024-03-06T07:32:44.560 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:32:44.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.765+0000 7f68ee200640 1 Processor -- start 2024-03-06T07:32:44.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.765+0000 7f68ee200640 1 -- start start 2024-03-06T07:32:44.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.766+0000 7f68ee200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f68e806c9f0 0x7f68e8070ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.766+0000 7f68ee200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f68e80b3a50 0x7f68e80acea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.766+0000 7f68ee200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e80ad3e0 0x7f68e806cdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.766+0000 7f68ee200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f68e815a9a0 con 0x7f68e80b3a50 2024-03-06T07:32:44.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.766+0000 7f68ee200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f68e8071370 con 0x7f68e806c9f0 2024-03-06T07:32:44.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.766+0000 7f68ee200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f68e8071650 con 0x7f68e80ad3e0 2024-03-06T07:32:44.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f68e80b3a50 0x7f68e80acea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f68e80b3a50 0x7f68e80acea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33602/0 (socket says 172.21.15.86:33602) 2024-03-06T07:32:44.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 -- 172.21.15.86:0/4049532480 learned_addr learned my addr 172.21.15.86:0/4049532480 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68e7400640 1 --2- 172.21.15.86:0/4049532480 >> v2:172.21.15.194:3300/0 conn(0x7f68e806c9f0 0x7f68e8070ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68e7e00640 1 --2- 172.21.15.86:0/4049532480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e80ad3e0 0x7f68e806cdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 -- 172.21.15.86:0/4049532480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e80ad3e0 msgr2=0x7f68e806cdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 --2- 172.21.15.86:0/4049532480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e80ad3e0 0x7f68e806cdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 -- 172.21.15.86:0/4049532480 >> v2:172.21.15.194:3300/0 conn(0x7f68e806c9f0 msgr2=0x7f68e8070ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 --2- 172.21.15.86:0/4049532480 >> v2:172.21.15.194:3300/0 conn(0x7f68e806c9f0 0x7f68e8070ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68dfe00640 1 -- 172.21.15.86:0/4049532480 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68e8071930 con 0x7f68e80b3a50 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.767+0000 7f68e7e00640 1 --2- 172.21.15.86:0/4049532480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e80ad3e0 0x7f68e806cdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:44.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.768+0000 7f68dfe00640 1 --2- 172.21.15.86:0/4049532480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e80acea0 secure :-1 s=READY pgs=1162 cs=0 l=1 rev1=1 crypto rx=0x7f68e806ede0 tx=0x7f68d002b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f924f6b86ef38c87 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:44.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68e6a00640 1 -- 172.21.15.86:0/4049532480 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68d0035020 con 0x7f68e80b3a50 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68e6a00640 1 -- 172.21.15.86:0/4049532480 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f68d002d040 con 0x7f68e80b3a50 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68e6a00640 1 -- 172.21.15.86:0/4049532480 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68d0031a80 con 0x7f68e80b3a50 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68ee200640 1 -- 172.21.15.86:0/4049532480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 msgr2=0x7f68e80acea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68ee200640 1 --2- 172.21.15.86:0/4049532480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e80acea0 secure :-1 s=READY pgs=1162 cs=0 l=1 rev1=1 crypto rx=0x7f68e806ede0 tx=0x7f68d002b900 comp rx=0 tx=0).stop 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68ee200640 1 -- 172.21.15.86:0/4049532480 shutdown_connections 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68ee200640 1 --2- 172.21.15.86:0/4049532480 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e80ad3e0 0x7f68e806cdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68ee200640 1 --2- 172.21.15.86:0/4049532480 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e80acea0 unknown :-1 s=CLOSED pgs=1162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68ee200640 1 --2- 172.21.15.86:0/4049532480 >> v2:172.21.15.194:3300/0 conn(0x7f68e806c9f0 0x7f68e8070ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.769+0000 7f68ee200640 1 -- 172.21.15.86:0/4049532480 >> 172.21.15.86:0/4049532480 conn(0x7f68e806b360 msgr2=0x7f68e806b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.770+0000 7f68ee200640 1 -- 172.21.15.86:0/4049532480 shutdown_connections 2024-03-06T07:32:44.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.770+0000 7f68ee200640 1 -- 172.21.15.86:0/4049532480 wait complete. 2024-03-06T07:32:44.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.771+0000 7f68ee200640 1 Processor -- start 2024-03-06T07:32:44.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.772+0000 7f68ee200640 1 -- start start 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.772+0000 7f68ee200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e806c9f0 0x7f68e809ab10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.772+0000 7f68ee200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f68e80ad3e0 0x7f68e81f7920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68ee200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e813efe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68ee200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68e80b3ee0 con 0x7f68e80b3a50 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68ee200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68e8071ef0 con 0x7f68e80ad3e0 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68ee200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f68e80b4060 con 0x7f68e806c9f0 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e813efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e813efe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33614/0 (socket says 172.21.15.86:33614) 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 -- 172.21.15.86:0/2148374199 learned_addr learned my addr 172.21.15.86:0/2148374199 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7400640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e806c9f0 0x7f68e809ab10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68dfe00640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f68e80ad3e0 0x7f68e81f7920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 -- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e806c9f0 msgr2=0x7f68e809ab10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e806c9f0 0x7f68e809ab10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 -- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f68e80ad3e0 msgr2=0x7f68e81f7920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f68e80ad3e0 0x7f68e81f7920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.773+0000 7f68e7e00640 1 -- 172.21.15.86:0/2148374199 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68e81456e0 con 0x7f68e80b3a50 2024-03-06T07:32:44.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.774+0000 7f68e7e00640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e813efe0 secure :-1 s=READY pgs=1163 cs=0 l=1 rev1=1 crypto rx=0x7f68e000ce40 tx=0x7f68e000a710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:44.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.774+0000 7f68df400640 1 -- 172.21.15.86:0/2148374199 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68e001a070 con 0x7f68e80b3a50 2024-03-06T07:32:44.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.774+0000 7f68df400640 1 -- 172.21.15.86:0/2148374199 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f68e000d840 con 0x7f68e80b3a50 2024-03-06T07:32:44.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.774+0000 7f68df400640 1 -- 172.21.15.86:0/2148374199 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68e000dc30 con 0x7f68e80b3a50 2024-03-06T07:32:44.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.774+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f68e813f520 con 0x7f68e80b3a50 2024-03-06T07:32:44.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.774+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f68e8141cb0 con 0x7f68e80b3a50 2024-03-06T07:32:44.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.775+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f68e8068070 con 0x7f68e80b3a50 2024-03-06T07:32:44.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.776+0000 7f68df400640 1 -- 172.21.15.86:0/2148374199 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f68e001d070 con 0x7f68e80b3a50 2024-03-06T07:32:44.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.777+0000 7f68df400640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f68c406b4d0 0x7f68c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:44.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.777+0000 7f68df400640 1 -- 172.21.15.86:0/2148374199 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f68e008a9a0 con 0x7f68e80b3a50 2024-03-06T07:32:44.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.777+0000 7f68e7400640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f68c406b4d0 0x7f68c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:44.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.778+0000 7f68e7400640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f68c406b4d0 0x7f68c406d990 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f68d8006fd0 tx=0x7f68d8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:44.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.781+0000 7f68df400640 1 -- 172.21.15.86:0/2148374199 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f68e0059800 con 0x7f68e80b3a50 2024-03-06T07:32:44.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.979+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f68e80aeb80 con 0x7f68e80b3a50 2024-03-06T07:32:44.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.980+0000 7f68df400640 1 -- 172.21.15.86:0/2148374199 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v98) v1 ==== 100+0+11 (secure 0 0 0) 0x7f68e0059100 con 0x7f68e80b3a50 2024-03-06T07:32:44.982 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:32:44.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f68c406b4d0 msgr2=0x7f68c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f68c406b4d0 0x7f68c406d990 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f68d8006fd0 tx=0x7f68d8008040 comp rx=0 tx=0).stop 2024-03-06T07:32:44.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 msgr2=0x7f68e813efe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:44.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e813efe0 secure :-1 s=READY pgs=1163 cs=0 l=1 rev1=1 crypto rx=0x7f68e000ce40 tx=0x7f68e000a710 comp rx=0 tx=0).stop 2024-03-06T07:32:44.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 shutdown_connections 2024-03-06T07:32:44.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f68c406b4d0 0x7f68c406d990 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f68e80b3a50 0x7f68e813efe0 unknown :-1 s=CLOSED pgs=1163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f68e80ad3e0 0x7f68e81f7920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 --2- 172.21.15.86:0/2148374199 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f68e806c9f0 0x7f68e809ab10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:44.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 >> 172.21.15.86:0/2148374199 conn(0x7f68e806b360 msgr2=0x7f68e8159c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:44.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 shutdown_connections 2024-03-06T07:32:44.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:44.982+0000 7f68ee200640 1 -- 172.21.15.86:0/2148374199 wait complete. 2024-03-06T07:32:45.005 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:32:45.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.206+0000 7f4709600640 1 Processor -- start 2024-03-06T07:32:45.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.207+0000 7f4709600640 1 -- start start 2024-03-06T07:32:45.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.208+0000 7f4709600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4704145920 0x7f470406c920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.208+0000 7f4709600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040cd980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.208+0000 7f4709600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f47040cdec0 con 0x7f47040cc490 2024-03-06T07:32:45.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.208+0000 7f4709600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f47041601f0 con 0x7f4704145920 2024-03-06T07:32:45.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.208+0000 7f4709600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47041604d0 con 0x7f470415faa0 2024-03-06T07:32:45.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.209+0000 7f46fb400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040cd980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.209+0000 7f4703e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4704145920 0x7f470406c920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.209+0000 7f4703e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4704145920 0x7f470406c920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:42932/0 (socket says 172.21.15.86:42932) 2024-03-06T07:32:45.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.209+0000 7f4703e00640 1 -- 172.21.15.86:0/3545347380 learned_addr learned my addr 172.21.15.86:0/3545347380 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:45.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.210+0000 7f4703e00640 1 -- 172.21.15.86:0/3545347380 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 msgr2=0x7f47040cd980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:45.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.210+0000 7f4703e00640 1 --2- 172.21.15.86:0/3545347380 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040cd980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.210+0000 7f4703e00640 1 -- 172.21.15.86:0/3545347380 >> v1:172.21.15.194:6789/0 conn(0x7f47040cc490 legacy=0x7f4704071dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.210+0000 7f4703e00640 1 -- 172.21.15.86:0/3545347380 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47041607b0 con 0x7f4704145920 2024-03-06T07:32:45.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.211+0000 7f4703e00640 1 --2- 172.21.15.86:0/3545347380 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4704145920 0x7f470406c920 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f46ec009870 tx=0x7f46ec02ba20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5e9b6738debf691e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:45.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.211+0000 7f4703400640 1 -- 172.21.15.86:0/3545347380 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46ec02f840 con 0x7f4704145920 2024-03-06T07:32:45.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4703400640 1 -- 172.21.15.86:0/3545347380 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46ec02f9e0 con 0x7f4704145920 2024-03-06T07:32:45.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4703400640 1 -- 172.21.15.86:0/3545347380 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46ec02fdb0 con 0x7f4704145920 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 -- 172.21.15.86:0/3545347380 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4704145920 msgr2=0x7f470406c920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 --2- 172.21.15.86:0/3545347380 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4704145920 0x7f470406c920 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f46ec009870 tx=0x7f46ec02ba20 comp rx=0 tx=0).stop 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 -- 172.21.15.86:0/3545347380 shutdown_connections 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 --2- 172.21.15.86:0/3545347380 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040cd980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 --2- 172.21.15.86:0/3545347380 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4704145920 0x7f470406c920 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 -- 172.21.15.86:0/3545347380 >> 172.21.15.86:0/3545347380 conn(0x7f47040b1770 msgr2=0x7f47040b1b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 -- 172.21.15.86:0/3545347380 shutdown_connections 2024-03-06T07:32:45.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.212+0000 7f4709600640 1 -- 172.21.15.86:0/3545347380 wait complete. 2024-03-06T07:32:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.214+0000 7f4709600640 1 Processor -- start 2024-03-06T07:32:45.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.214+0000 7f4709600640 1 -- start start 2024-03-06T07:32:45.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f4709600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f47040cc490 0x7f47040b2b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f4709600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4704145920 0x7f470415e580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f4709600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040b4670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f4709600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47040cc940 con 0x7f4704145920 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f4709600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f470415fec0 con 0x7f47040cc490 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f4709600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47040ccac0 con 0x7f470415faa0 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f46fb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4704145920 0x7f470415e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f46fb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4704145920 0x7f470415e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33628/0 (socket says 172.21.15.86:33628) 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.215+0000 7f46fb400640 1 -- 172.21.15.86:0/2295616323 learned_addr learned my addr 172.21.15.86:0/2295616323 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f4703e00640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f47040cc490 0x7f47040b2b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f4708c00640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040b4670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f46fb400640 1 -- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 msgr2=0x7f47040b4670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f46fb400640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040b4670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f46fb400640 1 -- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f47040cc490 msgr2=0x7f47040b2b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f46fb400640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f47040cc490 0x7f47040b2b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f46fb400640 1 -- 172.21.15.86:0/2295616323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47040b4bb0 con 0x7f4704145920 2024-03-06T07:32:45.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f4708c00640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040b4670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:45.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f46fb400640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4704145920 0x7f470415e580 secure :-1 s=READY pgs=1164 cs=0 l=1 rev1=1 crypto rx=0x7f46f400e840 tx=0x7f46f400ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:45.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f4700c00640 1 -- 172.21.15.86:0/2295616323 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46f4017070 con 0x7f4704145920 2024-03-06T07:32:45.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f4700c00640 1 -- 172.21.15.86:0/2295616323 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46f400c940 con 0x7f4704145920 2024-03-06T07:32:45.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.216+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47040b3150 con 0x7f4704145920 2024-03-06T07:32:45.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.217+0000 7f4700c00640 1 -- 172.21.15.86:0/2295616323 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46f400cd10 con 0x7f4704145920 2024-03-06T07:32:45.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.217+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47040b3710 con 0x7f4704145920 2024-03-06T07:32:45.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.218+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f47040cc210 con 0x7f4704145920 2024-03-06T07:32:45.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.219+0000 7f4700c00640 1 -- 172.21.15.86:0/2295616323 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f46f401a070 con 0x7f4704145920 2024-03-06T07:32:45.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.219+0000 7f4700c00640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46e006b4d0 0x7f46e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.220+0000 7f4700c00640 1 -- 172.21.15.86:0/2295616323 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f46f408a560 con 0x7f4704145920 2024-03-06T07:32:45.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.220+0000 7f4703e00640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46e006b4d0 0x7f46e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.221+0000 7f4703e00640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46e006b4d0 0x7f46e006d990 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f46ec036000 tx=0x7f46ec039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:45.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.224+0000 7f4700c00640 1 -- 172.21.15.86:0/2295616323 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f47040cc210 con 0x7f4704145920 2024-03-06T07:32:45.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.434+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f470404f330 con 0x7f4704145920 2024-03-06T07:32:45.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.434+0000 7f4700c00640 1 -- 172.21.15.86:0/2295616323 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 4 entries v98) v1 ==== 80+0+244 (secure 0 0 0) 0x7f46f4059320 con 0x7f4704145920 2024-03-06T07:32:45.437 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.15.86:6835/883748988 2024-03-07T07:32:20.051162+0000 2024-03-06T07:32:45.437 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.15.86:6834/883748988 2024-03-07T07:32:20.051162+0000 2024-03-06T07:32:45.437 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.15.194:6834/2712222804 2024-03-07T07:32:14.424511+0000 2024-03-06T07:32:45.437 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.15.194:6832/2712222804 2024-03-07T07:32:14.424511+0000 2024-03-06T07:32:45.437 INFO:teuthology.orchestra.run.smithi086.stderr:listed 4 entries 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46e006b4d0 msgr2=0x7f46e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46e006b4d0 0x7f46e006d990 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f46ec036000 tx=0x7f46ec039040 comp rx=0 tx=0).stop 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4704145920 msgr2=0x7f470415e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4704145920 0x7f470415e580 secure :-1 s=READY pgs=1164 cs=0 l=1 rev1=1 crypto rx=0x7f46f400e840 tx=0x7f46f400ed00 comp rx=0 tx=0).stop 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 shutdown_connections 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46e006b4d0 0x7f46e006d990 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f470415faa0 0x7f47040b4670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4704145920 0x7f470415e580 unknown :-1 s=CLOSED pgs=1164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 --2- 172.21.15.86:0/2295616323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f47040cc490 0x7f47040b2b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 >> 172.21.15.86:0/2295616323 conn(0x7f47040b1770 msgr2=0x7f47040b1b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:45.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 shutdown_connections 2024-03-06T07:32:45.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.437+0000 7f4709600640 1 -- 172.21.15.86:0/2295616323 wait complete. 2024-03-06T07:32:45.458 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:32:45.458 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:45.458 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:32:45.574 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:32:45.574 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:32:45.574 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:32:45.630 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:32:45.630 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:45.670 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:32:45.670 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:32:45.670 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:45.710 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:32:45.710 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:32:45.711 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:45.760 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:32:45.761 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:45.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.950+0000 7fb317800640 1 Processor -- start 2024-03-06T07:32:45.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.950+0000 7fb317800640 1 -- start start 2024-03-06T07:32:45.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb317800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb31015e2d0 0x7fb3100d1ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb317800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb3100cebd0 con 0x7fb31015e6b0 2024-03-06T07:32:45.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb317800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb3100d23e0 con 0x7fb3100d2db0 2024-03-06T07:32:45.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb317800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb3100d06f0 con 0x7fb31015e2d0 2024-03-06T07:32:45.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb307e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb31015e6b0 0x7fb3100d26a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44480/0 (socket says 172.21.15.86:44480) 2024-03-06T07:32:45.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb307e00640 1 -- 172.21.15.86:0/2383985234 learned_addr learned my addr 172.21.15.86:0/2383985234 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:45.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb314e00640 1 --2- 172.21.15.86:0/2383985234 >> v2:172.21.15.194:3300/0 conn(0x7fb31015e2d0 0x7fb3100d1ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3695698007 0 0) 0x7fb2f8001110 con 0x7fb31015e6b0 2024-03-06T07:32:45.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.952+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb2f4003520 con 0x7fb31015e6b0 2024-03-06T07:32:45.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 973573728 0 0) 0x7fb2f4003520 con 0x7fb31015e6b0 2024-03-06T07:32:45.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 >> v2:172.21.15.194:3300/0 conn(0x7fb31015e2d0 msgr2=0x7fb3100d1ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:45.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 --2- 172.21.15.86:0/2383985234 >> v2:172.21.15.194:3300/0 conn(0x7fb31015e2d0 0x7fb3100d1ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 >> v1:172.21.15.194:6789/0 conn(0x7fb3100d2db0 legacy=0x7fb3100cf070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3100d09d0 con 0x7fb31015e6b0 2024-03-06T07:32:45.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fb2f8001840 con 0x7fb31015e6b0 2024-03-06T07:32:45.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 >> v1:172.21.15.86:6789/0 conn(0x7fb31015e6b0 legacy=0x7fb3100d26a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2f40039b0 0x7fb2f4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f4004800 0x7fb2f4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2f40051c0 0x7fb2f400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3100d06f0 con 0x7fb2f40051c0 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2f400c060 con 0x7fb2f4004800 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb2f400c260 con 0x7fb2f40039b0 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.953+0000 7fb315800640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2f40051c0 0x7fb2f400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.954+0000 7fb314e00640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2f40039b0 0x7fb2f4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.954+0000 7fb307e00640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f4004800 0x7fb2f4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.954+0000 7fb315800640 1 -- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2f40039b0 msgr2=0x7fb2f4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.954+0000 7fb315800640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2f40039b0 0x7fb2f4003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.954+0000 7fb315800640 1 -- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f4004800 msgr2=0x7fb2f4004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.954+0000 7fb315800640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f4004800 0x7fb2f4004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.954+0000 7fb315800640 1 -- 172.21.15.86:0/2383985234 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2f4003520 con 0x7fb2f40051c0 2024-03-06T07:32:45.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.955+0000 7fb315800640 1 -- 172.21.15.86:0/2383985234 reap_dead start 2024-03-06T07:32:45.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.955+0000 7fb315800640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2f40051c0 0x7fb2f400b9c0 secure :-1 s=READY pgs=1166 cs=0 l=1 rev1=1 crypto rx=0x7fb308001e20 tx=0x7fb2f4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9c13cf9640f670a5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:45.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.955+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb308001a70 con 0x7fb2f40051c0 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.955+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb308003b70 con 0x7fb2f40051c0 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.955+0000 7fb30fe00640 1 -- 172.21.15.86:0/2383985234 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3080050d0 con 0x7fb2f40051c0 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.956+0000 7fb317800640 1 -- 172.21.15.86:0/2383985234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2f40051c0 msgr2=0x7fb2f400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.956+0000 7fb317800640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2f40051c0 0x7fb2f400b9c0 secure :-1 s=READY pgs=1166 cs=0 l=1 rev1=1 crypto rx=0x7fb308001e20 tx=0x7fb2f4005830 comp rx=0 tx=0).stop 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.956+0000 7fb317800640 1 -- 172.21.15.86:0/2383985234 shutdown_connections 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.956+0000 7fb317800640 1 --2- 172.21.15.86:0/2383985234 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2f40051c0 0x7fb2f400b9c0 unknown :-1 s=CLOSED pgs=1166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.956+0000 7fb317800640 1 -- 172.21.15.86:0/2383985234 >> 172.21.15.86:0/2383985234 conn(0x7fb31009d390 msgr2=0x7fb3100d40c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.956+0000 7fb317800640 1 -- 172.21.15.86:0/2383985234 shutdown_connections 2024-03-06T07:32:45.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.956+0000 7fb317800640 1 -- 172.21.15.86:0/2383985234 wait complete. 2024-03-06T07:32:45.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.958+0000 7fb317800640 1 Processor -- start 2024-03-06T07:32:45.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.958+0000 7fb317800640 1 -- start start 2024-03-06T07:32:45.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb317800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f40051c0 0x7fb3101e8ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb317800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb3101605b0 0x7fb310159540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb317800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb3100b1e00 0x7fb3100b2280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb317800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3100d1270 con 0x7fb3100b1e00 2024-03-06T07:32:45.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb317800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3100d0f90 con 0x7fb2f40051c0 2024-03-06T07:32:45.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb317800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb3100b3660 con 0x7fb3101605b0 2024-03-06T07:32:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb3100b1e00 0x7fb3100b2280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb3100b1e00 0x7fb3100b2280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48938/0 (socket says 172.21.15.86:48938) 2024-03-06T07:32:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 -- 172.21.15.86:0/1096702400 learned_addr learned my addr 172.21.15.86:0/1096702400 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb307e00640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb3101605b0 0x7fb310159540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb314e00640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f40051c0 0x7fb3101e8ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 -- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb3101605b0 msgr2=0x7fb310159540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb3101605b0 0x7fb310159540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 -- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f40051c0 msgr2=0x7fb3101e8ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f40051c0 0x7fb3101e8ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb315800640 1 -- 172.21.15.86:0/1096702400 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb310160d50 con 0x7fb3100b1e00 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.959+0000 7fb307e00640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb3101605b0 0x7fb310159540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.960+0000 7fb315800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb3100b1e00 0x7fb3100b2280 secure :-1 s=READY pgs=1167 cs=0 l=1 rev1=1 crypto rx=0x7fb308001f40 tx=0x7fb3080014a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.960+0000 7fb30d600640 1 -- 172.21.15.86:0/1096702400 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb308006b50 con 0x7fb3100b1e00 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.960+0000 7fb30d600640 1 -- 172.21.15.86:0/1096702400 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb308006cf0 con 0x7fb3100b1e00 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.960+0000 7fb30d600640 1 -- 172.21.15.86:0/1096702400 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb308007250 con 0x7fb3100b1e00 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.960+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb310161030 con 0x7fb3100b1e00 2024-03-06T07:32:45.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.960+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb3101615f0 con 0x7fb3100b1e00 2024-03-06T07:32:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.961+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb310068070 con 0x7fb3100b1e00 2024-03-06T07:32:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.962+0000 7fb30d600640 1 -- 172.21.15.86:0/1096702400 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb3080073f0 con 0x7fb3100b1e00 2024-03-06T07:32:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.963+0000 7fb30d600640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb2ec06b500 0x7fb2ec06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.963+0000 7fb30d600640 1 -- 172.21.15.86:0/1096702400 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fb3080abc50 con 0x7fb3100b1e00 2024-03-06T07:32:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.963+0000 7fb314e00640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb2ec06b500 0x7fb2ec06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.964+0000 7fb314e00640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb2ec06b500 0x7fb2ec06d9c0 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fb300009a10 tx=0x7fb300008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:45.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:45.971+0000 7fb30d600640 1 -- 172.21.15.86:0/1096702400 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb308033030 con 0x7fb3100b1e00 2024-03-06T07:32:46.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.205+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb3100ce480 con 0x7fb3100b1e00 2024-03-06T07:32:46.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.206+0000 7fb30d600640 1 -- 172.21.15.86:0/1096702400 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7fb30807a440 con 0x7fb3100b1e00 2024-03-06T07:32:46.208 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:46.209 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:46.209 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:46.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb2ec06b500 msgr2=0x7fb2ec06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb2ec06b500 0x7fb2ec06d9c0 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fb300009a10 tx=0x7fb300008040 comp rx=0 tx=0).stop 2024-03-06T07:32:46.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb3100b1e00 msgr2=0x7fb3100b2280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb3100b1e00 0x7fb3100b2280 secure :-1 s=READY pgs=1167 cs=0 l=1 rev1=1 crypto rx=0x7fb308001f40 tx=0x7fb3080014a0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 shutdown_connections 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb2ec06b500 0x7fb2ec06d9c0 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb3100b1e00 0x7fb3100b2280 unknown :-1 s=CLOSED pgs=1167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb3101605b0 0x7fb310159540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 --2- 172.21.15.86:0/1096702400 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2f40051c0 0x7fb3101e8ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.208+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 >> 172.21.15.86:0/1096702400 conn(0x7fb31009d390 msgr2=0x7fb31006bab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.209+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 shutdown_connections 2024-03-06T07:32:46.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.209+0000 7fb317800640 1 -- 172.21.15.86:0/1096702400 wait complete. 2024-03-06T07:32:46.229 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:46.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.418+0000 7f8866c00640 1 Processor -- start 2024-03-06T07:32:46.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.418+0000 7f8866c00640 1 -- start start 2024-03-06T07:32:46.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.420+0000 7f8866c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f88600d2400 0x7f88600d2840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.420+0000 7f8866c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600b4790 0x7f88600b4c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.420+0000 7f8866c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f88600b39d0 con 0x7f886015e230 2024-03-06T07:32:46.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.420+0000 7f8866c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f88600b3780 con 0x7f88600d2400 2024-03-06T07:32:46.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.420+0000 7f8866c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f88600d2d80 con 0x7f88600b4790 2024-03-06T07:32:46.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.420+0000 7f8864e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f88600d2400 0x7f88600d2840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.420+0000 7f8866200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600b4790 0x7f88600b4c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.421+0000 7f8864e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f88600d2400 0x7f88600d2840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:40602/0 (socket says 172.21.15.86:40602) 2024-03-06T07:32:46.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.421+0000 7f8864e00640 1 -- 172.21.15.86:0/3946881450 learned_addr learned my addr 172.21.15.86:0/3946881450 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:46.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.421+0000 7f8864e00640 1 -- 172.21.15.86:0/3946881450 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600b4790 msgr2=0x7f88600b4c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:46.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.421+0000 7f8864e00640 1 --2- 172.21.15.86:0/3946881450 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600b4790 0x7f88600b4c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.421+0000 7f8864e00640 1 -- 172.21.15.86:0/3946881450 >> v1:172.21.15.194:6789/0 conn(0x7f886015e230 legacy=0x7f88600d0e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:46.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.421+0000 7f8864e00640 1 -- 172.21.15.86:0/3946881450 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88600b3cb0 con 0x7f88600d2400 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.423+0000 7f8864e00640 1 --2- 172.21.15.86:0/3946881450 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600d2400 0x7f88600d2840 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f884c009870 tx=0x7f884c02b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=515f04c024e6f990 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.423+0000 7f885fe00640 1 -- 172.21.15.86:0/3946881450 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f884c034070 con 0x7f88600d2400 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.423+0000 7f885fe00640 1 -- 172.21.15.86:0/3946881450 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f884c030840 con 0x7f88600d2400 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.423+0000 7f885fe00640 1 -- 172.21.15.86:0/3946881450 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f884c030c10 con 0x7f88600d2400 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 -- 172.21.15.86:0/3946881450 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600d2400 msgr2=0x7f88600d2840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 --2- 172.21.15.86:0/3946881450 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600d2400 0x7f88600d2840 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f884c009870 tx=0x7f884c02b980 comp rx=0 tx=0).stop 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 -- 172.21.15.86:0/3946881450 shutdown_connections 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 --2- 172.21.15.86:0/3946881450 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600b4790 0x7f88600b4c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 --2- 172.21.15.86:0/3946881450 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600d2400 0x7f88600d2840 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 -- 172.21.15.86:0/3946881450 >> 172.21.15.86:0/3946881450 conn(0x7f88600d3e20 msgr2=0x7f88600d4230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:46.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 -- 172.21.15.86:0/3946881450 shutdown_connections 2024-03-06T07:32:46.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.424+0000 7f8866c00640 1 -- 172.21.15.86:0/3946881450 wait complete. 2024-03-06T07:32:46.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.425+0000 7f8866c00640 1 Processor -- start 2024-03-06T07:32:46.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.426+0000 7f8866c00640 1 -- start start 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.426+0000 7f8866c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600b4790 0x7f886014c2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.426+0000 7f8866c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600d2400 0x7f88600c7810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f886015e230 0x7f88600b1780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f886015e610 con 0x7f886015e230 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88600b4270 con 0x7f88600b4790 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f88600d1570 con 0x7f88600d2400 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f886015e230 0x7f88600b1780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f886015e230 0x7f88600b1780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48946/0 (socket says 172.21.15.86:48946) 2024-03-06T07:32:46.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 -- 172.21.15.86:0/3062384623 learned_addr learned my addr 172.21.15.86:0/3062384623 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8865800640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600b4790 0x7f886014c2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8864e00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600d2400 0x7f88600c7810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 -- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600d2400 msgr2=0x7f88600c7810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600d2400 0x7f88600c7810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 -- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600b4790 msgr2=0x7f886014c2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600b4790 0x7f886014c2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8866200640 1 -- 172.21.15.86:0/3062384623 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f886014c810 con 0x7f886015e230 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8865800640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600b4790 0x7f886014c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.427+0000 7f8864e00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600d2400 0x7f88600c7810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.428+0000 7f8866200640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f886015e230 0x7f88600b1780 secure :-1 s=READY pgs=1168 cs=0 l=1 rev1=1 crypto rx=0x7f885800efc0 tx=0x7f885800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.428+0000 7f885d600640 1 -- 172.21.15.86:0/3062384623 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8858018020 con 0x7f886015e230 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.428+0000 7f885d600640 1 -- 172.21.15.86:0/3062384623 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8858014480 con 0x7f886015e230 2024-03-06T07:32:46.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.428+0000 7f885d600640 1 -- 172.21.15.86:0/3062384623 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8858014870 con 0x7f886015e230 2024-03-06T07:32:46.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.428+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f886015e830 con 0x7f886015e230 2024-03-06T07:32:46.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.428+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f886015edf0 con 0x7f886015e230 2024-03-06T07:32:46.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.429+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f886004f330 con 0x7f886015e230 2024-03-06T07:32:46.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.430+0000 7f885d600640 1 -- 172.21.15.86:0/3062384623 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8858005d10 con 0x7f886015e230 2024-03-06T07:32:46.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.431+0000 7f885d600640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f884006b4d0 0x7f884006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.431+0000 7f885d600640 1 -- 172.21.15.86:0/3062384623 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f885808b1d0 con 0x7f886015e230 2024-03-06T07:32:46.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.431+0000 7f8865800640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f884006b4d0 0x7f884006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.432+0000 7f8865800640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f884006b4d0 0x7f884006d990 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7f885400a830 tx=0x7f8854008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:46.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.435+0000 7f885d600640 1 -- 172.21.15.86:0/3062384623 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8858022030 con 0x7f886015e230 2024-03-06T07:32:46.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.629+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f88600cfa10 con 0x7f886015e230 2024-03-06T07:32:46.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.631+0000 7f885d600640 1 -- 172.21.15.86:0/3062384623 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v98) v1 ==== 74+0+16879 (secure 0 0 0) 0x7f886014c810 con 0x7f886015e230 2024-03-06T07:32:46.634 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:46.634 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":98,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:20.051194+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:32:01.521762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-06T07:32:02.759692+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"94","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6835/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.86:6834/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.194:6834/2712222804":"2024-03-07T07:32:14.424511+0000","172.21.15.194:6832/2712222804":"2024-03-07T07:32:14.424511+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:46.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.633+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f884006b4d0 msgr2=0x7f884006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.633+0000 7f8866c00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f884006b4d0 0x7f884006d990 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7f885400a830 tx=0x7f8854008040 comp rx=0 tx=0).stop 2024-03-06T07:32:46.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.633+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f886015e230 msgr2=0x7f88600b1780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.633+0000 7f8866c00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f886015e230 0x7f88600b1780 secure :-1 s=READY pgs=1168 cs=0 l=1 rev1=1 crypto rx=0x7f885800efc0 tx=0x7f885800c490 comp rx=0 tx=0).stop 2024-03-06T07:32:46.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.633+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 shutdown_connections 2024-03-06T07:32:46.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.634+0000 7f8866c00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f884006b4d0 0x7f884006d990 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.634+0000 7f8866c00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f886015e230 0x7f88600b1780 unknown :-1 s=CLOSED pgs=1168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.634+0000 7f8866c00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88600d2400 0x7f88600c7810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.634+0000 7f8866c00640 1 --2- 172.21.15.86:0/3062384623 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f88600b4790 0x7f886014c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.634+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 >> 172.21.15.86:0/3062384623 conn(0x7f88600d3e20 msgr2=0x7f88600d1c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:46.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.634+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 shutdown_connections 2024-03-06T07:32:46.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.634+0000 7f8866c00640 1 -- 172.21.15.86:0/3062384623 wait complete. 2024-03-06T07:32:46.654 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 14, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:32:01.521762+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '93', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 15, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:32:02.759692+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '94', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:32:46.655 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:32:46.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.908+0000 7f90fbe00640 1 Processor -- start 2024-03-06T07:32:46.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.908+0000 7f90fbe00640 1 -- start start 2024-03-06T07:32:46.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.910+0000 7f90fbe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f90fc0d2400 0x7f90fc0d2840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.910+0000 7f90fbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 0x7f90fc0b4c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.910+0000 7f90fbe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f90fc0b39d0 con 0x7f90fc15e230 2024-03-06T07:32:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.910+0000 7f90fbe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f90fc0b3780 con 0x7f90fc0d2400 2024-03-06T07:32:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.910+0000 7f90fbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f90fc0d2d80 con 0x7f90fc0b4790 2024-03-06T07:32:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.910+0000 7f90faa00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f90fc15e230 0x7f90fc0d0e60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44494/0 (socket says 172.21.15.86:44494) 2024-03-06T07:32:46.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.910+0000 7f90faa00640 1 -- 172.21.15.86:0/2838306684 learned_addr learned my addr 172.21.15.86:0/2838306684 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4014140778 0 0) 0x7f90fc0b39d0 con 0x7f90fc15e230 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90f2000640 1 --2- 172.21.15.86:0/2838306684 >> v2:172.21.15.194:3300/0 conn(0x7f90fc0d2400 0x7f90fc0d2840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f90e0003520 con 0x7f90fc15e230 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fb400640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 0x7f90fc0b4c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3092237242 0 0) 0x7f90e0003520 con 0x7f90fc15e230 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 msgr2=0x7f90fc0b4c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 0x7f90fc0b4c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 >> v2:172.21.15.194:3300/0 conn(0x7f90fc0d2400 msgr2=0x7f90fc0d2840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 --2- 172.21.15.86:0/2838306684 >> v2:172.21.15.194:3300/0 conn(0x7f90fc0d2400 0x7f90fc0d2840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90fc0b3cb0 con 0x7f90fc15e230 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f90ec001670 con 0x7f90fc15e230 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 >> v1:172.21.15.86:6789/0 conn(0x7f90fc15e230 legacy=0x7f90fc0d0e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 0x7f90e00039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90e0003f80 0x7f90e0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0004d40 0x7f90e00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90fc0b3780 con 0x7f90e0003f80 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90fc0d2d80 con 0x7f90e0004d40 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.911+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f90e000ba40 con 0x7f90fc0b4790 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90f2000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90e0003f80 0x7f90e0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90faa00640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 0x7f90e00039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90fb400640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0004d40 0x7f90e00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90f2000640 1 -- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 msgr2=0x7f90e00039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90f2000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 0x7f90e00039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90f2000640 1 -- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0004d40 msgr2=0x7f90e00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:46.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90f2000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0004d40 0x7f90e00051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90f2000640 1 -- 172.21.15.86:0/2838306684 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90e0003520 con 0x7f90e0003f80 2024-03-06T07:32:46.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.912+0000 7f90faa00640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b4790 0x7f90e00039b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:46.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.913+0000 7f90f2000640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90e0003f80 0x7f90e0004800 secure :-1 s=READY pgs=1170 cs=0 l=1 rev1=1 crypto rx=0x7f90e4009ac0 tx=0x7f90e0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d0e63b4e330296e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:46.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.913+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90e4039070 con 0x7f90e0003f80 2024-03-06T07:32:46.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.913+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f90e402c710 con 0x7f90e0003f80 2024-03-06T07:32:46.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.913+0000 7f90fa000640 1 -- 172.21.15.86:0/2838306684 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90e4009690 con 0x7f90e0003f80 2024-03-06T07:32:46.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.914+0000 7f90fbe00640 1 -- 172.21.15.86:0/2838306684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90e0003f80 msgr2=0x7f90e0004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.914+0000 7f90fbe00640 1 --2- 172.21.15.86:0/2838306684 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90e0003f80 0x7f90e0004800 secure :-1 s=READY pgs=1170 cs=0 l=1 rev1=1 crypto rx=0x7f90e4009ac0 tx=0x7f90e0005830 comp rx=0 tx=0).stop 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.914+0000 7f90fb400640 1 -- 172.21.15.86:0/2838306684 reap_dead start 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.914+0000 7f90fbe00640 1 -- 172.21.15.86:0/2838306684 shutdown_connections 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.914+0000 7f90fbe00640 1 -- 172.21.15.86:0/2838306684 >> 172.21.15.86:0/2838306684 conn(0x7f90fc0d3e20 msgr2=0x7f90fc0d4230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.914+0000 7f90fbe00640 1 -- 172.21.15.86:0/2838306684 shutdown_connections 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.914+0000 7f90fbe00640 1 -- 172.21.15.86:0/2838306684 wait complete. 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.916+0000 7f90fbe00640 1 Processor -- start 2024-03-06T07:32:46.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.916+0000 7f90fbe00640 1 -- start start 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fbe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0003f80 0x7f90fc0b9330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b27e0 0x7f90fc15a740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90fc0b13e0 0x7f90fc0b1860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fbe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90fc15e610 con 0x7f90fc0b13e0 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fbe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90fc0b4270 con 0x7f90e0003f80 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f90fc0d1570 con 0x7f90fc0b27e0 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90fc0b13e0 0x7f90fc0b1860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90fc0b13e0 0x7f90fc0b1860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48970/0 (socket says 172.21.15.86:48970) 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.917+0000 7f90fb400640 1 -- 172.21.15.86:0/2545456177 learned_addr learned my addr 172.21.15.86:0/2545456177 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90faa00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0003f80 0x7f90fc0b9330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fb400640 1 -- 172.21.15.86:0/2545456177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b27e0 msgr2=0x7f90fc15a740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fb400640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b27e0 0x7f90fc15a740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fb400640 1 -- 172.21.15.86:0/2545456177 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0003f80 msgr2=0x7f90fc0b9330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fb400640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0003f80 0x7f90fc0b9330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fb400640 1 -- 172.21.15.86:0/2545456177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90fc160c90 con 0x7f90fc0b13e0 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fb400640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90fc0b13e0 0x7f90fc0b1860 secure :-1 s=READY pgs=1171 cs=0 l=1 rev1=1 crypto rx=0x7f90fc0b6a10 tx=0x7f90f400f4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90f3e00640 1 -- 172.21.15.86:0/2545456177 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90f401c070 con 0x7f90fc0b13e0 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90f3e00640 1 -- 172.21.15.86:0/2545456177 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f90f4017400 con 0x7f90fc0b13e0 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90fc160f70 con 0x7f90fc0b13e0 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90f3e00640 1 -- 172.21.15.86:0/2545456177 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90f40177f0 con 0x7f90fc0b13e0 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.918+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90fc161530 con 0x7f90fc0b13e0 2024-03-06T07:32:46.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.920+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f90fc04f330 con 0x7f90fc0b13e0 2024-03-06T07:32:46.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.920+0000 7f90f3e00640 1 -- 172.21.15.86:0/2545456177 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f90f401f070 con 0x7f90fc0b13e0 2024-03-06T07:32:46.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.922+0000 7f90f3e00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f90d806b4d0 0x7f90d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:46.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.922+0000 7f90f3e00640 1 -- 172.21.15.86:0/2545456177 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f90f408a1c0 con 0x7f90fc0b13e0 2024-03-06T07:32:46.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.922+0000 7f90faa00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f90d806b4d0 0x7f90d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:46.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.923+0000 7f90faa00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f90d806b4d0 0x7f90d806d990 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f90ec001eb0 tx=0x7f90ec009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:46.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:46.926+0000 7f90f3e00640 1 -- 172.21.15.86:0/2545456177 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f90f4012070 con 0x7f90fc0b13e0 2024-03-06T07:32:47.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.123+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f90fc161700 con 0x7f90fc0b13e0 2024-03-06T07:32:47.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.124+0000 7f90f3e00640 1 -- 172.21.15.86:0/2545456177 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v98) v1 ==== 93+0+10 (secure 0 0 0) 0x7f90f40589d0 con 0x7f90fc0b13e0 2024-03-06T07:32:47.127 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f90d806b4d0 msgr2=0x7f90d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f90d806b4d0 0x7f90d806d990 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f90ec001eb0 tx=0x7f90ec009040 comp rx=0 tx=0).stop 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90fc0b13e0 msgr2=0x7f90fc0b1860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90fc0b13e0 0x7f90fc0b1860 secure :-1 s=READY pgs=1171 cs=0 l=1 rev1=1 crypto rx=0x7f90fc0b6a10 tx=0x7f90f400f4a0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 shutdown_connections 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f90d806b4d0 0x7f90d806d990 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f90fc0b13e0 0x7f90fc0b1860 unknown :-1 s=CLOSED pgs=1171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f90fc0b27e0 0x7f90fc15a740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 --2- 172.21.15.86:0/2545456177 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f90e0003f80 0x7f90fc0b9330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 >> 172.21.15.86:0/2545456177 conn(0x7f90fc0d3e20 msgr2=0x7f90fc06bfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:47.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 shutdown_connections 2024-03-06T07:32:47.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.127+0000 7f90fbe00640 1 -- 172.21.15.86:0/2545456177 wait complete. 2024-03-06T07:32:47.155 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:32:47.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.355+0000 7f6ddca00640 1 Processor -- start 2024-03-06T07:32:47.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.355+0000 7f6ddca00640 1 -- start start 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6ddca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6dd815be90 0x7f6dd80b49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6ddca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd80acd60 0x7f6dd80d0070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6ddca00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6dd80d05b0 0x7f6dd806c9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6ddca00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6dd80b4f30 con 0x7f6dd815be90 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6ddca00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6dd80af240 con 0x7f6dd80d05b0 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6ddca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6dd80b5160 con 0x7f6dd80acd60 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6dd815be90 0x7f6dd80b49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6dd815be90 0x7f6dd80b49f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48976/0 (socket says 172.21.15.86:48976) 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 -- 172.21.15.86:0/3035453514 learned_addr learned my addr 172.21.15.86:0/3035453514 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 -- 172.21.15.86:0/3035453514 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd80acd60 msgr2=0x7f6dd80d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:47.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 --2- 172.21.15.86:0/3035453514 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd80acd60 0x7f6dd80d0070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 -- 172.21.15.86:0/3035453514 >> v2:172.21.15.194:3300/0 conn(0x7f6dd80d05b0 msgr2=0x7f6dd806c9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:47.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 --2- 172.21.15.86:0/3035453514 >> v2:172.21.15.194:3300/0 conn(0x7f6dd80d05b0 0x7f6dd806c9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.357+0000 7f6dd5e00640 1 -- 172.21.15.86:0/3035453514 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6dd80ae910 con 0x7f6dd815be90 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.358+0000 7f6dd5e00640 1 --2- 172.21.15.86:0/3035453514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd815be90 0x7f6dd80b49f0 secure :-1 s=READY pgs=1172 cs=0 l=1 rev1=1 crypto rx=0x7f6dd80af880 tx=0x7f6dc802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e514902bbdbbac30 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6dd5400640 1 -- 172.21.15.86:0/3035453514 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6dc802ca80 con 0x7f6dd815be90 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6dd5400640 1 -- 172.21.15.86:0/3035453514 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6dc802cc20 con 0x7f6dd815be90 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6dd5400640 1 -- 172.21.15.86:0/3035453514 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6dc80343f0 con 0x7f6dd815be90 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6ddca00640 1 -- 172.21.15.86:0/3035453514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd815be90 msgr2=0x7f6dd80b49f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6ddca00640 1 --2- 172.21.15.86:0/3035453514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd815be90 0x7f6dd80b49f0 secure :-1 s=READY pgs=1172 cs=0 l=1 rev1=1 crypto rx=0x7f6dd80af880 tx=0x7f6dc802bee0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6ddca00640 1 -- 172.21.15.86:0/3035453514 shutdown_connections 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6ddca00640 1 --2- 172.21.15.86:0/3035453514 >> v2:172.21.15.194:3300/0 conn(0x7f6dd80d05b0 0x7f6dd806c9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6ddca00640 1 --2- 172.21.15.86:0/3035453514 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd80acd60 0x7f6dd80d0070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6ddca00640 1 --2- 172.21.15.86:0/3035453514 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd815be90 0x7f6dd80b49f0 unknown :-1 s=CLOSED pgs=1172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.359+0000 7f6ddca00640 1 -- 172.21.15.86:0/3035453514 >> 172.21.15.86:0/3035453514 conn(0x7f6dd806b360 msgr2=0x7f6dd806b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:47.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.360+0000 7f6ddca00640 1 -- 172.21.15.86:0/3035453514 shutdown_connections 2024-03-06T07:32:47.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.360+0000 7f6ddca00640 1 -- 172.21.15.86:0/3035453514 wait complete. 2024-03-06T07:32:47.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.361+0000 7f6ddca00640 1 Processor -- start 2024-03-06T07:32:47.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.361+0000 7f6ddca00640 1 -- start start 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6ddca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd80acd60 0x7f6dd81492b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6ddca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6dd80d05b0 0x7f6dd8149ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6ddca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd815be90 0x7f6dd8149ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6ddca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6dd80b2c80 con 0x7f6dd80acd60 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6ddca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6dd80aeed0 con 0x7f6dd80d05b0 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6ddca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6dd80b2e00 con 0x7f6dd815be90 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6dd5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd80acd60 0x7f6dd81492b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.362+0000 7f6dd5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd80acd60 0x7f6dd81492b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48992/0 (socket says 172.21.15.86:48992) 2024-03-06T07:32:47.365 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dd5e00640 1 -- 172.21.15.86:0/1976509324 learned_addr learned my addr 172.21.15.86:0/1976509324 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dd5e00640 1 -- 172.21.15.86:0/1976509324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd815be90 msgr2=0x7f6dd8149ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dd5e00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd815be90 0x7f6dd8149ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dd5e00640 1 -- 172.21.15.86:0/1976509324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6dd80d05b0 msgr2=0x7f6dd8149ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dcd400640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6dd80d05b0 0x7f6dd8149ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dd5e00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6dd80d05b0 0x7f6dd8149ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dd5e00640 1 -- 172.21.15.86:0/1976509324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6dd80b0920 con 0x7f6dd80acd60 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dd5e00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd80acd60 0x7f6dd81492b0 secure :-1 s=READY pgs=1173 cs=0 l=1 rev1=1 crypto rx=0x7f6dc80093e0 tx=0x7f6dc8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dcea00640 1 -- 172.21.15.86:0/1976509324 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6dc8043020 con 0x7f6dd80acd60 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6dcea00640 1 -- 172.21.15.86:0/1976509324 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6dc803e450 con 0x7f6dd80acd60 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.363+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6dd80b0c00 con 0x7f6dd80acd60 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.364+0000 7f6dcea00640 1 -- 172.21.15.86:0/1976509324 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6dc803e840 con 0x7f6dd80acd60 2024-03-06T07:32:47.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.364+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6dd80b11c0 con 0x7f6dd80acd60 2024-03-06T07:32:47.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.365+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6dd8068070 con 0x7f6dd80acd60 2024-03-06T07:32:47.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.366+0000 7f6dcea00640 1 -- 172.21.15.86:0/1976509324 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6dc8046020 con 0x7f6dd80acd60 2024-03-06T07:32:47.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.366+0000 7f6dcea00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6db406b500 0x7f6db406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.367+0000 7f6dcea00640 1 -- 172.21.15.86:0/1976509324 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f6dc80aca30 con 0x7f6dd80acd60 2024-03-06T07:32:47.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.367+0000 7f6dcd400640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6db406b500 0x7f6db406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.367+0000 7f6dcd400640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6db406b500 0x7f6db406d9c0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f6dd80c9d30 tx=0x7f6dc0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:47.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.371+0000 7f6dcea00640 1 -- 172.21.15.86:0/1976509324 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6dc804d070 con 0x7f6dd80acd60 2024-03-06T07:32:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.672+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f6dd80718e0 con 0x7f6dd80acd60 2024-03-06T07:32:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.673+0000 7f6dcea00640 1 -- 172.21.15.86:0/1976509324 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v98) v1 ==== 104+0+11 (secure 0 0 0) 0x7f6dc807b8b0 con 0x7f6dd80acd60 2024-03-06T07:32:47.676 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6db406b500 msgr2=0x7f6db406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6db406b500 0x7f6db406d9c0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f6dd80c9d30 tx=0x7f6dc0008040 comp rx=0 tx=0).stop 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd80acd60 msgr2=0x7f6dd81492b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd80acd60 0x7f6dd81492b0 secure :-1 s=READY pgs=1173 cs=0 l=1 rev1=1 crypto rx=0x7f6dc80093e0 tx=0x7f6dc8004a40 comp rx=0 tx=0).stop 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 shutdown_connections 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6db406b500 0x7f6db406d9c0 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6dd815be90 0x7f6dd8149ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6dd80d05b0 0x7f6dd8149ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 --2- 172.21.15.86:0/1976509324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6dd80acd60 0x7f6dd81492b0 unknown :-1 s=CLOSED pgs=1173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 >> 172.21.15.86:0/1976509324 conn(0x7f6dd806b360 msgr2=0x7f6dd8070b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.676+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 shutdown_connections 2024-03-06T07:32:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.677+0000 7f6ddca00640 1 -- 172.21.15.86:0/1976509324 wait complete. 2024-03-06T07:32:47.703 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:32:47.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.893+0000 7fb750400640 1 Processor -- start 2024-03-06T07:32:47.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.893+0000 7fb750400640 1 -- start start 2024-03-06T07:32:47.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.894+0000 7fb750400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb74815e2d0 0x7fb7480d1ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.894+0000 7fb750400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb74815e6b0 0x7fb7480d26a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb750400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb748161fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb750400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb7480cebd0 con 0x7fb74815e6b0 2024-03-06T07:32:47.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb750400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb7480d23e0 con 0x7fb74815e2d0 2024-03-06T07:32:47.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb750400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb7480d06f0 con 0x7fb7480d2be0 2024-03-06T07:32:47.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb74815e6b0 0x7fb7480d26a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb74815e6b0 0x7fb7480d26a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49000/0 (socket says 172.21.15.86:49000) 2024-03-06T07:32:47.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 -- 172.21.15.86:0/1358802810 learned_addr learned my addr 172.21.15.86:0/1358802810 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74e400640 1 --2- 172.21.15.86:0/1358802810 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb748161fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74da00640 1 --2- 172.21.15.86:0/1358802810 >> v2:172.21.15.194:3300/0 conn(0x7fb74815e2d0 0x7fb7480d1ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 -- 172.21.15.86:0/1358802810 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 msgr2=0x7fb748161fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 --2- 172.21.15.86:0/1358802810 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb748161fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 -- 172.21.15.86:0/1358802810 >> v2:172.21.15.194:3300/0 conn(0x7fb74815e2d0 msgr2=0x7fb7480d1ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 --2- 172.21.15.86:0/1358802810 >> v2:172.21.15.194:3300/0 conn(0x7fb74815e2d0 0x7fb7480d1ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74d000640 1 -- 172.21.15.86:0/1358802810 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7480d09d0 con 0x7fb74815e6b0 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74e400640 1 --2- 172.21.15.86:0/1358802810 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb748161fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.895+0000 7fb74da00640 1 --2- 172.21.15.86:0/1358802810 >> v2:172.21.15.194:3300/0 conn(0x7fb74815e2d0 0x7fb7480d1ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:47.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.897+0000 7fb74d000640 1 --2- 172.21.15.86:0/1358802810 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb74815e6b0 0x7fb7480d26a0 secure :-1 s=READY pgs=1174 cs=0 l=1 rev1=1 crypto rx=0x7fb74808e490 tx=0x7fb73402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0468617cb047111 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.897+0000 7fb747e00640 1 -- 172.21.15.86:0/1358802810 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb734035020 con 0x7fb74815e6b0 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.897+0000 7fb747e00640 1 -- 172.21.15.86:0/1358802810 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb734031870 con 0x7fb74815e6b0 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.897+0000 7fb747e00640 1 -- 172.21.15.86:0/1358802810 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb734031c40 con 0x7fb74815e6b0 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.897+0000 7fb750400640 1 -- 172.21.15.86:0/1358802810 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb74815e6b0 msgr2=0x7fb7480d26a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.897+0000 7fb750400640 1 --2- 172.21.15.86:0/1358802810 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb74815e6b0 0x7fb7480d26a0 secure :-1 s=READY pgs=1174 cs=0 l=1 rev1=1 crypto rx=0x7fb74808e490 tx=0x7fb73402b910 comp rx=0 tx=0).stop 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.898+0000 7fb750400640 1 -- 172.21.15.86:0/1358802810 shutdown_connections 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.898+0000 7fb750400640 1 --2- 172.21.15.86:0/1358802810 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb748161fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.898+0000 7fb750400640 1 --2- 172.21.15.86:0/1358802810 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb74815e6b0 0x7fb7480d26a0 unknown :-1 s=CLOSED pgs=1174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.898+0000 7fb750400640 1 --2- 172.21.15.86:0/1358802810 >> v2:172.21.15.194:3300/0 conn(0x7fb74815e2d0 0x7fb7480d1ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.898+0000 7fb750400640 1 -- 172.21.15.86:0/1358802810 >> 172.21.15.86:0/1358802810 conn(0x7fb74809d390 msgr2=0x7fb7480d40c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:47.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.898+0000 7fb750400640 1 -- 172.21.15.86:0/1358802810 shutdown_connections 2024-03-06T07:32:47.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.898+0000 7fb750400640 1 -- 172.21.15.86:0/1358802810 wait complete. 2024-03-06T07:32:47.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.900+0000 7fb750400640 1 Processor -- start 2024-03-06T07:32:47.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.900+0000 7fb750400640 1 -- start start 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb750400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb7481552d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb750400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb74815e2d0 0x7fb74815fe10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb750400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb7481616b0 0x7fb7480b9050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb750400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7480d1270 con 0x7fb7481616b0 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb750400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7480d0f90 con 0x7fb74815e2d0 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb750400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb7480b3660 con 0x7fb7480d2be0 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb74e400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb7481616b0 0x7fb7480b9050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb74e400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb7481616b0 0x7fb7480b9050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49016/0 (socket says 172.21.15.86:49016) 2024-03-06T07:32:47.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.901+0000 7fb74e400640 1 -- 172.21.15.86:0/161038182 learned_addr learned my addr 172.21.15.86:0/161038182 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74d000640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb74815e2d0 0x7fb74815fe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74da00640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb7481552d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74e400640 1 -- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 msgr2=0x7fb7481552d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74e400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb7481552d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74e400640 1 -- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb74815e2d0 msgr2=0x7fb74815fe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74e400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb74815e2d0 0x7fb74815fe10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74e400640 1 -- 172.21.15.86:0/161038182 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb748160760 con 0x7fb7481616b0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb74e400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb7481616b0 0x7fb7480b9050 secure :-1 s=READY pgs=1175 cs=0 l=1 rev1=1 crypto rx=0x7fb74000cd10 tx=0x7fb74000a650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb745600640 1 -- 172.21.15.86:0/161038182 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb74000f680 con 0x7fb7481616b0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb745600640 1 -- 172.21.15.86:0/161038182 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb74000f840 con 0x7fb7481616b0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb748160a40 con 0x7fb7481616b0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.902+0000 7fb745600640 1 -- 172.21.15.86:0/161038182 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb74000fc30 con 0x7fb7481616b0 2024-03-06T07:32:47.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.903+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7480cb190 con 0x7fb7481616b0 2024-03-06T07:32:47.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.904+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb748068070 con 0x7fb7481616b0 2024-03-06T07:32:47.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.906+0000 7fb745600640 1 -- 172.21.15.86:0/161038182 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb7400213f0 con 0x7fb7481616b0 2024-03-06T07:32:47.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.907+0000 7fb745600640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb72806b500 0x7fb72806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:47.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.907+0000 7fb745600640 1 -- 172.21.15.86:0/161038182 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fb74008afb0 con 0x7fb7481616b0 2024-03-06T07:32:47.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.907+0000 7fb74da00640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb72806b500 0x7fb72806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:47.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.908+0000 7fb74da00640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb72806b500 0x7fb72806d9c0 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fb73c00b840 tx=0x7fb73c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:47.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:47.912+0000 7fb745600640 1 -- 172.21.15.86:0/161038182 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb740014030 con 0x7fb7481616b0 2024-03-06T07:32:48.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.110+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fb7480cfaa0 con 0x7fb7481616b0 2024-03-06T07:32:48.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.111+0000 7fb745600640 1 -- 172.21.15.86:0/161038182 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v98) v1 ==== 100+0+11 (secure 0 0 0) 0x7fb740059d70 con 0x7fb7481616b0 2024-03-06T07:32:48.113 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:32:48.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb72806b500 msgr2=0x7fb72806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb72806b500 0x7fb72806d9c0 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fb73c00b840 tx=0x7fb73c008040 comp rx=0 tx=0).stop 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb7481616b0 msgr2=0x7fb7480b9050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb7481616b0 0x7fb7480b9050 secure :-1 s=READY pgs=1175 cs=0 l=1 rev1=1 crypto rx=0x7fb74000cd10 tx=0x7fb74000a650 comp rx=0 tx=0).stop 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 shutdown_connections 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb72806b500 0x7fb72806d9c0 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb7481616b0 0x7fb7480b9050 unknown :-1 s=CLOSED pgs=1175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb74815e2d0 0x7fb74815fe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 --2- 172.21.15.86:0/161038182 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb7480d2be0 0x7fb7481552d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 >> 172.21.15.86:0/161038182 conn(0x7fb74809d390 msgr2=0x7fb74806bba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 shutdown_connections 2024-03-06T07:32:48.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.114+0000 7fb750400640 1 -- 172.21.15.86:0/161038182 wait complete. 2024-03-06T07:32:48.135 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:48.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.333+0000 7fde6d600640 1 Processor -- start 2024-03-06T07:32:48.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.333+0000 7fde6d600640 1 -- start start 2024-03-06T07:32:48.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.335+0000 7fde6d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 0x7fde680d0690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.335+0000 7fde6d600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fde680b5020 0x7fde680d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.335+0000 7fde6d600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde680d21c0 con 0x7fde680b44f0 2024-03-06T07:32:48.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.335+0000 7fde6d600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fde680d0000 con 0x7fde680b5020 2024-03-06T07:32:48.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.335+0000 7fde6d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde680b4e40 con 0x7fde680d2cc0 2024-03-06T07:32:48.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde66a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fde680b44f0 0x7fde680d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44498/0 (socket says 172.21.15.86:44498) 2024-03-06T07:32:48.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde66a00640 1 -- 172.21.15.86:0/2770130755 learned_addr learned my addr 172.21.15.86:0/2770130755 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde67400640 1 --2- 172.21.15.86:0/2770130755 >> v2:172.21.15.194:3300/0 conn(0x7fde680b5020 0x7fde680d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde5e000640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 0x7fde680d0690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3833103926 0 0) 0x7fde680d21c0 con 0x7fde680b44f0 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fde4c003520 con 0x7fde680b44f0 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1788987817 0 0) 0x7fde4c003520 con 0x7fde680b44f0 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.336+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 msgr2=0x7fde680d0690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 0x7fde680d0690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 >> v2:172.21.15.194:3300/0 conn(0x7fde680b5020 msgr2=0x7fde680d0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:48.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 --2- 172.21.15.86:0/2770130755 >> v2:172.21.15.194:3300/0 conn(0x7fde680b5020 0x7fde680d0bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde680b48d0 con 0x7fde680b44f0 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fde580019f0 con 0x7fde680b44f0 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 >> v1:172.21.15.86:6789/0 conn(0x7fde680b44f0 legacy=0x7fde680d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde4c0039e0 0x7fde4c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde4c004800 0x7fde4c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 0x7fde4c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde680d0000 con 0x7fde4c0039e0 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde680b4e40 con 0x7fde4c004800 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde4c00b9f0 con 0x7fde680d2cc0 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde66a00640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde4c0039e0 0x7fde4c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.337+0000 7fde5e000640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde4c004800 0x7fde4c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.338+0000 7fde67400640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 0x7fde4c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.338+0000 7fde66a00640 1 -- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 msgr2=0x7fde4c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.338+0000 7fde66a00640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 0x7fde4c0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.338+0000 7fde66a00640 1 -- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde4c004800 msgr2=0x7fde4c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:48.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.338+0000 7fde66a00640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde4c004800 0x7fde4c004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.338+0000 7fde66a00640 1 -- 172.21.15.86:0/2770130755 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde4c003520 con 0x7fde4c0039e0 2024-03-06T07:32:48.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.338+0000 7fde67400640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde680d2cc0 0x7fde4c0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:48.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.339+0000 7fde66a00640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde4c0039e0 0x7fde4c003e00 secure :-1 s=READY pgs=1177 cs=0 l=1 rev1=1 crypto rx=0x7fde5800a860 tx=0x7fde4c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a6a1815ecfe578ed server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:48.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.339+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde5800a480 con 0x7fde4c0039e0 2024-03-06T07:32:48.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.339+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde58034630 con 0x7fde4c0039e0 2024-03-06T07:32:48.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.339+0000 7fde66000640 1 -- 172.21.15.86:0/2770130755 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde58034a60 con 0x7fde4c0039e0 2024-03-06T07:32:48.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.340+0000 7fde6d600640 1 -- 172.21.15.86:0/2770130755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde4c0039e0 msgr2=0x7fde4c003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.340+0000 7fde6d600640 1 --2- 172.21.15.86:0/2770130755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde4c0039e0 0x7fde4c003e00 secure :-1 s=READY pgs=1177 cs=0 l=1 rev1=1 crypto rx=0x7fde5800a860 tx=0x7fde4c005830 comp rx=0 tx=0).stop 2024-03-06T07:32:48.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.340+0000 7fde67400640 1 -- 172.21.15.86:0/2770130755 reap_dead start 2024-03-06T07:32:48.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.340+0000 7fde6d600640 1 -- 172.21.15.86:0/2770130755 shutdown_connections 2024-03-06T07:32:48.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.340+0000 7fde6d600640 1 -- 172.21.15.86:0/2770130755 >> 172.21.15.86:0/2770130755 conn(0x7fde680d3fd0 msgr2=0x7fde680d43e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:48.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.340+0000 7fde6d600640 1 -- 172.21.15.86:0/2770130755 shutdown_connections 2024-03-06T07:32:48.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.340+0000 7fde6d600640 1 -- 172.21.15.86:0/2770130755 wait complete. 2024-03-06T07:32:48.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.342+0000 7fde6d600640 1 Processor -- start 2024-03-06T07:32:48.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.342+0000 7fde6d600640 1 -- start start 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde6d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde4c0039e0 0x7fde680cb310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde6d600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde6815d1e0 0x7fde680c9870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde6d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6815d660 0x7fde680c9db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde6d600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde680cfae0 con 0x7fde6815d660 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde6d600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde680cf800 con 0x7fde6815d1e0 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde6d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde680b4ad0 con 0x7fde4c0039e0 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6815d660 0x7fde680c9db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6815d660 0x7fde680c9db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49044/0 (socket says 172.21.15.86:49044) 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 -- 172.21.15.86:0/258801243 learned_addr learned my addr 172.21.15.86:0/258801243 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde66a00640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde4c0039e0 0x7fde680cb310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 -- 172.21.15.86:0/258801243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde4c0039e0 msgr2=0x7fde680cb310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde4c0039e0 0x7fde680cb310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 -- 172.21.15.86:0/258801243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde6815d1e0 msgr2=0x7fde680c9870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde6815d1e0 0x7fde680c9870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.343+0000 7fde67400640 1 -- 172.21.15.86:0/258801243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde6815dbf0 con 0x7fde6815d660 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.344+0000 7fde67400640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6815d660 0x7fde680c9db0 secure :-1 s=READY pgs=1178 cs=0 l=1 rev1=1 crypto rx=0x7fde680cb020 tx=0x7fde60010c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.344+0000 7fde5f400640 1 -- 172.21.15.86:0/258801243 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde6000f040 con 0x7fde6815d660 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.344+0000 7fde5f400640 1 -- 172.21.15.86:0/258801243 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde6001c070 con 0x7fde6815d660 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.344+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde6815ded0 con 0x7fde6815d660 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.344+0000 7fde5f400640 1 -- 172.21.15.86:0/258801243 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde6000eac0 con 0x7fde6815d660 2024-03-06T07:32:48.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.344+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde680cb850 con 0x7fde6815d660 2024-03-06T07:32:48.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.345+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde68068070 con 0x7fde6815d660 2024-03-06T07:32:48.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.347+0000 7fde5f400640 1 -- 172.21.15.86:0/258801243 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fde6000ec60 con 0x7fde6815d660 2024-03-06T07:32:48.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.348+0000 7fde5f400640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b500 0x7fde4406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.348+0000 7fde5f400640 1 -- 172.21.15.86:0/258801243 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fde60089eb0 con 0x7fde6815d660 2024-03-06T07:32:48.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.348+0000 7fde66a00640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b500 0x7fde4406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.349+0000 7fde66a00640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b500 0x7fde4406d9c0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fde58034000 tx=0x7fde5802f000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:48.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.353+0000 7fde5f400640 1 -- 172.21.15.86:0/258801243 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fde60026020 con 0x7fde6815d660 2024-03-06T07:32:48.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.587+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fde680691c0 con 0x7fde6815d660 2024-03-06T07:32:48.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.588+0000 7fde5f400640 1 -- 172.21.15.86:0/258801243 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7fde60058be0 con 0x7fde6815d660 2024-03-06T07:32:48.591 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:48.591 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:48.591 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b500 msgr2=0x7fde4406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b500 0x7fde4406d9c0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fde58034000 tx=0x7fde5802f000 comp rx=0 tx=0).stop 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6815d660 msgr2=0x7fde680c9db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6815d660 0x7fde680c9db0 secure :-1 s=READY pgs=1178 cs=0 l=1 rev1=1 crypto rx=0x7fde680cb020 tx=0x7fde60010c20 comp rx=0 tx=0).stop 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 shutdown_connections 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde4406b500 0x7fde4406d9c0 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde6815d660 0x7fde680c9db0 unknown :-1 s=CLOSED pgs=1178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde6815d1e0 0x7fde680c9870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 --2- 172.21.15.86:0/258801243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde4c0039e0 0x7fde680cb310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.590+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 >> 172.21.15.86:0/258801243 conn(0x7fde680d3fd0 msgr2=0x7fde680d0340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:48.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.591+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 shutdown_connections 2024-03-06T07:32:48.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.591+0000 7fde6d600640 1 -- 172.21.15.86:0/258801243 wait complete. 2024-03-06T07:32:48.612 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:48.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.811+0000 7f3d6ce00640 1 Processor -- start 2024-03-06T07:32:48.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.812+0000 7f3d6ce00640 1 -- start start 2024-03-06T07:32:48.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.813+0000 7f3d6ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d6815be90 0x7f3d680b49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.813+0000 7f3d6ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d680acd60 0x7f3d680d0070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.813+0000 7f3d6ce00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3d680b5160 con 0x7f3d680d05b0 2024-03-06T07:32:48.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.813+0000 7f3d6ce00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d680b4f30 con 0x7f3d6815be90 2024-03-06T07:32:48.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.813+0000 7f3d6ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d680af240 con 0x7f3d680acd60 2024-03-06T07:32:48.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d65e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d6815be90 0x7f3d680b49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d65e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d6815be90 0x7f3d680b49f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49060/0 (socket says 172.21.15.86:49060) 2024-03-06T07:32:48.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d65e00640 1 -- 172.21.15.86:0/2832225614 learned_addr learned my addr 172.21.15.86:0/2832225614 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:48.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d5d400640 1 --2- 172.21.15.86:0/2832225614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d680acd60 0x7f3d680d0070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d65e00640 1 -- 172.21.15.86:0/2832225614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d680acd60 msgr2=0x7f3d680d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:48.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d65e00640 1 --2- 172.21.15.86:0/2832225614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d680acd60 0x7f3d680d0070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d65e00640 1 -- 172.21.15.86:0/2832225614 >> v1:172.21.15.194:6789/0 conn(0x7f3d680d05b0 legacy=0x7f3d6806c9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.814+0000 7f3d65e00640 1 -- 172.21.15.86:0/2832225614 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d680ae910 con 0x7f3d6815be90 2024-03-06T07:32:48.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.815+0000 7f3d65e00640 1 --2- 172.21.15.86:0/2832225614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d6815be90 0x7f3d680b49f0 secure :-1 s=READY pgs=1179 cs=0 l=1 rev1=1 crypto rx=0x7f3d58009e50 tx=0x7f3d5802bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fed1efb8965ecdff server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:48.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d65400640 1 -- 172.21.15.86:0/2832225614 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d5802cb00 con 0x7f3d6815be90 2024-03-06T07:32:48.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d65400640 1 -- 172.21.15.86:0/2832225614 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d5802cca0 con 0x7f3d6815be90 2024-03-06T07:32:48.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d65400640 1 -- 172.21.15.86:0/2832225614 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d58034610 con 0x7f3d6815be90 2024-03-06T07:32:48.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d6ce00640 1 -- 172.21.15.86:0/2832225614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d6815be90 msgr2=0x7f3d680b49f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/2832225614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d6815be90 0x7f3d680b49f0 secure :-1 s=READY pgs=1179 cs=0 l=1 rev1=1 crypto rx=0x7f3d58009e50 tx=0x7f3d5802bf60 comp rx=0 tx=0).stop 2024-03-06T07:32:48.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d6ce00640 1 -- 172.21.15.86:0/2832225614 shutdown_connections 2024-03-06T07:32:48.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/2832225614 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d680acd60 0x7f3d680d0070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/2832225614 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d6815be90 0x7f3d680b49f0 unknown :-1 s=CLOSED pgs=1179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.816+0000 7f3d6ce00640 1 -- 172.21.15.86:0/2832225614 >> 172.21.15.86:0/2832225614 conn(0x7f3d6806b360 msgr2=0x7f3d6806b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:48.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.817+0000 7f3d6ce00640 1 -- 172.21.15.86:0/2832225614 shutdown_connections 2024-03-06T07:32:48.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.817+0000 7f3d6ce00640 1 -- 172.21.15.86:0/2832225614 wait complete. 2024-03-06T07:32:48.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.818+0000 7f3d6ce00640 1 Processor -- start 2024-03-06T07:32:48.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.818+0000 7f3d6ce00640 1 -- start start 2024-03-06T07:32:48.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.819+0000 7f3d6ce00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d680acd60 0x7f3d680ce940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.819+0000 7f3d6ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d680d05b0 0x7f3d68145820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.819+0000 7f3d6ce00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d6815be90 0x7f3d6815e2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.819+0000 7f3d6ce00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d680b2c80 con 0x7f3d680d05b0 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.819+0000 7f3d6ce00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d680aeed0 con 0x7f3d680acd60 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.819+0000 7f3d6ce00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d680b2e00 con 0x7f3d6815be90 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d680d05b0 0x7f3d68145820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d680d05b0 0x7f3d68145820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49068/0 (socket says 172.21.15.86:49068) 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 -- 172.21.15.86:0/1086220124 learned_addr learned my addr 172.21.15.86:0/1086220124 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d65e00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d680acd60 0x7f3d680ce940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d66800640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d6815be90 0x7f3d6815e2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 -- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d6815be90 msgr2=0x7f3d6815e2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d6815be90 0x7f3d6815e2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 -- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d680acd60 msgr2=0x7f3d680ce940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:48.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d680acd60 0x7f3d680ce940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 -- 172.21.15.86:0/1086220124 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d68154c10 con 0x7f3d680d05b0 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d65e00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d680acd60 0x7f3d680ce940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d66800640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d6815be90 0x7f3d6815e2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.820+0000 7f3d5d400640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d680d05b0 0x7f3d68145820 secure :-1 s=READY pgs=1180 cs=0 l=1 rev1=1 crypto rx=0x7f3d5000d800 tx=0x7f3d5000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.821+0000 7f3d5ea00640 1 -- 172.21.15.86:0/1086220124 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d5000b6c0 con 0x7f3d680d05b0 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.821+0000 7f3d5ea00640 1 -- 172.21.15.86:0/1086220124 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d50019070 con 0x7f3d680d05b0 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.821+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d6815d510 con 0x7f3d680d05b0 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.821+0000 7f3d5ea00640 1 -- 172.21.15.86:0/1086220124 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d5000ba80 con 0x7f3d680d05b0 2024-03-06T07:32:48.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.821+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d6815dad0 con 0x7f3d680d05b0 2024-03-06T07:32:48.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.822+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d68068070 con 0x7f3d680d05b0 2024-03-06T07:32:48.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.823+0000 7f3d5ea00640 1 -- 172.21.15.86:0/1086220124 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3d5000bc20 con 0x7f3d680d05b0 2024-03-06T07:32:48.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.824+0000 7f3d5ea00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d4406b4d0 0x7f3d4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:48.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.824+0000 7f3d5ea00640 1 -- 172.21.15.86:0/1086220124 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f3d5008a800 con 0x7f3d680d05b0 2024-03-06T07:32:48.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.824+0000 7f3d65e00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d4406b4d0 0x7f3d4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:48.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.825+0000 7f3d65e00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d4406b4d0 0x7f3d4406d990 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f3d58035000 tx=0x7f3d58030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:48.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:48.833+0000 7f3d5ea00640 1 -- 172.21.15.86:0/1086220124 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3d68154c10 con 0x7f3d680d05b0 2024-03-06T07:32:49.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.031+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f3d680b07d0 con 0x7f3d680d05b0 2024-03-06T07:32:49.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.034+0000 7f3d5ea00640 1 -- 172.21.15.86:0/1086220124 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v98) v1 ==== 74+0+16879 (secure 0 0 0) 0x7f3d50059530 con 0x7f3d680d05b0 2024-03-06T07:32:49.037 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:49.037 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":98,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:20.051194+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:32:01.521762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-06T07:32:02.759692+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"94","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6835/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.86:6834/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.194:6834/2712222804":"2024-03-07T07:32:14.424511+0000","172.21.15.194:6832/2712222804":"2024-03-07T07:32:14.424511+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d4406b4d0 msgr2=0x7f3d4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d4406b4d0 0x7f3d4406d990 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f3d58035000 tx=0x7f3d58030000 comp rx=0 tx=0).stop 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d680d05b0 msgr2=0x7f3d68145820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d680d05b0 0x7f3d68145820 secure :-1 s=READY pgs=1180 cs=0 l=1 rev1=1 crypto rx=0x7f3d5000d800 tx=0x7f3d5000dcc0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 shutdown_connections 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d4406b4d0 0x7f3d4406d990 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d6815be90 0x7f3d6815e2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d680d05b0 0x7f3d68145820 unknown :-1 s=CLOSED pgs=1180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 --2- 172.21.15.86:0/1086220124 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d680acd60 0x7f3d680ce940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 >> 172.21.15.86:0/1086220124 conn(0x7f3d6806b360 msgr2=0x7f3d68155170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:49.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 shutdown_connections 2024-03-06T07:32:49.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.037+0000 7f3d6ce00640 1 -- 172.21.15.86:0/1086220124 wait complete. 2024-03-06T07:32:49.057 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:32:49.058 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:49.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.255+0000 7f139ec00640 1 Processor -- start 2024-03-06T07:32:49.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.255+0000 7f139ec00640 1 -- start start 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.256+0000 7f139ec00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f13981448a0 0x7f13980b2a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.256+0000 7f139ec00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f139815c190 0x7f139806cc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.256+0000 7f139ec00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f13980b3430 con 0x7f13980b2fb0 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.256+0000 7f139ec00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f13980cc490 con 0x7f13981448a0 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.256+0000 7f139ec00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f13980cc6e0 con 0x7f139815c190 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139d800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f13981448a0 0x7f13980b2a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139ce00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f13980b2fb0 0x7f13980cea90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44502/0 (socket says 172.21.15.86:44502) 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139d800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f13981448a0 0x7f13980b2a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49076/0 (socket says 172.21.15.86:49076) 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139ce00640 1 -- 172.21.15.86:0/1151442105 learned_addr learned my addr 172.21.15.86:0/1151442105 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139e200640 1 --2- 172.21.15.86:0/1151442105 >> v2:172.21.15.194:3300/0 conn(0x7f139815c190 0x7f139806cc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139d800640 1 -- 172.21.15.86:0/1151442105 >> v2:172.21.15.194:3300/0 conn(0x7f139815c190 msgr2=0x7f139806cc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:49.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139d800640 1 --2- 172.21.15.86:0/1151442105 >> v2:172.21.15.194:3300/0 conn(0x7f139815c190 0x7f139806cc00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139d800640 1 -- 172.21.15.86:0/1151442105 >> v1:172.21.15.86:6789/0 conn(0x7f13980b2fb0 legacy=0x7f13980cea90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f139d800640 1 -- 172.21.15.86:0/1151442105 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13980cc9c0 con 0x7f13981448a0 2024-03-06T07:32:49.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.257+0000 7f1397e00640 1 -- 172.21.15.86:0/1151442105 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3503422310 0 0) 0x7f13980b3430 con 0x7f13980b2fb0 2024-03-06T07:32:49.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.258+0000 7f139d800640 1 --2- 172.21.15.86:0/1151442105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f13980b2a70 secure :-1 s=READY pgs=1182 cs=0 l=1 rev1=1 crypto rx=0x7f138c0097a0 tx=0x7f138c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=53df730396b23b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:49.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.259+0000 7f1397e00640 1 -- 172.21.15.86:0/1151442105 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f138c035020 con 0x7f13981448a0 2024-03-06T07:32:49.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.259+0000 7f1397e00640 1 -- 172.21.15.86:0/1151442105 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f138c031830 con 0x7f13981448a0 2024-03-06T07:32:49.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.259+0000 7f1397e00640 1 -- 172.21.15.86:0/1151442105 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f138c031c00 con 0x7f13981448a0 2024-03-06T07:32:49.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.259+0000 7f139ec00640 1 -- 172.21.15.86:0/1151442105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 msgr2=0x7f13980b2a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.259+0000 7f139ec00640 1 --2- 172.21.15.86:0/1151442105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f13980b2a70 secure :-1 s=READY pgs=1182 cs=0 l=1 rev1=1 crypto rx=0x7f138c0097a0 tx=0x7f138c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.260+0000 7f139ec00640 1 -- 172.21.15.86:0/1151442105 shutdown_connections 2024-03-06T07:32:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.260+0000 7f139ec00640 1 --2- 172.21.15.86:0/1151442105 >> v2:172.21.15.194:3300/0 conn(0x7f139815c190 0x7f139806cc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.260+0000 7f139ec00640 1 --2- 172.21.15.86:0/1151442105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f13980b2a70 unknown :-1 s=CLOSED pgs=1182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.260+0000 7f139ec00640 1 -- 172.21.15.86:0/1151442105 >> 172.21.15.86:0/1151442105 conn(0x7f13980b1770 msgr2=0x7f13980b1b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.260+0000 7f139ec00640 1 -- 172.21.15.86:0/1151442105 shutdown_connections 2024-03-06T07:32:49.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.260+0000 7f139ec00640 1 -- 172.21.15.86:0/1151442105 wait complete. 2024-03-06T07:32:49.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.261+0000 7f139ec00640 1 Processor -- start 2024-03-06T07:32:49.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.261+0000 7f139ec00640 1 -- start start 2024-03-06T07:32:49.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.262+0000 7f139ec00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13980b2fb0 0x7f13980712f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.262+0000 7f139ec00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f1398070c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.262+0000 7f139ec00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f139815c190 0x7f139815a570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.262+0000 7f139ec00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13980cd1c0 con 0x7f13981448a0 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.262+0000 7f139ec00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13980cf1a0 con 0x7f139815c190 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.262+0000 7f139ec00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f13980719f0 con 0x7f13980b2fb0 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f1398070c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f1398070c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49080/0 (socket says 172.21.15.86:49080) 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 -- 172.21.15.86:0/437700874 learned_addr learned my addr 172.21.15.86:0/437700874 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 -- 172.21.15.86:0/437700874 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13980b2fb0 msgr2=0x7f13980712f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13980b2fb0 0x7f13980712f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 -- 172.21.15.86:0/437700874 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f139815c190 msgr2=0x7f139815a570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f139815c190 0x7f139815a570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 -- 172.21.15.86:0/437700874 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1398071cd0 con 0x7f13981448a0 2024-03-06T07:32:49.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.263+0000 7f139ce00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f1398070c60 secure :-1 s=READY pgs=1183 cs=0 l=1 rev1=1 crypto rx=0x7f138400c4d0 tx=0x7f138400c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:49.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.264+0000 7f1395600640 1 -- 172.21.15.86:0/437700874 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1384018070 con 0x7f13981448a0 2024-03-06T07:32:49.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.264+0000 7f1395600640 1 -- 172.21.15.86:0/437700874 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1384007500 con 0x7f13981448a0 2024-03-06T07:32:49.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.264+0000 7f1395600640 1 -- 172.21.15.86:0/437700874 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1384004a20 con 0x7f13981448a0 2024-03-06T07:32:49.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.264+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1398071fb0 con 0x7f13981448a0 2024-03-06T07:32:49.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.264+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f139815aab0 con 0x7f13981448a0 2024-03-06T07:32:49.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.265+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f139804f330 con 0x7f13981448a0 2024-03-06T07:32:49.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.266+0000 7f1395600640 1 -- 172.21.15.86:0/437700874 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f138401b070 con 0x7f13981448a0 2024-03-06T07:32:49.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.267+0000 7f1395600640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f137806b500 0x7f137806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.267+0000 7f1395600640 1 -- 172.21.15.86:0/437700874 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f1384089570 con 0x7f13981448a0 2024-03-06T07:32:49.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.267+0000 7f139d800640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f137806b500 0x7f137806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.268+0000 7f139d800640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f137806b500 0x7f137806d9c0 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f138c036000 tx=0x7f138c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:49.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.271+0000 7f1395600640 1 -- 172.21.15.86:0/437700874 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1384058340 con 0x7f13981448a0 2024-03-06T07:32:49.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.507+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1398072990 con 0x7f13981448a0 2024-03-06T07:32:49.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.507+0000 7f1395600640 1 -- 172.21.15.86:0/437700874 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f1384057c40 con 0x7f13981448a0 2024-03-06T07:32:49.510 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:49.510 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:49.510 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f137806b500 msgr2=0x7f137806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f137806b500 0x7f137806d9c0 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f138c036000 tx=0x7f138c030000 comp rx=0 tx=0).stop 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 msgr2=0x7f1398070c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f1398070c60 secure :-1 s=READY pgs=1183 cs=0 l=1 rev1=1 crypto rx=0x7f138400c4d0 tx=0x7f138400c990 comp rx=0 tx=0).stop 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 shutdown_connections 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f137806b500 0x7f137806d9c0 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f139815c190 0x7f139815a570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f13981448a0 0x7f1398070c60 unknown :-1 s=CLOSED pgs=1183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 --2- 172.21.15.86:0/437700874 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f13980b2fb0 0x7f13980712f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 >> 172.21.15.86:0/437700874 conn(0x7f13980b1770 msgr2=0x7f13981439c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:49.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 shutdown_connections 2024-03-06T07:32:49.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.510+0000 7f139ec00640 1 -- 172.21.15.86:0/437700874 wait complete. 2024-03-06T07:32:49.530 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:49.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.723+0000 7f629d200640 1 Processor -- start 2024-03-06T07:32:49.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.723+0000 7f629d200640 1 -- start start 2024-03-06T07:32:49.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f629d200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f62980d0780 0x7f629815e720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f629d200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f629806d600 0x7f629815ec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f629d200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f629815f370 con 0x7f62980b3340 2024-03-06T07:32:49.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f629d200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f629815f650 con 0x7f629806d600 2024-03-06T07:32:49.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f629d200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f629806cb20 con 0x7f62980d0780 2024-03-06T07:32:49.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f6296800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f629806d600 0x7f629815ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f6296800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f629806d600 0x7f629815ec60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49082/0 (socket says 172.21.15.86:49082) 2024-03-06T07:32:49.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f6296800640 1 -- 172.21.15.86:0/3422861577 learned_addr learned my addr 172.21.15.86:0/3422861577 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:49.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.725+0000 7f6295400640 1 --2- 172.21.15.86:0/3422861577 >> v2:172.21.15.194:3300/0 conn(0x7f62980d0780 0x7f629815e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.726+0000 7f6296800640 1 -- 172.21.15.86:0/3422861577 >> v2:172.21.15.194:3300/0 conn(0x7f62980d0780 msgr2=0x7f629815e720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:49.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.726+0000 7f6296800640 1 --2- 172.21.15.86:0/3422861577 >> v2:172.21.15.194:3300/0 conn(0x7f62980d0780 0x7f629815e720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.726+0000 7f6296800640 1 -- 172.21.15.86:0/3422861577 >> v1:172.21.15.194:6789/0 conn(0x7f62980b3340 legacy=0x7f62980d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:49.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.726+0000 7f6296800640 1 -- 172.21.15.86:0/3422861577 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f629806ce00 con 0x7f629806d600 2024-03-06T07:32:49.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.727+0000 7f6296800640 1 --2- 172.21.15.86:0/3422861577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f629806d600 0x7f629815ec60 secure :-1 s=READY pgs=1184 cs=0 l=1 rev1=1 crypto rx=0x7f62980d0c20 tx=0x7f629002bdc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d187929ff13ba9aa server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:49.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.727+0000 7f6294a00640 1 -- 172.21.15.86:0/3422861577 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f629002c960 con 0x7f629806d600 2024-03-06T07:32:49.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.727+0000 7f6294a00640 1 -- 172.21.15.86:0/3422861577 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f629002cb00 con 0x7f629806d600 2024-03-06T07:32:49.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.727+0000 7f6294a00640 1 -- 172.21.15.86:0/3422861577 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62900343f0 con 0x7f629806d600 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 -- 172.21.15.86:0/3422861577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f629806d600 msgr2=0x7f629815ec60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 --2- 172.21.15.86:0/3422861577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f629806d600 0x7f629815ec60 secure :-1 s=READY pgs=1184 cs=0 l=1 rev1=1 crypto rx=0x7f62980d0c20 tx=0x7f629002bdc0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 -- 172.21.15.86:0/3422861577 shutdown_connections 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 --2- 172.21.15.86:0/3422861577 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f629806d600 0x7f629815ec60 unknown :-1 s=CLOSED pgs=1184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 --2- 172.21.15.86:0/3422861577 >> v2:172.21.15.194:3300/0 conn(0x7f62980d0780 0x7f629815e720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 -- 172.21.15.86:0/3422861577 >> 172.21.15.86:0/3422861577 conn(0x7f629806b360 msgr2=0x7f629806b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 -- 172.21.15.86:0/3422861577 shutdown_connections 2024-03-06T07:32:49.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.728+0000 7f629d200640 1 -- 172.21.15.86:0/3422861577 wait complete. 2024-03-06T07:32:49.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.729+0000 7f629d200640 1 Processor -- start 2024-03-06T07:32:49.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.730+0000 7f629d200640 1 -- start start 2024-03-06T07:32:49.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.730+0000 7f629d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f629806d600 0x7f629809d290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f629d200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62980b3340 0x7f6298157ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f629d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62980d0780 0x7f629809b3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f629d200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62980acd60 con 0x7f62980d0780 2024-03-06T07:32:49.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f629d200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62980b3720 con 0x7f62980b3340 2024-03-06T07:32:49.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f629d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62980acee0 con 0x7f629806d600 2024-03-06T07:32:49.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62980d0780 0x7f629809b3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62980d0780 0x7f629809b3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49092/0 (socket says 172.21.15.86:49092) 2024-03-06T07:32:49.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 -- 172.21.15.86:0/3545110395 learned_addr learned my addr 172.21.15.86:0/3545110395 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:49.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6295400640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62980b3340 0x7f6298157ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6295e00640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f629806d600 0x7f629809d290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 -- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f629806d600 msgr2=0x7f629809d290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f629806d600 0x7f629809d290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 -- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62980b3340 msgr2=0x7f6298157ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62980b3340 0x7f6298157ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6296800640 1 -- 172.21.15.86:0/3545110395 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6298159250 con 0x7f62980d0780 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6295e00640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f629806d600 0x7f629809d290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.731+0000 7f6295400640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62980b3340 0x7f6298157ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.732+0000 7f6296800640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62980d0780 0x7f629809b3b0 secure :-1 s=READY pgs=1185 cs=0 l=1 rev1=1 crypto rx=0x7f6290009f40 tx=0x7f6290004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.732+0000 7f628e000640 1 -- 172.21.15.86:0/3545110395 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6290043020 con 0x7f62980d0780 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.732+0000 7f628e000640 1 -- 172.21.15.86:0/3545110395 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f629003e450 con 0x7f62980d0780 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.732+0000 7f628e000640 1 -- 172.21.15.86:0/3545110395 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f629003e880 con 0x7f62980d0780 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.732+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6298159530 con 0x7f62980d0780 2024-03-06T07:32:49.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.732+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6298159af0 con 0x7f62980d0780 2024-03-06T07:32:49.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.733+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f629815d230 con 0x7f62980d0780 2024-03-06T07:32:49.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.734+0000 7f628e000640 1 -- 172.21.15.86:0/3545110395 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6290046020 con 0x7f62980d0780 2024-03-06T07:32:49.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.736+0000 7f628e000640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f627806b4d0 0x7f627806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:49.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.736+0000 7f628e000640 1 -- 172.21.15.86:0/3545110395 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f62900acac0 con 0x7f62980d0780 2024-03-06T07:32:49.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.736+0000 7f6295e00640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f627806b4d0 0x7f627806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:49.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.737+0000 7f6295e00640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f627806b4d0 0x7f627806d990 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f6288007fd0 tx=0x7f6288009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:49.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.743+0000 7f628e000640 1 -- 172.21.15.86:0/3545110395 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f629004d030 con 0x7f62980d0780 2024-03-06T07:32:49.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.979+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6298068070 con 0x7f62980d0780 2024-03-06T07:32:49.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.981+0000 7f628e000640 1 -- 172.21.15.86:0/3545110395 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 69 v69) v1 ==== 94+0+4986 (secure 0 0 0) 0x7f629007b880 con 0x7f62980d0780 2024-03-06T07:32:49.983 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:49.984 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":69,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66}],"filesystems":[{"mdsmap":{"epoch":69,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:32:03.747182+0000","modified":"2024-03-06T07:32:24.075822+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":98,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7164},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7164":{"gid":7164,"name":"c","rank":0,"incarnation":65,"state":"up:active","state_seq":6,"addr":"172.21.15.86:6835/2542557324","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2542557324},{"type":"v1","addr":"172.21.15.86:6835","nonce":2542557324}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7164,"qdb_cluster":[7164]},"id":7}]} 2024-03-06T07:32:49.984 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 69 2024-03-06T07:32:49.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f627806b4d0 msgr2=0x7f627806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f627806b4d0 0x7f627806d990 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f6288007fd0 tx=0x7f6288009040 comp rx=0 tx=0).stop 2024-03-06T07:32:49.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62980d0780 msgr2=0x7f629809b3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62980d0780 0x7f629809b3b0 secure :-1 s=READY pgs=1185 cs=0 l=1 rev1=1 crypto rx=0x7f6290009f40 tx=0x7f6290004a40 comp rx=0 tx=0).stop 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 shutdown_connections 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f627806b4d0 0x7f627806d990 unknown :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f62980d0780 0x7f629809b3b0 unknown :-1 s=CLOSED pgs=1185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f62980b3340 0x7f6298157ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 --2- 172.21.15.86:0/3545110395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f629806d600 0x7f629809d290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 >> 172.21.15.86:0/3545110395 conn(0x7f629806b360 msgr2=0x7f629806b740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 shutdown_connections 2024-03-06T07:32:49.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:49.984+0000 7f629d200640 1 -- 172.21.15.86:0/3545110395 wait complete. 2024-03-06T07:32:50.003 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:50.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.253+0000 7f0d69600640 1 Processor -- start 2024-03-06T07:32:50.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.253+0000 7f0d69600640 1 -- start start 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.255+0000 7f0d69600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d6415e980 0x7f0d6415ed60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.255+0000 7f0d69600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0d640b3be0 0x7f0d641547c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.255+0000 7f0d69600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0d640ccc30 con 0x7f0d6415f2a0 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.255+0000 7f0d69600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d64061e80 con 0x7f0d6415e980 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.255+0000 7f0d69600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d6406e190 con 0x7f0d640b3be0 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.255+0000 7f0d62a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d6415e980 0x7f0d6415ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.256+0000 7f0d62a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0d6415e980 0x7f0d6415ed60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49108/0 (socket says 172.21.15.86:49108) 2024-03-06T07:32:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.256+0000 7f0d62a00640 1 -- 172.21.15.86:0/3122672969 learned_addr learned my addr 172.21.15.86:0/3122672969 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:50.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.256+0000 7f0d62a00640 1 -- 172.21.15.86:0/3122672969 >> v2:172.21.15.194:3300/0 conn(0x7f0d640b3be0 msgr2=0x7f0d641547c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:50.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.256+0000 7f0d62a00640 1 --2- 172.21.15.86:0/3122672969 >> v2:172.21.15.194:3300/0 conn(0x7f0d640b3be0 0x7f0d641547c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.256+0000 7f0d62a00640 1 -- 172.21.15.86:0/3122672969 >> v1:172.21.15.194:6789/0 conn(0x7f0d6415f2a0 legacy=0x7f0d640b34d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:50.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.256+0000 7f0d62a00640 1 -- 172.21.15.86:0/3122672969 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d6406d5c0 con 0x7f0d6415e980 2024-03-06T07:32:50.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.257+0000 7f0d62a00640 1 --2- 172.21.15.86:0/3122672969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d6415e980 0x7f0d6415ed60 secure :-1 s=READY pgs=1186 cs=0 l=1 rev1=1 crypto rx=0x7f0d6406e100 tx=0x7f0d5402bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=141adab28dace55f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:50.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.257+0000 7f0d62000640 1 -- 172.21.15.86:0/3122672969 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d5402ca80 con 0x7f0d6415e980 2024-03-06T07:32:50.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.257+0000 7f0d62000640 1 -- 172.21.15.86:0/3122672969 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d5402cc20 con 0x7f0d6415e980 2024-03-06T07:32:50.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.257+0000 7f0d62000640 1 -- 172.21.15.86:0/3122672969 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d540343f0 con 0x7f0d6415e980 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 -- 172.21.15.86:0/3122672969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d6415e980 msgr2=0x7f0d6415ed60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 --2- 172.21.15.86:0/3122672969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d6415e980 0x7f0d6415ed60 secure :-1 s=READY pgs=1186 cs=0 l=1 rev1=1 crypto rx=0x7f0d6406e100 tx=0x7f0d5402bee0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 -- 172.21.15.86:0/3122672969 shutdown_connections 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 --2- 172.21.15.86:0/3122672969 >> v2:172.21.15.194:3300/0 conn(0x7f0d640b3be0 0x7f0d641547c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 --2- 172.21.15.86:0/3122672969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d6415e980 0x7f0d6415ed60 unknown :-1 s=CLOSED pgs=1186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 -- 172.21.15.86:0/3122672969 >> 172.21.15.86:0/3122672969 conn(0x7f0d640d0540 msgr2=0x7f0d640d0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 -- 172.21.15.86:0/3122672969 shutdown_connections 2024-03-06T07:32:50.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.258+0000 7f0d69600640 1 -- 172.21.15.86:0/3122672969 wait complete. 2024-03-06T07:32:50.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.260+0000 7f0d69600640 1 Processor -- start 2024-03-06T07:32:50.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.260+0000 7f0d69600640 1 -- start start 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d69600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d640b3be0 0x7f0d64141ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d69600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d6415e980 0x7f0d6414da70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d69600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d6415f2a0 0x7f0d6414ff30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d69600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d6406de20 con 0x7f0d640b3be0 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d69600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d6406db40 con 0x7f0d6415e980 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d69600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d6406eff0 con 0x7f0d6415f2a0 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d62a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d640b3be0 0x7f0d64141ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d62a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d640b3be0 0x7f0d64141ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49118/0 (socket says 172.21.15.86:49118) 2024-03-06T07:32:50.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.261+0000 7f0d62a00640 1 -- 172.21.15.86:0/1206303305 learned_addr learned my addr 172.21.15.86:0/1206303305 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:50.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d62a00640 1 -- 172.21.15.86:0/1206303305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d6415f2a0 msgr2=0x7f0d6414ff30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d62a00640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d6415f2a0 0x7f0d6414ff30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d62a00640 1 -- 172.21.15.86:0/1206303305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d6415e980 msgr2=0x7f0d6414da70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d62a00640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d6415e980 0x7f0d6414da70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d62a00640 1 -- 172.21.15.86:0/1206303305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d6414e510 con 0x7f0d640b3be0 2024-03-06T07:32:50.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d62a00640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d640b3be0 0x7f0d64141ff0 secure :-1 s=READY pgs=1187 cs=0 l=1 rev1=1 crypto rx=0x7f0d54036040 tx=0x7f0d54004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:50.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d5b400640 1 -- 172.21.15.86:0/1206303305 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d54043020 con 0x7f0d640b3be0 2024-03-06T07:32:50.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d5b400640 1 -- 172.21.15.86:0/1206303305 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d5403e450 con 0x7f0d640b3be0 2024-03-06T07:32:50.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d5b400640 1 -- 172.21.15.86:0/1206303305 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d5403e840 con 0x7f0d640b3be0 2024-03-06T07:32:50.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.262+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d6414e7f0 con 0x7f0d640b3be0 2024-03-06T07:32:50.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.263+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d6414edb0 con 0x7f0d640b3be0 2024-03-06T07:32:50.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.264+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d640b5280 con 0x7f0d640b3be0 2024-03-06T07:32:50.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.265+0000 7f0d5b400640 1 -- 172.21.15.86:0/1206303305 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0d54046020 con 0x7f0d640b3be0 2024-03-06T07:32:50.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.265+0000 7f0d5b400640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d4006b4d0 0x7f0d4006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.266+0000 7f0d5b400640 1 -- 172.21.15.86:0/1206303305 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f0d540acd10 con 0x7f0d640b3be0 2024-03-06T07:32:50.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.266+0000 7f0d5a000640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d4006b4d0 0x7f0d4006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.267+0000 7f0d5a000640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d4006b4d0 0x7f0d4006d990 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f0d6415fd90 tx=0x7f0d4c006ba0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:50.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.271+0000 7f0d5b400640 1 -- 172.21.15.86:0/1206303305 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0d640b5280 con 0x7f0d640b3be0 2024-03-06T07:32:50.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.464+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0d64068070 con 0x7f0d640b3be0 2024-03-06T07:32:50.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.467+0000 7f0d5b400640 1 -- 172.21.15.86:0/1206303305 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v98) v1 ==== 74+0+16879 (secure 0 0 0) 0x7f0d540b2020 con 0x7f0d640b3be0 2024-03-06T07:32:50.470 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:50.470 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":98,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:20.051194+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:32:01.521762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"93","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-06T07:32:02.759692+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"94","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6835/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.86:6834/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.194:6834/2712222804":"2024-03-07T07:32:14.424511+0000","172.21.15.194:6832/2712222804":"2024-03-07T07:32:14.424511+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d4006b4d0 msgr2=0x7f0d4006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d4006b4d0 0x7f0d4006d990 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f0d6415fd90 tx=0x7f0d4c006ba0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d640b3be0 msgr2=0x7f0d64141ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d640b3be0 0x7f0d64141ff0 secure :-1 s=READY pgs=1187 cs=0 l=1 rev1=1 crypto rx=0x7f0d54036040 tx=0x7f0d54004a40 comp rx=0 tx=0).stop 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 shutdown_connections 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0d4006b4d0 0x7f0d4006d990 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0d6415f2a0 0x7f0d6414ff30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0d6415e980 0x7f0d6414da70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 --2- 172.21.15.86:0/1206303305 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0d640b3be0 0x7f0d64141ff0 unknown :-1 s=CLOSED pgs=1187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 >> 172.21.15.86:0/1206303305 conn(0x7f0d640d0540 msgr2=0x7f0d640b1110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 shutdown_connections 2024-03-06T07:32:50.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.470+0000 7f0d69600640 1 -- 172.21.15.86:0/1206303305 wait complete. 2024-03-06T07:32:50.490 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:32:50.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.681+0000 7f9497600640 1 Processor -- start 2024-03-06T07:32:50.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.681+0000 7f9497600640 1 -- start start 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9497600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f949015e980 0x7f949015ed60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9497600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 0x7f94900b34d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9497600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f94900ccc30 con 0x7f949006eff0 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9497600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9490061e80 con 0x7f949015e980 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9497600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f949006e190 con 0x7f949015f2a0 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9494c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f949015e980 0x7f949015ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9494c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f949015e980 0x7f949015ed60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49126/0 (socket says 172.21.15.86:49126) 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9494c00640 1 -- 172.21.15.86:0/3078592579 learned_addr learned my addr 172.21.15.86:0/3078592579 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9494c00640 1 -- 172.21.15.86:0/3078592579 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 msgr2=0x7f94900b34d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.683+0000 7f9494c00640 1 --2- 172.21.15.86:0/3078592579 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 0x7f94900b34d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.684+0000 7f9494c00640 1 -- 172.21.15.86:0/3078592579 >> v1:172.21.15.194:6789/0 conn(0x7f949006eff0 legacy=0x7f94901547c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.684+0000 7f9494c00640 1 -- 172.21.15.86:0/3078592579 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f949006d5c0 con 0x7f949015e980 2024-03-06T07:32:50.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.685+0000 7f9494c00640 1 --2- 172.21.15.86:0/3078592579 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949015e980 0x7f949015ed60 secure :-1 s=READY pgs=1188 cs=0 l=1 rev1=1 crypto rx=0x7f9484009870 tx=0x7f948402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e0616c0ab67652e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.685+0000 7f948f400640 1 -- 172.21.15.86:0/3078592579 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9484035020 con 0x7f949015e980 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.685+0000 7f948f400640 1 -- 172.21.15.86:0/3078592579 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9484031640 con 0x7f949015e980 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.685+0000 7f948f400640 1 -- 172.21.15.86:0/3078592579 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f948402d040 con 0x7f949015e980 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.685+0000 7f9497600640 1 -- 172.21.15.86:0/3078592579 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949015e980 msgr2=0x7f949015ed60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.685+0000 7f9497600640 1 --2- 172.21.15.86:0/3078592579 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949015e980 0x7f949015ed60 secure :-1 s=READY pgs=1188 cs=0 l=1 rev1=1 crypto rx=0x7f9484009870 tx=0x7f948402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.686+0000 7f9497600640 1 -- 172.21.15.86:0/3078592579 shutdown_connections 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.686+0000 7f9497600640 1 --2- 172.21.15.86:0/3078592579 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 0x7f94900b34d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.686+0000 7f9497600640 1 --2- 172.21.15.86:0/3078592579 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949015e980 0x7f949015ed60 unknown :-1 s=CLOSED pgs=1188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.686+0000 7f9497600640 1 -- 172.21.15.86:0/3078592579 >> 172.21.15.86:0/3078592579 conn(0x7f94900d0540 msgr2=0x7f94900d0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.686+0000 7f9497600640 1 -- 172.21.15.86:0/3078592579 shutdown_connections 2024-03-06T07:32:50.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.686+0000 7f9497600640 1 -- 172.21.15.86:0/3078592579 wait complete. 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.687+0000 7f9497600640 1 Processor -- start 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.688+0000 7f9497600640 1 -- start start 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.688+0000 7f9497600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949006eff0 0x7f94901528b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.688+0000 7f9497600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f949015e980 0x7f94901502c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9497600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 0x7f9490140680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9497600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f949006de20 con 0x7f949006eff0 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9497600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f949006db40 con 0x7f949015e980 2024-03-06T07:32:50.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9497600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f94900b3a10 con 0x7f949015f2a0 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949006eff0 0x7f94901528b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949006eff0 0x7f94901528b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49138/0 (socket says 172.21.15.86:49138) 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 -- 172.21.15.86:0/3162917281 learned_addr learned my addr 172.21.15.86:0/3162917281 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 -- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 msgr2=0x7f9490140680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9495600640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 0x7f9490140680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f948fe00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f949015e980 0x7f94901502c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 0x7f9490140680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 -- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f949015e980 msgr2=0x7f94901502c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f949015e980 0x7f94901502c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f9494c00640 1 -- 172.21.15.86:0/3162917281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9490152e90 con 0x7f949006eff0 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.689+0000 7f948fe00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f949015e980 0x7f94901502c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.690+0000 7f9494c00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949006eff0 0x7f94901528b0 secure :-1 s=READY pgs=1189 cs=0 l=1 rev1=1 crypto rx=0x7f948402b930 tx=0x7f94840046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:50.694 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.690+0000 7f948cc00640 1 -- 172.21.15.86:0/3162917281 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f948403ac60 con 0x7f949006eff0 2024-03-06T07:32:50.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.690+0000 7f948cc00640 1 -- 172.21.15.86:0/3162917281 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f948403ae20 con 0x7f949006eff0 2024-03-06T07:32:50.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.690+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9490153170 con 0x7f949006eff0 2024-03-06T07:32:50.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.690+0000 7f948cc00640 1 -- 172.21.15.86:0/3162917281 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9484044630 con 0x7f949006eff0 2024-03-06T07:32:50.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.690+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f94900cdef0 con 0x7f949006eff0 2024-03-06T07:32:50.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.691+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9490068070 con 0x7f949006eff0 2024-03-06T07:32:50.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.692+0000 7f948cc00640 1 -- 172.21.15.86:0/3162917281 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f948404b060 con 0x7f949006eff0 2024-03-06T07:32:50.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.693+0000 7f948cc00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f947006b4d0 0x7f947006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:50.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.693+0000 7f948cc00640 1 -- 172.21.15.86:0/3162917281 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f94840ac220 con 0x7f949006eff0 2024-03-06T07:32:50.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.693+0000 7f948fe00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f947006b4d0 0x7f947006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:50.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.694+0000 7f948fe00640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f947006b4d0 0x7f947006d990 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f949006b600 tx=0x7f947c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:50.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.697+0000 7f948cc00640 1 -- 172.21.15.86:0/3162917281 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f948403f070 con 0x7f949006eff0 2024-03-06T07:32:50.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.929+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f949015e0e0 con 0x7f949006eff0 2024-03-06T07:32:50.933 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:32:50.931+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:32:50.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.933+0000 7f948cc00640 1 -- 172.21.15.86:0/3162917281 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v70) v1 ==== 161+0+0 (secure 0 0 0) 0x7f948402f300 con 0x7f949006eff0 2024-03-06T07:32:50.936 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:32:50.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f947006b4d0 msgr2=0x7f947006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f947006b4d0 0x7f947006d990 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f949006b600 tx=0x7f947c008040 comp rx=0 tx=0).stop 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949006eff0 msgr2=0x7f94901528b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949006eff0 0x7f94901528b0 secure :-1 s=READY pgs=1189 cs=0 l=1 rev1=1 crypto rx=0x7f948402b930 tx=0x7f94840046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 shutdown_connections 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f947006b4d0 0x7f947006d990 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f949015f2a0 0x7f9490140680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f949015e980 0x7f94901502c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 --2- 172.21.15.86:0/3162917281 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f949006eff0 0x7f94901528b0 unknown :-1 s=CLOSED pgs=1189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:50.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.937+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 >> 172.21.15.86:0/3162917281 conn(0x7f94900d0540 msgr2=0x7f94900b0140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:50.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.938+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 shutdown_connections 2024-03-06T07:32:50.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:50.938+0000 7f9497600640 1 -- 172.21.15.86:0/3162917281 wait complete. 2024-03-06T07:32:50.963 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:32:51.015 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:32:51.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.140+0000 7ffbd1600640 1 Processor -- start 2024-03-06T07:32:51.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.140+0000 7ffbd1600640 1 -- start start 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbd1600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ffbcc06d400 0x7ffbcc0d0070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbd1600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ffbcc0d05b0 0x7ffbcc0acd60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbd1600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffbcc1635a0 con 0x7ffbcc15cc90 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbd1600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffbcc163880 con 0x7ffbcc0d05b0 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbd1600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffbcc06cc00 con 0x7ffbcc06d400 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcb400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ffbcc0d05b0 0x7ffbcc0acd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcb400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ffbcc0d05b0 0x7ffbcc0acd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49166/0 (socket says 172.21.15.86:49166) 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcb400640 1 -- 172.21.15.86:0/823886341 learned_addr learned my addr 172.21.15.86:0/823886341 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:51.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcaa00640 1 --1- 172.21.15.86:0/823886341 >> v1:172.21.15.86:6789/0 conn(0x7ffbcc15cc90 0x7ffbcc162b00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44514/0 (socket says 172.21.15.86:44514) 2024-03-06T07:32:51.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcb400640 1 -- 172.21.15.86:0/823886341 >> v2:172.21.15.194:3300/0 conn(0x7ffbcc06d400 msgr2=0x7ffbcc0d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:51.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbca000640 1 --2- 172.21.15.86:0/823886341 >> v2:172.21.15.194:3300/0 conn(0x7ffbcc06d400 0x7ffbcc0d0070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:51.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcb400640 1 --2- 172.21.15.86:0/823886341 >> v2:172.21.15.194:3300/0 conn(0x7ffbcc06d400 0x7ffbcc0d0070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbc9600640 1 -- 172.21.15.86:0/823886341 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1359692527 0 0) 0x7ffbcc1635a0 con 0x7ffbcc15cc90 2024-03-06T07:32:51.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcb400640 1 -- 172.21.15.86:0/823886341 >> v1:172.21.15.86:6789/0 conn(0x7ffbcc15cc90 legacy=0x7ffbcc162b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:51.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbcb400640 1 -- 172.21.15.86:0/823886341 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffbcc06cee0 con 0x7ffbcc0d05b0 2024-03-06T07:32:51.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.142+0000 7ffbca000640 1 --2- 172.21.15.86:0/823886341 >> v2:172.21.15.194:3300/0 conn(0x7ffbcc06d400 0x7ffbcc0d0070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:51.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.143+0000 7ffbcb400640 1 --2- 172.21.15.86:0/823886341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc0acd60 secure :-1 s=READY pgs=1194 cs=0 l=1 rev1=1 crypto rx=0x7ffbcc0b2a40 tx=0x7ffbc402bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c3b7790df5fe814b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:51.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbc9600640 1 -- 172.21.15.86:0/823886341 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbc402cb20 con 0x7ffbcc0d05b0 2024-03-06T07:32:51.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbc9600640 1 -- 172.21.15.86:0/823886341 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffbc402ccc0 con 0x7ffbcc0d05b0 2024-03-06T07:32:51.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbc9600640 1 -- 172.21.15.86:0/823886341 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbc4034610 con 0x7ffbcc0d05b0 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbd1600640 1 -- 172.21.15.86:0/823886341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc0d05b0 msgr2=0x7ffbcc0acd60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbd1600640 1 --2- 172.21.15.86:0/823886341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc0acd60 secure :-1 s=READY pgs=1194 cs=0 l=1 rev1=1 crypto rx=0x7ffbcc0b2a40 tx=0x7ffbc402bf80 comp rx=0 tx=0).stop 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbd1600640 1 -- 172.21.15.86:0/823886341 shutdown_connections 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbd1600640 1 --2- 172.21.15.86:0/823886341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc0acd60 unknown :-1 s=CLOSED pgs=1194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbd1600640 1 --2- 172.21.15.86:0/823886341 >> v2:172.21.15.194:3300/0 conn(0x7ffbcc06d400 0x7ffbcc0d0070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.144+0000 7ffbd1600640 1 -- 172.21.15.86:0/823886341 >> 172.21.15.86:0/823886341 conn(0x7ffbcc06b360 msgr2=0x7ffbcc06b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.145+0000 7ffbd1600640 1 -- 172.21.15.86:0/823886341 shutdown_connections 2024-03-06T07:32:51.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.145+0000 7ffbd1600640 1 -- 172.21.15.86:0/823886341 wait complete. 2024-03-06T07:32:51.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.146+0000 7ffbd1600640 1 Processor -- start 2024-03-06T07:32:51.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.146+0000 7ffbd1600640 1 -- start start 2024-03-06T07:32:51.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbd1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffbcc06d400 0x7ffbcc151ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:51.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbd1600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc1ebf10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:51.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbd1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc15cc90 0x7ffbcc157020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:51.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbd1600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffbcc0b4e60 con 0x7ffbcc15cc90 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbd1600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffbcc163210 con 0x7ffbcc0d05b0 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbd1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffbcc0b4fe0 con 0x7ffbcc06d400 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbcb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc15cc90 0x7ffbcc157020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbcb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc15cc90 0x7ffbcc157020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49168/0 (socket says 172.21.15.86:49168) 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.147+0000 7ffbcb400640 1 -- 172.21.15.86:0/2835801180 learned_addr learned my addr 172.21.15.86:0/2835801180 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbca000640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc1ebf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbcb400640 1 -- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffbcc06d400 msgr2=0x7ffbcc151ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbcaa00640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffbcc06d400 0x7ffbcc151ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbcb400640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffbcc06d400 0x7ffbcc151ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbcb400640 1 -- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffbcc0d05b0 msgr2=0x7ffbcc1ebf10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbcb400640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc1ebf10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbcb400640 1 -- 172.21.15.86:0/2835801180 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffbcc156260 con 0x7ffbcc15cc90 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbca000640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc1ebf10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbcb400640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc15cc90 0x7ffbcc157020 secure :-1 s=READY pgs=1195 cs=0 l=1 rev1=1 crypto rx=0x7ffbc4036040 tx=0x7ffbc4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.148+0000 7ffbc2a00640 1 -- 172.21.15.86:0/2835801180 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbc4043030 con 0x7ffbcc15cc90 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.149+0000 7ffbc2a00640 1 -- 172.21.15.86:0/2835801180 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffbc403e450 con 0x7ffbcc15cc90 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.149+0000 7ffbc2a00640 1 -- 172.21.15.86:0/2835801180 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbc403e640 con 0x7ffbcc15cc90 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.149+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffbcc156540 con 0x7ffbcc15cc90 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.149+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffbcc156b00 con 0x7ffbcc15cc90 2024-03-06T07:32:51.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.150+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffbcc068070 con 0x7ffbcc15cc90 2024-03-06T07:32:51.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.151+0000 7ffbc2a00640 1 -- 172.21.15.86:0/2835801180 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffbc4046070 con 0x7ffbcc15cc90 2024-03-06T07:32:51.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.152+0000 7ffbc2a00640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffbac06b4d0 0x7ffbac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:51.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.152+0000 7ffbc2a00640 1 -- 172.21.15.86:0/2835801180 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(99..99 src has 1..99) v4 ==== 5967+0+0 (secure 0 0 0) 0x7ffbc40ac7e0 con 0x7ffbcc15cc90 2024-03-06T07:32:51.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.152+0000 7ffbcaa00640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffbac06b4d0 0x7ffbac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:51.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.152+0000 7ffbcaa00640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffbac06b4d0 0x7ffbac06d990 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7ffbbc007fd0 tx=0x7ffbbc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:51.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.156+0000 7ffbc2a00640 1 -- 172.21.15.86:0/2835801180 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffbc404d070 con 0x7ffbcc15cc90 2024-03-06T07:32:51.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.387+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7ffbcc0b16c0 con 0x7ffbcc15cc90 2024-03-06T07:32:51.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.944+0000 7ffbc2a00640 1 -- 172.21.15.86:0/2835801180 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v71) v1 ==== 104+0+0 (secure 0 0 0) 0x7ffbc407b710 con 0x7ffbcc15cc90 2024-03-06T07:32:51.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffbac06b4d0 msgr2=0x7ffbac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:51.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffbac06b4d0 0x7ffbac06d990 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7ffbbc007fd0 tx=0x7ffbbc009040 comp rx=0 tx=0).stop 2024-03-06T07:32:51.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc15cc90 msgr2=0x7ffbcc157020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:51.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc15cc90 0x7ffbcc157020 secure :-1 s=READY pgs=1195 cs=0 l=1 rev1=1 crypto rx=0x7ffbc4036040 tx=0x7ffbc4004a40 comp rx=0 tx=0).stop 2024-03-06T07:32:51.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 shutdown_connections 2024-03-06T07:32:51.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffbac06b4d0 0x7ffbac06d990 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffbcc15cc90 0x7ffbcc157020 unknown :-1 s=CLOSED pgs=1195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffbcc0d05b0 0x7ffbcc1ebf10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 --2- 172.21.15.86:0/2835801180 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffbcc06d400 0x7ffbcc151ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:51.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 >> 172.21.15.86:0/2835801180 conn(0x7ffbcc06b360 msgr2=0x7ffbcc0af7a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:51.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 shutdown_connections 2024-03-06T07:32:51.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:51.948+0000 7ffbd1600640 1 -- 172.21.15.86:0/2835801180 wait complete. 2024-03-06T07:32:51.972 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:32:52.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.174+0000 7f2f29e00640 1 Processor -- start 2024-03-06T07:32:52.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.175+0000 7f2f29e00640 1 -- start start 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.176+0000 7f2f29e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f24147800 0x7f2f240d0070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.176+0000 7f2f29e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2f2415ee40 0x7f2f240d05b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.176+0000 7f2f29e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2f2415f700 con 0x7f2f24147be0 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.176+0000 7f2f29e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2f24160050 con 0x7f2f2415ee40 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.176+0000 7f2f29e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2f2415f420 con 0x7f2f24147800 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.176+0000 7f2f23400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2f2415ee40 0x7f2f240d05b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f1be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2f24147be0 0x7f2f2415e730 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44522/0 (socket says 172.21.15.86:44522) 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f1be00640 1 -- 172.21.15.86:0/487641826 learned_addr learned my addr 172.21.15.86:0/487641826 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f22a00640 1 --2- 172.21.15.86:0/487641826 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f24147800 0x7f2f240d0070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f23400640 1 -- 172.21.15.86:0/487641826 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f24147800 msgr2=0x7f2f240d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:52.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f23400640 1 --2- 172.21.15.86:0/487641826 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f24147800 0x7f2f240d0070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:52.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f23400640 1 -- 172.21.15.86:0/487641826 >> v1:172.21.15.86:6789/0 conn(0x7f2f24147be0 legacy=0x7f2f2415e730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:52.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f23400640 1 -- 172.21.15.86:0/487641826 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f2415fa00 con 0x7f2f2415ee40 2024-03-06T07:32:52.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f22000640 1 -- 172.21.15.86:0/487641826 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1017282785 0 0) 0x7f2f2415f700 con 0x7f2f24147be0 2024-03-06T07:32:52.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.177+0000 7f2f22a00640 1 --2- 172.21.15.86:0/487641826 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f24147800 0x7f2f240d0070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.178+0000 7f2f23400640 1 --2- 172.21.15.86:0/487641826 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f2415ee40 0x7f2f240d05b0 secure :-1 s=READY pgs=1197 cs=0 l=1 rev1=1 crypto rx=0x7f2f1c009d90 tx=0x7f2f1c02beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b6ce34bb5c99f967 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.178+0000 7f2f22000640 1 -- 172.21.15.86:0/487641826 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f1c039070 con 0x7f2f2415ee40 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.178+0000 7f2f22000640 1 -- 172.21.15.86:0/487641826 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f1c02cc60 con 0x7f2f2415ee40 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.178+0000 7f2f22000640 1 -- 172.21.15.86:0/487641826 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f1c034660 con 0x7f2f2415ee40 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 -- 172.21.15.86:0/487641826 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f2415ee40 msgr2=0x7f2f240d05b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 --2- 172.21.15.86:0/487641826 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f2415ee40 0x7f2f240d05b0 secure :-1 s=READY pgs=1197 cs=0 l=1 rev1=1 crypto rx=0x7f2f1c009d90 tx=0x7f2f1c02beb0 comp rx=0 tx=0).stop 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 -- 172.21.15.86:0/487641826 shutdown_connections 2024-03-06T07:32:52.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 --2- 172.21.15.86:0/487641826 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f2415ee40 0x7f2f240d05b0 unknown :-1 s=CLOSED pgs=1197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 --2- 172.21.15.86:0/487641826 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f24147800 0x7f2f240d0070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 -- 172.21.15.86:0/487641826 >> 172.21.15.86:0/487641826 conn(0x7f2f2406b360 msgr2=0x7f2f2406b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 -- 172.21.15.86:0/487641826 shutdown_connections 2024-03-06T07:32:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.179+0000 7f2f29e00640 1 -- 172.21.15.86:0/487641826 wait complete. 2024-03-06T07:32:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.181+0000 7f2f29e00640 1 Processor -- start 2024-03-06T07:32:52.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.181+0000 7f2f29e00640 1 -- start start 2024-03-06T07:32:52.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f29e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f24147800 0x7f2f24136540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:52.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f29e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f24147be0 0x7f2f240b03c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:52.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f29e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2415ee40 0x7f2f240b0900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f29e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f24160660 con 0x7f2f24147be0 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f29e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f241604e0 con 0x7f2f24147800 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f29e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2f241607e0 con 0x7f2f2415ee40 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f24147be0 0x7f2f240b03c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f24147be0 0x7f2f240b03c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49190/0 (socket says 172.21.15.86:49190) 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 -- 172.21.15.86:0/1013676327 learned_addr learned my addr 172.21.15.86:0/1013676327 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 -- 172.21.15.86:0/1013676327 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2415ee40 msgr2=0x7f2f240b0900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2415ee40 0x7f2f240b0900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 -- 172.21.15.86:0/1013676327 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f24147800 msgr2=0x7f2f24136540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f24147800 0x7f2f24136540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.182+0000 7f2f1be00640 1 -- 172.21.15.86:0/1013676327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f241e5920 con 0x7f2f24147be0 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.183+0000 7f2f1be00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f24147be0 0x7f2f240b03c0 secure :-1 s=READY pgs=1198 cs=0 l=1 rev1=1 crypto rx=0x7f2f0c00c4d0 tx=0x7f2f0c00c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.183+0000 7f2f1b400640 1 -- 172.21.15.86:0/1013676327 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f0c018070 con 0x7f2f24147be0 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.183+0000 7f2f1b400640 1 -- 172.21.15.86:0/1013676327 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f0c007500 con 0x7f2f24147be0 2024-03-06T07:32:52.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.183+0000 7f2f1b400640 1 -- 172.21.15.86:0/1013676327 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f0c004a20 con 0x7f2f24147be0 2024-03-06T07:32:52.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.183+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f241e5c00 con 0x7f2f24147be0 2024-03-06T07:32:52.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.183+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f241e61c0 con 0x7f2f24147be0 2024-03-06T07:32:52.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.184+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2f24068070 con 0x7f2f24147be0 2024-03-06T07:32:52.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.185+0000 7f2f1b400640 1 -- 172.21.15.86:0/1013676327 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2f0c004bc0 con 0x7f2f24147be0 2024-03-06T07:32:52.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.186+0000 7f2f1b400640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f0006b4d0 0x7f2f0006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:52.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.186+0000 7f2f1b400640 1 -- 172.21.15.86:0/1013676327 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(99..99 src has 1..99) v4 ==== 5967+0+0 (secure 0 0 0) 0x7f2f0c089620 con 0x7f2f24147be0 2024-03-06T07:32:52.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.186+0000 7f2f22a00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f0006b4d0 0x7f2f0006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:52.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.187+0000 7f2f22a00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f0006b4d0 0x7f2f0006d990 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f2f14006fd0 tx=0x7f2f14008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:52.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.190+0000 7f2f1b400640 1 -- 172.21.15.86:0/1013676327 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2f0c058390 con 0x7f2f24147be0 2024-03-06T07:32:52.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:52.391+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f2f241466e0 con 0x7f2f24147be0 2024-03-06T07:32:53.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.244+0000 7f2f1b400640 1 -- 172.21.15.86:0/1013676327 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v100) v1 ==== 181+0+0 (secure 0 0 0) 0x7f2f0c057c90 con 0x7f2f24147be0 2024-03-06T07:32:53.247 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:32:53.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.247+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f0006b4d0 msgr2=0x7f2f0006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:53.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.247+0000 7f2f29e00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f0006b4d0 0x7f2f0006d990 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f2f14006fd0 tx=0x7f2f14008040 comp rx=0 tx=0).stop 2024-03-06T07:32:53.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f24147be0 msgr2=0x7f2f240b03c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:53.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f24147be0 0x7f2f240b03c0 secure :-1 s=READY pgs=1198 cs=0 l=1 rev1=1 crypto rx=0x7f2f0c00c4d0 tx=0x7f2f0c00c990 comp rx=0 tx=0).stop 2024-03-06T07:32:53.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 shutdown_connections 2024-03-06T07:32:53.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2f0006b4d0 0x7f2f0006d990 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2f2415ee40 0x7f2f240b0900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2f24147be0 0x7f2f240b03c0 unknown :-1 s=CLOSED pgs=1198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 --2- 172.21.15.86:0/1013676327 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2f24147800 0x7f2f24136540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 >> 172.21.15.86:0/1013676327 conn(0x7f2f2406b360 msgr2=0x7f2f24146ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:53.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 shutdown_connections 2024-03-06T07:32:53.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.248+0000 7f2f29e00640 1 -- 172.21.15.86:0/1013676327 wait complete. 2024-03-06T07:32:53.281 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:32:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.465+0000 7f0ca0c00640 1 Processor -- start 2024-03-06T07:32:53.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.465+0000 7f0ca0c00640 1 -- start start 2024-03-06T07:32:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.466+0000 7f0ca0c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 0x7f0c9c0ae630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.466+0000 7f0ca0c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0c9c15f320 0x7f0c9c0d04f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.466+0000 7f0ca0c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0c9c155870 con 0x7f0c9c0d0070 2024-03-06T07:32:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.466+0000 7f0ca0c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0c9c0ad640 con 0x7f0c9c15f320 2024-03-06T07:32:53.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.466+0000 7f0ca0c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c15f980 con 0x7f0c9c155490 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c91400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0c9c0d0070 0x7f0c9c15ec10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44530/0 (socket says 172.21.15.86:44530) 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c91400640 1 -- 172.21.15.86:0/3178700889 learned_addr learned my addr 172.21.15.86:0/3178700889 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1011099318 0 0) 0x7f0c9c155870 con 0x7f0c9c0d0070 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0c80003520 con 0x7f0c9c0d0070 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c9a800640 1 --2- 172.21.15.86:0/3178700889 >> v2:172.21.15.194:3300/0 conn(0x7f0c9c15f320 0x7f0c9c0d04f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99e00640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 0x7f0c9c0ae630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1151909909 0 0) 0x7f0c80003520 con 0x7f0c9c0d0070 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 msgr2=0x7f0c9c0ae630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c9c155490 0x7f0c9c0ae630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 >> v2:172.21.15.194:3300/0 conn(0x7f0c9c15f320 msgr2=0x7f0c9c0d04f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:53.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 --2- 172.21.15.86:0/3178700889 >> v2:172.21.15.194:3300/0 conn(0x7f0c9c15f320 0x7f0c9c0d04f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.467+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c9c0ad870 con 0x7f0c9c0d0070 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f0c84001670 con 0x7f0c9c0d0070 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c9a800640 1 --2- 172.21.15.86:0/3178700889 >> v2:172.21.15.194:3300/0 conn(0x7f0c9c15f320 0x7f0c9c0d04f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 >> v1:172.21.15.86:6789/0 conn(0x7f0c9c0d0070 legacy=0x7f0c9c15ec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c800039b0 0x7f0c80003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c80004800 0x7f0c80004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c800051c0 0x7f0c8000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c15f980 con 0x7f0c80004800 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c8000bfe0 con 0x7f0c800051c0 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0c8000c1e0 con 0x7f0c800039b0 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c91400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c80004800 0x7f0c80004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:53.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c9a800640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c800051c0 0x7f0c8000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c91400640 1 -- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c800039b0 msgr2=0x7f0c80003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c91400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c800039b0 0x7f0c80003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c91400640 1 -- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c800051c0 msgr2=0x7f0c8000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c91400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c800051c0 0x7f0c8000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.468+0000 7f0c91400640 1 -- 172.21.15.86:0/3178700889 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c80003520 con 0x7f0c80004800 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.469+0000 7f0c91400640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c80004800 0x7f0c80004c80 secure :-1 s=READY pgs=1200 cs=0 l=1 rev1=1 crypto rx=0x7f0c8400c190 tx=0x7f0c80005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7c98bca6acb1c553 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c84007c60 con 0x7f0c80004800 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0c84030070 con 0x7f0c80004800 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0c99400640 1 -- 172.21.15.86:0/3178700889 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c84005d70 con 0x7f0c80004800 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0ca0c00640 1 -- 172.21.15.86:0/3178700889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c80004800 msgr2=0x7f0c80004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/3178700889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c80004800 0x7f0c80004c80 secure :-1 s=READY pgs=1200 cs=0 l=1 rev1=1 crypto rx=0x7f0c8400c190 tx=0x7f0c80005830 comp rx=0 tx=0).stop 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0c9a800640 1 -- 172.21.15.86:0/3178700889 reap_dead start 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0ca0c00640 1 -- 172.21.15.86:0/3178700889 shutdown_connections 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0ca0c00640 1 -- 172.21.15.86:0/3178700889 >> 172.21.15.86:0/3178700889 conn(0x7f0c9c06b360 msgr2=0x7f0c9c06b770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:53.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0ca0c00640 1 -- 172.21.15.86:0/3178700889 shutdown_connections 2024-03-06T07:32:53.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.470+0000 7f0ca0c00640 1 -- 172.21.15.86:0/3178700889 wait complete. 2024-03-06T07:32:53.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.472+0000 7f0ca0c00640 1 Processor -- start 2024-03-06T07:32:53.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.472+0000 7f0ca0c00640 1 -- start start 2024-03-06T07:32:53.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0ca0c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c80004800 0x7f0c9c14aaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0ca0c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c0b05d0 0x7f0c9c138680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0ca0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c148f30 0x7f0c9c14eb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0ca0c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c0ae110 con 0x7f0c9c148f30 2024-03-06T07:32:53.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0ca0c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c0ade30 con 0x7f0c9c0b05d0 2024-03-06T07:32:53.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0ca0c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0c9c0ae630 con 0x7f0c80004800 2024-03-06T07:32:53.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0c9a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c148f30 0x7f0c9c14eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0c9a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c148f30 0x7f0c9c14eb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49210/0 (socket says 172.21.15.86:49210) 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.473+0000 7f0c9a800640 1 -- 172.21.15.86:0/702698931 learned_addr learned my addr 172.21.15.86:0/702698931 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c9a800640 1 -- 172.21.15.86:0/702698931 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c80004800 msgr2=0x7f0c9c14aaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c9a800640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c80004800 0x7f0c9c14aaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c9a800640 1 -- 172.21.15.86:0/702698931 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c0b05d0 msgr2=0x7f0c9c138680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c9a800640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c0b05d0 0x7f0c9c138680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c9a800640 1 -- 172.21.15.86:0/702698931 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c9c0af810 con 0x7f0c9c148f30 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c9a800640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c148f30 0x7f0c9c14eb00 secure :-1 s=READY pgs=1201 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c071710 tx=0x7f0c940046e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c92a00640 1 -- 172.21.15.86:0/702698931 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c94007cf0 con 0x7f0c9c148f30 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c92a00640 1 -- 172.21.15.86:0/702698931 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0c94019070 con 0x7f0c9c148f30 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0c92a00640 1 -- 172.21.15.86:0/702698931 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c94014630 con 0x7f0c9c148f30 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.474+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c9c0afaf0 con 0x7f0c9c148f30 2024-03-06T07:32:53.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.475+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c9c0b00b0 con 0x7f0c9c148f30 2024-03-06T07:32:53.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.476+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c9c0b0a10 con 0x7f0c9c148f30 2024-03-06T07:32:53.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.476+0000 7f0c92a00640 1 -- 172.21.15.86:0/702698931 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0c940147d0 con 0x7f0c9c148f30 2024-03-06T07:32:53.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.478+0000 7f0c92a00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7806b4d0 0x7f0c7806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:53.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.478+0000 7f0c92a00640 1 -- 172.21.15.86:0/702698931 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 5529+0+0 (secure 0 0 0) 0x7f0c9408a730 con 0x7f0c9c148f30 2024-03-06T07:32:53.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.478+0000 7f0c99e00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7806b4d0 0x7f0c7806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:53.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.479+0000 7f0c99e00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7806b4d0 0x7f0c7806d990 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c06ed50 tx=0x7f0c8c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:53.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.483+0000 7f0c92a00640 1 -- 172.21.15.86:0/702698931 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0c9c0b0a10 con 0x7f0c9c148f30 2024-03-06T07:32:53.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:53.681+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f0c9c068070 con 0x7f0c9c148f30 2024-03-06T07:32:54.039 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:32:54.047 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:32:54.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.248+0000 7f0c92a00640 1 -- 172.21.15.86:0/702698931 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v101) v1 ==== 169+0+0 (secure 0 0 0) 0x7f0c94059650 con 0x7f0c9c148f30 2024-03-06T07:32:54.250 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:32:54.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7806b4d0 msgr2=0x7f0c7806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:54.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7806b4d0 0x7f0c7806d990 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c06ed50 tx=0x7f0c8c008040 comp rx=0 tx=0).stop 2024-03-06T07:32:54.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c148f30 msgr2=0x7f0c9c14eb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:54.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c148f30 0x7f0c9c14eb00 secure :-1 s=READY pgs=1201 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c071710 tx=0x7f0c940046e0 comp rx=0 tx=0).stop 2024-03-06T07:32:54.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 shutdown_connections 2024-03-06T07:32:54.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0c7806b4d0 0x7f0c7806d990 unknown :-1 s=CLOSED pgs=521 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:54.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0c9c148f30 0x7f0c9c14eb00 unknown :-1 s=CLOSED pgs=1201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:54.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0c9c0b05d0 0x7f0c9c138680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:54.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 --2- 172.21.15.86:0/702698931 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0c80004800 0x7f0c9c14aaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:54.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.255+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 >> 172.21.15.86:0/702698931 conn(0x7f0c9c06b360 msgr2=0x7f0c9c0b2fb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:54.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.256+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 shutdown_connections 2024-03-06T07:32:54.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:54.256+0000 7f0ca0c00640 1 -- 172.21.15.86:0/702698931 wait complete. 2024-03-06T07:32:54.282 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:54.288 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: chdir = "" 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: auth supported = cephx 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: filestore xattr use omap = true 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: auth debug = true 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.427 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on old message = true 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on bug = true 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default size = 2 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow pg remap = true 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:54.428 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on too few osds = false 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: osd default data pool replay window = 5 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon allow pool delete = true 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: mon cluster log file level = debug 2024-03-06T07:32:54.429 INFO:teuthology.orchestra.run.smithi086.stdout: debug asserts on shutdown = true 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: mon health detail to clog = false 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on skipped message = False 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout:[osd] 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: osd journal size = 100 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub max interval = 600 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: osd recover clone overlap = true 2024-03-06T07:32:54.430 INFO:teuthology.orchestra.run.smithi086.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug shutdown = true 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug op order = true 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug trim objects = true 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd open classes on start = true 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug pg log writeout = true 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: osd map max advance = 10 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: journal zero on create = true 2024-03-06T07:32:54.431 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: bdev debug aio = true 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug misdirected ops = true 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: bdev async discard = True 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: bdev enable discard = True 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore allocator = bitmap 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore block size = 96636764160 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore fsck on mount = True 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluefs = 1/20 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluestore = 1/20 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.432 INFO:teuthology.orchestra.run.smithi086.stdout: debug osd = 20 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: debug rocksdb = 4/10 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: osd objectstore = bluestore 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr] 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: debug mgr = 20 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.433 INFO:teuthology.orchestra.run.smithi086.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout:[mon] 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: debug paxos = 20 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: mon data avail warn = 5 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd prime pg temp = true 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:54.434 INFO:teuthology.orchestra.run.smithi086.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: # 1m isn't quite enough 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: mon op complaint time = 120 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout:[client] 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: rgw cache enabled = true 2024-03-06T07:32:54.435 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable ops log = true 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable usage log = true 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: client mount timeout = 600 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: debug client = 20 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = "" 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.a] 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.b] 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.c] 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:32:54.436 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout:[mds] 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds = 20 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds balancer = 20 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug frag = True 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug scatterstat = True 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: mds op complaint time = 180 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: mds verify scatter = True 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_files = 6 2024-03-06T07:32:54.437 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:54.453 INFO:teuthology.orchestra.run.smithi194.stdout:[global] 2024-03-06T07:32:54.453 INFO:teuthology.orchestra.run.smithi194.stdout: chdir = "" 2024-03-06T07:32:54.453 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:54.453 INFO:teuthology.orchestra.run.smithi194.stdout: auth supported = cephx 2024-03-06T07:32:54.453 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.453 INFO:teuthology.orchestra.run.smithi194.stdout: filestore xattr use omap = true 2024-03-06T07:32:54.453 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: auth debug = true 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on old message = true 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on bug = true 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:54.454 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default size = 2 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow pg remap = true 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on too few osds = false 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:54.455 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: osd default data pool replay window = 5 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: mon allow pool delete = true 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: mon cluster log file level = debug 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: debug asserts on shutdown = true 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: mon health detail to clog = false 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on skipped message = False 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout:[osd] 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: osd journal size = 100 2024-03-06T07:32:54.456 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub max interval = 600 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd recover clone overlap = true 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug shutdown = true 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug op order = true 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug trim objects = true 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd open classes on start = true 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug pg log writeout = true 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.457 INFO:teuthology.orchestra.run.smithi194.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: osd map max advance = 10 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: journal zero on create = true 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: bdev debug aio = true 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug misdirected ops = true 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: bdev async discard = True 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: bdev enable discard = True 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore allocator = bitmap 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore block size = 96636764160 2024-03-06T07:32:54.458 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore fsck on mount = True 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluefs = 1/20 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluestore = 1/20 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug osd = 20 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug rocksdb = 4/10 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: osd objectstore = bluestore 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr] 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug mgr = 20 2024-03-06T07:32:54.459 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout:[mon] 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: debug paxos = 20 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mon data avail warn = 5 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:54.460 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd prime pg temp = true 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: # 1m isn't quite enough 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:54.461 INFO:teuthology.orchestra.run.smithi194.stdout: mon op complaint time = 120 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout:[client] 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: rgw cache enabled = true 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable ops log = true 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable usage log = true 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: client mount timeout = 600 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: debug client = 20 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = "" 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.a] 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.b] 2024-03-06T07:32:54.462 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.c] 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout:[mds] 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds = 20 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds balancer = 20 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug frag = True 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug scatterstat = True 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: mds op complaint time = 180 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: mds verify scatter = True 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.463 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.464 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_files = 6 2024-03-06T07:32:54.464 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:54.497 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:54.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:54.670 INFO:teuthology.orchestra.run.smithi194.stdout:[global] 2024-03-06T07:32:54.670 INFO:teuthology.orchestra.run.smithi194.stdout: chdir = "" 2024-03-06T07:32:54.670 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:54.670 INFO:teuthology.orchestra.run.smithi194.stdout: auth supported = cephx 2024-03-06T07:32:54.670 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.670 INFO:teuthology.orchestra.run.smithi194.stdout: filestore xattr use omap = true 2024-03-06T07:32:54.670 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: auth debug = true 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on old message = true 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on bug = true 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:54.671 INFO:teuthology.orchestra.run.smithi194.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default size = 2 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow pg remap = true 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on too few osds = false 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:54.672 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: osd default data pool replay window = 5 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: mon allow pool delete = true 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: mon cluster log file level = debug 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: debug asserts on shutdown = true 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: mon health detail to clog = false 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:54.673 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on skipped message = False 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout:[osd] 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: osd journal size = 100 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub max interval = 600 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:54.674 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd recover clone overlap = true 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug shutdown = true 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug op order = true 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug trim objects = true 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd open classes on start = true 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug pg log writeout = true 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:54.675 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: osd map max advance = 10 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: journal zero on create = true 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: bdev debug aio = true 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug misdirected ops = true 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: bdev async discard = True 2024-03-06T07:32:54.676 INFO:teuthology.orchestra.run.smithi194.stdout: bdev enable discard = True 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore allocator = bitmap 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore block size = 96636764160 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore fsck on mount = True 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluefs = 1/20 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluestore = 1/20 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: debug osd = 20 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: debug rocksdb = 4/10 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: osd objectstore = bluestore 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:54.677 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr] 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug mgr = 20 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout:[mon] 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug paxos = 20 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:54.678 INFO:teuthology.orchestra.run.smithi194.stdout: mon data avail warn = 5 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd prime pg temp = true 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: # 1m isn't quite enough 2024-03-06T07:32:54.679 INFO:teuthology.orchestra.run.smithi194.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: mon op complaint time = 120 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout:[client] 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: rgw cache enabled = true 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable ops log = true 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable usage log = true 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:54.680 INFO:teuthology.orchestra.run.smithi194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout: client mount timeout = 600 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout: debug client = 20 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = "" 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.a] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.b] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.c] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout:[mds] 2024-03-06T07:32:54.681 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds = 20 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds balancer = 20 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug frag = True 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug scatterstat = True 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: mds op complaint time = 180 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: mds verify scatter = True 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.682 INFO:teuthology.orchestra.run.smithi194.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: chdir = "" 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: auth supported = cephx 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: filestore xattr use omap = true 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.683 INFO:teuthology.orchestra.run.smithi086.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: auth debug = true 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on old message = true 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on bug = true 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default size = 2 2024-03-06T07:32:54.684 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow pg remap = true 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on too few osds = false 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: osd default data pool replay window = 5 2024-03-06T07:32:54.685 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: mon allow pool delete = true 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: mon cluster log file level = debug 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: debug asserts on shutdown = true 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: mon health detail to clog = false 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on skipped message = False 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout:[osd] 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: osd journal size = 100 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub max interval = 600 2024-03-06T07:32:54.686 INFO:teuthology.orchestra.run.smithi086.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd recover clone overlap = true 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug shutdown = true 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug op order = true 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug trim objects = true 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd open classes on start = true 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug pg log writeout = true 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.687 INFO:teuthology.orchestra.run.smithi086.stdout: osd map max advance = 10 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: journal zero on create = true 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: bdev debug aio = true 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug misdirected ops = true 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: bdev async discard = True 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: bdev enable discard = True 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore allocator = bitmap 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore block size = 96636764160 2024-03-06T07:32:54.688 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore fsck on mount = True 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluefs = 1/20 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluestore = 1/20 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug osd = 20 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug rocksdb = 4/10 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: osd objectstore = bluestore 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr] 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug mgr = 20 2024-03-06T07:32:54.689 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout:[mon] 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: debug paxos = 20 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon data avail warn = 5 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd prime pg temp = true 2024-03-06T07:32:54.690 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: # 1m isn't quite enough 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:54.691 INFO:teuthology.orchestra.run.smithi086.stdout: mon op complaint time = 120 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout:[client] 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: rgw cache enabled = true 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable ops log = true 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable usage log = true 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: client mount timeout = 600 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: debug client = 20 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = "" 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.a] 2024-03-06T07:32:54.692 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.b] 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.c] 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout:[mds] 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds = 20 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds balancer = 20 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug frag = True 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug scatterstat = True 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: mds op complaint time = 180 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: mds verify scatter = True 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.693 INFO:teuthology.orchestra.run.smithi086.stdout: mds_max_purge_ops = 100000000 2024-03-06T07:32:54.717 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:54.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-06T07:32:54.886 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-03-06T07:32:54.886 INFO:teuthology.orchestra.run.smithi086.stdout: chdir = "" 2024-03-06T07:32:54.886 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:54.886 INFO:teuthology.orchestra.run.smithi086.stdout: auth supported = cephx 2024-03-06T07:32:54.886 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.886 INFO:teuthology.orchestra.run.smithi086.stdout: filestore xattr use omap = true 2024-03-06T07:32:54.886 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: auth debug = true 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on old message = true 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on bug = true 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default size = 2 2024-03-06T07:32:54.887 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd allow pg remap = true 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon warn on too few osds = false 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:54.888 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: osd default data pool replay window = 5 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: mon allow pool delete = true 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: mon cluster log file level = debug 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: debug asserts on shutdown = true 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: mon health detail to clog = false 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: ms die on skipped message = False 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout:[osd] 2024-03-06T07:32:54.889 INFO:teuthology.orchestra.run.smithi086.stdout: osd journal size = 100 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd scrub max interval = 600 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd recover clone overlap = true 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug shutdown = true 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug op order = true 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug trim objects = true 2024-03-06T07:32:54.890 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: osd open classes on start = true 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug pg log writeout = true 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: osd map max advance = 10 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: journal zero on create = true 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: bdev debug aio = true 2024-03-06T07:32:54.891 INFO:teuthology.orchestra.run.smithi086.stdout: osd debug misdirected ops = true 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: bdev async discard = True 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: bdev enable discard = True 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore allocator = bitmap 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore block size = 96636764160 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: bluestore fsck on mount = True 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluefs = 1/20 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: debug bluestore = 1/20 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: debug osd = 20 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: debug rocksdb = 4/10 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:54.892 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: osd objectstore = bluestore 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout:[mgr] 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: debug mgr = 20 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout:[mon] 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.893 INFO:teuthology.orchestra.run.smithi086.stdout: debug mon = 20 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: debug paxos = 20 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: debug auth = 20 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon data avail warn = 5 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon osd prime pg temp = true 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:54.894 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: # 1m isn't quite enough 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: mon op complaint time = 120 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout:[client] 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: rgw cache enabled = true 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable ops log = true 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: rgw enable usage log = true 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: client mount timeout = 600 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: debug client = 20 2024-03-06T07:32:54.895 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout: pid file = "" 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.a] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.b] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mon.c] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.a] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.c] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.b] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mds.d] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout:[mds] 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds = 20 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout: debug mds balancer = 20 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout: debug ms = 1 2024-03-06T07:32:54.896 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug frag = True 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi086.stdout: mds debug scatterstat = True 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi086.stdout: mds op complaint time = 180 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi086.stdout: mds verify scatter = True 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi086.stdout: osd op complaint time = 180 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi086.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi086.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi194.stdout:[global] 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi194.stdout: chdir = "" 2024-03-06T07:32:54.897 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: auth supported = cephx 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: filestore xattr use omap = true 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: mon clock drift allowed = 1.000 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: osd crush chooseleaf type = 0 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: auth debug = true 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on old message = true 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on bug = true 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-06T07:32:54.898 INFO:teuthology.orchestra.run.smithi194.stdout: mon pg warn max object skew = 0 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: # disable pg_autoscaler by default for new pools 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default size = 2 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow primary affinity = true 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd allow pg remap = true 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on legacy crush tunables = false 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on crush straw calc version zero = false 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on no sortbitwise = false 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on osd down out interval zero = false 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon warn on too few osds = false 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-06T07:32:54.899 INFO:teuthology.orchestra.run.smithi194.stdout: mon_allow_pool_size_one = true 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: osd default data pool replay window = 5 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: mon allow pool delete = true 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: mon cluster log file level = debug 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: debug asserts on shutdown = true 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: mon health detail to clog = false 2024-03-06T07:32:54.900 INFO:teuthology.orchestra.run.smithi194.stdout: mon host = "172.21.15.86,172.21.15.194,[v2:172.21.15.194:3301,v1:172.21.15.194:6790]" 2024-03-06T07:32:54.901 INFO:teuthology.orchestra.run.smithi194.stdout: ms die on skipped message = False 2024-03-06T07:32:54.901 INFO:teuthology.orchestra.run.smithi194.stdout: fsid = 013155b5-b812-4a77-8af3-f8c90b517e11 2024-03-06T07:32:54.901 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.901 INFO:teuthology.orchestra.run.smithi194.stdout:[osd] 2024-03-06T07:32:54.901 INFO:teuthology.orchestra.run.smithi194.stdout: osd journal size = 100 2024-03-06T07:32:54.901 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.901 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub load threshold = 5.0 2024-03-06T07:32:54.902 INFO:teuthology.orchestra.run.smithi194.stdout: osd scrub max interval = 600 2024-03-06T07:32:54.902 INFO:teuthology.orchestra.run.smithi194.stdout: osd mclock profile = high_recovery_ops 2024-03-06T07:32:54.902 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.902 INFO:teuthology.orchestra.run.smithi194.stdout: osd recover clone overlap = true 2024-03-06T07:32:54.902 INFO:teuthology.orchestra.run.smithi194.stdout: osd recovery max chunk = 1048576 2024-03-06T07:32:54.902 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.903 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug shutdown = true 2024-03-06T07:32:54.903 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug op order = true 2024-03-06T07:32:54.903 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug verify stray on activate = true 2024-03-06T07:32:54.903 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug trim objects = true 2024-03-06T07:32:54.903 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.903 INFO:teuthology.orchestra.run.smithi194.stdout: osd open classes on start = true 2024-03-06T07:32:54.904 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug pg log writeout = true 2024-03-06T07:32:54.904 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.904 INFO:teuthology.orchestra.run.smithi194.stdout: osd deep scrub update digest min age = 30 2024-03-06T07:32:54.904 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.904 INFO:teuthology.orchestra.run.smithi194.stdout: osd map max advance = 10 2024-03-06T07:32:54.904 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.905 INFO:teuthology.orchestra.run.smithi194.stdout: journal zero on create = true 2024-03-06T07:32:54.905 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.905 INFO:teuthology.orchestra.run.smithi194.stdout: filestore ondisk finisher threads = 3 2024-03-06T07:32:54.905 INFO:teuthology.orchestra.run.smithi194.stdout: filestore apply finisher threads = 3 2024-03-06T07:32:54.905 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.905 INFO:teuthology.orchestra.run.smithi194.stdout: bdev debug aio = true 2024-03-06T07:32:54.906 INFO:teuthology.orchestra.run.smithi194.stdout: osd debug misdirected ops = true 2024-03-06T07:32:54.906 INFO:teuthology.orchestra.run.smithi194.stdout: bdev async discard = True 2024-03-06T07:32:54.906 INFO:teuthology.orchestra.run.smithi194.stdout: bdev enable discard = True 2024-03-06T07:32:54.906 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore allocator = bitmap 2024-03-06T07:32:54.906 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore block size = 96636764160 2024-03-06T07:32:54.906 INFO:teuthology.orchestra.run.smithi194.stdout: bluestore fsck on mount = True 2024-03-06T07:32:54.906 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluefs = 1/20 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug bluestore = 1/20 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug osd = 20 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug rocksdb = 4/10 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd backfillfull_ratio = 0.85 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd full ratio = 0.9 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd nearfull ratio = 0.8 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: osd failsafe full ratio = 0.95 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: osd objectstore = bluestore 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout:[mgr] 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug mgr = 20 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:54.907 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mgr/telemetry/nag = false 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout:[mon] 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: debug mon = 20 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: debug paxos = 20 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: debug auth = 20 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon data avail warn = 5 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon mgr mkfs grace = 240 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min pgs per osd = 4 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd reporter subtree level = osd 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon osd prime pg temp = true 2024-03-06T07:32:54.908 INFO:teuthology.orchestra.run.smithi194.stdout: mon reweight min bytes per osd = 10 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: auth mon ticket ttl = 660 # 11m 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: auth service ticket ttl = 240 # 4m 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: # don't complain about insecure global_id in the test suite 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: # 1m isn't quite enough 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: mon_down_mkfs_grace = 2m 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: mon_warn_on_filestore_osds = false 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: mon op complaint time = 120 2024-03-06T07:32:54.909 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout:[client] 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: rgw cache enabled = true 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable ops log = true 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: rgw enable usage log = true 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: client mount timeout = 600 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: debug client = 20 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: pid file = "" 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.910 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.a] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.b] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mon.c] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.a] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.c] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.b] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mds.d] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout:[mds] 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds = 20 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: debug mds balancer = 20 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: debug ms = 1 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug frag = True 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: mds debug scatterstat = True 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: mds op complaint time = 180 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: mds verify scatter = True 2024-03-06T07:32:54.911 INFO:teuthology.orchestra.run.smithi194.stdout: osd op complaint time = 180 2024-03-06T07:32:54.912 INFO:teuthology.orchestra.run.smithi194.stdout: rados mon op timeout = 900 2024-03-06T07:32:54.912 INFO:teuthology.orchestra.run.smithi194.stdout: rados osd op timeout = 900 2024-03-06T07:32:54.936 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-06T07:32:55.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.149+0000 7f15b3200640 1 Processor -- start 2024-03-06T07:32:55.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.150+0000 7f15b3200640 1 -- start start 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15b3200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f15ac0d07f0 0x7f15ac0d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15b3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac15e450 0x7f15ac0b4480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15b3200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f15ac0b3730 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15b3200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f15ac0d4930 con 0x7f15ac0d07f0 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15b3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f15ac067b20 con 0x7f15ac15e450 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15abe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f15ac0d07f0 0x7f15ac0d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15b1200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f15ac0d1bf0 0x7f15ac0b49c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:44534/0 (socket says 172.21.15.86:44534) 2024-03-06T07:32:55.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.151+0000 7f15b1200640 1 -- 172.21.15.86:0/234188936 learned_addr learned my addr 172.21.15.86:0/234188936 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:55.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.152+0000 7f15abe00640 1 -- 172.21.15.86:0/234188936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac15e450 msgr2=0x7f15ac0b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:55.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.152+0000 7f15ab400640 1 --2- 172.21.15.86:0/234188936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac15e450 0x7f15ac0b4480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.152+0000 7f15abe00640 1 --2- 172.21.15.86:0/234188936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac15e450 0x7f15ac0b4480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.152+0000 7f15abe00640 1 -- 172.21.15.86:0/234188936 >> v1:172.21.15.86:6789/0 conn(0x7f15ac0d1bf0 legacy=0x7f15ac0b49c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.152+0000 7f15abe00640 1 -- 172.21.15.86:0/234188936 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15ac0d1110 con 0x7f15ac0d07f0 2024-03-06T07:32:55.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.152+0000 7f15aaa00640 1 -- 172.21.15.86:0/234188936 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2119473071 0 0) 0x7f15ac0b3730 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.152+0000 7f15ab400640 1 --2- 172.21.15.86:0/234188936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac15e450 0x7f15ac0b4480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:32:55.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.153+0000 7f15abe00640 1 --2- 172.21.15.86:0/234188936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d07f0 0x7f15ac0d0bd0 secure :-1 s=READY pgs=1203 cs=0 l=1 rev1=1 crypto rx=0x7f159c009d90 tx=0x7f159c02be10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f9f2010d16bd05 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.153+0000 7f15aaa00640 1 -- 172.21.15.86:0/234188936 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f159c039070 con 0x7f15ac0d07f0 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.153+0000 7f15aaa00640 1 -- 172.21.15.86:0/234188936 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f159c02cbc0 con 0x7f15ac0d07f0 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.153+0000 7f15aaa00640 1 -- 172.21.15.86:0/234188936 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f159c034440 con 0x7f15ac0d07f0 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 -- 172.21.15.86:0/234188936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d07f0 msgr2=0x7f15ac0d0bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 --2- 172.21.15.86:0/234188936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d07f0 0x7f15ac0d0bd0 secure :-1 s=READY pgs=1203 cs=0 l=1 rev1=1 crypto rx=0x7f159c009d90 tx=0x7f159c02be10 comp rx=0 tx=0).stop 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 -- 172.21.15.86:0/234188936 shutdown_connections 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 --2- 172.21.15.86:0/234188936 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac15e450 0x7f15ac0b4480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 --2- 172.21.15.86:0/234188936 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d07f0 0x7f15ac0d0bd0 unknown :-1 s=CLOSED pgs=1203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 -- 172.21.15.86:0/234188936 >> 172.21.15.86:0/234188936 conn(0x7f15ac0d3e10 msgr2=0x7f15ac0d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 -- 172.21.15.86:0/234188936 shutdown_connections 2024-03-06T07:32:55.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.154+0000 7f15b3200640 1 -- 172.21.15.86:0/234188936 wait complete. 2024-03-06T07:32:55.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.156+0000 7f15b3200640 1 Processor -- start 2024-03-06T07:32:55.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.156+0000 7f15b3200640 1 -- start start 2024-03-06T07:32:55.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15b3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac0d07f0 0x7f15ac15c200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15b3200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d1bf0 0x7f15ac0ccab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15b3200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f15ac15e450 0x7f15ac09f120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15b3200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15ac0b3c80 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15b3200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15ac0b39a0 con 0x7f15ac15e450 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15b3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f15ac0b3f60 con 0x7f15ac0d07f0 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d1bf0 0x7f15ac0ccab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d1bf0 0x7f15ac0ccab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49226/0 (socket says 172.21.15.86:49226) 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 -- 172.21.15.86:0/4095595105 learned_addr learned my addr 172.21.15.86:0/4095595105 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15abe00640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac0d07f0 0x7f15ac15c200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 -- 172.21.15.86:0/4095595105 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac0d07f0 msgr2=0x7f15ac15c200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac0d07f0 0x7f15ac15c200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 -- 172.21.15.86:0/4095595105 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f15ac15e450 msgr2=0x7f15ac09f120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f15ac15e450 0x7f15ac09f120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.157+0000 7f15ab400640 1 -- 172.21.15.86:0/4095595105 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15ac0b9a00 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.158+0000 7f15ab400640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d1bf0 0x7f15ac0ccab0 secure :-1 s=READY pgs=1204 cs=0 l=1 rev1=1 crypto rx=0x7f159800c7f0 tx=0x7f159800ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:55.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.158+0000 7f15a3e00640 1 -- 172.21.15.86:0/4095595105 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1598015070 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.158+0000 7f15a3e00640 1 -- 172.21.15.86:0/4095595105 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1598007820 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.158+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f15ac0cbfd0 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.158+0000 7f15a3e00640 1 -- 172.21.15.86:0/4095595105 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1598007c30 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.158+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15ac0cc590 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.159+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f15ac04f330 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.160+0000 7f15a3e00640 1 -- 172.21.15.86:0/4095595105 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1598018070 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.161+0000 7f15a3e00640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f158c06b4d0 0x7f158c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.161+0000 7f15a3e00640 1 -- 172.21.15.86:0/4095595105 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f159808a0c0 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.161+0000 7f15abe00640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f158c06b4d0 0x7f158c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.162+0000 7f15abe00640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f158c06b4d0 0x7f158c06d990 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f159c036000 tx=0x7f159c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:55.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.165+0000 7f15a3e00640 1 -- 172.21.15.86:0/4095595105 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1598059180 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.368+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f15ac0d21d0 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.374+0000 7f15a3e00640 1 -- 172.21.15.86:0/4095595105 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v29) v1 ==== 104+0+0 (secure 0 0 0) 0x7f1598058a80 con 0x7f15ac0d1bf0 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.376+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f158c06b4d0 msgr2=0x7f158c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.376+0000 7f15b3200640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f158c06b4d0 0x7f158c06d990 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f159c036000 tx=0x7f159c030000 comp rx=0 tx=0).stop 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.376+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d1bf0 msgr2=0x7f15ac0ccab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.376+0000 7f15b3200640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d1bf0 0x7f15ac0ccab0 secure :-1 s=READY pgs=1204 cs=0 l=1 rev1=1 crypto rx=0x7f159800c7f0 tx=0x7f159800ccb0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 shutdown_connections 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f158c06b4d0 0x7f158c06d990 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f15ac15e450 0x7f15ac09f120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.379 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f15ac0d1bf0 0x7f15ac0ccab0 unknown :-1 s=CLOSED pgs=1204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 --2- 172.21.15.86:0/4095595105 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f15ac0d07f0 0x7f15ac15c200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 >> 172.21.15.86:0/4095595105 conn(0x7f15ac0d3e10 msgr2=0x7f15ac0d41f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 shutdown_connections 2024-03-06T07:32:55.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.377+0000 7f15b3200640 1 -- 172.21.15.86:0/4095595105 wait complete. 2024-03-06T07:32:55.395 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_files_throttle' 2024-03-06T07:32:55.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.609+0000 7f0a55800640 1 Processor -- start 2024-03-06T07:32:55.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.609+0000 7f0a55800640 1 -- start start 2024-03-06T07:32:55.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.610+0000 7f0a55800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0a5015acb0 0x7f0a500b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.610+0000 7f0a55800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0a5006ec00 0x7f0a500ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.610+0000 7f0a55800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0a5015b090 con 0x7f0a500b4680 2024-03-06T07:32:55.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.610+0000 7f0a55800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a500b2a30 con 0x7f0a5015acb0 2024-03-06T07:32:55.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.610+0000 7f0a55800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a500b4360 con 0x7f0a5006ec00 2024-03-06T07:32:55.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.611+0000 7f0a4ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0a5015acb0 0x7f0a500b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.611+0000 7f0a4ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0a5015acb0 0x7f0a500b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49232/0 (socket says 172.21.15.86:49232) 2024-03-06T07:32:55.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.611+0000 7f0a4ea00640 1 -- 172.21.15.86:0/2151156082 learned_addr learned my addr 172.21.15.86:0/2151156082 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:55.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.611+0000 7f0a4ea00640 1 -- 172.21.15.86:0/2151156082 >> v2:172.21.15.194:3300/0 conn(0x7f0a5006ec00 msgr2=0x7f0a500ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:55.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.611+0000 7f0a4ea00640 1 --2- 172.21.15.86:0/2151156082 >> v2:172.21.15.194:3300/0 conn(0x7f0a5006ec00 0x7f0a500ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.611+0000 7f0a4ea00640 1 -- 172.21.15.86:0/2151156082 >> v1:172.21.15.194:6789/0 conn(0x7f0a500b4680 legacy=0x7f0a5006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:55.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.611+0000 7f0a4ea00640 1 -- 172.21.15.86:0/2151156082 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a500b2cf0 con 0x7f0a5015acb0 2024-03-06T07:32:55.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.612+0000 7f0a4ea00640 1 --2- 172.21.15.86:0/2151156082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5015acb0 0x7f0a500b37d0 secure :-1 s=READY pgs=1205 cs=0 l=1 rev1=1 crypto rx=0x7f0a500b4e60 tx=0x7f0a4002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e69bf5aa77cf0178 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:55.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.612+0000 7f0a4e000640 1 -- 172.21.15.86:0/2151156082 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a40035020 con 0x7f0a5015acb0 2024-03-06T07:32:55.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.612+0000 7f0a4e000640 1 -- 172.21.15.86:0/2151156082 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a40031870 con 0x7f0a5015acb0 2024-03-06T07:32:55.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.612+0000 7f0a4e000640 1 -- 172.21.15.86:0/2151156082 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a40031c40 con 0x7f0a5015acb0 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 -- 172.21.15.86:0/2151156082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5015acb0 msgr2=0x7f0a500b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 --2- 172.21.15.86:0/2151156082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5015acb0 0x7f0a500b37d0 secure :-1 s=READY pgs=1205 cs=0 l=1 rev1=1 crypto rx=0x7f0a500b4e60 tx=0x7f0a4002b910 comp rx=0 tx=0).stop 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 -- 172.21.15.86:0/2151156082 shutdown_connections 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 --2- 172.21.15.86:0/2151156082 >> v2:172.21.15.194:3300/0 conn(0x7f0a5006ec00 0x7f0a500ccfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 --2- 172.21.15.86:0/2151156082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5015acb0 0x7f0a500b37d0 unknown :-1 s=CLOSED pgs=1205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 -- 172.21.15.86:0/2151156082 >> 172.21.15.86:0/2151156082 conn(0x7f0a500b1730 msgr2=0x7f0a500b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 -- 172.21.15.86:0/2151156082 shutdown_connections 2024-03-06T07:32:55.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.613+0000 7f0a55800640 1 -- 172.21.15.86:0/2151156082 wait complete. 2024-03-06T07:32:55.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.615+0000 7f0a55800640 1 Processor -- start 2024-03-06T07:32:55.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.615+0000 7f0a55800640 1 -- start start 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a55800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5006ec00 0x7f0a50158870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a55800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a500b4680 0x7f0a50146640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a55800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a5015acb0 0x7f0a50153e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a55800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a500b3d10 con 0x7f0a5006ec00 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a55800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a500b32b0 con 0x7f0a500b4680 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a55800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0a500b4010 con 0x7f0a5015acb0 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5006ec00 0x7f0a50158870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5006ec00 0x7f0a50158870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49248/0 (socket says 172.21.15.86:49248) 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 -- 172.21.15.86:0/4027309443 learned_addr learned my addr 172.21.15.86:0/4027309443 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 -- 172.21.15.86:0/4027309443 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a5015acb0 msgr2=0x7f0a50153e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a5015acb0 0x7f0a50153e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 -- 172.21.15.86:0/4027309443 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a500b4680 msgr2=0x7f0a50146640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a500b4680 0x7f0a50146640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:55.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.616+0000 7f0a4ea00640 1 -- 172.21.15.86:0/4027309443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a501546f0 con 0x7f0a5006ec00 2024-03-06T07:32:55.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.617+0000 7f0a4ea00640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5006ec00 0x7f0a50158870 secure :-1 s=READY pgs=1206 cs=0 l=1 rev1=1 crypto rx=0x7f0a40000c00 tx=0x7f0a400046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:55.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.617+0000 7f0a46a00640 1 -- 172.21.15.86:0/4027309443 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a4003ad60 con 0x7f0a5006ec00 2024-03-06T07:32:55.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.617+0000 7f0a46a00640 1 -- 172.21.15.86:0/4027309443 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a400443f0 con 0x7f0a5006ec00 2024-03-06T07:32:55.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.617+0000 7f0a46a00640 1 -- 172.21.15.86:0/4027309443 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a400447e0 con 0x7f0a5006ec00 2024-03-06T07:32:55.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.617+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0a50146de0 con 0x7f0a5006ec00 2024-03-06T07:32:55.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.617+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a501543a0 con 0x7f0a5006ec00 2024-03-06T07:32:55.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.618+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0a5004f330 con 0x7f0a5006ec00 2024-03-06T07:32:55.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.619+0000 7f0a46a00640 1 -- 172.21.15.86:0/4027309443 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0a4004b060 con 0x7f0a5006ec00 2024-03-06T07:32:55.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.620+0000 7f0a46a00640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2c06b4d0 0x7f0a2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:55.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.621+0000 7f0a46a00640 1 -- 172.21.15.86:0/4027309443 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f0a400ac210 con 0x7f0a5006ec00 2024-03-06T07:32:55.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.621+0000 7f0a47e00640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2c06b4d0 0x7f0a2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:55.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.621+0000 7f0a47e00640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2c06b4d0 0x7f0a2c06d990 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f0a50072a80 tx=0x7f0a38007990 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:55.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.625+0000 7f0a46a00640 1 -- 172.21.15.86:0/4027309443 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0a4003f070 con 0x7f0a5006ec00 2024-03-06T07:32:55.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:55.901+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]} v 0) v1 -- 0x7f0a5015a360 con 0x7f0a5006ec00 2024-03-06T07:32:56.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.276+0000 7f0a46a00640 1 -- 172.21.15.86:0/4027309443 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]}]=0 v335) v1 ==== 134+0+0 (secure 0 0 0) 0x7f0a4002f320 con 0x7f0a5006ec00 2024-03-06T07:32:56.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.279+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2c06b4d0 msgr2=0x7f0a2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.279+0000 7f0a55800640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2c06b4d0 0x7f0a2c06d990 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f0a50072a80 tx=0x7f0a38007990 comp rx=0 tx=0).stop 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.279+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5006ec00 msgr2=0x7f0a50158870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.279+0000 7f0a55800640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5006ec00 0x7f0a50158870 secure :-1 s=READY pgs=1206 cs=0 l=1 rev1=1 crypto rx=0x7f0a40000c00 tx=0x7f0a400046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 shutdown_connections 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2c06b4d0 0x7f0a2c06d990 unknown :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a5015acb0 0x7f0a50153e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a500b4680 0x7f0a50146640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 --2- 172.21.15.86:0/4027309443 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a5006ec00 0x7f0a50158870 unknown :-1 s=CLOSED pgs=1206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 >> 172.21.15.86:0/4027309443 conn(0x7f0a500b1730 msgr2=0x7f0a500ccf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 shutdown_connections 2024-03-06T07:32:56.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.280+0000 7f0a55800640 1 -- 172.21.15.86:0/4027309443 wait complete. 2024-03-06T07:32:56.303 INFO:tasks.cephfs_test_runner:test_files_throttle (tasks.cephfs.test_strays.TestStrays) ... ok 2024-03-06T07:32:56.303 INFO:tasks.cephfs_test_runner:Starting test: test_hardlink_reintegration (tasks.cephfs.test_strays.TestStrays) That removal of primary dentry of hardlinked inode results 2024-03-06T07:32:56.303 INFO:tasks.cephfs_test_runner:test_hardlink_reintegration (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:32:56.303 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:32:56.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.509+0000 7fa59ba00640 1 Processor -- start 2024-03-06T07:32:56.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.509+0000 7fa59ba00640 1 -- start start 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa59ba00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa5940af1c0 0x7fa59415eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa59ba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59406e4d0 0x7fa59406e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa59ba00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa594071d30 con 0x7fa594070d20 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa59ba00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa5940711d0 con 0x7fa5940af1c0 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa59ba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa594071490 con 0x7fa59406e4d0 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa5940af1c0 0x7fa59415eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa5940af1c0 0x7fa59415eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48292/0 (socket says 172.21.15.86:48292) 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599000640 1 -- 172.21.15.86:0/1426518324 learned_addr learned my addr 172.21.15.86:0/1426518324 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599a00640 1 --2- 172.21.15.86:0/1426518324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59406e4d0 0x7fa59406e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599000640 1 -- 172.21.15.86:0/1426518324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59406e4d0 msgr2=0x7fa59406e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:56.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599000640 1 --2- 172.21.15.86:0/1426518324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59406e4d0 0x7fa59406e960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599000640 1 -- 172.21.15.86:0/1426518324 >> v1:172.21.15.194:6789/0 conn(0x7fa594070d20 legacy=0x7fa59406cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:56.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.511+0000 7fa599000640 1 -- 172.21.15.86:0/1426518324 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa594071770 con 0x7fa5940af1c0 2024-03-06T07:32:56.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.512+0000 7fa599000640 1 --2- 172.21.15.86:0/1426518324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa5940af1c0 0x7fa59415eff0 secure :-1 s=READY pgs=1207 cs=0 l=1 rev1=1 crypto rx=0x7fa5880098a0 tx=0x7fa58802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d2e5a46479845b5e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:56.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.513+0000 7fa593400640 1 -- 172.21.15.86:0/1426518324 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa588035020 con 0x7fa5940af1c0 2024-03-06T07:32:56.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.513+0000 7fa593400640 1 -- 172.21.15.86:0/1426518324 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa588031640 con 0x7fa5940af1c0 2024-03-06T07:32:56.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.513+0000 7fa593400640 1 -- 172.21.15.86:0/1426518324 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa58802d040 con 0x7fa5940af1c0 2024-03-06T07:32:56.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.513+0000 7fa59ba00640 1 -- 172.21.15.86:0/1426518324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa5940af1c0 msgr2=0x7fa59415eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.513+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1426518324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa5940af1c0 0x7fa59415eff0 secure :-1 s=READY pgs=1207 cs=0 l=1 rev1=1 crypto rx=0x7fa5880098a0 tx=0x7fa58802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.514+0000 7fa59ba00640 1 -- 172.21.15.86:0/1426518324 shutdown_connections 2024-03-06T07:32:56.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.514+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1426518324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa59406e4d0 0x7fa59406e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.514+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1426518324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa5940af1c0 0x7fa59415eff0 unknown :-1 s=CLOSED pgs=1207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.514+0000 7fa59ba00640 1 -- 172.21.15.86:0/1426518324 >> 172.21.15.86:0/1426518324 conn(0x7fa5940b1730 msgr2=0x7fa5940b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:56.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.514+0000 7fa59ba00640 1 -- 172.21.15.86:0/1426518324 shutdown_connections 2024-03-06T07:32:56.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.514+0000 7fa59ba00640 1 -- 172.21.15.86:0/1426518324 wait complete. 2024-03-06T07:32:56.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.515+0000 7fa59ba00640 1 Processor -- start 2024-03-06T07:32:56.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.515+0000 7fa59ba00640 1 -- start start 2024-03-06T07:32:56.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.516+0000 7fa59ba00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59406e4d0 0x7fa59414b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.516+0000 7fa59ba00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa594070d20 0x7fa5941e67e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.516+0000 7fa59ba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa5940af1c0 0x7fa59415fdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.516+0000 7fa59ba00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa59406d370 con 0x7fa59406e4d0 2024-03-06T07:32:56.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.516+0000 7fa59ba00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa5940b30b0 con 0x7fa594070d20 2024-03-06T07:32:56.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.516+0000 7fa59ba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa5940b35d0 con 0x7fa5940af1c0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59406e4d0 0x7fa59414b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59406e4d0 0x7fa59414b3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48300/0 (socket says 172.21.15.86:48300) 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 -- 172.21.15.86:0/1194694201 learned_addr learned my addr 172.21.15.86:0/1194694201 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 -- 172.21.15.86:0/1194694201 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa5940af1c0 msgr2=0x7fa59415fdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa5940af1c0 0x7fa59415fdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 -- 172.21.15.86:0/1194694201 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa594070d20 msgr2=0x7fa5941e67e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa594070d20 0x7fa5941e67e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 -- 172.21.15.86:0/1194694201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa5941e71c0 con 0x7fa59406e4d0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa599000640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59406e4d0 0x7fa59414b3d0 secure :-1 s=READY pgs=1208 cs=0 l=1 rev1=1 crypto rx=0x7fa58802b930 tx=0x7fa5880046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.517+0000 7fa590c00640 1 -- 172.21.15.86:0/1194694201 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa588040070 con 0x7fa59406e4d0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.518+0000 7fa590c00640 1 -- 172.21.15.86:0/1194694201 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa588039c10 con 0x7fa59406e4d0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.518+0000 7fa590c00640 1 -- 172.21.15.86:0/1194694201 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa588035040 con 0x7fa59406e4d0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.518+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa5941e74a0 con 0x7fa59406e4d0 2024-03-06T07:32:56.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.518+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa5941e7a60 con 0x7fa59406e4d0 2024-03-06T07:32:56.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.519+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa5941e7da0 con 0x7fa59406e4d0 2024-03-06T07:32:56.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.520+0000 7fa590c00640 1 -- 172.21.15.86:0/1194694201 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa58804b060 con 0x7fa59406e4d0 2024-03-06T07:32:56.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.520+0000 7fa590c00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57406b4d0 0x7fa57406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.521+0000 7fa590c00640 1 -- 172.21.15.86:0/1194694201 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fa5880aba30 con 0x7fa59406e4d0 2024-03-06T07:32:56.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.521+0000 7fa593e00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57406b4d0 0x7fa57406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.521+0000 7fa593e00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57406b4d0 0x7fa57406d990 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7fa59415e130 tx=0x7fa5800079b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:56.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.525+0000 7fa590c00640 1 -- 172.21.15.86:0/1194694201 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa594071d30 con 0x7fa59406e4d0 2024-03-06T07:32:56.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.721+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa59404f330 con 0x7fa59406e4d0 2024-03-06T07:32:56.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.724+0000 7fa590c00640 1 -- 172.21.15.86:0/1194694201 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v101) v1 ==== 74+0+13294 (secure 0 0 0) 0x7fa58802f300 con 0x7fa59406e4d0 2024-03-06T07:32:56.725 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:56.726 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:32:54.245283+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6835/2542557324":"2024-03-07T07:32:50.931994+0000","172.21.15.86:6835/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.86:6834/2542557324":"2024-03-07T07:32:50.931994+0000","172.21.15.86:6834/883748988":"2024-03-07T07:32:20.051162+0000","172.21.15.194:6834/2712222804":"2024-03-07T07:32:14.424511+0000","172.21.15.194:6832/2712222804":"2024-03-07T07:32:14.424511+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:32:56.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.726+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57406b4d0 msgr2=0x7fa57406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.726+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57406b4d0 0x7fa57406d990 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7fa59415e130 tx=0x7fa5800079b0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.726+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59406e4d0 msgr2=0x7fa59414b3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.726+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59406e4d0 0x7fa59414b3d0 secure :-1 s=READY pgs=1208 cs=0 l=1 rev1=1 crypto rx=0x7fa58802b930 tx=0x7fa5880046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 shutdown_connections 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa57406b4d0 0x7fa57406d990 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa5940af1c0 0x7fa59415fdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa594070d20 0x7fa5941e67e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 --2- 172.21.15.86:0/1194694201 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa59406e4d0 0x7fa59414b3d0 unknown :-1 s=CLOSED pgs=1208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 >> 172.21.15.86:0/1194694201 conn(0x7fa5940b1730 msgr2=0x7fa5940cfa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 shutdown_connections 2024-03-06T07:32:56.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.727+0000 7fa59ba00640 1 -- 172.21.15.86:0/1194694201 wait complete. 2024-03-06T07:32:56.748 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:32:56.748 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:32:56.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.958+0000 7fd102e00640 1 Processor -- start 2024-03-06T07:32:56.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.959+0000 7fd102e00640 1 -- start start 2024-03-06T07:32:56.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.960+0000 7fd102e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd0fc0b2c40 0x7fd0fc15f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.960+0000 7fd102e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd0fc06d960 0x7fd0fc0b3020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.960+0000 7fd102e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd0fc0b3560 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.960+0000 7fd102e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd0fc15f700 con 0x7fd0fc0b2c40 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.960+0000 7fd102e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd0fc06d4d0 con 0x7fd0fc06d960 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.960+0000 7fd0fbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd0fc0b2c40 0x7fd0fc15f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0fbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd0fc0b2c40 0x7fd0fc15f1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48314/0 (socket says 172.21.15.86:48314) 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0fbe00640 1 -- 172.21.15.86:0/2762930229 learned_addr learned my addr 172.21.15.86:0/2762930229 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd100e00640 1 --1- 172.21.15.86:0/2762930229 >> v1:172.21.15.86:6789/0 conn(0x7fd0fc06e2f0 0x7fd0fc161d50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59862/0 (socket says 172.21.15.86:59862) 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0f3400640 1 --2- 172.21.15.86:0/2762930229 >> v2:172.21.15.194:3300/0 conn(0x7fd0fc06d960 0x7fd0fc0b3020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0fbe00640 1 -- 172.21.15.86:0/2762930229 >> v2:172.21.15.194:3300/0 conn(0x7fd0fc06d960 msgr2=0x7fd0fc0b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0fbe00640 1 --2- 172.21.15.86:0/2762930229 >> v2:172.21.15.194:3300/0 conn(0x7fd0fc06d960 0x7fd0fc0b3020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0fbe00640 1 -- 172.21.15.86:0/2762930229 >> v1:172.21.15.86:6789/0 conn(0x7fd0fc06e2f0 legacy=0x7fd0fc161d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0fbe00640 1 -- 172.21.15.86:0/2762930229 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0fc06cba0 con 0x7fd0fc0b2c40 2024-03-06T07:32:56.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.961+0000 7fd0fb400640 1 -- 172.21.15.86:0/2762930229 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 873767646 0 0) 0x7fd0fc0b3560 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.962+0000 7fd0fbe00640 1 --2- 172.21.15.86:0/2762930229 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc0b2c40 0x7fd0fc15f1c0 secure :-1 s=READY pgs=1210 cs=0 l=1 rev1=1 crypto rx=0x7fd0fc074a50 tx=0x7fd0ec02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=99062c2c6465f2df server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:56.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd0fb400640 1 -- 172.21.15.86:0/2762930229 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0ec034070 con 0x7fd0fc0b2c40 2024-03-06T07:32:56.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd0fb400640 1 -- 172.21.15.86:0/2762930229 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd0ec030860 con 0x7fd0fc0b2c40 2024-03-06T07:32:56.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd0fb400640 1 -- 172.21.15.86:0/2762930229 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0ec030c30 con 0x7fd0fc0b2c40 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 -- 172.21.15.86:0/2762930229 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc0b2c40 msgr2=0x7fd0fc15f1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 --2- 172.21.15.86:0/2762930229 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc0b2c40 0x7fd0fc15f1c0 secure :-1 s=READY pgs=1210 cs=0 l=1 rev1=1 crypto rx=0x7fd0fc074a50 tx=0x7fd0ec02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 -- 172.21.15.86:0/2762930229 shutdown_connections 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 --2- 172.21.15.86:0/2762930229 >> v2:172.21.15.194:3300/0 conn(0x7fd0fc06d960 0x7fd0fc0b3020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 --2- 172.21.15.86:0/2762930229 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc0b2c40 0x7fd0fc15f1c0 unknown :-1 s=CLOSED pgs=1210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 -- 172.21.15.86:0/2762930229 >> 172.21.15.86:0/2762930229 conn(0x7fd0fc0b1940 msgr2=0x7fd0fc0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 -- 172.21.15.86:0/2762930229 shutdown_connections 2024-03-06T07:32:56.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.963+0000 7fd102e00640 1 -- 172.21.15.86:0/2762930229 wait complete. 2024-03-06T07:32:56.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.965+0000 7fd102e00640 1 Processor -- start 2024-03-06T07:32:56.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.965+0000 7fd102e00640 1 -- start start 2024-03-06T07:32:56.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd102e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0fc06d960 0x7fd0fc152280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd102e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc06e2f0 0x7fd0fc154180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd102e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0fc0b2c40 0x7fd0fc1444b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd102e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0fc06e780 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd102e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0fc06d160 con 0x7fd0fc06d960 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd102e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0fc06e900 con 0x7fd0fc0b2c40 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd0f3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc06e2f0 0x7fd0fc154180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd0f3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc06e2f0 0x7fd0fc154180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48330/0 (socket says 172.21.15.86:48330) 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.966+0000 7fd0f3400640 1 -- 172.21.15.86:0/2855914282 learned_addr learned my addr 172.21.15.86:0/2855914282 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd100e00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0fc0b2c40 0x7fd0fc1444b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f3400640 1 -- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0fc0b2c40 msgr2=0x7fd0fc1444b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0fbe00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0fc06d960 0x7fd0fc152280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f3400640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0fc0b2c40 0x7fd0fc1444b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f3400640 1 -- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0fc06d960 msgr2=0x7fd0fc152280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f3400640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0fc06d960 0x7fd0fc152280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f3400640 1 -- 172.21.15.86:0/2855914282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0fc14e6d0 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f3400640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc06e2f0 0x7fd0fc154180 secure :-1 s=READY pgs=1211 cs=0 l=1 rev1=1 crypto rx=0x7fd0e400d620 tx=0x7fd0e400dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:56.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f8c00640 1 -- 172.21.15.86:0/2855914282 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0e400b620 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f8c00640 1 -- 172.21.15.86:0/2855914282 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd0e4019070 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd0f8c00640 1 -- 172.21.15.86:0/2855914282 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0e400ba00 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.967+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd0fc1449f0 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.968+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0fc152b40 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.969+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd0fc04f330 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.970+0000 7fd0f8c00640 1 -- 172.21.15.86:0/2855914282 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd0e400bba0 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.971+0000 7fd0f8c00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0d806b4d0 0x7fd0d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:56.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.971+0000 7fd0f8c00640 1 -- 172.21.15.86:0/2855914282 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fd0e408a390 con 0x7fd0fc06e2f0 2024-03-06T07:32:56.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.971+0000 7fd0fbe00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0d806b4d0 0x7fd0d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:56.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.972+0000 7fd0fbe00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0d806b4d0 0x7fd0d806d990 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec036000 tx=0x7fd0ec03b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:56.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:56.976+0000 7fd0f8c00640 1 -- 172.21.15.86:0/2855914282 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd0fc1449f0 con 0x7fd0fc06e2f0 2024-03-06T07:32:57.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.170+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd0fc15f460 con 0x7fd0fc06e2f0 2024-03-06T07:32:57.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.171+0000 7fd0f8c00640 1 -- 172.21.15.86:0/2855914282 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v101) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd0fc14e6d0 con 0x7fd0fc06e2f0 2024-03-06T07:32:57.173 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:32:57.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.173+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0d806b4d0 msgr2=0x7fd0d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.173+0000 7fd102e00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0d806b4d0 0x7fd0d806d990 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec036000 tx=0x7fd0ec03b040 comp rx=0 tx=0).stop 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc06e2f0 msgr2=0x7fd0fc154180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc06e2f0 0x7fd0fc154180 secure :-1 s=READY pgs=1211 cs=0 l=1 rev1=1 crypto rx=0x7fd0e400d620 tx=0x7fd0e400dae0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 shutdown_connections 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd0d806b4d0 0x7fd0d806d990 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd0fc0b2c40 0x7fd0fc1444b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0fc06e2f0 0x7fd0fc154180 unknown :-1 s=CLOSED pgs=1211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 --2- 172.21.15.86:0/2855914282 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd0fc06d960 0x7fd0fc152280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 >> 172.21.15.86:0/2855914282 conn(0x7fd0fc0b1940 msgr2=0x7fd0fc0b3020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 shutdown_connections 2024-03-06T07:32:57.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.174+0000 7fd102e00640 1 -- 172.21.15.86:0/2855914282 wait complete. 2024-03-06T07:32:57.194 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration' 2024-03-06T07:32:57.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.398+0000 7fa2ab200640 1 Processor -- start 2024-03-06T07:32:57.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.398+0000 7fa2ab200640 1 -- start start 2024-03-06T07:32:57.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.399+0000 7fa2ab200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa2ac0cc9d0 0x7fa2ac06d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.399+0000 7fa2ab200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa2ac15c240 con 0x7fa2ac1561f0 2024-03-06T07:32:57.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.399+0000 7fa2ab200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa2ac15c500 con 0x7fa2ac0aedc0 2024-03-06T07:32:57.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.399+0000 7fa2ab200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa2ac15ce30 con 0x7fa2ac0cc9d0 2024-03-06T07:32:57.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.400+0000 7fa2a9e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa2ac1561f0 0x7fa2ac0ae6b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59874/0 (socket says 172.21.15.86:59874) 2024-03-06T07:32:57.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.400+0000 7fa2a9e00640 1 -- 172.21.15.86:0/363038368 learned_addr learned my addr 172.21.15.86:0/363038368 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.400+0000 7fa2aa800640 1 --2- 172.21.15.86:0/363038368 >> v2:172.21.15.194:3300/0 conn(0x7fa2ac0cc9d0 0x7fa2ac06d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.400+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 267151669 0 0) 0x7fa2ac15c240 con 0x7fa2ac1561f0 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.400+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa290003520 con 0x7fa2ac1561f0 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3200977683 0 0) 0x7fa290003520 con 0x7fa2ac1561f0 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 >> v2:172.21.15.194:3300/0 conn(0x7fa2ac0cc9d0 msgr2=0x7fa2ac06d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 --2- 172.21.15.86:0/363038368 >> v2:172.21.15.194:3300/0 conn(0x7fa2ac0cc9d0 0x7fa2ac06d350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 >> v1:172.21.15.194:6789/0 conn(0x7fa2ac0aedc0 legacy=0x7fa2ac0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2ac15c7e0 con 0x7fa2ac1561f0 2024-03-06T07:32:57.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fa29c001670 con 0x7fa2ac1561f0 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 >> v1:172.21.15.86:6789/0 conn(0x7fa2ac1561f0 legacy=0x7fa2ac0ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa290003a00 0x7fa290003e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa290004800 0x7fa290004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2900051c0 0x7fa29000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2ac15ce30 con 0x7fa2900051c0 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa29000c0f0 con 0x7fa290004800 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa29000c2f0 con 0x7fa290003a00 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2aa800640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2900051c0 0x7fa29000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.401+0000 7fa2a9e00640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa290003a00 0x7fa290003e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.402+0000 7fa2aa800640 1 -- 172.21.15.86:0/363038368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa290003a00 msgr2=0x7fa290003e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.402+0000 7fa2aa800640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa290003a00 0x7fa290003e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.402+0000 7fa2aa800640 1 -- 172.21.15.86:0/363038368 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa290004800 msgr2=0x7fa290004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:57.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.402+0000 7fa2aa800640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa290004800 0x7fa290004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.402+0000 7fa2aa800640 1 -- 172.21.15.86:0/363038368 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa290003520 con 0x7fa2900051c0 2024-03-06T07:32:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.402+0000 7fa2aa800640 1 -- 172.21.15.86:0/363038368 reap_dead start 2024-03-06T07:32:57.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2aa800640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2900051c0 0x7fa29000b9c0 secure :-1 s=READY pgs=1213 cs=0 l=1 rev1=1 crypto rx=0x7fa2a4009fc0 tx=0x7fa290005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a20f4638295ba79c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2a4009c10 con 0x7fa2900051c0 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2a8a00640 1 -- 172.21.15.86:0/363038368 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa2a402cca0 con 0x7fa2900051c0 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2ab200640 1 -- 172.21.15.86:0/363038368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2900051c0 msgr2=0x7fa29000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2ab200640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2900051c0 0x7fa29000b9c0 secure :-1 s=READY pgs=1213 cs=0 l=1 rev1=1 crypto rx=0x7fa2a4009fc0 tx=0x7fa290005830 comp rx=0 tx=0).stop 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2ab200640 1 -- 172.21.15.86:0/363038368 shutdown_connections 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2ab200640 1 --2- 172.21.15.86:0/363038368 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2900051c0 0x7fa29000b9c0 unknown :-1 s=CLOSED pgs=1213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.403+0000 7fa2ab200640 1 -- 172.21.15.86:0/363038368 >> 172.21.15.86:0/363038368 conn(0x7fa2ac06afc0 msgr2=0x7fa2ac06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.404+0000 7fa2ab200640 1 -- 172.21.15.86:0/363038368 shutdown_connections 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.404+0000 7fa2ab200640 1 -- 172.21.15.86:0/363038368 wait complete. 2024-03-06T07:32:57.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.405+0000 7fa2ab200640 1 Processor -- start 2024-03-06T07:32:57.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.405+0000 7fa2ab200640 1 -- start start 2024-03-06T07:32:57.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.406+0000 7fa2ab200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa2900051c0 0x7fa2ac0adad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.406+0000 7fa2ab200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa2ac150250 0x7fa2ac1506d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.406+0000 7fa2ab200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2ac0cdeb0 0x7fa2ac0ce360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.406+0000 7fa2ab200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2ac156750 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.406+0000 7fa2ab200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2ac1565d0 con 0x7fa2900051c0 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.406+0000 7fa2ab200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa2ac0cc2c0 con 0x7fa2ac150250 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2ac0cdeb0 0x7fa2ac0ce360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2ac0cdeb0 0x7fa2ac0ce360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48354/0 (socket says 172.21.15.86:48354) 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 -- 172.21.15.86:0/3099515840 learned_addr learned my addr 172.21.15.86:0/3099515840 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2a9e00640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa2900051c0 0x7fa2ac0adad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 -- 172.21.15.86:0/3099515840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa2ac150250 msgr2=0x7fa2ac1506d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa2ac150250 0x7fa2ac1506d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 -- 172.21.15.86:0/3099515840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa2900051c0 msgr2=0x7fa2ac0adad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa2900051c0 0x7fa2ac0adad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 -- 172.21.15.86:0/3099515840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2ac0cf1f0 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.407+0000 7fa2aa800640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2ac0cdeb0 0x7fa2ac0ce360 secure :-1 s=READY pgs=1214 cs=0 l=1 rev1=1 crypto rx=0x7fa2a402c6d0 tx=0x7fa2a4009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.408+0000 7fa2a2000640 1 -- 172.21.15.86:0/3099515840 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2a4003990 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.408+0000 7fa2a2000640 1 -- 172.21.15.86:0/3099515840 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa2a4044070 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.408+0000 7fa2a2000640 1 -- 172.21.15.86:0/3099515840 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2a4003d90 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.408+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa2ac150c10 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.408+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa2ac0cef10 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.409+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa2ac04f330 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.410+0000 7fa2a2000640 1 -- 172.21.15.86:0/3099515840 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa2a4005d10 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.411+0000 7fa2a2000640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa28c06b500 0x7fa28c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:57.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.411+0000 7fa2a2000640 1 -- 172.21.15.86:0/3099515840 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fa2a40ac910 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.411+0000 7fa2a9e00640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa28c06b500 0x7fa28c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:57.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.412+0000 7fa2a9e00640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa28c06b500 0x7fa28c06d9c0 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fa29c0017e0 tx=0x7fa29c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:57.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.416+0000 7fa2a2000640 1 -- 172.21.15.86:0/3099515840 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa2a407b940 con 0x7fa2ac0cdeb0 2024-03-06T07:32:57.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:57.686+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]} v 0) v1 -- 0x7fa2ac06ce70 con 0x7fa2ac0cdeb0 2024-03-06T07:32:58.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.289+0000 7fa2a2000640 1 -- 172.21.15.86:0/3099515840 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]}]=0 v337) v1 ==== 145+0+0 (secure 0 0 0) 0x7fa2a407b240 con 0x7fa2ac0cdeb0 2024-03-06T07:32:58.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.292+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa28c06b500 msgr2=0x7fa28c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.292+0000 7fa2ab200640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa28c06b500 0x7fa28c06d9c0 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fa29c0017e0 tx=0x7fa29c009040 comp rx=0 tx=0).stop 2024-03-06T07:32:58.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.292+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2ac0cdeb0 msgr2=0x7fa2ac0ce360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.292+0000 7fa2ab200640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2ac0cdeb0 0x7fa2ac0ce360 secure :-1 s=READY pgs=1214 cs=0 l=1 rev1=1 crypto rx=0x7fa2a402c6d0 tx=0x7fa2a4009750 comp rx=0 tx=0).stop 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 shutdown_connections 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa28c06b500 0x7fa28c06d9c0 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa2ac0cdeb0 0x7fa2ac0ce360 unknown :-1 s=CLOSED pgs=1214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa2ac150250 0x7fa2ac1506d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 --2- 172.21.15.86:0/3099515840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa2900051c0 0x7fa2ac0adad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 >> 172.21.15.86:0/3099515840 conn(0x7fa2ac06afc0 msgr2=0x7fa2ac0b3720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 shutdown_connections 2024-03-06T07:32:58.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.293+0000 7fa2ab200640 1 -- 172.21.15.86:0/3099515840 wait complete. 2024-03-06T07:32:58.315 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:32:58.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.529+0000 7f46ec200640 1 Processor -- start 2024-03-06T07:32:58.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.529+0000 7f46ec200640 1 -- start start 2024-03-06T07:32:58.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.531+0000 7f46ec200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f46e40b3df0 0x7f46e40727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:58.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.531+0000 7f46ec200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40cda90 0x7f46e406c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:58.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.531+0000 7f46ec200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f46e40719e0 con 0x7f46e40b3650 2024-03-06T07:32:58.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.531+0000 7f46ec200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46e4072310 con 0x7f46e40b3df0 2024-03-06T07:32:58.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.531+0000 7f46ec200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46e4071cc0 con 0x7f46e40cda90 2024-03-06T07:32:58.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.531+0000 7f46e9800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f46e40b3df0 0x7f46e40727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.531+0000 7f46ea200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40cda90 0x7f46e406c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.532+0000 7f46ea200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40cda90 0x7f46e406c990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:43760/0 (socket says 172.21.15.86:43760) 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.532+0000 7f46ea200640 1 -- 172.21.15.86:0/3034347425 learned_addr learned my addr 172.21.15.86:0/3034347425 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.532+0000 7f46ea200640 1 -- 172.21.15.86:0/3034347425 >> v2:172.21.15.194:3300/0 conn(0x7f46e40b3df0 msgr2=0x7f46e40727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.532+0000 7f46ea200640 1 --2- 172.21.15.86:0/3034347425 >> v2:172.21.15.194:3300/0 conn(0x7f46e40b3df0 0x7f46e40727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.532+0000 7f46ea200640 1 -- 172.21.15.86:0/3034347425 >> v1:172.21.15.194:6789/0 conn(0x7f46e40b3650 legacy=0x7f46e40b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.533+0000 7f46ea200640 1 -- 172.21.15.86:0/3034347425 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46e4071fa0 con 0x7f46e40cda90 2024-03-06T07:32:58.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.532+0000 7f46e3e00640 1 -- 172.21.15.86:0/3034347425 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2916909360 0 0) 0x7f46e40719e0 con 0x7f46e40b3650 2024-03-06T07:32:58.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.534+0000 7f46ea200640 1 --2- 172.21.15.86:0/3034347425 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40cda90 0x7f46e406c990 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f46dc009870 tx=0x7f46dc02b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aae0599a32b777a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:58.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.534+0000 7f46e3e00640 1 -- 172.21.15.86:0/3034347425 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46dc034070 con 0x7f46e40cda90 2024-03-06T07:32:58.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.534+0000 7f46e3e00640 1 -- 172.21.15.86:0/3034347425 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46dc030840 con 0x7f46e40cda90 2024-03-06T07:32:58.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.534+0000 7f46e3e00640 1 -- 172.21.15.86:0/3034347425 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46dc030c10 con 0x7f46e40cda90 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 -- 172.21.15.86:0/3034347425 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40cda90 msgr2=0x7f46e406c990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 --2- 172.21.15.86:0/3034347425 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40cda90 0x7f46e406c990 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f46dc009870 tx=0x7f46dc02b980 comp rx=0 tx=0).stop 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 -- 172.21.15.86:0/3034347425 shutdown_connections 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 --2- 172.21.15.86:0/3034347425 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40cda90 0x7f46e406c990 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 --2- 172.21.15.86:0/3034347425 >> v2:172.21.15.194:3300/0 conn(0x7f46e40b3df0 0x7f46e40727a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 -- 172.21.15.86:0/3034347425 >> 172.21.15.86:0/3034347425 conn(0x7f46e40b1940 msgr2=0x7f46e40b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 -- 172.21.15.86:0/3034347425 shutdown_connections 2024-03-06T07:32:58.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.535+0000 7f46ec200640 1 -- 172.21.15.86:0/3034347425 wait complete. 2024-03-06T07:32:58.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.537+0000 7f46ec200640 1 Processor -- start 2024-03-06T07:32:58.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.537+0000 7f46ec200640 1 -- start start 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ec200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46e40b3650 0x7f46e415df10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ec200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40b3df0 0x7f46e414a7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ec200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46e40cda90 0x7f46e414ad00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ec200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46e415b3c0 con 0x7f46e40cda90 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ec200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46e415b240 con 0x7f46e40b3650 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ec200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46e40cf510 con 0x7f46e40b3df0 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ea200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46e40cda90 0x7f46e414ad00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ea200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46e40cda90 0x7f46e414ad00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48364/0 (socket says 172.21.15.86:48364) 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46ea200640 1 -- 172.21.15.86:0/445782775 learned_addr learned my addr 172.21.15.86:0/445782775 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46e8e00640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40b3df0 0x7f46e414a7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.538+0000 7f46e9800640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46e40b3650 0x7f46e415df10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:58.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ea200640 1 -- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40b3df0 msgr2=0x7f46e414a7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ea200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40b3df0 0x7f46e414a7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ea200640 1 -- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46e40b3650 msgr2=0x7f46e415df10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ea200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46e40b3650 0x7f46e415df10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ea200640 1 -- 172.21.15.86:0/445782775 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46e414a390 con 0x7f46e40cda90 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ea200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46e40cda90 0x7f46e414ad00 secure :-1 s=READY pgs=1215 cs=0 l=1 rev1=1 crypto rx=0x7f46dc02b9c0 tx=0x7f46dc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46e1600640 1 -- 172.21.15.86:0/445782775 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46dc034040 con 0x7f46e40cda90 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46e1600640 1 -- 172.21.15.86:0/445782775 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46dc03ada0 con 0x7f46e40cda90 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46e415e450 con 0x7f46e40cda90 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46e1600640 1 -- 172.21.15.86:0/445782775 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46dc044630 con 0x7f46e40cda90 2024-03-06T07:32:58.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.539+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46e409d310 con 0x7f46e40cda90 2024-03-06T07:32:58.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.541+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f46e404f330 con 0x7f46e40cda90 2024-03-06T07:32:58.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.541+0000 7f46e1600640 1 -- 172.21.15.86:0/445782775 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f46dc04b060 con 0x7f46e40cda90 2024-03-06T07:32:58.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.543+0000 7f46e1600640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46c406b4d0 0x7f46c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:58.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.543+0000 7f46e1600640 1 -- 172.21.15.86:0/445782775 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f46dc0abfe0 con 0x7f46e40cda90 2024-03-06T07:32:58.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.543+0000 7f46e9800640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46c406b4d0 0x7f46c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:58.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.543+0000 7f46e9800640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46c406b4d0 0x7f46c406d990 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f46d4009fd0 tx=0x7f46d400b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:58.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.548+0000 7f46e1600640 1 -- 172.21.15.86:0/445782775 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f46e414a390 con 0x7f46e40cda90 2024-03-06T07:32:58.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.732+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f46e40cc0e0 con 0x7f46e40cda90 2024-03-06T07:32:58.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.740+0000 7f46e1600640 1 -- 172.21.15.86:0/445782775 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v30)=0 v30) v1 ==== 122+0+0 (secure 0 0 0) 0x7f46dc03f070 con 0x7f46e40cda90 2024-03-06T07:32:58.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46c406b4d0 msgr2=0x7f46c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46c406b4d0 0x7f46c406d990 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f46d4009fd0 tx=0x7f46d400b040 comp rx=0 tx=0).stop 2024-03-06T07:32:58.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46e40cda90 msgr2=0x7f46e414ad00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:58.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46e40cda90 0x7f46e414ad00 secure :-1 s=READY pgs=1215 cs=0 l=1 rev1=1 crypto rx=0x7f46dc02b9c0 tx=0x7f46dc0046c0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 shutdown_connections 2024-03-06T07:32:58.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46c406b4d0 0x7f46c406d990 unknown :-1 s=CLOSED pgs=527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46e40cda90 0x7f46e414ad00 unknown :-1 s=CLOSED pgs=1215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46e40b3df0 0x7f46e414a7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 --2- 172.21.15.86:0/445782775 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46e40b3650 0x7f46e415df10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 >> 172.21.15.86:0/445782775 conn(0x7f46e40b1940 msgr2=0x7f46e40afc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.742+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 shutdown_connections 2024-03-06T07:32:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:58.743+0000 7f46ec200640 1 -- 172.21.15.86:0/445782775 wait complete. 2024-03-06T07:32:58.760 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:32:58.760 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:32:58.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:32:58.988 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:32:58.988 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:32:58.988 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:59.045 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:32:59.045 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:32:59.046 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:59.102 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:32:59.103 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:32:59.103 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:59.159 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:32:59.159 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:32:59.159 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:32:59.215 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:32:59.215 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:32:59.215 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:32:59.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.425+0000 7ff593600640 1 Processor -- start 2024-03-06T07:32:59.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.425+0000 7ff593600640 1 -- start start 2024-03-06T07:32:59.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff593600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 0x7ff58c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff593600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff58c15f9d0 0x7ff58c15fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff593600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff58c06ee70 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff593600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff58c160370 con 0x7ff58c15f9d0 2024-03-06T07:32:59.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff593600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff58c06cef0 con 0x7ff58c144c00 2024-03-06T07:32:59.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff583e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff58c15f9d0 0x7ff58c15fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff583e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff58c15f9d0 0x7ff58c15fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48366/0 (socket says 172.21.15.86:48366) 2024-03-06T07:32:59.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff583e00640 1 -- 172.21.15.86:0/893591766 learned_addr learned my addr 172.21.15.86:0/893591766 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:59.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff583e00640 1 -- 172.21.15.86:0/893591766 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 msgr2=0x7ff58c06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:59.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.427+0000 7ff590c00640 1 --2- 172.21.15.86:0/893591766 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 0x7ff58c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.428+0000 7ff583e00640 1 --2- 172.21.15.86:0/893591766 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 0x7ff58c06e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.428+0000 7ff583e00640 1 -- 172.21.15.86:0/893591766 >> v1:172.21.15.194:6789/0 conn(0x7ff58c0ce0a0 legacy=0x7ff58c0ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:59.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.428+0000 7ff583e00640 1 -- 172.21.15.86:0/893591766 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff58c06d1d0 con 0x7ff58c15f9d0 2024-03-06T07:32:59.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.429+0000 7ff583e00640 1 --2- 172.21.15.86:0/893591766 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c15f9d0 0x7ff58c15fe30 secure :-1 s=READY pgs=1216 cs=0 l=1 rev1=1 crypto rx=0x7ff574009870 tx=0x7ff57402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d4d94f393a5ebdf1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:59.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.429+0000 7ff58be00640 1 -- 172.21.15.86:0/893591766 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff574035020 con 0x7ff58c15f9d0 2024-03-06T07:32:59.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.429+0000 7ff58be00640 1 -- 172.21.15.86:0/893591766 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff574031640 con 0x7ff58c15f9d0 2024-03-06T07:32:59.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.429+0000 7ff58be00640 1 -- 172.21.15.86:0/893591766 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff57402d040 con 0x7ff58c15f9d0 2024-03-06T07:32:59.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 -- 172.21.15.86:0/893591766 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c15f9d0 msgr2=0x7ff58c15fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 --2- 172.21.15.86:0/893591766 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c15f9d0 0x7ff58c15fe30 secure :-1 s=READY pgs=1216 cs=0 l=1 rev1=1 crypto rx=0x7ff574009870 tx=0x7ff57402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 -- 172.21.15.86:0/893591766 shutdown_connections 2024-03-06T07:32:59.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 --2- 172.21.15.86:0/893591766 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c15f9d0 0x7ff58c15fe30 unknown :-1 s=CLOSED pgs=1216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 --2- 172.21.15.86:0/893591766 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 0x7ff58c06e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 -- 172.21.15.86:0/893591766 >> 172.21.15.86:0/893591766 conn(0x7ff58c0b1730 msgr2=0x7ff58c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:59.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 -- 172.21.15.86:0/893591766 shutdown_connections 2024-03-06T07:32:59.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.430+0000 7ff593600640 1 -- 172.21.15.86:0/893591766 wait complete. 2024-03-06T07:32:59.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.431+0000 7ff593600640 1 Processor -- start 2024-03-06T07:32:59.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.432+0000 7ff593600640 1 -- start start 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff593600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c0ce0a0 0x7ff58c139350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff593600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 0x7ff58c13d740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff593600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff58c15f9d0 0x7ff58c15d390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff593600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff58c160730 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff593600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff58c06d790 con 0x7ff58c15f9d0 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff593600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff58c06cb50 con 0x7ff58c144c00 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c0ce0a0 0x7ff58c139350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c0ce0a0 0x7ff58c139350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48372/0 (socket says 172.21.15.86:48372) 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 -- 172.21.15.86:0/937139434 learned_addr learned my addr 172.21.15.86:0/937139434 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 -- 172.21.15.86:0/937139434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 msgr2=0x7ff58c13d740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 0x7ff58c13d740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 -- 172.21.15.86:0/937139434 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff58c15f9d0 msgr2=0x7ff58c15d390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff58c15f9d0 0x7ff58c15d390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.433+0000 7ff590c00640 1 -- 172.21.15.86:0/937139434 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff58c13f580 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.434+0000 7ff590c00640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c0ce0a0 0x7ff58c139350 secure :-1 s=READY pgs=1217 cs=0 l=1 rev1=1 crypto rx=0x7ff57c00c7f0 tx=0x7ff57c00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:59.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.434+0000 7ff589600640 1 -- 172.21.15.86:0/937139434 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff57c015070 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.434+0000 7ff589600640 1 -- 172.21.15.86:0/937139434 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff57c004240 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.434+0000 7ff589600640 1 -- 172.21.15.86:0/937139434 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff57c007620 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.434+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff58c13f820 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.434+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff58c1400c0 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.435+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff58c13f9f0 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.436+0000 7ff589600640 1 -- 172.21.15.86:0/937139434 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff57c0079d0 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.437+0000 7ff589600640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff56806b4d0 0x7ff56806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.437+0000 7ff589600640 1 -- 172.21.15.86:0/937139434 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7ff57c08a440 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.438+0000 7ff583e00640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff56806b4d0 0x7ff56806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.438+0000 7ff583e00640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff56806b4d0 0x7ff56806d990 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7ff574036000 tx=0x7ff574030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:59.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.442+0000 7ff589600640 1 -- 172.21.15.86:0/937139434 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff57c059500 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.679+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff58c04f330 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.680+0000 7ff589600640 1 -- 172.21.15.86:0/937139434 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 71 v71) v1 ==== 94+0+3705 (secure 0 0 0) 0x7ff57c058e00 con 0x7ff58c0ce0a0 2024-03-06T07:32:59.682 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:32:59.682 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":71,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5228,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/1470479460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":1470479460},{"type":"v1","addr":"172.21.15.194:6835","nonce":1470479460}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":5905,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/2378871347","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":2378871347},{"type":"v1","addr":"172.21.15.194:6834","nonce":2378871347}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7185,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/3847343013","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":3847343013},{"type":"v1","addr":"172.21.15.86:6837","nonce":3847343013}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":66},{"gid":7383,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/1787919485","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":1787919485},{"type":"v1","addr":"172.21.15.86:6835","nonce":1787919485}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":71}],"filesystems":[]} 2024-03-06T07:32:59.682 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 71 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff56806b4d0 msgr2=0x7ff56806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff56806b4d0 0x7ff56806d990 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7ff574036000 tx=0x7ff574030000 comp rx=0 tx=0).stop 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c0ce0a0 msgr2=0x7ff58c139350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c0ce0a0 0x7ff58c139350 secure :-1 s=READY pgs=1217 cs=0 l=1 rev1=1 crypto rx=0x7ff57c00c7f0 tx=0x7ff57c00ccb0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 shutdown_connections 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff56806b4d0 0x7ff56806d990 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff58c15f9d0 0x7ff58c15d390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff58c144c00 0x7ff58c13d740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 --2- 172.21.15.86:0/937139434 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff58c0ce0a0 0x7ff58c139350 unknown :-1 s=CLOSED pgs=1217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 >> 172.21.15.86:0/937139434 conn(0x7ff58c0b1730 msgr2=0x7ff58c0b0eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:59.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 shutdown_connections 2024-03-06T07:32:59.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.683+0000 7ff593600640 1 -- 172.21.15.86:0/937139434 wait complete. 2024-03-06T07:32:59.702 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:32:59.702 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:32:59.703 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:32:59.703 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:59.703 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:32:59.703 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:32:59.703 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:32:59.703 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:59.704 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:32:59.704 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:32:59.704 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:32:59.704 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:59.704 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:32:59.704 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:32:59.705 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:32:59.705 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:32:59.719 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:32:59.719 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:32:59.721 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:32:59.721 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:32:59.725 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:32:59.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:32:59.736 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:32:59.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:32:59.769 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:32:59.769 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:59.774 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:32:59.775 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:59.783 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:32:59.784 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:59.789 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:32:59.790 DEBUG:teuthology.parallel:result is None 2024-03-06T07:32:59.790 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:32:59.921 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:32:59.927 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:32:59.939 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:32:59.952 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:32:59.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.971+0000 7fc190e00640 1 Processor -- start 2024-03-06T07:32:59.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.971+0000 7fc190e00640 1 -- start start 2024-03-06T07:32:59.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc190e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc18c15ac90 0x7fc18c06e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc190e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c15ea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc190e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc18c0cc000 con 0x7fc18c15ef50 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc190e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc18c0ce360 con 0x7fc18c15ac90 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc190e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc18c0ce640 con 0x7fc18c15b070 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc189e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc18c15ac90 0x7fc18c06e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc189e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc18c15ac90 0x7fc18c06e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48404/0 (socket says 172.21.15.86:48404) 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.972+0000 7fc189e00640 1 -- 172.21.15.86:0/3933289416 learned_addr learned my addr 172.21.15.86:0/3933289416 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.973+0000 7fc189400640 1 --2- 172.21.15.86:0/3933289416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c15ea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.973+0000 7fc189e00640 1 -- 172.21.15.86:0/3933289416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 msgr2=0x7fc18c15ea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.973+0000 7fc189e00640 1 --2- 172.21.15.86:0/3933289416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c15ea10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.973+0000 7fc189e00640 1 -- 172.21.15.86:0/3933289416 >> v1:172.21.15.194:6789/0 conn(0x7fc18c15ef50 legacy=0x7fc18c0cac60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.973+0000 7fc189e00640 1 -- 172.21.15.86:0/3933289416 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc18c0ce920 con 0x7fc18c15ac90 2024-03-06T07:32:59.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.973+0000 7fc189400640 1 --2- 172.21.15.86:0/3933289416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c15ea10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:32:59.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.974+0000 7fc189e00640 1 --2- 172.21.15.86:0/3933289416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c06e580 secure :-1 s=READY pgs=1225 cs=0 l=1 rev1=1 crypto rx=0x7fc17c009870 tx=0x7fc17c02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=73aa2aaf8309e41e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.974+0000 7fc188a00640 1 -- 172.21.15.86:0/3933289416 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc17c035020 con 0x7fc18c15ac90 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.974+0000 7fc188a00640 1 -- 172.21.15.86:0/3933289416 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc17c031640 con 0x7fc18c15ac90 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.974+0000 7fc188a00640 1 -- 172.21.15.86:0/3933289416 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc17c02d040 con 0x7fc18c15ac90 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 -- 172.21.15.86:0/3933289416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 msgr2=0x7fc18c06e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 --2- 172.21.15.86:0/3933289416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c06e580 secure :-1 s=READY pgs=1225 cs=0 l=1 rev1=1 crypto rx=0x7fc17c009870 tx=0x7fc17c02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 -- 172.21.15.86:0/3933289416 shutdown_connections 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 --2- 172.21.15.86:0/3933289416 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c15ea10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 --2- 172.21.15.86:0/3933289416 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c06e580 unknown :-1 s=CLOSED pgs=1225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 -- 172.21.15.86:0/3933289416 >> 172.21.15.86:0/3933289416 conn(0x7fc18c0cc4f0 msgr2=0x7fc18c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 -- 172.21.15.86:0/3933289416 shutdown_connections 2024-03-06T07:32:59.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.975+0000 7fc190e00640 1 -- 172.21.15.86:0/3933289416 wait complete. 2024-03-06T07:32:59.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.977+0000 7fc190e00640 1 Processor -- start 2024-03-06T07:32:59.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.977+0000 7fc190e00640 1 -- start start 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc190e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c1530f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc190e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c0ae510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc190e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc18c15ef50 0x7fc18c0aea50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc190e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc18c0c5a20 con 0x7fc18c15ac90 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc190e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc18c0ceee0 con 0x7fc18c15ef50 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc190e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc18c0c5ba0 con 0x7fc18c15b070 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c1530f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c1530f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48420/0 (socket says 172.21.15.86:48420) 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 -- 172.21.15.86:0/897973515 learned_addr learned my addr 172.21.15.86:0/897973515 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc18a800640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc18c15ef50 0x7fc18c0aea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189400640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c0ae510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 -- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 msgr2=0x7fc18c0ae510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c0ae510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 -- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc18c15ef50 msgr2=0x7fc18c0aea50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc18c15ef50 0x7fc18c0aea50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.978+0000 7fc189e00640 1 -- 172.21.15.86:0/897973515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc18c1f7860 con 0x7fc18c15ac90 2024-03-06T07:32:59.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.979+0000 7fc189e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c1530f0 secure :-1 s=READY pgs=1226 cs=0 l=1 rev1=1 crypto rx=0x7fc17c000c00 tx=0x7fc17c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:59.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.979+0000 7fc182000640 1 -- 172.21.15.86:0/897973515 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc17c040070 con 0x7fc18c15ac90 2024-03-06T07:32:59.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.979+0000 7fc182000640 1 -- 172.21.15.86:0/897973515 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc17c0399f0 con 0x7fc18c15ac90 2024-03-06T07:32:59.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.979+0000 7fc182000640 1 -- 172.21.15.86:0/897973515 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc17c035040 con 0x7fc18c15ac90 2024-03-06T07:32:59.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.979+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc18c1f7b40 con 0x7fc18c15ac90 2024-03-06T07:32:59.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.979+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc18c1f8100 con 0x7fc18c15ac90 2024-03-06T07:32:59.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.980+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc18c04f330 con 0x7fc18c15ac90 2024-03-06T07:32:59.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.981+0000 7fc182000640 1 -- 172.21.15.86:0/897973515 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc17c04b060 con 0x7fc18c15ac90 2024-03-06T07:32:59.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.982+0000 7fc182000640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc16c06b500 0x7fc16c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:32:59.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.982+0000 7fc182000640 1 -- 172.21.15.86:0/897973515 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fc17c0ab630 con 0x7fc18c15ac90 2024-03-06T07:32:59.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.982+0000 7fc189400640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc16c06b500 0x7fc16c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:32:59.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.983+0000 7fc189400640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc16c06b500 0x7fc16c06d9c0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7fc18c0b4720 tx=0x7fc178008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:32:59.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:32:59.986+0000 7fc182000640 1 -- 172.21.15.86:0/897973515 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc17c07a120 con 0x7fc18c15ac90 2024-03-06T07:33:00.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:00.194+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7fc18c0cd1c0 con 0x7fc18c15ac90 2024-03-06T07:33:01.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.328+0000 7fc182000640 1 -- 172.21.15.86:0/897973515 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v106) v1 ==== 97+0+0 (secure 0 0 0) 0x7fc17c0ab020 con 0x7fc18c15ac90 2024-03-06T07:33:01.330 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:33:01.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc16c06b500 msgr2=0x7fc16c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc16c06b500 0x7fc16c06d9c0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7fc18c0b4720 tx=0x7fc178008040 comp rx=0 tx=0).stop 2024-03-06T07:33:01.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 msgr2=0x7fc18c1530f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c1530f0 secure :-1 s=READY pgs=1226 cs=0 l=1 rev1=1 crypto rx=0x7fc17c000c00 tx=0x7fc17c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 shutdown_connections 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc16c06b500 0x7fc16c06d9c0 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc18c15ef50 0x7fc18c0aea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc18c15b070 0x7fc18c0ae510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 --2- 172.21.15.86:0/897973515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc18c15ac90 0x7fc18c1530f0 unknown :-1 s=CLOSED pgs=1226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.332+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 >> 172.21.15.86:0/897973515 conn(0x7fc18c0cc4f0 msgr2=0x7fc18c070d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.333+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 shutdown_connections 2024-03-06T07:33:01.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.333+0000 7fc190e00640 1 -- 172.21.15.86:0/897973515 wait complete. 2024-03-06T07:33:01.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:33:01.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.559+0000 7f09e5c00640 1 Processor -- start 2024-03-06T07:33:01.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.560+0000 7f09e5c00640 1 -- start start 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.561+0000 7f09e5c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f09e015a940 0x7f09e00b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.561+0000 7f09e5c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f09e00b2b30 con 0x7f09e00cf370 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.561+0000 7f09e5c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f09e00b2e10 con 0x7f09e00cc2a0 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.561+0000 7f09e5c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f09e00b28e0 con 0x7f09e015a940 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.561+0000 7f09dfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f09e015a940 0x7f09e00b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.562+0000 7f09df400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f09e00cf370 0x7f09e006d250 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59894/0 (socket says 172.21.15.86:59894) 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.562+0000 7f09dfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f09e015a940 0x7f09e00b45e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48436/0 (socket says 172.21.15.86:48436) 2024-03-06T07:33:01.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.562+0000 7f09df400640 1 -- 172.21.15.86:0/3438470164 learned_addr learned my addr 172.21.15.86:0/3438470164 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.562+0000 7f09dfe00640 1 -- 172.21.15.86:0/3438470164 >> v1:172.21.15.194:6789/0 conn(0x7f09e00cc2a0 legacy=0x7f09e00cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.562+0000 7f09dfe00640 1 -- 172.21.15.86:0/3438470164 >> v1:172.21.15.86:6789/0 conn(0x7f09e00cf370 legacy=0x7f09e006d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.562+0000 7f09dfe00640 1 -- 172.21.15.86:0/3438470164 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09e006ca50 con 0x7f09e015a940 2024-03-06T07:33:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.563+0000 7f09dfe00640 1 --2- 172.21.15.86:0/3438470164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e015a940 0x7f09e00b45e0 secure :-1 s=READY pgs=1228 cs=0 l=1 rev1=1 crypto rx=0x7f09d00097a0 tx=0x7f09d002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5f75b027d12acc14 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.563+0000 7f09dea00640 1 -- 172.21.15.86:0/3438470164 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d0035020 con 0x7f09e015a940 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.563+0000 7f09dea00640 1 -- 172.21.15.86:0/3438470164 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f09d0031830 con 0x7f09e015a940 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.563+0000 7f09dea00640 1 -- 172.21.15.86:0/3438470164 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d0031c00 con 0x7f09e015a940 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.564+0000 7f09e5c00640 1 -- 172.21.15.86:0/3438470164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e015a940 msgr2=0x7f09e00b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.564+0000 7f09e5c00640 1 --2- 172.21.15.86:0/3438470164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e015a940 0x7f09e00b45e0 secure :-1 s=READY pgs=1228 cs=0 l=1 rev1=1 crypto rx=0x7f09d00097a0 tx=0x7f09d002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.564+0000 7f09e5c00640 1 -- 172.21.15.86:0/3438470164 shutdown_connections 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.564+0000 7f09e5c00640 1 --2- 172.21.15.86:0/3438470164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e015a940 0x7f09e00b45e0 unknown :-1 s=CLOSED pgs=1228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.564+0000 7f09e5c00640 1 -- 172.21.15.86:0/3438470164 >> 172.21.15.86:0/3438470164 conn(0x7f09e006afc0 msgr2=0x7f09e006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.564+0000 7f09e5c00640 1 -- 172.21.15.86:0/3438470164 shutdown_connections 2024-03-06T07:33:01.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.564+0000 7f09e5c00640 1 -- 172.21.15.86:0/3438470164 wait complete. 2024-03-06T07:33:01.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.566+0000 7f09e5c00640 1 Processor -- start 2024-03-06T07:33:01.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.566+0000 7f09e5c00640 1 -- start start 2024-03-06T07:33:01.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09e5c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e00cc2a0 0x7f09e015b3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:01.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09e5c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09e00cf370 0x7f09e01520d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:01.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09e5c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09e014e310 0x7f09e0152610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:01.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09e5c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09e00b4b20 con 0x7f09e00cc2a0 2024-03-06T07:33:01.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09e5c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09e00b30f0 con 0x7f09e014e310 2024-03-06T07:33:01.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09e5c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09e015ad20 con 0x7f09e00cf370 2024-03-06T07:33:01.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e00cc2a0 0x7f09e015b3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:01.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e00cc2a0 0x7f09e015b3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48440/0 (socket says 172.21.15.86:48440) 2024-03-06T07:33:01.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 -- 172.21.15.86:0/2292180006 learned_addr learned my addr 172.21.15.86:0/2292180006 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:01.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 -- 172.21.15.86:0/2292180006 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09e00cf370 msgr2=0x7f09e01520d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09e00cf370 0x7f09e01520d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 -- 172.21.15.86:0/2292180006 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09e014e310 msgr2=0x7f09e0152610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09e014e310 0x7f09e0152610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.567+0000 7f09dfe00640 1 -- 172.21.15.86:0/2292180006 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09e015c550 con 0x7f09e00cc2a0 2024-03-06T07:33:01.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.568+0000 7f09dfe00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e00cc2a0 0x7f09e015b3c0 secure :-1 s=READY pgs=1229 cs=0 l=1 rev1=1 crypto rx=0x7f09d0009890 tx=0x7f09d00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:01.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.568+0000 7f09d7e00640 1 -- 172.21.15.86:0/2292180006 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d003ba70 con 0x7f09e00cc2a0 2024-03-06T07:33:01.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.568+0000 7f09d7e00640 1 -- 172.21.15.86:0/2292180006 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f09d003bc10 con 0x7f09e00cc2a0 2024-03-06T07:33:01.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.568+0000 7f09d7e00640 1 -- 172.21.15.86:0/2292180006 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d00453f0 con 0x7f09e00cc2a0 2024-03-06T07:33:01.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.568+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f09e015c830 con 0x7f09e00cc2a0 2024-03-06T07:33:01.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.568+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f09e015cdf0 con 0x7f09e00cc2a0 2024-03-06T07:33:01.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.569+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f09e004f330 con 0x7f09e00cc2a0 2024-03-06T07:33:01.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.570+0000 7f09d7e00640 1 -- 172.21.15.86:0/2292180006 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f09d0038020 con 0x7f09e00cc2a0 2024-03-06T07:33:01.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.571+0000 7f09d7e00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f09bc06b500 0x7f09bc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:01.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.571+0000 7f09d7e00640 1 -- 172.21.15.86:0/2292180006 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(106..106 src has 1..106) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f09d00abc20 con 0x7f09e00cc2a0 2024-03-06T07:33:01.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.571+0000 7f09df400640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f09bc06b500 0x7f09bc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:01.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.572+0000 7f09df400640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f09bc06b500 0x7f09bc06d9c0 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f09c8005c00 tx=0x7f09c80015b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:01.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.575+0000 7f09d7e00640 1 -- 172.21.15.86:0/2292180006 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f09d0040070 con 0x7f09e00cc2a0 2024-03-06T07:33:01.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.844+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f09e0072840 con 0x7f09e00cc2a0 2024-03-06T07:33:01.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.845+0000 7f09d7e00640 1 -- 172.21.15.86:0/2292180006 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v18) v1 ==== 80+0+5304 (secure 0 0 0) 0x7f09d007ad50 con 0x7f09e00cc2a0 2024-03-06T07:33:01.847 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:01.847 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:33:01.847 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:33:01.847 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.847 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:33:01.847 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:33:01.848 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.849 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.850 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.851 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:33:01.852 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.853 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.854 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.855 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow r", 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.856 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:33:01.857 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.858 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:33:01.859 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:33:01.860 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.860 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:01.860 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:33:01.860 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f09bc06b500 msgr2=0x7f09bc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f09bc06b500 0x7f09bc06d9c0 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f09c8005c00 tx=0x7f09c80015b0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e00cc2a0 msgr2=0x7f09e015b3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e00cc2a0 0x7f09e015b3c0 secure :-1 s=READY pgs=1229 cs=0 l=1 rev1=1 crypto rx=0x7f09d0009890 tx=0x7f09d00046c0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 shutdown_connections 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f09bc06b500 0x7f09bc06d9c0 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09e014e310 0x7f09e0152610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09e00cf370 0x7f09e01520d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 --2- 172.21.15.86:0/2292180006 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09e00cc2a0 0x7f09e015b3c0 unknown :-1 s=CLOSED pgs=1229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 >> 172.21.15.86:0/2292180006 conn(0x7f09e006afc0 msgr2=0x7f09e0070a20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 shutdown_connections 2024-03-06T07:33:01.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:01.848+0000 7f09e5c00640 1 -- 172.21.15.86:0/2292180006 wait complete. 2024-03-06T07:33:01.866 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:33:02.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.063+0000 7f1cf1000640 1 Processor -- start 2024-03-06T07:33:02.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.063+0000 7f1cf1000640 1 -- start start 2024-03-06T07:33:02.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.064+0000 7f1cf1000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1cec144c00 0x7f1cec06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.064+0000 7f1cf1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec15fe60 0x7f1cec0ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1cf1000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1cec06ee70 con 0x7f1cec15f9d0 2024-03-06T07:33:02.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1cf1000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1cec06cef0 con 0x7f1cec144c00 2024-03-06T07:33:02.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1cf1000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1cec06d1d0 con 0x7f1cec15fe60 2024-03-06T07:33:02.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1ce1400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1cec15f9d0 0x7f1cec0ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59908/0 (socket says 172.21.15.86:59908) 2024-03-06T07:33:02.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1ce1400640 1 -- 172.21.15.86:0/2696361914 learned_addr learned my addr 172.21.15.86:0/2696361914 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:02.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1cea800640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec15fe60 0x7f1cec0ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1ce9e00640 1 --2- 172.21.15.86:0/2696361914 >> v2:172.21.15.194:3300/0 conn(0x7f1cec144c00 0x7f1cec06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 26546052 0 0) 0x7f1cd4001110 con 0x7f1cec15f9d0 2024-03-06T07:33:02.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.065+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1cd0003520 con 0x7f1cec15f9d0 2024-03-06T07:33:02.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3158563954 0 0) 0x7f1cd0003520 con 0x7f1cec15f9d0 2024-03-06T07:33:02.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec15fe60 msgr2=0x7f1cec0ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec15fe60 0x7f1cec0ce7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 >> v2:172.21.15.194:3300/0 conn(0x7f1cec144c00 msgr2=0x7f1cec06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 --2- 172.21.15.86:0/2696361914 >> v2:172.21.15.194:3300/0 conn(0x7f1cec144c00 0x7f1cec06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cec06d4b0 con 0x7f1cec15f9d0 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1cd4001840 con 0x7f1cec15f9d0 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 >> v1:172.21.15.86:6789/0 conn(0x7f1cec15f9d0 legacy=0x7f1cec0ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cd00039b0 0x7f1cd0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cd0004800 0x7f1cd0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec15fe60 0x7f1cd00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cec06cef0 con 0x7f1cd0004800 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cec06d1d0 con 0x7f1cd00039b0 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1cd000b9c0 con 0x7f1cec15fe60 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce1400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cd0004800 0x7f1cd0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce1400640 1 -- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec15fe60 msgr2=0x7f1cd00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce1400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec15fe60 0x7f1cd00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce1400640 1 -- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cd00039b0 msgr2=0x7f1cd0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce1400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cd00039b0 0x7f1cd0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.066+0000 7f1ce1400640 1 -- 172.21.15.86:0/2696361914 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cd0003520 con 0x7f1cd0004800 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.067+0000 7f1ce1400640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cd0004800 0x7f1cd0004c80 secure :-1 s=READY pgs=1231 cs=0 l=1 rev1=1 crypto rx=0x7f1cd400c320 tx=0x7f1cd0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c3554c7b1cd9adf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cd402cc20 con 0x7f1cd0004800 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cd4030070 con 0x7f1cd0004800 2024-03-06T07:33:02.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1ce9400640 1 -- 172.21.15.86:0/2696361914 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cd40058d0 con 0x7f1cd0004800 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1cf1000640 1 -- 172.21.15.86:0/2696361914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cd0004800 msgr2=0x7f1cd0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1cf1000640 1 --2- 172.21.15.86:0/2696361914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cd0004800 0x7f1cd0004c80 secure :-1 s=READY pgs=1231 cs=0 l=1 rev1=1 crypto rx=0x7f1cd400c320 tx=0x7f1cd0005830 comp rx=0 tx=0).stop 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1cea800640 1 -- 172.21.15.86:0/2696361914 reap_dead start 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1cf1000640 1 -- 172.21.15.86:0/2696361914 shutdown_connections 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1cf1000640 1 -- 172.21.15.86:0/2696361914 >> 172.21.15.86:0/2696361914 conn(0x7f1cec0b1730 msgr2=0x7f1cec0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1cf1000640 1 -- 172.21.15.86:0/2696361914 shutdown_connections 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.068+0000 7f1cf1000640 1 -- 172.21.15.86:0/2696361914 wait complete. 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.070+0000 7f1cf1000640 1 Processor -- start 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.070+0000 7f1cf1000640 1 -- start start 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1cf1000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cd0004800 0x7f1cec15b540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1cf1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cec141ec0 0x7f1cec1f4400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1cf1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec142340 0x7f1cec1f4940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1cf1000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cec160730 con 0x7f1cec141ec0 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1cf1000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cec160290 con 0x7f1cd0004800 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1cf1000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1cec06cb50 con 0x7f1cec142340 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1ce1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cec141ec0 0x7f1cec1f4400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1ce1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cec141ec0 0x7f1cec1f4400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48452/0 (socket says 172.21.15.86:48452) 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.071+0000 7f1ce1400640 1 -- 172.21.15.86:0/1693431703 learned_addr learned my addr 172.21.15.86:0/1693431703 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce1400640 1 -- 172.21.15.86:0/1693431703 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec142340 msgr2=0x7f1cec1f4940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce1400640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec142340 0x7f1cec1f4940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce1400640 1 -- 172.21.15.86:0/1693431703 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cd0004800 msgr2=0x7f1cec15b540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce1400640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cd0004800 0x7f1cec15b540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce1400640 1 -- 172.21.15.86:0/1693431703 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cec13f990 con 0x7f1cec141ec0 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce1400640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cec141ec0 0x7f1cec1f4400 secure :-1 s=READY pgs=1232 cs=0 l=1 rev1=1 crypto rx=0x7f1cd400c440 tx=0x7f1cd4001b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce2a00640 1 -- 172.21.15.86:0/1693431703 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cd4004bf0 con 0x7f1cec141ec0 2024-03-06T07:33:02.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce2a00640 1 -- 172.21.15.86:0/1693431703 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cd4007e50 con 0x7f1cec141ec0 2024-03-06T07:33:02.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1cec13fc70 con 0x7f1cec141ec0 2024-03-06T07:33:02.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1ce2a00640 1 -- 172.21.15.86:0/1693431703 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cd4007070 con 0x7f1cec141ec0 2024-03-06T07:33:02.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.072+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1cec140230 con 0x7f1cec141ec0 2024-03-06T07:33:02.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.073+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1cec04f330 con 0x7f1cec141ec0 2024-03-06T07:33:02.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.074+0000 7f1ce2a00640 1 -- 172.21.15.86:0/1693431703 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1cd4047020 con 0x7f1cec141ec0 2024-03-06T07:33:02.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.075+0000 7f1ce2a00640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cc806b500 0x7f1cc806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.076+0000 7f1ce2a00640 1 -- 172.21.15.86:0/1693431703 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(106..106 src has 1..106) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f1cd40aafe0 con 0x7f1cec141ec0 2024-03-06T07:33:02.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.076+0000 7f1ce9e00640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cc806b500 0x7f1cc806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.076+0000 7f1ce9e00640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cc806b500 0x7f1cc806d9c0 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f1cdc006a90 tx=0x7f1cdc006a20 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.080+0000 7f1ce2a00640 1 -- 172.21.15.86:0/1693431703 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1cec06ee70 con 0x7f1cec141ec0 2024-03-06T07:33:02.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.276+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1cec15f7b0 con 0x7f1cec141ec0 2024-03-06T07:33:02.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.278+0000 7f1ce2a00640 1 -- 172.21.15.86:0/1693431703 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v106) v1 ==== 74+0+12905 (secure 0 0 0) 0x7f1cd407a1b0 con 0x7f1cec141ec0 2024-03-06T07:33:02.280 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:02.280 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":106,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:01.325666+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:33:02.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cc806b500 msgr2=0x7f1cc806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cc806b500 0x7f1cc806d9c0 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f1cdc006a90 tx=0x7f1cdc006a20 comp rx=0 tx=0).stop 2024-03-06T07:33:02.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cec141ec0 msgr2=0x7f1cec1f4400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cec141ec0 0x7f1cec1f4400 secure :-1 s=READY pgs=1232 cs=0 l=1 rev1=1 crypto rx=0x7f1cd400c440 tx=0x7f1cd4001b40 comp rx=0 tx=0).stop 2024-03-06T07:33:02.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 shutdown_connections 2024-03-06T07:33:02.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1cc806b500 0x7f1cc806d9c0 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1cec142340 0x7f1cec1f4940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1cec141ec0 0x7f1cec1f4400 unknown :-1 s=CLOSED pgs=1232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 --2- 172.21.15.86:0/1693431703 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1cd0004800 0x7f1cec15b540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 >> 172.21.15.86:0/1693431703 conn(0x7f1cec0b1730 msgr2=0x7f1cec0b0100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:02.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 shutdown_connections 2024-03-06T07:33:02.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.281+0000 7f1cf1000640 1 -- 172.21.15.86:0/1693431703 wait complete. 2024-03-06T07:33:02.301 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:33:02.301 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:33:02.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.503+0000 7fd78da00640 1 Processor -- start 2024-03-06T07:33:02.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.504+0000 7fd78da00640 1 -- start start 2024-03-06T07:33:02.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.505+0000 7fd78da00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7880b2c40 0x7fd78815f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.505+0000 7fd78da00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd78806cba0 con 0x7fd78806e2f0 2024-03-06T07:33:02.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.505+0000 7fd78da00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd78806ce80 con 0x7fd78806d960 2024-03-06T07:33:02.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.505+0000 7fd78da00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd78815f700 con 0x7fd7880b2c40 2024-03-06T07:33:02.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd787400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd78806e2f0 0x7fd7880cda90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59914/0 (socket says 172.21.15.86:59914) 2024-03-06T07:33:02.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd787400640 1 -- 172.21.15.86:0/900357077 learned_addr learned my addr 172.21.15.86:0/900357077 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd786a00640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7880b2c40 0x7fd78815f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3283785613 0 0) 0x7fd78806cba0 con 0x7fd78806e2f0 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd76c003520 con 0x7fd78806e2f0 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 183870551 0 0) 0x7fd76c003520 con 0x7fd78806e2f0 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7880b2c40 msgr2=0x7fd78815f1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7880b2c40 0x7fd78815f1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 >> v1:172.21.15.194:6789/0 conn(0x7fd78806d960 legacy=0x7fd7880b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd78806d160 con 0x7fd78806e2f0 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd780001670 con 0x7fd78806e2f0 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 >> v1:172.21.15.86:6789/0 conn(0x7fd78806e2f0 legacy=0x7fd7880cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7880b2c40 0x7fd76c0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd76c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.506+0000 7fd785600640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd76c004d40 0x7fd76c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd76c00ba70 con 0x7fd76c003f60 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd76c00bc20 con 0x7fd76c004d40 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd76c00be20 con 0x7fd7880b2c40 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd786000640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd76c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd786000640 1 -- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7880b2c40 msgr2=0x7fd76c0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd786000640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7880b2c40 0x7fd76c0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd786000640 1 -- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd76c004d40 msgr2=0x7fd76c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd786000640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd76c004d40 0x7fd76c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.507+0000 7fd786000640 1 -- 172.21.15.86:0/900357077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd76c003520 con 0x7fd76c003f60 2024-03-06T07:33:02.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.508+0000 7fd786000640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd76c004800 secure :-1 s=READY pgs=1234 cs=0 l=1 rev1=1 crypto rx=0x7fd778001e50 tx=0x7fd76c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1ccb60f265667cc9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.508+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd778001a70 con 0x7fd76c003f60 2024-03-06T07:33:02.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.508+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd778003b70 con 0x7fd76c003f60 2024-03-06T07:33:02.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.508+0000 7fd785600640 1 -- 172.21.15.86:0/900357077 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7780050d0 con 0x7fd76c003f60 2024-03-06T07:33:02.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.509+0000 7fd78da00640 1 -- 172.21.15.86:0/900357077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 msgr2=0x7fd76c004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.509+0000 7fd78da00640 1 --2- 172.21.15.86:0/900357077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd76c004800 secure :-1 s=READY pgs=1234 cs=0 l=1 rev1=1 crypto rx=0x7fd778001e50 tx=0x7fd76c005830 comp rx=0 tx=0).stop 2024-03-06T07:33:02.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.509+0000 7fd787400640 1 -- 172.21.15.86:0/900357077 reap_dead start 2024-03-06T07:33:02.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.509+0000 7fd78da00640 1 -- 172.21.15.86:0/900357077 shutdown_connections 2024-03-06T07:33:02.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.509+0000 7fd78da00640 1 -- 172.21.15.86:0/900357077 >> 172.21.15.86:0/900357077 conn(0x7fd7880b1940 msgr2=0x7fd7880b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:02.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.509+0000 7fd78da00640 1 -- 172.21.15.86:0/900357077 shutdown_connections 2024-03-06T07:33:02.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.509+0000 7fd78da00640 1 -- 172.21.15.86:0/900357077 wait complete. 2024-03-06T07:33:02.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.511+0000 7fd78da00640 1 Processor -- start 2024-03-06T07:33:02.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.511+0000 7fd78da00640 1 -- start start 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd78da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd78809ad20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd78da00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7881430c0 0x7fd7881610b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd78da00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7881615f0 0x7fd788161aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd78da00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd78806d4d0 con 0x7fd76c003f60 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd78da00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd78806e9a0 con 0x7fd7881615f0 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd78da00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd78815f1c0 con 0x7fd7881430c0 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd78809ad20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd78809ad20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48466/0 (socket says 172.21.15.86:48466) 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 -- 172.21.15.86:0/3128272594 learned_addr learned my addr 172.21.15.86:0/3128272594 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 -- 172.21.15.86:0/3128272594 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7881430c0 msgr2=0x7fd7881610b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7881430c0 0x7fd7881610b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 -- 172.21.15.86:0/3128272594 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7881615f0 msgr2=0x7fd788161aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7881615f0 0x7fd788161aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.512+0000 7fd786a00640 1 -- 172.21.15.86:0/3128272594 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd788161fe0 con 0x7fd76c003f60 2024-03-06T07:33:02.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.513+0000 7fd786a00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd78809ad20 secure :-1 s=READY pgs=1235 cs=0 l=1 rev1=1 crypto rx=0x7fd77000e7f0 tx=0x7fd77000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.513+0000 7fd77ea00640 1 -- 172.21.15.86:0/3128272594 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd770017070 con 0x7fd76c003f60 2024-03-06T07:33:02.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.513+0000 7fd77ea00640 1 -- 172.21.15.86:0/3128272594 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd77000c940 con 0x7fd76c003f60 2024-03-06T07:33:02.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.513+0000 7fd77ea00640 1 -- 172.21.15.86:0/3128272594 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd77000cd10 con 0x7fd76c003f60 2024-03-06T07:33:02.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.513+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd78815f6f0 con 0x7fd76c003f60 2024-03-06T07:33:02.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.513+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd7880acf70 con 0x7fd76c003f60 2024-03-06T07:33:02.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.514+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd78804f330 con 0x7fd76c003f60 2024-03-06T07:33:02.516 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.515+0000 7fd77ea00640 1 -- 172.21.15.86:0/3128272594 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd77001a070 con 0x7fd76c003f60 2024-03-06T07:33:02.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.516+0000 7fd77ea00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd76406b500 0x7fd76406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.516+0000 7fd77ea00640 1 -- 172.21.15.86:0/3128272594 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(106..106 src has 1..106) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fd770089d30 con 0x7fd76c003f60 2024-03-06T07:33:02.517 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.516+0000 7fd786000640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd76406b500 0x7fd76406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.517+0000 7fd786000640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd76406b500 0x7fd76406d9c0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7fd778034000 tx=0x7fd778030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.520+0000 7fd77ea00640 1 -- 172.21.15.86:0/3128272594 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd788161fe0 con 0x7fd76c003f60 2024-03-06T07:33:02.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.718+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd78815c540 con 0x7fd76c003f60 2024-03-06T07:33:02.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.719+0000 7fd77ea00640 1 -- 172.21.15.86:0/3128272594 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v106) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd770058ef0 con 0x7fd76c003f60 2024-03-06T07:33:02.721 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd76406b500 msgr2=0x7fd76406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd76406b500 0x7fd76406d9c0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7fd778034000 tx=0x7fd778030000 comp rx=0 tx=0).stop 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 msgr2=0x7fd78809ad20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd78809ad20 secure :-1 s=READY pgs=1235 cs=0 l=1 rev1=1 crypto rx=0x7fd77000e7f0 tx=0x7fd77000ecb0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 shutdown_connections 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd76406b500 0x7fd76406d9c0 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd7881615f0 0x7fd788161aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7881430c0 0x7fd7881610b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 --2- 172.21.15.86:0/3128272594 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd76c003f60 0x7fd78809ad20 unknown :-1 s=CLOSED pgs=1235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 >> 172.21.15.86:0/3128272594 conn(0x7fd7880b1940 msgr2=0x7fd7880b3d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:02.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 shutdown_connections 2024-03-06T07:33:02.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.722+0000 7fd78da00640 1 -- 172.21.15.86:0/3128272594 wait complete. 2024-03-06T07:33:02.742 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:33:02.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.935+0000 7fc411000640 1 Processor -- start 2024-03-06T07:33:02.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.936+0000 7fc411000640 1 -- start start 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc411000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc40c161380 0x7fc40c0ce430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc411000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc40c161760 0x7fc40c0ce970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc411000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc40c0cf000 con 0x7fc40c0b26a0 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc411000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc40c0cd570 con 0x7fc40c161760 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc411000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc40c0cd850 con 0x7fc40c161380 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc402a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc40c161760 0x7fc40c0ce970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc402a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc40c161760 0x7fc40c0ce970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48482/0 (socket says 172.21.15.86:48482) 2024-03-06T07:33:02.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.937+0000 7fc402a00640 1 -- 172.21.15.86:0/3998618232 learned_addr learned my addr 172.21.15.86:0/3998618232 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:02.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.938+0000 7fc402a00640 1 -- 172.21.15.86:0/3998618232 >> v2:172.21.15.194:3300/0 conn(0x7fc40c161380 msgr2=0x7fc40c0ce430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.938+0000 7fc402a00640 1 --2- 172.21.15.86:0/3998618232 >> v2:172.21.15.194:3300/0 conn(0x7fc40c161380 0x7fc40c0ce430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.938+0000 7fc402a00640 1 -- 172.21.15.86:0/3998618232 >> v1:172.21.15.194:6789/0 conn(0x7fc40c0b26a0 legacy=0x7fc40c161c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:02.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.938+0000 7fc402a00640 1 -- 172.21.15.86:0/3998618232 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc40c0cdb30 con 0x7fc40c161760 2024-03-06T07:33:02.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.939+0000 7fc402a00640 1 --2- 172.21.15.86:0/3998618232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c0ce970 secure :-1 s=READY pgs=1236 cs=0 l=1 rev1=1 crypto rx=0x7fc40c071ca0 tx=0x7fc3f402b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=10b9307b1957475c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.939+0000 7fc40aa00640 1 -- 172.21.15.86:0/3998618232 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f402e070 con 0x7fc40c161760 2024-03-06T07:33:02.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.939+0000 7fc40aa00640 1 -- 172.21.15.86:0/3998618232 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3f4037070 con 0x7fc40c161760 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.939+0000 7fc40aa00640 1 -- 172.21.15.86:0/3998618232 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f4031a00 con 0x7fc40c161760 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.939+0000 7fc411000640 1 -- 172.21.15.86:0/3998618232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 msgr2=0x7fc40c0ce970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.939+0000 7fc411000640 1 --2- 172.21.15.86:0/3998618232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c0ce970 secure :-1 s=READY pgs=1236 cs=0 l=1 rev1=1 crypto rx=0x7fc40c071ca0 tx=0x7fc3f402b880 comp rx=0 tx=0).stop 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.940+0000 7fc411000640 1 -- 172.21.15.86:0/3998618232 shutdown_connections 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.940+0000 7fc411000640 1 --2- 172.21.15.86:0/3998618232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c0ce970 unknown :-1 s=CLOSED pgs=1236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.940+0000 7fc411000640 1 --2- 172.21.15.86:0/3998618232 >> v2:172.21.15.194:3300/0 conn(0x7fc40c161380 0x7fc40c0ce430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.940+0000 7fc411000640 1 -- 172.21.15.86:0/3998618232 >> 172.21.15.86:0/3998618232 conn(0x7fc40c0b3690 msgr2=0x7fc40c0b3aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.940+0000 7fc411000640 1 -- 172.21.15.86:0/3998618232 shutdown_connections 2024-03-06T07:33:02.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.940+0000 7fc411000640 1 -- 172.21.15.86:0/3998618232 wait complete. 2024-03-06T07:33:02.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.941+0000 7fc411000640 1 Processor -- start 2024-03-06T07:33:02.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.941+0000 7fc411000640 1 -- start start 2024-03-06T07:33:02.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.942+0000 7fc411000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c0b26a0 0x7fc40c160780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.942+0000 7fc411000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c161380 0x7fc40c0b6830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.942+0000 7fc411000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c154720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.942+0000 7fc411000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc40c0d0a50 con 0x7fc40c161760 2024-03-06T07:33:02.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.942+0000 7fc411000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc40c0d08d0 con 0x7fc40c0b26a0 2024-03-06T07:33:02.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.942+0000 7fc411000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc40c0d0bd0 con 0x7fc40c161380 2024-03-06T07:33:02.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c154720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c154720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48486/0 (socket says 172.21.15.86:48486) 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 -- 172.21.15.86:0/30244048 learned_addr learned my addr 172.21.15.86:0/30244048 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc402a00640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c161380 0x7fc40c0b6830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 -- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c161380 msgr2=0x7fc40c0b6830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40b400640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c0b26a0 0x7fc40c160780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c161380 0x7fc40c0b6830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 -- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c0b26a0 msgr2=0x7fc40c160780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c0b26a0 0x7fc40c160780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 -- 172.21.15.86:0/30244048 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc40c0ca0c0 con 0x7fc40c161760 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc402a00640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c161380 0x7fc40c0b6830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc40be00640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c154720 secure :-1 s=READY pgs=1237 cs=0 l=1 rev1=1 crypto rx=0x7fc40400b990 tx=0x7fc40400be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.943+0000 7fc403e00640 1 -- 172.21.15.86:0/30244048 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc404019070 con 0x7fc40c161760 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.944+0000 7fc403e00640 1 -- 172.21.15.86:0/30244048 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc40400cba0 con 0x7fc40c161760 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.944+0000 7fc403e00640 1 -- 172.21.15.86:0/30244048 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc404014440 con 0x7fc40c161760 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.944+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc40c0b6f90 con 0x7fc40c161760 2024-03-06T07:33:02.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.944+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc40c0ca640 con 0x7fc40c161760 2024-03-06T07:33:02.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.945+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc40c04f330 con 0x7fc40c161760 2024-03-06T07:33:02.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.946+0000 7fc403e00640 1 -- 172.21.15.86:0/30244048 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc40401c070 con 0x7fc40c161760 2024-03-06T07:33:02.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.947+0000 7fc403e00640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3e806b4d0 0x7fc3e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:02.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.947+0000 7fc403e00640 1 -- 172.21.15.86:0/30244048 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(106..106 src has 1..106) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fc404089c40 con 0x7fc40c161760 2024-03-06T07:33:02.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.947+0000 7fc40b400640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3e806b4d0 0x7fc3e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:02.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.948+0000 7fc40b400640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3e806b4d0 0x7fc3e806d990 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7fc3fc00a770 tx=0x7fc3fc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:02.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:02.952+0000 7fc403e00640 1 -- 172.21.15.86:0/30244048 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc404058e00 con 0x7fc40c161760 2024-03-06T07:33:03.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.137+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7fc40c0b7ac0 con 0x7fc40c161760 2024-03-06T07:33:03.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.138+0000 7fc403e00640 1 -- 172.21.15.86:0/30244048 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v106) v1 ==== 84+0+65 (secure 0 0 0) 0x7fc404058700 con 0x7fc40c161760 2024-03-06T07:33:03.139 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:03.139 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:33:03.139 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:03.139 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:33:03.140 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:33:03.140 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:03.140 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:33:03.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.140+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3e806b4d0 msgr2=0x7fc3e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.140+0000 7fc411000640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3e806b4d0 0x7fc3e806d990 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7fc3fc00a770 tx=0x7fc3fc008040 comp rx=0 tx=0).stop 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.140+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 msgr2=0x7fc40c154720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.140+0000 7fc411000640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c154720 secure :-1 s=READY pgs=1237 cs=0 l=1 rev1=1 crypto rx=0x7fc40400b990 tx=0x7fc40400be50 comp rx=0 tx=0).stop 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 shutdown_connections 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3e806b4d0 0x7fc3e806d990 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c161760 0x7fc40c154720 unknown :-1 s=CLOSED pgs=1237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c161380 0x7fc40c0b6830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 --2- 172.21.15.86:0/30244048 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c0b26a0 0x7fc40c160780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 >> 172.21.15.86:0/30244048 conn(0x7fc40c0b3690 msgr2=0x7fc40c06c9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 shutdown_connections 2024-03-06T07:33:03.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.141+0000 7fc411000640 1 -- 172.21.15.86:0/30244048 wait complete. 2024-03-06T07:33:03.160 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:33:03.161 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:33:03.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.362+0000 7f4333000640 1 Processor -- start 2024-03-06T07:33:03.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.363+0000 7f4333000640 1 -- start start 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f4333000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f432c15e170 0x7f432c0ad860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f4333000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f432c06e570 0x7f432c0b4450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f4333000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f432c0b3690 con 0x7f432c0b4990 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f4333000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f432c06ea70 con 0x7f432c15e170 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f4333000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f432c0b3970 con 0x7f432c06e570 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f432be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f432c15e170 0x7f432c0ad860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f432be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f432c15e170 0x7f432c0ad860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48490/0 (socket says 172.21.15.86:48490) 2024-03-06T07:33:03.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.364+0000 7f432be00640 1 -- 172.21.15.86:0/3339564769 learned_addr learned my addr 172.21.15.86:0/3339564769 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:03.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.365+0000 7f432be00640 1 -- 172.21.15.86:0/3339564769 >> v2:172.21.15.194:3300/0 conn(0x7f432c06e570 msgr2=0x7f432c0b4450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:03.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.365+0000 7f432be00640 1 --2- 172.21.15.86:0/3339564769 >> v2:172.21.15.194:3300/0 conn(0x7f432c06e570 0x7f432c0b4450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.365+0000 7f432be00640 1 -- 172.21.15.86:0/3339564769 >> v1:172.21.15.194:6789/0 conn(0x7f432c0b4990 legacy=0x7f432c06cc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:03.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.365+0000 7f432be00640 1 -- 172.21.15.86:0/3339564769 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f432c0b3c50 con 0x7f432c15e170 2024-03-06T07:33:03.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.366+0000 7f432be00640 1 --2- 172.21.15.86:0/3339564769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c15e170 0x7f432c0ad860 secure :-1 s=READY pgs=1238 cs=0 l=1 rev1=1 crypto rx=0x7f432c072f50 tx=0x7f431c02be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e2dc458c4b5c457 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:03.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.366+0000 7f432aa00640 1 -- 172.21.15.86:0/3339564769 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f431c02c9f0 con 0x7f432c15e170 2024-03-06T07:33:03.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.366+0000 7f432aa00640 1 -- 172.21.15.86:0/3339564769 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f431c02cb90 con 0x7f432c15e170 2024-03-06T07:33:03.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.366+0000 7f432aa00640 1 -- 172.21.15.86:0/3339564769 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f431c0343f0 con 0x7f432c15e170 2024-03-06T07:33:03.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 -- 172.21.15.86:0/3339564769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c15e170 msgr2=0x7f432c0ad860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:03.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 --2- 172.21.15.86:0/3339564769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c15e170 0x7f432c0ad860 secure :-1 s=READY pgs=1238 cs=0 l=1 rev1=1 crypto rx=0x7f432c072f50 tx=0x7f431c02be50 comp rx=0 tx=0).stop 2024-03-06T07:33:03.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 -- 172.21.15.86:0/3339564769 shutdown_connections 2024-03-06T07:33:03.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 --2- 172.21.15.86:0/3339564769 >> v2:172.21.15.194:3300/0 conn(0x7f432c06e570 0x7f432c0b4450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 --2- 172.21.15.86:0/3339564769 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c15e170 0x7f432c0ad860 unknown :-1 s=CLOSED pgs=1238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 -- 172.21.15.86:0/3339564769 >> 172.21.15.86:0/3339564769 conn(0x7f432c0cfa80 msgr2=0x7f432c0cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:03.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 -- 172.21.15.86:0/3339564769 shutdown_connections 2024-03-06T07:33:03.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.367+0000 7f4333000640 1 -- 172.21.15.86:0/3339564769 wait complete. 2024-03-06T07:33:03.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.369+0000 7f4333000640 1 Processor -- start 2024-03-06T07:33:03.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.369+0000 7f4333000640 1 -- start start 2024-03-06T07:33:03.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f4333000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c06e570 0x7f432c1f7940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:03.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f4333000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f432c0b4990 0x7f432c147090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:03.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f4333000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f432c15e170 0x7f432c147990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:03.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f4333000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f432c06df80 con 0x7f432c06e570 2024-03-06T07:33:03.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f4333000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f432c0b4dd0 con 0x7f432c0b4990 2024-03-06T07:33:03.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f4333000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f432c0adda0 con 0x7f432c15e170 2024-03-06T07:33:03.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c06e570 0x7f432c1f7940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:03.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c06e570 0x7f432c1f7940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48506/0 (socket says 172.21.15.86:48506) 2024-03-06T07:33:03.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 -- 172.21.15.86:0/2157385159 learned_addr learned my addr 172.21.15.86:0/2157385159 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 -- 172.21.15.86:0/2157385159 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f432c15e170 msgr2=0x7f432c147990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f432c15e170 0x7f432c147990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 -- 172.21.15.86:0/2157385159 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f432c0b4990 msgr2=0x7f432c147090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f432c0b4990 0x7f432c147090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.370+0000 7f432be00640 1 -- 172.21.15.86:0/2157385159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f432c0c4e10 con 0x7f432c06e570 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.371+0000 7f432be00640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c06e570 0x7f432c1f7940 secure :-1 s=READY pgs=1239 cs=0 l=1 rev1=1 crypto rx=0x7f431c0093e0 tx=0x7f431c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.371+0000 7f4323e00640 1 -- 172.21.15.86:0/2157385159 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f431c043020 con 0x7f432c06e570 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.371+0000 7f4323e00640 1 -- 172.21.15.86:0/2157385159 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f431c03e450 con 0x7f432c06e570 2024-03-06T07:33:03.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.371+0000 7f4323e00640 1 -- 172.21.15.86:0/2157385159 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f431c03e840 con 0x7f432c06e570 2024-03-06T07:33:03.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.371+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f432c149f70 con 0x7f432c06e570 2024-03-06T07:33:03.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.371+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f432c14a530 con 0x7f432c06e570 2024-03-06T07:33:03.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.372+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f432c04f330 con 0x7f432c06e570 2024-03-06T07:33:03.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.373+0000 7f4323e00640 1 -- 172.21.15.86:0/2157385159 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f431c046020 con 0x7f432c06e570 2024-03-06T07:33:03.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.374+0000 7f4323e00640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f430806b4d0 0x7f430806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:03.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.374+0000 7f4323e00640 1 -- 172.21.15.86:0/2157385159 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(106..106 src has 1..106) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f431c0ac980 con 0x7f432c06e570 2024-03-06T07:33:03.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.374+0000 7f432b400640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f430806b4d0 0x7f430806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:03.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.375+0000 7f432b400640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f430806b4d0 0x7f430806d990 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f432c071870 tx=0x7f4314006c90 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:03.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.378+0000 7f4323e00640 1 -- 172.21.15.86:0/2157385159 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f431c04d030 con 0x7f432c06e570 2024-03-06T07:33:03.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:03.577+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f432c0b2e30 con 0x7f432c06e570 2024-03-06T07:33:04.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.361+0000 7f4323e00640 1 -- 172.21.15.86:0/2157385159 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v107) v1 ==== 138+0+0 (secure 0 0 0) 0x7f431c07bab0 con 0x7f432c06e570 2024-03-06T07:33:04.364 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:33:04.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f430806b4d0 msgr2=0x7f430806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:04.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f430806b4d0 0x7f430806d990 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f432c071870 tx=0x7f4314006c90 comp rx=0 tx=0).stop 2024-03-06T07:33:04.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c06e570 msgr2=0x7f432c1f7940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:04.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c06e570 0x7f432c1f7940 secure :-1 s=READY pgs=1239 cs=0 l=1 rev1=1 crypto rx=0x7f431c0093e0 tx=0x7f431c004a40 comp rx=0 tx=0).stop 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 shutdown_connections 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f430806b4d0 0x7f430806d990 unknown :-1 s=CLOSED pgs=538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f432c15e170 0x7f432c147990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f432c0b4990 0x7f432c147090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 --2- 172.21.15.86:0/2157385159 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f432c06e570 0x7f432c1f7940 unknown :-1 s=CLOSED pgs=1239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.365+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 >> 172.21.15.86:0/2157385159 conn(0x7f432c0cfa80 msgr2=0x7f432c15a490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.366+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 shutdown_connections 2024-03-06T07:33:04.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.366+0000 7f4333000640 1 -- 172.21.15.86:0/2157385159 wait complete. 2024-03-06T07:33:04.392 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:33:04.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.590+0000 7f83ce000640 1 Processor -- start 2024-03-06T07:33:04.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.591+0000 7f83ce000640 1 -- start start 2024-03-06T07:33:04.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.592+0000 7f83ce000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c8070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:04.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.592+0000 7f83ce000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f83c815fff0 0x7f83c8160450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:04.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.592+0000 7f83ce000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f83c80ce4a0 0x7f83c80ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:04.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.592+0000 7f83ce000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f83c80b46c0 con 0x7f83c80ce4a0 2024-03-06T07:33:04.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.592+0000 7f83ce000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f83c80b49a0 con 0x7f83c815fff0 2024-03-06T07:33:04.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.592+0000 7f83ce000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f83c8072c90 con 0x7f83c815fc10 2024-03-06T07:33:04.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f83c80ce4a0 0x7f83c80ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f83c80ce4a0 0x7f83c80ce930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48514/0 (socket says 172.21.15.86:48514) 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 -- 172.21.15.86:0/3541114948 learned_addr learned my addr 172.21.15.86:0/3541114948 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c6a00640 1 --2- 172.21.15.86:0/3541114948 >> v2:172.21.15.194:3300/0 conn(0x7f83c815fff0 0x7f83c8160450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7400640 1 --2- 172.21.15.86:0/3541114948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c8070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 -- 172.21.15.86:0/3541114948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 msgr2=0x7f83c8070ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 --2- 172.21.15.86:0/3541114948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c8070ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 -- 172.21.15.86:0/3541114948 >> v2:172.21.15.194:3300/0 conn(0x7f83c815fff0 msgr2=0x7f83c8160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 --2- 172.21.15.86:0/3541114948 >> v2:172.21.15.194:3300/0 conn(0x7f83c815fff0 0x7f83c8160450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7e00640 1 -- 172.21.15.86:0/3541114948 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83c8160990 con 0x7f83c80ce4a0 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c7400640 1 --2- 172.21.15.86:0/3541114948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c8070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:04.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.593+0000 7f83c6a00640 1 --2- 172.21.15.86:0/3541114948 >> v2:172.21.15.194:3300/0 conn(0x7f83c815fff0 0x7f83c8160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.594+0000 7f83c7e00640 1 --2- 172.21.15.86:0/3541114948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c80ce4a0 0x7f83c80ce930 secure :-1 s=READY pgs=1240 cs=0 l=1 rev1=1 crypto rx=0x7f83c80749e0 tx=0x7f83c002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec2e281463ddd7d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83c6000640 1 -- 172.21.15.86:0/3541114948 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83c002ca80 con 0x7f83c80ce4a0 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83c6000640 1 -- 172.21.15.86:0/3541114948 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f83c002cc20 con 0x7f83c80ce4a0 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83c6000640 1 -- 172.21.15.86:0/3541114948 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83c00343f0 con 0x7f83c80ce4a0 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83ce000640 1 -- 172.21.15.86:0/3541114948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c80ce4a0 msgr2=0x7f83c80ce930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83ce000640 1 --2- 172.21.15.86:0/3541114948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c80ce4a0 0x7f83c80ce930 secure :-1 s=READY pgs=1240 cs=0 l=1 rev1=1 crypto rx=0x7f83c80749e0 tx=0x7f83c002bee0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83ce000640 1 -- 172.21.15.86:0/3541114948 shutdown_connections 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83ce000640 1 --2- 172.21.15.86:0/3541114948 >> v2:172.21.15.194:3300/0 conn(0x7f83c815fff0 0x7f83c8160450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83ce000640 1 --2- 172.21.15.86:0/3541114948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c80ce4a0 0x7f83c80ce930 unknown :-1 s=CLOSED pgs=1240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83ce000640 1 --2- 172.21.15.86:0/3541114948 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c8070ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.595+0000 7f83ce000640 1 -- 172.21.15.86:0/3541114948 >> 172.21.15.86:0/3541114948 conn(0x7f83c80b1940 msgr2=0x7f83c80b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:04.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.596+0000 7f83ce000640 1 -- 172.21.15.86:0/3541114948 shutdown_connections 2024-03-06T07:33:04.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.596+0000 7f83ce000640 1 -- 172.21.15.86:0/3541114948 wait complete. 2024-03-06T07:33:04.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.597+0000 7f83ce000640 1 Processor -- start 2024-03-06T07:33:04.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.597+0000 7f83ce000640 1 -- start start 2024-03-06T07:33:04.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.598+0000 7f83ce000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83c80ce4a0 0x7f83c80cb7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:04.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.598+0000 7f83ce000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c81f7310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.598+0000 7f83ce000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c815fff0 0x7f83c81f7850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.598+0000 7f83ce000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83c8072a30 con 0x7f83c815fff0 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.598+0000 7f83ce000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83c80728b0 con 0x7f83c80ce4a0 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.598+0000 7f83ce000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f83c815c2e0 con 0x7f83c815fc10 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c815fff0 0x7f83c81f7850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c815fff0 0x7f83c81f7850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48526/0 (socket says 172.21.15.86:48526) 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 -- 172.21.15.86:0/4083504136 learned_addr learned my addr 172.21.15.86:0/4083504136 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c6a00640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c81f7310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7400640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83c80ce4a0 0x7f83c80cb7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 -- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 msgr2=0x7f83c81f7310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c81f7310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 -- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83c80ce4a0 msgr2=0x7f83c80cb7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83c80ce4a0 0x7f83c80cb7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:04.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 -- 172.21.15.86:0/4083504136 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83c81f8190 con 0x7f83c815fff0 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c6a00640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c81f7310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.599+0000 7f83c7e00640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c815fff0 0x7f83c81f7850 secure :-1 s=READY pgs=1241 cs=0 l=1 rev1=1 crypto rx=0x7f83c0036040 tx=0x7f83c0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.600+0000 7f83bf400640 1 -- 172.21.15.86:0/4083504136 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83c0043020 con 0x7f83c815fff0 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.600+0000 7f83bf400640 1 -- 172.21.15.86:0/4083504136 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f83c003e450 con 0x7f83c815fff0 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.600+0000 7f83bf400640 1 -- 172.21.15.86:0/4083504136 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83c003e820 con 0x7f83c815fff0 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.600+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83c80c92d0 con 0x7f83c815fff0 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.600+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83c80c9890 con 0x7f83c815fff0 2024-03-06T07:33:04.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.601+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f83c804f330 con 0x7f83c815fff0 2024-03-06T07:33:04.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.602+0000 7f83bf400640 1 -- 172.21.15.86:0/4083504136 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f83c0046020 con 0x7f83c815fff0 2024-03-06T07:33:04.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.603+0000 7f83bf400640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83a806b500 0x7f83a806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:04.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.603+0000 7f83bf400640 1 -- 172.21.15.86:0/4083504136 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(107..107 src has 1..107) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f83c00acdc0 con 0x7f83c815fff0 2024-03-06T07:33:04.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.603+0000 7f83c7400640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83a806b500 0x7f83a806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:04.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.604+0000 7f83c7400640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83a806b500 0x7f83a806d9c0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f83b8006b10 tx=0x7f83b8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:04.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.608+0000 7f83bf400640 1 -- 172.21.15.86:0/4083504136 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f83c004d070 con 0x7f83c815fff0 2024-03-06T07:33:04.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:04.808+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f83c806ef30 con 0x7f83c815fff0 2024-03-06T07:33:05.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.367+0000 7f83bf400640 1 -- 172.21.15.86:0/4083504136 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v108) v1 ==== 170+0+0 (secure 0 0 0) 0x7f83c007bf00 con 0x7f83c815fff0 2024-03-06T07:33:05.369 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:33:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.371+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83a806b500 msgr2=0x7f83a806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83a806b500 0x7f83a806d9c0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f83b8006b10 tx=0x7f83b8008040 comp rx=0 tx=0).stop 2024-03-06T07:33:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c815fff0 msgr2=0x7f83c81f7850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:05.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c815fff0 0x7f83c81f7850 secure :-1 s=READY pgs=1241 cs=0 l=1 rev1=1 crypto rx=0x7f83c0036040 tx=0x7f83c0004a40 comp rx=0 tx=0).stop 2024-03-06T07:33:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 shutdown_connections 2024-03-06T07:33:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f83a806b500 0x7f83a806d9c0 unknown :-1 s=CLOSED pgs=539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f83c815fff0 0x7f83c81f7850 unknown :-1 s=CLOSED pgs=1241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f83c815fc10 0x7f83c81f7310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 --2- 172.21.15.86:0/4083504136 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f83c80ce4a0 0x7f83c80cb7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.372+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 >> 172.21.15.86:0/4083504136 conn(0x7f83c80b1940 msgr2=0x7f83c81569a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:05.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.373+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 shutdown_connections 2024-03-06T07:33:05.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.373+0000 7f83ce000640 1 -- 172.21.15.86:0/4083504136 wait complete. 2024-03-06T07:33:05.408 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:33:05.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.588+0000 7f7d73e00640 1 Processor -- start 2024-03-06T07:33:05.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.588+0000 7f7d73e00640 1 -- start start 2024-03-06T07:33:05.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.590+0000 7f7d73e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7d6c15cb90 0x7f7d6c06d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:05.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.590+0000 7f7d73e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c0b2c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:05.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.590+0000 7f7d73e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d6c15cf70 con 0x7f7d6c06e060 2024-03-06T07:33:05.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.590+0000 7f7d73e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7d6c0b3890 con 0x7f7d6c15cb90 2024-03-06T07:33:05.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.590+0000 7f7d73e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d6c06c990 con 0x7f7d6c163800 2024-03-06T07:33:05.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d70a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c0b2c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:05.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d71e00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f7d6c06e060 0x7f7d6c0cda90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:54610/0 (socket says 172.21.15.86:54610) 2024-03-06T07:33:05.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d71e00640 1 -- 172.21.15.86:0/3603912286 learned_addr learned my addr 172.21.15.86:0/3603912286 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:05.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d71400640 1 --2- 172.21.15.86:0/3603912286 >> v2:172.21.15.194:3300/0 conn(0x7f7d6c15cb90 0x7f7d6c06d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:05.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d70a00640 1 -- 172.21.15.86:0/3603912286 >> v2:172.21.15.194:3300/0 conn(0x7f7d6c15cb90 msgr2=0x7f7d6c06d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:05.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d70a00640 1 --2- 172.21.15.86:0/3603912286 >> v2:172.21.15.194:3300/0 conn(0x7f7d6c15cb90 0x7f7d6c06d6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d6be00640 1 -- 172.21.15.86:0/3603912286 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2074583329 0 0) 0x7f7d6c15cf70 con 0x7f7d6c06e060 2024-03-06T07:33:05.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d70a00640 1 -- 172.21.15.86:0/3603912286 >> v1:172.21.15.194:6789/0 conn(0x7f7d6c06e060 legacy=0x7f7d6c0cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:05.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.591+0000 7f7d70a00640 1 -- 172.21.15.86:0/3603912286 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d6c0b35b0 con 0x7f7d6c163800 2024-03-06T07:33:05.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.593+0000 7f7d70a00640 1 --2- 172.21.15.86:0/3603912286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c0b2c40 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f7d580097a0 tx=0x7f7d5802b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bdb48094b3fc39da server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:05.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.593+0000 7f7d6be00640 1 -- 172.21.15.86:0/3603912286 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d58035020 con 0x7f7d6c163800 2024-03-06T07:33:05.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.593+0000 7f7d6be00640 1 -- 172.21.15.86:0/3603912286 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d58031830 con 0x7f7d6c163800 2024-03-06T07:33:05.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.593+0000 7f7d6be00640 1 -- 172.21.15.86:0/3603912286 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d58031c00 con 0x7f7d6c163800 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 -- 172.21.15.86:0/3603912286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 msgr2=0x7f7d6c0b2c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 --2- 172.21.15.86:0/3603912286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c0b2c40 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f7d580097a0 tx=0x7f7d5802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 -- 172.21.15.86:0/3603912286 shutdown_connections 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 --2- 172.21.15.86:0/3603912286 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c0b2c40 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 --2- 172.21.15.86:0/3603912286 >> v2:172.21.15.194:3300/0 conn(0x7f7d6c15cb90 0x7f7d6c06d6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 -- 172.21.15.86:0/3603912286 >> 172.21.15.86:0/3603912286 conn(0x7f7d6c0b1940 msgr2=0x7f7d6c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 -- 172.21.15.86:0/3603912286 shutdown_connections 2024-03-06T07:33:05.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.594+0000 7f7d73e00640 1 -- 172.21.15.86:0/3603912286 wait complete. 2024-03-06T07:33:05.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.596+0000 7f7d73e00640 1 Processor -- start 2024-03-06T07:33:05.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.596+0000 7f7d73e00640 1 -- start start 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d73e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d6c06e060 0x7f7d6c159240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d73e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d6c15cb90 0x7f7d6c1648c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d73e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c164e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d73e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d6c06cec0 con 0x7f7d6c06e060 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d73e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d6c06d230 con 0x7f7d6c15cb90 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d73e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d6c06dc00 con 0x7f7d6c163800 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d71400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d6c06e060 0x7f7d6c159240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d71400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d6c06e060 0x7f7d6c159240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48530/0 (socket says 172.21.15.86:48530) 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d71400640 1 -- 172.21.15.86:0/1581474572 learned_addr learned my addr 172.21.15.86:0/1581474572 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:05.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d71e00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c164e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d70a00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d6c15cb90 0x7f7d6c1648c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d71400640 1 -- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 msgr2=0x7f7d6c164e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.597+0000 7f7d71400640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c164e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d71400640 1 -- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d6c15cb90 msgr2=0x7f7d6c1648c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d71400640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d6c15cb90 0x7f7d6c1648c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d71400640 1 -- 172.21.15.86:0/1581474572 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d6c09b5b0 con 0x7f7d6c06e060 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d71400640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d6c06e060 0x7f7d6c159240 secure :-1 s=READY pgs=1242 cs=0 l=1 rev1=1 crypto rx=0x7f7d6000e9b0 tx=0x7f7d6000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d69600640 1 -- 172.21.15.86:0/1581474572 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d60017070 con 0x7f7d6c06e060 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d69600640 1 -- 172.21.15.86:0/1581474572 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d6000c9a0 con 0x7f7d6c06e060 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d69600640 1 -- 172.21.15.86:0/1581474572 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d6000cd90 con 0x7f7d6c06e060 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d6c09b890 con 0x7f7d6c06e060 2024-03-06T07:33:05.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.598+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d6c1fc130 con 0x7f7d6c06e060 2024-03-06T07:33:05.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.599+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d6c04f330 con 0x7f7d6c06e060 2024-03-06T07:33:05.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.600+0000 7f7d69600640 1 -- 172.21.15.86:0/1581474572 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7d6001a070 con 0x7f7d6c06e060 2024-03-06T07:33:05.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.602+0000 7f7d69600640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d4c06b4d0 0x7f7d4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:05.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.602+0000 7f7d69600640 1 -- 172.21.15.86:0/1581474572 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(108..108 src has 1..108) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f7d6008a590 con 0x7f7d6c06e060 2024-03-06T07:33:05.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.602+0000 7f7d70a00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d4c06b4d0 0x7f7d4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:05.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.603+0000 7f7d70a00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d4c06b4d0 0x7f7d4c06d990 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f7d58036000 tx=0x7f7d58008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:05.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.607+0000 7f7d69600640 1 -- 172.21.15.86:0/1581474572 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7d60059460 con 0x7f7d6c06e060 2024-03-06T07:33:05.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:05.829+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f7d6c074cc0 con 0x7f7d6c06e060 2024-03-06T07:33:06.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.411+0000 7f7d69600640 1 -- 172.21.15.86:0/1581474572 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 16 and data pool 17 v73) v1 ==== 174+0+0 (secure 0 0 0) 0x7f7d60058d60 con 0x7f7d6c06e060 2024-03-06T07:33:06.417 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 16 and data pool 17 2024-03-06T07:33:06.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.421+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d4c06b4d0 msgr2=0x7f7d4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.421+0000 7f7d73e00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d4c06b4d0 0x7f7d4c06d990 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f7d58036000 tx=0x7f7d58008030 comp rx=0 tx=0).stop 2024-03-06T07:33:06.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.421+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d6c06e060 msgr2=0x7f7d6c159240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.421+0000 7f7d73e00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d6c06e060 0x7f7d6c159240 secure :-1 s=READY pgs=1242 cs=0 l=1 rev1=1 crypto rx=0x7f7d6000e9b0 tx=0x7f7d6000ee70 comp rx=0 tx=0).stop 2024-03-06T07:33:06.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.422+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 shutdown_connections 2024-03-06T07:33:06.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.422+0000 7f7d73e00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d4c06b4d0 0x7f7d4c06d990 unknown :-1 s=CLOSED pgs=540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.422+0000 7f7d73e00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d6c163800 0x7f7d6c164e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.422+0000 7f7d73e00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d6c15cb90 0x7f7d6c1648c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.422+0000 7f7d73e00640 1 --2- 172.21.15.86:0/1581474572 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d6c06e060 0x7f7d6c159240 unknown :-1 s=CLOSED pgs=1242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.422+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 >> 172.21.15.86:0/1581474572 conn(0x7f7d6c0b1940 msgr2=0x7f7d6c074030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:06.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.425+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 shutdown_connections 2024-03-06T07:33:06.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.425+0000 7f7d73e00640 1 -- 172.21.15.86:0/1581474572 wait complete. 2024-03-06T07:33:06.461 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:33:06.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.636+0000 7f5268200640 1 Processor -- start 2024-03-06T07:33:06.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.636+0000 7f5268200640 1 -- start start 2024-03-06T07:33:06.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.637+0000 7f5268200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f526006cb50 0x7f526006cfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.637+0000 7f5268200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f52600717c0 con 0x7f52600cf430 2024-03-06T07:33:06.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.637+0000 7f5268200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5260071aa0 con 0x7f52600b2d10 2024-03-06T07:33:06.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.637+0000 7f5268200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5260071d80 con 0x7f526006cb50 2024-03-06T07:33:06.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.637+0000 7f5265800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f52600cf430 0x7f5260070d20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53574/0 (socket says 172.21.15.86:53574) 2024-03-06T07:33:06.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.637+0000 7f5265800640 1 -- 172.21.15.86:0/1149730852 learned_addr learned my addr 172.21.15.86:0/1149730852 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:06.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3542333664 0 0) 0x7f52600717c0 con 0x7f52600cf430 2024-03-06T07:33:06.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5248003520 con 0x7f52600cf430 2024-03-06T07:33:06.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f5264e00640 1 --2- 172.21.15.86:0/1149730852 >> v2:172.21.15.194:3300/0 conn(0x7f526006cb50 0x7f526006cfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:06.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 68617428 0 0) 0x7f5248003520 con 0x7f52600cf430 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 >> v2:172.21.15.194:3300/0 conn(0x7f526006cb50 msgr2=0x7f526006cfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 --2- 172.21.15.86:0/1149730852 >> v2:172.21.15.194:3300/0 conn(0x7f526006cb50 0x7f526006cfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 >> v1:172.21.15.194:6789/0 conn(0x7f52600b2d10 legacy=0x7f52600b3120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5260072060 con 0x7f52600cf430 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f5254001670 con 0x7f52600cf430 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 >> v1:172.21.15.86:6789/0 conn(0x7f52600cf430 legacy=0x7f5260070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f52480039b0 0x7f5248003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5248004800 0x7f5248004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52480051c0 0x7f524800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5260071d80 con 0x7f5248004800 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f524800c110 con 0x7f52480039b0 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f524800c310 con 0x7f52480051c0 2024-03-06T07:33:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.638+0000 7f5264e00640 1 --2- 172.21.15.86:0/1149730852 >> v2:172.21.15.194:3300/0 conn(0x7f526006cb50 0x7f526006cfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5264e00640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5248004800 0x7f5248004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5266200640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52480051c0 0x7f524800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5264e00640 1 -- 172.21.15.86:0/1149730852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52480051c0 msgr2=0x7f524800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5264e00640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52480051c0 0x7f524800b9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5264e00640 1 -- 172.21.15.86:0/1149730852 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f52480039b0 msgr2=0x7f5248003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5264e00640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f52480039b0 0x7f5248003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5264e00640 1 -- 172.21.15.86:0/1149730852 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5248003520 con 0x7f5248004800 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.639+0000 7f5266200640 1 -- 172.21.15.86:0/1149730852 reap_dead start 2024-03-06T07:33:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.640+0000 7f5264e00640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5248004800 0x7f5248004c80 secure :-1 s=READY pgs=1244 cs=0 l=1 rev1=1 crypto rx=0x7f524c0098a0 tx=0x7f5248005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c10d9e5845fa23f3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.640+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f524c034070 con 0x7f5248004800 2024-03-06T07:33:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.640+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f524c030610 con 0x7f5248004800 2024-03-06T07:33:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.640+0000 7f525fe00640 1 -- 172.21.15.86:0/1149730852 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f524c00f040 con 0x7f5248004800 2024-03-06T07:33:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.641+0000 7f5268200640 1 -- 172.21.15.86:0/1149730852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5248004800 msgr2=0x7f5248004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.641+0000 7f5268200640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5248004800 0x7f5248004c80 secure :-1 s=READY pgs=1244 cs=0 l=1 rev1=1 crypto rx=0x7f524c0098a0 tx=0x7f5248005830 comp rx=0 tx=0).stop 2024-03-06T07:33:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.641+0000 7f5268200640 1 -- 172.21.15.86:0/1149730852 shutdown_connections 2024-03-06T07:33:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.641+0000 7f5268200640 1 --2- 172.21.15.86:0/1149730852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5248004800 0x7f5248004c80 unknown :-1 s=CLOSED pgs=1244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.641+0000 7f5268200640 1 -- 172.21.15.86:0/1149730852 >> 172.21.15.86:0/1149730852 conn(0x7f52600b1730 msgr2=0x7f52600b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.641+0000 7f5268200640 1 -- 172.21.15.86:0/1149730852 shutdown_connections 2024-03-06T07:33:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.641+0000 7f5268200640 1 -- 172.21.15.86:0/1149730852 wait complete. 2024-03-06T07:33:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.643+0000 7f5268200640 1 Processor -- start 2024-03-06T07:33:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.643+0000 7f5268200640 1 -- start start 2024-03-06T07:33:06.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5268200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5248004800 0x7f5260161800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5268200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5260154f30 0x7f5260161d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5268200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52601462a0 0x7f5260146700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5268200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5260162cf0 con 0x7f5260154f30 2024-03-06T07:33:06.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5268200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5260162b70 con 0x7f5248004800 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5268200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f526006cfe0 con 0x7f52601462a0 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5264e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5260154f30 0x7f5260161d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5264e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5260154f30 0x7f5260161d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36144/0 (socket says 172.21.15.86:36144) 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.644+0000 7f5264e00640 1 -- 172.21.15.86:0/2758562044 learned_addr learned my addr 172.21.15.86:0/2758562044 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5266200640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52601462a0 0x7f5260146700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5264e00640 1 -- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52601462a0 msgr2=0x7f5260146700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5265800640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5248004800 0x7f5260161800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5264e00640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52601462a0 0x7f5260146700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5264e00640 1 -- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5248004800 msgr2=0x7f5260161800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5264e00640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5248004800 0x7f5260161800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5264e00640 1 -- 172.21.15.86:0/2758562044 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5260147180 con 0x7f5260154f30 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5265800640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5248004800 0x7f5260161800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5264e00640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5260154f30 0x7f5260161d40 secure :-1 s=READY pgs=1245 cs=0 l=1 rev1=1 crypto rx=0x7f524c02bf70 tx=0x7f524c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f525d600640 1 -- 172.21.15.86:0/2758562044 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f524c048070 con 0x7f5260154f30 2024-03-06T07:33:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f525d600640 1 -- 172.21.15.86:0/2758562044 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f524c030410 con 0x7f5260154f30 2024-03-06T07:33:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.645+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5260147460 con 0x7f5260154f30 2024-03-06T07:33:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.646+0000 7f525d600640 1 -- 172.21.15.86:0/2758562044 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f524c034040 con 0x7f5260154f30 2024-03-06T07:33:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.646+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f52600ada90 con 0x7f5260154f30 2024-03-06T07:33:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.647+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f526004f330 con 0x7f5260154f30 2024-03-06T07:33:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.648+0000 7f525d600640 1 -- 172.21.15.86:0/2758562044 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f524c04b070 con 0x7f5260154f30 2024-03-06T07:33:06.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.649+0000 7f525d600640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f524006b4d0 0x7f524006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:06.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.649+0000 7f525d600640 1 -- 172.21.15.86:0/2758562044 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f524c0aba30 con 0x7f5260154f30 2024-03-06T07:33:06.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.649+0000 7f5265800640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f524006b4d0 0x7f524006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:06.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.649+0000 7f5265800640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f524006b4d0 0x7f524006d990 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f5254007a90 tx=0x7f5254009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.653+0000 7f525d600640 1 -- 172.21.15.86:0/2758562044 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f524c0305e0 con 0x7f5260154f30 2024-03-06T07:33:06.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.842+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f52600b4a40 con 0x7f5260154f30 2024-03-06T07:33:06.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.845+0000 7f525d600640 1 -- 172.21.15.86:0/2758562044 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v109) v1 ==== 74+0+16652 (secure 0 0 0) 0x7f524c07a820 con 0x7f5260154f30 2024-03-06T07:33:06.847 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:06.847 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":109,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:05.830988+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f524006b4d0 msgr2=0x7f524006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f524006b4d0 0x7f524006d990 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f5254007a90 tx=0x7f5254009040 comp rx=0 tx=0).stop 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5260154f30 msgr2=0x7f5260161d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5260154f30 0x7f5260161d40 secure :-1 s=READY pgs=1245 cs=0 l=1 rev1=1 crypto rx=0x7f524c02bf70 tx=0x7f524c004730 comp rx=0 tx=0).stop 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 shutdown_connections 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f524006b4d0 0x7f524006d990 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f52601462a0 0x7f5260146700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5260154f30 0x7f5260161d40 unknown :-1 s=CLOSED pgs=1245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 --2- 172.21.15.86:0/2758562044 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5248004800 0x7f5260161800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 >> 172.21.15.86:0/2758562044 conn(0x7f52600b1730 msgr2=0x7f5260070d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 shutdown_connections 2024-03-06T07:33:06.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:06.848+0000 7f5268200640 1 -- 172.21.15.86:0/2758562044 wait complete. 2024-03-06T07:33:06.867 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:33:07.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.057+0000 7fb2c6200640 1 Processor -- start 2024-03-06T07:33:07.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.057+0000 7fb2c6200640 1 -- start start 2024-03-06T07:33:07.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.058+0000 7fb2c6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c006d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.058+0000 7fb2c6200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb2c00cc2a0 0x7fb2c00cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.058+0000 7fb2c6200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb2c00b2b30 con 0x7fb2c015a940 2024-03-06T07:33:07.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.058+0000 7fb2c6200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb2c00b28e0 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.058+0000 7fb2c6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb2c00b2e10 con 0x7fb2c00cf370 2024-03-06T07:33:07.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.059+0000 7fb2b7e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c006d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.059+0000 7fb2c5800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb2c00cc2a0 0x7fb2c00cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.059+0000 7fb2c4e00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7fb2c015a940 0x7fb2c00b45e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:52620/0 (socket says 172.21.15.86:52620) 2024-03-06T07:33:07.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.059+0000 7fb2c4e00640 1 -- 172.21.15.86:0/947112894 learned_addr learned my addr 172.21.15.86:0/947112894 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:07.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.060+0000 7fb2c5800640 1 -- 172.21.15.86:0/947112894 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 msgr2=0x7fb2c006d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:07.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.060+0000 7fb2c5800640 1 --2- 172.21.15.86:0/947112894 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c006d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.060+0000 7fb2c5800640 1 -- 172.21.15.86:0/947112894 >> v1:172.21.15.194:6789/0 conn(0x7fb2c015a940 legacy=0x7fb2c00b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.060+0000 7fb2c5800640 1 -- 172.21.15.86:0/947112894 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2c006ca50 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.061+0000 7fb2c5800640 1 --2- 172.21.15.86:0/947112894 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c00cc750 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fb2b8009d90 tx=0x7fb2b802bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8f12184d9dad0d33 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:07.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.062+0000 7fb2bfe00640 1 -- 172.21.15.86:0/947112894 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2b8039070 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.062+0000 7fb2bfe00640 1 -- 172.21.15.86:0/947112894 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb2b802cc50 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.062+0000 7fb2bfe00640 1 -- 172.21.15.86:0/947112894 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2b8034440 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.062+0000 7fb2c6200640 1 -- 172.21.15.86:0/947112894 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c00cc2a0 msgr2=0x7fb2c00cc750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.062+0000 7fb2c6200640 1 --2- 172.21.15.86:0/947112894 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c00cc750 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fb2b8009d90 tx=0x7fb2b802bea0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.063+0000 7fb2c6200640 1 -- 172.21.15.86:0/947112894 shutdown_connections 2024-03-06T07:33:07.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.063+0000 7fb2c6200640 1 --2- 172.21.15.86:0/947112894 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c00cc750 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.063+0000 7fb2c6200640 1 --2- 172.21.15.86:0/947112894 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c006d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.063+0000 7fb2c6200640 1 -- 172.21.15.86:0/947112894 >> 172.21.15.86:0/947112894 conn(0x7fb2c006afc0 msgr2=0x7fb2c006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:07.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.063+0000 7fb2c6200640 1 -- 172.21.15.86:0/947112894 shutdown_connections 2024-03-06T07:33:07.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.063+0000 7fb2c6200640 1 -- 172.21.15.86:0/947112894 wait complete. 2024-03-06T07:33:07.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.064+0000 7fb2c6200640 1 Processor -- start 2024-03-06T07:33:07.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.064+0000 7fb2c6200640 1 -- start start 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.065+0000 7fb2c6200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c0152f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.065+0000 7fb2c6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c00b0430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.065+0000 7fb2c6200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c015a940 0x7fb2c00b0970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.065+0000 7fb2c6200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2c006d790 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.065+0000 7fb2c6200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2c00b30f0 con 0x7fb2c015a940 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.065+0000 7fb2c6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb2c0154ea0 con 0x7fb2c00cf370 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c0152f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c0152f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36146/0 (socket says 172.21.15.86:36146) 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 -- 172.21.15.86:0/3442569120 learned_addr learned my addr 172.21.15.86:0/3442569120 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 -- 172.21.15.86:0/3442569120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 msgr2=0x7fb2c00b0430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2b7e00640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c00b0430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c00b0430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 -- 172.21.15.86:0/3442569120 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c015a940 msgr2=0x7fb2c00b0970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c015a940 0x7fb2c00b0970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 -- 172.21.15.86:0/3442569120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2c0154290 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.066+0000 7fb2c4e00640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c0152f10 secure :-1 s=READY pgs=1246 cs=0 l=1 rev1=1 crypto rx=0x7fb2b000c400 tx=0x7fb2b000c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:07.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.067+0000 7fb2bd600640 1 -- 172.21.15.86:0/3442569120 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2b0017070 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.067+0000 7fb2bd600640 1 -- 172.21.15.86:0/3442569120 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb2b0005ce0 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.067+0000 7fb2bd600640 1 -- 172.21.15.86:0/3442569120 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2b00072f0 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.067+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb2c0154570 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.067+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb2c0154b30 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.068+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb2c004f330 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.069+0000 7fb2bd600640 1 -- 172.21.15.86:0/3442569120 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb2b001a070 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.069+0000 7fb2bd600640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb29c06b4d0 0x7fb29c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.070+0000 7fb2bd600640 1 -- 172.21.15.86:0/3442569120 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb2b00895b0 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.070+0000 7fb2b7e00640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb29c06b4d0 0x7fb29c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.070+0000 7fb2b7e00640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb29c06b4d0 0x7fb29c06d990 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7fb2c0153fa0 tx=0x7fb2a8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:07.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.074+0000 7fb2bd600640 1 -- 172.21.15.86:0/3442569120 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb2b0058420 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.255+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fb2c006e180 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.257+0000 7fb2bd600640 1 -- 172.21.15.86:0/3442569120 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v109) v1 ==== 74+0+16652 (secure 0 0 0) 0x7fb2b0057d20 con 0x7fb2c00cc2a0 2024-03-06T07:33:07.259 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:07.259 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":109,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:05.830988+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:33:07.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb29c06b4d0 msgr2=0x7fb29c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb29c06b4d0 0x7fb29c06d990 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7fb2c0153fa0 tx=0x7fb2a8008040 comp rx=0 tx=0).stop 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2c00cc2a0 msgr2=0x7fb2c0152f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c0152f10 secure :-1 s=READY pgs=1246 cs=0 l=1 rev1=1 crypto rx=0x7fb2b000c400 tx=0x7fb2b000c8c0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 shutdown_connections 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb29c06b4d0 0x7fb29c06d990 unknown :-1 s=CLOSED pgs=542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb2c015a940 0x7fb2c00b0970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb2c00cf370 0x7fb2c00b0430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 --2- 172.21.15.86:0/3442569120 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb2c00cc2a0 0x7fb2c0152f10 unknown :-1 s=CLOSED pgs=1246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 >> 172.21.15.86:0/3442569120 conn(0x7fb2c006afc0 msgr2=0x7fb2c015eef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 shutdown_connections 2024-03-06T07:33:07.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.260+0000 7fb2c6200640 1 -- 172.21.15.86:0/3442569120 wait complete. 2024-03-06T07:33:07.281 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:33:07.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.492+0000 7f1f31a00640 1 Processor -- start 2024-03-06T07:33:07.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.492+0000 7f1f31a00640 1 -- start start 2024-03-06T07:33:07.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.493+0000 7f1f31a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 0x7f1f2c0727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.493+0000 7f1f31a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1f2c0b3650 0x7f1f2c0b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.493+0000 7f1f31a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1f2c0b4710 con 0x7f1f2c0cda90 2024-03-06T07:33:07.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.493+0000 7f1f31a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1f2c072310 con 0x7f1f2c0b3650 2024-03-06T07:33:07.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.493+0000 7f1f31a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0719e0 con 0x7f1f2c0b3df0 2024-03-06T07:33:07.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f2b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1f2c0cda90 0x7f1f2c06c990 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53582/0 (socket says 172.21.15.86:53582) 2024-03-06T07:33:07.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f2b400640 1 -- 172.21.15.86:0/2326866200 learned_addr learned my addr 172.21.15.86:0/2326866200 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:07.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 0x7f1f2c0727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f2a000640 1 --2- 172.21.15.86:0/2326866200 >> v2:172.21.15.194:3300/0 conn(0x7f1f2c0b3650 0x7f1f2c0b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 138703297 0 0) 0x7f1f2c0b4710 con 0x7f1f2c0cda90 2024-03-06T07:33:07.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1f10003520 con 0x7f1f2c0cda90 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1261746956 0 0) 0x7f1f10003520 con 0x7f1f2c0cda90 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 msgr2=0x7f1f2c0727a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 0x7f1f2c0727a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 >> v2:172.21.15.194:3300/0 conn(0x7f1f2c0b3650 msgr2=0x7f1f2c0b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 --2- 172.21.15.86:0/2326866200 >> v2:172.21.15.194:3300/0 conn(0x7f1f2c0b3650 0x7f1f2c0b41d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f2c071cc0 con 0x7f1f2c0cda90 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1f240019f0 con 0x7f1f2c0cda90 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 >> v1:172.21.15.86:6789/0 conn(0x7f1f2c0cda90 legacy=0x7f1f2c06c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.494+0000 7f1f29600640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f100039b0 0x7f1f10003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f29600640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f10004800 0x7f1f10004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f29600640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 0x7f1f100051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c072310 con 0x7f1f100039b0 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0719e0 con 0x7f1f10004800 2024-03-06T07:33:07.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f1000dbd0 con 0x7f1f2c0b3df0 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2a000640 1 --2- 172.21.15.86:0/2326866200 >> v2:172.21.15.194:3300/0 conn(0x7f1f2c0b3650 0x7f1f2c0b41d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f100039b0 0x7f1f10003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2b400640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 0x7f1f100051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2aa00640 1 -- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 msgr2=0x7f1f100051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f2c0b3df0 0x7f1f100051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2aa00640 1 -- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f10004800 msgr2=0x7f1f10004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f10004800 0x7f1f10004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.495+0000 7f1f2aa00640 1 -- 172.21.15.86:0/2326866200 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f10003520 con 0x7f1f100039b0 2024-03-06T07:33:07.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.496+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f100039b0 0x7f1f10003dd0 secure :-1 s=READY pgs=1248 cs=0 l=1 rev1=1 crypto rx=0x7f1f1c009ac0 tx=0x7f1f10005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1bcc870db23b9237 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:07.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f1c039070 con 0x7f1f100039b0 2024-03-06T07:33:07.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f1c02c6b0 con 0x7f1f100039b0 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f29600640 1 -- 172.21.15.86:0/2326866200 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f1c009690 con 0x7f1f100039b0 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f31a00640 1 -- 172.21.15.86:0/2326866200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f100039b0 msgr2=0x7f1f10003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f31a00640 1 --2- 172.21.15.86:0/2326866200 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f100039b0 0x7f1f10003dd0 secure :-1 s=READY pgs=1248 cs=0 l=1 rev1=1 crypto rx=0x7f1f1c009ac0 tx=0x7f1f10005830 comp rx=0 tx=0).stop 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f2b400640 1 -- 172.21.15.86:0/2326866200 reap_dead start 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f31a00640 1 -- 172.21.15.86:0/2326866200 shutdown_connections 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f31a00640 1 -- 172.21.15.86:0/2326866200 >> 172.21.15.86:0/2326866200 conn(0x7f1f2c0b1940 msgr2=0x7f1f2c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f31a00640 1 -- 172.21.15.86:0/2326866200 shutdown_connections 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.497+0000 7f1f31a00640 1 -- 172.21.15.86:0/2326866200 wait complete. 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.499+0000 7f1f31a00640 1 Processor -- start 2024-03-06T07:33:07.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.499+0000 7f1f31a00640 1 -- start start 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f31a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f100039b0 0x7f1f2c14f330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f31a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f2c15e540 0x7f1f2c0ca4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f31a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f2c0c9bd0 0x7f1f2c154780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f31a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c15b240 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f31a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0b3b30 con 0x7f1f2c15e540 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f31a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c15b3c0 con 0x7f1f100039b0 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f2b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f2c0c9bd0 0x7f1f2c154780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f2b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f2c0c9bd0 0x7f1f2c154780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36158/0 (socket says 172.21.15.86:36158) 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.500+0000 7f1f2b400640 1 -- 172.21.15.86:0/380658914 learned_addr learned my addr 172.21.15.86:0/380658914 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2a000640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f2c15e540 0x7f1f2c0ca4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f100039b0 0x7f1f2c14f330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2b400640 1 -- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f100039b0 msgr2=0x7f1f2c14f330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2b400640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f100039b0 0x7f1f2c14f330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2b400640 1 -- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f2c15e540 msgr2=0x7f1f2c0ca4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2b400640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f2c15e540 0x7f1f2c0ca4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2b400640 1 -- 172.21.15.86:0/380658914 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f2c0cad20 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f2b400640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f2c0c9bd0 0x7f1f2c154780 secure :-1 s=READY pgs=1249 cs=0 l=1 rev1=1 crypto rx=0x7f1f2c070ba0 tx=0x7f1f2400e530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f22a00640 1 -- 172.21.15.86:0/380658914 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f2400ebb0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f22a00640 1 -- 172.21.15.86:0/380658914 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f24018070 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f22a00640 1 -- 172.21.15.86:0/380658914 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f24013420 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.501+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1f2c0c90f0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.502+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1f2c0c96b0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.503+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1f2c0c98b0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.505 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.504+0000 7f1f22a00640 1 -- 172.21.15.86:0/380658914 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1f240135c0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.504+0000 7f1f22a00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f0c06b4d0 0x7f1f0c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:07.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.505+0000 7f1f22a00640 1 -- 172.21.15.86:0/380658914 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1f2408a4b0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.505+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f0c06b4d0 0x7f1f0c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:07.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.505+0000 7f1f2aa00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f0c06b4d0 0x7f1f0c06d990 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f1f1c036000 tx=0x7f1f1c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:07.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.509+0000 7f1f22a00640 1 -- 172.21.15.86:0/380658914 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1f2c0c98b0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:07.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:07.749+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f1f2c04f330 con 0x7f1f2c0c9bd0 2024-03-06T07:33:08.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.438+0000 7f1f22a00640 1 -- 172.21.15.86:0/380658914 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v76) v1 ==== 118+0+0 (secure 0 0 0) 0x7f1f240593c0 con 0x7f1f2c0c9bd0 2024-03-06T07:33:08.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f0c06b4d0 msgr2=0x7f1f0c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:08.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f0c06b4d0 0x7f1f0c06d990 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f1f1c036000 tx=0x7f1f1c030000 comp rx=0 tx=0).stop 2024-03-06T07:33:08.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f2c0c9bd0 msgr2=0x7f1f2c154780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:08.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f2c0c9bd0 0x7f1f2c154780 secure :-1 s=READY pgs=1249 cs=0 l=1 rev1=1 crypto rx=0x7f1f2c070ba0 tx=0x7f1f2400e530 comp rx=0 tx=0).stop 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 shutdown_connections 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1f0c06b4d0 0x7f1f0c06d990 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1f2c0c9bd0 0x7f1f2c154780 unknown :-1 s=CLOSED pgs=1249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1f2c15e540 0x7f1f2c0ca4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 --2- 172.21.15.86:0/380658914 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1f100039b0 0x7f1f2c14f330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 >> 172.21.15.86:0/380658914 conn(0x7f1f2c0b1940 msgr2=0x7f1f2c0b2a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 shutdown_connections 2024-03-06T07:33:08.445 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.442+0000 7f1f31a00640 1 -- 172.21.15.86:0/380658914 wait complete. 2024-03-06T07:33:08.465 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:33:08.465 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:33:08.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.662+0000 7f7d53800640 1 Processor -- start 2024-03-06T07:33:08.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.662+0000 7f7d53800640 1 -- start start 2024-03-06T07:33:08.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.664+0000 7f7d53800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7d4c0b2c40 0x7f7d4c15f1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.664+0000 7f7d53800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d4c06cba0 con 0x7f7d4c06d960 2024-03-06T07:33:08.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.664+0000 7f7d53800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d4c06ce80 con 0x7f7d4c06e2f0 2024-03-06T07:33:08.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.664+0000 7f7d53800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7d4c15f700 con 0x7f7d4c0b2c40 2024-03-06T07:33:08.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.664+0000 7f7d43e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7d4c06d960 0x7f7d4c0b3020 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53588/0 (socket says 172.21.15.86:53588) 2024-03-06T07:33:08.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.664+0000 7f7d43e00640 1 -- 172.21.15.86:0/306747289 learned_addr learned my addr 172.21.15.86:0/306747289 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.664+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1361115790 0 0) 0x7f7d4c06cba0 con 0x7f7d4c06d960 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7d30003520 con 0x7f7d4c06d960 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d50e00640 1 --2- 172.21.15.86:0/306747289 >> v2:172.21.15.194:3300/0 conn(0x7f7d4c0b2c40 0x7f7d4c15f1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1101651765 0 0) 0x7f7d30003520 con 0x7f7d4c06d960 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 >> v2:172.21.15.194:3300/0 conn(0x7f7d4c0b2c40 msgr2=0x7f7d4c15f1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 --2- 172.21.15.86:0/306747289 >> v2:172.21.15.194:3300/0 conn(0x7f7d4c0b2c40 0x7f7d4c15f1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 >> v1:172.21.15.194:6789/0 conn(0x7f7d4c06e2f0 legacy=0x7f7d4c0cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d4c06d160 con 0x7f7d4c06d960 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7d34001670 con 0x7f7d4c06d960 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 >> v1:172.21.15.86:6789/0 conn(0x7f7d4c06d960 legacy=0x7f7d4c0b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:08.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d30003a00 0x7f7d30003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30004800 0x7f7d30004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d30005210 0x7f7d3000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d3000c110 con 0x7f7d30003a00 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d3000c2c0 con 0x7f7d30005210 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d3000c4c0 con 0x7f7d30004800 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.665+0000 7f7d50e00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d30003a00 0x7f7d30003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d43e00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30004800 0x7f7d30004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d51800640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d30005210 0x7f7d3000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d50e00640 1 -- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30004800 msgr2=0x7f7d30004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d50e00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30004800 0x7f7d30004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d50e00640 1 -- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d30005210 msgr2=0x7f7d3000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d50e00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d30005210 0x7f7d3000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d50e00640 1 -- 172.21.15.86:0/306747289 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d30003520 con 0x7f7d30003a00 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d51800640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d30005210 0x7f7d3000b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:08.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.666+0000 7f7d51800640 1 -- 172.21.15.86:0/306747289 reap_dead start 2024-03-06T07:33:08.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.667+0000 7f7d50e00640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d30003a00 0x7f7d30003e00 secure :-1 s=READY pgs=1251 cs=0 l=1 rev1=1 crypto rx=0x7f7d3c00ba00 tx=0x7f7d30005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=93f0885bd950e1e3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:08.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.667+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d3c036020 con 0x7f7d30003a00 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.667+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d3c032600 con 0x7f7d30003a00 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.667+0000 7f7d4be00640 1 -- 172.21.15.86:0/306747289 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d3c00b600 con 0x7f7d30003a00 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.668+0000 7f7d53800640 1 -- 172.21.15.86:0/306747289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d30003a00 msgr2=0x7f7d30003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.668+0000 7f7d53800640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d30003a00 0x7f7d30003e00 secure :-1 s=READY pgs=1251 cs=0 l=1 rev1=1 crypto rx=0x7f7d3c00ba00 tx=0x7f7d30005830 comp rx=0 tx=0).stop 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.668+0000 7f7d53800640 1 -- 172.21.15.86:0/306747289 shutdown_connections 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.668+0000 7f7d53800640 1 --2- 172.21.15.86:0/306747289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d30003a00 0x7f7d30003e00 unknown :-1 s=CLOSED pgs=1251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.668+0000 7f7d53800640 1 -- 172.21.15.86:0/306747289 >> 172.21.15.86:0/306747289 conn(0x7f7d4c0b1940 msgr2=0x7f7d4c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.668+0000 7f7d53800640 1 -- 172.21.15.86:0/306747289 shutdown_connections 2024-03-06T07:33:08.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.668+0000 7f7d53800640 1 -- 172.21.15.86:0/306747289 wait complete. 2024-03-06T07:33:08.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.670+0000 7f7d53800640 1 Processor -- start 2024-03-06T07:33:08.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.670+0000 7f7d53800640 1 -- start start 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d53800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30003a00 0x7f7d4c09ce60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d53800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d4c15bc50 0x7f7d4c0cc630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d53800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d4c160260 0x7f7d4c1606e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d53800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d4c06d4d0 con 0x7f7d4c15bc50 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d53800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d4c06e990 con 0x7f7d4c160260 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d53800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d4c15f1c0 con 0x7f7d30003a00 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d43e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d4c15bc50 0x7f7d4c0cc630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d43e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d4c15bc50 0x7f7d4c0cc630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36172/0 (socket says 172.21.15.86:36172) 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.671+0000 7f7d43e00640 1 -- 172.21.15.86:0/2967162376 learned_addr learned my addr 172.21.15.86:0/2967162376 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:08.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d50e00640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30003a00 0x7f7d4c09ce60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d43e00640 1 -- 172.21.15.86:0/2967162376 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30003a00 msgr2=0x7f7d4c09ce60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d43e00640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30003a00 0x7f7d4c09ce60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d43e00640 1 -- 172.21.15.86:0/2967162376 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d4c160260 msgr2=0x7f7d4c1606e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d43e00640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d4c160260 0x7f7d4c1606e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d43e00640 1 -- 172.21.15.86:0/2967162376 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d4c09d3a0 con 0x7f7d4c15bc50 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d43e00640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d4c15bc50 0x7f7d4c0cc630 secure :-1 s=READY pgs=1252 cs=0 l=1 rev1=1 crypto rx=0x7f7d3400e830 tx=0x7f7d3400ecf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d49600640 1 -- 172.21.15.86:0/2967162376 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d3400c8d0 con 0x7f7d4c15bc50 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d49600640 1 -- 172.21.15.86:0/2967162376 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d34018070 con 0x7f7d4c15bc50 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d4c09bdc0 con 0x7f7d4c15bc50 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d49600640 1 -- 172.21.15.86:0/2967162376 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d3400ccb0 con 0x7f7d4c15bc50 2024-03-06T07:33:08.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.672+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d4c09c380 con 0x7f7d4c15bc50 2024-03-06T07:33:08.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.674+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d4c04f330 con 0x7f7d4c15bc50 2024-03-06T07:33:08.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.674+0000 7f7d49600640 1 -- 172.21.15.86:0/2967162376 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7d3401d460 con 0x7f7d4c15bc50 2024-03-06T07:33:08.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.675+0000 7f7d49600640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d2806b500 0x7f7d2806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:08.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.675+0000 7f7d49600640 1 -- 172.21.15.86:0/2967162376 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7d3408a1a0 con 0x7f7d4c15bc50 2024-03-06T07:33:08.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.676+0000 7f7d50e00640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d2806b500 0x7f7d2806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:08.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.676+0000 7f7d50e00640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d2806b500 0x7f7d2806d9c0 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f7d300057e0 tx=0x7f7d3c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:08.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:08.680+0000 7f7d49600640 1 -- 172.21.15.86:0/2967162376 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7d34058f80 con 0x7f7d4c15bc50 2024-03-06T07:33:09.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.044+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f7d4c0ad5c0 con 0x7f7d4c15bc50 2024-03-06T07:33:09.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.451+0000 7f7d49600640 1 -- 172.21.15.86:0/2967162376 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v77) v1 ==== 122+0+0 (secure 0 0 0) 0x7f7d34058880 con 0x7f7d4c15bc50 2024-03-06T07:33:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.454+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d2806b500 msgr2=0x7f7d2806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.454+0000 7f7d53800640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d2806b500 0x7f7d2806d9c0 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f7d300057e0 tx=0x7f7d3c030000 comp rx=0 tx=0).stop 2024-03-06T07:33:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.454+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d4c15bc50 msgr2=0x7f7d4c0cc630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.454+0000 7f7d53800640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d4c15bc50 0x7f7d4c0cc630 secure :-1 s=READY pgs=1252 cs=0 l=1 rev1=1 crypto rx=0x7f7d3400e830 tx=0x7f7d3400ecf0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 shutdown_connections 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7d2806b500 0x7f7d2806d9c0 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7d4c160260 0x7f7d4c1606e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d4c15bc50 0x7f7d4c0cc630 unknown :-1 s=CLOSED pgs=1252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 --2- 172.21.15.86:0/2967162376 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d30003a00 0x7f7d4c09ce60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 >> 172.21.15.86:0/2967162376 conn(0x7f7d4c0b1940 msgr2=0x7f7d4c0724a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 shutdown_connections 2024-03-06T07:33:09.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.455+0000 7f7d53800640 1 -- 172.21.15.86:0/2967162376 wait complete. 2024-03-06T07:33:09.473 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:33:09.473 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:09.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.663+0000 7f14b5200640 1 Processor -- start 2024-03-06T07:33:09.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.663+0000 7f14b5200640 1 -- start start 2024-03-06T07:33:09.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.665+0000 7f14b5200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b015e230 0x7f14b006edb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:09.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.665+0000 7f14b5200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f14b00b4320 0x7f14b00d0be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:09.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.665+0000 7f14b5200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f14b00cf340 con 0x7f14b00ce550 2024-03-06T07:33:09.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.665+0000 7f14b5200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14b015e610 con 0x7f14b00b4320 2024-03-06T07:33:09.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.665+0000 7f14b5200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14b00cfe60 con 0x7f14b015e230 2024-03-06T07:33:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.666+0000 7f14ade00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b015e230 0x7f14b006edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.666+0000 7f14ade00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b015e230 0x7f14b006edb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:57376/0 (socket says 172.21.15.86:57376) 2024-03-06T07:33:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.666+0000 7f14ade00640 1 -- 172.21.15.86:0/1722931888 learned_addr learned my addr 172.21.15.86:0/1722931888 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.666+0000 7f14ad400640 1 --2- 172.21.15.86:0/1722931888 >> v2:172.21.15.194:3300/0 conn(0x7f14b00b4320 0x7f14b00d0be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.667+0000 7f14ade00640 1 -- 172.21.15.86:0/1722931888 >> v2:172.21.15.194:3300/0 conn(0x7f14b00b4320 msgr2=0x7f14b00d0be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.667+0000 7f14ade00640 1 --2- 172.21.15.86:0/1722931888 >> v2:172.21.15.194:3300/0 conn(0x7f14b00b4320 0x7f14b00d0be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.667+0000 7f14ade00640 1 -- 172.21.15.86:0/1722931888 >> v1:172.21.15.194:6789/0 conn(0x7f14b00ce550 legacy=0x7f14b00cc320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.667+0000 7f14ade00640 1 -- 172.21.15.86:0/1722931888 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14b006f390 con 0x7f14b015e230 2024-03-06T07:33:09.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.668+0000 7f14ade00640 1 --2- 172.21.15.86:0/1722931888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b015e230 0x7f14b006edb0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f14a0009870 tx=0x7f14a002b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3fdb4bb52d2287c4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:09.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14aca00640 1 -- 172.21.15.86:0/1722931888 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14a0035020 con 0x7f14b015e230 2024-03-06T07:33:09.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14aca00640 1 -- 172.21.15.86:0/1722931888 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f14a0031640 con 0x7f14b015e230 2024-03-06T07:33:09.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14aca00640 1 -- 172.21.15.86:0/1722931888 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14a002d040 con 0x7f14b015e230 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 -- 172.21.15.86:0/1722931888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b015e230 msgr2=0x7f14b006edb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 --2- 172.21.15.86:0/1722931888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b015e230 0x7f14b006edb0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f14a0009870 tx=0x7f14a002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 -- 172.21.15.86:0/1722931888 shutdown_connections 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 --2- 172.21.15.86:0/1722931888 >> v2:172.21.15.194:3300/0 conn(0x7f14b00b4320 0x7f14b00d0be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 --2- 172.21.15.86:0/1722931888 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b015e230 0x7f14b006edb0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 -- 172.21.15.86:0/1722931888 >> 172.21.15.86:0/1722931888 conn(0x7f14b00b3800 msgr2=0x7f14b00b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 -- 172.21.15.86:0/1722931888 shutdown_connections 2024-03-06T07:33:09.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.669+0000 7f14b5200640 1 -- 172.21.15.86:0/1722931888 wait complete. 2024-03-06T07:33:09.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.671+0000 7f14b5200640 1 Processor -- start 2024-03-06T07:33:09.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.671+0000 7f14b5200640 1 -- start start 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14b5200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14b00b4320 0x7f14b01f42e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14b5200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b00ce550 0x7f14b009e0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14b5200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f14b015e230 0x7f14b009c670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14b5200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14b00d06c0 con 0x7f14b015e230 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14b5200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14b00d03e0 con 0x7f14b00b4320 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14b5200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14b00d1120 con 0x7f14b00ce550 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14ae800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f14b015e230 0x7f14b009c670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14ae800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f14b015e230 0x7f14b009c670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36180/0 (socket says 172.21.15.86:36180) 2024-03-06T07:33:09.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.672+0000 7f14ae800640 1 -- 172.21.15.86:0/3953788547 learned_addr learned my addr 172.21.15.86:0/3953788547 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ad400640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b00ce550 0x7f14b009e0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ade00640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14b00b4320 0x7f14b01f42e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ae800640 1 -- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b00ce550 msgr2=0x7f14b009e0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ae800640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b00ce550 0x7f14b009e0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ae800640 1 -- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14b00b4320 msgr2=0x7f14b01f42e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ae800640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14b00b4320 0x7f14b01f42e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ae800640 1 -- 172.21.15.86:0/3953788547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14b00b4990 con 0x7f14b015e230 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14ae800640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f14b015e230 0x7f14b009c670 secure :-1 s=READY pgs=1253 cs=0 l=1 rev1=1 crypto rx=0x7f14a800c6e0 tx=0x7f14a800cba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14a6000640 1 -- 172.21.15.86:0/3953788547 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14a8003b70 con 0x7f14b015e230 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14a6000640 1 -- 172.21.15.86:0/3953788547 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f14a8003d30 con 0x7f14b015e230 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14a6000640 1 -- 172.21.15.86:0/3953788547 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14a8005310 con 0x7f14b015e230 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.673+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f14b00b4c30 con 0x7f14b015e230 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.674+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14b00b51f0 con 0x7f14b015e230 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.675+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f14b00b53f0 con 0x7f14b015e230 2024-03-06T07:33:09.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.675+0000 7f14a6000640 1 -- 172.21.15.86:0/3953788547 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f14a80054b0 con 0x7f14b015e230 2024-03-06T07:33:09.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.676+0000 7f14a6000640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f149006b4d0 0x7f149006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:09.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.676+0000 7f14a6000640 1 -- 172.21.15.86:0/3953788547 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f14a8089fe0 con 0x7f14b015e230 2024-03-06T07:33:09.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.676+0000 7f14ade00640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f149006b4d0 0x7f149006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:09.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.677+0000 7f14ade00640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f149006b4d0 0x7f149006d990 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f14a0036000 tx=0x7f14a003c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:09.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.681+0000 7f14a6000640 1 -- 172.21.15.86:0/3953788547 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f14b00b53f0 con 0x7f14b015e230 2024-03-06T07:33:09.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.939+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f14b004f330 con 0x7f14b015e230 2024-03-06T07:33:09.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.940+0000 7f14a6000640 1 -- 172.21.15.86:0/3953788547 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 77 v77) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f14a8058e50 con 0x7f14b015e230 2024-03-06T07:33:09.942 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:09.942 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":77,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6043,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/906873619","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":906873619},{"type":"v1","addr":"172.21.15.194:6833","nonce":906873619}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":6046,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/398024320","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":398024320},{"type":"v1","addr":"172.21.15.194:6835","nonce":398024320}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":7464,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/1880188173","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":1880188173},{"type":"v1","addr":"172.21.15.86:6835","nonce":1880188173}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":77,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:09.045845+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7467},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7467":{"gid":7467,"name":"c","rank":0,"incarnation":73,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/221976790","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":221976790},{"type":"v1","addr":"172.21.15.86:6837","nonce":221976790}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7467,"qdb_cluster":[7467]},"id":8}]} 2024-03-06T07:33:09.942 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 77 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f149006b4d0 msgr2=0x7f149006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f149006b4d0 0x7f149006d990 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f14a0036000 tx=0x7f14a003c040 comp rx=0 tx=0).stop 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f14b015e230 msgr2=0x7f14b009c670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f14b015e230 0x7f14b009c670 secure :-1 s=READY pgs=1253 cs=0 l=1 rev1=1 crypto rx=0x7f14a800c6e0 tx=0x7f14a800cba0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 shutdown_connections 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f149006b4d0 0x7f149006d990 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f14b015e230 0x7f14b009c670 unknown :-1 s=CLOSED pgs=1253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14b00ce550 0x7f14b009e0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 --2- 172.21.15.86:0/3953788547 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14b00b4320 0x7f14b01f42e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 >> 172.21.15.86:0/3953788547 conn(0x7f14b00b3800 msgr2=0x7f14b015d0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 shutdown_connections 2024-03-06T07:33:09.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:09.943+0000 7f14b5200640 1 -- 172.21.15.86:0/3953788547 wait complete. 2024-03-06T07:33:09.962 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:33:10.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.158+0000 7fe166c00640 1 Processor -- start 2024-03-06T07:33:10.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.158+0000 7fe166c00640 1 -- start start 2024-03-06T07:33:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe166c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe160160460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe166c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe1600cf9a0 0x7fe1600cfe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe166c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe1600b3040 con 0x7fe16015f770 2024-03-06T07:33:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe166c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe16015fb50 con 0x7fe1600cf9a0 2024-03-06T07:33:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe166c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe1600cd3e0 con 0x7fe1600b4390 2024-03-06T07:33:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe164c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe1600cf9a0 0x7fe1600cfe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe164c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe1600cf9a0 0x7fe1600cfe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36194/0 (socket says 172.21.15.86:36194) 2024-03-06T07:33:10.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe164c00640 1 -- 172.21.15.86:0/3523713029 learned_addr learned my addr 172.21.15.86:0/3523713029 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe157e00640 1 --2- 172.21.15.86:0/3523713029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe160160460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe164c00640 1 -- 172.21.15.86:0/3523713029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 msgr2=0x7fe160160460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe164c00640 1 --2- 172.21.15.86:0/3523713029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe160160460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe164c00640 1 -- 172.21.15.86:0/3523713029 >> v1:172.21.15.194:6789/0 conn(0x7fe16015f770 legacy=0x7fe16015fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:10.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.160+0000 7fe164c00640 1 -- 172.21.15.86:0/3523713029 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1600cd6c0 con 0x7fe1600cf9a0 2024-03-06T07:33:10.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.161+0000 7fe164c00640 1 --2- 172.21.15.86:0/3523713029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1600cf9a0 0x7fe1600cfe30 secure :-1 s=READY pgs=1254 cs=0 l=1 rev1=1 crypto rx=0x7fe158009e50 tx=0x7fe15802be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44ccf78445e96553 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:10.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.162+0000 7fe15f400640 1 -- 172.21.15.86:0/3523713029 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe15802c9e0 con 0x7fe1600cf9a0 2024-03-06T07:33:10.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.162+0000 7fe15f400640 1 -- 172.21.15.86:0/3523713029 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe15802cb80 con 0x7fe1600cf9a0 2024-03-06T07:33:10.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.162+0000 7fe15f400640 1 -- 172.21.15.86:0/3523713029 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe1580343f0 con 0x7fe1600cf9a0 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.162+0000 7fe166c00640 1 -- 172.21.15.86:0/3523713029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1600cf9a0 msgr2=0x7fe1600cfe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.162+0000 7fe166c00640 1 --2- 172.21.15.86:0/3523713029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1600cf9a0 0x7fe1600cfe30 secure :-1 s=READY pgs=1254 cs=0 l=1 rev1=1 crypto rx=0x7fe158009e50 tx=0x7fe15802be40 comp rx=0 tx=0).stop 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.163+0000 7fe166c00640 1 -- 172.21.15.86:0/3523713029 shutdown_connections 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.163+0000 7fe166c00640 1 --2- 172.21.15.86:0/3523713029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1600cf9a0 0x7fe1600cfe30 unknown :-1 s=CLOSED pgs=1254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.163+0000 7fe166c00640 1 --2- 172.21.15.86:0/3523713029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe160160460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.163+0000 7fe166c00640 1 -- 172.21.15.86:0/3523713029 >> 172.21.15.86:0/3523713029 conn(0x7fe1600b3870 msgr2=0x7fe1600b3c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.163+0000 7fe166c00640 1 -- 172.21.15.86:0/3523713029 shutdown_connections 2024-03-06T07:33:10.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.163+0000 7fe166c00640 1 -- 172.21.15.86:0/3523713029 wait complete. 2024-03-06T07:33:10.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.165+0000 7fe166c00640 1 Processor -- start 2024-03-06T07:33:10.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.165+0000 7fe166c00640 1 -- start start 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe166c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe16014b970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe166c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1600cf9a0 0x7fe1600d1d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe166c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe16015f770 0x7fe1600d2260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe166c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1600b34a0 con 0x7fe16015f770 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe166c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1600b3320 con 0x7fe1600cf9a0 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe166c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe1601609a0 con 0x7fe1600b4390 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe164c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe16015f770 0x7fe1600d2260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe164c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe16015f770 0x7fe1600d2260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36200/0 (socket says 172.21.15.86:36200) 2024-03-06T07:33:10.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.167+0000 7fe164c00640 1 -- 172.21.15.86:0/668177096 learned_addr learned my addr 172.21.15.86:0/668177096 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe15fe00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe16014b970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe164c00640 1 -- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 msgr2=0x7fe16014b970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe157e00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1600cf9a0 0x7fe1600d1d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe164c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe16014b970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe164c00640 1 -- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1600cf9a0 msgr2=0x7fe1600d1d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe164c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1600cf9a0 0x7fe1600d1d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe164c00640 1 -- 172.21.15.86:0/668177096 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe160157160 con 0x7fe16015f770 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe164c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe16015f770 0x7fe1600d2260 secure :-1 s=READY pgs=1255 cs=0 l=1 rev1=1 crypto rx=0x7fe158036040 tx=0x7fe158004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe15cc00640 1 -- 172.21.15.86:0/668177096 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe158043020 con 0x7fe16015f770 2024-03-06T07:33:10.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe15cc00640 1 -- 172.21.15.86:0/668177096 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe15803e450 con 0x7fe16015f770 2024-03-06T07:33:10.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.168+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe160157440 con 0x7fe16015f770 2024-03-06T07:33:10.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.169+0000 7fe15cc00640 1 -- 172.21.15.86:0/668177096 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe158009510 con 0x7fe16015f770 2024-03-06T07:33:10.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.169+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe160157a00 con 0x7fe16015f770 2024-03-06T07:33:10.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.170+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe16004f330 con 0x7fe16015f770 2024-03-06T07:33:10.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.171+0000 7fe15cc00640 1 -- 172.21.15.86:0/668177096 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe158046020 con 0x7fe16015f770 2024-03-06T07:33:10.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.172+0000 7fe15cc00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe13c06b4d0 0x7fe13c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.172+0000 7fe15cc00640 1 -- 172.21.15.86:0/668177096 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe1580ac910 con 0x7fe16015f770 2024-03-06T07:33:10.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.172+0000 7fe15fe00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe13c06b4d0 0x7fe13c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.173+0000 7fe15fe00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe13c06b4d0 0x7fe13c06d990 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7fe15000a750 tx=0x7fe150008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:10.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.177+0000 7fe15cc00640 1 -- 172.21.15.86:0/668177096 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe15804d030 con 0x7fe16015f770 2024-03-06T07:33:10.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.365+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe1600ce130 con 0x7fe16015f770 2024-03-06T07:33:10.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.368+0000 7fe15cc00640 1 -- 172.21.15.86:0/668177096 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v110) v1 ==== 74+0+16639 (secure 0 0 0) 0x7fe15807b820 con 0x7fe16015f770 2024-03-06T07:33:10.370 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:10.370 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":110,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:07.410898+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"110","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe13c06b4d0 msgr2=0x7fe13c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe13c06b4d0 0x7fe13c06d990 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7fe15000a750 tx=0x7fe150008040 comp rx=0 tx=0).stop 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe16015f770 msgr2=0x7fe1600d2260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe16015f770 0x7fe1600d2260 secure :-1 s=READY pgs=1255 cs=0 l=1 rev1=1 crypto rx=0x7fe158036040 tx=0x7fe158004a40 comp rx=0 tx=0).stop 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 shutdown_connections 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe13c06b4d0 0x7fe13c06d990 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe16015f770 0x7fe1600d2260 unknown :-1 s=CLOSED pgs=1255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1600cf9a0 0x7fe1600d1d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 --2- 172.21.15.86:0/668177096 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1600b4390 0x7fe16014b970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 >> 172.21.15.86:0/668177096 conn(0x7fe1600b3870 msgr2=0x7fe1600b3c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:10.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 shutdown_connections 2024-03-06T07:33:10.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.372+0000 7fe166c00640 1 -- 172.21.15.86:0/668177096 wait complete. 2024-03-06T07:33:10.390 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:33:10.390 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:33:10.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.583+0000 7f39abc00640 1 Processor -- start 2024-03-06T07:33:10.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.584+0000 7f39abc00640 1 -- start start 2024-03-06T07:33:10.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.585+0000 7f39abc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f39a40b2de0 0x7f39a40d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.585+0000 7f39abc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 0x7f39a40b0f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.585+0000 7f39abc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39a40d4c50 con 0x7f39a40d24c0 2024-03-06T07:33:10.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.585+0000 7f39abc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f39a40d4a00 con 0x7f39a40b2de0 2024-03-06T07:33:10.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.585+0000 7f39abc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39a40cf7e0 con 0x7f39a40b0aa0 2024-03-06T07:33:10.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.585+0000 7f39a9200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f39a40d24c0 0x7f39a40d28a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53604/0 (socket says 172.21.15.86:53604) 2024-03-06T07:33:10.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.585+0000 7f39a9200640 1 -- 172.21.15.86:0/2289367275 learned_addr learned my addr 172.21.15.86:0/2289367275 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.586+0000 7f39a9c00640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 0x7f39a40b0f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.586+0000 7f39a3e00640 1 --2- 172.21.15.86:0/2289367275 >> v2:172.21.15.194:3300/0 conn(0x7f39a40b2de0 0x7f39a40d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.586+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1040123169 0 0) 0x7f39a40d4c50 con 0x7f39a40d24c0 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.586+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f398c003520 con 0x7f39a40d24c0 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.586+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 412072627 0 0) 0x7f398c003520 con 0x7f39a40d24c0 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.586+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 msgr2=0x7f39a40b0f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.586+0000 7f39a3400640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 0x7f39a40b0f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 >> v2:172.21.15.194:3300/0 conn(0x7f39a40b2de0 msgr2=0x7f39a40d1cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 --2- 172.21.15.86:0/2289367275 >> v2:172.21.15.194:3300/0 conn(0x7f39a40b2de0 0x7f39a40d1cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39a40cfa30 con 0x7f39a40d24c0 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3998001670 con 0x7f39a40d24c0 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 >> v1:172.21.15.86:6789/0 conn(0x7f39a40d24c0 legacy=0x7f39a40d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 0x7f398c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f398c003f30 0x7f398c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f398c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39a40d4a00 con 0x7f398c004d40 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39a40cf7e0 con 0x7f398c003f30 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f398c00ba10 con 0x7f39a40b0aa0 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a9c00640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f398c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a9200640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 0x7f398c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.587+0000 7f39a3e00640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f398c003f30 0x7f398c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.588+0000 7f39a9c00640 1 -- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 msgr2=0x7f398c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.588+0000 7f39a9c00640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b0aa0 0x7f398c0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.588+0000 7f39a9c00640 1 -- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f398c003f30 msgr2=0x7f398c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.588+0000 7f39a9c00640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f398c003f30 0x7f398c004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.588+0000 7f39a9c00640 1 -- 172.21.15.86:0/2289367275 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f398c003520 con 0x7f398c004d40 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.589+0000 7f39a9c00640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f398c0051c0 secure :-1 s=READY pgs=1257 cs=0 l=1 rev1=1 crypto rx=0x7f399c00e170 tx=0x7f398c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cee5dbfa6e062aa0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.589+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f399c039070 con 0x7f398c004d40 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.589+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f399c02cbf0 con 0x7f398c004d40 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.589+0000 7f39a3400640 1 -- 172.21.15.86:0/2289367275 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f399c009c10 con 0x7f398c004d40 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.589+0000 7f39abc00640 1 -- 172.21.15.86:0/2289367275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 msgr2=0x7f398c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.589+0000 7f39abc00640 1 --2- 172.21.15.86:0/2289367275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f398c0051c0 secure :-1 s=READY pgs=1257 cs=0 l=1 rev1=1 crypto rx=0x7f399c00e170 tx=0x7f398c005830 comp rx=0 tx=0).stop 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.590+0000 7f39a9c00640 1 -- 172.21.15.86:0/2289367275 reap_dead start 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.590+0000 7f39abc00640 1 -- 172.21.15.86:0/2289367275 shutdown_connections 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.590+0000 7f39abc00640 1 -- 172.21.15.86:0/2289367275 >> 172.21.15.86:0/2289367275 conn(0x7f39a40d3ee0 msgr2=0x7f39a40d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.590+0000 7f39abc00640 1 -- 172.21.15.86:0/2289367275 shutdown_connections 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.590+0000 7f39abc00640 1 -- 172.21.15.86:0/2289367275 wait complete. 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.591+0000 7f39abc00640 1 Processor -- start 2024-03-06T07:33:10.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.592+0000 7f39abc00640 1 -- start start 2024-03-06T07:33:10.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.592+0000 7f39abc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f39a414f430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.592+0000 7f39abc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a40c8420 0x7f39a40c88a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.592+0000 7f39abc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b5930 0x7f39a40b5d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.592+0000 7f39abc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39a415e5b0 con 0x7f398c004d40 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.592+0000 7f39abc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39a40d0080 con 0x7f39a40c8420 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.592+0000 7f39abc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39a415e730 con 0x7f39a40b5930 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f39a414f430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f39a414f430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36208/0 (socket says 172.21.15.86:36208) 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 -- 172.21.15.86:0/2539022041 learned_addr learned my addr 172.21.15.86:0/2539022041 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 -- 172.21.15.86:0/2539022041 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b5930 msgr2=0x7f39a40b5d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a3e00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a40c8420 0x7f39a40c88a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b5930 0x7f39a40b5d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 -- 172.21.15.86:0/2539022041 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a40c8420 msgr2=0x7f39a40c88a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a40c8420 0x7f39a40c88a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 -- 172.21.15.86:0/2539022041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39a40cd320 con 0x7f398c004d40 2024-03-06T07:33:10.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.593+0000 7f39a9200640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f39a414f430 secure :-1 s=READY pgs=1258 cs=0 l=1 rev1=1 crypto rx=0x7f399800e840 tx=0x7f399800ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:10.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.594+0000 7f39a0c00640 1 -- 172.21.15.86:0/2539022041 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f399800c7f0 con 0x7f398c004d40 2024-03-06T07:33:10.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.594+0000 7f39a0c00640 1 -- 172.21.15.86:0/2539022041 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3998018070 con 0x7f398c004d40 2024-03-06T07:33:10.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.594+0000 7f39a0c00640 1 -- 172.21.15.86:0/2539022041 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f399800cbf0 con 0x7f398c004d40 2024-03-06T07:33:10.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.594+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39a40c8f90 con 0x7f398c004d40 2024-03-06T07:33:10.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.594+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39a40c9550 con 0x7f398c004d40 2024-03-06T07:33:10.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.595+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f39a40b62d0 con 0x7f398c004d40 2024-03-06T07:33:10.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.596+0000 7f39a0c00640 1 -- 172.21.15.86:0/2539022041 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3998013420 con 0x7f398c004d40 2024-03-06T07:33:10.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.597+0000 7f39a0c00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f398406b500 0x7f398406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:10.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.597+0000 7f39a0c00640 1 -- 172.21.15.86:0/2539022041 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3998089ea0 con 0x7f398c004d40 2024-03-06T07:33:10.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.597+0000 7f39a3e00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f398406b500 0x7f398406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:10.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.598+0000 7f39a3e00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f398406b500 0x7f398406d9c0 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f39a40d1250 tx=0x7f3990008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:10.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.601+0000 7f39a0c00640 1 -- 172.21.15.86:0/2539022041 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3998058c80 con 0x7f398c004d40 2024-03-06T07:33:10.757 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:33:10.756+0000 7efc1c400640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:33:10.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.784+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f39a404f330 con 0x7f398406b500 2024-03-06T07:33:10.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.796+0000 7f39a0c00640 1 -- 172.21.15.86:0/2539022041 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+240297 (secure 0 0 0) 0x7f39a404f330 con 0x7f398406b500 2024-03-06T07:33:10.800 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:10.802 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f398406b500 msgr2=0x7f398406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f398406b500 0x7f398406d9c0 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f39a40d1250 tx=0x7f3990008040 comp rx=0 tx=0).stop 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 msgr2=0x7f39a414f430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f39a414f430 secure :-1 s=READY pgs=1258 cs=0 l=1 rev1=1 crypto rx=0x7f399800e840 tx=0x7f399800ed00 comp rx=0 tx=0).stop 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 shutdown_connections 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f398406b500 0x7f398406d9c0 unknown :-1 s=CLOSED pgs=547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a40b5930 0x7f39a40b5d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a40c8420 0x7f39a40c88a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 --2- 172.21.15.86:0/2539022041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f398c004d40 0x7f39a414f430 unknown :-1 s=CLOSED pgs=1258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 >> 172.21.15.86:0/2539022041 conn(0x7f39a40d3ee0 msgr2=0x7f39a40b51d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 shutdown_connections 2024-03-06T07:33:10.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:10.803+0000 7f39abc00640 1 -- 172.21.15.86:0/2539022041 wait complete. 2024-03-06T07:33:10.821 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":288,"stamp":"2024-03-06T07:33:08.960718+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":66,"acting":66,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":5,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":294256,"kb_used_data":11516,"kb_used_omap":12,"kb_used_meta":282611,"kb_avail":754680464,"statfs":{"total":773094113280,"available":772792795136,"internally_reserved":0,"allocated":11792384,"data_stored":5718143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":289394227},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.595948"},"pg_stats":[{"pgid":"17.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.3e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.31","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.25","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"17.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.1f","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.381911+0000","last_change":"2024-03-06T07:33:04.381911+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.381911+0000","last_undegraded":"2024-03-06T07:33:04.381911+0000","last_fullsized":"2024-03-06T07:33:04.381911+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.1e","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.378876+0000","last_change":"2024-03-06T07:33:04.378876+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.378876+0000","last_undegraded":"2024-03-06T07:33:04.378876+0000","last_fullsized":"2024-03-06T07:33:04.378876+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.1d","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.378610+0000","last_change":"2024-03-06T07:33:04.378610+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.378610+0000","last_undegraded":"2024-03-06T07:33:04.378610+0000","last_fullsized":"2024-03-06T07:33:04.378610+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.1c","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.380576+0000","last_change":"2024-03-06T07:33:04.380576+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.380576+0000","last_undegraded":"2024-03-06T07:33:04.380576+0000","last_fullsized":"2024-03-06T07:33:04.380576+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.1b","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.383017+0000","last_change":"2024-03-06T07:33:04.383017+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.383017+0000","last_undegraded":"2024-03-06T07:33:04.383017+0000","last_fullsized":"2024-03-06T07:33:04.383017+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.1a","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.371770+0000","last_change":"2024-03-06T07:33:04.371770+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.371770+0000","last_undegraded":"2024-03-06T07:33:04.371770+0000","last_fullsized":"2024-03-06T07:33:04.371770+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.19","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.368368+0000","last_change":"2024-03-06T07:33:04.368368+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.368368+0000","last_undegraded":"2024-03-06T07:33:04.368368+0000","last_fullsized":"2024-03-06T07:33:04.368368+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.18","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.376738+0000","last_change":"2024-03-06T07:33:04.376738+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.376738+0000","last_undegraded":"2024-03-06T07:33:04.376738+0000","last_fullsized":"2024-03-06T07:33:04.376738+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.17","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.378973+0000","last_change":"2024-03-06T07:33:04.378973+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.378973+0000","last_undegraded":"2024-03-06T07:33:04.378973+0000","last_fullsized":"2024-03-06T07:33:04.378973+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.16","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.371129+0000","last_change":"2024-03-06T07:33:04.371129+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.371129+0000","last_undegraded":"2024-03-06T07:33:04.371129+0000","last_fullsized":"2024-03-06T07:33:04.371129+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.15","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.376729+0000","last_change":"2024-03-06T07:33:04.376729+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.376729+0000","last_undegraded":"2024-03-06T07:33:04.376729+0000","last_fullsized":"2024-03-06T07:33:04.376729+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.14","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.375487+0000","last_change":"2024-03-06T07:33:04.375487+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.375487+0000","last_undegraded":"2024-03-06T07:33:04.375487+0000","last_fullsized":"2024-03-06T07:33:04.375487+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.13","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.373608+0000","last_change":"2024-03-06T07:33:04.373608+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.373608+0000","last_undegraded":"2024-03-06T07:33:04.373608+0000","last_fullsized":"2024-03-06T07:33:04.373608+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.12","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.376245+0000","last_change":"2024-03-06T07:33:04.376245+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.376245+0000","last_undegraded":"2024-03-06T07:33:04.376245+0000","last_fullsized":"2024-03-06T07:33:04.376245+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.10","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.380238+0000","last_change":"2024-03-06T07:33:04.380238+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.380238+0000","last_undegraded":"2024-03-06T07:33:04.380238+0000","last_fullsized":"2024-03-06T07:33:04.380238+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.f","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.375495+0000","last_change":"2024-03-06T07:33:04.375495+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.375495+0000","last_undegraded":"2024-03-06T07:33:04.375495+0000","last_fullsized":"2024-03-06T07:33:04.375495+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.e","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.381055+0000","last_change":"2024-03-06T07:33:04.381055+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.381055+0000","last_undegraded":"2024-03-06T07:33:04.381055+0000","last_fullsized":"2024-03-06T07:33:04.381055+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.d","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.376089+0000","last_change":"2024-03-06T07:33:04.376089+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.376089+0000","last_undegraded":"2024-03-06T07:33:04.376089+0000","last_fullsized":"2024-03-06T07:33:04.376089+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.c","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.381911+0000","last_change":"2024-03-06T07:33:04.381911+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.381911+0000","last_undegraded":"2024-03-06T07:33:04.381911+0000","last_fullsized":"2024-03-06T07:33:04.381911+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.b","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.372423+0000","last_change":"2024-03-06T07:33:04.372423+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.372423+0000","last_undegraded":"2024-03-06T07:33:04.372423+0000","last_fullsized":"2024-03-06T07:33:04.372423+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.a","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.379464+0000","last_change":"2024-03-06T07:33:04.379464+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.379464+0000","last_undegraded":"2024-03-06T07:33:04.379464+0000","last_fullsized":"2024-03-06T07:33:04.379464+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.9","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.372376+0000","last_change":"2024-03-06T07:33:04.372376+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.372376+0000","last_undegraded":"2024-03-06T07:33:04.372376+0000","last_fullsized":"2024-03-06T07:33:04.372376+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.8","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.376213+0000","last_change":"2024-03-06T07:33:04.376213+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.376213+0000","last_undegraded":"2024-03-06T07:33:04.376213+0000","last_fullsized":"2024-03-06T07:33:04.376213+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.7","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.371050+0000","last_change":"2024-03-06T07:33:04.371050+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.371050+0000","last_undegraded":"2024-03-06T07:33:04.371050+0000","last_fullsized":"2024-03-06T07:33:04.371050+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.6","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.376297+0000","last_change":"2024-03-06T07:33:04.376297+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.376297+0000","last_undegraded":"2024-03-06T07:33:04.376297+0000","last_fullsized":"2024-03-06T07:33:04.376297+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.5","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.375676+0000","last_change":"2024-03-06T07:33:04.375676+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.375676+0000","last_undegraded":"2024-03-06T07:33:04.375676+0000","last_fullsized":"2024-03-06T07:33:04.375676+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.4","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.371105+0000","last_change":"2024-03-06T07:33:04.371105+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.371105+0000","last_undegraded":"2024-03-06T07:33:04.371105+0000","last_fullsized":"2024-03-06T07:33:04.371105+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.3","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.381727+0000","last_change":"2024-03-06T07:33:04.381727+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.381727+0000","last_undegraded":"2024-03-06T07:33:04.381727+0000","last_fullsized":"2024-03-06T07:33:04.381727+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.2","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.368604+0000","last_change":"2024-03-06T07:33:04.368604+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.368604+0000","last_undegraded":"2024-03-06T07:33:04.368604+0000","last_fullsized":"2024-03-06T07:33:04.368604+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.1","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.379471+0000","last_change":"2024-03-06T07:33:04.379471+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.379471+0000","last_undegraded":"2024-03-06T07:33:04.379471+0000","last_fullsized":"2024-03-06T07:33:04.379471+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.0","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.375513+0000","last_change":"2024-03-06T07:33:04.375513+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.375513+0000","last_undegraded":"2024-03-06T07:33:04.375513+0000","last_fullsized":"2024-03-06T07:33:04.375513+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-06T07:33:05.361395+0000","last_change":"2024-03-06T07:33:05.361395+0000","last_active":"2024-03-06T07:33:05.361395+0000","last_peered":"2024-03-06T07:33:05.361395+0000","last_clean":"2024-03-06T07:33:05.361395+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:05.361395+0000","last_undegraded":"2024-03-06T07:33:05.361395+0000","last_fullsized":"2024-03-06T07:33:05.361395+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"16.11","version":"0'0","reported_seq":1,"reported_epoch":107,"state":"creating+peering","last_fresh":"2024-03-06T07:33:04.381663+0000","last_change":"2024-03-06T07:33:04.381663+0000","last_active":"2024-03-06T07:33:04.344828+0000","last_peered":"2024-03-06T07:33:04.344828+0000","last_clean":"2024-03-06T07:33:04.344828+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-06T07:33:04.381663+0000","last_undegraded":"2024-03-06T07:33:04.381663+0000","last_fullsized":"2024-03-06T07:33:04.381663+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":325,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-03-06T07:33:04.379840+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:33:04.379840+0000","last_peered":"2024-03-06T07:33:04.379840+0000","last_clean":"2024-03-06T07:33:04.379840+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:33:04.379840+0000","last_undegraded":"2024-03-06T07:33:04.379840+0000","last_fullsized":"2024-03-06T07:33:04.379840+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":17,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":16,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542221,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37220,"kb_used_data":1876,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334620,"statfs":{"total":96636764160,"available":96598650880,"internally_reserved":0,"allocated":1921024,"data_stored":1158271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542220,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36636,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335204,"statfs":{"total":96636764160,"available":96599248896,"internally_reserved":0,"allocated":1327104,"data_stored":567903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542221,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36636,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335204,"statfs":{"total":96636764160,"available":96599248896,"internally_reserved":0,"allocated":1327104,"data_stored":567903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542220,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36636,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335204,"statfs":{"total":96636764160,"available":96599248896,"internally_reserved":0,"allocated":1327104,"data_stored":567903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542220,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36640,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335200,"statfs":{"total":96636764160,"available":96599244800,"internally_reserved":0,"allocated":1327104,"data_stored":567903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542221,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36628,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335212,"statfs":{"total":96636764160,"available":96599257088,"internally_reserved":0,"allocated":1314816,"data_stored":562086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542222,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36644,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335196,"statfs":{"total":96636764160,"available":96599240704,"internally_reserved":0,"allocated":1327104,"data_stored":567903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542219,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37216,"kb_used_data":1876,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334624,"statfs":{"total":96636764160,"available":96598654976,"internally_reserved":0,"allocated":1921024,"data_stored":1158271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:33:10.828 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:33:11.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.066+0000 7f8890000640 1 Processor -- start 2024-03-06T07:33:11.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.067+0000 7f8890000640 1 -- start start 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f8890000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f888815e0d0 0x7f88880d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f8890000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f888815f470 0x7f88880d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f8890000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f88880d1200 con 0x7f88880d0df0 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f8890000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f88880d2f60 con 0x7f888815f470 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f8890000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f88880cea60 con 0x7f888815e0d0 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f888cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f888815f470 0x7f88880d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f888cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f888815f470 0x7f88880d08b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36214/0 (socket says 172.21.15.86:36214) 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f888cc00640 1 -- 172.21.15.86:0/511267256 learned_addr learned my addr 172.21.15.86:0/511267256 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:11.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.068+0000 7f888e000640 1 --1- 172.21.15.86:0/511267256 >> v1:172.21.15.86:6789/0 conn(0x7f88880d0df0 0x7f8888160dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53620/0 (socket says 172.21.15.86:53620) 2024-03-06T07:33:11.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.069+0000 7f888d600640 1 --2- 172.21.15.86:0/511267256 >> v2:172.21.15.194:3300/0 conn(0x7f888815e0d0 0x7f88880d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.069+0000 7f888cc00640 1 -- 172.21.15.86:0/511267256 >> v2:172.21.15.194:3300/0 conn(0x7f888815e0d0 msgr2=0x7f88880d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:11.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.069+0000 7f888cc00640 1 --2- 172.21.15.86:0/511267256 >> v2:172.21.15.194:3300/0 conn(0x7f888815e0d0 0x7f88880d1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.069+0000 7f888cc00640 1 -- 172.21.15.86:0/511267256 >> v1:172.21.15.86:6789/0 conn(0x7f88880d0df0 legacy=0x7f8888160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.069+0000 7f888cc00640 1 -- 172.21.15.86:0/511267256 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88880d2350 con 0x7f888815f470 2024-03-06T07:33:11.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.069+0000 7f8887e00640 1 -- 172.21.15.86:0/511267256 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3121553864 0 0) 0x7f8880001110 con 0x7f88880d0df0 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.070+0000 7f888cc00640 1 --2- 172.21.15.86:0/511267256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f888815f470 0x7f88880d08b0 secure :-1 s=READY pgs=1260 cs=0 l=1 rev1=1 crypto rx=0x7f88880b6a80 tx=0x7f887402ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc3fa3d31b3db5fd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.070+0000 7f8887e00640 1 -- 172.21.15.86:0/511267256 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f887402f850 con 0x7f888815f470 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.070+0000 7f8887e00640 1 -- 172.21.15.86:0/511267256 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f887402f9f0 con 0x7f888815f470 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.070+0000 7f8887e00640 1 -- 172.21.15.86:0/511267256 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f887402fdc0 con 0x7f888815f470 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 -- 172.21.15.86:0/511267256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f888815f470 msgr2=0x7f88880d08b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 --2- 172.21.15.86:0/511267256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f888815f470 0x7f88880d08b0 secure :-1 s=READY pgs=1260 cs=0 l=1 rev1=1 crypto rx=0x7f88880b6a80 tx=0x7f887402ba30 comp rx=0 tx=0).stop 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 -- 172.21.15.86:0/511267256 shutdown_connections 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 --2- 172.21.15.86:0/511267256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f888815f470 0x7f88880d08b0 unknown :-1 s=CLOSED pgs=1260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 --2- 172.21.15.86:0/511267256 >> v2:172.21.15.194:3300/0 conn(0x7f888815e0d0 0x7f88880d1a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 -- 172.21.15.86:0/511267256 >> 172.21.15.86:0/511267256 conn(0x7f88880d3ec0 msgr2=0x7f88880d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:11.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 -- 172.21.15.86:0/511267256 shutdown_connections 2024-03-06T07:33:11.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.071+0000 7f8890000640 1 -- 172.21.15.86:0/511267256 wait complete. 2024-03-06T07:33:11.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.073+0000 7f8890000640 1 Processor -- start 2024-03-06T07:33:11.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.073+0000 7f8890000640 1 -- start start 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f8890000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88880d0df0 0x7f8888155b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f8890000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f888815e0d0 0x7f88881528d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f8890000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f888815f470 0x7f88880b8c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f8890000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88880d2bf0 con 0x7f88880d0df0 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f8890000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88880d2910 con 0x7f888815f470 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f8890000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f888815fb30 con 0x7f888815e0d0 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f888d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88880d0df0 0x7f8888155b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f888d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88880d0df0 0x7f8888155b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36222/0 (socket says 172.21.15.86:36222) 2024-03-06T07:33:11.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.074+0000 7f888d600640 1 -- 172.21.15.86:0/1878578643 learned_addr learned my addr 172.21.15.86:0/1878578643 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888cc00640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f888815e0d0 0x7f88881528d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888d600640 1 -- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f888815e0d0 msgr2=0x7f88881528d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888d600640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f888815e0d0 0x7f88881528d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888d600640 1 -- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f888815f470 msgr2=0x7f88880b8c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888d600640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f888815f470 0x7f88880b8c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888d600640 1 -- 172.21.15.86:0/1878578643 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88880cb420 con 0x7f88880d0df0 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888cc00640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f888815e0d0 0x7f88881528d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f888d600640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88880d0df0 0x7f8888155b80 secure :-1 s=READY pgs=1261 cs=0 l=1 rev1=1 crypto rx=0x7f887c007ef0 tx=0x7f887c00e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f8885600640 1 -- 172.21.15.86:0/1878578643 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f887c014620 con 0x7f88880d0df0 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f8885600640 1 -- 172.21.15.86:0/1878578643 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f887c019070 con 0x7f88880d0df0 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f88880cb700 con 0x7f88880d0df0 2024-03-06T07:33:11.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.075+0000 7f8885600640 1 -- 172.21.15.86:0/1878578643 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f887c014a00 con 0x7f88880d0df0 2024-03-06T07:33:11.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.076+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f88880cbcc0 con 0x7f88880d0df0 2024-03-06T07:33:11.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.077+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f888804f330 con 0x7f88880d0df0 2024-03-06T07:33:11.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.077+0000 7f8885600640 1 -- 172.21.15.86:0/1878578643 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f887c014ba0 con 0x7f88880d0df0 2024-03-06T07:33:11.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.078+0000 7f8885600640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f886806b4d0 0x7f886806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.078+0000 7f8885600640 1 -- 172.21.15.86:0/1878578643 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f887c08abe0 con 0x7f88880d0df0 2024-03-06T07:33:11.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.079+0000 7f888cc00640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f886806b4d0 0x7f886806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.079+0000 7f888cc00640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f886806b4d0 0x7f886806d990 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f8874036000 tx=0x7f8874039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:11.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.083+0000 7f8885600640 1 -- 172.21.15.86:0/1878578643 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f887c0599c0 con 0x7f88880d0df0 2024-03-06T07:33:11.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.275+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f888814fbc0 con 0x7f886806b4d0 2024-03-06T07:33:11.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.284+0000 7f8885600640 1 -- 172.21.15.86:0/1878578643 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252483 (secure 0 0 0) 0x7f887402f890 con 0x7f886806b4d0 2024-03-06T07:33:11.291 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:11.292 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:33:11.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f886806b4d0 msgr2=0x7f886806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f886806b4d0 0x7f886806d990 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f8874036000 tx=0x7f8874039040 comp rx=0 tx=0).stop 2024-03-06T07:33:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88880d0df0 msgr2=0x7f8888155b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88880d0df0 0x7f8888155b80 secure :-1 s=READY pgs=1261 cs=0 l=1 rev1=1 crypto rx=0x7f887c007ef0 tx=0x7f887c00e4b0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 shutdown_connections 2024-03-06T07:33:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f886806b4d0 0x7f886806d990 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f888815f470 0x7f88880b8c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f888815e0d0 0x7f88881528d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 --2- 172.21.15.86:0/1878578643 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88880d0df0 0x7f8888155b80 unknown :-1 s=CLOSED pgs=1261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 >> 172.21.15.86:0/1878578643 conn(0x7f88880d3ec0 msgr2=0x7f88880d42a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 shutdown_connections 2024-03-06T07:33:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.290+0000 7f8890000640 1 -- 172.21.15.86:0/1878578643 wait complete. 2024-03-06T07:33:11.309 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":289,"stamp":"2024-03-06T07:33:10.961234+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":180,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":294656,"kb_used_data":11880,"kb_used_omap":12,"kb_used_meta":282611,"kb_avail":754680064,"statfs":{"total":773094113280,"available":772792385536,"internally_reserved":0,"allocated":12165120,"data_stored":5883580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":289394227},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":37,"apply_latency_ms":37,"commit_latency_ns":37000000,"apply_latency_ns":37000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001540"},"pg_stats":[{"pgid":"17.3f","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673862+0000","last_change":"2024-03-06T07:33:06.437174+0000","last_active":"2024-03-06T07:33:07.673862+0000","last_peered":"2024-03-06T07:33:07.673862+0000","last_clean":"2024-03-06T07:33:07.673862+0000","last_became_active":"2024-03-06T07:33:06.436914+0000","last_became_peered":"2024-03-06T07:33:06.436914+0000","last_unstale":"2024-03-06T07:33:07.673862+0000","last_undegraded":"2024-03-06T07:33:07.673862+0000","last_fullsized":"2024-03-06T07:33:07.673862+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:20:45.788483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.3e","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.422666+0000","last_change":"2024-03-06T07:33:06.427948+0000","last_active":"2024-03-06T07:33:07.422666+0000","last_peered":"2024-03-06T07:33:07.422666+0000","last_clean":"2024-03-06T07:33:07.422666+0000","last_became_active":"2024-03-06T07:33:06.427677+0000","last_became_peered":"2024-03-06T07:33:06.427677+0000","last_unstale":"2024-03-06T07:33:07.422666+0000","last_undegraded":"2024-03-06T07:33:07.422666+0000","last_fullsized":"2024-03-06T07:33:07.422666+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:11:04.349061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.3d","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674376+0000","last_change":"2024-03-06T07:33:06.437534+0000","last_active":"2024-03-06T07:33:07.674376+0000","last_peered":"2024-03-06T07:33:07.674376+0000","last_clean":"2024-03-06T07:33:07.674376+0000","last_became_active":"2024-03-06T07:33:06.436042+0000","last_became_peered":"2024-03-06T07:33:06.436042+0000","last_unstale":"2024-03-06T07:33:07.674376+0000","last_undegraded":"2024-03-06T07:33:07.674376+0000","last_fullsized":"2024-03-06T07:33:07.674376+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:34.836836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.3c","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673167+0000","last_change":"2024-03-06T07:33:06.436813+0000","last_active":"2024-03-06T07:33:07.673167+0000","last_peered":"2024-03-06T07:33:07.673167+0000","last_clean":"2024-03-06T07:33:07.673167+0000","last_became_active":"2024-03-06T07:33:06.431008+0000","last_became_peered":"2024-03-06T07:33:06.431008+0000","last_unstale":"2024-03-06T07:33:07.673167+0000","last_undegraded":"2024-03-06T07:33:07.673167+0000","last_fullsized":"2024-03-06T07:33:07.673167+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:43:46.084762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.3b","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.419841+0000","last_change":"2024-03-06T07:33:06.421065+0000","last_active":"2024-03-06T07:33:07.419841+0000","last_peered":"2024-03-06T07:33:07.419841+0000","last_clean":"2024-03-06T07:33:07.419841+0000","last_became_active":"2024-03-06T07:33:06.420751+0000","last_became_peered":"2024-03-06T07:33:06.420751+0000","last_unstale":"2024-03-06T07:33:07.419841+0000","last_undegraded":"2024-03-06T07:33:07.419841+0000","last_fullsized":"2024-03-06T07:33:07.419841+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:10:39.965505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.3a","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.671730+0000","last_change":"2024-03-06T07:33:06.431711+0000","last_active":"2024-03-06T07:33:07.671730+0000","last_peered":"2024-03-06T07:33:07.671730+0000","last_clean":"2024-03-06T07:33:07.671730+0000","last_became_active":"2024-03-06T07:33:06.431374+0000","last_became_peered":"2024-03-06T07:33:06.431374+0000","last_unstale":"2024-03-06T07:33:07.671730+0000","last_undegraded":"2024-03-06T07:33:07.671730+0000","last_fullsized":"2024-03-06T07:33:07.671730+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:15:36.751438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.39","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.445293+0000","last_change":"2024-03-06T07:33:06.420434+0000","last_active":"2024-03-06T07:33:07.445293+0000","last_peered":"2024-03-06T07:33:07.445293+0000","last_clean":"2024-03-06T07:33:07.445293+0000","last_became_active":"2024-03-06T07:33:06.420094+0000","last_became_peered":"2024-03-06T07:33:06.420094+0000","last_unstale":"2024-03-06T07:33:07.445293+0000","last_undegraded":"2024-03-06T07:33:07.445293+0000","last_fullsized":"2024-03-06T07:33:07.445293+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:28:06.784675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.38","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674913+0000","last_change":"2024-03-06T07:33:06.433711+0000","last_active":"2024-03-06T07:33:07.674913+0000","last_peered":"2024-03-06T07:33:07.674913+0000","last_clean":"2024-03-06T07:33:07.674913+0000","last_became_active":"2024-03-06T07:33:06.431252+0000","last_became_peered":"2024-03-06T07:33:06.431252+0000","last_unstale":"2024-03-06T07:33:07.674913+0000","last_undegraded":"2024-03-06T07:33:07.674913+0000","last_fullsized":"2024-03-06T07:33:07.674913+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:29:22.735453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.37","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673967+0000","last_change":"2024-03-06T07:33:06.428429+0000","last_active":"2024-03-06T07:33:07.673967+0000","last_peered":"2024-03-06T07:33:07.673967+0000","last_clean":"2024-03-06T07:33:07.673967+0000","last_became_active":"2024-03-06T07:33:06.428158+0000","last_became_peered":"2024-03-06T07:33:06.428158+0000","last_unstale":"2024-03-06T07:33:07.673967+0000","last_undegraded":"2024-03-06T07:33:07.673967+0000","last_fullsized":"2024-03-06T07:33:07.673967+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:37:11.242716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.36","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673793+0000","last_change":"2024-03-06T07:33:06.425287+0000","last_active":"2024-03-06T07:33:07.673793+0000","last_peered":"2024-03-06T07:33:07.673793+0000","last_clean":"2024-03-06T07:33:07.673793+0000","last_became_active":"2024-03-06T07:33:06.423800+0000","last_became_peered":"2024-03-06T07:33:06.423800+0000","last_unstale":"2024-03-06T07:33:07.673793+0000","last_undegraded":"2024-03-06T07:33:07.673793+0000","last_fullsized":"2024-03-06T07:33:07.673793+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:40:03.458821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.35","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.422323+0000","last_change":"2024-03-06T07:33:06.425545+0000","last_active":"2024-03-06T07:33:07.422323+0000","last_peered":"2024-03-06T07:33:07.422323+0000","last_clean":"2024-03-06T07:33:07.422323+0000","last_became_active":"2024-03-06T07:33:06.425218+0000","last_became_peered":"2024-03-06T07:33:06.425218+0000","last_unstale":"2024-03-06T07:33:07.422323+0000","last_undegraded":"2024-03-06T07:33:07.422323+0000","last_fullsized":"2024-03-06T07:33:07.422323+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:39:26.021667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.34","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673341+0000","last_change":"2024-03-06T07:33:06.433026+0000","last_active":"2024-03-06T07:33:07.673341+0000","last_peered":"2024-03-06T07:33:07.673341+0000","last_clean":"2024-03-06T07:33:07.673341+0000","last_became_active":"2024-03-06T07:33:06.430316+0000","last_became_peered":"2024-03-06T07:33:06.430316+0000","last_unstale":"2024-03-06T07:33:07.673341+0000","last_undegraded":"2024-03-06T07:33:07.673341+0000","last_fullsized":"2024-03-06T07:33:07.673341+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:01:29.439179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.33","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674608+0000","last_change":"2024-03-06T07:33:06.436421+0000","last_active":"2024-03-06T07:33:07.674608+0000","last_peered":"2024-03-06T07:33:07.674608+0000","last_clean":"2024-03-06T07:33:07.674608+0000","last_became_active":"2024-03-06T07:33:06.435627+0000","last_became_peered":"2024-03-06T07:33:06.435627+0000","last_unstale":"2024-03-06T07:33:07.674608+0000","last_undegraded":"2024-03-06T07:33:07.674608+0000","last_fullsized":"2024-03-06T07:33:07.674608+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:31:22.950437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.32","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674470+0000","last_change":"2024-03-06T07:33:06.431005+0000","last_active":"2024-03-06T07:33:07.674470+0000","last_peered":"2024-03-06T07:33:07.674470+0000","last_clean":"2024-03-06T07:33:07.674470+0000","last_became_active":"2024-03-06T07:33:06.429905+0000","last_became_peered":"2024-03-06T07:33:06.429905+0000","last_unstale":"2024-03-06T07:33:07.674470+0000","last_undegraded":"2024-03-06T07:33:07.674470+0000","last_fullsized":"2024-03-06T07:33:07.674470+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:46:19.780125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.31","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.671427+0000","last_change":"2024-03-06T07:33:06.425625+0000","last_active":"2024-03-06T07:33:07.671427+0000","last_peered":"2024-03-06T07:33:07.671427+0000","last_clean":"2024-03-06T07:33:07.671427+0000","last_became_active":"2024-03-06T07:33:06.425297+0000","last_became_peered":"2024-03-06T07:33:06.425297+0000","last_unstale":"2024-03-06T07:33:07.671427+0000","last_undegraded":"2024-03-06T07:33:07.671427+0000","last_fullsized":"2024-03-06T07:33:07.671427+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:17.511905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.30","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675350+0000","last_change":"2024-03-06T07:33:06.440432+0000","last_active":"2024-03-06T07:33:07.675350+0000","last_peered":"2024-03-06T07:33:07.675350+0000","last_clean":"2024-03-06T07:33:07.675350+0000","last_became_active":"2024-03-06T07:33:06.440148+0000","last_became_peered":"2024-03-06T07:33:06.440148+0000","last_unstale":"2024-03-06T07:33:07.675350+0000","last_undegraded":"2024-03-06T07:33:07.675350+0000","last_fullsized":"2024-03-06T07:33:07.675350+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:28:22.722869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.2f","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.422473+0000","last_change":"2024-03-06T07:33:06.431823+0000","last_active":"2024-03-06T07:33:07.422473+0000","last_peered":"2024-03-06T07:33:07.422473+0000","last_clean":"2024-03-06T07:33:07.422473+0000","last_became_active":"2024-03-06T07:33:06.430942+0000","last_became_peered":"2024-03-06T07:33:06.430942+0000","last_unstale":"2024-03-06T07:33:07.422473+0000","last_undegraded":"2024-03-06T07:33:07.422473+0000","last_fullsized":"2024-03-06T07:33:07.422473+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:52:21.710679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2e","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.439644+0000","last_change":"2024-03-06T07:33:06.430196+0000","last_active":"2024-03-06T07:33:07.439644+0000","last_peered":"2024-03-06T07:33:07.439644+0000","last_clean":"2024-03-06T07:33:07.439644+0000","last_became_active":"2024-03-06T07:33:06.429809+0000","last_became_peered":"2024-03-06T07:33:06.429809+0000","last_unstale":"2024-03-06T07:33:07.439644+0000","last_undegraded":"2024-03-06T07:33:07.439644+0000","last_fullsized":"2024-03-06T07:33:07.439644+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:06:22.674852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.2d","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674828+0000","last_change":"2024-03-06T07:33:06.430749+0000","last_active":"2024-03-06T07:33:07.674828+0000","last_peered":"2024-03-06T07:33:07.674828+0000","last_clean":"2024-03-06T07:33:07.674828+0000","last_became_active":"2024-03-06T07:33:06.430458+0000","last_became_peered":"2024-03-06T07:33:06.430458+0000","last_unstale":"2024-03-06T07:33:07.674828+0000","last_undegraded":"2024-03-06T07:33:07.674828+0000","last_fullsized":"2024-03-06T07:33:07.674828+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:39:26.021667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.2c","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.422357+0000","last_change":"2024-03-06T07:33:06.420718+0000","last_active":"2024-03-06T07:33:07.422357+0000","last_peered":"2024-03-06T07:33:07.422357+0000","last_clean":"2024-03-06T07:33:07.422357+0000","last_became_active":"2024-03-06T07:33:06.420366+0000","last_became_peered":"2024-03-06T07:33:06.420366+0000","last_unstale":"2024-03-06T07:33:07.422357+0000","last_undegraded":"2024-03-06T07:33:07.422357+0000","last_fullsized":"2024-03-06T07:33:07.422357+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:29:22.735453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2b","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.422703+0000","last_change":"2024-03-06T07:33:06.432500+0000","last_active":"2024-03-06T07:33:07.422703+0000","last_peered":"2024-03-06T07:33:07.422703+0000","last_clean":"2024-03-06T07:33:07.422703+0000","last_became_active":"2024-03-06T07:33:06.432205+0000","last_became_peered":"2024-03-06T07:33:06.432205+0000","last_unstale":"2024-03-06T07:33:07.422703+0000","last_undegraded":"2024-03-06T07:33:07.422703+0000","last_fullsized":"2024-03-06T07:33:07.422703+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:29:54.065514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2a","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673374+0000","last_change":"2024-03-06T07:33:06.430392+0000","last_active":"2024-03-06T07:33:07.673374+0000","last_peered":"2024-03-06T07:33:07.673374+0000","last_clean":"2024-03-06T07:33:07.673374+0000","last_became_active":"2024-03-06T07:33:06.429903+0000","last_became_peered":"2024-03-06T07:33:06.429903+0000","last_unstale":"2024-03-06T07:33:07.673374+0000","last_undegraded":"2024-03-06T07:33:07.673374+0000","last_fullsized":"2024-03-06T07:33:07.673374+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:48:17.105334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.29","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.421166+0000","last_change":"2024-03-06T07:33:06.422996+0000","last_active":"2024-03-06T07:33:07.421166+0000","last_peered":"2024-03-06T07:33:07.421166+0000","last_clean":"2024-03-06T07:33:07.421166+0000","last_became_active":"2024-03-06T07:33:06.422676+0000","last_became_peered":"2024-03-06T07:33:06.422676+0000","last_unstale":"2024-03-06T07:33:07.421166+0000","last_undegraded":"2024-03-06T07:33:07.421166+0000","last_fullsized":"2024-03-06T07:33:07.421166+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:08.137274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.28","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.671697+0000","last_change":"2024-03-06T07:33:06.438303+0000","last_active":"2024-03-06T07:33:07.671697+0000","last_peered":"2024-03-06T07:33:07.671697+0000","last_clean":"2024-03-06T07:33:07.671697+0000","last_became_active":"2024-03-06T07:33:06.437709+0000","last_became_peered":"2024-03-06T07:33:06.437709+0000","last_unstale":"2024-03-06T07:33:07.671697+0000","last_undegraded":"2024-03-06T07:33:07.671697+0000","last_fullsized":"2024-03-06T07:33:07.671697+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:24:16.841568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.27","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.422654+0000","last_change":"2024-03-06T07:33:06.431165+0000","last_active":"2024-03-06T07:33:07.422654+0000","last_peered":"2024-03-06T07:33:07.422654+0000","last_clean":"2024-03-06T07:33:07.422654+0000","last_became_active":"2024-03-06T07:33:06.430897+0000","last_became_peered":"2024-03-06T07:33:06.430897+0000","last_unstale":"2024-03-06T07:33:07.422654+0000","last_undegraded":"2024-03-06T07:33:07.422654+0000","last_fullsized":"2024-03-06T07:33:07.422654+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:30:13.145290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.26","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673960+0000","last_change":"2024-03-06T07:33:06.431011+0000","last_active":"2024-03-06T07:33:07.673960+0000","last_peered":"2024-03-06T07:33:07.673960+0000","last_clean":"2024-03-06T07:33:07.673960+0000","last_became_active":"2024-03-06T07:33:06.430743+0000","last_became_peered":"2024-03-06T07:33:06.430743+0000","last_unstale":"2024-03-06T07:33:07.673960+0000","last_undegraded":"2024-03-06T07:33:07.673960+0000","last_fullsized":"2024-03-06T07:33:07.673960+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:39:40.225627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.25","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672568+0000","last_change":"2024-03-06T07:33:06.455697+0000","last_active":"2024-03-06T07:33:07.672568+0000","last_peered":"2024-03-06T07:33:07.672568+0000","last_clean":"2024-03-06T07:33:07.672568+0000","last_became_active":"2024-03-06T07:33:06.454774+0000","last_became_peered":"2024-03-06T07:33:06.454774+0000","last_unstale":"2024-03-06T07:33:07.672568+0000","last_undegraded":"2024-03-06T07:33:07.672568+0000","last_fullsized":"2024-03-06T07:33:07.672568+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:25:06.948351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.24","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673322+0000","last_change":"2024-03-06T07:33:06.452147+0000","last_active":"2024-03-06T07:33:07.673322+0000","last_peered":"2024-03-06T07:33:07.673322+0000","last_clean":"2024-03-06T07:33:07.673322+0000","last_became_active":"2024-03-06T07:33:06.451779+0000","last_became_peered":"2024-03-06T07:33:06.451779+0000","last_unstale":"2024-03-06T07:33:07.673322+0000","last_undegraded":"2024-03-06T07:33:07.673322+0000","last_fullsized":"2024-03-06T07:33:07.673322+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:11:59.253806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.23","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.421844+0000","last_change":"2024-03-06T07:33:06.429405+0000","last_active":"2024-03-06T07:33:07.421844+0000","last_peered":"2024-03-06T07:33:07.421844+0000","last_clean":"2024-03-06T07:33:07.421844+0000","last_became_active":"2024-03-06T07:33:06.428592+0000","last_became_peered":"2024-03-06T07:33:06.428592+0000","last_unstale":"2024-03-06T07:33:07.421844+0000","last_undegraded":"2024-03-06T07:33:07.421844+0000","last_fullsized":"2024-03-06T07:33:07.421844+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:46:19.780125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.22","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.444945+0000","last_change":"2024-03-06T07:33:06.432095+0000","last_active":"2024-03-06T07:33:07.444945+0000","last_peered":"2024-03-06T07:33:07.444945+0000","last_clean":"2024-03-06T07:33:07.444945+0000","last_became_active":"2024-03-06T07:33:06.431810+0000","last_became_peered":"2024-03-06T07:33:06.431810+0000","last_unstale":"2024-03-06T07:33:07.444945+0000","last_undegraded":"2024-03-06T07:33:07.444945+0000","last_fullsized":"2024-03-06T07:33:07.444945+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:56:42.604726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.21","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672957+0000","last_change":"2024-03-06T07:33:06.435893+0000","last_active":"2024-03-06T07:33:07.672957+0000","last_peered":"2024-03-06T07:33:07.672957+0000","last_clean":"2024-03-06T07:33:07.672957+0000","last_became_active":"2024-03-06T07:33:06.435648+0000","last_became_peered":"2024-03-06T07:33:06.435648+0000","last_unstale":"2024-03-06T07:33:07.672957+0000","last_undegraded":"2024-03-06T07:33:07.672957+0000","last_fullsized":"2024-03-06T07:33:07.672957+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:05:06.543544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.20","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.671296+0000","last_change":"2024-03-06T07:33:06.428011+0000","last_active":"2024-03-06T07:33:07.671296+0000","last_peered":"2024-03-06T07:33:07.671296+0000","last_clean":"2024-03-06T07:33:07.671296+0000","last_became_active":"2024-03-06T07:33:06.427731+0000","last_became_peered":"2024-03-06T07:33:06.427731+0000","last_unstale":"2024-03-06T07:33:07.671296+0000","last_undegraded":"2024-03-06T07:33:07.671296+0000","last_fullsized":"2024-03-06T07:33:07.671296+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:09:23.660859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.1e","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672690+0000","last_change":"2024-03-06T07:33:06.426868+0000","last_active":"2024-03-06T07:33:07.672690+0000","last_peered":"2024-03-06T07:33:07.672690+0000","last_clean":"2024-03-06T07:33:07.672690+0000","last_became_active":"2024-03-06T07:33:06.426597+0000","last_became_peered":"2024-03-06T07:33:06.426597+0000","last_unstale":"2024-03-06T07:33:07.672690+0000","last_undegraded":"2024-03-06T07:33:07.672690+0000","last_fullsized":"2024-03-06T07:33:07.672690+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:24:56.178911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.1f","version":"109'4","reported_seq":18,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.445126+0000","last_change":"2024-03-06T07:33:05.400861+0000","last_active":"2024-03-06T07:33:07.445126+0000","last_peered":"2024-03-06T07:33:07.445126+0000","last_clean":"2024-03-06T07:33:07.445126+0000","last_became_active":"2024-03-06T07:33:05.400526+0000","last_became_peered":"2024-03-06T07:33:05.400526+0000","last_unstale":"2024-03-06T07:33:07.445126+0000","last_undegraded":"2024-03-06T07:33:07.445126+0000","last_fullsized":"2024-03-06T07:33:07.445126+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:16:21.672444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1f","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.440121+0000","last_change":"2024-03-06T07:33:06.436747+0000","last_active":"2024-03-06T07:33:07.440121+0000","last_peered":"2024-03-06T07:33:07.440121+0000","last_clean":"2024-03-06T07:33:07.440121+0000","last_became_active":"2024-03-06T07:33:06.436349+0000","last_became_peered":"2024-03-06T07:33:06.436349+0000","last_unstale":"2024-03-06T07:33:07.440121+0000","last_undegraded":"2024-03-06T07:33:07.440121+0000","last_fullsized":"2024-03-06T07:33:07.440121+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:15:40.369850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.1e","version":"109'1","reported_seq":15,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674298+0000","last_change":"2024-03-06T07:33:05.403114+0000","last_active":"2024-03-06T07:33:07.674298+0000","last_peered":"2024-03-06T07:33:07.674298+0000","last_clean":"2024-03-06T07:33:07.674298+0000","last_became_active":"2024-03-06T07:33:05.402730+0000","last_became_peered":"2024-03-06T07:33:05.402730+0000","last_unstale":"2024-03-06T07:33:07.674298+0000","last_undegraded":"2024-03-06T07:33:07.674298+0000","last_fullsized":"2024-03-06T07:33:07.674298+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:25:05.931783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.1c","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672726+0000","last_change":"2024-03-06T07:33:06.432773+0000","last_active":"2024-03-06T07:33:07.672726+0000","last_peered":"2024-03-06T07:33:07.672726+0000","last_clean":"2024-03-06T07:33:07.672726+0000","last_became_active":"2024-03-06T07:33:06.432481+0000","last_became_peered":"2024-03-06T07:33:06.432481+0000","last_unstale":"2024-03-06T07:33:07.672726+0000","last_undegraded":"2024-03-06T07:33:07.672726+0000","last_fullsized":"2024-03-06T07:33:07.672726+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:23:31.488780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.1d","version":"109'2","reported_seq":16,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.420316+0000","last_change":"2024-03-06T07:33:05.403006+0000","last_active":"2024-03-06T07:33:07.420316+0000","last_peered":"2024-03-06T07:33:07.420316+0000","last_clean":"2024-03-06T07:33:07.420316+0000","last_became_active":"2024-03-06T07:33:05.400500+0000","last_became_peered":"2024-03-06T07:33:05.400500+0000","last_unstale":"2024-03-06T07:33:07.420316+0000","last_undegraded":"2024-03-06T07:33:07.420316+0000","last_fullsized":"2024-03-06T07:33:07.420316+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:55:31.450463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1d","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674957+0000","last_change":"2024-03-06T07:33:06.433029+0000","last_active":"2024-03-06T07:33:07.674957+0000","last_peered":"2024-03-06T07:33:07.674957+0000","last_clean":"2024-03-06T07:33:07.674957+0000","last_became_active":"2024-03-06T07:33:06.432657+0000","last_became_peered":"2024-03-06T07:33:06.432657+0000","last_unstale":"2024-03-06T07:33:07.674957+0000","last_undegraded":"2024-03-06T07:33:07.674957+0000","last_fullsized":"2024-03-06T07:33:07.674957+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:52:21.710679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.1c","version":"109'2","reported_seq":16,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.420208+0000","last_change":"2024-03-06T07:33:05.400421+0000","last_active":"2024-03-06T07:33:07.420208+0000","last_peered":"2024-03-06T07:33:07.420208+0000","last_clean":"2024-03-06T07:33:07.420208+0000","last_became_active":"2024-03-06T07:33:05.400086+0000","last_became_peered":"2024-03-06T07:33:05.400086+0000","last_unstale":"2024-03-06T07:33:07.420208+0000","last_undegraded":"2024-03-06T07:33:07.420208+0000","last_fullsized":"2024-03-06T07:33:07.420208+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:47:21.953976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1a","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673128+0000","last_change":"2024-03-06T07:33:06.441132+0000","last_active":"2024-03-06T07:33:07.673128+0000","last_peered":"2024-03-06T07:33:07.673128+0000","last_clean":"2024-03-06T07:33:07.673128+0000","last_became_active":"2024-03-06T07:33:06.440887+0000","last_became_peered":"2024-03-06T07:33:06.440887+0000","last_unstale":"2024-03-06T07:33:07.673128+0000","last_undegraded":"2024-03-06T07:33:07.673128+0000","last_fullsized":"2024-03-06T07:33:07.673128+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:03:11.850137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.1b","version":"109'1","reported_seq":15,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.444533+0000","last_change":"2024-03-06T07:33:05.403032+0000","last_active":"2024-03-06T07:33:07.444533+0000","last_peered":"2024-03-06T07:33:07.444533+0000","last_clean":"2024-03-06T07:33:07.444533+0000","last_became_active":"2024-03-06T07:33:05.401844+0000","last_became_peered":"2024-03-06T07:33:05.401844+0000","last_unstale":"2024-03-06T07:33:07.444533+0000","last_undegraded":"2024-03-06T07:33:07.444533+0000","last_fullsized":"2024-03-06T07:33:07.444533+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:41:04.838825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1b","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.421972+0000","last_change":"2024-03-06T07:33:06.428868+0000","last_active":"2024-03-06T07:33:07.421972+0000","last_peered":"2024-03-06T07:33:07.421972+0000","last_clean":"2024-03-06T07:33:07.421972+0000","last_became_active":"2024-03-06T07:33:06.428172+0000","last_became_peered":"2024-03-06T07:33:06.428172+0000","last_unstale":"2024-03-06T07:33:07.421972+0000","last_undegraded":"2024-03-06T07:33:07.421972+0000","last_fullsized":"2024-03-06T07:33:07.421972+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:11:44.902634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.1a","version":"109'1","reported_seq":15,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673425+0000","last_change":"2024-03-06T07:33:05.402181+0000","last_active":"2024-03-06T07:33:07.673425+0000","last_peered":"2024-03-06T07:33:07.673425+0000","last_clean":"2024-03-06T07:33:07.673425+0000","last_became_active":"2024-03-06T07:33:05.401866+0000","last_became_peered":"2024-03-06T07:33:05.401866+0000","last_unstale":"2024-03-06T07:33:07.673425+0000","last_undegraded":"2024-03-06T07:33:07.673425+0000","last_fullsized":"2024-03-06T07:33:07.673425+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:07.120707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.18","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675509+0000","last_change":"2024-03-06T07:33:06.440398+0000","last_active":"2024-03-06T07:33:07.675509+0000","last_peered":"2024-03-06T07:33:07.675509+0000","last_clean":"2024-03-06T07:33:07.675509+0000","last_became_active":"2024-03-06T07:33:06.438824+0000","last_became_peered":"2024-03-06T07:33:06.438824+0000","last_unstale":"2024-03-06T07:33:07.675509+0000","last_undegraded":"2024-03-06T07:33:07.675509+0000","last_fullsized":"2024-03-06T07:33:07.675509+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:59:48.719092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.19","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.421144+0000","last_change":"2024-03-06T07:33:05.391222+0000","last_active":"2024-03-06T07:33:07.421144+0000","last_peered":"2024-03-06T07:33:07.421144+0000","last_clean":"2024-03-06T07:33:07.421144+0000","last_became_active":"2024-03-06T07:33:05.390818+0000","last_became_peered":"2024-03-06T07:33:05.390818+0000","last_unstale":"2024-03-06T07:33:07.421144+0000","last_undegraded":"2024-03-06T07:33:07.421144+0000","last_fullsized":"2024-03-06T07:33:07.421144+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:24:55.162344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.19","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.444729+0000","last_change":"2024-03-06T07:33:06.429132+0000","last_active":"2024-03-06T07:33:07.444729+0000","last_peered":"2024-03-06T07:33:07.444729+0000","last_clean":"2024-03-06T07:33:07.444729+0000","last_became_active":"2024-03-06T07:33:06.424533+0000","last_became_peered":"2024-03-06T07:33:06.424533+0000","last_unstale":"2024-03-06T07:33:07.444729+0000","last_undegraded":"2024-03-06T07:33:07.444729+0000","last_fullsized":"2024-03-06T07:33:07.444729+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:48:29.619012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.18","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675572+0000","last_change":"2024-03-06T07:33:05.391817+0000","last_active":"2024-03-06T07:33:07.675572+0000","last_peered":"2024-03-06T07:33:07.675572+0000","last_clean":"2024-03-06T07:33:07.675572+0000","last_became_active":"2024-03-06T07:33:05.390685+0000","last_became_peered":"2024-03-06T07:33:05.390685+0000","last_unstale":"2024-03-06T07:33:07.675572+0000","last_undegraded":"2024-03-06T07:33:07.675572+0000","last_fullsized":"2024-03-06T07:33:07.675572+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:30:12.128723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.16","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.445289+0000","last_change":"2024-03-06T07:33:06.426903+0000","last_active":"2024-03-06T07:33:07.445289+0000","last_peered":"2024-03-06T07:33:07.445289+0000","last_clean":"2024-03-06T07:33:07.445289+0000","last_became_active":"2024-03-06T07:33:06.426069+0000","last_became_peered":"2024-03-06T07:33:06.426069+0000","last_unstale":"2024-03-06T07:33:07.445289+0000","last_undegraded":"2024-03-06T07:33:07.445289+0000","last_fullsized":"2024-03-06T07:33:07.445289+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:17:16.111574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.17","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.420042+0000","last_change":"2024-03-06T07:33:05.408348+0000","last_active":"2024-03-06T07:33:07.420042+0000","last_peered":"2024-03-06T07:33:07.420042+0000","last_clean":"2024-03-06T07:33:07.420042+0000","last_became_active":"2024-03-06T07:33:05.407406+0000","last_became_peered":"2024-03-06T07:33:05.407406+0000","last_unstale":"2024-03-06T07:33:07.420042+0000","last_undegraded":"2024-03-06T07:33:07.420042+0000","last_fullsized":"2024-03-06T07:33:07.420042+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:43:29.803029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.17","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.440537+0000","last_change":"2024-03-06T07:33:06.456991+0000","last_active":"2024-03-06T07:33:07.440537+0000","last_peered":"2024-03-06T07:33:07.440537+0000","last_clean":"2024-03-06T07:33:07.440537+0000","last_became_active":"2024-03-06T07:33:06.456634+0000","last_became_peered":"2024-03-06T07:33:06.456634+0000","last_unstale":"2024-03-06T07:33:07.440537+0000","last_undegraded":"2024-03-06T07:33:07.440537+0000","last_fullsized":"2024-03-06T07:33:07.440537+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:57:55.567441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.16","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673028+0000","last_change":"2024-03-06T07:33:05.401010+0000","last_active":"2024-03-06T07:33:07.673028+0000","last_peered":"2024-03-06T07:33:07.673028+0000","last_clean":"2024-03-06T07:33:07.673028+0000","last_became_active":"2024-03-06T07:33:05.400730+0000","last_became_peered":"2024-03-06T07:33:05.400730+0000","last_unstale":"2024-03-06T07:33:07.673028+0000","last_undegraded":"2024-03-06T07:33:07.673028+0000","last_fullsized":"2024-03-06T07:33:07.673028+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:33.820269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.14","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672898+0000","last_change":"2024-03-06T07:33:06.423132+0000","last_active":"2024-03-06T07:33:07.672898+0000","last_peered":"2024-03-06T07:33:07.672898+0000","last_clean":"2024-03-06T07:33:07.672898+0000","last_became_active":"2024-03-06T07:33:06.422850+0000","last_became_peered":"2024-03-06T07:33:06.422850+0000","last_unstale":"2024-03-06T07:33:07.672898+0000","last_undegraded":"2024-03-06T07:33:07.672898+0000","last_fullsized":"2024-03-06T07:33:07.672898+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:45:28.992078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.15","version":"109'3","reported_seq":17,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673000+0000","last_change":"2024-03-06T07:33:05.389220+0000","last_active":"2024-03-06T07:33:07.673000+0000","last_peered":"2024-03-06T07:33:07.673000+0000","last_clean":"2024-03-06T07:33:07.673000+0000","last_became_active":"2024-03-06T07:33:05.388882+0000","last_became_peered":"2024-03-06T07:33:05.388882+0000","last_unstale":"2024-03-06T07:33:07.673000+0000","last_undegraded":"2024-03-06T07:33:07.673000+0000","last_fullsized":"2024-03-06T07:33:07.673000+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:15:18.571358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.15","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.445446+0000","last_change":"2024-03-06T07:33:06.434723+0000","last_active":"2024-03-06T07:33:07.445446+0000","last_peered":"2024-03-06T07:33:07.445446+0000","last_clean":"2024-03-06T07:33:07.445446+0000","last_became_active":"2024-03-06T07:33:06.434162+0000","last_became_peered":"2024-03-06T07:33:06.434162+0000","last_unstale":"2024-03-06T07:33:07.445446+0000","last_undegraded":"2024-03-06T07:33:07.445446+0000","last_fullsized":"2024-03-06T07:33:07.445446+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:38:34.607770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.14","version":"109'6","reported_seq":20,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673555+0000","last_change":"2024-03-06T07:33:05.399779+0000","last_active":"2024-03-06T07:33:07.673555+0000","last_peered":"2024-03-06T07:33:07.673555+0000","last_clean":"2024-03-06T07:33:07.673555+0000","last_became_active":"2024-03-06T07:33:05.391894+0000","last_became_peered":"2024-03-06T07:33:05.391894+0000","last_unstale":"2024-03-06T07:33:07.673555+0000","last_undegraded":"2024-03-06T07:33:07.673555+0000","last_fullsized":"2024-03-06T07:33:07.673555+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:45:27.975511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.12","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673351+0000","last_change":"2024-03-06T07:33:06.431009+0000","last_active":"2024-03-06T07:33:07.673351+0000","last_peered":"2024-03-06T07:33:07.673351+0000","last_clean":"2024-03-06T07:33:07.673351+0000","last_became_active":"2024-03-06T07:33:06.430747+0000","last_became_peered":"2024-03-06T07:33:06.430747+0000","last_unstale":"2024-03-06T07:33:07.673351+0000","last_undegraded":"2024-03-06T07:33:07.673351+0000","last_fullsized":"2024-03-06T07:33:07.673351+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:37:19.809594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.13","version":"109'2","reported_seq":16,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672388+0000","last_change":"2024-03-06T07:33:05.402517+0000","last_active":"2024-03-06T07:33:07.672388+0000","last_peered":"2024-03-06T07:33:07.672388+0000","last_clean":"2024-03-06T07:33:07.672388+0000","last_became_active":"2024-03-06T07:33:05.401198+0000","last_became_peered":"2024-03-06T07:33:05.401198+0000","last_unstale":"2024-03-06T07:33:07.672388+0000","last_undegraded":"2024-03-06T07:33:07.672388+0000","last_fullsized":"2024-03-06T07:33:07.672388+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:48:41.110207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.13","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673176+0000","last_change":"2024-03-06T07:33:06.438059+0000","last_active":"2024-03-06T07:33:07.673176+0000","last_peered":"2024-03-06T07:33:07.673176+0000","last_clean":"2024-03-06T07:33:07.673176+0000","last_became_active":"2024-03-06T07:33:06.437796+0000","last_became_peered":"2024-03-06T07:33:06.437796+0000","last_unstale":"2024-03-06T07:33:07.673176+0000","last_undegraded":"2024-03-06T07:33:07.673176+0000","last_fullsized":"2024-03-06T07:33:07.673176+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:57:50.977969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.12","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.439762+0000","last_change":"2024-03-06T07:33:05.410456+0000","last_active":"2024-03-06T07:33:07.439762+0000","last_peered":"2024-03-06T07:33:07.439762+0000","last_clean":"2024-03-06T07:33:07.439762+0000","last_became_active":"2024-03-06T07:33:05.410136+0000","last_became_peered":"2024-03-06T07:33:05.410136+0000","last_unstale":"2024-03-06T07:33:07.439762+0000","last_undegraded":"2024-03-06T07:33:07.439762+0000","last_fullsized":"2024-03-06T07:33:07.439762+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:11:40.692462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.11","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672160+0000","last_change":"2024-03-06T07:33:06.419583+0000","last_active":"2024-03-06T07:33:07.672160+0000","last_peered":"2024-03-06T07:33:07.672160+0000","last_clean":"2024-03-06T07:33:07.672160+0000","last_became_active":"2024-03-06T07:33:06.419212+0000","last_became_peered":"2024-03-06T07:33:06.419212+0000","last_unstale":"2024-03-06T07:33:07.672160+0000","last_undegraded":"2024-03-06T07:33:07.672160+0000","last_fullsized":"2024-03-06T07:33:07.672160+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:15:19.587926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.10","version":"109'8","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.419418+0000","last_change":"2024-03-06T07:33:05.410638+0000","last_active":"2024-03-06T07:33:07.419418+0000","last_peered":"2024-03-06T07:33:07.419418+0000","last_clean":"2024-03-06T07:33:07.419418+0000","last_became_active":"2024-03-06T07:33:05.410285+0000","last_became_peered":"2024-03-06T07:33:05.410285+0000","last_unstale":"2024-03-06T07:33:07.419418+0000","last_undegraded":"2024-03-06T07:33:07.419418+0000","last_fullsized":"2024-03-06T07:33:07.419418+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:03:43.011834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.e","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.420701+0000","last_change":"2024-03-06T07:33:06.451109+0000","last_active":"2024-03-06T07:33:07.420701+0000","last_peered":"2024-03-06T07:33:07.420701+0000","last_clean":"2024-03-06T07:33:07.420701+0000","last_became_active":"2024-03-06T07:33:06.449858+0000","last_became_peered":"2024-03-06T07:33:06.449858+0000","last_unstale":"2024-03-06T07:33:07.420701+0000","last_undegraded":"2024-03-06T07:33:07.420701+0000","last_fullsized":"2024-03-06T07:33:07.420701+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:40:39.397116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"16.f","version":"109'1","reported_seq":15,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674058+0000","last_change":"2024-03-06T07:33:05.411487+0000","last_active":"2024-03-06T07:33:07.674058+0000","last_peered":"2024-03-06T07:33:07.674058+0000","last_clean":"2024-03-06T07:33:07.674058+0000","last_became_active":"2024-03-06T07:33:05.411238+0000","last_became_peered":"2024-03-06T07:33:05.411238+0000","last_unstale":"2024-03-06T07:33:07.674058+0000","last_undegraded":"2024-03-06T07:33:07.674058+0000","last_fullsized":"2024-03-06T07:33:07.674058+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:41:26.086033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.f","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672807+0000","last_change":"2024-03-06T07:33:06.455014+0000","last_active":"2024-03-06T07:33:07.672807+0000","last_peered":"2024-03-06T07:33:07.672807+0000","last_clean":"2024-03-06T07:33:07.672807+0000","last_became_active":"2024-03-06T07:33:06.454631+0000","last_became_peered":"2024-03-06T07:33:06.454631+0000","last_unstale":"2024-03-06T07:33:07.672807+0000","last_undegraded":"2024-03-06T07:33:07.672807+0000","last_fullsized":"2024-03-06T07:33:07.672807+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:02:33.210189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.e","version":"109'2","reported_seq":16,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.420328+0000","last_change":"2024-03-06T07:33:05.406801+0000","last_active":"2024-03-06T07:33:07.420328+0000","last_peered":"2024-03-06T07:33:07.420328+0000","last_clean":"2024-03-06T07:33:07.420328+0000","last_became_active":"2024-03-06T07:33:05.406463+0000","last_became_peered":"2024-03-06T07:33:05.406463+0000","last_unstale":"2024-03-06T07:33:07.420328+0000","last_undegraded":"2024-03-06T07:33:07.420328+0000","last_fullsized":"2024-03-06T07:33:07.420328+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:25.625387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.c","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674611+0000","last_change":"2024-03-06T07:33:06.453922+0000","last_active":"2024-03-06T07:33:07.674611+0000","last_peered":"2024-03-06T07:33:07.674611+0000","last_clean":"2024-03-06T07:33:07.674611+0000","last_became_active":"2024-03-06T07:33:06.453657+0000","last_became_peered":"2024-03-06T07:33:06.453657+0000","last_unstale":"2024-03-06T07:33:07.674611+0000","last_undegraded":"2024-03-06T07:33:07.674611+0000","last_fullsized":"2024-03-06T07:33:07.674611+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:11:44.902634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.d","version":"109'1","reported_seq":15,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.440327+0000","last_change":"2024-03-06T07:33:05.401716+0000","last_active":"2024-03-06T07:33:07.440327+0000","last_peered":"2024-03-06T07:33:07.440327+0000","last_clean":"2024-03-06T07:33:07.440327+0000","last_became_active":"2024-03-06T07:33:05.400434+0000","last_became_peered":"2024-03-06T07:33:05.400434+0000","last_unstale":"2024-03-06T07:33:07.440327+0000","last_undegraded":"2024-03-06T07:33:07.440327+0000","last_fullsized":"2024-03-06T07:33:07.440327+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:02:34.041801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.d","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675079+0000","last_change":"2024-03-06T07:33:06.432848+0000","last_active":"2024-03-06T07:33:07.675079+0000","last_peered":"2024-03-06T07:33:07.675079+0000","last_clean":"2024-03-06T07:33:07.675079+0000","last_became_active":"2024-03-06T07:33:06.432294+0000","last_became_peered":"2024-03-06T07:33:06.432294+0000","last_unstale":"2024-03-06T07:33:07.675079+0000","last_undegraded":"2024-03-06T07:33:07.675079+0000","last_fullsized":"2024-03-06T07:33:07.675079+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:30:13.145290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.c","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.444613+0000","last_change":"2024-03-06T07:33:05.411091+0000","last_active":"2024-03-06T07:33:07.444613+0000","last_peered":"2024-03-06T07:33:07.444613+0000","last_clean":"2024-03-06T07:33:07.444613+0000","last_became_active":"2024-03-06T07:33:05.410689+0000","last_became_peered":"2024-03-06T07:33:05.410689+0000","last_unstale":"2024-03-06T07:33:07.444613+0000","last_undegraded":"2024-03-06T07:33:07.444613+0000","last_fullsized":"2024-03-06T07:33:07.444613+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:50:28.720928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.a","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674178+0000","last_change":"2024-03-06T07:33:06.431348+0000","last_active":"2024-03-06T07:33:07.674178+0000","last_peered":"2024-03-06T07:33:07.674178+0000","last_clean":"2024-03-06T07:33:07.674178+0000","last_became_active":"2024-03-06T07:33:06.428648+0000","last_became_peered":"2024-03-06T07:33:06.428648+0000","last_unstale":"2024-03-06T07:33:07.674178+0000","last_undegraded":"2024-03-06T07:33:07.674178+0000","last_fullsized":"2024-03-06T07:33:07.674178+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:56:32.420891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.b","version":"109'2","reported_seq":16,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672503+0000","last_change":"2024-03-06T07:33:05.407652+0000","last_active":"2024-03-06T07:33:07.672503+0000","last_peered":"2024-03-06T07:33:07.672503+0000","last_clean":"2024-03-06T07:33:07.672503+0000","last_became_active":"2024-03-06T07:33:05.407376+0000","last_became_peered":"2024-03-06T07:33:05.407376+0000","last_unstale":"2024-03-06T07:33:07.672503+0000","last_undegraded":"2024-03-06T07:33:07.672503+0000","last_fullsized":"2024-03-06T07:33:07.672503+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:09:52.458429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.b","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674154+0000","last_change":"2024-03-06T07:33:06.440035+0000","last_active":"2024-03-06T07:33:07.674154+0000","last_peered":"2024-03-06T07:33:07.674154+0000","last_clean":"2024-03-06T07:33:07.674154+0000","last_became_active":"2024-03-06T07:33:06.439262+0000","last_became_peered":"2024-03-06T07:33:06.439262+0000","last_unstale":"2024-03-06T07:33:07.674154+0000","last_undegraded":"2024-03-06T07:33:07.674154+0000","last_fullsized":"2024-03-06T07:33:07.674154+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:50:14.231658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.a","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.419399+0000","last_change":"2024-03-06T07:33:05.408109+0000","last_active":"2024-03-06T07:33:07.419399+0000","last_peered":"2024-03-06T07:33:07.419399+0000","last_clean":"2024-03-06T07:33:07.419399+0000","last_became_active":"2024-03-06T07:33:05.407788+0000","last_became_peered":"2024-03-06T07:33:05.407788+0000","last_unstale":"2024-03-06T07:33:07.419399+0000","last_undegraded":"2024-03-06T07:33:07.419399+0000","last_fullsized":"2024-03-06T07:33:07.419399+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:10:30.606686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.8","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674370+0000","last_change":"2024-03-06T07:33:06.431607+0000","last_active":"2024-03-06T07:33:07.674370+0000","last_peered":"2024-03-06T07:33:07.674370+0000","last_clean":"2024-03-06T07:33:07.674370+0000","last_became_active":"2024-03-06T07:33:06.430743+0000","last_became_peered":"2024-03-06T07:33:06.430743+0000","last_unstale":"2024-03-06T07:33:07.674370+0000","last_undegraded":"2024-03-06T07:33:07.674370+0000","last_fullsized":"2024-03-06T07:33:07.674370+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:56:08.137274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.9","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.671592+0000","last_change":"2024-03-06T07:33:05.399602+0000","last_active":"2024-03-06T07:33:07.671592+0000","last_peered":"2024-03-06T07:33:07.671592+0000","last_clean":"2024-03-06T07:33:07.671592+0000","last_became_active":"2024-03-06T07:33:05.399298+0000","last_became_peered":"2024-03-06T07:33:05.399298+0000","last_unstale":"2024-03-06T07:33:07.671592+0000","last_undegraded":"2024-03-06T07:33:07.671592+0000","last_fullsized":"2024-03-06T07:33:07.671592+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:22:51.704082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.9","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673779+0000","last_change":"2024-03-06T07:33:06.433399+0000","last_active":"2024-03-06T07:33:07.673779+0000","last_peered":"2024-03-06T07:33:07.673779+0000","last_clean":"2024-03-06T07:33:07.673779+0000","last_became_active":"2024-03-06T07:33:06.433120+0000","last_became_peered":"2024-03-06T07:33:06.433120+0000","last_unstale":"2024-03-06T07:33:07.673779+0000","last_undegraded":"2024-03-06T07:33:07.673779+0000","last_fullsized":"2024-03-06T07:33:07.673779+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:41:27.102600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.8","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675676+0000","last_change":"2024-03-06T07:33:05.410388+0000","last_active":"2024-03-06T07:33:07.675676+0000","last_peered":"2024-03-06T07:33:07.675676+0000","last_clean":"2024-03-06T07:33:07.675676+0000","last_became_active":"2024-03-06T07:33:05.410109+0000","last_became_peered":"2024-03-06T07:33:05.410109+0000","last_unstale":"2024-03-06T07:33:07.675676+0000","last_undegraded":"2024-03-06T07:33:07.675676+0000","last_fullsized":"2024-03-06T07:33:07.675676+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:29:53.048946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.6","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.439607+0000","last_change":"2024-03-06T07:33:06.457001+0000","last_active":"2024-03-06T07:33:07.439607+0000","last_peered":"2024-03-06T07:33:07.439607+0000","last_clean":"2024-03-06T07:33:07.439607+0000","last_became_active":"2024-03-06T07:33:06.456647+0000","last_became_peered":"2024-03-06T07:33:06.456647+0000","last_unstale":"2024-03-06T07:33:07.439607+0000","last_undegraded":"2024-03-06T07:33:07.439607+0000","last_fullsized":"2024-03-06T07:33:07.439607+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:50:59.047704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.7","version":"109'2","reported_seq":16,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673149+0000","last_change":"2024-03-06T07:33:05.398101+0000","last_active":"2024-03-06T07:33:07.673149+0000","last_peered":"2024-03-06T07:33:07.673149+0000","last_clean":"2024-03-06T07:33:07.673149+0000","last_became_active":"2024-03-06T07:33:05.397741+0000","last_became_peered":"2024-03-06T07:33:05.397741+0000","last_unstale":"2024-03-06T07:33:07.673149+0000","last_undegraded":"2024-03-06T07:33:07.673149+0000","last_fullsized":"2024-03-06T07:33:07.673149+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:56:31.404324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.7","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.674022+0000","last_change":"2024-03-06T07:33:06.434499+0000","last_active":"2024-03-06T07:33:07.674022+0000","last_peered":"2024-03-06T07:33:07.674022+0000","last_clean":"2024-03-06T07:33:07.674022+0000","last_became_active":"2024-03-06T07:33:06.434249+0000","last_became_peered":"2024-03-06T07:33:06.434249+0000","last_unstale":"2024-03-06T07:33:07.674022+0000","last_undegraded":"2024-03-06T07:33:07.674022+0000","last_fullsized":"2024-03-06T07:33:07.674022+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:17:34.492582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.6","version":"110'1","reported_seq":15,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.439945+0000","last_change":"2024-03-06T07:33:05.400042+0000","last_active":"2024-03-06T07:33:07.439945+0000","last_peered":"2024-03-06T07:33:07.439945+0000","last_clean":"2024-03-06T07:33:07.439945+0000","last_became_active":"2024-03-06T07:33:05.399581+0000","last_became_peered":"2024-03-06T07:33:05.399581+0000","last_unstale":"2024-03-06T07:33:07.439945+0000","last_undegraded":"2024-03-06T07:33:07.439945+0000","last_fullsized":"2024-03-06T07:33:07.439945+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:23:42.816989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.4","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673493+0000","last_change":"2024-03-06T07:33:06.436647+0000","last_active":"2024-03-06T07:33:07.673493+0000","last_peered":"2024-03-06T07:33:07.673493+0000","last_clean":"2024-03-06T07:33:07.673493+0000","last_became_active":"2024-03-06T07:33:06.430662+0000","last_became_peered":"2024-03-06T07:33:06.430662+0000","last_unstale":"2024-03-06T07:33:07.673493+0000","last_undegraded":"2024-03-06T07:33:07.673493+0000","last_fullsized":"2024-03-06T07:33:07.673493+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:01:41.237980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.5","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673360+0000","last_change":"2024-03-06T07:33:05.405665+0000","last_active":"2024-03-06T07:33:07.673360+0000","last_peered":"2024-03-06T07:33:07.673360+0000","last_clean":"2024-03-06T07:33:07.673360+0000","last_became_active":"2024-03-06T07:33:05.405387+0000","last_became_peered":"2024-03-06T07:33:05.405387+0000","last_unstale":"2024-03-06T07:33:07.673360+0000","last_undegraded":"2024-03-06T07:33:07.673360+0000","last_fullsized":"2024-03-06T07:33:07.673360+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:19:56.650304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.5","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.440033+0000","last_change":"2024-03-06T07:33:06.450287+0000","last_active":"2024-03-06T07:33:07.440033+0000","last_peered":"2024-03-06T07:33:07.440033+0000","last_clean":"2024-03-06T07:33:07.440033+0000","last_became_active":"2024-03-06T07:33:06.449969+0000","last_became_peered":"2024-03-06T07:33:06.449969+0000","last_unstale":"2024-03-06T07:33:07.440033+0000","last_undegraded":"2024-03-06T07:33:07.440033+0000","last_fullsized":"2024-03-06T07:33:07.440033+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:05:51.443961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.4","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672965+0000","last_change":"2024-03-06T07:33:05.401303+0000","last_active":"2024-03-06T07:33:07.672965+0000","last_peered":"2024-03-06T07:33:07.672965+0000","last_clean":"2024-03-06T07:33:07.672965+0000","last_became_active":"2024-03-06T07:33:05.399682+0000","last_became_peered":"2024-03-06T07:33:05.399682+0000","last_unstale":"2024-03-06T07:33:07.672965+0000","last_undegraded":"2024-03-06T07:33:07.672965+0000","last_fullsized":"2024-03-06T07:33:07.672965+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:46:18.763558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.2","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.421290+0000","last_change":"2024-03-06T07:33:06.427096+0000","last_active":"2024-03-06T07:33:07.421290+0000","last_peered":"2024-03-06T07:33:07.421290+0000","last_clean":"2024-03-06T07:33:07.421290+0000","last_became_active":"2024-03-06T07:33:06.420728+0000","last_became_peered":"2024-03-06T07:33:06.420728+0000","last_unstale":"2024-03-06T07:33:07.421290+0000","last_undegraded":"2024-03-06T07:33:07.421290+0000","last_fullsized":"2024-03-06T07:33:07.421290+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:54:34.836836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.3","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.444543+0000","last_change":"2024-03-06T07:33:05.403276+0000","last_active":"2024-03-06T07:33:07.444543+0000","last_peered":"2024-03-06T07:33:07.444543+0000","last_clean":"2024-03-06T07:33:07.444543+0000","last_became_active":"2024-03-06T07:33:05.402122+0000","last_became_peered":"2024-03-06T07:33:05.402122+0000","last_unstale":"2024-03-06T07:33:07.444543+0000","last_undegraded":"2024-03-06T07:33:07.444543+0000","last_fullsized":"2024-03-06T07:33:07.444543+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:41:27.977850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.3","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.422297+0000","last_change":"2024-03-06T07:33:06.415079+0000","last_active":"2024-03-06T07:33:07.422297+0000","last_peered":"2024-03-06T07:33:07.422297+0000","last_clean":"2024-03-06T07:33:07.422297+0000","last_became_active":"2024-03-06T07:33:06.414321+0000","last_became_peered":"2024-03-06T07:33:06.414321+0000","last_unstale":"2024-03-06T07:33:07.422297+0000","last_undegraded":"2024-03-06T07:33:07.422297+0000","last_fullsized":"2024-03-06T07:33:07.422297+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:31:22.950437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.2","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.421448+0000","last_change":"2024-03-06T07:33:05.399669+0000","last_active":"2024-03-06T07:33:07.421448+0000","last_peered":"2024-03-06T07:33:07.421448+0000","last_clean":"2024-03-06T07:33:07.421448+0000","last_became_active":"2024-03-06T07:33:05.399266+0000","last_became_peered":"2024-03-06T07:33:05.399266+0000","last_unstale":"2024-03-06T07:33:07.421448+0000","last_undegraded":"2024-03-06T07:33:07.421448+0000","last_fullsized":"2024-03-06T07:33:07.421448+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:25:05.931783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.0","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675069+0000","last_change":"2024-03-06T07:33:06.436078+0000","last_active":"2024-03-06T07:33:07.675069+0000","last_peered":"2024-03-06T07:33:07.675069+0000","last_clean":"2024-03-06T07:33:07.675069+0000","last_became_active":"2024-03-06T07:33:06.435602+0000","last_became_peered":"2024-03-06T07:33:06.435602+0000","last_unstale":"2024-03-06T07:33:07.675069+0000","last_undegraded":"2024-03-06T07:33:07.675069+0000","last_fullsized":"2024-03-06T07:33:07.675069+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:11:04.349061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.1","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.419447+0000","last_change":"2024-03-06T07:33:05.400646+0000","last_active":"2024-03-06T07:33:07.419447+0000","last_peered":"2024-03-06T07:33:07.419447+0000","last_clean":"2024-03-06T07:33:07.419447+0000","last_became_active":"2024-03-06T07:33:05.400167+0000","last_became_peered":"2024-03-06T07:33:05.400167+0000","last_unstale":"2024-03-06T07:33:07.419447+0000","last_undegraded":"2024-03-06T07:33:07.419447+0000","last_fullsized":"2024-03-06T07:33:07.419447+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:40:59.082170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.673128+0000","last_change":"2024-03-06T07:33:06.429672+0000","last_active":"2024-03-06T07:33:07.673128+0000","last_peered":"2024-03-06T07:33:07.673128+0000","last_clean":"2024-03-06T07:33:07.673128+0000","last_became_active":"2024-03-06T07:33:06.429394+0000","last_became_peered":"2024-03-06T07:33:06.429394+0000","last_unstale":"2024-03-06T07:33:07.673128+0000","last_undegraded":"2024-03-06T07:33:07.673128+0000","last_fullsized":"2024-03-06T07:33:07.673128+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:55:20.577728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.0","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675777+0000","last_change":"2024-03-06T07:33:05.409494+0000","last_active":"2024-03-06T07:33:07.675777+0000","last_peered":"2024-03-06T07:33:07.675777+0000","last_clean":"2024-03-06T07:33:07.675777+0000","last_became_active":"2024-03-06T07:33:05.405957+0000","last_became_peered":"2024-03-06T07:33:05.405957+0000","last_unstale":"2024-03-06T07:33:07.675777+0000","last_undegraded":"2024-03-06T07:33:07.675777+0000","last_fullsized":"2024-03-06T07:33:07.675777+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:43:38.350859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.10","version":"0'0","reported_seq":12,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.672601+0000","last_change":"2024-03-06T07:33:06.438513+0000","last_active":"2024-03-06T07:33:07.672601+0000","last_peered":"2024-03-06T07:33:07.672601+0000","last_clean":"2024-03-06T07:33:07.672601+0000","last_became_active":"2024-03-06T07:33:06.438068+0000","last_became_peered":"2024-03-06T07:33:06.438068+0000","last_unstale":"2024-03-06T07:33:07.672601+0000","last_undegraded":"2024-03-06T07:33:07.672601+0000","last_fullsized":"2024-03-06T07:33:07.672601+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":108,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:05.361395+0000","last_clean_scrub_stamp":"2024-03-06T07:33:05.361395+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:19:57.666871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.11","version":"0'0","reported_seq":14,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.445163+0000","last_change":"2024-03-06T07:33:05.410800+0000","last_active":"2024-03-06T07:33:07.445163+0000","last_peered":"2024-03-06T07:33:07.445163+0000","last_clean":"2024-03-06T07:33:07.445163+0000","last_became_active":"2024-03-06T07:33:05.410464+0000","last_became_peered":"2024-03-06T07:33:05.410464+0000","last_unstale":"2024-03-06T07:33:07.445163+0000","last_undegraded":"2024-03-06T07:33:07.445163+0000","last_fullsized":"2024-03-06T07:33:07.445163+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:33:04.344828+0000","last_clean_scrub_stamp":"2024-03-06T07:33:04.344828+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:54:09.194298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":331,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-03-06T07:33:07.675128+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:33:07.675128+0000","last_peered":"2024-03-06T07:33:07.675128+0000","last_clean":"2024-03-06T07:33:07.675128+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:33:07.675128+0000","last_undegraded":"2024-03-06T07:33:07.675128+0000","last_fullsized":"2024-03-06T07:33:07.675128+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":17,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":16,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542222,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37268,"kb_used_data":1920,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334572,"statfs":{"total":96636764160,"available":96598601728,"internally_reserved":0,"allocated":1966080,"data_stored":1178237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542221,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36680,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335160,"statfs":{"total":96636764160,"available":96599203840,"internally_reserved":0,"allocated":1368064,"data_stored":587323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542222,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36684,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335156,"statfs":{"total":96636764160,"available":96599199744,"internally_reserved":0,"allocated":1372160,"data_stored":587857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542221,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36680,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335160,"statfs":{"total":96636764160,"available":96599203840,"internally_reserved":0,"allocated":1363968,"data_stored":587253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542221,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36696,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335144,"statfs":{"total":96636764160,"available":96599187456,"internally_reserved":0,"allocated":1380352,"data_stored":588935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542222,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36680,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335160,"statfs":{"total":96636764160,"available":96599203840,"internally_reserved":0,"allocated":1363968,"data_stored":587253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542223,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36688,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335152,"statfs":{"total":96636764160,"available":96599195648,"internally_reserved":0,"allocated":1372160,"data_stored":587377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542220,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37280,"kb_used_data":1932,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334560,"statfs":{"total":96636764160,"available":96598589440,"internally_reserved":0,"allocated":1978368,"data_stored":1179345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:33:11.313 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:33:11.313 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:33:11.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.566+0000 7f9b1da00640 1 Processor -- start 2024-03-06T07:33:11.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.567+0000 7f9b1da00640 1 -- start start 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.568+0000 7f9b1da00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9b180cda90 0x7f9b180727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.568+0000 7f9b1da00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b180719e0 con 0x7f9b180b3df0 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.568+0000 7f9b1da00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b18071cc0 con 0x7f9b180b3650 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.568+0000 7f9b1da00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9b18072310 con 0x7f9b180cda90 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.569+0000 7f9b17400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9b180cda90 0x7f9b180727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.569+0000 7f9b16a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9b180b3df0 0x7f9b180b41d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53626/0 (socket says 172.21.15.86:53626) 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.569+0000 7f9b16a00640 1 -- 172.21.15.86:0/3467685869 learned_addr learned my addr 172.21.15.86:0/3467685869 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.569+0000 7f9b17400640 1 --2- 172.21.15.86:0/3467685869 >> v2:172.21.15.86:3300/0 conn(0x7f9b180cda90 0x7f9b180727a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36230/0 (socket says 172.21.15.86:36230) 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.569+0000 7f9b17400640 1 -- 172.21.15.86:0/3467685869 >> v1:172.21.15.194:6789/0 conn(0x7f9b180b3650 legacy=0x7f9b1806c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.569+0000 7f9b17400640 1 -- 172.21.15.86:0/3467685869 >> v1:172.21.15.86:6789/0 conn(0x7f9b180b3df0 legacy=0x7f9b180b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.569+0000 7f9b17400640 1 -- 172.21.15.86:0/3467685869 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b18071fa0 con 0x7f9b180cda90 2024-03-06T07:33:11.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.570+0000 7f9b17400640 1 --2- 172.21.15.86:0/3467685869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b180727a0 secure :-1 s=READY pgs=1263 cs=0 l=1 rev1=1 crypto rx=0x7f9b10009d90 tx=0x7f9b1002beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb9165379ed6be5f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.571+0000 7f9b16000640 1 -- 172.21.15.86:0/3467685869 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b10039070 con 0x7f9b180cda90 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.571+0000 7f9b16000640 1 -- 172.21.15.86:0/3467685869 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b1002cc60 con 0x7f9b180cda90 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.571+0000 7f9b16000640 1 -- 172.21.15.86:0/3467685869 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b10034660 con 0x7f9b180cda90 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.571+0000 7f9b1da00640 1 -- 172.21.15.86:0/3467685869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 msgr2=0x7f9b180727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.571+0000 7f9b1da00640 1 --2- 172.21.15.86:0/3467685869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b180727a0 secure :-1 s=READY pgs=1263 cs=0 l=1 rev1=1 crypto rx=0x7f9b10009d90 tx=0x7f9b1002beb0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.571+0000 7f9b1da00640 1 -- 172.21.15.86:0/3467685869 shutdown_connections 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.571+0000 7f9b1da00640 1 --2- 172.21.15.86:0/3467685869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b180727a0 unknown :-1 s=CLOSED pgs=1263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.572+0000 7f9b1da00640 1 -- 172.21.15.86:0/3467685869 >> 172.21.15.86:0/3467685869 conn(0x7f9b180b1940 msgr2=0x7f9b180b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.572+0000 7f9b1da00640 1 -- 172.21.15.86:0/3467685869 shutdown_connections 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.572+0000 7f9b1da00640 1 -- 172.21.15.86:0/3467685869 wait complete. 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.574+0000 7f9b1da00640 1 Processor -- start 2024-03-06T07:33:11.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.574+0000 7f9b1da00640 1 -- start start 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.575+0000 7f9b1da00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b180b3650 0x7f9b1809bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.575+0000 7f9b1da00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b180b3df0 0x7f9b1814e220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.575+0000 7f9b1da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b181e7be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.575+0000 7f9b1da00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b1815b3c0 con 0x7f9b180cda90 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.575+0000 7f9b1da00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b1815b240 con 0x7f9b180b3650 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.575+0000 7f9b1da00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b180cf510 con 0x7f9b180b3df0 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b181e7be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b181e7be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36240/0 (socket says 172.21.15.86:36240) 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 -- 172.21.15.86:0/487959036 learned_addr learned my addr 172.21.15.86:0/487959036 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b16a00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b180b3650 0x7f9b1809bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b0e000640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b180b3df0 0x7f9b1814e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 -- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b180b3df0 msgr2=0x7f9b1814e220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b180b3df0 0x7f9b1814e220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 -- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b180b3650 msgr2=0x7f9b1809bed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b180b3650 0x7f9b1809bed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b17400640 1 -- 172.21.15.86:0/487959036 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b1809b170 con 0x7f9b180cda90 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.576+0000 7f9b0e000640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b180b3df0 0x7f9b1814e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.577+0000 7f9b17400640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b181e7be0 secure :-1 s=READY pgs=1264 cs=0 l=1 rev1=1 crypto rx=0x7f9b10037040 tx=0x7f9b10004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.577+0000 7f9b0f400640 1 -- 172.21.15.86:0/487959036 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b10039050 con 0x7f9b180cda90 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.577+0000 7f9b0f400640 1 -- 172.21.15.86:0/487959036 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b1003a040 con 0x7f9b180cda90 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.577+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b1809cb70 con 0x7f9b180cda90 2024-03-06T07:33:11.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.577+0000 7f9b0f400640 1 -- 172.21.15.86:0/487959036 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b1003e610 con 0x7f9b180cda90 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.577+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b1809d130 con 0x7f9b180cda90 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.578+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b1804f330 con 0x7f9b180cda90 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.579+0000 7f9b0f400640 1 -- 172.21.15.86:0/487959036 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9b10047020 con 0x7f9b180cda90 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.580+0000 7f9b0f400640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9af406b4d0 0x7f9af406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.580+0000 7f9b0f400640 1 -- 172.21.15.86:0/487959036 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9b100ac680 con 0x7f9b180cda90 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.580+0000 7f9b16a00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9af406b4d0 0x7f9af406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.581+0000 7f9b16a00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9af406b4d0 0x7f9af406d990 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f9b08000f60 tx=0x7f9b08009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:11.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.589+0000 7f9b0f400640 1 -- 172.21.15.86:0/487959036 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9b1004e020 con 0x7f9b180cda90 2024-03-06T07:33:11.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.857+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f9b180cc0e0 con 0x7f9b180cda90 2024-03-06T07:33:11.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.862+0000 7f9b0f400640 1 -- 172.21.15.86:0/487959036 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v19) v1 ==== 190+0+147 (secure 0 0 0) 0x7f9b1007b520 con 0x7f9b180cda90 2024-03-06T07:33:11.864 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:33:11.864 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:33:11.864 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:33:11.864 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:33:11.864 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:33:11.864 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:33:11.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9af406b4d0 msgr2=0x7f9af406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9af406b4d0 0x7f9af406d990 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f9b08000f60 tx=0x7f9b08009040 comp rx=0 tx=0).stop 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 msgr2=0x7f9b181e7be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b181e7be0 secure :-1 s=READY pgs=1264 cs=0 l=1 rev1=1 crypto rx=0x7f9b10037040 tx=0x7f9b10004a40 comp rx=0 tx=0).stop 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 shutdown_connections 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9af406b4d0 0x7f9af406d990 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9b180cda90 0x7f9b181e7be0 unknown :-1 s=CLOSED pgs=1264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9b180b3df0 0x7f9b1814e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 --2- 172.21.15.86:0/487959036 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9b180b3650 0x7f9b1809bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:11.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 >> 172.21.15.86:0/487959036 conn(0x7f9b180b1940 msgr2=0x7f9b180727a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 shutdown_connections 2024-03-06T07:33:11.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:11.866+0000 7f9b1da00640 1 -- 172.21.15.86:0/487959036 wait complete. 2024-03-06T07:33:11.887 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:12.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.077+0000 7f6c17a00640 1 Processor -- start 2024-03-06T07:33:12.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.078+0000 7f6c17a00640 1 -- start start 2024-03-06T07:33:12.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.079+0000 7f6c17a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c10144c00 0x7f6c1006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.079+0000 7f6c17a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6c1015f9d0 0x7f6c1015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.079+0000 7f6c17a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6c1006ee70 con 0x7f6c100ce0a0 2024-03-06T07:33:12.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.079+0000 7f6c17a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c10160370 con 0x7f6c1015f9d0 2024-03-06T07:33:12.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.079+0000 7f6c17a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c1006cef0 con 0x7f6c10144c00 2024-03-06T07:33:12.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.080+0000 7f6c15000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c10144c00 0x7f6c1006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.080+0000 7f6c15000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c10144c00 0x7f6c1006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:57448/0 (socket says 172.21.15.86:57448) 2024-03-06T07:33:12.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.080+0000 7f6c15000640 1 -- 172.21.15.86:0/458802395 learned_addr learned my addr 172.21.15.86:0/458802395 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:12.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.080+0000 7f6c0fe00640 1 --2- 172.21.15.86:0/458802395 >> v2:172.21.15.194:3300/0 conn(0x7f6c1015f9d0 0x7f6c1015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.081+0000 7f6c15000640 1 -- 172.21.15.86:0/458802395 >> v2:172.21.15.194:3300/0 conn(0x7f6c1015f9d0 msgr2=0x7f6c1015fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:12.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.081+0000 7f6c15000640 1 --2- 172.21.15.86:0/458802395 >> v2:172.21.15.194:3300/0 conn(0x7f6c1015f9d0 0x7f6c1015fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.081+0000 7f6c15000640 1 -- 172.21.15.86:0/458802395 >> v1:172.21.15.194:6789/0 conn(0x7f6c100ce0a0 legacy=0x7f6c100ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.081+0000 7f6c15000640 1 -- 172.21.15.86:0/458802395 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c1006d1d0 con 0x7f6c10144c00 2024-03-06T07:33:12.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.082+0000 7f6c15000640 1 --2- 172.21.15.86:0/458802395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c10144c00 0x7f6c1006e930 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f6c04009870 tx=0x7f6c0402b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7c1c0bba3a1d9af6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:12.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c0f400640 1 -- 172.21.15.86:0/458802395 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c04034070 con 0x7f6c10144c00 2024-03-06T07:33:12.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c0f400640 1 -- 172.21.15.86:0/458802395 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c04030840 con 0x7f6c10144c00 2024-03-06T07:33:12.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c0f400640 1 -- 172.21.15.86:0/458802395 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c04030c10 con 0x7f6c10144c00 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c17a00640 1 -- 172.21.15.86:0/458802395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c10144c00 msgr2=0x7f6c1006e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c17a00640 1 --2- 172.21.15.86:0/458802395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c10144c00 0x7f6c1006e930 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f6c04009870 tx=0x7f6c0402b980 comp rx=0 tx=0).stop 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c17a00640 1 -- 172.21.15.86:0/458802395 shutdown_connections 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c17a00640 1 --2- 172.21.15.86:0/458802395 >> v2:172.21.15.194:3300/0 conn(0x7f6c1015f9d0 0x7f6c1015fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c17a00640 1 --2- 172.21.15.86:0/458802395 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c10144c00 0x7f6c1006e930 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.083+0000 7f6c17a00640 1 -- 172.21.15.86:0/458802395 >> 172.21.15.86:0/458802395 conn(0x7f6c100b1730 msgr2=0x7f6c100b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.084+0000 7f6c17a00640 1 -- 172.21.15.86:0/458802395 shutdown_connections 2024-03-06T07:33:12.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.084+0000 7f6c17a00640 1 -- 172.21.15.86:0/458802395 wait complete. 2024-03-06T07:33:12.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.086+0000 7f6c17a00640 1 Processor -- start 2024-03-06T07:33:12.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.086+0000 7f6c17a00640 1 -- start start 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.087+0000 7f6c17a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c100ce0a0 0x7f6c1013b160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.087+0000 7f6c17a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c10144c00 0x7f6c1015c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.087+0000 7f6c17a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c1015f9d0 0x7f6c1013e290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.087+0000 7f6c17a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c10160730 con 0x7f6c100ce0a0 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.087+0000 7f6c17a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c1006d790 con 0x7f6c10144c00 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.087+0000 7f6c17a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c1006cb50 con 0x7f6c1015f9d0 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c100ce0a0 0x7f6c1013b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c100ce0a0 0x7f6c1013b160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36254/0 (socket says 172.21.15.86:36254) 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 -- 172.21.15.86:0/1497612164 learned_addr learned my addr 172.21.15.86:0/1497612164 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c0fe00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c10144c00 0x7f6c1015c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 -- 172.21.15.86:0/1497612164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c1015f9d0 msgr2=0x7f6c1013e290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c1015f9d0 0x7f6c1013e290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 -- 172.21.15.86:0/1497612164 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c10144c00 msgr2=0x7f6c1015c3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c10144c00 0x7f6c1015c3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 -- 172.21.15.86:0/1497612164 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c1015efc0 con 0x7f6c100ce0a0 2024-03-06T07:33:12.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.088+0000 7f6c15000640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c100ce0a0 0x7f6c1013b160 secure :-1 s=READY pgs=1265 cs=0 l=1 rev1=1 crypto rx=0x7f6c04000c00 tx=0x7f6c04004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:12.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.089+0000 7f6c0cc00640 1 -- 172.21.15.86:0/1497612164 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c04048070 con 0x7f6c100ce0a0 2024-03-06T07:33:12.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.089+0000 7f6c0cc00640 1 -- 172.21.15.86:0/1497612164 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c04030640 con 0x7f6c100ce0a0 2024-03-06T07:33:12.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.089+0000 7f6c0cc00640 1 -- 172.21.15.86:0/1497612164 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c04034040 con 0x7f6c100ce0a0 2024-03-06T07:33:12.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.089+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6c1015f2a0 con 0x7f6c100ce0a0 2024-03-06T07:33:12.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.089+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c1014bfe0 con 0x7f6c100ce0a0 2024-03-06T07:33:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.090+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6c100cd9a0 con 0x7f6c100ce0a0 2024-03-06T07:33:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.092+0000 7f6c0cc00640 1 -- 172.21.15.86:0/1497612164 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6c04037070 con 0x7f6c100ce0a0 2024-03-06T07:33:12.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.093+0000 7f6c0cc00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6bf006b4d0 0x7f6bf006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.093+0000 7f6c0cc00640 1 -- 172.21.15.86:0/1497612164 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6c040abec0 con 0x7f6c100ce0a0 2024-03-06T07:33:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.093+0000 7f6c0fe00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6bf006b4d0 0x7f6bf006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.094+0000 7f6c0fe00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6bf006b4d0 0x7f6bf006d990 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f6c1013f3c0 tx=0x7f6bfc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:12.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.098+0000 7f6c0cc00640 1 -- 172.21.15.86:0/1497612164 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6c100cd9a0 con 0x7f6c100ce0a0 2024-03-06T07:33:12.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.328+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6c1004f330 con 0x7f6c100ce0a0 2024-03-06T07:33:12.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.329+0000 7f6c0cc00640 1 -- 172.21.15.86:0/1497612164 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 77 v77) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f6c0407adc0 con 0x7f6c100ce0a0 2024-03-06T07:33:12.332 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:12.332 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":77,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6043,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/906873619","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":906873619},{"type":"v1","addr":"172.21.15.194:6833","nonce":906873619}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":6046,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/398024320","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":398024320},{"type":"v1","addr":"172.21.15.194:6835","nonce":398024320}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":7464,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/1880188173","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":1880188173},{"type":"v1","addr":"172.21.15.86:6835","nonce":1880188173}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":77,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:09.045845+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7467},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7467":{"gid":7467,"name":"c","rank":0,"incarnation":73,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/221976790","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":221976790},{"type":"v1","addr":"172.21.15.86:6837","nonce":221976790}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7467,"qdb_cluster":[7467]},"id":8}]} 2024-03-06T07:33:12.332 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 77 2024-03-06T07:33:12.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6bf006b4d0 msgr2=0x7f6bf006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6bf006b4d0 0x7f6bf006d990 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f6c1013f3c0 tx=0x7f6bfc008040 comp rx=0 tx=0).stop 2024-03-06T07:33:12.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c100ce0a0 msgr2=0x7f6c1013b160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c100ce0a0 0x7f6c1013b160 secure :-1 s=READY pgs=1265 cs=0 l=1 rev1=1 crypto rx=0x7f6c04000c00 tx=0x7f6c04004730 comp rx=0 tx=0).stop 2024-03-06T07:33:12.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 shutdown_connections 2024-03-06T07:33:12.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6bf006b4d0 0x7f6bf006d990 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c1015f9d0 0x7f6c1013e290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c10144c00 0x7f6c1015c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 --2- 172.21.15.86:0/1497612164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c100ce0a0 0x7f6c1013b160 unknown :-1 s=CLOSED pgs=1265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 >> 172.21.15.86:0/1497612164 conn(0x7f6c100b1730 msgr2=0x7f6c100b03e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:12.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 shutdown_connections 2024-03-06T07:33:12.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.332+0000 7f6c17a00640 1 -- 172.21.15.86:0/1497612164 wait complete. 2024-03-06T07:33:12.755 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 77, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:33:05.831771+0000', 'modified': '2024-03-06T07:33:09.045845+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7467}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7467': {'gid': 7467, 'name': 'c', 'rank': 0, 'incarnation': 73, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/221976790', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6836', 'nonce': 221976790}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 221976790}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 7467, 'qdb_cluster': [7467]} 2024-03-06T07:33:12.755 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:33:12.755 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-06T07:33:12.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.960+0000 7f8b52000640 1 Processor -- start 2024-03-06T07:33:12.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.960+0000 7f8b52000640 1 -- start start 2024-03-06T07:33:12.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.962+0000 7f8b52000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8b4c164b30 0x7f8b4c166f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.962+0000 7f8b52000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8b4c145b30 con 0x7f8b4c0c4930 2024-03-06T07:33:12.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.962+0000 7f8b52000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8b4c142ce0 con 0x7f8b4c0c6460 2024-03-06T07:33:12.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.962+0000 7f8b52000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8b4c0c68c0 con 0x7f8b4c164b30 2024-03-06T07:33:12.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.962+0000 7f8b4b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8b4c0c4930 0x7f8b4c0c4d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53636/0 (socket says 172.21.15.86:53636) 2024-03-06T07:33:12.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.962+0000 7f8b4b400640 1 -- 172.21.15.86:0/3540163214 learned_addr learned my addr 172.21.15.86:0/3540163214 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:12.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2220325889 0 0) 0x7f8b3c001110 con 0x7f8b4c0c4930 2024-03-06T07:33:12.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8b2c003520 con 0x7f8b4c0c4930 2024-03-06T07:33:12.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4175785056 0 0) 0x7f8b2c003520 con 0x7f8b4c0c4930 2024-03-06T07:33:12.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3540163214 >> v2:172.21.15.194:3300/0 conn(0x7f8b4c164b30 0x7f8b4c166f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 >> v2:172.21.15.194:3300/0 conn(0x7f8b4c164b30 msgr2=0x7f8b4c166f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:12.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 --2- 172.21.15.86:0/3540163214 >> v2:172.21.15.194:3300/0 conn(0x7f8b4c164b30 0x7f8b4c166f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 >> v1:172.21.15.194:6789/0 conn(0x7f8b4c0c6460 legacy=0x7f8b4c0c5420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:12.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b4c0cbca0 con 0x7f8b4c0c4930 2024-03-06T07:33:12.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8b3c001840 con 0x7f8b4c0c4930 2024-03-06T07:33:12.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 >> v1:172.21.15.86:6789/0 conn(0x7f8b4c0c4930 legacy=0x7f8b4c0c4d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b2c0039b0 0x7f8b2c003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b2c004800 0x7f8b2c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0053c0 0x7f8b2c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b4c0c68c0 con 0x7f8b2c0039b0 2024-03-06T07:33:12.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b2c00c260 con 0x7f8b2c0053c0 2024-03-06T07:33:12.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.963+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8b2c00c460 con 0x7f8b2c004800 2024-03-06T07:33:12.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b2c0039b0 0x7f8b2c003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b2c004800 0x7f8b2c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0053c0 0x7f8b2c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4b400640 1 -- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b2c004800 msgr2=0x7f8b2c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:12.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b2c004800 0x7f8b2c004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4b400640 1 -- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0053c0 msgr2=0x7f8b2c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:12.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0053c0 0x7f8b2c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4b400640 1 -- 172.21.15.86:0/3540163214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b2c003520 con 0x7f8b2c0039b0 2024-03-06T07:33:12.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.964+0000 7f8b4be00640 1 -- 172.21.15.86:0/3540163214 reap_dead start 2024-03-06T07:33:12.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.965+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b2c0039b0 0x7f8b2c003db0 secure :-1 s=READY pgs=1267 cs=0 l=1 rev1=1 crypto rx=0x7f8b3c00c4c0 tx=0x7f8b2c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=76275661c894dfc5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:12.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.965+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b3c00c0e0 con 0x7f8b2c0039b0 2024-03-06T07:33:12.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.965+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b3c030070 con 0x7f8b2c0039b0 2024-03-06T07:33:12.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.965+0000 7f8b4a000640 1 -- 172.21.15.86:0/3540163214 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b3c0058d0 con 0x7f8b2c0039b0 2024-03-06T07:33:12.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.966+0000 7f8b52000640 1 -- 172.21.15.86:0/3540163214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b2c0039b0 msgr2=0x7f8b2c003db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.966+0000 7f8b52000640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b2c0039b0 0x7f8b2c003db0 secure :-1 s=READY pgs=1267 cs=0 l=1 rev1=1 crypto rx=0x7f8b3c00c4c0 tx=0x7f8b2c005830 comp rx=0 tx=0).stop 2024-03-06T07:33:12.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.966+0000 7f8b52000640 1 -- 172.21.15.86:0/3540163214 shutdown_connections 2024-03-06T07:33:12.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.966+0000 7f8b52000640 1 --2- 172.21.15.86:0/3540163214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b2c0039b0 0x7f8b2c003db0 unknown :-1 s=CLOSED pgs=1267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.966+0000 7f8b52000640 1 -- 172.21.15.86:0/3540163214 >> 172.21.15.86:0/3540163214 conn(0x7f8b4c0cc4f0 msgr2=0x7f8b4c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:12.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.966+0000 7f8b52000640 1 -- 172.21.15.86:0/3540163214 shutdown_connections 2024-03-06T07:33:12.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.966+0000 7f8b52000640 1 -- 172.21.15.86:0/3540163214 wait complete. 2024-03-06T07:33:12.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.968+0000 7f8b52000640 1 Processor -- start 2024-03-06T07:33:12.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.968+0000 7f8b52000640 1 -- start start 2024-03-06T07:33:12.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b52000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0039b0 0x7f8b4c13f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b52000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b4c13e4f0 0x7f8b4c0c7b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b52000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b4c13e970 0x7f8b4c0c8040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b52000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b4c0ca3b0 con 0x7f8b4c13e970 2024-03-06T07:33:12.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b52000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b4c0ca0d0 con 0x7f8b2c0039b0 2024-03-06T07:33:12.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b52000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8b4c0cc260 con 0x7f8b4c13e4f0 2024-03-06T07:33:12.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b4be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b4c13e970 0x7f8b4c0c8040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b4be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b4c13e970 0x7f8b4c0c8040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36260/0 (socket says 172.21.15.86:36260) 2024-03-06T07:33:12.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.969+0000 7f8b4be00640 1 -- 172.21.15.86:0/3587711592 learned_addr learned my addr 172.21.15.86:0/3587711592 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:12.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0039b0 0x7f8b4c13f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b4c13e4f0 0x7f8b4c0c7b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4be00640 1 -- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b4c13e4f0 msgr2=0x7f8b4c0c7b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b4c13e4f0 0x7f8b4c0c7b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4be00640 1 -- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0039b0 msgr2=0x7f8b4c13f140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:12.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0039b0 0x7f8b4c13f140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:12.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4be00640 1 -- 172.21.15.86:0/3587711592 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b4c142460 con 0x7f8b4c13e970 2024-03-06T07:33:12.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b4c13e970 0x7f8b4c0c8040 secure :-1 s=READY pgs=1268 cs=0 l=1 rev1=1 crypto rx=0x7f8b4c06eb30 tx=0x7f8b4400ef80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:12.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b43400640 1 -- 172.21.15.86:0/3587711592 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b44003990 con 0x7f8b4c13e970 2024-03-06T07:33:12.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.970+0000 7f8b43400640 1 -- 172.21.15.86:0/3587711592 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b44013070 con 0x7f8b4c13e970 2024-03-06T07:33:12.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.971+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8b4c141ea0 con 0x7f8b4c13e970 2024-03-06T07:33:12.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.971+0000 7f8b43400640 1 -- 172.21.15.86:0/3587711592 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b44003d70 con 0x7f8b4c13e970 2024-03-06T07:33:12.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.971+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8b4c1609c0 con 0x7f8b4c13e970 2024-03-06T07:33:12.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.973+0000 7f8b43400640 1 -- 172.21.15.86:0/3587711592 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8b44004030 con 0x7f8b4c13e970 2024-03-06T07:33:12.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.974+0000 7f8b43400640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b2806b500 0x7f8b2806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.974+0000 7f8b43400640 1 -- 172.21.15.86:0/3587711592 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8b4408aa20 con 0x7f8b4c13e970 2024-03-06T07:33:12.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.974+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b2806b500 0x7f8b2806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.974+0000 7f8b54c21740 1 Processor -- start 2024-03-06T07:33:12.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b2806b500 0x7f8b2806d9c0 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7f8b3c034000 tx=0x7f8b3c039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:12.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b54c21740 1 -- start start 2024-03-06T07:33:12.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d37b0 0x5566600a6750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600d6540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600a6c90 0x5566600d6a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:12.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b54c21740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x556660010600 con 0x5566600d37b0 2024-03-06T07:33:12.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55665ffb9930 con 0x5566600d3b90 2024-03-06T07:33:12.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.975+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55666000c6a0 con 0x5566600a6c90 2024-03-06T07:33:12.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d37b0 0x5566600a6750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:12.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d37b0 0x5566600a6750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36262/0 (socket says 172.21.15.86:36262) 2024-03-06T07:33:13.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 -- 172.21.15.86:0/479196217 learned_addr learned my addr 172.21.15.86:0/479196217 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:13.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600d6540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 -- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600a6c90 msgr2=0x5566600d6a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600a6c90 0x5566600d6a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 -- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 msgr2=0x5566600d6540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600d6540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 -- 172.21.15.86:0/479196217 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55665fd252d0 con 0x5566600d37b0 2024-03-06T07:33:13.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600d6540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:13.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.976+0000 7f8b4b400640 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d37b0 0x5566600a6750 secure :-1 s=READY pgs=1269 cs=0 l=1 rev1=1 crypto rx=0x7f8b3c02c720 tx=0x7f8b3c02e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=782d99d992ea88e3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:13.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.977+0000 7f8b3b400640 1 -- 172.21.15.86:0/479196217 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b3c007270 con 0x5566600d37b0 2024-03-06T07:33:13.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.977+0000 7f8b3b400640 1 -- 172.21.15.86:0/479196217 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b3c030040 con 0x5566600d37b0 2024-03-06T07:33:13.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.977+0000 7f8b3b400640 1 -- 172.21.15.86:0/479196217 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b3c007610 con 0x5566600d37b0 2024-03-06T07:33:13.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.977+0000 7f8b54c21740 1 -- 172.21.15.86:0/479196217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d37b0 msgr2=0x5566600a6750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.977+0000 7f8b54c21740 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d37b0 0x5566600a6750 secure :-1 s=READY pgs=1269 cs=0 l=1 rev1=1 crypto rx=0x7f8b3c02c720 tx=0x7f8b3c02e000 comp rx=0 tx=0).stop 2024-03-06T07:33:13.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.978+0000 7f8b54c21740 1 -- 172.21.15.86:0/479196217 shutdown_connections 2024-03-06T07:33:13.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.978+0000 7f8b54c21740 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600a6c90 0x5566600d6a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.978+0000 7f8b54c21740 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600d6540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.978+0000 7f8b54c21740 1 --2- 172.21.15.86:0/479196217 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d37b0 0x5566600a6750 unknown :-1 s=CLOSED pgs=1269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.978+0000 7f8b54c21740 1 -- 172.21.15.86:0/479196217 >> 172.21.15.86:0/479196217 conn(0x5566600a89c0 msgr2=0x5566600f1540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:13.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.978+0000 7f8b54c21740 1 -- 172.21.15.86:0/479196217 shutdown_connections 2024-03-06T07:33:13.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.978+0000 7f8b54c21740 1 -- 172.21.15.86:0/479196217 wait complete. 2024-03-06T07:33:13.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.979+0000 7f8b54c21740 1 -- start start 2024-03-06T07:33:13.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.979+0000 7f8b54c21740 1 Processor -- start 2024-03-06T07:33:13.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d37b0 0x5566600ec550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d3b90 0x5566600e8060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600e85a0 0x5566600d8be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b54c21740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566600aaf60 con 0x5566600d3b90 2024-03-06T07:33:13.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55665ffccd90 con 0x5566600d37b0 2024-03-06T07:33:13.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5566600aa350 con 0x5566600e85a0 2024-03-06T07:33:13.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b54c21740 10 client.0 _ll_register_callbacks cb 0x556660008350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:33:13.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b4aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d3b90 0x5566600e8060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b4aa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d3b90 0x5566600e8060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36278/0 (socket says 172.21.15.86:36278) 2024-03-06T07:33:13.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b4aa00640 1 -- 172.21.15.86:0/2497307550 learned_addr learned my addr 172.21.15.86:0/2497307550 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:13.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.981+0000 7f8b4b400640 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d37b0 0x5566600ec550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b4aa00640 1 -- 172.21.15.86:0/2497307550 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600e85a0 msgr2=0x5566600d8be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600e85a0 0x5566600d8be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b4aa00640 1 -- 172.21.15.86:0/2497307550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d37b0 msgr2=0x5566600ec550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d37b0 0x5566600ec550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b4aa00640 1 -- 172.21.15.86:0/2497307550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55665fdef6b0 con 0x5566600d3b90 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d3b90 0x5566600e8060 secure :-1 s=READY pgs=1270 cs=0 l=1 rev1=1 crypto rx=0x7f8b340051b0 tx=0x7f8b340105b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=30177689e874271 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b33e00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b33e00640 1 -- 172.21.15.86:0/2497307550 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b34019020 con 0x5566600d3b90 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b33e00640 1 -- 172.21.15.86:0/2497307550 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8b34015480 con 0x5566600d3b90 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8b3c030040 con 0x5566600d3b90 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b33e00640 1 -- 172.21.15.86:0/2497307550 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=77) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8b34023050 con 0x5566600d3b90 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.982+0000 7f8b54c21740 10 client.7599 fetch_fsmap learned FSMap version 77 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=77,monmap=2+}) v3 -- 0x5566600c2770 con 0x5566600d3b90 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b33e00640 1 -- 172.21.15.86:0/2497307550 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b3c030040 con 0x5566600d3b90 2024-03-06T07:33:13.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b33e00640 1 -- 172.21.15.86:0/2497307550 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 77) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f8b34026020 con 0x5566600d3b90 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b54c21740 10 client.7599 fetch_fsmap finished waiting for FSMap version 77 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b54c21740 10 client.7599 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b54c21740 20 client.7599 populate_metadata read hostname 'smithi086' 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b54c21740 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566600f1670 0x5566600c4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b54c21740 4 client.7599 mds_command: new command op to 7467 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 --> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55666001db40 con 0x5566600f1670 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.983+0000 7f8b4b400640 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566600f1670 0x5566600c4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.984+0000 7f8b4b400640 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566600f1670 0x5566600c4dd0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9919e2580050de06 server_cookie=65893ea914be483f in_seq=0 out_seq=0 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.984+0000 7f8b33e00640 10 client.7599 ms_handle_connect on v2:172.21.15.86:6836/221976790 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.985+0000 7f8b33e00640 1 -- 172.21.15.86:0/2497307550 <== mds.0 v2:172.21.15.86:6836/221976790 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x55666001db40 con 0x5566600f1670 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.985+0000 7f8b33e00640 10 client.7599 handle_command_reply: tid=0 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.985+0000 7f8b54c21740 1 client.7599 shutdown 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d3b90 msgr2=0x5566600e8060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d3b90 0x5566600e8060 secure :-1 s=READY pgs=1270 cs=0 l=1 rev1=1 crypto rx=0x7f8b340051b0 tx=0x7f8b340105b0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 shutdown_connections 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566600f1670 0x5566600c4dd0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600e85a0 0x5566600d8be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600d3b90 0x5566600e8060 unknown :-1 s=CLOSED pgs=1270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b33e00640 0 client.7599 ms_handle_reset on v2:172.21.15.86:6836/221976790 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 --2- 172.21.15.86:0/2497307550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d37b0 0x5566600ec550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 >> 172.21.15.86:0/2497307550 conn(0x5566600a6c90 msgr2=0x5566600d6530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 shutdown_connections 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 1 -- 172.21.15.86:0/2497307550 wait complete. 2024-03-06T07:33:13.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:12.986+0000 7f8b54c21740 20 client.7599 trim_cache size 0 max 16384 2024-03-06T07:33:13.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.014+0000 7f8b54c21740 1 Processor -- start 2024-03-06T07:33:13.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.014+0000 7f8b54c21740 1 -- start start 2024-03-06T07:33:13.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.015+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x556660116210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.015+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600a5620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.015+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600a5b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.015+0000 7f8b54c21740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b4c1fdb30 con 0x5566600e85a0 2024-03-06T07:33:13.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.015+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b4c161fd0 con 0x5566600d3b90 2024-03-06T07:33:13.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.015+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8b4c1fd5d0 con 0x5566600d37b0 2024-03-06T07:33:13.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600a5b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600a5b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36290/0 (socket says 172.21.15.86:36290) 2024-03-06T07:33:13.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 -- 172.21.15.86:0/3588494759 learned_addr learned my addr 172.21.15.86:0/3588494759 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:13.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4b400640 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x556660116210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600a5620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 -- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 msgr2=0x556660116210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x556660116210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 -- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 msgr2=0x5566600a5620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600a5620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 -- 172.21.15.86:0/3588494759 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5566600e83d0 con 0x5566600e85a0 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.016+0000 7f8b4be00640 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600a5b60 secure :-1 s=READY pgs=1271 cs=0 l=1 rev1=1 crypto rx=0x7f8b44007440 tx=0x7f8b44055430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8fd23d4ea8274342 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b3aa00640 1 -- 172.21.15.86:0/3588494759 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b4401f070 con 0x5566600e85a0 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b3aa00640 1 -- 172.21.15.86:0/3588494759 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b440618a0 con 0x5566600e85a0 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b3aa00640 1 -- 172.21.15.86:0/3588494759 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b44055800 con 0x5566600e85a0 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b54c21740 1 -- 172.21.15.86:0/3588494759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 msgr2=0x5566600a5b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b54c21740 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600a5b60 secure :-1 s=READY pgs=1271 cs=0 l=1 rev1=1 crypto rx=0x7f8b44007440 tx=0x7f8b44055430 comp rx=0 tx=0).stop 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b54c21740 1 -- 172.21.15.86:0/3588494759 shutdown_connections 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b54c21740 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600a5b60 unknown :-1 s=CLOSED pgs=1271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b54c21740 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600a5620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.017+0000 7f8b54c21740 1 --2- 172.21.15.86:0/3588494759 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x556660116210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.018+0000 7f8b54c21740 1 -- 172.21.15.86:0/3588494759 >> 172.21.15.86:0/3588494759 conn(0x5566600a6c90 msgr2=0x5566600ad770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:13.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.018+0000 7f8b54c21740 1 -- 172.21.15.86:0/3588494759 shutdown_connections 2024-03-06T07:33:13.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.018+0000 7f8b54c21740 1 -- 172.21.15.86:0/3588494759 wait complete. 2024-03-06T07:33:13.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.018+0000 7f8b54c21740 1 -- start start 2024-03-06T07:33:13.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.019+0000 7f8b54c21740 1 Processor -- start 2024-03-06T07:33:13.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x5566600c4120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600eb540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b54c21740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600eba80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b54c21740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566600f14f0 con 0x5566600e85a0 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566600c2b10 con 0x5566600d3b90 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b54c21740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5566600f1af0 con 0x5566600d37b0 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b54c21740 10 client.0 _ll_register_callbacks cb 0x7f8b34007c30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600eba80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600eba80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36292/0 (socket says 172.21.15.86:36292) 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 -- 172.21.15.86:0/1078292869 learned_addr learned my addr 172.21.15.86:0/1078292869 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4aa00640 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600eb540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4b400640 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x5566600c4120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 -- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 msgr2=0x5566600c4120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x5566600c4120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 -- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 msgr2=0x5566600eb540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600eb540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.020+0000 7f8b4be00640 1 -- 172.21.15.86:0/1078292869 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5566600f2130 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b4be00640 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600eba80 secure :-1 s=READY pgs=1272 cs=0 l=1 rev1=1 crypto rx=0x7f8b44061370 tx=0x7f8b4400c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ebd53e6c4abe425 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b33e00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b33e00640 1 -- 172.21.15.86:0/1078292869 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b44074070 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b33e00640 1 -- 172.21.15.86:0/1078292869 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8b440587e0 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8b440618a0 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b33e00640 1 -- 172.21.15.86:0/1078292869 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=77) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8b44080050 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b54c21740 10 client.7605 fetch_fsmap learned FSMap version 77 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=77,monmap=2+}) v3 -- 0x556660103e50 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b33e00640 1 -- 172.21.15.86:0/1078292869 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b440618a0 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.021+0000 7f8b33e00640 1 -- 172.21.15.86:0/1078292869 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 77) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f8b4405f280 con 0x5566600e85a0 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b54c21740 10 client.7605 fetch_fsmap finished waiting for FSMap version 77 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b54c21740 10 client.7605 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:33:13.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b54c21740 20 client.7605 populate_metadata read hostname 'smithi086' 2024-03-06T07:33:13.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b54c21740 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566601697e0 0x556660169bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b54c21740 4 client.7605 mds_command: new command op to 7467 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:33:13.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 --> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] -- command(tid 0: {"prefix": "status"}) v1 -- 0x556660017e90 con 0x5566601697e0 2024-03-06T07:33:13.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b4b400640 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566601697e0 0x556660169bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b4b400640 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566601697e0 0x556660169bc0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3e1df51349129527 server_cookie=70c27bb19d8551b7 in_seq=0 out_seq=0 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.022+0000 7f8b33e00640 10 client.7605 ms_handle_connect on v2:172.21.15.86:6836/221976790 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b33e00640 1 -- 172.21.15.86:0/1078292869 <== mds.0 v2:172.21.15.86:6836/221976790 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+327 (crc 0 0 0) 0x556660017e90 con 0x5566601697e0 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b33e00640 10 client.7605 handle_command_reply: tid=0 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 client.7605 shutdown 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 msgr2=0x5566600eba80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600eba80 secure :-1 s=READY pgs=1272 cs=0 l=1 rev1=1 crypto rx=0x7f8b44061370 tx=0x7f8b4400c920 comp rx=0 tx=0).stop 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 shutdown_connections 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:6836/221976790,v1:172.21.15.86:6837/221976790] conn(0x5566601697e0 0x556660169bc0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5566600e85a0 0x5566600eba80 unknown :-1 s=CLOSED pgs=1272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x5566600d3b90 0x5566600eb540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 --2- 172.21.15.86:0/1078292869 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x5566600d37b0 0x5566600c4120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 >> 172.21.15.86:0/1078292869 conn(0x5566600a6c90 msgr2=0x5566600d2bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b33e00640 0 client.7605 ms_handle_reset on v2:172.21.15.86:6836/221976790 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 shutdown_connections 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 1 -- 172.21.15.86:0/1078292869 wait complete. 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.023+0000 7f8b54c21740 20 client.7605 trim_cache size 0 max 16384 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:33:13.028 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 7467, 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 6.6082172440000004, 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 77, 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 110, 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 110, 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 13.092853921 2024-03-06T07:33:13.029 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b2806b500 msgr2=0x7f8b2806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b2806b500 0x7f8b2806d9c0 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7f8b3c034000 tx=0x7f8b3c039040 comp rx=0 tx=0).stop 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b4c13e970 msgr2=0x7f8b4c0c8040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b4c13e970 0x7f8b4c0c8040 secure :-1 s=READY pgs=1268 cs=0 l=1 rev1=1 crypto rx=0x7f8b4c06eb30 tx=0x7f8b4400ef80 comp rx=0 tx=0).stop 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 shutdown_connections 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8b2806b500 0x7f8b2806d9c0 unknown :-1 s=CLOSED pgs=551 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8b4c13e970 0x7f8b4c0c8040 unknown :-1 s=CLOSED pgs=1268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8b4c13e4f0 0x7f8b4c0c7b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 --2- 172.21.15.86:0/3587711592 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8b2c0039b0 0x7f8b4c13f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 >> 172.21.15.86:0/3587711592 conn(0x7f8b4c0cc4f0 msgr2=0x7f8b4c0718f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 shutdown_connections 2024-03-06T07:33:13.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.024+0000 7f8b52000640 1 -- 172.21.15.86:0/3587711592 wait complete. 2024-03-06T07:33:13.042 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:33:13.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.237+0000 7fc9cf600640 1 Processor -- start 2024-03-06T07:33:13.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.237+0000 7fc9cf600640 1 -- start start 2024-03-06T07:33:13.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.238+0000 7fc9cf600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc9c815acb0 0x7fc9c80b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.238+0000 7fc9cf600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9c815b090 con 0x7fc9c806ec00 2024-03-06T07:33:13.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.238+0000 7fc9cf600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9c80b2a30 con 0x7fc9c80b4680 2024-03-06T07:33:13.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.238+0000 7fc9cf600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc9c80b4360 con 0x7fc9c815acb0 2024-03-06T07:33:13.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.239+0000 7fc9ccc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc9c815acb0 0x7fc9c80b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.239+0000 7fc9ccc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc9c815acb0 0x7fc9c80b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36302/0 (socket says 172.21.15.86:36302) 2024-03-06T07:33:13.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.239+0000 7fc9ccc00640 1 -- 172.21.15.86:0/2006329279 learned_addr learned my addr 172.21.15.86:0/2006329279 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:13.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.239+0000 7fc9ccc00640 1 -- 172.21.15.86:0/2006329279 >> v1:172.21.15.194:6789/0 conn(0x7fc9c80b4680 legacy=0x7fc9c806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.239+0000 7fc9c7400640 1 -- 172.21.15.86:0/2006329279 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2753794396 0 0) 0x7fc9c815b090 con 0x7fc9c806ec00 2024-03-06T07:33:13.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.239+0000 7fc9ccc00640 1 -- 172.21.15.86:0/2006329279 >> v1:172.21.15.86:6789/0 conn(0x7fc9c806ec00 legacy=0x7fc9c80ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.239+0000 7fc9ccc00640 1 -- 172.21.15.86:0/2006329279 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9c80b2cf0 con 0x7fc9c815acb0 2024-03-06T07:33:13.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.240+0000 7fc9ccc00640 1 --2- 172.21.15.86:0/2006329279 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c815acb0 0x7fc9c80b37d0 secure :-1 s=READY pgs=1274 cs=0 l=1 rev1=1 crypto rx=0x7fc9bc00bb10 tx=0x7fc9bc02e1f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=11a85a55c0e472a0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:13.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9c7400640 1 -- 172.21.15.86:0/2006329279 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9bc03b070 con 0x7fc9c815acb0 2024-03-06T07:33:13.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9c7400640 1 -- 172.21.15.86:0/2006329279 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc9bc036440 con 0x7fc9c815acb0 2024-03-06T07:33:13.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9c7400640 1 -- 172.21.15.86:0/2006329279 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9bc036810 con 0x7fc9c815acb0 2024-03-06T07:33:13.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9cf600640 1 -- 172.21.15.86:0/2006329279 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c815acb0 msgr2=0x7fc9c80b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9cf600640 1 --2- 172.21.15.86:0/2006329279 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c815acb0 0x7fc9c80b37d0 secure :-1 s=READY pgs=1274 cs=0 l=1 rev1=1 crypto rx=0x7fc9bc00bb10 tx=0x7fc9bc02e1f0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9cf600640 1 -- 172.21.15.86:0/2006329279 shutdown_connections 2024-03-06T07:33:13.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9cf600640 1 --2- 172.21.15.86:0/2006329279 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c815acb0 0x7fc9c80b37d0 unknown :-1 s=CLOSED pgs=1274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9cf600640 1 -- 172.21.15.86:0/2006329279 >> 172.21.15.86:0/2006329279 conn(0x7fc9c80b1730 msgr2=0x7fc9c80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:13.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9cf600640 1 -- 172.21.15.86:0/2006329279 shutdown_connections 2024-03-06T07:33:13.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.241+0000 7fc9cf600640 1 -- 172.21.15.86:0/2006329279 wait complete. 2024-03-06T07:33:13.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.244+0000 7fc9cf600640 1 Processor -- start 2024-03-06T07:33:13.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.244+0000 7fc9cf600640 1 -- start start 2024-03-06T07:33:13.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9cf600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c806ec00 0x7fc9c8159870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9cf600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9c80b4680 0x7fc9c8159db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9cf600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc9c815acb0 0x7fc9c815cc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9cf600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9c80b3d10 con 0x7fc9c806ec00 2024-03-06T07:33:13.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9cf600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9c80b32b0 con 0x7fc9c80b4680 2024-03-06T07:33:13.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9cf600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9c80b4010 con 0x7fc9c815acb0 2024-03-06T07:33:13.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c806ec00 0x7fc9c8159870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c806ec00 0x7fc9c8159870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36316/0 (socket says 172.21.15.86:36316) 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 -- 172.21.15.86:0/1668408123 learned_addr learned my addr 172.21.15.86:0/1668408123 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 -- 172.21.15.86:0/1668408123 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc9c815acb0 msgr2=0x7fc9c815cc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc9c815acb0 0x7fc9c815cc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 -- 172.21.15.86:0/1668408123 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9c80b4680 msgr2=0x7fc9c8159db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9c80b4680 0x7fc9c8159db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.246+0000 7fc9ccc00640 1 -- 172.21.15.86:0/1668408123 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9c80cb840 con 0x7fc9c806ec00 2024-03-06T07:33:13.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.247+0000 7fc9ccc00640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c806ec00 0x7fc9c8159870 secure :-1 s=READY pgs=1275 cs=0 l=1 rev1=1 crypto rx=0x7fc9bc00b2c0 tx=0x7fc9bc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:13.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.247+0000 7fc9c4c00640 1 -- 172.21.15.86:0/1668408123 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9bc03b040 con 0x7fc9c806ec00 2024-03-06T07:33:13.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.247+0000 7fc9c4c00640 1 -- 172.21.15.86:0/1668408123 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc9bc046070 con 0x7fc9c806ec00 2024-03-06T07:33:13.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.247+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc9c815d970 con 0x7fc9c806ec00 2024-03-06T07:33:13.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.247+0000 7fc9c4c00640 1 -- 172.21.15.86:0/1668408123 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9bc034bd0 con 0x7fc9c806ec00 2024-03-06T07:33:13.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.247+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc9c8140880 con 0x7fc9c806ec00 2024-03-06T07:33:13.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.248+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc9c804f330 con 0x7fc9c806ec00 2024-03-06T07:33:13.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.250+0000 7fc9c4c00640 1 -- 172.21.15.86:0/1668408123 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc9bc04b460 con 0x7fc9c806ec00 2024-03-06T07:33:13.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.251+0000 7fc9c4c00640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9a806b500 0x7fc9a806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:13.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.251+0000 7fc9c4c00640 1 -- 172.21.15.86:0/1668408123 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc9bc0af170 con 0x7fc9c806ec00 2024-03-06T07:33:13.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.251+0000 7fc9c7e00640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9a806b500 0x7fc9a806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:13.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.252+0000 7fc9c7e00640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9a806b500 0x7fc9a806d9c0 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7fc9b4000f60 tx=0x7fc9b40079b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:13.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.256+0000 7fc9c4c00640 1 -- 172.21.15.86:0/1668408123 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc9bc049360 con 0x7fc9c806ec00 2024-03-06T07:33:13.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.500+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fc9c806ce60 con 0x7fc9c806ec00 2024-03-06T07:33:13.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.501+0000 7fc9c4c00640 1 -- 172.21.15.86:0/1668408123 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v77) v1 ==== 53+0+73 (secure 0 0 0) 0x7fc9bc07df50 con 0x7fc9c806ec00 2024-03-06T07:33:13.503 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:33:13.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9a806b500 msgr2=0x7fc9a806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9a806b500 0x7fc9a806d9c0 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7fc9b4000f60 tx=0x7fc9b40079b0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c806ec00 msgr2=0x7fc9c8159870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:13.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c806ec00 0x7fc9c8159870 secure :-1 s=READY pgs=1275 cs=0 l=1 rev1=1 crypto rx=0x7fc9bc00b2c0 tx=0x7fc9bc0046c0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 shutdown_connections 2024-03-06T07:33:13.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc9a806b500 0x7fc9a806d9c0 unknown :-1 s=CLOSED pgs=552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc9c815acb0 0x7fc9c815cc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9c80b4680 0x7fc9c8159db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 --2- 172.21.15.86:0/1668408123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9c806ec00 0x7fc9c8159870 unknown :-1 s=CLOSED pgs=1275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:13.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 >> 172.21.15.86:0/1668408123 conn(0x7fc9c80b1730 msgr2=0x7fc9c8156220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:13.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.504+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 shutdown_connections 2024-03-06T07:33:13.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:13.505+0000 7fc9cf600640 1 -- 172.21.15.86:0/1668408123 wait complete. 2024-03-06T07:33:13.524 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:33:13.524 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:33:13.524 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:33:13.524 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:33:13.524 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:13.525 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:33:13.525 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:33:13.525 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:33:13.525 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:33:13.525 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:13.525 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:33:13.567 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:33:13.567 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:33:13.567 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:33:13.568 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:13.568 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:33:13.568 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:13.568 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:33:13.568 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:13.568 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42073sec preferred_lft 42073sec 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:33:13.569 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:13.570 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:13.570 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:33:13.694 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:33:13.698 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:13.698 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:33:13.743 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:33:13.743 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:33:13.744 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:33:13.744 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:33:13.744 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:33:13.744 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:13.744 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:33:13.744 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:33:13.744 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:33:13.744 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:33:13.744 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:33:13.744 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:33:13.965 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:13.965 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:33:14.012 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:33:14.012 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:33:14.013 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:33:14.013 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:14.013 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:33:14.094 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:33:14.094 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:14.145 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:33:14.145 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:33:14.145 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:14.192 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:33:14.192 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:14.192 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:33:14.374 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:33:14.375 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:33:14.405 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:33:14.405 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:33:14.405 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:33:14.405 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:33:14.406 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:33:14.408 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:33:14.408 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:33:14.408 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:33:14.408 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:33:14.408 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:33:14.412 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:33:14.412 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:14.412 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:14.494 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:14.494 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:33:14.546 INFO:teuthology.orchestra.run.smithi086.stdout:client7617 2024-03-06T07:33:14.546 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:33:14.546 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:33:14.587 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:14.587 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec mkdir dir_1) 2024-03-06T07:33:14.687 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:33:14.687 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:33:14.734 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:14.734 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec dd if=/dev/urandom of=dir_1/file_a bs=1M conv=fdatasync count=8 seek=0) 2024-03-06T07:33:14.984 INFO:teuthology.orchestra.run.smithi086.stderr:8+0 records in 2024-03-06T07:33:14.984 INFO:teuthology.orchestra.run.smithi086.stderr:8+0 records out 2024-03-06T07:33:14.985 INFO:teuthology.orchestra.run.smithi086.stderr:8388608 bytes (8.4 MB, 8.0 MiB) copied, 0.152777 s, 54.9 MB/s 2024-03-06T07:33:14.986 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:33:14.986 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:33:14.986 DEBUG:teuthology.orchestra.run.smithi086:> import stat 2024-03-06T07:33:14.987 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:33:14.987 DEBUG:teuthology.orchestra.run.smithi086:> print(os.stat("/home/ubuntu/cephtest/mnt.0/dir_1/file_a").st_ino) 2024-03-06T07:33:14.987 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:33:15.142 INFO:teuthology.orchestra.run.smithi086.stdout:1099511627777 2024-03-06T07:33:15.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:33:15.159 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:15.160 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec mkdir dir_2) 2024-03-06T07:33:15.253 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:15.253 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec ln dir_1/file_a dir_2/file_b) 2024-03-06T07:33:15.304 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:33:15.304 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:33:15.304 DEBUG:teuthology.orchestra.run.smithi086:> import stat 2024-03-06T07:33:15.304 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:33:15.304 DEBUG:teuthology.orchestra.run.smithi086:> print(os.stat("/home/ubuntu/cephtest/mnt.0/dir_2/file_b").st_ino) 2024-03-06T07:33:15.304 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:33:15.424 INFO:teuthology.orchestra.run.smithi086.stdout:1099511627777 2024-03-06T07:33:15.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:33:15.436 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:15.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.675+0000 7f6a7f800640 1 Processor -- start 2024-03-06T07:33:15.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.675+0000 7f6a7f800640 1 -- start start 2024-03-06T07:33:15.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7f800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6a78147310 0x7f6a78072340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:15.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7f800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a780b39c0 con 0x7f6a7806d390 2024-03-06T07:33:15.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7f800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a780ad9e0 con 0x7f6a7806daa0 2024-03-06T07:33:15.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7f800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6a7806df30 con 0x7f6a78147310 2024-03-06T07:33:15.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6a78147310 0x7f6a78072340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:15.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6a78147310 0x7f6a78072340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36328/0 (socket says 172.21.15.86:36328) 2024-03-06T07:33:15.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7ce00640 1 -- 172.21.15.86:0/3479641948 learned_addr learned my addr 172.21.15.86:0/3479641948 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:15.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.677+0000 7f6a7d800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6a7806d390 0x7f6a780ae7a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:53654/0 (socket says 172.21.15.86:53654) 2024-03-06T07:33:15.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.678+0000 7f6a7ce00640 1 -- 172.21.15.86:0/3479641948 >> v1:172.21.15.194:6789/0 conn(0x7f6a7806daa0 legacy=0x7f6a7806cc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:15.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.678+0000 7f6a7ce00640 1 -- 172.21.15.86:0/3479641948 >> v1:172.21.15.86:6789/0 conn(0x7f6a7806d390 legacy=0x7f6a780ae7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:15.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.678+0000 7f6a7ce00640 1 -- 172.21.15.86:0/3479641948 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a780adcc0 con 0x7f6a78147310 2024-03-06T07:33:15.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.678+0000 7f6a77e00640 1 -- 172.21.15.86:0/3479641948 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1336155266 0 0) 0x7f6a70001490 con 0x7f6a7806d390 2024-03-06T07:33:15.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.679+0000 7f6a7ce00640 1 --2- 172.21.15.86:0/3479641948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a78147310 0x7f6a78072340 secure :-1 s=READY pgs=1278 cs=0 l=1 rev1=1 crypto rx=0x7f6a68009870 tx=0x7f6a6802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9e4906c11fbbead server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:15.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.679+0000 7f6a77e00640 1 -- 172.21.15.86:0/3479641948 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a68034070 con 0x7f6a78147310 2024-03-06T07:33:15.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.679+0000 7f6a77e00640 1 -- 172.21.15.86:0/3479641948 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a68030860 con 0x7f6a78147310 2024-03-06T07:33:15.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.679+0000 7f6a77e00640 1 -- 172.21.15.86:0/3479641948 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a68030c30 con 0x7f6a78147310 2024-03-06T07:33:15.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.680+0000 7f6a7f800640 1 -- 172.21.15.86:0/3479641948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a78147310 msgr2=0x7f6a78072340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:15.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.680+0000 7f6a7f800640 1 --2- 172.21.15.86:0/3479641948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a78147310 0x7f6a78072340 secure :-1 s=READY pgs=1278 cs=0 l=1 rev1=1 crypto rx=0x7f6a68009870 tx=0x7f6a6802b9a0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.680+0000 7f6a7f800640 1 -- 172.21.15.86:0/3479641948 shutdown_connections 2024-03-06T07:33:15.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.680+0000 7f6a7f800640 1 --2- 172.21.15.86:0/3479641948 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a78147310 0x7f6a78072340 unknown :-1 s=CLOSED pgs=1278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.680+0000 7f6a7f800640 1 -- 172.21.15.86:0/3479641948 >> 172.21.15.86:0/3479641948 conn(0x7f6a780b1be0 msgr2=0x7f6a780b1ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:15.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.680+0000 7f6a7f800640 1 -- 172.21.15.86:0/3479641948 shutdown_connections 2024-03-06T07:33:15.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.680+0000 7f6a7f800640 1 -- 172.21.15.86:0/3479641948 wait complete. 2024-03-06T07:33:15.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.683+0000 7f6a7f800640 1 Processor -- start 2024-03-06T07:33:15.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.683+0000 7f6a7f800640 1 -- start start 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.684+0000 7f6a7f800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a7806d390 0x7f6a7815d760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.684+0000 7f6a7f800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a7806daa0 0x7f6a7813fb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.684+0000 7f6a7f800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a78147310 0x7f6a780b52f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.684+0000 7f6a7f800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a781476f0 con 0x7f6a7806daa0 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.684+0000 7f6a7f800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a780ae280 con 0x7f6a7806d390 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.684+0000 7f6a7f800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a780aeeb0 con 0x7f6a78147310 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a7806daa0 0x7f6a7813fb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:15.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a7806daa0 0x7f6a7813fb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36338/0 (socket says 172.21.15.86:36338) 2024-03-06T07:33:15.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 -- 172.21.15.86:0/193463203 learned_addr learned my addr 172.21.15.86:0/193463203 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a7ce00640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a7806d390 0x7f6a7815d760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a7d800640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a78147310 0x7f6a780b52f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 -- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a78147310 msgr2=0x7f6a780b52f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a78147310 0x7f6a780b52f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 -- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a7806d390 msgr2=0x7f6a7815d760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a7806d390 0x7f6a7815d760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.685+0000 7f6a6fe00640 1 -- 172.21.15.86:0/193463203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a7813b2f0 con 0x7f6a7806daa0 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.686+0000 7f6a6fe00640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a7806daa0 0x7f6a7813fb80 secure :-1 s=READY pgs=1279 cs=0 l=1 rev1=1 crypto rx=0x7f6a6000c430 tx=0x7f6a6000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.686+0000 7f6a75600640 1 -- 172.21.15.86:0/193463203 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a60011790 con 0x7f6a7806daa0 2024-03-06T07:33:15.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.686+0000 7f6a75600640 1 -- 172.21.15.86:0/193463203 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a60017070 con 0x7f6a7806daa0 2024-03-06T07:33:15.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.686+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a7813c350 con 0x7f6a7806daa0 2024-03-06T07:33:15.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.686+0000 7f6a75600640 1 -- 172.21.15.86:0/193463203 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a60011b70 con 0x7f6a7806daa0 2024-03-06T07:33:15.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.686+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a7813b870 con 0x7f6a7806daa0 2024-03-06T07:33:15.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.687+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6a7804f330 con 0x7f6a7806daa0 2024-03-06T07:33:15.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.688+0000 7f6a75600640 1 -- 172.21.15.86:0/193463203 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6a60011d10 con 0x7f6a7806daa0 2024-03-06T07:33:15.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.689+0000 7f6a75600640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a5406b4d0 0x7f6a5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:15.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.689+0000 7f6a75600640 1 -- 172.21.15.86:0/193463203 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6a6008a160 con 0x7f6a7806daa0 2024-03-06T07:33:15.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.689+0000 7f6a7ce00640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a5406b4d0 0x7f6a5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:15.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.690+0000 7f6a7ce00640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a5406b4d0 0x7f6a5406d990 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f6a68036000 tx=0x7f6a6803b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:15.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.693+0000 7f6a75600640 1 -- 172.21.15.86:0/193463203 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6a60058f40 con 0x7f6a7806daa0 2024-03-06T07:33:15.928 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.926+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6a7806d880 con 0x7f6a7806daa0 2024-03-06T07:33:15.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.928+0000 7f6a75600640 1 -- 172.21.15.86:0/193463203 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 77 v77) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f6a60058840 con 0x7f6a7806daa0 2024-03-06T07:33:15.929 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:15.929 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":77,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6043,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/906873619","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":906873619},{"type":"v1","addr":"172.21.15.194:6833","nonce":906873619}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":6046,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/398024320","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":398024320},{"type":"v1","addr":"172.21.15.194:6835","nonce":398024320}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":7464,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/1880188173","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":1880188173},{"type":"v1","addr":"172.21.15.86:6835","nonce":1880188173}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":77,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:09.045845+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7467},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7467":{"gid":7467,"name":"c","rank":0,"incarnation":73,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/221976790","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":221976790},{"type":"v1","addr":"172.21.15.86:6837","nonce":221976790}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7467,"qdb_cluster":[7467]},"id":8}]} 2024-03-06T07:33:15.930 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 77 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a5406b4d0 msgr2=0x7f6a5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a5406b4d0 0x7f6a5406d990 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f6a68036000 tx=0x7f6a6803b040 comp rx=0 tx=0).stop 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a7806daa0 msgr2=0x7f6a7813fb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a7806daa0 0x7f6a7813fb80 secure :-1 s=READY pgs=1279 cs=0 l=1 rev1=1 crypto rx=0x7f6a6000c430 tx=0x7f6a6000c8f0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 shutdown_connections 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a5406b4d0 0x7f6a5406d990 unknown :-1 s=CLOSED pgs=553 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a78147310 0x7f6a780b52f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a7806daa0 0x7f6a7813fb80 unknown :-1 s=CLOSED pgs=1279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 --2- 172.21.15.86:0/193463203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a7806d390 0x7f6a7815d760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:15.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 >> 172.21.15.86:0/193463203 conn(0x7f6a780b1be0 msgr2=0x7f6a78161170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:15.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 shutdown_connections 2024-03-06T07:33:15.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:15.930+0000 7f6a7f800640 1 -- 172.21.15.86:0/193463203 wait complete. 2024-03-06T07:33:15.952 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:33:16.190 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:33:16.191 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:33:16.191 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-06T07:33:16.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.292+0000 7faaae246f40 1 Processor -- start 2024-03-06T07:33:16.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.292+0000 7faaae246f40 1 -- start start 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.293+0000 7faaae246f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa900022f0 0x556b13b03ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.293+0000 7faaae246f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x556b13af75e0 0x556b13af7f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.293+0000 7faaae246f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 0x556b13afe700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.293+0000 7faaae246f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b139d1fd0 con 0x556b13af75e0 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.293+0000 7faaae246f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b139d5f30 con 0x7faa900022f0 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.293+0000 7faaae246f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x556b13a74880 con 0x556b13af84c0 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x556b13af75e0 0x556b13af7f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x556b13af75e0 0x556b13af7f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60434/0 (socket says 172.21.15.86:60434) 2024-03-06T07:33:16.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 -- 172.21.15.86:0/768139858 learned_addr learned my addr 172.21.15.86:0/768139858 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaac00640 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa900022f0 0x556b13b03ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 -- 172.21.15.86:0/768139858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 msgr2=0x556b13afe700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 0x556b13afe700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 -- 172.21.15.86:0/768139858 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa900022f0 msgr2=0x556b13b03ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa900022f0 0x556b13b03ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 -- 172.21.15.86:0/768139858 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556b13af72a0 con 0x556b13af75e0 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.294+0000 7faaaa200640 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x556b13af75e0 0x556b13af7f80 secure :-1 s=READY pgs=1282 cs=0 l=1 rev1=1 crypto rx=0x7faa9800c580 tx=0x7faa9800ca40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=63a24c3559ead4c3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaa8e00640 1 -- 172.21.15.86:0/768139858 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa98012070 con 0x556b13af75e0 2024-03-06T07:33:16.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaa8e00640 1 -- 172.21.15.86:0/768139858 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faa9800e700 con 0x556b13af75e0 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaa8e00640 1 -- 172.21.15.86:0/768139858 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa9800eaf0 con 0x556b13af75e0 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 -- 172.21.15.86:0/768139858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x556b13af75e0 msgr2=0x556b13af7f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x556b13af75e0 0x556b13af7f80 secure :-1 s=READY pgs=1282 cs=0 l=1 rev1=1 crypto rx=0x7faa9800c580 tx=0x7faa9800ca40 comp rx=0 tx=0).stop 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 -- 172.21.15.86:0/768139858 shutdown_connections 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 0x556b13afe700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x556b13af75e0 0x556b13af7f80 unknown :-1 s=CLOSED pgs=1282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 --2- 172.21.15.86:0/768139858 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faa900022f0 0x556b13b03ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 -- 172.21.15.86:0/768139858 >> 172.21.15.86:0/768139858 conn(0x556b13a58bf0 msgr2=0x556b13a5a720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 -- 172.21.15.86:0/768139858 shutdown_connections 2024-03-06T07:33:16.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.295+0000 7faaae246f40 1 -- 172.21.15.86:0/768139858 wait complete. 2024-03-06T07:33:16.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.296+0000 7faaae246f40 1 Processor -- start 2024-03-06T07:33:16.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.296+0000 7faaae246f40 1 -- start start 2024-03-06T07:33:16.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaae246f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x556b13af75e0 0x556b13b9f450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaae246f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 0x556b13af5560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaae246f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa900022f0 0x556b13afc670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaae246f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b13a2ddc0 con 0x7faa900022f0 2024-03-06T07:33:16.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaae246f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b13a5b480 con 0x556b13af75e0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaae246f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x556b13a2e080 con 0x556b13af84c0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaab600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa900022f0 0x556b13afc670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.297+0000 7faaab600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa900022f0 0x556b13afc670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60442/0 (socket says 172.21.15.86:60442) 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaab600640 1 -- 172.21.15.86:0/2875231183 learned_addr learned my addr 172.21.15.86:0/2875231183 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaaa200640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 0x556b13af5560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaaac00640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x556b13af75e0 0x556b13b9f450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaab600640 1 -- 172.21.15.86:0/2875231183 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 msgr2=0x556b13af5560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaab600640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 0x556b13af5560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaab600640 1 -- 172.21.15.86:0/2875231183 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x556b13af75e0 msgr2=0x556b13b9f450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaab600640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x556b13af75e0 0x556b13b9f450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaab600640 1 -- 172.21.15.86:0/2875231183 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556b13b9fa30 con 0x7faa900022f0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaaa200640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x556b13af84c0 0x556b13af5560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.298+0000 7faaab600640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa900022f0 0x556b13afc670 secure :-1 s=READY pgs=1283 cs=0 l=1 rev1=1 crypto rx=0x556b13a19c70 tx=0x7faaa400b4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.299+0000 7faaa3400640 1 -- 172.21.15.86:0/2875231183 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faaa400bb60 con 0x7faa900022f0 2024-03-06T07:33:16.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.299+0000 7faaa3400640 1 -- 172.21.15.86:0/2875231183 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faaa400bd00 con 0x7faa900022f0 2024-03-06T07:33:16.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.299+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x556b13b9fd10 con 0x7faa900022f0 2024-03-06T07:33:16.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.299+0000 7faaa3400640 1 -- 172.21.15.86:0/2875231183 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faaa4004670 con 0x7faa900022f0 2024-03-06T07:33:16.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.299+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x556b13ba02d0 con 0x7faa900022f0 2024-03-06T07:33:16.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.301+0000 7faaa3400640 1 -- 172.21.15.86:0/2875231183 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faaa4010070 con 0x7faa900022f0 2024-03-06T07:33:16.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.302+0000 7faaa3400640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa8c069eb0 0x7faa8c06c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.302+0000 7faaa3400640 1 -- 172.21.15.86:0/2875231183 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7faaa4087ec0 con 0x7faa900022f0 2024-03-06T07:33:16.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.302+0000 7faaaac00640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa8c069eb0 0x7faa8c06c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.302+0000 7faaae246f40 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x556b13afcbb0 0x556b13afb520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.302+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e110) v9 -- 0x556b13afba60 con 0x556b13afcbb0 2024-03-06T07:33:16.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.302+0000 7faaaa200640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x556b13afcbb0 0x556b13afb520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.303+0000 7faaaac00640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa8c069eb0 0x7faa8c06c370 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7faa9c0099a0 tx=0x7faa9c04c070 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:16.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.303+0000 7faaaa200640 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x556b13afcbb0 0x556b13afb520 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:16.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.303+0000 7faaaa200640 1 -- 172.21.15.86:0/2875231183 <== osd.0 v2:172.21.15.86:6800/1601033517 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (93) out=93b] v0'0 uv3 ondisk = 0) v8 ==== 164+0+93 (crc 0 0 0) 0x7faa9800ea30 con 0x556b13afcbb0 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.303+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x556b13afcbb0 msgr2=0x556b13afb520 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.303+0000 7faaae246f40 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x556b13afcbb0 0x556b13afb520 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa8c069eb0 msgr2=0x7faa8c06c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faa8c069eb0 0x7faa8c06c370 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7faa9c0099a0 tx=0x7faa9c04c070 comp rx=0 tx=0).stop 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa900022f0 msgr2=0x556b13afc670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 --2- 172.21.15.86:0/2875231183 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faa900022f0 0x556b13afc670 secure :-1 s=READY pgs=1283 cs=0 l=1 rev1=1 crypto rx=0x556b13a19c70 tx=0x7faaa400b4b0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaab600640 1 -- 172.21.15.86:0/2875231183 reap_dead start 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 shutdown_connections 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 >> 172.21.15.86:0/2875231183 conn(0x556b13a58bf0 msgr2=0x556b13a15930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 shutdown_connections 2024-03-06T07:33:16.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.304+0000 7faaae246f40 1 -- 172.21.15.86:0/2875231183 wait complete. 2024-03-06T07:33:16.309 INFO:teuthology.orchestra.run.smithi086.stdout:Wfile_adir_1 2024-03-06T07:33:16.310 DEBUG:teuthology.orchestra.run.smithi086:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-06T07:33:16.492 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:33:16.492 INFO:teuthology.orchestra.run.smithi086.stdout: "ino": 1099511627777, 2024-03-06T07:33:16.492 INFO:teuthology.orchestra.run.smithi086.stdout: "ancestors": [ 2024-03-06T07:33:16.492 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:16.492 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 1099511627776, 2024-03-06T07:33:16.492 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "file_a", 2024-03-06T07:33:16.492 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 8 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 1, 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "dir_1", 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 15 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: "pool": 17, 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: "old_pools": [] 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:33:16.493 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:16.501 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:33:16.502 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:33:16.553 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:33:16.553 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:33:16.644 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:33:16.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.893+0000 7f356e600640 1 Processor -- start 2024-03-06T07:33:16.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.893+0000 7f356e600640 1 -- start start 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.894+0000 7f356e600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35680aedc0 0x7f35680cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f356e600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680cc800 0x7f356815f830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f356e600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f356815c240 con 0x7f35681561f0 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f356e600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f356815ce30 con 0x7f35680aedc0 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f356e600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f356815c500 con 0x7f35680cc800 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f3567e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35680aedc0 0x7f35680cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f3567e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35680aedc0 0x7f35680cc2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60452/0 (socket says 172.21.15.86:60452) 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f3567e00640 1 -- 172.21.15.86:0/1132040152 learned_addr learned my addr 172.21.15.86:0/1132040152 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f356dc00640 1 --2- 172.21.15.86:0/1132040152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680cc800 0x7f356815f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f3567e00640 1 -- 172.21.15.86:0/1132040152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680cc800 msgr2=0x7f356815f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:16.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f3567e00640 1 --2- 172.21.15.86:0/1132040152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680cc800 0x7f356815f830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f3567e00640 1 -- 172.21.15.86:0/1132040152 >> v1:172.21.15.194:6789/0 conn(0x7f35681561f0 legacy=0x7f35680ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:16.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.895+0000 7f3567e00640 1 -- 172.21.15.86:0/1132040152 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f356815c7e0 con 0x7f35680aedc0 2024-03-06T07:33:16.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.897+0000 7f3567e00640 1 --2- 172.21.15.86:0/1132040152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680aedc0 0x7f35680cc2c0 secure :-1 s=READY pgs=1284 cs=0 l=1 rev1=1 crypto rx=0x7f3554009870 tx=0x7f355402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ac9e532a6a55880a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:16.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.898+0000 7f3567400640 1 -- 172.21.15.86:0/1132040152 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3554035020 con 0x7f35680aedc0 2024-03-06T07:33:16.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.898+0000 7f3567400640 1 -- 172.21.15.86:0/1132040152 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3554031640 con 0x7f35680aedc0 2024-03-06T07:33:16.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.898+0000 7f3567400640 1 -- 172.21.15.86:0/1132040152 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f355402d040 con 0x7f35680aedc0 2024-03-06T07:33:16.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.898+0000 7f356e600640 1 -- 172.21.15.86:0/1132040152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680aedc0 msgr2=0x7f35680cc2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.898+0000 7f356e600640 1 --2- 172.21.15.86:0/1132040152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680aedc0 0x7f35680cc2c0 secure :-1 s=READY pgs=1284 cs=0 l=1 rev1=1 crypto rx=0x7f3554009870 tx=0x7f355402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.899+0000 7f356e600640 1 -- 172.21.15.86:0/1132040152 shutdown_connections 2024-03-06T07:33:16.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.899+0000 7f356e600640 1 --2- 172.21.15.86:0/1132040152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680cc800 0x7f356815f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.899+0000 7f356e600640 1 --2- 172.21.15.86:0/1132040152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680aedc0 0x7f35680cc2c0 unknown :-1 s=CLOSED pgs=1284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.899+0000 7f356e600640 1 -- 172.21.15.86:0/1132040152 >> 172.21.15.86:0/1132040152 conn(0x7f356806afc0 msgr2=0x7f356806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:16.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.899+0000 7f356e600640 1 -- 172.21.15.86:0/1132040152 shutdown_connections 2024-03-06T07:33:16.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.899+0000 7f356e600640 1 -- 172.21.15.86:0/1132040152 wait complete. 2024-03-06T07:33:16.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.901+0000 7f356e600640 1 Processor -- start 2024-03-06T07:33:16.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.901+0000 7f356e600640 1 -- start start 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356e600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680aedc0 0x7f35681f3ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356e600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680cc800 0x7f35680ce490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356e600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35681561f0 0x7f35680ce9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356e600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35681565d0 con 0x7f35680cc800 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356e600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35680ccc70 con 0x7f35681561f0 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356e600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3568156750 con 0x7f35680aedc0 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680cc800 0x7f35680ce490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680cc800 0x7f35680ce490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60462/0 (socket says 172.21.15.86:60462) 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 -- 172.21.15.86:0/453548289 learned_addr learned my addr 172.21.15.86:0/453548289 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356d200640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680aedc0 0x7f35681f3ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356dc00640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35681561f0 0x7f35680ce9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 -- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680aedc0 msgr2=0x7f35681f3ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680aedc0 0x7f35681f3ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 -- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35681561f0 msgr2=0x7f35680ce9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:16.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35681561f0 0x7f35680ce9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:16.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f3567e00640 1 -- 172.21.15.86:0/453548289 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35681f4510 con 0x7f35680cc800 2024-03-06T07:33:16.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.902+0000 7f356d200640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680aedc0 0x7f35681f3ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:16.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.903+0000 7f3567e00640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680cc800 0x7f35680ce490 secure :-1 s=READY pgs=1285 cs=0 l=1 rev1=1 crypto rx=0x7f355402b930 tx=0x7f35540046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:16.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.903+0000 7f3564c00640 1 -- 172.21.15.86:0/453548289 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3554040070 con 0x7f35680cc800 2024-03-06T07:33:16.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.903+0000 7f3564c00640 1 -- 172.21.15.86:0/453548289 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3554039bf0 con 0x7f35680cc800 2024-03-06T07:33:16.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.903+0000 7f3564c00640 1 -- 172.21.15.86:0/453548289 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3554035040 con 0x7f35680cc800 2024-03-06T07:33:16.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.903+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35681f47f0 con 0x7f35680cc800 2024-03-06T07:33:16.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.904+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35681f4db0 con 0x7f35680cc800 2024-03-06T07:33:16.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.905+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f356804f330 con 0x7f35680cc800 2024-03-06T07:33:16.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.906+0000 7f3564c00640 1 -- 172.21.15.86:0/453548289 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f355404b060 con 0x7f35680cc800 2024-03-06T07:33:16.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.906+0000 7f3564c00640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f354806b4d0 0x7f354806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:16.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.907+0000 7f3564c00640 1 -- 172.21.15.86:0/453548289 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f35540abed0 con 0x7f35680cc800 2024-03-06T07:33:16.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.907+0000 7f356d200640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f354806b4d0 0x7f354806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:16.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.907+0000 7f356d200640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f354806b4d0 0x7f354806d990 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f3568069290 tx=0x7f355c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:16.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:16.922+0000 7f3564c00640 1 -- 172.21.15.86:0/453548289 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f355402f300 con 0x7f35680cc800 2024-03-06T07:33:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.122+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f35680b2b10 con 0x7f35680cc800 2024-03-06T07:33:17.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.126+0000 7f3564c00640 1 -- 172.21.15.86:0/453548289 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v110) v1 ==== 74+0+16639 (secure 0 0 0) 0x7f355407ac30 con 0x7f35680cc800 2024-03-06T07:33:17.128 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:17.128 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":110,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:07.410898+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"110","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:33:17.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f354806b4d0 msgr2=0x7f354806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f354806b4d0 0x7f354806d990 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f3568069290 tx=0x7f355c008040 comp rx=0 tx=0).stop 2024-03-06T07:33:17.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680cc800 msgr2=0x7f35680ce490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680cc800 0x7f35680ce490 secure :-1 s=READY pgs=1285 cs=0 l=1 rev1=1 crypto rx=0x7f355402b930 tx=0x7f35540046c0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 shutdown_connections 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f354806b4d0 0x7f354806d990 unknown :-1 s=CLOSED pgs=555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35681561f0 0x7f35680ce9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35680cc800 0x7f35680ce490 unknown :-1 s=CLOSED pgs=1285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 --2- 172.21.15.86:0/453548289 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35680aedc0 0x7f35681f3ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.128+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 >> 172.21.15.86:0/453548289 conn(0x7f356806afc0 msgr2=0x7f356806de70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.129+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 shutdown_connections 2024-03-06T07:33:17.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.129+0000 7f356e600640 1 -- 172.21.15.86:0/453548289 wait complete. 2024-03-06T07:33:17.148 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 16, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:33:03.630003+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '109', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 17, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:33:04.861178+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '110', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:33:17.149 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:33:17.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.347+0000 7fd3f5a00640 1 Processor -- start 2024-03-06T07:33:17.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.348+0000 7fd3f5a00640 1 -- start start 2024-03-06T07:33:17.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.349+0000 7fd3f5a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 0x7fd3f00d0570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.349+0000 7fd3f5a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3f00ce1e0 con 0x7fd3f00b43b0 2024-03-06T07:33:17.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.349+0000 7fd3f5a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3f00ce4c0 con 0x7fd3f01605b0 2024-03-06T07:33:17.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.349+0000 7fd3f5a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f00cedf0 con 0x7fd3f015c330 2024-03-06T07:33:17.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.349+0000 7fd3e6000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd3f00b43b0 0x7fd3f015fea0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36716/0 (socket says 172.21.15.86:36716) 2024-03-06T07:33:17.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.349+0000 7fd3e6000640 1 -- 172.21.15.86:0/1001097420 learned_addr learned my addr 172.21.15.86:0/1001097420 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:17.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3eea00640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 0x7fd3f00d0570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1860559594 0 0) 0x7fd3f00ce1e0 con 0x7fd3f00b43b0 2024-03-06T07:33:17.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd3d4003520 con 0x7fd3f00b43b0 2024-03-06T07:33:17.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3165672941 0 0) 0x7fd3d4003520 con 0x7fd3f00b43b0 2024-03-06T07:33:17.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 msgr2=0x7fd3f00d0570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 0x7fd3f00d0570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 >> v1:172.21.15.194:6789/0 conn(0x7fd3f01605b0 legacy=0x7fd3f00cc870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3f00ce7a0 con 0x7fd3f00b43b0 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fd3d8001670 con 0x7fd3f00b43b0 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 >> v1:172.21.15.86:6789/0 conn(0x7fd3f00b43b0 legacy=0x7fd3f015fea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 0x7fd3d40039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d4003fb0 0x7fd3d4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d4004d40 0x7fd3d40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f00cedf0 con 0x7fd3d4003fb0 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3d400bb00 con 0x7fd3d4004d40 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.350+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3d400bd00 con 0x7fd3f015c330 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3e6000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d4003fb0 0x7fd3d4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3eea00640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 0x7fd3d40039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3ef400640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d4004d40 0x7fd3d40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3e6000640 1 -- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 msgr2=0x7fd3d40039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:17.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3e6000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 0x7fd3d40039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3e6000640 1 -- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d4004d40 msgr2=0x7fd3d40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:17.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3e6000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d4004d40 0x7fd3d40051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3e6000640 1 -- 172.21.15.86:0/1001097420 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3d4003520 con 0x7fd3d4003fb0 2024-03-06T07:33:17.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3eea00640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f015c330 0x7fd3d40039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:17.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.351+0000 7fd3ef400640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d4004d40 0x7fd3d40051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:17.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.352+0000 7fd3e6000640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d4003fb0 0x7fd3d4004800 secure :-1 s=READY pgs=1287 cs=0 l=1 rev1=1 crypto rx=0x7fd3d800c110 tx=0x7fd3d4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=335cb8f64b7452bd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:17.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.352+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3d8007c40 con 0x7fd3d4003fb0 2024-03-06T07:33:17.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.352+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3d8030070 con 0x7fd3d4003fb0 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.352+0000 7fd3ee000640 1 -- 172.21.15.86:0/1001097420 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3d8005d70 con 0x7fd3d4003fb0 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.353+0000 7fd3f5a00640 1 -- 172.21.15.86:0/1001097420 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d4003fb0 msgr2=0x7fd3d4004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.353+0000 7fd3f5a00640 1 --2- 172.21.15.86:0/1001097420 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d4003fb0 0x7fd3d4004800 secure :-1 s=READY pgs=1287 cs=0 l=1 rev1=1 crypto rx=0x7fd3d800c110 tx=0x7fd3d4005830 comp rx=0 tx=0).stop 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.353+0000 7fd3ef400640 1 -- 172.21.15.86:0/1001097420 reap_dead start 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.353+0000 7fd3f5a00640 1 -- 172.21.15.86:0/1001097420 shutdown_connections 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.353+0000 7fd3f5a00640 1 -- 172.21.15.86:0/1001097420 >> 172.21.15.86:0/1001097420 conn(0x7fd3f00b3790 msgr2=0x7fd3f00b3ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.353+0000 7fd3f5a00640 1 -- 172.21.15.86:0/1001097420 shutdown_connections 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.353+0000 7fd3f5a00640 1 -- 172.21.15.86:0/1001097420 wait complete. 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.355+0000 7fd3f5a00640 1 Processor -- start 2024-03-06T07:33:17.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.355+0000 7fd3f5a00640 1 -- start start 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3f5a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d4003fb0 0x7fd3f0152fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3f5a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f006d0a0 0x7fd3f006d500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3f5a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f006fca0 0x7fd3f0070120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3f5a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f00cf280 con 0x7fd3f006fca0 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3f5a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f015d720 con 0x7fd3f006d0a0 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3f5a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f00b5930 con 0x7fd3d4003fb0 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3ef400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f006fca0 0x7fd3f0070120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3ef400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f006fca0 0x7fd3f0070120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60486/0 (socket says 172.21.15.86:60486) 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.356+0000 7fd3ef400640 1 -- 172.21.15.86:0/2263629374 learned_addr learned my addr 172.21.15.86:0/2263629374 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3e6000640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f006d0a0 0x7fd3f006d500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3eea00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d4003fb0 0x7fd3f0152fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3ef400640 1 -- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d4003fb0 msgr2=0x7fd3f0152fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3ef400640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d4003fb0 0x7fd3f0152fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3ef400640 1 -- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f006d0a0 msgr2=0x7fd3f006d500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3ef400640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f006d0a0 0x7fd3f006d500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3ef400640 1 -- 172.21.15.86:0/2263629374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3f006c2e0 con 0x7fd3f006fca0 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3e6000640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f006d0a0 0x7fd3f006d500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3ef400640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f006fca0 0x7fd3f0070120 secure :-1 s=READY pgs=1288 cs=0 l=1 rev1=1 crypto rx=0x7fd3e8000f80 tx=0x7fd3e800dc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3e7400640 1 -- 172.21.15.86:0/2263629374 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e8006d50 con 0x7fd3f006fca0 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3e7400640 1 -- 172.21.15.86:0/2263629374 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3e8003970 con 0x7fd3f006fca0 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3e7400640 1 -- 172.21.15.86:0/2263629374 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e8003d60 con 0x7fd3f006fca0 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.357+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3f006c5c0 con 0x7fd3f006fca0 2024-03-06T07:33:17.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.358+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3f006cb80 con 0x7fd3f006fca0 2024-03-06T07:33:17.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.359+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd3f006cd80 con 0x7fd3f006fca0 2024-03-06T07:33:17.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.360+0000 7fd3e7400640 1 -- 172.21.15.86:0/2263629374 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd3e801b020 con 0x7fd3f006fca0 2024-03-06T07:33:17.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.361+0000 7fd3e7400640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3cc06b4d0 0x7fd3cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.361+0000 7fd3e7400640 1 -- 172.21.15.86:0/2263629374 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd3e8089cb0 con 0x7fd3f006fca0 2024-03-06T07:33:17.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.362+0000 7fd3eea00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3cc06b4d0 0x7fd3cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.362+0000 7fd3eea00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3cc06b4d0 0x7fd3cc06d990 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7fd3e00099c0 tx=0x7fd3e0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:17.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.366+0000 7fd3e7400640 1 -- 172.21.15.86:0/2263629374 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd3f006cd80 con 0x7fd3f006fca0 2024-03-06T07:33:17.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.564+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd3f004f330 con 0x7fd3f006fca0 2024-03-06T07:33:17.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.565+0000 7fd3e7400640 1 -- 172.21.15.86:0/2263629374 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v110) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd3e8058b30 con 0x7fd3f006fca0 2024-03-06T07:33:17.567 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3cc06b4d0 msgr2=0x7fd3cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3cc06b4d0 0x7fd3cc06d990 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7fd3e00099c0 tx=0x7fd3e0008040 comp rx=0 tx=0).stop 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f006fca0 msgr2=0x7fd3f0070120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f006fca0 0x7fd3f0070120 secure :-1 s=READY pgs=1288 cs=0 l=1 rev1=1 crypto rx=0x7fd3e8000f80 tx=0x7fd3e800dc00 comp rx=0 tx=0).stop 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 shutdown_connections 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3cc06b4d0 0x7fd3cc06d990 secure :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7fd3e00099c0 tx=0x7fd3e0008040 comp rx=0 tx=0).stop 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f006fca0 0x7fd3f0070120 unknown :-1 s=CLOSED pgs=1288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f006d0a0 0x7fd3f006d500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 --2- 172.21.15.86:0/2263629374 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d4003fb0 0x7fd3f0152fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 >> 172.21.15.86:0/2263629374 conn(0x7fd3f00b3790 msgr2=0x7fd3f015e700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 shutdown_connections 2024-03-06T07:33:17.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.568+0000 7fd3f5a00640 1 -- 172.21.15.86:0/2263629374 wait complete. 2024-03-06T07:33:17.589 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:33:17.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.795+0000 7f5379a00640 1 Processor -- start 2024-03-06T07:33:17.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.795+0000 7f5379a00640 1 -- start start 2024-03-06T07:33:17.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.796+0000 7f5379a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f537415be10 0x7f53740ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.796+0000 7f5379a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53740aef20 0x7f53740ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.796+0000 7f5379a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f537415c1f0 con 0x7f537406d330 2024-03-06T07:33:17.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.797+0000 7f5379a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f53740cd200 con 0x7f537415be10 2024-03-06T07:33:17.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.797+0000 7f5379a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f537406cac0 con 0x7f53740aef20 2024-03-06T07:33:17.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.797+0000 7f5373e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f537415be10 0x7f53740ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.797+0000 7f5373e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f537415be10 0x7f53740ccc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:36612/0 (socket says 172.21.15.86:36612) 2024-03-06T07:33:17.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.797+0000 7f5379000640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f537406d330 0x7f5374071ab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:46896/0 (socket says 172.21.15.86:46896) 2024-03-06T07:33:17.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.797+0000 7f5373e00640 1 -- 172.21.15.86:0/43519631 learned_addr learned my addr 172.21.15.86:0/43519631 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:17.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.797+0000 7f536b400640 1 --2- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53740aef20 0x7f53740ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.798+0000 7f5373e00640 1 -- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53740aef20 msgr2=0x7f53740ce2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:17.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.798+0000 7f5373e00640 1 --2- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53740aef20 0x7f53740ce2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.798+0000 7f5373e00640 1 -- 172.21.15.86:0/43519631 >> v1:172.21.15.194:6789/0 conn(0x7f537406d330 legacy=0x7f5374071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.798+0000 7f5373e00640 1 -- 172.21.15.86:0/43519631 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f537406cda0 con 0x7f537415be10 2024-03-06T07:33:17.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.799+0000 7f5373e00640 1 --2- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f53740ccc20 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f5364009e50 tx=0x7f536402bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=628707dc140f86e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:17.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.800+0000 7f5373400640 1 -- 172.21.15.86:0/43519631 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f536402cb00 con 0x7f537415be10 2024-03-06T07:33:17.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.800+0000 7f5373400640 1 -- 172.21.15.86:0/43519631 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f536402cca0 con 0x7f537415be10 2024-03-06T07:33:17.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.800+0000 7f5373400640 1 -- 172.21.15.86:0/43519631 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5364034610 con 0x7f537415be10 2024-03-06T07:33:17.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.800+0000 7f5379a00640 1 -- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 msgr2=0x7f53740ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.800+0000 7f5379a00640 1 --2- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f53740ccc20 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f5364009e50 tx=0x7f536402bf60 comp rx=0 tx=0).stop 2024-03-06T07:33:17.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.801+0000 7f5379a00640 1 -- 172.21.15.86:0/43519631 shutdown_connections 2024-03-06T07:33:17.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.801+0000 7f5379a00640 1 --2- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f53740aef20 0x7f53740ce2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.801+0000 7f5379a00640 1 --2- 172.21.15.86:0/43519631 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f53740ccc20 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.801+0000 7f5379a00640 1 -- 172.21.15.86:0/43519631 >> 172.21.15.86:0/43519631 conn(0x7f537406afc0 msgr2=0x7f537406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:17.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.801+0000 7f5379a00640 1 -- 172.21.15.86:0/43519631 shutdown_connections 2024-03-06T07:33:17.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.801+0000 7f5379a00640 1 -- 172.21.15.86:0/43519631 wait complete. 2024-03-06T07:33:17.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.802+0000 7f5379a00640 1 Processor -- start 2024-03-06T07:33:17.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.803+0000 7f5379a00640 1 -- start start 2024-03-06T07:33:17.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.803+0000 7f5379a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f537406d330 0x7f53741522b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5379a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f53740aef20 0x7f5374159630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5379a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f5374159b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5379a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53740cda90 con 0x7f53740aef20 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5379a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f537406d080 con 0x7f537415be10 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5379a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f537406d7c0 con 0x7f537406d330 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f53740aef20 0x7f5374159630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f53740aef20 0x7f5374159630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60490/0 (socket says 172.21.15.86:60490) 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 -- 172.21.15.86:0/2154175334 learned_addr learned my addr 172.21.15.86:0/2154175334 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 -- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f537406d330 msgr2=0x7f53741522b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5373e00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f537406d330 0x7f53741522b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5379000640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f5374159b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f537406d330 0x7f53741522b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 -- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 msgr2=0x7f5374159b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f5374159b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f536b400640 1 -- 172.21.15.86:0/2154175334 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f537415a0b0 con 0x7f53740aef20 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.804+0000 7f5379000640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f5374159b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.805+0000 7f536b400640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f53740aef20 0x7f5374159630 secure :-1 s=READY pgs=1289 cs=0 l=1 rev1=1 crypto rx=0x7f535c00e9e0 tx=0x7f535c00eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.805+0000 7f5370c00640 1 -- 172.21.15.86:0/2154175334 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f535c017070 con 0x7f53740aef20 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.805+0000 7f5370c00640 1 -- 172.21.15.86:0/2154175334 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f535c00c9d0 con 0x7f53740aef20 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.805+0000 7f5370c00640 1 -- 172.21.15.86:0/2154175334 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f535c00cdc0 con 0x7f53740aef20 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.805+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5374154050 con 0x7f53740aef20 2024-03-06T07:33:17.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.805+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f53741542f0 con 0x7f53740aef20 2024-03-06T07:33:17.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.806+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f537404f330 con 0x7f53740aef20 2024-03-06T07:33:17.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.808+0000 7f5370c00640 1 -- 172.21.15.86:0/2154175334 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f535c01a070 con 0x7f53740aef20 2024-03-06T07:33:17.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.809+0000 7f5370c00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f535006b4d0 0x7f535006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:17.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.809+0000 7f5370c00640 1 -- 172.21.15.86:0/2154175334 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f535c08a2e0 con 0x7f53740aef20 2024-03-06T07:33:17.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.809+0000 7f5373e00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f535006b4d0 0x7f535006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:17.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.810+0000 7f5373e00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f535006b4d0 0x7f535006d990 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f5364035000 tx=0x7f5364030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:17.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:17.814+0000 7f5370c00640 1 -- 172.21.15.86:0/2154175334 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f535c0590c0 con 0x7f53740aef20 2024-03-06T07:33:18.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.010+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f53740af9f0 con 0x7f53740aef20 2024-03-06T07:33:18.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.011+0000 7f5370c00640 1 -- 172.21.15.86:0/2154175334 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v110) v1 ==== 104+0+11 (secure 0 0 0) 0x7f535c0589c0 con 0x7f53740aef20 2024-03-06T07:33:18.013 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:33:18.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f535006b4d0 msgr2=0x7f535006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f535006b4d0 0x7f535006d990 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f5364035000 tx=0x7f5364030000 comp rx=0 tx=0).stop 2024-03-06T07:33:18.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f53740aef20 msgr2=0x7f5374159630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f53740aef20 0x7f5374159630 secure :-1 s=READY pgs=1289 cs=0 l=1 rev1=1 crypto rx=0x7f535c00e9e0 tx=0x7f535c00eea0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 shutdown_connections 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f535006b4d0 0x7f535006d990 unknown :-1 s=CLOSED pgs=557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f537415be10 0x7f5374159b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f53740aef20 0x7f5374159630 unknown :-1 s=CLOSED pgs=1289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 --2- 172.21.15.86:0/2154175334 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f537406d330 0x7f53741522b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.013+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 >> 172.21.15.86:0/2154175334 conn(0x7f537406afc0 msgr2=0x7f537406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.014+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 shutdown_connections 2024-03-06T07:33:18.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.014+0000 7f5379a00640 1 -- 172.21.15.86:0/2154175334 wait complete. 2024-03-06T07:33:18.034 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:33:18.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.242+0000 7ff12a800640 1 Processor -- start 2024-03-06T07:33:18.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.242+0000 7ff12a800640 1 -- start start 2024-03-06T07:33:18.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.244+0000 7ff12a800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff1240aef20 0x7ff1240ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.244+0000 7ff12a800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff12415c1f0 con 0x7ff12415be10 2024-03-06T07:33:18.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.244+0000 7ff12a800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff12406cac0 con 0x7ff12406da40 2024-03-06T07:33:18.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.244+0000 7ff12a800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff1240cd200 con 0x7ff1240aef20 2024-03-06T07:33:18.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.244+0000 7ff129e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff1240aef20 0x7ff1240ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.244+0000 7ff129400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff12415be10 0x7ff12406d330 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36720/0 (socket says 172.21.15.86:36720) 2024-03-06T07:33:18.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.244+0000 7ff129400640 1 -- 172.21.15.86:0/3265735760 learned_addr learned my addr 172.21.15.86:0/3265735760 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:18.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.245+0000 7ff129e00640 1 -- 172.21.15.86:0/3265735760 >> v1:172.21.15.194:6789/0 conn(0x7ff12406da40 legacy=0x7ff124071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:18.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.245+0000 7ff129e00640 1 -- 172.21.15.86:0/3265735760 >> v1:172.21.15.86:6789/0 conn(0x7ff12415be10 legacy=0x7ff12406d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.245+0000 7ff123e00640 1 -- 172.21.15.86:0/3265735760 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3677552481 0 0) 0x7ff12415c1f0 con 0x7ff12415be10 2024-03-06T07:33:18.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.245+0000 7ff129e00640 1 -- 172.21.15.86:0/3265735760 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff12406cda0 con 0x7ff1240aef20 2024-03-06T07:33:18.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.246+0000 7ff129e00640 1 --2- 172.21.15.86:0/3265735760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1240ccc20 secure :-1 s=READY pgs=1291 cs=0 l=1 rev1=1 crypto rx=0x7ff11c0097a0 tx=0x7ff11c02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e0ecb78426595102 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:18.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.246+0000 7ff123e00640 1 -- 172.21.15.86:0/3265735760 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff11c0315a0 con 0x7ff1240aef20 2024-03-06T07:33:18.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.246+0000 7ff123e00640 1 -- 172.21.15.86:0/3265735760 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff11c037070 con 0x7ff1240aef20 2024-03-06T07:33:18.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.246+0000 7ff123e00640 1 -- 172.21.15.86:0/3265735760 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff11c031960 con 0x7ff1240aef20 2024-03-06T07:33:18.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.247+0000 7ff12a800640 1 -- 172.21.15.86:0/3265735760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 msgr2=0x7ff1240ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.247+0000 7ff12a800640 1 --2- 172.21.15.86:0/3265735760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1240ccc20 secure :-1 s=READY pgs=1291 cs=0 l=1 rev1=1 crypto rx=0x7ff11c0097a0 tx=0x7ff11c02b840 comp rx=0 tx=0).stop 2024-03-06T07:33:18.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.247+0000 7ff12a800640 1 -- 172.21.15.86:0/3265735760 shutdown_connections 2024-03-06T07:33:18.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.247+0000 7ff12a800640 1 --2- 172.21.15.86:0/3265735760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1240ccc20 unknown :-1 s=CLOSED pgs=1291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.247+0000 7ff12a800640 1 -- 172.21.15.86:0/3265735760 >> 172.21.15.86:0/3265735760 conn(0x7ff12406afc0 msgr2=0x7ff12406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:18.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.247+0000 7ff12a800640 1 -- 172.21.15.86:0/3265735760 shutdown_connections 2024-03-06T07:33:18.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.247+0000 7ff12a800640 1 -- 172.21.15.86:0/3265735760 wait complete. 2024-03-06T07:33:18.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.249+0000 7ff12a800640 1 Processor -- start 2024-03-06T07:33:18.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.249+0000 7ff12a800640 1 -- start start 2024-03-06T07:33:18.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff12a800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff12406da40 0x7ff1241f1160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff12a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1241f46b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff12a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff12415be10 0x7ff1240b39c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff12a800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1240cda90 con 0x7ff1240aef20 2024-03-06T07:33:18.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff12a800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff12406d080 con 0x7ff12406da40 2024-03-06T07:33:18.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff12a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff1240ce2b0 con 0x7ff12415be10 2024-03-06T07:33:18.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff128a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1241f46b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff128a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1241f46b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60502/0 (socket says 172.21.15.86:60502) 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.250+0000 7ff128a00640 1 -- 172.21.15.86:0/4019715165 learned_addr learned my addr 172.21.15.86:0/4019715165 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff129e00640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff12415be10 0x7ff1240b39c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff129400640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff12406da40 0x7ff1241f1160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff128a00640 1 -- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff12415be10 msgr2=0x7ff1240b39c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff128a00640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff12415be10 0x7ff1240b39c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff128a00640 1 -- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff12406da40 msgr2=0x7ff1241f1160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff128a00640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff12406da40 0x7ff1241f1160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff128a00640 1 -- 172.21.15.86:0/4019715165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1241f4bf0 con 0x7ff1240aef20 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff129400640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff12406da40 0x7ff1241f1160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff128a00640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1241f46b0 secure :-1 s=READY pgs=1292 cs=0 l=1 rev1=1 crypto rx=0x7ff11000c430 tx=0x7ff11000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:18.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff121600640 1 -- 172.21.15.86:0/4019715165 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff110011790 con 0x7ff1240aef20 2024-03-06T07:33:18.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff121600640 1 -- 172.21.15.86:0/4019715165 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff110017070 con 0x7ff1240aef20 2024-03-06T07:33:18.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff124156400 con 0x7ff1240aef20 2024-03-06T07:33:18.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff121600640 1 -- 172.21.15.86:0/4019715165 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff110011bb0 con 0x7ff1240aef20 2024-03-06T07:33:18.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.251+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1241569c0 con 0x7ff1240aef20 2024-03-06T07:33:18.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.252+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff12404f330 con 0x7ff1240aef20 2024-03-06T07:33:18.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.253+0000 7ff121600640 1 -- 172.21.15.86:0/4019715165 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff11001c460 con 0x7ff1240aef20 2024-03-06T07:33:18.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.254+0000 7ff121600640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff10406b500 0x7ff10406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.254+0000 7ff121600640 1 -- 172.21.15.86:0/4019715165 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff11008a120 con 0x7ff1240aef20 2024-03-06T07:33:18.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.255+0000 7ff129400640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff10406b500 0x7ff10406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.255+0000 7ff129400640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff10406b500 0x7ff10406d9c0 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7ff114007fd0 tx=0x7ff114009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:18.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.259+0000 7ff121600640 1 -- 172.21.15.86:0/4019715165 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff1241f4bf0 con 0x7ff1240aef20 2024-03-06T07:33:18.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.455+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7ff12406b5b0 con 0x7ff1240aef20 2024-03-06T07:33:18.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.456+0000 7ff121600640 1 -- 172.21.15.86:0/4019715165 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v110) v1 ==== 100+0+11 (secure 0 0 0) 0x7ff110058f00 con 0x7ff1240aef20 2024-03-06T07:33:18.458 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:33:18.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff10406b500 msgr2=0x7ff10406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff10406b500 0x7ff10406d9c0 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7ff114007fd0 tx=0x7ff114009040 comp rx=0 tx=0).stop 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 msgr2=0x7ff1241f46b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1241f46b0 secure :-1 s=READY pgs=1292 cs=0 l=1 rev1=1 crypto rx=0x7ff11000c430 tx=0x7ff11000c8f0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 shutdown_connections 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff10406b500 0x7ff10406d9c0 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff12415be10 0x7ff1240b39c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1240aef20 0x7ff1241f46b0 unknown :-1 s=CLOSED pgs=1292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 --2- 172.21.15.86:0/4019715165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff12406da40 0x7ff1241f1160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.458+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 >> 172.21.15.86:0/4019715165 conn(0x7ff12406afc0 msgr2=0x7ff1240cf360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:18.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.459+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 shutdown_connections 2024-03-06T07:33:18.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.459+0000 7ff12a800640 1 -- 172.21.15.86:0/4019715165 wait complete. 2024-03-06T07:33:18.479 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:33:18.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.673+0000 7ff495000640 1 Processor -- start 2024-03-06T07:33:18.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.673+0000 7ff495000640 1 -- start start 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff495000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff4900c4930 0x7ff4900c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff495000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff4900c4d10 0x7ff4900c5170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff495000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c56b0 0x7ff49006d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff495000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff490145b30 con 0x7ff4900c4930 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff495000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff490142ce0 con 0x7ff4900c4d10 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff495000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4900cbca0 con 0x7ff4900c56b0 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff48de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff4900c4930 0x7ff4900c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.674+0000 7ff48de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff4900c4930 0x7ff4900c6460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60516/0 (socket says 172.21.15.86:60516) 2024-03-06T07:33:18.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff48de00640 1 -- 172.21.15.86:0/1317836153 learned_addr learned my addr 172.21.15.86:0/1317836153 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:18.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff485400640 1 --2- 172.21.15.86:0/1317836153 >> v2:172.21.15.194:3300/0 conn(0x7ff4900c4d10 0x7ff4900c5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff48e800640 1 --2- 172.21.15.86:0/1317836153 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c56b0 0x7ff49006d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff48de00640 1 -- 172.21.15.86:0/1317836153 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c56b0 msgr2=0x7ff49006d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:18.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff48de00640 1 --2- 172.21.15.86:0/1317836153 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c56b0 0x7ff49006d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff48de00640 1 -- 172.21.15.86:0/1317836153 >> v2:172.21.15.194:3300/0 conn(0x7ff4900c4d10 msgr2=0x7ff4900c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:18.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff48de00640 1 --2- 172.21.15.86:0/1317836153 >> v2:172.21.15.194:3300/0 conn(0x7ff4900c4d10 0x7ff4900c5170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.675+0000 7ff48de00640 1 -- 172.21.15.86:0/1317836153 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4900cbf80 con 0x7ff4900c4930 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.676+0000 7ff48de00640 1 --2- 172.21.15.86:0/1317836153 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4900c6460 secure :-1 s=READY pgs=1293 cs=0 l=1 rev1=1 crypto rx=0x7ff490068c90 tx=0x7ff48002b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d91b4fe83bcc81e5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff48d400640 1 -- 172.21.15.86:0/1317836153 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff48002e070 con 0x7ff4900c4930 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff48d400640 1 -- 172.21.15.86:0/1317836153 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff480037070 con 0x7ff4900c4930 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff48d400640 1 -- 172.21.15.86:0/1317836153 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff480031a00 con 0x7ff4900c4930 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 -- 172.21.15.86:0/1317836153 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 msgr2=0x7ff4900c6460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 --2- 172.21.15.86:0/1317836153 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4900c6460 secure :-1 s=READY pgs=1293 cs=0 l=1 rev1=1 crypto rx=0x7ff490068c90 tx=0x7ff48002b880 comp rx=0 tx=0).stop 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 -- 172.21.15.86:0/1317836153 shutdown_connections 2024-03-06T07:33:18.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 --2- 172.21.15.86:0/1317836153 >> v2:172.21.15.194:3300/0 conn(0x7ff4900c4d10 0x7ff4900c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 --2- 172.21.15.86:0/1317836153 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c56b0 0x7ff49006d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 --2- 172.21.15.86:0/1317836153 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4900c6460 unknown :-1 s=CLOSED pgs=1293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 -- 172.21.15.86:0/1317836153 >> 172.21.15.86:0/1317836153 conn(0x7ff4900cc4f0 msgr2=0x7ff4900cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:18.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 -- 172.21.15.86:0/1317836153 shutdown_connections 2024-03-06T07:33:18.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.677+0000 7ff495000640 1 -- 172.21.15.86:0/1317836153 wait complete. 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.679+0000 7ff495000640 1 Processor -- start 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.679+0000 7ff495000640 1 -- start start 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.680+0000 7ff495000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4901e2400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.680+0000 7ff495000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c4d10 0x7ff4900ae4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.680+0000 7ff495000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4900c56b0 0x7ff49013e030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.680+0000 7ff495000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4900cc260 con 0x7ff4900c4930 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.680+0000 7ff495000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4900ca3b0 con 0x7ff4900c56b0 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.680+0000 7ff495000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4900c5b40 con 0x7ff4900c4d10 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.680+0000 7ff48de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4901e2400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4901e2400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60530/0 (socket says 172.21.15.86:60530) 2024-03-06T07:33:18.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 -- 172.21.15.86:0/402395934 learned_addr learned my addr 172.21.15.86:0/402395934 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff485400640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c4d10 0x7ff4900ae4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48e800640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4900c56b0 0x7ff49013e030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 -- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c4d10 msgr2=0x7ff4900ae4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c4d10 0x7ff4900ae4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 -- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4900c56b0 msgr2=0x7ff49013e030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4900c56b0 0x7ff49013e030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 -- 172.21.15.86:0/402395934 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4901420c0 con 0x7ff4900c4930 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48e800640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4900c56b0 0x7ff49013e030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:18.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.681+0000 7ff48de00640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4901e2400 secure :-1 s=READY pgs=1294 cs=0 l=1 rev1=1 crypto rx=0x7ff480000c00 tx=0x7ff4800046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:18.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.682+0000 7ff486a00640 1 -- 172.21.15.86:0/402395934 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff480032670 con 0x7ff4900c4930 2024-03-06T07:33:18.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.682+0000 7ff486a00640 1 -- 172.21.15.86:0/402395934 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff480032850 con 0x7ff4900c4930 2024-03-06T07:33:18.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.682+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4900ada00 con 0x7ff4900c4930 2024-03-06T07:33:18.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.682+0000 7ff486a00640 1 -- 172.21.15.86:0/402395934 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff480032c40 con 0x7ff4900c4930 2024-03-06T07:33:18.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.682+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff49013e570 con 0x7ff4900c4930 2024-03-06T07:33:18.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.683+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4900c9cf0 con 0x7ff4900c4930 2024-03-06T07:33:18.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.684+0000 7ff486a00640 1 -- 172.21.15.86:0/402395934 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff4800453f0 con 0x7ff4900c4930 2024-03-06T07:33:18.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.685+0000 7ff486a00640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff46c06b4d0 0x7ff46c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:18.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.685+0000 7ff486a00640 1 -- 172.21.15.86:0/402395934 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff4800abf70 con 0x7ff4900c4930 2024-03-06T07:33:18.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.686+0000 7ff485400640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff46c06b4d0 0x7ff46c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:18.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.686+0000 7ff485400640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff46c06b4d0 0x7ff46c06d990 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7ff4901f7390 tx=0x7ff478008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:18.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.690+0000 7ff486a00640 1 -- 172.21.15.86:0/402395934 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff4900c9cf0 con 0x7ff4900c4930 2024-03-06T07:33:18.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.933+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7ff49004f330 con 0x7ff4900c4930 2024-03-06T07:33:18.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.933+0000 7ff486a00640 1 -- 172.21.15.86:0/402395934 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v110) v1 ==== 80+0+0 (secure 0 0 0) 0x7ff480040070 con 0x7ff4900c4930 2024-03-06T07:33:18.935 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff46c06b4d0 msgr2=0x7ff46c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff46c06b4d0 0x7ff46c06d990 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7ff4901f7390 tx=0x7ff478008040 comp rx=0 tx=0).stop 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 msgr2=0x7ff4901e2400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4901e2400 secure :-1 s=READY pgs=1294 cs=0 l=1 rev1=1 crypto rx=0x7ff480000c00 tx=0x7ff4800046c0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 shutdown_connections 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff46c06b4d0 0x7ff46c06d990 unknown :-1 s=CLOSED pgs=559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff4900c56b0 0x7ff49013e030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff4900c4d10 0x7ff4900ae4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 --2- 172.21.15.86:0/402395934 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff4900c4930 0x7ff4901e2400 unknown :-1 s=CLOSED pgs=1294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:18.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 >> 172.21.15.86:0/402395934 conn(0x7ff4900cc4f0 msgr2=0x7ff490070a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:18.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 shutdown_connections 2024-03-06T07:33:18.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:18.936+0000 7ff495000640 1 -- 172.21.15.86:0/402395934 wait complete. 2024-03-06T07:33:18.955 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:33:18.955 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:18.955 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:19.076 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:33:19.076 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:19.076 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:33:19.138 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:19.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.317+0000 7fbb79400640 1 Processor -- start 2024-03-06T07:33:19.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.317+0000 7fbb79400640 1 -- start start 2024-03-06T07:33:19.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.318+0000 7fbb79400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbb7415acb0 0x7fbb740b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:19.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb79400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbb7415b090 con 0x7fbb740b4680 2024-03-06T07:33:19.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb79400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbb740b2a30 con 0x7fbb7406ec00 2024-03-06T07:33:19.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb79400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbb740b4360 con 0x7fbb7415acb0 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb72800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbb7415acb0 0x7fbb740b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb72800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbb7415acb0 0x7fbb740b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60544/0 (socket says 172.21.15.86:60544) 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb72800640 1 -- 172.21.15.86:0/1337413000 learned_addr learned my addr 172.21.15.86:0/1337413000 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb69e00640 1 --1- 172.21.15.86:0/1337413000 >> v1:172.21.15.86:6789/0 conn(0x7fbb740b4680 0x7fbb7406e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36730/0 (socket says 172.21.15.86:36730) 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb72800640 1 -- 172.21.15.86:0/1337413000 >> v1:172.21.15.194:6789/0 conn(0x7fbb7406ec00 legacy=0x7fbb740ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb72800640 1 -- 172.21.15.86:0/1337413000 >> v1:172.21.15.86:6789/0 conn(0x7fbb740b4680 legacy=0x7fbb7406e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb72800640 1 -- 172.21.15.86:0/1337413000 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb740b2cf0 con 0x7fbb7415acb0 2024-03-06T07:33:19.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.319+0000 7fbb71e00640 1 -- 172.21.15.86:0/1337413000 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2289606947 0 0) 0x7fbb7415b090 con 0x7fbb740b4680 2024-03-06T07:33:19.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.321+0000 7fbb72800640 1 --2- 172.21.15.86:0/1337413000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb7415acb0 0x7fbb740b37d0 secure :-1 s=READY pgs=1296 cs=0 l=1 rev1=1 crypto rx=0x7fbb6400bb10 tx=0x7fbb6402e170 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a4082f131a7709f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:19.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.321+0000 7fbb71e00640 1 -- 172.21.15.86:0/1337413000 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb6403b070 con 0x7fbb7415acb0 2024-03-06T07:33:19.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.321+0000 7fbb71e00640 1 -- 172.21.15.86:0/1337413000 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbb64036440 con 0x7fbb7415acb0 2024-03-06T07:33:19.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.321+0000 7fbb71e00640 1 -- 172.21.15.86:0/1337413000 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb64036810 con 0x7fbb7415acb0 2024-03-06T07:33:19.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.321+0000 7fbb79400640 1 -- 172.21.15.86:0/1337413000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb7415acb0 msgr2=0x7fbb740b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:19.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.321+0000 7fbb79400640 1 --2- 172.21.15.86:0/1337413000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb7415acb0 0x7fbb740b37d0 secure :-1 s=READY pgs=1296 cs=0 l=1 rev1=1 crypto rx=0x7fbb6400bb10 tx=0x7fbb6402e170 comp rx=0 tx=0).stop 2024-03-06T07:33:19.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.322+0000 7fbb79400640 1 -- 172.21.15.86:0/1337413000 shutdown_connections 2024-03-06T07:33:19.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.322+0000 7fbb79400640 1 --2- 172.21.15.86:0/1337413000 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb7415acb0 0x7fbb740b37d0 unknown :-1 s=CLOSED pgs=1296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.322+0000 7fbb79400640 1 -- 172.21.15.86:0/1337413000 >> 172.21.15.86:0/1337413000 conn(0x7fbb740b1730 msgr2=0x7fbb740b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:19.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.322+0000 7fbb79400640 1 -- 172.21.15.86:0/1337413000 shutdown_connections 2024-03-06T07:33:19.325 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.322+0000 7fbb79400640 1 -- 172.21.15.86:0/1337413000 wait complete. 2024-03-06T07:33:19.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.324+0000 7fbb79400640 1 Processor -- start 2024-03-06T07:33:19.327 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.324+0000 7fbb79400640 1 -- start start 2024-03-06T07:33:19.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb79400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbb7406ec00 0x7fbb74143ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:19.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb79400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb740b4680 0x7fbb74159d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:19.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb79400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbb7415acb0 0x7fbb7415d180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:19.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb79400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb740b3d10 con 0x7fbb740b4680 2024-03-06T07:33:19.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb79400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb740b32b0 con 0x7fbb7406ec00 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb79400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbb740b4010 con 0x7fbb7415acb0 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb69e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb740b4680 0x7fbb74159d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb69e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb740b4680 0x7fbb74159d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60560/0 (socket says 172.21.15.86:60560) 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.326+0000 7fbb69e00640 1 -- 172.21.15.86:0/2564850237 learned_addr learned my addr 172.21.15.86:0/2564850237 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb72800640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbb7406ec00 0x7fbb74143ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb69e00640 1 -- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbb7415acb0 msgr2=0x7fbb7415d180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb69e00640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbb7415acb0 0x7fbb7415d180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb69e00640 1 -- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbb7406ec00 msgr2=0x7fbb74143ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb69e00640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbb7406ec00 0x7fbb74143ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb69e00640 1 -- 172.21.15.86:0/2564850237 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb741440e0 con 0x7fbb740b4680 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb72800640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbb7406ec00 0x7fbb74143ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:19.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb69e00640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb740b4680 0x7fbb74159d90 secure :-1 s=READY pgs=1297 cs=0 l=1 rev1=1 crypto rx=0x7fbb5c00c500 tx=0x7fbb5c00c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:19.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb6b400640 1 -- 172.21.15.86:0/2564850237 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb5c018070 con 0x7fbb740b4680 2024-03-06T07:33:19.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb6b400640 1 -- 172.21.15.86:0/2564850237 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbb5c007580 con 0x7fbb740b4680 2024-03-06T07:33:19.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb74158cf0 con 0x7fbb740b4680 2024-03-06T07:33:19.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb6b400640 1 -- 172.21.15.86:0/2564850237 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb5c004a20 con 0x7fbb740b4680 2024-03-06T07:33:19.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.327+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb74159590 con 0x7fbb740b4680 2024-03-06T07:33:19.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.329+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbb7404f330 con 0x7fbb740b4680 2024-03-06T07:33:19.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.330+0000 7fbb6b400640 1 -- 172.21.15.86:0/2564850237 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbb5c01b070 con 0x7fbb740b4680 2024-03-06T07:33:19.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.331+0000 7fbb6b400640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbb5006b4d0 0x7fbb5006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:19.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.331+0000 7fbb6b400640 1 -- 172.21.15.86:0/2564850237 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbb5c0894f0 con 0x7fbb740b4680 2024-03-06T07:33:19.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.332+0000 7fbb72800640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbb5006b4d0 0x7fbb5006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:19.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.332+0000 7fbb72800640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbb5006b4d0 0x7fbb5006d990 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7fbb64038000 tx=0x7fbb64008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:19.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.335+0000 7fbb6b400640 1 -- 172.21.15.86:0/2564850237 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbb5c058360 con 0x7fbb740b4680 2024-03-06T07:33:19.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.558+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbb7406ce60 con 0x7fbb740b4680 2024-03-06T07:33:19.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.559+0000 7fbb6b400640 1 -- 172.21.15.86:0/2564850237 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 77 v77) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fbb5c057c60 con 0x7fbb740b4680 2024-03-06T07:33:19.562 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:19.562 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":77,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6043,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/906873619","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":906873619},{"type":"v1","addr":"172.21.15.194:6833","nonce":906873619}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":6046,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/398024320","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":398024320},{"type":"v1","addr":"172.21.15.194:6835","nonce":398024320}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":7464,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/1880188173","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":1880188173},{"type":"v1","addr":"172.21.15.86:6835","nonce":1880188173}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":77,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:09.045845+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7467},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7467":{"gid":7467,"name":"c","rank":0,"incarnation":73,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/221976790","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":221976790},{"type":"v1","addr":"172.21.15.86:6837","nonce":221976790}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7467,"qdb_cluster":[7467]},"id":8}]} 2024-03-06T07:33:19.562 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 77 2024-03-06T07:33:19.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.561+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbb5006b4d0 msgr2=0x7fbb5006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:19.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.561+0000 7fbb79400640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbb5006b4d0 0x7fbb5006d990 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7fbb64038000 tx=0x7fbb64008040 comp rx=0 tx=0).stop 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.561+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb740b4680 msgr2=0x7fbb74159d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.561+0000 7fbb79400640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb740b4680 0x7fbb74159d90 secure :-1 s=READY pgs=1297 cs=0 l=1 rev1=1 crypto rx=0x7fbb5c00c500 tx=0x7fbb5c00c9c0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 shutdown_connections 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbb5006b4d0 0x7fbb5006d990 unknown :-1 s=CLOSED pgs=560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbb7415acb0 0x7fbb7415d180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbb740b4680 0x7fbb74159d90 unknown :-1 s=CLOSED pgs=1297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 --2- 172.21.15.86:0/2564850237 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbb7406ec00 0x7fbb74143ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 >> 172.21.15.86:0/2564850237 conn(0x7fbb740b1730 msgr2=0x7fbb740cdbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 shutdown_connections 2024-03-06T07:33:19.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:19.562+0000 7fbb79400640 1 -- 172.21.15.86:0/2564850237 wait complete. 2024-03-06T07:33:19.583 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:33:19.804 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:33:19.805 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:33:19.805 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:33:19.805 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:33:19.806 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:33:19.806 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:33:19.806 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:33:19.806 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:33:19.806 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:33:19.807 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:33:19.822 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:33:19.822 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail a 2024-03-06T07:33:19.825 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:33:19.825 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail c 2024-03-06T07:33:19.839 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:33:19.839 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail b 2024-03-06T07:33:19.863 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:33:19.863 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail d 2024-03-06T07:33:20.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.048+0000 7fd3cfe00640 1 Processor -- start 2024-03-06T07:33:20.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.048+0000 7fd3cfe00640 1 -- start start 2024-03-06T07:33:20.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3d0161580 0x7fd3d00d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cfe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd3d00d2a70 0x7fd3d00b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cfe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3d0161960 con 0x7fd3d00d1e90 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cfe00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3d00cf6f0 con 0x7fd3d0161580 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cfe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3d00cf9d0 con 0x7fd3d00d2a70 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3d0161580 0x7fd3d00d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3d0161580 0x7fd3d00d1950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60566/0 (socket says 172.21.15.86:60566) 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.049+0000 7fd3cea00640 1 -- 172.21.15.86:0/707090182 learned_addr learned my addr 172.21.15.86:0/707090182 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.050+0000 7fd3cf400640 1 --2- 172.21.15.86:0/707090182 >> v2:172.21.15.194:3300/0 conn(0x7fd3d00d2a70 0x7fd3d00b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.050+0000 7fd3cea00640 1 -- 172.21.15.86:0/707090182 >> v2:172.21.15.194:3300/0 conn(0x7fd3d00d2a70 msgr2=0x7fd3d00b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.050+0000 7fd3cea00640 1 --2- 172.21.15.86:0/707090182 >> v2:172.21.15.194:3300/0 conn(0x7fd3d00d2a70 0x7fd3d00b3ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.050+0000 7fd3cea00640 1 -- 172.21.15.86:0/707090182 >> v1:172.21.15.194:6789/0 conn(0x7fd3d00d1e90 legacy=0x7fd3d00d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.050+0000 7fd3cea00640 1 -- 172.21.15.86:0/707090182 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3d00cfcb0 con 0x7fd3d0161580 2024-03-06T07:33:20.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.050+0000 7f1de4e00640 1 Processor -- start 2024-03-06T07:33:20.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.050+0000 7f1de4e00640 1 -- start start 2024-03-06T07:33:20.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.051+0000 7fd3cea00640 1 --2- 172.21.15.86:0/707090182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00d1950 secure :-1 s=READY pgs=1298 cs=0 l=1 rev1=1 crypto rx=0x7fd3d008d180 tx=0x7fd3c002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e4fdaab3ffb693b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.051+0000 7fd3ce000640 1 -- 172.21.15.86:0/707090182 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3c0035020 con 0x7fd3d0161580 2024-03-06T07:33:20.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.051+0000 7fd3ce000640 1 -- 172.21.15.86:0/707090182 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3c0031870 con 0x7fd3d0161580 2024-03-06T07:33:20.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.051+0000 7fd3ce000640 1 -- 172.21.15.86:0/707090182 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3c0031c40 con 0x7fd3d0161580 2024-03-06T07:33:20.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f1de4e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1de015e360 0x7f1de00cec10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f1de4e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de00d12e0 0x7f1de0161d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f1de4e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1de00cf2b0 con 0x7f1de0162270 2024-03-06T07:33:20.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f1de4e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1de00cf590 con 0x7f1de015e360 2024-03-06T07:33:20.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f1de4e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1de00cf870 con 0x7f1de00d12e0 2024-03-06T07:33:20.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7fd3cfe00640 1 -- 172.21.15.86:0/707090182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 msgr2=0x7fd3d00d1950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/707090182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00d1950 secure :-1 s=READY pgs=1298 cs=0 l=1 rev1=1 crypto rx=0x7fd3d008d180 tx=0x7fd3c002b910 comp rx=0 tx=0).stop 2024-03-06T07:33:20.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f0b4da00640 1 Processor -- start 2024-03-06T07:33:20.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f1ddd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de00d12e0 0x7f1de0161d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7fd3cfe00640 1 -- 172.21.15.86:0/707090182 shutdown_connections 2024-03-06T07:33:20.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/707090182 >> v2:172.21.15.194:3300/0 conn(0x7fd3d00d2a70 0x7fd3d00b3ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/707090182 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00d1950 unknown :-1 s=CLOSED pgs=1298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7fd3cfe00640 1 -- 172.21.15.86:0/707090182 >> 172.21.15.86:0/707090182 conn(0x7fd3d00d3d80 msgr2=0x7fd3d00d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.052+0000 7f1ddde00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1de015e360 0x7f1de00cec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7f1ddd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de00d12e0 0x7f1de0161d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:58118/0 (socket says 172.21.15.86:58118) 2024-03-06T07:33:20.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7f1ddd400640 1 -- 172.21.15.86:0/2527477379 learned_addr learned my addr 172.21.15.86:0/2527477379 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7fd3cfe00640 1 -- 172.21.15.86:0/707090182 shutdown_connections 2024-03-06T07:33:20.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7fd3cfe00640 1 -- 172.21.15.86:0/707090182 wait complete. 2024-03-06T07:33:20.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7f0b4da00640 1 -- start start 2024-03-06T07:33:20.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7fa631000640 1 Processor -- start 2024-03-06T07:33:20.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7f1ddd400640 1 -- 172.21.15.86:0/2527477379 >> v2:172.21.15.194:3300/0 conn(0x7f1de015e360 msgr2=0x7f1de00cec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7f1ddd400640 1 --2- 172.21.15.86:0/2527477379 >> v2:172.21.15.194:3300/0 conn(0x7f1de015e360 0x7f1de00cec10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7f1ddd400640 1 -- 172.21.15.86:0/2527477379 >> v1:172.21.15.194:6789/0 conn(0x7f1de0162270 legacy=0x7f1de015e740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7f1ddd400640 1 -- 172.21.15.86:0/2527477379 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1de00cfb50 con 0x7f1de00d12e0 2024-03-06T07:33:20.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.053+0000 7fa631000640 1 -- start start 2024-03-06T07:33:20.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.054+0000 7f0b4da00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0b480b6d90 0x7f0b480d0810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.054+0000 7f0b4da00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0b480d0d50 0x7f0b480d2160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.054+0000 7f0b4da00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0b480d1160 con 0x7f0b481617d0 2024-03-06T07:33:20.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.054+0000 7f0b4da00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0b480cff70 con 0x7f0b480d0d50 2024-03-06T07:33:20.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.054+0000 7f0b4da00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0b480cf6d0 con 0x7f0b480b6d90 2024-03-06T07:33:20.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.054+0000 7f1ddde00640 1 --2- 172.21.15.86:0/2527477379 >> v2:172.21.15.194:3300/0 conn(0x7f1de015e360 0x7f1de00cec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:33:20.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.054+0000 7fa631000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c0d0c00 0x7fa62c0d4850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa631000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa62c0b5240 con 0x7fa62c0b2e50 2024-03-06T07:33:20.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa631000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa62c0cf4d0 con 0x7fa62c15e200 2024-03-06T07:33:20.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa631000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa62c0d2ec0 con 0x7fa62c0d0c00 2024-03-06T07:33:20.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fd3cfe00640 1 Processor -- start 2024-03-06T07:33:20.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa62be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa62c0b2e50 0x7fa62c0b3640 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36742/0 (socket says 172.21.15.86:36742) 2024-03-06T07:33:20.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa62be00640 1 -- 172.21.15.86:0/3926453063 learned_addr learned my addr 172.21.15.86:0/3926453063 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:20.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b47400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0b480d0d50 0x7f0b480d2160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b46a00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f0b481617d0 0x7f0b480ceb50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:46956/0 (socket says 172.21.15.86:46956) 2024-03-06T07:33:20.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b46a00640 1 -- 172.21.15.86:0/2542522716 learned_addr learned my addr 172.21.15.86:0/2542522716 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:20.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b46000640 1 --2- 172.21.15.86:0/2542522716 >> v2:172.21.15.194:3300/0 conn(0x7f0b480b6d90 0x7f0b480d0810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fd3cfe00640 1 -- start start 2024-03-06T07:33:20.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa623e00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c0d0c00 0x7fa62c0d4850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 737584706 0 0) 0x7fa62c0b5240 con 0x7fa62c0b2e50 2024-03-06T07:33:20.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa60c003520 con 0x7fa62c0b2e50 2024-03-06T07:33:20.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b47400640 1 -- 172.21.15.86:0/2542522716 >> v2:172.21.15.194:3300/0 conn(0x7f0b480b6d90 msgr2=0x7f0b480d0810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b47400640 1 --2- 172.21.15.86:0/2542522716 >> v2:172.21.15.194:3300/0 conn(0x7f0b480b6d90 0x7f0b480d0810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b47400640 1 -- 172.21.15.86:0/2542522716 >> v1:172.21.15.194:6789/0 conn(0x7f0b481617d0 legacy=0x7f0b480ceb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.055+0000 7f0b47400640 1 -- 172.21.15.86:0/2542522716 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b480cf9b0 con 0x7f0b480d0d50 2024-03-06T07:33:20.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7f1ddd400640 1 --2- 172.21.15.86:0/2527477379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de00d12e0 0x7f1de0161d30 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f1dd0009870 tx=0x7f1dd002ba30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cd79ac02a834637a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1092055460 0 0) 0x7fa62c0cf4d0 con 0x7fa62c15e200 2024-03-06T07:33:20.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa62c0b5240 con 0x7fa62c15e200 2024-03-06T07:33:20.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 706722708 0 0) 0x7fa60c003520 con 0x7fa62c0b2e50 2024-03-06T07:33:20.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c0d0c00 msgr2=0x7fa62c0d4850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c0d0c00 0x7fa62c0d4850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 >> v1:172.21.15.194:6789/0 conn(0x7fa62c15e200 legacy=0x7fa62c0d04f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa62c0b5500 con 0x7fa62c0b2e50 2024-03-06T07:33:20.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fa6240019f0 con 0x7fa62c0b2e50 2024-03-06T07:33:20.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 >> v1:172.21.15.86:6789/0 conn(0x7fa62c0b2e50 legacy=0x7fa62c0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60c003b70 0x7fa60c003f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa60c004800 0x7fa60c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.078 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c0051c0 0x7fa60c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa62c0d2ec0 con 0x7fa60c004800 2024-03-06T07:33:20.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa60c00bf60 con 0x7fa60c0051c0 2024-03-06T07:33:20.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa60c00c160 con 0x7fa60c003b70 2024-03-06T07:33:20.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7f1ddca00640 1 -- 172.21.15.86:0/2527477379 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dd002f830 con 0x7f1de00d12e0 2024-03-06T07:33:20.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7f1ddca00640 1 -- 172.21.15.86:0/2527477379 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1dd002f9d0 con 0x7f1de00d12e0 2024-03-06T07:33:20.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7f1ddca00640 1 -- 172.21.15.86:0/2527477379 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dd002fe00 con 0x7f1de00d12e0 2024-03-06T07:33:20.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7f0b47400640 1 --2- 172.21.15.86:0/2542522716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b480d0d50 0x7f0b480d2160 secure :-1 s=READY pgs=1300 cs=0 l=1 rev1=1 crypto rx=0x7f0b480b6a40 tx=0x7f0b4002be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=585133de95053b01 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fd3cfe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d00d1e90 0x7fd3d0158d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fd3cfe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d00d2a70 0x7fd3d00b7c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fd3cfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00cae50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fd3cfe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3d00cf3a0 con 0x7fd3d0161580 2024-03-06T07:33:20.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.056+0000 7fd3cfe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3d00b39a0 con 0x7fd3d00d1e90 2024-03-06T07:33:20.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fd3cfe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3d00b4400 con 0x7fd3d00d2a70 2024-03-06T07:33:20.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa623e00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa60c004800 0x7fa60c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa62be00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c0051c0 0x7fa60c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7f0b45600640 1 -- 172.21.15.86:0/2542522716 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b40039070 con 0x7f0b480d0d50 2024-03-06T07:33:20.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7f0b45600640 1 -- 172.21.15.86:0/2542522716 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b4002cc00 con 0x7f0b480d0d50 2024-03-06T07:33:20.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fd3cf400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00cae50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fd3cf400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00cae50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60588/0 (socket says 172.21.15.86:60588) 2024-03-06T07:33:20.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fd3cf400640 1 -- 172.21.15.86:0/2652145451 learned_addr learned my addr 172.21.15.86:0/2652145451 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7f0b45600640 1 -- 172.21.15.86:0/2542522716 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b40034440 con 0x7f0b480d0d50 2024-03-06T07:33:20.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa62b400640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60c003b70 0x7fa60c003f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fd3cea00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d00d1e90 0x7fd3d0158d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7f1de4e00640 1 -- 172.21.15.86:0/2527477379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de00d12e0 msgr2=0x7f1de0161d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2527477379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de00d12e0 0x7f1de0161d30 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f1dd0009870 tx=0x7f1dd002ba30 comp rx=0 tx=0).stop 2024-03-06T07:33:20.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fd3c6000640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d00d2a70 0x7fd3d00b7c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7fd3cf400640 1 -- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d00d2a70 msgr2=0x7fd3d00b7c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7fd3cf400640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d00d2a70 0x7fd3d00b7c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7fd3cf400640 1 -- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d00d1e90 msgr2=0x7fd3d0158d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7fd3cf400640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d00d1e90 0x7fd3d0158d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7fd3cf400640 1 -- 172.21.15.86:0/2652145451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3d00b6bc0 con 0x7fd3d0161580 2024-03-06T07:33:20.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f1de4e00640 1 -- 172.21.15.86:0/2527477379 shutdown_connections 2024-03-06T07:33:20.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2527477379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de00d12e0 0x7f1de0161d30 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2527477379 >> v2:172.21.15.194:3300/0 conn(0x7f1de015e360 0x7f1de00cec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f1de4e00640 1 -- 172.21.15.86:0/2527477379 >> 172.21.15.86:0/2527477379 conn(0x7f1de00d3f50 msgr2=0x7f1de00d4330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 -- 172.21.15.86:0/2542522716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b480d0d50 msgr2=0x7f0b480d2160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 --2- 172.21.15.86:0/2542522716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b480d0d50 0x7f0b480d2160 secure :-1 s=READY pgs=1300 cs=0 l=1 rev1=1 crypto rx=0x7f0b480b6a40 tx=0x7f0b4002be50 comp rx=0 tx=0).stop 2024-03-06T07:33:20.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f1de4e00640 1 -- 172.21.15.86:0/2527477379 shutdown_connections 2024-03-06T07:33:20.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f1de4e00640 1 -- 172.21.15.86:0/2527477379 wait complete. 2024-03-06T07:33:20.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 -- 172.21.15.86:0/2542522716 shutdown_connections 2024-03-06T07:33:20.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 --2- 172.21.15.86:0/2542522716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b480d0d50 0x7f0b480d2160 unknown :-1 s=CLOSED pgs=1300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 --2- 172.21.15.86:0/2542522716 >> v2:172.21.15.194:3300/0 conn(0x7f0b480b6d90 0x7f0b480d0810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 -- 172.21.15.86:0/2542522716 >> 172.21.15.86:0/2542522716 conn(0x7f0b4809d210 msgr2=0x7f0b480d3f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 -- 172.21.15.86:0/2542522716 shutdown_connections 2024-03-06T07:33:20.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7f0b4da00640 1 -- 172.21.15.86:0/2542522716 wait complete. 2024-03-06T07:33:20.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa623e00640 1 -- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60c003b70 msgr2=0x7fa60c003f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa623e00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60c003b70 0x7fa60c003f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa623e00640 1 -- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c0051c0 msgr2=0x7fa60c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa623e00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c0051c0 0x7fa60c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa623e00640 1 -- 172.21.15.86:0/3926453063 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa60c003520 con 0x7fa60c004800 2024-03-06T07:33:20.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.057+0000 7fa62b400640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa60c003b70 0x7fa60c003f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:20.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.058+0000 7fd3cf400640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00cae50 secure :-1 s=READY pgs=1301 cs=0 l=1 rev1=1 crypto rx=0x7fd3c800cd70 tx=0x7fd3c800a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fa623e00640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa60c004800 0x7fa60c004c80 secure :-1 s=READY pgs=1302 cs=0 l=1 rev1=1 crypto rx=0x7fa62c161f60 tx=0x7fa60c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e9af1c35a81149cd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fd3c7400640 1 -- 172.21.15.86:0/2652145451 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3c8003b90 con 0x7fd3d0161580 2024-03-06T07:33:20.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fd3c7400640 1 -- 172.21.15.86:0/2652145451 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3c8003d30 con 0x7fd3d0161580 2024-03-06T07:33:20.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fd3c7400640 1 -- 172.21.15.86:0/2652145451 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3c8004240 con 0x7fd3d0161580 2024-03-06T07:33:20.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3d00b6ea0 con 0x7fd3d0161580 2024-03-06T07:33:20.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa61c039070 con 0x7fa60c004800 2024-03-06T07:33:20.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa61c02c550 con 0x7fa60c004800 2024-03-06T07:33:20.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.059+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3d00b7460 con 0x7fd3d0161580 2024-03-06T07:33:20.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7fa62aa00640 1 -- 172.21.15.86:0/3926453063 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa61c009650 con 0x7fa60c004800 2024-03-06T07:33:20.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7fa631000640 1 -- 172.21.15.86:0/3926453063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa60c004800 msgr2=0x7fa60c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7fa631000640 1 --2- 172.21.15.86:0/3926453063 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa60c004800 0x7fa60c004c80 secure :-1 s=READY pgs=1302 cs=0 l=1 rev1=1 crypto rx=0x7fa62c161f60 tx=0x7fa60c005830 comp rx=0 tx=0).stop 2024-03-06T07:33:20.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7fa62be00640 1 -- 172.21.15.86:0/3926453063 reap_dead start 2024-03-06T07:33:20.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7fa631000640 1 -- 172.21.15.86:0/3926453063 shutdown_connections 2024-03-06T07:33:20.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7fa631000640 1 -- 172.21.15.86:0/3926453063 >> 172.21.15.86:0/3926453063 conn(0x7fa62c0d3d30 msgr2=0x7fa62c0d4140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7f0b4da00640 1 Processor -- start 2024-03-06T07:33:20.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd3d004f330 con 0x7fd3d0161580 2024-03-06T07:33:20.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7f1de4e00640 1 Processor -- start 2024-03-06T07:33:20.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.060+0000 7f1de4e00640 1 -- start start 2024-03-06T07:33:20.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.061+0000 7fa631000640 1 -- 172.21.15.86:0/3926453063 shutdown_connections 2024-03-06T07:33:20.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.061+0000 7f0b4da00640 1 -- start start 2024-03-06T07:33:20.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.061+0000 7fa631000640 1 -- 172.21.15.86:0/3926453063 wait complete. 2024-03-06T07:33:20.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b4da00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b480b6d90 0x7f0b4815d2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b4da00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b480d0d50 0x7f0b481fbbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b4da00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b481617d0 0x7f0b481fc120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b4da00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b48161bb0 con 0x7f0b481617d0 2024-03-06T07:33:20.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b4da00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b480d1c40 con 0x7f0b480d0d50 2024-03-06T07:33:20.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b4da00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b480b5350 con 0x7f0b480b6d90 2024-03-06T07:33:20.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f1de4e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1de00d12e0 0x7f1de00c98b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f1de4e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1de015e360 0x7f1de01e8460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f1de4e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de0162270 0x7f1de01e89a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f1de4e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1de00d18c0 con 0x7f1de015e360 2024-03-06T07:33:20.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f1de4e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1de00d1740 con 0x7f1de00d12e0 2024-03-06T07:33:20.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f1de4e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1de00d2ca0 con 0x7f1de0162270 2024-03-06T07:33:20.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b47400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b481617d0 0x7f0b481fc120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b47400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b481617d0 0x7f0b481fc120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60600/0 (socket says 172.21.15.86:60600) 2024-03-06T07:33:20.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b47400640 1 -- 172.21.15.86:0/3601955288 learned_addr learned my addr 172.21.15.86:0/3601955288 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f1ddd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1de015e360 0x7f1de01e8460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b46a00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b480b6d90 0x7f0b4815d2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.062+0000 7f0b46000640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b480d0d50 0x7f0b481fbbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b47400640 1 -- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b480b6d90 msgr2=0x7f0b4815d2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b47400640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b480b6d90 0x7f0b4815d2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b47400640 1 -- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b480d0d50 msgr2=0x7f0b481fbbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b47400640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b480d0d50 0x7f0b481fbbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b47400640 1 -- 172.21.15.86:0/3601955288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b4815fdb0 con 0x7f0b481617d0 2024-03-06T07:33:20.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1ddd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1de015e360 0x7f1de01e8460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60616/0 (socket says 172.21.15.86:60616) 2024-03-06T07:33:20.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1dde800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de0162270 0x7f1de01e89a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b46000640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b480d0d50 0x7f0b481fbbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:20.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1ddde00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1de00d12e0 0x7f1de00c98b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1dde800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de0162270 0x7f1de01e89a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:58158/0 (socket says 172.21.15.86:58158) 2024-03-06T07:33:20.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1dde800640 1 -- 172.21.15.86:0/2170495772 learned_addr learned my addr 172.21.15.86:0/2170495772 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b47400640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b481617d0 0x7f0b481fc120 secure :-1 s=READY pgs=1303 cs=0 l=1 rev1=1 crypto rx=0x7f0b40009e80 tx=0x7f0b40004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b3ea00640 1 -- 172.21.15.86:0/3601955288 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b40044020 con 0x7f0b481617d0 2024-03-06T07:33:20.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b3ea00640 1 -- 172.21.15.86:0/3601955288 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b4003f470 con 0x7f0b481617d0 2024-03-06T07:33:20.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b3ea00640 1 -- 172.21.15.86:0/3601955288 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b40039040 con 0x7f0b481617d0 2024-03-06T07:33:20.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0b48160090 con 0x7f0b481617d0 2024-03-06T07:33:20.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1ddd400640 1 -- 172.21.15.86:0/2170495772 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de0162270 msgr2=0x7f1de01e89a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1ddd400640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de0162270 0x7f1de01e89a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1ddd400640 1 -- 172.21.15.86:0/2170495772 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1de00d12e0 msgr2=0x7f1de00c98b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1ddd400640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1de00d12e0 0x7f1de00c98b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.063+0000 7f1ddd400640 1 -- 172.21.15.86:0/2170495772 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1de00b3800 con 0x7f1de015e360 2024-03-06T07:33:20.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.064+0000 7fa631000640 1 Processor -- start 2024-03-06T07:33:20.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.064+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0b48160650 con 0x7f0b481617d0 2024-03-06T07:33:20.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.064+0000 7fa631000640 1 -- start start 2024-03-06T07:33:20.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.064+0000 7fd3c7400640 1 -- 172.21.15.86:0/2652145451 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd3c80043e0 con 0x7fd3d0161580 2024-03-06T07:33:20.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.064+0000 7f1ddd400640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1de015e360 0x7f1de01e8460 secure :-1 s=READY pgs=1304 cs=0 l=1 rev1=1 crypto rx=0x7f1de00cf9f0 tx=0x7f1dd00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fd3c7400640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3ac06b4d0 0x7fd3ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fa631000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c004800 0x7fa62c1f7f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fa631000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa62c0b7330 0x7fa62c0b77b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fd3c7400640 1 -- 172.21.15.86:0/2652145451 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd3c808ad30 con 0x7fd3d0161580 2024-03-06T07:33:20.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0b4804f330 con 0x7f0b481617d0 2024-03-06T07:33:20.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fa631000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c06c1b0 0x7fa62c06c630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fa623e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa62c0b7330 0x7fa62c0b77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fd3cea00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3ac06b4d0 0x7fd3ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fa623e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa62c0b7330 0x7fa62c0b77b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60620/0 (socket says 172.21.15.86:60620) 2024-03-06T07:33:20.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.065+0000 7fa623e00640 1 -- 172.21.15.86:0/993324672 learned_addr learned my addr 172.21.15.86:0/993324672 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.066+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa62c0cfd60 con 0x7fa62c0b7330 2024-03-06T07:33:20.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.066+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa62c0cfa80 con 0x7fa60c004800 2024-03-06T07:33:20.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.066+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa62c15e5e0 con 0x7fa62c06c1b0 2024-03-06T07:33:20.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.066+0000 7fa62be00640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c06c1b0 0x7fa62c06c630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.066+0000 7fa62b400640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c004800 0x7fa62c1f7f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.066+0000 7fd3cea00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3ac06b4d0 0x7fd3ac06d990 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7fd3c0036000 tx=0x7fd3c003c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.067+0000 7fa62be00640 1 -- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c004800 msgr2=0x7fa62c1f7f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.068+0000 7f0b3ea00640 1 -- 172.21.15.86:0/3601955288 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0b40047020 con 0x7f0b481617d0 2024-03-06T07:33:20.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7fa62be00640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c004800 0x7fa62c1f7f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7fa62be00640 1 -- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa62c0b7330 msgr2=0x7fa62c0b77b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7fa62be00640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa62c0b7330 0x7fa62c0b77b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7fa62be00640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa62c1f84c0 con 0x7fa62c06c1b0 2024-03-06T07:33:20.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7fa623e00640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa62c0b7330 0x7fa62c0b77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:33:20.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7fa62b400640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c004800 0x7fa62c1f7f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:20.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7f0b3ea00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b2806b520 0x7f0b2806d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7f0b3ea00640 1 -- 172.21.15.86:0/3601955288 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0b400accc0 con 0x7f0b481617d0 2024-03-06T07:33:20.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7fa62be00640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c06c1b0 0x7fa62c06c630 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fa62c08c830 tx=0x7fa624007bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.069+0000 7f0b46a00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b2806b520 0x7f0b2806d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.070+0000 7fa623400640 1 -- 172.21.15.86:0/993324672 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa62400eb60 con 0x7fa62c06c1b0 2024-03-06T07:33:20.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.070+0000 7fa623400640 1 -- 172.21.15.86:0/993324672 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa62400ed00 con 0x7fa62c06c1b0 2024-03-06T07:33:20.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.070+0000 7fa623400640 1 -- 172.21.15.86:0/993324672 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6240146a0 con 0x7fa62c06c1b0 2024-03-06T07:33:20.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.070+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa62c158820 con 0x7fa62c06c1b0 2024-03-06T07:33:20.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.070+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa62c157a60 con 0x7fa62c06c1b0 2024-03-06T07:33:20.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.070+0000 7f0b46a00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b2806b520 0x7f0b2806d9e0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f0b38000f60 tx=0x7f0b38009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.071+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa62c04f330 con 0x7fa62c06c1b0 2024-03-06T07:33:20.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.072+0000 7f1dd6000640 1 -- 172.21.15.86:0/2170495772 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dd0047070 con 0x7f1de015e360 2024-03-06T07:33:20.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.072+0000 7fd3c7400640 1 -- 172.21.15.86:0/2652145451 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd3c8059ba0 con 0x7fd3d0161580 2024-03-06T07:33:20.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.072+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1de00b3aa0 con 0x7f1de015e360 2024-03-06T07:33:20.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.072+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1de00c9df0 con 0x7f1de015e360 2024-03-06T07:33:20.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.073+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1de00d3860 con 0x7f1de015e360 2024-03-06T07:33:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.073+0000 7fa623400640 1 -- 172.21.15.86:0/993324672 <== mon.2 v2:172.21.15.194:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa62401b060 con 0x7fa62c06c1b0 2024-03-06T07:33:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.074+0000 7fa623400640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b5a0 0x7fa60806da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.074+0000 7fa623400640 1 -- 172.21.15.86:0/993324672 <== mon.2 v2:172.21.15.194:3301/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa624089ad0 con 0x7fa62c06c1b0 2024-03-06T07:33:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.075+0000 7fa62b400640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b5a0 0x7fa60806da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.075+0000 7fa62b400640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b5a0 0x7fa60806da60 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7fa614000f60 tx=0x7fa614009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.076+0000 7f1dd6000640 1 -- 172.21.15.86:0/2170495772 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1dd0009490 con 0x7f1de015e360 2024-03-06T07:33:20.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.076+0000 7f1dd6000640 1 -- 172.21.15.86:0/2170495772 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1dd0051060 con 0x7f1de015e360 2024-03-06T07:33:20.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.077+0000 7f0b3ea00640 1 -- 172.21.15.86:0/3601955288 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0b4004e030 con 0x7f0b481617d0 2024-03-06T07:33:20.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.079+0000 7f1dd6000640 1 -- 172.21.15.86:0/2170495772 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1dd004a070 con 0x7f1de015e360 2024-03-06T07:33:20.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.080+0000 7f1dd6000640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dbc06b4d0 0x7f1dbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.080+0000 7f1dd6000640 1 -- 172.21.15.86:0/2170495772 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(110..110 src has 1..110) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1dd00ac120 con 0x7f1de015e360 2024-03-06T07:33:20.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.080+0000 7f1ddde00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dbc06b4d0 0x7f1dbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.081+0000 7f1ddde00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dbc06b4d0 0x7f1dbc06d990 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f1dd8009bd0 tx=0x7f1dd800b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.082+0000 7f1dd6000640 1 -- 172.21.15.86:0/2170495772 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1de00d3860 con 0x7f1de015e360 2024-03-06T07:33:20.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.089+0000 7fa623400640 1 -- 172.21.15.86:0/993324672 <== mon.2 v2:172.21.15.194:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa624058930 con 0x7fa62c06c1b0 2024-03-06T07:33:20.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.342+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "b"} v 0) v1 -- 0x7fd3d00d3b60 con 0x7fd3d0161580 2024-03-06T07:33:20.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.346+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "a"} v 0) v1 -- 0x7f0b480d15c0 con 0x7f0b481617d0 2024-03-06T07:33:20.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.356+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "d"} v 0) v1 -- 0x7f1de00d0230 con 0x7f1de015e360 2024-03-06T07:33:20.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.363+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "c"} v 0) v1 -- 0x7fa62c08d4c0 con 0x7fa62c06c1b0 2024-03-06T07:33:20.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.500+0000 7fd3c7400640 1 -- 172.21.15.86:0/2652145451 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "b"}]=0 failed mds gid 6046 v78) v1 ==== 95+0+0 (secure 0 0 0) 0x7fd3c80594a0 con 0x7fd3d0161580 2024-03-06T07:33:20.503 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 6046 2024-03-06T07:33:20.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.503+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3ac06b4d0 msgr2=0x7fd3ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.503+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3ac06b4d0 0x7fd3ac06d990 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7fd3c0036000 tx=0x7fd3c003c040 comp rx=0 tx=0).stop 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.503+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 msgr2=0x7fd3d00cae50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.503+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00cae50 secure :-1 s=READY pgs=1301 cs=0 l=1 rev1=1 crypto rx=0x7fd3c800cd70 tx=0x7fd3c800a660 comp rx=0 tx=0).stop 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 shutdown_connections 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3ac06b4d0 0x7fd3ac06d990 unknown :-1 s=CLOSED pgs=561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3d0161580 0x7fd3d00cae50 unknown :-1 s=CLOSED pgs=1301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3d00d2a70 0x7fd3d00b7c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 --2- 172.21.15.86:0/2652145451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3d00d1e90 0x7fd3d0158d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 >> 172.21.15.86:0/2652145451 conn(0x7fd3d00d3d80 msgr2=0x7fd3d006cc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 shutdown_connections 2024-03-06T07:33:20.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.504+0000 7fd3cfe00640 1 -- 172.21.15.86:0/2652145451 wait complete. 2024-03-06T07:33:20.525 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:33:20.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:33:20.535 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:33:20.535 DEBUG:teuthology.parallel:result is None 2024-03-06T07:33:20.539 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:33:20.537+0000 7f41a3400640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:33:20.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.542+0000 7f0b3ea00640 1 -- 172.21.15.86:0/3601955288 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "a"}]=0 failed mds gid 7464 v79) v1 ==== 95+0+0 (secure 0 0 0) 0x7f0b4007bb90 con 0x7f0b481617d0 2024-03-06T07:33:20.544 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 7464 2024-03-06T07:33:20.545 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.542+0000 7f1dd6000640 1 -- 172.21.15.86:0/2170495772 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "d"}]=0 failed mds gid 6043 v79) v1 ==== 95+0+0 (secure 0 0 0) 0x7f1dd0033030 con 0x7f1de015e360 2024-03-06T07:33:20.545 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 6043 2024-03-06T07:33:20.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.544+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b2806b520 msgr2=0x7f0b2806d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.544+0000 7f0b4da00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b2806b520 0x7f0b2806d9e0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f0b38000f60 tx=0x7f0b38009040 comp rx=0 tx=0).stop 2024-03-06T07:33:20.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.544+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b481617d0 msgr2=0x7f0b481fc120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.544+0000 7f0b4da00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b481617d0 0x7f0b481fc120 secure :-1 s=READY pgs=1303 cs=0 l=1 rev1=1 crypto rx=0x7f0b40009e80 tx=0x7f0b40004a40 comp rx=0 tx=0).stop 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.544+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 shutdown_connections 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f0b4da00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b2806b520 0x7f0b2806d9e0 unknown :-1 s=CLOSED pgs=562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f0b4da00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b481617d0 0x7f0b481fc120 unknown :-1 s=CLOSED pgs=1303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f0b4da00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b480d0d50 0x7f0b481fbbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f0b4da00640 1 --2- 172.21.15.86:0/3601955288 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b480b6d90 0x7f0b4815d2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 >> 172.21.15.86:0/3601955288 conn(0x7f0b4809d210 msgr2=0x7f0b4806c0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 shutdown_connections 2024-03-06T07:33:20.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f0b4da00640 1 -- 172.21.15.86:0/3601955288 wait complete. 2024-03-06T07:33:20.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dbc06b4d0 msgr2=0x7f1dbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dbc06b4d0 0x7f1dbc06d990 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f1dd8009bd0 tx=0x7f1dd800b040 comp rx=0 tx=0).stop 2024-03-06T07:33:20.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1de015e360 msgr2=0x7f1de01e8460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1de015e360 0x7f1de01e8460 secure :-1 s=READY pgs=1304 cs=0 l=1 rev1=1 crypto rx=0x7f1de00cf9f0 tx=0x7f1dd00046c0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 shutdown_connections 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1dbc06b4d0 0x7f1dbc06d990 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1de0162270 0x7f1de01e89a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1de015e360 0x7f1de01e8460 unknown :-1 s=CLOSED pgs=1304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 --2- 172.21.15.86:0/2170495772 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1de00d12e0 0x7f1de00c98b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.545+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 >> 172.21.15.86:0/2170495772 conn(0x7f1de00d3f50 msgr2=0x7f1de00d4330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.546+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 shutdown_connections 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.546+0000 7f1de4e00640 1 -- 172.21.15.86:0/2170495772 wait complete. 2024-03-06T07:33:20.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.547+0000 7fa623400640 1 -- 172.21.15.86:0/993324672 <== mon.2 v2:172.21.15.194:3301/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "c"}]=0 failed mds gid 7467 v79) v1 ==== 95+0+0 (secure 0 0 0) 0x7fa624058230 con 0x7fa62c06c1b0 2024-03-06T07:33:20.551 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 7467 2024-03-06T07:33:20.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.550+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b5a0 msgr2=0x7fa60806da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.550+0000 7fa631000640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b5a0 0x7fa60806da60 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7fa614000f60 tx=0x7fa614009040 comp rx=0 tx=0).stop 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.550+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c06c1b0 msgr2=0x7fa62c06c630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.550+0000 7fa631000640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c06c1b0 0x7fa62c06c630 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fa62c08c830 tx=0x7fa624007bc0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 shutdown_connections 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa60806b5a0 0x7fa60806da60 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa62c06c1b0 0x7fa62c06c630 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa62c0b7330 0x7fa62c0b77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 --2- 172.21.15.86:0/993324672 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa60c004800 0x7fa62c1f7f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 >> 172.21.15.86:0/993324672 conn(0x7fa62c0d3d30 msgr2=0x7fa62c0d1c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 shutdown_connections 2024-03-06T07:33:20.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.551+0000 7fa631000640 1 -- 172.21.15.86:0/993324672 wait complete. 2024-03-06T07:33:20.568 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:33:20.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:33:20.569 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:33:20.569 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:33:20.571 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:33:20.571 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:33:20.573 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:33:20.574 DEBUG:teuthology.parallel:result is None 2024-03-06T07:33:20.617 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:33:20.617 DEBUG:teuthology.parallel:result is None 2024-03-06T07:33:20.623 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:33:20.623 DEBUG:teuthology.parallel:result is None 2024-03-06T07:33:20.623 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:20.693 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:33:20.728 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:33:20.768 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:33:20.770 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:33:20.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.795+0000 7f7a7e000640 1 Processor -- start 2024-03-06T07:33:20.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.795+0000 7f7a7e000640 1 -- start start 2024-03-06T07:33:20.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a7e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7a78161580 0x7f7a780d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a7e000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7a78161960 con 0x7f7a780d1e90 2024-03-06T07:33:20.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a7e000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7a780cf6f0 con 0x7f7a780d2a70 2024-03-06T07:33:20.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a7e000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7a780cf9d0 con 0x7f7a78161580 2024-03-06T07:33:20.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a7cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7a78161580 0x7f7a780d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a7cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7a78161580 0x7f7a780d1950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60650/0 (socket says 172.21.15.86:60650) 2024-03-06T07:33:20.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a7cc00640 1 -- 172.21.15.86:0/2695443880 learned_addr learned my addr 172.21.15.86:0/2695443880 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.796+0000 7f7a77e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7a780d1e90 0x7f7a780d2360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36758/0 (socket says 172.21.15.86:36758) 2024-03-06T07:33:20.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.797+0000 7f7a7cc00640 1 -- 172.21.15.86:0/2695443880 >> v1:172.21.15.194:6789/0 conn(0x7f7a780d2a70 legacy=0x7f7a780b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:20.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.797+0000 7f7a7cc00640 1 -- 172.21.15.86:0/2695443880 >> v1:172.21.15.86:6789/0 conn(0x7f7a780d1e90 legacy=0x7f7a780d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.797+0000 7f7a7cc00640 1 -- 172.21.15.86:0/2695443880 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a780cfcb0 con 0x7f7a78161580 2024-03-06T07:33:20.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.797+0000 7f7a77400640 1 -- 172.21.15.86:0/2695443880 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1970271191 0 0) 0x7f7a78161960 con 0x7f7a780d1e90 2024-03-06T07:33:20.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.798+0000 7f7a7cc00640 1 --2- 172.21.15.86:0/2695443880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a78161580 0x7f7a780d1950 secure :-1 s=READY pgs=1312 cs=0 l=1 rev1=1 crypto rx=0x7f7a6c009870 tx=0x7f7a6c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3e4d5bf9986e4779 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.798+0000 7f7a77400640 1 -- 172.21.15.86:0/2695443880 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a6c034070 con 0x7f7a78161580 2024-03-06T07:33:20.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.798+0000 7f7a77400640 1 -- 172.21.15.86:0/2695443880 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a6c030860 con 0x7f7a78161580 2024-03-06T07:33:20.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.798+0000 7f7a77400640 1 -- 172.21.15.86:0/2695443880 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a6c030c30 con 0x7f7a78161580 2024-03-06T07:33:20.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.798+0000 7f7a7e000640 1 -- 172.21.15.86:0/2695443880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a78161580 msgr2=0x7f7a780d1950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.798+0000 7f7a7e000640 1 --2- 172.21.15.86:0/2695443880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a78161580 0x7f7a780d1950 secure :-1 s=READY pgs=1312 cs=0 l=1 rev1=1 crypto rx=0x7f7a6c009870 tx=0x7f7a6c02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.799+0000 7f7a7e000640 1 -- 172.21.15.86:0/2695443880 shutdown_connections 2024-03-06T07:33:20.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.799+0000 7f7a7e000640 1 --2- 172.21.15.86:0/2695443880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a78161580 0x7f7a780d1950 unknown :-1 s=CLOSED pgs=1312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.799+0000 7f7a7e000640 1 -- 172.21.15.86:0/2695443880 >> 172.21.15.86:0/2695443880 conn(0x7f7a780d3d80 msgr2=0x7f7a780d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:20.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.799+0000 7f7a7e000640 1 -- 172.21.15.86:0/2695443880 shutdown_connections 2024-03-06T07:33:20.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.799+0000 7f7a7e000640 1 -- 172.21.15.86:0/2695443880 wait complete. 2024-03-06T07:33:20.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.800+0000 7f7a7e000640 1 Processor -- start 2024-03-06T07:33:20.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.800+0000 7f7a7e000640 1 -- start start 2024-03-06T07:33:20.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.801+0000 7f7a7e000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7a780d1e90 0x7f7a780a14c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.801+0000 7f7a7e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a780d2a70 0x7f7a780ca700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.801+0000 7f7a7e000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7a78161580 0x7f7a7815ccd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.801+0000 7f7a7e000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a780cf3a0 con 0x7f7a780d2a70 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.801+0000 7f7a7e000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a780b39a0 con 0x7f7a780d1e90 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.801+0000 7f7a7e000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7a780b45d0 con 0x7f7a78161580 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a780d2a70 0x7f7a780ca700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a780d2a70 0x7f7a780ca700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60652/0 (socket says 172.21.15.86:60652) 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 -- 172.21.15.86:0/4069094620 learned_addr learned my addr 172.21.15.86:0/4069094620 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a7cc00640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7a780d1e90 0x7f7a780a14c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 -- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7a78161580 msgr2=0x7f7a7815ccd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a7d600640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7a78161580 0x7f7a7815ccd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7a78161580 0x7f7a7815ccd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 -- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7a780d1e90 msgr2=0x7f7a780a14c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:20.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7a780d1e90 0x7f7a780a14c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 -- 172.21.15.86:0/4069094620 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a780cac40 con 0x7f7a780d2a70 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a77e00640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a780d2a70 0x7f7a780ca700 secure :-1 s=READY pgs=1313 cs=0 l=1 rev1=1 crypto rx=0x7f7a6400c4d0 tx=0x7f7a6400c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.802+0000 7f7a74c00640 1 -- 172.21.15.86:0/4069094620 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a64018070 con 0x7f7a780d2a70 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.803+0000 7f7a74c00640 1 -- 172.21.15.86:0/4069094620 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a64007500 con 0x7f7a780d2a70 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.803+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a780c80e0 con 0x7f7a780d2a70 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.803+0000 7f7a74c00640 1 -- 172.21.15.86:0/4069094620 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a64004a20 con 0x7f7a780d2a70 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.803+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a780c7790 con 0x7f7a780d2a70 2024-03-06T07:33:20.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.804+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7a7804f330 con 0x7f7a780d2a70 2024-03-06T07:33:20.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.805+0000 7f7a74c00640 1 -- 172.21.15.86:0/4069094620 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7a6401b070 con 0x7f7a780d2a70 2024-03-06T07:33:20.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.806+0000 7f7a74c00640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7a5806b4d0 0x7f7a5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:20.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.806+0000 7f7a74c00640 1 -- 172.21.15.86:0/4069094620 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f7a64089640 con 0x7f7a780d2a70 2024-03-06T07:33:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.806+0000 7f7a7cc00640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7a5806b4d0 0x7f7a5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:20.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.807+0000 7f7a7cc00640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7a5806b4d0 0x7f7a5806d990 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f7a6c036000 tx=0x7f7a6c03b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:20.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:20.811+0000 7f7a74c00640 1 -- 172.21.15.86:0/4069094620 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7a64058330 con 0x7f7a780d2a70 2024-03-06T07:33:21.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.029+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7a780d3b60 con 0x7f7a780d2a70 2024-03-06T07:33:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.030+0000 7f7a74c00640 1 -- 172.21.15.86:0/4069094620 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 79 v79) v1 ==== 94+0+1802 (secure 0 0 0) 0x7f7a64057c30 con 0x7f7a780d2a70 2024-03-06T07:33:21.033 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:21.033 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":79,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":79,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:20.538472+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{},"failed":[0],"damaged":[],"stopped":[],"info":{},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":8}]} 2024-03-06T07:33:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 79 2024-03-06T07:33:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7a5806b4d0 msgr2=0x7f7a5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7a5806b4d0 0x7f7a5806d990 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f7a6c036000 tx=0x7f7a6c03b040 comp rx=0 tx=0).stop 2024-03-06T07:33:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a780d2a70 msgr2=0x7f7a780ca700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a780d2a70 0x7f7a780ca700 secure :-1 s=READY pgs=1313 cs=0 l=1 rev1=1 crypto rx=0x7f7a6400c4d0 tx=0x7f7a6400c990 comp rx=0 tx=0).stop 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 shutdown_connections 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7a5806b4d0 0x7f7a5806d990 unknown :-1 s=CLOSED pgs=565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7a78161580 0x7f7a7815ccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7a780d2a70 0x7f7a780ca700 unknown :-1 s=CLOSED pgs=1313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 --2- 172.21.15.86:0/4069094620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7a780d1e90 0x7f7a780a14c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.032+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 >> 172.21.15.86:0/4069094620 conn(0x7f7a780d3d80 msgr2=0x7f7a7815efd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.033+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 shutdown_connections 2024-03-06T07:33:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:21.033+0000 7f7a7e000640 1 -- 172.21.15.86:0/4069094620 wait complete. 2024-03-06T07:33:21.454 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 79, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:33:05.831771+0000', 'modified': '2024-03-06T07:33:20.538472+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 114, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {}, 'failed': [0], 'damaged': [], 'stopped': [], 'info': {}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-06T07:33:21.455 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 0/1 2024-03-06T07:33:22.456 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:22.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.668+0000 7f346be00640 1 Processor -- start 2024-03-06T07:33:22.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.668+0000 7f346be00640 1 -- start start 2024-03-06T07:33:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.670+0000 7f346be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 0x7f346406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.670+0000 7f346be00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f346415f9d0 0x7f346415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.670+0000 7f346be00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f346406ee70 con 0x7f34640ce0a0 2024-03-06T07:33:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.670+0000 7f346be00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3464160370 con 0x7f346415f9d0 2024-03-06T07:33:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.670+0000 7f346be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f346406cef0 con 0x7f3464144c00 2024-03-06T07:33:22.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.670+0000 7f3469e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f34640ce0a0 0x7f34640ce4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36760/0 (socket says 172.21.15.86:36760) 2024-03-06T07:33:22.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.670+0000 7f3469e00640 1 -- 172.21.15.86:0/2327489152 learned_addr learned my addr 172.21.15.86:0/2327489152 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:22.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3469400640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 0x7f346406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:22.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3468a00640 1 --2- 172.21.15.86:0/2327489152 >> v2:172.21.15.194:3300/0 conn(0x7f346415f9d0 0x7f346415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:22.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2816600196 0 0) 0x7f345c001490 con 0x7f34640ce0a0 2024-03-06T07:33:22.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f344c003520 con 0x7f34640ce0a0 2024-03-06T07:33:22.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2688783949 0 0) 0x7f344c003520 con 0x7f34640ce0a0 2024-03-06T07:33:22.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 msgr2=0x7f346406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:22.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 0x7f346406e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 >> v2:172.21.15.194:3300/0 conn(0x7f346415f9d0 msgr2=0x7f346415fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:22.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 --2- 172.21.15.86:0/2327489152 >> v2:172.21.15.194:3300/0 conn(0x7f346415f9d0 0x7f346415fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f346406d1d0 con 0x7f34640ce0a0 2024-03-06T07:33:22.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f345c001bc0 con 0x7f34640ce0a0 2024-03-06T07:33:22.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3468a00640 1 --2- 172.21.15.86:0/2327489152 >> v2:172.21.15.194:3300/0 conn(0x7f346415f9d0 0x7f346415fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:33:22.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 >> v1:172.21.15.86:6789/0 conn(0x7f34640ce0a0 legacy=0x7f34640ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:22.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f344c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f344c004800 0x7f344c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 0x7f344c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f346406cef0 con 0x7f344c0039b0 2024-03-06T07:33:22.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f344c00ba70 con 0x7f344c004800 2024-03-06T07:33:22.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f344c00bc40 con 0x7f3464144c00 2024-03-06T07:33:22.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.671+0000 7f3469400640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f344c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:22.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3468a00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f344c004800 0x7f344c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:22.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3469400640 1 -- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 msgr2=0x7f344c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:22.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3469e00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 0x7f344c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:22.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3469400640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 0x7f344c0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3469400640 1 -- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f344c004800 msgr2=0x7f344c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:22.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3469400640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f344c004800 0x7f344c004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3469400640 1 -- 172.21.15.86:0/2327489152 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f344c003520 con 0x7f344c0039b0 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3468a00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f344c004800 0x7f344c004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.672+0000 7f3469e00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3464144c00 0x7f344c0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.673+0000 7f3469400640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f344c003dd0 secure :-1 s=READY pgs=1315 cs=0 l=1 rev1=1 crypto rx=0x7f345800ba00 tx=0x7f344c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a36e5fc538ec32c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.673+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3458036020 con 0x7f344c0039b0 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.673+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f34580325f0 con 0x7f344c0039b0 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.673+0000 7f3463e00640 1 -- 172.21.15.86:0/2327489152 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f345800b600 con 0x7f344c0039b0 2024-03-06T07:33:22.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.674+0000 7f346be00640 1 -- 172.21.15.86:0/2327489152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 msgr2=0x7f344c003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:22.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.674+0000 7f346be00640 1 --2- 172.21.15.86:0/2327489152 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f344c003dd0 secure :-1 s=READY pgs=1315 cs=0 l=1 rev1=1 crypto rx=0x7f345800ba00 tx=0x7f344c005830 comp rx=0 tx=0).stop 2024-03-06T07:33:22.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.674+0000 7f3469e00640 1 -- 172.21.15.86:0/2327489152 reap_dead start 2024-03-06T07:33:22.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.674+0000 7f346be00640 1 -- 172.21.15.86:0/2327489152 shutdown_connections 2024-03-06T07:33:22.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.674+0000 7f346be00640 1 -- 172.21.15.86:0/2327489152 >> 172.21.15.86:0/2327489152 conn(0x7f34640b1730 msgr2=0x7f34640b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:22.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.674+0000 7f346be00640 1 -- 172.21.15.86:0/2327489152 shutdown_connections 2024-03-06T07:33:22.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.674+0000 7f346be00640 1 -- 172.21.15.86:0/2327489152 wait complete. 2024-03-06T07:33:22.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.676+0000 7f346be00640 1 Processor -- start 2024-03-06T07:33:22.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.676+0000 7f346be00640 1 -- start start 2024-03-06T07:33:22.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f346be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f346413d2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f346be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346413ab30 0x7f346415abb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.689 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f346be00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f346413fa20 0x7f346413fed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f346be00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3464160730 con 0x7f344c0039b0 2024-03-06T07:33:22.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f346be00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f346406d790 con 0x7f346413fa20 2024-03-06T07:33:22.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f346be00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f346406cb50 con 0x7f346413ab30 2024-03-06T07:33:22.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f3469400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f346413d2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:22.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f3469400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f346413d2a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60666/0 (socket says 172.21.15.86:60666) 2024-03-06T07:33:22.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f3469400640 1 -- 172.21.15.86:0/2247022615 learned_addr learned my addr 172.21.15.86:0/2247022615 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:22.690 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.677+0000 7f3469400640 1 -- 172.21.15.86:0/2247022615 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346413ab30 msgr2=0x7f346415abb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:22.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3469400640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346413ab30 0x7f346415abb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3469400640 1 -- 172.21.15.86:0/2247022615 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f346413fa20 msgr2=0x7f346413fed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:22.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3469400640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f346413fa20 0x7f346413fed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3469400640 1 -- 172.21.15.86:0/2247022615 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f346413afb0 con 0x7f344c0039b0 2024-03-06T07:33:22.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3469400640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f346413d2a0 secure :-1 s=READY pgs=1316 cs=0 l=1 rev1=1 crypto rx=0x7f345802bf40 tx=0x7f3458004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:22.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3461600640 1 -- 172.21.15.86:0/2247022615 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3458042070 con 0x7f344c0039b0 2024-03-06T07:33:22.691 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3461600640 1 -- 172.21.15.86:0/2247022615 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3458032e50 con 0x7f344c0039b0 2024-03-06T07:33:22.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f3461600640 1 -- 172.21.15.86:0/2247022615 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3458036040 con 0x7f344c0039b0 2024-03-06T07:33:22.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f346413d880 con 0x7f344c0039b0 2024-03-06T07:33:22.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.678+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f346413c750 con 0x7f344c0039b0 2024-03-06T07:33:22.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.680+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f346413cb30 con 0x7f344c0039b0 2024-03-06T07:33:22.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.680+0000 7f3461600640 1 -- 172.21.15.86:0/2247022615 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3458039020 con 0x7f344c0039b0 2024-03-06T07:33:22.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.681+0000 7f3461600640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:22.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.681+0000 7f3461600640 1 -- 172.21.15.86:0/2247022615 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f34580ac400 con 0x7f344c0039b0 2024-03-06T07:33:22.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.682+0000 7f3468a00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:22.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.682+0000 7f3468a00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7f3450006e80 tx=0x7f3450006e10 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:22.693 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.686+0000 7f3461600640 1 -- 172.21.15.86:0/2247022615 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f346413cb30 con 0x7f344c0039b0 2024-03-06T07:33:22.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.920+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f346404f330 con 0x7f344c0039b0 2024-03-06T07:33:22.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.920+0000 7f3461600640 1 -- 172.21.15.86:0/2247022615 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 82 v82) v1 ==== 94+0+4983 (secure 0 0 0) 0x7f346413afb0 con 0x7f344c0039b0 2024-03-06T07:33:22.923 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:22.923 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":82,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":82,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:21.575641+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:reconnect","state_seq":2,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":8}]} 2024-03-06T07:33:22.923 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 82 2024-03-06T07:33:22.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 msgr2=0x7f344406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:22.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7f3450006e80 tx=0x7f3450006e10 comp rx=0 tx=0).stop 2024-03-06T07:33:22.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 msgr2=0x7f346413d2a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:22.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f346413d2a0 secure :-1 s=READY pgs=1316 cs=0 l=1 rev1=1 crypto rx=0x7f345802bf40 tx=0x7f3458004730 comp rx=0 tx=0).stop 2024-03-06T07:33:22.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 shutdown_connections 2024-03-06T07:33:22.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f344406b4d0 0x7f344406d990 unknown :-1 s=CLOSED pgs=571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f346413fa20 0x7f346413fed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f346413ab30 0x7f346415abb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 --2- 172.21.15.86:0/2247022615 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f344c0039b0 0x7f346413d2a0 unknown :-1 s=CLOSED pgs=1316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:22.927 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 >> 172.21.15.86:0/2247022615 conn(0x7f34640b1730 msgr2=0x7f3464145250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:22.927 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.923+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 shutdown_connections 2024-03-06T07:33:22.927 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:22.924+0000 7f346be00640 1 -- 172.21.15.86:0/2247022615 wait complete. 2024-03-06T07:33:23.348 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 82, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:33:05.831771+0000', 'modified': '2024-03-06T07:33:21.575641+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 114, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7722}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7722': {'gid': 7722, 'name': 'c', 'rank': 0, 'incarnation': 81, 'state': 'up:reconnect', 'state_seq': 2, 'addr': '172.21.15.86:6837/2212362187', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6836', 'nonce': 2212362187}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 2212362187}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-06T07:33:23.348 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_7722:up:reconnect 2024-03-06T07:33:24.205 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:33:24.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:33:24.349 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:24.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.564+0000 7f55f5400640 1 Processor -- start 2024-03-06T07:33:24.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.565+0000 7f55f5400640 1 -- start start 2024-03-06T07:33:24.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.566+0000 7f55f5400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f55f015f470 0x7f55f00d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:24.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.566+0000 7f55f5400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55f00d1200 con 0x7f55f00d0df0 2024-03-06T07:33:24.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.566+0000 7f55f5400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55f00cea60 con 0x7f55f015e0d0 2024-03-06T07:33:24.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.566+0000 7f55f5400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f55f00d2f60 con 0x7f55f015f470 2024-03-06T07:33:24.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.566+0000 7f55ede00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f55f015f470 0x7f55f00d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:24.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.566+0000 7f55ef200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f55f00d0df0 0x7f55f0160dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36762/0 (socket says 172.21.15.86:36762) 2024-03-06T07:33:24.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.566+0000 7f55ef200640 1 -- 172.21.15.86:0/817877141 learned_addr learned my addr 172.21.15.86:0/817877141 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:24.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.567+0000 7f55ede00640 1 -- 172.21.15.86:0/817877141 >> v1:172.21.15.194:6789/0 conn(0x7f55f015e0d0 legacy=0x7f55f00d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:24.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.567+0000 7f55ede00640 1 -- 172.21.15.86:0/817877141 >> v1:172.21.15.86:6789/0 conn(0x7f55f00d0df0 legacy=0x7f55f0160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:24.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.567+0000 7f55ede00640 1 -- 172.21.15.86:0/817877141 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55f00d2350 con 0x7f55f015f470 2024-03-06T07:33:24.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.568+0000 7f55ede00640 1 --2- 172.21.15.86:0/817877141 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00d08b0 secure :-1 s=READY pgs=1318 cs=0 l=1 rev1=1 crypto rx=0x7f55f00c77f0 tx=0x7f55dc02b930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b1648caa3b0eabf0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:24.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.568+0000 7f55ed400640 1 -- 172.21.15.86:0/817877141 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55dc035020 con 0x7f55f015f470 2024-03-06T07:33:24.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.568+0000 7f55ed400640 1 -- 172.21.15.86:0/817877141 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f55dc031890 con 0x7f55f015f470 2024-03-06T07:33:24.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.568+0000 7f55ed400640 1 -- 172.21.15.86:0/817877141 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55dc031c60 con 0x7f55f015f470 2024-03-06T07:33:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.569+0000 7f55f5400640 1 -- 172.21.15.86:0/817877141 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 msgr2=0x7f55f00d08b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.569+0000 7f55f5400640 1 --2- 172.21.15.86:0/817877141 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00d08b0 secure :-1 s=READY pgs=1318 cs=0 l=1 rev1=1 crypto rx=0x7f55f00c77f0 tx=0x7f55dc02b930 comp rx=0 tx=0).stop 2024-03-06T07:33:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.569+0000 7f55f5400640 1 -- 172.21.15.86:0/817877141 shutdown_connections 2024-03-06T07:33:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.569+0000 7f55f5400640 1 --2- 172.21.15.86:0/817877141 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00d08b0 unknown :-1 s=CLOSED pgs=1318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.569+0000 7f55f5400640 1 -- 172.21.15.86:0/817877141 >> 172.21.15.86:0/817877141 conn(0x7f55f00d3ec0 msgr2=0x7f55f00d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.569+0000 7f55f5400640 1 -- 172.21.15.86:0/817877141 shutdown_connections 2024-03-06T07:33:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.569+0000 7f55f5400640 1 -- 172.21.15.86:0/817877141 wait complete. 2024-03-06T07:33:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.571+0000 7f55f5400640 1 Processor -- start 2024-03-06T07:33:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.571+0000 7f55f5400640 1 -- start start 2024-03-06T07:33:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55f5400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55f00d0df0 0x7f55f0155330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55f5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55f015e0d0 0x7f55f00b81a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55f5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00b3b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55f5400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55f00d2bf0 con 0x7f55f015f470 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55f5400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55f00d2910 con 0x7f55f00d0df0 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55f5400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55f00cff10 con 0x7f55f015e0d0 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55ef200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00b3b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55ef200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00b3b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60680/0 (socket says 172.21.15.86:60680) 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.572+0000 7f55ef200640 1 -- 172.21.15.86:0/2951590981 learned_addr learned my addr 172.21.15.86:0/2951590981 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ede00640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55f015e0d0 0x7f55f00b81a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ee800640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55f00d0df0 0x7f55f0155330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ef200640 1 -- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55f015e0d0 msgr2=0x7f55f00b81a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ef200640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55f015e0d0 0x7f55f00b81a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:24.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ef200640 1 -- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55f00d0df0 msgr2=0x7f55f0155330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:24.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ef200640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55f00d0df0 0x7f55f0155330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:24.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ef200640 1 -- 172.21.15.86:0/2951590981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55f00b9790 con 0x7f55f015f470 2024-03-06T07:33:24.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.573+0000 7f55ef200640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00b3b60 secure :-1 s=READY pgs=1319 cs=0 l=1 rev1=1 crypto rx=0x7f55e800c450 tx=0x7f55e800c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:24.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.574+0000 7f55e6a00640 1 -- 172.21.15.86:0/2951590981 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55e8017070 con 0x7f55f015f470 2024-03-06T07:33:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.574+0000 7f55e6a00640 1 -- 172.21.15.86:0/2951590981 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f55e8005ce0 con 0x7f55f015f470 2024-03-06T07:33:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.574+0000 7f55e6a00640 1 -- 172.21.15.86:0/2951590981 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55e8007270 con 0x7f55f015f470 2024-03-06T07:33:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.574+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f55f00b8c10 con 0x7f55f015f470 2024-03-06T07:33:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.574+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55f00b94b0 con 0x7f55f015f470 2024-03-06T07:33:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.575+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f55f004f330 con 0x7f55f015f470 2024-03-06T07:33:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.576+0000 7f55e6a00640 1 -- 172.21.15.86:0/2951590981 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f55e801a070 con 0x7f55f015f470 2024-03-06T07:33:24.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.577+0000 7f55e6a00640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f55d006b4d0 0x7f55d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:24.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.577+0000 7f55e6a00640 1 -- 172.21.15.86:0/2951590981 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f55e80893c0 con 0x7f55f015f470 2024-03-06T07:33:24.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.577+0000 7f55ee800640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f55d006b4d0 0x7f55d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:24.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.578+0000 7f55ee800640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f55d006b4d0 0x7f55d006d990 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7f55e0007fd0 tx=0x7f55e0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:24.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.581+0000 7f55e6a00640 1 -- 172.21.15.86:0/2951590981 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f55f00d1200 con 0x7f55f015f470 2024-03-06T07:33:24.821 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.819+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f55f008d920 con 0x7f55f015f470 2024-03-06T07:33:24.822 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.819+0000 7f55e6a00640 1 -- 172.21.15.86:0/2951590981 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f55e8057c20 con 0x7f55f015f470 2024-03-06T07:33:24.822 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:24.822 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:33:24.823 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f55d006b4d0 msgr2=0x7f55d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f55d006b4d0 0x7f55d006d990 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7f55e0007fd0 tx=0x7f55e0009040 comp rx=0 tx=0).stop 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 msgr2=0x7f55f00b3b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00b3b60 secure :-1 s=READY pgs=1319 cs=0 l=1 rev1=1 crypto rx=0x7f55e800c450 tx=0x7f55e800c910 comp rx=0 tx=0).stop 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 shutdown_connections 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f55d006b4d0 0x7f55d006d990 unknown :-1 s=CLOSED pgs=572 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f55f015f470 0x7f55f00b3b60 unknown :-1 s=CLOSED pgs=1319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f55f015e0d0 0x7f55f00b81a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 --2- 172.21.15.86:0/2951590981 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f55f00d0df0 0x7f55f0155330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 >> 172.21.15.86:0/2951590981 conn(0x7f55f00d3ec0 msgr2=0x7f55f00b2a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 shutdown_connections 2024-03-06T07:33:24.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:24.822+0000 7f55f5400640 1 -- 172.21.15.86:0/2951590981 wait complete. 2024-03-06T07:33:25.250 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 84, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:33:05.831771+0000', 'modified': '2024-03-06T07:33:24.568795+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 114, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7722}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7722': {'gid': 7722, 'name': 'c', 'rank': 0, 'incarnation': 81, 'state': 'up:active', 'state_seq': 4, 'addr': '172.21.15.86:6837/2212362187', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6836', 'nonce': 2212362187}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 2212362187}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 7722, 'qdb_cluster': [7722]} 2024-03-06T07:33:25.250 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:33:25.250 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-06T07:33:25.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.466+0000 7f0b77e00640 1 Processor -- start 2024-03-06T07:33:25.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.467+0000 7f0b77e00640 1 -- start start 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0b7815f8f0 0x7f0b780ce8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0b7806afc0 0x7f0b7806b400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0b7815fed0 con 0x7f0b78156320 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0b78071dc0 con 0x7f0b7806afc0 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0b7806b940 con 0x7f0b7815f8f0 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0b7806afc0 0x7f0b7806b400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0b7806afc0 0x7f0b7806b400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60684/0 (socket says 172.21.15.86:60684) 2024-03-06T07:33:25.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.468+0000 7f0b77400640 1 -- 172.21.15.86:0/871837335 learned_addr learned my addr 172.21.15.86:0/871837335 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:25.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.469+0000 7f0b77400640 1 -- 172.21.15.86:0/871837335 >> v2:172.21.15.194:3300/0 conn(0x7f0b7815f8f0 msgr2=0x7f0b780ce8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.469+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/871837335 >> v2:172.21.15.194:3300/0 conn(0x7f0b7815f8f0 0x7f0b780ce8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.469+0000 7f0b77400640 1 --2- 172.21.15.86:0/871837335 >> v2:172.21.15.194:3300/0 conn(0x7f0b7815f8f0 0x7f0b780ce8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.469+0000 7f0b77400640 1 -- 172.21.15.86:0/871837335 >> v1:172.21.15.194:6789/0 conn(0x7f0b78156320 legacy=0x7f0b7815cd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.469+0000 7f0b77400640 1 -- 172.21.15.86:0/871837335 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b781601b0 con 0x7f0b7806afc0 2024-03-06T07:33:25.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.469+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/871837335 >> v2:172.21.15.194:3300/0 conn(0x7f0b7815f8f0 0x7f0b780ce8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:25.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.470+0000 7f0b77400640 1 --2- 172.21.15.86:0/871837335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b7806b400 secure :-1 s=READY pgs=1320 cs=0 l=1 rev1=1 crypto rx=0x7f0b780b6000 tx=0x7f0b7002bdc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a0b4cb5cf87b82a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.470+0000 7f0b76000640 1 -- 172.21.15.86:0/871837335 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b7002c960 con 0x7f0b7806afc0 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.470+0000 7f0b76000640 1 -- 172.21.15.86:0/871837335 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b7002cb00 con 0x7f0b7806afc0 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.470+0000 7f0b76000640 1 -- 172.21.15.86:0/871837335 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b700343f0 con 0x7f0b7806afc0 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 -- 172.21.15.86:0/871837335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 msgr2=0x7f0b7806b400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 --2- 172.21.15.86:0/871837335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b7806b400 secure :-1 s=READY pgs=1320 cs=0 l=1 rev1=1 crypto rx=0x7f0b780b6000 tx=0x7f0b7002bdc0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 -- 172.21.15.86:0/871837335 shutdown_connections 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 --2- 172.21.15.86:0/871837335 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b7806b400 unknown :-1 s=CLOSED pgs=1320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 --2- 172.21.15.86:0/871837335 >> v2:172.21.15.194:3300/0 conn(0x7f0b7815f8f0 0x7f0b780ce8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 -- 172.21.15.86:0/871837335 >> 172.21.15.86:0/871837335 conn(0x7f0b7806c0d0 msgr2=0x7f0b7806c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 -- 172.21.15.86:0/871837335 shutdown_connections 2024-03-06T07:33:25.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.471+0000 7f0b77e00640 1 -- 172.21.15.86:0/871837335 wait complete. 2024-03-06T07:33:25.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.473+0000 7f0b77e00640 1 Processor -- start 2024-03-06T07:33:25.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.473+0000 7f0b77e00640 1 -- start start 2024-03-06T07:33:25.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b77e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b78143520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b77e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b78156320 0x7f0b781401c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b77e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b7815f8f0 0x7f0b781e77d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b77e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b78156700 con 0x7f0b7806afc0 2024-03-06T07:33:25.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b77e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b78160770 con 0x7f0b78156320 2024-03-06T07:33:25.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b77e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b780cee10 con 0x7f0b7815f8f0 2024-03-06T07:33:25.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b78143520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b78143520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60690/0 (socket says 172.21.15.86:60690) 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 -- 172.21.15.86:0/268643479 learned_addr learned my addr 172.21.15.86:0/268643479 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 -- 172.21.15.86:0/268643479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b7815f8f0 msgr2=0x7f0b781e77d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b7815f8f0 0x7f0b781e77d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 -- 172.21.15.86:0/268643479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b78156320 msgr2=0x7f0b781401c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b78156320 0x7f0b781401c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.474+0000 7f0b76a00640 1 -- 172.21.15.86:0/268643479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b78143a60 con 0x7f0b7806afc0 2024-03-06T07:33:25.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.475+0000 7f0b76a00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b78143520 secure :-1 s=READY pgs=1321 cs=0 l=1 rev1=1 crypto rx=0x7f0b6800b3f0 tx=0x7f0b6800b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.475+0000 7f0b6ea00640 1 -- 172.21.15.86:0/268643479 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b68015020 con 0x7f0b7806afc0 2024-03-06T07:33:25.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.475+0000 7f0b6ea00640 1 -- 172.21.15.86:0/268643479 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b68011830 con 0x7f0b7806afc0 2024-03-06T07:33:25.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.475+0000 7f0b6ea00640 1 -- 172.21.15.86:0/268643479 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b68011c20 con 0x7f0b7806afc0 2024-03-06T07:33:25.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.475+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0b78140df0 con 0x7f0b7806afc0 2024-03-06T07:33:25.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.475+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0b781413b0 con 0x7f0b7806afc0 2024-03-06T07:33:25.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.477+0000 7f0b6ea00640 1 -- 172.21.15.86:0/268643479 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0b68018020 con 0x7f0b7806afc0 2024-03-06T07:33:25.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.479+0000 7f0b6ea00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b5406b500 0x7f0b5406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.479+0000 7f0b6ea00640 1 -- 172.21.15.86:0/268643479 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f0b68089dd0 con 0x7f0b7806afc0 2024-03-06T07:33:25.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.479+0000 7f0b809cc740 1 Processor -- start 2024-03-06T07:33:25.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.479+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b5406b500 0x7f0b5406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.479+0000 7f0b809cc740 1 -- start start 2024-03-06T07:33:25.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b5406b500 0x7f0b5406d9c0 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7f0b60006fd0 tx=0x7f0b60008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7eb990 0x55723e7be930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e7ee720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7bee70 0x55723e7eec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b809cc740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55723e728600 con 0x55723e7eb990 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55723e6d1930 con 0x55723e7bee70 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55723e7246a0 con 0x55723e7ebd70 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.480+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7eb990 0x55723e7be930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7eb990 0x55723e7be930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60702/0 (socket says 172.21.15.86:60702) 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 -- 172.21.15.86:0/1316001903 learned_addr learned my addr 172.21.15.86:0/1316001903 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 -- 172.21.15.86:0/1316001903 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 msgr2=0x55723e7ee720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 --2- 172.21.15.86:0/1316001903 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e7ee720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 -- 172.21.15.86:0/1316001903 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7bee70 msgr2=0x55723e7eec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 --2- 172.21.15.86:0/1316001903 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7bee70 0x55723e7eec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 -- 172.21.15.86:0/1316001903 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55723e43d2d0 con 0x55723e7eb990 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b76a00640 1 --2- 172.21.15.86:0/1316001903 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7eb990 0x55723e7be930 secure :-1 s=READY pgs=1322 cs=0 l=1 rev1=1 crypto rx=0x7f0b68088350 tx=0x7f0b68058640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c7523c74b2b8741e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b67400640 1 -- 172.21.15.86:0/1316001903 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b68086070 con 0x55723e7eb990 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b67400640 1 -- 172.21.15.86:0/1316001903 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b68011dc0 con 0x55723e7eb990 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.481+0000 7f0b67400640 1 -- 172.21.15.86:0/1316001903 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b68070070 con 0x55723e7eb990 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 -- 172.21.15.86:0/1316001903 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7eb990 msgr2=0x55723e7be930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 --2- 172.21.15.86:0/1316001903 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7eb990 0x55723e7be930 secure :-1 s=READY pgs=1322 cs=0 l=1 rev1=1 crypto rx=0x7f0b68088350 tx=0x7f0b68058640 comp rx=0 tx=0).stop 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 -- 172.21.15.86:0/1316001903 shutdown_connections 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 --2- 172.21.15.86:0/1316001903 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7bee70 0x55723e7eec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 --2- 172.21.15.86:0/1316001903 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e7ee720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 --2- 172.21.15.86:0/1316001903 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7eb990 0x55723e7be930 unknown :-1 s=CLOSED pgs=1322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 -- 172.21.15.86:0/1316001903 >> 172.21.15.86:0/1316001903 conn(0x55723e7c0ba0 msgr2=0x55723e809720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 -- 172.21.15.86:0/1316001903 shutdown_connections 2024-03-06T07:33:25.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.482+0000 7f0b809cc740 1 -- 172.21.15.86:0/1316001903 wait complete. 2024-03-06T07:33:25.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.483+0000 7f0b809cc740 1 -- start start 2024-03-06T07:33:25.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.483+0000 7f0b809cc740 1 Processor -- start 2024-03-06T07:33:25.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7eb990 0x55723e7e71c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7ebd70 0x55723e7ef710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7e3c30 0x55723e7e40b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b809cc740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55723e7c3140 con 0x55723e7ebd70 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55723e6e4d90 con 0x55723e7e3c30 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55723e7c2530 con 0x55723e7eb990 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b809cc740 10 client.0 _ll_register_callbacks cb 0x55723e720350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.484+0000 7f0b6fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7ebd70 0x55723e7ef710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7ebd70 0x55723e7ef710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60716/0 (socket says 172.21.15.86:60716) 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 -- 172.21.15.86:0/2941061841 learned_addr learned my addr 172.21.15.86:0/2941061841 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b76a00640 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7eb990 0x55723e7e71c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 -- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7eb990 msgr2=0x55723e7e71c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7eb990 0x55723e7e71c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b77400640 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7e3c30 0x55723e7e40b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 -- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7e3c30 msgr2=0x55723e7e40b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7e3c30 0x55723e7e40b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 -- 172.21.15.86:0/2941061841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55723e5076b0 con 0x55723e7ebd70 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7ebd70 0x55723e7ef710 secure :-1 s=READY pgs=1323 cs=0 l=1 rev1=1 crypto rx=0x7f0b6000a530 tx=0x7f0b600137d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=764361d6aa90643 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b5be00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.485+0000 7f0b5be00640 1 -- 172.21.15.86:0/2941061841 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b6000ce60 con 0x55723e7ebd70 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b5be00640 1 -- 172.21.15.86:0/2941061841 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f0b6001d3f0 con 0x55723e7ebd70 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0b68011dc0 con 0x55723e7ebd70 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b5be00640 1 -- 172.21.15.86:0/2941061841 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=84) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0b60029050 con 0x55723e7ebd70 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 10 client.7758 fetch_fsmap learned FSMap version 84 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=84,monmap=2+}) v3 -- 0x55723e7f0b20 con 0x55723e7ebd70 2024-03-06T07:33:25.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b5be00640 1 -- 172.21.15.86:0/2941061841 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b68011dc0 con 0x55723e7ebd70 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b5be00640 1 -- 172.21.15.86:0/2941061841 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 84) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f0b6002c020 con 0x55723e7ebd70 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 10 client.7758 fetch_fsmap finished waiting for FSMap version 84 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 10 client.7758 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 20 client.7758 populate_metadata read hostname 'smithi086' 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e7f1bd0 0x55723e827eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 4 client.7758 mds_command: new command op to 7722 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.486+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 --> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55723e735b40 con 0x55723e7f1bd0 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.487+0000 7f0b76a00640 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e7f1bd0 0x55723e827eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.487+0000 7f0b76a00640 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e7f1bd0 0x55723e827eb0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=16cd17140a1840d2 server_cookie=8ce77705acd2942a in_seq=0 out_seq=0 2024-03-06T07:33:25.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.487+0000 7f0b5be00640 10 client.7758 ms_handle_connect on v2:172.21.15.86:6836/2212362187 2024-03-06T07:33:25.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b5be00640 1 -- 172.21.15.86:0/2941061841 <== mds.0 v2:172.21.15.86:6836/2212362187 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x55723e735b40 con 0x55723e7f1bd0 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b5be00640 10 client.7758 handle_command_reply: tid=0 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 client.7758 shutdown 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7ebd70 msgr2=0x55723e7ef710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7ebd70 0x55723e7ef710 secure :-1 s=READY pgs=1323 cs=0 l=1 rev1=1 crypto rx=0x7f0b6000a530 tx=0x7f0b600137d0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 shutdown_connections 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e7f1bd0 0x55723e827eb0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7e3c30 0x55723e7e40b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7ebd70 0x55723e7ef710 unknown :-1 s=CLOSED pgs=1323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 --2- 172.21.15.86:0/2941061841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7eb990 0x55723e7e71c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 >> 172.21.15.86:0/2941061841 conn(0x55723e7bee70 msgr2=0x55723e80cc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b5be00640 0 client.7758 ms_handle_reset on v2:172.21.15.86:6836/2212362187 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.488+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 shutdown_connections 2024-03-06T07:33:25.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.489+0000 7f0b809cc740 1 -- 172.21.15.86:0/2941061841 wait complete. 2024-03-06T07:33:25.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.489+0000 7f0b809cc740 20 client.7758 trim_cache size 0 max 16384 2024-03-06T07:33:25.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.507+0000 7f0b809cc740 1 Processor -- start 2024-03-06T07:33:25.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.507+0000 7f0b809cc740 1 -- start start 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.507+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7ec380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.507+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 0x55723e7ec8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e7c39a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b809cc740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b781e6110 con 0x55723e7e3c30 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b78142920 con 0x55723e7eb990 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b781e5b10 con 0x55723e7ebd70 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7ec380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7ec380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60724/0 (socket says 172.21.15.86:60724) 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 -- 172.21.15.86:0/3645367705 learned_addr learned my addr 172.21.15.86:0/3645367705 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 -- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 msgr2=0x55723e7c39a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b77400640 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e7c39a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e7c39a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 0x55723e7ec8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 -- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 msgr2=0x55723e7ec8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 0x55723e7ec8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.508+0000 7f0b76a00640 1 -- 172.21.15.86:0/3645367705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55723e7e6d90 con 0x55723e7e3c30 2024-03-06T07:33:25.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.509+0000 7f0b76a00640 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7ec380 secure :-1 s=READY pgs=1324 cs=0 l=1 rev1=1 crypto rx=0x7f0b68088270 tx=0x7f0b6805a000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=435a92a451399b17 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.509+0000 7f0b66a00640 1 -- 172.21.15.86:0/3645367705 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b68086040 con 0x55723e7e3c30 2024-03-06T07:33:25.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.509+0000 7f0b66a00640 1 -- 172.21.15.86:0/3645367705 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b68011dc0 con 0x55723e7e3c30 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.509+0000 7f0b66a00640 1 -- 172.21.15.86:0/3645367705 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b68070040 con 0x55723e7e3c30 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.509+0000 7f0b809cc740 1 -- 172.21.15.86:0/3645367705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 msgr2=0x55723e7ec380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.509+0000 7f0b809cc740 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7ec380 secure :-1 s=READY pgs=1324 cs=0 l=1 rev1=1 crypto rx=0x7f0b68088270 tx=0x7f0b6805a000 comp rx=0 tx=0).stop 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 -- 172.21.15.86:0/3645367705 shutdown_connections 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e7c39a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 0x55723e7ec8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 --2- 172.21.15.86:0/3645367705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7ec380 unknown :-1 s=CLOSED pgs=1324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 -- 172.21.15.86:0/3645367705 >> 172.21.15.86:0/3645367705 conn(0x55723e7bee70 msgr2=0x55723e7c2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 -- 172.21.15.86:0/3645367705 shutdown_connections 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 -- 172.21.15.86:0/3645367705 wait complete. 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.510+0000 7f0b809cc740 1 -- start start 2024-03-06T07:33:25.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.511+0000 7f0b809cc740 1 Processor -- start 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7e56b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 0x55723e7eaa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b809cc740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e80aab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b809cc740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55723e7f1a50 con 0x55723e7e3c30 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55723e7f15d0 con 0x55723e7eb990 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b809cc740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55723e7f2050 con 0x55723e7ebd70 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b809cc740 10 client.0 _ll_register_callbacks cb 0x7f0b6000a5c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7e56b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7e56b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60726/0 (socket says 172.21.15.86:60726) 2024-03-06T07:33:25.518 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 -- 172.21.15.86:0/4271512208 learned_addr learned my addr 172.21.15.86:0/4271512208 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 -- 172.21.15.86:0/4271512208 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 msgr2=0x55723e80aab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e80aab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 -- 172.21.15.86:0/4271512208 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 msgr2=0x55723e7eaa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 0x55723e7eaa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 -- 172.21.15.86:0/4271512208 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55723e7e5bf0 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b76a00640 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7e56b0 secure :-1 s=READY pgs=1325 cs=0 l=1 rev1=1 crypto rx=0x7f0b6800bdb0 tx=0x7f0b68006d70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=26475921933c92bf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.512+0000 7f0b5be00640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b5be00640 1 -- 172.21.15.86:0/4271512208 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b68086040 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b5be00640 1 -- 172.21.15.86:0/4271512208 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f0b68056880 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0b68011dc0 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b5be00640 1 -- 172.21.15.86:0/4271512208 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=84) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0b680172c0 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 10 client.7764 fetch_fsmap learned FSMap version 84 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=84,monmap=2+}) v3 -- 0x55723e809e10 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b5be00640 1 -- 172.21.15.86:0/4271512208 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b68011dc0 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b5be00640 1 -- 172.21.15.86:0/4271512208 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 84) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f0b68054bf0 con 0x55723e7e3c30 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 10 client.7764 fetch_fsmap finished waiting for FSMap version 84 2024-03-06T07:33:25.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 10 client.7764 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 20 client.7764 populate_metadata read hostname 'smithi086' 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e809220 0x55723e80aff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 4 client.7764 mds_command: new command op to 7722 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.513+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 --> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55723e72fe90 con 0x55723e809220 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e809220 0x55723e80aff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b6fe00640 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e809220 0x55723e80aff0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d55cb44c1d11ea1 server_cookie=fb6c93653cd35fe0 in_seq=0 out_seq=0 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b5be00640 10 client.7764 ms_handle_connect on v2:172.21.15.86:6836/2212362187 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b5be00640 1 -- 172.21.15.86:0/4271512208 <== mds.0 v2:172.21.15.86:6836/2212362187 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+326 (crc 0 0 0) 0x55723e72fe90 con 0x55723e809220 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b5be00640 10 client.7764 handle_command_reply: tid=0 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b809cc740 1 client.7764 shutdown 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 msgr2=0x55723e7e56b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.514+0000 7f0b809cc740 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7e56b0 secure :-1 s=READY pgs=1325 cs=0 l=1 rev1=1 crypto rx=0x7f0b6800bdb0 tx=0x7f0b68006d70 comp rx=0 tx=0).stop 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 shutdown_connections 2024-03-06T07:33:25.520 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:6836/2212362187,v1:172.21.15.86:6837/2212362187] conn(0x55723e809220 0x55723e80aff0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55723e7ebd70 0x55723e80aab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55723e7eb990 0x55723e7eaa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 --2- 172.21.15.86:0/4271512208 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55723e7e3c30 0x55723e7e56b0 unknown :-1 s=CLOSED pgs=1325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 >> 172.21.15.86:0/4271512208 conn(0x55723e7bee70 msgr2=0x55723e82d0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b5be00640 0 client.7764 ms_handle_reset on v2:172.21.15.86:6836/2212362187 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 shutdown_connections 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 1 -- 172.21.15.86:0/4271512208 wait complete. 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b809cc740 20 client.7764 trim_cache size 0 max 16384 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:33:25.521 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 7722, 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 3.9606334689999998, 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 84, 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 114, 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 114, 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 4.742646701 2024-03-06T07:33:25.522 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b5406b500 msgr2=0x7f0b5406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b77e00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b5406b500 0x7f0b5406d9c0 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7f0b60006fd0 tx=0x7f0b60008040 comp rx=0 tx=0).stop 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 msgr2=0x7f0b78143520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.515+0000 7f0b77e00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b78143520 secure :-1 s=READY pgs=1321 cs=0 l=1 rev1=1 crypto rx=0x7f0b6800b3f0 tx=0x7f0b6800b8b0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 shutdown_connections 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0b5406b500 0x7f0b5406d9c0 unknown :-1 s=CLOSED pgs=573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0b7815f8f0 0x7f0b781e77d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0b78156320 0x7f0b781401c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 --2- 172.21.15.86:0/268643479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0b7806afc0 0x7f0b78143520 unknown :-1 s=CLOSED pgs=1321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 >> 172.21.15.86:0/268643479 conn(0x7f0b7806c0d0 msgr2=0x7f0b7815ac90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 shutdown_connections 2024-03-06T07:33:25.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.516+0000 7f0b77e00640 1 -- 172.21.15.86:0/268643479 wait complete. 2024-03-06T07:33:25.535 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:33:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.728+0000 7f97fdc00640 1 Processor -- start 2024-03-06T07:33:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.728+0000 7f97fdc00640 1 -- start start 2024-03-06T07:33:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.729+0000 7f97fdc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f97f80cc9d0 0x7f97f806d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.729+0000 7f97fdc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f97f815c240 con 0x7f97f80aedc0 2024-03-06T07:33:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.729+0000 7f97fdc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f97f815c500 con 0x7f97f81561f0 2024-03-06T07:33:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.729+0000 7f97fdc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97f815ce30 con 0x7f97f80cc9d0 2024-03-06T07:33:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97fd200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f97f80cc9d0 0x7f97f806d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97ef400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f97f80aedc0 0x7f97f80cc2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36764/0 (socket says 172.21.15.86:36764) 2024-03-06T07:33:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97fd200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f97f80cc9d0 0x7f97f806d350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60738/0 (socket says 172.21.15.86:60738) 2024-03-06T07:33:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97ef400640 1 -- 172.21.15.86:0/1521734100 learned_addr learned my addr 172.21.15.86:0/1521734100 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97fd200640 1 -- 172.21.15.86:0/1521734100 >> v1:172.21.15.194:6789/0 conn(0x7f97f81561f0 legacy=0x7f97f80ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97fd200640 1 -- 172.21.15.86:0/1521734100 >> v1:172.21.15.86:6789/0 conn(0x7f97f80aedc0 legacy=0x7f97f80cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97fd200640 1 -- 172.21.15.86:0/1521734100 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97f815c7e0 con 0x7f97f80cc9d0 2024-03-06T07:33:25.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.730+0000 7f97f7400640 1 -- 172.21.15.86:0/1521734100 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4012997678 0 0) 0x7f97f815c240 con 0x7f97f80aedc0 2024-03-06T07:33:25.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.731+0000 7f97fd200640 1 --2- 172.21.15.86:0/1521734100 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f80cc9d0 0x7f97f806d350 secure :-1 s=READY pgs=1327 cs=0 l=1 rev1=1 crypto rx=0x7f97f815cfb0 tx=0x7f97f002bf30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=67d664d0b9918886 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97f7400640 1 -- 172.21.15.86:0/1521734100 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97f0039070 con 0x7f97f80cc9d0 2024-03-06T07:33:25.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97f7400640 1 -- 172.21.15.86:0/1521734100 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f97f002cce0 con 0x7f97f80cc9d0 2024-03-06T07:33:25.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97f7400640 1 -- 172.21.15.86:0/1521734100 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97f0034660 con 0x7f97f80cc9d0 2024-03-06T07:33:25.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97fdc00640 1 -- 172.21.15.86:0/1521734100 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f80cc9d0 msgr2=0x7f97f806d350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97fdc00640 1 --2- 172.21.15.86:0/1521734100 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f80cc9d0 0x7f97f806d350 secure :-1 s=READY pgs=1327 cs=0 l=1 rev1=1 crypto rx=0x7f97f815cfb0 tx=0x7f97f002bf30 comp rx=0 tx=0).stop 2024-03-06T07:33:25.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97fdc00640 1 -- 172.21.15.86:0/1521734100 shutdown_connections 2024-03-06T07:33:25.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97fdc00640 1 --2- 172.21.15.86:0/1521734100 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f80cc9d0 0x7f97f806d350 unknown :-1 s=CLOSED pgs=1327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97fdc00640 1 -- 172.21.15.86:0/1521734100 >> 172.21.15.86:0/1521734100 conn(0x7f97f806afc0 msgr2=0x7f97f806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97fdc00640 1 -- 172.21.15.86:0/1521734100 shutdown_connections 2024-03-06T07:33:25.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.732+0000 7f97fdc00640 1 -- 172.21.15.86:0/1521734100 wait complete. 2024-03-06T07:33:25.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.734+0000 7f97fdc00640 1 Processor -- start 2024-03-06T07:33:25.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.734+0000 7f97fdc00640 1 -- start start 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fdc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97f80aedc0 0x7f97f8145530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fdc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97f80cc9d0 0x7f97f8156900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fdc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f81561f0 0x7f97f815aeb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fdc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97f8156750 con 0x7f97f81561f0 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fdc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97f81565d0 con 0x7f97f80aedc0 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fdc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97f806d890 con 0x7f97f80cc9d0 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f81561f0 0x7f97f815aeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f81561f0 0x7f97f815aeb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60750/0 (socket says 172.21.15.86:60750) 2024-03-06T07:33:25.738 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.735+0000 7f97fd200640 1 -- 172.21.15.86:0/2989376653 learned_addr learned my addr 172.21.15.86:0/2989376653 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97fd200640 1 -- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97f80cc9d0 msgr2=0x7f97f8156900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97fd200640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97f80cc9d0 0x7f97f8156900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97f7e00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97f80aedc0 0x7f97f8145530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97fd200640 1 -- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97f80aedc0 msgr2=0x7f97f8145530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97fd200640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97f80aedc0 0x7f97f8145530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97fd200640 1 -- 172.21.15.86:0/2989376653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97f8145a70 con 0x7f97f81561f0 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97f7e00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97f80aedc0 0x7f97f8145530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.736+0000 7f97fd200640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f81561f0 0x7f97f815aeb0 secure :-1 s=READY pgs=1328 cs=0 l=1 rev1=1 crypto rx=0x7f97f0037040 tx=0x7f97f0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.737+0000 7f97f4c00640 1 -- 172.21.15.86:0/2989376653 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97f0039050 con 0x7f97f81561f0 2024-03-06T07:33:25.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.737+0000 7f97f4c00640 1 -- 172.21.15.86:0/2989376653 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f97f0045070 con 0x7f97f81561f0 2024-03-06T07:33:25.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.737+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97f8157150 con 0x7f97f81561f0 2024-03-06T07:33:25.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.737+0000 7f97f4c00640 1 -- 172.21.15.86:0/2989376653 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97f003e810 con 0x7f97f81561f0 2024-03-06T07:33:25.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.737+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f97f8157710 con 0x7f97f81561f0 2024-03-06T07:33:25.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.738+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f97f804f330 con 0x7f97f81561f0 2024-03-06T07:33:25.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.740+0000 7f97f4c00640 1 -- 172.21.15.86:0/2989376653 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f97f003e9b0 con 0x7f97f81561f0 2024-03-06T07:33:25.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.741+0000 7f97f4c00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f97d406b4d0 0x7f97d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:25.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.741+0000 7f97f4c00640 1 -- 172.21.15.86:0/2989376653 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f97f00ad400 con 0x7f97f81561f0 2024-03-06T07:33:25.743 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.741+0000 7f97f7e00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f97d406b4d0 0x7f97d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:25.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.742+0000 7f97f7e00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f97d406b4d0 0x7f97d406d990 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7f97e8000f60 tx=0x7f97e8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:25.748 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.745+0000 7f97f4c00640 1 -- 172.21.15.86:0/2989376653 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f97f007c330 con 0x7f97f81561f0 2024-03-06T07:33:25.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.981+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f97f80b2b10 con 0x7f97f81561f0 2024-03-06T07:33:25.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.981+0000 7f97f4c00640 1 -- 172.21.15.86:0/2989376653 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v84) v1 ==== 53+0+73 (secure 0 0 0) 0x7f97f007bc30 con 0x7f97f81561f0 2024-03-06T07:33:25.984 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:33:25.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f97d406b4d0 msgr2=0x7f97d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f97d406b4d0 0x7f97d406d990 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7f97e8000f60 tx=0x7f97e8009040 comp rx=0 tx=0).stop 2024-03-06T07:33:25.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f81561f0 msgr2=0x7f97f815aeb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:25.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f81561f0 0x7f97f815aeb0 secure :-1 s=READY pgs=1328 cs=0 l=1 rev1=1 crypto rx=0x7f97f0037040 tx=0x7f97f0004a40 comp rx=0 tx=0).stop 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 shutdown_connections 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f97d406b4d0 0x7f97d406d990 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97f81561f0 0x7f97f815aeb0 unknown :-1 s=CLOSED pgs=1328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97f80cc9d0 0x7f97f8156900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 --2- 172.21.15.86:0/2989376653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f97f80aedc0 0x7f97f8145530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.984+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 >> 172.21.15.86:0/2989376653 conn(0x7f97f806afc0 msgr2=0x7f97f80cc2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.985+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 shutdown_connections 2024-03-06T07:33:25.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:25.985+0000 7f97fdc00640 1 -- 172.21.15.86:0/2989376653 wait complete. 2024-03-06T07:33:26.013 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:33:26.013 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:33:26.013 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:33:26.013 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:33:26.013 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:26.013 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:33:26.014 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:33:26.014 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:33:26.014 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:33:26.014 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.014 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:26.063 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 42061sec preferred_lft 42061sec 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:26.064 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:33:26.065 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.065 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:33:26.192 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:33:26.197 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.197 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:33:26.251 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:33:26.252 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:33:26.252 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:33:26.252 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:33:26.252 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:33:26.252 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.252 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:33:26.252 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:33:26.252 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:33:26.252 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:33:26.253 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:33:26.253 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:33:26.431 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.431 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:33:26.477 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:33:26.477 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:33:26.477 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:33:26.477 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.477 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:33:26.569 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:33:26.566+0000 7f9c21600640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:33:26.602 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:33:26.602 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:26.650 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:33:26.650 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:33:26.650 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:26.742 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:33:26.743 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.743 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:33:26.870 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:33:26.870 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:33:26.901 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:33:26.901 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:33:26.901 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:33:26.901 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:33:26.901 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:33:26.904 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:33:26.904 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:33:26.904 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:33:26.904 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:33:26.904 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:33:26.908 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:33:26.908 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.908 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:33:26.988 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:26.988 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:33:27.041 INFO:teuthology.orchestra.run.smithi086.stdout:client7776 2024-03-06T07:33:27.041 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:33:27.041 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:33:27.092 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:33:27.092 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f dir_1/file_a) 2024-03-06T07:33:27.191 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:27.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.411+0000 7f8acbe00640 1 Processor -- start 2024-03-06T07:33:27.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.412+0000 7f8acbe00640 1 -- start start 2024-03-06T07:33:27.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.413+0000 7f8acbe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8acc0cc9d0 0x7f8acc06d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.413+0000 7f8acbe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8acc15c240 con 0x7f8acc1561f0 2024-03-06T07:33:27.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.413+0000 7f8acbe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8acc15c500 con 0x7f8acc0aedc0 2024-03-06T07:33:27.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.413+0000 7f8acbe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8acc15ce30 con 0x7f8acc0cc9d0 2024-03-06T07:33:27.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.413+0000 7f8acaa00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8acc1561f0 0x7f8acc0ae6b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55252/0 (socket says 172.21.15.86:55252) 2024-03-06T07:33:27.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.413+0000 7f8acaa00640 1 -- 172.21.15.86:0/528518783 learned_addr learned my addr 172.21.15.86:0/528518783 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3752666127 0 0) 0x7f8acc15c240 con 0x7f8acc1561f0 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8ab0003520 con 0x7f8acc1561f0 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8acb400640 1 --2- 172.21.15.86:0/528518783 >> v2:172.21.15.194:3300/0 conn(0x7f8acc0cc9d0 0x7f8acc06d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3293588476 0 0) 0x7f8ab0003520 con 0x7f8acc1561f0 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 >> v2:172.21.15.194:3300/0 conn(0x7f8acc0cc9d0 msgr2=0x7f8acc06d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 --2- 172.21.15.86:0/528518783 >> v2:172.21.15.194:3300/0 conn(0x7f8acc0cc9d0 0x7f8acc06d350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 >> v1:172.21.15.194:6789/0 conn(0x7f8acc0aedc0 legacy=0x7f8acc0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8acc15c7e0 con 0x7f8acc1561f0 2024-03-06T07:33:27.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8abc001670 con 0x7f8acc1561f0 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 >> v1:172.21.15.86:6789/0 conn(0x7f8acc1561f0 legacy=0x7f8acc0ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8ab00039b0 0x7f8ab0003df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab0004800 0x7f8ab0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8ab0005210 0x7f8ab000fdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8acc15ce30 con 0x7f8ab00039b0 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ab0010550 con 0x7f8ab0005210 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.414+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8ab0010750 con 0x7f8ab0004800 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.415+0000 7f8acaa00640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8ab00039b0 0x7f8ab0003df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.415+0000 7f8acaa00640 1 -- 172.21.15.86:0/528518783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab0004800 msgr2=0x7f8ab0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.415+0000 7f8acaa00640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab0004800 0x7f8ab0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.415+0000 7f8acaa00640 1 -- 172.21.15.86:0/528518783 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8ab0005210 msgr2=0x7f8ab000fdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.415+0000 7f8acaa00640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8ab0005210 0x7f8ab000fdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.415+0000 7f8acb400640 1 -- 172.21.15.86:0/528518783 reap_dead start 2024-03-06T07:33:27.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.415+0000 7f8acaa00640 1 -- 172.21.15.86:0/528518783 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8ab0003520 con 0x7f8ab00039b0 2024-03-06T07:33:27.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.416+0000 7f8acaa00640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8ab00039b0 0x7f8ab0003df0 secure :-1 s=READY pgs=1331 cs=0 l=1 rev1=1 crypto rx=0x7f8abc00c240 tx=0x7f8ab0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fcf18d06b9012e9e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:27.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.416+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8abc007d10 con 0x7f8ab00039b0 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8abc030070 con 0x7f8ab00039b0 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8aca000640 1 -- 172.21.15.86:0/528518783 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8abc02cda0 con 0x7f8ab00039b0 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8acbe00640 1 -- 172.21.15.86:0/528518783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8ab00039b0 msgr2=0x7f8ab0003df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8acbe00640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8ab00039b0 0x7f8ab0003df0 secure :-1 s=READY pgs=1331 cs=0 l=1 rev1=1 crypto rx=0x7f8abc00c240 tx=0x7f8ab0005830 comp rx=0 tx=0).stop 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8acbe00640 1 -- 172.21.15.86:0/528518783 shutdown_connections 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8acbe00640 1 --2- 172.21.15.86:0/528518783 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8ab00039b0 0x7f8ab0003df0 unknown :-1 s=CLOSED pgs=1331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8acbe00640 1 -- 172.21.15.86:0/528518783 >> 172.21.15.86:0/528518783 conn(0x7f8acc06afc0 msgr2=0x7f8acc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8acbe00640 1 -- 172.21.15.86:0/528518783 shutdown_connections 2024-03-06T07:33:27.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.417+0000 7f8acbe00640 1 -- 172.21.15.86:0/528518783 wait complete. 2024-03-06T07:33:27.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.419+0000 7f8acbe00640 1 Processor -- start 2024-03-06T07:33:27.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.419+0000 7f8acbe00640 1 -- start start 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8acbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab00039b0 0x7f8acc0cd290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8acbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8acc0cee60 0x7f8acc149700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8acbe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8acc158040 0x7f8acc06d350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8acbe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8acc156750 con 0x7f8acc0cee60 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8acbe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8acc1565d0 con 0x7f8acc158040 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8acbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8acc0cc2c0 con 0x7f8ab00039b0 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8ac2000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8acc0cee60 0x7f8acc149700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8ac2000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8acc0cee60 0x7f8acc149700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54606/0 (socket says 172.21.15.86:54606) 2024-03-06T07:33:27.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.420+0000 7f8ac2000640 1 -- 172.21.15.86:0/3873268527 learned_addr learned my addr 172.21.15.86:0/3873268527 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:27.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac2000640 1 -- 172.21.15.86:0/3873268527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab00039b0 msgr2=0x7f8acc0cd290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:27.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8acaa00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab00039b0 0x7f8acc0cd290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:27.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac2000640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab00039b0 0x7f8acc0cd290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac2000640 1 -- 172.21.15.86:0/3873268527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8acc158040 msgr2=0x7f8acc06d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac2000640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8acc158040 0x7f8acc06d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac2000640 1 -- 172.21.15.86:0/3873268527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8acc15d480 con 0x7f8acc0cee60 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac2000640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8acc0cee60 0x7f8acc149700 secure :-1 s=READY pgs=1332 cs=0 l=1 rev1=1 crypto rx=0x7f8ab400c3f0 tx=0x7f8ab400c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac3400640 1 -- 172.21.15.86:0/3873268527 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ab40052d0 con 0x7f8acc0cee60 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac3400640 1 -- 172.21.15.86:0/3873268527 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ab4005490 con 0x7f8acc0cee60 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8ac3400640 1 -- 172.21.15.86:0/3873268527 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ab4007260 con 0x7f8acc0cee60 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8acc0cd870 con 0x7f8acc0cee60 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.421+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8acc0cde30 con 0x7f8acc0cee60 2024-03-06T07:33:27.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.423+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8acc0b31e0 con 0x7f8acc0cee60 2024-03-06T07:33:27.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.424+0000 7f8ac3400640 1 -- 172.21.15.86:0/3873268527 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8ab4007400 con 0x7f8acc0cee60 2024-03-06T07:33:27.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.424+0000 7f8ac3400640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8aa806b4d0 0x7f8aa806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:27.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.425+0000 7f8ac3400640 1 -- 172.21.15.86:0/3873268527 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f8ab4089750 con 0x7f8acc0cee60 2024-03-06T07:33:27.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.425+0000 7f8acaa00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8aa806b4d0 0x7f8aa806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:27.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.426+0000 7f8acaa00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8aa806b4d0 0x7f8aa806d990 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f8abc034000 tx=0x7f8abc039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:27.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.429+0000 7f8ac3400640 1 -- 172.21.15.86:0/3873268527 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8acc0b31e0 con 0x7f8acc0cee60 2024-03-06T07:33:27.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.672+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8acc04f330 con 0x7f8acc0cee60 2024-03-06T07:33:27.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.673+0000 7f8ac3400640 1 -- 172.21.15.86:0/3873268527 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f8ab40584d0 con 0x7f8acc0cee60 2024-03-06T07:33:27.677 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:27.677 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:33:27.677 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:33:27.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.675+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8aa806b4d0 msgr2=0x7f8aa806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:27.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.675+0000 7f8acbe00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8aa806b4d0 0x7f8aa806d990 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f8abc034000 tx=0x7f8abc039040 comp rx=0 tx=0).stop 2024-03-06T07:33:27.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.675+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8acc0cee60 msgr2=0x7f8acc149700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:27.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8acc0cee60 0x7f8acc149700 secure :-1 s=READY pgs=1332 cs=0 l=1 rev1=1 crypto rx=0x7f8ab400c3f0 tx=0x7f8ab400c8b0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 shutdown_connections 2024-03-06T07:33:27.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8aa806b4d0 0x7f8aa806d990 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8acc158040 0x7f8acc06d350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8acc0cee60 0x7f8acc149700 unknown :-1 s=CLOSED pgs=1332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 --2- 172.21.15.86:0/3873268527 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8ab00039b0 0x7f8acc0cd290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:27.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 >> 172.21.15.86:0/3873268527 conn(0x7f8acc06afc0 msgr2=0x7f8acc15b350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:27.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 shutdown_connections 2024-03-06T07:33:27.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:27.676+0000 7f8acbe00640 1 -- 172.21.15.86:0/3873268527 wait complete. 2024-03-06T07:33:27.699 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-06T07:33:27.924 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"num_strays":1}} 2024-03-06T07:33:27.924 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 1 } } 2024-03-06T07:33:27.925 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:28.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.140+0000 7f338e200640 1 Processor -- start 2024-03-06T07:33:28.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.140+0000 7f338e200640 1 -- start start 2024-03-06T07:33:28.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.141+0000 7f338e200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3388144660 0x7f3388071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.141+0000 7f338e200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f33880cabf0 con 0x7f33880cf3e0 2024-03-06T07:33:28.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.141+0000 7f338e200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f33880ce900 con 0x7f33880ca5d0 2024-03-06T07:33:28.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.141+0000 7f338e200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f33880cebe0 con 0x7f3388144660 2024-03-06T07:33:28.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.142+0000 7f337ea00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f33880cf3e0 0x7f33880c9ec0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55254/0 (socket says 172.21.15.86:55254) 2024-03-06T07:33:28.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.142+0000 7f337ea00640 1 -- 172.21.15.86:0/941247383 learned_addr learned my addr 172.21.15.86:0/941247383 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:28.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.142+0000 7f3387400640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3388144660 0x7f3388071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.142+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3612328614 0 0) 0x7f33880cabf0 con 0x7f33880cf3e0 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.142+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f336c003520 con 0x7f33880cf3e0 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3366706145 0 0) 0x7f336c003520 con 0x7f33880cf3e0 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3388144660 msgr2=0x7f3388071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3388144660 0x7f3388071f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 >> v1:172.21.15.194:6789/0 conn(0x7f33880ca5d0 legacy=0x7f33880cc0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33880ceec0 con 0x7f33880cf3e0 2024-03-06T07:33:28.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3370001670 con 0x7f33880cf3e0 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 >> v1:172.21.15.86:6789/0 conn(0x7f33880cf3e0 legacy=0x7f33880c9ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3388144660 0x7f336c0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f336c003f60 0x7f336c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c004d40 0x7f336c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33880cebe0 con 0x7f336c003f60 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f336c00baa0 con 0x7f336c004d40 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f336c00bca0 con 0x7f3388144660 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.143+0000 7f337ea00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f336c003f60 0x7f336c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:28.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.144+0000 7f337ea00640 1 -- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3388144660 msgr2=0x7f336c0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:28.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.144+0000 7f337ea00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3388144660 0x7f336c0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.144+0000 7f337ea00640 1 -- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c004d40 msgr2=0x7f336c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:28.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.144+0000 7f337ea00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c004d40 0x7f336c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.144+0000 7f337ea00640 1 -- 172.21.15.86:0/941247383 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f336c003520 con 0x7f336c003f60 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.145+0000 7f337ea00640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f336c003f60 0x7f336c004800 secure :-1 s=READY pgs=1334 cs=0 l=1 rev1=1 crypto rx=0x7f337000c240 tx=0x7f336c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1a14bcc32696f76b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.145+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3370007cf0 con 0x7f336c003f60 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.145+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3370030070 con 0x7f336c003f60 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.145+0000 7f3386a00640 1 -- 172.21.15.86:0/941247383 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3370005d70 con 0x7f336c003f60 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.145+0000 7f338e200640 1 -- 172.21.15.86:0/941247383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f336c003f60 msgr2=0x7f336c004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.145+0000 7f338e200640 1 --2- 172.21.15.86:0/941247383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f336c003f60 0x7f336c004800 secure :-1 s=READY pgs=1334 cs=0 l=1 rev1=1 crypto rx=0x7f337000c240 tx=0x7f336c005830 comp rx=0 tx=0).stop 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.145+0000 7f3387e00640 1 -- 172.21.15.86:0/941247383 reap_dead start 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.146+0000 7f338e200640 1 -- 172.21.15.86:0/941247383 shutdown_connections 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.146+0000 7f338e200640 1 -- 172.21.15.86:0/941247383 >> 172.21.15.86:0/941247383 conn(0x7f33880d0b90 msgr2=0x7f33880d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:28.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.146+0000 7f338e200640 1 -- 172.21.15.86:0/941247383 shutdown_connections 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.146+0000 7f338e200640 1 -- 172.21.15.86:0/941247383 wait complete. 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.147+0000 7f338e200640 1 Processor -- start 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.147+0000 7f338e200640 1 -- start start 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.148+0000 7f338e200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c003f60 0x7f3388139760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.148+0000 7f338e200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f338813a8c0 0x7f338813ad20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.148+0000 7f338e200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f33881604a0 0x7f3388160920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.148+0000 7f338e200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33880cc980 con 0x7f33881604a0 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.148+0000 7f338e200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33880cc800 con 0x7f336c003f60 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.148+0000 7f338e200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f33880ccb00 con 0x7f338813a8c0 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f33881604a0 0x7f3388160920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f33881604a0 0x7f3388160920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54618/0 (socket says 172.21.15.86:54618) 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 -- 172.21.15.86:0/2880508987 learned_addr learned my addr 172.21.15.86:0/2880508987 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:28.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 -- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f338813a8c0 msgr2=0x7f338813ad20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:28.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f337ea00640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f338813a8c0 0x7f338813ad20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:28.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f338813a8c0 0x7f338813ad20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387400640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c003f60 0x7f3388139760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:28.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 -- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c003f60 msgr2=0x7f3388139760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:28.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c003f60 0x7f3388139760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 -- 172.21.15.86:0/2880508987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3388160e60 con 0x7f33881604a0 2024-03-06T07:33:28.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.149+0000 7f3387e00640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f33881604a0 0x7f3388160920 secure :-1 s=READY pgs=1335 cs=0 l=1 rev1=1 crypto rx=0x7f338806bbe0 tx=0x7f338000cb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:28.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.150+0000 7f337fe00640 1 -- 172.21.15.86:0/2880508987 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3380017070 con 0x7f33881604a0 2024-03-06T07:33:28.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.150+0000 7f337fe00640 1 -- 172.21.15.86:0/2880508987 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3380005d60 con 0x7f33881604a0 2024-03-06T07:33:28.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.150+0000 7f337fe00640 1 -- 172.21.15.86:0/2880508987 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f338000a850 con 0x7f33881604a0 2024-03-06T07:33:28.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.150+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f33880b4830 con 0x7f33881604a0 2024-03-06T07:33:28.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.150+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f33880b4df0 con 0x7f33881604a0 2024-03-06T07:33:28.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.151+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f338804f330 con 0x7f33881604a0 2024-03-06T07:33:28.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.152+0000 7f337fe00640 1 -- 172.21.15.86:0/2880508987 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f338000a9f0 con 0x7f33881604a0 2024-03-06T07:33:28.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.153+0000 7f337fe00640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f336406b4d0 0x7f336406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:28.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.154+0000 7f337fe00640 1 -- 172.21.15.86:0/2880508987 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f3380089380 con 0x7f33881604a0 2024-03-06T07:33:28.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.154+0000 7f3387400640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f336406b4d0 0x7f336406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:28.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.154+0000 7f3387400640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f336406b4d0 0x7f336406d990 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7f3378004770 tx=0x7f3378008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:28.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.158+0000 7f337fe00640 1 -- 172.21.15.86:0/2880508987 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3380058110 con 0x7f33881604a0 2024-03-06T07:33:28.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.390+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3388074030 con 0x7f33881604a0 2024-03-06T07:33:28.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.391+0000 7f337fe00640 1 -- 172.21.15.86:0/2880508987 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f3380057a10 con 0x7f33881604a0 2024-03-06T07:33:28.394 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:28.394 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:33:28.395 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:33:28.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.393+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f336406b4d0 msgr2=0x7f336406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.393+0000 7f338e200640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f336406b4d0 0x7f336406d990 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7f3378004770 tx=0x7f3378008040 comp rx=0 tx=0).stop 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.393+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f33881604a0 msgr2=0x7f3388160920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.393+0000 7f338e200640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f33881604a0 0x7f3388160920 secure :-1 s=READY pgs=1335 cs=0 l=1 rev1=1 crypto rx=0x7f338806bbe0 tx=0x7f338000cb90 comp rx=0 tx=0).stop 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 shutdown_connections 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f336406b4d0 0x7f336406d990 unknown :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f33881604a0 0x7f3388160920 unknown :-1 s=CLOSED pgs=1335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f338813a8c0 0x7f338813ad20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 --2- 172.21.15.86:0/2880508987 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f336c003f60 0x7f3388139760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 >> 172.21.15.86:0/2880508987 conn(0x7f33880d0b90 msgr2=0x7f338806dc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 shutdown_connections 2024-03-06T07:33:28.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:28.394+0000 7f338e200640 1 -- 172.21.15.86:0/2880508987 wait complete. 2024-03-06T07:33:28.415 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_created 2024-03-06T07:33:28.634 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_created":1}} 2024-03-06T07:33:28.635 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 1 } } 2024-03-06T07:33:54.349 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:33:54.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:33:58.636 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data ls 2024-03-06T07:33:58.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.741+0000 7f5e49a39f40 1 Processor -- start 2024-03-06T07:33:58.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.741+0000 7f5e49a39f40 1 -- start start 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e49a39f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff3bd7c0 0x55abff493440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e49a39f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff408b30 0x55abff495140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e49a39f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff494250 0x55abff490150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e49a39f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55abff365fd0 con 0x55abff408b30 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e49a39f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55abff369f30 con 0x55abff494250 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e49a39f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55abff3c1800 con 0x55abff3bd7c0 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff408b30 0x55abff495140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff408b30 0x55abff495140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35864/0 (socket says 172.21.15.86:35864) 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 -- 172.21.15.86:0/2903592725 learned_addr learned my addr 172.21.15.86:0/2903592725 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e46400640 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff3bd7c0 0x55abff493440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.745 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 -- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff3bd7c0 msgr2=0x55abff493440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff3bd7c0 0x55abff493440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 -- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff494250 msgr2=0x55abff490150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e46e00640 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff494250 0x55abff490150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff494250 0x55abff490150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.742+0000 7f5e45a00640 1 -- 172.21.15.86:0/2903592725 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55abff47c8f0 con 0x55abff408b30 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.743+0000 7f5e45a00640 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff408b30 0x55abff495140 secure :-1 s=READY pgs=1338 cs=0 l=1 rev1=1 crypto rx=0x7f5e34002de0 tx=0x7f5e3400b5b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7be2666829dadb07 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.743+0000 7f5e3fe00640 1 -- 172.21.15.86:0/2903592725 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e3400f400 con 0x55abff408b30 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.743+0000 7f5e3fe00640 1 -- 172.21.15.86:0/2903592725 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5e34014070 con 0x55abff408b30 2024-03-06T07:33:58.746 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.743+0000 7f5e3fe00640 1 -- 172.21.15.86:0/2903592725 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e3400f7e0 con 0x55abff408b30 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 -- 172.21.15.86:0/2903592725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff408b30 msgr2=0x55abff495140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff408b30 0x55abff495140 secure :-1 s=READY pgs=1338 cs=0 l=1 rev1=1 crypto rx=0x7f5e34002de0 tx=0x7f5e3400b5b0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 -- 172.21.15.86:0/2903592725 shutdown_connections 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff494250 0x55abff490150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff408b30 0x55abff495140 unknown :-1 s=CLOSED pgs=1338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/2903592725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff3bd7c0 0x55abff493440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 -- 172.21.15.86:0/2903592725 >> 172.21.15.86:0/2903592725 conn(0x55abff3ecbe0 msgr2=0x55abff3eecf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 -- 172.21.15.86:0/2903592725 shutdown_connections 2024-03-06T07:33:58.749 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.744+0000 7f5e49a39f40 1 -- 172.21.15.86:0/2903592725 wait complete. 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.745+0000 7f5e49a39f40 1 Processor -- start 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.745+0000 7f5e49a39f40 1 -- start start 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e49a39f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff3bd7c0 0x55abff495330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e49a39f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff408b30 0x55abff478360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e49a39f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff494250 0x55abff4788a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e49a39f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55abff47d1e0 con 0x55abff494250 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e49a39f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55abff491fb0 con 0x55abff3bd7c0 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e49a39f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55abff491000 con 0x55abff408b30 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff494250 0x55abff4788a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff494250 0x55abff4788a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35878/0 (socket says 172.21.15.86:35878) 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 learned_addr learned my addr 172.21.15.86:0/397544253 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff3bd7c0 0x55abff495330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff408b30 0x55abff478360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff408b30 msgr2=0x55abff478360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55abff408b30 0x55abff478360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff3bd7c0 msgr2=0x55abff495330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff3bd7c0 0x55abff495330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55abff491a30 con 0x55abff494250 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.746+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55abff3bd7c0 0x55abff495330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.747+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff494250 0x55abff4788a0 secure :-1 s=READY pgs=1339 cs=0 l=1 rev1=1 crypto rx=0x7f5e40004180 tx=0x7f5e4000c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.747+0000 7f5e3ea00640 1 -- 172.21.15.86:0/397544253 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e40017070 con 0x55abff494250 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.747+0000 7f5e3ea00640 1 -- 172.21.15.86:0/397544253 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5e400032f0 con 0x55abff494250 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.747+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x55abff5332d0 con 0x55abff494250 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.747+0000 7f5e3ea00640 1 -- 172.21.15.86:0/397544253 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e40001870 con 0x55abff494250 2024-03-06T07:33:58.751 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.747+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55abff3be410 con 0x55abff494250 2024-03-06T07:33:58.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.749+0000 7f5e3ea00640 1 -- 172.21.15.86:0/397544253 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5e4001a070 con 0x55abff494250 2024-03-06T07:33:58.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.750+0000 7f5e3ea00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e28069eb0 0x7f5e2806c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.750+0000 7f5e3ea00640 1 -- 172.21.15.86:0/397544253 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f5e400886a0 con 0x55abff494250 2024-03-06T07:33:58.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.750+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e28069eb0 0x7f5e2806c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.750+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55abff495870 0x55abff3be930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.752 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.750+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:1 17.0 17:00000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.751+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e28069eb0 0x7f5e2806c370 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7f5e38054000 tx=0x7f5e3804e000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.751+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55abff495870 0x55abff3be930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.752+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55abff495870 0x55abff3be930 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.753+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 1 ==== osd_op_reply(1 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.753+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7f5e34019b40 0x7f5e3400f910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.753+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:2 17.20 17:04000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34019b40 2024-03-06T07:33:58.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.753+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7f5e34019b40 0x7f5e3400f910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.755 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.754+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7f5e34019b40 0x7f5e3400f910 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.754+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.1 v2:172.21.15.86:6809/2555210107 1 ==== osd_op_reply(2 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34019b40 2024-03-06T07:33:58.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.754+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f5e3805c3e0 0x7f5e3805c860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.754+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:3 17.10 17:08000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.756 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.755+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f5e3805c3e0 0x7f5e3805c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.755+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f5e3805c3e0 0x7f5e3805c860 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.756+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 1 ==== osd_op_reply(3 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.756+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f5e34026380 0x7f5e340267c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.756+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:4 17.30 17:0c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.756+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f5e34026380 0x7f5e340267c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.756+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f5e34026380 0x7f5e340267c0 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.757+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 1 ==== osd_op_reply(4 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.758 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.757+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:5 17.8 17:10000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.758+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 2 ==== osd_op_reply(5 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.758+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:6 17.28 17:14000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.759 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.758+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 2 ==== osd_op_reply(6 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.758+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:7 17.18 17:18000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.759+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 2 ==== osd_op_reply(7 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.760 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.759+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:8 17.38 17:1c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.759+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 3 ==== osd_op_reply(8 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.761 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.759+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:9 17.4 17:20000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.760+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 3 ==== osd_op_reply(9 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.760+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:10 17.24 17:24000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.760+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 3 ==== osd_op_reply(10 [pgnls start_epoch 116 out=49b] v110'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.760+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:11 17.14 17:28000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.760+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 4 ==== osd_op_reply(11 [pgnls start_epoch 116 out=49b] v110'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.762 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.761+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:12 17.34 17:2c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.761+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 4 ==== osd_op_reply(12 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.761+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:13 17.c 17:30000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.763 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.762+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 4 ==== osd_op_reply(13 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.762+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f5e34035950 0x7f5e340343d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.762+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:14 17.2c 17:34000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.762+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f5e34035950 0x7f5e340343d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.763+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f5e34035950 0x7f5e340343d0 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.764+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 1 ==== osd_op_reply(14 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.764+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:15 17.1c 17:38000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34019b40 2024-03-06T07:33:58.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.765+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.1 v2:172.21.15.86:6809/2555210107 2 ==== osd_op_reply(15 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34019b40 2024-03-06T07:33:58.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.765+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:16 17.3c 17:3c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.765+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 5 ==== osd_op_reply(16 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.765+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:17 17.2 17:40000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.766+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 2 ==== osd_op_reply(17 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.766+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7f5e3806d040 0x7f5e3806d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.766+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:18 17.22 17:44000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3806d040 2024-03-06T07:33:58.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.767+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7f5e3806d040 0x7f5e3806d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.768+0000 7f5e45a00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7f5e3806d040 0x7f5e3806d930 crc :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.769+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.5 v2:172.21.15.194:6816/1278478503 1 ==== osd_op_reply(18 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3806d040 2024-03-06T07:33:58.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.769+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:19 17.12 17:48000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.769+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 5 ==== osd_op_reply(19 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.771 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.769+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:20 17.32 17:4c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.770+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 5 ==== osd_op_reply(20 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.770+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:21 17.a 17:50000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.771+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 6 ==== osd_op_reply(21 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.771+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:22 17.2a 17:54000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.771+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 6 ==== osd_op_reply(22 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.771+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:23 17.1a 17:58000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.772+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 6 ==== osd_op_reply(23 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.772+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:24 17.3a 17:5c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34019b40 2024-03-06T07:33:58.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.773+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.1 v2:172.21.15.86:6809/2555210107 3 ==== osd_op_reply(24 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34019b40 2024-03-06T07:33:58.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.773+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7f5e380788e0 0x7f5e3806a390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.773+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:25 17.6 17:60000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e380788e0 2024-03-06T07:33:58.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.773+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7f5e380788e0 0x7f5e3806a390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.774+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7f5e380788e0 0x7f5e3806a390 crc :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.775+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.6 v2:172.21.15.194:6808/2611339787 1 ==== osd_op_reply(25 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e380788e0 2024-03-06T07:33:58.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.775+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:26 17.26 17:64000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.776+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 7 ==== osd_op_reply(26 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.776+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:27 17.16 17:68000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3806d040 2024-03-06T07:33:58.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.777+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.5 v2:172.21.15.194:6816/1278478503 2 ==== osd_op_reply(27 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3806d040 2024-03-06T07:33:58.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.777+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:28 17.36 17:6c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.777+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 8 ==== osd_op_reply(28 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.777+0000 7f5e46e00640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f5e4001c060 0x7f5e40055c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:58.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.777+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- osd_op(unknown.0.0:29 17.e 17:70000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e4001c060 2024-03-06T07:33:58.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.778+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f5e4001c060 0x7f5e40055c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:58.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.778+0000 7f5e46400640 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f5e4001c060 0x7f5e40055c40 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:58.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.779+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.2 v2:172.21.15.86:6808/2264489672 1 ==== osd_op_reply(29 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e4001c060 2024-03-06T07:33:58.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.779+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:30 17.2e 17:74000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e380788e0 2024-03-06T07:33:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.780+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.6 v2:172.21.15.194:6808/2611339787 2 ==== osd_op_reply(30 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e380788e0 2024-03-06T07:33:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.780+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:31 17.1e 17:78000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34019b40 2024-03-06T07:33:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.780+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.1 v2:172.21.15.86:6809/2555210107 4 ==== osd_op_reply(31 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34019b40 2024-03-06T07:33:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.781+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:32 17.3e 17:7c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.782+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 3 ==== osd_op_reply(32 [pgnls start_epoch 116 out=49b] v110'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.782+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:33 17.1 17:80000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.782+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 9 ==== osd_op_reply(33 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.782+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:34 17.21 17:84000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.782+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 10 ==== osd_op_reply(34 [pgnls start_epoch 116 out=49b] v110'12 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.782+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:35 17.11 17:88000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.783+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 7 ==== osd_op_reply(35 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.783+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:36 17.31 17:8c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34019b40 2024-03-06T07:33:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.783+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.1 v2:172.21.15.86:6809/2555210107 5 ==== osd_op_reply(36 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34019b40 2024-03-06T07:33:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.783+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:37 17.9 17:90000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.784+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 8 ==== osd_op_reply(37 [pgnls start_epoch 116 out=49b] v110'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.784+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:38 17.29 17:94000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.785+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 4 ==== osd_op_reply(38 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.785+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:39 17.19 17:98000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3806d040 2024-03-06T07:33:58.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.786+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.5 v2:172.21.15.194:6816/1278478503 3 ==== osd_op_reply(39 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3806d040 2024-03-06T07:33:58.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.786+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:40 17.39 17:9c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3806d040 2024-03-06T07:33:58.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.787+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.5 v2:172.21.15.194:6816/1278478503 4 ==== osd_op_reply(40 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3806d040 2024-03-06T07:33:58.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.787+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:41 17.5 17:a0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e380788e0 2024-03-06T07:33:58.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.788+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.6 v2:172.21.15.194:6808/2611339787 3 ==== osd_op_reply(41 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e380788e0 2024-03-06T07:33:58.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.788+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:42 17.25 17:a4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34019b40 2024-03-06T07:33:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.788+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.1 v2:172.21.15.86:6809/2555210107 6 ==== osd_op_reply(42 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34019b40 2024-03-06T07:33:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.788+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:43 17.15 17:a8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3806d040 2024-03-06T07:33:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.789+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.5 v2:172.21.15.194:6816/1278478503 5 ==== osd_op_reply(43 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3806d040 2024-03-06T07:33:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.789+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:44 17.35 17:ac000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.790+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 5 ==== osd_op_reply(44 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.790+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:45 17.d 17:b0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.791+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 7 ==== osd_op_reply(45 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.791+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:46 17.2d 17:b4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.792+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 8 ==== osd_op_reply(46 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.792+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:47 17.1d 17:b8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.793+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 9 ==== osd_op_reply(47 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.793+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:48 17.3d 17:bc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.794+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 10 ==== osd_op_reply(48 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.794+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:49 17.3 17:c0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.795+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 6 ==== osd_op_reply(49 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.795+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:50 17.23 17:c4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.796+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 7 ==== osd_op_reply(50 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.796+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:51 17.13 17:c8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e3805c3e0 2024-03-06T07:33:58.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.797+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.3 v2:172.21.15.86:6824/3540554891 9 ==== osd_op_reply(51 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x7f5e3805c3e0 2024-03-06T07:33:58.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.797+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:52 17.33 17:cc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x55abff495870 2024-03-06T07:33:58.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.798+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 <== osd.7 v2:172.21.15.194:6819/3406102027 11 ==== osd_op_reply(52 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e34019400 con 0x55abff495870 2024-03-06T07:33:58.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.798+0000 7f5e45a00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:53 17.b 17:d0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.799+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 11 ==== osd_op_reply(53 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.799+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:54 17.2b 17:d4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.800+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 8 ==== osd_op_reply(54 [pgnls start_epoch 116 out=49b] v110'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.800+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:55 17.1b 17:d8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.801+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 9 ==== osd_op_reply(55 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.801+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- osd_op(unknown.0.0:56 17.3b 17:dc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e4001c060 2024-03-06T07:33:58.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.801+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.2 v2:172.21.15.86:6808/2264489672 2 ==== osd_op_reply(56 [pgnls start_epoch 116 out=49b] v110'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e4001c060 2024-03-06T07:33:58.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.801+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:57 17.7 17:e0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.802+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 12 ==== osd_op_reply(57 [pgnls start_epoch 116 out=81b] v110'8 uv3 ondisk = 1) v8 ==== 144+0+81 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.802+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:58 17.27 17:e4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.803+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 10 ==== osd_op_reply(58 [pgnls start_epoch 116 out=49b] v110'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.803+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:59 17.17 17:e8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e380788e0 2024-03-06T07:33:58.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.804+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.6 v2:172.21.15.194:6808/2611339787 4 ==== osd_op_reply(59 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e380788e0 2024-03-06T07:33:58.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.804+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:60 17.37 17:ec000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34026380 2024-03-06T07:33:58.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.805+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.0 v2:172.21.15.86:6800/1601033517 13 ==== osd_op_reply(60 [pgnls start_epoch 116 out=49b] v110'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e34026380 2024-03-06T07:33:58.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.805+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:61 17.f 17:f0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34019b40 2024-03-06T07:33:58.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.806+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.1 v2:172.21.15.86:6809/2555210107 7 ==== osd_op_reply(61 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34019b40 2024-03-06T07:33:58.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.806+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:62 17.2f 17:f4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e34035950 2024-03-06T07:33:58.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.807+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.4 v2:172.21.15.194:6800/2403584566 11 ==== osd_op_reply(62 [pgnls start_epoch 116 out=81b] v110'9 uv1 ondisk = 1) v8 ==== 144+0+81 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e34035950 2024-03-06T07:33:58.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.807+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:63 17.1f 17:f8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e380788e0 2024-03-06T07:33:58.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.808+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 <== osd.6 v2:172.21.15.194:6808/2611339787 5 ==== osd_op_reply(63 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e40019030 con 0x7f5e380788e0 2024-03-06T07:33:58.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.808+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- osd_op(unknown.0.0:64 17.3f 17:fc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55abff3bee70 con 0x7f5e4001c060 2024-03-06T07:33:58.810 INFO:teuthology.orchestra.run.smithi086.stdout:10000000001.00000000 2024-03-06T07:33:58.810 INFO:teuthology.orchestra.run.smithi086.stdout:10000000001.00000001 2024-03-06T07:33:58.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.808+0000 7f5e46400640 1 -- 172.21.15.86:0/397544253 <== osd.2 v2:172.21.15.86:6808/2264489672 3 ==== osd_op_reply(64 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f5e3805a040 con 0x7f5e4001c060 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f5e34026380 msgr2=0x7f5e340267c0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7f5e34026380 0x7f5e340267c0 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7f5e34019b40 msgr2=0x7f5e3400f910 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7f5e34019b40 0x7f5e3400f910 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f5e4001c060 msgr2=0x7f5e40055c40 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x7f5e4001c060 0x7f5e40055c40 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f5e3805c3e0 msgr2=0x7f5e3805c860 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7f5e3805c3e0 0x7f5e3805c860 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f5e34035950 msgr2=0x7f5e340343d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7f5e34035950 0x7f5e340343d0 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7f5e3806d040 msgr2=0x7f5e3806d930 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 reap_dead start 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7f5e3806d040 0x7f5e3806d930 crc :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7f5e380788e0 msgr2=0x7f5e3806a390 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7f5e380788e0 0x7f5e3806a390 crc :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55abff495870 msgr2=0x55abff3be930 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55abff495870 0x55abff3be930 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e28069eb0 msgr2=0x7f5e2806c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e28069eb0 0x7f5e2806c370 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7f5e38054000 tx=0x7f5e3804e000 comp rx=0 tx=0).stop 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff494250 msgr2=0x55abff4788a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e49a39f40 1 --2- 172.21.15.86:0/397544253 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55abff494250 0x55abff4788a0 secure :-1 s=READY pgs=1339 cs=0 l=1 rev1=1 crypto rx=0x7f5e40004180 tx=0x7f5e4000c520 comp rx=0 tx=0).stop 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.809+0000 7f5e46e00640 1 -- 172.21.15.86:0/397544253 reap_dead start 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.810+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 shutdown_connections 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.810+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 >> 172.21.15.86:0/397544253 conn(0x55abff3ecbe0 msgr2=0x55abff3edff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.810+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 shutdown_connections 2024-03-06T07:33:58.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:58.810+0000 7f5e49a39f40 1 -- 172.21.15.86:0/397544253 wait complete. 2024-03-06T07:33:58.818 DEBUG:tasks.cephfs.filesystem:All objects for ino 1099511627777 size 8388608 found 2024-03-06T07:33:58.818 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:59.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.008+0000 7fd726600640 1 Processor -- start 2024-03-06T07:33:59.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.008+0000 7fd726600640 1 -- start start 2024-03-06T07:33:59.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.009+0000 7fd726600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd72015e0d0 0x7fd7200d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd726600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd7200d0fc0 0x7fd720160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd726600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd7200cea60 con 0x7fd72015f470 2024-03-06T07:33:59.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd726600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7200d2f60 con 0x7fd72015e0d0 2024-03-06T07:33:59.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd726600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7200d2350 con 0x7fd7200d0fc0 2024-03-06T07:33:59.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd72015e0d0 0x7fd7200d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71ea00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd72015f470 0x7fd7200d08b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:43712/0 (socket says 172.21.15.86:43712) 2024-03-06T07:33:59.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd72015e0d0 0x7fd7200d1a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35894/0 (socket says 172.21.15.86:35894) 2024-03-06T07:33:59.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71ea00640 1 -- 172.21.15.86:0/331908337 learned_addr learned my addr 172.21.15.86:0/331908337 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:33:59.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71fe00640 1 --2- 172.21.15.86:0/331908337 >> v2:172.21.15.194:3300/0 conn(0x7fd7200d0fc0 0x7fd720160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71f400640 1 -- 172.21.15.86:0/331908337 >> v2:172.21.15.194:3300/0 conn(0x7fd7200d0fc0 msgr2=0x7fd720160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:59.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71f400640 1 --2- 172.21.15.86:0/331908337 >> v2:172.21.15.194:3300/0 conn(0x7fd7200d0fc0 0x7fd720160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.010+0000 7fd71f400640 1 -- 172.21.15.86:0/331908337 >> v1:172.21.15.86:6789/0 conn(0x7fd72015f470 legacy=0x7fd7200d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.011+0000 7fd71e000640 1 -- 172.21.15.86:0/331908337 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3363851843 0 0) 0x7fd7200cea60 con 0x7fd72015f470 2024-03-06T07:33:59.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.011+0000 7fd71f400640 1 -- 172.21.15.86:0/331908337 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7200d2630 con 0x7fd72015e0d0 2024-03-06T07:33:59.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.012+0000 7fd71f400640 1 --2- 172.21.15.86:0/331908337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd7200d1a90 secure :-1 s=READY pgs=1341 cs=0 l=1 rev1=1 crypto rx=0x7fd7200caf20 tx=0x7fd71002b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=475bd52c9ea2b428 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:59.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.012+0000 7fd71e000640 1 -- 172.21.15.86:0/331908337 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd710034070 con 0x7fd72015e0d0 2024-03-06T07:33:59.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.012+0000 7fd71e000640 1 -- 172.21.15.86:0/331908337 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd710030610 con 0x7fd72015e0d0 2024-03-06T07:33:59.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.012+0000 7fd71e000640 1 -- 172.21.15.86:0/331908337 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7100309e0 con 0x7fd72015e0d0 2024-03-06T07:33:59.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 -- 172.21.15.86:0/331908337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 msgr2=0x7fd7200d1a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 --2- 172.21.15.86:0/331908337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd7200d1a90 secure :-1 s=READY pgs=1341 cs=0 l=1 rev1=1 crypto rx=0x7fd7200caf20 tx=0x7fd71002b960 comp rx=0 tx=0).stop 2024-03-06T07:33:59.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 -- 172.21.15.86:0/331908337 shutdown_connections 2024-03-06T07:33:59.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 --2- 172.21.15.86:0/331908337 >> v2:172.21.15.194:3300/0 conn(0x7fd7200d0fc0 0x7fd720160dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 --2- 172.21.15.86:0/331908337 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd7200d1a90 unknown :-1 s=CLOSED pgs=1341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 -- 172.21.15.86:0/331908337 >> 172.21.15.86:0/331908337 conn(0x7fd7200d3ec0 msgr2=0x7fd7200d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:59.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 -- 172.21.15.86:0/331908337 shutdown_connections 2024-03-06T07:33:59.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.013+0000 7fd726600640 1 -- 172.21.15.86:0/331908337 wait complete. 2024-03-06T07:33:59.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.015+0000 7fd726600640 1 Processor -- start 2024-03-06T07:33:59.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.015+0000 7fd726600640 1 -- start start 2024-03-06T07:33:59.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd726600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7200d0fc0 0x7fd7200cbda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd726600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd72014e5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd726600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd72015f470 0x7fd7200cce70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd726600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd72015fb30 con 0x7fd72015e0d0 2024-03-06T07:33:59.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd726600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7200d2bf0 con 0x7fd72015f470 2024-03-06T07:33:59.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd726600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7200d1fd0 con 0x7fd7200d0fc0 2024-03-06T07:33:59.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd71ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd72014e5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd71ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd72014e5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35910/0 (socket says 172.21.15.86:35910) 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.016+0000 7fd71ea00640 1 -- 172.21.15.86:0/2973128270 learned_addr learned my addr 172.21.15.86:0/2973128270 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71f400640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7200d0fc0 0x7fd7200cbda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71fe00640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd72015f470 0x7fd7200cce70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71ea00640 1 -- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7200d0fc0 msgr2=0x7fd7200cbda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71ea00640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7200d0fc0 0x7fd7200cbda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71ea00640 1 -- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd72015f470 msgr2=0x7fd7200cce70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71ea00640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd72015f470 0x7fd7200cce70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71ea00640 1 -- 172.21.15.86:0/2973128270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7200cd3b0 con 0x7fd72015e0d0 2024-03-06T07:33:59.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd71ea00640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd72014e5b0 secure :-1 s=READY pgs=1342 cs=0 l=1 rev1=1 crypto rx=0x7fd70c00c460 tx=0x7fd70c00c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:59.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd717400640 1 -- 172.21.15.86:0/2973128270 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd70c018070 con 0x7fd72015e0d0 2024-03-06T07:33:59.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd717400640 1 -- 172.21.15.86:0/2973128270 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd70c007500 con 0x7fd72015e0d0 2024-03-06T07:33:59.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd717400640 1 -- 172.21.15.86:0/2973128270 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd70c004a20 con 0x7fd72015e0d0 2024-03-06T07:33:59.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.017+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd7200cd660 con 0x7fd72015e0d0 2024-03-06T07:33:59.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.018+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd7200c8ac0 con 0x7fd72015e0d0 2024-03-06T07:33:59.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.019+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd7200d0d40 con 0x7fd72015e0d0 2024-03-06T07:33:59.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.020+0000 7fd717400640 1 -- 172.21.15.86:0/2973128270 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd70c01b070 con 0x7fd72015e0d0 2024-03-06T07:33:59.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.021+0000 7fd717400640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd70006b4d0 0x7fd70006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.021+0000 7fd717400640 1 -- 172.21.15.86:0/2973128270 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fd70c089640 con 0x7fd72015e0d0 2024-03-06T07:33:59.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.021+0000 7fd71f400640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd70006b4d0 0x7fd70006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.022+0000 7fd71f400640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd70006b4d0 0x7fd70006d990 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7fd710036000 tx=0x7fd71003b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:59.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.026+0000 7fd717400640 1 -- 172.21.15.86:0/2973128270 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd7200d0d40 con 0x7fd72015e0d0 2024-03-06T07:33:59.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.260+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd72004f330 con 0x7fd72015e0d0 2024-03-06T07:33:59.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.261+0000 7fd717400640 1 -- 172.21.15.86:0/2973128270 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7fd70c058300 con 0x7fd72015e0d0 2024-03-06T07:33:59.263 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:59.263 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:33:59.263 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd70006b4d0 msgr2=0x7fd70006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd70006b4d0 0x7fd70006d990 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7fd710036000 tx=0x7fd71003b040 comp rx=0 tx=0).stop 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 msgr2=0x7fd72014e5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd72014e5b0 secure :-1 s=READY pgs=1342 cs=0 l=1 rev1=1 crypto rx=0x7fd70c00c460 tx=0x7fd70c00c920 comp rx=0 tx=0).stop 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 shutdown_connections 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd70006b4d0 0x7fd70006d990 unknown :-1 s=CLOSED pgs=578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd72015f470 0x7fd7200cce70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd72015e0d0 0x7fd72014e5b0 unknown :-1 s=CLOSED pgs=1342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 --2- 172.21.15.86:0/2973128270 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd7200d0fc0 0x7fd7200cbda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 >> 172.21.15.86:0/2973128270 conn(0x7fd7200d3ec0 msgr2=0x7fd72006bab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:59.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 shutdown_connections 2024-03-06T07:33:59.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.263+0000 7fd726600640 1 -- 172.21.15.86:0/2973128270 wait complete. 2024-03-06T07:33:59.283 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:33:59.493 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-06T07:33:59.493 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-06T07:33:59.493 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:33:59.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.695+0000 7fe6c2800640 1 Processor -- start 2024-03-06T07:33:59.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.695+0000 7fe6c2800640 1 -- start start 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe6bc15a940 0x7fe6bc0b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c2800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe6bc0cf370 0x7fe6bc06d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c2800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe6bc0b2b30 con 0x7fe6bc0cc2a0 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c2800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe6bc0b28e0 con 0x7fe6bc15a940 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c2800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe6bc0b2e10 con 0x7fe6bc0cf370 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c1400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe6bc15a940 0x7fe6bc0b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c1400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe6bc15a940 0x7fe6bc0b45e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35916/0 (socket says 172.21.15.86:35916) 2024-03-06T07:33:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.697+0000 7fe6c1400640 1 -- 172.21.15.86:0/400856427 learned_addr learned my addr 172.21.15.86:0/400856427 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:59.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.698+0000 7fe6c0a00640 1 --2- 172.21.15.86:0/400856427 >> v2:172.21.15.194:3300/0 conn(0x7fe6bc0cf370 0x7fe6bc06d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.698+0000 7fe6c1400640 1 -- 172.21.15.86:0/400856427 >> v2:172.21.15.194:3300/0 conn(0x7fe6bc0cf370 msgr2=0x7fe6bc06d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:59.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.698+0000 7fe6c1400640 1 --2- 172.21.15.86:0/400856427 >> v2:172.21.15.194:3300/0 conn(0x7fe6bc0cf370 0x7fe6bc06d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.698+0000 7fe6c1400640 1 -- 172.21.15.86:0/400856427 >> v1:172.21.15.194:6789/0 conn(0x7fe6bc0cc2a0 legacy=0x7fe6bc0cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:33:59.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.698+0000 7fe6c1400640 1 -- 172.21.15.86:0/400856427 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6bc06ca50 con 0x7fe6bc15a940 2024-03-06T07:33:59.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.699+0000 7fe6c1400640 1 --2- 172.21.15.86:0/400856427 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0b45e0 secure :-1 s=READY pgs=1343 cs=0 l=1 rev1=1 crypto rx=0x7fe6bc068400 tx=0x7fe6b002e0f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5a87e7ef36f9fd31 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.699+0000 7fe6bbe00640 1 -- 172.21.15.86:0/400856427 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6b002ea90 con 0x7fe6bc15a940 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.699+0000 7fe6bbe00640 1 -- 172.21.15.86:0/400856427 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6b002ec30 con 0x7fe6bc15a940 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.699+0000 7fe6bbe00640 1 -- 172.21.15.86:0/400856427 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6b0009600 con 0x7fe6bc15a940 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 -- 172.21.15.86:0/400856427 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 msgr2=0x7fe6bc0b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 --2- 172.21.15.86:0/400856427 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0b45e0 secure :-1 s=READY pgs=1343 cs=0 l=1 rev1=1 crypto rx=0x7fe6bc068400 tx=0x7fe6b002e0f0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 -- 172.21.15.86:0/400856427 shutdown_connections 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 --2- 172.21.15.86:0/400856427 >> v2:172.21.15.194:3300/0 conn(0x7fe6bc0cf370 0x7fe6bc06d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 --2- 172.21.15.86:0/400856427 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0b45e0 unknown :-1 s=CLOSED pgs=1343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 -- 172.21.15.86:0/400856427 >> 172.21.15.86:0/400856427 conn(0x7fe6bc06afc0 msgr2=0x7fe6bc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 -- 172.21.15.86:0/400856427 shutdown_connections 2024-03-06T07:33:59.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.700+0000 7fe6c2800640 1 -- 172.21.15.86:0/400856427 wait complete. 2024-03-06T07:33:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.702+0000 7fe6c2800640 1 Processor -- start 2024-03-06T07:33:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.702+0000 7fe6c2800640 1 -- start start 2024-03-06T07:33:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c2800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe6bc0cc2a0 0x7fe6bc147ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c2800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe6bc0cf370 0x7fe6bc0c4850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0c4d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c2800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6bc0b4b20 con 0x7fe6bc15a940 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c2800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6bc0b30f0 con 0x7fe6bc0cf370 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c2800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe6bc06d790 con 0x7fe6bc0cc2a0 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0c4d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0c4d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35920/0 (socket says 172.21.15.86:35920) 2024-03-06T07:33:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.703+0000 7fe6c1e00640 1 -- 172.21.15.86:0/2455975867 learned_addr learned my addr 172.21.15.86:0/2455975867 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:33:59.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c1400640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe6bc0cc2a0 0x7fe6bc147ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c0a00640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe6bc0cf370 0x7fe6bc0c4850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c1e00640 1 -- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe6bc0cc2a0 msgr2=0x7fe6bc147ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c1e00640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe6bc0cc2a0 0x7fe6bc147ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c1e00640 1 -- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe6bc0cf370 msgr2=0x7fe6bc0c4850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c1e00640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe6bc0cf370 0x7fe6bc0c4850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c1e00640 1 -- 172.21.15.86:0/2455975867 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6bc0aa500 con 0x7fe6bc15a940 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c0a00640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe6bc0cf370 0x7fe6bc0c4850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c1e00640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0c4d90 secure :-1 s=READY pgs=1344 cs=0 l=1 rev1=1 crypto rx=0x7fe6b400b410 tx=0x7fe6b400b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6b9600640 1 -- 172.21.15.86:0/2455975867 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6b4015030 con 0x7fe6bc15a940 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6b9600640 1 -- 172.21.15.86:0/2455975867 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6b4004020 con 0x7fe6bc15a940 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe6bc0aa7e0 con 0x7fe6bc15a940 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6b9600640 1 -- 172.21.15.86:0/2455975867 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6b4004410 con 0x7fe6bc15a940 2024-03-06T07:33:59.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.704+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe6bc0aada0 con 0x7fe6bc15a940 2024-03-06T07:33:59.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.706+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe6bc0ab180 con 0x7fe6bc15a940 2024-03-06T07:33:59.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.706+0000 7fe6b9600640 1 -- 172.21.15.86:0/2455975867 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe6b4018070 con 0x7fe6bc15a940 2024-03-06T07:33:59.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.707+0000 7fe6b9600640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe69c06b4d0 0x7fe69c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:33:59.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.707+0000 7fe6b9600640 1 -- 172.21.15.86:0/2455975867 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fe6b4089e40 con 0x7fe6bc15a940 2024-03-06T07:33:59.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.708+0000 7fe6c1400640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe69c06b4d0 0x7fe69c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:33:59.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.708+0000 7fe6c1400640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe69c06b4d0 0x7fe69c06d990 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7fe6b0037000 tx=0x7fe6b0032000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:33:59.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.712+0000 7fe6b9600640 1 -- 172.21.15.86:0/2455975867 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe6b4058b90 con 0x7fe6bc15a940 2024-03-06T07:33:59.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.949+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe6bc04f330 con 0x7fe6bc15a940 2024-03-06T07:33:59.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.949+0000 7fe6b9600640 1 -- 172.21.15.86:0/2455975867 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7fe6b4058490 con 0x7fe6bc15a940 2024-03-06T07:33:59.951 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:33:59.951 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:33:59.952 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:33:59.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe69c06b4d0 msgr2=0x7fe69c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe69c06b4d0 0x7fe69c06d990 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7fe6b0037000 tx=0x7fe6b0032000 comp rx=0 tx=0).stop 2024-03-06T07:33:59.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 msgr2=0x7fe6bc0c4d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:33:59.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0c4d90 secure :-1 s=READY pgs=1344 cs=0 l=1 rev1=1 crypto rx=0x7fe6b400b410 tx=0x7fe6b400b8d0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 shutdown_connections 2024-03-06T07:33:59.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe69c06b4d0 0x7fe69c06d990 unknown :-1 s=CLOSED pgs=579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe6bc15a940 0x7fe6bc0c4d90 unknown :-1 s=CLOSED pgs=1344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe6bc0cf370 0x7fe6bc0c4850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 --2- 172.21.15.86:0/2455975867 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe6bc0cc2a0 0x7fe6bc147ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:33:59.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 >> 172.21.15.86:0/2455975867 conn(0x7fe6bc06afc0 msgr2=0x7fe6bc0aee90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:33:59.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 shutdown_connections 2024-03-06T07:33:59.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:33:59.952+0000 7fe6c2800640 1 -- 172.21.15.86:0/2455975867 wait complete. 2024-03-06T07:33:59.979 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:34:00.215 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:34:00.215 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:34:00.215 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-06T07:34:00.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.301+0000 7fca6c17df40 1 Processor -- start 2024-03-06T07:34:00.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.301+0000 7fca6c17df40 1 -- start start 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca6c17df40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca500030f0 0x560263473e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca6c17df40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263502220 0x5602634e5fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca6c17df40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560263474f50 0x5602634e6510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca6c17df40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560263475520 con 0x7fca500030f0 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca6c17df40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5602633d4fd0 con 0x560263474f50 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca6c17df40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x5602635037b0 con 0x560263502220 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca500030f0 0x560263473e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca500030f0 0x560263473e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35934/0 (socket says 172.21.15.86:35934) 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 -- 172.21.15.86:0/3875456861 learned_addr learned my addr 172.21.15.86:0/3875456861 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 -- 172.21.15.86:0/3875456861 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263502220 msgr2=0x5602634e5fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:00.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 --2- 172.21.15.86:0/3875456861 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263502220 0x5602634e5fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 -- 172.21.15.86:0/3875456861 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560263474f50 msgr2=0x5602634e6510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:00.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 --2- 172.21.15.86:0/3875456861 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560263474f50 0x5602634e6510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 -- 172.21.15.86:0/3875456861 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56026345dfe0 con 0x7fca500030f0 2024-03-06T07:34:00.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.302+0000 7fca69400640 1 --2- 172.21.15.86:0/3875456861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca500030f0 0x560263473e20 secure :-1 s=READY pgs=1347 cs=0 l=1 rev1=1 crypto rx=0x7fca5c00c950 tx=0x7fca5c00ce10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=78208bcde0f49703 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:00.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca63400640 1 -- 172.21.15.86:0/3875456861 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca5c00eb40 con 0x7fca500030f0 2024-03-06T07:34:00.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca63400640 1 -- 172.21.15.86:0/3875456861 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca5c013070 con 0x7fca500030f0 2024-03-06T07:34:00.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca63400640 1 -- 172.21.15.86:0/3875456861 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca5c018470 con 0x7fca500030f0 2024-03-06T07:34:00.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 -- 172.21.15.86:0/3875456861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca500030f0 msgr2=0x560263473e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 --2- 172.21.15.86:0/3875456861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca500030f0 0x560263473e20 secure :-1 s=READY pgs=1347 cs=0 l=1 rev1=1 crypto rx=0x7fca5c00c950 tx=0x7fca5c00ce10 comp rx=0 tx=0).stop 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 -- 172.21.15.86:0/3875456861 shutdown_connections 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 --2- 172.21.15.86:0/3875456861 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x560263474f50 0x5602634e6510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 --2- 172.21.15.86:0/3875456861 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263502220 0x5602634e5fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 --2- 172.21.15.86:0/3875456861 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fca500030f0 0x560263473e20 unknown :-1 s=CLOSED pgs=1347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 -- 172.21.15.86:0/3875456861 >> 172.21.15.86:0/3875456861 conn(0x560263459690 msgr2=0x56026345c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 -- 172.21.15.86:0/3875456861 shutdown_connections 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.303+0000 7fca6c17df40 1 -- 172.21.15.86:0/3875456861 wait complete. 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.304+0000 7fca6c17df40 1 Processor -- start 2024-03-06T07:34:00.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.304+0000 7fca6c17df40 1 -- start start 2024-03-06T07:34:00.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.305+0000 7fca6c17df40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263474f50 0x5602634e3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.305+0000 7fca6c17df40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca500030f0 0x5602634e7150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.305+0000 7fca6c17df40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5602634e9760 0x560263504940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.305+0000 7fca6c17df40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x560263500a50 con 0x5602634e9760 2024-03-06T07:34:00.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.305+0000 7fca6c17df40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x5602635007b0 con 0x7fca500030f0 2024-03-06T07:34:00.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.305+0000 7fca6c17df40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x560263501d80 con 0x560263474f50 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5602634e9760 0x560263504940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5602634e9760 0x560263504940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35936/0 (socket says 172.21.15.86:35936) 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 -- 172.21.15.86:0/41171444 learned_addr learned my addr 172.21.15.86:0/41171444 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca68a00640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca500030f0 0x5602634e7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69400640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263474f50 0x5602634e3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 -- 172.21.15.86:0/41171444 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263474f50 msgr2=0x5602634e3650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x560263474f50 0x5602634e3650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 -- 172.21.15.86:0/41171444 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca500030f0 msgr2=0x5602634e7150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fca500030f0 0x5602634e7150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 -- 172.21.15.86:0/41171444 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5602634e7690 con 0x5602634e9760 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.306+0000 7fca69e00640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5602634e9760 0x560263504940 secure :-1 s=READY pgs=1348 cs=0 l=1 rev1=1 crypto rx=0x56026341b670 tx=0x7fca6400e5b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.307+0000 7fca62000640 1 -- 172.21.15.86:0/41171444 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca64017020 con 0x5602634e9760 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.307+0000 7fca62000640 1 -- 172.21.15.86:0/41171444 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca64013660 con 0x5602634e9760 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.307+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x5602634fb3b0 con 0x5602634e9760 2024-03-06T07:34:00.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.307+0000 7fca62000640 1 -- 172.21.15.86:0/41171444 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca64013a50 con 0x5602634e9760 2024-03-06T07:34:00.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.307+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5602634ea000 con 0x5602634e9760 2024-03-06T07:34:00.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.309+0000 7fca62000640 1 -- 172.21.15.86:0/41171444 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fca6401a020 con 0x5602634e9760 2024-03-06T07:34:00.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.310+0000 7fca62000640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca4c069eb0 0x7fca4c06c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.310+0000 7fca62000640 1 -- 172.21.15.86:0/41171444 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fca64088d20 con 0x5602634e9760 2024-03-06T07:34:00.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.310+0000 7fca69400640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca4c069eb0 0x7fca4c06c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.310+0000 7fca6c17df40 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x5602634fced0 0x5602634fd2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.310+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e116) v9 -- 0x5602634fd7f0 con 0x5602634fced0 2024-03-06T07:34:00.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.311+0000 7fca68a00640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x5602634fced0 0x5602634fd2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.311+0000 7fca69400640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca4c069eb0 0x7fca4c06c370 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7fca5c00ce50 tx=0x7fca5c009000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:00.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.311+0000 7fca68a00640 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x5602634fced0 0x5602634fd2b0 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca68a00640 1 -- 172.21.15.86:0/41171444 <== osd.0 v2:172.21.15.86:6800/1601033517 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (99) out=99b] v0'0 uv9 ondisk = 0) v8 ==== 164+0+99 (crc 0 0 0) 0x7fca5800a210 con 0x5602634fced0 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x5602634fced0 msgr2=0x5602634fd2b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x5602634fced0 0x5602634fd2b0 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca4c069eb0 msgr2=0x7fca4c06c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fca4c069eb0 0x7fca4c06c370 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7fca5c00ce50 tx=0x7fca5c009000 comp rx=0 tx=0).stop 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5602634e9760 msgr2=0x560263504940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 --2- 172.21.15.86:0/41171444 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x5602634e9760 0x560263504940 secure :-1 s=READY pgs=1348 cs=0 l=1 rev1=1 crypto rx=0x56026341b670 tx=0x7fca6400e5b0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca69e00640 1 -- 172.21.15.86:0/41171444 reap_dead start 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 shutdown_connections 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 >> 172.21.15.86:0/41171444 conn(0x560263459690 msgr2=0x560263473300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:00.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 shutdown_connections 2024-03-06T07:34:00.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.312+0000 7fca6c17df40 1 -- 172.21.15.86:0/41171444 wait complete. 2024-03-06T07:34:00.318 INFO:teuthology.orchestra.run.smithi086.stdout:] 10000000001 stray0 2024-03-06T07:34:00.318 DEBUG:teuthology.orchestra.run.smithi086:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: "ino": 1099511627777, 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: "ancestors": [ 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 1536, 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "10000000001", 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 13 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:00.404 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 256, 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "stray0", 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 5 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: "pool": 17, 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: "old_pools": [] 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:34:00.405 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:00.411 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:00.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.671+0000 7f2a44a00640 1 Processor -- start 2024-03-06T07:34:00.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.672+0000 7f2a44a00640 1 -- start start 2024-03-06T07:34:00.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.673+0000 7f2a44a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2a400ceb40 0x7f2a400cf8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a44a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161760 0x7f2a400cfe00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a44a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2a400cf000 con 0x7f2a40161380 2024-03-06T07:34:00.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a44a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2a400cd570 con 0x7f2a400ceb40 2024-03-06T07:34:00.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a44a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a400cd850 con 0x7f2a40161760 2024-03-06T07:34:00.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a35e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2a400ceb40 0x7f2a400cf8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a35e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2a400ceb40 0x7f2a400cf8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35952/0 (socket says 172.21.15.86:35952) 2024-03-06T07:34:00.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a35e00640 1 -- 172.21.15.86:0/3523592946 learned_addr learned my addr 172.21.15.86:0/3523592946 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:00.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.674+0000 7f2a3f200640 1 --2- 172.21.15.86:0/3523592946 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161760 0x7f2a400cfe00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.675+0000 7f2a35e00640 1 -- 172.21.15.86:0/3523592946 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161760 msgr2=0x7f2a400cfe00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:00.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.675+0000 7f2a35e00640 1 --2- 172.21.15.86:0/3523592946 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161760 0x7f2a400cfe00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.675+0000 7f2a35e00640 1 -- 172.21.15.86:0/3523592946 >> v1:172.21.15.194:6789/0 conn(0x7f2a40161380 legacy=0x7f2a400ce430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.675+0000 7f2a35e00640 1 -- 172.21.15.86:0/3523592946 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a400cdb30 con 0x7f2a400ceb40 2024-03-06T07:34:00.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.676+0000 7f2a35e00640 1 --2- 172.21.15.86:0/3523592946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400cf8c0 secure :-1 s=READY pgs=1349 cs=0 l=1 rev1=1 crypto rx=0x7f2a28009870 tx=0x7f2a2802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=228bae1ef67ad8b2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:00.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.676+0000 7f2a3de00640 1 -- 172.21.15.86:0/3523592946 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a28035020 con 0x7f2a400ceb40 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.676+0000 7f2a3de00640 1 -- 172.21.15.86:0/3523592946 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a28031640 con 0x7f2a400ceb40 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.676+0000 7f2a3de00640 1 -- 172.21.15.86:0/3523592946 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a2802d040 con 0x7f2a400ceb40 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 -- 172.21.15.86:0/3523592946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 msgr2=0x7f2a400cf8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 --2- 172.21.15.86:0/3523592946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400cf8c0 secure :-1 s=READY pgs=1349 cs=0 l=1 rev1=1 crypto rx=0x7f2a28009870 tx=0x7f2a2802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 -- 172.21.15.86:0/3523592946 shutdown_connections 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 --2- 172.21.15.86:0/3523592946 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161760 0x7f2a400cfe00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 --2- 172.21.15.86:0/3523592946 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400cf8c0 unknown :-1 s=CLOSED pgs=1349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 -- 172.21.15.86:0/3523592946 >> 172.21.15.86:0/3523592946 conn(0x7f2a400b3690 msgr2=0x7f2a400b3aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 -- 172.21.15.86:0/3523592946 shutdown_connections 2024-03-06T07:34:00.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.677+0000 7f2a44a00640 1 -- 172.21.15.86:0/3523592946 wait complete. 2024-03-06T07:34:00.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.679+0000 7f2a44a00640 1 Processor -- start 2024-03-06T07:34:00.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.679+0000 7f2a44a00640 1 -- start start 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a44a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400b6210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a44a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161380 0x7f2a400cb420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a44a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a40161760 0x7f2a4006f590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a44a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a400b2820 con 0x7f2a400ceb40 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a44a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a400b26a0 con 0x7f2a40161760 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a44a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a400b29a0 con 0x7f2a40161380 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a3e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400b6210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a3e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400b6210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35962/0 (socket says 172.21.15.86:35962) 2024-03-06T07:34:00.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.681+0000 7f2a3e800640 1 -- 172.21.15.86:0/4203138879 learned_addr learned my addr 172.21.15.86:0/4203138879 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a3f200640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a40161760 0x7f2a4006f590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a35e00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161380 0x7f2a400cb420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a3e800640 1 -- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161380 msgr2=0x7f2a400cb420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a3e800640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161380 0x7f2a400cb420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a3e800640 1 -- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a40161760 msgr2=0x7f2a4006f590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a3e800640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a40161760 0x7f2a4006f590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a3e800640 1 -- 172.21.15.86:0/4203138879 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a400af980 con 0x7f2a400ceb40 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a3e800640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400b6210 secure :-1 s=READY pgs=1350 cs=0 l=1 rev1=1 crypto rx=0x7f2a3000c3c0 tx=0x7f2a3000c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a37400640 1 -- 172.21.15.86:0/4203138879 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a300052d0 con 0x7f2a400ceb40 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a37400640 1 -- 172.21.15.86:0/4203138879 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a30005490 con 0x7f2a400ceb40 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a37400640 1 -- 172.21.15.86:0/4203138879 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a30007260 con 0x7f2a400ceb40 2024-03-06T07:34:00.685 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.682+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a400ca940 con 0x7f2a400ceb40 2024-03-06T07:34:00.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.683+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a400caf00 con 0x7f2a400ceb40 2024-03-06T07:34:00.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.684+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2a4004f330 con 0x7f2a400ceb40 2024-03-06T07:34:00.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.685+0000 7f2a37400640 1 -- 172.21.15.86:0/4203138879 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2a30007400 con 0x7f2a400ceb40 2024-03-06T07:34:00.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.686+0000 7f2a37400640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a1c06b4d0 0x7f2a1c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:00.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.686+0000 7f2a37400640 1 -- 172.21.15.86:0/4203138879 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f2a300898a0 con 0x7f2a400ceb40 2024-03-06T07:34:00.687 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.686+0000 7f2a35e00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a1c06b4d0 0x7f2a1c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:00.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.687+0000 7f2a35e00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a1c06b4d0 0x7f2a1c06d990 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f2a28036000 tx=0x7f2a28008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:00.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.690+0000 7f2a37400640 1 -- 172.21.15.86:0/4203138879 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2a30058560 con 0x7f2a400ceb40 2024-03-06T07:34:00.928 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.926+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2a400b7ac0 con 0x7f2a400ceb40 2024-03-06T07:34:00.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.927+0000 7f2a37400640 1 -- 172.21.15.86:0/4203138879 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f2a30057e60 con 0x7f2a400ceb40 2024-03-06T07:34:00.929 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:00.929 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:00.929 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:00.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a1c06b4d0 msgr2=0x7f2a1c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a1c06b4d0 0x7f2a1c06d990 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f2a28036000 tx=0x7f2a28008030 comp rx=0 tx=0).stop 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 msgr2=0x7f2a400b6210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400b6210 secure :-1 s=READY pgs=1350 cs=0 l=1 rev1=1 crypto rx=0x7f2a3000c3c0 tx=0x7f2a3000c880 comp rx=0 tx=0).stop 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 shutdown_connections 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2a1c06b4d0 0x7f2a1c06d990 unknown :-1 s=CLOSED pgs=581 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2a40161760 0x7f2a4006f590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2a40161380 0x7f2a400cb420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 --2- 172.21.15.86:0/4203138879 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2a400ceb40 0x7f2a400b6210 unknown :-1 s=CLOSED pgs=1350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:00.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 >> 172.21.15.86:0/4203138879 conn(0x7f2a400b3690 msgr2=0x7f2a400cf7a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:00.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 shutdown_connections 2024-03-06T07:34:00.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:00.930+0000 7f2a44a00640 1 -- 172.21.15.86:0/4203138879 wait complete. 2024-03-06T07:34:00.958 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_reintegrated 2024-03-06T07:34:01.181 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_reintegrated":0}} 2024-03-06T07:34:01.182 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_reintegrated": 0 } } 2024-03-06T07:34:01.182 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:01.182 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec mv dir_2/file_b dir_2/file_c) 2024-03-06T07:34:01.245 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:01.450 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.449+0000 7ff53fe00640 1 Processor -- start 2024-03-06T07:34:01.451 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.449+0000 7ff53fe00640 1 -- start start 2024-03-06T07:34:01.453 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff538151660 0x7ff538071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5380cf430 0x7ff53806d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53fe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff53806d920 0x7ff53815dff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53fe00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff5380b3ee0 con 0x7ff5380cf430 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53fe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff538071a60 con 0x7ff53806d920 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53fe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5380b35b0 con 0x7ff538151660 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53ca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5380cf430 0x7ff53806d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53ca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5380cf430 0x7ff53806d3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35976/0 (socket says 172.21.15.86:35976) 2024-03-06T07:34:01.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53ca00640 1 -- 172.21.15.86:0/1903648951 learned_addr learned my addr 172.21.15.86:0/1903648951 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:01.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53de00640 1 --2- 172.21.15.86:0/1903648951 >> v2:172.21.15.194:3300/0 conn(0x7ff53806d920 0x7ff53815dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:01.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53d400640 1 --2- 172.21.15.86:0/1903648951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff538151660 0x7ff538071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:01.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.452+0000 7ff53ca00640 1 -- 172.21.15.86:0/1903648951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff538151660 msgr2=0x7ff538071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:01.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.453+0000 7ff53ca00640 1 --2- 172.21.15.86:0/1903648951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff538151660 0x7ff538071520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.453+0000 7ff53ca00640 1 -- 172.21.15.86:0/1903648951 >> v2:172.21.15.194:3300/0 conn(0x7ff53806d920 msgr2=0x7ff53815dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:01.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.453+0000 7ff53ca00640 1 --2- 172.21.15.86:0/1903648951 >> v2:172.21.15.194:3300/0 conn(0x7ff53806d920 0x7ff53815dff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.455 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.453+0000 7ff53ca00640 1 -- 172.21.15.86:0/1903648951 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5380b3890 con 0x7ff5380cf430 2024-03-06T07:34:01.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.454+0000 7ff53ca00640 1 --2- 172.21.15.86:0/1903648951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff53806d3e0 secure :-1 s=READY pgs=1351 cs=0 l=1 rev1=1 crypto rx=0x7ff538154fc0 tx=0x7ff52402dfc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c52171b418dd54ba server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:01.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.454+0000 7ff537e00640 1 -- 172.21.15.86:0/1903648951 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff52403b070 con 0x7ff5380cf430 2024-03-06T07:34:01.456 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.454+0000 7ff537e00640 1 -- 172.21.15.86:0/1903648951 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff52402ed70 con 0x7ff5380cf430 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.454+0000 7ff537e00640 1 -- 172.21.15.86:0/1903648951 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff524036660 con 0x7ff5380cf430 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 -- 172.21.15.86:0/1903648951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 msgr2=0x7ff53806d3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 --2- 172.21.15.86:0/1903648951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff53806d3e0 secure :-1 s=READY pgs=1351 cs=0 l=1 rev1=1 crypto rx=0x7ff538154fc0 tx=0x7ff52402dfc0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 -- 172.21.15.86:0/1903648951 shutdown_connections 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 --2- 172.21.15.86:0/1903648951 >> v2:172.21.15.194:3300/0 conn(0x7ff53806d920 0x7ff53815dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 --2- 172.21.15.86:0/1903648951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff53806d3e0 unknown :-1 s=CLOSED pgs=1351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 --2- 172.21.15.86:0/1903648951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff538151660 0x7ff538071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 -- 172.21.15.86:0/1903648951 >> 172.21.15.86:0/1903648951 conn(0x7ff5380b1730 msgr2=0x7ff5380b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 -- 172.21.15.86:0/1903648951 shutdown_connections 2024-03-06T07:34:01.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.455+0000 7ff53fe00640 1 -- 172.21.15.86:0/1903648951 wait complete. 2024-03-06T07:34:01.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.456+0000 7ff53fe00640 1 Processor -- start 2024-03-06T07:34:01.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.457+0000 7ff53fe00640 1 -- start start 2024-03-06T07:34:01.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.457+0000 7ff53fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff53806d920 0x7ff5380cd190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:01.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.457+0000 7ff53fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff5380b0990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:01.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.457+0000 7ff53fe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff538151660 0x7ff5381e4f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:01.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53fe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff53815e530 con 0x7ff5380cf430 2024-03-06T07:34:01.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53fe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff538071d30 con 0x7ff538151660 2024-03-06T07:34:01.460 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53fe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff53815e6b0 con 0x7ff53806d920 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff5380b0990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff5380b0990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35988/0 (socket says 172.21.15.86:35988) 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 -- 172.21.15.86:0/3077258365 learned_addr learned my addr 172.21.15.86:0/3077258365 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53de00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff538151660 0x7ff5381e4f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53d400640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff53806d920 0x7ff5380cd190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 -- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff53806d920 msgr2=0x7ff5380cd190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff53806d920 0x7ff5380cd190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 -- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff538151660 msgr2=0x7ff5381e4f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff538151660 0x7ff5381e4f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53ca00640 1 -- 172.21.15.86:0/3077258365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5381e5980 con 0x7ff5380cf430 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.458+0000 7ff53d400640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff53806d920 0x7ff5380cd190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:01.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.459+0000 7ff53ca00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff5380b0990 secure :-1 s=READY pgs=1352 cs=0 l=1 rev1=1 crypto rx=0x7ff52402e000 tx=0x7ff5240046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:01.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.459+0000 7ff535600640 1 -- 172.21.15.86:0/3077258365 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff524045070 con 0x7ff5380cf430 2024-03-06T07:34:01.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.459+0000 7ff535600640 1 -- 172.21.15.86:0/3077258365 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff524036e20 con 0x7ff5380cf430 2024-03-06T07:34:01.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.459+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff5381e5c60 con 0x7ff5380cf430 2024-03-06T07:34:01.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.459+0000 7ff535600640 1 -- 172.21.15.86:0/3077258365 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff52403b040 con 0x7ff5380cf430 2024-03-06T07:34:01.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.460+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff5380ccc90 con 0x7ff5380cf430 2024-03-06T07:34:01.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.461+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff53804f330 con 0x7ff5380cf430 2024-03-06T07:34:01.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.462+0000 7ff535600640 1 -- 172.21.15.86:0/3077258365 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff524048070 con 0x7ff5380cf430 2024-03-06T07:34:01.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.463+0000 7ff535600640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff51806b4d0 0x7ff51806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:01.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.463+0000 7ff535600640 1 -- 172.21.15.86:0/3077258365 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7ff5240aece0 con 0x7ff5380cf430 2024-03-06T07:34:01.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.463+0000 7ff53d400640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff51806b4d0 0x7ff51806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:01.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.464+0000 7ff53d400640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff51806b4d0 0x7ff51806d990 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7ff52c008890 tx=0x7ff52c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:01.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.467+0000 7ff535600640 1 -- 172.21.15.86:0/3077258365 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff52407da30 con 0x7ff5380cf430 2024-03-06T07:34:01.702 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.701+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff538074be0 con 0x7ff5380cf430 2024-03-06T07:34:01.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.702+0000 7ff535600640 1 -- 172.21.15.86:0/3077258365 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7ff52407d330 con 0x7ff5380cf430 2024-03-06T07:34:01.704 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:01.704 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:01.704 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff51806b4d0 msgr2=0x7ff51806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff51806b4d0 0x7ff51806d990 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7ff52c008890 tx=0x7ff52c008040 comp rx=0 tx=0).stop 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 msgr2=0x7ff5380b0990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff5380b0990 secure :-1 s=READY pgs=1352 cs=0 l=1 rev1=1 crypto rx=0x7ff52402e000 tx=0x7ff5240046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 shutdown_connections 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff51806b4d0 0x7ff51806d990 unknown :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff538151660 0x7ff5381e4f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5380cf430 0x7ff5380b0990 unknown :-1 s=CLOSED pgs=1352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 --2- 172.21.15.86:0/3077258365 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff53806d920 0x7ff5380cd190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 >> 172.21.15.86:0/3077258365 conn(0x7ff5380b1730 msgr2=0x7ff53815c870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:01.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 shutdown_connections 2024-03-06T07:34:01.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:01.705+0000 7ff53fe00640 1 -- 172.21.15.86:0/3077258365 wait complete. 2024-03-06T07:34:01.727 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-06T07:34:01.959 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"num_strays":0}} 2024-03-06T07:34:01.959 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 0 } } 2024-03-06T07:34:01.960 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:02.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.157+0000 7fb532800640 1 Processor -- start 2024-03-06T07:34:02.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.157+0000 7fb532800640 1 -- start start 2024-03-06T07:34:02.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.158+0000 7fb532800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb52c15acb0 0x7fb52c0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.158+0000 7fb532800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c06ec00 0x7fb52c0ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.158+0000 7fb532800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb52c15b090 con 0x7fb52c0b4680 2024-03-06T07:34:02.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.158+0000 7fb532800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb52c0b2a30 con 0x7fb52c15acb0 2024-03-06T07:34:02.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.158+0000 7fb532800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb52c0b4360 con 0x7fb52c06ec00 2024-03-06T07:34:02.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb52c15acb0 0x7fb52c0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb522a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb52c0b4680 0x7fb52c06e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:43730/0 (socket says 172.21.15.86:43730) 2024-03-06T07:34:02.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb522a00640 1 -- 172.21.15.86:0/3103285510 learned_addr learned my addr 172.21.15.86:0/3103285510 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:02.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52be00640 1 --2- 172.21.15.86:0/3103285510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c06ec00 0x7fb52c0ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52b400640 1 -- 172.21.15.86:0/3103285510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c06ec00 msgr2=0x7fb52c0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:02.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52b400640 1 --2- 172.21.15.86:0/3103285510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c06ec00 0x7fb52c0ccfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52b400640 1 -- 172.21.15.86:0/3103285510 >> v1:172.21.15.86:6789/0 conn(0x7fb52c0b4680 legacy=0x7fb52c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52b400640 1 -- 172.21.15.86:0/3103285510 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb52c0b2cf0 con 0x7fb52c15acb0 2024-03-06T07:34:02.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52be00640 1 --2- 172.21.15.86:0/3103285510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c06ec00 0x7fb52c0ccfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:02.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.159+0000 7fb52aa00640 1 -- 172.21.15.86:0/3103285510 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1398023162 0 0) 0x7fb52c15b090 con 0x7fb52c0b4680 2024-03-06T07:34:02.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.160+0000 7fb52b400640 1 --2- 172.21.15.86:0/3103285510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c15acb0 0x7fb52c0b37d0 secure :-1 s=READY pgs=1354 cs=0 l=1 rev1=1 crypto rx=0x7fb51c009d90 tx=0x7fb51c02bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c4c53839cccb0772 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:02.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.160+0000 7fb52aa00640 1 -- 172.21.15.86:0/3103285510 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb51c039070 con 0x7fb52c15acb0 2024-03-06T07:34:02.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb52aa00640 1 -- 172.21.15.86:0/3103285510 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb51c02cc50 con 0x7fb52c15acb0 2024-03-06T07:34:02.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb52aa00640 1 -- 172.21.15.86:0/3103285510 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb51c034440 con 0x7fb52c15acb0 2024-03-06T07:34:02.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 -- 172.21.15.86:0/3103285510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c15acb0 msgr2=0x7fb52c0b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 --2- 172.21.15.86:0/3103285510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c15acb0 0x7fb52c0b37d0 secure :-1 s=READY pgs=1354 cs=0 l=1 rev1=1 crypto rx=0x7fb51c009d90 tx=0x7fb51c02bea0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 -- 172.21.15.86:0/3103285510 shutdown_connections 2024-03-06T07:34:02.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 --2- 172.21.15.86:0/3103285510 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c06ec00 0x7fb52c0ccfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 --2- 172.21.15.86:0/3103285510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c15acb0 0x7fb52c0b37d0 unknown :-1 s=CLOSED pgs=1354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 -- 172.21.15.86:0/3103285510 >> 172.21.15.86:0/3103285510 conn(0x7fb52c0b1730 msgr2=0x7fb52c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:02.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 -- 172.21.15.86:0/3103285510 shutdown_connections 2024-03-06T07:34:02.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.161+0000 7fb532800640 1 -- 172.21.15.86:0/3103285510 wait complete. 2024-03-06T07:34:02.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.163+0000 7fb532800640 1 Processor -- start 2024-03-06T07:34:02.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.163+0000 7fb532800640 1 -- start start 2024-03-06T07:34:02.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb532800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c06ec00 0x7fb52c166220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb532800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c0b4680 0x7fb52c14b190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb532800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb52c15acb0 0x7fb52c15c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb532800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb52c0b3d10 con 0x7fb52c06ec00 2024-03-06T07:34:02.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb532800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb52c0b32b0 con 0x7fb52c15acb0 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb532800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb52c0b4010 con 0x7fb52c0b4680 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb52b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c06ec00 0x7fb52c166220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb52b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c06ec00 0x7fb52c166220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36000/0 (socket says 172.21.15.86:36000) 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.164+0000 7fb52b400640 1 -- 172.21.15.86:0/291021332 learned_addr learned my addr 172.21.15.86:0/291021332 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb52b400640 1 -- 172.21.15.86:0/291021332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c0b4680 msgr2=0x7fb52c14b190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb52b400640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c0b4680 0x7fb52c14b190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb52b400640 1 -- 172.21.15.86:0/291021332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb52c15acb0 msgr2=0x7fb52c15c0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb52b400640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb52c15acb0 0x7fb52c15c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb52b400640 1 -- 172.21.15.86:0/291021332 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb52c150440 con 0x7fb52c06ec00 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb52b400640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c06ec00 0x7fb52c166220 secure :-1 s=READY pgs=1355 cs=0 l=1 rev1=1 crypto rx=0x7fb51c037040 tx=0x7fb51c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:02.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb523e00640 1 -- 172.21.15.86:0/291021332 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb51c039050 con 0x7fb52c06ec00 2024-03-06T07:34:02.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb523e00640 1 -- 172.21.15.86:0/291021332 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb51c03a040 con 0x7fb52c06ec00 2024-03-06T07:34:02.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb523e00640 1 -- 172.21.15.86:0/291021332 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb51c02ca50 con 0x7fb52c06ec00 2024-03-06T07:34:02.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb52c150720 con 0x7fb52c06ec00 2024-03-06T07:34:02.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.165+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb52c150fc0 con 0x7fb52c06ec00 2024-03-06T07:34:02.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.166+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb52c04f330 con 0x7fb52c06ec00 2024-03-06T07:34:02.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.167+0000 7fb523e00640 1 -- 172.21.15.86:0/291021332 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb51c047020 con 0x7fb52c06ec00 2024-03-06T07:34:02.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.168+0000 7fb523e00640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb50806b4d0 0x7fb50806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.168+0000 7fb523e00640 1 -- 172.21.15.86:0/291021332 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fb51c0ac4b0 con 0x7fb52c06ec00 2024-03-06T07:34:02.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.169+0000 7fb522a00640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb50806b4d0 0x7fb50806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.169+0000 7fb522a00640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb50806b4d0 0x7fb50806d990 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7fb514007fd0 tx=0x7fb514007920 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:02.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.177+0000 7fb523e00640 1 -- 172.21.15.86:0/291021332 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb51c04e020 con 0x7fb52c06ec00 2024-03-06T07:34:02.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.414+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb52c06ce60 con 0x7fb52c06ec00 2024-03-06T07:34:02.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.415+0000 7fb523e00640 1 -- 172.21.15.86:0/291021332 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7fb51c07b200 con 0x7fb52c06ec00 2024-03-06T07:34:02.417 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:02.417 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:02.418 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:02.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb50806b4d0 msgr2=0x7fb50806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb50806b4d0 0x7fb50806d990 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7fb514007fd0 tx=0x7fb514007920 comp rx=0 tx=0).stop 2024-03-06T07:34:02.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c06ec00 msgr2=0x7fb52c166220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c06ec00 0x7fb52c166220 secure :-1 s=READY pgs=1355 cs=0 l=1 rev1=1 crypto rx=0x7fb51c037040 tx=0x7fb51c004a40 comp rx=0 tx=0).stop 2024-03-06T07:34:02.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 shutdown_connections 2024-03-06T07:34:02.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb50806b4d0 0x7fb50806d990 unknown :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb52c15acb0 0x7fb52c15c0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb52c0b4680 0x7fb52c14b190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 --2- 172.21.15.86:0/291021332 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb52c06ec00 0x7fb52c166220 unknown :-1 s=CLOSED pgs=1355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 >> 172.21.15.86:0/291021332 conn(0x7fb52c0b1730 msgr2=0x7fb52c06d080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:02.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 shutdown_connections 2024-03-06T07:34:02.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.418+0000 7fb532800640 1 -- 172.21.15.86:0/291021332 wait complete. 2024-03-06T07:34:02.439 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_reintegrated 2024-03-06T07:34:02.701 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_reintegrated":1}} 2024-03-06T07:34:02.702 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_reintegrated": 1 } } 2024-03-06T07:34:02.702 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:02.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.894+0000 7f9f9ca00640 1 Processor -- start 2024-03-06T07:34:02.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.894+0000 7f9f9ca00640 1 -- start start 2024-03-06T07:34:02.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f9ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980b4140 0x7f9f980cebd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f9ca00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f980b1f00 con 0x7f9f9815f4e0 2024-03-06T07:34:02.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f9ca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f980b21a0 con 0x7f9f9815f100 2024-03-06T07:34:02.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f9ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f980b2480 con 0x7f9f980b4140 2024-03-06T07:34:02.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f97e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9f9815f4e0 0x7f9f980cca50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:43740/0 (socket says 172.21.15.86:43740) 2024-03-06T07:34:02.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f97e00640 1 -- 172.21.15.86:0/3631178525 learned_addr learned my addr 172.21.15.86:0/3631178525 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f96a00640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980b4140 0x7f9f980cebd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.896+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 258403538 0 0) 0x7f9f980b1f00 con 0x7f9f9815f4e0 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9f7c003520 con 0x7f9f9815f4e0 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 254536274 0 0) 0x7f9f7c003520 con 0x7f9f9815f4e0 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980b4140 msgr2=0x7f9f980cebd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980b4140 0x7f9f980cebd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 >> v1:172.21.15.194:6789/0 conn(0x7f9f9815f100 legacy=0x7f9f980ce4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f980b2760 con 0x7f9f9815f4e0 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9f90001690 con 0x7f9f9815f4e0 2024-03-06T07:34:02.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 >> v1:172.21.15.86:6789/0 conn(0x7f9f9815f4e0 legacy=0x7f9f980cca50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c0039e0 0x7f9f7c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980b4140 0x7f9f7c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f7c004d40 0x7f9f7c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f980b2480 con 0x7f9f7c004d40 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f7c00bb20 con 0x7f9f7c0039e0 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f7c00bd20 con 0x7f9f980b4140 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.897+0000 7f9f97e00640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f7c004d40 0x7f9f7c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.898+0000 7f9f97400640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c0039e0 0x7f9f7c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.898+0000 7f9f97e00640 1 -- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980b4140 msgr2=0x7f9f7c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.898+0000 7f9f97e00640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980b4140 0x7f9f7c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.898+0000 7f9f97e00640 1 -- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c0039e0 msgr2=0x7f9f7c003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:02.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.898+0000 7f9f97e00640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c0039e0 0x7f9f7c003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.898+0000 7f9f97e00640 1 -- 172.21.15.86:0/3631178525 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f7c003520 con 0x7f9f7c004d40 2024-03-06T07:34:02.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.898+0000 7f9f97400640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c0039e0 0x7f9f7c003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.899+0000 7f9f97e00640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f7c004d40 0x7f9f7c0051c0 secure :-1 s=READY pgs=1357 cs=0 l=1 rev1=1 crypto rx=0x7f9f90007fd0 tx=0x7f9f7c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=13494e9f166c6c65 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f90007d50 con 0x7f9f7c004d40 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f90030070 con 0x7f9f7c004d40 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f96000640 1 -- 172.21.15.86:0/3631178525 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f90005d30 con 0x7f9f7c004d40 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f9ca00640 1 -- 172.21.15.86:0/3631178525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f7c004d40 msgr2=0x7f9f7c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f9ca00640 1 --2- 172.21.15.86:0/3631178525 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f7c004d40 0x7f9f7c0051c0 secure :-1 s=READY pgs=1357 cs=0 l=1 rev1=1 crypto rx=0x7f9f90007fd0 tx=0x7f9f7c005830 comp rx=0 tx=0).stop 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f97e00640 1 -- 172.21.15.86:0/3631178525 reap_dead start 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f9ca00640 1 -- 172.21.15.86:0/3631178525 shutdown_connections 2024-03-06T07:34:02.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f9ca00640 1 -- 172.21.15.86:0/3631178525 >> 172.21.15.86:0/3631178525 conn(0x7f9f980b3620 msgr2=0x7f9f980b3a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:02.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f9ca00640 1 -- 172.21.15.86:0/3631178525 shutdown_connections 2024-03-06T07:34:02.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.900+0000 7f9f9ca00640 1 -- 172.21.15.86:0/3631178525 wait complete. 2024-03-06T07:34:02.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.902+0000 7f9f9ca00640 1 Processor -- start 2024-03-06T07:34:02.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.902+0000 7f9f9ca00640 1 -- start start 2024-03-06T07:34:02.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f9ca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c004d40 0x7f9f980afbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f9ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980cb640 0x7f9f980b4610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f9ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f9815aab0 0x7f9f9815af40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f9ca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f980b33b0 con 0x7f9f9815aab0 2024-03-06T07:34:02.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f9ca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f980b3230 con 0x7f9f7c004d40 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f9ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f980cebd0 con 0x7f9f980cb640 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f97e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f9815aab0 0x7f9f9815af40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f97e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f9815aab0 0x7f9f9815af40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36024/0 (socket says 172.21.15.86:36024) 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.903+0000 7f9f97e00640 1 -- 172.21.15.86:0/2187538467 learned_addr learned my addr 172.21.15.86:0/2187538467 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f97e00640 1 -- 172.21.15.86:0/2187538467 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980cb640 msgr2=0x7f9f980b4610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f97e00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980cb640 0x7f9f980b4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f97400640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c004d40 0x7f9f980afbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f97e00640 1 -- 172.21.15.86:0/2187538467 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c004d40 msgr2=0x7f9f980afbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f97e00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c004d40 0x7f9f980afbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:02.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f97e00640 1 -- 172.21.15.86:0/2187538467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f980b4f80 con 0x7f9f9815aab0 2024-03-06T07:34:02.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f97e00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f9815aab0 0x7f9f9815af40 secure :-1 s=READY pgs=1358 cs=0 l=1 rev1=1 crypto rx=0x7f9f9000c380 tx=0x7f9f90001a60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:02.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f8f400640 1 -- 172.21.15.86:0/2187538467 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f90007d50 con 0x7f9f9815aab0 2024-03-06T07:34:02.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f8f400640 1 -- 172.21.15.86:0/2187538467 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f90004c30 con 0x7f9f9815aab0 2024-03-06T07:34:02.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f8f400640 1 -- 172.21.15.86:0/2187538467 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f9002de60 con 0x7f9f9815aab0 2024-03-06T07:34:02.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.904+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f980b5260 con 0x7f9f9815aab0 2024-03-06T07:34:02.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.905+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f980b5820 con 0x7f9f9815aab0 2024-03-06T07:34:02.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.906+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f9804f330 con 0x7f9f9815aab0 2024-03-06T07:34:02.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.906+0000 7f9f8f400640 1 -- 172.21.15.86:0/2187538467 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9f90007070 con 0x7f9f9815aab0 2024-03-06T07:34:02.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.908+0000 7f9f8f400640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f7806b4d0 0x7f9f7806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:02.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.908+0000 7f9f8f400640 1 -- 172.21.15.86:0/2187538467 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f9f900ab490 con 0x7f9f9815aab0 2024-03-06T07:34:02.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.908+0000 7f9f97400640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f7806b4d0 0x7f9f7806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:02.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.909+0000 7f9f97400640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f7806b4d0 0x7f9f7806d990 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f9f88001d20 tx=0x7f9f88009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:02.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:02.912+0000 7f9f8f400640 1 -- 172.21.15.86:0/2187538467 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9f9007a270 con 0x7f9f9815aab0 2024-03-06T07:34:03.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.172+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9f9806fb60 con 0x7f9f9815aab0 2024-03-06T07:34:03.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.173+0000 7f9f8f400640 1 -- 172.21.15.86:0/2187538467 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f9f90079b70 con 0x7f9f9815aab0 2024-03-06T07:34:03.175 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:03.175 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:03.175 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:03.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.175+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f7806b4d0 msgr2=0x7f9f7806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.175+0000 7f9f9ca00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f7806b4d0 0x7f9f7806d990 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f9f88001d20 tx=0x7f9f88009040 comp rx=0 tx=0).stop 2024-03-06T07:34:03.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.175+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f9815aab0 msgr2=0x7f9f9815af40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.175+0000 7f9f9ca00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f9815aab0 0x7f9f9815af40 secure :-1 s=READY pgs=1358 cs=0 l=1 rev1=1 crypto rx=0x7f9f9000c380 tx=0x7f9f90001a60 comp rx=0 tx=0).stop 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 shutdown_connections 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f7806b4d0 0x7f9f7806d990 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f9815aab0 0x7f9f9815af40 unknown :-1 s=CLOSED pgs=1358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f980cb640 0x7f9f980b4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 --2- 172.21.15.86:0/2187538467 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9f7c004d40 0x7f9f980afbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 >> 172.21.15.86:0/2187538467 conn(0x7f9f980b3620 msgr2=0x7f9f9806fe40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 shutdown_connections 2024-03-06T07:34:03.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.176+0000 7f9f9ca00640 1 -- 172.21.15.86:0/2187538467 wait complete. 2024-03-06T07:34:03.201 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:34:03.434 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:34:03.435 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:34:03.435 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-06T07:34:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.530+0000 7f60b3c45f40 1 Processor -- start 2024-03-06T07:34:03.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.530+0000 7f60b3c45f40 1 -- start start 2024-03-06T07:34:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.531+0000 7f60b3c45f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d975ad990 0x557d975dbb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.531+0000 7f60b3c45f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975add70 0x557d975decb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.531+0000 7f60b3c45f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d9766b9b0 0x557d9766c470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.531+0000 7f60b3c45f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d97553fd0 con 0x557d975ad990 2024-03-06T07:34:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.531+0000 7f60b3c45f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d97557f30 con 0x557d975add70 2024-03-06T07:34:03.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.531+0000 7f60b3c45f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x557d975f7ba0 con 0x557d9766b9b0 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d975ad990 0x557d975dbb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d975ad990 0x557d975dbb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36034/0 (socket says 172.21.15.86:36034) 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 -- 172.21.15.86:0/3970730225 learned_addr learned my addr 172.21.15.86:0/3970730225 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60afc00640 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975add70 0x557d975decb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b1000640 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d9766b9b0 0x557d9766c470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 -- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d9766b9b0 msgr2=0x557d9766c470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d9766b9b0 0x557d9766c470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 -- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975add70 msgr2=0x557d975decb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975add70 0x557d975decb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 -- 172.21.15.86:0/3970730225 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d97681190 con 0x557d975ad990 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b1000640 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d9766b9b0 0x557d9766c470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60afc00640 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975add70 0x557d975decb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:03.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.532+0000 7f60b0600640 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d975ad990 0x557d975dbb00 secure :-1 s=READY pgs=1361 cs=0 l=1 rev1=1 crypto rx=0x7f60a000a430 tx=0x7f60a000a8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb38ac319e833048 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:03.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60ae800640 1 -- 172.21.15.86:0/3970730225 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60a000b920 con 0x557d975ad990 2024-03-06T07:34:03.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60ae800640 1 -- 172.21.15.86:0/3970730225 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f60a000f030 con 0x557d975ad990 2024-03-06T07:34:03.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60ae800640 1 -- 172.21.15.86:0/3970730225 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60a000bd20 con 0x557d975ad990 2024-03-06T07:34:03.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3970730225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d975ad990 msgr2=0x557d975dbb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d975ad990 0x557d975dbb00 secure :-1 s=READY pgs=1361 cs=0 l=1 rev1=1 crypto rx=0x7f60a000a430 tx=0x7f60a000a8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3970730225 shutdown_connections 2024-03-06T07:34:03.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d9766b9b0 0x557d9766c470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975add70 0x557d975decb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3970730225 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d975ad990 0x557d975dbb00 unknown :-1 s=CLOSED pgs=1361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3970730225 >> 172.21.15.86:0/3970730225 conn(0x557d975dcc50 msgr2=0x557d97683c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3970730225 shutdown_connections 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.533+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3970730225 wait complete. 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.534+0000 7f60b3c45f40 1 Processor -- start 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.534+0000 7f60b3c45f40 1 -- start start 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.535+0000 7f60b3c45f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975ad990 0x557d97663f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.535+0000 7f60b3c45f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d975add70 0x557d9767b200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.535+0000 7f60b3c45f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d9766b9b0 0x557d9767c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.535+0000 7f60b3c45f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d9766adc0 con 0x557d9766b9b0 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.535+0000 7f60b3c45f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d975de700 con 0x557d975ad990 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.535+0000 7f60b3c45f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x557d9766b040 con 0x557d975add70 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d9766b9b0 0x557d9767c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d9766b9b0 0x557d9767c0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36042/0 (socket says 172.21.15.86:36042) 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 -- 172.21.15.86:0/3744923324 learned_addr learned my addr 172.21.15.86:0/3744923324 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60afc00640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d975add70 0x557d9767b200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b0600640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975ad990 0x557d97663f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 -- 172.21.15.86:0/3744923324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d975add70 msgr2=0x557d9767b200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d975add70 0x557d9767b200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 -- 172.21.15.86:0/3744923324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975ad990 msgr2=0x557d97663f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x557d975ad990 0x557d97663f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 -- 172.21.15.86:0/3744923324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d9767bc20 con 0x557d9766b9b0 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60afc00640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x557d975add70 0x557d9767b200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.536+0000 7f60b1000640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d9766b9b0 0x557d9767c0e0 secure :-1 s=READY pgs=1362 cs=0 l=1 rev1=1 crypto rx=0x7f60a8007000 tx=0x7f60a8003330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.537+0000 7f60ad400640 1 -- 172.21.15.86:0/3744923324 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60a8066610 con 0x557d9766b9b0 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.537+0000 7f60ad400640 1 -- 172.21.15.86:0/3744923324 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f60a80667d0 con 0x557d9766b9b0 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.537+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x557d976681a0 con 0x557d9766b9b0 2024-03-06T07:34:03.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.537+0000 7f60ad400640 1 -- 172.21.15.86:0/3744923324 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60a8066bc0 con 0x557d9766b9b0 2024-03-06T07:34:03.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.537+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557d97668760 con 0x557d9766b9b0 2024-03-06T07:34:03.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.539+0000 7f60ad400640 1 -- 172.21.15.86:0/3744923324 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f60a8064020 con 0x557d9766b9b0 2024-03-06T07:34:03.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.540+0000 7f60ad400640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6090069eb0 0x7f609006c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.540+0000 7f60ad400640 1 -- 172.21.15.86:0/3744923324 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f60a80cb6d0 con 0x557d9766b9b0 2024-03-06T07:34:03.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.540+0000 7f60b0600640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6090069eb0 0x7f609006c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.540+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x557d9767b740 0x557d97668c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.540+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e116) v9 -- 0x557d976691c0 con 0x557d9767b740 2024-03-06T07:34:03.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.540+0000 7f60afc00640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x557d9767b740 0x557d97668c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.541+0000 7f60b0600640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6090069eb0 0x7f609006c370 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f60a000a930 tx=0x7f60a0016040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:03.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.541+0000 7f60afc00640 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x557d9767b740 0x557d97668c80 crc :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60afc00640 1 -- 172.21.15.86:0/3744923324 <== osd.0 v2:172.21.15.86:6800/1601033517 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (93) out=93b] v0'0 uv10 ondisk = 0) v8 ==== 164+0+93 (crc 0 0 0) 0x7f609c00d580 con 0x557d9767b740 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x557d9767b740 msgr2=0x557d97668c80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x557d9767b740 0x557d97668c80 crc :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6090069eb0 msgr2=0x7f609006c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6090069eb0 0x7f609006c370 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f60a000a930 tx=0x7f60a0016040 comp rx=0 tx=0).stop 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d9766b9b0 msgr2=0x557d9767c0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 --2- 172.21.15.86:0/3744923324 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x557d9766b9b0 0x557d9767c0e0 secure :-1 s=READY pgs=1362 cs=0 l=1 rev1=1 crypto rx=0x7f60a8007000 tx=0x7f60a8003330 comp rx=0 tx=0).stop 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b1000640 1 -- 172.21.15.86:0/3744923324 reap_dead start 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 shutdown_connections 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 >> 172.21.15.86:0/3744923324 conn(0x557d975dcc50 msgr2=0x557d975f9ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 shutdown_connections 2024-03-06T07:34:03.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.542+0000 7f60b3c45f40 1 -- 172.21.15.86:0/3744923324 wait complete. 2024-03-06T07:34:03.548 INFO:teuthology.orchestra.run.smithi086.stdout:W�file_cdir_2# 2024-03-06T07:34:03.548 DEBUG:teuthology.orchestra.run.smithi086:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-06T07:34:03.635 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:34:03.635 INFO:teuthology.orchestra.run.smithi086.stdout: "ino": 1099511627777, 2024-03-06T07:34:03.635 INFO:teuthology.orchestra.run.smithi086.stdout: "ancestors": [ 2024-03-06T07:34:03.635 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:03.635 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 1099511628278, 2024-03-06T07:34:03.635 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "file_c", 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 17 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 1, 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "dir_2", 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 35 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: "pool": 17, 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: "old_pools": [] 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:34:03.636 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:03.640 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:03.641 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec ln dir_2/file_c dir_2/file_d) 2024-03-06T07:34:03.693 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:03.693 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f dir_2/file_c) 2024-03-06T07:34:03.740 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:03.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.931+0000 7fc3bf800640 1 Processor -- start 2024-03-06T07:34:03.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.931+0000 7fc3bf800640 1 -- start start 2024-03-06T07:34:03.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.932+0000 7fc3bf800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80d24c0 0x7fc3b80d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.932+0000 7fc3bf800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc3b80d4c50 con 0x7fc3b80b2de0 2024-03-06T07:34:03.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.932+0000 7fc3bf800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc3b80cfa30 con 0x7fc3b80b0aa0 2024-03-06T07:34:03.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.932+0000 7fc3bf800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc3b80d4a00 con 0x7fc3b80d24c0 2024-03-06T07:34:03.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.933+0000 7fc3b7e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc3b80b2de0 0x7fc3b80d28a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:43754/0 (socket says 172.21.15.86:43754) 2024-03-06T07:34:03.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.933+0000 7fc3b7e00640 1 -- 172.21.15.86:0/2150605190 learned_addr learned my addr 172.21.15.86:0/2150605190 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:03.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.933+0000 7fc3bce00640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80d24c0 0x7fc3b80d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.933+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1732600170 0 0) 0x7fc3b80d4c50 con 0x7fc3b80b2de0 2024-03-06T07:34:03.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.933+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc3a0003520 con 0x7fc3b80b2de0 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2703867383 0 0) 0x7fc3a0003520 con 0x7fc3b80b2de0 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80d24c0 msgr2=0x7fc3b80d1cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80d24c0 0x7fc3b80d1cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 >> v1:172.21.15.194:6789/0 conn(0x7fc3b80b0aa0 legacy=0x7fc3b80b0f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3b80cfd10 con 0x7fc3b80b2de0 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc3a4001670 con 0x7fc3b80b2de0 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 >> v1:172.21.15.86:6789/0 conn(0x7fc3b80b2de0 legacy=0x7fc3b80d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3a00039e0 0x7fc3a0003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3a0004800 0x7fc3a0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3a000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3b80d4a00 con 0x7fc3a00052e0 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3a000c160 con 0x7fc3a00039e0 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc3a000c360 con 0x7fc3a0004800 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3bce00640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80d24c0 0x7fc3b80d1cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:34:03.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.934+0000 7fc3bd800640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3a000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3b7e00640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3a0004800 0x7fc3a0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3bd800640 1 -- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3a0004800 msgr2=0x7fc3a0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3bce00640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3a00039e0 0x7fc3a0003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3bd800640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3a0004800 0x7fc3a0004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3bd800640 1 -- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3a00039e0 msgr2=0x7fc3a0003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3bd800640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3a00039e0 0x7fc3a0003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3bd800640 1 -- 172.21.15.86:0/2150605190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3a0003520 con 0x7fc3a00052e0 2024-03-06T07:34:03.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.935+0000 7fc3bce00640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3a00039e0 0x7fc3a0003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:03.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.936+0000 7fc3bd800640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3a000b9c0 secure :-1 s=READY pgs=1364 cs=0 l=1 rev1=1 crypto rx=0x7fc3b0001e50 tx=0x7fc3a0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=39f1fe241500a124 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:03.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.936+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3b0001a70 con 0x7fc3a00052e0 2024-03-06T07:34:03.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.936+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3b0003b70 con 0x7fc3a00052e0 2024-03-06T07:34:03.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.936+0000 7fc3b7400640 1 -- 172.21.15.86:0/2150605190 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3b00050d0 con 0x7fc3a00052e0 2024-03-06T07:34:03.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.936+0000 7fc3bf800640 1 -- 172.21.15.86:0/2150605190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 msgr2=0x7fc3a000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.936+0000 7fc3bf800640 1 --2- 172.21.15.86:0/2150605190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3a000b9c0 secure :-1 s=READY pgs=1364 cs=0 l=1 rev1=1 crypto rx=0x7fc3b0001e50 tx=0x7fc3a0005830 comp rx=0 tx=0).stop 2024-03-06T07:34:03.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.936+0000 7fc3bd800640 1 -- 172.21.15.86:0/2150605190 reap_dead start 2024-03-06T07:34:03.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.937+0000 7fc3bf800640 1 -- 172.21.15.86:0/2150605190 shutdown_connections 2024-03-06T07:34:03.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.937+0000 7fc3bf800640 1 -- 172.21.15.86:0/2150605190 >> 172.21.15.86:0/2150605190 conn(0x7fc3b80d3ee0 msgr2=0x7fc3b80d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:03.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.937+0000 7fc3bf800640 1 -- 172.21.15.86:0/2150605190 shutdown_connections 2024-03-06T07:34:03.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.937+0000 7fc3bf800640 1 -- 172.21.15.86:0/2150605190 wait complete. 2024-03-06T07:34:03.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.939+0000 7fc3bf800640 1 Processor -- start 2024-03-06T07:34:03.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.939+0000 7fc3bf800640 1 -- start start 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bf800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3b815e840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bf800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3b80cafb0 0x7fc3b8150320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bf800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80b7050 0x7fc3b80b7500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bf800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3b80d0080 con 0x7fc3a00052e0 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bf800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3b80cf7e0 con 0x7fc3b80cafb0 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bf800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc3b80d1cc0 con 0x7fc3b80b7050 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3b815e840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3b815e840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:36058/0 (socket says 172.21.15.86:36058) 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 -- 172.21.15.86:0/3182225807 learned_addr learned my addr 172.21.15.86:0/3182225807 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:03.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 -- 172.21.15.86:0/3182225807 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80b7050 msgr2=0x7fc3b80b7500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80b7050 0x7fc3b80b7500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 -- 172.21.15.86:0/3182225807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3b80cafb0 msgr2=0x7fc3b8150320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3b80cafb0 0x7fc3b8150320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.940+0000 7fc3bce00640 1 -- 172.21.15.86:0/3182225807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3b8150860 con 0x7fc3a00052e0 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.941+0000 7fc3bce00640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3b815e840 secure :-1 s=READY pgs=1365 cs=0 l=1 rev1=1 crypto rx=0x7fc3ac00f4c0 tx=0x7fc3ac00f980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.941+0000 7fc3b4c00640 1 -- 172.21.15.86:0/3182225807 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3ac018020 con 0x7fc3a00052e0 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.941+0000 7fc3b4c00640 1 -- 172.21.15.86:0/3182225807 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3ac014890 con 0x7fc3a00052e0 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.941+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3b815ef40 con 0x7fc3a00052e0 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.941+0000 7fc3b4c00640 1 -- 172.21.15.86:0/3182225807 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3ac014c60 con 0x7fc3a00052e0 2024-03-06T07:34:03.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.941+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3b815f500 con 0x7fc3a00052e0 2024-03-06T07:34:03.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.942+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc3b804f330 con 0x7fc3a00052e0 2024-03-06T07:34:03.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.943+0000 7fc3b4c00640 1 -- 172.21.15.86:0/3182225807 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc3ac01b020 con 0x7fc3a00052e0 2024-03-06T07:34:03.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.944+0000 7fc3b4c00640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc39806b4d0 0x7fc39806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:03.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.944+0000 7fc3b4c00640 1 -- 172.21.15.86:0/3182225807 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fc3ac08a7b0 con 0x7fc3a00052e0 2024-03-06T07:34:03.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.945+0000 7fc3b7e00640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc39806b4d0 0x7fc39806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:03.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.945+0000 7fc3b7e00640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc39806b4d0 0x7fc39806d990 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7fc3a40017e0 tx=0x7fc3a4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:03.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:03.949+0000 7fc3b4c00640 1 -- 172.21.15.86:0/3182225807 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc3ac022020 con 0x7fc3a00052e0 2024-03-06T07:34:04.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.184+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc3b80d2270 con 0x7fc3a00052e0 2024-03-06T07:34:04.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.185+0000 7fc3b4c00640 1 -- 172.21.15.86:0/3182225807 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7fc3b8150860 con 0x7fc3a00052e0 2024-03-06T07:34:04.187 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:04.187 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:04.187 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc39806b4d0 msgr2=0x7fc39806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc39806b4d0 0x7fc39806d990 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7fc3a40017e0 tx=0x7fc3a4009040 comp rx=0 tx=0).stop 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 msgr2=0x7fc3b815e840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3b815e840 secure :-1 s=READY pgs=1365 cs=0 l=1 rev1=1 crypto rx=0x7fc3ac00f4c0 tx=0x7fc3ac00f980 comp rx=0 tx=0).stop 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 shutdown_connections 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc39806b4d0 0x7fc39806d990 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3b80b7050 0x7fc3b80b7500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc3b80cafb0 0x7fc3b8150320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:04.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 --2- 172.21.15.86:0/3182225807 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3a00052e0 0x7fc3b815e840 unknown :-1 s=CLOSED pgs=1365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:04.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 >> 172.21.15.86:0/3182225807 conn(0x7fc3b80d3ee0 msgr2=0x7fc3b80d42c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:04.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 shutdown_connections 2024-03-06T07:34:04.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:04.188+0000 7fc3bf800640 1 -- 172.21.15.86:0/3182225807 wait complete. 2024-03-06T07:34:04.208 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-06T07:34:04.417 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"num_strays":1}} 2024-03-06T07:34:04.417 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 1 } } 2024-03-06T07:34:04.417 DEBUG:tasks.ceph_test_case:wait_until_equal: 1 != 0, waiting (timeout=60)... 2024-03-06T07:34:09.418 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:09.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.630+0000 7f07d9a00640 1 Processor -- start 2024-03-06T07:34:09.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.630+0000 7f07d9a00640 1 -- start start 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.632+0000 7f07d9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f07d415aab0 0x7f07d40b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.632+0000 7f07d9a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f07d406d2e0 0x7f07d40cc2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.632+0000 7f07d9a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f07d415ae90 con 0x7f07d40cc7e0 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.632+0000 7f07d9a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f07d40ccc30 con 0x7f07d415aab0 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.632+0000 7f07d9a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f07d406ce50 con 0x7f07d406d2e0 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d9000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f07d40cc7e0 0x7f07d40cd460 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40424/0 (socket says 172.21.15.86:40424) 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d3e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f07d415aab0 0x7f07d40b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:09.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d9000640 1 -- 172.21.15.86:0/3330992137 learned_addr learned my addr 172.21.15.86:0/3330992137 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:09.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07cb400640 1 --2- 172.21.15.86:0/3330992137 >> v2:172.21.15.194:3300/0 conn(0x7f07d406d2e0 0x7f07d40cc2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:09.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d3400640 1 -- 172.21.15.86:0/3330992137 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2065951390 0 0) 0x7f07d415ae90 con 0x7f07d40cc7e0 2024-03-06T07:34:09.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d3400640 1 -- 172.21.15.86:0/3330992137 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f07b8003520 con 0x7f07d40cc7e0 2024-03-06T07:34:09.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d3e00640 1 -- 172.21.15.86:0/3330992137 >> v2:172.21.15.194:3300/0 conn(0x7f07d406d2e0 msgr2=0x7f07d40cc2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:09.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d3e00640 1 --2- 172.21.15.86:0/3330992137 >> v2:172.21.15.194:3300/0 conn(0x7f07d406d2e0 0x7f07d40cc2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:09.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d3e00640 1 -- 172.21.15.86:0/3330992137 >> v1:172.21.15.86:6789/0 conn(0x7f07d40cc7e0 legacy=0x7f07d40cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:09.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.633+0000 7f07d3e00640 1 -- 172.21.15.86:0/3330992137 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f07d40b3160 con 0x7f07d415aab0 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.634+0000 7f07d3e00640 1 --2- 172.21.15.86:0/3330992137 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d40b45e0 secure :-1 s=READY pgs=1367 cs=0 l=1 rev1=1 crypto rx=0x7f07d406de70 tx=0x7f07c402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9515e4bb4c62752b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d3400640 1 -- 172.21.15.86:0/3330992137 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07c4035020 con 0x7f07d415aab0 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d3400640 1 -- 172.21.15.86:0/3330992137 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f07c4031830 con 0x7f07d415aab0 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d3400640 1 -- 172.21.15.86:0/3330992137 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07c4031be0 con 0x7f07d415aab0 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d9a00640 1 -- 172.21.15.86:0/3330992137 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 msgr2=0x7f07d40b45e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d9a00640 1 --2- 172.21.15.86:0/3330992137 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d40b45e0 secure :-1 s=READY pgs=1367 cs=0 l=1 rev1=1 crypto rx=0x7f07d406de70 tx=0x7f07c402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d9a00640 1 -- 172.21.15.86:0/3330992137 shutdown_connections 2024-03-06T07:34:09.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d9a00640 1 --2- 172.21.15.86:0/3330992137 >> v2:172.21.15.194:3300/0 conn(0x7f07d406d2e0 0x7f07d40cc2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:09.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d9a00640 1 --2- 172.21.15.86:0/3330992137 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d40b45e0 unknown :-1 s=CLOSED pgs=1367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:09.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.635+0000 7f07d9a00640 1 -- 172.21.15.86:0/3330992137 >> 172.21.15.86:0/3330992137 conn(0x7f07d406afc0 msgr2=0x7f07d406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:09.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.636+0000 7f07d9a00640 1 -- 172.21.15.86:0/3330992137 shutdown_connections 2024-03-06T07:34:09.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.636+0000 7f07d9a00640 1 -- 172.21.15.86:0/3330992137 wait complete. 2024-03-06T07:34:09.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.638+0000 7f07d9a00640 1 Processor -- start 2024-03-06T07:34:09.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.638+0000 7f07d9a00640 1 -- start start 2024-03-06T07:34:09.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07d406d2e0 0x7f07d41431d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:09.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07d40cc7e0 0x7f07d4143f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:09.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d414bd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:09.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07d40b2e80 con 0x7f07d415aab0 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07d40b2ba0 con 0x7f07d40cc7e0 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f07d406cae0 con 0x7f07d406d2e0 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d414bd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d414bd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51288/0 (socket says 172.21.15.86:51288) 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.640+0000 7f07d9000640 1 -- 172.21.15.86:0/1300544940 learned_addr learned my addr 172.21.15.86:0/1300544940 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d3e00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07d406d2e0 0x7f07d41431d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9000640 1 -- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07d406d2e0 msgr2=0x7f07d41431d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9000640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07d406d2e0 0x7f07d41431d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9000640 1 -- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07d40cc7e0 msgr2=0x7f07d4143f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9000640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07d40cc7e0 0x7f07d4143f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9000640 1 -- 172.21.15.86:0/1300544940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f07d4143710 con 0x7f07d415aab0 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d3e00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07d406d2e0 0x7f07d41431d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:09.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9000640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d414bd60 secure :-1 s=READY pgs=1368 cs=0 l=1 rev1=1 crypto rx=0x7f07cc00cb60 tx=0x7f07cc00a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:09.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d0c00640 1 -- 172.21.15.86:0/1300544940 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07cc017070 con 0x7f07d415aab0 2024-03-06T07:34:09.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d0c00640 1 -- 172.21.15.86:0/1300544940 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f07cc00ad40 con 0x7f07d415aab0 2024-03-06T07:34:09.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f07d4070d50 con 0x7f07d415aab0 2024-03-06T07:34:09.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d0c00640 1 -- 172.21.15.86:0/1300544940 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07cc012600 con 0x7f07d415aab0 2024-03-06T07:34:09.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.641+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f07d4071310 con 0x7f07d415aab0 2024-03-06T07:34:09.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.643+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f07d404f330 con 0x7f07d415aab0 2024-03-06T07:34:09.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.644+0000 7f07d0c00640 1 -- 172.21.15.86:0/1300544940 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f07cc01a070 con 0x7f07d415aab0 2024-03-06T07:34:09.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.644+0000 7f07d0c00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f07b006b500 0x7f07b006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:09.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.645+0000 7f07d0c00640 1 -- 172.21.15.86:0/1300544940 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f07cc089f70 con 0x7f07d415aab0 2024-03-06T07:34:09.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.645+0000 7f07d3e00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f07b006b500 0x7f07b006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:09.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.645+0000 7f07d3e00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f07b006b500 0x7f07b006d9c0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f07c4036000 tx=0x7f07c4008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:09.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:09.649+0000 7f07d0c00640 1 -- 172.21.15.86:0/1300544940 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f07cc058670 con 0x7f07d415aab0 2024-03-06T07:34:10.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.004+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f07d40af690 con 0x7f07d415aab0 2024-03-06T07:34:10.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.005+0000 7f07d0c00640 1 -- 172.21.15.86:0/1300544940 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f07cc05c710 con 0x7f07d415aab0 2024-03-06T07:34:10.007 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:10.008 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:10.008 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:10.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.007+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f07b006b500 msgr2=0x7f07b006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:10.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.007+0000 7f07d9a00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f07b006b500 0x7f07b006d9c0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f07c4036000 tx=0x7f07c4008030 comp rx=0 tx=0).stop 2024-03-06T07:34:10.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.007+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 msgr2=0x7f07d414bd60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:10.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.007+0000 7f07d9a00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d414bd60 secure :-1 s=READY pgs=1368 cs=0 l=1 rev1=1 crypto rx=0x7f07cc00cb60 tx=0x7f07cc00a430 comp rx=0 tx=0).stop 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 shutdown_connections 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f07b006b500 0x7f07b006d9c0 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f07d415aab0 0x7f07d414bd60 unknown :-1 s=CLOSED pgs=1368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f07d40cc7e0 0x7f07d4143f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 --2- 172.21.15.86:0/1300544940 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f07d406d2e0 0x7f07d41431d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 >> 172.21.15.86:0/1300544940 conn(0x7f07d406afc0 msgr2=0x7f07d40b1e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 shutdown_connections 2024-03-06T07:34:10.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.008+0000 7f07d9a00640 1 -- 172.21.15.86:0/1300544940 wait complete. 2024-03-06T07:34:10.036 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-06T07:34:10.323 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"num_strays":0}} 2024-03-06T07:34:10.323 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 0 } } 2024-03-06T07:34:10.324 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:10.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.521+0000 7fe817000640 1 Processor -- start 2024-03-06T07:34:10.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.522+0000 7fe817000640 1 -- start start 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe817000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe81015acb0 0x7fe8100b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe817000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe8100b4680 0x7fe81006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe817000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe81015b090 con 0x7fe810072340 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe817000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe8100b2a30 con 0x7fe81015acb0 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe817000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe8100b4360 con 0x7fe8100b4680 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe80fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe81015acb0 0x7fe8100b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe80fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe81015acb0 0x7fe8100b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51294/0 (socket says 172.21.15.86:51294) 2024-03-06T07:34:10.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.523+0000 7fe80fe00640 1 -- 172.21.15.86:0/1461819646 learned_addr learned my addr 172.21.15.86:0/1461819646 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:10.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.524+0000 7fe80fe00640 1 -- 172.21.15.86:0/1461819646 >> v2:172.21.15.194:3300/0 conn(0x7fe8100b4680 msgr2=0x7fe81006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:10.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.524+0000 7fe80fe00640 1 --2- 172.21.15.86:0/1461819646 >> v2:172.21.15.194:3300/0 conn(0x7fe8100b4680 0x7fe81006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.524+0000 7fe80fe00640 1 -- 172.21.15.86:0/1461819646 >> v1:172.21.15.194:6789/0 conn(0x7fe810072340 legacy=0x7fe8100ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:10.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.524+0000 7fe80fe00640 1 -- 172.21.15.86:0/1461819646 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8100b2cf0 con 0x7fe81015acb0 2024-03-06T07:34:10.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.525+0000 7fe80fe00640 1 --2- 172.21.15.86:0/1461819646 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe81015acb0 0x7fe8100b37d0 secure :-1 s=READY pgs=1369 cs=0 l=1 rev1=1 crypto rx=0x7fe810068560 tx=0x7fe80002db20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=81f9f32cf9b2a2e0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:10.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.525+0000 7fe80ea00640 1 -- 172.21.15.86:0/1461819646 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe80002f8e0 con 0x7fe81015acb0 2024-03-06T07:34:10.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.525+0000 7fe80ea00640 1 -- 172.21.15.86:0/1461819646 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe80002fa80 con 0x7fe81015acb0 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.525+0000 7fe80ea00640 1 -- 172.21.15.86:0/1461819646 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe80002fe50 con 0x7fe81015acb0 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 -- 172.21.15.86:0/1461819646 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe81015acb0 msgr2=0x7fe8100b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 --2- 172.21.15.86:0/1461819646 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe81015acb0 0x7fe8100b37d0 secure :-1 s=READY pgs=1369 cs=0 l=1 rev1=1 crypto rx=0x7fe810068560 tx=0x7fe80002db20 comp rx=0 tx=0).stop 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 -- 172.21.15.86:0/1461819646 shutdown_connections 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 --2- 172.21.15.86:0/1461819646 >> v2:172.21.15.194:3300/0 conn(0x7fe8100b4680 0x7fe81006e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 --2- 172.21.15.86:0/1461819646 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe81015acb0 0x7fe8100b37d0 unknown :-1 s=CLOSED pgs=1369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 -- 172.21.15.86:0/1461819646 >> 172.21.15.86:0/1461819646 conn(0x7fe8100b1730 msgr2=0x7fe8100b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 -- 172.21.15.86:0/1461819646 shutdown_connections 2024-03-06T07:34:10.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.526+0000 7fe817000640 1 -- 172.21.15.86:0/1461819646 wait complete. 2024-03-06T07:34:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.528+0000 7fe817000640 1 Processor -- start 2024-03-06T07:34:10.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.528+0000 7fe817000640 1 -- start start 2024-03-06T07:34:10.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe817000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe810072340 0x7fe810143410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:10.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe817000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe8100b4680 0x7fe8101570e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:10.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe817000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe81015acb0 0x7fe810157620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:10.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe817000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8100b3d10 con 0x7fe810072340 2024-03-06T07:34:10.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe817000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8100b32b0 con 0x7fe81015acb0 2024-03-06T07:34:10.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe817000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe8100b4010 con 0x7fe8100b4680 2024-03-06T07:34:10.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe810072340 0x7fe810143410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:10.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe810072340 0x7fe810143410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51304/0 (socket says 172.21.15.86:51304) 2024-03-06T07:34:10.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 -- 172.21.15.86:0/2733273341 learned_addr learned my addr 172.21.15.86:0/2733273341 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:10.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 -- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe8100b4680 msgr2=0x7fe8101570e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:10.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe8100b4680 0x7fe8101570e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe815000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe81015acb0 0x7fe810157620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:10.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 -- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe81015acb0 msgr2=0x7fe810157620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe81015acb0 0x7fe810157620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe80fe00640 1 -- 172.21.15.86:0/2733273341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe81015db10 con 0x7fe810072340 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.529+0000 7fe815000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe81015acb0 0x7fe810157620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.530+0000 7fe80fe00640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe810072340 0x7fe810143410 secure :-1 s=READY pgs=1370 cs=0 l=1 rev1=1 crypto rx=0x7fe800000c00 tx=0x7fe8000046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.530+0000 7fe807e00640 1 -- 172.21.15.86:0/2733273341 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe800047070 con 0x7fe810072340 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.530+0000 7fe807e00640 1 -- 172.21.15.86:0/2733273341 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe80002f8a0 con 0x7fe810072340 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.530+0000 7fe807e00640 1 -- 172.21.15.86:0/2733273341 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe800040660 con 0x7fe810072340 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.530+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe81014fd80 con 0x7fe810072340 2024-03-06T07:34:10.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.530+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe810143950 con 0x7fe810072340 2024-03-06T07:34:10.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.532+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe81004f330 con 0x7fe810072340 2024-03-06T07:34:10.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.532+0000 7fe807e00640 1 -- 172.21.15.86:0/2733273341 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe80004a070 con 0x7fe810072340 2024-03-06T07:34:10.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.533+0000 7fe807e00640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7f006b4d0 0x7fe7f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:10.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.534+0000 7fe807e00640 1 -- 172.21.15.86:0/2733273341 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fe8000ae870 con 0x7fe810072340 2024-03-06T07:34:10.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.534+0000 7fe80f400640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7f006b4d0 0x7fe7f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:10.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.534+0000 7fe80f400640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7f006b4d0 0x7fe7f006d990 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7fe810151ea0 tx=0x7fe7fc006c90 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:10.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.538+0000 7fe807e00640 1 -- 172.21.15.86:0/2733273341 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe80002fa50 con 0x7fe810072340 2024-03-06T07:34:10.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.857+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe8100b1e80 con 0x7fe810072340 2024-03-06T07:34:10.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.858+0000 7fe807e00640 1 -- 172.21.15.86:0/2733273341 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7fe80007d5c0 con 0x7fe810072340 2024-03-06T07:34:10.861 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:10.861 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:10.861 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:10.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.861+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7f006b4d0 msgr2=0x7fe7f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.861+0000 7fe817000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7f006b4d0 0x7fe7f006d990 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7fe810151ea0 tx=0x7fe7fc006c90 comp rx=0 tx=0).stop 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.861+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe810072340 msgr2=0x7fe810143410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.861+0000 7fe817000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe810072340 0x7fe810143410 secure :-1 s=READY pgs=1370 cs=0 l=1 rev1=1 crypto rx=0x7fe800000c00 tx=0x7fe8000046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.861+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 shutdown_connections 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.862+0000 7fe817000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe7f006b4d0 0x7fe7f006d990 unknown :-1 s=CLOSED pgs=588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.862+0000 7fe817000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe81015acb0 0x7fe810157620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.862+0000 7fe817000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe8100b4680 0x7fe8101570e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.862+0000 7fe817000640 1 --2- 172.21.15.86:0/2733273341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe810072340 0x7fe810143410 unknown :-1 s=CLOSED pgs=1370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.862+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 >> 172.21.15.86:0/2733273341 conn(0x7fe8100b1730 msgr2=0x7fe8100ccf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.862+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 shutdown_connections 2024-03-06T07:34:10.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:10.862+0000 7fe817000640 1 -- 172.21.15.86:0/2733273341 wait complete. 2024-03-06T07:34:10.883 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_reintegrated 2024-03-06T07:34:11.098 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_reintegrated":2}} 2024-03-06T07:34:11.099 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_reintegrated": 2 } } 2024-03-06T07:34:11.099 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:11.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.289+0000 7fd5bd000640 1 Processor -- start 2024-03-06T07:34:11.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.290+0000 7fd5bd000640 1 -- start start 2024-03-06T07:34:11.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.292+0000 7fd5bd000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd5b80d24c0 0x7fd5b80d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.292+0000 7fd5bd000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd5b80b2de0 0x7fd5b80d28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.292+0000 7fd5bd000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80d07f0 0x7fd5b80d0c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.292+0000 7fd5bd000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd5b80d4a00 con 0x7fd5b80d24c0 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.292+0000 7fd5bd000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd5b80cf7e0 con 0x7fd5b80b2de0 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.292+0000 7fd5bd000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b80d0080 con 0x7fd5b80d07f0 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.292+0000 7fd5b5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd5b80d24c0 0x7fd5b80d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd5b80d24c0 0x7fd5b80d1cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51314/0 (socket says 172.21.15.86:51314) 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b5e00640 1 -- 172.21.15.86:0/2763553648 learned_addr learned my addr 172.21.15.86:0/2763553648 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5afe00640 1 --2- 172.21.15.86:0/2763553648 >> v2:172.21.15.194:3300/0 conn(0x7fd5b80b2de0 0x7fd5b80d28a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b6800640 1 --2- 172.21.15.86:0/2763553648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80d07f0 0x7fd5b80d0c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.296 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b5e00640 1 -- 172.21.15.86:0/2763553648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80d07f0 msgr2=0x7fd5b80d0c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b5e00640 1 --2- 172.21.15.86:0/2763553648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80d07f0 0x7fd5b80d0c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b5e00640 1 -- 172.21.15.86:0/2763553648 >> v2:172.21.15.194:3300/0 conn(0x7fd5b80b2de0 msgr2=0x7fd5b80d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b5e00640 1 --2- 172.21.15.86:0/2763553648 >> v2:172.21.15.194:3300/0 conn(0x7fd5b80b2de0 0x7fd5b80d28a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.293+0000 7fd5b5e00640 1 -- 172.21.15.86:0/2763553648 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5b80d2de0 con 0x7fd5b80d24c0 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.294+0000 7fd5b5e00640 1 --2- 172.21.15.86:0/2763553648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d24c0 0x7fd5b80d1cc0 secure :-1 s=READY pgs=1371 cs=0 l=1 rev1=1 crypto rx=0x7fd5b808d6a0 tx=0x7fd5a802b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4bee1318c69b354 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.294+0000 7fd5b5400640 1 -- 172.21.15.86:0/2763553648 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5a802e070 con 0x7fd5b80d24c0 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5b5400640 1 -- 172.21.15.86:0/2763553648 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd5a8037070 con 0x7fd5b80d24c0 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5b5400640 1 -- 172.21.15.86:0/2763553648 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5a8031a00 con 0x7fd5b80d24c0 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 -- 172.21.15.86:0/2763553648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d24c0 msgr2=0x7fd5b80d1cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 --2- 172.21.15.86:0/2763553648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d24c0 0x7fd5b80d1cc0 secure :-1 s=READY pgs=1371 cs=0 l=1 rev1=1 crypto rx=0x7fd5b808d6a0 tx=0x7fd5a802b880 comp rx=0 tx=0).stop 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 -- 172.21.15.86:0/2763553648 shutdown_connections 2024-03-06T07:34:11.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 --2- 172.21.15.86:0/2763553648 >> v2:172.21.15.194:3300/0 conn(0x7fd5b80b2de0 0x7fd5b80d28a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 --2- 172.21.15.86:0/2763553648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80d07f0 0x7fd5b80d0c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 --2- 172.21.15.86:0/2763553648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d24c0 0x7fd5b80d1cc0 unknown :-1 s=CLOSED pgs=1371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 -- 172.21.15.86:0/2763553648 >> 172.21.15.86:0/2763553648 conn(0x7fd5b80d3ee0 msgr2=0x7fd5b80d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:11.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 -- 172.21.15.86:0/2763553648 shutdown_connections 2024-03-06T07:34:11.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.295+0000 7fd5bd000640 1 -- 172.21.15.86:0/2763553648 wait complete. 2024-03-06T07:34:11.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.297+0000 7fd5bd000640 1 Processor -- start 2024-03-06T07:34:11.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.297+0000 7fd5bd000640 1 -- start start 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.298+0000 7fd5bd000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80b2de0 0x7fd5b8161260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.298+0000 7fd5bd000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d07f0 0x7fd5b80b7560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.298+0000 7fd5bd000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5b80d24c0 0x7fd5b80b7aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.298+0000 7fd5bd000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b80cfd10 con 0x7fd5b80d07f0 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.298+0000 7fd5bd000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b80cfa30 con 0x7fd5b80d24c0 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.298+0000 7fd5bd000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b815e5b0 con 0x7fd5b80b2de0 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d07f0 0x7fd5b80b7560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d07f0 0x7fd5b80b7560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51328/0 (socket says 172.21.15.86:51328) 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 -- 172.21.15.86:0/901401668 learned_addr learned my addr 172.21.15.86:0/901401668 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5b5e00640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80b2de0 0x7fd5b8161260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5b6800640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5b80d24c0 0x7fd5b80b7aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 -- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80b2de0 msgr2=0x7fd5b8161260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80b2de0 0x7fd5b8161260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 -- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5b80d24c0 msgr2=0x7fd5b80b7aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5b80d24c0 0x7fd5b80b7aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 -- 172.21.15.86:0/901401668 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5b80b8320 con 0x7fd5b80d07f0 2024-03-06T07:34:11.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.299+0000 7fd5afe00640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d07f0 0x7fd5b80b7560 secure :-1 s=READY pgs=1372 cs=0 l=1 rev1=1 crypto rx=0x7fd5a000d800 tx=0x7fd5a000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:11.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.300+0000 7fd5ae000640 1 -- 172.21.15.86:0/901401668 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5a0018030 con 0x7fd5b80d07f0 2024-03-06T07:34:11.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.300+0000 7fd5ae000640 1 -- 172.21.15.86:0/901401668 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd5a00096a0 con 0x7fd5b80d07f0 2024-03-06T07:34:11.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.300+0000 7fd5ae000640 1 -- 172.21.15.86:0/901401668 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5a0009870 con 0x7fd5b80d07f0 2024-03-06T07:34:11.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.300+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd5b80b8600 con 0x7fd5b80d07f0 2024-03-06T07:34:11.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.300+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd5b80b8bc0 con 0x7fd5b80d07f0 2024-03-06T07:34:11.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.301+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd5b804f330 con 0x7fd5b80d07f0 2024-03-06T07:34:11.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.302+0000 7fd5ae000640 1 -- 172.21.15.86:0/901401668 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd5a001b070 con 0x7fd5b80d07f0 2024-03-06T07:34:11.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.303+0000 7fd5ae000640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd59406b500 0x7fd59406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.303+0000 7fd5ae000640 1 -- 172.21.15.86:0/901401668 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fd5a008a470 con 0x7fd5b80d07f0 2024-03-06T07:34:11.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.303+0000 7fd5b5e00640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd59406b500 0x7fd59406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.304+0000 7fd5b5e00640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd59406b500 0x7fd59406d9c0 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8036000 tx=0x7fd5a803d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:11.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.308+0000 7fd5ae000640 1 -- 172.21.15.86:0/901401668 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd5a00592f0 con 0x7fd5b80d07f0 2024-03-06T07:34:11.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.543+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd5b815f320 con 0x7fd5b80d07f0 2024-03-06T07:34:11.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.544+0000 7fd5ae000640 1 -- 172.21.15.86:0/901401668 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7fd5a0058bf0 con 0x7fd5b80d07f0 2024-03-06T07:34:11.546 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:11.547 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:11.547 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd59406b500 msgr2=0x7fd59406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd59406b500 0x7fd59406d9c0 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8036000 tx=0x7fd5a803d040 comp rx=0 tx=0).stop 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d07f0 msgr2=0x7fd5b80b7560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d07f0 0x7fd5b80b7560 secure :-1 s=READY pgs=1372 cs=0 l=1 rev1=1 crypto rx=0x7fd5a000d800 tx=0x7fd5a000dcc0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 shutdown_connections 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd59406b500 0x7fd59406d9c0 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd5b80d24c0 0x7fd5b80b7aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd5b80d07f0 0x7fd5b80b7560 unknown :-1 s=CLOSED pgs=1372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 --2- 172.21.15.86:0/901401668 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd5b80b2de0 0x7fd5b8161260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 >> 172.21.15.86:0/901401668 conn(0x7fd5b80d3ee0 msgr2=0x7fd5b806ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:11.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 shutdown_connections 2024-03-06T07:34:11.551 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.547+0000 7fd5bd000640 1 -- 172.21.15.86:0/901401668 wait complete. 2024-03-06T07:34:11.570 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-06T07:34:11.816 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:34:11.817 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:34:11.817 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-06T07:34:11.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.902+0000 7f9f57c6cf40 1 Processor -- start 2024-03-06T07:34:11.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.902+0000 7f9f57c6cf40 1 -- start start 2024-03-06T07:34:11.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f57c6cf40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b533a70 0x55e33b5c4f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f57c6cf40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b5c8800 0x55e33b5c30b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f57c6cf40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8be0 0x55e33b5c35f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f57c6cf40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e33b494fd0 con 0x55e33b5c8be0 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f57c6cf40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e33b498f30 con 0x55e33b533a70 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f57c6cf40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55e33b5371e0 con 0x55e33b5c8800 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f55800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8be0 0x55e33b5c35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f55800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8be0 0x55e33b5c35f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51344/0 (socket says 172.21.15.86:51344) 2024-03-06T07:34:11.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.903+0000 7f9f55800640 1 -- 172.21.15.86:0/1043961840 learned_addr learned my addr 172.21.15.86:0/1043961840 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:11.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f55800640 1 -- 172.21.15.86:0/1043961840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b5c8800 msgr2=0x55e33b5c30b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f55800640 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b5c8800 0x55e33b5c30b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f54e00640 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b533a70 0x55e33b5c4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f55800640 1 -- 172.21.15.86:0/1043961840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b533a70 msgr2=0x55e33b5c4f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f55800640 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b533a70 0x55e33b5c4f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f55800640 1 -- 172.21.15.86:0/1043961840 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e33b534a60 con 0x55e33b5c8be0 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f55800640 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8be0 0x55e33b5c35f0 secure :-1 s=READY pgs=1375 cs=0 l=1 rev1=1 crypto rx=0x7f9f50006fc0 tx=0x7f9f50001ac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1bf7a6634fbb619a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f4f400640 1 -- 172.21.15.86:0/1043961840 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f50003650 con 0x55e33b5c8be0 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f4f400640 1 -- 172.21.15.86:0/1043961840 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f500037f0 con 0x55e33b5c8be0 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.904+0000 7f9f4f400640 1 -- 172.21.15.86:0/1043961840 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f50003bc0 con 0x55e33b5c8be0 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/1043961840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8be0 msgr2=0x55e33b5c35f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8be0 0x55e33b5c35f0 secure :-1 s=READY pgs=1375 cs=0 l=1 rev1=1 crypto rx=0x7f9f50006fc0 tx=0x7f9f50001ac0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/1043961840 shutdown_connections 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8be0 0x55e33b5c35f0 unknown :-1 s=CLOSED pgs=1375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b5c8800 0x55e33b5c30b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/1043961840 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b533a70 0x55e33b5c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/1043961840 >> 172.21.15.86:0/1043961840 conn(0x55e33b51bbf0 msgr2=0x55e33b4d8b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/1043961840 shutdown_connections 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.905+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/1043961840 wait complete. 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.906+0000 7f9f57c6cf40 1 Processor -- start 2024-03-06T07:34:11.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.906+0000 7f9f57c6cf40 1 -- start start 2024-03-06T07:34:11.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f57c6cf40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b533a70 0x55e33b5ac640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f57c6cf40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8800 0x55e33b5aedf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f57c6cf40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b5c8be0 0x55e33b5b1750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f57c6cf40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e33b5c5d30 con 0x55e33b5c8800 2024-03-06T07:34:11.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f57c6cf40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e33b534780 con 0x55e33b5c8be0 2024-03-06T07:34:11.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f57c6cf40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55e33b5c6010 con 0x55e33b533a70 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f4c400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8800 0x55e33b5aedf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f4c400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8800 0x55e33b5aedf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51358/0 (socket says 172.21.15.86:51358) 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f4c400640 1 -- 172.21.15.86:0/297297414 learned_addr learned my addr 172.21.15.86:0/297297414 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f54e00640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b533a70 0x55e33b5ac640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f4c400640 1 -- 172.21.15.86:0/297297414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b533a70 msgr2=0x55e33b5ac640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f4c400640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55e33b533a70 0x55e33b5ac640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f55800640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b5c8be0 0x55e33b5b1750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f4c400640 1 -- 172.21.15.86:0/297297414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b5c8be0 msgr2=0x55e33b5b1750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.907+0000 7f9f4c400640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55e33b5c8be0 0x55e33b5b1750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.908+0000 7f9f4c400640 1 -- 172.21.15.86:0/297297414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e33b5acff0 con 0x55e33b5c8800 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.908+0000 7f9f4c400640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8800 0x55e33b5aedf0 secure :-1 s=READY pgs=1376 cs=0 l=1 rev1=1 crypto rx=0x7f9f3c055040 tx=0x7f9f3c00a560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.908+0000 7f9f4e000640 1 -- 172.21.15.86:0/297297414 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f3c057040 con 0x55e33b5c8800 2024-03-06T07:34:11.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.908+0000 7f9f4e000640 1 -- 172.21.15.86:0/297297414 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f3c052ba0 con 0x55e33b5c8800 2024-03-06T07:34:11.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.908+0000 7f9f4e000640 1 -- 172.21.15.86:0/297297414 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f3c05d470 con 0x55e33b5c8800 2024-03-06T07:34:11.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.908+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x55e33b5ad2d0 con 0x55e33b5c8800 2024-03-06T07:34:11.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.908+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55e33b5ad890 con 0x55e33b5c8800 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.910+0000 7f9f4e000640 1 -- 172.21.15.86:0/297297414 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9f3c05d610 con 0x55e33b5c8800 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.911+0000 7f9f4e000640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f30069eb0 0x7f9f3006c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.911+0000 7f9f4e000640 1 -- 172.21.15.86:0/297297414 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f9f3c0cb2d0 con 0x55e33b5c8800 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.911+0000 7f9f54e00640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f30069eb0 0x7f9f3006c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.912+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x55e33b5af330 0x55e33b65eb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.912+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e116) v9 -- 0x55e33b65f050 con 0x55e33b5af330 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.912+0000 7f9f55800640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x55e33b5af330 0x55e33b65eb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.912+0000 7f9f54e00640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f30069eb0 0x7f9f3006c370 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f9f4400d2c0 tx=0x7f9f44002400 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.912+0000 7f9f55800640 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x55e33b5af330 0x55e33b65eb10 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:11.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f55800640 1 -- 172.21.15.86:0/297297414 <== osd.0 v2:172.21.15.86:6800/1601033517 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (93) out=93b] v0'0 uv11 ondisk = 0) v8 ==== 164+0+93 (crc 0 0 0) 0x7f9f5000f030 con 0x55e33b5af330 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x55e33b5af330 msgr2=0x55e33b65eb10 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x55e33b5af330 0x55e33b65eb10 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f30069eb0 msgr2=0x7f9f3006c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9f30069eb0 0x7f9f3006c370 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f9f4400d2c0 tx=0x7f9f44002400 comp rx=0 tx=0).stop 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8800 msgr2=0x55e33b5aedf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 --2- 172.21.15.86:0/297297414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55e33b5c8800 0x55e33b5aedf0 secure :-1 s=READY pgs=1376 cs=0 l=1 rev1=1 crypto rx=0x7f9f3c055040 tx=0x7f9f3c00a560 comp rx=0 tx=0).stop 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f55800640 1 -- 172.21.15.86:0/297297414 reap_dead start 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 shutdown_connections 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 >> 172.21.15.86:0/297297414 conn(0x55e33b51bbf0 msgr2=0x55e33b531c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 shutdown_connections 2024-03-06T07:34:11.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:11.913+0000 7f9f57c6cf40 1 -- 172.21.15.86:0/297297414 wait complete. 2024-03-06T07:34:11.920 INFO:teuthology.orchestra.run.smithi086.stdout:W�file_ddir_2. 2024-03-06T07:34:11.921 DEBUG:teuthology.orchestra.run.smithi086:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-06T07:34:12.009 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:34:12.009 INFO:teuthology.orchestra.run.smithi086.stdout: "ino": 1099511627777, 2024-03-06T07:34:12.009 INFO:teuthology.orchestra.run.smithi086.stdout: "ancestors": [ 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 1099511628278, 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "file_d", 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 27 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "dirino": 1, 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "dname": "dir_2", 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "version": 46 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "pool": 17, 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: "old_pools": [] 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:34:12.010 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:12.015 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:12.015 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f dir_2/file_d) 2024-03-06T07:34:12.065 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:12.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.266+0000 7f1722200640 1 Processor -- start 2024-03-06T07:34:12.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.266+0000 7f1722200640 1 -- start start 2024-03-06T07:34:12.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.267+0000 7f1722200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f171c15f9d0 0x7f171c15fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:12.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.267+0000 7f1722200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c070b10 0x7f171c070f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:12.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.267+0000 7f1722200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f171c06cef0 con 0x7f171c144c00 2024-03-06T07:34:12.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.267+0000 7f1722200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f171c06d1d0 con 0x7f171c15f9d0 2024-03-06T07:34:12.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.267+0000 7f1722200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f171c06d4b0 con 0x7f171c070b10 2024-03-06T07:34:12.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.268+0000 7f171be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c070b10 0x7f171c070f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:12.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.268+0000 7f171be00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c070b10 0x7f171c070f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:43878/0 (socket says 172.21.15.86:43878) 2024-03-06T07:34:12.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.268+0000 7f171be00640 1 -- 172.21.15.86:0/3039927840 learned_addr learned my addr 172.21.15.86:0/3039927840 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:12.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.268+0000 7f171aa00640 1 --2- 172.21.15.86:0/3039927840 >> v2:172.21.15.194:3300/0 conn(0x7f171c15f9d0 0x7f171c15fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:12.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.269+0000 7f171a000640 1 -- 172.21.15.86:0/3039927840 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1526790150 0 0) 0x7f171c06cef0 con 0x7f171c144c00 2024-03-06T07:34:12.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.269+0000 7f171a000640 1 -- 172.21.15.86:0/3039927840 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1700003520 con 0x7f171c144c00 2024-03-06T07:34:12.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.269+0000 7f171be00640 1 -- 172.21.15.86:0/3039927840 >> v2:172.21.15.194:3300/0 conn(0x7f171c15f9d0 msgr2=0x7f171c15fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:12.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.269+0000 7f171be00640 1 --2- 172.21.15.86:0/3039927840 >> v2:172.21.15.194:3300/0 conn(0x7f171c15f9d0 0x7f171c15fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.269+0000 7f171be00640 1 -- 172.21.15.86:0/3039927840 >> v1:172.21.15.194:6789/0 conn(0x7f171c144c00 legacy=0x7f171c06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:12.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.269+0000 7f171be00640 1 -- 172.21.15.86:0/3039927840 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f171c06d790 con 0x7f171c070b10 2024-03-06T07:34:12.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.270+0000 7f171be00640 1 --2- 172.21.15.86:0/3039927840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c070b10 0x7f171c070f40 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f1714009870 tx=0x7f171402b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1eba36312b8f5ffb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:12.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.271+0000 7f171a000640 1 -- 172.21.15.86:0/3039927840 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1714034070 con 0x7f171c070b10 2024-03-06T07:34:12.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.271+0000 7f171a000640 1 -- 172.21.15.86:0/3039927840 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1714030840 con 0x7f171c070b10 2024-03-06T07:34:12.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.271+0000 7f171a000640 1 -- 172.21.15.86:0/3039927840 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1714030c10 con 0x7f171c070b10 2024-03-06T07:34:12.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.271+0000 7f1722200640 1 -- 172.21.15.86:0/3039927840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c070b10 msgr2=0x7f171c070f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:12.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.271+0000 7f1722200640 1 --2- 172.21.15.86:0/3039927840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c070b10 0x7f171c070f40 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f1714009870 tx=0x7f171402b980 comp rx=0 tx=0).stop 2024-03-06T07:34:12.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.272+0000 7f1722200640 1 -- 172.21.15.86:0/3039927840 shutdown_connections 2024-03-06T07:34:12.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.272+0000 7f1722200640 1 --2- 172.21.15.86:0/3039927840 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c070b10 0x7f171c070f40 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.272+0000 7f1722200640 1 --2- 172.21.15.86:0/3039927840 >> v2:172.21.15.194:3300/0 conn(0x7f171c15f9d0 0x7f171c15fe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.272+0000 7f1722200640 1 -- 172.21.15.86:0/3039927840 >> 172.21.15.86:0/3039927840 conn(0x7f171c0b1730 msgr2=0x7f171c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:12.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.272+0000 7f1722200640 1 -- 172.21.15.86:0/3039927840 shutdown_connections 2024-03-06T07:34:12.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.272+0000 7f1722200640 1 -- 172.21.15.86:0/3039927840 wait complete. 2024-03-06T07:34:12.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.273+0000 7f1722200640 1 Processor -- start 2024-03-06T07:34:12.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.274+0000 7f1722200640 1 -- start start 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.274+0000 7f1722200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f171c070b10 0x7f171c14da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.274+0000 7f1722200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f171c144c00 0x7f171c13c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f1722200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c15f9d0 0x7f171c0af5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f1722200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f171c06ccd0 con 0x7f171c070b10 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f1722200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f171c06cb50 con 0x7f171c144c00 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f1722200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f171c06df90 con 0x7f171c15f9d0 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f171c070b10 0x7f171c14da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f171c070b10 0x7f171c14da90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:51362/0 (socket says 172.21.15.86:51362) 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 -- 172.21.15.86:0/394295821 learned_addr learned my addr 172.21.15.86:0/394295821 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171be00640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c15f9d0 0x7f171c0af5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:12.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171aa00640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f171c144c00 0x7f171c13c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:12.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 -- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c15f9d0 msgr2=0x7f171c0af5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:12.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c15f9d0 0x7f171c0af5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 -- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f171c144c00 msgr2=0x7f171c13c380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:12.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f171c144c00 0x7f171c13c380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171b400640 1 -- 172.21.15.86:0/394295821 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f171c14dfd0 con 0x7f171c070b10 2024-03-06T07:34:12.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171aa00640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f171c144c00 0x7f171c13c380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:12.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.275+0000 7f171be00640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c15f9d0 0x7f171c0af5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:12.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.276+0000 7f171b400640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f171c070b10 0x7f171c14da90 secure :-1 s=READY pgs=1377 cs=0 l=1 rev1=1 crypto rx=0x7f170c00c8a0 tx=0x7f170c00cd60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:12.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.276+0000 7f1713400640 1 -- 172.21.15.86:0/394295821 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f170c003b70 con 0x7f171c070b10 2024-03-06T07:34:12.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.276+0000 7f1713400640 1 -- 172.21.15.86:0/394295821 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f170c003d30 con 0x7f171c070b10 2024-03-06T07:34:12.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.276+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f171c1f1be0 con 0x7f171c070b10 2024-03-06T07:34:12.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.276+0000 7f1713400640 1 -- 172.21.15.86:0/394295821 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f170c00aa90 con 0x7f171c070b10 2024-03-06T07:34:12.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.276+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f171c1f21a0 con 0x7f171c070b10 2024-03-06T07:34:12.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.277+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f171c04f330 con 0x7f171c070b10 2024-03-06T07:34:12.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.278+0000 7f1713400640 1 -- 172.21.15.86:0/394295821 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f170c00ac30 con 0x7f171c070b10 2024-03-06T07:34:12.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.280+0000 7f1713400640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16f806b4d0 0x7f16f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:12.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.280+0000 7f1713400640 1 -- 172.21.15.86:0/394295821 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f170c08a210 con 0x7f171c070b10 2024-03-06T07:34:12.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.280+0000 7f171aa00640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16f806b4d0 0x7f16f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:12.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.281+0000 7f171aa00640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16f806b4d0 0x7f16f806d990 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f171c0b0190 tx=0x7f1704008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:12.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.284+0000 7f1713400640 1 -- 172.21.15.86:0/394295821 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f170c059090 con 0x7f171c070b10 2024-03-06T07:34:12.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.520+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f171c14b040 con 0x7f171c070b10 2024-03-06T07:34:12.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.521+0000 7f1713400640 1 -- 172.21.15.86:0/394295821 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f171c14b040 con 0x7f171c070b10 2024-03-06T07:34:12.524 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:12.524 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:12.524 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16f806b4d0 msgr2=0x7f16f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16f806b4d0 0x7f16f806d990 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f171c0b0190 tx=0x7f1704008040 comp rx=0 tx=0).stop 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f171c070b10 msgr2=0x7f171c14da90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f171c070b10 0x7f171c14da90 secure :-1 s=READY pgs=1377 cs=0 l=1 rev1=1 crypto rx=0x7f170c00c8a0 tx=0x7f170c00cd60 comp rx=0 tx=0).stop 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 shutdown_connections 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16f806b4d0 0x7f16f806d990 unknown :-1 s=CLOSED pgs=591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f171c15f9d0 0x7f171c0af5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f171c144c00 0x7f171c13c380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 --2- 172.21.15.86:0/394295821 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f171c070b10 0x7f171c14da90 unknown :-1 s=CLOSED pgs=1377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:12.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 >> 172.21.15.86:0/394295821 conn(0x7f171c0b1730 msgr2=0x7f171c06e810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:12.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 shutdown_connections 2024-03-06T07:34:12.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:12.523+0000 7f1722200640 1 -- 172.21.15.86:0/394295821 wait complete. 2024-03-06T07:34:12.544 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:34:12.759 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-06T07:34:12.759 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-06T07:34:12.759 DEBUG:tasks.ceph_test_case:wait_until_equal: 0 != 1, waiting (timeout=60)... 2024-03-06T07:34:17.760 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:17.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.969+0000 7f3147a00640 1 Processor -- start 2024-03-06T07:34:17.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.969+0000 7f3147a00640 1 -- start start 2024-03-06T07:34:17.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.970+0000 7f3147a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3140144c00 0x7f314006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:17.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.970+0000 7f3147a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f314006ee70 con 0x7f314015f9d0 2024-03-06T07:34:17.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.970+0000 7f3147a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f314006cef0 con 0x7f314015fe60 2024-03-06T07:34:17.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.970+0000 7f3147a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f314006d1d0 con 0x7f3140144c00 2024-03-06T07:34:17.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.971+0000 7f3145000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3140144c00 0x7f314006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:17.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.971+0000 7f3145000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3140144c00 0x7f314006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48176/0 (socket says 172.21.15.86:48176) 2024-03-06T07:34:17.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.971+0000 7f3145000640 1 -- 172.21.15.86:0/438280038 learned_addr learned my addr 172.21.15.86:0/438280038 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:17.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.971+0000 7f3145000640 1 -- 172.21.15.86:0/438280038 >> v1:172.21.15.194:6789/0 conn(0x7f314015fe60 legacy=0x7f31400ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:17.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.971+0000 7f3145000640 1 -- 172.21.15.86:0/438280038 >> v1:172.21.15.86:6789/0 conn(0x7f314015f9d0 legacy=0x7f31400ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:17.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.971+0000 7f3145000640 1 -- 172.21.15.86:0/438280038 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f314006d4b0 con 0x7f3140144c00 2024-03-06T07:34:17.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.972+0000 7f3145000640 1 --2- 172.21.15.86:0/438280038 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3140144c00 0x7f314006e930 secure :-1 s=READY pgs=1379 cs=0 l=1 rev1=1 crypto rx=0x7f3134009870 tx=0x7f313402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c366fdeeafb9cb5d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:17.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.972+0000 7f313f400640 1 -- 172.21.15.86:0/438280038 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3134034070 con 0x7f3140144c00 2024-03-06T07:34:17.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f313f400640 1 -- 172.21.15.86:0/438280038 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3134030860 con 0x7f3140144c00 2024-03-06T07:34:17.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f313f400640 1 -- 172.21.15.86:0/438280038 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3134030c30 con 0x7f3140144c00 2024-03-06T07:34:17.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f3147a00640 1 -- 172.21.15.86:0/438280038 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3140144c00 msgr2=0x7f314006e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:17.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f3147a00640 1 --2- 172.21.15.86:0/438280038 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3140144c00 0x7f314006e930 secure :-1 s=READY pgs=1379 cs=0 l=1 rev1=1 crypto rx=0x7f3134009870 tx=0x7f313402b9a0 comp rx=0 tx=0).stop 2024-03-06T07:34:17.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f3147a00640 1 -- 172.21.15.86:0/438280038 shutdown_connections 2024-03-06T07:34:17.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f3147a00640 1 --2- 172.21.15.86:0/438280038 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3140144c00 0x7f314006e930 unknown :-1 s=CLOSED pgs=1379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:17.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f3147a00640 1 -- 172.21.15.86:0/438280038 >> 172.21.15.86:0/438280038 conn(0x7f31400b1730 msgr2=0x7f31400b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:17.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f3147a00640 1 -- 172.21.15.86:0/438280038 shutdown_connections 2024-03-06T07:34:17.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.973+0000 7f3147a00640 1 -- 172.21.15.86:0/438280038 wait complete. 2024-03-06T07:34:17.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.975+0000 7f3147a00640 1 Processor -- start 2024-03-06T07:34:17.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.975+0000 7f3147a00640 1 -- start start 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3147a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3140144c00 0x7f31401e61d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3147a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314015f9d0 0x7f31401f4090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3147a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f314015fe60 0x7f31401e7230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3147a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3140160730 con 0x7f314015fe60 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3147a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3140160270 con 0x7f314015f9d0 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3147a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f31400ceec0 con 0x7f3140144c00 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3145a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f314015fe60 0x7f31401e7230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3145a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f314015fe60 0x7f31401e7230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48180/0 (socket says 172.21.15.86:48180) 2024-03-06T07:34:17.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.976+0000 7f3145a00640 1 -- 172.21.15.86:0/3434726642 learned_addr learned my addr 172.21.15.86:0/3434726642 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:17.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145000640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3140144c00 0x7f31401e61d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:17.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145a00640 1 -- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3140144c00 msgr2=0x7f31401e61d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:17.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3140144c00 0x7f31401e61d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:17.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f313fe00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314015f9d0 0x7f31401f4090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:17.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145a00640 1 -- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314015f9d0 msgr2=0x7f31401f4090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:17.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314015f9d0 0x7f31401f4090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:17.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145a00640 1 -- 172.21.15.86:0/3434726642 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f31401e79a0 con 0x7f314015fe60 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145000640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3140144c00 0x7f31401e61d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f3145a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f314015fe60 0x7f31401e7230 secure :-1 s=READY pgs=1380 cs=0 l=1 rev1=1 crypto rx=0x7f313800c7b0 tx=0x7f313800cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.977+0000 7f313cc00640 1 -- 172.21.15.86:0/3434726642 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3138004b60 con 0x7f314015fe60 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.978+0000 7f313cc00640 1 -- 172.21.15.86:0/3434726642 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3138004d00 con 0x7f314015fe60 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.978+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f31401e7c80 con 0x7f314015fe60 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.978+0000 7f313cc00640 1 -- 172.21.15.86:0/3434726642 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f313800aa90 con 0x7f314015fe60 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.978+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31401e8240 con 0x7f314015fe60 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.979+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3140067830 con 0x7f314015fe60 2024-03-06T07:34:17.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.980+0000 7f313cc00640 1 -- 172.21.15.86:0/3434726642 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f313800ac30 con 0x7f314015fe60 2024-03-06T07:34:17.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.981+0000 7f313cc00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312006b4d0 0x7f312006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:17.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.981+0000 7f313cc00640 1 -- 172.21.15.86:0/3434726642 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f313808a670 con 0x7f314015fe60 2024-03-06T07:34:17.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.981+0000 7f3145000640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312006b4d0 0x7f312006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:17.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.982+0000 7f3145000640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312006b4d0 0x7f312006d990 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f3134036000 tx=0x7f313403b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:17.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:17.986+0000 7f313cc00640 1 -- 172.21.15.86:0/3434726642 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3140067830 con 0x7f314015fe60 2024-03-06T07:34:18.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.219+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f31400b4b60 con 0x7f314015fe60 2024-03-06T07:34:18.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.220+0000 7f313cc00640 1 -- 172.21.15.86:0/3434726642 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f31400b4b60 con 0x7f314015fe60 2024-03-06T07:34:18.223 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:18.223 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:18.224 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312006b4d0 msgr2=0x7f312006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312006b4d0 0x7f312006d990 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f3134036000 tx=0x7f313403b040 comp rx=0 tx=0).stop 2024-03-06T07:34:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f314015fe60 msgr2=0x7f31401e7230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f314015fe60 0x7f31401e7230 secure :-1 s=READY pgs=1380 cs=0 l=1 rev1=1 crypto rx=0x7f313800c7b0 tx=0x7f313800cc70 comp rx=0 tx=0).stop 2024-03-06T07:34:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 shutdown_connections 2024-03-06T07:34:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f312006b4d0 0x7f312006d990 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f314015fe60 0x7f31401e7230 unknown :-1 s=CLOSED pgs=1380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f314015f9d0 0x7f31401f4090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 --2- 172.21.15.86:0/3434726642 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3140144c00 0x7f31401e61d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 >> 172.21.15.86:0/3434726642 conn(0x7f31400b1730 msgr2=0x7f31400aff70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 shutdown_connections 2024-03-06T07:34:18.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.223+0000 7f3147a00640 1 -- 172.21.15.86:0/3434726642 wait complete. 2024-03-06T07:34:18.244 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:34:18.465 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":1}} 2024-03-06T07:34:18.659 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 1 } } 2024-03-06T07:34:18.659 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:18.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.862+0000 7ff5dc200640 1 Processor -- start 2024-03-06T07:34:18.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.862+0000 7ff5dc200640 1 -- start start 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5dc200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5d415f9d0 0x7ff5d415fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5dc200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d4070b10 0x7ff5d4070f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5dc200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff5d406cef0 con 0x7ff5d4144c00 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5dc200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff5d406d1d0 con 0x7ff5d415f9d0 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5dc200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5d406d4b0 con 0x7ff5d4070b10 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5d8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5d415f9d0 0x7ff5d415fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5d9800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff5d4144c00 0x7ff5d406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34106/0 (socket says 172.21.15.86:34106) 2024-03-06T07:34:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.864+0000 7ff5d9800640 1 -- 172.21.15.86:0/324176077 learned_addr learned my addr 172.21.15.86:0/324176077 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:18.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.865+0000 7ff5da200640 1 --2- 172.21.15.86:0/324176077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d4070b10 0x7ff5d4070f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:18.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.865+0000 7ff5d3e00640 1 -- 172.21.15.86:0/324176077 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1871563555 0 0) 0x7ff5d406cef0 con 0x7ff5d4144c00 2024-03-06T07:34:18.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.865+0000 7ff5d8e00640 1 -- 172.21.15.86:0/324176077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d4070b10 msgr2=0x7ff5d4070f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:18.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.865+0000 7ff5d8e00640 1 --2- 172.21.15.86:0/324176077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d4070b10 0x7ff5d4070f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.865+0000 7ff5d8e00640 1 -- 172.21.15.86:0/324176077 >> v1:172.21.15.86:6789/0 conn(0x7ff5d4144c00 legacy=0x7ff5d406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:18.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.865+0000 7ff5d8e00640 1 -- 172.21.15.86:0/324176077 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5d406d790 con 0x7ff5d415f9d0 2024-03-06T07:34:18.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.866+0000 7ff5d8e00640 1 --2- 172.21.15.86:0/324176077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d415f9d0 0x7ff5d415fe60 secure :-1 s=READY pgs=1382 cs=0 l=1 rev1=1 crypto rx=0x7ff5c00097a0 tx=0x7ff5c002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1401685480cbbc19 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:18.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5d3e00640 1 -- 172.21.15.86:0/324176077 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5c0035020 con 0x7ff5d415f9d0 2024-03-06T07:34:18.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5d3e00640 1 -- 172.21.15.86:0/324176077 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5c0031830 con 0x7ff5d415f9d0 2024-03-06T07:34:18.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5d3e00640 1 -- 172.21.15.86:0/324176077 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5c0031c00 con 0x7ff5d415f9d0 2024-03-06T07:34:18.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5dc200640 1 -- 172.21.15.86:0/324176077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d415f9d0 msgr2=0x7ff5d415fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:18.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5dc200640 1 --2- 172.21.15.86:0/324176077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d415f9d0 0x7ff5d415fe60 secure :-1 s=READY pgs=1382 cs=0 l=1 rev1=1 crypto rx=0x7ff5c00097a0 tx=0x7ff5c002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5dc200640 1 -- 172.21.15.86:0/324176077 shutdown_connections 2024-03-06T07:34:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5dc200640 1 --2- 172.21.15.86:0/324176077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d4070b10 0x7ff5d4070f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.867+0000 7ff5dc200640 1 --2- 172.21.15.86:0/324176077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d415f9d0 0x7ff5d415fe60 unknown :-1 s=CLOSED pgs=1382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.868+0000 7ff5dc200640 1 -- 172.21.15.86:0/324176077 >> 172.21.15.86:0/324176077 conn(0x7ff5d40b1730 msgr2=0x7ff5d40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.868+0000 7ff5dc200640 1 -- 172.21.15.86:0/324176077 shutdown_connections 2024-03-06T07:34:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.868+0000 7ff5dc200640 1 -- 172.21.15.86:0/324176077 wait complete. 2024-03-06T07:34:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.869+0000 7ff5dc200640 1 Processor -- start 2024-03-06T07:34:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.869+0000 7ff5dc200640 1 -- start start 2024-03-06T07:34:18.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.870+0000 7ff5dc200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5d4070b10 0x7ff5d4071680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.870+0000 7ff5dc200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d4144c00 0x7ff5d415dae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.870+0000 7ff5dc200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d415f9d0 0x7ff5d4139d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.870+0000 7ff5dc200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5d406ccd0 con 0x7ff5d4144c00 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.870+0000 7ff5dc200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5d406cb50 con 0x7ff5d4070b10 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.870+0000 7ff5dc200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5d406df90 con 0x7ff5d415f9d0 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.870+0000 7ff5d8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d4144c00 0x7ff5d415dae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d4144c00 0x7ff5d415dae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48194/0 (socket says 172.21.15.86:48194) 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 -- 172.21.15.86:0/615549801 learned_addr learned my addr 172.21.15.86:0/615549801 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5da200640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d415f9d0 0x7ff5d4139d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d9800640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5d4070b10 0x7ff5d4071680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 -- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d415f9d0 msgr2=0x7ff5d4139d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d415f9d0 0x7ff5d4139d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 -- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5d4070b10 msgr2=0x7ff5d4071680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5d4070b10 0x7ff5d4071680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 -- 172.21.15.86:0/615549801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5d40b2b30 con 0x7ff5d4144c00 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d9800640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5d4070b10 0x7ff5d4071680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.871+0000 7ff5d8e00640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d4144c00 0x7ff5d415dae0 secure :-1 s=READY pgs=1383 cs=0 l=1 rev1=1 crypto rx=0x7ff5c00097a0 tx=0x7ff5c00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.872+0000 7ff5d1600640 1 -- 172.21.15.86:0/615549801 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5c0035070 con 0x7ff5d4144c00 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.872+0000 7ff5d1600640 1 -- 172.21.15.86:0/615549801 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5c0004230 con 0x7ff5d4144c00 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.872+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff5d415d350 con 0x7ff5d4144c00 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.872+0000 7ff5d1600640 1 -- 172.21.15.86:0/615549801 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5c003a6f0 con 0x7ff5d4144c00 2024-03-06T07:34:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.872+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff5d415cab0 con 0x7ff5d4144c00 2024-03-06T07:34:18.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.873+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff5d41537a0 con 0x7ff5d4144c00 2024-03-06T07:34:18.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.874+0000 7ff5d1600640 1 -- 172.21.15.86:0/615549801 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff5c0038020 con 0x7ff5d4144c00 2024-03-06T07:34:18.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.875+0000 7ff5d1600640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5b406b4d0 0x7ff5b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:18.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.875+0000 7ff5d1600640 1 -- 172.21.15.86:0/615549801 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7ff5c00ac090 con 0x7ff5d4144c00 2024-03-06T07:34:18.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.876+0000 7ff5d9800640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5b406b4d0 0x7ff5b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:18.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.876+0000 7ff5d9800640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5b406b4d0 0x7ff5b406d990 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7ff5c8000f60 tx=0x7ff5c8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:18.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:18.879+0000 7ff5d1600640 1 -- 172.21.15.86:0/615549801 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff5c0040070 con 0x7ff5d4144c00 2024-03-06T07:34:19.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.166+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff5d404f330 con 0x7ff5d4144c00 2024-03-06T07:34:19.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.167+0000 7ff5d1600640 1 -- 172.21.15.86:0/615549801 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7ff5c007ade0 con 0x7ff5d4144c00 2024-03-06T07:34:19.170 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:19.170 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:19.170 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:19.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5b406b4d0 msgr2=0x7ff5b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5b406b4d0 0x7ff5b406d990 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7ff5c8000f60 tx=0x7ff5c8009040 comp rx=0 tx=0).stop 2024-03-06T07:34:19.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d4144c00 msgr2=0x7ff5d415dae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d4144c00 0x7ff5d415dae0 secure :-1 s=READY pgs=1383 cs=0 l=1 rev1=1 crypto rx=0x7ff5c00097a0 tx=0x7ff5c00046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 shutdown_connections 2024-03-06T07:34:19.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5b406b4d0 0x7ff5b406d990 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5d415f9d0 0x7ff5d4139d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5d4144c00 0x7ff5d415dae0 unknown :-1 s=CLOSED pgs=1383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 --2- 172.21.15.86:0/615549801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5d4070b10 0x7ff5d4071680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 >> 172.21.15.86:0/615549801 conn(0x7ff5d40b1730 msgr2=0x7ff5d40afe90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:19.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.170+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 shutdown_connections 2024-03-06T07:34:19.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.171+0000 7ff5dc200640 1 -- 172.21.15.86:0/615549801 wait complete. 2024-03-06T07:34:19.191 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump purge_queue pq_executed 2024-03-06T07:34:19.401 INFO:teuthology.orchestra.run.smithi086.stdout:{"purge_queue":{"pq_executed":1}} 2024-03-06T07:34:19.402 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 1 } } 2024-03-06T07:34:19.402 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:19.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.610+0000 7efef6a00640 1 Processor -- start 2024-03-06T07:34:19.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.610+0000 7efef6a00640 1 -- start start 2024-03-06T07:34:19.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.611+0000 7efef6a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef006b6d0 0x7efef0162700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:19.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.611+0000 7efef6a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7efef00affe0 0x7efef00d06f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:19.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.611+0000 7efef6a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efef00cef70 con 0x7efef015dfd0 2024-03-06T07:34:19.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.611+0000 7efef6a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efef0162270 con 0x7efef00affe0 2024-03-06T07:34:19.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.611+0000 7efef6a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efef015e3b0 con 0x7efef006b6d0 2024-03-06T07:34:19.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.612+0000 7efef6000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7efef00affe0 0x7efef00d06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:19.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.612+0000 7efef4c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef006b6d0 0x7efef0162700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:19.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.612+0000 7efef5600640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7efef015dfd0 0x7efef006afc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:49556/0 (socket says 172.21.15.86:49556) 2024-03-06T07:34:19.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.612+0000 7efef5600640 1 -- 172.21.15.86:0/3803343734 learned_addr learned my addr 172.21.15.86:0/3803343734 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:19.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.613+0000 7efef6000640 1 -- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef006b6d0 msgr2=0x7efef0162700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:19.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.613+0000 7efef6000640 1 --2- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef006b6d0 0x7efef0162700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.613+0000 7efef6000640 1 -- 172.21.15.86:0/3803343734 >> v1:172.21.15.194:6789/0 conn(0x7efef015dfd0 legacy=0x7efef006afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.613+0000 7efef6000640 1 -- 172.21.15.86:0/3803343734 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efef0161c60 con 0x7efef00affe0 2024-03-06T07:34:19.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.613+0000 7efef4c00640 1 --2- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef006b6d0 0x7efef0162700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:34:19.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.614+0000 7efef6000640 1 --2- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef00affe0 0x7efef00d06f0 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7efee8009d90 tx=0x7efee802bd90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6569f61e09942ef1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:19.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.615+0000 7efeefe00640 1 -- 172.21.15.86:0/3803343734 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efee8039070 con 0x7efef00affe0 2024-03-06T07:34:19.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.615+0000 7efeefe00640 1 -- 172.21.15.86:0/3803343734 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efee802cb40 con 0x7efef00affe0 2024-03-06T07:34:19.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.615+0000 7efeefe00640 1 -- 172.21.15.86:0/3803343734 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efee8034440 con 0x7efef00affe0 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.615+0000 7efef6a00640 1 -- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef00affe0 msgr2=0x7efef00d06f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.616+0000 7efef6a00640 1 --2- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef00affe0 0x7efef00d06f0 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7efee8009d90 tx=0x7efee802bd90 comp rx=0 tx=0).stop 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.616+0000 7efef6a00640 1 -- 172.21.15.86:0/3803343734 shutdown_connections 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.616+0000 7efef6a00640 1 --2- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef00affe0 0x7efef00d06f0 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.616+0000 7efef6a00640 1 --2- 172.21.15.86:0/3803343734 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef006b6d0 0x7efef0162700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.616+0000 7efef6a00640 1 -- 172.21.15.86:0/3803343734 >> 172.21.15.86:0/3803343734 conn(0x7efef006c0d0 msgr2=0x7efef006c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.616+0000 7efef6a00640 1 -- 172.21.15.86:0/3803343734 shutdown_connections 2024-03-06T07:34:19.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.616+0000 7efef6a00640 1 -- 172.21.15.86:0/3803343734 wait complete. 2024-03-06T07:34:19.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.617+0000 7efef6a00640 1 Processor -- start 2024-03-06T07:34:19.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.618+0000 7efef6a00640 1 -- start start 2024-03-06T07:34:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.618+0000 7efef6a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef006b6d0 0x7efef015bb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.618+0000 7efef6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efef00affe0 0x7efef015d290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef6a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef015dfd0 0x7efef015a060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:19.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef6a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efef00ce8d0 con 0x7efef00affe0 2024-03-06T07:34:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef6a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efef0161f00 con 0x7efef006b6d0 2024-03-06T07:34:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef6a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efef00cea50 con 0x7efef015dfd0 2024-03-06T07:34:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efef00affe0 0x7efef015d290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efef00affe0 0x7efef015d290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48198/0 (socket says 172.21.15.86:48198) 2024-03-06T07:34:19.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 -- 172.21.15.86:0/458382369 learned_addr learned my addr 172.21.15.86:0/458382369 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:19.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef6000640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef015dfd0 0x7efef015a060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:19.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef5600640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef006b6d0 0x7efef015bb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:19.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 -- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef015dfd0 msgr2=0x7efef015a060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef015dfd0 0x7efef015a060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 -- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef006b6d0 msgr2=0x7efef015bb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef006b6d0 0x7efef015bb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef4c00640 1 -- 172.21.15.86:0/458382369 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efef015d7d0 con 0x7efef00affe0 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.619+0000 7efef5600640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef006b6d0 0x7efef015bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.620+0000 7efef4c00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efef00affe0 0x7efef015d290 secure :-1 s=READY pgs=1384 cs=0 l=1 rev1=1 crypto rx=0x7efedc00ec10 tx=0x7efedc00c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.620+0000 7efeed600640 1 -- 172.21.15.86:0/458382369 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efedc017070 con 0x7efef00affe0 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.620+0000 7efeed600640 1 -- 172.21.15.86:0/458382369 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efedc00cc30 con 0x7efef00affe0 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.620+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efef015c4d0 con 0x7efef00affe0 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.620+0000 7efeed600640 1 -- 172.21.15.86:0/458382369 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efedc013470 con 0x7efef00affe0 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.620+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efef015ca90 con 0x7efef00affe0 2024-03-06T07:34:19.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.622+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efef004f330 con 0x7efef00affe0 2024-03-06T07:34:19.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.622+0000 7efeed600640 1 -- 172.21.15.86:0/458382369 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efedc01a070 con 0x7efef00affe0 2024-03-06T07:34:19.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.623+0000 7efeed600640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efed006b4d0 0x7efed006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:19.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.623+0000 7efeed600640 1 -- 172.21.15.86:0/458382369 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7efedc08a2c0 con 0x7efef00affe0 2024-03-06T07:34:19.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.623+0000 7efef5600640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efed006b4d0 0x7efed006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:19.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.624+0000 7efef5600640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efed006b4d0 0x7efed006d990 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7efee4000f60 tx=0x7efee4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:19.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.627+0000 7efeed600640 1 -- 172.21.15.86:0/458382369 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efedc059010 con 0x7efef00affe0 2024-03-06T07:34:19.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.971+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7efef00cdea0 con 0x7efef00affe0 2024-03-06T07:34:19.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.972+0000 7efeed600640 1 -- 172.21.15.86:0/458382369 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7efedc058910 con 0x7efef00affe0 2024-03-06T07:34:19.975 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:19.975 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:19.975 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:19.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.974+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efed006b4d0 msgr2=0x7efed006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.974+0000 7efef6a00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efed006b4d0 0x7efed006d990 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7efee4000f60 tx=0x7efee4009040 comp rx=0 tx=0).stop 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.974+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efef00affe0 msgr2=0x7efef015d290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.974+0000 7efef6a00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efef00affe0 0x7efef015d290 secure :-1 s=READY pgs=1384 cs=0 l=1 rev1=1 crypto rx=0x7efedc00ec10 tx=0x7efedc00c520 comp rx=0 tx=0).stop 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 shutdown_connections 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7efed006b4d0 0x7efed006d990 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7efef015dfd0 0x7efef015a060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7efef00affe0 0x7efef015d290 unknown :-1 s=CLOSED pgs=1384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 --2- 172.21.15.86:0/458382369 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7efef006b6d0 0x7efef015bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 >> 172.21.15.86:0/458382369 conn(0x7efef006c0d0 msgr2=0x7efef00b0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 shutdown_connections 2024-03-06T07:34:19.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:19.975+0000 7efef6a00640 1 -- 172.21.15.86:0/458382369 wait complete. 2024-03-06T07:34:19.997 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache 2024-03-06T07:34:20.212 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":0,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":3,"strays_enqueued":1,"strays_reintegrated":2,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0}} 2024-03-06T07:34:20.212 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 0, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3, "strays_enqueued": 1, "strays_migrated": 0, "strays_reintegrated": 2 } } 2024-03-06T07:34:20.212 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:20.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.422+0000 7f6b92800640 1 Processor -- start 2024-03-06T07:34:20.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.422+0000 7f6b92800640 1 -- start start 2024-03-06T07:34:20.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b92800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6b8c0af1c0 0x7f6b8c15eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:20.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b92800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b8c070d20 0x7f6b8c06cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:20.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b92800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b8c071d30 con 0x7f6b8c06d1a0 2024-03-06T07:34:20.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b92800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b8c0711d0 con 0x7f6b8c070d20 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b92800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b8c071490 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b8aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b8c070d20 0x7f6b8c06cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b8aa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b8c070d20 0x7f6b8c06cc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48202/0 (socket says 172.21.15.86:48202) 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b8aa00640 1 -- 172.21.15.86:0/1534532839 learned_addr learned my addr 172.21.15.86:0/1534532839 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.424+0000 7f6b8be00640 1 --1- 172.21.15.86:0/1534532839 >> v1:172.21.15.86:6789/0 conn(0x7f6b8c06d1a0 0x7f6b8c06e4d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34118/0 (socket says 172.21.15.86:34118) 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.425+0000 7f6b8b400640 1 --2- 172.21.15.86:0/1534532839 >> v2:172.21.15.194:3300/0 conn(0x7f6b8c0af1c0 0x7f6b8c15eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.425+0000 7f6b8aa00640 1 -- 172.21.15.86:0/1534532839 >> v2:172.21.15.194:3300/0 conn(0x7f6b8c0af1c0 msgr2=0x7f6b8c15eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.425+0000 7f6b8aa00640 1 --2- 172.21.15.86:0/1534532839 >> v2:172.21.15.194:3300/0 conn(0x7f6b8c0af1c0 0x7f6b8c15eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.425+0000 7f6b8aa00640 1 -- 172.21.15.86:0/1534532839 >> v1:172.21.15.86:6789/0 conn(0x7f6b8c06d1a0 legacy=0x7f6b8c06e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:20.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.425+0000 7f6b8aa00640 1 -- 172.21.15.86:0/1534532839 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b8c071770 con 0x7f6b8c070d20 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.426+0000 7f6b8aa00640 1 --2- 172.21.15.86:0/1534532839 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c070d20 0x7f6b8c06cc60 secure :-1 s=READY pgs=1386 cs=0 l=1 rev1=1 crypto rx=0x7f6b78009870 tx=0x7f6b7802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=542a76f60f27a20a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.426+0000 7f6b8a000640 1 -- 172.21.15.86:0/1534532839 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b78034070 con 0x7f6b8c070d20 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.426+0000 7f6b8a000640 1 -- 172.21.15.86:0/1534532839 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b78030860 con 0x7f6b8c070d20 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.426+0000 7f6b8a000640 1 -- 172.21.15.86:0/1534532839 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b78030c30 con 0x7f6b8c070d20 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 -- 172.21.15.86:0/1534532839 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c070d20 msgr2=0x7f6b8c06cc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 --2- 172.21.15.86:0/1534532839 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c070d20 0x7f6b8c06cc60 secure :-1 s=READY pgs=1386 cs=0 l=1 rev1=1 crypto rx=0x7f6b78009870 tx=0x7f6b7802b9a0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 -- 172.21.15.86:0/1534532839 shutdown_connections 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 --2- 172.21.15.86:0/1534532839 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c070d20 0x7f6b8c06cc60 unknown :-1 s=CLOSED pgs=1386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 --2- 172.21.15.86:0/1534532839 >> v2:172.21.15.194:3300/0 conn(0x7f6b8c0af1c0 0x7f6b8c15eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 -- 172.21.15.86:0/1534532839 >> 172.21.15.86:0/1534532839 conn(0x7f6b8c0b1730 msgr2=0x7f6b8c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 -- 172.21.15.86:0/1534532839 shutdown_connections 2024-03-06T07:34:20.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.427+0000 7f6b92800640 1 -- 172.21.15.86:0/1534532839 wait complete. 2024-03-06T07:34:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.429+0000 7f6b92800640 1 Processor -- start 2024-03-06T07:34:20.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.429+0000 7f6b92800640 1 -- start start 2024-03-06T07:34:20.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b92800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8c06d1a0 0x7f6b8c1484d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b92800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b8c070d20 0x7f6b8c0cce00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b92800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c0af1c0 0x7f6b8c152410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b92800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b8c06ebe0 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b92800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b8c0b30b0 con 0x7f6b8c070d20 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b92800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b8c06ed60 con 0x7f6b8c06d1a0 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c0af1c0 0x7f6b8c152410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c0af1c0 0x7f6b8c152410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48218/0 (socket says 172.21.15.86:48218) 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 -- 172.21.15.86:0/4219028163 learned_addr learned my addr 172.21.15.86:0/4219028163 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8aa00640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b8c070d20 0x7f6b8c0cce00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8b400640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8c06d1a0 0x7f6b8c1484d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 -- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8c06d1a0 msgr2=0x7f6b8c1484d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8c06d1a0 0x7f6b8c1484d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 -- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b8c070d20 msgr2=0x7f6b8c0cce00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b8c070d20 0x7f6b8c0cce00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8be00640 1 -- 172.21.15.86:0/4219028163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b8c0cd340 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.430+0000 7f6b8b400640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8c06d1a0 0x7f6b8c1484d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.431+0000 7f6b8be00640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c0af1c0 0x7f6b8c152410 secure :-1 s=READY pgs=1387 cs=0 l=1 rev1=1 crypto rx=0x7f6b8400c760 tx=0x7f6b8400cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.431+0000 7f6b83400640 1 -- 172.21.15.86:0/4219028163 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b84017070 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.431+0000 7f6b83400640 1 -- 172.21.15.86:0/4219028163 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b8400a640 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.431+0000 7f6b83400640 1 -- 172.21.15.86:0/4219028163 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b8400a9f0 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.431+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b8c1552e0 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.431+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b8c158e90 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.432+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b8c04f330 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.434+0000 7f6b83400640 1 -- 172.21.15.86:0/4219028163 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b8401a070 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.436+0000 7f6b83400640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b6c06b4d0 0x7f6b6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.436+0000 7f6b83400640 1 -- 172.21.15.86:0/4219028163 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f6b840896d0 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.436+0000 7f6b8b400640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b6c06b4d0 0x7f6b6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:20.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.437+0000 7f6b8b400640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b6c06b4d0 0x7f6b6c06d990 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f6b7c008890 tx=0x7f6b7c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:20.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.441+0000 7f6b83400640 1 -- 172.21.15.86:0/4219028163 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6b8c071d30 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.674+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6b8c15ec20 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.675+0000 7f6b83400640 1 -- 172.21.15.86:0/4219028163 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f6b84057e70 con 0x7f6b8c0af1c0 2024-03-06T07:34:20.679 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:20.679 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:20.680 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:20.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b6c06b4d0 msgr2=0x7f6b6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:20.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b6c06b4d0 0x7f6b6c06d990 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f6b7c008890 tx=0x7f6b7c008040 comp rx=0 tx=0).stop 2024-03-06T07:34:20.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c0af1c0 msgr2=0x7f6b8c152410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:20.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c0af1c0 0x7f6b8c152410 secure :-1 s=READY pgs=1387 cs=0 l=1 rev1=1 crypto rx=0x7f6b8400c760 tx=0x7f6b8400cc20 comp rx=0 tx=0).stop 2024-03-06T07:34:20.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 shutdown_connections 2024-03-06T07:34:20.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b6c06b4d0 0x7f6b6c06d990 unknown :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b8c0af1c0 0x7f6b8c152410 unknown :-1 s=CLOSED pgs=1387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b8c070d20 0x7f6b8c0cce00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 --2- 172.21.15.86:0/4219028163 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b8c06d1a0 0x7f6b8c1484d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:20.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 >> 172.21.15.86:0/4219028163 conn(0x7f6b8c0b1730 msgr2=0x7f6b8c0afa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:20.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 shutdown_connections 2024-03-06T07:34:20.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:20.678+0000 7f6b92800640 1 -- 172.21.15.86:0/4219028163 wait complete. 2024-03-06T07:34:20.699 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump purge_queue 2024-03-06T07:34:20.919 INFO:teuthology.orchestra.run.smithi086.stdout:{"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":2,"pq_executing":0,"pq_executing_high_water":1,"pq_executed_ops":2,"pq_executed":1,"pq_item_in_journal":0}} 2024-03-06T07:34:20.920 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 1, "pq_executed_ops": 2, "pq_executing": 0, "pq_executing_high_water": 1, "pq_executing_ops": 0, "pq_executing_ops_high_water": 2, "pq_item_in_journal": 0 } } 2024-03-06T07:34:20.920 DEBUG:teuthology.orchestra.run.smithi086:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data ls 2024-03-06T07:34:21.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.009+0000 7fd9c2cf7f40 1 Processor -- start 2024-03-06T07:34:21.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.009+0000 7fd9c2cf7f40 1 -- start start 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9c2cf7f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9a4003ab0 0x55a6e6009240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9c2cf7f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55a6e60affa0 0x55a6e60b0380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9c2cf7f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e60ab610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9c2cf7f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a6e5f81fd0 con 0x55a6e60affa0 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9c2cf7f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a6e5f85f30 con 0x7fd9a4003ab0 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9c2cf7f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55a6e60245d0 con 0x55a6e60ab190 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55a6e60affa0 0x55a6e60b0380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55a6e60affa0 0x55a6e60b0380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48242/0 (socket says 172.21.15.86:48242) 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.010+0000 7fd9b7e00640 1 -- 172.21.15.86:0/3636107323 learned_addr learned my addr 172.21.15.86:0/3636107323 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:21.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9a4003ab0 0x55a6e6009240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9b7e00640 1 -- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 msgr2=0x55a6e60ab610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9c0800640 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e60ab610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e60ab610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9b7e00640 1 -- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9a4003ab0 msgr2=0x55a6e6009240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9a4003ab0 0x55a6e6009240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9b7e00640 1 -- 172.21.15.86:0/3636107323 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a6e6025420 con 0x55a6e60affa0 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9a4003ab0 0x55a6e6009240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55a6e60affa0 0x55a6e60b0380 secure :-1 s=READY pgs=1390 cs=0 l=1 rev1=1 crypto rx=0x7fd9a8001200 tx=0x7fd9a800cb40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=870b2e57c1f481db server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.011+0000 7fd9bea00640 1 -- 172.21.15.86:0/3636107323 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9a8012070 con 0x55a6e60affa0 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9bea00640 1 -- 172.21.15.86:0/3636107323 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd9a800a860 con 0x55a6e60affa0 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9bea00640 1 -- 172.21.15.86:0/3636107323 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9a800ac30 con 0x55a6e60affa0 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/3636107323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55a6e60affa0 msgr2=0x55a6e60b0380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55a6e60affa0 0x55a6e60b0380 secure :-1 s=READY pgs=1390 cs=0 l=1 rev1=1 crypto rx=0x7fd9a8001200 tx=0x7fd9a800cb40 comp rx=0 tx=0).stop 2024-03-06T07:34:21.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/3636107323 shutdown_connections 2024-03-06T07:34:21.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e60ab610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55a6e60affa0 0x55a6e60b0380 unknown :-1 s=CLOSED pgs=1390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/3636107323 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd9a4003ab0 0x55a6e6009240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/3636107323 >> 172.21.15.86:0/3636107323 conn(0x55a6e6008be0 msgr2=0x55a6e5fddc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:21.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/3636107323 shutdown_connections 2024-03-06T07:34:21.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.012+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/3636107323 wait complete. 2024-03-06T07:34:21.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.013+0000 7fd9c2cf7f40 1 Processor -- start 2024-03-06T07:34:21.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.013+0000 7fd9c2cf7f40 1 -- start start 2024-03-06T07:34:21.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c2cf7f40 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e6009c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c2cf7f40 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55a6e60affa0 0x55a6e600a180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c2cf7f40 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9a4003ab0 0x55a6e60a5630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c2cf7f40 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a6e60251c0 con 0x7fd9a4003ab0 2024-03-06T07:34:21.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c2cf7f40 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a6e60aad00 con 0x55a6e60affa0 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c2cf7f40 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55a6e60aa990 con 0x55a6e60ab190 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c0800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9a4003ab0 0x55a6e60a5630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c0800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9a4003ab0 0x55a6e60a5630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48252/0 (socket says 172.21.15.86:48252) 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.014+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 learned_addr learned my addr 172.21.15.86:0/4288642787 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e6009c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 msgr2=0x55a6e6009c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e6009c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55a6e60affa0 msgr2=0x55a6e600a180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55a6e60affa0 0x55a6e600a180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55a6e60affa0 0x55a6e600a180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a6e60a3110 con 0x7fd9a4003ab0 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55a6e60ab190 0x55a6e6009c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9a4003ab0 0x55a6e60a5630 secure :-1 s=READY pgs=1391 cs=0 l=1 rev1=1 crypto rx=0x7fd9b801d0b0 tx=0x7fd9b8005d80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9bd600640 1 -- 172.21.15.86:0/4288642787 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9b800a3b0 con 0x7fd9a4003ab0 2024-03-06T07:34:21.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9bd600640 1 -- 172.21.15.86:0/4288642787 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd9b8052620 con 0x7fd9a4003ab0 2024-03-06T07:34:21.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9bd600640 1 -- 172.21.15.86:0/4288642787 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9b8005a80 con 0x7fd9a4003ab0 2024-03-06T07:34:21.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.015+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x55a6e60a2e70 con 0x7fd9a4003ab0 2024-03-06T07:34:21.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.016+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55a6e60aa140 con 0x7fd9a4003ab0 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.018+0000 7fd9bd600640 1 -- 172.21.15.86:0/4288642787 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd9b805a040 con 0x7fd9a4003ab0 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.018+0000 7fd9bd600640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd99c069ee0 0x7fd99c06c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.018+0000 7fd9bd600640 1 -- 172.21.15.86:0/4288642787 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fd9b80ca760 con 0x7fd9a4003ab0 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.019+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd99c069ee0 0x7fd99c06c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.019+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55a6e60a5b70 0x55a6e60a5f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.019+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:1 17.0 17:00000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.019+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd99c069ee0 0x7fd99c06c3a0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x55a6e60a3e60 tx=0x7fd9b0003530 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.020+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55a6e60a5b70 0x55a6e60a5f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.021+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55a6e60a5b70 0x55a6e60a5f80 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.021+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 1 ==== osd_op_reply(1 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.021+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fd9a800e440 0x7fd9a800e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.021+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:2 17.20 17:04000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a800e440 2024-03-06T07:34:21.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.022+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fd9a800e440 0x7fd9a800e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.022+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fd9a800e440 0x7fd9a800e8a0 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.023+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.1 v2:172.21.15.86:6809/2555210107 1 ==== osd_op_reply(2 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b0010770 con 0x7fd9a800e440 2024-03-06T07:34:21.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.023+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7fd9b001c920 0x7fd9b001b3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.023+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:3 17.10 17:08000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.023+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7fd9b001c920 0x7fd9b001b3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.024+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7fd9b001c920 0x7fd9b001b3a0 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.024+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 1 ==== osd_op_reply(3 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.024+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7fd9a802c960 0x7fd9a80293c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.024+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:4 17.30 17:0c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.024+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7fd9a802c960 0x7fd9a80293c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.025+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7fd9a802c960 0x7fd9a80293c0 crc :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.025+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 1 ==== osd_op_reply(4 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b8064020 con 0x7fd9a802c960 2024-03-06T07:34:21.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.025+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:5 17.8 17:10000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.026+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 2 ==== osd_op_reply(5 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.026+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:6 17.28 17:14000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.026+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 2 ==== osd_op_reply(6 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.026+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:7 17.18 17:18000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.027+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 2 ==== osd_op_reply(7 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.027+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:8 17.38 17:1c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.028+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 3 ==== osd_op_reply(8 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.028+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:9 17.4 17:20000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.028+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 3 ==== osd_op_reply(9 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.028+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:10 17.24 17:24000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.028+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 3 ==== osd_op_reply(10 [pgnls start_epoch 116 out=49b] v110'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.028+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:11 17.14 17:28000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.029+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 4 ==== osd_op_reply(11 [pgnls start_epoch 116 out=49b] v110'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.029+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:12 17.34 17:2c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.029+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 4 ==== osd_op_reply(12 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.029+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:13 17.c 17:30000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.030+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 4 ==== osd_op_reply(13 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.030+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7fd9a802b3e0 0x7fd9a802b860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.030+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:14 17.2c 17:34000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.031+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7fd9a802b3e0 0x7fd9a802b860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.032+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7fd9a802b3e0 0x7fd9a802b860 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.033+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 1 ==== osd_op_reply(14 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b002a040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.033+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:15 17.1c 17:38000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a800e440 2024-03-06T07:34:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.033+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.1 v2:172.21.15.86:6809/2555210107 2 ==== osd_op_reply(15 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b002a040 con 0x7fd9a800e440 2024-03-06T07:34:21.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.033+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:16 17.3c 17:3c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.033+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 5 ==== osd_op_reply(16 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.033+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:17 17.2 17:40000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.034+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 2 ==== osd_op_reply(17 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b002a040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.034+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd9b000f8b0 0x7fd9b002c3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.034+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:18 17.22 17:44000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b000f8b0 2024-03-06T07:34:21.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.035+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd9b000f8b0 0x7fd9b002c3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.036+0000 7fd9b7e00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd9b000f8b0 0x7fd9b002c3b0 crc :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.037+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.5 v2:172.21.15.194:6816/1278478503 1 ==== osd_op_reply(18 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b000f8b0 2024-03-06T07:34:21.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.037+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:19 17.12 17:48000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.037+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 5 ==== osd_op_reply(19 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.037+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:20 17.32 17:4c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.038+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 5 ==== osd_op_reply(20 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.038+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:21 17.a 17:50000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.039+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 6 ==== osd_op_reply(21 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.039+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:22 17.2a 17:54000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.040+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 6 ==== osd_op_reply(22 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.040+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:23 17.1a 17:58000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.040+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 6 ==== osd_op_reply(23 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.040+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:24 17.3a 17:5c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a800e440 2024-03-06T07:34:21.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.041+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.1 v2:172.21.15.86:6809/2555210107 3 ==== osd_op_reply(24 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b002a040 con 0x7fd9a800e440 2024-03-06T07:34:21.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.041+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fd9b0031450 0x7fd9b0031890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.041+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:25 17.6 17:60000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b0031450 2024-03-06T07:34:21.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.042+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fd9b0031450 0x7fd9b0031890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.045 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.043+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fd9b0031450 0x7fd9b0031890 crc :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.043+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.6 v2:172.21.15.194:6808/2611339787 1 ==== osd_op_reply(25 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9b0031450 2024-03-06T07:34:21.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.043+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:26 17.26 17:64000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.044+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 7 ==== osd_op_reply(26 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.044+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:27 17.16 17:68000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b000f8b0 2024-03-06T07:34:21.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.045+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.5 v2:172.21.15.194:6816/1278478503 2 ==== osd_op_reply(27 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b000f8b0 2024-03-06T07:34:21.047 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.045+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:28 17.36 17:6c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.045+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 8 ==== osd_op_reply(28 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.045+0000 7fd9c0800640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x55a6e6020730 0x7fd9b8097e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.045+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- osd_op(unknown.0.0:29 17.e 17:70000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e6020730 2024-03-06T07:34:21.048 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.046+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x55a6e6020730 0x7fd9b8097e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.046+0000 7fd9bfe00640 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x55a6e6020730 0x7fd9b8097e30 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.047+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.2 v2:172.21.15.86:6808/2264489672 1 ==== osd_op_reply(29 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x55a6e6020730 2024-03-06T07:34:21.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.047+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:30 17.2e 17:74000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b0031450 2024-03-06T07:34:21.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.048+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.6 v2:172.21.15.194:6808/2611339787 2 ==== osd_op_reply(30 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9b0031450 2024-03-06T07:34:21.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.048+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:31 17.1e 17:78000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a800e440 2024-03-06T07:34:21.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.048+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.1 v2:172.21.15.86:6809/2555210107 4 ==== osd_op_reply(31 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a800e440 2024-03-06T07:34:21.051 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.048+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:32 17.3e 17:7c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.049+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 3 ==== osd_op_reply(32 [pgnls start_epoch 116 out=49b] v110'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.049+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:33 17.1 17:80000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.052 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.050+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 9 ==== osd_op_reply(33 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.050+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:34 17.21 17:84000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.050+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 10 ==== osd_op_reply(34 [pgnls start_epoch 116 out=49b] v110'12 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.050+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:35 17.11 17:88000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.051+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 7 ==== osd_op_reply(35 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.051+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:36 17.31 17:8c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a800e440 2024-03-06T07:34:21.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.051+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.1 v2:172.21.15.86:6809/2555210107 5 ==== osd_op_reply(36 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a800e440 2024-03-06T07:34:21.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.051+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:37 17.9 17:90000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.051+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 8 ==== osd_op_reply(37 [pgnls start_epoch 116 out=49b] v110'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.051+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:38 17.29 17:94000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.052+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 4 ==== osd_op_reply(38 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.052+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:39 17.19 17:98000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b000f8b0 2024-03-06T07:34:21.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.053+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.5 v2:172.21.15.194:6816/1278478503 3 ==== osd_op_reply(39 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b000f8b0 2024-03-06T07:34:21.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.053+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:40 17.39 17:9c000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b000f8b0 2024-03-06T07:34:21.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.054+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.5 v2:172.21.15.194:6816/1278478503 4 ==== osd_op_reply(40 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b000f8b0 2024-03-06T07:34:21.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.054+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:41 17.5 17:a0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b0031450 2024-03-06T07:34:21.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.055+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.6 v2:172.21.15.194:6808/2611339787 3 ==== osd_op_reply(41 [pgnls start_epoch 116 out=49b] v110'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9b0031450 2024-03-06T07:34:21.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.055+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:42 17.25 17:a4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a800e440 2024-03-06T07:34:21.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.056+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.1 v2:172.21.15.86:6809/2555210107 6 ==== osd_op_reply(42 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a800e440 2024-03-06T07:34:21.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.056+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] -- osd_op(unknown.0.0:43 17.15 17:a8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b000f8b0 2024-03-06T07:34:21.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.057+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.5 v2:172.21.15.194:6816/1278478503 5 ==== osd_op_reply(43 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b000f8b0 2024-03-06T07:34:21.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.057+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:44 17.35 17:ac000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.058+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 5 ==== osd_op_reply(44 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.058+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:45 17.d 17:b0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.059+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 7 ==== osd_op_reply(45 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.059+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:46 17.2d 17:b4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.060+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 8 ==== osd_op_reply(46 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.060+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:47 17.1d 17:b8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.061+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 9 ==== osd_op_reply(47 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.061+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:48 17.3d 17:bc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.062+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 10 ==== osd_op_reply(48 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.062+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:49 17.3 17:c0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.063+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 6 ==== osd_op_reply(49 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.063+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:50 17.23 17:c4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.064+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 7 ==== osd_op_reply(50 [pgnls start_epoch 116 out=49b] v110'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.064+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] -- osd_op(unknown.0.0:51 17.13 17:c8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b001c920 2024-03-06T07:34:21.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.064+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.3 v2:172.21.15.86:6824/3540554891 9 ==== osd_op_reply(51 [pgnls start_epoch 116 out=49b] v110'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x7fd9b001c920 2024-03-06T07:34:21.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.064+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] -- osd_op(unknown.0.0:52 17.33 17:cc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e60a5b70 2024-03-06T07:34:21.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.065+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 <== osd.7 v2:172.21.15.194:6819/3406102027 11 ==== osd_op_reply(52 [pgnls start_epoch 116 out=49b] v110'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9a800aa10 con 0x55a6e60a5b70 2024-03-06T07:34:21.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.065+0000 7fd9b7e00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:53 17.b 17:d0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.066+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 11 ==== osd_op_reply(53 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.066+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:54 17.2b 17:d4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.067+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 8 ==== osd_op_reply(54 [pgnls start_epoch 116 out=49b] v110'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.067+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:55 17.1b 17:d8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.068+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 9 ==== osd_op_reply(55 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b003e040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.068+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- osd_op(unknown.0.0:56 17.3b 17:dc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e6020730 2024-03-06T07:34:21.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.068+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.2 v2:172.21.15.86:6808/2264489672 2 ==== osd_op_reply(56 [pgnls start_epoch 116 out=49b] v110'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b0050040 con 0x55a6e6020730 2024-03-06T07:34:21.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.068+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:57 17.7 17:e0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.069+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 12 ==== osd_op_reply(57 [pgnls start_epoch 116 out=49b] v116'12 uv12 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.069+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:58 17.27 17:e4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.070+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 10 ==== osd_op_reply(58 [pgnls start_epoch 116 out=49b] v110'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b0050040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.070+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:59 17.17 17:e8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b0031450 2024-03-06T07:34:21.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.071+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.6 v2:172.21.15.194:6808/2611339787 4 ==== osd_op_reply(59 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9b0031450 2024-03-06T07:34:21.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.071+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] -- osd_op(unknown.0.0:60 17.37 17:ec000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802c960 2024-03-06T07:34:21.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.072+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.0 v2:172.21.15.86:6800/1601033517 13 ==== osd_op_reply(60 [pgnls start_epoch 116 out=49b] v110'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9a802c960 2024-03-06T07:34:21.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.072+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] -- osd_op(unknown.0.0:61 17.f 17:f0000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a800e440 2024-03-06T07:34:21.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.073+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.1 v2:172.21.15.86:6809/2555210107 7 ==== osd_op_reply(61 [pgnls start_epoch 116 out=49b] v110'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b0050040 con 0x7fd9a800e440 2024-03-06T07:34:21.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.073+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] -- osd_op(unknown.0.0:62 17.2f 17:f4000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9a802b3e0 2024-03-06T07:34:21.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.074+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.4 v2:172.21.15.194:6800/2403584566 11 ==== osd_op_reply(62 [pgnls start_epoch 116 out=49b] v116'10 uv10 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b0050040 con 0x7fd9a802b3e0 2024-03-06T07:34:21.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.074+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] -- osd_op(unknown.0.0:63 17.1f 17:f8000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x7fd9b0031450 2024-03-06T07:34:21.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.075+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 <== osd.6 v2:172.21.15.194:6808/2611339787 5 ==== osd_op_reply(63 [pgnls start_epoch 116 out=49b] v110'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b809c040 con 0x7fd9b0031450 2024-03-06T07:34:21.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.075+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 --> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] -- osd_op(unknown.0.0:64 17.3f 17:fc000000::::head [pgnls start_epoch 116 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e116) v9 -- 0x55a6e6155bf0 con 0x55a6e6020730 2024-03-06T07:34:21.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.075+0000 7fd9bfe00640 1 -- 172.21.15.86:0/4288642787 <== osd.2 v2:172.21.15.86:6808/2264489672 3 ==== osd_op_reply(64 [pgnls start_epoch 116 out=49b] v110'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fd9b0050040 con 0x55a6e6020730 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7fd9a802c960 msgr2=0x7fd9a80293c0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6800/1601033517,v1:172.21.15.86:6801/1601033517] conn(0x7fd9a802c960 0x7fd9a80293c0 crc :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fd9a800e440 msgr2=0x7fd9a800e8a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6809/2555210107,v1:172.21.15.86:6811/2555210107] conn(0x7fd9a800e440 0x7fd9a800e8a0 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x55a6e6020730 msgr2=0x7fd9b8097e30 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6808/2264489672,v1:172.21.15.86:6810/2264489672] conn(0x55a6e6020730 0x7fd9b8097e30 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7fd9b001c920 msgr2=0x7fd9b001b3a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6824/3540554891,v1:172.21.15.86:6825/3540554891] conn(0x7fd9b001c920 0x7fd9b001b3a0 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7fd9a802b3e0 msgr2=0x7fd9a802b860 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6800/2403584566,v1:172.21.15.194:6801/2403584566] conn(0x7fd9a802b3e0 0x7fd9a802b860 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd9b000f8b0 msgr2=0x7fd9b002c3b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6816/1278478503,v1:172.21.15.194:6817/1278478503] conn(0x7fd9b000f8b0 0x7fd9b002c3b0 crc :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fd9b0031450 msgr2=0x7fd9b0031890 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6808/2611339787,v1:172.21.15.194:6809/2611339787] conn(0x7fd9b0031450 0x7fd9b0031890 crc :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 reap_dead start 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55a6e60a5b70 msgr2=0x55a6e60a5f80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55a6e60a5b70 0x55a6e60a5f80 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c0800640 1 -- 172.21.15.86:0/4288642787 reap_dead start 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd99c069ee0 msgr2=0x7fd99c06c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd99c069ee0 0x7fd99c06c3a0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x55a6e60a3e60 tx=0x7fd9b0003530 comp rx=0 tx=0).stop 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9a4003ab0 msgr2=0x55a6e60a5630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9a4003ab0 0x55a6e60a5630 secure :-1 s=READY pgs=1391 cs=0 l=1 rev1=1 crypto rx=0x7fd9b801d0b0 tx=0x7fd9b8005d80 comp rx=0 tx=0).stop 2024-03-06T07:34:21.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 shutdown_connections 2024-03-06T07:34:21.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.194:6819/3406102027,v1:172.21.15.194:6821/3406102027] conn(0x55a6e60a5b70 0x55a6e60a5f80 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd99c069ee0 0x7fd99c06c3a0 secure :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0x55a6e60a3e60 tx=0x7fd9b0003530 comp rx=0 tx=0).stop 2024-03-06T07:34:21.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 --2- 172.21.15.86:0/4288642787 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd9a4003ab0 0x55a6e60a5630 unknown :-1 s=CLOSED pgs=1391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.076+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 >> 172.21.15.86:0/4288642787 conn(0x55a6e6008be0 msgr2=0x55a6e60af0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:21.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.077+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 shutdown_connections 2024-03-06T07:34:21.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.077+0000 7fd9c2cf7f40 1 -- 172.21.15.86:0/4288642787 wait complete. 2024-03-06T07:34:21.084 DEBUG:tasks.cephfs.filesystem:All objects for ino 1099511627777 size 8388608 are absent 2024-03-06T07:34:21.084 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:21.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.272+0000 7ffab8a00640 1 Processor -- start 2024-03-06T07:34:21.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.272+0000 7ffab8a00640 1 -- start start 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab8a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ffab4144c00 0x7ffab406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab8a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ffab415fe60 0x7ffab40ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab8a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffab406ee70 con 0x7ffab415f9d0 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab8a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffab406cef0 con 0x7ffab415fe60 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab8a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffab406d1d0 con 0x7ffab4144c00 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ffab415fe60 0x7ffab40ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ffab415fe60 0x7ffab40ce7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48268/0 (socket says 172.21.15.86:48268) 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffaa9400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ffab415f9d0 0x7ffab40ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34128/0 (socket says 172.21.15.86:34128) 2024-03-06T07:34:21.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.274+0000 7ffab2800640 1 -- 172.21.15.86:0/753426203 learned_addr learned my addr 172.21.15.86:0/753426203 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.275+0000 7ffab1e00640 1 --2- 172.21.15.86:0/753426203 >> v2:172.21.15.194:3300/0 conn(0x7ffab4144c00 0x7ffab406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.275+0000 7ffab2800640 1 -- 172.21.15.86:0/753426203 >> v2:172.21.15.194:3300/0 conn(0x7ffab4144c00 msgr2=0x7ffab406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.275+0000 7ffab2800640 1 --2- 172.21.15.86:0/753426203 >> v2:172.21.15.194:3300/0 conn(0x7ffab4144c00 0x7ffab406e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.275+0000 7ffab2800640 1 -- 172.21.15.86:0/753426203 >> v1:172.21.15.86:6789/0 conn(0x7ffab415f9d0 legacy=0x7ffab40ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.275+0000 7ffab2800640 1 -- 172.21.15.86:0/753426203 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffab406d4b0 con 0x7ffab415fe60 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.276+0000 7ffab2800640 1 --2- 172.21.15.86:0/753426203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab40ce7b0 secure :-1 s=READY pgs=1393 cs=0 l=1 rev1=1 crypto rx=0x7ffab40b25f0 tx=0x7ffaac02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=75ace0df7b0d200e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.276+0000 7ffab1400640 1 -- 172.21.15.86:0/753426203 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaac034070 con 0x7ffab415fe60 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.276+0000 7ffab1400640 1 -- 172.21.15.86:0/753426203 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffaac030860 con 0x7ffab415fe60 2024-03-06T07:34:21.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.276+0000 7ffab1400640 1 -- 172.21.15.86:0/753426203 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaac030c30 con 0x7ffab415fe60 2024-03-06T07:34:21.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 -- 172.21.15.86:0/753426203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 msgr2=0x7ffab40ce7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 --2- 172.21.15.86:0/753426203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab40ce7b0 secure :-1 s=READY pgs=1393 cs=0 l=1 rev1=1 crypto rx=0x7ffab40b25f0 tx=0x7ffaac02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 -- 172.21.15.86:0/753426203 shutdown_connections 2024-03-06T07:34:21.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 --2- 172.21.15.86:0/753426203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab40ce7b0 unknown :-1 s=CLOSED pgs=1393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 --2- 172.21.15.86:0/753426203 >> v2:172.21.15.194:3300/0 conn(0x7ffab4144c00 0x7ffab406e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 -- 172.21.15.86:0/753426203 >> 172.21.15.86:0/753426203 conn(0x7ffab40b1730 msgr2=0x7ffab40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:21.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 -- 172.21.15.86:0/753426203 shutdown_connections 2024-03-06T07:34:21.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.277+0000 7ffab8a00640 1 -- 172.21.15.86:0/753426203 wait complete. 2024-03-06T07:34:21.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.279+0000 7ffab8a00640 1 Processor -- start 2024-03-06T07:34:21.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.279+0000 7ffab8a00640 1 -- start start 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab8a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffab4144c00 0x7ffab41f5290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab8a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffab415f9d0 0x7ffab413ac20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab413b160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab8a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffab4160730 con 0x7ffab415fe60 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab8a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffab4160270 con 0x7ffab4144c00 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab8a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffab406cb50 con 0x7ffab415f9d0 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab413b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab413b160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48276/0 (socket says 172.21.15.86:48276) 2024-03-06T07:34:21.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.280+0000 7ffab2800640 1 -- 172.21.15.86:0/241356103 learned_addr learned my addr 172.21.15.86:0/241356103 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab2800640 1 -- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffab415f9d0 msgr2=0x7ffab413ac20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffaa9400640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffab415f9d0 0x7ffab413ac20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab1e00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffab4144c00 0x7ffab41f5290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab2800640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffab415f9d0 0x7ffab413ac20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab2800640 1 -- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffab4144c00 msgr2=0x7ffab41f5290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab2800640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffab4144c00 0x7ffab41f5290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab2800640 1 -- 172.21.15.86:0/241356103 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffab40b4360 con 0x7ffab415fe60 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab2800640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab413b160 secure :-1 s=READY pgs=1394 cs=0 l=1 rev1=1 crypto rx=0x7ffaac009960 tx=0x7ffaac004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffaaaa00640 1 -- 172.21.15.86:0/241356103 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaac048070 con 0x7ffab415fe60 2024-03-06T07:34:21.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffaaaa00640 1 -- 172.21.15.86:0/241356103 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffaac039d90 con 0x7ffab415fe60 2024-03-06T07:34:21.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffab40b4640 con 0x7ffab415fe60 2024-03-06T07:34:21.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.281+0000 7ffaaaa00640 1 -- 172.21.15.86:0/241356103 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaac034040 con 0x7ffab415fe60 2024-03-06T07:34:21.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.282+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffab40ada00 con 0x7ffab415fe60 2024-03-06T07:34:21.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.283+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffab40b4b60 con 0x7ffab415fe60 2024-03-06T07:34:21.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.284+0000 7ffaaaa00640 1 -- 172.21.15.86:0/241356103 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffaac04b070 con 0x7ffab415fe60 2024-03-06T07:34:21.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.285+0000 7ffaaaa00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa9006b500 0x7ffa9006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.286+0000 7ffaaaa00640 1 -- 172.21.15.86:0/241356103 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7ffaac0ac570 con 0x7ffab415fe60 2024-03-06T07:34:21.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.286+0000 7ffab1e00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa9006b500 0x7ffa9006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.286+0000 7ffab1e00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa9006b500 0x7ffa9006d9c0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7ffaa4008e50 tx=0x7ffaa40083a0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.289+0000 7ffaaaa00640 1 -- 172.21.15.86:0/241356103 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffab40b4b60 con 0x7ffab415fe60 2024-03-06T07:34:21.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.524+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ffab404f330 con 0x7ffab415fe60 2024-03-06T07:34:21.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.525+0000 7ffaaaa00640 1 -- 172.21.15.86:0/241356103 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7ffaac07b3e0 con 0x7ffab415fe60 2024-03-06T07:34:21.527 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:21.528 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:21.528 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:21.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa9006b500 msgr2=0x7ffa9006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa9006b500 0x7ffa9006d9c0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7ffaa4008e50 tx=0x7ffaa40083a0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 msgr2=0x7ffab413b160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab413b160 secure :-1 s=READY pgs=1394 cs=0 l=1 rev1=1 crypto rx=0x7ffaac009960 tx=0x7ffaac004730 comp rx=0 tx=0).stop 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 shutdown_connections 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffa9006b500 0x7ffa9006d9c0 unknown :-1 s=CLOSED pgs=597 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffab415fe60 0x7ffab413b160 unknown :-1 s=CLOSED pgs=1394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffab415f9d0 0x7ffab413ac20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 --2- 172.21.15.86:0/241356103 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffab4144c00 0x7ffab41f5290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.527+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 >> 172.21.15.86:0/241356103 conn(0x7ffab40b1730 msgr2=0x7ffab406de90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.528+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 shutdown_connections 2024-03-06T07:34:21.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.528+0000 7ffab8a00640 1 -- 172.21.15.86:0/241356103 wait complete. 2024-03-06T07:34:21.550 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_created 2024-03-06T07:34:21.765 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_created":3}} 2024-03-06T07:34:21.765 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 3 } } 2024-03-06T07:34:21.765 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:21.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.968+0000 7f6152400640 1 Processor -- start 2024-03-06T07:34:21.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.968+0000 7f6152400640 1 -- start start 2024-03-06T07:34:21.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.970+0000 7f6152400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 0x7f614c06db80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.970+0000 7f6152400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f614c06e160 con 0x7f614c06afc0 2024-03-06T07:34:21.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.970+0000 7f6152400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f614c06e440 con 0x7f614c0aec50 2024-03-06T07:34:21.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.970+0000 7f6152400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f614c06ed70 con 0x7f614c15e1d0 2024-03-06T07:34:21.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.970+0000 7f614b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f614c06afc0 0x7f614c06b3a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34136/0 (socket says 172.21.15.86:34136) 2024-03-06T07:34:21.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.970+0000 7f614b400640 1 -- 172.21.15.86:0/2290579485 learned_addr learned my addr 172.21.15.86:0/2290579485 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:21.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.970+0000 7f614aa00640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 0x7f614c06db80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2795756285 0 0) 0x7f614c06e160 con 0x7f614c06afc0 2024-03-06T07:34:21.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6130003520 con 0x7f614c06afc0 2024-03-06T07:34:21.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1110311122 0 0) 0x7f6130003520 con 0x7f614c06afc0 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 msgr2=0x7f614c06db80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 0x7f614c06db80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 >> v1:172.21.15.194:6789/0 conn(0x7f614c0aec50 legacy=0x7f614c167b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f614c06e720 con 0x7f614c06afc0 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f613c001670 con 0x7f614c06afc0 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 >> v1:172.21.15.86:6789/0 conn(0x7f614c06afc0 legacy=0x7f614c06b3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61300039e0 0x7f6130003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 0x7f6130004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6130004d40 0x7f61300051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f614c06ed70 con 0x7f61300039e0 2024-03-06T07:34:21.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f613000baa0 con 0x7f6130004d40 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.971+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f613000bca0 con 0x7f614c15e1d0 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614b400640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61300039e0 0x7f6130003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614aa00640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 0x7f6130004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614be00640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6130004d40 0x7f61300051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614b400640 1 -- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 msgr2=0x7f6130004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614b400640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f614c15e1d0 0x7f6130004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614b400640 1 -- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6130004d40 msgr2=0x7f61300051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614b400640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6130004d40 0x7f61300051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.972+0000 7f614b400640 1 -- 172.21.15.86:0/2290579485 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6130003520 con 0x7f61300039e0 2024-03-06T07:34:21.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.974+0000 7f614b400640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61300039e0 0x7f6130003e00 secure :-1 s=READY pgs=1396 cs=0 l=1 rev1=1 crypto rx=0x7f613c00c210 tx=0x7f6130005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44f9ef22559ed32d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f613c007d10 con 0x7f61300039e0 2024-03-06T07:34:21.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f613c030070 con 0x7f61300039e0 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f614a000640 1 -- 172.21.15.86:0/2290579485 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f613c005d50 con 0x7f61300039e0 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f6152400640 1 -- 172.21.15.86:0/2290579485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61300039e0 msgr2=0x7f6130003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f6152400640 1 --2- 172.21.15.86:0/2290579485 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f61300039e0 0x7f6130003e00 secure :-1 s=READY pgs=1396 cs=0 l=1 rev1=1 crypto rx=0x7f613c00c210 tx=0x7f6130005830 comp rx=0 tx=0).stop 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f614be00640 1 -- 172.21.15.86:0/2290579485 reap_dead start 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f6152400640 1 -- 172.21.15.86:0/2290579485 shutdown_connections 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f6152400640 1 -- 172.21.15.86:0/2290579485 >> 172.21.15.86:0/2290579485 conn(0x7f614c0d0b40 msgr2=0x7f614c0d0f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f6152400640 1 -- 172.21.15.86:0/2290579485 shutdown_connections 2024-03-06T07:34:21.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.975+0000 7f6152400640 1 -- 172.21.15.86:0/2290579485 wait complete. 2024-03-06T07:34:21.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.977+0000 7f6152400640 1 Processor -- start 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.977+0000 7f6152400640 1 -- start start 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f6152400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61300039e0 0x7f614c154650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f6152400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f614c1504c0 0x7f614c160050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f6152400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f614c160590 0x7f614c152f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f6152400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f614c0d1630 con 0x7f614c160590 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f6152400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f614c0704b0 con 0x7f614c1504c0 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f6152400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f614c0d17b0 con 0x7f61300039e0 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f614be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f614c160590 0x7f614c152f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f614be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f614c160590 0x7f614c152f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48302/0 (socket says 172.21.15.86:48302) 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.978+0000 7f614be00640 1 -- 172.21.15.86:0/3194427081 learned_addr learned my addr 172.21.15.86:0/3194427081 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614b400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61300039e0 0x7f614c154650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614aa00640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f614c1504c0 0x7f614c160050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614be00640 1 -- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61300039e0 msgr2=0x7f614c154650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614be00640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61300039e0 0x7f614c154650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614be00640 1 -- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f614c1504c0 msgr2=0x7f614c160050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:21.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614be00640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f614c1504c0 0x7f614c160050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:21.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614be00640 1 -- 172.21.15.86:0/3194427081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f614c15e5e0 con 0x7f614c160590 2024-03-06T07:34:21.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.979+0000 7f614be00640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f614c160590 0x7f614c152f40 secure :-1 s=READY pgs=1397 cs=0 l=1 rev1=1 crypto rx=0x7f614c068cd0 tx=0x7f614400ec10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.980+0000 7f6143400640 1 -- 172.21.15.86:0/3194427081 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6144003970 con 0x7f614c160590 2024-03-06T07:34:21.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.980+0000 7f6143400640 1 -- 172.21.15.86:0/3194427081 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6144018070 con 0x7f614c160590 2024-03-06T07:34:21.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.980+0000 7f6143400640 1 -- 172.21.15.86:0/3194427081 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6144003d50 con 0x7f614c160590 2024-03-06T07:34:21.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.980+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f614c15e8c0 con 0x7f614c160590 2024-03-06T07:34:21.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.980+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f614c15ee80 con 0x7f614c160590 2024-03-06T07:34:21.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.981+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f614c04f330 con 0x7f614c160590 2024-03-06T07:34:21.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.982+0000 7f6143400640 1 -- 172.21.15.86:0/3194427081 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f61440050d0 con 0x7f614c160590 2024-03-06T07:34:21.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.983+0000 7f6143400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f612c06b4d0 0x7f612c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:21.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.983+0000 7f6143400640 1 -- 172.21.15.86:0/3194427081 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f6144089ff0 con 0x7f614c160590 2024-03-06T07:34:21.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.983+0000 7f614b400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f612c06b4d0 0x7f612c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:21.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.984+0000 7f614b400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f612c06b4d0 0x7f612c06d990 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7f613c034000 tx=0x7f613c009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:21.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:21.987+0000 7f6143400640 1 -- 172.21.15.86:0/3194427081 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6144058d80 con 0x7f614c160590 2024-03-06T07:34:22.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.221+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f614c0b11f0 con 0x7f614c160590 2024-03-06T07:34:22.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.222+0000 7f6143400640 1 -- 172.21.15.86:0/3194427081 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f6144058680 con 0x7f614c160590 2024-03-06T07:34:22.225 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:22.225 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:22.225 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.224+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f612c06b4d0 msgr2=0x7f612c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.224+0000 7f6152400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f612c06b4d0 0x7f612c06d990 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7f613c034000 tx=0x7f613c009030 comp rx=0 tx=0).stop 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.224+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f614c160590 msgr2=0x7f614c152f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.224+0000 7f6152400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f614c160590 0x7f614c152f40 secure :-1 s=READY pgs=1397 cs=0 l=1 rev1=1 crypto rx=0x7f614c068cd0 tx=0x7f614400ec10 comp rx=0 tx=0).stop 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 shutdown_connections 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f612c06b4d0 0x7f612c06d990 unknown :-1 s=CLOSED pgs=598 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f614c160590 0x7f614c152f40 unknown :-1 s=CLOSED pgs=1397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f614c1504c0 0x7f614c160050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 --2- 172.21.15.86:0/3194427081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f61300039e0 0x7f614c154650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 >> 172.21.15.86:0/3194427081 conn(0x7f614c0d0b40 msgr2=0x7f614c0b0740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 shutdown_connections 2024-03-06T07:34:22.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.225+0000 7f6152400640 1 -- 172.21.15.86:0/3194427081 wait complete. 2024-03-06T07:34:22.245 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:34:22.456 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":1}} 2024-03-06T07:34:22.457 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 1 } } 2024-03-06T07:34:22.457 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:34:22.457 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:34:22.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:34:22.683 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:34:22.683 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:34:22.684 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:22.737 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:34:22.737 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:22.788 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:22.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.985+0000 7f67dd400640 1 Processor -- start 2024-03-06T07:34:22.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.985+0000 7f67dd400640 1 -- start start 2024-03-06T07:34:22.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.987+0000 7f67dd400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f67d815e160 0x7f67d8162620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:22.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.987+0000 7f67dd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 0x7f67d806b580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:22.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.987+0000 7f67dd400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67d8162b60 con 0x7f67d80af820 2024-03-06T07:34:22.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.987+0000 7f67dd400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f67d806b0f0 con 0x7f67d815e160 2024-03-06T07:34:22.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.987+0000 7f67dd400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67d80b5620 con 0x7f67d80b3ac0 2024-03-06T07:34:22.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.987+0000 7f67d6800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f67d815e160 0x7f67d8162620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:22.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d6800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f67d815e160 0x7f67d8162620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48304/0 (socket says 172.21.15.86:48304) 2024-03-06T07:34:22.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d6800640 1 -- 172.21.15.86:0/185479636 learned_addr learned my addr 172.21.15.86:0/185479636 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:22.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d7200640 1 --2- 172.21.15.86:0/185479636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 0x7f67d806b580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:22.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d6800640 1 -- 172.21.15.86:0/185479636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 msgr2=0x7f67d806b580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:22.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d6800640 1 --2- 172.21.15.86:0/185479636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 0x7f67d806b580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d6800640 1 -- 172.21.15.86:0/185479636 >> v1:172.21.15.86:6789/0 conn(0x7f67d80af820 legacy=0x7f67d80b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:22.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d5400640 1 -- 172.21.15.86:0/185479636 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3395727673 0 0) 0x7f67c4001110 con 0x7f67d80af820 2024-03-06T07:34:22.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.988+0000 7f67d6800640 1 -- 172.21.15.86:0/185479636 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67d806f1f0 con 0x7f67d815e160 2024-03-06T07:34:22.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.989+0000 7f67d6800640 1 --2- 172.21.15.86:0/185479636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d815e160 0x7f67d8162620 secure :-1 s=READY pgs=1399 cs=0 l=1 rev1=1 crypto rx=0x7f67c8009870 tx=0x7f67c802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2ad20bc35cbd203c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:22.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67d5400640 1 -- 172.21.15.86:0/185479636 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67c8035020 con 0x7f67d815e160 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67d5400640 1 -- 172.21.15.86:0/185479636 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f67c8031640 con 0x7f67d815e160 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67d5400640 1 -- 172.21.15.86:0/185479636 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67c802d040 con 0x7f67d815e160 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67dd400640 1 -- 172.21.15.86:0/185479636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d815e160 msgr2=0x7f67d8162620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67dd400640 1 --2- 172.21.15.86:0/185479636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d815e160 0x7f67d8162620 secure :-1 s=READY pgs=1399 cs=0 l=1 rev1=1 crypto rx=0x7f67c8009870 tx=0x7f67c802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67dd400640 1 -- 172.21.15.86:0/185479636 shutdown_connections 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67dd400640 1 --2- 172.21.15.86:0/185479636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 0x7f67d806b580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67dd400640 1 --2- 172.21.15.86:0/185479636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d815e160 0x7f67d8162620 unknown :-1 s=CLOSED pgs=1399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.990+0000 7f67dd400640 1 -- 172.21.15.86:0/185479636 >> 172.21.15.86:0/185479636 conn(0x7f67d80d0340 msgr2=0x7f67d80d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.991+0000 7f67dd400640 1 -- 172.21.15.86:0/185479636 shutdown_connections 2024-03-06T07:34:22.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.991+0000 7f67dd400640 1 -- 172.21.15.86:0/185479636 wait complete. 2024-03-06T07:34:22.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.992+0000 7f67dd400640 1 Processor -- start 2024-03-06T07:34:22.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.992+0000 7f67dd400640 1 -- start start 2024-03-06T07:34:22.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67dd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d80af820 0x7f67d80ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:22.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67dd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 0x7f67d81602f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:22.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67dd400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67d815e160 0x7f67d81479d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:22.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67dd400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67d806ef10 con 0x7f67d80af820 2024-03-06T07:34:22.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67dd400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67d806ec30 con 0x7f67d815e160 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67dd400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67d806bac0 con 0x7f67d80b3ac0 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67d6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d80af820 0x7f67d80ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67d6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d80af820 0x7f67d80ccbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48320/0 (socket says 172.21.15.86:48320) 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.994+0000 7f67d6800640 1 -- 172.21.15.86:0/3407553083 learned_addr learned my addr 172.21.15.86:0/3407553083 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67d6800640 1 -- 172.21.15.86:0/3407553083 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 msgr2=0x7f67d81602f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67d6800640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 0x7f67d81602f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67d6800640 1 -- 172.21.15.86:0/3407553083 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67d815e160 msgr2=0x7f67d81479d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67d6800640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67d815e160 0x7f67d81479d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67d6800640 1 -- 172.21.15.86:0/3407553083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67d80cd310 con 0x7f67d80af820 2024-03-06T07:34:22.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67d6800640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d80af820 0x7f67d80ccbc0 secure :-1 s=READY pgs=1400 cs=0 l=1 rev1=1 crypto rx=0x7f67c8034040 tx=0x7f67c80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:22.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67cea00640 1 -- 172.21.15.86:0/3407553083 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67c8040070 con 0x7f67d80af820 2024-03-06T07:34:22.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67cea00640 1 -- 172.21.15.86:0/3407553083 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f67c80399f0 con 0x7f67d80af820 2024-03-06T07:34:22.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f67d80cd5f0 con 0x7f67d80af820 2024-03-06T07:34:22.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67cea00640 1 -- 172.21.15.86:0/3407553083 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67c8035040 con 0x7f67d80af820 2024-03-06T07:34:22.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.995+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f67d80cdbb0 con 0x7f67d80af820 2024-03-06T07:34:23.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.997+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f67d804f330 con 0x7f67d80af820 2024-03-06T07:34:23.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.997+0000 7f67cea00640 1 -- 172.21.15.86:0/3407553083 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f67c8038020 con 0x7f67d80af820 2024-03-06T07:34:23.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.998+0000 7f67cea00640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f67b806b4d0 0x7f67b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.998+0000 7f67cea00640 1 -- 172.21.15.86:0/3407553083 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f67c80abc90 con 0x7f67d80af820 2024-03-06T07:34:23.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.998+0000 7f67d5e00640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f67b806b4d0 0x7f67b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:23.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:22.999+0000 7f67d5e00640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f67b806b4d0 0x7f67b806d990 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7f67d80ad980 tx=0x7f67c4007b60 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:23.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.003+0000 7f67cea00640 1 -- 172.21.15.86:0/3407553083 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f67c807a9e0 con 0x7f67d80af820 2024-03-06T07:34:23.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.299+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f67d80b42f0 con 0x7f67d80af820 2024-03-06T07:34:23.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.301+0000 7f67cea00640 1 -- 172.21.15.86:0/3407553083 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v116) v1 ==== 74+0+16920 (secure 0 0 0) 0x7f67c807a2e0 con 0x7f67d80af820 2024-03-06T07:34:23.304 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:23.304 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":116,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:55.692842+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"110","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.3f","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/221976790":"2024-03-07T07:33:20.538234+0000","172.21.15.86:6836/221976790":"2024-03-07T07:33:20.538234+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f67b806b4d0 msgr2=0x7f67b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f67b806b4d0 0x7f67b806d990 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7f67d80ad980 tx=0x7f67c4007b60 comp rx=0 tx=0).stop 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d80af820 msgr2=0x7f67d80ccbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d80af820 0x7f67d80ccbc0 secure :-1 s=READY pgs=1400 cs=0 l=1 rev1=1 crypto rx=0x7f67c8034040 tx=0x7f67c80046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 shutdown_connections 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f67b806b4d0 0x7f67b806d990 unknown :-1 s=CLOSED pgs=599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f67d815e160 0x7f67d81479d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f67d80b3ac0 0x7f67d81602f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 --2- 172.21.15.86:0/3407553083 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f67d80af820 0x7f67d80ccbc0 unknown :-1 s=CLOSED pgs=1400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 >> 172.21.15.86:0/3407553083 conn(0x7f67d80d0340 msgr2=0x7f67d806be80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:23.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 shutdown_connections 2024-03-06T07:34:23.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.305+0000 7f67dd400640 1 -- 172.21.15.86:0/3407553083 wait complete. 2024-03-06T07:34:23.327 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 16, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:33:03.630003+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '109', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 17, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:33:04.861178+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '110', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:34:23.327 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:34:23.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.580+0000 7f75e8a00640 1 Processor -- start 2024-03-06T07:34:23.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.580+0000 7f75e8a00640 1 -- start start 2024-03-06T07:34:23.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e8a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75e4158570 0x7f75e40ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e8a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 0x7f75e40cf6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e8a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f75e40aebf0 0x7f75e40cfc10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e8a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75e415b130 con 0x7f75e4158570 2024-03-06T07:34:23.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e8a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75e40af170 con 0x7f75e40aebf0 2024-03-06T07:34:23.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e8a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75e415b3f0 con 0x7f75e4158950 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e3400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75e4158570 0x7f75e40ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e3400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75e4158570 0x7f75e40ae6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48332/0 (socket says 172.21.15.86:48332) 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.582+0000 7f75e3400640 1 -- 172.21.15.86:0/1409373785 learned_addr learned my addr 172.21.15.86:0/1409373785 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.583+0000 7f75e3400640 1 -- 172.21.15.86:0/1409373785 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 msgr2=0x7f75e40cf6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.583+0000 7f75e3400640 1 --2- 172.21.15.86:0/1409373785 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 0x7f75e40cf6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.583+0000 7f75e3400640 1 -- 172.21.15.86:0/1409373785 >> v2:172.21.15.194:3300/0 conn(0x7f75e40aebf0 msgr2=0x7f75e40cfc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.583+0000 7f75e3400640 1 --2- 172.21.15.86:0/1409373785 >> v2:172.21.15.194:3300/0 conn(0x7f75e40aebf0 0x7f75e40cfc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.583+0000 7f75e3400640 1 -- 172.21.15.86:0/1409373785 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75e415b6d0 con 0x7f75e4158570 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.584+0000 7f75e3400640 1 --2- 172.21.15.86:0/1409373785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e40ae6b0 secure :-1 s=READY pgs=1401 cs=0 l=1 rev1=1 crypto rx=0x7f75e40b4190 tx=0x7f75d402da90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c672b3c90f2b2f26 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.584+0000 7f75e2a00640 1 -- 172.21.15.86:0/1409373785 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75d402f850 con 0x7f75e4158570 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.584+0000 7f75e2a00640 1 -- 172.21.15.86:0/1409373785 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75d402f9f0 con 0x7f75e4158570 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.584+0000 7f75e2a00640 1 -- 172.21.15.86:0/1409373785 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75d402fdc0 con 0x7f75e4158570 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 -- 172.21.15.86:0/1409373785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 msgr2=0x7f75e40ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1409373785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e40ae6b0 secure :-1 s=READY pgs=1401 cs=0 l=1 rev1=1 crypto rx=0x7f75e40b4190 tx=0x7f75d402da90 comp rx=0 tx=0).stop 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 -- 172.21.15.86:0/1409373785 shutdown_connections 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1409373785 >> v2:172.21.15.194:3300/0 conn(0x7f75e40aebf0 0x7f75e40cfc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1409373785 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 0x7f75e40cf6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1409373785 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e40ae6b0 unknown :-1 s=CLOSED pgs=1401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 -- 172.21.15.86:0/1409373785 >> 172.21.15.86:0/1409373785 conn(0x7f75e406afc0 msgr2=0x7f75e406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 -- 172.21.15.86:0/1409373785 shutdown_connections 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.585+0000 7f75e8a00640 1 -- 172.21.15.86:0/1409373785 wait complete. 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.587+0000 7f75e8a00640 1 Processor -- start 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.587+0000 7f75e8a00640 1 -- start start 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e8a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75e40aebf0 0x7f75e415c2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e415a7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e8a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 0x7f75e415da50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e8a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75e415bed0 con 0x7f75e4158570 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e8a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75e40b4950 con 0x7f75e40aebf0 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e8a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75e415c050 con 0x7f75e4158950 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e415a7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:23.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e415a7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48346/0 (socket says 172.21.15.86:48346) 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 -- 172.21.15.86:0/1984433760 learned_addr learned my addr 172.21.15.86:0/1984433760 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e3400640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75e40aebf0 0x7f75e415c2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 -- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 msgr2=0x7f75e415da50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75e3e00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 0x7f75e415da50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 0x7f75e415da50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 -- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75e40aebf0 msgr2=0x7f75e415c2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75e40aebf0 0x7f75e415c2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.588+0000 7f75dbe00640 1 -- 172.21.15.86:0/1984433760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75e415c9b0 con 0x7f75e4158570 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.589+0000 7f75dbe00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e415a7c0 secure :-1 s=READY pgs=1402 cs=0 l=1 rev1=1 crypto rx=0x7f75cc00b4a0 tx=0x7f75cc00b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.589+0000 7f75db400640 1 -- 172.21.15.86:0/1984433760 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75cc014070 con 0x7f75e4158570 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.589+0000 7f75db400640 1 -- 172.21.15.86:0/1984433760 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75cc010610 con 0x7f75e4158570 2024-03-06T07:34:23.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.589+0000 7f75db400640 1 -- 172.21.15.86:0/1984433760 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75cc010a00 con 0x7f75e4158570 2024-03-06T07:34:23.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.589+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f75e415cc90 con 0x7f75e4158570 2024-03-06T07:34:23.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.589+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f75e415d250 con 0x7f75e4158570 2024-03-06T07:34:23.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.591+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f75e404f330 con 0x7f75e4158570 2024-03-06T07:34:23.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.591+0000 7f75db400640 1 -- 172.21.15.86:0/1984433760 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f75cc017070 con 0x7f75e4158570 2024-03-06T07:34:23.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.592+0000 7f75db400640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75c006b500 0x7f75c006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:23.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.592+0000 7f75db400640 1 -- 172.21.15.86:0/1984433760 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f75cc089d40 con 0x7f75e4158570 2024-03-06T07:34:23.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.592+0000 7f75e3400640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75c006b500 0x7f75c006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:23.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.593+0000 7f75e3400640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75c006b500 0x7f75c006d9c0 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f75d4038000 tx=0x7f75d403b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.601+0000 7f75db400640 1 -- 172.21.15.86:0/1984433760 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f75cc058aa0 con 0x7f75e4158570 2024-03-06T07:34:23.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.898+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f75e40ced50 con 0x7f75e4158570 2024-03-06T07:34:23.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.898+0000 7f75db400640 1 -- 172.21.15.86:0/1984433760 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v116) v1 ==== 93+0+10 (secure 0 0 0) 0x7f75cc0583a0 con 0x7f75e4158570 2024-03-06T07:34:23.901 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75c006b500 msgr2=0x7f75c006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75c006b500 0x7f75c006d9c0 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f75d4038000 tx=0x7f75d403b040 comp rx=0 tx=0).stop 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 msgr2=0x7f75e415a7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e415a7c0 secure :-1 s=READY pgs=1402 cs=0 l=1 rev1=1 crypto rx=0x7f75cc00b4a0 tx=0x7f75cc00b960 comp rx=0 tx=0).stop 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 shutdown_connections 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75c006b500 0x7f75c006d9c0 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75e4158950 0x7f75e415da50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75e4158570 0x7f75e415a7c0 unknown :-1 s=CLOSED pgs=1402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 --2- 172.21.15.86:0/1984433760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75e40aebf0 0x7f75e415c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:23.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 >> 172.21.15.86:0/1984433760 conn(0x7f75e406afc0 msgr2=0x7f75e40cd830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:23.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 shutdown_connections 2024-03-06T07:34:23.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:23.901+0000 7f75e8a00640 1 -- 172.21.15.86:0/1984433760 wait complete. 2024-03-06T07:34:23.929 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:34:24.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.135+0000 7fdb18600640 1 Processor -- start 2024-03-06T07:34:24.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.135+0000 7fdb18600640 1 -- start start 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb18600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdb100d07f0 0x7fdb100d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb18600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fdb100d1bf0 0x7fdb100cfa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb18600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdb100b3730 con 0x7fdb1015e450 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb18600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdb100d4930 con 0x7fdb100d07f0 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb18600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdb10067b20 con 0x7fdb100d1bf0 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb15c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdb100d07f0 0x7fdb100d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb15200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fdb1015e450 0x7fdb100b4480 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34150/0 (socket says 172.21.15.86:34150) 2024-03-06T07:34:24.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.137+0000 7fdb15200640 1 -- 172.21.15.86:0/2509231468 learned_addr learned my addr 172.21.15.86:0/2509231468 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:24.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.138+0000 7fdb0fe00640 1 -- 172.21.15.86:0/2509231468 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 131680934 0 0) 0x7fdb100b3730 con 0x7fdb1015e450 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.138+0000 7fdb15c00640 1 -- 172.21.15.86:0/2509231468 >> v2:172.21.15.194:3300/0 conn(0x7fdb100d1bf0 msgr2=0x7fdb100cfa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.138+0000 7fdb16600640 1 --2- 172.21.15.86:0/2509231468 >> v2:172.21.15.194:3300/0 conn(0x7fdb100d1bf0 0x7fdb100cfa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.138+0000 7fdb15c00640 1 --2- 172.21.15.86:0/2509231468 >> v2:172.21.15.194:3300/0 conn(0x7fdb100d1bf0 0x7fdb100cfa30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.138+0000 7fdb15c00640 1 -- 172.21.15.86:0/2509231468 >> v1:172.21.15.86:6789/0 conn(0x7fdb1015e450 legacy=0x7fdb100b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.138+0000 7fdb15c00640 1 -- 172.21.15.86:0/2509231468 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb100d1110 con 0x7fdb100d07f0 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.138+0000 7fdb16600640 1 --2- 172.21.15.86:0/2509231468 >> v2:172.21.15.194:3300/0 conn(0x7fdb100d1bf0 0x7fdb100cfa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.139+0000 7fdb15c00640 1 --2- 172.21.15.86:0/2509231468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb100d07f0 0x7fdb100d0bd0 secure :-1 s=READY pgs=1404 cs=0 l=1 rev1=1 crypto rx=0x7fdb100cae50 tx=0x7fdb0402be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=36c60a55ac89ba58 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.139+0000 7fdb0fe00640 1 -- 172.21.15.86:0/2509231468 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb04039070 con 0x7fdb100d07f0 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.139+0000 7fdb0fe00640 1 -- 172.21.15.86:0/2509231468 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdb0402cbe0 con 0x7fdb100d07f0 2024-03-06T07:34:24.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.139+0000 7fdb0fe00640 1 -- 172.21.15.86:0/2509231468 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb04034440 con 0x7fdb100d07f0 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 -- 172.21.15.86:0/2509231468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb100d07f0 msgr2=0x7fdb100d0bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 --2- 172.21.15.86:0/2509231468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb100d07f0 0x7fdb100d0bd0 secure :-1 s=READY pgs=1404 cs=0 l=1 rev1=1 crypto rx=0x7fdb100cae50 tx=0x7fdb0402be30 comp rx=0 tx=0).stop 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 -- 172.21.15.86:0/2509231468 shutdown_connections 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 --2- 172.21.15.86:0/2509231468 >> v2:172.21.15.194:3300/0 conn(0x7fdb100d1bf0 0x7fdb100cfa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 --2- 172.21.15.86:0/2509231468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb100d07f0 0x7fdb100d0bd0 unknown :-1 s=CLOSED pgs=1404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 -- 172.21.15.86:0/2509231468 >> 172.21.15.86:0/2509231468 conn(0x7fdb100d3e10 msgr2=0x7fdb100d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 -- 172.21.15.86:0/2509231468 shutdown_connections 2024-03-06T07:34:24.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.140+0000 7fdb18600640 1 -- 172.21.15.86:0/2509231468 wait complete. 2024-03-06T07:34:24.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.142+0000 7fdb18600640 1 Processor -- start 2024-03-06T07:34:24.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.142+0000 7fdb18600640 1 -- start start 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb18600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdb100d07f0 0x7fdb100b1f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb18600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdb100d1bf0 0x7fdb100b6a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb18600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb1015e450 0x7fdb1015b1e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb18600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb100b3c80 con 0x7fdb1015e450 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb18600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb100b39a0 con 0x7fdb100d1bf0 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb18600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdb100b3f60 con 0x7fdb100d07f0 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb16600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb1015e450 0x7fdb1015b1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb16600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb1015e450 0x7fdb1015b1e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48354/0 (socket says 172.21.15.86:48354) 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb16600640 1 -- 172.21.15.86:0/1086576004 learned_addr learned my addr 172.21.15.86:0/1086576004 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb16600640 1 -- 172.21.15.86:0/1086576004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdb100d07f0 msgr2=0x7fdb100b1f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.143+0000 7fdb16600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdb100d07f0 0x7fdb100b1f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb16600640 1 -- 172.21.15.86:0/1086576004 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdb100d1bf0 msgr2=0x7fdb100b6a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb16600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdb100d1bf0 0x7fdb100b6a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb16600640 1 -- 172.21.15.86:0/1086576004 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb100b7480 con 0x7fdb1015e450 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb16600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb1015e450 0x7fdb1015b1e0 secure :-1 s=READY pgs=1405 cs=0 l=1 rev1=1 crypto rx=0x7fdb0800c7f0 tx=0x7fdb0800ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb0d600640 1 -- 172.21.15.86:0/1086576004 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb08015070 con 0x7fdb1015e450 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb0d600640 1 -- 172.21.15.86:0/1086576004 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdb08004240 con 0x7fdb1015e450 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb0d600640 1 -- 172.21.15.86:0/1086576004 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb08007620 con 0x7fdb1015e450 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdb100b7760 con 0x7fdb1015e450 2024-03-06T07:34:24.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.144+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdb100cb2f0 con 0x7fdb1015e450 2024-03-06T07:34:24.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.145+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdb1004f330 con 0x7fdb1015e450 2024-03-06T07:34:24.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.146+0000 7fdb0d600640 1 -- 172.21.15.86:0/1086576004 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdb080079d0 con 0x7fdb1015e450 2024-03-06T07:34:24.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.147+0000 7fdb0d600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdaf006b4d0 0x7fdaf006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.147+0000 7fdb0d600640 1 -- 172.21.15.86:0/1086576004 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fdb0808a370 con 0x7fdb1015e450 2024-03-06T07:34:24.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.148+0000 7fdb15c00640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdaf006b4d0 0x7fdaf006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.148+0000 7fdb15c00640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdaf006b4d0 0x7fdaf006d990 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fdb04036000 tx=0x7fdb0402e000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:24.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.156+0000 7fdb0d600640 1 -- 172.21.15.86:0/1086576004 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdb080591f0 con 0x7fdb1015e450 2024-03-06T07:34:24.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.352+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fdb100d21d0 con 0x7fdb1015e450 2024-03-06T07:34:24.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.353+0000 7fdb0d600640 1 -- 172.21.15.86:0/1086576004 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v116) v1 ==== 104+0+11 (secure 0 0 0) 0x7fdb08058af0 con 0x7fdb1015e450 2024-03-06T07:34:24.356 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:34:24.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.355+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdaf006b4d0 msgr2=0x7fdaf006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.355+0000 7fdb18600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdaf006b4d0 0x7fdaf006d990 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fdb04036000 tx=0x7fdb0402e000 comp rx=0 tx=0).stop 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.355+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb1015e450 msgr2=0x7fdb1015b1e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.355+0000 7fdb18600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb1015e450 0x7fdb1015b1e0 secure :-1 s=READY pgs=1405 cs=0 l=1 rev1=1 crypto rx=0x7fdb0800c7f0 tx=0x7fdb0800ccb0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 shutdown_connections 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdaf006b4d0 0x7fdaf006d990 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdb1015e450 0x7fdb1015b1e0 unknown :-1 s=CLOSED pgs=1405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdb100d1bf0 0x7fdb100b6a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 --2- 172.21.15.86:0/1086576004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdb100d07f0 0x7fdb100b1f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 >> 172.21.15.86:0/1086576004 conn(0x7fdb100d3e10 msgr2=0x7fdb1006d280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 shutdown_connections 2024-03-06T07:34:24.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.356+0000 7fdb18600640 1 -- 172.21.15.86:0/1086576004 wait complete. 2024-03-06T07:34:24.377 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:34:24.495 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:34:24.502 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:34:24.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.568+0000 7f1055600640 1 Processor -- start 2024-03-06T07:34:24.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.568+0000 7f1055600640 1 -- start start 2024-03-06T07:34:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.569+0000 7f1055600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f10500b1c50 0x7f1050168fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.569+0000 7f1055600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f10500b32f0 con 0x7f10500b2d80 2024-03-06T07:34:24.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.569+0000 7f1055600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f10500d0f70 con 0x7f10501618e0 2024-03-06T07:34:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.569+0000 7f1055600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f10500d0d20 con 0x7f10500b1c50 2024-03-06T07:34:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.570+0000 7f104e000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f10500b2d80 0x7f10500cf1b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34160/0 (socket says 172.21.15.86:34160) 2024-03-06T07:34:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.570+0000 7f104e000640 1 -- 172.21.15.86:0/3149365947 learned_addr learned my addr 172.21.15.86:0/3149365947 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.570+0000 7f104f400640 1 --2- 172.21.15.86:0/3149365947 >> v2:172.21.15.86:3300/0 conn(0x7f10500b1c50 0x7f1050168fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.570+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3549451599 0 0) 0x7f10500b32f0 con 0x7f10500b2d80 2024-03-06T07:34:24.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.570+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1034003520 con 0x7f10500b2d80 2024-03-06T07:34:24.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.570+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3835726461 0 0) 0x7f10500d0f70 con 0x7f10501618e0 2024-03-06T07:34:24.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.570+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f10500b32f0 con 0x7f10501618e0 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1209826800 0 0) 0x7f1034003520 con 0x7f10500b2d80 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 >> v2:172.21.15.86:3300/0 conn(0x7f10500b1c50 msgr2=0x7f1050168fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 --2- 172.21.15.86:0/3149365947 >> v2:172.21.15.86:3300/0 conn(0x7f10500b1c50 0x7f1050168fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 >> v1:172.21.15.194:6789/0 conn(0x7f10501618e0 legacy=0x7f10500ceaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10500d1250 con 0x7f10500b2d80 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f103c0015f0 con 0x7f10500b2d80 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 >> v1:172.21.15.86:6789/0 conn(0x7f10500b2d80 legacy=0x7f10500cf1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1034003b90 0x7f1034003f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034004800 0x7f1034004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10340051c0 0x7f103400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10500d0d20 con 0x7f1034003b90 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f103400c090 con 0x7f1034004800 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f103400c290 con 0x7f10340051c0 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104ea00640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1034003b90 0x7f1034003f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104e000640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034004800 0x7f1034004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104ea00640 1 -- 172.21.15.86:0/3149365947 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10340051c0 msgr2=0x7f103400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104ea00640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10340051c0 0x7f103400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104ea00640 1 -- 172.21.15.86:0/3149365947 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034004800 msgr2=0x7f1034004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104ea00640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034004800 0x7f1034004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104ea00640 1 -- 172.21.15.86:0/3149365947 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1034003520 con 0x7f1034003b90 2024-03-06T07:34:24.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.571+0000 7f104f400640 1 -- 172.21.15.86:0/3149365947 reap_dead start 2024-03-06T07:34:24.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.572+0000 7f104ea00640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1034003b90 0x7f1034003f90 secure :-1 s=READY pgs=1407 cs=0 l=1 rev1=1 crypto rx=0x7f10400018d0 tx=0x7f1034005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=42b5eba6602d5b02 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:24.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.573+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1040037070 con 0x7f1034003b90 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.573+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f10400052d0 con 0x7f1034003b90 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.573+0000 7f104d600640 1 -- 172.21.15.86:0/3149365947 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10400014f0 con 0x7f1034003b90 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.574+0000 7f1055600640 1 -- 172.21.15.86:0/3149365947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1034003b90 msgr2=0x7f1034003f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.574+0000 7f1055600640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1034003b90 0x7f1034003f90 secure :-1 s=READY pgs=1407 cs=0 l=1 rev1=1 crypto rx=0x7f10400018d0 tx=0x7f1034005830 comp rx=0 tx=0).stop 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.574+0000 7f1055600640 1 -- 172.21.15.86:0/3149365947 shutdown_connections 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.574+0000 7f1055600640 1 --2- 172.21.15.86:0/3149365947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1034003b90 0x7f1034003f90 unknown :-1 s=CLOSED pgs=1407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.574+0000 7f1055600640 1 -- 172.21.15.86:0/3149365947 >> 172.21.15.86:0/3149365947 conn(0x7f10500d3e90 msgr2=0x7f10500d42a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.574+0000 7f1055600640 1 -- 172.21.15.86:0/3149365947 shutdown_connections 2024-03-06T07:34:24.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.574+0000 7f1055600640 1 -- 172.21.15.86:0/3149365947 wait complete. 2024-03-06T07:34:24.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.576+0000 7f1055600640 1 Processor -- start 2024-03-06T07:34:24.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.576+0000 7f1055600640 1 -- start start 2024-03-06T07:34:24.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f1055600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034003b90 0x7f10500ca2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f1055600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10501544c0 0x7f10500b7eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f1055600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f105015f520 0x7f105015f9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f1055600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10500b0c40 con 0x7f105015f520 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f1055600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1050161cc0 con 0x7f1034003b90 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f1055600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10500b0dc0 con 0x7f10501544c0 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f104f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f105015f520 0x7f105015f9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f104f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f105015f520 0x7f105015f9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48372/0 (socket says 172.21.15.86:48372) 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f104f400640 1 -- 172.21.15.86:0/252577738 learned_addr learned my addr 172.21.15.86:0/252577738 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f104ea00640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034003b90 0x7f10500ca2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.577+0000 7f104e000640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10501544c0 0x7f10500b7eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f104f400640 1 -- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10501544c0 msgr2=0x7f10500b7eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f104f400640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10501544c0 0x7f10500b7eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f104f400640 1 -- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034003b90 msgr2=0x7f10500ca2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f104f400640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034003b90 0x7f10500ca2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f104f400640 1 -- 172.21.15.86:0/252577738 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10500b9b60 con 0x7f105015f520 2024-03-06T07:34:24.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f104f400640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f105015f520 0x7f105015f9a0 secure :-1 s=READY pgs=1408 cs=0 l=1 rev1=1 crypto rx=0x7f10500d3e60 tx=0x7f104800b480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:24.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f1046a00640 1 -- 172.21.15.86:0/252577738 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f104800ba80 con 0x7f105015f520 2024-03-06T07:34:24.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f1046a00640 1 -- 172.21.15.86:0/252577738 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f104801a070 con 0x7f105015f520 2024-03-06T07:34:24.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10500cb100 con 0x7f105015f520 2024-03-06T07:34:24.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f1046a00640 1 -- 172.21.15.86:0/252577738 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f104800be40 con 0x7f105015f520 2024-03-06T07:34:24.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.578+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10500cb6c0 con 0x7f105015f520 2024-03-06T07:34:24.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.580+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f105004f330 con 0x7f105015f520 2024-03-06T07:34:24.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.581+0000 7f1046a00640 1 -- 172.21.15.86:0/252577738 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1048013070 con 0x7f105015f520 2024-03-06T07:34:24.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.582+0000 7f1046a00640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103006b500 0x7f103006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:24.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.582+0000 7f1046a00640 1 -- 172.21.15.86:0/252577738 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f104808a370 con 0x7f105015f520 2024-03-06T07:34:24.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.583+0000 7f104ea00640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103006b500 0x7f103006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:24.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.584+0000 7f104ea00640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103006b500 0x7f103006d9c0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f1040034000 tx=0x7f1040031000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:24.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.587+0000 7f1046a00640 1 -- 172.21.15.86:0/252577738 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1048058a70 con 0x7f105015f520 2024-03-06T07:34:24.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.783+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f10501548a0 con 0x7f105015f520 2024-03-06T07:34:24.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.784+0000 7f1046a00640 1 -- 172.21.15.86:0/252577738 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v116) v1 ==== 100+0+11 (secure 0 0 0) 0x7f104805cb10 con 0x7f105015f520 2024-03-06T07:34:24.786 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:34:24.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103006b500 msgr2=0x7f103006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103006b500 0x7f103006d9c0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f1040034000 tx=0x7f1040031000 comp rx=0 tx=0).stop 2024-03-06T07:34:24.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f105015f520 msgr2=0x7f105015f9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:24.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f105015f520 0x7f105015f9a0 secure :-1 s=READY pgs=1408 cs=0 l=1 rev1=1 crypto rx=0x7f10500d3e60 tx=0x7f104800b480 comp rx=0 tx=0).stop 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 shutdown_connections 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f103006b500 0x7f103006d9c0 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f105015f520 0x7f105015f9a0 unknown :-1 s=CLOSED pgs=1408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f10501544c0 0x7f10500b7eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 --2- 172.21.15.86:0/252577738 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1034003b90 0x7f10500ca2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 >> 172.21.15.86:0/252577738 conn(0x7f10500d3e90 msgr2=0x7f10500d1d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 shutdown_connections 2024-03-06T07:34:24.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:24.786+0000 7f1055600640 1 -- 172.21.15.86:0/252577738 wait complete. 2024-03-06T07:34:24.807 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:34:25.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.010+0000 7fde99400640 1 Processor -- start 2024-03-06T07:34:25.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.011+0000 7fde99400640 1 -- start start 2024-03-06T07:34:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.012+0000 7fde99400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde9415f510 0x7fde940b1410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.012+0000 7fde99400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde940b32c0 con 0x7fde940af640 2024-03-06T07:34:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.012+0000 7fde99400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde9415f8f0 con 0x7fde940d2830 2024-03-06T07:34:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.012+0000 7fde99400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fde940aeb60 con 0x7fde9415f510 2024-03-06T07:34:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.012+0000 7fde92800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde9415f510 0x7fde940b1410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.012+0000 7fde92800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde9415f510 0x7fde940b1410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48382/0 (socket says 172.21.15.86:48382) 2024-03-06T07:34:25.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.012+0000 7fde92800640 1 -- 172.21.15.86:0/3231364870 learned_addr learned my addr 172.21.15.86:0/3231364870 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:25.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.013+0000 7fde92800640 1 -- 172.21.15.86:0/3231364870 >> v1:172.21.15.194:6789/0 conn(0x7fde940d2830 legacy=0x7fde940b1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:25.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.013+0000 7fde92800640 1 -- 172.21.15.86:0/3231364870 >> v1:172.21.15.86:6789/0 conn(0x7fde940af640 legacy=0x7fde940afab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.013+0000 7fde92800640 1 -- 172.21.15.86:0/3231364870 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde940aee40 con 0x7fde9415f510 2024-03-06T07:34:25.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.014+0000 7fde92800640 1 --2- 172.21.15.86:0/3231364870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde940b1410 secure :-1 s=READY pgs=1410 cs=0 l=1 rev1=1 crypto rx=0x7fde7c009870 tx=0x7fde7c02b990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9810c53989c7d6a5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:25.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.014+0000 7fde91e00640 1 -- 172.21.15.86:0/3231364870 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde7c034070 con 0x7fde9415f510 2024-03-06T07:34:25.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.014+0000 7fde91e00640 1 -- 172.21.15.86:0/3231364870 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde7c030850 con 0x7fde9415f510 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.014+0000 7fde91e00640 1 -- 172.21.15.86:0/3231364870 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde7c030c20 con 0x7fde9415f510 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.015+0000 7fde99400640 1 -- 172.21.15.86:0/3231364870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 msgr2=0x7fde940b1410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.015+0000 7fde99400640 1 --2- 172.21.15.86:0/3231364870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde940b1410 secure :-1 s=READY pgs=1410 cs=0 l=1 rev1=1 crypto rx=0x7fde7c009870 tx=0x7fde7c02b990 comp rx=0 tx=0).stop 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.015+0000 7fde99400640 1 -- 172.21.15.86:0/3231364870 shutdown_connections 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.015+0000 7fde99400640 1 --2- 172.21.15.86:0/3231364870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde940b1410 unknown :-1 s=CLOSED pgs=1410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.015+0000 7fde99400640 1 -- 172.21.15.86:0/3231364870 >> 172.21.15.86:0/3231364870 conn(0x7fde940d1d10 msgr2=0x7fde940d2120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.015+0000 7fde99400640 1 -- 172.21.15.86:0/3231364870 shutdown_connections 2024-03-06T07:34:25.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.015+0000 7fde99400640 1 -- 172.21.15.86:0/3231364870 wait complete. 2024-03-06T07:34:25.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.017+0000 7fde99400640 1 Processor -- start 2024-03-06T07:34:25.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.017+0000 7fde99400640 1 -- start start 2024-03-06T07:34:25.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde99400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde940af640 0x7fde940cc710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde99400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde940d2830 0x7fde94070010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde99400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde94070550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde99400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde940b0fc0 con 0x7fde9415f510 2024-03-06T07:34:25.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde99400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde940b3560 con 0x7fde940af640 2024-03-06T07:34:25.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde99400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde940b1140 con 0x7fde940d2830 2024-03-06T07:34:25.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde93200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde94070550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde93200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde94070550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48398/0 (socket says 172.21.15.86:48398) 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.018+0000 7fde93200640 1 -- 172.21.15.86:0/3658539842 learned_addr learned my addr 172.21.15.86:0/3658539842 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde92800640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde940af640 0x7fde940cc710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde89e00640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde940d2830 0x7fde94070010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde93200640 1 -- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde940d2830 msgr2=0x7fde94070010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde93200640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde940d2830 0x7fde94070010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde93200640 1 -- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde940af640 msgr2=0x7fde940cc710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde93200640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde940af640 0x7fde940cc710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde93200640 1 -- 172.21.15.86:0/3658539842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde940cd0f0 con 0x7fde9415f510 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde93200640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde94070550 secure :-1 s=READY pgs=1411 cs=0 l=1 rev1=1 crypto rx=0x7fde8c00c730 tx=0x7fde8c00cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde8b400640 1 -- 172.21.15.86:0/3658539842 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde8c003b90 con 0x7fde9415f510 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde8b400640 1 -- 172.21.15.86:0/3658539842 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fde8c003d30 con 0x7fde9415f510 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde8b400640 1 -- 172.21.15.86:0/3658539842 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde8c00aa90 con 0x7fde9415f510 2024-03-06T07:34:25.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.019+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde940cd3d0 con 0x7fde9415f510 2024-03-06T07:34:25.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.020+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde940cd990 con 0x7fde9415f510 2024-03-06T07:34:25.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.021+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde940d0d10 con 0x7fde9415f510 2024-03-06T07:34:25.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.022+0000 7fde8b400640 1 -- 172.21.15.86:0/3658539842 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fde8c00ac30 con 0x7fde9415f510 2024-03-06T07:34:25.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.022+0000 7fde8b400640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde7006b4d0 0x7fde7006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.023+0000 7fde8b400640 1 -- 172.21.15.86:0/3658539842 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fde8c08a220 con 0x7fde9415f510 2024-03-06T07:34:25.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.023+0000 7fde92800640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde7006b4d0 0x7fde7006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.023+0000 7fde92800640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde7006b4d0 0x7fde7006d990 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7fde7c036000 tx=0x7fde7c03b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:25.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.027+0000 7fde8b400640 1 -- 172.21.15.86:0/3658539842 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fde940d0d10 con 0x7fde9415f510 2024-03-06T07:34:25.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.233+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fde9404f330 con 0x7fde9415f510 2024-03-06T07:34:25.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.234+0000 7fde8b400640 1 -- 172.21.15.86:0/3658539842 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 2 entries v116) v1 ==== 80+0+120 (secure 0 0 0) 0x7fde8c058f70 con 0x7fde9415f510 2024-03-06T07:34:25.237 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.15.86:6837/221976790 2024-03-07T07:33:20.538234+0000 2024-03-06T07:34:25.237 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.15.86:6836/221976790 2024-03-07T07:33:20.538234+0000 2024-03-06T07:34:25.237 INFO:teuthology.orchestra.run.smithi086.stderr:listed 2 entries 2024-03-06T07:34:25.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde7006b4d0 msgr2=0x7fde7006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde7006b4d0 0x7fde7006d990 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7fde7c036000 tx=0x7fde7c03b040 comp rx=0 tx=0).stop 2024-03-06T07:34:25.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 msgr2=0x7fde94070550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde94070550 secure :-1 s=READY pgs=1411 cs=0 l=1 rev1=1 crypto rx=0x7fde8c00c730 tx=0x7fde8c00cbf0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 shutdown_connections 2024-03-06T07:34:25.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fde7006b4d0 0x7fde7006d990 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde9415f510 0x7fde94070550 unknown :-1 s=CLOSED pgs=1411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde940d2830 0x7fde94070010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 --2- 172.21.15.86:0/3658539842 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde940af640 0x7fde940cc710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 >> 172.21.15.86:0/3658539842 conn(0x7fde940d1d10 msgr2=0x7fde940c7850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:25.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 shutdown_connections 2024-03-06T07:34:25.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.237+0000 7fde99400640 1 -- 172.21.15.86:0/3658539842 wait complete. 2024-03-06T07:34:25.256 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:34:25.256 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:25.257 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:34:25.344 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi086.front.sepia.ceph.com 2024-03-06T07:34:25.344 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:25.345 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-06T07:34:25.389 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:34:25.389 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:25.446 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:34:25.447 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:34:25.447 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:25.503 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:34:25.504 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:34:25.504 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:25.562 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:34:25.562 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:25.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.771+0000 7fdc5ac00640 1 Processor -- start 2024-03-06T07:34:25.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.771+0000 7fdc5ac00640 1 -- start start 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.772+0000 7fdc5ac00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fdc5415e160 0x7fdc54162620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.772+0000 7fdc5ac00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdc540af820 0x7fdc540b33b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.772+0000 7fdc5ac00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdc54162b60 con 0x7fdc540b38f0 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc5ac00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdc5406b0f0 con 0x7fdc540af820 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc5ac00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdc540b5620 con 0x7fdc5415e160 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc53400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdc540af820 0x7fdc540b33b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc53400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fdc540af820 0x7fdc540b33b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48410/0 (socket says 172.21.15.86:48410) 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc53400640 1 -- 172.21.15.86:0/1792870476 learned_addr learned my addr 172.21.15.86:0/1792870476 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:25.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc53e00640 1 --2- 172.21.15.86:0/1792870476 >> v2:172.21.15.194:3300/0 conn(0x7fdc5415e160 0x7fdc54162620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc53400640 1 -- 172.21.15.86:0/1792870476 >> v2:172.21.15.194:3300/0 conn(0x7fdc5415e160 msgr2=0x7fdc54162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:25.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.773+0000 7fdc53400640 1 --2- 172.21.15.86:0/1792870476 >> v2:172.21.15.194:3300/0 conn(0x7fdc5415e160 0x7fdc54162620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.774+0000 7fdc53400640 1 -- 172.21.15.86:0/1792870476 >> v1:172.21.15.194:6789/0 conn(0x7fdc540b38f0 legacy=0x7fdc5406d3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:25.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.774+0000 7fdc53400640 1 -- 172.21.15.86:0/1792870476 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdc540b3da0 con 0x7fdc540af820 2024-03-06T07:34:25.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.775+0000 7fdc53400640 1 --2- 172.21.15.86:0/1792870476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc540af820 0x7fdc540b33b0 secure :-1 s=READY pgs=1412 cs=0 l=1 rev1=1 crypto rx=0x7fdc540d0310 tx=0x7fdc4002b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d154108886c6082a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:25.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.775+0000 7fdc52a00640 1 -- 172.21.15.86:0/1792870476 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc4002e070 con 0x7fdc540af820 2024-03-06T07:34:25.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.775+0000 7fdc52a00640 1 -- 172.21.15.86:0/1792870476 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdc40037070 con 0x7fdc540af820 2024-03-06T07:34:25.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.775+0000 7fdc52a00640 1 -- 172.21.15.86:0/1792870476 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc40031a00 con 0x7fdc540af820 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 -- 172.21.15.86:0/1792870476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc540af820 msgr2=0x7fdc540b33b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/1792870476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc540af820 0x7fdc540b33b0 secure :-1 s=READY pgs=1412 cs=0 l=1 rev1=1 crypto rx=0x7fdc540d0310 tx=0x7fdc4002b880 comp rx=0 tx=0).stop 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 -- 172.21.15.86:0/1792870476 shutdown_connections 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/1792870476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc540af820 0x7fdc540b33b0 unknown :-1 s=CLOSED pgs=1412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/1792870476 >> v2:172.21.15.194:3300/0 conn(0x7fdc5415e160 0x7fdc54162620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 -- 172.21.15.86:0/1792870476 >> 172.21.15.86:0/1792870476 conn(0x7fdc540d0340 msgr2=0x7fdc540d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 -- 172.21.15.86:0/1792870476 shutdown_connections 2024-03-06T07:34:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.776+0000 7fdc5ac00640 1 -- 172.21.15.86:0/1792870476 wait complete. 2024-03-06T07:34:25.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.778+0000 7fdc5ac00640 1 Processor -- start 2024-03-06T07:34:25.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.778+0000 7fdc5ac00640 1 -- start start 2024-03-06T07:34:25.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc5ac00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdc540af820 0x7fdc54145f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc5ac00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdc540b38f0 0x7fdc540ccd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc5ac00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc5415e160 0x7fdc541600a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc5ac00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc5406ec30 con 0x7fdc5415e160 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc5ac00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc540afce0 con 0x7fdc540af820 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc5ac00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdc5406ef10 con 0x7fdc540b38f0 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc58c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc5415e160 0x7fdc541600a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc58c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc5415e160 0x7fdc541600a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:48412/0 (socket says 172.21.15.86:48412) 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.779+0000 7fdc58c00640 1 -- 172.21.15.86:0/2014466648 learned_addr learned my addr 172.21.15.86:0/2014466648 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc53400640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdc540b38f0 0x7fdc540ccd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc53e00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdc540af820 0x7fdc54145f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc58c00640 1 -- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdc540b38f0 msgr2=0x7fdc540ccd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc58c00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdc540b38f0 0x7fdc540ccd30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc58c00640 1 -- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdc540af820 msgr2=0x7fdc54145f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc58c00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdc540af820 0x7fdc54145f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:25.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc58c00640 1 -- 172.21.15.86:0/2014466648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdc540cd270 con 0x7fdc5415e160 2024-03-06T07:34:25.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc58c00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc5415e160 0x7fdc541600a0 secure :-1 s=READY pgs=1413 cs=0 l=1 rev1=1 crypto rx=0x7fdc4c00c780 tx=0x7fdc4c00cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:25.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc4be00640 1 -- 172.21.15.86:0/2014466648 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc4c004b10 con 0x7fdc5415e160 2024-03-06T07:34:25.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc4be00640 1 -- 172.21.15.86:0/2014466648 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdc4c004cb0 con 0x7fdc5415e160 2024-03-06T07:34:25.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.780+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdc5415ea90 con 0x7fdc5415e160 2024-03-06T07:34:25.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.781+0000 7fdc4be00640 1 -- 172.21.15.86:0/2014466648 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc4c00aa90 con 0x7fdc5415e160 2024-03-06T07:34:25.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.781+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdc54161290 con 0x7fdc5415e160 2024-03-06T07:34:25.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.782+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdc54074ae0 con 0x7fdc5415e160 2024-03-06T07:34:25.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.783+0000 7fdc4be00640 1 -- 172.21.15.86:0/2014466648 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdc4c00ac30 con 0x7fdc5415e160 2024-03-06T07:34:25.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.783+0000 7fdc4be00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdc3406b4d0 0x7fdc3406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:25.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.784+0000 7fdc4be00640 1 -- 172.21.15.86:0/2014466648 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7fdc4c08a510 con 0x7fdc5415e160 2024-03-06T07:34:25.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.784+0000 7fdc53e00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdc3406b4d0 0x7fdc3406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:25.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.784+0000 7fdc53e00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdc3406b4d0 0x7fdc3406d990 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7fdc44006fd0 tx=0x7fdc44008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:25.788+0000 7fdc4be00640 1 -- 172.21.15.86:0/2014466648 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdc4c020070 con 0x7fdc5415e160 2024-03-06T07:34:26.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.025+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fdc541614d0 con 0x7fdc5415e160 2024-03-06T07:34:26.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.026+0000 7fdc4be00640 1 -- 172.21.15.86:0/2014466648 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7fdc4c059260 con 0x7fdc5415e160 2024-03-06T07:34:26.029 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:26.029 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:26.029 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdc3406b4d0 msgr2=0x7fdc3406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdc3406b4d0 0x7fdc3406d990 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7fdc44006fd0 tx=0x7fdc44008040 comp rx=0 tx=0).stop 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc5415e160 msgr2=0x7fdc541600a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc5415e160 0x7fdc541600a0 secure :-1 s=READY pgs=1413 cs=0 l=1 rev1=1 crypto rx=0x7fdc4c00c780 tx=0x7fdc4c00cc40 comp rx=0 tx=0).stop 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 shutdown_connections 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdc3406b4d0 0x7fdc3406d990 unknown :-1 s=CLOSED pgs=604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdc5415e160 0x7fdc541600a0 unknown :-1 s=CLOSED pgs=1413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdc540b38f0 0x7fdc540ccd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 --2- 172.21.15.86:0/2014466648 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdc540af820 0x7fdc54145f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 >> 172.21.15.86:0/2014466648 conn(0x7fdc540d0340 msgr2=0x7fdc5406b580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:26.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 shutdown_connections 2024-03-06T07:34:26.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.029+0000 7fdc5ac00640 1 -- 172.21.15.86:0/2014466648 wait complete. 2024-03-06T07:34:26.050 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:26.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.247+0000 7f6a2d600640 1 Processor -- start 2024-03-06T07:34:26.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.247+0000 7f6a2d600640 1 -- start start 2024-03-06T07:34:26.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a2d600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6a2815f9d0 0x7f6a2815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a2d600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6a28070b10 0x7f6a28070f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a2d600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a2806cef0 con 0x7f6a28144c00 2024-03-06T07:34:26.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a2d600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6a2806d1d0 con 0x7f6a2815f9d0 2024-03-06T07:34:26.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a2d600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6a2806d4b0 con 0x7f6a28070b10 2024-03-06T07:34:26.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a1fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6a2815f9d0 0x7f6a2815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a1fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6a2815f9d0 0x7f6a2815fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54622/0 (socket says 172.21.15.86:54622) 2024-03-06T07:34:26.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.249+0000 7f6a1fe00640 1 -- 172.21.15.86:0/4284032494 learned_addr learned my addr 172.21.15.86:0/4284032494 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:26.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.250+0000 7f6a1fe00640 1 -- 172.21.15.86:0/4284032494 >> v2:172.21.15.194:3300/0 conn(0x7f6a28070b10 msgr2=0x7f6a28070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:26.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.250+0000 7f6a1fe00640 1 --2- 172.21.15.86:0/4284032494 >> v2:172.21.15.194:3300/0 conn(0x7f6a28070b10 0x7f6a28070f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.250+0000 7f6a1fe00640 1 -- 172.21.15.86:0/4284032494 >> v1:172.21.15.194:6789/0 conn(0x7f6a28144c00 legacy=0x7f6a2806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:26.253 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.250+0000 7f6a1fe00640 1 -- 172.21.15.86:0/4284032494 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a2806d790 con 0x7f6a2815f9d0 2024-03-06T07:34:26.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.251+0000 7f6a1fe00640 1 --2- 172.21.15.86:0/4284032494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a2815fe60 secure :-1 s=READY pgs=1414 cs=0 l=1 rev1=1 crypto rx=0x7f6a2806e500 tx=0x7f6a1002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ecd6b97056f9f118 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:26.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.251+0000 7f6a26000640 1 -- 172.21.15.86:0/4284032494 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a10035020 con 0x7f6a2815f9d0 2024-03-06T07:34:26.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.251+0000 7f6a26000640 1 -- 172.21.15.86:0/4284032494 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a10031870 con 0x7f6a2815f9d0 2024-03-06T07:34:26.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.251+0000 7f6a26000640 1 -- 172.21.15.86:0/4284032494 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a10031c40 con 0x7f6a2815f9d0 2024-03-06T07:34:26.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.251+0000 7f6a2d600640 1 -- 172.21.15.86:0/4284032494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 msgr2=0x7f6a2815fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.251+0000 7f6a2d600640 1 --2- 172.21.15.86:0/4284032494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a2815fe60 secure :-1 s=READY pgs=1414 cs=0 l=1 rev1=1 crypto rx=0x7f6a2806e500 tx=0x7f6a1002b910 comp rx=0 tx=0).stop 2024-03-06T07:34:26.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.252+0000 7f6a2d600640 1 -- 172.21.15.86:0/4284032494 shutdown_connections 2024-03-06T07:34:26.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.252+0000 7f6a2d600640 1 --2- 172.21.15.86:0/4284032494 >> v2:172.21.15.194:3300/0 conn(0x7f6a28070b10 0x7f6a28070f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.252+0000 7f6a2d600640 1 --2- 172.21.15.86:0/4284032494 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a2815fe60 unknown :-1 s=CLOSED pgs=1414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.252+0000 7f6a2d600640 1 -- 172.21.15.86:0/4284032494 >> 172.21.15.86:0/4284032494 conn(0x7f6a280b1730 msgr2=0x7f6a280b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:26.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.252+0000 7f6a2d600640 1 -- 172.21.15.86:0/4284032494 shutdown_connections 2024-03-06T07:34:26.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.252+0000 7f6a2d600640 1 -- 172.21.15.86:0/4284032494 wait complete. 2024-03-06T07:34:26.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.253+0000 7f6a2d600640 1 Processor -- start 2024-03-06T07:34:26.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.253+0000 7f6a2d600640 1 -- start start 2024-03-06T07:34:26.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.254+0000 7f6a2d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a28070b10 0x7f6a280aed80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.254+0000 7f6a2d600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a28144c00 0x7f6a281e5650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.254+0000 7f6a2d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a28141240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.254+0000 7f6a2d600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a2806ccd0 con 0x7f6a2815f9d0 2024-03-06T07:34:26.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.254+0000 7f6a2d600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a2806cb50 con 0x7f6a28144c00 2024-03-06T07:34:26.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.254+0000 7f6a2d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a2806df90 con 0x7f6a28070b10 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.254+0000 7f6a27400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a28141240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a28141240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54634/0 (socket says 172.21.15.86:54634) 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 -- 172.21.15.86:0/1463030077 learned_addr learned my addr 172.21.15.86:0/1463030077 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a1fe00640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a28144c00 0x7f6a281e5650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a26a00640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a28070b10 0x7f6a280aed80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 -- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a28070b10 msgr2=0x7f6a280aed80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a28070b10 0x7f6a280aed80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 -- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a28144c00 msgr2=0x7f6a281e5650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a28144c00 0x7f6a281e5650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 -- 172.21.15.86:0/1463030077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a281e5df0 con 0x7f6a2815f9d0 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a26a00640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a28070b10 0x7f6a280aed80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.255+0000 7f6a27400640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a28141240 secure :-1 s=READY pgs=1415 cs=0 l=1 rev1=1 crypto rx=0x7f6a2000b970 tx=0x7f6a2000be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.256+0000 7f6a1ea00640 1 -- 172.21.15.86:0/1463030077 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a20019070 con 0x7f6a2815f9d0 2024-03-06T07:34:26.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.256+0000 7f6a1ea00640 1 -- 172.21.15.86:0/1463030077 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a2000cb60 con 0x7f6a2815f9d0 2024-03-06T07:34:26.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.256+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a281e60d0 con 0x7f6a2815f9d0 2024-03-06T07:34:26.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.256+0000 7f6a1ea00640 1 -- 172.21.15.86:0/1463030077 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a20014440 con 0x7f6a2815f9d0 2024-03-06T07:34:26.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.256+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a281e6690 con 0x7f6a2815f9d0 2024-03-06T07:34:26.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.257+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6a2804f330 con 0x7f6a2815f9d0 2024-03-06T07:34:26.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.258+0000 7f6a1ea00640 1 -- 172.21.15.86:0/1463030077 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6a2001c070 con 0x7f6a2815f9d0 2024-03-06T07:34:26.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.258+0000 7f6a1ea00640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a0406b4d0 0x7f6a0406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.259+0000 7f6a1ea00640 1 -- 172.21.15.86:0/1463030077 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f6a20089fe0 con 0x7f6a2815f9d0 2024-03-06T07:34:26.261 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.259+0000 7f6a26a00640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a0406b4d0 0x7f6a0406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.262 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.259+0000 7f6a26a00640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a0406b4d0 0x7f6a0406d990 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7f6a28143ff0 tx=0x7f6a18009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:26.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.267+0000 7f6a1ea00640 1 -- 172.21.15.86:0/1463030077 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6a20058dc0 con 0x7f6a2815f9d0 2024-03-06T07:34:26.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.484+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6a28072310 con 0x7f6a2815f9d0 2024-03-06T07:34:26.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.486+0000 7f6a1ea00640 1 -- 172.21.15.86:0/1463030077 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v116) v1 ==== 74+0+16920 (secure 0 0 0) 0x7f6a200586c0 con 0x7f6a2815f9d0 2024-03-06T07:34:26.489 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:26.489 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":116,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:55.692842+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"110","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.3f","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/221976790":"2024-03-07T07:33:20.538234+0000","172.21.15.86:6836/221976790":"2024-03-07T07:33:20.538234+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a0406b4d0 msgr2=0x7f6a0406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a0406b4d0 0x7f6a0406d990 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7f6a28143ff0 tx=0x7f6a18009040 comp rx=0 tx=0).stop 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 msgr2=0x7f6a28141240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a28141240 secure :-1 s=READY pgs=1415 cs=0 l=1 rev1=1 crypto rx=0x7f6a2000b970 tx=0x7f6a2000be30 comp rx=0 tx=0).stop 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 shutdown_connections 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6a0406b4d0 0x7f6a0406d990 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6a2815f9d0 0x7f6a28141240 unknown :-1 s=CLOSED pgs=1415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6a28144c00 0x7f6a281e5650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 --2- 172.21.15.86:0/1463030077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6a28070b10 0x7f6a280aed80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 >> 172.21.15.86:0/1463030077 conn(0x7f6a280b1730 msgr2=0x7f6a2815ee30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:26.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 shutdown_connections 2024-03-06T07:34:26.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.489+0000 7f6a2d600640 1 -- 172.21.15.86:0/1463030077 wait complete. 2024-03-06T07:34:26.511 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 16, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:33:03.630003+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '109', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 17, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:33:04.861178+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '110', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:34:26.511 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:34:26.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.717+0000 7f5eb7e00640 1 Processor -- start 2024-03-06T07:34:26.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.717+0000 7f5eb7e00640 1 -- start start 2024-03-06T07:34:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.718+0000 7f5eb7e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5eb806da40 0x7f5eb80ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.718+0000 7f5eb7e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb80aef20 0x7f5eb80ce2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb7e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5eb815c1f0 con 0x7f5eb815be10 2024-03-06T07:34:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb7e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5eb80cd200 con 0x7f5eb806da40 2024-03-06T07:34:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb7e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5eb806cac0 con 0x7f5eb80aef20 2024-03-06T07:34:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb6000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5eb806da40 0x7f5eb80ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb6000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5eb806da40 0x7f5eb80ccc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54640/0 (socket says 172.21.15.86:54640) 2024-03-06T07:34:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb6000640 1 -- 172.21.15.86:0/3607723907 learned_addr learned my addr 172.21.15.86:0/3607723907 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:26.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb7400640 1 --2- 172.21.15.86:0/3607723907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb80aef20 0x7f5eb80ce2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb6000640 1 -- 172.21.15.86:0/3607723907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb80aef20 msgr2=0x7f5eb80ce2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb6000640 1 --2- 172.21.15.86:0/3607723907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb80aef20 0x7f5eb80ce2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb6000640 1 -- 172.21.15.86:0/3607723907 >> v1:172.21.15.194:6789/0 conn(0x7f5eb815be10 legacy=0x7f5eb806d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:26.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.719+0000 7f5eb6000640 1 -- 172.21.15.86:0/3607723907 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5eb806cda0 con 0x7f5eb806da40 2024-03-06T07:34:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.721+0000 7f5eb6000640 1 --2- 172.21.15.86:0/3607723907 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80ccc20 secure :-1 s=READY pgs=1416 cs=0 l=1 rev1=1 crypto rx=0x7f5ea4009870 tx=0x7f5ea402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=72a4dc61006f14e1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.721+0000 7f5eb5600640 1 -- 172.21.15.86:0/3607723907 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ea4035020 con 0x7f5eb806da40 2024-03-06T07:34:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.721+0000 7f5eb5600640 1 -- 172.21.15.86:0/3607723907 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5ea4031640 con 0x7f5eb806da40 2024-03-06T07:34:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.721+0000 7f5eb5600640 1 -- 172.21.15.86:0/3607723907 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ea402d040 con 0x7f5eb806da40 2024-03-06T07:34:26.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.721+0000 7f5eb7e00640 1 -- 172.21.15.86:0/3607723907 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 msgr2=0x7f5eb80ccc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.721+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/3607723907 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80ccc20 secure :-1 s=READY pgs=1416 cs=0 l=1 rev1=1 crypto rx=0x7f5ea4009870 tx=0x7f5ea402b8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.722+0000 7f5eb7e00640 1 -- 172.21.15.86:0/3607723907 shutdown_connections 2024-03-06T07:34:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.722+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/3607723907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb80aef20 0x7f5eb80ce2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.722+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/3607723907 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80ccc20 unknown :-1 s=CLOSED pgs=1416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.722+0000 7f5eb7e00640 1 -- 172.21.15.86:0/3607723907 >> 172.21.15.86:0/3607723907 conn(0x7f5eb806afc0 msgr2=0x7f5eb806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.722+0000 7f5eb7e00640 1 -- 172.21.15.86:0/3607723907 shutdown_connections 2024-03-06T07:34:26.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.722+0000 7f5eb7e00640 1 -- 172.21.15.86:0/3607723907 wait complete. 2024-03-06T07:34:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.724+0000 7f5eb7e00640 1 Processor -- start 2024-03-06T07:34:26.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.725+0000 7f5eb7e00640 1 -- start start 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80b1e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb7e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5eb80aef20 0x7f5eb81569c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb7e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb815be10 0x7f5eb8147050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb7e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5eb80cda90 con 0x7f5eb806da40 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb7e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5eb806d080 con 0x7f5eb80aef20 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb7e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5eb8071ab0 con 0x7f5eb815be10 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80b1e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb6a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80b1e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54654/0 (socket says 172.21.15.86:54654) 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.726+0000 7f5eb6a00640 1 -- 172.21.15.86:0/1178661550 learned_addr learned my addr 172.21.15.86:0/1178661550 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:26.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6000640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5eb80aef20 0x7f5eb81569c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6a00640 1 -- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb815be10 msgr2=0x7f5eb8147050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6a00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb815be10 0x7f5eb8147050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6a00640 1 -- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5eb80aef20 msgr2=0x7f5eb81569c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6a00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5eb80aef20 0x7f5eb81569c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6a00640 1 -- 172.21.15.86:0/1178661550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5eb80b23a0 con 0x7f5eb806da40 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6000640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5eb80aef20 0x7f5eb81569c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb6a00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80b1e60 secure :-1 s=READY pgs=1417 cs=0 l=1 rev1=1 crypto rx=0x7f5ea800b3f0 tx=0x7f5ea800b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eaea00640 1 -- 172.21.15.86:0/1178661550 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ea8015020 con 0x7f5eb806da40 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eaea00640 1 -- 172.21.15.86:0/1178661550 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5ea8011830 con 0x7f5eb806da40 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eaea00640 1 -- 172.21.15.86:0/1178661550 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ea8011c20 con 0x7f5eb806da40 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.727+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5eb8158b10 con 0x7f5eb806da40 2024-03-06T07:34:26.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.728+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5eb81545c0 con 0x7f5eb806da40 2024-03-06T07:34:26.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.729+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5eb804f330 con 0x7f5eb806da40 2024-03-06T07:34:26.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.730+0000 7f5eaea00640 1 -- 172.21.15.86:0/1178661550 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5ea8018020 con 0x7f5eb806da40 2024-03-06T07:34:26.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.730+0000 7f5eaea00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e9806b500 0x7f5e9806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:26.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.730+0000 7f5eaea00640 1 -- 172.21.15.86:0/1178661550 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f5ea8089f30 con 0x7f5eb806da40 2024-03-06T07:34:26.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.731+0000 7f5eb6000640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e9806b500 0x7f5e9806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:26.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.731+0000 7f5eb6000640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e9806b500 0x7f5e9806d9c0 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f5ea4036000 tx=0x7f5ea4008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:26.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.735+0000 7f5eaea00640 1 -- 172.21.15.86:0/1178661550 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5ea801f030 con 0x7f5eb806da40 2024-03-06T07:34:26.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.930+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f5eb80cecd0 con 0x7f5eb806da40 2024-03-06T07:34:26.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.931+0000 7f5eaea00640 1 -- 172.21.15.86:0/1178661550 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v116) v1 ==== 93+0+10 (secure 0 0 0) 0x7f5eb80b23a0 con 0x7f5eb806da40 2024-03-06T07:34:26.933 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:34:26.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e9806b500 msgr2=0x7f5e9806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e9806b500 0x7f5e9806d9c0 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f5ea4036000 tx=0x7f5ea4008030 comp rx=0 tx=0).stop 2024-03-06T07:34:26.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 msgr2=0x7f5eb80b1e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80b1e60 secure :-1 s=READY pgs=1417 cs=0 l=1 rev1=1 crypto rx=0x7f5ea800b3f0 tx=0x7f5ea800b8b0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 shutdown_connections 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5e9806b500 0x7f5e9806d9c0 unknown :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5eb815be10 0x7f5eb8147050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5eb80aef20 0x7f5eb81569c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 --2- 172.21.15.86:0/1178661550 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5eb806da40 0x7f5eb80b1e60 unknown :-1 s=CLOSED pgs=1417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.933+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 >> 172.21.15.86:0/1178661550 conn(0x7f5eb806afc0 msgr2=0x7f5eb80cf4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.934+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 shutdown_connections 2024-03-06T07:34:26.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:26.934+0000 7f5eb7e00640 1 -- 172.21.15.86:0/1178661550 wait complete. 2024-03-06T07:34:26.954 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:34:27.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.152+0000 7f448e000640 1 Processor -- start 2024-03-06T07:34:27.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.152+0000 7f448e000640 1 -- start start 2024-03-06T07:34:27.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.153+0000 7f448e000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44880ce890 0x7f44880d39a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.153+0000 7f448e000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f44880d3ee0 0x7f44880b3de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.153+0000 7f448e000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f44880cf020 con 0x7f448815e000 2024-03-06T07:34:27.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.153+0000 7f448e000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f44880cf300 con 0x7f44880ce890 2024-03-06T07:34:27.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.153+0000 7f448e000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f44880cf5e0 con 0x7f44880d3ee0 2024-03-06T07:34:27.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44880ce890 0x7f44880d39a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f44880ce890 0x7f44880d39a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54668/0 (socket says 172.21.15.86:54668) 2024-03-06T07:34:27.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487e00640 1 -- 172.21.15.86:0/3933645025 learned_addr learned my addr 172.21.15.86:0/3933645025 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:27.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f448d600640 1 --2- 172.21.15.86:0/3933645025 >> v2:172.21.15.194:3300/0 conn(0x7f44880d3ee0 0x7f44880b3de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487e00640 1 -- 172.21.15.86:0/3933645025 >> v2:172.21.15.194:3300/0 conn(0x7f44880d3ee0 msgr2=0x7f44880b3de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:27.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487e00640 1 --2- 172.21.15.86:0/3933645025 >> v2:172.21.15.194:3300/0 conn(0x7f44880d3ee0 0x7f44880b3de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487e00640 1 -- 172.21.15.86:0/3933645025 >> v1:172.21.15.86:6789/0 conn(0x7f448815e000 legacy=0x7f44880d1b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487e00640 1 -- 172.21.15.86:0/3933645025 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44880cf8c0 con 0x7f44880ce890 2024-03-06T07:34:27.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.154+0000 7f4487400640 1 -- 172.21.15.86:0/3933645025 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3234095292 0 0) 0x7f44880cf020 con 0x7f448815e000 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.156+0000 7f4487e00640 1 --2- 172.21.15.86:0/3933645025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44880ce890 0x7f44880d39a0 secure :-1 s=READY pgs=1419 cs=0 l=1 rev1=1 crypto rx=0x7f448808dfd0 tx=0x7f447402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d50fb2ad1efb980b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.156+0000 7f4487400640 1 -- 172.21.15.86:0/3933645025 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4474034070 con 0x7f44880ce890 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.156+0000 7f4487400640 1 -- 172.21.15.86:0/3933645025 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4474030860 con 0x7f44880ce890 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.156+0000 7f4487400640 1 -- 172.21.15.86:0/3933645025 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4474030c30 con 0x7f44880ce890 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.156+0000 7f448e000640 1 -- 172.21.15.86:0/3933645025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44880ce890 msgr2=0x7f44880d39a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.156+0000 7f448e000640 1 --2- 172.21.15.86:0/3933645025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44880ce890 0x7f44880d39a0 secure :-1 s=READY pgs=1419 cs=0 l=1 rev1=1 crypto rx=0x7f448808dfd0 tx=0x7f447402b9a0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.156+0000 7f448e000640 1 -- 172.21.15.86:0/3933645025 shutdown_connections 2024-03-06T07:34:27.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.157+0000 7f448e000640 1 --2- 172.21.15.86:0/3933645025 >> v2:172.21.15.194:3300/0 conn(0x7f44880d3ee0 0x7f44880b3de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.157+0000 7f448e000640 1 --2- 172.21.15.86:0/3933645025 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44880ce890 0x7f44880d39a0 unknown :-1 s=CLOSED pgs=1419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.157+0000 7f448e000640 1 -- 172.21.15.86:0/3933645025 >> 172.21.15.86:0/3933645025 conn(0x7f448808d740 msgr2=0x7f448809d060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:27.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.157+0000 7f448e000640 1 -- 172.21.15.86:0/3933645025 shutdown_connections 2024-03-06T07:34:27.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.157+0000 7f448e000640 1 -- 172.21.15.86:0/3933645025 wait complete. 2024-03-06T07:34:27.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.158+0000 7f448e000640 1 Processor -- start 2024-03-06T07:34:27.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.158+0000 7f448e000640 1 -- start start 2024-03-06T07:34:27.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.159+0000 7f448e000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44880ce890 0x7f4488153210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.159+0000 7f448e000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44880d3ee0 0x7f44880c8380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.159+0000 7f448e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448815e000 0x7f448806c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.159+0000 7f448e000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44880cecf0 con 0x7f448815e000 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.159+0000 7f448e000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44880d4320 con 0x7f44880ce890 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.159+0000 7f448e000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f44880b4320 con 0x7f44880d3ee0 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.159+0000 7f448d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448815e000 0x7f448806c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448815e000 0x7f448806c8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54676/0 (socket says 172.21.15.86:54676) 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 -- 172.21.15.86:0/1011834077 learned_addr learned my addr 172.21.15.86:0/1011834077 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 -- 172.21.15.86:0/1011834077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44880d3ee0 msgr2=0x7f44880c8380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44880d3ee0 0x7f44880c8380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 -- 172.21.15.86:0/1011834077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44880ce890 msgr2=0x7f4488153210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44880ce890 0x7f4488153210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 -- 172.21.15.86:0/1011834077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44880b20e0 con 0x7f448815e000 2024-03-06T07:34:27.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f448d600640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448815e000 0x7f448806c8d0 secure :-1 s=READY pgs=1420 cs=0 l=1 rev1=1 crypto rx=0x7f4480007e70 tx=0x7f448000e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:27.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.160+0000 7f4484c00640 1 -- 172.21.15.86:0/1011834077 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4480014620 con 0x7f448815e000 2024-03-06T07:34:27.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.161+0000 7f4484c00640 1 -- 172.21.15.86:0/1011834077 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4480019070 con 0x7f448815e000 2024-03-06T07:34:27.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.161+0000 7f4484c00640 1 -- 172.21.15.86:0/1011834077 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4480014a20 con 0x7f448815e000 2024-03-06T07:34:27.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.161+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44880b23c0 con 0x7f448815e000 2024-03-06T07:34:27.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.161+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44880b2980 con 0x7f448815e000 2024-03-06T07:34:27.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.162+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f448804f330 con 0x7f448815e000 2024-03-06T07:34:27.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.163+0000 7f4484c00640 1 -- 172.21.15.86:0/1011834077 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4480014bc0 con 0x7f448815e000 2024-03-06T07:34:27.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.164+0000 7f4484c00640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446806b4d0 0x7f446806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.164+0000 7f4484c00640 1 -- 172.21.15.86:0/1011834077 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f448008ac30 con 0x7f448815e000 2024-03-06T07:34:27.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.164+0000 7f448cc00640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446806b4d0 0x7f446806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.165+0000 7f448cc00640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446806b4d0 0x7f446806d990 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f44880cde20 tx=0x7f447c007930 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:27.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.168+0000 7f4484c00640 1 -- 172.21.15.86:0/1011834077 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4480059980 con 0x7f448815e000 2024-03-06T07:34:27.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.383+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f44880d1ce0 con 0x7f448815e000 2024-03-06T07:34:27.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.384+0000 7f4484c00640 1 -- 172.21.15.86:0/1011834077 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v116) v1 ==== 104+0+11 (secure 0 0 0) 0x7f4480059280 con 0x7f448815e000 2024-03-06T07:34:27.387 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446806b4d0 msgr2=0x7f446806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446806b4d0 0x7f446806d990 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f44880cde20 tx=0x7f447c007930 comp rx=0 tx=0).stop 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448815e000 msgr2=0x7f448806c8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448815e000 0x7f448806c8d0 secure :-1 s=READY pgs=1420 cs=0 l=1 rev1=1 crypto rx=0x7f4480007e70 tx=0x7f448000e4b0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 shutdown_connections 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f446806b4d0 0x7f446806d990 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f448815e000 0x7f448806c8d0 unknown :-1 s=CLOSED pgs=1420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f44880d3ee0 0x7f44880c8380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 --2- 172.21.15.86:0/1011834077 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f44880ce890 0x7f4488153210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 >> 172.21.15.86:0/1011834077 conn(0x7f448808d740 msgr2=0x7f44880d0490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 shutdown_connections 2024-03-06T07:34:27.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.387+0000 7f448e000640 1 -- 172.21.15.86:0/1011834077 wait complete. 2024-03-06T07:34:27.406 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:34:27.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.605+0000 7f3ff9800640 1 Processor -- start 2024-03-06T07:34:27.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.605+0000 7f3ff9800640 1 -- start start 2024-03-06T07:34:27.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.607+0000 7f3ff9800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3ff40b3b40 0x7f3ff40b3fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.607+0000 7f3ff9800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff406dda0 0x7f3ff406e230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.607+0000 7f3ff9800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3ff40b04b0 con 0x7f3ff4158770 2024-03-06T07:34:27.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.607+0000 7f3ff9800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3ff415ef20 con 0x7f3ff40b3b40 2024-03-06T07:34:27.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.607+0000 7f3ff9800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ff415e8d0 con 0x7f3ff406dda0 2024-03-06T07:34:27.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.608+0000 7f3ff3400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff406dda0 0x7f3ff406e230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.608+0000 7f3ff2a00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f3ff4158770 0x7f3ff40afa10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:39380/0 (socket says 172.21.15.86:39380) 2024-03-06T07:34:27.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.608+0000 7f3ff2a00640 1 -- 172.21.15.86:0/3225876989 learned_addr learned my addr 172.21.15.86:0/3225876989 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:27.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.608+0000 7f3ff3400640 1 --2- 172.21.15.86:0/3225876989 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff406dda0 0x7f3ff406e230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:43788/0 (socket says 172.21.15.86:43788) 2024-03-06T07:34:27.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.608+0000 7f3febe00640 1 --2- 172.21.15.86:0/3225876989 >> v2:172.21.15.194:3300/0 conn(0x7f3ff40b3b40 0x7f3ff40b3fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.609+0000 7f3ff3400640 1 -- 172.21.15.86:0/3225876989 >> v2:172.21.15.194:3300/0 conn(0x7f3ff40b3b40 msgr2=0x7f3ff40b3fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:27.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.609+0000 7f3ff3400640 1 --2- 172.21.15.86:0/3225876989 >> v2:172.21.15.194:3300/0 conn(0x7f3ff40b3b40 0x7f3ff40b3fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.609+0000 7f3ff3400640 1 -- 172.21.15.86:0/3225876989 >> v1:172.21.15.194:6789/0 conn(0x7f3ff4158770 legacy=0x7f3ff40afa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.609+0000 7f3ff3400640 1 -- 172.21.15.86:0/3225876989 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ff415ebb0 con 0x7f3ff406dda0 2024-03-06T07:34:27.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.610+0000 7f3ff3400640 1 --2- 172.21.15.86:0/3225876989 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff406dda0 0x7f3ff406e230 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f3fec009d90 tx=0x7f3fec02bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6c63bcc15e6d4bdc server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:27.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.611+0000 7f3ff2000640 1 -- 172.21.15.86:0/3225876989 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fec039070 con 0x7f3ff406dda0 2024-03-06T07:34:27.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.611+0000 7f3ff2000640 1 -- 172.21.15.86:0/3225876989 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3fec02cc50 con 0x7f3ff406dda0 2024-03-06T07:34:27.613 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.611+0000 7f3ff2000640 1 -- 172.21.15.86:0/3225876989 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fec034440 con 0x7f3ff406dda0 2024-03-06T07:34:27.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.611+0000 7f3ff9800640 1 -- 172.21.15.86:0/3225876989 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff406dda0 msgr2=0x7f3ff406e230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.611+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3225876989 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff406dda0 0x7f3ff406e230 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f3fec009d90 tx=0x7f3fec02bea0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.612+0000 7f3ff9800640 1 -- 172.21.15.86:0/3225876989 shutdown_connections 2024-03-06T07:34:27.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.612+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3225876989 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff406dda0 0x7f3ff406e230 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.612+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3225876989 >> v2:172.21.15.194:3300/0 conn(0x7f3ff40b3b40 0x7f3ff40b3fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.612+0000 7f3ff9800640 1 -- 172.21.15.86:0/3225876989 >> 172.21.15.86:0/3225876989 conn(0x7f3ff40b2840 msgr2=0x7f3ff40b2c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:27.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.612+0000 7f3ff9800640 1 -- 172.21.15.86:0/3225876989 shutdown_connections 2024-03-06T07:34:27.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.612+0000 7f3ff9800640 1 -- 172.21.15.86:0/3225876989 wait complete. 2024-03-06T07:34:27.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.614+0000 7f3ff9800640 1 Processor -- start 2024-03-06T07:34:27.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.614+0000 7f3ff9800640 1 -- start start 2024-03-06T07:34:27.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.615+0000 7f3ff9800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3ff406dda0 0x7f3ff413fc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.615+0000 7f3ff9800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff40b3b40 0x7f3ff40ae400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.615+0000 7f3ff9800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3ff4158770 0x7f3ff413f3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.615+0000 7f3ff9800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ff40b0120 con 0x7f3ff4158770 2024-03-06T07:34:27.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.615+0000 7f3ff9800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ff40b5d70 con 0x7f3ff406dda0 2024-03-06T07:34:27.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.615+0000 7f3ff9800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ff415f3b0 con 0x7f3ff40b3b40 2024-03-06T07:34:27.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.615+0000 7f3ff3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3ff4158770 0x7f3ff413f3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3ff4158770 0x7f3ff413f3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54688/0 (socket says 172.21.15.86:54688) 2024-03-06T07:34:27.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff3400640 1 -- 172.21.15.86:0/3782820865 learned_addr learned my addr 172.21.15.86:0/3782820865 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:27.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff2a00640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3ff406dda0 0x7f3ff413fc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3febe00640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff40b3b40 0x7f3ff40ae400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff3400640 1 -- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff40b3b40 msgr2=0x7f3ff40ae400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff3400640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff40b3b40 0x7f3ff40ae400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff3400640 1 -- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3ff406dda0 msgr2=0x7f3ff413fc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff3400640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3ff406dda0 0x7f3ff413fc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.616+0000 7f3ff3400640 1 -- 172.21.15.86:0/3782820865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ff4140370 con 0x7f3ff4158770 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.617+0000 7f3ff3400640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3ff4158770 0x7f3ff413f3b0 secure :-1 s=READY pgs=1421 cs=0 l=1 rev1=1 crypto rx=0x7f3fec009d90 tx=0x7f3fec004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.617+0000 7f3feaa00640 1 -- 172.21.15.86:0/3782820865 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fec039050 con 0x7f3ff4158770 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.617+0000 7f3feaa00640 1 -- 172.21.15.86:0/3782820865 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3fec045070 con 0x7f3ff4158770 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.617+0000 7f3feaa00640 1 -- 172.21.15.86:0/3782820865 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3fec009450 con 0x7f3ff4158770 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.617+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3ff4140650 con 0x7f3ff4158770 2024-03-06T07:34:27.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.617+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3ff4140c10 con 0x7f3ff4158770 2024-03-06T07:34:27.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.618+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3ff404f330 con 0x7f3ff4158770 2024-03-06T07:34:27.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.619+0000 7f3feaa00640 1 -- 172.21.15.86:0/3782820865 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3fec03e610 con 0x7f3ff4158770 2024-03-06T07:34:27.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.620+0000 7f3feaa00640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3fd006b4d0 0x7f3fd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:27.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.621+0000 7f3feaa00640 1 -- 172.21.15.86:0/3782820865 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f3fec0acca0 con 0x7f3ff4158770 2024-03-06T07:34:27.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.621+0000 7f3ff2a00640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3fd006b4d0 0x7f3fd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:27.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.621+0000 7f3ff2a00640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3fd006b4d0 0x7f3fd006d990 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f3fe4007fd0 tx=0x7f3fe4007e80 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:27.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.625+0000 7f3feaa00640 1 -- 172.21.15.86:0/3782820865 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3fec07b9f0 con 0x7f3ff4158770 2024-03-06T07:34:27.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.862+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f3ff40b4050 con 0x7f3ff4158770 2024-03-06T07:34:27.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.863+0000 7f3feaa00640 1 -- 172.21.15.86:0/3782820865 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v116) v1 ==== 100+0+11 (secure 0 0 0) 0x7f3fec07b2f0 con 0x7f3ff4158770 2024-03-06T07:34:27.866 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:34:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.865+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3fd006b4d0 msgr2=0x7f3fd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.865+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3fd006b4d0 0x7f3fd006d990 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f3fe4007fd0 tx=0x7f3fe4007e80 comp rx=0 tx=0).stop 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.865+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3ff4158770 msgr2=0x7f3ff413f3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.865+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3ff4158770 0x7f3ff413f3b0 secure :-1 s=READY pgs=1421 cs=0 l=1 rev1=1 crypto rx=0x7f3fec009d90 tx=0x7f3fec004a40 comp rx=0 tx=0).stop 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 shutdown_connections 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3fd006b4d0 0x7f3fd006d990 unknown :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3ff4158770 0x7f3ff413f3b0 unknown :-1 s=CLOSED pgs=1421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3ff40b3b40 0x7f3ff40ae400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 --2- 172.21.15.86:0/3782820865 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3ff406dda0 0x7f3ff413fc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 >> 172.21.15.86:0/3782820865 conn(0x7f3ff40b2840 msgr2=0x7f3ff40b15d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 shutdown_connections 2024-03-06T07:34:27.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:27.866+0000 7f3ff9800640 1 -- 172.21.15.86:0/3782820865 wait complete. 2024-03-06T07:34:27.894 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:28.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.089+0000 7f9248800640 1 Processor -- start 2024-03-06T07:34:28.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.090+0000 7f9248800640 1 -- start start 2024-03-06T07:34:28.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.091+0000 7f9248800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9240144c00 0x7f924006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.091+0000 7f9248800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f924015fe30 0x7f92400ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.091+0000 7f9248800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f924006ee70 con 0x7f924015f9d0 2024-03-06T07:34:28.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.091+0000 7f9248800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f924006cef0 con 0x7f924015fe30 2024-03-06T07:34:28.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.091+0000 7f9248800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f924006d1d0 con 0x7f9240144c00 2024-03-06T07:34:28.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.092+0000 7f9245400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f924015f9d0 0x7f92400ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42952/0 (socket says 172.21.15.86:42952) 2024-03-06T07:34:28.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.092+0000 7f9245400640 1 -- 172.21.15.86:0/267967339 learned_addr learned my addr 172.21.15.86:0/267967339 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.092+0000 7f9246800640 1 --2- 172.21.15.86:0/267967339 >> v2:172.21.15.194:3300/0 conn(0x7f924015fe30 0x7f92400ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.092+0000 7f9245e00640 1 --2- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9240144c00 0x7f924006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.092+0000 7f9244a00640 1 -- 172.21.15.86:0/267967339 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1251924230 0 0) 0x7f9230001110 con 0x7f924015f9d0 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.092+0000 7f9244a00640 1 -- 172.21.15.86:0/267967339 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9228003520 con 0x7f924015f9d0 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.093+0000 7f9244a00640 1 -- 172.21.15.86:0/267967339 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1097180374 0 0) 0x7f9228003520 con 0x7f924015f9d0 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.093+0000 7f9246800640 1 -- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9240144c00 msgr2=0x7f924006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.093+0000 7f9246800640 1 --2- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9240144c00 0x7f924006e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.093+0000 7f9246800640 1 -- 172.21.15.86:0/267967339 >> v1:172.21.15.86:6789/0 conn(0x7f924015f9d0 legacy=0x7f92400ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.093+0000 7f9246800640 1 -- 172.21.15.86:0/267967339 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f924006d4b0 con 0x7f924015fe30 2024-03-06T07:34:28.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.094+0000 7f9246800640 1 --2- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015fe30 0x7f92400ce7b0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f923c009d90 tx=0x7f923c02bd70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1e7ad834ac25ff3d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:28.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.094+0000 7f9244a00640 1 -- 172.21.15.86:0/267967339 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f923c039070 con 0x7f924015fe30 2024-03-06T07:34:28.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9244a00640 1 -- 172.21.15.86:0/267967339 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f923c02c8a0 con 0x7f924015fe30 2024-03-06T07:34:28.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9244a00640 1 -- 172.21.15.86:0/267967339 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9230001840 con 0x7f924015fe30 2024-03-06T07:34:28.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 -- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015fe30 msgr2=0x7f92400ce7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 --2- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015fe30 0x7f92400ce7b0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f923c009d90 tx=0x7f923c02bd70 comp rx=0 tx=0).stop 2024-03-06T07:34:28.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 -- 172.21.15.86:0/267967339 shutdown_connections 2024-03-06T07:34:28.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 --2- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015fe30 0x7f92400ce7b0 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 --2- 172.21.15.86:0/267967339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9240144c00 0x7f924006e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 -- 172.21.15.86:0/267967339 >> 172.21.15.86:0/267967339 conn(0x7f92400b1730 msgr2=0x7f92400b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:28.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 -- 172.21.15.86:0/267967339 shutdown_connections 2024-03-06T07:34:28.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.095+0000 7f9248800640 1 -- 172.21.15.86:0/267967339 wait complete. 2024-03-06T07:34:28.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.098+0000 7f9248800640 1 Processor -- start 2024-03-06T07:34:28.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.098+0000 7f9248800640 1 -- start start 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9248800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9240144c00 0x7f92400b4060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9248800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015f9d0 0x7f92400ae240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9248800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f924015fe30 0x7f9240139370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9248800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9240160730 con 0x7f9240144c00 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9248800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9240160280 con 0x7f924015f9d0 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9248800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f924006cb50 con 0x7f924015fe30 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9245e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9240144c00 0x7f92400b4060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9245e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9240144c00 0x7f92400b4060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54690/0 (socket says 172.21.15.86:54690) 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.100+0000 7f9245e00640 1 -- 172.21.15.86:0/2172458988 learned_addr learned my addr 172.21.15.86:0/2172458988 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9246800640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f924015fe30 0x7f9240139370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245400640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015f9d0 0x7f92400ae240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245e00640 1 -- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f924015fe30 msgr2=0x7f9240139370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245e00640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f924015fe30 0x7f9240139370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245e00640 1 -- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015f9d0 msgr2=0x7f92400ae240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245e00640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015f9d0 0x7f92400ae240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245e00640 1 -- 172.21.15.86:0/2172458988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f92400b2fc0 con 0x7f9240144c00 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245400640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015f9d0 0x7f92400ae240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9245e00640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9240144c00 0x7f92400b4060 secure :-1 s=READY pgs=1423 cs=0 l=1 rev1=1 crypto rx=0x7f923400e870 tx=0x7f923400ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f923a000640 1 -- 172.21.15.86:0/2172458988 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9234017070 con 0x7f9240144c00 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f923a000640 1 -- 172.21.15.86:0/2172458988 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f923400c940 con 0x7f9240144c00 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f92400b32a0 con 0x7f9240144c00 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.101+0000 7f923a000640 1 -- 172.21.15.86:0/2172458988 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f923400cd30 con 0x7f9240144c00 2024-03-06T07:34:28.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.102+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f92400b3860 con 0x7f9240144c00 2024-03-06T07:34:28.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.103+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f924004f330 con 0x7f9240144c00 2024-03-06T07:34:28.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.104+0000 7f923a000640 1 -- 172.21.15.86:0/2172458988 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f923401a070 con 0x7f9240144c00 2024-03-06T07:34:28.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.105+0000 7f923a000640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f922406b4d0 0x7f922406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.105+0000 7f923a000640 1 -- 172.21.15.86:0/2172458988 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f923408a640 con 0x7f9240144c00 2024-03-06T07:34:28.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.105+0000 7f9245400640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f922406b4d0 0x7f922406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.106+0000 7f9245400640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f922406b4d0 0x7f922406d990 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f9230001eb0 tx=0x7f9230009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:28.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.110+0000 7f923a000640 1 -- 172.21.15.86:0/2172458988 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9234059300 con 0x7f9240144c00 2024-03-06T07:34:28.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.343+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9240072890 con 0x7f9240144c00 2024-03-06T07:34:28.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.344+0000 7f923a000640 1 -- 172.21.15.86:0/2172458988 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f9234058c00 con 0x7f9240144c00 2024-03-06T07:34:28.347 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:28.347 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:28.347 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:28.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f922406b4d0 msgr2=0x7f922406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f922406b4d0 0x7f922406d990 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f9230001eb0 tx=0x7f9230009040 comp rx=0 tx=0).stop 2024-03-06T07:34:28.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9240144c00 msgr2=0x7f92400b4060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9240144c00 0x7f92400b4060 secure :-1 s=READY pgs=1423 cs=0 l=1 rev1=1 crypto rx=0x7f923400e870 tx=0x7f923400ed30 comp rx=0 tx=0).stop 2024-03-06T07:34:28.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 shutdown_connections 2024-03-06T07:34:28.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f922406b4d0 0x7f922406d990 unknown :-1 s=CLOSED pgs=609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f924015fe30 0x7f9240139370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f924015f9d0 0x7f92400ae240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 --2- 172.21.15.86:0/2172458988 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9240144c00 0x7f92400b4060 unknown :-1 s=CLOSED pgs=1423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.346+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 >> 172.21.15.86:0/2172458988 conn(0x7f92400b1730 msgr2=0x7f92400b0480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:28.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.347+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 shutdown_connections 2024-03-06T07:34:28.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.347+0000 7f9248800640 1 -- 172.21.15.86:0/2172458988 wait complete. 2024-03-06T07:34:28.368 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:28.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.561+0000 7f84c1a00640 1 Processor -- start 2024-03-06T07:34:28.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.561+0000 7f84c1a00640 1 -- start start 2024-03-06T07:34:28.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84c1a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f84bc0af1c0 0x7f84bc15eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84c1a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc06e4d0 0x7f84bc06e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84c1a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f84bc071d30 con 0x7f84bc070d20 2024-03-06T07:34:28.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84c1a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f84bc0711d0 con 0x7f84bc0af1c0 2024-03-06T07:34:28.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84c1a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f84bc071490 con 0x7f84bc06e4d0 2024-03-06T07:34:28.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84baa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f84bc0af1c0 0x7f84bc15eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84b2000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f84bc070d20 0x7f84bc06cc60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42968/0 (socket says 172.21.15.86:42968) 2024-03-06T07:34:28.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.563+0000 7f84b2000640 1 -- 172.21.15.86:0/323211035 learned_addr learned my addr 172.21.15.86:0/323211035 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:28.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.564+0000 7f84bb400640 1 --2- 172.21.15.86:0/323211035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc06e4d0 0x7f84bc06e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.564+0000 7f84baa00640 1 -- 172.21.15.86:0/323211035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc06e4d0 msgr2=0x7f84bc06e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:28.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.564+0000 7f84baa00640 1 --2- 172.21.15.86:0/323211035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc06e4d0 0x7f84bc06e960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.564+0000 7f84baa00640 1 -- 172.21.15.86:0/323211035 >> v1:172.21.15.86:6789/0 conn(0x7f84bc070d20 legacy=0x7f84bc06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.564+0000 7f84baa00640 1 -- 172.21.15.86:0/323211035 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84bc071770 con 0x7f84bc0af1c0 2024-03-06T07:34:28.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.564+0000 7f84bb400640 1 --2- 172.21.15.86:0/323211035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc06e4d0 0x7f84bc06e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:28.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.565+0000 7f84baa00640 1 --2- 172.21.15.86:0/323211035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc0af1c0 0x7f84bc15eff0 secure :-1 s=READY pgs=1425 cs=0 l=1 rev1=1 crypto rx=0x7f84ac009dc0 tx=0x7f84ac02be80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c34a91059bc714e6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:28.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.565+0000 7f84ba000640 1 -- 172.21.15.86:0/323211035 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84ac039070 con 0x7f84bc0af1c0 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.565+0000 7f84ba000640 1 -- 172.21.15.86:0/323211035 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f84ac02cc30 con 0x7f84bc0af1c0 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.565+0000 7f84ba000640 1 -- 172.21.15.86:0/323211035 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84ac034440 con 0x7f84bc0af1c0 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 -- 172.21.15.86:0/323211035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc0af1c0 msgr2=0x7f84bc15eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 --2- 172.21.15.86:0/323211035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc0af1c0 0x7f84bc15eff0 secure :-1 s=READY pgs=1425 cs=0 l=1 rev1=1 crypto rx=0x7f84ac009dc0 tx=0x7f84ac02be80 comp rx=0 tx=0).stop 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 -- 172.21.15.86:0/323211035 shutdown_connections 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 --2- 172.21.15.86:0/323211035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc06e4d0 0x7f84bc06e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 --2- 172.21.15.86:0/323211035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc0af1c0 0x7f84bc15eff0 unknown :-1 s=CLOSED pgs=1425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 -- 172.21.15.86:0/323211035 >> 172.21.15.86:0/323211035 conn(0x7f84bc0b1730 msgr2=0x7f84bc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 -- 172.21.15.86:0/323211035 shutdown_connections 2024-03-06T07:34:28.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.566+0000 7f84c1a00640 1 -- 172.21.15.86:0/323211035 wait complete. 2024-03-06T07:34:28.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.568+0000 7f84c1a00640 1 Processor -- start 2024-03-06T07:34:28.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.568+0000 7f84c1a00640 1 -- start start 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.568+0000 7f84c1a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84bc06e4d0 0x7f84bc14b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84c1a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc070d20 0x7f84bc0cdaa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84c1a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc0af1c0 0x7f84bc15fe10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84c1a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84bc06d370 con 0x7f84bc070d20 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84c1a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84bc0b30b0 con 0x7f84bc06e4d0 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84c1a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f84bc0b35d0 con 0x7f84bc0af1c0 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc070d20 0x7f84bc0cdaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc070d20 0x7f84bc0cdaa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54716/0 (socket says 172.21.15.86:54716) 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 -- 172.21.15.86:0/3039324964 learned_addr learned my addr 172.21.15.86:0/3039324964 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84baa00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84bc06e4d0 0x7f84bc14b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84bb400640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc0af1c0 0x7f84bc15fe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 -- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc0af1c0 msgr2=0x7f84bc15fe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc0af1c0 0x7f84bc15fe10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 -- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84bc06e4d0 msgr2=0x7f84bc14b3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84bc06e4d0 0x7f84bc14b3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.569+0000 7f84b2000640 1 -- 172.21.15.86:0/3039324964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84bc0ce540 con 0x7f84bc070d20 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.570+0000 7f84b2000640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc070d20 0x7f84bc0cdaa0 secure :-1 s=READY pgs=1426 cs=0 l=1 rev1=1 crypto rx=0x7f84a400c430 tx=0x7f84a400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.570+0000 7f84b3400640 1 -- 172.21.15.86:0/3039324964 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84a4017070 con 0x7f84bc070d20 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.570+0000 7f84b3400640 1 -- 172.21.15.86:0/3039324964 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f84a4005ce0 con 0x7f84bc070d20 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.570+0000 7f84b3400640 1 -- 172.21.15.86:0/3039324964 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84a40072f0 con 0x7f84bc070d20 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.570+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84bc0ce820 con 0x7f84bc070d20 2024-03-06T07:34:28.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.570+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84bc0cede0 con 0x7f84bc070d20 2024-03-06T07:34:28.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.572+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f84bc04f330 con 0x7f84bc070d20 2024-03-06T07:34:28.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.573+0000 7f84b3400640 1 -- 172.21.15.86:0/3039324964 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f84a401a070 con 0x7f84bc070d20 2024-03-06T07:34:28.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.574+0000 7f84b3400640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f849806b4d0 0x7f849806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:28.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.574+0000 7f84b3400640 1 -- 172.21.15.86:0/3039324964 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f84a40896c0 con 0x7f84bc070d20 2024-03-06T07:34:28.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.574+0000 7f84baa00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f849806b4d0 0x7f849806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:28.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.575+0000 7f84baa00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f849806b4d0 0x7f849806d990 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f84ac036000 tx=0x7f84ac030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:28.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.579+0000 7f84b3400640 1 -- 172.21.15.86:0/3039324964 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f84bc071d30 con 0x7f84bc070d20 2024-03-06T07:34:28.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.774+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f84bc0af920 con 0x7f84bc070d20 2024-03-06T07:34:28.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.776+0000 7f84b3400640 1 -- 172.21.15.86:0/3039324964 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v116) v1 ==== 74+0+16920 (secure 0 0 0) 0x7f84a40584a0 con 0x7f84bc070d20 2024-03-06T07:34:28.779 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:28.779 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":116,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:55.692842+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"110","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.3f","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/221976790":"2024-03-07T07:33:20.538234+0000","172.21.15.86:6836/221976790":"2024-03-07T07:33:20.538234+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:28.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.778+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f849806b4d0 msgr2=0x7f849806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.778+0000 7f84c1a00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f849806b4d0 0x7f849806d990 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f84ac036000 tx=0x7f84ac030000 comp rx=0 tx=0).stop 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc070d20 msgr2=0x7f84bc0cdaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc070d20 0x7f84bc0cdaa0 secure :-1 s=READY pgs=1426 cs=0 l=1 rev1=1 crypto rx=0x7f84a400c430 tx=0x7f84a400c8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 shutdown_connections 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f849806b4d0 0x7f849806d990 unknown :-1 s=CLOSED pgs=610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84bc0af1c0 0x7f84bc15fe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84bc070d20 0x7f84bc0cdaa0 unknown :-1 s=CLOSED pgs=1426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 --2- 172.21.15.86:0/3039324964 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84bc06e4d0 0x7f84bc14b3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 >> 172.21.15.86:0/3039324964 conn(0x7f84bc0b1730 msgr2=0x7f84bc15beb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 shutdown_connections 2024-03-06T07:34:28.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.779+0000 7f84c1a00640 1 -- 172.21.15.86:0/3039324964 wait complete. 2024-03-06T07:34:28.800 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-06T07:34:28.800 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:29.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.999+0000 7f6c39600640 1 Processor -- start 2024-03-06T07:34:29.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:28.999+0000 7f6c39600640 1 -- start start 2024-03-06T07:34:29.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c39600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6c3415e7a0 0x7f6c340b6430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c39600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 0x7f6c3415eb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c39600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6c3415f0c0 0x7f6c3406ec90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c39600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c3406d0a0 con 0x7f6c3415e7a0 2024-03-06T07:34:29.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c39600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c3406c490 con 0x7f6c3415f0c0 2024-03-06T07:34:29.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c39600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c3406c770 con 0x7f6c3406bf90 2024-03-06T07:34:29.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c33e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6c3415e7a0 0x7f6c340b6430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c33e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6c3415e7a0 0x7f6c340b6430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54726/0 (socket says 172.21.15.86:54726) 2024-03-06T07:34:29.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.002+0000 7f6c33e00640 1 -- 172.21.15.86:0/1849954346 learned_addr learned my addr 172.21.15.86:0/1849954346 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:29.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.003+0000 7f6c33e00640 1 -- 172.21.15.86:0/1849954346 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 msgr2=0x7f6c3415eb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:29.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.003+0000 7f6c33e00640 1 --2- 172.21.15.86:0/1849954346 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 0x7f6c3415eb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.003+0000 7f6c33e00640 1 -- 172.21.15.86:0/1849954346 >> v2:172.21.15.194:3300/0 conn(0x7f6c3415f0c0 msgr2=0x7f6c3406ec90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:29.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.003+0000 7f6c33e00640 1 --2- 172.21.15.86:0/1849954346 >> v2:172.21.15.194:3300/0 conn(0x7f6c3415f0c0 0x7f6c3406ec90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.003+0000 7f6c33e00640 1 -- 172.21.15.86:0/1849954346 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c3406ca50 con 0x7f6c3415e7a0 2024-03-06T07:34:29.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.004+0000 7f6c33e00640 1 --2- 172.21.15.86:0/1849954346 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b6430 secure :-1 s=READY pgs=1427 cs=0 l=1 rev1=1 crypto rx=0x7f6c3406bf20 tx=0x7f6c2402be70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c265265773def209 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.004+0000 7f6c33400640 1 -- 172.21.15.86:0/1849954346 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c2402ca10 con 0x7f6c3415e7a0 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.004+0000 7f6c33400640 1 -- 172.21.15.86:0/1849954346 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c2402cbb0 con 0x7f6c3415e7a0 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.004+0000 7f6c33400640 1 -- 172.21.15.86:0/1849954346 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c240343f0 con 0x7f6c3415e7a0 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.004+0000 7f6c39600640 1 -- 172.21.15.86:0/1849954346 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 msgr2=0x7f6c340b6430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.004+0000 7f6c39600640 1 --2- 172.21.15.86:0/1849954346 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b6430 secure :-1 s=READY pgs=1427 cs=0 l=1 rev1=1 crypto rx=0x7f6c3406bf20 tx=0x7f6c2402be70 comp rx=0 tx=0).stop 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.005+0000 7f6c39600640 1 -- 172.21.15.86:0/1849954346 shutdown_connections 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.005+0000 7f6c39600640 1 --2- 172.21.15.86:0/1849954346 >> v2:172.21.15.194:3300/0 conn(0x7f6c3415f0c0 0x7f6c3406ec90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.005+0000 7f6c39600640 1 --2- 172.21.15.86:0/1849954346 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 0x7f6c3415eb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.005+0000 7f6c39600640 1 --2- 172.21.15.86:0/1849954346 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b6430 unknown :-1 s=CLOSED pgs=1427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.005+0000 7f6c39600640 1 -- 172.21.15.86:0/1849954346 >> 172.21.15.86:0/1849954346 conn(0x7f6c3409a9f0 msgr2=0x7f6c3409ae00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.005+0000 7f6c39600640 1 -- 172.21.15.86:0/1849954346 shutdown_connections 2024-03-06T07:34:29.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.005+0000 7f6c39600640 1 -- 172.21.15.86:0/1849954346 wait complete. 2024-03-06T07:34:29.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.007+0000 7f6c39600640 1 Processor -- start 2024-03-06T07:34:29.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.008+0000 7f6c39600640 1 -- start start 2024-03-06T07:34:29.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.009+0000 7f6c39600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 0x7f6c3415a6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.009+0000 7f6c39600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b3130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.009+0000 7f6c39600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c3415f0c0 0x7f6c340b3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.009+0000 7f6c39600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c3406d530 con 0x7f6c3415e7a0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.009+0000 7f6c39600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c340b5750 con 0x7f6c3415f0c0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.009+0000 7f6c39600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c3406d6b0 con 0x7f6c3406bf90 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.009+0000 7f6c2b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b3130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b3130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54734/0 (socket says 172.21.15.86:54734) 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 -- 172.21.15.86:0/2277305101 learned_addr learned my addr 172.21.15.86:0/2277305101 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c33e00640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 0x7f6c3415a6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c38c00640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c3415f0c0 0x7f6c340b3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 -- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 msgr2=0x7f6c3415a6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 0x7f6c3415a6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 -- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c3415f0c0 msgr2=0x7f6c340b3670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c3415f0c0 0x7f6c340b3670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 -- 172.21.15.86:0/2277305101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c3415af00 con 0x7f6c3415e7a0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.010+0000 7f6c2b400640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b3130 secure :-1 s=READY pgs=1428 cs=0 l=1 rev1=1 crypto rx=0x7f6c1c00b4a0 tx=0x7f6c1c00b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.011+0000 7f6c30c00640 1 -- 172.21.15.86:0/2277305101 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c1c014070 con 0x7f6c3415e7a0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.011+0000 7f6c30c00640 1 -- 172.21.15.86:0/2277305101 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c1c010610 con 0x7f6c3415e7a0 2024-03-06T07:34:29.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.011+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6c3415b1e0 con 0x7f6c3415e7a0 2024-03-06T07:34:29.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.011+0000 7f6c30c00640 1 -- 172.21.15.86:0/2277305101 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c1c010a00 con 0x7f6c3415e7a0 2024-03-06T07:34:29.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.011+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c3415b7a0 con 0x7f6c3415e7a0 2024-03-06T07:34:29.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.012+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6c3404f330 con 0x7f6c3415e7a0 2024-03-06T07:34:29.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.013+0000 7f6c30c00640 1 -- 172.21.15.86:0/2277305101 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6c1c017070 con 0x7f6c3415e7a0 2024-03-06T07:34:29.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.014+0000 7f6c30c00640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c1006b4d0 0x7f6c1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.014+0000 7f6c30c00640 1 -- 172.21.15.86:0/2277305101 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f6c1c089d70 con 0x7f6c3415e7a0 2024-03-06T07:34:29.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.014+0000 7f6c33e00640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c1006b4d0 0x7f6c1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.015+0000 7f6c33e00640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c1006b4d0 0x7f6c1006d990 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f6c24035000 tx=0x7f6c24030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.019+0000 7f6c30c00640 1 -- 172.21.15.86:0/2277305101 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6c1c058ac0 con 0x7f6c3415e7a0 2024-03-06T07:34:29.243 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.242+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6c340ce100 con 0x7f6c3415e7a0 2024-03-06T07:34:29.244 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.243+0000 7f6c30c00640 1 -- 172.21.15.86:0/2277305101 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7f6c1c0583c0 con 0x7f6c3415e7a0 2024-03-06T07:34:29.245 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:29.245 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:29.245 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:29.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c1006b4d0 msgr2=0x7f6c1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c1006b4d0 0x7f6c1006d990 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f6c24035000 tx=0x7f6c24030000 comp rx=0 tx=0).stop 2024-03-06T07:34:29.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 msgr2=0x7f6c340b3130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b3130 secure :-1 s=READY pgs=1428 cs=0 l=1 rev1=1 crypto rx=0x7f6c1c00b4a0 tx=0x7f6c1c00b960 comp rx=0 tx=0).stop 2024-03-06T07:34:29.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 shutdown_connections 2024-03-06T07:34:29.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6c1006b4d0 0x7f6c1006d990 unknown :-1 s=CLOSED pgs=611 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6c3415f0c0 0x7f6c340b3670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6c3415e7a0 0x7f6c340b3130 unknown :-1 s=CLOSED pgs=1428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 --2- 172.21.15.86:0/2277305101 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6c3406bf90 0x7f6c3415a6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.245+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 >> 172.21.15.86:0/2277305101 conn(0x7f6c3409a9f0 msgr2=0x7f6c34156900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:29.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.246+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 shutdown_connections 2024-03-06T07:34:29.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.246+0000 7f6c39600640 1 -- 172.21.15.86:0/2277305101 wait complete. 2024-03-06T07:34:29.267 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:29.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.455+0000 7ffb3c600640 1 Processor -- start 2024-03-06T07:34:29.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.455+0000 7ffb3c600640 1 -- start start 2024-03-06T07:34:29.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.457+0000 7ffb3c600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb3415f9d0 0x7ffb3415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.457+0000 7ffb3c600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffb3406cef0 con 0x7ffb34070b10 2024-03-06T07:34:29.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.457+0000 7ffb3c600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffb3406d1d0 con 0x7ffb34144c00 2024-03-06T07:34:29.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.457+0000 7ffb3c600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb34160370 con 0x7ffb3415f9d0 2024-03-06T07:34:29.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.457+0000 7ffb3a600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ffb34070b10 0x7ffb34070f60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42978/0 (socket says 172.21.15.86:42978) 2024-03-06T07:34:29.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.457+0000 7ffb3a600640 1 -- 172.21.15.86:0/1762441166 learned_addr learned my addr 172.21.15.86:0/1762441166 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:29.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb39200640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb3415f9d0 0x7ffb3415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 729592742 0 0) 0x7ffb3406cef0 con 0x7ffb34070b10 2024-03-06T07:34:29.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ffb1c003520 con 0x7ffb34070b10 2024-03-06T07:34:29.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1764998152 0 0) 0x7ffb1c003520 con 0x7ffb34070b10 2024-03-06T07:34:29.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb3415f9d0 msgr2=0x7ffb3415fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:29.461 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb3415f9d0 0x7ffb3415fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 >> v1:172.21.15.194:6789/0 conn(0x7ffb34144c00 legacy=0x7ffb3406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb3406d4b0 con 0x7ffb34070b10 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7ffb2c0019f0 con 0x7ffb34070b10 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 >> v1:172.21.15.86:6789/0 conn(0x7ffb34070b10 legacy=0x7ffb34070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c0039e0 0x7ffb1c003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb1c004800 0x7ffb1c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb3415f9d0 0x7ffb1c005250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb1c00bb50 con 0x7ffb1c004800 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb1c00bd00 con 0x7ffb1c0039e0 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.458+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb1c00bf00 con 0x7ffb3415f9d0 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39200640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb1c004800 0x7ffb1c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39c00640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c0039e0 0x7ffb1c003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39200640 1 -- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb3415f9d0 msgr2=0x7ffb1c005250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:29.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39200640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb3415f9d0 0x7ffb1c005250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39200640 1 -- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c0039e0 msgr2=0x7ffb1c003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:29.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39200640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c0039e0 0x7ffb1c003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39200640 1 -- 172.21.15.86:0/1762441166 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb1c003520 con 0x7ffb1c004800 2024-03-06T07:34:29.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.459+0000 7ffb39c00640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c0039e0 0x7ffb1c003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.460+0000 7ffb39200640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb1c004800 0x7ffb1c004c80 secure :-1 s=READY pgs=1430 cs=0 l=1 rev1=1 crypto rx=0x7ffb2800ba30 tx=0x7ffb1c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9b1364bd402a81e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.460+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb28036020 con 0x7ffb1c004800 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffb28032680 con 0x7ffb1c004800 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb33e00640 1 -- 172.21.15.86:0/1762441166 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb2800b600 con 0x7ffb1c004800 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb3c600640 1 -- 172.21.15.86:0/1762441166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb1c004800 msgr2=0x7ffb1c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb3c600640 1 --2- 172.21.15.86:0/1762441166 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb1c004800 0x7ffb1c004c80 secure :-1 s=READY pgs=1430 cs=0 l=1 rev1=1 crypto rx=0x7ffb2800ba30 tx=0x7ffb1c005830 comp rx=0 tx=0).stop 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb3a600640 1 -- 172.21.15.86:0/1762441166 reap_dead start 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb3c600640 1 -- 172.21.15.86:0/1762441166 shutdown_connections 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb3c600640 1 -- 172.21.15.86:0/1762441166 >> 172.21.15.86:0/1762441166 conn(0x7ffb340b1730 msgr2=0x7ffb340b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb3c600640 1 -- 172.21.15.86:0/1762441166 shutdown_connections 2024-03-06T07:34:29.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.461+0000 7ffb3c600640 1 -- 172.21.15.86:0/1762441166 wait complete. 2024-03-06T07:34:29.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.464+0000 7ffb3c600640 1 Processor -- start 2024-03-06T07:34:29.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.464+0000 7ffb3c600640 1 -- start start 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.465+0000 7ffb3c600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c004800 0x7ffb340cc430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.465+0000 7ffb3c600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb3413f220 0x7ffb341345d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.465+0000 7ffb3c600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb341324f0 0x7ffb34132970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.465+0000 7ffb3c600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb3406cb50 con 0x7ffb3413f220 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.465+0000 7ffb3c600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb34160730 con 0x7ffb1c004800 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.465+0000 7ffb3c600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb3406ccd0 con 0x7ffb341324f0 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb3413f220 0x7ffb341345d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb3413f220 0x7ffb341345d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54746/0 (socket says 172.21.15.86:54746) 2024-03-06T07:34:29.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 -- 172.21.15.86:0/1613061315 learned_addr learned my addr 172.21.15.86:0/1613061315 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39c00640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c004800 0x7ffb340cc430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb3a600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb341324f0 0x7ffb34132970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 -- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb341324f0 msgr2=0x7ffb34132970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb341324f0 0x7ffb34132970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 -- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c004800 msgr2=0x7ffb340cc430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c004800 0x7ffb340cc430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.466+0000 7ffb39200640 1 -- 172.21.15.86:0/1613061315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb34131450 con 0x7ffb3413f220 2024-03-06T07:34:29.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.467+0000 7ffb39200640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb3413f220 0x7ffb341345d0 secure :-1 s=READY pgs=1431 cs=0 l=1 rev1=1 crypto rx=0x7ffb2800bb20 tx=0x7ffb28004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.467+0000 7ffb31600640 1 -- 172.21.15.86:0/1613061315 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb28036040 con 0x7ffb3413f220 2024-03-06T07:34:29.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.467+0000 7ffb31600640 1 -- 172.21.15.86:0/1613061315 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffb28030ac0 con 0x7ffb3413f220 2024-03-06T07:34:29.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.467+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffb34131730 con 0x7ffb3413f220 2024-03-06T07:34:29.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.467+0000 7ffb31600640 1 -- 172.21.15.86:0/1613061315 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb2800b600 con 0x7ffb3413f220 2024-03-06T07:34:29.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.467+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb34131cf0 con 0x7ffb3413f220 2024-03-06T07:34:29.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.468+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffb3404f330 con 0x7ffb3413f220 2024-03-06T07:34:29.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.469+0000 7ffb31600640 1 -- 172.21.15.86:0/1613061315 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffb28039020 con 0x7ffb3413f220 2024-03-06T07:34:29.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.471+0000 7ffb31600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffb1806b500 0x7ffb1806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.471+0000 7ffb31600640 1 -- 172.21.15.86:0/1613061315 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7ffb280acee0 con 0x7ffb3413f220 2024-03-06T07:34:29.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.471+0000 7ffb39c00640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffb1806b500 0x7ffb1806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.472+0000 7ffb39c00640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffb1806b500 0x7ffb1806d9c0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7ffb24000f60 tx=0x7ffb24009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.475+0000 7ffb31600640 1 -- 172.21.15.86:0/1613061315 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffb2807bcd0 con 0x7ffb3413f220 2024-03-06T07:34:29.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.707+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ffb34072890 con 0x7ffb3413f220 2024-03-06T07:34:29.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.708+0000 7ffb31600640 1 -- 172.21.15.86:0/1613061315 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4987 (secure 0 0 0) 0x7ffb2804fe30 con 0x7ffb3413f220 2024-03-06T07:34:29.710 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:29.711 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":84,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:33:05.831771+0000","modified":"2024-03-06T07:33:24.568795+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":114,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7722},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7722":{"gid":7722,"name":"c","rank":0,"incarnation":81,"state":"up:active","state_seq":4,"addr":"172.21.15.86:6837/2212362187","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":2212362187},{"type":"v1","addr":"172.21.15.86:6837","nonce":2212362187}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7722,"qdb_cluster":[7722]},"id":8}]} 2024-03-06T07:34:29.711 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 84 2024-03-06T07:34:29.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffb1806b500 msgr2=0x7ffb1806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffb1806b500 0x7ffb1806d9c0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7ffb24000f60 tx=0x7ffb24009040 comp rx=0 tx=0).stop 2024-03-06T07:34:29.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb3413f220 msgr2=0x7ffb341345d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb3413f220 0x7ffb341345d0 secure :-1 s=READY pgs=1431 cs=0 l=1 rev1=1 crypto rx=0x7ffb2800bb20 tx=0x7ffb28004730 comp rx=0 tx=0).stop 2024-03-06T07:34:29.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 shutdown_connections 2024-03-06T07:34:29.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ffb1806b500 0x7ffb1806d9c0 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ffb341324f0 0x7ffb34132970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ffb3413f220 0x7ffb341345d0 unknown :-1 s=CLOSED pgs=1431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 --2- 172.21.15.86:0/1613061315 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ffb1c004800 0x7ffb340cc430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.711+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 >> 172.21.15.86:0/1613061315 conn(0x7ffb340b1730 msgr2=0x7ffb3406dc90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:29.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.712+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 shutdown_connections 2024-03-06T07:34:29.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.712+0000 7ffb3c600640 1 -- 172.21.15.86:0/1613061315 wait complete. 2024-03-06T07:34:29.731 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:29.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.934+0000 7f05dba00640 1 Processor -- start 2024-03-06T07:34:29.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.934+0000 7f05dba00640 1 -- start start 2024-03-06T07:34:29.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.936+0000 7f05dba00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f05d40cff20 0x7f05d40b6ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.936+0000 7f05dba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b7860 0x7f05d40d0a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.936+0000 7f05dba00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f05d40b5e00 con 0x7f05d40b73e0 2024-03-06T07:34:29.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.936+0000 7f05dba00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f05d40b6a10 con 0x7f05d40cff20 2024-03-06T07:34:29.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.936+0000 7f05dba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05d40b60e0 con 0x7f05d40b7860 2024-03-06T07:34:29.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.936+0000 7f05d9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b7860 0x7f05d40d0a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.937+0000 7f05d9a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b7860 0x7f05d40d0a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:43884/0 (socket says 172.21.15.86:43884) 2024-03-06T07:34:29.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.937+0000 7f05d9a00640 1 -- 172.21.15.86:0/3524469898 learned_addr learned my addr 172.21.15.86:0/3524469898 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:29.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.937+0000 7f05d9000640 1 --2- 172.21.15.86:0/3524469898 >> v2:172.21.15.194:3300/0 conn(0x7f05d40cff20 0x7f05d40b6ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.937+0000 7f05d9a00640 1 -- 172.21.15.86:0/3524469898 >> v2:172.21.15.194:3300/0 conn(0x7f05d40cff20 msgr2=0x7f05d40b6ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:29.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.937+0000 7f05d9a00640 1 --2- 172.21.15.86:0/3524469898 >> v2:172.21.15.194:3300/0 conn(0x7f05d40cff20 0x7f05d40b6ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.937+0000 7f05d9a00640 1 -- 172.21.15.86:0/3524469898 >> v1:172.21.15.194:6789/0 conn(0x7f05d40b73e0 legacy=0x7f05d40d0300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.939 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.937+0000 7f05d9a00640 1 -- 172.21.15.86:0/3524469898 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05d40b63c0 con 0x7f05d40b7860 2024-03-06T07:34:29.940 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.939+0000 7f05d9a00640 1 --2- 172.21.15.86:0/3524469898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b7860 0x7f05d40d0a10 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f05cc009e50 tx=0x7f05cc02bf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5e66b54dd878b9fa server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.939+0000 7f05d3e00640 1 -- 172.21.15.86:0/3524469898 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05cc02cb00 con 0x7f05d40b7860 2024-03-06T07:34:29.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.939+0000 7f05d3e00640 1 -- 172.21.15.86:0/3524469898 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f05cc02cca0 con 0x7f05d40b7860 2024-03-06T07:34:29.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.939+0000 7f05d3e00640 1 -- 172.21.15.86:0/3524469898 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05cc034610 con 0x7f05d40b7860 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 -- 172.21.15.86:0/3524469898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b7860 msgr2=0x7f05d40d0a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 --2- 172.21.15.86:0/3524469898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b7860 0x7f05d40d0a10 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f05cc009e50 tx=0x7f05cc02bf60 comp rx=0 tx=0).stop 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 -- 172.21.15.86:0/3524469898 shutdown_connections 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 --2- 172.21.15.86:0/3524469898 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b7860 0x7f05d40d0a10 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 --2- 172.21.15.86:0/3524469898 >> v2:172.21.15.194:3300/0 conn(0x7f05d40cff20 0x7f05d40b6ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 -- 172.21.15.86:0/3524469898 >> 172.21.15.86:0/3524469898 conn(0x7f05d40d1c90 msgr2=0x7f05d40d20a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 -- 172.21.15.86:0/3524469898 shutdown_connections 2024-03-06T07:34:29.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.940+0000 7f05dba00640 1 -- 172.21.15.86:0/3524469898 wait complete. 2024-03-06T07:34:29.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.942+0000 7f05dba00640 1 Processor -- start 2024-03-06T07:34:29.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.942+0000 7f05dba00640 1 -- start start 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05dba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b73e0 0x7f05d4146f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05dba00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05d40b7860 0x7f05d415cc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05dba00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05d40cff20 0x7f05d415d190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05dba00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05d4161a50 con 0x7f05d40b7860 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05dba00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05d40b5af0 con 0x7f05d40cff20 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05dba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05d40d0f50 con 0x7f05d40b73e0 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05cbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05d40b7860 0x7f05d415cc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05cbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05d40b7860 0x7f05d415cc50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54750/0 (socket says 172.21.15.86:54750) 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.943+0000 7f05cbe00640 1 -- 172.21.15.86:0/806453215 learned_addr learned my addr 172.21.15.86:0/806453215 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05d9000640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b73e0 0x7f05d4146f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05cbe00640 1 -- 172.21.15.86:0/806453215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b73e0 msgr2=0x7f05d4146f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05cbe00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b73e0 0x7f05d4146f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05cbe00640 1 -- 172.21.15.86:0/806453215 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05d40cff20 msgr2=0x7f05d415d190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05cbe00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05d40cff20 0x7f05d415d190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05cbe00640 1 -- 172.21.15.86:0/806453215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05d409c7e0 con 0x7f05d40b7860 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05cbe00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05d40b7860 0x7f05d415cc50 secure :-1 s=READY pgs=1432 cs=0 l=1 rev1=1 crypto rx=0x7f05bc00c400 tx=0x7f05bc00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05d1600640 1 -- 172.21.15.86:0/806453215 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05bc0052d0 con 0x7f05d40b7860 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05d1600640 1 -- 172.21.15.86:0/806453215 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f05bc005490 con 0x7f05d40b7860 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05d1600640 1 -- 172.21.15.86:0/806453215 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05bc007260 con 0x7f05d40b7860 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f05d4147960 con 0x7f05d40b7860 2024-03-06T07:34:29.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.944+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f05d4147f20 con 0x7f05d40b7860 2024-03-06T07:34:29.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.946+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f05d4148120 con 0x7f05d40b7860 2024-03-06T07:34:29.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.947+0000 7f05d1600640 1 -- 172.21.15.86:0/806453215 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f05bc007400 con 0x7f05d40b7860 2024-03-06T07:34:29.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.948+0000 7f05d1600640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05b006b4d0 0x7f05b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:29.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.948+0000 7f05d1600640 1 -- 172.21.15.86:0/806453215 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f05bc089750 con 0x7f05d40b7860 2024-03-06T07:34:29.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.948+0000 7f05d9000640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05b006b4d0 0x7f05b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:29.950 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.949+0000 7f05d9000640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05b006b4d0 0x7f05b006d990 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f05d41f75e0 tx=0x7f05c4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:29.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:29.953+0000 7f05d1600640 1 -- 172.21.15.86:0/806453215 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f05d4148120 con 0x7f05d40b7860 2024-03-06T07:34:30.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.168+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f05d404f330 con 0x7f05d40b7860 2024-03-06T07:34:30.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.171+0000 7f05d1600640 1 -- 172.21.15.86:0/806453215 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v116) v1 ==== 74+0+16920 (secure 0 0 0) 0x7f05bc0585d0 con 0x7f05d40b7860 2024-03-06T07:34:30.173 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:30.173 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":116,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:33:55.692842+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:33:03.630003+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"109","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-06T07:33:04.861178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"110","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.3f","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/221976790":"2024-03-07T07:33:20.538234+0000","172.21.15.86:6836/221976790":"2024-03-07T07:33:20.538234+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05b006b4d0 msgr2=0x7f05b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05b006b4d0 0x7f05b006d990 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f05d41f75e0 tx=0x7f05c4008040 comp rx=0 tx=0).stop 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05d40b7860 msgr2=0x7f05d415cc50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05d40b7860 0x7f05d415cc50 secure :-1 s=READY pgs=1432 cs=0 l=1 rev1=1 crypto rx=0x7f05bc00c400 tx=0x7f05bc00c8c0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 shutdown_connections 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05b006b4d0 0x7f05b006d990 unknown :-1 s=CLOSED pgs=613 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05d40cff20 0x7f05d415d190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05d40b7860 0x7f05d415cc50 unknown :-1 s=CLOSED pgs=1432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 --2- 172.21.15.86:0/806453215 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05d40b73e0 0x7f05d4146f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.173+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 >> 172.21.15.86:0/806453215 conn(0x7f05d40d1c90 msgr2=0x7f05d4160ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:30.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.174+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 shutdown_connections 2024-03-06T07:34:30.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.174+0000 7f05dba00640 1 -- 172.21.15.86:0/806453215 wait complete. 2024-03-06T07:34:30.195 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-06T07:34:30.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.388+0000 7f1844000640 1 Processor -- start 2024-03-06T07:34:30.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.389+0000 7f1844000640 1 -- start start 2024-03-06T07:34:30.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.390+0000 7f1844000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f183c0c6460 0x7f183c0c5420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.390+0000 7f1844000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f183c145b30 con 0x7f183c0c4930 2024-03-06T07:34:30.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.390+0000 7f1844000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f183c142ce0 con 0x7f183c0c5960 2024-03-06T07:34:30.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.390+0000 7f1844000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f183c0c68c0 con 0x7f183c0c6460 2024-03-06T07:34:30.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.390+0000 7f1841600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f183c0c4930 0x7f183c0c4d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42980/0 (socket says 172.21.15.86:42980) 2024-03-06T07:34:30.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.390+0000 7f1841600640 1 -- 172.21.15.86:0/3312776883 learned_addr learned my addr 172.21.15.86:0/3312776883 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:30.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1458297993 0 0) 0x7f1830001110 con 0x7f183c0c4930 2024-03-06T07:34:30.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1824003520 con 0x7f183c0c4930 2024-03-06T07:34:30.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f1840c00640 1 --2- 172.21.15.86:0/3312776883 >> v2:172.21.15.194:3300/0 conn(0x7f183c0c6460 0x7f183c0c5420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2839342109 0 0) 0x7f1824003520 con 0x7f183c0c4930 2024-03-06T07:34:30.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 >> v2:172.21.15.194:3300/0 conn(0x7f183c0c6460 msgr2=0x7f183c0c5420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:30.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 --2- 172.21.15.86:0/3312776883 >> v2:172.21.15.194:3300/0 conn(0x7f183c0c6460 0x7f183c0c5420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 >> v1:172.21.15.194:6789/0 conn(0x7f183c0c5960 legacy=0x7f183c165b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f183c0cbca0 con 0x7f183c0c4930 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1830001840 con 0x7f183c0c4930 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 >> v1:172.21.15.86:6789/0 conn(0x7f183c0c4930 legacy=0x7f183c0c4d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1824003a00 0x7f1824003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1824004800 0x7f1824004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f182400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f183c0c68c0 con 0x7f18240052e0 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1824010610 con 0x7f1824004800 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.391+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1824010810 con 0x7f1824003a00 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1842000640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f182400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1840c00640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1824004800 0x7f1824004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1841600640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1824003a00 0x7f1824003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1842000640 1 -- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1824003a00 msgr2=0x7f1824003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:30.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1842000640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1824003a00 0x7f1824003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1842000640 1 -- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1824004800 msgr2=0x7f1824004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:30.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1842000640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1824004800 0x7f1824004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.392+0000 7f1842000640 1 -- 172.21.15.86:0/3312776883 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1824003520 con 0x7f18240052e0 2024-03-06T07:34:30.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.393+0000 7f1842000640 1 -- 172.21.15.86:0/3312776883 reap_dead start 2024-03-06T07:34:30.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.393+0000 7f1842000640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f182400b9c0 secure :-1 s=READY pgs=1434 cs=0 l=1 rev1=1 crypto rx=0x7f1834001e50 tx=0x7f1824005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c3eee44fe1e606f2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:30.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1834001a70 con 0x7f18240052e0 2024-03-06T07:34:30.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1834003bb0 con 0x7f18240052e0 2024-03-06T07:34:30.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f183be00640 1 -- 172.21.15.86:0/3312776883 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18340050d0 con 0x7f18240052e0 2024-03-06T07:34:30.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f1844000640 1 -- 172.21.15.86:0/3312776883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 msgr2=0x7f182400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f1844000640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f182400b9c0 secure :-1 s=READY pgs=1434 cs=0 l=1 rev1=1 crypto rx=0x7f1834001e50 tx=0x7f1824005830 comp rx=0 tx=0).stop 2024-03-06T07:34:30.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f1844000640 1 -- 172.21.15.86:0/3312776883 shutdown_connections 2024-03-06T07:34:30.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f1844000640 1 --2- 172.21.15.86:0/3312776883 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f182400b9c0 unknown :-1 s=CLOSED pgs=1434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f1844000640 1 -- 172.21.15.86:0/3312776883 >> 172.21.15.86:0/3312776883 conn(0x7f183c0cc4f0 msgr2=0x7f183c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:30.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f1844000640 1 -- 172.21.15.86:0/3312776883 shutdown_connections 2024-03-06T07:34:30.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.394+0000 7f1844000640 1 -- 172.21.15.86:0/3312776883 wait complete. 2024-03-06T07:34:30.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.397+0000 7f1844000640 1 Processor -- start 2024-03-06T07:34:30.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.397+0000 7f1844000640 1 -- start start 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1844000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f183c13e510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1844000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f183c0c9020 0x7f183c0c94a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1844000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f183c0c6fe0 0x7f183c0c7490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1844000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f183c0ca3b0 con 0x7f18240052e0 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1844000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f183c0ca0d0 con 0x7f183c0c6fe0 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1844000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f183c0cc260 con 0x7f183c0c9020 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f183c13e510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f183c13e510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54766/0 (socket says 172.21.15.86:54766) 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 -- 172.21.15.86:0/3233768847 learned_addr learned my addr 172.21.15.86:0/3233768847 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 -- 172.21.15.86:0/3233768847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f183c0c9020 msgr2=0x7f183c0c94a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f183c0c9020 0x7f183c0c94a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 -- 172.21.15.86:0/3233768847 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f183c0c6fe0 msgr2=0x7f183c0c7490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f183c0c6fe0 0x7f183c0c7490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.399+0000 7f1841600640 1 -- 172.21.15.86:0/3233768847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f183c071770 con 0x7f18240052e0 2024-03-06T07:34:30.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.400+0000 7f1841600640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f183c13e510 secure :-1 s=READY pgs=1435 cs=0 l=1 rev1=1 crypto rx=0x7f1830001050 tx=0x7f183000ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:30.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.400+0000 7f1839600640 1 -- 172.21.15.86:0/3233768847 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f183000ee70 con 0x7f18240052e0 2024-03-06T07:34:30.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.400+0000 7f1839600640 1 -- 172.21.15.86:0/3233768847 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1830018070 con 0x7f18240052e0 2024-03-06T07:34:30.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.400+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f183c071a50 con 0x7f18240052e0 2024-03-06T07:34:30.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.400+0000 7f1839600640 1 -- 172.21.15.86:0/3233768847 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f183000cab0 con 0x7f18240052e0 2024-03-06T07:34:30.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.400+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f183c072010 con 0x7f18240052e0 2024-03-06T07:34:30.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.401+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f183c04f330 con 0x7f18240052e0 2024-03-06T07:34:30.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.402+0000 7f1839600640 1 -- 172.21.15.86:0/3233768847 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f183000cc50 con 0x7f18240052e0 2024-03-06T07:34:30.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.403+0000 7f1839600640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f181c06b500 0x7f181c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.404+0000 7f1839600640 1 -- 172.21.15.86:0/3233768847 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f183008a0f0 con 0x7f18240052e0 2024-03-06T07:34:30.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.404+0000 7f1840c00640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f181c06b500 0x7f181c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.404+0000 7f1840c00640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f181c06b500 0x7f181c06d9c0 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f183c15ca70 tx=0x7f182800d280 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:30.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.407+0000 7f1839600640 1 -- 172.21.15.86:0/3233768847 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f183c145b30 con 0x7f18240052e0 2024-03-06T07:34:30.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.639+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f183c15e540 con 0x7f18240052e0 2024-03-06T07:34:30.641 INFO:tasks.ceph.mon.a.smithi086.stderr:2024-03-06T07:34:30.640+0000 7f41a6600640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-06T07:34:30.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.643+0000 7f1839600640 1 -- 172.21.15.86:0/3233768847 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v85) v1 ==== 161+0+0 (secure 0 0 0) 0x7f1830058ed0 con 0x7f18240052e0 2024-03-06T07:34:30.644 INFO:teuthology.orchestra.run.smithi086.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.645+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f181c06b500 msgr2=0x7f181c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f181c06b500 0x7f181c06d9c0 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f183c15ca70 tx=0x7f182800d280 comp rx=0 tx=0).stop 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 msgr2=0x7f183c13e510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f183c13e510 secure :-1 s=READY pgs=1435 cs=0 l=1 rev1=1 crypto rx=0x7f1830001050 tx=0x7f183000ec80 comp rx=0 tx=0).stop 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 shutdown_connections 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f181c06b500 0x7f181c06d9c0 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f183c0c6fe0 0x7f183c0c7490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f183c0c9020 0x7f183c0c94a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 --2- 172.21.15.86:0/3233768847 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f18240052e0 0x7f183c13e510 unknown :-1 s=CLOSED pgs=1435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 >> 172.21.15.86:0/3233768847 conn(0x7f183c0cc4f0 msgr2=0x7f183c0cef60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.646+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 shutdown_connections 2024-03-06T07:34:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.647+0000 7f1844000640 1 -- 172.21.15.86:0/3233768847 wait complete. 2024-03-06T07:34:30.661 INFO:tasks.ceph.mds.c.smithi086.stderr: -8201> 2024-03-06T07:33:26.566+0000 7f9c21600640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:34:30.665 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-06T07:34:30.730 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:34:30.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.851+0000 7f9887200640 1 Processor -- start 2024-03-06T07:34:30.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.851+0000 7f9887200640 1 -- start start 2024-03-06T07:34:30.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.852+0000 7f9887200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f98800b3650 0x7f98800727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.852+0000 7f9887200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f98800cda90 0x7f988006c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f9887200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f98800719e0 con 0x7f98800b3df0 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f9887200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9880072310 con 0x7f98800b3650 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f9887200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9880071cc0 con 0x7f98800cda90 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f987f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f98800b3650 0x7f98800727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f987f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f98800b3650 0x7f98800727a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54786/0 (socket says 172.21.15.86:54786) 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f987f400640 1 -- 172.21.15.86:0/4118440755 learned_addr learned my addr 172.21.15.86:0/4118440755 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f987f400640 1 -- 172.21.15.86:0/4118440755 >> v2:172.21.15.194:3300/0 conn(0x7f98800cda90 msgr2=0x7f988006c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f987f400640 1 --2- 172.21.15.86:0/4118440755 >> v2:172.21.15.194:3300/0 conn(0x7f98800cda90 0x7f988006c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f987f400640 1 -- 172.21.15.86:0/4118440755 >> v1:172.21.15.194:6789/0 conn(0x7f98800b3df0 legacy=0x7f98800b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:30.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.853+0000 7f987f400640 1 -- 172.21.15.86:0/4118440755 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9880071fa0 con 0x7f98800b3650 2024-03-06T07:34:30.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.854+0000 7f987f400640 1 --2- 172.21.15.86:0/4118440755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800b3650 0x7f98800727a0 secure :-1 s=READY pgs=1438 cs=0 l=1 rev1=1 crypto rx=0x7f988006ed00 tx=0x7f986802db20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=77bc9b3c1ff65668 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:30.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f987ea00640 1 -- 172.21.15.86:0/4118440755 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f986802f8e0 con 0x7f98800b3650 2024-03-06T07:34:30.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f987ea00640 1 -- 172.21.15.86:0/4118440755 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f986802fa80 con 0x7f98800b3650 2024-03-06T07:34:30.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f987ea00640 1 -- 172.21.15.86:0/4118440755 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f986802fe50 con 0x7f98800b3650 2024-03-06T07:34:30.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 -- 172.21.15.86:0/4118440755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800b3650 msgr2=0x7f98800727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 --2- 172.21.15.86:0/4118440755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800b3650 0x7f98800727a0 secure :-1 s=READY pgs=1438 cs=0 l=1 rev1=1 crypto rx=0x7f988006ed00 tx=0x7f986802db20 comp rx=0 tx=0).stop 2024-03-06T07:34:30.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 -- 172.21.15.86:0/4118440755 shutdown_connections 2024-03-06T07:34:30.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 --2- 172.21.15.86:0/4118440755 >> v2:172.21.15.194:3300/0 conn(0x7f98800cda90 0x7f988006c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 --2- 172.21.15.86:0/4118440755 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800b3650 0x7f98800727a0 unknown :-1 s=CLOSED pgs=1438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 -- 172.21.15.86:0/4118440755 >> 172.21.15.86:0/4118440755 conn(0x7f98800b1940 msgr2=0x7f98800b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 -- 172.21.15.86:0/4118440755 shutdown_connections 2024-03-06T07:34:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.855+0000 7f9887200640 1 -- 172.21.15.86:0/4118440755 wait complete. 2024-03-06T07:34:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.857+0000 7f9887200640 1 Processor -- start 2024-03-06T07:34:30.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.857+0000 7f9887200640 1 -- start start 2024-03-06T07:34:30.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9887200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f98800b3650 0x7f98800c9970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9887200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f98800b3df0 0x7f98800cb0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9887200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800cda90 0x7f988009beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9887200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f988015b3c0 con 0x7f98800cda90 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9887200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f988015b240 con 0x7f98800b3650 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9887200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f98800cf510 con 0x7f98800b3df0 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800cda90 0x7f988009beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800cda90 0x7f988009beb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54800/0 (socket says 172.21.15.86:54800) 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 -- 172.21.15.86:0/764751013 learned_addr learned my addr 172.21.15.86:0/764751013 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 -- 172.21.15.86:0/764751013 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f98800b3df0 msgr2=0x7f98800cb0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f98800b3df0 0x7f98800cb0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 -- 172.21.15.86:0/764751013 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f98800b3650 msgr2=0x7f98800c9970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:30.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f98800b3650 0x7f98800c9970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.858+0000 7f9885200640 1 -- 172.21.15.86:0/764751013 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f98800ca040 con 0x7f98800cda90 2024-03-06T07:34:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.859+0000 7f9885200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800cda90 0x7f988009beb0 secure :-1 s=READY pgs=1439 cs=0 l=1 rev1=1 crypto rx=0x7f987800b970 tx=0x7f987800be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.859+0000 7f9877e00640 1 -- 172.21.15.86:0/764751013 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9878019070 con 0x7f98800cda90 2024-03-06T07:34:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.859+0000 7f9877e00640 1 -- 172.21.15.86:0/764751013 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f987800cb60 con 0x7f98800cda90 2024-03-06T07:34:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.859+0000 7f9877e00640 1 -- 172.21.15.86:0/764751013 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9878014440 con 0x7f98800cda90 2024-03-06T07:34:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.859+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f98800ca320 con 0x7f98800cda90 2024-03-06T07:34:30.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.859+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f98800ca8e0 con 0x7f98800cda90 2024-03-06T07:34:30.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.860+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9880150f50 con 0x7f98800cda90 2024-03-06T07:34:30.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.861+0000 7f9877e00640 1 -- 172.21.15.86:0/764751013 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f987801c070 con 0x7f98800cda90 2024-03-06T07:34:30.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.862+0000 7f9877e00640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f985c06b4d0 0x7f985c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:30.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.862+0000 7f9877e00640 1 -- 172.21.15.86:0/764751013 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(117..117 src has 1..117) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f987808a0d0 con 0x7f98800cda90 2024-03-06T07:34:30.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.862+0000 7f987fe00640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f985c06b4d0 0x7f985c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:30.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.863+0000 7f987fe00640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f985c06b4d0 0x7f985c06d990 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f9870009650 tx=0x7f9870006ba0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:30.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:30.867+0000 7f9877e00640 1 -- 172.21.15.86:0/764751013 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9878058dc0 con 0x7f98800cda90 2024-03-06T07:34:31.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.084+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f988004f330 con 0x7f98800cda90 2024-03-06T07:34:31.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.646+0000 7f9877e00640 1 -- 172.21.15.86:0/764751013 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v86) v1 ==== 104+0+0 (secure 0 0 0) 0x7f98780586c0 con 0x7f98800cda90 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f985c06b4d0 msgr2=0x7f985c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f985c06b4d0 0x7f985c06d990 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f9870009650 tx=0x7f9870006ba0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800cda90 msgr2=0x7f988009beb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800cda90 0x7f988009beb0 secure :-1 s=READY pgs=1439 cs=0 l=1 rev1=1 crypto rx=0x7f987800b970 tx=0x7f987800be30 comp rx=0 tx=0).stop 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 shutdown_connections 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f985c06b4d0 0x7f985c06d990 unknown :-1 s=CLOSED pgs=615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f98800cda90 0x7f988009beb0 unknown :-1 s=CLOSED pgs=1439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f98800b3df0 0x7f98800cb0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 --2- 172.21.15.86:0/764751013 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f98800b3650 0x7f98800c9970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 >> 172.21.15.86:0/764751013 conn(0x7f98800b1940 msgr2=0x7f9880156bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:31.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 shutdown_connections 2024-03-06T07:34:31.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.650+0000 7f9887200640 1 -- 172.21.15.86:0/764751013 wait complete. 2024-03-06T07:34:31.671 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-06T07:34:31.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.876+0000 7fcaec800640 1 Processor -- start 2024-03-06T07:34:31.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.876+0000 7fcaec800640 1 -- start start 2024-03-06T07:34:31.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcaec800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fcae40cea90 0x7fcae40cef20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcaec800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcae41405c0 con 0x7fcae415b3c0 2024-03-06T07:34:31.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcaec800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcae41408a0 con 0x7fcae413dd40 2024-03-06T07:34:31.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcaec800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcae4140b80 con 0x7fcae40cea90 2024-03-06T07:34:31.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcaea800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fcae415b3c0 0x7fcae415b830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42988/0 (socket says 172.21.15.86:42988) 2024-03-06T07:34:31.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcaea800640 1 -- 172.21.15.86:0/206865829 learned_addr learned my addr 172.21.15.86:0/206865829 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:31.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1844245003 0 0) 0x7fcae41405c0 con 0x7fcae415b3c0 2024-03-06T07:34:31.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.878+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcacc003520 con 0x7fcae415b3c0 2024-03-06T07:34:31.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae9400640 1 --2- 172.21.15.86:0/206865829 >> v2:172.21.15.194:3300/0 conn(0x7fcae40cea90 0x7fcae40cef20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1386060895 0 0) 0x7fcacc003520 con 0x7fcae415b3c0 2024-03-06T07:34:31.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 >> v2:172.21.15.194:3300/0 conn(0x7fcae40cea90 msgr2=0x7fcae40cef20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:31.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 --2- 172.21.15.86:0/206865829 >> v2:172.21.15.194:3300/0 conn(0x7fcae40cea90 0x7fcae40cef20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 >> v1:172.21.15.194:6789/0 conn(0x7fcae413dd40 legacy=0x7fcae40b2b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcae4140e60 con 0x7fcae415b3c0 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fcae00019f0 con 0x7fcae415b3c0 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 >> v1:172.21.15.86:6789/0 conn(0x7fcae415b3c0 legacy=0x7fcae415b830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcacc003a20 0x7fcacc003e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcacc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcacc005230 0x7fcacc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcae4140b80 con 0x7fcacc004800 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcacc00c190 con 0x7fcacc005230 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcacc00c390 con 0x7fcacc003a20 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.879+0000 7fcae9400640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcacc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcae9e00640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcacc003a20 0x7fcacc003e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcae9400640 1 -- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcacc003a20 msgr2=0x7fcacc003e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcaea800640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcacc005230 0x7fcacc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcae9400640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcacc003a20 0x7fcacc003e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcae9400640 1 -- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcacc005230 msgr2=0x7fcacc00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:31.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcae9400640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcacc005230 0x7fcacc00b9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcaea800640 1 -- 172.21.15.86:0/206865829 reap_dead start 2024-03-06T07:34:31.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcae9400640 1 -- 172.21.15.86:0/206865829 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcacc003520 con 0x7fcacc004800 2024-03-06T07:34:31.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.880+0000 7fcae9e00640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcacc003a20 0x7fcacc003e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:31.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.882+0000 7fcae9400640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcacc004c80 secure :-1 s=READY pgs=1441 cs=0 l=1 rev1=1 crypto rx=0x7fcad400b5c0 tx=0x7fcacc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=389a68b0e0359386 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:31.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.883+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad4034070 con 0x7fcacc004800 2024-03-06T07:34:31.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.883+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcad402e730 con 0x7fcacc004800 2024-03-06T07:34:31.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.883+0000 7fcae8a00640 1 -- 172.21.15.86:0/206865829 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad400b600 con 0x7fcacc004800 2024-03-06T07:34:31.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.883+0000 7fcaec800640 1 -- 172.21.15.86:0/206865829 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 msgr2=0x7fcacc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:31.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.883+0000 7fcaec800640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcacc004c80 secure :-1 s=READY pgs=1441 cs=0 l=1 rev1=1 crypto rx=0x7fcad400b5c0 tx=0x7fcacc005830 comp rx=0 tx=0).stop 2024-03-06T07:34:31.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.884+0000 7fcaec800640 1 -- 172.21.15.86:0/206865829 shutdown_connections 2024-03-06T07:34:31.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.884+0000 7fcaec800640 1 --2- 172.21.15.86:0/206865829 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcacc004c80 unknown :-1 s=CLOSED pgs=1441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.884+0000 7fcaec800640 1 -- 172.21.15.86:0/206865829 >> 172.21.15.86:0/206865829 conn(0x7fcae40cfa80 msgr2=0x7fcae40cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:31.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.884+0000 7fcaec800640 1 -- 172.21.15.86:0/206865829 shutdown_connections 2024-03-06T07:34:31.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.884+0000 7fcaec800640 1 -- 172.21.15.86:0/206865829 wait complete. 2024-03-06T07:34:31.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.885+0000 7fcaec800640 1 Processor -- start 2024-03-06T07:34:31.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.886+0000 7fcaec800640 1 -- start start 2024-03-06T07:34:31.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.886+0000 7fcaec800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcae40b1610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.886+0000 7fcaec800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcae414cd30 0x7fcae413d3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.886+0000 7fcaec800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcae4154d10 0x7fcae4155190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.886+0000 7fcaec800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcae40b35b0 con 0x7fcacc004800 2024-03-06T07:34:31.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.886+0000 7fcaec800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcae4141380 con 0x7fcae414cd30 2024-03-06T07:34:31.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.886+0000 7fcaec800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcae40b3730 con 0x7fcae4154d10 2024-03-06T07:34:31.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcae40b1610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcae40b1610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54816/0 (socket says 172.21.15.86:54816) 2024-03-06T07:34:31.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 -- 172.21.15.86:0/3853043379 learned_addr learned my addr 172.21.15.86:0/3853043379 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:31.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 -- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcae4154d10 msgr2=0x7fcae4155190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:31.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcaea800640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcae4154d10 0x7fcae4155190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9400640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcae414cd30 0x7fcae413d3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcae4154d10 0x7fcae4155190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 -- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcae414cd30 msgr2=0x7fcae413d3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:31.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcae414cd30 0x7fcae413d3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:31.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 -- 172.21.15.86:0/3853043379 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcae40b1e40 con 0x7fcacc004800 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.887+0000 7fcae9e00640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcae40b1610 secure :-1 s=READY pgs=1442 cs=0 l=1 rev1=1 crypto rx=0x7fcad800d4d0 tx=0x7fcad800d990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.888+0000 7fcade000640 1 -- 172.21.15.86:0/3853043379 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad8019070 con 0x7fcacc004800 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.888+0000 7fcade000640 1 -- 172.21.15.86:0/3853043379 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcad8005480 con 0x7fcacc004800 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.888+0000 7fcade000640 1 -- 172.21.15.86:0/3853043379 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad8007230 con 0x7fcacc004800 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.888+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcae414d410 con 0x7fcacc004800 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.888+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcae414d9d0 con 0x7fcacc004800 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.889+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcae404f3a0 con 0x7fcacc004800 2024-03-06T07:34:31.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.890+0000 7fcade000640 1 -- 172.21.15.86:0/3853043379 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcad80073d0 con 0x7fcacc004800 2024-03-06T07:34:31.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.891+0000 7fcade000640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcac806b500 0x7fcac806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:31.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.891+0000 7fcade000640 1 -- 172.21.15.86:0/3853043379 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(117..117 src has 1..117) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fcad80898e0 con 0x7fcacc004800 2024-03-06T07:34:31.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.891+0000 7fcae9400640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcac806b500 0x7fcac806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:31.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.892+0000 7fcae9400640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcac806b500 0x7fcac806d9c0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fcad400ba00 tx=0x7fcad4034000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:31.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:31.895+0000 7fcade000640 1 -- 172.21.15.86:0/3853043379 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcad8058660 con 0x7fcacc004800 2024-03-06T07:34:32.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.098+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fcae41583e0 con 0x7fcacc004800 2024-03-06T07:34:32.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.892+0000 7fcade000640 1 -- 172.21.15.86:0/3853043379 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v118) v1 ==== 181+0+0 (secure 0 0 0) 0x7fcad8057f60 con 0x7fcacc004800 2024-03-06T07:34:32.894 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' removed 2024-03-06T07:34:32.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.897+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcac806b500 msgr2=0x7fcac806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:32.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcac806b500 0x7fcac806d9c0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fcad400ba00 tx=0x7fcad4034000 comp rx=0 tx=0).stop 2024-03-06T07:34:32.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 msgr2=0x7fcae40b1610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:32.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcae40b1610 secure :-1 s=READY pgs=1442 cs=0 l=1 rev1=1 crypto rx=0x7fcad800d4d0 tx=0x7fcad800d990 comp rx=0 tx=0).stop 2024-03-06T07:34:32.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 shutdown_connections 2024-03-06T07:34:32.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fcac806b500 0x7fcac806d9c0 unknown :-1 s=CLOSED pgs=617 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:32.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fcae4154d10 0x7fcae4155190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:32.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fcae414cd30 0x7fcae413d3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:32.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 --2- 172.21.15.86:0/3853043379 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fcacc004800 0x7fcae40b1610 unknown :-1 s=CLOSED pgs=1442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:32.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.898+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 >> 172.21.15.86:0/3853043379 conn(0x7fcae40cfa80 msgr2=0x7fcae4157360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:32.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.899+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 shutdown_connections 2024-03-06T07:34:32.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:32.899+0000 7fcaec800640 1 -- 172.21.15.86:0/3853043379 wait complete. 2024-03-06T07:34:32.922 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-06T07:34:33.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.133+0000 7fa954a00640 1 Processor -- start 2024-03-06T07:34:33.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.133+0000 7fa954a00640 1 -- start start 2024-03-06T07:34:33.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.135+0000 7fa954a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa950144940 0x7fa950070d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.135+0000 7fa954a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 0x7fa950071860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.135+0000 7fa954a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa950072090 con 0x7fa95006afc0 2024-03-06T07:34:33.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.135+0000 7fa954a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa95015ffe0 con 0x7fa950144940 2024-03-06T07:34:33.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.135+0000 7fa954a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9501608d0 con 0x7fa950160470 2024-03-06T07:34:33.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.135+0000 7fa94f200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa95006afc0 0x7fa9500b3640 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42998/0 (socket says 172.21.15.86:42998) 2024-03-06T07:34:33.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.135+0000 7fa94f200640 1 -- 172.21.15.86:0/3686780687 learned_addr learned my addr 172.21.15.86:0/3686780687 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:33.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1480409291 0 0) 0x7fa950072090 con 0x7fa95006afc0 2024-03-06T07:34:33.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa934003520 con 0x7fa95006afc0 2024-03-06T07:34:33.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94de00640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 0x7fa950071860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94e800640 1 --2- 172.21.15.86:0/3686780687 >> v2:172.21.15.194:3300/0 conn(0x7fa950144940 0x7fa950070d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 668087552 0 0) 0x7fa934003520 con 0x7fa95006afc0 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 msgr2=0x7fa950071860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 0x7fa950071860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 >> v2:172.21.15.194:3300/0 conn(0x7fa950144940 msgr2=0x7fa950070d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 --2- 172.21.15.86:0/3686780687 >> v2:172.21.15.194:3300/0 conn(0x7fa950144940 0x7fa950070d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa950072370 con 0x7fa95006afc0 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fa9480019f0 con 0x7fa95006afc0 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 >> v1:172.21.15.86:6789/0 conn(0x7fa95006afc0 legacy=0x7fa9500b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9340039b0 0x7fa934003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 0x7fa934004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa9340051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa95015ffe0 con 0x7fa934004d40 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9501608d0 con 0x7fa9340039b0 2024-03-06T07:34:33.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa93400b9c0 con 0x7fa950160470 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.136+0000 7fa94f200640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa9340051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.137+0000 7fa94e800640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9340039b0 0x7fa934003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.137+0000 7fa94de00640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 0x7fa934004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.137+0000 7fa94f200640 1 -- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 msgr2=0x7fa934004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.137+0000 7fa94f200640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa950160470 0x7fa934004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.137+0000 7fa94f200640 1 -- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9340039b0 msgr2=0x7fa934003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.137+0000 7fa94f200640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9340039b0 0x7fa934003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.137+0000 7fa94f200640 1 -- 172.21.15.86:0/3686780687 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa934003520 con 0x7fa934004d40 2024-03-06T07:34:33.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.138+0000 7fa94f200640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa9340051c0 secure :-1 s=READY pgs=1444 cs=0 l=1 rev1=1 crypto rx=0x7fa94800a8b0 tx=0x7fa934005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=40690a69deb247a9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:33.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.138+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa94800a4a0 con 0x7fa934004d40 2024-03-06T07:34:33.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.138+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa948034630 con 0x7fa934004d40 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.138+0000 7fa94d400640 1 -- 172.21.15.86:0/3686780687 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa948034a60 con 0x7fa934004d40 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.139+0000 7fa954a00640 1 -- 172.21.15.86:0/3686780687 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 msgr2=0x7fa9340051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.139+0000 7fa954a00640 1 --2- 172.21.15.86:0/3686780687 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa9340051c0 secure :-1 s=READY pgs=1444 cs=0 l=1 rev1=1 crypto rx=0x7fa94800a8b0 tx=0x7fa934005830 comp rx=0 tx=0).stop 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.139+0000 7fa94f200640 1 -- 172.21.15.86:0/3686780687 reap_dead start 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.139+0000 7fa954a00640 1 -- 172.21.15.86:0/3686780687 shutdown_connections 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.139+0000 7fa954a00640 1 -- 172.21.15.86:0/3686780687 >> 172.21.15.86:0/3686780687 conn(0x7fa9500b18a0 msgr2=0x7fa9500b1cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.139+0000 7fa954a00640 1 -- 172.21.15.86:0/3686780687 shutdown_connections 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.139+0000 7fa954a00640 1 -- 172.21.15.86:0/3686780687 wait complete. 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.141+0000 7fa954a00640 1 Processor -- start 2024-03-06T07:34:33.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.141+0000 7fa954a00640 1 -- start start 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa954a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa95013a470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa954a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9501f7b70 0x7fa9500ce790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa954a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9500cecd0 0x7fa9500ad860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa954a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa950144d20 con 0x7fa934004d40 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa954a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa95015fc70 con 0x7fa9500cecd0 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa954a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa950144ea0 con 0x7fa9501f7b70 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa94e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa95013a470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa94e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa95013a470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54826/0 (socket says 172.21.15.86:54826) 2024-03-06T07:34:33.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa94e800640 1 -- 172.21.15.86:0/3427910008 learned_addr learned my addr 172.21.15.86:0/3427910008 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa94f200640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9500cecd0 0x7fa9500ad860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa94e800640 1 -- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9501f7b70 msgr2=0x7fa9500ce790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.142+0000 7fa94de00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9501f7b70 0x7fa9500ce790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa94e800640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9501f7b70 0x7fa9500ce790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa94e800640 1 -- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9500cecd0 msgr2=0x7fa9500ad860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa94e800640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9500cecd0 0x7fa9500ad860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa94e800640 1 -- 172.21.15.86:0/3427910008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa95015c540 con 0x7fa934004d40 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa94f200640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9500cecd0 0x7fa9500ad860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa94e800640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa95013a470 secure :-1 s=READY pgs=1445 cs=0 l=1 rev1=1 crypto rx=0x7fa940011630 tx=0x7fa940011af0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa946a00640 1 -- 172.21.15.86:0/3427910008 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa94000f920 con 0x7fa934004d40 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa946a00640 1 -- 172.21.15.86:0/3427910008 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa94000fac0 con 0x7fa934004d40 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa946a00640 1 -- 172.21.15.86:0/3427910008 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa94000fe70 con 0x7fa934004d40 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa95015c820 con 0x7fa934004d40 2024-03-06T07:34:33.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.143+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa95015cde0 con 0x7fa934004d40 2024-03-06T07:34:33.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.145+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa95004f330 con 0x7fa934004d40 2024-03-06T07:34:33.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.146+0000 7fa946a00640 1 -- 172.21.15.86:0/3427910008 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa940015460 con 0x7fa934004d40 2024-03-06T07:34:33.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.147+0000 7fa946a00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa93006b4d0 0x7fa93006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:33.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.147+0000 7fa946a00640 1 -- 172.21.15.86:0/3427910008 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(118..118 src has 1..118) v4 ==== 5501+0+0 (secure 0 0 0) 0x7fa94008aab0 con 0x7fa934004d40 2024-03-06T07:34:33.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.147+0000 7fa94de00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa93006b4d0 0x7fa93006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:33.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.148+0000 7fa94de00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa93006b4d0 0x7fa93006d990 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7fa95013d6d0 tx=0x7fa93c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:33.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.152+0000 7fa946a00640 1 -- 172.21.15.86:0/3427910008 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa940019030 con 0x7fa934004d40 2024-03-06T07:34:33.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.348+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fa9500cf0b0 con 0x7fa934004d40 2024-03-06T07:34:33.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.902+0000 7fa946a00640 1 -- 172.21.15.86:0/3427910008 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v119) v1 ==== 169+0+0 (secure 0 0 0) 0x7fa9400599e0 con 0x7fa934004d40 2024-03-06T07:34:33.904 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' removed 2024-03-06T07:34:33.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.906+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa93006b4d0 msgr2=0x7fa93006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:33.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.906+0000 7fa954a00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa93006b4d0 0x7fa93006d990 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7fa95013d6d0 tx=0x7fa93c008040 comp rx=0 tx=0).stop 2024-03-06T07:34:33.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.906+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 msgr2=0x7fa95013a470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:33.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.906+0000 7fa954a00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa95013a470 secure :-1 s=READY pgs=1445 cs=0 l=1 rev1=1 crypto rx=0x7fa940011630 tx=0x7fa940011af0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 shutdown_connections 2024-03-06T07:34:33.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa93006b4d0 0x7fa93006d990 unknown :-1 s=CLOSED pgs=618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9500cecd0 0x7fa9500ad860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9501f7b70 0x7fa9500ce790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 --2- 172.21.15.86:0/3427910008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa934004d40 0x7fa95013a470 unknown :-1 s=CLOSED pgs=1445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:33.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 >> 172.21.15.86:0/3427910008 conn(0x7fa9500b18a0 msgr2=0x7fa950140af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:33.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 shutdown_connections 2024-03-06T07:34:33.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:33.907+0000 7fa954a00640 1 -- 172.21.15.86:0/3427910008 wait complete. 2024-03-06T07:34:33.931 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-06T07:34:34.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.133+0000 7f7af0c00640 1 Processor -- start 2024-03-06T07:34:34.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.133+0000 7f7af0c00640 1 -- start start 2024-03-06T07:34:34.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.135+0000 7f7af0c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 0x7f7aec0ccc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.135+0000 7f7af0c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7aec15c1f0 con 0x7f7aec0aef20 2024-03-06T07:34:34.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.135+0000 7f7af0c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7aec06cac0 con 0x7f7aec15be10 2024-03-06T07:34:34.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.135+0000 7f7af0c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7aec0cd200 con 0x7f7aec06da40 2024-03-06T07:34:34.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.135+0000 7f7aebe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7aec0aef20 0x7f7aec071ab0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:43014/0 (socket says 172.21.15.86:43014) 2024-03-06T07:34:34.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.135+0000 7f7aebe00640 1 -- 172.21.15.86:0/1076965387 learned_addr learned my addr 172.21.15.86:0/1076965387 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:34.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7ae2a00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 0x7f7aec0ccc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1964953911 0 0) 0x7f7aec15c1f0 con 0x7f7aec0aef20 2024-03-06T07:34:34.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7ad0003520 con 0x7f7aec0aef20 2024-03-06T07:34:34.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4027445731 0 0) 0x7f7ad0003520 con 0x7f7aec0aef20 2024-03-06T07:34:34.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 msgr2=0x7f7aec0ccc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:34.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 0x7f7aec0ccc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 >> v1:172.21.15.194:6789/0 conn(0x7f7aec15be10 legacy=0x7f7aec06d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7aec06cda0 con 0x7f7aec0aef20 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7ae40019f0 con 0x7f7aec0aef20 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 >> v1:172.21.15.86:6789/0 conn(0x7f7aec0aef20 legacy=0x7f7aec071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ad00039e0 0x7f7ad0003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad0004800 0x7f7ad0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 0x7f7ad00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7aec0cd200 con 0x7f7ad00039e0 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ad000ba70 con 0x7f7ad0004800 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.136+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ad000bc70 con 0x7f7aec06da40 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7aeb400640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ad00039e0 0x7f7ad0003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7ae2a00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad0004800 0x7f7ad0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7aebe00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 0x7f7ad00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7aeb400640 1 -- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 msgr2=0x7f7ad00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7aeb400640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec06da40 0x7f7ad00051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7aeb400640 1 -- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad0004800 msgr2=0x7f7ad0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7aeb400640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad0004800 0x7f7ad0004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7aeb400640 1 -- 172.21.15.86:0/1076965387 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ad0003520 con 0x7f7ad00039e0 2024-03-06T07:34:34.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.137+0000 7f7ae2a00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad0004800 0x7f7ad0004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.138+0000 7f7aeb400640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ad00039e0 0x7f7ad0003e00 secure :-1 s=READY pgs=1447 cs=0 l=1 rev1=1 crypto rx=0x7f7adc001720 tx=0x7f7ad0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7562f5a39250731 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.138+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7adc001370 con 0x7f7ad00039e0 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7adc0052d0 con 0x7f7ad00039e0 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7aeaa00640 1 -- 172.21.15.86:0/1076965387 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7adc003830 con 0x7f7ad00039e0 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7af0c00640 1 -- 172.21.15.86:0/1076965387 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ad00039e0 msgr2=0x7f7ad0003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7af0c00640 1 --2- 172.21.15.86:0/1076965387 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7ad00039e0 0x7f7ad0003e00 secure :-1 s=READY pgs=1447 cs=0 l=1 rev1=1 crypto rx=0x7f7adc001720 tx=0x7f7ad0005830 comp rx=0 tx=0).stop 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7aebe00640 1 -- 172.21.15.86:0/1076965387 reap_dead start 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7af0c00640 1 -- 172.21.15.86:0/1076965387 shutdown_connections 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7af0c00640 1 -- 172.21.15.86:0/1076965387 >> 172.21.15.86:0/1076965387 conn(0x7f7aec06afc0 msgr2=0x7f7aec06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7af0c00640 1 -- 172.21.15.86:0/1076965387 shutdown_connections 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.139+0000 7f7af0c00640 1 -- 172.21.15.86:0/1076965387 wait complete. 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.141+0000 7f7af0c00640 1 Processor -- start 2024-03-06T07:34:34.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.141+0000 7f7af0c00640 1 -- start start 2024-03-06T07:34:34.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7af0c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad00039e0 0x7f7aec158e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7af0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7aec159360 0x7f7aec1547b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7af0c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec154cf0 0x7f7aec155170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7af0c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7aec0cda90 con 0x7f7aec159360 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7af0c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7aec06d080 con 0x7f7ad00039e0 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7af0c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7aec0ccc20 con 0x7f7aec154cf0 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7ae2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7aec159360 0x7f7aec1547b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7ae2a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7aec159360 0x7f7aec1547b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54844/0 (socket says 172.21.15.86:54844) 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.142+0000 7f7ae2a00640 1 -- 172.21.15.86:0/1510543692 learned_addr learned my addr 172.21.15.86:0/1510543692 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae2a00640 1 -- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec154cf0 msgr2=0x7f7aec155170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7aebe00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec154cf0 0x7f7aec155170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7aeb400640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad00039e0 0x7f7aec158e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae2a00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec154cf0 0x7f7aec155170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae2a00640 1 -- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad00039e0 msgr2=0x7f7aec158e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae2a00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad00039e0 0x7f7aec158e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae2a00640 1 -- 172.21.15.86:0/1510543692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7aec1556b0 con 0x7f7aec159360 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae2a00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7aec159360 0x7f7aec1547b0 secure :-1 s=READY pgs=1448 cs=0 l=1 rev1=1 crypto rx=0x7f7ad40108c0 tx=0x7f7ad4010d80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:34.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae3e00640 1 -- 172.21.15.86:0/1510543692 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ad400f040 con 0x7f7aec159360 2024-03-06T07:34:34.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae3e00640 1 -- 172.21.15.86:0/1510543692 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ad400eb00 con 0x7f7aec159360 2024-03-06T07:34:34.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7aec159870 con 0x7f7aec159360 2024-03-06T07:34:34.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.143+0000 7f7ae3e00640 1 -- 172.21.15.86:0/1510543692 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ad401c400 con 0x7f7aec159360 2024-03-06T07:34:34.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.144+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7aec159e30 con 0x7f7aec159360 2024-03-06T07:34:34.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.145+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7aec0b23a0 con 0x7f7aec159360 2024-03-06T07:34:34.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.146+0000 7f7ae3e00640 1 -- 172.21.15.86:0/1510543692 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7ad401c5a0 con 0x7f7aec159360 2024-03-06T07:34:34.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.146+0000 7f7ae3e00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ac806b4d0 0x7f7ac806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.147+0000 7f7ae3e00640 1 -- 172.21.15.86:0/1510543692 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f7ad408a460 con 0x7f7aec159360 2024-03-06T07:34:34.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.147+0000 7f7aeb400640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ac806b4d0 0x7f7ac806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.149 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.147+0000 7f7aeb400640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ac806b4d0 0x7f7ac806d990 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f7adc033000 tx=0x7f7adc009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:34.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.151+0000 7f7ae3e00640 1 -- 172.21.15.86:0/1510543692 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7aec0b23a0 con 0x7f7aec159360 2024-03-06T07:34:34.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.345+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f7aec04f330 con 0x7f7aec159360 2024-03-06T07:34:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.351+0000 7f7ae3e00640 1 -- 172.21.15.86:0/1510543692 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v31) v1 ==== 104+0+0 (secure 0 0 0) 0x7f7ad4018030 con 0x7f7aec159360 2024-03-06T07:34:34.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ac806b4d0 msgr2=0x7f7ac806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ac806b4d0 0x7f7ac806d990 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f7adc033000 tx=0x7f7adc009030 comp rx=0 tx=0).stop 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7aec159360 msgr2=0x7f7aec1547b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7aec159360 0x7f7aec1547b0 secure :-1 s=READY pgs=1448 cs=0 l=1 rev1=1 crypto rx=0x7f7ad40108c0 tx=0x7f7ad4010d80 comp rx=0 tx=0).stop 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 shutdown_connections 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ac806b4d0 0x7f7ac806d990 unknown :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7aec154cf0 0x7f7aec155170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7aec159360 0x7f7aec1547b0 unknown :-1 s=CLOSED pgs=1448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 --2- 172.21.15.86:0/1510543692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7ad00039e0 0x7f7aec158e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 >> 172.21.15.86:0/1510543692 conn(0x7f7aec06afc0 msgr2=0x7f7aec0b0920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 shutdown_connections 2024-03-06T07:34:34.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.354+0000 7f7af0c00640 1 -- 172.21.15.86:0/1510543692 wait complete. 2024-03-06T07:34:34.375 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration' 2024-03-06T07:34:34.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.581+0000 7f6d70600640 1 Processor -- start 2024-03-06T07:34:34.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.581+0000 7f6d70600640 1 -- start start 2024-03-06T07:34:34.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d70600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6d68144660 0x7f6d68071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d70600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d680cf3e0 0x7f6d680724b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d70600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6d680cabf0 con 0x7f6d680c9ec0 2024-03-06T07:34:34.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d70600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6d680ce900 con 0x7f6d68144660 2024-03-06T07:34:34.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d70600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6d680cebe0 con 0x7f6d680cf3e0 2024-03-06T07:34:34.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d6e600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6d680c9ec0 0x7f6d680ca310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:43018/0 (socket says 172.21.15.86:43018) 2024-03-06T07:34:34.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d6e600640 1 -- 172.21.15.86:0/2557656458 learned_addr learned my addr 172.21.15.86:0/2557656458 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:34.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d6d200640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d680cf3e0 0x7f6d680724b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d6dc00640 1 --2- 172.21.15.86:0/2557656458 >> v2:172.21.15.194:3300/0 conn(0x7f6d68144660 0x7f6d68071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 938793749 0 0) 0x7f6d680cabf0 con 0x7f6d680c9ec0 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.583+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6d4c003520 con 0x7f6d680c9ec0 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3738824512 0 0) 0x7f6d4c003520 con 0x7f6d680c9ec0 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d680cf3e0 msgr2=0x7f6d680724b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d680cf3e0 0x7f6d680724b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 >> v2:172.21.15.194:3300/0 conn(0x7f6d68144660 msgr2=0x7f6d68071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 --2- 172.21.15.86:0/2557656458 >> v2:172.21.15.194:3300/0 conn(0x7f6d68144660 0x7f6d68071f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d680ceec0 con 0x7f6d680c9ec0 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6d600019f0 con 0x7f6d680c9ec0 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 >> v1:172.21.15.86:6789/0 conn(0x7f6d680c9ec0 legacy=0x7f6d680ca310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c0039b0 0x7f6d4c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d4c004800 0x7f6d4c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d680cf3e0 0x7f6d4c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d680ce900 con 0x7f6d4c004800 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d680cebe0 con 0x7f6d4c0039b0 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6d4c00b9c0 con 0x7f6d680cf3e0 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.584+0000 7f6d6d200640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d4c004800 0x7f6d4c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.585+0000 7f6d6dc00640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c0039b0 0x7f6d4c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.585+0000 7f6d6d200640 1 -- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d680cf3e0 msgr2=0x7f6d4c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.585+0000 7f6d6d200640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d680cf3e0 0x7f6d4c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.585+0000 7f6d6d200640 1 -- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c0039b0 msgr2=0x7f6d4c003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.585+0000 7f6d6d200640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c0039b0 0x7f6d4c003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.585+0000 7f6d6d200640 1 -- 172.21.15.86:0/2557656458 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d4c003520 con 0x7f6d4c004800 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.586+0000 7f6d6d200640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d4c004800 0x7f6d4c004c80 secure :-1 s=READY pgs=1450 cs=0 l=1 rev1=1 crypto rx=0x7f6d54009ac0 tx=0x7f6d4c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=41e95a8876c90d84 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.586+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d54039070 con 0x7f6d4c004800 2024-03-06T07:34:34.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.586+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d5402c710 con 0x7f6d4c004800 2024-03-06T07:34:34.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.586+0000 7f6d67e00640 1 -- 172.21.15.86:0/2557656458 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d54009690 con 0x7f6d4c004800 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.587+0000 7f6d70600640 1 -- 172.21.15.86:0/2557656458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d4c004800 msgr2=0x7f6d4c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.587+0000 7f6d70600640 1 --2- 172.21.15.86:0/2557656458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d4c004800 0x7f6d4c004c80 secure :-1 s=READY pgs=1450 cs=0 l=1 rev1=1 crypto rx=0x7f6d54009ac0 tx=0x7f6d4c005830 comp rx=0 tx=0).stop 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.587+0000 7f6d6e600640 1 -- 172.21.15.86:0/2557656458 reap_dead start 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.587+0000 7f6d70600640 1 -- 172.21.15.86:0/2557656458 shutdown_connections 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.587+0000 7f6d70600640 1 -- 172.21.15.86:0/2557656458 >> 172.21.15.86:0/2557656458 conn(0x7f6d680d0b90 msgr2=0x7f6d680d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.587+0000 7f6d70600640 1 -- 172.21.15.86:0/2557656458 shutdown_connections 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.587+0000 7f6d70600640 1 -- 172.21.15.86:0/2557656458 wait complete. 2024-03-06T07:34:34.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.589+0000 7f6d70600640 1 Processor -- start 2024-03-06T07:34:34.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.589+0000 7f6d70600640 1 -- start start 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d70600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c004800 0x7f6d6815d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d70600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d680b07b0 0x7f6d680b0c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d70600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d6815fec0 0x7f6d68160340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d70600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d68072630 con 0x7f6d680b07b0 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d70600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d680724b0 con 0x7f6d4c004800 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d70600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6d680727b0 con 0x7f6d6815fec0 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d680b07b0 0x7f6d680b0c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d680b07b0 0x7f6d680b0c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54874/0 (socket says 172.21.15.86:54874) 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 -- 172.21.15.86:0/830099612 learned_addr learned my addr 172.21.15.86:0/830099612 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6dc00640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c004800 0x7f6d6815d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6e600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d6815fec0 0x7f6d68160340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 -- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d6815fec0 msgr2=0x7f6d68160340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d6815fec0 0x7f6d68160340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 -- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c004800 msgr2=0x7f6d6815d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c004800 0x7f6d6815d6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.590+0000 7f6d6d200640 1 -- 172.21.15.86:0/830099612 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d6815dc00 con 0x7f6d680b07b0 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.591+0000 7f6d6d200640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d680b07b0 0x7f6d680b0c10 secure :-1 s=READY pgs=1451 cs=0 l=1 rev1=1 crypto rx=0x7f6d54037040 tx=0x7f6d54004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.591+0000 7f6d65600640 1 -- 172.21.15.86:0/830099612 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d54042020 con 0x7f6d680b07b0 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.591+0000 7f6d65600640 1 -- 172.21.15.86:0/830099612 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d5403a040 con 0x7f6d680b07b0 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.591+0000 7f6d65600640 1 -- 172.21.15.86:0/830099612 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d54039070 con 0x7f6d680b07b0 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.591+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6d68160c30 con 0x7f6d680b07b0 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.591+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6d681611f0 con 0x7f6d680b07b0 2024-03-06T07:34:34.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.592+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6d6804f330 con 0x7f6d680b07b0 2024-03-06T07:34:34.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.593+0000 7f6d65600640 1 -- 172.21.15.86:0/830099612 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6d54045020 con 0x7f6d680b07b0 2024-03-06T07:34:34.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.594+0000 7f6d65600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6d4806b500 0x7f6d4806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:34.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.594+0000 7f6d65600640 1 -- 172.21.15.86:0/830099612 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f6d540ab3f0 con 0x7f6d680b07b0 2024-03-06T07:34:34.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.594+0000 7f6d6dc00640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6d4806b500 0x7f6d4806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:34.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.595+0000 7f6d6dc00640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6d4806b500 0x7f6d4806d9c0 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f6d58009a90 tx=0x7f6d58008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:34.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.598+0000 7f6d65600640 1 -- 172.21.15.86:0/830099612 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6d5404c070 con 0x7f6d680b07b0 2024-03-06T07:34:34.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.860+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]} v 0) v1 -- 0x7f6d68162860 con 0x7f6d680b07b0 2024-03-06T07:34:34.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.905+0000 7f6d65600640 1 -- 172.21.15.86:0/830099612 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]}]=0 v410) v1 ==== 142+0+0 (secure 0 0 0) 0x7f6d6815dc00 con 0x7f6d680b07b0 2024-03-06T07:34:34.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6d4806b500 msgr2=0x7f6d4806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6d4806b500 0x7f6d4806d9c0 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f6d58009a90 tx=0x7f6d58008040 comp rx=0 tx=0).stop 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d680b07b0 msgr2=0x7f6d680b0c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d680b07b0 0x7f6d680b0c10 secure :-1 s=READY pgs=1451 cs=0 l=1 rev1=1 crypto rx=0x7f6d54037040 tx=0x7f6d54004730 comp rx=0 tx=0).stop 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 shutdown_connections 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6d4806b500 0x7f6d4806d9c0 unknown :-1 s=CLOSED pgs=620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d6815fec0 0x7f6d68160340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6d680b07b0 0x7f6d680b0c10 unknown :-1 s=CLOSED pgs=1451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 --2- 172.21.15.86:0/830099612 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d4c004800 0x7f6d6815d6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 >> 172.21.15.86:0/830099612 conn(0x7f6d680d0b90 msgr2=0x7f6d68143800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 shutdown_connections 2024-03-06T07:34:34.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:34.908+0000 7f6d70600640 1 -- 172.21.15.86:0/830099612 wait complete. 2024-03-06T07:34:34.929 INFO:tasks.cephfs_test_runner:That removal of primary dentry of hardlinked inode results ... ok 2024-03-06T07:34:34.929 INFO:tasks.cephfs_test_runner:Starting test: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) Reproduce https://tracker.ceph.com/issues/53597 2024-03-06T07:34:34.929 INFO:tasks.cephfs_test_runner:test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) 2024-03-06T07:34:34.929 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:35.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.134+0000 7f974b000640 1 Processor -- start 2024-03-06T07:34:35.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.134+0000 7f974b000640 1 -- start start 2024-03-06T07:34:35.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f974b000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f974415acb0 0x7f97440b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f974b000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f974406ec00 0x7f97440ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f974b000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f974415b090 con 0x7f97440b4680 2024-03-06T07:34:35.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f974b000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97440b2a30 con 0x7f974406ec00 2024-03-06T07:34:35.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f974b000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97440b4360 con 0x7f974415acb0 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f9749000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f974406ec00 0x7f97440ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f9743400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f97440b4680 0x7f974406e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:43030/0 (socket says 172.21.15.86:43030) 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f9743400640 1 -- 172.21.15.86:0/1178309641 learned_addr learned my addr 172.21.15.86:0/1178309641 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.136+0000 7f9743e00640 1 --2- 172.21.15.86:0/1178309641 >> v2:172.21.15.194:3300/0 conn(0x7f974415acb0 0x7f97440b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.137+0000 7f9749000640 1 -- 172.21.15.86:0/1178309641 >> v2:172.21.15.194:3300/0 conn(0x7f974415acb0 msgr2=0x7f97440b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.137+0000 7f9749000640 1 --2- 172.21.15.86:0/1178309641 >> v2:172.21.15.194:3300/0 conn(0x7f974415acb0 0x7f97440b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.137+0000 7f9749000640 1 -- 172.21.15.86:0/1178309641 >> v1:172.21.15.86:6789/0 conn(0x7f97440b4680 legacy=0x7f974406e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.137+0000 7f9749000640 1 -- 172.21.15.86:0/1178309641 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97440b2cf0 con 0x7f974406ec00 2024-03-06T07:34:35.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f9749000640 1 --2- 172.21.15.86:0/1178309641 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f974406ec00 0x7f97440ccfb0 secure :-1 s=READY pgs=1453 cs=0 l=1 rev1=1 crypto rx=0x7f97440b27f0 tx=0x7f973c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e9009f42abb6f97 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:35.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f9742a00640 1 -- 172.21.15.86:0/1178309641 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f973c039070 con 0x7f974406ec00 2024-03-06T07:34:35.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f9742a00640 1 -- 172.21.15.86:0/1178309641 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f973c02cc80 con 0x7f974406ec00 2024-03-06T07:34:35.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f9742a00640 1 -- 172.21.15.86:0/1178309641 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f973c034660 con 0x7f974406ec00 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f974b000640 1 -- 172.21.15.86:0/1178309641 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f974406ec00 msgr2=0x7f97440ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f974b000640 1 --2- 172.21.15.86:0/1178309641 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f974406ec00 0x7f97440ccfb0 secure :-1 s=READY pgs=1453 cs=0 l=1 rev1=1 crypto rx=0x7f97440b27f0 tx=0x7f973c02bed0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f974b000640 1 -- 172.21.15.86:0/1178309641 shutdown_connections 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f974b000640 1 --2- 172.21.15.86:0/1178309641 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f974406ec00 0x7f97440ccfb0 unknown :-1 s=CLOSED pgs=1453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.138+0000 7f974b000640 1 --2- 172.21.15.86:0/1178309641 >> v2:172.21.15.194:3300/0 conn(0x7f974415acb0 0x7f97440b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.139+0000 7f974b000640 1 -- 172.21.15.86:0/1178309641 >> 172.21.15.86:0/1178309641 conn(0x7f97440b1730 msgr2=0x7f97440b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.139+0000 7f974b000640 1 -- 172.21.15.86:0/1178309641 shutdown_connections 2024-03-06T07:34:35.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.139+0000 7f974b000640 1 -- 172.21.15.86:0/1178309641 wait complete. 2024-03-06T07:34:35.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.140+0000 7f974b000640 1 Processor -- start 2024-03-06T07:34:35.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.140+0000 7f974b000640 1 -- start start 2024-03-06T07:34:35.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.141+0000 7f974b000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f974406ec00 0x7f9744159770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.141+0000 7f974b000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97440b4680 0x7f974415be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.141+0000 7f974b000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f974415acb0 0x7f9744145090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.141+0000 7f974b000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97440b3d10 con 0x7f97440b4680 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.141+0000 7f974b000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97440b32b0 con 0x7f974406ec00 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.141+0000 7f974b000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97440b4010 con 0x7f974415acb0 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97440b4680 0x7f974415be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97440b4680 0x7f974415be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54892/0 (socket says 172.21.15.86:54892) 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 -- 172.21.15.86:0/3482489179 learned_addr learned my addr 172.21.15.86:0/3482489179 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9749000640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f974415acb0 0x7f9744145090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743e00640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f974406ec00 0x7f9744159770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 -- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f974415acb0 msgr2=0x7f9744145090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f974415acb0 0x7f9744145090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 -- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f974406ec00 msgr2=0x7f9744159770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f974406ec00 0x7f9744159770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 -- 172.21.15.86:0/3482489179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f974415cab0 con 0x7f97440b4680 2024-03-06T07:34:35.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.142+0000 7f9743400640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97440b4680 0x7f974415be30 secure :-1 s=READY pgs=1454 cs=0 l=1 rev1=1 crypto rx=0x7f972c00c400 tx=0x7f972c00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.143+0000 7f973be00640 1 -- 172.21.15.86:0/3482489179 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f972c017070 con 0x7f97440b4680 2024-03-06T07:34:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.143+0000 7f973be00640 1 -- 172.21.15.86:0/3482489179 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f972c005ce0 con 0x7f97440b4680 2024-03-06T07:34:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.143+0000 7f973be00640 1 -- 172.21.15.86:0/3482489179 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f972c0072f0 con 0x7f97440b4680 2024-03-06T07:34:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.143+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f974415cd90 con 0x7f97440b4680 2024-03-06T07:34:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.143+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f974415d350 con 0x7f97440b4680 2024-03-06T07:34:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.144+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f974404f330 con 0x7f97440b4680 2024-03-06T07:34:35.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.145+0000 7f973be00640 1 -- 172.21.15.86:0/3482489179 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f972c01a070 con 0x7f97440b4680 2024-03-06T07:34:35.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.146+0000 7f973be00640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f972006b4d0 0x7f972006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.146+0000 7f973be00640 1 -- 172.21.15.86:0/3482489179 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f972c0893e0 con 0x7f97440b4680 2024-03-06T07:34:35.147 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.146+0000 7f9743e00640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f972006b4d0 0x7f972006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.147+0000 7f9743e00640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f972006b4d0 0x7f972006d990 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7f9734006fd0 tx=0x7f9734008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:35.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.150+0000 7f973be00640 1 -- 172.21.15.86:0/3482489179 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f972c058470 con 0x7f97440b4680 2024-03-06T07:34:35.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.345+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f97440d0450 con 0x7f97440b4680 2024-03-06T07:34:35.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.346+0000 7f973be00640 1 -- 172.21.15.86:0/3482489179 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v119) v1 ==== 74+0+13170 (secure 0 0 0) 0x7f972c057d70 con 0x7f97440b4680 2024-03-06T07:34:35.348 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:35.348 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":119,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:34:33.893227+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:6837/2212362187":"2024-03-07T07:34:30.640325+0000","172.21.15.86:6836/2212362187":"2024-03-07T07:34:30.640325+0000","172.21.15.86:6837/221976790":"2024-03-07T07:33:20.538234+0000","172.21.15.86:6836/221976790":"2024-03-07T07:33:20.538234+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f972006b4d0 msgr2=0x7f972006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f972006b4d0 0x7f972006d990 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7f9734006fd0 tx=0x7f9734008040 comp rx=0 tx=0).stop 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97440b4680 msgr2=0x7f974415be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97440b4680 0x7f974415be30 secure :-1 s=READY pgs=1454 cs=0 l=1 rev1=1 crypto rx=0x7f972c00c400 tx=0x7f972c00c8c0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 shutdown_connections 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f972006b4d0 0x7f972006d990 unknown :-1 s=CLOSED pgs=621 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f974415acb0 0x7f9744145090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f97440b4680 0x7f974415be30 unknown :-1 s=CLOSED pgs=1454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 --2- 172.21.15.86:0/3482489179 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f974406ec00 0x7f9744159770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 >> 172.21.15.86:0/3482489179 conn(0x7f97440b1730 msgr2=0x7f9744147f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 shutdown_connections 2024-03-06T07:34:35.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.349+0000 7f974b000640 1 -- 172.21.15.86:0/3482489179 wait complete. 2024-03-06T07:34:35.368 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:34:35.368 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:34:35.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.579+0000 7f9765e00640 1 Processor -- start 2024-03-06T07:34:35.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.580+0000 7f9765e00640 1 -- start start 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f9765e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f976015fc10 0x7f9760070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f9765e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 0x7f97600ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f9765e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f97600b46c0 con 0x7f976015fff0 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f9765e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97600b49a0 con 0x7f976015fc10 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f9765e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9760072c90 con 0x7f97600b3f70 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f975ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f976015fc10 0x7f9760070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f975ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f976015fc10 0x7f9760070ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54894/0 (socket says 172.21.15.86:54894) 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.581+0000 7f975ea00640 1 -- 172.21.15.86:0/513177545 learned_addr learned my addr 172.21.15.86:0/513177545 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:35.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.582+0000 7f975ea00640 1 -- 172.21.15.86:0/513177545 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 msgr2=0x7f97600ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:35.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.582+0000 7f975ea00640 1 --2- 172.21.15.86:0/513177545 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 0x7f97600ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.582+0000 7f975ea00640 1 -- 172.21.15.86:0/513177545 >> v1:172.21.15.194:6789/0 conn(0x7f976015fff0 legacy=0x7f9760160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:35.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.582+0000 7f975ea00640 1 -- 172.21.15.86:0/513177545 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f976009dd80 con 0x7f976015fc10 2024-03-06T07:34:35.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.583+0000 7f975ea00640 1 --2- 172.21.15.86:0/513177545 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f9760070ff0 secure :-1 s=READY pgs=1455 cs=0 l=1 rev1=1 crypto rx=0x7f9750009870 tx=0x7f975002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3595c723223540a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:35.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.583+0000 7f975e000640 1 -- 172.21.15.86:0/513177545 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9750035020 con 0x7f976015fc10 2024-03-06T07:34:35.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.583+0000 7f975e000640 1 -- 172.21.15.86:0/513177545 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9750031640 con 0x7f976015fc10 2024-03-06T07:34:35.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.583+0000 7f975e000640 1 -- 172.21.15.86:0/513177545 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f975002d040 con 0x7f976015fc10 2024-03-06T07:34:35.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 -- 172.21.15.86:0/513177545 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 msgr2=0x7f9760070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 --2- 172.21.15.86:0/513177545 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f9760070ff0 secure :-1 s=READY pgs=1455 cs=0 l=1 rev1=1 crypto rx=0x7f9750009870 tx=0x7f975002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 -- 172.21.15.86:0/513177545 shutdown_connections 2024-03-06T07:34:35.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 --2- 172.21.15.86:0/513177545 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 0x7f97600ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 --2- 172.21.15.86:0/513177545 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f9760070ff0 unknown :-1 s=CLOSED pgs=1455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 -- 172.21.15.86:0/513177545 >> 172.21.15.86:0/513177545 conn(0x7f97600b1940 msgr2=0x7f97600b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:35.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 -- 172.21.15.86:0/513177545 shutdown_connections 2024-03-06T07:34:35.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.584+0000 7f9765e00640 1 -- 172.21.15.86:0/513177545 wait complete. 2024-03-06T07:34:35.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.586+0000 7f9765e00640 1 Processor -- start 2024-03-06T07:34:35.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.586+0000 7f9765e00640 1 -- start start 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9765e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 0x7f97601e5a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9765e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f97600aded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9765e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f976015fff0 0x7f9760147140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9765e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f976015c460 con 0x7f976015fc10 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9765e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f976015c2e0 con 0x7f976015fff0 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9765e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97600ce9e0 con 0x7f97600b3f70 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9757e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f97600aded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9757e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f97600aded0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54904/0 (socket says 172.21.15.86:54904) 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.587+0000 7f9757e00640 1 -- 172.21.15.86:0/1549990391 learned_addr learned my addr 172.21.15.86:0/1549990391 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f975ea00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 0x7f97601e5a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f975f400640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f976015fff0 0x7f9760147140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757e00640 1 -- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 msgr2=0x7f97601e5a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 0x7f97601e5a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757e00640 1 -- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f976015fff0 msgr2=0x7f9760147140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f976015fff0 0x7f9760147140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757e00640 1 -- 172.21.15.86:0/1549990391 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9760152960 con 0x7f976015fc10 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f975f400640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f976015fff0 0x7f9760147140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f97600aded0 secure :-1 s=READY pgs=1456 cs=0 l=1 rev1=1 crypto rx=0x7f974800b3e0 tx=0x7f974800b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757400640 1 -- 172.21.15.86:0/1549990391 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9748015020 con 0x7f976015fc10 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757400640 1 -- 172.21.15.86:0/1549990391 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9748011850 con 0x7f976015fc10 2024-03-06T07:34:35.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9757400640 1 -- 172.21.15.86:0/1549990391 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9748011c40 con 0x7f976015fc10 2024-03-06T07:34:35.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.588+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f976015f350 con 0x7f976015fc10 2024-03-06T07:34:35.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.589+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f976006dd20 con 0x7f976015fc10 2024-03-06T07:34:35.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.590+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f976004f330 con 0x7f976015fc10 2024-03-06T07:34:35.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.591+0000 7f9757400640 1 -- 172.21.15.86:0/1549990391 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9748018020 con 0x7f976015fc10 2024-03-06T07:34:35.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.592+0000 7f9757400640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f973c06b4d0 0x7f973c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:35.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.592+0000 7f9757400640 1 -- 172.21.15.86:0/1549990391 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f9748089e70 con 0x7f976015fc10 2024-03-06T07:34:35.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.592+0000 7f975ea00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f973c06b4d0 0x7f973c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:35.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.592+0000 7f975ea00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f973c06b4d0 0x7f973c06d990 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f9750036000 tx=0x7f975003c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:35.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.596+0000 7f9757400640 1 -- 172.21.15.86:0/1549990391 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f974801f020 con 0x7f976015fc10 2024-03-06T07:34:35.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.794+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f97600b2d40 con 0x7f976015fc10 2024-03-06T07:34:35.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.795+0000 7f9757400640 1 -- 172.21.15.86:0/1549990391 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v119) v1 ==== 93+0+10 (secure 0 0 0) 0x7f9760152960 con 0x7f976015fc10 2024-03-06T07:34:35.797 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:34:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f973c06b4d0 msgr2=0x7f973c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f973c06b4d0 0x7f973c06d990 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f9750036000 tx=0x7f975003c040 comp rx=0 tx=0).stop 2024-03-06T07:34:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 msgr2=0x7f97600aded0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f97600aded0 secure :-1 s=READY pgs=1456 cs=0 l=1 rev1=1 crypto rx=0x7f974800b3e0 tx=0x7f974800b8a0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 shutdown_connections 2024-03-06T07:34:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f973c06b4d0 0x7f973c06d990 unknown :-1 s=CLOSED pgs=622 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f976015fff0 0x7f9760147140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f976015fc10 0x7f97600aded0 unknown :-1 s=CLOSED pgs=1456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 --2- 172.21.15.86:0/1549990391 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f97600b3f70 0x7f97601e5a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:35.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 >> 172.21.15.86:0/1549990391 conn(0x7f97600b1940 msgr2=0x7f97600b0aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:35.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 shutdown_connections 2024-03-06T07:34:35.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:35.798+0000 7f9765e00640 1 -- 172.21.15.86:0/1549990391 wait complete. 2024-03-06T07:34:35.817 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir' 2024-03-06T07:34:36.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.018+0000 7f57cd400640 1 Processor -- start 2024-03-06T07:34:36.020 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.018+0000 7f57cd400640 1 -- start start 2024-03-06T07:34:36.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57cd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 0x7f57c815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:36.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57cd400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f57c8070b10 0x7f57c8070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:36.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57cd400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f57c806cef0 con 0x7f57c8144c00 2024-03-06T07:34:36.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57cd400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f57c8160370 con 0x7f57c8070b10 2024-03-06T07:34:36.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57cd400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f57c806d1d0 con 0x7f57c815f9d0 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57c6800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f57c8144c00 0x7f57c806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:58994/0 (socket says 172.21.15.86:58994) 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57c6800640 1 -- 172.21.15.86:0/1190680924 learned_addr learned my addr 172.21.15.86:0/1190680924 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.020+0000 7f57c7200640 1 --2- 172.21.15.86:0/1190680924 >> v2:172.21.15.86:3300/0 conn(0x7f57c8070b10 0x7f57c8070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.021+0000 7f57c5e00640 1 --2- 172.21.15.86:0/1190680924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 0x7f57c815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.021+0000 7f57c7200640 1 -- 172.21.15.86:0/1190680924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 msgr2=0x7f57c815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.021+0000 7f57c7200640 1 --2- 172.21.15.86:0/1190680924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 0x7f57c815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.021+0000 7f57c7200640 1 -- 172.21.15.86:0/1190680924 >> v1:172.21.15.86:6789/0 conn(0x7f57c8144c00 legacy=0x7f57c806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:36.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.021+0000 7f57c7200640 1 -- 172.21.15.86:0/1190680924 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57c806d4b0 con 0x7f57c8070b10 2024-03-06T07:34:36.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.022+0000 7f57c7200640 1 --2- 172.21.15.86:0/1190680924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8070b10 0x7f57c8070f60 secure :-1 s=READY pgs=1458 cs=0 l=1 rev1=1 crypto rx=0x7f57c00097a0 tx=0x7f57c002b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cbeb4959d88fa3ad server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:36.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.022+0000 7f57c5400640 1 -- 172.21.15.86:0/1190680924 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57c0035020 con 0x7f57c8070b10 2024-03-06T07:34:36.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.022+0000 7f57c5400640 1 -- 172.21.15.86:0/1190680924 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f57c0031810 con 0x7f57c8070b10 2024-03-06T07:34:36.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.022+0000 7f57c5400640 1 -- 172.21.15.86:0/1190680924 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57c0031be0 con 0x7f57c8070b10 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 -- 172.21.15.86:0/1190680924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8070b10 msgr2=0x7f57c8070f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 --2- 172.21.15.86:0/1190680924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8070b10 0x7f57c8070f60 secure :-1 s=READY pgs=1458 cs=0 l=1 rev1=1 crypto rx=0x7f57c00097a0 tx=0x7f57c002b8b0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 -- 172.21.15.86:0/1190680924 shutdown_connections 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 --2- 172.21.15.86:0/1190680924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8070b10 0x7f57c8070f60 unknown :-1 s=CLOSED pgs=1458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 --2- 172.21.15.86:0/1190680924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 0x7f57c815fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 -- 172.21.15.86:0/1190680924 >> 172.21.15.86:0/1190680924 conn(0x7f57c80b1730 msgr2=0x7f57c80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 -- 172.21.15.86:0/1190680924 shutdown_connections 2024-03-06T07:34:36.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.023+0000 7f57cd400640 1 -- 172.21.15.86:0/1190680924 wait complete. 2024-03-06T07:34:36.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.025+0000 7f57cd400640 1 Processor -- start 2024-03-06T07:34:36.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.025+0000 7f57cd400640 1 -- start start 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57cd400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f57c8070b10 0x7f57c815dda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57cd400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8144c00 0x7f57c81e8b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57cd400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 0x7f57c81e90a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57cd400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57c806cb50 con 0x7f57c8144c00 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57cd400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57c8160730 con 0x7f57c8070b10 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57cd400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f57c806ccd0 con 0x7f57c815f9d0 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8144c00 0x7f57c81e8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8144c00 0x7f57c81e8b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54578/0 (socket says 172.21.15.86:54578) 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 -- 172.21.15.86:0/3504747776 learned_addr learned my addr 172.21.15.86:0/3504747776 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 -- 172.21.15.86:0/3504747776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 msgr2=0x7f57c81e90a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 0x7f57c81e90a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 -- 172.21.15.86:0/3504747776 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f57c8070b10 msgr2=0x7f57c815dda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f57c8070b10 0x7f57c815dda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.026+0000 7f57c5e00640 1 -- 172.21.15.86:0/3504747776 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57c815e780 con 0x7f57c8144c00 2024-03-06T07:34:36.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.027+0000 7f57c5e00640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8144c00 0x7f57c81e8b60 secure :-1 s=READY pgs=1459 cs=0 l=1 rev1=1 crypto rx=0x7f57b400d800 tx=0x7f57b400dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:36.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.027+0000 7f57bea00640 1 -- 172.21.15.86:0/3504747776 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57b400b6c0 con 0x7f57c8144c00 2024-03-06T07:34:36.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.027+0000 7f57bea00640 1 -- 172.21.15.86:0/3504747776 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f57b4019070 con 0x7f57c8144c00 2024-03-06T07:34:36.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.027+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f57c815ea60 con 0x7f57c8144c00 2024-03-06T07:34:36.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.027+0000 7f57bea00640 1 -- 172.21.15.86:0/3504747776 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57b400ba80 con 0x7f57c8144c00 2024-03-06T07:34:36.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.027+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f57c81e66c0 con 0x7f57c8144c00 2024-03-06T07:34:36.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.028+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f57c804f330 con 0x7f57c8144c00 2024-03-06T07:34:36.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.029+0000 7f57bea00640 1 -- 172.21.15.86:0/3504747776 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f57b400bc20 con 0x7f57c8144c00 2024-03-06T07:34:36.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.030+0000 7f57bea00640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57a806b4d0 0x7f57a806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:36.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.030+0000 7f57bea00640 1 -- 172.21.15.86:0/3504747776 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f57b408a420 con 0x7f57c8144c00 2024-03-06T07:34:36.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.030+0000 7f57c6800640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57a806b4d0 0x7f57a806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:36.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.031+0000 7f57c6800640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57a806b4d0 0x7f57a806d990 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7f57b80012e0 tx=0x7f57b8001630 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:36.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.035+0000 7f57bea00640 1 -- 172.21.15.86:0/3504747776 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f57b4059530 con 0x7f57c8144c00 2024-03-06T07:34:36.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.300+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir"]} v 0) v1 -- 0x7f57c815be20 con 0x7f57c8144c00 2024-03-06T07:34:36.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.917+0000 7f57bea00640 1 -- 172.21.15.86:0/3504747776 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir"]}]=0 v412) v1 ==== 143+0+0 (secure 0 0 0) 0x7f57b4058e30 con 0x7f57c8144c00 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57a806b4d0 msgr2=0x7f57a806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57a806b4d0 0x7f57a806d990 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7f57b80012e0 tx=0x7f57b8001630 comp rx=0 tx=0).stop 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8144c00 msgr2=0x7f57c81e8b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8144c00 0x7f57c81e8b60 secure :-1 s=READY pgs=1459 cs=0 l=1 rev1=1 crypto rx=0x7f57b400d800 tx=0x7f57b400dcc0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 shutdown_connections 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f57a806b4d0 0x7f57a806d990 unknown :-1 s=CLOSED pgs=623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f57c815f9d0 0x7f57c81e90a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f57c8144c00 0x7f57c81e8b60 unknown :-1 s=CLOSED pgs=1459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 --2- 172.21.15.86:0/3504747776 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f57c8070b10 0x7f57c815dda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 >> 172.21.15.86:0/3504747776 conn(0x7f57c80b1730 msgr2=0x7f57c80ae6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 shutdown_connections 2024-03-06T07:34:36.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:36.921+0000 7f57cd400640 1 -- 172.21.15.86:0/3504747776 wait complete. 2024-03-06T07:34:36.943 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-06T07:34:37.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.151+0000 7f96a3c00640 1 Processor -- start 2024-03-06T07:34:37.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.151+0000 7f96a3c00640 1 -- start start 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f96a3c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f969c0b3f70 0x7f969c160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f96a3c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c070ff0 0x7f969c0ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f96a3c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f969c0b46c0 con 0x7f969c15fc10 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f96a3c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f969c071480 con 0x7f969c0b3f70 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f96a3c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f969c0b49a0 con 0x7f969c070ff0 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f969be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f969c0b3f70 0x7f969c160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f969be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f969c0b3f70 0x7f969c160700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54584/0 (socket says 172.21.15.86:54584) 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f969be00640 1 -- 172.21.15.86:0/1976043176 learned_addr learned my addr 172.21.15.86:0/1976043176 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:37.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.153+0000 7f96a1200640 1 --1- 172.21.15.86:0/1976043176 >> v1:172.21.15.86:6789/0 conn(0x7f969c15fc10 0x7f969c15fff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59006/0 (socket says 172.21.15.86:59006) 2024-03-06T07:34:37.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.154+0000 7f96a1c00640 1 --2- 172.21.15.86:0/1976043176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c070ff0 0x7f969c0ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:37.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.154+0000 7f969be00640 1 -- 172.21.15.86:0/1976043176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c070ff0 msgr2=0x7f969c0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:37.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.154+0000 7f969be00640 1 --2- 172.21.15.86:0/1976043176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c070ff0 0x7f969c0ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.154+0000 7f969be00640 1 -- 172.21.15.86:0/1976043176 >> v1:172.21.15.86:6789/0 conn(0x7f969c15fc10 legacy=0x7f969c15fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:37.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.154+0000 7f969b400640 1 -- 172.21.15.86:0/1976043176 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 972093479 0 0) 0x7f969c0b46c0 con 0x7f969c15fc10 2024-03-06T07:34:37.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.154+0000 7f969be00640 1 -- 172.21.15.86:0/1976043176 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f969c072c90 con 0x7f969c0b3f70 2024-03-06T07:34:37.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.154+0000 7f96a1c00640 1 --2- 172.21.15.86:0/1976043176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c070ff0 0x7f969c0ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:37.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.156+0000 7f969be00640 1 --2- 172.21.15.86:0/1976043176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c0b3f70 0x7f969c160700 secure :-1 s=READY pgs=1461 cs=0 l=1 rev1=1 crypto rx=0x7f968c009870 tx=0x7f968c02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=df1bf0f84a6646ad server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:37.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f969b400640 1 -- 172.21.15.86:0/1976043176 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f968c034070 con 0x7f969c0b3f70 2024-03-06T07:34:37.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f969b400640 1 -- 172.21.15.86:0/1976043176 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f968c030840 con 0x7f969c0b3f70 2024-03-06T07:34:37.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f969b400640 1 -- 172.21.15.86:0/1976043176 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f968c030c10 con 0x7f969c0b3f70 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f96a3c00640 1 -- 172.21.15.86:0/1976043176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c0b3f70 msgr2=0x7f969c160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f96a3c00640 1 --2- 172.21.15.86:0/1976043176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c0b3f70 0x7f969c160700 secure :-1 s=READY pgs=1461 cs=0 l=1 rev1=1 crypto rx=0x7f968c009870 tx=0x7f968c02b980 comp rx=0 tx=0).stop 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f96a3c00640 1 -- 172.21.15.86:0/1976043176 shutdown_connections 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f96a3c00640 1 --2- 172.21.15.86:0/1976043176 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c070ff0 0x7f969c0ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f96a3c00640 1 --2- 172.21.15.86:0/1976043176 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c0b3f70 0x7f969c160700 unknown :-1 s=CLOSED pgs=1461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.157+0000 7f96a3c00640 1 -- 172.21.15.86:0/1976043176 >> 172.21.15.86:0/1976043176 conn(0x7f969c0b1940 msgr2=0x7f969c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.158+0000 7f96a3c00640 1 -- 172.21.15.86:0/1976043176 shutdown_connections 2024-03-06T07:34:37.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.158+0000 7f96a3c00640 1 -- 172.21.15.86:0/1976043176 wait complete. 2024-03-06T07:34:37.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.159+0000 7f96a3c00640 1 Processor -- start 2024-03-06T07:34:37.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.159+0000 7f96a3c00640 1 -- start start 2024-03-06T07:34:37.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.160+0000 7f96a3c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f969c070ff0 0x7f969c13ddd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:37.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.160+0000 7f96a3c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c0b3f70 0x7f969c158030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:37.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.160+0000 7f96a3c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c15fc10 0x7f969c158570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:37.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.160+0000 7f96a3c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f969c072a30 con 0x7f969c15fc10 2024-03-06T07:34:37.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.160+0000 7f96a3c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f969c0728b0 con 0x7f969c070ff0 2024-03-06T07:34:37.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.160+0000 7f96a3c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f969c15c2e0 con 0x7f969c0b3f70 2024-03-06T07:34:37.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c15fc10 0x7f969c158570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:37.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c15fc10 0x7f969c158570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54600/0 (socket says 172.21.15.86:54600) 2024-03-06T07:34:37.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 -- 172.21.15.86:0/3343667515 learned_addr learned my addr 172.21.15.86:0/3343667515 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:37.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 -- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c0b3f70 msgr2=0x7f969c158030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:37.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f969be00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c0b3f70 0x7f969c158030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:37.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1200640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f969c070ff0 0x7f969c13ddd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:37.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c0b3f70 0x7f969c158030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 -- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f969c070ff0 msgr2=0x7f969c13ddd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:37.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f969c070ff0 0x7f969c13ddd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1c00640 1 -- 172.21.15.86:0/3343667515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f969c15f410 con 0x7f969c15fc10 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.161+0000 7f96a1200640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f969c070ff0 0x7f969c13ddd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.162+0000 7f96a1c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c15fc10 0x7f969c158570 secure :-1 s=READY pgs=1462 cs=0 l=1 rev1=1 crypto rx=0x7f969400cef0 tx=0x7f969400a710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.162+0000 7f9698c00640 1 -- 172.21.15.86:0/3343667515 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969401a070 con 0x7f969c15fc10 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.162+0000 7f9698c00640 1 -- 172.21.15.86:0/3343667515 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f969400d860 con 0x7f969c15fc10 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.162+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f969c13e5a0 con 0x7f969c15fc10 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.162+0000 7f9698c00640 1 -- 172.21.15.86:0/3343667515 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969400dc50 con 0x7f969c15fc10 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.162+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f969c13eb60 con 0x7f969c15fc10 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.163+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f969c04f330 con 0x7f969c15fc10 2024-03-06T07:34:37.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.164+0000 7f9698c00640 1 -- 172.21.15.86:0/3343667515 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f969401d070 con 0x7f969c15fc10 2024-03-06T07:34:37.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.165+0000 7f9698c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f968006b4d0 0x7f968006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:37.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.165+0000 7f9698c00640 1 -- 172.21.15.86:0/3343667515 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f969408a910 con 0x7f969c15fc10 2024-03-06T07:34:37.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.165+0000 7f96a1200640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f968006b4d0 0x7f968006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:37.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.166+0000 7f96a1200640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f968006b4d0 0x7f968006d990 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f9690000f60 tx=0x7f9690009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:37.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.170+0000 7f9698c00640 1 -- 172.21.15.86:0/3343667515 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f96940599c0 con 0x7f969c15fc10 2024-03-06T07:34:37.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.367+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f969c0b2d40 con 0x7f969c15fc10 2024-03-06T07:34:37.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.372+0000 7f9698c00640 1 -- 172.21.15.86:0/3343667515 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v32)=0 v32) v1 ==== 122+0+0 (secure 0 0 0) 0x7f96940592c0 con 0x7f969c15fc10 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f968006b4d0 msgr2=0x7f968006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f968006b4d0 0x7f968006d990 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f9690000f60 tx=0x7f9690009040 comp rx=0 tx=0).stop 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c15fc10 msgr2=0x7f969c158570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c15fc10 0x7f969c158570 secure :-1 s=READY pgs=1462 cs=0 l=1 rev1=1 crypto rx=0x7f969400cef0 tx=0x7f969400a710 comp rx=0 tx=0).stop 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 shutdown_connections 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f968006b4d0 0x7f968006d990 unknown :-1 s=CLOSED pgs=624 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f969c15fc10 0x7f969c158570 unknown :-1 s=CLOSED pgs=1462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f969c0b3f70 0x7f969c158030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 --2- 172.21.15.86:0/3343667515 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f969c070ff0 0x7f969c13ddd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 >> 172.21.15.86:0/3343667515 conn(0x7f969c0b1940 msgr2=0x7f969c0ace90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 shutdown_connections 2024-03-06T07:34:37.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:37.375+0000 7f96a3c00640 1 -- 172.21.15.86:0/3343667515 wait complete. 2024-03-06T07:34:37.395 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:34:37.395 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:34:37.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:34:37.623 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:34:37.624 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:34:37.624 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:37.680 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:34:37.680 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:34:37.680 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:37.737 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:34:37.737 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:34:37.737 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:37.791 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053427,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249180k,nr_inodes=812295,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498364k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-06T07:34:37.791 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-03-06T07:34:37.791 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:37.848 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4053432,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 ramfs /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3249184k,nr_inodes=812296,mode=700,uid=1000,gid=1301 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6498372k,nr_inodes=819200,mode=755 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-06T07:34:37.848 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-06T07:34:37.849 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:38.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.063+0000 7f3c90800640 1 Processor -- start 2024-03-06T07:34:38.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.063+0000 7f3c90800640 1 -- start start 2024-03-06T07:34:38.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.064+0000 7f3c90800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.064+0000 7f3c90800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3c88070b10 0x7f3c88070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.064+0000 7f3c90800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c8806cef0 con 0x7f3c88144c00 2024-03-06T07:34:38.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.064+0000 7f3c90800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c88160370 con 0x7f3c88070b10 2024-03-06T07:34:38.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.064+0000 7f3c90800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8806d1d0 con 0x7f3c8815f9d0 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.065+0000 7f3c8d400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.065+0000 7f3c8de00640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f3c88144c00 0x7f3c8806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:57120/0 (socket says 172.21.15.86:57120) 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.065+0000 7f3c8de00640 1 -- 172.21.15.86:0/3870828456 learned_addr learned my addr 172.21.15.86:0/3870828456 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.065+0000 7f3c8e800640 1 --2- 172.21.15.86:0/3870828456 >> v2:172.21.15.194:3300/0 conn(0x7f3c88070b10 0x7f3c88070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.066+0000 7f3c8ca00640 1 -- 172.21.15.86:0/3870828456 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2828062644 0 0) 0x7f3c8806cef0 con 0x7f3c88144c00 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.066+0000 7f3c8ca00640 1 -- 172.21.15.86:0/3870828456 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3c70003520 con 0x7f3c88144c00 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.066+0000 7f3c8e800640 1 -- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 msgr2=0x7f3c8815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.066+0000 7f3c8e800640 1 --2- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.066+0000 7f3c8e800640 1 -- 172.21.15.86:0/3870828456 >> v1:172.21.15.194:6789/0 conn(0x7f3c88144c00 legacy=0x7f3c8806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.066+0000 7f3c8e800640 1 -- 172.21.15.86:0/3870828456 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c8806d4b0 con 0x7f3c88070b10 2024-03-06T07:34:38.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.066+0000 7f3c8d400640 1 --2- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:34:38.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.068+0000 7f3c8e800640 1 --2- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c88070f60 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f3c84009d90 tx=0x7f3c8402beb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=77654e9a5a838ae0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:38.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.068+0000 7f3c8ca00640 1 -- 172.21.15.86:0/3870828456 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c84039070 con 0x7f3c88070b10 2024-03-06T07:34:38.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.068+0000 7f3c8ca00640 1 -- 172.21.15.86:0/3870828456 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c8402cc60 con 0x7f3c88070b10 2024-03-06T07:34:38.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.068+0000 7f3c8ca00640 1 -- 172.21.15.86:0/3870828456 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c84034640 con 0x7f3c88070b10 2024-03-06T07:34:38.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 -- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 msgr2=0x7f3c88070f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 --2- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c88070f60 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f3c84009d90 tx=0x7f3c8402beb0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 -- 172.21.15.86:0/3870828456 shutdown_connections 2024-03-06T07:34:38.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 --2- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c88070f60 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 --2- 172.21.15.86:0/3870828456 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 -- 172.21.15.86:0/3870828456 >> 172.21.15.86:0/3870828456 conn(0x7f3c880b1730 msgr2=0x7f3c880b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:38.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 -- 172.21.15.86:0/3870828456 shutdown_connections 2024-03-06T07:34:38.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.069+0000 7f3c90800640 1 -- 172.21.15.86:0/3870828456 wait complete. 2024-03-06T07:34:38.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.071+0000 7f3c90800640 1 Processor -- start 2024-03-06T07:34:38.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.071+0000 7f3c90800640 1 -- start start 2024-03-06T07:34:38.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.072+0000 7f3c90800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c880ce2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.072+0000 7f3c90800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c88144c00 0x7f3c880b2b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.072+0000 7f3c90800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8815f9d0 0x7f3c880b3070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.072+0000 7f3c90800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8806cb50 con 0x7f3c8815f9d0 2024-03-06T07:34:38.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.072+0000 7f3c90800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c88160730 con 0x7f3c88070b10 2024-03-06T07:34:38.074 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.072+0000 7f3c90800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8806ccd0 con 0x7f3c88144c00 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8815f9d0 0x7f3c880b3070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8815f9d0 0x7f3c880b3070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54612/0 (socket says 172.21.15.86:54612) 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 -- 172.21.15.86:0/1315850047 learned_addr learned my addr 172.21.15.86:0/1315850047 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8de00640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c880ce2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8d400640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c88144c00 0x7f3c880b2b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 -- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c88144c00 msgr2=0x7f3c880b2b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c88144c00 0x7f3c880b2b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 -- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 msgr2=0x7f3c880ce2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c880ce2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8e800640 1 -- 172.21.15.86:0/1315850047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c880b35b0 con 0x7f3c8815f9d0 2024-03-06T07:34:38.076 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.073+0000 7f3c8de00640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c880ce2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.074+0000 7f3c8e800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8815f9d0 0x7f3c880b3070 secure :-1 s=READY pgs=1463 cs=0 l=1 rev1=1 crypto rx=0x7f3c84009d90 tx=0x7f3c84004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.074+0000 7f3c82000640 1 -- 172.21.15.86:0/1315850047 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c84039050 con 0x7f3c8815f9d0 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.074+0000 7f3c82000640 1 -- 172.21.15.86:0/1315850047 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c84045070 con 0x7f3c8815f9d0 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.074+0000 7f3c82000640 1 -- 172.21.15.86:0/1315850047 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c84009450 con 0x7f3c8815f9d0 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.074+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c880b3b20 con 0x7f3c8815f9d0 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.074+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c880b40e0 con 0x7f3c8815f9d0 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.075+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c880b38a0 con 0x7f3c8815f9d0 2024-03-06T07:34:38.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.076+0000 7f3c82000640 1 -- 172.21.15.86:0/1315850047 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3c8403e830 con 0x7f3c8815f9d0 2024-03-06T07:34:38.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.077+0000 7f3c82000640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b500 0x7f3c6c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.077+0000 7f3c82000640 1 -- 172.21.15.86:0/1315850047 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f3c840acc70 con 0x7f3c8815f9d0 2024-03-06T07:34:38.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.077+0000 7f3c8de00640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b500 0x7f3c6c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.078+0000 7f3c8de00640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b500 0x7f3c6c06d9c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f3c7c0011e0 tx=0x7f3c7c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:38.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.082+0000 7f3c82000640 1 -- 172.21.15.86:0/1315850047 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3c880b38a0 con 0x7f3c8815f9d0 2024-03-06T07:34:38.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.316+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3c88072890 con 0x7f3c8815f9d0 2024-03-06T07:34:38.319 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.317+0000 7f3c82000640 1 -- 172.21.15.86:0/1315850047 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 86 v86) v1 ==== 94+0+3702 (secure 0 0 0) 0x7f3c84032e20 con 0x7f3c8815f9d0 2024-03-06T07:34:38.319 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:38.319 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":86,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6169,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6833/3355855670","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":3355855670},{"type":"v1","addr":"172.21.15.194:6833","nonce":3355855670}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":6175,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2134306690","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6834","nonce":2134306690},{"type":"v1","addr":"172.21.15.194:6835","nonce":2134306690}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":7719,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6835/2002989031","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":2002989031},{"type":"v1","addr":"172.21.15.86:6835","nonce":2002989031}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":80},{"gid":8073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6837/235260164","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6836","nonce":235260164},{"type":"v1","addr":"172.21.15.86:6837","nonce":235260164}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":86}],"filesystems":[]} 2024-03-06T07:34:38.320 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 86 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b500 msgr2=0x7f3c6c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b500 0x7f3c6c06d9c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f3c7c0011e0 tx=0x7f3c7c009040 comp rx=0 tx=0).stop 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8815f9d0 msgr2=0x7f3c880b3070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8815f9d0 0x7f3c880b3070 secure :-1 s=READY pgs=1463 cs=0 l=1 rev1=1 crypto rx=0x7f3c84009d90 tx=0x7f3c84004a40 comp rx=0 tx=0).stop 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 shutdown_connections 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6c06b500 0x7f3c6c06d9c0 unknown :-1 s=CLOSED pgs=625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c8815f9d0 0x7f3c880b3070 unknown :-1 s=CLOSED pgs=1463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c88144c00 0x7f3c880b2b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 --2- 172.21.15.86:0/1315850047 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c88070b10 0x7f3c880ce2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 >> 172.21.15.86:0/1315850047 conn(0x7f3c880b1730 msgr2=0x7f3c880b0f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:38.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.320+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 shutdown_connections 2024-03-06T07:34:38.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.321+0000 7f3c90800640 1 -- 172.21.15.86:0/1315850047 wait complete. 2024-03-06T07:34:38.340 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-06T07:34:38.340 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-06T07:34:38.340 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-06T07:34:38.340 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:34:38.340 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-06T07:34:38.341 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-06T07:34:38.341 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-06T07:34:38.341 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:34:38.341 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-06T07:34:38.341 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-06T07:34:38.341 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:34:38.341 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:34:38.342 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-06T07:34:38.342 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-06T07:34:38.342 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:34:38.342 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:34:38.363 INFO:tasks.ceph.mds.a:Stopped 2024-03-06T07:34:38.363 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-06T07:34:38.365 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:34:38.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-06T07:34:38.365 INFO:tasks.ceph.mds.c:Stopped 2024-03-06T07:34:38.365 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-06T07:34:38.367 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:34:38.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-06T07:34:38.412 INFO:tasks.ceph.mds.a:Started 2024-03-06T07:34:38.412 DEBUG:teuthology.parallel:result is None 2024-03-06T07:34:38.414 INFO:tasks.ceph.mds.b:Started 2024-03-06T07:34:38.414 DEBUG:teuthology.parallel:result is None 2024-03-06T07:34:38.418 INFO:tasks.ceph.mds.c:Started 2024-03-06T07:34:38.418 DEBUG:teuthology.parallel:result is None 2024-03-06T07:34:38.420 INFO:tasks.ceph.mds.d:Started 2024-03-06T07:34:38.420 DEBUG:teuthology.parallel:result is None 2024-03-06T07:34:38.421 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-06T07:34:38.566 INFO:tasks.ceph.mds.a.smithi086.stdout:starting mds.a at 2024-03-06T07:34:38.566 INFO:tasks.ceph.mds.c.smithi086.stdout:starting mds.c at 2024-03-06T07:34:38.568 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:34:38.566+0000 7f3a10c00640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6834/0: (98) Address already in use 2024-03-06T07:34:38.577 INFO:tasks.ceph.mds.b.smithi194.stdout:starting mds.b at 2024-03-06T07:34:38.578 INFO:tasks.ceph.mds.d.smithi194.stdout:starting mds.d at 2024-03-06T07:34:38.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.595+0000 7f503ee00640 1 Processor -- start 2024-03-06T07:34:38.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.595+0000 7f503ee00640 1 -- start start 2024-03-06T07:34:38.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.596+0000 7f503ee00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f503806e470 0x7f50380cbab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.596+0000 7f503ee00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f503815b9e0 con 0x7f503806db80 2024-03-06T07:34:38.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.596+0000 7f503ee00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f50380cc220 con 0x7f50381581d0 2024-03-06T07:34:38.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.596+0000 7f503ee00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f50380cbff0 con 0x7f503806e470 2024-03-06T07:34:38.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.596+0000 7f5037400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f503806db80 0x7f50380ca240 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59018/0 (socket says 172.21.15.86:59018) 2024-03-06T07:34:38.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.596+0000 7f503ce00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f503806e470 0x7f50380cbab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.597+0000 7f5037400640 1 -- 172.21.15.86:0/2737515565 learned_addr learned my addr 172.21.15.86:0/2737515565 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:38.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.597+0000 7f503ce00640 1 -- 172.21.15.86:0/2737515565 >> v1:172.21.15.194:6789/0 conn(0x7f50381581d0 legacy=0x7f503806d470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.597+0000 7f503ce00640 1 -- 172.21.15.86:0/2737515565 >> v1:172.21.15.86:6789/0 conn(0x7f503806db80 legacy=0x7f50380ca240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.597+0000 7f503ce00640 1 -- 172.21.15.86:0/2737515565 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f503815ac20 con 0x7f503806e470 2024-03-06T07:34:38.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.598+0000 7f503ce00640 1 --2- 172.21.15.86:0/2737515565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806e470 0x7f50380cbab0 secure :-1 s=READY pgs=1472 cs=0 l=1 rev1=1 crypto rx=0x7f5030009d90 tx=0x7f503002e150 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bc2746957c7bb5e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:38.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.598+0000 7f5036a00640 1 -- 172.21.15.86:0/2737515565 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f503003b070 con 0x7f503806e470 2024-03-06T07:34:38.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.598+0000 7f5036a00640 1 -- 172.21.15.86:0/2737515565 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5030036440 con 0x7f503806e470 2024-03-06T07:34:38.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.598+0000 7f5036a00640 1 -- 172.21.15.86:0/2737515565 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5030036810 con 0x7f503806e470 2024-03-06T07:34:38.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.599+0000 7f503ee00640 1 -- 172.21.15.86:0/2737515565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806e470 msgr2=0x7f50380cbab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.599+0000 7f503ee00640 1 --2- 172.21.15.86:0/2737515565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806e470 0x7f50380cbab0 secure :-1 s=READY pgs=1472 cs=0 l=1 rev1=1 crypto rx=0x7f5030009d90 tx=0x7f503002e150 comp rx=0 tx=0).stop 2024-03-06T07:34:38.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.599+0000 7f503ee00640 1 -- 172.21.15.86:0/2737515565 shutdown_connections 2024-03-06T07:34:38.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.599+0000 7f503ee00640 1 --2- 172.21.15.86:0/2737515565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806e470 0x7f50380cbab0 unknown :-1 s=CLOSED pgs=1472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.599+0000 7f503ee00640 1 -- 172.21.15.86:0/2737515565 >> 172.21.15.86:0/2737515565 conn(0x7f50380cc4f0 msgr2=0x7f50380cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:38.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.599+0000 7f503ee00640 1 -- 172.21.15.86:0/2737515565 shutdown_connections 2024-03-06T07:34:38.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.599+0000 7f503ee00640 1 -- 172.21.15.86:0/2737515565 wait complete. 2024-03-06T07:34:38.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.600+0000 7f503ee00640 1 Processor -- start 2024-03-06T07:34:38.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.601+0000 7f503ee00640 1 -- start start 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.601+0000 7f503ee00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806db80 0x7f50380b21c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.601+0000 7f503ee00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f503806e470 0x7f50380b0020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.601+0000 7f503ee00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f50381581d0 0x7f50380ade60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.601+0000 7f503ee00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f503815b4c0 con 0x7f503806db80 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.601+0000 7f503ee00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f503815b1e0 con 0x7f503806e470 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.601+0000 7f503ee00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f503806cbe0 con 0x7f50381581d0 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806db80 0x7f50380b21c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806db80 0x7f50380b21c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54670/0 (socket says 172.21.15.86:54670) 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 -- 172.21.15.86:0/368644762 learned_addr learned my addr 172.21.15.86:0/368644762 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037400640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f503806e470 0x7f50380b0020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f503ce00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f50381581d0 0x7f50380ade60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 -- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f50381581d0 msgr2=0x7f50380ade60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f50381581d0 0x7f50380ade60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 -- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f503806e470 msgr2=0x7f50380b0020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f503806e470 0x7f50380b0020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 -- 172.21.15.86:0/368644762 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50380b1120 con 0x7f503806db80 2024-03-06T07:34:38.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f5037e00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806db80 0x7f50380b21c0 secure :-1 s=READY pgs=1473 cs=0 l=1 rev1=1 crypto rx=0x7f502800c3c0 tx=0x7f502800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:38.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.602+0000 7f502fe00640 1 -- 172.21.15.86:0/368644762 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f50280052d0 con 0x7f503806db80 2024-03-06T07:34:38.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.603+0000 7f502fe00640 1 -- 172.21.15.86:0/368644762 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5028005490 con 0x7f503806db80 2024-03-06T07:34:38.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.603+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f50380b1400 con 0x7f503806db80 2024-03-06T07:34:38.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.603+0000 7f502fe00640 1 -- 172.21.15.86:0/368644762 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5028007260 con 0x7f503806db80 2024-03-06T07:34:38.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.603+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f50380b19c0 con 0x7f503806db80 2024-03-06T07:34:38.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.604+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f503806e1b0 con 0x7f503806db80 2024-03-06T07:34:38.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.605+0000 7f502fe00640 1 -- 172.21.15.86:0/368644762 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5028007400 con 0x7f503806db80 2024-03-06T07:34:38.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.606+0000 7f502fe00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f501806b4d0 0x7f501806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:38.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.606+0000 7f502fe00640 1 -- 172.21.15.86:0/368644762 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f50280892c0 con 0x7f503806db80 2024-03-06T07:34:38.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.606+0000 7f5037400640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f501806b4d0 0x7f501806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:38.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.606+0000 7f5037400640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f501806b4d0 0x7f501806d990 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f50380aeb10 tx=0x7f5024009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:38.611 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.610+0000 7f502fe00640 1 -- 172.21.15.86:0/368644762 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f503806e1b0 con 0x7f503806db80 2024-03-06T07:34:38.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.809+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f503804f330 con 0x7f503806db80 2024-03-06T07:34:38.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.952+0000 7f502fe00640 1 -- 172.21.15.86:0/368644762 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v121) v1 ==== 97+0+0 (secure 0 0 0) 0x7f5028057e20 con 0x7f503806db80 2024-03-06T07:34:38.954 INFO:teuthology.orchestra.run.smithi086.stderr: removed all blocklist entries 2024-03-06T07:34:38.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f501806b4d0 msgr2=0x7f501806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f501806b4d0 0x7f501806d990 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f50380aeb10 tx=0x7f5024009040 comp rx=0 tx=0).stop 2024-03-06T07:34:38.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806db80 msgr2=0x7f50380b21c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:38.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806db80 0x7f50380b21c0 secure :-1 s=READY pgs=1473 cs=0 l=1 rev1=1 crypto rx=0x7f502800c3c0 tx=0x7f502800c880 comp rx=0 tx=0).stop 2024-03-06T07:34:38.958 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 shutdown_connections 2024-03-06T07:34:38.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f501806b4d0 0x7f501806d990 unknown :-1 s=CLOSED pgs=626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f50381581d0 0x7f50380ade60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f503806e470 0x7f50380b0020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 --2- 172.21.15.86:0/368644762 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f503806db80 0x7f50380b21c0 unknown :-1 s=CLOSED pgs=1473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:38.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.956+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 >> 172.21.15.86:0/368644762 conn(0x7f50380cc4f0 msgr2=0x7f5038157590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:38.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.957+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 shutdown_connections 2024-03-06T07:34:38.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:38.957+0000 7f503ee00640 1 -- 172.21.15.86:0/368644762 wait complete. 2024-03-06T07:34:38.980 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-06T07:34:39.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.173+0000 7f9de9000640 1 Processor -- start 2024-03-06T07:34:39.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.173+0000 7f9de9000640 1 -- start start 2024-03-06T07:34:39.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de9000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9de4144c00 0x7f9de406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de9000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9de406ee70 con 0x7f9de415fe60 2024-03-06T07:34:39.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de9000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9de406cef0 con 0x7f9de415f9d0 2024-03-06T07:34:39.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de9000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9de406d1d0 con 0x7f9de4144c00 2024-03-06T07:34:39.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de2800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9de415fe60 0x7f9de40ce7b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59026/0 (socket says 172.21.15.86:59026) 2024-03-06T07:34:39.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de2800640 1 -- 172.21.15.86:0/2027923882 learned_addr learned my addr 172.21.15.86:0/2027923882 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:39.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3120693136 0 0) 0x7f9ddc001110 con 0x7f9de415fe60 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9dc8003520 con 0x7f9de415fe60 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.175+0000 7f9de1e00640 1 --2- 172.21.15.86:0/2027923882 >> v2:172.21.15.194:3300/0 conn(0x7f9de4144c00 0x7f9de406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2601957728 0 0) 0x7f9dc8003520 con 0x7f9de415fe60 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 >> v2:172.21.15.194:3300/0 conn(0x7f9de4144c00 msgr2=0x7f9de406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 --2- 172.21.15.86:0/2027923882 >> v2:172.21.15.194:3300/0 conn(0x7f9de4144c00 0x7f9de406e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 >> v1:172.21.15.194:6789/0 conn(0x7f9de415f9d0 legacy=0x7f9de40ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9de406d4b0 con 0x7f9de415fe60 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9ddc001840 con 0x7f9de415fe60 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 >> v1:172.21.15.86:6789/0 conn(0x7f9de415fe60 legacy=0x7f9de40ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9dc80039b0 0x7f9dc8003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9dc8004800 0x7f9dc8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80051c0 0x7f9dc800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9de406d1d0 con 0x7f9dc80039b0 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dc800c0f0 con 0x7f9dc8004800 2024-03-06T07:34:39.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9dc800c2f0 con 0x7f9dc80051c0 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de1e00640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9dc80039b0 0x7f9dc8003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de1400640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9dc8004800 0x7f9dc8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.176+0000 7f9de2800640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80051c0 0x7f9dc800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.177+0000 7f9de1e00640 1 -- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80051c0 msgr2=0x7f9dc800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.177+0000 7f9de1e00640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80051c0 0x7f9dc800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.177+0000 7f9de1e00640 1 -- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9dc8004800 msgr2=0x7f9dc8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.177+0000 7f9de1e00640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9dc8004800 0x7f9dc8004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.177+0000 7f9de1e00640 1 -- 172.21.15.86:0/2027923882 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9dc8003520 con 0x7f9dc80039b0 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.177+0000 7f9de2800640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80051c0 0x7f9dc800b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:39.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.177+0000 7f9de2800640 1 -- 172.21.15.86:0/2027923882 reap_dead start 2024-03-06T07:34:39.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.178+0000 7f9de1e00640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9dc80039b0 0x7f9dc8003db0 secure :-1 s=READY pgs=1475 cs=0 l=1 rev1=1 crypto rx=0x7f9dd4007330 tx=0x7f9dc8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9dc9fcda78c9540e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:39.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.178+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd4039070 con 0x7f9dc80039b0 2024-03-06T07:34:39.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.178+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9dd402cc70 con 0x7f9dc80039b0 2024-03-06T07:34:39.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.178+0000 7f9de0a00640 1 -- 172.21.15.86:0/2027923882 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd4009be0 con 0x7f9dc80039b0 2024-03-06T07:34:39.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.179+0000 7f9de9000640 1 -- 172.21.15.86:0/2027923882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9dc80039b0 msgr2=0x7f9dc8003db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.179+0000 7f9de9000640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9dc80039b0 0x7f9dc8003db0 secure :-1 s=READY pgs=1475 cs=0 l=1 rev1=1 crypto rx=0x7f9dd4007330 tx=0x7f9dc8005830 comp rx=0 tx=0).stop 2024-03-06T07:34:39.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.179+0000 7f9de9000640 1 -- 172.21.15.86:0/2027923882 shutdown_connections 2024-03-06T07:34:39.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.179+0000 7f9de9000640 1 --2- 172.21.15.86:0/2027923882 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9dc80039b0 0x7f9dc8003db0 unknown :-1 s=CLOSED pgs=1475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.179+0000 7f9de9000640 1 -- 172.21.15.86:0/2027923882 >> 172.21.15.86:0/2027923882 conn(0x7f9de40b1730 msgr2=0x7f9de40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:39.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.179+0000 7f9de9000640 1 -- 172.21.15.86:0/2027923882 shutdown_connections 2024-03-06T07:34:39.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.179+0000 7f9de9000640 1 -- 172.21.15.86:0/2027923882 wait complete. 2024-03-06T07:34:39.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.181+0000 7f9de9000640 1 Processor -- start 2024-03-06T07:34:39.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.181+0000 7f9de9000640 1 -- start start 2024-03-06T07:34:39.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de9000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80039b0 0x7f9de415b290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9de40b3a20 0x7f9de40b4160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de9000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9de40b46a0 0x7f9de40ae1a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de9000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9de4160730 con 0x7f9de40b3a20 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de9000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9de4160270 con 0x7f9de40b46a0 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de9000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9de406cb50 con 0x7f9dc80039b0 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9de40b3a20 0x7f9de40b4160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9de40b3a20 0x7f9de40b4160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54674/0 (socket says 172.21.15.86:54674) 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 -- 172.21.15.86:0/1480817371 learned_addr learned my addr 172.21.15.86:0/1480817371 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1e00640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80039b0 0x7f9de415b290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 -- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80039b0 msgr2=0x7f9de415b290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80039b0 0x7f9de415b290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 -- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9de40b46a0 msgr2=0x7f9de40ae1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de2800640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9de40b46a0 0x7f9de40ae1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9de40b46a0 0x7f9de40ae1a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.182+0000 7f9de1400640 1 -- 172.21.15.86:0/1480817371 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9de413e8d0 con 0x7f9de40b3a20 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.183+0000 7f9de1400640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9de40b3a20 0x7f9de40b4160 secure :-1 s=READY pgs=1476 cs=0 l=1 rev1=1 crypto rx=0x7f9dd000e4b0 tx=0x7f9dd000e970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.183+0000 7f9dda000640 1 -- 172.21.15.86:0/1480817371 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd00135f0 con 0x7f9de40b3a20 2024-03-06T07:34:39.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.183+0000 7f9dda000640 1 -- 172.21.15.86:0/1480817371 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9dd001a070 con 0x7f9de40b3a20 2024-03-06T07:34:39.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.183+0000 7f9dda000640 1 -- 172.21.15.86:0/1480817371 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd00139f0 con 0x7f9de40b3a20 2024-03-06T07:34:39.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.183+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9de413ebb0 con 0x7f9de40b3a20 2024-03-06T07:34:39.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.183+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9de413f170 con 0x7f9de40b3a20 2024-03-06T07:34:39.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.184+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9de404f330 con 0x7f9de40b3a20 2024-03-06T07:34:39.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.185+0000 7f9dda000640 1 -- 172.21.15.86:0/1480817371 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9dd0013b90 con 0x7f9de40b3a20 2024-03-06T07:34:39.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.186+0000 7f9dda000640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9dc406b4d0 0x7f9dc406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.186+0000 7f9dda000640 1 -- 172.21.15.86:0/1480817371 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f9dd0089fe0 con 0x7f9de40b3a20 2024-03-06T07:34:39.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.186+0000 7f9de1e00640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9dc406b4d0 0x7f9dc406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.187+0000 7f9de1e00640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9dc406b4d0 0x7f9dc406d990 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f9dd4036000 tx=0x7f9dd4030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:39.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.190+0000 7f9dda000640 1 -- 172.21.15.86:0/1480817371 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9dd0024020 con 0x7f9de40b3a20 2024-03-06T07:34:39.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.436+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f9de40b4a80 con 0x7f9de40b3a20 2024-03-06T07:34:39.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.437+0000 7f9dda000640 1 -- 172.21.15.86:0/1480817371 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v19) v1 ==== 80+0+5304 (secure 0 0 0) 0x7f9dd00591e0 con 0x7f9de40b3a20 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "auth_dump": [ 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.a", 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlr0LCNRAAJON7Lf+9WKad4t+JYxHujw==", 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.439 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.b", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlkQ8xGxAADGKYI5YDkiZiUFVzRn06Ug==", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.c", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlF+GSChAA1fOwS8/b871Het0Asui8iA==", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:34:39.440 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mds.d", 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlzH3RKxAA/j4HObHxySA1Zm7sudZW4Q==", 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.441 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.0", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCJGuhlzadgHhAAVJU0dzPapNIBqvztNwIpoQ==", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.1", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCMGuhlAhT4NhAAS3sgJq5drB9eApwRvR++5w==", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.442 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.2", 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCQGuhlGaYyExAABKCQwoKGNWolsSA1zTlgAA==", 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.3", 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCTGuhlY2EqHRAAKQq0BmwyYPQ34J+bAWSB2g==", 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.443 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.4", 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCaGuhlX2oZKxAArrbsix9rvdFyFS9bDdtXcA==", 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.444 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.5", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCeGuhl2dLvCRAABLRUH5SjIEut7cWq0gALNQ==", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.6", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQChGuhlM0zTIRAAzTG85IiOvzO67xczSNUhJg==", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.445 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "osd.7", 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCkGuhlnfDvOhAA6VtAJMq5eU8kZyI8/3VH5g==", 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow profile osd", 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile osd", 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.0", 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw==", 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.446 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow r", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.1", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlAcUZExAAsaSrFXVjBTRGckunnf9cNg==", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.2", 2024-03-06T07:34:39.447 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlXa1XCBAAWU4+perDJO0oYdlBfsl54w==", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.3", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCFGuhlT/TpHBAAku18cRj6NKnF+srJevraaQ==", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow r", 2024-03-06T07:34:39.448 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow rw", 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow rwx" 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "client.admin", 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCCGuhlhGrTKBAAxvNbIsa7SbPK7D4Zm7oHow==", 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": "allow *", 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow *", 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.x", 2024-03-06T07:34:39.449 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlxAVZKhAAGcgajI4raUzOAwcaQaph4A==", 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "entity": "mgr.y", 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "AQCDGuhlWZVRHhAATyrWAmAN+iB+ttZwWRymkQ==", 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "caps": { 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": "allow *", 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": "allow profile mgr", 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": "allow *" 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:39.450 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-03-06T07:34:39.451 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9dc406b4d0 msgr2=0x7f9dc406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9dc406b4d0 0x7f9dc406d990 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f9dd4036000 tx=0x7f9dd4030000 comp rx=0 tx=0).stop 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9de40b3a20 msgr2=0x7f9de40b4160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9de40b3a20 0x7f9de40b4160 secure :-1 s=READY pgs=1476 cs=0 l=1 rev1=1 crypto rx=0x7f9dd000e4b0 tx=0x7f9dd000e970 comp rx=0 tx=0).stop 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 shutdown_connections 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9dc406b4d0 0x7f9dc406d990 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9de40b46a0 0x7f9de40ae1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9de40b3a20 0x7f9de40b4160 unknown :-1 s=CLOSED pgs=1476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 --2- 172.21.15.86:0/1480817371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9dc80039b0 0x7f9de415b290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 >> 172.21.15.86:0/1480817371 conn(0x7f9de40b1730 msgr2=0x7f9de40afc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.439+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 shutdown_connections 2024-03-06T07:34:39.452 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.440+0000 7f9de9000640 1 -- 172.21.15.86:0/1480817371 wait complete. 2024-03-06T07:34:39.458 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:39.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.654+0000 7f7f85600640 1 Processor -- start 2024-03-06T07:34:39.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.654+0000 7f7f85600640 1 -- start start 2024-03-06T07:34:39.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f85600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f7f8006ec00 0x7f7f800ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f85600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7f8015b090 con 0x7f7f8015acb0 2024-03-06T07:34:39.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f85600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7f800b2a30 con 0x7f7f800b4680 2024-03-06T07:34:39.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f85600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7f800b4360 con 0x7f7f8006ec00 2024-03-06T07:34:39.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f7ea00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7f8015acb0 0x7f7f800b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59032/0 (socket says 172.21.15.86:59032) 2024-03-06T07:34:39.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f7ea00640 1 -- 172.21.15.86:0/2476752947 learned_addr learned my addr 172.21.15.86:0/2476752947 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:39.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f7f400640 1 --2- 172.21.15.86:0/2476752947 >> v2:172.21.15.86:3300/0 conn(0x7f7f8006ec00 0x7f7f800ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f7f400640 1 -- 172.21.15.86:0/2476752947 >> v1:172.21.15.194:6789/0 conn(0x7f7f800b4680 legacy=0x7f7f8006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:39.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.656+0000 7f7f7f400640 1 -- 172.21.15.86:0/2476752947 >> v1:172.21.15.86:6789/0 conn(0x7f7f8015acb0 legacy=0x7f7f800b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.657+0000 7f7f7f400640 1 -- 172.21.15.86:0/2476752947 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f800b2cf0 con 0x7f7f8006ec00 2024-03-06T07:34:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.657+0000 7f7f7d600640 1 -- 172.21.15.86:0/2476752947 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3607285508 0 0) 0x7f7f8015b090 con 0x7f7f8015acb0 2024-03-06T07:34:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.658+0000 7f7f7f400640 1 --2- 172.21.15.86:0/2476752947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f8006ec00 0x7f7f800ccfb0 secure :-1 s=READY pgs=1478 cs=0 l=1 rev1=1 crypto rx=0x7f7f78009d90 tx=0x7f7f7802beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7d0e0583d2e1ecad server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:39.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.658+0000 7f7f7d600640 1 -- 172.21.15.86:0/2476752947 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f78039070 con 0x7f7f8006ec00 2024-03-06T07:34:39.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.658+0000 7f7f7d600640 1 -- 172.21.15.86:0/2476752947 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f7802cc60 con 0x7f7f8006ec00 2024-03-06T07:34:39.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.658+0000 7f7f7d600640 1 -- 172.21.15.86:0/2476752947 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f78034660 con 0x7f7f8006ec00 2024-03-06T07:34:39.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.659+0000 7f7f85600640 1 -- 172.21.15.86:0/2476752947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f8006ec00 msgr2=0x7f7f800ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.659+0000 7f7f85600640 1 --2- 172.21.15.86:0/2476752947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f8006ec00 0x7f7f800ccfb0 secure :-1 s=READY pgs=1478 cs=0 l=1 rev1=1 crypto rx=0x7f7f78009d90 tx=0x7f7f7802beb0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.659+0000 7f7f85600640 1 -- 172.21.15.86:0/2476752947 shutdown_connections 2024-03-06T07:34:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.659+0000 7f7f85600640 1 --2- 172.21.15.86:0/2476752947 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f8006ec00 0x7f7f800ccfb0 unknown :-1 s=CLOSED pgs=1478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.659+0000 7f7f85600640 1 -- 172.21.15.86:0/2476752947 >> 172.21.15.86:0/2476752947 conn(0x7f7f800b1730 msgr2=0x7f7f800b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.659+0000 7f7f85600640 1 -- 172.21.15.86:0/2476752947 shutdown_connections 2024-03-06T07:34:39.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.659+0000 7f7f85600640 1 -- 172.21.15.86:0/2476752947 wait complete. 2024-03-06T07:34:39.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.660+0000 7f7f85600640 1 Processor -- start 2024-03-06T07:34:39.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.661+0000 7f7f85600640 1 -- start start 2024-03-06T07:34:39.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.661+0000 7f7f85600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f8006ec00 0x7f7f80153a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.661+0000 7f7f85600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f800b4680 0x7f7f80150f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f85600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f8015acb0 0x7f7f8015d220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f85600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f800b4010 con 0x7f7f800b4680 2024-03-06T07:34:39.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f85600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f800b32b0 con 0x7f7f8006ec00 2024-03-06T07:34:39.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f85600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7f80072340 con 0x7f7f8015acb0 2024-03-06T07:34:39.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f800b4680 0x7f7f80150f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f800b4680 0x7f7f80150f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54680/0 (socket says 172.21.15.86:54680) 2024-03-06T07:34:39.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 -- 172.21.15.86:0/1058734705 learned_addr learned my addr 172.21.15.86:0/1058734705 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7f400640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f8015acb0 0x7f7f8015d220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7ea00640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f8006ec00 0x7f7f80153a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 -- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f8015acb0 msgr2=0x7f7f8015d220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f8015acb0 0x7f7f8015d220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 -- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f8006ec00 msgr2=0x7f7f80153a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f8006ec00 0x7f7f80153a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.662+0000 7f7f7e000640 1 -- 172.21.15.86:0/1058734705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f8015d760 con 0x7f7f800b4680 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.663+0000 7f7f7e000640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f800b4680 0x7f7f80150f70 secure :-1 s=READY pgs=1479 cs=0 l=1 rev1=1 crypto rx=0x7f7f6c00c430 tx=0x7f7f6c00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.663+0000 7f7f76a00640 1 -- 172.21.15.86:0/1058734705 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f6c011790 con 0x7f7f800b4680 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.663+0000 7f7f76a00640 1 -- 172.21.15.86:0/1058734705 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f6c017070 con 0x7f7f800b4680 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.663+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7f8015da00 con 0x7f7f800b4680 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.663+0000 7f7f76a00640 1 -- 172.21.15.86:0/1058734705 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f6c011bb0 con 0x7f7f800b4680 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.663+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7f8015dfc0 con 0x7f7f800b4680 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.664+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7f8004f330 con 0x7f7f800b4680 2024-03-06T07:34:39.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.665+0000 7f7f76a00640 1 -- 172.21.15.86:0/1058734705 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7f6c01c460 con 0x7f7f800b4680 2024-03-06T07:34:39.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.666+0000 7f7f76a00640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f6006b4d0 0x7f7f6006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:39.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.666+0000 7f7f76a00640 1 -- 172.21.15.86:0/1058734705 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f7f6c089da0 con 0x7f7f800b4680 2024-03-06T07:34:39.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.667+0000 7f7f7ea00640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f6006b4d0 0x7f7f6006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:39.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.667+0000 7f7f7ea00640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f6006b4d0 0x7f7f6006d990 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f7f70000f60 tx=0x7f7f70009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:39.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.671+0000 7f7f76a00640 1 -- 172.21.15.86:0/1058734705 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7f6c058f60 con 0x7f7f800b4680 2024-03-06T07:34:39.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.867+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7f80157b40 con 0x7f7f800b4680 2024-03-06T07:34:39.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.869+0000 7f7f76a00640 1 -- 172.21.15.86:0/1058734705 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v123) v1 ==== 74+0+12913 (secure 0 0 0) 0x7f7f6c058860 con 0x7f7f800b4680 2024-03-06T07:34:39.870 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:39.871 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":123,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:34:38.966167+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":115,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:39.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f6006b4d0 msgr2=0x7f7f6006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f6006b4d0 0x7f7f6006d990 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f7f70000f60 tx=0x7f7f70009040 comp rx=0 tx=0).stop 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f800b4680 msgr2=0x7f7f80150f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f800b4680 0x7f7f80150f70 secure :-1 s=READY pgs=1479 cs=0 l=1 rev1=1 crypto rx=0x7f7f6c00c430 tx=0x7f7f6c00c8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 shutdown_connections 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7f6006b4d0 0x7f7f6006d990 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7f8015acb0 0x7f7f8015d220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7f800b4680 0x7f7f80150f70 unknown :-1 s=CLOSED pgs=1479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 --2- 172.21.15.86:0/1058734705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7f8006ec00 0x7f7f80153a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.871+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 >> 172.21.15.86:0/1058734705 conn(0x7f7f800b1730 msgr2=0x7f7f800af730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.872+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 shutdown_connections 2024-03-06T07:34:39.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:39.872+0000 7f7f85600640 1 -- 172.21.15.86:0/1058734705 wait complete. 2024-03-06T07:34:39.891 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:34:39.891 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:34:40.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.103+0000 7f0a47000640 1 Processor -- start 2024-03-06T07:34:40.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.103+0000 7f0a47000640 1 -- start start 2024-03-06T07:34:40.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a47000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0a400b3f70 0x7f0a40160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a47000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0a40070ff0 0x7f0a400ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a47000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0a40071440 con 0x7f0a4015fc10 2024-03-06T07:34:40.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a47000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a400b46c0 con 0x7f0a40070ff0 2024-03-06T07:34:40.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a47000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a400b49a0 con 0x7f0a400b3f70 2024-03-06T07:34:40.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a45000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0a40070ff0 0x7f0a400ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a45000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0a40070ff0 0x7f0a400ce4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54686/0 (socket says 172.21.15.86:54686) 2024-03-06T07:34:40.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a45000640 1 -- 172.21.15.86:0/999890969 learned_addr learned my addr 172.21.15.86:0/999890969 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:40.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a3f400640 1 --2- 172.21.15.86:0/999890969 >> v2:172.21.15.194:3300/0 conn(0x7f0a400b3f70 0x7f0a40160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a45000640 1 -- 172.21.15.86:0/999890969 >> v2:172.21.15.194:3300/0 conn(0x7f0a400b3f70 msgr2=0x7f0a40160700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:40.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a45000640 1 --2- 172.21.15.86:0/999890969 >> v2:172.21.15.194:3300/0 conn(0x7f0a400b3f70 0x7f0a40160700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a45000640 1 -- 172.21.15.86:0/999890969 >> v1:172.21.15.194:6789/0 conn(0x7f0a4015fc10 legacy=0x7f0a4015fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:40.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.105+0000 7f0a45000640 1 -- 172.21.15.86:0/999890969 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a40072c90 con 0x7f0a40070ff0 2024-03-06T07:34:40.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.106+0000 7f0a45000640 1 --2- 172.21.15.86:0/999890969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a400ce4a0 secure :-1 s=READY pgs=1480 cs=0 l=1 rev1=1 crypto rx=0x7f0a40074a50 tx=0x7f0a3802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=182ea40fdb6ff715 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:40.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.107+0000 7f0a3ea00640 1 -- 172.21.15.86:0/999890969 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a3802ca80 con 0x7f0a40070ff0 2024-03-06T07:34:40.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.107+0000 7f0a3ea00640 1 -- 172.21.15.86:0/999890969 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a3802cc20 con 0x7f0a40070ff0 2024-03-06T07:34:40.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.107+0000 7f0a3ea00640 1 -- 172.21.15.86:0/999890969 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a380343f0 con 0x7f0a40070ff0 2024-03-06T07:34:40.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.107+0000 7f0a47000640 1 -- 172.21.15.86:0/999890969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 msgr2=0x7f0a400ce4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.107+0000 7f0a47000640 1 --2- 172.21.15.86:0/999890969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a400ce4a0 secure :-1 s=READY pgs=1480 cs=0 l=1 rev1=1 crypto rx=0x7f0a40074a50 tx=0x7f0a3802bee0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.108+0000 7f0a47000640 1 -- 172.21.15.86:0/999890969 shutdown_connections 2024-03-06T07:34:40.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.108+0000 7f0a47000640 1 --2- 172.21.15.86:0/999890969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a400ce4a0 unknown :-1 s=CLOSED pgs=1480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.108+0000 7f0a47000640 1 --2- 172.21.15.86:0/999890969 >> v2:172.21.15.194:3300/0 conn(0x7f0a400b3f70 0x7f0a40160700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.108+0000 7f0a47000640 1 -- 172.21.15.86:0/999890969 >> 172.21.15.86:0/999890969 conn(0x7f0a400b1940 msgr2=0x7f0a400b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:40.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.108+0000 7f0a47000640 1 -- 172.21.15.86:0/999890969 shutdown_connections 2024-03-06T07:34:40.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.108+0000 7f0a47000640 1 -- 172.21.15.86:0/999890969 wait complete. 2024-03-06T07:34:40.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.109+0000 7f0a47000640 1 Processor -- start 2024-03-06T07:34:40.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.110+0000 7f0a47000640 1 -- start start 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.110+0000 7f0a47000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a4014beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.110+0000 7f0a47000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a400b3f70 0x7f0a401598d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.110+0000 7f0a47000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a4015fc10 0x7f0a4015a480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.110+0000 7f0a47000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a40072a30 con 0x7f0a40070ff0 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.110+0000 7f0a47000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a400728b0 con 0x7f0a400b3f70 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.110+0000 7f0a47000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0a4015c2e0 con 0x7f0a4015fc10 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a4014beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a4014beb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54692/0 (socket says 172.21.15.86:54692) 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 -- 172.21.15.86:0/2079365033 learned_addr learned my addr 172.21.15.86:0/2079365033 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3f400640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a400b3f70 0x7f0a401598d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 -- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a4015fc10 msgr2=0x7f0a4015a480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a4015fc10 0x7f0a4015a480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 -- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a400b3f70 msgr2=0x7f0a401598d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a400b3f70 0x7f0a401598d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 -- 172.21.15.86:0/2079365033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a400cba40 con 0x7f0a40070ff0 2024-03-06T07:34:40.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3f400640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a400b3f70 0x7f0a401598d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:40.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.111+0000 7f0a3fe00640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a4014beb0 secure :-1 s=READY pgs=1481 cs=0 l=1 rev1=1 crypto rx=0x7f0a3000b410 tx=0x7f0a3000b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:40.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.112+0000 7f0a37e00640 1 -- 172.21.15.86:0/2079365033 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a30015020 con 0x7f0a40070ff0 2024-03-06T07:34:40.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.112+0000 7f0a37e00640 1 -- 172.21.15.86:0/2079365033 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a30011830 con 0x7f0a40070ff0 2024-03-06T07:34:40.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.112+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0a4014c750 con 0x7f0a40070ff0 2024-03-06T07:34:40.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.112+0000 7f0a37e00640 1 -- 172.21.15.86:0/2079365033 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a30011c20 con 0x7f0a40070ff0 2024-03-06T07:34:40.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.112+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a400ca4a0 con 0x7f0a40070ff0 2024-03-06T07:34:40.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.113+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0a4004f330 con 0x7f0a40070ff0 2024-03-06T07:34:40.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.114+0000 7f0a37e00640 1 -- 172.21.15.86:0/2079365033 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0a30018020 con 0x7f0a40070ff0 2024-03-06T07:34:40.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.115+0000 7f0a37e00640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2006b4d0 0x7f0a2006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.115+0000 7f0a37e00640 1 -- 172.21.15.86:0/2079365033 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f0a30089d90 con 0x7f0a40070ff0 2024-03-06T07:34:40.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.115+0000 7f0a3f400640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2006b4d0 0x7f0a2006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.116+0000 7f0a3f400640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2006b4d0 0x7f0a2006d990 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f0a4015a200 tx=0x7f0a2c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:40.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.119+0000 7f0a37e00640 1 -- 172.21.15.86:0/2079365033 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0a3001f020 con 0x7f0a40070ff0 2024-03-06T07:34:40.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.318+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f0a4006ef30 con 0x7f0a40070ff0 2024-03-06T07:34:40.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.319+0000 7f0a37e00640 1 -- 172.21.15.86:0/2079365033 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v123) v1 ==== 93+0+10 (secure 0 0 0) 0x7f0a400cba40 con 0x7f0a40070ff0 2024-03-06T07:34:40.321 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2006b4d0 msgr2=0x7f0a2006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2006b4d0 0x7f0a2006d990 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f0a4015a200 tx=0x7f0a2c008040 comp rx=0 tx=0).stop 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 msgr2=0x7f0a4014beb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a4014beb0 secure :-1 s=READY pgs=1481 cs=0 l=1 rev1=1 crypto rx=0x7f0a3000b410 tx=0x7f0a3000b8d0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 shutdown_connections 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0a2006b4d0 0x7f0a2006d990 unknown :-1 s=CLOSED pgs=633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0a4015fc10 0x7f0a4015a480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0a400b3f70 0x7f0a401598d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 --2- 172.21.15.86:0/2079365033 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0a40070ff0 0x7f0a4014beb0 unknown :-1 s=CLOSED pgs=1481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 >> 172.21.15.86:0/2079365033 conn(0x7f0a400b1940 msgr2=0x7f0a400acdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 shutdown_connections 2024-03-06T07:34:40.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.322+0000 7f0a47000640 1 -- 172.21.15.86:0/2079365033 wait complete. 2024-03-06T07:34:40.341 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-06T07:34:40.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.542+0000 7f5998400640 1 Processor -- start 2024-03-06T07:34:40.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.542+0000 7f5998400640 1 -- start start 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5998400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5990144c00 0x7f599006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5998400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f599015f9d0 0x7f599015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5998400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f59900ce0a0 0x7f59900ce4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5998400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5990160370 con 0x7f599015f9d0 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5998400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f599006ee70 con 0x7f59900ce0a0 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5998400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f599006cef0 con 0x7f5990144c00 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5995000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f599015f9d0 0x7f599015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5995000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f599015f9d0 0x7f599015fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54708/0 (socket says 172.21.15.86:54708) 2024-03-06T07:34:40.546 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.544+0000 7f5995000640 1 -- 172.21.15.86:0/3175376468 learned_addr learned my addr 172.21.15.86:0/3175376468 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5995a00640 1 --2- 172.21.15.86:0/3175376468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5990144c00 0x7f599006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5995000640 1 -- 172.21.15.86:0/3175376468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5990144c00 msgr2=0x7f599006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5995000640 1 --2- 172.21.15.86:0/3175376468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5990144c00 0x7f599006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5995000640 1 -- 172.21.15.86:0/3175376468 >> v2:172.21.15.194:3300/0 conn(0x7f59900ce0a0 msgr2=0x7f59900ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5996400640 1 --2- 172.21.15.86:0/3175376468 >> v2:172.21.15.194:3300/0 conn(0x7f59900ce0a0 0x7f59900ce4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5995000640 1 --2- 172.21.15.86:0/3175376468 >> v2:172.21.15.194:3300/0 conn(0x7f59900ce0a0 0x7f59900ce4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5995000640 1 -- 172.21.15.86:0/3175376468 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f599006d1d0 con 0x7f599015f9d0 2024-03-06T07:34:40.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.545+0000 7f5996400640 1 --2- 172.21.15.86:0/3175376468 >> v2:172.21.15.194:3300/0 conn(0x7f59900ce0a0 0x7f59900ce4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:40.548 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.546+0000 7f5995000640 1 --2- 172.21.15.86:0/3175376468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f599015f9d0 0x7f599015fe30 secure :-1 s=READY pgs=1482 cs=0 l=1 rev1=1 crypto rx=0x7f59900737b0 tx=0x7f597c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=254af238219d55bd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.546+0000 7f598fe00640 1 -- 172.21.15.86:0/3175376468 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f597c035020 con 0x7f599015f9d0 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.546+0000 7f598fe00640 1 -- 172.21.15.86:0/3175376468 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f597c031870 con 0x7f599015f9d0 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.546+0000 7f598fe00640 1 -- 172.21.15.86:0/3175376468 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f597c031c40 con 0x7f599015f9d0 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 -- 172.21.15.86:0/3175376468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f599015f9d0 msgr2=0x7f599015fe30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 --2- 172.21.15.86:0/3175376468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f599015f9d0 0x7f599015fe30 secure :-1 s=READY pgs=1482 cs=0 l=1 rev1=1 crypto rx=0x7f59900737b0 tx=0x7f597c02b910 comp rx=0 tx=0).stop 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 -- 172.21.15.86:0/3175376468 shutdown_connections 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 --2- 172.21.15.86:0/3175376468 >> v2:172.21.15.194:3300/0 conn(0x7f59900ce0a0 0x7f59900ce4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 --2- 172.21.15.86:0/3175376468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f599015f9d0 0x7f599015fe30 unknown :-1 s=CLOSED pgs=1482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 --2- 172.21.15.86:0/3175376468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5990144c00 0x7f599006e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 -- 172.21.15.86:0/3175376468 >> 172.21.15.86:0/3175376468 conn(0x7f59900b1730 msgr2=0x7f59900b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 -- 172.21.15.86:0/3175376468 shutdown_connections 2024-03-06T07:34:40.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.547+0000 7f5998400640 1 -- 172.21.15.86:0/3175376468 wait complete. 2024-03-06T07:34:40.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.549+0000 7f5998400640 1 Processor -- start 2024-03-06T07:34:40.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.549+0000 7f5998400640 1 -- start start 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5998400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f59900ce0a0 0x7f59901390b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5998400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990144c00 0x7f599015a900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5998400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f599015f9d0 0x7f5990155e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5998400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5990160730 con 0x7f5990144c00 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5998400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f599006d790 con 0x7f59900ce0a0 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5998400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f599006cb50 con 0x7f599015f9d0 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5995000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990144c00 0x7f599015a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5995000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990144c00 0x7f599015a900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54722/0 (socket says 172.21.15.86:54722) 2024-03-06T07:34:40.552 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.550+0000 7f5995000640 1 -- 172.21.15.86:0/4137880275 learned_addr learned my addr 172.21.15.86:0/4137880275 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:40.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995a00640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f59900ce0a0 0x7f59901390b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5996400640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f599015f9d0 0x7f5990155e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995000640 1 -- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f599015f9d0 msgr2=0x7f5990155e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995000640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f599015f9d0 0x7f5990155e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995000640 1 -- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f59900ce0a0 msgr2=0x7f59901390b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995000640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f59900ce0a0 0x7f59901390b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995000640 1 -- 172.21.15.86:0/4137880275 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f599013bde0 con 0x7f5990144c00 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995a00640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f59900ce0a0 0x7f59901390b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5995000640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990144c00 0x7f599015a900 secure :-1 s=READY pgs=1483 cs=0 l=1 rev1=1 crypto rx=0x7f597c000c00 tx=0x7f597c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f598d600640 1 -- 172.21.15.86:0/4137880275 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f597c035040 con 0x7f5990144c00 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f598d600640 1 -- 172.21.15.86:0/4137880275 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f597c046480 con 0x7f5990144c00 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f598d600640 1 -- 172.21.15.86:0/4137880275 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f597c046870 con 0x7f5990144c00 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.551+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f599013c0c0 con 0x7f5990144c00 2024-03-06T07:34:40.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.552+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f599015ae40 con 0x7f5990144c00 2024-03-06T07:34:40.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.553+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f599004f330 con 0x7f5990144c00 2024-03-06T07:34:40.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.554+0000 7f598d600640 1 -- 172.21.15.86:0/4137880275 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f597c038020 con 0x7f5990144c00 2024-03-06T07:34:40.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.554+0000 7f598d600640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f597006b500 0x7f597006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:40.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.555+0000 7f598d600640 1 -- 172.21.15.86:0/4137880275 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f597c0ac450 con 0x7f5990144c00 2024-03-06T07:34:40.556 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.555+0000 7f5995a00640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f597006b500 0x7f597006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:40.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.555+0000 7f5995a00640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f597006b500 0x7f597006d9c0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f5984008890 tx=0x7f5984008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:40.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.560+0000 7f598d600640 1 -- 172.21.15.86:0/4137880275 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f597c07b580 con 0x7f5990144c00 2024-03-06T07:34:40.764 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.763+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f599013d310 con 0x7f5990144c00 2024-03-06T07:34:40.765 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.764+0000 7f598d600640 1 -- 172.21.15.86:0/4137880275 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v123) v1 ==== 84+0+65 (secure 0 0 0) 0x7f597c07ae80 con 0x7f5990144c00 2024-03-06T07:34:40.765 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:40.765 INFO:teuthology.orchestra.run.smithi086.stdout:[ 2024-03-06T07:34:40.765 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-03-06T07:34:40.766 INFO:teuthology.orchestra.run.smithi086.stdout: "poolnum": 1, 2024-03-06T07:34:40.766 INFO:teuthology.orchestra.run.smithi086.stdout: "poolname": ".mgr" 2024-03-06T07:34:40.766 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-03-06T07:34:40.766 INFO:teuthology.orchestra.run.smithi086.stdout:] 2024-03-06T07:34:40.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.766+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f597006b500 msgr2=0x7f597006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.768 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.766+0000 7f5998400640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f597006b500 0x7f597006d9c0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f5984008890 tx=0x7f5984008040 comp rx=0 tx=0).stop 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.766+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990144c00 msgr2=0x7f599015a900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.766+0000 7f5998400640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990144c00 0x7f599015a900 secure :-1 s=READY pgs=1483 cs=0 l=1 rev1=1 crypto rx=0x7f597c000c00 tx=0x7f597c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 shutdown_connections 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f597006b500 0x7f597006d9c0 unknown :-1 s=CLOSED pgs=634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f599015f9d0 0x7f5990155e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5990144c00 0x7f599015a900 unknown :-1 s=CLOSED pgs=1483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 --2- 172.21.15.86:0/4137880275 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f59900ce0a0 0x7f59901390b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 >> 172.21.15.86:0/4137880275 conn(0x7f59900b1730 msgr2=0x7f599006dcb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 shutdown_connections 2024-03-06T07:34:40.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:40.767+0000 7f5998400640 1 -- 172.21.15.86:0/4137880275 wait complete. 2024-03-06T07:34:40.786 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-06T07:34:40.787 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-06T07:34:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.001+0000 7f665de00640 1 Processor -- start 2024-03-06T07:34:41.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.001+0000 7f665de00640 1 -- start start 2024-03-06T07:34:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.002+0000 7f665de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6658145980 0x7f665815fa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.002+0000 7f665de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 0x7f665806dec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.002+0000 7f665de00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66580cf1e0 con 0x7f66580ced20 2024-03-06T07:34:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.002+0000 7f665de00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6658070e40 con 0x7f6658145980 2024-03-06T07:34:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.002+0000 7f665de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6658160370 con 0x7f665815ff40 2024-03-06T07:34:41.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6658145980 0x7f665815fa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6658145980 0x7f665815fa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54732/0 (socket says 172.21.15.86:54732) 2024-03-06T07:34:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656a00640 1 -- 172.21.15.86:0/579255172 learned_addr learned my addr 172.21.15.86:0/579255172 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f664e000640 1 --1- 172.21.15.86:0/579255172 >> v1:172.21.15.86:6789/0 conn(0x7f66580ced20 0x7f66580712d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59046/0 (socket says 172.21.15.86:59046) 2024-03-06T07:34:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6657400640 1 --2- 172.21.15.86:0/579255172 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 0x7f665806dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656a00640 1 -- 172.21.15.86:0/579255172 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 msgr2=0x7f665806dec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656a00640 1 --2- 172.21.15.86:0/579255172 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 0x7f665806dec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656a00640 1 -- 172.21.15.86:0/579255172 >> v1:172.21.15.86:6789/0 conn(0x7f66580ced20 legacy=0x7f66580712d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656a00640 1 -- 172.21.15.86:0/579255172 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66580cf4c0 con 0x7f6658145980 2024-03-06T07:34:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.003+0000 7f6656000640 1 -- 172.21.15.86:0/579255172 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 107586399 0 0) 0x7f66580cf1e0 con 0x7f66580ced20 2024-03-06T07:34:41.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.005+0000 7f6656a00640 1 --2- 172.21.15.86:0/579255172 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f665815fa00 secure :-1 s=READY pgs=1485 cs=0 l=1 rev1=1 crypto rx=0x7f6648009870 tx=0x7f664802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0f31a454cba8f69 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:41.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.005+0000 7f6656000640 1 -- 172.21.15.86:0/579255172 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6648034070 con 0x7f6658145980 2024-03-06T07:34:41.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.005+0000 7f6656000640 1 -- 172.21.15.86:0/579255172 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6648030840 con 0x7f6658145980 2024-03-06T07:34:41.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.005+0000 7f6656000640 1 -- 172.21.15.86:0/579255172 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6648030c10 con 0x7f6658145980 2024-03-06T07:34:41.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.005+0000 7f665de00640 1 -- 172.21.15.86:0/579255172 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 msgr2=0x7f665815fa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:41.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.005+0000 7f665de00640 1 --2- 172.21.15.86:0/579255172 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f665815fa00 secure :-1 s=READY pgs=1485 cs=0 l=1 rev1=1 crypto rx=0x7f6648009870 tx=0x7f664802b980 comp rx=0 tx=0).stop 2024-03-06T07:34:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.006+0000 7f665de00640 1 -- 172.21.15.86:0/579255172 shutdown_connections 2024-03-06T07:34:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.006+0000 7f665de00640 1 --2- 172.21.15.86:0/579255172 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 0x7f665806dec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.006+0000 7f665de00640 1 --2- 172.21.15.86:0/579255172 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f665815fa00 unknown :-1 s=CLOSED pgs=1485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.006+0000 7f665de00640 1 -- 172.21.15.86:0/579255172 >> 172.21.15.86:0/579255172 conn(0x7f66580cfa80 msgr2=0x7f66580cfe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.006+0000 7f665de00640 1 -- 172.21.15.86:0/579255172 shutdown_connections 2024-03-06T07:34:41.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.006+0000 7f665de00640 1 -- 172.21.15.86:0/579255172 wait complete. 2024-03-06T07:34:41.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.007+0000 7f665de00640 1 Processor -- start 2024-03-06T07:34:41.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.008+0000 7f665de00640 1 -- start start 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.008+0000 7f665de00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66580ced20 0x7f66581e6160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.008+0000 7f665de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f6658143c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.008+0000 7f665de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 0x7f66581f1150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.008+0000 7f665de00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6658160740 con 0x7f6658145980 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.008+0000 7f665de00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6658070ad0 con 0x7f66580ced20 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.008+0000 7f665de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66581608c0 con 0x7f665815ff40 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f6658143c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f6658143c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54740/0 (socket says 172.21.15.86:54740) 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 -- 172.21.15.86:0/3447534662 learned_addr learned my addr 172.21.15.86:0/3447534662 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 -- 172.21.15.86:0/3447534662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 msgr2=0x7f66581f1150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 0x7f66581f1150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.011 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 -- 172.21.15.86:0/3447534662 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66580ced20 msgr2=0x7f66581e6160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:41.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66580ced20 0x7f66581e6160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 -- 172.21.15.86:0/3447534662 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66581e6a70 con 0x7f6658145980 2024-03-06T07:34:41.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.009+0000 7f664e000640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f6658143c40 secure :-1 s=READY pgs=1486 cs=0 l=1 rev1=1 crypto rx=0x7f664000c430 tx=0x7f664000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:41.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.010+0000 7f664fe00640 1 -- 172.21.15.86:0/3447534662 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66400074e0 con 0x7f6658145980 2024-03-06T07:34:41.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.010+0000 7f664fe00640 1 -- 172.21.15.86:0/3447534662 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66400047e0 con 0x7f6658145980 2024-03-06T07:34:41.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.010+0000 7f664fe00640 1 -- 172.21.15.86:0/3447534662 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6640004bd0 con 0x7f6658145980 2024-03-06T07:34:41.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.010+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66581e6d50 con 0x7f6658145980 2024-03-06T07:34:41.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.010+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66581e66a0 con 0x7f6658145980 2024-03-06T07:34:41.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.011+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f665804f330 con 0x7f6658145980 2024-03-06T07:34:41.013 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.012+0000 7f664fe00640 1 -- 172.21.15.86:0/3447534662 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6640011070 con 0x7f6658145980 2024-03-06T07:34:41.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.013+0000 7f664fe00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f663406b4d0 0x7f663406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:41.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.013+0000 7f664fe00640 1 -- 172.21.15.86:0/3447534662 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f6640089b80 con 0x7f6658145980 2024-03-06T07:34:41.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.013+0000 7f6656a00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f663406b4d0 0x7f663406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:41.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.014+0000 7f6656a00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f663406b4d0 0x7f663406d990 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f6648036000 tx=0x7f6648030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:41.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.017+0000 7f664fe00640 1 -- 172.21.15.86:0/3447534662 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6640058d40 con 0x7f6658145980 2024-03-06T07:34:41.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.210+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f6658072920 con 0x7f6658145980 2024-03-06T07:34:41.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.969+0000 7f664fe00640 1 -- 172.21.15.86:0/3447534662 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v124) v1 ==== 138+0+0 (secure 0 0 0) 0x7f6640058640 con 0x7f6658145980 2024-03-06T07:34:41.971 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_metadata' created 2024-03-06T07:34:41.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.973+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f663406b4d0 msgr2=0x7f663406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:41.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.973+0000 7f665de00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f663406b4d0 0x7f663406d990 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f6648036000 tx=0x7f6648030000 comp rx=0 tx=0).stop 2024-03-06T07:34:41.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.973+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 msgr2=0x7f6658143c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:41.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.973+0000 7f665de00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f6658143c40 secure :-1 s=READY pgs=1486 cs=0 l=1 rev1=1 crypto rx=0x7f664000c430 tx=0x7f664000c8f0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 shutdown_connections 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f663406b4d0 0x7f663406d990 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f665815ff40 0x7f66581f1150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6658145980 0x7f6658143c40 unknown :-1 s=CLOSED pgs=1486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 --2- 172.21.15.86:0/3447534662 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66580ced20 0x7f66581e6160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 >> 172.21.15.86:0/3447534662 conn(0x7f66580cfa80 msgr2=0x7f665806d3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 shutdown_connections 2024-03-06T07:34:41.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:41.974+0000 7f665de00640 1 -- 172.21.15.86:0/3447534662 wait complete. 2024-03-06T07:34:42.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-06T07:34:42.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.196+0000 7f6e50c00640 1 Processor -- start 2024-03-06T07:34:42.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.196+0000 7f6e50c00640 1 -- start start 2024-03-06T07:34:42.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.198+0000 7f6e50c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6e4c0b3f70 0x7f6e4c160700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:42.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.198+0000 7f6e50c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c070ff0 0x7f6e4c0ce4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:42.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.198+0000 7f6e50c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6e4c0b46c0 con 0x7f6e4c15fc10 2024-03-06T07:34:42.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.198+0000 7f6e50c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6e4c071480 con 0x7f6e4c0b3f70 2024-03-06T07:34:42.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.198+0000 7f6e50c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6e4c0b49a0 con 0x7f6e4c070ff0 2024-03-06T07:34:42.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.198+0000 7f6e49400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6e4c0b3f70 0x7f6e4c160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:42.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.198+0000 7f6e4a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c070ff0 0x7f6e4c0ce4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:42.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.199+0000 7f6e4a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c070ff0 0x7f6e4c0ce4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:55706/0 (socket says 172.21.15.86:55706) 2024-03-06T07:34:42.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.199+0000 7f6e4a800640 1 -- 172.21.15.86:0/2036218353 learned_addr learned my addr 172.21.15.86:0/2036218353 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:42.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.199+0000 7f6e49400640 1 -- 172.21.15.86:0/2036218353 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c070ff0 msgr2=0x7f6e4c0ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:42.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.199+0000 7f6e49400640 1 --2- 172.21.15.86:0/2036218353 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c070ff0 0x7f6e4c0ce4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.199+0000 7f6e49400640 1 -- 172.21.15.86:0/2036218353 >> v1:172.21.15.194:6789/0 conn(0x7f6e4c15fc10 legacy=0x7f6e4c15fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:42.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.199+0000 7f6e49400640 1 -- 172.21.15.86:0/2036218353 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e4c072c90 con 0x7f6e4c0b3f70 2024-03-06T07:34:42.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.201+0000 7f6e49400640 1 --2- 172.21.15.86:0/2036218353 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c0b3f70 0x7f6e4c160700 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7f6e380097a0 tx=0x7f6e3802b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6a7e9d1192fb110a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:42.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.201+0000 7f6e48a00640 1 -- 172.21.15.86:0/2036218353 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e38035020 con 0x7f6e4c0b3f70 2024-03-06T07:34:42.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.201+0000 7f6e48a00640 1 -- 172.21.15.86:0/2036218353 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6e38031830 con 0x7f6e4c0b3f70 2024-03-06T07:34:42.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.201+0000 7f6e48a00640 1 -- 172.21.15.86:0/2036218353 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e38031c00 con 0x7f6e4c0b3f70 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 -- 172.21.15.86:0/2036218353 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c0b3f70 msgr2=0x7f6e4c160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2036218353 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c0b3f70 0x7f6e4c160700 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7f6e380097a0 tx=0x7f6e3802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 -- 172.21.15.86:0/2036218353 shutdown_connections 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2036218353 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c070ff0 0x7f6e4c0ce4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2036218353 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c0b3f70 0x7f6e4c160700 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 -- 172.21.15.86:0/2036218353 >> 172.21.15.86:0/2036218353 conn(0x7f6e4c0b1940 msgr2=0x7f6e4c0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 -- 172.21.15.86:0/2036218353 shutdown_connections 2024-03-06T07:34:42.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.202+0000 7f6e50c00640 1 -- 172.21.15.86:0/2036218353 wait complete. 2024-03-06T07:34:42.205 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.204+0000 7f6e50c00640 1 Processor -- start 2024-03-06T07:34:42.205 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.204+0000 7f6e50c00640 1 -- start start 2024-03-06T07:34:42.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e50c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c070ff0 0x7f6e4c0b0270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:42.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e50c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c0b3f70 0x7f6e4c14baf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:42.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e50c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6e4c15fc10 0x7f6e4c15a950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:42.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e50c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e4c072a30 con 0x7f6e4c15fc10 2024-03-06T07:34:42.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e50c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e4c0728b0 con 0x7f6e4c070ff0 2024-03-06T07:34:42.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e50c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6e4c15c2e0 con 0x7f6e4c0b3f70 2024-03-06T07:34:42.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6e4c15fc10 0x7f6e4c15a950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:42.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6e4c15fc10 0x7f6e4c15a950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54752/0 (socket says 172.21.15.86:54752) 2024-03-06T07:34:42.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 -- 172.21.15.86:0/2982355818 learned_addr learned my addr 172.21.15.86:0/2982355818 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:42.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e49400640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c0b3f70 0x7f6e4c14baf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:42.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e49e00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c070ff0 0x7f6e4c0b0270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 -- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c0b3f70 msgr2=0x7f6e4c14baf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c0b3f70 0x7f6e4c14baf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 -- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c070ff0 msgr2=0x7f6e4c0b0270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c070ff0 0x7f6e4c0b0270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.205+0000 7f6e4a800640 1 -- 172.21.15.86:0/2982355818 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e4c159900 con 0x7f6e4c15fc10 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.206+0000 7f6e4a800640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6e4c15fc10 0x7f6e4c15a950 secure :-1 s=READY pgs=1487 cs=0 l=1 rev1=1 crypto rx=0x7f6e4400efc0 tx=0x7f6e4400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.206+0000 7f6e42000640 1 -- 172.21.15.86:0/2982355818 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e4400cd20 con 0x7f6e4c15fc10 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.206+0000 7f6e42000640 1 -- 172.21.15.86:0/2982355818 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6e44003990 con 0x7f6e4c15fc10 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.206+0000 7f6e42000640 1 -- 172.21.15.86:0/2982355818 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e44003d80 con 0x7f6e4c15fc10 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.206+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6e4c159e90 con 0x7f6e4c15fc10 2024-03-06T07:34:42.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.206+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6e4c15a450 con 0x7f6e4c15fc10 2024-03-06T07:34:42.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.207+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6e4c04f330 con 0x7f6e4c15fc10 2024-03-06T07:34:42.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.209+0000 7f6e42000640 1 -- 172.21.15.86:0/2982355818 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6e44005d10 con 0x7f6e4c15fc10 2024-03-06T07:34:42.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.209+0000 7f6e42000640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6e2c06b4d0 0x7f6e2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:42.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.210+0000 7f6e42000640 1 -- 172.21.15.86:0/2982355818 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(124..124 src has 1..124) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f6e4408b360 con 0x7f6e4c15fc10 2024-03-06T07:34:42.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.210+0000 7f6e49e00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6e2c06b4d0 0x7f6e2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:42.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.210+0000 7f6e49e00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6e2c06b4d0 0x7f6e2c06d990 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f6e4c14c870 tx=0x7f6e3c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:42.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.213+0000 7f6e42000640 1 -- 172.21.15.86:0/2982355818 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6e44022020 con 0x7f6e4c15fc10 2024-03-06T07:34:42.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.411+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f6e4c159690 con 0x7f6e4c15fc10 2024-03-06T07:34:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.986+0000 7f6e42000640 1 -- 172.21.15.86:0/2982355818 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v125) v1 ==== 170+0+0 (secure 0 0 0) 0x7f6e44010040 con 0x7f6e4c15fc10 2024-03-06T07:34:42.988 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'cephfs_data' created 2024-03-06T07:34:42.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.991+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6e2c06b4d0 msgr2=0x7f6e2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:42.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.991+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6e2c06b4d0 0x7f6e2c06d990 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f6e4c14c870 tx=0x7f6e3c009040 comp rx=0 tx=0).stop 2024-03-06T07:34:42.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.991+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6e4c15fc10 msgr2=0x7f6e4c15a950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:42.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.991+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6e4c15fc10 0x7f6e4c15a950 secure :-1 s=READY pgs=1487 cs=0 l=1 rev1=1 crypto rx=0x7f6e4400efc0 tx=0x7f6e4400c490 comp rx=0 tx=0).stop 2024-03-06T07:34:42.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.992+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 shutdown_connections 2024-03-06T07:34:42.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.992+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6e2c06b4d0 0x7f6e2c06d990 unknown :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.992+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6e4c15fc10 0x7f6e4c15a950 unknown :-1 s=CLOSED pgs=1487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.992+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6e4c0b3f70 0x7f6e4c14baf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.992+0000 7f6e50c00640 1 --2- 172.21.15.86:0/2982355818 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6e4c070ff0 0x7f6e4c0b0270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:42.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.992+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 >> 172.21.15.86:0/2982355818 conn(0x7f6e4c0b1940 msgr2=0x7f6e4c0cc690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:42.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.992+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 shutdown_connections 2024-03-06T07:34:42.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:42.993+0000 7f6e50c00640 1 -- 172.21.15.86:0/2982355818 wait complete. 2024-03-06T07:34:43.030 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-06T07:34:43.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.205+0000 7f39c1400640 1 Processor -- start 2024-03-06T07:34:43.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.205+0000 7f39c1400640 1 -- start start 2024-03-06T07:34:43.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.207+0000 7f39c1400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f39bc15cb90 0x7f39bc06d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.207+0000 7f39c1400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39bc15cf70 con 0x7f39bc06e060 2024-03-06T07:34:43.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.207+0000 7f39c1400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39bc0b35b0 con 0x7f39bc163800 2024-03-06T07:34:43.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.207+0000 7f39c1400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f39bc0b3890 con 0x7f39bc15cb90 2024-03-06T07:34:43.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.207+0000 7f39bb200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f39bc06e060 0x7f39bc0cda90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59062/0 (socket says 172.21.15.86:59062) 2024-03-06T07:34:43.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.207+0000 7f39bb200640 1 -- 172.21.15.86:0/3656394049 learned_addr learned my addr 172.21.15.86:0/3656394049 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39ba800640 1 --2- 172.21.15.86:0/3656394049 >> v2:172.21.15.194:3300/0 conn(0x7f39bc15cb90 0x7f39bc06d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1139584788 0 0) 0x7f39bc15cf70 con 0x7f39bc06e060 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f39a0003520 con 0x7f39bc06e060 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3459400222 0 0) 0x7f39a0003520 con 0x7f39bc06e060 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 >> v2:172.21.15.194:3300/0 conn(0x7f39bc15cb90 msgr2=0x7f39bc06d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 --2- 172.21.15.86:0/3656394049 >> v2:172.21.15.194:3300/0 conn(0x7f39bc15cb90 0x7f39bc06d6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 >> v1:172.21.15.194:6789/0 conn(0x7f39bc163800 legacy=0x7f39bc0b2c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39bc06cbe0 con 0x7f39bc06e060 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f39b40019f0 con 0x7f39bc06e060 2024-03-06T07:34:43.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 >> v1:172.21.15.86:6789/0 conn(0x7f39bc06e060 legacy=0x7f39bc0cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a00039b0 0x7f39a0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39a0004800 0x7f39a0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a00051c0 0x7f39a000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39bc0b3890 con 0x7f39a0004800 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39a000c060 con 0x7f39a00051c0 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.208+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39a000c260 con 0x7f39a00039b0 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39b1e00640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39a0004800 0x7f39a0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39bb200640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a00051c0 0x7f39a000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39ba800640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a00039b0 0x7f39a0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39b1e00640 1 -- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a00039b0 msgr2=0x7f39a0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39b1e00640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a00039b0 0x7f39a0003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39b1e00640 1 -- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a00051c0 msgr2=0x7f39a000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:43.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39b1e00640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a00051c0 0x7f39a000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:43.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39b1e00640 1 -- 172.21.15.86:0/3656394049 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39a0003520 con 0x7f39a0004800 2024-03-06T07:34:43.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39bb200640 1 -- 172.21.15.86:0/3656394049 reap_dead start 2024-03-06T07:34:43.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.209+0000 7f39ba800640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39a00039b0 0x7f39a0003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.210+0000 7f39b1e00640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39a0004800 0x7f39a0004c80 secure :-1 s=READY pgs=1489 cs=0 l=1 rev1=1 crypto rx=0x7f39a4001750 tx=0x7f39a0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=82115e26fc357d3c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39a4001370 con 0x7f39a0004800 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f39a40052d0 con 0x7f39a0004800 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39b9e00640 1 -- 172.21.15.86:0/3656394049 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39a4003830 con 0x7f39a0004800 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39c1400640 1 -- 172.21.15.86:0/3656394049 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39a0004800 msgr2=0x7f39a0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39c1400640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39a0004800 0x7f39a0004c80 secure :-1 s=READY pgs=1489 cs=0 l=1 rev1=1 crypto rx=0x7f39a4001750 tx=0x7f39a0005830 comp rx=0 tx=0).stop 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39c1400640 1 -- 172.21.15.86:0/3656394049 shutdown_connections 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39c1400640 1 --2- 172.21.15.86:0/3656394049 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39a0004800 0x7f39a0004c80 unknown :-1 s=CLOSED pgs=1489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39c1400640 1 -- 172.21.15.86:0/3656394049 >> 172.21.15.86:0/3656394049 conn(0x7f39bc0b1940 msgr2=0x7f39bc0b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39c1400640 1 -- 172.21.15.86:0/3656394049 shutdown_connections 2024-03-06T07:34:43.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.211+0000 7f39c1400640 1 -- 172.21.15.86:0/3656394049 wait complete. 2024-03-06T07:34:43.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.213+0000 7f39c1400640 1 Processor -- start 2024-03-06T07:34:43.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.213+0000 7f39c1400640 1 -- start start 2024-03-06T07:34:43.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39c1400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a0004800 0x7f39bc1ed8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39c1400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39bc15bca0 0x7f39bc15b030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39c1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39bc1ebf60 0x7f39bc0add70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39c1400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39bc06d230 con 0x7f39bc1ebf60 2024-03-06T07:34:43.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39c1400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39bc06c990 con 0x7f39a0004800 2024-03-06T07:34:43.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39c1400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39bc06e4d0 con 0x7f39bc15bca0 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39bb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39bc1ebf60 0x7f39bc0add70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39bb200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39bc1ebf60 0x7f39bc0add70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54758/0 (socket says 172.21.15.86:54758) 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.214+0000 7f39bb200640 1 -- 172.21.15.86:0/2576617774 learned_addr learned my addr 172.21.15.86:0/2576617774 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39b1e00640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39bc15bca0 0x7f39bc15b030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39ba800640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a0004800 0x7f39bc1ed8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39bb200640 1 -- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39bc15bca0 msgr2=0x7f39bc15b030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39bb200640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39bc15bca0 0x7f39bc15b030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39bb200640 1 -- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a0004800 msgr2=0x7f39bc1ed8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39bb200640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a0004800 0x7f39bc1ed8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39bb200640 1 -- 172.21.15.86:0/2576617774 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39bc1ec840 con 0x7f39bc1ebf60 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39b1e00640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39bc15bca0 0x7f39bc15b030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39bb200640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39bc1ebf60 0x7f39bc0add70 secure :-1 s=READY pgs=1490 cs=0 l=1 rev1=1 crypto rx=0x7f39bc06ec20 tx=0x7f39b4007bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:43.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39b3400640 1 -- 172.21.15.86:0/2576617774 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39b400ead0 con 0x7f39bc1ebf60 2024-03-06T07:34:43.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39b3400640 1 -- 172.21.15.86:0/2576617774 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f39b400ec70 con 0x7f39bc1ebf60 2024-03-06T07:34:43.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39b3400640 1 -- 172.21.15.86:0/2576617774 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39b4014430 con 0x7f39bc1ebf60 2024-03-06T07:34:43.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.215+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39bc1ecb20 con 0x7f39bc1ebf60 2024-03-06T07:34:43.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.216+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39bc1ed0e0 con 0x7f39bc1ebf60 2024-03-06T07:34:43.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.217+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f39bc09dec0 con 0x7f39bc1ebf60 2024-03-06T07:34:43.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.218+0000 7f39b3400640 1 -- 172.21.15.86:0/2576617774 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f39b40145d0 con 0x7f39bc1ebf60 2024-03-06T07:34:43.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.218+0000 7f39b3400640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f399806b4d0 0x7f399806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:43.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.219+0000 7f39b3400640 1 -- 172.21.15.86:0/2576617774 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f39b4089f50 con 0x7f39bc1ebf60 2024-03-06T07:34:43.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.219+0000 7f39ba800640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f399806b4d0 0x7f399806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:43.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.219+0000 7f39ba800640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f399806b4d0 0x7f399806d990 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7f39bc0686c0 tx=0x7f39ac008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:43.225 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.224+0000 7f39b3400640 1 -- 172.21.15.86:0/2576617774 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f39bc09dec0 con 0x7f39bc1ebf60 2024-03-06T07:34:43.454 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:43.453+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f39bc04f330 con 0x7f39bc1ebf60 2024-03-06T07:34:44.016 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.014+0000 7f39b3400640 1 -- 172.21.15.86:0/2576617774 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 18 and data pool 19 v89) v1 ==== 174+0+0 (secure 0 0 0) 0x7f39b4058d90 con 0x7f39bc1ebf60 2024-03-06T07:34:44.018 INFO:teuthology.orchestra.run.smithi086.stderr:new fs with metadata pool 18 and data pool 19 2024-03-06T07:34:44.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.023+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f399806b4d0 msgr2=0x7f399806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.023+0000 7f39c1400640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f399806b4d0 0x7f399806d990 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7f39bc0686c0 tx=0x7f39ac008040 comp rx=0 tx=0).stop 2024-03-06T07:34:44.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.023+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39bc1ebf60 msgr2=0x7f39bc0add70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.023+0000 7f39c1400640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39bc1ebf60 0x7f39bc0add70 secure :-1 s=READY pgs=1490 cs=0 l=1 rev1=1 crypto rx=0x7f39bc06ec20 tx=0x7f39b4007bc0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.028+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 shutdown_connections 2024-03-06T07:34:44.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.028+0000 7f39c1400640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f399806b4d0 0x7f399806d990 unknown :-1 s=CLOSED pgs=637 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.028+0000 7f39c1400640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39bc1ebf60 0x7f39bc0add70 unknown :-1 s=CLOSED pgs=1490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.028+0000 7f39c1400640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39bc15bca0 0x7f39bc15b030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.028+0000 7f39c1400640 1 --2- 172.21.15.86:0/2576617774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39a0004800 0x7f39bc1ed8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.028+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 >> 172.21.15.86:0/2576617774 conn(0x7f39bc0b1940 msgr2=0x7f39bc0cef50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:44.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.030+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 shutdown_connections 2024-03-06T07:34:44.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.030+0000 7f39c1400640 1 -- 172.21.15.86:0/2576617774 wait complete. 2024-03-06T07:34:44.059 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:44.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.228+0000 7f4ff2200640 1 Processor -- start 2024-03-06T07:34:44.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.228+0000 7f4ff2200640 1 -- start start 2024-03-06T07:34:44.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.229+0000 7f4ff2200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4fec144c00 0x7f4fec06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.229+0000 7f4ff2200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 0x7f4fec15fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.229+0000 7f4ff2200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4fec0ce0a0 0x7f4fec0ce4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.229+0000 7f4ff2200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4fec160370 con 0x7f4fec144c00 2024-03-06T07:34:44.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4ff2200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4fec06ee70 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4ff2200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4fec06cef0 con 0x7f4fec15f9d0 2024-03-06T07:34:44.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4fec144c00 0x7f4fec06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4fec144c00 0x7f4fec06e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54772/0 (socket says 172.21.15.86:54772) 2024-03-06T07:34:44.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 -- 172.21.15.86:0/2806997164 learned_addr learned my addr 172.21.15.86:0/2806997164 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:44.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 -- 172.21.15.86:0/2806997164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 msgr2=0x7f4fec15fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:44.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 --2- 172.21.15.86:0/2806997164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 0x7f4fec15fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 -- 172.21.15.86:0/2806997164 >> v2:172.21.15.194:3300/0 conn(0x7f4fec0ce0a0 msgr2=0x7f4fec0ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:44.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 --2- 172.21.15.86:0/2806997164 >> v2:172.21.15.194:3300/0 conn(0x7f4fec0ce0a0 0x7f4fec0ce4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.230+0000 7f4feb400640 1 -- 172.21.15.86:0/2806997164 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4fec06d1d0 con 0x7f4fec144c00 2024-03-06T07:34:44.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.231+0000 7f4feb400640 1 --2- 172.21.15.86:0/2806997164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec144c00 0x7f4fec06e930 secure :-1 s=READY pgs=1491 cs=0 l=1 rev1=1 crypto rx=0x7f4fec0737b0 tx=0x7f4fdc02b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aac887517b388e6e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:44.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4feaa00640 1 -- 172.21.15.86:0/2806997164 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc035020 con 0x7f4fec144c00 2024-03-06T07:34:44.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4feaa00640 1 -- 172.21.15.86:0/2806997164 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4fdc02d040 con 0x7f4fec144c00 2024-03-06T07:34:44.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4feaa00640 1 -- 172.21.15.86:0/2806997164 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc031a80 con 0x7f4fec144c00 2024-03-06T07:34:44.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4ff2200640 1 -- 172.21.15.86:0/2806997164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec144c00 msgr2=0x7f4fec06e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4ff2200640 1 --2- 172.21.15.86:0/2806997164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec144c00 0x7f4fec06e930 secure :-1 s=READY pgs=1491 cs=0 l=1 rev1=1 crypto rx=0x7f4fec0737b0 tx=0x7f4fdc02b900 comp rx=0 tx=0).stop 2024-03-06T07:34:44.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4ff2200640 1 -- 172.21.15.86:0/2806997164 shutdown_connections 2024-03-06T07:34:44.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4ff2200640 1 --2- 172.21.15.86:0/2806997164 >> v2:172.21.15.194:3300/0 conn(0x7f4fec0ce0a0 0x7f4fec0ce4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4ff2200640 1 --2- 172.21.15.86:0/2806997164 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 0x7f4fec15fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4ff2200640 1 --2- 172.21.15.86:0/2806997164 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec144c00 0x7f4fec06e930 unknown :-1 s=CLOSED pgs=1491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.232+0000 7f4ff2200640 1 -- 172.21.15.86:0/2806997164 >> 172.21.15.86:0/2806997164 conn(0x7f4fec0b1730 msgr2=0x7f4fec0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:44.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.233+0000 7f4ff2200640 1 -- 172.21.15.86:0/2806997164 shutdown_connections 2024-03-06T07:34:44.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.233+0000 7f4ff2200640 1 -- 172.21.15.86:0/2806997164 wait complete. 2024-03-06T07:34:44.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.234+0000 7f4ff2200640 1 Processor -- start 2024-03-06T07:34:44.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.234+0000 7f4ff2200640 1 -- start start 2024-03-06T07:34:44.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.235+0000 7f4ff2200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec0ce0a0 0x7f4fec0711c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.235+0000 7f4ff2200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fec144c00 0x7f4fec15b240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.235+0000 7f4ff2200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 0x7f4fec15b780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.235+0000 7f4ff2200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4fec160730 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.235+0000 7f4ff2200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4fec06d790 con 0x7f4fec144c00 2024-03-06T07:34:44.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.235+0000 7f4ff2200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4fec06cb50 con 0x7f4fec15f9d0 2024-03-06T07:34:44.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec0ce0a0 0x7f4fec0711c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec0ce0a0 0x7f4fec0711c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54774/0 (socket says 172.21.15.86:54774) 2024-03-06T07:34:44.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 -- 172.21.15.86:0/366542596 learned_addr learned my addr 172.21.15.86:0/366542596 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 -- 172.21.15.86:0/366542596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 msgr2=0x7f4fec15b780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 0x7f4fec15b780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 -- 172.21.15.86:0/366542596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fec144c00 msgr2=0x7f4fec15b240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fec144c00 0x7f4fec15b240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.236+0000 7f4feb400640 1 -- 172.21.15.86:0/366542596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4fec13f5f0 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.237+0000 7f4feb400640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec0ce0a0 0x7f4fec0711c0 secure :-1 s=READY pgs=1492 cs=0 l=1 rev1=1 crypto rx=0x7f4fdc000c00 tx=0x7f4fdc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.237+0000 7f4fe3400640 1 -- 172.21.15.86:0/366542596 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc03ac60 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.237+0000 7f4fe3400640 1 -- 172.21.15.86:0/366542596 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4fdc03ae00 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.237+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4fec13b220 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.237+0000 7f4fe3400640 1 -- 172.21.15.86:0/366542596 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fdc044630 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.237+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4fec13b7e0 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.238+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4fec04f330 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.240 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.239+0000 7f4fe3400640 1 -- 172.21.15.86:0/366542596 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4fdc04b060 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.240+0000 7f4fe3400640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc806b4d0 0x7f4fc806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.240+0000 7f4fe3400640 1 -- 172.21.15.86:0/366542596 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(126..126 src has 1..126) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4fdc0ac260 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.240+0000 7f4fe3e00640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc806b4d0 0x7f4fc806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.242 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.241+0000 7f4fe3e00640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc806b4d0 0x7f4fc806d990 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f4fec1f7a30 tx=0x7f4fd4006c90 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:44.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.244+0000 7f4fe3400640 1 -- 172.21.15.86:0/366542596 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4fdc03f070 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.432+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4fec0afbd0 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.434+0000 7f4fe3400640 1 -- 172.21.15.86:0/366542596 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v126) v1 ==== 74+0+16704 (secure 0 0 0) 0x7f4fdc07b0e0 con 0x7f4fec0ce0a0 2024-03-06T07:34:44.437 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:44.437 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":126,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:34:43.454697+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:34:41.263023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"126","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-06T07:34:42.462100+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"126","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:44.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc806b4d0 msgr2=0x7f4fc806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc806b4d0 0x7f4fc806d990 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f4fec1f7a30 tx=0x7f4fd4006c90 comp rx=0 tx=0).stop 2024-03-06T07:34:44.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec0ce0a0 msgr2=0x7f4fec0711c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec0ce0a0 0x7f4fec0711c0 secure :-1 s=READY pgs=1492 cs=0 l=1 rev1=1 crypto rx=0x7f4fdc000c00 tx=0x7f4fdc0046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 shutdown_connections 2024-03-06T07:34:44.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4fc806b4d0 0x7f4fc806d990 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fec15f9d0 0x7f4fec15b780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fec144c00 0x7f4fec15b240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 --2- 172.21.15.86:0/366542596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fec0ce0a0 0x7f4fec0711c0 unknown :-1 s=CLOSED pgs=1492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.436+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 >> 172.21.15.86:0/366542596 conn(0x7f4fec0b1730 msgr2=0x7f4fec1431e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:44.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.437+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 shutdown_connections 2024-03-06T07:34:44.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.437+0000 7f4ff2200640 1 -- 172.21.15.86:0/366542596 wait complete. 2024-03-06T07:34:44.456 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:44.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.659+0000 7f1aa7200640 1 Processor -- start 2024-03-06T07:34:44.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.659+0000 7f1aa7200640 1 -- start start 2024-03-06T07:34:44.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.660+0000 7f1aa7200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1aa015dfa0 0x7f1aa00cc980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.660+0000 7f1aa7200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa015f0f0 0x7f1aa00ce2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.660+0000 7f1aa7200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1aa015f540 con 0x7f1aa00ce7e0 2024-03-06T07:34:44.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.660+0000 7f1aa7200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1aa00cd060 con 0x7f1aa015dfa0 2024-03-06T07:34:44.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa7200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1aa00cd340 con 0x7f1aa015f0f0 2024-03-06T07:34:44.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1aa015dfa0 0x7f1aa00cc980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1aa015dfa0 0x7f1aa00cc980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54782/0 (socket says 172.21.15.86:54782) 2024-03-06T07:34:44.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5e00640 1 -- 172.21.15.86:0/234423155 learned_addr learned my addr 172.21.15.86:0/234423155 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:44.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5400640 1 --2- 172.21.15.86:0/234423155 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa015f0f0 0x7f1aa00ce2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5e00640 1 -- 172.21.15.86:0/234423155 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa015f0f0 msgr2=0x7f1aa00ce2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:44.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5e00640 1 --2- 172.21.15.86:0/234423155 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa015f0f0 0x7f1aa00ce2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5e00640 1 -- 172.21.15.86:0/234423155 >> v1:172.21.15.194:6789/0 conn(0x7f1aa00ce7e0 legacy=0x7f1aa0161940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:44.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.661+0000 7f1aa5e00640 1 -- 172.21.15.86:0/234423155 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1aa00cd620 con 0x7f1aa015dfa0 2024-03-06T07:34:44.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.663+0000 7f1aa5e00640 1 --2- 172.21.15.86:0/234423155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00cc980 secure :-1 s=READY pgs=1493 cs=0 l=1 rev1=1 crypto rx=0x7f1a940098a0 tx=0x7f1a9402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cfcba693471f06c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:44.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.663+0000 7f1aa4a00640 1 -- 172.21.15.86:0/234423155 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a94034070 con 0x7f1aa015dfa0 2024-03-06T07:34:44.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa4a00640 1 -- 172.21.15.86:0/234423155 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a94030840 con 0x7f1aa015dfa0 2024-03-06T07:34:44.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa4a00640 1 -- 172.21.15.86:0/234423155 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a94030c10 con 0x7f1aa015dfa0 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 -- 172.21.15.86:0/234423155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 msgr2=0x7f1aa00cc980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 --2- 172.21.15.86:0/234423155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00cc980 secure :-1 s=READY pgs=1493 cs=0 l=1 rev1=1 crypto rx=0x7f1a940098a0 tx=0x7f1a9402b980 comp rx=0 tx=0).stop 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 -- 172.21.15.86:0/234423155 shutdown_connections 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 --2- 172.21.15.86:0/234423155 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa015f0f0 0x7f1aa00ce2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 --2- 172.21.15.86:0/234423155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00cc980 unknown :-1 s=CLOSED pgs=1493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 -- 172.21.15.86:0/234423155 >> 172.21.15.86:0/234423155 conn(0x7f1aa00b3620 msgr2=0x7f1aa00b3a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 -- 172.21.15.86:0/234423155 shutdown_connections 2024-03-06T07:34:44.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.664+0000 7f1aa7200640 1 -- 172.21.15.86:0/234423155 wait complete. 2024-03-06T07:34:44.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.666+0000 7f1aa7200640 1 Processor -- start 2024-03-06T07:34:44.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.666+0000 7f1aa7200640 1 -- start start 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa7200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa00ce7e0 0x7f1aa00b6250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00c9c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa7200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aa015f0f0 0x7f1aa00ca140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa7200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1aa00d0700 con 0x7f1aa015dfa0 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa7200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1aa00cdbe0 con 0x7f1aa015f0f0 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa7200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1aa00d0880 con 0x7f1aa00ce7e0 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00c9c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa5400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00c9c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54784/0 (socket says 172.21.15.86:54784) 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.667+0000 7f1aa5400640 1 -- 172.21.15.86:0/4263349264 learned_addr learned my addr 172.21.15.86:0/4263349264 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:44.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa5400640 1 -- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa00ce7e0 msgr2=0x7f1aa00b6250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa5e00640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa00ce7e0 0x7f1aa00b6250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa5400640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa00ce7e0 0x7f1aa00b6250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa6800640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aa015f0f0 0x7f1aa00ca140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa5400640 1 -- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aa015f0f0 msgr2=0x7f1aa00ca140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa5400640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aa015f0f0 0x7f1aa00ca140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa5400640 1 -- 172.21.15.86:0/4263349264 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1aa00b4660 con 0x7f1aa015dfa0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa5400640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00c9c00 secure :-1 s=READY pgs=1494 cs=0 l=1 rev1=1 crypto rx=0x7f1a9000d5f0 tx=0x7f1a9000dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1a9a000640 1 -- 172.21.15.86:0/4263349264 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a900125f0 con 0x7f1aa015dfa0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1a9a000640 1 -- 172.21.15.86:0/4263349264 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a90019070 con 0x7f1aa015dfa0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1aa00b4940 con 0x7f1aa015dfa0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1a9a000640 1 -- 172.21.15.86:0/4263349264 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a900129d0 con 0x7f1aa015dfa0 2024-03-06T07:34:44.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.668+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1aa00b4f00 con 0x7f1aa015dfa0 2024-03-06T07:34:44.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.669+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1aa00b5360 con 0x7f1aa015dfa0 2024-03-06T07:34:44.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.670+0000 7f1a9a000640 1 -- 172.21.15.86:0/4263349264 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1a90012b70 con 0x7f1aa015dfa0 2024-03-06T07:34:44.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.672+0000 7f1a9a000640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8406b4d0 0x7f1a8406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:44.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.672+0000 7f1a9a000640 1 -- 172.21.15.86:0/4263349264 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(126..126 src has 1..126) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1a9008a620 con 0x7f1aa015dfa0 2024-03-06T07:34:44.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.673+0000 7f1aa5e00640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8406b4d0 0x7f1a8406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:44.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.673+0000 7f1aa5e00640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8406b4d0 0x7f1a8406d990 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f1a94036000 tx=0x7f1a9403b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:44.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.676+0000 7f1a9a000640 1 -- 172.21.15.86:0/4263349264 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1a90059490 con 0x7f1aa015dfa0 2024-03-06T07:34:44.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.872+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1aa004f330 con 0x7f1aa015dfa0 2024-03-06T07:34:44.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.874+0000 7f1a9a000640 1 -- 172.21.15.86:0/4263349264 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v126) v1 ==== 74+0+16704 (secure 0 0 0) 0x7f1a90058d90 con 0x7f1aa015dfa0 2024-03-06T07:34:44.876 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:44.876 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":126,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:34:43.454697+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:34:41.263023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"126","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-06T07:34:42.462100+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"126","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8406b4d0 msgr2=0x7f1a8406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8406b4d0 0x7f1a8406d990 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f1a94036000 tx=0x7f1a9403b040 comp rx=0 tx=0).stop 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 msgr2=0x7f1aa00c9c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00c9c00 secure :-1 s=READY pgs=1494 cs=0 l=1 rev1=1 crypto rx=0x7f1a9000d5f0 tx=0x7f1a9000dab0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 shutdown_connections 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8406b4d0 0x7f1a8406d990 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aa015f0f0 0x7f1aa00ca140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aa015dfa0 0x7f1aa00c9c00 unknown :-1 s=CLOSED pgs=1494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 --2- 172.21.15.86:0/4263349264 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aa00ce7e0 0x7f1aa00b6250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:44.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 >> 172.21.15.86:0/4263349264 conn(0x7f1aa00b3620 msgr2=0x7f1aa00d1a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:44.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 shutdown_connections 2024-03-06T07:34:44.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:44.877+0000 7f1aa7200640 1 -- 172.21.15.86:0/4263349264 wait complete. 2024-03-06T07:34:44.898 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-06T07:34:45.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.114+0000 7f3be6200640 1 Processor -- start 2024-03-06T07:34:45.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.114+0000 7f3be6200640 1 -- start start 2024-03-06T07:34:45.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.116+0000 7f3be6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be00d0570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:45.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.116+0000 7f3be6200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3be00b43b0 0x7f3be015fea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:45.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.116+0000 7f3be6200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3be00ce1e0 con 0x7f3be01603e0 2024-03-06T07:34:45.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.116+0000 7f3be6200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3be0160830 con 0x7f3be00b43b0 2024-03-06T07:34:45.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.116+0000 7f3be6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3be00cedf0 con 0x7f3be015c330 2024-03-06T07:34:45.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bd7e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3be00b43b0 0x7f3be015fea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:45.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bdf400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be00d0570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:45.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bd7e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3be00b43b0 0x7f3be015fea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:58268/0 (socket says 172.21.15.86:58268) 2024-03-06T07:34:45.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bd7e00640 1 -- 172.21.15.86:0/62004924 learned_addr learned my addr 172.21.15.86:0/62004924 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:45.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bd7e00640 1 -- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 msgr2=0x7f3be00d0570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:45.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bd7e00640 1 --2- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be00d0570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:45.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bd7e00640 1 -- 172.21.15.86:0/62004924 >> v1:172.21.15.194:6789/0 conn(0x7f3be01603e0 legacy=0x7f3be015c710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:45.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bd7e00640 1 -- 172.21.15.86:0/62004924 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3be00ce4c0 con 0x7f3be00b43b0 2024-03-06T07:34:45.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.117+0000 7f3bdf400640 1 --2- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be00d0570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:34:45.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.119+0000 7f3bd7e00640 1 --2- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 0x7f3be015fea0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8009870 tx=0x7f3bc802b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=359eb057b42052e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:45.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.119+0000 7f3bdea00640 1 -- 172.21.15.86:0/62004924 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bc8034070 con 0x7f3be00b43b0 2024-03-06T07:34:45.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.119+0000 7f3bdea00640 1 -- 172.21.15.86:0/62004924 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3bc8030840 con 0x7f3be00b43b0 2024-03-06T07:34:45.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.119+0000 7f3bdea00640 1 -- 172.21.15.86:0/62004924 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bc8030c10 con 0x7f3be00b43b0 2024-03-06T07:34:45.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 -- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 msgr2=0x7f3be015fea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:45.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 --2- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 0x7f3be015fea0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8009870 tx=0x7f3bc802b980 comp rx=0 tx=0).stop 2024-03-06T07:34:45.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 -- 172.21.15.86:0/62004924 shutdown_connections 2024-03-06T07:34:45.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 --2- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 0x7f3be015fea0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:45.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 --2- 172.21.15.86:0/62004924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be00d0570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:45.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 -- 172.21.15.86:0/62004924 >> 172.21.15.86:0/62004924 conn(0x7f3be00b3790 msgr2=0x7f3be00b3ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:45.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 -- 172.21.15.86:0/62004924 shutdown_connections 2024-03-06T07:34:45.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.120+0000 7f3be6200640 1 -- 172.21.15.86:0/62004924 wait complete. 2024-03-06T07:34:45.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.122+0000 7f3be6200640 1 Processor -- start 2024-03-06T07:34:45.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.122+0000 7f3be6200640 1 -- start start 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3be6200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 0x7f3be015ad40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3be6200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be015e5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3be6200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be01603e0 0x7f3be00b54c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3be6200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3be00cd880 con 0x7f3be01603e0 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3be6200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3be00cea80 con 0x7f3be00b43b0 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3be6200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3be00cf280 con 0x7f3be015c330 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3bdfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be01603e0 0x7f3be00b54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3bdfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be01603e0 0x7f3be00b54c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54792/0 (socket says 172.21.15.86:54792) 2024-03-06T07:34:45.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.123+0000 7f3bdfe00640 1 -- 172.21.15.86:0/3814469232 learned_addr learned my addr 172.21.15.86:0/3814469232 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:45.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bd7e00640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be015e5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bdf400640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 0x7f3be015ad40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bdfe00640 1 -- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 msgr2=0x7f3be015e5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bdfe00640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be015e5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bdfe00640 1 -- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 msgr2=0x7f3be015ad40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bdfe00640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 0x7f3be015ad40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bdfe00640 1 -- 172.21.15.86:0/3814469232 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3be00b5cf0 con 0x7f3be01603e0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bdfe00640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be01603e0 0x7f3be00b54c0 secure :-1 s=READY pgs=1495 cs=0 l=1 rev1=1 crypto rx=0x7f3bd800c6b0 tx=0x7f3bd800cb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bd7400640 1 -- 172.21.15.86:0/3814469232 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bd8003b70 con 0x7f3be01603e0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.124+0000 7f3bd7400640 1 -- 172.21.15.86:0/3814469232 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3bd8003d30 con 0x7f3be01603e0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.125+0000 7f3bd7400640 1 -- 172.21.15.86:0/3814469232 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bd8005310 con 0x7f3be01603e0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.125+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3be00b5fd0 con 0x7f3be01603e0 2024-03-06T07:34:45.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.125+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3be00b6590 con 0x7f3be01603e0 2024-03-06T07:34:45.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.126+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3be004f330 con 0x7f3be01603e0 2024-03-06T07:34:45.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.128+0000 7f3bd7400640 1 -- 172.21.15.86:0/3814469232 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3bd80054b0 con 0x7f3be01603e0 2024-03-06T07:34:45.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.129+0000 7f3bd7400640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:45.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.129+0000 7f3bd7400640 1 -- 172.21.15.86:0/3814469232 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3bd808a100 con 0x7f3be01603e0 2024-03-06T07:34:45.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.130+0000 7f3bdf400640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:45.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.130+0000 7f3bdf400640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f3bd0009560 tx=0x7f3bd0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:45.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.134+0000 7f3bd7400640 1 -- 172.21.15.86:0/3814469232 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3bd8058ee0 con 0x7f3be01603e0 2024-03-06T07:34:45.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:45.372+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f3be015f520 con 0x7f3be01603e0 2024-03-06T07:34:46.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.033+0000 7f3bd7400640 1 -- 172.21.15.86:0/3814469232 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v92) v1 ==== 118+0+0 (secure 0 0 0) 0x7f3bd80587e0 con 0x7f3be01603e0 2024-03-06T07:34:46.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 msgr2=0x7f3bbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:46.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f3bd0009560 tx=0x7f3bd0008040 comp rx=0 tx=0).stop 2024-03-06T07:34:46.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be01603e0 msgr2=0x7f3be00b54c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:46.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be01603e0 0x7f3be00b54c0 secure :-1 s=READY pgs=1495 cs=0 l=1 rev1=1 crypto rx=0x7f3bd800c6b0 tx=0x7f3bd800cb70 comp rx=0 tx=0).stop 2024-03-06T07:34:46.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 shutdown_connections 2024-03-06T07:34:46.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3bbc06b4d0 0x7f3bbc06d990 unknown :-1 s=CLOSED pgs=640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3be01603e0 0x7f3be00b54c0 unknown :-1 s=CLOSED pgs=1495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3be015c330 0x7f3be015e5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 --2- 172.21.15.86:0/3814469232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3be00b43b0 0x7f3be015ad40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 >> 172.21.15.86:0/3814469232 conn(0x7f3be00b3790 msgr2=0x7f3be006d040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:46.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 shutdown_connections 2024-03-06T07:34:46.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.038+0000 7f3be6200640 1 -- 172.21.15.86:0/3814469232 wait complete. 2024-03-06T07:34:46.067 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-06T07:34:46.068 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-06T07:34:46.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.269+0000 7f3e7e600640 1 Processor -- start 2024-03-06T07:34:46.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.269+0000 7f3e7e600640 1 -- start start 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e7e600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e780b3df0 0x7f3e780727a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e7e600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3e780cda90 0x7f3e7806c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e7e600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3e780719e0 con 0x7f3e780b3650 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e7e600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e78072310 con 0x7f3e780b3df0 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e7e600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e78071cc0 con 0x7f3e780cda90 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e77400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e780b3df0 0x7f3e780727a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e77400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e780b3df0 0x7f3e780727a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42886/0 (socket says 172.21.15.86:42886) 2024-03-06T07:34:46.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e77400640 1 -- 172.21.15.86:0/3866937479 learned_addr learned my addr 172.21.15.86:0/3866937479 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:46.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e77e00640 1 --2- 172.21.15.86:0/3866937479 >> v2:172.21.15.194:3300/0 conn(0x7f3e780cda90 0x7f3e7806c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:46.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.271+0000 7f3e77400640 1 -- 172.21.15.86:0/3866937479 >> v2:172.21.15.194:3300/0 conn(0x7f3e780cda90 msgr2=0x7f3e7806c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:46.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.272+0000 7f3e77400640 1 --2- 172.21.15.86:0/3866937479 >> v2:172.21.15.194:3300/0 conn(0x7f3e780cda90 0x7f3e7806c990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.272+0000 7f3e77400640 1 -- 172.21.15.86:0/3866937479 >> v1:172.21.15.86:6789/0 conn(0x7f3e780b3650 legacy=0x7f3e780b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:46.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.272+0000 7f3e77400640 1 -- 172.21.15.86:0/3866937479 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e78071fa0 con 0x7f3e780b3df0 2024-03-06T07:34:46.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.272+0000 7f3e76a00640 1 -- 172.21.15.86:0/3866937479 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2166152498 0 0) 0x7f3e780719e0 con 0x7f3e780b3650 2024-03-06T07:34:46.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.273+0000 7f3e77400640 1 --2- 172.21.15.86:0/3866937479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780b3df0 0x7f3e780727a0 secure :-1 s=READY pgs=1497 cs=0 l=1 rev1=1 crypto rx=0x7f3e7806ed00 tx=0x7f3e6802e170 comp rx=0 tx=0).ready entity=mon.0 client_cookie=76a4426d597afe18 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:46.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.273+0000 7f3e76a00640 1 -- 172.21.15.86:0/3866937479 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e6803b070 con 0x7f3e780b3df0 2024-03-06T07:34:46.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.273+0000 7f3e76a00640 1 -- 172.21.15.86:0/3866937479 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e68036440 con 0x7f3e780b3df0 2024-03-06T07:34:46.275 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.273+0000 7f3e76a00640 1 -- 172.21.15.86:0/3866937479 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e68036810 con 0x7f3e780b3df0 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.273+0000 7f3e7e600640 1 -- 172.21.15.86:0/3866937479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780b3df0 msgr2=0x7f3e780727a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.273+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3866937479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780b3df0 0x7f3e780727a0 secure :-1 s=READY pgs=1497 cs=0 l=1 rev1=1 crypto rx=0x7f3e7806ed00 tx=0x7f3e6802e170 comp rx=0 tx=0).stop 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.274+0000 7f3e7e600640 1 -- 172.21.15.86:0/3866937479 shutdown_connections 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.274+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3866937479 >> v2:172.21.15.194:3300/0 conn(0x7f3e780cda90 0x7f3e7806c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.274+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3866937479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780b3df0 0x7f3e780727a0 unknown :-1 s=CLOSED pgs=1497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.274+0000 7f3e7e600640 1 -- 172.21.15.86:0/3866937479 >> 172.21.15.86:0/3866937479 conn(0x7f3e780b1940 msgr2=0x7f3e780b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.274+0000 7f3e7e600640 1 -- 172.21.15.86:0/3866937479 shutdown_connections 2024-03-06T07:34:46.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.274+0000 7f3e7e600640 1 -- 172.21.15.86:0/3866937479 wait complete. 2024-03-06T07:34:46.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.275+0000 7f3e7e600640 1 Processor -- start 2024-03-06T07:34:46.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.275+0000 7f3e7e600640 1 -- start start 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.276+0000 7f3e7e600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e780b3650 0x7f3e7815df10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.276+0000 7f3e7e600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e780b3df0 0x7f3e78150e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.276+0000 7f3e7e600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780cda90 0x7f3e781f41c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.276+0000 7f3e7e600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e7815b3c0 con 0x7f3e780cda90 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.276+0000 7f3e7e600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e7815b240 con 0x7f3e780b3650 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.276+0000 7f3e7e600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3e780cf510 con 0x7f3e780b3df0 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780cda90 0x7f3e781f41c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780cda90 0x7f3e781f41c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42902/0 (socket says 172.21.15.86:42902) 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 -- 172.21.15.86:0/3614219186 learned_addr learned my addr 172.21.15.86:0/3614219186 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:46.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e6fe00640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e780b3df0 0x7f3e78150e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:46.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77400640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e780b3650 0x7f3e7815df10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:46.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 -- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e780b3df0 msgr2=0x7f3e78150e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:46.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e780b3df0 0x7f3e78150e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 -- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e780b3650 msgr2=0x7f3e7815df10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:46.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e780b3650 0x7f3e7815df10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:46.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.277+0000 7f3e77e00640 1 -- 172.21.15.86:0/3614219186 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e781f4cf0 con 0x7f3e780cda90 2024-03-06T07:34:46.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.278+0000 7f3e77e00640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780cda90 0x7f3e781f41c0 secure :-1 s=READY pgs=1498 cs=0 l=1 rev1=1 crypto rx=0x7f3e7000cd70 tx=0x7f3e7000a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:46.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.278+0000 7f3e6f400640 1 -- 172.21.15.86:0/3614219186 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e70003b90 con 0x7f3e780cda90 2024-03-06T07:34:46.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.278+0000 7f3e6f400640 1 -- 172.21.15.86:0/3614219186 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e70003d30 con 0x7f3e780cda90 2024-03-06T07:34:46.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.278+0000 7f3e6f400640 1 -- 172.21.15.86:0/3614219186 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e70004240 con 0x7f3e780cda90 2024-03-06T07:34:46.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.278+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e781f4fd0 con 0x7f3e780cda90 2024-03-06T07:34:46.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.278+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e780cba50 con 0x7f3e780cda90 2024-03-06T07:34:46.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.279+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3e7804f330 con 0x7f3e780cda90 2024-03-06T07:34:46.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.280+0000 7f3e6f400640 1 -- 172.21.15.86:0/3614219186 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3e700043e0 con 0x7f3e780cda90 2024-03-06T07:34:46.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.282+0000 7f3e6f400640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e5406b4d0 0x7f3e5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:46.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.282+0000 7f3e6f400640 1 -- 172.21.15.86:0/3614219186 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3e7008ad80 con 0x7f3e780cda90 2024-03-06T07:34:46.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.282+0000 7f3e77400640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e5406b4d0 0x7f3e5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:46.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.283+0000 7f3e77400640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e5406b4d0 0x7f3e5406d990 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7f3e68038000 tx=0x7f3e68008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:46.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.289+0000 7f3e6f400640 1 -- 172.21.15.86:0/3614219186 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3e7001d340 con 0x7f3e780cda90 2024-03-06T07:34:46.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:46.517+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f3e780b4a70 con 0x7f3e780cda90 2024-03-06T07:34:47.049 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.048+0000 7f3e6f400640 1 -- 172.21.15.86:0/3614219186 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v93) v1 ==== 122+0+0 (secure 0 0 0) 0x7f3e70059c00 con 0x7f3e780cda90 2024-03-06T07:34:47.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e5406b4d0 msgr2=0x7f3e5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.053 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e5406b4d0 0x7f3e5406d990 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7f3e68038000 tx=0x7f3e68008040 comp rx=0 tx=0).stop 2024-03-06T07:34:47.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780cda90 msgr2=0x7f3e781f41c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780cda90 0x7f3e781f41c0 secure :-1 s=READY pgs=1498 cs=0 l=1 rev1=1 crypto rx=0x7f3e7000cd70 tx=0x7f3e7000a660 comp rx=0 tx=0).stop 2024-03-06T07:34:47.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 shutdown_connections 2024-03-06T07:34:47.054 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e5406b4d0 0x7f3e5406d990 unknown :-1 s=CLOSED pgs=641 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e780cda90 0x7f3e781f41c0 unknown :-1 s=CLOSED pgs=1498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e780b3df0 0x7f3e78150e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 --2- 172.21.15.86:0/3614219186 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e780b3650 0x7f3e7815df10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.052+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 >> 172.21.15.86:0/3614219186 conn(0x7f3e780b1940 msgr2=0x7f3e780b0ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:47.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.053+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 shutdown_connections 2024-03-06T07:34:47.055 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.053+0000 7f3e7e600640 1 -- 172.21.15.86:0/3614219186 wait complete. 2024-03-06T07:34:47.073 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-06T07:34:47.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:47.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.275+0000 7f3a2d000640 1 Processor -- start 2024-03-06T07:34:47.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.275+0000 7f3a2d000640 1 -- start start 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.276+0000 7f3a2d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3a280b4680 0x7f3a2806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.276+0000 7f3a2d000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3a28072340 0x7f3a2806ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.276+0000 7f3a2d000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3a2815b090 con 0x7f3a2815acb0 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a2d000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a280b2a30 con 0x7f3a280b4680 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a2d000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a280b4360 con 0x7f3a28072340 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a1d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3a280b4680 0x7f3a2806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a1d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3a280b4680 0x7f3a2806e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42918/0 (socket says 172.21.15.86:42918) 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a1d400640 1 -- 172.21.15.86:0/1993530565 learned_addr learned my addr 172.21.15.86:0/1993530565 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:47.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a26800640 1 --2- 172.21.15.86:0/1993530565 >> v2:172.21.15.194:3300/0 conn(0x7f3a28072340 0x7f3a2806ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a1d400640 1 -- 172.21.15.86:0/1993530565 >> v2:172.21.15.194:3300/0 conn(0x7f3a28072340 msgr2=0x7f3a2806ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:47.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a1d400640 1 --2- 172.21.15.86:0/1993530565 >> v2:172.21.15.194:3300/0 conn(0x7f3a28072340 0x7f3a2806ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a1d400640 1 -- 172.21.15.86:0/1993530565 >> v1:172.21.15.194:6789/0 conn(0x7f3a2815acb0 legacy=0x7f3a280b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:47.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.277+0000 7f3a1d400640 1 -- 172.21.15.86:0/1993530565 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a280b2cf0 con 0x7f3a280b4680 2024-03-06T07:34:47.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.279+0000 7f3a1d400640 1 --2- 172.21.15.86:0/1993530565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a280b4680 0x7f3a2806e4f0 secure :-1 s=READY pgs=1499 cs=0 l=1 rev1=1 crypto rx=0x7f3a280b4e60 tx=0x7f3a1002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c6a9766e24b303a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:47.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.279+0000 7f3a25400640 1 -- 172.21.15.86:0/1993530565 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a10035020 con 0x7f3a280b4680 2024-03-06T07:34:47.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.279+0000 7f3a25400640 1 -- 172.21.15.86:0/1993530565 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a10031870 con 0x7f3a280b4680 2024-03-06T07:34:47.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.279+0000 7f3a25400640 1 -- 172.21.15.86:0/1993530565 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a10031c40 con 0x7f3a280b4680 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 -- 172.21.15.86:0/1993530565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a280b4680 msgr2=0x7f3a2806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 --2- 172.21.15.86:0/1993530565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a280b4680 0x7f3a2806e4f0 secure :-1 s=READY pgs=1499 cs=0 l=1 rev1=1 crypto rx=0x7f3a280b4e60 tx=0x7f3a1002b910 comp rx=0 tx=0).stop 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 -- 172.21.15.86:0/1993530565 shutdown_connections 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 --2- 172.21.15.86:0/1993530565 >> v2:172.21.15.194:3300/0 conn(0x7f3a28072340 0x7f3a2806ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 --2- 172.21.15.86:0/1993530565 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a280b4680 0x7f3a2806e4f0 unknown :-1 s=CLOSED pgs=1499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 -- 172.21.15.86:0/1993530565 >> 172.21.15.86:0/1993530565 conn(0x7f3a280b1730 msgr2=0x7f3a280b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 -- 172.21.15.86:0/1993530565 shutdown_connections 2024-03-06T07:34:47.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.280+0000 7f3a2d000640 1 -- 172.21.15.86:0/1993530565 wait complete. 2024-03-06T07:34:47.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.283+0000 7f3a2d000640 1 Processor -- start 2024-03-06T07:34:47.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.283+0000 7f3a2d000640 1 -- start start 2024-03-06T07:34:47.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.284+0000 7f3a2d000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a28072340 0x7f3a2815dbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.284+0000 7f3a2d000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a280b4680 0x7f3a28144fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.284+0000 7f3a2d000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a2815acb0 0x7f3a28152360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.284+0000 7f3a2d000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a280b4010 con 0x7f3a2815acb0 2024-03-06T07:34:47.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.284+0000 7f3a2d000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a280b32b0 con 0x7f3a280b4680 2024-03-06T07:34:47.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.284+0000 7f3a2d000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3a280ccfb0 con 0x7f3a28072340 2024-03-06T07:34:47.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a2815acb0 0x7f3a28152360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a2815acb0 0x7f3a28152360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42922/0 (socket says 172.21.15.86:42922) 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 -- 172.21.15.86:0/2875136364 learned_addr learned my addr 172.21.15.86:0/2875136364 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 -- 172.21.15.86:0/2875136364 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a28072340 msgr2=0x7f3a2815dbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a28072340 0x7f3a2815dbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 -- 172.21.15.86:0/2875136364 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a280b4680 msgr2=0x7f3a28144fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a280b4680 0x7f3a28144fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 -- 172.21.15.86:0/2875136364 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a281512c0 con 0x7f3a2815acb0 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a26800640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a2815acb0 0x7f3a28152360 secure :-1 s=READY pgs=1500 cs=0 l=1 rev1=1 crypto rx=0x7f3a2000d830 tx=0x7f3a2000dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.285+0000 7f3a1ea00640 1 -- 172.21.15.86:0/2875136364 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a2000b6c0 con 0x7f3a2815acb0 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.286+0000 7f3a1ea00640 1 -- 172.21.15.86:0/2875136364 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a20019070 con 0x7f3a2815acb0 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.286+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a281515a0 con 0x7f3a2815acb0 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.286+0000 7f3a1ea00640 1 -- 172.21.15.86:0/2875136364 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a2000bac0 con 0x7f3a2815acb0 2024-03-06T07:34:47.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.286+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a28151b60 con 0x7f3a2815acb0 2024-03-06T07:34:47.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.287+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a2804f330 con 0x7f3a2815acb0 2024-03-06T07:34:47.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.288+0000 7f3a1ea00640 1 -- 172.21.15.86:0/2875136364 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3a2001e460 con 0x7f3a2815acb0 2024-03-06T07:34:47.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.289+0000 7f3a1ea00640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a0406b4d0 0x7f3a0406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.289+0000 7f3a1ea00640 1 -- 172.21.15.86:0/2875136364 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3a2008a890 con 0x7f3a2815acb0 2024-03-06T07:34:47.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.290+0000 7f3a25e00640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a0406b4d0 0x7f3a0406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.290+0000 7f3a25e00640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a0406b4d0 0x7f3a0406d990 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7f3a1800abb0 tx=0x7f3a18009250 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:47.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.293+0000 7f3a1ea00640 1 -- 172.21.15.86:0/2875136364 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3a20059710 con 0x7f3a2815acb0 2024-03-06T07:34:47.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.561+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3a2806ce60 con 0x7f3a2815acb0 2024-03-06T07:34:47.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.562+0000 7f3a1ea00640 1 -- 172.21.15.86:0/2875136364 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 93 v93) v1 ==== 94+0+4982 (secure 0 0 0) 0x7f3a20059010 con 0x7f3a2815acb0 2024-03-06T07:34:47.564 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:47.564 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":93,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5495,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/374549834","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":374549834},{"type":"v1","addr":"172.21.15.194:6834","nonce":374549834}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88},{"gid":5498,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2544629493","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":2544629493},{"type":"v1","addr":"172.21.15.194:6835","nonce":2544629493}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88},{"gid":8148,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":87}],"filesystems":[{"mdsmap":{"epoch":93,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:46.519048+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8151},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:34:47.565 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 93 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a0406b4d0 msgr2=0x7f3a0406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a0406b4d0 0x7f3a0406d990 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7f3a1800abb0 tx=0x7f3a18009250 comp rx=0 tx=0).stop 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a2815acb0 msgr2=0x7f3a28152360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a2815acb0 0x7f3a28152360 secure :-1 s=READY pgs=1500 cs=0 l=1 rev1=1 crypto rx=0x7f3a2000d830 tx=0x7f3a2000dcf0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 shutdown_connections 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3a0406b4d0 0x7f3a0406d990 unknown :-1 s=CLOSED pgs=642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3a2815acb0 0x7f3a28152360 unknown :-1 s=CLOSED pgs=1500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3a280b4680 0x7f3a28144fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 --2- 172.21.15.86:0/2875136364 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3a28072340 0x7f3a2815dbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 >> 172.21.15.86:0/2875136364 conn(0x7f3a280b1730 msgr2=0x7f3a280affd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:47.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 shutdown_connections 2024-03-06T07:34:47.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.565+0000 7f3a2d000640 1 -- 172.21.15.86:0/2875136364 wait complete. 2024-03-06T07:34:47.588 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:34:47.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.794+0000 7f08bbe00640 1 Processor -- start 2024-03-06T07:34:47.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.794+0000 7f08bbe00640 1 -- start start 2024-03-06T07:34:47.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.796+0000 7f08bbe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f08bc15a940 0x7f08bc0b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.796+0000 7f08bbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc0cc750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.796+0000 7f08bbe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f08bc0b2b30 con 0x7f08bc0cf370 2024-03-06T07:34:47.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.796+0000 7f08bbe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f08bc0b28e0 con 0x7f08bc15a940 2024-03-06T07:34:47.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.796+0000 7f08bbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f08bc0b2e10 con 0x7f08bc0cc2a0 2024-03-06T07:34:47.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.797+0000 7f08bb400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc0cc750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.797+0000 7f08baa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f08bc15a940 0x7f08bc0b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.797+0000 7f08baa00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f08bc15a940 0x7f08bc0b45e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:48874/0 (socket says 172.21.15.86:48874) 2024-03-06T07:34:47.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.797+0000 7f08baa00640 1 -- 172.21.15.86:0/853709537 learned_addr learned my addr 172.21.15.86:0/853709537 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:47.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.798+0000 7f08bb400640 1 -- 172.21.15.86:0/853709537 >> v2:172.21.15.194:3300/0 conn(0x7f08bc15a940 msgr2=0x7f08bc0b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:47.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.798+0000 7f08bb400640 1 --2- 172.21.15.86:0/853709537 >> v2:172.21.15.194:3300/0 conn(0x7f08bc15a940 0x7f08bc0b45e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.798+0000 7f08bb400640 1 -- 172.21.15.86:0/853709537 >> v1:172.21.15.194:6789/0 conn(0x7f08bc0cf370 legacy=0x7f08bc06d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.798+0000 7f08bb400640 1 -- 172.21.15.86:0/853709537 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08bc06ca50 con 0x7f08bc0cc2a0 2024-03-06T07:34:47.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.798+0000 7f08baa00640 1 --2- 172.21.15.86:0/853709537 >> v2:172.21.15.194:3300/0 conn(0x7f08bc15a940 0x7f08bc0b45e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:34:47.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.799+0000 7f08bb400640 1 --2- 172.21.15.86:0/853709537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc0cc750 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f08b4009d90 tx=0x7f08b402bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ae48b1e1140c4e00 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:47.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.799+0000 7f08b9600640 1 -- 172.21.15.86:0/853709537 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08b4039070 con 0x7f08bc0cc2a0 2024-03-06T07:34:47.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08b9600640 1 -- 172.21.15.86:0/853709537 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08b402cc50 con 0x7f08bc0cc2a0 2024-03-06T07:34:47.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08b9600640 1 -- 172.21.15.86:0/853709537 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08b4034440 con 0x7f08bc0cc2a0 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 -- 172.21.15.86:0/853709537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 msgr2=0x7f08bc0cc750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 --2- 172.21.15.86:0/853709537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc0cc750 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f08b4009d90 tx=0x7f08b402bea0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 -- 172.21.15.86:0/853709537 shutdown_connections 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 --2- 172.21.15.86:0/853709537 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc0cc750 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 --2- 172.21.15.86:0/853709537 >> v2:172.21.15.194:3300/0 conn(0x7f08bc15a940 0x7f08bc0b45e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 -- 172.21.15.86:0/853709537 >> 172.21.15.86:0/853709537 conn(0x7f08bc06afc0 msgr2=0x7f08bc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 -- 172.21.15.86:0/853709537 shutdown_connections 2024-03-06T07:34:47.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.800+0000 7f08bbe00640 1 -- 172.21.15.86:0/853709537 wait complete. 2024-03-06T07:34:47.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.802+0000 7f08bbe00640 1 Processor -- start 2024-03-06T07:34:47.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.802+0000 7f08bbe00640 1 -- start start 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08bbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc154bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08bbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08bc0cf370 0x7f08bc070d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08bbe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08bc15a940 0x7f08bc0712c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08bbe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08bc0b4b20 con 0x7f08bc0cf370 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08bbe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08bc0b30f0 con 0x7f08bc15a940 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08bbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f08bc15ad20 con 0x7f08bc0cc2a0 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08ba000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08bc0cf370 0x7f08bc070d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08ba000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08bc0cf370 0x7f08bc070d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42926/0 (socket says 172.21.15.86:42926) 2024-03-06T07:34:47.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.803+0000 7f08ba000640 1 -- 172.21.15.86:0/2646184566 learned_addr learned my addr 172.21.15.86:0/2646184566 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08bb400640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08bc15a940 0x7f08bc0712c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08baa00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc154bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08ba000640 1 -- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 msgr2=0x7f08bc154bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08ba000640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc154bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08ba000640 1 -- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08bc15a940 msgr2=0x7f08bc0712c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08ba000640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08bc15a940 0x7f08bc0712c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08ba000640 1 -- 172.21.15.86:0/2646184566 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08bc15d2d0 con 0x7f08bc0cf370 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08baa00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc154bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08ba000640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08bc0cf370 0x7f08bc070d80 secure :-1 s=READY pgs=1501 cs=0 l=1 rev1=1 crypto rx=0x7f08a800c400 tx=0x7f08a800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08b3400640 1 -- 172.21.15.86:0/2646184566 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08a80052d0 con 0x7f08bc0cf370 2024-03-06T07:34:47.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08b3400640 1 -- 172.21.15.86:0/2646184566 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08a8005490 con 0x7f08bc0cf370 2024-03-06T07:34:47.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08bc15d5b0 con 0x7f08bc0cf370 2024-03-06T07:34:47.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.804+0000 7f08b3400640 1 -- 172.21.15.86:0/2646184566 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08a8007260 con 0x7f08bc0cf370 2024-03-06T07:34:47.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.805+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08bc15db70 con 0x7f08bc0cf370 2024-03-06T07:34:47.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.806+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f08bc04f330 con 0x7f08bc0cf370 2024-03-06T07:34:47.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.807+0000 7f08b3400640 1 -- 172.21.15.86:0/2646184566 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f08a8007400 con 0x7f08bc0cf370 2024-03-06T07:34:47.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.808+0000 7f08b3400640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f089c06b500 0x7f089c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:47.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.808+0000 7f08b3400640 1 -- 172.21.15.86:0/2646184566 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f08a80897c0 con 0x7f08bc0cf370 2024-03-06T07:34:47.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.808+0000 7f08baa00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f089c06b500 0x7f089c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:47.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.809+0000 7f08baa00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f089c06b500 0x7f089c06d9c0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7f08bc14e7d0 tx=0x7f08ac008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:47.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:47.816+0000 7f08b3400640 1 -- 172.21.15.86:0/2646184566 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f08a8058520 con 0x7f08bc0cf370 2024-03-06T07:34:48.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.193+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f08bc0721f0 con 0x7f08bc0cf370 2024-03-06T07:34:48.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.195+0000 7f08b3400640 1 -- 172.21.15.86:0/2646184566 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v127) v1 ==== 74+0+16691 (secure 0 0 0) 0x7f08a8057e20 con 0x7f08bc0cf370 2024-03-06T07:34:48.198 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:48.198 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":127,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:34:45.008922+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:34:41.263023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"126","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-06T07:34:42.462100+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"127","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f089c06b500 msgr2=0x7f089c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f089c06b500 0x7f089c06d9c0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7f08bc14e7d0 tx=0x7f08ac008040 comp rx=0 tx=0).stop 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08bc0cf370 msgr2=0x7f08bc070d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08bc0cf370 0x7f08bc070d80 secure :-1 s=READY pgs=1501 cs=0 l=1 rev1=1 crypto rx=0x7f08a800c400 tx=0x7f08a800c8c0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 shutdown_connections 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f089c06b500 0x7f089c06d9c0 unknown :-1 s=CLOSED pgs=643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08bc15a940 0x7f08bc0712c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08bc0cf370 0x7f08bc070d80 unknown :-1 s=CLOSED pgs=1501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 --2- 172.21.15.86:0/2646184566 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08bc0cc2a0 0x7f08bc154bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 >> 172.21.15.86:0/2646184566 conn(0x7f08bc06afc0 msgr2=0x7f08bc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 shutdown_connections 2024-03-06T07:34:48.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.199+0000 7f08bbe00640 1 -- 172.21.15.86:0/2646184566 wait complete. 2024-03-06T07:34:48.225 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-06T07:34:48.226 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:34:48.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.406+0000 7ff19e600640 1 Processor -- start 2024-03-06T07:34:48.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.406+0000 7ff19e600640 1 -- start start 2024-03-06T07:34:48.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.408+0000 7ff19e600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff19815ac50 0x7ff19806f6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.408+0000 7ff19e600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff19815f610 con 0x7ff19815efb0 2024-03-06T07:34:48.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff19e600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff19806ebe0 con 0x7ff1980d09f0 2024-03-06T07:34:48.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff19e600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff19806e990 con 0x7ff19815ac50 2024-03-06T07:34:48.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff197400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff19815ac50 0x7ff19806f6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff197400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff19815ac50 0x7ff19806f6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42936/0 (socket says 172.21.15.86:42936) 2024-03-06T07:34:48.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff197400640 1 -- 172.21.15.86:0/3394397070 learned_addr learned my addr 172.21.15.86:0/3394397070 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:48.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff197400640 1 -- 172.21.15.86:0/3394397070 >> v1:172.21.15.194:6789/0 conn(0x7ff1980d09f0 legacy=0x7ff19815e8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:48.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff197400640 1 -- 172.21.15.86:0/3394397070 >> v1:172.21.15.86:6789/0 conn(0x7ff19815efb0 legacy=0x7ff1980cdfc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.409+0000 7ff197400640 1 -- 172.21.15.86:0/3394397070 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff19806eec0 con 0x7ff19815ac50 2024-03-06T07:34:48.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.410+0000 7ff197400640 1 --2- 172.21.15.86:0/3394397070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff19815ac50 0x7ff19806f6c0 secure :-1 s=READY pgs=1503 cs=0 l=1 rev1=1 crypto rx=0x7ff188009870 tx=0x7ff18802b990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6418aca78f7d4dd6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:48.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff196000640 1 -- 172.21.15.86:0/3394397070 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff188034070 con 0x7ff19815ac50 2024-03-06T07:34:48.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff196000640 1 -- 172.21.15.86:0/3394397070 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff188030850 con 0x7ff19815ac50 2024-03-06T07:34:48.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff196000640 1 -- 172.21.15.86:0/3394397070 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff188030c20 con 0x7ff19815ac50 2024-03-06T07:34:48.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff19e600640 1 -- 172.21.15.86:0/3394397070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff19815ac50 msgr2=0x7ff19806f6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff19e600640 1 --2- 172.21.15.86:0/3394397070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff19815ac50 0x7ff19806f6c0 secure :-1 s=READY pgs=1503 cs=0 l=1 rev1=1 crypto rx=0x7ff188009870 tx=0x7ff18802b990 comp rx=0 tx=0).stop 2024-03-06T07:34:48.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff19e600640 1 -- 172.21.15.86:0/3394397070 shutdown_connections 2024-03-06T07:34:48.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff19e600640 1 --2- 172.21.15.86:0/3394397070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff19815ac50 0x7ff19806f6c0 unknown :-1 s=CLOSED pgs=1503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff19e600640 1 -- 172.21.15.86:0/3394397070 >> 172.21.15.86:0/3394397070 conn(0x7ff19806c0d0 msgr2=0x7ff19806c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:48.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff19e600640 1 -- 172.21.15.86:0/3394397070 shutdown_connections 2024-03-06T07:34:48.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.411+0000 7ff19e600640 1 -- 172.21.15.86:0/3394397070 wait complete. 2024-03-06T07:34:48.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.413+0000 7ff19e600640 1 Processor -- start 2024-03-06T07:34:48.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.413+0000 7ff19e600640 1 -- start start 2024-03-06T07:34:48.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff19e600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1980d09f0 0x7ff1980cd4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff19e600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff19815ac50 0x7ff19815da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff19e600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff19815efb0 0x7ff198163f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff19e600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff19815b030 con 0x7ff1980d09f0 2024-03-06T07:34:48.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff19e600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff19815f3c0 con 0x7ff19815efb0 2024-03-06T07:34:48.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff19e600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff1980d0e50 con 0x7ff19815ac50 2024-03-06T07:34:48.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff197400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1980d09f0 0x7ff1980cd4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff197400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1980d09f0 0x7ff1980cd4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42950/0 (socket says 172.21.15.86:42950) 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.414+0000 7ff197400640 1 -- 172.21.15.86:0/336622886 learned_addr learned my addr 172.21.15.86:0/336622886 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff196a00640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff19815ac50 0x7ff19815da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff197e00640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff19815efb0 0x7ff198163f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff197400640 1 -- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff19815ac50 msgr2=0x7ff19815da90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff197400640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff19815ac50 0x7ff19815da90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff197400640 1 -- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff19815efb0 msgr2=0x7ff198163f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff197400640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff19815efb0 0x7ff198163f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff197400640 1 -- 172.21.15.86:0/336622886 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff198167680 con 0x7ff1980d09f0 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff196a00640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff19815ac50 0x7ff19815da90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff197400640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1980d09f0 0x7ff1980cd4a0 secure :-1 s=READY pgs=1504 cs=0 l=1 rev1=1 crypto rx=0x7ff188009870 tx=0x7ff188004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff18f400640 1 -- 172.21.15.86:0/336622886 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff188048070 con 0x7ff1980d09f0 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff18f400640 1 -- 172.21.15.86:0/336622886 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff188030650 con 0x7ff1980d09f0 2024-03-06T07:34:48.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff198167960 con 0x7ff1980d09f0 2024-03-06T07:34:48.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.415+0000 7ff18f400640 1 -- 172.21.15.86:0/336622886 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff188034040 con 0x7ff1980d09f0 2024-03-06T07:34:48.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.416+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff198167f20 con 0x7ff1980d09f0 2024-03-06T07:34:48.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.417+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff19804f330 con 0x7ff1980d09f0 2024-03-06T07:34:48.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.418+0000 7ff18f400640 1 -- 172.21.15.86:0/336622886 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff18804b070 con 0x7ff1980d09f0 2024-03-06T07:34:48.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.419+0000 7ff18f400640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff17806b4d0 0x7ff17806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.419+0000 7ff18f400640 1 -- 172.21.15.86:0/336622886 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff1880ac370 con 0x7ff1980d09f0 2024-03-06T07:34:48.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.419+0000 7ff196a00640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff17806b4d0 0x7ff17806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.420+0000 7ff196a00640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff17806b4d0 0x7ff17806d990 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7ff184007fd0 tx=0x7ff184009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:48.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.426+0000 7ff18f400640 1 -- 172.21.15.86:0/336622886 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff18807b150 con 0x7ff1980d09f0 2024-03-06T07:34:48.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.617+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7ff19806e300 con 0x7ff17806b4d0 2024-03-06T07:34:48.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.625+0000 7ff18f400640 1 -- 172.21.15.86:0/336622886 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252420 (secure 0 0 0) 0x7ff19806e300 con 0x7ff17806b4d0 2024-03-06T07:34:48.630 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:48.631 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.629+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff17806b4d0 msgr2=0x7ff17806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.629+0000 7ff19e600640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff17806b4d0 0x7ff17806d990 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7ff184007fd0 tx=0x7ff184009040 comp rx=0 tx=0).stop 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.629+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1980d09f0 msgr2=0x7ff1980cd4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.629+0000 7ff19e600640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1980d09f0 0x7ff1980cd4a0 secure :-1 s=READY pgs=1504 cs=0 l=1 rev1=1 crypto rx=0x7ff188009870 tx=0x7ff188004730 comp rx=0 tx=0).stop 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 shutdown_connections 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff17806b4d0 0x7ff17806d990 unknown :-1 s=CLOSED pgs=644 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff19815efb0 0x7ff198163f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff19815ac50 0x7ff19815da90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 --2- 172.21.15.86:0/336622886 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff1980d09f0 0x7ff1980cd4a0 unknown :-1 s=CLOSED pgs=1504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 >> 172.21.15.86:0/336622886 conn(0x7ff19806c0d0 msgr2=0x7ff19815b1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 shutdown_connections 2024-03-06T07:34:48.633 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.630+0000 7ff19e600640 1 -- 172.21.15.86:0/336622886 wait complete. 2024-03-06T07:34:48.648 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":354,"stamp":"2024-03-06T07:34:46.993028+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":70,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":296232,"kb_used_data":13480,"kb_used_omap":12,"kb_used_meta":282611,"kb_avail":754678488,"statfs":{"total":773094113280,"available":772790771712,"internally_reserved":0,"allocated":13803520,"data_stored":6631196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":289394227},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.021965"},"pg_stats":[{"pgid":"19.3f","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.042663+0000","last_change":"2024-03-06T07:34:44.037084+0000","last_active":"2024-03-06T07:34:44.042663+0000","last_peered":"2024-03-06T07:34:44.042663+0000","last_clean":"2024-03-06T07:34:44.042663+0000","last_became_active":"2024-03-06T07:34:44.031300+0000","last_became_peered":"2024-03-06T07:34:44.031300+0000","last_unstale":"2024-03-06T07:34:44.042663+0000","last_undegraded":"2024-03-06T07:34:44.042663+0000","last_fullsized":"2024-03-06T07:34:44.042663+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:12:41.958115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3e","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039875+0000","last_change":"2024-03-06T07:34:44.032281+0000","last_active":"2024-03-06T07:34:44.039875+0000","last_peered":"2024-03-06T07:34:44.039875+0000","last_clean":"2024-03-06T07:34:44.039875+0000","last_became_active":"2024-03-06T07:34:44.031988+0000","last_became_peered":"2024-03-06T07:34:44.031988+0000","last_unstale":"2024-03-06T07:34:44.039875+0000","last_undegraded":"2024-03-06T07:34:44.039875+0000","last_fullsized":"2024-03-06T07:34:44.039875+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:41:41.067874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3d","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028243+0000","last_change":"2024-03-06T07:34:44.024789+0000","last_active":"2024-03-06T07:34:45.028243+0000","last_peered":"2024-03-06T07:34:45.028243+0000","last_clean":"2024-03-06T07:34:45.028243+0000","last_became_active":"2024-03-06T07:34:44.021439+0000","last_became_peered":"2024-03-06T07:34:44.021439+0000","last_unstale":"2024-03-06T07:34:45.028243+0000","last_undegraded":"2024-03-06T07:34:45.028243+0000","last_fullsized":"2024-03-06T07:34:45.028243+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:54:21.297081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.3c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.027121+0000","last_change":"2024-03-06T07:34:44.023829+0000","last_active":"2024-03-06T07:34:44.027121+0000","last_peered":"2024-03-06T07:34:44.027121+0000","last_clean":"2024-03-06T07:34:44.027121+0000","last_became_active":"2024-03-06T07:34:44.023528+0000","last_became_peered":"2024-03-06T07:34:44.023528+0000","last_unstale":"2024-03-06T07:34:44.027121+0000","last_undegraded":"2024-03-06T07:34:44.027121+0000","last_fullsized":"2024-03-06T07:34:44.027121+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:58:10.029945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.037731+0000","last_change":"2024-03-06T07:34:44.031849+0000","last_active":"2024-03-06T07:34:44.037731+0000","last_peered":"2024-03-06T07:34:44.037731+0000","last_clean":"2024-03-06T07:34:44.037731+0000","last_became_active":"2024-03-06T07:34:44.027460+0000","last_became_peered":"2024-03-06T07:34:44.027460+0000","last_unstale":"2024-03-06T07:34:44.037731+0000","last_undegraded":"2024-03-06T07:34:44.037731+0000","last_fullsized":"2024-03-06T07:34:44.037731+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:38.410784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.3a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033280+0000","last_change":"2024-03-06T07:34:44.027413+0000","last_active":"2024-03-06T07:34:44.033280+0000","last_peered":"2024-03-06T07:34:44.033280+0000","last_clean":"2024-03-06T07:34:44.033280+0000","last_became_active":"2024-03-06T07:34:44.025864+0000","last_became_peered":"2024-03-06T07:34:44.025864+0000","last_unstale":"2024-03-06T07:34:44.033280+0000","last_undegraded":"2024-03-06T07:34:44.033280+0000","last_fullsized":"2024-03-06T07:34:44.033280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:38.410784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.39","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030834+0000","last_change":"2024-03-06T07:34:44.024017+0000","last_active":"2024-03-06T07:34:44.030834+0000","last_peered":"2024-03-06T07:34:44.030834+0000","last_clean":"2024-03-06T07:34:44.030834+0000","last_became_active":"2024-03-06T07:34:44.023744+0000","last_became_peered":"2024-03-06T07:34:44.023744+0000","last_unstale":"2024-03-06T07:34:44.030834+0000","last_undegraded":"2024-03-06T07:34:44.030834+0000","last_fullsized":"2024-03-06T07:34:44.030834+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:23.962585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.38","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028092+0000","last_change":"2024-03-06T07:34:44.029784+0000","last_active":"2024-03-06T07:34:45.028092+0000","last_peered":"2024-03-06T07:34:45.028092+0000","last_clean":"2024-03-06T07:34:45.028092+0000","last_became_active":"2024-03-06T07:34:44.029232+0000","last_became_peered":"2024-03-06T07:34:44.029232+0000","last_unstale":"2024-03-06T07:34:45.028092+0000","last_undegraded":"2024-03-06T07:34:45.028092+0000","last_fullsized":"2024-03-06T07:34:45.028092+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:43:43.782358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.37","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028222+0000","last_change":"2024-03-06T07:34:44.027094+0000","last_active":"2024-03-06T07:34:45.028222+0000","last_peered":"2024-03-06T07:34:45.028222+0000","last_clean":"2024-03-06T07:34:45.028222+0000","last_became_active":"2024-03-06T07:34:44.025393+0000","last_became_peered":"2024-03-06T07:34:44.025393+0000","last_unstale":"2024-03-06T07:34:45.028222+0000","last_undegraded":"2024-03-06T07:34:45.028222+0000","last_fullsized":"2024-03-06T07:34:45.028222+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:12:11.776779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.36","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.020354+0000","last_change":"2024-03-06T07:34:44.018805+0000","last_active":"2024-03-06T07:34:44.020354+0000","last_peered":"2024-03-06T07:34:44.020354+0000","last_clean":"2024-03-06T07:34:44.020354+0000","last_became_active":"2024-03-06T07:34:44.018438+0000","last_became_peered":"2024-03-06T07:34:44.018438+0000","last_unstale":"2024-03-06T07:34:44.020354+0000","last_undegraded":"2024-03-06T07:34:44.020354+0000","last_fullsized":"2024-03-06T07:34:44.020354+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:19.983769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.35","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.022441+0000","last_change":"2024-03-06T07:34:44.020291+0000","last_active":"2024-03-06T07:34:44.022441+0000","last_peered":"2024-03-06T07:34:44.022441+0000","last_clean":"2024-03-06T07:34:44.022441+0000","last_became_active":"2024-03-06T07:34:44.019931+0000","last_became_peered":"2024-03-06T07:34:44.019931+0000","last_unstale":"2024-03-06T07:34:44.022441+0000","last_undegraded":"2024-03-06T07:34:44.022441+0000","last_fullsized":"2024-03-06T07:34:44.022441+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:07:16.630980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.34","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028499+0000","last_change":"2024-03-06T07:34:44.032320+0000","last_active":"2024-03-06T07:34:45.028499+0000","last_peered":"2024-03-06T07:34:45.028499+0000","last_clean":"2024-03-06T07:34:45.028499+0000","last_became_active":"2024-03-06T07:34:44.031970+0000","last_became_peered":"2024-03-06T07:34:44.031970+0000","last_unstale":"2024-03-06T07:34:45.028499+0000","last_undegraded":"2024-03-06T07:34:45.028499+0000","last_fullsized":"2024-03-06T07:34:45.028499+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:49:26.666996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.33","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041782+0000","last_change":"2024-03-06T07:34:44.035648+0000","last_active":"2024-03-06T07:34:44.041782+0000","last_peered":"2024-03-06T07:34:44.041782+0000","last_clean":"2024-03-06T07:34:44.041782+0000","last_became_active":"2024-03-06T07:34:44.027272+0000","last_became_peered":"2024-03-06T07:34:44.027272+0000","last_unstale":"2024-03-06T07:34:44.041782+0000","last_undegraded":"2024-03-06T07:34:44.041782+0000","last_fullsized":"2024-03-06T07:34:44.041782+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:53:59.319732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.32","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039584+0000","last_change":"2024-03-06T07:34:44.024324+0000","last_active":"2024-03-06T07:34:44.039584+0000","last_peered":"2024-03-06T07:34:44.039584+0000","last_clean":"2024-03-06T07:34:44.039584+0000","last_became_active":"2024-03-06T07:34:44.022698+0000","last_became_peered":"2024-03-06T07:34:44.022698+0000","last_unstale":"2024-03-06T07:34:44.039584+0000","last_undegraded":"2024-03-06T07:34:44.039584+0000","last_fullsized":"2024-03-06T07:34:44.039584+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:56:58.186782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.31","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.034144+0000","last_change":"2024-03-06T07:34:44.028668+0000","last_active":"2024-03-06T07:34:44.034144+0000","last_peered":"2024-03-06T07:34:44.034144+0000","last_clean":"2024-03-06T07:34:44.034144+0000","last_became_active":"2024-03-06T07:34:44.025564+0000","last_became_peered":"2024-03-06T07:34:44.025564+0000","last_unstale":"2024-03-06T07:34:44.034144+0000","last_undegraded":"2024-03-06T07:34:44.034144+0000","last_fullsized":"2024-03-06T07:34:44.034144+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:57:45.746327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.30","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035674+0000","last_change":"2024-03-06T07:34:44.031123+0000","last_active":"2024-03-06T07:34:44.035674+0000","last_peered":"2024-03-06T07:34:44.035674+0000","last_clean":"2024-03-06T07:34:44.035674+0000","last_became_active":"2024-03-06T07:34:44.028040+0000","last_became_peered":"2024-03-06T07:34:44.028040+0000","last_unstale":"2024-03-06T07:34:44.035674+0000","last_undegraded":"2024-03-06T07:34:44.035674+0000","last_fullsized":"2024-03-06T07:34:44.035674+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:41.158147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2f","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023977+0000","last_change":"2024-03-06T07:34:44.016530+0000","last_active":"2024-03-06T07:34:44.023977+0000","last_peered":"2024-03-06T07:34:44.023977+0000","last_clean":"2024-03-06T07:34:44.023977+0000","last_became_active":"2024-03-06T07:34:44.016052+0000","last_became_peered":"2024-03-06T07:34:44.016052+0000","last_unstale":"2024-03-06T07:34:44.023977+0000","last_undegraded":"2024-03-06T07:34:44.023977+0000","last_fullsized":"2024-03-06T07:34:44.023977+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:01:26.328146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.2e","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035426+0000","last_change":"2024-03-06T07:34:44.026550+0000","last_active":"2024-03-06T07:34:44.035426+0000","last_peered":"2024-03-06T07:34:44.035426+0000","last_clean":"2024-03-06T07:34:44.035426+0000","last_became_active":"2024-03-06T07:34:44.026259+0000","last_became_peered":"2024-03-06T07:34:44.026259+0000","last_unstale":"2024-03-06T07:34:44.035426+0000","last_undegraded":"2024-03-06T07:34:44.035426+0000","last_fullsized":"2024-03-06T07:34:44.035426+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:52:58.683080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2d","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.018578+0000","last_change":"2024-03-06T07:34:44.012280+0000","last_active":"2024-03-06T07:34:44.018578+0000","last_peered":"2024-03-06T07:34:44.018578+0000","last_clean":"2024-03-06T07:34:44.018578+0000","last_became_active":"2024-03-06T07:34:44.011683+0000","last_became_peered":"2024-03-06T07:34:44.011683+0000","last_unstale":"2024-03-06T07:34:44.018578+0000","last_undegraded":"2024-03-06T07:34:44.018578+0000","last_fullsized":"2024-03-06T07:34:44.018578+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:06:19.965303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035615+0000","last_change":"2024-03-06T07:34:44.028349+0000","last_active":"2024-03-06T07:34:44.035615+0000","last_peered":"2024-03-06T07:34:44.035615+0000","last_clean":"2024-03-06T07:34:44.035615+0000","last_became_active":"2024-03-06T07:34:44.027507+0000","last_became_peered":"2024-03-06T07:34:44.027507+0000","last_unstale":"2024-03-06T07:34:44.035615+0000","last_undegraded":"2024-03-06T07:34:44.035615+0000","last_fullsized":"2024-03-06T07:34:44.035615+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:17.080362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.026281+0000","last_change":"2024-03-06T07:34:44.025230+0000","last_active":"2024-03-06T07:34:44.026281+0000","last_peered":"2024-03-06T07:34:44.026281+0000","last_clean":"2024-03-06T07:34:44.026281+0000","last_became_active":"2024-03-06T07:34:44.024796+0000","last_became_peered":"2024-03-06T07:34:44.024796+0000","last_unstale":"2024-03-06T07:34:44.026281+0000","last_undegraded":"2024-03-06T07:34:44.026281+0000","last_fullsized":"2024-03-06T07:34:44.026281+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:38:44.146167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.2a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023296+0000","last_change":"2024-03-06T07:34:44.021547+0000","last_active":"2024-03-06T07:34:44.023296+0000","last_peered":"2024-03-06T07:34:44.023296+0000","last_clean":"2024-03-06T07:34:44.023296+0000","last_became_active":"2024-03-06T07:34:44.021233+0000","last_became_peered":"2024-03-06T07:34:44.021233+0000","last_unstale":"2024-03-06T07:34:44.023296+0000","last_undegraded":"2024-03-06T07:34:44.023296+0000","last_fullsized":"2024-03-06T07:34:44.023296+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:13:48.555138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.29","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.027651+0000","last_change":"2024-03-06T07:34:44.031844+0000","last_active":"2024-03-06T07:34:45.027651+0000","last_peered":"2024-03-06T07:34:45.027651+0000","last_clean":"2024-03-06T07:34:45.027651+0000","last_became_active":"2024-03-06T07:34:44.027082+0000","last_became_peered":"2024-03-06T07:34:44.027082+0000","last_unstale":"2024-03-06T07:34:45.027651+0000","last_undegraded":"2024-03-06T07:34:45.027651+0000","last_fullsized":"2024-03-06T07:34:45.027651+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:37:23.571379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.28","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030116+0000","last_change":"2024-03-06T07:34:44.023318+0000","last_active":"2024-03-06T07:34:44.030116+0000","last_peered":"2024-03-06T07:34:44.030116+0000","last_clean":"2024-03-06T07:34:44.030116+0000","last_became_active":"2024-03-06T07:34:44.022223+0000","last_became_peered":"2024-03-06T07:34:44.022223+0000","last_unstale":"2024-03-06T07:34:44.030116+0000","last_undegraded":"2024-03-06T07:34:44.030116+0000","last_fullsized":"2024-03-06T07:34:44.030116+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:36.089536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.27","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039406+0000","last_change":"2024-03-06T07:34:44.022845+0000","last_active":"2024-03-06T07:34:44.039406+0000","last_peered":"2024-03-06T07:34:44.039406+0000","last_clean":"2024-03-06T07:34:44.039406+0000","last_became_active":"2024-03-06T07:34:44.019249+0000","last_became_peered":"2024-03-06T07:34:44.019249+0000","last_unstale":"2024-03-06T07:34:44.039406+0000","last_undegraded":"2024-03-06T07:34:44.039406+0000","last_fullsized":"2024-03-06T07:34:44.039406+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:43:43.768052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.26","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041382+0000","last_change":"2024-03-06T07:34:44.037055+0000","last_active":"2024-03-06T07:34:44.041382+0000","last_peered":"2024-03-06T07:34:44.041382+0000","last_clean":"2024-03-06T07:34:44.041382+0000","last_became_active":"2024-03-06T07:34:44.036432+0000","last_became_peered":"2024-03-06T07:34:44.036432+0000","last_unstale":"2024-03-06T07:34:44.041382+0000","last_undegraded":"2024-03-06T07:34:44.041382+0000","last_fullsized":"2024-03-06T07:34:44.041382+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:30:21.585235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.25","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035805+0000","last_change":"2024-03-06T07:34:44.029379+0000","last_active":"2024-03-06T07:34:44.035805+0000","last_peered":"2024-03-06T07:34:44.035805+0000","last_clean":"2024-03-06T07:34:44.035805+0000","last_became_active":"2024-03-06T07:34:44.029116+0000","last_became_peered":"2024-03-06T07:34:44.029116+0000","last_unstale":"2024-03-06T07:34:44.035805+0000","last_undegraded":"2024-03-06T07:34:44.035805+0000","last_fullsized":"2024-03-06T07:34:44.035805+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:12.445889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.24","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033542+0000","last_change":"2024-03-06T07:34:44.030753+0000","last_active":"2024-03-06T07:34:44.033542+0000","last_peered":"2024-03-06T07:34:44.033542+0000","last_clean":"2024-03-06T07:34:44.033542+0000","last_became_active":"2024-03-06T07:34:44.030360+0000","last_became_peered":"2024-03-06T07:34:44.030360+0000","last_unstale":"2024-03-06T07:34:44.033542+0000","last_undegraded":"2024-03-06T07:34:44.033542+0000","last_fullsized":"2024-03-06T07:34:44.033542+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:47:39.121616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.23","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.038202+0000","last_change":"2024-03-06T07:34:44.033970+0000","last_active":"2024-03-06T07:34:44.038202+0000","last_peered":"2024-03-06T07:34:44.038202+0000","last_clean":"2024-03-06T07:34:44.038202+0000","last_became_active":"2024-03-06T07:34:44.026804+0000","last_became_peered":"2024-03-06T07:34:44.026804+0000","last_unstale":"2024-03-06T07:34:44.038202+0000","last_undegraded":"2024-03-06T07:34:44.038202+0000","last_fullsized":"2024-03-06T07:34:44.038202+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:33:00.559490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.22","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041605+0000","last_change":"2024-03-06T07:34:44.034657+0000","last_active":"2024-03-06T07:34:44.041605+0000","last_peered":"2024-03-06T07:34:44.041605+0000","last_clean":"2024-03-06T07:34:44.041605+0000","last_became_active":"2024-03-06T07:34:44.033275+0000","last_became_peered":"2024-03-06T07:34:44.033275+0000","last_unstale":"2024-03-06T07:34:44.041605+0000","last_undegraded":"2024-03-06T07:34:44.041605+0000","last_fullsized":"2024-03-06T07:34:44.041605+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:41:03.630720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.21","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023848+0000","last_change":"2024-03-06T07:34:44.021116+0000","last_active":"2024-03-06T07:34:44.023848+0000","last_peered":"2024-03-06T07:34:44.023848+0000","last_clean":"2024-03-06T07:34:44.023848+0000","last_became_active":"2024-03-06T07:34:44.020866+0000","last_became_peered":"2024-03-06T07:34:44.020866+0000","last_unstale":"2024-03-06T07:34:44.023848+0000","last_undegraded":"2024-03-06T07:34:44.023848+0000","last_fullsized":"2024-03-06T07:34:44.023848+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:01:26.328146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.20","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033461+0000","last_change":"2024-03-06T07:34:44.027612+0000","last_active":"2024-03-06T07:34:44.033461+0000","last_peered":"2024-03-06T07:34:44.033461+0000","last_clean":"2024-03-06T07:34:44.033461+0000","last_became_active":"2024-03-06T07:34:44.025754+0000","last_became_peered":"2024-03-06T07:34:44.025754+0000","last_unstale":"2024-03-06T07:34:44.033461+0000","last_undegraded":"2024-03-06T07:34:44.033461+0000","last_fullsized":"2024-03-06T07:34:44.033461+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:23.688678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.1e","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035575+0000","last_change":"2024-03-06T07:34:44.030083+0000","last_active":"2024-03-06T07:34:44.035575+0000","last_peered":"2024-03-06T07:34:44.035575+0000","last_clean":"2024-03-06T07:34:44.035575+0000","last_became_active":"2024-03-06T07:34:44.029812+0000","last_became_peered":"2024-03-06T07:34:44.029812+0000","last_unstale":"2024-03-06T07:34:44.035575+0000","last_undegraded":"2024-03-06T07:34:44.035575+0000","last_fullsized":"2024-03-06T07:34:44.035575+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:03:39.406427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1f","version":"126'4","reported_seq":16,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.044403+0000","last_change":"2024-03-06T07:34:43.027908+0000","last_active":"2024-03-06T07:34:44.044403+0000","last_peered":"2024-03-06T07:34:44.044403+0000","last_clean":"2024-03-06T07:34:44.044403+0000","last_became_active":"2024-03-06T07:34:43.027598+0000","last_became_peered":"2024-03-06T07:34:43.027598+0000","last_unstale":"2024-03-06T07:34:44.044403+0000","last_undegraded":"2024-03-06T07:34:44.044403+0000","last_fullsized":"2024-03-06T07:34:44.044403+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:19:11.087872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.1f","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028764+0000","last_change":"2024-03-06T07:34:44.028470+0000","last_active":"2024-03-06T07:34:45.028764+0000","last_peered":"2024-03-06T07:34:45.028764+0000","last_clean":"2024-03-06T07:34:45.028764+0000","last_became_active":"2024-03-06T07:34:44.028174+0000","last_became_peered":"2024-03-06T07:34:44.028174+0000","last_unstale":"2024-03-06T07:34:45.028764+0000","last_undegraded":"2024-03-06T07:34:45.028764+0000","last_fullsized":"2024-03-06T07:34:45.028764+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:41:09.887782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.1e","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041547+0000","last_change":"2024-03-06T07:34:43.014220+0000","last_active":"2024-03-06T07:34:44.041547+0000","last_peered":"2024-03-06T07:34:44.041547+0000","last_clean":"2024-03-06T07:34:44.041547+0000","last_became_active":"2024-03-06T07:34:43.013789+0000","last_became_peered":"2024-03-06T07:34:43.013789+0000","last_unstale":"2024-03-06T07:34:44.041547+0000","last_undegraded":"2024-03-06T07:34:44.041547+0000","last_fullsized":"2024-03-06T07:34:44.041547+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:28.402445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.1c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036961+0000","last_change":"2024-03-06T07:34:44.034445+0000","last_active":"2024-03-06T07:34:44.036961+0000","last_peered":"2024-03-06T07:34:44.036961+0000","last_clean":"2024-03-06T07:34:44.036961+0000","last_became_active":"2024-03-06T07:34:44.031375+0000","last_became_peered":"2024-03-06T07:34:44.031375+0000","last_unstale":"2024-03-06T07:34:44.036961+0000","last_undegraded":"2024-03-06T07:34:44.036961+0000","last_fullsized":"2024-03-06T07:34:44.036961+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:10.782135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1d","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043443+0000","last_change":"2024-03-06T07:34:43.013750+0000","last_active":"2024-03-06T07:34:44.043443+0000","last_peered":"2024-03-06T07:34:44.043443+0000","last_clean":"2024-03-06T07:34:44.043443+0000","last_became_active":"2024-03-06T07:34:43.013462+0000","last_became_peered":"2024-03-06T07:34:43.013462+0000","last_unstale":"2024-03-06T07:34:44.043443+0000","last_undegraded":"2024-03-06T07:34:44.043443+0000","last_fullsized":"2024-03-06T07:34:44.043443+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:00:03.977004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.1d","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041407+0000","last_change":"2024-03-06T07:34:44.040486+0000","last_active":"2024-03-06T07:34:44.041407+0000","last_peered":"2024-03-06T07:34:44.041407+0000","last_clean":"2024-03-06T07:34:44.041407+0000","last_became_active":"2024-03-06T07:34:44.040148+0000","last_became_peered":"2024-03-06T07:34:44.040148+0000","last_unstale":"2024-03-06T07:34:44.041407+0000","last_undegraded":"2024-03-06T07:34:44.041407+0000","last_fullsized":"2024-03-06T07:34:44.041407+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:01:19.169150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1c","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043669+0000","last_change":"2024-03-06T07:34:43.012887+0000","last_active":"2024-03-06T07:34:44.043669+0000","last_peered":"2024-03-06T07:34:44.043669+0000","last_clean":"2024-03-06T07:34:44.043669+0000","last_became_active":"2024-03-06T07:34:43.012621+0000","last_became_peered":"2024-03-06T07:34:43.012621+0000","last_unstale":"2024-03-06T07:34:44.043669+0000","last_undegraded":"2024-03-06T07:34:44.043669+0000","last_fullsized":"2024-03-06T07:34:44.043669+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:17:57.709369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.040478+0000","last_change":"2024-03-06T07:34:44.032328+0000","last_active":"2024-03-06T07:34:44.040478+0000","last_peered":"2024-03-06T07:34:44.040478+0000","last_clean":"2024-03-06T07:34:44.040478+0000","last_became_active":"2024-03-06T07:34:44.032062+0000","last_became_peered":"2024-03-06T07:34:44.032062+0000","last_unstale":"2024-03-06T07:34:44.040478+0000","last_undegraded":"2024-03-06T07:34:44.040478+0000","last_fullsized":"2024-03-06T07:34:44.040478+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:33:52.163428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1b","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041555+0000","last_change":"2024-03-06T07:34:43.011300+0000","last_active":"2024-03-06T07:34:44.041555+0000","last_peered":"2024-03-06T07:34:44.041555+0000","last_clean":"2024-03-06T07:34:44.041555+0000","last_became_active":"2024-03-06T07:34:43.010209+0000","last_became_peered":"2024-03-06T07:34:43.010209+0000","last_unstale":"2024-03-06T07:34:44.041555+0000","last_undegraded":"2024-03-06T07:34:44.041555+0000","last_fullsized":"2024-03-06T07:34:44.041555+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:45:21.882196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023325+0000","last_change":"2024-03-06T07:34:44.016896+0000","last_active":"2024-03-06T07:34:44.023325+0000","last_peered":"2024-03-06T07:34:44.023325+0000","last_clean":"2024-03-06T07:34:44.023325+0000","last_became_active":"2024-03-06T07:34:44.016484+0000","last_became_peered":"2024-03-06T07:34:44.016484+0000","last_unstale":"2024-03-06T07:34:44.023325+0000","last_undegraded":"2024-03-06T07:34:44.023325+0000","last_fullsized":"2024-03-06T07:34:44.023325+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:38:44.146167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.1a","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.040506+0000","last_change":"2024-03-06T07:34:43.019384+0000","last_active":"2024-03-06T07:34:44.040506+0000","last_peered":"2024-03-06T07:34:44.040506+0000","last_clean":"2024-03-06T07:34:44.040506+0000","last_became_active":"2024-03-06T07:34:43.017323+0000","last_became_peered":"2024-03-06T07:34:43.017323+0000","last_unstale":"2024-03-06T07:34:44.040506+0000","last_undegraded":"2024-03-06T07:34:44.040506+0000","last_fullsized":"2024-03-06T07:34:44.040506+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:20:56.487188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.18","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028157+0000","last_change":"2024-03-06T07:34:44.029535+0000","last_active":"2024-03-06T07:34:45.028157+0000","last_peered":"2024-03-06T07:34:45.028157+0000","last_clean":"2024-03-06T07:34:45.028157+0000","last_became_active":"2024-03-06T07:34:44.028800+0000","last_became_peered":"2024-03-06T07:34:44.028800+0000","last_unstale":"2024-03-06T07:34:45.028157+0000","last_undegraded":"2024-03-06T07:34:45.028157+0000","last_fullsized":"2024-03-06T07:34:45.028157+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:24:06.945371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.19","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.998925+0000","last_change":"2024-03-06T07:34:43.024474+0000","last_active":"2024-03-06T07:34:43.998925+0000","last_peered":"2024-03-06T07:34:43.998925+0000","last_clean":"2024-03-06T07:34:43.998925+0000","last_became_active":"2024-03-06T07:34:43.024161+0000","last_became_peered":"2024-03-06T07:34:43.024161+0000","last_unstale":"2024-03-06T07:34:43.998925+0000","last_undegraded":"2024-03-06T07:34:43.998925+0000","last_fullsized":"2024-03-06T07:34:43.998925+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:29:59.318159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.19","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039519+0000","last_change":"2024-03-06T07:34:44.025122+0000","last_active":"2024-03-06T07:34:44.039519+0000","last_peered":"2024-03-06T07:34:44.039519+0000","last_clean":"2024-03-06T07:34:44.039519+0000","last_became_active":"2024-03-06T07:34:44.024425+0000","last_became_peered":"2024-03-06T07:34:44.024425+0000","last_unstale":"2024-03-06T07:34:44.039519+0000","last_undegraded":"2024-03-06T07:34:44.039519+0000","last_fullsized":"2024-03-06T07:34:44.039519+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:45:23.693815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.18","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.008373+0000","last_change":"2024-03-06T07:34:43.017180+0000","last_active":"2024-03-06T07:34:44.008373+0000","last_peered":"2024-03-06T07:34:44.008373+0000","last_clean":"2024-03-06T07:34:44.008373+0000","last_became_active":"2024-03-06T07:34:43.016896+0000","last_became_peered":"2024-03-06T07:34:43.016896+0000","last_unstale":"2024-03-06T07:34:44.008373+0000","last_undegraded":"2024-03-06T07:34:44.008373+0000","last_fullsized":"2024-03-06T07:34:44.008373+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:54:37.437177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.16","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.037894+0000","last_change":"2024-03-06T07:34:44.033645+0000","last_active":"2024-03-06T07:34:44.037894+0000","last_peered":"2024-03-06T07:34:44.037894+0000","last_clean":"2024-03-06T07:34:44.037894+0000","last_became_active":"2024-03-06T07:34:44.026172+0000","last_became_peered":"2024-03-06T07:34:44.026172+0000","last_unstale":"2024-03-06T07:34:44.037894+0000","last_undegraded":"2024-03-06T07:34:44.037894+0000","last_fullsized":"2024-03-06T07:34:44.037894+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:47:57.389178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.17","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.017365+0000","last_change":"2024-03-06T07:34:43.009102+0000","last_active":"2024-03-06T07:34:44.017365+0000","last_peered":"2024-03-06T07:34:44.017365+0000","last_clean":"2024-03-06T07:34:44.017365+0000","last_became_active":"2024-03-06T07:34:43.008057+0000","last_became_peered":"2024-03-06T07:34:43.008057+0000","last_unstale":"2024-03-06T07:34:44.017365+0000","last_undegraded":"2024-03-06T07:34:44.017365+0000","last_fullsized":"2024-03-06T07:34:44.017365+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:29:26.160005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.17","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.032833+0000","last_change":"2024-03-06T07:34:44.028355+0000","last_active":"2024-03-06T07:34:44.032833+0000","last_peered":"2024-03-06T07:34:44.032833+0000","last_clean":"2024-03-06T07:34:44.032833+0000","last_became_active":"2024-03-06T07:34:44.028035+0000","last_became_peered":"2024-03-06T07:34:44.028035+0000","last_unstale":"2024-03-06T07:34:44.032833+0000","last_undegraded":"2024-03-06T07:34:44.032833+0000","last_fullsized":"2024-03-06T07:34:44.032833+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:54:17.379795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.16","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.017050+0000","last_change":"2024-03-06T07:34:43.018858+0000","last_active":"2024-03-06T07:34:44.017050+0000","last_peered":"2024-03-06T07:34:44.017050+0000","last_clean":"2024-03-06T07:34:44.017050+0000","last_became_active":"2024-03-06T07:34:43.018590+0000","last_became_peered":"2024-03-06T07:34:43.018590+0000","last_unstale":"2024-03-06T07:34:44.017050+0000","last_undegraded":"2024-03-06T07:34:44.017050+0000","last_fullsized":"2024-03-06T07:34:44.017050+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:43:00.056746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.14","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036940+0000","last_change":"2024-03-06T07:34:44.034274+0000","last_active":"2024-03-06T07:34:44.036940+0000","last_peered":"2024-03-06T07:34:44.036940+0000","last_clean":"2024-03-06T07:34:44.036940+0000","last_became_active":"2024-03-06T07:34:44.033601+0000","last_became_peered":"2024-03-06T07:34:44.033601+0000","last_unstale":"2024-03-06T07:34:44.036940+0000","last_undegraded":"2024-03-06T07:34:44.036940+0000","last_fullsized":"2024-03-06T07:34:44.036940+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:25:59.779783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.15","version":"126'3","reported_seq":15,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043648+0000","last_change":"2024-03-06T07:34:43.013971+0000","last_active":"2024-03-06T07:34:44.043648+0000","last_peered":"2024-03-06T07:34:44.043648+0000","last_clean":"2024-03-06T07:34:44.043648+0000","last_became_active":"2024-03-06T07:34:43.013498+0000","last_became_peered":"2024-03-06T07:34:43.013498+0000","last_unstale":"2024-03-06T07:34:44.043648+0000","last_undegraded":"2024-03-06T07:34:44.043648+0000","last_fullsized":"2024-03-06T07:34:44.043648+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:18:01.619257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.15","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036043+0000","last_change":"2024-03-06T07:34:44.032330+0000","last_active":"2024-03-06T07:34:44.036043+0000","last_peered":"2024-03-06T07:34:44.036043+0000","last_clean":"2024-03-06T07:34:44.036043+0000","last_became_active":"2024-03-06T07:34:44.031526+0000","last_became_peered":"2024-03-06T07:34:44.031526+0000","last_unstale":"2024-03-06T07:34:44.036043+0000","last_undegraded":"2024-03-06T07:34:44.036043+0000","last_fullsized":"2024-03-06T07:34:44.036043+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:26:11.944719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.14","version":"126'6","reported_seq":20,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.027730+0000","last_change":"2024-03-06T07:34:43.023234+0000","last_active":"2024-03-06T07:34:45.027730+0000","last_peered":"2024-03-06T07:34:45.027730+0000","last_clean":"2024-03-06T07:34:45.027730+0000","last_became_active":"2024-03-06T07:34:43.022878+0000","last_became_peered":"2024-03-06T07:34:43.022878+0000","last_unstale":"2024-03-06T07:34:45.027730+0000","last_undegraded":"2024-03-06T07:34:45.027730+0000","last_fullsized":"2024-03-06T07:34:45.027730+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:27:07.066055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.13","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.024957+0000","last_change":"2024-03-06T07:34:44.020973+0000","last_active":"2024-03-06T07:34:44.024957+0000","last_peered":"2024-03-06T07:34:44.024957+0000","last_clean":"2024-03-06T07:34:44.024957+0000","last_became_active":"2024-03-06T07:34:44.020626+0000","last_became_peered":"2024-03-06T07:34:44.020626+0000","last_unstale":"2024-03-06T07:34:44.024957+0000","last_undegraded":"2024-03-06T07:34:44.024957+0000","last_fullsized":"2024-03-06T07:34:44.024957+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:13:57.055771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.12","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.009509+0000","last_change":"2024-03-06T07:34:43.015723+0000","last_active":"2024-03-06T07:34:44.009509+0000","last_peered":"2024-03-06T07:34:44.009509+0000","last_clean":"2024-03-06T07:34:44.009509+0000","last_became_active":"2024-03-06T07:34:43.015412+0000","last_became_peered":"2024-03-06T07:34:43.015412+0000","last_unstale":"2024-03-06T07:34:44.009509+0000","last_undegraded":"2024-03-06T07:34:44.009509+0000","last_fullsized":"2024-03-06T07:34:44.009509+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:36:59.942128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.10","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039707+0000","last_change":"2024-03-06T07:34:44.026778+0000","last_active":"2024-03-06T07:34:44.039707+0000","last_peered":"2024-03-06T07:34:44.039707+0000","last_clean":"2024-03-06T07:34:44.039707+0000","last_became_active":"2024-03-06T07:34:44.026494+0000","last_became_peered":"2024-03-06T07:34:44.026494+0000","last_unstale":"2024-03-06T07:34:44.039707+0000","last_undegraded":"2024-03-06T07:34:44.039707+0000","last_fullsized":"2024-03-06T07:34:44.039707+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:03:07.048232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.11","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.011130+0000","last_change":"2024-03-06T07:34:43.022773+0000","last_active":"2024-03-06T07:34:44.011130+0000","last_peered":"2024-03-06T07:34:44.011130+0000","last_clean":"2024-03-06T07:34:44.011130+0000","last_became_active":"2024-03-06T07:34:43.022426+0000","last_became_peered":"2024-03-06T07:34:43.022426+0000","last_unstale":"2024-03-06T07:34:44.011130+0000","last_undegraded":"2024-03-06T07:34:44.011130+0000","last_fullsized":"2024-03-06T07:34:44.011130+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:20:15.694812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.11","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.027658+0000","last_change":"2024-03-06T07:34:44.024381+0000","last_active":"2024-03-06T07:34:45.027658+0000","last_peered":"2024-03-06T07:34:45.027658+0000","last_clean":"2024-03-06T07:34:45.027658+0000","last_became_active":"2024-03-06T07:34:44.024094+0000","last_became_peered":"2024-03-06T07:34:44.024094+0000","last_unstale":"2024-03-06T07:34:45.027658+0000","last_undegraded":"2024-03-06T07:34:45.027658+0000","last_fullsized":"2024-03-06T07:34:45.027658+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:32:43.917666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.10","version":"126'8","reported_seq":22,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028418+0000","last_change":"2024-03-06T07:34:43.016707+0000","last_active":"2024-03-06T07:34:45.028418+0000","last_peered":"2024-03-06T07:34:45.028418+0000","last_clean":"2024-03-06T07:34:45.028418+0000","last_became_active":"2024-03-06T07:34:43.016205+0000","last_became_peered":"2024-03-06T07:34:43.016205+0000","last_unstale":"2024-03-06T07:34:45.028418+0000","last_undegraded":"2024-03-06T07:34:45.028418+0000","last_fullsized":"2024-03-06T07:34:45.028418+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:31:15.042351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.e","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028622+0000","last_change":"2024-03-06T07:34:44.032521+0000","last_active":"2024-03-06T07:34:45.028622+0000","last_peered":"2024-03-06T07:34:45.028622+0000","last_clean":"2024-03-06T07:34:45.028622+0000","last_became_active":"2024-03-06T07:34:44.031853+0000","last_became_peered":"2024-03-06T07:34:44.031853+0000","last_unstale":"2024-03-06T07:34:45.028622+0000","last_undegraded":"2024-03-06T07:34:45.028622+0000","last_fullsized":"2024-03-06T07:34:45.028622+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:42:18.108074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.f","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.042087+0000","last_change":"2024-03-06T07:34:43.020026+0000","last_active":"2024-03-06T07:34:44.042087+0000","last_peered":"2024-03-06T07:34:44.042087+0000","last_clean":"2024-03-06T07:34:44.042087+0000","last_became_active":"2024-03-06T07:34:43.018358+0000","last_became_peered":"2024-03-06T07:34:43.018358+0000","last_unstale":"2024-03-06T07:34:44.042087+0000","last_undegraded":"2024-03-06T07:34:44.042087+0000","last_fullsized":"2024-03-06T07:34:44.042087+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:33:14.485031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.f","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030109+0000","last_change":"2024-03-06T07:34:44.024205+0000","last_active":"2024-03-06T07:34:44.030109+0000","last_peered":"2024-03-06T07:34:44.030109+0000","last_clean":"2024-03-06T07:34:44.030109+0000","last_became_active":"2024-03-06T07:34:44.023586+0000","last_became_peered":"2024-03-06T07:34:44.023586+0000","last_unstale":"2024-03-06T07:34:44.030109+0000","last_undegraded":"2024-03-06T07:34:44.030109+0000","last_fullsized":"2024-03-06T07:34:44.030109+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:18:31.327765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.e","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.042959+0000","last_change":"2024-03-06T07:34:43.021714+0000","last_active":"2024-03-06T07:34:44.042959+0000","last_peered":"2024-03-06T07:34:44.042959+0000","last_clean":"2024-03-06T07:34:44.042959+0000","last_became_active":"2024-03-06T07:34:43.021362+0000","last_became_peered":"2024-03-06T07:34:43.021362+0000","last_unstale":"2024-03-06T07:34:44.042959+0000","last_undegraded":"2024-03-06T07:34:44.042959+0000","last_fullsized":"2024-03-06T07:34:44.042959+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:29:36.165337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035635+0000","last_change":"2024-03-06T07:34:44.028636+0000","last_active":"2024-03-06T07:34:44.035635+0000","last_peered":"2024-03-06T07:34:44.035635+0000","last_clean":"2024-03-06T07:34:44.035635+0000","last_became_active":"2024-03-06T07:34:44.027948+0000","last_became_peered":"2024-03-06T07:34:44.027948+0000","last_unstale":"2024-03-06T07:34:44.035635+0000","last_undegraded":"2024-03-06T07:34:44.035635+0000","last_fullsized":"2024-03-06T07:34:44.035635+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:16.286843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.d","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.044889+0000","last_change":"2024-03-06T07:34:43.015685+0000","last_active":"2024-03-06T07:34:44.044889+0000","last_peered":"2024-03-06T07:34:44.044889+0000","last_clean":"2024-03-06T07:34:44.044889+0000","last_became_active":"2024-03-06T07:34:43.015355+0000","last_became_peered":"2024-03-06T07:34:43.015355+0000","last_unstale":"2024-03-06T07:34:44.044889+0000","last_undegraded":"2024-03-06T07:34:44.044889+0000","last_fullsized":"2024-03-06T07:34:44.044889+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:51:50.826948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.d","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039463+0000","last_change":"2024-03-06T07:34:44.023479+0000","last_active":"2024-03-06T07:34:44.039463+0000","last_peered":"2024-03-06T07:34:44.039463+0000","last_clean":"2024-03-06T07:34:44.039463+0000","last_became_active":"2024-03-06T07:34:44.022456+0000","last_became_peered":"2024-03-06T07:34:44.022456+0000","last_unstale":"2024-03-06T07:34:44.039463+0000","last_undegraded":"2024-03-06T07:34:44.039463+0000","last_fullsized":"2024-03-06T07:34:44.039463+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:06:44.152597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.c","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.017287+0000","last_change":"2024-03-06T07:34:43.017624+0000","last_active":"2024-03-06T07:34:44.017287+0000","last_peered":"2024-03-06T07:34:44.017287+0000","last_clean":"2024-03-06T07:34:44.017287+0000","last_became_active":"2024-03-06T07:34:43.014807+0000","last_became_peered":"2024-03-06T07:34:43.014807+0000","last_unstale":"2024-03-06T07:34:44.017287+0000","last_undegraded":"2024-03-06T07:34:44.017287+0000","last_fullsized":"2024-03-06T07:34:44.017287+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:55.305546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.038651+0000","last_change":"2024-03-06T07:34:44.033797+0000","last_active":"2024-03-06T07:34:44.038651+0000","last_peered":"2024-03-06T07:34:44.038651+0000","last_clean":"2024-03-06T07:34:44.038651+0000","last_became_active":"2024-03-06T07:34:44.025891+0000","last_became_peered":"2024-03-06T07:34:44.025891+0000","last_unstale":"2024-03-06T07:34:44.038651+0000","last_undegraded":"2024-03-06T07:34:44.038651+0000","last_fullsized":"2024-03-06T07:34:44.038651+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:13:22.511687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.b","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043762+0000","last_change":"2024-03-06T07:34:43.018128+0000","last_active":"2024-03-06T07:34:44.043762+0000","last_peered":"2024-03-06T07:34:44.043762+0000","last_clean":"2024-03-06T07:34:44.043762+0000","last_became_active":"2024-03-06T07:34:43.010332+0000","last_became_peered":"2024-03-06T07:34:43.010332+0000","last_unstale":"2024-03-06T07:34:44.043762+0000","last_undegraded":"2024-03-06T07:34:44.043762+0000","last_fullsized":"2024-03-06T07:34:44.043762+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:56:08.846529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033699+0000","last_change":"2024-03-06T07:34:44.026526+0000","last_active":"2024-03-06T07:34:44.033699+0000","last_peered":"2024-03-06T07:34:44.033699+0000","last_clean":"2024-03-06T07:34:44.033699+0000","last_became_active":"2024-03-06T07:34:44.026197+0000","last_became_peered":"2024-03-06T07:34:44.026197+0000","last_unstale":"2024-03-06T07:34:44.033699+0000","last_undegraded":"2024-03-06T07:34:44.033699+0000","last_fullsized":"2024-03-06T07:34:44.033699+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:25:59.779783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.a","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.016555+0000","last_change":"2024-03-06T07:34:43.020405+0000","last_active":"2024-03-06T07:34:44.016555+0000","last_peered":"2024-03-06T07:34:44.016555+0000","last_clean":"2024-03-06T07:34:44.016555+0000","last_became_active":"2024-03-06T07:34:43.020136+0000","last_became_peered":"2024-03-06T07:34:43.020136+0000","last_unstale":"2024-03-06T07:34:44.016555+0000","last_undegraded":"2024-03-06T07:34:44.016555+0000","last_fullsized":"2024-03-06T07:34:44.016555+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:44:09.577857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.8","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.020280+0000","last_change":"2024-03-06T07:34:44.012720+0000","last_active":"2024-03-06T07:34:44.020280+0000","last_peered":"2024-03-06T07:34:44.020280+0000","last_clean":"2024-03-06T07:34:44.020280+0000","last_became_active":"2024-03-06T07:34:44.012174+0000","last_became_peered":"2024-03-06T07:34:44.012174+0000","last_unstale":"2024-03-06T07:34:44.020280+0000","last_undegraded":"2024-03-06T07:34:44.020280+0000","last_fullsized":"2024-03-06T07:34:44.020280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:43:41.376718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.9","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.015787+0000","last_change":"2024-03-06T07:34:43.020805+0000","last_active":"2024-03-06T07:34:44.015787+0000","last_peered":"2024-03-06T07:34:44.015787+0000","last_clean":"2024-03-06T07:34:44.015787+0000","last_became_active":"2024-03-06T07:34:43.020389+0000","last_became_peered":"2024-03-06T07:34:43.020389+0000","last_unstale":"2024-03-06T07:34:44.015787+0000","last_undegraded":"2024-03-06T07:34:44.015787+0000","last_fullsized":"2024-03-06T07:34:44.015787+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:35:32.564536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.9","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030661+0000","last_change":"2024-03-06T07:34:44.027454+0000","last_active":"2024-03-06T07:34:44.030661+0000","last_peered":"2024-03-06T07:34:44.030661+0000","last_clean":"2024-03-06T07:34:44.030661+0000","last_became_active":"2024-03-06T07:34:44.027125+0000","last_became_peered":"2024-03-06T07:34:44.027125+0000","last_unstale":"2024-03-06T07:34:44.030661+0000","last_undegraded":"2024-03-06T07:34:44.030661+0000","last_fullsized":"2024-03-06T07:34:44.030661+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:23.962585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.8","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.995584+0000","last_change":"2024-03-06T07:34:43.019990+0000","last_active":"2024-03-06T07:34:43.995584+0000","last_peered":"2024-03-06T07:34:43.995584+0000","last_clean":"2024-03-06T07:34:43.995584+0000","last_became_active":"2024-03-06T07:34:43.019716+0000","last_became_peered":"2024-03-06T07:34:43.019716+0000","last_unstale":"2024-03-06T07:34:43.995584+0000","last_undegraded":"2024-03-06T07:34:43.995584+0000","last_fullsized":"2024-03-06T07:34:43.995584+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:44:31.128029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.6","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030786+0000","last_change":"2024-03-06T07:34:44.025077+0000","last_active":"2024-03-06T07:34:44.030786+0000","last_peered":"2024-03-06T07:34:44.030786+0000","last_clean":"2024-03-06T07:34:44.030786+0000","last_became_active":"2024-03-06T07:34:44.024802+0000","last_became_peered":"2024-03-06T07:34:44.024802+0000","last_unstale":"2024-03-06T07:34:44.030786+0000","last_undegraded":"2024-03-06T07:34:44.030786+0000","last_fullsized":"2024-03-06T07:34:44.030786+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:26:11.944719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.7","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043173+0000","last_change":"2024-03-06T07:34:43.007731+0000","last_active":"2024-03-06T07:34:44.043173+0000","last_peered":"2024-03-06T07:34:44.043173+0000","last_clean":"2024-03-06T07:34:44.043173+0000","last_became_active":"2024-03-06T07:34:43.007354+0000","last_became_peered":"2024-03-06T07:34:43.007354+0000","last_unstale":"2024-03-06T07:34:44.043173+0000","last_undegraded":"2024-03-06T07:34:44.043173+0000","last_fullsized":"2024-03-06T07:34:44.043173+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:33:00.437444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.7","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041726+0000","last_change":"2024-03-06T07:34:44.035217+0000","last_active":"2024-03-06T07:34:44.041726+0000","last_peered":"2024-03-06T07:34:44.041726+0000","last_clean":"2024-03-06T07:34:44.041726+0000","last_became_active":"2024-03-06T07:34:44.032137+0000","last_became_peered":"2024-03-06T07:34:44.032137+0000","last_unstale":"2024-03-06T07:34:44.041726+0000","last_undegraded":"2024-03-06T07:34:44.041726+0000","last_fullsized":"2024-03-06T07:34:44.041726+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:31:00.344506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.6","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.019874+0000","last_change":"2024-03-06T07:34:43.026213+0000","last_active":"2024-03-06T07:34:44.019874+0000","last_peered":"2024-03-06T07:34:44.019874+0000","last_clean":"2024-03-06T07:34:44.019874+0000","last_became_active":"2024-03-06T07:34:43.025963+0000","last_became_peered":"2024-03-06T07:34:43.025963+0000","last_unstale":"2024-03-06T07:34:44.019874+0000","last_undegraded":"2024-03-06T07:34:44.019874+0000","last_fullsized":"2024-03-06T07:34:44.019874+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:29:59.318159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.4","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039659+0000","last_change":"2024-03-06T07:34:44.024167+0000","last_active":"2024-03-06T07:34:44.039659+0000","last_peered":"2024-03-06T07:34:44.039659+0000","last_clean":"2024-03-06T07:34:44.039659+0000","last_became_active":"2024-03-06T07:34:44.023851+0000","last_became_peered":"2024-03-06T07:34:44.023851+0000","last_unstale":"2024-03-06T07:34:44.039659+0000","last_undegraded":"2024-03-06T07:34:44.039659+0000","last_fullsized":"2024-03-06T07:34:44.039659+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:49.459190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.5","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.009623+0000","last_change":"2024-03-06T07:34:43.017104+0000","last_active":"2024-03-06T07:34:44.009623+0000","last_peered":"2024-03-06T07:34:44.009623+0000","last_clean":"2024-03-06T07:34:44.009623+0000","last_became_active":"2024-03-06T07:34:43.016821+0000","last_became_peered":"2024-03-06T07:34:43.016821+0000","last_unstale":"2024-03-06T07:34:44.009623+0000","last_undegraded":"2024-03-06T07:34:44.009623+0000","last_fullsized":"2024-03-06T07:34:44.009623+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:14:52.827203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.5","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.032246+0000","last_change":"2024-03-06T07:34:44.024358+0000","last_active":"2024-03-06T07:34:44.032246+0000","last_peered":"2024-03-06T07:34:44.032246+0000","last_clean":"2024-03-06T07:34:44.032246+0000","last_became_active":"2024-03-06T07:34:44.024057+0000","last_became_peered":"2024-03-06T07:34:44.024057+0000","last_unstale":"2024-03-06T07:34:44.032246+0000","last_undegraded":"2024-03-06T07:34:44.032246+0000","last_fullsized":"2024-03-06T07:34:44.032246+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:17:38.038578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.4","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.996257+0000","last_change":"2024-03-06T07:34:43.016147+0000","last_active":"2024-03-06T07:34:43.996257+0000","last_peered":"2024-03-06T07:34:43.996257+0000","last_clean":"2024-03-06T07:34:43.996257+0000","last_became_active":"2024-03-06T07:34:43.015638+0000","last_became_peered":"2024-03-06T07:34:43.015638+0000","last_unstale":"2024-03-06T07:34:43.996257+0000","last_undegraded":"2024-03-06T07:34:43.996257+0000","last_fullsized":"2024-03-06T07:34:43.996257+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:35:47.937364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036989+0000","last_change":"2024-03-06T07:34:44.032338+0000","last_active":"2024-03-06T07:34:44.036989+0000","last_peered":"2024-03-06T07:34:44.036989+0000","last_clean":"2024-03-06T07:34:44.036989+0000","last_became_active":"2024-03-06T07:34:44.031240+0000","last_became_peered":"2024-03-06T07:34:44.031240+0000","last_unstale":"2024-03-06T07:34:44.036989+0000","last_undegraded":"2024-03-06T07:34:44.036989+0000","last_fullsized":"2024-03-06T07:34:44.036989+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:23.688678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.3","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.010543+0000","last_change":"2024-03-06T07:34:43.019136+0000","last_active":"2024-03-06T07:34:44.010543+0000","last_peered":"2024-03-06T07:34:44.010543+0000","last_clean":"2024-03-06T07:34:44.010543+0000","last_became_active":"2024-03-06T07:34:43.018780+0000","last_became_peered":"2024-03-06T07:34:43.018780+0000","last_unstale":"2024-03-06T07:34:44.010543+0000","last_undegraded":"2024-03-06T07:34:44.010543+0000","last_fullsized":"2024-03-06T07:34:44.010543+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:04:04.884683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.032815+0000","last_change":"2024-03-06T07:34:44.029923+0000","last_active":"2024-03-06T07:34:44.032815+0000","last_peered":"2024-03-06T07:34:44.032815+0000","last_clean":"2024-03-06T07:34:44.032815+0000","last_became_active":"2024-03-06T07:34:44.028803+0000","last_became_peered":"2024-03-06T07:34:44.028803+0000","last_unstale":"2024-03-06T07:34:44.032815+0000","last_undegraded":"2024-03-06T07:34:44.032815+0000","last_fullsized":"2024-03-06T07:34:44.032815+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:43.352719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.2","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.018462+0000","last_change":"2024-03-06T07:34:43.028247+0000","last_active":"2024-03-06T07:34:44.018462+0000","last_peered":"2024-03-06T07:34:44.018462+0000","last_clean":"2024-03-06T07:34:44.018462+0000","last_became_active":"2024-03-06T07:34:43.027967+0000","last_became_peered":"2024-03-06T07:34:43.027967+0000","last_unstale":"2024-03-06T07:34:44.018462+0000","last_undegraded":"2024-03-06T07:34:44.018462+0000","last_fullsized":"2024-03-06T07:34:44.018462+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:47.838006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.0","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039070+0000","last_change":"2024-03-06T07:34:44.032660+0000","last_active":"2024-03-06T07:34:44.039070+0000","last_peered":"2024-03-06T07:34:44.039070+0000","last_clean":"2024-03-06T07:34:44.039070+0000","last_became_active":"2024-03-06T07:34:44.027677+0000","last_became_peered":"2024-03-06T07:34:44.027677+0000","last_unstale":"2024-03-06T07:34:44.039070+0000","last_undegraded":"2024-03-06T07:34:44.039070+0000","last_fullsized":"2024-03-06T07:34:44.039070+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:37.503853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.015953+0000","last_change":"2024-03-06T07:34:43.020327+0000","last_active":"2024-03-06T07:34:44.015953+0000","last_peered":"2024-03-06T07:34:44.015953+0000","last_clean":"2024-03-06T07:34:44.015953+0000","last_became_active":"2024-03-06T07:34:43.017714+0000","last_became_peered":"2024-03-06T07:34:43.017714+0000","last_unstale":"2024-03-06T07:34:44.015953+0000","last_undegraded":"2024-03-06T07:34:44.015953+0000","last_fullsized":"2024-03-06T07:34:44.015953+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:33:59.259143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.1","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039597+0000","last_change":"2024-03-06T07:34:44.027736+0000","last_active":"2024-03-06T07:34:44.039597+0000","last_peered":"2024-03-06T07:34:44.039597+0000","last_clean":"2024-03-06T07:34:44.039597+0000","last_became_active":"2024-03-06T07:34:44.024826+0000","last_became_peered":"2024-03-06T07:34:44.024826+0000","last_unstale":"2024-03-06T07:34:44.039597+0000","last_undegraded":"2024-03-06T07:34:44.039597+0000","last_fullsized":"2024-03-06T07:34:44.039597+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:03:18.847033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.0","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.995998+0000","last_change":"2024-03-06T07:34:43.018763+0000","last_active":"2024-03-06T07:34:43.995998+0000","last_peered":"2024-03-06T07:34:43.995998+0000","last_clean":"2024-03-06T07:34:43.995998+0000","last_became_active":"2024-03-06T07:34:43.018451+0000","last_became_peered":"2024-03-06T07:34:43.018451+0000","last_unstale":"2024-03-06T07:34:43.995998+0000","last_undegraded":"2024-03-06T07:34:43.995998+0000","last_fullsized":"2024-03-06T07:34:43.995998+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:39:42.752837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.12","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036473+0000","last_change":"2024-03-06T07:34:44.033135+0000","last_active":"2024-03-06T07:34:44.036473+0000","last_peered":"2024-03-06T07:34:44.036473+0000","last_clean":"2024-03-06T07:34:44.036473+0000","last_became_active":"2024-03-06T07:34:44.032781+0000","last_became_peered":"2024-03-06T07:34:44.032781+0000","last_unstale":"2024-03-06T07:34:44.036473+0000","last_undegraded":"2024-03-06T07:34:44.036473+0000","last_fullsized":"2024-03-06T07:34:44.036473+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:17:38.038578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.13","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043960+0000","last_change":"2024-03-06T07:34:43.025819+0000","last_active":"2024-03-06T07:34:44.043960+0000","last_peered":"2024-03-06T07:34:44.043960+0000","last_clean":"2024-03-06T07:34:44.043960+0000","last_became_active":"2024-03-06T07:34:43.025462+0000","last_became_peered":"2024-03-06T07:34:43.025462+0000","last_unstale":"2024-03-06T07:34:44.043960+0000","last_undegraded":"2024-03-06T07:34:44.043960+0000","last_fullsized":"2024-03-06T07:34:44.043960+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:51:50.826948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":357,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.008349+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:34:44.008349+0000","last_peered":"2024-03-06T07:34:44.008349+0000","last_clean":"2024-03-06T07:34:44.008349+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:34:44.008349+0000","last_undegraded":"2024-03-06T07:34:44.008349+0000","last_fullsized":"2024-03-06T07:34:44.008349+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":19,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":18,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542241,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37484,"kb_used_data":2136,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334356,"statfs":{"total":96636764160,"available":96598380544,"internally_reserved":0,"allocated":2187264,"data_stored":1273323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542240,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36888,"kb_used_data":1544,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334952,"statfs":{"total":96636764160,"available":96598990848,"internally_reserved":0,"allocated":1581056,"data_stored":682317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542241,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36884,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334956,"statfs":{"total":96636764160,"available":96598994944,"internally_reserved":0,"allocated":1576960,"data_stored":680817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542240,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36884,"kb_used_data":1536,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334956,"statfs":{"total":96636764160,"available":96598994944,"internally_reserved":0,"allocated":1572864,"data_stored":680775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542240,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36888,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334952,"statfs":{"total":96636764160,"available":96598990848,"internally_reserved":0,"allocated":1576960,"data_stored":680865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542241,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36876,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334964,"statfs":{"total":96636764160,"available":96599003136,"internally_reserved":0,"allocated":1576960,"data_stored":681321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542242,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36880,"kb_used_data":1532,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334960,"statfs":{"total":96636764160,"available":96598999040,"internally_reserved":0,"allocated":1568768,"data_stored":680705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542239,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37448,"kb_used_data":2112,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334392,"statfs":{"total":96636764160,"available":96598417408,"internally_reserved":0,"allocated":2162688,"data_stored":1271073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":2250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:34:48.653 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-06T07:34:48.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.886+0000 7fa553400640 1 Processor -- start 2024-03-06T07:34:48.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.886+0000 7fa553400640 1 -- start start 2024-03-06T07:34:48.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.888+0000 7fa553400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa54c15eff0 0x7fa54c070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.888+0000 7fa553400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa54c071d30 con 0x7fa54c0af1c0 2024-03-06T07:34:48.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.888+0000 7fa553400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa54c15f4a0 con 0x7fa54c06e4d0 2024-03-06T07:34:48.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.888+0000 7fa553400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa54c071260 con 0x7fa54c15eff0 2024-03-06T07:34:48.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.888+0000 7fa550a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa54c0af1c0 0x7fa54c06cc60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40928/0 (socket says 172.21.15.86:40928) 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.888+0000 7fa550a00640 1 -- 172.21.15.86:0/1055497339 learned_addr learned my addr 172.21.15.86:0/1055497339 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.888+0000 7fa543e00640 1 --2- 172.21.15.86:0/1055497339 >> v2:172.21.15.194:3300/0 conn(0x7fa54c15eff0 0x7fa54c070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1990637402 0 0) 0x7fa53c001110 con 0x7fa54c0af1c0 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa530003520 con 0x7fa54c0af1c0 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2520891968 0 0) 0x7fa530003520 con 0x7fa54c0af1c0 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 >> v2:172.21.15.194:3300/0 conn(0x7fa54c15eff0 msgr2=0x7fa54c070d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 --2- 172.21.15.86:0/1055497339 >> v2:172.21.15.194:3300/0 conn(0x7fa54c15eff0 0x7fa54c070d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 >> v1:172.21.15.194:6789/0 conn(0x7fa54c06e4d0 legacy=0x7fa54c0b35d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa54c071490 con 0x7fa54c0af1c0 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fa53c001840 con 0x7fa54c0af1c0 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 >> v1:172.21.15.86:6789/0 conn(0x7fa54c0af1c0 legacy=0x7fa54c06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa530003a30 0x7fa530003e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa530004800 0x7fa530004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.893 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa5300051c0 0x7fa53000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa53000c0c0 con 0x7fa530003a30 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa53000c270 con 0x7fa530004800 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa53000c470 con 0x7fa5300051c0 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.889+0000 7fa550a00640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa530003a30 0x7fa530003e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.890+0000 7fa550a00640 1 -- 172.21.15.86:0/1055497339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa5300051c0 msgr2=0x7fa53000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.890+0000 7fa550a00640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa5300051c0 0x7fa53000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.890+0000 7fa550a00640 1 -- 172.21.15.86:0/1055497339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa530004800 msgr2=0x7fa530004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.890+0000 7fa550a00640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa530004800 0x7fa530004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.890+0000 7fa550a00640 1 -- 172.21.15.86:0/1055497339 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa530003520 con 0x7fa530003a30 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.890+0000 7fa551400640 1 -- 172.21.15.86:0/1055497339 reap_dead start 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.891+0000 7fa550a00640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa530003a30 0x7fa530003e50 secure :-1 s=READY pgs=1506 cs=0 l=1 rev1=1 crypto rx=0x7fa53c007fd0 tx=0x7fa530005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bcb2c7b687e2ed00 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.891+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53c007e30 con 0x7fa530003a30 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.891+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa53c030070 con 0x7fa530003a30 2024-03-06T07:34:48.894 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.891+0000 7fa54be00640 1 -- 172.21.15.86:0/1055497339 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53c0058d0 con 0x7fa530003a30 2024-03-06T07:34:48.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.891+0000 7fa553400640 1 -- 172.21.15.86:0/1055497339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa530003a30 msgr2=0x7fa530003e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.891+0000 7fa553400640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa530003a30 0x7fa530003e50 secure :-1 s=READY pgs=1506 cs=0 l=1 rev1=1 crypto rx=0x7fa53c007fd0 tx=0x7fa530005830 comp rx=0 tx=0).stop 2024-03-06T07:34:48.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.892+0000 7fa553400640 1 -- 172.21.15.86:0/1055497339 shutdown_connections 2024-03-06T07:34:48.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.892+0000 7fa553400640 1 --2- 172.21.15.86:0/1055497339 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa530003a30 0x7fa530003e50 unknown :-1 s=CLOSED pgs=1506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.892+0000 7fa553400640 1 -- 172.21.15.86:0/1055497339 >> 172.21.15.86:0/1055497339 conn(0x7fa54c0b1730 msgr2=0x7fa54c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.892+0000 7fa553400640 1 -- 172.21.15.86:0/1055497339 shutdown_connections 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.892+0000 7fa553400640 1 -- 172.21.15.86:0/1055497339 wait complete. 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.893+0000 7fa553400640 1 Processor -- start 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.893+0000 7fa553400640 1 -- start start 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.894+0000 7fa553400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa530003a30 0x7fa54c143510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.894+0000 7fa553400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa54c0b05f0 0x7fa54c0b0a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.894+0000 7fa553400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa54c0ce730 0x7fa54c0cebb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.894+0000 7fa553400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa54c0b30b0 con 0x7fa54c0ce730 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.894+0000 7fa553400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa54c071a50 con 0x7fa54c0b05f0 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.894+0000 7fa553400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa54c070d20 con 0x7fa530003a30 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa54c0ce730 0x7fa54c0cebb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa54c0ce730 0x7fa54c0cebb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42966/0 (socket says 172.21.15.86:42966) 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 -- 172.21.15.86:0/2909930518 learned_addr learned my addr 172.21.15.86:0/2909930518 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa550a00640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa530003a30 0x7fa54c143510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.898 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa543e00640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa54c0b05f0 0x7fa54c0b0a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 -- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa530003a30 msgr2=0x7fa54c143510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa530003a30 0x7fa54c143510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 -- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa54c0b05f0 msgr2=0x7fa54c0b0a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa54c0b05f0 0x7fa54c0b0a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 -- 172.21.15.86:0/2909930518 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa54c0cf380 con 0x7fa54c0ce730 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.895+0000 7fa551400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa54c0ce730 0x7fa54c0cebb0 secure :-1 s=READY pgs=1507 cs=0 l=1 rev1=1 crypto rx=0x7fa54c15f3d0 tx=0x7fa54400cb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.896+0000 7fa549600640 1 -- 172.21.15.86:0/2909930518 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa544017070 con 0x7fa54c0ce730 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.896+0000 7fa549600640 1 -- 172.21.15.86:0/2909930518 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa544005d60 con 0x7fa54c0ce730 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.896+0000 7fa549600640 1 -- 172.21.15.86:0/2909930518 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa54400a850 con 0x7fa54c0ce730 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.896+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa54c1f8860 con 0x7fa54c0ce730 2024-03-06T07:34:48.899 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.896+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa54c1f7aa0 con 0x7fa54c0ce730 2024-03-06T07:34:48.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.897+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa54c074030 con 0x7fa54c0ce730 2024-03-06T07:34:48.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.899+0000 7fa549600640 1 -- 172.21.15.86:0/2909930518 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa54400a9f0 con 0x7fa54c0ce730 2024-03-06T07:34:48.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.900+0000 7fa549600640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa52806b4d0 0x7fa52806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:48.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.900+0000 7fa549600640 1 -- 172.21.15.86:0/2909930518 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa5440891b0 con 0x7fa54c0ce730 2024-03-06T07:34:48.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.900+0000 7fa550a00640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa52806b4d0 0x7fa52806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:48.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.900+0000 7fa550a00640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa52806b4d0 0x7fa52806d990 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fa53c034000 tx=0x7fa53c009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:48.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:48.904+0000 7fa549600640 1 -- 172.21.15.86:0/2909930518 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa54c074030 con 0x7fa54c0ce730 2024-03-06T07:34:49.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.097+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 --> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fa54c0710b0 con 0x7fa52806b4d0 2024-03-06T07:34:49.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.104+0000 7fa549600640 1 -- 172.21.15.86:0/2909930518 <== mgr.4101 v2:172.21.15.86:6832/2881780473 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252420 (secure 0 0 0) 0x7fa54c0710b0 con 0x7fa52806b4d0 2024-03-06T07:34:49.107 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:49.111 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.110+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa52806b4d0 msgr2=0x7fa52806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.110+0000 7fa553400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa52806b4d0 0x7fa52806d990 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fa53c034000 tx=0x7fa53c009030 comp rx=0 tx=0).stop 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa54c0ce730 msgr2=0x7fa54c0cebb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa54c0ce730 0x7fa54c0cebb0 secure :-1 s=READY pgs=1507 cs=0 l=1 rev1=1 crypto rx=0x7fa54c15f3d0 tx=0x7fa54400cb90 comp rx=0 tx=0).stop 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 shutdown_connections 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa52806b4d0 0x7fa52806d990 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa54c0ce730 0x7fa54c0cebb0 unknown :-1 s=CLOSED pgs=1507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa54c0b05f0 0x7fa54c0b0a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 --2- 172.21.15.86:0/2909930518 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa530003a30 0x7fa54c143510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 >> 172.21.15.86:0/2909930518 conn(0x7fa54c0b1730 msgr2=0x7fa54c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:49.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 shutdown_connections 2024-03-06T07:34:49.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.111+0000 7fa553400640 1 -- 172.21.15.86:0/2909930518 wait complete. 2024-03-06T07:34:49.130 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":355,"stamp":"2024-03-06T07:34:48.993664+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":70,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":296232,"kb_used_data":13480,"kb_used_omap":12,"kb_used_meta":282611,"kb_avail":754678488,"statfs":{"total":773094113280,"available":772790771712,"internally_reserved":0,"allocated":13803520,"data_stored":6631196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12749,"internal_metadata":289394227},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.005019"},"pg_stats":[{"pgid":"19.3f","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.042663+0000","last_change":"2024-03-06T07:34:44.037084+0000","last_active":"2024-03-06T07:34:44.042663+0000","last_peered":"2024-03-06T07:34:44.042663+0000","last_clean":"2024-03-06T07:34:44.042663+0000","last_became_active":"2024-03-06T07:34:44.031300+0000","last_became_peered":"2024-03-06T07:34:44.031300+0000","last_unstale":"2024-03-06T07:34:44.042663+0000","last_undegraded":"2024-03-06T07:34:44.042663+0000","last_fullsized":"2024-03-06T07:34:44.042663+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:12:41.958115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3e","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039875+0000","last_change":"2024-03-06T07:34:44.032281+0000","last_active":"2024-03-06T07:34:44.039875+0000","last_peered":"2024-03-06T07:34:44.039875+0000","last_clean":"2024-03-06T07:34:44.039875+0000","last_became_active":"2024-03-06T07:34:44.031988+0000","last_became_peered":"2024-03-06T07:34:44.031988+0000","last_unstale":"2024-03-06T07:34:44.039875+0000","last_undegraded":"2024-03-06T07:34:44.039875+0000","last_fullsized":"2024-03-06T07:34:44.039875+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:41:41.067874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3d","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028243+0000","last_change":"2024-03-06T07:34:44.024789+0000","last_active":"2024-03-06T07:34:45.028243+0000","last_peered":"2024-03-06T07:34:45.028243+0000","last_clean":"2024-03-06T07:34:45.028243+0000","last_became_active":"2024-03-06T07:34:44.021439+0000","last_became_peered":"2024-03-06T07:34:44.021439+0000","last_unstale":"2024-03-06T07:34:45.028243+0000","last_undegraded":"2024-03-06T07:34:45.028243+0000","last_fullsized":"2024-03-06T07:34:45.028243+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:54:21.297081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.3c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.027121+0000","last_change":"2024-03-06T07:34:44.023829+0000","last_active":"2024-03-06T07:34:44.027121+0000","last_peered":"2024-03-06T07:34:44.027121+0000","last_clean":"2024-03-06T07:34:44.027121+0000","last_became_active":"2024-03-06T07:34:44.023528+0000","last_became_peered":"2024-03-06T07:34:44.023528+0000","last_unstale":"2024-03-06T07:34:44.027121+0000","last_undegraded":"2024-03-06T07:34:44.027121+0000","last_fullsized":"2024-03-06T07:34:44.027121+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:58:10.029945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.037731+0000","last_change":"2024-03-06T07:34:44.031849+0000","last_active":"2024-03-06T07:34:44.037731+0000","last_peered":"2024-03-06T07:34:44.037731+0000","last_clean":"2024-03-06T07:34:44.037731+0000","last_became_active":"2024-03-06T07:34:44.027460+0000","last_became_peered":"2024-03-06T07:34:44.027460+0000","last_unstale":"2024-03-06T07:34:44.037731+0000","last_undegraded":"2024-03-06T07:34:44.037731+0000","last_fullsized":"2024-03-06T07:34:44.037731+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:38.410784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.3a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033280+0000","last_change":"2024-03-06T07:34:44.027413+0000","last_active":"2024-03-06T07:34:44.033280+0000","last_peered":"2024-03-06T07:34:44.033280+0000","last_clean":"2024-03-06T07:34:44.033280+0000","last_became_active":"2024-03-06T07:34:44.025864+0000","last_became_peered":"2024-03-06T07:34:44.025864+0000","last_unstale":"2024-03-06T07:34:44.033280+0000","last_undegraded":"2024-03-06T07:34:44.033280+0000","last_fullsized":"2024-03-06T07:34:44.033280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:46:38.410784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.39","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030834+0000","last_change":"2024-03-06T07:34:44.024017+0000","last_active":"2024-03-06T07:34:44.030834+0000","last_peered":"2024-03-06T07:34:44.030834+0000","last_clean":"2024-03-06T07:34:44.030834+0000","last_became_active":"2024-03-06T07:34:44.023744+0000","last_became_peered":"2024-03-06T07:34:44.023744+0000","last_unstale":"2024-03-06T07:34:44.030834+0000","last_undegraded":"2024-03-06T07:34:44.030834+0000","last_fullsized":"2024-03-06T07:34:44.030834+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:23.962585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.38","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028092+0000","last_change":"2024-03-06T07:34:44.029784+0000","last_active":"2024-03-06T07:34:45.028092+0000","last_peered":"2024-03-06T07:34:45.028092+0000","last_clean":"2024-03-06T07:34:45.028092+0000","last_became_active":"2024-03-06T07:34:44.029232+0000","last_became_peered":"2024-03-06T07:34:44.029232+0000","last_unstale":"2024-03-06T07:34:45.028092+0000","last_undegraded":"2024-03-06T07:34:45.028092+0000","last_fullsized":"2024-03-06T07:34:45.028092+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:43:43.782358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.37","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028222+0000","last_change":"2024-03-06T07:34:44.027094+0000","last_active":"2024-03-06T07:34:45.028222+0000","last_peered":"2024-03-06T07:34:45.028222+0000","last_clean":"2024-03-06T07:34:45.028222+0000","last_became_active":"2024-03-06T07:34:44.025393+0000","last_became_peered":"2024-03-06T07:34:44.025393+0000","last_unstale":"2024-03-06T07:34:45.028222+0000","last_undegraded":"2024-03-06T07:34:45.028222+0000","last_fullsized":"2024-03-06T07:34:45.028222+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:12:11.776779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.36","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.020354+0000","last_change":"2024-03-06T07:34:44.018805+0000","last_active":"2024-03-06T07:34:44.020354+0000","last_peered":"2024-03-06T07:34:44.020354+0000","last_clean":"2024-03-06T07:34:44.020354+0000","last_became_active":"2024-03-06T07:34:44.018438+0000","last_became_peered":"2024-03-06T07:34:44.018438+0000","last_unstale":"2024-03-06T07:34:44.020354+0000","last_undegraded":"2024-03-06T07:34:44.020354+0000","last_fullsized":"2024-03-06T07:34:44.020354+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:19.983769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.35","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.022441+0000","last_change":"2024-03-06T07:34:44.020291+0000","last_active":"2024-03-06T07:34:44.022441+0000","last_peered":"2024-03-06T07:34:44.022441+0000","last_clean":"2024-03-06T07:34:44.022441+0000","last_became_active":"2024-03-06T07:34:44.019931+0000","last_became_peered":"2024-03-06T07:34:44.019931+0000","last_unstale":"2024-03-06T07:34:44.022441+0000","last_undegraded":"2024-03-06T07:34:44.022441+0000","last_fullsized":"2024-03-06T07:34:44.022441+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:07:16.630980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.34","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028499+0000","last_change":"2024-03-06T07:34:44.032320+0000","last_active":"2024-03-06T07:34:45.028499+0000","last_peered":"2024-03-06T07:34:45.028499+0000","last_clean":"2024-03-06T07:34:45.028499+0000","last_became_active":"2024-03-06T07:34:44.031970+0000","last_became_peered":"2024-03-06T07:34:44.031970+0000","last_unstale":"2024-03-06T07:34:45.028499+0000","last_undegraded":"2024-03-06T07:34:45.028499+0000","last_fullsized":"2024-03-06T07:34:45.028499+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:49:26.666996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.33","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041782+0000","last_change":"2024-03-06T07:34:44.035648+0000","last_active":"2024-03-06T07:34:44.041782+0000","last_peered":"2024-03-06T07:34:44.041782+0000","last_clean":"2024-03-06T07:34:44.041782+0000","last_became_active":"2024-03-06T07:34:44.027272+0000","last_became_peered":"2024-03-06T07:34:44.027272+0000","last_unstale":"2024-03-06T07:34:44.041782+0000","last_undegraded":"2024-03-06T07:34:44.041782+0000","last_fullsized":"2024-03-06T07:34:44.041782+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:53:59.319732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.32","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039584+0000","last_change":"2024-03-06T07:34:44.024324+0000","last_active":"2024-03-06T07:34:44.039584+0000","last_peered":"2024-03-06T07:34:44.039584+0000","last_clean":"2024-03-06T07:34:44.039584+0000","last_became_active":"2024-03-06T07:34:44.022698+0000","last_became_peered":"2024-03-06T07:34:44.022698+0000","last_unstale":"2024-03-06T07:34:44.039584+0000","last_undegraded":"2024-03-06T07:34:44.039584+0000","last_fullsized":"2024-03-06T07:34:44.039584+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:56:58.186782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.31","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.034144+0000","last_change":"2024-03-06T07:34:44.028668+0000","last_active":"2024-03-06T07:34:44.034144+0000","last_peered":"2024-03-06T07:34:44.034144+0000","last_clean":"2024-03-06T07:34:44.034144+0000","last_became_active":"2024-03-06T07:34:44.025564+0000","last_became_peered":"2024-03-06T07:34:44.025564+0000","last_unstale":"2024-03-06T07:34:44.034144+0000","last_undegraded":"2024-03-06T07:34:44.034144+0000","last_fullsized":"2024-03-06T07:34:44.034144+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:57:45.746327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.30","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035674+0000","last_change":"2024-03-06T07:34:44.031123+0000","last_active":"2024-03-06T07:34:44.035674+0000","last_peered":"2024-03-06T07:34:44.035674+0000","last_clean":"2024-03-06T07:34:44.035674+0000","last_became_active":"2024-03-06T07:34:44.028040+0000","last_became_peered":"2024-03-06T07:34:44.028040+0000","last_unstale":"2024-03-06T07:34:44.035674+0000","last_undegraded":"2024-03-06T07:34:44.035674+0000","last_fullsized":"2024-03-06T07:34:44.035674+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:41.158147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2f","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023977+0000","last_change":"2024-03-06T07:34:44.016530+0000","last_active":"2024-03-06T07:34:44.023977+0000","last_peered":"2024-03-06T07:34:44.023977+0000","last_clean":"2024-03-06T07:34:44.023977+0000","last_became_active":"2024-03-06T07:34:44.016052+0000","last_became_peered":"2024-03-06T07:34:44.016052+0000","last_unstale":"2024-03-06T07:34:44.023977+0000","last_undegraded":"2024-03-06T07:34:44.023977+0000","last_fullsized":"2024-03-06T07:34:44.023977+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:01:26.328146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.2e","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035426+0000","last_change":"2024-03-06T07:34:44.026550+0000","last_active":"2024-03-06T07:34:44.035426+0000","last_peered":"2024-03-06T07:34:44.035426+0000","last_clean":"2024-03-06T07:34:44.035426+0000","last_became_active":"2024-03-06T07:34:44.026259+0000","last_became_peered":"2024-03-06T07:34:44.026259+0000","last_unstale":"2024-03-06T07:34:44.035426+0000","last_undegraded":"2024-03-06T07:34:44.035426+0000","last_fullsized":"2024-03-06T07:34:44.035426+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:52:58.683080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2d","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.018578+0000","last_change":"2024-03-06T07:34:44.012280+0000","last_active":"2024-03-06T07:34:44.018578+0000","last_peered":"2024-03-06T07:34:44.018578+0000","last_clean":"2024-03-06T07:34:44.018578+0000","last_became_active":"2024-03-06T07:34:44.011683+0000","last_became_peered":"2024-03-06T07:34:44.011683+0000","last_unstale":"2024-03-06T07:34:44.018578+0000","last_undegraded":"2024-03-06T07:34:44.018578+0000","last_fullsized":"2024-03-06T07:34:44.018578+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:06:19.965303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035615+0000","last_change":"2024-03-06T07:34:44.028349+0000","last_active":"2024-03-06T07:34:44.035615+0000","last_peered":"2024-03-06T07:34:44.035615+0000","last_clean":"2024-03-06T07:34:44.035615+0000","last_became_active":"2024-03-06T07:34:44.027507+0000","last_became_peered":"2024-03-06T07:34:44.027507+0000","last_unstale":"2024-03-06T07:34:44.035615+0000","last_undegraded":"2024-03-06T07:34:44.035615+0000","last_fullsized":"2024-03-06T07:34:44.035615+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:14:17.080362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.026281+0000","last_change":"2024-03-06T07:34:44.025230+0000","last_active":"2024-03-06T07:34:44.026281+0000","last_peered":"2024-03-06T07:34:44.026281+0000","last_clean":"2024-03-06T07:34:44.026281+0000","last_became_active":"2024-03-06T07:34:44.024796+0000","last_became_peered":"2024-03-06T07:34:44.024796+0000","last_unstale":"2024-03-06T07:34:44.026281+0000","last_undegraded":"2024-03-06T07:34:44.026281+0000","last_fullsized":"2024-03-06T07:34:44.026281+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:38:44.146167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.2a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023296+0000","last_change":"2024-03-06T07:34:44.021547+0000","last_active":"2024-03-06T07:34:44.023296+0000","last_peered":"2024-03-06T07:34:44.023296+0000","last_clean":"2024-03-06T07:34:44.023296+0000","last_became_active":"2024-03-06T07:34:44.021233+0000","last_became_peered":"2024-03-06T07:34:44.021233+0000","last_unstale":"2024-03-06T07:34:44.023296+0000","last_undegraded":"2024-03-06T07:34:44.023296+0000","last_fullsized":"2024-03-06T07:34:44.023296+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:13:48.555138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.29","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.027651+0000","last_change":"2024-03-06T07:34:44.031844+0000","last_active":"2024-03-06T07:34:45.027651+0000","last_peered":"2024-03-06T07:34:45.027651+0000","last_clean":"2024-03-06T07:34:45.027651+0000","last_became_active":"2024-03-06T07:34:44.027082+0000","last_became_peered":"2024-03-06T07:34:44.027082+0000","last_unstale":"2024-03-06T07:34:45.027651+0000","last_undegraded":"2024-03-06T07:34:45.027651+0000","last_fullsized":"2024-03-06T07:34:45.027651+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:37:23.571379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.28","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030116+0000","last_change":"2024-03-06T07:34:44.023318+0000","last_active":"2024-03-06T07:34:44.030116+0000","last_peered":"2024-03-06T07:34:44.030116+0000","last_clean":"2024-03-06T07:34:44.030116+0000","last_became_active":"2024-03-06T07:34:44.022223+0000","last_became_peered":"2024-03-06T07:34:44.022223+0000","last_unstale":"2024-03-06T07:34:44.030116+0000","last_undegraded":"2024-03-06T07:34:44.030116+0000","last_fullsized":"2024-03-06T07:34:44.030116+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:43:36.089536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.27","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039406+0000","last_change":"2024-03-06T07:34:44.022845+0000","last_active":"2024-03-06T07:34:44.039406+0000","last_peered":"2024-03-06T07:34:44.039406+0000","last_clean":"2024-03-06T07:34:44.039406+0000","last_became_active":"2024-03-06T07:34:44.019249+0000","last_became_peered":"2024-03-06T07:34:44.019249+0000","last_unstale":"2024-03-06T07:34:44.039406+0000","last_undegraded":"2024-03-06T07:34:44.039406+0000","last_fullsized":"2024-03-06T07:34:44.039406+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:43:43.768052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.26","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041382+0000","last_change":"2024-03-06T07:34:44.037055+0000","last_active":"2024-03-06T07:34:44.041382+0000","last_peered":"2024-03-06T07:34:44.041382+0000","last_clean":"2024-03-06T07:34:44.041382+0000","last_became_active":"2024-03-06T07:34:44.036432+0000","last_became_peered":"2024-03-06T07:34:44.036432+0000","last_unstale":"2024-03-06T07:34:44.041382+0000","last_undegraded":"2024-03-06T07:34:44.041382+0000","last_fullsized":"2024-03-06T07:34:44.041382+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:30:21.585235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.25","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035805+0000","last_change":"2024-03-06T07:34:44.029379+0000","last_active":"2024-03-06T07:34:44.035805+0000","last_peered":"2024-03-06T07:34:44.035805+0000","last_clean":"2024-03-06T07:34:44.035805+0000","last_became_active":"2024-03-06T07:34:44.029116+0000","last_became_peered":"2024-03-06T07:34:44.029116+0000","last_unstale":"2024-03-06T07:34:44.035805+0000","last_undegraded":"2024-03-06T07:34:44.035805+0000","last_fullsized":"2024-03-06T07:34:44.035805+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:56:12.445889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.24","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033542+0000","last_change":"2024-03-06T07:34:44.030753+0000","last_active":"2024-03-06T07:34:44.033542+0000","last_peered":"2024-03-06T07:34:44.033542+0000","last_clean":"2024-03-06T07:34:44.033542+0000","last_became_active":"2024-03-06T07:34:44.030360+0000","last_became_peered":"2024-03-06T07:34:44.030360+0000","last_unstale":"2024-03-06T07:34:44.033542+0000","last_undegraded":"2024-03-06T07:34:44.033542+0000","last_fullsized":"2024-03-06T07:34:44.033542+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:47:39.121616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.23","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.038202+0000","last_change":"2024-03-06T07:34:44.033970+0000","last_active":"2024-03-06T07:34:44.038202+0000","last_peered":"2024-03-06T07:34:44.038202+0000","last_clean":"2024-03-06T07:34:44.038202+0000","last_became_active":"2024-03-06T07:34:44.026804+0000","last_became_peered":"2024-03-06T07:34:44.026804+0000","last_unstale":"2024-03-06T07:34:44.038202+0000","last_undegraded":"2024-03-06T07:34:44.038202+0000","last_fullsized":"2024-03-06T07:34:44.038202+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:33:00.559490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.22","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041605+0000","last_change":"2024-03-06T07:34:44.034657+0000","last_active":"2024-03-06T07:34:44.041605+0000","last_peered":"2024-03-06T07:34:44.041605+0000","last_clean":"2024-03-06T07:34:44.041605+0000","last_became_active":"2024-03-06T07:34:44.033275+0000","last_became_peered":"2024-03-06T07:34:44.033275+0000","last_unstale":"2024-03-06T07:34:44.041605+0000","last_undegraded":"2024-03-06T07:34:44.041605+0000","last_fullsized":"2024-03-06T07:34:44.041605+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:41:03.630720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.21","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023848+0000","last_change":"2024-03-06T07:34:44.021116+0000","last_active":"2024-03-06T07:34:44.023848+0000","last_peered":"2024-03-06T07:34:44.023848+0000","last_clean":"2024-03-06T07:34:44.023848+0000","last_became_active":"2024-03-06T07:34:44.020866+0000","last_became_peered":"2024-03-06T07:34:44.020866+0000","last_unstale":"2024-03-06T07:34:44.023848+0000","last_undegraded":"2024-03-06T07:34:44.023848+0000","last_fullsized":"2024-03-06T07:34:44.023848+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:01:26.328146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.20","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033461+0000","last_change":"2024-03-06T07:34:44.027612+0000","last_active":"2024-03-06T07:34:44.033461+0000","last_peered":"2024-03-06T07:34:44.033461+0000","last_clean":"2024-03-06T07:34:44.033461+0000","last_became_active":"2024-03-06T07:34:44.025754+0000","last_became_peered":"2024-03-06T07:34:44.025754+0000","last_unstale":"2024-03-06T07:34:44.033461+0000","last_undegraded":"2024-03-06T07:34:44.033461+0000","last_fullsized":"2024-03-06T07:34:44.033461+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:23.688678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.1e","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035575+0000","last_change":"2024-03-06T07:34:44.030083+0000","last_active":"2024-03-06T07:34:44.035575+0000","last_peered":"2024-03-06T07:34:44.035575+0000","last_clean":"2024-03-06T07:34:44.035575+0000","last_became_active":"2024-03-06T07:34:44.029812+0000","last_became_peered":"2024-03-06T07:34:44.029812+0000","last_unstale":"2024-03-06T07:34:44.035575+0000","last_undegraded":"2024-03-06T07:34:44.035575+0000","last_fullsized":"2024-03-06T07:34:44.035575+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:03:39.406427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1f","version":"126'4","reported_seq":16,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.044403+0000","last_change":"2024-03-06T07:34:43.027908+0000","last_active":"2024-03-06T07:34:44.044403+0000","last_peered":"2024-03-06T07:34:44.044403+0000","last_clean":"2024-03-06T07:34:44.044403+0000","last_became_active":"2024-03-06T07:34:43.027598+0000","last_became_peered":"2024-03-06T07:34:43.027598+0000","last_unstale":"2024-03-06T07:34:44.044403+0000","last_undegraded":"2024-03-06T07:34:44.044403+0000","last_fullsized":"2024-03-06T07:34:44.044403+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:19:11.087872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.1f","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028764+0000","last_change":"2024-03-06T07:34:44.028470+0000","last_active":"2024-03-06T07:34:45.028764+0000","last_peered":"2024-03-06T07:34:45.028764+0000","last_clean":"2024-03-06T07:34:45.028764+0000","last_became_active":"2024-03-06T07:34:44.028174+0000","last_became_peered":"2024-03-06T07:34:44.028174+0000","last_unstale":"2024-03-06T07:34:45.028764+0000","last_undegraded":"2024-03-06T07:34:45.028764+0000","last_fullsized":"2024-03-06T07:34:45.028764+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:41:09.887782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.1e","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041547+0000","last_change":"2024-03-06T07:34:43.014220+0000","last_active":"2024-03-06T07:34:44.041547+0000","last_peered":"2024-03-06T07:34:44.041547+0000","last_clean":"2024-03-06T07:34:44.041547+0000","last_became_active":"2024-03-06T07:34:43.013789+0000","last_became_peered":"2024-03-06T07:34:43.013789+0000","last_unstale":"2024-03-06T07:34:44.041547+0000","last_undegraded":"2024-03-06T07:34:44.041547+0000","last_fullsized":"2024-03-06T07:34:44.041547+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:00:28.402445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.1c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036961+0000","last_change":"2024-03-06T07:34:44.034445+0000","last_active":"2024-03-06T07:34:44.036961+0000","last_peered":"2024-03-06T07:34:44.036961+0000","last_clean":"2024-03-06T07:34:44.036961+0000","last_became_active":"2024-03-06T07:34:44.031375+0000","last_became_peered":"2024-03-06T07:34:44.031375+0000","last_unstale":"2024-03-06T07:34:44.036961+0000","last_undegraded":"2024-03-06T07:34:44.036961+0000","last_fullsized":"2024-03-06T07:34:44.036961+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:05:10.782135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1d","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043443+0000","last_change":"2024-03-06T07:34:43.013750+0000","last_active":"2024-03-06T07:34:44.043443+0000","last_peered":"2024-03-06T07:34:44.043443+0000","last_clean":"2024-03-06T07:34:44.043443+0000","last_became_active":"2024-03-06T07:34:43.013462+0000","last_became_peered":"2024-03-06T07:34:43.013462+0000","last_unstale":"2024-03-06T07:34:44.043443+0000","last_undegraded":"2024-03-06T07:34:44.043443+0000","last_fullsized":"2024-03-06T07:34:44.043443+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:00:03.977004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.1d","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041407+0000","last_change":"2024-03-06T07:34:44.040486+0000","last_active":"2024-03-06T07:34:44.041407+0000","last_peered":"2024-03-06T07:34:44.041407+0000","last_clean":"2024-03-06T07:34:44.041407+0000","last_became_active":"2024-03-06T07:34:44.040148+0000","last_became_peered":"2024-03-06T07:34:44.040148+0000","last_unstale":"2024-03-06T07:34:44.041407+0000","last_undegraded":"2024-03-06T07:34:44.041407+0000","last_fullsized":"2024-03-06T07:34:44.041407+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:01:19.169150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1c","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043669+0000","last_change":"2024-03-06T07:34:43.012887+0000","last_active":"2024-03-06T07:34:44.043669+0000","last_peered":"2024-03-06T07:34:44.043669+0000","last_clean":"2024-03-06T07:34:44.043669+0000","last_became_active":"2024-03-06T07:34:43.012621+0000","last_became_peered":"2024-03-06T07:34:43.012621+0000","last_unstale":"2024-03-06T07:34:44.043669+0000","last_undegraded":"2024-03-06T07:34:44.043669+0000","last_fullsized":"2024-03-06T07:34:44.043669+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:17:57.709369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.040478+0000","last_change":"2024-03-06T07:34:44.032328+0000","last_active":"2024-03-06T07:34:44.040478+0000","last_peered":"2024-03-06T07:34:44.040478+0000","last_clean":"2024-03-06T07:34:44.040478+0000","last_became_active":"2024-03-06T07:34:44.032062+0000","last_became_peered":"2024-03-06T07:34:44.032062+0000","last_unstale":"2024-03-06T07:34:44.040478+0000","last_undegraded":"2024-03-06T07:34:44.040478+0000","last_fullsized":"2024-03-06T07:34:44.040478+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:33:52.163428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1b","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041555+0000","last_change":"2024-03-06T07:34:43.011300+0000","last_active":"2024-03-06T07:34:44.041555+0000","last_peered":"2024-03-06T07:34:44.041555+0000","last_clean":"2024-03-06T07:34:44.041555+0000","last_became_active":"2024-03-06T07:34:43.010209+0000","last_became_peered":"2024-03-06T07:34:43.010209+0000","last_unstale":"2024-03-06T07:34:44.041555+0000","last_undegraded":"2024-03-06T07:34:44.041555+0000","last_fullsized":"2024-03-06T07:34:44.041555+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:45:21.882196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.023325+0000","last_change":"2024-03-06T07:34:44.016896+0000","last_active":"2024-03-06T07:34:44.023325+0000","last_peered":"2024-03-06T07:34:44.023325+0000","last_clean":"2024-03-06T07:34:44.023325+0000","last_became_active":"2024-03-06T07:34:44.016484+0000","last_became_peered":"2024-03-06T07:34:44.016484+0000","last_unstale":"2024-03-06T07:34:44.023325+0000","last_undegraded":"2024-03-06T07:34:44.023325+0000","last_fullsized":"2024-03-06T07:34:44.023325+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T07:38:44.146167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.1a","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.040506+0000","last_change":"2024-03-06T07:34:43.019384+0000","last_active":"2024-03-06T07:34:44.040506+0000","last_peered":"2024-03-06T07:34:44.040506+0000","last_clean":"2024-03-06T07:34:44.040506+0000","last_became_active":"2024-03-06T07:34:43.017323+0000","last_became_peered":"2024-03-06T07:34:43.017323+0000","last_unstale":"2024-03-06T07:34:44.040506+0000","last_undegraded":"2024-03-06T07:34:44.040506+0000","last_fullsized":"2024-03-06T07:34:44.040506+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:20:56.487188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.18","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028157+0000","last_change":"2024-03-06T07:34:44.029535+0000","last_active":"2024-03-06T07:34:45.028157+0000","last_peered":"2024-03-06T07:34:45.028157+0000","last_clean":"2024-03-06T07:34:45.028157+0000","last_became_active":"2024-03-06T07:34:44.028800+0000","last_became_peered":"2024-03-06T07:34:44.028800+0000","last_unstale":"2024-03-06T07:34:45.028157+0000","last_undegraded":"2024-03-06T07:34:45.028157+0000","last_fullsized":"2024-03-06T07:34:45.028157+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:24:06.945371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.19","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.998925+0000","last_change":"2024-03-06T07:34:43.024474+0000","last_active":"2024-03-06T07:34:43.998925+0000","last_peered":"2024-03-06T07:34:43.998925+0000","last_clean":"2024-03-06T07:34:43.998925+0000","last_became_active":"2024-03-06T07:34:43.024161+0000","last_became_peered":"2024-03-06T07:34:43.024161+0000","last_unstale":"2024-03-06T07:34:43.998925+0000","last_undegraded":"2024-03-06T07:34:43.998925+0000","last_fullsized":"2024-03-06T07:34:43.998925+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:29:59.318159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.19","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039519+0000","last_change":"2024-03-06T07:34:44.025122+0000","last_active":"2024-03-06T07:34:44.039519+0000","last_peered":"2024-03-06T07:34:44.039519+0000","last_clean":"2024-03-06T07:34:44.039519+0000","last_became_active":"2024-03-06T07:34:44.024425+0000","last_became_peered":"2024-03-06T07:34:44.024425+0000","last_unstale":"2024-03-06T07:34:44.039519+0000","last_undegraded":"2024-03-06T07:34:44.039519+0000","last_fullsized":"2024-03-06T07:34:44.039519+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:45:23.693815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.18","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.008373+0000","last_change":"2024-03-06T07:34:43.017180+0000","last_active":"2024-03-06T07:34:44.008373+0000","last_peered":"2024-03-06T07:34:44.008373+0000","last_clean":"2024-03-06T07:34:44.008373+0000","last_became_active":"2024-03-06T07:34:43.016896+0000","last_became_peered":"2024-03-06T07:34:43.016896+0000","last_unstale":"2024-03-06T07:34:44.008373+0000","last_undegraded":"2024-03-06T07:34:44.008373+0000","last_fullsized":"2024-03-06T07:34:44.008373+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:54:37.437177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.16","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.037894+0000","last_change":"2024-03-06T07:34:44.033645+0000","last_active":"2024-03-06T07:34:44.037894+0000","last_peered":"2024-03-06T07:34:44.037894+0000","last_clean":"2024-03-06T07:34:44.037894+0000","last_became_active":"2024-03-06T07:34:44.026172+0000","last_became_peered":"2024-03-06T07:34:44.026172+0000","last_unstale":"2024-03-06T07:34:44.037894+0000","last_undegraded":"2024-03-06T07:34:44.037894+0000","last_fullsized":"2024-03-06T07:34:44.037894+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:47:57.389178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.17","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.017365+0000","last_change":"2024-03-06T07:34:43.009102+0000","last_active":"2024-03-06T07:34:44.017365+0000","last_peered":"2024-03-06T07:34:44.017365+0000","last_clean":"2024-03-06T07:34:44.017365+0000","last_became_active":"2024-03-06T07:34:43.008057+0000","last_became_peered":"2024-03-06T07:34:43.008057+0000","last_unstale":"2024-03-06T07:34:44.017365+0000","last_undegraded":"2024-03-06T07:34:44.017365+0000","last_fullsized":"2024-03-06T07:34:44.017365+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:29:26.160005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.17","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.032833+0000","last_change":"2024-03-06T07:34:44.028355+0000","last_active":"2024-03-06T07:34:44.032833+0000","last_peered":"2024-03-06T07:34:44.032833+0000","last_clean":"2024-03-06T07:34:44.032833+0000","last_became_active":"2024-03-06T07:34:44.028035+0000","last_became_peered":"2024-03-06T07:34:44.028035+0000","last_unstale":"2024-03-06T07:34:44.032833+0000","last_undegraded":"2024-03-06T07:34:44.032833+0000","last_fullsized":"2024-03-06T07:34:44.032833+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:54:17.379795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.16","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.017050+0000","last_change":"2024-03-06T07:34:43.018858+0000","last_active":"2024-03-06T07:34:44.017050+0000","last_peered":"2024-03-06T07:34:44.017050+0000","last_clean":"2024-03-06T07:34:44.017050+0000","last_became_active":"2024-03-06T07:34:43.018590+0000","last_became_peered":"2024-03-06T07:34:43.018590+0000","last_unstale":"2024-03-06T07:34:44.017050+0000","last_undegraded":"2024-03-06T07:34:44.017050+0000","last_fullsized":"2024-03-06T07:34:44.017050+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:43:00.056746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.14","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036940+0000","last_change":"2024-03-06T07:34:44.034274+0000","last_active":"2024-03-06T07:34:44.036940+0000","last_peered":"2024-03-06T07:34:44.036940+0000","last_clean":"2024-03-06T07:34:44.036940+0000","last_became_active":"2024-03-06T07:34:44.033601+0000","last_became_peered":"2024-03-06T07:34:44.033601+0000","last_unstale":"2024-03-06T07:34:44.036940+0000","last_undegraded":"2024-03-06T07:34:44.036940+0000","last_fullsized":"2024-03-06T07:34:44.036940+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:25:59.779783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.15","version":"126'3","reported_seq":15,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043648+0000","last_change":"2024-03-06T07:34:43.013971+0000","last_active":"2024-03-06T07:34:44.043648+0000","last_peered":"2024-03-06T07:34:44.043648+0000","last_clean":"2024-03-06T07:34:44.043648+0000","last_became_active":"2024-03-06T07:34:43.013498+0000","last_became_peered":"2024-03-06T07:34:43.013498+0000","last_unstale":"2024-03-06T07:34:44.043648+0000","last_undegraded":"2024-03-06T07:34:44.043648+0000","last_fullsized":"2024-03-06T07:34:44.043648+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:18:01.619257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.15","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036043+0000","last_change":"2024-03-06T07:34:44.032330+0000","last_active":"2024-03-06T07:34:44.036043+0000","last_peered":"2024-03-06T07:34:44.036043+0000","last_clean":"2024-03-06T07:34:44.036043+0000","last_became_active":"2024-03-06T07:34:44.031526+0000","last_became_peered":"2024-03-06T07:34:44.031526+0000","last_unstale":"2024-03-06T07:34:44.036043+0000","last_undegraded":"2024-03-06T07:34:44.036043+0000","last_fullsized":"2024-03-06T07:34:44.036043+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:26:11.944719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.14","version":"126'6","reported_seq":20,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.027730+0000","last_change":"2024-03-06T07:34:43.023234+0000","last_active":"2024-03-06T07:34:45.027730+0000","last_peered":"2024-03-06T07:34:45.027730+0000","last_clean":"2024-03-06T07:34:45.027730+0000","last_became_active":"2024-03-06T07:34:43.022878+0000","last_became_peered":"2024-03-06T07:34:43.022878+0000","last_unstale":"2024-03-06T07:34:45.027730+0000","last_undegraded":"2024-03-06T07:34:45.027730+0000","last_fullsized":"2024-03-06T07:34:45.027730+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:27:07.066055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.13","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.024957+0000","last_change":"2024-03-06T07:34:44.020973+0000","last_active":"2024-03-06T07:34:44.024957+0000","last_peered":"2024-03-06T07:34:44.024957+0000","last_clean":"2024-03-06T07:34:44.024957+0000","last_became_active":"2024-03-06T07:34:44.020626+0000","last_became_peered":"2024-03-06T07:34:44.020626+0000","last_unstale":"2024-03-06T07:34:44.024957+0000","last_undegraded":"2024-03-06T07:34:44.024957+0000","last_fullsized":"2024-03-06T07:34:44.024957+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:13:57.055771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.12","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.009509+0000","last_change":"2024-03-06T07:34:43.015723+0000","last_active":"2024-03-06T07:34:44.009509+0000","last_peered":"2024-03-06T07:34:44.009509+0000","last_clean":"2024-03-06T07:34:44.009509+0000","last_became_active":"2024-03-06T07:34:43.015412+0000","last_became_peered":"2024-03-06T07:34:43.015412+0000","last_unstale":"2024-03-06T07:34:44.009509+0000","last_undegraded":"2024-03-06T07:34:44.009509+0000","last_fullsized":"2024-03-06T07:34:44.009509+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T18:36:59.942128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.10","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039707+0000","last_change":"2024-03-06T07:34:44.026778+0000","last_active":"2024-03-06T07:34:44.039707+0000","last_peered":"2024-03-06T07:34:44.039707+0000","last_clean":"2024-03-06T07:34:44.039707+0000","last_became_active":"2024-03-06T07:34:44.026494+0000","last_became_peered":"2024-03-06T07:34:44.026494+0000","last_unstale":"2024-03-06T07:34:44.039707+0000","last_undegraded":"2024-03-06T07:34:44.039707+0000","last_fullsized":"2024-03-06T07:34:44.039707+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:03:07.048232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.11","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.011130+0000","last_change":"2024-03-06T07:34:43.022773+0000","last_active":"2024-03-06T07:34:44.011130+0000","last_peered":"2024-03-06T07:34:44.011130+0000","last_clean":"2024-03-06T07:34:44.011130+0000","last_became_active":"2024-03-06T07:34:43.022426+0000","last_became_peered":"2024-03-06T07:34:43.022426+0000","last_unstale":"2024-03-06T07:34:44.011130+0000","last_undegraded":"2024-03-06T07:34:44.011130+0000","last_fullsized":"2024-03-06T07:34:44.011130+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:20:15.694812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.11","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.027658+0000","last_change":"2024-03-06T07:34:44.024381+0000","last_active":"2024-03-06T07:34:45.027658+0000","last_peered":"2024-03-06T07:34:45.027658+0000","last_clean":"2024-03-06T07:34:45.027658+0000","last_became_active":"2024-03-06T07:34:44.024094+0000","last_became_peered":"2024-03-06T07:34:44.024094+0000","last_unstale":"2024-03-06T07:34:45.027658+0000","last_undegraded":"2024-03-06T07:34:45.027658+0000","last_fullsized":"2024-03-06T07:34:45.027658+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:32:43.917666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.10","version":"126'8","reported_seq":22,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028418+0000","last_change":"2024-03-06T07:34:43.016707+0000","last_active":"2024-03-06T07:34:45.028418+0000","last_peered":"2024-03-06T07:34:45.028418+0000","last_clean":"2024-03-06T07:34:45.028418+0000","last_became_active":"2024-03-06T07:34:43.016205+0000","last_became_peered":"2024-03-06T07:34:43.016205+0000","last_unstale":"2024-03-06T07:34:45.028418+0000","last_undegraded":"2024-03-06T07:34:45.028418+0000","last_fullsized":"2024-03-06T07:34:45.028418+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:31:15.042351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.e","version":"0'0","reported_seq":12,"reported_epoch":127,"state":"active+clean","last_fresh":"2024-03-06T07:34:45.028622+0000","last_change":"2024-03-06T07:34:44.032521+0000","last_active":"2024-03-06T07:34:45.028622+0000","last_peered":"2024-03-06T07:34:45.028622+0000","last_clean":"2024-03-06T07:34:45.028622+0000","last_became_active":"2024-03-06T07:34:44.031853+0000","last_became_peered":"2024-03-06T07:34:44.031853+0000","last_unstale":"2024-03-06T07:34:45.028622+0000","last_undegraded":"2024-03-06T07:34:45.028622+0000","last_fullsized":"2024-03-06T07:34:45.028622+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:42:18.108074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.f","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.042087+0000","last_change":"2024-03-06T07:34:43.020026+0000","last_active":"2024-03-06T07:34:44.042087+0000","last_peered":"2024-03-06T07:34:44.042087+0000","last_clean":"2024-03-06T07:34:44.042087+0000","last_became_active":"2024-03-06T07:34:43.018358+0000","last_became_peered":"2024-03-06T07:34:43.018358+0000","last_unstale":"2024-03-06T07:34:44.042087+0000","last_undegraded":"2024-03-06T07:34:44.042087+0000","last_fullsized":"2024-03-06T07:34:44.042087+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:33:14.485031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.f","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030109+0000","last_change":"2024-03-06T07:34:44.024205+0000","last_active":"2024-03-06T07:34:44.030109+0000","last_peered":"2024-03-06T07:34:44.030109+0000","last_clean":"2024-03-06T07:34:44.030109+0000","last_became_active":"2024-03-06T07:34:44.023586+0000","last_became_peered":"2024-03-06T07:34:44.023586+0000","last_unstale":"2024-03-06T07:34:44.030109+0000","last_undegraded":"2024-03-06T07:34:44.030109+0000","last_fullsized":"2024-03-06T07:34:44.030109+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:18:31.327765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.e","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.042959+0000","last_change":"2024-03-06T07:34:43.021714+0000","last_active":"2024-03-06T07:34:44.042959+0000","last_peered":"2024-03-06T07:34:44.042959+0000","last_clean":"2024-03-06T07:34:44.042959+0000","last_became_active":"2024-03-06T07:34:43.021362+0000","last_became_peered":"2024-03-06T07:34:43.021362+0000","last_unstale":"2024-03-06T07:34:44.042959+0000","last_undegraded":"2024-03-06T07:34:44.042959+0000","last_fullsized":"2024-03-06T07:34:44.042959+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:29:36.165337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.c","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.035635+0000","last_change":"2024-03-06T07:34:44.028636+0000","last_active":"2024-03-06T07:34:44.035635+0000","last_peered":"2024-03-06T07:34:44.035635+0000","last_clean":"2024-03-06T07:34:44.035635+0000","last_became_active":"2024-03-06T07:34:44.027948+0000","last_became_peered":"2024-03-06T07:34:44.027948+0000","last_unstale":"2024-03-06T07:34:44.035635+0000","last_undegraded":"2024-03-06T07:34:44.035635+0000","last_fullsized":"2024-03-06T07:34:44.035635+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:05:16.286843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.d","version":"126'1","reported_seq":13,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.044889+0000","last_change":"2024-03-06T07:34:43.015685+0000","last_active":"2024-03-06T07:34:44.044889+0000","last_peered":"2024-03-06T07:34:44.044889+0000","last_clean":"2024-03-06T07:34:44.044889+0000","last_became_active":"2024-03-06T07:34:43.015355+0000","last_became_peered":"2024-03-06T07:34:43.015355+0000","last_unstale":"2024-03-06T07:34:44.044889+0000","last_undegraded":"2024-03-06T07:34:44.044889+0000","last_fullsized":"2024-03-06T07:34:44.044889+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:51:50.826948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.d","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039463+0000","last_change":"2024-03-06T07:34:44.023479+0000","last_active":"2024-03-06T07:34:44.039463+0000","last_peered":"2024-03-06T07:34:44.039463+0000","last_clean":"2024-03-06T07:34:44.039463+0000","last_became_active":"2024-03-06T07:34:44.022456+0000","last_became_peered":"2024-03-06T07:34:44.022456+0000","last_unstale":"2024-03-06T07:34:44.039463+0000","last_undegraded":"2024-03-06T07:34:44.039463+0000","last_fullsized":"2024-03-06T07:34:44.039463+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:06:44.152597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.c","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.017287+0000","last_change":"2024-03-06T07:34:43.017624+0000","last_active":"2024-03-06T07:34:44.017287+0000","last_peered":"2024-03-06T07:34:44.017287+0000","last_clean":"2024-03-06T07:34:44.017287+0000","last_became_active":"2024-03-06T07:34:43.014807+0000","last_became_peered":"2024-03-06T07:34:43.014807+0000","last_unstale":"2024-03-06T07:34:44.017287+0000","last_undegraded":"2024-03-06T07:34:44.017287+0000","last_fullsized":"2024-03-06T07:34:44.017287+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:35:55.305546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.a","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.038651+0000","last_change":"2024-03-06T07:34:44.033797+0000","last_active":"2024-03-06T07:34:44.038651+0000","last_peered":"2024-03-06T07:34:44.038651+0000","last_clean":"2024-03-06T07:34:44.038651+0000","last_became_active":"2024-03-06T07:34:44.025891+0000","last_became_peered":"2024-03-06T07:34:44.025891+0000","last_unstale":"2024-03-06T07:34:44.038651+0000","last_undegraded":"2024-03-06T07:34:44.038651+0000","last_fullsized":"2024-03-06T07:34:44.038651+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:13:22.511687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.b","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043762+0000","last_change":"2024-03-06T07:34:43.018128+0000","last_active":"2024-03-06T07:34:44.043762+0000","last_peered":"2024-03-06T07:34:44.043762+0000","last_clean":"2024-03-06T07:34:44.043762+0000","last_became_active":"2024-03-06T07:34:43.010332+0000","last_became_peered":"2024-03-06T07:34:43.010332+0000","last_unstale":"2024-03-06T07:34:44.043762+0000","last_undegraded":"2024-03-06T07:34:44.043762+0000","last_fullsized":"2024-03-06T07:34:44.043762+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:56:08.846529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.b","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.033699+0000","last_change":"2024-03-06T07:34:44.026526+0000","last_active":"2024-03-06T07:34:44.033699+0000","last_peered":"2024-03-06T07:34:44.033699+0000","last_clean":"2024-03-06T07:34:44.033699+0000","last_became_active":"2024-03-06T07:34:44.026197+0000","last_became_peered":"2024-03-06T07:34:44.026197+0000","last_unstale":"2024-03-06T07:34:44.033699+0000","last_undegraded":"2024-03-06T07:34:44.033699+0000","last_fullsized":"2024-03-06T07:34:44.033699+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T19:25:59.779783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.a","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.016555+0000","last_change":"2024-03-06T07:34:43.020405+0000","last_active":"2024-03-06T07:34:44.016555+0000","last_peered":"2024-03-06T07:34:44.016555+0000","last_clean":"2024-03-06T07:34:44.016555+0000","last_became_active":"2024-03-06T07:34:43.020136+0000","last_became_peered":"2024-03-06T07:34:43.020136+0000","last_unstale":"2024-03-06T07:34:44.016555+0000","last_undegraded":"2024-03-06T07:34:44.016555+0000","last_fullsized":"2024-03-06T07:34:44.016555+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:44:09.577857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.8","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.020280+0000","last_change":"2024-03-06T07:34:44.012720+0000","last_active":"2024-03-06T07:34:44.020280+0000","last_peered":"2024-03-06T07:34:44.020280+0000","last_clean":"2024-03-06T07:34:44.020280+0000","last_became_active":"2024-03-06T07:34:44.012174+0000","last_became_peered":"2024-03-06T07:34:44.012174+0000","last_unstale":"2024-03-06T07:34:44.020280+0000","last_undegraded":"2024-03-06T07:34:44.020280+0000","last_fullsized":"2024-03-06T07:34:44.020280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:43:41.376718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.9","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.015787+0000","last_change":"2024-03-06T07:34:43.020805+0000","last_active":"2024-03-06T07:34:44.015787+0000","last_peered":"2024-03-06T07:34:44.015787+0000","last_clean":"2024-03-06T07:34:44.015787+0000","last_became_active":"2024-03-06T07:34:43.020389+0000","last_became_peered":"2024-03-06T07:34:43.020389+0000","last_unstale":"2024-03-06T07:34:44.015787+0000","last_undegraded":"2024-03-06T07:34:44.015787+0000","last_fullsized":"2024-03-06T07:34:44.015787+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T14:35:32.564536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.9","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030661+0000","last_change":"2024-03-06T07:34:44.027454+0000","last_active":"2024-03-06T07:34:44.030661+0000","last_peered":"2024-03-06T07:34:44.030661+0000","last_clean":"2024-03-06T07:34:44.030661+0000","last_became_active":"2024-03-06T07:34:44.027125+0000","last_became_peered":"2024-03-06T07:34:44.027125+0000","last_unstale":"2024-03-06T07:34:44.030661+0000","last_undegraded":"2024-03-06T07:34:44.030661+0000","last_fullsized":"2024-03-06T07:34:44.030661+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:42:23.962585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.8","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.995584+0000","last_change":"2024-03-06T07:34:43.019990+0000","last_active":"2024-03-06T07:34:43.995584+0000","last_peered":"2024-03-06T07:34:43.995584+0000","last_clean":"2024-03-06T07:34:43.995584+0000","last_became_active":"2024-03-06T07:34:43.019716+0000","last_became_peered":"2024-03-06T07:34:43.019716+0000","last_unstale":"2024-03-06T07:34:43.995584+0000","last_undegraded":"2024-03-06T07:34:43.995584+0000","last_fullsized":"2024-03-06T07:34:43.995584+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:44:31.128029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.6","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.030786+0000","last_change":"2024-03-06T07:34:44.025077+0000","last_active":"2024-03-06T07:34:44.030786+0000","last_peered":"2024-03-06T07:34:44.030786+0000","last_clean":"2024-03-06T07:34:44.030786+0000","last_became_active":"2024-03-06T07:34:44.024802+0000","last_became_peered":"2024-03-06T07:34:44.024802+0000","last_unstale":"2024-03-06T07:34:44.030786+0000","last_undegraded":"2024-03-06T07:34:44.030786+0000","last_fullsized":"2024-03-06T07:34:44.030786+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:26:11.944719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.7","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043173+0000","last_change":"2024-03-06T07:34:43.007731+0000","last_active":"2024-03-06T07:34:44.043173+0000","last_peered":"2024-03-06T07:34:44.043173+0000","last_clean":"2024-03-06T07:34:44.043173+0000","last_became_active":"2024-03-06T07:34:43.007354+0000","last_became_peered":"2024-03-06T07:34:43.007354+0000","last_unstale":"2024-03-06T07:34:44.043173+0000","last_undegraded":"2024-03-06T07:34:44.043173+0000","last_fullsized":"2024-03-06T07:34:44.043173+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:33:00.437444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.7","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.041726+0000","last_change":"2024-03-06T07:34:44.035217+0000","last_active":"2024-03-06T07:34:44.041726+0000","last_peered":"2024-03-06T07:34:44.041726+0000","last_clean":"2024-03-06T07:34:44.041726+0000","last_became_active":"2024-03-06T07:34:44.032137+0000","last_became_peered":"2024-03-06T07:34:44.032137+0000","last_unstale":"2024-03-06T07:34:44.041726+0000","last_undegraded":"2024-03-06T07:34:44.041726+0000","last_fullsized":"2024-03-06T07:34:44.041726+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:31:00.344506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.6","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.019874+0000","last_change":"2024-03-06T07:34:43.026213+0000","last_active":"2024-03-06T07:34:44.019874+0000","last_peered":"2024-03-06T07:34:44.019874+0000","last_clean":"2024-03-06T07:34:44.019874+0000","last_became_active":"2024-03-06T07:34:43.025963+0000","last_became_peered":"2024-03-06T07:34:43.025963+0000","last_unstale":"2024-03-06T07:34:44.019874+0000","last_undegraded":"2024-03-06T07:34:44.019874+0000","last_fullsized":"2024-03-06T07:34:44.019874+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:29:59.318159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.4","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039659+0000","last_change":"2024-03-06T07:34:44.024167+0000","last_active":"2024-03-06T07:34:44.039659+0000","last_peered":"2024-03-06T07:34:44.039659+0000","last_clean":"2024-03-06T07:34:44.039659+0000","last_became_active":"2024-03-06T07:34:44.023851+0000","last_became_peered":"2024-03-06T07:34:44.023851+0000","last_unstale":"2024-03-06T07:34:44.039659+0000","last_undegraded":"2024-03-06T07:34:44.039659+0000","last_fullsized":"2024-03-06T07:34:44.039659+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:04:49.459190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.5","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.009623+0000","last_change":"2024-03-06T07:34:43.017104+0000","last_active":"2024-03-06T07:34:44.009623+0000","last_peered":"2024-03-06T07:34:44.009623+0000","last_clean":"2024-03-06T07:34:44.009623+0000","last_became_active":"2024-03-06T07:34:43.016821+0000","last_became_peered":"2024-03-06T07:34:43.016821+0000","last_unstale":"2024-03-06T07:34:44.009623+0000","last_undegraded":"2024-03-06T07:34:44.009623+0000","last_fullsized":"2024-03-06T07:34:44.009623+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:14:52.827203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.5","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.032246+0000","last_change":"2024-03-06T07:34:44.024358+0000","last_active":"2024-03-06T07:34:44.032246+0000","last_peered":"2024-03-06T07:34:44.032246+0000","last_clean":"2024-03-06T07:34:44.032246+0000","last_became_active":"2024-03-06T07:34:44.024057+0000","last_became_peered":"2024-03-06T07:34:44.024057+0000","last_unstale":"2024-03-06T07:34:44.032246+0000","last_undegraded":"2024-03-06T07:34:44.032246+0000","last_fullsized":"2024-03-06T07:34:44.032246+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:17:38.038578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.4","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.996257+0000","last_change":"2024-03-06T07:34:43.016147+0000","last_active":"2024-03-06T07:34:43.996257+0000","last_peered":"2024-03-06T07:34:43.996257+0000","last_clean":"2024-03-06T07:34:43.996257+0000","last_became_active":"2024-03-06T07:34:43.015638+0000","last_became_peered":"2024-03-06T07:34:43.015638+0000","last_unstale":"2024-03-06T07:34:43.996257+0000","last_undegraded":"2024-03-06T07:34:43.996257+0000","last_fullsized":"2024-03-06T07:34:43.996257+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T15:35:47.937364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036989+0000","last_change":"2024-03-06T07:34:44.032338+0000","last_active":"2024-03-06T07:34:44.036989+0000","last_peered":"2024-03-06T07:34:44.036989+0000","last_clean":"2024-03-06T07:34:44.036989+0000","last_became_active":"2024-03-06T07:34:44.031240+0000","last_became_peered":"2024-03-06T07:34:44.031240+0000","last_unstale":"2024-03-06T07:34:44.036989+0000","last_undegraded":"2024-03-06T07:34:44.036989+0000","last_fullsized":"2024-03-06T07:34:44.036989+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T10:41:23.688678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.3","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.010543+0000","last_change":"2024-03-06T07:34:43.019136+0000","last_active":"2024-03-06T07:34:44.010543+0000","last_peered":"2024-03-06T07:34:44.010543+0000","last_clean":"2024-03-06T07:34:44.010543+0000","last_became_active":"2024-03-06T07:34:43.018780+0000","last_became_peered":"2024-03-06T07:34:43.018780+0000","last_unstale":"2024-03-06T07:34:44.010543+0000","last_undegraded":"2024-03-06T07:34:44.010543+0000","last_fullsized":"2024-03-06T07:34:44.010543+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:04:04.884683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.032815+0000","last_change":"2024-03-06T07:34:44.029923+0000","last_active":"2024-03-06T07:34:44.032815+0000","last_peered":"2024-03-06T07:34:44.032815+0000","last_clean":"2024-03-06T07:34:44.032815+0000","last_became_active":"2024-03-06T07:34:44.028803+0000","last_became_peered":"2024-03-06T07:34:44.028803+0000","last_unstale":"2024-03-06T07:34:44.032815+0000","last_undegraded":"2024-03-06T07:34:44.032815+0000","last_fullsized":"2024-03-06T07:34:44.032815+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T13:21:43.352719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.2","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.018462+0000","last_change":"2024-03-06T07:34:43.028247+0000","last_active":"2024-03-06T07:34:44.018462+0000","last_peered":"2024-03-06T07:34:44.018462+0000","last_clean":"2024-03-06T07:34:44.018462+0000","last_became_active":"2024-03-06T07:34:43.027967+0000","last_became_peered":"2024-03-06T07:34:43.027967+0000","last_unstale":"2024-03-06T07:34:44.018462+0000","last_undegraded":"2024-03-06T07:34:44.018462+0000","last_fullsized":"2024-03-06T07:34:44.018462+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T16:38:47.838006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.0","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039070+0000","last_change":"2024-03-06T07:34:44.032660+0000","last_active":"2024-03-06T07:34:44.039070+0000","last_peered":"2024-03-06T07:34:44.039070+0000","last_clean":"2024-03-06T07:34:44.039070+0000","last_became_active":"2024-03-06T07:34:44.027677+0000","last_became_peered":"2024-03-06T07:34:44.027677+0000","last_unstale":"2024-03-06T07:34:44.039070+0000","last_undegraded":"2024-03-06T07:34:44.039070+0000","last_fullsized":"2024-03-06T07:34:44.039070+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:10:37.503853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.015953+0000","last_change":"2024-03-06T07:34:43.020327+0000","last_active":"2024-03-06T07:34:44.015953+0000","last_peered":"2024-03-06T07:34:44.015953+0000","last_clean":"2024-03-06T07:34:44.015953+0000","last_became_active":"2024-03-06T07:34:43.017714+0000","last_became_peered":"2024-03-06T07:34:43.017714+0000","last_unstale":"2024-03-06T07:34:44.015953+0000","last_undegraded":"2024-03-06T07:34:44.015953+0000","last_fullsized":"2024-03-06T07:34:44.015953+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T11:33:59.259143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.1","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.039597+0000","last_change":"2024-03-06T07:34:44.027736+0000","last_active":"2024-03-06T07:34:44.039597+0000","last_peered":"2024-03-06T07:34:44.039597+0000","last_clean":"2024-03-06T07:34:44.039597+0000","last_became_active":"2024-03-06T07:34:44.024826+0000","last_became_peered":"2024-03-06T07:34:44.024826+0000","last_unstale":"2024-03-06T07:34:44.039597+0000","last_undegraded":"2024-03-06T07:34:44.039597+0000","last_fullsized":"2024-03-06T07:34:44.039597+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:03:18.847033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.0","version":"0'0","reported_seq":12,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:43.995998+0000","last_change":"2024-03-06T07:34:43.018763+0000","last_active":"2024-03-06T07:34:43.995998+0000","last_peered":"2024-03-06T07:34:43.995998+0000","last_clean":"2024-03-06T07:34:43.995998+0000","last_became_active":"2024-03-06T07:34:43.018451+0000","last_became_peered":"2024-03-06T07:34:43.018451+0000","last_unstale":"2024-03-06T07:34:43.995998+0000","last_undegraded":"2024-03-06T07:34:43.995998+0000","last_fullsized":"2024-03-06T07:34:43.995998+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T09:39:42.752837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.12","version":"0'0","reported_seq":9,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.036473+0000","last_change":"2024-03-06T07:34:44.033135+0000","last_active":"2024-03-06T07:34:44.036473+0000","last_peered":"2024-03-06T07:34:44.036473+0000","last_clean":"2024-03-06T07:34:44.036473+0000","last_became_active":"2024-03-06T07:34:44.032781+0000","last_became_peered":"2024-03-06T07:34:44.032781+0000","last_unstale":"2024-03-06T07:34:44.036473+0000","last_undegraded":"2024-03-06T07:34:44.036473+0000","last_fullsized":"2024-03-06T07:34:44.036473+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":125,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:42.970448+0000","last_clean_scrub_stamp":"2024-03-06T07:34:42.970448+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T08:17:38.038578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.13","version":"126'2","reported_seq":14,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.043960+0000","last_change":"2024-03-06T07:34:43.025819+0000","last_active":"2024-03-06T07:34:44.043960+0000","last_peered":"2024-03-06T07:34:44.043960+0000","last_clean":"2024-03-06T07:34:44.043960+0000","last_became_active":"2024-03-06T07:34:43.025462+0000","last_became_peered":"2024-03-06T07:34:43.025462+0000","last_unstale":"2024-03-06T07:34:44.043960+0000","last_undegraded":"2024-03-06T07:34:44.043960+0000","last_fullsized":"2024-03-06T07:34:44.043960+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:34:41.956685+0000","last_clean_scrub_stamp":"2024-03-06T07:34:41.956685+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T17:51:50.826948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"18'117","reported_seq":357,"reported_epoch":126,"state":"active+clean","last_fresh":"2024-03-06T07:34:44.008349+0000","last_change":"2024-03-06T07:27:02.709745+0000","last_active":"2024-03-06T07:34:44.008349+0000","last_peered":"2024-03-06T07:34:44.008349+0000","last_clean":"2024-03-06T07:34:44.008349+0000","last_became_active":"2024-03-06T07:27:02.707656+0000","last_became_peered":"2024-03-06T07:27:02.707656+0000","last_unstale":"2024-03-06T07:34:44.008349+0000","last_undegraded":"2024-03-06T07:34:44.008349+0000","last_fullsized":"2024-03-06T07:34:44.008349+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-06T07:27:01.599005+0000","last_clean_scrub_stamp":"2024-03-06T07:27:01.599005+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-07T12:35:06.334981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":19,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":18,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":14,"seq":60129542241,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37484,"kb_used_data":2136,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334356,"statfs":{"total":96636764160,"available":96598380544,"internally_reserved":0,"allocated":2187264,"data_stored":1273323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":14,"seq":60129542240,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36888,"kb_used_data":1544,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334952,"statfs":{"total":96636764160,"available":96598990848,"internally_reserved":0,"allocated":1581056,"data_stored":682317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":14,"seq":60129542241,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36884,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334956,"statfs":{"total":96636764160,"available":96598994944,"internally_reserved":0,"allocated":1576960,"data_stored":680817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":14,"seq":60129542240,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36884,"kb_used_data":1536,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334956,"statfs":{"total":96636764160,"available":96598994944,"internally_reserved":0,"allocated":1572864,"data_stored":680775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":14,"seq":60129542240,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36888,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334952,"statfs":{"total":96636764160,"available":96598990848,"internally_reserved":0,"allocated":1576960,"data_stored":680865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":14,"seq":60129542241,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36876,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334964,"statfs":{"total":96636764160,"available":96599003136,"internally_reserved":0,"allocated":1576960,"data_stored":681321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":14,"seq":60129542242,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36880,"kb_used_data":1532,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334960,"statfs":{"total":96636764160,"available":96598999040,"internally_reserved":0,"allocated":1568768,"data_stored":680705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":14,"seq":60129542239,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37448,"kb_used_data":2112,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334392,"statfs":{"total":96636764160,"available":96598417408,"internally_reserved":0,"allocated":2162688,"data_stored":1271073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":36174279},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":2250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-06T07:34:49.134 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-06T07:34:49.134 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-06T07:34:49.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.379+0000 7f79da200640 1 Processor -- start 2024-03-06T07:34:49.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.379+0000 7f79da200640 1 -- start start 2024-03-06T07:34:49.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.380+0000 7f79da200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f79d415dde0 0x7f79d4161ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79da200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d406cbf0 0x7f79d406d6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79da200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f79d406afc0 0x7f79d406b450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79da200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f79d4162300 con 0x7f79d415dde0 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79da200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f79d41625e0 con 0x7f79d406afc0 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79da200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79d41628c0 con 0x7f79d406cbf0 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79d8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f79d415dde0 0x7f79d4161ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79d8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f79d415dde0 0x7f79d4161ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42972/0 (socket says 172.21.15.86:42972) 2024-03-06T07:34:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79d8e00640 1 -- 172.21.15.86:0/3231821920 learned_addr learned my addr 172.21.15.86:0/3231821920 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79d3e00640 1 --2- 172.21.15.86:0/3231821920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d406cbf0 0x7f79d406d6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.381+0000 7f79d9800640 1 --2- 172.21.15.86:0/3231821920 >> v2:172.21.15.194:3300/0 conn(0x7f79d406afc0 0x7f79d406b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.382+0000 7f79d8e00640 1 -- 172.21.15.86:0/3231821920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d406cbf0 msgr2=0x7f79d406d6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.382+0000 7f79d8e00640 1 --2- 172.21.15.86:0/3231821920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d406cbf0 0x7f79d406d6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.382+0000 7f79d8e00640 1 -- 172.21.15.86:0/3231821920 >> v2:172.21.15.194:3300/0 conn(0x7f79d406afc0 msgr2=0x7f79d406b450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.382+0000 7f79d8e00640 1 --2- 172.21.15.86:0/3231821920 >> v2:172.21.15.194:3300/0 conn(0x7f79d406afc0 0x7f79d406b450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.382+0000 7f79d8e00640 1 -- 172.21.15.86:0/3231821920 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79d40ad860 con 0x7f79d415dde0 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.382+0000 7f79d3e00640 1 --2- 172.21.15.86:0/3231821920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d406cbf0 0x7f79d406d6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.383+0000 7f79d8e00640 1 --2- 172.21.15.86:0/3231821920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d415dde0 0x7f79d4161ad0 secure :-1 s=READY pgs=1508 cs=0 l=1 rev1=1 crypto rx=0x7f79d4162ba0 tx=0x7f79c802b870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=61abdeb32ea5a52c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.383+0000 7f79d3400640 1 -- 172.21.15.86:0/3231821920 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79c80317d0 con 0x7f79d415dde0 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.383+0000 7f79d3400640 1 -- 172.21.15.86:0/3231821920 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f79c8037070 con 0x7f79d415dde0 2024-03-06T07:34:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.383+0000 7f79d3400640 1 -- 172.21.15.86:0/3231821920 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79c8031b90 con 0x7f79d415dde0 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 -- 172.21.15.86:0/3231821920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d415dde0 msgr2=0x7f79d4161ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 --2- 172.21.15.86:0/3231821920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d415dde0 0x7f79d4161ad0 secure :-1 s=READY pgs=1508 cs=0 l=1 rev1=1 crypto rx=0x7f79d4162ba0 tx=0x7f79c802b870 comp rx=0 tx=0).stop 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 -- 172.21.15.86:0/3231821920 shutdown_connections 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 --2- 172.21.15.86:0/3231821920 >> v2:172.21.15.194:3300/0 conn(0x7f79d406afc0 0x7f79d406b450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 --2- 172.21.15.86:0/3231821920 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d406cbf0 0x7f79d406d6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 --2- 172.21.15.86:0/3231821920 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d415dde0 0x7f79d4161ad0 unknown :-1 s=CLOSED pgs=1508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 -- 172.21.15.86:0/3231821920 >> 172.21.15.86:0/3231821920 conn(0x7f79d406c0d0 msgr2=0x7f79d406c4e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 -- 172.21.15.86:0/3231821920 shutdown_connections 2024-03-06T07:34:49.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.384+0000 7f79da200640 1 -- 172.21.15.86:0/3231821920 wait complete. 2024-03-06T07:34:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.386+0000 7f79da200640 1 Processor -- start 2024-03-06T07:34:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.387+0000 7f79da200640 1 -- start start 2024-03-06T07:34:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79da200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d406afc0 0x7f79d414c5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79da200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79d406cbf0 0x7f79d415c480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79da200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d415dde0 0x7f79d415d0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79da200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79d406dc20 con 0x7f79d406afc0 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79da200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79d4162010 con 0x7f79d406cbf0 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79da200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79d406dda0 con 0x7f79d415dde0 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79d8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d406afc0 0x7f79d414c5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79d8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d406afc0 0x7f79d414c5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42986/0 (socket says 172.21.15.86:42986) 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.388+0000 7f79d8e00640 1 -- 172.21.15.86:0/2300614340 learned_addr learned my addr 172.21.15.86:0/2300614340 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d8e00640 1 -- 172.21.15.86:0/2300614340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d415dde0 msgr2=0x7f79d415d0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d8e00640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d415dde0 0x7f79d415d0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d8e00640 1 -- 172.21.15.86:0/2300614340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79d406cbf0 msgr2=0x7f79d415c480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d8e00640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79d406cbf0 0x7f79d415c480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d8e00640 1 -- 172.21.15.86:0/2300614340 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79d415d5f0 con 0x7f79d406afc0 2024-03-06T07:34:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d8e00640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d406afc0 0x7f79d414c5f0 secure :-1 s=READY pgs=1509 cs=0 l=1 rev1=1 crypto rx=0x7f79c802b8b0 tx=0x7f79c80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:49.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d0c00640 1 -- 172.21.15.86:0/2300614340 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79c8032870 con 0x7f79d406afc0 2024-03-06T07:34:49.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d0c00640 1 -- 172.21.15.86:0/2300614340 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f79c8032a30 con 0x7f79d406afc0 2024-03-06T07:34:49.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f79d415b3e0 con 0x7f79d406afc0 2024-03-06T07:34:49.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.389+0000 7f79d0c00640 1 -- 172.21.15.86:0/2300614340 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79c8032e20 con 0x7f79d406afc0 2024-03-06T07:34:49.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.390+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f79d415b9a0 con 0x7f79d406afc0 2024-03-06T07:34:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.391+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f79d4075450 con 0x7f79d406afc0 2024-03-06T07:34:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.391+0000 7f79d0c00640 1 -- 172.21.15.86:0/2300614340 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f79c804c060 con 0x7f79d406afc0 2024-03-06T07:34:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.392+0000 7f79d0c00640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f79b406b4d0 0x7f79b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.392+0000 7f79d0c00640 1 -- 172.21.15.86:0/2300614340 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f79c80abdf0 con 0x7f79d406afc0 2024-03-06T07:34:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.392+0000 7f79d3e00640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f79b406b4d0 0x7f79b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.393+0000 7f79d3e00640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f79b406b4d0 0x7f79b406d990 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f79c0006fd0 tx=0x7f79c0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:49.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.397+0000 7f79d0c00640 1 -- 172.21.15.86:0/2300614340 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f79c8040070 con 0x7f79d406afc0 2024-03-06T07:34:49.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.651+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f79d406bca0 con 0x7f79d406afc0 2024-03-06T07:34:49.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.656+0000 7f79d0c00640 1 -- 172.21.15.86:0/2300614340 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v21) v1 ==== 190+0+147 (secure 0 0 0) 0x7f79c807ac60 con 0x7f79d406afc0 2024-03-06T07:34:49.658 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-03-06T07:34:49.658 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQCEGuhlxvlJORAAltCfTP4uo3ruhagEBNqPbw== 2024-03-06T07:34:49.658 INFO:teuthology.orchestra.run.smithi086.stdout: caps mds = "allow" 2024-03-06T07:34:49.658 INFO:teuthology.orchestra.run.smithi086.stdout: caps mon = "allow r" 2024-03-06T07:34:49.658 INFO:teuthology.orchestra.run.smithi086.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-06T07:34:49.659 INFO:teuthology.orchestra.run.smithi086.stderr:updated caps for client.0 2024-03-06T07:34:49.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f79b406b4d0 msgr2=0x7f79b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f79b406b4d0 0x7f79b406d990 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f79c0006fd0 tx=0x7f79c0008040 comp rx=0 tx=0).stop 2024-03-06T07:34:49.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d406afc0 msgr2=0x7f79d414c5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d406afc0 0x7f79d414c5f0 secure :-1 s=READY pgs=1509 cs=0 l=1 rev1=1 crypto rx=0x7f79c802b8b0 tx=0x7f79c80046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 shutdown_connections 2024-03-06T07:34:49.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f79b406b4d0 0x7f79b406d990 unknown :-1 s=CLOSED pgs=646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f79d415dde0 0x7f79d415d0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f79d406cbf0 0x7f79d415c480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 --2- 172.21.15.86:0/2300614340 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f79d406afc0 0x7f79d414c5f0 unknown :-1 s=CLOSED pgs=1509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 >> 172.21.15.86:0/2300614340 conn(0x7f79d406c0d0 msgr2=0x7f79d41618d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:49.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 shutdown_connections 2024-03-06T07:34:49.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.659+0000 7f79da200640 1 -- 172.21.15.86:0/2300614340 wait complete. 2024-03-06T07:34:49.680 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:49.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.867+0000 7fbfad400640 1 Processor -- start 2024-03-06T07:34:49.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.867+0000 7fbfad400640 1 -- start start 2024-03-06T07:34:49.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.868+0000 7fbfad400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbfa815acb0 0x7fbfa80b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.868+0000 7fbfad400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbfa80b4680 0x7fbfa806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.868+0000 7fbfad400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbfa815b090 con 0x7fbfa8072340 2024-03-06T07:34:49.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.868+0000 7fbfad400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbfa80b2a30 con 0x7fbfa80b4680 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.868+0000 7fbfad400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbfa80b4360 con 0x7fbfa815acb0 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.868+0000 7fbfa5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbfa80b4680 0x7fbfa806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.869+0000 7fbfa5e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbfa80b4680 0x7fbfa806e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42990/0 (socket says 172.21.15.86:42990) 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.869+0000 7fbfa5e00640 1 -- 172.21.15.86:0/3549473759 learned_addr learned my addr 172.21.15.86:0/3549473759 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.869+0000 7fbfa5e00640 1 -- 172.21.15.86:0/3549473759 >> v2:172.21.15.194:3300/0 conn(0x7fbfa815acb0 msgr2=0x7fbfa80b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.869+0000 7fbfa5e00640 1 --2- 172.21.15.86:0/3549473759 >> v2:172.21.15.194:3300/0 conn(0x7fbfa815acb0 0x7fbfa80b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.869+0000 7fbfa5e00640 1 -- 172.21.15.86:0/3549473759 >> v1:172.21.15.194:6789/0 conn(0x7fbfa8072340 legacy=0x7fbfa80ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:49.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.869+0000 7fbfa5e00640 1 -- 172.21.15.86:0/3549473759 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbfa80b2cf0 con 0x7fbfa80b4680 2024-03-06T07:34:49.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.870+0000 7fbfa5e00640 1 --2- 172.21.15.86:0/3549473759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa806e4f0 secure :-1 s=READY pgs=1510 cs=0 l=1 rev1=1 crypto rx=0x7fbfa80b27f0 tx=0x7fbf9402e0e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8cf5f49c9b4726df server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:49.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.870+0000 7fbfa5400640 1 -- 172.21.15.86:0/3549473759 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf9403b070 con 0x7fbfa80b4680 2024-03-06T07:34:49.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.870+0000 7fbfa5400640 1 -- 172.21.15.86:0/3549473759 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf94036440 con 0x7fbfa80b4680 2024-03-06T07:34:49.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.870+0000 7fbfa5400640 1 -- 172.21.15.86:0/3549473759 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf94036810 con 0x7fbfa80b4680 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 -- 172.21.15.86:0/3549473759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 msgr2=0x7fbfa806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 --2- 172.21.15.86:0/3549473759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa806e4f0 secure :-1 s=READY pgs=1510 cs=0 l=1 rev1=1 crypto rx=0x7fbfa80b27f0 tx=0x7fbf9402e0e0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 -- 172.21.15.86:0/3549473759 shutdown_connections 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 --2- 172.21.15.86:0/3549473759 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa806e4f0 unknown :-1 s=CLOSED pgs=1510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 --2- 172.21.15.86:0/3549473759 >> v2:172.21.15.194:3300/0 conn(0x7fbfa815acb0 0x7fbfa80b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 -- 172.21.15.86:0/3549473759 >> 172.21.15.86:0/3549473759 conn(0x7fbfa80b1730 msgr2=0x7fbfa80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 -- 172.21.15.86:0/3549473759 shutdown_connections 2024-03-06T07:34:49.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.871+0000 7fbfad400640 1 -- 172.21.15.86:0/3549473759 wait complete. 2024-03-06T07:34:49.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.873+0000 7fbfad400640 1 Processor -- start 2024-03-06T07:34:49.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.873+0000 7fbfad400640 1 -- start start 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfad400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfa8072340 0x7fbfa814deb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfad400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa8071180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfad400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfa815acb0 0x7fbfa80716c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfad400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbfa80b3d10 con 0x7fbfa80b4680 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfad400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbfa80b32b0 con 0x7fbfa815acb0 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfad400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbfa80b4010 con 0x7fbfa8072340 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa8071180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa8071180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42992/0 (socket says 172.21.15.86:42992) 2024-03-06T07:34:49.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 -- 172.21.15.86:0/2105290268 learned_addr learned my addr 172.21.15.86:0/2105290268 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:49.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 -- 172.21.15.86:0/2105290268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfa8072340 msgr2=0x7fbfa814deb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfa8072340 0x7fbfa814deb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 -- 172.21.15.86:0/2105290268 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfa815acb0 msgr2=0x7fbfa80716c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:49.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfa815acb0 0x7fbfa80716c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:49.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.874+0000 7fbfa5e00640 1 -- 172.21.15.86:0/2105290268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbfa80ce1f0 con 0x7fbfa80b4680 2024-03-06T07:34:49.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.875+0000 7fbfa5e00640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa8071180 secure :-1 s=READY pgs=1511 cs=0 l=1 rev1=1 crypto rx=0x7fbf9402e120 tx=0x7fbf94004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:49.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.875+0000 7fbf9ea00640 1 -- 172.21.15.86:0/2105290268 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf94046070 con 0x7fbfa80b4680 2024-03-06T07:34:49.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.875+0000 7fbf9ea00640 1 -- 172.21.15.86:0/2105290268 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf94034990 con 0x7fbfa80b4680 2024-03-06T07:34:49.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.875+0000 7fbf9ea00640 1 -- 172.21.15.86:0/2105290268 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf9403b070 con 0x7fbfa80b4680 2024-03-06T07:34:49.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.875+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbfa80cf070 con 0x7fbfa80b4680 2024-03-06T07:34:49.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.875+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbfa815cdb0 con 0x7fbfa80b4680 2024-03-06T07:34:49.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.876+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbfa804f330 con 0x7fbfa80b4680 2024-03-06T07:34:49.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.877+0000 7fbf9ea00640 1 -- 172.21.15.86:0/2105290268 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbf94049070 con 0x7fbfa80b4680 2024-03-06T07:34:49.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.878+0000 7fbf9ea00640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf8806b4d0 0x7fbf8806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:49.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.878+0000 7fbf9ea00640 1 -- 172.21.15.86:0/2105290268 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbf940ac510 con 0x7fbfa80b4680 2024-03-06T07:34:49.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.878+0000 7fbfa6800640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf8806b4d0 0x7fbf8806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:49.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.879+0000 7fbfa6800640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf8806b4d0 0x7fbf8806d990 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7fbf98009710 tx=0x7fbf98006c90 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:49.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:49.883+0000 7fbf9ea00640 1 -- 172.21.15.86:0/2105290268 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbf9407b2f0 con 0x7fbfa80b4680 2024-03-06T07:34:49.962 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:34:49.961+0000 7f3a09400640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-06T07:34:50.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.182+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbfa806ce60 con 0x7fbfa80b4680 2024-03-06T07:34:50.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.183+0000 7fbf9ea00640 1 -- 172.21.15.86:0/2105290268 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 93 v93) v1 ==== 94+0+4982 (secure 0 0 0) 0x7fbf9407abf0 con 0x7fbfa80b4680 2024-03-06T07:34:50.185 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:50.185 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":93,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5495,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/374549834","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":374549834},{"type":"v1","addr":"172.21.15.194:6834","nonce":374549834}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88},{"gid":5498,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2544629493","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":2544629493},{"type":"v1","addr":"172.21.15.194:6835","nonce":2544629493}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88},{"gid":8148,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":87}],"filesystems":[{"mdsmap":{"epoch":93,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:46.519048+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8151},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:34:50.185 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 93 2024-03-06T07:34:50.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.185+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf8806b4d0 msgr2=0x7fbf8806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.185+0000 7fbfad400640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf8806b4d0 0x7fbf8806d990 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7fbf98009710 tx=0x7fbf98006c90 comp rx=0 tx=0).stop 2024-03-06T07:34:50.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 msgr2=0x7fbfa8071180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa8071180 secure :-1 s=READY pgs=1511 cs=0 l=1 rev1=1 crypto rx=0x7fbf9402e120 tx=0x7fbf94004730 comp rx=0 tx=0).stop 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 shutdown_connections 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbf8806b4d0 0x7fbf8806d990 unknown :-1 s=CLOSED pgs=647 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbfa815acb0 0x7fbfa80716c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbfa80b4680 0x7fbfa8071180 unknown :-1 s=CLOSED pgs=1511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 --2- 172.21.15.86:0/2105290268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbfa8072340 0x7fbfa814deb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 >> 172.21.15.86:0/2105290268 conn(0x7fbfa80b1730 msgr2=0x7fbfa80aec30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 shutdown_connections 2024-03-06T07:34:50.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.186+0000 7fbfad400640 1 -- 172.21.15.86:0/2105290268 wait complete. 2024-03-06T07:34:50.608 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 93, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:34:46.519048+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 8151}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:34:50.608 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-06T07:34:50.608 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-06T07:34:50.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.818+0000 7f8afba00640 1 Processor -- start 2024-03-06T07:34:50.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.818+0000 7f8afba00640 1 -- start start 2024-03-06T07:34:50.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8afba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af40c6460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8afba00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8af40c4d10 0x7f8af40c5170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8afba00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8af40c56b0 0x7f8af406d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8afba00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8af4145b30 con 0x7f8af40c56b0 2024-03-06T07:34:50.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8afba00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8af4142ce0 con 0x7f8af40c4d10 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8afba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8af40cbca0 con 0x7f8af40c4930 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8af9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8af40c56b0 0x7f8af406d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8af9a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8af40c56b0 0x7f8af406d5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43002/0 (socket says 172.21.15.86:43002) 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8af9a00640 1 -- 172.21.15.86:0/3135237761 learned_addr learned my addr 172.21.15.86:0/3135237761 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8aebe00640 1 --2- 172.21.15.86:0/3135237761 >> v2:172.21.15.194:3300/0 conn(0x7f8af40c4d10 0x7f8af40c5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8af9000640 1 --2- 172.21.15.86:0/3135237761 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af40c6460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.820+0000 7f8af9a00640 1 -- 172.21.15.86:0/3135237761 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 msgr2=0x7f8af40c6460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.821+0000 7f8af9a00640 1 --2- 172.21.15.86:0/3135237761 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af40c6460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.821+0000 7f8af9a00640 1 -- 172.21.15.86:0/3135237761 >> v2:172.21.15.194:3300/0 conn(0x7f8af40c4d10 msgr2=0x7f8af40c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.821+0000 7f8af9a00640 1 --2- 172.21.15.86:0/3135237761 >> v2:172.21.15.194:3300/0 conn(0x7f8af40c4d10 0x7f8af40c5170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.821+0000 7f8af9a00640 1 -- 172.21.15.86:0/3135237761 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8af40cbf80 con 0x7f8af40c56b0 2024-03-06T07:34:50.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.822+0000 7f8af9a00640 1 --2- 172.21.15.86:0/3135237761 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af406d5f0 secure :-1 s=READY pgs=1512 cs=0 l=1 rev1=1 crypto rx=0x7f8af40d0aa0 tx=0x7f8aec02b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b0e0663ef5328dd5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:50.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.822+0000 7f8af3e00640 1 -- 172.21.15.86:0/3135237761 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8aec02e070 con 0x7f8af40c56b0 2024-03-06T07:34:50.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.822+0000 7f8af3e00640 1 -- 172.21.15.86:0/3135237761 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8aec037070 con 0x7f8af40c56b0 2024-03-06T07:34:50.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.822+0000 7f8af3e00640 1 -- 172.21.15.86:0/3135237761 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8aec031a00 con 0x7f8af40c56b0 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.822+0000 7f8afba00640 1 -- 172.21.15.86:0/3135237761 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 msgr2=0x7f8af406d5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.822+0000 7f8afba00640 1 --2- 172.21.15.86:0/3135237761 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af406d5f0 secure :-1 s=READY pgs=1512 cs=0 l=1 rev1=1 crypto rx=0x7f8af40d0aa0 tx=0x7f8aec02b880 comp rx=0 tx=0).stop 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.823+0000 7f8afba00640 1 -- 172.21.15.86:0/3135237761 shutdown_connections 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.823+0000 7f8afba00640 1 --2- 172.21.15.86:0/3135237761 >> v2:172.21.15.194:3300/0 conn(0x7f8af40c4d10 0x7f8af40c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.823+0000 7f8afba00640 1 --2- 172.21.15.86:0/3135237761 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af406d5f0 unknown :-1 s=CLOSED pgs=1512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.823+0000 7f8afba00640 1 --2- 172.21.15.86:0/3135237761 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af40c6460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.823+0000 7f8afba00640 1 -- 172.21.15.86:0/3135237761 >> 172.21.15.86:0/3135237761 conn(0x7f8af40cc4f0 msgr2=0x7f8af40cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.823+0000 7f8afba00640 1 -- 172.21.15.86:0/3135237761 shutdown_connections 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.823+0000 7f8afba00640 1 -- 172.21.15.86:0/3135237761 wait complete. 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.825+0000 7f8afba00640 1 Processor -- start 2024-03-06T07:34:50.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.825+0000 7f8afba00640 1 -- start start 2024-03-06T07:34:50.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8afba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af413c460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8afba00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8af40c4d10 0x7f8af40c80f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8afba00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af40c8630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.829 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8afba00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af40cc260 con 0x7f8af40c56b0 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8afba00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af40ca3b0 con 0x7f8af40c4d10 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8afba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8af40c5b40 con 0x7f8af40c4930 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af40c8630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af40c8630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43004/0 (socket says 172.21.15.86:43004) 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 -- 172.21.15.86:0/578107598 learned_addr learned my addr 172.21.15.86:0/578107598 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9000640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af413c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 -- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 msgr2=0x7f8af413c460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af413c460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 -- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8af40c4d10 msgr2=0x7f8af40c80f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8aebe00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8af40c4d10 0x7f8af40c80f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8af40c4d10 0x7f8af40c80f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.826+0000 7f8af9a00640 1 -- 172.21.15.86:0/578107598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8af40c8c60 con 0x7f8af40c56b0 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.827+0000 7f8af9a00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af40c8630 secure :-1 s=READY pgs=1513 cs=0 l=1 rev1=1 crypto rx=0x7f8aec02b8c0 tx=0x7f8aec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:50.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.827+0000 7f8af1600640 1 -- 172.21.15.86:0/578107598 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8aec032880 con 0x7f8af40c56b0 2024-03-06T07:34:50.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.827+0000 7f8af1600640 1 -- 172.21.15.86:0/578107598 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8aec032a20 con 0x7f8af40c56b0 2024-03-06T07:34:50.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.827+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8af40c8f40 con 0x7f8af40c56b0 2024-03-06T07:34:50.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.827+0000 7f8af1600640 1 -- 172.21.15.86:0/578107598 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8aec032e10 con 0x7f8af40c56b0 2024-03-06T07:34:50.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.827+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8af40c9500 con 0x7f8af40c56b0 2024-03-06T07:34:50.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.830+0000 7f8af1600640 1 -- 172.21.15.86:0/578107598 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8aec0453f0 con 0x7f8af40c56b0 2024-03-06T07:34:50.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.831+0000 7f8af1600640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8ad006b4d0 0x7f8ad006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.831+0000 7f8af1600640 1 -- 172.21.15.86:0/578107598 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8aec0ac130 con 0x7f8af40c56b0 2024-03-06T07:34:50.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.831+0000 7f8af9000640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8ad006b4d0 0x7f8ad006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.831+0000 7f8afe779740 1 Processor -- start 2024-03-06T07:34:50.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.832+0000 7f8afe779740 1 -- start start 2024-03-06T07:34:50.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.832+0000 7f8af9000640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8ad006b4d0 0x7f8ad006d990 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4008d40 tx=0x7f8ae4008290 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:50.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.832+0000 7f8afe779740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05bb750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.832+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05e8b90 0x55bee05eb540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05bbc90 0x55bee05eba80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8afe779740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee0525600 con 0x55bee05e87b0 2024-03-06T07:34:50.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee04ce930 con 0x55bee05e8b90 2024-03-06T07:34:50.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55bee05216a0 con 0x55bee05bbc90 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05bb750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05bb750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43020/0 (socket says 172.21.15.86:43020) 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 -- 172.21.15.86:0/3582853815 learned_addr learned my addr 172.21.15.86:0/3582853815 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8aebe00640 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05e8b90 0x55bee05eb540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 -- 172.21.15.86:0/3582853815 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05bbc90 msgr2=0x55bee05eba80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05bbc90 0x55bee05eba80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 -- 172.21.15.86:0/3582853815 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05e8b90 msgr2=0x55bee05eb540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05e8b90 0x55bee05eb540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 -- 172.21.15.86:0/3582853815 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bee023a2d0 con 0x55bee05e87b0 2024-03-06T07:34:50.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.833+0000 7f8af9000640 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05bb750 secure :-1 s=READY pgs=1514 cs=0 l=1 rev1=1 crypto rx=0x7f8ae400de60 tx=0x7f8ae4010000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb3e795c182d3e17 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:50.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8ae9600640 1 -- 172.21.15.86:0/3582853815 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae4019050 con 0x55bee05e87b0 2024-03-06T07:34:50.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8ae9600640 1 -- 172.21.15.86:0/3582853815 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ae4023070 con 0x55bee05e87b0 2024-03-06T07:34:50.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8ae9600640 1 -- 172.21.15.86:0/3582853815 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae401e880 con 0x55bee05e87b0 2024-03-06T07:34:50.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8afe779740 1 -- 172.21.15.86:0/3582853815 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 msgr2=0x55bee05bb750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8afe779740 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05bb750 secure :-1 s=READY pgs=1514 cs=0 l=1 rev1=1 crypto rx=0x7f8ae400de60 tx=0x7f8ae4010000 comp rx=0 tx=0).stop 2024-03-06T07:34:50.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8afe779740 1 -- 172.21.15.86:0/3582853815 shutdown_connections 2024-03-06T07:34:50.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8afe779740 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05bbc90 0x55bee05eba80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.834+0000 7f8afe779740 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05e8b90 0x55bee05eb540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.835+0000 7f8afe779740 1 --2- 172.21.15.86:0/3582853815 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05bb750 unknown :-1 s=CLOSED pgs=1514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.835+0000 7f8afe779740 1 -- 172.21.15.86:0/3582853815 >> 172.21.15.86:0/3582853815 conn(0x55bee05bd9c0 msgr2=0x55bee0606540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.835+0000 7f8afe779740 1 -- 172.21.15.86:0/3582853815 shutdown_connections 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.835+0000 7f8afe779740 1 -- 172.21.15.86:0/3582853815 wait complete. 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.835+0000 7f8afe779740 1 -- start start 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.835+0000 7f8afe779740 1 Processor -- start 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.836+0000 7f8afe779740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05d9d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.836+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 0x55bee05fe700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.836+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d71d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.836+0000 7f8afe779740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee05bff60 con 0x55bee05e87b0 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.836+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee04e1d90 con 0x55bee05fec40 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55bee05bf350 con 0x55bee05e8b90 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8afe779740 10 client.0 _ll_register_callbacks cb 0x55bee051d350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05d9d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05d9d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43032/0 (socket says 172.21.15.86:43032) 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 -- 172.21.15.86:0/2086369956 learned_addr learned my addr 172.21.15.86:0/2086369956 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:50.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8aebe00640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 0x55bee05fe700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9a00640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d71d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 -- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 msgr2=0x55bee05fe700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 0x55bee05fe700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 -- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 msgr2=0x55bee05d71d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d71d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9000640 1 -- 172.21.15.86:0/2086369956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55bee03046b0 con 0x55bee05e87b0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.837+0000 7f8af9a00640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d71d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8af9000640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05d9d00 secure :-1 s=READY pgs=1515 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4008010 tx=0x7f8ae4004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=37b677a279f9c7d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8ae2000640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8ae2000640 1 -- 172.21.15.86:0/2086369956 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae4033650 con 0x55bee05e87b0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8ae2000640 1 -- 172.21.15.86:0/2086369956 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8ae40337f0 con 0x55bee05e87b0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8ae4023070 con 0x55bee05e87b0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8ae2000640 1 -- 172.21.15.86:0/2086369956 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=93) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8ae4019020 con 0x55bee05e87b0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8afe779740 10 client.8289 fetch_fsmap learned FSMap version 93 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.838+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=93,monmap=2+}) v3 -- 0x55bee0600420 con 0x55bee05e87b0 2024-03-06T07:34:50.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8ae2000640 1 -- 172.21.15.86:0/2086369956 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ae4023070 con 0x55bee05e87b0 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8ae2000640 1 -- 172.21.15.86:0/2086369956 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 93) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f8ae40339f0 con 0x55bee05e87b0 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8afe779740 10 client.8289 fetch_fsmap finished waiting for FSMap version 93 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8afe779740 10 client.8289 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8afe779740 20 client.8289 populate_metadata read hostname 'smithi086' 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8afe779740 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee0622590 0x55bee0624980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8afe779740 4 client.8289 mds_command: new command op to 8151 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 --> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55bee0532b40 con 0x55bee0622590 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.839+0000 7f8aebe00640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee0622590 0x55bee0624980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.840+0000 7f8aebe00640 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee0622590 0x55bee0624980 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=71b9ad58e36263a0 server_cookie=ac154848a22887c in_seq=0 out_seq=0 2024-03-06T07:34:50.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.840+0000 7f8ae2000640 10 client.8289 ms_handle_connect on v2:172.21.15.86:6835/1131889927 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.840+0000 7f8ae2000640 1 -- 172.21.15.86:0/2086369956 <== mds.0 v2:172.21.15.86:6835/1131889927 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x55bee0532b40 con 0x55bee0622590 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.840+0000 7f8ae2000640 10 client.8289 handle_command_reply: tid=0 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.840+0000 7f8afe779740 1 client.8289 shutdown 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 msgr2=0x55bee05d9d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05d9d00 secure :-1 s=READY pgs=1515 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4008010 tx=0x7f8ae4004ab0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 shutdown_connections 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee0622590 0x55bee0624980 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d71d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 0x55bee05fe700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8ae2000640 0 client.8289 ms_handle_reset on v2:172.21.15.86:6835/1131889927 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 --2- 172.21.15.86:0/2086369956 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee05d9d00 unknown :-1 s=CLOSED pgs=1515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 >> 172.21.15.86:0/2086369956 conn(0x55bee05bbc90 msgr2=0x55bee0609a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:50.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 shutdown_connections 2024-03-06T07:34:50.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 1 -- 172.21.15.86:0/2086369956 wait complete. 2024-03-06T07:34:50.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.841+0000 7f8afe779740 20 client.8289 trim_cache size 0 max 16384 2024-03-06T07:34:50.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.859+0000 7f8afe779740 1 Processor -- start 2024-03-06T07:34:50.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.859+0000 7f8afe779740 1 -- start start 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e87b0 0x55bee0629ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8afe779740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e8b90 0x55bee05d8e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d93b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8afe779740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af42003a0 con 0x55bee05e8b90 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af41e1750 con 0x55bee05fec40 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8af40ae6e0 con 0x55bee05e87b0 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8aebe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e8b90 0x55bee05d8e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8aebe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e8b90 0x55bee05d8e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43036/0 (socket says 172.21.15.86:43036) 2024-03-06T07:34:50.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.860+0000 7f8aebe00640 1 -- 172.21.15.86:0/4184368430 learned_addr learned my addr 172.21.15.86:0/4184368430 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:50.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8aebe00640 1 -- 172.21.15.86:0/4184368430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e87b0 msgr2=0x55bee0629ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8aebe00640 1 --2- 172.21.15.86:0/4184368430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e87b0 0x55bee0629ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8aebe00640 1 -- 172.21.15.86:0/4184368430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 msgr2=0x55bee05d93b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8aebe00640 1 --2- 172.21.15.86:0/4184368430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d93b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8aebe00640 1 -- 172.21.15.86:0/4184368430 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bee05fd470 con 0x55bee05e8b90 2024-03-06T07:34:50.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8aebe00640 1 --2- 172.21.15.86:0/4184368430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e8b90 0x55bee05d8e70 secure :-1 s=READY pgs=1516 cs=0 l=1 rev1=1 crypto rx=0x7f8af41df280 tx=0x7f8adc008040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dcf8eb62c06b26c5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:50.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8ae8c00640 1 -- 172.21.15.86:0/4184368430 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8adc01ae60 con 0x55bee05e8b90 2024-03-06T07:34:50.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8ae8c00640 1 -- 172.21.15.86:0/4184368430 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8adc01a250 con 0x55bee05e8b90 2024-03-06T07:34:50.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.861+0000 7f8ae8c00640 1 -- 172.21.15.86:0/4184368430 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8adc013640 con 0x55bee05e8b90 2024-03-06T07:34:50.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 -- 172.21.15.86:0/4184368430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e8b90 msgr2=0x55bee05d8e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 --2- 172.21.15.86:0/4184368430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e8b90 0x55bee05d8e70 secure :-1 s=READY pgs=1516 cs=0 l=1 rev1=1 crypto rx=0x7f8af41df280 tx=0x7f8adc008040 comp rx=0 tx=0).stop 2024-03-06T07:34:50.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 -- 172.21.15.86:0/4184368430 shutdown_connections 2024-03-06T07:34:50.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 --2- 172.21.15.86:0/4184368430 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05d93b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 --2- 172.21.15.86:0/4184368430 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e8b90 0x55bee05d8e70 unknown :-1 s=CLOSED pgs=1516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 --2- 172.21.15.86:0/4184368430 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e87b0 0x55bee0629ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 -- 172.21.15.86:0/4184368430 >> 172.21.15.86:0/4184368430 conn(0x55bee05bbc90 msgr2=0x55bee05d9b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:50.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 -- 172.21.15.86:0/4184368430 shutdown_connections 2024-03-06T07:34:50.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.862+0000 7f8afe779740 1 -- 172.21.15.86:0/4184368430 wait complete. 2024-03-06T07:34:50.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.863+0000 7f8afe779740 1 -- start start 2024-03-06T07:34:50.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.863+0000 7f8afe779740 1 Processor -- start 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8afe779740 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee0675610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 0x55bee05ec530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8afe779740 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05eca70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8afe779740 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee05ff940 con 0x55bee05e87b0 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af41e1150 con 0x55bee05fec40 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8afe779740 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x55bee05eebd0 con 0x55bee05e8b90 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8afe779740 10 client.0 _ll_register_callbacks cb 0x7f8ae400dd80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-06T07:34:50.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee0675610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee0675610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43042/0 (socket says 172.21.15.86:43042) 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 -- 172.21.15.86:0/3673172107 learned_addr learned my addr 172.21.15.86:0/3673172107 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 -- 172.21.15.86:0/3673172107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 msgr2=0x55bee05ec530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 0x55bee05ec530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 -- 172.21.15.86:0/3673172107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 msgr2=0x55bee05eca70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05eca70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.864+0000 7f8af9000640 1 -- 172.21.15.86:0/3673172107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55bee05ed340 con 0x55bee05e87b0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8af9000640 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee0675610 secure :-1 s=READY pgs=1517 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4003f60 tx=0x7f8ae4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=622957100782078f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8ae2000640 10 client.0 ms_handle_connect on v2:172.21.15.86:3300/0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8ae2000640 1 -- 172.21.15.86:0/3673172107 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae4043020 con 0x55bee05e87b0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8ae2000640 1 -- 172.21.15.86:0/3673172107 <== mon.0 v2:172.21.15.86:3300/0 2 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8ae4033860 con 0x55bee05e87b0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8adc01a250 con 0x55bee05e87b0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8ae2000640 1 -- 172.21.15.86:0/3673172107 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_get_version_reply(handle=1 version=93) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8ae401e750 con 0x55bee05e87b0 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8afe779740 10 client.8295 fetch_fsmap learned FSMap version 93 2024-03-06T07:34:50.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.865+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,fsmap=93,monmap=2+}) v3 -- 0x55bee05ecfb0 con 0x55bee05e87b0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8ae2000640 1 -- 172.21.15.86:0/3673172107 <== mon.0 v2:172.21.15.86:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8adc01a250 con 0x55bee05e87b0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8ae2000640 1 -- 172.21.15.86:0/3673172107 <== mon.0 v2:172.21.15.86:3300/0 5 ==== fsmap(e 93) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f8ae4004040 con 0x55bee05e87b0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8afe779740 10 client.8295 fetch_fsmap finished waiting for FSMap version 93 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8afe779740 10 client.8295 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8afe779740 20 client.8295 populate_metadata read hostname 'smithi086' 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8afe779740 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee05fc8f0 0x55bee05c03f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8afe779740 4 client.8295 mds_command: new command op to 8151 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 --> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55bee052ce90 con 0x55bee05fc8f0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.866+0000 7f8aebe00640 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee05fc8f0 0x55bee05c03f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8aebe00640 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee05fc8f0 0x55bee05c03f0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7b7931f9359ce585 server_cookie=95dd6f0921d618de in_seq=0 out_seq=0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8ae2000640 10 client.8295 ms_handle_connect on v2:172.21.15.86:6835/1131889927 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8ae2000640 1 -- 172.21.15.86:0/3673172107 <== mds.0 v2:172.21.15.86:6835/1131889927 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+333 (crc 0 0 0) 0x55bee052ce90 con 0x55bee05fc8f0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8ae2000640 10 client.8295 handle_command_reply: tid=0 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 client.8295 shutdown 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 msgr2=0x55bee0675610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee0675610 secure :-1 s=READY pgs=1517 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4003f60 tx=0x7f8ae4004a40 comp rx=0 tx=0).stop 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 shutdown_connections 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:6835/1131889927,v1:172.21.15.86:6837/1131889927] conn(0x55bee05fc8f0 0x55bee05c03f0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x55bee05fec40 0x55bee05eca70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x55bee05e8b90 0x55bee05ec530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 --2- 172.21.15.86:0/3673172107 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x55bee05e87b0 0x55bee0675610 unknown :-1 s=CLOSED pgs=1517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8ae2000640 0 client.8295 ms_handle_reset on v2:172.21.15.86:6835/1131889927 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 >> 172.21.15.86:0/3673172107 conn(0x55bee05bbc90 msgr2=0x55bee0609a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 shutdown_connections 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 1 -- 172.21.15.86:0/3673172107 wait complete. 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.867+0000 7f8afe779740 20 client.8295 trim_cache size 0 max 16384 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-03-06T07:34:50.875 INFO:teuthology.orchestra.run.smithi086.stdout: "cluster_fsid": "013155b5-b812-4a77-8af3-f8c90b517e11", 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "whoami": 0, 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "id": 8151, 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "want_state": "up:active", 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "state": "up:active", 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "fs_name": "cephfs", 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "rank_uptime": 6.8539556920000004, 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "mdsmap_epoch": 93, 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch": 127, 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "osdmap_epoch_barrier": 127, 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout: "uptime": 12.298065084999999 2024-03-06T07:34:50.876 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-03-06T07:34:50.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8ad006b4d0 msgr2=0x7f8ad006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8ad006b4d0 0x7f8ad006d990 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4008d40 tx=0x7f8ae4008290 comp rx=0 tx=0).stop 2024-03-06T07:34:50.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 msgr2=0x7f8af40c8630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:50.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af40c8630 secure :-1 s=READY pgs=1513 cs=0 l=1 rev1=1 crypto rx=0x7f8aec02b8c0 tx=0x7f8aec0046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 shutdown_connections 2024-03-06T07:34:50.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8ad006b4d0 0x7f8ad006d990 unknown :-1 s=CLOSED pgs=648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8af40c56b0 0x7f8af40c8630 unknown :-1 s=CLOSED pgs=1513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8af40c4d10 0x7f8af40c80f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 --2- 172.21.15.86:0/578107598 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8af40c4930 0x7f8af413c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:50.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 >> 172.21.15.86:0/578107598 conn(0x7f8af40cc4f0 msgr2=0x7f8af4140220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:50.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 shutdown_connections 2024-03-06T07:34:50.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:50.868+0000 7f8afba00640 1 -- 172.21.15.86:0/578107598 wait complete. 2024-03-06T07:34:50.888 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-06T07:34:51.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.092+0000 7f2025400640 1 Processor -- start 2024-03-06T07:34:51.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.092+0000 7f2025400640 1 -- start start 2024-03-06T07:34:51.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.093+0000 7f2025400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 0x7f202006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.093+0000 7f2025400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f202006ee70 con 0x7f202015fe30 2024-03-06T07:34:51.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.093+0000 7f2025400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f202006cef0 con 0x7f202015f9d0 2024-03-06T07:34:51.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.093+0000 7f2025400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f202006d1d0 con 0x7f2020144c00 2024-03-06T07:34:51.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.094+0000 7f201f200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f202015fe30 0x7f20200ce7b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40944/0 (socket says 172.21.15.86:40944) 2024-03-06T07:34:51.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.094+0000 7f201f200640 1 -- 172.21.15.86:0/4120911015 learned_addr learned my addr 172.21.15.86:0/4120911015 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:51.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.094+0000 7f201e800640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 0x7f202006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.094+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2871720485 0 0) 0x7f2018001110 con 0x7f202015fe30 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.094+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2004003520 con 0x7f202015fe30 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1373202799 0 0) 0x7f2004003520 con 0x7f202015fe30 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 msgr2=0x7f202006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 0x7f202006e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 >> v1:172.21.15.194:6789/0 conn(0x7f202015f9d0 legacy=0x7f20200ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f202006d4b0 con 0x7f202015fe30 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f2018001840 con 0x7f202015fe30 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 >> v1:172.21.15.86:6789/0 conn(0x7f202015fe30 legacy=0x7f20200ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 0x7f20040039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2004003f60 0x7f2004004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2004004d40 0x7f20040051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f202006d1d0 con 0x7f2004004d40 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f200400baa0 con 0x7f2004003f60 2024-03-06T07:34:51.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f200400bca0 con 0x7f2020144c00 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201f200640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2004004d40 0x7f20040051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.095+0000 7f201e800640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 0x7f20040039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.096+0000 7f2015e00640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2004003f60 0x7f2004004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.096+0000 7f201f200640 1 -- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 msgr2=0x7f20040039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.096+0000 7f201f200640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2020144c00 0x7f20040039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.096+0000 7f201f200640 1 -- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2004003f60 msgr2=0x7f2004004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.096+0000 7f201f200640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2004003f60 0x7f2004004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.096+0000 7f201f200640 1 -- 172.21.15.86:0/4120911015 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2004003520 con 0x7f2004004d40 2024-03-06T07:34:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.097+0000 7f201f200640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2004004d40 0x7f20040051c0 secure :-1 s=READY pgs=1519 cs=0 l=1 rev1=1 crypto rx=0x7f201800c4c0 tx=0x7f2004005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e82ecc6f9027e707 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.097+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f201800c0e0 con 0x7f2004004d40 2024-03-06T07:34:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.097+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2018030070 con 0x7f2004004d40 2024-03-06T07:34:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.097+0000 7f201de00640 1 -- 172.21.15.86:0/4120911015 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20180058d0 con 0x7f2004004d40 2024-03-06T07:34:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.098+0000 7f2025400640 1 -- 172.21.15.86:0/4120911015 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2004004d40 msgr2=0x7f20040051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.098+0000 7f2025400640 1 --2- 172.21.15.86:0/4120911015 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2004004d40 0x7f20040051c0 secure :-1 s=READY pgs=1519 cs=0 l=1 rev1=1 crypto rx=0x7f201800c4c0 tx=0x7f2004005830 comp rx=0 tx=0).stop 2024-03-06T07:34:51.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.098+0000 7f201f200640 1 -- 172.21.15.86:0/4120911015 reap_dead start 2024-03-06T07:34:51.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.098+0000 7f2025400640 1 -- 172.21.15.86:0/4120911015 shutdown_connections 2024-03-06T07:34:51.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.098+0000 7f2025400640 1 -- 172.21.15.86:0/4120911015 >> 172.21.15.86:0/4120911015 conn(0x7f20200b1730 msgr2=0x7f20200b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:51.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.098+0000 7f2025400640 1 -- 172.21.15.86:0/4120911015 shutdown_connections 2024-03-06T07:34:51.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.098+0000 7f2025400640 1 -- 172.21.15.86:0/4120911015 wait complete. 2024-03-06T07:34:51.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.100+0000 7f2025400640 1 Processor -- start 2024-03-06T07:34:51.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.100+0000 7f2025400640 1 -- start start 2024-03-06T07:34:51.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.100+0000 7f2025400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2020139a30 0x7f20201f2200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f2025400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f20201f2740 0x7f2020141ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f2025400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f20201420e0 0x7f202013b9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f2025400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2020160730 con 0x7f20201420e0 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f2025400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2020160280 con 0x7f2020139a30 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f2025400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20200ceec0 con 0x7f20201f2740 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f20201420e0 0x7f202013b9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f20201420e0 0x7f202013b9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43062/0 (socket says 172.21.15.86:43062) 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 -- 172.21.15.86:0/341176187 learned_addr learned my addr 172.21.15.86:0/341176187 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201e800640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2020139a30 0x7f20201f2200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f2015e00640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f20201f2740 0x7f2020141ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 -- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f20201f2740 msgr2=0x7f2020141ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f20201f2740 0x7f2020141ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 -- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2020139a30 msgr2=0x7f20201f2200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2020139a30 0x7f20201f2200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.101+0000 7f201f200640 1 -- 172.21.15.86:0/341176187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2020139e10 con 0x7f20201420e0 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.102+0000 7f201f200640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f20201420e0 0x7f202013b9d0 secure :-1 s=READY pgs=1520 cs=0 l=1 rev1=1 crypto rx=0x7f201802c740 tx=0x7f2018001c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:51.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.102+0000 7f2017400640 1 -- 172.21.15.86:0/341176187 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2018004cb0 con 0x7f20201420e0 2024-03-06T07:34:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.102+0000 7f2017400640 1 -- 172.21.15.86:0/341176187 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2018000ed0 con 0x7f20201420e0 2024-03-06T07:34:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.102+0000 7f2017400640 1 -- 172.21.15.86:0/341176187 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2018007290 con 0x7f20201420e0 2024-03-06T07:34:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.102+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20201f2c80 con 0x7f20201420e0 2024-03-06T07:34:51.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.102+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20201f3240 con 0x7f20201420e0 2024-03-06T07:34:51.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.103+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f202006cb50 con 0x7f20201420e0 2024-03-06T07:34:51.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.104+0000 7f2017400640 1 -- 172.21.15.86:0/341176187 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2018007430 con 0x7f20201420e0 2024-03-06T07:34:51.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.105+0000 7f2017400640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ffc06b500 0x7f1ffc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:51.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.105+0000 7f2017400640 1 -- 172.21.15.86:0/341176187 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f20180ab0c0 con 0x7f20201420e0 2024-03-06T07:34:51.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.105+0000 7f201e800640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ffc06b500 0x7f1ffc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:51.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.106+0000 7f201e800640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ffc06b500 0x7f1ffc06d9c0 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7f2010007520 tx=0x7f201000b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:51.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.113+0000 7f2017400640 1 -- 172.21.15.86:0/341176187 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f202006cb50 con 0x7f20201420e0 2024-03-06T07:34:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.346+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f202004f330 con 0x7f20201420e0 2024-03-06T07:34:51.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.347+0000 7f2017400640 1 -- 172.21.15.86:0/341176187 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v93) v1 ==== 53+0+73 (secure 0 0 0) 0x7f202006ee70 con 0x7f20201420e0 2024-03-06T07:34:51.349 INFO:teuthology.orchestra.run.smithi086.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-06T07:34:51.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ffc06b500 msgr2=0x7f1ffc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:51.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ffc06b500 0x7f1ffc06d9c0 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7f2010007520 tx=0x7f201000b040 comp rx=0 tx=0).stop 2024-03-06T07:34:51.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f20201420e0 msgr2=0x7f202013b9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:51.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f20201420e0 0x7f202013b9d0 secure :-1 s=READY pgs=1520 cs=0 l=1 rev1=1 crypto rx=0x7f201802c740 tx=0x7f2018001c10 comp rx=0 tx=0).stop 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 shutdown_connections 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1ffc06b500 0x7f1ffc06d9c0 unknown :-1 s=CLOSED pgs=649 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f20201420e0 0x7f202013b9d0 unknown :-1 s=CLOSED pgs=1520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f20201f2740 0x7f2020141ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 --2- 172.21.15.86:0/341176187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2020139a30 0x7f20201f2200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 >> 172.21.15.86:0/341176187 conn(0x7f20200b1730 msgr2=0x7f20200ce0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.349+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 shutdown_connections 2024-03-06T07:34:51.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:51.350+0000 7f2025400640 1 -- 172.21.15.86:0/341176187 wait complete. 2024-03-06T07:34:51.368 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2024-03-06T07:34:51.368 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-06T07:34:51.369 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi086.front.sepia.ceph.com 2024-03-06T07:34:51.369 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-06T07:34:51.369 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-06T07:34:51.369 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2024-03-06T07:34:51.369 INFO:tasks.cephfs.mount:self.cephfs_mntpt = / 2024-03-06T07:34:51.369 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-06T07:34:51.369 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:34:51.369 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:51.369 DEBUG:teuthology.orchestra.run.smithi086:> ip addr 2024-03-06T07:34:51.414 INFO:teuthology.orchestra.run.smithi086.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-06T07:34:51.414 INFO:teuthology.orchestra.run.smithi086.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-06T07:34:51.414 INFO:teuthology.orchestra.run.smithi086.stdout: inet 127.0.0.1/8 scope host lo 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 ::1/128 scope host 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f6 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f0 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:d8:da:f7 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: altname enp7s0f1 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:08 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-06T07:34:51.415 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0c:c4:7a:8f:d8:09 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: inet 172.21.15.86/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft 41975sec preferred_lft 41975sec 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link proto kernel_ll 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::983b:d7ff:fea8:5039/64 scope link proto kernel_ll 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 6a:a0:03:ef:c4:23 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-06T07:34:51.416 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::68a0:3ff:feef:c423/64 scope link proto kernel_ll 2024-03-06T07:34:51.417 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:34:51.417 INFO:teuthology.orchestra.run.smithi086.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-06T07:34:51.417 INFO:teuthology.orchestra.run.smithi086.stdout: link/ether 0e:4c:26:89:75:66 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-06T07:34:51.417 INFO:teuthology.orchestra.run.smithi086.stdout: inet6 fe80::c4c:26ff:fe89:7566/64 scope link proto kernel_ll 2024-03-06T07:34:51.417 INFO:teuthology.orchestra.run.smithi086.stdout: valid_lft forever preferred_lft forever 2024-03-06T07:34:51.418 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:51.418 DEBUG:teuthology.orchestra.run.smithi086:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-06T07:34:51.545 INFO:teuthology.orchestra.run.smithi086.stdout:1 2024-03-06T07:34:51.548 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:51.549 DEBUG:teuthology.orchestra.run.smithi086:> route 2024-03-06T07:34:51.595 INFO:teuthology.orchestra.run.smithi086.stdout:Kernel IP routing table 2024-03-06T07:34:51.595 INFO:teuthology.orchestra.run.smithi086.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-06T07:34:51.595 INFO:teuthology.orchestra.run.smithi086.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-06T07:34:51.595 INFO:teuthology.orchestra.run.smithi086.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-06T07:34:51.595 INFO:teuthology.orchestra.run.smithi086.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-06T07:34:51.595 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:51.596 DEBUG:teuthology.orchestra.run.smithi086:> (cd / && exec bash -c ' 2024-03-06T07:34:51.596 DEBUG:teuthology.orchestra.run.smithi086:> set -e 2024-03-06T07:34:51.596 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-06T07:34:51.596 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-06T07:34:51.596 DEBUG:teuthology.orchestra.run.smithi086:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-06T07:34:51.596 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:34:51.814 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:51.814 DEBUG:teuthology.orchestra.run.smithi086:> ip netns list 2024-03-06T07:34:51.861 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-06T07:34:51.861 INFO:teuthology.orchestra.run.smithi086.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-06T07:34:51.862 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:34:51.863 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:51.863 DEBUG:teuthology.orchestra.run.smithi086:> sudo ip link set brx.0 up 2024-03-06T07:34:51.942 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:34:51.943 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-06T07:34:51.991 INFO:teuthology.orchestra.run.smithi086.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-06T07:34:51.992 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-06T07:34:51.992 DEBUG:teuthology.orchestra.run.smithi086:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:34:52.082 INFO:tasks.cephfs.kernel_mount:mounting using device: 0@.cephfs=/ 2024-03-06T07:34:52.082 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:52.082 DEBUG:teuthology.orchestra.run.smithi086:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph 0@.cephfs=/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:34:52.196 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:34:52.196 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:34:52.228 INFO:teuthology.orchestra.run.smithi086.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,nofallback 2024-03-06T07:34:52.228 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes". 2024-03-06T07:34:52.229 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: resolved to: "172.21.15.86:3300,172.21.15.194:3300,172.21.15.194:3301" 2024-03-06T07:34:52.229 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: trying mount with new device syntax: 0@013155b5-b812-4a77-8af3-f8c90b517e11.cephfs=/ 2024-03-06T07:34:52.229 INFO:teuthology.orchestra.run.smithi086.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,ms_mode=prefer-crc,key=0,mon_addr=172.21.15.86:3300/172.21.15.194:3300/172.21.15.194:3301" will pass to kernel 2024-03-06T07:34:52.231 INFO:teuthology.orchestra.run.smithi086.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-06T07:34:52.231 INFO:teuthology.orchestra.run.smithi086.stdout: You just mounted a file system that supports labels which does not 2024-03-06T07:34:52.231 INFO:teuthology.orchestra.run.smithi086.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-06T07:34:52.232 INFO:teuthology.orchestra.run.smithi086.stdout: applications will generate AVC messages and not be allowed access to 2024-03-06T07:34:52.232 INFO:teuthology.orchestra.run.smithi086.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-06T07:34:52.235 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-06T07:34:52.235 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:52.236 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-06T07:34:52.310 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:34:52.311 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-06T07:34:52.368 INFO:teuthology.orchestra.run.smithi086.stdout:client5534 2024-03-06T07:34:52.368 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:34:52.369 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:34:52.414 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 2 2024-03-06T07:34:52.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.662+0000 7fc51de00640 1 Processor -- start 2024-03-06T07:34:52.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.662+0000 7fc51de00640 1 -- start start 2024-03-06T07:34:52.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.663+0000 7fc51de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc5180b3c30 0x7fc51815c370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:52.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.663+0000 7fc51de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc5180cda90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:52.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc51de00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc51815fa50 con 0x7fc5180cc760 2024-03-06T07:34:52.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc51de00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5180cdfd0 con 0x7fc5180b3c30 2024-03-06T07:34:52.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc51de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5180cea60 con 0x7fc5180cd660 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc516000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc5180cc760 0x7fc5180ce350 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40948/0 (socket says 172.21.15.86:40948) 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc516000640 1 -- 172.21.15.86:0/777742246 learned_addr learned my addr 172.21.15.86:0/777742246 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc516a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc5180b3c30 0x7fc51815c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc517400640 1 --2- 172.21.15.86:0/777742246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc5180cda90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.664+0000 7fc515600640 1 -- 172.21.15.86:0/777742246 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 500876584 0 0) 0x7fc51815fa50 con 0x7fc5180cc760 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.665+0000 7fc516a00640 1 -- 172.21.15.86:0/777742246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 msgr2=0x7fc5180cda90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.665+0000 7fc516a00640 1 --2- 172.21.15.86:0/777742246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc5180cda90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.665+0000 7fc516a00640 1 -- 172.21.15.86:0/777742246 >> v1:172.21.15.86:6789/0 conn(0x7fc5180cc760 legacy=0x7fc5180ce350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:52.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.665+0000 7fc516a00640 1 -- 172.21.15.86:0/777742246 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc51815ca10 con 0x7fc5180b3c30 2024-03-06T07:34:52.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.666+0000 7fc516a00640 1 --2- 172.21.15.86:0/777742246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51815c370 secure :-1 s=READY pgs=1522 cs=0 l=1 rev1=1 crypto rx=0x7fc5080097a0 tx=0x7fc50802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fc9b772c92f3217 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:52.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.666+0000 7fc515600640 1 -- 172.21.15.86:0/777742246 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc508035020 con 0x7fc5180b3c30 2024-03-06T07:34:52.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.666+0000 7fc515600640 1 -- 172.21.15.86:0/777742246 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc508031830 con 0x7fc5180b3c30 2024-03-06T07:34:52.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.666+0000 7fc515600640 1 -- 172.21.15.86:0/777742246 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc508031c00 con 0x7fc5180b3c30 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 -- 172.21.15.86:0/777742246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 msgr2=0x7fc51815c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 --2- 172.21.15.86:0/777742246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51815c370 secure :-1 s=READY pgs=1522 cs=0 l=1 rev1=1 crypto rx=0x7fc5080097a0 tx=0x7fc50802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 -- 172.21.15.86:0/777742246 shutdown_connections 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 --2- 172.21.15.86:0/777742246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc5180cda90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 --2- 172.21.15.86:0/777742246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51815c370 unknown :-1 s=CLOSED pgs=1522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 -- 172.21.15.86:0/777742246 >> 172.21.15.86:0/777742246 conn(0x7fc5180b2890 msgr2=0x7fc5180ccf50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 -- 172.21.15.86:0/777742246 shutdown_connections 2024-03-06T07:34:52.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.667+0000 7fc51de00640 1 -- 172.21.15.86:0/777742246 wait complete. 2024-03-06T07:34:52.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.669+0000 7fc51de00640 1 Processor -- start 2024-03-06T07:34:52.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.669+0000 7fc51de00640 1 -- start start 2024-03-06T07:34:52.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc51de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51813f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:52.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc51de00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5180cc760 0x7fc51806e760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:52.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc51de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc51806eca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:52.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc51de00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5180b4010 con 0x7fc5180b3c30 2024-03-06T07:34:52.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc51de00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc51815cfd0 con 0x7fc5180cc760 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc51de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc51815d2b0 con 0x7fc5180cd660 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc516a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51813f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc516a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51813f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43070/0 (socket says 172.21.15.86:43070) 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.670+0000 7fc516a00640 1 -- 172.21.15.86:0/3902977607 learned_addr learned my addr 172.21.15.86:0/3902977607 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc516a00640 1 -- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 msgr2=0x7fc51806eca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc517400640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc51806eca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc516000640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5180cc760 0x7fc51806e760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc516a00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc51806eca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc516a00640 1 -- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5180cc760 msgr2=0x7fc51806e760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc516a00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5180cc760 0x7fc51806e760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc516a00640 1 -- 172.21.15.86:0/3902977607 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5181f3bf0 con 0x7fc5180b3c30 2024-03-06T07:34:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.671+0000 7fc516a00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51813f020 secure :-1 s=READY pgs=1523 cs=0 l=1 rev1=1 crypto rx=0x7fc508000c00 tx=0x7fc5080046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.672+0000 7fc50f400640 1 -- 172.21.15.86:0/3902977607 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc508035070 con 0x7fc5180b3c30 2024-03-06T07:34:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.672+0000 7fc50f400640 1 -- 172.21.15.86:0/3902977607 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc508004230 con 0x7fc5180b3c30 2024-03-06T07:34:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.672+0000 7fc50f400640 1 -- 172.21.15.86:0/3902977607 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc50803a6f0 con 0x7fc5180b3c30 2024-03-06T07:34:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.672+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5181f3e70 con 0x7fc5180b3c30 2024-03-06T07:34:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.672+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5181f4430 con 0x7fc5180b3c30 2024-03-06T07:34:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.673+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc51804f330 con 0x7fc5180b3c30 2024-03-06T07:34:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.674+0000 7fc50f400640 1 -- 172.21.15.86:0/3902977607 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc508038020 con 0x7fc5180b3c30 2024-03-06T07:34:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.675+0000 7fc50f400640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4f806b4d0 0x7fc4f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.675+0000 7fc50f400640 1 -- 172.21.15.86:0/3902977607 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc5080abe60 con 0x7fc5180b3c30 2024-03-06T07:34:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.675+0000 7fc516000640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4f806b4d0 0x7fc4f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.676+0000 7fc516000640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4f806b4d0 0x7fc4f806d990 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7fc51806f2f0 tx=0x7fc504009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.679+0000 7fc50f400640 1 -- 172.21.15.86:0/3902977607 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc5181f3bf0 con 0x7fc5180b3c30 2024-03-06T07:34:52.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:52.916+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"} v 0) v1 -- 0x7fc51806bd50 con 0x7fc5180b3c30 2024-03-06T07:34:53.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.091+0000 7fc50f400640 1 -- 172.21.15.86:0/3902977607 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"}]=0 v94) v1 ==== 105+0+0 (secure 0 0 0) 0x7fc508040070 con 0x7fc5180b3c30 2024-03-06T07:34:53.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4f806b4d0 msgr2=0x7fc4f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:53.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4f806b4d0 0x7fc4f806d990 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7fc51806f2f0 tx=0x7fc504009040 comp rx=0 tx=0).stop 2024-03-06T07:34:53.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 msgr2=0x7fc51813f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:53.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51813f020 secure :-1 s=READY pgs=1523 cs=0 l=1 rev1=1 crypto rx=0x7fc508000c00 tx=0x7fc5080046c0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 shutdown_connections 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc4f806b4d0 0x7fc4f806d990 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc5180cd660 0x7fc51806eca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc5180cc760 0x7fc51806e760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 --2- 172.21.15.86:0/3902977607 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc5180b3c30 0x7fc51813f020 unknown :-1 s=CLOSED pgs=1523 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.094+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 >> 172.21.15.86:0/3902977607 conn(0x7fc5180b2890 msgr2=0x7fc5180b0170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.095+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 shutdown_connections 2024-03-06T07:34:53.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.095+0000 7fc51de00640 1 -- 172.21.15.86:0/3902977607 wait complete. 2024-03-06T07:34:53.122 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:53.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.302+0000 7faf66400640 1 Processor -- start 2024-03-06T07:34:53.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.303+0000 7faf66400640 1 -- start start 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf66400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7faf600d24c0 0x7faf600d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf66400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7faf600b2de0 0x7faf600d28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf66400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600d07f0 0x7faf600d0c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf66400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faf600d4a00 con 0x7faf600d24c0 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf66400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faf600cf7e0 con 0x7faf600b2de0 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf66400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faf600d0080 con 0x7faf600d07f0 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf5f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7faf600d24c0 0x7faf600d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:53.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf5f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7faf600d24c0 0x7faf600d1cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43074/0 (socket says 172.21.15.86:43074) 2024-03-06T07:34:53.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.304+0000 7faf5f400640 1 -- 172.21.15.86:0/2226454197 learned_addr learned my addr 172.21.15.86:0/2226454197 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:53.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.305+0000 7faf5f400640 1 -- 172.21.15.86:0/2226454197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600d07f0 msgr2=0x7faf600d0c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:53.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.305+0000 7faf5f400640 1 --2- 172.21.15.86:0/2226454197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600d07f0 0x7faf600d0c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.305+0000 7faf5f400640 1 -- 172.21.15.86:0/2226454197 >> v2:172.21.15.194:3300/0 conn(0x7faf600b2de0 msgr2=0x7faf600d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:53.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.305+0000 7faf5f400640 1 --2- 172.21.15.86:0/2226454197 >> v2:172.21.15.194:3300/0 conn(0x7faf600b2de0 0x7faf600d28a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.305+0000 7faf5f400640 1 -- 172.21.15.86:0/2226454197 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf600d2de0 con 0x7faf600d24c0 2024-03-06T07:34:53.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.306+0000 7faf5f400640 1 --2- 172.21.15.86:0/2226454197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf600d1cc0 secure :-1 s=READY pgs=1524 cs=0 l=1 rev1=1 crypto rx=0x7faf6008d6a0 tx=0x7faf5002b890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=931e22f7729d247e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:53.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.306+0000 7faf5ea00640 1 -- 172.21.15.86:0/2226454197 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf500317f0 con 0x7faf600d24c0 2024-03-06T07:34:53.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.306+0000 7faf5ea00640 1 -- 172.21.15.86:0/2226454197 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faf50037070 con 0x7faf600d24c0 2024-03-06T07:34:53.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.306+0000 7faf5ea00640 1 -- 172.21.15.86:0/2226454197 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf50031bb0 con 0x7faf600d24c0 2024-03-06T07:34:53.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.306+0000 7faf66400640 1 -- 172.21.15.86:0/2226454197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 msgr2=0x7faf600d1cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:53.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.306+0000 7faf66400640 1 --2- 172.21.15.86:0/2226454197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf600d1cc0 secure :-1 s=READY pgs=1524 cs=0 l=1 rev1=1 crypto rx=0x7faf6008d6a0 tx=0x7faf5002b890 comp rx=0 tx=0).stop 2024-03-06T07:34:53.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.307+0000 7faf66400640 1 -- 172.21.15.86:0/2226454197 shutdown_connections 2024-03-06T07:34:53.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.307+0000 7faf66400640 1 --2- 172.21.15.86:0/2226454197 >> v2:172.21.15.194:3300/0 conn(0x7faf600b2de0 0x7faf600d28a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.307+0000 7faf66400640 1 --2- 172.21.15.86:0/2226454197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600d07f0 0x7faf600d0c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.307+0000 7faf66400640 1 --2- 172.21.15.86:0/2226454197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf600d1cc0 unknown :-1 s=CLOSED pgs=1524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.307+0000 7faf66400640 1 -- 172.21.15.86:0/2226454197 >> 172.21.15.86:0/2226454197 conn(0x7faf600d3ee0 msgr2=0x7faf600d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:53.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.307+0000 7faf66400640 1 -- 172.21.15.86:0/2226454197 shutdown_connections 2024-03-06T07:34:53.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.307+0000 7faf66400640 1 -- 172.21.15.86:0/2226454197 wait complete. 2024-03-06T07:34:53.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.308+0000 7faf66400640 1 Processor -- start 2024-03-06T07:34:53.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.308+0000 7faf66400640 1 -- start start 2024-03-06T07:34:53.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.309+0000 7faf66400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600b2de0 0x7faf600ca620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:53.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.309+0000 7faf66400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faf600d07f0 0x7faf600cb9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:53.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.309+0000 7faf66400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf601f3ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:53.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.309+0000 7faf66400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf600cfd10 con 0x7faf600d24c0 2024-03-06T07:34:53.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.309+0000 7faf66400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf600cfa30 con 0x7faf600d07f0 2024-03-06T07:34:53.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.309+0000 7faf66400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faf6015e5b0 con 0x7faf600b2de0 2024-03-06T07:34:53.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf601f3ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:53.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf601f3ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43088/0 (socket says 172.21.15.86:43088) 2024-03-06T07:34:53.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 -- 172.21.15.86:0/1261500035 learned_addr learned my addr 172.21.15.86:0/1261500035 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5f400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600b2de0 0x7faf600ca620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf57e00640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faf600d07f0 0x7faf600cb9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 -- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600b2de0 msgr2=0x7faf600ca620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600b2de0 0x7faf600ca620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 -- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faf600d07f0 msgr2=0x7faf600cb9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faf600d07f0 0x7faf600cb9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 -- 172.21.15.86:0/1261500035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf600cbf00 con 0x7faf600d24c0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.310+0000 7faf5fe00640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf601f3ba0 secure :-1 s=READY pgs=1525 cs=0 l=1 rev1=1 crypto rx=0x7faf5800b970 tx=0x7faf5800be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.311+0000 7faf57400640 1 -- 172.21.15.86:0/1261500035 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf58019070 con 0x7faf600d24c0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.311+0000 7faf57400640 1 -- 172.21.15.86:0/1261500035 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faf5800cb60 con 0x7faf600d24c0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.311+0000 7faf57400640 1 -- 172.21.15.86:0/1261500035 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf58014440 con 0x7faf600d24c0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.311+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faf600cc1a0 con 0x7faf600d24c0 2024-03-06T07:34:53.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.311+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faf600cc760 con 0x7faf600d24c0 2024-03-06T07:34:53.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.312+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faf6004f330 con 0x7faf600d24c0 2024-03-06T07:34:53.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.314+0000 7faf57400640 1 -- 172.21.15.86:0/1261500035 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faf5801c070 con 0x7faf600d24c0 2024-03-06T07:34:53.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.315+0000 7faf57400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faf3c06b4d0 0x7faf3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:53.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.315+0000 7faf57400640 1 -- 172.21.15.86:0/1261500035 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5709+0+0 (secure 0 0 0) 0x7faf58089e90 con 0x7faf600d24c0 2024-03-06T07:34:53.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.315+0000 7faf5f400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faf3c06b4d0 0x7faf3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:53.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.316+0000 7faf5f400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faf3c06b4d0 0x7faf3c06d990 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7faf50036000 tx=0x7faf5003d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:53.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.320+0000 7faf57400640 1 -- 172.21.15.86:0/1261500035 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faf58058da0 con 0x7faf600d24c0 2024-03-06T07:34:53.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.552+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7faf600d22a0 con 0x7faf600d24c0 2024-03-06T07:34:53.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.553+0000 7faf57400640 1 -- 172.21.15.86:0/1261500035 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 95 v95) v1 ==== 94+0+4998 (secure 0 0 0) 0x7faf580586a0 con 0x7faf600d24c0 2024-03-06T07:34:53.555 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:53.555 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":95,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5495,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/374549834","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":374549834},{"type":"v1","addr":"172.21.15.194:6834","nonce":374549834}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88},{"gid":5498,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2544629493","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":2544629493},{"type":"v1","addr":"172.21.15.194:6835","nonce":2544629493}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88}],"filesystems":[{"mdsmap":{"epoch":95,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:53.094863+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:creating","state_seq":1,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:34:53.555 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 95 2024-03-06T07:34:53.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faf3c06b4d0 msgr2=0x7faf3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:53.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faf3c06b4d0 0x7faf3c06d990 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7faf50036000 tx=0x7faf5003d040 comp rx=0 tx=0).stop 2024-03-06T07:34:53.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 msgr2=0x7faf601f3ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:53.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf601f3ba0 secure :-1 s=READY pgs=1525 cs=0 l=1 rev1=1 crypto rx=0x7faf5800b970 tx=0x7faf5800be30 comp rx=0 tx=0).stop 2024-03-06T07:34:53.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 shutdown_connections 2024-03-06T07:34:53.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7faf3c06b4d0 0x7faf3c06d990 unknown :-1 s=CLOSED pgs=651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7faf600d24c0 0x7faf601f3ba0 unknown :-1 s=CLOSED pgs=1525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7faf600d07f0 0x7faf600cb9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.555+0000 7faf66400640 1 --2- 172.21.15.86:0/1261500035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7faf600b2de0 0x7faf600ca620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:53.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.556+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 >> 172.21.15.86:0/1261500035 conn(0x7faf600d3ee0 msgr2=0x7faf600b1cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:53.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.556+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 shutdown_connections 2024-03-06T07:34:53.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:53.556+0000 7faf66400640 1 -- 172.21.15.86:0/1261500035 wait complete. 2024-03-06T07:34:53.581 DEBUG:tasks.ceph_test_case:wait_until_equal: 1 != 2, waiting (timeout=30)... 2024-03-06T07:34:54.643 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:34:54.650 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:34:54.963 INFO:tasks.ceph.mds.c.smithi086.stderr:2024-03-06T07:34:54.960+0000 7f3a09400640 -1 mds.pinger is_rank_lagging: rank=1 was never sent ping request. 2024-03-06T07:34:58.582 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:58.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.777+0000 7f811a200640 1 Processor -- start 2024-03-06T07:34:58.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.777+0000 7f811a200640 1 -- start start 2024-03-06T07:34:58.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.778+0000 7f811a200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:58.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.778+0000 7f811a200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f811415fe30 0x7f81140ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:58.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.778+0000 7f811a200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f811406ee70 con 0x7f811415f9d0 2024-03-06T07:34:58.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.778+0000 7f811a200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f811406cef0 con 0x7f811415fe30 2024-03-06T07:34:58.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.778+0000 7f811a200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f811406d1d0 con 0x7f8114144c00 2024-03-06T07:34:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.779+0000 7f8113e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f811415fe30 0x7f81140ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.779+0000 7f8113e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f811415fe30 0x7f81140ce7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:45742/0 (socket says 172.21.15.86:45742) 2024-03-06T07:34:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.779+0000 7f8113e00640 1 -- 172.21.15.86:0/564301232 learned_addr learned my addr 172.21.15.86:0/564301232 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.779+0000 7f8113400640 1 --2- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.780+0000 7f8113e00640 1 -- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 msgr2=0x7f811406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.780+0000 7f8113e00640 1 --2- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811406e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:58.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.780+0000 7f8113e00640 1 -- 172.21.15.86:0/564301232 >> v1:172.21.15.194:6789/0 conn(0x7f811415f9d0 legacy=0x7f81140ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:58.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.780+0000 7f8113e00640 1 -- 172.21.15.86:0/564301232 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f811406d4b0 con 0x7f811415fe30 2024-03-06T07:34:58.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.781+0000 7f8113e00640 1 --2- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 0x7f81140ce7b0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f810c009e50 tx=0x7f810c02bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fbc0c9f56d16d5d6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.781+0000 7f8112a00640 1 -- 172.21.15.86:0/564301232 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f810c02cb00 con 0x7f811415fe30 2024-03-06T07:34:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f8112a00640 1 -- 172.21.15.86:0/564301232 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f810c02cca0 con 0x7f811415fe30 2024-03-06T07:34:58.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f8112a00640 1 -- 172.21.15.86:0/564301232 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f810c034610 con 0x7f811415fe30 2024-03-06T07:34:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 -- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 msgr2=0x7f81140ce7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 --2- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 0x7f81140ce7b0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f810c009e50 tx=0x7f810c02bf60 comp rx=0 tx=0).stop 2024-03-06T07:34:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 -- 172.21.15.86:0/564301232 shutdown_connections 2024-03-06T07:34:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 --2- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 0x7f81140ce7b0 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 --2- 172.21.15.86:0/564301232 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811406e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 -- 172.21.15.86:0/564301232 >> 172.21.15.86:0/564301232 conn(0x7f81140b1730 msgr2=0x7f81140b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:58.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 -- 172.21.15.86:0/564301232 shutdown_connections 2024-03-06T07:34:58.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.782+0000 7f811a200640 1 -- 172.21.15.86:0/564301232 wait complete. 2024-03-06T07:34:58.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.784+0000 7f811a200640 1 Processor -- start 2024-03-06T07:34:58.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.784+0000 7f811a200640 1 -- start start 2024-03-06T07:34:58.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.785+0000 7f811a200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811413c480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:58.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.785+0000 7f811a200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f811415f9d0 0x7f811414dbb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:58.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.785+0000 7f811a200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 0x7f811415e760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.785+0000 7f811a200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8114160730 con 0x7f811415f9d0 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.785+0000 7f811a200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8114160280 con 0x7f811415fe30 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.785+0000 7f811a200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f811406cb50 con 0x7f8114144c00 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.785+0000 7f810be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f811415f9d0 0x7f811414dbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f811415f9d0 0x7f811414dbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33394/0 (socket says 172.21.15.86:33394) 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 -- 172.21.15.86:0/1523003029 learned_addr learned my addr 172.21.15.86:0/1523003029 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f8113400640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811413c480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f8113e00640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 0x7f811415e760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 -- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 msgr2=0x7f811413c480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811413c480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:58.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 -- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 msgr2=0x7f811415e760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 0x7f811415e760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 -- 172.21.15.86:0/1523003029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f811414e0f0 con 0x7f811415f9d0 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.786+0000 7f810be00640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f811415f9d0 0x7f811414dbb0 secure :-1 s=READY pgs=1526 cs=0 l=1 rev1=1 crypto rx=0x7f80fc00c400 tx=0x7f80fc00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.787+0000 7f810b400640 1 -- 172.21.15.86:0/1523003029 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80fc0052d0 con 0x7f811415f9d0 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.787+0000 7f810b400640 1 -- 172.21.15.86:0/1523003029 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f80fc005490 con 0x7f811415f9d0 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.787+0000 7f810b400640 1 -- 172.21.15.86:0/1523003029 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80fc007260 con 0x7f811415f9d0 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.787+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f811415eca0 con 0x7f811415f9d0 2024-03-06T07:34:58.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.787+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f811415d9a0 con 0x7f811415f9d0 2024-03-06T07:34:58.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.788+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f811404f330 con 0x7f811415f9d0 2024-03-06T07:34:58.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.790+0000 7f810b400640 1 -- 172.21.15.86:0/1523003029 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f80fc007400 con 0x7f811415f9d0 2024-03-06T07:34:58.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.791+0000 7f810b400640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f80f006b4d0 0x7f80f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:58.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.791+0000 7f810b400640 1 -- 172.21.15.86:0/1523003029 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(129..129 src has 1..129) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f80fc0897f0 con 0x7f811415f9d0 2024-03-06T07:34:58.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.791+0000 7f8113400640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f80f006b4d0 0x7f80f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:58.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.792+0000 7f8113400640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f80f006b4d0 0x7f80f006d990 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f8104009ad0 tx=0x7f8104008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:58.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:58.799+0000 7f810b400640 1 -- 172.21.15.86:0/1523003029 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f811406ee70 con 0x7f811415f9d0 2024-03-06T07:34:59.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.118+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8114072890 con 0x7f811415f9d0 2024-03-06T07:34:59.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.119+0000 7f810b400640 1 -- 172.21.15.86:0/1523003029 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 96 v96) v1 ==== 94+0+5001 (secure 0 0 0) 0x7f80fc058590 con 0x7f811415f9d0 2024-03-06T07:34:59.121 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:59.122 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":96,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5495,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/374549834","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":374549834},{"type":"v1","addr":"172.21.15.194:6834","nonce":374549834}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88},{"gid":5498,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2544629493","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":2544629493},{"type":"v1","addr":"172.21.15.194:6835","nonce":2544629493}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88}],"filesystems":[{"mdsmap":{"epoch":96,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:54.098168+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:active","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8148,8151]},"id":9}]} 2024-03-06T07:34:59.122 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 96 2024-03-06T07:34:59.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f80f006b4d0 msgr2=0x7f80f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f80f006b4d0 0x7f80f006d990 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f8104009ad0 tx=0x7f8104008040 comp rx=0 tx=0).stop 2024-03-06T07:34:59.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f811415f9d0 msgr2=0x7f811414dbb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f811415f9d0 0x7f811414dbb0 secure :-1 s=READY pgs=1526 cs=0 l=1 rev1=1 crypto rx=0x7f80fc00c400 tx=0x7f80fc00c8c0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 shutdown_connections 2024-03-06T07:34:59.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f80f006b4d0 0x7f80f006d990 unknown :-1 s=CLOSED pgs=652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f811415fe30 0x7f811415e760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f811415f9d0 0x7f811414dbb0 unknown :-1 s=CLOSED pgs=1526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 --2- 172.21.15.86:0/1523003029 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8114144c00 0x7f811413c480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 >> 172.21.15.86:0/1523003029 conn(0x7f81140b1730 msgr2=0x7f8114145190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:59.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 shutdown_connections 2024-03-06T07:34:59.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.122+0000 7f811a200640 1 -- 172.21.15.86:0/1523003029 wait complete. 2024-03-06T07:34:59.144 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:34:59.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.345+0000 7ff454600640 1 Processor -- start 2024-03-06T07:34:59.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.346+0000 7ff454600640 1 -- start start 2024-03-06T07:34:59.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff454600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c15e0d0 0x7ff44c0d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff454600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff44c0d0fc0 0x7ff44c160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff454600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff44c0cea60 con 0x7ff44c15f470 2024-03-06T07:34:59.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff454600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff44c0d2f60 con 0x7ff44c0d0fc0 2024-03-06T07:34:59.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff454600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff44c0d2350 con 0x7ff44c15e0d0 2024-03-06T07:34:59.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff452600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff44c0d0fc0 0x7ff44c160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff451c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c15e0d0 0x7ff44c0d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.347+0000 7ff452600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff44c0d0fc0 0x7ff44c160dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:45770/0 (socket says 172.21.15.86:45770) 2024-03-06T07:34:59.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.348+0000 7ff452600640 1 -- 172.21.15.86:0/225683371 learned_addr learned my addr 172.21.15.86:0/225683371 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:59.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.348+0000 7ff451c00640 1 -- 172.21.15.86:0/225683371 >> v2:172.21.15.194:3300/0 conn(0x7ff44c0d0fc0 msgr2=0x7ff44c160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:59.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.348+0000 7ff451c00640 1 --2- 172.21.15.86:0/225683371 >> v2:172.21.15.194:3300/0 conn(0x7ff44c0d0fc0 0x7ff44c160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.348+0000 7ff451c00640 1 -- 172.21.15.86:0/225683371 >> v1:172.21.15.194:6789/0 conn(0x7ff44c15f470 legacy=0x7ff44c0d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.348+0000 7ff451c00640 1 -- 172.21.15.86:0/225683371 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff44c0d2630 con 0x7ff44c15e0d0 2024-03-06T07:34:59.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.350+0000 7ff451c00640 1 --2- 172.21.15.86:0/225683371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c15e0d0 0x7ff44c0d1a90 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7ff43c009d90 tx=0x7ff43c02bf30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eda0f7450f138ed4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:59.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.350+0000 7ff44be00640 1 -- 172.21.15.86:0/225683371 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff43c039070 con 0x7ff44c15e0d0 2024-03-06T07:34:59.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.350+0000 7ff44be00640 1 -- 172.21.15.86:0/225683371 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff43c02cce0 con 0x7ff44c15e0d0 2024-03-06T07:34:59.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.350+0000 7ff44be00640 1 -- 172.21.15.86:0/225683371 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff43c034660 con 0x7ff44c15e0d0 2024-03-06T07:34:59.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 -- 172.21.15.86:0/225683371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c15e0d0 msgr2=0x7ff44c0d1a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 --2- 172.21.15.86:0/225683371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c15e0d0 0x7ff44c0d1a90 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7ff43c009d90 tx=0x7ff43c02bf30 comp rx=0 tx=0).stop 2024-03-06T07:34:59.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 -- 172.21.15.86:0/225683371 shutdown_connections 2024-03-06T07:34:59.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 --2- 172.21.15.86:0/225683371 >> v2:172.21.15.194:3300/0 conn(0x7ff44c0d0fc0 0x7ff44c160dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 --2- 172.21.15.86:0/225683371 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c15e0d0 0x7ff44c0d1a90 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 -- 172.21.15.86:0/225683371 >> 172.21.15.86:0/225683371 conn(0x7ff44c0d3ec0 msgr2=0x7ff44c0d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:59.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 -- 172.21.15.86:0/225683371 shutdown_connections 2024-03-06T07:34:59.354 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.351+0000 7ff454600640 1 -- 172.21.15.86:0/225683371 wait complete. 2024-03-06T07:34:59.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.353+0000 7ff454600640 1 Processor -- start 2024-03-06T07:34:59.356 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.353+0000 7ff454600640 1 -- start start 2024-03-06T07:34:59.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff454600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c0d0fc0 0x7ff44c155d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff454600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff44c15e0d0 0x7ff44c0c9fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff454600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff44c15f470 0x7ff44c1f75f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff454600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff44c15fb30 con 0x7ff44c15e0d0 2024-03-06T07:34:59.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff454600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff44c0d2bf0 con 0x7ff44c15f470 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff454600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff44c0d1fd0 con 0x7ff44c0d0fc0 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff451200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff44c15e0d0 0x7ff44c0c9fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff451200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff44c15e0d0 0x7ff44c0c9fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33396/0 (socket says 172.21.15.86:33396) 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff451200640 1 -- 172.21.15.86:0/3656011447 learned_addr learned my addr 172.21.15.86:0/3656011447 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.355+0000 7ff451c00640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c0d0fc0 0x7ff44c155d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff452600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff44c15f470 0x7ff44c1f75f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff451200640 1 -- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c0d0fc0 msgr2=0x7ff44c155d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff451200640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c0d0fc0 0x7ff44c155d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff451200640 1 -- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff44c15f470 msgr2=0x7ff44c1f75f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff451200640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff44c15f470 0x7ff44c1f75f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff451200640 1 -- 172.21.15.86:0/3656011447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff44c156240 con 0x7ff44c15e0d0 2024-03-06T07:34:59.359 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff451200640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff44c15e0d0 0x7ff44c0c9fc0 secure :-1 s=READY pgs=1527 cs=0 l=1 rev1=1 crypto rx=0x7ff43800c400 tx=0x7ff43800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:59.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff449600640 1 -- 172.21.15.86:0/3656011447 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4380052d0 con 0x7ff44c15e0d0 2024-03-06T07:34:59.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff449600640 1 -- 172.21.15.86:0/3656011447 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff438005490 con 0x7ff44c15e0d0 2024-03-06T07:34:59.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff44c0cd4a0 con 0x7ff44c15e0d0 2024-03-06T07:34:59.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.356+0000 7ff449600640 1 -- 172.21.15.86:0/3656011447 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff438007260 con 0x7ff44c15e0d0 2024-03-06T07:34:59.360 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.357+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff44c0cda60 con 0x7ff44c15e0d0 2024-03-06T07:34:59.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.358+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff44c0cdc60 con 0x7ff44c15e0d0 2024-03-06T07:34:59.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.359+0000 7ff449600640 1 -- 172.21.15.86:0/3656011447 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff438007400 con 0x7ff44c15e0d0 2024-03-06T07:34:59.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.359+0000 7ff449600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff42c06b4d0 0x7ff42c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.359+0000 7ff449600640 1 -- 172.21.15.86:0/3656011447 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(129..129 src has 1..129) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff4380897d0 con 0x7ff44c15e0d0 2024-03-06T07:34:59.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.360+0000 7ff451c00640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff42c06b4d0 0x7ff42c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.360+0000 7ff451c00640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff42c06b4d0 0x7ff42c06d990 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7ff43c036000 tx=0x7ff43c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:59.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.364+0000 7ff449600640 1 -- 172.21.15.86:0/3656011447 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff44c0cdc60 con 0x7ff44c15e0d0 2024-03-06T07:34:59.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.600+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff44c04f330 con 0x7ff44c15e0d0 2024-03-06T07:34:59.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.601+0000 7ff449600640 1 -- 172.21.15.86:0/3656011447 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 96 v96) v1 ==== 94+0+5001 (secure 0 0 0) 0x7ff438058600 con 0x7ff44c15e0d0 2024-03-06T07:34:59.604 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:34:59.604 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":96,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5495,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6834/374549834","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6832","nonce":374549834},{"type":"v1","addr":"172.21.15.194:6834","nonce":374549834}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88},{"gid":5498,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.194:6835/2544629493","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6833","nonce":2544629493},{"type":"v1","addr":"172.21.15.194:6835","nonce":2544629493}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":88}],"filesystems":[{"mdsmap":{"epoch":96,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:54.098168+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:active","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8148,8151]},"id":9}]} 2024-03-06T07:34:59.604 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 96 2024-03-06T07:34:59.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff42c06b4d0 msgr2=0x7ff42c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff42c06b4d0 0x7ff42c06d990 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7ff43c036000 tx=0x7ff43c030000 comp rx=0 tx=0).stop 2024-03-06T07:34:59.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff44c15e0d0 msgr2=0x7ff44c0c9fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff44c15e0d0 0x7ff44c0c9fc0 secure :-1 s=READY pgs=1527 cs=0 l=1 rev1=1 crypto rx=0x7ff43800c400 tx=0x7ff43800c8c0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 shutdown_connections 2024-03-06T07:34:59.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff42c06b4d0 0x7ff42c06d990 unknown :-1 s=CLOSED pgs=653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff44c15f470 0x7ff44c1f75f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff44c15e0d0 0x7ff44c0c9fc0 unknown :-1 s=CLOSED pgs=1527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 --2- 172.21.15.86:0/3656011447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff44c0d0fc0 0x7ff44c155d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 >> 172.21.15.86:0/3656011447 conn(0x7ff44c0d3ec0 msgr2=0x7ff44c0b4e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:59.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.604+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 shutdown_connections 2024-03-06T07:34:59.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.605+0000 7ff454600640 1 -- 172.21.15.86:0/3656011447 wait complete. 2024-03-06T07:34:59.627 INFO:tasks.cephfs.test_strays:Ranks 0 and 1 are c and a 2024-03-06T07:34:59.627 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-06T07:34:59.628 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:34:59.656 INFO:tasks.ceph.mds.b:Stopped 2024-03-06T07:34:59.656 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail b 2024-03-06T07:34:59.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.844+0000 7f63d8c00640 1 Processor -- start 2024-03-06T07:34:59.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.844+0000 7f63d8c00640 1 -- start start 2024-03-06T07:34:59.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.845+0000 7f63d8c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f63d415b070 0x7f63d406e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.845+0000 7f63d8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d40c4a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.845+0000 7f63d8c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63d40cc000 con 0x7f63d415ac90 2024-03-06T07:34:59.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.845+0000 7f63d8c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63d40ce360 con 0x7f63d415b070 2024-03-06T07:34:59.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.845+0000 7f63d8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63d40ce640 con 0x7f63d415f120 2024-03-06T07:34:59.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.846+0000 7f63d2800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d40c4a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.846+0000 7f63c9400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f63d415b070 0x7f63d406e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.846+0000 7f63d2800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d40c4a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:42226/0 (socket says 172.21.15.86:42226) 2024-03-06T07:34:59.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.846+0000 7f63c9400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f63d415b070 0x7f63d406e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:45790/0 (socket says 172.21.15.86:45790) 2024-03-06T07:34:59.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.846+0000 7f63d2800640 1 -- 172.21.15.86:0/3444925792 learned_addr learned my addr 172.21.15.86:0/3444925792 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:59.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.847+0000 7f63c9400640 1 -- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 msgr2=0x7f63d40c4a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:34:59.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.847+0000 7f63c9400640 1 --2- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d40c4a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.847+0000 7f63c9400640 1 -- 172.21.15.86:0/3444925792 >> v1:172.21.15.194:6789/0 conn(0x7f63d415ac90 legacy=0x7f63d415ea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.847+0000 7f63c9400640 1 -- 172.21.15.86:0/3444925792 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63d40ce920 con 0x7f63d415b070 2024-03-06T07:34:59.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.847+0000 7f63d2800640 1 --2- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d40c4a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:34:59.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.848+0000 7f63c9400640 1 --2- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 0x7f63d406e580 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f63bc0097a0 tx=0x7f63bc02b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2631a621725dd4a9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:59.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.849+0000 7f63d1400640 1 -- 172.21.15.86:0/3444925792 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63bc031780 con 0x7f63d415b070 2024-03-06T07:34:59.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.849+0000 7f63d1400640 1 -- 172.21.15.86:0/3444925792 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f63bc037070 con 0x7f63d415b070 2024-03-06T07:34:59.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.849+0000 7f63d1400640 1 -- 172.21.15.86:0/3444925792 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63bc031b40 con 0x7f63d415b070 2024-03-06T07:34:59.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 -- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 msgr2=0x7f63d406e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 --2- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 0x7f63d406e580 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f63bc0097a0 tx=0x7f63bc02b820 comp rx=0 tx=0).stop 2024-03-06T07:34:59.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 -- 172.21.15.86:0/3444925792 shutdown_connections 2024-03-06T07:34:59.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 --2- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d40c4a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 --2- 172.21.15.86:0/3444925792 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 0x7f63d406e580 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 -- 172.21.15.86:0/3444925792 >> 172.21.15.86:0/3444925792 conn(0x7f63d40cc4f0 msgr2=0x7f63d40cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:34:59.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 -- 172.21.15.86:0/3444925792 shutdown_connections 2024-03-06T07:34:59.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.850+0000 7f63d8c00640 1 -- 172.21.15.86:0/3444925792 wait complete. 2024-03-06T07:34:59.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.852+0000 7f63d8c00640 1 Processor -- start 2024-03-06T07:34:59.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.852+0000 7f63d8c00640 1 -- start start 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d8c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63d415ac90 0x7f63d40b3a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d8c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 0x7f63d406d470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d406d9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d8c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63d415f580 con 0x7f63d415ac90 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d8c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63d40ceee0 con 0x7f63d415b070 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63d415f700 con 0x7f63d415f120 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63d415ac90 0x7f63d40b3a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63d415ac90 0x7f63d40b3a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33408/0 (socket says 172.21.15.86:33408) 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d1e00640 1 -- 172.21.15.86:0/909814760 learned_addr learned my addr 172.21.15.86:0/909814760 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63c9400640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 0x7f63d406d470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.853+0000 7f63d2800640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d406d9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d1e00640 1 -- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 msgr2=0x7f63d406d9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d1e00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d406d9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d1e00640 1 -- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 msgr2=0x7f63d406d470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d1e00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 0x7f63d406d470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d1e00640 1 -- 172.21.15.86:0/909814760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63d40b5680 con 0x7f63d415ac90 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d2800640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d406d9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d1e00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63d415ac90 0x7f63d40b3a80 secure :-1 s=READY pgs=1528 cs=0 l=1 rev1=1 crypto rx=0x7f63c400c790 tx=0x7f63c400cc50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63caa00640 1 -- 172.21.15.86:0/909814760 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63c4016020 con 0x7f63d415ac90 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63caa00640 1 -- 172.21.15.86:0/909814760 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f63c4006d50 con 0x7f63d415ac90 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63caa00640 1 -- 172.21.15.86:0/909814760 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63c4003bb0 con 0x7f63d415ac90 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63d41f48b0 con 0x7f63d415ac90 2024-03-06T07:34:59.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.854+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63d40ad860 con 0x7f63d415ac90 2024-03-06T07:34:59.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.856+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f63d404f330 con 0x7f63d415ac90 2024-03-06T07:34:59.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.857+0000 7f63caa00640 1 -- 172.21.15.86:0/909814760 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f63c400a850 con 0x7f63d415ac90 2024-03-06T07:34:59.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.858+0000 7f63caa00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63b006b4d0 0x7f63b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:34:59.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.858+0000 7f63caa00640 1 -- 172.21.15.86:0/909814760 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(129..129 src has 1..129) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f63c4089eb0 con 0x7f63d415ac90 2024-03-06T07:34:59.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.858+0000 7f63c9400640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63b006b4d0 0x7f63b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:34:59.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.859+0000 7f63c9400640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63b006b4d0 0x7f63b006d990 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f63bc036000 tx=0x7f63bc03d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:34:59.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:34:59.867+0000 7f63caa00640 1 -- 172.21.15.86:0/909814760 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f63c4020020 con 0x7f63d415ac90 2024-03-06T07:35:00.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:00.167+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "b"} v 0) v1 -- 0x7f63d4071610 con 0x7f63d415ac90 2024-03-06T07:35:01.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.165+0000 7f63caa00640 1 -- 172.21.15.86:0/909814760 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "b"}]=0 failed mds gid 5495 v97) v1 ==== 95+0+0 (secure 0 0 0) 0x7f63c4019310 con 0x7f63d415ac90 2024-03-06T07:35:01.168 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 5495 2024-03-06T07:35:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63b006b4d0 msgr2=0x7f63b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63b006b4d0 0x7f63b006d990 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f63bc036000 tx=0x7f63bc03d040 comp rx=0 tx=0).stop 2024-03-06T07:35:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63d415ac90 msgr2=0x7f63d40b3a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63d415ac90 0x7f63d40b3a80 secure :-1 s=READY pgs=1528 cs=0 l=1 rev1=1 crypto rx=0x7f63c400c790 tx=0x7f63c400cc50 comp rx=0 tx=0).stop 2024-03-06T07:35:01.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 shutdown_connections 2024-03-06T07:35:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f63b006b4d0 0x7f63b006d990 unknown :-1 s=CLOSED pgs=654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f63d415f120 0x7f63d406d9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f63d415b070 0x7f63d406d470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 --2- 172.21.15.86:0/909814760 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f63d415ac90 0x7f63d40b3a80 unknown :-1 s=CLOSED pgs=1528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.169+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 >> 172.21.15.86:0/909814760 conn(0x7f63d40cc4f0 msgr2=0x7f63d4070c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.170+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 shutdown_connections 2024-03-06T07:35:01.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.170+0000 7f63d8c00640 1 -- 172.21.15.86:0/909814760 wait complete. 2024-03-06T07:35:01.192 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-06T07:35:01.193 INFO:teuthology.orchestra.run:waiting for 300 2024-03-06T07:35:01.218 INFO:tasks.ceph.mds.d:Stopped 2024-03-06T07:35:01.218 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail d 2024-03-06T07:35:01.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.419+0000 7f6d1ca00640 1 Processor -- start 2024-03-06T07:35:01.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.419+0000 7f6d1ca00640 1 -- start start 2024-03-06T07:35:01.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.420+0000 7f6d1ca00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6d18160400 0x7f6d1806afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d1ca00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6d1806b920 con 0x7f6d1813ef20 2024-03-06T07:35:01.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d1ca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6d1815f940 con 0x7f6d1806b500 2024-03-06T07:35:01.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d1ca00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6d1806e9c0 con 0x7f6d18160400 2024-03-06T07:35:01.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d15e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6d1813ef20 0x7f6d1815c350 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:41458/0 (socket says 172.21.15.86:41458) 2024-03-06T07:35:01.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d15e00640 1 -- 172.21.15.86:0/2556588705 learned_addr learned my addr 172.21.15.86:0/2556588705 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:01.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d0fe00640 1 --2- 172.21.15.86:0/2556588705 >> v2:172.21.15.194:3300/0 conn(0x7f6d18160400 0x7f6d1806afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:01.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2178804816 0 0) 0x7f6d1806b920 con 0x7f6d1813ef20 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.421+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6cfc003520 con 0x7f6d1813ef20 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3491151459 0 0) 0x7f6cfc003520 con 0x7f6d1813ef20 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 >> v2:172.21.15.194:3300/0 conn(0x7f6d18160400 msgr2=0x7f6d1806afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 --2- 172.21.15.86:0/2556588705 >> v2:172.21.15.194:3300/0 conn(0x7f6d18160400 0x7f6d1806afc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 >> v1:172.21.15.194:6789/0 conn(0x7f6d1806b500 legacy=0x7f6d181414b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d1815ce80 con 0x7f6d1813ef20 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f6d08001670 con 0x7f6d1813ef20 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 >> v1:172.21.15.86:6789/0 conn(0x7f6d1813ef20 legacy=0x7f6d1815c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6cfc003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cfc004800 0x7f6cfc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cfc0051c0 0x7f6cfc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cfc00c090 con 0x7f6cfc0039e0 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cfc00c210 con 0x7f6cfc0051c0 2024-03-06T07:35:01.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6cfc00c410 con 0x7f6cfc004800 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.422+0000 7f6d15e00640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6cfc003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d0fe00640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cfc004800 0x7f6cfc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d16800640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cfc0051c0 0x7f6cfc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d15e00640 1 -- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cfc004800 msgr2=0x7f6cfc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d15e00640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6cfc004800 0x7f6cfc004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d15e00640 1 -- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cfc0051c0 msgr2=0x7f6cfc00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d15e00640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cfc0051c0 0x7f6cfc00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d15e00640 1 -- 172.21.15.86:0/2556588705 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cfc003520 con 0x7f6cfc0039e0 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d16800640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6cfc0051c0 0x7f6cfc00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:01.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.423+0000 7f6d16800640 1 -- 172.21.15.86:0/2556588705 reap_dead start 2024-03-06T07:35:01.428 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.424+0000 7f6d15e00640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6cfc003e00 secure :-1 s=READY pgs=1530 cs=0 l=1 rev1=1 crypto rx=0x7f6d0800c240 tx=0x7f6cfc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a3b9141088c6f366 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.424+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d08007d10 con 0x7f6cfc0039e0 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.424+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d08030070 con 0x7f6cfc0039e0 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.424+0000 7f6d15400640 1 -- 172.21.15.86:0/2556588705 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d0802cd80 con 0x7f6cfc0039e0 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.425+0000 7f6d1ca00640 1 -- 172.21.15.86:0/2556588705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 msgr2=0x7f6cfc003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.425+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6cfc003e00 secure :-1 s=READY pgs=1530 cs=0 l=1 rev1=1 crypto rx=0x7f6d0800c240 tx=0x7f6cfc005830 comp rx=0 tx=0).stop 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.425+0000 7f6d1ca00640 1 -- 172.21.15.86:0/2556588705 shutdown_connections 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.425+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/2556588705 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6cfc003e00 unknown :-1 s=CLOSED pgs=1530 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.425+0000 7f6d1ca00640 1 -- 172.21.15.86:0/2556588705 >> 172.21.15.86:0/2556588705 conn(0x7f6d180d0b90 msgr2=0x7f6d180d0f70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.425+0000 7f6d1ca00640 1 -- 172.21.15.86:0/2556588705 shutdown_connections 2024-03-06T07:35:01.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.425+0000 7f6d1ca00640 1 -- 172.21.15.86:0/2556588705 wait complete. 2024-03-06T07:35:01.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.427+0000 7f6d1ca00640 1 Processor -- start 2024-03-06T07:35:01.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.427+0000 7f6d1ca00640 1 -- start start 2024-03-06T07:35:01.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d1ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6d1813a0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d1ca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d18140d50 0x7f6d180b1a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d1ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d1813ca50 0x7f6d180b44b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d1ca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d1815fee0 con 0x7f6cfc0039e0 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d1ca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d1815fc00 con 0x7f6d18140d50 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d1ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6d180700e0 con 0x7f6d1813ca50 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d15e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6d1813a0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d15e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6d1813a0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33420/0 (socket says 172.21.15.86:33420) 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.429+0000 7f6d15e00640 1 -- 172.21.15.86:0/1396931924 learned_addr learned my addr 172.21.15.86:0/1396931924 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d15e00640 1 -- 172.21.15.86:0/1396931924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d1813ca50 msgr2=0x7f6d180b44b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d15e00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d1813ca50 0x7f6d180b44b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d15e00640 1 -- 172.21.15.86:0/1396931924 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d18140d50 msgr2=0x7f6d180b1a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d15e00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d18140d50 0x7f6d180b1a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d15e00640 1 -- 172.21.15.86:0/1396931924 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d18141510 con 0x7f6cfc0039e0 2024-03-06T07:35:01.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d15e00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6d1813a0f0 secure :-1 s=READY pgs=1531 cs=0 l=1 rev1=1 crypto rx=0x7f6d0800c360 tx=0x7f6d1815d050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:01.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d0e000640 1 -- 172.21.15.86:0/1396931924 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d08045070 con 0x7f6cfc0039e0 2024-03-06T07:35:01.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d0e000640 1 -- 172.21.15.86:0/1396931924 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d080049f0 con 0x7f6cfc0039e0 2024-03-06T07:35:01.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d0e000640 1 -- 172.21.15.86:0/1396931924 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d08007d10 con 0x7f6cfc0039e0 2024-03-06T07:35:01.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.430+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6d181417f0 con 0x7f6cfc0039e0 2024-03-06T07:35:01.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.431+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6d18141ad0 con 0x7f6cfc0039e0 2024-03-06T07:35:01.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.432+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6d1804f330 con 0x7f6cfc0039e0 2024-03-06T07:35:01.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.433+0000 7f6d0e000640 1 -- 172.21.15.86:0/1396931924 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6d08007070 con 0x7f6cfc0039e0 2024-03-06T07:35:01.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.434+0000 7f6d0e000640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cf406b4d0 0x7f6cf406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:01.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.434+0000 7f6d0e000640 1 -- 172.21.15.86:0/1396931924 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6d080aadb0 con 0x7f6cfc0039e0 2024-03-06T07:35:01.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.434+0000 7f6d0fe00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cf406b4d0 0x7f6cf406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:01.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.435+0000 7f6d0fe00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cf406b4d0 0x7f6cf406d990 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f6d000098f0 tx=0x7f6d00008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:01.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.438+0000 7f6d0e000640 1 -- 172.21.15.86:0/1396931924 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6d080af070 con 0x7f6cfc0039e0 2024-03-06T07:35:01.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:01.710+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "d"} v 0) v1 -- 0x7f6d180d1aa0 con 0x7f6cfc0039e0 2024-03-06T07:35:02.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.169+0000 7f6d0e000640 1 -- 172.21.15.86:0/1396931924 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "d"}]=0 failed mds gid 5498 v98) v1 ==== 95+0+0 (secure 0 0 0) 0x7f6d08079590 con 0x7f6cfc0039e0 2024-03-06T07:35:02.172 INFO:teuthology.orchestra.run.smithi086.stderr:failed mds gid 5498 2024-03-06T07:35:02.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.172+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cf406b4d0 msgr2=0x7f6cf406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.172+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cf406b4d0 0x7f6cf406d990 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f6d000098f0 tx=0x7f6d00008040 comp rx=0 tx=0).stop 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.172+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 msgr2=0x7f6d1813a0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.172+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6d1813a0f0 secure :-1 s=READY pgs=1531 cs=0 l=1 rev1=1 crypto rx=0x7f6d0800c360 tx=0x7f6d1815d050 comp rx=0 tx=0).stop 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 shutdown_connections 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6cf406b4d0 0x7f6cf406d990 unknown :-1 s=CLOSED pgs=655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6d1813ca50 0x7f6d180b44b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6d18140d50 0x7f6d180b1a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 --2- 172.21.15.86:0/1396931924 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6cfc0039e0 0x7f6d1813a0f0 unknown :-1 s=CLOSED pgs=1531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 >> 172.21.15.86:0/1396931924 conn(0x7f6d180d0b90 msgr2=0x7f6d1813b010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 shutdown_connections 2024-03-06T07:35:02.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.173+0000 7f6d1ca00640 1 -- 172.21.15.86:0/1396931924 wait complete. 2024-03-06T07:35:02.201 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:35:02.201 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c ' 2024-03-06T07:35:02.201 DEBUG:teuthology.orchestra.run.smithi086:> mkdir pin 2024-03-06T07:35:02.201 DEBUG:teuthology.orchestra.run.smithi086:> touch pin/placeholder 2024-03-06T07:35:02.201 DEBUG:teuthology.orchestra.run.smithi086:> ') 2024-03-06T07:35:02.280 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:35:02.280 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec setfattr -n ceph.dir.pin -v 1 pin) 2024-03-06T07:35:02.332 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:02.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.519+0000 7f4418e00640 1 Processor -- start 2024-03-06T07:35:02.522 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.519+0000 7f4418e00640 1 -- start start 2024-03-06T07:35:02.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.520+0000 7f4418e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4414161580 0x7f44140d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.520+0000 7f4418e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4414161960 con 0x7f44140d1e90 2024-03-06T07:35:02.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.520+0000 7f4418e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f44140cf6f0 con 0x7f44140d2a70 2024-03-06T07:35:02.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.520+0000 7f4418e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f44140cf9d0 con 0x7f4414161580 2024-03-06T07:35:02.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.521+0000 7f4412a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f44140d1e90 0x7f44140d2360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:41472/0 (socket says 172.21.15.86:41472) 2024-03-06T07:35:02.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.521+0000 7f4412a00640 1 -- 172.21.15.86:0/3347857993 learned_addr learned my addr 172.21.15.86:0/3347857993 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.521+0000 7f4413400640 1 --2- 172.21.15.86:0/3347857993 >> v2:172.21.15.194:3300/0 conn(0x7f4414161580 0x7f44140d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.521+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 293012178 0 0) 0x7f4414161960 con 0x7f44140d1e90 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.521+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f43f8003520 con 0x7f44140d1e90 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1146419292 0 0) 0x7f43f8003520 con 0x7f44140d1e90 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 >> v2:172.21.15.194:3300/0 conn(0x7f4414161580 msgr2=0x7f44140d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 --2- 172.21.15.86:0/3347857993 >> v2:172.21.15.194:3300/0 conn(0x7f4414161580 0x7f44140d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 >> v1:172.21.15.194:6789/0 conn(0x7f44140d2a70 legacy=0x7f44140b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44140cfcb0 con 0x7f44140d1e90 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f4400001670 con 0x7f44140d1e90 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 >> v1:172.21.15.86:6789/0 conn(0x7f44140d1e90 legacy=0x7f44140d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43f8003a00 0x7f43f8003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8004800 0x7f43f8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43f80051c0 0x7f43f800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44140cf9d0 con 0x7f43f8003a00 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43f800c0c0 con 0x7f43f8004800 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43f800c2c0 con 0x7f43f80051c0 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.522+0000 7f4413400640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43f8003a00 0x7f43f8003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4412a00640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8004800 0x7f43f8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413e00640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43f80051c0 0x7f43f800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413400640 1 -- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43f80051c0 msgr2=0x7f43f800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413400640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43f80051c0 0x7f43f800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413400640 1 -- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8004800 msgr2=0x7f43f8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413400640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8004800 0x7f43f8004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413400640 1 -- 172.21.15.86:0/3347857993 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43f8003520 con 0x7f43f8003a00 2024-03-06T07:35:02.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413e00640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f43f80051c0 0x7f43f800b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:02.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.523+0000 7f4413e00640 1 -- 172.21.15.86:0/3347857993 reap_dead start 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.524+0000 7f4413400640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43f8003a00 0x7f43f8003e20 secure :-1 s=READY pgs=1533 cs=0 l=1 rev1=1 crypto rx=0x7f4404009ac0 tx=0x7f43f8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ecb64a98b4e15e86 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.524+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4404039070 con 0x7f43f8003a00 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.524+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f440402c6b0 con 0x7f43f8003a00 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.524+0000 7f4412000640 1 -- 172.21.15.86:0/3347857993 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4404009690 con 0x7f43f8003a00 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.525+0000 7f4418e00640 1 -- 172.21.15.86:0/3347857993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43f8003a00 msgr2=0x7f43f8003e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.525+0000 7f4418e00640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43f8003a00 0x7f43f8003e20 secure :-1 s=READY pgs=1533 cs=0 l=1 rev1=1 crypto rx=0x7f4404009ac0 tx=0x7f43f8005830 comp rx=0 tx=0).stop 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.525+0000 7f4418e00640 1 -- 172.21.15.86:0/3347857993 shutdown_connections 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.525+0000 7f4418e00640 1 --2- 172.21.15.86:0/3347857993 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f43f8003a00 0x7f43f8003e20 unknown :-1 s=CLOSED pgs=1533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.525+0000 7f4418e00640 1 -- 172.21.15.86:0/3347857993 >> 172.21.15.86:0/3347857993 conn(0x7f44140d3d80 msgr2=0x7f44140d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.525+0000 7f4418e00640 1 -- 172.21.15.86:0/3347857993 shutdown_connections 2024-03-06T07:35:02.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.525+0000 7f4418e00640 1 -- 172.21.15.86:0/3347857993 wait complete. 2024-03-06T07:35:02.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.528+0000 7f4418e00640 1 Processor -- start 2024-03-06T07:35:02.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.528+0000 7f4418e00640 1 -- start start 2024-03-06T07:35:02.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4418e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8003a00 0x7f44141520d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4418e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44140b88d0 0x7f44140b8d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4418e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f441415cb20 0x7f441415cfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4418e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44140cf3a0 con 0x7f44140b88d0 2024-03-06T07:35:02.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4418e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44140b39a0 con 0x7f43f8003a00 2024-03-06T07:35:02.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4418e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f44140d1950 con 0x7f441415cb20 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4412a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44140b88d0 0x7f44140b8d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4412a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44140b88d0 0x7f44140b8d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33432/0 (socket says 172.21.15.86:33432) 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4412a00640 1 -- 172.21.15.86:0/423411479 learned_addr learned my addr 172.21.15.86:0/423411479 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4413400640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8003a00 0x7f44141520d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4413e00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f441415cb20 0x7f441415cfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4412a00640 1 -- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f441415cb20 msgr2=0x7f441415cfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4412a00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f441415cb20 0x7f441415cfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4412a00640 1 -- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8003a00 msgr2=0x7f44141520d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.530+0000 7f4412a00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8003a00 0x7f44141520d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.531+0000 7f4412a00640 1 -- 172.21.15.86:0/423411479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44140ccc30 con 0x7f44140b88d0 2024-03-06T07:35:02.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.531+0000 7f4412a00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44140b88d0 0x7f44140b8d50 secure :-1 s=READY pgs=1534 cs=0 l=1 rev1=1 crypto rx=0x7f440000e710 tx=0x7f440000ebd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:02.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.531+0000 7f440b400640 1 -- 172.21.15.86:0/423411479 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f440000ee20 con 0x7f44140b88d0 2024-03-06T07:35:02.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.531+0000 7f440b400640 1 -- 172.21.15.86:0/423411479 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4400018070 con 0x7f44140b88d0 2024-03-06T07:35:02.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.531+0000 7f440b400640 1 -- 172.21.15.86:0/423411479 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f440000caf0 con 0x7f44140b88d0 2024-03-06T07:35:02.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.531+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44140cbb90 con 0x7f44140b88d0 2024-03-06T07:35:02.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.531+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44140cc150 con 0x7f44140b88d0 2024-03-06T07:35:02.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.532+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f441404f330 con 0x7f44140b88d0 2024-03-06T07:35:02.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.534+0000 7f440b400640 1 -- 172.21.15.86:0/423411479 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f440000cc90 con 0x7f44140b88d0 2024-03-06T07:35:02.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.535+0000 7f440b400640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43f406b4d0 0x7f43f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:02.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.535+0000 7f440b400640 1 -- 172.21.15.86:0/423411479 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f440008a130 con 0x7f44140b88d0 2024-03-06T07:35:02.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.535+0000 7f4413400640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43f406b4d0 0x7f43f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:02.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.536+0000 7f4413400640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43f406b4d0 0x7f43f406d990 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x7f4404036000 tx=0x7f4404030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:02.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.539+0000 7f440b400640 1 -- 172.21.15.86:0/423411479 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4400058f70 con 0x7f44140b88d0 2024-03-06T07:35:02.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.853+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f44140d3b60 con 0x7f44140b88d0 2024-03-06T07:35:02.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.854+0000 7f440b400640 1 -- 172.21.15.86:0/423411479 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 98 v98) v1 ==== 94+0+3417 (secure 0 0 0) 0x7f4400058870 con 0x7f44140b88d0 2024-03-06T07:35:02.857 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:02.857 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":98,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":96,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:54.098168+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:active","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8148,8151]},"id":9}]} 2024-03-06T07:35:02.857 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 98 2024-03-06T07:35:02.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.856+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43f406b4d0 msgr2=0x7f43f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43f406b4d0 0x7f43f406d990 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x7f4404036000 tx=0x7f4404030000 comp rx=0 tx=0).stop 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44140b88d0 msgr2=0x7f44140b8d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44140b88d0 0x7f44140b8d50 secure :-1 s=READY pgs=1534 cs=0 l=1 rev1=1 crypto rx=0x7f440000e710 tx=0x7f440000ebd0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 shutdown_connections 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f43f406b4d0 0x7f43f406d990 unknown :-1 s=CLOSED pgs=656 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f441415cb20 0x7f441415cfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f44140b88d0 0x7f44140b8d50 unknown :-1 s=CLOSED pgs=1534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 --2- 172.21.15.86:0/423411479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f43f8003a00 0x7f44141520d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 >> 172.21.15.86:0/423411479 conn(0x7f44140d3d80 msgr2=0x7f441406d0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 shutdown_connections 2024-03-06T07:35:02.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:02.857+0000 7f4418e00640 1 -- 172.21.15.86:0/423411479 wait complete. 2024-03-06T07:35:02.885 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json get subtrees 2024-03-06T07:35:03.100 INFO:teuthology.orchestra.run.smithi086.stdout:[{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}}] 2024-03-06T07:35:03.101 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-06T07:35:03.101 INFO:tasks.cephfs.cephfs_test_case:[] =?= [('/pin', 1)] 2024-03-06T07:35:05.102 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:05.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.312+0000 7fc0a7800640 1 Processor -- start 2024-03-06T07:35:05.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.312+0000 7fc0a7800640 1 -- start start 2024-03-06T07:35:05.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.314+0000 7fc0a7800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 0x7fc0a00d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc0a7800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc0a015f470 0x7fc0a00d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc0a7800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc0a00cea60 con 0x7fc0a00d0df0 2024-03-06T07:35:05.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc0a7800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc0a00d1260 con 0x7fc0a015f470 2024-03-06T07:35:05.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc0a7800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc0a00d2f60 con 0x7fc0a015e0d0 2024-03-06T07:35:05.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc0a5800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc0a00d0df0 0x7fc0a0160dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:41488/0 (socket says 172.21.15.86:41488) 2024-03-06T07:35:05.318 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc0a5800640 1 -- 172.21.15.86:0/1217095620 learned_addr learned my addr 172.21.15.86:0/1217095620 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc0a4e00640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 0x7fc0a00d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.315+0000 7fc09fe00640 1 --2- 172.21.15.86:0/1217095620 >> v2:172.21.15.194:3300/0 conn(0x7fc0a015f470 0x7fc0a00d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2067430244 0 0) 0x7fc0a00cea60 con 0x7fc0a00d0df0 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc088003520 con 0x7fc0a00d0df0 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3658820909 0 0) 0x7fc088003520 con 0x7fc0a00d0df0 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 msgr2=0x7fc0a00d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 0x7fc0a00d1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 >> v2:172.21.15.194:3300/0 conn(0x7fc0a015f470 msgr2=0x7fc0a00d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 --2- 172.21.15.86:0/1217095620 >> v2:172.21.15.194:3300/0 conn(0x7fc0a015f470 0x7fc0a00d08b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0a00d2350 con 0x7fc0a00d0df0 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc0980019f0 con 0x7fc0a00d0df0 2024-03-06T07:35:05.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 >> v1:172.21.15.86:6789/0 conn(0x7fc0a00d0df0 legacy=0x7fc0a0160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0880039b0 0x7fc088003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 0x7fc088004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc088004d40 0x7fc0880051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0a00d1260 con 0x7fc088004d40 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc08800b9c0 con 0x7fc0880039b0 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.316+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc08800bb70 con 0x7fc0a015e0d0 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a5800640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc088004d40 0x7fc0880051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc09fe00640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 0x7fc088004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a5800640 1 -- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 msgr2=0x7fc088004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a4e00640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0880039b0 0x7fc088003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a5800640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a015e0d0 0x7fc088004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a5800640 1 -- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0880039b0 msgr2=0x7fc088003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a5800640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0880039b0 0x7fc088003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.321 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a5800640 1 -- 172.21.15.86:0/1217095620 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc088003520 con 0x7fc088004d40 2024-03-06T07:35:05.322 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.317+0000 7fc0a4e00640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0880039b0 0x7fc088003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.318+0000 7fc0a5800640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc088004d40 0x7fc0880051c0 secure :-1 s=READY pgs=1536 cs=0 l=1 rev1=1 crypto rx=0x7fc09800a7d0 tx=0x7fc088005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=580758153434195 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.318+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc09800a3f0 con 0x7fc088004d40 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.318+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc098034630 con 0x7fc088004d40 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.318+0000 7fc09f400640 1 -- 172.21.15.86:0/1217095620 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc09802ce60 con 0x7fc088004d40 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.319+0000 7fc0a7800640 1 -- 172.21.15.86:0/1217095620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc088004d40 msgr2=0x7fc0880051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.319+0000 7fc0a7800640 1 --2- 172.21.15.86:0/1217095620 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc088004d40 0x7fc0880051c0 secure :-1 s=READY pgs=1536 cs=0 l=1 rev1=1 crypto rx=0x7fc09800a7d0 tx=0x7fc088005830 comp rx=0 tx=0).stop 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.319+0000 7fc0a5800640 1 -- 172.21.15.86:0/1217095620 reap_dead start 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.319+0000 7fc0a7800640 1 -- 172.21.15.86:0/1217095620 shutdown_connections 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.319+0000 7fc0a7800640 1 -- 172.21.15.86:0/1217095620 >> 172.21.15.86:0/1217095620 conn(0x7fc0a00d3ec0 msgr2=0x7fc0a00d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.319+0000 7fc0a7800640 1 -- 172.21.15.86:0/1217095620 shutdown_connections 2024-03-06T07:35:05.323 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.319+0000 7fc0a7800640 1 -- 172.21.15.86:0/1217095620 wait complete. 2024-03-06T07:35:05.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.322+0000 7fc0a7800640 1 Processor -- start 2024-03-06T07:35:05.324 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.322+0000 7fc0a7800640 1 -- start start 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a7800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc088004d40 0x7fc0a014aa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a7800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0a00b1120 0x7fc0a00b1580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a7800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a00c9890 0x7fc0a00c9d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a7800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0a00d2bf0 con 0x7fc0a00b1120 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a7800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0a00d2910 con 0x7fc088004d40 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a7800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc0a015fb30 con 0x7fc0a00c9890 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0a00b1120 0x7fc0a00b1580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0a00b1120 0x7fc0a00b1580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33458/0 (socket says 172.21.15.86:33458) 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 -- 172.21.15.86:0/2822302716 learned_addr learned my addr 172.21.15.86:0/2822302716 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a4e00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc088004d40 0x7fc0a014aa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.328 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 -- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a00c9890 msgr2=0x7fc0a00c9d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc0a5800640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a00c9890 0x7fc0a00c9d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a00c9890 0x7fc0a00c9d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 -- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc088004d40 msgr2=0x7fc0a014aa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc088004d40 0x7fc0a014aa10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.324+0000 7fc09fe00640 1 -- 172.21.15.86:0/2822302716 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0a00b1be0 con 0x7fc0a00b1120 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.325+0000 7fc0a4e00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc088004d40 0x7fc0a014aa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.325+0000 7fc09fe00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0a00b1120 0x7fc0a00b1580 secure :-1 s=READY pgs=1537 cs=0 l=1 rev1=1 crypto rx=0x7fc08c0108e0 tx=0x7fc08c010da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.325+0000 7fc09cc00640 1 -- 172.21.15.86:0/2822302716 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc08c00f040 con 0x7fc0a00b1120 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.325+0000 7fc09cc00640 1 -- 172.21.15.86:0/2822302716 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc08c00eb60 con 0x7fc0a00b1120 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.325+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0a00b1ec0 con 0x7fc0a00b1120 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.325+0000 7fc09cc00640 1 -- 172.21.15.86:0/2822302716 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc08c01c400 con 0x7fc0a00b1120 2024-03-06T07:35:05.329 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.325+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0a00b2480 con 0x7fc0a00b1120 2024-03-06T07:35:05.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.327+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc0a004f330 con 0x7fc0a00b1120 2024-03-06T07:35:05.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.327+0000 7fc09cc00640 1 -- 172.21.15.86:0/2822302716 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc08c01c5a0 con 0x7fc0a00b1120 2024-03-06T07:35:05.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.328+0000 7fc09cc00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc08006b4d0 0x7fc08006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:05.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.328+0000 7fc09cc00640 1 -- 172.21.15.86:0/2822302716 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc08c08a9d0 con 0x7fc0a00b1120 2024-03-06T07:35:05.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.329+0000 7fc0a4e00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc08006b4d0 0x7fc08006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:05.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.329+0000 7fc0a4e00640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc08006b4d0 0x7fc08006d990 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fc094009aa0 tx=0x7fc094008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:05.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.338+0000 7fc09cc00640 1 -- 172.21.15.86:0/2822302716 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc08c018030 con 0x7fc0a00b1120 2024-03-06T07:35:05.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.574+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc0a008d920 con 0x7fc0a00b1120 2024-03-06T07:35:05.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.575+0000 7fc09cc00640 1 -- 172.21.15.86:0/2822302716 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 98 v98) v1 ==== 94+0+3417 (secure 0 0 0) 0x7fc08c059880 con 0x7fc0a00b1120 2024-03-06T07:35:05.578 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:05.578 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":98,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":96,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:54.098168+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:active","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8148,8151]},"id":9}]} 2024-03-06T07:35:05.578 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 98 2024-03-06T07:35:05.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.577+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc08006b4d0 msgr2=0x7fc08006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.577+0000 7fc0a7800640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc08006b4d0 0x7fc08006d990 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fc094009aa0 tx=0x7fc094008040 comp rx=0 tx=0).stop 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0a00b1120 msgr2=0x7fc0a00b1580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0a00b1120 0x7fc0a00b1580 secure :-1 s=READY pgs=1537 cs=0 l=1 rev1=1 crypto rx=0x7fc08c0108e0 tx=0x7fc08c010da0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 shutdown_connections 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc08006b4d0 0x7fc08006d990 unknown :-1 s=CLOSED pgs=657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0a00c9890 0x7fc0a00c9d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0a00b1120 0x7fc0a00b1580 unknown :-1 s=CLOSED pgs=1537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 --2- 172.21.15.86:0/2822302716 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc088004d40 0x7fc0a014aa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 >> 172.21.15.86:0/2822302716 conn(0x7fc0a00d3ec0 msgr2=0x7fc0a00ca820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 shutdown_connections 2024-03-06T07:35:05.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:05.578+0000 7fc0a7800640 1 -- 172.21.15.86:0/2822302716 wait complete. 2024-03-06T07:35:05.607 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json get subtrees 2024-03-06T07:35:05.815 INFO:teuthology.orchestra.run.smithi086.stdout:[{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}}] 2024-03-06T07:35:05.816 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-06T07:35:05.816 INFO:tasks.cephfs.cephfs_test_case:[] =?= [('/pin', 1)] 2024-03-06T07:35:07.817 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:08.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.019+0000 7febe3200640 1 Processor -- start 2024-03-06T07:35:08.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.019+0000 7febe3200640 1 -- start start 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.020+0000 7febe3200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7febe415e200 0x7febe40d4850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe3200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7febe40d0c00 0x7febe40b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe3200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7febe40b5240 con 0x7febe40b2e50 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe3200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7febe40d2ec0 con 0x7febe40d0c00 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe3200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7febe40cf4d0 con 0x7febe415e200 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7febe40d0c00 0x7febe40b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe1400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7febe40b2e50 0x7febe40d04f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42258/0 (socket says 172.21.15.86:42258) 2024-03-06T07:35:08.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe1400640 1 -- 172.21.15.86:0/1568356468 learned_addr learned my addr 172.21.15.86:0/1568356468 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:08.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.021+0000 7febe1e00640 1 --2- 172.21.15.86:0/1568356468 >> v2:172.21.15.194:3300/0 conn(0x7febe415e200 0x7febe40d4850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:08.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.022+0000 7febe2800640 1 -- 172.21.15.86:0/1568356468 >> v2:172.21.15.194:3300/0 conn(0x7febe415e200 msgr2=0x7febe40d4850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:08.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.022+0000 7febe2800640 1 --2- 172.21.15.86:0/1568356468 >> v2:172.21.15.194:3300/0 conn(0x7febe415e200 0x7febe40d4850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.022+0000 7febe2800640 1 -- 172.21.15.86:0/1568356468 >> v1:172.21.15.86:6789/0 conn(0x7febe40b2e50 legacy=0x7febe40d04f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:08.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.022+0000 7febe0a00640 1 -- 172.21.15.86:0/1568356468 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2783561685 0 0) 0x7febe40b5240 con 0x7febe40b2e50 2024-03-06T07:35:08.025 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.022+0000 7febe2800640 1 -- 172.21.15.86:0/1568356468 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7febe40b5500 con 0x7febe40d0c00 2024-03-06T07:35:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.023+0000 7febe2800640 1 --2- 172.21.15.86:0/1568356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe40d0c00 0x7febe40b3640 secure :-1 s=READY pgs=1539 cs=0 l=1 rev1=1 crypto rx=0x7febe40d3d00 tx=0x7febdc02bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=65e2bb20794d69a6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.024+0000 7febe0a00640 1 -- 172.21.15.86:0/1568356468 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febdc039070 con 0x7febe40d0c00 2024-03-06T07:35:08.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.024+0000 7febe0a00640 1 -- 172.21.15.86:0/1568356468 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7febdc02cd10 con 0x7febe40d0c00 2024-03-06T07:35:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.024+0000 7febe0a00640 1 -- 172.21.15.86:0/1568356468 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febdc034660 con 0x7febe40d0c00 2024-03-06T07:35:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 -- 172.21.15.86:0/1568356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe40d0c00 msgr2=0x7febe40b3640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 --2- 172.21.15.86:0/1568356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe40d0c00 0x7febe40b3640 secure :-1 s=READY pgs=1539 cs=0 l=1 rev1=1 crypto rx=0x7febe40d3d00 tx=0x7febdc02bf60 comp rx=0 tx=0).stop 2024-03-06T07:35:08.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 -- 172.21.15.86:0/1568356468 shutdown_connections 2024-03-06T07:35:08.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 --2- 172.21.15.86:0/1568356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe40d0c00 0x7febe40b3640 unknown :-1 s=CLOSED pgs=1539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 --2- 172.21.15.86:0/1568356468 >> v2:172.21.15.194:3300/0 conn(0x7febe415e200 0x7febe40d4850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 -- 172.21.15.86:0/1568356468 >> 172.21.15.86:0/1568356468 conn(0x7febe40d3d30 msgr2=0x7febe40d4140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:08.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 -- 172.21.15.86:0/1568356468 shutdown_connections 2024-03-06T07:35:08.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.025+0000 7febe3200640 1 -- 172.21.15.86:0/1568356468 wait complete. 2024-03-06T07:35:08.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.027+0000 7febe3200640 1 Processor -- start 2024-03-06T07:35:08.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.027+0000 7febe3200640 1 -- start start 2024-03-06T07:35:08.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe3200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febe40b2e50 0x7febe415c970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:08.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febe40d0c00 0x7febe40cd3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:08.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe3200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe415e200 0x7febe40b8580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:08.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe3200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7febe40cfd60 con 0x7febe415e200 2024-03-06T07:35:08.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe3200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7febe40cfa80 con 0x7febe40b2e50 2024-03-06T07:35:08.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7febe415e5e0 con 0x7febe40d0c00 2024-03-06T07:35:08.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe415e200 0x7febe40b8580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe415e200 0x7febe40b8580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43186/0 (socket says 172.21.15.86:43186) 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe2800640 1 -- 172.21.15.86:0/3310942419 learned_addr learned my addr 172.21.15.86:0/3310942419 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe1400640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febe40d0c00 0x7febe40cd3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.029+0000 7febe1e00640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febe40b2e50 0x7febe415c970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febe2800640 1 -- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febe40d0c00 msgr2=0x7febe40cd3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febe2800640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febe40d0c00 0x7febe40cd3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febe2800640 1 -- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febe40b2e50 msgr2=0x7febe415c970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febe2800640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febe40b2e50 0x7febe415c970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febe2800640 1 -- 172.21.15.86:0/3310942419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7febe40b6aa0 con 0x7febe415e200 2024-03-06T07:35:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febe2800640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe415e200 0x7febe40b8580 secure :-1 s=READY pgs=1540 cs=0 l=1 rev1=1 crypto rx=0x7febdc037040 tx=0x7febdc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febda000640 1 -- 172.21.15.86:0/3310942419 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febdc039050 con 0x7febe415e200 2024-03-06T07:35:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febda000640 1 -- 172.21.15.86:0/3310942419 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7febdc045070 con 0x7febe415e200 2024-03-06T07:35:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febda000640 1 -- 172.21.15.86:0/3310942419 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febdc03e830 con 0x7febe415e200 2024-03-06T07:35:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.030+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7febe40b6d80 con 0x7febe415e200 2024-03-06T07:35:08.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.031+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7febe40b7340 con 0x7febe415e200 2024-03-06T07:35:08.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.032+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7febe415ddb0 con 0x7febe415e200 2024-03-06T07:35:08.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.033+0000 7febda000640 1 -- 172.21.15.86:0/3310942419 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7febdc03e9d0 con 0x7febe415e200 2024-03-06T07:35:08.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.035+0000 7febda000640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febc406b500 0x7febc406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:08.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.035+0000 7febda000640 1 -- 172.21.15.86:0/3310942419 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7febdc0ac8a0 con 0x7febe415e200 2024-03-06T07:35:08.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.035+0000 7febe1e00640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febc406b500 0x7febc406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:08.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.036+0000 7febe1e00640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febc406b500 0x7febc406d9c0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7febd4006fd0 tx=0x7febd4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:08.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.040+0000 7febda000640 1 -- 172.21.15.86:0/3310942419 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7febe415ddb0 con 0x7febe415e200 2024-03-06T07:35:08.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.343+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7febe406d210 con 0x7febe415e200 2024-03-06T07:35:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.345+0000 7febda000640 1 -- 172.21.15.86:0/3310942419 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 98 v98) v1 ==== 94+0+3417 (secure 0 0 0) 0x7febe406d210 con 0x7febe415e200 2024-03-06T07:35:08.347 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:08.347 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":98,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":96,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:34:54.098168+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:active","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8148,8151]},"id":9}]} 2024-03-06T07:35:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 98 2024-03-06T07:35:08.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febc406b500 msgr2=0x7febc406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:08.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febc406b500 0x7febc406d9c0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7febd4006fd0 tx=0x7febd4008040 comp rx=0 tx=0).stop 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe415e200 msgr2=0x7febe40b8580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe415e200 0x7febe40b8580 secure :-1 s=READY pgs=1540 cs=0 l=1 rev1=1 crypto rx=0x7febdc037040 tx=0x7febdc004a40 comp rx=0 tx=0).stop 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 shutdown_connections 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febc406b500 0x7febc406d9c0 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febe415e200 0x7febe40b8580 unknown :-1 s=CLOSED pgs=1540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febe40d0c00 0x7febe40cd3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 --2- 172.21.15.86:0/3310942419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febe40b2e50 0x7febe415c970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.347+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 >> 172.21.15.86:0/3310942419 conn(0x7febe40d3d30 msgr2=0x7febe406b970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.348+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 shutdown_connections 2024-03-06T07:35:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:08.348+0000 7febe3200640 1 -- 172.21.15.86:0/3310942419 wait complete. 2024-03-06T07:35:08.369 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json get subtrees 2024-03-06T07:35:08.573 INFO:teuthology.orchestra.run.smithi086.stdout:[{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}}] 2024-03-06T07:35:08.573 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-06T07:35:08.573 INFO:tasks.cephfs.cephfs_test_case:[] =?= [('/pin', 1)] 2024-03-06T07:35:10.575 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:10.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.780+0000 7fde05e00640 1 Processor -- start 2024-03-06T07:35:10.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.780+0000 7fde05e00640 1 -- start start 2024-03-06T07:35:10.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fde05e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde0015acb0 0x7fde000b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:10.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fde05e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fde0006ec00 0x7fde000ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:10.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fde05e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fde0015b090 con 0x7fde000b4680 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fde05e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fde000b2a30 con 0x7fde0015acb0 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fde05e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fde000b4360 con 0x7fde0006ec00 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fddfea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde0015acb0 0x7fde000b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fddfea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fde0015acb0 0x7fde000b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43190/0 (socket says 172.21.15.86:43190) 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.781+0000 7fddfea00640 1 -- 172.21.15.86:0/1093136781 learned_addr learned my addr 172.21.15.86:0/1093136781 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.782+0000 7fddfea00640 1 -- 172.21.15.86:0/1093136781 >> v2:172.21.15.194:3300/0 conn(0x7fde0006ec00 msgr2=0x7fde000ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.782+0000 7fddfea00640 1 --2- 172.21.15.86:0/1093136781 >> v2:172.21.15.194:3300/0 conn(0x7fde0006ec00 0x7fde000ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.782+0000 7fddfea00640 1 -- 172.21.15.86:0/1093136781 >> v1:172.21.15.194:6789/0 conn(0x7fde000b4680 legacy=0x7fde0006e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.782+0000 7fddfea00640 1 -- 172.21.15.86:0/1093136781 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde000b2cf0 con 0x7fde0015acb0 2024-03-06T07:35:10.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.783+0000 7fddfea00640 1 --2- 172.21.15.86:0/1093136781 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde0015acb0 0x7fde000b37d0 secure :-1 s=READY pgs=1541 cs=0 l=1 rev1=1 crypto rx=0x7fde000b4e60 tx=0x7fddf002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9fce8975620d4bee server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:10.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.783+0000 7fddfe000640 1 -- 172.21.15.86:0/1093136781 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fddf0035020 con 0x7fde0015acb0 2024-03-06T07:35:10.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.783+0000 7fddfe000640 1 -- 172.21.15.86:0/1093136781 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fddf0031870 con 0x7fde0015acb0 2024-03-06T07:35:10.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.783+0000 7fddfe000640 1 -- 172.21.15.86:0/1093136781 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fddf0031c40 con 0x7fde0015acb0 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 -- 172.21.15.86:0/1093136781 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde0015acb0 msgr2=0x7fde000b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 --2- 172.21.15.86:0/1093136781 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde0015acb0 0x7fde000b37d0 secure :-1 s=READY pgs=1541 cs=0 l=1 rev1=1 crypto rx=0x7fde000b4e60 tx=0x7fddf002b910 comp rx=0 tx=0).stop 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 -- 172.21.15.86:0/1093136781 shutdown_connections 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 --2- 172.21.15.86:0/1093136781 >> v2:172.21.15.194:3300/0 conn(0x7fde0006ec00 0x7fde000ccfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 --2- 172.21.15.86:0/1093136781 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde0015acb0 0x7fde000b37d0 unknown :-1 s=CLOSED pgs=1541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 -- 172.21.15.86:0/1093136781 >> 172.21.15.86:0/1093136781 conn(0x7fde000b1730 msgr2=0x7fde000b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 -- 172.21.15.86:0/1093136781 shutdown_connections 2024-03-06T07:35:10.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.784+0000 7fde05e00640 1 -- 172.21.15.86:0/1093136781 wait complete. 2024-03-06T07:35:10.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.786+0000 7fde05e00640 1 Processor -- start 2024-03-06T07:35:10.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.786+0000 7fde05e00640 1 -- start start 2024-03-06T07:35:10.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.787+0000 7fde05e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde0006ec00 0x7fde0015ce90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:10.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.787+0000 7fde05e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000b4680 0x7fde000cdb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:10.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.787+0000 7fde05e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde0015acb0 0x7fde000ce090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.787+0000 7fde05e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde000b3d10 con 0x7fde000b4680 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.787+0000 7fde05e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde000b32b0 con 0x7fde0006ec00 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.787+0000 7fde05e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fde000b4010 con 0x7fde0015acb0 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000b4680 0x7fde000cdb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000b4680 0x7fde000cdb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43206/0 (socket says 172.21.15.86:43206) 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 -- 172.21.15.86:0/2313085878 learned_addr learned my addr 172.21.15.86:0/2313085878 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddfea00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde0006ec00 0x7fde0015ce90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddff400640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde0015acb0 0x7fde000ce090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 -- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde0015acb0 msgr2=0x7fde000ce090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde0015acb0 0x7fde000ce090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 -- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde0006ec00 msgr2=0x7fde0015ce90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde0006ec00 0x7fde0015ce90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddf7e00640 1 -- 172.21.15.86:0/2313085878 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde0015d670 con 0x7fde000b4680 2024-03-06T07:35:10.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.788+0000 7fddfea00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde0006ec00 0x7fde0015ce90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.789+0000 7fddf7e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000b4680 0x7fde000cdb50 secure :-1 s=READY pgs=1542 cs=0 l=1 rev1=1 crypto rx=0x7fdde800c410 tx=0x7fdde800c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.789+0000 7fddf7400640 1 -- 172.21.15.86:0/2313085878 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdde8017070 con 0x7fde000b4680 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.789+0000 7fddf7400640 1 -- 172.21.15.86:0/2313085878 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdde80035f0 con 0x7fde000b4680 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.789+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde00146f90 con 0x7fde000b4680 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.789+0000 7fddf7400640 1 -- 172.21.15.86:0/2313085878 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdde8003a20 con 0x7fde000b4680 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.789+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde00147230 con 0x7fde000b4680 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.790+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde00067830 con 0x7fde000b4680 2024-03-06T07:35:10.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.791+0000 7fddf7400640 1 -- 172.21.15.86:0/2313085878 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdde801a070 con 0x7fde000b4680 2024-03-06T07:35:10.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.793+0000 7fddf7400640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdddc06b4d0 0x7fdddc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:10.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.793+0000 7fddf7400640 1 -- 172.21.15.86:0/2313085878 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fdde8089730 con 0x7fde000b4680 2024-03-06T07:35:10.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.793+0000 7fddfea00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdddc06b4d0 0x7fdddc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:10.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.794+0000 7fddfea00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdddc06b4d0 0x7fdddc06d990 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7fddf0036000 tx=0x7fddf003c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:10.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:10.796+0000 7fddf7400640 1 -- 172.21.15.86:0/2313085878 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fde00067830 con 0x7fde000b4680 2024-03-06T07:35:11.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.032+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fde0004f330 con 0x7fde000b4680 2024-03-06T07:35:11.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.033+0000 7fddf7400640 1 -- 172.21.15.86:0/2313085878 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 99 v99) v1 ==== 94+0+3418 (secure 0 0 0) 0x7fdde8058560 con 0x7fde000b4680 2024-03-06T07:35:11.036 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:11.036 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":99,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":99,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:08.574422+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:active","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151,8148]},"id":9}]} 2024-03-06T07:35:11.037 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 99 2024-03-06T07:35:11.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdddc06b4d0 msgr2=0x7fdddc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:11.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdddc06b4d0 0x7fdddc06d990 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7fddf0036000 tx=0x7fddf003c040 comp rx=0 tx=0).stop 2024-03-06T07:35:11.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000b4680 msgr2=0x7fde000cdb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:11.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000b4680 0x7fde000cdb50 secure :-1 s=READY pgs=1542 cs=0 l=1 rev1=1 crypto rx=0x7fdde800c410 tx=0x7fdde800c8d0 comp rx=0 tx=0).stop 2024-03-06T07:35:11.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 shutdown_connections 2024-03-06T07:35:11.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdddc06b4d0 0x7fdddc06d990 unknown :-1 s=CLOSED pgs=659 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:11.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fde0015acb0 0x7fde000ce090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:11.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fde000b4680 0x7fde000cdb50 unknown :-1 s=CLOSED pgs=1542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:11.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 --2- 172.21.15.86:0/2313085878 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fde0006ec00 0x7fde0015ce90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:11.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 >> 172.21.15.86:0/2313085878 conn(0x7fde000b1730 msgr2=0x7fde000b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:11.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 shutdown_connections 2024-03-06T07:35:11.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:11.037+0000 7fde05e00640 1 -- 172.21.15.86:0/2313085878 wait complete. 2024-03-06T07:35:11.062 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json get subtrees 2024-03-06T07:35:11.315 INFO:teuthology.orchestra.run.smithi086.stdout:[{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}},{"is_auth":false,"auth_first":0,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"","dirfrag":"0x1","snapid_first":2,"projected_version":"0","version":"8","committing_version":"0","committed_version":"0","is_rep":true,"dir_auth":"0","states":[],"is_auth":false,"auth_state":{"replicas":{}},"replica_state":{"authority":[0,-2],"replica_nonce":1},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1},"nref":2}},{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":1,"dir":{"path":"/pin","dirfrag":"0x10000000000","snapid_first":2,"projected_version":"3","version":"3","committing_version":"0","committed_version":"0","is_rep":false,"dir_auth":"1","states":["auth","dirty","complete"],"is_auth":true,"auth_state":{"replicas":{"0":1}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"ptrwaiter":0,"child":1,"frozen":0,"subtree":1,"importing":0,"replicated":1,"dirty":1},"nref":4}}] 2024-03-06T07:35:11.315 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false }, { "auth_first": 0, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "0", "committing_version": "0", "dir_auth": "0", "dirfrag": "0x1", "is_auth": false, "is_freezing": false, "is_frozen": false, "is_rep": true, "nref": 2, "path": "", "pins": { "child": 1, "subtree": 1 }, "projected_version": "0", "replica_state": { "authority": [ 0, -2 ], "replica_nonce": 1 }, "snapid_first": 2, "states": [], "version": "8" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": false, "random_ephemeral_pin": false }, { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": { "0": 1 } }, "committed_version": "0", "committing_version": "0", "dir_auth": "1", "dirfrag": "0x10000000000", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 4, "path": "/pin", "pins": { "child": 1, "dirty": 1, "frozen": 0, "importing": 0, "ptrwaiter": 0, "replicated": 1, "subtree": 1 }, "projected_version": "3", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "dirty", "complete" ], "version": "3" }, "distributed_ephemeral_pin": false, "export_pin": 1, "export_pin_target": 1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-06T07:35:11.315 INFO:tasks.cephfs.cephfs_test_case:[('/pin', 1)] =?= [('/pin', 1)] 2024-03-06T07:35:11.316 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:35:11.316 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> stdin-killer -- python3 -c ' 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> import fcntl 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> import sys 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> import time 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> import os 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> fcntl.fcntl(sys.stdin, fcntl.F_SETFL, fcntl.fcntl(sys.stdin, fcntl.F_GETFL) | os.O_NONBLOCK) 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> os.mkdir("/home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked") 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> fd = os.open("/home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked", os.O_RDONLY) 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> while True: 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> print("open_dir_background: keeping dir open", file=sys.stderr) 2024-03-06T07:35:11.362 DEBUG:teuthology.orchestra.run.smithi086:> try: 2024-03-06T07:35:11.363 DEBUG:teuthology.orchestra.run.smithi086:> if os.read(0, 4096) == b"": 2024-03-06T07:35:11.363 DEBUG:teuthology.orchestra.run.smithi086:> break 2024-03-06T07:35:11.363 DEBUG:teuthology.orchestra.run.smithi086:> except BlockingIOError: 2024-03-06T07:35:11.363 DEBUG:teuthology.orchestra.run.smithi086:> pass 2024-03-06T07:35:11.363 DEBUG:teuthology.orchestra.run.smithi086:> time.sleep(2) 2024-03-06T07:35:11.363 DEBUG:teuthology.orchestra.run.smithi086:> ' 2024-03-06T07:35:11.407 DEBUG:teuthology.orchestra.run.smithi086:> stat /home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked 2024-03-06T07:35:11.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-06T07:35:11.451 INFO:teuthology.orchestra.run.smithi086.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked': No such file or directory 2024-03-06T07:35:11.524 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:12.451 DEBUG:teuthology.orchestra.run.smithi086:> stat /home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked 2024-03-06T07:35:12.504 INFO:teuthology.orchestra.run.smithi086.stdout: File: /home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 0 Blocks: 0 IO Block: 65536 directory 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 29h/41d Inode: 2199023255552 Links: 2 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0755/drwxr-xr-x) Uid: ( 1000/ ubuntu) Gid: ( 1301/ ubuntu) 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout:Context: unconfined_u:object_r:unlabeled_t:s0 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-03-06 07:35:11.517641549 +0000 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-03-06 07:35:11.517641549 +0000 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-03-06 07:35:11.517641549 +0000 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-03-06 07:35:11.517641549 +0000 2024-03-06T07:35:12.505 INFO:tasks.cephfs.mount:File pin/to-be-unlinked became visible from 0 after 1s 2024-03-06T07:35:12.505 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:35:12.505 DEBUG:teuthology.orchestra.run.smithi086:> (cd /home/ubuntu/cephtest/mnt.0 && exec rmdir pin/to-be-unlinked) 2024-03-06T07:35:12.552 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache num_strays 2024-03-06T07:35:12.816 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"num_strays":1}} 2024-03-06T07:35:12.816 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 1 } } 2024-03-06T07:35:12.816 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_created 2024-03-06T07:35:13.112 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_created":1}} 2024-03-06T07:35:13.113 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 1 } } 2024-03-06T07:35:13.113 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_enqueued 2024-03-06T07:35:13.335 INFO:teuthology.orchestra.run.smithi086.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-06T07:35:13.335 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-06T07:35:13.336 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json flush journal 2024-03-06T07:35:13.526 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:13.575 INFO:teuthology.orchestra.run.smithi086.stdout:{"message":"","return_code":0} 2024-03-06T07:35:13.575 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-06T07:35:13.576 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json cache drop 2024-03-06T07:35:13.790 INFO:teuthology.orchestra.run.smithi086.stdout:{"client_recall":{"return_code":0,"message":"(0) Success","recalled":0},"flush_journal":{"return_code":0,"message":""},"trim_cache":{"trimmed":0},"cache":{"pool":{"items":378,"bytes":65362}},"duration":0.013278145999999999} 2024-03-06T07:35:13.790 DEBUG:tasks.cephfs.filesystem:_json_asok output { "cache": { "pool": { "bytes": 65362, "items": 378 } }, "client_recall": { "message": "(0) Success", "recalled": 0, "return_code": 0 }, "duration": 0.013278146, "flush_journal": { "message": "", "return_code": 0 }, "trim_cache": { "trimmed": 0 } } 2024-03-06T07:35:13.790 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 1 2024-03-06T07:35:13.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.984+0000 7f75a9c00640 1 Processor -- start 2024-03-06T07:35:13.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.984+0000 7f75a9c00640 1 -- start start 2024-03-06T07:35:13.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.985+0000 7f75a9c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75a40d23c0 0x7f75a40d2850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:13.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.985+0000 7f75a9c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75a40d1e60 con 0x7f75a41616e0 2024-03-06T07:35:13.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.985+0000 7f75a9c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75a40cee60 con 0x7f75a40b31d0 2024-03-06T07:35:13.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a9c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75a40cf140 con 0x7f75a40d23c0 2024-03-06T07:35:13.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a3400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f75a40d23c0 0x7f75a40d2850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:13.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a3e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f75a41616e0 0x7f75a40d4700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42270/0 (socket says 172.21.15.86:42270) 2024-03-06T07:35:13.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a3e00640 1 -- 172.21.15.86:0/4214738504 learned_addr learned my addr 172.21.15.86:0/4214738504 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:13.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a2a00640 1 -- 172.21.15.86:0/4214738504 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2140782086 0 0) 0x7f75a40d1e60 con 0x7f75a41616e0 2024-03-06T07:35:13.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a2a00640 1 -- 172.21.15.86:0/4214738504 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7588003520 con 0x7f75a41616e0 2024-03-06T07:35:13.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a3400640 1 -- 172.21.15.86:0/4214738504 >> v1:172.21.15.194:6789/0 conn(0x7f75a40b31d0 legacy=0x7f75a40b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:13.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a3400640 1 -- 172.21.15.86:0/4214738504 >> v1:172.21.15.86:6789/0 conn(0x7f75a41616e0 legacy=0x7f75a40d4700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:13.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.986+0000 7f75a3400640 1 -- 172.21.15.86:0/4214738504 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75a40cf420 con 0x7f75a40d23c0 2024-03-06T07:35:13.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.988+0000 7f75a3400640 1 --2- 172.21.15.86:0/4214738504 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40d2850 secure :-1 s=READY pgs=1544 cs=0 l=1 rev1=1 crypto rx=0x7f75900097a0 tx=0x7f759002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=82e4a4e6d67befef server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.988+0000 7f75a2a00640 1 -- 172.21.15.86:0/4214738504 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7590035020 con 0x7f75a40d23c0 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.988+0000 7f75a2a00640 1 -- 172.21.15.86:0/4214738504 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7590031830 con 0x7f75a40d23c0 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.988+0000 7f75a2a00640 1 -- 172.21.15.86:0/4214738504 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7590031c00 con 0x7f75a40d23c0 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.988+0000 7f75a9c00640 1 -- 172.21.15.86:0/4214738504 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 msgr2=0x7f75a40d2850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.988+0000 7f75a9c00640 1 --2- 172.21.15.86:0/4214738504 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40d2850 secure :-1 s=READY pgs=1544 cs=0 l=1 rev1=1 crypto rx=0x7f75900097a0 tx=0x7f759002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.989+0000 7f75a9c00640 1 -- 172.21.15.86:0/4214738504 shutdown_connections 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.989+0000 7f75a9c00640 1 --2- 172.21.15.86:0/4214738504 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40d2850 unknown :-1 s=CLOSED pgs=1544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.989+0000 7f75a9c00640 1 -- 172.21.15.86:0/4214738504 >> 172.21.15.86:0/4214738504 conn(0x7f75a409d050 msgr2=0x7f75a40d3de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.989+0000 7f75a9c00640 1 -- 172.21.15.86:0/4214738504 shutdown_connections 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.989+0000 7f75a9c00640 1 -- 172.21.15.86:0/4214738504 wait complete. 2024-03-06T07:35:13.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.990+0000 7f75a9c00640 1 Processor -- start 2024-03-06T07:35:13.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.991+0000 7f75a9c00640 1 -- start start 2024-03-06T07:35:13.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.991+0000 7f75a9c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75a40b31d0 0x7f75a41f7470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:13.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.991+0000 7f75a9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40b2040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:13.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.991+0000 7f75a9c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75a41616e0 0x7f75a4159280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:13.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.991+0000 7f75a9c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75a4161ac0 con 0x7f75a40d23c0 2024-03-06T07:35:13.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.991+0000 7f75a9c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75a40cf9e0 con 0x7f75a41616e0 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.991+0000 7f75a9c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75a4161c40 con 0x7f75a40b31d0 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40b2040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40b2040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43228/0 (socket says 172.21.15.86:43228) 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 -- 172.21.15.86:0/990866725 learned_addr learned my addr 172.21.15.86:0/990866725 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3e00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75a40b31d0 0x7f75a41f7470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a9200640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75a41616e0 0x7f75a4159280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 -- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75a40b31d0 msgr2=0x7f75a41f7470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75a40b31d0 0x7f75a41f7470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 -- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75a41616e0 msgr2=0x7f75a4159280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75a41616e0 0x7f75a4159280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.992+0000 7f75a3400640 1 -- 172.21.15.86:0/990866725 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75a40cca40 con 0x7f75a40d23c0 2024-03-06T07:35:13.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.993+0000 7f75a3400640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40b2040 secure :-1 s=READY pgs=1545 cs=0 l=1 rev1=1 crypto rx=0x7f75900098c0 tx=0x7f75900046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:13.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.993+0000 7f759be00640 1 -- 172.21.15.86:0/990866725 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7590041070 con 0x7f75a40d23c0 2024-03-06T07:35:13.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.993+0000 7f759be00640 1 -- 172.21.15.86:0/990866725 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f759003b920 con 0x7f75a40d23c0 2024-03-06T07:35:13.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.993+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f75a41f7be0 con 0x7f75a40d23c0 2024-03-06T07:35:13.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.993+0000 7f759be00640 1 -- 172.21.15.86:0/990866725 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7590035040 con 0x7f75a40d23c0 2024-03-06T07:35:13.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.993+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f75a41f81a0 con 0x7f75a40d23c0 2024-03-06T07:35:13.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.994+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f75a4067ae0 con 0x7f75a40d23c0 2024-03-06T07:35:13.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.995+0000 7f759be00640 1 -- 172.21.15.86:0/990866725 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f759004c060 con 0x7f75a40d23c0 2024-03-06T07:35:13.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.996+0000 7f759be00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f758406b4d0 0x7f758406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:13.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.996+0000 7f759be00640 1 -- 172.21.15.86:0/990866725 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f75900abbe0 con 0x7f75a40d23c0 2024-03-06T07:35:13.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.996+0000 7f75a3e00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f758406b4d0 0x7f758406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:14.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:13.997+0000 7f75a3e00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f758406b4d0 0x7f758406d990 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f7594005880 tx=0x7f7594009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:14.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:14.001+0000 7f759be00640 1 -- 172.21.15.86:0/990866725 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f75a4067ae0 con 0x7f75a40d23c0 2024-03-06T07:35:14.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:14.327+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "1"} v 0) v1 -- 0x7f75a40ce9f0 con 0x7f75a40d23c0 2024-03-06T07:35:15.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.228+0000 7f759be00640 1 -- 172.21.15.86:0/990866725 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "1"}]=0 v100) v1 ==== 105+0+0 (secure 0 0 0) 0x7f75a40ce9f0 con 0x7f75a40d23c0 2024-03-06T07:35:15.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.231+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f758406b4d0 msgr2=0x7f758406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.231+0000 7f75a9c00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f758406b4d0 0x7f758406d990 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f7594005880 tx=0x7f7594009040 comp rx=0 tx=0).stop 2024-03-06T07:35:15.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 msgr2=0x7f75a40b2040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40b2040 secure :-1 s=READY pgs=1545 cs=0 l=1 rev1=1 crypto rx=0x7f75900098c0 tx=0x7f75900046c0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 shutdown_connections 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f758406b4d0 0x7f758406d990 unknown :-1 s=CLOSED pgs=660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75a41616e0 0x7f75a4159280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75a40d23c0 0x7f75a40b2040 unknown :-1 s=CLOSED pgs=1545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 --2- 172.21.15.86:0/990866725 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75a40b31d0 0x7f75a41f7470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 >> 172.21.15.86:0/990866725 conn(0x7f75a409d050 msgr2=0x7f75a40d4700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 shutdown_connections 2024-03-06T07:35:15.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.232+0000 7f75a9c00640 1 -- 172.21.15.86:0/990866725 wait complete. 2024-03-06T07:35:15.259 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:15.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.461+0000 7f3c8f600640 1 Processor -- start 2024-03-06T07:35:15.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.461+0000 7f3c8f600640 1 -- start start 2024-03-06T07:35:15.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.462+0000 7f3c8f600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3c88144c00 0x7f3c8806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.462+0000 7f3c8f600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.462+0000 7f3c8f600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c8806ee70 con 0x7f3c880ce0a0 2024-03-06T07:35:15.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.462+0000 7f3c8f600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c88160370 con 0x7f3c88144c00 2024-03-06T07:35:15.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.462+0000 7f3c8f600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8806cef0 con 0x7f3c8815f9d0 2024-03-06T07:35:15.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.463+0000 7f3c8d600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3c880ce0a0 0x7f3c880ce4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:42280/0 (socket says 172.21.15.86:42280) 2024-03-06T07:35:15.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.463+0000 7f3c8d600640 1 -- 172.21.15.86:0/3890616499 learned_addr learned my addr 172.21.15.86:0/3890616499 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:15.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.463+0000 7f3c87e00640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:15.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.463+0000 7f3c8cc00640 1 --2- 172.21.15.86:0/3890616499 >> v2:172.21.15.194:3300/0 conn(0x7f3c88144c00 0x7f3c8806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:15.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.463+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3685348037 0 0) 0x7f3c80001490 con 0x7f3c880ce0a0 2024-03-06T07:35:15.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.463+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3c6c003520 con 0x7f3c880ce0a0 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3121734317 0 0) 0x7f3c6c003520 con 0x7f3c880ce0a0 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 msgr2=0x7f3c8815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c8815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 >> v2:172.21.15.194:3300/0 conn(0x7f3c88144c00 msgr2=0x7f3c8806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 --2- 172.21.15.86:0/3890616499 >> v2:172.21.15.194:3300/0 conn(0x7f3c88144c00 0x7f3c8806e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c8806d1d0 con 0x7f3c880ce0a0 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3c80001bc0 con 0x7f3c880ce0a0 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 >> v1:172.21.15.86:6789/0 conn(0x7f3c880ce0a0 legacy=0x7f3c880ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c6c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c6c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c6c004d40 0x7f3c6c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8806cef0 con 0x7f3c6c003f30 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c6c00b9c0 con 0x7f3c6c004d40 2024-03-06T07:35:15.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c6c00bb40 con 0x7f3c8815f9d0 2024-03-06T07:35:15.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.464+0000 7f3c87e00640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c6c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:15.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.465+0000 7f3c8cc00640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c6c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:15.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.465+0000 7f3c87e00640 1 -- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 msgr2=0x7f3c6c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:15.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.465+0000 7f3c87e00640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c8815f9d0 0x7f3c6c0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.465+0000 7f3c87e00640 1 -- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c6c004d40 msgr2=0x7f3c6c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:15.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.465+0000 7f3c87e00640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c6c004d40 0x7f3c6c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.465+0000 7f3c87e00640 1 -- 172.21.15.86:0/3890616499 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c6c003520 con 0x7f3c6c003f30 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.466+0000 7f3c87e00640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c6c004800 secure :-1 s=READY pgs=1547 cs=0 l=1 rev1=1 crypto rx=0x7f3c74009ac0 tx=0x7f3c6c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f499d5c0ecdd59d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.466+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c74039070 con 0x7f3c6c003f30 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.466+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c7402c6b0 con 0x7f3c6c003f30 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.466+0000 7f3c87400640 1 -- 172.21.15.86:0/3890616499 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c74009690 con 0x7f3c6c003f30 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.466+0000 7f3c8f600640 1 -- 172.21.15.86:0/3890616499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 msgr2=0x7f3c6c004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.467+0000 7f3c8f600640 1 --2- 172.21.15.86:0/3890616499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c6c004800 secure :-1 s=READY pgs=1547 cs=0 l=1 rev1=1 crypto rx=0x7f3c74009ac0 tx=0x7f3c6c005830 comp rx=0 tx=0).stop 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.467+0000 7f3c8d600640 1 -- 172.21.15.86:0/3890616499 reap_dead start 2024-03-06T07:35:15.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.467+0000 7f3c8f600640 1 -- 172.21.15.86:0/3890616499 shutdown_connections 2024-03-06T07:35:15.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.467+0000 7f3c8f600640 1 -- 172.21.15.86:0/3890616499 >> 172.21.15.86:0/3890616499 conn(0x7f3c880b1730 msgr2=0x7f3c880b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:15.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.467+0000 7f3c8f600640 1 -- 172.21.15.86:0/3890616499 shutdown_connections 2024-03-06T07:35:15.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.467+0000 7f3c8f600640 1 -- 172.21.15.86:0/3890616499 wait complete. 2024-03-06T07:35:15.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.469+0000 7f3c8f600640 1 Processor -- start 2024-03-06T07:35:15.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.469+0000 7f3c8f600640 1 -- start start 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8f600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c881312f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8f600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c880ae8b0 0x7f3c881f4430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8f600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c881f7130 0x7f3c881f75b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8f600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c88160730 con 0x7f3c6c003f30 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8f600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8806d790 con 0x7f3c880ae8b0 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8f600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8806cb50 con 0x7f3c881f7130 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c881312f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c881312f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:43254/0 (socket says 172.21.15.86:43254) 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 -- 172.21.15.86:0/2618565532 learned_addr learned my addr 172.21.15.86:0/2618565532 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 -- 172.21.15.86:0/2618565532 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c881f7130 msgr2=0x7f3c881f75b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c881f7130 0x7f3c881f75b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c87e00640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c880ae8b0 0x7f3c881f4430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:15.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 -- 172.21.15.86:0/2618565532 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c880ae8b0 msgr2=0x7f3c881f4430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c880ae8b0 0x7f3c881f4430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.470+0000 7f3c8cc00640 1 -- 172.21.15.86:0/2618565532 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c880af100 con 0x7f3c6c003f30 2024-03-06T07:35:15.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.471+0000 7f3c8cc00640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c881312f0 secure :-1 s=READY pgs=1548 cs=0 l=1 rev1=1 crypto rx=0x7f3c780116d0 tx=0x7f3c78011b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:15.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.471+0000 7f3c84c00640 1 -- 172.21.15.86:0/2618565532 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c7800f9c0 con 0x7f3c6c003f30 2024-03-06T07:35:15.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.471+0000 7f3c84c00640 1 -- 172.21.15.86:0/2618565532 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c7800fb60 con 0x7f3c6c003f30 2024-03-06T07:35:15.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.471+0000 7f3c84c00640 1 -- 172.21.15.86:0/2618565532 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c78015460 con 0x7f3c6c003f30 2024-03-06T07:35:15.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.471+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c8815d530 con 0x7f3c6c003f30 2024-03-06T07:35:15.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.471+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c8815daf0 con 0x7f3c6c003f30 2024-03-06T07:35:15.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.472+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c8804f330 con 0x7f3c6c003f30 2024-03-06T07:35:15.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.474+0000 7f3c84c00640 1 -- 172.21.15.86:0/2618565532 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3c78015600 con 0x7f3c6c003f30 2024-03-06T07:35:15.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.475+0000 7f3c84c00640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6806b500 0x7f3c6806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:15.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.475+0000 7f3c84c00640 1 -- 172.21.15.86:0/2618565532 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3c7808abe0 con 0x7f3c6c003f30 2024-03-06T07:35:15.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.475+0000 7f3c87e00640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6806b500 0x7f3c6806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:15.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.476+0000 7f3c87e00640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6806b500 0x7f3c6806d9c0 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f3c74036000 tx=0x7f3c74030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:15.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.480+0000 7f3c84c00640 1 -- 172.21.15.86:0/2618565532 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3c78019030 con 0x7f3c6c003f30 2024-03-06T07:35:15.528 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:15.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.714+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3c88072890 con 0x7f3c6c003f30 2024-03-06T07:35:15.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.715+0000 7f3c84c00640 1 -- 172.21.15.86:0/2618565532 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f3c78059a20 con 0x7f3c6c003f30 2024-03-06T07:35:15.718 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:15.718 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:15.718 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:15.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6806b500 msgr2=0x7f3c6806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6806b500 0x7f3c6806d9c0 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f3c74036000 tx=0x7f3c74030000 comp rx=0 tx=0).stop 2024-03-06T07:35:15.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 msgr2=0x7f3c881312f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:15.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c881312f0 secure :-1 s=READY pgs=1548 cs=0 l=1 rev1=1 crypto rx=0x7f3c780116d0 tx=0x7f3c78011b90 comp rx=0 tx=0).stop 2024-03-06T07:35:15.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 shutdown_connections 2024-03-06T07:35:15.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3c6806b500 0x7f3c6806d9c0 unknown :-1 s=CLOSED pgs=661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3c881f7130 0x7f3c881f75b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3c880ae8b0 0x7f3c881f4430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 --2- 172.21.15.86:0/2618565532 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3c6c003f30 0x7f3c881312f0 unknown :-1 s=CLOSED pgs=1548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:15.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 >> 172.21.15.86:0/2618565532 conn(0x7f3c880b1730 msgr2=0x7f3c880afc00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:15.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 shutdown_connections 2024-03-06T07:35:15.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:15.718+0000 7f3c8f600640 1 -- 172.21.15.86:0/2618565532 wait complete. 2024-03-06T07:35:15.941 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:15.942 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:16.943 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:17.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.163+0000 7f08ea200640 1 Processor -- start 2024-03-06T07:35:17.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.163+0000 7f08ea200640 1 -- start start 2024-03-06T07:35:17.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08ea200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f08e40aef40 0x7f08e40ce090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:17.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08ea200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f08e406e1b0 con 0x7f08e406dbf0 2024-03-06T07:35:17.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08ea200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f08e40b4530 con 0x7f08e406eaa0 2024-03-06T07:35:17.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08ea200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f08e40d06a0 con 0x7f08e40aef40 2024-03-06T07:35:17.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08e9800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f08e40aef40 0x7f08e40ce090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:17.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08e9800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f08e40aef40 0x7f08e40ce090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55582/0 (socket says 172.21.15.86:55582) 2024-03-06T07:35:17.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08e3e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f08e406dbf0 0x7f08e40cd980 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40538/0 (socket says 172.21.15.86:40538) 2024-03-06T07:35:17.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.165+0000 7f08e9800640 1 -- 172.21.15.86:0/491552076 learned_addr learned my addr 172.21.15.86:0/491552076 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:17.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.166+0000 7f08e9800640 1 -- 172.21.15.86:0/491552076 >> v1:172.21.15.194:6789/0 conn(0x7f08e406eaa0 legacy=0x7f08e4159cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:17.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.166+0000 7f08e9800640 1 -- 172.21.15.86:0/491552076 >> v1:172.21.15.86:6789/0 conn(0x7f08e406dbf0 legacy=0x7f08e40cd980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:17.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.166+0000 7f08e9800640 1 -- 172.21.15.86:0/491552076 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08e40b4810 con 0x7f08e40aef40 2024-03-06T07:35:17.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.167+0000 7f08e9800640 1 --2- 172.21.15.86:0/491552076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40ce090 secure :-1 s=READY pgs=1550 cs=0 l=1 rev1=1 crypto rx=0x7f08dc009e50 tx=0x7f08dc02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a8f978d100f299ee server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:17.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.167+0000 7f08e3400640 1 -- 172.21.15.86:0/491552076 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08dc02ca80 con 0x7f08e40aef40 2024-03-06T07:35:17.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.167+0000 7f08e3400640 1 -- 172.21.15.86:0/491552076 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08dc02cc20 con 0x7f08e40aef40 2024-03-06T07:35:17.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.167+0000 7f08e3400640 1 -- 172.21.15.86:0/491552076 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08dc0343f0 con 0x7f08e40aef40 2024-03-06T07:35:17.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.168+0000 7f08ea200640 1 -- 172.21.15.86:0/491552076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 msgr2=0x7f08e40ce090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:17.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.168+0000 7f08ea200640 1 --2- 172.21.15.86:0/491552076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40ce090 secure :-1 s=READY pgs=1550 cs=0 l=1 rev1=1 crypto rx=0x7f08dc009e50 tx=0x7f08dc02bee0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.168+0000 7f08ea200640 1 -- 172.21.15.86:0/491552076 shutdown_connections 2024-03-06T07:35:17.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.168+0000 7f08ea200640 1 --2- 172.21.15.86:0/491552076 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40ce090 unknown :-1 s=CLOSED pgs=1550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.168+0000 7f08ea200640 1 -- 172.21.15.86:0/491552076 >> 172.21.15.86:0/491552076 conn(0x7f08e406afc0 msgr2=0x7f08e406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:17.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.168+0000 7f08ea200640 1 -- 172.21.15.86:0/491552076 shutdown_connections 2024-03-06T07:35:17.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.168+0000 7f08ea200640 1 -- 172.21.15.86:0/491552076 wait complete. 2024-03-06T07:35:17.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.170+0000 7f08ea200640 1 Processor -- start 2024-03-06T07:35:17.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.170+0000 7f08ea200640 1 -- start start 2024-03-06T07:35:17.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08ea200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08e406dbf0 0x7f08e414aca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:17.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08ea200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08e406eaa0 0x7f08e40af770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:17.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08ea200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40afcb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:17.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08ea200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08e40cd4f0 con 0x7f08e40aef40 2024-03-06T07:35:17.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08ea200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08e40af3b0 con 0x7f08e406eaa0 2024-03-06T07:35:17.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08ea200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f08e40ce5d0 con 0x7f08e406dbf0 2024-03-06T07:35:17.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08e9800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40afcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:17.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08e9800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40afcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55584/0 (socket says 172.21.15.86:55584) 2024-03-06T07:35:17.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.172+0000 7f08e9800640 1 -- 172.21.15.86:0/4211579563 learned_addr learned my addr 172.21.15.86:0/4211579563 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:17.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e9800640 1 -- 172.21.15.86:0/4211579563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08e406dbf0 msgr2=0x7f08e414aca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:17.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e9800640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08e406dbf0 0x7f08e414aca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e9800640 1 -- 172.21.15.86:0/4211579563 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08e406eaa0 msgr2=0x7f08e40af770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:17.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e9800640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08e406eaa0 0x7f08e40af770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e9800640 1 -- 172.21.15.86:0/4211579563 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08e4142290 con 0x7f08e40aef40 2024-03-06T07:35:17.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e9800640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40afcb0 secure :-1 s=READY pgs=1551 cs=0 l=1 rev1=1 crypto rx=0x7f08dc009e50 tx=0x7f08dc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:17.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e0c00640 1 -- 172.21.15.86:0/4211579563 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08dc043020 con 0x7f08e40aef40 2024-03-06T07:35:17.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e0c00640 1 -- 172.21.15.86:0/4211579563 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08dc03e450 con 0x7f08e40aef40 2024-03-06T07:35:17.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08e0c00640 1 -- 172.21.15.86:0/4211579563 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08dc03e840 con 0x7f08e40aef40 2024-03-06T07:35:17.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08e4142570 con 0x7f08e40aef40 2024-03-06T07:35:17.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.173+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08e4142b30 con 0x7f08e40aef40 2024-03-06T07:35:17.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.175+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f08e4147d10 con 0x7f08e40aef40 2024-03-06T07:35:17.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.176+0000 7f08e0c00640 1 -- 172.21.15.86:0/4211579563 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f08dc046020 con 0x7f08e40aef40 2024-03-06T07:35:17.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.176+0000 7f08e0c00640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f08c406b4d0 0x7f08c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:17.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.177+0000 7f08e0c00640 1 -- 172.21.15.86:0/4211579563 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f08dc0accc0 con 0x7f08e40aef40 2024-03-06T07:35:17.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.177+0000 7f08e8e00640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f08c406b4d0 0x7f08c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:17.179 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.177+0000 7f08e8e00640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f08c406b4d0 0x7f08c406d990 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f08e40ae7a0 tx=0x7f08d80079b0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:17.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.181+0000 7f08e0c00640 1 -- 172.21.15.86:0/4211579563 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f08dc0b3080 con 0x7f08e40aef40 2024-03-06T07:35:17.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.420+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f08e404f330 con 0x7f08e40aef40 2024-03-06T07:35:17.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.421+0000 7f08e0c00640 1 -- 172.21.15.86:0/4211579563 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f08dc04d030 con 0x7f08e40aef40 2024-03-06T07:35:17.423 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:17.423 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:17.423 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f08c406b4d0 msgr2=0x7f08c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f08c406b4d0 0x7f08c406d990 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f08e40ae7a0 tx=0x7f08d80079b0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 msgr2=0x7f08e40afcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40afcb0 secure :-1 s=READY pgs=1551 cs=0 l=1 rev1=1 crypto rx=0x7f08dc009e50 tx=0x7f08dc004a40 comp rx=0 tx=0).stop 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 shutdown_connections 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f08c406b4d0 0x7f08c406d990 unknown :-1 s=CLOSED pgs=662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08e40aef40 0x7f08e40afcb0 unknown :-1 s=CLOSED pgs=1551 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f08e406eaa0 0x7f08e40af770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 --2- 172.21.15.86:0/4211579563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f08e406dbf0 0x7f08e414aca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:17.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 >> 172.21.15.86:0/4211579563 conn(0x7f08e406afc0 msgr2=0x7f08e40cd980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:17.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 shutdown_connections 2024-03-06T07:35:17.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:17.424+0000 7f08ea200640 1 -- 172.21.15.86:0/4211579563 wait complete. 2024-03-06T07:35:17.530 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:17.646 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:17.646 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:18.648 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:18.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.859+0000 7f6b6e800640 1 Processor -- start 2024-03-06T07:35:18.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.859+0000 7f6b6e800640 1 -- start start 2024-03-06T07:35:18.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.861+0000 7f6b6e800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b6815acb0 0x7f6b680b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:18.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.861+0000 7f6b6e800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6b680b4680 0x7f6b6806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:18.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.861+0000 7f6b6e800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b6815b090 con 0x7f6b68072340 2024-03-06T07:35:18.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.861+0000 7f6b6e800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b680b2a30 con 0x7f6b680b4680 2024-03-06T07:35:18.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.861+0000 7f6b6e800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b680b4360 con 0x7f6b6815acb0 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b5ea00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6b680b4680 0x7f6b6806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b67400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b6815acb0 0x7f6b680b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b5ea00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6b680b4680 0x7f6b6806e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:43276/0 (socket says 172.21.15.86:43276) 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b5ea00640 1 -- 172.21.15.86:0/3094733169 learned_addr learned my addr 172.21.15.86:0/3094733169 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b5ea00640 1 -- 172.21.15.86:0/3094733169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b6815acb0 msgr2=0x7f6b680b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b5ea00640 1 --2- 172.21.15.86:0/3094733169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b6815acb0 0x7f6b680b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b5ea00640 1 -- 172.21.15.86:0/3094733169 >> v1:172.21.15.194:6789/0 conn(0x7f6b68072340 legacy=0x7f6b680ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:18.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.862+0000 7f6b5ea00640 1 -- 172.21.15.86:0/3094733169 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b680b2cf0 con 0x7f6b680b4680 2024-03-06T07:35:18.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.864+0000 7f6b5ea00640 1 --2- 172.21.15.86:0/3094733169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6806e4f0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f6b5000bb10 tx=0x7f6b5002e150 comp rx=0 tx=0).ready entity=mon.1 client_cookie=63905c87a718d7e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:18.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.864+0000 7f6b66a00640 1 -- 172.21.15.86:0/3094733169 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b5003b070 con 0x7f6b680b4680 2024-03-06T07:35:18.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.864+0000 7f6b66a00640 1 -- 172.21.15.86:0/3094733169 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b50036440 con 0x7f6b680b4680 2024-03-06T07:35:18.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.864+0000 7f6b66a00640 1 -- 172.21.15.86:0/3094733169 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b50036810 con 0x7f6b680b4680 2024-03-06T07:35:18.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 -- 172.21.15.86:0/3094733169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 msgr2=0x7f6b6806e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 --2- 172.21.15.86:0/3094733169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6806e4f0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f6b5000bb10 tx=0x7f6b5002e150 comp rx=0 tx=0).stop 2024-03-06T07:35:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 -- 172.21.15.86:0/3094733169 shutdown_connections 2024-03-06T07:35:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 --2- 172.21.15.86:0/3094733169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6806e4f0 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 --2- 172.21.15.86:0/3094733169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b6815acb0 0x7f6b680b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 -- 172.21.15.86:0/3094733169 >> 172.21.15.86:0/3094733169 conn(0x7f6b680b1730 msgr2=0x7f6b680b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 -- 172.21.15.86:0/3094733169 shutdown_connections 2024-03-06T07:35:18.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.865+0000 7f6b6e800640 1 -- 172.21.15.86:0/3094733169 wait complete. 2024-03-06T07:35:18.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.867+0000 7f6b6e800640 1 Processor -- start 2024-03-06T07:35:18.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.867+0000 7f6b6e800640 1 -- start start 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.868+0000 7f6b6e800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b68072340 0x7f6b68143ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b6e800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6815bc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b6e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b6815acb0 0x7f6b680713f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b6e800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b680b3d10 con 0x7f6b6815acb0 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b6e800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b680b32b0 con 0x7f6b680b4680 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b6e800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b680b4010 con 0x7f6b68072340 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b67e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b6815acb0 0x7f6b680713f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b67e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b6815acb0 0x7f6b680713f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55598/0 (socket says 172.21.15.86:55598) 2024-03-06T07:35:18.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b67e00640 1 -- 172.21.15.86:0/2057810481 learned_addr learned my addr 172.21.15.86:0/2057810481 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b67400640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b68072340 0x7f6b68143ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.869+0000 7f6b5ea00640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6815bc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b67e00640 1 -- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b68072340 msgr2=0x7f6b68143ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b67e00640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b68072340 0x7f6b68143ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b67e00640 1 -- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 msgr2=0x7f6b6815bc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b67e00640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6815bc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b67e00640 1 -- 172.21.15.86:0/2057810481 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b6815e0b0 con 0x7f6b6815acb0 2024-03-06T07:35:18.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b5ea00640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6815bc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:35:18.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b67e00640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b6815acb0 0x7f6b680713f0 secure :-1 s=READY pgs=1552 cs=0 l=1 rev1=1 crypto rx=0x7f6b60001eb0 tx=0x7f6b6000c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:18.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b5fe00640 1 -- 172.21.15.86:0/2057810481 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b60017070 con 0x7f6b6815acb0 2024-03-06T07:35:18.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b5fe00640 1 -- 172.21.15.86:0/2057810481 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b600052d0 con 0x7f6b6815acb0 2024-03-06T07:35:18.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b6815c350 con 0x7f6b6815acb0 2024-03-06T07:35:18.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b5fe00640 1 -- 172.21.15.86:0/2057810481 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b60003810 con 0x7f6b6815acb0 2024-03-06T07:35:18.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.870+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b68158910 con 0x7f6b6815acb0 2024-03-06T07:35:18.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.872+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b6804f330 con 0x7f6b6815acb0 2024-03-06T07:35:18.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.874+0000 7f6b5fe00640 1 -- 172.21.15.86:0/2057810481 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b6001a070 con 0x7f6b6815acb0 2024-03-06T07:35:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.874+0000 7f6b5fe00640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b4406b500 0x7f6b4406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.875+0000 7f6b5fe00640 1 -- 172.21.15.86:0/2057810481 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6b60089480 con 0x7f6b6815acb0 2024-03-06T07:35:18.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.875+0000 7f6b67400640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b4406b500 0x7f6b4406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:18.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.875+0000 7f6b67400640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b4406b500 0x7f6b4406d9c0 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f6b58011340 tx=0x7f6b5800b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:18.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:18.879+0000 7f6b5fe00640 1 -- 172.21.15.86:0/2057810481 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6b6815e0b0 con 0x7f6b6815acb0 2024-03-06T07:35:19.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.117+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6b6806ce60 con 0x7f6b6815acb0 2024-03-06T07:35:19.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.118+0000 7f6b5fe00640 1 -- 172.21.15.86:0/2057810481 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f6b60057ce0 con 0x7f6b6815acb0 2024-03-06T07:35:19.120 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:19.120 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:19.120 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:19.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b4406b500 msgr2=0x7f6b4406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:19.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b4406b500 0x7f6b4406d9c0 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f6b58011340 tx=0x7f6b5800b040 comp rx=0 tx=0).stop 2024-03-06T07:35:19.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b6815acb0 msgr2=0x7f6b680713f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:19.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b6815acb0 0x7f6b680713f0 secure :-1 s=READY pgs=1552 cs=0 l=1 rev1=1 crypto rx=0x7f6b60001eb0 tx=0x7f6b6000c890 comp rx=0 tx=0).stop 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 shutdown_connections 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6b4406b500 0x7f6b4406d9c0 unknown :-1 s=CLOSED pgs=663 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b6815acb0 0x7f6b680713f0 unknown :-1 s=CLOSED pgs=1552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b680b4680 0x7f6b6815bc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 --2- 172.21.15.86:0/2057810481 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b68072340 0x7f6b68143ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.121+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 >> 172.21.15.86:0/2057810481 conn(0x7f6b680b1730 msgr2=0x7f6b680af790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.122+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 shutdown_connections 2024-03-06T07:35:19.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:19.122+0000 7f6b6e800640 1 -- 172.21.15.86:0/2057810481 wait complete. 2024-03-06T07:35:19.350 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:19.350 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:19.532 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:20.351 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:20.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.563+0000 7fa6a6000640 1 Processor -- start 2024-03-06T07:35:20.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.564+0000 7fa6a6000640 1 -- start start 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa6a6000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa6a0161580 0x7fa6a00d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa6a6000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa6a0161960 con 0x7fa6a00d1e90 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa6a6000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa6a00cf6f0 con 0x7fa6a00d2a70 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa6a6000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6a00cf9d0 con 0x7fa6a0161580 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa697e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa6a00d1e90 0x7fa6a00d2360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40546/0 (socket says 172.21.15.86:40546) 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa697e00640 1 -- 172.21.15.86:0/1372993347 learned_addr learned my addr 172.21.15.86:0/1372993347 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 907012931 0 0) 0x7fa6a0161960 con 0x7fa6a00d1e90 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa6a4c00640 1 --2- 172.21.15.86:0/1372993347 >> v2:172.21.15.194:3300/0 conn(0x7fa6a0161580 0x7fa6a00d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.566+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa684003520 con 0x7fa6a00d1e90 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3846361361 0 0) 0x7fa684003520 con 0x7fa6a00d1e90 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 >> v2:172.21.15.194:3300/0 conn(0x7fa6a0161580 msgr2=0x7fa6a00d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 --2- 172.21.15.86:0/1372993347 >> v2:172.21.15.194:3300/0 conn(0x7fa6a0161580 0x7fa6a00d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 >> v1:172.21.15.194:6789/0 conn(0x7fa6a00d2a70 legacy=0x7fa6a00b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6a00cfcb0 con 0x7fa6a00d1e90 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fa688001670 con 0x7fa6a00d1e90 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 >> v1:172.21.15.86:6789/0 conn(0x7fa6a00d1e90 legacy=0x7fa6a00d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6840039b0 0x7fa684003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa684004800 0x7fa684004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6840051c0 0x7fa68400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6a00cf9d0 con 0x7fa6840051c0 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa68400c0f0 con 0x7fa684004800 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa68400c2f0 con 0x7fa6840039b0 2024-03-06T07:35:20.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.567+0000 7fa6a5600640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6840051c0 0x7fa68400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa697e00640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa684004800 0x7fa684004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa6a4c00640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6840039b0 0x7fa684003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa6a5600640 1 -- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6840039b0 msgr2=0x7fa684003db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa6a5600640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6840039b0 0x7fa684003db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa6a5600640 1 -- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa684004800 msgr2=0x7fa684004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa6a5600640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa684004800 0x7fa684004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa6a5600640 1 -- 172.21.15.86:0/1372993347 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa684003520 con 0x7fa6840051c0 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.568+0000 7fa6a4c00640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6840039b0 0x7fa684003db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.569+0000 7fa6a5600640 1 -- 172.21.15.86:0/1372993347 reap_dead start 2024-03-06T07:35:20.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.569+0000 7fa6a5600640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6840051c0 0x7fa68400b9c0 secure :-1 s=READY pgs=1554 cs=0 l=1 rev1=1 crypto rx=0x7fa6a0073710 tx=0x7fa684005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=946d498ad43bc6f9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.569+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa698035030 con 0x7fa6840051c0 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.569+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa698006b50 con 0x7fa6840051c0 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.569+0000 7fa69fe00640 1 -- 172.21.15.86:0/1372993347 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa698004080 con 0x7fa6840051c0 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.569+0000 7fa6a6000640 1 -- 172.21.15.86:0/1372993347 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6840051c0 msgr2=0x7fa68400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.569+0000 7fa6a6000640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6840051c0 0x7fa68400b9c0 secure :-1 s=READY pgs=1554 cs=0 l=1 rev1=1 crypto rx=0x7fa6a0073710 tx=0x7fa684005830 comp rx=0 tx=0).stop 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.570+0000 7fa6a6000640 1 -- 172.21.15.86:0/1372993347 shutdown_connections 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.570+0000 7fa6a6000640 1 --2- 172.21.15.86:0/1372993347 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6840051c0 0x7fa68400b9c0 unknown :-1 s=CLOSED pgs=1554 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.570+0000 7fa6a6000640 1 -- 172.21.15.86:0/1372993347 >> 172.21.15.86:0/1372993347 conn(0x7fa6a00d3d80 msgr2=0x7fa6a00d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.570+0000 7fa6a6000640 1 -- 172.21.15.86:0/1372993347 shutdown_connections 2024-03-06T07:35:20.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.570+0000 7fa6a6000640 1 -- 172.21.15.86:0/1372993347 wait complete. 2024-03-06T07:35:20.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.572+0000 7fa6a6000640 1 Processor -- start 2024-03-06T07:35:20.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.573+0000 7fa6a6000640 1 -- start start 2024-03-06T07:35:20.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.574+0000 7fa6a6000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa6840051c0 0x7fa6a00b82a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.574+0000 7fa6a6000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6a00b6b40 0x7fa6a00b9560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.574+0000 7fa6a6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6a00c8d70 0x7fa6a00c91f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.574+0000 7fa6a6000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6a00cf3a0 con 0x7fa6a00c8d70 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.574+0000 7fa6a6000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6a00b39a0 con 0x7fa6840051c0 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.574+0000 7fa6a6000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6a00d1950 con 0x7fa6a00b6b40 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.574+0000 7fa6a5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6a00c8d70 0x7fa6a00c91f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6a00c8d70 0x7fa6a00c91f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55622/0 (socket says 172.21.15.86:55622) 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 -- 172.21.15.86:0/4289581895 learned_addr learned my addr 172.21.15.86:0/4289581895 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a4c00640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa6840051c0 0x7fa6a00b82a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa697e00640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6a00b6b40 0x7fa6a00b9560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 -- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6a00b6b40 msgr2=0x7fa6a00b9560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6a00b6b40 0x7fa6a00b9560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 -- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa6840051c0 msgr2=0x7fa6a00b82a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:20.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa6840051c0 0x7fa6a00b82a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 -- 172.21.15.86:0/4289581895 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6a00b7200 con 0x7fa6a00c8d70 2024-03-06T07:35:20.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa6a5600640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6a00c8d70 0x7fa6a00c91f0 secure :-1 s=READY pgs=1555 cs=0 l=1 rev1=1 crypto rx=0x7fa698004570 tx=0x7fa698003a70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:20.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.575+0000 7fa69d600640 1 -- 172.21.15.86:0/4289581895 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa698035040 con 0x7fa6a00c8d70 2024-03-06T07:35:20.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.576+0000 7fa69d600640 1 -- 172.21.15.86:0/4289581895 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa698042070 con 0x7fa6a00c8d70 2024-03-06T07:35:20.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.576+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa6a00b74e0 con 0x7fa6a00c8d70 2024-03-06T07:35:20.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.576+0000 7fa69d600640 1 -- 172.21.15.86:0/4289581895 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa698004080 con 0x7fa6a00c8d70 2024-03-06T07:35:20.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.576+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6a00b7aa0 con 0x7fa6a00c8d70 2024-03-06T07:35:20.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.577+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa6a00cf0b0 con 0x7fa6a00c8d70 2024-03-06T07:35:20.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.578+0000 7fa69d600640 1 -- 172.21.15.86:0/4289581895 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa698038070 con 0x7fa6a00c8d70 2024-03-06T07:35:20.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.579+0000 7fa69d600640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa67c06b500 0x7fa67c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:20.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.579+0000 7fa69d600640 1 -- 172.21.15.86:0/4289581895 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa6980ab500 con 0x7fa6a00c8d70 2024-03-06T07:35:20.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.579+0000 7fa6a4c00640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa67c06b500 0x7fa67c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:20.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.580+0000 7fa6a4c00640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa67c06b500 0x7fa67c06d9c0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7fa6a01f7130 tx=0x7fa690008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:20.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.584+0000 7fa69d600640 1 -- 172.21.15.86:0/4289581895 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa6a00cf0b0 con 0x7fa6a00c8d70 2024-03-06T07:35:20.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.840+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa6a004f330 con 0x7fa6a00c8d70 2024-03-06T07:35:20.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.841+0000 7fa69d600640 1 -- 172.21.15.86:0/4289581895 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7fa69807a450 con 0x7fa6a00c8d70 2024-03-06T07:35:20.843 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:20.843 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:20.843 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa67c06b500 msgr2=0x7fa67c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa67c06b500 0x7fa67c06d9c0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7fa6a01f7130 tx=0x7fa690008040 comp rx=0 tx=0).stop 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6a00c8d70 msgr2=0x7fa6a00c91f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6a00c8d70 0x7fa6a00c91f0 secure :-1 s=READY pgs=1555 cs=0 l=1 rev1=1 crypto rx=0x7fa698004570 tx=0x7fa698003a70 comp rx=0 tx=0).stop 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 shutdown_connections 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa67c06b500 0x7fa67c06d9c0 unknown :-1 s=CLOSED pgs=664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa6a00c8d70 0x7fa6a00c91f0 unknown :-1 s=CLOSED pgs=1555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa6a00b6b40 0x7fa6a00b9560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 --2- 172.21.15.86:0/4289581895 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa6840051c0 0x7fa6a00b82a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 >> 172.21.15.86:0/4289581895 conn(0x7fa6a00d3d80 msgr2=0x7fa6a00b3dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 shutdown_connections 2024-03-06T07:35:20.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:20.844+0000 7fa6a6000640 1 -- 172.21.15.86:0/4289581895 wait complete. 2024-03-06T07:35:21.066 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:21.066 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:21.534 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:22.067 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:22.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.274+0000 7f60dfe00640 1 Processor -- start 2024-03-06T07:35:22.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.274+0000 7f60dfe00640 1 -- start start 2024-03-06T07:35:22.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.275+0000 7f60dfe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f60e015e200 0x7f60e00d4850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.275+0000 7f60dfe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f60e00b2e50 0x7f60e00d04f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.275+0000 7f60dfe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f60e00b5240 con 0x7f60e015e5e0 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60dfe00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f60e00d2ec0 con 0x7f60e00b2e50 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60dfe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f60e00cf4d0 con 0x7f60e015e200 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60d6000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f60e00b2e50 0x7f60e00d04f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60d6000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f60e00b2e50 0x7f60e00d04f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55632/0 (socket says 172.21.15.86:55632) 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60d6000640 1 -- 172.21.15.86:0/3357237209 learned_addr learned my addr 172.21.15.86:0/3357237209 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60d6000640 1 -- 172.21.15.86:0/3357237209 >> v2:172.21.15.194:3300/0 conn(0x7f60e015e200 msgr2=0x7f60e00d4850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60d6000640 1 --2- 172.21.15.86:0/3357237209 >> v2:172.21.15.194:3300/0 conn(0x7f60e015e200 0x7f60e00d4850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60d6000640 1 -- 172.21.15.86:0/3357237209 >> v1:172.21.15.194:6789/0 conn(0x7f60e015e5e0 legacy=0x7f60e00d1c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:22.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.276+0000 7f60d6000640 1 -- 172.21.15.86:0/3357237209 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60e00b5500 con 0x7f60e00b2e50 2024-03-06T07:35:22.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.277+0000 7f60d6000640 1 --2- 172.21.15.86:0/3357237209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e00d04f0 secure :-1 s=READY pgs=1556 cs=0 l=1 rev1=1 crypto rx=0x7f60e00d3d00 tx=0x7f60c802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e43f6b116e22622c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:22.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.277+0000 7f60de000640 1 -- 172.21.15.86:0/3357237209 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60c8034070 con 0x7f60e00b2e50 2024-03-06T07:35:22.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60de000640 1 -- 172.21.15.86:0/3357237209 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f60c8030860 con 0x7f60e00b2e50 2024-03-06T07:35:22.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60de000640 1 -- 172.21.15.86:0/3357237209 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60c8030c30 con 0x7f60e00b2e50 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 -- 172.21.15.86:0/3357237209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 msgr2=0x7f60e00d04f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3357237209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e00d04f0 secure :-1 s=READY pgs=1556 cs=0 l=1 rev1=1 crypto rx=0x7f60e00d3d00 tx=0x7f60c802b9a0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 -- 172.21.15.86:0/3357237209 shutdown_connections 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3357237209 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e00d04f0 unknown :-1 s=CLOSED pgs=1556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3357237209 >> v2:172.21.15.194:3300/0 conn(0x7f60e015e200 0x7f60e00d4850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 -- 172.21.15.86:0/3357237209 >> 172.21.15.86:0/3357237209 conn(0x7f60e00d3d30 msgr2=0x7f60e00d4140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 -- 172.21.15.86:0/3357237209 shutdown_connections 2024-03-06T07:35:22.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.278+0000 7f60dfe00640 1 -- 172.21.15.86:0/3357237209 wait complete. 2024-03-06T07:35:22.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.280+0000 7f60dfe00640 1 Processor -- start 2024-03-06T07:35:22.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.280+0000 7f60dfe00640 1 -- start start 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dfe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e015c4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dfe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f60e015e200 0x7f60e00b7b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dfe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f60e015e5e0 0x7f60e01f31c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dfe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60e00cfd60 con 0x7f60e00b2e50 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dfe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60e00cfa80 con 0x7f60e015e5e0 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dfe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f60e00b4650 con 0x7f60e015e200 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e015c4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e015c4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55648/0 (socket says 172.21.15.86:55648) 2024-03-06T07:35:22.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.281+0000 7f60dea00640 1 -- 172.21.15.86:0/3082811320 learned_addr learned my addr 172.21.15.86:0/3082811320 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60dea00640 1 -- 172.21.15.86:0/3082811320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f60e015e200 msgr2=0x7f60e00b7b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60dea00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f60e015e200 0x7f60e00b7b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60dea00640 1 -- 172.21.15.86:0/3082811320 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f60e015e5e0 msgr2=0x7f60e01f31c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60df400640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f60e015e5e0 0x7f60e01f31c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60dea00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f60e015e5e0 0x7f60e01f31c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60dea00640 1 -- 172.21.15.86:0/3082811320 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60e01f4d10 con 0x7f60e00b2e50 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60dea00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e015c4a0 secure :-1 s=READY pgs=1557 cs=0 l=1 rev1=1 crypto rx=0x7f60d000ba50 tx=0x7f60d000bf10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60d7400640 1 -- 172.21.15.86:0/3082811320 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60d0019070 con 0x7f60e00b2e50 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60d7400640 1 -- 172.21.15.86:0/3082811320 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f60d000cc00 con 0x7f60e00b2e50 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60d7400640 1 -- 172.21.15.86:0/3082811320 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60d0014440 con 0x7f60e00b2e50 2024-03-06T07:35:22.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.282+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f60e01f41f0 con 0x7f60e00b2e50 2024-03-06T07:35:22.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.283+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f60e01f4a30 con 0x7f60e00b2e50 2024-03-06T07:35:22.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.284+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f60e004f330 con 0x7f60e00b2e50 2024-03-06T07:35:22.286 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.285+0000 7f60d7400640 1 -- 172.21.15.86:0/3082811320 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f60d001c070 con 0x7f60e00b2e50 2024-03-06T07:35:22.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.286+0000 7f60d7400640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f60bc06b500 0x7f60bc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:22.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.286+0000 7f60d7400640 1 -- 172.21.15.86:0/3082811320 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f60d0089fb0 con 0x7f60e00b2e50 2024-03-06T07:35:22.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.287+0000 7f60d6000640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f60bc06b500 0x7f60bc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:22.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.287+0000 7f60d6000640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f60bc06b500 0x7f60bc06d9c0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f60c8036000 tx=0x7f60c803b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:22.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.290+0000 7f60d7400640 1 -- 172.21.15.86:0/3082811320 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f60e01f41f0 con 0x7f60e00b2e50 2024-03-06T07:35:22.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.524+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f60e00c8e10 con 0x7f60e00b2e50 2024-03-06T07:35:22.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.525+0000 7f60d7400640 1 -- 172.21.15.86:0/3082811320 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f60d0058de0 con 0x7f60e00b2e50 2024-03-06T07:35:22.527 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:22.527 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:22.527 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:22.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f60bc06b500 msgr2=0x7f60bc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:22.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f60bc06b500 0x7f60bc06d9c0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f60c8036000 tx=0x7f60c803b040 comp rx=0 tx=0).stop 2024-03-06T07:35:22.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 msgr2=0x7f60e015c4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:22.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e015c4a0 secure :-1 s=READY pgs=1557 cs=0 l=1 rev1=1 crypto rx=0x7f60d000ba50 tx=0x7f60d000bf10 comp rx=0 tx=0).stop 2024-03-06T07:35:22.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 shutdown_connections 2024-03-06T07:35:22.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f60bc06b500 0x7f60bc06d9c0 unknown :-1 s=CLOSED pgs=665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f60e015e5e0 0x7f60e01f31c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f60e015e200 0x7f60e00b7b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 --2- 172.21.15.86:0/3082811320 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f60e00b2e50 0x7f60e015c4a0 unknown :-1 s=CLOSED pgs=1557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:22.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 >> 172.21.15.86:0/3082811320 conn(0x7f60e00d3d30 msgr2=0x7f60e015aa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:22.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 shutdown_connections 2024-03-06T07:35:22.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:22.528+0000 7f60dfe00640 1 -- 172.21.15.86:0/3082811320 wait complete. 2024-03-06T07:35:22.747 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:22.747 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:23.537 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:23.748 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:23.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.960+0000 7f36bbc00640 1 Processor -- start 2024-03-06T07:35:23.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.961+0000 7f36bbc00640 1 -- start start 2024-03-06T07:35:23.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.962+0000 7f36bbc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f36b40cce10 0x7f36b415fd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.962+0000 7f36bbc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36b4070de0 con 0x7f36b415f4f0 2024-03-06T07:35:23.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.962+0000 7f36bbc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36b4160a20 con 0x7f36b415f8d0 2024-03-06T07:35:23.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.962+0000 7f36bbc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f36b406ef50 con 0x7f36b40cce10 2024-03-06T07:35:23.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.962+0000 7f36b9200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f36b415f4f0 0x7f36b40706d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:40550/0 (socket says 172.21.15.86:40550) 2024-03-06T07:35:23.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.962+0000 7f36b9200640 1 -- 172.21.15.86:0/2488087190 learned_addr learned my addr 172.21.15.86:0/2488087190 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:23.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b9c00640 1 --2- 172.21.15.86:0/2488087190 >> v2:172.21.15.194:3300/0 conn(0x7f36b40cce10 0x7f36b415fd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:23.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3142966631 0 0) 0x7f36b4070de0 con 0x7f36b415f4f0 2024-03-06T07:35:23.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3698003520 con 0x7f36b415f4f0 2024-03-06T07:35:23.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3338775077 0 0) 0x7f3698003520 con 0x7f36b415f4f0 2024-03-06T07:35:23.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 >> v2:172.21.15.194:3300/0 conn(0x7f36b40cce10 msgr2=0x7f36b415fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:23.970 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 --2- 172.21.15.86:0/2488087190 >> v2:172.21.15.194:3300/0 conn(0x7f36b40cce10 0x7f36b415fd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 >> v1:172.21.15.194:6789/0 conn(0x7f36b415f8d0 legacy=0x7f36b40cc700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36b406f230 con 0x7f36b415f4f0 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f36a4001670 con 0x7f36b415f4f0 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 >> v1:172.21.15.86:6789/0 conn(0x7f36b415f4f0 legacy=0x7f36b40706d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980039b0 0x7f3698003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3698004800 0x7f3698004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.963+0000 7f36b3e00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36980051c0 0x7f369800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36b406ef50 con 0x7f36980051c0 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f369800c060 con 0x7f3698004800 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f369800c260 con 0x7f36980039b0 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b9c00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36980051c0 0x7f369800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36abe00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3698004800 0x7f3698004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b9200640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980039b0 0x7f3698003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:23.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b9c00640 1 -- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980039b0 msgr2=0x7f3698003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b9c00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980039b0 0x7f3698003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b9c00640 1 -- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3698004800 msgr2=0x7f3698004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b9c00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3698004800 0x7f3698004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.964+0000 7f36b9c00640 1 -- 172.21.15.86:0/2488087190 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3698003520 con 0x7f36980051c0 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.965+0000 7f36b9c00640 1 -- 172.21.15.86:0/2488087190 reap_dead start 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.965+0000 7f36b9c00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36980051c0 0x7f369800b9c0 secure :-1 s=READY pgs=1559 cs=0 l=1 rev1=1 crypto rx=0x7f36ac009fc0 tx=0x7f3698005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea263b223f534024 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36ac009c10 con 0x7f36980051c0 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36ac02cca0 con 0x7f36980051c0 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36b3e00640 1 -- 172.21.15.86:0/2488087190 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36ac003bd0 con 0x7f36980051c0 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36bbc00640 1 -- 172.21.15.86:0/2488087190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36980051c0 msgr2=0x7f369800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36bbc00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36980051c0 0x7f369800b9c0 secure :-1 s=READY pgs=1559 cs=0 l=1 rev1=1 crypto rx=0x7f36ac009fc0 tx=0x7f3698005830 comp rx=0 tx=0).stop 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36bbc00640 1 -- 172.21.15.86:0/2488087190 shutdown_connections 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36bbc00640 1 --2- 172.21.15.86:0/2488087190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36980051c0 0x7f369800b9c0 unknown :-1 s=CLOSED pgs=1559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36bbc00640 1 -- 172.21.15.86:0/2488087190 >> 172.21.15.86:0/2488087190 conn(0x7f36b40b3800 msgr2=0x7f36b40b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.966+0000 7f36bbc00640 1 -- 172.21.15.86:0/2488087190 shutdown_connections 2024-03-06T07:35:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.967+0000 7f36bbc00640 1 -- 172.21.15.86:0/2488087190 wait complete. 2024-03-06T07:35:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.968+0000 7f36bbc00640 1 Processor -- start 2024-03-06T07:35:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.968+0000 7f36bbc00640 1 -- start start 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.969+0000 7f36bbc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980051c0 0x7f36b40b0aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.969+0000 7f36bbc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36b40b4760 0x7f36b4159270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.969+0000 7f36bbc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36b40b62f0 0x7f36b40b6770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.969+0000 7f36bbc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36b4070a60 con 0x7f36b40b62f0 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.969+0000 7f36bbc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36b40708e0 con 0x7f36b40b4760 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.969+0000 7f36bbc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36b4070be0 con 0x7f36980051c0 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.969+0000 7f36b9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36b40b62f0 0x7f36b40b6770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36b40b62f0 0x7f36b40b6770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55652/0 (socket says 172.21.15.86:55652) 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 -- 172.21.15.86:0/3398326066 learned_addr learned my addr 172.21.15.86:0/3398326066 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9200640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980051c0 0x7f36b40b0aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:23.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 -- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980051c0 msgr2=0x7f36b40b0aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980051c0 0x7f36b40b0aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 -- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36b40b4760 msgr2=0x7f36b4159270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36b40b4760 0x7f36b4159270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 -- 172.21.15.86:0/3398326066 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36b40b7660 con 0x7f36b40b62f0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9200640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980051c0 0x7f36b40b0aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.970+0000 7f36b9c00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36b40b62f0 0x7f36b40b6770 secure :-1 s=READY pgs=1560 cs=0 l=1 rev1=1 crypto rx=0x7f36ac00e200 tx=0x7f36ac009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.971+0000 7f36b1600640 1 -- 172.21.15.86:0/3398326066 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36ac042020 con 0x7f36b40b62f0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.971+0000 7f36b1600640 1 -- 172.21.15.86:0/3398326066 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36ac02ce50 con 0x7f36b40b62f0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.971+0000 7f36b1600640 1 -- 172.21.15.86:0/3398326066 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36ac003dc0 con 0x7f36b40b62f0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.971+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36b40b4be0 con 0x7f36b40b62f0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.971+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f36b40b7380 con 0x7f36b40b62f0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.972+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f36b404f330 con 0x7f36b40b62f0 2024-03-06T07:35:23.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.973+0000 7f36b1600640 1 -- 172.21.15.86:0/3398326066 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f36ac045020 con 0x7f36b40b62f0 2024-03-06T07:35:23.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.974+0000 7f36b1600640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f369006b500 0x7f369006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:23.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.974+0000 7f36b1600640 1 -- 172.21.15.86:0/3398326066 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f36ac0acc20 con 0x7f36b40b62f0 2024-03-06T07:35:23.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.975+0000 7f36b9200640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f369006b500 0x7f369006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:23.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.975+0000 7f36b9200640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f369006b500 0x7f369006d9c0 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f36a4001200 tx=0x7f36a4000f70 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:23.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:23.979+0000 7f36b1600640 1 -- 172.21.15.86:0/3398326066 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f36ac04c020 con 0x7f36b40b62f0 2024-03-06T07:35:24.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.274+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f36b40cff00 con 0x7f36b40b62f0 2024-03-06T07:35:24.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.275+0000 7f36b1600640 1 -- 172.21.15.86:0/3398326066 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f36ac07bad0 con 0x7f36b40b62f0 2024-03-06T07:35:24.277 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:24.277 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:24.277 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:24.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.277+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f369006b500 msgr2=0x7f369006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:24.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.277+0000 7f36bbc00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f369006b500 0x7f369006d9c0 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f36a4001200 tx=0x7f36a4000f70 comp rx=0 tx=0).stop 2024-03-06T07:35:24.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.277+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36b40b62f0 msgr2=0x7f36b40b6770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.277+0000 7f36bbc00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36b40b62f0 0x7f36b40b6770 secure :-1 s=READY pgs=1560 cs=0 l=1 rev1=1 crypto rx=0x7f36ac00e200 tx=0x7f36ac009750 comp rx=0 tx=0).stop 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 shutdown_connections 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f369006b500 0x7f369006d9c0 unknown :-1 s=CLOSED pgs=666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36b40b62f0 0x7f36b40b6770 unknown :-1 s=CLOSED pgs=1560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36b40b4760 0x7f36b4159270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 --2- 172.21.15.86:0/3398326066 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36980051c0 0x7f36b40b0aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 >> 172.21.15.86:0/3398326066 conn(0x7f36b40b3800 msgr2=0x7f36b40b1650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 shutdown_connections 2024-03-06T07:35:24.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:24.278+0000 7f36bbc00640 1 -- 172.21.15.86:0/3398326066 wait complete. 2024-03-06T07:35:24.501 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:24.501 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:24.780 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:35:24.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:35:25.502 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:25.538 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:25.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.718+0000 7fb908e00640 1 Processor -- start 2024-03-06T07:35:25.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.718+0000 7fb908e00640 1 -- start start 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.719+0000 7fb908e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb9040af1c0 0x7fb90415eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.719+0000 7fb908e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb904070d20 0x7fb90406cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.719+0000 7fb908e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb90406d1a0 0x7fb90406e4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.719+0000 7fb908e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb904071d30 con 0x7fb904070d20 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.719+0000 7fb908e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb9040711d0 con 0x7fb9040af1c0 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.719+0000 7fb908e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb904071490 con 0x7fb90406d1a0 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb904070d20 0x7fb90406cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb904070d20 0x7fb90406cc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55666/0 (socket says 172.21.15.86:55666) 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 -- 172.21.15.86:0/562364905 learned_addr learned my addr 172.21.15.86:0/562364905 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb901e00640 1 --2- 172.21.15.86:0/562364905 >> v2:172.21.15.194:3300/0 conn(0x7fb9040af1c0 0x7fb90415eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:25.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb902800640 1 --2- 172.21.15.86:0/562364905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb90406d1a0 0x7fb90406e4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 -- 172.21.15.86:0/562364905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb90406d1a0 msgr2=0x7fb90406e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 --2- 172.21.15.86:0/562364905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb90406d1a0 0x7fb90406e4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 -- 172.21.15.86:0/562364905 >> v2:172.21.15.194:3300/0 conn(0x7fb9040af1c0 msgr2=0x7fb90415eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 --2- 172.21.15.86:0/562364905 >> v2:172.21.15.194:3300/0 conn(0x7fb9040af1c0 0x7fb90415eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.720+0000 7fb8f9400640 1 -- 172.21.15.86:0/562364905 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb904071770 con 0x7fb904070d20 2024-03-06T07:35:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.722+0000 7fb8f9400640 1 --2- 172.21.15.86:0/562364905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb90406cc60 secure :-1 s=READY pgs=1561 cs=0 l=1 rev1=1 crypto rx=0x7fb8ec009870 tx=0x7fb8ec02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d8e26d328f57d70e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.722+0000 7fb901400640 1 -- 172.21.15.86:0/562364905 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8ec035020 con 0x7fb904070d20 2024-03-06T07:35:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.722+0000 7fb901400640 1 -- 172.21.15.86:0/562364905 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb8ec031640 con 0x7fb904070d20 2024-03-06T07:35:25.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.722+0000 7fb901400640 1 -- 172.21.15.86:0/562364905 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8ec02d040 con 0x7fb904070d20 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 -- 172.21.15.86:0/562364905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 msgr2=0x7fb90406cc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 --2- 172.21.15.86:0/562364905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb90406cc60 secure :-1 s=READY pgs=1561 cs=0 l=1 rev1=1 crypto rx=0x7fb8ec009870 tx=0x7fb8ec02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 -- 172.21.15.86:0/562364905 shutdown_connections 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 --2- 172.21.15.86:0/562364905 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb90406d1a0 0x7fb90406e4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 --2- 172.21.15.86:0/562364905 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb90406cc60 unknown :-1 s=CLOSED pgs=1561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 --2- 172.21.15.86:0/562364905 >> v2:172.21.15.194:3300/0 conn(0x7fb9040af1c0 0x7fb90415eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 -- 172.21.15.86:0/562364905 >> 172.21.15.86:0/562364905 conn(0x7fb9040b1730 msgr2=0x7fb9040b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 -- 172.21.15.86:0/562364905 shutdown_connections 2024-03-06T07:35:25.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.723+0000 7fb908e00640 1 -- 172.21.15.86:0/562364905 wait complete. 2024-03-06T07:35:25.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.725+0000 7fb908e00640 1 Processor -- start 2024-03-06T07:35:25.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.725+0000 7fb908e00640 1 -- start start 2024-03-06T07:35:25.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.726+0000 7fb908e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb90406d1a0 0x7fb90414a440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:25.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.726+0000 7fb908e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb9040cc430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.726+0000 7fb908e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9040af1c0 0x7fb90415fd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.726+0000 7fb908e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb90406ea10 con 0x7fb904070d20 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.726+0000 7fb908e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9040b30b0 con 0x7fb90406d1a0 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.726+0000 7fb908e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb90406eb90 con 0x7fb9040af1c0 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb9040cc430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb9040cc430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:55680/0 (socket says 172.21.15.86:55680) 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 -- 172.21.15.86:0/3841574479 learned_addr learned my addr 172.21.15.86:0/3841574479 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb901e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb90406d1a0 0x7fb90414a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb902800640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9040af1c0 0x7fb90415fd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 -- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9040af1c0 msgr2=0x7fb90415fd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9040af1c0 0x7fb90415fd90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 -- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb90406d1a0 msgr2=0x7fb90414a440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb90406d1a0 0x7fb90414a440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.727+0000 7fb8f9400640 1 -- 172.21.15.86:0/3841574479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9040ccd30 con 0x7fb904070d20 2024-03-06T07:35:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.728+0000 7fb8f9400640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb9040cc430 secure :-1 s=READY pgs=1562 cs=0 l=1 rev1=1 crypto rx=0x7fb8ec000c00 tx=0x7fb8ec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.728+0000 7fb8faa00640 1 -- 172.21.15.86:0/3841574479 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8ec040070 con 0x7fb904070d20 2024-03-06T07:35:25.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.728+0000 7fb8faa00640 1 -- 172.21.15.86:0/3841574479 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb8ec039bf0 con 0x7fb904070d20 2024-03-06T07:35:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.728+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb9040cd010 con 0x7fb904070d20 2024-03-06T07:35:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.728+0000 7fb8faa00640 1 -- 172.21.15.86:0/3841574479 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8ec035040 con 0x7fb904070d20 2024-03-06T07:35:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.728+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9040cd5d0 con 0x7fb904070d20 2024-03-06T07:35:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.729+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb90404f330 con 0x7fb904070d20 2024-03-06T07:35:25.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.730+0000 7fb8faa00640 1 -- 172.21.15.86:0/3841574479 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb8ec04b060 con 0x7fb904070d20 2024-03-06T07:35:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.731+0000 7fb8faa00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb8e006b4d0 0x7fb8e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:25.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.731+0000 7fb8faa00640 1 -- 172.21.15.86:0/3841574479 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb8ec0abd50 con 0x7fb904070d20 2024-03-06T07:35:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.731+0000 7fb901e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb8e006b4d0 0x7fb8e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:25.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.732+0000 7fb901e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb8e006b4d0 0x7fb8e006d990 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7fb8f4007130 tx=0x7fb8f4007060 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:25.741 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.740+0000 7fb8faa00640 1 -- 172.21.15.86:0/3841574479 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb8ec031610 con 0x7fb904070d20 2024-03-06T07:35:25.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.961+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb9040ae5a0 con 0x7fb904070d20 2024-03-06T07:35:25.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.962+0000 7fb8faa00640 1 -- 172.21.15.86:0/3841574479 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7fb8ec07ab70 con 0x7fb904070d20 2024-03-06T07:35:25.964 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:25.964 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:25.964 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb8e006b4d0 msgr2=0x7fb8e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb8e006b4d0 0x7fb8e006d990 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7fb8f4007130 tx=0x7fb8f4007060 comp rx=0 tx=0).stop 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 msgr2=0x7fb9040cc430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb9040cc430 secure :-1 s=READY pgs=1562 cs=0 l=1 rev1=1 crypto rx=0x7fb8ec000c00 tx=0x7fb8ec0046c0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 shutdown_connections 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb8e006b4d0 0x7fb8e006d990 unknown :-1 s=CLOSED pgs=667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb9040af1c0 0x7fb90415fd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb904070d20 0x7fb9040cc430 unknown :-1 s=CLOSED pgs=1562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 --2- 172.21.15.86:0/3841574479 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb90406d1a0 0x7fb90414a440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 >> 172.21.15.86:0/3841574479 conn(0x7fb9040b1730 msgr2=0x7fb9040b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 shutdown_connections 2024-03-06T07:35:25.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:25.965+0000 7fb908e00640 1 -- 172.21.15.86:0/3841574479 wait complete. 2024-03-06T07:35:26.185 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:26.185 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:27.186 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.389+0000 7fb5bda00640 1 Processor -- start 2024-03-06T07:35:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.389+0000 7fb5bda00640 1 -- start start 2024-03-06T07:35:27.392 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5bda00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fb5b80d1cc0 0x7fb5b80b11b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5bda00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5b80d4c50 con 0x7fb5b80d24c0 2024-03-06T07:35:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5bda00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5b80cfa30 con 0x7fb5b80b2de0 2024-03-06T07:35:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5bda00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5b80d4a00 con 0x7fb5b80d1cc0 2024-03-06T07:35:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5b6a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb5b80d24c0 0x7fb5b80d28a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59744/0 (socket says 172.21.15.86:59744) 2024-03-06T07:35:27.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5b6a00640 1 -- 172.21.15.86:0/375267187 learned_addr learned my addr 172.21.15.86:0/375267187 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 912898137 0 0) 0x7fb5b80d4c50 con 0x7fb5b80d24c0 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.391+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb59c003520 con 0x7fb5b80d24c0 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b7400640 1 --2- 172.21.15.86:0/375267187 >> v2:172.21.15.194:3300/0 conn(0x7fb5b80d1cc0 0x7fb5b80b11b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1411839998 0 0) 0x7fb59c003520 con 0x7fb5b80d24c0 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 >> v2:172.21.15.194:3300/0 conn(0x7fb5b80d1cc0 msgr2=0x7fb5b80b11b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 --2- 172.21.15.86:0/375267187 >> v2:172.21.15.194:3300/0 conn(0x7fb5b80d1cc0 0x7fb5b80b11b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 >> v1:172.21.15.194:6789/0 conn(0x7fb5b80b2de0 legacy=0x7fb5b80b0aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5b80cfd10 con 0x7fb5b80d24c0 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fb5a80019f0 con 0x7fb5b80d24c0 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 >> v1:172.21.15.86:6789/0 conn(0x7fb5b80d24c0 legacy=0x7fb5b80d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb59c003a00 0x7fb59c003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb59c004800 0x7fb59c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c0052e0 0x7fb59c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5b80d4a00 con 0x7fb59c004800 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb59c00c200 con 0x7fb59c0052e0 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb59c00c400 con 0x7fb59c003a00 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.392+0000 7fb5ae000640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb59c004800 0x7fb59c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5b6a00640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb59c003a00 0x7fb59c003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5b7400640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c0052e0 0x7fb59c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5ae000640 1 -- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb59c003a00 msgr2=0x7fb59c003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:27.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5ae000640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb59c003a00 0x7fb59c003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5ae000640 1 -- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c0052e0 msgr2=0x7fb59c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:27.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5ae000640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c0052e0 0x7fb59c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5ae000640 1 -- 172.21.15.86:0/375267187 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb59c003520 con 0x7fb59c004800 2024-03-06T07:35:27.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.393+0000 7fb5b7400640 1 -- 172.21.15.86:0/375267187 reap_dead start 2024-03-06T07:35:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.394+0000 7fb5ae000640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb59c004800 0x7fb59c004c80 secure :-1 s=READY pgs=1564 cs=0 l=1 rev1=1 crypto rx=0x7fb5a0001750 tx=0x7fb59c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=410c0d63f8571294 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.394+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5a0001370 con 0x7fb59c004800 2024-03-06T07:35:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.394+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5a00052d0 con 0x7fb59c004800 2024-03-06T07:35:27.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.394+0000 7fb5b6000640 1 -- 172.21.15.86:0/375267187 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5a0003830 con 0x7fb59c004800 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.395+0000 7fb5bda00640 1 -- 172.21.15.86:0/375267187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb59c004800 msgr2=0x7fb59c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.395+0000 7fb5bda00640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb59c004800 0x7fb59c004c80 secure :-1 s=READY pgs=1564 cs=0 l=1 rev1=1 crypto rx=0x7fb5a0001750 tx=0x7fb59c005830 comp rx=0 tx=0).stop 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.395+0000 7fb5bda00640 1 -- 172.21.15.86:0/375267187 shutdown_connections 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.395+0000 7fb5bda00640 1 --2- 172.21.15.86:0/375267187 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb59c004800 0x7fb59c004c80 unknown :-1 s=CLOSED pgs=1564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.395+0000 7fb5bda00640 1 -- 172.21.15.86:0/375267187 >> 172.21.15.86:0/375267187 conn(0x7fb5b80d3ee0 msgr2=0x7fb5b80d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.395+0000 7fb5bda00640 1 -- 172.21.15.86:0/375267187 shutdown_connections 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.395+0000 7fb5bda00640 1 -- 172.21.15.86:0/375267187 wait complete. 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.397+0000 7fb5bda00640 1 Processor -- start 2024-03-06T07:35:27.399 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.397+0000 7fb5bda00640 1 -- start start 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.397+0000 7fb5bda00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c004800 0x7fb5b815bad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.397+0000 7fb5bda00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5b80b74d0 0x7fb5b80b7950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5bda00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5b80ccc90 0x7fb5b80cd110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5bda00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5b80d0080 con 0x7fb5b80b74d0 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5bda00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5b80cf7e0 con 0x7fb59c004800 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5bda00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5b80d28a0 con 0x7fb5b80ccc90 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5b80b74d0 0x7fb5b80b7950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5b80b74d0 0x7fb5b80b7950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40188/0 (socket says 172.21.15.86:40188) 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 -- 172.21.15.86:0/2070887583 learned_addr learned my addr 172.21.15.86:0/2070887583 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 -- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5b80ccc90 msgr2=0x7fb5b80cd110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5b7400640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5b80ccc90 0x7fb5b80cd110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5b6a00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c004800 0x7fb5b815bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5b80ccc90 0x7fb5b80cd110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 -- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c004800 msgr2=0x7fb5b815bad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c004800 0x7fb5b815bad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.398+0000 7fb5ae000640 1 -- 172.21.15.86:0/2070887583 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5b815c010 con 0x7fb5b80b74d0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.399+0000 7fb5ae000640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5b80b74d0 0x7fb5b80b7950 secure :-1 s=READY pgs=1565 cs=0 l=1 rev1=1 crypto rx=0x7fb5a0005040 tx=0x7fb5a0001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.399+0000 7fb5af400640 1 -- 172.21.15.86:0/2070887583 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5a0041070 con 0x7fb5b80b74d0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.399+0000 7fb5af400640 1 -- 172.21.15.86:0/2070887583 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5a0005480 con 0x7fb5b80b74d0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.399+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5b80b7f90 con 0x7fb5b80b74d0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.399+0000 7fb5af400640 1 -- 172.21.15.86:0/2070887583 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5a0007a30 con 0x7fb5b80b74d0 2024-03-06T07:35:27.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.399+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5b80b8550 con 0x7fb5b80b74d0 2024-03-06T07:35:27.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.400+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5b804f330 con 0x7fb5b80b74d0 2024-03-06T07:35:27.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.401+0000 7fb5af400640 1 -- 172.21.15.86:0/2070887583 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb5a0044070 con 0x7fb5b80b74d0 2024-03-06T07:35:27.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.403+0000 7fb5af400640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb59406b4d0 0x7fb59406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:27.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.403+0000 7fb5af400640 1 -- 172.21.15.86:0/2070887583 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb5a00abdd0 con 0x7fb5b80b74d0 2024-03-06T07:35:27.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.403+0000 7fb5b6a00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb59406b4d0 0x7fb59406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:27.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.403+0000 7fb5b6a00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb59406b4d0 0x7fb59406d990 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7fb5a8001580 tx=0x7fb5a80014d0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:27.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.407+0000 7fb5af400640 1 -- 172.21.15.86:0/2070887583 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb5b815c010 con 0x7fb5b80b74d0 2024-03-06T07:35:27.540 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:27.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.638+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb5b80d22a0 con 0x7fb5b80b74d0 2024-03-06T07:35:27.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.640+0000 7fb5af400640 1 -- 172.21.15.86:0/2070887583 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7fb5a007ab70 con 0x7fb5b80b74d0 2024-03-06T07:35:27.641 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:27.641 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:27.642 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb59406b4d0 msgr2=0x7fb59406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb59406b4d0 0x7fb59406d990 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7fb5a8001580 tx=0x7fb5a80014d0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5b80b74d0 msgr2=0x7fb5b80b7950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5b80b74d0 0x7fb5b80b7950 secure :-1 s=READY pgs=1565 cs=0 l=1 rev1=1 crypto rx=0x7fb5a0005040 tx=0x7fb5a0001120 comp rx=0 tx=0).stop 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 shutdown_connections 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb59406b4d0 0x7fb59406d990 unknown :-1 s=CLOSED pgs=668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb5b80ccc90 0x7fb5b80cd110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb5b80b74d0 0x7fb5b80b7950 unknown :-1 s=CLOSED pgs=1565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 --2- 172.21.15.86:0/2070887583 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb59c004800 0x7fb5b815bad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:27.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 >> 172.21.15.86:0/2070887583 conn(0x7fb5b80d3ee0 msgr2=0x7fb5b80d0510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:27.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 shutdown_connections 2024-03-06T07:35:27.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:27.642+0000 7fb5bda00640 1 -- 172.21.15.86:0/2070887583 wait complete. 2024-03-06T07:35:27.863 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:27.863 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:28.864 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:29.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.078+0000 7f082c400640 1 Processor -- start 2024-03-06T07:35:29.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.078+0000 7f082c400640 1 -- start start 2024-03-06T07:35:29.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.079+0000 7f082c400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f08240af1c0 0x7f082415eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:29.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.079+0000 7f082c400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0824070d20 0x7f082406cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:29.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.079+0000 7f082c400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0824071d30 con 0x7f082406d1a0 2024-03-06T07:35:29.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f082c400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0824071490 con 0x7f08240af1c0 2024-03-06T07:35:29.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f082c400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0824071770 con 0x7f0824070d20 2024-03-06T07:35:29.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f08240af1c0 0x7f082415eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:29.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f08240af1c0 0x7f082415eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40204/0 (socket says 172.21.15.86:40204) 2024-03-06T07:35:29.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829a00640 1 -- 172.21.15.86:0/1238958688 learned_addr learned my addr 172.21.15.86:0/1238958688 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:29.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829000640 1 --2- 172.21.15.86:0/1238958688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0824070d20 0x7f082406cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:29.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829a00640 1 -- 172.21.15.86:0/1238958688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0824070d20 msgr2=0x7f082406cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:29.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829a00640 1 --2- 172.21.15.86:0/1238958688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0824070d20 0x7f082406cc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829a00640 1 -- 172.21.15.86:0/1238958688 >> v1:172.21.15.194:6789/0 conn(0x7f082406d1a0 legacy=0x7f082406e4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:29.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.080+0000 7f0829a00640 1 -- 172.21.15.86:0/1238958688 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0824071a50 con 0x7f08240af1c0 2024-03-06T07:35:29.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.081+0000 7f0829a00640 1 --2- 172.21.15.86:0/1238958688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415eff0 secure :-1 s=READY pgs=1566 cs=0 l=1 rev1=1 crypto rx=0x7f0818009870 tx=0x7f081802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8b2ca1f391c8c530 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:29.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.082+0000 7f0823e00640 1 -- 172.21.15.86:0/1238958688 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0818035020 con 0x7f08240af1c0 2024-03-06T07:35:29.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.082+0000 7f0823e00640 1 -- 172.21.15.86:0/1238958688 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0818031640 con 0x7f08240af1c0 2024-03-06T07:35:29.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.082+0000 7f0823e00640 1 -- 172.21.15.86:0/1238958688 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f081802d040 con 0x7f08240af1c0 2024-03-06T07:35:29.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.082+0000 7f082c400640 1 -- 172.21.15.86:0/1238958688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 msgr2=0x7f082415eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:29.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.082+0000 7f082c400640 1 --2- 172.21.15.86:0/1238958688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415eff0 secure :-1 s=READY pgs=1566 cs=0 l=1 rev1=1 crypto rx=0x7f0818009870 tx=0x7f081802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.083+0000 7f082c400640 1 -- 172.21.15.86:0/1238958688 shutdown_connections 2024-03-06T07:35:29.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.083+0000 7f082c400640 1 --2- 172.21.15.86:0/1238958688 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0824070d20 0x7f082406cc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.083+0000 7f082c400640 1 --2- 172.21.15.86:0/1238958688 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415eff0 unknown :-1 s=CLOSED pgs=1566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.083+0000 7f082c400640 1 -- 172.21.15.86:0/1238958688 >> 172.21.15.86:0/1238958688 conn(0x7f08240b1730 msgr2=0x7f08240b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:29.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.083+0000 7f082c400640 1 -- 172.21.15.86:0/1238958688 shutdown_connections 2024-03-06T07:35:29.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.083+0000 7f082c400640 1 -- 172.21.15.86:0/1238958688 wait complete. 2024-03-06T07:35:29.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.085+0000 7f082c400640 1 Processor -- start 2024-03-06T07:35:29.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.085+0000 7f082c400640 1 -- start start 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082c400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f082406d1a0 0x7f0824148b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082c400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0824070d20 0x7f08240cdea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082c400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415cfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082c400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f082406ebe0 con 0x7f08240af1c0 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082c400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08240711a0 con 0x7f0824070d20 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082c400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f082406ed60 con 0x7f082406d1a0 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415cfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415cfc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40214/0 (socket says 172.21.15.86:40214) 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 -- 172.21.15.86:0/164330659 learned_addr learned my addr 172.21.15.86:0/164330659 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f0829a00640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f082406d1a0 0x7f0824148b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f0829000640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0824070d20 0x7f08240cdea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 -- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f082406d1a0 msgr2=0x7f0824148b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f082406d1a0 0x7f0824148b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 -- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0824070d20 msgr2=0x7f08240cdea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0824070d20 0x7f08240cdea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.087+0000 7f082a400640 1 -- 172.21.15.86:0/164330659 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08240ce3e0 con 0x7f08240af1c0 2024-03-06T07:35:29.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.088+0000 7f082a400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415cfc0 secure :-1 s=READY pgs=1567 cs=0 l=1 rev1=1 crypto rx=0x7f081c00c730 tx=0x7f081c00cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:29.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.088+0000 7f0821600640 1 -- 172.21.15.86:0/164330659 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f081c016020 con 0x7f08240af1c0 2024-03-06T07:35:29.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.088+0000 7f0821600640 1 -- 172.21.15.86:0/164330659 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f081c00aa50 con 0x7f08240af1c0 2024-03-06T07:35:29.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.088+0000 7f0821600640 1 -- 172.21.15.86:0/164330659 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f081c00ce40 con 0x7f08240af1c0 2024-03-06T07:35:29.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.088+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08240ce680 con 0x7f08240af1c0 2024-03-06T07:35:29.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.088+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08240cec40 con 0x7f08240af1c0 2024-03-06T07:35:29.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.090+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f082404f330 con 0x7f08240af1c0 2024-03-06T07:35:29.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.090+0000 7f0821600640 1 -- 172.21.15.86:0/164330659 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f081c019020 con 0x7f08240af1c0 2024-03-06T07:35:29.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.091+0000 7f0821600640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f080406b4d0 0x7f080406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.091+0000 7f0821600640 1 -- 172.21.15.86:0/164330659 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f081c08a250 con 0x7f08240af1c0 2024-03-06T07:35:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.091+0000 7f0829a00640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f080406b4d0 0x7f080406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.092+0000 7f0829a00640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f080406b4d0 0x7f080406d990 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f0818036000 tx=0x7f081803c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:29.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.096+0000 7f0821600640 1 -- 172.21.15.86:0/164330659 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f081c020030 con 0x7f08240af1c0 2024-03-06T07:35:29.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.335+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f082415beb0 con 0x7f08240af1c0 2024-03-06T07:35:29.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.336+0000 7f0821600640 1 -- 172.21.15.86:0/164330659 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f081c0212b0 con 0x7f08240af1c0 2024-03-06T07:35:29.338 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:29.338 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:29.338 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f080406b4d0 msgr2=0x7f080406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f080406b4d0 0x7f080406d990 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f0818036000 tx=0x7f081803c040 comp rx=0 tx=0).stop 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 msgr2=0x7f082415cfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415cfc0 secure :-1 s=READY pgs=1567 cs=0 l=1 rev1=1 crypto rx=0x7f081c00c730 tx=0x7f081c00cbf0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 shutdown_connections 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f080406b4d0 0x7f080406d990 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f08240af1c0 0x7f082415cfc0 unknown :-1 s=CLOSED pgs=1567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0824070d20 0x7f08240cdea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 --2- 172.21.15.86:0/164330659 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f082406d1a0 0x7f0824148b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:29.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 >> 172.21.15.86:0/164330659 conn(0x7f08240b1730 msgr2=0x7f08240afd80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:29.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 shutdown_connections 2024-03-06T07:35:29.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:29.339+0000 7f082c400640 1 -- 172.21.15.86:0/164330659 wait complete. 2024-03-06T07:35:29.542 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:29.562 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:29.562 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:30.562 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:30.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.770+0000 7fd88c600640 1 Processor -- start 2024-03-06T07:35:30.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.770+0000 7fd88c600640 1 -- start start 2024-03-06T07:35:30.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd88c600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd88415e160 0x7fd8840b2e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:30.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd88c600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd8840cec00 0x7fd8840cc630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:30.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd88c600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd88406cc80 con 0x7fd88415e540 2024-03-06T07:35:30.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd88c600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd88406cf60 con 0x7fd88415e160 2024-03-06T07:35:30.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd88c600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd88406d240 con 0x7fd8840cec00 2024-03-06T07:35:30.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd889c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd88415e160 0x7fd8840b2e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:30.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd889200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd88415e540 0x7fd8840ce4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59760/0 (socket says 172.21.15.86:59760) 2024-03-06T07:35:30.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd889200640 1 -- 172.21.15.86:0/704066930 learned_addr learned my addr 172.21.15.86:0/704066930 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:30.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.772+0000 7fd88a600640 1 --2- 172.21.15.86:0/704066930 >> v2:172.21.15.194:3300/0 conn(0x7fd8840cec00 0x7fd8840cc630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:30.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.773+0000 7fd889c00640 1 -- 172.21.15.86:0/704066930 >> v2:172.21.15.194:3300/0 conn(0x7fd8840cec00 msgr2=0x7fd8840cc630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.773+0000 7fd889c00640 1 --2- 172.21.15.86:0/704066930 >> v2:172.21.15.194:3300/0 conn(0x7fd8840cec00 0x7fd8840cc630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.773+0000 7fd889c00640 1 -- 172.21.15.86:0/704066930 >> v1:172.21.15.86:6789/0 conn(0x7fd88415e540 legacy=0x7fd8840ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.773+0000 7fd889c00640 1 -- 172.21.15.86:0/704066930 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8840b33a0 con 0x7fd88415e160 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.773+0000 7fd88a600640 1 --2- 172.21.15.86:0/704066930 >> v2:172.21.15.194:3300/0 conn(0x7fd8840cec00 0x7fd8840cc630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.773+0000 7fd883e00640 1 -- 172.21.15.86:0/704066930 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1027963956 0 0) 0x7fd88406cc80 con 0x7fd88415e540 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.774+0000 7fd889c00640 1 --2- 172.21.15.86:0/704066930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd8840b2e60 secure :-1 s=READY pgs=1569 cs=0 l=1 rev1=1 crypto rx=0x7fd8840cfbf0 tx=0x7fd87802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=34e1bd7d4d38b852 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.774+0000 7fd883e00640 1 -- 172.21.15.86:0/704066930 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd878035020 con 0x7fd88415e160 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.774+0000 7fd883e00640 1 -- 172.21.15.86:0/704066930 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd878031830 con 0x7fd88415e160 2024-03-06T07:35:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.774+0000 7fd883e00640 1 -- 172.21.15.86:0/704066930 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd878031c00 con 0x7fd88415e160 2024-03-06T07:35:30.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.774+0000 7fd88c600640 1 -- 172.21.15.86:0/704066930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 msgr2=0x7fd8840b2e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:30.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.774+0000 7fd88c600640 1 --2- 172.21.15.86:0/704066930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd8840b2e60 secure :-1 s=READY pgs=1569 cs=0 l=1 rev1=1 crypto rx=0x7fd8840cfbf0 tx=0x7fd87802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:35:30.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.775+0000 7fd88c600640 1 -- 172.21.15.86:0/704066930 shutdown_connections 2024-03-06T07:35:30.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.775+0000 7fd88c600640 1 --2- 172.21.15.86:0/704066930 >> v2:172.21.15.194:3300/0 conn(0x7fd8840cec00 0x7fd8840cc630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:30.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.775+0000 7fd88c600640 1 --2- 172.21.15.86:0/704066930 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd8840b2e60 unknown :-1 s=CLOSED pgs=1569 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:30.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.775+0000 7fd88c600640 1 -- 172.21.15.86:0/704066930 >> 172.21.15.86:0/704066930 conn(0x7fd8840b3800 msgr2=0x7fd8840b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:30.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.775+0000 7fd88c600640 1 -- 172.21.15.86:0/704066930 shutdown_connections 2024-03-06T07:35:30.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.775+0000 7fd88c600640 1 -- 172.21.15.86:0/704066930 wait complete. 2024-03-06T07:35:30.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.776+0000 7fd88c600640 1 Processor -- start 2024-03-06T07:35:30.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.777+0000 7fd88c600640 1 -- start start 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.777+0000 7fd88c600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8840cec00 0x7fd8840b4740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.777+0000 7fd88c600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd88415c460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.777+0000 7fd88c600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd88415e540 0x7fd8840c63d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.777+0000 7fd88c600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd8840ccb70 con 0x7fd88415e160 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.777+0000 7fd88c600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd8840cf3c0 con 0x7fd88415e540 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.777+0000 7fd88c600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd8840cccf0 con 0x7fd8840cec00 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd88415c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd88415c460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40224/0 (socket says 172.21.15.86:40224) 2024-03-06T07:35:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 -- 172.21.15.86:0/3579100880 learned_addr learned my addr 172.21.15.86:0/3579100880 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd88a600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd88415e540 0x7fd8840c63d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889c00640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8840cec00 0x7fd8840b4740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 -- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8840cec00 msgr2=0x7fd8840b4740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8840cec00 0x7fd8840b4740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 -- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd88415e540 msgr2=0x7fd8840c63d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd88415e540 0x7fd8840c63d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 -- 172.21.15.86:0/3579100880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8840c92b0 con 0x7fd88415e160 2024-03-06T07:35:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.778+0000 7fd889200640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd88415c460 secure :-1 s=READY pgs=1570 cs=0 l=1 rev1=1 crypto rx=0x7fd87000c430 tx=0x7fd87000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:30.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.779+0000 7fd881600640 1 -- 172.21.15.86:0/3579100880 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd8700074e0 con 0x7fd88415e160 2024-03-06T07:35:30.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.779+0000 7fd881600640 1 -- 172.21.15.86:0/3579100880 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd8700047e0 con 0x7fd88415e160 2024-03-06T07:35:30.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.779+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd8840c6bc0 con 0x7fd88415e160 2024-03-06T07:35:30.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.779+0000 7fd881600640 1 -- 172.21.15.86:0/3579100880 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd870004bd0 con 0x7fd88415e160 2024-03-06T07:35:30.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.779+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd8840c7180 con 0x7fd88415e160 2024-03-06T07:35:30.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.780+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd88404f330 con 0x7fd88415e160 2024-03-06T07:35:30.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.781+0000 7fd881600640 1 -- 172.21.15.86:0/3579100880 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd870011070 con 0x7fd88415e160 2024-03-06T07:35:30.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.782+0000 7fd881600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd86406b4d0 0x7fd86406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:30.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.782+0000 7fd881600640 1 -- 172.21.15.86:0/3579100880 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd870089f10 con 0x7fd88415e160 2024-03-06T07:35:30.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.782+0000 7fd889c00640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd86406b4d0 0x7fd86406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:30.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.783+0000 7fd889c00640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd86406b4d0 0x7fd86406d990 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7fd878036000 tx=0x7fd878008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:30.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:30.790+0000 7fd881600640 1 -- 172.21.15.86:0/3579100880 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd870058d40 con 0x7fd88415e160 2024-03-06T07:35:31.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.029+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd884070770 con 0x7fd88415e160 2024-03-06T07:35:31.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.030+0000 7fd881600640 1 -- 172.21.15.86:0/3579100880 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7fd870058640 con 0x7fd88415e160 2024-03-06T07:35:31.032 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:31.032 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:31.032 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd86406b4d0 msgr2=0x7fd86406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd86406b4d0 0x7fd86406d990 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7fd878036000 tx=0x7fd878008030 comp rx=0 tx=0).stop 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 msgr2=0x7fd88415c460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd88415c460 secure :-1 s=READY pgs=1570 cs=0 l=1 rev1=1 crypto rx=0x7fd87000c430 tx=0x7fd87000c8f0 comp rx=0 tx=0).stop 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 shutdown_connections 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd86406b4d0 0x7fd86406d990 unknown :-1 s=CLOSED pgs=670 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd88415e540 0x7fd8840c63d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:31.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd88415e160 0x7fd88415c460 unknown :-1 s=CLOSED pgs=1570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:31.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 --2- 172.21.15.86:0/3579100880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd8840cec00 0x7fd8840b4740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:31.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 >> 172.21.15.86:0/3579100880 conn(0x7fd8840b3800 msgr2=0x7fd8840d0520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:31.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 shutdown_connections 2024-03-06T07:35:31.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:31.033+0000 7fd88c600640 1 -- 172.21.15.86:0/3579100880 wait complete. 2024-03-06T07:35:31.255 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:31.255 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:31.544 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:32.256 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:32.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.466+0000 7f3e3f600640 1 Processor -- start 2024-03-06T07:35:32.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.466+0000 7f3e3f600640 1 -- start start 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.467+0000 7f3e3f600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e38151660 0x7f3e38071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.467+0000 7f3e3f600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3e380cf430 0x7f3e3806d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3f600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3e38071a60 con 0x7f3e3806d920 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3f600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e380b3ee0 con 0x7f3e38151660 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3f600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e380b35b0 con 0x7f3e380cf430 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e38151660 0x7f3e38071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3d600640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3e3806d920 0x7f3e3815dff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59772/0 (socket says 172.21.15.86:59772) 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3d600640 1 -- 172.21.15.86:0/3293350877 learned_addr learned my addr 172.21.15.86:0/3293350877 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:32.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e37e00640 1 --2- 172.21.15.86:0/3293350877 >> v2:172.21.15.194:3300/0 conn(0x7f3e380cf430 0x7f3e3806d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:32.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3cc00640 1 -- 172.21.15.86:0/3293350877 >> v2:172.21.15.194:3300/0 conn(0x7f3e380cf430 msgr2=0x7f3e3806d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:32.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.468+0000 7f3e3cc00640 1 --2- 172.21.15.86:0/3293350877 >> v2:172.21.15.194:3300/0 conn(0x7f3e380cf430 0x7f3e3806d3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.469+0000 7f3e3cc00640 1 -- 172.21.15.86:0/3293350877 >> v1:172.21.15.86:6789/0 conn(0x7f3e3806d920 legacy=0x7f3e3815dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:32.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.469+0000 7f3e3cc00640 1 -- 172.21.15.86:0/3293350877 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e380b3890 con 0x7f3e38151660 2024-03-06T07:35:32.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.469+0000 7f3e37400640 1 -- 172.21.15.86:0/3293350877 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 421676188 0 0) 0x7f3e38071a60 con 0x7f3e3806d920 2024-03-06T07:35:32.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.470+0000 7f3e3cc00640 1 --2- 172.21.15.86:0/3293350877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e38151660 0x7f3e38071520 secure :-1 s=READY pgs=1572 cs=0 l=1 rev1=1 crypto rx=0x7f3e3806c8e0 tx=0x7f3e2802bdb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=68bb704eb0a855dd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.470+0000 7f3e37400640 1 -- 172.21.15.86:0/3293350877 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e28039070 con 0x7f3e38151660 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.470+0000 7f3e37400640 1 -- 172.21.15.86:0/3293350877 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e2802cb60 con 0x7f3e38151660 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.470+0000 7f3e37400640 1 -- 172.21.15.86:0/3293350877 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e28034440 con 0x7f3e38151660 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.470+0000 7f3e3f600640 1 -- 172.21.15.86:0/3293350877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e38151660 msgr2=0x7f3e38071520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.470+0000 7f3e3f600640 1 --2- 172.21.15.86:0/3293350877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e38151660 0x7f3e38071520 secure :-1 s=READY pgs=1572 cs=0 l=1 rev1=1 crypto rx=0x7f3e3806c8e0 tx=0x7f3e2802bdb0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.471+0000 7f3e3f600640 1 -- 172.21.15.86:0/3293350877 shutdown_connections 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.471+0000 7f3e3f600640 1 --2- 172.21.15.86:0/3293350877 >> v2:172.21.15.194:3300/0 conn(0x7f3e380cf430 0x7f3e3806d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.471+0000 7f3e3f600640 1 --2- 172.21.15.86:0/3293350877 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e38151660 0x7f3e38071520 unknown :-1 s=CLOSED pgs=1572 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.471+0000 7f3e3f600640 1 -- 172.21.15.86:0/3293350877 >> 172.21.15.86:0/3293350877 conn(0x7f3e380b1730 msgr2=0x7f3e380b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.471+0000 7f3e3f600640 1 -- 172.21.15.86:0/3293350877 shutdown_connections 2024-03-06T07:35:32.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.471+0000 7f3e3f600640 1 -- 172.21.15.86:0/3293350877 wait complete. 2024-03-06T07:35:32.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.472+0000 7f3e3f600640 1 Processor -- start 2024-03-06T07:35:32.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.473+0000 7f3e3f600640 1 -- start start 2024-03-06T07:35:32.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.473+0000 7f3e3f600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e3806d920 0x7f3e3814bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3f600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e380cf430 0x7f3e3813eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3f600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e38151660 0x7f3e381418a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3f600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e3815e700 con 0x7f3e3806d920 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3f600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e38071d30 con 0x7f3e380cf430 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3f600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3e3815e880 con 0x7f3e38151660 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e3806d920 0x7f3e3814bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e3806d920 0x7f3e3814bed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40254/0 (socket says 172.21.15.86:40254) 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 -- 172.21.15.86:0/488859596 learned_addr learned my addr 172.21.15.86:0/488859596 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e37e00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e380cf430 0x7f3e3813eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3d600640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e38151660 0x7f3e381418a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 -- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e38151660 msgr2=0x7f3e381418a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e38151660 0x7f3e381418a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 -- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e380cf430 msgr2=0x7f3e3813eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e380cf430 0x7f3e3813eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e3cc00640 1 -- 172.21.15.86:0/488859596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e3814e470 con 0x7f3e3806d920 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.474+0000 7f3e37e00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e380cf430 0x7f3e3813eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.475+0000 7f3e3cc00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e3806d920 0x7f3e3814bed0 secure :-1 s=READY pgs=1573 cs=0 l=1 rev1=1 crypto rx=0x7f3e28037040 tx=0x7f3e28004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.475+0000 7f3e34c00640 1 -- 172.21.15.86:0/488859596 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e28039050 con 0x7f3e3806d920 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.475+0000 7f3e34c00640 1 -- 172.21.15.86:0/488859596 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e28045070 con 0x7f3e3806d920 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.475+0000 7f3e34c00640 1 -- 172.21.15.86:0/488859596 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e2803ea50 con 0x7f3e3806d920 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.475+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e3813f530 con 0x7f3e3806d920 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.475+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e3814e9d0 con 0x7f3e3806d920 2024-03-06T07:35:32.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.476+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3e3804f330 con 0x7f3e3806d920 2024-03-06T07:35:32.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.478+0000 7f3e34c00640 1 -- 172.21.15.86:0/488859596 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3e2802cd10 con 0x7f3e3806d920 2024-03-06T07:35:32.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.479+0000 7f3e34c00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e1406b4d0 0x7f3e1406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:32.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.479+0000 7f3e34c00640 1 -- 172.21.15.86:0/488859596 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3e280aceb0 con 0x7f3e3806d920 2024-03-06T07:35:32.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.480+0000 7f3e37e00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e1406b4d0 0x7f3e1406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:32.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.480+0000 7f3e37e00640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e1406b4d0 0x7f3e1406d990 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f3e20006fd0 tx=0x7f3e20008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:32.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.484+0000 7f3e34c00640 1 -- 172.21.15.86:0/488859596 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3e3813f530 con 0x7f3e3806d920 2024-03-06T07:35:32.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.719+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3e3806e4f0 con 0x7f3e3806d920 2024-03-06T07:35:32.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.720+0000 7f3e34c00640 1 -- 172.21.15.86:0/488859596 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f3e3814e470 con 0x7f3e3806d920 2024-03-06T07:35:32.722 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:32.722 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:32.722 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:32.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e1406b4d0 msgr2=0x7f3e1406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:32.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e1406b4d0 0x7f3e1406d990 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f3e20006fd0 tx=0x7f3e20008040 comp rx=0 tx=0).stop 2024-03-06T07:35:32.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e3806d920 msgr2=0x7f3e3814bed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:32.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e3806d920 0x7f3e3814bed0 secure :-1 s=READY pgs=1573 cs=0 l=1 rev1=1 crypto rx=0x7f3e28037040 tx=0x7f3e28004a40 comp rx=0 tx=0).stop 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 shutdown_connections 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e1406b4d0 0x7f3e1406d990 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e38151660 0x7f3e381418a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e380cf430 0x7f3e3813eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.722+0000 7f3e3f600640 1 --2- 172.21.15.86:0/488859596 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e3806d920 0x7f3e3814bed0 unknown :-1 s=CLOSED pgs=1573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.723+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 >> 172.21.15.86:0/488859596 conn(0x7f3e380b1730 msgr2=0x7f3e3806e710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.723+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 shutdown_connections 2024-03-06T07:35:32.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:32.723+0000 7f3e3f600640 1 -- 172.21.15.86:0/488859596 wait complete. 2024-03-06T07:35:32.945 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:32.945 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:33.546 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:33.946 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:34.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.152+0000 7f5c31e00640 1 Processor -- start 2024-03-06T07:35:34.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.152+0000 7f5c31e00640 1 -- start start 2024-03-06T07:35:34.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.154+0000 7f5c31e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5c2c161580 0x7f5c2c0d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:34.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.154+0000 7f5c31e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c2c161960 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.154+0000 7f5c31e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c2c0cf6f0 con 0x7f5c2c0d1e90 2024-03-06T07:35:34.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.154+0000 7f5c31e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5c2c0cf9d0 con 0x7f5c2c161580 2024-03-06T07:35:34.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.154+0000 7f5c30a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5c2c161580 0x7f5c2c0d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:34.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.154+0000 7f5c31400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f5c2c0d2a70 0x7f5c2c0b3ec0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:59784/0 (socket says 172.21.15.86:59784) 2024-03-06T07:35:34.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.154+0000 7f5c31400640 1 -- 172.21.15.86:0/2585689129 learned_addr learned my addr 172.21.15.86:0/2585689129 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:34.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.155+0000 7f5c30a00640 1 -- 172.21.15.86:0/2585689129 >> v1:172.21.15.194:6789/0 conn(0x7f5c2c0d1e90 legacy=0x7f5c2c0d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:34.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.155+0000 7f5c30a00640 1 -- 172.21.15.86:0/2585689129 >> v1:172.21.15.86:6789/0 conn(0x7f5c2c0d2a70 legacy=0x7f5c2c0b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:34.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.155+0000 7f5c30a00640 1 -- 172.21.15.86:0/2585689129 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c2c0cfcb0 con 0x7f5c2c161580 2024-03-06T07:35:34.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.155+0000 7f5c2be00640 1 -- 172.21.15.86:0/2585689129 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2078573603 0 0) 0x7f5c2c161960 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.156+0000 7f5c30a00640 1 --2- 172.21.15.86:0/2585689129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c161580 0x7f5c2c0d1950 secure :-1 s=READY pgs=1575 cs=0 l=1 rev1=1 crypto rx=0x7f5c1c0097a0 tx=0x7f5c1c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=623a16dd1e7f5fe2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:34.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.156+0000 7f5c2be00640 1 -- 172.21.15.86:0/2585689129 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c1c035020 con 0x7f5c2c161580 2024-03-06T07:35:34.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.156+0000 7f5c2be00640 1 -- 172.21.15.86:0/2585689129 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c1c031830 con 0x7f5c2c161580 2024-03-06T07:35:34.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.156+0000 7f5c2be00640 1 -- 172.21.15.86:0/2585689129 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c1c031c00 con 0x7f5c2c161580 2024-03-06T07:35:34.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.157+0000 7f5c31e00640 1 -- 172.21.15.86:0/2585689129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c161580 msgr2=0x7f5c2c0d1950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:34.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.157+0000 7f5c31e00640 1 --2- 172.21.15.86:0/2585689129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c161580 0x7f5c2c0d1950 secure :-1 s=READY pgs=1575 cs=0 l=1 rev1=1 crypto rx=0x7f5c1c0097a0 tx=0x7f5c1c02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.157+0000 7f5c31e00640 1 -- 172.21.15.86:0/2585689129 shutdown_connections 2024-03-06T07:35:34.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.157+0000 7f5c31e00640 1 --2- 172.21.15.86:0/2585689129 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c161580 0x7f5c2c0d1950 unknown :-1 s=CLOSED pgs=1575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.157+0000 7f5c31e00640 1 -- 172.21.15.86:0/2585689129 >> 172.21.15.86:0/2585689129 conn(0x7f5c2c0d3d80 msgr2=0x7f5c2c0d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:34.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.157+0000 7f5c31e00640 1 -- 172.21.15.86:0/2585689129 shutdown_connections 2024-03-06T07:35:34.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.157+0000 7f5c31e00640 1 -- 172.21.15.86:0/2585689129 wait complete. 2024-03-06T07:35:34.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.159+0000 7f5c31e00640 1 Processor -- start 2024-03-06T07:35:34.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.159+0000 7f5c31e00640 1 -- start start 2024-03-06T07:35:34.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.159+0000 7f5c31e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c2c0d1e90 0x7f5c2c14e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:34.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c31e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c0d2a70 0x7f5c2c0cd2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:34.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c31e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c2c161580 0x7f5c2c0b9450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:34.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c31e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c2c0cf3a0 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c31e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c2c0b39a0 con 0x7f5c2c0d1e90 2024-03-06T07:35:34.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c31e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c2c0b45d0 con 0x7f5c2c161580 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c0d2a70 0x7f5c2c0cd2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c0d2a70 0x7f5c2c0cd2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40280/0 (socket says 172.21.15.86:40280) 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 -- 172.21.15.86:0/1939240426 learned_addr learned my addr 172.21.15.86:0/1939240426 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 -- 172.21.15.86:0/1939240426 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c2c161580 msgr2=0x7f5c2c0b9450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c2c161580 0x7f5c2c0b9450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 -- 172.21.15.86:0/1939240426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c2c0d1e90 msgr2=0x7f5c2c14e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c2c0d1e90 0x7f5c2c14e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 -- 172.21.15.86:0/1939240426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c2c0cd7f0 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.160+0000 7f5c23e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c0d2a70 0x7f5c2c0cd2b0 secure :-1 s=READY pgs=1576 cs=0 l=1 rev1=1 crypto rx=0x7f5c1400c460 tx=0x7f5c1400c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.161+0000 7f5c29600640 1 -- 172.21.15.86:0/1939240426 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c14016020 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.161+0000 7f5c29600640 1 -- 172.21.15.86:0/1939240426 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c14012870 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.161+0000 7f5c29600640 1 -- 172.21.15.86:0/1939240426 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c14012c60 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.161+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c2c0cc210 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.161+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c2c0cc7d0 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.162+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c2c04f330 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.163+0000 7f5c29600640 1 -- 172.21.15.86:0/1939240426 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5c14019020 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.164+0000 7f5c29600640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0806b4d0 0x7f5c0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:34.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.164+0000 7f5c29600640 1 -- 172.21.15.86:0/1939240426 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f5c1408a0a0 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.164+0000 7f5c30a00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0806b4d0 0x7f5c0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:34.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.165+0000 7f5c30a00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0806b4d0 0x7f5c0806d990 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f5c1c036000 tx=0x7f5c1c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:34.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.169+0000 7f5c29600640 1 -- 172.21.15.86:0/1939240426 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5c14020030 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.403+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5c2c0d3b60 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.404+0000 7f5c29600640 1 -- 172.21.15.86:0/1939240426 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f5c14058db0 con 0x7f5c2c0d2a70 2024-03-06T07:35:34.406 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:34.406 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:34.407 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0806b4d0 msgr2=0x7f5c0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0806b4d0 0x7f5c0806d990 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f5c1c036000 tx=0x7f5c1c030000 comp rx=0 tx=0).stop 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c0d2a70 msgr2=0x7f5c2c0cd2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c0d2a70 0x7f5c2c0cd2b0 secure :-1 s=READY pgs=1576 cs=0 l=1 rev1=1 crypto rx=0x7f5c1400c460 tx=0x7f5c1400c920 comp rx=0 tx=0).stop 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 shutdown_connections 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5c0806b4d0 0x7f5c0806d990 unknown :-1 s=CLOSED pgs=672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5c2c161580 0x7f5c2c0b9450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5c2c0d2a70 0x7f5c2c0cd2b0 unknown :-1 s=CLOSED pgs=1576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 --2- 172.21.15.86:0/1939240426 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5c2c0d1e90 0x7f5c2c14e2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:34.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 >> 172.21.15.86:0/1939240426 conn(0x7f5c2c0d3d80 msgr2=0x7f5c2c06d0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:34.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 shutdown_connections 2024-03-06T07:35:34.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:34.407+0000 7f5c31e00640 1 -- 172.21.15.86:0/1939240426 wait complete. 2024-03-06T07:35:34.630 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:34.630 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:35.548 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:35.632 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:35.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.836+0000 7fec19400640 1 Processor -- start 2024-03-06T07:35:35.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.837+0000 7fec19400640 1 -- start start 2024-03-06T07:35:35.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.838+0000 7fec19400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fec140d2360 0x7fec140d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:35.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.838+0000 7fec19400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fec140b3ec0 0x7fec140b4310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:35.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.838+0000 7fec19400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fec14161960 con 0x7fec14161580 2024-03-06T07:35:35.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.838+0000 7fec19400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fec140d2ce0 con 0x7fec140d2360 2024-03-06T07:35:35.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.838+0000 7fec19400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fec140cf6f0 con 0x7fec140b3ec0 2024-03-06T07:35:35.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.838+0000 7fec0b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fec140d2360 0x7fec140d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:35.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.839+0000 7fec0b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fec140d2360 0x7fec140d27a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40294/0 (socket says 172.21.15.86:40294) 2024-03-06T07:35:35.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.839+0000 7fec0b400640 1 -- 172.21.15.86:0/3044260889 learned_addr learned my addr 172.21.15.86:0/3044260889 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:35.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.839+0000 7fec18a00640 1 --2- 172.21.15.86:0/3044260889 >> v2:172.21.15.194:3300/0 conn(0x7fec140b3ec0 0x7fec140b4310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:35.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.839+0000 7fec0b400640 1 -- 172.21.15.86:0/3044260889 >> v2:172.21.15.194:3300/0 conn(0x7fec140b3ec0 msgr2=0x7fec140b4310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:35.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.839+0000 7fec0b400640 1 --2- 172.21.15.86:0/3044260889 >> v2:172.21.15.194:3300/0 conn(0x7fec140b3ec0 0x7fec140b4310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:35.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.839+0000 7fec0b400640 1 -- 172.21.15.86:0/3044260889 >> v1:172.21.15.194:6789/0 conn(0x7fec14161580 legacy=0x7fec140d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:35.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.839+0000 7fec0b400640 1 -- 172.21.15.86:0/3044260889 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec140cf9d0 con 0x7fec140d2360 2024-03-06T07:35:35.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.840+0000 7fec0b400640 1 --2- 172.21.15.86:0/3044260889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec140d2360 0x7fec140d27a0 secure :-1 s=READY pgs=1577 cs=0 l=1 rev1=1 crypto rx=0x7fec140d2e60 tx=0x7febfc02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29f695d6cff4c1d8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:35.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.840+0000 7fec13400640 1 -- 172.21.15.86:0/3044260889 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febfc035020 con 0x7fec140d2360 2024-03-06T07:35:35.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec13400640 1 -- 172.21.15.86:0/3044260889 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7febfc031870 con 0x7fec140d2360 2024-03-06T07:35:35.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec13400640 1 -- 172.21.15.86:0/3044260889 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febfc031c40 con 0x7fec140d2360 2024-03-06T07:35:35.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec19400640 1 -- 172.21.15.86:0/3044260889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec140d2360 msgr2=0x7fec140d27a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:35.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec19400640 1 --2- 172.21.15.86:0/3044260889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec140d2360 0x7fec140d27a0 secure :-1 s=READY pgs=1577 cs=0 l=1 rev1=1 crypto rx=0x7fec140d2e60 tx=0x7febfc02b910 comp rx=0 tx=0).stop 2024-03-06T07:35:35.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec19400640 1 -- 172.21.15.86:0/3044260889 shutdown_connections 2024-03-06T07:35:35.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec19400640 1 --2- 172.21.15.86:0/3044260889 >> v2:172.21.15.194:3300/0 conn(0x7fec140b3ec0 0x7fec140b4310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:35.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec19400640 1 --2- 172.21.15.86:0/3044260889 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec140d2360 0x7fec140d27a0 unknown :-1 s=CLOSED pgs=1577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:35.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec19400640 1 -- 172.21.15.86:0/3044260889 >> 172.21.15.86:0/3044260889 conn(0x7fec140d3d80 msgr2=0x7fec140d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:35.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.841+0000 7fec19400640 1 -- 172.21.15.86:0/3044260889 shutdown_connections 2024-03-06T07:35:35.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.842+0000 7fec19400640 1 -- 172.21.15.86:0/3044260889 wait complete. 2024-03-06T07:35:35.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.843+0000 7fec19400640 1 Processor -- start 2024-03-06T07:35:35.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.843+0000 7fec19400640 1 -- start start 2024-03-06T07:35:35.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.844+0000 7fec19400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fec140b3ec0 0x7fec140c9ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:35.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.844+0000 7fec19400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fec140d2360 0x7fec141f1180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:35.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.844+0000 7fec19400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec14161580 0x7fec141f43a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:35.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.844+0000 7fec19400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec140b39a0 con 0x7fec14161580 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.844+0000 7fec19400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec140b36c0 con 0x7fec140b3ec0 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.844+0000 7fec19400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fec140cf3a0 con 0x7fec140d2360 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec14161580 0x7fec141f43a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec14161580 0x7fec141f43a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:40310/0 (socket says 172.21.15.86:40310) 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 -- 172.21.15.86:0/1265610969 learned_addr learned my addr 172.21.15.86:0/1265610969 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec13e00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fec140b3ec0 0x7fec140c9ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec0b400640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fec140d2360 0x7fec141f1180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 -- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fec140d2360 msgr2=0x7fec141f1180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fec140d2360 0x7fec141f1180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 -- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fec140b3ec0 msgr2=0x7fec140c9ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fec140b3ec0 0x7fec140c9ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 -- 172.21.15.86:0/1265610969 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec140c90e0 con 0x7fec14161580 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec13e00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fec140b3ec0 0x7fec140c9ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:35.848 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.845+0000 7fec18a00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec14161580 0x7fec141f43a0 secure :-1 s=READY pgs=1578 cs=0 l=1 rev1=1 crypto rx=0x7fec0c00d600 tx=0x7fec0c00dac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:35.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.846+0000 7fec10c00640 1 -- 172.21.15.86:0/1265610969 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec0c0125f0 con 0x7fec14161580 2024-03-06T07:35:35.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.846+0000 7fec10c00640 1 -- 172.21.15.86:0/1265610969 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fec0c019070 con 0x7fec14161580 2024-03-06T07:35:35.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.846+0000 7fec10c00640 1 -- 172.21.15.86:0/1265610969 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec0c0129f0 con 0x7fec14161580 2024-03-06T07:35:35.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.846+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec140c93c0 con 0x7fec14161580 2024-03-06T07:35:35.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.846+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec140c9980 con 0x7fec14161580 2024-03-06T07:35:35.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.847+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fec140c9b80 con 0x7fec14161580 2024-03-06T07:35:35.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.849+0000 7fec10c00640 1 -- 172.21.15.86:0/1265610969 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fec0c012b90 con 0x7fec14161580 2024-03-06T07:35:35.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.850+0000 7fec10c00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febf006b4d0 0x7febf006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:35.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.850+0000 7fec10c00640 1 -- 172.21.15.86:0/1265610969 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fec0c08b0c0 con 0x7fec14161580 2024-03-06T07:35:35.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.850+0000 7fec13e00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febf006b4d0 0x7febf006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:35.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.851+0000 7fec13e00640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febf006b4d0 0x7febf006d990 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7fec1406d6b0 tx=0x7fec040073c0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:35.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:35.853+0000 7fec10c00640 1 -- 172.21.15.86:0/1265610969 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fec140c9b80 con 0x7fec14161580 2024-03-06T07:35:36.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.089+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fec1415f880 con 0x7fec14161580 2024-03-06T07:35:36.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.090+0000 7fec10c00640 1 -- 172.21.15.86:0/1265610969 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7fec1415f880 con 0x7fec14161580 2024-03-06T07:35:36.092 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:36.092 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:36.092 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febf006b4d0 msgr2=0x7febf006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febf006b4d0 0x7febf006d990 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7fec1406d6b0 tx=0x7fec040073c0 comp rx=0 tx=0).stop 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec14161580 msgr2=0x7fec141f43a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec14161580 0x7fec141f43a0 secure :-1 s=READY pgs=1578 cs=0 l=1 rev1=1 crypto rx=0x7fec0c00d600 tx=0x7fec0c00dac0 comp rx=0 tx=0).stop 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 shutdown_connections 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febf006b4d0 0x7febf006d990 unknown :-1 s=CLOSED pgs=673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fec14161580 0x7fec141f43a0 unknown :-1 s=CLOSED pgs=1578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:36.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fec140d2360 0x7fec141f1180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:36.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 --2- 172.21.15.86:0/1265610969 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fec140b3ec0 0x7fec140c9ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:36.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 >> 172.21.15.86:0/1265610969 conn(0x7fec140d3d80 msgr2=0x7fec14160850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:36.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 shutdown_connections 2024-03-06T07:35:36.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:36.093+0000 7fec19400640 1 -- 172.21.15.86:0/1265610969 wait complete. 2024-03-06T07:35:36.316 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:36.316 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:37.317 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:37.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.521+0000 7f8af4000640 1 Processor -- start 2024-03-06T07:35:37.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.521+0000 7f8af4000640 1 -- start start 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af4000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8aec0af1c0 0x7f8aec15eff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af4000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f8aec06e4d0 0x7f8aec06e940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af4000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8aec071d30 con 0x7f8aec070d20 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af4000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8aec06ee80 con 0x7f8aec0af1c0 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af4000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8aec0711d0 con 0x7f8aec06e4d0 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af1600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8aec0af1c0 0x7f8aec15eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af1600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8aec0af1c0 0x7f8aec15eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49872/0 (socket says 172.21.15.86:49872) 2024-03-06T07:35:37.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.523+0000 7f8af1600640 1 -- 172.21.15.86:0/2152042257 learned_addr learned my addr 172.21.15.86:0/2152042257 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:37.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.524+0000 7f8af2000640 1 --2- 172.21.15.86:0/2152042257 >> v2:172.21.15.194:3300/0 conn(0x7f8aec06e4d0 0x7f8aec06e940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:37.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.524+0000 7f8af1600640 1 -- 172.21.15.86:0/2152042257 >> v2:172.21.15.194:3300/0 conn(0x7f8aec06e4d0 msgr2=0x7f8aec06e940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:37.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.524+0000 7f8af1600640 1 --2- 172.21.15.86:0/2152042257 >> v2:172.21.15.194:3300/0 conn(0x7f8aec06e4d0 0x7f8aec06e940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.524+0000 7f8af1600640 1 -- 172.21.15.86:0/2152042257 >> v1:172.21.15.194:6789/0 conn(0x7f8aec070d20 legacy=0x7f8aec06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:37.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.524+0000 7f8af1600640 1 -- 172.21.15.86:0/2152042257 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8aec071490 con 0x7f8aec0af1c0 2024-03-06T07:35:37.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.525+0000 7f8af1600640 1 --2- 172.21.15.86:0/2152042257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec0af1c0 0x7f8aec15eff0 secure :-1 s=READY pgs=1579 cs=0 l=1 rev1=1 crypto rx=0x7f8ae0009870 tx=0x7f8ae002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4b5f48908bae3f2e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:37.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.525+0000 7f8aebe00640 1 -- 172.21.15.86:0/2152042257 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae0035020 con 0x7f8aec0af1c0 2024-03-06T07:35:37.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.525+0000 7f8aebe00640 1 -- 172.21.15.86:0/2152042257 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ae0031870 con 0x7f8aec0af1c0 2024-03-06T07:35:37.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.525+0000 7f8aebe00640 1 -- 172.21.15.86:0/2152042257 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae0031c40 con 0x7f8aec0af1c0 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 -- 172.21.15.86:0/2152042257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec0af1c0 msgr2=0x7f8aec15eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 --2- 172.21.15.86:0/2152042257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec0af1c0 0x7f8aec15eff0 secure :-1 s=READY pgs=1579 cs=0 l=1 rev1=1 crypto rx=0x7f8ae0009870 tx=0x7f8ae002b910 comp rx=0 tx=0).stop 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 -- 172.21.15.86:0/2152042257 shutdown_connections 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 --2- 172.21.15.86:0/2152042257 >> v2:172.21.15.194:3300/0 conn(0x7f8aec06e4d0 0x7f8aec06e940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 --2- 172.21.15.86:0/2152042257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec0af1c0 0x7f8aec15eff0 unknown :-1 s=CLOSED pgs=1579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 -- 172.21.15.86:0/2152042257 >> 172.21.15.86:0/2152042257 conn(0x7f8aec0b1730 msgr2=0x7f8aec0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 -- 172.21.15.86:0/2152042257 shutdown_connections 2024-03-06T07:35:37.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.526+0000 7f8af4000640 1 -- 172.21.15.86:0/2152042257 wait complete. 2024-03-06T07:35:37.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.528+0000 7f8af4000640 1 Processor -- start 2024-03-06T07:35:37.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.528+0000 7f8af4000640 1 -- start start 2024-03-06T07:35:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af4000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8aec06e4d0 0x7f8aec150190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af4000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec070d20 0x7f8aec1506d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af4000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8aec0af1c0 0x7f8aec0cc690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af4000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8aec0b30b0 con 0x7f8aec070d20 2024-03-06T07:35:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af4000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8aec071a50 con 0x7f8aec06e4d0 2024-03-06T07:35:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af4000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8aec06d370 con 0x7f8aec0af1c0 2024-03-06T07:35:37.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec070d20 0x7f8aec1506d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af0c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec070d20 0x7f8aec1506d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49888/0 (socket says 172.21.15.86:49888) 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.529+0000 7f8af0c00640 1 -- 172.21.15.86:0/1430552587 learned_addr learned my addr 172.21.15.86:0/1430552587 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af1600640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8aec06e4d0 0x7f8aec150190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af2000640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8aec0af1c0 0x7f8aec0cc690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af0c00640 1 -- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8aec0af1c0 msgr2=0x7f8aec0cc690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af0c00640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8aec0af1c0 0x7f8aec0cc690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af0c00640 1 -- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8aec06e4d0 msgr2=0x7f8aec150190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af0c00640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8aec06e4d0 0x7f8aec150190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af0c00640 1 -- 172.21.15.86:0/1430552587 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8aec156ee0 con 0x7f8aec070d20 2024-03-06T07:35:37.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8af0c00640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec070d20 0x7f8aec1506d0 secure :-1 s=READY pgs=1580 cs=0 l=1 rev1=1 crypto rx=0x7f8ad800c410 tx=0x7f8ad800c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:37.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.530+0000 7f8ae9600640 1 -- 172.21.15.86:0/1430552587 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ad8017070 con 0x7f8aec070d20 2024-03-06T07:35:37.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.531+0000 7f8ae9600640 1 -- 172.21.15.86:0/1430552587 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ad80035f0 con 0x7f8aec070d20 2024-03-06T07:35:37.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.531+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8aec1571c0 con 0x7f8aec070d20 2024-03-06T07:35:37.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.531+0000 7f8ae9600640 1 -- 172.21.15.86:0/1430552587 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ad80039e0 con 0x7f8aec070d20 2024-03-06T07:35:37.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.531+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8aec15e900 con 0x7f8aec070d20 2024-03-06T07:35:37.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.532+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8aec04f330 con 0x7f8aec070d20 2024-03-06T07:35:37.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.533+0000 7f8ae9600640 1 -- 172.21.15.86:0/1430552587 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8ad801a070 con 0x7f8aec070d20 2024-03-06T07:35:37.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.534+0000 7f8ae9600640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8acc06b4d0 0x7f8acc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:37.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.534+0000 7f8ae9600640 1 -- 172.21.15.86:0/1430552587 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f8ad8089640 con 0x7f8aec070d20 2024-03-06T07:35:37.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.534+0000 7f8af1600640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8acc06b4d0 0x7f8acc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:37.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.535+0000 7f8af1600640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8acc06b4d0 0x7f8acc06d990 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7f8ae0036000 tx=0x7f8ae003c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:37.543 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.542+0000 7f8ae9600640 1 -- 172.21.15.86:0/1430552587 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8ad8058510 con 0x7f8aec070d20 2024-03-06T07:35:37.550 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:37.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.771+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8aec15aeb0 con 0x7f8aec070d20 2024-03-06T07:35:37.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.772+0000 7f8ae9600640 1 -- 172.21.15.86:0/1430552587 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f8ad8057e10 con 0x7f8aec070d20 2024-03-06T07:35:37.774 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:37.774 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:37.774 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:37.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.774+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8acc06b4d0 msgr2=0x7f8acc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:37.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.774+0000 7f8af4000640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8acc06b4d0 0x7f8acc06d990 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7f8ae0036000 tx=0x7f8ae003c040 comp rx=0 tx=0).stop 2024-03-06T07:35:37.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.774+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec070d20 msgr2=0x7f8aec1506d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:37.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.774+0000 7f8af4000640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec070d20 0x7f8aec1506d0 secure :-1 s=READY pgs=1580 cs=0 l=1 rev1=1 crypto rx=0x7f8ad800c410 tx=0x7f8ad800c8d0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 shutdown_connections 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8acc06b4d0 0x7f8acc06d990 unknown :-1 s=CLOSED pgs=674 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8aec0af1c0 0x7f8aec0cc690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8aec070d20 0x7f8aec1506d0 unknown :-1 s=CLOSED pgs=1580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 --2- 172.21.15.86:0/1430552587 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8aec06e4d0 0x7f8aec150190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 >> 172.21.15.86:0/1430552587 conn(0x7f8aec0b1730 msgr2=0x7f8aec0ad860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 shutdown_connections 2024-03-06T07:35:37.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:37.775+0000 7f8af4000640 1 -- 172.21.15.86:0/1430552587 wait complete. 2024-03-06T07:35:37.995 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:37.995 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:38.996 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:39.205 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.203+0000 7fc485400640 1 Processor -- start 2024-03-06T07:35:39.205 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.203+0000 7fc485400640 1 -- start start 2024-03-06T07:35:39.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.205+0000 7fc485400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc48015f9d0 0x7fc48015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.205+0000 7fc485400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc480070b10 0x7fc480070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.205+0000 7fc485400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc48006cef0 con 0x7fc480144c00 2024-03-06T07:35:39.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.205+0000 7fc485400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc480160370 con 0x7fc480070b10 2024-03-06T07:35:39.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.205+0000 7fc485400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc48006d1d0 con 0x7fc48015f9d0 2024-03-06T07:35:39.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.205+0000 7fc47e800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc480144c00 0x7fc48006e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34024/0 (socket says 172.21.15.86:34024) 2024-03-06T07:35:39.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.205+0000 7fc47e800640 1 -- 172.21.15.86:0/994924899 learned_addr learned my addr 172.21.15.86:0/994924899 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47f200640 1 --2- 172.21.15.86:0/994924899 >> v2:172.21.15.194:3300/0 conn(0x7fc480070b10 0x7fc480070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc475e00640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc48015f9d0 0x7fc48015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 766921617 0 0) 0x7fc48006cef0 con 0x7fc480144c00 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc464003520 con 0x7fc480144c00 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1392760408 0 0) 0x7fc464003520 con 0x7fc480144c00 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc48015f9d0 msgr2=0x7fc48015fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc48015f9d0 0x7fc48015fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 >> v2:172.21.15.194:3300/0 conn(0x7fc480070b10 msgr2=0x7fc480070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 --2- 172.21.15.86:0/994924899 >> v2:172.21.15.194:3300/0 conn(0x7fc480070b10 0x7fc480070f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc48006d4b0 con 0x7fc480144c00 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc4700019f0 con 0x7fc480144c00 2024-03-06T07:35:39.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 >> v1:172.21.15.86:6789/0 conn(0x7fc480144c00 legacy=0x7fc48006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4640039b0 0x7fc464003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc464004800 0x7fc464004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc48015f9d0 0x7fc4640051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc48006d1d0 con 0x7fc4640039b0 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc46400b9c0 con 0x7fc464004800 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.206+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc46400bb40 con 0x7fc48015f9d0 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.207+0000 7fc47e800640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4640039b0 0x7fc464003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.207+0000 7fc47e800640 1 -- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc48015f9d0 msgr2=0x7fc4640051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.207+0000 7fc47e800640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc48015f9d0 0x7fc4640051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.207+0000 7fc47e800640 1 -- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc464004800 msgr2=0x7fc464004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.207+0000 7fc47e800640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc464004800 0x7fc464004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.207+0000 7fc47e800640 1 -- 172.21.15.86:0/994924899 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc464003520 con 0x7fc4640039b0 2024-03-06T07:35:39.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.208+0000 7fc47e800640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4640039b0 0x7fc464003dd0 secure :-1 s=READY pgs=1582 cs=0 l=1 rev1=1 crypto rx=0x7fc4700013d0 tx=0x7fc464005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9710e5675083a81 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:39.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.208+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc47000a4e0 con 0x7fc4640039b0 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.208+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc470034650 con 0x7fc4640039b0 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.208+0000 7fc47de00640 1 -- 172.21.15.86:0/994924899 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc470034a40 con 0x7fc4640039b0 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.209+0000 7fc485400640 1 -- 172.21.15.86:0/994924899 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4640039b0 msgr2=0x7fc464003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.209+0000 7fc485400640 1 --2- 172.21.15.86:0/994924899 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4640039b0 0x7fc464003dd0 secure :-1 s=READY pgs=1582 cs=0 l=1 rev1=1 crypto rx=0x7fc4700013d0 tx=0x7fc464005830 comp rx=0 tx=0).stop 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.209+0000 7fc47f200640 1 -- 172.21.15.86:0/994924899 reap_dead start 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.209+0000 7fc485400640 1 -- 172.21.15.86:0/994924899 shutdown_connections 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.209+0000 7fc485400640 1 -- 172.21.15.86:0/994924899 >> 172.21.15.86:0/994924899 conn(0x7fc4800b1730 msgr2=0x7fc4800b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.209+0000 7fc485400640 1 -- 172.21.15.86:0/994924899 shutdown_connections 2024-03-06T07:35:39.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.209+0000 7fc485400640 1 -- 172.21.15.86:0/994924899 wait complete. 2024-03-06T07:35:39.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.211+0000 7fc485400640 1 Processor -- start 2024-03-06T07:35:39.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.211+0000 7fc485400640 1 -- start start 2024-03-06T07:35:39.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.212+0000 7fc485400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4640039b0 0x7fc48013d740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.214 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.212+0000 7fc485400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4801f2590 0x7fc480158560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.212+0000 7fc485400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4801f29f0 0x7fc480158aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.212+0000 7fc485400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc48006cb50 con 0x7fc4801f2590 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.212+0000 7fc485400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc480160730 con 0x7fc4640039b0 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.212+0000 7fc485400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc48006ccd0 con 0x7fc4801f29f0 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.212+0000 7fc475e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4801f2590 0x7fc480158560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4801f2590 0x7fc480158560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49910/0 (socket says 172.21.15.86:49910) 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 -- 172.21.15.86:0/1244506801 learned_addr learned my addr 172.21.15.86:0/1244506801 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 -- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4801f29f0 msgr2=0x7fc480158aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc47e800640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4640039b0 0x7fc48013d740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc47f200640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4801f29f0 0x7fc480158aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4801f29f0 0x7fc480158aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 -- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4640039b0 msgr2=0x7fc48013d740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:39.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4640039b0 0x7fc48013d740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 -- 172.21.15.86:0/1244506801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc48013f7d0 con 0x7fc4801f2590 2024-03-06T07:35:39.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc475e00640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4801f2590 0x7fc480158560 secure :-1 s=READY pgs=1583 cs=0 l=1 rev1=1 crypto rx=0x7fc46800e7a0 tx=0x7fc46800ec60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:39.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.213+0000 7fc477400640 1 -- 172.21.15.86:0/1244506801 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc468017070 con 0x7fc4801f2590 2024-03-06T07:35:39.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.214+0000 7fc477400640 1 -- 172.21.15.86:0/1244506801 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc46800c940 con 0x7fc4801f2590 2024-03-06T07:35:39.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.214+0000 7fc477400640 1 -- 172.21.15.86:0/1244506801 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc46800cd10 con 0x7fc4801f2590 2024-03-06T07:35:39.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.214+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc480132e20 con 0x7fc4801f2590 2024-03-06T07:35:39.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.214+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc4801333e0 con 0x7fc4801f2590 2024-03-06T07:35:39.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.215+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc48004f330 con 0x7fc4801f2590 2024-03-06T07:35:39.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.216+0000 7fc477400640 1 -- 172.21.15.86:0/1244506801 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc46801a070 con 0x7fc4801f2590 2024-03-06T07:35:39.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.217+0000 7fc477400640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45c06b500 0x7fc45c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:39.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.217+0000 7fc477400640 1 -- 172.21.15.86:0/1244506801 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc46808a180 con 0x7fc4801f2590 2024-03-06T07:35:39.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.217+0000 7fc47e800640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45c06b500 0x7fc45c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:39.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.218+0000 7fc47e800640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45c06b500 0x7fc45c06d9c0 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7fc470034000 tx=0x7fc470009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:39.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.222+0000 7fc477400640 1 -- 172.21.15.86:0/1244506801 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc468058f20 con 0x7fc4801f2590 2024-03-06T07:35:39.457 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.456+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc480158fe0 con 0x7fc4801f2590 2024-03-06T07:35:39.458 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.457+0000 7fc477400640 1 -- 172.21.15.86:0/1244506801 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7fc468058820 con 0x7fc4801f2590 2024-03-06T07:35:39.458 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:39.459 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:39.459 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45c06b500 msgr2=0x7fc45c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45c06b500 0x7fc45c06d9c0 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7fc470034000 tx=0x7fc470009030 comp rx=0 tx=0).stop 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4801f2590 msgr2=0x7fc480158560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4801f2590 0x7fc480158560 secure :-1 s=READY pgs=1583 cs=0 l=1 rev1=1 crypto rx=0x7fc46800e7a0 tx=0x7fc46800ec60 comp rx=0 tx=0).stop 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 shutdown_connections 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc45c06b500 0x7fc45c06d9c0 unknown :-1 s=CLOSED pgs=675 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc4801f29f0 0x7fc480158aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc4801f2590 0x7fc480158560 unknown :-1 s=CLOSED pgs=1583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 --2- 172.21.15.86:0/1244506801 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc4640039b0 0x7fc48013d740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 >> 172.21.15.86:0/1244506801 conn(0x7fc4800b1730 msgr2=0x7fc480145040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:39.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 shutdown_connections 2024-03-06T07:35:39.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:39.461+0000 7fc485400640 1 -- 172.21.15.86:0/1244506801 wait complete. 2024-03-06T07:35:39.551 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:39.690 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:39.690 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:40.691 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.900+0000 7fd395800640 1 Processor -- start 2024-03-06T07:35:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.900+0000 7fd395800640 1 -- start start 2024-03-06T07:35:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.901+0000 7fd395800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd39015e0d0 0x7fd3900d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd395800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd390160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd395800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3900cea60 con 0x7fd39015f470 2024-03-06T07:35:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd395800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3900d2f60 con 0x7fd39015e0d0 2024-03-06T07:35:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd395800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3900d2350 con 0x7fd3900d0fc0 2024-03-06T07:35:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd38ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd39015e0d0 0x7fd3900d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd38ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd39015e0d0 0x7fd3900d1a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49924/0 (socket says 172.21.15.86:49924) 2024-03-06T07:35:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd38ea00640 1 -- 172.21.15.86:0/662579967 learned_addr learned my addr 172.21.15.86:0/662579967 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.902+0000 7fd38f400640 1 --2- 172.21.15.86:0/662579967 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd390160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.903+0000 7fd38ea00640 1 -- 172.21.15.86:0/662579967 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 msgr2=0x7fd390160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.903+0000 7fd38ea00640 1 --2- 172.21.15.86:0/662579967 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd390160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.903+0000 7fd38ea00640 1 -- 172.21.15.86:0/662579967 >> v1:172.21.15.194:6789/0 conn(0x7fd39015f470 legacy=0x7fd3900d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.903+0000 7fd38ea00640 1 -- 172.21.15.86:0/662579967 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3900d2630 con 0x7fd39015e0d0 2024-03-06T07:35:40.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.905+0000 7fd38ea00640 1 --2- 172.21.15.86:0/662579967 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd3900d1a90 secure :-1 s=READY pgs=1584 cs=0 l=1 rev1=1 crypto rx=0x7fd380009870 tx=0x7fd38002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d15f0937a73377cb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:40.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.906+0000 7fd38e000640 1 -- 172.21.15.86:0/662579967 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd38002f840 con 0x7fd39015e0d0 2024-03-06T07:35:40.907 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.906+0000 7fd38e000640 1 -- 172.21.15.86:0/662579967 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd38002f9e0 con 0x7fd39015e0d0 2024-03-06T07:35:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.906+0000 7fd38e000640 1 -- 172.21.15.86:0/662579967 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd38002fdb0 con 0x7fd39015e0d0 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.906+0000 7fd395800640 1 -- 172.21.15.86:0/662579967 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 msgr2=0x7fd3900d1a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.906+0000 7fd395800640 1 --2- 172.21.15.86:0/662579967 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd3900d1a90 secure :-1 s=READY pgs=1584 cs=0 l=1 rev1=1 crypto rx=0x7fd380009870 tx=0x7fd38002ba20 comp rx=0 tx=0).stop 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.907+0000 7fd395800640 1 -- 172.21.15.86:0/662579967 shutdown_connections 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.907+0000 7fd395800640 1 --2- 172.21.15.86:0/662579967 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd390160dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.907+0000 7fd395800640 1 --2- 172.21.15.86:0/662579967 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd3900d1a90 unknown :-1 s=CLOSED pgs=1584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.907+0000 7fd395800640 1 -- 172.21.15.86:0/662579967 >> 172.21.15.86:0/662579967 conn(0x7fd3900d3ec0 msgr2=0x7fd3900d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.907+0000 7fd395800640 1 -- 172.21.15.86:0/662579967 shutdown_connections 2024-03-06T07:35:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.907+0000 7fd395800640 1 -- 172.21.15.86:0/662579967 wait complete. 2024-03-06T07:35:40.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.909+0000 7fd395800640 1 Processor -- start 2024-03-06T07:35:40.911 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.910+0000 7fd395800640 1 -- start start 2024-03-06T07:35:40.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.911+0000 7fd395800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd3900ca080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:40.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.911+0000 7fd395800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd390155140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:40.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.911+0000 7fd395800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd39015f470 0x7fd3900c8230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:40.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.911+0000 7fd395800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd39015fb30 con 0x7fd39015e0d0 2024-03-06T07:35:40.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.911+0000 7fd395800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3900d2bf0 con 0x7fd39015f470 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.911+0000 7fd395800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3900d1fd0 con 0x7fd3900d0fc0 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd390155140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd390155140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49936/0 (socket says 172.21.15.86:49936) 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 -- 172.21.15.86:0/1480484169 learned_addr learned my addr 172.21.15.86:0/1480484169 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd38ea00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd3900ca080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd38f400640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd39015f470 0x7fd3900c8230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 -- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 msgr2=0x7fd3900ca080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:40.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd3900ca080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 -- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd39015f470 msgr2=0x7fd3900c8230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd39015f470 0x7fd3900c8230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 -- 172.21.15.86:0/1480484169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3900ca5c0 con 0x7fd39015e0d0 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd38f400640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd39015f470 0x7fd3900c8230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd38ea00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd3900ca080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.912+0000 7fd387e00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd390155140 secure :-1 s=READY pgs=1585 cs=0 l=1 rev1=1 crypto rx=0x7fd37800c3f0 tx=0x7fd37800c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.913+0000 7fd386a00640 1 -- 172.21.15.86:0/1480484169 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3780052d0 con 0x7fd39015e0d0 2024-03-06T07:35:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.913+0000 7fd386a00640 1 -- 172.21.15.86:0/1480484169 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd378005490 con 0x7fd39015e0d0 2024-03-06T07:35:40.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.913+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3900c8fe0 con 0x7fd39015e0d0 2024-03-06T07:35:40.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.913+0000 7fd386a00640 1 -- 172.21.15.86:0/1480484169 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd378007260 con 0x7fd39015e0d0 2024-03-06T07:35:40.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.913+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3900c95a0 con 0x7fd39015e0d0 2024-03-06T07:35:40.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.914+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd39004f330 con 0x7fd39015e0d0 2024-03-06T07:35:40.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.915+0000 7fd386a00640 1 -- 172.21.15.86:0/1480484169 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd378007400 con 0x7fd39015e0d0 2024-03-06T07:35:40.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.916+0000 7fd386a00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd36c06b4d0 0x7fd36c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:40.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.916+0000 7fd386a00640 1 -- 172.21.15.86:0/1480484169 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd378089700 con 0x7fd39015e0d0 2024-03-06T07:35:40.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.916+0000 7fd38ea00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd36c06b4d0 0x7fd36c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:40.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.917+0000 7fd38ea00640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd36c06b4d0 0x7fd36c06d990 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fd380036000 tx=0x7fd380039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:40.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:40.920+0000 7fd386a00640 1 -- 172.21.15.86:0/1480484169 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd378058540 con 0x7fd39015e0d0 2024-03-06T07:35:41.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.153+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd39008d920 con 0x7fd39015e0d0 2024-03-06T07:35:41.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.154+0000 7fd386a00640 1 -- 172.21.15.86:0/1480484169 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7fd378057e40 con 0x7fd39015e0d0 2024-03-06T07:35:41.156 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:41.156 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd36c06b4d0 msgr2=0x7fd36c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd36c06b4d0 0x7fd36c06d990 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fd380036000 tx=0x7fd380039040 comp rx=0 tx=0).stop 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 msgr2=0x7fd390155140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd390155140 secure :-1 s=READY pgs=1585 cs=0 l=1 rev1=1 crypto rx=0x7fd37800c3f0 tx=0x7fd37800c8b0 comp rx=0 tx=0).stop 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 shutdown_connections 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd36c06b4d0 0x7fd36c06d990 unknown :-1 s=CLOSED pgs=676 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd39015f470 0x7fd3900c8230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd39015e0d0 0x7fd390155140 unknown :-1 s=CLOSED pgs=1585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 --2- 172.21.15.86:0/1480484169 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3900d0fc0 0x7fd3900ca080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:41.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 >> 172.21.15.86:0/1480484169 conn(0x7fd3900d3ec0 msgr2=0x7fd3900b3be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:41.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 shutdown_connections 2024-03-06T07:35:41.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:41.157+0000 7fd395800640 1 -- 172.21.15.86:0/1480484169 wait complete. 2024-03-06T07:35:41.379 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:41.380 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:41.553 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:42.380 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:42.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.577+0000 7f1ab3600640 1 Processor -- start 2024-03-06T07:35:42.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.577+0000 7f1ab3600640 1 -- start start 2024-03-06T07:35:42.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.578+0000 7f1ab3600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f1aac15f9d0 0x7f1aac15fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:42.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.578+0000 7f1ab3600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f1aac070b10 0x7f1aac070f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:42.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.578+0000 7f1ab3600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1aac06cef0 con 0x7f1aac144c00 2024-03-06T07:35:42.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.578+0000 7f1ab3600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1aac06d1d0 con 0x7f1aac15f9d0 2024-03-06T07:35:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.578+0000 7f1ab3600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1aac06d4b0 con 0x7f1aac070b10 2024-03-06T07:35:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1ab0c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f1aac144c00 0x7f1aac06e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:34030/0 (socket says 172.21.15.86:34030) 2024-03-06T07:35:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1ab0c00640 1 -- 172.21.15.86:0/678432481 learned_addr learned my addr 172.21.15.86:0/678432481 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1aa3e00640 1 --2- 172.21.15.86:0/678432481 >> v2:172.21.15.86:3300/0 conn(0x7f1aac15f9d0 0x7f1aac15fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:42.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1ab1600640 1 --2- 172.21.15.86:0/678432481 >> v2:172.21.15.194:3300/0 conn(0x7f1aac070b10 0x7f1aac070f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:42.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1aabe00640 1 -- 172.21.15.86:0/678432481 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 126793502 0 0) 0x7f1aac06cef0 con 0x7f1aac144c00 2024-03-06T07:35:42.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1aabe00640 1 -- 172.21.15.86:0/678432481 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1a90003520 con 0x7f1aac144c00 2024-03-06T07:35:42.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1aa3e00640 1 -- 172.21.15.86:0/678432481 >> v2:172.21.15.194:3300/0 conn(0x7f1aac070b10 msgr2=0x7f1aac070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:42.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1aa3e00640 1 --2- 172.21.15.86:0/678432481 >> v2:172.21.15.194:3300/0 conn(0x7f1aac070b10 0x7f1aac070f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1aa3e00640 1 -- 172.21.15.86:0/678432481 >> v1:172.21.15.86:6789/0 conn(0x7f1aac144c00 legacy=0x7f1aac06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:42.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.579+0000 7f1aa3e00640 1 -- 172.21.15.86:0/678432481 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1aac06d790 con 0x7f1aac15f9d0 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.581+0000 7f1aa3e00640 1 --2- 172.21.15.86:0/678432481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15fe60 secure :-1 s=READY pgs=1587 cs=0 l=1 rev1=1 crypto rx=0x7f1aac06e500 tx=0x7f1a9402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=888fea7e9e9adbb0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.581+0000 7f1aabe00640 1 -- 172.21.15.86:0/678432481 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a94035020 con 0x7f1aac15f9d0 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.581+0000 7f1aabe00640 1 -- 172.21.15.86:0/678432481 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a94031830 con 0x7f1aac15f9d0 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.581+0000 7f1aabe00640 1 -- 172.21.15.86:0/678432481 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a94031c00 con 0x7f1aac15f9d0 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 -- 172.21.15.86:0/678432481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 msgr2=0x7f1aac15fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 --2- 172.21.15.86:0/678432481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15fe60 secure :-1 s=READY pgs=1587 cs=0 l=1 rev1=1 crypto rx=0x7f1aac06e500 tx=0x7f1a9402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 -- 172.21.15.86:0/678432481 shutdown_connections 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 --2- 172.21.15.86:0/678432481 >> v2:172.21.15.194:3300/0 conn(0x7f1aac070b10 0x7f1aac070f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 --2- 172.21.15.86:0/678432481 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15fe60 unknown :-1 s=CLOSED pgs=1587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 -- 172.21.15.86:0/678432481 >> 172.21.15.86:0/678432481 conn(0x7f1aac0b1730 msgr2=0x7f1aac0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:42.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 -- 172.21.15.86:0/678432481 shutdown_connections 2024-03-06T07:35:42.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.582+0000 7f1ab3600640 1 -- 172.21.15.86:0/678432481 wait complete. 2024-03-06T07:35:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.585+0000 7f1ab3600640 1 Processor -- start 2024-03-06T07:35:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.585+0000 7f1ab3600640 1 -- start start 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.586+0000 7f1ab3600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aac070b10 0x7f1aac15b770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.586+0000 7f1ab3600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aac144c00 0x7f1aac15de40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab3600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15e380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab3600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1aac06ccd0 con 0x7f1aac15f9d0 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab3600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1aac06cb50 con 0x7f1aac070b10 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab3600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1aac06df90 con 0x7f1aac144c00 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15e380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15e380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49952/0 (socket says 172.21.15.86:49952) 2024-03-06T07:35:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 -- 172.21.15.86:0/4042359051 learned_addr learned my addr 172.21.15.86:0/4042359051 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab0c00640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aac070b10 0x7f1aac15b770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1aa3e00640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aac144c00 0x7f1aac15de40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 -- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aac144c00 msgr2=0x7f1aac15de40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aac144c00 0x7f1aac15de40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 -- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aac070b10 msgr2=0x7f1aac15b770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aac070b10 0x7f1aac15b770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab1600640 1 -- 172.21.15.86:0/4042359051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1aac0aead0 con 0x7f1aac15f9d0 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.587+0000 7f1ab0c00640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aac070b10 0x7f1aac15b770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.588+0000 7f1ab1600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15e380 secure :-1 s=READY pgs=1588 cs=0 l=1 rev1=1 crypto rx=0x7f1aa4007e40 tx=0x7f1aa400e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.588+0000 7f1aa9600640 1 -- 172.21.15.86:0/4042359051 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1aa4014620 con 0x7f1aac15f9d0 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.588+0000 7f1aa9600640 1 -- 172.21.15.86:0/4042359051 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1aa4019070 con 0x7f1aac15f9d0 2024-03-06T07:35:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.588+0000 7f1aa9600640 1 -- 172.21.15.86:0/4042359051 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1aa4014a20 con 0x7f1aac15f9d0 2024-03-06T07:35:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.588+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1aac0aedb0 con 0x7f1aac15f9d0 2024-03-06T07:35:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.588+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1aac0ae750 con 0x7f1aac15f9d0 2024-03-06T07:35:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.589+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1aac04f330 con 0x7f1aac15f9d0 2024-03-06T07:35:42.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.590+0000 7f1aa9600640 1 -- 172.21.15.86:0/4042359051 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1aa4014bc0 con 0x7f1aac15f9d0 2024-03-06T07:35:42.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.591+0000 7f1aa9600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8806b500 0x7f1a8806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:42.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.591+0000 7f1aa9600640 1 -- 172.21.15.86:0/4042359051 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1aa408ab20 con 0x7f1aac15f9d0 2024-03-06T07:35:42.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.591+0000 7f1ab0c00640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8806b500 0x7f1a8806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:42.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.592+0000 7f1ab0c00640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8806b500 0x7f1a8806d9c0 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7f1a9c005880 tx=0x7f1a9c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:42.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.595+0000 7f1aa9600640 1 -- 172.21.15.86:0/4042359051 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1aa4059950 con 0x7f1aac15f9d0 2024-03-06T07:35:42.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.831+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1aac072890 con 0x7f1aac15f9d0 2024-03-06T07:35:42.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.832+0000 7f1aa9600640 1 -- 172.21.15.86:0/4042359051 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 101 v101) v1 ==== 95+0+3417 (secure 0 0 0) 0x7f1aa4059250 con 0x7f1aac15f9d0 2024-03-06T07:35:42.835 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:42.835 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":101,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":101,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:15.234758+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:42.835 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 101 2024-03-06T07:35:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.834+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8806b500 msgr2=0x7f1a8806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.834+0000 7f1ab3600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8806b500 0x7f1a8806d9c0 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7f1a9c005880 tx=0x7f1a9c009040 comp rx=0 tx=0).stop 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.834+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 msgr2=0x7f1aac15e380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.834+0000 7f1ab3600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15e380 secure :-1 s=READY pgs=1588 cs=0 l=1 rev1=1 crypto rx=0x7f1aa4007e40 tx=0x7f1aa400e4b0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 shutdown_connections 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f1a8806b500 0x7f1a8806d9c0 unknown :-1 s=CLOSED pgs=677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1aac15f9d0 0x7f1aac15e380 unknown :-1 s=CLOSED pgs=1588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1aac144c00 0x7f1aac15de40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 --2- 172.21.15.86:0/4042359051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1aac070b10 0x7f1aac15b770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 >> 172.21.15.86:0/4042359051 conn(0x7f1aac0b1730 msgr2=0x7f1aac0ce060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 shutdown_connections 2024-03-06T07:35:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:42.835+0000 7f1ab3600640 1 -- 172.21.15.86:0/4042359051 wait complete. 2024-03-06T07:35:43.056 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 101, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:15.234758+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:43.057 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:43.555 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:44.057 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.275+0000 7fe524a00640 1 Processor -- start 2024-03-06T07:35:44.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.275+0000 7fe524a00640 1 -- start start 2024-03-06T07:35:44.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe524a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe5200d26e0 0x7fe5200ceb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:44.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe524a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe52015d750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:44.280 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe524a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe52006c060 con 0x7fe5200d00c0 2024-03-06T07:35:44.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe524a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe52006be10 con 0x7fe5200d26e0 2024-03-06T07:35:44.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe524a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe52006c340 con 0x7fe5200b4930 2024-03-06T07:35:44.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe516a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fe5200d26e0 0x7fe5200ceb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:44.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe51fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe52015d750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:44.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.278+0000 7fe51f400640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7fe5200d00c0 0x7fe5200d04a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:34722/0 (socket says 172.21.15.86:34722) 2024-03-06T07:35:44.281 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.279+0000 7fe51f400640 1 -- 172.21.15.86:0/662045741 learned_addr learned my addr 172.21.15.86:0/662045741 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:44.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.279+0000 7fe516a00640 1 -- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 msgr2=0x7fe52015d750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:44.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.279+0000 7fe516a00640 1 --2- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe52015d750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.279+0000 7fe516a00640 1 -- 172.21.15.86:0/662045741 >> v1:172.21.15.194:6789/0 conn(0x7fe5200d00c0 legacy=0x7fe5200d04a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:44.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.279+0000 7fe516a00640 1 -- 172.21.15.86:0/662045741 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe52006c620 con 0x7fe5200d26e0 2024-03-06T07:35:44.282 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.279+0000 7fe51fe00640 1 --2- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe52015d750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:35:44.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.281+0000 7fe516a00640 1 --2- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d26e0 0x7fe5200ceb40 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7fe5080097a0 tx=0x7fe50802b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e8d14b8e662f90e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:44.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.281+0000 7fe51ea00640 1 -- 172.21.15.86:0/662045741 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe508035020 con 0x7fe5200d26e0 2024-03-06T07:35:44.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.281+0000 7fe51ea00640 1 -- 172.21.15.86:0/662045741 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe508031810 con 0x7fe5200d26e0 2024-03-06T07:35:44.284 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.281+0000 7fe51ea00640 1 -- 172.21.15.86:0/662045741 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe508031be0 con 0x7fe5200d26e0 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 -- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d26e0 msgr2=0x7fe5200ceb40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 --2- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d26e0 0x7fe5200ceb40 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7fe5080097a0 tx=0x7fe50802b8b0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 -- 172.21.15.86:0/662045741 shutdown_connections 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 --2- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe52015d750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 --2- 172.21.15.86:0/662045741 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d26e0 0x7fe5200ceb40 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 -- 172.21.15.86:0/662045741 >> 172.21.15.86:0/662045741 conn(0x7fe5200d1ae0 msgr2=0x7fe5200d1ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 -- 172.21.15.86:0/662045741 shutdown_connections 2024-03-06T07:35:44.285 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.282+0000 7fe524a00640 1 -- 172.21.15.86:0/662045741 wait complete. 2024-03-06T07:35:44.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.285+0000 7fe524a00640 1 Processor -- start 2024-03-06T07:35:44.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.285+0000 7fe524a00640 1 -- start start 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe524a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe520154930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe524a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d00c0 0x7fe52015cb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe524a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5200d26e0 0x7fe5200ca080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe524a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5200b7a10 con 0x7fe5200d26e0 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe524a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5200b7890 con 0x7fe5200d00c0 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe524a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5200cf8c0 con 0x7fe5200b4930 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5200d26e0 0x7fe5200ca080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5200d26e0 0x7fe5200ca080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49954/0 (socket says 172.21.15.86:49954) 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51fe00640 1 -- 172.21.15.86:0/1456331850 learned_addr learned my addr 172.21.15.86:0/1456331850 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:44.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51f400640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe520154930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe516a00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d00c0 0x7fe52015cb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51fe00640 1 -- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 msgr2=0x7fe520154930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51fe00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe520154930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51fe00640 1 -- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d00c0 msgr2=0x7fe52015cb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.287+0000 7fe51fe00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d00c0 0x7fe52015cb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.288+0000 7fe51fe00640 1 -- 172.21.15.86:0/1456331850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe52015d090 con 0x7fe5200d26e0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.288+0000 7fe51fe00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5200d26e0 0x7fe5200ca080 secure :-1 s=READY pgs=1589 cs=0 l=1 rev1=1 crypto rx=0x7fe51800efc0 tx=0x7fe51800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.288+0000 7fe517e00640 1 -- 172.21.15.86:0/1456331850 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe518018030 con 0x7fe5200d26e0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.288+0000 7fe517e00640 1 -- 172.21.15.86:0/1456331850 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe51800cd90 con 0x7fe5200d26e0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.288+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe52015d530 con 0x7fe5200d26e0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.288+0000 7fe517e00640 1 -- 172.21.15.86:0/1456331850 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe518003bb0 con 0x7fe5200d26e0 2024-03-06T07:35:44.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.288+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe52015daf0 con 0x7fe5200d26e0 2024-03-06T07:35:44.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.289+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe52004f330 con 0x7fe5200d26e0 2024-03-06T07:35:44.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.291+0000 7fe517e00640 1 -- 172.21.15.86:0/1456331850 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe518005d10 con 0x7fe5200d26e0 2024-03-06T07:35:44.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.291+0000 7fe517e00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4fc06b4d0 0x7fe4fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:44.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.292+0000 7fe517e00640 1 -- 172.21.15.86:0/1456331850 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fe51808ac20 con 0x7fe5200d26e0 2024-03-06T07:35:44.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.292+0000 7fe51f400640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4fc06b4d0 0x7fe4fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:44.295 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.292+0000 7fe51f400640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4fc06b4d0 0x7fe4fc06d990 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7fe510005880 tx=0x7fe510009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:44.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.297+0000 7fe517e00640 1 -- 172.21.15.86:0/1456331850 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe518022070 con 0x7fe5200d26e0 2024-03-06T07:35:44.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.527+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe5200cf3a0 con 0x7fe5200d26e0 2024-03-06T07:35:44.530 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.528+0000 7fe517e00640 1 -- 172.21.15.86:0/1456331850 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fe52015d090 con 0x7fe5200d26e0 2024-03-06T07:35:44.530 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:44.530 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:44.531 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:44.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.530+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4fc06b4d0 msgr2=0x7fe4fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:44.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.530+0000 7fe524a00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4fc06b4d0 0x7fe4fc06d990 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7fe510005880 tx=0x7fe510009040 comp rx=0 tx=0).stop 2024-03-06T07:35:44.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.530+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5200d26e0 msgr2=0x7fe5200ca080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:44.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.530+0000 7fe524a00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5200d26e0 0x7fe5200ca080 secure :-1 s=READY pgs=1589 cs=0 l=1 rev1=1 crypto rx=0x7fe51800efc0 tx=0x7fe51800c490 comp rx=0 tx=0).stop 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 shutdown_connections 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe4fc06b4d0 0x7fe4fc06d990 unknown :-1 s=CLOSED pgs=678 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe5200d26e0 0x7fe5200ca080 unknown :-1 s=CLOSED pgs=1589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe5200d00c0 0x7fe52015cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 --2- 172.21.15.86:0/1456331850 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe5200b4930 0x7fe520154930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 >> 172.21.15.86:0/1456331850 conn(0x7fe5200d1ae0 msgr2=0x7fe520161200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 shutdown_connections 2024-03-06T07:35:44.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:44.531+0000 7fe524a00640 1 -- 172.21.15.86:0/1456331850 wait complete. 2024-03-06T07:35:44.753 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:44.753 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:45.558 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:45.754 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:45.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.973+0000 7f19aca00640 1 Processor -- start 2024-03-06T07:35:45.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.973+0000 7f19aca00640 1 -- start start 2024-03-06T07:35:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.974+0000 7f19aca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80d0a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19aca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f19a80ce840 0x7f19a80cecb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:45.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19aca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f19a809d5e0 con 0x7f19a81615c0 2024-03-06T07:35:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19aca00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f19a80d4c00 con 0x7f19a80ce840 2024-03-06T07:35:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19aca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f19a80b40b0 con 0x7f19a80d0600 2024-03-06T07:35:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a7200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f19a80ce840 0x7f19a80cecb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a7200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f19a80ce840 0x7f19a80cecb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34510/0 (socket says 172.21.15.86:34510) 2024-03-06T07:35:45.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a7200640 1 -- 172.21.15.86:0/1980520233 learned_addr learned my addr 172.21.15.86:0/1980520233 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:45.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a5e00640 1 --2- 172.21.15.86:0/1980520233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80d0a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:45.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a7200640 1 -- 172.21.15.86:0/1980520233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 msgr2=0x7f19a80d0a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:45.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a7200640 1 --2- 172.21.15.86:0/1980520233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80d0a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:45.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a7200640 1 -- 172.21.15.86:0/1980520233 >> v1:172.21.15.194:6789/0 conn(0x7f19a81615c0 legacy=0x7f19a80d4440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:45.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.975+0000 7f19a7200640 1 -- 172.21.15.86:0/1980520233 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19a80b3780 con 0x7f19a80ce840 2024-03-06T07:35:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.977+0000 7f19a7200640 1 --2- 172.21.15.86:0/1980520233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a80ce840 0x7f19a80cecb0 secure :-1 s=READY pgs=1590 cs=0 l=1 rev1=1 crypto rx=0x7f19a80d4d80 tx=0x7f19a002be00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=55409134aef9765e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.977+0000 7f19a5400640 1 -- 172.21.15.86:0/1980520233 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19a002c9a0 con 0x7f19a80ce840 2024-03-06T07:35:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.977+0000 7f19a5400640 1 -- 172.21.15.86:0/1980520233 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f19a002cb40 con 0x7f19a80ce840 2024-03-06T07:35:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.977+0000 7f19a5400640 1 -- 172.21.15.86:0/1980520233 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19a00343f0 con 0x7f19a80ce840 2024-03-06T07:35:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.977+0000 7f19aca00640 1 -- 172.21.15.86:0/1980520233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a80ce840 msgr2=0x7f19a80cecb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:45.980 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.977+0000 7f19aca00640 1 --2- 172.21.15.86:0/1980520233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a80ce840 0x7f19a80cecb0 secure :-1 s=READY pgs=1590 cs=0 l=1 rev1=1 crypto rx=0x7f19a80d4d80 tx=0x7f19a002be00 comp rx=0 tx=0).stop 2024-03-06T07:35:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.978+0000 7f19aca00640 1 -- 172.21.15.86:0/1980520233 shutdown_connections 2024-03-06T07:35:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.978+0000 7f19aca00640 1 --2- 172.21.15.86:0/1980520233 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a80ce840 0x7f19a80cecb0 unknown :-1 s=CLOSED pgs=1590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.978+0000 7f19aca00640 1 --2- 172.21.15.86:0/1980520233 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80d0a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.978+0000 7f19aca00640 1 -- 172.21.15.86:0/1980520233 >> 172.21.15.86:0/1980520233 conn(0x7f19a809d080 msgr2=0x7f19a80d3d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.978+0000 7f19aca00640 1 -- 172.21.15.86:0/1980520233 shutdown_connections 2024-03-06T07:35:45.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.978+0000 7f19aca00640 1 -- 172.21.15.86:0/1980520233 wait complete. 2024-03-06T07:35:45.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.979+0000 7f19aca00640 1 Processor -- start 2024-03-06T07:35:45.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.980+0000 7f19aca00640 1 -- start start 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.980+0000 7f19aca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f19a80ce840 0x7f19a815d130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.980+0000 7f19aca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80cb860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.980+0000 7f19aca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a81615c0 0x7f19a80cca90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.980+0000 7f19aca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19a80b4540 con 0x7f19a81615c0 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.980+0000 7f19aca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19a80b3d40 con 0x7f19a80ce840 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.980+0000 7f19aca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f19a80b46c0 con 0x7f19a80d0600 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a81615c0 0x7f19a80cca90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a81615c0 0x7f19a80cca90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34524/0 (socket says 172.21.15.86:34524) 2024-03-06T07:35:45.984 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 -- 172.21.15.86:0/3102995848 learned_addr learned my addr 172.21.15.86:0/3102995848 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a5e00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80cb860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a6800640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f19a80ce840 0x7f19a815d130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 -- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 msgr2=0x7f19a80cb860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80cb860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 -- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f19a80ce840 msgr2=0x7f19a815d130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f19a80ce840 0x7f19a815d130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.981+0000 7f19a7200640 1 -- 172.21.15.86:0/3102995848 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19a80cd710 con 0x7f19a81615c0 2024-03-06T07:35:45.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.982+0000 7f19a7200640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a81615c0 0x7f19a80cca90 secure :-1 s=READY pgs=1591 cs=0 l=1 rev1=1 crypto rx=0x7f19a0009f10 tx=0x7f19a0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.982+0000 7f199ea00640 1 -- 172.21.15.86:0/3102995848 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19a0043020 con 0x7f19a81615c0 2024-03-06T07:35:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.982+0000 7f199ea00640 1 -- 172.21.15.86:0/3102995848 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f19a003e450 con 0x7f19a81615c0 2024-03-06T07:35:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.982+0000 7f199ea00640 1 -- 172.21.15.86:0/3102995848 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19a0009510 con 0x7f19a81615c0 2024-03-06T07:35:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.982+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f19a80cd9f0 con 0x7f19a81615c0 2024-03-06T07:35:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.982+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f19a80cc290 con 0x7f19a81615c0 2024-03-06T07:35:45.986 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.983+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f19a804f330 con 0x7f19a81615c0 2024-03-06T07:35:45.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.984+0000 7f199ea00640 1 -- 172.21.15.86:0/3102995848 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f19a0046020 con 0x7f19a81615c0 2024-03-06T07:35:45.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.985+0000 7f199ea00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f198806b4d0 0x7f198806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:45.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.985+0000 7f199ea00640 1 -- 172.21.15.86:0/3102995848 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f19a00aca60 con 0x7f19a81615c0 2024-03-06T07:35:45.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.985+0000 7f19a6800640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f198806b4d0 0x7f198806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:45.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.986+0000 7f19a6800640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f198806b4d0 0x7f198806d990 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f1998006fd0 tx=0x7f1998008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:45.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:45.990+0000 7f199ea00640 1 -- 172.21.15.86:0/3102995848 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f19a004d030 con 0x7f19a81615c0 2024-03-06T07:35:46.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.227+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f19a80b9bb0 con 0x7f19a81615c0 2024-03-06T07:35:46.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.228+0000 7f199ea00640 1 -- 172.21.15.86:0/3102995848 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f19a007b890 con 0x7f19a81615c0 2024-03-06T07:35:46.231 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:46.231 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:46.231 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:46.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f198806b4d0 msgr2=0x7f198806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:46.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f198806b4d0 0x7f198806d990 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f1998006fd0 tx=0x7f1998008040 comp rx=0 tx=0).stop 2024-03-06T07:35:46.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a81615c0 msgr2=0x7f19a80cca90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a81615c0 0x7f19a80cca90 secure :-1 s=READY pgs=1591 cs=0 l=1 rev1=1 crypto rx=0x7f19a0009f10 tx=0x7f19a0004a40 comp rx=0 tx=0).stop 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 shutdown_connections 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f198806b4d0 0x7f198806d990 unknown :-1 s=CLOSED pgs=679 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f19a81615c0 0x7f19a80cca90 unknown :-1 s=CLOSED pgs=1591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f19a80d0600 0x7f19a80cb860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 --2- 172.21.15.86:0/3102995848 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f19a80ce840 0x7f19a815d130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 >> 172.21.15.86:0/3102995848 conn(0x7f19a809d080 msgr2=0x7f19a8160410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 shutdown_connections 2024-03-06T07:35:46.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:46.231+0000 7f19aca00640 1 -- 172.21.15.86:0/3102995848 wait complete. 2024-03-06T07:35:46.454 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:46.454 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:47.456 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:47.559 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:47.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.666+0000 7f7616a00640 1 Processor -- start 2024-03-06T07:35:47.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.666+0000 7f7616a00640 1 -- start start 2024-03-06T07:35:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.667+0000 7f7616a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7610151660 0x7f7610071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f7616a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f76100cf430 0x7f761006d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f7616a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7610071a60 con 0x7f761006d920 2024-03-06T07:35:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f7616a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f76100b3ee0 con 0x7f76100cf430 2024-03-06T07:35:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f7616a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76100b35b0 con 0x7f7610151660 2024-03-06T07:35:47.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f7614a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f761006d920 0x7f761015dff0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38012/0 (socket says 172.21.15.86:38012) 2024-03-06T07:35:47.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f7614a00640 1 -- 172.21.15.86:0/557745618 learned_addr learned my addr 172.21.15.86:0/557745618 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:47.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f760fe00640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7610151660 0x7f7610071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2221491967 0 0) 0x7f7610071a60 con 0x7f761006d920 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f75f4003520 con 0x7f761006d920 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.668+0000 7f7607400640 1 --2- 172.21.15.86:0/557745618 >> v2:172.21.15.194:3300/0 conn(0x7f76100cf430 0x7f761006d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2473078963 0 0) 0x7f75f4003520 con 0x7f761006d920 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7610151660 msgr2=0x7f7610071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7610151660 0x7f7610071520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 >> v2:172.21.15.194:3300/0 conn(0x7f76100cf430 msgr2=0x7f761006d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 --2- 172.21.15.86:0/557745618 >> v2:172.21.15.194:3300/0 conn(0x7f76100cf430 0x7f761006d3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76100b3890 con 0x7f761006d920 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7608001670 con 0x7f761006d920 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 >> v1:172.21.15.86:6789/0 conn(0x7f761006d920 legacy=0x7f761015dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75f40039b0 0x7f75f4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4004800 0x7f75f4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75f4005210 0x7f75f400dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76100b3ee0 con 0x7f75f4005210 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76100b35b0 con 0x7f75f40039b0 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75f400e3a0 con 0x7f75f4004800 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f760fe00640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7610151660 0x7f7610071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.669+0000 7f7614a00640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75f4005210 0x7f75f400dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.670+0000 7f7607400640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4004800 0x7f75f4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.670+0000 7f7614a00640 1 -- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4004800 msgr2=0x7f75f4004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.670+0000 7f7614a00640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4004800 0x7f75f4004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.670+0000 7f7614a00640 1 -- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75f40039b0 msgr2=0x7f75f4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.670+0000 7f7614a00640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f75f40039b0 0x7f75f4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.670+0000 7f7614a00640 1 -- 172.21.15.86:0/557745618 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75f4003520 con 0x7f75f4005210 2024-03-06T07:35:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.671+0000 7f7614a00640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75f4005210 0x7f75f400dbd0 secure :-1 s=READY pgs=1593 cs=0 l=1 rev1=1 crypto rx=0x7f760800c190 tx=0x7f75f4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80f531f2bf71d76b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.671+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7608007c80 con 0x7f75f4005210 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.671+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7608030070 con 0x7f75f4005210 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.671+0000 7f760f400640 1 -- 172.21.15.86:0/557745618 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7608005d10 con 0x7f75f4005210 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.671+0000 7f7616a00640 1 -- 172.21.15.86:0/557745618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75f4005210 msgr2=0x7f75f400dbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.671+0000 7f7616a00640 1 --2- 172.21.15.86:0/557745618 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f75f4005210 0x7f75f400dbd0 secure :-1 s=READY pgs=1593 cs=0 l=1 rev1=1 crypto rx=0x7f760800c190 tx=0x7f75f4005830 comp rx=0 tx=0).stop 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.671+0000 7f7614a00640 1 -- 172.21.15.86:0/557745618 reap_dead start 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.672+0000 7f7616a00640 1 -- 172.21.15.86:0/557745618 shutdown_connections 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.672+0000 7f7616a00640 1 -- 172.21.15.86:0/557745618 >> 172.21.15.86:0/557745618 conn(0x7f76100b1730 msgr2=0x7f76100b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.672+0000 7f7616a00640 1 -- 172.21.15.86:0/557745618 shutdown_connections 2024-03-06T07:35:47.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.672+0000 7f7616a00640 1 -- 172.21.15.86:0/557745618 wait complete. 2024-03-06T07:35:47.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.673+0000 7f7616a00640 1 Processor -- start 2024-03-06T07:35:47.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.674+0000 7f7616a00640 1 -- start start 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.674+0000 7f7616a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4005210 0x7f76100cd630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.674+0000 7f7616a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76100acc40 0x7f761014a0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.674+0000 7f7616a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7610153d60 0x7f761014a600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.674+0000 7f7616a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f761015e700 con 0x7f76100acc40 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.674+0000 7f7616a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7610071d30 con 0x7f7610153d60 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.674+0000 7f7616a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f761015e880 con 0x7f75f4005210 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76100acc40 0x7f761014a0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76100acc40 0x7f761014a0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34546/0 (socket says 172.21.15.86:34546) 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 -- 172.21.15.86:0/562326526 learned_addr learned my addr 172.21.15.86:0/562326526 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f760fe00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4005210 0x7f76100cd630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7614a00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7610153d60 0x7f761014a600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 -- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4005210 msgr2=0x7f76100cd630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4005210 0x7f76100cd630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 -- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7610153d60 msgr2=0x7f761014a600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:47.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7610153d60 0x7f761014a600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 -- 172.21.15.86:0/562326526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76101542b0 con 0x7f76100acc40 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f760fe00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4005210 0x7f76100cd630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.675+0000 7f7607400640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76100acc40 0x7f761014a0c0 secure :-1 s=READY pgs=1594 cs=0 l=1 rev1=1 crypto rx=0x7f75f800e8e0 tx=0x7f75f800eda0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.676+0000 7f760cc00640 1 -- 172.21.15.86:0/562326526 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75f8018020 con 0x7f76100acc40 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.676+0000 7f760cc00640 1 -- 172.21.15.86:0/562326526 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75f800abe0 con 0x7f76100acc40 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.676+0000 7f760cc00640 1 -- 172.21.15.86:0/562326526 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75f800b040 con 0x7f76100acc40 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.676+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7610154590 con 0x7f76100acc40 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.676+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7610154b50 con 0x7f76100acc40 2024-03-06T07:35:47.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.677+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f76100a9510 con 0x7f76100acc40 2024-03-06T07:35:47.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.678+0000 7f760cc00640 1 -- 172.21.15.86:0/562326526 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f75f801b020 con 0x7f76100acc40 2024-03-06T07:35:47.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.679+0000 7f760cc00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75ec06b500 0x7f75ec06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:47.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.679+0000 7f760cc00640 1 -- 172.21.15.86:0/562326526 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f75f808a3d0 con 0x7f76100acc40 2024-03-06T07:35:47.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.679+0000 7f760fe00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75ec06b500 0x7f75ec06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:47.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.680+0000 7f760fe00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75ec06b500 0x7f75ec06d9c0 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f76000094c0 tx=0x7f760000b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:47.686 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.684+0000 7f760cc00640 1 -- 172.21.15.86:0/562326526 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f75f8022020 con 0x7f76100acc40 2024-03-06T07:35:47.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.918+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f761004f330 con 0x7f76100acc40 2024-03-06T07:35:47.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.919+0000 7f760cc00640 1 -- 172.21.15.86:0/562326526 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f75f8058c50 con 0x7f76100acc40 2024-03-06T07:35:47.922 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:47.922 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:47.923 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:47.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75ec06b500 msgr2=0x7f75ec06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:47.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75ec06b500 0x7f75ec06d9c0 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f76000094c0 tx=0x7f760000b040 comp rx=0 tx=0).stop 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76100acc40 msgr2=0x7f761014a0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76100acc40 0x7f761014a0c0 secure :-1 s=READY pgs=1594 cs=0 l=1 rev1=1 crypto rx=0x7f75f800e8e0 tx=0x7f75f800eda0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 shutdown_connections 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f75ec06b500 0x7f75ec06d9c0 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7610153d60 0x7f761014a600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f76100acc40 0x7f761014a0c0 unknown :-1 s=CLOSED pgs=1594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 --2- 172.21.15.86:0/562326526 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f75f4005210 0x7f76100cd630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.922+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 >> 172.21.15.86:0/562326526 conn(0x7f76100b1730 msgr2=0x7f761006d080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.923+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 shutdown_connections 2024-03-06T07:35:47.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:47.923+0000 7f7616a00640 1 -- 172.21.15.86:0/562326526 wait complete. 2024-03-06T07:35:48.149 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:48.149 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:49.150 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:49.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.363+0000 7fbc68a00640 1 Processor -- start 2024-03-06T07:35:49.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.363+0000 7fbc68a00640 1 -- start start 2024-03-06T07:35:49.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.364+0000 7fbc68a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbc640af5a0 0x7fbc640b5db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:49.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc68a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fbc64072670 0x7fbc64072ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:49.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc68a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbc64066f50 con 0x7fbc6415be20 2024-03-06T07:35:49.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc68a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbc64073000 con 0x7fbc640af5a0 2024-03-06T07:35:49.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc68a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbc64066d00 con 0x7fbc64072670 2024-03-06T07:35:49.368 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc61e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbc640af5a0 0x7fbc640b5db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:49.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc61e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbc640af5a0 0x7fbc640b5db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34562/0 (socket says 172.21.15.86:34562) 2024-03-06T07:35:49.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc61e00640 1 -- 172.21.15.86:0/3996522326 learned_addr learned my addr 172.21.15.86:0/3996522326 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:49.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc63200640 1 --2- 172.21.15.86:0/3996522326 >> v2:172.21.15.194:3300/0 conn(0x7fbc64072670 0x7fbc64072ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:49.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc61e00640 1 -- 172.21.15.86:0/3996522326 >> v2:172.21.15.194:3300/0 conn(0x7fbc64072670 msgr2=0x7fbc64072ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:49.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc61e00640 1 --2- 172.21.15.86:0/3996522326 >> v2:172.21.15.194:3300/0 conn(0x7fbc64072670 0x7fbc64072ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.365+0000 7fbc61e00640 1 -- 172.21.15.86:0/3996522326 >> v1:172.21.15.194:6789/0 conn(0x7fbc6415be20 legacy=0x7fbc640b16a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:49.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.366+0000 7fbc61e00640 1 -- 172.21.15.86:0/3996522326 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc64067230 con 0x7fbc640af5a0 2024-03-06T07:35:49.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.367+0000 7fbc61e00640 1 --2- 172.21.15.86:0/3996522326 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640b5db0 secure :-1 s=READY pgs=1595 cs=0 l=1 rev1=1 crypto rx=0x7fbc640afc40 tx=0x7fbc5002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=381a6dfe3dedc3ea server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:49.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.367+0000 7fbc61400640 1 -- 172.21.15.86:0/3996522326 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc50035020 con 0x7fbc640af5a0 2024-03-06T07:35:49.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.367+0000 7fbc61400640 1 -- 172.21.15.86:0/3996522326 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc50031870 con 0x7fbc640af5a0 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.367+0000 7fbc61400640 1 -- 172.21.15.86:0/3996522326 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc50031c40 con 0x7fbc640af5a0 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 -- 172.21.15.86:0/3996522326 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 msgr2=0x7fbc640b5db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 --2- 172.21.15.86:0/3996522326 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640b5db0 secure :-1 s=READY pgs=1595 cs=0 l=1 rev1=1 crypto rx=0x7fbc640afc40 tx=0x7fbc5002b910 comp rx=0 tx=0).stop 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 -- 172.21.15.86:0/3996522326 shutdown_connections 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 --2- 172.21.15.86:0/3996522326 >> v2:172.21.15.194:3300/0 conn(0x7fbc64072670 0x7fbc64072ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 --2- 172.21.15.86:0/3996522326 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640b5db0 unknown :-1 s=CLOSED pgs=1595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 -- 172.21.15.86:0/3996522326 >> 172.21.15.86:0/3996522326 conn(0x7fbc640b4fe0 msgr2=0x7fbc640b56a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 -- 172.21.15.86:0/3996522326 shutdown_connections 2024-03-06T07:35:49.371 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.368+0000 7fbc68a00640 1 -- 172.21.15.86:0/3996522326 wait complete. 2024-03-06T07:35:49.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.371+0000 7fbc68a00640 1 Processor -- start 2024-03-06T07:35:49.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.371+0000 7fbc68a00640 1 -- start start 2024-03-06T07:35:49.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.372+0000 7fbc68a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc64072670 0x7fbc640c9500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:49.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.372+0000 7fbc68a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640cb990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:49.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.372+0000 7fbc68a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc6415be20 0x7fbc640ad160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:49.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.372+0000 7fbc68a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc640b0b50 con 0x7fbc640af5a0 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.372+0000 7fbc68a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc640b1180 con 0x7fbc6415be20 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.372+0000 7fbc68a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbc6415c200 con 0x7fbc64072670 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640cb990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640cb990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34564/0 (socket says 172.21.15.86:34564) 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 -- 172.21.15.86:0/1090140799 learned_addr learned my addr 172.21.15.86:0/1090140799 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc63200640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc6415be20 0x7fbc640ad160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc62800640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc64072670 0x7fbc640c9500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 -- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc64072670 msgr2=0x7fbc640c9500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc64072670 0x7fbc640c9500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 -- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc6415be20 msgr2=0x7fbc640ad160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc6415be20 0x7fbc640ad160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 -- 172.21.15.86:0/1090140799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc6409c040 con 0x7fbc640af5a0 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.373+0000 7fbc61e00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640cb990 secure :-1 s=READY pgs=1596 cs=0 l=1 rev1=1 crypto rx=0x7fbc50000c00 tx=0x7fbc500046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.374+0000 7fbc5aa00640 1 -- 172.21.15.86:0/1090140799 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc50040070 con 0x7fbc640af5a0 2024-03-06T07:35:49.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.374+0000 7fbc5aa00640 1 -- 172.21.15.86:0/1090140799 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc50030d00 con 0x7fbc640af5a0 2024-03-06T07:35:49.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.374+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc640c9e50 con 0x7fbc640af5a0 2024-03-06T07:35:49.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.374+0000 7fbc5aa00640 1 -- 172.21.15.86:0/1090140799 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc50035040 con 0x7fbc640af5a0 2024-03-06T07:35:49.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.374+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc6409c2e0 con 0x7fbc640af5a0 2024-03-06T07:35:49.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.375+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc640afe90 con 0x7fbc640af5a0 2024-03-06T07:35:49.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.376+0000 7fbc5aa00640 1 -- 172.21.15.86:0/1090140799 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbc5004b060 con 0x7fbc640af5a0 2024-03-06T07:35:49.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.378+0000 7fbc5aa00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc4406b4d0 0x7fbc4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:49.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.378+0000 7fbc5aa00640 1 -- 172.21.15.86:0/1090140799 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbc500ac0e0 con 0x7fbc640af5a0 2024-03-06T07:35:49.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.378+0000 7fbc62800640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc4406b4d0 0x7fbc4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:49.381 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.379+0000 7fbc62800640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc4406b4d0 0x7fbc4406d990 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7fbc5400a750 tx=0x7fbc54008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.382+0000 7fbc5aa00640 1 -- 172.21.15.86:0/1090140799 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbc640afe90 con 0x7fbc640af5a0 2024-03-06T07:35:49.561 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:49.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.617+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbc6404f330 con 0x7fbc640af5a0 2024-03-06T07:35:49.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.618+0000 7fbc5aa00640 1 -- 172.21.15.86:0/1090140799 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fbc5007ae80 con 0x7fbc640af5a0 2024-03-06T07:35:49.620 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:49.620 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:49.621 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:49.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc4406b4d0 msgr2=0x7fbc4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:49.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc4406b4d0 0x7fbc4406d990 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7fbc5400a750 tx=0x7fbc54008040 comp rx=0 tx=0).stop 2024-03-06T07:35:49.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 msgr2=0x7fbc640cb990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:49.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640cb990 secure :-1 s=READY pgs=1596 cs=0 l=1 rev1=1 crypto rx=0x7fbc50000c00 tx=0x7fbc500046c0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 shutdown_connections 2024-03-06T07:35:49.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbc4406b4d0 0x7fbc4406d990 unknown :-1 s=CLOSED pgs=681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbc6415be20 0x7fbc640ad160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbc640af5a0 0x7fbc640cb990 unknown :-1 s=CLOSED pgs=1596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 --2- 172.21.15.86:0/1090140799 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbc64072670 0x7fbc640c9500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:49.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.620+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 >> 172.21.15.86:0/1090140799 conn(0x7fbc640b4fe0 msgr2=0x7fbc641594a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:49.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.621+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 shutdown_connections 2024-03-06T07:35:49.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:49.621+0000 7fbc68a00640 1 -- 172.21.15.86:0/1090140799 wait complete. 2024-03-06T07:35:49.842 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:49.843 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:50.844 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:51.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.056+0000 7f8c54a00640 1 Processor -- start 2024-03-06T07:35:51.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.056+0000 7f8c54a00640 1 -- start start 2024-03-06T07:35:51.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.058+0000 7f8c54a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 0x7f8c500cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.058+0000 7f8c54a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8c5015c240 con 0x7f8c501561f0 2024-03-06T07:35:51.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.058+0000 7f8c54a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8c5015c500 con 0x7f8c500cc800 2024-03-06T07:35:51.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.058+0000 7f8c54a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c5015ce30 con 0x7f8c500aedc0 2024-03-06T07:35:51.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.059+0000 7f8c4f400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f8c501561f0 0x7f8c500ae6b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:38016/0 (socket says 172.21.15.86:38016) 2024-03-06T07:35:51.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.059+0000 7f8c4f400640 1 -- 172.21.15.86:0/189784757 learned_addr learned my addr 172.21.15.86:0/189784757 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:51.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.059+0000 7f8c4ea00640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 0x7f8c500cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.063 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.059+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2150375801 0 0) 0x7f8c5015c240 con 0x7f8c501561f0 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.059+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8c34003520 con 0x7f8c501561f0 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1269373198 0 0) 0x7f8c34003520 con 0x7f8c501561f0 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 msgr2=0x7f8c500cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 0x7f8c500cc2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 >> v1:172.21.15.194:6789/0 conn(0x7f8c500cc800 legacy=0x7f8c5015f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c5015c7e0 con 0x7f8c501561f0 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8c40001670 con 0x7f8c501561f0 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 >> v1:172.21.15.86:6789/0 conn(0x7f8c501561f0 legacy=0x7f8c500ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340039e0 0x7f8c34003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 0x7f8c34004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c34004d40 0x7f8c340051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c5015ce30 con 0x7f8c340039e0 2024-03-06T07:35:51.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c3400bb50 con 0x7f8c34004d40 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c3400bd20 con 0x7f8c500aedc0 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.060+0000 7f8c4f400640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340039e0 0x7f8c34003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4ea00640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 0x7f8c34004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4fe00640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c34004d40 0x7f8c340051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4f400640 1 -- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 msgr2=0x7f8c34004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4f400640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 0x7f8c34004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4f400640 1 -- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c34004d40 msgr2=0x7f8c340051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4f400640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c34004d40 0x7f8c340051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4f400640 1 -- 172.21.15.86:0/189784757 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c34003520 con 0x7f8c340039e0 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4ea00640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c500aedc0 0x7f8c34004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:51.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.061+0000 7f8c4fe00640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c34004d40 0x7f8c340051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:51.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.062+0000 7f8c4f400640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340039e0 0x7f8c34003e00 secure :-1 s=READY pgs=1598 cs=0 l=1 rev1=1 crypto rx=0x7f8c4000c210 tx=0x7f8c34005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9039b6bc393365d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:51.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.062+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c40007d30 con 0x7f8c340039e0 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.062+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c40030070 con 0x7f8c340039e0 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.062+0000 7f8c4e000640 1 -- 172.21.15.86:0/189784757 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c40005cf0 con 0x7f8c340039e0 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.063+0000 7f8c54a00640 1 -- 172.21.15.86:0/189784757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340039e0 msgr2=0x7f8c34003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.063+0000 7f8c54a00640 1 --2- 172.21.15.86:0/189784757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c340039e0 0x7f8c34003e00 secure :-1 s=READY pgs=1598 cs=0 l=1 rev1=1 crypto rx=0x7f8c4000c210 tx=0x7f8c34005830 comp rx=0 tx=0).stop 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.063+0000 7f8c4fe00640 1 -- 172.21.15.86:0/189784757 reap_dead start 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.063+0000 7f8c54a00640 1 -- 172.21.15.86:0/189784757 shutdown_connections 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.063+0000 7f8c54a00640 1 -- 172.21.15.86:0/189784757 >> 172.21.15.86:0/189784757 conn(0x7f8c5006afc0 msgr2=0x7f8c5006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.063+0000 7f8c54a00640 1 -- 172.21.15.86:0/189784757 shutdown_connections 2024-03-06T07:35:51.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.063+0000 7f8c54a00640 1 -- 172.21.15.86:0/189784757 wait complete. 2024-03-06T07:35:51.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.066+0000 7f8c54a00640 1 Processor -- start 2024-03-06T07:35:51.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.066+0000 7f8c54a00640 1 -- start start 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.067+0000 7f8c54a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340039e0 0x7f8c50154620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.067+0000 7f8c54a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c500ce620 0x7f8c500ceaa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.067+0000 7f8c54a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c500ad860 0x7f8c500adce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.067+0000 7f8c54a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c5006e360 con 0x7f8c500ad860 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.067+0000 7f8c54a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c500ccc90 con 0x7f8c500ce620 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.067+0000 7f8c54a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c501565d0 con 0x7f8c340039e0 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c500ad860 0x7f8c500adce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c500ad860 0x7f8c500adce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34576/0 (socket says 172.21.15.86:34576) 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 -- 172.21.15.86:0/1417861451 learned_addr learned my addr 172.21.15.86:0/1417861451 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4f400640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340039e0 0x7f8c50154620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4ea00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c500ce620 0x7f8c500ceaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 -- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340039e0 msgr2=0x7f8c50154620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:51.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340039e0 0x7f8c50154620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 -- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c500ce620 msgr2=0x7f8c500ceaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:51.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c500ce620 0x7f8c500ceaa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 -- 172.21.15.86:0/1417861451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c500cd840 con 0x7f8c500ad860 2024-03-06T07:35:51.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4f400640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340039e0 0x7f8c50154620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:51.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.068+0000 7f8c4fe00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c500ad860 0x7f8c500adce0 secure :-1 s=READY pgs=1599 cs=0 l=1 rev1=1 crypto rx=0x7f8c500cc1f0 tx=0x7f8c4800dc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:51.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.069+0000 7f8c47400640 1 -- 172.21.15.86:0/1417861451 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c48006d50 con 0x7f8c500ad860 2024-03-06T07:35:51.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.069+0000 7f8c47400640 1 -- 172.21.15.86:0/1417861451 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c48003970 con 0x7f8c500ad860 2024-03-06T07:35:51.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.069+0000 7f8c47400640 1 -- 172.21.15.86:0/1417861451 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c48003d60 con 0x7f8c500ad860 2024-03-06T07:35:51.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.069+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c500cdb20 con 0x7f8c500ad860 2024-03-06T07:35:51.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.069+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c500cd460 con 0x7f8c500ad860 2024-03-06T07:35:51.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.070+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c500ae220 con 0x7f8c500ad860 2024-03-06T07:35:51.073 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.071+0000 7f8c47400640 1 -- 172.21.15.86:0/1417861451 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8c4801b020 con 0x7f8c500ad860 2024-03-06T07:35:51.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.072+0000 7f8c47400640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c3006b4d0 0x7f8c3006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:51.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.072+0000 7f8c47400640 1 -- 172.21.15.86:0/1417861451 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f8c48089d50 con 0x7f8c500ad860 2024-03-06T07:35:51.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.073+0000 7f8c4f400640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c3006b4d0 0x7f8c3006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:51.075 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.073+0000 7f8c4f400640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c3006b4d0 0x7f8c3006d990 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f8c40034000 tx=0x7f8c40007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.077+0000 7f8c47400640 1 -- 172.21.15.86:0/1417861451 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8c4808d030 con 0x7f8c500ad860 2024-03-06T07:35:51.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.337+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8c500ce040 con 0x7f8c500ad860 2024-03-06T07:35:51.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.338+0000 7f8c47400640 1 -- 172.21.15.86:0/1417861451 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f8c500ce040 con 0x7f8c500ad860 2024-03-06T07:35:51.341 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:51.341 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:51.342 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c3006b4d0 msgr2=0x7f8c3006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c3006b4d0 0x7f8c3006d990 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f8c40034000 tx=0x7f8c40007000 comp rx=0 tx=0).stop 2024-03-06T07:35:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c500ad860 msgr2=0x7f8c500adce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c500ad860 0x7f8c500adce0 secure :-1 s=READY pgs=1599 cs=0 l=1 rev1=1 crypto rx=0x7f8c500cc1f0 tx=0x7f8c4800dc00 comp rx=0 tx=0).stop 2024-03-06T07:35:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 shutdown_connections 2024-03-06T07:35:51.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8c3006b4d0 0x7f8c3006d990 unknown :-1 s=CLOSED pgs=682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c500ad860 0x7f8c500adce0 unknown :-1 s=CLOSED pgs=1599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c500ce620 0x7f8c500ceaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 --2- 172.21.15.86:0/1417861451 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c340039e0 0x7f8c50154620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 >> 172.21.15.86:0/1417861451 conn(0x7f8c5006afc0 msgr2=0x7f8c500ae6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.341+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 shutdown_connections 2024-03-06T07:35:51.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:51.342+0000 7f8c54a00640 1 -- 172.21.15.86:0/1417861451 wait complete. 2024-03-06T07:35:51.563 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:51.564 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:51.564 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:52.565 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:52.778 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.776+0000 7ff93ca00640 1 Processor -- start 2024-03-06T07:35:52.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.776+0000 7ff93ca00640 1 -- start start 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff93ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff93815e160 0x7ff938162620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff93ca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff9380b3ac0 0x7ff93806b580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff93ca00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff938162b60 con 0x7ff9380af820 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff93ca00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff93806b0f0 con 0x7ff9380b3ac0 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff93ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9380b5620 con 0x7ff93815e160 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff936800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff9380b3ac0 0x7ff93806b580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff936800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff9380b3ac0 0x7ff93806b580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34578/0 (socket says 172.21.15.86:34578) 2024-03-06T07:35:52.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.778+0000 7ff936800640 1 -- 172.21.15.86:0/64216192 learned_addr learned my addr 172.21.15.86:0/64216192 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:52.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.779+0000 7ff935e00640 1 --2- 172.21.15.86:0/64216192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff93815e160 0x7ff938162620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:52.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.779+0000 7ff936800640 1 -- 172.21.15.86:0/64216192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff93815e160 msgr2=0x7ff938162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:52.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.779+0000 7ff936800640 1 --2- 172.21.15.86:0/64216192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff93815e160 0x7ff938162620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:52.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.779+0000 7ff934a00640 1 -- 172.21.15.86:0/64216192 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3339950605 0 0) 0x7ff920001490 con 0x7ff9380af820 2024-03-06T07:35:52.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.779+0000 7ff936800640 1 -- 172.21.15.86:0/64216192 >> v1:172.21.15.86:6789/0 conn(0x7ff9380af820 legacy=0x7ff9380b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:52.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.779+0000 7ff936800640 1 -- 172.21.15.86:0/64216192 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff93806f1f0 con 0x7ff9380b3ac0 2024-03-06T07:35:52.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.779+0000 7ff935e00640 1 --2- 172.21.15.86:0/64216192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff93815e160 0x7ff938162620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:52.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.780+0000 7ff936800640 1 --2- 172.21.15.86:0/64216192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9380b3ac0 0x7ff93806b580 secure :-1 s=READY pgs=1601 cs=0 l=1 rev1=1 crypto rx=0x7ff930009870 tx=0x7ff93002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cd87d7e86cc3c605 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.780+0000 7ff934a00640 1 -- 172.21.15.86:0/64216192 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff930035020 con 0x7ff9380b3ac0 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.780+0000 7ff934a00640 1 -- 172.21.15.86:0/64216192 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff930031640 con 0x7ff9380b3ac0 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.780+0000 7ff934a00640 1 -- 172.21.15.86:0/64216192 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff93002d040 con 0x7ff9380b3ac0 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 -- 172.21.15.86:0/64216192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9380b3ac0 msgr2=0x7ff93806b580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 --2- 172.21.15.86:0/64216192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9380b3ac0 0x7ff93806b580 secure :-1 s=READY pgs=1601 cs=0 l=1 rev1=1 crypto rx=0x7ff930009870 tx=0x7ff93002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 -- 172.21.15.86:0/64216192 shutdown_connections 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 --2- 172.21.15.86:0/64216192 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff9380b3ac0 0x7ff93806b580 unknown :-1 s=CLOSED pgs=1601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 --2- 172.21.15.86:0/64216192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff93815e160 0x7ff938162620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 -- 172.21.15.86:0/64216192 >> 172.21.15.86:0/64216192 conn(0x7ff9380d0340 msgr2=0x7ff9380d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 -- 172.21.15.86:0/64216192 shutdown_connections 2024-03-06T07:35:52.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.781+0000 7ff93ca00640 1 -- 172.21.15.86:0/64216192 wait complete. 2024-03-06T07:35:52.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.783+0000 7ff93ca00640 1 Processor -- start 2024-03-06T07:35:52.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.783+0000 7ff93ca00640 1 -- start start 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff93ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9380af820 0x7ff938151e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff93ca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9380b3ac0 0x7ff938151800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff93ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff93815e160 0x7ff93815e970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff93ca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff93806ef10 con 0x7ff93815e160 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff93ca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff93806ec30 con 0x7ff9380b3ac0 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff93ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff93806bac0 con 0x7ff9380af820 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff93815e160 0x7ff93815e970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff93815e160 0x7ff93815e970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34586/0 (socket says 172.21.15.86:34586) 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 -- 172.21.15.86:0/4155568711 learned_addr learned my addr 172.21.15.86:0/4155568711 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:52.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff935400640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9380b3ac0 0x7ff938151800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:52.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff935e00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9380af820 0x7ff938151e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:52.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 -- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9380af820 msgr2=0x7ff938151e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:52.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9380af820 0x7ff938151e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:52.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 -- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9380b3ac0 msgr2=0x7ff938151800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:52.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9380b3ac0 0x7ff938151800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:52.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.785+0000 7ff936800640 1 -- 172.21.15.86:0/4155568711 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9380cf9e0 con 0x7ff93815e160 2024-03-06T07:35:52.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.786+0000 7ff936800640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff93815e160 0x7ff93815e970 secure :-1 s=READY pgs=1602 cs=0 l=1 rev1=1 crypto rx=0x7ff930009870 tx=0x7ff9300046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:52.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.786+0000 7ff92e000640 1 -- 172.21.15.86:0/4155568711 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff930040070 con 0x7ff93815e160 2024-03-06T07:35:52.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.786+0000 7ff92e000640 1 -- 172.21.15.86:0/4155568711 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff930004230 con 0x7ff93815e160 2024-03-06T07:35:52.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.786+0000 7ff92e000640 1 -- 172.21.15.86:0/4155568711 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff930035040 con 0x7ff93815e160 2024-03-06T07:35:52.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.786+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff938154dd0 con 0x7ff93815e160 2024-03-06T07:35:52.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.786+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff938155390 con 0x7ff93815e160 2024-03-06T07:35:52.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.788+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff938074ae0 con 0x7ff93815e160 2024-03-06T07:35:52.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.788+0000 7ff92e000640 1 -- 172.21.15.86:0/4155568711 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff930038020 con 0x7ff93815e160 2024-03-06T07:35:52.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.789+0000 7ff92e000640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff91406b4d0 0x7ff91406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:52.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.789+0000 7ff92e000640 1 -- 172.21.15.86:0/4155568711 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff9300abd40 con 0x7ff93815e160 2024-03-06T07:35:52.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.790+0000 7ff935e00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff91406b4d0 0x7ff91406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:52.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.790+0000 7ff935e00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff91406b4d0 0x7ff91406d990 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7ff9380b34c0 tx=0x7ff928008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:52.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:52.798+0000 7ff92e000640 1 -- 172.21.15.86:0/4155568711 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff93007ab70 con 0x7ff93815e160 2024-03-06T07:35:53.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.135+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff9380d11e0 con 0x7ff93815e160 2024-03-06T07:35:53.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.136+0000 7ff92e000640 1 -- 172.21.15.86:0/4155568711 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7ff93007a470 con 0x7ff93815e160 2024-03-06T07:35:53.139 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:53.139 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:53.139 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:53.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.139+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff91406b4d0 msgr2=0x7ff91406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:53.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.139+0000 7ff93ca00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff91406b4d0 0x7ff91406d990 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7ff9380b34c0 tx=0x7ff928008040 comp rx=0 tx=0).stop 2024-03-06T07:35:53.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.139+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff93815e160 msgr2=0x7ff93815e970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.139+0000 7ff93ca00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff93815e160 0x7ff93815e970 secure :-1 s=READY pgs=1602 cs=0 l=1 rev1=1 crypto rx=0x7ff930009870 tx=0x7ff9300046c0 comp rx=0 tx=0).stop 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 shutdown_connections 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff91406b4d0 0x7ff91406d990 unknown :-1 s=CLOSED pgs=683 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff93815e160 0x7ff93815e970 unknown :-1 s=CLOSED pgs=1602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff9380b3ac0 0x7ff938151800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 --2- 172.21.15.86:0/4155568711 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff9380af820 0x7ff938151e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 >> 172.21.15.86:0/4155568711 conn(0x7ff9380d0340 msgr2=0x7ff93806be80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 shutdown_connections 2024-03-06T07:35:53.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:53.140+0000 7ff93ca00640 1 -- 172.21.15.86:0/4155568711 wait complete. 2024-03-06T07:35:53.363 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:53.363 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:53.566 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:54.365 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:54.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.571+0000 7f02e3a00640 1 Processor -- start 2024-03-06T07:35:54.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.571+0000 7f02e3a00640 1 -- start start 2024-03-06T07:35:54.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e3a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f02dc144c00 0x7f02dc06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:54.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e3a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:54.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e3a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f02dc06ee70 con 0x7f02dc15f9d0 2024-03-06T07:35:54.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e3a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f02dc06cef0 con 0x7f02dc144c00 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e3a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f02dc06d1d0 con 0x7f02dc15fe60 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e1000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f02dc144c00 0x7f02dc06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e1000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f02dc144c00 0x7f02dc06e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34598/0 (socket says 172.21.15.86:34598) 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.573+0000 7f02e1000640 1 -- 172.21.15.86:0/2641960830 learned_addr learned my addr 172.21.15.86:0/2641960830 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.574+0000 7f02e1a00640 1 --2- 172.21.15.86:0/2641960830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.574+0000 7f02e1000640 1 -- 172.21.15.86:0/2641960830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 msgr2=0x7f02dc0ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.574+0000 7f02e1000640 1 --2- 172.21.15.86:0/2641960830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0ce7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.574+0000 7f02e1000640 1 -- 172.21.15.86:0/2641960830 >> v1:172.21.15.194:6789/0 conn(0x7f02dc15f9d0 legacy=0x7f02dc0ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.574+0000 7f02e1000640 1 -- 172.21.15.86:0/2641960830 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02dc06d4b0 con 0x7f02dc144c00 2024-03-06T07:35:54.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.574+0000 7f02e1a00640 1 --2- 172.21.15.86:0/2641960830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0ce7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.575+0000 7f02e1000640 1 --2- 172.21.15.86:0/2641960830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc06e930 secure :-1 s=READY pgs=1603 cs=0 l=1 rev1=1 crypto rx=0x7f02cc009870 tx=0x7f02cc02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c428d78d9a38745 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.575+0000 7f02dbe00640 1 -- 172.21.15.86:0/2641960830 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02cc035020 con 0x7f02dc144c00 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.575+0000 7f02dbe00640 1 -- 172.21.15.86:0/2641960830 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f02cc031640 con 0x7f02dc144c00 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.575+0000 7f02dbe00640 1 -- 172.21.15.86:0/2641960830 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02cc02d040 con 0x7f02dc144c00 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 -- 172.21.15.86:0/2641960830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 msgr2=0x7f02dc06e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 --2- 172.21.15.86:0/2641960830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc06e930 secure :-1 s=READY pgs=1603 cs=0 l=1 rev1=1 crypto rx=0x7f02cc009870 tx=0x7f02cc02b8f0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 -- 172.21.15.86:0/2641960830 shutdown_connections 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 --2- 172.21.15.86:0/2641960830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0ce7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 --2- 172.21.15.86:0/2641960830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc06e930 unknown :-1 s=CLOSED pgs=1603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 -- 172.21.15.86:0/2641960830 >> 172.21.15.86:0/2641960830 conn(0x7f02dc0b1730 msgr2=0x7f02dc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 -- 172.21.15.86:0/2641960830 shutdown_connections 2024-03-06T07:35:54.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.576+0000 7f02e3a00640 1 -- 172.21.15.86:0/2641960830 wait complete. 2024-03-06T07:35:54.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.578+0000 7f02e3a00640 1 Processor -- start 2024-03-06T07:35:54.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.578+0000 7f02e3a00640 1 -- start start 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e3a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc139570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e3a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02dc15f9d0 0x7f02dc0aecf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e3a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0aff70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e3a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02dc160730 con 0x7f02dc144c00 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e3a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02dc160290 con 0x7f02dc15f9d0 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e3a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f02dc06cb50 con 0x7f02dc15fe60 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc139570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc139570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34612/0 (socket says 172.21.15.86:34612) 2024-03-06T07:35:54.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.579+0000 7f02e1000640 1 -- 172.21.15.86:0/1337354439 learned_addr learned my addr 172.21.15.86:0/1337354439 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02d3e00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02dc15f9d0 0x7f02dc0aecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1000640 1 -- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 msgr2=0x7f02dc0aff70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0aff70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1000640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0aff70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1000640 1 -- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02dc15f9d0 msgr2=0x7f02dc0aecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1000640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02dc15f9d0 0x7f02dc0aecf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1000640 1 -- 172.21.15.86:0/1337354439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02dc0b4340 con 0x7f02dc144c00 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02d3e00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02dc15f9d0 0x7f02dc0aecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0aff70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02e1000640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc139570 secure :-1 s=READY pgs=1604 cs=0 l=1 rev1=1 crypto rx=0x7f02cc000c00 tx=0x7f02cc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.580+0000 7f02d9600640 1 -- 172.21.15.86:0/1337354439 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02cc040070 con 0x7f02dc144c00 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.581+0000 7f02d9600640 1 -- 172.21.15.86:0/1337354439 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f02cc039c10 con 0x7f02dc144c00 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.581+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f02dc0b2fc0 con 0x7f02dc144c00 2024-03-06T07:35:54.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.581+0000 7f02d9600640 1 -- 172.21.15.86:0/1337354439 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02cc035040 con 0x7f02dc144c00 2024-03-06T07:35:54.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.581+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02dc0b3580 con 0x7f02dc144c00 2024-03-06T07:35:54.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.582+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f02dc04f330 con 0x7f02dc144c00 2024-03-06T07:35:54.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.583+0000 7f02d9600640 1 -- 172.21.15.86:0/1337354439 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f02cc038020 con 0x7f02dc144c00 2024-03-06T07:35:54.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.584+0000 7f02d9600640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02b806b4d0 0x7f02b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:54.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.584+0000 7f02d9600640 1 -- 172.21.15.86:0/1337354439 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f02cc0abda0 con 0x7f02dc144c00 2024-03-06T07:35:54.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.584+0000 7f02d3e00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02b806b4d0 0x7f02b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:54.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.585+0000 7f02d3e00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02b806b4d0 0x7f02b806d990 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f02dc0b0ca0 tx=0x7f02c4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:54.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.588+0000 7f02d9600640 1 -- 172.21.15.86:0/1337354439 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f02dc06ee70 con 0x7f02dc144c00 2024-03-06T07:35:54.850 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.848+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f02dc072890 con 0x7f02dc144c00 2024-03-06T07:35:54.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.849+0000 7f02d9600640 1 -- 172.21.15.86:0/1337354439 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f02cc07abd0 con 0x7f02dc144c00 2024-03-06T07:35:54.851 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:54.851 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:54.852 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02b806b4d0 msgr2=0x7f02b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02b806b4d0 0x7f02b806d990 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f02dc0b0ca0 tx=0x7f02c4009040 comp rx=0 tx=0).stop 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 msgr2=0x7f02dc139570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc139570 secure :-1 s=READY pgs=1604 cs=0 l=1 rev1=1 crypto rx=0x7f02cc000c00 tx=0x7f02cc0046c0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 shutdown_connections 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f02b806b4d0 0x7f02b806d990 unknown :-1 s=CLOSED pgs=684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f02dc15fe60 0x7f02dc0aff70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f02dc15f9d0 0x7f02dc0aecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 --2- 172.21.15.86:0/1337354439 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f02dc144c00 0x7f02dc139570 unknown :-1 s=CLOSED pgs=1604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 >> 172.21.15.86:0/1337354439 conn(0x7f02dc0b1730 msgr2=0x7f02dc145120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 shutdown_connections 2024-03-06T07:35:54.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:54.851+0000 7f02e3a00640 1 -- 172.21.15.86:0/1337354439 wait complete. 2024-03-06T07:35:54.935 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:35:54.941 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:35:55.078 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:55.078 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:55.567 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:56.079 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:56.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.285+0000 7f0f00a00640 1 Processor -- start 2024-03-06T07:35:56.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.285+0000 7f0f00a00640 1 -- start start 2024-03-06T07:35:56.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.286+0000 7f0f00a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0efc15acb0 0x7f0efc0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.286+0000 7f0f00a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0efc0b4680 0x7f0efc06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.286+0000 7f0f00a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0efc15b090 con 0x7f0efc072340 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.286+0000 7f0f00a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0efc0b2a30 con 0x7f0efc15acb0 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.286+0000 7f0f00a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0efc0b4360 con 0x7f0efc0b4680 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0ef9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0efc15acb0 0x7f0efc0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0efa800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0efc072340 0x7f0efc0ccfb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37668/0 (socket says 172.21.15.86:37668) 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0efa800640 1 -- 172.21.15.86:0/3494709959 learned_addr learned my addr 172.21.15.86:0/3494709959 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0ef3e00640 1 --2- 172.21.15.86:0/3494709959 >> v2:172.21.15.194:3300/0 conn(0x7f0efc0b4680 0x7f0efc06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:56.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0ef9e00640 1 -- 172.21.15.86:0/3494709959 >> v2:172.21.15.194:3300/0 conn(0x7f0efc0b4680 msgr2=0x7f0efc06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0ef9e00640 1 --2- 172.21.15.86:0/3494709959 >> v2:172.21.15.194:3300/0 conn(0x7f0efc0b4680 0x7f0efc06e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0ef9e00640 1 -- 172.21.15.86:0/3494709959 >> v1:172.21.15.86:6789/0 conn(0x7f0efc072340 legacy=0x7f0efc0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.287+0000 7f0ef9e00640 1 -- 172.21.15.86:0/3494709959 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0efc0b2cf0 con 0x7f0efc15acb0 2024-03-06T07:35:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.289+0000 7f0ef9e00640 1 --2- 172.21.15.86:0/3494709959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc15acb0 0x7f0efc0b37d0 secure :-1 s=READY pgs=1606 cs=0 l=1 rev1=1 crypto rx=0x7f0efc068560 tx=0x7f0eec02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e8eb2e472cad7693 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.289+0000 7f0ef9400640 1 -- 172.21.15.86:0/3494709959 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0eec035020 con 0x7f0efc15acb0 2024-03-06T07:35:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.289+0000 7f0ef9400640 1 -- 172.21.15.86:0/3494709959 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0eec031830 con 0x7f0efc15acb0 2024-03-06T07:35:56.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.289+0000 7f0ef9400640 1 -- 172.21.15.86:0/3494709959 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0eec031c00 con 0x7f0efc15acb0 2024-03-06T07:35:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.289+0000 7f0f00a00640 1 -- 172.21.15.86:0/3494709959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc15acb0 msgr2=0x7f0efc0b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:56.292 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.289+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3494709959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc15acb0 0x7f0efc0b37d0 secure :-1 s=READY pgs=1606 cs=0 l=1 rev1=1 crypto rx=0x7f0efc068560 tx=0x7f0eec02b8d0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.290+0000 7f0f00a00640 1 -- 172.21.15.86:0/3494709959 shutdown_connections 2024-03-06T07:35:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.290+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3494709959 >> v2:172.21.15.194:3300/0 conn(0x7f0efc0b4680 0x7f0efc06e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.290+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3494709959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc15acb0 0x7f0efc0b37d0 unknown :-1 s=CLOSED pgs=1606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.290+0000 7f0f00a00640 1 -- 172.21.15.86:0/3494709959 >> 172.21.15.86:0/3494709959 conn(0x7f0efc0b1730 msgr2=0x7f0efc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.290+0000 7f0f00a00640 1 -- 172.21.15.86:0/3494709959 shutdown_connections 2024-03-06T07:35:56.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.290+0000 7f0f00a00640 1 -- 172.21.15.86:0/3494709959 wait complete. 2024-03-06T07:35:56.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.291+0000 7f0f00a00640 1 Processor -- start 2024-03-06T07:35:56.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.291+0000 7f0f00a00640 1 -- start start 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0f00a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc072340 0x7f0efc1552a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0f00a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0efc0b4680 0x7f0efc0cab50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0f00a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0efc15acb0 0x7f0efc0cb090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0f00a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0efc0b3d10 con 0x7f0efc072340 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0f00a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0efc0b32b0 con 0x7f0efc0b4680 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0f00a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0efc0b4010 con 0x7f0efc15acb0 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0ef9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc072340 0x7f0efc1552a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:56.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0ef9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc072340 0x7f0efc1552a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34886/0 (socket says 172.21.15.86:34886) 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.293+0000 7f0ef9e00640 1 -- 172.21.15.86:0/3228835692 learned_addr learned my addr 172.21.15.86:0/3228835692 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef9e00640 1 -- 172.21.15.86:0/3228835692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0efc15acb0 msgr2=0x7f0efc0cb090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef9e00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0efc15acb0 0x7f0efc0cb090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef9e00640 1 -- 172.21.15.86:0/3228835692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0efc0b4680 msgr2=0x7f0efc0cab50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef9e00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0efc0b4680 0x7f0efc0cab50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef9e00640 1 -- 172.21.15.86:0/3228835692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0efc0cb5d0 con 0x7f0efc072340 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef9e00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc072340 0x7f0efc1552a0 secure :-1 s=READY pgs=1607 cs=0 l=1 rev1=1 crypto rx=0x7f0eec009890 tx=0x7f0eec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef2000640 1 -- 172.21.15.86:0/3228835692 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0eec041070 con 0x7f0efc072340 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef2000640 1 -- 172.21.15.86:0/3228835692 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0eec03aa30 con 0x7f0efc072340 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0ef2000640 1 -- 172.21.15.86:0/3228835692 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0eec035040 con 0x7f0efc072340 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.294+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0efc0cb880 con 0x7f0efc072340 2024-03-06T07:35:56.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.295+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0efc0cbe40 con 0x7f0efc072340 2024-03-06T07:35:56.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.296+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0efc04f330 con 0x7f0efc072340 2024-03-06T07:35:56.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.297+0000 7f0ef2000640 1 -- 172.21.15.86:0/3228835692 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0eec038020 con 0x7f0efc072340 2024-03-06T07:35:56.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.298+0000 7f0ef2000640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ed806b4d0 0x7f0ed806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:56.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.298+0000 7f0ef2000640 1 -- 172.21.15.86:0/3228835692 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0eec0abd10 con 0x7f0efc072340 2024-03-06T07:35:56.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.298+0000 7f0ef3e00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ed806b4d0 0x7f0ed806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:56.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.299+0000 7f0ef3e00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ed806b4d0 0x7f0ed806d990 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f0ee4006fd0 tx=0x7f0ee4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:56.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.302+0000 7f0ef2000640 1 -- 172.21.15.86:0/3228835692 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0eec0b1020 con 0x7f0efc072340 2024-03-06T07:35:56.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.537+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0efc0ceb60 con 0x7f0efc072340 2024-03-06T07:35:56.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.538+0000 7f0ef2000640 1 -- 172.21.15.86:0/3228835692 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f0eec07aab0 con 0x7f0efc072340 2024-03-06T07:35:56.541 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:56.541 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:56.541 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.540+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ed806b4d0 msgr2=0x7f0ed806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.540+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ed806b4d0 0x7f0ed806d990 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f0ee4006fd0 tx=0x7f0ee4008040 comp rx=0 tx=0).stop 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.540+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc072340 msgr2=0x7f0efc1552a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.540+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc072340 0x7f0efc1552a0 secure :-1 s=READY pgs=1607 cs=0 l=1 rev1=1 crypto rx=0x7f0eec009890 tx=0x7f0eec0046c0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 shutdown_connections 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0ed806b4d0 0x7f0ed806d990 unknown :-1 s=CLOSED pgs=685 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0efc15acb0 0x7f0efc0cb090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0efc0b4680 0x7f0efc0cab50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 --2- 172.21.15.86:0/3228835692 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0efc072340 0x7f0efc1552a0 unknown :-1 s=CLOSED pgs=1607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 >> 172.21.15.86:0/3228835692 conn(0x7f0efc0b1730 msgr2=0x7f0efc15e4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 shutdown_connections 2024-03-06T07:35:56.544 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:56.541+0000 7f0f00a00640 1 -- 172.21.15.86:0/3228835692 wait complete. 2024-03-06T07:35:56.767 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:56.767 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:57.569 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:57.768 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:57.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.992+0000 7f6458c00640 1 Processor -- start 2024-03-06T07:35:57.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.992+0000 7f6458c00640 1 -- start start 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6458c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6454066f10 0x7f645406ed70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6458c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f64540ce8f0 0x7f64540ced80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6458c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f645406e5b0 con 0x7f645415f930 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6458c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f64540ccd90 con 0x7f64540ce8f0 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6458c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f64540afc00 con 0x7f6454066f10 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6452800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f64540ce8f0 0x7f64540ced80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6452800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f64540ce8f0 0x7f64540ced80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34894/0 (socket says 172.21.15.86:34894) 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6452800640 1 -- 172.21.15.86:0/670216241 learned_addr learned my addr 172.21.15.86:0/670216241 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:57.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6449400640 1 --2- 172.21.15.86:0/670216241 >> v2:172.21.15.194:3300/0 conn(0x7f6454066f10 0x7f645406ed70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:57.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6452800640 1 -- 172.21.15.86:0/670216241 >> v2:172.21.15.194:3300/0 conn(0x7f6454066f10 msgr2=0x7f645406ed70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:57.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6452800640 1 --2- 172.21.15.86:0/670216241 >> v2:172.21.15.194:3300/0 conn(0x7f6454066f10 0x7f645406ed70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:57.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6452800640 1 -- 172.21.15.86:0/670216241 >> v1:172.21.15.194:6789/0 conn(0x7f645415f930 legacy=0x7f645415fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:35:57.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.994+0000 7f6452800640 1 -- 172.21.15.86:0/670216241 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64540ccfe0 con 0x7f64540ce8f0 2024-03-06T07:35:57.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.996+0000 7f6452800640 1 --2- 172.21.15.86:0/670216241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64540ce8f0 0x7f64540ced80 secure :-1 s=READY pgs=1608 cs=0 l=1 rev1=1 crypto rx=0x7f64540679d0 tx=0x7f644c02be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=33f9c801578d4d8c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:57.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.996+0000 7f6451400640 1 -- 172.21.15.86:0/670216241 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f644c02c9f0 con 0x7f64540ce8f0 2024-03-06T07:35:57.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.996+0000 7f6451400640 1 -- 172.21.15.86:0/670216241 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f644c02cb90 con 0x7f64540ce8f0 2024-03-06T07:35:57.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.996+0000 7f6451400640 1 -- 172.21.15.86:0/670216241 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f644c0343f0 con 0x7f64540ce8f0 2024-03-06T07:35:57.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.996+0000 7f6458c00640 1 -- 172.21.15.86:0/670216241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64540ce8f0 msgr2=0x7f64540ced80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:57.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.996+0000 7f6458c00640 1 --2- 172.21.15.86:0/670216241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64540ce8f0 0x7f64540ced80 secure :-1 s=READY pgs=1608 cs=0 l=1 rev1=1 crypto rx=0x7f64540679d0 tx=0x7f644c02be50 comp rx=0 tx=0).stop 2024-03-06T07:35:58.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.996+0000 7f6458c00640 1 -- 172.21.15.86:0/670216241 shutdown_connections 2024-03-06T07:35:58.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.997+0000 7f6458c00640 1 --2- 172.21.15.86:0/670216241 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f64540ce8f0 0x7f64540ced80 unknown :-1 s=CLOSED pgs=1608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.997+0000 7f6458c00640 1 --2- 172.21.15.86:0/670216241 >> v2:172.21.15.194:3300/0 conn(0x7f6454066f10 0x7f645406ed70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.997+0000 7f6458c00640 1 -- 172.21.15.86:0/670216241 >> 172.21.15.86:0/670216241 conn(0x7f64540d0340 msgr2=0x7f64540d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:58.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.997+0000 7f6458c00640 1 -- 172.21.15.86:0/670216241 shutdown_connections 2024-03-06T07:35:58.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.997+0000 7f6458c00640 1 -- 172.21.15.86:0/670216241 wait complete. 2024-03-06T07:35:58.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:57.999+0000 7f6458c00640 1 Processor -- start 2024-03-06T07:35:58.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.000+0000 7f6458c00640 1 -- start start 2024-03-06T07:35:58.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6458c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6454066f10 0x7f6454146050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:58.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6458c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f64540ce8f0 0x7f64541508f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6458c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f645415f930 0x7f64541552e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6458c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f645406e090 con 0x7f645415f930 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6458c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f645406ddb0 con 0x7f6454066f10 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6458c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f645406c780 con 0x7f64540ce8f0 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6452800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f645415f930 0x7f64541552e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6452800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f645415f930 0x7f64541552e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34902/0 (socket says 172.21.15.86:34902) 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.001+0000 7f6452800640 1 -- 172.21.15.86:0/4076524943 learned_addr learned my addr 172.21.15.86:0/4076524943 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6449400640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f64540ce8f0 0x7f64541508f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6451e00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6454066f10 0x7f6454146050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6452800640 1 -- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f64540ce8f0 msgr2=0x7f64541508f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6452800640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f64540ce8f0 0x7f64541508f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6452800640 1 -- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6454066f10 msgr2=0x7f6454146050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:58.006 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6452800640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6454066f10 0x7f6454146050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6452800640 1 -- 172.21.15.86:0/4076524943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6454151160 con 0x7f645415f930 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6449400640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f64540ce8f0 0x7f64541508f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6451e00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6454066f10 0x7f6454146050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.002+0000 7f6452800640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f645415f930 0x7f64541552e0 secure :-1 s=READY pgs=1609 cs=0 l=1 rev1=1 crypto rx=0x7f644c036040 tx=0x7f644c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.003+0000 7f644aa00640 1 -- 172.21.15.86:0/4076524943 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f644c043020 con 0x7f645415f930 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.003+0000 7f644aa00640 1 -- 172.21.15.86:0/4076524943 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f644c03e450 con 0x7f645415f930 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.003+0000 7f644aa00640 1 -- 172.21.15.86:0/4076524943 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f644c009510 con 0x7f645415f930 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.003+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6454151440 con 0x7f645415f930 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.003+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6454151a00 con 0x7f645415f930 2024-03-06T07:35:58.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.004+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6454074ae0 con 0x7f645415f930 2024-03-06T07:35:58.008 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.006+0000 7f644aa00640 1 -- 172.21.15.86:0/4076524943 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f644c046020 con 0x7f645415f930 2024-03-06T07:35:58.009 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.007+0000 7f644aa00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f643006b500 0x7f643006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:58.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.007+0000 7f644aa00640 1 -- 172.21.15.86:0/4076524943 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f644c0ac780 con 0x7f645415f930 2024-03-06T07:35:58.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.007+0000 7f6451e00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f643006b500 0x7f643006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:58.010 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.008+0000 7f6451e00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f643006b500 0x7f643006d9c0 secure :-1 s=READY pgs=686 cs=0 l=1 rev1=1 crypto rx=0x7f644400a830 tx=0x7f6444008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:58.014 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.012+0000 7f644aa00640 1 -- 172.21.15.86:0/4076524943 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f644c04d070 con 0x7f645415f930 2024-03-06T07:35:58.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.248+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f645404f330 con 0x7f645415f930 2024-03-06T07:35:58.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.249+0000 7f644aa00640 1 -- 172.21.15.86:0/4076524943 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f644c07b650 con 0x7f645415f930 2024-03-06T07:35:58.252 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:58.252 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:58.252 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:58.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f643006b500 msgr2=0x7f643006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:58.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f643006b500 0x7f643006d9c0 secure :-1 s=READY pgs=686 cs=0 l=1 rev1=1 crypto rx=0x7f644400a830 tx=0x7f6444008040 comp rx=0 tx=0).stop 2024-03-06T07:35:58.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f645415f930 msgr2=0x7f64541552e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:58.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f645415f930 0x7f64541552e0 secure :-1 s=READY pgs=1609 cs=0 l=1 rev1=1 crypto rx=0x7f644c036040 tx=0x7f644c004a40 comp rx=0 tx=0).stop 2024-03-06T07:35:58.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 shutdown_connections 2024-03-06T07:35:58.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f643006b500 0x7f643006d9c0 unknown :-1 s=CLOSED pgs=686 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f645415f930 0x7f64541552e0 unknown :-1 s=CLOSED pgs=1609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f64540ce8f0 0x7f64541508f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 --2- 172.21.15.86:0/4076524943 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6454066f10 0x7f6454146050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:58.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 >> 172.21.15.86:0/4076524943 conn(0x7f64540d0340 msgr2=0x7f6454154a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:58.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.251+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 shutdown_connections 2024-03-06T07:35:58.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:58.252+0000 7f6458c00640 1 -- 172.21.15.86:0/4076524943 wait complete. 2024-03-06T07:35:58.473 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:35:58.473 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:35:59.474 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:35:59.571 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:35:59.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.689+0000 7f21fc000640 1 Processor -- start 2024-03-06T07:35:59.692 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.689+0000 7f21fc000640 1 -- start start 2024-03-06T07:35:59.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.692+0000 7f21fc000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f4151660 0x7f21f4071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:59.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.692+0000 7f21fc000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21f4071a60 con 0x7f21f415dff0 2024-03-06T07:35:59.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.692+0000 7f21fc000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21f40b35b0 con 0x7f21f40cf430 2024-03-06T07:35:59.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.692+0000 7f21fc000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21f40b3ee0 con 0x7f21f4151660 2024-03-06T07:35:59.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.692+0000 7f21fa000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f21f415dff0 0x7f21f40b4370 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37678/0 (socket says 172.21.15.86:37678) 2024-03-06T07:35:59.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.692+0000 7f21fa000640 1 -- 172.21.15.86:0/456222636 learned_addr learned my addr 172.21.15.86:0/456222636 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:35:59.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.692+0000 7f21f9600640 1 --2- 172.21.15.86:0/456222636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f4151660 0x7f21f4071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:59.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.693+0000 7f21f3e00640 1 -- 172.21.15.86:0/456222636 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3780630519 0 0) 0x7f21f4071a60 con 0x7f21f415dff0 2024-03-06T07:35:59.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.693+0000 7f21f3e00640 1 -- 172.21.15.86:0/456222636 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f21dc003520 con 0x7f21f415dff0 2024-03-06T07:35:59.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.693+0000 7f21f3e00640 1 -- 172.21.15.86:0/456222636 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1116526330 0 0) 0x7f21dc003520 con 0x7f21f415dff0 2024-03-06T07:35:59.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.693+0000 7f21f9600640 1 -- 172.21.15.86:0/456222636 >> v1:172.21.15.194:6789/0 conn(0x7f21f40cf430 legacy=0x7f21f406d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:59.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.693+0000 7f21f9600640 1 -- 172.21.15.86:0/456222636 >> v1:172.21.15.86:6789/0 conn(0x7f21f415dff0 legacy=0x7f21f40b4370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:59.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.693+0000 7f21f9600640 1 -- 172.21.15.86:0/456222636 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21f40b3890 con 0x7f21f4151660 2024-03-06T07:35:59.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.695+0000 7f21f9600640 1 --2- 172.21.15.86:0/456222636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f4151660 0x7f21f4071520 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f21e80097d0 tx=0x7f21e802b6e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c9cbb9bf14f3cdb1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:59.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.695+0000 7f21f3e00640 1 -- 172.21.15.86:0/456222636 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21e8037070 con 0x7f21f4151660 2024-03-06T07:35:59.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.695+0000 7f21f3e00640 1 -- 172.21.15.86:0/456222636 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f21e8030510 con 0x7f21f4151660 2024-03-06T07:35:59.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.695+0000 7f21f3e00640 1 -- 172.21.15.86:0/456222636 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21ec0019f0 con 0x7f21f4151660 2024-03-06T07:35:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.696+0000 7f21fc000640 1 -- 172.21.15.86:0/456222636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f4151660 msgr2=0x7f21f4071520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.696+0000 7f21fc000640 1 --2- 172.21.15.86:0/456222636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f4151660 0x7f21f4071520 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f21e80097d0 tx=0x7f21e802b6e0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.696+0000 7f21fc000640 1 -- 172.21.15.86:0/456222636 shutdown_connections 2024-03-06T07:35:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.696+0000 7f21fc000640 1 --2- 172.21.15.86:0/456222636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f4151660 0x7f21f4071520 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.696+0000 7f21fc000640 1 -- 172.21.15.86:0/456222636 >> 172.21.15.86:0/456222636 conn(0x7f21f40b1730 msgr2=0x7f21f40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.696+0000 7f21fc000640 1 -- 172.21.15.86:0/456222636 shutdown_connections 2024-03-06T07:35:59.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.696+0000 7f21fc000640 1 -- 172.21.15.86:0/456222636 wait complete. 2024-03-06T07:35:59.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.697+0000 7f21fc000640 1 Processor -- start 2024-03-06T07:35:59.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.698+0000 7f21fc000640 1 -- start start 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.698+0000 7f21fc000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f40cf430 0x7f21f41406e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fc000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21f4151660 0x7f21f41e8ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fc000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21f415dff0 0x7f21f41e9010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fc000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21f406daf0 con 0x7f21f415dff0 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fc000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21f4071d30 con 0x7f21f4151660 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fc000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21f40b4a80 con 0x7f21f40cf430 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21f415dff0 0x7f21f41e9010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21f415dff0 0x7f21f41e9010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34912/0 (socket says 172.21.15.86:34912) 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 -- 172.21.15.86:0/716863191 learned_addr learned my addr 172.21.15.86:0/716863191 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:35:59.703 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21f9600640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f40cf430 0x7f21f41406e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 -- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f40cf430 msgr2=0x7f21f41406e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f40cf430 0x7f21f41406e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 -- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21f4151660 msgr2=0x7f21f41e8ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21f8c00640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21f4151660 0x7f21f41e8ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21f4151660 0x7f21f41e8ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.699+0000 7f21fa000640 1 -- 172.21.15.86:0/716863191 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21f41e9840 con 0x7f21f415dff0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.700+0000 7f21fa000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21f415dff0 0x7f21f41e9010 secure :-1 s=READY pgs=1611 cs=0 l=1 rev1=1 crypto rx=0x7f21ec00cc90 tx=0x7f21ec00a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.700+0000 7f21f1600640 1 -- 172.21.15.86:0/716863191 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21ec00ce40 con 0x7f21f415dff0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.700+0000 7f21f1600640 1 -- 172.21.15.86:0/716863191 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f21ec012400 con 0x7f21f415dff0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.700+0000 7f21f1600640 1 -- 172.21.15.86:0/716863191 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21ec0127f0 con 0x7f21f415dff0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.700+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21f41e9b20 con 0x7f21f415dff0 2024-03-06T07:35:59.704 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.700+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21f41e4df0 con 0x7f21f415dff0 2024-03-06T07:35:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.701+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f21f404f330 con 0x7f21f415dff0 2024-03-06T07:35:59.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.702+0000 7f21f1600640 1 -- 172.21.15.86:0/716863191 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f21ec012990 con 0x7f21f415dff0 2024-03-06T07:35:59.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.703+0000 7f21f1600640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21d406b4d0 0x7f21d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:35:59.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.703+0000 7f21f1600640 1 -- 172.21.15.86:0/716863191 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f21ec08aa70 con 0x7f21f415dff0 2024-03-06T07:35:59.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.703+0000 7f21f9600640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21d406b4d0 0x7f21d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:35:59.706 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.704+0000 7f21f9600640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21d406b4d0 0x7f21d406d990 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f21e8034000 tx=0x7f21e8008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:35:59.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.708+0000 7f21f1600640 1 -- 172.21.15.86:0/716863191 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f21ec0598a0 con 0x7f21f415dff0 2024-03-06T07:35:59.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.939+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f21f41e4fc0 con 0x7f21f415dff0 2024-03-06T07:35:59.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.940+0000 7f21f1600640 1 -- 172.21.15.86:0/716863191 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f21ec0591a0 con 0x7f21f415dff0 2024-03-06T07:35:59.943 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:35:59.943 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:35:59.943 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21d406b4d0 msgr2=0x7f21d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21d406b4d0 0x7f21d406d990 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f21e8034000 tx=0x7f21e8008030 comp rx=0 tx=0).stop 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21f415dff0 msgr2=0x7f21f41e9010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21f415dff0 0x7f21f41e9010 secure :-1 s=READY pgs=1611 cs=0 l=1 rev1=1 crypto rx=0x7f21ec00cc90 tx=0x7f21ec00a430 comp rx=0 tx=0).stop 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 shutdown_connections 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21d406b4d0 0x7f21d406d990 unknown :-1 s=CLOSED pgs=687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21f415dff0 0x7f21f41e9010 unknown :-1 s=CLOSED pgs=1611 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21f4151660 0x7f21f41e8ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 --2- 172.21.15.86:0/716863191 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21f40cf430 0x7f21f41406e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 >> 172.21.15.86:0/716863191 conn(0x7f21f40b1730 msgr2=0x7f21f40cf9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:35:59.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 shutdown_connections 2024-03-06T07:35:59.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:35:59.942+0000 7f21fc000640 1 -- 172.21.15.86:0/716863191 wait complete. 2024-03-06T07:36:00.165 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:00.165 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:01.166 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:01.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.375+0000 7f0fcba00640 1 Processor -- start 2024-03-06T07:36:01.378 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.376+0000 7f0fcba00640 1 -- start start 2024-03-06T07:36:01.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.377+0000 7f0fcba00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f0fc415f770 0x7f0fc415fd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.377+0000 7f0fcba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc40b4390 0x7f0fc4160290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.377+0000 7f0fcba00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0fc40b3040 con 0x7f0fc41607d0 2024-03-06T07:36:01.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.377+0000 7f0fcba00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0fc415fb50 con 0x7f0fc415f770 2024-03-06T07:36:01.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.377+0000 7f0fcba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0fc40cd3e0 con 0x7f0fc40b4390 2024-03-06T07:36:01.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.377+0000 7f0fc9a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f0fc41607d0 0x7f0fc40ce560 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37686/0 (socket says 172.21.15.86:37686) 2024-03-06T07:36:01.380 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.377+0000 7f0fc9a00640 1 -- 172.21.15.86:0/712267196 learned_addr learned my addr 172.21.15.86:0/712267196 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:01.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3012802893 0 0) 0x7f0fc40b3040 con 0x7f0fc41607d0 2024-03-06T07:36:01.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0fa8003520 con 0x7f0fc41607d0 2024-03-06T07:36:01.382 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fbbe00640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc40b4390 0x7f0fc4160290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc9000640 1 --2- 172.21.15.86:0/712267196 >> v2:172.21.15.194:3300/0 conn(0x7f0fc415f770 0x7f0fc415fd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1393460228 0 0) 0x7f0fa8003520 con 0x7f0fc41607d0 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc40b4390 msgr2=0x7f0fc4160290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc40b4390 0x7f0fc4160290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 >> v2:172.21.15.194:3300/0 conn(0x7f0fc415f770 msgr2=0x7f0fc415fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 --2- 172.21.15.86:0/712267196 >> v2:172.21.15.194:3300/0 conn(0x7f0fc415f770 0x7f0fc415fd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0fc40cd6c0 con 0x7f0fc41607d0 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f0fbc0019f0 con 0x7f0fc41607d0 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 >> v1:172.21.15.86:6789/0 conn(0x7f0fc41607d0 legacy=0x7f0fc40ce560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fa80039b0 0x7f0fa8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa8004800 0x7f0fa8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc40b4390 0x7f0fa80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fc40cd3e0 con 0x7f0fa80039b0 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fa800baa0 con 0x7f0fa8004800 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.378+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0fa800bca0 con 0x7f0fc40b4390 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.379+0000 7f0fc9000640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fa80039b0 0x7f0fa8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.379+0000 7f0fbbe00640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa8004800 0x7f0fa8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.379+0000 7f0fc9000640 1 -- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc40b4390 msgr2=0x7f0fa80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.379+0000 7f0fc9000640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc40b4390 0x7f0fa80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.379+0000 7f0fc9000640 1 -- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa8004800 msgr2=0x7f0fa8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.379+0000 7f0fc9000640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa8004800 0x7f0fa8004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.379+0000 7f0fc9000640 1 -- 172.21.15.86:0/712267196 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0fa8003520 con 0x7f0fa80039b0 2024-03-06T07:36:01.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.380+0000 7f0fc9000640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fa80039b0 0x7f0fa8003dd0 secure :-1 s=READY pgs=1613 cs=0 l=1 rev1=1 crypto rx=0x7f0fb400ba00 tx=0x7f0fa8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b8d9da40e9c33f9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:01.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fb4036020 con 0x7f0fa80039b0 2024-03-06T07:36:01.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0fb4032620 con 0x7f0fa80039b0 2024-03-06T07:36:01.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fc3e00640 1 -- 172.21.15.86:0/712267196 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fb400b600 con 0x7f0fa80039b0 2024-03-06T07:36:01.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fcba00640 1 -- 172.21.15.86:0/712267196 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fa80039b0 msgr2=0x7f0fa8003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:01.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fcba00640 1 --2- 172.21.15.86:0/712267196 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fa80039b0 0x7f0fa8003dd0 secure :-1 s=READY pgs=1613 cs=0 l=1 rev1=1 crypto rx=0x7f0fb400ba00 tx=0x7f0fa8005830 comp rx=0 tx=0).stop 2024-03-06T07:36:01.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fc9a00640 1 -- 172.21.15.86:0/712267196 reap_dead start 2024-03-06T07:36:01.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fcba00640 1 -- 172.21.15.86:0/712267196 shutdown_connections 2024-03-06T07:36:01.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fcba00640 1 -- 172.21.15.86:0/712267196 >> 172.21.15.86:0/712267196 conn(0x7f0fc40b3870 msgr2=0x7f0fc40b3c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:01.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fcba00640 1 -- 172.21.15.86:0/712267196 shutdown_connections 2024-03-06T07:36:01.386 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.381+0000 7f0fcba00640 1 -- 172.21.15.86:0/712267196 wait complete. 2024-03-06T07:36:01.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.383+0000 7f0fcba00640 1 Processor -- start 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.383+0000 7f0fcba00640 1 -- start start 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.384+0000 7f0fcba00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa80039b0 0x7f0fc415e930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.384+0000 7f0fcba00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fc4156a70 0x7f0fc406c0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.384+0000 7f0fcba00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc406c610 0x7f0fc406ca90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.384+0000 7f0fcba00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fc40b34a0 con 0x7f0fc4156a70 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.384+0000 7f0fcba00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fc40b3320 con 0x7f0fa80039b0 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.384+0000 7f0fcba00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0fc40d09b0 con 0x7f0fc406c610 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fc4156a70 0x7f0fc406c0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fc4156a70 0x7f0fc406c0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34938/0 (socket says 172.21.15.86:34938) 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 -- 172.21.15.86:0/1168356468 learned_addr learned my addr 172.21.15.86:0/1168356468 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fc9000640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa80039b0 0x7f0fc415e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fc9a00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc406c610 0x7f0fc406ca90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 -- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc406c610 msgr2=0x7f0fc406ca90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:01.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc406c610 0x7f0fc406ca90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 -- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa80039b0 msgr2=0x7f0fc415e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:01.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa80039b0 0x7f0fc415e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.385+0000 7f0fbbe00640 1 -- 172.21.15.86:0/1168356468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0fc40cb490 con 0x7f0fc4156a70 2024-03-06T07:36:01.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.386+0000 7f0fbbe00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fc4156a70 0x7f0fc406c0d0 secure :-1 s=READY pgs=1614 cs=0 l=1 rev1=1 crypto rx=0x7f0fac00e600 tx=0x7f0fac00eac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:01.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.386+0000 7f0fc1600640 1 -- 172.21.15.86:0/1168356468 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fac017070 con 0x7f0fc4156a70 2024-03-06T07:36:01.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.386+0000 7f0fc1600640 1 -- 172.21.15.86:0/1168356468 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0fac013610 con 0x7f0fc4156a70 2024-03-06T07:36:01.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.386+0000 7f0fc1600640 1 -- 172.21.15.86:0/1168356468 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fac00f040 con 0x7f0fc4156a70 2024-03-06T07:36:01.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.386+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0fc40cb730 con 0x7f0fc4156a70 2024-03-06T07:36:01.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.386+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0fc40cbcb0 con 0x7f0fc4156a70 2024-03-06T07:36:01.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.387+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0fc404f330 con 0x7f0fc4156a70 2024-03-06T07:36:01.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.389+0000 7f0fc1600640 1 -- 172.21.15.86:0/1168356468 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0fac01a070 con 0x7f0fc4156a70 2024-03-06T07:36:01.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.390+0000 7f0fc1600640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fa006b500 0x7f0fa006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:01.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.390+0000 7f0fc1600640 1 -- 172.21.15.86:0/1168356468 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0fac089e50 con 0x7f0fc4156a70 2024-03-06T07:36:01.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.390+0000 7f0fc9000640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fa006b500 0x7f0fa006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:01.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.391+0000 7f0fc9000640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fa006b500 0x7f0fa006d9c0 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f0fa80057e0 tx=0x7f0fb403d040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:01.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.394+0000 7f0fc1600640 1 -- 172.21.15.86:0/1168356468 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0fac058bf0 con 0x7f0fc4156a70 2024-03-06T07:36:01.573 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:01.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.628+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0fc40ce130 con 0x7f0fc4156a70 2024-03-06T07:36:01.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.629+0000 7f0fc1600640 1 -- 172.21.15.86:0/1168356468 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f0fac0584f0 con 0x7f0fc4156a70 2024-03-06T07:36:01.632 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:01.632 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:01.633 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:01.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fa006b500 msgr2=0x7f0fa006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:01.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fa006b500 0x7f0fa006d9c0 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f0fa80057e0 tx=0x7f0fb403d040 comp rx=0 tx=0).stop 2024-03-06T07:36:01.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fc4156a70 msgr2=0x7f0fc406c0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:01.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fc4156a70 0x7f0fc406c0d0 secure :-1 s=READY pgs=1614 cs=0 l=1 rev1=1 crypto rx=0x7f0fac00e600 tx=0x7f0fac00eac0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 shutdown_connections 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0fa006b500 0x7f0fa006d9c0 unknown :-1 s=CLOSED pgs=688 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0fc406c610 0x7f0fc406ca90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0fc4156a70 0x7f0fc406c0d0 unknown :-1 s=CLOSED pgs=1614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 --2- 172.21.15.86:0/1168356468 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0fa80039b0 0x7f0fc415e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.632+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 >> 172.21.15.86:0/1168356468 conn(0x7f0fc40b3870 msgr2=0x7f0fc415a7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.633+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 shutdown_connections 2024-03-06T07:36:01.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:01.633+0000 7f0fcba00640 1 -- 172.21.15.86:0/1168356468 wait complete. 2024-03-06T07:36:01.863 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:01.863 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:02.864 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:03.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.086+0000 7f4d98400640 1 Processor -- start 2024-03-06T07:36:03.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.086+0000 7f4d98400640 1 -- start start 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.087+0000 7f4d98400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d90144660 0x7f4d90071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.087+0000 7f4d98400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4d900ca5d0 0x7f4d900724b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.087+0000 7f4d98400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4d900cabf0 con 0x7f4d900cf3e0 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d98400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4d900ce900 con 0x7f4d900ca5d0 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d98400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d900cebe0 con 0x7f4d90144660 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d96400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4d900ca5d0 0x7f4d900724b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d96400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4d900ca5d0 0x7f4d900724b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34940/0 (socket says 172.21.15.86:34940) 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d96400640 1 -- 172.21.15.86:0/4040227986 learned_addr learned my addr 172.21.15.86:0/4040227986 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d96400640 1 -- 172.21.15.86:0/4040227986 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d90144660 msgr2=0x7f4d90071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:03.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d96400640 1 --2- 172.21.15.86:0/4040227986 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d90144660 0x7f4d90071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d96400640 1 -- 172.21.15.86:0/4040227986 >> v1:172.21.15.194:6789/0 conn(0x7f4d900cf3e0 legacy=0x7f4d900c9ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:03.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.088+0000 7f4d96400640 1 -- 172.21.15.86:0/4040227986 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d900ceec0 con 0x7f4d900ca5d0 2024-03-06T07:36:03.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.089+0000 7f4d96400640 1 --2- 172.21.15.86:0/4040227986 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d900ca5d0 0x7f4d900724b0 secure :-1 s=READY pgs=1615 cs=0 l=1 rev1=1 crypto rx=0x7f4d88009e50 tx=0x7f4d8802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b0ab50aebbd16059 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:03.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d8fe00640 1 -- 172.21.15.86:0/4040227986 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d8802ca70 con 0x7f4d900ca5d0 2024-03-06T07:36:03.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d8fe00640 1 -- 172.21.15.86:0/4040227986 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d8802cc10 con 0x7f4d900ca5d0 2024-03-06T07:36:03.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d8fe00640 1 -- 172.21.15.86:0/4040227986 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d880343f0 con 0x7f4d900ca5d0 2024-03-06T07:36:03.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d98400640 1 -- 172.21.15.86:0/4040227986 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d900ca5d0 msgr2=0x7f4d900724b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:03.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d98400640 1 --2- 172.21.15.86:0/4040227986 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d900ca5d0 0x7f4d900724b0 secure :-1 s=READY pgs=1615 cs=0 l=1 rev1=1 crypto rx=0x7f4d88009e50 tx=0x7f4d8802bed0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d98400640 1 -- 172.21.15.86:0/4040227986 shutdown_connections 2024-03-06T07:36:03.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d98400640 1 --2- 172.21.15.86:0/4040227986 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d900ca5d0 0x7f4d900724b0 unknown :-1 s=CLOSED pgs=1615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d98400640 1 --2- 172.21.15.86:0/4040227986 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d90144660 0x7f4d90071f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.090+0000 7f4d98400640 1 -- 172.21.15.86:0/4040227986 >> 172.21.15.86:0/4040227986 conn(0x7f4d900d0b90 msgr2=0x7f4d900d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:03.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.091+0000 7f4d98400640 1 -- 172.21.15.86:0/4040227986 shutdown_connections 2024-03-06T07:36:03.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.091+0000 7f4d98400640 1 -- 172.21.15.86:0/4040227986 wait complete. 2024-03-06T07:36:03.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.092+0000 7f4d98400640 1 Processor -- start 2024-03-06T07:36:03.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.092+0000 7f4d98400640 1 -- start start 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.093+0000 7f4d98400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d900ca5d0 0x7f4d901602e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.093+0000 7f4d98400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d900cf3e0 0x7f4d9013f730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.093+0000 7f4d98400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d90144660 0x7f4d900b4320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.093+0000 7f4d98400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d900cc270 con 0x7f4d90144660 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.093+0000 7f4d98400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d900cc0f0 con 0x7f4d900cf3e0 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.093+0000 7f4d98400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d900cc3f0 con 0x7f4d900ca5d0 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d90144660 0x7f4d900b4320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d90144660 0x7f4d900b4320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34944/0 (socket says 172.21.15.86:34944) 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 -- 172.21.15.86:0/2320353746 learned_addr learned my addr 172.21.15.86:0/2320353746 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d95a00640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d900ca5d0 0x7f4d901602e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d95000640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d900cf3e0 0x7f4d9013f730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 -- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d900ca5d0 msgr2=0x7f4d901602e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d900ca5d0 0x7f4d901602e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 -- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d900cf3e0 msgr2=0x7f4d9013f730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:03.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d900cf3e0 0x7f4d9013f730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.094+0000 7f4d96400640 1 -- 172.21.15.86:0/2320353746 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d900b4860 con 0x7f4d90144660 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.095+0000 7f4d96400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d90144660 0x7f4d900b4320 secure :-1 s=READY pgs=1616 cs=0 l=1 rev1=1 crypto rx=0x7f4d88036040 tx=0x7f4d88004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.095+0000 7f4d8d600640 1 -- 172.21.15.86:0/2320353746 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d88043020 con 0x7f4d90144660 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.095+0000 7f4d8d600640 1 -- 172.21.15.86:0/2320353746 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d8803e450 con 0x7f4d90144660 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.095+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d900b4b00 con 0x7f4d90144660 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.095+0000 7f4d8d600640 1 -- 172.21.15.86:0/2320353746 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d88009510 con 0x7f4d90144660 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.095+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4d900b50c0 con 0x7f4d90144660 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.096+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4d9004f330 con 0x7f4d90144660 2024-03-06T07:36:03.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.097+0000 7f4d8d600640 1 -- 172.21.15.86:0/2320353746 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4d88046020 con 0x7f4d90144660 2024-03-06T07:36:03.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.098+0000 7f4d8d600640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d7006b500 0x7f4d7006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:03.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.098+0000 7f4d8d600640 1 -- 172.21.15.86:0/2320353746 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4d880ac940 con 0x7f4d90144660 2024-03-06T07:36:03.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.098+0000 7f4d95a00640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d7006b500 0x7f4d7006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:03.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.099+0000 7f4d95a00640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d7006b500 0x7f4d7006d9c0 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7f4d9006fd50 tx=0x7f4d84008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:03.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.102+0000 7f4d8d600640 1 -- 172.21.15.86:0/2320353746 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4d8804d070 con 0x7f4d90144660 2024-03-06T07:36:03.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.338+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4d90143bc0 con 0x7f4d90144660 2024-03-06T07:36:03.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.339+0000 7f4d8d600640 1 -- 172.21.15.86:0/2320353746 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f4d880ac090 con 0x7f4d90144660 2024-03-06T07:36:03.341 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:03.341 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:03.342 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:03.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d7006b500 msgr2=0x7f4d7006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:03.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d7006b500 0x7f4d7006d9c0 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7f4d9006fd50 tx=0x7f4d84008040 comp rx=0 tx=0).stop 2024-03-06T07:36:03.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d90144660 msgr2=0x7f4d900b4320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:03.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d90144660 0x7f4d900b4320 secure :-1 s=READY pgs=1616 cs=0 l=1 rev1=1 crypto rx=0x7f4d88036040 tx=0x7f4d88004a40 comp rx=0 tx=0).stop 2024-03-06T07:36:03.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 shutdown_connections 2024-03-06T07:36:03.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4d7006b500 0x7f4d7006d9c0 unknown :-1 s=CLOSED pgs=689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4d90144660 0x7f4d900b4320 unknown :-1 s=CLOSED pgs=1616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4d900cf3e0 0x7f4d9013f730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 --2- 172.21.15.86:0/2320353746 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4d900ca5d0 0x7f4d901602e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:03.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.341+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 >> 172.21.15.86:0/2320353746 conn(0x7f4d900d0b90 msgr2=0x7f4d9006ffc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:03.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.342+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 shutdown_connections 2024-03-06T07:36:03.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:03.342+0000 7f4d98400640 1 -- 172.21.15.86:0/2320353746 wait complete. 2024-03-06T07:36:03.564 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:03.564 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:03.575 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:04.565 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:04.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.768+0000 7f6b0b000640 1 Processor -- start 2024-03-06T07:36:04.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.768+0000 7f6b0b000640 1 -- start start 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b0b000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b04066f10 0x7f6b0406ed70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b0b000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6b040ce8f0 0x7f6b040ced80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b0b000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b0406e5b0 con 0x7f6b0415f930 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b0b000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b040ccd90 con 0x7f6b04066f10 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b0b000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b040afc00 con 0x7f6b040ce8f0 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b03400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b04066f10 0x7f6b0406ed70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b03e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6b0415f930 0x7f6b0415fd10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37692/0 (socket says 172.21.15.86:37692) 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b03400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6b04066f10 0x7f6b0406ed70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34954/0 (socket says 172.21.15.86:34954) 2024-03-06T07:36:04.772 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b03e00640 1 -- 172.21.15.86:0/583220975 learned_addr learned my addr 172.21.15.86:0/583220975 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:04.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b03400640 1 -- 172.21.15.86:0/583220975 >> v2:172.21.15.194:3300/0 conn(0x7f6b040ce8f0 msgr2=0x7f6b040ced80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:04.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b03400640 1 --2- 172.21.15.86:0/583220975 >> v2:172.21.15.194:3300/0 conn(0x7f6b040ce8f0 0x7f6b040ced80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:04.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.770+0000 7f6b03400640 1 -- 172.21.15.86:0/583220975 >> v1:172.21.15.86:6789/0 conn(0x7f6b0415f930 legacy=0x7f6b0415fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:04.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.771+0000 7f6b03400640 1 -- 172.21.15.86:0/583220975 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b040ccfe0 con 0x7f6b04066f10 2024-03-06T07:36:04.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.772+0000 7f6b03400640 1 --2- 172.21.15.86:0/583220975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b04066f10 0x7f6b0406ed70 secure :-1 s=READY pgs=1618 cs=0 l=1 rev1=1 crypto rx=0x7f6b040b3460 tx=0x7f6af002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=49a92db7dd4db158 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:04.773 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.772+0000 7f6b02a00640 1 -- 172.21.15.86:0/583220975 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6af00315a0 con 0x7f6b04066f10 2024-03-06T07:36:04.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.772+0000 7f6b02a00640 1 -- 172.21.15.86:0/583220975 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6af0037070 con 0x7f6b04066f10 2024-03-06T07:36:04.774 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.772+0000 7f6b02a00640 1 -- 172.21.15.86:0/583220975 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6af0031960 con 0x7f6b04066f10 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.773+0000 7f6b0b000640 1 -- 172.21.15.86:0/583220975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b04066f10 msgr2=0x7f6b0406ed70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.773+0000 7f6b0b000640 1 --2- 172.21.15.86:0/583220975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b04066f10 0x7f6b0406ed70 secure :-1 s=READY pgs=1618 cs=0 l=1 rev1=1 crypto rx=0x7f6b040b3460 tx=0x7f6af002b840 comp rx=0 tx=0).stop 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.773+0000 7f6b0b000640 1 -- 172.21.15.86:0/583220975 shutdown_connections 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.773+0000 7f6b0b000640 1 --2- 172.21.15.86:0/583220975 >> v2:172.21.15.194:3300/0 conn(0x7f6b040ce8f0 0x7f6b040ced80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.773+0000 7f6b0b000640 1 --2- 172.21.15.86:0/583220975 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b04066f10 0x7f6b0406ed70 unknown :-1 s=CLOSED pgs=1618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.773+0000 7f6b0b000640 1 -- 172.21.15.86:0/583220975 >> 172.21.15.86:0/583220975 conn(0x7f6b040d0340 msgr2=0x7f6b040d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.773+0000 7f6b0b000640 1 -- 172.21.15.86:0/583220975 shutdown_connections 2024-03-06T07:36:04.775 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.774+0000 7f6b0b000640 1 -- 172.21.15.86:0/583220975 wait complete. 2024-03-06T07:36:04.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.775+0000 7f6b0b000640 1 Processor -- start 2024-03-06T07:36:04.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.775+0000 7f6b0b000640 1 -- start start 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b0b000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b04066f10 0x7f6b040b4c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b0b000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b040ce8f0 0x7f6b040b34b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b0b000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b0415f930 0x7f6b04151570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b0b000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b0406e090 con 0x7f6b040ce8f0 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b0b000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b0406ddb0 con 0x7f6b0415f930 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b0b000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b0406c780 con 0x7f6b04066f10 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b03400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b040ce8f0 0x7f6b040b34b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b03400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b040ce8f0 0x7f6b040b34b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:34964/0 (socket says 172.21.15.86:34964) 2024-03-06T07:36:04.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.776+0000 7f6b03400640 1 -- 172.21.15.86:0/695590221 learned_addr learned my addr 172.21.15.86:0/695590221 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6b03400640 1 -- 172.21.15.86:0/695590221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b04066f10 msgr2=0x7f6b040b4c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6b03400640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b04066f10 0x7f6b040b4c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6b03400640 1 -- 172.21.15.86:0/695590221 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b0415f930 msgr2=0x7f6b04151570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6b03400640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b0415f930 0x7f6b04151570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6b03400640 1 -- 172.21.15.86:0/695590221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b040b3bb0 con 0x7f6b040ce8f0 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6b03400640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b040ce8f0 0x7f6b040b34b0 secure :-1 s=READY pgs=1619 cs=0 l=1 rev1=1 crypto rx=0x7f6af0009890 tx=0x7f6af00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6afbe00640 1 -- 172.21.15.86:0/695590221 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6af0032840 con 0x7f6b040ce8f0 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6afbe00640 1 -- 172.21.15.86:0/695590221 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6af0032a00 con 0x7f6b040ce8f0 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6afbe00640 1 -- 172.21.15.86:0/695590221 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6af0032df0 con 0x7f6b040ce8f0 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.777+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b040b3e90 con 0x7f6b040ce8f0 2024-03-06T07:36:04.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.778+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b040b4450 con 0x7f6b040ce8f0 2024-03-06T07:36:04.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.779+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b04074ae0 con 0x7f6b040ce8f0 2024-03-06T07:36:04.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.779+0000 7f6afbe00640 1 -- 172.21.15.86:0/695590221 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6af004c020 con 0x7f6b040ce8f0 2024-03-06T07:36:04.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.780+0000 7f6afbe00640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6ae406b4d0 0x7f6ae406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:04.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.780+0000 7f6afbe00640 1 -- 172.21.15.86:0/695590221 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6af002f020 con 0x7f6b040ce8f0 2024-03-06T07:36:04.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.781+0000 7f6b03e00640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6ae406b4d0 0x7f6ae406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:04.783 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.781+0000 7f6b03e00640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6ae406b4d0 0x7f6ae406d990 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7f6af4000f60 tx=0x7f6af4007a80 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:04.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:04.785+0000 7f6afbe00640 1 -- 172.21.15.86:0/695590221 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6af00af030 con 0x7f6b040ce8f0 2024-03-06T07:36:05.064 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.063+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6b040d0fa0 con 0x7f6b040ce8f0 2024-03-06T07:36:05.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.063+0000 7f6afbe00640 1 -- 172.21.15.86:0/695590221 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f6af0040070 con 0x7f6b040ce8f0 2024-03-06T07:36:05.065 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:05.065 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:05.066 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6ae406b4d0 msgr2=0x7f6ae406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6ae406b4d0 0x7f6ae406d990 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7f6af4000f60 tx=0x7f6af4007a80 comp rx=0 tx=0).stop 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b040ce8f0 msgr2=0x7f6b040b34b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b040ce8f0 0x7f6b040b34b0 secure :-1 s=READY pgs=1619 cs=0 l=1 rev1=1 crypto rx=0x7f6af0009890 tx=0x7f6af00046c0 comp rx=0 tx=0).stop 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 shutdown_connections 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6ae406b4d0 0x7f6ae406d990 unknown :-1 s=CLOSED pgs=690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6b0415f930 0x7f6b04151570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:05.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6b040ce8f0 0x7f6b040b34b0 unknown :-1 s=CLOSED pgs=1619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:05.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 --2- 172.21.15.86:0/695590221 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6b04066f10 0x7f6b040b4c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:05.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 >> 172.21.15.86:0/695590221 conn(0x7f6b040d0340 msgr2=0x7f6b0406ea40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:05.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 shutdown_connections 2024-03-06T07:36:05.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:05.066+0000 7f6b0b000640 1 -- 172.21.15.86:0/695590221 wait complete. 2024-03-06T07:36:05.292 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:05.292 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:05.578 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:06.292 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:06.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.510+0000 7ff6e4c00640 1 Processor -- start 2024-03-06T07:36:06.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.510+0000 7ff6e4c00640 1 -- start start 2024-03-06T07:36:06.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.512+0000 7ff6e4c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff6e0144c00 0x7ff6e006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.512+0000 7ff6e4c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e015f9d0 0x7ff6e015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.512+0000 7ff6e4c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff6e006ee70 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.512+0000 7ff6e4c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff6e0160370 con 0x7ff6e0144c00 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.512+0000 7ff6e4c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff6e006cef0 con 0x7ff6e015f9d0 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.512+0000 7ff6d5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e015f9d0 0x7ff6e015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.512+0000 7ff6dde00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff6e0144c00 0x7ff6e006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6d5400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e015f9d0 0x7ff6e015fe30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:49212/0 (socket says 172.21.15.86:49212) 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6dde00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff6e0144c00 0x7ff6e006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:53966/0 (socket says 172.21.15.86:53966) 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6d5400640 1 -- 172.21.15.86:0/3363830356 learned_addr learned my addr 172.21.15.86:0/3363830356 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6dde00640 1 -- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e015f9d0 msgr2=0x7ff6e015fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6dde00640 1 --2- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e015f9d0 0x7ff6e015fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6dde00640 1 -- 172.21.15.86:0/3363830356 >> v1:172.21.15.194:6789/0 conn(0x7ff6e00ce0a0 legacy=0x7ff6e00ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6dde00640 1 -- 172.21.15.86:0/3363830356 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6e006d1d0 con 0x7ff6e0144c00 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.513+0000 7ff6d5400640 1 --2- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e015f9d0 0x7ff6e015fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-06T07:36:06.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.515+0000 7ff6dde00640 1 --2- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e0144c00 0x7ff6e006e930 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7ff6d00097a0 tx=0x7ff6d002b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=efbd2bbe1a14a6f8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.515+0000 7ff6dd400640 1 -- 172.21.15.86:0/3363830356 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6d0035020 con 0x7ff6e0144c00 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.515+0000 7ff6dd400640 1 -- 172.21.15.86:0/3363830356 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff6d0031830 con 0x7ff6e0144c00 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.515+0000 7ff6dd400640 1 -- 172.21.15.86:0/3363830356 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6d0031c00 con 0x7ff6e0144c00 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 -- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e0144c00 msgr2=0x7ff6e006e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e0144c00 0x7ff6e006e930 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7ff6d00097a0 tx=0x7ff6d002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 -- 172.21.15.86:0/3363830356 shutdown_connections 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e015f9d0 0x7ff6e015fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/3363830356 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e0144c00 0x7ff6e006e930 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 -- 172.21.15.86:0/3363830356 >> 172.21.15.86:0/3363830356 conn(0x7ff6e00b1730 msgr2=0x7ff6e00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 -- 172.21.15.86:0/3363830356 shutdown_connections 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.516+0000 7ff6e4c00640 1 -- 172.21.15.86:0/3363830356 wait complete. 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.518+0000 7ff6e4c00640 1 Processor -- start 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.518+0000 7ff6e4c00640 1 -- start start 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6e4c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6e00ce0a0 0x7ff6e015aed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6e4c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e0144c00 0x7ff6e013c4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:06.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6e4c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e015f9d0 0x7ff6e0071c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6e4c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6e0160730 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6e4c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6e006d790 con 0x7ff6e015f9d0 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6e4c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff6e006cb50 con 0x7ff6e0144c00 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6dde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6e00ce0a0 0x7ff6e015aed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6dde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6e00ce0a0 0x7ff6e015aed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56912/0 (socket says 172.21.15.86:56912) 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.520+0000 7ff6dde00640 1 -- 172.21.15.86:0/1333015023 learned_addr learned my addr 172.21.15.86:0/1333015023 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6d5400640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e0144c00 0x7ff6e013c4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6dde00640 1 -- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e0144c00 msgr2=0x7ff6e013c4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6dde00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e0144c00 0x7ff6e013c4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6dde00640 1 -- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e015f9d0 msgr2=0x7ff6e0071c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6de800640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e015f9d0 0x7ff6e0071c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6dde00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e015f9d0 0x7ff6e0071c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6dde00640 1 -- 172.21.15.86:0/1333015023 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6e013cd30 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6de800640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e015f9d0 0x7ff6e0071c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6dde00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6e00ce0a0 0x7ff6e015aed0 secure :-1 s=READY pgs=1620 cs=0 l=1 rev1=1 crypto rx=0x7ff6d0009890 tx=0x7ff6d00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:06.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6d6a00640 1 -- 172.21.15.86:0/1333015023 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6d0035040 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6d6a00640 1 -- 172.21.15.86:0/1333015023 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff6d003b920 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6d6a00640 1 -- 172.21.15.86:0/1333015023 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6d003bd10 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.521+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff6e013d010 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.522+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff6e015d590 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.523+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff6e004f330 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.525+0000 7ff6d6a00640 1 -- 172.21.15.86:0/1333015023 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff6d0038020 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.526+0000 7ff6d6a00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6bc06b4d0 0x7ff6bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.526+0000 7ff6d6a00640 1 -- 172.21.15.86:0/1333015023 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff6d00ac450 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.526+0000 7ff6d5400640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6bc06b4d0 0x7ff6bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.527+0000 7ff6d5400640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6bc06b4d0 0x7ff6bc06d990 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7ff6e00afbd0 tx=0x7ff6c8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:06.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.531+0000 7ff6d6a00640 1 -- 172.21.15.86:0/1333015023 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff6d007b200 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.766 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.765+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff6e0072890 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.767 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.765+0000 7ff6d6a00640 1 -- 172.21.15.86:0/1333015023 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7ff6d0031800 con 0x7ff6e00ce0a0 2024-03-06T07:36:06.767 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:06.767 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:06.767 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:06.769 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6bc06b4d0 msgr2=0x7ff6bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6bc06b4d0 0x7ff6bc06d990 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7ff6e00afbd0 tx=0x7ff6c8008040 comp rx=0 tx=0).stop 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6e00ce0a0 msgr2=0x7ff6e015aed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6e00ce0a0 0x7ff6e015aed0 secure :-1 s=READY pgs=1620 cs=0 l=1 rev1=1 crypto rx=0x7ff6d0009890 tx=0x7ff6d00046c0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 shutdown_connections 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff6bc06b4d0 0x7ff6bc06d990 unknown :-1 s=CLOSED pgs=691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff6e015f9d0 0x7ff6e0071c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff6e0144c00 0x7ff6e013c4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 --2- 172.21.15.86:0/1333015023 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff6e00ce0a0 0x7ff6e015aed0 unknown :-1 s=CLOSED pgs=1620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 >> 172.21.15.86:0/1333015023 conn(0x7ff6e00b1730 msgr2=0x7ff6e0145250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 shutdown_connections 2024-03-06T07:36:06.770 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:06.768+0000 7ff6e4c00640 1 -- 172.21.15.86:0/1333015023 wait complete. 2024-03-06T07:36:06.989 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:06.989 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:07.580 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:07.989 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:08.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.202+0000 7f146a800640 1 Processor -- start 2024-03-06T07:36:08.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.203+0000 7f146a800640 1 -- start start 2024-03-06T07:36:08.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.204+0000 7f146a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 0x7f146406e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.204+0000 7f146a800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f146415fe30 0x7f14640ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.204+0000 7f146a800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f146406ee70 con 0x7f146415f9d0 2024-03-06T07:36:08.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.204+0000 7f146a800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f146406cef0 con 0x7f146415fe30 2024-03-06T07:36:08.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.204+0000 7f146a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f146406d1d0 con 0x7f1464144c00 2024-03-06T07:36:08.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.205+0000 7f1462a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f146415f9d0 0x7f14640ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:48156/0 (socket says 172.21.15.86:48156) 2024-03-06T07:36:08.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.205+0000 7f1462a00640 1 -- 172.21.15.86:0/3416143676 learned_addr learned my addr 172.21.15.86:0/3416143676 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:08.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.205+0000 7f1463400640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 0x7f146406e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:08.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.205+0000 7f1463e00640 1 --2- 172.21.15.86:0/3416143676 >> v2:172.21.15.194:3300/0 conn(0x7f146415fe30 0x7f14640ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:08.208 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.205+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4163765175 0 0) 0x7f1450001110 con 0x7f146415f9d0 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.205+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1448003520 con 0x7f146415f9d0 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.205+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4011666763 0 0) 0x7f1448003520 con 0x7f146415f9d0 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 msgr2=0x7f146406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 0x7f146406e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 >> v2:172.21.15.194:3300/0 conn(0x7f146415fe30 msgr2=0x7f14640ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 --2- 172.21.15.86:0/3416143676 >> v2:172.21.15.194:3300/0 conn(0x7f146415fe30 0x7f14640ce7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f146406d4b0 con 0x7f146415f9d0 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1450001840 con 0x7f146415f9d0 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 >> v1:172.21.15.86:6789/0 conn(0x7f146415f9d0 legacy=0x7f14640ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14480039b0 0x7f1448003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 0x7f1448004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1448004d40 0x7f14480051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f146406cef0 con 0x7f1448004d40 2024-03-06T07:36:08.209 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f146406d1d0 con 0x7f14480039b0 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f144800b9c0 con 0x7f1464144c00 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1463e00640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1448004d40 0x7f14480051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1462a00640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 0x7f1448004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1463400640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14480039b0 0x7f1448003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.206+0000 7f1463e00640 1 -- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 msgr2=0x7f1448004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.207+0000 7f1463e00640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f1464144c00 0x7f1448004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.207+0000 7f1463e00640 1 -- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14480039b0 msgr2=0x7f1448003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.207+0000 7f1463e00640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f14480039b0 0x7f1448003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.210 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.207+0000 7f1463e00640 1 -- 172.21.15.86:0/3416143676 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1448003520 con 0x7f1448004d40 2024-03-06T07:36:08.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f1463e00640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1448004d40 0x7f14480051c0 secure :-1 s=READY pgs=1622 cs=0 l=1 rev1=1 crypto rx=0x7f145c009fc0 tx=0x7f1448005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=34890718ce3e916a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:08.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f145c039070 con 0x7f1448004d40 2024-03-06T07:36:08.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f145c02cc60 con 0x7f1448004d40 2024-03-06T07:36:08.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f1462000640 1 -- 172.21.15.86:0/3416143676 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f145c009c10 con 0x7f1448004d40 2024-03-06T07:36:08.211 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f146a800640 1 -- 172.21.15.86:0/3416143676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1448004d40 msgr2=0x7f14480051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:08.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f146a800640 1 --2- 172.21.15.86:0/3416143676 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1448004d40 0x7f14480051c0 secure :-1 s=READY pgs=1622 cs=0 l=1 rev1=1 crypto rx=0x7f145c009fc0 tx=0x7f1448005830 comp rx=0 tx=0).stop 2024-03-06T07:36:08.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f1463e00640 1 -- 172.21.15.86:0/3416143676 reap_dead start 2024-03-06T07:36:08.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f146a800640 1 -- 172.21.15.86:0/3416143676 shutdown_connections 2024-03-06T07:36:08.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.208+0000 7f146a800640 1 -- 172.21.15.86:0/3416143676 >> 172.21.15.86:0/3416143676 conn(0x7f14640b1730 msgr2=0x7f14640b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:08.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.209+0000 7f146a800640 1 -- 172.21.15.86:0/3416143676 shutdown_connections 2024-03-06T07:36:08.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.209+0000 7f146a800640 1 -- 172.21.15.86:0/3416143676 wait complete. 2024-03-06T07:36:08.212 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.211+0000 7f146a800640 1 Processor -- start 2024-03-06T07:36:08.213 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.212+0000 7f146a800640 1 -- start start 2024-03-06T07:36:08.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f146a800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f146413c2c0 0x7f14640aedf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f146a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f146413cb10 0x7f14640b0210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.215 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f146a800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14640b0750 0x7f14641417c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f146a800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1464160730 con 0x7f146413cb10 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f146a800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1464160280 con 0x7f146413c2c0 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f146a800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f146406cb50 con 0x7f14640b0750 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f1462a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f146413cb10 0x7f14640b0210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f1462a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f146413cb10 0x7f14640b0210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56920/0 (socket says 172.21.15.86:56920) 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.213+0000 7f1462a00640 1 -- 172.21.15.86:0/4179183378 learned_addr learned my addr 172.21.15.86:0/4179183378 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f1462a00640 1 -- 172.21.15.86:0/4179183378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14640b0750 msgr2=0x7f14641417c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f1462a00640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14640b0750 0x7f14641417c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f1462a00640 1 -- 172.21.15.86:0/4179183378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f146413c2c0 msgr2=0x7f14640aedf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f1462a00640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f146413c2c0 0x7f14640aedf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f1462a00640 1 -- 172.21.15.86:0/4179183378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14640b2b30 con 0x7f146413cb10 2024-03-06T07:36:08.216 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f1462a00640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f146413cb10 0x7f14640b0210 secure :-1 s=READY pgs=1623 cs=0 l=1 rev1=1 crypto rx=0x7f145000dae0 tx=0x7f1450001830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:08.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f145b400640 1 -- 172.21.15.86:0/4179183378 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1450019070 con 0x7f146413cb10 2024-03-06T07:36:08.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f145b400640 1 -- 172.21.15.86:0/4179183378 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1450004cb0 con 0x7f146413cb10 2024-03-06T07:36:08.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f145b400640 1 -- 172.21.15.86:0/4179183378 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1450004e60 con 0x7f146413cb10 2024-03-06T07:36:08.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.214+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f14640ae600 con 0x7f146413cb10 2024-03-06T07:36:08.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.215+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14640b2df0 con 0x7f146413cb10 2024-03-06T07:36:08.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.216+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f146404f330 con 0x7f146413cb10 2024-03-06T07:36:08.218 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.216+0000 7f145b400640 1 -- 172.21.15.86:0/4179183378 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f145001c070 con 0x7f146413cb10 2024-03-06T07:36:08.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.217+0000 7f145b400640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f144406b4d0 0x7f144406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:08.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.217+0000 7f145b400640 1 -- 172.21.15.86:0/4179183378 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1450089d70 con 0x7f146413cb10 2024-03-06T07:36:08.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.218+0000 7f1463400640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f144406b4d0 0x7f144406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:08.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.218+0000 7f1463400640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f144406b4d0 0x7f144406d990 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f1454009910 tx=0x7f1454008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:08.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.227+0000 7f145b400640 1 -- 172.21.15.86:0/4179183378 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f146406ee70 con 0x7f146413cb10 2024-03-06T07:36:08.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.556+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1464072890 con 0x7f146413cb10 2024-03-06T07:36:08.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.557+0000 7f145b400640 1 -- 172.21.15.86:0/4179183378 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f1450058b10 con 0x7f146413cb10 2024-03-06T07:36:08.558 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:08.559 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:08.559 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f144406b4d0 msgr2=0x7f144406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f144406b4d0 0x7f144406d990 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f1454009910 tx=0x7f1454008040 comp rx=0 tx=0).stop 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f146413cb10 msgr2=0x7f14640b0210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f146413cb10 0x7f14640b0210 secure :-1 s=READY pgs=1623 cs=0 l=1 rev1=1 crypto rx=0x7f145000dae0 tx=0x7f1450001830 comp rx=0 tx=0).stop 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 shutdown_connections 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f144406b4d0 0x7f144406d990 unknown :-1 s=CLOSED pgs=692 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f14640b0750 0x7f14641417c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f146413cb10 0x7f14640b0210 unknown :-1 s=CLOSED pgs=1623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 --2- 172.21.15.86:0/4179183378 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f146413c2c0 0x7f14640aedf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 >> 172.21.15.86:0/4179183378 conn(0x7f14640b1730 msgr2=0x7f14640cede0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 shutdown_connections 2024-03-06T07:36:08.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:08.559+0000 7f146a800640 1 -- 172.21.15.86:0/4179183378 wait complete. 2024-03-06T07:36:08.781 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:08.781 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:09.581 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:09.782 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:09.989 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.988+0000 7f5d60200640 1 Processor -- start 2024-03-06T07:36:09.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.988+0000 7f5d60200640 1 -- start start 2024-03-06T07:36:09.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d60200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5d580b0300 0x7f5d5806f5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:09.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d60200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5d580b6090 con 0x7f5d5806afc0 2024-03-06T07:36:09.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d60200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5d580b5db0 con 0x7f5d58147430 2024-03-06T07:36:09.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d60200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5d580d09c0 con 0x7f5d580b0300 2024-03-06T07:36:09.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d5d800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5d580b0300 0x7f5d5806f5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:09.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d5d800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f5d580b0300 0x7f5d5806f5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56926/0 (socket says 172.21.15.86:56926) 2024-03-06T07:36:09.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d5d800640 1 -- 172.21.15.86:0/2564267394 learned_addr learned my addr 172.21.15.86:0/2564267394 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:09.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.991+0000 7f5d5e200640 1 --1- 172.21.15.86:0/2564267394 >> v1:172.21.15.86:6789/0 conn(0x7f5d5806afc0 0x7f5d5806d920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:48164/0 (socket says 172.21.15.86:48164) 2024-03-06T07:36:09.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.992+0000 7f5d5d800640 1 -- 172.21.15.86:0/2564267394 >> v1:172.21.15.194:6789/0 conn(0x7f5d58147430 legacy=0x7f5d580d14a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:09.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.992+0000 7f5d5d800640 1 -- 172.21.15.86:0/2564267394 >> v1:172.21.15.86:6789/0 conn(0x7f5d5806afc0 legacy=0x7f5d5806d920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:09.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.992+0000 7f5d5d800640 1 -- 172.21.15.86:0/2564267394 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5d580d0ca0 con 0x7f5d580b0300 2024-03-06T07:36:09.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.993+0000 7f5d5d800640 1 --2- 172.21.15.86:0/2564267394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d580b0300 0x7f5d5806f5e0 secure :-1 s=READY pgs=1625 cs=0 l=1 rev1=1 crypto rx=0x7f5d4c009870 tx=0x7f5d4c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=251af3d304901d5a server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:09.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.993+0000 7f5d57e00640 1 -- 172.21.15.86:0/2564267394 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d4c034070 con 0x7f5d580b0300 2024-03-06T07:36:09.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.993+0000 7f5d57e00640 1 -- 172.21.15.86:0/2564267394 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5d4c030860 con 0x7f5d580b0300 2024-03-06T07:36:09.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.993+0000 7f5d57e00640 1 -- 172.21.15.86:0/2564267394 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d4c030c30 con 0x7f5d580b0300 2024-03-06T07:36:09.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.994+0000 7f5d60200640 1 -- 172.21.15.86:0/2564267394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d580b0300 msgr2=0x7f5d5806f5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:09.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.994+0000 7f5d60200640 1 --2- 172.21.15.86:0/2564267394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d580b0300 0x7f5d5806f5e0 secure :-1 s=READY pgs=1625 cs=0 l=1 rev1=1 crypto rx=0x7f5d4c009870 tx=0x7f5d4c02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:36:09.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.994+0000 7f5d60200640 1 -- 172.21.15.86:0/2564267394 shutdown_connections 2024-03-06T07:36:09.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.994+0000 7f5d60200640 1 --2- 172.21.15.86:0/2564267394 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d580b0300 0x7f5d5806f5e0 unknown :-1 s=CLOSED pgs=1625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:09.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.994+0000 7f5d60200640 1 -- 172.21.15.86:0/2564267394 >> 172.21.15.86:0/2564267394 conn(0x7f5d5806c0d0 msgr2=0x7f5d5806c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:09.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.994+0000 7f5d60200640 1 -- 172.21.15.86:0/2564267394 shutdown_connections 2024-03-06T07:36:09.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.994+0000 7f5d60200640 1 -- 172.21.15.86:0/2564267394 wait complete. 2024-03-06T07:36:09.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.995+0000 7f5d60200640 1 Processor -- start 2024-03-06T07:36:09.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.996+0000 7f5d60200640 1 -- start start 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.997+0000 7f5d60200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5d5806afc0 0x7f5d5815dca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.997+0000 7f5d60200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5d580b0300 0x7f5d580c61c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.997+0000 7f5d60200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d58147430 0x7f5d580c6700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.997+0000 7f5d60200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d580b5b50 con 0x7f5d58147430 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.997+0000 7f5d60200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d580b59d0 con 0x7f5d580b0300 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.997+0000 7f5d60200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5d5806b470 con 0x7f5d5806afc0 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.997+0000 7f5d5e200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d58147430 0x7f5d580c6700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d58147430 0x7f5d580c6700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56940/0 (socket says 172.21.15.86:56940) 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 -- 172.21.15.86:0/318581257 learned_addr learned my addr 172.21.15.86:0/318581257 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5ce00640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5d580b0300 0x7f5d580c61c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 -- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5d5806afc0 msgr2=0x7f5d5815dca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5d5806afc0 0x7f5d5815dca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 -- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5d580b0300 msgr2=0x7f5d580c61c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:10.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5d580b0300 0x7f5d580c61c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:10.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 -- 172.21.15.86:0/318581257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5d5815fd90 con 0x7f5d58147430 2024-03-06T07:36:10.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5ce00640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5d580b0300 0x7f5d580c61c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:10.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d5e200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d58147430 0x7f5d580c6700 secure :-1 s=READY pgs=1626 cs=0 l=1 rev1=1 crypto rx=0x7f5d5000c7b0 tx=0x7f5d5000cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:10.001 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.998+0000 7f5d55600640 1 -- 172.21.15.86:0/318581257 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d50017070 con 0x7f5d58147430 2024-03-06T07:36:10.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.999+0000 7f5d55600640 1 -- 172.21.15.86:0/318581257 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5d5000a640 con 0x7f5d58147430 2024-03-06T07:36:10.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.999+0000 7f5d55600640 1 -- 172.21.15.86:0/318581257 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d5000a9f0 con 0x7f5d58147430 2024-03-06T07:36:10.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.999+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5d58160a90 con 0x7f5d58147430 2024-03-06T07:36:10.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:09.999+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5d5815e1e0 con 0x7f5d58147430 2024-03-06T07:36:10.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.000+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5d5804f330 con 0x7f5d58147430 2024-03-06T07:36:10.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.001+0000 7f5d55600640 1 -- 172.21.15.86:0/318581257 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5d5001a070 con 0x7f5d58147430 2024-03-06T07:36:10.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.002+0000 7f5d55600640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5d3806b4d0 0x7f5d3806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:10.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.002+0000 7f5d55600640 1 -- 172.21.15.86:0/318581257 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f5d50089760 con 0x7f5d58147430 2024-03-06T07:36:10.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.002+0000 7f5d5d800640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5d3806b4d0 0x7f5d3806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:10.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.003+0000 7f5d5d800640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5d3806b4d0 0x7f5d3806d990 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f5d4c036000 tx=0x7f5d4c03b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:10.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.006+0000 7f5d55600640 1 -- 172.21.15.86:0/318581257 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5d50057f40 con 0x7f5d58147430 2024-03-06T07:36:10.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.272+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5d58072890 con 0x7f5d58147430 2024-03-06T07:36:10.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.273+0000 7f5d55600640 1 -- 172.21.15.86:0/318581257 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f5d5005bfe0 con 0x7f5d58147430 2024-03-06T07:36:10.274 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:10.274 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:10.275 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:10.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5d3806b4d0 msgr2=0x7f5d3806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:10.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5d3806b4d0 0x7f5d3806d990 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f5d4c036000 tx=0x7f5d4c03b040 comp rx=0 tx=0).stop 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d58147430 msgr2=0x7f5d580c6700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d58147430 0x7f5d580c6700 secure :-1 s=READY pgs=1626 cs=0 l=1 rev1=1 crypto rx=0x7f5d5000c7b0 tx=0x7f5d5000cc70 comp rx=0 tx=0).stop 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 shutdown_connections 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f5d3806b4d0 0x7f5d3806d990 unknown :-1 s=CLOSED pgs=693 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5d58147430 0x7f5d580c6700 unknown :-1 s=CLOSED pgs=1626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5d580b0300 0x7f5d580c61c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 --2- 172.21.15.86:0/318581257 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5d5806afc0 0x7f5d5815dca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.275+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 >> 172.21.15.86:0/318581257 conn(0x7f5d5806c0d0 msgr2=0x7f5d580b1880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.276+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 shutdown_connections 2024-03-06T07:36:10.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:10.276+0000 7f5d60200640 1 -- 172.21.15.86:0/318581257 wait complete. 2024-03-06T07:36:10.497 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:10.497 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:11.498 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:11.583 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:11.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.704+0000 7f8143600640 1 Processor -- start 2024-03-06T07:36:11.705 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.704+0000 7f8143600640 1 -- start start 2024-03-06T07:36:11.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.705+0000 7f8143600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f813c15acb0 0x7f813c0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:11.707 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.705+0000 7f8143600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f813c0b4680 0x7f813c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.705+0000 7f8143600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 0x7f813c06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.705+0000 7f8143600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f813c15b090 con 0x7f813c0b4680 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.705+0000 7f8143600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f813c0b2a30 con 0x7f813c15acb0 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.705+0000 7f8143600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f813c0b4360 con 0x7f813c072340 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f813c0b4680 0x7f813c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f813c0b4680 0x7f813c06e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56954/0 (socket says 172.21.15.86:56954) 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 -- 172.21.15.86:0/1894278959 learned_addr learned my addr 172.21.15.86:0/1894278959 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 -- 172.21.15.86:0/1894278959 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 msgr2=0x7f813c06ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 --2- 172.21.15.86:0/1894278959 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 0x7f813c06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 -- 172.21.15.86:0/1894278959 >> v2:172.21.15.194:3300/0 conn(0x7f813c15acb0 msgr2=0x7f813c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 --2- 172.21.15.86:0/1894278959 >> v2:172.21.15.194:3300/0 conn(0x7f813c15acb0 0x7f813c0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:11.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.706+0000 7f8133e00640 1 -- 172.21.15.86:0/1894278959 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f813c0b2cf0 con 0x7f813c0b4680 2024-03-06T07:36:11.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.707+0000 7f8133e00640 1 --2- 172.21.15.86:0/1894278959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c06e4f0 secure :-1 s=READY pgs=1627 cs=0 l=1 rev1=1 crypto rx=0x7f813c0b27f0 tx=0x7f812402db10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d166385ad6664296 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:11.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f813be00640 1 -- 172.21.15.86:0/1894278959 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f812402f8d0 con 0x7f813c0b4680 2024-03-06T07:36:11.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f813be00640 1 -- 172.21.15.86:0/1894278959 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f812402fa70 con 0x7f813c0b4680 2024-03-06T07:36:11.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f813be00640 1 -- 172.21.15.86:0/1894278959 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f812402fe40 con 0x7f813c0b4680 2024-03-06T07:36:11.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 -- 172.21.15.86:0/1894278959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 msgr2=0x7f813c06e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:11.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 --2- 172.21.15.86:0/1894278959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c06e4f0 secure :-1 s=READY pgs=1627 cs=0 l=1 rev1=1 crypto rx=0x7f813c0b27f0 tx=0x7f812402db10 comp rx=0 tx=0).stop 2024-03-06T07:36:11.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 -- 172.21.15.86:0/1894278959 shutdown_connections 2024-03-06T07:36:11.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 --2- 172.21.15.86:0/1894278959 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 0x7f813c06ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:11.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 --2- 172.21.15.86:0/1894278959 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c06e4f0 unknown :-1 s=CLOSED pgs=1627 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:11.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 --2- 172.21.15.86:0/1894278959 >> v2:172.21.15.194:3300/0 conn(0x7f813c15acb0 0x7f813c0b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:11.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 -- 172.21.15.86:0/1894278959 >> 172.21.15.86:0/1894278959 conn(0x7f813c0b1730 msgr2=0x7f813c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:11.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 -- 172.21.15.86:0/1894278959 shutdown_connections 2024-03-06T07:36:11.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.708+0000 7f8143600640 1 -- 172.21.15.86:0/1894278959 wait complete. 2024-03-06T07:36:11.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.710+0000 7f8143600640 1 Processor -- start 2024-03-06T07:36:11.711 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.710+0000 7f8143600640 1 -- start start 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8143600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 0x7f813c14e0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8143600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c1592d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8143600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f813c15acb0 0x7f813c1538e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8143600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f813c0b3d10 con 0x7f813c0b4680 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8143600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f813c0b32b0 con 0x7f813c15acb0 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8143600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f813c0b4010 con 0x7f813c072340 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8133e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c1592d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8133e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c1592d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56960/0 (socket says 172.21.15.86:56960) 2024-03-06T07:36:11.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.711+0000 7f8133e00640 1 -- 172.21.15.86:0/3406797951 learned_addr learned my addr 172.21.15.86:0/3406797951 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8140c00640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 0x7f813c14e0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8141600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f813c15acb0 0x7f813c1538e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8133e00640 1 -- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 msgr2=0x7f813c14e0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8133e00640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 0x7f813c14e0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8133e00640 1 -- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f813c15acb0 msgr2=0x7f813c1538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8133e00640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f813c15acb0 0x7f813c1538e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8133e00640 1 -- 172.21.15.86:0/3406797951 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f813c159810 con 0x7f813c0b4680 2024-03-06T07:36:11.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8141600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f813c15acb0 0x7f813c1538e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:11.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8133e00640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c1592d0 secure :-1 s=READY pgs=1628 cs=0 l=1 rev1=1 crypto rx=0x7f8124000c00 tx=0x7f81240046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:11.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.712+0000 7f8139600640 1 -- 172.21.15.86:0/3406797951 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8124046020 con 0x7f813c0b4680 2024-03-06T07:36:11.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.713+0000 7f8139600640 1 -- 172.21.15.86:0/3406797951 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8124037db0 con 0x7f813c0b4680 2024-03-06T07:36:11.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.713+0000 7f8139600640 1 -- 172.21.15.86:0/3406797951 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81240426b0 con 0x7f813c0b4680 2024-03-06T07:36:11.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.713+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f813c152800 con 0x7f813c0b4680 2024-03-06T07:36:11.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.713+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f813c152dc0 con 0x7f813c0b4680 2024-03-06T07:36:11.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.714+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f813c04f330 con 0x7f813c0b4680 2024-03-06T07:36:11.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.715+0000 7f8139600640 1 -- 172.21.15.86:0/3406797951 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8124049020 con 0x7f813c0b4680 2024-03-06T07:36:11.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.716+0000 7f8139600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f811806b4d0 0x7f811806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:11.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.716+0000 7f8139600640 1 -- 172.21.15.86:0/3406797951 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f81240acf10 con 0x7f813c0b4680 2024-03-06T07:36:11.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.716+0000 7f8140c00640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f811806b4d0 0x7f811806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:11.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.717+0000 7f8140c00640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f811806b4d0 0x7f811806d990 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f812c00b810 tx=0x7f812c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:11.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:11.720+0000 7f8139600640 1 -- 172.21.15.86:0/3406797951 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8124050030 con 0x7f813c0b4680 2024-03-06T07:36:12.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.021+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f813c06ce60 con 0x7f813c0b4680 2024-03-06T07:36:12.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.021+0000 7f8139600640 1 -- 172.21.15.86:0/3406797951 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f812407bcb0 con 0x7f813c0b4680 2024-03-06T07:36:12.023 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:12.023 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:12.024 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:12.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f811806b4d0 msgr2=0x7f811806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:12.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f811806b4d0 0x7f811806d990 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f812c00b810 tx=0x7f812c008040 comp rx=0 tx=0).stop 2024-03-06T07:36:12.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 msgr2=0x7f813c1592d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:12.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c1592d0 secure :-1 s=READY pgs=1628 cs=0 l=1 rev1=1 crypto rx=0x7f8124000c00 tx=0x7f81240046c0 comp rx=0 tx=0).stop 2024-03-06T07:36:12.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 shutdown_connections 2024-03-06T07:36:12.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f811806b4d0 0x7f811806d990 unknown :-1 s=CLOSED pgs=694 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:12.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f813c15acb0 0x7f813c1538e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:12.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f813c0b4680 0x7f813c1592d0 unknown :-1 s=CLOSED pgs=1628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:12.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 --2- 172.21.15.86:0/3406797951 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f813c072340 0x7f813c14e0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:12.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.024+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 >> 172.21.15.86:0/3406797951 conn(0x7f813c0b1730 msgr2=0x7f813c0b0600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:12.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.025+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 shutdown_connections 2024-03-06T07:36:12.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:12.025+0000 7f8143600640 1 -- 172.21.15.86:0/3406797951 wait complete. 2024-03-06T07:36:12.247 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:12.247 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:13.248 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:13.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.461+0000 7fc330800640 1 Processor -- start 2024-03-06T07:36:13.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.462+0000 7fc330800640 1 -- start start 2024-03-06T07:36:13.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.463+0000 7fc330800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 0x7fc32815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.463+0000 7fc330800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc328070b10 0x7fc328070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.463+0000 7fc330800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc32806cef0 con 0x7fc328144c00 2024-03-06T07:36:13.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.463+0000 7fc330800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc328160370 con 0x7fc328070b10 2024-03-06T07:36:13.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.463+0000 7fc330800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc32806d1d0 con 0x7fc32815f9d0 2024-03-06T07:36:13.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.463+0000 7fc32de00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc328144c00 0x7fc32806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:48168/0 (socket says 172.21.15.86:48168) 2024-03-06T07:36:13.465 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.463+0000 7fc32de00640 1 -- 172.21.15.86:0/2755654304 learned_addr learned my addr 172.21.15.86:0/2755654304 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:13.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32d400640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 0x7fc32815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32e800640 1 --2- 172.21.15.86:0/2755654304 >> v2:172.21.15.194:3300/0 conn(0x7fc328070b10 0x7fc328070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 116802923 0 0) 0x7fc32806cef0 con 0x7fc328144c00 2024-03-06T07:36:13.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc310003520 con 0x7fc328144c00 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 57631434 0 0) 0x7fc310003520 con 0x7fc328144c00 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 msgr2=0x7fc32815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 0x7fc32815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 >> v2:172.21.15.194:3300/0 conn(0x7fc328070b10 msgr2=0x7fc328070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 --2- 172.21.15.86:0/2755654304 >> v2:172.21.15.194:3300/0 conn(0x7fc328070b10 0x7fc328070f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc32806d4b0 con 0x7fc328144c00 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.464+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc31c001670 con 0x7fc328144c00 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 >> v1:172.21.15.86:6789/0 conn(0x7fc328144c00 legacy=0x7fc32806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32ca00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3100039b0 0x7fc310003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32ca00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 0x7fc310004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32ca00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc310004d40 0x7fc3100051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc32806d1d0 con 0x7fc3100039b0 2024-03-06T07:36:13.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc31000b9c0 con 0x7fc310004d40 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc31000bb40 con 0x7fc32815f9d0 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32de00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3100039b0 0x7fc310003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32d400640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 0x7fc310004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32de00640 1 -- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 msgr2=0x7fc310004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32e800640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc310004d40 0x7fc3100051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32de00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc32815f9d0 0x7fc310004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32de00640 1 -- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc310004d40 msgr2=0x7fc3100051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32de00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc310004d40 0x7fc3100051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.465+0000 7fc32de00640 1 -- 172.21.15.86:0/2755654304 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc310003520 con 0x7fc3100039b0 2024-03-06T07:36:13.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.466+0000 7fc32de00640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3100039b0 0x7fc310003dd0 secure :-1 s=READY pgs=1630 cs=0 l=1 rev1=1 crypto rx=0x7fc31c00c240 tx=0x7fc310005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be711e7aef4e9117 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:13.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc31c007d50 con 0x7fc3100039b0 2024-03-06T07:36:13.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc31c030070 con 0x7fc3100039b0 2024-03-06T07:36:13.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc32ca00640 1 -- 172.21.15.86:0/2755654304 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc31c02cd80 con 0x7fc3100039b0 2024-03-06T07:36:13.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc330800640 1 -- 172.21.15.86:0/2755654304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3100039b0 msgr2=0x7fc310003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:13.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc330800640 1 --2- 172.21.15.86:0/2755654304 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3100039b0 0x7fc310003dd0 secure :-1 s=READY pgs=1630 cs=0 l=1 rev1=1 crypto rx=0x7fc31c00c240 tx=0x7fc310005830 comp rx=0 tx=0).stop 2024-03-06T07:36:13.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc32e800640 1 -- 172.21.15.86:0/2755654304 reap_dead start 2024-03-06T07:36:13.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc330800640 1 -- 172.21.15.86:0/2755654304 shutdown_connections 2024-03-06T07:36:13.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc330800640 1 -- 172.21.15.86:0/2755654304 >> 172.21.15.86:0/2755654304 conn(0x7fc3280b1730 msgr2=0x7fc3280b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:13.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.467+0000 7fc330800640 1 -- 172.21.15.86:0/2755654304 shutdown_connections 2024-03-06T07:36:13.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.468+0000 7fc330800640 1 -- 172.21.15.86:0/2755654304 wait complete. 2024-03-06T07:36:13.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.469+0000 7fc330800640 1 Processor -- start 2024-03-06T07:36:13.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.469+0000 7fc330800640 1 -- start start 2024-03-06T07:36:13.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.470+0000 7fc330800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3100039b0 0x7fc32813ee50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.470+0000 7fc330800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc32815b700 0x7fc32813e0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.470+0000 7fc330800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3280b2b30 0x7fc32815ded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.470+0000 7fc330800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc32806cb50 con 0x7fc3280b2b30 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.470+0000 7fc330800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc328160730 con 0x7fc32815b700 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.470+0000 7fc330800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc32806ccd0 con 0x7fc3100039b0 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.470+0000 7fc32e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3280b2b30 0x7fc32815ded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3280b2b30 0x7fc32815ded0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56974/0 (socket says 172.21.15.86:56974) 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 -- 172.21.15.86:0/2501359268 learned_addr learned my addr 172.21.15.86:0/2501359268 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32de00640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3100039b0 0x7fc32813ee50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32d400640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc32815b700 0x7fc32813e0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 -- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3100039b0 msgr2=0x7fc32813ee50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3100039b0 0x7fc32813ee50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 -- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc32815b700 msgr2=0x7fc32813e0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc32815b700 0x7fc32813e0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 -- 172.21.15.86:0/2501359268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc32813e5e0 con 0x7fc3280b2b30 2024-03-06T07:36:13.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32de00640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3100039b0 0x7fc32813ee50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:13.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.471+0000 7fc32e800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3280b2b30 0x7fc32815ded0 secure :-1 s=READY pgs=1631 cs=0 l=1 rev1=1 crypto rx=0x7fc3280d0ac0 tx=0x7fc32400f4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:13.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.472+0000 7fc322000640 1 -- 172.21.15.86:0/2501359268 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc32401c070 con 0x7fc3280b2b30 2024-03-06T07:36:13.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.472+0000 7fc322000640 1 -- 172.21.15.86:0/2501359268 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc324017400 con 0x7fc3280b2b30 2024-03-06T07:36:13.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.472+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3280ad860 con 0x7fc3280b2b30 2024-03-06T07:36:13.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.472+0000 7fc322000640 1 -- 172.21.15.86:0/2501359268 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3240177f0 con 0x7fc3280b2b30 2024-03-06T07:36:13.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.472+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3280adb20 con 0x7fc3280b2b30 2024-03-06T07:36:13.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.473+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc32804f330 con 0x7fc3280b2b30 2024-03-06T07:36:13.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.474+0000 7fc322000640 1 -- 172.21.15.86:0/2501359268 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc32401f070 con 0x7fc3280b2b30 2024-03-06T07:36:13.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.475+0000 7fc322000640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc30c06b500 0x7fc30c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:13.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.475+0000 7fc322000640 1 -- 172.21.15.86:0/2501359268 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc32408a280 con 0x7fc3280b2b30 2024-03-06T07:36:13.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.475+0000 7fc32de00640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc30c06b500 0x7fc30c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:13.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.476+0000 7fc32de00640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc30c06b500 0x7fc30c06d9c0 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7fc31c034000 tx=0x7fc31c007000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:13.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.480+0000 7fc322000640 1 -- 172.21.15.86:0/2501359268 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc32813e5e0 con 0x7fc3280b2b30 2024-03-06T07:36:13.585 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:13.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.715+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc3280cedd0 con 0x7fc3280b2b30 2024-03-06T07:36:13.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.716+0000 7fc322000640 1 -- 172.21.15.86:0/2501359268 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fc324012070 con 0x7fc3280b2b30 2024-03-06T07:36:13.718 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:13.718 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:13.718 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:13.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.718+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc30c06b500 msgr2=0x7fc30c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.718+0000 7fc330800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc30c06b500 0x7fc30c06d9c0 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7fc31c034000 tx=0x7fc31c007000 comp rx=0 tx=0).stop 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.718+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3280b2b30 msgr2=0x7fc32815ded0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.718+0000 7fc330800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3280b2b30 0x7fc32815ded0 secure :-1 s=READY pgs=1631 cs=0 l=1 rev1=1 crypto rx=0x7fc3280d0ac0 tx=0x7fc32400f4a0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 shutdown_connections 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc30c06b500 0x7fc30c06d9c0 unknown :-1 s=CLOSED pgs=695 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc3280b2b30 0x7fc32815ded0 unknown :-1 s=CLOSED pgs=1631 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc32815b700 0x7fc32813e0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 --2- 172.21.15.86:0/2501359268 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc3100039b0 0x7fc32813ee50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:13.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 >> 172.21.15.86:0/2501359268 conn(0x7fc3280b1730 msgr2=0x7fc328145040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:13.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 shutdown_connections 2024-03-06T07:36:13.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:13.719+0000 7fc330800640 1 -- 172.21.15.86:0/2501359268 wait complete. 2024-03-06T07:36:13.939 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:13.939 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:14.940 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:15.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.155+0000 7f9270a00640 1 Processor -- start 2024-03-06T07:36:15.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.155+0000 7f9270a00640 1 -- start start 2024-03-06T07:36:15.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f9270a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f926c0aedc0 0x7f926c0cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f9270a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f926c15c240 con 0x7f926c0cc800 2024-03-06T07:36:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f9270a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f926c15c500 con 0x7f926c1561f0 2024-03-06T07:36:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f9270a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f926c15ce30 con 0x7f926c0aedc0 2024-03-06T07:36:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f9262a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f926c0aedc0 0x7f926c0cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f926be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f926c0cc800 0x7f926c15f830 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:48172/0 (socket says 172.21.15.86:48172) 2024-03-06T07:36:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f9262a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f926c0aedc0 0x7f926c0cc2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56978/0 (socket says 172.21.15.86:56978) 2024-03-06T07:36:15.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.157+0000 7f926be00640 1 -- 172.21.15.86:0/1429904186 learned_addr learned my addr 172.21.15.86:0/1429904186 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.158+0000 7f9262a00640 1 -- 172.21.15.86:0/1429904186 >> v1:172.21.15.194:6789/0 conn(0x7f926c1561f0 legacy=0x7f926c0ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.158+0000 7f9262a00640 1 -- 172.21.15.86:0/1429904186 >> v1:172.21.15.86:6789/0 conn(0x7f926c0cc800 legacy=0x7f926c15f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.158+0000 7f9262a00640 1 -- 172.21.15.86:0/1429904186 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f926c15c7e0 con 0x7f926c0aedc0 2024-03-06T07:36:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.158+0000 7f926aa00640 1 -- 172.21.15.86:0/1429904186 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 500287781 0 0) 0x7f926c15c240 con 0x7f926c0cc800 2024-03-06T07:36:15.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.159+0000 7f9262a00640 1 --2- 172.21.15.86:0/1429904186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c0aedc0 0x7f926c0cc2c0 secure :-1 s=READY pgs=1633 cs=0 l=1 rev1=1 crypto rx=0x7f926c15cfb0 tx=0x7f925402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44fd4140899a2b82 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.159+0000 7f926aa00640 1 -- 172.21.15.86:0/1429904186 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9254034070 con 0x7f926c0aedc0 2024-03-06T07:36:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.159+0000 7f926aa00640 1 -- 172.21.15.86:0/1429904186 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9254030610 con 0x7f926c0aedc0 2024-03-06T07:36:15.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.159+0000 7f926aa00640 1 -- 172.21.15.86:0/1429904186 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f92540309e0 con 0x7f926c0aedc0 2024-03-06T07:36:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.160+0000 7f9270a00640 1 -- 172.21.15.86:0/1429904186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c0aedc0 msgr2=0x7f926c0cc2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.160+0000 7f9270a00640 1 --2- 172.21.15.86:0/1429904186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c0aedc0 0x7f926c0cc2c0 secure :-1 s=READY pgs=1633 cs=0 l=1 rev1=1 crypto rx=0x7f926c15cfb0 tx=0x7f925402b960 comp rx=0 tx=0).stop 2024-03-06T07:36:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.160+0000 7f9270a00640 1 -- 172.21.15.86:0/1429904186 shutdown_connections 2024-03-06T07:36:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.160+0000 7f9270a00640 1 --2- 172.21.15.86:0/1429904186 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c0aedc0 0x7f926c0cc2c0 unknown :-1 s=CLOSED pgs=1633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.160+0000 7f9270a00640 1 -- 172.21.15.86:0/1429904186 >> 172.21.15.86:0/1429904186 conn(0x7f926c06afc0 msgr2=0x7f926c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.160+0000 7f9270a00640 1 -- 172.21.15.86:0/1429904186 shutdown_connections 2024-03-06T07:36:15.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.160+0000 7f9270a00640 1 -- 172.21.15.86:0/1429904186 wait complete. 2024-03-06T07:36:15.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.162+0000 7f9270a00640 1 Processor -- start 2024-03-06T07:36:15.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.162+0000 7f9270a00640 1 -- start start 2024-03-06T07:36:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f9270a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0aedc0 0x7f926c1492e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f9270a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c0cc800 0x7f926c1590a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f9270a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c1561f0 0x7f926c15aeb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f9270a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f926c0ccc30 con 0x7f926c1561f0 2024-03-06T07:36:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f9270a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f926c06e360 con 0x7f926c0cc800 2024-03-06T07:36:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f9270a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f926c1565d0 con 0x7f926c0aedc0 2024-03-06T07:36:15.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c1561f0 0x7f926c15aeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c1561f0 0x7f926c15aeb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56988/0 (socket says 172.21.15.86:56988) 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 -- 172.21.15.86:0/2316870653 learned_addr learned my addr 172.21.15.86:0/2316870653 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f9262a00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c0cc800 0x7f926c1590a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926b400640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0aedc0 0x7f926c1492e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 -- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0aedc0 msgr2=0x7f926c1492e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0aedc0 0x7f926c1492e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 -- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c0cc800 msgr2=0x7f926c1590a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c0cc800 0x7f926c1590a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926be00640 1 -- 172.21.15.86:0/2316870653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f926c1f78e0 con 0x7f926c1561f0 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.163+0000 7f926b400640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0aedc0 0x7f926c1492e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:15.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.164+0000 7f926be00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c1561f0 0x7f926c15aeb0 secure :-1 s=READY pgs=1634 cs=0 l=1 rev1=1 crypto rx=0x7f926400c7a0 tx=0x7f926400cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.164+0000 7f9263e00640 1 -- 172.21.15.86:0/2316870653 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9264018070 con 0x7f926c1561f0 2024-03-06T07:36:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.164+0000 7f9263e00640 1 -- 172.21.15.86:0/2316870653 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f926400a640 con 0x7f926c1561f0 2024-03-06T07:36:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.164+0000 7f9263e00640 1 -- 172.21.15.86:0/2316870653 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f926400aab0 con 0x7f926c1561f0 2024-03-06T07:36:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.164+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f926c1f7bc0 con 0x7f926c1561f0 2024-03-06T07:36:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.164+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f926c1f8180 con 0x7f926c1561f0 2024-03-06T07:36:15.167 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.165+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f926c04f330 con 0x7f926c1561f0 2024-03-06T07:36:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.166+0000 7f9263e00640 1 -- 172.21.15.86:0/2316870653 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f926400ac50 con 0x7f926c1561f0 2024-03-06T07:36:15.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.167+0000 7f9263e00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:15.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.167+0000 7f9263e00640 1 -- 172.21.15.86:0/2316870653 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9264089940 con 0x7f926c1561f0 2024-03-06T07:36:15.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.167+0000 7f926b400640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:15.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.168+0000 7f926b400640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f925c000f60 tx=0x7f925c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.172+0000 7f9263e00640 1 -- 172.21.15.86:0/2316870653 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9264058770 con 0x7f926c1561f0 2024-03-06T07:36:15.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.407+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f926c0b2b10 con 0x7f926c1561f0 2024-03-06T07:36:15.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.408+0000 7f9263e00640 1 -- 172.21.15.86:0/2316870653 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f9264089030 con 0x7f926c1561f0 2024-03-06T07:36:15.410 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:15.410 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:15.410 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:15.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.411+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 msgr2=0x7f924806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:15.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.411+0000 7f9270a00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f925c000f60 tx=0x7f925c009040 comp rx=0 tx=0).stop 2024-03-06T07:36:15.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.411+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c1561f0 msgr2=0x7f926c15aeb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:15.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.411+0000 7f9270a00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c1561f0 0x7f926c15aeb0 secure :-1 s=READY pgs=1634 cs=0 l=1 rev1=1 crypto rx=0x7f926400c7a0 tx=0x7f926400cc60 comp rx=0 tx=0).stop 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.411+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 shutdown_connections 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.412+0000 7f9270a00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 unknown :-1 s=CLOSED pgs=696 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.412+0000 7f9270a00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f926c1561f0 0x7f926c15aeb0 unknown :-1 s=CLOSED pgs=1634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.412+0000 7f9270a00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c0cc800 0x7f926c1590a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.412+0000 7f9270a00640 1 --2- 172.21.15.86:0/2316870653 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0aedc0 0x7f926c1492e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.412+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 >> 172.21.15.86:0/2316870653 conn(0x7f926c06afc0 msgr2=0x7f926c1550e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.412+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 shutdown_connections 2024-03-06T07:36:15.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:15.412+0000 7f9270a00640 1 -- 172.21.15.86:0/2316870653 wait complete. 2024-03-06T07:36:15.587 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:15.632 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:15.632 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:16.633 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:16.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.849+0000 7fb454400640 1 Processor -- start 2024-03-06T07:36:16.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.849+0000 7fb454400640 1 -- start start 2024-03-06T07:36:16.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.850+0000 7fb454400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb44c15bfa0 0x7fb44c0cf190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:16.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.850+0000 7fb454400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c0cd240 0x7fb44c0ce580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:16.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.850+0000 7fb454400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb44c0cdc50 con 0x7fb44c0ceac0 2024-03-06T07:36:16.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.850+0000 7fb454400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb44c0b5240 con 0x7fb44c15bfa0 2024-03-06T07:36:16.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.850+0000 7fb454400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb44c0cd970 con 0x7fb44c0cd240 2024-03-06T07:36:16.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb44c15bfa0 0x7fb44c0cf190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:16.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb44c15bfa0 0x7fb44c0cf190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42728/0 (socket says 172.21.15.86:42728) 2024-03-06T07:36:16.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451a00640 1 -- 172.21.15.86:0/2169678733 learned_addr learned my addr 172.21.15.86:0/2169678733 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:16.853 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451000640 1 --2- 172.21.15.86:0/2169678733 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c0cd240 0x7fb44c0ce580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451a00640 1 -- 172.21.15.86:0/2169678733 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c0cd240 msgr2=0x7fb44c0ce580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451a00640 1 --2- 172.21.15.86:0/2169678733 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c0cd240 0x7fb44c0ce580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451a00640 1 -- 172.21.15.86:0/2169678733 >> v1:172.21.15.194:6789/0 conn(0x7fb44c0ceac0 legacy=0x7fb44c0cf860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.851+0000 7fb451a00640 1 -- 172.21.15.86:0/2169678733 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb44c0b4bf0 con 0x7fb44c15bfa0 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.852+0000 7fb451a00640 1 --2- 172.21.15.86:0/2169678733 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c15bfa0 0x7fb44c0cf190 secure :-1 s=READY pgs=1635 cs=0 l=1 rev1=1 crypto rx=0x7fb440009870 tx=0x7fb44002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d0ee1ed83eb3d34 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.853+0000 7fb44be00640 1 -- 172.21.15.86:0/2169678733 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb440035020 con 0x7fb44c15bfa0 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.853+0000 7fb44be00640 1 -- 172.21.15.86:0/2169678733 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb440031640 con 0x7fb44c15bfa0 2024-03-06T07:36:16.854 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.853+0000 7fb44be00640 1 -- 172.21.15.86:0/2169678733 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb44002d040 con 0x7fb44c15bfa0 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 -- 172.21.15.86:0/2169678733 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c15bfa0 msgr2=0x7fb44c0cf190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 --2- 172.21.15.86:0/2169678733 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c15bfa0 0x7fb44c0cf190 secure :-1 s=READY pgs=1635 cs=0 l=1 rev1=1 crypto rx=0x7fb440009870 tx=0x7fb44002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 -- 172.21.15.86:0/2169678733 shutdown_connections 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 --2- 172.21.15.86:0/2169678733 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c0cd240 0x7fb44c0ce580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 --2- 172.21.15.86:0/2169678733 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c15bfa0 0x7fb44c0cf190 unknown :-1 s=CLOSED pgs=1635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 -- 172.21.15.86:0/2169678733 >> 172.21.15.86:0/2169678733 conn(0x7fb44c0d1c90 msgr2=0x7fb44c0d20a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 -- 172.21.15.86:0/2169678733 shutdown_connections 2024-03-06T07:36:16.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.854+0000 7fb454400640 1 -- 172.21.15.86:0/2169678733 wait complete. 2024-03-06T07:36:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.855+0000 7fb454400640 1 Processor -- start 2024-03-06T07:36:16.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.856+0000 7fb454400640 1 -- start start 2024-03-06T07:36:16.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.856+0000 7fb454400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c0cd240 0x7fb44c14f220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:16.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb454400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb44c0ceac0 0x7fb44c0cb560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:16.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb454400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c15bfa0 0x7fb44c15e0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:16.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb454400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb44c15c380 con 0x7fb44c0cd240 2024-03-06T07:36:16.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb454400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb44c0cd680 con 0x7fb44c0ceac0 2024-03-06T07:36:16.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb454400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb44c0b66e0 con 0x7fb44c15bfa0 2024-03-06T07:36:16.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb451a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c0cd240 0x7fb44c14f220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:16.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb451a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c0cd240 0x7fb44c14f220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42740/0 (socket says 172.21.15.86:42740) 2024-03-06T07:36:16.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb451a00640 1 -- 172.21.15.86:0/3566620203 learned_addr learned my addr 172.21.15.86:0/3566620203 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:16.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.857+0000 7fb451000640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb44c0ceac0 0x7fb44c0cb560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:16.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb451a00640 1 -- 172.21.15.86:0/3566620203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c15bfa0 msgr2=0x7fb44c15e0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:16.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb451a00640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c15bfa0 0x7fb44c15e0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:16.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb451a00640 1 -- 172.21.15.86:0/3566620203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb44c0ceac0 msgr2=0x7fb44c0cb560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:16.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb451a00640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb44c0ceac0 0x7fb44c0cb560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:16.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb451a00640 1 -- 172.21.15.86:0/3566620203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb44c0ca4c0 con 0x7fb44c0cd240 2024-03-06T07:36:16.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb451a00640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c0cd240 0x7fb44c14f220 secure :-1 s=READY pgs=1636 cs=0 l=1 rev1=1 crypto rx=0x7fb44002b930 tx=0x7fb4400046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:16.860 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb449600640 1 -- 172.21.15.86:0/3566620203 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb44003ac60 con 0x7fb44c0cd240 2024-03-06T07:36:16.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb449600640 1 -- 172.21.15.86:0/3566620203 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb44003ae20 con 0x7fb44c0cd240 2024-03-06T07:36:16.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb449600640 1 -- 172.21.15.86:0/3566620203 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb440044630 con 0x7fb44c0cd240 2024-03-06T07:36:16.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.858+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb44c0ca7a0 con 0x7fb44c0cd240 2024-03-06T07:36:16.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.859+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb44c0cad60 con 0x7fb44c0cd240 2024-03-06T07:36:16.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.860+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb44c0ccff0 con 0x7fb44c0cd240 2024-03-06T07:36:16.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.861+0000 7fb449600640 1 -- 172.21.15.86:0/3566620203 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb44004b060 con 0x7fb44c0cd240 2024-03-06T07:36:16.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.862+0000 7fb449600640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb42c06b4d0 0x7fb42c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:16.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.862+0000 7fb449600640 1 -- 172.21.15.86:0/3566620203 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb4400ac050 con 0x7fb44c0cd240 2024-03-06T07:36:16.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.862+0000 7fb451000640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb42c06b4d0 0x7fb42c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:16.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.863+0000 7fb451000640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb42c06b4d0 0x7fb42c06d990 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7fb44c1585c0 tx=0x7fb438008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:16.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:16.867+0000 7fb449600640 1 -- 172.21.15.86:0/3566620203 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb44c0ccff0 con 0x7fb44c0cd240 2024-03-06T07:36:17.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.117+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb44c0684a0 con 0x7fb44c0cd240 2024-03-06T07:36:17.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.119+0000 7fb449600640 1 -- 172.21.15.86:0/3566620203 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fb44003f070 con 0x7fb44c0cd240 2024-03-06T07:36:17.121 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:17.121 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:17.121 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb42c06b4d0 msgr2=0x7fb42c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb42c06b4d0 0x7fb42c06d990 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7fb44c1585c0 tx=0x7fb438008040 comp rx=0 tx=0).stop 2024-03-06T07:36:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c0cd240 msgr2=0x7fb44c14f220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c0cd240 0x7fb44c14f220 secure :-1 s=READY pgs=1636 cs=0 l=1 rev1=1 crypto rx=0x7fb44002b930 tx=0x7fb4400046c0 comp rx=0 tx=0).stop 2024-03-06T07:36:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 shutdown_connections 2024-03-06T07:36:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb42c06b4d0 0x7fb42c06d990 unknown :-1 s=CLOSED pgs=697 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:17.124 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb44c15bfa0 0x7fb44c15e0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:17.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb44c0ceac0 0x7fb44c0cb560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:17.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 --2- 172.21.15.86:0/3566620203 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb44c0cd240 0x7fb44c14f220 unknown :-1 s=CLOSED pgs=1636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:17.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 >> 172.21.15.86:0/3566620203 conn(0x7fb44c0d1c90 msgr2=0x7fb44c0b7280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:17.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 shutdown_connections 2024-03-06T07:36:17.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:17.122+0000 7fb454400640 1 -- 172.21.15.86:0/3566620203 wait complete. 2024-03-06T07:36:17.343 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:17.343 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:17.590 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:18.344 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.556+0000 7f2123200640 1 Processor -- start 2024-03-06T07:36:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.556+0000 7f2123200640 1 -- start start 2024-03-06T07:36:18.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.558+0000 7f2123200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c144c00 0x7f211c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.558+0000 7f2123200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f211c06ee70 con 0x7f211c15f9d0 2024-03-06T07:36:18.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.558+0000 7f2123200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f211c06cef0 con 0x7f211c15fe30 2024-03-06T07:36:18.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.558+0000 7f2123200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f211c06d1d0 con 0x7f211c144c00 2024-03-06T07:36:18.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.558+0000 7f2113400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f211c15f9d0 0x7f211c0ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54362/0 (socket says 172.21.15.86:54362) 2024-03-06T07:36:18.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.558+0000 7f2113400640 1 -- 172.21.15.86:0/2923991534 learned_addr learned my addr 172.21.15.86:0/2923991534 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211be00640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c144c00 0x7f211c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 514798299 0 0) 0x7f2104001110 con 0x7f211c15f9d0 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2100003520 con 0x7f211c15f9d0 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3095215705 0 0) 0x7f2100003520 con 0x7f211c15f9d0 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c144c00 msgr2=0x7f211c06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c144c00 0x7f211c06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 >> v1:172.21.15.194:6789/0 conn(0x7f211c15fe30 legacy=0x7f211c0ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f211c06d4b0 con 0x7f211c15f9d0 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f2104001840 con 0x7f211c15f9d0 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 >> v1:172.21.15.86:6789/0 conn(0x7f211c15f9d0 legacy=0x7f211c0ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21000039e0 0x7f2100003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2100004800 0x7f2100004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c144c00 0x7f21000051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f211c06d1d0 con 0x7f21000039e0 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f210000ba70 con 0x7f2100004800 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.559+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f210000bc70 con 0x7f211c144c00 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f211be00640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21000039e0 0x7f2100003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f2113400640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2100004800 0x7f2100004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f211be00640 1 -- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c144c00 msgr2=0x7f21000051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f211be00640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c144c00 0x7f21000051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f211be00640 1 -- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2100004800 msgr2=0x7f2100004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f211be00640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2100004800 0x7f2100004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f211be00640 1 -- 172.21.15.86:0/2923991534 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2100003520 con 0x7f21000039e0 2024-03-06T07:36:18.563 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.560+0000 7f2113400640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2100004800 0x7f2100004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.561+0000 7f211be00640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21000039e0 0x7f2100003e00 secure :-1 s=READY pgs=1638 cs=0 l=1 rev1=1 crypto rx=0x7f210c009f90 tx=0x7f2100005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad6ae07351c2baf6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.561+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f210c039070 con 0x7f21000039e0 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.561+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f210c02cbf0 con 0x7f21000039e0 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.561+0000 7f211b400640 1 -- 172.21.15.86:0/2923991534 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f210c009c10 con 0x7f21000039e0 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.562+0000 7f2123200640 1 -- 172.21.15.86:0/2923991534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21000039e0 msgr2=0x7f2100003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.562+0000 7f2123200640 1 --2- 172.21.15.86:0/2923991534 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21000039e0 0x7f2100003e00 secure :-1 s=READY pgs=1638 cs=0 l=1 rev1=1 crypto rx=0x7f210c009f90 tx=0x7f2100005830 comp rx=0 tx=0).stop 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.562+0000 7f2121200640 1 -- 172.21.15.86:0/2923991534 reap_dead start 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.563+0000 7f2123200640 1 -- 172.21.15.86:0/2923991534 shutdown_connections 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.563+0000 7f2123200640 1 -- 172.21.15.86:0/2923991534 >> 172.21.15.86:0/2923991534 conn(0x7f211c0b1730 msgr2=0x7f211c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.563+0000 7f2123200640 1 -- 172.21.15.86:0/2923991534 shutdown_connections 2024-03-06T07:36:18.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.563+0000 7f2123200640 1 -- 172.21.15.86:0/2923991534 wait complete. 2024-03-06T07:36:18.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.564+0000 7f2123200640 1 Processor -- start 2024-03-06T07:36:18.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.565+0000 7f2123200640 1 -- start start 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.565+0000 7f2123200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21000039e0 0x7f211c1f0710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.565+0000 7f2123200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c1f1270 0x7f211c15c810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2123200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f211c15e820 0x7f211c15ecb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2123200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f211c160730 con 0x7f211c15e820 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2123200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f211c160280 con 0x7f21000039e0 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2123200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f211c0ceec0 con 0x7f211c1f1270 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f211c15e820 0x7f211c15ecb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:18.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f211c15e820 0x7f211c15ecb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42758/0 (socket says 172.21.15.86:42758) 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 -- 172.21.15.86:0/3505571226 learned_addr learned my addr 172.21.15.86:0/3505571226 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f211be00640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21000039e0 0x7f211c1f0710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2113400640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c1f1270 0x7f211c15c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 -- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c1f1270 msgr2=0x7f211c15c810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c1f1270 0x7f211c15c810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 -- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21000039e0 msgr2=0x7f211c1f0710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21000039e0 0x7f211c1f0710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2121200640 1 -- 172.21.15.86:0/3505571226 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f211c15cf10 con 0x7f211c15e820 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.566+0000 7f2113400640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c1f1270 0x7f211c15c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:18.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.567+0000 7f2121200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f211c15e820 0x7f211c15ecb0 secure :-1 s=READY pgs=1639 cs=0 l=1 rev1=1 crypto rx=0x7f211c160210 tx=0x7f211400c820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:18.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.567+0000 7f2118c00640 1 -- 172.21.15.86:0/3505571226 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2114004220 con 0x7f211c15e820 2024-03-06T07:36:18.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.567+0000 7f2118c00640 1 -- 172.21.15.86:0/3505571226 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f21140043e0 con 0x7f211c15e820 2024-03-06T07:36:18.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.567+0000 7f2118c00640 1 -- 172.21.15.86:0/3505571226 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2114007310 con 0x7f211c15e820 2024-03-06T07:36:18.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.567+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f211c15d1f0 con 0x7f211c15e820 2024-03-06T07:36:18.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.567+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f211c15d7b0 con 0x7f211c15e820 2024-03-06T07:36:18.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.568+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f211c0cd9a0 con 0x7f211c15e820 2024-03-06T07:36:18.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.569+0000 7f2118c00640 1 -- 172.21.15.86:0/3505571226 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f21140074b0 con 0x7f211c15e820 2024-03-06T07:36:18.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.570+0000 7f2118c00640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20f806b500 0x7f20f806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:18.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.570+0000 7f2118c00640 1 -- 172.21.15.86:0/3505571226 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f2114089450 con 0x7f211c15e820 2024-03-06T07:36:18.572 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.571+0000 7f211be00640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20f806b500 0x7f20f806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:18.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.571+0000 7f211be00640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20f806b500 0x7f20f806d9c0 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f210c036000 tx=0x7f210c02e000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:18.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.575+0000 7f2118c00640 1 -- 172.21.15.86:0/3505571226 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f211c0cd9a0 con 0x7f211c15e820 2024-03-06T07:36:18.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.814+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f211c04f330 con 0x7f211c15e820 2024-03-06T07:36:18.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.815+0000 7f2118c00640 1 -- 172.21.15.86:0/3505571226 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f21140581f0 con 0x7f211c15e820 2024-03-06T07:36:18.817 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:18.817 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:18.817 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:18.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20f806b500 msgr2=0x7f20f806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:18.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20f806b500 0x7f20f806d9c0 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f210c036000 tx=0x7f210c02e000 comp rx=0 tx=0).stop 2024-03-06T07:36:18.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f211c15e820 msgr2=0x7f211c15ecb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:18.819 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f211c15e820 0x7f211c15ecb0 secure :-1 s=READY pgs=1639 cs=0 l=1 rev1=1 crypto rx=0x7f211c160210 tx=0x7f211400c820 comp rx=0 tx=0).stop 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 shutdown_connections 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20f806b500 0x7f20f806d9c0 unknown :-1 s=CLOSED pgs=698 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f211c15e820 0x7f211c15ecb0 unknown :-1 s=CLOSED pgs=1639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f211c1f1270 0x7f211c15c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 --2- 172.21.15.86:0/3505571226 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21000039e0 0x7f211c1f0710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.817+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 >> 172.21.15.86:0/3505571226 conn(0x7f211c0b1730 msgr2=0x7f211c071ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.818+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 shutdown_connections 2024-03-06T07:36:18.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:18.818+0000 7f2123200640 1 -- 172.21.15.86:0/3505571226 wait complete. 2024-03-06T07:36:19.037 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:19.038 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:19.592 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:20.038 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:20.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.243+0000 7f398ca00640 1 Processor -- start 2024-03-06T07:36:20.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.243+0000 7f398ca00640 1 -- start start 2024-03-06T07:36:20.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.245+0000 7f398ca00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f39880aedc0 0x7f39880cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:20.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.245+0000 7f398ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39880cc800 0x7f398815f830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:20.246 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.245+0000 7f398ca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f398815c240 con 0x7f39881561f0 2024-03-06T07:36:20.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.245+0000 7f398ca00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f398815ce30 con 0x7f39880aedc0 2024-03-06T07:36:20.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.245+0000 7f398ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f398815c500 con 0x7f39880cc800 2024-03-06T07:36:20.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.245+0000 7f3985e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f39880aedc0 0x7f39880cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:20.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.245+0000 7f3987200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39880cc800 0x7f398815f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:20.247 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.246+0000 7f3987200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39880cc800 0x7f398815f830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:48162/0 (socket says 172.21.15.86:48162) 2024-03-06T07:36:20.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.246+0000 7f3987200640 1 -- 172.21.15.86:0/2177227613 learned_addr learned my addr 172.21.15.86:0/2177227613 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:20.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.246+0000 7f3987200640 1 -- 172.21.15.86:0/2177227613 >> v2:172.21.15.194:3300/0 conn(0x7f39880aedc0 msgr2=0x7f39880cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:20.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.246+0000 7f3987200640 1 --2- 172.21.15.86:0/2177227613 >> v2:172.21.15.194:3300/0 conn(0x7f39880aedc0 0x7f39880cc2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.246+0000 7f3987200640 1 -- 172.21.15.86:0/2177227613 >> v1:172.21.15.194:6789/0 conn(0x7f39881561f0 legacy=0x7f39880ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:20.248 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.246+0000 7f3987200640 1 -- 172.21.15.86:0/2177227613 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f398815c7e0 con 0x7f39880cc800 2024-03-06T07:36:20.249 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.248+0000 7f3987200640 1 --2- 172.21.15.86:0/2177227613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39880cc800 0x7f398815f830 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f3980009870 tx=0x7f398002b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b173896a6ab9fb7f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:20.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.248+0000 7f3985400640 1 -- 172.21.15.86:0/2177227613 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3980034070 con 0x7f39880cc800 2024-03-06T07:36:20.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.248+0000 7f3985400640 1 -- 172.21.15.86:0/2177227613 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3980030840 con 0x7f39880cc800 2024-03-06T07:36:20.250 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.248+0000 7f3985400640 1 -- 172.21.15.86:0/2177227613 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3980030c10 con 0x7f39880cc800 2024-03-06T07:36:20.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 -- 172.21.15.86:0/2177227613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39880cc800 msgr2=0x7f398815f830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:20.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 --2- 172.21.15.86:0/2177227613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39880cc800 0x7f398815f830 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f3980009870 tx=0x7f398002b980 comp rx=0 tx=0).stop 2024-03-06T07:36:20.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 -- 172.21.15.86:0/2177227613 shutdown_connections 2024-03-06T07:36:20.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 --2- 172.21.15.86:0/2177227613 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39880cc800 0x7f398815f830 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 --2- 172.21.15.86:0/2177227613 >> v2:172.21.15.194:3300/0 conn(0x7f39880aedc0 0x7f39880cc2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 -- 172.21.15.86:0/2177227613 >> 172.21.15.86:0/2177227613 conn(0x7f398806afc0 msgr2=0x7f398806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:20.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 -- 172.21.15.86:0/2177227613 shutdown_connections 2024-03-06T07:36:20.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.249+0000 7f398ca00640 1 -- 172.21.15.86:0/2177227613 wait complete. 2024-03-06T07:36:20.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.251+0000 7f398ca00640 1 Processor -- start 2024-03-06T07:36:20.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.251+0000 7f398ca00640 1 -- start start 2024-03-06T07:36:20.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f398ca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39880aedc0 0x7f39881535f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:20.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f398ca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39880cc800 0x7f398815d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:20.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f398ca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39881561f0 0x7f398815d920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:20.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f398ca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39881565d0 con 0x7f39880aedc0 2024-03-06T07:36:20.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f398ca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39880ccc70 con 0x7f39880cc800 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f398ca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3988156750 con 0x7f39881561f0 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f3986800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39880aedc0 0x7f39881535f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f3986800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39880aedc0 0x7f39881535f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42766/0 (socket says 172.21.15.86:42766) 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.252+0000 7f3986800640 1 -- 172.21.15.86:0/1457514945 learned_addr learned my addr 172.21.15.86:0/1457514945 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3987200640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39881561f0 0x7f398815d920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3985e00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39880cc800 0x7f398815d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3986800640 1 -- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39881561f0 msgr2=0x7f398815d920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3986800640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39881561f0 0x7f398815d920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3986800640 1 -- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39880cc800 msgr2=0x7f398815d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3986800640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39880cc800 0x7f398815d3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3986800640 1 -- 172.21.15.86:0/1457514945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39881568d0 con 0x7f39880aedc0 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3985e00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39880cc800 0x7f398815d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3987200640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39881561f0 0x7f398815d920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f3986800640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39880aedc0 0x7f39881535f0 secure :-1 s=READY pgs=1640 cs=0 l=1 rev1=1 crypto rx=0x7f397800c730 tx=0x7f397800cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f397ea00640 1 -- 172.21.15.86:0/1457514945 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3978016020 con 0x7f39880aedc0 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f397ea00640 1 -- 172.21.15.86:0/1457514945 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3978006d50 con 0x7f39880aedc0 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f397ea00640 1 -- 172.21.15.86:0/1457514945 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3978003bb0 con 0x7f39880aedc0 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.253+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3988156bb0 con 0x7f39880aedc0 2024-03-06T07:36:20.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.254+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3988157170 con 0x7f39880aedc0 2024-03-06T07:36:20.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.255+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f398804f330 con 0x7f39880aedc0 2024-03-06T07:36:20.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.256+0000 7f397ea00640 1 -- 172.21.15.86:0/1457514945 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f397800a850 con 0x7f39880aedc0 2024-03-06T07:36:20.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.256+0000 7f397ea00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f396406b4d0 0x7f396406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:20.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.257+0000 7f397ea00640 1 -- 172.21.15.86:0/1457514945 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3978089ef0 con 0x7f39880aedc0 2024-03-06T07:36:20.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.257+0000 7f3985e00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f396406b4d0 0x7f396406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:20.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.257+0000 7f3985e00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f396406b4d0 0x7f396406d990 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7f3988072650 tx=0x7f3970008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:20.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.261+0000 7f397ea00640 1 -- 172.21.15.86:0/1457514945 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3978020020 con 0x7f39880aedc0 2024-03-06T07:36:20.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.494+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3988159bc0 con 0x7f39880aedc0 2024-03-06T07:36:20.496 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.495+0000 7f397ea00640 1 -- 172.21.15.86:0/1457514945 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f3978019310 con 0x7f39880aedc0 2024-03-06T07:36:20.497 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:20.497 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:20.497 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:20.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.497+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f396406b4d0 msgr2=0x7f396406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f396406b4d0 0x7f396406d990 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7f3988072650 tx=0x7f3970008040 comp rx=0 tx=0).stop 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39880aedc0 msgr2=0x7f39881535f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39880aedc0 0x7f39881535f0 secure :-1 s=READY pgs=1640 cs=0 l=1 rev1=1 crypto rx=0x7f397800c730 tx=0x7f397800cbf0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 shutdown_connections 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f396406b4d0 0x7f396406d990 unknown :-1 s=CLOSED pgs=699 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f39881561f0 0x7f398815d920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f39880cc800 0x7f398815d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 --2- 172.21.15.86:0/1457514945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f39880aedc0 0x7f39881535f0 unknown :-1 s=CLOSED pgs=1640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 >> 172.21.15.86:0/1457514945 conn(0x7f398806afc0 msgr2=0x7f3988155450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:20.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 shutdown_connections 2024-03-06T07:36:20.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:20.498+0000 7f398ca00640 1 -- 172.21.15.86:0/1457514945 wait complete. 2024-03-06T07:36:20.720 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:20.720 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:21.593 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:21.721 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:21.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.928+0000 7f66e8c00640 1 Processor -- start 2024-03-06T07:36:21.930 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.928+0000 7f66e8c00640 1 -- start start 2024-03-06T07:36:21.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.929+0000 7f66e8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 0x7f66e415fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.929+0000 7f66e8c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f66e4070b10 0x7f66e4070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.929+0000 7f66e8c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66e406cef0 con 0x7f66e4144c00 2024-03-06T07:36:21.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.929+0000 7f66e8c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f66e4160370 con 0x7f66e4070b10 2024-03-06T07:36:21.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.929+0000 7f66e8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66e406d1d0 con 0x7f66e415f9d0 2024-03-06T07:36:21.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.930+0000 7f66e1e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f66e4144c00 0x7f66e406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54370/0 (socket says 172.21.15.86:54370) 2024-03-06T07:36:21.931 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.930+0000 7f66e1e00640 1 -- 172.21.15.86:0/3148058752 learned_addr learned my addr 172.21.15.86:0/3148058752 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:21.933 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.930+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1293381987 0 0) 0x7f66e406cef0 con 0x7f66e4144c00 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.930+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f66c8003520 con 0x7f66e4144c00 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.930+0000 7f66e2800640 1 --2- 172.21.15.86:0/3148058752 >> v2:172.21.15.194:3300/0 conn(0x7f66e4070b10 0x7f66e4070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.930+0000 7f66d9400640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 0x7f66e415fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.930+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2353697276 0 0) 0x7f66c8003520 con 0x7f66e4144c00 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 msgr2=0x7f66e415fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 0x7f66e415fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 >> v2:172.21.15.194:3300/0 conn(0x7f66e4070b10 msgr2=0x7f66e4070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 --2- 172.21.15.86:0/3148058752 >> v2:172.21.15.194:3300/0 conn(0x7f66e4070b10 0x7f66e4070f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66e406d4b0 con 0x7f66e4144c00 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f66d40019f0 con 0x7f66e4144c00 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 >> v1:172.21.15.86:6789/0 conn(0x7f66e4144c00 legacy=0x7f66e406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66c8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8004800 0x7f66c8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 0x7f66c80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66e406d1d0 con 0x7f66c80039b0 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66c800ba70 con 0x7f66c8004800 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66c800bc70 con 0x7f66e415f9d0 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e1e00640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66c8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.931+0000 7f66e2800640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 0x7f66c80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.932+0000 7f66d9400640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8004800 0x7f66c8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.932+0000 7f66e1e00640 1 -- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 msgr2=0x7f66c80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.932+0000 7f66e1e00640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e415f9d0 0x7f66c80051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.932+0000 7f66e1e00640 1 -- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8004800 msgr2=0x7f66c8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.932+0000 7f66e1e00640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8004800 0x7f66c8004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.932+0000 7f66e1e00640 1 -- 172.21.15.86:0/3148058752 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66c8003520 con 0x7f66c80039b0 2024-03-06T07:36:21.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.932+0000 7f66d9400640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66c8004800 0x7f66c8004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:21.936 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.933+0000 7f66e1e00640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66c8003dd0 secure :-1 s=READY pgs=1642 cs=0 l=1 rev1=1 crypto rx=0x7f66d400a7a0 tx=0x7f66c8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7ae55be4645443db server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.933+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66d400a3f0 con 0x7f66c80039b0 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66d4034630 con 0x7f66c80039b0 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e1400640 1 -- 172.21.15.86:0/3148058752 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66d4034a00 con 0x7f66c80039b0 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e8c00640 1 -- 172.21.15.86:0/3148058752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 msgr2=0x7f66c8003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e8c00640 1 --2- 172.21.15.86:0/3148058752 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66c8003dd0 secure :-1 s=READY pgs=1642 cs=0 l=1 rev1=1 crypto rx=0x7f66d400a7a0 tx=0x7f66c8005830 comp rx=0 tx=0).stop 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e2800640 1 -- 172.21.15.86:0/3148058752 reap_dead start 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e8c00640 1 -- 172.21.15.86:0/3148058752 shutdown_connections 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e8c00640 1 -- 172.21.15.86:0/3148058752 >> 172.21.15.86:0/3148058752 conn(0x7f66e40b1730 msgr2=0x7f66e40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e8c00640 1 -- 172.21.15.86:0/3148058752 shutdown_connections 2024-03-06T07:36:21.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.934+0000 7f66e8c00640 1 -- 172.21.15.86:0/3148058752 wait complete. 2024-03-06T07:36:21.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.937+0000 7f66e8c00640 1 Processor -- start 2024-03-06T07:36:21.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.937+0000 7f66e8c00640 1 -- start start 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e8c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66e4143310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e8c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e40b4570 0x7f66e413d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e8c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66e413a480 0x7f66e40ce300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e8c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66e406cb50 con 0x7f66c80039b0 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e8c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66e4160730 con 0x7f66e413a480 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e8c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66e406ccd0 con 0x7f66e40b4570 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66e4143310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66e4143310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42794/0 (socket says 172.21.15.86:42794) 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 -- 172.21.15.86:0/557861211 learned_addr learned my addr 172.21.15.86:0/557861211 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66d9400640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e40b4570 0x7f66e413d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 -- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e40b4570 msgr2=0x7f66e413d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:21.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e40b4570 0x7f66e413d9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 -- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66e413a480 msgr2=0x7f66e40ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e2800640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66e413a480 0x7f66e40ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66e413a480 0x7f66e40ce300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.939+0000 7f66e1e00640 1 -- 172.21.15.86:0/557861211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66e413df20 con 0x7f66c80039b0 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.940+0000 7f66e1e00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66e4143310 secure :-1 s=READY pgs=1643 cs=0 l=1 rev1=1 crypto rx=0x7f66d400a8c0 tx=0x7f66d4001cf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.940+0000 7f66daa00640 1 -- 172.21.15.86:0/557861211 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66d4043020 con 0x7f66c80039b0 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.940+0000 7f66daa00640 1 -- 172.21.15.86:0/557861211 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66d4034430 con 0x7f66c80039b0 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.940+0000 7f66daa00640 1 -- 172.21.15.86:0/557861211 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66d4034bf0 con 0x7f66c80039b0 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.940+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66e413adf0 con 0x7f66c80039b0 2024-03-06T07:36:21.943 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.940+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66e413d4c0 con 0x7f66c80039b0 2024-03-06T07:36:21.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.941+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f66e404f330 con 0x7f66c80039b0 2024-03-06T07:36:21.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.942+0000 7f66daa00640 1 -- 172.21.15.86:0/557861211 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f66d4046020 con 0x7f66c80039b0 2024-03-06T07:36:21.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.943+0000 7f66daa00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66c006b470 0x7f66c006d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:21.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.943+0000 7f66daa00640 1 -- 172.21.15.86:0/557861211 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f66d40acc20 con 0x7f66c80039b0 2024-03-06T07:36:21.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.943+0000 7f66d9400640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66c006b470 0x7f66c006d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:21.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.944+0000 7f66d9400640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66c006b470 0x7f66c006d930 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f66e415f9a0 tx=0x7f66cc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:21.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:21.948+0000 7f66daa00640 1 -- 172.21.15.86:0/557861211 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f66d404d030 con 0x7f66c80039b0 2024-03-06T07:36:22.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.194+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f66e40b4950 con 0x7f66c80039b0 2024-03-06T07:36:22.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.195+0000 7f66daa00640 1 -- 172.21.15.86:0/557861211 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f66d407bb70 con 0x7f66c80039b0 2024-03-06T07:36:22.196 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:22.196 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:22.197 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66c006b470 msgr2=0x7f66c006d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66c006b470 0x7f66c006d930 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f66e415f9a0 tx=0x7f66cc008040 comp rx=0 tx=0).stop 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 msgr2=0x7f66e4143310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66e4143310 secure :-1 s=READY pgs=1643 cs=0 l=1 rev1=1 crypto rx=0x7f66d400a8c0 tx=0x7f66d4001cf0 comp rx=0 tx=0).stop 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 shutdown_connections 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f66c006b470 0x7f66c006d930 unknown :-1 s=CLOSED pgs=700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66e413a480 0x7f66e40ce300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f66e40b4570 0x7f66e413d9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:22.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 --2- 172.21.15.86:0/557861211 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f66c80039b0 0x7f66e4143310 unknown :-1 s=CLOSED pgs=1643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:22.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 >> 172.21.15.86:0/557861211 conn(0x7f66e40b1730 msgr2=0x7f66e4140690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:22.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 shutdown_connections 2024-03-06T07:36:22.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:22.197+0000 7f66e8c00640 1 -- 172.21.15.86:0/557861211 wait complete. 2024-03-06T07:36:22.419 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:22.419 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:23.420 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:23.596 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:23.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.635+0000 7f9ecf200640 1 Processor -- start 2024-03-06T07:36:23.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.635+0000 7f9ecf200640 1 -- start start 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ecf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 0x7f9ed00d27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ecf200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9ed00b3ec0 0x7f9ed00b4350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ecf200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9ed0161960 con 0x7f9ed0161580 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ecf200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9ed00d2ce0 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ecf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9ed00cf6f0 con 0x7f9ed00d2360 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ece800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9ed00b3ec0 0x7f9ed00b4350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ece800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9ed00b3ec0 0x7f9ed00b4350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42810/0 (socket says 172.21.15.86:42810) 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ece800640 1 -- 172.21.15.86:0/2813240245 learned_addr learned my addr 172.21.15.86:0/2813240245 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:23.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.637+0000 7f9ecde00640 1 --1- 172.21.15.86:0/2813240245 >> v1:172.21.15.86:6789/0 conn(0x7f9ed0161580 0x7f9ed00d1950 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54376/0 (socket says 172.21.15.86:54376) 2024-03-06T07:36:23.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.638+0000 7f9ec5400640 1 --2- 172.21.15.86:0/2813240245 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 0x7f9ed00d27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:23.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.638+0000 7f9ece800640 1 -- 172.21.15.86:0/2813240245 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 msgr2=0x7f9ed00d27a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:23.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.638+0000 7f9ece800640 1 --2- 172.21.15.86:0/2813240245 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 0x7f9ed00d27a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:23.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.638+0000 7f9ece800640 1 -- 172.21.15.86:0/2813240245 >> v1:172.21.15.86:6789/0 conn(0x7f9ed0161580 legacy=0x7f9ed00d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:23.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.638+0000 7f9ecd400640 1 -- 172.21.15.86:0/2813240245 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1456824875 0 0) 0x7f9ed0161960 con 0x7f9ed0161580 2024-03-06T07:36:23.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.638+0000 7f9ece800640 1 -- 172.21.15.86:0/2813240245 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ed00cf9d0 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.639+0000 7f9ece800640 1 --2- 172.21.15.86:0/2813240245 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed00b4350 secure :-1 s=READY pgs=1645 cs=0 l=1 rev1=1 crypto rx=0x7f9ec8009e50 tx=0x7f9ec802bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=50443a5c1d974b86 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:23.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.639+0000 7f9ecd400640 1 -- 172.21.15.86:0/2813240245 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec802cb00 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.639+0000 7f9ecd400640 1 -- 172.21.15.86:0/2813240245 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9ec802cca0 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.639+0000 7f9ecd400640 1 -- 172.21.15.86:0/2813240245 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec8034610 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 -- 172.21.15.86:0/2813240245 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 msgr2=0x7f9ed00b4350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 --2- 172.21.15.86:0/2813240245 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed00b4350 secure :-1 s=READY pgs=1645 cs=0 l=1 rev1=1 crypto rx=0x7f9ec8009e50 tx=0x7f9ec802bf60 comp rx=0 tx=0).stop 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 -- 172.21.15.86:0/2813240245 shutdown_connections 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 --2- 172.21.15.86:0/2813240245 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed00b4350 unknown :-1 s=CLOSED pgs=1645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 --2- 172.21.15.86:0/2813240245 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 0x7f9ed00d27a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 -- 172.21.15.86:0/2813240245 >> 172.21.15.86:0/2813240245 conn(0x7f9ed00d3d80 msgr2=0x7f9ed00d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 -- 172.21.15.86:0/2813240245 shutdown_connections 2024-03-06T07:36:23.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.640+0000 7f9ecf200640 1 -- 172.21.15.86:0/2813240245 wait complete. 2024-03-06T07:36:23.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.641+0000 7f9ecf200640 1 Processor -- start 2024-03-06T07:36:23.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.642+0000 7f9ecf200640 1 -- start start 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.642+0000 7f9ecf200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed014d120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.642+0000 7f9ecf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 0x7f9ed014fa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecf200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ed0161580 0x7f9ed014ffa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecf200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ed00b39a0 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecf200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ed00b36c0 con 0x7f9ed0161580 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9ed00cf3a0 con 0x7f9ed00d2360 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed014d120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed014d120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42820/0 (socket says 172.21.15.86:42820) 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 -- 172.21.15.86:0/1005974214 learned_addr learned my addr 172.21.15.86:0/1005974214 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 -- 172.21.15.86:0/1005974214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 msgr2=0x7f9ed014fa60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 0x7f9ed014fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:23.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 -- 172.21.15.86:0/1005974214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ed0161580 msgr2=0x7f9ed014ffa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:23.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ed0161580 0x7f9ed014ffa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:23.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 -- 172.21.15.86:0/1005974214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ed00b91f0 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.643+0000 7f9ecde00640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed014d120 secure :-1 s=READY pgs=1646 cs=0 l=1 rev1=1 crypto rx=0x7f9ec000c4d0 tx=0x7f9ec000c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:23.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.644+0000 7f9ec6a00640 1 -- 172.21.15.86:0/1005974214 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec0018070 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.644+0000 7f9ec6a00640 1 -- 172.21.15.86:0/1005974214 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9ec0007500 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.644+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9ed014d9d0 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.644+0000 7f9ec6a00640 1 -- 172.21.15.86:0/1005974214 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec0004a20 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.644+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ed014df90 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.645+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9ed004f330 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.647+0000 7f9ec6a00640 1 -- 172.21.15.86:0/1005974214 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9ec001b070 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.648+0000 7f9ec6a00640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9eac06b4d0 0x7f9eac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:23.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.648+0000 7f9ec6a00640 1 -- 172.21.15.86:0/1005974214 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9ec0089590 con 0x7f9ed00b3ec0 2024-03-06T07:36:23.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.648+0000 7f9ec5400640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9eac06b4d0 0x7f9eac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:23.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.649+0000 7f9ec5400640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9eac06b4d0 0x7f9eac06d990 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7f9eb8006fd0 tx=0x7f9eb8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:23.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:23.653+0000 7f9ec6a00640 1 -- 172.21.15.86:0/1005974214 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9ec00583c0 con 0x7f9ed00b3ec0 2024-03-06T07:36:24.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.095+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9ed00d3b60 con 0x7f9ed00b3ec0 2024-03-06T07:36:24.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.095+0000 7f9ec6a00640 1 -- 172.21.15.86:0/1005974214 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f9ec0006e20 con 0x7f9ed00b3ec0 2024-03-06T07:36:24.097 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:24.097 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:24.097 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9eac06b4d0 msgr2=0x7f9eac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9eac06b4d0 0x7f9eac06d990 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7f9eb8006fd0 tx=0x7f9eb8008040 comp rx=0 tx=0).stop 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 msgr2=0x7f9ed014d120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed014d120 secure :-1 s=READY pgs=1646 cs=0 l=1 rev1=1 crypto rx=0x7f9ec000c4d0 tx=0x7f9ec000c990 comp rx=0 tx=0).stop 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 shutdown_connections 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9eac06b4d0 0x7f9eac06d990 unknown :-1 s=CLOSED pgs=701 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ed0161580 0x7f9ed014ffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ed00d2360 0x7f9ed014fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:24.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 --2- 172.21.15.86:0/1005974214 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ed00b3ec0 0x7f9ed014d120 unknown :-1 s=CLOSED pgs=1646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:24.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 >> 172.21.15.86:0/1005974214 conn(0x7f9ed00d3d80 msgr2=0x7f9ed00d0260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:24.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 shutdown_connections 2024-03-06T07:36:24.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:24.099+0000 7f9ecf200640 1 -- 172.21.15.86:0/1005974214 wait complete. 2024-03-06T07:36:24.319 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:24.320 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:25.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:36:25.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:36:25.321 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:25.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.527+0000 7ff0a6400640 1 Processor -- start 2024-03-06T07:36:25.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.528+0000 7ff0a6400640 1 -- start start 2024-03-06T07:36:25.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.529+0000 7ff0a6400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff0a00cf250 0x7ff0a00cd7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:25.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.529+0000 7ff0a6400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0a00cc7c0 con 0x7ff0a00d0270 2024-03-06T07:36:25.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.529+0000 7ff0a6400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0a00cca60 con 0x7ff0a015c030 2024-03-06T07:36:25.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.529+0000 7ff0a6400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0a00d00d0 con 0x7ff0a00cf250 2024-03-06T07:36:25.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.529+0000 7ff09ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff0a00cf250 0x7ff0a00cd7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:25.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.529+0000 7ff09fe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7ff0a00d0270 0x7ff0a006ed60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:54380/0 (socket says 172.21.15.86:54380) 2024-03-06T07:36:25.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.529+0000 7ff09fe00640 1 -- 172.21.15.86:0/1042988500 learned_addr learned my addr 172.21.15.86:0/1042988500 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:25.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.530+0000 7ff09e000640 1 -- 172.21.15.86:0/1042988500 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2827639782 0 0) 0x7ff0a00cc7c0 con 0x7ff0a00d0270 2024-03-06T07:36:25.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.530+0000 7ff09e000640 1 -- 172.21.15.86:0/1042988500 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff084003520 con 0x7ff0a00d0270 2024-03-06T07:36:25.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.530+0000 7ff09ea00640 1 -- 172.21.15.86:0/1042988500 >> v1:172.21.15.194:6789/0 conn(0x7ff0a015c030 legacy=0x7ff0a00cf880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:25.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.530+0000 7ff09ea00640 1 -- 172.21.15.86:0/1042988500 >> v1:172.21.15.86:6789/0 conn(0x7ff0a00d0270 legacy=0x7ff0a006ed60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:25.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.530+0000 7ff09ea00640 1 -- 172.21.15.86:0/1042988500 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0a00ccd00 con 0x7ff0a00cf250 2024-03-06T07:36:25.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.531+0000 7ff09ea00640 1 --2- 172.21.15.86:0/1042988500 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a00cf250 0x7ff0a00cd7e0 secure :-1 s=READY pgs=1648 cs=0 l=1 rev1=1 crypto rx=0x7ff08c0097d0 tx=0x7ff08c02b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6eb8bd6f8a588a8c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:25.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.531+0000 7ff09e000640 1 -- 172.21.15.86:0/1042988500 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff08c035020 con 0x7ff0a00cf250 2024-03-06T07:36:25.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.531+0000 7ff09e000640 1 -- 172.21.15.86:0/1042988500 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff08c031810 con 0x7ff0a00cf250 2024-03-06T07:36:25.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.531+0000 7ff09e000640 1 -- 172.21.15.86:0/1042988500 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff08c031be0 con 0x7ff0a00cf250 2024-03-06T07:36:25.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.532+0000 7ff0a6400640 1 -- 172.21.15.86:0/1042988500 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a00cf250 msgr2=0x7ff0a00cd7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:25.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.532+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1042988500 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a00cf250 0x7ff0a00cd7e0 secure :-1 s=READY pgs=1648 cs=0 l=1 rev1=1 crypto rx=0x7ff08c0097d0 tx=0x7ff08c02b8b0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.532+0000 7ff0a6400640 1 -- 172.21.15.86:0/1042988500 shutdown_connections 2024-03-06T07:36:25.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.532+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1042988500 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a00cf250 0x7ff0a00cd7e0 unknown :-1 s=CLOSED pgs=1648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.532+0000 7ff0a6400640 1 -- 172.21.15.86:0/1042988500 >> 172.21.15.86:0/1042988500 conn(0x7ff0a00d1cb0 msgr2=0x7ff0a00d20c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:25.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.532+0000 7ff0a6400640 1 -- 172.21.15.86:0/1042988500 shutdown_connections 2024-03-06T07:36:25.534 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.532+0000 7ff0a6400640 1 -- 172.21.15.86:0/1042988500 wait complete. 2024-03-06T07:36:25.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.534+0000 7ff0a6400640 1 Processor -- start 2024-03-06T07:36:25.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.534+0000 7ff0a6400640 1 -- start start 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff0a6400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0a00cf250 0x7ff0a00ca9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff0a6400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0a00d0270 0x7ff0a00c6a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff0a6400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a015c030 0x7ff0a00cb170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff0a6400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0a00cd2c0 con 0x7ff0a015c030 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff0a6400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0a015c410 con 0x7ff0a00d0270 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff0a6400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0a00d06c0 con 0x7ff0a00cf250 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff09fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a015c030 0x7ff0a00cb170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff09fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a015c030 0x7ff0a00cb170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:42846/0 (socket says 172.21.15.86:42846) 2024-03-06T07:36:25.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.535+0000 7ff09fe00640 1 -- 172.21.15.86:0/1788014414 learned_addr learned my addr 172.21.15.86:0/1788014414 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:25.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff09fe00640 1 -- 172.21.15.86:0/1788014414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0a00cf250 msgr2=0x7ff0a00ca9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:25.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff09ea00640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0a00d0270 0x7ff0a00c6a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:25.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff09fe00640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0a00cf250 0x7ff0a00ca9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff09fe00640 1 -- 172.21.15.86:0/1788014414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0a00d0270 msgr2=0x7ff0a00c6a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:25.538 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff09fe00640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0a00d0270 0x7ff0a00c6a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff09fe00640 1 -- 172.21.15.86:0/1788014414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0a00c5960 con 0x7ff0a015c030 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff09fe00640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a015c030 0x7ff0a00cb170 secure :-1 s=READY pgs=1649 cs=0 l=1 rev1=1 crypto rx=0x7ff09800cb40 tx=0x7ff09800a060 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff097400640 1 -- 172.21.15.86:0/1788014414 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff098017070 con 0x7ff0a015c030 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff097400640 1 -- 172.21.15.86:0/1788014414 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff09800aca0 con 0x7ff0a015c030 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff0a00c5c40 con 0x7ff0a015c030 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff097400640 1 -- 172.21.15.86:0/1788014414 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff098012620 con 0x7ff0a015c030 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.536+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff0a00c6200 con 0x7ff0a015c030 2024-03-06T07:36:25.539 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.538+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff0a00b7250 con 0x7ff0a015c030 2024-03-06T07:36:25.540 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.539+0000 7ff097400640 1 -- 172.21.15.86:0/1788014414 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff09801a070 con 0x7ff0a015c030 2024-03-06T07:36:25.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.539+0000 7ff097400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff08006b4d0 0x7ff08006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:25.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.540+0000 7ff097400640 1 -- 172.21.15.86:0/1788014414 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff09808a160 con 0x7ff0a015c030 2024-03-06T07:36:25.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.540+0000 7ff09f400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff08006b4d0 0x7ff08006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:25.542 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.540+0000 7ff09f400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff08006b4d0 0x7ff08006d990 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7ff090006fd0 tx=0x7ff090006bc0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:25.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.548+0000 7ff097400640 1 -- 172.21.15.86:0/1788014414 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff0a00b7250 con 0x7ff0a015c030 2024-03-06T07:36:25.597 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:25.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.783+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff0a004f330 con 0x7ff0a015c030 2024-03-06T07:36:25.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.784+0000 7ff097400640 1 -- 172.21.15.86:0/1788014414 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7ff098058f10 con 0x7ff0a015c030 2024-03-06T07:36:25.786 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:25.786 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:25.786 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:25.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff08006b4d0 msgr2=0x7ff08006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:25.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff08006b4d0 0x7ff08006d990 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7ff090006fd0 tx=0x7ff090006bc0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a015c030 msgr2=0x7ff0a00cb170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:25.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a015c030 0x7ff0a00cb170 secure :-1 s=READY pgs=1649 cs=0 l=1 rev1=1 crypto rx=0x7ff09800cb40 tx=0x7ff09800a060 comp rx=0 tx=0).stop 2024-03-06T07:36:25.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 shutdown_connections 2024-03-06T07:36:25.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff08006b4d0 0x7ff08006d990 unknown :-1 s=CLOSED pgs=702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0a015c030 0x7ff0a00cb170 unknown :-1 s=CLOSED pgs=1649 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0a00d0270 0x7ff0a00c6a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 --2- 172.21.15.86:0/1788014414 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0a00cf250 0x7ff0a00ca9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 >> 172.21.15.86:0/1788014414 conn(0x7ff0a00d1cb0 msgr2=0x7ff0a00b6b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.787+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 shutdown_connections 2024-03-06T07:36:25.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:25.788+0000 7ff0a6400640 1 -- 172.21.15.86:0/1788014414 wait complete. 2024-03-06T07:36:26.008 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:26.008 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:27.009 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:27.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.216+0000 7f46d3200640 1 Processor -- start 2024-03-06T07:36:27.217 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.216+0000 7f46d3200640 1 -- start start 2024-03-06T07:36:27.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.217+0000 7f46d3200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f46cc0cf430 0x7f46cc070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:27.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.217+0000 7f46d3200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f46cc071260 0x7f46cc06cb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:27.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.217+0000 7f46d3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc06d090 0x7f46cc165f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:27.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.217+0000 7f46d3200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46cc06d4e0 con 0x7f46cc0cf430 2024-03-06T07:36:27.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.217+0000 7f46d3200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46cc0717c0 con 0x7f46cc071260 2024-03-06T07:36:27.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.217+0000 7f46d3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46cc071aa0 con 0x7f46cc06d090 2024-03-06T07:36:27.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f46cc0cf430 0x7f46cc070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:27.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f46cc0cf430 0x7f46cc070d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50770/0 (socket says 172.21.15.86:50770) 2024-03-06T07:36:27.220 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 -- 172.21.15.86:0/3782954928 learned_addr learned my addr 172.21.15.86:0/3782954928 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46d1200640 1 --2- 172.21.15.86:0/3782954928 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc06d090 0x7f46cc165f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cb400640 1 --2- 172.21.15.86:0/3782954928 >> v2:172.21.15.194:3300/0 conn(0x7f46cc071260 0x7f46cc06cb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 -- 172.21.15.86:0/3782954928 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc06d090 msgr2=0x7f46cc165f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 --2- 172.21.15.86:0/3782954928 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc06d090 0x7f46cc165f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 -- 172.21.15.86:0/3782954928 >> v2:172.21.15.194:3300/0 conn(0x7f46cc071260 msgr2=0x7f46cc06cb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 --2- 172.21.15.86:0/3782954928 >> v2:172.21.15.194:3300/0 conn(0x7f46cc071260 0x7f46cc06cb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cbe00640 1 -- 172.21.15.86:0/3782954928 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46cc071d80 con 0x7f46cc0cf430 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46d1200640 1 --2- 172.21.15.86:0/3782954928 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc06d090 0x7f46cc165f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.218+0000 7f46cb400640 1 --2- 172.21.15.86:0/3782954928 >> v2:172.21.15.194:3300/0 conn(0x7f46cc071260 0x7f46cc06cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:27.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.219+0000 7f46cbe00640 1 --2- 172.21.15.86:0/3782954928 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc0cf430 0x7f46cc070d20 secure :-1 s=READY pgs=1650 cs=0 l=1 rev1=1 crypto rx=0x7f46cc06dc80 tx=0x7f46bc02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b6f41904cb5f934d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:27.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.220+0000 7f46caa00640 1 -- 172.21.15.86:0/3782954928 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46bc02ca80 con 0x7f46cc0cf430 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.220+0000 7f46caa00640 1 -- 172.21.15.86:0/3782954928 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46bc02cc20 con 0x7f46cc0cf430 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.220+0000 7f46caa00640 1 -- 172.21.15.86:0/3782954928 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46bc0343f0 con 0x7f46cc0cf430 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.220+0000 7f46d3200640 1 -- 172.21.15.86:0/3782954928 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc0cf430 msgr2=0x7f46cc070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.220+0000 7f46d3200640 1 --2- 172.21.15.86:0/3782954928 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc0cf430 0x7f46cc070d20 secure :-1 s=READY pgs=1650 cs=0 l=1 rev1=1 crypto rx=0x7f46cc06dc80 tx=0x7f46bc02bee0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.221+0000 7f46d3200640 1 -- 172.21.15.86:0/3782954928 shutdown_connections 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.221+0000 7f46d3200640 1 --2- 172.21.15.86:0/3782954928 >> v2:172.21.15.194:3300/0 conn(0x7f46cc071260 0x7f46cc06cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.221+0000 7f46d3200640 1 --2- 172.21.15.86:0/3782954928 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc06d090 0x7f46cc165f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.221+0000 7f46d3200640 1 --2- 172.21.15.86:0/3782954928 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc0cf430 0x7f46cc070d20 unknown :-1 s=CLOSED pgs=1650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.221+0000 7f46d3200640 1 -- 172.21.15.86:0/3782954928 >> 172.21.15.86:0/3782954928 conn(0x7f46cc0b1730 msgr2=0x7f46cc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.221+0000 7f46d3200640 1 -- 172.21.15.86:0/3782954928 shutdown_connections 2024-03-06T07:36:27.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.221+0000 7f46d3200640 1 -- 172.21.15.86:0/3782954928 wait complete. 2024-03-06T07:36:27.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.222+0000 7f46d3200640 1 Processor -- start 2024-03-06T07:36:27.224 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.223+0000 7f46d3200640 1 -- start start 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.223+0000 7f46d3200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc06d090 0x7f46cc159c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.223+0000 7f46d3200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46cc071260 0x7f46cc15bbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.223+0000 7f46d3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc0cf430 0x7f46cc15c120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46d3200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46cc162b70 con 0x7f46cc06d090 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46d3200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46cc072340 con 0x7f46cc071260 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46d3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46cc162cf0 con 0x7f46cc0cf430 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc06d090 0x7f46cc159c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc06d090 0x7f46cc159c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50774/0 (socket says 172.21.15.86:50774) 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 -- 172.21.15.86:0/281413927 learned_addr learned my addr 172.21.15.86:0/281413927 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46d1200640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc0cf430 0x7f46cc15c120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:27.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cb400640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46cc071260 0x7f46cc15bbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:27.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 -- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc0cf430 msgr2=0x7f46cc15c120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:27.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc0cf430 0x7f46cc15c120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 -- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46cc071260 msgr2=0x7f46cc15bbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:27.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46cc071260 0x7f46cc15bbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.224+0000 7f46cbe00640 1 -- 172.21.15.86:0/281413927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46cc15c660 con 0x7f46cc06d090 2024-03-06T07:36:27.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.225+0000 7f46cbe00640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc06d090 0x7f46cc159c30 secure :-1 s=READY pgs=1651 cs=0 l=1 rev1=1 crypto rx=0x7f46bc009f40 tx=0x7f46bc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:27.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.225+0000 7f46c3e00640 1 -- 172.21.15.86:0/281413927 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46bc043020 con 0x7f46cc06d090 2024-03-06T07:36:27.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.225+0000 7f46c3e00640 1 -- 172.21.15.86:0/281413927 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46bc03e450 con 0x7f46cc06d090 2024-03-06T07:36:27.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.225+0000 7f46c3e00640 1 -- 172.21.15.86:0/281413927 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46bc03e820 con 0x7f46cc06d090 2024-03-06T07:36:27.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.225+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46cc140e10 con 0x7f46cc06d090 2024-03-06T07:36:27.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.225+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46cc1413d0 con 0x7f46cc06d090 2024-03-06T07:36:27.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.227+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f46cc06ddf0 con 0x7f46cc06d090 2024-03-06T07:36:27.229 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.227+0000 7f46c3e00640 1 -- 172.21.15.86:0/281413927 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f46bc046020 con 0x7f46cc06d090 2024-03-06T07:36:27.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.229+0000 7f46c3e00640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46ac06b4d0 0x7f46ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:27.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.229+0000 7f46c3e00640 1 -- 172.21.15.86:0/281413927 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f46bc0acbf0 con 0x7f46cc06d090 2024-03-06T07:36:27.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.229+0000 7f46cb400640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46ac06b4d0 0x7f46ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:27.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.230+0000 7f46cb400640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46ac06b4d0 0x7f46ac06d990 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f46cc1554f0 tx=0x7f46b8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:27.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.233+0000 7f46c3e00640 1 -- 172.21.15.86:0/281413927 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f46cc06ddf0 con 0x7f46cc06d090 2024-03-06T07:36:27.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.467+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f46cc04f330 con 0x7f46cc06d090 2024-03-06T07:36:27.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.468+0000 7f46c3e00640 1 -- 172.21.15.86:0/281413927 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f46bc04d030 con 0x7f46cc06d090 2024-03-06T07:36:27.470 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:27.470 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:27.470 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46ac06b4d0 msgr2=0x7f46ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46ac06b4d0 0x7f46ac06d990 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f46cc1554f0 tx=0x7f46b8008040 comp rx=0 tx=0).stop 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc06d090 msgr2=0x7f46cc159c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc06d090 0x7f46cc159c30 secure :-1 s=READY pgs=1651 cs=0 l=1 rev1=1 crypto rx=0x7f46bc009f40 tx=0x7f46bc004a40 comp rx=0 tx=0).stop 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 shutdown_connections 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f46ac06b4d0 0x7f46ac06d990 unknown :-1 s=CLOSED pgs=703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f46cc0cf430 0x7f46cc15c120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f46cc071260 0x7f46cc15bbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 --2- 172.21.15.86:0/281413927 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f46cc06d090 0x7f46cc159c30 unknown :-1 s=CLOSED pgs=1651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:27.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 >> 172.21.15.86:0/281413927 conn(0x7f46cc0b1730 msgr2=0x7f46cc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:27.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 shutdown_connections 2024-03-06T07:36:27.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:27.471+0000 7f46d3200640 1 -- 172.21.15.86:0/281413927 wait complete. 2024-03-06T07:36:27.600 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:27.693 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:27.694 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:28.695 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:28.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.911+0000 7f3f80600640 1 Processor -- start 2024-03-06T07:36:28.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.911+0000 7f3f80600640 1 -- start start 2024-03-06T07:36:28.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.913+0000 7f3f80600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f7815f730 0x7f3f780cf840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:28.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.913+0000 7f3f80600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3f780b2e60 0x7f3f780cfd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:28.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.913+0000 7f3f80600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3f780d02c0 0x7f3f780ce660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:28.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.913+0000 7f3f80600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3f780cd320 con 0x7f3f780b2e60 2024-03-06T07:36:28.915 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.913+0000 7f3f80600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3f780cdbc0 con 0x7f3f780d02c0 2024-03-06T07:36:28.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.913+0000 7f3f80600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f780b3340 con 0x7f3f7815f730 2024-03-06T07:36:28.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3f780b2e60 0x7f3f780cfd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:28.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3f780b2e60 0x7f3f780cfd80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50784/0 (socket says 172.21.15.86:50784) 2024-03-06T07:36:28.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 -- 172.21.15.86:0/3373028921 learned_addr learned my addr 172.21.15.86:0/3373028921 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:28.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7dc00640 1 --2- 172.21.15.86:0/3373028921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f7815f730 0x7f3f780cf840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:28.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7e600640 1 --2- 172.21.15.86:0/3373028921 >> v2:172.21.15.194:3300/0 conn(0x7f3f780d02c0 0x7f3f780ce660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:28.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 -- 172.21.15.86:0/3373028921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f7815f730 msgr2=0x7f3f780cf840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:28.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 --2- 172.21.15.86:0/3373028921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f7815f730 0x7f3f780cf840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:28.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 -- 172.21.15.86:0/3373028921 >> v2:172.21.15.194:3300/0 conn(0x7f3f780d02c0 msgr2=0x7f3f780ce660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:28.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 --2- 172.21.15.86:0/3373028921 >> v2:172.21.15.194:3300/0 conn(0x7f3f780d02c0 0x7f3f780ce660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:28.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.914+0000 7f3f7d200640 1 -- 172.21.15.86:0/3373028921 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f780cd570 con 0x7f3f780b2e60 2024-03-06T07:36:28.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.916+0000 7f3f7d200640 1 --2- 172.21.15.86:0/3373028921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f780cfd80 secure :-1 s=READY pgs=1652 cs=0 l=1 rev1=1 crypto rx=0x7f3f780726e0 tx=0x7f3f6402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cb74bbb50bcf4281 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:28.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f77e00640 1 -- 172.21.15.86:0/3373028921 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f64035020 con 0x7f3f780b2e60 2024-03-06T07:36:28.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f77e00640 1 -- 172.21.15.86:0/3373028921 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f64031870 con 0x7f3f780b2e60 2024-03-06T07:36:28.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f77e00640 1 -- 172.21.15.86:0/3373028921 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f64031c40 con 0x7f3f780b2e60 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f80600640 1 -- 172.21.15.86:0/3373028921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 msgr2=0x7f3f780cfd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f80600640 1 --2- 172.21.15.86:0/3373028921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f780cfd80 secure :-1 s=READY pgs=1652 cs=0 l=1 rev1=1 crypto rx=0x7f3f780726e0 tx=0x7f3f6402b910 comp rx=0 tx=0).stop 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f80600640 1 -- 172.21.15.86:0/3373028921 shutdown_connections 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f80600640 1 --2- 172.21.15.86:0/3373028921 >> v2:172.21.15.194:3300/0 conn(0x7f3f780d02c0 0x7f3f780ce660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f80600640 1 --2- 172.21.15.86:0/3373028921 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f780cfd80 unknown :-1 s=CLOSED pgs=1652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f80600640 1 --2- 172.21.15.86:0/3373028921 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f7815f730 0x7f3f780cf840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.917+0000 7f3f80600640 1 -- 172.21.15.86:0/3373028921 >> 172.21.15.86:0/3373028921 conn(0x7f3f780b3800 msgr2=0x7f3f780b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:28.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.918+0000 7f3f80600640 1 -- 172.21.15.86:0/3373028921 shutdown_connections 2024-03-06T07:36:28.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.918+0000 7f3f80600640 1 -- 172.21.15.86:0/3373028921 wait complete. 2024-03-06T07:36:28.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.919+0000 7f3f80600640 1 Processor -- start 2024-03-06T07:36:28.921 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.919+0000 7f3f80600640 1 -- start start 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.920+0000 7f3f80600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f7815ebc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.920+0000 7f3f80600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f780d02c0 0x7f3f780b48d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.920+0000 7f3f80600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3f7815f730 0x7f3f780ce170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.920+0000 7f3f80600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f7815fb10 con 0x7f3f780b2e60 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.920+0000 7f3f80600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f780d0ae0 con 0x7f3f7815f730 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.920+0000 7f3f80600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f7815fc90 con 0x7f3f780d02c0 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f7815ebc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:28.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f7815ebc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50794/0 (socket says 172.21.15.86:50794) 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 -- 172.21.15.86:0/3346008046 learned_addr learned my addr 172.21.15.86:0/3346008046 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7d200640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f780d02c0 0x7f3f780b48d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7e600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3f7815f730 0x7f3f780ce170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 -- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f780d02c0 msgr2=0x7f3f780b48d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f780d02c0 0x7f3f780b48d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 -- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3f7815f730 msgr2=0x7f3f780ce170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3f7815f730 0x7f3f780ce170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7dc00640 1 -- 172.21.15.86:0/3346008046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f780b5100 con 0x7f3f780b2e60 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7d200640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f780d02c0 0x7f3f780b48d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.921+0000 7f3f7e600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3f7815f730 0x7f3f780ce170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:28.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.922+0000 7f3f7dc00640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f7815ebc0 secure :-1 s=READY pgs=1653 cs=0 l=1 rev1=1 crypto rx=0x7f3f6c00d800 tx=0x7f3f6c00dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:28.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.922+0000 7f3f75600640 1 -- 172.21.15.86:0/3346008046 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f6c00b6c0 con 0x7f3f780b2e60 2024-03-06T07:36:28.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.922+0000 7f3f75600640 1 -- 172.21.15.86:0/3346008046 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f6c019070 con 0x7f3f780b2e60 2024-03-06T07:36:28.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.922+0000 7f3f75600640 1 -- 172.21.15.86:0/3346008046 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f6c00ba80 con 0x7f3f780b2e60 2024-03-06T07:36:28.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.922+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f780b53e0 con 0x7f3f780b2e60 2024-03-06T07:36:28.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.922+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f780b59a0 con 0x7f3f780b2e60 2024-03-06T07:36:28.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.924+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3f7806bbc0 con 0x7f3f780b2e60 2024-03-06T07:36:28.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.924+0000 7f3f75600640 1 -- 172.21.15.86:0/3346008046 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3f6c00bc20 con 0x7f3f780b2e60 2024-03-06T07:36:28.927 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.925+0000 7f3f75600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f5806b500 0x7f3f5806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:28.927 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.925+0000 7f3f75600640 1 -- 172.21.15.86:0/3346008046 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3f6c08a670 con 0x7f3f780b2e60 2024-03-06T07:36:28.927 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.925+0000 7f3f7d200640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f5806b500 0x7f3f5806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:28.928 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.926+0000 7f3f7d200640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f5806b500 0x7f3f5806d9c0 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7f3f64036000 tx=0x7f3f6403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:28.932 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:28.930+0000 7f3f75600640 1 -- 172.21.15.86:0/3346008046 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3f6c01c360 con 0x7f3f780b2e60 2024-03-06T07:36:29.202 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.201+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3f780b5dc0 con 0x7f3f780b2e60 2024-03-06T07:36:29.203 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.202+0000 7f3f75600640 1 -- 172.21.15.86:0/3346008046 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f3f6c059540 con 0x7f3f780b2e60 2024-03-06T07:36:29.204 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:29.204 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:29.204 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:29.206 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f5806b500 msgr2=0x7f3f5806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f5806b500 0x7f3f5806d9c0 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7f3f64036000 tx=0x7f3f6403c040 comp rx=0 tx=0).stop 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 msgr2=0x7f3f7815ebc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f7815ebc0 secure :-1 s=READY pgs=1653 cs=0 l=1 rev1=1 crypto rx=0x7f3f6c00d800 tx=0x7f3f6c00dcc0 comp rx=0 tx=0).stop 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 shutdown_connections 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3f5806b500 0x7f3f5806d9c0 unknown :-1 s=CLOSED pgs=704 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3f7815f730 0x7f3f780ce170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3f780d02c0 0x7f3f780b48d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 --2- 172.21.15.86:0/3346008046 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3f780b2e60 0x7f3f7815ebc0 unknown :-1 s=CLOSED pgs=1653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 >> 172.21.15.86:0/3346008046 conn(0x7f3f780b3800 msgr2=0x7f3f7815a3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 shutdown_connections 2024-03-06T07:36:29.207 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:29.205+0000 7f3f80600640 1 -- 172.21.15.86:0/3346008046 wait complete. 2024-03-06T07:36:29.427 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:29.427 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:29.602 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:30.428 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:30.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.637+0000 7f09ada00640 1 Processor -- start 2024-03-06T07:36:30.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.637+0000 7f09ada00640 1 -- start start 2024-03-06T07:36:30.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.639+0000 7f09ada00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:30.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.639+0000 7f09ada00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f09a815fe30 0x7f09a80ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:30.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.639+0000 7f09ada00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f09a806ee70 con 0x7f09a815f9d0 2024-03-06T07:36:30.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.639+0000 7f09ada00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f09a806cef0 con 0x7f09a815fe30 2024-03-06T07:36:30.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.639+0000 7f09ada00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09a806d1d0 con 0x7f09a8144c00 2024-03-06T07:36:30.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.640+0000 7f09a6a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:30.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.640+0000 7f09a7400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f09a815fe30 0x7f09a80ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:30.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.640+0000 7f09a6a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a806e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:53914/0 (socket says 172.21.15.86:53914) 2024-03-06T07:36:30.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.640+0000 7f09a7400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f09a815fe30 0x7f09a80ce7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:56496/0 (socket says 172.21.15.86:56496) 2024-03-06T07:36:30.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.640+0000 7f09a7400640 1 -- 172.21.15.86:0/530518412 learned_addr learned my addr 172.21.15.86:0/530518412 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:30.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.641+0000 7f09a6a00640 1 -- 172.21.15.86:0/530518412 >> v2:172.21.15.194:3300/0 conn(0x7f09a815fe30 msgr2=0x7f09a80ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:30.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.641+0000 7f09a6a00640 1 --2- 172.21.15.86:0/530518412 >> v2:172.21.15.194:3300/0 conn(0x7f09a815fe30 0x7f09a80ce7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.641+0000 7f09a6a00640 1 -- 172.21.15.86:0/530518412 >> v1:172.21.15.194:6789/0 conn(0x7f09a815f9d0 legacy=0x7f09a80ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:30.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.641+0000 7f09a6a00640 1 -- 172.21.15.86:0/530518412 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09a806d4b0 con 0x7f09a8144c00 2024-03-06T07:36:30.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.642+0000 7f09a6a00640 1 --2- 172.21.15.86:0/530518412 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a806e930 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f0998009d90 tx=0x7f099802bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=97c7b5f491d91d68 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:30.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.642+0000 7f09a6000640 1 -- 172.21.15.86:0/530518412 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0998039070 con 0x7f09a8144c00 2024-03-06T07:36:30.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.642+0000 7f09a6000640 1 -- 172.21.15.86:0/530518412 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f099802cc50 con 0x7f09a8144c00 2024-03-06T07:36:30.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.642+0000 7f09a6000640 1 -- 172.21.15.86:0/530518412 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0998034440 con 0x7f09a8144c00 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 -- 172.21.15.86:0/530518412 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 msgr2=0x7f09a806e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 --2- 172.21.15.86:0/530518412 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a806e930 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f0998009d90 tx=0x7f099802bea0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 -- 172.21.15.86:0/530518412 shutdown_connections 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 --2- 172.21.15.86:0/530518412 >> v2:172.21.15.194:3300/0 conn(0x7f09a815fe30 0x7f09a80ce7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 --2- 172.21.15.86:0/530518412 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a806e930 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 -- 172.21.15.86:0/530518412 >> 172.21.15.86:0/530518412 conn(0x7f09a80b1730 msgr2=0x7f09a80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 -- 172.21.15.86:0/530518412 shutdown_connections 2024-03-06T07:36:30.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.643+0000 7f09ada00640 1 -- 172.21.15.86:0/530518412 wait complete. 2024-03-06T07:36:30.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.645+0000 7f09ada00640 1 Processor -- start 2024-03-06T07:36:30.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.645+0000 7f09ada00640 1 -- start start 2024-03-06T07:36:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09ada00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a81418c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09ada00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09a815f9d0 0x7f09a815c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09ada00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09a815fe30 0x7f09a814d780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09ada00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09a8160730 con 0x7f09a815fe30 2024-03-06T07:36:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09ada00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09a8160280 con 0x7f09a815f9d0 2024-03-06T07:36:30.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09ada00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09a806cb50 con 0x7f09a8144c00 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09a7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09a815fe30 0x7f09a814d780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09a7400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09a815fe30 0x7f09a814d780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50808/0 (socket says 172.21.15.86:50808) 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09a7400640 1 -- 172.21.15.86:0/2878530306 learned_addr learned my addr 172.21.15.86:0/2878530306 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f09a6a00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a81418c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.646+0000 7f099e000640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09a815f9d0 0x7f09a815c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f09a7400640 1 -- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 msgr2=0x7f09a81418c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f09a7400640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a81418c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f09a7400640 1 -- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09a815f9d0 msgr2=0x7f09a815c340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f09a7400640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09a815f9d0 0x7f09a815c340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f09a7400640 1 -- 172.21.15.86:0/2878530306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09a815f190 con 0x7f09a815fe30 2024-03-06T07:36:30.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f09a7400640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09a815fe30 0x7f09a814d780 secure :-1 s=READY pgs=1654 cs=0 l=1 rev1=1 crypto rx=0x7f09a000e870 tx=0x7f09a000ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:30.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f099f400640 1 -- 172.21.15.86:0/2878530306 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09a0017070 con 0x7f09a815fe30 2024-03-06T07:36:30.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f099f400640 1 -- 172.21.15.86:0/2878530306 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f09a000c940 con 0x7f09a815fe30 2024-03-06T07:36:30.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f099f400640 1 -- 172.21.15.86:0/2878530306 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09a000cd30 con 0x7f09a815fe30 2024-03-06T07:36:30.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.647+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f09a815f470 con 0x7f09a815fe30 2024-03-06T07:36:30.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.648+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f09a814dcc0 con 0x7f09a815fe30 2024-03-06T07:36:30.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.649+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f09a804f330 con 0x7f09a815fe30 2024-03-06T07:36:30.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.650+0000 7f099f400640 1 -- 172.21.15.86:0/2878530306 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f09a001a070 con 0x7f09a815fe30 2024-03-06T07:36:30.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.650+0000 7f099f400640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f098406b4d0 0x7f098406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:30.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.651+0000 7f099f400640 1 -- 172.21.15.86:0/2878530306 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f09a008a540 con 0x7f09a815fe30 2024-03-06T07:36:30.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.651+0000 7f09a6a00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f098406b4d0 0x7f098406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:30.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.652+0000 7f09a6a00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f098406b4d0 0x7f098406d990 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7f0998036000 tx=0x7f0998030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:30.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.655+0000 7f099f400640 1 -- 172.21.15.86:0/2878530306 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f09a0059410 con 0x7f09a815fe30 2024-03-06T07:36:30.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.971+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f09a8072890 con 0x7f09a815fe30 2024-03-06T07:36:30.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.972+0000 7f099f400640 1 -- 172.21.15.86:0/2878530306 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f09a0058d10 con 0x7f09a815fe30 2024-03-06T07:36:30.974 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:30.975 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:30.975 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:30.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.974+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f098406b4d0 msgr2=0x7f098406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:30.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.974+0000 7f09ada00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f098406b4d0 0x7f098406d990 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7f0998036000 tx=0x7f0998030000 comp rx=0 tx=0).stop 2024-03-06T07:36:30.977 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.974+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09a815fe30 msgr2=0x7f09a814d780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.974+0000 7f09ada00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09a815fe30 0x7f09a814d780 secure :-1 s=READY pgs=1654 cs=0 l=1 rev1=1 crypto rx=0x7f09a000e870 tx=0x7f09a000ed30 comp rx=0 tx=0).stop 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 shutdown_connections 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f098406b4d0 0x7f098406d990 unknown :-1 s=CLOSED pgs=705 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f09a815fe30 0x7f09a814d780 unknown :-1 s=CLOSED pgs=1654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f09a815f9d0 0x7f09a815c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 --2- 172.21.15.86:0/2878530306 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f09a8144c00 0x7f09a81418c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 >> 172.21.15.86:0/2878530306 conn(0x7f09a80b1730 msgr2=0x7f09a8071870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 shutdown_connections 2024-03-06T07:36:30.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:30.975+0000 7f09ada00640 1 -- 172.21.15.86:0/2878530306 wait complete. 2024-03-06T07:36:31.194 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:31.194 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:31.604 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:32.195 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:32.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.401+0000 7fa146e00640 1 Processor -- start 2024-03-06T07:36:32.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.401+0000 7fa146e00640 1 -- start start 2024-03-06T07:36:32.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.403+0000 7fa146e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 0x7fa140071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.403+0000 7fa146e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa1400ca5d0 0x7fa1400724b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.403+0000 7fa146e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa1400cabf0 con 0x7fa1400cf3e0 2024-03-06T07:36:32.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.403+0000 7fa146e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa1400ce900 con 0x7fa1400ca5d0 2024-03-06T07:36:32.406 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.403+0000 7fa146e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa1400cebe0 con 0x7fa140144660 2024-03-06T07:36:32.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.403+0000 7fa13f400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa1400cf3e0 0x7fa1400c9ec0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47276/0 (socket says 172.21.15.86:47276) 2024-03-06T07:36:32.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.403+0000 7fa13f400640 1 -- 172.21.15.86:0/1579741082 learned_addr learned my addr 172.21.15.86:0/1579741082 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:32.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13fe00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 0x7fa140071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa144e00640 1 --2- 172.21.15.86:0/1579741082 >> v2:172.21.15.194:3300/0 conn(0x7fa1400ca5d0 0x7fa1400724b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1936411459 0 0) 0x7fa1400cabf0 con 0x7fa1400cf3e0 2024-03-06T07:36:32.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa124003520 con 0x7fa1400cf3e0 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1254680633 0 0) 0x7fa124003520 con 0x7fa1400cf3e0 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 msgr2=0x7fa140071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 0x7fa140071f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 >> v2:172.21.15.194:3300/0 conn(0x7fa1400ca5d0 msgr2=0x7fa1400724b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 --2- 172.21.15.86:0/1579741082 >> v2:172.21.15.194:3300/0 conn(0x7fa1400ca5d0 0x7fa1400724b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1400ceec0 con 0x7fa1400cf3e0 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fa12c001670 con 0x7fa1400cf3e0 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 >> v1:172.21.15.86:6789/0 conn(0x7fa1400cf3e0 legacy=0x7fa1400c9ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.404+0000 7fa13ea00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1240039b0 0x7fa124003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13ea00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 0x7fa124004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13ea00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1240051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1400ce900 con 0x7fa124004d40 2024-03-06T07:36:32.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1400cebe0 con 0x7fa1240039b0 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa12400b9c0 con 0x7fa140144660 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa144e00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1240051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13fe00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1240039b0 0x7fa124003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13f400640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 0x7fa124004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa144e00640 1 -- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 msgr2=0x7fa124004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa144e00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 0x7fa124004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa144e00640 1 -- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1240039b0 msgr2=0x7fa124003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa144e00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1240039b0 0x7fa124003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa144e00640 1 -- 172.21.15.86:0/1579741082 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa124003520 con 0x7fa124004d40 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13f400640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa140144660 0x7fa124004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.405+0000 7fa13fe00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1240039b0 0x7fa124003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:36:32.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.406+0000 7fa144e00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1240051c0 secure :-1 s=READY pgs=1656 cs=0 l=1 rev1=1 crypto rx=0x7fa138009fc0 tx=0x7fa124005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=33f3ef675805e302 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa138039070 con 0x7fa124004d40 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa13802cc30 con 0x7fa124004d40 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa13ea00640 1 -- 172.21.15.86:0/1579741082 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa138009c10 con 0x7fa124004d40 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa146e00640 1 -- 172.21.15.86:0/1579741082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 msgr2=0x7fa1240051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa146e00640 1 --2- 172.21.15.86:0/1579741082 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1240051c0 secure :-1 s=READY pgs=1656 cs=0 l=1 rev1=1 crypto rx=0x7fa138009fc0 tx=0x7fa124005830 comp rx=0 tx=0).stop 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa144e00640 1 -- 172.21.15.86:0/1579741082 reap_dead start 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa146e00640 1 -- 172.21.15.86:0/1579741082 shutdown_connections 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.407+0000 7fa146e00640 1 -- 172.21.15.86:0/1579741082 >> 172.21.15.86:0/1579741082 conn(0x7fa1400d0b90 msgr2=0x7fa1400d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.408+0000 7fa146e00640 1 -- 172.21.15.86:0/1579741082 shutdown_connections 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.408+0000 7fa146e00640 1 -- 172.21.15.86:0/1579741082 wait complete. 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.409+0000 7fa146e00640 1 Processor -- start 2024-03-06T07:36:32.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.409+0000 7fa146e00640 1 -- start start 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.410+0000 7fa146e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1400b6610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.410+0000 7fa146e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1400b3dd0 0x7fa140161b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.410+0000 7fa146e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1400b4250 0x7fa140162090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.410+0000 7fa146e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1400720f0 con 0x7fa124004d40 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.410+0000 7fa146e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa140071f70 con 0x7fa1400b4250 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.410+0000 7fa146e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa140072270 con 0x7fa1400b3dd0 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1400b6610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1400b6610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50836/0 (socket says 172.21.15.86:50836) 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 -- 172.21.15.86:0/3588157197 learned_addr learned my addr 172.21.15.86:0/3588157197 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13f400640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1400b3dd0 0x7fa140161b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa144e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1400b4250 0x7fa140162090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 -- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1400b3dd0 msgr2=0x7fa140161b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1400b3dd0 0x7fa140161b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 -- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1400b4250 msgr2=0x7fa140162090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1400b4250 0x7fa140162090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 -- 172.21.15.86:0/3588157197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1400b58b0 con 0x7fa124004d40 2024-03-06T07:36:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13f400640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1400b3dd0 0x7fa140161b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.411+0000 7fa13fe00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1400b6610 secure :-1 s=READY pgs=1657 cs=0 l=1 rev1=1 crypto rx=0x7fa130011540 tx=0x7fa130011a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:32.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.412+0000 7fa137e00640 1 -- 172.21.15.86:0/3588157197 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa130015870 con 0x7fa124004d40 2024-03-06T07:36:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.412+0000 7fa137e00640 1 -- 172.21.15.86:0/3588157197 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa130015a10 con 0x7fa124004d40 2024-03-06T07:36:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.412+0000 7fa137e00640 1 -- 172.21.15.86:0/3588157197 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa130015dc0 con 0x7fa124004d40 2024-03-06T07:36:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.412+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa1400b6b50 con 0x7fa124004d40 2024-03-06T07:36:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.412+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa1400b5b30 con 0x7fa124004d40 2024-03-06T07:36:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.413+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa14004f330 con 0x7fa124004d40 2024-03-06T07:36:32.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.414+0000 7fa137e00640 1 -- 172.21.15.86:0/3588157197 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa13000f460 con 0x7fa124004d40 2024-03-06T07:36:32.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.415+0000 7fa137e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa12006b4d0 0x7fa12006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:32.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.415+0000 7fa137e00640 1 -- 172.21.15.86:0/3588157197 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa13008abc0 con 0x7fa124004d40 2024-03-06T07:36:32.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.415+0000 7fa13f400640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa12006b4d0 0x7fa12006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:32.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.416+0000 7fa13f400640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa12006b4d0 0x7fa12006d990 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7fa12c001eb0 tx=0x7fa12c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:32.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.419+0000 7fa137e00640 1 -- 172.21.15.86:0/3588157197 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa1400b58b0 con 0x7fa124004d40 2024-03-06T07:36:32.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.652+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa1401424e0 con 0x7fa124004d40 2024-03-06T07:36:32.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.653+0000 7fa137e00640 1 -- 172.21.15.86:0/3588157197 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fa130019030 con 0x7fa124004d40 2024-03-06T07:36:32.656 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:32.656 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:32.656 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:32.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.655+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa12006b4d0 msgr2=0x7fa12006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.655+0000 7fa146e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa12006b4d0 0x7fa12006d990 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7fa12c001eb0 tx=0x7fa12c009040 comp rx=0 tx=0).stop 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.655+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 msgr2=0x7fa1400b6610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.655+0000 7fa146e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1400b6610 secure :-1 s=READY pgs=1657 cs=0 l=1 rev1=1 crypto rx=0x7fa130011540 tx=0x7fa130011a00 comp rx=0 tx=0).stop 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 shutdown_connections 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa12006b4d0 0x7fa12006d990 unknown :-1 s=CLOSED pgs=706 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1400b4250 0x7fa140162090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1400b3dd0 0x7fa140161b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 --2- 172.21.15.86:0/3588157197 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa124004d40 0x7fa1400b6610 unknown :-1 s=CLOSED pgs=1657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 >> 172.21.15.86:0/3588157197 conn(0x7fa1400d0b90 msgr2=0x7fa14006f950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 shutdown_connections 2024-03-06T07:36:32.659 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:32.656+0000 7fa146e00640 1 -- 172.21.15.86:0/3588157197 wait complete. 2024-03-06T07:36:32.879 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:32.879 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:33.606 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:33.880 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:34.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.096+0000 7f9904a00640 1 Processor -- start 2024-03-06T07:36:34.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.096+0000 7f9904a00640 1 -- start start 2024-03-06T07:36:34.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.097+0000 7f9904a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f990015eff0 0x7f9900070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:34.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.097+0000 7f9904a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9900071d30 con 0x7f990006e4d0 2024-03-06T07:36:34.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.097+0000 7f9904a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f990015f470 con 0x7f99000af1c0 2024-03-06T07:36:34.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.097+0000 7f9904a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9900071260 con 0x7f990015eff0 2024-03-06T07:36:34.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.098+0000 7f98fe800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f990006e4d0 0x7f99000b35d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:47280/0 (socket says 172.21.15.86:47280) 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.098+0000 7f98fe800640 1 -- 172.21.15.86:0/2046144192 learned_addr learned my addr 172.21.15.86:0/2046144192 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.098+0000 7f98f5400640 1 --2- 172.21.15.86:0/2046144192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f990015eff0 0x7f9900070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.098+0000 7f98fd400640 1 -- 172.21.15.86:0/2046144192 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1226389123 0 0) 0x7f98f8001110 con 0x7f990006e4d0 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.099+0000 7f98fd400640 1 -- 172.21.15.86:0/2046144192 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f98e4003520 con 0x7f990006e4d0 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.099+0000 7f98fd400640 1 -- 172.21.15.86:0/2046144192 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2293309277 0 0) 0x7f990015f470 con 0x7f99000af1c0 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.099+0000 7f98fd400640 1 -- 172.21.15.86:0/2046144192 --> v1:172.21.15.194:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f98f8001110 con 0x7f99000af1c0 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.099+0000 7f98f5400640 1 -- 172.21.15.86:0/2046144192 >> v1:172.21.15.194:6789/0 conn(0x7f99000af1c0 legacy=0x7f990006cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.099+0000 7f98f5400640 1 -- 172.21.15.86:0/2046144192 >> v1:172.21.15.86:6789/0 conn(0x7f990006e4d0 legacy=0x7f99000b35d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:34.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.099+0000 7f98f5400640 1 -- 172.21.15.86:0/2046144192 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9900071490 con 0x7f990015eff0 2024-03-06T07:36:34.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.100+0000 7f98f5400640 1 --2- 172.21.15.86:0/2046144192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f990015eff0 0x7f9900070d20 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f98e80097d0 tx=0x7f98e802b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=765b1dcfd699cfc6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:34.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f98fd400640 1 -- 172.21.15.86:0/2046144192 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98e8035020 con 0x7f990015eff0 2024-03-06T07:36:34.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f98fd400640 1 -- 172.21.15.86:0/2046144192 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f98e8031890 con 0x7f990015eff0 2024-03-06T07:36:34.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f98fd400640 1 -- 172.21.15.86:0/2046144192 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98e8031c60 con 0x7f990015eff0 2024-03-06T07:36:34.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f9904a00640 1 -- 172.21.15.86:0/2046144192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f990015eff0 msgr2=0x7f9900070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:34.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f9904a00640 1 --2- 172.21.15.86:0/2046144192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f990015eff0 0x7f9900070d20 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f98e80097d0 tx=0x7f98e802b930 comp rx=0 tx=0).stop 2024-03-06T07:36:34.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f9904a00640 1 -- 172.21.15.86:0/2046144192 shutdown_connections 2024-03-06T07:36:34.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f9904a00640 1 --2- 172.21.15.86:0/2046144192 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f990015eff0 0x7f9900070d20 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:34.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f9904a00640 1 -- 172.21.15.86:0/2046144192 >> 172.21.15.86:0/2046144192 conn(0x7f99000b1730 msgr2=0x7f99000b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:34.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f9904a00640 1 -- 172.21.15.86:0/2046144192 shutdown_connections 2024-03-06T07:36:34.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.101+0000 7f9904a00640 1 -- 172.21.15.86:0/2046144192 wait complete. 2024-03-06T07:36:34.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.103+0000 7f9904a00640 1 Processor -- start 2024-03-06T07:36:34.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.103+0000 7f9904a00640 1 -- start start 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f9904a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f990006e4d0 0x7f99000ce620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f9904a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99000af1c0 0x7f99001f3fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f9904a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f990015eff0 0x7f99000af810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f9904a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99000b30b0 con 0x7f990006e4d0 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f9904a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9900071a50 con 0x7f990015eff0 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f9904a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f990006d370 con 0x7f99000af1c0 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f98fde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f990006e4d0 0x7f99000ce620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f98fde00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f990006e4d0 0x7f99000ce620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50840/0 (socket says 172.21.15.86:50840) 2024-03-06T07:36:34.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.104+0000 7f98fde00640 1 -- 172.21.15.86:0/4026921447 learned_addr learned my addr 172.21.15.86:0/4026921447 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fe800640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f990015eff0 0x7f99000af810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98f5400640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99000af1c0 0x7f99001f3fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fde00640 1 -- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99000af1c0 msgr2=0x7f99001f3fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fde00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99000af1c0 0x7f99001f3fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fde00640 1 -- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f990015eff0 msgr2=0x7f99000af810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fde00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f990015eff0 0x7f99000af810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fde00640 1 -- 172.21.15.86:0/4026921447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99000cee70 con 0x7f990006e4d0 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fe800640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f990015eff0 0x7f99000af810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98fde00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f990006e4d0 0x7f99000ce620 secure :-1 s=READY pgs=1659 cs=0 l=1 rev1=1 crypto rx=0x7f98f000c7c0 tx=0x7f98f000cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98f6a00640 1 -- 172.21.15.86:0/4026921447 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98f0003b70 con 0x7f990006e4d0 2024-03-06T07:36:34.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98f6a00640 1 -- 172.21.15.86:0/4026921447 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f98f0003d10 con 0x7f990006e4d0 2024-03-06T07:36:34.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f98f6a00640 1 -- 172.21.15.86:0/4026921447 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98f000aa90 con 0x7f990006e4d0 2024-03-06T07:36:34.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.105+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99001e6910 con 0x7f990006e4d0 2024-03-06T07:36:34.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.106+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99001e6ed0 con 0x7f990006e4d0 2024-03-06T07:36:34.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.107+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f99000cebc0 con 0x7f990006e4d0 2024-03-06T07:36:34.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.108+0000 7f98f6a00640 1 -- 172.21.15.86:0/4026921447 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f98f000ac30 con 0x7f990006e4d0 2024-03-06T07:36:34.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.108+0000 7f98f6a00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f98dc06b4d0 0x7f98dc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:34.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.109+0000 7f98f6a00640 1 -- 172.21.15.86:0/4026921447 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f98f008a230 con 0x7f990006e4d0 2024-03-06T07:36:34.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.109+0000 7f98f5400640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f98dc06b4d0 0x7f98dc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:34.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.109+0000 7f98f5400640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f98dc06b4d0 0x7f98dc06d990 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f98e8036000 tx=0x7f98e8008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:34.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.113+0000 7f98f6a00640 1 -- 172.21.15.86:0/4026921447 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f98f00590e0 con 0x7f990006e4d0 2024-03-06T07:36:34.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.346+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f990004f330 con 0x7f990006e4d0 2024-03-06T07:36:34.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.346+0000 7f98f6a00640 1 -- 172.21.15.86:0/4026921447 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f98f00589e0 con 0x7f990006e4d0 2024-03-06T07:36:34.349 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:34.349 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:34.349 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f98dc06b4d0 msgr2=0x7f98dc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f98dc06b4d0 0x7f98dc06d990 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f98e8036000 tx=0x7f98e8008030 comp rx=0 tx=0).stop 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f990006e4d0 msgr2=0x7f99000ce620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f990006e4d0 0x7f99000ce620 secure :-1 s=READY pgs=1659 cs=0 l=1 rev1=1 crypto rx=0x7f98f000c7c0 tx=0x7f98f000cc80 comp rx=0 tx=0).stop 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 shutdown_connections 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f98dc06b4d0 0x7f98dc06d990 unknown :-1 s=CLOSED pgs=707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f990015eff0 0x7f99000af810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f99000af1c0 0x7f99001f3fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:34.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 --2- 172.21.15.86:0/4026921447 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f990006e4d0 0x7f99000ce620 unknown :-1 s=CLOSED pgs=1659 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:34.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 >> 172.21.15.86:0/4026921447 conn(0x7f99000b1730 msgr2=0x7f9900074030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:34.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.349+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 shutdown_connections 2024-03-06T07:36:34.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:34.350+0000 7f9904a00640 1 -- 172.21.15.86:0/4026921447 wait complete. 2024-03-06T07:36:34.570 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:34.570 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:35.571 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:35.608 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:35.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.781+0000 7f1461c00640 1 Processor -- start 2024-03-06T07:36:35.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.781+0000 7f1461c00640 1 -- start start 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f1461c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0cff20 0x7f145c0b6ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f1461c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f145c0b7860 0x7f145c0d0a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f1461c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f145c0b5e00 con 0x7f145c0b73e0 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f1461c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f145c0b6a10 con 0x7f145c0b7860 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f1461c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f145c0b60e0 con 0x7f145c0cff20 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f145b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f145c0b7860 0x7f145c0d0a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f145b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f145c0b7860 0x7f145c0d0a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50856/0 (socket says 172.21.15.86:50856) 2024-03-06T07:36:35.786 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.783+0000 7f145b400640 1 -- 172.21.15.86:0/263362142 learned_addr learned my addr 172.21.15.86:0/263362142 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:35.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.784+0000 7f145aa00640 1 --2- 172.21.15.86:0/263362142 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0cff20 0x7f145c0b6ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:35.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.784+0000 7f145b400640 1 -- 172.21.15.86:0/263362142 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0cff20 msgr2=0x7f145c0b6ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:35.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.784+0000 7f145b400640 1 --2- 172.21.15.86:0/263362142 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0cff20 0x7f145c0b6ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:35.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.784+0000 7f145b400640 1 -- 172.21.15.86:0/263362142 >> v1:172.21.15.86:6789/0 conn(0x7f145c0b73e0 legacy=0x7f145c0d0300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:35.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.784+0000 7f145b400640 1 -- 172.21.15.86:0/263362142 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f145c0b63c0 con 0x7f145c0b7860 2024-03-06T07:36:35.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.784+0000 7f145a000640 1 -- 172.21.15.86:0/263362142 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3844726014 0 0) 0x7f145c0b5e00 con 0x7f145c0b73e0 2024-03-06T07:36:35.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.785+0000 7f145b400640 1 --2- 172.21.15.86:0/263362142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c0d0a10 secure :-1 s=READY pgs=1661 cs=0 l=1 rev1=1 crypto rx=0x7f1454009e80 tx=0x7f145402bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=65310388d4fb8eff server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:35.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.785+0000 7f145a000640 1 -- 172.21.15.86:0/263362142 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f145402caf0 con 0x7f145c0b7860 2024-03-06T07:36:35.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.785+0000 7f145a000640 1 -- 172.21.15.86:0/263362142 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f145402cc90 con 0x7f145c0b7860 2024-03-06T07:36:35.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.785+0000 7f145a000640 1 -- 172.21.15.86:0/263362142 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1454034610 con 0x7f145c0b7860 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 -- 172.21.15.86:0/263362142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 msgr2=0x7f145c0d0a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 --2- 172.21.15.86:0/263362142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c0d0a10 secure :-1 s=READY pgs=1661 cs=0 l=1 rev1=1 crypto rx=0x7f1454009e80 tx=0x7f145402bf50 comp rx=0 tx=0).stop 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 -- 172.21.15.86:0/263362142 shutdown_connections 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 --2- 172.21.15.86:0/263362142 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c0d0a10 unknown :-1 s=CLOSED pgs=1661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 --2- 172.21.15.86:0/263362142 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0cff20 0x7f145c0b6ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 -- 172.21.15.86:0/263362142 >> 172.21.15.86:0/263362142 conn(0x7f145c0d1c90 msgr2=0x7f145c0d20a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 -- 172.21.15.86:0/263362142 shutdown_connections 2024-03-06T07:36:35.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.786+0000 7f1461c00640 1 -- 172.21.15.86:0/263362142 wait complete. 2024-03-06T07:36:35.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.788+0000 7f1461c00640 1 Processor -- start 2024-03-06T07:36:35.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.788+0000 7f1461c00640 1 -- start start 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1461c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0b73e0 0x7f145c14b1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1461c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c09f390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1461c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f145c0cff20 0x7f145c14bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1461c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f145c161a50 con 0x7f145c0b7860 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1461c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f145c0b5af0 con 0x7f145c0cff20 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1461c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f145c0d0f50 con 0x7f145c0b73e0 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1452000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c09f390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1452000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c09f390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:50858/0 (socket says 172.21.15.86:50858) 2024-03-06T07:36:35.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.789+0000 7f1452000640 1 -- 172.21.15.86:0/1248567575 learned_addr learned my addr 172.21.15.86:0/1248567575 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f145aa00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0b73e0 0x7f145c14b1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1452000640 1 -- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0b73e0 msgr2=0x7f145c14b1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1452000640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0b73e0 0x7f145c14b1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1452000640 1 -- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f145c0cff20 msgr2=0x7f145c14bb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f145b400640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f145c0cff20 0x7f145c14bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1452000640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f145c0cff20 0x7f145c14bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1452000640 1 -- 172.21.15.86:0/1248567575 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f145c157e00 con 0x7f145c0b7860 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1452000640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c09f390 secure :-1 s=READY pgs=1662 cs=0 l=1 rev1=1 crypto rx=0x7f144400c4b0 tx=0x7f144400c970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1453400640 1 -- 172.21.15.86:0/1248567575 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1444017070 con 0x7f145c0b7860 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1453400640 1 -- 172.21.15.86:0/1248567575 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1444005480 con 0x7f145c0b7860 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1453400640 1 -- 172.21.15.86:0/1248567575 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1444003830 con 0x7f145c0b7860 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.790+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f145c1580e0 con 0x7f145c0b7860 2024-03-06T07:36:35.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.791+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f145c1586a0 con 0x7f145c0b7860 2024-03-06T07:36:35.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.792+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f145c04f330 con 0x7f145c0b7860 2024-03-06T07:36:35.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.793+0000 7f1453400640 1 -- 172.21.15.86:0/1248567575 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f144401a070 con 0x7f145c0b7860 2024-03-06T07:36:35.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.794+0000 7f1453400640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f143806b4d0 0x7f143806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:35.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.794+0000 7f1453400640 1 -- 172.21.15.86:0/1248567575 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1444089560 con 0x7f145c0b7860 2024-03-06T07:36:35.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.794+0000 7f145aa00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f143806b4d0 0x7f143806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:35.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.794+0000 7f145aa00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f143806b4d0 0x7f143806d990 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7f144c006fd0 tx=0x7f144c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:35.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:35.798+0000 7f1453400640 1 -- 172.21.15.86:0/1248567575 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1444058430 con 0x7f145c0b7860 2024-03-06T07:36:36.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.036+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f145c0684a0 con 0x7f145c0b7860 2024-03-06T07:36:36.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.037+0000 7f1453400640 1 -- 172.21.15.86:0/1248567575 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f1444057d30 con 0x7f145c0b7860 2024-03-06T07:36:36.040 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:36.040 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:36.040 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:36.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f143806b4d0 msgr2=0x7f143806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f143806b4d0 0x7f143806d990 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7f144c006fd0 tx=0x7f144c008040 comp rx=0 tx=0).stop 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 msgr2=0x7f145c09f390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c09f390 secure :-1 s=READY pgs=1662 cs=0 l=1 rev1=1 crypto rx=0x7f144400c4b0 tx=0x7f144400c970 comp rx=0 tx=0).stop 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 shutdown_connections 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f143806b4d0 0x7f143806d990 unknown :-1 s=CLOSED pgs=708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f145c0cff20 0x7f145c14bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f145c0b7860 0x7f145c09f390 unknown :-1 s=CLOSED pgs=1662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 --2- 172.21.15.86:0/1248567575 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f145c0b73e0 0x7f145c14b1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 >> 172.21.15.86:0/1248567575 conn(0x7f145c0d1c90 msgr2=0x7f145c070230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 shutdown_connections 2024-03-06T07:36:36.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:36.040+0000 7f1461c00640 1 -- 172.21.15.86:0/1248567575 wait complete. 2024-03-06T07:36:36.262 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:36.262 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:37.263 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:37.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.464+0000 7fbd6dc00640 1 Processor -- start 2024-03-06T07:36:37.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.464+0000 7fbd6dc00640 1 -- start start 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.465+0000 7fbd6dc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.465+0000 7fbd6dc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbd680b0aa0 0x7fbd680b0f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.465+0000 7fbd6dc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbd680d4c50 con 0x7fbd680b2de0 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.465+0000 7fbd6dc00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbd680d4a00 con 0x7fbd680b0aa0 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.465+0000 7fbd6dc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbd680cf7e0 con 0x7fbd680d24c0 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd5e000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fbd680b2de0 0x7fbd680d28a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:45190/0 (socket says 172.21.15.86:45190) 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd5e000640 1 -- 172.21.15.86:0/900855636 learned_addr learned my addr 172.21.15.86:0/900855636 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd67400640 1 --2- 172.21.15.86:0/900855636 >> v2:172.21.15.86:3300/0 conn(0x7fbd680b0aa0 0x7fbd680b0f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd66a00640 1 --2- 172.21.15.86:0/900855636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd66000640 1 -- 172.21.15.86:0/900855636 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1485556945 0 0) 0x7fbd680d4c50 con 0x7fbd680b2de0 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd66000640 1 -- 172.21.15.86:0/900855636 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbd4c003520 con 0x7fbd680b2de0 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd67400640 1 -- 172.21.15.86:0/900855636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 msgr2=0x7fbd680d1cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd67400640 1 --2- 172.21.15.86:0/900855636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680d1cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd67400640 1 -- 172.21.15.86:0/900855636 >> v1:172.21.15.86:6789/0 conn(0x7fbd680b2de0 legacy=0x7fbd680d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:37.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.466+0000 7fbd67400640 1 -- 172.21.15.86:0/900855636 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd680cfa30 con 0x7fbd680b0aa0 2024-03-06T07:36:37.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.467+0000 7fbd67400640 1 --2- 172.21.15.86:0/900855636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b0aa0 0x7fbd680b0f30 secure :-1 s=READY pgs=1664 cs=0 l=1 rev1=1 crypto rx=0x7fbd60009d90 tx=0x7fbd6002be20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=51d90b27b9740c1e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:37.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd66000640 1 -- 172.21.15.86:0/900855636 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd60039070 con 0x7fbd680b0aa0 2024-03-06T07:36:37.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd66000640 1 -- 172.21.15.86:0/900855636 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbd6002cbd0 con 0x7fbd680b0aa0 2024-03-06T07:36:37.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd66000640 1 -- 172.21.15.86:0/900855636 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd60034440 con 0x7fbd680b0aa0 2024-03-06T07:36:37.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd6dc00640 1 -- 172.21.15.86:0/900855636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b0aa0 msgr2=0x7fbd680b0f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:37.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/900855636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b0aa0 0x7fbd680b0f30 secure :-1 s=READY pgs=1664 cs=0 l=1 rev1=1 crypto rx=0x7fbd60009d90 tx=0x7fbd6002be20 comp rx=0 tx=0).stop 2024-03-06T07:36:37.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd6dc00640 1 -- 172.21.15.86:0/900855636 shutdown_connections 2024-03-06T07:36:37.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/900855636 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b0aa0 0x7fbd680b0f30 unknown :-1 s=CLOSED pgs=1664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/900855636 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680d1cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.468+0000 7fbd6dc00640 1 -- 172.21.15.86:0/900855636 >> 172.21.15.86:0/900855636 conn(0x7fbd680d3ee0 msgr2=0x7fbd680d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:37.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.469+0000 7fbd6dc00640 1 -- 172.21.15.86:0/900855636 shutdown_connections 2024-03-06T07:36:37.472 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.469+0000 7fbd6dc00640 1 -- 172.21.15.86:0/900855636 wait complete. 2024-03-06T07:36:37.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.470+0000 7fbd6dc00640 1 Processor -- start 2024-03-06T07:36:37.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.470+0000 7fbd6dc00640 1 -- start start 2024-03-06T07:36:37.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd6dc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbd680b0aa0 0x7fbd68153720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:37.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd6dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b2de0 0x7fbd680cafb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:37.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd6dc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680c9d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:37.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd6dc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd6815e5b0 con 0x7fbd680b2de0 2024-03-06T07:36:37.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd6dc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd680d0080 con 0x7fbd680b0aa0 2024-03-06T07:36:37.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd6dc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbd6815e730 con 0x7fbd680d24c0 2024-03-06T07:36:37.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd5e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b2de0 0x7fbd680cafb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd5e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b2de0 0x7fbd680cafb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49150/0 (socket says 172.21.15.86:49150) 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.472+0000 7fbd5e000640 1 -- 172.21.15.86:0/3296255021 learned_addr learned my addr 172.21.15.86:0/3296255021 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd66a00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbd680b0aa0 0x7fbd68153720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd5e000640 1 -- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 msgr2=0x7fbd680c9d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd67400640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680c9d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd5e000640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680c9d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd5e000640 1 -- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbd680b0aa0 msgr2=0x7fbd68153720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd5e000640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbd680b0aa0 0x7fbd68153720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd5e000640 1 -- 172.21.15.86:0/3296255021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd68153c60 con 0x7fbd680b2de0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd66a00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbd680b0aa0 0x7fbd68153720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd5e000640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b2de0 0x7fbd680cafb0 secure :-1 s=READY pgs=1665 cs=0 l=1 rev1=1 crypto rx=0x7fbd5000c390 tx=0x7fbd5000c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.473+0000 7fbd5f400640 1 -- 172.21.15.86:0/3296255021 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd50017070 con 0x7fbd680b2de0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.474+0000 7fbd5f400640 1 -- 172.21.15.86:0/3296255021 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbd50005ce0 con 0x7fbd680b2de0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.474+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbd680c8900 con 0x7fbd680b2de0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.474+0000 7fbd5f400640 1 -- 172.21.15.86:0/3296255021 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd500072f0 con 0x7fbd680b2de0 2024-03-06T07:36:37.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.474+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd680b8180 con 0x7fbd680b2de0 2024-03-06T07:36:37.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.475+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbd6804f330 con 0x7fbd680b2de0 2024-03-06T07:36:37.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.476+0000 7fbd5f400640 1 -- 172.21.15.86:0/3296255021 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbd5001a070 con 0x7fbd680b2de0 2024-03-06T07:36:37.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.477+0000 7fbd5f400640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbd4406b500 0x7fbd4406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:37.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.477+0000 7fbd5f400640 1 -- 172.21.15.86:0/3296255021 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbd50089690 con 0x7fbd680b2de0 2024-03-06T07:36:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.477+0000 7fbd66a00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbd4406b500 0x7fbd4406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:37.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.478+0000 7fbd66a00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbd4406b500 0x7fbd4406d9c0 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7fbd58006fd0 tx=0x7fbd58008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:37.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.485+0000 7fbd5f400640 1 -- 172.21.15.86:0/3296255021 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbd50058430 con 0x7fbd680b2de0 2024-03-06T07:36:37.610 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:37.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.718+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbd680d22a0 con 0x7fbd680b2de0 2024-03-06T07:36:37.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.720+0000 7fbd5f400640 1 -- 172.21.15.86:0/3296255021 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fbd50057d30 con 0x7fbd680b2de0 2024-03-06T07:36:37.722 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:37.722 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:37.723 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbd4406b500 msgr2=0x7fbd4406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbd4406b500 0x7fbd4406d9c0 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7fbd58006fd0 tx=0x7fbd58008040 comp rx=0 tx=0).stop 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b2de0 msgr2=0x7fbd680cafb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b2de0 0x7fbd680cafb0 secure :-1 s=READY pgs=1665 cs=0 l=1 rev1=1 crypto rx=0x7fbd5000c390 tx=0x7fbd5000c850 comp rx=0 tx=0).stop 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 shutdown_connections 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbd4406b500 0x7fbd4406d9c0 unknown :-1 s=CLOSED pgs=709 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbd680d24c0 0x7fbd680c9d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbd680b2de0 0x7fbd680cafb0 unknown :-1 s=CLOSED pgs=1665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 --2- 172.21.15.86:0/3296255021 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbd680b0aa0 0x7fbd68153720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 >> 172.21.15.86:0/3296255021 conn(0x7fbd680d3ee0 msgr2=0x7fbd6815d7a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:37.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 shutdown_connections 2024-03-06T07:36:37.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:37.722+0000 7fbd6dc00640 1 -- 172.21.15.86:0/3296255021 wait complete. 2024-03-06T07:36:37.946 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:37.946 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:38.947 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:39.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.157+0000 7fa1e8e00640 1 Processor -- start 2024-03-06T07:36:39.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.157+0000 7fa1e8e00640 1 -- start start 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.158+0000 7fa1e8e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa1e415acb0 0x7fa1e40b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e8e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa1e406ec00 0x7fa1e40ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e8e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa1e415b090 con 0x7fa1e40b4680 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e8e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa1e40b2a30 con 0x7fa1e406ec00 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e8e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa1e40b4360 con 0x7fa1e415acb0 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa1e406ec00 0x7fa1e40ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa1e406ec00 0x7fa1e40ccfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49166/0 (socket says 172.21.15.86:49166) 2024-03-06T07:36:39.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e2800640 1 -- 172.21.15.86:0/1537589089 learned_addr learned my addr 172.21.15.86:0/1537589089 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:39.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e1e00640 1 --2- 172.21.15.86:0/1537589089 >> v2:172.21.15.194:3300/0 conn(0x7fa1e415acb0 0x7fa1e40b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:39.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e2800640 1 -- 172.21.15.86:0/1537589089 >> v2:172.21.15.194:3300/0 conn(0x7fa1e415acb0 msgr2=0x7fa1e40b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:39.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e2800640 1 --2- 172.21.15.86:0/1537589089 >> v2:172.21.15.194:3300/0 conn(0x7fa1e415acb0 0x7fa1e40b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e2800640 1 -- 172.21.15.86:0/1537589089 >> v1:172.21.15.194:6789/0 conn(0x7fa1e40b4680 legacy=0x7fa1e406e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:39.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.159+0000 7fa1e2800640 1 -- 172.21.15.86:0/1537589089 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1e40b2cf0 con 0x7fa1e406ec00 2024-03-06T07:36:39.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.161+0000 7fa1e2800640 1 --2- 172.21.15.86:0/1537589089 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e406ec00 0x7fa1e40ccfb0 secure :-1 s=READY pgs=1666 cs=0 l=1 rev1=1 crypto rx=0x7fa1e40b27f0 tx=0x7fa1dc02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9cb739698f5a42f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:39.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.161+0000 7fa1e0a00640 1 -- 172.21.15.86:0/1537589089 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1dc02ca80 con 0x7fa1e406ec00 2024-03-06T07:36:39.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.161+0000 7fa1e0a00640 1 -- 172.21.15.86:0/1537589089 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa1dc02cc20 con 0x7fa1e406ec00 2024-03-06T07:36:39.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.161+0000 7fa1e0a00640 1 -- 172.21.15.86:0/1537589089 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1dc0343f0 con 0x7fa1e406ec00 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 -- 172.21.15.86:0/1537589089 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e406ec00 msgr2=0x7fa1e40ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/1537589089 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e406ec00 0x7fa1e40ccfb0 secure :-1 s=READY pgs=1666 cs=0 l=1 rev1=1 crypto rx=0x7fa1e40b27f0 tx=0x7fa1dc02bee0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 -- 172.21.15.86:0/1537589089 shutdown_connections 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/1537589089 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e406ec00 0x7fa1e40ccfb0 unknown :-1 s=CLOSED pgs=1666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/1537589089 >> v2:172.21.15.194:3300/0 conn(0x7fa1e415acb0 0x7fa1e40b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 -- 172.21.15.86:0/1537589089 >> 172.21.15.86:0/1537589089 conn(0x7fa1e40b1730 msgr2=0x7fa1e40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 -- 172.21.15.86:0/1537589089 shutdown_connections 2024-03-06T07:36:39.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.162+0000 7fa1e8e00640 1 -- 172.21.15.86:0/1537589089 wait complete. 2024-03-06T07:36:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.163+0000 7fa1e8e00640 1 Processor -- start 2024-03-06T07:36:39.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.164+0000 7fa1e8e00640 1 -- start start 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.164+0000 7fa1e8e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1e406ec00 0x7fa1e40b0be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e40b4680 0x7fa1e4144db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e8e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1e415acb0 0x7fa1e414aff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e8e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1e40b3d10 con 0x7fa1e40b4680 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e8e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1e40b32b0 con 0x7fa1e415acb0 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e8e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa1e40b4010 con 0x7fa1e406ec00 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e40b4680 0x7fa1e4144db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e40b4680 0x7fa1e4144db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49168/0 (socket says 172.21.15.86:49168) 2024-03-06T07:36:39.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 -- 172.21.15.86:0/3668278666 learned_addr learned my addr 172.21.15.86:0/3668278666 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1e406ec00 0x7fa1e40b0be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e2800640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1e415acb0 0x7fa1e414aff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 -- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1e406ec00 msgr2=0x7fa1e40b0be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1e406ec00 0x7fa1e40b0be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 -- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1e415acb0 msgr2=0x7fa1e414aff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1e415acb0 0x7fa1e414aff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e1400640 1 -- 172.21.15.86:0/3668278666 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1e40cf0c0 con 0x7fa1e40b4680 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.165+0000 7fa1e2800640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1e415acb0 0x7fa1e414aff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:39.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.166+0000 7fa1e1400640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e40b4680 0x7fa1e4144db0 secure :-1 s=READY pgs=1667 cs=0 l=1 rev1=1 crypto rx=0x7fa1cc00b3e0 tx=0x7fa1cc00b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:39.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.166+0000 7fa1da000640 1 -- 172.21.15.86:0/3668278666 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1cc015020 con 0x7fa1e40b4680 2024-03-06T07:36:39.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.166+0000 7fa1da000640 1 -- 172.21.15.86:0/3668278666 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa1cc011850 con 0x7fa1e40b4680 2024-03-06T07:36:39.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.166+0000 7fa1da000640 1 -- 172.21.15.86:0/3668278666 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1cc011c40 con 0x7fa1e40b4680 2024-03-06T07:36:39.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.166+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa1e40ceb00 con 0x7fa1e40b4680 2024-03-06T07:36:39.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.166+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa1e415cf50 con 0x7fa1e40b4680 2024-03-06T07:36:39.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.167+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa1e404f330 con 0x7fa1e40b4680 2024-03-06T07:36:39.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.168+0000 7fa1da000640 1 -- 172.21.15.86:0/3668278666 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa1cc018020 con 0x7fa1e40b4680 2024-03-06T07:36:39.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.170+0000 7fa1da000640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa1c006b4d0 0x7fa1c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:39.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.170+0000 7fa1da000640 1 -- 172.21.15.86:0/3668278666 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa1cc08a1f0 con 0x7fa1e40b4680 2024-03-06T07:36:39.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.170+0000 7fa1e1e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa1c006b4d0 0x7fa1c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:39.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.171+0000 7fa1e1e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa1c006b4d0 0x7fa1c006d990 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7fa1d4006fd0 tx=0x7fa1d4008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:39.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.174+0000 7fa1da000640 1 -- 172.21.15.86:0/3668278666 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa1cc01f020 con 0x7fa1e40b4680 2024-03-06T07:36:39.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.456+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa1e406ce60 con 0x7fa1e40b4680 2024-03-06T07:36:39.459 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.457+0000 7fa1da000640 1 -- 172.21.15.86:0/3668278666 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fa1cc0172c0 con 0x7fa1e40b4680 2024-03-06T07:36:39.460 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:39.460 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:39.460 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:39.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.459+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa1c006b4d0 msgr2=0x7fa1c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:39.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.459+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa1c006b4d0 0x7fa1c006d990 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7fa1d4006fd0 tx=0x7fa1d4008040 comp rx=0 tx=0).stop 2024-03-06T07:36:39.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.459+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e40b4680 msgr2=0x7fa1e4144db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:39.462 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.459+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e40b4680 0x7fa1e4144db0 secure :-1 s=READY pgs=1667 cs=0 l=1 rev1=1 crypto rx=0x7fa1cc00b3e0 tx=0x7fa1cc00b8a0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 shutdown_connections 2024-03-06T07:36:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa1c006b4d0 0x7fa1c006d990 unknown :-1 s=CLOSED pgs=710 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa1e415acb0 0x7fa1e414aff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.463 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa1e40b4680 0x7fa1e4144db0 unknown :-1 s=CLOSED pgs=1667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 --2- 172.21.15.86:0/3668278666 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa1e406ec00 0x7fa1e40b0be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:39.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 >> 172.21.15.86:0/3668278666 conn(0x7fa1e40b1730 msgr2=0x7fa1e4071180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:39.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 shutdown_connections 2024-03-06T07:36:39.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:39.460+0000 7fa1e8e00640 1 -- 172.21.15.86:0/3668278666 wait complete. 2024-03-06T07:36:39.612 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:39.683 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:39.683 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:40.684 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:40.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.897+0000 7fd649c00640 1 Processor -- start 2024-03-06T07:36:40.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.897+0000 7fd649c00640 1 -- start start 2024-03-06T07:36:40.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd649c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd64415acb0 0x7fd6440b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd649c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd6440b4680 0x7fd64406e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd649c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd64415b090 con 0x7fd644072340 2024-03-06T07:36:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd649c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6440b2a30 con 0x7fd6440b4680 2024-03-06T07:36:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd649c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd6440b4360 con 0x7fd64415acb0 2024-03-06T07:36:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd642000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd6440b4680 0x7fd64406e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd642000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd6440b4680 0x7fd64406e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49170/0 (socket says 172.21.15.86:49170) 2024-03-06T07:36:40.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd642000640 1 -- 172.21.15.86:0/2773989727 learned_addr learned my addr 172.21.15.86:0/2773989727 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:40.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.899+0000 7fd642a00640 1 --2- 172.21.15.86:0/2773989727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd64415acb0 0x7fd6440b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:40.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.900+0000 7fd642000640 1 -- 172.21.15.86:0/2773989727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd64415acb0 msgr2=0x7fd6440b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:40.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.900+0000 7fd642000640 1 --2- 172.21.15.86:0/2773989727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd64415acb0 0x7fd6440b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:40.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.900+0000 7fd642000640 1 -- 172.21.15.86:0/2773989727 >> v1:172.21.15.194:6789/0 conn(0x7fd644072340 legacy=0x7fd6440ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:40.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.900+0000 7fd642000640 1 -- 172.21.15.86:0/2773989727 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6440b2cf0 con 0x7fd6440b4680 2024-03-06T07:36:40.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.901+0000 7fd642000640 1 --2- 172.21.15.86:0/2773989727 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd64406e4f0 secure :-1 s=READY pgs=1668 cs=0 l=1 rev1=1 crypto rx=0x7fd630009870 tx=0x7fd63002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fa50d5303879446b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:40.903 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.901+0000 7fd641600640 1 -- 172.21.15.86:0/2773989727 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd630035020 con 0x7fd6440b4680 2024-03-06T07:36:40.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.901+0000 7fd641600640 1 -- 172.21.15.86:0/2773989727 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd630031640 con 0x7fd6440b4680 2024-03-06T07:36:40.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.901+0000 7fd641600640 1 -- 172.21.15.86:0/2773989727 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd63002d040 con 0x7fd6440b4680 2024-03-06T07:36:40.904 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 -- 172.21.15.86:0/2773989727 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 msgr2=0x7fd64406e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 --2- 172.21.15.86:0/2773989727 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd64406e4f0 secure :-1 s=READY pgs=1668 cs=0 l=1 rev1=1 crypto rx=0x7fd630009870 tx=0x7fd63002b8f0 comp rx=0 tx=0).stop 2024-03-06T07:36:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 -- 172.21.15.86:0/2773989727 shutdown_connections 2024-03-06T07:36:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 --2- 172.21.15.86:0/2773989727 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd64406e4f0 unknown :-1 s=CLOSED pgs=1668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 --2- 172.21.15.86:0/2773989727 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd64415acb0 0x7fd6440b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 -- 172.21.15.86:0/2773989727 >> 172.21.15.86:0/2773989727 conn(0x7fd6440b1730 msgr2=0x7fd6440b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 -- 172.21.15.86:0/2773989727 shutdown_connections 2024-03-06T07:36:40.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.902+0000 7fd649c00640 1 -- 172.21.15.86:0/2773989727 wait complete. 2024-03-06T07:36:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.903+0000 7fd649c00640 1 Processor -- start 2024-03-06T07:36:40.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.904+0000 7fd649c00640 1 -- start start 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd649c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd644072340 0x7fd6440713f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd649c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd6440ce7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd649c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd64415acb0 0x7fd6440ced10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd649c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6440b3d10 con 0x7fd6440b4680 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd649c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6440b32b0 con 0x7fd64415acb0 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd649c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd6440b4010 con 0x7fd644072340 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd6440ce7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd6440ce7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49174/0 (socket says 172.21.15.86:49174) 2024-03-06T07:36:40.908 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 -- 172.21.15.86:0/1295121351 learned_addr learned my addr 172.21.15.86:0/1295121351 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642a00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd644072340 0x7fd6440713f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd643400640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd64415acb0 0x7fd6440ced10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 -- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd644072340 msgr2=0x7fd6440713f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd644072340 0x7fd6440713f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 -- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd64415acb0 msgr2=0x7fd6440ced10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd64415acb0 0x7fd6440ced10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:40.909 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.905+0000 7fd642000640 1 -- 172.21.15.86:0/1295121351 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd64415d250 con 0x7fd6440b4680 2024-03-06T07:36:40.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.906+0000 7fd642000640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd6440ce7d0 secure :-1 s=READY pgs=1669 cs=0 l=1 rev1=1 crypto rx=0x7fd63002b930 tx=0x7fd6300046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:40.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.906+0000 7fd63aa00640 1 -- 172.21.15.86:0/1295121351 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd630040070 con 0x7fd6440b4680 2024-03-06T07:36:40.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.906+0000 7fd63aa00640 1 -- 172.21.15.86:0/1295121351 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd630039c10 con 0x7fd6440b4680 2024-03-06T07:36:40.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.906+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd64415d4d0 con 0x7fd6440b4680 2024-03-06T07:36:40.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.906+0000 7fd63aa00640 1 -- 172.21.15.86:0/1295121351 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd630035040 con 0x7fd6440b4680 2024-03-06T07:36:40.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.907+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd64415da70 con 0x7fd6440b4680 2024-03-06T07:36:40.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.908+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd64404f330 con 0x7fd6440b4680 2024-03-06T07:36:40.912 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.910+0000 7fd63aa00640 1 -- 172.21.15.86:0/1295121351 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd63004b060 con 0x7fd6440b4680 2024-03-06T07:36:40.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.910+0000 7fd63aa00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd62406b4d0 0x7fd62406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:40.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.911+0000 7fd63aa00640 1 -- 172.21.15.86:0/1295121351 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd6300abdd0 con 0x7fd6440b4680 2024-03-06T07:36:40.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.911+0000 7fd642a00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd62406b4d0 0x7fd62406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:40.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.912+0000 7fd642a00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd62406b4d0 0x7fd62406d990 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7fd63400a970 tx=0x7fd634008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:40.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:40.915+0000 7fd63aa00640 1 -- 172.21.15.86:0/1295121351 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd64415d250 con 0x7fd6440b4680 2024-03-06T07:36:41.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.149+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd644072120 con 0x7fd6440b4680 2024-03-06T07:36:41.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.150+0000 7fd63aa00640 1 -- 172.21.15.86:0/1295121351 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fd6300ad090 con 0x7fd6440b4680 2024-03-06T07:36:41.153 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:41.153 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:41.153 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:41.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd62406b4d0 msgr2=0x7fd62406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:41.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd62406b4d0 0x7fd62406d990 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7fd63400a970 tx=0x7fd634008040 comp rx=0 tx=0).stop 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 msgr2=0x7fd6440ce7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd6440ce7d0 secure :-1 s=READY pgs=1669 cs=0 l=1 rev1=1 crypto rx=0x7fd63002b930 tx=0x7fd6300046c0 comp rx=0 tx=0).stop 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 shutdown_connections 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd62406b4d0 0x7fd62406d990 unknown :-1 s=CLOSED pgs=711 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd64415acb0 0x7fd6440ced10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6440b4680 0x7fd6440ce7d0 unknown :-1 s=CLOSED pgs=1669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.153+0000 7fd649c00640 1 --2- 172.21.15.86:0/1295121351 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd644072340 0x7fd6440713f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.154+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 >> 172.21.15.86:0/1295121351 conn(0x7fd6440b1730 msgr2=0x7fd6440af940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.154+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 shutdown_connections 2024-03-06T07:36:41.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:41.154+0000 7fd649c00640 1 -- 172.21.15.86:0/1295121351 wait complete. 2024-03-06T07:36:41.376 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:41.376 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:41.614 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:42.377 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:42.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.577+0000 7f9f19a00640 1 Processor -- start 2024-03-06T07:36:42.580 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.577+0000 7f9f19a00640 1 -- start start 2024-03-06T07:36:42.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.579+0000 7f9f19a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 0x7f9f14162620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.579+0000 7f9f19a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f14162b60 con 0x7f9f140b3ac0 2024-03-06T07:36:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.579+0000 7f9f19a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f1406f1f0 con 0x7f9f140af820 2024-03-06T07:36:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.579+0000 7f9f19a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f1406b0f0 con 0x7f9f1415e160 2024-03-06T07:36:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.579+0000 7f9f13400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9f140b3ac0 0x7f9f1406b580 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:45194/0 (socket says 172.21.15.86:45194) 2024-03-06T07:36:42.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.579+0000 7f9f13400640 1 -- 172.21.15.86:0/359757693 learned_addr learned my addr 172.21.15.86:0/359757693 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12a00640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 0x7f9f14162620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 256193726 0 0) 0x7f9f0c001110 con 0x7f9f140b3ac0 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9ef8003520 con 0x7f9f140b3ac0 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3107954708 0 0) 0x7f9ef8003520 con 0x7f9f140b3ac0 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 msgr2=0x7f9f14162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 0x7f9f14162620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 >> v1:172.21.15.194:6789/0 conn(0x7f9f140af820 legacy=0x7f9f140b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f140afce0 con 0x7f9f140b3ac0 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9f0c001840 con 0x7f9f140b3ac0 2024-03-06T07:36:42.584 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 >> v1:172.21.15.86:6789/0 conn(0x7f9f140b3ac0 legacy=0x7f9f1406b580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ef80039e0 0x7f9ef8003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 0x7f9ef8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef8004d40 0x7f9ef80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f1406b0f0 con 0x7f9ef80039e0 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ef800bb50 con 0x7f9ef8004d40 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9ef800bd50 con 0x7f9f1415e160 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.580+0000 7f9f12a00640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ef80039e0 0x7f9ef8003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f13400640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef8004d40 0x7f9ef80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f12a00640 1 -- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 msgr2=0x7f9ef8004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f0a000640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 0x7f9ef8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f12a00640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415e160 0x7f9ef8004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f12a00640 1 -- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef8004d40 msgr2=0x7f9ef80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f12a00640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef8004d40 0x7f9ef80051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f12a00640 1 -- 172.21.15.86:0/359757693 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ef8003520 con 0x7f9ef80039e0 2024-03-06T07:36:42.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.581+0000 7f9f13400640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef8004d40 0x7f9ef80051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.582+0000 7f9f12a00640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ef80039e0 0x7f9ef8003e00 secure :-1 s=READY pgs=1671 cs=0 l=1 rev1=1 crypto rx=0x7f9f04009fc0 tx=0x7f9ef8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=97e7f48c1b32bbd4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.582+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f04039070 con 0x7f9ef80039e0 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.582+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f0402cc20 con 0x7f9ef80039e0 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.582+0000 7f9f12000640 1 -- 172.21.15.86:0/359757693 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f04009c10 con 0x7f9ef80039e0 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.583+0000 7f9f19a00640 1 -- 172.21.15.86:0/359757693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ef80039e0 msgr2=0x7f9ef8003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.583+0000 7f9f19a00640 1 --2- 172.21.15.86:0/359757693 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ef80039e0 0x7f9ef8003e00 secure :-1 s=READY pgs=1671 cs=0 l=1 rev1=1 crypto rx=0x7f9f04009fc0 tx=0x7f9ef8005830 comp rx=0 tx=0).stop 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.583+0000 7f9f13400640 1 -- 172.21.15.86:0/359757693 reap_dead start 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.583+0000 7f9f19a00640 1 -- 172.21.15.86:0/359757693 shutdown_connections 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.583+0000 7f9f19a00640 1 -- 172.21.15.86:0/359757693 >> 172.21.15.86:0/359757693 conn(0x7f9f140d0340 msgr2=0x7f9f140d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.583+0000 7f9f19a00640 1 -- 172.21.15.86:0/359757693 shutdown_connections 2024-03-06T07:36:42.587 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.583+0000 7f9f19a00640 1 -- 172.21.15.86:0/359757693 wait complete. 2024-03-06T07:36:42.588 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.586+0000 7f9f19a00640 1 Processor -- start 2024-03-06T07:36:42.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.586+0000 7f9f19a00640 1 -- start start 2024-03-06T07:36:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f19a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef80039e0 0x7f9f141f3dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f19a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f141f4300 0x7f9f1415f950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f19a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415fe90 0x7f9f14155030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f19a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f1406ef10 con 0x7f9f141f4300 2024-03-06T07:36:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f19a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f140b5620 con 0x7f9ef80039e0 2024-03-06T07:36:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f19a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f14162620 con 0x7f9f1415fe90 2024-03-06T07:36:42.591 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f141f4300 0x7f9f1415f950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f141f4300 0x7f9f1415f950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49194/0 (socket says 172.21.15.86:49194) 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 -- 172.21.15.86:0/2094555980 learned_addr learned my addr 172.21.15.86:0/2094555980 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 -- 172.21.15.86:0/2094555980 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415fe90 msgr2=0x7f9f14155030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415fe90 0x7f9f14155030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 -- 172.21.15.86:0/2094555980 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef80039e0 msgr2=0x7f9f141f3dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef80039e0 0x7f9f141f3dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.588+0000 7f9f0a000640 1 -- 172.21.15.86:0/2094555980 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f141e80c0 con 0x7f9f141f4300 2024-03-06T07:36:42.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.589+0000 7f9f0a000640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f141f4300 0x7f9f1415f950 secure :-1 s=READY pgs=1672 cs=0 l=1 rev1=1 crypto rx=0x7f9efc00d480 tx=0x7f9efc00d940 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.589+0000 7f9f0b400640 1 -- 172.21.15.86:0/2094555980 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9efc005480 con 0x7f9f141f4300 2024-03-06T07:36:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.589+0000 7f9f0b400640 1 -- 172.21.15.86:0/2094555980 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9efc017070 con 0x7f9f141f4300 2024-03-06T07:36:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.589+0000 7f9f0b400640 1 -- 172.21.15.86:0/2094555980 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9efc003830 con 0x7f9f141f4300 2024-03-06T07:36:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.589+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f141604a0 con 0x7f9f141f4300 2024-03-06T07:36:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.589+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f14160a60 con 0x7f9f141f4300 2024-03-06T07:36:42.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.591+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f14074ae0 con 0x7f9f141f4300 2024-03-06T07:36:42.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.591+0000 7f9f0b400640 1 -- 172.21.15.86:0/2094555980 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9efc0039d0 con 0x7f9f141f4300 2024-03-06T07:36:42.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.592+0000 7f9f0b400640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef006b500 0x7f9ef006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:42.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.592+0000 7f9f0b400640 1 -- 172.21.15.86:0/2094555980 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9efc089fd0 con 0x7f9f141f4300 2024-03-06T07:36:42.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.593+0000 7f9f12a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef006b500 0x7f9ef006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:42.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.593+0000 7f9f12a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef006b500 0x7f9ef006d9c0 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7f9f04036000 tx=0x7f9f0402e070 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:42.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.597+0000 7f9f0b400640 1 -- 172.21.15.86:0/2094555980 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9efc058ed0 con 0x7f9f141f4300 2024-03-06T07:36:42.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.831+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9f140b4230 con 0x7f9f141f4300 2024-03-06T07:36:42.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.832+0000 7f9f0b400640 1 -- 172.21.15.86:0/2094555980 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f9efc0587d0 con 0x7f9f141f4300 2024-03-06T07:36:42.834 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:42.834 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:42.835 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef006b500 msgr2=0x7f9ef006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef006b500 0x7f9ef006d9c0 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7f9f04036000 tx=0x7f9f0402e070 comp rx=0 tx=0).stop 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f141f4300 msgr2=0x7f9f1415f950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f141f4300 0x7f9f1415f950 secure :-1 s=READY pgs=1672 cs=0 l=1 rev1=1 crypto rx=0x7f9efc00d480 tx=0x7f9efc00d940 comp rx=0 tx=0).stop 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 shutdown_connections 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ef006b500 0x7f9ef006d9c0 unknown :-1 s=CLOSED pgs=712 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9f1415fe90 0x7f9f14155030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9f141f4300 0x7f9f1415f950 unknown :-1 s=CLOSED pgs=1672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 --2- 172.21.15.86:0/2094555980 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ef80039e0 0x7f9f141f3dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 >> 172.21.15.86:0/2094555980 conn(0x7f9f140d0340 msgr2=0x7f9f14071380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:42.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 shutdown_connections 2024-03-06T07:36:42.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:42.834+0000 7f9f19a00640 1 -- 172.21.15.86:0/2094555980 wait complete. 2024-03-06T07:36:43.056 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:43.056 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:43.616 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:44.056 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:44.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.262+0000 7fd697000640 1 Processor -- start 2024-03-06T07:36:44.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.262+0000 7fd697000640 1 -- start start 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd697000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd690144660 0x7fd6900b3b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd697000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd6900cf170 0x7fd690071c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd697000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6900cf5e0 0x7fd690072140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd697000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6900cfe30 con 0x7fd690144660 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd697000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6900cfb50 con 0x7fd6900cf170 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd697000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd69006ed90 con 0x7fd6900cf5e0 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd68fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd690144660 0x7fd6900b3b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd68fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd690144660 0x7fd6900b3b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49210/0 (socket says 172.21.15.86:49210) 2024-03-06T07:36:44.268 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.265+0000 7fd68fe00640 1 -- 172.21.15.86:0/2597764190 learned_addr learned my addr 172.21.15.86:0/2597764190 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:44.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd68f400640 1 --2- 172.21.15.86:0/2597764190 >> v2:172.21.15.194:3300/0 conn(0x7fd6900cf170 0x7fd690071c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:44.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd695000640 1 --2- 172.21.15.86:0/2597764190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6900cf5e0 0x7fd690072140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:44.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd68fe00640 1 -- 172.21.15.86:0/2597764190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6900cf5e0 msgr2=0x7fd690072140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:44.269 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd68fe00640 1 --2- 172.21.15.86:0/2597764190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6900cf5e0 0x7fd690072140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd68fe00640 1 -- 172.21.15.86:0/2597764190 >> v2:172.21.15.194:3300/0 conn(0x7fd6900cf170 msgr2=0x7fd690071c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:44.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd68fe00640 1 --2- 172.21.15.86:0/2597764190 >> v2:172.21.15.194:3300/0 conn(0x7fd6900cf170 0x7fd690071c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd68fe00640 1 -- 172.21.15.86:0/2597764190 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd69006f070 con 0x7fd690144660 2024-03-06T07:36:44.270 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.266+0000 7fd695000640 1 --2- 172.21.15.86:0/2597764190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6900cf5e0 0x7fd690072140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:44.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.267+0000 7fd68fe00640 1 --2- 172.21.15.86:0/2597764190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd690144660 0x7fd6900b3b40 secure :-1 s=READY pgs=1673 cs=0 l=1 rev1=1 crypto rx=0x7fd69006d870 tx=0x7fd68002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=adb24929f0fc6380 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:44.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd68ea00640 1 -- 172.21.15.86:0/2597764190 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd680035020 con 0x7fd690144660 2024-03-06T07:36:44.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd68ea00640 1 -- 172.21.15.86:0/2597764190 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd680031870 con 0x7fd690144660 2024-03-06T07:36:44.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd68ea00640 1 -- 172.21.15.86:0/2597764190 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd680031c40 con 0x7fd690144660 2024-03-06T07:36:44.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd697000640 1 -- 172.21.15.86:0/2597764190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd690144660 msgr2=0x7fd6900b3b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:44.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd697000640 1 --2- 172.21.15.86:0/2597764190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd690144660 0x7fd6900b3b40 secure :-1 s=READY pgs=1673 cs=0 l=1 rev1=1 crypto rx=0x7fd69006d870 tx=0x7fd68002b910 comp rx=0 tx=0).stop 2024-03-06T07:36:44.271 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd697000640 1 -- 172.21.15.86:0/2597764190 shutdown_connections 2024-03-06T07:36:44.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd697000640 1 --2- 172.21.15.86:0/2597764190 >> v2:172.21.15.194:3300/0 conn(0x7fd6900cf170 0x7fd690071c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd697000640 1 --2- 172.21.15.86:0/2597764190 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6900cf5e0 0x7fd690072140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd697000640 1 --2- 172.21.15.86:0/2597764190 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd690144660 0x7fd6900b3b40 unknown :-1 s=CLOSED pgs=1673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.268+0000 7fd697000640 1 -- 172.21.15.86:0/2597764190 >> 172.21.15.86:0/2597764190 conn(0x7fd6900d0b90 msgr2=0x7fd6900d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:44.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.269+0000 7fd697000640 1 -- 172.21.15.86:0/2597764190 shutdown_connections 2024-03-06T07:36:44.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.269+0000 7fd697000640 1 -- 172.21.15.86:0/2597764190 wait complete. 2024-03-06T07:36:44.272 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.270+0000 7fd697000640 1 Processor -- start 2024-03-06T07:36:44.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.270+0000 7fd697000640 1 -- start start 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.271+0000 7fd697000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6900cf170 0x7fd69015c110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.271+0000 7fd697000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6900cf5e0 0x7fd690142510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.271+0000 7fd697000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd690144660 0x7fd690142a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.271+0000 7fd697000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6900cc6d0 con 0x7fd6900cf5e0 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.271+0000 7fd697000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6900cc550 con 0x7fd6900cf170 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.271+0000 7fd697000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd69006f870 con 0x7fd690144660 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6900cf5e0 0x7fd690142510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6900cf5e0 0x7fd690142510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49216/0 (socket says 172.21.15.86:49216) 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 -- 172.21.15.86:0/3991616382 learned_addr learned my addr 172.21.15.86:0/3991616382 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:44.276 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd695000640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd690144660 0x7fd690142a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68fe00640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6900cf170 0x7fd69015c110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 -- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd690144660 msgr2=0x7fd690142a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd690144660 0x7fd690142a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 -- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6900cf170 msgr2=0x7fd69015c110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6900cf170 0x7fd69015c110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.272+0000 7fd68f400640 1 -- 172.21.15.86:0/3991616382 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd69013a330 con 0x7fd6900cf5e0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.273+0000 7fd68f400640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6900cf5e0 0x7fd690142510 secure :-1 s=READY pgs=1674 cs=0 l=1 rev1=1 crypto rx=0x7fd67c00d800 tx=0x7fd67c00dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.273+0000 7fd687e00640 1 -- 172.21.15.86:0/3991616382 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd67c00b6c0 con 0x7fd6900cf5e0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.273+0000 7fd687e00640 1 -- 172.21.15.86:0/3991616382 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd67c019070 con 0x7fd6900cf5e0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.273+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd69015c6f0 con 0x7fd6900cf5e0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.273+0000 7fd687e00640 1 -- 172.21.15.86:0/3991616382 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd67c00ba80 con 0x7fd6900cf5e0 2024-03-06T07:36:44.277 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.273+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd69015ccb0 con 0x7fd6900cf5e0 2024-03-06T07:36:44.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.274+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd69004f330 con 0x7fd6900cf5e0 2024-03-06T07:36:44.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.275+0000 7fd687e00640 1 -- 172.21.15.86:0/3991616382 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd67c00bc20 con 0x7fd6900cf5e0 2024-03-06T07:36:44.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.276+0000 7fd687e00640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd67006b4d0 0x7fd67006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:44.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.276+0000 7fd687e00640 1 -- 172.21.15.86:0/3991616382 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd67c08a730 con 0x7fd6900cf5e0 2024-03-06T07:36:44.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.276+0000 7fd68fe00640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd67006b4d0 0x7fd67006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:44.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.277+0000 7fd68fe00640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd67006b4d0 0x7fd67006d990 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7fd680036000 tx=0x7fd680008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:44.283 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.281+0000 7fd687e00640 1 -- 172.21.15.86:0/3991616382 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd67c059560 con 0x7fd6900cf5e0 2024-03-06T07:36:44.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.520+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd690141df0 con 0x7fd6900cf5e0 2024-03-06T07:36:44.524 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.521+0000 7fd687e00640 1 -- 172.21.15.86:0/3991616382 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fd67c058e60 con 0x7fd6900cf5e0 2024-03-06T07:36:44.524 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:44.524 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:44.524 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:44.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd67006b4d0 msgr2=0x7fd67006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:44.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd67006b4d0 0x7fd67006d990 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7fd680036000 tx=0x7fd680008030 comp rx=0 tx=0).stop 2024-03-06T07:36:44.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6900cf5e0 msgr2=0x7fd690142510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:44.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6900cf5e0 0x7fd690142510 secure :-1 s=READY pgs=1674 cs=0 l=1 rev1=1 crypto rx=0x7fd67c00d800 tx=0x7fd67c00dcc0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 shutdown_connections 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd67006b4d0 0x7fd67006d990 unknown :-1 s=CLOSED pgs=713 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd690144660 0x7fd690142a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6900cf5e0 0x7fd690142510 unknown :-1 s=CLOSED pgs=1674 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 --2- 172.21.15.86:0/3991616382 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6900cf170 0x7fd69015c110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 >> 172.21.15.86:0/3991616382 conn(0x7fd6900d0b90 msgr2=0x7fd6900b4de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 shutdown_connections 2024-03-06T07:36:44.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:44.524+0000 7fd697000640 1 -- 172.21.15.86:0/3991616382 wait complete. 2024-03-06T07:36:44.748 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:44.748 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:45.618 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:45.749 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:45.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.956+0000 7f523a400640 1 Processor -- start 2024-03-06T07:36:45.959 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.957+0000 7f523a400640 1 -- start start 2024-03-06T07:36:45.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.959+0000 7f523a400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 0x7f523406b3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f523a400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f523406ea00 con 0x7f52340cc470 2024-03-06T07:36:45.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f523a400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f523406b8e0 con 0x7f523406f1b0 2024-03-06T07:36:45.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f523a400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5234071ba0 con 0x7f523406afc0 2024-03-06T07:36:45.962 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f5233e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f52340cc470 0x7f523406e2f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55000/0 (socket says 172.21.15.86:55000) 2024-03-06T07:36:45.963 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f5233e00640 1 -- 172.21.15.86:0/2070243 learned_addr learned my addr 172.21.15.86:0/2070243 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f5233400640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 0x7f523406b3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2090267965 0 0) 0x7f523406ea00 con 0x7f52340cc470 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.960+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5218003520 con 0x7f52340cc470 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 13131107 0 0) 0x7f5218003520 con 0x7f52340cc470 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 msgr2=0x7f523406b3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 0x7f523406b3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 >> v1:172.21.15.194:6789/0 conn(0x7f523406f1b0 legacy=0x7f523406dbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5234071df0 con 0x7f52340cc470 2024-03-06T07:36:45.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f522c0019f0 con 0x7f52340cc470 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 >> v1:172.21.15.86:6789/0 conn(0x7f52340cc470 legacy=0x7f523406e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f52180039e0 0x7f5218003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f5218004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 0x7f52180051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5234071ba0 con 0x7f5218004800 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f521800baa0 con 0x7f52180039e0 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f521800bca0 con 0x7f523406afc0 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.961+0000 7f5232a00640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f5218004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.962+0000 7f5233400640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f52180039e0 0x7f5218003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.962+0000 7f5233e00640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 0x7f52180051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.962+0000 7f5232a00640 1 -- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 msgr2=0x7f52180051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.962+0000 7f5232a00640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f523406afc0 0x7f52180051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.962+0000 7f5232a00640 1 -- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f52180039e0 msgr2=0x7f5218003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:45.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.962+0000 7f5232a00640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f52180039e0 0x7f5218003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:45.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.962+0000 7f5232a00640 1 -- 172.21.15.86:0/2070243 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5218003520 con 0x7f5218004800 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.963+0000 7f5232a00640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f5218004c80 secure :-1 s=READY pgs=1676 cs=0 l=1 rev1=1 crypto rx=0x7f5220001720 tx=0x7f5218005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=381f5b9cc4f28f40 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.963+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5220001370 con 0x7f5218004800 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.963+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f52200052d0 con 0x7f5218004800 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.963+0000 7f5232000640 1 -- 172.21.15.86:0/2070243 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5220003830 con 0x7f5218004800 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.964+0000 7f523a400640 1 -- 172.21.15.86:0/2070243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 msgr2=0x7f5218004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.964+0000 7f523a400640 1 --2- 172.21.15.86:0/2070243 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f5218004c80 secure :-1 s=READY pgs=1676 cs=0 l=1 rev1=1 crypto rx=0x7f5220001720 tx=0x7f5218005830 comp rx=0 tx=0).stop 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.964+0000 7f5233e00640 1 -- 172.21.15.86:0/2070243 reap_dead start 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.964+0000 7f523a400640 1 -- 172.21.15.86:0/2070243 shutdown_connections 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.964+0000 7f523a400640 1 -- 172.21.15.86:0/2070243 >> 172.21.15.86:0/2070243 conn(0x7f52340d0ba0 msgr2=0x7f52340d0fb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.964+0000 7f523a400640 1 -- 172.21.15.86:0/2070243 shutdown_connections 2024-03-06T07:36:45.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.964+0000 7f523a400640 1 -- 172.21.15.86:0/2070243 wait complete. 2024-03-06T07:36:45.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.966+0000 7f523a400640 1 Processor -- start 2024-03-06T07:36:45.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.966+0000 7f523a400640 1 -- start start 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f523a400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f52341f44e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f523a400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5234142920 0x7f52340ae2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f523a400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5234142da0 0x7f52340ae7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f523a400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f52340723b0 con 0x7f5218004800 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f523a400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5234072a00 con 0x7f5234142da0 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f523a400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5234072690 con 0x7f5234142920 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f5233400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f52341f44e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.971 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f5233400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f52341f44e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38318/0 (socket says 172.21.15.86:38318) 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.967+0000 7f5233400640 1 -- 172.21.15.86:0/3389475256 learned_addr learned my addr 172.21.15.86:0/3389475256 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5232a00640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5234142920 0x7f52340ae2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5233400640 1 -- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5234142920 msgr2=0x7f52340ae2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5233e00640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5234142da0 0x7f52340ae7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5233400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5234142920 0x7f52340ae2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5233400640 1 -- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5234142da0 msgr2=0x7f52340ae7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5233400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5234142da0 0x7f52340ae7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5233400640 1 -- 172.21.15.86:0/3389475256 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f52341f4a20 con 0x7f5218004800 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f5233400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f52341f44e0 secure :-1 s=READY pgs=1677 cs=0 l=1 rev1=1 crypto rx=0x7f5224011630 tx=0x7f5224011af0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f522b400640 1 -- 172.21.15.86:0/3389475256 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f522400f920 con 0x7f5218004800 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f522b400640 1 -- 172.21.15.86:0/3389475256 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f522400fac0 con 0x7f5218004800 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5234143370 con 0x7f5218004800 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.968+0000 7f522b400640 1 -- 172.21.15.86:0/3389475256 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f522400fe70 con 0x7f5218004800 2024-03-06T07:36:45.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.969+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5234143930 con 0x7f5218004800 2024-03-06T07:36:45.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.970+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f523404f330 con 0x7f5218004800 2024-03-06T07:36:45.974 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.972+0000 7f522b400640 1 -- 172.21.15.86:0/3389475256 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5224015460 con 0x7f5218004800 2024-03-06T07:36:45.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.973+0000 7f522b400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f521406b500 0x7f521406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:45.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.973+0000 7f522b400640 1 -- 172.21.15.86:0/3389475256 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f522408ab60 con 0x7f5218004800 2024-03-06T07:36:45.975 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.973+0000 7f5232a00640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f521406b500 0x7f521406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:45.976 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.974+0000 7f5232a00640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f521406b500 0x7f521406d9c0 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f5220033000 tx=0x7f5220009030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:45.983 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:45.981+0000 7f522b400640 1 -- 172.21.15.86:0/3389475256 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5224019030 con 0x7f5218004800 2024-03-06T07:36:46.221 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.218+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f523406f870 con 0x7f5218004800 2024-03-06T07:36:46.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.219+0000 7f522b400640 1 -- 172.21.15.86:0/3389475256 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f5224059990 con 0x7f5218004800 2024-03-06T07:36:46.222 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:46.222 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:46.222 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f521406b500 msgr2=0x7f521406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f521406b500 0x7f521406d9c0 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f5220033000 tx=0x7f5220009030 comp rx=0 tx=0).stop 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 msgr2=0x7f52341f44e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f52341f44e0 secure :-1 s=READY pgs=1677 cs=0 l=1 rev1=1 crypto rx=0x7f5224011630 tx=0x7f5224011af0 comp rx=0 tx=0).stop 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 shutdown_connections 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f521406b500 0x7f521406d9c0 unknown :-1 s=CLOSED pgs=714 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f5234142da0 0x7f52340ae7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5234142920 0x7f52340ae2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 --2- 172.21.15.86:0/3389475256 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f5218004800 0x7f52341f44e0 unknown :-1 s=CLOSED pgs=1677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 >> 172.21.15.86:0/3389475256 conn(0x7f52340d0ba0 msgr2=0x7f5234144a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:46.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 shutdown_connections 2024-03-06T07:36:46.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:46.223+0000 7f523a400640 1 -- 172.21.15.86:0/3389475256 wait complete. 2024-03-06T07:36:46.454 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:46.454 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:47.455 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:47.620 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:47.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.662+0000 7fc86c200640 1 Processor -- start 2024-03-06T07:36:47.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.662+0000 7fc86c200640 1 -- start start 2024-03-06T07:36:47.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc86c200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 0x7fc8640b2e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:47.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc86c200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc8640cec00 0x7fc8640cc630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:47.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc86c200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc86406cc80 con 0x7fc86415e540 2024-03-06T07:36:47.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc86c200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc86406cf60 con 0x7fc8640cec00 2024-03-06T07:36:47.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc86c200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc86406d240 con 0x7fc86415e160 2024-03-06T07:36:47.666 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc86a200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc8640cec00 0x7fc8640cc630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:47.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc868e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc86415e540 0x7fc8640ce4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55012/0 (socket says 172.21.15.86:55012) 2024-03-06T07:36:47.667 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc868e00640 1 -- 172.21.15.86:0/217947922 learned_addr learned my addr 172.21.15.86:0/217947922 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:47.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.664+0000 7fc869800640 1 --2- 172.21.15.86:0/217947922 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 0x7fc8640b2e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:47.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.665+0000 7fc86a200640 1 -- 172.21.15.86:0/217947922 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 msgr2=0x7fc8640b2e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:47.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.665+0000 7fc86a200640 1 --2- 172.21.15.86:0/217947922 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 0x7fc8640b2e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.665+0000 7fc86a200640 1 -- 172.21.15.86:0/217947922 >> v1:172.21.15.86:6789/0 conn(0x7fc86415e540 legacy=0x7fc8640ce4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:47.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.665+0000 7fc86a200640 1 -- 172.21.15.86:0/217947922 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc8640b33a0 con 0x7fc8640cec00 2024-03-06T07:36:47.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.665+0000 7fc863e00640 1 -- 172.21.15.86:0/217947922 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2970695345 0 0) 0x7fc86406cc80 con 0x7fc86415e540 2024-03-06T07:36:47.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.666+0000 7fc86a200640 1 --2- 172.21.15.86:0/217947922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640cc630 secure :-1 s=READY pgs=1679 cs=0 l=1 rev1=1 crypto rx=0x7fc85c009d90 tx=0x7fc85c02bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=64161cfcbf3b4565 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:47.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.666+0000 7fc863e00640 1 -- 172.21.15.86:0/217947922 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc85c039070 con 0x7fc8640cec00 2024-03-06T07:36:47.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.666+0000 7fc863e00640 1 -- 172.21.15.86:0/217947922 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc85c02cc50 con 0x7fc8640cec00 2024-03-06T07:36:47.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.666+0000 7fc863e00640 1 -- 172.21.15.86:0/217947922 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc85c034440 con 0x7fc8640cec00 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 -- 172.21.15.86:0/217947922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 msgr2=0x7fc8640cc630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 --2- 172.21.15.86:0/217947922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640cc630 secure :-1 s=READY pgs=1679 cs=0 l=1 rev1=1 crypto rx=0x7fc85c009d90 tx=0x7fc85c02bea0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 -- 172.21.15.86:0/217947922 shutdown_connections 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 --2- 172.21.15.86:0/217947922 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640cc630 unknown :-1 s=CLOSED pgs=1679 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 --2- 172.21.15.86:0/217947922 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 0x7fc8640b2e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 -- 172.21.15.86:0/217947922 >> 172.21.15.86:0/217947922 conn(0x7fc8640b3800 msgr2=0x7fc8640b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 -- 172.21.15.86:0/217947922 shutdown_connections 2024-03-06T07:36:47.670 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.667+0000 7fc86c200640 1 -- 172.21.15.86:0/217947922 wait complete. 2024-03-06T07:36:47.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.669+0000 7fc86c200640 1 Processor -- start 2024-03-06T07:36:47.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.669+0000 7fc86c200640 1 -- start start 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc86c200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640b7530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc86c200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 0x7fc8641e4210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc86c200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc86415e540 0x7fc8640b59d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc86c200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8640ccb70 con 0x7fc8640cec00 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc86c200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8640cf380 con 0x7fc86415e540 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc86c200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc8640cccf0 con 0x7fc86415e160 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640b7530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:47.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640b7530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38332/0 (socket says 172.21.15.86:38332) 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 -- 172.21.15.86:0/511376992 learned_addr learned my addr 172.21.15.86:0/511376992 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 -- 172.21.15.86:0/511376992 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 msgr2=0x7fc8641e4210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 0x7fc8641e4210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 -- 172.21.15.86:0/511376992 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc86415e540 msgr2=0x7fc8640b59d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc86415e540 0x7fc8640b59d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.670+0000 7fc869800640 1 -- 172.21.15.86:0/511376992 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc8640b6490 con 0x7fc8640cec00 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.671+0000 7fc869800640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640b7530 secure :-1 s=READY pgs=1680 cs=0 l=1 rev1=1 crypto rx=0x7fc85800d830 tx=0x7fc85800dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:47.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.671+0000 7fc861600640 1 -- 172.21.15.86:0/511376992 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc85800b6c0 con 0x7fc8640cec00 2024-03-06T07:36:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.671+0000 7fc861600640 1 -- 172.21.15.86:0/511376992 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc858019070 con 0x7fc8640cec00 2024-03-06T07:36:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.671+0000 7fc861600640 1 -- 172.21.15.86:0/511376992 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc85800ba80 con 0x7fc8640cec00 2024-03-06T07:36:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.671+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc8640b6770 con 0x7fc8640cec00 2024-03-06T07:36:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.671+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc8640b6d30 con 0x7fc8640cec00 2024-03-06T07:36:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.673+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc86404f330 con 0x7fc8640cec00 2024-03-06T07:36:47.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.673+0000 7fc861600640 1 -- 172.21.15.86:0/511376992 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc85801e460 con 0x7fc8640cec00 2024-03-06T07:36:47.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.675+0000 7fc861600640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc84406b4d0 0x7fc84406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:47.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.675+0000 7fc861600640 1 -- 172.21.15.86:0/511376992 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc85808a900 con 0x7fc8640cec00 2024-03-06T07:36:47.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.675+0000 7fc868e00640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc84406b4d0 0x7fc84406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:47.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.675+0000 7fc868e00640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc84406b4d0 0x7fc84406d990 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7fc850007fd0 tx=0x7fc850007960 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:47.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.679+0000 7fc861600640 1 -- 172.21.15.86:0/511376992 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc858059730 con 0x7fc8640cec00 2024-03-06T07:36:47.941 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.939+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc8640cbf80 con 0x7fc8640cec00 2024-03-06T07:36:47.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.940+0000 7fc861600640 1 -- 172.21.15.86:0/511376992 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fc858059030 con 0x7fc8640cec00 2024-03-06T07:36:47.943 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:47.943 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:47.943 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc84406b4d0 msgr2=0x7fc84406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc84406b4d0 0x7fc84406d990 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7fc850007fd0 tx=0x7fc850007960 comp rx=0 tx=0).stop 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 msgr2=0x7fc8640b7530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640b7530 secure :-1 s=READY pgs=1680 cs=0 l=1 rev1=1 crypto rx=0x7fc85800d830 tx=0x7fc85800dcf0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 shutdown_connections 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc84406b4d0 0x7fc84406d990 unknown :-1 s=CLOSED pgs=715 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc86415e540 0x7fc8640b59d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc86415e160 0x7fc8641e4210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 --2- 172.21.15.86:0/511376992 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc8640cec00 0x7fc8640b7530 unknown :-1 s=CLOSED pgs=1680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:47.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 >> 172.21.15.86:0/511376992 conn(0x7fc8640b3800 msgr2=0x7fc8640cd280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:47.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 shutdown_connections 2024-03-06T07:36:47.947 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:47.943+0000 7fc86c200640 1 -- 172.21.15.86:0/511376992 wait complete. 2024-03-06T07:36:48.164 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:48.164 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:49.165 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.380+0000 7fc744e00640 1 Processor -- start 2024-03-06T07:36:49.383 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.380+0000 7fc744e00640 1 -- start start 2024-03-06T07:36:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.382+0000 7fc744e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc74015e0d0 0x7fc7400d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.382+0000 7fc744e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc7400cea60 con 0x7fc7400d0fc0 2024-03-06T07:36:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.382+0000 7fc744e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc7400d2350 con 0x7fc74015f470 2024-03-06T07:36:49.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.382+0000 7fc744e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc7400d2f60 con 0x7fc74015e0d0 2024-03-06T07:36:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.382+0000 7fc73e800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc7400d0fc0 0x7fc740160dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55024/0 (socket says 172.21.15.86:55024) 2024-03-06T07:36:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.382+0000 7fc73e800640 1 -- 172.21.15.86:0/3235848293 learned_addr learned my addr 172.21.15.86:0/3235848293 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.382+0000 7fc73de00640 1 --2- 172.21.15.86:0/3235848293 >> v2:172.21.15.194:3300/0 conn(0x7fc74015e0d0 0x7fc7400d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 729344430 0 0) 0x7fc7400cea60 con 0x7fc7400d0fc0 2024-03-06T07:36:49.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc724003520 con 0x7fc7400d0fc0 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3806042174 0 0) 0x7fc724003520 con 0x7fc7400d0fc0 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 >> v2:172.21.15.194:3300/0 conn(0x7fc74015e0d0 msgr2=0x7fc7400d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 --2- 172.21.15.86:0/3235848293 >> v2:172.21.15.194:3300/0 conn(0x7fc74015e0d0 0x7fc7400d1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 >> v1:172.21.15.194:6789/0 conn(0x7fc74015f470 legacy=0x7fc7400d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7400d2630 con 0x7fc7400d0fc0 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc7380019f0 con 0x7fc7400d0fc0 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 >> v1:172.21.15.86:6789/0 conn(0x7fc7400d0fc0 legacy=0x7fc740160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:49.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc724003a00 0x7fc724003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc724004800 0x7fc724004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7240051c0 0x7fc72400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc72400c070 con 0x7fc724003a00 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc72400c220 con 0x7fc7240051c0 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.383+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc72400c420 con 0x7fc724004800 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73de00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc724003a00 0x7fc724003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73e800640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7240051c0 0x7fc72400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73de00640 1 -- 172.21.15.86:0/3235848293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc724004800 msgr2=0x7fc724004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73de00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc724004800 0x7fc724004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73de00640 1 -- 172.21.15.86:0/3235848293 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7240051c0 msgr2=0x7fc72400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73de00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc7240051c0 0x7fc72400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73de00640 1 -- 172.21.15.86:0/3235848293 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc724003520 con 0x7fc724003a00 2024-03-06T07:36:49.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.384+0000 7fc73e800640 1 -- 172.21.15.86:0/3235848293 reap_dead start 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.385+0000 7fc73de00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc724003a00 0x7fc724003e20 secure :-1 s=READY pgs=1682 cs=0 l=1 rev1=1 crypto rx=0x7fc730009ac0 tx=0x7fc724005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2b9c5b781e0188b8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.385+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc730039070 con 0x7fc724003a00 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.385+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc73002c6b0 con 0x7fc724003a00 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc73ca00640 1 -- 172.21.15.86:0/3235848293 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc730009690 con 0x7fc724003a00 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc744e00640 1 -- 172.21.15.86:0/3235848293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc724003a00 msgr2=0x7fc724003e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc744e00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc724003a00 0x7fc724003e20 secure :-1 s=READY pgs=1682 cs=0 l=1 rev1=1 crypto rx=0x7fc730009ac0 tx=0x7fc724005830 comp rx=0 tx=0).stop 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc744e00640 1 -- 172.21.15.86:0/3235848293 shutdown_connections 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc744e00640 1 --2- 172.21.15.86:0/3235848293 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc724003a00 0x7fc724003e20 unknown :-1 s=CLOSED pgs=1682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc744e00640 1 -- 172.21.15.86:0/3235848293 >> 172.21.15.86:0/3235848293 conn(0x7fc7400d3ec0 msgr2=0x7fc7400d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc744e00640 1 -- 172.21.15.86:0/3235848293 shutdown_connections 2024-03-06T07:36:49.390 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.386+0000 7fc744e00640 1 -- 172.21.15.86:0/3235848293 wait complete. 2024-03-06T07:36:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.388+0000 7fc744e00640 1 Processor -- start 2024-03-06T07:36:49.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.388+0000 7fc744e00640 1 -- start start 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc744e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc724003a00 0x7fc74006cdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc744e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc740150e20 0x7fc7401512a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc744e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7400cd370 0x7fc7400cd7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc744e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc74015fb30 con 0x7fc7400cd370 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc744e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7400d2bf0 con 0x7fc724003a00 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc744e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc74015e4b0 con 0x7fc740150e20 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc73e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7400cd370 0x7fc7400cd7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc73e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7400cd370 0x7fc7400cd7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38360/0 (socket says 172.21.15.86:38360) 2024-03-06T07:36:49.393 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.389+0000 7fc73e800640 1 -- 172.21.15.86:0/1444039841 learned_addr learned my addr 172.21.15.86:0/1444039841 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73de00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc724003a00 0x7fc74006cdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73d400640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc740150e20 0x7fc7401512a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73e800640 1 -- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc740150e20 msgr2=0x7fc7401512a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73e800640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc740150e20 0x7fc7401512a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73e800640 1 -- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc724003a00 msgr2=0x7fc74006cdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73e800640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc724003a00 0x7fc74006cdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73e800640 1 -- 172.21.15.86:0/1444039841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7400cdd30 con 0x7fc7400cd370 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73d400640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc740150e20 0x7fc7401512a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc73e800640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7400cd370 0x7fc7400cd7f0 secure :-1 s=READY pgs=1683 cs=0 l=1 rev1=1 crypto rx=0x7fc7400d3e90 tx=0x7fc73800e530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc736000640 1 -- 172.21.15.86:0/1444039841 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc738001a30 con 0x7fc7400cd370 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc736000640 1 -- 172.21.15.86:0/1444039841 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc73800ee20 con 0x7fc7400cd370 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.390+0000 7fc736000640 1 -- 172.21.15.86:0/1444039841 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc738014650 con 0x7fc7400cd370 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.391+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc740161ea0 con 0x7fc7400cd370 2024-03-06T07:36:49.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.391+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc740162460 con 0x7fc7400cd370 2024-03-06T07:36:49.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.392+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc74004f330 con 0x7fc7400cd370 2024-03-06T07:36:49.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.393+0000 7fc736000640 1 -- 172.21.15.86:0/1444039841 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc7380147f0 con 0x7fc7400cd370 2024-03-06T07:36:49.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.394+0000 7fc736000640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc72006b4d0 0x7fc72006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:49.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.394+0000 7fc736000640 1 -- 172.21.15.86:0/1444039841 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc73808a110 con 0x7fc7400cd370 2024-03-06T07:36:49.396 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.394+0000 7fc73de00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc72006b4d0 0x7fc72006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:49.397 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.395+0000 7fc73de00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc72006b4d0 0x7fc72006d990 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7fc730036000 tx=0x7fc730030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:49.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.398+0000 7fc736000640 1 -- 172.21.15.86:0/1444039841 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc738058f50 con 0x7fc7400cd370 2024-03-06T07:36:49.622 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:49.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.635+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc74008d920 con 0x7fc7400cd370 2024-03-06T07:36:49.638 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.636+0000 7fc736000640 1 -- 172.21.15.86:0/1444039841 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fc738058850 con 0x7fc7400cd370 2024-03-06T07:36:49.638 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:49.639 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:49.639 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:49.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc72006b4d0 msgr2=0x7fc72006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:49.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc72006b4d0 0x7fc72006d990 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7fc730036000 tx=0x7fc730030000 comp rx=0 tx=0).stop 2024-03-06T07:36:49.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7400cd370 msgr2=0x7fc7400cd7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:49.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7400cd370 0x7fc7400cd7f0 secure :-1 s=READY pgs=1683 cs=0 l=1 rev1=1 crypto rx=0x7fc7400d3e90 tx=0x7fc73800e530 comp rx=0 tx=0).stop 2024-03-06T07:36:49.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 shutdown_connections 2024-03-06T07:36:49.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc72006b4d0 0x7fc72006d990 unknown :-1 s=CLOSED pgs=716 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc7400cd370 0x7fc7400cd7f0 unknown :-1 s=CLOSED pgs=1683 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc740150e20 0x7fc7401512a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 --2- 172.21.15.86:0/1444039841 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc724003a00 0x7fc74006cdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:49.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.638+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 >> 172.21.15.86:0/1444039841 conn(0x7fc7400d3ec0 msgr2=0x7fc7400b48f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:49.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.639+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 shutdown_connections 2024-03-06T07:36:49.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:49.639+0000 7fc744e00640 1 -- 172.21.15.86:0/1444039841 wait complete. 2024-03-06T07:36:49.860 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:49.860 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:50.862 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:51.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.074+0000 7f2999200640 1 Processor -- start 2024-03-06T07:36:51.077 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.074+0000 7f2999200640 1 -- start start 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2999200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2994161580 0x7f29940d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2999200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f29940d2a70 0x7f29940b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2999200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2994161960 con 0x7f29940d1e90 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2999200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f29940cf6f0 con 0x7f2994161580 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2999200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f29940cf9d0 con 0x7f29940d2a70 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2993400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2994161580 0x7f29940d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2993400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2994161580 0x7f29940d1950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38368/0 (socket says 172.21.15.86:38368) 2024-03-06T07:36:51.079 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2993400640 1 -- 172.21.15.86:0/2288422267 learned_addr learned my addr 172.21.15.86:0/2288422267 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:51.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.076+0000 7f2993e00640 1 --2- 172.21.15.86:0/2288422267 >> v2:172.21.15.194:3300/0 conn(0x7f29940d2a70 0x7f29940b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:51.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.077+0000 7f2993400640 1 -- 172.21.15.86:0/2288422267 >> v2:172.21.15.194:3300/0 conn(0x7f29940d2a70 msgr2=0x7f29940b3ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:51.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.077+0000 7f2993400640 1 --2- 172.21.15.86:0/2288422267 >> v2:172.21.15.194:3300/0 conn(0x7f29940d2a70 0x7f29940b3ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.077+0000 7f2993400640 1 -- 172.21.15.86:0/2288422267 >> v1:172.21.15.194:6789/0 conn(0x7f29940d1e90 legacy=0x7f29940d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:51.080 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.077+0000 7f2993400640 1 -- 172.21.15.86:0/2288422267 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29940cfcb0 con 0x7f2994161580 2024-03-06T07:36:51.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.078+0000 7f2993400640 1 --2- 172.21.15.86:0/2288422267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940d1950 secure :-1 s=READY pgs=1684 cs=0 l=1 rev1=1 crypto rx=0x7f299408d180 tx=0x7f298402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f0c3eff45c714347 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:51.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.078+0000 7f2992a00640 1 -- 172.21.15.86:0/2288422267 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2984035020 con 0x7f2994161580 2024-03-06T07:36:51.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.078+0000 7f2992a00640 1 -- 172.21.15.86:0/2288422267 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2984031870 con 0x7f2994161580 2024-03-06T07:36:51.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.078+0000 7f2992a00640 1 -- 172.21.15.86:0/2288422267 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2984031c40 con 0x7f2994161580 2024-03-06T07:36:51.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 -- 172.21.15.86:0/2288422267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 msgr2=0x7f29940d1950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:51.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 --2- 172.21.15.86:0/2288422267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940d1950 secure :-1 s=READY pgs=1684 cs=0 l=1 rev1=1 crypto rx=0x7f299408d180 tx=0x7f298402b910 comp rx=0 tx=0).stop 2024-03-06T07:36:51.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 -- 172.21.15.86:0/2288422267 shutdown_connections 2024-03-06T07:36:51.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 --2- 172.21.15.86:0/2288422267 >> v2:172.21.15.194:3300/0 conn(0x7f29940d2a70 0x7f29940b3ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 --2- 172.21.15.86:0/2288422267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940d1950 unknown :-1 s=CLOSED pgs=1684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 -- 172.21.15.86:0/2288422267 >> 172.21.15.86:0/2288422267 conn(0x7f29940d3d80 msgr2=0x7f29940d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:51.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 -- 172.21.15.86:0/2288422267 shutdown_connections 2024-03-06T07:36:51.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.079+0000 7f2999200640 1 -- 172.21.15.86:0/2288422267 wait complete. 2024-03-06T07:36:51.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.081+0000 7f2999200640 1 Processor -- start 2024-03-06T07:36:51.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.081+0000 7f2999200640 1 -- start start 2024-03-06T07:36:51.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2999200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29940d1e90 0x7f299415db20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:51.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2999200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29940d2a70 0x7f29940b90c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:51.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2999200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940c7db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:51.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2999200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29940cf3a0 con 0x7f2994161580 2024-03-06T07:36:51.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2999200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29940b39a0 con 0x7f29940d2a70 2024-03-06T07:36:51.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2999200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29940b4400 con 0x7f29940d1e90 2024-03-06T07:36:51.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940c7db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:51.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940c7db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38380/0 (socket says 172.21.15.86:38380) 2024-03-06T07:36:51.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 -- 172.21.15.86:0/3905779119 learned_addr learned my addr 172.21.15.86:0/3905779119 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993400640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29940d1e90 0x7f299415db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f298be00640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29940d2a70 0x7f29940b90c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 -- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29940d1e90 msgr2=0x7f299415db20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29940d1e90 0x7f299415db20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 -- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29940d2a70 msgr2=0x7f29940b90c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29940d2a70 0x7f29940b90c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.082+0000 7f2993e00640 1 -- 172.21.15.86:0/3905779119 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29940b8640 con 0x7f2994161580 2024-03-06T07:36:51.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.083+0000 7f2993e00640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940c7db0 secure :-1 s=READY pgs=1685 cs=0 l=1 rev1=1 crypto rx=0x7f298c00cd70 tx=0x7f298c00a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:51.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.083+0000 7f298b400640 1 -- 172.21.15.86:0/3905779119 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f298c003b90 con 0x7f2994161580 2024-03-06T07:36:51.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.083+0000 7f298b400640 1 -- 172.21.15.86:0/3905779119 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f298c003d30 con 0x7f2994161580 2024-03-06T07:36:51.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.083+0000 7f298b400640 1 -- 172.21.15.86:0/3905779119 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f298c004240 con 0x7f2994161580 2024-03-06T07:36:51.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.083+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29940c7410 con 0x7f2994161580 2024-03-06T07:36:51.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.083+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f29940c79d0 con 0x7f2994161580 2024-03-06T07:36:51.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.084+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f299404f330 con 0x7f2994161580 2024-03-06T07:36:51.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.085+0000 7f298b400640 1 -- 172.21.15.86:0/3905779119 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f298c0043e0 con 0x7f2994161580 2024-03-06T07:36:51.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.086+0000 7f298b400640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f297006b4d0 0x7f297006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:51.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.086+0000 7f298b400640 1 -- 172.21.15.86:0/3905779119 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f298c08ad50 con 0x7f2994161580 2024-03-06T07:36:51.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.086+0000 7f2993400640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f297006b4d0 0x7f297006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:51.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.087+0000 7f2993400640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f297006b4d0 0x7f297006d990 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f2984036000 tx=0x7f298403c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:51.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.095+0000 7f298b400640 1 -- 172.21.15.86:0/3905779119 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f29940b8640 con 0x7f2994161580 2024-03-06T07:36:51.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.329+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f29940d3b60 con 0x7f2994161580 2024-03-06T07:36:51.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.330+0000 7f298b400640 1 -- 172.21.15.86:0/3905779119 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f298c059b80 con 0x7f2994161580 2024-03-06T07:36:51.332 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:51.332 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:51.333 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f297006b4d0 msgr2=0x7f297006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f297006b4d0 0x7f297006d990 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f2984036000 tx=0x7f298403c040 comp rx=0 tx=0).stop 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 msgr2=0x7f29940c7db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940c7db0 secure :-1 s=READY pgs=1685 cs=0 l=1 rev1=1 crypto rx=0x7f298c00cd70 tx=0x7f298c00a660 comp rx=0 tx=0).stop 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 shutdown_connections 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f297006b4d0 0x7f297006d990 unknown :-1 s=CLOSED pgs=717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2994161580 0x7f29940c7db0 unknown :-1 s=CLOSED pgs=1685 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f29940d2a70 0x7f29940b90c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 --2- 172.21.15.86:0/3905779119 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f29940d1e90 0x7f299415db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 >> 172.21.15.86:0/3905779119 conn(0x7f29940d3d80 msgr2=0x7f299415e600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 shutdown_connections 2024-03-06T07:36:51.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:51.332+0000 7f2999200640 1 -- 172.21.15.86:0/3905779119 wait complete. 2024-03-06T07:36:51.553 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:51.553 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:51.624 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:52.553 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:52.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.800+0000 7f3645000640 1 Processor -- start 2024-03-06T07:36:52.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.800+0000 7f3645000640 1 -- start start 2024-03-06T07:36:52.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.802+0000 7f3645000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f36400cc9d0 0x7f364006d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:52.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.802+0000 7f3645000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f364015c240 con 0x7f36400aedc0 2024-03-06T07:36:52.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.802+0000 7f3645000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f364015c500 con 0x7f36401561f0 2024-03-06T07:36:52.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.802+0000 7f3645000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f364015ce30 con 0x7f36400cc9d0 2024-03-06T07:36:52.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.802+0000 7f3637e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f36400aedc0 0x7f36400cc2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55028/0 (socket says 172.21.15.86:55028) 2024-03-06T07:36:52.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.802+0000 7f3637e00640 1 -- 172.21.15.86:0/955679390 learned_addr learned my addr 172.21.15.86:0/955679390 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:52.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.803+0000 7f363fe00640 1 --2- 172.21.15.86:0/955679390 >> v2:172.21.15.194:3300/0 conn(0x7f36400cc9d0 0x7f364006d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:52.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.803+0000 7f363ea00640 1 -- 172.21.15.86:0/955679390 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4105184566 0 0) 0x7f364015c240 con 0x7f36400aedc0 2024-03-06T07:36:52.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.803+0000 7f363ea00640 1 -- 172.21.15.86:0/955679390 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3624003520 con 0x7f36400aedc0 2024-03-06T07:36:52.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.803+0000 7f363fe00640 1 -- 172.21.15.86:0/955679390 >> v1:172.21.15.194:6789/0 conn(0x7f36401561f0 legacy=0x7f36400ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:52.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.803+0000 7f363fe00640 1 -- 172.21.15.86:0/955679390 >> v1:172.21.15.86:6789/0 conn(0x7f36400aedc0 legacy=0x7f36400cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:52.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.803+0000 7f363fe00640 1 -- 172.21.15.86:0/955679390 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f364015c7e0 con 0x7f36400cc9d0 2024-03-06T07:36:52.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.805+0000 7f363fe00640 1 --2- 172.21.15.86:0/955679390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f364006d350 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f364015cfb0 tx=0x7f363802bf30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6d6493bf96af6c4f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:52.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.805+0000 7f363ea00640 1 -- 172.21.15.86:0/955679390 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3638039070 con 0x7f36400cc9d0 2024-03-06T07:36:52.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.805+0000 7f363ea00640 1 -- 172.21.15.86:0/955679390 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f363802cce0 con 0x7f36400cc9d0 2024-03-06T07:36:52.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.805+0000 7f363ea00640 1 -- 172.21.15.86:0/955679390 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3638034640 con 0x7f36400cc9d0 2024-03-06T07:36:52.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.806+0000 7f3645000640 1 -- 172.21.15.86:0/955679390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 msgr2=0x7f364006d350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:52.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.806+0000 7f3645000640 1 --2- 172.21.15.86:0/955679390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f364006d350 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f364015cfb0 tx=0x7f363802bf30 comp rx=0 tx=0).stop 2024-03-06T07:36:52.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.806+0000 7f3645000640 1 -- 172.21.15.86:0/955679390 shutdown_connections 2024-03-06T07:36:52.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.806+0000 7f3645000640 1 --2- 172.21.15.86:0/955679390 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f364006d350 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:52.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.806+0000 7f3645000640 1 -- 172.21.15.86:0/955679390 >> 172.21.15.86:0/955679390 conn(0x7f364006afc0 msgr2=0x7f364006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:52.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.806+0000 7f3645000640 1 -- 172.21.15.86:0/955679390 shutdown_connections 2024-03-06T07:36:52.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.806+0000 7f3645000640 1 -- 172.21.15.86:0/955679390 wait complete. 2024-03-06T07:36:52.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.808+0000 7f3645000640 1 Processor -- start 2024-03-06T07:36:52.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.808+0000 7f3645000640 1 -- start start 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f3645000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36400aedc0 0x7f36401493c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f3645000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f36400cee10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f3645000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36401561f0 0x7f36400cd460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f3645000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3640156750 con 0x7f36401561f0 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f3645000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36401565d0 con 0x7f36400cc9d0 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f3645000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f364006d890 con 0x7f36400aedc0 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f363fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36401561f0 0x7f36400cd460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f363fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36401561f0 0x7f36400cd460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38386/0 (socket says 172.21.15.86:38386) 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.809+0000 7f363fe00640 1 -- 172.21.15.86:0/133193350 learned_addr learned my addr 172.21.15.86:0/133193350 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:52.812 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363f400640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36400aedc0 0x7f36401493c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f3637e00640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f36400cee10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363fe00640 1 -- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36400aedc0 msgr2=0x7f36401493c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363fe00640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36400aedc0 0x7f36401493c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363fe00640 1 -- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 msgr2=0x7f36400cee10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363fe00640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f36400cee10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363fe00640 1 -- 172.21.15.86:0/133193350 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36400cdd70 con 0x7f36401561f0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f3637e00640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f36400cee10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363f400640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36400aedc0 0x7f36401493c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f363fe00640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36401561f0 0x7f36400cd460 secure :-1 s=READY pgs=1687 cs=0 l=1 rev1=1 crypto rx=0x7f3638037040 tx=0x7f3638004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f3637400640 1 -- 172.21.15.86:0/133193350 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3638044020 con 0x7f36401561f0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f3637400640 1 -- 172.21.15.86:0/133193350 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f363803a040 con 0x7f36401561f0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f3637400640 1 -- 172.21.15.86:0/133193350 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3638039050 con 0x7f36401561f0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.810+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36400ce050 con 0x7f36401561f0 2024-03-06T07:36:52.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.811+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f36400ce610 con 0x7f36401561f0 2024-03-06T07:36:52.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.812+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f364006b4a0 con 0x7f36401561f0 2024-03-06T07:36:52.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.813+0000 7f3637400640 1 -- 172.21.15.86:0/133193350 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3638003b90 con 0x7f36401561f0 2024-03-06T07:36:52.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.814+0000 7f3637400640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f361c06b4d0 0x7f361c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:52.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.814+0000 7f3637400640 1 -- 172.21.15.86:0/133193350 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f36380aca00 con 0x7f36401561f0 2024-03-06T07:36:52.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.814+0000 7f363f400640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f361c06b4d0 0x7f361c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:52.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.815+0000 7f363f400640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f361c06b4d0 0x7f361c06d990 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7f36300015e0 tx=0x7f3630009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:52.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:52.822+0000 7f3637400640 1 -- 172.21.15.86:0/133193350 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f364006b4a0 con 0x7f36401561f0 2024-03-06T07:36:53.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.198+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f364004f330 con 0x7f36401561f0 2024-03-06T07:36:53.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.199+0000 7f3637400640 1 -- 172.21.15.86:0/133193350 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f363804e020 con 0x7f36401561f0 2024-03-06T07:36:53.202 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:53.202 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:53.202 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f361c06b4d0 msgr2=0x7f361c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f361c06b4d0 0x7f361c06d990 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7f36300015e0 tx=0x7f3630009040 comp rx=0 tx=0).stop 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36401561f0 msgr2=0x7f36400cd460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36401561f0 0x7f36400cd460 secure :-1 s=READY pgs=1687 cs=0 l=1 rev1=1 crypto rx=0x7f3638037040 tx=0x7f3638004a40 comp rx=0 tx=0).stop 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 shutdown_connections 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f361c06b4d0 0x7f361c06d990 unknown :-1 s=CLOSED pgs=718 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f36401561f0 0x7f36400cd460 unknown :-1 s=CLOSED pgs=1687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f36400cc9d0 0x7f36400cee10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 --2- 172.21.15.86:0/133193350 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f36400aedc0 0x7f36401493c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 >> 172.21.15.86:0/133193350 conn(0x7f364006afc0 msgr2=0x7f3640154cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 shutdown_connections 2024-03-06T07:36:53.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:53.202+0000 7f3645000640 1 -- 172.21.15.86:0/133193350 wait complete. 2024-03-06T07:36:53.427 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:53.427 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:53.626 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:54.429 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:54.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.638+0000 7f9bc8a00640 1 Processor -- start 2024-03-06T07:36:54.639 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.638+0000 7f9bc8a00640 1 -- start start 2024-03-06T07:36:54.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.639+0000 7f9bc8a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9bc40cf3e0 0x7f9bc40ca5d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc8a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9bc40cabf0 con 0x7f9bc4144660 2024-03-06T07:36:54.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc8a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9bc40ce900 con 0x7f9bc4072680 2024-03-06T07:36:54.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc8a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9bc40cebe0 con 0x7f9bc40cf3e0 2024-03-06T07:36:54.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc1e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9bc4144660 0x7f9bc4071f70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55040/0 (socket says 172.21.15.86:55040) 2024-03-06T07:36:54.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc1e00640 1 -- 172.21.15.86:0/3165031634 learned_addr learned my addr 172.21.15.86:0/3165031634 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:54.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc2800640 1 --2- 172.21.15.86:0/3165031634 >> v2:172.21.15.86:3300/0 conn(0x7f9bc40cf3e0 0x7f9bc40ca5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:54.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3913426565 0 0) 0x7f9bc40cabf0 con 0x7f9bc4144660 2024-03-06T07:36:54.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.640+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9ba8003520 con 0x7f9bc4144660 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4237942181 0 0) 0x7f9ba8003520 con 0x7f9bc4144660 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 >> v2:172.21.15.86:3300/0 conn(0x7f9bc40cf3e0 msgr2=0x7f9bc40ca5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 --2- 172.21.15.86:0/3165031634 >> v2:172.21.15.86:3300/0 conn(0x7f9bc40cf3e0 0x7f9bc40ca5d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 >> v1:172.21.15.194:6789/0 conn(0x7f9bc4072680 legacy=0x7f9bc40c9ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9bc40ceec0 con 0x7f9bc4144660 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9bb40019f0 con 0x7f9bc4144660 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 >> v1:172.21.15.86:6789/0 conn(0x7f9bc4144660 legacy=0x7f9bc4071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80039b0 0x7f9ba8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ba8004800 0x7f9ba8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba80051c0 0x7f9ba800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc40cebe0 con 0x7f9ba80051c0 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ba800c060 con 0x7f9ba80039b0 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9ba800c260 con 0x7f9ba8004800 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.641+0000 7f9bc2800640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba80051c0 0x7f9ba800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.642+0000 7f9bc1e00640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80039b0 0x7f9ba8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:54.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.642+0000 7f9bc2800640 1 -- 172.21.15.86:0/3165031634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ba8004800 msgr2=0x7f9ba8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:54.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.642+0000 7f9bc2800640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9ba8004800 0x7f9ba8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.642+0000 7f9bc2800640 1 -- 172.21.15.86:0/3165031634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80039b0 msgr2=0x7f9ba8003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:54.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.642+0000 7f9bc2800640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80039b0 0x7f9ba8003dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.642+0000 7f9bc2800640 1 -- 172.21.15.86:0/3165031634 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ba8003520 con 0x7f9ba80051c0 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.643+0000 7f9bc2800640 1 -- 172.21.15.86:0/3165031634 reap_dead start 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.643+0000 7f9bc2800640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba80051c0 0x7f9ba800b9c0 secure :-1 s=READY pgs=1689 cs=0 l=1 rev1=1 crypto rx=0x7f9bbc009ac0 tx=0x7f9ba8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7fa00f414c14b897 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.643+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bbc009690 con 0x7f9ba80051c0 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.643+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9bbc004020 con 0x7f9ba80051c0 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.643+0000 7f9bc0a00640 1 -- 172.21.15.86:0/3165031634 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bbc004410 con 0x7f9ba80051c0 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.644+0000 7f9bc8a00640 1 -- 172.21.15.86:0/3165031634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba80051c0 msgr2=0x7f9ba800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.644+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba80051c0 0x7f9ba800b9c0 secure :-1 s=READY pgs=1689 cs=0 l=1 rev1=1 crypto rx=0x7f9bbc009ac0 tx=0x7f9ba8005830 comp rx=0 tx=0).stop 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.644+0000 7f9bc8a00640 1 -- 172.21.15.86:0/3165031634 shutdown_connections 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.644+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/3165031634 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9ba80051c0 0x7f9ba800b9c0 unknown :-1 s=CLOSED pgs=1689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.644+0000 7f9bc8a00640 1 -- 172.21.15.86:0/3165031634 >> 172.21.15.86:0/3165031634 conn(0x7f9bc40d0b90 msgr2=0x7f9bc40d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:54.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.644+0000 7f9bc8a00640 1 -- 172.21.15.86:0/3165031634 shutdown_connections 2024-03-06T07:36:54.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.644+0000 7f9bc8a00640 1 -- 172.21.15.86:0/3165031634 wait complete. 2024-03-06T07:36:54.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.645+0000 7f9bc8a00640 1 Processor -- start 2024-03-06T07:36:54.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.646+0000 7f9bc8a00640 1 -- start start 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.646+0000 7f9bc8a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80051c0 0x7f9bc413c670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc8a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160bb0 0x7f9bc4161030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9bc415e530 0x7f9bc415e9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc8a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc40720f0 con 0x7f9bc415e530 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc8a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc4071f70 con 0x7f9ba80051c0 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc8a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9bc4072270 con 0x7f9bc4160bb0 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9bc415e530 0x7f9bc415e9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9bc415e530 0x7f9bc415e9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38408/0 (socket says 172.21.15.86:38408) 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 -- 172.21.15.86:0/2645619645 learned_addr learned my addr 172.21.15.86:0/2645619645 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:54.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc1e00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80051c0 0x7f9bc413c670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:54.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc1400640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160bb0 0x7f9bc4161030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:54.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 -- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160bb0 msgr2=0x7f9bc4161030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:54.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160bb0 0x7f9bc4161030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 -- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80051c0 msgr2=0x7f9bc413c670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:54.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80051c0 0x7f9bc413c670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.647+0000 7f9bc2800640 1 -- 172.21.15.86:0/2645619645 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9bc40b3c00 con 0x7f9bc415e530 2024-03-06T07:36:54.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.648+0000 7f9bc2800640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9bc415e530 0x7f9bc415e9b0 secure :-1 s=READY pgs=1690 cs=0 l=1 rev1=1 crypto rx=0x7f9bbc02c380 tx=0x7f9bbc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:54.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.648+0000 7f9bba000640 1 -- 172.21.15.86:0/2645619645 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bbc02cc50 con 0x7f9bc415e530 2024-03-06T07:36:54.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.648+0000 7f9bba000640 1 -- 172.21.15.86:0/2645619645 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9bbc02cdf0 con 0x7f9bc415e530 2024-03-06T07:36:54.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.648+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9bc415f110 con 0x7f9bc415e530 2024-03-06T07:36:54.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.648+0000 7f9bba000640 1 -- 172.21.15.86:0/2645619645 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bbc003850 con 0x7f9bc415e530 2024-03-06T07:36:54.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.648+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bc415f6d0 con 0x7f9bc415e530 2024-03-06T07:36:54.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.649+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9bc404f330 con 0x7f9bc415e530 2024-03-06T07:36:54.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.650+0000 7f9bba000640 1 -- 172.21.15.86:0/2645619645 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9bbc0039f0 con 0x7f9bc415e530 2024-03-06T07:36:54.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.651+0000 7f9bba000640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba006b500 0x7f9ba006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:54.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.652+0000 7f9bba000640 1 -- 172.21.15.86:0/2645619645 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9bbc0aafa0 con 0x7f9bc415e530 2024-03-06T07:36:54.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.652+0000 7f9bc1e00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba006b500 0x7f9ba006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:54.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.652+0000 7f9bc1e00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba006b500 0x7f9ba006d9c0 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7f9bb4002020 tx=0x7f9bb4009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:54.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.656+0000 7f9bba000640 1 -- 172.21.15.86:0/2645619645 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9bbc079cb0 con 0x7f9bc415e530 2024-03-06T07:36:54.934 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.932+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9bc40ce000 con 0x7f9bc415e530 2024-03-06T07:36:54.935 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.933+0000 7f9bba000640 1 -- 172.21.15.86:0/2645619645 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f9bbc0795b0 con 0x7f9bc415e530 2024-03-06T07:36:54.935 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:54.935 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:54.935 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:54.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba006b500 msgr2=0x7f9ba006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:54.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba006b500 0x7f9ba006d9c0 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7f9bb4002020 tx=0x7f9bb4009040 comp rx=0 tx=0).stop 2024-03-06T07:36:54.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9bc415e530 msgr2=0x7f9bc415e9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:54.937 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9bc415e530 0x7f9bc415e9b0 secure :-1 s=READY pgs=1690 cs=0 l=1 rev1=1 crypto rx=0x7f9bbc02c380 tx=0x7f9bbc004730 comp rx=0 tx=0).stop 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 shutdown_connections 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9ba006b500 0x7f9ba006d9c0 unknown :-1 s=CLOSED pgs=719 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9bc415e530 0x7f9bc415e9b0 unknown :-1 s=CLOSED pgs=1690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9bc4160bb0 0x7f9bc4161030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 --2- 172.21.15.86:0/2645619645 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9ba80051c0 0x7f9bc413c670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 >> 172.21.15.86:0/2645619645 conn(0x7f9bc40d0b90 msgr2=0x7f9bc4143700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 shutdown_connections 2024-03-06T07:36:54.938 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:54.936+0000 7f9bc8a00640 1 -- 172.21.15.86:0/2645619645 wait complete. 2024-03-06T07:36:55.156 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:55.156 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:55.205 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:36:55.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:36:55.628 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:56.157 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:56.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.362+0000 7fed75a00640 1 Processor -- start 2024-03-06T07:36:56.364 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.362+0000 7fed75a00640 1 -- start start 2024-03-06T07:36:56.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed75a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fed70144c00 0x7fed7006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:56.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed75a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fed7015fe60 0x7fed700ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed75a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fed7006ee70 con 0x7fed7015f9d0 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed75a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fed7006cef0 con 0x7fed7015fe60 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed75a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fed7006d1d0 con 0x7fed70144c00 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed6f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fed7015fe60 0x7fed700ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed6f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fed7015fe60 0x7fed700ce7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49600/0 (socket says 172.21.15.86:49600) 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.364+0000 7fed6f400640 1 -- 172.21.15.86:0/362809757 learned_addr learned my addr 172.21.15.86:0/362809757 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.365+0000 7fed6ea00640 1 --2- 172.21.15.86:0/362809757 >> v2:172.21.15.194:3300/0 conn(0x7fed70144c00 0x7fed7006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.365+0000 7fed6f400640 1 -- 172.21.15.86:0/362809757 >> v2:172.21.15.194:3300/0 conn(0x7fed70144c00 msgr2=0x7fed7006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.365+0000 7fed6f400640 1 --2- 172.21.15.86:0/362809757 >> v2:172.21.15.194:3300/0 conn(0x7fed70144c00 0x7fed7006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.365+0000 7fed6f400640 1 -- 172.21.15.86:0/362809757 >> v1:172.21.15.194:6789/0 conn(0x7fed7015f9d0 legacy=0x7fed700ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:56.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.365+0000 7fed6f400640 1 -- 172.21.15.86:0/362809757 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed7006d4b0 con 0x7fed7015fe60 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.366+0000 7fed6f400640 1 --2- 172.21.15.86:0/362809757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed7015fe60 0x7fed700ce7b0 secure :-1 s=READY pgs=1691 cs=0 l=1 rev1=1 crypto rx=0x7fed700b25f0 tx=0x7fed6802b930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=78c7617470828d4f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.366+0000 7fed6e000640 1 -- 172.21.15.86:0/362809757 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed68035020 con 0x7fed7015fe60 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.366+0000 7fed6e000640 1 -- 172.21.15.86:0/362809757 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fed68031890 con 0x7fed7015fe60 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.366+0000 7fed6e000640 1 -- 172.21.15.86:0/362809757 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed68031c60 con 0x7fed7015fe60 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 -- 172.21.15.86:0/362809757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed7015fe60 msgr2=0x7fed700ce7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 --2- 172.21.15.86:0/362809757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed7015fe60 0x7fed700ce7b0 secure :-1 s=READY pgs=1691 cs=0 l=1 rev1=1 crypto rx=0x7fed700b25f0 tx=0x7fed6802b930 comp rx=0 tx=0).stop 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 -- 172.21.15.86:0/362809757 shutdown_connections 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 --2- 172.21.15.86:0/362809757 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed7015fe60 0x7fed700ce7b0 unknown :-1 s=CLOSED pgs=1691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 --2- 172.21.15.86:0/362809757 >> v2:172.21.15.194:3300/0 conn(0x7fed70144c00 0x7fed7006e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 -- 172.21.15.86:0/362809757 >> 172.21.15.86:0/362809757 conn(0x7fed700b1730 msgr2=0x7fed700b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:56.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 -- 172.21.15.86:0/362809757 shutdown_connections 2024-03-06T07:36:56.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.367+0000 7fed75a00640 1 -- 172.21.15.86:0/362809757 wait complete. 2024-03-06T07:36:56.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.369+0000 7fed75a00640 1 Processor -- start 2024-03-06T07:36:56.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.369+0000 7fed75a00640 1 -- start start 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed75a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed70144c00 0x7fed7015e210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed75a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed7015f9d0 0x7fed7015c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed75a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed7015fe60 0x7fed700cecd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed75a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed70160730 con 0x7fed70144c00 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed75a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed70160270 con 0x7fed7015f9d0 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed75a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fed7006cb50 con 0x7fed7015fe60 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed6ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed70144c00 0x7fed7015e210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed6ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed70144c00 0x7fed7015e210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49610/0 (socket says 172.21.15.86:49610) 2024-03-06T07:36:56.372 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.370+0000 7fed6ea00640 1 -- 172.21.15.86:0/3865158720 learned_addr learned my addr 172.21.15.86:0/3865158720 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed6f400640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed7015fe60 0x7fed700cecd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed66000640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed7015f9d0 0x7fed7015c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed6ea00640 1 -- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed7015fe60 msgr2=0x7fed700cecd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed6ea00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed7015fe60 0x7fed700cecd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed6ea00640 1 -- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed7015f9d0 msgr2=0x7fed7015c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed6ea00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed7015f9d0 0x7fed7015c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed6ea00640 1 -- 172.21.15.86:0/3865158720 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed7015ccb0 con 0x7fed70144c00 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed66000640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed7015f9d0 0x7fed7015c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed6ea00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed70144c00 0x7fed7015e210 secure :-1 s=READY pgs=1692 cs=0 l=1 rev1=1 crypto rx=0x7fed6000ccd0 tx=0x7fed6000a5c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed67400640 1 -- 172.21.15.86:0/3865158720 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed60003990 con 0x7fed70144c00 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed67400640 1 -- 172.21.15.86:0/3865158720 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fed60003b30 con 0x7fed70144c00 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed67400640 1 -- 172.21.15.86:0/3865158720 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed60004020 con 0x7fed70144c00 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fed7015e880 con 0x7fed70144c00 2024-03-06T07:36:56.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.371+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fed7015ee40 con 0x7fed70144c00 2024-03-06T07:36:56.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.373+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fed7004f330 con 0x7fed70144c00 2024-03-06T07:36:56.375 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.374+0000 7fed67400640 1 -- 172.21.15.86:0/3865158720 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fed600041c0 con 0x7fed70144c00 2024-03-06T07:36:56.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.374+0000 7fed67400640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed4c06b4d0 0x7fed4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:56.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.375+0000 7fed67400640 1 -- 172.21.15.86:0/3865158720 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fed6008a700 con 0x7fed70144c00 2024-03-06T07:36:56.376 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.375+0000 7fed66000640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed4c06b4d0 0x7fed4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:56.377 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.375+0000 7fed66000640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed4c06b4d0 0x7fed4c06d990 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7fed58007fd0 tx=0x7fed58009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:56.387 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.383+0000 7fed67400640 1 -- 172.21.15.86:0/3865158720 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fed60014030 con 0x7fed70144c00 2024-03-06T07:36:56.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.618+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fed70072890 con 0x7fed70144c00 2024-03-06T07:36:56.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.619+0000 7fed67400640 1 -- 172.21.15.86:0/3865158720 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fed60059540 con 0x7fed70144c00 2024-03-06T07:36:56.621 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:56.621 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:56.621 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:56.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed4c06b4d0 msgr2=0x7fed4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed4c06b4d0 0x7fed4c06d990 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7fed58007fd0 tx=0x7fed58009040 comp rx=0 tx=0).stop 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed70144c00 msgr2=0x7fed7015e210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed70144c00 0x7fed7015e210 secure :-1 s=READY pgs=1692 cs=0 l=1 rev1=1 crypto rx=0x7fed6000ccd0 tx=0x7fed6000a5c0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 shutdown_connections 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fed4c06b4d0 0x7fed4c06d990 unknown :-1 s=CLOSED pgs=720 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fed7015fe60 0x7fed700cecd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fed7015f9d0 0x7fed7015c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 --2- 172.21.15.86:0/3865158720 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fed70144c00 0x7fed7015e210 unknown :-1 s=CLOSED pgs=1692 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 >> 172.21.15.86:0/3865158720 conn(0x7fed700b1730 msgr2=0x7fed7006df90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 shutdown_connections 2024-03-06T07:36:56.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:56.622+0000 7fed75a00640 1 -- 172.21.15.86:0/3865158720 wait complete. 2024-03-06T07:36:56.842 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:56.842 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:57.630 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:57.843 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:58.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.054+0000 7fe19ee00640 1 Processor -- start 2024-03-06T07:36:58.056 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.054+0000 7fe19ee00640 1 -- start start 2024-03-06T07:36:58.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.055+0000 7fe19ee00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe1980d24c0 0x7fe1980d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:58.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.055+0000 7fe19ee00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe1980d4c50 con 0x7fe1980b0aa0 2024-03-06T07:36:58.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe19ee00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe1980cfa30 con 0x7fe1980b2de0 2024-03-06T07:36:58.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe19ee00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe1980d4a00 con 0x7fe1980d24c0 2024-03-06T07:36:58.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe197e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fe1980d24c0 0x7fe1980d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:58.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe19ce00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fe1980b0aa0 0x7fe1980b0f30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36036/0 (socket says 172.21.15.86:36036) 2024-03-06T07:36:58.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe19ce00640 1 -- 172.21.15.86:0/3589230306 learned_addr learned my addr 172.21.15.86:0/3589230306 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:58.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe197e00640 1 -- 172.21.15.86:0/3589230306 >> v1:172.21.15.194:6789/0 conn(0x7fe1980b2de0 legacy=0x7fe1980d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:58.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe197e00640 1 -- 172.21.15.86:0/3589230306 >> v1:172.21.15.86:6789/0 conn(0x7fe1980b0aa0 legacy=0x7fe1980b0f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:58.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe197e00640 1 -- 172.21.15.86:0/3589230306 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1980cfd10 con 0x7fe1980d24c0 2024-03-06T07:36:58.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.056+0000 7fe196a00640 1 -- 172.21.15.86:0/3589230306 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2050036181 0 0) 0x7fe1980d4c50 con 0x7fe1980b0aa0 2024-03-06T07:36:58.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe197e00640 1 --2- 172.21.15.86:0/3589230306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1980d1cc0 secure :-1 s=READY pgs=1694 cs=0 l=1 rev1=1 crypto rx=0x7fe1880097a0 tx=0x7fe18802b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d40163917e5da683 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:58.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe196a00640 1 -- 172.21.15.86:0/3589230306 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe1880315a0 con 0x7fe1980d24c0 2024-03-06T07:36:58.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe196a00640 1 -- 172.21.15.86:0/3589230306 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe188037070 con 0x7fe1980d24c0 2024-03-06T07:36:58.059 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe196a00640 1 -- 172.21.15.86:0/3589230306 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe188031960 con 0x7fe1980d24c0 2024-03-06T07:36:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe19ee00640 1 -- 172.21.15.86:0/3589230306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 msgr2=0x7fe1980d1cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe19ee00640 1 --2- 172.21.15.86:0/3589230306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1980d1cc0 secure :-1 s=READY pgs=1694 cs=0 l=1 rev1=1 crypto rx=0x7fe1880097a0 tx=0x7fe18802b840 comp rx=0 tx=0).stop 2024-03-06T07:36:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe19ee00640 1 -- 172.21.15.86:0/3589230306 shutdown_connections 2024-03-06T07:36:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe19ee00640 1 --2- 172.21.15.86:0/3589230306 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1980d1cc0 unknown :-1 s=CLOSED pgs=1694 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.058+0000 7fe19ee00640 1 -- 172.21.15.86:0/3589230306 >> 172.21.15.86:0/3589230306 conn(0x7fe1980d3ee0 msgr2=0x7fe1980d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.059+0000 7fe19ee00640 1 -- 172.21.15.86:0/3589230306 shutdown_connections 2024-03-06T07:36:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.059+0000 7fe19ee00640 1 -- 172.21.15.86:0/3589230306 wait complete. 2024-03-06T07:36:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.061+0000 7fe19ee00640 1 Processor -- start 2024-03-06T07:36:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.061+0000 7fe19ee00640 1 -- start start 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ee00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1980b0aa0 0x7fe1980c9260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ee00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1980b2de0 0x7fe1980c97a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ee00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1981489e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ee00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1980d0080 con 0x7fe1980d24c0 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ee00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1980cf7e0 con 0x7fe1980b2de0 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ee00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe1980b1640 con 0x7fe1980b0aa0 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1981489e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:58.065 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ce00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1981489e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49634/0 (socket says 172.21.15.86:49634) 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.062+0000 7fe19ce00640 1 -- 172.21.15.86:0/1664055911 learned_addr learned my addr 172.21.15.86:0/1664055911 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe197400640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1980b2de0 0x7fe1980c97a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe197e00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1980b0aa0 0x7fe1980c9260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe19ce00640 1 -- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1980b0aa0 msgr2=0x7fe1980c9260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe19ce00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1980b0aa0 0x7fe1980c9260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe19ce00640 1 -- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1980b2de0 msgr2=0x7fe1980c97a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe19ce00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1980b2de0 0x7fe1980c97a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe19ce00640 1 -- 172.21.15.86:0/1664055911 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1980c9f30 con 0x7fe1980d24c0 2024-03-06T07:36:58.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe197e00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1980b0aa0 0x7fe1980c9260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe19ce00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1981489e0 secure :-1 s=READY pgs=1695 cs=0 l=1 rev1=1 crypto rx=0x7fe19000c7a0 tx=0x7fe19000cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe18fe00640 1 -- 172.21.15.86:0/1664055911 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe190018070 con 0x7fe1980d24c0 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe18fe00640 1 -- 172.21.15.86:0/1664055911 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe19000a5e0 con 0x7fe1980d24c0 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe18fe00640 1 -- 172.21.15.86:0/1664055911 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe19000aa50 con 0x7fe1980d24c0 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.063+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe1980ca210 con 0x7fe1980d24c0 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.064+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe19815ea50 con 0x7fe1980d24c0 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.065+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe19804f330 con 0x7fe1980d24c0 2024-03-06T07:36:58.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.066+0000 7fe18fe00640 1 -- 172.21.15.86:0/1664055911 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe19000abf0 con 0x7fe1980d24c0 2024-03-06T07:36:58.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.066+0000 7fe18fe00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe17806b4d0 0x7fe17806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:58.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.067+0000 7fe18fe00640 1 -- 172.21.15.86:0/1664055911 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fe1900898a0 con 0x7fe1980d24c0 2024-03-06T07:36:58.068 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.067+0000 7fe197e00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe17806b4d0 0x7fe17806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:58.069 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.067+0000 7fe197e00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe17806b4d0 0x7fe17806d990 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7fe188036000 tx=0x7fe188008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:58.072 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.071+0000 7fe18fe00640 1 -- 172.21.15.86:0/1664055911 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe1900586e0 con 0x7fe1980d24c0 2024-03-06T07:36:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.343+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe1980d22a0 con 0x7fe1980d24c0 2024-03-06T07:36:58.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.344+0000 7fe18fe00640 1 -- 172.21.15.86:0/1664055911 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fe190089030 con 0x7fe1980d24c0 2024-03-06T07:36:58.346 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:36:58.346 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:36:58.346 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:36:58.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe17806b4d0 msgr2=0x7fe17806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:58.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe17806b4d0 0x7fe17806d990 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7fe188036000 tx=0x7fe188008030 comp rx=0 tx=0).stop 2024-03-06T07:36:58.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 msgr2=0x7fe1981489e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:58.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1981489e0 secure :-1 s=READY pgs=1695 cs=0 l=1 rev1=1 crypto rx=0x7fe19000c7a0 tx=0x7fe19000cc60 comp rx=0 tx=0).stop 2024-03-06T07:36:58.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 shutdown_connections 2024-03-06T07:36:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fe17806b4d0 0x7fe17806d990 unknown :-1 s=CLOSED pgs=721 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fe1980d24c0 0x7fe1981489e0 unknown :-1 s=CLOSED pgs=1695 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fe1980b2de0 0x7fe1980c97a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 --2- 172.21.15.86:0/1664055911 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fe1980b0aa0 0x7fe1980c9260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.346+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 >> 172.21.15.86:0/1664055911 conn(0x7fe1980d3ee0 msgr2=0x7fe1980d28a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.347+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 shutdown_connections 2024-03-06T07:36:58.349 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:58.347+0000 7fe19ee00640 1 -- 172.21.15.86:0/1664055911 wait complete. 2024-03-06T07:36:58.567 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:36:58.567 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:36:59.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:36:59.632 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:36:59.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.795+0000 7f88a8600640 1 Processor -- start 2024-03-06T07:36:59.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.795+0000 7f88a8600640 1 -- start start 2024-03-06T07:36:59.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.797+0000 7f88a8600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 0x7f88a006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.797+0000 7f88a8600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f88a015fe30 0x7f88a00ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.797+0000 7f88a8600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f88a006ee70 con 0x7f88a015f9d0 2024-03-06T07:36:59.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.797+0000 7f88a8600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f88a006cef0 con 0x7f88a015fe30 2024-03-06T07:36:59.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.797+0000 7f88a8600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f88a006d1d0 con 0x7f88a0144c00 2024-03-06T07:36:59.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.797+0000 7f88a5200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f88a015f9d0 0x7f88a00ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36042/0 (socket says 172.21.15.86:36042) 2024-03-06T07:36:59.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.797+0000 7f88a5200640 1 -- 172.21.15.86:0/1787319971 learned_addr learned my addr 172.21.15.86:0/1787319971 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:36:59.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f88a5c00640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 0x7f88a006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f88a6600640 1 --2- 172.21.15.86:0/1787319971 >> v2:172.21.15.194:3300/0 conn(0x7f88a015fe30 0x7f88a00ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3497667772 0 0) 0x7f888c001110 con 0x7f88a015f9d0 2024-03-06T07:36:59.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8888003520 con 0x7f88a015f9d0 2024-03-06T07:36:59.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3984594577 0 0) 0x7f8888003520 con 0x7f88a015f9d0 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 msgr2=0x7f88a006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 0x7f88a006e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 >> v2:172.21.15.194:3300/0 conn(0x7f88a015fe30 msgr2=0x7f88a00ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 --2- 172.21.15.86:0/1787319971 >> v2:172.21.15.194:3300/0 conn(0x7f88a015fe30 0x7f88a00ce7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88a006d4b0 con 0x7f88a015f9d0 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f888c001840 con 0x7f88a015f9d0 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 >> v1:172.21.15.86:6789/0 conn(0x7f88a015f9d0 legacy=0x7f88a00ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 0x7f88880039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8888003f30 0x7f8888004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888004d40 0x7f88880051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88a006cef0 con 0x7f8888003f30 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88a006d1d0 con 0x7f8888004d40 2024-03-06T07:36:59.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.798+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f888800b9c0 con 0x7f88a0144c00 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a5200640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8888003f30 0x7f8888004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a6600640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888004d40 0x7f88880051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a5c00640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 0x7f88880039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a5200640 1 -- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 msgr2=0x7f88880039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a5200640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a0144c00 0x7f88880039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a5200640 1 -- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888004d40 msgr2=0x7f88880051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a5200640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888004d40 0x7f88880051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a5200640 1 -- 172.21.15.86:0/1787319971 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8888003520 con 0x7f8888003f30 2024-03-06T07:36:59.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.799+0000 7f88a6600640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888004d40 0x7f88880051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.800+0000 7f88a5200640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8888003f30 0x7f8888004800 secure :-1 s=READY pgs=1697 cs=0 l=1 rev1=1 crypto rx=0x7f888c00c3e0 tx=0x7f8888005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9529292aadf94bfa server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.800+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f888c00c0e0 con 0x7f8888003f30 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.800+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f888c030070 con 0x7f8888003f30 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.800+0000 7f889fe00640 1 -- 172.21.15.86:0/1787319971 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f888c0058d0 con 0x7f8888003f30 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.801+0000 7f88a8600640 1 -- 172.21.15.86:0/1787319971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8888003f30 msgr2=0x7f8888004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.801+0000 7f88a8600640 1 --2- 172.21.15.86:0/1787319971 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8888003f30 0x7f8888004800 secure :-1 s=READY pgs=1697 cs=0 l=1 rev1=1 crypto rx=0x7f888c00c3e0 tx=0x7f8888005830 comp rx=0 tx=0).stop 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.801+0000 7f88a6600640 1 -- 172.21.15.86:0/1787319971 reap_dead start 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.801+0000 7f88a8600640 1 -- 172.21.15.86:0/1787319971 shutdown_connections 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.801+0000 7f88a8600640 1 -- 172.21.15.86:0/1787319971 >> 172.21.15.86:0/1787319971 conn(0x7f88a00b1730 msgr2=0x7f88a00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.801+0000 7f88a8600640 1 -- 172.21.15.86:0/1787319971 shutdown_connections 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.801+0000 7f88a8600640 1 -- 172.21.15.86:0/1787319971 wait complete. 2024-03-06T07:36:59.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.803+0000 7f88a8600640 1 Processor -- start 2024-03-06T07:36:59.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.803+0000 7f88a8600640 1 -- start start 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a8600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888003f30 0x7f88a015aaa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a8600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a00b3eb0 0x7f88a0138fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a8600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88a00b4330 0x7f88a0139510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a8600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88a0160730 con 0x7f88a00b4330 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a8600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88a0160280 con 0x7f8888003f30 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a8600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f88a006cb50 con 0x7f88a00b3eb0 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a6600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88a00b4330 0x7f88a0139510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a6600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88a00b4330 0x7f88a0139510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49660/0 (socket says 172.21.15.86:49660) 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.804+0000 7f88a6600640 1 -- 172.21.15.86:0/470283061 learned_addr learned my addr 172.21.15.86:0/470283061 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a5c00640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888003f30 0x7f88a015aaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a5200640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a00b3eb0 0x7f88a0138fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a6600640 1 -- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a00b3eb0 msgr2=0x7f88a0138fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a6600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a00b3eb0 0x7f88a0138fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a6600640 1 -- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888003f30 msgr2=0x7f88a015aaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a6600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888003f30 0x7f88a015aaa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a6600640 1 -- 172.21.15.86:0/470283061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88a00b2b30 con 0x7f88a00b4330 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a5200640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a00b3eb0 0x7f88a0138fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a5c00640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888003f30 0x7f88a015aaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a6600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88a00b4330 0x7f88a0139510 secure :-1 s=READY pgs=1698 cs=0 l=1 rev1=1 crypto rx=0x7f88a0160210 tx=0x7f889800f520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f889d600640 1 -- 172.21.15.86:0/470283061 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8898016620 con 0x7f88a00b4330 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f889d600640 1 -- 172.21.15.86:0/470283061 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f889801b070 con 0x7f88a00b4330 2024-03-06T07:36:59.809 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f88a00b2e10 con 0x7f88a00b4330 2024-03-06T07:36:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.805+0000 7f889d600640 1 -- 172.21.15.86:0/470283061 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f88980169e0 con 0x7f88a00b4330 2024-03-06T07:36:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.806+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f88a00b33d0 con 0x7f88a00b4330 2024-03-06T07:36:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.807+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f88a004f330 con 0x7f88a00b4330 2024-03-06T07:36:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.808+0000 7f889d600640 1 -- 172.21.15.86:0/470283061 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8898005d10 con 0x7f88a00b4330 2024-03-06T07:36:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.809+0000 7f889d600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f888006b4d0 0x7f888006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:36:59.810 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.809+0000 7f889d600640 1 -- 172.21.15.86:0/470283061 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f889808af10 con 0x7f88a00b4330 2024-03-06T07:36:59.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.809+0000 7f88a5c00640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f888006b4d0 0x7f888006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:36:59.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.810+0000 7f88a5c00640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f888006b4d0 0x7f888006d990 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7f8894009910 tx=0x7f8894008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:36:59.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:36:59.814+0000 7f889d600640 1 -- 172.21.15.86:0/470283061 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8898025020 con 0x7f88a00b4330 2024-03-06T07:37:00.057 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.056+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f88a0145450 con 0x7f88a00b4330 2024-03-06T07:37:00.058 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.056+0000 7f889d600640 1 -- 172.21.15.86:0/470283061 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f8898059cb0 con 0x7f88a00b4330 2024-03-06T07:37:00.058 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:00.058 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:00.058 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:00.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f888006b4d0 msgr2=0x7f888006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:00.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f888006b4d0 0x7f888006d990 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7f8894009910 tx=0x7f8894008040 comp rx=0 tx=0).stop 2024-03-06T07:37:00.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88a00b4330 msgr2=0x7f88a0139510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:00.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88a00b4330 0x7f88a0139510 secure :-1 s=READY pgs=1698 cs=0 l=1 rev1=1 crypto rx=0x7f88a0160210 tx=0x7f889800f520 comp rx=0 tx=0).stop 2024-03-06T07:37:00.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 shutdown_connections 2024-03-06T07:37:00.061 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f888006b4d0 0x7f888006d990 unknown :-1 s=CLOSED pgs=722 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:00.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f88a00b4330 0x7f88a0139510 unknown :-1 s=CLOSED pgs=1698 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:00.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f88a00b3eb0 0x7f88a0138fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:00.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 --2- 172.21.15.86:0/470283061 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8888003f30 0x7f88a015aaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:00.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 >> 172.21.15.86:0/470283061 conn(0x7f88a00b1730 msgr2=0x7f88a00afa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:00.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.059+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 shutdown_connections 2024-03-06T07:37:00.062 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:00.060+0000 7f88a8600640 1 -- 172.21.15.86:0/470283061 wait complete. 2024-03-06T07:37:00.280 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:00.280 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:01.282 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.499+0000 7f9270e00640 1 Processor -- start 2024-03-06T07:37:01.501 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.499+0000 7f9270e00640 1 -- start start 2024-03-06T07:37:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.501+0000 7f9270e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f926c15a4b0 0x7f926c071380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.501+0000 7f9270e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 0x7f926c0b3d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.501+0000 7f9270e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f926c06c940 con 0x7f926c0b42d0 2024-03-06T07:37:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.501+0000 7f9270e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f926c0ce680 con 0x7f926c15a4b0 2024-03-06T07:37:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.501+0000 7f9270e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f926c0cec40 con 0x7f926c0cbb60 2024-03-06T07:37:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.501+0000 7f926be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f926c0b42d0 0x7f926c0cd460 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36052/0 (socket says 172.21.15.86:36052) 2024-03-06T07:37:01.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.501+0000 7f926be00640 1 -- 172.21.15.86:0/1717142094 learned_addr learned my addr 172.21.15.86:0/1717142094 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:01.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f9262a00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 0x7f926c0b3d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.504 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926b400640 1 --2- 172.21.15.86:0/1717142094 >> v2:172.21.15.194:3300/0 conn(0x7f926c15a4b0 0x7f926c071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2422584761 0 0) 0x7f9264001490 con 0x7f926c0b42d0 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9250005730 con 0x7f926c0b42d0 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1671627461 0 0) 0x7f9250005730 con 0x7f926c0b42d0 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 msgr2=0x7f926c0b3d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 0x7f926c0b3d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 >> v2:172.21.15.194:3300/0 conn(0x7f926c15a4b0 msgr2=0x7f926c071380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 --2- 172.21.15.86:0/1717142094 >> v2:172.21.15.194:3300/0 conn(0x7f926c15a4b0 0x7f926c071380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f926c071980 con 0x7f926c0b42d0 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f9264001bc0 con 0x7f926c0b42d0 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 >> v1:172.21.15.86:6789/0 conn(0x7f926c0b42d0 legacy=0x7f926c0cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.502+0000 7f926aa00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9250005bc0 0x7f9250005fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f926aa00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f9250006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f926aa00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 0x7f92500073d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f926c0ce680 con 0x7f9250006a10 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f926c0cec40 con 0x7f9250005bc0 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f925000dbd0 con 0x7f926c0cbb60 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f9262a00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f9250006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f926b400640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9250005bc0 0x7f9250005fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f926be00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 0x7f92500073d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f9262a00640 1 -- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 msgr2=0x7f92500073d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f9262a00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c0cbb60 0x7f92500073d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f9262a00640 1 -- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9250005bc0 msgr2=0x7f9250005fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f9262a00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9250005bc0 0x7f9250005fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.503+0000 7f9262a00640 1 -- 172.21.15.86:0/1717142094 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9250005730 con 0x7f9250006a10 2024-03-06T07:37:01.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.504+0000 7f9262a00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f9250006e90 secure :-1 s=READY pgs=1700 cs=0 l=1 rev1=1 crypto rx=0x7f925c009af0 tx=0x7f9250007a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1faa2afd3b318e9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f925c039070 con 0x7f9250006a10 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f925c02c6b0 con 0x7f9250006a10 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f926aa00640 1 -- 172.21.15.86:0/1717142094 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f925c009690 con 0x7f9250006a10 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f9270e00640 1 -- 172.21.15.86:0/1717142094 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 msgr2=0x7f9250006e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f9270e00640 1 --2- 172.21.15.86:0/1717142094 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f9250006e90 secure :-1 s=READY pgs=1700 cs=0 l=1 rev1=1 crypto rx=0x7f925c009af0 tx=0x7f9250007a40 comp rx=0 tx=0).stop 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f926be00640 1 -- 172.21.15.86:0/1717142094 reap_dead start 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f9270e00640 1 -- 172.21.15.86:0/1717142094 shutdown_connections 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f9270e00640 1 -- 172.21.15.86:0/1717142094 >> 172.21.15.86:0/1717142094 conn(0x7f926c06afc0 msgr2=0x7f926c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.505+0000 7f9270e00640 1 -- 172.21.15.86:0/1717142094 shutdown_connections 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.506+0000 7f9270e00640 1 -- 172.21.15.86:0/1717142094 wait complete. 2024-03-06T07:37:01.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.507+0000 7f9270e00640 1 Processor -- start 2024-03-06T07:37:01.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.507+0000 7f9270e00640 1 -- start start 2024-03-06T07:37:01.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.508+0000 7f9270e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f926c15a900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.508+0000 7f9270e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c14fce0 0x7f926c1598a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.508+0000 7f9270e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c14e0c0 0x7f926c144e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.508+0000 7f9270e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f926c0b4760 con 0x7f9250006a10 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.508+0000 7f9270e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f926c0ce8d0 con 0x7f926c14fce0 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.508+0000 7f9270e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f926c0d00e0 con 0x7f926c14e0c0 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f926c15a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f926c15a900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49678/0 (socket says 172.21.15.86:49678) 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 -- 172.21.15.86:0/1412812533 learned_addr learned my addr 172.21.15.86:0/1412812533 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f9262a00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c14fce0 0x7f926c1598a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 -- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c14e0c0 msgr2=0x7f926c144e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c14e0c0 0x7f926c144e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 -- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c14fce0 msgr2=0x7f926c1598a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c14fce0 0x7f926c1598a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 -- 172.21.15.86:0/1412812533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f926c150230 con 0x7f9250006a10 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f9262a00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c14fce0 0x7f926c1598a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.509+0000 7f926b400640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f926c15a900 secure :-1 s=READY pgs=1701 cs=0 l=1 rev1=1 crypto rx=0x7f9254011730 tx=0x7f9254011bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.510+0000 7f9263e00640 1 -- 172.21.15.86:0/1412812533 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f925400f9c0 con 0x7f9250006a10 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.510+0000 7f9263e00640 1 -- 172.21.15.86:0/1412812533 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f925400fb60 con 0x7f9250006a10 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.510+0000 7f9263e00640 1 -- 172.21.15.86:0/1412812533 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9254015460 con 0x7f9250006a10 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.510+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f926c0c9ec0 con 0x7f9250006a10 2024-03-06T07:37:01.513 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.510+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f926c0ca480 con 0x7f9250006a10 2024-03-06T07:37:01.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.511+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f926c04f330 con 0x7f9250006a10 2024-03-06T07:37:01.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.512+0000 7f9263e00640 1 -- 172.21.15.86:0/1412812533 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9254015600 con 0x7f9250006a10 2024-03-06T07:37:01.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.513+0000 7f9263e00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:01.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.513+0000 7f9263e00640 1 -- 172.21.15.86:0/1412812533 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f925408ac90 con 0x7f9250006a10 2024-03-06T07:37:01.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.513+0000 7f9262a00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:01.515 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.514+0000 7f9262a00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7f925c036000 tx=0x7f925c030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:01.519 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.517+0000 7f9263e00640 1 -- 172.21.15.86:0/1412812533 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9254019030 con 0x7f9250006a10 2024-03-06T07:37:01.634 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:01.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.751+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f926c1560e0 con 0x7f9250006a10 2024-03-06T07:37:01.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.753+0000 7f9263e00640 1 -- 172.21.15.86:0/1412812533 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f9254059ac0 con 0x7f9250006a10 2024-03-06T07:37:01.754 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:01.754 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:01.755 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 msgr2=0x7f924806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7f925c036000 tx=0x7f925c030000 comp rx=0 tx=0).stop 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 msgr2=0x7f926c15a900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f926c15a900 secure :-1 s=READY pgs=1701 cs=0 l=1 rev1=1 crypto rx=0x7f9254011730 tx=0x7f9254011bf0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 shutdown_connections 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f924806b4d0 0x7f924806d990 unknown :-1 s=CLOSED pgs=723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f926c14e0c0 0x7f926c144e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f926c14fce0 0x7f926c1598a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 --2- 172.21.15.86:0/1412812533 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9250006a10 0x7f926c15a900 unknown :-1 s=CLOSED pgs=1701 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 >> 172.21.15.86:0/1412812533 conn(0x7f926c06afc0 msgr2=0x7f926c06d3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 shutdown_connections 2024-03-06T07:37:01.757 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:01.755+0000 7f9270e00640 1 -- 172.21.15.86:0/1412812533 wait complete. 2024-03-06T07:37:01.976 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:01.976 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:02.977 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:03.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.186+0000 7f16fc200640 1 Processor -- start 2024-03-06T07:37:03.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.187+0000 7f16fc200640 1 -- start start 2024-03-06T07:37:03.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.188+0000 7f16fc200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f40af820 0x7f16f40b33b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:03.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.188+0000 7f16fc200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16f40b38f0 0x7f16f406b580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:03.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.188+0000 7f16fc200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16f40b3d70 con 0x7f16f415e160 2024-03-06T07:37:03.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.188+0000 7f16fc200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f16f406b0f0 con 0x7f16f40b38f0 2024-03-06T07:37:03.190 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.188+0000 7f16fc200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16f40b5620 con 0x7f16f40af820 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16fa200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16f40b38f0 0x7f16f406b580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16fa200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f16f40b38f0 0x7f16f406b580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49694/0 (socket says 172.21.15.86:49694) 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16fa200640 1 -- 172.21.15.86:0/3687211610 learned_addr learned my addr 172.21.15.86:0/3687211610 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16f8e00640 1 --2- 172.21.15.86:0/3687211610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f40af820 0x7f16f40b33b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16fa200640 1 -- 172.21.15.86:0/3687211610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f40af820 msgr2=0x7f16f40b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16fa200640 1 --2- 172.21.15.86:0/3687211610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f40af820 0x7f16f40b33b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16fa200640 1 -- 172.21.15.86:0/3687211610 >> v1:172.21.15.194:6789/0 conn(0x7f16f415e160 legacy=0x7f16f4162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:03.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.189+0000 7f16fa200640 1 -- 172.21.15.86:0/3687211610 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16f406f1f0 con 0x7f16f40b38f0 2024-03-06T07:37:03.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.191+0000 7f16fa200640 1 --2- 172.21.15.86:0/3687211610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f406b580 secure :-1 s=READY pgs=1702 cs=0 l=1 rev1=1 crypto rx=0x7f16ec009e50 tx=0x7f16ec02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a264910ce74e06e0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:03.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.191+0000 7f16f3e00640 1 -- 172.21.15.86:0/3687211610 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16ec02ca70 con 0x7f16f40b38f0 2024-03-06T07:37:03.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.191+0000 7f16f3e00640 1 -- 172.21.15.86:0/3687211610 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16ec02cc10 con 0x7f16f40b38f0 2024-03-06T07:37:03.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.191+0000 7f16f3e00640 1 -- 172.21.15.86:0/3687211610 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16ec0343f0 con 0x7f16f40b38f0 2024-03-06T07:37:03.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 -- 172.21.15.86:0/3687211610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 msgr2=0x7f16f406b580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:03.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 --2- 172.21.15.86:0/3687211610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f406b580 secure :-1 s=READY pgs=1702 cs=0 l=1 rev1=1 crypto rx=0x7f16ec009e50 tx=0x7f16ec02bed0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 -- 172.21.15.86:0/3687211610 shutdown_connections 2024-03-06T07:37:03.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 --2- 172.21.15.86:0/3687211610 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f406b580 unknown :-1 s=CLOSED pgs=1702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 --2- 172.21.15.86:0/3687211610 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f40af820 0x7f16f40b33b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 -- 172.21.15.86:0/3687211610 >> 172.21.15.86:0/3687211610 conn(0x7f16f40d0340 msgr2=0x7f16f40d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:03.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 -- 172.21.15.86:0/3687211610 shutdown_connections 2024-03-06T07:37:03.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.192+0000 7f16fc200640 1 -- 172.21.15.86:0/3687211610 wait complete. 2024-03-06T07:37:03.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.194+0000 7f16fc200640 1 Processor -- start 2024-03-06T07:37:03.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.194+0000 7f16fc200640 1 -- start start 2024-03-06T07:37:03.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16fc200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16f40af820 0x7f16f40cf8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:03.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16fc200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f40cd5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:03.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16fc200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f415e160 0x7f16f4149e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:03.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16fc200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16f406ef10 con 0x7f16f40b38f0 2024-03-06T07:37:03.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16fc200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16f406ec30 con 0x7f16f40af820 2024-03-06T07:37:03.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16fc200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16f406bac0 con 0x7f16f415e160 2024-03-06T07:37:03.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16f8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f40cd5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:03.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16f8e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f40cd5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49706/0 (socket says 172.21.15.86:49706) 2024-03-06T07:37:03.197 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16f8e00640 1 -- 172.21.15.86:0/1187334941 learned_addr learned my addr 172.21.15.86:0/1187334941 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:03.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16fa200640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f415e160 0x7f16f4149e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:03.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16f8e00640 1 -- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f415e160 msgr2=0x7f16f4149e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:03.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16f8e00640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f415e160 0x7f16f4149e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16f9800640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16f40af820 0x7f16f40cf8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:03.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.195+0000 7f16f8e00640 1 -- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16f40af820 msgr2=0x7f16f40cf8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:03.198 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16f8e00640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16f40af820 0x7f16f40cf8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16f8e00640 1 -- 172.21.15.86:0/1187334941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16f40ce050 con 0x7f16f40b38f0 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16f8e00640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f40cd5d0 secure :-1 s=READY pgs=1703 cs=0 l=1 rev1=1 crypto rx=0x7f16e000d7d0 tx=0x7f16e000dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16f1600640 1 -- 172.21.15.86:0/1187334941 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16e000b6c0 con 0x7f16f40b38f0 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16f1600640 1 -- 172.21.15.86:0/1187334941 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16e0019070 con 0x7f16f40b38f0 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16f40cdd70 con 0x7f16f40b38f0 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16f1600640 1 -- 172.21.15.86:0/1187334941 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16e000ba80 con 0x7f16f40b38f0 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.196+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16f4155660 con 0x7f16f40b38f0 2024-03-06T07:37:03.199 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.198+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16f4074ae0 con 0x7f16f40b38f0 2024-03-06T07:37:03.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.198+0000 7f16f1600640 1 -- 172.21.15.86:0/1187334941 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f16e000bc20 con 0x7f16f40b38f0 2024-03-06T07:37:03.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.199+0000 7f16f1600640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16d406b4d0 0x7f16d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:03.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.199+0000 7f16f1600640 1 -- 172.21.15.86:0/1187334941 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f16e008a740 con 0x7f16f40b38f0 2024-03-06T07:37:03.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.199+0000 7f16f9800640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16d406b4d0 0x7f16d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:03.201 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.200+0000 7f16f9800640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16d406b4d0 0x7f16d406d990 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7f16e8001500 tx=0x7f16e8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:03.205 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.204+0000 7f16f1600640 1 -- 172.21.15.86:0/1187334941 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f16e001c360 con 0x7f16f40b38f0 2024-03-06T07:37:03.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.437+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f16f40d11e0 con 0x7f16f40b38f0 2024-03-06T07:37:03.440 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.438+0000 7f16f1600640 1 -- 172.21.15.86:0/1187334941 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f16e00594e0 con 0x7f16f40b38f0 2024-03-06T07:37:03.440 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:03.440 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:03.440 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16d406b4d0 msgr2=0x7f16d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16d406b4d0 0x7f16d406d990 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7f16e8001500 tx=0x7f16e8009040 comp rx=0 tx=0).stop 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 msgr2=0x7f16f40cd5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f40cd5d0 secure :-1 s=READY pgs=1703 cs=0 l=1 rev1=1 crypto rx=0x7f16e000d7d0 tx=0x7f16e000dc90 comp rx=0 tx=0).stop 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 shutdown_connections 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f16d406b4d0 0x7f16d406d990 unknown :-1 s=CLOSED pgs=724 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f16f415e160 0x7f16f4149e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f16f40b38f0 0x7f16f40cd5d0 unknown :-1 s=CLOSED pgs=1703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.443 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 --2- 172.21.15.86:0/1187334941 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f16f40af820 0x7f16f40cf8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:03.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 >> 172.21.15.86:0/1187334941 conn(0x7f16f40d0340 msgr2=0x7f16f40b1540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:03.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 shutdown_connections 2024-03-06T07:37:03.444 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:03.441+0000 7f16fc200640 1 -- 172.21.15.86:0/1187334941 wait complete. 2024-03-06T07:37:03.636 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:03.662 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:03.662 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:04.663 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:04.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.876+0000 7f84dd800640 1 Processor -- start 2024-03-06T07:37:04.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.876+0000 7f84dd800640 1 -- start start 2024-03-06T07:37:04.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84dd800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d806e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:04.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84dd800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f84d815fe30 0x7f84d80ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:04.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84dd800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f84d806ee70 con 0x7f84d815f9d0 2024-03-06T07:37:04.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84dd800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f84d806cef0 con 0x7f84d815fe30 2024-03-06T07:37:04.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84dd800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f84d806d1d0 con 0x7f84d8144c00 2024-03-06T07:37:04.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84d6000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f84d815f9d0 0x7f84d80ce0a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36060/0 (socket says 172.21.15.86:36060) 2024-03-06T07:37:04.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84d6000640 1 -- 172.21.15.86:0/2241541907 learned_addr learned my addr 172.21.15.86:0/2241541907 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:04.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.878+0000 7f84d7400640 1 --2- 172.21.15.86:0/2241541907 >> v2:172.21.15.194:3300/0 conn(0x7f84d815fe30 0x7f84d80ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:04.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d6a00640 1 --2- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d806e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:04.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d5600640 1 -- 172.21.15.86:0/2241541907 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4224492318 0 0) 0x7f84c0001110 con 0x7f84d815f9d0 2024-03-06T07:37:04.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d5600640 1 -- 172.21.15.86:0/2241541907 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f84bc003520 con 0x7f84d815f9d0 2024-03-06T07:37:04.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d7400640 1 -- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 msgr2=0x7f84d806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:04.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d7400640 1 --2- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d806e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:04.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d7400640 1 -- 172.21.15.86:0/2241541907 >> v1:172.21.15.86:6789/0 conn(0x7f84d815f9d0 legacy=0x7f84d80ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:04.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d5600640 1 -- 172.21.15.86:0/2241541907 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2481820585 0 0) 0x7f84bc003520 con 0x7f84d815f9d0 2024-03-06T07:37:04.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.879+0000 7f84d7400640 1 -- 172.21.15.86:0/2241541907 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84d806d4b0 con 0x7f84d815fe30 2024-03-06T07:37:04.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.881+0000 7f84d7400640 1 --2- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 0x7f84d80ce7b0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f84d0009d90 tx=0x7f84d002bd70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=45eac9a4db0ec55d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:04.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.881+0000 7f84d5600640 1 -- 172.21.15.86:0/2241541907 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84d0039070 con 0x7f84d815fe30 2024-03-06T07:37:04.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.881+0000 7f84d5600640 1 -- 172.21.15.86:0/2241541907 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f84d002c8a0 con 0x7f84d815fe30 2024-03-06T07:37:04.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.881+0000 7f84d5600640 1 -- 172.21.15.86:0/2241541907 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84c0001840 con 0x7f84d815fe30 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 -- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 msgr2=0x7f84d80ce7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 --2- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 0x7f84d80ce7b0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f84d0009d90 tx=0x7f84d002bd70 comp rx=0 tx=0).stop 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 -- 172.21.15.86:0/2241541907 shutdown_connections 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 --2- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 0x7f84d80ce7b0 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 --2- 172.21.15.86:0/2241541907 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d806e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 -- 172.21.15.86:0/2241541907 >> 172.21.15.86:0/2241541907 conn(0x7f84d80b1730 msgr2=0x7f84d80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 -- 172.21.15.86:0/2241541907 shutdown_connections 2024-03-06T07:37:04.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.882+0000 7f84dd800640 1 -- 172.21.15.86:0/2241541907 wait complete. 2024-03-06T07:37:04.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.884+0000 7f84dd800640 1 Processor -- start 2024-03-06T07:37:04.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.884+0000 7f84dd800640 1 -- start start 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84dd800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d81f81d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84dd800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84d815f9d0 0x7f84d813d390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84dd800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 0x7f84d81f7630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84dd800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84d8160730 con 0x7f84d815f9d0 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84dd800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84d8160280 con 0x7f84d815fe30 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84dd800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f84d806cb50 con 0x7f84d8144c00 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84d6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84d815f9d0 0x7f84d813d390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84d6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84d815f9d0 0x7f84d813d390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:49718/0 (socket says 172.21.15.86:49718) 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84d6000640 1 -- 172.21.15.86:0/3308726116 learned_addr learned my addr 172.21.15.86:0/3308726116 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84d6a00640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d81f81d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:04.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.885+0000 7f84d7400640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 0x7f84d81f7630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:04.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84d6000640 1 -- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 msgr2=0x7f84d81f81d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:04.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84d6000640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d81f81d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84d6000640 1 -- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 msgr2=0x7f84d81f7630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84d6000640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 0x7f84d81f7630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84d6000640 1 -- 172.21.15.86:0/3308726116 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84d813d8d0 con 0x7f84d815f9d0 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84d6000640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84d815f9d0 0x7f84d813d390 secure :-1 s=READY pgs=1705 cs=0 l=1 rev1=1 crypto rx=0x7f84c000c750 tx=0x7f84c000cc10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84cea00640 1 -- 172.21.15.86:0/3308726116 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84c0017070 con 0x7f84d815f9d0 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84cea00640 1 -- 172.21.15.86:0/3308726116 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f84c0005af0 con 0x7f84d815f9d0 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84cea00640 1 -- 172.21.15.86:0/3308726116 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84c0007070 con 0x7f84d815f9d0 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84d815c3e0 con 0x7f84d815f9d0 2024-03-06T07:37:04.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.886+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84d815f0e0 con 0x7f84d815f9d0 2024-03-06T07:37:04.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.887+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f84d80b0dc0 con 0x7f84d815f9d0 2024-03-06T07:37:04.890 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.888+0000 7f84cea00640 1 -- 172.21.15.86:0/3308726116 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f84c001a070 con 0x7f84d815f9d0 2024-03-06T07:37:04.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.889+0000 7f84cea00640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84b406b4d0 0x7f84b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:04.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.889+0000 7f84cea00640 1 -- 172.21.15.86:0/3308726116 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f84c0089830 con 0x7f84d815f9d0 2024-03-06T07:37:04.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.890+0000 7f84d6a00640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84b406b4d0 0x7f84b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:04.891 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.890+0000 7f84d6a00640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84b406b4d0 0x7f84b406d990 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f84c8009a30 tx=0x7f84c8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:04.895 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:04.893+0000 7f84cea00640 1 -- 172.21.15.86:0/3308726116 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f84d80b0dc0 con 0x7f84d815f9d0 2024-03-06T07:37:05.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.188+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f84d804f330 con 0x7f84d815f9d0 2024-03-06T07:37:05.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.189+0000 7f84cea00640 1 -- 172.21.15.86:0/3308726116 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f84d806ee70 con 0x7f84d815f9d0 2024-03-06T07:37:05.191 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:05.191 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:05.191 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84b406b4d0 msgr2=0x7f84b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84b406b4d0 0x7f84b406d990 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f84c8009a30 tx=0x7f84c8008040 comp rx=0 tx=0).stop 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84d815f9d0 msgr2=0x7f84d813d390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84d815f9d0 0x7f84d813d390 secure :-1 s=READY pgs=1705 cs=0 l=1 rev1=1 crypto rx=0x7f84c000c750 tx=0x7f84c000cc10 comp rx=0 tx=0).stop 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 shutdown_connections 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f84b406b4d0 0x7f84b406d990 unknown :-1 s=CLOSED pgs=725 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f84d815fe30 0x7f84d81f7630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f84d815f9d0 0x7f84d813d390 unknown :-1 s=CLOSED pgs=1705 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 --2- 172.21.15.86:0/3308726116 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f84d8144c00 0x7f84d81f81d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 >> 172.21.15.86:0/3308726116 conn(0x7f84d80b1730 msgr2=0x7f84d80713d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 shutdown_connections 2024-03-06T07:37:05.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:05.192+0000 7f84dd800640 1 -- 172.21.15.86:0/3308726116 wait complete. 2024-03-06T07:37:05.422 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:05.422 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:05.638 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:06.424 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.640+0000 7f2ec5600640 1 Processor -- start 2024-03-06T07:37:06.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.640+0000 7f2ec5600640 1 -- start start 2024-03-06T07:37:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.641+0000 7f2ec5600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec00b4680 0x7f2ec006e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.641+0000 7f2ec5600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2ec015b090 con 0x7f2ec015acb0 2024-03-06T07:37:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.641+0000 7f2ec5600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2ec00b2a30 con 0x7f2ec0072340 2024-03-06T07:37:06.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.641+0000 7f2ec5600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ec00b4360 con 0x7f2ec00b4680 2024-03-06T07:37:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.642+0000 7f2ebea00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2ec015acb0 0x7f2ec00b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:58830/0 (socket says 172.21.15.86:58830) 2024-03-06T07:37:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.642+0000 7f2ebea00640 1 -- 172.21.15.86:0/282916266 learned_addr learned my addr 172.21.15.86:0/282916266 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.642+0000 7f2eb6000640 1 --2- 172.21.15.86:0/282916266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec00b4680 0x7f2ec006e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:06.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.643+0000 7f2ebe000640 1 -- 172.21.15.86:0/282916266 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1619785019 0 0) 0x7f2ec015b090 con 0x7f2ec015acb0 2024-03-06T07:37:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.643+0000 7f2ebe000640 1 -- 172.21.15.86:0/282916266 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2ea4003520 con 0x7f2ec015acb0 2024-03-06T07:37:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.643+0000 7f2eb6000640 1 -- 172.21.15.86:0/282916266 >> v1:172.21.15.194:6789/0 conn(0x7f2ec0072340 legacy=0x7f2ec00ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.643+0000 7f2eb6000640 1 -- 172.21.15.86:0/282916266 >> v1:172.21.15.86:6789/0 conn(0x7f2ec015acb0 legacy=0x7f2ec00b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:06.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.643+0000 7f2eb6000640 1 -- 172.21.15.86:0/282916266 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ec00b2cf0 con 0x7f2ec00b4680 2024-03-06T07:37:06.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.644+0000 7f2eb6000640 1 --2- 172.21.15.86:0/282916266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec00b4680 0x7f2ec006e4f0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f2ea80097d0 tx=0x7f2ea802b940 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e94a3259e73a5d73 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:06.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.645+0000 7f2ebe000640 1 -- 172.21.15.86:0/282916266 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ea8034070 con 0x7f2ec00b4680 2024-03-06T07:37:06.646 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.645+0000 7f2ebe000640 1 -- 172.21.15.86:0/282916266 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2ea8030800 con 0x7f2ec00b4680 2024-03-06T07:37:06.647 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.645+0000 7f2ebe000640 1 -- 172.21.15.86:0/282916266 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ea8030bd0 con 0x7f2ec00b4680 2024-03-06T07:37:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.646+0000 7f2ec5600640 1 -- 172.21.15.86:0/282916266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec00b4680 msgr2=0x7f2ec006e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.646+0000 7f2ec5600640 1 --2- 172.21.15.86:0/282916266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec00b4680 0x7f2ec006e4f0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f2ea80097d0 tx=0x7f2ea802b940 comp rx=0 tx=0).stop 2024-03-06T07:37:06.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.646+0000 7f2ec5600640 1 -- 172.21.15.86:0/282916266 shutdown_connections 2024-03-06T07:37:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.646+0000 7f2ec5600640 1 --2- 172.21.15.86:0/282916266 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec00b4680 0x7f2ec006e4f0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.646+0000 7f2ec5600640 1 -- 172.21.15.86:0/282916266 >> 172.21.15.86:0/282916266 conn(0x7f2ec00b1730 msgr2=0x7f2ec00b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.646+0000 7f2ec5600640 1 -- 172.21.15.86:0/282916266 shutdown_connections 2024-03-06T07:37:06.649 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.646+0000 7f2ec5600640 1 -- 172.21.15.86:0/282916266 wait complete. 2024-03-06T07:37:06.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.649+0000 7f2ec5600640 1 Processor -- start 2024-03-06T07:37:06.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.649+0000 7f2ec5600640 1 -- start start 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ec5600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec0072340 0x7f2ec01e4620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ec5600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2ec00b4680 0x7f2ec00ce230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ec5600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2ec015acb0 0x7f2ec0147350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ec5600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ec00b4010 con 0x7f2ec00b4680 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ec5600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ec00b32b0 con 0x7f2ec015acb0 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ec5600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ec006ea30 con 0x7f2ec0072340 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2eb6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2ec00b4680 0x7f2ec00ce230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2eb6000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2ec00b4680 0x7f2ec00ce230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53948/0 (socket says 172.21.15.86:53948) 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2eb6000640 1 -- 172.21.15.86:0/3848183830 learned_addr learned my addr 172.21.15.86:0/3848183830 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ebf400640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2ec015acb0 0x7f2ec0147350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.650+0000 7f2ebea00640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec0072340 0x7f2ec01e4620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb6000640 1 -- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec0072340 msgr2=0x7f2ec01e4620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:06.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb6000640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec0072340 0x7f2ec01e4620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb6000640 1 -- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2ec015acb0 msgr2=0x7f2ec0147350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb6000640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2ec015acb0 0x7f2ec0147350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb6000640 1 -- 172.21.15.86:0/3848183830 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ec0147f20 con 0x7f2ec00b4680 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb6000640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2ec00b4680 0x7f2ec00ce230 secure :-1 s=READY pgs=1707 cs=0 l=1 rev1=1 crypto rx=0x7f2ea802bf50 tx=0x7f2ea80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb7400640 1 -- 172.21.15.86:0/3848183830 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ea8034040 con 0x7f2ec00b4680 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb7400640 1 -- 172.21.15.86:0/3848183830 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2ea8030600 con 0x7f2ec00b4680 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2eb7400640 1 -- 172.21.15.86:0/3848183830 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ea8006e10 con 0x7f2ec00b4680 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ec0148200 con 0x7f2ec00b4680 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.651+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ec01487c0 con 0x7f2ec00b4680 2024-03-06T07:37:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.652+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ec004f330 con 0x7f2ec00b4680 2024-03-06T07:37:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.653+0000 7f2eb7400640 1 -- 172.21.15.86:0/3848183830 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2ea8037070 con 0x7f2ec00b4680 2024-03-06T07:37:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.654+0000 7f2eb7400640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e9c06b4d0 0x7f2e9c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.654+0000 7f2eb7400640 1 -- 172.21.15.86:0/3848183830 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f2ea80ac300 con 0x7f2ec00b4680 2024-03-06T07:37:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.655+0000 7f2ebea00640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e9c06b4d0 0x7f2e9c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:06.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.655+0000 7f2ebea00640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e9c06b4d0 0x7f2e9c06d990 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f2eb0001560 tx=0x7f2eb0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:06.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.663+0000 7f2eb7400640 1 -- 172.21.15.86:0/3848183830 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2ea807b130 con 0x7f2ec00b4680 2024-03-06T07:37:06.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.898+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2ec006ce60 con 0x7f2ec00b4680 2024-03-06T07:37:06.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.899+0000 7f2eb7400640 1 -- 172.21.15.86:0/3848183830 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f2ea807aa30 con 0x7f2ec00b4680 2024-03-06T07:37:06.901 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:06.901 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:06.902 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:06.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e9c06b4d0 msgr2=0x7f2e9c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:06.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e9c06b4d0 0x7f2e9c06d990 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f2eb0001560 tx=0x7f2eb0009040 comp rx=0 tx=0).stop 2024-03-06T07:37:06.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2ec00b4680 msgr2=0x7f2ec00ce230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:06.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2ec00b4680 0x7f2ec00ce230 secure :-1 s=READY pgs=1707 cs=0 l=1 rev1=1 crypto rx=0x7f2ea802bf50 tx=0x7f2ea80046c0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 shutdown_connections 2024-03-06T07:37:06.905 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e9c06b4d0 0x7f2e9c06d990 unknown :-1 s=CLOSED pgs=726 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2ec015acb0 0x7f2ec0147350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2ec00b4680 0x7f2ec00ce230 unknown :-1 s=CLOSED pgs=1707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 --2- 172.21.15.86:0/3848183830 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2ec0072340 0x7f2ec01e4620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:06.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 >> 172.21.15.86:0/3848183830 conn(0x7f2ec00b1730 msgr2=0x7f2ec0071b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:06.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 shutdown_connections 2024-03-06T07:37:06.906 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:06.902+0000 7f2ec5600640 1 -- 172.21.15.86:0/3848183830 wait complete. 2024-03-06T07:37:07.125 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:07.125 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:07.639 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:08.126 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:08.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.340+0000 7f7174800640 1 Processor -- start 2024-03-06T07:37:08.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.340+0000 7f7174800640 1 -- start start 2024-03-06T07:37:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.341+0000 7f7174800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.341+0000 7f7174800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f716c0b4680 0x7f716c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.341+0000 7f7174800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f716c072340 0x7f716c06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.341+0000 7f7174800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f716c15b090 con 0x7f716c0b4680 2024-03-06T07:37:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.341+0000 7f7174800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f716c0b2a30 con 0x7f716c072340 2024-03-06T07:37:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.341+0000 7f7174800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f716c0b4360 con 0x7f716c15acb0 2024-03-06T07:37:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f716c0b4680 0x7f716c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:08.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f716c0b4680 0x7f716c06e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53962/0 (socket says 172.21.15.86:53962) 2024-03-06T07:37:08.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 -- 172.21.15.86:0/238649301 learned_addr learned my addr 172.21.15.86:0/238649301 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:08.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171e00640 1 --2- 172.21.15.86:0/238649301 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:08.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7172800640 1 --2- 172.21.15.86:0/238649301 >> v2:172.21.15.194:3300/0 conn(0x7f716c072340 0x7f716c06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:08.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 -- 172.21.15.86:0/238649301 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 msgr2=0x7f716c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:08.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 --2- 172.21.15.86:0/238649301 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c0b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 -- 172.21.15.86:0/238649301 >> v2:172.21.15.194:3300/0 conn(0x7f716c072340 msgr2=0x7f716c06ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:08.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 --2- 172.21.15.86:0/238649301 >> v2:172.21.15.194:3300/0 conn(0x7f716c072340 0x7f716c06ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.342+0000 7f7171400640 1 -- 172.21.15.86:0/238649301 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f716c0b2cf0 con 0x7f716c0b4680 2024-03-06T07:37:08.345 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.344+0000 7f7171400640 1 --2- 172.21.15.86:0/238649301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c06e4f0 secure :-1 s=READY pgs=1708 cs=0 l=1 rev1=1 crypto rx=0x7f716c070aa0 tx=0x7f715c02b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4c720b5874c6e618 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:08.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.344+0000 7f7170a00640 1 -- 172.21.15.86:0/238649301 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f715c035020 con 0x7f716c0b4680 2024-03-06T07:37:08.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.344+0000 7f7170a00640 1 -- 172.21.15.86:0/238649301 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f715c02d040 con 0x7f716c0b4680 2024-03-06T07:37:08.346 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.344+0000 7f7170a00640 1 -- 172.21.15.86:0/238649301 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f715c031a80 con 0x7f716c0b4680 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 -- 172.21.15.86:0/238649301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 msgr2=0x7f716c06e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 --2- 172.21.15.86:0/238649301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c06e4f0 secure :-1 s=READY pgs=1708 cs=0 l=1 rev1=1 crypto rx=0x7f716c070aa0 tx=0x7f715c02b900 comp rx=0 tx=0).stop 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 -- 172.21.15.86:0/238649301 shutdown_connections 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 --2- 172.21.15.86:0/238649301 >> v2:172.21.15.194:3300/0 conn(0x7f716c072340 0x7f716c06ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 --2- 172.21.15.86:0/238649301 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c06e4f0 unknown :-1 s=CLOSED pgs=1708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 --2- 172.21.15.86:0/238649301 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c0b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 -- 172.21.15.86:0/238649301 >> 172.21.15.86:0/238649301 conn(0x7f716c0b1730 msgr2=0x7f716c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 -- 172.21.15.86:0/238649301 shutdown_connections 2024-03-06T07:37:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.345+0000 7f7174800640 1 -- 172.21.15.86:0/238649301 wait complete. 2024-03-06T07:37:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.347+0000 7f7174800640 1 Processor -- start 2024-03-06T07:37:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.347+0000 7f7174800640 1 -- start start 2024-03-06T07:37:08.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7174800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f716c072340 0x7f716c14a580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:08.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7174800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c0ae270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:08.350 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7174800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c15c040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7174800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f716c0b3d10 con 0x7f716c0b4680 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7174800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f716c0b32b0 con 0x7f716c072340 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7174800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f716c0b4010 con 0x7f716c15acb0 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7171400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c0ae270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7171400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c0ae270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53976/0 (socket says 172.21.15.86:53976) 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.348+0000 7f7171400640 1 -- 172.21.15.86:0/2567741600 learned_addr learned my addr 172.21.15.86:0/2567741600 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171e00640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f716c072340 0x7f716c14a580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171400640 1 -- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 msgr2=0x7f716c15c040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7172800640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c15c040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171400640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c15c040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171400640 1 -- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f716c072340 msgr2=0x7f716c14a580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171400640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f716c072340 0x7f716c14a580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171400640 1 -- 172.21.15.86:0/2567741600 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f716c0af1a0 con 0x7f716c0b4680 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171e00640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f716c072340 0x7f716c14a580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:08.351 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7171400640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c0ae270 secure :-1 s=READY pgs=1709 cs=0 l=1 rev1=1 crypto rx=0x7f715c02b940 tx=0x7f715c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:08.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7166000640 1 -- 172.21.15.86:0/2567741600 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f715c040070 con 0x7f716c0b4680 2024-03-06T07:37:08.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7166000640 1 -- 172.21.15.86:0/2567741600 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f715c039bf0 con 0x7f716c0b4680 2024-03-06T07:37:08.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7166000640 1 -- 172.21.15.86:0/2567741600 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f715c035040 con 0x7f716c0b4680 2024-03-06T07:37:08.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.349+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f716c0aebe0 con 0x7f716c0b4680 2024-03-06T07:37:08.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.350+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f716c14b070 con 0x7f716c0b4680 2024-03-06T07:37:08.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.351+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f716c0b1b10 con 0x7f716c0b4680 2024-03-06T07:37:08.353 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.352+0000 7f7166000640 1 -- 172.21.15.86:0/2567741600 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f715c038020 con 0x7f716c0b4680 2024-03-06T07:37:08.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.353+0000 7f7166000640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f715006b4d0 0x7f715006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:08.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.353+0000 7f7166000640 1 -- 172.21.15.86:0/2567741600 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f715c0abea0 con 0x7f716c0b4680 2024-03-06T07:37:08.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.353+0000 7f7171e00640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f715006b4d0 0x7f715006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:08.355 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.354+0000 7f7171e00640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f715006b4d0 0x7f715006d990 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f7160006fd0 tx=0x7f7160008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:08.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.360+0000 7f7166000640 1 -- 172.21.15.86:0/2567741600 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f716c0b1b10 con 0x7f716c0b4680 2024-03-06T07:37:08.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.594+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f716c15a810 con 0x7f716c0b4680 2024-03-06T07:37:08.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.595+0000 7f7166000640 1 -- 172.21.15.86:0/2567741600 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f716c15a810 con 0x7f716c0b4680 2024-03-06T07:37:08.596 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:08.597 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:08.597 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:08.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.597+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f715006b4d0 msgr2=0x7f715006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:08.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.597+0000 7f7174800640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f715006b4d0 0x7f715006d990 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f7160006fd0 tx=0x7f7160008040 comp rx=0 tx=0).stop 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.597+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 msgr2=0x7f716c0ae270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.597+0000 7f7174800640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c0ae270 secure :-1 s=READY pgs=1709 cs=0 l=1 rev1=1 crypto rx=0x7f715c02b940 tx=0x7f715c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.597+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 shutdown_connections 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.597+0000 7f7174800640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f715006b4d0 0x7f715006d990 unknown :-1 s=CLOSED pgs=727 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.598+0000 7f7174800640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f716c15acb0 0x7f716c15c040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.598+0000 7f7174800640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f716c0b4680 0x7f716c0ae270 unknown :-1 s=CLOSED pgs=1709 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.598+0000 7f7174800640 1 --2- 172.21.15.86:0/2567741600 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f716c072340 0x7f716c14a580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.598+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 >> 172.21.15.86:0/2567741600 conn(0x7f716c0b1730 msgr2=0x7f716c0b06b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.598+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 shutdown_connections 2024-03-06T07:37:08.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:08.598+0000 7f7174800640 1 -- 172.21.15.86:0/2567741600 wait complete. 2024-03-06T07:37:08.820 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:08.820 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:09.642 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:09.821 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:10.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.028+0000 7fc19ae00640 1 Processor -- start 2024-03-06T07:37:10.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.029+0000 7fc19ae00640 1 -- start start 2024-03-06T07:37:10.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.030+0000 7fc19ae00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc19415e070 0x7fc194066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.030+0000 7fc19ae00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc1940b5010 con 0x7fc1940cd860 2024-03-06T07:37:10.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.030+0000 7fc19ae00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc1940b52f0 con 0x7fc19406b6d0 2024-03-06T07:37:10.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.030+0000 7fc19ae00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc1940b55d0 con 0x7fc19415e070 2024-03-06T07:37:10.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.030+0000 7fc18b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc1940cd860 0x7fc19406afc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:58834/0 (socket says 172.21.15.86:58834) 2024-03-06T07:37:10.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.030+0000 7fc18b400640 1 -- 172.21.15.86:0/347452079 learned_addr learned my addr 172.21.15.86:0/347452079 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193e00640 1 --2- 172.21.15.86:0/347452079 >> v2:172.21.15.194:3300/0 conn(0x7fc19415e070 0x7fc194066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4267323006 0 0) 0x7fc1940b5010 con 0x7fc1940cd860 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc178003520 con 0x7fc1940cd860 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2370136185 0 0) 0x7fc178003520 con 0x7fc1940cd860 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 >> v2:172.21.15.194:3300/0 conn(0x7fc19415e070 msgr2=0x7fc194066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 --2- 172.21.15.86:0/347452079 >> v2:172.21.15.194:3300/0 conn(0x7fc19415e070 0x7fc194066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 >> v1:172.21.15.194:6789/0 conn(0x7fc19406b6d0 legacy=0x7fc19406db00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc1940b58b0 con 0x7fc1940cd860 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc17c001a10 con 0x7fc1940cd860 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 >> v1:172.21.15.86:6789/0 conn(0x7fc1940cd860 legacy=0x7fc19406afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc178003a00 0x7fc178003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc178004800 0x7fc178004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1780051c0 0x7fc17800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1940b55d0 con 0x7fc1780051c0 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc17800c0c0 con 0x7fc178003a00 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.031+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc17800c2c0 con 0x7fc178004800 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc198e00640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1780051c0 0x7fc17800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc193e00640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc178003a00 0x7fc178003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc18b400640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc178004800 0x7fc178004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc198e00640 1 -- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc178004800 msgr2=0x7fc178004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc198e00640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc178004800 0x7fc178004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc198e00640 1 -- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc178003a00 msgr2=0x7fc178003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc198e00640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc178003a00 0x7fc178003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.032+0000 7fc198e00640 1 -- 172.21.15.86:0/347452079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc178003520 con 0x7fc1780051c0 2024-03-06T07:37:10.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.034+0000 7fc198e00640 1 -- 172.21.15.86:0/347452079 reap_dead start 2024-03-06T07:37:10.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.034+0000 7fc198e00640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1780051c0 0x7fc17800b9c0 secure :-1 s=READY pgs=1711 cs=0 l=1 rev1=1 crypto rx=0x7fc18c001750 tx=0x7fc178005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=91d45e82d3c2bbff server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.034+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc18c001370 con 0x7fc1780051c0 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.034+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc18c005310 con 0x7fc1780051c0 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.034+0000 7fc193400640 1 -- 172.21.15.86:0/347452079 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc18c003830 con 0x7fc1780051c0 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.035+0000 7fc19ae00640 1 -- 172.21.15.86:0/347452079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1780051c0 msgr2=0x7fc17800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.035+0000 7fc19ae00640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1780051c0 0x7fc17800b9c0 secure :-1 s=READY pgs=1711 cs=0 l=1 rev1=1 crypto rx=0x7fc18c001750 tx=0x7fc178005830 comp rx=0 tx=0).stop 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.035+0000 7fc19ae00640 1 -- 172.21.15.86:0/347452079 shutdown_connections 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.035+0000 7fc19ae00640 1 --2- 172.21.15.86:0/347452079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc1780051c0 0x7fc17800b9c0 unknown :-1 s=CLOSED pgs=1711 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.035+0000 7fc19ae00640 1 -- 172.21.15.86:0/347452079 >> 172.21.15.86:0/347452079 conn(0x7fc1940b1e10 msgr2=0x7fc1940b2220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.035+0000 7fc19ae00640 1 -- 172.21.15.86:0/347452079 shutdown_connections 2024-03-06T07:37:10.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.035+0000 7fc19ae00640 1 -- 172.21.15.86:0/347452079 wait complete. 2024-03-06T07:37:10.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.038+0000 7fc19ae00640 1 Processor -- start 2024-03-06T07:37:10.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.038+0000 7fc19ae00640 1 -- start start 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.039+0000 7fc19ae00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1780051c0 0x7fc1940d0b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.039+0000 7fc19ae00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc194156920 0x7fc194151a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.039+0000 7fc19ae00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc194099b60 0x7fc194157140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.039+0000 7fc19ae00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1940b60b0 con 0x7fc194099b60 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.039+0000 7fc19ae00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc19415e450 con 0x7fc194156920 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.039+0000 7fc19ae00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc194151f40 con 0x7fc1780051c0 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc194099b60 0x7fc194157140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc194099b60 0x7fc194157140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53998/0 (socket says 172.21.15.86:53998) 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 -- 172.21.15.86:0/1789526419 learned_addr learned my addr 172.21.15.86:0/1789526419 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc193e00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1780051c0 0x7fc1940d0b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.042 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc18b400640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc194156920 0x7fc194151a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 -- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1780051c0 msgr2=0x7fc1940d0b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:10.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1780051c0 0x7fc1940d0b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 -- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc194156920 msgr2=0x7fc194151a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:10.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc194156920 0x7fc194151a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 -- 172.21.15.86:0/1789526419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc194155b60 con 0x7fc194099b60 2024-03-06T07:37:10.043 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.040+0000 7fc198e00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc194099b60 0x7fc194157140 secure :-1 s=READY pgs=1712 cs=0 l=1 rev1=1 crypto rx=0x7fc18c001840 tx=0x7fc18c001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:10.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.041+0000 7fc190c00640 1 -- 172.21.15.86:0/1789526419 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc18c001370 con 0x7fc194099b60 2024-03-06T07:37:10.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.041+0000 7fc190c00640 1 -- 172.21.15.86:0/1789526419 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc18c007240 con 0x7fc194099b60 2024-03-06T07:37:10.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.041+0000 7fc190c00640 1 -- 172.21.15.86:0/1789526419 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc18c007a30 con 0x7fc194099b60 2024-03-06T07:37:10.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.041+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc194155e40 con 0x7fc194099b60 2024-03-06T07:37:10.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.041+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc194156400 con 0x7fc194099b60 2024-03-06T07:37:10.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.042+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc19404f330 con 0x7fc194099b60 2024-03-06T07:37:10.044 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.043+0000 7fc190c00640 1 -- 172.21.15.86:0/1789526419 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc18c0067d0 con 0x7fc194099b60 2024-03-06T07:37:10.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.044+0000 7fc190c00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc17006b4d0 0x7fc17006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:10.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.044+0000 7fc190c00640 1 -- 172.21.15.86:0/1789526419 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc18c03a020 con 0x7fc194099b60 2024-03-06T07:37:10.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.044+0000 7fc193e00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc17006b4d0 0x7fc17006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:10.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.045+0000 7fc193e00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc17006b4d0 0x7fc17006d990 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7fc184009990 tx=0x7fc184008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:10.050 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.049+0000 7fc190c00640 1 -- 172.21.15.86:0/1789526419 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc18c079a50 con 0x7fc194099b60 2024-03-06T07:37:10.289 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.287+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc1941565d0 con 0x7fc194099b60 2024-03-06T07:37:10.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.288+0000 7fc190c00640 1 -- 172.21.15.86:0/1789526419 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7fc18c079350 con 0x7fc194099b60 2024-03-06T07:37:10.290 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:10.290 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:10.291 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc17006b4d0 msgr2=0x7fc17006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc17006b4d0 0x7fc17006d990 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7fc184009990 tx=0x7fc184008040 comp rx=0 tx=0).stop 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc194099b60 msgr2=0x7fc194157140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc194099b60 0x7fc194157140 secure :-1 s=READY pgs=1712 cs=0 l=1 rev1=1 crypto rx=0x7fc18c001840 tx=0x7fc18c001120 comp rx=0 tx=0).stop 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 shutdown_connections 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc17006b4d0 0x7fc17006d990 unknown :-1 s=CLOSED pgs=728 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc194099b60 0x7fc194157140 unknown :-1 s=CLOSED pgs=1712 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.293 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc194156920 0x7fc194151a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 --2- 172.21.15.86:0/1789526419 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc1780051c0 0x7fc1940d0b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:10.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.291+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 >> 172.21.15.86:0/1789526419 conn(0x7fc1940b1e10 msgr2=0x7fc1940add90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:10.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.292+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 shutdown_connections 2024-03-06T07:37:10.294 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:10.292+0000 7fc19ae00640 1 -- 172.21.15.86:0/1789526419 wait complete. 2024-03-06T07:37:10.513 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:10.513 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:11.515 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:11.643 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:11.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.718+0000 7f45d0600640 1 Processor -- start 2024-03-06T07:37:11.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.719+0000 7f45d0600640 1 -- start start 2024-03-06T07:37:11.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.720+0000 7f45d0600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f45c815f9d0 0x7f45c815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:11.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.720+0000 7f45d0600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f45c8070b10 0x7f45c8070f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:11.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.720+0000 7f45d0600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f45c806cef0 con 0x7f45c8144c00 2024-03-06T07:37:11.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.720+0000 7f45d0600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f45c806d1d0 con 0x7f45c815f9d0 2024-03-06T07:37:11.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.720+0000 7f45d0600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f45c806d4b0 con 0x7f45c8070b10 2024-03-06T07:37:11.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.720+0000 7f45cd200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f45c815f9d0 0x7f45c815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:11.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45cdc00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f45c8144c00 0x7f45c806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:58836/0 (socket says 172.21.15.86:58836) 2024-03-06T07:37:11.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45cdc00640 1 -- 172.21.15.86:0/1467964963 learned_addr learned my addr 172.21.15.86:0/1467964963 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:11.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45ce600640 1 --2- 172.21.15.86:0/1467964963 >> v2:172.21.15.194:3300/0 conn(0x7f45c8070b10 0x7f45c8070f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:11.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45cd200640 1 -- 172.21.15.86:0/1467964963 >> v2:172.21.15.194:3300/0 conn(0x7f45c8070b10 msgr2=0x7f45c8070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:11.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45cd200640 1 --2- 172.21.15.86:0/1467964963 >> v2:172.21.15.194:3300/0 conn(0x7f45c8070b10 0x7f45c8070f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45cd200640 1 -- 172.21.15.86:0/1467964963 >> v1:172.21.15.86:6789/0 conn(0x7f45c8144c00 legacy=0x7f45c806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:11.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45c7e00640 1 -- 172.21.15.86:0/1467964963 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 411670132 0 0) 0x7f45c806cef0 con 0x7f45c8144c00 2024-03-06T07:37:11.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45cd200640 1 -- 172.21.15.86:0/1467964963 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f45c806d790 con 0x7f45c815f9d0 2024-03-06T07:37:11.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.721+0000 7f45ce600640 1 --2- 172.21.15.86:0/1467964963 >> v2:172.21.15.194:3300/0 conn(0x7f45c8070b10 0x7f45c8070f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.722+0000 7f45cd200640 1 --2- 172.21.15.86:0/1467964963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c815f9d0 0x7f45c815fe60 secure :-1 s=READY pgs=1714 cs=0 l=1 rev1=1 crypto rx=0x7f45c806e500 tx=0x7f45b402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e44063d2edcbdfb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.722+0000 7f45c7e00640 1 -- 172.21.15.86:0/1467964963 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45b4035020 con 0x7f45c815f9d0 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45c7e00640 1 -- 172.21.15.86:0/1467964963 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f45b4031830 con 0x7f45c815f9d0 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45c7e00640 1 -- 172.21.15.86:0/1467964963 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45b4031c00 con 0x7f45c815f9d0 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 -- 172.21.15.86:0/1467964963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c815f9d0 msgr2=0x7f45c815fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 --2- 172.21.15.86:0/1467964963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c815f9d0 0x7f45c815fe60 secure :-1 s=READY pgs=1714 cs=0 l=1 rev1=1 crypto rx=0x7f45c806e500 tx=0x7f45b402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 -- 172.21.15.86:0/1467964963 shutdown_connections 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 --2- 172.21.15.86:0/1467964963 >> v2:172.21.15.194:3300/0 conn(0x7f45c8070b10 0x7f45c8070f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 --2- 172.21.15.86:0/1467964963 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c815f9d0 0x7f45c815fe60 unknown :-1 s=CLOSED pgs=1714 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 -- 172.21.15.86:0/1467964963 >> 172.21.15.86:0/1467964963 conn(0x7f45c80b1730 msgr2=0x7f45c80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:11.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 -- 172.21.15.86:0/1467964963 shutdown_connections 2024-03-06T07:37:11.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.723+0000 7f45d0600640 1 -- 172.21.15.86:0/1467964963 wait complete. 2024-03-06T07:37:11.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.725+0000 7f45d0600640 1 Processor -- start 2024-03-06T07:37:11.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.725+0000 7f45d0600640 1 -- start start 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45d0600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45c8070b10 0x7f45c8139450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45d0600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c8144c00 0x7f45c813c260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45d0600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f45c815f9d0 0x7f45c813a580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45d0600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45c806ccd0 con 0x7f45c8144c00 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45d0600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45c806cb50 con 0x7f45c8070b10 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45d0600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f45c806df90 con 0x7f45c815f9d0 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45cd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c8144c00 0x7f45c813c260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45cd200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c8144c00 0x7f45c813c260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54010/0 (socket says 172.21.15.86:54010) 2024-03-06T07:37:11.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.726+0000 7f45cd200640 1 -- 172.21.15.86:0/1179822747 learned_addr learned my addr 172.21.15.86:0/1179822747 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cdc00640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45c8070b10 0x7f45c8139450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cd200640 1 -- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f45c815f9d0 msgr2=0x7f45c813a580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45ce600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f45c815f9d0 0x7f45c813a580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cd200640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f45c815f9d0 0x7f45c813a580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cd200640 1 -- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45c8070b10 msgr2=0x7f45c8139450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cd200640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45c8070b10 0x7f45c8139450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cd200640 1 -- 172.21.15.86:0/1179822747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f45c81e6100 con 0x7f45c8144c00 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cdc00640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45c8070b10 0x7f45c8139450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45cd200640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c8144c00 0x7f45c813c260 secure :-1 s=READY pgs=1715 cs=0 l=1 rev1=1 crypto rx=0x7f45b4000c00 tx=0x7f45b40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45c5600640 1 -- 172.21.15.86:0/1179822747 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45b4041070 con 0x7f45c8144c00 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45c5600640 1 -- 172.21.15.86:0/1179822747 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f45b4004230 con 0x7f45c8144c00 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45c5600640 1 -- 172.21.15.86:0/1179822747 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45b4035040 con 0x7f45c8144c00 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.727+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f45c81e63e0 con 0x7f45c8144c00 2024-03-06T07:37:11.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.728+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f45c8139c10 con 0x7f45c8144c00 2024-03-06T07:37:11.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.729+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f45c804f330 con 0x7f45c8144c00 2024-03-06T07:37:11.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.731+0000 7f45c5600640 1 -- 172.21.15.86:0/1179822747 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f45b4038020 con 0x7f45c8144c00 2024-03-06T07:37:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.732+0000 7f45c5600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f45a806b4d0 0x7f45a806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.732+0000 7f45c5600640 1 -- 172.21.15.86:0/1179822747 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f45b40abde0 con 0x7f45c8144c00 2024-03-06T07:37:11.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.732+0000 7f45cdc00640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f45a806b4d0 0x7f45a806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:11.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.732+0000 7f45cdc00640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f45a806b4d0 0x7f45a806d990 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f45bc000f60 tx=0x7f45bc009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:11.742 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.741+0000 7f45c5600640 1 -- 172.21.15.86:0/1179822747 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f45b407ab80 con 0x7f45c8144c00 2024-03-06T07:37:11.978 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.976+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f45c8072890 con 0x7f45c8144c00 2024-03-06T07:37:11.979 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.977+0000 7f45c5600640 1 -- 172.21.15.86:0/1179822747 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f45b407a480 con 0x7f45c8144c00 2024-03-06T07:37:11.979 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:11.979 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:11.979 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:11.981 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f45a806b4d0 msgr2=0x7f45a806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f45a806b4d0 0x7f45a806d990 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f45bc000f60 tx=0x7f45bc009040 comp rx=0 tx=0).stop 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c8144c00 msgr2=0x7f45c813c260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c8144c00 0x7f45c813c260 secure :-1 s=READY pgs=1715 cs=0 l=1 rev1=1 crypto rx=0x7f45b4000c00 tx=0x7f45b40046c0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 shutdown_connections 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f45a806b4d0 0x7f45a806d990 unknown :-1 s=CLOSED pgs=729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f45c815f9d0 0x7f45c813a580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f45c8144c00 0x7f45c813c260 unknown :-1 s=CLOSED pgs=1715 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 --2- 172.21.15.86:0/1179822747 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f45c8070b10 0x7f45c8139450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 >> 172.21.15.86:0/1179822747 conn(0x7f45c80b1730 msgr2=0x7f45c80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 shutdown_connections 2024-03-06T07:37:11.982 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:11.980+0000 7f45d0600640 1 -- 172.21.15.86:0/1179822747 wait complete. 2024-03-06T07:37:12.200 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:12.201 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:13.202 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:13.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.410+0000 7f5798200640 1 Processor -- start 2024-03-06T07:37:13.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.411+0000 7f5798200640 1 -- start start 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5798200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f579015f4f0 0x7f57900706d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5798200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790070c10 0x7f57900cc700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5798200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f579015f8d0 0x7f57900ccc40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5798200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5790160a20 con 0x7f579015f4f0 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5798200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f579006ef50 con 0x7f579015f8d0 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5798200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f579006f230 con 0x7f5790070c10 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5795800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f579015f4f0 0x7f57900706d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5795800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f579015f4f0 0x7f57900706d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54022/0 (socket says 172.21.15.86:54022) 2024-03-06T07:37:13.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.412+0000 7f5795800640 1 -- 172.21.15.86:0/2127553583 learned_addr learned my addr 172.21.15.86:0/2127553583 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.413+0000 7f5794e00640 1 --2- 172.21.15.86:0/2127553583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790070c10 0x7f57900cc700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.413+0000 7f5796200640 1 --2- 172.21.15.86:0/2127553583 >> v2:172.21.15.194:3300/0 conn(0x7f579015f8d0 0x7f57900ccc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.413+0000 7f5795800640 1 -- 172.21.15.86:0/2127553583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790070c10 msgr2=0x7f57900cc700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.413+0000 7f5795800640 1 --2- 172.21.15.86:0/2127553583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790070c10 0x7f57900cc700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.413+0000 7f5795800640 1 -- 172.21.15.86:0/2127553583 >> v2:172.21.15.194:3300/0 conn(0x7f579015f8d0 msgr2=0x7f57900ccc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.413+0000 7f5795800640 1 --2- 172.21.15.86:0/2127553583 >> v2:172.21.15.194:3300/0 conn(0x7f579015f8d0 0x7f57900ccc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.413+0000 7f5795800640 1 -- 172.21.15.86:0/2127553583 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f579015fd50 con 0x7f579015f4f0 2024-03-06T07:37:13.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.414+0000 7f5795800640 1 --2- 172.21.15.86:0/2127553583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f4f0 0x7f57900706d0 secure :-1 s=READY pgs=1716 cs=0 l=1 rev1=1 crypto rx=0x7f579006ecc0 tx=0x7f578402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5312f3d736431e9b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.414+0000 7f578fe00640 1 -- 172.21.15.86:0/2127553583 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5784035020 con 0x7f579015f4f0 2024-03-06T07:37:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.414+0000 7f578fe00640 1 -- 172.21.15.86:0/2127553583 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5784031870 con 0x7f579015f4f0 2024-03-06T07:37:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.414+0000 7f578fe00640 1 -- 172.21.15.86:0/2127553583 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5784031c40 con 0x7f579015f4f0 2024-03-06T07:37:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 -- 172.21.15.86:0/2127553583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f4f0 msgr2=0x7f57900706d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 --2- 172.21.15.86:0/2127553583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f4f0 0x7f57900706d0 secure :-1 s=READY pgs=1716 cs=0 l=1 rev1=1 crypto rx=0x7f579006ecc0 tx=0x7f578402b910 comp rx=0 tx=0).stop 2024-03-06T07:37:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 -- 172.21.15.86:0/2127553583 shutdown_connections 2024-03-06T07:37:13.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 --2- 172.21.15.86:0/2127553583 >> v2:172.21.15.194:3300/0 conn(0x7f579015f8d0 0x7f57900ccc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 --2- 172.21.15.86:0/2127553583 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790070c10 0x7f57900cc700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 --2- 172.21.15.86:0/2127553583 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f4f0 0x7f57900706d0 unknown :-1 s=CLOSED pgs=1716 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 -- 172.21.15.86:0/2127553583 >> 172.21.15.86:0/2127553583 conn(0x7f57900b3800 msgr2=0x7f57900b3c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:13.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.415+0000 7f5798200640 1 -- 172.21.15.86:0/2127553583 shutdown_connections 2024-03-06T07:37:13.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.416+0000 7f5798200640 1 -- 172.21.15.86:0/2127553583 wait complete. 2024-03-06T07:37:13.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.418+0000 7f5798200640 1 Processor -- start 2024-03-06T07:37:13.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.418+0000 7f5798200640 1 -- start start 2024-03-06T07:37:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5798200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f579015f4f0 0x7f5790156b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:13.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5798200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f8d0 0x7f57900b48f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5798200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790159dc0 0x7f579015a250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5798200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5790160450 con 0x7f579015f8d0 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5798200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57901602d0 con 0x7f579015f4f0 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5798200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f57901605d0 con 0x7f5790159dc0 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5794e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f8d0 0x7f57900b48f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5794e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f8d0 0x7f57900b48f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54034/0 (socket says 172.21.15.86:54034) 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5794e00640 1 -- 172.21.15.86:0/1427365484 learned_addr learned my addr 172.21.15.86:0/1427365484 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.419+0000 7f5795800640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f579015f4f0 0x7f5790156b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5796200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790159dc0 0x7f579015a250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5794e00640 1 -- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790159dc0 msgr2=0x7f579015a250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5794e00640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790159dc0 0x7f579015a250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5794e00640 1 -- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f579015f4f0 msgr2=0x7f5790156b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:13.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5794e00640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f579015f4f0 0x7f5790156b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5794e00640 1 -- 172.21.15.86:0/1427365484 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57901e5bb0 con 0x7f579015f8d0 2024-03-06T07:37:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5796200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790159dc0 0x7f579015a250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5795800640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f579015f4f0 0x7f5790156b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:13.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5794e00640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f8d0 0x7f57900b48f0 secure :-1 s=READY pgs=1717 cs=0 l=1 rev1=1 crypto rx=0x7f577c00d800 tx=0x7f577c00dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:13.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f578d600640 1 -- 172.21.15.86:0/1427365484 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f577c004020 con 0x7f579015f8d0 2024-03-06T07:37:13.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f578d600640 1 -- 172.21.15.86:0/1427365484 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f577c0041c0 con 0x7f579015f8d0 2024-03-06T07:37:13.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f578d600640 1 -- 172.21.15.86:0/1427365484 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f577c004390 con 0x7f579015f8d0 2024-03-06T07:37:13.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f57901e5e90 con 0x7f579015f8d0 2024-03-06T07:37:13.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.420+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f57901e6450 con 0x7f579015f8d0 2024-03-06T07:37:13.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.422+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f57900b35b0 con 0x7f579015f8d0 2024-03-06T07:37:13.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.422+0000 7f578d600640 1 -- 172.21.15.86:0/1427365484 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f577c01b020 con 0x7f579015f8d0 2024-03-06T07:37:13.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.423+0000 7f578d600640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f577006b500 0x7f577006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:13.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.423+0000 7f578d600640 1 -- 172.21.15.86:0/1427365484 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f577c08a080 con 0x7f579015f8d0 2024-03-06T07:37:13.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.424+0000 7f5795800640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f577006b500 0x7f577006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:13.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.424+0000 7f5795800640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f577006b500 0x7f577006d9c0 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f5784036000 tx=0x7f5784008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:13.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.428+0000 7f578d600640 1 -- 172.21.15.86:0/1427365484 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f57900b35b0 con 0x7f579015f8d0 2024-03-06T07:37:13.645 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:13.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.663+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f579004f330 con 0x7f579015f8d0 2024-03-06T07:37:13.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.663+0000 7f578d600640 1 -- 172.21.15.86:0/1427365484 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 102 v102) v1 ==== 95+0+3465 (secure 0 0 0) 0x7f577c058ec0 con 0x7f579015f8d0 2024-03-06T07:37:13.665 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:13.665 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":102,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":102,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:35:42.924360+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:13.666 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 102 2024-03-06T07:37:13.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f577006b500 msgr2=0x7f577006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f577006b500 0x7f577006d9c0 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f5784036000 tx=0x7f5784008030 comp rx=0 tx=0).stop 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f8d0 msgr2=0x7f57900b48f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f8d0 0x7f57900b48f0 secure :-1 s=READY pgs=1717 cs=0 l=1 rev1=1 crypto rx=0x7f577c00d800 tx=0x7f577c00dcc0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 shutdown_connections 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f577006b500 0x7f577006d9c0 unknown :-1 s=CLOSED pgs=730 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f5790159dc0 0x7f579015a250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f579015f8d0 0x7f57900b48f0 unknown :-1 s=CLOSED pgs=1717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 --2- 172.21.15.86:0/1427365484 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f579015f4f0 0x7f5790156b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.666+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 >> 172.21.15.86:0/1427365484 conn(0x7f57900b3800 msgr2=0x7f57900b1650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.667+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 shutdown_connections 2024-03-06T07:37:13.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:13.667+0000 7f5798200640 1 -- 172.21.15.86:0/1427365484 wait complete. 2024-03-06T07:37:13.887 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 102, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:35:42.924360+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:13.887 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:14.887 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.103+0000 7f5161600640 1 Processor -- start 2024-03-06T07:37:15.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.103+0000 7f5161600640 1 -- start start 2024-03-06T07:37:15.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f5161600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c15acb0 0x7f515c0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:15.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f5161600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f515c0b4680 0x7f515c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f5161600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f515c15b090 con 0x7f515c072340 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f5161600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f515c0b2a30 con 0x7f515c0b4680 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f5161600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f515c0b4360 con 0x7f515c15acb0 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f515a000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f515c0b4680 0x7f515c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f515b400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f515c072340 0x7f515c0ccfb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:58842/0 (socket says 172.21.15.86:58842) 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.105+0000 7f515b400640 1 -- 172.21.15.86:0/1563200418 learned_addr learned my addr 172.21.15.86:0/1563200418 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.106+0000 7f515aa00640 1 --2- 172.21.15.86:0/1563200418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c15acb0 0x7f515c0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:15.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.106+0000 7f515a000640 1 -- 172.21.15.86:0/1563200418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c15acb0 msgr2=0x7f515c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:15.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.106+0000 7f515a000640 1 --2- 172.21.15.86:0/1563200418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c15acb0 0x7f515c0b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.106+0000 7f515a000640 1 -- 172.21.15.86:0/1563200418 >> v1:172.21.15.86:6789/0 conn(0x7f515c072340 legacy=0x7f515c0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:15.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.106+0000 7f515a000640 1 -- 172.21.15.86:0/1563200418 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f515c0b2cf0 con 0x7f515c0b4680 2024-03-06T07:37:15.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.107+0000 7f515a000640 1 --2- 172.21.15.86:0/1563200418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c06e4f0 secure :-1 s=READY pgs=1719 cs=0 l=1 rev1=1 crypto rx=0x7f51480097a0 tx=0x7f514802dac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2f1659660c36407 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:15.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.107+0000 7f5159600640 1 -- 172.21.15.86:0/1563200418 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f514802f880 con 0x7f515c0b4680 2024-03-06T07:37:15.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.107+0000 7f5159600640 1 -- 172.21.15.86:0/1563200418 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f514802fa20 con 0x7f515c0b4680 2024-03-06T07:37:15.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.107+0000 7f5159600640 1 -- 172.21.15.86:0/1563200418 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f514802fdf0 con 0x7f515c0b4680 2024-03-06T07:37:15.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 -- 172.21.15.86:0/1563200418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 msgr2=0x7f515c06e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:15.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 --2- 172.21.15.86:0/1563200418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c06e4f0 secure :-1 s=READY pgs=1719 cs=0 l=1 rev1=1 crypto rx=0x7f51480097a0 tx=0x7f514802dac0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 -- 172.21.15.86:0/1563200418 shutdown_connections 2024-03-06T07:37:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 --2- 172.21.15.86:0/1563200418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c06e4f0 unknown :-1 s=CLOSED pgs=1719 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 --2- 172.21.15.86:0/1563200418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c15acb0 0x7f515c0b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 -- 172.21.15.86:0/1563200418 >> 172.21.15.86:0/1563200418 conn(0x7f515c0b1730 msgr2=0x7f515c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 -- 172.21.15.86:0/1563200418 shutdown_connections 2024-03-06T07:37:15.111 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.108+0000 7f5161600640 1 -- 172.21.15.86:0/1563200418 wait complete. 2024-03-06T07:37:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.110+0000 7f5161600640 1 Processor -- start 2024-03-06T07:37:15.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.110+0000 7f5161600640 1 -- start start 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f5161600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c072340 0x7f515c1478f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f5161600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c0ae710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f5161600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f515c15acb0 0x7f515c071180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f5161600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f515c0b3d10 con 0x7f515c0b4680 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f5161600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f515c0b32b0 con 0x7f515c15acb0 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f5161600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f515c0b4010 con 0x7f515c072340 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c0ae710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c0ae710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:54050/0 (socket says 172.21.15.86:54050) 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 -- 172.21.15.86:0/497109513 learned_addr learned my addr 172.21.15.86:0/497109513 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:15.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515aa00640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c072340 0x7f515c1478f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515b400640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f515c15acb0 0x7f515c071180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 -- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c072340 msgr2=0x7f515c1478f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c072340 0x7f515c1478f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 -- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f515c15acb0 msgr2=0x7f515c071180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f515c15acb0 0x7f515c071180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515a000640 1 -- 172.21.15.86:0/497109513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f515c15bfc0 con 0x7f515c0b4680 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.111+0000 7f515aa00640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c072340 0x7f515c1478f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:15.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.112+0000 7f515a000640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c0ae710 secure :-1 s=READY pgs=1720 cs=0 l=1 rev1=1 crypto rx=0x7f5148009890 tx=0x7f51480046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.112+0000 7f5152a00640 1 -- 172.21.15.86:0/497109513 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5148047070 con 0x7f515c0b4680 2024-03-06T07:37:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.112+0000 7f5152a00640 1 -- 172.21.15.86:0/497109513 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5148030b00 con 0x7f515c0b4680 2024-03-06T07:37:15.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.112+0000 7f5152a00640 1 -- 172.21.15.86:0/497109513 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5148042410 con 0x7f515c0b4680 2024-03-06T07:37:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.112+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f515c0aef60 con 0x7f515c0b4680 2024-03-06T07:37:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.113+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f515c1584f0 con 0x7f515c0b4680 2024-03-06T07:37:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.114+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f515c04f330 con 0x7f515c0b4680 2024-03-06T07:37:15.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.115+0000 7f5152a00640 1 -- 172.21.15.86:0/497109513 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5148039400 con 0x7f515c0b4680 2024-03-06T07:37:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.116+0000 7f5152a00640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f513c06b4d0 0x7f513c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.116+0000 7f5152a00640 1 -- 172.21.15.86:0/497109513 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f51480abbc0 con 0x7f515c0b4680 2024-03-06T07:37:15.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.117+0000 7f515aa00640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f513c06b4d0 0x7f513c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:15.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.118+0000 7f515aa00640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f513c06b4d0 0x7f513c06d990 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f514c006fd0 tx=0x7f514c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:15.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.121+0000 7f5152a00640 1 -- 172.21.15.86:0/497109513 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f515c15bfc0 con 0x7f515c0b4680 2024-03-06T07:37:15.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.383+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f515c06ce60 con 0x7f515c0b4680 2024-03-06T07:37:15.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.384+0000 7f5152a00640 1 -- 172.21.15.86:0/497109513 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f5148035030 con 0x7f515c0b4680 2024-03-06T07:37:15.386 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:15.386 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:15.386 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:15.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f513c06b4d0 msgr2=0x7f513c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:15.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f513c06b4d0 0x7f513c06d990 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f514c006fd0 tx=0x7f514c008040 comp rx=0 tx=0).stop 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 msgr2=0x7f515c0ae710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c0ae710 secure :-1 s=READY pgs=1720 cs=0 l=1 rev1=1 crypto rx=0x7f5148009890 tx=0x7f51480046c0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 shutdown_connections 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f513c06b4d0 0x7f513c06d990 unknown :-1 s=CLOSED pgs=731 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f515c15acb0 0x7f515c071180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f515c0b4680 0x7f515c0ae710 unknown :-1 s=CLOSED pgs=1720 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 --2- 172.21.15.86:0/497109513 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f515c072340 0x7f515c1478f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.386+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 >> 172.21.15.86:0/497109513 conn(0x7f515c0b1730 msgr2=0x7f515c0af790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.387+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 shutdown_connections 2024-03-06T07:37:15.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:15.387+0000 7f5161600640 1 -- 172.21.15.86:0/497109513 wait complete. 2024-03-06T07:37:15.610 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:15.610 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:15.647 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:16.611 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:16.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.828+0000 7f3e46800640 1 Processor -- start 2024-03-06T07:37:16.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.828+0000 7f3e46800640 1 -- start start 2024-03-06T07:37:16.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e46800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3e4015a4b0 0x7f3e40071380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:16.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e46800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e400cbb60 0x7f3e400b3d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:16.831 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e46800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3e4006c940 con 0x7f3e400b42d0 2024-03-06T07:37:16.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e46800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e400ce680 con 0x7f3e400cbb60 2024-03-06T07:37:16.832 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e46800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3e400cec40 con 0x7f3e4015a4b0 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e44a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3e400cbb60 0x7f3e400b3d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e45e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3e400b42d0 0x7f3e400cd460 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46634/0 (socket says 172.21.15.86:46634) 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.830+0000 7f3e45e00640 1 -- 172.21.15.86:0/3484717945 learned_addr learned my addr 172.21.15.86:0/3484717945 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.831+0000 7f3e45400640 1 --2- 172.21.15.86:0/3484717945 >> v2:172.21.15.194:3300/0 conn(0x7f3e4015a4b0 0x7f3e40071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.831+0000 7f3e44a00640 1 -- 172.21.15.86:0/3484717945 >> v2:172.21.15.194:3300/0 conn(0x7f3e4015a4b0 msgr2=0x7f3e40071380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.831+0000 7f3e44a00640 1 --2- 172.21.15.86:0/3484717945 >> v2:172.21.15.194:3300/0 conn(0x7f3e4015a4b0 0x7f3e40071380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.831+0000 7f3e44a00640 1 -- 172.21.15.86:0/3484717945 >> v1:172.21.15.86:6789/0 conn(0x7f3e400b42d0 legacy=0x7f3e400cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.831+0000 7f3e44a00640 1 -- 172.21.15.86:0/3484717945 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e40071980 con 0x7f3e400cbb60 2024-03-06T07:37:16.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.831+0000 7f3e45400640 1 --2- 172.21.15.86:0/3484717945 >> v2:172.21.15.194:3300/0 conn(0x7f3e4015a4b0 0x7f3e40071380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.832+0000 7f3e44a00640 1 --2- 172.21.15.86:0/3484717945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400cbb60 0x7f3e400b3d90 secure :-1 s=READY pgs=1722 cs=0 l=1 rev1=1 crypto rx=0x7f3e40068400 tx=0x7f3e2c02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=88b715aa350498eb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.832+0000 7f3e3fe00640 1 -- 172.21.15.86:0/3484717945 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e2c0315a0 con 0x7f3e400cbb60 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.832+0000 7f3e3fe00640 1 -- 172.21.15.86:0/3484717945 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e2c037070 con 0x7f3e400cbb60 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.832+0000 7f3e3fe00640 1 -- 172.21.15.86:0/3484717945 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e2c031960 con 0x7f3e400cbb60 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 -- 172.21.15.86:0/3484717945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400cbb60 msgr2=0x7f3e400b3d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 --2- 172.21.15.86:0/3484717945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400cbb60 0x7f3e400b3d90 secure :-1 s=READY pgs=1722 cs=0 l=1 rev1=1 crypto rx=0x7f3e40068400 tx=0x7f3e2c02b840 comp rx=0 tx=0).stop 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 -- 172.21.15.86:0/3484717945 shutdown_connections 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 --2- 172.21.15.86:0/3484717945 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400cbb60 0x7f3e400b3d90 unknown :-1 s=CLOSED pgs=1722 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 --2- 172.21.15.86:0/3484717945 >> v2:172.21.15.194:3300/0 conn(0x7f3e4015a4b0 0x7f3e40071380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 -- 172.21.15.86:0/3484717945 >> 172.21.15.86:0/3484717945 conn(0x7f3e4006afc0 msgr2=0x7f3e4006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 -- 172.21.15.86:0/3484717945 shutdown_connections 2024-03-06T07:37:16.835 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.833+0000 7f3e46800640 1 -- 172.21.15.86:0/3484717945 wait complete. 2024-03-06T07:37:16.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.836+0000 7f3e46800640 1 Processor -- start 2024-03-06T07:37:16.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.836+0000 7f3e46800640 1 -- start start 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e46800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400b42d0 0x7f3e40145710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e46800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e400cbb60 0x7f3e4015a890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e46800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e4015a4b0 0x7f3e400cd440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e46800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e400d00e0 con 0x7f3e400b42d0 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e46800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e400ce8d0 con 0x7f3e4015a4b0 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e46800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3e400b46e0 con 0x7f3e400cbb60 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400b42d0 0x7f3e40145710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400b42d0 0x7f3e40145710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58046/0 (socket says 172.21.15.86:58046) 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 -- 172.21.15.86:0/257611569 learned_addr learned my addr 172.21.15.86:0/257611569 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 -- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e400cbb60 msgr2=0x7f3e4015a890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e44a00640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e400cbb60 0x7f3e4015a890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:16.841 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45e00640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e4015a4b0 0x7f3e400cd440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e400cbb60 0x7f3e4015a890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 -- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e4015a4b0 msgr2=0x7f3e400cd440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e4015a4b0 0x7f3e400cd440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45400640 1 -- 172.21.15.86:0/257611569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e40157ab0 con 0x7f3e400b42d0 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.838+0000 7f3e45e00640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e4015a4b0 0x7f3e400cd440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.839+0000 7f3e45400640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400b42d0 0x7f3e40145710 secure :-1 s=READY pgs=1723 cs=0 l=1 rev1=1 crypto rx=0x7f3e3400c820 tx=0x7f3e3400cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.839+0000 7f3e3d600640 1 -- 172.21.15.86:0/257611569 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e34015070 con 0x7f3e400b42d0 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.839+0000 7f3e3d600640 1 -- 172.21.15.86:0/257611569 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e34007a30 con 0x7f3e400b42d0 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.839+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e40157d90 con 0x7f3e400b42d0 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.839+0000 7f3e3d600640 1 -- 172.21.15.86:0/257611569 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e34007e00 con 0x7f3e400b42d0 2024-03-06T07:37:16.842 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.839+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e40158350 con 0x7f3e400b42d0 2024-03-06T07:37:16.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.841+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3e4004f330 con 0x7f3e400b42d0 2024-03-06T07:37:16.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.841+0000 7f3e3d600640 1 -- 172.21.15.86:0/257611569 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3e34018070 con 0x7f3e400b42d0 2024-03-06T07:37:16.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.842+0000 7f3e3d600640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e2006b500 0x7f3e2006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:16.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.843+0000 7f3e3d600640 1 -- 172.21.15.86:0/257611569 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3e3408a270 con 0x7f3e400b42d0 2024-03-06T07:37:16.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.843+0000 7f3e44a00640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e2006b500 0x7f3e2006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:16.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.844+0000 7f3e44a00640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e2006b500 0x7f3e2006d9c0 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f3e2c036000 tx=0x7f3e2c008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:16.849 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:16.847+0000 7f3e3d600640 1 -- 172.21.15.86:0/257611569 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3e34059010 con 0x7f3e400b42d0 2024-03-06T07:37:17.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.108+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3e400b3130 con 0x7f3e400b42d0 2024-03-06T07:37:17.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.108+0000 7f3e3d600640 1 -- 172.21.15.86:0/257611569 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f3e34058910 con 0x7f3e400b42d0 2024-03-06T07:37:17.110 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:17.110 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:17.111 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e2006b500 msgr2=0x7f3e2006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e2006b500 0x7f3e2006d9c0 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f3e2c036000 tx=0x7f3e2c008030 comp rx=0 tx=0).stop 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400b42d0 msgr2=0x7f3e40145710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400b42d0 0x7f3e40145710 secure :-1 s=READY pgs=1723 cs=0 l=1 rev1=1 crypto rx=0x7f3e3400c820 tx=0x7f3e3400cc20 comp rx=0 tx=0).stop 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 shutdown_connections 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3e2006b500 0x7f3e2006d9c0 unknown :-1 s=CLOSED pgs=732 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3e4015a4b0 0x7f3e400cd440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3e400cbb60 0x7f3e4015a890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:17.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 --2- 172.21.15.86:0/257611569 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3e400b42d0 0x7f3e40145710 unknown :-1 s=CLOSED pgs=1723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:17.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 >> 172.21.15.86:0/257611569 conn(0x7f3e4006afc0 msgr2=0x7f3e400b17d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:17.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 shutdown_connections 2024-03-06T07:37:17.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:17.111+0000 7f3e46800640 1 -- 172.21.15.86:0/257611569 wait complete. 2024-03-06T07:37:17.333 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:17.333 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:17.650 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:18.334 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:18.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.544+0000 7f1289600640 1 Processor -- start 2024-03-06T07:37:18.547 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.545+0000 7f1289600640 1 -- start start 2024-03-06T07:37:18.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.546+0000 7f1289600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 0x7f1284071220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.549 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.546+0000 7f1289600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f128406cef0 con 0x7f128415f9d0 2024-03-06T07:37:18.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.546+0000 7f1289600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f128406d1d0 con 0x7f1284144c00 2024-03-06T07:37:18.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.546+0000 7f1289600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f128406d4b0 con 0x7f128415fe60 2024-03-06T07:37:18.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.546+0000 7f1282000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f128415f9d0 0x7f1284070b10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46650/0 (socket says 172.21.15.86:46650) 2024-03-06T07:37:18.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1282000640 1 -- 172.21.15.86:0/109342717 learned_addr learned my addr 172.21.15.86:0/109342717 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:18.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1283400640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 0x7f1284071220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 790717753 0 0) 0x7f128406cef0 con 0x7f128415f9d0 2024-03-06T07:37:18.550 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1268003520 con 0x7f128415f9d0 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1717264320 0 0) 0x7f1268003520 con 0x7f128415f9d0 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 msgr2=0x7f1284071220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 0x7f1284071220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 >> v1:172.21.15.194:6789/0 conn(0x7f1284144c00 legacy=0x7f128406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f128406d790 con 0x7f128415f9d0 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.547+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f1270001670 con 0x7f128415f9d0 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 >> v1:172.21.15.86:6789/0 conn(0x7f128415f9d0 legacy=0x7f1284070b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1281600640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12680039e0 0x7f1268003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1281600640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1268004800 0x7f1268004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1281600640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 0x7f12680051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f128406d4b0 con 0x7f12680039e0 2024-03-06T07:37:18.553 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f126800baa0 con 0x7f1268004800 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f126800bca0 con 0x7f128415fe60 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1282a00640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12680039e0 0x7f1268003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1283400640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 0x7f12680051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1282000640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1268004800 0x7f1268004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1282a00640 1 -- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 msgr2=0x7f12680051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1282a00640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f128415fe60 0x7f12680051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1282a00640 1 -- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1268004800 msgr2=0x7f1268004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1282a00640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1268004800 0x7f1268004c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.548+0000 7f1282a00640 1 -- 172.21.15.86:0/109342717 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1268003520 con 0x7f12680039e0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.549+0000 7f1282a00640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12680039e0 0x7f1268003e00 secure :-1 s=READY pgs=1725 cs=0 l=1 rev1=1 crypto rx=0x7f1274001e20 tx=0x7f1268005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cd1a7d2e57ba5dd1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1274001a70 con 0x7f12680039e0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1274003b70 con 0x7f12680039e0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1281600640 1 -- 172.21.15.86:0/109342717 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12740050d0 con 0x7f12680039e0 2024-03-06T07:37:18.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1289600640 1 -- 172.21.15.86:0/109342717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12680039e0 msgr2=0x7f1268003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:18.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1289600640 1 --2- 172.21.15.86:0/109342717 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f12680039e0 0x7f1268003e00 secure :-1 s=READY pgs=1725 cs=0 l=1 rev1=1 crypto rx=0x7f1274001e20 tx=0x7f1268005830 comp rx=0 tx=0).stop 2024-03-06T07:37:18.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1283400640 1 -- 172.21.15.86:0/109342717 reap_dead start 2024-03-06T07:37:18.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1289600640 1 -- 172.21.15.86:0/109342717 shutdown_connections 2024-03-06T07:37:18.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1289600640 1 -- 172.21.15.86:0/109342717 >> 172.21.15.86:0/109342717 conn(0x7f12840b1730 msgr2=0x7f12840b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:18.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1289600640 1 -- 172.21.15.86:0/109342717 shutdown_connections 2024-03-06T07:37:18.555 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.550+0000 7f1289600640 1 -- 172.21.15.86:0/109342717 wait complete. 2024-03-06T07:37:18.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.552+0000 7f1289600640 1 Processor -- start 2024-03-06T07:37:18.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.552+0000 7f1289600640 1 -- start start 2024-03-06T07:37:18.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.553+0000 7f1289600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12680039e0 0x7f12841e8cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.553+0000 7f1289600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12840cbef0 0x7f12840cc350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.553+0000 7f1289600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f128415dda0 0x7f128415e250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.557 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.553+0000 7f1289600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f128406cb50 con 0x7f128415dda0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.553+0000 7f1289600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1284160730 con 0x7f12680039e0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.553+0000 7f1289600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f128406ccd0 con 0x7f12840cbef0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f128415dda0 0x7f128415e250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f128415dda0 0x7f128415e250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58060/0 (socket says 172.21.15.86:58060) 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 -- 172.21.15.86:0/2475748043 learned_addr learned my addr 172.21.15.86:0/2475748043 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1282000640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12840cbef0 0x7f12840cc350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 -- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12840cbef0 msgr2=0x7f12840cc350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12840cbef0 0x7f12840cc350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 -- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12680039e0 msgr2=0x7f12841e8cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1282a00640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12680039e0 0x7f12841e8cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12680039e0 0x7f12841e8cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 -- 172.21.15.86:0/2475748043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f128415ead0 con 0x7f128415dda0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.554+0000 7f1283400640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f128415dda0 0x7f128415e250 secure :-1 s=READY pgs=1726 cs=0 l=1 rev1=1 crypto rx=0x7f12840d0ac0 tx=0x7f127c010c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.555+0000 7f127aa00640 1 -- 172.21.15.86:0/2475748043 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f127c00f040 con 0x7f128415dda0 2024-03-06T07:37:18.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.555+0000 7f127aa00640 1 -- 172.21.15.86:0/2475748043 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f127c01c070 con 0x7f128415dda0 2024-03-06T07:37:18.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.555+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1284130330 con 0x7f128415dda0 2024-03-06T07:37:18.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.555+0000 7f127aa00640 1 -- 172.21.15.86:0/2475748043 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f127c00eb30 con 0x7f128415dda0 2024-03-06T07:37:18.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.555+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12841308f0 con 0x7f128415dda0 2024-03-06T07:37:18.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.556+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f128404f330 con 0x7f128415dda0 2024-03-06T07:37:18.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.557+0000 7f127aa00640 1 -- 172.21.15.86:0/2475748043 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f127c00ecd0 con 0x7f128415dda0 2024-03-06T07:37:18.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.558+0000 7f127aa00640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f126406b500 0x7f126406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:18.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.558+0000 7f127aa00640 1 -- 172.21.15.86:0/2475748043 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f127c089ff0 con 0x7f128415dda0 2024-03-06T07:37:18.561 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.558+0000 7f1282a00640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f126406b500 0x7f126406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:18.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.559+0000 7f1282a00640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f126406b500 0x7f126406d9c0 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f1274034000 tx=0x7f1274039040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:18.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.564+0000 7f127aa00640 1 -- 172.21.15.86:0/2475748043 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f127c058dd0 con 0x7f128415dda0 2024-03-06T07:37:18.896 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.894+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1284130ac0 con 0x7f128415dda0 2024-03-06T07:37:18.897 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.895+0000 7f127aa00640 1 -- 172.21.15.86:0/2475748043 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f127c0586d0 con 0x7f128415dda0 2024-03-06T07:37:18.897 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:18.898 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:18.898 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:18.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f126406b500 msgr2=0x7f126406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:18.900 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f126406b500 0x7f126406d9c0 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f1274034000 tx=0x7f1274039040 comp rx=0 tx=0).stop 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f128415dda0 msgr2=0x7f128415e250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f128415dda0 0x7f128415e250 secure :-1 s=READY pgs=1726 cs=0 l=1 rev1=1 crypto rx=0x7f12840d0ac0 tx=0x7f127c010c20 comp rx=0 tx=0).stop 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 shutdown_connections 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f126406b500 0x7f126406d9c0 unknown :-1 s=CLOSED pgs=733 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f128415dda0 0x7f128415e250 unknown :-1 s=CLOSED pgs=1726 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12840cbef0 0x7f12840cc350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 --2- 172.21.15.86:0/2475748043 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12680039e0 0x7f12841e8cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 >> 172.21.15.86:0/2475748043 conn(0x7f12840b1730 msgr2=0x7f12840b0f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:18.901 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 shutdown_connections 2024-03-06T07:37:18.902 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:18.898+0000 7f1289600640 1 -- 172.21.15.86:0/2475748043 wait complete. 2024-03-06T07:37:19.120 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:19.120 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:19.651 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:20.121 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:20.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.327+0000 7fdd0cc00640 1 Processor -- start 2024-03-06T07:37:20.330 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.327+0000 7fdd0cc00640 1 -- start start 2024-03-06T07:37:20.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.328+0000 7fdd0cc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 0x7fdd08160380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.331 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd0cc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fdd0806b3a0 0x7fdd0815c3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd0cc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdd081608c0 con 0x7fdd0815c900 2024-03-06T07:37:20.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd0cc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdd08145b00 con 0x7fdd0806b3a0 2024-03-06T07:37:20.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd0cc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd080d02e0 con 0x7fdd0806afc0 2024-03-06T07:37:20.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd06800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fdd0815c900 0x7fdd0815d750 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46656/0 (socket says 172.21.15.86:46656) 2024-03-06T07:37:20.332 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd06800640 1 -- 172.21.15.86:0/3948235230 learned_addr learned my addr 172.21.15.86:0/3948235230 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:20.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd05400640 1 --2- 172.21.15.86:0/3948235230 >> v2:172.21.15.194:3300/0 conn(0x7fdd0806b3a0 0x7fdd0815c3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:20.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd05e00640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 0x7fdd08160380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:20.333 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4082687261 0 0) 0x7fdd081608c0 con 0x7fdd0815c900 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.329+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fdcec003520 con 0x7fdd0815c900 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2464583715 0 0) 0x7fdcec003520 con 0x7fdd0815c900 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 msgr2=0x7fdd08160380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 0x7fdd08160380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 >> v2:172.21.15.194:3300/0 conn(0x7fdd0806b3a0 msgr2=0x7fdd0815c3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 --2- 172.21.15.86:0/3948235230 >> v2:172.21.15.194:3300/0 conn(0x7fdd0806b3a0 0x7fdd0815c3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd080cf6d0 con 0x7fdd0815c900 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fdd000019f0 con 0x7fdd0815c900 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 >> v1:172.21.15.86:6789/0 conn(0x7fdd0815c900 legacy=0x7fdd0815d750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 0x7fdcec0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdcec004030 0x7fdcec004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdcec004d40 0x7fdcec0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd08145b00 con 0x7fdcec004030 2024-03-06T07:37:20.334 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd080d02e0 con 0x7fdcec004d40 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdcec00bb20 con 0x7fdd0806afc0 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.330+0000 7fdd05400640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdcec004030 0x7fdcec004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.331+0000 7fdd05e00640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 0x7fdcec0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.331+0000 7fdd05400640 1 -- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 msgr2=0x7fdcec0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.331+0000 7fdd05400640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdd0806afc0 0x7fdcec0039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.331+0000 7fdd05400640 1 -- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdcec004d40 msgr2=0x7fdcec0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.331+0000 7fdd05400640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdcec004d40 0x7fdcec0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.335 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.331+0000 7fdd05400640 1 -- 172.21.15.86:0/3948235230 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdcec003520 con 0x7fdcec004030 2024-03-06T07:37:20.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.332+0000 7fdd05400640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdcec004030 0x7fdcec004800 secure :-1 s=READY pgs=1728 cs=0 l=1 rev1=1 crypto rx=0x7fdcf400ba30 tx=0x7fdcec005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e3584963e51c80e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:20.336 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.332+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdcf4036020 con 0x7fdcec004030 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.332+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdcf4032680 con 0x7fdcec004030 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.332+0000 7fdd04a00640 1 -- 172.21.15.86:0/3948235230 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdcf400b600 con 0x7fdcec004030 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.333+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3948235230 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdcec004030 msgr2=0x7fdcec004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.333+0000 7fdd0cc00640 1 --2- 172.21.15.86:0/3948235230 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdcec004030 0x7fdcec004800 secure :-1 s=READY pgs=1728 cs=0 l=1 rev1=1 crypto rx=0x7fdcf400ba30 tx=0x7fdcec005830 comp rx=0 tx=0).stop 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.333+0000 7fdd06800640 1 -- 172.21.15.86:0/3948235230 reap_dead start 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.333+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3948235230 shutdown_connections 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.333+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3948235230 >> 172.21.15.86:0/3948235230 conn(0x7fdd080d0b40 msgr2=0x7fdd080d0f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.333+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3948235230 shutdown_connections 2024-03-06T07:37:20.337 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.333+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3948235230 wait complete. 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.335+0000 7fdd0cc00640 1 Processor -- start 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.335+0000 7fdd0cc00640 1 -- start start 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd0cc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdcec004030 0x7fdd081f6fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd0cc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd081f4350 0x7fdd080afbb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd0cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd081377a0 0x7fdd08137c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd0cc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd080cff70 con 0x7fdd081377a0 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd0cc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd080cfc90 con 0x7fdd081f4350 2024-03-06T07:37:20.338 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd0cc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd0806b820 con 0x7fdcec004030 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd081377a0 0x7fdd08137c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd081377a0 0x7fdd08137c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58088/0 (socket says 172.21.15.86:58088) 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 -- 172.21.15.86:0/3863570383 learned_addr learned my addr 172.21.15.86:0/3863570383 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd05400640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd081f4350 0x7fdd080afbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 -- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdcec004030 msgr2=0x7fdd081f6fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdcec004030 0x7fdd081f6fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 -- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd081f4350 msgr2=0x7fdd080afbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd081f4350 0x7fdd080afbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd06800640 1 -- 172.21.15.86:0/3863570383 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd0813a760 con 0x7fdd081377a0 2024-03-06T07:37:20.340 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.336+0000 7fdd05400640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd081f4350 0x7fdd080afbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:20.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.337+0000 7fdd06800640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd081377a0 0x7fdd08137c20 secure :-1 s=READY pgs=1729 cs=0 l=1 rev1=1 crypto rx=0x7fdd080cea10 tx=0x7fdd0000cdc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:20.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.337+0000 7fdcfe000640 1 -- 172.21.15.86:0/3863570383 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd0000a610 con 0x7fdd081377a0 2024-03-06T07:37:20.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.337+0000 7fdcfe000640 1 -- 172.21.15.86:0/3863570383 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd0000a7b0 con 0x7fdd081377a0 2024-03-06T07:37:20.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.337+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdd0813aa40 con 0x7fdd081377a0 2024-03-06T07:37:20.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.337+0000 7fdcfe000640 1 -- 172.21.15.86:0/3863570383 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd0000abc0 con 0x7fdd081377a0 2024-03-06T07:37:20.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.337+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdd08143280 con 0x7fdd081377a0 2024-03-06T07:37:20.341 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.338+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdd0804f330 con 0x7fdd081377a0 2024-03-06T07:37:20.342 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.339+0000 7fdcfe000640 1 -- 172.21.15.86:0/3863570383 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdd0001c3f0 con 0x7fdd081377a0 2024-03-06T07:37:20.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.340+0000 7fdcfe000640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdce806b4d0 0x7fdce806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:20.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.341+0000 7fdcfe000640 1 -- 172.21.15.86:0/3863570383 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fdd00089c60 con 0x7fdd081377a0 2024-03-06T07:37:20.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.341+0000 7fdd05e00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdce806b4d0 0x7fdce806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:20.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.341+0000 7fdd05e00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdce806b4d0 0x7fdce806d990 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7fdcf8006f00 tx=0x7fdcf8006e10 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:20.352 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.349+0000 7fdcfe000640 1 -- 172.21.15.86:0/3863570383 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdd000583b0 con 0x7fdd081377a0 2024-03-06T07:37:20.585 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.582+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fdd0806baf0 con 0x7fdd081377a0 2024-03-06T07:37:20.586 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.583+0000 7fdcfe000640 1 -- 172.21.15.86:0/3863570383 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fdd0005c450 con 0x7fdd081377a0 2024-03-06T07:37:20.586 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:20.586 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:20.587 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdce806b4d0 msgr2=0x7fdce806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdce806b4d0 0x7fdce806d990 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7fdcf8006f00 tx=0x7fdcf8006e10 comp rx=0 tx=0).stop 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd081377a0 msgr2=0x7fdd08137c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd081377a0 0x7fdd08137c20 secure :-1 s=READY pgs=1729 cs=0 l=1 rev1=1 crypto rx=0x7fdd080cea10 tx=0x7fdd0000cdc0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 shutdown_connections 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fdce806b4d0 0x7fdce806d990 unknown :-1 s=CLOSED pgs=734 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fdd081377a0 0x7fdd08137c20 unknown :-1 s=CLOSED pgs=1729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fdd081f4350 0x7fdd080afbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 --2- 172.21.15.86:0/3863570383 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fdcec004030 0x7fdd081f6fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 >> 172.21.15.86:0/3863570383 conn(0x7fdd080d0b40 msgr2=0x7fdd081420a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 shutdown_connections 2024-03-06T07:37:20.589 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:20.586+0000 7fdd0cc00640 1 -- 172.21.15.86:0/3863570383 wait complete. 2024-03-06T07:37:20.810 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:20.810 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:21.652 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:21.811 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:22.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.023+0000 7f21d1000640 1 Processor -- start 2024-03-06T07:37:22.026 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.023+0000 7f21d1000640 1 -- start start 2024-03-06T07:37:22.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.025+0000 7f21d1000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f21cc15eff0 0x7f21cc070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:22.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.025+0000 7f21d1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc06e4d0 0x7f21cc06e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:22.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.025+0000 7f21d1000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21cc071d30 con 0x7f21cc0af1c0 2024-03-06T07:37:22.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.025+0000 7f21d1000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21cc071260 con 0x7f21cc15eff0 2024-03-06T07:37:22.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.025+0000 7f21d1000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21cc15f4a0 con 0x7f21cc06e4d0 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.026+0000 7f21c1400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f21cc15eff0 0x7f21cc070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.026+0000 7f21ca800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc06e4d0 0x7f21cc06e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.026+0000 7f21c1400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f21cc15eff0 0x7f21cc070d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:46912/0 (socket says 172.21.15.86:46912) 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.026+0000 7f21c1400640 1 -- 172.21.15.86:0/913388505 learned_addr learned my addr 172.21.15.86:0/913388505 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.027+0000 7f21c1400640 1 -- 172.21.15.86:0/913388505 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc06e4d0 msgr2=0x7f21cc06e960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.027+0000 7f21c1400640 1 --2- 172.21.15.86:0/913388505 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc06e4d0 0x7f21cc06e960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.027+0000 7f21c1400640 1 -- 172.21.15.86:0/913388505 >> v1:172.21.15.194:6789/0 conn(0x7f21cc0af1c0 legacy=0x7f21cc06cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:22.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.027+0000 7f21c1400640 1 -- 172.21.15.86:0/913388505 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21cc071490 con 0x7f21cc15eff0 2024-03-06T07:37:22.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.028+0000 7f21c1400640 1 --2- 172.21.15.86:0/913388505 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 0x7f21cc070d20 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f21b40098a0 tx=0x7f21b402b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a3abf8a14ff2f69d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:22.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.028+0000 7f21c9400640 1 -- 172.21.15.86:0/913388505 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21b4034070 con 0x7f21cc15eff0 2024-03-06T07:37:22.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.028+0000 7f21c9400640 1 -- 172.21.15.86:0/913388505 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f21b4030840 con 0x7f21cc15eff0 2024-03-06T07:37:22.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.028+0000 7f21c9400640 1 -- 172.21.15.86:0/913388505 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21b4030c10 con 0x7f21cc15eff0 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 -- 172.21.15.86:0/913388505 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 msgr2=0x7f21cc070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 --2- 172.21.15.86:0/913388505 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 0x7f21cc070d20 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f21b40098a0 tx=0x7f21b402b980 comp rx=0 tx=0).stop 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 -- 172.21.15.86:0/913388505 shutdown_connections 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 --2- 172.21.15.86:0/913388505 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc06e4d0 0x7f21cc06e960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 --2- 172.21.15.86:0/913388505 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 0x7f21cc070d20 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 -- 172.21.15.86:0/913388505 >> 172.21.15.86:0/913388505 conn(0x7f21cc0b1730 msgr2=0x7f21cc0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 -- 172.21.15.86:0/913388505 shutdown_connections 2024-03-06T07:37:22.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.029+0000 7f21d1000640 1 -- 172.21.15.86:0/913388505 wait complete. 2024-03-06T07:37:22.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.031+0000 7f21d1000640 1 Processor -- start 2024-03-06T07:37:22.033 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.031+0000 7f21d1000640 1 -- start start 2024-03-06T07:37:22.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21d1000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21cc06e4d0 0x7f21cc1e72d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:22.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21d1000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc0af1c0 0x7f21cc14b0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:22.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21d1000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 0x7f21cc14b610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:22.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21d1000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21cc0b30b0 con 0x7f21cc06e4d0 2024-03-06T07:37:22.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21d1000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21cc071a50 con 0x7f21cc15eff0 2024-03-06T07:37:22.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21d1000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21cc06d370 con 0x7f21cc0af1c0 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21c9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21cc06e4d0 0x7f21cc1e72d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21c9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21cc06e4d0 0x7f21cc1e72d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58096/0 (socket says 172.21.15.86:58096) 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21c9e00640 1 -- 172.21.15.86:0/3885478283 learned_addr learned my addr 172.21.15.86:0/3885478283 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.032+0000 7f21c1400640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc0af1c0 0x7f21cc14b0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21ca800640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 0x7f21cc14b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c9e00640 1 -- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc0af1c0 msgr2=0x7f21cc14b0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c9e00640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc0af1c0 0x7f21cc14b0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c9e00640 1 -- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 msgr2=0x7f21cc14b610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:22.036 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c9e00640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 0x7f21cc14b610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c9e00640 1 -- 172.21.15.86:0/3885478283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21cc0ce100 con 0x7f21cc06e4d0 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c9e00640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21cc06e4d0 0x7f21cc1e72d0 secure :-1 s=READY pgs=1730 cs=0 l=1 rev1=1 crypto rx=0x7f21bc00c7c0 tx=0x7f21bc00cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c2a00640 1 -- 172.21.15.86:0/3885478283 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21bc016020 con 0x7f21cc06e4d0 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c2a00640 1 -- 172.21.15.86:0/3885478283 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f21bc006d50 con 0x7f21cc06e4d0 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21cc0ce3e0 con 0x7f21cc06e4d0 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.033+0000 7f21c2a00640 1 -- 172.21.15.86:0/3885478283 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21bc003bb0 con 0x7f21cc06e4d0 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.034+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21cc0ce9a0 con 0x7f21cc06e4d0 2024-03-06T07:37:22.037 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.035+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f21cc04f330 con 0x7f21cc06e4d0 2024-03-06T07:37:22.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.036+0000 7f21c2a00640 1 -- 172.21.15.86:0/3885478283 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f21bc00a850 con 0x7f21cc06e4d0 2024-03-06T07:37:22.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.038+0000 7f21c2a00640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21a806b4d0 0x7f21a806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:22.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.038+0000 7f21c2a00640 1 -- 172.21.15.86:0/3885478283 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f21bc089f90 con 0x7f21cc06e4d0 2024-03-06T07:37:22.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.038+0000 7f21c1400640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21a806b4d0 0x7f21a806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:22.041 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.039+0000 7f21c1400640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21a806b4d0 0x7f21a806d990 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7f21b4036000 tx=0x7f21b403b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:22.046 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.043+0000 7f21c2a00640 1 -- 172.21.15.86:0/3885478283 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f21bc020020 con 0x7f21cc06e4d0 2024-03-06T07:37:22.287 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.284+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f21cc074be0 con 0x7f21cc06e4d0 2024-03-06T07:37:22.288 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.285+0000 7f21c2a00640 1 -- 172.21.15.86:0/3885478283 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f21cc071d30 con 0x7f21cc06e4d0 2024-03-06T07:37:22.288 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:22.288 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:22.288 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:22.290 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21a806b4d0 msgr2=0x7f21a806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21a806b4d0 0x7f21a806d990 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7f21b4036000 tx=0x7f21b403b040 comp rx=0 tx=0).stop 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21cc06e4d0 msgr2=0x7f21cc1e72d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21cc06e4d0 0x7f21cc1e72d0 secure :-1 s=READY pgs=1730 cs=0 l=1 rev1=1 crypto rx=0x7f21bc00c7c0 tx=0x7f21bc00cc80 comp rx=0 tx=0).stop 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 shutdown_connections 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f21a806b4d0 0x7f21a806d990 unknown :-1 s=CLOSED pgs=735 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f21cc15eff0 0x7f21cc14b610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f21cc0af1c0 0x7f21cc14b0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 --2- 172.21.15.86:0/3885478283 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21cc06e4d0 0x7f21cc1e72d0 unknown :-1 s=CLOSED pgs=1730 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 >> 172.21.15.86:0/3885478283 conn(0x7f21cc0b1730 msgr2=0x7f21cc15e550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 shutdown_connections 2024-03-06T07:37:22.291 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:22.288+0000 7f21d1000640 1 -- 172.21.15.86:0/3885478283 wait complete. 2024-03-06T07:37:22.508 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:22.508 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:23.509 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:23.654 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:23.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.716+0000 7fb0bf200640 1 Processor -- start 2024-03-06T07:37:23.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.716+0000 7fb0bf200640 1 -- start start 2024-03-06T07:37:23.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.718+0000 7fb0bf200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fb0b815dff0 0x7fb0b815e480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:23.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.718+0000 7fb0bf200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb0b80b35b0 con 0x7fb0b8151660 2024-03-06T07:37:23.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.718+0000 7fb0bf200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb0b80b3890 con 0x7fb0b80cf430 2024-03-06T07:37:23.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.718+0000 7fb0bf200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0b80b3ee0 con 0x7fb0b815dff0 2024-03-06T07:37:23.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0b7e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fb0b8151660 0x7fb0b8071520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46670/0 (socket says 172.21.15.86:46670) 2024-03-06T07:37:23.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0b7e00640 1 -- 172.21.15.86:0/1952494264 learned_addr learned my addr 172.21.15.86:0/1952494264 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:23.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0bd200640 1 --2- 172.21.15.86:0/1952494264 >> v2:172.21.15.86:3300/0 conn(0x7fb0b815dff0 0x7fb0b815e480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:23.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0b7400640 1 -- 172.21.15.86:0/1952494264 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4208429160 0 0) 0x7fb0b80b35b0 con 0x7fb0b8151660 2024-03-06T07:37:23.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0b7400640 1 -- 172.21.15.86:0/1952494264 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb09c003440 con 0x7fb0b8151660 2024-03-06T07:37:23.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0bd200640 1 -- 172.21.15.86:0/1952494264 >> v1:172.21.15.194:6789/0 conn(0x7fb0b80cf430 legacy=0x7fb0b806d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:23.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0bd200640 1 -- 172.21.15.86:0/1952494264 >> v1:172.21.15.86:6789/0 conn(0x7fb0b8151660 legacy=0x7fb0b8071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:23.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.719+0000 7fb0bd200640 1 -- 172.21.15.86:0/1952494264 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0b80b3b70 con 0x7fb0b815dff0 2024-03-06T07:37:23.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.721+0000 7fb0bd200640 1 --2- 172.21.15.86:0/1952494264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b815dff0 0x7fb0b815e480 secure :-1 s=READY pgs=1732 cs=0 l=1 rev1=1 crypto rx=0x7fb0b00097a0 tx=0x7fb0b002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9b8c899be14de8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:23.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.721+0000 7fb0b7400640 1 -- 172.21.15.86:0/1952494264 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0b00315a0 con 0x7fb0b815dff0 2024-03-06T07:37:23.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.721+0000 7fb0b7400640 1 -- 172.21.15.86:0/1952494264 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0b0037070 con 0x7fb0b815dff0 2024-03-06T07:37:23.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.721+0000 7fb0b7400640 1 -- 172.21.15.86:0/1952494264 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0b0031960 con 0x7fb0b815dff0 2024-03-06T07:37:23.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.722+0000 7fb0bf200640 1 -- 172.21.15.86:0/1952494264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b815dff0 msgr2=0x7fb0b815e480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:23.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.722+0000 7fb0bf200640 1 --2- 172.21.15.86:0/1952494264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b815dff0 0x7fb0b815e480 secure :-1 s=READY pgs=1732 cs=0 l=1 rev1=1 crypto rx=0x7fb0b00097a0 tx=0x7fb0b002b840 comp rx=0 tx=0).stop 2024-03-06T07:37:23.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.722+0000 7fb0bf200640 1 -- 172.21.15.86:0/1952494264 shutdown_connections 2024-03-06T07:37:23.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.722+0000 7fb0bf200640 1 --2- 172.21.15.86:0/1952494264 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b815dff0 0x7fb0b815e480 unknown :-1 s=CLOSED pgs=1732 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:23.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.722+0000 7fb0bf200640 1 -- 172.21.15.86:0/1952494264 >> 172.21.15.86:0/1952494264 conn(0x7fb0b80b1730 msgr2=0x7fb0b80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:23.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.722+0000 7fb0bf200640 1 -- 172.21.15.86:0/1952494264 shutdown_connections 2024-03-06T07:37:23.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.722+0000 7fb0bf200640 1 -- 172.21.15.86:0/1952494264 wait complete. 2024-03-06T07:37:23.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.724+0000 7fb0bf200640 1 Processor -- start 2024-03-06T07:37:23.726 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.724+0000 7fb0bf200640 1 -- start start 2024-03-06T07:37:23.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0bf200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b80cf430 0x7fb0b8145910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:23.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0bf200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0b8151660 0x7fb0b81f79f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:23.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0bf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0b815dff0 0x7fb0b81f4b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:23.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0bf200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b806cb50 con 0x7fb0b80cf430 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0bf200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b806daf0 con 0x7fb0b8151660 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0bf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b806ccd0 con 0x7fb0b815dff0 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b80cf430 0x7fb0b8145910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0b7e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b80cf430 0x7fb0b8145910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58104/0 (socket says 172.21.15.86:58104) 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0b7e00640 1 -- 172.21.15.86:0/262026768 learned_addr learned my addr 172.21.15.86:0/262026768 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0af400640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0b8151660 0x7fb0b81f79f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0b7e00640 1 -- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0b815dff0 msgr2=0x7fb0b81f4b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0b7e00640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0b815dff0 0x7fb0b81f4b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.725+0000 7fb0b7e00640 1 -- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0b8151660 msgr2=0x7fb0b81f79f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0b7e00640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0b8151660 0x7fb0b81f79f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0b7e00640 1 -- 172.21.15.86:0/262026768 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0b81f88b0 con 0x7fb0b80cf430 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0af400640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0b8151660 0x7fb0b81f79f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:23.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0b7e00640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b80cf430 0x7fb0b8145910 secure :-1 s=READY pgs=1733 cs=0 l=1 rev1=1 crypto rx=0x7fb0a800cbf0 tx=0x7fb0a800a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:23.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0b4c00640 1 -- 172.21.15.86:0/262026768 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0a8017070 con 0x7fb0b80cf430 2024-03-06T07:37:23.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0b4c00640 1 -- 172.21.15.86:0/262026768 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0a800ad50 con 0x7fb0b80cf430 2024-03-06T07:37:23.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0b81f82f0 con 0x7fb0b80cf430 2024-03-06T07:37:23.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0b4c00640 1 -- 172.21.15.86:0/262026768 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0a8012620 con 0x7fb0b80cf430 2024-03-06T07:37:23.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.726+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0b8145110 con 0x7fb0b80cf430 2024-03-06T07:37:23.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.727+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb0b806c780 con 0x7fb0b80cf430 2024-03-06T07:37:23.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.729+0000 7fb0b4c00640 1 -- 172.21.15.86:0/262026768 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb0a801a070 con 0x7fb0b80cf430 2024-03-06T07:37:23.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.729+0000 7fb0b4c00640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09406b4d0 0x7fb09406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:23.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.730+0000 7fb0b4c00640 1 -- 172.21.15.86:0/262026768 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb0a808a130 con 0x7fb0b80cf430 2024-03-06T07:37:23.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.730+0000 7fb0af400640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09406b4d0 0x7fb09406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:23.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.730+0000 7fb0af400640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09406b4d0 0x7fb09406d990 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7fb0b80b1080 tx=0x7fb0a0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:23.737 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.734+0000 7fb0b4c00640 1 -- 172.21.15.86:0/262026768 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb0b806c780 con 0x7fb0b80cf430 2024-03-06T07:37:23.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.963+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb0b815e480 con 0x7fb0b80cf430 2024-03-06T07:37:23.966 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.963+0000 7fb0b4c00640 1 -- 172.21.15.86:0/262026768 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fb0b815e480 con 0x7fb0b80cf430 2024-03-06T07:37:23.966 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:23.966 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:23.966 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09406b4d0 msgr2=0x7fb09406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09406b4d0 0x7fb09406d990 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7fb0b80b1080 tx=0x7fb0a0009040 comp rx=0 tx=0).stop 2024-03-06T07:37:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b80cf430 msgr2=0x7fb0b8145910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:23.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b80cf430 0x7fb0b8145910 secure :-1 s=READY pgs=1733 cs=0 l=1 rev1=1 crypto rx=0x7fb0a800cbf0 tx=0x7fb0a800a430 comp rx=0 tx=0).stop 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 shutdown_connections 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fb09406b4d0 0x7fb09406d990 unknown :-1 s=CLOSED pgs=736 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fb0b815dff0 0x7fb0b81f4b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fb0b8151660 0x7fb0b81f79f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 --2- 172.21.15.86:0/262026768 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fb0b80cf430 0x7fb0b8145910 unknown :-1 s=CLOSED pgs=1733 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 >> 172.21.15.86:0/262026768 conn(0x7fb0b80b1730 msgr2=0x7fb0b8071fb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 shutdown_connections 2024-03-06T07:37:23.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:23.967+0000 7fb0bf200640 1 -- 172.21.15.86:0/262026768 wait complete. 2024-03-06T07:37:24.188 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:24.188 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:25.188 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:25.383 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:37:25.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:37:25.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.402+0000 7f05cb000640 1 Processor -- start 2024-03-06T07:37:25.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.402+0000 7f05cb000640 1 -- start start 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.403+0000 7f05cb000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f05c40cd860 0x7f05c406afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.403+0000 7f05cb000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c406db00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.403+0000 7f05cb000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f05c40b5010 con 0x7f05c415e070 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05cb000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f05c406b950 con 0x7f05c40cd860 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05cb000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05c40b52f0 con 0x7f05c406b500 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f05c40cd860 0x7f05c406afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f05c40cd860 0x7f05c406afc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58110/0 (socket says 172.21.15.86:58110) 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3400640 1 -- 172.21.15.86:0/1601248904 learned_addr learned my addr 172.21.15.86:0/1601248904 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f05c415e070 0x7f05c4066d00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:46680/0 (socket says 172.21.15.86:46680) 2024-03-06T07:37:25.407 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c9000640 1 --2- 172.21.15.86:0/1601248904 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c406db00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:25.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3400640 1 -- 172.21.15.86:0/1601248904 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 msgr2=0x7f05c406db00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:25.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3400640 1 --2- 172.21.15.86:0/1601248904 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c406db00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3400640 1 -- 172.21.15.86:0/1601248904 >> v1:172.21.15.86:6789/0 conn(0x7f05c415e070 legacy=0x7f05c4066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:25.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.404+0000 7f05c3400640 1 -- 172.21.15.86:0/1601248904 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05c40b55d0 con 0x7f05c40cd860 2024-03-06T07:37:25.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.406+0000 7f05c3400640 1 --2- 172.21.15.86:0/1601248904 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c40cd860 0x7f05c406afc0 secure :-1 s=READY pgs=1735 cs=0 l=1 rev1=1 crypto rx=0x7f05b0009870 tx=0x7f05b002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d3a99d527a8c1af3 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:25.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.406+0000 7f05c2a00640 1 -- 172.21.15.86:0/1601248904 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05b0034070 con 0x7f05c40cd860 2024-03-06T07:37:25.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.406+0000 7f05c2a00640 1 -- 172.21.15.86:0/1601248904 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f05b0030840 con 0x7f05c40cd860 2024-03-06T07:37:25.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.406+0000 7f05c2a00640 1 -- 172.21.15.86:0/1601248904 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05b0030c10 con 0x7f05c40cd860 2024-03-06T07:37:25.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 -- 172.21.15.86:0/1601248904 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c40cd860 msgr2=0x7f05c406afc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:25.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 --2- 172.21.15.86:0/1601248904 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c40cd860 0x7f05c406afc0 secure :-1 s=READY pgs=1735 cs=0 l=1 rev1=1 crypto rx=0x7f05b0009870 tx=0x7f05b002b980 comp rx=0 tx=0).stop 2024-03-06T07:37:25.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 -- 172.21.15.86:0/1601248904 shutdown_connections 2024-03-06T07:37:25.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 --2- 172.21.15.86:0/1601248904 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c406db00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 --2- 172.21.15.86:0/1601248904 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c40cd860 0x7f05c406afc0 unknown :-1 s=CLOSED pgs=1735 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 -- 172.21.15.86:0/1601248904 >> 172.21.15.86:0/1601248904 conn(0x7f05c40b1e10 msgr2=0x7f05c40b2220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:25.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 -- 172.21.15.86:0/1601248904 shutdown_connections 2024-03-06T07:37:25.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.407+0000 7f05cb000640 1 -- 172.21.15.86:0/1601248904 wait complete. 2024-03-06T07:37:25.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.409+0000 7f05cb000640 1 Processor -- start 2024-03-06T07:37:25.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.409+0000 7f05cb000640 1 -- start start 2024-03-06T07:37:25.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05cb000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c4159000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:25.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05cb000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05c40cd860 0x7f05c40992a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:25.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05cb000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c415e070 0x7f05c4159a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05cb000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05c415e450 con 0x7f05c415e070 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05cb000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05c40b5b90 con 0x7f05c40cd860 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05cb000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05c40b60b0 con 0x7f05c406b500 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05c9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c415e070 0x7f05c4159a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05c9000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c415e070 0x7f05c4159a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:58120/0 (socket says 172.21.15.86:58120) 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.410+0000 7f05c9000640 1 -- 172.21.15.86:0/3098581219 learned_addr learned my addr 172.21.15.86:0/3098581219 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c3e00640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c4159000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c3400640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05c40cd860 0x7f05c40992a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c9000640 1 -- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 msgr2=0x7f05c4159000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c9000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c4159000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c9000640 1 -- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05c40cd860 msgr2=0x7f05c40992a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:25.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c9000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05c40cd860 0x7f05c40992a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c9000640 1 -- 172.21.15.86:0/3098581219 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05c415aab0 con 0x7f05c415e070 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c3400640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05c40cd860 0x7f05c40992a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05c9000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c415e070 0x7f05c4159a00 secure :-1 s=READY pgs=1736 cs=0 l=1 rev1=1 crypto rx=0x7f05bc007e60 tx=0x7f05bc00e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05bbe00640 1 -- 172.21.15.86:0/3098581219 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05bc014420 con 0x7f05c415e070 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.411+0000 7f05bbe00640 1 -- 172.21.15.86:0/3098581219 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f05bc019070 con 0x7f05c415e070 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.412+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f05c415ad90 con 0x7f05c415e070 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.412+0000 7f05bbe00640 1 -- 172.21.15.86:0/3098581219 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05bc014820 con 0x7f05c415e070 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.412+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f05c415b350 con 0x7f05c415e070 2024-03-06T07:37:25.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.413+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f05c404f330 con 0x7f05c415e070 2024-03-06T07:37:25.417 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.415+0000 7f05bbe00640 1 -- 172.21.15.86:0/3098581219 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f05bc0149c0 con 0x7f05c415e070 2024-03-06T07:37:25.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.416+0000 7f05bbe00640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05a406b4d0 0x7f05a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:25.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.416+0000 7f05bbe00640 1 -- 172.21.15.86:0/3098581219 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f05bc08aba0 con 0x7f05c415e070 2024-03-06T07:37:25.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.416+0000 7f05c3e00640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05a406b4d0 0x7f05a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:25.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.417+0000 7f05c3e00640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05a406b4d0 0x7f05a406d990 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7f05b4007fd0 tx=0x7f05b4007c40 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:25.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.421+0000 7f05bbe00640 1 -- 172.21.15.86:0/3098581219 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f05bc059940 con 0x7f05c415e070 2024-03-06T07:37:25.656 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:25.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.662+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f05c41597c0 con 0x7f05c415e070 2024-03-06T07:37:25.665 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.663+0000 7f05bbe00640 1 -- 172.21.15.86:0/3098581219 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f05bc059240 con 0x7f05c415e070 2024-03-06T07:37:25.665 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:25.665 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:25.666 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:25.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.665+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05a406b4d0 msgr2=0x7f05a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.665+0000 7f05cb000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05a406b4d0 0x7f05a406d990 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7f05b4007fd0 tx=0x7f05b4007c40 comp rx=0 tx=0).stop 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.665+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c415e070 msgr2=0x7f05c4159a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.665+0000 7f05cb000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c415e070 0x7f05c4159a00 secure :-1 s=READY pgs=1736 cs=0 l=1 rev1=1 crypto rx=0x7f05bc007e60 tx=0x7f05bc00e4b0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 shutdown_connections 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f05a406b4d0 0x7f05a406d990 unknown :-1 s=CLOSED pgs=737 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f05c415e070 0x7f05c4159a00 unknown :-1 s=CLOSED pgs=1736 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f05c40cd860 0x7f05c40992a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 --2- 172.21.15.86:0/3098581219 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f05c406b500 0x7f05c4159000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 >> 172.21.15.86:0/3098581219 conn(0x7f05c40b1e10 msgr2=0x7f05c40afd80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 shutdown_connections 2024-03-06T07:37:25.669 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:25.666+0000 7f05cb000640 1 -- 172.21.15.86:0/3098581219 wait complete. 2024-03-06T07:37:25.887 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:25.888 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:26.889 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:27.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.107+0000 7fc413800640 1 Processor -- start 2024-03-06T07:37:27.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.107+0000 7fc413800640 1 -- start start 2024-03-06T07:37:27.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.109+0000 7fc413800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c15acb0 0x7fc40c0b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:27.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc413800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc40c06ec00 0x7fc40c0ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:27.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc413800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc40c15b090 con 0x7fc40c0b4680 2024-03-06T07:37:27.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc413800640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc40c0b2a30 con 0x7fc40c06ec00 2024-03-06T07:37:27.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc413800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc40c0b4360 con 0x7fc40c15acb0 2024-03-06T07:37:27.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc411800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc40c06ec00 0x7fc40c0ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:27.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc410e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c15acb0 0x7fc40c0b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:27.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc411800640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc40c06ec00 0x7fc40c0ccfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:60286/0 (socket says 172.21.15.86:60286) 2024-03-06T07:37:27.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.110+0000 7fc411800640 1 -- 172.21.15.86:0/2044638238 learned_addr learned my addr 172.21.15.86:0/2044638238 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:27.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.111+0000 7fc411800640 1 -- 172.21.15.86:0/2044638238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c15acb0 msgr2=0x7fc40c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:27.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.111+0000 7fc411800640 1 --2- 172.21.15.86:0/2044638238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c15acb0 0x7fc40c0b37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.111+0000 7fc411800640 1 -- 172.21.15.86:0/2044638238 >> v1:172.21.15.194:6789/0 conn(0x7fc40c0b4680 legacy=0x7fc40c06e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:27.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.111+0000 7fc411800640 1 -- 172.21.15.86:0/2044638238 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc40c0b2cf0 con 0x7fc40c06ec00 2024-03-06T07:37:27.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.113+0000 7fc411800640 1 --2- 172.21.15.86:0/2044638238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c06ec00 0x7fc40c0ccfb0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fc404009e50 tx=0x7fc40402bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c156ea0fdce8edc9 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:27.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.113+0000 7fc40b400640 1 -- 172.21.15.86:0/2044638238 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc40402cb00 con 0x7fc40c06ec00 2024-03-06T07:37:27.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.113+0000 7fc40b400640 1 -- 172.21.15.86:0/2044638238 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc40402cca0 con 0x7fc40c06ec00 2024-03-06T07:37:27.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.113+0000 7fc40b400640 1 -- 172.21.15.86:0/2044638238 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc404034610 con 0x7fc40c06ec00 2024-03-06T07:37:27.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.114+0000 7fc413800640 1 -- 172.21.15.86:0/2044638238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c06ec00 msgr2=0x7fc40c0ccfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:27.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.114+0000 7fc413800640 1 --2- 172.21.15.86:0/2044638238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c06ec00 0x7fc40c0ccfb0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fc404009e50 tx=0x7fc40402bf60 comp rx=0 tx=0).stop 2024-03-06T07:37:27.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.114+0000 7fc413800640 1 -- 172.21.15.86:0/2044638238 shutdown_connections 2024-03-06T07:37:27.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.114+0000 7fc413800640 1 --2- 172.21.15.86:0/2044638238 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c06ec00 0x7fc40c0ccfb0 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.114+0000 7fc413800640 1 --2- 172.21.15.86:0/2044638238 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c15acb0 0x7fc40c0b37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.114+0000 7fc413800640 1 -- 172.21.15.86:0/2044638238 >> 172.21.15.86:0/2044638238 conn(0x7fc40c0b1730 msgr2=0x7fc40c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:27.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.115+0000 7fc413800640 1 -- 172.21.15.86:0/2044638238 shutdown_connections 2024-03-06T07:37:27.118 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.115+0000 7fc413800640 1 -- 172.21.15.86:0/2044638238 wait complete. 2024-03-06T07:37:27.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.118+0000 7fc413800640 1 Processor -- start 2024-03-06T07:37:27.121 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.118+0000 7fc413800640 1 -- start start 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc413800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c06ec00 0x7fc40c15ba60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc413800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c0b4680 0x7fc40c155940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc413800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c15acb0 0x7fc40c1f4100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc413800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc40c0b3d10 con 0x7fc40c0b4680 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc413800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc40c0b32b0 con 0x7fc40c15acb0 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc413800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc40c0b4010 con 0x7fc40c06ec00 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc40be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c0b4680 0x7fc40c155940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc40be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c0b4680 0x7fc40c155940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38752/0 (socket says 172.21.15.86:38752) 2024-03-06T07:37:27.123 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.120+0000 7fc40be00640 1 -- 172.21.15.86:0/2185709974 learned_addr learned my addr 172.21.15.86:0/2185709974 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc411800640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c15acb0 0x7fc40c1f4100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc410e00640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c06ec00 0x7fc40c15ba60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc40be00640 1 -- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c06ec00 msgr2=0x7fc40c15ba60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc40be00640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c06ec00 0x7fc40c15ba60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc40be00640 1 -- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c15acb0 msgr2=0x7fc40c1f4100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc40be00640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c15acb0 0x7fc40c1f4100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc40be00640 1 -- 172.21.15.86:0/2185709974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc40c15cae0 con 0x7fc40c0b4680 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.121+0000 7fc40be00640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c0b4680 0x7fc40c155940 secure :-1 s=READY pgs=1737 cs=0 l=1 rev1=1 crypto rx=0x7fc3f800c400 tx=0x7fc3f800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.122+0000 7fc408c00640 1 -- 172.21.15.86:0/2185709974 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f80052d0 con 0x7fc40c0b4680 2024-03-06T07:37:27.125 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.122+0000 7fc408c00640 1 -- 172.21.15.86:0/2185709974 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3f8005490 con 0x7fc40c0b4680 2024-03-06T07:37:27.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.122+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc40c15cdc0 con 0x7fc40c0b4680 2024-03-06T07:37:27.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.122+0000 7fc408c00640 1 -- 172.21.15.86:0/2185709974 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f8007260 con 0x7fc40c0b4680 2024-03-06T07:37:27.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.122+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc40c15b540 con 0x7fc40c0b4680 2024-03-06T07:37:27.126 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.124+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc40c04f330 con 0x7fc40c0b4680 2024-03-06T07:37:27.127 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.124+0000 7fc408c00640 1 -- 172.21.15.86:0/2185709974 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc3f8007400 con 0x7fc40c0b4680 2024-03-06T07:37:27.128 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.126+0000 7fc408c00640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:27.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.126+0000 7fc408c00640 1 -- 172.21.15.86:0/2185709974 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc3f8089700 con 0x7fc40c0b4680 2024-03-06T07:37:27.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.126+0000 7fc410e00640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:27.129 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.127+0000 7fc410e00640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 secure :-1 s=READY pgs=738 cs=0 l=1 rev1=1 crypto rx=0x7fc40c072a80 tx=0x7fc400008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:27.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.130+0000 7fc408c00640 1 -- 172.21.15.86:0/2185709974 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc3f80585d0 con 0x7fc40c0b4680 2024-03-06T07:37:27.369 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.366+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc40c06ce60 con 0x7fc40c0b4680 2024-03-06T07:37:27.370 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.368+0000 7fc408c00640 1 -- 172.21.15.86:0/2185709974 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fc3f8057ed0 con 0x7fc40c0b4680 2024-03-06T07:37:27.371 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:27.371 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:27.371 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:27.373 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3ec06b4d0 msgr2=0x7fc3ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 secure :-1 s=READY pgs=738 cs=0 l=1 rev1=1 crypto rx=0x7fc40c072a80 tx=0x7fc400008040 comp rx=0 tx=0).stop 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c0b4680 msgr2=0x7fc40c155940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c0b4680 0x7fc40c155940 secure :-1 s=READY pgs=1737 cs=0 l=1 rev1=1 crypto rx=0x7fc3f800c400 tx=0x7fc3f800c8c0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 shutdown_connections 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 unknown :-1 s=CLOSED pgs=738 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc40c15acb0 0x7fc40c1f4100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.370+0000 7fc413800640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc40c0b4680 0x7fc40c155940 unknown :-1 s=CLOSED pgs=1737 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.371+0000 7fc413800640 1 --2- 172.21.15.86:0/2185709974 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc40c06ec00 0x7fc40c15ba60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.371+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 >> 172.21.15.86:0/2185709974 conn(0x7fc40c0b1730 msgr2=0x7fc40c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.371+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 shutdown_connections 2024-03-06T07:37:27.374 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:27.371+0000 7fc413800640 1 -- 172.21.15.86:0/2185709974 wait complete. 2024-03-06T07:37:27.592 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:27.592 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:27.658 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:28.593 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:28.833 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.831+0000 7f604ae00640 1 Processor -- start 2024-03-06T07:37:28.834 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.831+0000 7f604ae00640 1 -- start start 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f604ae00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f604415f9d0 0x7f604415fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f604ae00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f604406cef0 con 0x7f6044144c00 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f604ae00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f604406d1d0 con 0x7f6044070b10 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f604ae00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f604406d4b0 con 0x7f604415f9d0 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f6043400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f604415f9d0 0x7f604415fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f6043400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f604415f9d0 0x7f604415fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38768/0 (socket says 172.21.15.86:38768) 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f6043400640 1 -- 172.21.15.86:0/1024279702 learned_addr learned my addr 172.21.15.86:0/1024279702 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:28.836 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.833+0000 7f6043e00640 1 --1- 172.21.15.86:0/1024279702 >> v1:172.21.15.86:6789/0 conn(0x7f6044144c00 0x7f604406e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57704/0 (socket says 172.21.15.86:57704) 2024-03-06T07:37:28.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.834+0000 7f6043400640 1 -- 172.21.15.86:0/1024279702 >> v1:172.21.15.194:6789/0 conn(0x7f6044070b10 legacy=0x7f6044070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:28.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.834+0000 7f6043400640 1 -- 172.21.15.86:0/1024279702 >> v1:172.21.15.86:6789/0 conn(0x7f6044144c00 legacy=0x7f604406e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:28.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.834+0000 7f6043400640 1 -- 172.21.15.86:0/1024279702 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f604406d790 con 0x7f604415f9d0 2024-03-06T07:37:28.837 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.835+0000 7f6043400640 1 --2- 172.21.15.86:0/1024279702 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f604415f9d0 0x7f604415fe60 secure :-1 s=READY pgs=1739 cs=0 l=1 rev1=1 crypto rx=0x7f602c009870 tx=0x7f602c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=21d1569e65d689a8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:28.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.835+0000 7f6042a00640 1 -- 172.21.15.86:0/1024279702 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f602c034070 con 0x7f604415f9d0 2024-03-06T07:37:28.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.835+0000 7f6042a00640 1 -- 172.21.15.86:0/1024279702 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f602c030860 con 0x7f604415f9d0 2024-03-06T07:37:28.838 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.835+0000 7f6042a00640 1 -- 172.21.15.86:0/1024279702 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f602c030c30 con 0x7f604415f9d0 2024-03-06T07:37:28.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.836+0000 7f604ae00640 1 -- 172.21.15.86:0/1024279702 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f604415f9d0 msgr2=0x7f604415fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:28.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.836+0000 7f604ae00640 1 --2- 172.21.15.86:0/1024279702 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f604415f9d0 0x7f604415fe60 secure :-1 s=READY pgs=1739 cs=0 l=1 rev1=1 crypto rx=0x7f602c009870 tx=0x7f602c02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:37:28.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.836+0000 7f604ae00640 1 -- 172.21.15.86:0/1024279702 shutdown_connections 2024-03-06T07:37:28.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.836+0000 7f604ae00640 1 --2- 172.21.15.86:0/1024279702 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f604415f9d0 0x7f604415fe60 unknown :-1 s=CLOSED pgs=1739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:28.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.836+0000 7f604ae00640 1 -- 172.21.15.86:0/1024279702 >> 172.21.15.86:0/1024279702 conn(0x7f60440b1730 msgr2=0x7f60440b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:28.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.836+0000 7f604ae00640 1 -- 172.21.15.86:0/1024279702 shutdown_connections 2024-03-06T07:37:28.839 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.836+0000 7f604ae00640 1 -- 172.21.15.86:0/1024279702 wait complete. 2024-03-06T07:37:28.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.838+0000 7f604ae00640 1 Processor -- start 2024-03-06T07:37:28.840 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.838+0000 7f604ae00640 1 -- start start 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f604ae00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6044070b10 0x7f604413d180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f604ae00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6044144c00 0x7f604413ef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f604ae00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f604415f9d0 0x7f604413f4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f604ae00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f604406ccd0 con 0x7f6044070b10 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f604ae00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f604406cb50 con 0x7f604415f9d0 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f604ae00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f604406df90 con 0x7f6044144c00 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f6043e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6044070b10 0x7f604413d180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f6043e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6044070b10 0x7f604413d180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38782/0 (socket says 172.21.15.86:38782) 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f6043e00640 1 -- 172.21.15.86:0/4253578113 learned_addr learned my addr 172.21.15.86:0/4253578113 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f6043400640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6044144c00 0x7f604413ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f6043e00640 1 -- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6044144c00 msgr2=0x7f604413ef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:28.843 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.839+0000 7f6048e00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f604415f9d0 0x7f604413f4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f6043e00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6044144c00 0x7f604413ef70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f6043e00640 1 -- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f604415f9d0 msgr2=0x7f604413f4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f6043e00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f604415f9d0 0x7f604413f4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f6043e00640 1 -- 172.21.15.86:0/4253578113 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f604413fb30 con 0x7f6044070b10 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f6048e00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f604415f9d0 0x7f604413f4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f6043e00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6044070b10 0x7f604413d180 secure :-1 s=READY pgs=1740 cs=0 l=1 rev1=1 crypto rx=0x7f603400c790 tx=0x7f603400cc50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f603be00640 1 -- 172.21.15.86:0/4253578113 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6034017070 con 0x7f6044070b10 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f603be00640 1 -- 172.21.15.86:0/4253578113 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f603400a640 con 0x7f6044070b10 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f603be00640 1 -- 172.21.15.86:0/4253578113 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f603400a9f0 con 0x7f6044070b10 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f604413fe10 con 0x7f6044070b10 2024-03-06T07:37:28.844 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.840+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f604415d5e0 con 0x7f6044070b10 2024-03-06T07:37:28.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.842+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f604415d7e0 con 0x7f6044070b10 2024-03-06T07:37:28.845 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.843+0000 7f603be00640 1 -- 172.21.15.86:0/4253578113 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f603401a070 con 0x7f6044070b10 2024-03-06T07:37:28.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.843+0000 7f603be00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f602006b4d0 0x7f602006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:28.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.844+0000 7f603be00640 1 -- 172.21.15.86:0/4253578113 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6034089810 con 0x7f6044070b10 2024-03-06T07:37:28.846 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.844+0000 7f6043400640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f602006b4d0 0x7f602006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:28.847 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.845+0000 7f6043400640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f602006b4d0 0x7f602006d990 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f602c036000 tx=0x7f602c03b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:28.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:28.849+0000 7f603be00640 1 -- 172.21.15.86:0/4253578113 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6034058000 con 0x7f6044070b10 2024-03-06T07:37:29.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.086+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6044072890 con 0x7f6044070b10 2024-03-06T07:37:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.087+0000 7f603be00640 1 -- 172.21.15.86:0/4253578113 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f603405c0a0 con 0x7f6044070b10 2024-03-06T07:37:29.090 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:29.090 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:29.090 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f602006b4d0 msgr2=0x7f602006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f602006b4d0 0x7f602006d990 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f602c036000 tx=0x7f602c03b040 comp rx=0 tx=0).stop 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6044070b10 msgr2=0x7f604413d180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6044070b10 0x7f604413d180 secure :-1 s=READY pgs=1740 cs=0 l=1 rev1=1 crypto rx=0x7f603400c790 tx=0x7f603400cc50 comp rx=0 tx=0).stop 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 shutdown_connections 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f602006b4d0 0x7f602006d990 unknown :-1 s=CLOSED pgs=739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f604415f9d0 0x7f604413f4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6044144c00 0x7f604413ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 --2- 172.21.15.86:0/4253578113 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6044070b10 0x7f604413d180 unknown :-1 s=CLOSED pgs=1740 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 >> 172.21.15.86:0/4253578113 conn(0x7f60440b1730 msgr2=0x7f6044145040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 shutdown_connections 2024-03-06T07:37:29.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:29.090+0000 7f604ae00640 1 -- 172.21.15.86:0/4253578113 wait complete. 2024-03-06T07:37:29.311 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:29.311 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:29.660 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:30.312 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:30.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.520+0000 7fd458400640 1 Processor -- start 2024-03-06T07:37:30.523 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.520+0000 7fd458400640 1 -- start start 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd458400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd450144c00 0x7fd45006e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd458400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd45006ee70 con 0x7fd45015fe60 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd458400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd45006cef0 con 0x7fd45015f9d0 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd458400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd45006d1d0 con 0x7fd450144c00 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd455a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd450144c00 0x7fd45006e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd456400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd45015fe60 0x7fd4500ce7b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57720/0 (socket says 172.21.15.86:57720) 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd455a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd450144c00 0x7fd45006e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38786/0 (socket says 172.21.15.86:38786) 2024-03-06T07:37:30.525 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.522+0000 7fd456400640 1 -- 172.21.15.86:0/3188290965 learned_addr learned my addr 172.21.15.86:0/3188290965 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:30.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.523+0000 7fd455a00640 1 -- 172.21.15.86:0/3188290965 >> v1:172.21.15.194:6789/0 conn(0x7fd45015f9d0 legacy=0x7fd4500ce0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:30.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.523+0000 7fd44fe00640 1 -- 172.21.15.86:0/3188290965 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 337041922 0 0) 0x7fd448001110 con 0x7fd45015fe60 2024-03-06T07:37:30.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.523+0000 7fd455a00640 1 -- 172.21.15.86:0/3188290965 >> v1:172.21.15.86:6789/0 conn(0x7fd45015fe60 legacy=0x7fd4500ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:30.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.523+0000 7fd455a00640 1 -- 172.21.15.86:0/3188290965 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd45006d4b0 con 0x7fd450144c00 2024-03-06T07:37:30.526 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.524+0000 7fd455a00640 1 --2- 172.21.15.86:0/3188290965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd450144c00 0x7fd45006e930 secure :-1 s=READY pgs=1742 cs=0 l=1 rev1=1 crypto rx=0x7fd444009d90 tx=0x7fd44402beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=42c2b06bec8f4ca6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:30.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.524+0000 7fd44fe00640 1 -- 172.21.15.86:0/3188290965 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd444039070 con 0x7fd450144c00 2024-03-06T07:37:30.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.524+0000 7fd44fe00640 1 -- 172.21.15.86:0/3188290965 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd44402cc60 con 0x7fd450144c00 2024-03-06T07:37:30.527 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.524+0000 7fd44fe00640 1 -- 172.21.15.86:0/3188290965 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd444034660 con 0x7fd450144c00 2024-03-06T07:37:30.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.525+0000 7fd458400640 1 -- 172.21.15.86:0/3188290965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd450144c00 msgr2=0x7fd45006e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:30.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.525+0000 7fd458400640 1 --2- 172.21.15.86:0/3188290965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd450144c00 0x7fd45006e930 secure :-1 s=READY pgs=1742 cs=0 l=1 rev1=1 crypto rx=0x7fd444009d90 tx=0x7fd44402beb0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.525+0000 7fd458400640 1 -- 172.21.15.86:0/3188290965 shutdown_connections 2024-03-06T07:37:30.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.525+0000 7fd458400640 1 --2- 172.21.15.86:0/3188290965 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd450144c00 0x7fd45006e930 unknown :-1 s=CLOSED pgs=1742 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.525+0000 7fd458400640 1 -- 172.21.15.86:0/3188290965 >> 172.21.15.86:0/3188290965 conn(0x7fd4500b1730 msgr2=0x7fd4500b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:30.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.525+0000 7fd458400640 1 -- 172.21.15.86:0/3188290965 shutdown_connections 2024-03-06T07:37:30.528 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.525+0000 7fd458400640 1 -- 172.21.15.86:0/3188290965 wait complete. 2024-03-06T07:37:30.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.527+0000 7fd458400640 1 Processor -- start 2024-03-06T07:37:30.529 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.527+0000 7fd458400640 1 -- start start 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd458400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd450144c00 0x7fd45015eb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd458400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd45015f9d0 0x7fd45013a7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd458400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd45015fe60 0x7fd4501f7090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd458400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd450160730 con 0x7fd45015f9d0 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd458400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd450160270 con 0x7fd45015fe60 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd458400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd4500ceec0 con 0x7fd450144c00 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd455000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd45015f9d0 0x7fd45013a7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd455000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd45015f9d0 0x7fd45013a7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38802/0 (socket says 172.21.15.86:38802) 2024-03-06T07:37:30.531 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.528+0000 7fd455000640 1 -- 172.21.15.86:0/1274579740 learned_addr learned my addr 172.21.15.86:0/1274579740 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:30.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd456400640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd45015fe60 0x7fd4501f7090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:30.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd455a00640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd450144c00 0x7fd45015eb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:30.532 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd455000640 1 -- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd450144c00 msgr2=0x7fd45015eb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd455000640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd450144c00 0x7fd45015eb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd455000640 1 -- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd45015fe60 msgr2=0x7fd4501f7090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd455000640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd45015fe60 0x7fd4501f7090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd455000640 1 -- 172.21.15.86:0/1274579740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd45013c3a0 con 0x7fd45015f9d0 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd455000640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd45015f9d0 0x7fd45013a7a0 secure :-1 s=READY pgs=1743 cs=0 l=1 rev1=1 crypto rx=0x7fd43c00c430 tx=0x7fd43c00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd44d600640 1 -- 172.21.15.86:0/1274579740 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd43c011790 con 0x7fd45015f9d0 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd44d600640 1 -- 172.21.15.86:0/1274579740 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd43c017070 con 0x7fd45015f9d0 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4500b2cd0 con 0x7fd45015f9d0 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd44d600640 1 -- 172.21.15.86:0/1274579740 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd43c011bb0 con 0x7fd45015f9d0 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.529+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd45013c900 con 0x7fd45015f9d0 2024-03-06T07:37:30.533 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.531+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd45004f330 con 0x7fd45015f9d0 2024-03-06T07:37:30.535 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.533+0000 7fd44d600640 1 -- 172.21.15.86:0/1274579740 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd43c01c460 con 0x7fd45015f9d0 2024-03-06T07:37:30.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.533+0000 7fd44d600640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd43006b4d0 0x7fd43006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:30.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.533+0000 7fd44d600640 1 -- 172.21.15.86:0/1274579740 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd43c08a230 con 0x7fd45015f9d0 2024-03-06T07:37:30.536 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.534+0000 7fd455a00640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd43006b4d0 0x7fd43006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:30.537 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.534+0000 7fd455a00640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd43006b4d0 0x7fd43006d990 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7fd444036000 tx=0x7fd444030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:30.541 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.538+0000 7fd44d600640 1 -- 172.21.15.86:0/1274579740 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd45013c3a0 con 0x7fd45015f9d0 2024-03-06T07:37:30.776 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.774+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd450074be0 con 0x7fd45015f9d0 2024-03-06T07:37:30.777 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.775+0000 7fd44d600640 1 -- 172.21.15.86:0/1274579740 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fd43c058fd0 con 0x7fd45015f9d0 2024-03-06T07:37:30.778 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:30.778 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:30.778 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.777+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd43006b4d0 msgr2=0x7fd43006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.777+0000 7fd458400640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd43006b4d0 0x7fd43006d990 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7fd444036000 tx=0x7fd444030000 comp rx=0 tx=0).stop 2024-03-06T07:37:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.777+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd45015f9d0 msgr2=0x7fd45013a7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.777+0000 7fd458400640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd45015f9d0 0x7fd45013a7a0 secure :-1 s=READY pgs=1743 cs=0 l=1 rev1=1 crypto rx=0x7fd43c00c430 tx=0x7fd43c00c8f0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.780 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.777+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 shutdown_connections 2024-03-06T07:37:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.777+0000 7fd458400640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd43006b4d0 0x7fd43006d990 unknown :-1 s=CLOSED pgs=740 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.778+0000 7fd458400640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd45015fe60 0x7fd4501f7090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.778+0000 7fd458400640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd45015f9d0 0x7fd45013a7a0 unknown :-1 s=CLOSED pgs=1743 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.778+0000 7fd458400640 1 --2- 172.21.15.86:0/1274579740 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd450144c00 0x7fd45015eb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.778+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 >> 172.21.15.86:0/1274579740 conn(0x7fd4500b1730 msgr2=0x7fd4501452e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.778+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 shutdown_connections 2024-03-06T07:37:30.781 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:30.778+0000 7fd458400640 1 -- 172.21.15.86:0/1274579740 wait complete. 2024-03-06T07:37:31.001 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:31.001 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:31.662 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:32.001 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:32.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.223+0000 7f8958200640 1 Processor -- start 2024-03-06T07:37:32.226 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.223+0000 7f8958200640 1 -- start start 2024-03-06T07:37:32.227 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.225+0000 7f8958200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f895015f470 0x7f89500d08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.225+0000 7f8958200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 0x7f8950160dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.225+0000 7f8958200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f89500cea60 con 0x7f895015e0d0 2024-03-06T07:37:32.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.225+0000 7f8958200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f89500d2f60 con 0x7f895015f470 2024-03-06T07:37:32.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.225+0000 7f8958200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89500d1240 con 0x7f89500d0df0 2024-03-06T07:37:32.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.225+0000 7f8955800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f895015e0d0 0x7f89500d1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:57730/0 (socket says 172.21.15.86:57730) 2024-03-06T07:37:32.228 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.225+0000 7f8955800640 1 -- 172.21.15.86:0/2409457207 learned_addr learned my addr 172.21.15.86:0/2409457207 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f8954e00640 1 --2- 172.21.15.86:0/2409457207 >> v2:172.21.15.194:3300/0 conn(0x7f895015f470 0x7f89500d08b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f8956200640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 0x7f8950160dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1324981273 0 0) 0x7f89500cea60 con 0x7f895015e0d0 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8938003520 con 0x7f895015e0d0 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3937134208 0 0) 0x7f8938003520 con 0x7f895015e0d0 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 msgr2=0x7f8950160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 0x7f8950160dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 >> v2:172.21.15.194:3300/0 conn(0x7f895015f470 msgr2=0x7f89500d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 --2- 172.21.15.86:0/2409457207 >> v2:172.21.15.194:3300/0 conn(0x7f895015f470 0x7f89500d08b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89500d2350 con 0x7f895015e0d0 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f8944001670 con 0x7f895015e0d0 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f8956200640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 0x7f8950160dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:37:32.230 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 >> v1:172.21.15.86:6789/0 conn(0x7f895015e0d0 legacy=0x7f89500d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f8938003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 0x7f8938004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8938004d40 0x7f89380051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89500d1240 con 0x7f89380039b0 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f893800ba50 con 0x7f8938004d40 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f893800bc00 con 0x7f89500d0df0 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.226+0000 7f8955800640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f8938003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.227+0000 7f8954e00640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 0x7f8938004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.227+0000 7f8955800640 1 -- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 msgr2=0x7f8938004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.227+0000 7f8955800640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f89500d0df0 0x7f8938004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.227+0000 7f8955800640 1 -- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8938004d40 msgr2=0x7f89380051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.227+0000 7f8955800640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8938004d40 0x7f89380051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.231 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.227+0000 7f8955800640 1 -- 172.21.15.86:0/2409457207 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8938003520 con 0x7f89380039b0 2024-03-06T07:37:32.232 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.228+0000 7f8955800640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f8938003dd0 secure :-1 s=READY pgs=1745 cs=0 l=1 rev1=1 crypto rx=0x7f894400c190 tx=0x7f8938005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ef81fb74bf4628d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.228+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8944007c80 con 0x7f89380039b0 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.228+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8944030070 con 0x7f89380039b0 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.228+0000 7f894fe00640 1 -- 172.21.15.86:0/2409457207 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8944005d30 con 0x7f89380039b0 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.229+0000 7f8958200640 1 -- 172.21.15.86:0/2409457207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 msgr2=0x7f8938003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.229+0000 7f8958200640 1 --2- 172.21.15.86:0/2409457207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f8938003dd0 secure :-1 s=READY pgs=1745 cs=0 l=1 rev1=1 crypto rx=0x7f894400c190 tx=0x7f8938005830 comp rx=0 tx=0).stop 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.229+0000 7f8956200640 1 -- 172.21.15.86:0/2409457207 reap_dead start 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.229+0000 7f8958200640 1 -- 172.21.15.86:0/2409457207 shutdown_connections 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.229+0000 7f8958200640 1 -- 172.21.15.86:0/2409457207 >> 172.21.15.86:0/2409457207 conn(0x7f89500d3ec0 msgr2=0x7f89500d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.229+0000 7f8958200640 1 -- 172.21.15.86:0/2409457207 shutdown_connections 2024-03-06T07:37:32.233 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.229+0000 7f8958200640 1 -- 172.21.15.86:0/2409457207 wait complete. 2024-03-06T07:37:32.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.232+0000 7f8958200640 1 Processor -- start 2024-03-06T07:37:32.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.232+0000 7f8958200640 1 -- start start 2024-03-06T07:37:32.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.233+0000 7f8958200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f89500b9360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.233+0000 7f8958200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8950155260 0x7f89500cd610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.233+0000 7f8958200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8950154d10 0x7f89500cdb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.233+0000 7f8958200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89500d2bf0 con 0x7f89380039b0 2024-03-06T07:37:32.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.233+0000 7f8958200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89500d2910 con 0x7f8950154d10 2024-03-06T07:37:32.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.233+0000 7f8958200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f895015fb30 con 0x7f8950155260 2024-03-06T07:37:32.236 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f89500b9360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:32.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f89500b9360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38812/0 (socket says 172.21.15.86:38812) 2024-03-06T07:37:32.237 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 -- 172.21.15.86:0/1453305131 learned_addr learned my addr 172.21.15.86:0/1453305131 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8954e00640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8950155260 0x7f89500cd610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 -- 172.21.15.86:0/1453305131 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8950155260 msgr2=0x7f89500cd610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8950155260 0x7f89500cd610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 -- 172.21.15.86:0/1453305131 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8950154d10 msgr2=0x7f89500cdb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8950154d10 0x7f89500cdb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 -- 172.21.15.86:0/1453305131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89501559e0 con 0x7f89380039b0 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.234+0000 7f8955800640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f89500b9360 secure :-1 s=READY pgs=1746 cs=0 l=1 rev1=1 crypto rx=0x7f894400c2b0 tx=0x7f89440019e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.235+0000 7f894d600640 1 -- 172.21.15.86:0/1453305131 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8944046070 con 0x7f89380039b0 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.235+0000 7f894d600640 1 -- 172.21.15.86:0/1453305131 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8944004bf0 con 0x7f89380039b0 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.235+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f895015c1f0 con 0x7f89380039b0 2024-03-06T07:37:32.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.235+0000 7f894d600640 1 -- 172.21.15.86:0/1453305131 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8944001bc0 con 0x7f89380039b0 2024-03-06T07:37:32.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.235+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89500cbdb0 con 0x7f89380039b0 2024-03-06T07:37:32.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.236+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f895004f330 con 0x7f89380039b0 2024-03-06T07:37:32.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.237+0000 7f894d600640 1 -- 172.21.15.86:0/1453305131 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8944007070 con 0x7f89380039b0 2024-03-06T07:37:32.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.238+0000 7f894d600640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f893006b500 0x7f893006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:32.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.238+0000 7f894d600640 1 -- 172.21.15.86:0/1453305131 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f89440ab840 con 0x7f89380039b0 2024-03-06T07:37:32.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.238+0000 7f8954e00640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f893006b500 0x7f893006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:32.241 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.239+0000 7f8954e00640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f893006b500 0x7f893006d9c0 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f89500b7620 tx=0x7f893c008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:32.245 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.243+0000 7f894d600640 1 -- 172.21.15.86:0/1453305131 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f894407a670 con 0x7f89380039b0 2024-03-06T07:37:32.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.576+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f895008d920 con 0x7f89380039b0 2024-03-06T07:37:32.579 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.576+0000 7f894d600640 1 -- 172.21.15.86:0/1453305131 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f8944079f70 con 0x7f89380039b0 2024-03-06T07:37:32.579 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:32.579 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:32.579 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:32.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f893006b500 msgr2=0x7f893006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:32.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f893006b500 0x7f893006d9c0 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f89500b7620 tx=0x7f893c008040 comp rx=0 tx=0).stop 2024-03-06T07:37:32.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 msgr2=0x7f89500b9360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:32.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f89500b9360 secure :-1 s=READY pgs=1746 cs=0 l=1 rev1=1 crypto rx=0x7f894400c2b0 tx=0x7f89440019e0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 shutdown_connections 2024-03-06T07:37:32.582 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f893006b500 0x7f893006d9c0 unknown :-1 s=CLOSED pgs=741 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8950154d10 0x7f89500cdb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8950155260 0x7f89500cd610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 --2- 172.21.15.86:0/1453305131 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f89380039b0 0x7f89500b9360 unknown :-1 s=CLOSED pgs=1746 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:32.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.579+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 >> 172.21.15.86:0/1453305131 conn(0x7f89500d3ec0 msgr2=0x7f89500b3700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:32.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.580+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 shutdown_connections 2024-03-06T07:37:32.583 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:32.580+0000 7f8958200640 1 -- 172.21.15.86:0/1453305131 wait complete. 2024-03-06T07:37:32.802 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:32.802 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:33.665 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:33.803 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:34.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.012+0000 7ff5e5200640 1 Processor -- start 2024-03-06T07:37:34.015 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.013+0000 7ff5e5200640 1 -- start start 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.014+0000 7ff5e5200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5e015dcd0 0x7ff5e00b5860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.014+0000 7ff5e5200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7ff5e015e0b0 0x7ff5e006f290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.014+0000 7ff5e5200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff5e006e780 con 0x7ff5e00b39d0 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.014+0000 7ff5e5200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff5e006d770 con 0x7ff5e015dcd0 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.014+0000 7ff5e5200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff5e006ee50 con 0x7ff5e015e0b0 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.014+0000 7ff5df400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5e015dcd0 0x7ff5e00b5860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5df400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff5e015dcd0 0x7ff5e00b5860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38824/0 (socket says 172.21.15.86:38824) 2024-03-06T07:37:34.017 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5df400640 1 -- 172.21.15.86:0/699450776 learned_addr learned my addr 172.21.15.86:0/699450776 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:34.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5dea00640 1 --2- 172.21.15.86:0/699450776 >> v2:172.21.15.194:3300/0 conn(0x7ff5e015e0b0 0x7ff5e006f290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:34.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5df400640 1 -- 172.21.15.86:0/699450776 >> v2:172.21.15.194:3300/0 conn(0x7ff5e015e0b0 msgr2=0x7ff5e006f290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:34.018 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5de000640 1 -- 172.21.15.86:0/699450776 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2360811177 0 0) 0x7ff5e006e780 con 0x7ff5e00b39d0 2024-03-06T07:37:34.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5df400640 1 --2- 172.21.15.86:0/699450776 >> v2:172.21.15.194:3300/0 conn(0x7ff5e015e0b0 0x7ff5e006f290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5df400640 1 -- 172.21.15.86:0/699450776 >> v1:172.21.15.86:6789/0 conn(0x7ff5e00b39d0 legacy=0x7ff5e0161b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:34.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5df400640 1 -- 172.21.15.86:0/699450776 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5e006d9c0 con 0x7ff5e015dcd0 2024-03-06T07:37:34.019 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.015+0000 7ff5dea00640 1 --2- 172.21.15.86:0/699450776 >> v2:172.21.15.194:3300/0 conn(0x7ff5e015e0b0 0x7ff5e006f290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:34.021 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.019+0000 7ff5df400640 1 --2- 172.21.15.86:0/699450776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015dcd0 0x7ff5e00b5860 secure :-1 s=READY pgs=1748 cs=0 l=1 rev1=1 crypto rx=0x7ff5e006bd40 tx=0x7ff5d002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=856a65ca82da606e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:34.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.019+0000 7ff5de000640 1 -- 172.21.15.86:0/699450776 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5d0035020 con 0x7ff5e015dcd0 2024-03-06T07:37:34.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.019+0000 7ff5de000640 1 -- 172.21.15.86:0/699450776 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5d0031870 con 0x7ff5e015dcd0 2024-03-06T07:37:34.022 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.019+0000 7ff5de000640 1 -- 172.21.15.86:0/699450776 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5d0031c40 con 0x7ff5e015dcd0 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 -- 172.21.15.86:0/699450776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015dcd0 msgr2=0x7ff5e00b5860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 --2- 172.21.15.86:0/699450776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015dcd0 0x7ff5e00b5860 secure :-1 s=READY pgs=1748 cs=0 l=1 rev1=1 crypto rx=0x7ff5e006bd40 tx=0x7ff5d002b910 comp rx=0 tx=0).stop 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 -- 172.21.15.86:0/699450776 shutdown_connections 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 --2- 172.21.15.86:0/699450776 >> v2:172.21.15.194:3300/0 conn(0x7ff5e015e0b0 0x7ff5e006f290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 --2- 172.21.15.86:0/699450776 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015dcd0 0x7ff5e00b5860 unknown :-1 s=CLOSED pgs=1748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 -- 172.21.15.86:0/699450776 >> 172.21.15.86:0/699450776 conn(0x7ff5e006c0d0 msgr2=0x7ff5e006c4e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 -- 172.21.15.86:0/699450776 shutdown_connections 2024-03-06T07:37:34.023 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.020+0000 7ff5e5200640 1 -- 172.21.15.86:0/699450776 wait complete. 2024-03-06T07:37:34.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.022+0000 7ff5e5200640 1 Processor -- start 2024-03-06T07:37:34.024 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.022+0000 7ff5e5200640 1 -- start start 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5e5200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e00b39d0 0x7ff5e0148730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5e5200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015e0b0 0x7ff5e00b1140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5e5200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e00b1680 0x7ff5e00d04d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5e5200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5e006e260 con 0x7ff5e015e0b0 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5e5200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5e006df80 con 0x7ff5e00b39d0 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5e5200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5e00d16f0 con 0x7ff5e00b1680 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015e0b0 0x7ff5e00b1140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:34.027 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5dea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015e0b0 0x7ff5e00b1140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38826/0 (socket says 172.21.15.86:38826) 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.023+0000 7ff5dea00640 1 -- 172.21.15.86:0/3489588308 learned_addr learned my addr 172.21.15.86:0/3489588308 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5df400640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e00b39d0 0x7ff5e0148730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5dfe00640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e00b1680 0x7ff5e00d04d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5dea00640 1 -- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e00b1680 msgr2=0x7ff5e00d04d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5dea00640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e00b1680 0x7ff5e00d04d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5dea00640 1 -- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e00b39d0 msgr2=0x7ff5e0148730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5dea00640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e00b39d0 0x7ff5e0148730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5dea00640 1 -- 172.21.15.86:0/3489588308 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5e014b0b0 con 0x7ff5e015e0b0 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5df400640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e00b39d0 0x7ff5e0148730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5dea00640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015e0b0 0x7ff5e00b1140 secure :-1 s=READY pgs=1749 cs=0 l=1 rev1=1 crypto rx=0x7ff5cc00c7f0 tx=0x7ff5cc00cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5d7400640 1 -- 172.21.15.86:0/3489588308 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5cc005b90 con 0x7ff5e015e0b0 2024-03-06T07:37:34.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5d7400640 1 -- 172.21.15.86:0/3489588308 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5cc005d30 con 0x7ff5e015e0b0 2024-03-06T07:37:34.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5d7400640 1 -- 172.21.15.86:0/3489588308 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5cc007a30 con 0x7ff5e015e0b0 2024-03-06T07:37:34.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.024+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff5e014b390 con 0x7ff5e015e0b0 2024-03-06T07:37:34.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.025+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff5e014b950 con 0x7ff5e015e0b0 2024-03-06T07:37:34.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.026+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff5e004f330 con 0x7ff5e015e0b0 2024-03-06T07:37:34.029 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.027+0000 7ff5d7400640 1 -- 172.21.15.86:0/3489588308 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff5cc007bd0 con 0x7ff5e015e0b0 2024-03-06T07:37:34.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.027+0000 7ff5d7400640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5c006b4d0 0x7ff5c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:34.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.028+0000 7ff5d7400640 1 -- 172.21.15.86:0/3489588308 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff5cc08a340 con 0x7ff5e015e0b0 2024-03-06T07:37:34.030 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.028+0000 7ff5df400640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5c006b4d0 0x7ff5c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:34.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.029+0000 7ff5df400640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5c006b4d0 0x7ff5c006d990 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7ff5d0036000 tx=0x7ff5d0008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:34.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.032+0000 7ff5d7400640 1 -- 172.21.15.86:0/3489588308 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff5cc015020 con 0x7ff5e015e0b0 2024-03-06T07:37:34.273 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.271+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff5e00749e0 con 0x7ff5e015e0b0 2024-03-06T07:37:34.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.272+0000 7ff5d7400640 1 -- 172.21.15.86:0/3489588308 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7ff5cc059170 con 0x7ff5e015e0b0 2024-03-06T07:37:34.275 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:34.275 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:34.275 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5c006b4d0 msgr2=0x7ff5c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5c006b4d0 0x7ff5c006d990 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7ff5d0036000 tx=0x7ff5d0008030 comp rx=0 tx=0).stop 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015e0b0 msgr2=0x7ff5e00b1140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015e0b0 0x7ff5e00b1140 secure :-1 s=READY pgs=1749 cs=0 l=1 rev1=1 crypto rx=0x7ff5cc00c7f0 tx=0x7ff5cc00cbf0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 shutdown_connections 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff5c006b4d0 0x7ff5c006d990 unknown :-1 s=CLOSED pgs=742 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff5e00b1680 0x7ff5e00d04d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff5e015e0b0 0x7ff5e00b1140 unknown :-1 s=CLOSED pgs=1749 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.278 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 --2- 172.21.15.86:0/3489588308 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff5e00b39d0 0x7ff5e0148730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:34.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 >> 172.21.15.86:0/3489588308 conn(0x7ff5e006c0d0 msgr2=0x7ff5e00b5860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:34.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 shutdown_connections 2024-03-06T07:37:34.279 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:34.275+0000 7ff5e5200640 1 -- 172.21.15.86:0/3489588308 wait complete. 2024-03-06T07:37:34.499 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:34.499 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:35.500 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:35.667 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:35.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.710+0000 7f0dfe000640 1 Processor -- start 2024-03-06T07:37:35.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.710+0000 7f0dfe000640 1 -- start start 2024-03-06T07:37:35.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.712+0000 7f0dfe000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0df815eff0 0x7f0df8070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:35.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.712+0000 7f0dfe000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0df8071d30 con 0x7f0df806e4d0 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.712+0000 7f0dfe000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0df815f4a0 con 0x7f0df80af1c0 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.712+0000 7f0dfe000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0df8071260 con 0x7f0df815eff0 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.712+0000 7f0deea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0df815eff0 0x7f0df8070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.712+0000 7f0deea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f0df815eff0 0x7f0df8070d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38828/0 (socket says 172.21.15.86:38828) 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.712+0000 7f0deea00640 1 -- 172.21.15.86:0/237082310 learned_addr learned my addr 172.21.15.86:0/237082310 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.713+0000 7f0deea00640 1 -- 172.21.15.86:0/237082310 >> v1:172.21.15.194:6789/0 conn(0x7f0df80af1c0 legacy=0x7f0df806cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.713+0000 7f0deea00640 1 -- 172.21.15.86:0/237082310 >> v1:172.21.15.86:6789/0 conn(0x7f0df806e4d0 legacy=0x7f0df80b35d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.713+0000 7f0deea00640 1 -- 172.21.15.86:0/237082310 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0df8071490 con 0x7f0df815eff0 2024-03-06T07:37:35.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.714+0000 7f0deea00640 1 --2- 172.21.15.86:0/237082310 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df815eff0 0x7f0df8070d20 secure :-1 s=READY pgs=1751 cs=0 l=1 rev1=1 crypto rx=0x7f0de00098a0 tx=0x7f0de002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=10d4ed21d2e544bf server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.714+0000 7f0df6a00640 1 -- 172.21.15.86:0/237082310 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de0034070 con 0x7f0df815eff0 2024-03-06T07:37:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.714+0000 7f0df6a00640 1 -- 172.21.15.86:0/237082310 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0de0030840 con 0x7f0df815eff0 2024-03-06T07:37:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.714+0000 7f0df6a00640 1 -- 172.21.15.86:0/237082310 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de0030c10 con 0x7f0df815eff0 2024-03-06T07:37:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.715+0000 7f0dfe000640 1 -- 172.21.15.86:0/237082310 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df815eff0 msgr2=0x7f0df8070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:35.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.715+0000 7f0dfe000640 1 --2- 172.21.15.86:0/237082310 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df815eff0 0x7f0df8070d20 secure :-1 s=READY pgs=1751 cs=0 l=1 rev1=1 crypto rx=0x7f0de00098a0 tx=0x7f0de002b980 comp rx=0 tx=0).stop 2024-03-06T07:37:35.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.715+0000 7f0dfe000640 1 -- 172.21.15.86:0/237082310 shutdown_connections 2024-03-06T07:37:35.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.715+0000 7f0dfe000640 1 --2- 172.21.15.86:0/237082310 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df815eff0 0x7f0df8070d20 unknown :-1 s=CLOSED pgs=1751 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.715+0000 7f0dfe000640 1 -- 172.21.15.86:0/237082310 >> 172.21.15.86:0/237082310 conn(0x7f0df80b1730 msgr2=0x7f0df80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:35.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.715+0000 7f0dfe000640 1 -- 172.21.15.86:0/237082310 shutdown_connections 2024-03-06T07:37:35.718 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.715+0000 7f0dfe000640 1 -- 172.21.15.86:0/237082310 wait complete. 2024-03-06T07:37:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.717+0000 7f0dfe000640 1 Processor -- start 2024-03-06T07:37:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.718+0000 7f0dfe000640 1 -- start start 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0dfe000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0df806e4d0 0x7f0df81590d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0dfe000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df80af1c0 0x7f0df814b590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0dfe000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0df815eff0 0x7f0df815beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0dfe000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0df80b30b0 con 0x7f0df80af1c0 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0dfe000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0df8071a50 con 0x7f0df815eff0 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0dfe000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0df806d370 con 0x7f0df806e4d0 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0deea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df80af1c0 0x7f0df814b590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.719+0000 7f0deea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df80af1c0 0x7f0df814b590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38830/0 (socket says 172.21.15.86:38830) 2024-03-06T07:37:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0deea00640 1 -- 172.21.15.86:0/883525372 learned_addr learned my addr 172.21.15.86:0/883525372 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0df7e00640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0df815eff0 0x7f0df815beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0df7400640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0df806e4d0 0x7f0df81590d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0deea00640 1 -- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0df806e4d0 msgr2=0x7f0df81590d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0deea00640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0df806e4d0 0x7f0df81590d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0deea00640 1 -- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0df815eff0 msgr2=0x7f0df815beb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0deea00640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0df815eff0 0x7f0df815beb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0deea00640 1 -- 172.21.15.86:0/883525372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0df80ce490 con 0x7f0df80af1c0 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0df7e00640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0df815eff0 0x7f0df815beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.720+0000 7f0deea00640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df80af1c0 0x7f0df814b590 secure :-1 s=READY pgs=1752 cs=0 l=1 rev1=1 crypto rx=0x7f0de0000c00 tx=0x7f0de00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.721+0000 7f0defe00640 1 -- 172.21.15.86:0/883525372 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de0034040 con 0x7f0df80af1c0 2024-03-06T07:37:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.721+0000 7f0defe00640 1 -- 172.21.15.86:0/883525372 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0de00443f0 con 0x7f0df80af1c0 2024-03-06T07:37:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.721+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0df8159900 con 0x7f0df80af1c0 2024-03-06T07:37:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.721+0000 7f0defe00640 1 -- 172.21.15.86:0/883525372 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de00447e0 con 0x7f0df80af1c0 2024-03-06T07:37:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.721+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0df8159ec0 con 0x7f0df80af1c0 2024-03-06T07:37:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.722+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0df804f330 con 0x7f0df80af1c0 2024-03-06T07:37:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.723+0000 7f0defe00640 1 -- 172.21.15.86:0/883525372 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0de004b060 con 0x7f0df80af1c0 2024-03-06T07:37:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.724+0000 7f0defe00640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dd406b4d0 0x7f0dd406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.725+0000 7f0defe00640 1 -- 172.21.15.86:0/883525372 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0de00ac410 con 0x7f0df80af1c0 2024-03-06T07:37:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.725+0000 7f0df7400640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dd406b4d0 0x7f0dd406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.726+0000 7f0df7400640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dd406b4d0 0x7f0dd406d990 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f0df81f7530 tx=0x7f0de8009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:35.731 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.729+0000 7f0defe00640 1 -- 172.21.15.86:0/883525372 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0de003f070 con 0x7f0df80af1c0 2024-03-06T07:37:35.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.961+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0df80b1c40 con 0x7f0df80af1c0 2024-03-06T07:37:35.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.962+0000 7f0defe00640 1 -- 172.21.15.86:0/883525372 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f0df815f4a0 con 0x7f0df80af1c0 2024-03-06T07:37:35.964 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:35.965 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:35.965 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:35.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.964+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dd406b4d0 msgr2=0x7f0dd406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:35.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.964+0000 7f0dfe000640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dd406b4d0 0x7f0dd406d990 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f0df81f7530 tx=0x7f0de8009040 comp rx=0 tx=0).stop 2024-03-06T07:37:35.967 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.964+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df80af1c0 msgr2=0x7f0df814b590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.964+0000 7f0dfe000640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df80af1c0 0x7f0df814b590 secure :-1 s=READY pgs=1752 cs=0 l=1 rev1=1 crypto rx=0x7f0de0000c00 tx=0x7f0de00046c0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 shutdown_connections 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f0dd406b4d0 0x7f0dd406d990 unknown :-1 s=CLOSED pgs=743 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f0df815eff0 0x7f0df815beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f0df80af1c0 0x7f0df814b590 unknown :-1 s=CLOSED pgs=1752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 --2- 172.21.15.86:0/883525372 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f0df806e4d0 0x7f0df81590d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 >> 172.21.15.86:0/883525372 conn(0x7f0df80b1730 msgr2=0x7f0df815e5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 shutdown_connections 2024-03-06T07:37:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:35.965+0000 7f0dfe000640 1 -- 172.21.15.86:0/883525372 wait complete. 2024-03-06T07:37:36.188 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:36.188 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:37.189 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:37.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.406+0000 7f3d12c00640 1 Processor -- start 2024-03-06T07:37:37.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.406+0000 7f3d12c00640 1 -- start start 2024-03-06T07:37:37.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.409+0000 7f3d12c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3d0c151660 0x7f3d0c071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:37.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.409+0000 7f3d12c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c0cf430 0x7f3d0c06d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:37.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.409+0000 7f3d12c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d0c06d920 0x7f3d0c15dff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:37.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.409+0000 7f3d12c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d0c0b3ee0 con 0x7f3d0c06d920 2024-03-06T07:37:37.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.409+0000 7f3d12c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d0c071a60 con 0x7f3d0c151660 2024-03-06T07:37:37.413 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.409+0000 7f3d12c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d0c0b35b0 con 0x7f3d0c0cf430 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d0c06d920 0x7f3d0c15dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f3d0c06d920 0x7f3d0c15dff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38942/0 (socket says 172.21.15.86:38942) 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 -- 172.21.15.86:0/1705431845 learned_addr learned my addr 172.21.15.86:0/1705431845 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d0be00640 1 --2- 172.21.15.86:0/1705431845 >> v2:172.21.15.194:3300/0 conn(0x7f3d0c151660 0x7f3d0c071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 -- 172.21.15.86:0/1705431845 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c0cf430 msgr2=0x7f3d0c06d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d03e00640 1 --2- 172.21.15.86:0/1705431845 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c0cf430 0x7f3d0c06d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 --2- 172.21.15.86:0/1705431845 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c0cf430 0x7f3d0c06d3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 -- 172.21.15.86:0/1705431845 >> v2:172.21.15.194:3300/0 conn(0x7f3d0c151660 msgr2=0x7f3d0c071520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 --2- 172.21.15.86:0/1705431845 >> v2:172.21.15.194:3300/0 conn(0x7f3d0c151660 0x7f3d0c071520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d10c00640 1 -- 172.21.15.86:0/1705431845 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d0c0b3890 con 0x7f3d0c06d920 2024-03-06T07:37:37.414 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.410+0000 7f3d0be00640 1 --2- 172.21.15.86:0/1705431845 >> v2:172.21.15.194:3300/0 conn(0x7f3d0c151660 0x7f3d0c071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.411+0000 7f3d10c00640 1 --2- 172.21.15.86:0/1705431845 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c06d920 0x7f3d0c15dff0 secure :-1 s=READY pgs=1753 cs=0 l=1 rev1=1 crypto rx=0x7f3d0c072fc0 tx=0x7f3d0402bdb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=744a42d91dae1d41 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:37.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.412+0000 7f3d0b400640 1 -- 172.21.15.86:0/1705431845 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d0402c950 con 0x7f3d0c06d920 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.412+0000 7f3d0b400640 1 -- 172.21.15.86:0/1705431845 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d0402caf0 con 0x7f3d0c06d920 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.412+0000 7f3d0b400640 1 -- 172.21.15.86:0/1705431845 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d040343f0 con 0x7f3d0c06d920 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.412+0000 7f3d12c00640 1 -- 172.21.15.86:0/1705431845 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c06d920 msgr2=0x7f3d0c15dff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.412+0000 7f3d12c00640 1 --2- 172.21.15.86:0/1705431845 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c06d920 0x7f3d0c15dff0 secure :-1 s=READY pgs=1753 cs=0 l=1 rev1=1 crypto rx=0x7f3d0c072fc0 tx=0x7f3d0402bdb0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.413+0000 7f3d12c00640 1 -- 172.21.15.86:0/1705431845 shutdown_connections 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.413+0000 7f3d12c00640 1 --2- 172.21.15.86:0/1705431845 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c06d920 0x7f3d0c15dff0 unknown :-1 s=CLOSED pgs=1753 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.413+0000 7f3d12c00640 1 --2- 172.21.15.86:0/1705431845 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c0cf430 0x7f3d0c06d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.413+0000 7f3d12c00640 1 --2- 172.21.15.86:0/1705431845 >> v2:172.21.15.194:3300/0 conn(0x7f3d0c151660 0x7f3d0c071520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.413+0000 7f3d12c00640 1 -- 172.21.15.86:0/1705431845 >> 172.21.15.86:0/1705431845 conn(0x7f3d0c0b1730 msgr2=0x7f3d0c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.413+0000 7f3d12c00640 1 -- 172.21.15.86:0/1705431845 shutdown_connections 2024-03-06T07:37:37.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.413+0000 7f3d12c00640 1 -- 172.21.15.86:0/1705431845 wait complete. 2024-03-06T07:37:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.415+0000 7f3d12c00640 1 Processor -- start 2024-03-06T07:37:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.416+0000 7f3d12c00640 1 -- start start 2024-03-06T07:37:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.417+0000 7f3d12c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d0c06d920 0x7f3d0c149b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.417+0000 7f3d12c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c0cf430 0x7f3d0c15c400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.417+0000 7f3d12c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c151660 0x7f3d0c147e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.417+0000 7f3d12c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d0c15e530 con 0x7f3d0c0cf430 2024-03-06T07:37:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.417+0000 7f3d12c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d0c071d30 con 0x7f3d0c06d920 2024-03-06T07:37:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.417+0000 7f3d12c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d0c15e6b0 con 0x7f3d0c151660 2024-03-06T07:37:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.417+0000 7f3d03e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c0cf430 0x7f3d0c15c400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c0cf430 0x7f3d0c15c400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38948/0 (socket says 172.21.15.86:38948) 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 -- 172.21.15.86:0/2553922923 learned_addr learned my addr 172.21.15.86:0/2553922923 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d0be00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d0c06d920 0x7f3d0c149b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d10c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c151660 0x7f3d0c147e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 -- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c151660 msgr2=0x7f3d0c147e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c151660 0x7f3d0c147e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 -- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d0c06d920 msgr2=0x7f3d0c149b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d0c06d920 0x7f3d0c149b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 -- 172.21.15.86:0/2553922923 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d0c151db0 con 0x7f3d0c0cf430 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d03e00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c0cf430 0x7f3d0c15c400 secure :-1 s=READY pgs=1754 cs=0 l=1 rev1=1 crypto rx=0x7f3cf400c7f0 tx=0x7f3cf400ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.418+0000 7f3d08c00640 1 -- 172.21.15.86:0/2553922923 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cf4015070 con 0x7f3d0c0cf430 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.419+0000 7f3d08c00640 1 -- 172.21.15.86:0/2553922923 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3cf4004220 con 0x7f3d0c0cf430 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.419+0000 7f3d08c00640 1 -- 172.21.15.86:0/2553922923 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cf4007620 con 0x7f3d0c0cf430 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.419+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d0c15ce50 con 0x7f3d0c0cf430 2024-03-06T07:37:37.422 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.419+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d0c149050 con 0x7f3d0c0cf430 2024-03-06T07:37:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.420+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d0c06d5d0 con 0x7f3d0c0cf430 2024-03-06T07:37:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.421+0000 7f3d08c00640 1 -- 172.21.15.86:0/2553922923 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3cf40079d0 con 0x7f3d0c0cf430 2024-03-06T07:37:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.422+0000 7f3d08c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ce806b500 0x7f3ce806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.422+0000 7f3d08c00640 1 -- 172.21.15.86:0/2553922923 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3cf408a350 con 0x7f3d0c0cf430 2024-03-06T07:37:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.422+0000 7f3d0be00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ce806b500 0x7f3ce806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:37.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.423+0000 7f3d0be00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ce806b500 0x7f3ce806d9c0 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f3d0c0b0cc0 tx=0x7f3cfc008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:37.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.426+0000 7f3d08c00640 1 -- 172.21.15.86:0/2553922923 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3d0c06d5d0 con 0x7f3d0c0cf430 2024-03-06T07:37:37.669 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:37.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.694+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3d0c166250 con 0x7f3d0c0cf430 2024-03-06T07:37:37.697 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.695+0000 7f3d08c00640 1 -- 172.21.15.86:0/2553922923 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f3cf4059060 con 0x7f3d0c0cf430 2024-03-06T07:37:37.698 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:37.698 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:37.698 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:37.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ce806b500 msgr2=0x7f3ce806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:37.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ce806b500 0x7f3ce806d9c0 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f3d0c0b0cc0 tx=0x7f3cfc008040 comp rx=0 tx=0).stop 2024-03-06T07:37:37.700 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c0cf430 msgr2=0x7f3d0c15c400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c0cf430 0x7f3d0c15c400 secure :-1 s=READY pgs=1754 cs=0 l=1 rev1=1 crypto rx=0x7f3cf400c7f0 tx=0x7f3cf400ccb0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 shutdown_connections 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ce806b500 0x7f3ce806d9c0 unknown :-1 s=CLOSED pgs=744 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d0c151660 0x7f3d0c147e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d0c0cf430 0x7f3d0c15c400 unknown :-1 s=CLOSED pgs=1754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.697+0000 7f3d12c00640 1 --2- 172.21.15.86:0/2553922923 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d0c06d920 0x7f3d0c149b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.698+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 >> 172.21.15.86:0/2553922923 conn(0x7f3d0c0b1730 msgr2=0x7f3d0c06cec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.698+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 shutdown_connections 2024-03-06T07:37:37.701 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:37.698+0000 7f3d12c00640 1 -- 172.21.15.86:0/2553922923 wait complete. 2024-03-06T07:37:37.919 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:37.919 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:38.920 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:39.130 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.128+0000 7f4fb1600640 1 Processor -- start 2024-03-06T07:37:39.131 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.128+0000 7f4fb1600640 1 -- start start 2024-03-06T07:37:39.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.129+0000 7f4fb1600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4fac0cf430 0x7f4fac070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:39.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.129+0000 7f4fb1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac06d260 0x7f4fac0b2d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:39.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.129+0000 7f4fb1600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4fac0717c0 con 0x7f4fac071260 2024-03-06T07:37:39.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.129+0000 7f4fb1600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4fac071aa0 con 0x7f4fac0cf430 2024-03-06T07:37:39.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.129+0000 7f4fb1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4fac071d80 con 0x7f4fac06d260 2024-03-06T07:37:39.132 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4faaa00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4fac0cf430 0x7f4fac070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:39.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4fa2000640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4fac071260 0x7f4fac06cb50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49226/0 (socket says 172.21.15.86:49226) 2024-03-06T07:37:39.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4fa2000640 1 -- 172.21.15.86:0/232584123 learned_addr learned my addr 172.21.15.86:0/232584123 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:39.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4fab400640 1 --2- 172.21.15.86:0/232584123 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac06d260 0x7f4fac0b2d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:39.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4faa000640 1 -- 172.21.15.86:0/232584123 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3101030714 0 0) 0x7f4fac0717c0 con 0x7f4fac071260 2024-03-06T07:37:39.133 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4faa000640 1 -- 172.21.15.86:0/232584123 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4f90003520 con 0x7f4fac071260 2024-03-06T07:37:39.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4faaa00640 1 -- 172.21.15.86:0/232584123 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac06d260 msgr2=0x7f4fac0b2d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:39.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4faaa00640 1 --2- 172.21.15.86:0/232584123 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac06d260 0x7f4fac0b2d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4faaa00640 1 -- 172.21.15.86:0/232584123 >> v1:172.21.15.86:6789/0 conn(0x7f4fac071260 legacy=0x7f4fac06cb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:39.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.130+0000 7f4faaa00640 1 -- 172.21.15.86:0/232584123 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4fac072060 con 0x7f4fac0cf430 2024-03-06T07:37:39.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.131+0000 7f4faaa00640 1 --2- 172.21.15.86:0/232584123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac0cf430 0x7f4fac070d20 secure :-1 s=READY pgs=1756 cs=0 l=1 rev1=1 crypto rx=0x7f4f9c0097a0 tx=0x7f4f9c02b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a0e69756021b7318 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:39.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4faa000640 1 -- 172.21.15.86:0/232584123 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f9c035020 con 0x7f4fac0cf430 2024-03-06T07:37:39.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4faa000640 1 -- 172.21.15.86:0/232584123 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f9c031810 con 0x7f4fac0cf430 2024-03-06T07:37:39.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4faa000640 1 -- 172.21.15.86:0/232584123 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f9c031be0 con 0x7f4fac0cf430 2024-03-06T07:37:39.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4fb1600640 1 -- 172.21.15.86:0/232584123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac0cf430 msgr2=0x7f4fac070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:39.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4fb1600640 1 --2- 172.21.15.86:0/232584123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac0cf430 0x7f4fac070d20 secure :-1 s=READY pgs=1756 cs=0 l=1 rev1=1 crypto rx=0x7f4f9c0097a0 tx=0x7f4f9c02b8b0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4fb1600640 1 -- 172.21.15.86:0/232584123 shutdown_connections 2024-03-06T07:37:39.135 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4fb1600640 1 --2- 172.21.15.86:0/232584123 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac06d260 0x7f4fac0b2d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.132+0000 7f4fb1600640 1 --2- 172.21.15.86:0/232584123 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac0cf430 0x7f4fac070d20 unknown :-1 s=CLOSED pgs=1756 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.133+0000 7f4fb1600640 1 -- 172.21.15.86:0/232584123 >> 172.21.15.86:0/232584123 conn(0x7f4fac0b1730 msgr2=0x7f4fac0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:39.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.133+0000 7f4fb1600640 1 -- 172.21.15.86:0/232584123 shutdown_connections 2024-03-06T07:37:39.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.133+0000 7f4fb1600640 1 -- 172.21.15.86:0/232584123 wait complete. 2024-03-06T07:37:39.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.134+0000 7f4fb1600640 1 Processor -- start 2024-03-06T07:37:39.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.134+0000 7f4fb1600640 1 -- start start 2024-03-06T07:37:39.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4fb1600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac06d260 0x7f4fac144a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:39.138 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4fb1600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac071260 0x7f4fac1441d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4fb1600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fac0cf430 0x7f4fac147450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4fb1600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4fac162cf0 con 0x7f4fac06d260 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4fb1600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4fac162b70 con 0x7f4fac0cf430 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4fb1600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4fac0b3250 con 0x7f4fac071260 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4faaa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac06d260 0x7f4fac144a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.135+0000 7f4faaa00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac06d260 0x7f4fac144a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38958/0 (socket says 172.21.15.86:38958) 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4faaa00640 1 -- 172.21.15.86:0/947596307 learned_addr learned my addr 172.21.15.86:0/947596307 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4faaa00640 1 -- 172.21.15.86:0/947596307 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac071260 msgr2=0x7f4fac1441d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4faaa00640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac071260 0x7f4fac1441d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4faaa00640 1 -- 172.21.15.86:0/947596307 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fac0cf430 msgr2=0x7f4fac147450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4faaa00640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fac0cf430 0x7f4fac147450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4faaa00640 1 -- 172.21.15.86:0/947596307 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4fac1602c0 con 0x7f4fac06d260 2024-03-06T07:37:39.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4faaa00640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac06d260 0x7f4fac144a10 secure :-1 s=READY pgs=1757 cs=0 l=1 rev1=1 crypto rx=0x7f4f9c034040 tx=0x7f4f9c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:39.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4fa3400640 1 -- 172.21.15.86:0/947596307 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f9c040070 con 0x7f4fac06d260 2024-03-06T07:37:39.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4fa3400640 1 -- 172.21.15.86:0/947596307 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f9c039cf0 con 0x7f4fac06d260 2024-03-06T07:37:39.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4fac15b4c0 con 0x7f4fac06d260 2024-03-06T07:37:39.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.136+0000 7f4fa3400640 1 -- 172.21.15.86:0/947596307 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f9c035040 con 0x7f4fac06d260 2024-03-06T07:37:39.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.137+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4fac15fd30 con 0x7f4fac06d260 2024-03-06T07:37:39.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.138+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4fac04f330 con 0x7f4fac06d260 2024-03-06T07:37:39.141 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.139+0000 7f4fa3400640 1 -- 172.21.15.86:0/947596307 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4f9c038020 con 0x7f4fac06d260 2024-03-06T07:37:39.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.140+0000 7f4fa3400640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f8806b4d0 0x7f4f8806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:39.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.140+0000 7f4fa3400640 1 -- 172.21.15.86:0/947596307 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4f9c0abe40 con 0x7f4fac06d260 2024-03-06T07:37:39.142 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.140+0000 7f4fa2000640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f8806b4d0 0x7f4f8806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:39.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.140+0000 7f4fa2000640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f8806b4d0 0x7f4f8806d990 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f4f94000f60 tx=0x7f4f940078f0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:39.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.150+0000 7f4fa3400640 1 -- 172.21.15.86:0/947596307 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4f9c07ac70 con 0x7f4fac06d260 2024-03-06T07:37:39.384 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.382+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4fac0b4a40 con 0x7f4fac06d260 2024-03-06T07:37:39.385 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.383+0000 7f4fa3400640 1 -- 172.21.15.86:0/947596307 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f4f9c07a570 con 0x7f4fac06d260 2024-03-06T07:37:39.385 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:39.386 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:39.386 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f8806b4d0 msgr2=0x7f4f8806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:39.388 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f8806b4d0 0x7f4f8806d990 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f4f94000f60 tx=0x7f4f940078f0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac06d260 msgr2=0x7f4fac144a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac06d260 0x7f4fac144a10 secure :-1 s=READY pgs=1757 cs=0 l=1 rev1=1 crypto rx=0x7f4f9c034040 tx=0x7f4f9c0046c0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 shutdown_connections 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4f8806b4d0 0x7f4f8806d990 unknown :-1 s=CLOSED pgs=745 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4fac0cf430 0x7f4fac147450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4fac071260 0x7f4fac1441d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 --2- 172.21.15.86:0/947596307 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4fac06d260 0x7f4fac144a10 unknown :-1 s=CLOSED pgs=1757 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 >> 172.21.15.86:0/947596307 conn(0x7f4fac0b1730 msgr2=0x7f4fac0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 shutdown_connections 2024-03-06T07:37:39.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:39.386+0000 7f4fb1600640 1 -- 172.21.15.86:0/947596307 wait complete. 2024-03-06T07:37:39.610 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:39.610 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:39.669 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:40.611 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:40.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.811+0000 7fd3fbe00640 1 Processor -- start 2024-03-06T07:37:40.813 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.811+0000 7fd3fbe00640 1 -- start start 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.812+0000 7fd3fbe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd3f415e160 0x7fd3f4162620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.812+0000 7fd3fbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3f40b3ac0 0x7fd3f406b580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.812+0000 7fd3fbe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3f4162b60 con 0x7fd3f40af820 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.812+0000 7fd3fbe00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3f406b0f0 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.812+0000 7fd3fbe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3f40b5620 con 0x7fd3f415e160 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3f40b3ac0 0x7fd3f406b580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f9e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd3f40b3ac0 0x7fd3f406b580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38966/0 (socket says 172.21.15.86:38966) 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f8a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd3f40af820 0x7fd3f40b33b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49238/0 (socket says 172.21.15.86:49238) 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f9e00640 1 -- 172.21.15.86:0/3310851155 learned_addr learned my addr 172.21.15.86:0/3310851155 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:40.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f9e00640 1 -- 172.21.15.86:0/3310851155 >> v2:172.21.15.194:3300/0 conn(0x7fd3f415e160 msgr2=0x7fd3f4162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:40.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f9e00640 1 --2- 172.21.15.86:0/3310851155 >> v2:172.21.15.194:3300/0 conn(0x7fd3f415e160 0x7fd3f4162620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:40.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f9e00640 1 -- 172.21.15.86:0/3310851155 >> v1:172.21.15.86:6789/0 conn(0x7fd3f40af820 legacy=0x7fd3f40b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:40.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.813+0000 7fd3f9e00640 1 -- 172.21.15.86:0/3310851155 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3f406f1f0 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.814+0000 7fd3f9e00640 1 --2- 172.21.15.86:0/3310851155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f406b580 secure :-1 s=READY pgs=1759 cs=0 l=1 rev1=1 crypto rx=0x7fd3f40d0310 tx=0x7fd3ec02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6b4de19fdf2ddd4 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:40.816 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.815+0000 7fd3f3e00640 1 -- 172.21.15.86:0/3310851155 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3ec035020 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.815+0000 7fd3f3e00640 1 -- 172.21.15.86:0/3310851155 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3ec031870 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.815+0000 7fd3f3e00640 1 -- 172.21.15.86:0/3310851155 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3ec031c40 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.815+0000 7fd3fbe00640 1 -- 172.21.15.86:0/3310851155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 msgr2=0x7fd3f406b580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.815+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/3310851155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f406b580 secure :-1 s=READY pgs=1759 cs=0 l=1 rev1=1 crypto rx=0x7fd3f40d0310 tx=0x7fd3ec02b910 comp rx=0 tx=0).stop 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.816+0000 7fd3fbe00640 1 -- 172.21.15.86:0/3310851155 shutdown_connections 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.816+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/3310851155 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f406b580 unknown :-1 s=CLOSED pgs=1759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.816+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/3310851155 >> v2:172.21.15.194:3300/0 conn(0x7fd3f415e160 0x7fd3f4162620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.816+0000 7fd3fbe00640 1 -- 172.21.15.86:0/3310851155 >> 172.21.15.86:0/3310851155 conn(0x7fd3f40d0340 msgr2=0x7fd3f40d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.816+0000 7fd3fbe00640 1 -- 172.21.15.86:0/3310851155 shutdown_connections 2024-03-06T07:37:40.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.816+0000 7fd3fbe00640 1 -- 172.21.15.86:0/3310851155 wait complete. 2024-03-06T07:37:40.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.818+0000 7fd3fbe00640 1 Processor -- start 2024-03-06T07:37:40.820 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.818+0000 7fd3fbe00640 1 -- start start 2024-03-06T07:37:40.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3fbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f40af820 0x7fd3f40cf710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:40.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f4142940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:40.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3fbe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f415e160 0x7fd3f41583c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:40.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3fbe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f406ef10 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.823 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3fbe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f406ec30 con 0x7fd3f415e160 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3fbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f406bac0 con 0x7fd3f40af820 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3f8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f4142940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3f8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f4142940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38980/0 (socket says 172.21.15.86:38980) 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.820+0000 7fd3f8a00640 1 -- 172.21.15.86:0/1294651357 learned_addr learned my addr 172.21.15.86:0/1294651357 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f9e00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f415e160 0x7fd3f41583c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f8a00640 1 -- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f40af820 msgr2=0x7fd3f40cf710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f9400640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f40af820 0x7fd3f40cf710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f8a00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f40af820 0x7fd3f40cf710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f8a00640 1 -- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f415e160 msgr2=0x7fd3f41583c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f8a00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f415e160 0x7fd3f41583c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f8a00640 1 -- 172.21.15.86:0/1294651357 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3f4142e80 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.824 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f9e00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f415e160 0x7fd3f41583c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:40.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f8a00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f4142940 secure :-1 s=READY pgs=1760 cs=0 l=1 rev1=1 crypto rx=0x7fd3e000c440 tx=0x7fd3e0001e70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:40.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f1600640 1 -- 172.21.15.86:0/1294651357 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e00074e0 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f1600640 1 -- 172.21.15.86:0/1294651357 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3e0007680 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3f1600640 1 -- 172.21.15.86:0/1294651357 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e0004a20 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.821+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3f415e5e0 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.825 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.822+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3f4157bc0 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.823+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd3f4074ae0 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.823+0000 7fd3f1600640 1 -- 172.21.15.86:0/1294651357 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd3e001a020 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.824+0000 7fd3f1600640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3d406b4d0 0x7fd3d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:40.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.824+0000 7fd3f1600640 1 -- 172.21.15.86:0/1294651357 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd3e0089b40 con 0x7fd3f40b3ac0 2024-03-06T07:37:40.826 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.825+0000 7fd3f9400640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3d406b4d0 0x7fd3d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:40.827 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.825+0000 7fd3f9400640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3d406b4d0 0x7fd3d406d990 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7fd3e8007dc0 tx=0x7fd3e80072f0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:40.830 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:40.829+0000 7fd3f1600640 1 -- 172.21.15.86:0/1294651357 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd3f4142e80 con 0x7fd3f40b3ac0 2024-03-06T07:37:41.066 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.065+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd3f404f330 con 0x7fd3f40b3ac0 2024-03-06T07:37:41.067 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.066+0000 7fd3f1600640 1 -- 172.21.15.86:0/1294651357 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fd3f4162b60 con 0x7fd3f40b3ac0 2024-03-06T07:37:41.068 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:41.068 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:41.068 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:41.070 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.068+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3d406b4d0 msgr2=0x7fd3d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.068+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3d406b4d0 0x7fd3d406d990 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7fd3e8007dc0 tx=0x7fd3e80072f0 comp rx=0 tx=0).stop 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 msgr2=0x7fd3f4142940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f4142940 secure :-1 s=READY pgs=1760 cs=0 l=1 rev1=1 crypto rx=0x7fd3e000c440 tx=0x7fd3e0001e70 comp rx=0 tx=0).stop 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 shutdown_connections 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd3d406b4d0 0x7fd3d406d990 unknown :-1 s=CLOSED pgs=746 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd3f415e160 0x7fd3f41583c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd3f40b3ac0 0x7fd3f4142940 unknown :-1 s=CLOSED pgs=1760 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 --2- 172.21.15.86:0/1294651357 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd3f40af820 0x7fd3f40cf710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 >> 172.21.15.86:0/1294651357 conn(0x7fd3f40d0340 msgr2=0x7fd3f40aeae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 shutdown_connections 2024-03-06T07:37:41.071 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:41.069+0000 7fd3fbe00640 1 -- 172.21.15.86:0/1294651357 wait complete. 2024-03-06T07:37:41.291 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:41.291 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:41.671 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:42.291 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:42.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.496+0000 7f2e31c00640 1 Processor -- start 2024-03-06T07:37:42.497 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.496+0000 7f2e31c00640 1 -- start start 2024-03-06T07:37:42.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e31c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c15a940 0x7f2e2c0b45e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:42.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e31c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2e2c0cf370 0x7f2e2c06d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:42.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e31c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2e2c0b2b30 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.499 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e31c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2e2c0b28e0 con 0x7f2e2c0cf370 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e31c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c0b2e10 con 0x7f2e2c15a940 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e2b400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f2e2c0cf370 0x7f2e2c06d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e31200640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f2e2c0cc2a0 0x7f2e2c0cd460 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49252/0 (socket says 172.21.15.86:49252) 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.497+0000 7f2e31200640 1 -- 172.21.15.86:0/4184697004 learned_addr learned my addr 172.21.15.86:0/4184697004 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.498+0000 7f2e2be00640 1 --2- 172.21.15.86:0/4184697004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c15a940 0x7f2e2c0b45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.498+0000 7f2e2b400640 1 -- 172.21.15.86:0/4184697004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c15a940 msgr2=0x7f2e2c0b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.498+0000 7f2e2b400640 1 --2- 172.21.15.86:0/4184697004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c15a940 0x7f2e2c0b45e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.498+0000 7f2e2b400640 1 -- 172.21.15.86:0/4184697004 >> v1:172.21.15.86:6789/0 conn(0x7f2e2c0cc2a0 legacy=0x7f2e2c0cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:42.500 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.498+0000 7f2e2b400640 1 -- 172.21.15.86:0/4184697004 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e2c06ca50 con 0x7f2e2c0cf370 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.499+0000 7f2e2b400640 1 --2- 172.21.15.86:0/4184697004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cf370 0x7f2e2c06d250 secure :-1 s=READY pgs=1762 cs=0 l=1 rev1=1 crypto rx=0x7f2e1400ba40 tx=0x7f2e1402e080 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a143ee476093e821 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.499+0000 7f2e2aa00640 1 -- 172.21.15.86:0/4184697004 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1403b070 con 0x7f2e2c0cf370 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.499+0000 7f2e2aa00640 1 -- 172.21.15.86:0/4184697004 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e1402ee30 con 0x7f2e2c0cf370 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.499+0000 7f2e2aa00640 1 -- 172.21.15.86:0/4184697004 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e14036640 con 0x7f2e2c0cf370 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 -- 172.21.15.86:0/4184697004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cf370 msgr2=0x7f2e2c06d250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 --2- 172.21.15.86:0/4184697004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cf370 0x7f2e2c06d250 secure :-1 s=READY pgs=1762 cs=0 l=1 rev1=1 crypto rx=0x7f2e1400ba40 tx=0x7f2e1402e080 comp rx=0 tx=0).stop 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 -- 172.21.15.86:0/4184697004 shutdown_connections 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 --2- 172.21.15.86:0/4184697004 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cf370 0x7f2e2c06d250 unknown :-1 s=CLOSED pgs=1762 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 --2- 172.21.15.86:0/4184697004 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c15a940 0x7f2e2c0b45e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.502 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 -- 172.21.15.86:0/4184697004 >> 172.21.15.86:0/4184697004 conn(0x7f2e2c06afc0 msgr2=0x7f2e2c06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:42.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 -- 172.21.15.86:0/4184697004 shutdown_connections 2024-03-06T07:37:42.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.500+0000 7f2e31c00640 1 -- 172.21.15.86:0/4184697004 wait complete. 2024-03-06T07:37:42.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.502+0000 7f2e31c00640 1 Processor -- start 2024-03-06T07:37:42.503 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.502+0000 7f2e31c00640 1 -- start start 2024-03-06T07:37:42.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e31c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cc2a0 0x7f2e2c151080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:42.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e31c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c0cf370 0x7f2e2c1527b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:42.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e31c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e2c15a940 0x7f2e2c1530a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:42.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e31c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c0b4b20 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e31c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c0b30f0 con 0x7f2e2c15a940 2024-03-06T07:37:42.506 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e31c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c06d790 con 0x7f2e2c0cf370 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e2be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cc2a0 0x7f2e2c151080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e2be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cc2a0 0x7f2e2c151080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39010/0 (socket says 172.21.15.86:39010) 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.503+0000 7f2e2be00640 1 -- 172.21.15.86:0/2042617035 learned_addr learned my addr 172.21.15.86:0/2042617035 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e2be00640 1 -- 172.21.15.86:0/2042617035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c0cf370 msgr2=0x7f2e2c1527b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e2be00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c0cf370 0x7f2e2c1527b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e2be00640 1 -- 172.21.15.86:0/2042617035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e2c15a940 msgr2=0x7f2e2c1530a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e2be00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e2c15a940 0x7f2e2c1530a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e2be00640 1 -- 172.21.15.86:0/2042617035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e2c152cf0 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e2be00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cc2a0 0x7f2e2c151080 secure :-1 s=READY pgs=1763 cs=0 l=1 rev1=1 crypto rx=0x7f2e1c007e40 tx=0x7f2e1c00e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e23e00640 1 -- 172.21.15.86:0/2042617035 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1c014620 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e23e00640 1 -- 172.21.15.86:0/2042617035 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e1c019070 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2e2c1535e0 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e23e00640 1 -- 172.21.15.86:0/2042617035 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1c014a00 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.504+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2e2c153b60 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.505+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2e2c04f330 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.506+0000 7f2e23e00640 1 -- 172.21.15.86:0/2042617035 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2e1c014ba0 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.508+0000 7f2e23e00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e0806b4d0 0x7f2e0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:42.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.508+0000 7f2e23e00640 1 -- 172.21.15.86:0/2042617035 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f2e1c08ab80 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.509 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.508+0000 7f2e2b400640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e0806b4d0 0x7f2e0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:42.510 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.509+0000 7f2e2b400640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e0806b4d0 0x7f2e0806d990 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7f2e14038000 tx=0x7f2e14030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:42.514 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.512+0000 7f2e23e00640 1 -- 172.21.15.86:0/2042617035 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2e2c152cf0 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.748+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2e2c06e180 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.750 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.748+0000 7f2e23e00640 1 -- 172.21.15.86:0/2042617035 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f2e1c059920 con 0x7f2e2c0cc2a0 2024-03-06T07:37:42.750 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:42.750 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:42.751 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:42.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e0806b4d0 msgr2=0x7f2e0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:42.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e0806b4d0 0x7f2e0806d990 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7f2e14038000 tx=0x7f2e14030000 comp rx=0 tx=0).stop 2024-03-06T07:37:42.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cc2a0 msgr2=0x7f2e2c151080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:42.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cc2a0 0x7f2e2c151080 secure :-1 s=READY pgs=1763 cs=0 l=1 rev1=1 crypto rx=0x7f2e1c007e40 tx=0x7f2e1c00e4b0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 shutdown_connections 2024-03-06T07:37:42.753 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f2e0806b4d0 0x7f2e0806d990 unknown :-1 s=CLOSED pgs=747 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2e2c15a940 0x7f2e2c1530a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f2e2c0cf370 0x7f2e2c1527b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 --2- 172.21.15.86:0/2042617035 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f2e2c0cc2a0 0x7f2e2c151080 unknown :-1 s=CLOSED pgs=1763 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:42.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 >> 172.21.15.86:0/2042617035 conn(0x7f2e2c06afc0 msgr2=0x7f2e2c0d0120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:42.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 shutdown_connections 2024-03-06T07:37:42.754 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:42.751+0000 7f2e31c00640 1 -- 172.21.15.86:0/2042617035 wait complete. 2024-03-06T07:37:42.971 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:42.971 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:43.673 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:43.972 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.179+0000 7f7d0b200640 1 Processor -- start 2024-03-06T07:37:44.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.179+0000 7f7d0b200640 1 -- start start 2024-03-06T07:37:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.181+0000 7f7d0b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0d0600 0x7f7d0c0d4830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.181+0000 7f7d0b200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f7d0c0b2a60 0x7f7d0c0d09e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.181+0000 7f7d0b200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d0c0d0f20 con 0x7f7d0c0d18e0 2024-03-06T07:37:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.181+0000 7f7d0b200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7d0c0cf090 con 0x7f7d0c0b2a60 2024-03-06T07:37:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.181+0000 7f7d0b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c15f030 con 0x7f7d0c0d0600 2024-03-06T07:37:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.181+0000 7f7d0a800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f7d0c0d18e0 0x7f7d0c0cf520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49264/0 (socket says 172.21.15.86:49264) 2024-03-06T07:37:44.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.181+0000 7f7d0a800640 1 -- 172.21.15.86:0/3478360624 learned_addr learned my addr 172.21.15.86:0/3478360624 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d03e00640 1 --2- 172.21.15.86:0/3478360624 >> v2:172.21.15.194:3300/0 conn(0x7f7d0c0b2a60 0x7f7d0c0d09e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09e00640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0d0600 0x7f7d0c0d4830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1097308936 0 0) 0x7f7d0c0d0f20 con 0x7f7d0c0d18e0 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7cf0003520 con 0x7f7d0c0d18e0 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 732077812 0 0) 0x7f7cf0003520 con 0x7f7d0c0d18e0 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0d0600 msgr2=0x7f7d0c0d4830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0d0600 0x7f7d0c0d4830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 >> v2:172.21.15.194:3300/0 conn(0x7f7d0c0b2a60 msgr2=0x7f7d0c0d09e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 --2- 172.21.15.86:0/3478360624 >> v2:172.21.15.194:3300/0 conn(0x7f7d0c0b2a60 0x7f7d0c0d09e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d0c15f310 con 0x7f7d0c0d18e0 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f7d040019f0 con 0x7f7d0c0d18e0 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 >> v1:172.21.15.86:6789/0 conn(0x7f7d0c0d18e0 legacy=0x7f7d0c0cf520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7cf00039b0 0x7f7cf0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.182+0000 7f7d09400640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf0004800 0x7f7cf0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09400640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0d0600 0x7f7cf00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c0cf090 con 0x7f7cf00039b0 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c15f030 con 0x7f7cf0004800 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7cf000b9c0 con 0x7f7d0c0d0600 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09e00640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7cf00039b0 0x7f7cf0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09e00640 1 -- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0d0600 msgr2=0x7f7cf00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09e00640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0d0600 0x7f7cf00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09e00640 1 -- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf0004800 msgr2=0x7f7cf0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09e00640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf0004800 0x7f7cf0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.186 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.183+0000 7f7d09e00640 1 -- 172.21.15.86:0/3478360624 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7cf0003520 con 0x7f7cf00039b0 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.184+0000 7f7d09e00640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7cf00039b0 0x7f7cf0003dd0 secure :-1 s=READY pgs=1765 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc009ac0 tx=0x7f7cf0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=56165dded4819e5d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.184+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cfc039070 con 0x7f7cf00039b0 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.184+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7cfc02c6d0 con 0x7f7cf00039b0 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.184+0000 7f7d09400640 1 -- 172.21.15.86:0/3478360624 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cfc009690 con 0x7f7cf00039b0 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.185+0000 7f7d0b200640 1 -- 172.21.15.86:0/3478360624 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7cf00039b0 msgr2=0x7f7cf0003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.185+0000 7f7d0b200640 1 --2- 172.21.15.86:0/3478360624 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7cf00039b0 0x7f7cf0003dd0 secure :-1 s=READY pgs=1765 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc009ac0 tx=0x7f7cf0005830 comp rx=0 tx=0).stop 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.185+0000 7f7d0a800640 1 -- 172.21.15.86:0/3478360624 reap_dead start 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.185+0000 7f7d0b200640 1 -- 172.21.15.86:0/3478360624 shutdown_connections 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.185+0000 7f7d0b200640 1 -- 172.21.15.86:0/3478360624 >> 172.21.15.86:0/3478360624 conn(0x7f7d0c0d3d10 msgr2=0x7f7d0c0d4120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.185+0000 7f7d0b200640 1 -- 172.21.15.86:0/3478360624 shutdown_connections 2024-03-06T07:37:44.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.185+0000 7f7d0b200640 1 -- 172.21.15.86:0/3478360624 wait complete. 2024-03-06T07:37:44.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.187+0000 7f7d0b200640 1 Processor -- start 2024-03-06T07:37:44.189 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.187+0000 7f7d0b200640 1 -- start start 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d0b200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf00039b0 0x7f7d0c1f5970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d0b200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d0c0c8d40 0x7f7d0c1f7da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d0b200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0b4480 0x7f7d0c150fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d0b200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c0b3250 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d0b200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c0ced20 con 0x7f7cf00039b0 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d0b200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c15ecf0 con 0x7f7d0c0b4480 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d03e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d0c0c8d40 0x7f7d0c1f7da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d03e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d0c0c8d40 0x7f7d0c1f7da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:39020/0 (socket says 172.21.15.86:39020) 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.188+0000 7f7d03e00640 1 -- 172.21.15.86:0/1591759681 learned_addr learned my addr 172.21.15.86:0/1591759681 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d09e00640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf00039b0 0x7f7d0c1f5970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d0a800640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0b4480 0x7f7d0c150fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d03e00640 1 -- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0b4480 msgr2=0x7f7d0c150fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:44.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d03e00640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0b4480 0x7f7d0c150fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d03e00640 1 -- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf00039b0 msgr2=0x7f7d0c1f5970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:44.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d03e00640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf00039b0 0x7f7d0c1f5970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d03e00640 1 -- 172.21.15.86:0/1591759681 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d0c1f5eb0 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.189+0000 7f7d03e00640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d0c0c8d40 0x7f7d0c1f7da0 secure :-1 s=READY pgs=1766 cs=0 l=1 rev1=1 crypto rx=0x7f7cf400e890 tx=0x7f7cf400ed50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:44.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.190+0000 7f7d02000640 1 -- 172.21.15.86:0/1591759681 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cf4017070 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.190+0000 7f7d02000640 1 -- 172.21.15.86:0/1591759681 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7cf400c940 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.190+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d0c1f86a0 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.190+0000 7f7d02000640 1 -- 172.21.15.86:0/1591759681 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cf400cd10 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.190+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d0c1f8c60 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.191+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d0c04f330 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.194 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.193+0000 7f7d02000640 1 -- 172.21.15.86:0/1591759681 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7cf401a070 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.194+0000 7f7d02000640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ce806b500 0x7f7ce806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:44.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.194+0000 7f7d02000640 1 -- 172.21.15.86:0/1591759681 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f7cf408a400 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.194+0000 7f7d09e00640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ce806b500 0x7f7ce806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:44.196 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.195+0000 7f7d09e00640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ce806b500 0x7f7ce806d9c0 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc036000 tx=0x7f7cfc030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:44.200 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.199+0000 7f7d02000640 1 -- 172.21.15.86:0/1591759681 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7cf4059240 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.432 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.430+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7d0c08d4d0 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.431+0000 7f7d02000640 1 -- 172.21.15.86:0/1591759681 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f7cf4058b40 con 0x7f7d0c0c8d40 2024-03-06T07:37:44.433 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:44.433 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:44.433 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ce806b500 msgr2=0x7f7ce806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ce806b500 0x7f7ce806d9c0 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc036000 tx=0x7f7cfc030000 comp rx=0 tx=0).stop 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d0c0c8d40 msgr2=0x7f7d0c1f7da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d0c0c8d40 0x7f7d0c1f7da0 secure :-1 s=READY pgs=1766 cs=0 l=1 rev1=1 crypto rx=0x7f7cf400e890 tx=0x7f7cf400ed50 comp rx=0 tx=0).stop 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 shutdown_connections 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f7ce806b500 0x7f7ce806d9c0 unknown :-1 s=CLOSED pgs=748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7d0c0b4480 0x7f7d0c150fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7d0c0c8d40 0x7f7d0c1f7da0 unknown :-1 s=CLOSED pgs=1766 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 --2- 172.21.15.86:0/1591759681 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7cf00039b0 0x7f7d0c1f5970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:44.436 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 >> 172.21.15.86:0/1591759681 conn(0x7f7d0c0d3d10 msgr2=0x7f7d0c160ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:44.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 shutdown_connections 2024-03-06T07:37:44.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:44.434+0000 7f7d0b200640 1 -- 172.21.15.86:0/1591759681 wait complete. 2024-03-06T07:37:44.661 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:44.661 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:45.662 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:45.675 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:45.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.873+0000 7f00dca00640 1 Processor -- start 2024-03-06T07:37:45.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.873+0000 7f00dca00640 1 -- start start 2024-03-06T07:37:45.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.875+0000 7f00dca00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f00d815a060 0x7f00d806cc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:45.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.875+0000 7f00dca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815e460 0x7f00d806d1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:45.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.875+0000 7f00dca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f00d815eac0 con 0x7f00d8071d20 2024-03-06T07:37:45.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.875+0000 7f00dca00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f00d8070f60 con 0x7f00d815a060 2024-03-06T07:37:45.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.875+0000 7f00dca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f00d8071240 con 0x7f00d815e460 2024-03-06T07:37:45.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d6800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815e460 0x7f00d806d1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:45.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d5e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f00d815a060 0x7f00d806cc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:45.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d6800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815e460 0x7f00d806d1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:59130/0 (socket says 172.21.15.86:59130) 2024-03-06T07:37:45.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d6800640 1 -- 172.21.15.86:0/3147923185 learned_addr learned my addr 172.21.15.86:0/3147923185 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:45.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d6800640 1 -- 172.21.15.86:0/3147923185 >> v2:172.21.15.194:3300/0 conn(0x7f00d815a060 msgr2=0x7f00d806cc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:45.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d6800640 1 --2- 172.21.15.86:0/3147923185 >> v2:172.21.15.194:3300/0 conn(0x7f00d815a060 0x7f00d806cc80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:45.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d6800640 1 -- 172.21.15.86:0/3147923185 >> v1:172.21.15.194:6789/0 conn(0x7f00d8071d20 legacy=0x7f00d815dd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:45.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.876+0000 7f00d6800640 1 -- 172.21.15.86:0/3147923185 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f00d8071520 con 0x7f00d815e460 2024-03-06T07:37:45.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.878+0000 7f00d6800640 1 --2- 172.21.15.86:0/3147923185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815e460 0x7f00d806d1c0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f00d000aa10 tx=0x7f00d002ce60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=53503d8724af3a14 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:45.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.878+0000 7f00d5400640 1 -- 172.21.15.86:0/3147923185 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00d003a070 con 0x7f00d815e460 2024-03-06T07:37:45.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.878+0000 7f00d5400640 1 -- 172.21.15.86:0/3147923185 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f00d002dc10 con 0x7f00d815e460 2024-03-06T07:37:45.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.878+0000 7f00d5400640 1 -- 172.21.15.86:0/3147923185 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00d000b040 con 0x7f00d815e460 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 -- 172.21.15.86:0/3147923185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815e460 msgr2=0x7f00d806d1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 --2- 172.21.15.86:0/3147923185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815e460 0x7f00d806d1c0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f00d000aa10 tx=0x7f00d002ce60 comp rx=0 tx=0).stop 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 -- 172.21.15.86:0/3147923185 shutdown_connections 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 --2- 172.21.15.86:0/3147923185 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815e460 0x7f00d806d1c0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 --2- 172.21.15.86:0/3147923185 >> v2:172.21.15.194:3300/0 conn(0x7f00d815a060 0x7f00d806cc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 -- 172.21.15.86:0/3147923185 >> 172.21.15.86:0/3147923185 conn(0x7f00d80b1be0 msgr2=0x7f00d80b1ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 -- 172.21.15.86:0/3147923185 shutdown_connections 2024-03-06T07:37:45.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.879+0000 7f00dca00640 1 -- 172.21.15.86:0/3147923185 wait complete. 2024-03-06T07:37:45.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.881+0000 7f00dca00640 1 Processor -- start 2024-03-06T07:37:45.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.881+0000 7f00dca00640 1 -- start start 2024-03-06T07:37:45.883 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00dca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815a060 0x7f00d81408f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00dca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f00d815e460 0x7f00d80b0ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00dca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f00d80b1220 0x7f00d814c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00dca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00d80688a0 con 0x7f00d80b1220 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00dca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00d8068720 con 0x7f00d815e460 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00dca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f00d815a440 con 0x7f00d815a060 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00d6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f00d80b1220 0x7f00d814c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00d6800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f00d80b1220 0x7f00d814c730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56474/0 (socket says 172.21.15.86:56474) 2024-03-06T07:37:45.884 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00d6800640 1 -- 172.21.15.86:0/2438428267 learned_addr learned my addr 172.21.15.86:0/2438428267 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:45.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00d5e00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815a060 0x7f00d81408f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:45.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.882+0000 7f00cd400640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f00d815e460 0x7f00d80b0ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:45.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00d6800640 1 -- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815a060 msgr2=0x7f00d81408f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00d6800640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815a060 0x7f00d81408f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00d6800640 1 -- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f00d815e460 msgr2=0x7f00d80b0ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00d6800640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f00d815e460 0x7f00d80b0ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00d6800640 1 -- 172.21.15.86:0/2438428267 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f00d815a940 con 0x7f00d80b1220 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00d6800640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f00d80b1220 0x7f00d814c730 secure :-1 s=READY pgs=1767 cs=0 l=1 rev1=1 crypto rx=0x7f00d000aa10 tx=0x7f00d0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00cea00640 1 -- 172.21.15.86:0/2438428267 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00d0043070 con 0x7f00d80b1220 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00cea00640 1 -- 172.21.15.86:0/2438428267 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f00d002da10 con 0x7f00d80b1220 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00cea00640 1 -- 172.21.15.86:0/2438428267 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00d003a030 con 0x7f00d80b1220 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f00d815ac20 con 0x7f00d80b1220 2024-03-06T07:37:45.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.883+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f00d815b1e0 con 0x7f00d80b1220 2024-03-06T07:37:45.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.885+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f00d80b16a0 con 0x7f00d80b1220 2024-03-06T07:37:45.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.885+0000 7f00cea00640 1 -- 172.21.15.86:0/2438428267 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f00d0046070 con 0x7f00d80b1220 2024-03-06T07:37:45.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.887+0000 7f00cea00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f00b406b500 0x7f00b406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:45.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.887+0000 7f00cea00640 1 -- 172.21.15.86:0/2438428267 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f00d00ab4f0 con 0x7f00d80b1220 2024-03-06T07:37:45.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.887+0000 7f00d5e00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f00b406b500 0x7f00b406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:45.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.888+0000 7f00d5e00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f00b406b500 0x7f00b406d9c0 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7f00c8009fd0 tx=0x7f00c800b040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:45.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:45.891+0000 7f00cea00640 1 -- 172.21.15.86:0/2438428267 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f00d007a290 con 0x7f00d80b1220 2024-03-06T07:37:46.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.153+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f00d8159ba0 con 0x7f00d80b1220 2024-03-06T07:37:46.156 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.154+0000 7f00cea00640 1 -- 172.21.15.86:0/2438428267 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f00d002dbe0 con 0x7f00d80b1220 2024-03-06T07:37:46.156 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:46.156 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:46.156 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f00b406b500 msgr2=0x7f00b406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f00b406b500 0x7f00b406d9c0 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7f00c8009fd0 tx=0x7f00c800b040 comp rx=0 tx=0).stop 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f00d80b1220 msgr2=0x7f00d814c730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f00d80b1220 0x7f00d814c730 secure :-1 s=READY pgs=1767 cs=0 l=1 rev1=1 crypto rx=0x7f00d000aa10 tx=0x7f00d0004730 comp rx=0 tx=0).stop 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 shutdown_connections 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f00b406b500 0x7f00b406d9c0 unknown :-1 s=CLOSED pgs=749 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f00d80b1220 0x7f00d814c730 unknown :-1 s=CLOSED pgs=1767 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f00d815e460 0x7f00d80b0ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 --2- 172.21.15.86:0/2438428267 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f00d815a060 0x7f00d81408f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 >> 172.21.15.86:0/2438428267 conn(0x7f00d80b1be0 msgr2=0x7f00d80b3dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:46.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 shutdown_connections 2024-03-06T07:37:46.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:46.157+0000 7f00dca00640 1 -- 172.21.15.86:0/2438428267 wait complete. 2024-03-06T07:37:46.389 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:46.389 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:47.389 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:47.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.601+0000 7f8053200640 1 Processor -- start 2024-03-06T07:37:47.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.602+0000 7f8053200640 1 -- start start 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8053200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8053200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f804c0cd860 0x7f804c06afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8053200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f804c06b500 0x7f804c161c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8053200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f804c06b970 con 0x7f804c06b500 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8053200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f804c0b5010 con 0x7f804c0cd860 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8053200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f804c0b52f0 con 0x7f804c15e070 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8051200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f804c06b500 0x7f804c161c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8051200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f804c06b500 0x7f804c161c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56484/0 (socket says 172.21.15.86:56484) 2024-03-06T07:37:47.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.603+0000 7f8051200640 1 -- 172.21.15.86:0/3523136322 learned_addr learned my addr 172.21.15.86:0/3523136322 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:47.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f804be00640 1 --2- 172.21.15.86:0/3523136322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:47.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f8043e00640 1 --2- 172.21.15.86:0/3523136322 >> v2:172.21.15.194:3300/0 conn(0x7f804c0cd860 0x7f804c06afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:47.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f8051200640 1 -- 172.21.15.86:0/3523136322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 msgr2=0x7f804c066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:47.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f8051200640 1 --2- 172.21.15.86:0/3523136322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c066d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f8051200640 1 -- 172.21.15.86:0/3523136322 >> v2:172.21.15.194:3300/0 conn(0x7f804c0cd860 msgr2=0x7f804c06afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:47.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f8051200640 1 --2- 172.21.15.86:0/3523136322 >> v2:172.21.15.194:3300/0 conn(0x7f804c0cd860 0x7f804c06afc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f8051200640 1 -- 172.21.15.86:0/3523136322 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f804c0b55d0 con 0x7f804c06b500 2024-03-06T07:37:47.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.604+0000 7f8043e00640 1 --2- 172.21.15.86:0/3523136322 >> v2:172.21.15.194:3300/0 conn(0x7f804c0cd860 0x7f804c06afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:47.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.606+0000 7f8051200640 1 --2- 172.21.15.86:0/3523136322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c161c30 secure :-1 s=READY pgs=1768 cs=0 l=1 rev1=1 crypto rx=0x7f804c070b40 tx=0x7f804402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=229e05945e88ff96 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:47.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.606+0000 7f804b400640 1 -- 172.21.15.86:0/3523136322 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f804402ca70 con 0x7f804c06b500 2024-03-06T07:37:47.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.606+0000 7f804b400640 1 -- 172.21.15.86:0/3523136322 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f804402cc10 con 0x7f804c06b500 2024-03-06T07:37:47.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.606+0000 7f804b400640 1 -- 172.21.15.86:0/3523136322 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80440343f0 con 0x7f804c06b500 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 -- 172.21.15.86:0/3523136322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 msgr2=0x7f804c161c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 --2- 172.21.15.86:0/3523136322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c161c30 secure :-1 s=READY pgs=1768 cs=0 l=1 rev1=1 crypto rx=0x7f804c070b40 tx=0x7f804402bed0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 -- 172.21.15.86:0/3523136322 shutdown_connections 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 --2- 172.21.15.86:0/3523136322 >> v2:172.21.15.194:3300/0 conn(0x7f804c0cd860 0x7f804c06afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 --2- 172.21.15.86:0/3523136322 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c161c30 unknown :-1 s=CLOSED pgs=1768 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 --2- 172.21.15.86:0/3523136322 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 -- 172.21.15.86:0/3523136322 >> 172.21.15.86:0/3523136322 conn(0x7f804c0b1e10 msgr2=0x7f804c0b2220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 -- 172.21.15.86:0/3523136322 shutdown_connections 2024-03-06T07:37:47.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.607+0000 7f8053200640 1 -- 172.21.15.86:0/3523136322 wait complete. 2024-03-06T07:37:47.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.610+0000 7f8053200640 1 Processor -- start 2024-03-06T07:37:47.612 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.610+0000 7f8053200640 1 -- start start 2024-03-06T07:37:47.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f8053200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c0ad6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:47.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f8053200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f804c0cd860 0x7f804c1564b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f8053200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c0ce170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f8053200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f804c15e450 con 0x7f804c06b500 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f8053200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f804c0b5b90 con 0x7f804c0cd860 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f8053200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f804c067240 con 0x7f804c15e070 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f804be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c0ad6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f804be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c0ad6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56486/0 (socket says 172.21.15.86:56486) 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.612+0000 7f804be00640 1 -- 172.21.15.86:0/3028329445 learned_addr learned my addr 172.21.15.86:0/3028329445 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f8051200640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c0ce170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f804be00640 1 -- 172.21.15.86:0/3028329445 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 msgr2=0x7f804c0ce170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f804be00640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c0ce170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f804be00640 1 -- 172.21.15.86:0/3028329445 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f804c0cd860 msgr2=0x7f804c1564b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f804be00640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f804c0cd860 0x7f804c1564b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.615 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f804be00640 1 -- 172.21.15.86:0/3028329445 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f804c0c3e70 con 0x7f804c06b500 2024-03-06T07:37:47.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f804be00640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c0ad6d0 secure :-1 s=READY pgs=1769 cs=0 l=1 rev1=1 crypto rx=0x7f803c00c7f0 tx=0x7f803c00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:47.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.613+0000 7f8048c00640 1 -- 172.21.15.86:0/3028329445 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f803c015070 con 0x7f804c06b500 2024-03-06T07:37:47.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.614+0000 7f8048c00640 1 -- 172.21.15.86:0/3028329445 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f803c004240 con 0x7f804c06b500 2024-03-06T07:37:47.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.614+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f804c0c4110 con 0x7f804c06b500 2024-03-06T07:37:47.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.614+0000 7f8048c00640 1 -- 172.21.15.86:0/3028329445 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f803c007620 con 0x7f804c06b500 2024-03-06T07:37:47.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.614+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f804c0c49b0 con 0x7f804c06b500 2024-03-06T07:37:47.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.615+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f804c0c42e0 con 0x7f804c06b500 2024-03-06T07:37:47.617 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.616+0000 7f8048c00640 1 -- 172.21.15.86:0/3028329445 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f803c0079d0 con 0x7f804c06b500 2024-03-06T07:37:47.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.616+0000 7f8048c00640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f802806b500 0x7f802806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:47.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.617+0000 7f8048c00640 1 -- 172.21.15.86:0/3028329445 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f803c08a340 con 0x7f804c06b500 2024-03-06T07:37:47.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.617+0000 7f8043e00640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f802806b500 0x7f802806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:47.619 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.618+0000 7f8043e00640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f802806b500 0x7f802806d9c0 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7f8034006b10 tx=0x7f8034008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:47.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.621+0000 7f8048c00640 1 -- 172.21.15.86:0/3028329445 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f803c0590e0 con 0x7f804c06b500 2024-03-06T07:37:47.677 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:47.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.855+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f804c0b2ae0 con 0x7f804c06b500 2024-03-06T07:37:47.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.855+0000 7f8048c00640 1 -- 172.21.15.86:0/3028329445 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f803c0589e0 con 0x7f804c06b500 2024-03-06T07:37:47.857 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:47.857 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:47.858 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f802806b500 msgr2=0x7f802806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f802806b500 0x7f802806d9c0 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7f8034006b10 tx=0x7f8034008040 comp rx=0 tx=0).stop 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 msgr2=0x7f804c0ad6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c0ad6d0 secure :-1 s=READY pgs=1769 cs=0 l=1 rev1=1 crypto rx=0x7f803c00c7f0 tx=0x7f803c00ccb0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 shutdown_connections 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f802806b500 0x7f802806d9c0 unknown :-1 s=CLOSED pgs=750 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f804c15e070 0x7f804c0ce170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f804c0cd860 0x7f804c1564b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 --2- 172.21.15.86:0/3028329445 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f804c06b500 0x7f804c0ad6d0 unknown :-1 s=CLOSED pgs=1769 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 >> 172.21.15.86:0/3028329445 conn(0x7f804c0b1e10 msgr2=0x7f804c15be70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 shutdown_connections 2024-03-06T07:37:47.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:47.859+0000 7f8053200640 1 -- 172.21.15.86:0/3028329445 wait complete. 2024-03-06T07:37:48.081 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:48.081 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:49.082 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:49.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.295+0000 7f7898e00640 1 Processor -- start 2024-03-06T07:37:49.297 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.295+0000 7f7898e00640 1 -- start start 2024-03-06T07:37:49.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.297+0000 7f7898e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7894161580 0x7f78940d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:49.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.297+0000 7f7898e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f78940d1e90 0x7f78940d2360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:49.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.297+0000 7f7898e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7894161960 con 0x7f78940d28a0 2024-03-06T07:37:49.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.297+0000 7f7898e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f78940cf6f0 con 0x7f78940d1e90 2024-03-06T07:37:49.298 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.297+0000 7f7898e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f78940cf9d0 con 0x7f7894161580 2024-03-06T07:37:49.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.297+0000 7f7893400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7894161580 0x7f78940d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:49.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.297+0000 7f7893400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7894161580 0x7f78940d1950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:59162/0 (socket says 172.21.15.86:59162) 2024-03-06T07:37:49.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.298+0000 7f7893400640 1 -- 172.21.15.86:0/2150872470 learned_addr learned my addr 172.21.15.86:0/2150872470 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:49.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.298+0000 7f7892a00640 1 --2- 172.21.15.86:0/2150872470 >> v2:172.21.15.194:3300/0 conn(0x7f78940d1e90 0x7f78940d2360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:49.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.298+0000 7f7892a00640 1 -- 172.21.15.86:0/2150872470 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7894161580 msgr2=0x7f78940d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:49.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.298+0000 7f7892a00640 1 --2- 172.21.15.86:0/2150872470 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7894161580 0x7f78940d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.298+0000 7f7892a00640 1 -- 172.21.15.86:0/2150872470 >> v1:172.21.15.194:6789/0 conn(0x7f78940d28a0 legacy=0x7f78940d06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:49.300 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.298+0000 7f7892a00640 1 -- 172.21.15.86:0/2150872470 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78940cfcb0 con 0x7f78940d1e90 2024-03-06T07:37:49.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.300+0000 7f7892a00640 1 --2- 172.21.15.86:0/2150872470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78940d1e90 0x7f78940d2360 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f78800097a0 tx=0x7f788002b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f483159441f3075f server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:49.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.300+0000 7f7892000640 1 -- 172.21.15.86:0/2150872470 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7880035020 con 0x7f78940d1e90 2024-03-06T07:37:49.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.300+0000 7f7892000640 1 -- 172.21.15.86:0/2150872470 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7880031830 con 0x7f78940d1e90 2024-03-06T07:37:49.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.300+0000 7f7892000640 1 -- 172.21.15.86:0/2150872470 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7880031c00 con 0x7f78940d1e90 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 -- 172.21.15.86:0/2150872470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78940d1e90 msgr2=0x7f78940d2360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 --2- 172.21.15.86:0/2150872470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78940d1e90 0x7f78940d2360 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f78800097a0 tx=0x7f788002b8d0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 -- 172.21.15.86:0/2150872470 shutdown_connections 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 --2- 172.21.15.86:0/2150872470 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f78940d1e90 0x7f78940d2360 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 --2- 172.21.15.86:0/2150872470 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7894161580 0x7f78940d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 -- 172.21.15.86:0/2150872470 >> 172.21.15.86:0/2150872470 conn(0x7f78940d3d80 msgr2=0x7f78940d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 -- 172.21.15.86:0/2150872470 shutdown_connections 2024-03-06T07:37:49.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.301+0000 7f7898e00640 1 -- 172.21.15.86:0/2150872470 wait complete. 2024-03-06T07:37:49.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.303+0000 7f7898e00640 1 Processor -- start 2024-03-06T07:37:49.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.303+0000 7f7898e00640 1 -- start start 2024-03-06T07:37:49.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7898e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78940d1e90 0x7f78941553c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:49.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7898e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78940d28a0 0x7f78940c7410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:49.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7898e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7894161580 0x7f78940c96d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:49.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7898e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78940cf3a0 con 0x7f78940d1e90 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7898e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78940b39a0 con 0x7f7894161580 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7898e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f78940b4ed0 con 0x7f78940d28a0 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7893400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78940d1e90 0x7f78941553c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7893400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78940d1e90 0x7f78941553c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56496/0 (socket says 172.21.15.86:56496) 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7893400640 1 -- 172.21.15.86:0/1441359674 learned_addr learned my addr 172.21.15.86:0/1441359674 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7893e00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7894161580 0x7f78940c96d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.305+0000 7f7892a00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78940d28a0 0x7f78940c7410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.306+0000 7f7893400640 1 -- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78940d28a0 msgr2=0x7f78940c7410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.306+0000 7f7893400640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78940d28a0 0x7f78940c7410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.306+0000 7f7893400640 1 -- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7894161580 msgr2=0x7f78940c96d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.306+0000 7f7893400640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7894161580 0x7f78940c96d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.306+0000 7f7893400640 1 -- 172.21.15.86:0/1441359674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7894155900 con 0x7f78940d1e90 2024-03-06T07:37:49.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.306+0000 7f7893400640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78940d1e90 0x7f78941553c0 secure :-1 s=READY pgs=1770 cs=0 l=1 rev1=1 crypto rx=0x7f788400e840 tx=0x7f788400ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:49.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.307+0000 7f788b400640 1 -- 172.21.15.86:0/1441359674 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7884017070 con 0x7f78940d1e90 2024-03-06T07:37:49.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.307+0000 7f788b400640 1 -- 172.21.15.86:0/1441359674 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f788400c8e0 con 0x7f78940d1e90 2024-03-06T07:37:49.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.307+0000 7f788b400640 1 -- 172.21.15.86:0/1441359674 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f788400ccd0 con 0x7f78940d1e90 2024-03-06T07:37:49.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.307+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78940c7c60 con 0x7f78940d1e90 2024-03-06T07:37:49.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.307+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78940c8220 con 0x7f78940d1e90 2024-03-06T07:37:49.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.308+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f789404f330 con 0x7f78940d1e90 2024-03-06T07:37:49.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.309+0000 7f788b400640 1 -- 172.21.15.86:0/1441359674 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f788401a070 con 0x7f78940d1e90 2024-03-06T07:37:49.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.310+0000 7f788b400640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f787406b500 0x7f787406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:49.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.310+0000 7f788b400640 1 -- 172.21.15.86:0/1441359674 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f788408a1e0 con 0x7f78940d1e90 2024-03-06T07:37:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.310+0000 7f7892a00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f787406b500 0x7f787406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:49.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.311+0000 7f7892a00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f787406b500 0x7f787406d9c0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7f7880036000 tx=0x7f7880008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:49.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.314+0000 7f788b400640 1 -- 172.21.15.86:0/1441359674 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7884058ef0 con 0x7f78940d1e90 2024-03-06T07:37:49.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.552+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f78940d3b60 con 0x7f78940d1e90 2024-03-06T07:37:49.554 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.553+0000 7f788b400640 1 -- 172.21.15.86:0/1441359674 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f78840587f0 con 0x7f78940d1e90 2024-03-06T07:37:49.555 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:49.555 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:49.555 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:49.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f787406b500 msgr2=0x7f787406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:49.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f787406b500 0x7f787406d9c0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7f7880036000 tx=0x7f7880008030 comp rx=0 tx=0).stop 2024-03-06T07:37:49.558 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78940d1e90 msgr2=0x7f78941553c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78940d1e90 0x7f78941553c0 secure :-1 s=READY pgs=1770 cs=0 l=1 rev1=1 crypto rx=0x7f788400e840 tx=0x7f788400ed00 comp rx=0 tx=0).stop 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 shutdown_connections 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f787406b500 0x7f787406d9c0 unknown :-1 s=CLOSED pgs=751 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f7894161580 0x7f78940c96d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f78940d28a0 0x7f78940c7410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 --2- 172.21.15.86:0/1441359674 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f78940d1e90 0x7f78941553c0 unknown :-1 s=CLOSED pgs=1770 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.556+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 >> 172.21.15.86:0/1441359674 conn(0x7f78940d3d80 msgr2=0x7f78940cfe80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.557+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 shutdown_connections 2024-03-06T07:37:49.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:49.557+0000 7f7898e00640 1 -- 172.21.15.86:0/1441359674 wait complete. 2024-03-06T07:37:49.679 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:49.779 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:49.779 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:50.780 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:50.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.983+0000 7fbe1a200640 1 Processor -- start 2024-03-06T07:37:50.985 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.984+0000 7fbe1a200640 1 -- start start 2024-03-06T07:37:50.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe1a200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe140cf370 0x7fbe1406d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:50.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe1a200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe140b2b30 con 0x7fbe140cc2a0 2024-03-06T07:37:50.987 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe1a200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe140b2e10 con 0x7fbe1415a940 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe1a200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbe140b28e0 con 0x7fbe140cf370 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe0be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe140cf370 0x7fbe1406d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe0be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fbe140cf370 0x7fbe1406d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56510/0 (socket says 172.21.15.86:56510) 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe19800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fbe140cc2a0 0x7fbe140cd460 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:39336/0 (socket says 172.21.15.86:39336) 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.985+0000 7fbe0be00640 1 -- 172.21.15.86:0/2175454650 learned_addr learned my addr 172.21.15.86:0/2175454650 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.986+0000 7fbe0be00640 1 -- 172.21.15.86:0/2175454650 >> v1:172.21.15.194:6789/0 conn(0x7fbe1415a940 legacy=0x7fbe140b45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.986+0000 7fbe0be00640 1 -- 172.21.15.86:0/2175454650 >> v1:172.21.15.86:6789/0 conn(0x7fbe140cc2a0 legacy=0x7fbe140cd460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.986+0000 7fbe0be00640 1 -- 172.21.15.86:0/2175454650 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe1406ca50 con 0x7fbe140cf370 2024-03-06T07:37:50.988 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.986+0000 7fbe13e00640 1 -- 172.21.15.86:0/2175454650 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1563716594 0 0) 0x7fbe140b2b30 con 0x7fbe140cc2a0 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.987+0000 7fbe0be00640 1 --2- 172.21.15.86:0/2175454650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe140cf370 0x7fbe1406d250 secure :-1 s=READY pgs=1772 cs=0 l=1 rev1=1 crypto rx=0x7fbdfc009870 tx=0x7fbdfc02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=49fdc0fa3d1b08c2 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.987+0000 7fbe13e00640 1 -- 172.21.15.86:0/2175454650 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdfc034070 con 0x7fbe140cf370 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.987+0000 7fbe13e00640 1 -- 172.21.15.86:0/2175454650 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbdfc030860 con 0x7fbe140cf370 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.987+0000 7fbe13e00640 1 -- 172.21.15.86:0/2175454650 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdfc030c30 con 0x7fbe140cf370 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.988+0000 7fbe1a200640 1 -- 172.21.15.86:0/2175454650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe140cf370 msgr2=0x7fbe1406d250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.988+0000 7fbe1a200640 1 --2- 172.21.15.86:0/2175454650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe140cf370 0x7fbe1406d250 secure :-1 s=READY pgs=1772 cs=0 l=1 rev1=1 crypto rx=0x7fbdfc009870 tx=0x7fbdfc02b9a0 comp rx=0 tx=0).stop 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.988+0000 7fbe1a200640 1 -- 172.21.15.86:0/2175454650 shutdown_connections 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.988+0000 7fbe1a200640 1 --2- 172.21.15.86:0/2175454650 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe140cf370 0x7fbe1406d250 unknown :-1 s=CLOSED pgs=1772 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:50.990 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.988+0000 7fbe1a200640 1 -- 172.21.15.86:0/2175454650 >> 172.21.15.86:0/2175454650 conn(0x7fbe1406afc0 msgr2=0x7fbe1406b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:50.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.988+0000 7fbe1a200640 1 -- 172.21.15.86:0/2175454650 shutdown_connections 2024-03-06T07:37:50.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.988+0000 7fbe1a200640 1 -- 172.21.15.86:0/2175454650 wait complete. 2024-03-06T07:37:50.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.990+0000 7fbe1a200640 1 Processor -- start 2024-03-06T07:37:50.991 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.990+0000 7fbe1a200640 1 -- start start 2024-03-06T07:37:50.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe1a200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe140cc2a0 0x7fbe141f7690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:50.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe1a200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe140cf370 0x7fbe14071b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:50.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe1a200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe1415a940 0x7fbe14072050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:50.994 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe1a200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe1406d790 con 0x7fbe1415a940 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe1a200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe140b30f0 con 0x7fbe140cf370 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe1a200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe141e4520 con 0x7fbe140cc2a0 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe19800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe1415a940 0x7fbe14072050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe19800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe1415a940 0x7fbe14072050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56516/0 (socket says 172.21.15.86:56516) 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe19800640 1 -- 172.21.15.86:0/906828008 learned_addr learned my addr 172.21.15.86:0/906828008 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe18e00640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe140cc2a0 0x7fbe141f7690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe0be00640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe140cf370 0x7fbe14071b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe19800640 1 -- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe140cc2a0 msgr2=0x7fbe141f7690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe19800640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe140cc2a0 0x7fbe141f7690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe19800640 1 -- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe140cf370 msgr2=0x7fbe14071b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:50.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.991+0000 7fbe19800640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe140cf370 0x7fbe14071b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:50.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.992+0000 7fbe19800640 1 -- 172.21.15.86:0/906828008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe141e4800 con 0x7fbe1415a940 2024-03-06T07:37:50.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.992+0000 7fbe19800640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe1415a940 0x7fbe14072050 secure :-1 s=READY pgs=1773 cs=0 l=1 rev1=1 crypto rx=0x7fbe0c00c7a0 tx=0x7fbe0c00cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:50.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.992+0000 7fbe11600640 1 -- 172.21.15.86:0/906828008 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe0c018070 con 0x7fbe1415a940 2024-03-06T07:37:50.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.992+0000 7fbe11600640 1 -- 172.21.15.86:0/906828008 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe0c00a640 con 0x7fbe1415a940 2024-03-06T07:37:50.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.992+0000 7fbe11600640 1 -- 172.21.15.86:0/906828008 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe0c00aa70 con 0x7fbe1415a940 2024-03-06T07:37:50.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.992+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbe141f6e90 con 0x7fbe1415a940 2024-03-06T07:37:50.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.992+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe141f7bd0 con 0x7fbe1415a940 2024-03-06T07:37:50.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.993+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbe1404f330 con 0x7fbe1415a940 2024-03-06T07:37:50.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.994+0000 7fbe11600640 1 -- 172.21.15.86:0/906828008 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbe0c00ac10 con 0x7fbe1415a940 2024-03-06T07:37:50.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.995+0000 7fbe11600640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbdf006b4d0 0x7fbdf006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:50.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.995+0000 7fbe11600640 1 -- 172.21.15.86:0/906828008 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbe0c089820 con 0x7fbe1415a940 2024-03-06T07:37:50.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.996+0000 7fbe18e00640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbdf006b4d0 0x7fbdf006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:50.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:50.996+0000 7fbe18e00640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbdf006b4d0 0x7fbdf006d990 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7fbe04006fd0 tx=0x7fbe04008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:51.002 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.000+0000 7fbe11600640 1 -- 172.21.15.86:0/906828008 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbe0c0586f0 con 0x7fbe1415a940 2024-03-06T07:37:51.234 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.232+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbe141e49d0 con 0x7fbe1415a940 2024-03-06T07:37:51.235 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.233+0000 7fbe11600640 1 -- 172.21.15.86:0/906828008 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fbe0c057ff0 con 0x7fbe1415a940 2024-03-06T07:37:51.235 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:51.236 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:51.236 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbdf006b4d0 msgr2=0x7fbdf006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbdf006b4d0 0x7fbdf006d990 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7fbe04006fd0 tx=0x7fbe04008040 comp rx=0 tx=0).stop 2024-03-06T07:37:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe1415a940 msgr2=0x7fbe14072050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe1415a940 0x7fbe14072050 secure :-1 s=READY pgs=1773 cs=0 l=1 rev1=1 crypto rx=0x7fbe0c00c7a0 tx=0x7fbe0c00cc60 comp rx=0 tx=0).stop 2024-03-06T07:37:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 shutdown_connections 2024-03-06T07:37:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fbdf006b4d0 0x7fbdf006d990 unknown :-1 s=CLOSED pgs=752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fbe1415a940 0x7fbe14072050 unknown :-1 s=CLOSED pgs=1773 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fbe140cf370 0x7fbe14071b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 --2- 172.21.15.86:0/906828008 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fbe140cc2a0 0x7fbe141f7690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 >> 172.21.15.86:0/906828008 conn(0x7fbe1406afc0 msgr2=0x7fbe14156a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 shutdown_connections 2024-03-06T07:37:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:51.236+0000 7fbe1a200640 1 -- 172.21.15.86:0/906828008 wait complete. 2024-03-06T07:37:51.465 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:51.465 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:51.680 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:52.466 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:52.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.670+0000 7fa96d600640 1 Processor -- start 2024-03-06T07:37:52.672 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.670+0000 7fa96d600640 1 -- start start 2024-03-06T07:37:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.671+0000 7fa96d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa968161580 0x7fa9680d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.671+0000 7fa96d600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fa9680d1e90 0x7fa9680d2360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:52.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.671+0000 7fa96d600640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa9680d28a0 0x7fa9680d06e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.671+0000 7fa96d600640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa968161960 con 0x7fa9680d28a0 2024-03-06T07:37:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.671+0000 7fa96d600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa9680cf6f0 con 0x7fa9680d1e90 2024-03-06T07:37:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.671+0000 7fa96d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9680cf9d0 con 0x7fa968161580 2024-03-06T07:37:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa9680d28a0 0x7fa9680d06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa9680d28a0 0x7fa9680d06e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56528/0 (socket says 172.21.15.86:56528) 2024-03-06T07:37:52.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 -- 172.21.15.86:0/616210563 learned_addr learned my addr 172.21.15.86:0/616210563 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa967e00640 1 --2- 172.21.15.86:0/616210563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa968161580 0x7fa9680d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 -- 172.21.15.86:0/616210563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa968161580 msgr2=0x7fa9680d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa967400640 1 --2- 172.21.15.86:0/616210563 >> v2:172.21.15.194:3300/0 conn(0x7fa9680d1e90 0x7fa9680d2360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 --2- 172.21.15.86:0/616210563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa968161580 0x7fa9680d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 -- 172.21.15.86:0/616210563 >> v2:172.21.15.194:3300/0 conn(0x7fa9680d1e90 msgr2=0x7fa9680d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:52.675 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 --2- 172.21.15.86:0/616210563 >> v2:172.21.15.194:3300/0 conn(0x7fa9680d1e90 0x7fa9680d2360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa96cc00640 1 -- 172.21.15.86:0/616210563 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9680cfcb0 con 0x7fa9680d28a0 2024-03-06T07:37:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.672+0000 7fa967e00640 1 --2- 172.21.15.86:0/616210563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa968161580 0x7fa9680d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.673+0000 7fa96cc00640 1 --2- 172.21.15.86:0/616210563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9680d28a0 0x7fa9680d06e0 secure :-1 s=READY pgs=1774 cs=0 l=1 rev1=1 crypto rx=0x7fa96808c8e0 tx=0x7fa96002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=73561f5fe8f4f04d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa966a00640 1 -- 172.21.15.86:0/616210563 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa96002ca70 con 0x7fa9680d28a0 2024-03-06T07:37:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa966a00640 1 -- 172.21.15.86:0/616210563 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa96002cc10 con 0x7fa9680d28a0 2024-03-06T07:37:52.676 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa966a00640 1 -- 172.21.15.86:0/616210563 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9600343f0 con 0x7fa9680d28a0 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 -- 172.21.15.86:0/616210563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9680d28a0 msgr2=0x7fa9680d06e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 --2- 172.21.15.86:0/616210563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9680d28a0 0x7fa9680d06e0 secure :-1 s=READY pgs=1774 cs=0 l=1 rev1=1 crypto rx=0x7fa96808c8e0 tx=0x7fa96002bed0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 -- 172.21.15.86:0/616210563 shutdown_connections 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 --2- 172.21.15.86:0/616210563 >> v2:172.21.15.194:3300/0 conn(0x7fa9680d1e90 0x7fa9680d2360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 --2- 172.21.15.86:0/616210563 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa9680d28a0 0x7fa9680d06e0 unknown :-1 s=CLOSED pgs=1774 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 --2- 172.21.15.86:0/616210563 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa968161580 0x7fa9680d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 -- 172.21.15.86:0/616210563 >> 172.21.15.86:0/616210563 conn(0x7fa9680d3d80 msgr2=0x7fa9680d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 -- 172.21.15.86:0/616210563 shutdown_connections 2024-03-06T07:37:52.677 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.674+0000 7fa96d600640 1 -- 172.21.15.86:0/616210563 wait complete. 2024-03-06T07:37:52.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.676+0000 7fa96d600640 1 Processor -- start 2024-03-06T07:37:52.678 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.676+0000 7fa96d600640 1 -- start start 2024-03-06T07:37:52.679 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96d600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9680d1e90 0x7fa968151f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96d600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9680d28a0 0x7fa968151410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96d600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa968161580 0x7fa9680b6970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96d600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9680cf3a0 con 0x7fa968161580 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96d600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9680b39a0 con 0x7fa9680d1e90 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96d600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9680d0c20 con 0x7fa9680d28a0 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa968161580 0x7fa9680b6970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96cc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa968161580 0x7fa9680b6970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56536/0 (socket says 172.21.15.86:56536) 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.677+0000 7fa96cc00640 1 -- 172.21.15.86:0/2680431081 learned_addr learned my addr 172.21.15.86:0/2680431081 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa967e00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9680d1e90 0x7fa968151f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa967400640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9680d28a0 0x7fa968151410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa96cc00640 1 -- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9680d28a0 msgr2=0x7fa968151410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:52.680 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa96cc00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9680d28a0 0x7fa968151410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa96cc00640 1 -- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9680d1e90 msgr2=0x7fa968151f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:52.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa96cc00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9680d1e90 0x7fa968151f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:52.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa96cc00640 1 -- 172.21.15.86:0/2680431081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9680b72c0 con 0x7fa968161580 2024-03-06T07:37:52.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa967e00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9680d1e90 0x7fa968151f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:52.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa967400640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9680d28a0 0x7fa968151410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:52.681 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa96cc00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa968161580 0x7fa9680b6970 secure :-1 s=READY pgs=1775 cs=0 l=1 rev1=1 crypto rx=0x7fa960036040 tx=0x7fa960004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:52.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa95fe00640 1 -- 172.21.15.86:0/2680431081 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa960043020 con 0x7fa968161580 2024-03-06T07:37:52.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa95fe00640 1 -- 172.21.15.86:0/2680431081 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa96003e450 con 0x7fa968161580 2024-03-06T07:37:52.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.678+0000 7fa95fe00640 1 -- 172.21.15.86:0/2680431081 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa960009510 con 0x7fa968161580 2024-03-06T07:37:52.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.679+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa9680b75a0 con 0x7fa968161580 2024-03-06T07:37:52.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.679+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9680b8530 con 0x7fa968161580 2024-03-06T07:37:52.682 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.680+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa96804f330 con 0x7fa968161580 2024-03-06T07:37:52.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.681+0000 7fa95fe00640 1 -- 172.21.15.86:0/2680431081 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa960005d10 con 0x7fa968161580 2024-03-06T07:37:52.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.682+0000 7fa95fe00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa94806b4d0 0x7fa94806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:52.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.682+0000 7fa95fe00640 1 -- 172.21.15.86:0/2680431081 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa9600ace10 con 0x7fa968161580 2024-03-06T07:37:52.683 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.682+0000 7fa967e00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa94806b4d0 0x7fa94806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:52.684 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.683+0000 7fa967e00640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa94806b4d0 0x7fa94806d990 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7fa958006fd0 tx=0x7fa958008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:52.688 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:52.686+0000 7fa95fe00640 1 -- 172.21.15.86:0/2680431081 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa96007bc50 con 0x7fa968161580 2024-03-06T07:37:53.034 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.032+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa9680d3b60 con 0x7fa968161580 2024-03-06T07:37:53.035 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.033+0000 7fa95fe00640 1 -- 172.21.15.86:0/2680431081 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fa96007b550 con 0x7fa968161580 2024-03-06T07:37:53.035 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:53.035 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:53.036 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa94806b4d0 msgr2=0x7fa94806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa94806b4d0 0x7fa94806d990 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7fa958006fd0 tx=0x7fa958008040 comp rx=0 tx=0).stop 2024-03-06T07:37:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa968161580 msgr2=0x7fa9680b6970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa968161580 0x7fa9680b6970 secure :-1 s=READY pgs=1775 cs=0 l=1 rev1=1 crypto rx=0x7fa960036040 tx=0x7fa960004a40 comp rx=0 tx=0).stop 2024-03-06T07:37:53.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 shutdown_connections 2024-03-06T07:37:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa94806b4d0 0x7fa94806d990 unknown :-1 s=CLOSED pgs=753 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa968161580 0x7fa9680b6970 unknown :-1 s=CLOSED pgs=1775 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa9680d28a0 0x7fa968151410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 --2- 172.21.15.86:0/2680431081 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa9680d1e90 0x7fa968151f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.036+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 >> 172.21.15.86:0/2680431081 conn(0x7fa9680d3d80 msgr2=0x7fa9680d1050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.037+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 shutdown_connections 2024-03-06T07:37:53.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:53.037+0000 7fa96d600640 1 -- 172.21.15.86:0/2680431081 wait complete. 2024-03-06T07:37:53.264 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:53.265 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:53.682 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:54.266 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:54.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.476+0000 7fc05d000640 1 Processor -- start 2024-03-06T07:37:54.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.476+0000 7fc05d000640 1 -- start start 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc05d000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d24c0 0x7fc0580d1cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc05d000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc0580b2de0 0x7fc0580d28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc05d000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc0580d07f0 0x7fc0580d0c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc05d000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc0580d4a00 con 0x7fc0580b2de0 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc05d000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc0580cf7e0 con 0x7fc0580d07f0 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc05d000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc0580d0080 con 0x7fc0580d24c0 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc04fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc0580b2de0 0x7fc0580d28a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc04fe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc0580b2de0 0x7fc0580d28a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56550/0 (socket says 172.21.15.86:56550) 2024-03-06T07:37:54.480 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.478+0000 7fc04fe00640 1 -- 172.21.15.86:0/2130114410 learned_addr learned my addr 172.21.15.86:0/2130114410 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:54.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.479+0000 7fc04fe00640 1 -- 172.21.15.86:0/2130114410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d24c0 msgr2=0x7fc0580d1cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:54.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.479+0000 7fc04fe00640 1 --2- 172.21.15.86:0/2130114410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d24c0 0x7fc0580d1cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.479+0000 7fc056800640 1 --2- 172.21.15.86:0/2130114410 >> v2:172.21.15.194:3300/0 conn(0x7fc0580d07f0 0x7fc0580d0c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:54.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.479+0000 7fc04fe00640 1 -- 172.21.15.86:0/2130114410 >> v2:172.21.15.194:3300/0 conn(0x7fc0580d07f0 msgr2=0x7fc0580d0c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:54.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.479+0000 7fc04fe00640 1 --2- 172.21.15.86:0/2130114410 >> v2:172.21.15.194:3300/0 conn(0x7fc0580d07f0 0x7fc0580d0c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.479+0000 7fc04fe00640 1 -- 172.21.15.86:0/2130114410 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0580d2de0 con 0x7fc0580b2de0 2024-03-06T07:37:54.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.480+0000 7fc04fe00640 1 --2- 172.21.15.86:0/2130114410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580b2de0 0x7fc0580d28a0 secure :-1 s=READY pgs=1776 cs=0 l=1 rev1=1 crypto rx=0x7fc0580ce760 tx=0x7fc04002b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6ff89aa51500ddcb server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:54.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.480+0000 7fc055400640 1 -- 172.21.15.86:0/2130114410 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc04002e070 con 0x7fc0580b2de0 2024-03-06T07:37:54.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.480+0000 7fc055400640 1 -- 172.21.15.86:0/2130114410 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc040037070 con 0x7fc0580b2de0 2024-03-06T07:37:54.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.480+0000 7fc055400640 1 -- 172.21.15.86:0/2130114410 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc040031a00 con 0x7fc0580b2de0 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 -- 172.21.15.86:0/2130114410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580b2de0 msgr2=0x7fc0580d28a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 --2- 172.21.15.86:0/2130114410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580b2de0 0x7fc0580d28a0 secure :-1 s=READY pgs=1776 cs=0 l=1 rev1=1 crypto rx=0x7fc0580ce760 tx=0x7fc04002b880 comp rx=0 tx=0).stop 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 -- 172.21.15.86:0/2130114410 shutdown_connections 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 --2- 172.21.15.86:0/2130114410 >> v2:172.21.15.194:3300/0 conn(0x7fc0580d07f0 0x7fc0580d0c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 --2- 172.21.15.86:0/2130114410 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580b2de0 0x7fc0580d28a0 unknown :-1 s=CLOSED pgs=1776 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 --2- 172.21.15.86:0/2130114410 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d24c0 0x7fc0580d1cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 -- 172.21.15.86:0/2130114410 >> 172.21.15.86:0/2130114410 conn(0x7fc0580d3ee0 msgr2=0x7fc0580d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 -- 172.21.15.86:0/2130114410 shutdown_connections 2024-03-06T07:37:54.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.481+0000 7fc05d000640 1 -- 172.21.15.86:0/2130114410 wait complete. 2024-03-06T07:37:54.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.483+0000 7fc05d000640 1 Processor -- start 2024-03-06T07:37:54.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.483+0000 7fc05d000640 1 -- start start 2024-03-06T07:37:54.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.484+0000 7fc05d000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0580b2de0 0x7fc0580b8810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:54.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.484+0000 7fc05d000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d07f0 0x7fc0580cd8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:54.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.484+0000 7fc05d000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580d24c0 0x7fc05814ce00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.484+0000 7fc05d000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0580cfd10 con 0x7fc0580d24c0 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.484+0000 7fc05d000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0580cfa30 con 0x7fc0580b2de0 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.484+0000 7fc05d000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc05815e5b0 con 0x7fc0580d07f0 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.484+0000 7fc056800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580d24c0 0x7fc05814ce00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580d24c0 0x7fc05814ce00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:56556/0 (socket says 172.21.15.86:56556) 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 -- 172.21.15.86:0/3894498418 learned_addr learned my addr 172.21.15.86:0/3894498418 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc04fe00640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d07f0 0x7fc0580cd8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc055e00640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0580b2de0 0x7fc0580b8810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 -- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d07f0 msgr2=0x7fc0580cd8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d07f0 0x7fc0580cd8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 -- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0580b2de0 msgr2=0x7fc0580b8810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0580b2de0 0x7fc0580b8810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 -- 172.21.15.86:0/3894498418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0580b8ea0 con 0x7fc0580d24c0 2024-03-06T07:37:54.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.485+0000 7fc056800640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580d24c0 0x7fc05814ce00 secure :-1 s=READY pgs=1777 cs=0 l=1 rev1=1 crypto rx=0x7fc05000cc60 tx=0x7fc05000a5c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:54.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.486+0000 7fc04e000640 1 -- 172.21.15.86:0/3894498418 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc050003990 con 0x7fc0580d24c0 2024-03-06T07:37:54.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.486+0000 7fc04e000640 1 -- 172.21.15.86:0/3894498418 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc050003b30 con 0x7fc0580d24c0 2024-03-06T07:37:54.488 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.486+0000 7fc04e000640 1 -- 172.21.15.86:0/3894498418 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc050004020 con 0x7fc0580d24c0 2024-03-06T07:37:54.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.486+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0580b9180 con 0x7fc0580d24c0 2024-03-06T07:37:54.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.486+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc05814d340 con 0x7fc0580d24c0 2024-03-06T07:37:54.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.487+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc05804f330 con 0x7fc0580d24c0 2024-03-06T07:37:54.489 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.488+0000 7fc04e000640 1 -- 172.21.15.86:0/3894498418 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc0500041c0 con 0x7fc0580d24c0 2024-03-06T07:37:54.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.489+0000 7fc04e000640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc03406b4d0 0x7fc03406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:54.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.489+0000 7fc04e000640 1 -- 172.21.15.86:0/3894498418 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc05008a7d0 con 0x7fc0580d24c0 2024-03-06T07:37:54.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.489+0000 7fc055e00640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc03406b4d0 0x7fc03406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:54.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.490+0000 7fc055e00640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc03406b4d0 0x7fc03406d990 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7fc048009890 tx=0x7fc048008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:54.495 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.494+0000 7fc04e000640 1 -- 172.21.15.86:0/3894498418 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc050014030 con 0x7fc0580d24c0 2024-03-06T07:37:54.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.728+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc0580d22a0 con 0x7fc0580d24c0 2024-03-06T07:37:54.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.728+0000 7fc04e000640 1 -- 172.21.15.86:0/3894498418 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fc050059570 con 0x7fc0580d24c0 2024-03-06T07:37:54.730 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:54.730 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:54.730 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc03406b4d0 msgr2=0x7fc03406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc03406b4d0 0x7fc03406d990 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7fc048009890 tx=0x7fc048008040 comp rx=0 tx=0).stop 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580d24c0 msgr2=0x7fc05814ce00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580d24c0 0x7fc05814ce00 secure :-1 s=READY pgs=1777 cs=0 l=1 rev1=1 crypto rx=0x7fc05000cc60 tx=0x7fc05000a5c0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 shutdown_connections 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc03406b4d0 0x7fc03406d990 unknown :-1 s=CLOSED pgs=754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc0580d24c0 0x7fc05814ce00 unknown :-1 s=CLOSED pgs=1777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc0580d07f0 0x7fc0580cd8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 --2- 172.21.15.86:0/3894498418 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc0580b2de0 0x7fc0580b8810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 >> 172.21.15.86:0/3894498418 conn(0x7fc0580d3ee0 msgr2=0x7fc0580b31e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:54.733 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 shutdown_connections 2024-03-06T07:37:54.734 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:54.731+0000 7fc05d000640 1 -- 172.21.15.86:0/3894498418 wait complete. 2024-03-06T07:37:54.955 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:54.955 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:55.529 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:37:55.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:37:55.684 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:55.956 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:56.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.166+0000 7f210dc00640 1 Processor -- start 2024-03-06T07:37:56.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.166+0000 7f210dc00640 1 -- start start 2024-03-06T07:37:56.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.167+0000 7f210dc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:56.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.167+0000 7f210dc00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f21080b4680 0x7f210806e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:56.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.167+0000 7f210dc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f210815b090 con 0x7f2108072340 2024-03-06T07:37:56.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f210dc00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21080b2a30 con 0x7f21080b4680 2024-03-06T07:37:56.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f210dc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21080b4360 con 0x7f210815acb0 2024-03-06T07:37:56.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f2106000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f21080b4680 0x7f210806e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:56.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f2106a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:56.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f2106000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f21080b4680 0x7f210806e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:52186/0 (socket says 172.21.15.86:52186) 2024-03-06T07:37:56.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f2107400640 1 --1- >> v1:172.21.15.194:6789/0 conn(0x7f2108072340 0x7f21080ccfb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.194:6789/0 says I am v1:172.21.15.86:41956/0 (socket says 172.21.15.86:41956) 2024-03-06T07:37:56.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f2106a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080b37d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3301/0 says I am v2:172.21.15.86:49792/0 (socket says 172.21.15.86:49792) 2024-03-06T07:37:56.171 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.168+0000 7f2106000640 1 -- 172.21.15.86:0/2398198574 learned_addr learned my addr 172.21.15.86:0/2398198574 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:56.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.169+0000 7f2105600640 1 -- 172.21.15.86:0/2398198574 <== mon.1 v1:172.21.15.194:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3350238833 0 0) 0x7f210815b090 con 0x7f2108072340 2024-03-06T07:37:56.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.169+0000 7f2106a00640 1 -- 172.21.15.86:0/2398198574 >> v2:172.21.15.194:3300/0 conn(0x7f21080b4680 msgr2=0x7f210806e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:56.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.169+0000 7f2106a00640 1 --2- 172.21.15.86:0/2398198574 >> v2:172.21.15.194:3300/0 conn(0x7f21080b4680 0x7f210806e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.169+0000 7f2106a00640 1 -- 172.21.15.86:0/2398198574 >> v1:172.21.15.194:6789/0 conn(0x7f2108072340 legacy=0x7f21080ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:56.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.169+0000 7f2106a00640 1 -- 172.21.15.86:0/2398198574 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21080b2cf0 con 0x7f210815acb0 2024-03-06T07:37:56.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.170+0000 7f2106a00640 1 --2- 172.21.15.86:0/2398198574 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080b37d0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f20f80097a0 tx=0x7f20f802b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eb23c81747f268 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:56.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.171+0000 7f2105600640 1 -- 172.21.15.86:0/2398198574 <== mon.2 v2:172.21.15.194:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20f8035020 con 0x7f210815acb0 2024-03-06T07:37:56.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.171+0000 7f2105600640 1 -- 172.21.15.86:0/2398198574 <== mon.2 v2:172.21.15.194:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20f8031830 con 0x7f210815acb0 2024-03-06T07:37:56.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.171+0000 7f2105600640 1 -- 172.21.15.86:0/2398198574 <== mon.2 v2:172.21.15.194:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20f8031c00 con 0x7f210815acb0 2024-03-06T07:37:56.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 -- 172.21.15.86:0/2398198574 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 msgr2=0x7f21080b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:56.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 --2- 172.21.15.86:0/2398198574 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080b37d0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f20f80097a0 tx=0x7f20f802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 -- 172.21.15.86:0/2398198574 shutdown_connections 2024-03-06T07:37:56.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 --2- 172.21.15.86:0/2398198574 >> v2:172.21.15.194:3300/0 conn(0x7f21080b4680 0x7f210806e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 --2- 172.21.15.86:0/2398198574 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080b37d0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 -- 172.21.15.86:0/2398198574 >> 172.21.15.86:0/2398198574 conn(0x7f21080b1730 msgr2=0x7f21080b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:56.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 -- 172.21.15.86:0/2398198574 shutdown_connections 2024-03-06T07:37:56.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.172+0000 7f210dc00640 1 -- 172.21.15.86:0/2398198574 wait complete. 2024-03-06T07:37:56.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.174+0000 7f210dc00640 1 Processor -- start 2024-03-06T07:37:56.176 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.175+0000 7f210dc00640 1 -- start start 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.176+0000 7f210dc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2108072340 0x7f210815d490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.176+0000 7f210dc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21080b4680 0x7f21080c8760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.176+0000 7f210dc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080c49a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.176+0000 7f210dc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21080b3d10 con 0x7f21080b4680 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.176+0000 7f210dc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21080b32b0 con 0x7f2108072340 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.176+0000 7f210dc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21080b4010 con 0x7f210815acb0 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21080b4680 0x7f21080c8760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21080b4680 0x7f21080c8760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38544/0 (socket says 172.21.15.86:38544) 2024-03-06T07:37:56.180 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 -- 172.21.15.86:0/4044314502 learned_addr learned my addr 172.21.15.86:0/4044314502 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106a00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2108072340 0x7f210815d490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 -- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 msgr2=0x7f21080c49a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080c49a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 -- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2108072340 msgr2=0x7f210815d490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2108072340 0x7f210815d490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 -- 172.21.15.86:0/4044314502 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21080c9140 con 0x7f21080b4680 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106a00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2108072340 0x7f210815d490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f2106000640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21080b4680 0x7f21080c8760 secure :-1 s=READY pgs=1778 cs=0 l=1 rev1=1 crypto rx=0x7f20f4009fd0 tx=0x7f20f400ed40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.177+0000 7f20fea00640 1 -- 172.21.15.86:0/4044314502 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20f4017070 con 0x7f21080b4680 2024-03-06T07:37:56.181 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.178+0000 7f20fea00640 1 -- 172.21.15.86:0/4044314502 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20f400c940 con 0x7f21080b4680 2024-03-06T07:37:56.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.178+0000 7f20fea00640 1 -- 172.21.15.86:0/4044314502 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20f400cd10 con 0x7f21080b4680 2024-03-06T07:37:56.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.178+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21080c9420 con 0x7f21080b4680 2024-03-06T07:37:56.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.178+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21080c99e0 con 0x7f21080b4680 2024-03-06T07:37:56.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.179+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f210804f330 con 0x7f21080b4680 2024-03-06T07:37:56.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.180+0000 7f20fea00640 1 -- 172.21.15.86:0/4044314502 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f20f401a070 con 0x7f21080b4680 2024-03-06T07:37:56.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.180+0000 7f20fea00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20e806b500 0x7f20e806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:56.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.181+0000 7f20fea00640 1 -- 172.21.15.86:0/4044314502 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f20f408a5b0 con 0x7f21080b4680 2024-03-06T07:37:56.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.181+0000 7f2106a00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20e806b500 0x7f20e806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:56.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.181+0000 7f2106a00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20e806b500 0x7f20e806d9c0 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f20f8036000 tx=0x7f20f8008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:56.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.190+0000 7f20fea00640 1 -- 172.21.15.86:0/4044314502 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f20f4059350 con 0x7f21080b4680 2024-03-06T07:37:56.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.424+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f210806ce60 con 0x7f21080b4680 2024-03-06T07:37:56.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.425+0000 7f20fea00640 1 -- 172.21.15.86:0/4044314502 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f20f4088e20 con 0x7f21080b4680 2024-03-06T07:37:56.427 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:56.427 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:56.428 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20e806b500 msgr2=0x7f20e806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20e806b500 0x7f20e806d9c0 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f20f8036000 tx=0x7f20f8008030 comp rx=0 tx=0).stop 2024-03-06T07:37:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21080b4680 msgr2=0x7f21080c8760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21080b4680 0x7f21080c8760 secure :-1 s=READY pgs=1778 cs=0 l=1 rev1=1 crypto rx=0x7f20f4009fd0 tx=0x7f20f400ed40 comp rx=0 tx=0).stop 2024-03-06T07:37:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 shutdown_connections 2024-03-06T07:37:56.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f20e806b500 0x7f20e806d9c0 unknown :-1 s=CLOSED pgs=755 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f210815acb0 0x7f21080c49a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f21080b4680 0x7f21080c8760 unknown :-1 s=CLOSED pgs=1778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 --2- 172.21.15.86:0/4044314502 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f2108072340 0x7f210815d490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 >> 172.21.15.86:0/4044314502 conn(0x7f21080b1730 msgr2=0x7f210814e200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 shutdown_connections 2024-03-06T07:37:56.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:56.428+0000 7f210dc00640 1 -- 172.21.15.86:0/4044314502 wait complete. 2024-03-06T07:37:56.650 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:56.650 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:57.651 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:57.686 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:57.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.862+0000 7feeacc00640 1 Processor -- start 2024-03-06T07:37:57.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.862+0000 7feeacc00640 1 -- start start 2024-03-06T07:37:57.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.863+0000 7feeacc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 0x7feea815fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.863+0000 7feeacc00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feea806cef0 con 0x7feea8070b10 2024-03-06T07:37:57.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.864+0000 7feeacc00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feea806d1d0 con 0x7feea8144c00 2024-03-06T07:37:57.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.864+0000 7feeacc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feea8160370 con 0x7feea815f9d0 2024-03-06T07:37:57.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.864+0000 7feea6800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7feea8070b10 0x7feea8070f60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55326/0 (socket says 172.21.15.86:55326) 2024-03-06T07:37:57.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.864+0000 7feea6800640 1 -- 172.21.15.86:0/2040701440 learned_addr learned my addr 172.21.15.86:0/2040701440 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:57.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.864+0000 7fee9d400640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 0x7feea815fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.864+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1793420530 0 0) 0x7feea806cef0 con 0x7feea8070b10 2024-03-06T07:37:57.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.864+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fee8c003520 con 0x7feea8070b10 2024-03-06T07:37:57.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1061933170 0 0) 0x7fee8c003520 con 0x7feea8070b10 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 msgr2=0x7feea815fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 0x7feea815fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 >> v1:172.21.15.194:6789/0 conn(0x7feea8144c00 legacy=0x7feea806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feea806d4b0 con 0x7feea8070b10 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7feea00019f0 con 0x7feea8070b10 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 >> v1:172.21.15.86:6789/0 conn(0x7feea8070b10 legacy=0x7feea8070f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 0x7fee8c0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fee8c003f60 0x7fee8c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7fee8c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee8c00ba70 con 0x7fee8c004d40 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee8c00bc20 con 0x7fee8c003f60 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fee8c00be20 con 0x7feea815f9d0 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.865+0000 7feea6800640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7fee8c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7fee9d400640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fee8c003f60 0x7fee8c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7feea5e00640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 0x7fee8c0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7feea6800640 1 -- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 msgr2=0x7fee8c0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:57.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7feea6800640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 0x7fee8c0039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:57.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7feea6800640 1 -- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fee8c003f60 msgr2=0x7fee8c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:57.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7feea6800640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fee8c003f60 0x7fee8c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:57.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7feea6800640 1 -- 172.21.15.86:0/2040701440 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fee8c003520 con 0x7fee8c004d40 2024-03-06T07:37:57.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.866+0000 7feea5e00640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea815f9d0 0x7fee8c0039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:37:57.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.868+0000 7feea6800640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7fee8c0051c0 secure :-1 s=READY pgs=1780 cs=0 l=1 rev1=1 crypto rx=0x7feea000a850 tx=0x7fee8c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec4548afc03c95b8 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:57.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.868+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feea000a4a0 con 0x7fee8c004d40 2024-03-06T07:37:57.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.868+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feea0034650 con 0x7fee8c004d40 2024-03-06T07:37:57.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.868+0000 7feea5400640 1 -- 172.21.15.86:0/2040701440 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feea0034a80 con 0x7fee8c004d40 2024-03-06T07:37:57.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.868+0000 7feeacc00640 1 -- 172.21.15.86:0/2040701440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 msgr2=0x7fee8c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:57.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.868+0000 7feeacc00640 1 --2- 172.21.15.86:0/2040701440 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7fee8c0051c0 secure :-1 s=READY pgs=1780 cs=0 l=1 rev1=1 crypto rx=0x7feea000a850 tx=0x7fee8c005830 comp rx=0 tx=0).stop 2024-03-06T07:37:57.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.868+0000 7feea6800640 1 -- 172.21.15.86:0/2040701440 reap_dead start 2024-03-06T07:37:57.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.869+0000 7feeacc00640 1 -- 172.21.15.86:0/2040701440 shutdown_connections 2024-03-06T07:37:57.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.869+0000 7feeacc00640 1 -- 172.21.15.86:0/2040701440 >> 172.21.15.86:0/2040701440 conn(0x7feea80b1730 msgr2=0x7feea80b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:57.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.869+0000 7feeacc00640 1 -- 172.21.15.86:0/2040701440 shutdown_connections 2024-03-06T07:37:57.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.869+0000 7feeacc00640 1 -- 172.21.15.86:0/2040701440 wait complete. 2024-03-06T07:37:57.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.870+0000 7feeacc00640 1 Processor -- start 2024-03-06T07:37:57.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.870+0000 7feeacc00640 1 -- start start 2024-03-06T07:37:57.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.871+0000 7feeacc00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7feea815b670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.871+0000 7feeacc00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feea81f16f0 0x7feea80a9450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.871+0000 7feeacc00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea80a9990 0x7feea80ce060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.871+0000 7feeacc00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feea806cb50 con 0x7fee8c004d40 2024-03-06T07:37:57.873 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.871+0000 7feeacc00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feea8160730 con 0x7feea81f16f0 2024-03-06T07:37:57.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.871+0000 7feeacc00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feea806ccd0 con 0x7feea80a9990 2024-03-06T07:37:57.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7feea815b670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7feea815b670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38556/0 (socket says 172.21.15.86:38556) 2024-03-06T07:37:57.874 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 -- 172.21.15.86:0/1066426982 learned_addr learned my addr 172.21.15.86:0/1066426982 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7fee9d400640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feea81f16f0 0x7feea80a9450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea6800640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea80a9990 0x7feea80ce060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 -- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea80a9990 msgr2=0x7feea80ce060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea80a9990 0x7feea80ce060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 -- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feea81f16f0 msgr2=0x7feea80a9450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feea81f16f0 0x7feea80a9450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 -- 172.21.15.86:0/1066426982 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feea80a9ea0 con 0x7fee8c004d40 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.872+0000 7feea5e00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7feea815b670 secure :-1 s=READY pgs=1781 cs=0 l=1 rev1=1 crypto rx=0x7fee9000d7a0 tx=0x7fee9000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:57.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.873+0000 7fee9ea00640 1 -- 172.21.15.86:0/1066426982 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee90005270 con 0x7fee8c004d40 2024-03-06T07:37:57.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.873+0000 7fee9ea00640 1 -- 172.21.15.86:0/1066426982 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fee90017070 con 0x7fee8c004d40 2024-03-06T07:37:57.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.873+0000 7fee9ea00640 1 -- 172.21.15.86:0/1066426982 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee9000b6b0 con 0x7fee8c004d40 2024-03-06T07:37:57.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.873+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feea80aa180 con 0x7fee8c004d40 2024-03-06T07:37:57.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.873+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feea80aa740 con 0x7fee8c004d40 2024-03-06T07:37:57.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.874+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feea804f330 con 0x7fee8c004d40 2024-03-06T07:37:57.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.875+0000 7fee9ea00640 1 -- 172.21.15.86:0/1066426982 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fee9000b850 con 0x7fee8c004d40 2024-03-06T07:37:57.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.876+0000 7fee9ea00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fee8406b500 0x7fee8406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:57.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.876+0000 7fee9ea00640 1 -- 172.21.15.86:0/1066426982 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fee9008a250 con 0x7fee8c004d40 2024-03-06T07:37:57.877 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.876+0000 7fee9d400640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fee8406b500 0x7fee8406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:57.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.877+0000 7fee9d400640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fee8406b500 0x7fee8406d9c0 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7feea80af4f0 tx=0x7fee98008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:57.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:57.881+0000 7fee9ea00640 1 -- 172.21.15.86:0/1066426982 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fee90058ff0 con 0x7fee8c004d40 2024-03-06T07:37:58.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.176+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7feea806ec80 con 0x7fee8c004d40 2024-03-06T07:37:58.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.177+0000 7fee9ea00640 1 -- 172.21.15.86:0/1066426982 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fee9001ae40 con 0x7fee8c004d40 2024-03-06T07:37:58.179 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:58.179 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:58.179 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fee8406b500 msgr2=0x7fee8406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fee8406b500 0x7fee8406d9c0 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7feea80af4f0 tx=0x7fee98008040 comp rx=0 tx=0).stop 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 msgr2=0x7feea815b670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7feea815b670 secure :-1 s=READY pgs=1781 cs=0 l=1 rev1=1 crypto rx=0x7fee9000d7a0 tx=0x7fee9000dc60 comp rx=0 tx=0).stop 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 shutdown_connections 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fee8406b500 0x7fee8406d9c0 unknown :-1 s=CLOSED pgs=756 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7feea80a9990 0x7feea80ce060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7feea81f16f0 0x7feea80a9450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 --2- 172.21.15.86:0/1066426982 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fee8c004d40 0x7feea815b670 unknown :-1 s=CLOSED pgs=1781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:58.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.180+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 >> 172.21.15.86:0/1066426982 conn(0x7feea80b1730 msgr2=0x7feea80cef20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:58.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.181+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 shutdown_connections 2024-03-06T07:37:58.183 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:58.181+0000 7feeacc00640 1 -- 172.21.15.86:0/1066426982 wait complete. 2024-03-06T07:37:58.402 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:37:58.402 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:37:59.404 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:37:59.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.614+0000 7fc6cf200640 1 Processor -- start 2024-03-06T07:37:59.616 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.615+0000 7fc6cf200640 1 -- start start 2024-03-06T07:37:59.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.616+0000 7fc6cf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d01583d0 0x7fc6d0072820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.616+0000 7fc6cf200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc6d006afc0 0x7fc6d006b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.616+0000 7fc6cf200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc6d006b920 con 0x7fc6d0161a80 2024-03-06T07:37:59.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.616+0000 7fc6cf200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc6d0072d60 con 0x7fc6d006afc0 2024-03-06T07:37:59.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.616+0000 7fc6cf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc6d0071a80 con 0x7fc6d01583d0 2024-03-06T07:37:59.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.616+0000 7fc6ce800640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc6d0161a80 0x7fc6d0161f10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55332/0 (socket says 172.21.15.86:55332) 2024-03-06T07:37:59.618 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.616+0000 7fc6ce800640 1 -- 172.21.15.86:0/2711189913 learned_addr learned my addr 172.21.15.86:0/2711189913 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:37:59.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1852130758 0 0) 0x7fc6d006b920 con 0x7fc6d0161a80 2024-03-06T07:37:59.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cd400640 1 --2- 172.21.15.86:0/2711189913 >> v2:172.21.15.194:3300/0 conn(0x7fc6d006afc0 0x7fc6d006b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.620 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cde00640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d01583d0 0x7fc6d0072820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc6b4003520 con 0x7fc6d0161a80 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2413172460 0 0) 0x7fc6b4003520 con 0x7fc6d0161a80 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d01583d0 msgr2=0x7fc6d0072820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d01583d0 0x7fc6d0072820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 >> v2:172.21.15.194:3300/0 conn(0x7fc6d006afc0 msgr2=0x7fc6d006b3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 --2- 172.21.15.86:0/2711189913 >> v2:172.21.15.194:3300/0 conn(0x7fc6d006afc0 0x7fc6d006b3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6d01627d0 con 0x7fc6d0161a80 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc6c8001670 con 0x7fc6d0161a80 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 >> v1:172.21.15.86:6789/0 conn(0x7fc6d0161a80 legacy=0x7fc6d0161f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d01583d0 0x7fc6b40039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4003fe0 0x7fc6b4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.617+0000 7fc6cca00640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6b4004d40 0x7fc6b40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6d0072d60 con 0x7fc6b4004d40 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6d0071a80 con 0x7fc6b4003fe0 2024-03-06T07:37:59.621 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc6b400bac0 con 0x7fc6d01583d0 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6cd400640 1 --2- 172.21.15.86:0/2711189913 >> v2:172.21.15.194:3300/0 conn(0x7fc6d006afc0 0x7fc6d006b3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6ce800640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6b4004d40 0x7fc6b40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6ce800640 1 -- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d01583d0 msgr2=0x7fc6b40039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6ce800640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d01583d0 0x7fc6b40039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6ce800640 1 -- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4003fe0 msgr2=0x7fc6b4004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6cd400640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4003fe0 0x7fc6b4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6ce800640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4003fe0 0x7fc6b4004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.622 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.618+0000 7fc6ce800640 1 -- 172.21.15.86:0/2711189913 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6b4003520 con 0x7fc6b4004d40 2024-03-06T07:37:59.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.619+0000 7fc6ce800640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6b4004d40 0x7fc6b40051c0 secure :-1 s=READY pgs=1783 cs=0 l=1 rev1=1 crypto rx=0x7fc6c800c240 tx=0x7fc6b4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4e919fc83adcdf00 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:59.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6c8007d30 con 0x7fc6b4004d40 2024-03-06T07:37:59.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc6c8030070 con 0x7fc6b4004d40 2024-03-06T07:37:59.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6cca00640 1 -- 172.21.15.86:0/2711189913 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6c802cda0 con 0x7fc6b4004d40 2024-03-06T07:37:59.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6cf200640 1 -- 172.21.15.86:0/2711189913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6b4004d40 msgr2=0x7fc6b40051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:59.623 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6cf200640 1 --2- 172.21.15.86:0/2711189913 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6b4004d40 0x7fc6b40051c0 secure :-1 s=READY pgs=1783 cs=0 l=1 rev1=1 crypto rx=0x7fc6c800c240 tx=0x7fc6b4005830 comp rx=0 tx=0).stop 2024-03-06T07:37:59.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6ce800640 1 -- 172.21.15.86:0/2711189913 reap_dead start 2024-03-06T07:37:59.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6cf200640 1 -- 172.21.15.86:0/2711189913 shutdown_connections 2024-03-06T07:37:59.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.620+0000 7fc6cf200640 1 -- 172.21.15.86:0/2711189913 >> 172.21.15.86:0/2711189913 conn(0x7fc6d006c0d0 msgr2=0x7fc6d006c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:59.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.621+0000 7fc6cf200640 1 -- 172.21.15.86:0/2711189913 shutdown_connections 2024-03-06T07:37:59.624 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.621+0000 7fc6cf200640 1 -- 172.21.15.86:0/2711189913 wait complete. 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.622+0000 7fc6cf200640 1 Processor -- start 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.622+0000 7fc6cf200640 1 -- start start 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.623+0000 7fc6cf200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4004d40 0x7fc6d015a9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.623+0000 7fc6cf200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d015cce0 0x7fc6d0155520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.623+0000 7fc6cf200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6d00ae8b0 0x7fc6d00aed30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.623+0000 7fc6cf200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6d0072300 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.623+0000 7fc6cf200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6d0072020 con 0x7fc6b4004d40 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.623+0000 7fc6cf200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc6d01587b0 con 0x7fc6d015cce0 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6d00ae8b0 0x7fc6d00aed30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.625 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6d00ae8b0 0x7fc6d00aed30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38562/0 (socket says 172.21.15.86:38562) 2024-03-06T07:37:59.626 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 -- 172.21.15.86:0/2648015835 learned_addr learned my addr 172.21.15.86:0/2648015835 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6cde00640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4004d40 0x7fc6d015a9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6cd400640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d015cce0 0x7fc6d0155520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 -- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d015cce0 msgr2=0x7fc6d0155520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d015cce0 0x7fc6d0155520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 -- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4004d40 msgr2=0x7fc6d015a9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4004d40 0x7fc6d015a9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.624+0000 7fc6ce800640 1 -- 172.21.15.86:0/2648015835 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6d015d160 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.625+0000 7fc6ce800640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6d00ae8b0 0x7fc6d00aed30 secure :-1 s=READY pgs=1784 cs=0 l=1 rev1=1 crypto rx=0x7fc6c80017e0 tx=0x7fc6d0072ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.625+0000 7fc6c6000640 1 -- 172.21.15.86:0/2648015835 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6c8007d30 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.625+0000 7fc6c6000640 1 -- 172.21.15.86:0/2648015835 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc6c8004bf0 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.625+0000 7fc6c6000640 1 -- 172.21.15.86:0/2648015835 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6c8001c70 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.625+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc6d015b0c0 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.625+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc6d015b680 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.626+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc6d004f330 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.628 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.627+0000 7fc6c6000640 1 -- 172.21.15.86:0/2648015835 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc6c8007070 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.628+0000 7fc6c6000640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6ac06b4d0 0x7fc6ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:37:59.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.629+0000 7fc6c6000640 1 -- 172.21.15.86:0/2648015835 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc6c80aafe0 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.629+0000 7fc6cde00640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6ac06b4d0 0x7fc6ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:37:59.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.629+0000 7fc6cde00640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6ac06b4d0 0x7fc6ac06d990 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7fc6b80047b0 tx=0x7fc6b8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:37:59.634 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.633+0000 7fc6c6000640 1 -- 172.21.15.86:0/2648015835 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc6c8048030 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.688 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:37:59.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.866+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc6d00cde40 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.868 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.867+0000 7fc6c6000640 1 -- 172.21.15.86:0/2648015835 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fc6c8079cf0 con 0x7fc6d00ae8b0 2024-03-06T07:37:59.869 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:37:59.869 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:37:59.869 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:37:59.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.869+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6ac06b4d0 msgr2=0x7fc6ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:59.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.869+0000 7fc6cf200640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6ac06b4d0 0x7fc6ac06d990 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7fc6b80047b0 tx=0x7fc6b8008040 comp rx=0 tx=0).stop 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.869+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6d00ae8b0 msgr2=0x7fc6d00aed30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.869+0000 7fc6cf200640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6d00ae8b0 0x7fc6d00aed30 secure :-1 s=READY pgs=1784 cs=0 l=1 rev1=1 crypto rx=0x7fc6c80017e0 tx=0x7fc6d0072ee0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 shutdown_connections 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc6ac06b4d0 0x7fc6ac06d990 unknown :-1 s=CLOSED pgs=757 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc6d00ae8b0 0x7fc6d00aed30 unknown :-1 s=CLOSED pgs=1784 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc6d015cce0 0x7fc6d0155520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 --2- 172.21.15.86:0/2648015835 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc6b4004d40 0x7fc6d015a9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 >> 172.21.15.86:0/2648015835 conn(0x7fc6d006c0d0 msgr2=0x7fc6d0156090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 shutdown_connections 2024-03-06T07:37:59.872 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:37:59.870+0000 7fc6cf200640 1 -- 172.21.15.86:0/2648015835 wait complete. 2024-03-06T07:38:00.090 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:00.090 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:01.090 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:01.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.301+0000 7fa83d200640 1 Processor -- start 2024-03-06T07:38:01.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.301+0000 7fa83d200640 1 -- start start 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa83d200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa83815f9d0 0x7fa83815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa83d200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa83806cef0 con 0x7fa838144c00 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa83d200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa83806d1d0 con 0x7fa838070b10 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa83d200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa83806d4b0 con 0x7fa83815f9d0 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa82d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa83815f9d0 0x7fa83815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa82d400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa83815f9d0 0x7fa83815fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38574/0 (socket says 172.21.15.86:38574) 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa82d400640 1 -- 172.21.15.86:0/4193217736 learned_addr learned my addr 172.21.15.86:0/4193217736 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:01.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.303+0000 7fa835e00640 1 --1- 172.21.15.86:0/4193217736 >> v1:172.21.15.86:6789/0 conn(0x7fa838144c00 0x7fa83806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55336/0 (socket says 172.21.15.86:55336) 2024-03-06T07:38:01.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.304+0000 7fa82d400640 1 -- 172.21.15.86:0/4193217736 >> v1:172.21.15.194:6789/0 conn(0x7fa838070b10 legacy=0x7fa838070f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:01.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.304+0000 7fa82d400640 1 -- 172.21.15.86:0/4193217736 >> v1:172.21.15.86:6789/0 conn(0x7fa838144c00 legacy=0x7fa83806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:01.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.304+0000 7fa82d400640 1 -- 172.21.15.86:0/4193217736 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa83806d790 con 0x7fa83815f9d0 2024-03-06T07:38:01.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.305+0000 7fa82d400640 1 --2- 172.21.15.86:0/4193217736 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815fe60 secure :-1 s=READY pgs=1786 cs=0 l=1 rev1=1 crypto rx=0x7fa820009870 tx=0x7fa82002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b07bae98ce1b8572 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:01.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.305+0000 7fa835400640 1 -- 172.21.15.86:0/4193217736 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa820034070 con 0x7fa83815f9d0 2024-03-06T07:38:01.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.305+0000 7fa835400640 1 -- 172.21.15.86:0/4193217736 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa820030860 con 0x7fa83815f9d0 2024-03-06T07:38:01.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.305+0000 7fa835400640 1 -- 172.21.15.86:0/4193217736 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa820030c30 con 0x7fa83815f9d0 2024-03-06T07:38:01.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.306+0000 7fa83d200640 1 -- 172.21.15.86:0/4193217736 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 msgr2=0x7fa83815fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:01.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.306+0000 7fa83d200640 1 --2- 172.21.15.86:0/4193217736 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815fe60 secure :-1 s=READY pgs=1786 cs=0 l=1 rev1=1 crypto rx=0x7fa820009870 tx=0x7fa82002b9a0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.306+0000 7fa83d200640 1 -- 172.21.15.86:0/4193217736 shutdown_connections 2024-03-06T07:38:01.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.306+0000 7fa83d200640 1 --2- 172.21.15.86:0/4193217736 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815fe60 unknown :-1 s=CLOSED pgs=1786 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.306+0000 7fa83d200640 1 -- 172.21.15.86:0/4193217736 >> 172.21.15.86:0/4193217736 conn(0x7fa8380b1730 msgr2=0x7fa8380b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:01.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.306+0000 7fa83d200640 1 -- 172.21.15.86:0/4193217736 shutdown_connections 2024-03-06T07:38:01.308 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.306+0000 7fa83d200640 1 -- 172.21.15.86:0/4193217736 wait complete. 2024-03-06T07:38:01.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.308+0000 7fa83d200640 1 Processor -- start 2024-03-06T07:38:01.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.308+0000 7fa83d200640 1 -- start start 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa83d200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa838070b10 0x7fa83815b070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa83d200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa838144c00 0x7fa83815e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa83d200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815e800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa83d200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa83806ccd0 con 0x7fa83815f9d0 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa83d200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa83806cb50 con 0x7fa838070b10 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa83d200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa83806df90 con 0x7fa838144c00 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815e800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815e800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38580/0 (socket says 172.21.15.86:38580) 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 -- 172.21.15.86:0/3896164167 learned_addr learned my addr 172.21.15.86:0/3896164167 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:01.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa835e00640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa838070b10 0x7fa83815b070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa82d400640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa838144c00 0x7fa83815e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 -- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa838144c00 msgr2=0x7fa83815e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa838144c00 0x7fa83815e2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 -- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa838070b10 msgr2=0x7fa83815b070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa838070b10 0x7fa83815b070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.309+0000 7fa836800640 1 -- 172.21.15.86:0/3896164167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa83815f440 con 0x7fa83815f9d0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.310+0000 7fa836800640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815e800 secure :-1 s=READY pgs=1787 cs=0 l=1 rev1=1 crypto rx=0x7fa83000c390 tx=0x7fa83000c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.310+0000 7fa82ea00640 1 -- 172.21.15.86:0/3896164167 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa830017070 con 0x7fa83815f9d0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.310+0000 7fa82ea00640 1 -- 172.21.15.86:0/3896164167 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8300035f0 con 0x7fa83815f9d0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.310+0000 7fa82ea00640 1 -- 172.21.15.86:0/3896164167 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8300039e0 con 0x7fa83815f9d0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.310+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa83815ee80 con 0x7fa83815f9d0 2024-03-06T07:38:01.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.310+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8380b2c60 con 0x7fa83815f9d0 2024-03-06T07:38:01.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.311+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa83804f330 con 0x7fa83815f9d0 2024-03-06T07:38:01.314 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.312+0000 7fa82ea00640 1 -- 172.21.15.86:0/3896164167 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa830003b80 con 0x7fa83815f9d0 2024-03-06T07:38:01.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.314+0000 7fa82ea00640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa81406b4d0 0x7fa81406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:01.315 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.314+0000 7fa82ea00640 1 -- 172.21.15.86:0/3896164167 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa830089570 con 0x7fa83815f9d0 2024-03-06T07:38:01.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.314+0000 7fa835e00640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa81406b4d0 0x7fa81406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:01.316 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.315+0000 7fa835e00640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa81406b4d0 0x7fa81406d990 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7fa8280012e0 tx=0x7fa828009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:01.320 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.319+0000 7fa82ea00640 1 -- 172.21.15.86:0/3896164167 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa83815f440 con 0x7fa83815f9d0 2024-03-06T07:38:01.559 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.557+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa8380cf260 con 0x7fa83815f9d0 2024-03-06T07:38:01.560 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.558+0000 7fa82ea00640 1 -- 172.21.15.86:0/3896164167 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fa8300583a0 con 0x7fa83815f9d0 2024-03-06T07:38:01.560 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:01.560 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:01.560 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa81406b4d0 msgr2=0x7fa81406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa81406b4d0 0x7fa81406d990 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7fa8280012e0 tx=0x7fa828009040 comp rx=0 tx=0).stop 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 msgr2=0x7fa83815e800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815e800 secure :-1 s=READY pgs=1787 cs=0 l=1 rev1=1 crypto rx=0x7fa83000c390 tx=0x7fa83000c850 comp rx=0 tx=0).stop 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 shutdown_connections 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa81406b4d0 0x7fa81406d990 unknown :-1 s=CLOSED pgs=758 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa83815f9d0 0x7fa83815e800 unknown :-1 s=CLOSED pgs=1787 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa838144c00 0x7fa83815e2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 --2- 172.21.15.86:0/3896164167 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa838070b10 0x7fa83815b070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 >> 172.21.15.86:0/3896164167 conn(0x7fa8380b1730 msgr2=0x7fa83806e7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:01.564 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 shutdown_connections 2024-03-06T07:38:01.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:01.562+0000 7fa83d200640 1 -- 172.21.15.86:0/3896164167 wait complete. 2024-03-06T07:38:01.690 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:01.783 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:01.783 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:02.784 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:02.992 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.991+0000 7f4b8f200640 1 Processor -- start 2024-03-06T07:38:02.993 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.991+0000 7f4b8f200640 1 -- start start 2024-03-06T07:38:02.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.992+0000 7f4b8f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:02.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b8f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4b900d1e90 0x7f4b900d2360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:02.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b8f200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b90161960 con 0x7f4b900d28a0 2024-03-06T07:38:02.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b8f200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b900cf6f0 con 0x7f4b900d1e90 2024-03-06T07:38:02.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b8f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b900cf9d0 con 0x7f4b90161580 2024-03-06T07:38:02.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b87e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4b900d1e90 0x7f4b900d2360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:02.995 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b87e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f4b900d1e90 0x7f4b900d2360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38588/0 (socket says 172.21.15.86:38588) 2024-03-06T07:38:02.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b87e00640 1 -- 172.21.15.86:0/761269954 learned_addr learned my addr 172.21.15.86:0/761269954 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:02.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b8de00640 1 --2- 172.21.15.86:0/761269954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:02.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b87e00640 1 -- 172.21.15.86:0/761269954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 msgr2=0x7f4b900d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:02.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b87e00640 1 --2- 172.21.15.86:0/761269954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:02.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b87e00640 1 -- 172.21.15.86:0/761269954 >> v1:172.21.15.194:6789/0 conn(0x7f4b900d28a0 legacy=0x7f4b900d06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:02.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b87e00640 1 -- 172.21.15.86:0/761269954 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b900cfcb0 con 0x7f4b900d1e90 2024-03-06T07:38:02.996 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.993+0000 7f4b8de00640 1 --2- 172.21.15.86:0/761269954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:02.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.995+0000 7f4b87e00640 1 --2- 172.21.15.86:0/761269954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900d2360 secure :-1 s=READY pgs=1788 cs=0 l=1 rev1=1 crypto rx=0x7f4b78009870 tx=0x7f4b7802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e72b3645ef6a636 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:02.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.996+0000 7f4b8d400640 1 -- 172.21.15.86:0/761269954 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b78035020 con 0x7f4b900d1e90 2024-03-06T07:38:02.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.996+0000 7f4b8d400640 1 -- 172.21.15.86:0/761269954 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b78031640 con 0x7f4b900d1e90 2024-03-06T07:38:02.997 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.996+0000 7f4b8d400640 1 -- 172.21.15.86:0/761269954 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b7802d040 con 0x7f4b900d1e90 2024-03-06T07:38:02.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.996+0000 7f4b8f200640 1 -- 172.21.15.86:0/761269954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 msgr2=0x7f4b900d2360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:02.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.996+0000 7f4b8f200640 1 --2- 172.21.15.86:0/761269954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900d2360 secure :-1 s=READY pgs=1788 cs=0 l=1 rev1=1 crypto rx=0x7f4b78009870 tx=0x7f4b7802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:38:02.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.997+0000 7f4b8f200640 1 -- 172.21.15.86:0/761269954 shutdown_connections 2024-03-06T07:38:02.998 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.997+0000 7f4b8f200640 1 --2- 172.21.15.86:0/761269954 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900d2360 unknown :-1 s=CLOSED pgs=1788 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:02.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.997+0000 7f4b8f200640 1 --2- 172.21.15.86:0/761269954 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:02.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.997+0000 7f4b8f200640 1 -- 172.21.15.86:0/761269954 >> 172.21.15.86:0/761269954 conn(0x7f4b900d3d80 msgr2=0x7f4b900d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:02.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.997+0000 7f4b8f200640 1 -- 172.21.15.86:0/761269954 shutdown_connections 2024-03-06T07:38:02.999 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.997+0000 7f4b8f200640 1 -- 172.21.15.86:0/761269954 wait complete. 2024-03-06T07:38:03.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.998+0000 7f4b8f200640 1 Processor -- start 2024-03-06T07:38:03.000 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.999+0000 7f4b8f200640 1 -- start start 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:02.999+0000 7f4b8f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900b9530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8f200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b900d28a0 0x7f4b90152170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900c9cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8f200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b900cf3a0 con 0x7f4b900d1e90 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8f200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b900b39a0 con 0x7f4b900d28a0 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b900b4ed0 con 0x7f4b90161580 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900b9530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900b9530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38598/0 (socket says 172.21.15.86:38598) 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 -- 172.21.15.86:0/1294400406 learned_addr learned my addr 172.21.15.86:0/1294400406 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b87e00640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b900d28a0 0x7f4b90152170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8e800640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900c9cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 -- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 msgr2=0x7f4b900c9cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:03.003 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900c9cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 -- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b900d28a0 msgr2=0x7f4b90152170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b900d28a0 0x7f4b90152170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8de00640 1 -- 172.21.15.86:0/1294400406 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b901526b0 con 0x7f4b900d1e90 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b87e00640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b900d28a0 0x7f4b90152170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.000+0000 7f4b8e800640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900c9cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.001+0000 7f4b8de00640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900b9530 secure :-1 s=READY pgs=1789 cs=0 l=1 rev1=1 crypto rx=0x7f4b8000c820 tx=0x7f4b8000cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.001+0000 7f4b86000640 1 -- 172.21.15.86:0/1294400406 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b80015070 con 0x7f4b900d1e90 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.001+0000 7f4b86000640 1 -- 172.21.15.86:0/1294400406 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b80007a30 con 0x7f4b900d1e90 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.001+0000 7f4b86000640 1 -- 172.21.15.86:0/1294400406 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b80007de0 con 0x7f4b900d1e90 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.001+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b900c8c30 con 0x7f4b900d1e90 2024-03-06T07:38:03.004 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.001+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b900c91f0 con 0x7f4b900d1e90 2024-03-06T07:38:03.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.002+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b9004f330 con 0x7f4b900d1e90 2024-03-06T07:38:03.005 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.004+0000 7f4b86000640 1 -- 172.21.15.86:0/1294400406 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b80018070 con 0x7f4b900d1e90 2024-03-06T07:38:03.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.005+0000 7f4b86000640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b6c06b4d0 0x7f4b6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:03.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.005+0000 7f4b86000640 1 -- 172.21.15.86:0/1294400406 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4b8008a190 con 0x7f4b900d1e90 2024-03-06T07:38:03.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.005+0000 7f4b87e00640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b6c06b4d0 0x7f4b6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:03.007 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.006+0000 7f4b87e00640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b6c06b4d0 0x7f4b6c06d990 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f4b78036000 tx=0x7f4b78008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:03.012 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.010+0000 7f4b86000640 1 -- 172.21.15.86:0/1294400406 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b80058f30 con 0x7f4b900d1e90 2024-03-06T07:38:03.251 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.249+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4b900b3020 con 0x7f4b900d1e90 2024-03-06T07:38:03.252 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.250+0000 7f4b86000640 1 -- 172.21.15.86:0/1294400406 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f4b80058830 con 0x7f4b900d1e90 2024-03-06T07:38:03.252 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:03.252 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:03.252 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:03.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b6c06b4d0 msgr2=0x7f4b6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:03.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b6c06b4d0 0x7f4b6c06d990 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f4b78036000 tx=0x7f4b78008030 comp rx=0 tx=0).stop 2024-03-06T07:38:03.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 msgr2=0x7f4b900b9530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:03.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900b9530 secure :-1 s=READY pgs=1789 cs=0 l=1 rev1=1 crypto rx=0x7f4b8000c820 tx=0x7f4b8000cce0 comp rx=0 tx=0).stop 2024-03-06T07:38:03.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 shutdown_connections 2024-03-06T07:38:03.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4b6c06b4d0 0x7f4b6c06d990 unknown :-1 s=CLOSED pgs=759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:03.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4b90161580 0x7f4b900c9cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:03.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4b900d28a0 0x7f4b90152170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:03.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 --2- 172.21.15.86:0/1294400406 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4b900d1e90 0x7f4b900b9530 unknown :-1 s=CLOSED pgs=1789 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:03.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.253+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 >> 172.21.15.86:0/1294400406 conn(0x7f4b900d3d80 msgr2=0x7f4b900d0b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:03.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.254+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 shutdown_connections 2024-03-06T07:38:03.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:03.254+0000 7f4b8f200640 1 -- 172.21.15.86:0/1294400406 wait complete. 2024-03-06T07:38:03.482 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:03.482 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:03.692 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:04.483 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:04.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.706+0000 7fc95e000640 1 Processor -- start 2024-03-06T07:38:04.708 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.706+0000 7fc95e000640 1 -- start start 2024-03-06T07:38:04.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.707+0000 7fc95e000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc9580cea80 0x7fc9580d0b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.709 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.707+0000 7fc95e000640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9580ceec0 con 0x7fc9580d07b0 2024-03-06T07:38:04.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.708+0000 7fc95e000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9580d10d0 con 0x7fc9580d1c20 2024-03-06T07:38:04.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.708+0000 7fc95e000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc9580cfd50 con 0x7fc9580cea80 2024-03-06T07:38:04.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.708+0000 7fc957400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fc9580d07b0 0x7fc95815f110 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:55346/0 (socket says 172.21.15.86:55346) 2024-03-06T07:38:04.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.708+0000 7fc957400640 1 -- 172.21.15.86:0/2201464341 learned_addr learned my addr 172.21.15.86:0/2201464341 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:04.710 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.708+0000 7fc956a00640 1 --2- 172.21.15.86:0/2201464341 >> v2:172.21.15.194:3300/0 conn(0x7fc9580cea80 0x7fc9580d0b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.708+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 775735253 0 0) 0x7fc948001110 con 0x7fc9580d07b0 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.708+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc93c003520 con 0x7fc9580d07b0 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 478048474 0 0) 0x7fc93c003520 con 0x7fc9580d07b0 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 >> v2:172.21.15.194:3300/0 conn(0x7fc9580cea80 msgr2=0x7fc9580d0b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 --2- 172.21.15.86:0/2201464341 >> v2:172.21.15.194:3300/0 conn(0x7fc9580cea80 0x7fc9580d0b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 >> v1:172.21.15.194:6789/0 conn(0x7fc9580d1c20 legacy=0x7fc9580d20b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9580cf140 con 0x7fc9580d07b0 2024-03-06T07:38:04.712 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fc948001840 con 0x7fc9580d07b0 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 >> v1:172.21.15.86:6789/0 conn(0x7fc9580d07b0 legacy=0x7fc95815f110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc93c0039b0 0x7fc93c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c004800 0x7fc93c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc93c0051c0 0x7fc93c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9580cfd50 con 0x7fc93c0051c0 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc93c00c060 con 0x7fc93c0039b0 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc93c00c260 con 0x7fc93c004800 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.709+0000 7fc957e00640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc93c0051c0 0x7fc93c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc956a00640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c004800 0x7fc93c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc957400640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc93c0039b0 0x7fc93c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc957e00640 1 -- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c004800 msgr2=0x7fc93c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc957e00640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c004800 0x7fc93c004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc957e00640 1 -- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc93c0039b0 msgr2=0x7fc93c003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc957e00640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc93c0039b0 0x7fc93c003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.713 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc957e00640 1 -- 172.21.15.86:0/2201464341 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc93c003520 con 0x7fc93c0051c0 2024-03-06T07:38:04.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc956a00640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c004800 0x7fc93c004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:04.714 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.710+0000 7fc957400640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc93c0039b0 0x7fc93c003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.711+0000 7fc957e00640 1 -- 172.21.15.86:0/2201464341 reap_dead start 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.711+0000 7fc957e00640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc93c0051c0 0x7fc93c00b9c0 secure :-1 s=READY pgs=1791 cs=0 l=1 rev1=1 crypto rx=0x7fc950001e50 tx=0x7fc93c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=690304a3e640f75e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.712+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc950001a70 con 0x7fc93c0051c0 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.712+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc950003b90 con 0x7fc93c0051c0 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.712+0000 7fc956000640 1 -- 172.21.15.86:0/2201464341 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9500050d0 con 0x7fc93c0051c0 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.712+0000 7fc95e000640 1 -- 172.21.15.86:0/2201464341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc93c0051c0 msgr2=0x7fc93c00b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.712+0000 7fc95e000640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc93c0051c0 0x7fc93c00b9c0 secure :-1 s=READY pgs=1791 cs=0 l=1 rev1=1 crypto rx=0x7fc950001e50 tx=0x7fc93c005830 comp rx=0 tx=0).stop 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.713+0000 7fc95e000640 1 -- 172.21.15.86:0/2201464341 shutdown_connections 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.713+0000 7fc95e000640 1 --2- 172.21.15.86:0/2201464341 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc93c0051c0 0x7fc93c00b9c0 unknown :-1 s=CLOSED pgs=1791 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.713+0000 7fc95e000640 1 -- 172.21.15.86:0/2201464341 >> 172.21.15.86:0/2201464341 conn(0x7fc9580d3ee0 msgr2=0x7fc9580d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:04.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.713+0000 7fc95e000640 1 -- 172.21.15.86:0/2201464341 shutdown_connections 2024-03-06T07:38:04.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.713+0000 7fc95e000640 1 -- 172.21.15.86:0/2201464341 wait complete. 2024-03-06T07:38:04.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.715+0000 7fc95e000640 1 Processor -- start 2024-03-06T07:38:04.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.715+0000 7fc95e000640 1 -- start start 2024-03-06T07:38:04.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc95e000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c0051c0 0x7fc9580cafb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc95e000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9581f4fe0 0x7fc9581f3a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc95e000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9581f3fa0 0x7fc9580c9910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc95e000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9580cf9e0 con 0x7fc9581f3fa0 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc95e000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9580cf700 con 0x7fc9581f4fe0 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc95e000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc95815e390 con 0x7fc93c0051c0 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc957e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9581f3fa0 0x7fc9580c9910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc957e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9581f3fa0 0x7fc9580c9910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:38620/0 (socket says 172.21.15.86:38620) 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.717+0000 7fc957e00640 1 -- 172.21.15.86:0/4055091846 learned_addr learned my addr 172.21.15.86:0/4055091846 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc956a00640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9581f4fe0 0x7fc9581f3a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc957400640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c0051c0 0x7fc9580cafb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc957e00640 1 -- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c0051c0 msgr2=0x7fc9580cafb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc957e00640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c0051c0 0x7fc9580cafb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc957e00640 1 -- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9581f4fe0 msgr2=0x7fc9581f3a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc957e00640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9581f4fe0 0x7fc9581f3a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc957e00640 1 -- 172.21.15.86:0/4055091846 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9580cb920 con 0x7fc9581f3fa0 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc956a00640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9581f4fe0 0x7fc9581f3a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc957e00640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9581f3fa0 0x7fc9580c9910 secure :-1 s=READY pgs=1792 cs=0 l=1 rev1=1 crypto rx=0x7fc950001f70 tx=0x7fc9500014a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc94f400640 1 -- 172.21.15.86:0/4055091846 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc950007db0 con 0x7fc9581f3fa0 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc94f400640 1 -- 172.21.15.86:0/4055091846 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc950006d50 con 0x7fc9581f3fa0 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc94f400640 1 -- 172.21.15.86:0/4055091846 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9500396c0 con 0x7fc9581f3fa0 2024-03-06T07:38:04.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.718+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc9580cbc00 con 0x7fc9581f3fa0 2024-03-06T07:38:04.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.719+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc9580cc4a0 con 0x7fc9581f3fa0 2024-03-06T07:38:04.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.720+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc9580cbdd0 con 0x7fc9581f3fa0 2024-03-06T07:38:04.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.721+0000 7fc94f400640 1 -- 172.21.15.86:0/4055091846 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc950039a70 con 0x7fc9581f3fa0 2024-03-06T07:38:04.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.722+0000 7fc94f400640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc93806b4d0 0x7fc93806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:04.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.722+0000 7fc94f400640 1 -- 172.21.15.86:0/4055091846 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc9500ac000 con 0x7fc9581f3fa0 2024-03-06T07:38:04.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.722+0000 7fc957400640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc93806b4d0 0x7fc93806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:04.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.723+0000 7fc957400640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc93806b4d0 0x7fc93806d990 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7fc948000f60 tx=0x7fc948009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:04.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.726+0000 7fc94f400640 1 -- 172.21.15.86:0/4055091846 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc95003e070 con 0x7fc9581f3fa0 2024-03-06T07:38:04.960 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.959+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc95804f330 con 0x7fc9581f3fa0 2024-03-06T07:38:04.961 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.959+0000 7fc94f400640 1 -- 172.21.15.86:0/4055091846 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fc950033030 con 0x7fc9581f3fa0 2024-03-06T07:38:04.961 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:04.962 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:04.962 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:04.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.962+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc93806b4d0 msgr2=0x7fc93806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:04.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.962+0000 7fc95e000640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc93806b4d0 0x7fc93806d990 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7fc948000f60 tx=0x7fc948009040 comp rx=0 tx=0).stop 2024-03-06T07:38:04.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.962+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9581f3fa0 msgr2=0x7fc9580c9910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:04.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.962+0000 7fc95e000640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9581f3fa0 0x7fc9580c9910 secure :-1 s=READY pgs=1792 cs=0 l=1 rev1=1 crypto rx=0x7fc950001f70 tx=0x7fc9500014a0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 shutdown_connections 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc93806b4d0 0x7fc93806d990 unknown :-1 s=CLOSED pgs=760 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc9581f3fa0 0x7fc9580c9910 unknown :-1 s=CLOSED pgs=1792 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc9581f4fe0 0x7fc9581f3a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 --2- 172.21.15.86:0/4055091846 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc93c0051c0 0x7fc9580cafb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 >> 172.21.15.86:0/4055091846 conn(0x7fc9580d3ee0 msgr2=0x7fc958157080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 shutdown_connections 2024-03-06T07:38:04.965 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:04.963+0000 7fc95e000640 1 -- 172.21.15.86:0/4055091846 wait complete. 2024-03-06T07:38:05.182 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:05.183 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:05.694 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:06.184 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:06.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.395+0000 7fc630a00640 1 Processor -- start 2024-03-06T07:38:06.398 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.395+0000 7fc630a00640 1 -- start start 2024-03-06T07:38:06.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc630a00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fc62c144c00 0x7fc62c06e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:06.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc630a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc62c15f9d0 0x7fc62c15fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:06.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc630a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc62c06ee70 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.400 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc630a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc62c06cef0 con 0x7fc62c15f9d0 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc630a00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc62c06d1d0 con 0x7fc62c144c00 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc629400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc62c15f9d0 0x7fc62c15fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc629400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fc62c15f9d0 0x7fc62c15fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33660/0 (socket says 172.21.15.86:33660) 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.397+0000 7fc629400640 1 -- 172.21.15.86:0/2369598204 learned_addr learned my addr 172.21.15.86:0/2369598204 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.398+0000 7fc629e00640 1 --2- 172.21.15.86:0/2369598204 >> v2:172.21.15.194:3300/0 conn(0x7fc62c144c00 0x7fc62c06e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.398+0000 7fc629400640 1 -- 172.21.15.86:0/2369598204 >> v2:172.21.15.194:3300/0 conn(0x7fc62c144c00 msgr2=0x7fc62c06e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.398+0000 7fc629400640 1 --2- 172.21.15.86:0/2369598204 >> v2:172.21.15.194:3300/0 conn(0x7fc62c144c00 0x7fc62c06e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.398+0000 7fc629400640 1 -- 172.21.15.86:0/2369598204 >> v1:172.21.15.194:6789/0 conn(0x7fc62c0ce0a0 legacy=0x7fc62c0ce4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:06.401 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.398+0000 7fc629400640 1 -- 172.21.15.86:0/2369598204 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc62c06d4b0 con 0x7fc62c15f9d0 2024-03-06T07:38:06.402 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.400+0000 7fc629400640 1 --2- 172.21.15.86:0/2369598204 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c15f9d0 0x7fc62c15fe60 secure :-1 s=READY pgs=1793 cs=0 l=1 rev1=1 crypto rx=0x7fc62c0b25f0 tx=0x7fc61802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6f411af2799d9bf6 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:06.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.400+0000 7fc628a00640 1 -- 172.21.15.86:0/2369598204 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc618035020 con 0x7fc62c15f9d0 2024-03-06T07:38:06.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.400+0000 7fc628a00640 1 -- 172.21.15.86:0/2369598204 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc618031870 con 0x7fc62c15f9d0 2024-03-06T07:38:06.403 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.400+0000 7fc628a00640 1 -- 172.21.15.86:0/2369598204 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc618031c40 con 0x7fc62c15f9d0 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 -- 172.21.15.86:0/2369598204 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c15f9d0 msgr2=0x7fc62c15fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 --2- 172.21.15.86:0/2369598204 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c15f9d0 0x7fc62c15fe60 secure :-1 s=READY pgs=1793 cs=0 l=1 rev1=1 crypto rx=0x7fc62c0b25f0 tx=0x7fc61802b910 comp rx=0 tx=0).stop 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 -- 172.21.15.86:0/2369598204 shutdown_connections 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 --2- 172.21.15.86:0/2369598204 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c15f9d0 0x7fc62c15fe60 unknown :-1 s=CLOSED pgs=1793 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 --2- 172.21.15.86:0/2369598204 >> v2:172.21.15.194:3300/0 conn(0x7fc62c144c00 0x7fc62c06e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 -- 172.21.15.86:0/2369598204 >> 172.21.15.86:0/2369598204 conn(0x7fc62c0b1730 msgr2=0x7fc62c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 -- 172.21.15.86:0/2369598204 shutdown_connections 2024-03-06T07:38:06.404 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.401+0000 7fc630a00640 1 -- 172.21.15.86:0/2369598204 wait complete. 2024-03-06T07:38:06.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.402+0000 7fc630a00640 1 Processor -- start 2024-03-06T07:38:06.405 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.403+0000 7fc630a00640 1 -- start start 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.403+0000 7fc630a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c0ce0a0 0x7fc62c1434e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.403+0000 7fc630a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc62c144c00 0x7fc62c1f6ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.403+0000 7fc630a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc62c15f9d0 0x7fc62c13d3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.403+0000 7fc630a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc62c06cb50 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.403+0000 7fc630a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc62c160730 con 0x7fc62c15f9d0 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.403+0000 7fc630a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc62c06ccd0 con 0x7fc62c144c00 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c0ce0a0 0x7fc62c1434e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:06.408 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c0ce0a0 0x7fc62c1434e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33670/0 (socket says 172.21.15.86:33670) 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 -- 172.21.15.86:0/203009213 learned_addr learned my addr 172.21.15.86:0/203009213 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 -- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc62c144c00 msgr2=0x7fc62c1f6ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc62a800640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc62c15f9d0 0x7fc62c13d3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629400640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc62c144c00 0x7fc62c1f6ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc62c144c00 0x7fc62c1f6ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 -- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc62c15f9d0 msgr2=0x7fc62c13d3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc62c15f9d0 0x7fc62c13d3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc629e00640 1 -- 172.21.15.86:0/203009213 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc62c13fcc0 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.404+0000 7fc62a800640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc62c15f9d0 0x7fc62c13d3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:06.409 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.405+0000 7fc629e00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c0ce0a0 0x7fc62c1434e0 secure :-1 s=READY pgs=1794 cs=0 l=1 rev1=1 crypto rx=0x7fc61c00d800 tx=0x7fc61c00dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:06.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.405+0000 7fc622000640 1 -- 172.21.15.86:0/203009213 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc61c00b6c0 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.405+0000 7fc622000640 1 -- 172.21.15.86:0/203009213 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc61c019070 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.405+0000 7fc622000640 1 -- 172.21.15.86:0/203009213 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc61c00ba80 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.405+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc62c13ffa0 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.410 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.405+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc62c13d8e0 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.406+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc62c04f330 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.411 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.407+0000 7fc622000640 1 -- 172.21.15.86:0/203009213 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc61c00bc20 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.408+0000 7fc622000640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc60c06b500 0x7fc60c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:06.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.408+0000 7fc622000640 1 -- 172.21.15.86:0/203009213 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc61c08a710 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.408+0000 7fc629400640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc60c06b500 0x7fc60c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:06.412 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.409+0000 7fc629400640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc60c06b500 0x7fc60c06d9c0 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7fc618036000 tx=0x7fc618008030 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:06.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.412+0000 7fc622000640 1 -- 172.21.15.86:0/203009213 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc61c01c360 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.648+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc62c072890 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.649+0000 7fc622000640 1 -- 172.21.15.86:0/203009213 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fc61c059540 con 0x7fc62c0ce0a0 2024-03-06T07:38:06.651 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:06.652 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:06.652 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.651+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc60c06b500 msgr2=0x7fc60c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.651+0000 7fc630a00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc60c06b500 0x7fc60c06d9c0 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7fc618036000 tx=0x7fc618008030 comp rx=0 tx=0).stop 2024-03-06T07:38:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.651+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c0ce0a0 msgr2=0x7fc62c1434e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:06.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.651+0000 7fc630a00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c0ce0a0 0x7fc62c1434e0 secure :-1 s=READY pgs=1794 cs=0 l=1 rev1=1 crypto rx=0x7fc61c00d800 tx=0x7fc61c00dcc0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 shutdown_connections 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fc60c06b500 0x7fc60c06d9c0 unknown :-1 s=CLOSED pgs=761 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fc62c15f9d0 0x7fc62c13d3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fc62c144c00 0x7fc62c1f6ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 --2- 172.21.15.86:0/203009213 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fc62c0ce0a0 0x7fc62c1434e0 unknown :-1 s=CLOSED pgs=1794 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 >> 172.21.15.86:0/203009213 conn(0x7fc62c0b1730 msgr2=0x7fc62c145370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 shutdown_connections 2024-03-06T07:38:06.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:06.652+0000 7fc630a00640 1 -- 172.21.15.86:0/203009213 wait complete. 2024-03-06T07:38:06.874 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:06.874 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:07.696 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:07.875 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:08.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.089+0000 7f71ab400640 1 Processor -- start 2024-03-06T07:38:08.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.089+0000 7f71ab400640 1 -- start start 2024-03-06T07:38:08.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71ab400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f71a4151660 0x7f71a4071520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:08.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71ab400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a40cf430 0x7f71a406d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:08.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71ab400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71a4071a60 con 0x7f71a406d920 2024-03-06T07:38:08.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71ab400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f71a40b3ee0 con 0x7f71a4151660 2024-03-06T07:38:08.094 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71ab400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f71a40b35b0 con 0x7f71a40cf430 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71a8a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f71a4151660 0x7f71a4071520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71a8a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f71a4151660 0x7f71a4071520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33684/0 (socket says 172.21.15.86:33684) 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.091+0000 7f71a8a00640 1 -- 172.21.15.86:0/4083506318 learned_addr learned my addr 172.21.15.86:0/4083506318 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.092+0000 7f71a3e00640 1 --2- 172.21.15.86:0/4083506318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a40cf430 0x7f71a406d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.092+0000 7f71a8a00640 1 -- 172.21.15.86:0/4083506318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a40cf430 msgr2=0x7f71a406d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.092+0000 7f71a8a00640 1 --2- 172.21.15.86:0/4083506318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a40cf430 0x7f71a406d3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.092+0000 7f71a8a00640 1 -- 172.21.15.86:0/4083506318 >> v1:172.21.15.194:6789/0 conn(0x7f71a406d920 legacy=0x7f71a415dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:08.095 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.092+0000 7f71a8a00640 1 -- 172.21.15.86:0/4083506318 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71a40b3890 con 0x7f71a4151660 2024-03-06T07:38:08.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.093+0000 7f71a8a00640 1 --2- 172.21.15.86:0/4083506318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a4151660 0x7f71a4071520 secure :-1 s=READY pgs=1795 cs=0 l=1 rev1=1 crypto rx=0x7f7198009870 tx=0x7f719802b7e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b9251d8982dab159 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:08.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71a3400640 1 -- 172.21.15.86:0/4083506318 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7198038070 con 0x7f71a4151660 2024-03-06T07:38:08.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71a3400640 1 -- 172.21.15.86:0/4083506318 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7198031740 con 0x7f71a4151660 2024-03-06T07:38:08.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71a3400640 1 -- 172.21.15.86:0/4083506318 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f719802f050 con 0x7f71a4151660 2024-03-06T07:38:08.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71ab400640 1 -- 172.21.15.86:0/4083506318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a4151660 msgr2=0x7f71a4071520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:08.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71ab400640 1 --2- 172.21.15.86:0/4083506318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a4151660 0x7f71a4071520 secure :-1 s=READY pgs=1795 cs=0 l=1 rev1=1 crypto rx=0x7f7198009870 tx=0x7f719802b7e0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71ab400640 1 -- 172.21.15.86:0/4083506318 shutdown_connections 2024-03-06T07:38:08.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71ab400640 1 --2- 172.21.15.86:0/4083506318 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a40cf430 0x7f71a406d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71ab400640 1 --2- 172.21.15.86:0/4083506318 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a4151660 0x7f71a4071520 unknown :-1 s=CLOSED pgs=1795 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.094+0000 7f71ab400640 1 -- 172.21.15.86:0/4083506318 >> 172.21.15.86:0/4083506318 conn(0x7f71a40b1730 msgr2=0x7f71a40b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:08.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.095+0000 7f71ab400640 1 -- 172.21.15.86:0/4083506318 shutdown_connections 2024-03-06T07:38:08.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.095+0000 7f71ab400640 1 -- 172.21.15.86:0/4083506318 wait complete. 2024-03-06T07:38:08.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.096+0000 7f71ab400640 1 Processor -- start 2024-03-06T07:38:08.099 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.096+0000 7f71ab400640 1 -- start start 2024-03-06T07:38:08.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.097+0000 7f71ab400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a406d920 0x7f71a40affb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:08.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.097+0000 7f71ab400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71a40cf430 0x7f71a4137ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:08.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.097+0000 7f71ab400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a4151660 0x7f71a4138020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:08.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.097+0000 7f71ab400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71a415e700 con 0x7f71a406d920 2024-03-06T07:38:08.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.097+0000 7f71ab400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71a4071d30 con 0x7f71a40cf430 2024-03-06T07:38:08.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.097+0000 7f71ab400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f71a415e880 con 0x7f71a4151660 2024-03-06T07:38:08.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.097+0000 7f71a8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a406d920 0x7f71a40affb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:08.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a406d920 0x7f71a40affb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33698/0 (socket says 172.21.15.86:33698) 2024-03-06T07:38:08.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 -- 172.21.15.86:0/2213153582 learned_addr learned my addr 172.21.15.86:0/2213153582 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 -- 172.21.15.86:0/2213153582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a4151660 msgr2=0x7f71a4138020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a4151660 0x7f71a4138020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 -- 172.21.15.86:0/2213153582 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71a40cf430 msgr2=0x7f71a4137ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71a40cf430 0x7f71a4137ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 -- 172.21.15.86:0/2213153582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71a40b0ab0 con 0x7f71a406d920 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.098+0000 7f71a8a00640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a406d920 0x7f71a40affb0 secure :-1 s=READY pgs=1796 cs=0 l=1 rev1=1 crypto rx=0x7f7198000c00 tx=0x7f71980046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.099+0000 7f71a0c00640 1 -- 172.21.15.86:0/2213153582 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7198042070 con 0x7f71a406d920 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.099+0000 7f71a0c00640 1 -- 172.21.15.86:0/2213153582 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7198033c70 con 0x7f71a406d920 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.099+0000 7f71a0c00640 1 -- 172.21.15.86:0/2213153582 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7198038040 con 0x7f71a406d920 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.099+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f71a40b0d90 con 0x7f71a406d920 2024-03-06T07:38:08.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.099+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f71a414b9e0 con 0x7f71a406d920 2024-03-06T07:38:08.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.100+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f71a406d710 con 0x7f71a406d920 2024-03-06T07:38:08.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.101+0000 7f71a0c00640 1 -- 172.21.15.86:0/2213153582 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7198045070 con 0x7f71a406d920 2024-03-06T07:38:08.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.102+0000 7f71a0c00640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f718406b4d0 0x7f718406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:08.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.102+0000 7f71a0c00640 1 -- 172.21.15.86:0/2213153582 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f71980ac070 con 0x7f71a406d920 2024-03-06T07:38:08.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.102+0000 7f71a3e00640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f718406b4d0 0x7f718406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:08.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.103+0000 7f71a3e00640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f718406b4d0 0x7f718406d990 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f7190006fd0 tx=0x7f7190008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:08.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.107+0000 7f71a0c00640 1 -- 172.21.15.86:0/2213153582 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f71a406d710 con 0x7f71a406d920 2024-03-06T07:38:08.343 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.340+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f71a404f330 con 0x7f71a406d920 2024-03-06T07:38:08.344 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.341+0000 7f71a0c00640 1 -- 172.21.15.86:0/2213153582 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f719804c030 con 0x7f71a406d920 2024-03-06T07:38:08.344 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:08.344 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:08.345 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f718406b4d0 msgr2=0x7f718406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f718406b4d0 0x7f718406d990 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f7190006fd0 tx=0x7f7190008040 comp rx=0 tx=0).stop 2024-03-06T07:38:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a406d920 msgr2=0x7f71a40affb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:08.347 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a406d920 0x7f71a40affb0 secure :-1 s=READY pgs=1796 cs=0 l=1 rev1=1 crypto rx=0x7f7198000c00 tx=0x7f71980046c0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 shutdown_connections 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f718406b4d0 0x7f718406d990 unknown :-1 s=CLOSED pgs=762 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f71a4151660 0x7f71a4138020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f71a40cf430 0x7f71a4137ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 --2- 172.21.15.86:0/2213153582 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f71a406d920 0x7f71a40affb0 unknown :-1 s=CLOSED pgs=1796 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 >> 172.21.15.86:0/2213153582 conn(0x7f71a40b1730 msgr2=0x7f71a406e6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 shutdown_connections 2024-03-06T07:38:08.348 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:08.344+0000 7f71ab400640 1 -- 172.21.15.86:0/2213153582 wait complete. 2024-03-06T07:38:08.567 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:08.567 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:09.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:09.698 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:09.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.776+0000 7f34f4400640 1 Processor -- start 2024-03-06T07:38:09.779 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.776+0000 7f34f4400640 1 -- start start 2024-03-06T07:38:09.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.779+0000 7f34f4400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f34ec0b75b0 0x7f34ec0b4ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.779+0000 7f34f4400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f34ec0b5e00 con 0x7f34ec0cff20 2024-03-06T07:38:09.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.779+0000 7f34f4400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f34ec0b60e0 con 0x7f34ec0b5020 2024-03-06T07:38:09.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.779+0000 7f34f4400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f34ec0b5bc0 con 0x7f34ec0b75b0 2024-03-06T07:38:09.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.779+0000 7f34f1a00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f34ec0cff20 0x7f34ec0b6ea0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49932/0 (socket says 172.21.15.86:49932) 2024-03-06T07:38:09.782 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.779+0000 7f34f1a00640 1 -- 172.21.15.86:0/813372774 learned_addr learned my addr 172.21.15.86:0/813372774 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1407126064 0 0) 0x7f34ec0b5e00 con 0x7f34ec0cff20 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34f1000640 1 --2- 172.21.15.86:0/813372774 >> v2:172.21.15.194:3300/0 conn(0x7f34ec0b75b0 0x7f34ec0b4ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f34d4003520 con 0x7f34ec0cff20 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3188175814 0 0) 0x7f34d4003520 con 0x7f34ec0cff20 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 >> v2:172.21.15.194:3300/0 conn(0x7f34ec0b75b0 msgr2=0x7f34ec0b4ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 --2- 172.21.15.86:0/813372774 >> v2:172.21.15.194:3300/0 conn(0x7f34ec0b75b0 0x7f34ec0b4ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 >> v1:172.21.15.194:6789/0 conn(0x7f34ec0b5020 legacy=0x7f34ec0b54b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34ec0b63c0 con 0x7f34ec0cff20 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f34e0001670 con 0x7f34ec0cff20 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 >> v1:172.21.15.86:6789/0 conn(0x7f34ec0cff20 legacy=0x7f34ec0b6ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34d4003a00 0x7f34d4003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.784 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34d4004800 0x7f34d4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4005210 0x7f34d400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34ec0b5bc0 con 0x7f34d4004800 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34d400c160 con 0x7f34d4005210 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.780+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f34d400c360 con 0x7f34d4003a00 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f1000640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34d4004800 0x7f34d4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f2400640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4005210 0x7f34d400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f1a00640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34d4003a00 0x7f34d4003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f1000640 1 -- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34d4003a00 msgr2=0x7f34d4003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f1000640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34d4003a00 0x7f34d4003e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f1000640 1 -- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4005210 msgr2=0x7f34d400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f1000640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4005210 0x7f34d400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f1000640 1 -- 172.21.15.86:0/813372774 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34d4003520 con 0x7f34d4004800 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f2400640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4005210 0x7f34d400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:09.785 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.781+0000 7f34f2400640 1 -- 172.21.15.86:0/813372774 reap_dead start 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.782+0000 7f34f1000640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34d4004800 0x7f34d4004c80 secure :-1 s=READY pgs=1798 cs=0 l=1 rev1=1 crypto rx=0x7f34d8009a90 tx=0x7f34d4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8a41199bc6ee52ce server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.782+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34d8039070 con 0x7f34d4004800 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.782+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f34d802c680 con 0x7f34d4004800 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.782+0000 7f34ebe00640 1 -- 172.21.15.86:0/813372774 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34d8009690 con 0x7f34d4004800 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.783+0000 7f34f4400640 1 -- 172.21.15.86:0/813372774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34d4004800 msgr2=0x7f34d4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.783+0000 7f34f4400640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34d4004800 0x7f34d4004c80 secure :-1 s=READY pgs=1798 cs=0 l=1 rev1=1 crypto rx=0x7f34d8009a90 tx=0x7f34d4005830 comp rx=0 tx=0).stop 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.783+0000 7f34f4400640 1 -- 172.21.15.86:0/813372774 shutdown_connections 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.783+0000 7f34f4400640 1 --2- 172.21.15.86:0/813372774 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34d4004800 0x7f34d4004c80 unknown :-1 s=CLOSED pgs=1798 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.783+0000 7f34f4400640 1 -- 172.21.15.86:0/813372774 >> 172.21.15.86:0/813372774 conn(0x7f34ec0d1c90 msgr2=0x7f34ec0d20a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.783+0000 7f34f4400640 1 -- 172.21.15.86:0/813372774 shutdown_connections 2024-03-06T07:38:09.787 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.783+0000 7f34f4400640 1 -- 172.21.15.86:0/813372774 wait complete. 2024-03-06T07:38:09.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.785+0000 7f34f4400640 1 Processor -- start 2024-03-06T07:38:09.788 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.785+0000 7f34f4400640 1 -- start start 2024-03-06T07:38:09.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f4400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4004800 0x7f34ec158290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.789 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f4400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34ec158940 0x7f34ec1f7290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f4400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34ec1f77d0 0x7f34ec09bd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f4400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34ec161a50 con 0x7f34ec1f77d0 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f4400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34ec0b6a10 con 0x7f34d4004800 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f4400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f34ec0b7a30 con 0x7f34ec158940 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f2400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34ec1f77d0 0x7f34ec09bd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f2400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34ec1f77d0 0x7f34ec09bd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33724/0 (socket says 172.21.15.86:33724) 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.786+0000 7f34f2400640 1 -- 172.21.15.86:0/2592727075 learned_addr learned my addr 172.21.15.86:0/2592727075 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f1000640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34ec158940 0x7f34ec1f7290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f1a00640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4004800 0x7f34ec158290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f2400640 1 -- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34ec158940 msgr2=0x7f34ec1f7290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f2400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34ec158940 0x7f34ec1f7290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f2400640 1 -- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4004800 msgr2=0x7f34ec158290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f2400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4004800 0x7f34ec158290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f2400640 1 -- 172.21.15.86:0/2592727075 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34ec09cac0 con 0x7f34ec1f77d0 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f1a00640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4004800 0x7f34ec158290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:09.790 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f2400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34ec1f77d0 0x7f34ec09bd90 secure :-1 s=READY pgs=1799 cs=0 l=1 rev1=1 crypto rx=0x7f34ec0d2c60 tx=0x7f34e400d900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:09.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34e9600640 1 -- 172.21.15.86:0/2592727075 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34e4018070 con 0x7f34ec1f77d0 2024-03-06T07:38:09.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34e9600640 1 -- 172.21.15.86:0/2592727075 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f34e4003da0 con 0x7f34ec1f77d0 2024-03-06T07:38:09.791 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f34ec0c8e00 con 0x7f34ec1f77d0 2024-03-06T07:38:09.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.787+0000 7f34e9600640 1 -- 172.21.15.86:0/2592727075 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34e4006d90 con 0x7f34ec1f77d0 2024-03-06T07:38:09.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.788+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f34ec0c93c0 con 0x7f34ec1f77d0 2024-03-06T07:38:09.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.789+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f34ec04f330 con 0x7f34ec1f77d0 2024-03-06T07:38:09.792 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.790+0000 7f34e9600640 1 -- 172.21.15.86:0/2592727075 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f34e4005070 con 0x7f34ec1f77d0 2024-03-06T07:38:09.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.791+0000 7f34e9600640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34cc06b500 0x7f34cc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:09.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.791+0000 7f34e9600640 1 -- 172.21.15.86:0/2592727075 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f34e40892a0 con 0x7f34ec1f77d0 2024-03-06T07:38:09.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.791+0000 7f34f1a00640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34cc06b500 0x7f34cc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:09.794 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.792+0000 7f34f1a00640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34cc06b500 0x7f34cc06d9c0 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7f34e0001eb0 tx=0x7f34e0009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:09.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:09.799+0000 7f34e9600640 1 -- 172.21.15.86:0/2592727075 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f34e40580d0 con 0x7f34ec1f77d0 2024-03-06T07:38:10.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.033+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f34ec06c8c0 con 0x7f34ec1f77d0 2024-03-06T07:38:10.038 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.034+0000 7f34e9600640 1 -- 172.21.15.86:0/2592727075 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f34e40579d0 con 0x7f34ec1f77d0 2024-03-06T07:38:10.038 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:10.038 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:10.038 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:10.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34cc06b500 msgr2=0x7f34cc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:10.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34cc06b500 0x7f34cc06d9c0 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7f34e0001eb0 tx=0x7f34e0009040 comp rx=0 tx=0).stop 2024-03-06T07:38:10.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34ec1f77d0 msgr2=0x7f34ec09bd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:10.039 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34ec1f77d0 0x7f34ec09bd90 secure :-1 s=READY pgs=1799 cs=0 l=1 rev1=1 crypto rx=0x7f34ec0d2c60 tx=0x7f34e400d900 comp rx=0 tx=0).stop 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 shutdown_connections 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34cc06b500 0x7f34cc06d9c0 unknown :-1 s=CLOSED pgs=763 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34ec1f77d0 0x7f34ec09bd90 unknown :-1 s=CLOSED pgs=1799 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34ec158940 0x7f34ec1f7290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 --2- 172.21.15.86:0/2592727075 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34d4004800 0x7f34ec158290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.036+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 >> 172.21.15.86:0/2592727075 conn(0x7f34ec0d1c90 msgr2=0x7f34ec0d2070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.037+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 shutdown_connections 2024-03-06T07:38:10.040 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:10.037+0000 7f34f4400640 1 -- 172.21.15.86:0/2592727075 wait complete. 2024-03-06T07:38:10.261 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:10.261 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:11.263 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:11.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.476+0000 7f12a6c00640 1 Processor -- start 2024-03-06T07:38:11.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.477+0000 7f12a6c00640 1 -- start start 2024-03-06T07:38:11.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.478+0000 7f12a6c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f12a006afc0 0x7f12a006db80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.478+0000 7f12a6c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 0x7f12a006b3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.478+0000 7f12a6c00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f12a006b8e0 con 0x7f12a00d1630 2024-03-06T07:38:11.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.478+0000 7f12a6c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f12a006ed70 con 0x7f12a006afc0 2024-03-06T07:38:11.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.478+0000 7f12a6c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f12a006e160 con 0x7f12a015e1d0 2024-03-06T07:38:11.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f12a4c00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f12a00d1630 0x7f12a00adc40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49934/0 (socket says 172.21.15.86:49934) 2024-03-06T07:38:11.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f12a4c00640 1 -- 172.21.15.86:0/481443028 learned_addr learned my addr 172.21.15.86:0/481443028 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:11.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129f400640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 0x7f12a006b3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129fe00640 1 --2- 172.21.15.86:0/481443028 >> v2:172.21.15.194:3300/0 conn(0x7f12a006afc0 0x7f12a006db80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3341728464 0 0) 0x7f12a006b8e0 con 0x7f12a00d1630 2024-03-06T07:38:11.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1284003520 con 0x7f12a00d1630 2024-03-06T07:38:11.483 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3269551556 0 0) 0x7f1284003520 con 0x7f12a00d1630 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 msgr2=0x7f12a006b3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 0x7f12a006b3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 >> v2:172.21.15.194:3300/0 conn(0x7f12a006afc0 msgr2=0x7f12a006db80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 --2- 172.21.15.86:0/481443028 >> v2:172.21.15.194:3300/0 conn(0x7f12a006afc0 0x7f12a006db80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12a006e440 con 0x7f12a00d1630 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.479+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f12980019f0 con 0x7f12a00d1630 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 >> v1:172.21.15.86:6789/0 conn(0x7f12a00d1630 legacy=0x7f12a00adc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129ea00640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 0x7f12840039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129ea00640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f1284004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129ea00640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1284004d40 0x7f12840051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a006ed70 con 0x7f1284003f30 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a006e160 con 0x7f1284004d40 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f128400b9c0 con 0x7f12a015e1d0 2024-03-06T07:38:11.484 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129f400640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f1284004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f12a4c00640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1284004d40 0x7f12840051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129fe00640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 0x7f12840039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129f400640 1 -- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 msgr2=0x7f12840039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:11.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129f400640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a015e1d0 0x7f12840039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129f400640 1 -- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1284004d40 msgr2=0x7f12840051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:11.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129f400640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f1284004d40 0x7f12840051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.485 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.480+0000 7f129f400640 1 -- 172.21.15.86:0/481443028 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1284003520 con 0x7f1284003f30 2024-03-06T07:38:11.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.482+0000 7f129f400640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f1284004800 secure :-1 s=READY pgs=1801 cs=0 l=1 rev1=1 crypto rx=0x7f1288009a90 tx=0x7f1284005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1dd156053d880e54 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:11.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.482+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1288039070 con 0x7f1284003f30 2024-03-06T07:38:11.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.482+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f128802c6b0 con 0x7f1284003f30 2024-03-06T07:38:11.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.482+0000 7f129ea00640 1 -- 172.21.15.86:0/481443028 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1288009690 con 0x7f1284003f30 2024-03-06T07:38:11.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.483+0000 7f12a6c00640 1 -- 172.21.15.86:0/481443028 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 msgr2=0x7f1284004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:11.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.483+0000 7f12a6c00640 1 --2- 172.21.15.86:0/481443028 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f1284004800 secure :-1 s=READY pgs=1801 cs=0 l=1 rev1=1 crypto rx=0x7f1288009a90 tx=0x7f1284005830 comp rx=0 tx=0).stop 2024-03-06T07:38:11.486 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.483+0000 7f12a4c00640 1 -- 172.21.15.86:0/481443028 reap_dead start 2024-03-06T07:38:11.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.483+0000 7f12a6c00640 1 -- 172.21.15.86:0/481443028 shutdown_connections 2024-03-06T07:38:11.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.483+0000 7f12a6c00640 1 -- 172.21.15.86:0/481443028 >> 172.21.15.86:0/481443028 conn(0x7f12a00d0b40 msgr2=0x7f12a00d0f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:11.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.483+0000 7f12a6c00640 1 -- 172.21.15.86:0/481443028 shutdown_connections 2024-03-06T07:38:11.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.483+0000 7f12a6c00640 1 -- 172.21.15.86:0/481443028 wait complete. 2024-03-06T07:38:11.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.485+0000 7f12a6c00640 1 Processor -- start 2024-03-06T07:38:11.487 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.485+0000 7f12a6c00640 1 -- start start 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f12a6c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f12a0156d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f12a6c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12a01485e0 0x7f12a0155180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f12a6c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a01556c0 0x7f12a00af580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f12a6c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a00704b0 con 0x7f1284003f30 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f12a6c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a006ea00 con 0x7f12a01485e0 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f12a6c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f12a006b3a0 con 0x7f12a01556c0 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f129fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f12a0156d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f129fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f12a0156d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33754/0 (socket says 172.21.15.86:33754) 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.486+0000 7f129fe00640 1 -- 172.21.15.86:0/1577196145 learned_addr learned my addr 172.21.15.86:0/1577196145 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f129fe00640 1 -- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a01556c0 msgr2=0x7f12a00af580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f129f400640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12a01485e0 0x7f12a0155180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.490 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f12a4c00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a01556c0 0x7f12a00af580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f129fe00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a01556c0 0x7f12a00af580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f129fe00640 1 -- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12a01485e0 msgr2=0x7f12a0155180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f129fe00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12a01485e0 0x7f12a0155180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f129fe00640 1 -- 172.21.15.86:0/1577196145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12a0154e10 con 0x7f1284003f30 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f129fe00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f12a0156d90 secure :-1 s=READY pgs=1802 cs=0 l=1 rev1=1 crypto rx=0x7f1290011700 tx=0x7f1290011bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f1297e00640 1 -- 172.21.15.86:0/1577196145 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f129000f9c0 con 0x7f1284003f30 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f1297e00640 1 -- 172.21.15.86:0/1577196145 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f129000fb60 con 0x7f1284003f30 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f12a0155cf0 con 0x7f1284003f30 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f1297e00640 1 -- 172.21.15.86:0/1577196145 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1290015460 con 0x7f1284003f30 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.487+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12a01562b0 con 0x7f1284003f30 2024-03-06T07:38:11.491 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.489+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f12a004f330 con 0x7f1284003f30 2024-03-06T07:38:11.492 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.490+0000 7f1297e00640 1 -- 172.21.15.86:0/1577196145 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1290015600 con 0x7f1284003f30 2024-03-06T07:38:11.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.490+0000 7f1297e00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f127c06b500 0x7f127c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:11.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.491+0000 7f1297e00640 1 -- 172.21.15.86:0/1577196145 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f129008ac10 con 0x7f1284003f30 2024-03-06T07:38:11.493 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.491+0000 7f129f400640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f127c06b500 0x7f127c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:11.494 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.492+0000 7f129f400640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f127c06b500 0x7f127c06d9c0 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7f1288036000 tx=0x7f1288030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:11.498 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.495+0000 7f1297e00640 1 -- 172.21.15.86:0/1577196145 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1290019030 con 0x7f1284003f30 2024-03-06T07:38:11.700 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:11.735 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.732+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f12a0159c60 con 0x7f1284003f30 2024-03-06T07:38:11.736 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.734+0000 7f1297e00640 1 -- 172.21.15.86:0/1577196145 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f1290059a50 con 0x7f1284003f30 2024-03-06T07:38:11.737 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:11.737 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:11.737 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:11.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f127c06b500 msgr2=0x7f127c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:11.739 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f127c06b500 0x7f127c06d9c0 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7f1288036000 tx=0x7f1288030000 comp rx=0 tx=0).stop 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 msgr2=0x7f12a0156d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f12a0156d90 secure :-1 s=READY pgs=1802 cs=0 l=1 rev1=1 crypto rx=0x7f1290011700 tx=0x7f1290011bc0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 shutdown_connections 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f127c06b500 0x7f127c06d9c0 unknown :-1 s=CLOSED pgs=764 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f12a01556c0 0x7f12a00af580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f12a01485e0 0x7f12a0155180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 --2- 172.21.15.86:0/1577196145 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f1284003f30 0x7f12a0156d90 unknown :-1 s=CLOSED pgs=1802 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.736+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 >> 172.21.15.86:0/1577196145 conn(0x7f12a00d0b40 msgr2=0x7f12a00ae4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.737+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 shutdown_connections 2024-03-06T07:38:11.740 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:11.737+0000 7f12a6c00640 1 -- 172.21.15.86:0/1577196145 wait complete. 2024-03-06T07:38:11.966 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:11.966 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:12.967 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:13.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.179+0000 7fd6d1c00640 1 Processor -- start 2024-03-06T07:38:13.182 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.179+0000 7fd6d1c00640 1 -- start start 2024-03-06T07:38:13.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.180+0000 7fd6d1c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd6cc1561f0 0x7fd6cc0ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:13.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.180+0000 7fd6d1c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fd6cc0aebf0 0x7fd6cc0cc2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:13.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.180+0000 7fd6d1c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd6cc15c240 con 0x7fd6cc0cc800 2024-03-06T07:38:13.184 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.180+0000 7fd6d1c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6cc15ce30 con 0x7fd6cc1561f0 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.180+0000 7fd6d1c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6cc15c500 con 0x7fd6cc0aebf0 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd6cc1561f0 0x7fd6cc0ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cbe00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd6cc1561f0 0x7fd6cc0ae6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33768/0 (socket says 172.21.15.86:33768) 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cbe00640 1 -- 172.21.15.86:0/323367925 learned_addr learned my addr 172.21.15.86:0/323367925 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cb400640 1 --2- 172.21.15.86:0/323367925 >> v2:172.21.15.194:3300/0 conn(0x7fd6cc0aebf0 0x7fd6cc0cc2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cbe00640 1 -- 172.21.15.86:0/323367925 >> v2:172.21.15.194:3300/0 conn(0x7fd6cc0aebf0 msgr2=0x7fd6cc0cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cbe00640 1 --2- 172.21.15.86:0/323367925 >> v2:172.21.15.194:3300/0 conn(0x7fd6cc0aebf0 0x7fd6cc0cc2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cbe00640 1 -- 172.21.15.86:0/323367925 >> v1:172.21.15.194:6789/0 conn(0x7fd6cc0cc800 legacy=0x7fd6cc0cf800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:13.185 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.181+0000 7fd6cbe00640 1 -- 172.21.15.86:0/323367925 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6cc15c7e0 con 0x7fd6cc1561f0 2024-03-06T07:38:13.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.183+0000 7fd6cbe00640 1 --2- 172.21.15.86:0/323367925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc1561f0 0x7fd6cc0ae6b0 secure :-1 s=READY pgs=1803 cs=0 l=1 rev1=1 crypto rx=0x7fd6cc068400 tx=0x7fd6bc02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c95f410df3d65718 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:13.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.183+0000 7fd6caa00640 1 -- 172.21.15.86:0/323367925 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd6bc035020 con 0x7fd6cc1561f0 2024-03-06T07:38:13.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.183+0000 7fd6caa00640 1 -- 172.21.15.86:0/323367925 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd6bc031870 con 0x7fd6cc1561f0 2024-03-06T07:38:13.187 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.183+0000 7fd6caa00640 1 -- 172.21.15.86:0/323367925 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd6bc031c40 con 0x7fd6cc1561f0 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 -- 172.21.15.86:0/323367925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc1561f0 msgr2=0x7fd6cc0ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/323367925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc1561f0 0x7fd6cc0ae6b0 secure :-1 s=READY pgs=1803 cs=0 l=1 rev1=1 crypto rx=0x7fd6cc068400 tx=0x7fd6bc02b910 comp rx=0 tx=0).stop 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 -- 172.21.15.86:0/323367925 shutdown_connections 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/323367925 >> v2:172.21.15.194:3300/0 conn(0x7fd6cc0aebf0 0x7fd6cc0cc2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/323367925 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc1561f0 0x7fd6cc0ae6b0 unknown :-1 s=CLOSED pgs=1803 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 -- 172.21.15.86:0/323367925 >> 172.21.15.86:0/323367925 conn(0x7fd6cc06afc0 msgr2=0x7fd6cc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 -- 172.21.15.86:0/323367925 shutdown_connections 2024-03-06T07:38:13.188 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.184+0000 7fd6d1c00640 1 -- 172.21.15.86:0/323367925 wait complete. 2024-03-06T07:38:13.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.186+0000 7fd6d1c00640 1 Processor -- start 2024-03-06T07:38:13.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.186+0000 7fd6d1c00640 1 -- start start 2024-03-06T07:38:13.191 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6d1c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6cc0aebf0 0x7fd6cc1591f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6d1c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc0cc800 0x7fd6cc149220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6d1c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6cc1561f0 0x7fd6cc0ad860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6d1c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6cc1565d0 con 0x7fd6cc0cc800 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6d1c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6cc0ccc50 con 0x7fd6cc0aebf0 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6d1c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd6cc156750 con 0x7fd6cc1561f0 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc0cc800 0x7fd6cc149220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc0cc800 0x7fd6cc149220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33776/0 (socket says 172.21.15.86:33776) 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 -- 172.21.15.86:0/3460056171 learned_addr learned my addr 172.21.15.86:0/3460056171 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:13.192 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 -- 172.21.15.86:0/3460056171 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6cc1561f0 msgr2=0x7fd6cc0ad860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6cc1561f0 0x7fd6cc0ad860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 -- 172.21.15.86:0/3460056171 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6cc0aebf0 msgr2=0x7fd6cc1591f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6cc0aebf0 0x7fd6cc1591f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.187+0000 7fd6cb400640 1 -- 172.21.15.86:0/3460056171 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6cc153650 con 0x7fd6cc0cc800 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.188+0000 7fd6cb400640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc0cc800 0x7fd6cc149220 secure :-1 s=READY pgs=1804 cs=0 l=1 rev1=1 crypto rx=0x7fd6b800d800 tx=0x7fd6b800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.188+0000 7fd6c3e00640 1 -- 172.21.15.86:0/3460056171 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd6b800b6c0 con 0x7fd6cc0cc800 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.188+0000 7fd6c3e00640 1 -- 172.21.15.86:0/3460056171 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd6b8019070 con 0x7fd6cc0cc800 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.188+0000 7fd6c3e00640 1 -- 172.21.15.86:0/3460056171 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd6b800ba80 con 0x7fd6cc0cc800 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.188+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd6cc159730 con 0x7fd6cc0cc800 2024-03-06T07:38:13.193 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.188+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6cc156b20 con 0x7fd6cc0cc800 2024-03-06T07:38:13.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.189+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd6cc04f330 con 0x7fd6cc0cc800 2024-03-06T07:38:13.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.190+0000 7fd6c3e00640 1 -- 172.21.15.86:0/3460056171 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd6b800bc20 con 0x7fd6cc0cc800 2024-03-06T07:38:13.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.191+0000 7fd6c3e00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd6ac06b500 0x7fd6ac06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:13.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.191+0000 7fd6c3e00640 1 -- 172.21.15.86:0/3460056171 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd6b808a7a0 con 0x7fd6cc0cc800 2024-03-06T07:38:13.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.191+0000 7fd6cbe00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd6ac06b500 0x7fd6ac06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:13.195 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.192+0000 7fd6cbe00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd6ac06b500 0x7fd6ac06d9c0 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7fd6bc036000 tx=0x7fd6bc030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:13.204 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.201+0000 7fd6c3e00640 1 -- 172.21.15.86:0/3460056171 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd6b801c360 con 0x7fd6cc0cc800 2024-03-06T07:38:13.437 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.434+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd6cc0b2b10 con 0x7fd6cc0cc800 2024-03-06T07:38:13.438 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.435+0000 7fd6c3e00640 1 -- 172.21.15.86:0/3460056171 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fd6b8059540 con 0x7fd6cc0cc800 2024-03-06T07:38:13.438 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:13.439 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:13.439 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:13.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd6ac06b500 msgr2=0x7fd6ac06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:13.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd6ac06b500 0x7fd6ac06d9c0 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7fd6bc036000 tx=0x7fd6bc030000 comp rx=0 tx=0).stop 2024-03-06T07:38:13.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc0cc800 msgr2=0x7fd6cc149220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:13.441 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc0cc800 0x7fd6cc149220 secure :-1 s=READY pgs=1804 cs=0 l=1 rev1=1 crypto rx=0x7fd6b800d800 tx=0x7fd6b800dcc0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 shutdown_connections 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd6ac06b500 0x7fd6ac06d9c0 unknown :-1 s=CLOSED pgs=765 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd6cc1561f0 0x7fd6cc0ad860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd6cc0cc800 0x7fd6cc149220 unknown :-1 s=CLOSED pgs=1804 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 --2- 172.21.15.86:0/3460056171 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd6cc0aebf0 0x7fd6cc1591f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 >> 172.21.15.86:0/3460056171 conn(0x7fd6cc06afc0 msgr2=0x7fd6cc155290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 shutdown_connections 2024-03-06T07:38:13.442 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:13.438+0000 7fd6d1c00640 1 -- 172.21.15.86:0/3460056171 wait complete. 2024-03-06T07:38:13.661 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:13.662 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:13.702 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:14.663 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:14.875 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.873+0000 7fa36f200640 1 Processor -- start 2024-03-06T07:38:14.876 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.873+0000 7fa36f200640 1 -- start start 2024-03-06T07:38:14.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.874+0000 7fa36f200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fa3701620c0 0x7fa3700b28e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:14.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.874+0000 7fa36f200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa370161900 con 0x7fa3700b40c0 2024-03-06T07:38:14.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.874+0000 7fa36f200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa37015ebd0 con 0x7fa3700ae670 2024-03-06T07:38:14.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36f200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa37015eeb0 con 0x7fa3701620c0 2024-03-06T07:38:14.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36de00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fa3700b40c0 0x7fa3700b44a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49948/0 (socket says 172.21.15.86:49948) 2024-03-06T07:38:14.878 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36de00640 1 -- 172.21.15.86:0/970711207 learned_addr learned my addr 172.21.15.86:0/970711207 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:14.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36e800640 1 --2- 172.21.15.86:0/970711207 >> v2:172.21.15.86:3300/0 conn(0x7fa3701620c0 0x7fa3700b28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:14.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36d400640 1 -- 172.21.15.86:0/970711207 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 672821726 0 0) 0x7fa370161900 con 0x7fa3700b40c0 2024-03-06T07:38:14.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36d400640 1 -- 172.21.15.86:0/970711207 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa354003520 con 0x7fa3700b40c0 2024-03-06T07:38:14.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36e800640 1 -- 172.21.15.86:0/970711207 >> v1:172.21.15.194:6789/0 conn(0x7fa3700ae670 legacy=0x7fa37015e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:14.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36e800640 1 -- 172.21.15.86:0/970711207 >> v1:172.21.15.86:6789/0 conn(0x7fa3700b40c0 legacy=0x7fa3700b44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:14.879 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.875+0000 7fa36e800640 1 -- 172.21.15.86:0/970711207 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa37015f190 con 0x7fa3701620c0 2024-03-06T07:38:14.880 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.877+0000 7fa36e800640 1 --2- 172.21.15.86:0/970711207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa3700b28e0 secure :-1 s=READY pgs=1806 cs=0 l=1 rev1=1 crypto rx=0x7fa3680097a0 tx=0x7fa36802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c0e2ef3ce2087354 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.877+0000 7fa36d400640 1 -- 172.21.15.86:0/970711207 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa368034070 con 0x7fa3701620c0 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.877+0000 7fa36d400640 1 -- 172.21.15.86:0/970711207 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa368030610 con 0x7fa3701620c0 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.877+0000 7fa36d400640 1 -- 172.21.15.86:0/970711207 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3680309e0 con 0x7fa3701620c0 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.878+0000 7fa36f200640 1 -- 172.21.15.86:0/970711207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 msgr2=0x7fa3700b28e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.878+0000 7fa36f200640 1 --2- 172.21.15.86:0/970711207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa3700b28e0 secure :-1 s=READY pgs=1806 cs=0 l=1 rev1=1 crypto rx=0x7fa3680097a0 tx=0x7fa36802b960 comp rx=0 tx=0).stop 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.878+0000 7fa36f200640 1 -- 172.21.15.86:0/970711207 shutdown_connections 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.878+0000 7fa36f200640 1 --2- 172.21.15.86:0/970711207 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa3700b28e0 unknown :-1 s=CLOSED pgs=1806 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.878+0000 7fa36f200640 1 -- 172.21.15.86:0/970711207 >> 172.21.15.86:0/970711207 conn(0x7fa37006afc0 msgr2=0x7fa37006b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.878+0000 7fa36f200640 1 -- 172.21.15.86:0/970711207 shutdown_connections 2024-03-06T07:38:14.881 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.878+0000 7fa36f200640 1 -- 172.21.15.86:0/970711207 wait complete. 2024-03-06T07:38:14.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.879+0000 7fa36f200640 1 Processor -- start 2024-03-06T07:38:14.882 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.880+0000 7fa36f200640 1 -- start start 2024-03-06T07:38:14.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36f200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3700ae670 0x7fa3701f0e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:14.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36f200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3700b40c0 0x7fa3700b24a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:14.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36f200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa370153170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:14.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36f200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3701587e0 con 0x7fa3701620c0 2024-03-06T07:38:14.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36f200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa370158660 con 0x7fa3700ae670 2024-03-06T07:38:14.885 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36f200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa370162570 con 0x7fa3700b40c0 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa370153170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa370153170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:33796/0 (socket says 172.21.15.86:33796) 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 -- 172.21.15.86:0/3795701770 learned_addr learned my addr 172.21.15.86:0/3795701770 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36de00640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3700ae670 0x7fa3701f0e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa365400640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3700b40c0 0x7fa3700b24a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 -- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3700b40c0 msgr2=0x7fa3700b24a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3700b40c0 0x7fa3700b24a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 -- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3700ae670 msgr2=0x7fa3701f0e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3700ae670 0x7fa3701f0e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.881+0000 7fa36e800640 1 -- 172.21.15.86:0/3795701770 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3701f1370 con 0x7fa3701620c0 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.882+0000 7fa36e800640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa370153170 secure :-1 s=READY pgs=1807 cs=0 l=1 rev1=1 crypto rx=0x7fa3680098c0 tx=0x7fa3680046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:14.886 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.882+0000 7fa366a00640 1 -- 172.21.15.86:0/3795701770 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa368034040 con 0x7fa3701620c0 2024-03-06T07:38:14.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.882+0000 7fa366a00640 1 -- 172.21.15.86:0/3795701770 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa36803aa20 con 0x7fa3701620c0 2024-03-06T07:38:14.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.882+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa370070a30 con 0x7fa3701620c0 2024-03-06T07:38:14.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.882+0000 7fa366a00640 1 -- 172.21.15.86:0/3795701770 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa36803adf0 con 0x7fa3701620c0 2024-03-06T07:38:14.887 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.882+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa3700b3180 con 0x7fa3701620c0 2024-03-06T07:38:14.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.884+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa3700b3380 con 0x7fa3701620c0 2024-03-06T07:38:14.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.884+0000 7fa366a00640 1 -- 172.21.15.86:0/3795701770 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa368037070 con 0x7fa3701620c0 2024-03-06T07:38:14.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.885+0000 7fa366a00640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa34c06b4d0 0x7fa34c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:14.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.885+0000 7fa366a00640 1 -- 172.21.15.86:0/3795701770 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa3680ac080 con 0x7fa3701620c0 2024-03-06T07:38:14.888 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.885+0000 7fa36de00640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa34c06b4d0 0x7fa34c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:14.889 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.886+0000 7fa36de00640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa34c06b4d0 0x7fa34c06d990 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7fa360001560 tx=0x7fa360009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:14.892 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:14.890+0000 7fa366a00640 1 -- 172.21.15.86:0/3795701770 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa36803f070 con 0x7fa3701620c0 2024-03-06T07:38:15.173 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.170+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa37004f330 con 0x7fa3701620c0 2024-03-06T07:38:15.174 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.171+0000 7fa366a00640 1 -- 172.21.15.86:0/3795701770 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fa36807ae20 con 0x7fa3701620c0 2024-03-06T07:38:15.174 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:15.174 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:15.174 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa34c06b4d0 msgr2=0x7fa34c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa34c06b4d0 0x7fa34c06d990 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7fa360001560 tx=0x7fa360009040 comp rx=0 tx=0).stop 2024-03-06T07:38:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 msgr2=0x7fa370153170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:15.177 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa370153170 secure :-1 s=READY pgs=1807 cs=0 l=1 rev1=1 crypto rx=0x7fa3680098c0 tx=0x7fa3680046c0 comp rx=0 tx=0).stop 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 shutdown_connections 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fa34c06b4d0 0x7fa34c06d990 unknown :-1 s=CLOSED pgs=766 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fa3701620c0 0x7fa370153170 unknown :-1 s=CLOSED pgs=1807 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fa3700b40c0 0x7fa3700b24a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 --2- 172.21.15.86:0/3795701770 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fa3700ae670 0x7fa3701f0e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 >> 172.21.15.86:0/3795701770 conn(0x7fa37006afc0 msgr2=0x7fa3700b0220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 shutdown_connections 2024-03-06T07:38:15.178 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:15.174+0000 7fa36f200640 1 -- 172.21.15.86:0/3795701770 wait complete. 2024-03-06T07:38:15.398 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:15.398 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:15.705 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:16.399 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:16.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.624+0000 7ff0eca00640 1 Processor -- start 2024-03-06T07:38:16.627 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.624+0000 7ff0eca00640 1 -- start start 2024-03-06T07:38:16.629 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.626+0000 7ff0eca00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff0e81561f0 0x7ff0e80ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.626+0000 7ff0eca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80cc9d0 0x7ff0e815f830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.626+0000 7ff0eca00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0e815c240 con 0x7ff0e80aebf0 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.626+0000 7ff0eca00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0e815ce30 con 0x7ff0e81561f0 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.626+0000 7ff0eca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0e815c500 con 0x7ff0e80cc9d0 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.626+0000 7ff0e6800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff0e81561f0 0x7ff0e80ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.627+0000 7ff0e6800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff0e81561f0 0x7ff0e80ae6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57862/0 (socket says 172.21.15.86:57862) 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.627+0000 7ff0e6800640 1 -- 172.21.15.86:0/4034967571 learned_addr learned my addr 172.21.15.86:0/4034967571 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.627+0000 7ff0e7200640 1 --2- 172.21.15.86:0/4034967571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80cc9d0 0x7ff0e815f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.627+0000 7ff0e6800640 1 -- 172.21.15.86:0/4034967571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80cc9d0 msgr2=0x7ff0e815f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.627+0000 7ff0e6800640 1 --2- 172.21.15.86:0/4034967571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80cc9d0 0x7ff0e815f830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:16.630 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.627+0000 7ff0e6800640 1 -- 172.21.15.86:0/4034967571 >> v1:172.21.15.194:6789/0 conn(0x7ff0e80aebf0 legacy=0x7ff0e80cc2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:16.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.627+0000 7ff0e6800640 1 -- 172.21.15.86:0/4034967571 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0e815c7e0 con 0x7ff0e81561f0 2024-03-06T07:38:16.631 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.629+0000 7ff0e6800640 1 --2- 172.21.15.86:0/4034967571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80ae6b0 secure :-1 s=READY pgs=1808 cs=0 l=1 rev1=1 crypto rx=0x7ff0d8009870 tx=0x7ff0d802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b11ac15862aeeff0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:16.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.629+0000 7ff0e5400640 1 -- 172.21.15.86:0/4034967571 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0d8035020 con 0x7ff0e81561f0 2024-03-06T07:38:16.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.629+0000 7ff0e5400640 1 -- 172.21.15.86:0/4034967571 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0d8031640 con 0x7ff0e81561f0 2024-03-06T07:38:16.632 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.629+0000 7ff0e5400640 1 -- 172.21.15.86:0/4034967571 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0d802d040 con 0x7ff0e81561f0 2024-03-06T07:38:16.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 -- 172.21.15.86:0/4034967571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 msgr2=0x7ff0e80ae6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:16.635 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 --2- 172.21.15.86:0/4034967571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80ae6b0 secure :-1 s=READY pgs=1808 cs=0 l=1 rev1=1 crypto rx=0x7ff0d8009870 tx=0x7ff0d802b8f0 comp rx=0 tx=0).stop 2024-03-06T07:38:16.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 -- 172.21.15.86:0/4034967571 shutdown_connections 2024-03-06T07:38:16.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 --2- 172.21.15.86:0/4034967571 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80cc9d0 0x7ff0e815f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:16.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 --2- 172.21.15.86:0/4034967571 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80ae6b0 unknown :-1 s=CLOSED pgs=1808 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:16.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 -- 172.21.15.86:0/4034967571 >> 172.21.15.86:0/4034967571 conn(0x7ff0e806afc0 msgr2=0x7ff0e806b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:16.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 -- 172.21.15.86:0/4034967571 shutdown_connections 2024-03-06T07:38:16.636 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.630+0000 7ff0eca00640 1 -- 172.21.15.86:0/4034967571 wait complete. 2024-03-06T07:38:16.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.635+0000 7ff0eca00640 1 Processor -- start 2024-03-06T07:38:16.637 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.635+0000 7ff0eca00640 1 -- start start 2024-03-06T07:38:16.640 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.636+0000 7ff0eca00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80aebf0 0x7ff0e8153fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.636+0000 7ff0eca00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80cc9d0 0x7ff0e8157580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.636+0000 7ff0eca00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80b0440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.636+0000 7ff0eca00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0e8156750 con 0x7ff0e81561f0 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.636+0000 7ff0eca00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0e81565d0 con 0x7ff0e80cc9d0 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.636+0000 7ff0eca00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0e815fd70 con 0x7ff0e80aebf0 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80b0440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80b0440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57866/0 (socket says 172.21.15.86:57866) 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 -- 172.21.15.86:0/2572440118 learned_addr learned my addr 172.21.15.86:0/2572440118 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e6800640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80aebf0 0x7ff0e8153fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e5e00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80cc9d0 0x7ff0e8157580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 -- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80aebf0 msgr2=0x7ff0e8153fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:16.641 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80aebf0 0x7ff0e8153fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 -- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80cc9d0 msgr2=0x7ff0e8157580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80cc9d0 0x7ff0e8157580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e7200640 1 -- 172.21.15.86:0/2572440118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0e81f2b00 con 0x7ff0e81561f0 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.637+0000 7ff0e5e00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80cc9d0 0x7ff0e8157580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.638+0000 7ff0e7200640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80b0440 secure :-1 s=READY pgs=1809 cs=0 l=1 rev1=1 crypto rx=0x7ff0e0007e40 tx=0x7ff0e000e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.638+0000 7ff0dea00640 1 -- 172.21.15.86:0/2572440118 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0e0014620 con 0x7ff0e81561f0 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.638+0000 7ff0dea00640 1 -- 172.21.15.86:0/2572440118 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0e0019070 con 0x7ff0e81561f0 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.638+0000 7ff0dea00640 1 -- 172.21.15.86:0/2572440118 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0e0014a20 con 0x7ff0e81561f0 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.638+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff0e8156aa0 con 0x7ff0e81561f0 2024-03-06T07:38:16.642 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.638+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff0e80b0980 con 0x7ff0e81561f0 2024-03-06T07:38:16.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.639+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff0e804f330 con 0x7ff0e81561f0 2024-03-06T07:38:16.643 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.640+0000 7ff0dea00640 1 -- 172.21.15.86:0/2572440118 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff0e0014bc0 con 0x7ff0e81561f0 2024-03-06T07:38:16.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.641+0000 7ff0dea00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c406b4d0 0x7ff0c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:16.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.641+0000 7ff0dea00640 1 -- 172.21.15.86:0/2572440118 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff0e008abc0 con 0x7ff0e81561f0 2024-03-06T07:38:16.644 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.641+0000 7ff0e6800640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c406b4d0 0x7ff0c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:16.645 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.642+0000 7ff0e6800640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c406b4d0 0x7ff0c406d990 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7ff0d8036000 tx=0x7ff0d803c040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:16.648 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:16.646+0000 7ff0dea00640 1 -- 172.21.15.86:0/2572440118 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff0e0059960 con 0x7ff0e81561f0 2024-03-06T07:38:17.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.025+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff0e806b3a0 con 0x7ff0e81561f0 2024-03-06T07:38:17.028 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.026+0000 7ff0dea00640 1 -- 172.21.15.86:0/2572440118 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7ff0e0059260 con 0x7ff0e81561f0 2024-03-06T07:38:17.029 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:17.029 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:17.029 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c406b4d0 msgr2=0x7ff0c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c406b4d0 0x7ff0c406d990 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7ff0d8036000 tx=0x7ff0d803c040 comp rx=0 tx=0).stop 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 msgr2=0x7ff0e80b0440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80b0440 secure :-1 s=READY pgs=1809 cs=0 l=1 rev1=1 crypto rx=0x7ff0e0007e40 tx=0x7ff0e000e4b0 comp rx=0 tx=0).stop 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 shutdown_connections 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff0c406b4d0 0x7ff0c406d990 unknown :-1 s=CLOSED pgs=767 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff0e81561f0 0x7ff0e80b0440 unknown :-1 s=CLOSED pgs=1809 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff0e80cc9d0 0x7ff0e8157580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:17.031 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 --2- 172.21.15.86:0/2572440118 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff0e80aebf0 0x7ff0e8153fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:17.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 >> 172.21.15.86:0/2572440118 conn(0x7ff0e806afc0 msgr2=0x7ff0e806d350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:17.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 shutdown_connections 2024-03-06T07:38:17.032 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:17.028+0000 7ff0eca00640 1 -- 172.21.15.86:0/2572440118 wait complete. 2024-03-06T07:38:17.251 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:17.251 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:17.707 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:18.252 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.461+0000 7f6f55000640 1 Processor -- start 2024-03-06T07:38:18.464 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.462+0000 7f6f55000640 1 -- start start 2024-03-06T07:38:18.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.463+0000 7f6f55000640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5015eff0 0x7f6f50070d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:18.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.463+0000 7f6f55000640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f6f5006e4d0 0x7f6f5006e940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:18.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.463+0000 7f6f55000640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6f50071d30 con 0x7f6f500af1c0 2024-03-06T07:38:18.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.463+0000 7f6f55000640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6f5006ee80 con 0x7f6f5015eff0 2024-03-06T07:38:18.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.463+0000 7f6f55000640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6f50071260 con 0x7f6f5006e4d0 2024-03-06T07:38:18.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f45400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5015eff0 0x7f6f50070d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:18.466 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f45400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f6f5015eff0 0x7f6f50070d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57882/0 (socket says 172.21.15.86:57882) 2024-03-06T07:38:18.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f45400640 1 -- 172.21.15.86:0/2107163932 learned_addr learned my addr 172.21.15.86:0/2107163932 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:18.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f4e800640 1 --2- 172.21.15.86:0/2107163932 >> v2:172.21.15.194:3300/0 conn(0x7f6f5006e4d0 0x7f6f5006e940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:18.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f45400640 1 -- 172.21.15.86:0/2107163932 >> v2:172.21.15.194:3300/0 conn(0x7f6f5006e4d0 msgr2=0x7f6f5006e940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:18.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f45400640 1 --2- 172.21.15.86:0/2107163932 >> v2:172.21.15.194:3300/0 conn(0x7f6f5006e4d0 0x7f6f5006e940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.467 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f45400640 1 -- 172.21.15.86:0/2107163932 >> v1:172.21.15.194:6789/0 conn(0x7f6f500af1c0 legacy=0x7f6f5006cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:18.468 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.464+0000 7f6f45400640 1 -- 172.21.15.86:0/2107163932 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f5015f4a0 con 0x7f6f5015eff0 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.465+0000 7f6f45400640 1 --2- 172.21.15.86:0/2107163932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5015eff0 0x7f6f50070d20 secure :-1 s=READY pgs=1810 cs=0 l=1 rev1=1 crypto rx=0x7f6f40009870 tx=0x7f6f4002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=894027bc80e34a51 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f4d400640 1 -- 172.21.15.86:0/2107163932 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f40035020 con 0x7f6f5015eff0 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f4d400640 1 -- 172.21.15.86:0/2107163932 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6f40031870 con 0x7f6f5015eff0 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f4d400640 1 -- 172.21.15.86:0/2107163932 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f40031c40 con 0x7f6f5015eff0 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 -- 172.21.15.86:0/2107163932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5015eff0 msgr2=0x7f6f50070d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 --2- 172.21.15.86:0/2107163932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5015eff0 0x7f6f50070d20 secure :-1 s=READY pgs=1810 cs=0 l=1 rev1=1 crypto rx=0x7f6f40009870 tx=0x7f6f4002b910 comp rx=0 tx=0).stop 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 -- 172.21.15.86:0/2107163932 shutdown_connections 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 --2- 172.21.15.86:0/2107163932 >> v2:172.21.15.194:3300/0 conn(0x7f6f5006e4d0 0x7f6f5006e940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 --2- 172.21.15.86:0/2107163932 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5015eff0 0x7f6f50070d20 unknown :-1 s=CLOSED pgs=1810 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 -- 172.21.15.86:0/2107163932 >> 172.21.15.86:0/2107163932 conn(0x7f6f500b1730 msgr2=0x7f6f500b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:18.469 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 -- 172.21.15.86:0/2107163932 shutdown_connections 2024-03-06T07:38:18.470 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.466+0000 7f6f55000640 1 -- 172.21.15.86:0/2107163932 wait complete. 2024-03-06T07:38:18.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.468+0000 7f6f55000640 1 Processor -- start 2024-03-06T07:38:18.471 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.468+0000 7f6f55000640 1 -- start start 2024-03-06T07:38:18.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.469+0000 7f6f55000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5006e4d0 0x7f6f50160ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:18.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.469+0000 7f6f55000640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f500af1c0 0x7f6f50146790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:18.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.469+0000 7f6f55000640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5015eff0 0x7f6f50146cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:18.473 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.469+0000 7f6f55000640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f50071a50 con 0x7f6f5006e4d0 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.469+0000 7f6f55000640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f50071770 con 0x7f6f500af1c0 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.469+0000 7f6f55000640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6f500b30b0 con 0x7f6f5015eff0 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.469+0000 7f6f4de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5006e4d0 0x7f6f50160ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5006e4d0 0x7f6f50160ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57884/0 (socket says 172.21.15.86:57884) 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 -- 172.21.15.86:0/2626332041 learned_addr learned my addr 172.21.15.86:0/2626332041 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 -- 172.21.15.86:0/2626332041 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5015eff0 msgr2=0x7f6f50146cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5015eff0 0x7f6f50146cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 -- 172.21.15.86:0/2626332041 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f500af1c0 msgr2=0x7f6f50146790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f500af1c0 0x7f6f50146790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 -- 172.21.15.86:0/2626332041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f50147580 con 0x7f6f5006e4d0 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f4de00640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5006e4d0 0x7f6f50160ea0 secure :-1 s=READY pgs=1811 cs=0 l=1 rev1=1 crypto rx=0x7f6f3800c430 tx=0x7f6f3800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:18.474 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.470+0000 7f6f46a00640 1 -- 172.21.15.86:0/2626332041 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f38011790 con 0x7f6f5006e4d0 2024-03-06T07:38:18.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.471+0000 7f6f46a00640 1 -- 172.21.15.86:0/2626332041 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6f38017070 con 0x7f6f5006e4d0 2024-03-06T07:38:18.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.471+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6f50147860 con 0x7f6f5006e4d0 2024-03-06T07:38:18.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.471+0000 7f6f46a00640 1 -- 172.21.15.86:0/2626332041 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f38011bb0 con 0x7f6f5006e4d0 2024-03-06T07:38:18.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.471+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6f501613e0 con 0x7f6f5006e4d0 2024-03-06T07:38:18.475 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.472+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6f5004f330 con 0x7f6f5006e4d0 2024-03-06T07:38:18.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.473+0000 7f6f46a00640 1 -- 172.21.15.86:0/2626332041 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6f3801c460 con 0x7f6f5006e4d0 2024-03-06T07:38:18.476 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.474+0000 7f6f46a00640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f2c06b4d0 0x7f6f2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:18.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.474+0000 7f6f46a00640 1 -- 172.21.15.86:0/2626332041 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6f3808a1c0 con 0x7f6f5006e4d0 2024-03-06T07:38:18.477 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.474+0000 7f6f45400640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f2c06b4d0 0x7f6f2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:18.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.475+0000 7f6f45400640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f2c06b4d0 0x7f6f2c06d990 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7f6f40036000 tx=0x7f6f40030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:18.481 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.478+0000 7f6f46a00640 1 -- 172.21.15.86:0/2626332041 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6f38058f60 con 0x7f6f5006e4d0 2024-03-06T07:38:18.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.713+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6f500b3720 con 0x7f6f5006e4d0 2024-03-06T07:38:18.716 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.714+0000 7f6f46a00640 1 -- 172.21.15.86:0/2626332041 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f6f38058860 con 0x7f6f5006e4d0 2024-03-06T07:38:18.717 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:18.717 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:18.717 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f2c06b4d0 msgr2=0x7f6f2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f2c06b4d0 0x7f6f2c06d990 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7f6f40036000 tx=0x7f6f40030000 comp rx=0 tx=0).stop 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5006e4d0 msgr2=0x7f6f50160ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5006e4d0 0x7f6f50160ea0 secure :-1 s=READY pgs=1811 cs=0 l=1 rev1=1 crypto rx=0x7f6f3800c430 tx=0x7f6f3800c8f0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 shutdown_connections 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f6f2c06b4d0 0x7f6f2c06d990 unknown :-1 s=CLOSED pgs=768 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6f5015eff0 0x7f6f50146cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6f500af1c0 0x7f6f50146790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 --2- 172.21.15.86:0/2626332041 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6f5006e4d0 0x7f6f50160ea0 unknown :-1 s=CLOSED pgs=1811 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:18.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 >> 172.21.15.86:0/2626332041 conn(0x7f6f500b1730 msgr2=0x7f6f500b3f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:18.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 shutdown_connections 2024-03-06T07:38:18.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:18.717+0000 7f6f55000640 1 -- 172.21.15.86:0/2626332041 wait complete. 2024-03-06T07:38:18.940 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:18.940 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:19.709 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:19.941 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:20.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.155+0000 7f3595e00640 1 Processor -- start 2024-03-06T07:38:20.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.155+0000 7f3595e00640 1 -- start start 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.156+0000 7f3595e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f35900cec90 0x7f35900cf190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.156+0000 7f3595e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35900cd240 0x7f35900b56d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.156+0000 7f3595e00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f35900cdc50 con 0x7f359015bfa0 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.156+0000 7f3595e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35900b5240 con 0x7f35900cd240 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.156+0000 7f3595e00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35900cd6b0 con 0x7f35900cec90 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f358f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35900cd240 0x7f35900b56d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f358f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f35900cd240 0x7f35900b56d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57886/0 (socket says 172.21.15.86:57886) 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f358f400640 1 -- 172.21.15.86:0/2339343556 learned_addr learned my addr 172.21.15.86:0/2339343556 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:20.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f3586000640 1 --2- 172.21.15.86:0/2339343556 >> v2:172.21.15.194:3300/0 conn(0x7f35900cec90 0x7f35900cf190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:20.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f358f400640 1 -- 172.21.15.86:0/2339343556 >> v2:172.21.15.194:3300/0 conn(0x7f35900cec90 msgr2=0x7f35900cf190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:20.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f358f400640 1 --2- 172.21.15.86:0/2339343556 >> v2:172.21.15.194:3300/0 conn(0x7f35900cec90 0x7f35900cf190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f358f400640 1 -- 172.21.15.86:0/2339343556 >> v1:172.21.15.194:6789/0 conn(0x7f359015bfa0 legacy=0x7f35900ce580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:20.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.157+0000 7f358f400640 1 -- 172.21.15.86:0/2339343556 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35900cd970 con 0x7f35900cd240 2024-03-06T07:38:20.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.159+0000 7f358f400640 1 --2- 172.21.15.86:0/2339343556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35900cd240 0x7f35900b56d0 secure :-1 s=READY pgs=1812 cs=0 l=1 rev1=1 crypto rx=0x7f3590071c50 tx=0x7f358802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ff8fbbca6cceb45 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:20.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.159+0000 7f358e000640 1 -- 172.21.15.86:0/2339343556 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f358802ca80 con 0x7f35900cd240 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.159+0000 7f358e000640 1 -- 172.21.15.86:0/2339343556 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f358802cc20 con 0x7f35900cd240 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.159+0000 7f358e000640 1 -- 172.21.15.86:0/2339343556 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35880343f0 con 0x7f35900cd240 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 -- 172.21.15.86:0/2339343556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35900cd240 msgr2=0x7f35900b56d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 --2- 172.21.15.86:0/2339343556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35900cd240 0x7f35900b56d0 secure :-1 s=READY pgs=1812 cs=0 l=1 rev1=1 crypto rx=0x7f3590071c50 tx=0x7f358802bee0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 -- 172.21.15.86:0/2339343556 shutdown_connections 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 --2- 172.21.15.86:0/2339343556 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f35900cd240 0x7f35900b56d0 unknown :-1 s=CLOSED pgs=1812 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 --2- 172.21.15.86:0/2339343556 >> v2:172.21.15.194:3300/0 conn(0x7f35900cec90 0x7f35900cf190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 -- 172.21.15.86:0/2339343556 >> 172.21.15.86:0/2339343556 conn(0x7f35900d1c90 msgr2=0x7f35900d20a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 -- 172.21.15.86:0/2339343556 shutdown_connections 2024-03-06T07:38:20.163 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.160+0000 7f3595e00640 1 -- 172.21.15.86:0/2339343556 wait complete. 2024-03-06T07:38:20.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.162+0000 7f3595e00640 1 Processor -- start 2024-03-06T07:38:20.164 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.162+0000 7f3595e00640 1 -- start start 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f3595e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35900cd240 0x7f35900c5d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f3595e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35900cec90 0x7f359014bfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f3595e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f359015bfa0 0x7f35900b2c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f3595e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f359015c380 con 0x7f359015bfa0 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f3595e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35900b4ed0 con 0x7f35900cec90 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f3595e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35900b5c10 con 0x7f35900cd240 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f358f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f359015bfa0 0x7f35900b2c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:20.168 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f358f400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f359015bfa0 0x7f35900b2c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57902/0 (socket says 172.21.15.86:57902) 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f358f400640 1 -- 172.21.15.86:0/3914052713 learned_addr learned my addr 172.21.15.86:0/3914052713 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f358ea00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35900cd240 0x7f35900c5d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.164+0000 7f3586000640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35900cec90 0x7f359014bfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f358f400640 1 -- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35900cd240 msgr2=0x7f35900c5d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f358f400640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35900cd240 0x7f35900c5d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f358f400640 1 -- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35900cec90 msgr2=0x7f359014bfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f358f400640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35900cec90 0x7f359014bfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f358f400640 1 -- 172.21.15.86:0/3914052713 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35900c6a30 con 0x7f359015bfa0 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f3586000640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35900cec90 0x7f359014bfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f358f400640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f359015bfa0 0x7f35900b2c30 secure :-1 s=READY pgs=1813 cs=0 l=1 rev1=1 crypto rx=0x7f3588036040 tx=0x7f3588004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f3587e00640 1 -- 172.21.15.86:0/3914052713 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3588043020 con 0x7f359015bfa0 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f3587e00640 1 -- 172.21.15.86:0/3914052713 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f358803e450 con 0x7f359015bfa0 2024-03-06T07:38:20.169 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35900c6d10 con 0x7f359015bfa0 2024-03-06T07:38:20.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.165+0000 7f3587e00640 1 -- 172.21.15.86:0/3914052713 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3588009510 con 0x7f359015bfa0 2024-03-06T07:38:20.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.166+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35900c72d0 con 0x7f359015bfa0 2024-03-06T07:38:20.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.167+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f359004f330 con 0x7f359015bfa0 2024-03-06T07:38:20.170 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.168+0000 7f3587e00640 1 -- 172.21.15.86:0/3914052713 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3588046020 con 0x7f359015bfa0 2024-03-06T07:38:20.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.169+0000 7f3587e00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f356c06b4d0 0x7f356c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:20.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.169+0000 7f3587e00640 1 -- 172.21.15.86:0/3914052713 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f35880acb30 con 0x7f359015bfa0 2024-03-06T07:38:20.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.169+0000 7f358ea00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f356c06b4d0 0x7f356c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:20.172 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.170+0000 7f358ea00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f356c06b4d0 0x7f356c06d990 secure :-1 s=READY pgs=769 cs=0 l=1 rev1=1 crypto rx=0x7f358000a750 tx=0x7f3580008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:20.175 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.173+0000 7f3587e00640 1 -- 172.21.15.86:0/3914052713 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f358804d030 con 0x7f359015bfa0 2024-03-06T07:38:20.478 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.475+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f35900684a0 con 0x7f359015bfa0 2024-03-06T07:38:20.479 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.476+0000 7f3587e00640 1 -- 172.21.15.86:0/3914052713 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f358807b9f0 con 0x7f359015bfa0 2024-03-06T07:38:20.479 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:20.479 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:20.479 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f356c06b4d0 msgr2=0x7f356c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f356c06b4d0 0x7f356c06d990 secure :-1 s=READY pgs=769 cs=0 l=1 rev1=1 crypto rx=0x7f358000a750 tx=0x7f3580008040 comp rx=0 tx=0).stop 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f359015bfa0 msgr2=0x7f35900b2c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f359015bfa0 0x7f35900b2c30 secure :-1 s=READY pgs=1813 cs=0 l=1 rev1=1 crypto rx=0x7f3588036040 tx=0x7f3588004a40 comp rx=0 tx=0).stop 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 shutdown_connections 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f356c06b4d0 0x7f356c06d990 unknown :-1 s=CLOSED pgs=769 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f359015bfa0 0x7f35900b2c30 unknown :-1 s=CLOSED pgs=1813 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f35900cec90 0x7f359014bfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 --2- 172.21.15.86:0/3914052713 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f35900cd240 0x7f35900c5d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 >> 172.21.15.86:0/3914052713 conn(0x7f35900d1c90 msgr2=0x7f35900b6f70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 shutdown_connections 2024-03-06T07:38:20.482 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:20.479+0000 7f3595e00640 1 -- 172.21.15.86:0/3914052713 wait complete. 2024-03-06T07:38:20.700 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:20.700 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:21.701 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:21.710 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:21.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.908+0000 7f662fe00640 1 Processor -- start 2024-03-06T07:38:21.910 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.908+0000 7f662fe00640 1 -- start start 2024-03-06T07:38:21.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.910+0000 7f662fe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f662815f9d0 0x7f662815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 0x7f6628070f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662fe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f662806cef0 con 0x7f6628144c00 2024-03-06T07:38:21.913 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662fe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f662806d1d0 con 0x7f662815f9d0 2024-03-06T07:38:21.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662fe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f662806d4b0 con 0x7f6628070b10 2024-03-06T07:38:21.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662d400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f6628144c00 0x7f662806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49800/0 (socket says 172.21.15.86:49800) 2024-03-06T07:38:21.914 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662d400640 1 -- 172.21.15.86:0/415032870 learned_addr learned my addr 172.21.15.86:0/415032870 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662ca00640 1 --2- 172.21.15.86:0/415032870 >> v2:172.21.15.194:3300/0 conn(0x7f662815f9d0 0x7f662815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1987071979 0 0) 0x7f662806cef0 con 0x7f6628144c00 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f662de00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 0x7f6628070f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.911+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6610003520 con 0x7f6628144c00 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2140684342 0 0) 0x7f6610003520 con 0x7f6628144c00 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 msgr2=0x7f6628070f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 0x7f6628070f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 >> v2:172.21.15.194:3300/0 conn(0x7f662815f9d0 msgr2=0x7f662815fe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 --2- 172.21.15.86:0/415032870 >> v2:172.21.15.194:3300/0 conn(0x7f662815f9d0 0x7f662815fe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f662806d790 con 0x7f6628144c00 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f661c001670 con 0x7f6628144c00 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 >> v1:172.21.15.86:6789/0 conn(0x7f6628144c00 legacy=0x7f662806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:21.916 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66100039b0 0x7f6610003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6610004800 0x7f6610004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 0x7f66100051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f662806d1d0 con 0x7f6610004800 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f662806d4b0 con 0x7f66100039b0 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f661000ba10 con 0x7f6628070b10 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.912+0000 7f662ca00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6610004800 0x7f6610004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662d400640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66100039b0 0x7f6610003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662ca00640 1 -- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 msgr2=0x7f66100051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662de00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 0x7f66100051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662ca00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6628070b10 0x7f66100051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662ca00640 1 -- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66100039b0 msgr2=0x7f6610003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662ca00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66100039b0 0x7f6610003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:21.917 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662ca00640 1 -- 172.21.15.86:0/415032870 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6610003520 con 0x7f6610004800 2024-03-06T07:38:21.918 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.913+0000 7f662d400640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f66100039b0 0x7f6610003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.914+0000 7f662ca00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6610004800 0x7f6610004c80 secure :-1 s=READY pgs=1815 cs=0 l=1 rev1=1 crypto rx=0x7f6614009ac0 tx=0x7f6610005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6794cbfb95665bb1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.914+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6614039070 con 0x7f6610004800 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.914+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f661402c710 con 0x7f6610004800 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.914+0000 7f6627e00640 1 -- 172.21.15.86:0/415032870 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6614009690 con 0x7f6610004800 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.915+0000 7f662fe00640 1 -- 172.21.15.86:0/415032870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6610004800 msgr2=0x7f6610004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.915+0000 7f662fe00640 1 --2- 172.21.15.86:0/415032870 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f6610004800 0x7f6610004c80 secure :-1 s=READY pgs=1815 cs=0 l=1 rev1=1 crypto rx=0x7f6614009ac0 tx=0x7f6610005830 comp rx=0 tx=0).stop 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.915+0000 7f662de00640 1 -- 172.21.15.86:0/415032870 reap_dead start 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.915+0000 7f662fe00640 1 -- 172.21.15.86:0/415032870 shutdown_connections 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.915+0000 7f662fe00640 1 -- 172.21.15.86:0/415032870 >> 172.21.15.86:0/415032870 conn(0x7f66280b1730 msgr2=0x7f66280b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:21.919 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.915+0000 7f662fe00640 1 -- 172.21.15.86:0/415032870 shutdown_connections 2024-03-06T07:38:21.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.915+0000 7f662fe00640 1 -- 172.21.15.86:0/415032870 wait complete. 2024-03-06T07:38:21.920 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.917+0000 7f662fe00640 1 Processor -- start 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.917+0000 7f662fe00640 1 -- start start 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662fe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6610004800 0x7f66281f6d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662fe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f662815ef70 0x7f66281f3d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f662815f3f0 0x7f66281f42b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662fe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f662806ccd0 con 0x7f662815f3f0 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662fe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f662806cb50 con 0x7f662815ef70 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662fe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f662806df90 con 0x7f6610004800 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f662815f3f0 0x7f66281f42b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f662815f3f0 0x7f66281f42b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57918/0 (socket says 172.21.15.86:57918) 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 -- 172.21.15.86:0/1936656856 learned_addr learned my addr 172.21.15.86:0/1936656856 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 -- 172.21.15.86:0/1936656856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6610004800 msgr2=0x7f66281f6d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:21.922 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662d400640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6610004800 0x7f66281f6d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6610004800 0x7f66281f6d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:21.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 -- 172.21.15.86:0/1936656856 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f662815ef70 msgr2=0x7f66281f3d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:21.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f662815ef70 0x7f66281f3d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:21.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.918+0000 7f662de00640 1 -- 172.21.15.86:0/1936656856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6628141bd0 con 0x7f662815f3f0 2024-03-06T07:38:21.923 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.919+0000 7f662de00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f662815f3f0 0x7f66281f42b0 secure :-1 s=READY pgs=1816 cs=0 l=1 rev1=1 crypto rx=0x7f66280d0ac0 tx=0x7f6620009cd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:21.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.919+0000 7f6625600640 1 -- 172.21.15.86:0/1936656856 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6620019070 con 0x7f662815f3f0 2024-03-06T07:38:21.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.919+0000 7f6625600640 1 -- 172.21.15.86:0/1936656856 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6620007d40 con 0x7f662815f3f0 2024-03-06T07:38:21.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.919+0000 7f6625600640 1 -- 172.21.15.86:0/1936656856 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6620014640 con 0x7f662815f3f0 2024-03-06T07:38:21.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.919+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6628141eb0 con 0x7f662815f3f0 2024-03-06T07:38:21.924 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.919+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6628142470 con 0x7f662815f3f0 2024-03-06T07:38:21.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.920+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f662804f330 con 0x7f662815f3f0 2024-03-06T07:38:21.925 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.921+0000 7f6625600640 1 -- 172.21.15.86:0/1936656856 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f662001c070 con 0x7f662815f3f0 2024-03-06T07:38:21.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.923+0000 7f6625600640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f660806b4d0 0x7f660806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:21.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.923+0000 7f6625600640 1 -- 172.21.15.86:0/1936656856 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6620089bb0 con 0x7f662815f3f0 2024-03-06T07:38:21.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.923+0000 7f662d400640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f660806b4d0 0x7f660806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:21.926 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.924+0000 7f662d400640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f660806b4d0 0x7f660806d990 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f661c007990 tx=0x7f661c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:21.929 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:21.927+0000 7f6625600640 1 -- 172.21.15.86:0/1936656856 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f66200589e0 con 0x7f662815f3f0 2024-03-06T07:38:22.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.159+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6628072890 con 0x7f662815f3f0 2024-03-06T07:38:22.162 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.160+0000 7f6625600640 1 -- 172.21.15.86:0/1936656856 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f66200582e0 con 0x7f662815f3f0 2024-03-06T07:38:22.162 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:22.162 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:22.163 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:22.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.162+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f660806b4d0 msgr2=0x7f660806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:22.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.162+0000 7f662fe00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f660806b4d0 0x7f660806d990 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f661c007990 tx=0x7f661c009040 comp rx=0 tx=0).stop 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.162+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f662815f3f0 msgr2=0x7f66281f42b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.162+0000 7f662fe00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f662815f3f0 0x7f66281f42b0 secure :-1 s=READY pgs=1816 cs=0 l=1 rev1=1 crypto rx=0x7f66280d0ac0 tx=0x7f6620009cd0 comp rx=0 tx=0).stop 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 shutdown_connections 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f660806b4d0 0x7f660806d990 unknown :-1 s=CLOSED pgs=770 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f662815f3f0 0x7f66281f42b0 unknown :-1 s=CLOSED pgs=1816 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f662815ef70 0x7f66281f3d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 --2- 172.21.15.86:0/1936656856 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6610004800 0x7f66281f6d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 >> 172.21.15.86:0/1936656856 conn(0x7f66280b1730 msgr2=0x7f66280af5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 shutdown_connections 2024-03-06T07:38:22.166 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:22.163+0000 7f662fe00640 1 -- 172.21.15.86:0/1936656856 wait complete. 2024-03-06T07:38:22.385 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:22.385 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:23.387 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:23.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.595+0000 7ff67e400640 1 Processor -- start 2024-03-06T07:38:23.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.595+0000 7ff67e400640 1 -- start start 2024-03-06T07:38:23.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.596+0000 7ff67e400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff67815f9d0 0x7ff67815fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:23.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff67e400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff678070f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:23.599 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff67e400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff67806cef0 con 0x7ff678144c00 2024-03-06T07:38:23.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff67e400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff67806d1d0 con 0x7ff67815f9d0 2024-03-06T07:38:23.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff67e400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff67806d4b0 con 0x7ff678070b10 2024-03-06T07:38:23.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff66ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff67815f9d0 0x7ff67815fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:23.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff66ea00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7ff67815f9d0 0x7ff67815fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57928/0 (socket says 172.21.15.86:57928) 2024-03-06T07:38:23.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff66ea00640 1 -- 172.21.15.86:0/1183547671 learned_addr learned my addr 172.21.15.86:0/1183547671 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:23.600 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff677400640 1 --1- 172.21.15.86:0/1183547671 >> v1:172.21.15.86:6789/0 conn(0x7ff678144c00 0x7ff67806e930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49810/0 (socket says 172.21.15.86:49810) 2024-03-06T07:38:23.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff677e00640 1 --2- 172.21.15.86:0/1183547671 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff678070f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:23.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff66ea00640 1 -- 172.21.15.86:0/1183547671 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 msgr2=0x7ff678070f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:23.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff66ea00640 1 --2- 172.21.15.86:0/1183547671 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff678070f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.597+0000 7ff66ea00640 1 -- 172.21.15.86:0/1183547671 >> v1:172.21.15.86:6789/0 conn(0x7ff678144c00 legacy=0x7ff67806e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.598+0000 7ff66ea00640 1 -- 172.21.15.86:0/1183547671 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff67806d790 con 0x7ff67815f9d0 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.598+0000 7ff676a00640 1 -- 172.21.15.86:0/1183547671 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1043710066 0 0) 0x7ff67806cef0 con 0x7ff678144c00 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.598+0000 7ff677e00640 1 --2- 172.21.15.86:0/1183547671 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff678070f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.599+0000 7ff66ea00640 1 --2- 172.21.15.86:0/1183547671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67815fe60 secure :-1 s=READY pgs=1818 cs=0 l=1 rev1=1 crypto rx=0x7ff660009870 tx=0x7ff66002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=719872a258bc073d server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.599+0000 7ff676a00640 1 -- 172.21.15.86:0/1183547671 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff660034070 con 0x7ff67815f9d0 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.599+0000 7ff676a00640 1 -- 172.21.15.86:0/1183547671 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff660030840 con 0x7ff67815f9d0 2024-03-06T07:38:23.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.599+0000 7ff676a00640 1 -- 172.21.15.86:0/1183547671 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff660030c10 con 0x7ff67815f9d0 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.599+0000 7ff67e400640 1 -- 172.21.15.86:0/1183547671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 msgr2=0x7ff67815fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.599+0000 7ff67e400640 1 --2- 172.21.15.86:0/1183547671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67815fe60 secure :-1 s=READY pgs=1818 cs=0 l=1 rev1=1 crypto rx=0x7ff660009870 tx=0x7ff66002b980 comp rx=0 tx=0).stop 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.600+0000 7ff67e400640 1 -- 172.21.15.86:0/1183547671 shutdown_connections 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.600+0000 7ff67e400640 1 --2- 172.21.15.86:0/1183547671 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff678070f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.600+0000 7ff67e400640 1 --2- 172.21.15.86:0/1183547671 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67815fe60 unknown :-1 s=CLOSED pgs=1818 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.600+0000 7ff67e400640 1 -- 172.21.15.86:0/1183547671 >> 172.21.15.86:0/1183547671 conn(0x7ff6780b1730 msgr2=0x7ff6780b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.600+0000 7ff67e400640 1 -- 172.21.15.86:0/1183547671 shutdown_connections 2024-03-06T07:38:23.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.600+0000 7ff67e400640 1 -- 172.21.15.86:0/1183547671 wait complete. 2024-03-06T07:38:23.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.602+0000 7ff67e400640 1 Processor -- start 2024-03-06T07:38:23.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.602+0000 7ff67e400640 1 -- start start 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff67e400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff67815e8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff67e400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff678144c00 0x7ff678141020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff67e400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67812ed80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff67e400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff67806ccd0 con 0x7ff67815f9d0 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff67e400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff67806cb50 con 0x7ff678144c00 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff67e400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff67806df90 con 0x7ff678070b10 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67812ed80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:23.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67812ed80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57930/0 (socket says 172.21.15.86:57930) 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 -- 172.21.15.86:0/2029493070 learned_addr learned my addr 172.21.15.86:0/2029493070 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff67815e8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 -- 172.21.15.86:0/2029493070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 msgr2=0x7ff67815e8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff67815e8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 -- 172.21.15.86:0/2029493070 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff678144c00 msgr2=0x7ff678141020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff678144c00 0x7ff678141020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.603+0000 7ff677e00640 1 -- 172.21.15.86:0/2029493070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff678141560 con 0x7ff67815f9d0 2024-03-06T07:38:23.607 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.604+0000 7ff677e00640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67812ed80 secure :-1 s=READY pgs=1819 cs=0 l=1 rev1=1 crypto rx=0x7ff67000cd80 tx=0x7ff67000a650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:23.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.604+0000 7ff66fe00640 1 -- 172.21.15.86:0/2029493070 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff67000f680 con 0x7ff67815f9d0 2024-03-06T07:38:23.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.604+0000 7ff66fe00640 1 -- 172.21.15.86:0/2029493070 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff67000f840 con 0x7ff67815f9d0 2024-03-06T07:38:23.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.604+0000 7ff66fe00640 1 -- 172.21.15.86:0/2029493070 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff67000fc30 con 0x7ff67815f9d0 2024-03-06T07:38:23.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.604+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff678141800 con 0x7ff67815f9d0 2024-03-06T07:38:23.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.604+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff678141dc0 con 0x7ff67815f9d0 2024-03-06T07:38:23.608 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.606+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff678145270 con 0x7ff67815f9d0 2024-03-06T07:38:23.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.606+0000 7ff66fe00640 1 -- 172.21.15.86:0/2029493070 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff6700213f0 con 0x7ff67815f9d0 2024-03-06T07:38:23.609 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.607+0000 7ff66fe00640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff65406b4d0 0x7ff65406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:23.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.607+0000 7ff66fe00640 1 -- 172.21.15.86:0/2029493070 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff67008af80 con 0x7ff67815f9d0 2024-03-06T07:38:23.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.607+0000 7ff677400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff65406b4d0 0x7ff65406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:23.610 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.608+0000 7ff677400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff65406b4d0 0x7ff65406d990 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7ff668007fd0 tx=0x7ff668009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:23.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.612+0000 7ff66fe00640 1 -- 172.21.15.86:0/2029493070 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff678145270 con 0x7ff67815f9d0 2024-03-06T07:38:23.712 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:23.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.863+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff678072890 con 0x7ff67815f9d0 2024-03-06T07:38:23.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.863+0000 7ff66fe00640 1 -- 172.21.15.86:0/2029493070 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7ff670014030 con 0x7ff67815f9d0 2024-03-06T07:38:23.866 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:23.866 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:23.866 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff65406b4d0 msgr2=0x7ff65406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff65406b4d0 0x7ff65406d990 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7ff668007fd0 tx=0x7ff668009040 comp rx=0 tx=0).stop 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 msgr2=0x7ff67812ed80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67812ed80 secure :-1 s=READY pgs=1819 cs=0 l=1 rev1=1 crypto rx=0x7ff67000cd80 tx=0x7ff67000a650 comp rx=0 tx=0).stop 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 shutdown_connections 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7ff65406b4d0 0x7ff65406d990 unknown :-1 s=CLOSED pgs=771 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7ff67815f9d0 0x7ff67812ed80 unknown :-1 s=CLOSED pgs=1819 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.870 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7ff678144c00 0x7ff678141020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 --2- 172.21.15.86:0/2029493070 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7ff678070b10 0x7ff67815e8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:23.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 >> 172.21.15.86:0/2029493070 conn(0x7ff6780b1730 msgr2=0x7ff6780ae400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:23.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 shutdown_connections 2024-03-06T07:38:23.871 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:23.867+0000 7ff67e400640 1 -- 172.21.15.86:0/2029493070 wait complete. 2024-03-06T07:38:24.089 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:24.089 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:25.090 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:25.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.297+0000 7f6593e00640 1 Processor -- start 2024-03-06T07:38:25.299 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.297+0000 7f6593e00640 1 -- start start 2024-03-06T07:38:25.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.298+0000 7f6593e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6594161580 0x7f65940d1950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:25.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.298+0000 7f6593e00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f65940d2a70 0x7f65940b3ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:25.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.298+0000 7f6593e00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6594161960 con 0x7f65940d1e90 2024-03-06T07:38:25.301 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.298+0000 7f6593e00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f65940cf6f0 con 0x7f65940d2a70 2024-03-06T07:38:25.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.298+0000 7f6593e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f65940cf9d0 con 0x7f6594161580 2024-03-06T07:38:25.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6593400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f65940d2a70 0x7f65940b3ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:25.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6593400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f65940d2a70 0x7f65940b3ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57936/0 (socket says 172.21.15.86:57936) 2024-03-06T07:38:25.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f658be00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f65940d1e90 0x7f65940d2360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:49814/0 (socket says 172.21.15.86:49814) 2024-03-06T07:38:25.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6593400640 1 -- 172.21.15.86:0/3605445997 learned_addr learned my addr 172.21.15.86:0/3605445997 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:25.302 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6592a00640 1 --2- 172.21.15.86:0/3605445997 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6594161580 0x7f65940d1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:25.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6593400640 1 -- 172.21.15.86:0/3605445997 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6594161580 msgr2=0x7f65940d1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:25.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6593400640 1 --2- 172.21.15.86:0/3605445997 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6594161580 0x7f65940d1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6593400640 1 -- 172.21.15.86:0/3605445997 >> v1:172.21.15.86:6789/0 conn(0x7f65940d1e90 legacy=0x7f65940d2360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:25.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6593400640 1 -- 172.21.15.86:0/3605445997 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65940cfcb0 con 0x7f65940d2a70 2024-03-06T07:38:25.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.299+0000 7f6592000640 1 -- 172.21.15.86:0/3605445997 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2454758437 0 0) 0x7f6594161960 con 0x7f65940d1e90 2024-03-06T07:38:25.303 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.300+0000 7f6593400640 1 --2- 172.21.15.86:0/3605445997 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d2a70 0x7f65940b3ec0 secure :-1 s=READY pgs=1821 cs=0 l=1 rev1=1 crypto rx=0x7f658c009e50 tx=0x7f658c02bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=911c18ba52c8f5cd server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:25.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6592000640 1 -- 172.21.15.86:0/3605445997 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f658c02cb00 con 0x7f65940d2a70 2024-03-06T07:38:25.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6592000640 1 -- 172.21.15.86:0/3605445997 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f658c02cca0 con 0x7f65940d2a70 2024-03-06T07:38:25.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6592000640 1 -- 172.21.15.86:0/3605445997 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f658c034610 con 0x7f65940d2a70 2024-03-06T07:38:25.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 -- 172.21.15.86:0/3605445997 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d2a70 msgr2=0x7f65940b3ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:25.304 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 --2- 172.21.15.86:0/3605445997 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d2a70 0x7f65940b3ec0 secure :-1 s=READY pgs=1821 cs=0 l=1 rev1=1 crypto rx=0x7f658c009e50 tx=0x7f658c02bf60 comp rx=0 tx=0).stop 2024-03-06T07:38:25.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 -- 172.21.15.86:0/3605445997 shutdown_connections 2024-03-06T07:38:25.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 --2- 172.21.15.86:0/3605445997 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d2a70 0x7f65940b3ec0 unknown :-1 s=CLOSED pgs=1821 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 --2- 172.21.15.86:0/3605445997 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f6594161580 0x7f65940d1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 -- 172.21.15.86:0/3605445997 >> 172.21.15.86:0/3605445997 conn(0x7f65940d3d80 msgr2=0x7f65940d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:25.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 -- 172.21.15.86:0/3605445997 shutdown_connections 2024-03-06T07:38:25.305 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.301+0000 7f6593e00640 1 -- 172.21.15.86:0/3605445997 wait complete. 2024-03-06T07:38:25.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.304+0000 7f6593e00640 1 Processor -- start 2024-03-06T07:38:25.307 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.304+0000 7f6593e00640 1 -- start start 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6593e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d1e90 0x7f65941517d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6593e00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f65940d2a70 0x7f65940cae50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6593e00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6594161580 0x7f65940b1a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6593e00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65940cf3a0 con 0x7f65940d1e90 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6593e00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65940b39a0 con 0x7f6594161580 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6593e00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f65940b4400 con 0x7f65940d2a70 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6592a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d1e90 0x7f65941517d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6592a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d1e90 0x7f65941517d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:57944/0 (socket says 172.21.15.86:57944) 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.306+0000 7f6592a00640 1 -- 172.21.15.86:0/4049935051 learned_addr learned my addr 172.21.15.86:0/4049935051 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6592a00640 1 -- 172.21.15.86:0/4049935051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f65940d2a70 msgr2=0x7f65940cae50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:25.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6592a00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f65940d2a70 0x7f65940cae50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6592a00640 1 -- 172.21.15.86:0/4049935051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6594161580 msgr2=0x7f65940b1a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6592a00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6594161580 0x7f65940b1a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6592a00640 1 -- 172.21.15.86:0/4049935051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6594152660 con 0x7f65940d1e90 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6592a00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d1e90 0x7f65941517d0 secure :-1 s=READY pgs=1822 cs=0 l=1 rev1=1 crypto rx=0x7f658400d650 tx=0x7f658400db10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f658b400640 1 -- 172.21.15.86:0/4049935051 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f658400b6c0 con 0x7f65940d1e90 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f658b400640 1 -- 172.21.15.86:0/4049935051 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6584019070 con 0x7f65940d1e90 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f658b400640 1 -- 172.21.15.86:0/4049935051 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f658400baa0 con 0x7f65940d1e90 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65940cb530 con 0x7f65940d1e90 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.307+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65940cbaf0 con 0x7f65940d1e90 2024-03-06T07:38:25.311 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.309+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f659415f8e0 con 0x7f65940d1e90 2024-03-06T07:38:25.312 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.309+0000 7f658b400640 1 -- 172.21.15.86:0/4049935051 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f658400bc40 con 0x7f65940d1e90 2024-03-06T07:38:25.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.310+0000 7f658b400640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f657006b500 0x7f657006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:25.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.310+0000 7f658b400640 1 -- 172.21.15.86:0/4049935051 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f658408a530 con 0x7f65940d1e90 2024-03-06T07:38:25.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.311+0000 7f658be00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f657006b500 0x7f657006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:25.313 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.311+0000 7f658be00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f657006b500 0x7f657006d9c0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f657c007fd0 tx=0x7f657c007920 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:25.317 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.315+0000 7f658b400640 1 -- 172.21.15.86:0/4049935051 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f659415f8e0 con 0x7f65940d1e90 2024-03-06T07:38:25.681 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:38:25.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:38:25.695 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.692+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f659404f330 con 0x7f65940d1e90 2024-03-06T07:38:25.696 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.693+0000 7f658b400640 1 -- 172.21.15.86:0/4049935051 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f6594152660 con 0x7f65940d1e90 2024-03-06T07:38:25.696 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:25.696 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:25.697 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:25.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f657006b500 msgr2=0x7f657006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:25.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f657006b500 0x7f657006d9c0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f657c007fd0 tx=0x7f657c007920 comp rx=0 tx=0).stop 2024-03-06T07:38:25.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d1e90 msgr2=0x7f65941517d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:25.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d1e90 0x7f65941517d0 secure :-1 s=READY pgs=1822 cs=0 l=1 rev1=1 crypto rx=0x7f658400d650 tx=0x7f658400db10 comp rx=0 tx=0).stop 2024-03-06T07:38:25.698 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 shutdown_connections 2024-03-06T07:38:25.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f657006b500 0x7f657006d9c0 unknown :-1 s=CLOSED pgs=772 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f6594161580 0x7f65940b1a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f65940d2a70 0x7f65940cae50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 --2- 172.21.15.86:0/4049935051 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f65940d1e90 0x7f65941517d0 unknown :-1 s=CLOSED pgs=1822 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:25.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 >> 172.21.15.86:0/4049935051 conn(0x7f65940d3d80 msgr2=0x7f659415fb80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:25.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 shutdown_connections 2024-03-06T07:38:25.699 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:25.695+0000 7f6593e00640 1 -- 172.21.15.86:0/4049935051 wait complete. 2024-03-06T07:38:25.714 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:25.916 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:25.916 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:26.917 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:27.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.132+0000 7f9a6c200640 1 Processor -- start 2024-03-06T07:38:27.134 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.132+0000 7f9a6c200640 1 -- start start 2024-03-06T07:38:27.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.132+0000 7f9a6c200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9a6415acb0 0x7f9a640b37d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:27.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a6c200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f9a6406ec00 0x7f9a640ccfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:27.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a6c200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9a6415b090 con 0x7f9a640b4680 2024-03-06T07:38:27.136 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a6c200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9a640b2a30 con 0x7f9a6415acb0 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a6c200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9a640b4360 con 0x7f9a6406ec00 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a69800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9a6415acb0 0x7f9a640b37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a68e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f9a640b4680 0x7f9a6406e4f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36774/0 (socket says 172.21.15.86:36774) 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a68e00640 1 -- 172.21.15.86:0/2678821951 learned_addr learned my addr 172.21.15.86:0/2678821951 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.133+0000 7f9a6a200640 1 --2- 172.21.15.86:0/2678821951 >> v2:172.21.15.194:3300/0 conn(0x7f9a6406ec00 0x7f9a640ccfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.134+0000 7f9a69800640 1 -- 172.21.15.86:0/2678821951 >> v2:172.21.15.194:3300/0 conn(0x7f9a6406ec00 msgr2=0x7f9a640ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.134+0000 7f9a69800640 1 --2- 172.21.15.86:0/2678821951 >> v2:172.21.15.194:3300/0 conn(0x7f9a6406ec00 0x7f9a640ccfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.134+0000 7f9a69800640 1 -- 172.21.15.86:0/2678821951 >> v1:172.21.15.86:6789/0 conn(0x7f9a640b4680 legacy=0x7f9a6406e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.134+0000 7f9a69800640 1 -- 172.21.15.86:0/2678821951 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a640b2cf0 con 0x7f9a6415acb0 2024-03-06T07:38:27.137 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.134+0000 7f9a63e00640 1 -- 172.21.15.86:0/2678821951 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3250967361 0 0) 0x7f9a6415b090 con 0x7f9a640b4680 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.135+0000 7f9a69800640 1 --2- 172.21.15.86:0/2678821951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6415acb0 0x7f9a640b37d0 secure :-1 s=READY pgs=1824 cs=0 l=1 rev1=1 crypto rx=0x7f9a640b4e60 tx=0x7f9a5802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8aef8492ba15a3e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.135+0000 7f9a63e00640 1 -- 172.21.15.86:0/2678821951 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a58035020 con 0x7f9a6415acb0 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.135+0000 7f9a63e00640 1 -- 172.21.15.86:0/2678821951 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9a58031830 con 0x7f9a6415acb0 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.135+0000 7f9a63e00640 1 -- 172.21.15.86:0/2678821951 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a58031c00 con 0x7f9a6415acb0 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 -- 172.21.15.86:0/2678821951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6415acb0 msgr2=0x7f9a640b37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2678821951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6415acb0 0x7f9a640b37d0 secure :-1 s=READY pgs=1824 cs=0 l=1 rev1=1 crypto rx=0x7f9a640b4e60 tx=0x7f9a5802b8d0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 -- 172.21.15.86:0/2678821951 shutdown_connections 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2678821951 >> v2:172.21.15.194:3300/0 conn(0x7f9a6406ec00 0x7f9a640ccfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2678821951 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6415acb0 0x7f9a640b37d0 unknown :-1 s=CLOSED pgs=1824 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 -- 172.21.15.86:0/2678821951 >> 172.21.15.86:0/2678821951 conn(0x7f9a640b1730 msgr2=0x7f9a640b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:27.139 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 -- 172.21.15.86:0/2678821951 shutdown_connections 2024-03-06T07:38:27.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.136+0000 7f9a6c200640 1 -- 172.21.15.86:0/2678821951 wait complete. 2024-03-06T07:38:27.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.138+0000 7f9a6c200640 1 Processor -- start 2024-03-06T07:38:27.140 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.138+0000 7f9a6c200640 1 -- start start 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a6c200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6406ec00 0x7f9a641442d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a6c200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a640b4680 0x7f9a6415cec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a6c200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a6415acb0 0x7f9a6415d400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a6c200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a640b3d10 con 0x7f9a6406ec00 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a6c200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a640b32b0 con 0x7f9a6415acb0 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a6c200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a640b4010 con 0x7f9a640b4680 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6406ec00 0x7f9a641442d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6406ec00 0x7f9a641442d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53518/0 (socket says 172.21.15.86:53518) 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 -- 172.21.15.86:0/2360611216 learned_addr learned my addr 172.21.15.86:0/2360611216 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 -- 172.21.15.86:0/2360611216 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a640b4680 msgr2=0x7f9a6415cec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a640b4680 0x7f9a6415cec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.143 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 -- 172.21.15.86:0/2360611216 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a6415acb0 msgr2=0x7f9a6415d400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a6415acb0 0x7f9a6415d400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.139+0000 7f9a69800640 1 -- 172.21.15.86:0/2360611216 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a640c8fc0 con 0x7f9a6406ec00 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.140+0000 7f9a69800640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6406ec00 0x7f9a641442d0 secure :-1 s=READY pgs=1825 cs=0 l=1 rev1=1 crypto rx=0x7f9a58009890 tx=0x7f9a580046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.140+0000 7f9a61600640 1 -- 172.21.15.86:0/2360611216 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a58041070 con 0x7f9a6406ec00 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.140+0000 7f9a61600640 1 -- 172.21.15.86:0/2360611216 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9a5803aa30 con 0x7f9a6406ec00 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.140+0000 7f9a61600640 1 -- 172.21.15.86:0/2360611216 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a58035040 con 0x7f9a6406ec00 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.140+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9a640c92a0 con 0x7f9a6406ec00 2024-03-06T07:38:27.144 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.140+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9a640c9860 con 0x7f9a6406ec00 2024-03-06T07:38:27.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.141+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9a6406cba0 con 0x7f9a6406ec00 2024-03-06T07:38:27.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.142+0000 7f9a61600640 1 -- 172.21.15.86:0/2360611216 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9a58038020 con 0x7f9a6406ec00 2024-03-06T07:38:27.145 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.143+0000 7f9a61600640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a4406b4d0 0x7f9a4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:27.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.143+0000 7f9a61600640 1 -- 172.21.15.86:0/2360611216 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9a580abb90 con 0x7f9a6406ec00 2024-03-06T07:38:27.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.143+0000 7f9a68e00640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a4406b4d0 0x7f9a4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:27.146 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.144+0000 7f9a68e00640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a4406b4d0 0x7f9a4406d990 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7f9a50007fd0 tx=0x7f9a50007960 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:27.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.152+0000 7f9a61600640 1 -- 172.21.15.86:0/2360611216 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9a6406cba0 con 0x7f9a6406ec00 2024-03-06T07:38:27.389 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.387+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9a6404f330 con 0x7f9a6406ec00 2024-03-06T07:38:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.388+0000 7f9a61600640 1 -- 172.21.15.86:0/2360611216 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f9a5807aa60 con 0x7f9a6406ec00 2024-03-06T07:38:27.391 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:27.391 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:27.391 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.391+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a4406b4d0 msgr2=0x7f9a4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.391+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a4406b4d0 0x7f9a4406d990 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7f9a50007fd0 tx=0x7f9a50007960 comp rx=0 tx=0).stop 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.391+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6406ec00 msgr2=0x7f9a641442d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.391+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6406ec00 0x7f9a641442d0 secure :-1 s=READY pgs=1825 cs=0 l=1 rev1=1 crypto rx=0x7f9a58009890 tx=0x7f9a580046c0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 shutdown_connections 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9a4406b4d0 0x7f9a4406d990 unknown :-1 s=CLOSED pgs=773 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9a6415acb0 0x7f9a6415d400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9a640b4680 0x7f9a6415cec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.394 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 --2- 172.21.15.86:0/2360611216 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9a6406ec00 0x7f9a641442d0 unknown :-1 s=CLOSED pgs=1825 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 >> 172.21.15.86:0/2360611216 conn(0x7f9a640b1730 msgr2=0x7f9a64155e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 shutdown_connections 2024-03-06T07:38:27.395 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:27.392+0000 7f9a6c200640 1 -- 172.21.15.86:0/2360611216 wait complete. 2024-03-06T07:38:27.614 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:27.614 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:27.716 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:28.615 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:28.851 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.850+0000 7f3e03200640 1 Processor -- start 2024-03-06T07:38:28.852 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.850+0000 7f3e03200640 1 -- start start 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.851+0000 7f3e03200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 0x7f3dfc0b44a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e03200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3dfc161900 con 0x7f3dfc0ae670 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e03200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3dfc0b49e0 con 0x7f3dfc1620c0 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e03200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc15ebd0 con 0x7f3dfc0b40c0 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e01400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f3dfc0ae670 0x7f3dfc15e2c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36784/0 (socket says 172.21.15.86:36784) 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e01400640 1 -- 172.21.15.86:0/4101102535 learned_addr learned my addr 172.21.15.86:0/4101102535 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e01e00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 0x7f3dfc0b44a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:28.855 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3585336145 0 0) 0x7f3dfc161900 con 0x7f3dfc0ae670 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.852+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3de4003520 con 0x7f3dfc0ae670 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2476384026 0 0) 0x7f3de4003520 con 0x7f3dfc0ae670 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 msgr2=0x7f3dfc0b44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 0x7f3dfc0b44a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 >> v1:172.21.15.194:6789/0 conn(0x7f3dfc1620c0 legacy=0x7f3dfc0b28e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3dfc15eeb0 con 0x7f3dfc0ae670 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f3de8001670 con 0x7f3dfc0ae670 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 >> v1:172.21.15.86:6789/0 conn(0x7f3dfc0ae670 legacy=0x7f3dfc15e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3de40039e0 0x7f3de4003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 0x7f3de4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de4004d40 0x7f3de40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc15ebd0 con 0x7f3de40039e0 2024-03-06T07:38:28.856 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3de400baa0 con 0x7f3de4004d40 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3de400bca0 con 0x7f3dfc0b40c0 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.853+0000 7f3e01e00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3de40039e0 0x7f3de4003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.854+0000 7f3e01400640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 0x7f3de4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.854+0000 7f3e02800640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de4004d40 0x7f3de40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.854+0000 7f3e01e00640 1 -- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 msgr2=0x7f3de4004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.854+0000 7f3e01e00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc0b40c0 0x7f3de4004800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.854+0000 7f3e01e00640 1 -- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de4004d40 msgr2=0x7f3de40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.854+0000 7f3e01e00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de4004d40 0x7f3de40051c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:28.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.854+0000 7f3e01e00640 1 -- 172.21.15.86:0/4101102535 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3de4003520 con 0x7f3de40039e0 2024-03-06T07:38:28.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.855+0000 7f3e01e00640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3de40039e0 0x7f3de4003e00 secure :-1 s=READY pgs=1827 cs=0 l=1 rev1=1 crypto rx=0x7f3df0009a90 tx=0x7f3de4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=661748f86d13f24c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:28.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.855+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3df0039070 con 0x7f3de40039e0 2024-03-06T07:38:28.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.855+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3df002c680 con 0x7f3de40039e0 2024-03-06T07:38:28.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.855+0000 7f3e00a00640 1 -- 172.21.15.86:0/4101102535 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3df0009690 con 0x7f3de40039e0 2024-03-06T07:38:28.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.855+0000 7f3e03200640 1 -- 172.21.15.86:0/4101102535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3de40039e0 msgr2=0x7f3de4003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.855+0000 7f3e03200640 1 --2- 172.21.15.86:0/4101102535 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3de40039e0 0x7f3de4003e00 secure :-1 s=READY pgs=1827 cs=0 l=1 rev1=1 crypto rx=0x7f3df0009a90 tx=0x7f3de4005830 comp rx=0 tx=0).stop 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.855+0000 7f3e02800640 1 -- 172.21.15.86:0/4101102535 reap_dead start 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.856+0000 7f3e03200640 1 -- 172.21.15.86:0/4101102535 shutdown_connections 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.856+0000 7f3e03200640 1 -- 172.21.15.86:0/4101102535 >> 172.21.15.86:0/4101102535 conn(0x7f3dfc06afc0 msgr2=0x7f3dfc06b3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.856+0000 7f3e03200640 1 -- 172.21.15.86:0/4101102535 shutdown_connections 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.856+0000 7f3e03200640 1 -- 172.21.15.86:0/4101102535 wait complete. 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.857+0000 7f3e03200640 1 Processor -- start 2024-03-06T07:38:28.859 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.858+0000 7f3e03200640 1 -- start start 2024-03-06T07:38:28.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.858+0000 7f3e03200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de40039e0 0x7f3dfc140d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.858+0000 7f3e03200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc14e340 0x7f3dfc1f6b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.858+0000 7f3e03200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3dfc15aa00 0x7f3dfc071d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.858+0000 7f3e03200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc158660 con 0x7f3dfc15aa00 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.858+0000 7f3e03200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc161ba0 con 0x7f3de40039e0 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.858+0000 7f3e03200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc1587e0 con 0x7f3dfc14e340 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3dfc15aa00 0x7f3dfc071d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3dfc15aa00 0x7f3dfc071d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53536/0 (socket says 172.21.15.86:53536) 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 -- 172.21.15.86:0/841208507 learned_addr learned my addr 172.21.15.86:0/841208507 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e01e00640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de40039e0 0x7f3dfc140d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 -- 172.21.15.86:0/841208507 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc14e340 msgr2=0x7f3dfc1f6b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc14e340 0x7f3dfc1f6b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 -- 172.21.15.86:0/841208507 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de40039e0 msgr2=0x7f3dfc140d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de40039e0 0x7f3dfc140d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 -- 172.21.15.86:0/841208507 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3dfc1402f0 con 0x7f3dfc15aa00 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.859+0000 7f3e02800640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3dfc15aa00 0x7f3dfc071d80 secure :-1 s=READY pgs=1828 cs=0 l=1 rev1=1 crypto rx=0x7f3dfc06ec30 tx=0x7f3df800dc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.860+0000 7f3df6000640 1 -- 172.21.15.86:0/841208507 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3df8018070 con 0x7f3dfc15aa00 2024-03-06T07:38:28.862 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.860+0000 7f3df6000640 1 -- 172.21.15.86:0/841208507 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3df8003b70 con 0x7f3dfc15aa00 2024-03-06T07:38:28.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.860+0000 7f3df6000640 1 -- 172.21.15.86:0/841208507 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3df8005070 con 0x7f3dfc15aa00 2024-03-06T07:38:28.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.860+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3dfc15b470 con 0x7f3dfc15aa00 2024-03-06T07:38:28.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.860+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3dfc140840 con 0x7f3dfc15aa00 2024-03-06T07:38:28.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.861+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3dfc15b810 con 0x7f3dfc15aa00 2024-03-06T07:38:28.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.862+0000 7f3df6000640 1 -- 172.21.15.86:0/841208507 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3df801b070 con 0x7f3dfc15aa00 2024-03-06T07:38:28.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.863+0000 7f3df6000640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ddc06b4d0 0x7f3ddc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:28.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.863+0000 7f3df6000640 1 -- 172.21.15.86:0/841208507 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3df8089b70 con 0x7f3dfc15aa00 2024-03-06T07:38:28.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.863+0000 7f3e01e00640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ddc06b4d0 0x7f3ddc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:28.865 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.864+0000 7f3e01e00640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ddc06b4d0 0x7f3ddc06d990 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f3df0036000 tx=0x7f3df0030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:28.869 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:28.868+0000 7f3df6000640 1 -- 172.21.15.86:0/841208507 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3df8058910 con 0x7f3dfc15aa00 2024-03-06T07:38:29.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.114+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3dfc04f330 con 0x7f3dfc15aa00 2024-03-06T07:38:29.116 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.114+0000 7f3df6000640 1 -- 172.21.15.86:0/841208507 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f3df8058210 con 0x7f3dfc15aa00 2024-03-06T07:38:29.116 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:29.116 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:29.116 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:29.119 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.117+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ddc06b4d0 msgr2=0x7f3ddc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.117+0000 7f3e03200640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ddc06b4d0 0x7f3ddc06d990 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f3df0036000 tx=0x7f3df0030000 comp rx=0 tx=0).stop 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.117+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3dfc15aa00 msgr2=0x7f3dfc071d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.117+0000 7f3e03200640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3dfc15aa00 0x7f3dfc071d80 secure :-1 s=READY pgs=1828 cs=0 l=1 rev1=1 crypto rx=0x7f3dfc06ec30 tx=0x7f3df800dc00 comp rx=0 tx=0).stop 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 shutdown_connections 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3ddc06b4d0 0x7f3ddc06d990 unknown :-1 s=CLOSED pgs=774 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3dfc15aa00 0x7f3dfc071d80 unknown :-1 s=CLOSED pgs=1828 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3dfc14e340 0x7f3dfc1f6b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 --2- 172.21.15.86:0/841208507 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3de40039e0 0x7f3dfc140d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 >> 172.21.15.86:0/841208507 conn(0x7f3dfc06afc0 msgr2=0x7f3dfc0b2690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 shutdown_connections 2024-03-06T07:38:29.120 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:29.118+0000 7f3e03200640 1 -- 172.21.15.86:0/841208507 wait complete. 2024-03-06T07:38:29.338 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:29.339 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:29.719 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:30.340 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:30.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.563+0000 7f737de00640 1 Processor -- start 2024-03-06T07:38:30.565 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.563+0000 7f737de00640 1 -- start start 2024-03-06T07:38:30.566 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f737de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7378144660 0x7f7378071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f737de00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f73780cf3e0 0x7f73780724b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f737de00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f73780c9ec0 0x7f73780ca310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f737de00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f73780cabf0 con 0x7f73780c9ec0 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f737de00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f73780ce900 con 0x7f73780cf3e0 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f737de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f73780cebe0 con 0x7f7378144660 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f7377400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f73780c9ec0 0x7f73780ca310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f7377400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f73780c9ec0 0x7f73780ca310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53550/0 (socket says 172.21.15.86:53550) 2024-03-06T07:38:30.567 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f7377400640 1 -- 172.21.15.86:0/3760733333 learned_addr learned my addr 172.21.15.86:0/3760733333 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:30.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f736fe00640 1 --2- 172.21.15.86:0/3760733333 >> v2:172.21.15.194:3300/0 conn(0x7f73780cf3e0 0x7f73780724b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:30.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.565+0000 7f7376a00640 1 --2- 172.21.15.86:0/3760733333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7378144660 0x7f7378071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:30.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.566+0000 7f7377400640 1 -- 172.21.15.86:0/3760733333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7378144660 msgr2=0x7f7378071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:30.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.566+0000 7f7377400640 1 --2- 172.21.15.86:0/3760733333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7378144660 0x7f7378071f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.566+0000 7f7377400640 1 -- 172.21.15.86:0/3760733333 >> v2:172.21.15.194:3300/0 conn(0x7f73780cf3e0 msgr2=0x7f73780724b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:30.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.566+0000 7f7377400640 1 --2- 172.21.15.86:0/3760733333 >> v2:172.21.15.194:3300/0 conn(0x7f73780cf3e0 0x7f73780724b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.568 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.566+0000 7f7377400640 1 -- 172.21.15.86:0/3760733333 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73780ceec0 con 0x7f73780c9ec0 2024-03-06T07:38:30.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.567+0000 7f7377400640 1 --2- 172.21.15.86:0/3760733333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73780c9ec0 0x7f73780ca310 secure :-1 s=READY pgs=1829 cs=0 l=1 rev1=1 crypto rx=0x7f7378068530 tx=0x7f737002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b6ec9613f148f7b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:30.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.567+0000 7f7376000640 1 -- 172.21.15.86:0/3760733333 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f737002ca70 con 0x7f73780c9ec0 2024-03-06T07:38:30.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.567+0000 7f7376000640 1 -- 172.21.15.86:0/3760733333 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f737002cc10 con 0x7f73780c9ec0 2024-03-06T07:38:30.569 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.567+0000 7f7376000640 1 -- 172.21.15.86:0/3760733333 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73700343f0 con 0x7f73780c9ec0 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 -- 172.21.15.86:0/3760733333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73780c9ec0 msgr2=0x7f73780ca310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 --2- 172.21.15.86:0/3760733333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73780c9ec0 0x7f73780ca310 secure :-1 s=READY pgs=1829 cs=0 l=1 rev1=1 crypto rx=0x7f7378068530 tx=0x7f737002bed0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 -- 172.21.15.86:0/3760733333 shutdown_connections 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 --2- 172.21.15.86:0/3760733333 >> v2:172.21.15.194:3300/0 conn(0x7f73780cf3e0 0x7f73780724b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 --2- 172.21.15.86:0/3760733333 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f73780c9ec0 0x7f73780ca310 unknown :-1 s=CLOSED pgs=1829 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 --2- 172.21.15.86:0/3760733333 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f7378144660 0x7f7378071f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 -- 172.21.15.86:0/3760733333 >> 172.21.15.86:0/3760733333 conn(0x7f73780d0b90 msgr2=0x7f73780d0fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 -- 172.21.15.86:0/3760733333 shutdown_connections 2024-03-06T07:38:30.570 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.568+0000 7f737de00640 1 -- 172.21.15.86:0/3760733333 wait complete. 2024-03-06T07:38:30.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.570+0000 7f737de00640 1 Processor -- start 2024-03-06T07:38:30.571 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.570+0000 7f737de00640 1 -- start start 2024-03-06T07:38:30.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f737de00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f73780c9ec0 0x7f7378160490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:30.573 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f737de00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f73780cf3e0 0x7f737815a010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f737de00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7378144660 0x7f73781407d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f737de00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73780ca9d0 con 0x7f7378144660 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f737de00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73780ca850 con 0x7f73780c9ec0 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f737de00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7378161bf0 con 0x7f73780cf3e0 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f7377400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7378144660 0x7f73781407d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f7377400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7378144660 0x7f73781407d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53562/0 (socket says 172.21.15.86:53562) 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.571+0000 7f7377400640 1 -- 172.21.15.86:0/3755186321 learned_addr learned my addr 172.21.15.86:0/3755186321 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f7376a00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f73780c9ec0 0x7f7378160490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f736fe00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f73780cf3e0 0x7f737815a010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f7377400640 1 -- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f73780cf3e0 msgr2=0x7f737815a010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f7377400640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f73780cf3e0 0x7f737815a010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f7377400640 1 -- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f73780c9ec0 msgr2=0x7f7378160490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f7377400640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f73780c9ec0 0x7f7378160490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f7377400640 1 -- 172.21.15.86:0/3755186321 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7378140d10 con 0x7f7378144660 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f736fe00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f73780cf3e0 0x7f737815a010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:30.574 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f7377400640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7378144660 0x7f73781407d0 secure :-1 s=READY pgs=1830 cs=0 l=1 rev1=1 crypto rx=0x7f7370036040 tx=0x7f7370004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:30.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f736f400640 1 -- 172.21.15.86:0/3755186321 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7370043020 con 0x7f7378144660 2024-03-06T07:38:30.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f736f400640 1 -- 172.21.15.86:0/3755186321 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f737003e450 con 0x7f7378144660 2024-03-06T07:38:30.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f736f400640 1 -- 172.21.15.86:0/3755186321 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7370009510 con 0x7f7378144660 2024-03-06T07:38:30.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.572+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7378140fe0 con 0x7f7378144660 2024-03-06T07:38:30.575 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.573+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f73781415a0 con 0x7f7378144660 2024-03-06T07:38:30.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.574+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f737804f330 con 0x7f7378144660 2024-03-06T07:38:30.576 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.575+0000 7f736f400640 1 -- 172.21.15.86:0/3755186321 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7370046020 con 0x7f7378144660 2024-03-06T07:38:30.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.576+0000 7f736f400640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f735406b500 0x7f735406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:30.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.576+0000 7f736f400640 1 -- 172.21.15.86:0/3755186321 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f73700aca30 con 0x7f7378144660 2024-03-06T07:38:30.577 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.576+0000 7f7376a00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f735406b500 0x7f735406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:30.578 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.577+0000 7f7376a00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f735406b500 0x7f735406d9c0 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f7368006fd0 tx=0x7f7368008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:30.581 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.580+0000 7f736f400640 1 -- 172.21.15.86:0/3755186321 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f737004d030 con 0x7f7378144660 2024-03-06T07:38:30.814 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.812+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f73780cfb70 con 0x7f7378144660 2024-03-06T07:38:30.815 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.813+0000 7f736f400640 1 -- 172.21.15.86:0/3755186321 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f737007b900 con 0x7f7378144660 2024-03-06T07:38:30.815 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:30.815 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:30.815 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:30.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.815+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f735406b500 msgr2=0x7f735406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:30.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.815+0000 7f737de00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f735406b500 0x7f735406d9c0 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f7368006fd0 tx=0x7f7368008040 comp rx=0 tx=0).stop 2024-03-06T07:38:30.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.815+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7378144660 msgr2=0x7f73781407d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:30.817 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.815+0000 7f737de00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7378144660 0x7f73781407d0 secure :-1 s=READY pgs=1830 cs=0 l=1 rev1=1 crypto rx=0x7f7370036040 tx=0x7f7370004a40 comp rx=0 tx=0).stop 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 shutdown_connections 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f735406b500 0x7f735406d9c0 unknown :-1 s=CLOSED pgs=775 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f7378144660 0x7f73781407d0 unknown :-1 s=CLOSED pgs=1830 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f73780cf3e0 0x7f737815a010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 --2- 172.21.15.86:0/3755186321 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f73780c9ec0 0x7f7378160490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 >> 172.21.15.86:0/3755186321 conn(0x7f73780d0b90 msgr2=0x7f73780cd1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 shutdown_connections 2024-03-06T07:38:30.818 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:30.816+0000 7f737de00640 1 -- 172.21.15.86:0/3755186321 wait complete. 2024-03-06T07:38:31.039 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:31.040 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:31.721 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:32.041 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:32.254 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.253+0000 7f9db5c00640 1 Processor -- start 2024-03-06T07:38:32.255 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.253+0000 7f9db5c00640 1 -- start start 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.254+0000 7f9db5c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db00af820 0x7f9db00b33b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.254+0000 7f9db5c00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9db00b38f0 0x7f9db006b580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.254+0000 7f9db5c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9db00b3d70 con 0x7f9db015e160 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.254+0000 7f9db5c00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9db006b0f0 con 0x7f9db00b38f0 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.254+0000 7f9db5c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9db00b5620 con 0x7f9db00af820 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.254+0000 7f9daf400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9db00b38f0 0x7f9db006b580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.255+0000 7f9daf400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f9db00b38f0 0x7f9db006b580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53572/0 (socket says 172.21.15.86:53572) 2024-03-06T07:38:32.256 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.255+0000 7f9daf400640 1 -- 172.21.15.86:0/2894079487 learned_addr learned my addr 172.21.15.86:0/2894079487 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:32.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.255+0000 7f9da6000640 1 --2- 172.21.15.86:0/2894079487 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db00af820 0x7f9db00b33b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:32.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.255+0000 7f9daf400640 1 -- 172.21.15.86:0/2894079487 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db00af820 msgr2=0x7f9db00b33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:32.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.255+0000 7f9daf400640 1 --2- 172.21.15.86:0/2894079487 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db00af820 0x7f9db00b33b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.255+0000 7f9daf400640 1 -- 172.21.15.86:0/2894079487 >> v1:172.21.15.194:6789/0 conn(0x7f9db015e160 legacy=0x7f9db0162620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:32.257 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.255+0000 7f9daf400640 1 -- 172.21.15.86:0/2894079487 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9db006f1f0 con 0x7f9db00b38f0 2024-03-06T07:38:32.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.256+0000 7f9daf400640 1 --2- 172.21.15.86:0/2894079487 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00b38f0 0x7f9db006b580 secure :-1 s=READY pgs=1831 cs=0 l=1 rev1=1 crypto rx=0x7f9da8009e50 tx=0x7f9da802be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=10105223fec07c89 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:32.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.256+0000 7f9dae000640 1 -- 172.21.15.86:0/2894079487 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9da802c9f0 con 0x7f9db00b38f0 2024-03-06T07:38:32.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.256+0000 7f9dae000640 1 -- 172.21.15.86:0/2894079487 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9da802cb90 con 0x7f9db00b38f0 2024-03-06T07:38:32.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.256+0000 7f9dae000640 1 -- 172.21.15.86:0/2894079487 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9da80343f0 con 0x7f9db00b38f0 2024-03-06T07:38:32.258 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 -- 172.21.15.86:0/2894079487 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00b38f0 msgr2=0x7f9db006b580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:32.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 --2- 172.21.15.86:0/2894079487 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00b38f0 0x7f9db006b580 secure :-1 s=READY pgs=1831 cs=0 l=1 rev1=1 crypto rx=0x7f9da8009e50 tx=0x7f9da802be50 comp rx=0 tx=0).stop 2024-03-06T07:38:32.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 -- 172.21.15.86:0/2894079487 shutdown_connections 2024-03-06T07:38:32.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 --2- 172.21.15.86:0/2894079487 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00b38f0 0x7f9db006b580 unknown :-1 s=CLOSED pgs=1831 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 --2- 172.21.15.86:0/2894079487 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db00af820 0x7f9db00b33b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.259 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 -- 172.21.15.86:0/2894079487 >> 172.21.15.86:0/2894079487 conn(0x7f9db00d0340 msgr2=0x7f9db00d0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:32.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 -- 172.21.15.86:0/2894079487 shutdown_connections 2024-03-06T07:38:32.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.257+0000 7f9db5c00640 1 -- 172.21.15.86:0/2894079487 wait complete. 2024-03-06T07:38:32.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.259+0000 7f9db5c00640 1 Processor -- start 2024-03-06T07:38:32.260 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.259+0000 7f9db5c00640 1 -- start start 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9db5c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00af820 0x7f9db0154170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9db5c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9db00b38f0 0x7f9db0155910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9db5c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db015e160 0x7f9db01577a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9db5c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9db006ef10 con 0x7f9db00af820 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9db5c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9db006ec30 con 0x7f9db00b38f0 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9db5c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9db006bac0 con 0x7f9db015e160 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00af820 0x7f9db0154170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:32.263 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00af820 0x7f9db0154170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53574/0 (socket says 172.21.15.86:53574) 2024-03-06T07:38:32.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 -- 172.21.15.86:0/4077194520 learned_addr learned my addr 172.21.15.86:0/4077194520 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:32.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 -- 172.21.15.86:0/4077194520 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db015e160 msgr2=0x7f9db01577a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:32.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db015e160 0x7f9db01577a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 -- 172.21.15.86:0/4077194520 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9db00b38f0 msgr2=0x7f9db0155910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:32.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9db00b38f0 0x7f9db0155910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.261+0000 7f9daea00640 1 -- 172.21.15.86:0/4077194520 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9db0154870 con 0x7f9db00af820 2024-03-06T07:38:32.264 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.262+0000 7f9daea00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00af820 0x7f9db0154170 secure :-1 s=READY pgs=1832 cs=0 l=1 rev1=1 crypto rx=0x7f9d9800b420 tx=0x7f9d9800b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:32.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.262+0000 7f9da7400640 1 -- 172.21.15.86:0/4077194520 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9d98011830 con 0x7f9db00af820 2024-03-06T07:38:32.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.262+0000 7f9da7400640 1 -- 172.21.15.86:0/4077194520 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9d98016070 con 0x7f9db00af820 2024-03-06T07:38:32.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.262+0000 7f9da7400640 1 -- 172.21.15.86:0/4077194520 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9d98011c10 con 0x7f9db00af820 2024-03-06T07:38:32.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.262+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9db0154b50 con 0x7f9db00af820 2024-03-06T07:38:32.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.262+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9db0155110 con 0x7f9db00af820 2024-03-06T07:38:32.265 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.264+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9db004f330 con 0x7f9db00af820 2024-03-06T07:38:32.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.264+0000 7f9da7400640 1 -- 172.21.15.86:0/4077194520 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9d9801b460 con 0x7f9db00af820 2024-03-06T07:38:32.266 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.265+0000 7f9da7400640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9d8c06b4d0 0x7f9d8c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:32.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.265+0000 7f9da7400640 1 -- 172.21.15.86:0/4077194520 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9d98089e50 con 0x7f9db00af820 2024-03-06T07:38:32.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.265+0000 7f9da6000640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9d8c06b4d0 0x7f9d8c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:32.267 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.266+0000 7f9da6000640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9d8c06b4d0 0x7f9d8c06d990 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f9db0157290 tx=0x7f9da0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:32.274 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.273+0000 7f9da7400640 1 -- 172.21.15.86:0/4077194520 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9d98020020 con 0x7f9db00af820 2024-03-06T07:38:32.507 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.505+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9db00d11e0 con 0x7f9db00af820 2024-03-06T07:38:32.508 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.506+0000 7f9da7400640 1 -- 172.21.15.86:0/4077194520 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f9d98058bf0 con 0x7f9db00af820 2024-03-06T07:38:32.508 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:32.508 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:32.508 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.508+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9d8c06b4d0 msgr2=0x7f9d8c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9d8c06b4d0 0x7f9d8c06d990 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f9db0157290 tx=0x7f9da0008040 comp rx=0 tx=0).stop 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00af820 msgr2=0x7f9db0154170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00af820 0x7f9db0154170 secure :-1 s=READY pgs=1832 cs=0 l=1 rev1=1 crypto rx=0x7f9d9800b420 tx=0x7f9d9800b8e0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 shutdown_connections 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f9d8c06b4d0 0x7f9d8c06d990 unknown :-1 s=CLOSED pgs=776 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f9db015e160 0x7f9db01577a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f9db00b38f0 0x7f9db0155910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 --2- 172.21.15.86:0/4077194520 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f9db00af820 0x7f9db0154170 unknown :-1 s=CLOSED pgs=1832 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 >> 172.21.15.86:0/4077194520 conn(0x7f9db00d0340 msgr2=0x7f9db006be80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:32.511 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 shutdown_connections 2024-03-06T07:38:32.512 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:32.509+0000 7f9db5c00640 1 -- 172.21.15.86:0/4077194520 wait complete. 2024-03-06T07:38:32.730 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:32.731 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:33.723 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:33.731 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:33.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.940+0000 7f8c15800640 1 Processor -- start 2024-03-06T07:38:33.942 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.940+0000 7f8c15800640 1 -- start start 2024-03-06T07:38:33.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.941+0000 7f8c15800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 0x7f8c1015fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:33.944 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.941+0000 7f8c15800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8c10070b10 0x7f8c10070f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.941+0000 7f8c15800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8c1006cef0 con 0x7f8c10144c00 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.942+0000 7f8c15800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8c10160370 con 0x7f8c10070b10 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.942+0000 7f8c15800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c1006d1d0 con 0x7f8c1015f9d0 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.942+0000 7f8c0f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8c10070b10 0x7f8c10070f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.942+0000 7f8c0f400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f8c10070b10 0x7f8c10070f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53590/0 (socket says 172.21.15.86:53590) 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.942+0000 7f8c0f400640 1 -- 172.21.15.86:0/1391222880 learned_addr learned my addr 172.21.15.86:0/1391222880 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.942+0000 7f8c06000640 1 --2- 172.21.15.86:0/1391222880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 0x7f8c1015fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.943+0000 7f8c0f400640 1 -- 172.21.15.86:0/1391222880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 msgr2=0x7f8c1015fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.943+0000 7f8c0f400640 1 --2- 172.21.15.86:0/1391222880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 0x7f8c1015fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.943+0000 7f8c0f400640 1 -- 172.21.15.86:0/1391222880 >> v1:172.21.15.194:6789/0 conn(0x7f8c10144c00 legacy=0x7f8c1006e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:33.945 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.943+0000 7f8c0f400640 1 -- 172.21.15.86:0/1391222880 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c1006d4b0 con 0x7f8c10070b10 2024-03-06T07:38:33.946 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.945+0000 7f8c0f400640 1 --2- 172.21.15.86:0/1391222880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c10070f60 secure :-1 s=READY pgs=1833 cs=0 l=1 rev1=1 crypto rx=0x7f8c08009870 tx=0x7f8c0802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71d4d0b84e547bd7 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.945+0000 7f8c0e000640 1 -- 172.21.15.86:0/1391222880 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c08034070 con 0x7f8c10070b10 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.945+0000 7f8c0e000640 1 -- 172.21.15.86:0/1391222880 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c08030840 con 0x7f8c10070b10 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.945+0000 7f8c0e000640 1 -- 172.21.15.86:0/1391222880 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c08030c10 con 0x7f8c10070b10 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 -- 172.21.15.86:0/1391222880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 msgr2=0x7f8c10070f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 --2- 172.21.15.86:0/1391222880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c10070f60 secure :-1 s=READY pgs=1833 cs=0 l=1 rev1=1 crypto rx=0x7f8c08009870 tx=0x7f8c0802b980 comp rx=0 tx=0).stop 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 -- 172.21.15.86:0/1391222880 shutdown_connections 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 --2- 172.21.15.86:0/1391222880 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c10070f60 unknown :-1 s=CLOSED pgs=1833 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 --2- 172.21.15.86:0/1391222880 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 0x7f8c1015fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:33.948 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 -- 172.21.15.86:0/1391222880 >> 172.21.15.86:0/1391222880 conn(0x7f8c100b1730 msgr2=0x7f8c100b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:33.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 -- 172.21.15.86:0/1391222880 shutdown_connections 2024-03-06T07:38:33.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.946+0000 7f8c15800640 1 -- 172.21.15.86:0/1391222880 wait complete. 2024-03-06T07:38:33.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.948+0000 7f8c15800640 1 Processor -- start 2024-03-06T07:38:33.949 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.948+0000 7f8c15800640 1 -- start start 2024-03-06T07:38:33.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c15800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c100aeec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:33.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c15800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c10144c00 0x7f8c1015c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:33.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c15800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 0x7f8c1013c190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:33.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c15800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c1006cb50 con 0x7f8c10070b10 2024-03-06T07:38:33.951 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c15800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c10160730 con 0x7f8c10144c00 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c15800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c1006ccd0 con 0x7f8c1015f9d0 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c0ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c100aeec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c0ea00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c100aeec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53596/0 (socket says 172.21.15.86:53596) 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c0ea00640 1 -- 172.21.15.86:0/2048420476 learned_addr learned my addr 172.21.15.86:0/2048420476 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c0ea00640 1 -- 172.21.15.86:0/2048420476 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 msgr2=0x7f8c1013c190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c0ea00640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 0x7f8c1013c190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c0ea00640 1 -- 172.21.15.86:0/2048420476 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c10144c00 msgr2=0x7f8c1015c340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.949+0000 7f8c06000640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c10144c00 0x7f8c1015c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c0ea00640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c10144c00 0x7f8c1015c340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c0ea00640 1 -- 172.21.15.86:0/2048420476 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c1015df40 con 0x7f8c10070b10 2024-03-06T07:38:33.952 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c0ea00640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c100aeec0 secure :-1 s=READY pgs=1834 cs=0 l=1 rev1=1 crypto rx=0x7f8c0000c8a0 tx=0x7f8c0000cd60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:33.953 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c07400640 1 -- 172.21.15.86:0/2048420476 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c00003b70 con 0x7f8c10070b10 2024-03-06T07:38:33.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c07400640 1 -- 172.21.15.86:0/2048420476 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c00003d30 con 0x7f8c10070b10 2024-03-06T07:38:33.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c07400640 1 -- 172.21.15.86:0/2048420476 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c0000aa90 con 0x7f8c10070b10 2024-03-06T07:38:33.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c1015e1e0 con 0x7f8c10070b10 2024-03-06T07:38:33.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.950+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c1015e7a0 con 0x7f8c10070b10 2024-03-06T07:38:33.954 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.952+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c1004f330 con 0x7f8c10070b10 2024-03-06T07:38:33.955 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.953+0000 7f8c07400640 1 -- 172.21.15.86:0/2048420476 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8c0000ac30 con 0x7f8c10070b10 2024-03-06T07:38:33.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.954+0000 7f8c07400640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bec06b4d0 0x7f8bec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:33.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.954+0000 7f8c07400640 1 -- 172.21.15.86:0/2048420476 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f8c0008a170 con 0x7f8c10070b10 2024-03-06T07:38:33.956 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.955+0000 7f8c06000640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bec06b4d0 0x7f8bec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:33.957 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.955+0000 7f8c06000640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bec06b4d0 0x7f8bec06d990 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f8c101f74a0 tx=0x7f8bf8008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:33.964 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:33.963+0000 7f8c07400640 1 -- 172.21.15.86:0/2048420476 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8c00059020 con 0x7f8c10070b10 2024-03-06T07:38:34.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.217+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8c10072890 con 0x7f8c10070b10 2024-03-06T07:38:34.219 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.218+0000 7f8c07400640 1 -- 172.21.15.86:0/2048420476 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f8c00058920 con 0x7f8c10070b10 2024-03-06T07:38:34.220 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:34.220 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:34.220 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:34.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.220+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bec06b4d0 msgr2=0x7f8bec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:34.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.220+0000 7f8c15800640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bec06b4d0 0x7f8bec06d990 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f8c101f74a0 tx=0x7f8bf8008040 comp rx=0 tx=0).stop 2024-03-06T07:38:34.222 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.220+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 msgr2=0x7f8c100aeec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.220+0000 7f8c15800640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c100aeec0 secure :-1 s=READY pgs=1834 cs=0 l=1 rev1=1 crypto rx=0x7f8c0000c8a0 tx=0x7f8c0000cd60 comp rx=0 tx=0).stop 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 shutdown_connections 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f8bec06b4d0 0x7f8bec06d990 unknown :-1 s=CLOSED pgs=777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f8c1015f9d0 0x7f8c1013c190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f8c10144c00 0x7f8c1015c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 --2- 172.21.15.86:0/2048420476 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f8c10070b10 0x7f8c100aeec0 unknown :-1 s=CLOSED pgs=1834 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 >> 172.21.15.86:0/2048420476 conn(0x7f8c100b1730 msgr2=0x7f8c100ce060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 shutdown_connections 2024-03-06T07:38:34.223 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:34.221+0000 7f8c15800640 1 -- 172.21.15.86:0/2048420476 wait complete. 2024-03-06T07:38:34.442 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:34.442 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:35.442 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.713+0000 7febde400640 1 Processor -- start 2024-03-06T07:38:35.715 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.714+0000 7febde400640 1 -- start start 2024-03-06T07:38:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.715+0000 7febde400640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7febd80b2a60 0x7febd80d4830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.715+0000 7febde400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7febd815f030 con 0x7febd80d18e0 2024-03-06T07:38:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.715+0000 7febde400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7febd815f310 con 0x7febd80d0600 2024-03-06T07:38:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.715+0000 7febde400640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7febd80cf090 con 0x7febd80b2a60 2024-03-06T07:38:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febdda00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7febd80d18e0 0x7febd80cf520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36798/0 (socket says 172.21.15.86:36798) 2024-03-06T07:38:35.717 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febdda00640 1 -- 172.21.15.86:0/1971716325 learned_addr learned my addr 172.21.15.86:0/1971716325 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:35.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febcfe00640 1 --2- 172.21.15.86:0/1971716325 >> v2:172.21.15.194:3300/0 conn(0x7febd80b2a60 0x7febd80d4830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3255510866 0 0) 0x7febd815f030 con 0x7febd80d18e0 2024-03-06T07:38:35.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7febbc003520 con 0x7febd80d18e0 2024-03-06T07:38:35.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2562835254 0 0) 0x7febbc003520 con 0x7febd80d18e0 2024-03-06T07:38:35.719 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 >> v2:172.21.15.194:3300/0 conn(0x7febd80b2a60 msgr2=0x7febd80d4830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.716+0000 7febd7e00640 1 --2- 172.21.15.86:0/1971716325 >> v2:172.21.15.194:3300/0 conn(0x7febd80b2a60 0x7febd80d4830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 >> v1:172.21.15.194:6789/0 conn(0x7febd80d0600 legacy=0x7febd80d09e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7febd80cea40 con 0x7febd80d18e0 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7febd00019f0 con 0x7febd80d18e0 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 >> v1:172.21.15.86:6789/0 conn(0x7febd80d18e0 legacy=0x7febd80cf520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febbc0039b0 0x7febbc003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febbc004800 0x7febbc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febbc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7febd80cf090 con 0x7febbc0051c0 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7febbc00c060 con 0x7febbc004800 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7febbc00c260 con 0x7febbc0039b0 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febdda00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febbc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.720 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febdd000640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febbc0039b0 0x7febbc003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febcfe00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febbc004800 0x7febbc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febdda00640 1 -- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febbc0039b0 msgr2=0x7febbc003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:35.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febdda00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febbc0039b0 0x7febbc003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febdda00640 1 -- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febbc004800 msgr2=0x7febbc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:35.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febdda00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febbc004800 0x7febbc004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.721 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.717+0000 7febdda00640 1 -- 172.21.15.86:0/1971716325 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7febbc003520 con 0x7febbc0051c0 2024-03-06T07:38:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.718+0000 7febdda00640 1 -- 172.21.15.86:0/1971716325 reap_dead start 2024-03-06T07:38:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.719+0000 7febdda00640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febbc00b9c0 secure :-1 s=READY pgs=1836 cs=0 l=1 rev1=1 crypto rx=0x7febd000a840 tx=0x7febbc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9cc7ac1c7eccc219 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.719+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febd000a460 con 0x7febbc0051c0 2024-03-06T07:38:35.722 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.719+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7febd0034650 con 0x7febbc0051c0 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.719+0000 7febd7e00640 1 -- 172.21.15.86:0/1971716325 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febd0034a40 con 0x7febbc0051c0 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.719+0000 7febde400640 1 -- 172.21.15.86:0/1971716325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 msgr2=0x7febbc00b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.719+0000 7febde400640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febbc00b9c0 secure :-1 s=READY pgs=1836 cs=0 l=1 rev1=1 crypto rx=0x7febd000a840 tx=0x7febbc005830 comp rx=0 tx=0).stop 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.720+0000 7febde400640 1 -- 172.21.15.86:0/1971716325 shutdown_connections 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.720+0000 7febde400640 1 --2- 172.21.15.86:0/1971716325 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febbc00b9c0 unknown :-1 s=CLOSED pgs=1836 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.720+0000 7febde400640 1 -- 172.21.15.86:0/1971716325 >> 172.21.15.86:0/1971716325 conn(0x7febd80d3d10 msgr2=0x7febd80d4120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.720+0000 7febde400640 1 -- 172.21.15.86:0/1971716325 shutdown_connections 2024-03-06T07:38:35.723 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.720+0000 7febde400640 1 -- 172.21.15.86:0/1971716325 wait complete. 2024-03-06T07:38:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.721+0000 7febde400640 1 Processor -- start 2024-03-06T07:38:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.722+0000 7febde400640 1 -- start start 2024-03-06T07:38:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.722+0000 7febde400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febd80b8df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.724 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.722+0000 7febde400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febd80b1d60 0x7febd80b21c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.722+0000 7febde400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febd80c7af0 0x7febd80c7f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.722+0000 7febde400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7febd815ecf0 con 0x7febbc0051c0 2024-03-06T07:38:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.722+0000 7febde400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7febd80b3250 con 0x7febd80b1d60 2024-03-06T07:38:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.722+0000 7febde400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7febd80d1d70 con 0x7febd80c7af0 2024-03-06T07:38:35.725 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febd80b8df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febd80b8df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:53620/0 (socket says 172.21.15.86:53620) 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 -- 172.21.15.86:0/3639452246 learned_addr learned my addr 172.21.15.86:0/3639452246 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febcfe00640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febd80b1d60 0x7febd80b21c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdda00640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febd80c7af0 0x7febd80c7f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 -- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febd80c7af0 msgr2=0x7febd80c7f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febd80c7af0 0x7febd80c7f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 -- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febd80b1d60 msgr2=0x7febd80b21c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:35.727 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febd80b1d60 0x7febd80b21c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 -- 172.21.15.86:0/3639452246 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7febd81f4520 con 0x7febbc0051c0 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.723+0000 7febdd000640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febd80b8df0 secure :-1 s=READY pgs=1837 cs=0 l=1 rev1=1 crypto rx=0x7febc800e4b0 tx=0x7febc800e970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.724+0000 7febd5600640 1 -- 172.21.15.86:0/3639452246 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febc80052e0 con 0x7febbc0051c0 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.724+0000 7febd5600640 1 -- 172.21.15.86:0/3639452246 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7febc8018070 con 0x7febbc0051c0 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.724+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7febd81f4800 con 0x7febbc0051c0 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.724+0000 7febd5600640 1 -- 172.21.15.86:0/3639452246 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7febc8003830 con 0x7febbc0051c0 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.724+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7febd80c84b0 con 0x7febbc0051c0 2024-03-06T07:38:35.728 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.725+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7febd804f330 con 0x7febbc0051c0 2024-03-06T07:38:35.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.726+0000 7febd5600640 1 -- 172.21.15.86:0/3639452246 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7febc80039d0 con 0x7febbc0051c0 2024-03-06T07:38:35.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.727+0000 7febd5600640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febb406b4d0 0x7febb406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:35.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.728+0000 7febd5600640 1 -- 172.21.15.86:0/3639452246 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7febc8089e40 con 0x7febbc0051c0 2024-03-06T07:38:35.729 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.728+0000 7febcfe00640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febb406b4d0 0x7febb406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:35.730 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.728+0000 7febcfe00640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febb406b4d0 0x7febb406d990 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7febd8161950 tx=0x7febc0008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:35.732 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.731+0000 7febd5600640 1 -- 172.21.15.86:0/3639452246 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7febc8058c70 con 0x7febbc0051c0 2024-03-06T07:38:35.968 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.966+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7febd808d7e0 con 0x7febbc0051c0 2024-03-06T07:38:35.969 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.967+0000 7febd5600640 1 -- 172.21.15.86:0/3639452246 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7febc8058570 con 0x7febbc0051c0 2024-03-06T07:38:35.969 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:35.969 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:35.970 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febb406b4d0 msgr2=0x7febb406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febb406b4d0 0x7febb406d990 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7febd8161950 tx=0x7febc0008040 comp rx=0 tx=0).stop 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 msgr2=0x7febd80b8df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febd80b8df0 secure :-1 s=READY pgs=1837 cs=0 l=1 rev1=1 crypto rx=0x7febc800e4b0 tx=0x7febc800e970 comp rx=0 tx=0).stop 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 shutdown_connections 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7febb406b4d0 0x7febb406d990 unknown :-1 s=CLOSED pgs=778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7febd80c7af0 0x7febd80c7f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7febd80b1d60 0x7febd80b21c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.972 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 --2- 172.21.15.86:0/3639452246 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7febbc0051c0 0x7febd80b8df0 unknown :-1 s=CLOSED pgs=1837 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:35.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 >> 172.21.15.86:0/3639452246 conn(0x7febd80d3d10 msgr2=0x7febd80d09e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:35.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 shutdown_connections 2024-03-06T07:38:35.973 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:35.970+0000 7febde400640 1 -- 172.21.15.86:0/3639452246 wait complete. 2024-03-06T07:38:36.198 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:36.198 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:37.199 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.416+0000 7f4c15200640 1 Processor -- start 2024-03-06T07:38:37.418 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.417+0000 7f4c15200640 1 -- start start 2024-03-06T07:38:37.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.419+0000 7f4c15200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f4c1015e0d0 0x7f4c100d1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.419+0000 7f4c15200640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4c100cea60 con 0x7f4c1015f470 2024-03-06T07:38:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.419+0000 7f4c15200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4c100d2350 con 0x7f4c100d0fc0 2024-03-06T07:38:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.419+0000 7f4c15200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4c100d2f60 con 0x7f4c1015e0d0 2024-03-06T07:38:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.419+0000 7f4c07e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f4c1015f470 0x7f4c100d08b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35976/0 (socket says 172.21.15.86:35976) 2024-03-06T07:38:37.421 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.419+0000 7f4c07e00640 1 -- 172.21.15.86:0/307602852 learned_addr learned my addr 172.21.15.86:0/307602852 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0de00640 1 --2- 172.21.15.86:0/307602852 >> v2:172.21.15.194:3300/0 conn(0x7f4c1015e0d0 0x7f4c100d1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2266410565 0 0) 0x7f4c100cea60 con 0x7f4c1015f470 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4bf4003520 con 0x7f4c1015f470 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3727407161 0 0) 0x7f4bf4003520 con 0x7f4c1015f470 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 >> v2:172.21.15.194:3300/0 conn(0x7f4c1015e0d0 msgr2=0x7f4c100d1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 --2- 172.21.15.86:0/307602852 >> v2:172.21.15.194:3300/0 conn(0x7f4c1015e0d0 0x7f4c100d1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 >> v1:172.21.15.194:6789/0 conn(0x7f4c100d0fc0 legacy=0x7f4c10160dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:37.423 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c100d2630 con 0x7f4c1015f470 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f4bf8001670 con 0x7f4c1015f470 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 >> v1:172.21.15.86:6789/0 conn(0x7f4c1015f470 legacy=0x7f4c100d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4bf40039b0 0x7f4bf4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4bf4004800 0x7f4bf4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf40051c0 0x7f4bf400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4bf400c040 con 0x7f4bf4004800 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4bf400c1f0 con 0x7f4bf40051c0 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.420+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4bf400c3f0 con 0x7f4bf40039b0 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c07e00640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4bf4004800 0x7f4bf4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c0e800640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf40051c0 0x7f4bf400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c0de00640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4bf40039b0 0x7f4bf4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c07e00640 1 -- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4bf40039b0 msgr2=0x7f4bf4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:37.424 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c07e00640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4bf40039b0 0x7f4bf4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c07e00640 1 -- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf40051c0 msgr2=0x7f4bf400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:37.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c07e00640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf40051c0 0x7f4bf400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c07e00640 1 -- 172.21.15.86:0/307602852 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4bf4003520 con 0x7f4bf4004800 2024-03-06T07:38:37.425 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.421+0000 7f4c0e800640 1 -- 172.21.15.86:0/307602852 reap_dead start 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.423+0000 7f4c07e00640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4bf4004800 0x7f4bf4004c80 secure :-1 s=READY pgs=1839 cs=0 l=1 rev1=1 crypto rx=0x7f4bf800c210 tx=0x7f4bf4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=23e55c683e96962e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.423+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bf8007d30 con 0x7f4bf4004800 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.423+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4bf8030070 con 0x7f4bf4004800 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.423+0000 7f4c0d400640 1 -- 172.21.15.86:0/307602852 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bf802cda0 con 0x7f4bf4004800 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.424+0000 7f4c15200640 1 -- 172.21.15.86:0/307602852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4bf4004800 msgr2=0x7f4bf4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.424+0000 7f4c15200640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4bf4004800 0x7f4bf4004c80 secure :-1 s=READY pgs=1839 cs=0 l=1 rev1=1 crypto rx=0x7f4bf800c210 tx=0x7f4bf4005830 comp rx=0 tx=0).stop 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.424+0000 7f4c15200640 1 -- 172.21.15.86:0/307602852 shutdown_connections 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.424+0000 7f4c15200640 1 --2- 172.21.15.86:0/307602852 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4bf4004800 0x7f4bf4004c80 unknown :-1 s=CLOSED pgs=1839 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.424+0000 7f4c15200640 1 -- 172.21.15.86:0/307602852 >> 172.21.15.86:0/307602852 conn(0x7f4c100d3ec0 msgr2=0x7f4c100d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.424+0000 7f4c15200640 1 -- 172.21.15.86:0/307602852 shutdown_connections 2024-03-06T07:38:37.426 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.424+0000 7f4c15200640 1 -- 172.21.15.86:0/307602852 wait complete. 2024-03-06T07:38:37.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.426+0000 7f4c15200640 1 Processor -- start 2024-03-06T07:38:37.427 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.426+0000 7f4c15200640 1 -- start start 2024-03-06T07:38:37.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c15200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf4004800 0x7f4c10151ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c15200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4c1015fcb0 0x7f4c100b1c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c15200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4c10160130 0x7f4c100b2140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c15200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c1015fb30 con 0x7f4c10160130 2024-03-06T07:38:37.429 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c15200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c100d2bf0 con 0x7f4bf4004800 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c15200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4c1015e4b0 con 0x7f4c1015fcb0 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c0e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4c10160130 0x7f4c100b2140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c0e800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4c10160130 0x7f4c100b2140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60702/0 (socket says 172.21.15.86:60702) 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c0e800640 1 -- 172.21.15.86:0/3943968996 learned_addr learned my addr 172.21.15.86:0/3943968996 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c07e00640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4c1015fcb0 0x7f4c100b1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.427+0000 7f4c0de00640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf4004800 0x7f4c10151ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c0e800640 1 -- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4c1015fcb0 msgr2=0x7f4c100b1c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c0e800640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4c1015fcb0 0x7f4c100b1c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c0e800640 1 -- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf4004800 msgr2=0x7f4c10151ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c0e800640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf4004800 0x7f4c10151ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c0e800640 1 -- 172.21.15.86:0/3943968996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c10162ba0 con 0x7f4c10160130 2024-03-06T07:38:37.430 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c0e800640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4c10160130 0x7f4c100b2140 secure :-1 s=READY pgs=1840 cs=0 l=1 rev1=1 crypto rx=0x7f4c1008d000 tx=0x7f4c0800ea00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:37.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c06000640 1 -- 172.21.15.86:0/3943968996 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c08006d60 con 0x7f4c10160130 2024-03-06T07:38:37.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c06000640 1 -- 172.21.15.86:0/3943968996 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4c08003970 con 0x7f4c10160130 2024-03-06T07:38:37.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c06000640 1 -- 172.21.15.86:0/3943968996 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c08003d60 con 0x7f4c10160130 2024-03-06T07:38:37.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4c10160690 con 0x7f4c10160130 2024-03-06T07:38:37.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.428+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4c100c81c0 con 0x7f4c10160130 2024-03-06T07:38:37.431 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.429+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4c1004f330 con 0x7f4c10160130 2024-03-06T07:38:37.433 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.431+0000 7f4c06000640 1 -- 172.21.15.86:0/3943968996 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4c08005070 con 0x7f4c10160130 2024-03-06T07:38:37.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.433+0000 7f4c06000640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4bec06b4d0 0x7f4bec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:37.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.433+0000 7f4c06000640 1 -- 172.21.15.86:0/3943968996 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4c08089a10 con 0x7f4c10160130 2024-03-06T07:38:37.434 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.433+0000 7f4c0de00640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4bec06b4d0 0x7f4bec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:37.435 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.434+0000 7f4c0de00640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4bec06b4d0 0x7f4bec06d990 secure :-1 s=READY pgs=779 cs=0 l=1 rev1=1 crypto rx=0x7f4c100686c0 tx=0x7f4c00008040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:37.439 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.437+0000 7f4c06000640 1 -- 172.21.15.86:0/3943968996 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4c080587b0 con 0x7f4c10160130 2024-03-06T07:38:37.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.668+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4c100ca940 con 0x7f4c10160130 2024-03-06T07:38:37.671 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.669+0000 7f4c06000640 1 -- 172.21.15.86:0/3943968996 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f4c080580b0 con 0x7f4c10160130 2024-03-06T07:38:37.672 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:37.672 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4bec06b4d0 msgr2=0x7f4bec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4bec06b4d0 0x7f4bec06d990 secure :-1 s=READY pgs=779 cs=0 l=1 rev1=1 crypto rx=0x7f4c100686c0 tx=0x7f4c00008040 comp rx=0 tx=0).stop 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4c10160130 msgr2=0x7f4c100b2140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4c10160130 0x7f4c100b2140 secure :-1 s=READY pgs=1840 cs=0 l=1 rev1=1 crypto rx=0x7f4c1008d000 tx=0x7f4c0800ea00 comp rx=0 tx=0).stop 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 shutdown_connections 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f4bec06b4d0 0x7f4bec06d990 unknown :-1 s=CLOSED pgs=779 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f4c10160130 0x7f4c100b2140 unknown :-1 s=CLOSED pgs=1840 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f4c1015fcb0 0x7f4c100b1c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.673 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 --2- 172.21.15.86:0/3943968996 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f4bf4004800 0x7f4c10151ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:37.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 >> 172.21.15.86:0/3943968996 conn(0x7f4c100d3ec0 msgr2=0x7f4c100cfcb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:37.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 shutdown_connections 2024-03-06T07:38:37.674 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:37.671+0000 7f4c15200640 1 -- 172.21.15.86:0/3943968996 wait complete. 2024-03-06T07:38:37.726 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:37.891 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:37.891 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:38.893 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:39.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.101+0000 7fd043800640 1 Processor -- start 2024-03-06T07:38:39.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.102+0000 7fd043800640 1 -- start start 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd043800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd03c0b4680 0x7fd03c06e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd043800640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd03c15b090 con 0x7fd03c15acb0 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd043800640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd03c0b2a30 con 0x7fd03c072340 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd043800640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd03c0b4360 con 0x7fd03c0b4680 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd03be00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd03c0b4680 0x7fd03c06e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd040e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd03c15acb0 0x7fd03c0b37d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35978/0 (socket says 172.21.15.86:35978) 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd040e00640 1 -- 172.21.15.86:0/2262077510 learned_addr learned my addr 172.21.15.86:0/2262077510 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.104+0000 7fd03b400640 1 -- 172.21.15.86:0/2262077510 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1370557888 0 0) 0x7fd03c15b090 con 0x7fd03c15acb0 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.105+0000 7fd03b400640 1 -- 172.21.15.86:0/2262077510 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd020005730 con 0x7fd03c15acb0 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.105+0000 7fd03be00640 1 -- 172.21.15.86:0/2262077510 >> v1:172.21.15.194:6789/0 conn(0x7fd03c072340 legacy=0x7fd03c0ccfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.105+0000 7fd03be00640 1 -- 172.21.15.86:0/2262077510 >> v1:172.21.15.86:6789/0 conn(0x7fd03c15acb0 legacy=0x7fd03c0b37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:39.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.105+0000 7fd03be00640 1 -- 172.21.15.86:0/2262077510 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd03c0b2cf0 con 0x7fd03c0b4680 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.106+0000 7fd03be00640 1 --2- 172.21.15.86:0/2262077510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c06e4f0 secure :-1 s=READY pgs=1842 cs=0 l=1 rev1=1 crypto rx=0x7fd0240097a0 tx=0x7fd02402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98e1ab53abe9eaa5 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.106+0000 7fd03b400640 1 -- 172.21.15.86:0/2262077510 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd024035020 con 0x7fd03c0b4680 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.106+0000 7fd03b400640 1 -- 172.21.15.86:0/2262077510 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd024031830 con 0x7fd03c0b4680 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.106+0000 7fd03b400640 1 -- 172.21.15.86:0/2262077510 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd024031be0 con 0x7fd03c0b4680 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.107+0000 7fd043800640 1 -- 172.21.15.86:0/2262077510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 msgr2=0x7fd03c06e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.107+0000 7fd043800640 1 --2- 172.21.15.86:0/2262077510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c06e4f0 secure :-1 s=READY pgs=1842 cs=0 l=1 rev1=1 crypto rx=0x7fd0240097a0 tx=0x7fd02402b8d0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.107+0000 7fd043800640 1 -- 172.21.15.86:0/2262077510 shutdown_connections 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.107+0000 7fd043800640 1 --2- 172.21.15.86:0/2262077510 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c06e4f0 unknown :-1 s=CLOSED pgs=1842 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.107+0000 7fd043800640 1 -- 172.21.15.86:0/2262077510 >> 172.21.15.86:0/2262077510 conn(0x7fd03c0b1730 msgr2=0x7fd03c0b1b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.107+0000 7fd043800640 1 -- 172.21.15.86:0/2262077510 shutdown_connections 2024-03-06T07:38:39.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.107+0000 7fd043800640 1 -- 172.21.15.86:0/2262077510 wait complete. 2024-03-06T07:38:39.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.109+0000 7fd043800640 1 Processor -- start 2024-03-06T07:38:39.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.109+0000 7fd043800640 1 -- start start 2024-03-06T07:38:39.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd043800640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd03c072340 0x7fd03c146860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:39.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd043800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c0cefa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:39.112 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd043800640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd03c15acb0 0x7fd03c0cd4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd043800640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd03c0b4010 con 0x7fd03c0b4680 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd043800640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd03c0b32b0 con 0x7fd03c072340 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd043800640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd03c06ea30 con 0x7fd03c15acb0 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c0cefa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c0cefa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60718/0 (socket says 172.21.15.86:60718) 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 -- 172.21.15.86:0/2663001195 learned_addr learned my addr 172.21.15.86:0/2663001195 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd040e00640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd03c072340 0x7fd03c146860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd041800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd03c15acb0 0x7fd03c0cd4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 -- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd03c15acb0 msgr2=0x7fd03c0cd4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd03c15acb0 0x7fd03c0cd4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 -- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd03c072340 msgr2=0x7fd03c146860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd03c072340 0x7fd03c146860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd03be00640 1 -- 172.21.15.86:0/2663001195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd03c0cdf00 con 0x7fd03c0b4680 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.110+0000 7fd041800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd03c15acb0 0x7fd03c0cd4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:39.113 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.111+0000 7fd03be00640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c0cefa0 secure :-1 s=READY pgs=1843 cs=0 l=1 rev1=1 crypto rx=0x7fd02402b910 tx=0x7fd0240046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:39.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.111+0000 7fd038c00640 1 -- 172.21.15.86:0/2663001195 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd024041070 con 0x7fd03c0b4680 2024-03-06T07:38:39.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.111+0000 7fd038c00640 1 -- 172.21.15.86:0/2663001195 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd02403b840 con 0x7fd03c0b4680 2024-03-06T07:38:39.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.111+0000 7fd038c00640 1 -- 172.21.15.86:0/2663001195 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd024035040 con 0x7fd03c0b4680 2024-03-06T07:38:39.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.111+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd03c0ce1e0 con 0x7fd03c0b4680 2024-03-06T07:38:39.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.111+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd03c0ce7a0 con 0x7fd03c0b4680 2024-03-06T07:38:39.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.113+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd03c04f330 con 0x7fd03c0b4680 2024-03-06T07:38:39.115 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.114+0000 7fd038c00640 1 -- 172.21.15.86:0/2663001195 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd024038020 con 0x7fd03c0b4680 2024-03-06T07:38:39.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.115+0000 7fd038c00640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd01806b4d0 0x7fd01806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:39.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.115+0000 7fd038c00640 1 -- 172.21.15.86:0/2663001195 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd0240abb40 con 0x7fd03c0b4680 2024-03-06T07:38:39.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.115+0000 7fd040e00640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd01806b4d0 0x7fd01806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:39.117 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.116+0000 7fd040e00640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd01806b4d0 0x7fd01806d990 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7fd02c0012e0 tx=0x7fd02c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:39.122 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.120+0000 7fd038c00640 1 -- 172.21.15.86:0/2663001195 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd024031800 con 0x7fd03c0b4680 2024-03-06T07:38:39.357 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.355+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd03c15f6a0 con 0x7fd03c0b4680 2024-03-06T07:38:39.358 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.356+0000 7fd038c00640 1 -- 172.21.15.86:0/2663001195 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7fd02407a980 con 0x7fd03c0b4680 2024-03-06T07:38:39.358 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:39.358 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:39.359 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd01806b4d0 msgr2=0x7fd01806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd01806b4d0 0x7fd01806d990 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7fd02c0012e0 tx=0x7fd02c009040 comp rx=0 tx=0).stop 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 msgr2=0x7fd03c0cefa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c0cefa0 secure :-1 s=READY pgs=1843 cs=0 l=1 rev1=1 crypto rx=0x7fd02402b910 tx=0x7fd0240046c0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 shutdown_connections 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd01806b4d0 0x7fd01806d990 unknown :-1 s=CLOSED pgs=780 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd03c15acb0 0x7fd03c0cd4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd03c0b4680 0x7fd03c0cefa0 unknown :-1 s=CLOSED pgs=1843 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 --2- 172.21.15.86:0/2663001195 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd03c072340 0x7fd03c146860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:39.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 >> 172.21.15.86:0/2663001195 conn(0x7fd03c0b1730 msgr2=0x7fd03c0af6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:39.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 shutdown_connections 2024-03-06T07:38:39.362 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:39.359+0000 7fd043800640 1 -- 172.21.15.86:0/2663001195 wait complete. 2024-03-06T07:38:39.584 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:39.584 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:39.728 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:40.585 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-06T07:38:40.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.791+0000 7f28fbe00640 1 Processor -- start 2024-03-06T07:38:40.793 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.791+0000 7f28fbe00640 1 -- start start 2024-03-06T07:38:40.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28fbe00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f28f40d1cc0 0x7f28f40b11b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28fbe00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28f40d4c50 con 0x7f28f40d24c0 2024-03-06T07:38:40.795 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28fbe00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28f40cfa30 con 0x7f28f40b2de0 2024-03-06T07:38:40.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28fbe00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f28f40d4a00 con 0x7f28f40d1cc0 2024-03-06T07:38:40.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28f9400640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7f28f40d24c0 0x7f28f40d28a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:35986/0 (socket says 172.21.15.86:35986) 2024-03-06T07:38:40.796 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28f9400640 1 -- 172.21.15.86:0/2838646635 learned_addr learned my addr 172.21.15.86:0/2838646635 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:40.797 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28f9e00640 1 --2- 172.21.15.86:0/2838646635 >> v2:172.21.15.194:3300/0 conn(0x7f28f40d1cc0 0x7f28f40b11b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1042307471 0 0) 0x7f28f40d4c50 con 0x7f28f40d24c0 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.794+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f28dc003520 con 0x7f28f40d24c0 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2054514853 0 0) 0x7f28dc003520 con 0x7f28f40d24c0 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 >> v2:172.21.15.194:3300/0 conn(0x7f28f40d1cc0 msgr2=0x7f28f40b11b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 --2- 172.21.15.86:0/2838646635 >> v2:172.21.15.194:3300/0 conn(0x7f28f40d1cc0 0x7f28f40b11b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 >> v1:172.21.15.194:6789/0 conn(0x7f28f40b2de0 legacy=0x7f28f40b0aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28f40cfd10 con 0x7f28f40d24c0 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7f28e4001670 con 0x7f28f40d24c0 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 >> v1:172.21.15.86:6789/0 conn(0x7f28f40d24c0 legacy=0x7f28f40d28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28dc003a00 0x7f28dc003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc004800 0x7f28dc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28dc005210 0x7f28dc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28f40d4a00 con 0x7f28dc005210 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28dc00c160 con 0x7f28dc004800 2024-03-06T07:38:40.798 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f28dc00c360 con 0x7f28dc003a00 2024-03-06T07:38:40.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.795+0000 7f28f9e00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28dc005210 0x7f28dc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:40.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.796+0000 7f28f9e00640 1 -- 172.21.15.86:0/2838646635 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28dc003a00 msgr2=0x7f28dc003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:40.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.796+0000 7f28f9e00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28dc003a00 0x7f28dc003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:40.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.796+0000 7f28f9e00640 1 -- 172.21.15.86:0/2838646635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc004800 msgr2=0x7f28dc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:40.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.796+0000 7f28f9e00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc004800 0x7f28dc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:40.799 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.796+0000 7f28f9e00640 1 -- 172.21.15.86:0/2838646635 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28dc003520 con 0x7f28dc005210 2024-03-06T07:38:40.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.797+0000 7f28f9e00640 1 -- 172.21.15.86:0/2838646635 reap_dead start 2024-03-06T07:38:40.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.797+0000 7f28f9e00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28dc005210 0x7f28dc00b9c0 secure :-1 s=READY pgs=1845 cs=0 l=1 rev1=1 crypto rx=0x7f28ec009a90 tx=0x7f28dc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b1185d4728c90e5e server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:40.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.797+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28ec009690 con 0x7f28dc005210 2024-03-06T07:38:40.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.797+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f28ec004020 con 0x7f28dc005210 2024-03-06T07:38:40.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.797+0000 7f28f3e00640 1 -- 172.21.15.86:0/2838646635 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28ec004410 con 0x7f28dc005210 2024-03-06T07:38:40.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.797+0000 7f28fbe00640 1 -- 172.21.15.86:0/2838646635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28dc005210 msgr2=0x7f28dc00b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:40.800 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.797+0000 7f28fbe00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28dc005210 0x7f28dc00b9c0 secure :-1 s=READY pgs=1845 cs=0 l=1 rev1=1 crypto rx=0x7f28ec009a90 tx=0x7f28dc005830 comp rx=0 tx=0).stop 2024-03-06T07:38:40.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.798+0000 7f28fbe00640 1 -- 172.21.15.86:0/2838646635 shutdown_connections 2024-03-06T07:38:40.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.798+0000 7f28fbe00640 1 --2- 172.21.15.86:0/2838646635 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28dc005210 0x7f28dc00b9c0 unknown :-1 s=CLOSED pgs=1845 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:40.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.798+0000 7f28fbe00640 1 -- 172.21.15.86:0/2838646635 >> 172.21.15.86:0/2838646635 conn(0x7f28f40d3ee0 msgr2=0x7f28f40d42f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:40.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.798+0000 7f28fbe00640 1 -- 172.21.15.86:0/2838646635 shutdown_connections 2024-03-06T07:38:40.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.798+0000 7f28fbe00640 1 -- 172.21.15.86:0/2838646635 wait complete. 2024-03-06T07:38:40.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.799+0000 7f28fbe00640 1 Processor -- start 2024-03-06T07:38:40.801 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.800+0000 7f28fbe00640 1 -- start start 2024-03-06T07:38:40.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.800+0000 7f28fbe00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc005210 0x7f28f4160c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28fbe00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28f40b6ba0 0x7f28f40b7020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.802 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28fbe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28f415a700 0x7f28f415ab80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28fbe00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28f40d0080 con 0x7f28f415a700 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28fbe00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28f40cf7e0 con 0x7f28dc005210 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28fbe00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f28f40d28a0 con 0x7f28f40b6ba0 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28f415a700 0x7f28f415ab80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28f415a700 0x7f28f415ab80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60734/0 (socket says 172.21.15.86:60734) 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 -- 172.21.15.86:0/3658002079 learned_addr learned my addr 172.21.15.86:0/3658002079 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9400640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc005210 0x7f28f4160c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:40.803 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f8a00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28f40b6ba0 0x7f28f40b7020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 -- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28f40b6ba0 msgr2=0x7f28f40b7020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28f40b6ba0 0x7f28f40b7020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 -- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc005210 msgr2=0x7f28f4160c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc005210 0x7f28f4160c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.801+0000 7f28f9e00640 1 -- 172.21.15.86:0/3658002079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28f41611c0 con 0x7f28f415a700 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.802+0000 7f28f9e00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28f415a700 0x7f28f415ab80 secure :-1 s=READY pgs=1846 cs=0 l=1 rev1=1 crypto rx=0x7f28ec009b80 tx=0x7f28ec004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.802+0000 7f28f1600640 1 -- 172.21.15.86:0/3658002079 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28ec02cbf0 con 0x7f28f415a700 2024-03-06T07:38:40.804 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.802+0000 7f28f1600640 1 -- 172.21.15.86:0/3658002079 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f28ec02cdb0 con 0x7f28f415a700 2024-03-06T07:38:40.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.802+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28f415b630 con 0x7f28f415a700 2024-03-06T07:38:40.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.802+0000 7f28f1600640 1 -- 172.21.15.86:0/3658002079 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28ec003850 con 0x7f28f415a700 2024-03-06T07:38:40.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.802+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28f40cbb00 con 0x7f28f415a700 2024-03-06T07:38:40.805 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.803+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f28f404f330 con 0x7f28f415a700 2024-03-06T07:38:40.806 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.804+0000 7f28f1600640 1 -- 172.21.15.86:0/3658002079 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f28ec0039f0 con 0x7f28f415a700 2024-03-06T07:38:40.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.806+0000 7f28f1600640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f28d806b4d0 0x7f28d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:40.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.806+0000 7f28f1600640 1 -- 172.21.15.86:0/3658002079 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f28ec0aafc0 con 0x7f28f415a700 2024-03-06T07:38:40.807 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.806+0000 7f28f9400640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f28d806b4d0 0x7f28d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:40.808 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.807+0000 7f28f9400640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f28d806b4d0 0x7f28d806d990 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f28e40018a0 tx=0x7f28e40016d0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:40.811 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:40.810+0000 7f28f1600640 1 -- 172.21.15.86:0/3658002079 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f28ec079e90 con 0x7f28f415a700 2024-03-06T07:38:41.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.099+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f28f40d22a0 con 0x7f28f415a700 2024-03-06T07:38:41.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.100+0000 7f28f1600640 1 -- 172.21.15.86:0/3658002079 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 103 v103) v1 ==== 95+0+3464 (secure 0 0 0) 0x7f28ec079790 con 0x7f28f415a700 2024-03-06T07:38:41.102 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:41.102 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":103,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":103,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-06T07:34:43.455814+0000","modified":"2024-03-06T07:37:13.576501+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8151,"mds_1":8148},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8148":{"gid":8148,"name":"a","rank":1,"incarnation":95,"state":"up:stopping","state_seq":5,"addr":"172.21.15.86:6836/3099697148","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6834","nonce":3099697148},{"type":"v1","addr":"172.21.15.86:6836","nonce":3099697148}]},"join_fscid":-1,"laggy_since":"2024-03-06T07:35:42.924359+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8151":{"gid":8151,"name":"c","rank":0,"incarnation":89,"state":"up:active","state_seq":3,"addr":"172.21.15.86:6837/1131889927","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6835","nonce":1131889927},{"type":"v1","addr":"172.21.15.86:6837","nonce":1131889927}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8151,"qdb_cluster":[8151]},"id":9}]} 2024-03-06T07:38:41.102 INFO:teuthology.orchestra.run.smithi086.stderr:dumped fsmap epoch 103 2024-03-06T07:38:41.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.102+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f28d806b4d0 msgr2=0x7f28d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:41.104 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.102+0000 7f28fbe00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f28d806b4d0 0x7f28d806d990 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f28e40018a0 tx=0x7f28e40016d0 comp rx=0 tx=0).stop 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.102+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28f415a700 msgr2=0x7f28f415ab80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.102+0000 7f28fbe00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28f415a700 0x7f28f415ab80 secure :-1 s=READY pgs=1846 cs=0 l=1 rev1=1 crypto rx=0x7f28ec009b80 tx=0x7f28ec004730 comp rx=0 tx=0).stop 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 shutdown_connections 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f28d806b4d0 0x7f28d806d990 unknown :-1 s=CLOSED pgs=781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f28f415a700 0x7f28f415ab80 unknown :-1 s=CLOSED pgs=1846 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f28f40b6ba0 0x7f28f40b7020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 --2- 172.21.15.86:0/3658002079 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f28dc005210 0x7f28f4160c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 >> 172.21.15.86:0/3658002079 conn(0x7f28f40d3ee0 msgr2=0x7f28f40b31c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 shutdown_connections 2024-03-06T07:38:41.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:41.103+0000 7f28fbe00640 1 -- 172.21.15.86:0/3658002079 wait complete. 2024-03-06T07:38:41.326 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 103, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-06T07:34:43.455814+0000', 'modified': '2024-03-06T07:37:13.576501+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8151, 'mds_1': 8148}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8148': {'gid': 8148, 'name': 'a', 'rank': 1, 'incarnation': 95, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.86:6836/3099697148', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6834', 'nonce': 3099697148}, {'type': 'v1', 'addr': '172.21.15.86:6836', 'nonce': 3099697148}]}, 'join_fscid': -1, 'laggy_since': '2024-03-06T07:35:42.924359+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8151': {'gid': 8151, 'name': 'c', 'rank': 0, 'incarnation': 89, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.86:6837/1131889927', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.86:6835', 'nonce': 1131889927}, {'type': 'v1', 'addr': '172.21.15.86:6837', 'nonce': 1131889927}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8151, 'qdb_cluster': [8151]} 2024-03-06T07:38:41.326 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8148:up:stopping 2024-03-06T07:38:41.730 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:42.327 DEBUG:tasks.cephfs.filesystem:status = { "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "default_fscid": 9, "epoch": 103, "feature_flags": { "enable_multiple": true, "ever_enabled_multiple": true }, "filesystems": [ { "id": 9, "mdsmap": { "bal_rank_mask": "-1", "balancer": "", "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_7": "mds uses inline data", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "created": "2024-03-06T07:34:43.455814+0000", "damaged": [], "data_pools": [ 19 ], "enabled": true, "epoch": 103, "ever_allowed_features": 0, "explicitly_allowed_features": 0, "failed": [], "flags": 18, "flags_state": { "allow_multimds_snaps": true, "allow_snaps": true, "allow_standby_replay": false, "balance_automate": false, "joinable": true, "refuse_client_session": false, "refuse_standby_for_another_fs": false }, "fs_name": "cephfs", "in": [ 0, 1 ], "info": { "gid_8148": { "addr": "172.21.15.86:6836/3099697148", "addrs": { "addrvec": [ { "addr": "172.21.15.86:6834", "nonce": 3099697148, "type": "v2" }, { "addr": "172.21.15.86:6836", "nonce": 3099697148, "type": "v1" } ] }, "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_7": "mds uses inline data", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "export_targets": [], "features": 4540701547738038271, "flags": 0, "gid": 8148, "incarnation": 95, "join_fscid": -1, "laggy_since": "2024-03-06T07:35:42.924359+0000", "name": "a", "rank": 1, "state": "up:stopping", "state_seq": 5 }, "gid_8151": { "addr": "172.21.15.86:6837/1131889927", "addrs": { "addrvec": [ { "addr": "172.21.15.86:6835", "nonce": 1131889927, "type": "v2" }, { "addr": "172.21.15.86:6837", "nonce": 1131889927, "type": "v1" } ] }, "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_7": "mds uses inline data", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "export_targets": [], "features": 4540701547738038271, "flags": 0, "gid": 8151, "incarnation": 89, "join_fscid": -1, "name": "c", "rank": 0, "state": "up:active", "state_seq": 3 } }, "last_failure": 0, "last_failure_osd_epoch": 0, "max_file_size": 1099511627776, "max_mds": 1, "max_xattr_size": 65536, "metadata_pool": 18, "modified": "2024-03-06T07:37:13.576501+0000", "qdb_cluster": [ 8151 ], "qdb_leader": 8151, "required_client_features": {}, "root": 0, "session_autoclose": 300, "session_timeout": 60, "standby_count_wanted": 0, "stopped": [], "tableserver": 0, "up": { "mds_0": 8151, "mds_1": 8148 } } } ], "standbys": [] } 2024-03-06T07:38:42.328 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-06T07:38:42.328 DEBUG:teuthology.orchestra.run.smithi086:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:38:42.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-06T07:38:42.559 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-06T07:38:42.559 INFO:tasks.cephfs.mount:Terminating background process 2024-03-06T07:38:42.562 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:42 stdin-killer INFO: expiration expected; waiting 5 seconds for command to complete 2024-03-06T07:38:43.733 INFO:teuthology.orchestra.run.smithi086.stderr:open_dir_background: keeping dir open 2024-03-06T07:38:43.744 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:43 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-06T07:38:43.765 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:38:43.765 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:38:43.860 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-03-06T07:38:43.860 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-06T07:38:43.907 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-06T07:38:44.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.147+0000 7f34f3200640 1 Processor -- start 2024-03-06T07:38:44.148 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.147+0000 7f34f3200640 1 -- start start 2024-03-06T07:38:44.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.148+0000 7f34f3200640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f34f41615c0 0x7f34f40d4440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.148+0000 7f34f3200640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f34f40d0ab0 0x7f34f40d2340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.148+0000 7f34f3200640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f34f40d4980 con 0x7f34f40d0600 2024-03-06T07:38:44.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.148+0000 7f34f3200640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f34f40d4c00 con 0x7f34f40d0ab0 2024-03-06T07:38:44.150 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.148+0000 7f34f3200640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f34f40b40b0 con 0x7f34f41615c0 2024-03-06T07:38:44.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f34f40d0ab0 0x7f34f40d2340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f2800640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f34f40d0ab0 0x7f34f40d2340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60740/0 (socket says 172.21.15.86:60740) 2024-03-06T07:38:44.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f2800640 1 -- 172.21.15.86:0/283877541 learned_addr learned my addr 172.21.15.86:0/283877541 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:44.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f1e00640 1 --2- 172.21.15.86:0/283877541 >> v2:172.21.15.194:3300/0 conn(0x7f34f41615c0 0x7f34f40d4440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.151 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f2800640 1 -- 172.21.15.86:0/283877541 >> v2:172.21.15.194:3300/0 conn(0x7f34f41615c0 msgr2=0x7f34f40d4440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:44.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f2800640 1 --2- 172.21.15.86:0/283877541 >> v2:172.21.15.194:3300/0 conn(0x7f34f41615c0 0x7f34f40d4440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f2800640 1 -- 172.21.15.86:0/283877541 >> v1:172.21.15.194:6789/0 conn(0x7f34f40d0600 legacy=0x7f34f40d1c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:44.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.149+0000 7f34f2800640 1 -- 172.21.15.86:0/283877541 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34f409d5e0 con 0x7f34f40d0ab0 2024-03-06T07:38:44.152 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.151+0000 7f34f2800640 1 --2- 172.21.15.86:0/283877541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f40d0ab0 0x7f34f40d2340 secure :-1 s=READY pgs=1847 cs=0 l=1 rev1=1 crypto rx=0x7f34f408dcf0 tx=0x7f34ec02b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=25df13c4199b36c server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:44.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.151+0000 7f34f1400640 1 -- 172.21.15.86:0/283877541 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34ec02e070 con 0x7f34f40d0ab0 2024-03-06T07:38:44.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.151+0000 7f34f1400640 1 -- 172.21.15.86:0/283877541 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f34ec037070 con 0x7f34f40d0ab0 2024-03-06T07:38:44.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.151+0000 7f34f1400640 1 -- 172.21.15.86:0/283877541 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34ec031a00 con 0x7f34f40d0ab0 2024-03-06T07:38:44.153 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.151+0000 7f34f3200640 1 -- 172.21.15.86:0/283877541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f40d0ab0 msgr2=0x7f34f40d2340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.151+0000 7f34f3200640 1 --2- 172.21.15.86:0/283877541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f40d0ab0 0x7f34f40d2340 secure :-1 s=READY pgs=1847 cs=0 l=1 rev1=1 crypto rx=0x7f34f408dcf0 tx=0x7f34ec02b880 comp rx=0 tx=0).stop 2024-03-06T07:38:44.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.152+0000 7f34f3200640 1 -- 172.21.15.86:0/283877541 shutdown_connections 2024-03-06T07:38:44.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.152+0000 7f34f3200640 1 --2- 172.21.15.86:0/283877541 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f40d0ab0 0x7f34f40d2340 unknown :-1 s=CLOSED pgs=1847 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.152+0000 7f34f3200640 1 --2- 172.21.15.86:0/283877541 >> v2:172.21.15.194:3300/0 conn(0x7f34f41615c0 0x7f34f40d4440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.152+0000 7f34f3200640 1 -- 172.21.15.86:0/283877541 >> 172.21.15.86:0/283877541 conn(0x7f34f409d080 msgr2=0x7f34f40d3d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:44.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.152+0000 7f34f3200640 1 -- 172.21.15.86:0/283877541 shutdown_connections 2024-03-06T07:38:44.154 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.152+0000 7f34f3200640 1 -- 172.21.15.86:0/283877541 wait complete. 2024-03-06T07:38:44.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.153+0000 7f34f3200640 1 Processor -- start 2024-03-06T07:38:44.155 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.154+0000 7f34f3200640 1 -- start start 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.154+0000 7f34f3200640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34f40d0600 0x7f34f40b93f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.154+0000 7f34f3200640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34f40d0ab0 0x7f34f40c8390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.154+0000 7f34f3200640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f41615c0 0x7f34f414e190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.154+0000 7f34f3200640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34f40b3d40 con 0x7f34f41615c0 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.154+0000 7f34f3200640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34f40b3a60 con 0x7f34f40d0ab0 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.154+0000 7f34f3200640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f34f40cf850 con 0x7f34f40d0600 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f41615c0 0x7f34f414e190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.157 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f41615c0 0x7f34f414e190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60754/0 (socket says 172.21.15.86:60754) 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 -- 172.21.15.86:0/3915267458 learned_addr learned my addr 172.21.15.86:0/3915267458 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34e9400640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34f40d0ab0 0x7f34f40c8390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f1e00640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34f40d0600 0x7f34f40b93f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 -- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34f40d0600 msgr2=0x7f34f40b93f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34f40d0600 0x7f34f40b93f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 -- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34f40d0ab0 msgr2=0x7f34f40c8390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34f40d0ab0 0x7f34f40c8390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.155+0000 7f34f2800640 1 -- 172.21.15.86:0/3915267458 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34f40cbe50 con 0x7f34f41615c0 2024-03-06T07:38:44.158 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.156+0000 7f34f2800640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f41615c0 0x7f34f414e190 secure :-1 s=READY pgs=1848 cs=0 l=1 rev1=1 crypto rx=0x7f34ec0090b0 tx=0x7f34ec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:44.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.156+0000 7f34eaa00640 1 -- 172.21.15.86:0/3915267458 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34ec032670 con 0x7f34f41615c0 2024-03-06T07:38:44.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.156+0000 7f34eaa00640 1 -- 172.21.15.86:0/3915267458 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f34ec032810 con 0x7f34f41615c0 2024-03-06T07:38:44.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.156+0000 7f34eaa00640 1 -- 172.21.15.86:0/3915267458 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34ec032c00 con 0x7f34f41615c0 2024-03-06T07:38:44.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.156+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f34f40b9b10 con 0x7f34f41615c0 2024-03-06T07:38:44.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.156+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f34f40cc3f0 con 0x7f34f41615c0 2024-03-06T07:38:44.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.157+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f34f404f330 con 0x7f34f41615c0 2024-03-06T07:38:44.159 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.158+0000 7f34eaa00640 1 -- 172.21.15.86:0/3915267458 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f34ec04c060 con 0x7f34f41615c0 2024-03-06T07:38:44.160 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.159+0000 7f34eaa00640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34d006b500 0x7f34d006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.159+0000 7f34eaa00640 1 -- 172.21.15.86:0/3915267458 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f34ec0abb60 con 0x7f34f41615c0 2024-03-06T07:38:44.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.159+0000 7f34f1e00640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34d006b500 0x7f34d006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.161 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.160+0000 7f34f1e00640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34d006b500 0x7f34d006d9c0 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7f34e40099c0 tx=0x7f34e4003990 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:44.165 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.164+0000 7f34eaa00640 1 -- 172.21.15.86:0/3915267458 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f34ec040070 con 0x7f34f41615c0 2024-03-06T07:38:44.361 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.360+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f34f414e6d0 con 0x7f34f41615c0 2024-03-06T07:38:44.363 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.361+0000 7f34eaa00640 1 -- 172.21.15.86:0/3915267458 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v131) v1 ==== 74+0+16732 (secure 0 0 0) 0x7f34f40d4980 con 0x7f34f41615c0 2024-03-06T07:38:44.363 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-03-06T07:38:44.364 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":131,"fsid":"013155b5-b812-4a77-8af3-f8c90b517e11","created":"2024-03-06T07:26:47.831907+0000","modified":"2024-03-06T07:35:01.712964+0000","last_up_change":"2024-03-06T07:26:59.589976+0000","last_in_change":"2024-03-06T07:26:51.348822+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-06T07:27:00.921596+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-06T07:34:41.263023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"126","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-06T07:34:42.462100+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"127","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.25,"score_stable":1.25,"optimal_score":1,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ca3d759d-401c-42df-893e-5c63b2bbab14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":128,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6801","nonce":1601033517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6803","nonce":1601033517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6807","nonce":1601033517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":1601033517},{"type":"v1","addr":"172.21.15.86:6805","nonce":1601033517}]},"public_addr":"172.21.15.86:6801/1601033517","cluster_addr":"172.21.15.86:6803/1601033517","heartbeat_back_addr":"172.21.15.86:6807/1601033517","heartbeat_front_addr":"172.21.15.86:6805/1601033517","state":["exists","up"]},{"osd":1,"uuid":"a4b124ea-80de-4143-892b-1cbf191ab49e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6809","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6811","nonce":2555210107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6813","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6815","nonce":2555210107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6821","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6823","nonce":2555210107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6817","nonce":2555210107},{"type":"v1","addr":"172.21.15.86:6819","nonce":2555210107}]},"public_addr":"172.21.15.86:6811/2555210107","cluster_addr":"172.21.15.86:6815/2555210107","heartbeat_back_addr":"172.21.15.86:6823/2555210107","heartbeat_front_addr":"172.21.15.86:6819/2555210107","state":["exists","up"]},{"osd":2,"uuid":"52f80d9b-399d-40fa-ba7e-0119d56617a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6810","nonce":2264489672}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6814","nonce":2264489672}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6822","nonce":2264489672}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2264489672},{"type":"v1","addr":"172.21.15.86:6818","nonce":2264489672}]},"public_addr":"172.21.15.86:6810/2264489672","cluster_addr":"172.21.15.86:6814/2264489672","heartbeat_back_addr":"172.21.15.86:6822/2264489672","heartbeat_front_addr":"172.21.15.86:6818/2264489672","state":["exists","up"]},{"osd":3,"uuid":"771a4b29-0323-4910-acff-3261da6fee51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6825","nonce":3540554891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6827","nonce":3540554891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6831","nonce":3540554891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":3540554891},{"type":"v1","addr":"172.21.15.86:6829","nonce":3540554891}]},"public_addr":"172.21.15.86:6825/3540554891","cluster_addr":"172.21.15.86:6827/3540554891","heartbeat_back_addr":"172.21.15.86:6831/3540554891","heartbeat_front_addr":"172.21.15.86:6829/3540554891","state":["exists","up"]},{"osd":4,"uuid":"79765dea-3964-41f3-a109-a064f2fb7c89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6801","nonce":2403584566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6803","nonce":2403584566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6807","nonce":2403584566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":2403584566},{"type":"v1","addr":"172.21.15.194:6805","nonce":2403584566}]},"public_addr":"172.21.15.194:6801/2403584566","cluster_addr":"172.21.15.194:6803/2403584566","heartbeat_back_addr":"172.21.15.194:6807/2403584566","heartbeat_front_addr":"172.21.15.194:6805/2403584566","state":["exists","up"]},{"osd":5,"uuid":"dbfaf0fd-5805-4e2d-8903-0b4a5c522443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6817","nonce":1278478503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6820","nonce":1278478503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6828","nonce":1278478503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":1278478503},{"type":"v1","addr":"172.21.15.194:6824","nonce":1278478503}]},"public_addr":"172.21.15.194:6817/1278478503","cluster_addr":"172.21.15.194:6820/1278478503","heartbeat_back_addr":"172.21.15.194:6828/1278478503","heartbeat_front_addr":"172.21.15.194:6824/1278478503","state":["exists","up"]},{"osd":6,"uuid":"70fa6d13-7e1a-4720-9009-24c3c629e40f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":128,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6809","nonce":2611339787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6811","nonce":2611339787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6815","nonce":2611339787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":2611339787},{"type":"v1","addr":"172.21.15.194:6813","nonce":2611339787}]},"public_addr":"172.21.15.194:6809/2611339787","cluster_addr":"172.21.15.194:6811/2611339787","heartbeat_back_addr":"172.21.15.194:6815/2611339787","heartbeat_front_addr":"172.21.15.194:6813/2611339787","state":["exists","up"]},{"osd":7,"uuid":"c6cb9f27-28ca-4d7d-9064-bb8f18aea0c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6819","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6821","nonce":3406102027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6823","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6825","nonce":3406102027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6831","nonce":3406102027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6827","nonce":3406102027},{"type":"v1","addr":"172.21.15.194:6829","nonce":3406102027}]},"public_addr":"172.21.15.194:6821/3406102027","cluster_addr":"172.21.15.194:6825/3406102027","heartbeat_back_addr":"172.21.15.194:6831/3406102027","heartbeat_front_addr":"172.21.15.194:6829/3406102027","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.607450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.664828+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.596013+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.688505+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.704111+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:57.001222+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.889920+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-06T07:26:56.735043+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"19.e","mappings":[{"from":2,"to":0}]},{"pgid":"19.25","mappings":[{"from":2,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34d006b500 msgr2=0x7f34d006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34d006b500 0x7f34d006d9c0 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7f34e40099c0 tx=0x7f34e4003990 comp rx=0 tx=0).stop 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f41615c0 msgr2=0x7f34f414e190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f41615c0 0x7f34f414e190 secure :-1 s=READY pgs=1848 cs=0 l=1 rev1=1 crypto rx=0x7f34ec0090b0 tx=0x7f34ec0046c0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 shutdown_connections 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f34d006b500 0x7f34d006d9c0 unknown :-1 s=CLOSED pgs=782 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f34f41615c0 0x7f34f414e190 unknown :-1 s=CLOSED pgs=1848 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f34f40d0ab0 0x7f34f40c8390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 --2- 172.21.15.86:0/3915267458 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f34f40d0600 0x7f34f40b93f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 >> 172.21.15.86:0/3915267458 conn(0x7f34f409d080 msgr2=0x7f34f40b21f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:44.366 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 shutdown_connections 2024-03-06T07:38:44.367 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.364+0000 7f34f3200640 1 -- 172.21.15.86:0/3915267458 wait complete. 2024-03-06T07:38:44.383 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-06T07:27:00.921596+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '18', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 18, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-06T07:34:41.263023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '126', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 19, 'pool_name': 'cephfs_data', 'create_time': '2024-03-06T07:34:42.462100+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '127', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.25, 'score_stable': 1.25, 'optimal_score': 1, 'raw_score_acting': 1.25, 'raw_score_stable': 1.25, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-06T07:38:44.383 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-06T07:38:44.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.589+0000 7fd07e400640 1 Processor -- start 2024-03-06T07:38:44.590 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.589+0000 7fd07e400640 1 -- start start 2024-03-06T07:38:44.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.590+0000 7fd07e400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7fd07815fc10 0x7fd078070ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.590+0000 7fd07e400640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd0780b46c0 con 0x7fd0780b3f70 2024-03-06T07:38:44.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.590+0000 7fd07e400640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd0780b49a0 con 0x7fd07815fff0 2024-03-06T07:38:44.592 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.590+0000 7fd07e400640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd078072c90 con 0x7fd07815fc10 2024-03-06T07:38:44.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd077e00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fd0780b3f70 0x7fd0780ce4a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36002/0 (socket says 172.21.15.86:36002) 2024-03-06T07:38:44.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd077e00640 1 -- 172.21.15.86:0/2021440386 learned_addr learned my addr 172.21.15.86:0/2021440386 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:44.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd077400640 1 --2- 172.21.15.86:0/2021440386 >> v2:172.21.15.86:3300/0 conn(0x7fd07815fc10 0x7fd078070ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd076a00640 1 -- 172.21.15.86:0/2021440386 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1134491141 0 0) 0x7fd0780b46c0 con 0x7fd0780b3f70 2024-03-06T07:38:44.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd076a00640 1 -- 172.21.15.86:0/2021440386 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd05c003520 con 0x7fd0780b3f70 2024-03-06T07:38:44.593 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd077400640 1 -- 172.21.15.86:0/2021440386 >> v1:172.21.15.194:6789/0 conn(0x7fd07815fff0 legacy=0x7fd078160450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd077400640 1 -- 172.21.15.86:0/2021440386 >> v1:172.21.15.86:6789/0 conn(0x7fd0780b3f70 legacy=0x7fd0780ce4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.594 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.591+0000 7fd077400640 1 -- 172.21.15.86:0/2021440386 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd07809dd80 con 0x7fd07815fc10 2024-03-06T07:38:44.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.593+0000 7fd077400640 1 --2- 172.21.15.86:0/2021440386 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd07815fc10 0x7fd078070ff0 secure :-1 s=READY pgs=1850 cs=0 l=1 rev1=1 crypto rx=0x7fd0680097a0 tx=0x7fd06802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f86ebb60f9e89e01 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:44.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.593+0000 7fd076a00640 1 -- 172.21.15.86:0/2021440386 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd068034070 con 0x7fd07815fc10 2024-03-06T07:38:44.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd076a00640 1 -- 172.21.15.86:0/2021440386 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd068030610 con 0x7fd07815fc10 2024-03-06T07:38:44.595 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd076a00640 1 -- 172.21.15.86:0/2021440386 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0680309e0 con 0x7fd07815fc10 2024-03-06T07:38:44.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd07e400640 1 -- 172.21.15.86:0/2021440386 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd07815fc10 msgr2=0x7fd078070ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd07e400640 1 --2- 172.21.15.86:0/2021440386 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd07815fc10 0x7fd078070ff0 secure :-1 s=READY pgs=1850 cs=0 l=1 rev1=1 crypto rx=0x7fd0680097a0 tx=0x7fd06802b960 comp rx=0 tx=0).stop 2024-03-06T07:38:44.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd07e400640 1 -- 172.21.15.86:0/2021440386 shutdown_connections 2024-03-06T07:38:44.596 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd07e400640 1 --2- 172.21.15.86:0/2021440386 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd07815fc10 0x7fd078070ff0 unknown :-1 s=CLOSED pgs=1850 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd07e400640 1 -- 172.21.15.86:0/2021440386 >> 172.21.15.86:0/2021440386 conn(0x7fd0780b1940 msgr2=0x7fd0780b1d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:44.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd07e400640 1 -- 172.21.15.86:0/2021440386 shutdown_connections 2024-03-06T07:38:44.597 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.594+0000 7fd07e400640 1 -- 172.21.15.86:0/2021440386 wait complete. 2024-03-06T07:38:44.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.597+0000 7fd07e400640 1 Processor -- start 2024-03-06T07:38:44.598 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.597+0000 7fd07e400640 1 -- start start 2024-03-06T07:38:44.601 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd07e400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0780b3f70 0x7fd07813dbb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd07e400640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd07815fc10 0x7fd07815c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd07e400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd07815fff0 0x7fd07815cb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd07e400640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd07815c460 con 0x7fd0780b3f70 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd07e400640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd07815c2e0 con 0x7fd07815fc10 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd07e400640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0780cebb0 con 0x7fd07815fff0 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0780b3f70 0x7fd07813dbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0780b3f70 0x7fd07813dbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60778/0 (socket says 172.21.15.86:60778) 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 -- 172.21.15.86:0/3399698165 learned_addr learned my addr 172.21.15.86:0/3399698165 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 -- 172.21.15.86:0/3399698165 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd07815fff0 msgr2=0x7fd07815cb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd07815fff0 0x7fd07815cb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 -- 172.21.15.86:0/3399698165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd07815fc10 msgr2=0x7fd07815c5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.602 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd07815fc10 0x7fd07815c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.599+0000 7fd077400640 1 -- 172.21.15.86:0/3399698165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd078150710 con 0x7fd0780b3f70 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.600+0000 7fd077400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0780b3f70 0x7fd07813dbb0 secure :-1 s=READY pgs=1851 cs=0 l=1 rev1=1 crypto rx=0x7fd06802bf70 tx=0x7fd0680046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.600+0000 7fd06fe00640 1 -- 172.21.15.86:0/3399698165 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd068034040 con 0x7fd0780b3f70 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.600+0000 7fd06fe00640 1 -- 172.21.15.86:0/3399698165 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd068030410 con 0x7fd0780b3f70 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.600+0000 7fd06fe00640 1 -- 172.21.15.86:0/3399698165 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd068039c50 con 0x7fd0780b3f70 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.600+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd07815d150 con 0x7fd0780b3f70 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.600+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd07815d710 con 0x7fd0780b3f70 2024-03-06T07:38:44.603 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.602+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd07804f330 con 0x7fd0780b3f70 2024-03-06T07:38:44.604 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.602+0000 7fd06fe00640 1 -- 172.21.15.86:0/3399698165 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd068037070 con 0x7fd0780b3f70 2024-03-06T07:38:44.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.603+0000 7fd06fe00640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd05406b4d0 0x7fd05406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:44.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.603+0000 7fd06fe00640 1 -- 172.21.15.86:0/3399698165 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd0680ac2f0 con 0x7fd0780b3f70 2024-03-06T07:38:44.605 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.604+0000 7fd06ea00640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd05406b4d0 0x7fd05406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:44.606 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.604+0000 7fd06ea00640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd05406b4d0 0x7fd05406d990 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fd060000f60 tx=0x7fd0600079d0 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:44.614 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.612+0000 7fd06fe00640 1 -- 172.21.15.86:0/3399698165 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd0680305e0 con 0x7fd0780b3f70 2024-03-06T07:38:44.857 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.856+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd07806ef30 con 0x7fd0780b3f70 2024-03-06T07:38:44.858 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.856+0000 7fd06fe00640 1 -- 172.21.15.86:0/3399698165 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v131) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd06807b000 con 0x7fd0780b3f70 2024-03-06T07:38:44.858 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd05406b4d0 msgr2=0x7fd05406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd05406b4d0 0x7fd05406d990 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fd060000f60 tx=0x7fd0600079d0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0780b3f70 msgr2=0x7fd07813dbb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0780b3f70 0x7fd07813dbb0 secure :-1 s=READY pgs=1851 cs=0 l=1 rev1=1 crypto rx=0x7fd06802bf70 tx=0x7fd0680046c0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 shutdown_connections 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fd05406b4d0 0x7fd05406d990 unknown :-1 s=CLOSED pgs=783 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fd07815fff0 0x7fd07815cb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fd07815fc10 0x7fd07815c5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 --2- 172.21.15.86:0/3399698165 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fd0780b3f70 0x7fd07813dbb0 unknown :-1 s=CLOSED pgs=1851 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 >> 172.21.15.86:0/3399698165 conn(0x7fd0780b1940 msgr2=0x7fd0781519a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 shutdown_connections 2024-03-06T07:38:44.861 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:44.859+0000 7fd07e400640 1 -- 172.21.15.86:0/3399698165 wait complete. 2024-03-06T07:38:44.878 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-06T07:38:45.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.094+0000 7fae46600640 1 Processor -- start 2024-03-06T07:38:45.096 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.094+0000 7fae46600640 1 -- start start 2024-03-06T07:38:45.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.095+0000 7fae46600640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7fae400d07f0 0x7fae400d0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.095+0000 7fae46600640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fae400b3730 con 0x7fae400d1bf0 2024-03-06T07:38:45.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.095+0000 7fae46600640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fae40067b20 con 0x7fae4015e450 2024-03-06T07:38:45.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.095+0000 7fae46600640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fae400d4930 con 0x7fae400d07f0 2024-03-06T07:38:45.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.096+0000 7fae3fe00640 1 --1- >> v1:172.21.15.86:6789/0 conn(0x7fae400d1bf0 0x7fae400b4b90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:36010/0 (socket says 172.21.15.86:36010) 2024-03-06T07:38:45.098 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.096+0000 7fae3fe00640 1 -- 172.21.15.86:0/3337053699 learned_addr learned my addr 172.21.15.86:0/3337053699 (peer_addr_for_me v1:172.21.15.86:0/0) 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.096+0000 7fae3f400640 1 --2- 172.21.15.86:0/3337053699 >> v2:172.21.15.194:3300/0 conn(0x7fae400d07f0 0x7fae400d0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.096+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1901556006 0 0) 0x7fae400b3730 con 0x7fae400d1bf0 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.096+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 --> v1:172.21.15.86:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fae24003520 con 0x7fae400d1bf0 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 <== mon.0 v1:172.21.15.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4039183088 0 0) 0x7fae24003520 con 0x7fae400d1bf0 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 >> v2:172.21.15.194:3300/0 conn(0x7fae400d07f0 msgr2=0x7fae400d0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 --2- 172.21.15.86:0/3337053699 >> v2:172.21.15.194:3300/0 conn(0x7fae400d07f0 0x7fae400d0bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 >> v1:172.21.15.194:6789/0 conn(0x7fae4015e450 legacy=0x7fae400b4480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 --> v1:172.21.15.86:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae400d1110 con 0x7fae400d1bf0 2024-03-06T07:38:45.100 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 <== mon.0 v1:172.21.15.86:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 2663644978 0 0) 0x7fae380019f0 con 0x7fae400d1bf0 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 >> v1:172.21.15.86:6789/0 conn(0x7fae400d1bf0 legacy=0x7fae400b4b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae240039b0 0x7fae24003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae24004800 0x7fae24004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240051c0 0x7fae2400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae400d4930 con 0x7fae240039b0 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae2400c160 con 0x7fae24004800 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fae2400c360 con 0x7fae240051c0 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.097+0000 7fae3f400640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae240039b0 0x7fae24003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae3f400640 1 -- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240051c0 msgr2=0x7fae2400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae3f400640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240051c0 0x7fae2400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae37e00640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae24004800 0x7fae24004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae3f400640 1 -- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae24004800 msgr2=0x7fae24004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae3f400640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae24004800 0x7fae24004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.101 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae3fe00640 1 -- 172.21.15.86:0/3337053699 reap_dead start 2024-03-06T07:38:45.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae3f400640 1 -- 172.21.15.86:0/3337053699 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae24003520 con 0x7fae240039b0 2024-03-06T07:38:45.102 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.098+0000 7fae37e00640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae24004800 0x7fae24004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.099+0000 7fae3f400640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae240039b0 0x7fae24003dd0 secure :-1 s=READY pgs=1853 cs=0 l=1 rev1=1 crypto rx=0x7fae30009ac0 tx=0x7fae24005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2f8c36527e549f6b server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.099+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae30039070 con 0x7fae240039b0 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.099+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fae3002c6b0 con 0x7fae240039b0 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.099+0000 7fae3ea00640 1 -- 172.21.15.86:0/3337053699 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae30009690 con 0x7fae240039b0 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.100+0000 7fae46600640 1 -- 172.21.15.86:0/3337053699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae240039b0 msgr2=0x7fae24003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.100+0000 7fae46600640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae240039b0 0x7fae24003dd0 secure :-1 s=READY pgs=1853 cs=0 l=1 rev1=1 crypto rx=0x7fae30009ac0 tx=0x7fae24005830 comp rx=0 tx=0).stop 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.100+0000 7fae46600640 1 -- 172.21.15.86:0/3337053699 shutdown_connections 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.100+0000 7fae46600640 1 --2- 172.21.15.86:0/3337053699 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae240039b0 0x7fae24003dd0 unknown :-1 s=CLOSED pgs=1853 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.100+0000 7fae46600640 1 -- 172.21.15.86:0/3337053699 >> 172.21.15.86:0/3337053699 conn(0x7fae400d3e10 msgr2=0x7fae400d4220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.100+0000 7fae46600640 1 -- 172.21.15.86:0/3337053699 shutdown_connections 2024-03-06T07:38:45.103 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.100+0000 7fae46600640 1 -- 172.21.15.86:0/3337053699 wait complete. 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.102+0000 7fae46600640 1 Processor -- start 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.102+0000 7fae46600640 1 -- start start 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae46600640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240039b0 0x7fae40151180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae46600640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae400b8df0 0x7fae400b7300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae46600640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae400b7840 0x7fae401e67b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae46600640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae400b3c80 con 0x7fae400b7840 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae46600640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae400b39a0 con 0x7fae400b8df0 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae46600640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fae400b3f60 con 0x7fae240039b0 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae400b7840 0x7fae401e67b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.105 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae3fe00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae400b7840 0x7fae401e67b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60806/0 (socket says 172.21.15.86:60806) 2024-03-06T07:38:45.106 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.103+0000 7fae3fe00640 1 -- 172.21.15.86:0/1940921040 learned_addr learned my addr 172.21.15.86:0/1940921040 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae37e00640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae400b8df0 0x7fae400b7300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae3f400640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240039b0 0x7fae40151180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae3fe00640 1 -- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240039b0 msgr2=0x7fae40151180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae3fe00640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240039b0 0x7fae40151180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae3fe00640 1 -- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae400b8df0 msgr2=0x7fae400b7300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae3fe00640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae400b8df0 0x7fae400b7300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae3fe00640 1 -- 172.21.15.86:0/1940921040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae400b7d50 con 0x7fae400b7840 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae3fe00640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae400b7840 0x7fae401e67b0 secure :-1 s=READY pgs=1854 cs=0 l=1 rev1=1 crypto rx=0x7fae400b9ec0 tx=0x7fae3800cf00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae37400640 1 -- 172.21.15.86:0/1940921040 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae3800a5b0 con 0x7fae400b7840 2024-03-06T07:38:45.107 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae37400640 1 -- 172.21.15.86:0/1940921040 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fae3800a750 con 0x7fae400b7840 2024-03-06T07:38:45.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fae400b8030 con 0x7fae400b7840 2024-03-06T07:38:45.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.104+0000 7fae37400640 1 -- 172.21.15.86:0/1940921040 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae3800ab40 con 0x7fae400b7840 2024-03-06T07:38:45.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.105+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae400b85f0 con 0x7fae400b7840 2024-03-06T07:38:45.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.106+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fae4004f330 con 0x7fae400b7840 2024-03-06T07:38:45.108 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.107+0000 7fae37400640 1 -- 172.21.15.86:0/1940921040 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fae38011070 con 0x7fae400b7840 2024-03-06T07:38:45.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.108+0000 7fae37400640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae1c06b500 0x7fae1c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.108+0000 7fae37400640 1 -- 172.21.15.86:0/1940921040 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fae38089cf0 con 0x7fae400b7840 2024-03-06T07:38:45.109 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.108+0000 7fae3f400640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae1c06b500 0x7fae1c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.110 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.109+0000 7fae3f400640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae1c06b500 0x7fae1c06d9c0 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7fae30036000 tx=0x7fae30030000 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:45.114 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.112+0000 7fae37400640 1 -- 172.21.15.86:0/1940921040 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fae3808e070 con 0x7fae400b7840 2024-03-06T07:38:45.415 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.413+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fae400d21d0 con 0x7fae400b7840 2024-03-06T07:38:45.416 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.414+0000 7fae37400640 1 -- 172.21.15.86:0/1940921040 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v131) v1 ==== 104+0+11 (secure 0 0 0) 0x7fae3801a060 con 0x7fae400b7840 2024-03-06T07:38:45.416 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 32 2024-03-06T07:38:45.419 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae1c06b500 msgr2=0x7fae1c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae1c06b500 0x7fae1c06d9c0 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7fae30036000 tx=0x7fae30030000 comp rx=0 tx=0).stop 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae400b7840 msgr2=0x7fae401e67b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae400b7840 0x7fae401e67b0 secure :-1 s=READY pgs=1854 cs=0 l=1 rev1=1 crypto rx=0x7fae400b9ec0 tx=0x7fae3800cf00 comp rx=0 tx=0).stop 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 shutdown_connections 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7fae1c06b500 0x7fae1c06d9c0 unknown :-1 s=CLOSED pgs=784 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7fae400b7840 0x7fae401e67b0 unknown :-1 s=CLOSED pgs=1854 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7fae400b8df0 0x7fae400b7300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 --2- 172.21.15.86:0/1940921040 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7fae240039b0 0x7fae40151180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 >> 172.21.15.86:0/1940921040 conn(0x7fae400d3e10 msgr2=0x7fae4006ba00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 shutdown_connections 2024-03-06T07:38:45.420 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.418+0000 7fae46600640 1 -- 172.21.15.86:0/1940921040 wait complete. 2024-03-06T07:38:45.446 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-06T07:38:45.650 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.649+0000 7f3d88c00640 1 Processor -- start 2024-03-06T07:38:45.651 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.649+0000 7f3d88c00640 1 -- start start 2024-03-06T07:38:45.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.650+0000 7f3d88c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840cf5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d88c00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3d840d25b0 0x7f3d840c7970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d88c00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3d840d13e0 con 0x7f3d840cfaf0 2024-03-06T07:38:45.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d88c00640 1 -- --> v2:172.21.15.194:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d840cf120 con 0x7f3d840d25b0 2024-03-06T07:38:45.652 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d88c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d840cead0 con 0x7f3d8415e130 2024-03-06T07:38:45.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d83e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3d840d25b0 0x7f3d840c7970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d83400640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840cf5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d83e00640 1 --2- >> v2:172.21.15.194:3300/0 conn(0x7f3d840d25b0 0x7f3d840c7970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.194:3300/0 says I am v2:172.21.15.86:42286/0 (socket says 172.21.15.86:42286) 2024-03-06T07:38:45.653 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.651+0000 7f3d83e00640 1 -- 172.21.15.86:0/3706552463 learned_addr learned my addr 172.21.15.86:0/3706552463 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:45.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.652+0000 7f3d83e00640 1 -- 172.21.15.86:0/3706552463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 msgr2=0x7f3d840cf5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:45.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.652+0000 7f3d83e00640 1 --2- 172.21.15.86:0/3706552463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840cf5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.652+0000 7f3d83e00640 1 -- 172.21.15.86:0/3706552463 >> v1:172.21.15.194:6789/0 conn(0x7f3d840cfaf0 legacy=0x7f3d840d1ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.654 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.652+0000 7f3d83e00640 1 -- 172.21.15.86:0/3706552463 --> v2:172.21.15.194:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d840cedb0 con 0x7f3d840d25b0 2024-03-06T07:38:45.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.653+0000 7f3d83e00640 1 --2- 172.21.15.86:0/3706552463 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840c7970 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f3d7c009e50 tx=0x7f3d7c02bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a86a22c7fe7d1fc1 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:45.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.654+0000 7f3d82000640 1 -- 172.21.15.86:0/3706552463 <== mon.1 v2:172.21.15.194:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d7c02cb00 con 0x7f3d840d25b0 2024-03-06T07:38:45.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.654+0000 7f3d82000640 1 -- 172.21.15.86:0/3706552463 <== mon.1 v2:172.21.15.194:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d7c02cca0 con 0x7f3d840d25b0 2024-03-06T07:38:45.655 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.654+0000 7f3d82000640 1 -- 172.21.15.86:0/3706552463 <== mon.1 v2:172.21.15.194:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d7c034610 con 0x7f3d840d25b0 2024-03-06T07:38:45.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.654+0000 7f3d88c00640 1 -- 172.21.15.86:0/3706552463 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 msgr2=0x7f3d840c7970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.656 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.654+0000 7f3d88c00640 1 --2- 172.21.15.86:0/3706552463 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840c7970 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f3d7c009e50 tx=0x7f3d7c02bf60 comp rx=0 tx=0).stop 2024-03-06T07:38:45.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.655+0000 7f3d88c00640 1 -- 172.21.15.86:0/3706552463 shutdown_connections 2024-03-06T07:38:45.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.655+0000 7f3d88c00640 1 --2- 172.21.15.86:0/3706552463 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840c7970 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.655+0000 7f3d88c00640 1 --2- 172.21.15.86:0/3706552463 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840cf5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.655+0000 7f3d88c00640 1 -- 172.21.15.86:0/3706552463 >> 172.21.15.86:0/3706552463 conn(0x7f3d8409d070 msgr2=0x7f3d840d3e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:45.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.655+0000 7f3d88c00640 1 -- 172.21.15.86:0/3706552463 shutdown_connections 2024-03-06T07:38:45.657 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.655+0000 7f3d88c00640 1 -- 172.21.15.86:0/3706552463 wait complete. 2024-03-06T07:38:45.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.656+0000 7f3d88c00640 1 Processor -- start 2024-03-06T07:38:45.658 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.656+0000 7f3d88c00640 1 -- start start 2024-03-06T07:38:45.660 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.657+0000 7f3d88c00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d840cfaf0 0x7f3d8414e360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.657+0000 7f3d88c00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840b89b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.657+0000 7f3d88c00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840b8ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.657+0000 7f3d88c00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d840c8030 con 0x7f3d840cfaf0 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.657+0000 7f3d88c00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d840c7eb0 con 0x7f3d840d25b0 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.657+0000 7f3d88c00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d840c81b0 con 0x7f3d8415e130 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d840cfaf0 0x7f3d8414e360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d840cfaf0 0x7f3d8414e360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:60818/0 (socket says 172.21.15.86:60818) 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 -- 172.21.15.86:0/2084444499 learned_addr learned my addr 172.21.15.86:0/2084444499 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83e00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840b8ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d82a00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840b89b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 -- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 msgr2=0x7f3d840b8ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840b8ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 -- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 msgr2=0x7f3d840b89b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.661 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840b89b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83400640 1 -- 172.21.15.86:0/2084444499 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d840cbf30 con 0x7f3d840cfaf0 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d83e00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840b8ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.658+0000 7f3d82a00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840b89b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.659+0000 7f3d83400640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d840cfaf0 0x7f3d8414e360 secure :-1 s=READY pgs=1855 cs=0 l=1 rev1=1 crypto rx=0x7f3d7400e9b0 tx=0x7f3d7400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.659+0000 7f3d7b400640 1 -- 172.21.15.86:0/2084444499 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d74017070 con 0x7f3d840cfaf0 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.659+0000 7f3d7b400640 1 -- 172.21.15.86:0/2084444499 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d7400c9a0 con 0x7f3d840cfaf0 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.659+0000 7f3d7b400640 1 -- 172.21.15.86:0/2084444499 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d7400cd90 con 0x7f3d840cfaf0 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.659+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d840cc210 con 0x7f3d840cfaf0 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.659+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d840cb1b0 con 0x7f3d840cfaf0 2024-03-06T07:38:45.662 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.660+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d841f40f0 con 0x7f3d840cfaf0 2024-03-06T07:38:45.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.661+0000 7f3d7b400640 1 -- 172.21.15.86:0/2084444499 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3d7401a070 con 0x7f3d840cfaf0 2024-03-06T07:38:45.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.662+0000 7f3d7b400640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d6406b4d0 0x7f3d6406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:45.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.662+0000 7f3d7b400640 1 -- 172.21.15.86:0/2084444499 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3d7408a730 con 0x7f3d840cfaf0 2024-03-06T07:38:45.663 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.662+0000 7f3d82a00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d6406b4d0 0x7f3d6406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:45.664 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.663+0000 7f3d82a00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d6406b4d0 0x7f3d6406d990 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7f3d840a1d80 tx=0x7f3d70009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:45.668 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.667+0000 7f3d7b400640 1 -- 172.21.15.86:0/2084444499 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3d841f40f0 con 0x7f3d840cfaf0 2024-03-06T07:38:45.863 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.861+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f3d840d2d00 con 0x7f3d840cfaf0 2024-03-06T07:38:45.864 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.862+0000 7f3d7b400640 1 -- 172.21.15.86:0/2084444499 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v131) v1 ==== 100+0+11 (secure 0 0 0) 0x7f3d840d2d00 con 0x7f3d840cfaf0 2024-03-06T07:38:45.864 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 64 2024-03-06T07:38:45.866 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d6406b4d0 msgr2=0x7f3d6406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d6406b4d0 0x7f3d6406d990 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7f3d840a1d80 tx=0x7f3d70009040 comp rx=0 tx=0).stop 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d840cfaf0 msgr2=0x7f3d8414e360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d840cfaf0 0x7f3d8414e360 secure :-1 s=READY pgs=1855 cs=0 l=1 rev1=1 crypto rx=0x7f3d7400e9b0 tx=0x7f3d7400ee70 comp rx=0 tx=0).stop 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 shutdown_connections 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f3d6406b4d0 0x7f3d6406d990 secure :-1 s=CLOSED pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7f3d840a1d80 tx=0x7f3d70009040 comp rx=0 tx=0).stop 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f3d8415e130 0x7f3d840b8ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f3d840d25b0 0x7f3d840b89b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 --2- 172.21.15.86:0/2084444499 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f3d840cfaf0 0x7f3d8414e360 unknown :-1 s=CLOSED pgs=1855 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 >> 172.21.15.86:0/2084444499 conn(0x7f3d8409d070 msgr2=0x7f3d8406c750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 shutdown_connections 2024-03-06T07:38:45.867 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:45.865+0000 7f3d88c00640 1 -- 172.21.15.86:0/2084444499 wait complete. 2024-03-06T07:38:45.886 DEBUG:teuthology.orchestra.run.smithi086:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-06T07:38:46.081 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.080+0000 7f9570a00640 1 Processor -- start 2024-03-06T07:38:46.082 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.080+0000 7f9570a00640 1 -- start start 2024-03-06T07:38:46.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.081+0000 7f9570a00640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f956c15b070 0x7f956c06e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:46.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.081+0000 7f9570a00640 1 -- --> v1:172.21.15.86:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f956c0cc000 con 0x7f956c15f120 2024-03-06T07:38:46.083 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.081+0000 7f9570a00640 1 -- --> v1:172.21.15.194:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f956c0ce360 con 0x7f956c15ac90 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.081+0000 7f9570a00640 1 -- --> v2:172.21.15.86:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f956c0ce640 con 0x7f956c15b070 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.081+0000 7f9569400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f956c15b070 0x7f956c06e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.082+0000 7f9569400640 1 --2- >> v2:172.21.15.86:3300/0 conn(0x7f956c15b070 0x7f956c06e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35838/0 (socket says 172.21.15.86:35838) 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.082+0000 7f9569400640 1 -- 172.21.15.86:0/319013086 learned_addr learned my addr 172.21.15.86:0/319013086 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.082+0000 7f956a800640 1 --1- 172.21.15.86:0/319013086 >> v1:172.21.15.86:6789/0 conn(0x7f956c15f120 0x7f956c0c4a10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.86:6789/0 says I am v1:172.21.15.86:37418/0 (socket says 172.21.15.86:37418) 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.082+0000 7f9569400640 1 -- 172.21.15.86:0/319013086 >> v1:172.21.15.194:6789/0 conn(0x7f956c15ac90 legacy=0x7f956c15ea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.082+0000 7f9569400640 1 -- 172.21.15.86:0/319013086 >> v1:172.21.15.86:6789/0 conn(0x7f956c15f120 legacy=0x7f956c0c4a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.082+0000 7f9568a00640 1 -- 172.21.15.86:0/319013086 <== mon.0 v1:172.21.15.86:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3450649172 0 0) 0x7f956c0cc000 con 0x7f956c15f120 2024-03-06T07:38:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.082+0000 7f9569400640 1 -- 172.21.15.86:0/319013086 --> v2:172.21.15.86:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f956c0ce920 con 0x7f956c15b070 2024-03-06T07:38:46.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.083+0000 7f9569400640 1 --2- 172.21.15.86:0/319013086 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15b070 0x7f956c06e580 secure :-1 s=READY pgs=1857 cs=0 l=1 rev1=1 crypto rx=0x7f9558009870 tx=0x7f955802b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=24efaf86d41b3990 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:46.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.083+0000 7f9568a00640 1 -- 172.21.15.86:0/319013086 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9558035020 con 0x7f956c15b070 2024-03-06T07:38:46.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.083+0000 7f9568a00640 1 -- 172.21.15.86:0/319013086 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f955802d040 con 0x7f956c15b070 2024-03-06T07:38:46.085 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.083+0000 7f9568a00640 1 -- 172.21.15.86:0/319013086 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9558031a80 con 0x7f956c15b070 2024-03-06T07:38:46.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.084+0000 7f9570a00640 1 -- 172.21.15.86:0/319013086 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15b070 msgr2=0x7f956c06e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:46.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.084+0000 7f9570a00640 1 --2- 172.21.15.86:0/319013086 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15b070 0x7f956c06e580 secure :-1 s=READY pgs=1857 cs=0 l=1 rev1=1 crypto rx=0x7f9558009870 tx=0x7f955802b900 comp rx=0 tx=0).stop 2024-03-06T07:38:46.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.084+0000 7f9570a00640 1 -- 172.21.15.86:0/319013086 shutdown_connections 2024-03-06T07:38:46.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.084+0000 7f9570a00640 1 --2- 172.21.15.86:0/319013086 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15b070 0x7f956c06e580 unknown :-1 s=CLOSED pgs=1857 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:46.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.084+0000 7f9570a00640 1 -- 172.21.15.86:0/319013086 >> 172.21.15.86:0/319013086 conn(0x7f956c0cc4f0 msgr2=0x7f956c0cc900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:46.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.084+0000 7f9570a00640 1 -- 172.21.15.86:0/319013086 shutdown_connections 2024-03-06T07:38:46.086 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.084+0000 7f9570a00640 1 -- 172.21.15.86:0/319013086 wait complete. 2024-03-06T07:38:46.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.086+0000 7f9570a00640 1 Processor -- start 2024-03-06T07:38:46.087 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.086+0000 7f9570a00640 1 -- start start 2024-03-06T07:38:46.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f9570a00640 1 --2- >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f956c15ac90 0x7f956c0b2010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:46.088 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f9570a00640 1 --2- >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f956c15b070 0x7f956c14e5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:46.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f9570a00640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15f120 0x7f956c0b3b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:46.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f9570a00640 1 -- --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f956c15f5b0 con 0x7f956c15f120 2024-03-06T07:38:46.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f9570a00640 1 -- --> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f956c0ceee0 con 0x7f956c15ac90 2024-03-06T07:38:46.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f9570a00640 1 -- --> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f956c15f730 con 0x7f956c15b070 2024-03-06T07:38:46.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15f120 0x7f956c0b3b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:46.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 --2- >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15f120 0x7f956c0b3b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.86:3300/0 says I am v2:172.21.15.86:35848/0 (socket says 172.21.15.86:35848) 2024-03-06T07:38:46.089 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 -- 172.21.15.86:0/161938784 learned_addr learned my addr 172.21.15.86:0/161938784 (peer_addr_for_me v2:172.21.15.86:0/0) 2024-03-06T07:38:46.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f9569e00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f956c15ac90 0x7f956c0b2010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:46.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 -- 172.21.15.86:0/161938784 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f956c15b070 msgr2=0x7f956c14e5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:46.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f956c15b070 0x7f956c14e5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:46.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 -- 172.21.15.86:0/161938784 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f956c15ac90 msgr2=0x7f956c0b2010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:46.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f956c15ac90 0x7f956c0b2010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:46.090 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.087+0000 7f956a800640 1 -- 172.21.15.86:0/161938784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f956c14eb30 con 0x7f956c15f120 2024-03-06T07:38:46.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.088+0000 7f956a800640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15f120 0x7f956c0b3b90 secure :-1 s=READY pgs=1858 cs=0 l=1 rev1=1 crypto rx=0x7f956400c780 tx=0x7f956400cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:46.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.088+0000 7f9562000640 1 -- 172.21.15.86:0/161938784 <== mon.0 v2:172.21.15.86:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9564018070 con 0x7f956c15f120 2024-03-06T07:38:46.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.088+0000 7f9562000640 1 -- 172.21.15.86:0/161938784 <== mon.0 v2:172.21.15.86:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f956400ce30 con 0x7f956c15f120 2024-03-06T07:38:46.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.088+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f956c0b2af0 con 0x7f956c15f120 2024-03-06T07:38:46.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.088+0000 7f9562000640 1 -- 172.21.15.86:0/161938784 <== mon.0 v2:172.21.15.86:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f956400a840 con 0x7f956c15f120 2024-03-06T07:38:46.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.088+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f956c0b30b0 con 0x7f956c15f120 2024-03-06T07:38:46.091 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.089+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f956c04f330 con 0x7f956c15f120 2024-03-06T07:38:46.092 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.090+0000 7f9562000640 1 -- 172.21.15.86:0/161938784 <== mon.0 v2:172.21.15.86:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f956401b070 con 0x7f956c15f120 2024-03-06T07:38:46.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.091+0000 7f9562000640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f954c06b4d0 0x7f954c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-06T07:38:46.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.091+0000 7f9562000640 1 -- 172.21.15.86:0/161938784 <== mon.0 v2:172.21.15.86:3300/0 5 ==== osd_map(131..131 src has 1..131) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f95640897e0 con 0x7f956c15f120 2024-03-06T07:38:46.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.091+0000 7f9569e00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f954c06b4d0 0x7f954c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-06T07:38:46.093 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.092+0000 7f9569e00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f954c06b4d0 0x7f954c06d990 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7f955c003d90 tx=0x7f955c009040 comp rx=0 tx=0).ready entity=mgr.4101 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-06T07:38:46.097 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.095+0000 7f9562000640 1 -- 172.21.15.86:0/161938784 <== mon.0 v2:172.21.15.86:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9564058610 con 0x7f956c15f120 2024-03-06T07:38:46.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.304+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 --> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f956c0cd1c0 con 0x7f956c15f120 2024-03-06T07:38:46.306 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.305+0000 7f9562000640 1 -- 172.21.15.86:0/161938784 <== mon.0 v2:172.21.15.86:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v131) v1 ==== 80+0+0 (secure 0 0 0) 0x7f9564057f10 con 0x7f956c15f120 2024-03-06T07:38:46.306 INFO:teuthology.orchestra.run.smithi086.stderr:listed 0 entries 2024-03-06T07:38:46.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f954c06b4d0 msgr2=0x7f954c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:46.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f954c06b4d0 0x7f954c06d990 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7f955c003d90 tx=0x7f955c009040 comp rx=0 tx=0).stop 2024-03-06T07:38:46.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15f120 msgr2=0x7f956c0b3b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-06T07:38:46.309 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15f120 0x7f956c0b3b90 secure :-1 s=READY pgs=1858 cs=0 l=1 rev1=1 crypto rx=0x7f956400c780 tx=0x7f956400cc40 comp rx=0 tx=0).stop 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 shutdown_connections 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:6832/2881780473,v1:172.21.15.86:6833/2881780473] conn(0x7f954c06b4d0 0x7f954c06d990 unknown :-1 s=CLOSED pgs=786 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] conn(0x7f956c15f120 0x7f956c0b3b90 unknown :-1 s=CLOSED pgs=1858 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.194:3301/0,v1:172.21.15.194:6790/0] conn(0x7f956c15b070 0x7f956c14e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.307+0000 7f9570a00640 1 --2- 172.21.15.86:0/161938784 >> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] conn(0x7f956c15ac90 0x7f956c0b2010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.308+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 >> 172.21.15.86:0/161938784 conn(0x7f956c0cc4f0 msgr2=0x7f956c0c5540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.308+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 shutdown_connections 2024-03-06T07:38:46.310 INFO:teuthology.orchestra.run.smithi086.stderr:2024-03-06T07:38:46.308+0000 7f9570a00640 1 -- 172.21.15.86:0/161938784 wait complete. 2024-03-06T07:38:46.333 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-06T07:38:46.333 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-06T07:38:46.333 DEBUG:teuthology.orchestra.run.smithi086:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-06T07:38:55.831 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:38:55.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:39:25.977 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:39:25.986 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:39:56.125 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:39:56.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:40:26.261 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:40:26.269 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:40:56.433 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:40:56.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:41:26.572 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:41:26.583 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:41:56.710 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:41:56.718 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:42:26.854 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:42:26.863 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:42:57.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:42:57.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:43:27.140 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:43:27.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:43:46.340 ERROR:teuthology:Uncaught exception (Hub) Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 745, in recv_stderr out = self.in_stderr_buffer.read(nbytes, self.timeout) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/buffered_pipe.py", line 154, in read raise PipeTimeout() paramiko.buffered_pipe.PipeTimeout During handling of the above exception, another exception occurred: Traceback (most recent call last): File "src/gevent/greenlet.py", line 908, in gevent._gevent_cgreenlet.Greenlet.run File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 323, in copy_file_to copy_to_log(src, logger, capture=stream, quiet=quiet) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 276, in copy_to_log for line in f: File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 109, in __next__ line = self.readline() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 275, in readline new_data = self._read(n) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 1374, in _read return self.channel.recv_stderr(size) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 747, in recv_stderr raise socket.timeout() socket.timeout 2024-03-06T07:43:46.424 ERROR:teuthology:Uncaught exception (Hub) Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 697, in recv out = self.in_buffer.read(nbytes, self.timeout) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/buffered_pipe.py", line 154, in read raise PipeTimeout() paramiko.buffered_pipe.PipeTimeout During handling of the above exception, another exception occurred: Traceback (most recent call last): File "src/gevent/greenlet.py", line 908, in gevent._gevent_cgreenlet.Greenlet.run File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 323, in copy_file_to copy_to_log(src, logger, capture=stream, quiet=quiet) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 276, in copy_to_log for line in f: File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 109, in __next__ line = self.readline() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 275, in readline new_data = self._read(n) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 1359, in _read return self.channel.recv(size) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 699, in recv raise socket.timeout() socket.timeout 2024-03-06T07:43:57.286 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:43:57.295 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:44:27.415 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:44:27.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:44:57.561 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:44:57.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:45:27.724 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:45:27.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:45:57.955 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:45:57.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:46:28.106 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:46:28.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:46:58.240 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:46:58.248 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:47:28.377 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:47:28.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:47:58.524 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:47:58.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:48:28.703 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:48:28.714 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:48:58.910 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:48:58.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:49:29.048 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:49:29.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:49:59.202 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:49:59.211 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:50:29.355 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:50:29.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:50:59.502 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:50:59.510 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:51:29.642 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:51:29.654 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:51:59.788 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:51:59.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:52:29.933 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:52:29.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:53:00.080 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:53:00.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:53:30.233 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:53:30.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:54:00.368 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:54:00.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:54:30.509 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:54:30.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:55:00.661 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:55:00.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:55:30.804 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:55:30.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:56:00.949 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:56:00.958 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:56:31.104 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:56:31.113 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:57:01.243 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:57:01.251 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:57:31.383 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:57:31.395 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:58:01.528 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:58:01.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:58:31.674 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:58:31.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:59:01.844 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:59:01.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:59:31.990 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T07:59:31.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:00:02.138 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:00:02.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:00:32.293 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:00:32.301 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:01:02.429 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:01:02.438 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:01:32.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:01:32.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:02:02.714 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:02:02.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:02:32.869 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:02:32.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:03:03.008 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:03:03.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:03:33.145 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:03:33.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:04:03.292 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:04:03.301 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:04:33.421 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:04:33.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:05:03.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:05:03.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:05:33.730 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:05:33.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:06:03.978 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:06:03.990 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:06:34.191 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:06:34.202 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:07:04.337 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:07:04.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:07:34.475 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:07:34.482 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:08:04.627 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:08:04.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:08:35.052 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:08:35.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:09:05.202 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:09:05.210 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:09:35.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:09:35.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:10:05.495 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:10:05.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:10:35.654 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:10:35.663 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:11:05.790 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:11:05.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:11:35.931 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:11:35.941 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:12:06.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:12:06.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:12:36.269 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:12:36.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:13:06.503 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:13:06.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:13:36.724 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:13:36.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:14:06.862 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:14:06.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:14:37.046 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:14:37.055 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:15:07.365 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:15:07.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:15:37.511 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:15:37.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:16:07.692 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:16:07.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:16:37.915 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:16:37.926 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:17:08.115 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:17:08.125 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:17:38.284 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:17:38.295 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:18:08.500 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:18:08.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:18:38.672 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:18:38.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:19:08.878 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:19:08.887 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:19:39.291 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:19:39.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:20:09.445 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:20:09.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:20:39.654 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:20:39.666 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:21:09.808 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:21:09.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:21:40.009 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:21:40.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:22:10.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:22:10.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:22:40.778 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:22:40.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:23:10.934 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:23:10.943 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:23:41.132 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:23:41.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:24:11.346 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:24:11.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:24:41.488 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:24:41.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:25:11.684 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:25:11.692 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:25:41.881 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:25:41.890 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:26:12.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:26:12.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:26:42.453 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:26:42.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:27:12.646 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:27:12.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:27:42.885 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:27:42.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:28:13.081 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:28:13.090 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:28:43.283 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:28:43.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:29:13.424 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:29:13.433 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:29:43.620 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:29:43.631 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:30:13.918 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:30:13.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:30:44.072 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:30:44.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:31:14.287 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:31:14.296 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:31:44.487 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:31:44.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:32:14.676 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:32:14.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:32:44.906 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:32:44.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:33:15.096 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:33:15.110 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:33:45.274 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:33:45.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:34:15.509 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:34:15.519 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:34:45.697 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:34:45.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:34:48.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:48.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:48.391145+0000) 2024-03-06T08:34:49.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:49.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:49.391367+0000) 2024-03-06T08:34:50.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:50.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:50.391626+0000) 2024-03-06T08:34:51.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:51.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:51.391861+0000) 2024-03-06T08:34:52.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:52.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:52.392082+0000) 2024-03-06T08:34:53.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:53.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:53.392312+0000) 2024-03-06T08:34:54.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:54.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:54.392562+0000) 2024-03-06T08:34:55.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:55.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:55.392806+0000) 2024-03-06T08:34:56.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:56.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:56.393003+0000) 2024-03-06T08:34:57.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:57.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:57.393229+0000) 2024-03-06T08:34:58.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:58.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:58.393466+0000) 2024-03-06T08:34:59.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:34:59.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:34:59.393701+0000) 2024-03-06T08:35:00.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:00.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:00.393904+0000) 2024-03-06T08:35:01.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:01.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:01.394070+0000) 2024-03-06T08:35:02.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:02.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:02.394282+0000) 2024-03-06T08:35:03.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:03.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:03.394568+0000) 2024-03-06T08:35:04.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:04.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:04.394768+0000) 2024-03-06T08:35:05.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:05.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:05.395012+0000) 2024-03-06T08:35:06.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:06.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:06.395249+0000) 2024-03-06T08:35:07.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:07.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:07.395428+0000) 2024-03-06T08:35:08.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:08.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:08.395646+0000) 2024-03-06T08:35:09.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:09.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:09.395838+0000) 2024-03-06T08:35:10.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:10.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:10.396068+0000) 2024-03-06T08:35:11.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:11.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:11.396293+0000) 2024-03-06T08:35:12.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:12.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:12.396514+0000) 2024-03-06T08:35:13.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:13.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:13.396808+0000) 2024-03-06T08:35:14.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:14.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:14.397039+0000) 2024-03-06T08:35:15.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:15.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:15.397286+0000) 2024-03-06T08:35:15.910 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:35:15.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:35:16.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:16.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:16.397485+0000) 2024-03-06T08:35:17.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:17.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:17.397693+0000) 2024-03-06T08:35:18.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:18.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:18.397928+0000) 2024-03-06T08:35:19.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:19.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:19.398152+0000) 2024-03-06T08:35:20.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:20.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:20.398332+0000) 2024-03-06T08:35:21.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:21.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:21.398595+0000) 2024-03-06T08:35:22.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:22.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:22.398817+0000) 2024-03-06T08:35:23.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:23.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:23.399015+0000) 2024-03-06T08:35:24.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:24.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:24.399197+0000) 2024-03-06T08:35:25.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:25.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:25.399427+0000) 2024-03-06T08:35:26.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:26.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:26.399655+0000) 2024-03-06T08:35:27.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:27.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:27.399879+0000) 2024-03-06T08:35:28.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:28.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:28.400086+0000) 2024-03-06T08:35:29.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:29.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:29.400316+0000) 2024-03-06T08:35:30.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:30.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:30.400577+0000) 2024-03-06T08:35:31.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:31.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:31.400808+0000) 2024-03-06T08:35:32.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:32.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:32.401047+0000) 2024-03-06T08:35:33.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:33.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:33.401308+0000) 2024-03-06T08:35:34.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:34.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:34.401586+0000) 2024-03-06T08:35:35.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:35.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:35.401795+0000) 2024-03-06T08:35:36.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:36.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:36.401997+0000) 2024-03-06T08:35:37.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:37.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:37.402210+0000) 2024-03-06T08:35:38.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:38.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:38.402395+0000) 2024-03-06T08:35:39.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:39.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:39.402647+0000) 2024-03-06T08:35:40.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:40.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:40.402835+0000) 2024-03-06T08:35:41.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:41.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:41.403032+0000) 2024-03-06T08:35:42.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:42.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:42.403236+0000) 2024-03-06T08:35:43.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:43.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:43.403483+0000) 2024-03-06T08:35:44.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:44.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:44.403707+0000) 2024-03-06T08:35:45.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:45.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:45.403941+0000) 2024-03-06T08:35:46.113 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:35:46.121 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:35:46.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:46.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:46.404186+0000) 2024-03-06T08:35:47.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:47.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:47.404369+0000) 2024-03-06T08:35:48.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:48.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:48.404616+0000) 2024-03-06T08:35:49.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:49.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:49.404838+0000) 2024-03-06T08:35:50.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:50.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:50.405063+0000) 2024-03-06T08:35:51.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:51.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:51.405260+0000) 2024-03-06T08:35:52.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:52.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:52.405455+0000) 2024-03-06T08:35:53.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:53.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:53.405703+0000) 2024-03-06T08:35:54.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:54.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:54.405940+0000) 2024-03-06T08:35:55.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:55.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:55.406169+0000) 2024-03-06T08:35:56.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:56.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:56.406450+0000) 2024-03-06T08:35:57.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:57.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:57.406682+0000) 2024-03-06T08:35:58.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:58.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:58.406890+0000) 2024-03-06T08:35:59.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:35:59.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:35:59.407113+0000) 2024-03-06T08:36:00.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:00.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:00.407336+0000) 2024-03-06T08:36:01.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:01.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:01.407564+0000) 2024-03-06T08:36:02.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:02.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:02.407784+0000) 2024-03-06T08:36:03.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:03.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:03.407986+0000) 2024-03-06T08:36:04.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:04.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:04.408163+0000) 2024-03-06T08:36:05.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:05.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:05.408387+0000) 2024-03-06T08:36:06.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:06.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:06.408656+0000) 2024-03-06T08:36:07.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:07.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:07.408852+0000) 2024-03-06T08:36:08.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:08.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:08.409089+0000) 2024-03-06T08:36:09.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:09.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:09.409320+0000) 2024-03-06T08:36:10.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:10.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:10.409614+0000) 2024-03-06T08:36:11.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:11.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:11.409821+0000) 2024-03-06T08:36:12.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:12.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:12.410008+0000) 2024-03-06T08:36:13.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:13.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:13.410217+0000) 2024-03-06T08:36:14.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:14.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:14.410454+0000) 2024-03-06T08:36:15.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:15.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:15.410695+0000) 2024-03-06T08:36:16.315 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:36:16.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:36:16.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:16.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:16.410887+0000) 2024-03-06T08:36:17.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:17.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:17.411082+0000) 2024-03-06T08:36:18.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:18.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:18.411281+0000) 2024-03-06T08:36:19.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:19.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:19.411517+0000) 2024-03-06T08:36:20.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:20.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:20.411789+0000) 2024-03-06T08:36:21.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:21.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:21.412010+0000) 2024-03-06T08:36:22.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:22.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:22.412282+0000) 2024-03-06T08:36:23.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:23.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:23.412524+0000) 2024-03-06T08:36:24.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:24.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:24.412816+0000) 2024-03-06T08:36:25.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:25.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:25.413076+0000) 2024-03-06T08:36:26.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:26.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:26.413289+0000) 2024-03-06T08:36:27.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:27.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:27.413528+0000) 2024-03-06T08:36:28.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:28.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:28.413806+0000) 2024-03-06T08:36:29.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:29.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:29.414028+0000) 2024-03-06T08:36:30.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:30.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:30.414222+0000) 2024-03-06T08:36:31.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:31.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:31.414460+0000) 2024-03-06T08:36:32.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:32.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:32.414681+0000) 2024-03-06T08:36:33.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:33.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:33.414914+0000) 2024-03-06T08:36:34.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:34.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:34.415136+0000) 2024-03-06T08:36:35.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:35.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:35.415367+0000) 2024-03-06T08:36:36.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:36.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:36.415637+0000) 2024-03-06T08:36:37.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:37.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:37.415819+0000) 2024-03-06T08:36:38.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:38.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:38.416042+0000) 2024-03-06T08:36:39.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:39.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:39.416273+0000) 2024-03-06T08:36:40.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:40.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:40.416508+0000) 2024-03-06T08:36:41.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:41.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:41.416795+0000) 2024-03-06T08:36:42.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:42.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:42.417011+0000) 2024-03-06T08:36:43.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:43.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:43.417270+0000) 2024-03-06T08:36:44.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:44.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:44.417449+0000) 2024-03-06T08:36:45.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:45.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:45.417678+0000) 2024-03-06T08:36:46.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:46.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:46.417871+0000) 2024-03-06T08:36:46.546 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:36:46.554 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:36:47.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:47.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:47.418109+0000) 2024-03-06T08:36:48.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:48.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:48.418335+0000) 2024-03-06T08:36:49.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:49.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:49.418602+0000) 2024-03-06T08:36:50.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:50.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:50.418848+0000) 2024-03-06T08:36:51.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:51.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:51.419040+0000) 2024-03-06T08:36:52.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:52.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:52.419281+0000) 2024-03-06T08:36:53.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:53.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:53.419517+0000) 2024-03-06T08:36:54.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:54.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:54.419779+0000) 2024-03-06T08:36:55.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:55.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:55.420011+0000) 2024-03-06T08:36:56.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:56.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:56.420238+0000) 2024-03-06T08:36:57.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:57.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:57.420443+0000) 2024-03-06T08:36:58.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:58.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:58.420672+0000) 2024-03-06T08:36:59.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:36:59.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:36:59.420907+0000) 2024-03-06T08:37:00.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:00.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:00.421135+0000) 2024-03-06T08:37:01.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:01.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:01.421407+0000) 2024-03-06T08:37:02.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:02.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:02.421686+0000) 2024-03-06T08:37:03.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:03.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:03.421901+0000) 2024-03-06T08:37:04.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:04.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:04.422113+0000) 2024-03-06T08:37:05.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:05.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:05.422332+0000) 2024-03-06T08:37:06.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:06.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:06.422655+0000) 2024-03-06T08:37:07.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:07.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:07.422935+0000) 2024-03-06T08:37:08.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:08.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:08.423191+0000) 2024-03-06T08:37:09.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:09.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:09.423441+0000) 2024-03-06T08:37:10.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:10.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:10.423637+0000) 2024-03-06T08:37:11.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:11.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:11.423833+0000) 2024-03-06T08:37:12.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:12.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:12.424067+0000) 2024-03-06T08:37:13.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:13.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:13.424276+0000) 2024-03-06T08:37:14.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:14.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:14.424493+0000) 2024-03-06T08:37:15.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:15.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:15.424820+0000) 2024-03-06T08:37:16.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:16.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:16.425053+0000) 2024-03-06T08:37:16.742 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:37:16.750 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:37:17.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:17.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:17.425283+0000) 2024-03-06T08:37:18.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:18.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:18.425522+0000) 2024-03-06T08:37:19.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:19.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:19.425817+0000) 2024-03-06T08:37:20.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:20.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:20.426060+0000) 2024-03-06T08:37:21.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:21.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:21.426261+0000) 2024-03-06T08:37:22.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:22.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:22.426437+0000) 2024-03-06T08:37:23.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:23.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:23.426674+0000) 2024-03-06T08:37:24.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:24.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:24.426902+0000) 2024-03-06T08:37:25.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:25.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:25.427136+0000) 2024-03-06T08:37:26.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:26.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:26.427321+0000) 2024-03-06T08:37:27.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:27.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:27.427495+0000) 2024-03-06T08:37:28.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:28.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:28.427688+0000) 2024-03-06T08:37:29.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:29.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:29.427901+0000) 2024-03-06T08:37:30.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:30.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:30.428149+0000) 2024-03-06T08:37:31.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:31.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:31.428392+0000) 2024-03-06T08:37:32.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:32.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:32.428644+0000) 2024-03-06T08:37:33.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:33.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:33.428969+0000) 2024-03-06T08:37:34.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:34.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:34.429213+0000) 2024-03-06T08:37:35.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:35.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:35.429424+0000) 2024-03-06T08:37:36.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:36.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:36.429662+0000) 2024-03-06T08:37:37.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:37.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:37.429903+0000) 2024-03-06T08:37:38.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:38.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:38.430142+0000) 2024-03-06T08:37:39.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:39.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:39.430381+0000) 2024-03-06T08:37:40.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:40.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:40.430671+0000) 2024-03-06T08:37:41.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:41.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:41.430882+0000) 2024-03-06T08:37:42.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:42.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:42.431133+0000) 2024-03-06T08:37:43.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:43.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:43.431373+0000) 2024-03-06T08:37:44.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:44.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:44.431660+0000) 2024-03-06T08:37:45.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:45.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:45.431903+0000) 2024-03-06T08:37:46.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:46.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:46.432150+0000) 2024-03-06T08:37:46.881 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:37:46.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:37:47.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:47.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:47.432398+0000) 2024-03-06T08:37:48.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:48.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:48.432650+0000) 2024-03-06T08:37:49.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:49.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:49.432849+0000) 2024-03-06T08:37:50.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:50.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:50.433080+0000) 2024-03-06T08:37:51.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:51.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:51.433304+0000) 2024-03-06T08:37:52.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:52.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:52.433567+0000) 2024-03-06T08:37:53.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:53.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:53.433787+0000) 2024-03-06T08:37:54.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:54.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:54.434027+0000) 2024-03-06T08:37:55.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:55.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:55.434233+0000) 2024-03-06T08:37:56.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:56.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:56.434434+0000) 2024-03-06T08:37:57.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:57.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:57.434668+0000) 2024-03-06T08:37:58.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:58.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:58.434949+0000) 2024-03-06T08:37:59.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:37:59.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:37:59.435131+0000) 2024-03-06T08:38:00.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:00.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:00.435361+0000) 2024-03-06T08:38:01.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:01.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:01.435626+0000) 2024-03-06T08:38:02.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:02.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:02.435851+0000) 2024-03-06T08:38:03.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:03.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:03.436117+0000) 2024-03-06T08:38:04.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:04.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:04.436354+0000) 2024-03-06T08:38:05.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:05.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:05.436639+0000) 2024-03-06T08:38:06.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:06.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:06.436886+0000) 2024-03-06T08:38:07.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:07.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:07.437125+0000) 2024-03-06T08:38:08.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:08.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:08.437362+0000) 2024-03-06T08:38:09.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:09.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:09.437643+0000) 2024-03-06T08:38:10.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:10.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:10.437848+0000) 2024-03-06T08:38:11.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:11.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:11.438040+0000) 2024-03-06T08:38:12.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:12.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:12.438230+0000) 2024-03-06T08:38:13.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:13.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:13.438465+0000) 2024-03-06T08:38:14.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:14.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:14.438673+0000) 2024-03-06T08:38:15.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:15.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:15.438872+0000) 2024-03-06T08:38:16.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:16.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:16.439087+0000) 2024-03-06T08:38:17.038 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:38:17.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:38:17.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:17.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:17.439372+0000) 2024-03-06T08:38:18.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:18.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:18.439625+0000) 2024-03-06T08:38:19.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:19.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:19.439876+0000) 2024-03-06T08:38:20.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:20.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:20.440103+0000) 2024-03-06T08:38:21.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:21.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:21.440283+0000) 2024-03-06T08:38:22.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:22.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:22.440575+0000) 2024-03-06T08:38:23.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:23.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:23.440854+0000) 2024-03-06T08:38:24.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:24.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:24.441090+0000) 2024-03-06T08:38:25.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:25.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:25.441295+0000) 2024-03-06T08:38:26.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:26.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:26.441515+0000) 2024-03-06T08:38:27.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:27.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:27.441778+0000) 2024-03-06T08:38:28.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:28.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:28.441970+0000) 2024-03-06T08:38:29.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:29.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:29.442202+0000) 2024-03-06T08:38:30.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:30.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:30.442434+0000) 2024-03-06T08:38:31.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:31.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:31.442649+0000) 2024-03-06T08:38:32.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:32.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:32.442869+0000) 2024-03-06T08:38:33.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:33.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:33.443102+0000) 2024-03-06T08:38:34.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:34.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:34.443340+0000) 2024-03-06T08:38:35.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:35.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:35.443591+0000) 2024-03-06T08:38:36.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:36.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:36.443823+0000) 2024-03-06T08:38:37.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:37.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:37.444053+0000) 2024-03-06T08:38:38.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:38.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:38.444246+0000) 2024-03-06T08:38:39.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:39.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:39.444449+0000) 2024-03-06T08:38:40.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:40.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:40.444662+0000) 2024-03-06T08:38:41.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:41.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:41.444856+0000) 2024-03-06T08:38:42.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:42.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:42.445081+0000) 2024-03-06T08:38:43.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:43.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:43.445287+0000) 2024-03-06T08:38:44.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:44.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:44.445526+0000) 2024-03-06T08:38:45.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:45.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:45.445777+0000) 2024-03-06T08:38:46.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:46.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:46.446013+0000) 2024-03-06T08:38:47.243 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:38:47.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:38:47.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:47.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:47.446299+0000) 2024-03-06T08:38:48.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:48.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:48.446525+0000) 2024-03-06T08:38:49.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:49.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:49.446776+0000) 2024-03-06T08:38:50.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:50.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:50.447018+0000) 2024-03-06T08:38:51.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:51.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:51.447262+0000) 2024-03-06T08:38:52.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:52.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:52.447468+0000) 2024-03-06T08:38:53.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:53.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:53.447701+0000) 2024-03-06T08:38:54.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:54.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:54.447861+0000) 2024-03-06T08:38:55.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:55.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:55.448091+0000) 2024-03-06T08:38:56.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:56.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:56.448331+0000) 2024-03-06T08:38:57.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:57.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:57.448674+0000) 2024-03-06T08:38:58.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:58.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:58.448903+0000) 2024-03-06T08:38:59.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:38:59.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:38:59.449096+0000) 2024-03-06T08:39:00.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:00.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:00.449334+0000) 2024-03-06T08:39:01.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:01.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:01.449598+0000) 2024-03-06T08:39:02.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:02.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:02.449783+0000) 2024-03-06T08:39:03.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:03.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:03.449998+0000) 2024-03-06T08:39:04.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:04.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:04.450218+0000) 2024-03-06T08:39:05.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:05.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:05.450449+0000) 2024-03-06T08:39:06.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:06.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:06.450680+0000) 2024-03-06T08:39:07.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:07.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:07.450875+0000) 2024-03-06T08:39:08.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:08.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:08.451065+0000) 2024-03-06T08:39:09.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:09.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:09.451290+0000) 2024-03-06T08:39:10.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:10.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:10.451510+0000) 2024-03-06T08:39:11.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:11.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:11.451755+0000) 2024-03-06T08:39:12.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:12.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:12.451966+0000) 2024-03-06T08:39:13.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:13.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:13.452157+0000) 2024-03-06T08:39:14.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:14.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:14.452383+0000) 2024-03-06T08:39:15.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:15.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:15.452648+0000) 2024-03-06T08:39:16.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:16.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:16.452845+0000) 2024-03-06T08:39:17.381 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:39:17.391 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:39:17.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:17.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:17.453002+0000) 2024-03-06T08:39:18.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:18.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:18.453221+0000) 2024-03-06T08:39:19.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:19.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:19.453447+0000) 2024-03-06T08:39:20.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:20.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:20.453687+0000) 2024-03-06T08:39:21.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:21.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:21.453884+0000) 2024-03-06T08:39:22.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:22.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:22.454096+0000) 2024-03-06T08:39:23.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:23.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:23.454358+0000) 2024-03-06T08:39:24.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:24.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:24.454658+0000) 2024-03-06T08:39:25.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:25.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:25.454885+0000) 2024-03-06T08:39:26.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:26.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:26.455129+0000) 2024-03-06T08:39:27.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:27.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:27.455356+0000) 2024-03-06T08:39:28.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:28.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:28.455627+0000) 2024-03-06T08:39:29.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:29.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:29.455867+0000) 2024-03-06T08:39:30.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:30.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:30.456079+0000) 2024-03-06T08:39:31.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:31.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:31.456350+0000) 2024-03-06T08:39:32.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:32.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:32.456587+0000) 2024-03-06T08:39:33.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:33.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:33.456799+0000) 2024-03-06T08:39:34.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:34.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:34.457039+0000) 2024-03-06T08:39:35.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:35.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:35.457276+0000) 2024-03-06T08:39:36.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:36.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:36.457470+0000) 2024-03-06T08:39:37.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:37.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:37.457686+0000) 2024-03-06T08:39:38.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:38.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:38.457909+0000) 2024-03-06T08:39:39.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:39.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:39.458110+0000) 2024-03-06T08:39:40.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:40.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:40.458340+0000) 2024-03-06T08:39:41.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:41.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:41.458587+0000) 2024-03-06T08:39:42.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:42.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:42.458820+0000) 2024-03-06T08:39:43.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:43.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:43.459043+0000) 2024-03-06T08:39:44.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:44.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:44.459311+0000) 2024-03-06T08:39:45.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:45.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:45.459574+0000) 2024-03-06T08:39:46.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:46.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:46.459761+0000) 2024-03-06T08:39:47.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:47.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:47.459956+0000) 2024-03-06T08:39:47.548 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:39:47.556 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:39:48.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:48.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:48.460182+0000) 2024-03-06T08:39:49.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:49.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:49.460423+0000) 2024-03-06T08:39:50.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:50.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:50.460706+0000) 2024-03-06T08:39:51.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:51.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:51.460937+0000) 2024-03-06T08:39:52.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:52.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:52.461154+0000) 2024-03-06T08:39:53.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:53.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:53.461394+0000) 2024-03-06T08:39:54.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:54.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:54.461691+0000) 2024-03-06T08:39:55.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:55.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:55.461928+0000) 2024-03-06T08:39:56.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:56.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:56.462159+0000) 2024-03-06T08:39:57.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:57.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:57.462333+0000) 2024-03-06T08:39:58.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:58.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:58.462560+0000) 2024-03-06T08:39:59.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:39:59.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:39:59.462780+0000) 2024-03-06T08:40:00.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:00.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:00.462966+0000) 2024-03-06T08:40:01.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:01.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:01.463161+0000) 2024-03-06T08:40:02.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:02.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:02.463381+0000) 2024-03-06T08:40:03.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:03.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:03.463673+0000) 2024-03-06T08:40:04.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:04.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:04.463896+0000) 2024-03-06T08:40:05.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:05.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:05.464140+0000) 2024-03-06T08:40:06.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:06.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:06.464382+0000) 2024-03-06T08:40:07.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:07.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:07.464665+0000) 2024-03-06T08:40:08.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:08.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:08.464862+0000) 2024-03-06T08:40:09.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:09.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:09.465133+0000) 2024-03-06T08:40:10.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:10.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:10.465406+0000) 2024-03-06T08:40:11.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:11.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:11.465655+0000) 2024-03-06T08:40:12.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:12.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:12.465868+0000) 2024-03-06T08:40:13.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:13.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:13.466171+0000) 2024-03-06T08:40:14.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:14.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:14.466471+0000) 2024-03-06T08:40:15.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:15.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:15.466690+0000) 2024-03-06T08:40:16.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:16.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:16.466924+0000) 2024-03-06T08:40:17.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:17.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:17.467170+0000) 2024-03-06T08:40:17.692 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:40:17.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:40:18.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:18.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:18.467418+0000) 2024-03-06T08:40:19.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:19.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:19.467672+0000) 2024-03-06T08:40:20.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:20.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:20.467916+0000) 2024-03-06T08:40:21.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:21.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:21.468166+0000) 2024-03-06T08:40:22.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:22.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:22.468388+0000) 2024-03-06T08:40:23.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:23.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:23.468658+0000) 2024-03-06T08:40:24.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:24.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:24.468887+0000) 2024-03-06T08:40:25.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:25.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:25.469129+0000) 2024-03-06T08:40:26.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:26.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:26.469324+0000) 2024-03-06T08:40:27.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:27.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:27.469595+0000) 2024-03-06T08:40:28.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:28.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:28.469789+0000) 2024-03-06T08:40:29.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:29.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:29.470050+0000) 2024-03-06T08:40:30.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:30.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:30.470263+0000) 2024-03-06T08:40:31.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:31.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:31.470495+0000) 2024-03-06T08:40:32.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:32.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:32.470679+0000) 2024-03-06T08:40:33.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:33.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:33.470857+0000) 2024-03-06T08:40:34.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:34.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:34.471077+0000) 2024-03-06T08:40:35.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:35.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:35.471321+0000) 2024-03-06T08:40:36.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:36.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:36.471612+0000) 2024-03-06T08:40:37.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:37.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:37.471850+0000) 2024-03-06T08:40:38.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:38.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:38.472174+0000) 2024-03-06T08:40:39.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:39.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:39.472432+0000) 2024-03-06T08:40:40.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:40.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:40.472657+0000) 2024-03-06T08:40:41.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:41.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:41.472831+0000) 2024-03-06T08:40:42.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:42.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:42.472977+0000) 2024-03-06T08:40:43.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:43.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:43.473222+0000) 2024-03-06T08:40:44.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:44.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:44.473453+0000) 2024-03-06T08:40:45.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:45.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:45.473693+0000) 2024-03-06T08:40:46.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:46.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:46.473887+0000) 2024-03-06T08:40:47.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:47.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:47.474097+0000) 2024-03-06T08:40:47.901 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:40:47.909 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:40:48.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:48.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:48.474335+0000) 2024-03-06T08:40:49.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:49.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:49.474626+0000) 2024-03-06T08:40:50.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:50.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:50.474856+0000) 2024-03-06T08:40:51.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:51.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:51.475083+0000) 2024-03-06T08:40:52.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:52.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:52.475308+0000) 2024-03-06T08:40:53.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:53.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:53.475569+0000) 2024-03-06T08:40:54.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:54.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:54.475801+0000) 2024-03-06T08:40:55.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:55.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:55.476050+0000) 2024-03-06T08:40:56.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:56.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:56.476242+0000) 2024-03-06T08:40:57.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:57.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:57.476435+0000) 2024-03-06T08:40:58.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:58.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:58.476676+0000) 2024-03-06T08:40:59.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:40:59.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:40:59.476878+0000) 2024-03-06T08:41:00.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:00.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:00.477105+0000) 2024-03-06T08:41:01.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:01.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:01.477349+0000) 2024-03-06T08:41:02.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:02.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:02.477598+0000) 2024-03-06T08:41:03.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:03.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:03.477783+0000) 2024-03-06T08:41:04.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:04.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:04.478007+0000) 2024-03-06T08:41:05.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:05.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:05.478243+0000) 2024-03-06T08:41:06.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:06.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:06.478435+0000) 2024-03-06T08:41:07.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:07.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:07.478678+0000) 2024-03-06T08:41:08.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:08.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:08.478880+0000) 2024-03-06T08:41:09.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:09.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:09.479126+0000) 2024-03-06T08:41:10.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:10.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:10.479383+0000) 2024-03-06T08:41:11.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:11.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:11.479677+0000) 2024-03-06T08:41:12.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:12.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:12.479915+0000) 2024-03-06T08:41:13.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:13.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:13.480161+0000) 2024-03-06T08:41:14.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:14.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:14.480403+0000) 2024-03-06T08:41:15.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:15.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:15.480647+0000) 2024-03-06T08:41:16.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:16.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:16.480886+0000) 2024-03-06T08:41:17.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:17.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:17.481125+0000) 2024-03-06T08:41:18.098 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:41:18.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:41:18.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:18.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:18.481392+0000) 2024-03-06T08:41:19.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:19.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:19.481645+0000) 2024-03-06T08:41:20.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:20.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:20.481902+0000) 2024-03-06T08:41:21.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:21.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:21.482108+0000) 2024-03-06T08:41:22.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:22.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:22.482324+0000) 2024-03-06T08:41:23.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:23.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:23.482615+0000) 2024-03-06T08:41:24.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:24.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:24.482793+0000) 2024-03-06T08:41:25.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:25.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:25.483019+0000) 2024-03-06T08:41:26.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:26.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:26.483246+0000) 2024-03-06T08:41:27.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:27.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:27.483432+0000) 2024-03-06T08:41:28.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:28.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:28.483659+0000) 2024-03-06T08:41:29.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:29.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:29.483859+0000) 2024-03-06T08:41:30.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:30.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:30.484081+0000) 2024-03-06T08:41:31.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:31.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:31.484355+0000) 2024-03-06T08:41:32.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:32.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:32.484655+0000) 2024-03-06T08:41:33.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:33.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:33.484860+0000) 2024-03-06T08:41:34.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:34.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:34.485048+0000) 2024-03-06T08:41:35.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:35.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:35.485281+0000) 2024-03-06T08:41:36.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:36.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:36.485471+0000) 2024-03-06T08:41:37.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:37.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:37.485719+0000) 2024-03-06T08:41:38.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:38.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:38.485946+0000) 2024-03-06T08:41:39.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:39.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:39.486186+0000) 2024-03-06T08:41:40.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:40.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:40.486436+0000) 2024-03-06T08:41:41.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:41.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:41.486710+0000) 2024-03-06T08:41:42.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:42.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:42.486954+0000) 2024-03-06T08:41:43.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:43.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:43.487235+0000) 2024-03-06T08:41:44.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:44.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:44.487508+0000) 2024-03-06T08:41:45.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:45.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:45.487796+0000) 2024-03-06T08:41:46.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:46.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:46.488006+0000) 2024-03-06T08:41:47.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:47.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:47.488248+0000) 2024-03-06T08:41:48.297 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:41:48.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:41:48.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:48.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:48.488470+0000) 2024-03-06T08:41:49.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:49.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:49.488685+0000) 2024-03-06T08:41:50.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:50.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:50.488911+0000) 2024-03-06T08:41:51.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:51.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:51.489117+0000) 2024-03-06T08:41:52.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:52.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:52.489353+0000) 2024-03-06T08:41:53.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:53.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:53.489621+0000) 2024-03-06T08:41:54.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:54.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:54.489820+0000) 2024-03-06T08:41:55.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:55.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:55.490074+0000) 2024-03-06T08:41:56.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:56.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:56.490318+0000) 2024-03-06T08:41:57.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:57.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:57.490570+0000) 2024-03-06T08:41:58.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:58.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:58.490798+0000) 2024-03-06T08:41:59.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:41:59.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:41:59.491025+0000) 2024-03-06T08:42:00.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:00.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:00.491280+0000) 2024-03-06T08:42:01.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:01.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:01.491524+0000) 2024-03-06T08:42:02.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:02.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:02.491787+0000) 2024-03-06T08:42:03.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:03.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:03.492023+0000) 2024-03-06T08:42:04.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:04.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:04.492282+0000) 2024-03-06T08:42:05.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:05.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:05.492510+0000) 2024-03-06T08:42:06.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:06.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:06.492691+0000) 2024-03-06T08:42:07.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:07.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:07.492886+0000) 2024-03-06T08:42:08.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:08.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:08.493116+0000) 2024-03-06T08:42:09.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:09.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:09.493347+0000) 2024-03-06T08:42:10.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:10.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:10.493618+0000) 2024-03-06T08:42:11.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:11.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:11.493822+0000) 2024-03-06T08:42:12.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:12.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:12.494068+0000) 2024-03-06T08:42:13.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:13.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:13.494303+0000) 2024-03-06T08:42:14.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:14.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:14.494524+0000) 2024-03-06T08:42:15.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:15.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:15.494824+0000) 2024-03-06T08:42:16.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:16.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:16.495065+0000) 2024-03-06T08:42:17.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:17.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:17.495310+0000) 2024-03-06T08:42:18.476 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:42:18.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:42:18.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:18.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:18.495481+0000) 2024-03-06T08:42:19.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:19.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:19.495716+0000) 2024-03-06T08:42:20.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:20.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:20.495936+0000) 2024-03-06T08:42:21.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:21.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:21.496222+0000) 2024-03-06T08:42:22.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:22.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:22.496463+0000) 2024-03-06T08:42:23.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:23.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:23.496682+0000) 2024-03-06T08:42:24.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:24.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:24.496960+0000) 2024-03-06T08:42:25.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:25.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:25.497188+0000) 2024-03-06T08:42:26.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:26.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:26.497428+0000) 2024-03-06T08:42:27.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:27.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:27.497675+0000) 2024-03-06T08:42:28.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:28.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:28.497886+0000) 2024-03-06T08:42:29.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:29.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:29.498125+0000) 2024-03-06T08:42:30.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:30.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:30.498336+0000) 2024-03-06T08:42:31.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:31.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:31.498668+0000) 2024-03-06T08:42:32.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:32.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:32.498836+0000) 2024-03-06T08:42:33.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:33.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:33.499061+0000) 2024-03-06T08:42:34.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:34.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:34.499260+0000) 2024-03-06T08:42:35.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:35.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:35.499489+0000) 2024-03-06T08:42:36.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:36.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:36.499688+0000) 2024-03-06T08:42:37.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:37.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:37.499883+0000) 2024-03-06T08:42:38.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:38.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:38.500098+0000) 2024-03-06T08:42:39.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:39.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:39.500323+0000) 2024-03-06T08:42:40.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:40.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:40.500508+0000) 2024-03-06T08:42:41.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:41.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:41.500811+0000) 2024-03-06T08:42:42.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:42.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:42.501088+0000) 2024-03-06T08:42:43.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:43.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:43.501381+0000) 2024-03-06T08:42:44.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:44.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:44.501666+0000) 2024-03-06T08:42:45.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:45.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:45.501861+0000) 2024-03-06T08:42:46.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:46.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:46.502121+0000) 2024-03-06T08:42:47.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:47.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:47.502360+0000) 2024-03-06T08:42:48.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:48.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:48.502665+0000) 2024-03-06T08:42:48.624 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:42:48.632 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:42:49.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:49.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:49.502857+0000) 2024-03-06T08:42:50.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:50.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:50.503085+0000) 2024-03-06T08:42:51.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:51.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:51.503317+0000) 2024-03-06T08:42:52.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:52.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:52.503583+0000) 2024-03-06T08:42:53.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:53.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:53.503805+0000) 2024-03-06T08:42:54.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:54.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:54.504009+0000) 2024-03-06T08:42:55.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:55.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:55.504277+0000) 2024-03-06T08:42:56.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:56.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:56.504488+0000) 2024-03-06T08:42:57.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:57.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:57.504767+0000) 2024-03-06T08:42:58.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:58.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:58.504954+0000) 2024-03-06T08:42:59.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:42:59.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:42:59.505126+0000) 2024-03-06T08:43:00.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:00.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:00.505347+0000) 2024-03-06T08:43:01.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:01.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:01.505526+0000) 2024-03-06T08:43:02.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:02.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:02.505755+0000) 2024-03-06T08:43:03.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:03.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:03.505956+0000) 2024-03-06T08:43:04.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:04.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:04.506168+0000) 2024-03-06T08:43:05.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:05.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:05.506394+0000) 2024-03-06T08:43:06.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:06.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:06.506614+0000) 2024-03-06T08:43:07.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:07.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:07.506847+0000) 2024-03-06T08:43:08.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:08.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:08.507067+0000) 2024-03-06T08:43:09.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:09.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:09.507289+0000) 2024-03-06T08:43:10.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:10.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:10.507513+0000) 2024-03-06T08:43:11.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:11.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:11.507754+0000) 2024-03-06T08:43:12.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:12.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:12.507990+0000) 2024-03-06T08:43:13.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:13.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:13.508273+0000) 2024-03-06T08:43:14.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:14.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:14.508512+0000) 2024-03-06T08:43:15.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:15.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:15.508770+0000) 2024-03-06T08:43:16.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:16.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:16.508997+0000) 2024-03-06T08:43:17.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:17.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:17.509222+0000) 2024-03-06T08:43:18.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:18.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:18.509456+0000) 2024-03-06T08:43:18.816 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:43:18.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:43:19.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:19.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:19.509687+0000) 2024-03-06T08:43:20.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:20.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:20.509886+0000) 2024-03-06T08:43:21.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:21.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:21.510123+0000) 2024-03-06T08:43:22.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:22.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:22.510349+0000) 2024-03-06T08:43:23.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:23.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:23.510650+0000) 2024-03-06T08:43:24.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:24.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:24.510871+0000) 2024-03-06T08:43:25.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:25.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:25.511060+0000) 2024-03-06T08:43:26.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:26.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:26.511265+0000) 2024-03-06T08:43:27.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:27.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:27.511512+0000) 2024-03-06T08:43:28.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:28.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:28.511771+0000) 2024-03-06T08:43:29.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:29.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:29.511956+0000) 2024-03-06T08:43:30.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:30.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:30.512177+0000) 2024-03-06T08:43:31.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:31.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:31.512409+0000) 2024-03-06T08:43:32.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:32.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:32.512643+0000) 2024-03-06T08:43:33.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:33.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:33.512834+0000) 2024-03-06T08:43:34.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:34.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:34.513007+0000) 2024-03-06T08:43:35.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:35.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:35.513195+0000) 2024-03-06T08:43:36.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:36.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:36.513376+0000) 2024-03-06T08:43:37.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:37.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:37.513655+0000) 2024-03-06T08:43:38.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:38.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:38.513839+0000) 2024-03-06T08:43:39.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:39.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:39.514066+0000) 2024-03-06T08:43:40.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:40.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:40.514295+0000) 2024-03-06T08:43:41.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:41.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:41.514579+0000) 2024-03-06T08:43:42.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:42.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:42.514830+0000) 2024-03-06T08:43:43.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:43.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:43.515064+0000) 2024-03-06T08:43:44.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:44.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:44.515282+0000) 2024-03-06T08:43:45.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:45.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:45.515511+0000) 2024-03-06T08:43:46.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:46.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:46.515774+0000) 2024-03-06T08:43:47.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:47.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:47.516005+0000) 2024-03-06T08:43:48.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:48.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:48.516231+0000) 2024-03-06T08:43:49.011 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:43:49.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:43:49.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:49.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:49.516457+0000) 2024-03-06T08:43:50.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:50.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:50.516688+0000) 2024-03-06T08:43:51.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:51.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:51.516909+0000) 2024-03-06T08:43:52.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:52.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:52.517129+0000) 2024-03-06T08:43:53.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:53.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:53.517367+0000) 2024-03-06T08:43:54.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:54.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:54.517628+0000) 2024-03-06T08:43:55.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:55.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:55.517839+0000) 2024-03-06T08:43:56.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:56.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:56.518023+0000) 2024-03-06T08:43:57.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:57.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:57.518250+0000) 2024-03-06T08:43:58.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:58.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:58.518457+0000) 2024-03-06T08:43:59.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:43:59.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:43:59.518726+0000) 2024-03-06T08:44:00.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:00.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:00.518943+0000) 2024-03-06T08:44:01.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:01.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:01.519168+0000) 2024-03-06T08:44:02.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:02.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:02.519386+0000) 2024-03-06T08:44:03.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:03.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:03.519653+0000) 2024-03-06T08:44:04.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:04.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:04.519855+0000) 2024-03-06T08:44:05.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:05.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:05.520282+0000) 2024-03-06T08:44:06.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:06.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:06.520580+0000) 2024-03-06T08:44:07.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:07.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:07.520817+0000) 2024-03-06T08:44:08.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:08.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:08.521053+0000) 2024-03-06T08:44:09.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:09.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:09.521278+0000) 2024-03-06T08:44:10.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:10.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:10.521500+0000) 2024-03-06T08:44:11.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:11.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:11.521782+0000) 2024-03-06T08:44:12.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:12.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:12.521976+0000) 2024-03-06T08:44:13.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:13.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:13.522210+0000) 2024-03-06T08:44:14.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:14.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:14.522431+0000) 2024-03-06T08:44:15.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:15.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:15.522672+0000) 2024-03-06T08:44:16.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:16.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:16.522854+0000) 2024-03-06T08:44:17.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:17.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:17.523051+0000) 2024-03-06T08:44:18.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:18.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:18.523289+0000) 2024-03-06T08:44:19.158 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:44:19.166 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:44:19.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:19.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:19.523575+0000) 2024-03-06T08:44:20.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:20.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:20.523835+0000) 2024-03-06T08:44:21.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:21.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:21.524087+0000) 2024-03-06T08:44:22.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:22.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:22.524316+0000) 2024-03-06T08:44:23.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:23.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:23.524603+0000) 2024-03-06T08:44:24.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:24.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:24.524851+0000) 2024-03-06T08:44:25.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:25.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:25.525047+0000) 2024-03-06T08:44:26.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:26.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:26.525280+0000) 2024-03-06T08:44:27.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:27.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:27.525572+0000) 2024-03-06T08:44:28.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:28.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:28.525818+0000) 2024-03-06T08:44:29.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:29.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:29.526025+0000) 2024-03-06T08:44:30.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:30.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:30.526269+0000) 2024-03-06T08:44:31.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:31.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:31.526495+0000) 2024-03-06T08:44:32.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:32.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:32.526754+0000) 2024-03-06T08:44:33.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:33.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:33.526993+0000) 2024-03-06T08:44:34.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:34.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:34.527229+0000) 2024-03-06T08:44:35.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:35.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:35.527474+0000) 2024-03-06T08:44:36.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:36.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:36.527717+0000) 2024-03-06T08:44:37.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:37.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:37.527986+0000) 2024-03-06T08:44:38.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:38.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:38.528227+0000) 2024-03-06T08:44:39.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:39.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:39.528455+0000) 2024-03-06T08:44:40.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:40.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:40.528707+0000) 2024-03-06T08:44:41.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:41.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:41.528919+0000) 2024-03-06T08:44:42.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:42.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:42.529149+0000) 2024-03-06T08:44:43.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:43.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:43.529358+0000) 2024-03-06T08:44:44.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:44.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:44.529628+0000) 2024-03-06T08:44:45.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:45.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:45.529862+0000) 2024-03-06T08:44:46.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:46.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:46.530088+0000) 2024-03-06T08:44:47.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:47.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:47.530311+0000) 2024-03-06T08:44:48.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:48.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:48.530576+0000) 2024-03-06T08:44:49.426 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:44:49.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:44:49.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:49.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:49.530821+0000) 2024-03-06T08:44:50.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:50.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:50.531063+0000) 2024-03-06T08:44:51.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:51.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:51.531304+0000) 2024-03-06T08:44:52.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:52.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:52.531516+0000) 2024-03-06T08:44:53.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:53.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:53.531765+0000) 2024-03-06T08:44:54.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:54.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:54.532056+0000) 2024-03-06T08:44:55.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:55.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:55.532245+0000) 2024-03-06T08:44:56.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:56.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:56.532463+0000) 2024-03-06T08:44:57.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:57.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:57.532692+0000) 2024-03-06T08:44:58.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:58.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:58.532918+0000) 2024-03-06T08:44:59.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:44:59.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:44:59.533141+0000) 2024-03-06T08:45:00.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:00.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:00.533365+0000) 2024-03-06T08:45:01.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:01.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:01.533576+0000) 2024-03-06T08:45:02.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:02.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:02.533815+0000) 2024-03-06T08:45:03.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:03.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:03.534038+0000) 2024-03-06T08:45:04.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:04.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:04.534273+0000) 2024-03-06T08:45:05.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:05.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:05.534528+0000) 2024-03-06T08:45:06.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:06.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:06.534812+0000) 2024-03-06T08:45:07.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:07.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:07.535009+0000) 2024-03-06T08:45:08.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:08.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:08.535188+0000) 2024-03-06T08:45:09.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:09.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:09.535416+0000) 2024-03-06T08:45:10.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:10.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:10.535649+0000) 2024-03-06T08:45:11.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:11.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:11.535875+0000) 2024-03-06T08:45:12.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:12.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:12.536094+0000) 2024-03-06T08:45:13.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:13.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:13.536319+0000) 2024-03-06T08:45:14.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:14.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:14.536498+0000) 2024-03-06T08:45:15.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:15.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:15.536769+0000) 2024-03-06T08:45:16.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:16.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:16.536964+0000) 2024-03-06T08:45:17.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:17.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:17.537193+0000) 2024-03-06T08:45:18.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:18.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:18.537412+0000) 2024-03-06T08:45:19.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:19.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:19.537637+0000) 2024-03-06T08:45:19.629 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:45:19.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:45:20.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:20.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:20.537858+0000) 2024-03-06T08:45:21.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:21.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:21.538078+0000) 2024-03-06T08:45:22.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:22.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:22.538296+0000) 2024-03-06T08:45:23.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:23.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:23.538529+0000) 2024-03-06T08:45:24.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:24.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:24.538799+0000) 2024-03-06T08:45:25.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:25.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:25.539041+0000) 2024-03-06T08:45:26.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:26.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:26.539269+0000) 2024-03-06T08:45:27.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:27.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:27.539498+0000) 2024-03-06T08:45:28.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:28.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:28.539727+0000) 2024-03-06T08:45:29.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:29.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:29.539951+0000) 2024-03-06T08:45:30.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:30.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:30.540192+0000) 2024-03-06T08:45:31.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:31.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:31.540411+0000) 2024-03-06T08:45:32.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:32.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:32.540636+0000) 2024-03-06T08:45:33.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:33.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:33.540869+0000) 2024-03-06T08:45:34.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:34.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:34.541088+0000) 2024-03-06T08:45:35.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:35.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:35.541329+0000) 2024-03-06T08:45:36.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:36.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:36.541576+0000) 2024-03-06T08:45:37.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:37.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:37.541786+0000) 2024-03-06T08:45:38.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:38.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:38.542023+0000) 2024-03-06T08:45:39.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:39.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:39.542205+0000) 2024-03-06T08:45:40.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:40.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:40.542448+0000) 2024-03-06T08:45:41.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:41.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:41.542732+0000) 2024-03-06T08:45:42.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:42.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:42.542974+0000) 2024-03-06T08:45:43.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:43.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:43.543228+0000) 2024-03-06T08:45:44.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:44.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:44.543462+0000) 2024-03-06T08:45:45.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:45.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:45.543710+0000) 2024-03-06T08:45:46.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:46.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:46.543930+0000) 2024-03-06T08:45:47.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:47.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:47.544209+0000) 2024-03-06T08:45:48.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:48.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:48.544427+0000) 2024-03-06T08:45:49.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:49.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:49.544647+0000) 2024-03-06T08:45:49.779 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:45:49.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:45:50.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:50.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:50.544870+0000) 2024-03-06T08:45:51.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:51.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:51.545110+0000) 2024-03-06T08:45:52.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:52.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:52.545324+0000) 2024-03-06T08:45:53.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:53.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:53.545591+0000) 2024-03-06T08:45:54.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:54.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:54.545810+0000) 2024-03-06T08:45:55.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:55.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:55.546059+0000) 2024-03-06T08:45:56.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:56.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:56.546286+0000) 2024-03-06T08:45:57.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:57.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:57.546507+0000) 2024-03-06T08:45:58.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:58.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:58.546776+0000) 2024-03-06T08:45:59.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:45:59.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:45:59.546988+0000) 2024-03-06T08:46:00.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:00.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:00.547221+0000) 2024-03-06T08:46:01.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:01.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:01.547430+0000) 2024-03-06T08:46:02.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:02.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:02.547654+0000) 2024-03-06T08:46:03.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:03.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:03.547888+0000) 2024-03-06T08:46:04.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:04.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:04.548067+0000) 2024-03-06T08:46:05.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:05.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:05.548334+0000) 2024-03-06T08:46:06.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:06.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:06.548607+0000) 2024-03-06T08:46:07.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:07.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:07.548812+0000) 2024-03-06T08:46:08.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:08.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:08.549030+0000) 2024-03-06T08:46:09.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:09.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:09.549212+0000) 2024-03-06T08:46:10.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:10.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:10.549437+0000) 2024-03-06T08:46:11.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:11.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:11.549677+0000) 2024-03-06T08:46:12.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:12.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:12.549914+0000) 2024-03-06T08:46:13.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:13.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:13.550155+0000) 2024-03-06T08:46:14.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:14.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:14.550381+0000) 2024-03-06T08:46:15.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:15.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:15.550652+0000) 2024-03-06T08:46:16.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:16.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:16.550875+0000) 2024-03-06T08:46:17.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:17.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:17.551100+0000) 2024-03-06T08:46:18.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:18.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:18.551295+0000) 2024-03-06T08:46:19.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:19.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:19.551576+0000) 2024-03-06T08:46:19.934 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:46:19.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:46:20.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:20.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:20.551815+0000) 2024-03-06T08:46:21.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:21.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:21.552046+0000) 2024-03-06T08:46:22.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:22.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:22.552232+0000) 2024-03-06T08:46:23.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:23.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:23.552493+0000) 2024-03-06T08:46:24.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:24.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:24.552794+0000) 2024-03-06T08:46:25.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:25.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:25.553086+0000) 2024-03-06T08:46:26.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:26.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:26.553360+0000) 2024-03-06T08:46:27.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:27.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:27.553684+0000) 2024-03-06T08:46:28.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:28.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:28.553877+0000) 2024-03-06T08:46:29.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:29.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:29.554061+0000) 2024-03-06T08:46:30.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:30.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:30.554284+0000) 2024-03-06T08:46:31.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:31.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:31.554516+0000) 2024-03-06T08:46:32.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:32.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:32.554785+0000) 2024-03-06T08:46:33.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:33.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:33.555021+0000) 2024-03-06T08:46:34.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:34.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:34.555246+0000) 2024-03-06T08:46:35.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:35.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:35.555477+0000) 2024-03-06T08:46:36.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:36.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:36.555685+0000) 2024-03-06T08:46:37.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:37.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:37.555920+0000) 2024-03-06T08:46:38.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:38.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:38.556141+0000) 2024-03-06T08:46:39.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:39.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:39.556365+0000) 2024-03-06T08:46:40.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:40.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:40.556622+0000) 2024-03-06T08:46:41.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:41.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:41.556910+0000) 2024-03-06T08:46:42.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:42.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:42.557160+0000) 2024-03-06T08:46:43.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:43.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:43.557419+0000) 2024-03-06T08:46:44.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:44.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:44.557664+0000) 2024-03-06T08:46:45.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:45.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:45.557904+0000) 2024-03-06T08:46:46.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:46.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:46.558105+0000) 2024-03-06T08:46:47.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:47.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:47.558331+0000) 2024-03-06T08:46:48.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:48.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:48.558600+0000) 2024-03-06T08:46:49.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:49.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:49.558795+0000) 2024-03-06T08:46:50.078 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:46:50.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:46:50.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:50.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:50.559021+0000) 2024-03-06T08:46:51.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:51.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:51.559250+0000) 2024-03-06T08:46:52.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:52.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:52.559474+0000) 2024-03-06T08:46:53.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:53.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:53.559675+0000) 2024-03-06T08:46:54.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:54.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:54.559874+0000) 2024-03-06T08:46:55.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:55.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:55.560108+0000) 2024-03-06T08:46:56.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:56.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:56.560323+0000) 2024-03-06T08:46:57.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:57.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:57.560575+0000) 2024-03-06T08:46:58.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:58.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:58.560809+0000) 2024-03-06T08:46:59.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:46:59.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:46:59.560992+0000) 2024-03-06T08:47:00.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:00.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:00.561189+0000) 2024-03-06T08:47:01.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:01.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:01.561388+0000) 2024-03-06T08:47:02.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:02.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:02.561649+0000) 2024-03-06T08:47:03.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:03.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:03.561928+0000) 2024-03-06T08:47:04.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:04.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:04.562185+0000) 2024-03-06T08:47:05.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:05.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:05.562452+0000) 2024-03-06T08:47:06.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:06.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:06.562727+0000) 2024-03-06T08:47:07.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:07.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:07.563007+0000) 2024-03-06T08:47:08.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:08.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:08.563286+0000) 2024-03-06T08:47:09.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:09.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:09.563597+0000) 2024-03-06T08:47:10.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:10.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:10.563879+0000) 2024-03-06T08:47:11.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:11.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:11.564159+0000) 2024-03-06T08:47:12.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:12.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:12.564435+0000) 2024-03-06T08:47:13.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:13.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:13.564659+0000) 2024-03-06T08:47:14.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:14.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:14.564934+0000) 2024-03-06T08:47:15.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:15.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:15.565204+0000) 2024-03-06T08:47:16.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:16.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:16.565482+0000) 2024-03-06T08:47:17.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:17.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:17.565727+0000) 2024-03-06T08:47:18.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:18.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:18.565917+0000) 2024-03-06T08:47:19.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:19.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:19.566165+0000) 2024-03-06T08:47:20.223 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:47:20.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:47:20.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:20.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:20.566442+0000) 2024-03-06T08:47:21.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:21.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:21.566713+0000) 2024-03-06T08:47:22.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:22.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:22.566975+0000) 2024-03-06T08:47:23.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:23.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:23.567208+0000) 2024-03-06T08:47:24.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:24.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:24.567442+0000) 2024-03-06T08:47:25.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:25.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:25.567680+0000) 2024-03-06T08:47:26.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:26.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:26.567887+0000) 2024-03-06T08:47:27.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:27.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:27.568096+0000) 2024-03-06T08:47:28.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:28.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:28.568324+0000) 2024-03-06T08:47:29.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:29.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:29.568661+0000) 2024-03-06T08:47:30.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:30.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:30.568870+0000) 2024-03-06T08:47:31.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:31.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:31.569097+0000) 2024-03-06T08:47:32.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:32.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:32.569324+0000) 2024-03-06T08:47:33.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:33.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:33.569508+0000) 2024-03-06T08:47:34.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:34.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:34.569750+0000) 2024-03-06T08:47:35.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:35.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:35.569975+0000) 2024-03-06T08:47:36.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:36.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:36.570171+0000) 2024-03-06T08:47:37.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:37.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:37.570394+0000) 2024-03-06T08:47:38.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:38.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:38.570651+0000) 2024-03-06T08:47:39.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:39.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:39.570831+0000) 2024-03-06T08:47:40.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:40.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:40.571053+0000) 2024-03-06T08:47:41.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:41.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:41.571283+0000) 2024-03-06T08:47:42.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:42.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:42.571522+0000) 2024-03-06T08:47:43.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:43.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:43.571799+0000) 2024-03-06T08:47:44.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:44.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:44.572026+0000) 2024-03-06T08:47:45.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:45.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:45.572305+0000) 2024-03-06T08:47:46.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:46.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:46.572611+0000) 2024-03-06T08:47:47.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:47.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:47.572884+0000) 2024-03-06T08:47:48.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:48.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:48.573152+0000) 2024-03-06T08:47:49.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:49.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:49.573417+0000) 2024-03-06T08:47:50.424 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:47:50.432 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:47:50.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:50.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:50.573599+0000) 2024-03-06T08:47:51.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:51.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:51.573842+0000) 2024-03-06T08:47:52.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:52.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:52.574064+0000) 2024-03-06T08:47:53.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:53.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:53.574305+0000) 2024-03-06T08:47:54.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:54.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:54.574607+0000) 2024-03-06T08:47:55.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:55.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:55.574774+0000) 2024-03-06T08:47:56.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:56.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:56.574996+0000) 2024-03-06T08:47:57.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:57.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:57.575184+0000) 2024-03-06T08:47:58.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:58.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:58.575398+0000) 2024-03-06T08:47:59.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:47:59.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:47:59.575639+0000) 2024-03-06T08:48:00.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:00.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:00.575864+0000) 2024-03-06T08:48:01.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:01.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:01.576100+0000) 2024-03-06T08:48:02.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:02.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:02.576282+0000) 2024-03-06T08:48:03.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:03.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:03.576519+0000) 2024-03-06T08:48:04.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:04.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:04.576737+0000) 2024-03-06T08:48:05.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:05.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:05.576929+0000) 2024-03-06T08:48:06.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:06.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:06.577164+0000) 2024-03-06T08:48:07.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:07.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:07.577395+0000) 2024-03-06T08:48:08.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:08.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:08.577653+0000) 2024-03-06T08:48:09.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:09.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:09.577867+0000) 2024-03-06T08:48:10.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:10.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:10.578068+0000) 2024-03-06T08:48:11.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:11.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:11.578259+0000) 2024-03-06T08:48:12.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:12.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:12.578489+0000) 2024-03-06T08:48:13.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:13.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:13.578723+0000) 2024-03-06T08:48:14.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:14.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:14.578922+0000) 2024-03-06T08:48:15.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:15.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:15.579127+0000) 2024-03-06T08:48:16.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:16.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:16.579337+0000) 2024-03-06T08:48:17.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:17.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:17.579622+0000) 2024-03-06T08:48:18.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:18.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:18.579843+0000) 2024-03-06T08:48:19.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:19.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:19.580068+0000) 2024-03-06T08:48:20.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:20.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:20.580294+0000) 2024-03-06T08:48:20.625 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:48:20.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:48:21.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:21.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:21.580529+0000) 2024-03-06T08:48:22.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:22.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:22.580875+0000) 2024-03-06T08:48:23.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:23.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:23.581087+0000) 2024-03-06T08:48:24.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:24.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:24.581323+0000) 2024-03-06T08:48:25.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:25.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:25.581587+0000) 2024-03-06T08:48:26.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:26.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:26.581773+0000) 2024-03-06T08:48:27.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:27.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:27.582044+0000) 2024-03-06T08:48:28.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:28.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:28.582263+0000) 2024-03-06T08:48:29.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:29.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:29.582501+0000) 2024-03-06T08:48:30.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:30.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:30.582737+0000) 2024-03-06T08:48:31.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:31.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:31.582965+0000) 2024-03-06T08:48:32.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:32.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:32.583190+0000) 2024-03-06T08:48:33.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:33.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:33.583430+0000) 2024-03-06T08:48:34.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:34.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:34.583642+0000) 2024-03-06T08:48:35.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:35.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:35.583839+0000) 2024-03-06T08:48:36.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:36.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:36.584016+0000) 2024-03-06T08:48:37.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:37.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:37.584258+0000) 2024-03-06T08:48:38.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:38.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:38.584494+0000) 2024-03-06T08:48:39.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:39.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:39.584764+0000) 2024-03-06T08:48:40.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:40.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:40.584997+0000) 2024-03-06T08:48:41.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:41.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:41.585245+0000) 2024-03-06T08:48:42.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:42.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:42.585456+0000) 2024-03-06T08:48:43.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:43.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:43.585690+0000) 2024-03-06T08:48:44.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:44.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:44.585919+0000) 2024-03-06T08:48:45.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:45.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:45.586107+0000) 2024-03-06T08:48:46.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:46.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:46.586290+0000) 2024-03-06T08:48:47.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:47.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:47.586510+0000) 2024-03-06T08:48:48.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:48.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:48.586787+0000) 2024-03-06T08:48:49.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:49.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:49.587032+0000) 2024-03-06T08:48:50.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:50.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:50.587268+0000) 2024-03-06T08:48:50.817 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:48:50.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:48:51.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:51.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:51.587512+0000) 2024-03-06T08:48:52.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:52.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:52.587795+0000) 2024-03-06T08:48:53.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:53.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:53.588014+0000) 2024-03-06T08:48:54.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:54.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:54.588249+0000) 2024-03-06T08:48:55.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:55.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:55.588487+0000) 2024-03-06T08:48:56.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:56.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:56.588701+0000) 2024-03-06T08:48:57.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:57.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:57.588899+0000) 2024-03-06T08:48:58.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:58.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:58.589131+0000) 2024-03-06T08:48:59.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:48:59.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:48:59.589381+0000) 2024-03-06T08:49:00.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:00.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:00.589674+0000) 2024-03-06T08:49:01.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:01.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:01.589876+0000) 2024-03-06T08:49:02.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:02.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:02.590054+0000) 2024-03-06T08:49:03.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:03.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:03.590270+0000) 2024-03-06T08:49:04.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:04.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:04.590484+0000) 2024-03-06T08:49:05.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:05.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:05.590698+0000) 2024-03-06T08:49:06.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:06.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:06.590924+0000) 2024-03-06T08:49:07.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:07.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:07.591155+0000) 2024-03-06T08:49:08.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:08.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:08.591377+0000) 2024-03-06T08:49:09.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:09.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:09.591617+0000) 2024-03-06T08:49:10.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:10.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:10.591875+0000) 2024-03-06T08:49:11.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:11.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:11.592061+0000) 2024-03-06T08:49:12.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:12.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:12.592314+0000) 2024-03-06T08:49:13.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:13.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:13.592578+0000) 2024-03-06T08:49:14.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:14.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:14.592803+0000) 2024-03-06T08:49:15.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:15.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:15.592992+0000) 2024-03-06T08:49:16.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:16.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:16.593214+0000) 2024-03-06T08:49:17.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:17.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:17.593401+0000) 2024-03-06T08:49:18.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:18.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:18.593642+0000) 2024-03-06T08:49:19.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:19.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:19.593887+0000) 2024-03-06T08:49:20.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:20.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:20.594112+0000) 2024-03-06T08:49:21.053 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:49:21.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:49:21.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:21.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:21.594342+0000) 2024-03-06T08:49:22.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:22.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:22.594629+0000) 2024-03-06T08:49:23.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:23.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:23.594880+0000) 2024-03-06T08:49:24.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:24.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:24.595118+0000) 2024-03-06T08:49:25.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:25.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:25.595360+0000) 2024-03-06T08:49:26.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:26.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:26.595644+0000) 2024-03-06T08:49:27.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:27.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:27.595891+0000) 2024-03-06T08:49:28.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:28.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:28.596130+0000) 2024-03-06T08:49:29.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:29.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:29.596374+0000) 2024-03-06T08:49:30.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:30.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:30.596644+0000) 2024-03-06T08:49:31.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:31.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:31.596887+0000) 2024-03-06T08:49:32.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:32.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:32.597105+0000) 2024-03-06T08:49:33.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:33.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:33.597350+0000) 2024-03-06T08:49:34.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:34.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:34.597624+0000) 2024-03-06T08:49:35.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:35.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:35.597847+0000) 2024-03-06T08:49:36.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:36.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:36.598100+0000) 2024-03-06T08:49:37.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:37.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:37.598326+0000) 2024-03-06T08:49:38.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:38.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:38.598511+0000) 2024-03-06T08:49:39.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:39.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:39.598696+0000) 2024-03-06T08:49:40.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:40.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:40.598881+0000) 2024-03-06T08:49:41.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:41.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:41.599111+0000) 2024-03-06T08:49:42.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:42.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:42.599303+0000) 2024-03-06T08:49:43.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:43.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:43.599517+0000) 2024-03-06T08:49:44.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:44.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:44.599803+0000) 2024-03-06T08:49:45.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:45.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:45.600030+0000) 2024-03-06T08:49:46.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:46.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:46.600260+0000) 2024-03-06T08:49:47.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:47.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:47.600488+0000) 2024-03-06T08:49:48.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:48.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:48.600710+0000) 2024-03-06T08:49:49.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:49.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:49.600937+0000) 2024-03-06T08:49:50.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:50.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:50.601155+0000) 2024-03-06T08:49:51.207 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:49:51.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:49:51.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:51.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:51.601388+0000) 2024-03-06T08:49:52.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:52.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:52.601658+0000) 2024-03-06T08:49:53.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:53.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:53.601902+0000) 2024-03-06T08:49:54.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:54.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:54.602122+0000) 2024-03-06T08:49:55.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:55.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:55.602342+0000) 2024-03-06T08:49:56.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:56.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:56.602613+0000) 2024-03-06T08:49:57.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:57.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:57.602843+0000) 2024-03-06T08:49:58.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:58.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:58.603034+0000) 2024-03-06T08:49:59.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:49:59.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:49:59.603269+0000) 2024-03-06T08:50:00.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:00.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:00.603507+0000) 2024-03-06T08:50:01.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:01.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:01.603756+0000) 2024-03-06T08:50:02.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:02.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:02.603938+0000) 2024-03-06T08:50:03.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:03.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:03.604242+0000) 2024-03-06T08:50:04.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:04.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:04.604480+0000) 2024-03-06T08:50:05.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:05.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:05.604728+0000) 2024-03-06T08:50:06.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:06.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:06.604931+0000) 2024-03-06T08:50:07.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:07.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:07.605161+0000) 2024-03-06T08:50:08.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:08.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:08.605380+0000) 2024-03-06T08:50:09.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:09.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:09.605653+0000) 2024-03-06T08:50:10.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:10.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:10.605842+0000) 2024-03-06T08:50:11.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:11.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:11.606079+0000) 2024-03-06T08:50:12.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:12.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:12.606285+0000) 2024-03-06T08:50:13.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:13.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:13.606577+0000) 2024-03-06T08:50:14.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:14.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:14.606777+0000) 2024-03-06T08:50:15.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:15.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:15.606968+0000) 2024-03-06T08:50:16.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:16.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:16.607210+0000) 2024-03-06T08:50:17.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:17.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:17.607455+0000) 2024-03-06T08:50:18.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:18.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:18.607697+0000) 2024-03-06T08:50:19.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:19.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:19.607911+0000) 2024-03-06T08:50:20.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:20.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:20.608150+0000) 2024-03-06T08:50:21.405 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:50:21.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:50:21.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:21.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:21.608402+0000) 2024-03-06T08:50:22.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:22.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:22.608646+0000) 2024-03-06T08:50:23.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:23.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:23.608844+0000) 2024-03-06T08:50:24.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:24.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:24.609066+0000) 2024-03-06T08:50:25.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:25.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:25.609274+0000) 2024-03-06T08:50:26.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:26.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:26.609501+0000) 2024-03-06T08:50:27.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:27.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:27.609695+0000) 2024-03-06T08:50:28.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:28.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:28.609882+0000) 2024-03-06T08:50:29.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:29.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:29.610085+0000) 2024-03-06T08:50:30.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:30.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:30.610320+0000) 2024-03-06T08:50:31.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:31.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:31.610579+0000) 2024-03-06T08:50:32.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:32.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:32.610774+0000) 2024-03-06T08:50:33.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:33.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:33.610992+0000) 2024-03-06T08:50:34.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:34.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:34.611194+0000) 2024-03-06T08:50:35.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:35.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:35.611437+0000) 2024-03-06T08:50:36.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:36.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:36.611679+0000) 2024-03-06T08:50:37.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:37.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:37.611960+0000) 2024-03-06T08:50:38.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:38.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:38.612238+0000) 2024-03-06T08:50:39.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:39.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:39.612516+0000) 2024-03-06T08:50:40.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:40.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:40.612848+0000) 2024-03-06T08:50:41.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:41.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:41.613075+0000) 2024-03-06T08:50:42.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:42.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:42.613340+0000) 2024-03-06T08:50:43.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:43.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:43.613669+0000) 2024-03-06T08:50:44.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:44.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:44.613876+0000) 2024-03-06T08:50:45.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:45.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:45.614116+0000) 2024-03-06T08:50:46.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:46.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:46.614355+0000) 2024-03-06T08:50:47.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:47.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:47.614635+0000) 2024-03-06T08:50:48.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:48.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:48.614911+0000) 2024-03-06T08:50:49.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:49.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:49.615188+0000) 2024-03-06T08:50:50.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:50.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:50.615387+0000) 2024-03-06T08:50:51.610 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:50:51.618 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:50:51.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:51.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:51.615622+0000) 2024-03-06T08:50:52.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:52.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:52.615840+0000) 2024-03-06T08:50:53.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:53.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:53.616127+0000) 2024-03-06T08:50:54.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:54.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:54.616336+0000) 2024-03-06T08:50:55.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:55.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:55.616607+0000) 2024-03-06T08:50:56.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:56.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:56.616843+0000) 2024-03-06T08:50:57.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:57.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:57.617115+0000) 2024-03-06T08:50:58.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:58.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:58.617340+0000) 2024-03-06T08:50:59.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:50:59.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:50:59.617604+0000) 2024-03-06T08:51:00.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:00.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:00.617837+0000) 2024-03-06T08:51:01.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:01.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:01.618072+0000) 2024-03-06T08:51:02.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:02.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:02.618291+0000) 2024-03-06T08:51:03.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:03.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:03.618523+0000) 2024-03-06T08:51:04.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:04.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:04.618744+0000) 2024-03-06T08:51:05.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:05.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:05.618961+0000) 2024-03-06T08:51:06.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:06.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:06.619238+0000) 2024-03-06T08:51:07.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:07.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:07.619507+0000) 2024-03-06T08:51:08.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:08.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:08.619824+0000) 2024-03-06T08:51:09.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:09.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:09.620099+0000) 2024-03-06T08:51:10.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:10.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:10.620372+0000) 2024-03-06T08:51:11.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:11.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:11.620651+0000) 2024-03-06T08:51:12.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:12.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:12.620887+0000) 2024-03-06T08:51:13.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:13.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:13.621124+0000) 2024-03-06T08:51:14.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:14.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:14.621347+0000) 2024-03-06T08:51:15.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:15.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:15.621526+0000) 2024-03-06T08:51:16.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:16.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:16.621830+0000) 2024-03-06T08:51:17.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:17.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:17.622114+0000) 2024-03-06T08:51:18.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:18.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:18.622334+0000) 2024-03-06T08:51:19.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:19.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:19.622647+0000) 2024-03-06T08:51:20.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:20.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:20.622923+0000) 2024-03-06T08:51:21.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:21.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:21.623209+0000) 2024-03-06T08:51:21.757 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:51:21.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:51:22.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:22.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:22.623485+0000) 2024-03-06T08:51:23.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:23.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:23.623834+0000) 2024-03-06T08:51:24.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:24.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:24.624113+0000) 2024-03-06T08:51:25.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:25.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:25.624393+0000) 2024-03-06T08:51:26.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:26.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:26.624681+0000) 2024-03-06T08:51:27.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:27.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:27.624959+0000) 2024-03-06T08:51:28.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:28.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:28.625187+0000) 2024-03-06T08:51:29.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:29.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:29.625455+0000) 2024-03-06T08:51:30.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:30.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:30.625732+0000) 2024-03-06T08:51:31.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:31.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:31.625994+0000) 2024-03-06T08:51:32.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:32.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:32.626277+0000) 2024-03-06T08:51:33.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:33.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:33.626495+0000) 2024-03-06T08:51:34.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:34.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:34.626764+0000) 2024-03-06T08:51:35.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:35.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:35.626958+0000) 2024-03-06T08:51:36.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:36.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:36.627180+0000) 2024-03-06T08:51:37.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:37.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:37.627430+0000) 2024-03-06T08:51:38.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:38.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:38.627651+0000) 2024-03-06T08:51:39.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:39.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:39.627880+0000) 2024-03-06T08:51:40.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:40.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:40.628106+0000) 2024-03-06T08:51:41.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:41.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:41.628366+0000) 2024-03-06T08:51:42.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:42.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:42.628654+0000) 2024-03-06T08:51:43.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:43.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:43.628912+0000) 2024-03-06T08:51:44.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:44.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:44.629132+0000) 2024-03-06T08:51:45.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:45.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:45.629412+0000) 2024-03-06T08:51:46.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:46.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:46.629683+0000) 2024-03-06T08:51:47.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:47.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:47.629906+0000) 2024-03-06T08:51:48.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:48.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:48.630141+0000) 2024-03-06T08:51:49.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:49.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:49.630392+0000) 2024-03-06T08:51:50.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:50.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:50.630657+0000) 2024-03-06T08:51:51.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:51.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:51.630901+0000) 2024-03-06T08:51:51.955 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:51:51.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:51:52.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:52.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:52.631148+0000) 2024-03-06T08:51:53.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:53.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:53.631391+0000) 2024-03-06T08:51:54.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:54.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:54.631649+0000) 2024-03-06T08:51:55.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:55.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:55.631899+0000) 2024-03-06T08:51:56.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:56.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:56.632172+0000) 2024-03-06T08:51:57.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:57.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:57.632443+0000) 2024-03-06T08:51:58.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:58.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:58.632728+0000) 2024-03-06T08:51:59.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:51:59.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:51:59.633006+0000) 2024-03-06T08:52:00.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:00.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:00.633286+0000) 2024-03-06T08:52:01.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:01.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:01.633618+0000) 2024-03-06T08:52:02.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:02.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:02.633900+0000) 2024-03-06T08:52:03.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:03.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:03.634204+0000) 2024-03-06T08:52:04.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:04.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:04.634482+0000) 2024-03-06T08:52:05.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:05.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:05.634853+0000) 2024-03-06T08:52:06.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:06.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:06.635132+0000) 2024-03-06T08:52:07.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:07.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:07.635416+0000) 2024-03-06T08:52:08.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:08.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:08.635649+0000) 2024-03-06T08:52:09.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:09.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:09.635888+0000) 2024-03-06T08:52:10.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:10.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:10.636107+0000) 2024-03-06T08:52:11.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:11.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:11.636347+0000) 2024-03-06T08:52:12.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:12.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:12.636645+0000) 2024-03-06T08:52:13.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:13.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:13.636937+0000) 2024-03-06T08:52:14.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:14.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:14.637206+0000) 2024-03-06T08:52:15.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:15.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:15.637502+0000) 2024-03-06T08:52:16.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:16.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:16.637810+0000) 2024-03-06T08:52:17.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:17.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:17.638039+0000) 2024-03-06T08:52:18.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:18.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:18.638288+0000) 2024-03-06T08:52:19.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:19.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:19.638510+0000) 2024-03-06T08:52:20.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:20.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:20.638768+0000) 2024-03-06T08:52:21.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:21.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:21.638964+0000) 2024-03-06T08:52:22.144 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:52:22.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:52:22.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:22.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:22.639150+0000) 2024-03-06T08:52:23.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:23.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:23.639364+0000) 2024-03-06T08:52:24.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:24.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:24.639680+0000) 2024-03-06T08:52:25.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:25.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:25.639946+0000) 2024-03-06T08:52:26.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:26.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:26.640234+0000) 2024-03-06T08:52:27.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:27.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:27.640480+0000) 2024-03-06T08:52:28.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:28.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:28.640686+0000) 2024-03-06T08:52:29.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:29.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:29.640876+0000) 2024-03-06T08:52:30.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:30.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:30.641070+0000) 2024-03-06T08:52:31.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:31.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:31.641294+0000) 2024-03-06T08:52:32.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:32.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:32.641514+0000) 2024-03-06T08:52:33.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:33.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:33.641800+0000) 2024-03-06T08:52:34.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:34.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:34.642014+0000) 2024-03-06T08:52:35.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:35.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:35.642241+0000) 2024-03-06T08:52:36.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:36.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:36.642462+0000) 2024-03-06T08:52:37.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:37.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:37.642690+0000) 2024-03-06T08:52:38.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:38.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:38.642969+0000) 2024-03-06T08:52:39.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:39.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:39.643167+0000) 2024-03-06T08:52:40.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:40.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:40.643407+0000) 2024-03-06T08:52:41.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:41.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:41.643643+0000) 2024-03-06T08:52:42.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:42.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:42.643883+0000) 2024-03-06T08:52:43.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:43.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:43.644052+0000) 2024-03-06T08:52:44.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:44.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:44.644285+0000) 2024-03-06T08:52:45.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:45.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:45.644516+0000) 2024-03-06T08:52:46.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:46.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:46.644721+0000) 2024-03-06T08:52:47.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:47.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:47.644914+0000) 2024-03-06T08:52:48.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:48.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:48.645097+0000) 2024-03-06T08:52:49.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:49.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:49.645320+0000) 2024-03-06T08:52:50.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:50.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:50.645473+0000) 2024-03-06T08:52:51.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:51.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:51.645691+0000) 2024-03-06T08:52:52.351 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:52:52.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:52:52.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:52.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:52.645887+0000) 2024-03-06T08:52:53.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:53.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:53.646062+0000) 2024-03-06T08:52:54.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:54.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:54.646300+0000) 2024-03-06T08:52:55.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:55.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:55.646524+0000) 2024-03-06T08:52:56.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:56.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:56.646736+0000) 2024-03-06T08:52:57.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:57.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:57.646942+0000) 2024-03-06T08:52:58.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:58.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:58.647146+0000) 2024-03-06T08:52:59.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:52:59.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:52:59.647353+0000) 2024-03-06T08:53:00.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:00.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:00.647612+0000) 2024-03-06T08:53:01.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:01.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:01.647794+0000) 2024-03-06T08:53:02.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:02.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:02.647981+0000) 2024-03-06T08:53:03.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:03.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:03.648180+0000) 2024-03-06T08:53:04.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:04.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:04.648376+0000) 2024-03-06T08:53:05.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:05.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:05.648668+0000) 2024-03-06T08:53:06.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:06.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:06.648867+0000) 2024-03-06T08:53:07.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:07.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:07.649111+0000) 2024-03-06T08:53:08.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:08.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:08.649345+0000) 2024-03-06T08:53:09.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:09.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:09.649616+0000) 2024-03-06T08:53:10.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:10.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:10.649840+0000) 2024-03-06T08:53:11.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:11.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:11.650069+0000) 2024-03-06T08:53:12.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:12.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:12.650296+0000) 2024-03-06T08:53:13.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:13.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:13.650527+0000) 2024-03-06T08:53:14.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:14.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:14.650772+0000) 2024-03-06T08:53:15.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:15.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:15.650970+0000) 2024-03-06T08:53:16.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:16.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:16.651190+0000) 2024-03-06T08:53:17.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:17.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:17.651422+0000) 2024-03-06T08:53:18.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:18.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:18.651641+0000) 2024-03-06T08:53:19.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:19.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:19.651847+0000) 2024-03-06T08:53:20.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:20.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:20.652086+0000) 2024-03-06T08:53:21.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:21.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:21.652324+0000) 2024-03-06T08:53:22.565 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:53:22.573 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:53:22.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:22.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:22.652522+0000) 2024-03-06T08:53:23.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:23.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:23.652733+0000) 2024-03-06T08:53:24.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:24.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:24.652936+0000) 2024-03-06T08:53:25.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:25.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:25.653179+0000) 2024-03-06T08:53:26.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:26.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:26.653456+0000) 2024-03-06T08:53:27.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:27.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:27.653692+0000) 2024-03-06T08:53:28.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:28.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:28.653896+0000) 2024-03-06T08:53:29.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:29.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:29.654141+0000) 2024-03-06T08:53:30.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:30.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:30.654357+0000) 2024-03-06T08:53:31.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:31.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:31.654567+0000) 2024-03-06T08:53:32.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:32.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:32.654767+0000) 2024-03-06T08:53:33.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:33.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:33.654962+0000) 2024-03-06T08:53:34.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:34.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:34.655212+0000) 2024-03-06T08:53:35.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:35.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:35.655423+0000) 2024-03-06T08:53:36.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:36.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:36.655624+0000) 2024-03-06T08:53:37.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:37.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:37.655798+0000) 2024-03-06T08:53:38.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:38.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:38.656033+0000) 2024-03-06T08:53:39.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:39.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:39.656258+0000) 2024-03-06T08:53:40.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:40.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:40.656478+0000) 2024-03-06T08:53:41.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:41.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:41.656674+0000) 2024-03-06T08:53:42.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:42.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:42.656880+0000) 2024-03-06T08:53:43.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:43.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:43.657126+0000) 2024-03-06T08:53:44.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:44.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:44.657300+0000) 2024-03-06T08:53:45.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:45.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:45.657528+0000) 2024-03-06T08:53:46.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:46.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:46.657746+0000) 2024-03-06T08:53:47.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:47.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:47.657940+0000) 2024-03-06T08:53:48.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:48.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:48.658159+0000) 2024-03-06T08:53:49.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:49.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:49.658381+0000) 2024-03-06T08:53:50.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:50.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:50.658637+0000) 2024-03-06T08:53:51.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:51.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:51.658871+0000) 2024-03-06T08:53:52.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:52.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:52.659097+0000) 2024-03-06T08:53:52.779 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:53:52.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:53:53.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:53.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:53.659335+0000) 2024-03-06T08:53:54.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:54.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:54.659581+0000) 2024-03-06T08:53:55.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:55.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:55.659773+0000) 2024-03-06T08:53:56.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:56.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:56.659993+0000) 2024-03-06T08:53:57.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:57.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:57.660204+0000) 2024-03-06T08:53:58.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:58.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:58.660435+0000) 2024-03-06T08:53:59.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:53:59.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:53:59.660687+0000) 2024-03-06T08:54:00.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:00.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:00.660887+0000) 2024-03-06T08:54:01.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:01.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:01.661097+0000) 2024-03-06T08:54:02.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:02.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:02.661304+0000) 2024-03-06T08:54:03.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:03.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:03.661573+0000) 2024-03-06T08:54:04.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:04.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:04.661770+0000) 2024-03-06T08:54:05.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:05.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:05.661974+0000) 2024-03-06T08:54:06.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:06.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:06.662214+0000) 2024-03-06T08:54:07.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:07.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:07.662461+0000) 2024-03-06T08:54:08.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:08.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:08.662692+0000) 2024-03-06T08:54:09.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:09.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:09.662888+0000) 2024-03-06T08:54:10.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:10.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:10.663066+0000) 2024-03-06T08:54:11.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:11.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:11.663278+0000) 2024-03-06T08:54:12.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:12.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:12.663499+0000) 2024-03-06T08:54:13.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:13.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:13.663758+0000) 2024-03-06T08:54:14.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:14.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:14.663960+0000) 2024-03-06T08:54:15.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:15.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:15.664203+0000) 2024-03-06T08:54:16.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:16.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:16.664428+0000) 2024-03-06T08:54:17.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:17.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:17.664671+0000) 2024-03-06T08:54:18.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:18.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:18.664920+0000) 2024-03-06T08:54:19.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:19.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:19.665200+0000) 2024-03-06T08:54:20.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:20.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:20.665417+0000) 2024-03-06T08:54:21.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:21.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:21.665634+0000) 2024-03-06T08:54:22.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:22.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:22.665833+0000) 2024-03-06T08:54:22.961 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:54:22.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:54:23.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:23.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:23.666078+0000) 2024-03-06T08:54:24.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:24.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:24.666293+0000) 2024-03-06T08:54:25.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:25.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:25.666520+0000) 2024-03-06T08:54:26.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:26.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:26.666778+0000) 2024-03-06T08:54:27.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:27.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:27.667049+0000) 2024-03-06T08:54:28.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:28.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:28.667232+0000) 2024-03-06T08:54:29.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:29.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:29.667422+0000) 2024-03-06T08:54:30.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:30.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:30.667690+0000) 2024-03-06T08:54:31.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:31.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:31.667961+0000) 2024-03-06T08:54:32.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:32.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:32.668232+0000) 2024-03-06T08:54:33.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:33.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:33.668448+0000) 2024-03-06T08:54:34.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:34.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:34.668689+0000) 2024-03-06T08:54:35.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:35.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:35.668890+0000) 2024-03-06T08:54:36.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:36.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:36.669088+0000) 2024-03-06T08:54:37.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:37.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:37.669292+0000) 2024-03-06T08:54:38.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:38.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:38.669508+0000) 2024-03-06T08:54:39.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:39.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:39.669740+0000) 2024-03-06T08:54:40.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:40.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:40.669922+0000) 2024-03-06T08:54:41.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:41.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:41.670117+0000) 2024-03-06T08:54:42.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:42.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:42.670345+0000) 2024-03-06T08:54:43.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:43.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:43.670616+0000) 2024-03-06T08:54:44.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:44.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:44.670843+0000) 2024-03-06T08:54:45.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:45.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:45.671031+0000) 2024-03-06T08:54:46.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:46.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:46.671259+0000) 2024-03-06T08:54:47.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:47.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:47.671486+0000) 2024-03-06T08:54:48.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:48.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:48.671706+0000) 2024-03-06T08:54:49.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:49.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:49.671933+0000) 2024-03-06T08:54:50.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:50.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:50.672117+0000) 2024-03-06T08:54:51.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:51.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:51.672306+0000) 2024-03-06T08:54:52.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:52.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:52.672568+0000) 2024-03-06T08:54:53.096 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:54:53.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:54:53.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:53.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:53.672769+0000) 2024-03-06T08:54:54.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:54.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:54.672957+0000) 2024-03-06T08:54:55.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:55.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:55.673137+0000) 2024-03-06T08:54:56.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:56.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:56.673355+0000) 2024-03-06T08:54:57.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:57.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:57.673626+0000) 2024-03-06T08:54:58.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:58.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:58.673829+0000) 2024-03-06T08:54:59.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:54:59.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:54:59.674027+0000) 2024-03-06T08:55:00.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:00.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:00.674245+0000) 2024-03-06T08:55:01.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:01.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:01.674471+0000) 2024-03-06T08:55:02.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:02.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:02.674646+0000) 2024-03-06T08:55:03.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:03.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:03.674844+0000) 2024-03-06T08:55:04.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:04.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:04.675041+0000) 2024-03-06T08:55:05.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:05.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:05.675280+0000) 2024-03-06T08:55:06.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:06.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:06.675489+0000) 2024-03-06T08:55:07.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:07.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:07.675691+0000) 2024-03-06T08:55:08.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:08.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:08.675874+0000) 2024-03-06T08:55:09.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:09.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:09.676046+0000) 2024-03-06T08:55:10.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:10.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:10.676236+0000) 2024-03-06T08:55:11.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:11.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:11.676476+0000) 2024-03-06T08:55:12.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:12.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:12.676709+0000) 2024-03-06T08:55:13.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:13.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:13.677009+0000) 2024-03-06T08:55:14.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:14.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:14.677231+0000) 2024-03-06T08:55:15.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:15.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:15.677512+0000) 2024-03-06T08:55:16.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:16.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:16.677785+0000) 2024-03-06T08:55:17.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:17.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:17.678033+0000) 2024-03-06T08:55:18.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:18.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:18.678296+0000) 2024-03-06T08:55:19.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:19.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:19.678511+0000) 2024-03-06T08:55:20.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:20.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:20.678738+0000) 2024-03-06T08:55:21.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:21.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:21.678934+0000) 2024-03-06T08:55:22.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:22.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:22.679165+0000) 2024-03-06T08:55:23.245 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:55:23.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:55:23.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:23.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:23.679379+0000) 2024-03-06T08:55:24.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:24.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:24.679682+0000) 2024-03-06T08:55:25.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:25.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:25.679905+0000) 2024-03-06T08:55:26.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:26.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:26.680093+0000) 2024-03-06T08:55:27.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:27.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:27.680335+0000) 2024-03-06T08:55:28.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:28.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:28.680596+0000) 2024-03-06T08:55:29.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:29.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:29.680810+0000) 2024-03-06T08:55:30.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:30.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:30.681042+0000) 2024-03-06T08:55:31.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:31.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:31.681239+0000) 2024-03-06T08:55:32.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:32.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:32.681436+0000) 2024-03-06T08:55:33.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:33.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:33.681670+0000) 2024-03-06T08:55:34.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:34.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:34.681924+0000) 2024-03-06T08:55:35.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:35.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:35.682168+0000) 2024-03-06T08:55:36.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:36.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:36.682396+0000) 2024-03-06T08:55:37.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:37.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:37.682647+0000) 2024-03-06T08:55:38.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:38.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:38.682849+0000) 2024-03-06T08:55:39.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:39.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:39.683083+0000) 2024-03-06T08:55:40.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:40.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:40.683325+0000) 2024-03-06T08:55:41.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:41.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:41.683603+0000) 2024-03-06T08:55:42.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:42.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:42.683843+0000) 2024-03-06T08:55:43.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:43.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:43.684050+0000) 2024-03-06T08:55:44.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:44.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:44.684268+0000) 2024-03-06T08:55:45.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:45.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:45.684496+0000) 2024-03-06T08:55:46.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:46.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:46.684680+0000) 2024-03-06T08:55:47.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:47.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:47.684920+0000) 2024-03-06T08:55:48.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:48.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:48.685093+0000) 2024-03-06T08:55:49.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:49.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:49.685280+0000) 2024-03-06T08:55:50.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:50.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:50.685501+0000) 2024-03-06T08:55:51.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:51.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:51.685733+0000) 2024-03-06T08:55:52.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:52.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:52.685926+0000) 2024-03-06T08:55:53.390 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:55:53.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:55:53.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:53.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:53.686129+0000) 2024-03-06T08:55:54.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:54.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:54.686346+0000) 2024-03-06T08:55:55.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:55.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:55.686571+0000) 2024-03-06T08:55:56.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:56.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:56.686770+0000) 2024-03-06T08:55:57.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:57.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:57.686975+0000) 2024-03-06T08:55:58.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:58.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:58.687156+0000) 2024-03-06T08:55:59.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:55:59.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:55:59.687342+0000) 2024-03-06T08:56:00.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:00.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:00.687597+0000) 2024-03-06T08:56:01.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:01.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:01.687838+0000) 2024-03-06T08:56:02.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:02.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:02.688025+0000) 2024-03-06T08:56:03.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:03.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:03.688256+0000) 2024-03-06T08:56:04.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:04.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:04.688473+0000) 2024-03-06T08:56:05.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:05.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:05.688684+0000) 2024-03-06T08:56:06.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:06.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:06.688870+0000) 2024-03-06T08:56:07.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:07.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:07.689077+0000) 2024-03-06T08:56:08.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:08.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:08.689250+0000) 2024-03-06T08:56:09.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:09.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:09.689470+0000) 2024-03-06T08:56:10.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:10.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:10.689678+0000) 2024-03-06T08:56:11.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:11.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:11.689874+0000) 2024-03-06T08:56:12.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:12.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:12.690063+0000) 2024-03-06T08:56:13.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:13.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:13.690307+0000) 2024-03-06T08:56:14.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:14.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:14.690474+0000) 2024-03-06T08:56:15.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:15.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:15.690676+0000) 2024-03-06T08:56:16.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:16.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:16.690853+0000) 2024-03-06T08:56:17.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:17.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:17.691052+0000) 2024-03-06T08:56:18.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:18.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:18.691269+0000) 2024-03-06T08:56:19.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:19.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:19.691495+0000) 2024-03-06T08:56:20.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:20.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:20.691679+0000) 2024-03-06T08:56:21.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:21.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:21.691865+0000) 2024-03-06T08:56:22.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:22.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:22.692098+0000) 2024-03-06T08:56:23.608 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:56:23.620 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:56:23.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:23.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:23.692286+0000) 2024-03-06T08:56:24.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:24.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:24.692495+0000) 2024-03-06T08:56:25.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:25.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:25.692686+0000) 2024-03-06T08:56:26.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:26.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:26.692897+0000) 2024-03-06T08:56:27.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:27.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:27.693092+0000) 2024-03-06T08:56:28.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:28.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:28.693315+0000) 2024-03-06T08:56:29.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:29.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:29.693494+0000) 2024-03-06T08:56:30.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:30.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:30.693692+0000) 2024-03-06T08:56:31.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:31.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:31.693882+0000) 2024-03-06T08:56:32.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:32.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:32.694101+0000) 2024-03-06T08:56:33.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:33.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:33.694337+0000) 2024-03-06T08:56:34.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:34.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:34.694594+0000) 2024-03-06T08:56:35.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:35.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:35.694772+0000) 2024-03-06T08:56:36.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:36.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:36.694987+0000) 2024-03-06T08:56:37.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:37.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:37.695258+0000) 2024-03-06T08:56:38.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:38.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:38.695452+0000) 2024-03-06T08:56:39.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:39.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:39.695686+0000) 2024-03-06T08:56:40.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:40.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:40.695908+0000) 2024-03-06T08:56:41.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:41.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:41.696103+0000) 2024-03-06T08:56:42.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:42.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:42.696343+0000) 2024-03-06T08:56:43.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:43.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:43.696610+0000) 2024-03-06T08:56:44.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:44.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:44.696835+0000) 2024-03-06T08:56:45.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:45.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:45.697065+0000) 2024-03-06T08:56:46.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:46.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:46.697280+0000) 2024-03-06T08:56:47.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:47.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:47.697486+0000) 2024-03-06T08:56:48.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:48.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:48.697702+0000) 2024-03-06T08:56:49.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:49.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:49.697924+0000) 2024-03-06T08:56:50.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:50.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:50.698140+0000) 2024-03-06T08:56:51.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:51.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:51.698340+0000) 2024-03-06T08:56:52.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:52.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:52.698633+0000) 2024-03-06T08:56:53.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:53.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:53.698872+0000) 2024-03-06T08:56:53.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:56:53.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:56:54.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:54.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:54.699144+0000) 2024-03-06T08:56:55.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:55.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:55.699336+0000) 2024-03-06T08:56:56.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:56.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:56.699608+0000) 2024-03-06T08:56:57.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:57.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:57.699857+0000) 2024-03-06T08:56:58.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:58.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:58.700029+0000) 2024-03-06T08:56:59.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:56:59.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:56:59.700216+0000) 2024-03-06T08:57:00.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:00.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:00.700409+0000) 2024-03-06T08:57:01.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:01.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:01.700627+0000) 2024-03-06T08:57:02.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:02.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:02.700841+0000) 2024-03-06T08:57:03.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:03.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:03.701059+0000) 2024-03-06T08:57:04.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:04.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:04.701292+0000) 2024-03-06T08:57:05.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:05.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:05.701493+0000) 2024-03-06T08:57:06.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:06.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:06.701678+0000) 2024-03-06T08:57:07.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:07.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:07.701932+0000) 2024-03-06T08:57:08.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:08.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:08.702156+0000) 2024-03-06T08:57:09.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:09.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:09.702373+0000) 2024-03-06T08:57:10.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:10.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:10.702634+0000) 2024-03-06T08:57:11.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:11.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:11.702790+0000) 2024-03-06T08:57:12.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:12.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:12.703007+0000) 2024-03-06T08:57:13.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:13.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:13.703268+0000) 2024-03-06T08:57:14.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:14.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:14.703502+0000) 2024-03-06T08:57:15.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:15.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:15.703743+0000) 2024-03-06T08:57:16.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:16.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:16.703945+0000) 2024-03-06T08:57:17.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:17.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:17.704171+0000) 2024-03-06T08:57:18.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:18.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:18.704404+0000) 2024-03-06T08:57:19.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:19.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:19.704648+0000) 2024-03-06T08:57:20.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:20.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:20.704843+0000) 2024-03-06T08:57:21.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:21.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:21.705061+0000) 2024-03-06T08:57:22.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:22.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:22.705302+0000) 2024-03-06T08:57:23.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:23.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:23.705498+0000) 2024-03-06T08:57:24.056 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:57:24.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:57:24.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:24.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:24.705734+0000) 2024-03-06T08:57:25.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:25.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:25.705972+0000) 2024-03-06T08:57:26.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:26.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:26.706202+0000) 2024-03-06T08:57:27.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:27.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:27.706465+0000) 2024-03-06T08:57:28.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:28.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:28.706676+0000) 2024-03-06T08:57:29.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:29.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:29.706886+0000) 2024-03-06T08:57:30.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:30.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:30.707078+0000) 2024-03-06T08:57:31.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:31.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:31.707325+0000) 2024-03-06T08:57:32.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:32.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:32.707569+0000) 2024-03-06T08:57:33.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:33.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:33.707796+0000) 2024-03-06T08:57:34.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:34.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:34.707996+0000) 2024-03-06T08:57:35.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:35.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:35.708247+0000) 2024-03-06T08:57:36.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:36.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:36.708461+0000) 2024-03-06T08:57:37.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:37.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:37.708702+0000) 2024-03-06T08:57:38.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:38.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:38.708881+0000) 2024-03-06T08:57:39.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:39.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:39.709104+0000) 2024-03-06T08:57:40.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:40.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:40.709329+0000) 2024-03-06T08:57:41.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:41.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:41.709574+0000) 2024-03-06T08:57:42.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:42.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:42.709827+0000) 2024-03-06T08:57:43.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:43.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:43.710020+0000) 2024-03-06T08:57:44.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:44.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:44.710250+0000) 2024-03-06T08:57:45.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:45.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:45.710471+0000) 2024-03-06T08:57:46.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:46.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:46.710657+0000) 2024-03-06T08:57:47.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:47.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:47.710871+0000) 2024-03-06T08:57:48.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:48.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:48.711098+0000) 2024-03-06T08:57:49.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:49.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:49.711347+0000) 2024-03-06T08:57:50.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:50.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:50.711594+0000) 2024-03-06T08:57:51.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:51.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:51.711784+0000) 2024-03-06T08:57:52.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:52.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:52.712017+0000) 2024-03-06T08:57:53.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:53.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:53.712263+0000) 2024-03-06T08:57:54.193 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:57:54.201 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:57:54.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:54.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:54.712486+0000) 2024-03-06T08:57:55.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:55.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:55.712684+0000) 2024-03-06T08:57:56.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:56.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:56.712878+0000) 2024-03-06T08:57:57.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:57.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:57.713094+0000) 2024-03-06T08:57:58.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:58.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:58.713344+0000) 2024-03-06T08:57:59.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:57:59.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:57:59.713630+0000) 2024-03-06T08:58:00.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:00.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:00.713821+0000) 2024-03-06T08:58:01.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:01.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:01.714008+0000) 2024-03-06T08:58:02.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:02.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:02.714238+0000) 2024-03-06T08:58:03.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:03.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:03.714516+0000) 2024-03-06T08:58:04.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:04.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:04.714766+0000) 2024-03-06T08:58:05.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:05.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:05.714960+0000) 2024-03-06T08:58:06.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:06.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:06.715143+0000) 2024-03-06T08:58:07.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:07.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:07.715342+0000) 2024-03-06T08:58:08.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:08.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:08.715595+0000) 2024-03-06T08:58:09.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:09.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:09.715786+0000) 2024-03-06T08:58:10.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:10.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:10.715960+0000) 2024-03-06T08:58:11.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:11.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:11.716200+0000) 2024-03-06T08:58:12.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:12.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:12.716437+0000) 2024-03-06T08:58:13.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:13.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:13.716704+0000) 2024-03-06T08:58:14.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:14.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:14.716881+0000) 2024-03-06T08:58:15.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:15.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:15.717097+0000) 2024-03-06T08:58:16.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:16.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:16.717328+0000) 2024-03-06T08:58:17.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:17.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:17.717578+0000) 2024-03-06T08:58:18.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:18.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:18.717776+0000) 2024-03-06T08:58:19.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:19.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:19.717980+0000) 2024-03-06T08:58:20.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:20.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:20.718202+0000) 2024-03-06T08:58:21.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:21.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:21.718422+0000) 2024-03-06T08:58:22.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:22.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:22.718664+0000) 2024-03-06T08:58:23.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:23.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:23.718860+0000) 2024-03-06T08:58:24.382 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:58:24.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:58:24.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:24.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:24.719083+0000) 2024-03-06T08:58:25.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:25.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:25.719312+0000) 2024-03-06T08:58:26.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:26.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:26.719528+0000) 2024-03-06T08:58:27.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:27.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:27.719744+0000) 2024-03-06T08:58:28.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:28.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:28.719927+0000) 2024-03-06T08:58:29.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:29.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:29.720150+0000) 2024-03-06T08:58:30.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:30.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:30.720372+0000) 2024-03-06T08:58:31.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:31.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:31.720622+0000) 2024-03-06T08:58:32.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:32.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:32.720803+0000) 2024-03-06T08:58:33.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:33.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:33.721004+0000) 2024-03-06T08:58:34.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:34.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:34.721222+0000) 2024-03-06T08:58:35.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:35.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:35.721501+0000) 2024-03-06T08:58:36.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:36.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:36.721730+0000) 2024-03-06T08:58:37.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:37.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:37.721924+0000) 2024-03-06T08:58:38.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:38.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:38.722112+0000) 2024-03-06T08:58:39.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:39.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:39.722328+0000) 2024-03-06T08:58:40.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:40.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:40.722572+0000) 2024-03-06T08:58:41.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:41.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:41.722765+0000) 2024-03-06T08:58:42.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:42.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:42.722943+0000) 2024-03-06T08:58:43.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:43.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:43.723151+0000) 2024-03-06T08:58:44.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:44.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:44.723369+0000) 2024-03-06T08:58:45.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:45.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:45.723619+0000) 2024-03-06T08:58:46.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:46.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:46.723803+0000) 2024-03-06T08:58:47.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:47.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:47.724010+0000) 2024-03-06T08:58:48.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:48.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:48.724245+0000) 2024-03-06T08:58:49.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:49.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:49.724473+0000) 2024-03-06T08:58:50.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:50.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:50.724693+0000) 2024-03-06T08:58:51.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:51.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:51.724882+0000) 2024-03-06T08:58:52.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:52.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:52.725101+0000) 2024-03-06T08:58:53.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:53.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:53.725354+0000) 2024-03-06T08:58:54.579 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:58:54.590 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:58:54.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:54.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:54.725624+0000) 2024-03-06T08:58:55.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:55.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:55.725847+0000) 2024-03-06T08:58:56.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:56.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:56.726058+0000) 2024-03-06T08:58:57.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:57.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:57.726288+0000) 2024-03-06T08:58:58.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:58.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:58.726509+0000) 2024-03-06T08:58:59.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:58:59.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:58:59.726715+0000) 2024-03-06T08:59:00.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:00.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:00.726939+0000) 2024-03-06T08:59:01.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:01.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:01.727125+0000) 2024-03-06T08:59:02.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:02.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:02.727311+0000) 2024-03-06T08:59:03.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:03.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:03.727603+0000) 2024-03-06T08:59:04.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:04.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:04.727820+0000) 2024-03-06T08:59:05.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:05.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:05.728049+0000) 2024-03-06T08:59:06.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:06.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:06.728265+0000) 2024-03-06T08:59:07.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:07.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:07.728486+0000) 2024-03-06T08:59:08.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:08.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:08.728674+0000) 2024-03-06T08:59:09.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:09.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:09.728861+0000) 2024-03-06T08:59:10.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:10.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:10.729082+0000) 2024-03-06T08:59:11.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:11.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:11.729306+0000) 2024-03-06T08:59:12.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:12.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:12.729473+0000) 2024-03-06T08:59:13.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:13.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:13.729687+0000) 2024-03-06T08:59:14.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:14.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:14.729900+0000) 2024-03-06T08:59:15.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:15.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:15.730084+0000) 2024-03-06T08:59:16.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:16.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:16.730307+0000) 2024-03-06T08:59:17.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:17.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:17.730566+0000) 2024-03-06T08:59:18.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:18.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:18.730792+0000) 2024-03-06T08:59:19.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:19.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:19.731014+0000) 2024-03-06T08:59:20.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:20.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:20.731192+0000) 2024-03-06T08:59:21.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:21.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:21.731379+0000) 2024-03-06T08:59:22.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:22.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:22.731582+0000) 2024-03-06T08:59:23.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:23.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:23.731784+0000) 2024-03-06T08:59:24.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:24.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:24.731975+0000) 2024-03-06T08:59:24.776 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:59:24.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:59:25.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:25.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:25.732185+0000) 2024-03-06T08:59:26.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:26.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:26.732362+0000) 2024-03-06T08:59:27.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:27.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:27.732581+0000) 2024-03-06T08:59:28.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:28.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:28.732804+0000) 2024-03-06T08:59:29.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:29.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:29.733108+0000) 2024-03-06T08:59:30.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:30.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:30.733325+0000) 2024-03-06T08:59:31.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:31.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:31.733571+0000) 2024-03-06T08:59:32.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:32.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:32.733813+0000) 2024-03-06T08:59:33.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:33.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:33.734052+0000) 2024-03-06T08:59:34.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:34.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:34.734315+0000) 2024-03-06T08:59:35.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:35.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:35.734596+0000) 2024-03-06T08:59:36.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:36.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:36.734783+0000) 2024-03-06T08:59:37.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:37.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:37.734979+0000) 2024-03-06T08:59:38.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:38.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:38.735171+0000) 2024-03-06T08:59:39.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:39.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:39.735373+0000) 2024-03-06T08:59:40.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:40.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:40.735637+0000) 2024-03-06T08:59:41.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:41.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:41.735830+0000) 2024-03-06T08:59:42.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:42.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:42.736100+0000) 2024-03-06T08:59:43.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:43.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:43.736332+0000) 2024-03-06T08:59:44.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:44.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:44.736567+0000) 2024-03-06T08:59:45.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:45.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:45.736735+0000) 2024-03-06T08:59:46.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:46.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:46.736935+0000) 2024-03-06T08:59:47.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:47.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:47.737137+0000) 2024-03-06T08:59:48.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:48.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:48.737379+0000) 2024-03-06T08:59:49.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:49.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:49.737621+0000) 2024-03-06T08:59:50.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:50.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:50.737805+0000) 2024-03-06T08:59:51.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:51.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:51.738014+0000) 2024-03-06T08:59:52.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:52.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:52.738207+0000) 2024-03-06T08:59:53.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:53.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:53.738461+0000) 2024-03-06T08:59:54.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:54.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:54.738681+0000) 2024-03-06T08:59:54.992 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:59:55.002 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T08:59:55.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:55.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:55.738892+0000) 2024-03-06T08:59:56.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:56.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:56.739117+0000) 2024-03-06T08:59:57.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:57.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:57.739340+0000) 2024-03-06T08:59:58.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:58.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:58.739595+0000) 2024-03-06T08:59:59.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T08:59:59.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T07:59:59.739820+0000) 2024-03-06T09:00:00.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:00.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:00.740019+0000) 2024-03-06T09:00:01.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:01.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:01.740242+0000) 2024-03-06T09:00:02.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:02.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:02.740461+0000) 2024-03-06T09:00:03.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:03.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:03.740699+0000) 2024-03-06T09:00:04.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:04.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:04.740888+0000) 2024-03-06T09:00:05.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:05.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:05.741080+0000) 2024-03-06T09:00:06.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:06.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:06.741307+0000) 2024-03-06T09:00:07.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:07.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:07.741577+0000) 2024-03-06T09:00:08.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:08.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:08.741792+0000) 2024-03-06T09:00:09.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:09.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:09.741991+0000) 2024-03-06T09:00:10.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:10.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:10.742208+0000) 2024-03-06T09:00:11.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:11.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:11.742430+0000) 2024-03-06T09:00:12.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:12.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:12.742652+0000) 2024-03-06T09:00:13.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:13.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:13.742890+0000) 2024-03-06T09:00:14.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:14.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:14.743076+0000) 2024-03-06T09:00:15.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:15.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:15.743297+0000) 2024-03-06T09:00:16.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:16.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:16.743470+0000) 2024-03-06T09:00:17.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:17.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:17.743898+0000) 2024-03-06T09:00:18.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:18.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:18.744071+0000) 2024-03-06T09:00:19.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:19.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:19.744303+0000) 2024-03-06T09:00:20.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:20.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:20.744583+0000) 2024-03-06T09:00:21.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:21.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:21.744851+0000) 2024-03-06T09:00:22.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:22.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:22.745118+0000) 2024-03-06T09:00:23.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:23.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:23.745341+0000) 2024-03-06T09:00:24.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:24.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:24.745575+0000) 2024-03-06T09:00:25.126 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:00:25.134 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:00:25.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:25.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:25.745815+0000) 2024-03-06T09:00:26.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:26.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:26.746037+0000) 2024-03-06T09:00:27.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:27.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:27.746224+0000) 2024-03-06T09:00:28.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:28.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:28.746464+0000) 2024-03-06T09:00:29.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:29.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:29.746683+0000) 2024-03-06T09:00:30.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:30.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:30.746892+0000) 2024-03-06T09:00:31.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:31.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:31.747096+0000) 2024-03-06T09:00:32.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:32.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:32.747326+0000) 2024-03-06T09:00:33.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:33.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:33.747575+0000) 2024-03-06T09:00:34.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:34.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:34.747783+0000) 2024-03-06T09:00:35.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:35.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:35.748016+0000) 2024-03-06T09:00:36.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:36.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:36.748257+0000) 2024-03-06T09:00:37.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:37.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:37.748478+0000) 2024-03-06T09:00:38.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:38.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:38.748704+0000) 2024-03-06T09:00:39.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:39.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:39.748893+0000) 2024-03-06T09:00:40.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:40.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:40.749087+0000) 2024-03-06T09:00:41.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:41.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:41.749323+0000) 2024-03-06T09:00:42.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:42.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:42.749529+0000) 2024-03-06T09:00:43.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:43.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:43.749815+0000) 2024-03-06T09:00:44.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:44.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:44.750000+0000) 2024-03-06T09:00:45.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:45.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:45.750225+0000) 2024-03-06T09:00:46.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:46.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:46.750423+0000) 2024-03-06T09:00:47.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:47.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:47.750669+0000) 2024-03-06T09:00:48.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:48.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:48.750868+0000) 2024-03-06T09:00:49.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:49.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:49.751072+0000) 2024-03-06T09:00:50.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:50.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:50.751298+0000) 2024-03-06T09:00:51.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:51.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:51.751527+0000) 2024-03-06T09:00:52.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:52.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:52.751791+0000) 2024-03-06T09:00:53.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:53.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:53.752041+0000) 2024-03-06T09:00:54.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:54.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:54.752287+0000) 2024-03-06T09:00:55.298 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:00:55.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:00:55.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:55.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:55.752587+0000) 2024-03-06T09:00:56.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:56.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:56.752836+0000) 2024-03-06T09:00:57.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:57.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:57.753064+0000) 2024-03-06T09:00:58.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:58.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:58.753290+0000) 2024-03-06T09:00:59.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:00:59.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:00:59.753503+0000) 2024-03-06T09:01:00.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:00.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:00.753753+0000) 2024-03-06T09:01:01.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:01.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:01.753951+0000) 2024-03-06T09:01:02.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:02.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:02.754144+0000) 2024-03-06T09:01:03.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:03.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:03.754386+0000) 2024-03-06T09:01:04.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:04.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:04.754611+0000) 2024-03-06T09:01:05.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:05.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:05.754858+0000) 2024-03-06T09:01:06.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:06.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:06.755061+0000) 2024-03-06T09:01:07.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:07.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:07.755290+0000) 2024-03-06T09:01:08.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:08.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:08.755487+0000) 2024-03-06T09:01:09.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:09.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:09.755743+0000) 2024-03-06T09:01:10.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:10.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:10.756049+0000) 2024-03-06T09:01:11.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:11.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:11.756279+0000) 2024-03-06T09:01:12.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:12.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:12.756448+0000) 2024-03-06T09:01:13.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:13.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:13.756705+0000) 2024-03-06T09:01:14.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:14.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:14.756931+0000) 2024-03-06T09:01:15.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:15.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:15.757155+0000) 2024-03-06T09:01:16.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:16.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:16.757391+0000) 2024-03-06T09:01:17.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:17.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:17.757621+0000) 2024-03-06T09:01:18.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:18.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:18.757812+0000) 2024-03-06T09:01:19.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:19.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:19.758007+0000) 2024-03-06T09:01:20.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:20.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:20.758238+0000) 2024-03-06T09:01:21.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:21.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:21.758420+0000) 2024-03-06T09:01:22.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:22.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:22.758646+0000) 2024-03-06T09:01:23.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:23.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:23.758887+0000) 2024-03-06T09:01:24.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:24.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:24.759111+0000) 2024-03-06T09:01:25.434 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:01:25.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:01:25.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:25.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:25.759350+0000) 2024-03-06T09:01:26.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:26.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:26.759610+0000) 2024-03-06T09:01:27.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:27.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:27.760011+0000) 2024-03-06T09:01:28.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:28.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:28.760241+0000) 2024-03-06T09:01:29.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:29.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:29.760465+0000) 2024-03-06T09:01:30.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:30.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:30.760683+0000) 2024-03-06T09:01:31.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:31.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:31.761366+0000) 2024-03-06T09:01:32.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:32.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:32.761644+0000) 2024-03-06T09:01:33.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:33.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:33.761844+0000) 2024-03-06T09:01:34.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:34.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:34.762080+0000) 2024-03-06T09:01:35.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:35.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:35.762306+0000) 2024-03-06T09:01:36.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:36.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:36.762523+0000) 2024-03-06T09:01:37.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:37.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:37.762763+0000) 2024-03-06T09:01:38.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:38.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:38.762931+0000) 2024-03-06T09:01:39.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:39.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:39.763173+0000) 2024-03-06T09:01:40.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:40.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:40.763312+0000) 2024-03-06T09:01:41.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:41.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:41.763574+0000) 2024-03-06T09:01:42.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:42.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:42.763768+0000) 2024-03-06T09:01:43.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:43.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:43.764012+0000) 2024-03-06T09:01:44.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:44.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:44.764240+0000) 2024-03-06T09:01:45.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:45.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:45.764470+0000) 2024-03-06T09:01:46.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:46.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:46.764688+0000) 2024-03-06T09:01:47.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:47.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:47.764867+0000) 2024-03-06T09:01:48.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:48.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:48.765071+0000) 2024-03-06T09:01:49.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:49.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:49.765306+0000) 2024-03-06T09:01:50.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:50.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:50.765524+0000) 2024-03-06T09:01:51.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:51.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:51.765737+0000) 2024-03-06T09:01:52.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:52.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:52.765951+0000) 2024-03-06T09:01:53.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:53.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:53.766265+0000) 2024-03-06T09:01:54.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:54.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:54.766482+0000) 2024-03-06T09:01:55.632 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:01:55.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:01:55.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:55.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:55.766719+0000) 2024-03-06T09:01:56.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:56.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:56.766922+0000) 2024-03-06T09:01:57.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:57.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:57.767201+0000) 2024-03-06T09:01:58.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:58.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:58.767524+0000) 2024-03-06T09:01:59.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:01:59.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:01:59.767761+0000) 2024-03-06T09:02:00.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:00.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:00.768014+0000) 2024-03-06T09:02:01.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:01.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:01.768243+0000) 2024-03-06T09:02:02.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:02.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:02.768458+0000) 2024-03-06T09:02:03.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:03.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:03.768702+0000) 2024-03-06T09:02:04.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:04.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:04.768870+0000) 2024-03-06T09:02:05.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:05.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:05.769074+0000) 2024-03-06T09:02:06.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:06.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:06.769292+0000) 2024-03-06T09:02:07.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:07.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:07.769515+0000) 2024-03-06T09:02:08.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:08.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:08.769731+0000) 2024-03-06T09:02:09.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:09.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:09.769883+0000) 2024-03-06T09:02:10.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:10.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:10.770089+0000) 2024-03-06T09:02:11.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:11.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:11.770320+0000) 2024-03-06T09:02:12.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:12.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:12.770579+0000) 2024-03-06T09:02:13.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:13.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:13.770783+0000) 2024-03-06T09:02:14.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:14.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:14.770973+0000) 2024-03-06T09:02:15.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:15.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:15.771177+0000) 2024-03-06T09:02:16.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:16.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:16.771378+0000) 2024-03-06T09:02:17.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:17.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:17.771571+0000) 2024-03-06T09:02:18.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:18.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:18.771778+0000) 2024-03-06T09:02:19.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:19.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:19.771944+0000) 2024-03-06T09:02:20.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:20.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:20.772212+0000) 2024-03-06T09:02:21.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:21.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:21.772489+0000) 2024-03-06T09:02:22.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:22.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:22.772695+0000) 2024-03-06T09:02:23.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:23.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:23.772926+0000) 2024-03-06T09:02:24.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:24.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:24.773184+0000) 2024-03-06T09:02:25.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:25.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:25.773451+0000) 2024-03-06T09:02:25.837 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:02:25.848 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:02:26.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:26.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:26.773729+0000) 2024-03-06T09:02:27.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:27.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:27.773969+0000) 2024-03-06T09:02:28.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:28.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:28.774245+0000) 2024-03-06T09:02:29.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:29.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:29.774573+0000) 2024-03-06T09:02:30.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:30.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:30.774813+0000) 2024-03-06T09:02:31.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:31.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:31.775091+0000) 2024-03-06T09:02:32.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:32.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:32.775366+0000) 2024-03-06T09:02:33.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:33.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:33.775714+0000) 2024-03-06T09:02:34.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:34.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:34.775989+0000) 2024-03-06T09:02:35.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:35.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:35.776254+0000) 2024-03-06T09:02:36.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:36.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:36.776516+0000) 2024-03-06T09:02:37.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:37.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:37.776769+0000) 2024-03-06T09:02:38.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:38.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:38.777016+0000) 2024-03-06T09:02:39.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:39.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:39.777242+0000) 2024-03-06T09:02:40.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:40.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:40.777399+0000) 2024-03-06T09:02:41.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:41.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:41.777638+0000) 2024-03-06T09:02:42.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:42.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:42.777830+0000) 2024-03-06T09:02:43.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:43.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:43.778076+0000) 2024-03-06T09:02:44.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:44.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:44.778269+0000) 2024-03-06T09:02:45.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:45.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:45.778419+0000) 2024-03-06T09:02:46.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:46.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:46.778667+0000) 2024-03-06T09:02:47.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:47.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:47.778864+0000) 2024-03-06T09:02:48.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:48.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:48.779034+0000) 2024-03-06T09:02:49.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:49.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:49.779245+0000) 2024-03-06T09:02:50.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:50.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:50.779451+0000) 2024-03-06T09:02:51.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:51.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:51.779742+0000) 2024-03-06T09:02:52.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:52.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:52.779933+0000) 2024-03-06T09:02:53.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:53.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:53.780160+0000) 2024-03-06T09:02:54.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:54.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:54.780355+0000) 2024-03-06T09:02:55.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:55.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:55.780631+0000) 2024-03-06T09:02:56.069 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:02:56.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:02:56.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:56.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:56.780824+0000) 2024-03-06T09:02:57.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:57.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:57.781017+0000) 2024-03-06T09:02:58.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:58.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:58.781267+0000) 2024-03-06T09:02:59.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:02:59.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:02:59.781467+0000) 2024-03-06T09:03:00.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:00.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:00.781671+0000) 2024-03-06T09:03:01.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:01.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:01.781832+0000) 2024-03-06T09:03:02.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:02.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:02.782094+0000) 2024-03-06T09:03:03.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:03.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:03.782339+0000) 2024-03-06T09:03:04.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:04.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:04.782613+0000) 2024-03-06T09:03:05.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:05.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:05.782836+0000) 2024-03-06T09:03:06.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:06.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:06.783085+0000) 2024-03-06T09:03:07.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:07.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:07.783306+0000) 2024-03-06T09:03:08.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:08.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:08.783514+0000) 2024-03-06T09:03:09.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:09.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:09.783751+0000) 2024-03-06T09:03:10.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:10.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:10.783967+0000) 2024-03-06T09:03:11.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:11.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:11.784248+0000) 2024-03-06T09:03:12.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:12.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:12.784481+0000) 2024-03-06T09:03:13.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:13.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:13.784758+0000) 2024-03-06T09:03:14.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:14.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:14.784931+0000) 2024-03-06T09:03:15.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:15.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:15.785142+0000) 2024-03-06T09:03:16.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:16.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:16.785416+0000) 2024-03-06T09:03:17.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:17.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:17.785665+0000) 2024-03-06T09:03:18.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:18.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:18.785912+0000) 2024-03-06T09:03:19.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:19.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:19.786159+0000) 2024-03-06T09:03:20.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:20.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:20.786442+0000) 2024-03-06T09:03:21.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:21.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:21.786660+0000) 2024-03-06T09:03:22.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:22.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:22.786870+0000) 2024-03-06T09:03:23.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:23.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:23.787097+0000) 2024-03-06T09:03:24.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:24.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:24.787347+0000) 2024-03-06T09:03:25.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:25.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:25.787650+0000) 2024-03-06T09:03:26.346 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:03:26.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:03:26.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:26.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:26.787865+0000) 2024-03-06T09:03:27.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:27.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:27.788079+0000) 2024-03-06T09:03:28.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:28.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:28.788270+0000) 2024-03-06T09:03:29.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:29.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:29.788443+0000) 2024-03-06T09:03:30.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:30.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:30.788677+0000) 2024-03-06T09:03:31.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:31.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:31.788852+0000) 2024-03-06T09:03:32.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:32.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:32.789072+0000) 2024-03-06T09:03:33.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:33.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:33.789314+0000) 2024-03-06T09:03:34.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:34.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:34.789577+0000) 2024-03-06T09:03:35.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:35.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:35.789786+0000) 2024-03-06T09:03:36.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:36.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:36.789989+0000) 2024-03-06T09:03:37.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:37.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:37.790180+0000) 2024-03-06T09:03:38.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:38.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:38.790391+0000) 2024-03-06T09:03:39.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:39.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:39.790644+0000) 2024-03-06T09:03:40.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:40.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:40.790870+0000) 2024-03-06T09:03:41.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:41.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:41.791054+0000) 2024-03-06T09:03:42.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:42.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:42.791271+0000) 2024-03-06T09:03:43.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:43.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:43.791502+0000) 2024-03-06T09:03:44.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:44.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:44.791734+0000) 2024-03-06T09:03:45.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:45.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:45.791976+0000) 2024-03-06T09:03:46.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:46.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:46.792164+0000) 2024-03-06T09:03:47.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:47.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:47.792391+0000) 2024-03-06T09:03:48.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:48.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:48.792691+0000) 2024-03-06T09:03:49.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:49.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:49.792872+0000) 2024-03-06T09:03:50.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:50.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:50.793061+0000) 2024-03-06T09:03:51.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:51.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:51.793249+0000) 2024-03-06T09:03:52.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:52.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:52.793496+0000) 2024-03-06T09:03:53.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:53.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:53.793763+0000) 2024-03-06T09:03:54.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:54.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:54.793945+0000) 2024-03-06T09:03:55.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:55.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:55.794141+0000) 2024-03-06T09:03:56.546 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:03:56.554 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:03:56.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:56.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:56.794402+0000) 2024-03-06T09:03:57.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:57.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:57.794699+0000) 2024-03-06T09:03:58.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:58.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:58.794899+0000) 2024-03-06T09:03:59.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:03:59.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:03:59.795120+0000) 2024-03-06T09:04:00.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:00.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:00.795340+0000) 2024-03-06T09:04:01.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:01.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:01.795608+0000) 2024-03-06T09:04:02.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:02.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:02.795832+0000) 2024-03-06T09:04:03.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:03.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:03.795996+0000) 2024-03-06T09:04:04.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:04.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:04.796156+0000) 2024-03-06T09:04:05.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:05.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:05.796317+0000) 2024-03-06T09:04:06.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:06.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:06.796516+0000) 2024-03-06T09:04:07.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:07.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:07.796763+0000) 2024-03-06T09:04:08.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:08.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:08.796918+0000) 2024-03-06T09:04:09.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:09.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:09.797118+0000) 2024-03-06T09:04:10.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:10.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:10.797338+0000) 2024-03-06T09:04:11.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:11.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:11.797498+0000) 2024-03-06T09:04:12.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:12.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:12.797761+0000) 2024-03-06T09:04:13.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:13.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:13.797957+0000) 2024-03-06T09:04:14.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:14.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:14.798125+0000) 2024-03-06T09:04:15.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:15.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:15.798363+0000) 2024-03-06T09:04:16.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:16.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:16.798607+0000) 2024-03-06T09:04:17.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:17.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:17.798775+0000) 2024-03-06T09:04:18.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:18.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:18.798938+0000) 2024-03-06T09:04:19.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:19.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:19.799127+0000) 2024-03-06T09:04:20.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:20.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:20.799346+0000) 2024-03-06T09:04:21.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:21.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:21.799602+0000) 2024-03-06T09:04:22.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:22.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:22.799828+0000) 2024-03-06T09:04:23.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:23.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:23.800024+0000) 2024-03-06T09:04:24.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:24.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:24.800246+0000) 2024-03-06T09:04:25.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:25.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:25.800467+0000) 2024-03-06T09:04:26.720 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:04:26.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:04:26.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:26.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:26.800655+0000) 2024-03-06T09:04:27.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:27.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:27.800810+0000) 2024-03-06T09:04:28.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:28.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:28.801010+0000) 2024-03-06T09:04:29.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:29.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:29.801230+0000) 2024-03-06T09:04:30.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:30.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:30.801451+0000) 2024-03-06T09:04:31.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:31.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:31.801660+0000) 2024-03-06T09:04:32.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:32.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:32.801861+0000) 2024-03-06T09:04:33.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:33.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:33.802157+0000) 2024-03-06T09:04:34.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:34.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:34.802350+0000) 2024-03-06T09:04:35.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:35.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:35.802633+0000) 2024-03-06T09:04:36.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:36.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:36.802788+0000) 2024-03-06T09:04:37.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:37.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:37.802984+0000) 2024-03-06T09:04:38.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:38.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:38.803184+0000) 2024-03-06T09:04:39.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:39.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:39.803418+0000) 2024-03-06T09:04:40.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:40.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:40.803639+0000) 2024-03-06T09:04:41.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:41.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:41.803831+0000) 2024-03-06T09:04:42.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:42.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:42.804048+0000) 2024-03-06T09:04:43.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:43.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:43.804295+0000) 2024-03-06T09:04:44.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:44.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:44.804522+0000) 2024-03-06T09:04:45.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:45.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:45.804797+0000) 2024-03-06T09:04:46.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:46.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:46.804985+0000) 2024-03-06T09:04:47.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:47.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:47.805205+0000) 2024-03-06T09:04:48.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:48.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:48.805350+0000) 2024-03-06T09:04:49.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:49.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:49.805623+0000) 2024-03-06T09:04:50.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:50.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:50.805807+0000) 2024-03-06T09:04:51.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:51.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:51.806005+0000) 2024-03-06T09:04:52.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:52.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:52.806209+0000) 2024-03-06T09:04:53.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:53.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:53.806444+0000) 2024-03-06T09:04:54.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:54.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:54.806672+0000) 2024-03-06T09:04:55.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:55.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:55.806859+0000) 2024-03-06T09:04:56.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:56.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:56.807048+0000) 2024-03-06T09:04:56.970 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:04:56.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:04:57.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:57.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:57.807288+0000) 2024-03-06T09:04:58.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:58.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:58.807515+0000) 2024-03-06T09:04:59.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:04:59.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:04:59.807730+0000) 2024-03-06T09:05:00.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:00.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:00.807961+0000) 2024-03-06T09:05:01.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:01.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:01.808145+0000) 2024-03-06T09:05:02.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:02.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:02.808297+0000) 2024-03-06T09:05:03.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:03.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:03.808569+0000) 2024-03-06T09:05:04.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:04.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:04.808765+0000) 2024-03-06T09:05:05.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:05.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:05.808962+0000) 2024-03-06T09:05:06.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:06.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:06.809178+0000) 2024-03-06T09:05:07.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:07.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:07.809409+0000) 2024-03-06T09:05:08.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:08.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:08.809639+0000) 2024-03-06T09:05:09.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:09.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:09.809838+0000) 2024-03-06T09:05:10.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:10.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:10.810085+0000) 2024-03-06T09:05:11.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:11.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:11.810262+0000) 2024-03-06T09:05:12.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:12.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:12.810493+0000) 2024-03-06T09:05:13.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:13.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:13.810693+0000) 2024-03-06T09:05:14.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:14.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:14.810867+0000) 2024-03-06T09:05:15.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:15.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:15.811060+0000) 2024-03-06T09:05:16.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:16.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:16.811282+0000) 2024-03-06T09:05:17.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:17.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:17.811493+0000) 2024-03-06T09:05:18.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:18.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:18.811680+0000) 2024-03-06T09:05:19.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:19.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:19.811888+0000) 2024-03-06T09:05:20.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:20.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:20.812091+0000) 2024-03-06T09:05:21.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:21.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:21.812310+0000) 2024-03-06T09:05:22.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:22.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:22.812574+0000) 2024-03-06T09:05:23.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:23.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:23.812772+0000) 2024-03-06T09:05:24.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:24.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:24.812954+0000) 2024-03-06T09:05:25.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:25.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:25.813177+0000) 2024-03-06T09:05:26.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:26.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:26.813447+0000) 2024-03-06T09:05:27.199 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:05:27.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:05:27.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:27.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:27.813662+0000) 2024-03-06T09:05:28.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:28.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:28.813894+0000) 2024-03-06T09:05:29.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:29.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:29.814082+0000) 2024-03-06T09:05:30.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:30.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:30.814317+0000) 2024-03-06T09:05:31.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:31.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:31.814564+0000) 2024-03-06T09:05:32.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:32.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:32.814738+0000) 2024-03-06T09:05:33.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:33.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:33.814933+0000) 2024-03-06T09:05:34.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:34.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:34.815151+0000) 2024-03-06T09:05:35.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:35.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:35.815345+0000) 2024-03-06T09:05:36.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:36.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:36.815638+0000) 2024-03-06T09:05:37.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:37.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:37.815862+0000) 2024-03-06T09:05:38.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:38.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:38.816082+0000) 2024-03-06T09:05:39.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:39.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:39.816280+0000) 2024-03-06T09:05:40.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:40.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:40.816516+0000) 2024-03-06T09:05:41.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:41.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:41.816763+0000) 2024-03-06T09:05:42.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:42.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:42.816981+0000) 2024-03-06T09:05:43.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:43.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:43.817213+0000) 2024-03-06T09:05:44.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:44.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:44.817438+0000) 2024-03-06T09:05:45.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:45.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:45.817682+0000) 2024-03-06T09:05:46.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:46.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:46.817902+0000) 2024-03-06T09:05:47.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:47.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:47.818112+0000) 2024-03-06T09:05:48.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:48.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:48.818346+0000) 2024-03-06T09:05:49.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:49.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:49.818508+0000) 2024-03-06T09:05:50.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:50.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:50.818715+0000) 2024-03-06T09:05:51.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:51.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:51.818948+0000) 2024-03-06T09:05:52.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:52.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:52.819103+0000) 2024-03-06T09:05:53.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:53.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:53.819356+0000) 2024-03-06T09:05:54.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:54.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:54.819634+0000) 2024-03-06T09:05:55.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:55.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:55.819872+0000) 2024-03-06T09:05:56.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:56.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:56.820090+0000) 2024-03-06T09:05:57.662 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:05:57.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:05:57.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:57.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:57.820243+0000) 2024-03-06T09:05:58.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:58.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:58.820438+0000) 2024-03-06T09:05:59.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:05:59.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:05:59.820683+0000) 2024-03-06T09:06:00.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:00.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:00.820875+0000) 2024-03-06T09:06:01.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:01.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:01.821043+0000) 2024-03-06T09:06:02.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:02.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:02.821286+0000) 2024-03-06T09:06:03.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:03.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:03.821458+0000) 2024-03-06T09:06:04.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:04.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:04.821679+0000) 2024-03-06T09:06:05.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:05.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:05.821876+0000) 2024-03-06T09:06:06.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:06.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:06.822036+0000) 2024-03-06T09:06:07.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:07.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:07.822272+0000) 2024-03-06T09:06:08.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:08.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:08.822495+0000) 2024-03-06T09:06:09.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:09.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:09.822707+0000) 2024-03-06T09:06:10.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:10.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:10.822928+0000) 2024-03-06T09:06:11.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:11.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:11.823078+0000) 2024-03-06T09:06:12.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:12.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:12.823344+0000) 2024-03-06T09:06:13.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:13.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:13.823657+0000) 2024-03-06T09:06:14.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:14.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:14.823937+0000) 2024-03-06T09:06:15.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:15.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:15.824186+0000) 2024-03-06T09:06:16.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:16.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:16.824418+0000) 2024-03-06T09:06:17.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:17.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:17.824642+0000) 2024-03-06T09:06:18.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:18.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:18.824828+0000) 2024-03-06T09:06:19.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:19.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:19.824979+0000) 2024-03-06T09:06:20.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:20.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:20.825232+0000) 2024-03-06T09:06:21.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:21.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:21.825412+0000) 2024-03-06T09:06:22.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:22.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:22.825653+0000) 2024-03-06T09:06:23.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:23.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:23.825854+0000) 2024-03-06T09:06:24.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:24.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:24.826022+0000) 2024-03-06T09:06:25.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:25.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:25.826263+0000) 2024-03-06T09:06:26.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:26.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:26.826492+0000) 2024-03-06T09:06:27.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:27.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:27.826696+0000) 2024-03-06T09:06:27.850 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:06:27.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:06:28.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:28.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:28.826869+0000) 2024-03-06T09:06:29.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:29.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:29.827023+0000) 2024-03-06T09:06:30.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:30.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:30.827187+0000) 2024-03-06T09:06:31.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:31.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:31.827416+0000) 2024-03-06T09:06:32.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:32.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:32.827599+0000) 2024-03-06T09:06:33.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:33.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:33.827876+0000) 2024-03-06T09:06:34.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:34.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:34.828089+0000) 2024-03-06T09:06:35.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:35.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:35.828305+0000) 2024-03-06T09:06:36.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:36.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:36.828527+0000) 2024-03-06T09:06:37.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:37.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:37.828764+0000) 2024-03-06T09:06:38.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:38.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:38.828952+0000) 2024-03-06T09:06:39.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:39.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:39.829180+0000) 2024-03-06T09:06:40.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:40.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:40.829400+0000) 2024-03-06T09:06:41.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:41.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:41.829652+0000) 2024-03-06T09:06:42.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:42.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:42.829863+0000) 2024-03-06T09:06:43.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:43.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:43.830098+0000) 2024-03-06T09:06:44.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:44.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:44.830323+0000) 2024-03-06T09:06:45.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:45.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:45.830511+0000) 2024-03-06T09:06:46.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:46.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:46.830737+0000) 2024-03-06T09:06:47.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:47.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:47.830941+0000) 2024-03-06T09:06:48.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:48.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:48.831162+0000) 2024-03-06T09:06:49.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:49.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:49.831392+0000) 2024-03-06T09:06:50.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:50.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:50.831689+0000) 2024-03-06T09:06:51.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:51.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:51.831890+0000) 2024-03-06T09:06:52.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:52.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:52.832126+0000) 2024-03-06T09:06:53.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:53.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:53.832356+0000) 2024-03-06T09:06:54.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:54.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:54.832622+0000) 2024-03-06T09:06:55.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:55.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:55.832818+0000) 2024-03-06T09:06:56.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:56.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:56.833033+0000) 2024-03-06T09:06:57.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:57.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:57.833262+0000) 2024-03-06T09:06:58.063 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:06:58.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:06:58.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:58.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:58.833430+0000) 2024-03-06T09:06:59.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:06:59.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:06:59.833669+0000) 2024-03-06T09:07:00.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:00.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:00.833860+0000) 2024-03-06T09:07:01.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:01.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:01.834048+0000) 2024-03-06T09:07:02.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:02.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:02.834249+0000) 2024-03-06T09:07:03.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:03.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:03.834495+0000) 2024-03-06T09:07:04.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:04.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:04.834697+0000) 2024-03-06T09:07:05.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:05.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:05.834947+0000) 2024-03-06T09:07:06.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:06.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:06.835161+0000) 2024-03-06T09:07:07.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:07.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:07.835408+0000) 2024-03-06T09:07:08.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:08.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:08.835657+0000) 2024-03-06T09:07:09.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:09.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:09.835825+0000) 2024-03-06T09:07:10.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:10.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:10.836046+0000) 2024-03-06T09:07:11.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:11.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:11.836238+0000) 2024-03-06T09:07:12.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:12.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:12.836457+0000) 2024-03-06T09:07:13.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:13.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:13.836694+0000) 2024-03-06T09:07:14.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:14.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:14.836952+0000) 2024-03-06T09:07:15.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:15.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:15.837198+0000) 2024-03-06T09:07:16.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:16.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:16.837463+0000) 2024-03-06T09:07:17.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:17.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:17.837666+0000) 2024-03-06T09:07:18.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:18.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:18.837877+0000) 2024-03-06T09:07:19.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:19.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:19.838045+0000) 2024-03-06T09:07:20.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:20.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:20.838295+0000) 2024-03-06T09:07:21.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:21.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:21.838519+0000) 2024-03-06T09:07:22.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:22.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:22.838751+0000) 2024-03-06T09:07:23.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:23.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:23.838948+0000) 2024-03-06T09:07:24.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:24.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:24.839175+0000) 2024-03-06T09:07:25.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:25.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:25.839403+0000) 2024-03-06T09:07:26.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:26.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:26.839656+0000) 2024-03-06T09:07:27.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:27.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:27.839883+0000) 2024-03-06T09:07:28.284 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:07:28.292 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:07:28.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:28.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:28.840087+0000) 2024-03-06T09:07:29.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:29.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:29.840338+0000) 2024-03-06T09:07:30.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:30.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:30.840594+0000) 2024-03-06T09:07:31.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:31.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:31.840784+0000) 2024-03-06T09:07:32.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:32.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:32.840939+0000) 2024-03-06T09:07:33.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:33.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:33.841187+0000) 2024-03-06T09:07:34.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:34.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:34.841414+0000) 2024-03-06T09:07:35.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:35.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:35.841639+0000) 2024-03-06T09:07:36.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:36.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:36.841828+0000) 2024-03-06T09:07:37.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:37.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:37.842046+0000) 2024-03-06T09:07:38.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:38.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:38.842267+0000) 2024-03-06T09:07:39.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:39.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:39.842501+0000) 2024-03-06T09:07:40.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:40.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:40.842732+0000) 2024-03-06T09:07:41.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:41.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:41.842923+0000) 2024-03-06T09:07:42.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:42.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:42.843078+0000) 2024-03-06T09:07:43.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:43.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:43.843325+0000) 2024-03-06T09:07:44.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:44.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:44.843574+0000) 2024-03-06T09:07:45.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:45.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:45.843816+0000) 2024-03-06T09:07:46.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:46.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:46.843999+0000) 2024-03-06T09:07:47.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:47.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:47.844175+0000) 2024-03-06T09:07:48.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:48.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:48.844342+0000) 2024-03-06T09:07:49.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:49.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:49.844648+0000) 2024-03-06T09:07:50.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:50.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:50.844895+0000) 2024-03-06T09:07:51.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:51.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:51.845080+0000) 2024-03-06T09:07:52.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:52.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:52.845309+0000) 2024-03-06T09:07:53.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:53.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:53.845496+0000) 2024-03-06T09:07:54.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:54.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:54.845676+0000) 2024-03-06T09:07:55.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:55.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:55.845882+0000) 2024-03-06T09:07:56.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:56.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:56.846079+0000) 2024-03-06T09:07:57.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:57.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:57.846313+0000) 2024-03-06T09:07:58.731 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:07:58.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:07:58.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:58.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:58.846481+0000) 2024-03-06T09:07:59.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:07:59.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:07:59.846691+0000) 2024-03-06T09:08:00.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:00.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:00.846917+0000) 2024-03-06T09:08:01.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:01.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:01.847187+0000) 2024-03-06T09:08:02.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:02.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:02.847378+0000) 2024-03-06T09:08:03.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:03.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:03.847644+0000) 2024-03-06T09:08:04.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:04.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:04.847875+0000) 2024-03-06T09:08:05.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:05.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:05.848071+0000) 2024-03-06T09:08:06.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:06.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:06.848294+0000) 2024-03-06T09:08:07.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:07.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:07.848521+0000) 2024-03-06T09:08:08.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:08.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:08.848737+0000) 2024-03-06T09:08:09.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:09.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:09.848940+0000) 2024-03-06T09:08:10.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:10.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:10.849128+0000) 2024-03-06T09:08:11.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:11.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:11.849360+0000) 2024-03-06T09:08:12.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:12.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:12.849619+0000) 2024-03-06T09:08:13.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:13.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:13.849820+0000) 2024-03-06T09:08:14.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:14.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:14.850018+0000) 2024-03-06T09:08:15.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:15.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:15.850245+0000) 2024-03-06T09:08:16.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:16.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:16.850465+0000) 2024-03-06T09:08:17.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:17.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:17.850681+0000) 2024-03-06T09:08:18.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:18.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:18.850903+0000) 2024-03-06T09:08:19.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:19.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:19.851158+0000) 2024-03-06T09:08:20.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:20.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:20.851372+0000) 2024-03-06T09:08:21.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:21.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:21.851597+0000) 2024-03-06T09:08:22.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:22.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:22.851810+0000) 2024-03-06T09:08:23.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:23.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:23.852052+0000) 2024-03-06T09:08:24.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:24.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:24.852344+0000) 2024-03-06T09:08:25.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:25.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:25.852618+0000) 2024-03-06T09:08:26.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:26.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:26.852845+0000) 2024-03-06T09:08:27.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:27.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:27.853033+0000) 2024-03-06T09:08:28.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:28.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:28.853257+0000) 2024-03-06T09:08:28.913 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:08:28.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:08:29.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:29.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:29.853501+0000) 2024-03-06T09:08:30.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:30.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:30.853688+0000) 2024-03-06T09:08:31.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:31.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:31.853875+0000) 2024-03-06T09:08:32.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:32.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:32.854064+0000) 2024-03-06T09:08:33.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:33.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:33.854316+0000) 2024-03-06T09:08:34.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:34.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:34.854580+0000) 2024-03-06T09:08:35.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:35.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:35.854759+0000) 2024-03-06T09:08:36.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:36.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:36.854985+0000) 2024-03-06T09:08:37.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:37.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:37.855208+0000) 2024-03-06T09:08:38.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:38.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:38.855433+0000) 2024-03-06T09:08:39.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:39.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:39.855682+0000) 2024-03-06T09:08:40.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:40.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:40.855887+0000) 2024-03-06T09:08:41.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:41.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:41.856124+0000) 2024-03-06T09:08:42.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:42.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:42.856347+0000) 2024-03-06T09:08:43.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:43.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:43.856631+0000) 2024-03-06T09:08:44.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:44.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:44.856846+0000) 2024-03-06T09:08:45.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:45.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:45.857074+0000) 2024-03-06T09:08:46.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:46.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:46.857294+0000) 2024-03-06T09:08:47.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:47.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:47.857503+0000) 2024-03-06T09:08:48.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:48.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:48.857731+0000) 2024-03-06T09:08:49.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:49.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:49.857991+0000) 2024-03-06T09:08:50.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:50.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:50.858135+0000) 2024-03-06T09:08:51.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:51.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:51.858365+0000) 2024-03-06T09:08:52.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:52.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:52.858641+0000) 2024-03-06T09:08:53.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:53.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:53.858846+0000) 2024-03-06T09:08:54.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:54.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:54.859086+0000) 2024-03-06T09:08:55.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:55.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:55.859257+0000) 2024-03-06T09:08:56.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:56.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:56.859497+0000) 2024-03-06T09:08:57.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:57.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:57.859697+0000) 2024-03-06T09:08:58.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:58.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:58.859895+0000) 2024-03-06T09:08:59.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:08:59.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:08:59.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:08:59.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:08:59.860100+0000) 2024-03-06T09:09:00.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:00.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:00.860335+0000) 2024-03-06T09:09:01.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:01.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:01.860580+0000) 2024-03-06T09:09:02.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:02.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:02.860765+0000) 2024-03-06T09:09:03.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:03.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:03.860959+0000) 2024-03-06T09:09:04.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:04.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:04.861203+0000) 2024-03-06T09:09:05.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:05.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:05.861477+0000) 2024-03-06T09:09:06.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:06.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:06.861674+0000) 2024-03-06T09:09:07.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:07.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:07.861955+0000) 2024-03-06T09:09:08.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:08.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:08.862236+0000) 2024-03-06T09:09:09.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:09.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:09.862489+0000) 2024-03-06T09:09:10.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:10.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:10.862719+0000) 2024-03-06T09:09:11.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:11.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:11.862910+0000) 2024-03-06T09:09:12.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:12.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:12.863097+0000) 2024-03-06T09:09:13.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:13.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:13.863325+0000) 2024-03-06T09:09:14.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:14.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:14.863497+0000) 2024-03-06T09:09:15.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:15.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:15.863780+0000) 2024-03-06T09:09:16.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:16.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:16.863979+0000) 2024-03-06T09:09:17.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:17.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:17.864214+0000) 2024-03-06T09:09:18.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:18.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:18.864373+0000) 2024-03-06T09:09:19.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:19.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:19.864664+0000) 2024-03-06T09:09:20.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:20.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:20.864938+0000) 2024-03-06T09:09:21.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:21.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:21.865167+0000) 2024-03-06T09:09:22.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:22.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:22.865433+0000) 2024-03-06T09:09:23.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:23.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:23.865764+0000) 2024-03-06T09:09:24.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:24.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:24.865979+0000) 2024-03-06T09:09:25.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:25.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:25.866242+0000) 2024-03-06T09:09:26.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:26.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:26.866521+0000) 2024-03-06T09:09:27.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:27.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:27.866800+0000) 2024-03-06T09:09:28.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:28.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:28.867076+0000) 2024-03-06T09:09:29.299 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:09:29.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:09:29.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:29.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:29.867279+0000) 2024-03-06T09:09:30.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:30.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:30.867477+0000) 2024-03-06T09:09:31.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:31.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:31.867722+0000) 2024-03-06T09:09:32.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:32.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:32.868012+0000) 2024-03-06T09:09:33.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:33.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:33.868300+0000) 2024-03-06T09:09:34.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:34.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:34.868611+0000) 2024-03-06T09:09:35.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:35.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:35.868855+0000) 2024-03-06T09:09:36.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:36.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:36.869026+0000) 2024-03-06T09:09:37.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:37.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:37.869229+0000) 2024-03-06T09:09:38.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:38.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:38.869467+0000) 2024-03-06T09:09:39.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:39.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:39.869691+0000) 2024-03-06T09:09:40.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:40.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:40.869850+0000) 2024-03-06T09:09:41.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:41.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:41.870081+0000) 2024-03-06T09:09:42.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:42.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:42.870366+0000) 2024-03-06T09:09:43.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:43.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:43.870624+0000) 2024-03-06T09:09:44.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:44.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:44.870797+0000) 2024-03-06T09:09:45.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:45.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:45.871029+0000) 2024-03-06T09:09:46.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:46.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:46.871211+0000) 2024-03-06T09:09:47.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:47.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:47.871479+0000) 2024-03-06T09:09:48.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:48.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:48.871709+0000) 2024-03-06T09:09:49.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:49.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:49.871945+0000) 2024-03-06T09:09:50.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:50.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:50.872191+0000) 2024-03-06T09:09:51.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:51.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:51.872463+0000) 2024-03-06T09:09:52.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:52.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:52.872711+0000) 2024-03-06T09:09:53.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:53.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:53.872956+0000) 2024-03-06T09:09:54.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:54.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:54.873221+0000) 2024-03-06T09:09:55.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:55.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:55.873498+0000) 2024-03-06T09:09:56.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:56.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:56.873748+0000) 2024-03-06T09:09:57.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:57.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:57.873971+0000) 2024-03-06T09:09:58.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:58.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:58.874205+0000) 2024-03-06T09:09:59.446 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:09:59.454 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:09:59.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:09:59.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:09:59.874468+0000) 2024-03-06T09:10:00.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:00.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:00.874662+0000) 2024-03-06T09:10:01.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:01.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:01.874877+0000) 2024-03-06T09:10:02.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:02.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:02.875039+0000) 2024-03-06T09:10:03.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:03.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:03.875291+0000) 2024-03-06T09:10:04.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:04.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:04.875474+0000) 2024-03-06T09:10:05.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:05.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:05.875683+0000) 2024-03-06T09:10:06.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:06.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:06.875867+0000) 2024-03-06T09:10:07.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:07.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:07.876012+0000) 2024-03-06T09:10:08.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:08.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:08.876241+0000) 2024-03-06T09:10:09.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:09.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:09.876461+0000) 2024-03-06T09:10:10.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:10.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:10.876683+0000) 2024-03-06T09:10:11.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:11.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:11.876895+0000) 2024-03-06T09:10:12.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:12.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:12.877104+0000) 2024-03-06T09:10:13.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:13.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:13.877344+0000) 2024-03-06T09:10:14.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:14.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:14.877608+0000) 2024-03-06T09:10:15.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:15.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:15.877821+0000) 2024-03-06T09:10:16.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:16.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:16.878054+0000) 2024-03-06T09:10:17.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:17.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:17.878209+0000) 2024-03-06T09:10:18.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:18.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:18.878448+0000) 2024-03-06T09:10:19.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:19.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:19.878720+0000) 2024-03-06T09:10:20.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:20.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:20.878965+0000) 2024-03-06T09:10:21.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:21.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:21.879242+0000) 2024-03-06T09:10:22.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:22.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:22.879402+0000) 2024-03-06T09:10:23.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:23.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:23.879664+0000) 2024-03-06T09:10:24.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:24.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:24.879860+0000) 2024-03-06T09:10:25.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:25.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:25.880089+0000) 2024-03-06T09:10:26.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:26.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:26.880309+0000) 2024-03-06T09:10:27.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:27.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:27.880578+0000) 2024-03-06T09:10:28.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:28.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:28.880771+0000) 2024-03-06T09:10:29.606 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:10:29.614 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:10:29.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:29.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:29.881005+0000) 2024-03-06T09:10:30.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:30.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:30.881230+0000) 2024-03-06T09:10:31.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:31.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:31.881455+0000) 2024-03-06T09:10:32.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:32.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:32.881678+0000) 2024-03-06T09:10:33.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:33.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:33.881880+0000) 2024-03-06T09:10:34.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:34.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:34.882068+0000) 2024-03-06T09:10:35.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:35.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:35.882317+0000) 2024-03-06T09:10:36.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:36.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:36.882612+0000) 2024-03-06T09:10:37.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:37.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:37.882778+0000) 2024-03-06T09:10:38.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:38.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:38.882990+0000) 2024-03-06T09:10:39.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:39.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:39.883216+0000) 2024-03-06T09:10:40.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:40.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:40.883456+0000) 2024-03-06T09:10:41.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:41.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:41.883673+0000) 2024-03-06T09:10:42.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:42.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:42.883865+0000) 2024-03-06T09:10:43.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:43.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:43.884097+0000) 2024-03-06T09:10:44.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:44.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:44.884323+0000) 2024-03-06T09:10:45.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:45.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:45.884589+0000) 2024-03-06T09:10:46.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:46.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:46.884791+0000) 2024-03-06T09:10:47.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:47.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:47.885020+0000) 2024-03-06T09:10:48.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:48.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:48.885242+0000) 2024-03-06T09:10:49.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:49.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:49.885479+0000) 2024-03-06T09:10:50.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:50.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:50.885684+0000) 2024-03-06T09:10:51.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:51.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:51.885927+0000) 2024-03-06T09:10:52.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:52.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:52.886112+0000) 2024-03-06T09:10:53.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:53.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:53.886340+0000) 2024-03-06T09:10:54.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:54.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:54.886589+0000) 2024-03-06T09:10:55.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:55.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:55.886819+0000) 2024-03-06T09:10:56.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:56.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:56.887004+0000) 2024-03-06T09:10:57.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:57.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:57.887193+0000) 2024-03-06T09:10:58.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:58.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:58.887418+0000) 2024-03-06T09:10:59.823 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:10:59.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:10:59.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:10:59.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:10:59.887631+0000) 2024-03-06T09:11:00.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:00.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:00.887810+0000) 2024-03-06T09:11:01.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:01.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:01.888009+0000) 2024-03-06T09:11:02.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:02.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:02.888232+0000) 2024-03-06T09:11:03.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:03.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:03.888466+0000) 2024-03-06T09:11:04.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:04.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:04.888694+0000) 2024-03-06T09:11:05.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:05.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:05.888887+0000) 2024-03-06T09:11:06.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:06.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:06.889040+0000) 2024-03-06T09:11:07.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:07.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:07.889279+0000) 2024-03-06T09:11:08.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:08.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:08.889500+0000) 2024-03-06T09:11:09.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:09.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:09.889735+0000) 2024-03-06T09:11:10.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:10.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:10.890011+0000) 2024-03-06T09:11:11.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:11.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:11.890279+0000) 2024-03-06T09:11:12.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:12.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:12.890515+0000) 2024-03-06T09:11:13.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:13.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:13.890786+0000) 2024-03-06T09:11:14.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:14.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:14.891069+0000) 2024-03-06T09:11:15.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:15.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:15.891247+0000) 2024-03-06T09:11:16.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:16.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:16.891487+0000) 2024-03-06T09:11:17.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:17.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:17.891688+0000) 2024-03-06T09:11:18.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:18.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:18.891874+0000) 2024-03-06T09:11:19.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:19.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:19.892060+0000) 2024-03-06T09:11:20.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:20.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:20.892285+0000) 2024-03-06T09:11:21.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:21.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:21.892509+0000) 2024-03-06T09:11:22.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:22.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:22.892723+0000) 2024-03-06T09:11:23.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:23.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:23.893016+0000) 2024-03-06T09:11:24.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:24.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:24.893237+0000) 2024-03-06T09:11:25.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:25.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:25.893467+0000) 2024-03-06T09:11:26.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:26.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:26.893688+0000) 2024-03-06T09:11:27.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:27.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:27.893872+0000) 2024-03-06T09:11:28.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:28.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:28.894066+0000) 2024-03-06T09:11:29.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:29.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:29.894302+0000) 2024-03-06T09:11:30.011 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:11:30.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:11:30.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:30.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:30.894494+0000) 2024-03-06T09:11:31.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:31.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:31.894682+0000) 2024-03-06T09:11:32.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:32.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:32.894905+0000) 2024-03-06T09:11:33.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:33.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:33.895099+0000) 2024-03-06T09:11:34.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:34.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:34.895306+0000) 2024-03-06T09:11:35.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:35.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:35.895617+0000) 2024-03-06T09:11:36.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:36.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:36.895842+0000) 2024-03-06T09:11:37.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:37.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:37.896060+0000) 2024-03-06T09:11:38.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:38.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:38.896273+0000) 2024-03-06T09:11:39.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:39.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:39.896492+0000) 2024-03-06T09:11:40.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:40.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:40.896767+0000) 2024-03-06T09:11:41.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:41.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:41.896958+0000) 2024-03-06T09:11:42.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:42.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:42.897176+0000) 2024-03-06T09:11:43.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:43.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:43.897407+0000) 2024-03-06T09:11:44.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:44.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:44.897634+0000) 2024-03-06T09:11:45.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:45.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:45.897837+0000) 2024-03-06T09:11:46.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:46.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:46.898057+0000) 2024-03-06T09:11:47.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:47.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:47.898254+0000) 2024-03-06T09:11:48.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:48.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:48.898490+0000) 2024-03-06T09:11:49.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:49.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:49.898691+0000) 2024-03-06T09:11:50.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:50.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:50.898882+0000) 2024-03-06T09:11:51.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:51.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:51.899111+0000) 2024-03-06T09:11:52.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:52.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:52.899340+0000) 2024-03-06T09:11:53.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:53.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:53.899617+0000) 2024-03-06T09:11:54.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:54.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:54.899814+0000) 2024-03-06T09:11:55.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:55.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:55.900061+0000) 2024-03-06T09:11:56.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:56.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:56.900280+0000) 2024-03-06T09:11:57.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:57.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:57.900505+0000) 2024-03-06T09:11:58.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:58.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:58.900781+0000) 2024-03-06T09:11:59.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:11:59.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:11:59.901004+0000) 2024-03-06T09:12:00.168 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:12:00.176 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:12:00.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:00.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:00.901228+0000) 2024-03-06T09:12:01.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:01.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:01.901455+0000) 2024-03-06T09:12:02.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:02.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:02.901661+0000) 2024-03-06T09:12:03.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:03.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:03.901865+0000) 2024-03-06T09:12:04.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:04.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:04.902088+0000) 2024-03-06T09:12:05.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:05.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:05.902298+0000) 2024-03-06T09:12:06.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:06.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:06.902472+0000) 2024-03-06T09:12:07.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:07.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:07.902704+0000) 2024-03-06T09:12:08.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:08.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:08.902885+0000) 2024-03-06T09:12:09.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:09.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:09.903066+0000) 2024-03-06T09:12:10.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:10.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:10.903291+0000) 2024-03-06T09:12:11.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:11.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:11.903496+0000) 2024-03-06T09:12:12.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:12.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:12.903720+0000) 2024-03-06T09:12:13.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:13.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:13.903923+0000) 2024-03-06T09:12:14.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:14.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:14.904115+0000) 2024-03-06T09:12:15.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:15.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:15.904354+0000) 2024-03-06T09:12:16.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:16.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:16.904605+0000) 2024-03-06T09:12:17.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:17.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:17.904878+0000) 2024-03-06T09:12:18.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:18.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:18.905106+0000) 2024-03-06T09:12:19.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:19.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:19.905320+0000) 2024-03-06T09:12:20.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:20.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:20.905463+0000) 2024-03-06T09:12:21.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:21.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:21.905671+0000) 2024-03-06T09:12:22.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:22.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:22.905864+0000) 2024-03-06T09:12:23.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:23.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:23.906078+0000) 2024-03-06T09:12:24.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:24.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:24.906298+0000) 2024-03-06T09:12:25.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:25.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:25.906492+0000) 2024-03-06T09:12:26.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:26.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:26.906681+0000) 2024-03-06T09:12:27.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:27.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:27.906872+0000) 2024-03-06T09:12:28.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:28.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:28.907049+0000) 2024-03-06T09:12:29.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:29.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:29.907202+0000) 2024-03-06T09:12:30.333 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:12:30.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:12:30.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:30.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:30.907417+0000) 2024-03-06T09:12:31.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:31.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:31.907638+0000) 2024-03-06T09:12:32.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:32.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:32.907824+0000) 2024-03-06T09:12:33.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:33.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:33.908064+0000) 2024-03-06T09:12:34.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:34.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:34.908284+0000) 2024-03-06T09:12:35.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:35.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:35.908514+0000) 2024-03-06T09:12:36.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:36.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:36.908756+0000) 2024-03-06T09:12:37.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:37.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:37.908960+0000) 2024-03-06T09:12:38.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:38.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:38.909123+0000) 2024-03-06T09:12:39.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:39.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:39.909338+0000) 2024-03-06T09:12:40.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:40.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:40.909617+0000) 2024-03-06T09:12:41.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:41.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:41.909804+0000) 2024-03-06T09:12:42.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:42.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:42.909990+0000) 2024-03-06T09:12:43.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:43.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:43.910193+0000) 2024-03-06T09:12:44.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:44.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:44.910422+0000) 2024-03-06T09:12:45.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:45.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:45.910663+0000) 2024-03-06T09:12:46.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:46.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:46.910899+0000) 2024-03-06T09:12:47.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:47.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:47.911133+0000) 2024-03-06T09:12:48.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:48.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:48.911362+0000) 2024-03-06T09:12:49.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:49.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:49.911574+0000) 2024-03-06T09:12:50.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:50.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:50.911784+0000) 2024-03-06T09:12:51.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:51.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:51.911975+0000) 2024-03-06T09:12:52.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:52.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:52.912128+0000) 2024-03-06T09:12:53.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:53.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:53.912371+0000) 2024-03-06T09:12:54.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:54.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:54.912632+0000) 2024-03-06T09:12:55.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:55.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:55.912863+0000) 2024-03-06T09:12:56.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:56.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:56.913130+0000) 2024-03-06T09:12:57.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:57.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:57.913369+0000) 2024-03-06T09:12:58.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:58.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:58.913676+0000) 2024-03-06T09:12:59.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:12:59.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:12:59.913884+0000) 2024-03-06T09:13:00.547 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:13:00.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:13:00.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:00.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:00.914110+0000) 2024-03-06T09:13:01.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:01.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:01.914300+0000) 2024-03-06T09:13:02.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:02.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:02.914525+0000) 2024-03-06T09:13:03.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:03.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:03.914814+0000) 2024-03-06T09:13:04.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:04.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:04.915000+0000) 2024-03-06T09:13:05.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:05.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:05.915280+0000) 2024-03-06T09:13:06.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:06.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:06.915501+0000) 2024-03-06T09:13:07.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:07.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:07.915660+0000) 2024-03-06T09:13:08.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:08.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:08.915851+0000) 2024-03-06T09:13:09.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:09.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:09.916041+0000) 2024-03-06T09:13:10.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:10.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:10.916244+0000) 2024-03-06T09:13:11.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:11.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:11.916463+0000) 2024-03-06T09:13:12.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:12.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:12.916694+0000) 2024-03-06T09:13:13.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:13.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:13.916897+0000) 2024-03-06T09:13:14.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:14.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:14.917131+0000) 2024-03-06T09:13:15.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:15.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:15.917344+0000) 2024-03-06T09:13:16.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:16.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:16.917605+0000) 2024-03-06T09:13:17.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:17.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:17.917797+0000) 2024-03-06T09:13:18.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:18.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:18.917986+0000) 2024-03-06T09:13:19.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:19.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:19.918207+0000) 2024-03-06T09:13:20.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:20.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:20.918394+0000) 2024-03-06T09:13:21.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:21.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:21.918650+0000) 2024-03-06T09:13:22.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:22.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:22.918861+0000) 2024-03-06T09:13:23.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:23.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:23.919072+0000) 2024-03-06T09:13:24.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:24.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:24.919293+0000) 2024-03-06T09:13:25.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:25.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:25.919527+0000) 2024-03-06T09:13:26.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:26.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:26.919770+0000) 2024-03-06T09:13:27.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:27.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:27.919950+0000) 2024-03-06T09:13:28.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:28.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:28.920171+0000) 2024-03-06T09:13:29.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:29.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:29.920392+0000) 2024-03-06T09:13:30.765 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:13:30.773 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:13:30.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:30.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:30.920693+0000) 2024-03-06T09:13:31.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:31.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:31.920878+0000) 2024-03-06T09:13:32.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:32.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:32.921063+0000) 2024-03-06T09:13:33.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:33.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:33.921295+0000) 2024-03-06T09:13:34.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:34.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:34.921500+0000) 2024-03-06T09:13:35.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:35.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:35.921767+0000) 2024-03-06T09:13:36.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:36.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:36.922011+0000) 2024-03-06T09:13:37.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:37.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:37.922168+0000) 2024-03-06T09:13:38.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:38.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:38.922429+0000) 2024-03-06T09:13:39.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:39.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:39.922665+0000) 2024-03-06T09:13:40.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:40.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:40.922840+0000) 2024-03-06T09:13:41.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:41.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:41.923080+0000) 2024-03-06T09:13:42.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:42.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:42.923305+0000) 2024-03-06T09:13:43.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:43.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:43.923572+0000) 2024-03-06T09:13:44.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:44.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:44.923759+0000) 2024-03-06T09:13:45.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:45.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:45.923975+0000) 2024-03-06T09:13:46.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:46.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:46.924209+0000) 2024-03-06T09:13:47.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:47.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:47.924435+0000) 2024-03-06T09:13:48.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:48.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:48.924635+0000) 2024-03-06T09:13:49.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:49.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:49.924869+0000) 2024-03-06T09:13:50.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:50.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:50.925106+0000) 2024-03-06T09:13:51.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:51.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:51.925338+0000) 2024-03-06T09:13:52.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:52.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:52.925611+0000) 2024-03-06T09:13:53.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:53.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:53.925816+0000) 2024-03-06T09:13:54.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:54.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:54.926037+0000) 2024-03-06T09:13:55.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:55.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:55.926323+0000) 2024-03-06T09:13:56.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:56.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:56.926653+0000) 2024-03-06T09:13:57.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:57.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:57.926866+0000) 2024-03-06T09:13:58.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:58.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:58.927210+0000) 2024-03-06T09:13:59.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:13:59.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:13:59.927491+0000) 2024-03-06T09:14:00.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:00.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:00.927777+0000) 2024-03-06T09:14:00.950 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:14:00.958 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:14:01.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:01.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:01.928021+0000) 2024-03-06T09:14:02.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:02.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:02.928296+0000) 2024-03-06T09:14:03.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:03.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:03.928620+0000) 2024-03-06T09:14:04.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:04.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:04.928850+0000) 2024-03-06T09:14:05.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:05.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:05.929084+0000) 2024-03-06T09:14:06.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:06.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:06.929329+0000) 2024-03-06T09:14:07.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:07.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:07.929636+0000) 2024-03-06T09:14:08.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:08.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:08.929874+0000) 2024-03-06T09:14:09.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:09.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:09.930147+0000) 2024-03-06T09:14:10.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:10.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:10.930439+0000) 2024-03-06T09:14:11.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:11.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:11.930682+0000) 2024-03-06T09:14:12.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:12.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:12.930947+0000) 2024-03-06T09:14:13.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:13.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:13.931147+0000) 2024-03-06T09:14:14.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:14.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:14.931373+0000) 2024-03-06T09:14:15.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:15.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:15.931696+0000) 2024-03-06T09:14:16.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:16.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:16.931973+0000) 2024-03-06T09:14:17.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:17.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:17.932248+0000) 2024-03-06T09:14:18.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:18.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:18.932520+0000) 2024-03-06T09:14:19.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:19.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:19.932804+0000) 2024-03-06T09:14:20.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:20.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:20.933096+0000) 2024-03-06T09:14:21.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:21.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:21.933374+0000) 2024-03-06T09:14:22.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:22.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:22.933686+0000) 2024-03-06T09:14:23.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:23.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:23.933961+0000) 2024-03-06T09:14:24.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:24.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:24.934128+0000) 2024-03-06T09:14:25.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:25.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:25.934423+0000) 2024-03-06T09:14:26.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:26.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:26.934704+0000) 2024-03-06T09:14:27.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:27.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:27.934945+0000) 2024-03-06T09:14:28.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:28.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:28.935144+0000) 2024-03-06T09:14:29.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:29.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:29.935400+0000) 2024-03-06T09:14:30.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:30.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:30.935670+0000) 2024-03-06T09:14:31.087 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:14:31.095 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:14:31.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:31.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:31.935918+0000) 2024-03-06T09:14:32.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:32.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:32.936183+0000) 2024-03-06T09:14:33.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:33.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:33.936471+0000) 2024-03-06T09:14:34.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:34.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:34.936739+0000) 2024-03-06T09:14:35.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:35.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:35.936937+0000) 2024-03-06T09:14:36.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:36.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:36.937189+0000) 2024-03-06T09:14:37.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:37.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:37.937416+0000) 2024-03-06T09:14:38.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:38.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:38.937643+0000) 2024-03-06T09:14:39.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:39.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:39.937835+0000) 2024-03-06T09:14:40.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:40.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:40.938078+0000) 2024-03-06T09:14:41.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:41.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:41.938255+0000) 2024-03-06T09:14:42.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:42.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:42.938482+0000) 2024-03-06T09:14:43.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:43.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:43.938720+0000) 2024-03-06T09:14:44.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:44.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:44.938915+0000) 2024-03-06T09:14:45.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:45.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:45.939134+0000) 2024-03-06T09:14:46.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:46.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:46.939412+0000) 2024-03-06T09:14:47.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:47.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:47.939686+0000) 2024-03-06T09:14:48.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:48.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:48.939893+0000) 2024-03-06T09:14:49.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:49.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:49.940123+0000) 2024-03-06T09:14:50.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:50.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:50.940336+0000) 2024-03-06T09:14:51.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:51.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:51.940606+0000) 2024-03-06T09:14:52.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:52.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:52.940794+0000) 2024-03-06T09:14:53.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:53.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:53.941031+0000) 2024-03-06T09:14:54.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:54.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:54.941298+0000) 2024-03-06T09:14:55.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:55.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:55.941586+0000) 2024-03-06T09:14:56.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:56.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:56.941792+0000) 2024-03-06T09:14:57.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:57.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:57.941981+0000) 2024-03-06T09:14:58.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:58.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:58.942152+0000) 2024-03-06T09:14:59.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:14:59.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:14:59.942345+0000) 2024-03-06T09:15:00.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:00.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:00.942623+0000) 2024-03-06T09:15:01.258 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:15:01.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:15:01.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:01.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:01.942891+0000) 2024-03-06T09:15:02.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:02.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:02.943093+0000) 2024-03-06T09:15:03.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:03.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:03.943326+0000) 2024-03-06T09:15:04.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:04.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:04.943485+0000) 2024-03-06T09:15:05.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:05.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:05.943704+0000) 2024-03-06T09:15:06.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:06.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:06.943877+0000) 2024-03-06T09:15:07.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:07.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:07.944088+0000) 2024-03-06T09:15:08.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:08.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:08.944292+0000) 2024-03-06T09:15:09.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:09.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:09.944454+0000) 2024-03-06T09:15:10.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:10.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:10.944658+0000) 2024-03-06T09:15:11.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:11.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:11.944867+0000) 2024-03-06T09:15:12.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:12.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:12.945064+0000) 2024-03-06T09:15:13.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:13.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:13.945316+0000) 2024-03-06T09:15:14.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:14.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:14.945587+0000) 2024-03-06T09:15:15.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:15.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:15.945796+0000) 2024-03-06T09:15:16.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:16.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:16.945991+0000) 2024-03-06T09:15:17.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:17.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:17.946176+0000) 2024-03-06T09:15:18.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:18.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:18.946381+0000) 2024-03-06T09:15:19.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:19.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:19.946677+0000) 2024-03-06T09:15:20.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:20.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:20.946875+0000) 2024-03-06T09:15:21.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:21.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:21.947102+0000) 2024-03-06T09:15:22.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:22.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:22.947299+0000) 2024-03-06T09:15:23.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:23.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:23.947574+0000) 2024-03-06T09:15:24.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:24.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:24.947780+0000) 2024-03-06T09:15:25.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:25.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:25.947981+0000) 2024-03-06T09:15:26.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:26.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:26.948220+0000) 2024-03-06T09:15:27.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:27.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:27.948443+0000) 2024-03-06T09:15:28.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:28.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:28.948675+0000) 2024-03-06T09:15:29.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:29.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:29.948873+0000) 2024-03-06T09:15:30.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:30.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:30.949102+0000) 2024-03-06T09:15:31.470 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:15:31.478 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:15:31.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:31.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:31.949300+0000) 2024-03-06T09:15:32.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:32.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:32.949518+0000) 2024-03-06T09:15:33.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:33.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:33.949757+0000) 2024-03-06T09:15:34.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:34.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:34.949943+0000) 2024-03-06T09:15:35.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:35.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:35.950171+0000) 2024-03-06T09:15:36.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:36.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:36.950340+0000) 2024-03-06T09:15:37.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:37.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:37.950606+0000) 2024-03-06T09:15:38.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:38.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:38.950808+0000) 2024-03-06T09:15:39.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:39.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:39.951111+0000) 2024-03-06T09:15:40.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:40.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:40.951314+0000) 2024-03-06T09:15:41.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:41.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:41.951520+0000) 2024-03-06T09:15:42.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:42.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:42.951767+0000) 2024-03-06T09:15:43.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:43.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:43.951978+0000) 2024-03-06T09:15:44.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:44.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:44.952178+0000) 2024-03-06T09:15:45.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:45.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:45.952405+0000) 2024-03-06T09:15:46.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:46.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:46.952658+0000) 2024-03-06T09:15:47.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:47.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:47.952847+0000) 2024-03-06T09:15:48.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:48.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:48.953041+0000) 2024-03-06T09:15:49.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:49.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:49.953259+0000) 2024-03-06T09:15:50.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:50.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:50.953483+0000) 2024-03-06T09:15:51.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:51.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:51.953673+0000) 2024-03-06T09:15:52.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:52.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:52.953891+0000) 2024-03-06T09:15:53.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:53.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:53.954108+0000) 2024-03-06T09:15:54.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:54.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:54.954320+0000) 2024-03-06T09:15:55.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:55.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:55.954583+0000) 2024-03-06T09:15:56.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:56.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:56.954774+0000) 2024-03-06T09:15:57.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:57.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:57.954955+0000) 2024-03-06T09:15:58.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:58.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:58.955145+0000) 2024-03-06T09:15:59.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:15:59.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:15:59.955382+0000) 2024-03-06T09:16:00.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:00.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:00.955629+0000) 2024-03-06T09:16:01.660 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:16:01.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:16:01.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:01.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:01.955823+0000) 2024-03-06T09:16:02.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:02.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:02.956007+0000) 2024-03-06T09:16:03.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:03.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:03.956217+0000) 2024-03-06T09:16:04.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:04.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:04.956437+0000) 2024-03-06T09:16:05.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:05.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:05.956656+0000) 2024-03-06T09:16:06.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:06.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:06.956868+0000) 2024-03-06T09:16:07.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:07.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:07.957068+0000) 2024-03-06T09:16:08.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:08.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:08.957273+0000) 2024-03-06T09:16:09.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:09.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:09.957502+0000) 2024-03-06T09:16:10.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:10.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:10.957725+0000) 2024-03-06T09:16:11.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:11.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:11.957919+0000) 2024-03-06T09:16:12.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:12.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:12.958144+0000) 2024-03-06T09:16:13.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:13.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:13.958371+0000) 2024-03-06T09:16:14.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:14.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:14.958627+0000) 2024-03-06T09:16:15.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:15.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:15.958815+0000) 2024-03-06T09:16:16.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:16.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:16.959004+0000) 2024-03-06T09:16:17.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:17.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:17.959222+0000) 2024-03-06T09:16:18.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:18.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:18.959449+0000) 2024-03-06T09:16:19.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:19.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:19.959695+0000) 2024-03-06T09:16:20.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:20.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:20.959888+0000) 2024-03-06T09:16:21.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:21.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:21.960080+0000) 2024-03-06T09:16:22.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:22.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:22.960276+0000) 2024-03-06T09:16:23.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:23.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:23.960492+0000) 2024-03-06T09:16:24.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:24.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:24.960725+0000) 2024-03-06T09:16:25.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:25.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:25.960921+0000) 2024-03-06T09:16:26.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:26.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:26.961120+0000) 2024-03-06T09:16:27.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:27.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:27.961341+0000) 2024-03-06T09:16:28.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:28.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:28.961597+0000) 2024-03-06T09:16:29.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:29.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:29.961775+0000) 2024-03-06T09:16:30.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:30.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:30.961984+0000) 2024-03-06T09:16:31.856 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:16:31.864 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:16:31.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:31.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:31.962226+0000) 2024-03-06T09:16:32.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:32.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:32.962443+0000) 2024-03-06T09:16:33.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:33.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:33.962734+0000) 2024-03-06T09:16:34.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:34.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:34.962926+0000) 2024-03-06T09:16:35.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:35.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:35.963149+0000) 2024-03-06T09:16:36.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:36.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:36.963373+0000) 2024-03-06T09:16:37.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:37.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:37.963606+0000) 2024-03-06T09:16:38.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:38.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:38.963819+0000) 2024-03-06T09:16:39.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:39.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:39.964040+0000) 2024-03-06T09:16:40.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:40.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:40.964207+0000) 2024-03-06T09:16:41.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:41.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:41.964395+0000) 2024-03-06T09:16:42.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:42.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:42.964624+0000) 2024-03-06T09:16:43.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:43.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:43.964836+0000) 2024-03-06T09:16:44.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:44.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:44.964997+0000) 2024-03-06T09:16:45.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:45.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:45.965228+0000) 2024-03-06T09:16:46.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:46.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:46.965491+0000) 2024-03-06T09:16:47.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:47.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:47.965658+0000) 2024-03-06T09:16:48.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:48.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:48.965855+0000) 2024-03-06T09:16:49.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:49.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:49.966083+0000) 2024-03-06T09:16:50.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:50.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:50.966278+0000) 2024-03-06T09:16:51.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:51.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:51.966524+0000) 2024-03-06T09:16:52.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:52.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:52.966769+0000) 2024-03-06T09:16:53.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:53.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:53.966970+0000) 2024-03-06T09:16:54.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:54.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:54.967133+0000) 2024-03-06T09:16:55.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:55.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:55.967372+0000) 2024-03-06T09:16:56.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:56.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:56.967614+0000) 2024-03-06T09:16:57.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:57.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:57.967818+0000) 2024-03-06T09:16:58.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:58.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:58.968040+0000) 2024-03-06T09:16:59.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:16:59.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:16:59.968291+0000) 2024-03-06T09:17:00.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:00.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:00.968483+0000) 2024-03-06T09:17:01.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:01.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:01.968704+0000) 2024-03-06T09:17:02.054 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:17:02.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:17:02.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:02.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:02.968897+0000) 2024-03-06T09:17:03.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:03.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:03.969100+0000) 2024-03-06T09:17:04.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:04.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:04.969319+0000) 2024-03-06T09:17:05.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:05.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:05.969565+0000) 2024-03-06T09:17:06.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:06.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:06.969761+0000) 2024-03-06T09:17:07.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:07.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:07.969932+0000) 2024-03-06T09:17:08.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:08.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:08.970174+0000) 2024-03-06T09:17:09.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:09.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:09.970336+0000) 2024-03-06T09:17:10.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:10.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:10.970616+0000) 2024-03-06T09:17:11.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:11.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:11.970822+0000) 2024-03-06T09:17:12.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:12.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:12.971020+0000) 2024-03-06T09:17:13.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:13.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:13.971223+0000) 2024-03-06T09:17:14.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:14.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:14.971440+0000) 2024-03-06T09:17:15.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:15.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:15.971682+0000) 2024-03-06T09:17:16.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:16.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:16.971889+0000) 2024-03-06T09:17:17.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:17.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:17.972047+0000) 2024-03-06T09:17:18.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:18.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:18.972278+0000) 2024-03-06T09:17:19.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:19.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:19.972487+0000) 2024-03-06T09:17:20.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:20.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:20.972690+0000) 2024-03-06T09:17:21.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:21.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:21.972929+0000) 2024-03-06T09:17:22.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:22.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:22.973128+0000) 2024-03-06T09:17:23.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:23.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:23.973360+0000) 2024-03-06T09:17:24.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:24.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:24.973628+0000) 2024-03-06T09:17:25.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:25.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:25.973819+0000) 2024-03-06T09:17:26.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:26.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:26.974028+0000) 2024-03-06T09:17:27.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:27.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:27.974263+0000) 2024-03-06T09:17:28.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:28.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:28.974499+0000) 2024-03-06T09:17:29.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:29.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:29.974726+0000) 2024-03-06T09:17:30.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:30.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:30.974910+0000) 2024-03-06T09:17:31.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:31.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:31.975080+0000) 2024-03-06T09:17:32.195 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:17:32.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:17:32.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:32.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:32.975315+0000) 2024-03-06T09:17:33.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:33.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:33.975580+0000) 2024-03-06T09:17:34.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:34.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:34.975738+0000) 2024-03-06T09:17:35.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:35.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:35.975939+0000) 2024-03-06T09:17:36.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:36.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:36.976165+0000) 2024-03-06T09:17:37.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:37.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:37.976447+0000) 2024-03-06T09:17:38.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:38.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:38.976720+0000) 2024-03-06T09:17:39.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:39.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:39.976959+0000) 2024-03-06T09:17:40.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:40.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:40.977223+0000) 2024-03-06T09:17:41.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:41.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:41.977497+0000) 2024-03-06T09:17:42.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:42.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:42.977784+0000) 2024-03-06T09:17:43.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:43.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:43.977969+0000) 2024-03-06T09:17:44.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:44.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:44.978199+0000) 2024-03-06T09:17:45.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:45.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:45.978422+0000) 2024-03-06T09:17:46.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:46.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:46.978648+0000) 2024-03-06T09:17:47.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:47.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:47.978824+0000) 2024-03-06T09:17:48.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:48.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:48.979029+0000) 2024-03-06T09:17:49.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:49.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:49.979220+0000) 2024-03-06T09:17:50.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:50.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:50.979453+0000) 2024-03-06T09:17:51.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:51.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:51.979695+0000) 2024-03-06T09:17:52.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:52.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:52.979872+0000) 2024-03-06T09:17:53.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:53.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:53.980074+0000) 2024-03-06T09:17:54.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:54.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:54.980295+0000) 2024-03-06T09:17:55.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:55.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:55.980503+0000) 2024-03-06T09:17:56.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:56.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:56.980758+0000) 2024-03-06T09:17:57.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:57.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:57.980947+0000) 2024-03-06T09:17:58.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:58.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:58.981138+0000) 2024-03-06T09:17:59.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:17:59.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:17:59.981360+0000) 2024-03-06T09:18:00.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:00.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:00.981613+0000) 2024-03-06T09:18:01.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:01.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:01.981854+0000) 2024-03-06T09:18:02.349 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:18:02.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:18:02.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:02.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:02.982052+0000) 2024-03-06T09:18:03.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:03.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:03.982285+0000) 2024-03-06T09:18:04.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:04.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:04.982513+0000) 2024-03-06T09:18:05.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:05.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:05.982784+0000) 2024-03-06T09:18:06.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:06.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:06.982965+0000) 2024-03-06T09:18:07.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:07.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:07.983211+0000) 2024-03-06T09:18:08.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:08.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:08.983372+0000) 2024-03-06T09:18:09.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:09.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:09.983627+0000) 2024-03-06T09:18:10.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:10.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:10.983827+0000) 2024-03-06T09:18:11.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:11.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:11.984049+0000) 2024-03-06T09:18:12.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:12.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:12.984253+0000) 2024-03-06T09:18:13.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:13.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:13.984431+0000) 2024-03-06T09:18:14.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:14.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:14.984710+0000) 2024-03-06T09:18:15.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:15.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:15.984889+0000) 2024-03-06T09:18:16.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:16.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:16.985105+0000) 2024-03-06T09:18:17.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:17.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:17.985343+0000) 2024-03-06T09:18:18.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:18.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:18.985612+0000) 2024-03-06T09:18:19.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:19.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:19.985765+0000) 2024-03-06T09:18:20.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:20.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:20.986006+0000) 2024-03-06T09:18:21.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:21.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:21.986211+0000) 2024-03-06T09:18:22.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:22.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:22.986434+0000) 2024-03-06T09:18:23.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:23.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:23.986728+0000) 2024-03-06T09:18:24.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:24.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:24.986926+0000) 2024-03-06T09:18:25.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:25.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:25.987162+0000) 2024-03-06T09:18:26.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:26.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:26.987426+0000) 2024-03-06T09:18:27.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:27.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:27.987721+0000) 2024-03-06T09:18:28.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:28.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:28.987950+0000) 2024-03-06T09:18:29.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:29.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:29.988180+0000) 2024-03-06T09:18:30.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:30.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:30.988396+0000) 2024-03-06T09:18:31.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:31.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:31.988661+0000) 2024-03-06T09:18:32.492 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:18:32.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:18:32.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:32.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:32.988874+0000) 2024-03-06T09:18:33.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:33.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:33.989065+0000) 2024-03-06T09:18:34.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:34.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:34.989260+0000) 2024-03-06T09:18:35.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:35.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:35.989483+0000) 2024-03-06T09:18:36.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:36.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:36.989709+0000) 2024-03-06T09:18:37.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:37.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:37.989903+0000) 2024-03-06T09:18:38.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:38.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:38.990121+0000) 2024-03-06T09:18:39.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:39.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:39.990342+0000) 2024-03-06T09:18:40.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:40.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:40.990651+0000) 2024-03-06T09:18:41.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:41.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:41.990871+0000) 2024-03-06T09:18:42.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:42.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:42.991095+0000) 2024-03-06T09:18:43.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:43.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:43.991325+0000) 2024-03-06T09:18:44.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:44.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:44.991488+0000) 2024-03-06T09:18:45.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:45.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:45.991662+0000) 2024-03-06T09:18:46.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:46.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:46.991886+0000) 2024-03-06T09:18:47.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:47.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:47.992090+0000) 2024-03-06T09:18:48.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:48.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:48.992309+0000) 2024-03-06T09:18:49.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:49.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:49.992559+0000) 2024-03-06T09:18:50.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:50.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:50.992819+0000) 2024-03-06T09:18:51.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:51.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:51.993086+0000) 2024-03-06T09:18:52.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:52.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:52.993396+0000) 2024-03-06T09:18:53.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:53.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:53.993696+0000) 2024-03-06T09:18:54.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:54.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:54.993971+0000) 2024-03-06T09:18:55.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:55.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:55.994218+0000) 2024-03-06T09:18:56.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:56.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:56.994450+0000) 2024-03-06T09:18:57.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:57.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:57.994721+0000) 2024-03-06T09:18:58.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:58.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:58.994963+0000) 2024-03-06T09:18:59.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:18:59.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:18:59.995189+0000) 2024-03-06T09:19:00.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:00.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:00.995454+0000) 2024-03-06T09:19:01.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:01.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:01.995667+0000) 2024-03-06T09:19:02.686 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:19:02.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:19:02.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:02.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:02.995962+0000) 2024-03-06T09:19:03.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:03.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:03.996162+0000) 2024-03-06T09:19:04.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:04.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:04.996390+0000) 2024-03-06T09:19:05.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:05.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:05.996614+0000) 2024-03-06T09:19:06.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:06.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:06.996815+0000) 2024-03-06T09:19:07.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:07.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:07.997042+0000) 2024-03-06T09:19:08.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:08.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:08.997262+0000) 2024-03-06T09:19:09.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:09.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:09.997481+0000) 2024-03-06T09:19:10.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:10.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:10.997688+0000) 2024-03-06T09:19:11.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:11.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:11.997888+0000) 2024-03-06T09:19:12.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:12.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:12.998110+0000) 2024-03-06T09:19:13.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:13.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:13.998345+0000) 2024-03-06T09:19:14.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:14.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:14.998614+0000) 2024-03-06T09:19:15.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:15.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:15.998829+0000) 2024-03-06T09:19:16.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:16.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:16.999060+0000) 2024-03-06T09:19:17.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:17.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:17.999281+0000) 2024-03-06T09:19:18.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:18.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:18.999505+0000) 2024-03-06T09:19:19.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:19.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:19.999745+0000) 2024-03-06T09:19:20.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:20.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:20.999937+0000) 2024-03-06T09:19:21.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:21.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:22.000172+0000) 2024-03-06T09:19:22.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:22.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:23.000394+0000) 2024-03-06T09:19:24.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:24.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:24.000681+0000) 2024-03-06T09:19:25.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:25.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:25.000909+0000) 2024-03-06T09:19:26.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:26.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:26.001132+0000) 2024-03-06T09:19:27.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:27.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:27.001366+0000) 2024-03-06T09:19:28.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:28.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:28.001630+0000) 2024-03-06T09:19:29.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:29.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:29.001773+0000) 2024-03-06T09:19:30.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:30.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:30.001977+0000) 2024-03-06T09:19:31.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:31.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:31.002145+0000) 2024-03-06T09:19:32.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:32.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:32.002404+0000) 2024-03-06T09:19:32.899 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:19:32.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:19:33.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:33.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:33.002569+0000) 2024-03-06T09:19:34.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:34.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:34.002771+0000) 2024-03-06T09:19:35.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:35.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:35.002978+0000) 2024-03-06T09:19:36.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:36.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:36.003213+0000) 2024-03-06T09:19:37.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:37.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:37.003476+0000) 2024-03-06T09:19:38.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:38.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:38.003683+0000) 2024-03-06T09:19:39.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:39.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:39.003872+0000) 2024-03-06T09:19:40.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:40.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:40.004067+0000) 2024-03-06T09:19:41.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:41.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:41.004225+0000) 2024-03-06T09:19:42.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:42.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:42.004425+0000) 2024-03-06T09:19:43.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:43.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:43.004676+0000) 2024-03-06T09:19:44.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:44.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:44.004883+0000) 2024-03-06T09:19:45.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:45.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:45.005106+0000) 2024-03-06T09:19:46.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:46.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:46.005335+0000) 2024-03-06T09:19:47.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:47.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:47.005615+0000) 2024-03-06T09:19:48.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:48.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:48.005795+0000) 2024-03-06T09:19:49.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:49.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:49.006003+0000) 2024-03-06T09:19:50.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:50.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:50.006241+0000) 2024-03-06T09:19:51.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:51.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:51.006463+0000) 2024-03-06T09:19:52.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:52.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:52.006729+0000) 2024-03-06T09:19:53.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:53.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:53.006930+0000) 2024-03-06T09:19:54.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:54.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:54.007170+0000) 2024-03-06T09:19:55.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:55.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:55.007343+0000) 2024-03-06T09:19:56.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:56.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:56.007685+0000) 2024-03-06T09:19:57.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:57.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:57.007855+0000) 2024-03-06T09:19:58.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:58.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:58.008074+0000) 2024-03-06T09:19:59.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:19:59.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:19:59.008291+0000) 2024-03-06T09:20:00.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:00.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:00.008528+0000) 2024-03-06T09:20:01.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:01.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:01.008753+0000) 2024-03-06T09:20:02.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:02.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:02.008962+0000) 2024-03-06T09:20:03.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:03.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:03.009182+0000) 2024-03-06T09:20:03.349 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:20:03.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:20:04.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:04.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:04.009386+0000) 2024-03-06T09:20:05.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:05.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:05.009640+0000) 2024-03-06T09:20:06.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:06.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:06.009850+0000) 2024-03-06T09:20:07.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:07.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:07.010073+0000) 2024-03-06T09:20:08.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:08.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:08.010292+0000) 2024-03-06T09:20:09.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:09.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:09.010514+0000) 2024-03-06T09:20:10.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:10.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:10.010737+0000) 2024-03-06T09:20:11.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:11.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:11.010936+0000) 2024-03-06T09:20:12.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:12.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:12.011126+0000) 2024-03-06T09:20:13.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:13.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:13.011352+0000) 2024-03-06T09:20:14.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:14.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:14.011527+0000) 2024-03-06T09:20:15.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:15.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:15.011764+0000) 2024-03-06T09:20:16.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:16.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:16.011953+0000) 2024-03-06T09:20:17.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:17.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:17.012153+0000) 2024-03-06T09:20:18.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:18.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:18.012385+0000) 2024-03-06T09:20:19.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:19.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:19.012636+0000) 2024-03-06T09:20:20.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:20.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:20.012825+0000) 2024-03-06T09:20:21.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:21.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:21.013054+0000) 2024-03-06T09:20:22.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:22.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:22.013285+0000) 2024-03-06T09:20:23.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:23.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:23.013514+0000) 2024-03-06T09:20:24.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:24.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:24.013756+0000) 2024-03-06T09:20:25.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:25.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:25.013954+0000) 2024-03-06T09:20:26.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:26.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:26.014181+0000) 2024-03-06T09:20:27.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:27.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:27.014411+0000) 2024-03-06T09:20:28.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:28.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:28.014648+0000) 2024-03-06T09:20:29.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:29.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:29.014799+0000) 2024-03-06T09:20:30.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:30.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:30.014993+0000) 2024-03-06T09:20:31.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:31.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:31.015216+0000) 2024-03-06T09:20:32.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:32.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:32.015440+0000) 2024-03-06T09:20:33.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:33.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:33.015640+0000) 2024-03-06T09:20:33.545 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:20:33.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:20:34.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:34.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:34.015882+0000) 2024-03-06T09:20:35.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:35.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:35.016106+0000) 2024-03-06T09:20:36.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:36.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:36.016332+0000) 2024-03-06T09:20:37.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:37.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:37.016622+0000) 2024-03-06T09:20:38.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:38.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:38.016833+0000) 2024-03-06T09:20:39.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:39.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:39.017055+0000) 2024-03-06T09:20:40.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:40.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:40.017214+0000) 2024-03-06T09:20:41.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:41.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:41.017448+0000) 2024-03-06T09:20:42.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:42.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:42.017687+0000) 2024-03-06T09:20:43.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:43.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:43.017874+0000) 2024-03-06T09:20:44.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:44.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:44.018076+0000) 2024-03-06T09:20:45.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:45.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:45.018261+0000) 2024-03-06T09:20:46.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:46.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:46.018492+0000) 2024-03-06T09:20:47.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:47.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:47.018685+0000) 2024-03-06T09:20:48.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:48.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:48.018877+0000) 2024-03-06T09:20:49.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:49.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:49.019101+0000) 2024-03-06T09:20:50.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:50.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:50.019394+0000) 2024-03-06T09:20:51.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:51.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:51.019647+0000) 2024-03-06T09:20:52.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:52.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:52.019876+0000) 2024-03-06T09:20:53.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:53.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:53.020070+0000) 2024-03-06T09:20:54.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:54.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:54.020305+0000) 2024-03-06T09:20:55.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:55.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:55.020505+0000) 2024-03-06T09:20:56.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:56.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:56.020751+0000) 2024-03-06T09:20:57.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:57.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:57.020963+0000) 2024-03-06T09:20:58.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:58.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:58.021169+0000) 2024-03-06T09:20:59.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:20:59.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:20:59.021372+0000) 2024-03-06T09:21:00.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:00.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:00.021643+0000) 2024-03-06T09:21:01.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:01.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:01.021834+0000) 2024-03-06T09:21:02.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:02.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:02.022048+0000) 2024-03-06T09:21:03.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:03.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:03.022283+0000) 2024-03-06T09:21:03.760 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:21:03.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:21:04.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:04.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:04.022518+0000) 2024-03-06T09:21:05.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:05.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:05.022738+0000) 2024-03-06T09:21:06.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:06.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:06.022932+0000) 2024-03-06T09:21:07.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:07.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:07.023124+0000) 2024-03-06T09:21:08.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:08.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:08.023306+0000) 2024-03-06T09:21:09.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:09.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:09.023527+0000) 2024-03-06T09:21:10.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:10.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:10.023751+0000) 2024-03-06T09:21:11.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:11.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:11.023932+0000) 2024-03-06T09:21:12.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:12.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:12.024208+0000) 2024-03-06T09:21:13.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:13.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:13.024482+0000) 2024-03-06T09:21:14.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:14.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:14.024767+0000) 2024-03-06T09:21:15.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:15.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:15.025028+0000) 2024-03-06T09:21:16.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:16.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:16.025213+0000) 2024-03-06T09:21:17.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:17.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:17.025487+0000) 2024-03-06T09:21:18.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:18.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:18.025754+0000) 2024-03-06T09:21:19.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:19.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:19.025959+0000) 2024-03-06T09:21:20.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:20.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:20.026201+0000) 2024-03-06T09:21:21.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:21.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:21.026469+0000) 2024-03-06T09:21:22.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:22.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:22.026674+0000) 2024-03-06T09:21:23.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:23.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:23.026933+0000) 2024-03-06T09:21:24.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:24.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:24.027167+0000) 2024-03-06T09:21:25.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:25.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:25.027394+0000) 2024-03-06T09:21:26.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:26.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:26.027648+0000) 2024-03-06T09:21:27.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:27.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:27.027864+0000) 2024-03-06T09:21:28.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:28.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:28.028048+0000) 2024-03-06T09:21:29.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:29.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:29.028221+0000) 2024-03-06T09:21:30.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:30.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:30.028429+0000) 2024-03-06T09:21:31.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:31.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:31.028711+0000) 2024-03-06T09:21:32.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:32.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:32.028944+0000) 2024-03-06T09:21:33.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:33.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:33.029186+0000) 2024-03-06T09:21:33.971 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:21:33.980 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:21:34.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:34.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:34.029384+0000) 2024-03-06T09:21:35.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:35.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:35.029570+0000) 2024-03-06T09:21:36.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:36.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:36.029766+0000) 2024-03-06T09:21:37.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:37.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:37.029965+0000) 2024-03-06T09:21:38.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:38.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:38.030243+0000) 2024-03-06T09:21:39.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:39.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:39.030423+0000) 2024-03-06T09:21:40.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:40.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:40.030679+0000) 2024-03-06T09:21:41.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:41.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:41.030896+0000) 2024-03-06T09:21:42.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:42.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:42.031086+0000) 2024-03-06T09:21:43.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:43.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:43.031308+0000) 2024-03-06T09:21:44.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:44.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:44.031556+0000) 2024-03-06T09:21:45.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:45.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:45.031783+0000) 2024-03-06T09:21:46.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:46.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:46.032002+0000) 2024-03-06T09:21:47.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:47.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:47.032220+0000) 2024-03-06T09:21:48.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:48.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:48.032448+0000) 2024-03-06T09:21:49.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:49.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:49.032666+0000) 2024-03-06T09:21:50.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:50.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:50.032919+0000) 2024-03-06T09:21:51.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:51.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:51.033195+0000) 2024-03-06T09:21:52.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:52.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:52.033429+0000) 2024-03-06T09:21:53.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:53.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:53.033656+0000) 2024-03-06T09:21:54.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:54.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:54.033872+0000) 2024-03-06T09:21:55.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:55.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:55.034142+0000) 2024-03-06T09:21:56.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:56.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:56.034423+0000) 2024-03-06T09:21:57.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:57.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:57.034660+0000) 2024-03-06T09:21:58.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:58.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:58.034909+0000) 2024-03-06T09:21:59.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:21:59.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:21:59.035133+0000) 2024-03-06T09:22:00.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:00.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:00.035362+0000) 2024-03-06T09:22:01.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:01.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:01.035679+0000) 2024-03-06T09:22:02.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:02.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:02.035889+0000) 2024-03-06T09:22:03.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:03.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:03.036113+0000) 2024-03-06T09:22:04.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:04.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:04.036358+0000) 2024-03-06T09:22:04.192 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:22:04.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:22:05.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:05.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:05.036619+0000) 2024-03-06T09:22:06.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:06.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:06.036883+0000) 2024-03-06T09:22:07.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:07.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:07.037118+0000) 2024-03-06T09:22:08.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:08.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:08.037339+0000) 2024-03-06T09:22:09.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:09.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:09.037600+0000) 2024-03-06T09:22:10.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:10.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:10.037860+0000) 2024-03-06T09:22:11.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:11.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:11.038137+0000) 2024-03-06T09:22:12.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:12.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:12.038413+0000) 2024-03-06T09:22:13.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:13.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:13.038649+0000) 2024-03-06T09:22:14.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:14.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:14.038915+0000) 2024-03-06T09:22:15.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:15.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:15.039195+0000) 2024-03-06T09:22:16.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:16.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:16.039477+0000) 2024-03-06T09:22:17.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:17.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:17.039689+0000) 2024-03-06T09:22:18.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:18.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:18.039949+0000) 2024-03-06T09:22:19.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:19.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:19.040224+0000) 2024-03-06T09:22:20.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:20.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:20.040434+0000) 2024-03-06T09:22:21.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:21.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:21.040640+0000) 2024-03-06T09:22:22.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:22.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:22.040836+0000) 2024-03-06T09:22:23.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:23.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:23.041049+0000) 2024-03-06T09:22:24.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:24.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:24.041337+0000) 2024-03-06T09:22:25.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:25.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:25.041644+0000) 2024-03-06T09:22:26.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:26.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:26.041915+0000) 2024-03-06T09:22:27.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:27.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:27.042154+0000) 2024-03-06T09:22:28.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:28.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:28.042418+0000) 2024-03-06T09:22:29.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:29.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:29.042702+0000) 2024-03-06T09:22:30.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:30.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:30.042981+0000) 2024-03-06T09:22:31.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:31.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:31.043299+0000) 2024-03-06T09:22:32.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:32.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:32.043609+0000) 2024-03-06T09:22:33.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:33.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:33.043890+0000) 2024-03-06T09:22:34.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:34.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:34.044177+0000) 2024-03-06T09:22:34.332 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:22:34.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:22:35.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:35.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:35.044461+0000) 2024-03-06T09:22:36.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:36.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:36.044735+0000) 2024-03-06T09:22:37.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:37.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:37.045003+0000) 2024-03-06T09:22:38.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:38.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:38.045208+0000) 2024-03-06T09:22:39.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:39.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:39.045428+0000) 2024-03-06T09:22:40.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:40.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:40.045657+0000) 2024-03-06T09:22:41.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:41.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:41.045872+0000) 2024-03-06T09:22:42.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:42.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:42.046064+0000) 2024-03-06T09:22:43.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:43.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:43.046282+0000) 2024-03-06T09:22:44.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:44.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:44.046520+0000) 2024-03-06T09:22:45.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:45.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:45.046775+0000) 2024-03-06T09:22:46.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:46.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:46.047002+0000) 2024-03-06T09:22:47.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:47.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:47.047226+0000) 2024-03-06T09:22:48.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:48.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:48.047456+0000) 2024-03-06T09:22:49.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:49.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:49.047730+0000) 2024-03-06T09:22:50.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:50.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:50.047998+0000) 2024-03-06T09:22:51.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:51.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:51.048272+0000) 2024-03-06T09:22:52.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:52.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:52.048584+0000) 2024-03-06T09:22:53.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:53.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:53.048799+0000) 2024-03-06T09:22:54.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:54.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:54.049017+0000) 2024-03-06T09:22:55.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:55.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:55.049236+0000) 2024-03-06T09:22:56.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:56.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:56.049460+0000) 2024-03-06T09:22:57.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:57.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:57.049661+0000) 2024-03-06T09:22:58.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:58.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:58.049905+0000) 2024-03-06T09:22:59.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:22:59.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:22:59.050129+0000) 2024-03-06T09:23:00.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:00.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:00.050351+0000) 2024-03-06T09:23:01.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:01.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:01.050566+0000) 2024-03-06T09:23:02.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:02.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:02.050798+0000) 2024-03-06T09:23:03.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:03.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:03.051025+0000) 2024-03-06T09:23:04.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:04.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:04.051260+0000) 2024-03-06T09:23:04.536 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:23:04.544 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:23:05.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:05.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:05.051481+0000) 2024-03-06T09:23:06.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:06.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:06.051712+0000) 2024-03-06T09:23:07.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:07.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:07.051998+0000) 2024-03-06T09:23:08.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:08.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:08.052280+0000) 2024-03-06T09:23:09.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:09.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:09.052524+0000) 2024-03-06T09:23:10.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:10.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:10.052819+0000) 2024-03-06T09:23:11.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:11.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:11.053056+0000) 2024-03-06T09:23:12.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:12.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:12.053281+0000) 2024-03-06T09:23:13.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:13.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:13.053572+0000) 2024-03-06T09:23:14.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:14.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:14.053850+0000) 2024-03-06T09:23:15.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:15.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:15.054074+0000) 2024-03-06T09:23:16.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:16.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:16.054350+0000) 2024-03-06T09:23:17.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:17.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:17.054625+0000) 2024-03-06T09:23:18.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:18.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:18.054830+0000) 2024-03-06T09:23:19.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:19.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:19.055102+0000) 2024-03-06T09:23:20.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:20.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:20.055332+0000) 2024-03-06T09:23:21.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:21.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:21.055589+0000) 2024-03-06T09:23:22.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:22.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:22.055811+0000) 2024-03-06T09:23:23.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:23.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:23.056038+0000) 2024-03-06T09:23:24.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:24.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:24.056271+0000) 2024-03-06T09:23:25.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:25.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:25.056559+0000) 2024-03-06T09:23:26.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:26.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:26.056829+0000) 2024-03-06T09:23:27.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:27.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:27.057048+0000) 2024-03-06T09:23:28.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:28.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:28.057285+0000) 2024-03-06T09:23:29.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:29.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:29.057510+0000) 2024-03-06T09:23:30.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:30.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:30.057775+0000) 2024-03-06T09:23:31.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:31.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:31.058044+0000) 2024-03-06T09:23:32.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:32.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:32.058321+0000) 2024-03-06T09:23:33.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:33.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:33.058523+0000) 2024-03-06T09:23:34.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:34.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:34.058799+0000) 2024-03-06T09:23:34.731 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:23:34.739 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:23:35.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:35.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:35.059033+0000) 2024-03-06T09:23:36.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:36.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:36.059243+0000) 2024-03-06T09:23:37.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:37.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:37.059446+0000) 2024-03-06T09:23:38.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:38.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:38.059691+0000) 2024-03-06T09:23:39.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:39.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:39.059884+0000) 2024-03-06T09:23:40.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:40.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:40.060113+0000) 2024-03-06T09:23:41.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:41.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:41.060345+0000) 2024-03-06T09:23:42.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:42.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:42.060608+0000) 2024-03-06T09:23:43.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:43.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:43.060819+0000) 2024-03-06T09:23:44.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:44.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:44.061108+0000) 2024-03-06T09:23:45.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:45.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:45.061388+0000) 2024-03-06T09:23:46.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:46.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:46.061684+0000) 2024-03-06T09:23:47.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:47.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:47.061894+0000) 2024-03-06T09:23:48.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:48.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:48.062139+0000) 2024-03-06T09:23:49.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:49.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:49.062368+0000) 2024-03-06T09:23:50.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:50.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:50.062644+0000) 2024-03-06T09:23:51.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:51.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:51.062910+0000) 2024-03-06T09:23:52.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:52.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:52.063186+0000) 2024-03-06T09:23:53.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:53.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:53.063403+0000) 2024-03-06T09:23:54.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:54.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:54.063663+0000) 2024-03-06T09:23:55.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:55.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:55.063895+0000) 2024-03-06T09:23:56.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:56.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:56.064126+0000) 2024-03-06T09:23:57.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:57.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:57.064406+0000) 2024-03-06T09:23:58.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:58.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:58.064678+0000) 2024-03-06T09:23:59.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:23:59.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:23:59.064920+0000) 2024-03-06T09:24:00.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:00.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:00.065162+0000) 2024-03-06T09:24:01.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:01.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:01.065366+0000) 2024-03-06T09:24:02.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:02.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:02.065628+0000) 2024-03-06T09:24:03.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:03.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:03.065910+0000) 2024-03-06T09:24:04.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:04.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:04.066202+0000) 2024-03-06T09:24:04.927 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:24:04.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:24:05.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:05.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:05.066408+0000) 2024-03-06T09:24:06.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:06.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:06.066637+0000) 2024-03-06T09:24:07.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:07.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:07.066865+0000) 2024-03-06T09:24:08.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:08.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:08.067074+0000) 2024-03-06T09:24:09.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:09.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:09.067319+0000) 2024-03-06T09:24:10.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:10.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:10.067574+0000) 2024-03-06T09:24:11.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:11.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:11.067793+0000) 2024-03-06T09:24:12.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:12.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:12.068117+0000) 2024-03-06T09:24:13.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:13.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:13.068395+0000) 2024-03-06T09:24:14.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:14.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:14.068692+0000) 2024-03-06T09:24:15.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:15.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:15.068889+0000) 2024-03-06T09:24:16.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:16.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:16.069139+0000) 2024-03-06T09:24:17.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:17.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:17.069366+0000) 2024-03-06T09:24:18.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:18.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:18.069634+0000) 2024-03-06T09:24:19.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:19.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:19.069914+0000) 2024-03-06T09:24:20.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:20.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:20.070190+0000) 2024-03-06T09:24:21.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:21.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:21.070471+0000) 2024-03-06T09:24:22.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:22.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:22.070688+0000) 2024-03-06T09:24:23.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:23.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:23.070921+0000) 2024-03-06T09:24:24.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:24.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:24.071210+0000) 2024-03-06T09:24:25.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:25.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:25.071422+0000) 2024-03-06T09:24:26.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:26.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:26.071664+0000) 2024-03-06T09:24:27.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:27.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:27.071886+0000) 2024-03-06T09:24:28.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:28.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:28.072093+0000) 2024-03-06T09:24:29.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:29.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:29.072374+0000) 2024-03-06T09:24:30.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:30.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:30.072689+0000) 2024-03-06T09:24:31.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:31.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:31.072968+0000) 2024-03-06T09:24:32.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:32.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:32.073229+0000) 2024-03-06T09:24:33.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:33.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:33.073446+0000) 2024-03-06T09:24:34.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:34.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:34.073740+0000) 2024-03-06T09:24:35.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:35.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:35.074015+0000) 2024-03-06T09:24:35.132 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:24:35.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:24:36.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:36.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:36.074276+0000) 2024-03-06T09:24:37.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:37.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:37.074587+0000) 2024-03-06T09:24:38.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:38.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:38.074860+0000) 2024-03-06T09:24:39.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:39.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:39.075077+0000) 2024-03-06T09:24:40.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:40.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:40.075316+0000) 2024-03-06T09:24:41.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:41.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:41.075567+0000) 2024-03-06T09:24:42.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:42.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:42.075771+0000) 2024-03-06T09:24:43.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:43.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:43.075969+0000) 2024-03-06T09:24:44.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:44.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:44.076205+0000) 2024-03-06T09:24:45.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:45.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:45.076413+0000) 2024-03-06T09:24:46.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:46.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:46.076693+0000) 2024-03-06T09:24:47.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:47.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:47.076929+0000) 2024-03-06T09:24:48.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:48.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:48.077159+0000) 2024-03-06T09:24:49.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:49.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:49.077419+0000) 2024-03-06T09:24:50.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:50.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:50.077684+0000) 2024-03-06T09:24:51.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:51.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:51.077960+0000) 2024-03-06T09:24:52.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:52.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:52.078243+0000) 2024-03-06T09:24:53.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:53.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:53.078525+0000) 2024-03-06T09:24:54.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:54.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:54.078868+0000) 2024-03-06T09:24:55.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:55.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:55.079078+0000) 2024-03-06T09:24:56.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:56.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:56.079304+0000) 2024-03-06T09:24:57.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:57.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:57.079565+0000) 2024-03-06T09:24:58.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:58.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:58.079834+0000) 2024-03-06T09:24:59.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:24:59.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:24:59.080115+0000) 2024-03-06T09:25:00.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:00.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:00.080354+0000) 2024-03-06T09:25:01.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:01.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:01.080674+0000) 2024-03-06T09:25:02.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:02.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:02.080876+0000) 2024-03-06T09:25:03.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:03.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:03.081107+0000) 2024-03-06T09:25:04.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:04.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:04.081319+0000) 2024-03-06T09:25:05.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:05.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:05.081601+0000) 2024-03-06T09:25:05.337 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:25:05.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:25:06.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:06.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:06.081882+0000) 2024-03-06T09:25:07.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:07.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:07.082110+0000) 2024-03-06T09:25:08.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:08.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:08.082341+0000) 2024-03-06T09:25:09.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:09.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:09.082601+0000) 2024-03-06T09:25:10.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:10.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:10.082768+0000) 2024-03-06T09:25:11.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:11.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:11.082989+0000) 2024-03-06T09:25:12.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:12.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:12.083244+0000) 2024-03-06T09:25:13.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:13.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:13.083519+0000) 2024-03-06T09:25:14.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:14.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:14.083886+0000) 2024-03-06T09:25:15.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:15.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:15.084164+0000) 2024-03-06T09:25:16.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:16.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:16.084436+0000) 2024-03-06T09:25:17.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:17.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:17.084659+0000) 2024-03-06T09:25:18.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:18.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:18.084893+0000) 2024-03-06T09:25:19.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:19.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:19.085101+0000) 2024-03-06T09:25:20.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:20.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:20.085309+0000) 2024-03-06T09:25:21.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:21.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:21.085507+0000) 2024-03-06T09:25:22.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:22.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:22.085778+0000) 2024-03-06T09:25:23.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:23.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:23.085974+0000) 2024-03-06T09:25:24.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:24.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:24.086189+0000) 2024-03-06T09:25:25.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:25.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:25.086433+0000) 2024-03-06T09:25:26.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:26.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:26.086660+0000) 2024-03-06T09:25:27.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:27.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:27.086884+0000) 2024-03-06T09:25:28.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:28.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:28.087077+0000) 2024-03-06T09:25:29.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:29.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:29.087298+0000) 2024-03-06T09:25:30.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:30.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:30.087522+0000) 2024-03-06T09:25:31.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:31.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:31.087782+0000) 2024-03-06T09:25:32.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:32.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:32.088010+0000) 2024-03-06T09:25:33.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:33.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:33.088208+0000) 2024-03-06T09:25:34.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:34.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:34.088436+0000) 2024-03-06T09:25:35.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:35.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:35.088666+0000) 2024-03-06T09:25:35.466 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:25:35.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:25:36.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:36.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:36.088865+0000) 2024-03-06T09:25:37.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:37.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:37.089106+0000) 2024-03-06T09:25:38.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:38.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:38.089350+0000) 2024-03-06T09:25:39.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:39.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:39.089622+0000) 2024-03-06T09:25:40.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:40.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:40.089827+0000) 2024-03-06T09:25:41.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:41.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:41.090071+0000) 2024-03-06T09:25:42.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:42.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:42.090297+0000) 2024-03-06T09:25:43.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:43.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:43.090567+0000) 2024-03-06T09:25:44.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:44.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:44.090789+0000) 2024-03-06T09:25:45.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:45.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:45.091001+0000) 2024-03-06T09:25:46.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:46.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:46.091195+0000) 2024-03-06T09:25:47.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:47.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:47.091411+0000) 2024-03-06T09:25:48.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:48.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:48.091659+0000) 2024-03-06T09:25:49.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:49.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:49.091883+0000) 2024-03-06T09:25:50.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:50.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:50.092110+0000) 2024-03-06T09:25:51.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:51.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:51.092333+0000) 2024-03-06T09:25:52.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:52.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:52.092518+0000) 2024-03-06T09:25:53.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:53.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:53.092755+0000) 2024-03-06T09:25:54.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:54.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:54.093027+0000) 2024-03-06T09:25:55.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:55.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:55.093305+0000) 2024-03-06T09:25:56.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:56.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:56.093527+0000) 2024-03-06T09:25:57.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:57.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:57.093835+0000) 2024-03-06T09:25:58.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:58.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:58.094039+0000) 2024-03-06T09:25:59.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:25:59.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:25:59.094260+0000) 2024-03-06T09:26:00.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:00.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:00.094484+0000) 2024-03-06T09:26:01.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:01.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:01.094709+0000) 2024-03-06T09:26:02.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:02.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:02.094857+0000) 2024-03-06T09:26:03.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:03.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:03.095093+0000) 2024-03-06T09:26:04.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:04.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:04.095334+0000) 2024-03-06T09:26:05.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:05.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:05.095565+0000) 2024-03-06T09:26:05.679 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:26:05.686 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:26:06.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:06.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:06.095754+0000) 2024-03-06T09:26:07.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:07.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:07.095946+0000) 2024-03-06T09:26:08.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:08.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:08.096155+0000) 2024-03-06T09:26:09.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:09.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:09.096361+0000) 2024-03-06T09:26:10.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:10.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:10.096644+0000) 2024-03-06T09:26:11.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:11.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:11.096873+0000) 2024-03-06T09:26:12.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:12.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:12.097103+0000) 2024-03-06T09:26:13.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:13.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:13.097330+0000) 2024-03-06T09:26:14.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:14.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:14.097585+0000) 2024-03-06T09:26:15.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:15.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:15.097771+0000) 2024-03-06T09:26:16.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:16.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:16.097961+0000) 2024-03-06T09:26:17.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:17.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:17.098190+0000) 2024-03-06T09:26:18.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:18.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:18.098422+0000) 2024-03-06T09:26:19.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:19.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:19.098641+0000) 2024-03-06T09:26:20.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:20.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:20.098873+0000) 2024-03-06T09:26:21.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:21.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:21.099096+0000) 2024-03-06T09:26:22.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:22.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:22.099283+0000) 2024-03-06T09:26:23.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:23.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:23.099507+0000) 2024-03-06T09:26:24.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:24.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:24.099741+0000) 2024-03-06T09:26:25.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:25.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:25.099958+0000) 2024-03-06T09:26:26.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:26.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:26.100159+0000) 2024-03-06T09:26:27.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:27.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:27.100398+0000) 2024-03-06T09:26:28.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:28.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:28.100662+0000) 2024-03-06T09:26:29.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:29.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:29.100892+0000) 2024-03-06T09:26:30.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:30.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:30.101073+0000) 2024-03-06T09:26:31.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:31.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:31.101298+0000) 2024-03-06T09:26:32.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:32.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:32.101486+0000) 2024-03-06T09:26:33.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:33.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:33.101717+0000) 2024-03-06T09:26:34.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:34.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:34.101915+0000) 2024-03-06T09:26:35.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:35.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:35.102141+0000) 2024-03-06T09:26:35.875 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:26:35.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:26:36.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:36.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:36.102339+0000) 2024-03-06T09:26:37.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:37.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:37.102596+0000) 2024-03-06T09:26:38.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:38.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:38.102832+0000) 2024-03-06T09:26:39.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:39.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:39.103013+0000) 2024-03-06T09:26:40.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:40.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:40.103236+0000) 2024-03-06T09:26:41.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:41.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:41.103457+0000) 2024-03-06T09:26:42.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:42.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:42.103694+0000) 2024-03-06T09:26:43.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:43.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:43.103924+0000) 2024-03-06T09:26:44.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:44.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:44.104140+0000) 2024-03-06T09:26:45.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:45.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:45.104419+0000) 2024-03-06T09:26:46.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:46.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:46.104695+0000) 2024-03-06T09:26:47.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:47.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:47.105017+0000) 2024-03-06T09:26:48.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:48.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:48.105296+0000) 2024-03-06T09:26:49.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:49.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:49.105514+0000) 2024-03-06T09:26:50.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:50.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:50.105832+0000) 2024-03-06T09:26:51.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:51.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:51.106029+0000) 2024-03-06T09:26:52.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:52.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:52.106258+0000) 2024-03-06T09:26:53.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:53.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:53.106449+0000) 2024-03-06T09:26:54.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:54.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:54.106730+0000) 2024-03-06T09:26:55.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:55.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:55.107004+0000) 2024-03-06T09:26:56.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:56.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:56.107279+0000) 2024-03-06T09:26:57.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:57.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:57.107590+0000) 2024-03-06T09:26:58.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:58.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:58.107876+0000) 2024-03-06T09:26:59.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:26:59.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:26:59.108153+0000) 2024-03-06T09:27:00.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:00.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:00.108433+0000) 2024-03-06T09:27:01.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:01.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:01.108719+0000) 2024-03-06T09:27:02.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:02.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:02.108981+0000) 2024-03-06T09:27:03.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:03.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:03.109252+0000) 2024-03-06T09:27:04.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:04.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:04.109584+0000) 2024-03-06T09:27:05.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:05.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:05.109830+0000) 2024-03-06T09:27:06.088 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:27:06.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:27:06.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:06.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:06.109976+0000) 2024-03-06T09:27:07.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:07.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:07.110209+0000) 2024-03-06T09:27:08.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:08.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:08.110469+0000) 2024-03-06T09:27:09.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:09.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:09.110744+0000) 2024-03-06T09:27:10.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:10.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:10.111018+0000) 2024-03-06T09:27:11.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:11.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:11.111219+0000) 2024-03-06T09:27:12.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:12.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:12.111482+0000) 2024-03-06T09:27:13.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:13.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:13.111768+0000) 2024-03-06T09:27:14.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:14.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:14.112058+0000) 2024-03-06T09:27:15.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:15.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:15.112332+0000) 2024-03-06T09:27:16.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:16.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:16.112640+0000) 2024-03-06T09:27:17.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:17.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:17.112912+0000) 2024-03-06T09:27:18.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:18.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:18.113145+0000) 2024-03-06T09:27:19.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:19.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:19.113415+0000) 2024-03-06T09:27:20.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:20.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:20.113695+0000) 2024-03-06T09:27:21.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:21.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:21.113975+0000) 2024-03-06T09:27:22.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:22.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:22.114168+0000) 2024-03-06T09:27:23.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:23.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:23.114425+0000) 2024-03-06T09:27:24.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:24.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:24.114709+0000) 2024-03-06T09:27:25.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:25.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:25.114973+0000) 2024-03-06T09:27:26.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:26.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:26.115151+0000) 2024-03-06T09:27:27.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:27.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:27.115391+0000) 2024-03-06T09:27:28.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:28.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:28.115649+0000) 2024-03-06T09:27:29.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:29.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:29.115846+0000) 2024-03-06T09:27:30.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:30.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:30.116095+0000) 2024-03-06T09:27:31.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:31.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:31.116339+0000) 2024-03-06T09:27:32.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:32.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:32.116597+0000) 2024-03-06T09:27:33.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:33.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:33.116801+0000) 2024-03-06T09:27:34.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:34.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:34.117081+0000) 2024-03-06T09:27:35.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:35.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:35.117403+0000) 2024-03-06T09:27:36.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:36.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:36.117686+0000) 2024-03-06T09:27:36.285 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:27:36.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:27:37.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:37.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:37.117922+0000) 2024-03-06T09:27:38.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:38.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:38.118200+0000) 2024-03-06T09:27:39.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:39.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:39.118473+0000) 2024-03-06T09:27:40.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:40.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:40.118748+0000) 2024-03-06T09:27:41.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:41.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:41.118981+0000) 2024-03-06T09:27:42.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:42.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:42.119185+0000) 2024-03-06T09:27:43.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:43.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:43.119427+0000) 2024-03-06T09:27:44.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:44.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:44.119661+0000) 2024-03-06T09:27:45.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:45.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:45.119911+0000) 2024-03-06T09:27:46.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:46.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:46.120167+0000) 2024-03-06T09:27:47.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:47.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:47.120377+0000) 2024-03-06T09:27:48.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:48.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:48.120654+0000) 2024-03-06T09:27:49.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:49.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:49.120887+0000) 2024-03-06T09:27:50.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:50.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:50.121094+0000) 2024-03-06T09:27:51.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:51.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:51.121376+0000) 2024-03-06T09:27:52.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:52.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:52.121673+0000) 2024-03-06T09:27:53.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:53.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:53.121937+0000) 2024-03-06T09:27:54.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:54.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:54.122236+0000) 2024-03-06T09:27:55.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:55.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:55.122498+0000) 2024-03-06T09:27:56.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:56.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:56.122804+0000) 2024-03-06T09:27:57.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:57.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:57.123000+0000) 2024-03-06T09:27:58.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:58.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:58.123178+0000) 2024-03-06T09:27:59.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:27:59.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:27:59.123404+0000) 2024-03-06T09:28:00.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:00.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:00.123665+0000) 2024-03-06T09:28:01.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:01.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:01.123932+0000) 2024-03-06T09:28:02.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:02.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:02.124204+0000) 2024-03-06T09:28:03.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:03.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:03.124476+0000) 2024-03-06T09:28:04.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:04.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:04.124814+0000) 2024-03-06T09:28:05.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:05.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:05.125017+0000) 2024-03-06T09:28:06.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:06.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:06.125243+0000) 2024-03-06T09:28:06.490 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:28:06.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:28:07.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:07.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:07.125440+0000) 2024-03-06T09:28:08.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:08.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:08.125660+0000) 2024-03-06T09:28:09.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:09.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:09.125885+0000) 2024-03-06T09:28:10.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:10.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:10.126116+0000) 2024-03-06T09:28:11.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:11.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:11.126380+0000) 2024-03-06T09:28:12.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:12.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:12.126686+0000) 2024-03-06T09:28:13.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:13.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:13.126961+0000) 2024-03-06T09:28:14.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:14.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:14.127247+0000) 2024-03-06T09:28:15.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:15.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:15.127513+0000) 2024-03-06T09:28:16.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:16.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:16.127759+0000) 2024-03-06T09:28:17.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:17.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:17.127987+0000) 2024-03-06T09:28:18.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:18.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:18.128208+0000) 2024-03-06T09:28:19.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:19.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:19.128466+0000) 2024-03-06T09:28:20.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:20.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:20.128734+0000) 2024-03-06T09:28:21.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:21.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:21.129001+0000) 2024-03-06T09:28:22.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:22.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:22.129278+0000) 2024-03-06T09:28:23.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:23.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:23.129596+0000) 2024-03-06T09:28:24.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:24.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:24.129901+0000) 2024-03-06T09:28:25.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:25.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:25.130187+0000) 2024-03-06T09:28:26.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:26.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:26.130458+0000) 2024-03-06T09:28:27.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:27.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:27.130745+0000) 2024-03-06T09:28:28.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:28.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:28.131067+0000) 2024-03-06T09:28:29.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:29.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:29.131272+0000) 2024-03-06T09:28:30.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:30.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:30.131503+0000) 2024-03-06T09:28:31.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:31.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:31.131731+0000) 2024-03-06T09:28:32.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:32.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:32.131954+0000) 2024-03-06T09:28:33.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:33.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:33.132231+0000) 2024-03-06T09:28:34.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:34.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:34.132513+0000) 2024-03-06T09:28:35.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:35.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:35.132833+0000) 2024-03-06T09:28:36.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:36.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:36.133041+0000) 2024-03-06T09:28:36.718 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:28:36.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:28:37.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:37.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:37.133273+0000) 2024-03-06T09:28:38.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:38.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:38.133572+0000) 2024-03-06T09:28:39.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:39.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:39.133837+0000) 2024-03-06T09:28:40.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:40.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:40.134115+0000) 2024-03-06T09:28:41.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:41.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:41.134373+0000) 2024-03-06T09:28:42.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:42.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:42.134659+0000) 2024-03-06T09:28:43.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:43.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:43.134899+0000) 2024-03-06T09:28:44.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:44.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:44.135176+0000) 2024-03-06T09:28:45.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:45.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:45.135461+0000) 2024-03-06T09:28:46.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:46.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:46.135692+0000) 2024-03-06T09:28:47.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:47.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:47.135913+0000) 2024-03-06T09:28:48.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:48.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:48.136141+0000) 2024-03-06T09:28:49.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:49.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:49.136337+0000) 2024-03-06T09:28:50.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:50.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:50.136609+0000) 2024-03-06T09:28:51.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:51.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:51.136832+0000) 2024-03-06T09:28:52.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:52.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:52.137057+0000) 2024-03-06T09:28:53.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:53.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:53.137280+0000) 2024-03-06T09:28:54.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:54.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:54.137592+0000) 2024-03-06T09:28:55.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:55.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:55.137826+0000) 2024-03-06T09:28:56.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:56.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:56.138044+0000) 2024-03-06T09:28:57.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:57.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:57.138321+0000) 2024-03-06T09:28:58.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:58.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:58.138627+0000) 2024-03-06T09:28:59.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:28:59.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:28:59.138906+0000) 2024-03-06T09:29:00.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:00.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:00.139179+0000) 2024-03-06T09:29:01.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:01.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:01.139455+0000) 2024-03-06T09:29:02.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:02.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:02.139710+0000) 2024-03-06T09:29:03.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:03.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:03.139957+0000) 2024-03-06T09:29:04.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:04.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:04.140183+0000) 2024-03-06T09:29:05.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:05.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:05.140410+0000) 2024-03-06T09:29:06.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:06.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:06.140664+0000) 2024-03-06T09:29:06.957 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:29:06.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:29:07.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:07.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:07.140871+0000) 2024-03-06T09:29:08.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:08.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:08.141104+0000) 2024-03-06T09:29:09.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:09.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:09.141327+0000) 2024-03-06T09:29:10.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:10.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:10.141603+0000) 2024-03-06T09:29:11.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:11.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:11.141817+0000) 2024-03-06T09:29:12.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:12.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:12.142042+0000) 2024-03-06T09:29:13.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:13.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:13.142272+0000) 2024-03-06T09:29:14.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:14.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:14.142594+0000) 2024-03-06T09:29:15.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:15.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:15.142881+0000) 2024-03-06T09:29:16.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:16.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:16.143122+0000) 2024-03-06T09:29:17.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:17.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:17.143407+0000) 2024-03-06T09:29:18.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:18.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:18.143644+0000) 2024-03-06T09:29:19.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:19.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:19.143864+0000) 2024-03-06T09:29:20.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:20.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:20.144093+0000) 2024-03-06T09:29:21.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:21.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:21.144307+0000) 2024-03-06T09:29:22.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:22.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:22.144614+0000) 2024-03-06T09:29:23.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:23.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:23.144857+0000) 2024-03-06T09:29:24.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:24.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:24.145094+0000) 2024-03-06T09:29:25.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:25.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:25.145293+0000) 2024-03-06T09:29:26.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:26.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:26.145517+0000) 2024-03-06T09:29:27.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:27.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:27.145766+0000) 2024-03-06T09:29:28.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:28.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:28.146004+0000) 2024-03-06T09:29:29.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:29.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:29.146223+0000) 2024-03-06T09:29:30.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:30.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:30.146443+0000) 2024-03-06T09:29:31.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:31.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:31.146658+0000) 2024-03-06T09:29:32.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:32.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:32.146898+0000) 2024-03-06T09:29:33.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:33.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:33.147181+0000) 2024-03-06T09:29:34.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:34.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:34.147471+0000) 2024-03-06T09:29:35.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:35.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:35.147692+0000) 2024-03-06T09:29:36.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:36.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:36.147913+0000) 2024-03-06T09:29:37.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:37.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:37.148138+0000) 2024-03-06T09:29:37.170 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:29:37.178 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:29:38.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:38.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:38.148397+0000) 2024-03-06T09:29:39.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:39.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:39.148650+0000) 2024-03-06T09:29:40.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:40.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:40.148935+0000) 2024-03-06T09:29:41.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:41.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:41.149212+0000) 2024-03-06T09:29:42.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:42.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:42.149414+0000) 2024-03-06T09:29:43.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:43.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:43.149631+0000) 2024-03-06T09:29:44.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:44.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:44.149835+0000) 2024-03-06T09:29:45.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:45.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:45.150019+0000) 2024-03-06T09:29:46.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:46.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:46.150247+0000) 2024-03-06T09:29:47.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:47.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:47.150450+0000) 2024-03-06T09:29:48.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:48.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:48.150714+0000) 2024-03-06T09:29:49.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:49.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:49.150998+0000) 2024-03-06T09:29:50.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:50.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:50.151169+0000) 2024-03-06T09:29:51.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:51.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:51.151386+0000) 2024-03-06T09:29:52.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:52.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:52.151631+0000) 2024-03-06T09:29:53.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:53.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:53.151858+0000) 2024-03-06T09:29:54.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:54.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:54.152087+0000) 2024-03-06T09:29:55.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:55.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:55.152322+0000) 2024-03-06T09:29:56.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:56.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:56.152578+0000) 2024-03-06T09:29:57.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:57.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:57.152826+0000) 2024-03-06T09:29:58.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:58.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:58.153052+0000) 2024-03-06T09:29:59.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:29:59.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:29:59.153280+0000) 2024-03-06T09:30:00.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:00.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:00.153506+0000) 2024-03-06T09:30:01.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:01.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:01.153762+0000) 2024-03-06T09:30:02.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:02.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:02.153986+0000) 2024-03-06T09:30:03.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:03.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:03.154249+0000) 2024-03-06T09:30:04.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:04.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:04.154580+0000) 2024-03-06T09:30:05.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:05.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:05.154838+0000) 2024-03-06T09:30:06.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:06.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:06.155065+0000) 2024-03-06T09:30:07.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:07.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:07.155263+0000) 2024-03-06T09:30:07.368 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:30:07.376 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:30:08.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:08.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:08.155503+0000) 2024-03-06T09:30:09.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:09.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:09.155731+0000) 2024-03-06T09:30:10.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:10.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:10.156026+0000) 2024-03-06T09:30:11.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:11.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:11.156262+0000) 2024-03-06T09:30:12.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:12.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:12.156501+0000) 2024-03-06T09:30:13.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:13.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:13.156740+0000) 2024-03-06T09:30:14.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:14.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:14.156970+0000) 2024-03-06T09:30:15.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:15.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:15.157206+0000) 2024-03-06T09:30:16.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:16.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:16.157405+0000) 2024-03-06T09:30:17.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:17.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:17.157663+0000) 2024-03-06T09:30:18.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:18.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:18.157942+0000) 2024-03-06T09:30:19.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:19.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:19.158213+0000) 2024-03-06T09:30:20.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:20.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:20.158421+0000) 2024-03-06T09:30:21.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:21.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:21.158647+0000) 2024-03-06T09:30:22.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:22.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:22.158870+0000) 2024-03-06T09:30:23.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:23.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:23.159149+0000) 2024-03-06T09:30:24.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:24.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:24.159427+0000) 2024-03-06T09:30:25.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:25.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:25.159697+0000) 2024-03-06T09:30:26.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:26.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:26.159893+0000) 2024-03-06T09:30:27.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:27.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:27.160131+0000) 2024-03-06T09:30:28.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:28.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:28.160405+0000) 2024-03-06T09:30:29.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:29.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:29.160679+0000) 2024-03-06T09:30:30.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:30.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:30.160964+0000) 2024-03-06T09:30:31.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:31.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:31.161223+0000) 2024-03-06T09:30:32.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:32.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:32.161476+0000) 2024-03-06T09:30:33.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:33.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:33.161740+0000) 2024-03-06T09:30:34.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:34.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:34.162021+0000) 2024-03-06T09:30:35.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:35.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:35.162284+0000) 2024-03-06T09:30:36.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:36.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:36.162593+0000) 2024-03-06T09:30:37.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:37.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:37.162879+0000) 2024-03-06T09:30:37.505 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:30:37.512 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:30:38.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:38.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:38.163153+0000) 2024-03-06T09:30:39.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:39.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:39.163360+0000) 2024-03-06T09:30:40.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:40.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:40.163581+0000) 2024-03-06T09:30:41.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:41.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:41.163826+0000) 2024-03-06T09:30:42.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:42.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:42.164097+0000) 2024-03-06T09:30:43.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:43.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:43.164370+0000) 2024-03-06T09:30:44.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:44.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:44.164648+0000) 2024-03-06T09:30:45.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:45.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:45.164889+0000) 2024-03-06T09:30:46.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:46.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:46.165082+0000) 2024-03-06T09:30:47.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:47.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:47.165348+0000) 2024-03-06T09:30:48.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:48.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:48.165656+0000) 2024-03-06T09:30:49.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:49.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:49.165868+0000) 2024-03-06T09:30:50.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:50.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:50.166098+0000) 2024-03-06T09:30:51.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:51.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:51.166319+0000) 2024-03-06T09:30:52.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:52.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:52.166575+0000) 2024-03-06T09:30:53.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:53.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:53.166801+0000) 2024-03-06T09:30:54.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:54.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:54.167036+0000) 2024-03-06T09:30:55.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:55.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:55.167256+0000) 2024-03-06T09:30:56.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:56.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:56.167467+0000) 2024-03-06T09:30:57.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:57.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:57.167692+0000) 2024-03-06T09:30:58.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:58.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:58.167917+0000) 2024-03-06T09:30:59.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:30:59.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:30:59.168147+0000) 2024-03-06T09:31:00.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:00.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:00.168377+0000) 2024-03-06T09:31:01.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:01.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:01.168636+0000) 2024-03-06T09:31:02.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:02.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:02.168827+0000) 2024-03-06T09:31:03.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:03.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:03.169084+0000) 2024-03-06T09:31:04.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:04.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:04.169327+0000) 2024-03-06T09:31:05.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:05.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:05.169638+0000) 2024-03-06T09:31:06.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:06.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:06.169917+0000) 2024-03-06T09:31:07.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:07.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:07.170197+0000) 2024-03-06T09:31:07.708 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:31:07.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:31:08.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:08.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:08.170471+0000) 2024-03-06T09:31:09.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:09.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:09.170754+0000) 2024-03-06T09:31:10.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:10.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:10.171038+0000) 2024-03-06T09:31:11.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:11.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:11.171314+0000) 2024-03-06T09:31:12.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:12.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:12.171518+0000) 2024-03-06T09:31:13.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:13.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:13.171751+0000) 2024-03-06T09:31:14.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:14.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:14.172008+0000) 2024-03-06T09:31:15.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:15.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:15.172249+0000) 2024-03-06T09:31:16.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:16.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:16.172492+0000) 2024-03-06T09:31:17.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:17.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:17.172721+0000) 2024-03-06T09:31:18.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:18.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:18.172957+0000) 2024-03-06T09:31:19.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:19.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:19.173315+0000) 2024-03-06T09:31:20.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:20.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:20.173647+0000) 2024-03-06T09:31:21.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:21.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:21.173858+0000) 2024-03-06T09:31:22.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:22.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:22.174103+0000) 2024-03-06T09:31:23.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:23.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:23.174295+0000) 2024-03-06T09:31:24.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:24.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:24.174517+0000) 2024-03-06T09:31:25.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:25.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:25.174754+0000) 2024-03-06T09:31:26.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:26.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:26.174978+0000) 2024-03-06T09:31:27.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:27.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:27.175172+0000) 2024-03-06T09:31:28.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:28.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:28.175353+0000) 2024-03-06T09:31:29.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:29.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:29.175559+0000) 2024-03-06T09:31:30.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:30.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:30.175744+0000) 2024-03-06T09:31:31.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:31.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:31.175921+0000) 2024-03-06T09:31:32.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:32.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:32.176099+0000) 2024-03-06T09:31:33.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:33.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:33.176280+0000) 2024-03-06T09:31:34.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:34.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:34.176498+0000) 2024-03-06T09:31:35.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:35.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:35.176738+0000) 2024-03-06T09:31:36.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:36.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:36.176943+0000) 2024-03-06T09:31:37.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:37.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:37.177142+0000) 2024-03-06T09:31:37.955 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:31:37.962 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:31:38.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:38.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:38.177333+0000) 2024-03-06T09:31:39.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:39.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:39.177646+0000) 2024-03-06T09:31:40.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:40.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:40.177856+0000) 2024-03-06T09:31:41.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:41.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:41.178052+0000) 2024-03-06T09:31:42.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:42.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:42.178240+0000) 2024-03-06T09:31:43.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:43.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:43.178466+0000) 2024-03-06T09:31:44.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:44.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:44.178660+0000) 2024-03-06T09:31:45.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:45.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:45.178839+0000) 2024-03-06T09:31:46.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:46.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:46.179055+0000) 2024-03-06T09:31:47.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:47.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:47.179270+0000) 2024-03-06T09:31:48.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:48.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:48.179469+0000) 2024-03-06T09:31:49.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:49.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:49.179727+0000) 2024-03-06T09:31:50.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:50.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:50.179952+0000) 2024-03-06T09:31:51.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:51.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:51.180223+0000) 2024-03-06T09:31:52.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:52.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:52.180445+0000) 2024-03-06T09:31:53.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:53.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:53.180644+0000) 2024-03-06T09:31:54.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:54.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:54.180835+0000) 2024-03-06T09:31:55.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:55.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:55.181034+0000) 2024-03-06T09:31:56.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:56.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:56.181223+0000) 2024-03-06T09:31:57.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:57.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:57.181417+0000) 2024-03-06T09:31:58.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:58.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:58.181667+0000) 2024-03-06T09:31:59.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:31:59.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:31:59.181879+0000) 2024-03-06T09:32:00.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:00.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:00.182127+0000) 2024-03-06T09:32:01.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:01.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:01.182354+0000) 2024-03-06T09:32:02.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:02.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:02.182637+0000) 2024-03-06T09:32:03.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:03.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:03.182848+0000) 2024-03-06T09:32:04.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:04.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:04.183070+0000) 2024-03-06T09:32:05.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:05.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:05.183316+0000) 2024-03-06T09:32:06.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:06.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:06.183599+0000) 2024-03-06T09:32:07.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:07.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:07.183810+0000) 2024-03-06T09:32:08.180 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:32:08.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:08.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:08.184049+0000) 2024-03-06T09:32:08.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:32:09.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:09.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:09.184286+0000) 2024-03-06T09:32:10.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:10.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:10.184525+0000) 2024-03-06T09:32:11.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:11.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:11.184804+0000) 2024-03-06T09:32:12.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:12.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:12.185032+0000) 2024-03-06T09:32:13.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:13.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:13.185272+0000) 2024-03-06T09:32:14.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:14.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:14.185506+0000) 2024-03-06T09:32:15.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:15.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:15.185742+0000) 2024-03-06T09:32:16.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:16.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:16.185961+0000) 2024-03-06T09:32:17.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:17.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:17.186184+0000) 2024-03-06T09:32:18.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:18.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:18.186408+0000) 2024-03-06T09:32:19.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:19.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:19.186637+0000) 2024-03-06T09:32:20.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:20.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:20.186832+0000) 2024-03-06T09:32:21.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:21.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:21.187057+0000) 2024-03-06T09:32:22.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:22.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:22.187278+0000) 2024-03-06T09:32:23.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:23.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:23.187456+0000) 2024-03-06T09:32:24.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:24.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:24.187714+0000) 2024-03-06T09:32:25.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:25.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:25.187948+0000) 2024-03-06T09:32:26.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:26.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:26.188177+0000) 2024-03-06T09:32:27.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:27.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:27.188358+0000) 2024-03-06T09:32:28.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:28.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:28.188634+0000) 2024-03-06T09:32:29.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:29.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:29.188860+0000) 2024-03-06T09:32:30.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:30.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:30.189095+0000) 2024-03-06T09:32:31.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:31.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:31.189320+0000) 2024-03-06T09:32:32.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:32.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:32.189567+0000) 2024-03-06T09:32:33.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:33.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:33.189789+0000) 2024-03-06T09:32:34.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:34.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:34.190031+0000) 2024-03-06T09:32:35.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:35.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:35.190214+0000) 2024-03-06T09:32:36.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:36.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:36.190460+0000) 2024-03-06T09:32:37.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:37.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:37.190689+0000) 2024-03-06T09:32:38.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:38.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:38.190904+0000) 2024-03-06T09:32:38.374 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:32:38.382 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:32:39.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:39.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:39.191155+0000) 2024-03-06T09:32:40.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:40.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:40.191422+0000) 2024-03-06T09:32:41.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:41.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:41.191656+0000) 2024-03-06T09:32:42.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:42.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:42.191843+0000) 2024-03-06T09:32:43.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:43.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:43.192079+0000) 2024-03-06T09:32:44.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:44.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:44.192325+0000) 2024-03-06T09:32:45.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:45.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:45.192505+0000) 2024-03-06T09:32:46.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:46.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:46.192761+0000) 2024-03-06T09:32:47.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:47.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:47.192954+0000) 2024-03-06T09:32:48.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:48.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:48.193177+0000) 2024-03-06T09:32:49.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:49.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:49.193401+0000) 2024-03-06T09:32:50.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:50.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:50.193654+0000) 2024-03-06T09:32:51.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:51.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:51.193848+0000) 2024-03-06T09:32:52.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:52.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:52.194072+0000) 2024-03-06T09:32:53.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:53.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:53.194295+0000) 2024-03-06T09:32:54.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:54.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:54.194566+0000) 2024-03-06T09:32:55.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:55.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:55.194802+0000) 2024-03-06T09:32:56.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:56.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:56.195025+0000) 2024-03-06T09:32:57.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:57.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:57.195250+0000) 2024-03-06T09:32:58.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:58.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:58.195475+0000) 2024-03-06T09:32:59.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:32:59.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:32:59.195698+0000) 2024-03-06T09:33:00.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:00.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:00.195893+0000) 2024-03-06T09:33:01.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:01.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:01.196079+0000) 2024-03-06T09:33:02.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:02.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:02.196297+0000) 2024-03-06T09:33:03.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:03.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:03.196516+0000) 2024-03-06T09:33:04.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:04.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:04.196745+0000) 2024-03-06T09:33:05.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:05.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:05.196929+0000) 2024-03-06T09:33:06.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:06.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:06.197116+0000) 2024-03-06T09:33:07.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:07.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:07.197335+0000) 2024-03-06T09:33:08.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:08.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:08.197569+0000) 2024-03-06T09:33:08.596 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:33:08.604 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:33:09.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:09.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:09.197764+0000) 2024-03-06T09:33:10.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:10.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:10.197997+0000) 2024-03-06T09:33:11.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:11.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:11.198229+0000) 2024-03-06T09:33:12.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:12.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:12.198455+0000) 2024-03-06T09:33:13.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:13.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:13.198655+0000) 2024-03-06T09:33:14.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:14.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:14.198836+0000) 2024-03-06T09:33:15.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:15.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:15.199052+0000) 2024-03-06T09:33:16.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:16.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:16.199323+0000) 2024-03-06T09:33:17.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:17.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:17.199515+0000) 2024-03-06T09:33:18.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:18.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:18.199757+0000) 2024-03-06T09:33:19.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:19.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:19.199994+0000) 2024-03-06T09:33:20.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:20.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:20.200223+0000) 2024-03-06T09:33:21.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:21.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:21.200441+0000) 2024-03-06T09:33:22.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:22.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:22.200669+0000) 2024-03-06T09:33:23.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:23.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:23.200892+0000) 2024-03-06T09:33:24.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:24.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:24.201118+0000) 2024-03-06T09:33:25.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:25.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:25.201344+0000) 2024-03-06T09:33:26.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:26.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:26.201570+0000) 2024-03-06T09:33:27.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:27.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:27.201770+0000) 2024-03-06T09:33:28.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:28.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:28.202011+0000) 2024-03-06T09:33:29.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:29.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:29.202252+0000) 2024-03-06T09:33:30.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:30.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:30.202501+0000) 2024-03-06T09:33:31.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:31.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:31.202776+0000) 2024-03-06T09:33:32.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:32.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:32.203025+0000) 2024-03-06T09:33:33.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:33.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:33.203306+0000) 2024-03-06T09:33:34.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:34.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:34.203635+0000) 2024-03-06T09:33:35.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:35.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:35.203915+0000) 2024-03-06T09:33:36.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:36.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:36.204195+0000) 2024-03-06T09:33:37.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:37.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:37.204472+0000) 2024-03-06T09:33:38.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:38.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:38.204766+0000) 2024-03-06T09:33:38.815 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:33:38.824 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:33:39.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:39.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:39.204926+0000) 2024-03-06T09:33:40.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:40.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:40.205152+0000) 2024-03-06T09:33:41.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:41.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:41.205368+0000) 2024-03-06T09:33:42.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:42.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:42.205599+0000) 2024-03-06T09:33:43.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:43.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:43.205813+0000) 2024-03-06T09:33:44.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:44.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:44.206066+0000) 2024-03-06T09:33:45.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:45.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:45.206275+0000) 2024-03-06T09:33:46.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:46.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:46.206526+0000) 2024-03-06T09:33:47.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:47.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:47.206766+0000) 2024-03-06T09:33:48.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:48.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:48.206955+0000) 2024-03-06T09:33:49.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:49.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:49.207125+0000) 2024-03-06T09:33:50.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:50.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:50.207310+0000) 2024-03-06T09:33:51.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:51.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:51.207482+0000) 2024-03-06T09:33:52.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:52.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:52.207645+0000) 2024-03-06T09:33:53.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:53.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:53.207841+0000) 2024-03-06T09:33:54.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:54.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:54.208042+0000) 2024-03-06T09:33:55.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:55.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:55.208229+0000) 2024-03-06T09:33:56.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:56.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:56.208454+0000) 2024-03-06T09:33:57.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:57.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:57.208699+0000) 2024-03-06T09:33:58.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:58.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:58.208880+0000) 2024-03-06T09:33:59.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:33:59.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:33:59.209061+0000) 2024-03-06T09:34:00.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:00.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:00.209263+0000) 2024-03-06T09:34:01.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:01.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:01.209465+0000) 2024-03-06T09:34:02.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:02.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:02.209691+0000) 2024-03-06T09:34:03.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:03.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:03.209877+0000) 2024-03-06T09:34:04.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:04.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:04.210077+0000) 2024-03-06T09:34:05.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:05.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:05.210304+0000) 2024-03-06T09:34:06.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:06.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:06.210528+0000) 2024-03-06T09:34:07.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:07.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:07.210765+0000) 2024-03-06T09:34:08.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:08.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:08.210987+0000) 2024-03-06T09:34:09.163 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:34:09.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:34:09.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:09.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:09.211138+0000) 2024-03-06T09:34:10.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:10.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:10.211358+0000) 2024-03-06T09:34:11.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:11.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:11.211627+0000) 2024-03-06T09:34:12.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:12.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:12.211813+0000) 2024-03-06T09:34:13.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:13.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:13.212045+0000) 2024-03-06T09:34:14.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:14.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:14.212228+0000) 2024-03-06T09:34:15.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:15.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:15.212412+0000) 2024-03-06T09:34:16.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:16.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:16.212639+0000) 2024-03-06T09:34:17.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:17.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:17.212902+0000) 2024-03-06T09:34:18.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:18.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:18.213178+0000) 2024-03-06T09:34:19.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:19.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:19.213384+0000) 2024-03-06T09:34:20.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:20.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:20.213652+0000) 2024-03-06T09:34:21.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:21.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:21.213876+0000) 2024-03-06T09:34:22.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:22.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:22.214140+0000) 2024-03-06T09:34:23.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:23.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:23.214348+0000) 2024-03-06T09:34:24.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:24.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:24.214596+0000) 2024-03-06T09:34:25.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:25.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:25.214872+0000) 2024-03-06T09:34:26.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:26.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:26.215198+0000) 2024-03-06T09:34:27.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:27.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:27.215476+0000) 2024-03-06T09:34:28.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:28.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:28.215758+0000) 2024-03-06T09:34:29.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:29.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:29.216034+0000) 2024-03-06T09:34:30.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:30.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:30.216315+0000) 2024-03-06T09:34:31.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:31.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:31.216619+0000) 2024-03-06T09:34:32.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:32.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:32.216790+0000) 2024-03-06T09:34:33.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:33.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:33.217005+0000) 2024-03-06T09:34:34.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:34.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:34.217298+0000) 2024-03-06T09:34:35.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:35.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:35.217613+0000) 2024-03-06T09:34:36.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:36.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:36.217897+0000) 2024-03-06T09:34:37.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:37.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:37.218175+0000) 2024-03-06T09:34:38.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:38.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:38.218456+0000) 2024-03-06T09:34:39.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:39.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:39.218736+0000) 2024-03-06T09:34:39.375 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:34:39.383 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:34:40.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:40.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:40.219002+0000) 2024-03-06T09:34:41.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:41.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:41.219281+0000) 2024-03-06T09:34:42.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:42.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:42.219594+0000) 2024-03-06T09:34:43.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:43.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:43.219879+0000) 2024-03-06T09:34:44.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:44.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:44.220170+0000) 2024-03-06T09:34:45.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:45.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:45.220394+0000) 2024-03-06T09:34:46.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:46.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:46.220648+0000) 2024-03-06T09:34:47.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:47.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:47.220815+0000) 2024-03-06T09:34:48.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:48.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:48.221035+0000) 2024-03-06T09:34:49.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:49.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:49.221259+0000) 2024-03-06T09:34:50.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:50.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:50.221465+0000) 2024-03-06T09:34:51.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:51.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:51.221690+0000) 2024-03-06T09:34:52.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:52.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:52.221916+0000) 2024-03-06T09:34:53.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:53.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:53.222178+0000) 2024-03-06T09:34:54.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:54.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:54.222448+0000) 2024-03-06T09:34:55.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:55.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:55.222700+0000) 2024-03-06T09:34:56.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:56.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:56.222901+0000) 2024-03-06T09:34:57.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:57.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:57.223090+0000) 2024-03-06T09:34:58.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:58.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:58.223318+0000) 2024-03-06T09:34:59.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:34:59.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:34:59.223621+0000) 2024-03-06T09:35:00.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:00.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:00.223845+0000) 2024-03-06T09:35:01.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:01.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:01.224090+0000) 2024-03-06T09:35:02.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:02.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:02.224293+0000) 2024-03-06T09:35:03.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:03.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:03.224563+0000) 2024-03-06T09:35:04.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:04.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:04.224804+0000) 2024-03-06T09:35:05.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:05.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:05.225033+0000) 2024-03-06T09:35:06.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:06.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:06.225257+0000) 2024-03-06T09:35:07.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:07.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:07.225478+0000) 2024-03-06T09:35:08.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:08.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:08.225717+0000) 2024-03-06T09:35:09.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:09.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:09.225941+0000) 2024-03-06T09:35:09.541 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:35:09.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:35:10.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:10.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:10.226167+0000) 2024-03-06T09:35:11.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:11.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:11.226391+0000) 2024-03-06T09:35:12.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:12.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:12.226609+0000) 2024-03-06T09:35:13.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:13.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:13.226797+0000) 2024-03-06T09:35:14.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:14.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:14.227036+0000) 2024-03-06T09:35:15.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:15.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:15.227262+0000) 2024-03-06T09:35:16.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:16.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:16.227488+0000) 2024-03-06T09:35:17.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:17.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:17.227684+0000) 2024-03-06T09:35:18.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:18.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:18.227927+0000) 2024-03-06T09:35:19.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:19.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:19.228251+0000) 2024-03-06T09:35:20.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:20.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:20.228524+0000) 2024-03-06T09:35:21.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:21.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:21.228858+0000) 2024-03-06T09:35:22.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:22.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:22.229065+0000) 2024-03-06T09:35:23.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:23.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:23.229287+0000) 2024-03-06T09:35:24.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:24.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:24.229569+0000) 2024-03-06T09:35:25.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:25.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:25.229788+0000) 2024-03-06T09:35:26.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:26.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:26.230067+0000) 2024-03-06T09:35:27.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:27.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:27.230340+0000) 2024-03-06T09:35:28.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:28.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:28.230652+0000) 2024-03-06T09:35:29.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:29.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:29.230890+0000) 2024-03-06T09:35:30.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:30.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:30.231095+0000) 2024-03-06T09:35:31.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:31.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:31.231348+0000) 2024-03-06T09:35:32.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:32.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:32.231637+0000) 2024-03-06T09:35:33.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:33.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:33.231890+0000) 2024-03-06T09:35:34.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:34.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:34.232141+0000) 2024-03-06T09:35:35.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:35.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:35.232370+0000) 2024-03-06T09:35:36.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:36.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:36.232604+0000) 2024-03-06T09:35:37.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:37.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:37.232830+0000) 2024-03-06T09:35:38.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:38.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:38.233099+0000) 2024-03-06T09:35:39.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:39.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:39.233369+0000) 2024-03-06T09:35:39.734 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:35:39.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:35:40.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:40.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:40.233623+0000) 2024-03-06T09:35:41.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:41.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:41.233846+0000) 2024-03-06T09:35:42.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:42.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:42.234063+0000) 2024-03-06T09:35:43.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:43.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:43.234342+0000) 2024-03-06T09:35:44.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:44.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:44.234661+0000) 2024-03-06T09:35:45.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:45.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:45.234884+0000) 2024-03-06T09:35:46.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:46.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:46.235124+0000) 2024-03-06T09:35:47.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:47.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:47.235347+0000) 2024-03-06T09:35:48.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:48.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:48.235562+0000) 2024-03-06T09:35:49.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:49.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:49.235759+0000) 2024-03-06T09:35:50.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:50.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:50.236027+0000) 2024-03-06T09:35:51.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:51.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:51.236294+0000) 2024-03-06T09:35:52.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:52.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:52.236496+0000) 2024-03-06T09:35:53.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:53.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:53.236689+0000) 2024-03-06T09:35:54.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:54.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:54.236919+0000) 2024-03-06T09:35:55.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:55.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:55.237122+0000) 2024-03-06T09:35:56.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:56.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:56.237360+0000) 2024-03-06T09:35:57.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:57.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:57.237634+0000) 2024-03-06T09:35:58.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:58.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:58.237785+0000) 2024-03-06T09:35:59.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:35:59.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:35:59.237972+0000) 2024-03-06T09:36:00.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:00.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:00.238201+0000) 2024-03-06T09:36:01.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:01.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:01.238418+0000) 2024-03-06T09:36:02.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:02.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:02.238705+0000) 2024-03-06T09:36:03.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:03.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:03.238897+0000) 2024-03-06T09:36:04.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:04.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:04.239107+0000) 2024-03-06T09:36:05.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:05.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:05.239356+0000) 2024-03-06T09:36:06.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:06.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:06.239621+0000) 2024-03-06T09:36:07.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:07.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:07.239930+0000) 2024-03-06T09:36:08.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:08.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:08.240173+0000) 2024-03-06T09:36:09.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:09.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:09.240370+0000) 2024-03-06T09:36:09.964 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:36:09.973 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:36:10.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:10.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:10.240642+0000) 2024-03-06T09:36:11.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:11.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:11.240832+0000) 2024-03-06T09:36:12.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:12.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:12.241058+0000) 2024-03-06T09:36:13.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:13.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:13.241281+0000) 2024-03-06T09:36:14.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:14.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:14.241514+0000) 2024-03-06T09:36:15.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:15.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:15.241788+0000) 2024-03-06T09:36:16.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:16.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:16.242013+0000) 2024-03-06T09:36:17.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:17.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:17.242233+0000) 2024-03-06T09:36:18.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:18.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:18.242437+0000) 2024-03-06T09:36:19.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:19.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:19.242704+0000) 2024-03-06T09:36:20.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:20.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:20.242915+0000) 2024-03-06T09:36:21.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:21.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:21.243176+0000) 2024-03-06T09:36:22.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:22.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:22.243460+0000) 2024-03-06T09:36:23.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:23.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:23.243713+0000) 2024-03-06T09:36:24.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:24.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:24.243969+0000) 2024-03-06T09:36:25.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:25.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:25.244236+0000) 2024-03-06T09:36:26.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:26.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:26.244499+0000) 2024-03-06T09:36:27.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:27.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:27.244773+0000) 2024-03-06T09:36:28.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:28.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:28.245041+0000) 2024-03-06T09:36:29.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:29.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:29.245306+0000) 2024-03-06T09:36:30.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:30.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:30.245516+0000) 2024-03-06T09:36:31.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:31.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:31.245769+0000) 2024-03-06T09:36:32.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:32.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:32.245993+0000) 2024-03-06T09:36:33.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:33.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:33.246215+0000) 2024-03-06T09:36:34.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:34.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:34.246468+0000) 2024-03-06T09:36:35.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:35.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:35.246696+0000) 2024-03-06T09:36:36.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:36.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:36.246878+0000) 2024-03-06T09:36:37.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:37.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:37.247077+0000) 2024-03-06T09:36:38.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:38.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:38.247259+0000) 2024-03-06T09:36:39.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:39.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:39.247479+0000) 2024-03-06T09:36:40.178 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:36:40.186 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:36:40.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:40.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:40.247652+0000) 2024-03-06T09:36:41.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:41.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:41.247881+0000) 2024-03-06T09:36:42.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:42.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:42.248070+0000) 2024-03-06T09:36:43.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:43.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:43.248264+0000) 2024-03-06T09:36:44.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:44.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:44.248521+0000) 2024-03-06T09:36:45.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:45.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:45.248747+0000) 2024-03-06T09:36:46.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:46.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:46.248984+0000) 2024-03-06T09:36:47.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:47.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:47.249218+0000) 2024-03-06T09:36:48.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:48.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:48.249457+0000) 2024-03-06T09:36:49.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:49.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:49.249697+0000) 2024-03-06T09:36:50.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:50.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:50.249950+0000) 2024-03-06T09:36:51.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:51.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:51.250213+0000) 2024-03-06T09:36:52.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:52.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:52.250417+0000) 2024-03-06T09:36:53.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:53.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:53.250672+0000) 2024-03-06T09:36:54.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:54.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:54.250960+0000) 2024-03-06T09:36:55.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:55.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:55.251238+0000) 2024-03-06T09:36:56.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:56.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:56.251475+0000) 2024-03-06T09:36:57.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:57.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:57.251746+0000) 2024-03-06T09:36:58.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:58.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:58.251981+0000) 2024-03-06T09:36:59.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:36:59.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:36:59.252239+0000) 2024-03-06T09:37:00.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:00.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:00.252576+0000) 2024-03-06T09:37:01.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:01.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:01.252783+0000) 2024-03-06T09:37:02.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:02.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:02.252966+0000) 2024-03-06T09:37:03.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:03.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:03.253183+0000) 2024-03-06T09:37:04.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:04.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:04.253412+0000) 2024-03-06T09:37:05.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:05.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:05.253649+0000) 2024-03-06T09:37:06.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:06.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:06.253882+0000) 2024-03-06T09:37:07.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:07.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:07.254103+0000) 2024-03-06T09:37:08.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:08.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:08.254329+0000) 2024-03-06T09:37:09.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:09.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:09.254582+0000) 2024-03-06T09:37:10.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:10.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:10.254794+0000) 2024-03-06T09:37:10.391 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:37:10.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:37:11.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:11.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:11.254988+0000) 2024-03-06T09:37:12.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:12.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:12.255184+0000) 2024-03-06T09:37:13.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:13.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:13.255408+0000) 2024-03-06T09:37:14.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:14.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:14.255647+0000) 2024-03-06T09:37:15.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:15.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:15.255877+0000) 2024-03-06T09:37:16.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:16.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:16.256079+0000) 2024-03-06T09:37:17.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:17.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:17.256321+0000) 2024-03-06T09:37:18.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:18.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:18.256584+0000) 2024-03-06T09:37:19.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:19.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:19.256781+0000) 2024-03-06T09:37:20.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:20.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:20.257014+0000) 2024-03-06T09:37:21.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:21.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:21.257234+0000) 2024-03-06T09:37:22.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:22.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:22.257435+0000) 2024-03-06T09:37:23.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:23.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:23.257684+0000) 2024-03-06T09:37:24.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:24.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:24.257917+0000) 2024-03-06T09:37:25.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:25.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:25.258188+0000) 2024-03-06T09:37:26.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:26.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:26.258469+0000) 2024-03-06T09:37:27.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:27.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:27.258737+0000) 2024-03-06T09:37:28.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:28.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:28.259019+0000) 2024-03-06T09:37:29.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:29.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:29.259230+0000) 2024-03-06T09:37:30.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:30.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:30.259507+0000) 2024-03-06T09:37:31.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:31.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:31.259788+0000) 2024-03-06T09:37:32.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:32.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:32.259937+0000) 2024-03-06T09:37:33.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:33.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:33.260133+0000) 2024-03-06T09:37:34.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:34.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:34.260419+0000) 2024-03-06T09:37:35.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:35.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:35.260676+0000) 2024-03-06T09:37:36.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:36.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:36.260905+0000) 2024-03-06T09:37:37.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:37.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:37.261114+0000) 2024-03-06T09:37:38.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:38.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:38.261349+0000) 2024-03-06T09:37:39.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:39.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:39.261647+0000) 2024-03-06T09:37:40.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:40.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:40.261934+0000) 2024-03-06T09:37:40.561 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:37:40.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:37:41.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:41.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:41.262138+0000) 2024-03-06T09:37:42.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:42.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:42.262332+0000) 2024-03-06T09:37:43.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:43.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:43.262523+0000) 2024-03-06T09:37:44.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:44.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:44.262839+0000) 2024-03-06T09:37:45.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:45.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:45.263088+0000) 2024-03-06T09:37:46.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:46.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:46.263288+0000) 2024-03-06T09:37:47.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:47.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:47.263581+0000) 2024-03-06T09:37:48.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:48.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:48.263822+0000) 2024-03-06T09:37:49.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:49.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:49.264100+0000) 2024-03-06T09:37:50.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:50.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:50.264374+0000) 2024-03-06T09:37:51.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:51.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:51.264676+0000) 2024-03-06T09:37:52.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:52.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:52.264909+0000) 2024-03-06T09:37:53.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:53.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:53.265134+0000) 2024-03-06T09:37:54.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:54.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:54.265477+0000) 2024-03-06T09:37:55.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:55.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:55.265724+0000) 2024-03-06T09:37:56.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:56.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:56.265965+0000) 2024-03-06T09:37:57.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:57.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:57.266198+0000) 2024-03-06T09:37:58.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:58.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:58.266468+0000) 2024-03-06T09:37:59.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:37:59.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:37:59.266667+0000) 2024-03-06T09:38:00.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:00.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:00.266929+0000) 2024-03-06T09:38:01.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:01.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:01.267209+0000) 2024-03-06T09:38:02.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:02.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:02.267493+0000) 2024-03-06T09:38:03.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:03.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:03.267797+0000) 2024-03-06T09:38:04.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:04.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:04.268044+0000) 2024-03-06T09:38:05.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:05.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:05.268256+0000) 2024-03-06T09:38:06.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:06.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:06.268483+0000) 2024-03-06T09:38:07.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:07.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:07.268697+0000) 2024-03-06T09:38:08.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:08.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:08.268921+0000) 2024-03-06T09:38:09.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:09.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:09.269099+0000) 2024-03-06T09:38:10.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:10.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:10.269331+0000) 2024-03-06T09:38:10.791 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:38:10.800 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:38:11.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:11.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:11.269570+0000) 2024-03-06T09:38:12.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:12.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:12.269805+0000) 2024-03-06T09:38:13.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:13.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:13.270079+0000) 2024-03-06T09:38:14.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:14.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:14.270367+0000) 2024-03-06T09:38:15.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:15.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:15.270684+0000) 2024-03-06T09:38:16.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:16.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:16.270881+0000) 2024-03-06T09:38:17.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:17.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:17.271118+0000) 2024-03-06T09:38:18.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:18.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:18.271354+0000) 2024-03-06T09:38:19.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:19.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:19.271619+0000) 2024-03-06T09:38:20.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:20.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:20.271861+0000) 2024-03-06T09:38:21.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:21.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:21.272134+0000) 2024-03-06T09:38:22.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:22.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:22.272414+0000) 2024-03-06T09:38:23.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:23.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:23.272693+0000) 2024-03-06T09:38:24.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:24.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:24.272904+0000) 2024-03-06T09:38:25.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:25.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:25.273107+0000) 2024-03-06T09:38:26.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:26.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:26.273357+0000) 2024-03-06T09:38:27.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:27.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:27.273636+0000) 2024-03-06T09:38:28.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:28.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:28.273841+0000) 2024-03-06T09:38:29.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:29.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:29.274078+0000) 2024-03-06T09:38:30.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:30.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:30.274325+0000) 2024-03-06T09:38:31.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:31.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:31.274587+0000) 2024-03-06T09:38:32.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:32.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:32.274781+0000) 2024-03-06T09:38:33.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:33.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:33.275007+0000) 2024-03-06T09:38:34.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:34.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:34.275295+0000) 2024-03-06T09:38:35.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:35.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:35.275497+0000) 2024-03-06T09:38:36.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:36.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:36.275702+0000) 2024-03-06T09:38:37.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:37.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:37.275902+0000) 2024-03-06T09:38:38.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:38.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:38.276131+0000) 2024-03-06T09:38:39.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:39.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:39.276357+0000) 2024-03-06T09:38:40.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:40.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:40.276653+0000) 2024-03-06T09:38:41.004 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:38:41.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:38:41.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:41.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:41.276890+0000) 2024-03-06T09:38:42.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:42.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:42.277144+0000) 2024-03-06T09:38:43.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:43.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:43.277379+0000) 2024-03-06T09:38:44.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:44.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:44.277643+0000) 2024-03-06T09:38:45.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:45.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:45.277878+0000) 2024-03-06T09:38:46.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:46.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:46.278099+0000) 2024-03-06T09:38:47.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:47.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:47.278325+0000) 2024-03-06T09:38:48.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:48.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:48.278587+0000) 2024-03-06T09:38:49.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:49.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:49.278782+0000) 2024-03-06T09:38:50.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:50.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:50.279009+0000) 2024-03-06T09:38:51.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:51.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:51.279238+0000) 2024-03-06T09:38:52.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:52.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:52.279439+0000) 2024-03-06T09:38:53.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:53.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:53.279687+0000) 2024-03-06T09:38:54.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:54.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:54.279936+0000) 2024-03-06T09:38:55.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:55.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:55.280176+0000) 2024-03-06T09:38:56.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:56.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:56.280394+0000) 2024-03-06T09:38:57.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:57.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:57.280637+0000) 2024-03-06T09:38:58.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:58.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:58.280847+0000) 2024-03-06T09:38:59.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:38:59.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:38:59.281025+0000) 2024-03-06T09:39:00.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:00.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:00.281273+0000) 2024-03-06T09:39:01.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:01.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:01.281584+0000) 2024-03-06T09:39:02.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:02.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:02.281803+0000) 2024-03-06T09:39:03.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:03.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:03.282032+0000) 2024-03-06T09:39:04.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:04.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:04.282250+0000) 2024-03-06T09:39:05.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:05.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:05.282484+0000) 2024-03-06T09:39:06.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:06.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:06.282713+0000) 2024-03-06T09:39:07.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:07.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:07.282926+0000) 2024-03-06T09:39:08.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:08.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:08.283143+0000) 2024-03-06T09:39:09.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:09.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:09.283391+0000) 2024-03-06T09:39:10.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:10.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:10.283652+0000) 2024-03-06T09:39:11.218 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:39:11.226 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:39:11.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:11.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:11.283834+0000) 2024-03-06T09:39:12.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:12.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:12.284049+0000) 2024-03-06T09:39:13.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:13.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:13.284273+0000) 2024-03-06T09:39:14.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:14.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:14.284513+0000) 2024-03-06T09:39:15.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:15.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:15.284787+0000) 2024-03-06T09:39:16.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:16.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:16.285019+0000) 2024-03-06T09:39:17.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:17.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:17.285201+0000) 2024-03-06T09:39:18.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:18.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:18.285419+0000) 2024-03-06T09:39:19.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:19.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:19.285672+0000) 2024-03-06T09:39:20.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:20.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:20.285891+0000) 2024-03-06T09:39:21.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:21.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:21.286094+0000) 2024-03-06T09:39:22.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:22.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:22.286313+0000) 2024-03-06T09:39:23.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:23.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:23.286562+0000) 2024-03-06T09:39:24.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:24.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:24.286819+0000) 2024-03-06T09:39:25.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:25.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:25.287048+0000) 2024-03-06T09:39:26.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:26.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:26.287275+0000) 2024-03-06T09:39:27.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:27.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:27.287508+0000) 2024-03-06T09:39:28.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:28.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:28.287834+0000) 2024-03-06T09:39:29.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:29.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:29.288071+0000) 2024-03-06T09:39:30.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:30.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:30.288297+0000) 2024-03-06T09:39:31.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:31.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:31.288562+0000) 2024-03-06T09:39:32.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:32.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:32.288794+0000) 2024-03-06T09:39:33.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:33.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:33.289018+0000) 2024-03-06T09:39:34.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:34.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:34.289279+0000) 2024-03-06T09:39:35.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:35.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:35.289603+0000) 2024-03-06T09:39:36.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:36.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:36.289828+0000) 2024-03-06T09:39:37.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:37.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:37.290082+0000) 2024-03-06T09:39:38.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:38.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:38.290331+0000) 2024-03-06T09:39:39.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:39.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:39.290636+0000) 2024-03-06T09:39:40.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:40.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:40.290849+0000) 2024-03-06T09:39:41.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:41.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:41.291068+0000) 2024-03-06T09:39:41.372 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:39:41.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:39:42.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:42.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:42.291328+0000) 2024-03-06T09:39:43.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:43.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:43.291591+0000) 2024-03-06T09:39:44.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:44.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:44.291832+0000) 2024-03-06T09:39:45.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:45.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:45.292123+0000) 2024-03-06T09:39:46.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:46.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:46.292390+0000) 2024-03-06T09:39:47.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:47.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:47.292637+0000) 2024-03-06T09:39:48.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:48.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:48.292854+0000) 2024-03-06T09:39:49.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:49.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:49.293089+0000) 2024-03-06T09:39:50.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:50.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:50.293283+0000) 2024-03-06T09:39:51.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:51.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:51.293582+0000) 2024-03-06T09:39:52.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:52.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:52.293857+0000) 2024-03-06T09:39:53.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:53.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:53.294131+0000) 2024-03-06T09:39:54.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:54.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:54.294416+0000) 2024-03-06T09:39:55.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:55.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:55.294661+0000) 2024-03-06T09:39:56.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:56.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:56.294891+0000) 2024-03-06T09:39:57.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:57.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:57.295129+0000) 2024-03-06T09:39:58.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:58.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:58.295371+0000) 2024-03-06T09:39:59.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:39:59.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:39:59.295649+0000) 2024-03-06T09:40:00.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:00.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:00.295855+0000) 2024-03-06T09:40:01.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:01.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:01.296081+0000) 2024-03-06T09:40:02.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:02.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:02.296304+0000) 2024-03-06T09:40:03.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:03.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:03.296526+0000) 2024-03-06T09:40:04.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:04.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:04.296784+0000) 2024-03-06T09:40:05.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:05.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:05.296992+0000) 2024-03-06T09:40:06.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:06.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:06.297182+0000) 2024-03-06T09:40:07.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:07.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:07.297407+0000) 2024-03-06T09:40:08.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:08.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:08.297646+0000) 2024-03-06T09:40:09.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:09.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:09.297829+0000) 2024-03-06T09:40:10.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:10.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:10.298025+0000) 2024-03-06T09:40:11.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:11.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:11.298227+0000) 2024-03-06T09:40:11.596 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:40:11.604 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:40:12.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:12.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:12.298415+0000) 2024-03-06T09:40:13.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:13.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:13.298655+0000) 2024-03-06T09:40:14.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:14.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:14.298858+0000) 2024-03-06T09:40:15.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:15.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:15.299055+0000) 2024-03-06T09:40:16.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:16.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:16.299283+0000) 2024-03-06T09:40:17.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:17.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:17.299505+0000) 2024-03-06T09:40:18.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:18.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:18.299737+0000) 2024-03-06T09:40:19.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:19.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:19.299931+0000) 2024-03-06T09:40:20.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:20.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:20.300157+0000) 2024-03-06T09:40:21.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:21.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:21.300393+0000) 2024-03-06T09:40:22.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:22.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:22.300629+0000) 2024-03-06T09:40:23.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:23.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:23.300864+0000) 2024-03-06T09:40:24.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:24.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:24.301095+0000) 2024-03-06T09:40:25.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:25.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:25.301321+0000) 2024-03-06T09:40:26.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:26.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:26.301575+0000) 2024-03-06T09:40:27.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:27.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:27.301795+0000) 2024-03-06T09:40:28.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:28.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:28.302036+0000) 2024-03-06T09:40:29.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:29.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:29.302267+0000) 2024-03-06T09:40:30.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:30.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:30.302490+0000) 2024-03-06T09:40:31.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:31.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:31.302697+0000) 2024-03-06T09:40:32.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:32.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:32.302899+0000) 2024-03-06T09:40:33.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:33.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:33.303131+0000) 2024-03-06T09:40:34.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:34.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:34.303308+0000) 2024-03-06T09:40:35.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:35.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:35.303571+0000) 2024-03-06T09:40:36.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:36.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:36.303772+0000) 2024-03-06T09:40:37.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:37.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:37.303999+0000) 2024-03-06T09:40:38.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:38.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:38.304218+0000) 2024-03-06T09:40:39.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:39.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:39.304446+0000) 2024-03-06T09:40:40.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:40.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:40.304662+0000) 2024-03-06T09:40:41.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:41.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:41.304884+0000) 2024-03-06T09:40:41.819 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:40:41.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:40:42.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:42.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:42.305110+0000) 2024-03-06T09:40:43.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:43.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:43.305296+0000) 2024-03-06T09:40:44.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:44.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:44.305572+0000) 2024-03-06T09:40:45.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:45.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:45.305802+0000) 2024-03-06T09:40:46.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:46.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:46.305987+0000) 2024-03-06T09:40:47.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:47.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:47.306213+0000) 2024-03-06T09:40:48.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:48.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:48.306437+0000) 2024-03-06T09:40:49.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:49.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:49.306630+0000) 2024-03-06T09:40:50.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:50.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:50.306875+0000) 2024-03-06T09:40:51.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:51.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:51.307095+0000) 2024-03-06T09:40:52.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:52.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:52.307321+0000) 2024-03-06T09:40:53.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:53.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:53.307586+0000) 2024-03-06T09:40:54.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:54.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:54.307823+0000) 2024-03-06T09:40:55.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:55.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:55.308055+0000) 2024-03-06T09:40:56.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:56.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:56.308282+0000) 2024-03-06T09:40:57.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:57.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:57.308483+0000) 2024-03-06T09:40:58.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:58.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:58.308724+0000) 2024-03-06T09:40:59.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:40:59.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:40:59.308949+0000) 2024-03-06T09:41:00.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:00.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:00.309178+0000) 2024-03-06T09:41:01.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:01.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:01.309345+0000) 2024-03-06T09:41:02.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:02.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:02.309565+0000) 2024-03-06T09:41:03.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:03.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:03.309767+0000) 2024-03-06T09:41:04.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:04.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:04.310002+0000) 2024-03-06T09:41:05.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:05.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:05.310230+0000) 2024-03-06T09:41:06.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:06.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:06.310450+0000) 2024-03-06T09:41:07.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:07.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:07.310707+0000) 2024-03-06T09:41:08.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:08.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:08.310946+0000) 2024-03-06T09:41:09.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:09.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:09.311138+0000) 2024-03-06T09:41:10.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:10.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:10.311364+0000) 2024-03-06T09:41:11.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:11.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:11.311645+0000) 2024-03-06T09:41:12.060 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:41:12.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:41:12.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:12.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:12.311846+0000) 2024-03-06T09:41:13.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:13.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:13.312036+0000) 2024-03-06T09:41:14.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:14.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:14.312314+0000) 2024-03-06T09:41:15.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:15.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:15.312514+0000) 2024-03-06T09:41:16.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:16.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:16.312806+0000) 2024-03-06T09:41:17.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:17.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:17.313004+0000) 2024-03-06T09:41:18.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:18.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:18.313230+0000) 2024-03-06T09:41:19.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:19.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:19.313479+0000) 2024-03-06T09:41:20.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:20.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:20.313692+0000) 2024-03-06T09:41:21.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:21.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:21.313897+0000) 2024-03-06T09:41:22.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:22.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:22.314112+0000) 2024-03-06T09:41:23.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:23.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:23.314351+0000) 2024-03-06T09:41:24.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:24.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:24.314629+0000) 2024-03-06T09:41:25.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:25.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:25.314885+0000) 2024-03-06T09:41:26.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:26.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:26.315142+0000) 2024-03-06T09:41:27.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:27.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:27.315383+0000) 2024-03-06T09:41:28.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:28.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:28.315651+0000) 2024-03-06T09:41:29.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:29.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:29.315893+0000) 2024-03-06T09:41:30.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:30.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:30.316136+0000) 2024-03-06T09:41:31.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:31.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:31.316382+0000) 2024-03-06T09:41:32.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:32.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:32.316648+0000) 2024-03-06T09:41:33.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:33.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:33.316874+0000) 2024-03-06T09:41:34.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:34.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:34.317140+0000) 2024-03-06T09:41:35.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:35.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:35.317423+0000) 2024-03-06T09:41:36.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:36.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:36.317695+0000) 2024-03-06T09:41:37.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:37.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:37.317972+0000) 2024-03-06T09:41:38.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:38.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:38.318245+0000) 2024-03-06T09:41:39.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:39.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:39.318524+0000) 2024-03-06T09:41:40.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:40.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:40.318777+0000) 2024-03-06T09:41:41.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:41.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:41.319027+0000) 2024-03-06T09:41:42.265 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:41:42.274 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:41:42.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:42.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:42.319188+0000) 2024-03-06T09:41:43.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:43.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:43.319443+0000) 2024-03-06T09:41:44.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:44.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:44.319727+0000) 2024-03-06T09:41:45.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:45.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:45.320006+0000) 2024-03-06T09:41:46.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:46.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:46.320273+0000) 2024-03-06T09:41:47.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:47.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:47.320587+0000) 2024-03-06T09:41:48.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:48.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:48.320859+0000) 2024-03-06T09:41:49.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:49.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:49.321094+0000) 2024-03-06T09:41:50.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:50.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:50.321362+0000) 2024-03-06T09:41:51.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:51.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:51.321683+0000) 2024-03-06T09:41:52.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:52.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:52.321941+0000) 2024-03-06T09:41:53.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:53.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:53.322104+0000) 2024-03-06T09:41:54.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:54.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:54.322342+0000) 2024-03-06T09:41:55.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:55.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:55.322633+0000) 2024-03-06T09:41:56.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:56.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:56.322837+0000) 2024-03-06T09:41:57.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:57.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:57.323118+0000) 2024-03-06T09:41:58.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:58.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:58.323313+0000) 2024-03-06T09:41:59.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:41:59.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:41:59.323575+0000) 2024-03-06T09:42:00.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:00.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:00.323810+0000) 2024-03-06T09:42:01.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:01.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:01.324037+0000) 2024-03-06T09:42:02.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:02.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:02.324265+0000) 2024-03-06T09:42:03.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:03.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:03.324487+0000) 2024-03-06T09:42:04.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:04.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:04.324710+0000) 2024-03-06T09:42:05.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:05.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:05.324923+0000) 2024-03-06T09:42:06.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:06.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:06.325115+0000) 2024-03-06T09:42:07.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:07.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:07.325339+0000) 2024-03-06T09:42:08.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:08.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:08.325604+0000) 2024-03-06T09:42:09.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:09.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:09.325841+0000) 2024-03-06T09:42:10.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:10.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:10.326178+0000) 2024-03-06T09:42:11.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:11.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:11.326449+0000) 2024-03-06T09:42:12.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:12.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:12.326746+0000) 2024-03-06T09:42:12.470 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:42:12.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:42:13.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:13.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:13.327025+0000) 2024-03-06T09:42:14.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:14.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:14.327314+0000) 2024-03-06T09:42:15.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:15.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:15.327620+0000) 2024-03-06T09:42:16.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:16.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:16.327855+0000) 2024-03-06T09:42:17.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:17.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:17.328084+0000) 2024-03-06T09:42:18.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:18.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:18.328285+0000) 2024-03-06T09:42:19.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:19.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:19.328621+0000) 2024-03-06T09:42:20.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:20.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:20.328897+0000) 2024-03-06T09:42:21.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:21.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:21.329167+0000) 2024-03-06T09:42:22.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:22.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:22.329439+0000) 2024-03-06T09:42:23.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:23.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:23.329702+0000) 2024-03-06T09:42:24.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:24.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:24.329954+0000) 2024-03-06T09:42:25.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:25.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:25.330234+0000) 2024-03-06T09:42:26.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:26.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:26.330523+0000) 2024-03-06T09:42:27.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:27.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:27.330843+0000) 2024-03-06T09:42:28.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:28.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:28.331092+0000) 2024-03-06T09:42:29.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:29.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:29.331372+0000) 2024-03-06T09:42:30.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:30.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:30.331627+0000) 2024-03-06T09:42:31.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:31.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:31.331874+0000) 2024-03-06T09:42:32.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:32.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:32.332070+0000) 2024-03-06T09:42:33.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:33.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:33.332291+0000) 2024-03-06T09:42:34.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:34.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:34.332507+0000) 2024-03-06T09:42:35.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:35.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:35.332777+0000) 2024-03-06T09:42:36.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:36.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:36.333052+0000) 2024-03-06T09:42:37.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:37.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:37.333340+0000) 2024-03-06T09:42:38.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:38.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:38.333560+0000) 2024-03-06T09:42:39.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:39.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:39.333749+0000) 2024-03-06T09:42:40.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:40.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:40.333948+0000) 2024-03-06T09:42:41.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:41.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:41.334153+0000) 2024-03-06T09:42:42.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:42.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:42.334379+0000) 2024-03-06T09:42:42.683 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:42:42.691 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:42:43.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:43.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:43.334663+0000) 2024-03-06T09:42:44.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:44.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:44.334873+0000) 2024-03-06T09:42:45.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:45.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:45.335091+0000) 2024-03-06T09:42:46.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:46.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:46.335283+0000) 2024-03-06T09:42:47.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:47.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:47.335473+0000) 2024-03-06T09:42:48.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:48.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:48.335663+0000) 2024-03-06T09:42:49.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:49.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:49.335838+0000) 2024-03-06T09:42:50.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:50.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:50.336019+0000) 2024-03-06T09:42:51.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:51.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:51.336220+0000) 2024-03-06T09:42:52.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:52.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:52.336454+0000) 2024-03-06T09:42:53.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:53.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:53.336707+0000) 2024-03-06T09:42:54.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:54.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:54.336956+0000) 2024-03-06T09:42:55.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:55.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:55.337188+0000) 2024-03-06T09:42:56.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:56.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:56.337423+0000) 2024-03-06T09:42:57.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:57.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:57.337683+0000) 2024-03-06T09:42:58.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:58.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:58.337973+0000) 2024-03-06T09:42:59.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:42:59.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:42:59.338181+0000) 2024-03-06T09:43:00.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:00.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:00.338411+0000) 2024-03-06T09:43:01.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:01.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:01.338665+0000) 2024-03-06T09:43:02.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:02.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:02.338893+0000) 2024-03-06T09:43:03.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:03.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:03.339083+0000) 2024-03-06T09:43:04.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:04.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:04.339353+0000) 2024-03-06T09:43:05.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:05.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:05.339675+0000) 2024-03-06T09:43:06.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:06.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:06.339974+0000) 2024-03-06T09:43:07.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:07.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:07.340255+0000) 2024-03-06T09:43:08.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:08.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:08.340505+0000) 2024-03-06T09:43:09.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:09.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:09.340766+0000) 2024-03-06T09:43:10.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:10.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:10.341037+0000) 2024-03-06T09:43:11.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:11.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:11.341255+0000) 2024-03-06T09:43:12.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:12.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:12.341489+0000) 2024-03-06T09:43:12.900 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:43:12.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:43:13.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:13.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:13.341717+0000) 2024-03-06T09:43:14.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:14.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:14.341910+0000) 2024-03-06T09:43:15.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:15.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:15.342181+0000) 2024-03-06T09:43:16.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:16.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:16.342463+0000) 2024-03-06T09:43:17.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:17.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:17.342718+0000) 2024-03-06T09:43:18.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:18.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:18.342950+0000) 2024-03-06T09:43:19.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:19.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:19.343179+0000) 2024-03-06T09:43:20.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:20.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:20.343468+0000) 2024-03-06T09:43:21.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:21.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:21.343753+0000) 2024-03-06T09:43:22.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:22.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:22.343967+0000) 2024-03-06T09:43:23.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:23.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:23.344187+0000) 2024-03-06T09:43:24.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:24.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:24.344429+0000) 2024-03-06T09:43:25.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:25.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:25.344660+0000) 2024-03-06T09:43:26.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:26.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:26.344855+0000) 2024-03-06T09:43:27.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:27.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:27.345067+0000) 2024-03-06T09:43:28.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:28.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:28.345281+0000) 2024-03-06T09:43:29.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:29.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:29.345487+0000) 2024-03-06T09:43:30.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:30.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:30.345697+0000) 2024-03-06T09:43:31.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:31.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:31.345908+0000) 2024-03-06T09:43:32.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:32.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:32.346152+0000) 2024-03-06T09:43:33.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:33.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:33.346385+0000) 2024-03-06T09:43:34.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:34.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:34.346659+0000) 2024-03-06T09:43:35.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:35.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:35.346866+0000) 2024-03-06T09:43:36.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:36.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:36.347108+0000) 2024-03-06T09:43:37.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:37.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:37.347348+0000) 2024-03-06T09:43:38.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:38.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:38.347620+0000) 2024-03-06T09:43:39.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:39.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:39.347815+0000) 2024-03-06T09:43:40.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:40.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:40.348042+0000) 2024-03-06T09:43:41.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:41.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:41.348273+0000) 2024-03-06T09:43:42.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:42.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:42.348462+0000) 2024-03-06T09:43:43.114 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:43:43.123 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:43:43.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:43.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:43.348698+0000) 2024-03-06T09:43:44.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:44.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:44.348895+0000) 2024-03-06T09:43:45.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:45.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:45.349131+0000) 2024-03-06T09:43:46.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:46.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:46.349383+0000) 2024-03-06T09:43:47.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:47.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:47.349668+0000) 2024-03-06T09:43:48.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:48.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:48.349888+0000) 2024-03-06T09:43:49.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:49.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:49.350126+0000) 2024-03-06T09:43:50.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:50.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:50.350359+0000) 2024-03-06T09:43:51.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:51.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:51.350695+0000) 2024-03-06T09:43:52.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:52.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:52.350902+0000) 2024-03-06T09:43:53.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:53.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:53.351090+0000) 2024-03-06T09:43:54.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:54.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:54.351349+0000) 2024-03-06T09:43:55.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:55.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:55.351643+0000) 2024-03-06T09:43:56.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:56.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:56.351889+0000) 2024-03-06T09:43:57.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:57.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:57.352169+0000) 2024-03-06T09:43:58.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:58.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:58.352448+0000) 2024-03-06T09:43:59.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:43:59.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:43:59.352741+0000) 2024-03-06T09:44:00.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:00.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:00.353023+0000) 2024-03-06T09:44:01.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:01.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:01.353303+0000) 2024-03-06T09:44:02.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:02.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:02.353632+0000) 2024-03-06T09:44:03.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:03.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:03.353904+0000) 2024-03-06T09:44:04.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:04.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:04.354196+0000) 2024-03-06T09:44:05.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:05.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:05.354478+0000) 2024-03-06T09:44:06.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:06.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:06.354758+0000) 2024-03-06T09:44:07.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:07.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:07.355043+0000) 2024-03-06T09:44:08.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:08.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:08.355321+0000) 2024-03-06T09:44:09.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:09.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:09.355638+0000) 2024-03-06T09:44:10.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:10.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:10.355814+0000) 2024-03-06T09:44:11.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:11.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:11.356043+0000) 2024-03-06T09:44:12.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:12.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:12.356274+0000) 2024-03-06T09:44:13.314 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:44:13.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:44:13.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:13.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:13.356492+0000) 2024-03-06T09:44:14.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:14.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:14.356844+0000) 2024-03-06T09:44:15.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:15.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:15.357070+0000) 2024-03-06T09:44:16.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:16.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:16.357298+0000) 2024-03-06T09:44:17.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:17.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:17.357529+0000) 2024-03-06T09:44:18.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:18.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:18.357863+0000) 2024-03-06T09:44:19.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:19.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:19.358085+0000) 2024-03-06T09:44:20.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:20.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:20.358284+0000) 2024-03-06T09:44:21.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:21.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:21.358474+0000) 2024-03-06T09:44:22.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:22.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:22.358722+0000) 2024-03-06T09:44:23.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:23.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:23.358955+0000) 2024-03-06T09:44:24.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:24.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:24.359196+0000) 2024-03-06T09:44:25.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:25.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:25.359430+0000) 2024-03-06T09:44:26.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:26.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:26.359656+0000) 2024-03-06T09:44:27.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:27.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:27.359858+0000) 2024-03-06T09:44:28.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:28.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:28.360057+0000) 2024-03-06T09:44:29.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:29.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:29.360244+0000) 2024-03-06T09:44:30.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:30.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:30.360483+0000) 2024-03-06T09:44:31.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:31.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:31.360736+0000) 2024-03-06T09:44:32.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:32.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:32.360932+0000) 2024-03-06T09:44:33.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:33.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:33.361158+0000) 2024-03-06T09:44:34.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:34.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:34.361372+0000) 2024-03-06T09:44:35.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:35.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:35.361650+0000) 2024-03-06T09:44:36.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:36.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:36.361869+0000) 2024-03-06T09:44:37.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:37.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:37.362049+0000) 2024-03-06T09:44:38.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:38.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:38.362285+0000) 2024-03-06T09:44:39.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:39.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:39.362647+0000) 2024-03-06T09:44:40.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:40.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:40.362918+0000) 2024-03-06T09:44:41.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:41.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:41.363189+0000) 2024-03-06T09:44:42.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:42.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:42.363457+0000) 2024-03-06T09:44:43.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:43.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:43.363749+0000) 2024-03-06T09:44:43.545 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:44:43.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:44:44.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:44.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:44.363998+0000) 2024-03-06T09:44:45.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:45.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:45.364186+0000) 2024-03-06T09:44:46.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:46.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:46.364412+0000) 2024-03-06T09:44:47.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:47.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:47.364641+0000) 2024-03-06T09:44:48.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:48.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:48.364868+0000) 2024-03-06T09:44:49.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:49.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:49.365054+0000) 2024-03-06T09:44:50.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:50.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:50.365300+0000) 2024-03-06T09:44:51.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:51.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:51.365518+0000) 2024-03-06T09:44:52.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:52.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:52.365792+0000) 2024-03-06T09:44:53.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:53.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:53.366014+0000) 2024-03-06T09:44:54.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:54.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:54.366247+0000) 2024-03-06T09:44:55.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:55.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:55.366444+0000) 2024-03-06T09:44:56.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:56.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:56.366655+0000) 2024-03-06T09:44:57.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:57.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:57.366930+0000) 2024-03-06T09:44:58.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:58.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:58.367160+0000) 2024-03-06T09:44:59.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:44:59.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:44:59.367434+0000) 2024-03-06T09:45:00.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:00.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:00.367698+0000) 2024-03-06T09:45:01.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:01.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:01.367898+0000) 2024-03-06T09:45:02.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:02.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:02.368126+0000) 2024-03-06T09:45:03.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:03.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:03.368321+0000) 2024-03-06T09:45:04.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:04.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:04.368601+0000) 2024-03-06T09:45:05.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:05.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:05.368824+0000) 2024-03-06T09:45:06.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:06.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:06.369005+0000) 2024-03-06T09:45:07.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:07.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:07.369268+0000) 2024-03-06T09:45:08.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:08.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:08.369581+0000) 2024-03-06T09:45:09.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:09.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:09.369820+0000) 2024-03-06T09:45:10.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:10.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:10.370060+0000) 2024-03-06T09:45:11.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:11.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:11.370241+0000) 2024-03-06T09:45:12.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:12.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:12.370470+0000) 2024-03-06T09:45:13.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:13.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:13.370694+0000) 2024-03-06T09:45:13.774 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:45:13.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:45:14.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:14.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:14.370935+0000) 2024-03-06T09:45:15.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:15.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:15.371157+0000) 2024-03-06T09:45:16.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:16.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:16.371377+0000) 2024-03-06T09:45:17.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:17.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:17.371691+0000) 2024-03-06T09:45:18.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:18.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:18.371965+0000) 2024-03-06T09:45:19.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:19.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:19.372238+0000) 2024-03-06T09:45:20.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:20.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:20.372479+0000) 2024-03-06T09:45:21.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:21.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:21.372692+0000) 2024-03-06T09:45:22.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:22.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:22.372875+0000) 2024-03-06T09:45:23.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:23.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:23.373118+0000) 2024-03-06T09:45:24.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:24.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:24.373408+0000) 2024-03-06T09:45:25.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:25.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:25.373684+0000) 2024-03-06T09:45:26.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:26.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:26.373955+0000) 2024-03-06T09:45:27.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:27.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:27.374198+0000) 2024-03-06T09:45:28.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:28.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:28.374395+0000) 2024-03-06T09:45:29.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:29.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:29.374652+0000) 2024-03-06T09:45:30.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:30.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:30.374867+0000) 2024-03-06T09:45:31.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:31.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:31.375089+0000) 2024-03-06T09:45:32.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:32.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:32.375358+0000) 2024-03-06T09:45:33.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:33.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:33.375629+0000) 2024-03-06T09:45:34.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:34.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:34.375886+0000) 2024-03-06T09:45:35.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:35.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:35.376084+0000) 2024-03-06T09:45:36.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:36.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:36.376273+0000) 2024-03-06T09:45:37.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:37.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:37.376486+0000) 2024-03-06T09:45:38.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:38.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:38.376708+0000) 2024-03-06T09:45:39.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:39.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:39.376960+0000) 2024-03-06T09:45:40.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:40.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:40.377207+0000) 2024-03-06T09:45:41.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:41.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:41.377469+0000) 2024-03-06T09:45:42.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:42.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:42.377718+0000) 2024-03-06T09:45:43.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:43.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:43.377946+0000) 2024-03-06T09:45:43.996 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:45:44.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:45:44.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:44.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:44.378169+0000) 2024-03-06T09:45:45.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:45.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:45.378375+0000) 2024-03-06T09:45:46.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:46.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:46.378605+0000) 2024-03-06T09:45:47.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:47.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:47.378829+0000) 2024-03-06T09:45:48.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:48.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:48.379015+0000) 2024-03-06T09:45:49.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:49.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:49.379200+0000) 2024-03-06T09:45:50.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:50.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:50.379440+0000) 2024-03-06T09:45:51.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:51.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:51.379725+0000) 2024-03-06T09:45:52.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:52.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:52.379990+0000) 2024-03-06T09:45:53.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:53.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:53.380206+0000) 2024-03-06T09:45:54.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:54.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:54.380464+0000) 2024-03-06T09:45:55.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:55.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:55.380702+0000) 2024-03-06T09:45:56.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:56.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:56.380890+0000) 2024-03-06T09:45:57.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:57.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:57.381091+0000) 2024-03-06T09:45:58.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:58.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:58.381319+0000) 2024-03-06T09:45:59.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:45:59.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:45:59.381583+0000) 2024-03-06T09:46:00.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:00.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:00.381817+0000) 2024-03-06T09:46:01.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:01.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:01.382042+0000) 2024-03-06T09:46:02.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:02.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:02.382273+0000) 2024-03-06T09:46:03.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:03.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:03.382509+0000) 2024-03-06T09:46:04.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:04.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:04.382753+0000) 2024-03-06T09:46:05.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:05.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:05.382992+0000) 2024-03-06T09:46:06.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:06.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:06.383217+0000) 2024-03-06T09:46:07.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:07.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:07.383451+0000) 2024-03-06T09:46:08.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:08.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:08.383676+0000) 2024-03-06T09:46:09.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:09.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:09.383869+0000) 2024-03-06T09:46:10.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:10.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:10.384116+0000) 2024-03-06T09:46:11.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:11.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:11.384340+0000) 2024-03-06T09:46:12.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:12.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:12.384592+0000) 2024-03-06T09:46:13.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:13.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:13.384808+0000) 2024-03-06T09:46:14.136 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:46:14.145 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:46:14.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:14.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:14.385062+0000) 2024-03-06T09:46:15.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:15.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:15.385297+0000) 2024-03-06T09:46:16.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:16.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:16.385522+0000) 2024-03-06T09:46:17.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:17.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:17.385746+0000) 2024-03-06T09:46:18.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:18.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:18.385937+0000) 2024-03-06T09:46:19.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:19.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:19.386162+0000) 2024-03-06T09:46:20.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:20.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:20.386685+0000) 2024-03-06T09:46:21.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:21.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:21.386897+0000) 2024-03-06T09:46:22.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:22.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:22.387129+0000) 2024-03-06T09:46:23.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:23.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:23.387347+0000) 2024-03-06T09:46:24.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:24.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:24.387631+0000) 2024-03-06T09:46:25.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:25.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:25.387822+0000) 2024-03-06T09:46:26.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:26.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:26.388093+0000) 2024-03-06T09:46:27.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:27.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:27.388263+0000) 2024-03-06T09:46:28.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:28.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:28.388621+0000) 2024-03-06T09:46:29.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:29.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:29.388865+0000) 2024-03-06T09:46:30.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:30.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:30.389148+0000) 2024-03-06T09:46:31.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:31.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:31.389425+0000) 2024-03-06T09:46:32.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:32.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:32.389705+0000) 2024-03-06T09:46:33.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:33.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:33.389946+0000) 2024-03-06T09:46:34.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:34.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:34.390251+0000) 2024-03-06T09:46:35.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:35.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:35.390524+0000) 2024-03-06T09:46:36.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:36.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:36.390799+0000) 2024-03-06T09:46:37.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:37.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:37.391025+0000) 2024-03-06T09:46:38.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:38.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:38.391244+0000) 2024-03-06T09:46:39.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:39.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:39.391491+0000) 2024-03-06T09:46:40.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:40.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:40.391794+0000) 2024-03-06T09:46:41.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:41.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:41.392032+0000) 2024-03-06T09:46:42.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:42.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:42.392294+0000) 2024-03-06T09:46:43.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:43.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:43.392578+0000) 2024-03-06T09:46:44.359 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:46:44.367 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:46:44.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:44.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:44.392889+0000) 2024-03-06T09:46:45.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:45.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:45.393107+0000) 2024-03-06T09:46:46.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:46.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:46.393390+0000) 2024-03-06T09:46:47.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:47.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:47.393638+0000) 2024-03-06T09:46:48.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:48.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:48.393849+0000) 2024-03-06T09:46:49.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:49.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:49.394124+0000) 2024-03-06T09:46:50.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:50.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:50.394404+0000) 2024-03-06T09:46:51.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:51.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:51.394694+0000) 2024-03-06T09:46:52.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:52.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:52.394976+0000) 2024-03-06T09:46:53.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:53.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:53.395186+0000) 2024-03-06T09:46:54.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:54.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:54.395427+0000) 2024-03-06T09:46:55.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:55.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:55.395655+0000) 2024-03-06T09:46:56.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:56.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:56.395848+0000) 2024-03-06T09:46:57.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:57.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:57.396086+0000) 2024-03-06T09:46:58.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:58.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:58.396320+0000) 2024-03-06T09:46:59.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:46:59.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:46:59.396584+0000) 2024-03-06T09:47:00.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:00.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:00.396821+0000) 2024-03-06T09:47:01.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:01.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:01.397055+0000) 2024-03-06T09:47:02.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:02.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:02.397290+0000) 2024-03-06T09:47:03.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:03.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:03.397518+0000) 2024-03-06T09:47:04.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:04.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:04.397791+0000) 2024-03-06T09:47:05.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:05.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:05.398012+0000) 2024-03-06T09:47:06.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:06.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:06.398233+0000) 2024-03-06T09:47:07.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:07.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:07.398446+0000) 2024-03-06T09:47:08.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:08.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:08.398662+0000) 2024-03-06T09:47:09.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:09.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:09.398901+0000) 2024-03-06T09:47:10.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:10.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:10.399181+0000) 2024-03-06T09:47:11.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:11.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:11.399456+0000) 2024-03-06T09:47:12.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:12.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:12.399703+0000) 2024-03-06T09:47:13.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:13.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:13.399891+0000) 2024-03-06T09:47:14.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:14.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:14.400227+0000) 2024-03-06T09:47:14.575 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:47:14.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:47:15.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:15.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:15.400459+0000) 2024-03-06T09:47:16.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:16.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:16.400693+0000) 2024-03-06T09:47:17.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:17.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:17.400936+0000) 2024-03-06T09:47:18.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:18.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:18.401172+0000) 2024-03-06T09:47:19.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:19.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:19.401410+0000) 2024-03-06T09:47:20.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:20.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:20.401648+0000) 2024-03-06T09:47:21.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:21.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:21.401847+0000) 2024-03-06T09:47:22.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:22.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:22.402085+0000) 2024-03-06T09:47:23.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:23.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:23.402279+0000) 2024-03-06T09:47:24.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:24.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:24.402456+0000) 2024-03-06T09:47:25.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:25.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:25.402693+0000) 2024-03-06T09:47:26.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:26.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:26.402916+0000) 2024-03-06T09:47:27.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:27.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:27.403143+0000) 2024-03-06T09:47:28.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:28.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:28.403380+0000) 2024-03-06T09:47:29.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:29.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:29.403644+0000) 2024-03-06T09:47:30.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:30.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:30.403844+0000) 2024-03-06T09:47:31.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:31.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:31.404072+0000) 2024-03-06T09:47:32.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:32.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:32.404290+0000) 2024-03-06T09:47:33.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:33.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:33.404614+0000) 2024-03-06T09:47:34.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:34.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:34.404903+0000) 2024-03-06T09:47:35.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:35.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:35.405180+0000) 2024-03-06T09:47:36.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:36.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:36.405457+0000) 2024-03-06T09:47:37.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:37.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:37.405727+0000) 2024-03-06T09:47:38.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:38.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:38.405975+0000) 2024-03-06T09:47:39.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:39.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:39.406198+0000) 2024-03-06T09:47:40.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:40.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:40.406414+0000) 2024-03-06T09:47:41.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:41.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:41.406674+0000) 2024-03-06T09:47:42.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:42.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:42.406926+0000) 2024-03-06T09:47:43.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:43.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:43.407154+0000) 2024-03-06T09:47:44.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:44.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:44.407441+0000) 2024-03-06T09:47:44.729 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:47:44.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:47:45.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:45.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:45.407730+0000) 2024-03-06T09:47:46.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:46.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:46.407980+0000) 2024-03-06T09:47:47.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:47.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:47.408221+0000) 2024-03-06T09:47:48.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:48.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:48.408454+0000) 2024-03-06T09:47:49.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:49.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:49.408720+0000) 2024-03-06T09:47:50.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:50.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:50.408973+0000) 2024-03-06T09:47:51.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:51.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:51.409241+0000) 2024-03-06T09:47:52.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:52.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:52.409525+0000) 2024-03-06T09:47:53.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:53.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:53.409870+0000) 2024-03-06T09:47:54.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:54.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:54.410158+0000) 2024-03-06T09:47:55.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:55.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:55.410427+0000) 2024-03-06T09:47:56.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:56.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:56.410718+0000) 2024-03-06T09:47:57.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:57.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:57.410975+0000) 2024-03-06T09:47:58.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:58.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:58.411257+0000) 2024-03-06T09:47:59.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:47:59.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:47:59.411582+0000) 2024-03-06T09:48:00.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:00.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:00.411838+0000) 2024-03-06T09:48:01.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:01.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:01.412121+0000) 2024-03-06T09:48:02.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:02.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:02.412403+0000) 2024-03-06T09:48:03.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:03.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:03.412678+0000) 2024-03-06T09:48:04.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:04.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:04.412941+0000) 2024-03-06T09:48:05.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:05.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:05.413184+0000) 2024-03-06T09:48:06.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:06.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:06.413407+0000) 2024-03-06T09:48:07.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:07.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:07.413767+0000) 2024-03-06T09:48:08.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:08.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:08.414049+0000) 2024-03-06T09:48:09.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:09.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:09.414333+0000) 2024-03-06T09:48:10.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:10.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:10.414645+0000) 2024-03-06T09:48:11.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:11.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:11.414886+0000) 2024-03-06T09:48:12.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:12.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:12.415127+0000) 2024-03-06T09:48:13.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:13.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:13.415357+0000) 2024-03-06T09:48:14.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:14.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:14.415644+0000) 2024-03-06T09:48:14.950 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:48:14.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:48:15.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:15.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:15.415914+0000) 2024-03-06T09:48:16.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:16.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:16.416114+0000) 2024-03-06T09:48:17.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:17.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:17.416309+0000) 2024-03-06T09:48:18.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:18.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:18.416572+0000) 2024-03-06T09:48:19.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:19.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:19.416842+0000) 2024-03-06T09:48:20.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:20.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:20.417135+0000) 2024-03-06T09:48:21.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:21.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:21.417411+0000) 2024-03-06T09:48:22.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:22.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:22.417693+0000) 2024-03-06T09:48:23.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:23.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:23.417963+0000) 2024-03-06T09:48:24.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:24.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:24.418193+0000) 2024-03-06T09:48:25.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:25.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:25.418426+0000) 2024-03-06T09:48:26.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:26.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:26.418688+0000) 2024-03-06T09:48:27.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:27.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:27.418897+0000) 2024-03-06T09:48:28.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:28.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:28.419107+0000) 2024-03-06T09:48:29.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:29.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:29.419342+0000) 2024-03-06T09:48:30.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:30.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:30.419579+0000) 2024-03-06T09:48:31.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:31.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:31.419792+0000) 2024-03-06T09:48:32.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:32.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:32.420045+0000) 2024-03-06T09:48:33.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:33.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:33.420262+0000) 2024-03-06T09:48:34.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:34.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:34.420469+0000) 2024-03-06T09:48:35.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:35.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:35.420698+0000) 2024-03-06T09:48:36.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:36.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:36.420902+0000) 2024-03-06T09:48:37.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:37.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:37.421136+0000) 2024-03-06T09:48:38.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:38.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:38.421361+0000) 2024-03-06T09:48:39.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:39.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:39.421650+0000) 2024-03-06T09:48:40.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:40.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:40.421850+0000) 2024-03-06T09:48:41.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:41.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:41.422040+0000) 2024-03-06T09:48:42.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:42.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:42.422280+0000) 2024-03-06T09:48:43.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:43.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:43.422513+0000) 2024-03-06T09:48:44.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:44.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:44.422800+0000) 2024-03-06T09:48:45.180 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:48:45.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:48:45.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:45.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:45.422999+0000) 2024-03-06T09:48:46.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:46.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:46.423195+0000) 2024-03-06T09:48:47.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:47.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:47.423445+0000) 2024-03-06T09:48:48.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:48.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:48.423649+0000) 2024-03-06T09:48:49.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:49.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:49.423852+0000) 2024-03-06T09:48:50.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:50.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:50.424043+0000) 2024-03-06T09:48:51.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:51.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:51.424275+0000) 2024-03-06T09:48:52.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:52.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:52.424525+0000) 2024-03-06T09:48:53.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:53.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:53.424802+0000) 2024-03-06T09:48:54.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:54.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:54.425006+0000) 2024-03-06T09:48:55.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:55.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:55.425244+0000) 2024-03-06T09:48:56.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:56.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:56.425493+0000) 2024-03-06T09:48:57.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:57.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:57.425745+0000) 2024-03-06T09:48:58.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:58.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:58.425943+0000) 2024-03-06T09:48:59.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:48:59.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:48:59.426171+0000) 2024-03-06T09:49:00.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:00.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:00.426361+0000) 2024-03-06T09:49:01.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:01.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:01.426631+0000) 2024-03-06T09:49:02.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:02.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:02.426838+0000) 2024-03-06T09:49:03.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:03.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:03.427068+0000) 2024-03-06T09:49:04.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:04.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:04.427259+0000) 2024-03-06T09:49:05.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:05.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:05.427505+0000) 2024-03-06T09:49:06.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:06.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:06.427785+0000) 2024-03-06T09:49:07.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:07.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:07.428044+0000) 2024-03-06T09:49:08.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:08.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:08.428264+0000) 2024-03-06T09:49:09.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:09.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:09.428459+0000) 2024-03-06T09:49:10.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:10.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:10.428656+0000) 2024-03-06T09:49:11.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:11.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:11.428871+0000) 2024-03-06T09:49:12.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:12.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:12.429108+0000) 2024-03-06T09:49:13.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:13.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:13.429331+0000) 2024-03-06T09:49:14.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:14.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:14.429612+0000) 2024-03-06T09:49:15.384 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:49:15.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:49:15.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:15.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:15.429783+0000) 2024-03-06T09:49:16.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:16.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:16.430019+0000) 2024-03-06T09:49:17.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:17.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:17.430207+0000) 2024-03-06T09:49:18.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:18.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:18.430442+0000) 2024-03-06T09:49:19.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:19.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:19.430671+0000) 2024-03-06T09:49:20.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:20.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:20.430874+0000) 2024-03-06T09:49:21.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:21.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:21.431099+0000) 2024-03-06T09:49:22.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:22.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:22.431331+0000) 2024-03-06T09:49:23.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:23.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:23.431628+0000) 2024-03-06T09:49:24.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:24.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:24.431874+0000) 2024-03-06T09:49:25.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:25.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:25.432106+0000) 2024-03-06T09:49:26.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:26.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:26.432336+0000) 2024-03-06T09:49:27.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:27.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:27.432662+0000) 2024-03-06T09:49:28.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:28.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:28.432865+0000) 2024-03-06T09:49:29.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:29.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:29.433094+0000) 2024-03-06T09:49:30.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:30.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:30.433325+0000) 2024-03-06T09:49:31.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:31.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:31.433523+0000) 2024-03-06T09:49:32.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:32.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:32.433776+0000) 2024-03-06T09:49:33.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:33.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:33.434014+0000) 2024-03-06T09:49:34.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:34.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:34.434251+0000) 2024-03-06T09:49:35.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:35.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:35.434477+0000) 2024-03-06T09:49:36.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:36.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:36.434729+0000) 2024-03-06T09:49:37.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:37.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:37.434983+0000) 2024-03-06T09:49:38.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:38.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:38.435164+0000) 2024-03-06T09:49:39.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:39.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:39.435426+0000) 2024-03-06T09:49:40.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:40.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:40.435697+0000) 2024-03-06T09:49:41.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:41.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:41.435893+0000) 2024-03-06T09:49:42.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:42.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:42.436115+0000) 2024-03-06T09:49:43.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:43.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:43.436336+0000) 2024-03-06T09:49:44.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:44.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:44.436663+0000) 2024-03-06T09:49:45.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:45.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:45.436950+0000) 2024-03-06T09:49:45.588 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:49:45.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:49:46.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:46.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:46.437185+0000) 2024-03-06T09:49:47.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:47.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:47.437474+0000) 2024-03-06T09:49:48.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:48.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:48.437781+0000) 2024-03-06T09:49:49.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:49.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:49.437962+0000) 2024-03-06T09:49:50.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:50.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:50.438188+0000) 2024-03-06T09:49:51.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:51.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:51.438413+0000) 2024-03-06T09:49:52.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:52.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:52.438694+0000) 2024-03-06T09:49:53.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:53.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:53.438873+0000) 2024-03-06T09:49:54.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:54.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:54.439114+0000) 2024-03-06T09:49:55.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:55.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:55.439403+0000) 2024-03-06T09:49:56.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:56.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:56.439666+0000) 2024-03-06T09:49:57.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:57.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:57.439849+0000) 2024-03-06T09:49:58.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:58.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:58.440077+0000) 2024-03-06T09:49:59.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:49:59.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:49:59.440339+0000) 2024-03-06T09:50:00.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:00.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:00.440571+0000) 2024-03-06T09:50:01.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:01.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:01.440854+0000) 2024-03-06T09:50:02.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:02.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:02.441132+0000) 2024-03-06T09:50:03.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:03.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:03.441412+0000) 2024-03-06T09:50:04.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:04.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:04.441710+0000) 2024-03-06T09:50:05.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:05.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:05.441917+0000) 2024-03-06T09:50:06.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:06.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:06.442117+0000) 2024-03-06T09:50:07.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:07.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:07.442308+0000) 2024-03-06T09:50:08.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:08.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:08.442575+0000) 2024-03-06T09:50:09.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:09.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:09.442783+0000) 2024-03-06T09:50:10.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:10.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:10.442979+0000) 2024-03-06T09:50:11.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:11.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:11.443242+0000) 2024-03-06T09:50:12.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:12.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:12.443566+0000) 2024-03-06T09:50:13.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:13.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:13.443849+0000) 2024-03-06T09:50:14.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:14.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:14.444131+0000) 2024-03-06T09:50:15.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:15.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:15.444382+0000) 2024-03-06T09:50:15.760 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:50:15.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:50:16.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:16.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:16.444666+0000) 2024-03-06T09:50:17.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:17.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:17.444850+0000) 2024-03-06T09:50:18.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:18.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:18.445051+0000) 2024-03-06T09:50:19.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:19.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:19.445307+0000) 2024-03-06T09:50:20.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:20.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:20.445607+0000) 2024-03-06T09:50:21.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:21.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:21.445810+0000) 2024-03-06T09:50:22.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:22.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:22.446047+0000) 2024-03-06T09:50:23.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:23.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:23.446292+0000) 2024-03-06T09:50:24.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:24.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:24.446597+0000) 2024-03-06T09:50:25.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:25.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:25.446851+0000) 2024-03-06T09:50:26.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:26.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:26.447100+0000) 2024-03-06T09:50:27.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:27.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:27.447347+0000) 2024-03-06T09:50:28.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:28.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:28.447675+0000) 2024-03-06T09:50:29.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:29.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:29.447858+0000) 2024-03-06T09:50:30.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:30.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:30.448107+0000) 2024-03-06T09:50:31.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:31.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:31.448344+0000) 2024-03-06T09:50:32.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:32.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:32.448627+0000) 2024-03-06T09:50:33.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:33.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:33.448871+0000) 2024-03-06T09:50:34.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:34.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:34.449130+0000) 2024-03-06T09:50:35.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:35.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:35.449359+0000) 2024-03-06T09:50:36.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:36.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:36.449607+0000) 2024-03-06T09:50:37.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:37.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:37.449820+0000) 2024-03-06T09:50:38.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:38.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:38.450072+0000) 2024-03-06T09:50:39.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:39.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:39.450317+0000) 2024-03-06T09:50:40.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:40.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:40.450624+0000) 2024-03-06T09:50:41.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:41.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:41.450876+0000) 2024-03-06T09:50:42.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:42.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:42.451162+0000) 2024-03-06T09:50:43.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:43.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:43.451392+0000) 2024-03-06T09:50:44.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:44.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:44.451665+0000) 2024-03-06T09:50:45.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:45.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:45.451901+0000) 2024-03-06T09:50:46.152 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:50:46.161 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:50:46.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:46.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:46.452144+0000) 2024-03-06T09:50:47.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:47.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:47.452342+0000) 2024-03-06T09:50:48.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:48.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:48.452586+0000) 2024-03-06T09:50:49.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:49.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:49.452827+0000) 2024-03-06T09:50:50.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:50.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:50.453065+0000) 2024-03-06T09:50:51.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:51.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:51.453272+0000) 2024-03-06T09:50:52.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:52.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:52.453519+0000) 2024-03-06T09:50:53.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:53.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:53.453800+0000) 2024-03-06T09:50:54.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:54.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:54.454039+0000) 2024-03-06T09:50:55.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:55.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:55.454270+0000) 2024-03-06T09:50:56.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:56.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:56.454496+0000) 2024-03-06T09:50:57.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:57.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:57.454725+0000) 2024-03-06T09:50:58.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:58.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:58.454933+0000) 2024-03-06T09:50:59.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:50:59.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:50:59.455179+0000) 2024-03-06T09:51:00.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:00.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:00.455415+0000) 2024-03-06T09:51:01.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:01.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:01.455639+0000) 2024-03-06T09:51:02.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:02.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:02.455787+0000) 2024-03-06T09:51:03.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:03.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:03.456036+0000) 2024-03-06T09:51:04.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:04.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:04.456287+0000) 2024-03-06T09:51:05.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:05.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:05.456514+0000) 2024-03-06T09:51:06.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:06.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:06.456792+0000) 2024-03-06T09:51:07.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:07.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:07.457077+0000) 2024-03-06T09:51:08.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:08.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:08.457365+0000) 2024-03-06T09:51:09.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:09.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:09.457682+0000) 2024-03-06T09:51:10.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:10.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:10.457965+0000) 2024-03-06T09:51:11.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:11.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:11.458249+0000) 2024-03-06T09:51:12.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:12.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:12.458527+0000) 2024-03-06T09:51:13.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:13.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:13.458871+0000) 2024-03-06T09:51:14.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:14.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:14.459161+0000) 2024-03-06T09:51:15.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:15.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:15.459449+0000) 2024-03-06T09:51:16.302 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:51:16.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:51:16.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:16.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:16.459686+0000) 2024-03-06T09:51:17.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:17.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:17.459919+0000) 2024-03-06T09:51:18.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:18.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:18.460144+0000) 2024-03-06T09:51:19.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:19.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:19.460382+0000) 2024-03-06T09:51:20.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:20.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:20.460680+0000) 2024-03-06T09:51:21.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:21.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:21.460864+0000) 2024-03-06T09:51:22.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:22.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:22.461146+0000) 2024-03-06T09:51:23.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:23.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:23.461417+0000) 2024-03-06T09:51:24.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:24.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:24.461718+0000) 2024-03-06T09:51:25.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:25.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:25.461958+0000) 2024-03-06T09:51:26.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:26.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:26.462179+0000) 2024-03-06T09:51:27.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:27.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:27.462408+0000) 2024-03-06T09:51:28.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:28.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:28.462648+0000) 2024-03-06T09:51:29.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:29.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:29.462875+0000) 2024-03-06T09:51:30.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:30.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:30.463181+0000) 2024-03-06T09:51:31.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:31.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:31.463475+0000) 2024-03-06T09:51:32.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:32.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:32.463703+0000) 2024-03-06T09:51:33.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:33.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:33.463904+0000) 2024-03-06T09:51:34.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:34.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:34.464158+0000) 2024-03-06T09:51:35.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:35.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:35.464386+0000) 2024-03-06T09:51:36.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:36.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:36.464661+0000) 2024-03-06T09:51:37.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:37.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:37.464849+0000) 2024-03-06T09:51:38.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:38.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:38.465075+0000) 2024-03-06T09:51:39.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:39.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:39.465271+0000) 2024-03-06T09:51:40.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:40.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:40.465486+0000) 2024-03-06T09:51:41.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:41.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:41.465682+0000) 2024-03-06T09:51:42.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:42.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:42.465903+0000) 2024-03-06T09:51:43.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:43.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:43.466126+0000) 2024-03-06T09:51:44.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:44.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:44.466367+0000) 2024-03-06T09:51:45.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:45.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:45.466642+0000) 2024-03-06T09:51:46.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:46.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:46.466877+0000) 2024-03-06T09:51:46.508 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:51:46.517 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:51:47.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:47.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:47.467124+0000) 2024-03-06T09:51:48.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:48.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:48.467362+0000) 2024-03-06T09:51:49.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:49.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:49.467636+0000) 2024-03-06T09:51:50.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:50.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:50.467870+0000) 2024-03-06T09:51:51.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:51.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:51.468100+0000) 2024-03-06T09:51:52.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:52.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:52.468275+0000) 2024-03-06T09:51:53.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:53.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:53.468510+0000) 2024-03-06T09:51:54.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:54.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:54.468863+0000) 2024-03-06T09:51:55.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:55.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:55.469147+0000) 2024-03-06T09:51:56.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:56.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:56.469380+0000) 2024-03-06T09:51:57.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:57.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:57.469695+0000) 2024-03-06T09:51:58.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:58.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:58.469946+0000) 2024-03-06T09:51:59.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:51:59.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:51:59.470231+0000) 2024-03-06T09:52:00.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:00.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:00.470515+0000) 2024-03-06T09:52:01.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:01.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:01.470837+0000) 2024-03-06T09:52:02.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:02.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:02.471049+0000) 2024-03-06T09:52:03.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:03.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:03.471284+0000) 2024-03-06T09:52:04.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:04.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:04.471494+0000) 2024-03-06T09:52:05.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:05.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:05.471688+0000) 2024-03-06T09:52:06.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:06.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:06.471851+0000) 2024-03-06T09:52:07.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:07.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:07.472070+0000) 2024-03-06T09:52:08.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:08.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:08.472302+0000) 2024-03-06T09:52:09.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:09.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:09.472488+0000) 2024-03-06T09:52:10.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:10.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:10.472724+0000) 2024-03-06T09:52:11.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:11.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:11.472968+0000) 2024-03-06T09:52:12.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:12.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:12.473198+0000) 2024-03-06T09:52:13.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:13.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:13.473454+0000) 2024-03-06T09:52:14.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:14.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:14.473683+0000) 2024-03-06T09:52:15.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:15.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:15.473900+0000) 2024-03-06T09:52:16.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:16.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:16.474140+0000) 2024-03-06T09:52:16.723 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:52:16.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:52:17.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:17.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:17.474364+0000) 2024-03-06T09:52:18.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:18.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:18.474668+0000) 2024-03-06T09:52:19.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:19.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:19.474868+0000) 2024-03-06T09:52:20.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:20.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:20.475119+0000) 2024-03-06T09:52:21.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:21.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:21.475348+0000) 2024-03-06T09:52:22.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:22.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:22.475607+0000) 2024-03-06T09:52:23.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:23.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:23.475897+0000) 2024-03-06T09:52:24.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:24.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:24.476102+0000) 2024-03-06T09:52:25.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:25.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:25.476346+0000) 2024-03-06T09:52:26.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:26.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:26.476612+0000) 2024-03-06T09:52:27.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:27.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:27.476839+0000) 2024-03-06T09:52:28.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:28.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:28.477081+0000) 2024-03-06T09:52:29.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:29.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:29.477302+0000) 2024-03-06T09:52:30.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:30.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:30.477566+0000) 2024-03-06T09:52:31.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:31.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:31.477803+0000) 2024-03-06T09:52:32.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:32.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:32.478033+0000) 2024-03-06T09:52:33.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:33.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:33.478277+0000) 2024-03-06T09:52:34.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:34.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:34.478526+0000) 2024-03-06T09:52:35.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:35.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:35.478823+0000) 2024-03-06T09:52:36.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:36.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:36.479071+0000) 2024-03-06T09:52:37.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:37.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:37.479263+0000) 2024-03-06T09:52:38.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:38.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:38.479498+0000) 2024-03-06T09:52:39.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:39.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:39.479698+0000) 2024-03-06T09:52:40.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:40.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:40.479884+0000) 2024-03-06T09:52:41.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:41.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:41.480107+0000) 2024-03-06T09:52:42.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:42.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:42.480331+0000) 2024-03-06T09:52:43.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:43.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:43.480601+0000) 2024-03-06T09:52:44.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:44.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:44.480813+0000) 2024-03-06T09:52:45.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:45.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:45.481013+0000) 2024-03-06T09:52:46.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:46.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:46.481301+0000) 2024-03-06T09:52:46.859 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:52:46.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:52:47.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:47.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:47.481635+0000) 2024-03-06T09:52:48.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:48.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:48.481864+0000) 2024-03-06T09:52:49.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:49.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:49.482118+0000) 2024-03-06T09:52:50.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:50.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:50.482355+0000) 2024-03-06T09:52:51.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:51.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:51.482668+0000) 2024-03-06T09:52:52.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:52.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:52.482913+0000) 2024-03-06T09:52:53.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:53.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:53.483194+0000) 2024-03-06T09:52:54.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:54.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:54.483441+0000) 2024-03-06T09:52:55.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:55.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:55.483697+0000) 2024-03-06T09:52:56.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:56.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:56.483913+0000) 2024-03-06T09:52:57.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:57.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:57.484180+0000) 2024-03-06T09:52:58.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:58.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:58.484457+0000) 2024-03-06T09:52:59.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:52:59.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:52:59.484715+0000) 2024-03-06T09:53:00.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:00.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:00.484964+0000) 2024-03-06T09:53:01.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:01.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:01.485225+0000) 2024-03-06T09:53:02.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:02.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:02.485494+0000) 2024-03-06T09:53:03.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:03.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:03.485763+0000) 2024-03-06T09:53:04.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:04.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:04.485987+0000) 2024-03-06T09:53:05.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:05.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:05.486218+0000) 2024-03-06T09:53:06.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:06.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:06.486444+0000) 2024-03-06T09:53:07.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:07.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:07.486653+0000) 2024-03-06T09:53:08.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:08.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:08.486883+0000) 2024-03-06T09:53:09.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:09.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:09.487114+0000) 2024-03-06T09:53:10.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:10.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:10.487377+0000) 2024-03-06T09:53:11.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:11.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:11.487705+0000) 2024-03-06T09:53:12.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:12.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:12.487912+0000) 2024-03-06T09:53:13.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:13.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:13.488160+0000) 2024-03-06T09:53:14.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:14.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:14.488402+0000) 2024-03-06T09:53:15.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:15.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:15.488683+0000) 2024-03-06T09:53:16.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:16.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:16.488930+0000) 2024-03-06T09:53:17.064 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:53:17.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:53:17.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:17.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:17.489149+0000) 2024-03-06T09:53:18.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:18.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:18.489397+0000) 2024-03-06T09:53:19.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:19.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:19.489671+0000) 2024-03-06T09:53:20.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:20.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:20.489926+0000) 2024-03-06T09:53:21.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:21.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:21.490148+0000) 2024-03-06T09:53:22.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:22.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:22.490377+0000) 2024-03-06T09:53:23.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:23.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:23.490627+0000) 2024-03-06T09:53:24.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:24.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:24.490835+0000) 2024-03-06T09:53:25.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:25.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:25.491102+0000) 2024-03-06T09:53:26.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:26.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:26.491380+0000) 2024-03-06T09:53:27.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:27.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:27.491683+0000) 2024-03-06T09:53:28.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:28.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:28.491968+0000) 2024-03-06T09:53:29.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:29.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:29.492251+0000) 2024-03-06T09:53:30.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:30.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:30.492459+0000) 2024-03-06T09:53:31.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:31.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:31.492690+0000) 2024-03-06T09:53:32.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:32.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:32.492946+0000) 2024-03-06T09:53:33.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:33.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:33.493188+0000) 2024-03-06T09:53:34.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:34.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:34.493430+0000) 2024-03-06T09:53:35.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:35.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:35.493657+0000) 2024-03-06T09:53:36.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:36.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:36.493858+0000) 2024-03-06T09:53:37.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:37.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:37.494123+0000) 2024-03-06T09:53:38.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:38.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:38.494351+0000) 2024-03-06T09:53:39.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:39.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:39.494668+0000) 2024-03-06T09:53:40.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:40.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:40.494917+0000) 2024-03-06T09:53:41.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:41.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:41.495186+0000) 2024-03-06T09:53:42.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:42.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:42.495460+0000) 2024-03-06T09:53:43.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:43.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:43.495733+0000) 2024-03-06T09:53:44.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:44.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:44.495981+0000) 2024-03-06T09:53:45.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:45.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:45.496248+0000) 2024-03-06T09:53:46.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:46.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:46.496523+0000) 2024-03-06T09:53:47.269 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:53:47.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:53:47.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:47.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:47.496783+0000) 2024-03-06T09:53:48.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:48.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:48.497059+0000) 2024-03-06T09:53:49.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:49.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:49.497342+0000) 2024-03-06T09:53:50.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:50.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:50.497634+0000) 2024-03-06T09:53:51.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:51.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:51.497852+0000) 2024-03-06T09:53:52.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:52.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:52.498077+0000) 2024-03-06T09:53:53.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:53.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:53.498312+0000) 2024-03-06T09:53:54.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:54.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:54.498595+0000) 2024-03-06T09:53:55.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:55.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:55.498835+0000) 2024-03-06T09:53:56.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:56.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:56.499121+0000) 2024-03-06T09:53:57.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:57.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:57.499404+0000) 2024-03-06T09:53:58.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:58.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:58.499694+0000) 2024-03-06T09:53:59.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:53:59.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:53:59.499935+0000) 2024-03-06T09:54:00.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:00.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:00.500206+0000) 2024-03-06T09:54:01.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:01.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:01.500468+0000) 2024-03-06T09:54:02.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:02.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:02.500721+0000) 2024-03-06T09:54:03.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:03.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:03.500976+0000) 2024-03-06T09:54:04.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:04.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:04.501317+0000) 2024-03-06T09:54:05.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:05.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:05.501635+0000) 2024-03-06T09:54:06.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:06.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:06.501877+0000) 2024-03-06T09:54:07.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:07.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:07.502140+0000) 2024-03-06T09:54:08.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:08.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:08.502417+0000) 2024-03-06T09:54:09.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:09.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:09.502671+0000) 2024-03-06T09:54:10.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:10.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:10.502880+0000) 2024-03-06T09:54:11.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:11.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:11.503109+0000) 2024-03-06T09:54:12.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:12.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:12.503343+0000) 2024-03-06T09:54:13.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:13.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:13.503635+0000) 2024-03-06T09:54:14.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:14.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:14.503835+0000) 2024-03-06T09:54:15.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:15.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:15.504016+0000) 2024-03-06T09:54:16.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:16.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:16.504273+0000) 2024-03-06T09:54:17.489 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:54:17.497 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:54:17.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:17.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:17.504521+0000) 2024-03-06T09:54:18.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:18.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:18.504788+0000) 2024-03-06T09:54:19.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:19.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:19.505013+0000) 2024-03-06T09:54:20.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:20.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:20.505250+0000) 2024-03-06T09:54:21.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:21.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:21.505425+0000) 2024-03-06T09:54:22.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:22.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:22.505659+0000) 2024-03-06T09:54:23.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:23.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:23.505925+0000) 2024-03-06T09:54:24.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:24.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:24.506153+0000) 2024-03-06T09:54:25.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:25.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:25.506393+0000) 2024-03-06T09:54:26.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:26.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:26.506645+0000) 2024-03-06T09:54:27.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:27.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:27.506848+0000) 2024-03-06T09:54:28.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:28.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:28.507108+0000) 2024-03-06T09:54:29.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:29.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:29.507352+0000) 2024-03-06T09:54:30.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:30.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:30.507667+0000) 2024-03-06T09:54:31.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:31.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:31.507905+0000) 2024-03-06T09:54:32.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:32.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:32.508145+0000) 2024-03-06T09:54:33.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:33.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:33.508401+0000) 2024-03-06T09:54:34.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:34.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:34.508647+0000) 2024-03-06T09:54:35.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:35.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:35.508875+0000) 2024-03-06T09:54:36.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:36.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:36.509083+0000) 2024-03-06T09:54:37.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:37.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:37.509275+0000) 2024-03-06T09:54:38.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:38.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:38.509521+0000) 2024-03-06T09:54:39.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:39.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:39.509796+0000) 2024-03-06T09:54:40.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:40.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:40.510023+0000) 2024-03-06T09:54:41.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:41.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:41.510243+0000) 2024-03-06T09:54:42.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:42.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:42.510472+0000) 2024-03-06T09:54:43.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:43.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:43.510722+0000) 2024-03-06T09:54:44.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:44.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:44.510963+0000) 2024-03-06T09:54:45.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:45.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:45.511195+0000) 2024-03-06T09:54:46.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:46.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:46.511375+0000) 2024-03-06T09:54:47.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:47.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:47.511610+0000) 2024-03-06T09:54:47.636 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:54:47.644 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:54:48.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:48.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:48.511863+0000) 2024-03-06T09:54:49.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:49.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:49.512094+0000) 2024-03-06T09:54:50.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:50.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:50.512321+0000) 2024-03-06T09:54:51.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:51.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:51.512612+0000) 2024-03-06T09:54:52.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:52.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:52.512863+0000) 2024-03-06T09:54:53.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:53.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:53.513154+0000) 2024-03-06T09:54:54.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:54.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:54.513374+0000) 2024-03-06T09:54:55.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:55.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:55.513639+0000) 2024-03-06T09:54:56.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:56.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:56.513843+0000) 2024-03-06T09:54:57.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:57.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:57.514070+0000) 2024-03-06T09:54:58.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:58.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:58.514358+0000) 2024-03-06T09:54:59.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:54:59.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:54:59.514629+0000) 2024-03-06T09:55:00.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:00.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:00.514865+0000) 2024-03-06T09:55:01.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:01.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:01.515078+0000) 2024-03-06T09:55:02.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:02.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:02.515296+0000) 2024-03-06T09:55:03.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:03.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:03.515576+0000) 2024-03-06T09:55:04.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:04.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:04.515809+0000) 2024-03-06T09:55:05.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:05.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:05.516048+0000) 2024-03-06T09:55:06.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:06.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:06.516272+0000) 2024-03-06T09:55:07.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:07.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:07.516444+0000) 2024-03-06T09:55:08.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:08.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:08.516717+0000) 2024-03-06T09:55:09.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:09.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:09.516970+0000) 2024-03-06T09:55:10.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:10.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:10.517257+0000) 2024-03-06T09:55:11.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:11.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:11.517521+0000) 2024-03-06T09:55:12.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:12.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:12.517797+0000) 2024-03-06T09:55:13.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:13.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:13.518039+0000) 2024-03-06T09:55:14.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:14.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:14.518282+0000) 2024-03-06T09:55:15.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:15.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:15.518637+0000) 2024-03-06T09:55:16.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:16.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:16.518883+0000) 2024-03-06T09:55:17.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:17.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:17.519069+0000) 2024-03-06T09:55:17.784 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:55:17.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:55:18.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:18.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:18.519300+0000) 2024-03-06T09:55:19.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:19.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:19.519527+0000) 2024-03-06T09:55:20.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:20.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:20.519807+0000) 2024-03-06T09:55:21.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:21.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:21.520031+0000) 2024-03-06T09:55:22.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:22.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:22.520255+0000) 2024-03-06T09:55:23.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:23.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:23.520484+0000) 2024-03-06T09:55:24.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:24.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:24.520689+0000) 2024-03-06T09:55:25.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:25.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:25.520883+0000) 2024-03-06T09:55:26.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:26.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:26.521107+0000) 2024-03-06T09:55:27.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:27.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:27.521308+0000) 2024-03-06T09:55:28.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:28.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:28.521517+0000) 2024-03-06T09:55:29.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:29.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:29.521743+0000) 2024-03-06T09:55:30.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:30.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:30.521943+0000) 2024-03-06T09:55:31.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:31.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:31.522130+0000) 2024-03-06T09:55:32.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:32.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:32.522312+0000) 2024-03-06T09:55:33.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:33.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:33.522577+0000) 2024-03-06T09:55:34.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:34.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:34.522791+0000) 2024-03-06T09:55:35.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:35.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:35.523031+0000) 2024-03-06T09:55:36.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:36.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:36.523298+0000) 2024-03-06T09:55:37.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:37.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:37.523622+0000) 2024-03-06T09:55:38.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:38.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:38.523869+0000) 2024-03-06T09:55:39.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:39.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:39.524145+0000) 2024-03-06T09:55:40.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:40.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:40.524395+0000) 2024-03-06T09:55:41.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:41.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:41.524638+0000) 2024-03-06T09:55:42.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:42.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:42.524889+0000) 2024-03-06T09:55:43.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:43.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:43.525167+0000) 2024-03-06T09:55:44.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:44.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:44.525461+0000) 2024-03-06T09:55:45.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:45.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:45.525664+0000) 2024-03-06T09:55:46.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:46.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:46.525941+0000) 2024-03-06T09:55:47.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:47.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:47.526168+0000) 2024-03-06T09:55:48.095 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:55:48.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:55:48.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:48.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:48.526405+0000) 2024-03-06T09:55:49.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:49.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:49.526656+0000) 2024-03-06T09:55:50.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:50.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:50.526866+0000) 2024-03-06T09:55:51.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:51.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:51.527052+0000) 2024-03-06T09:55:52.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:52.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:52.527260+0000) 2024-03-06T09:55:53.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:53.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:53.527416+0000) 2024-03-06T09:55:54.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:54.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:54.527658+0000) 2024-03-06T09:55:55.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:55.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:55.527841+0000) 2024-03-06T09:55:56.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:56.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:56.528031+0000) 2024-03-06T09:55:57.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:57.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:57.528227+0000) 2024-03-06T09:55:58.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:58.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:58.528412+0000) 2024-03-06T09:55:59.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:55:59.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:55:59.528664+0000) 2024-03-06T09:56:00.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:00.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:00.528899+0000) 2024-03-06T09:56:01.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:01.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:01.529117+0000) 2024-03-06T09:56:02.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:02.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:02.529341+0000) 2024-03-06T09:56:03.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:03.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:03.529518+0000) 2024-03-06T09:56:04.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:04.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:04.529830+0000) 2024-03-06T09:56:05.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:05.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:05.530079+0000) 2024-03-06T09:56:06.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:06.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:06.530296+0000) 2024-03-06T09:56:07.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:07.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:07.530568+0000) 2024-03-06T09:56:08.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:08.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:08.530783+0000) 2024-03-06T09:56:09.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:09.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:09.530988+0000) 2024-03-06T09:56:10.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:10.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:10.531238+0000) 2024-03-06T09:56:11.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:11.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:11.531483+0000) 2024-03-06T09:56:12.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:12.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:12.531736+0000) 2024-03-06T09:56:13.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:13.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:13.531957+0000) 2024-03-06T09:56:14.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:14.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:14.532193+0000) 2024-03-06T09:56:15.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:15.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:15.532419+0000) 2024-03-06T09:56:16.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:16.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:16.532638+0000) 2024-03-06T09:56:17.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:17.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:17.532845+0000) 2024-03-06T09:56:18.308 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:56:18.316 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:56:18.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:18.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:18.533093+0000) 2024-03-06T09:56:19.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:19.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:19.533316+0000) 2024-03-06T09:56:20.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:20.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:20.533579+0000) 2024-03-06T09:56:21.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:21.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:21.533801+0000) 2024-03-06T09:56:22.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:22.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:22.534025+0000) 2024-03-06T09:56:23.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:23.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:23.534249+0000) 2024-03-06T09:56:24.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:24.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:24.534479+0000) 2024-03-06T09:56:25.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:25.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:25.534708+0000) 2024-03-06T09:56:26.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:26.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:26.534949+0000) 2024-03-06T09:56:27.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:27.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:27.535162+0000) 2024-03-06T09:56:28.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:28.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:28.535345+0000) 2024-03-06T09:56:29.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:29.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:29.535611+0000) 2024-03-06T09:56:30.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:30.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:30.535884+0000) 2024-03-06T09:56:31.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:31.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:31.536165+0000) 2024-03-06T09:56:32.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:32.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:32.536445+0000) 2024-03-06T09:56:33.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:33.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:33.536733+0000) 2024-03-06T09:56:34.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:34.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:34.536968+0000) 2024-03-06T09:56:35.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:35.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:35.537166+0000) 2024-03-06T09:56:36.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:36.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:36.537395+0000) 2024-03-06T09:56:37.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:37.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:37.537695+0000) 2024-03-06T09:56:38.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:38.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:38.537961+0000) 2024-03-06T09:56:39.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:39.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:39.538294+0000) 2024-03-06T09:56:40.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:40.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:40.538624+0000) 2024-03-06T09:56:41.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:41.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:41.538832+0000) 2024-03-06T09:56:42.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:42.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:42.539041+0000) 2024-03-06T09:56:43.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:43.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:43.539259+0000) 2024-03-06T09:56:44.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:44.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:44.539498+0000) 2024-03-06T09:56:45.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:45.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:45.539735+0000) 2024-03-06T09:56:46.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:46.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:46.539969+0000) 2024-03-06T09:56:47.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:47.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:47.540188+0000) 2024-03-06T09:56:48.493 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:56:48.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:56:48.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:48.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:48.540378+0000) 2024-03-06T09:56:49.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:49.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:49.540597+0000) 2024-03-06T09:56:50.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:50.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:50.540808+0000) 2024-03-06T09:56:51.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:51.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:51.541019+0000) 2024-03-06T09:56:52.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:52.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:52.541253+0000) 2024-03-06T09:56:53.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:53.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:53.541484+0000) 2024-03-06T09:56:54.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:54.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:54.541733+0000) 2024-03-06T09:56:55.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:55.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:55.541973+0000) 2024-03-06T09:56:56.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:56.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:56.542198+0000) 2024-03-06T09:56:57.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:57.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:57.542429+0000) 2024-03-06T09:56:58.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:58.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:58.542662+0000) 2024-03-06T09:56:59.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:56:59.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:56:59.542874+0000) 2024-03-06T09:57:00.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:00.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:00.543059+0000) 2024-03-06T09:57:01.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:01.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:01.543269+0000) 2024-03-06T09:57:02.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:02.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:02.543510+0000) 2024-03-06T09:57:03.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:03.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:03.543764+0000) 2024-03-06T09:57:04.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:04.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:04.544029+0000) 2024-03-06T09:57:05.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:05.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:05.544273+0000) 2024-03-06T09:57:06.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:06.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:06.544522+0000) 2024-03-06T09:57:07.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:07.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:07.544826+0000) 2024-03-06T09:57:08.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:08.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:08.545041+0000) 2024-03-06T09:57:09.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:09.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:09.545289+0000) 2024-03-06T09:57:10.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:10.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:10.545566+0000) 2024-03-06T09:57:11.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:11.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:11.545804+0000) 2024-03-06T09:57:12.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:12.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:12.545999+0000) 2024-03-06T09:57:13.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:13.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:13.546188+0000) 2024-03-06T09:57:14.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:14.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:14.546440+0000) 2024-03-06T09:57:15.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:15.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:15.546689+0000) 2024-03-06T09:57:16.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:16.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:16.546892+0000) 2024-03-06T09:57:17.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:17.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:17.547084+0000) 2024-03-06T09:57:18.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:18.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:18.547320+0000) 2024-03-06T09:57:18.691 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:57:18.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:57:19.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:19.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:19.547591+0000) 2024-03-06T09:57:20.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:20.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:20.547831+0000) 2024-03-06T09:57:21.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:21.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:21.548057+0000) 2024-03-06T09:57:22.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:22.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:22.548284+0000) 2024-03-06T09:57:23.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:23.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:23.548517+0000) 2024-03-06T09:57:24.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:24.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:24.548866+0000) 2024-03-06T09:57:25.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:25.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:25.549062+0000) 2024-03-06T09:57:26.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:26.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:26.549288+0000) 2024-03-06T09:57:27.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:27.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:27.549592+0000) 2024-03-06T09:57:28.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:28.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:28.549817+0000) 2024-03-06T09:57:29.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:29.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:29.550044+0000) 2024-03-06T09:57:30.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:30.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:30.550277+0000) 2024-03-06T09:57:31.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:31.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:31.550519+0000) 2024-03-06T09:57:32.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:32.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:32.550808+0000) 2024-03-06T09:57:33.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:33.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:33.551023+0000) 2024-03-06T09:57:34.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:34.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:34.551288+0000) 2024-03-06T09:57:35.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:35.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:35.551509+0000) 2024-03-06T09:57:36.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:36.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:36.551788+0000) 2024-03-06T09:57:37.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:37.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:37.552031+0000) 2024-03-06T09:57:38.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:38.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:38.552275+0000) 2024-03-06T09:57:39.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:39.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:39.552526+0000) 2024-03-06T09:57:40.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:40.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:40.552806+0000) 2024-03-06T09:57:41.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:41.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:41.553054+0000) 2024-03-06T09:57:42.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:42.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:42.553323+0000) 2024-03-06T09:57:43.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:43.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:43.553600+0000) 2024-03-06T09:57:44.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:44.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:44.553806+0000) 2024-03-06T09:57:45.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:45.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:45.553985+0000) 2024-03-06T09:57:46.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:46.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:46.554170+0000) 2024-03-06T09:57:47.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:47.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:47.554400+0000) 2024-03-06T09:57:48.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:48.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:48.554718+0000) 2024-03-06T09:57:48.880 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:57:48.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:57:49.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:49.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:49.554945+0000) 2024-03-06T09:57:50.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:50.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:50.555173+0000) 2024-03-06T09:57:51.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:51.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:51.555393+0000) 2024-03-06T09:57:52.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:52.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:52.555658+0000) 2024-03-06T09:57:53.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:53.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:53.555880+0000) 2024-03-06T09:57:54.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:54.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:54.556130+0000) 2024-03-06T09:57:55.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:55.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:55.556354+0000) 2024-03-06T09:57:56.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:56.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:56.556626+0000) 2024-03-06T09:57:57.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:57.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:57.556819+0000) 2024-03-06T09:57:58.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:58.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:58.557060+0000) 2024-03-06T09:57:59.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:57:59.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:57:59.557271+0000) 2024-03-06T09:58:00.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:00.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:00.557484+0000) 2024-03-06T09:58:01.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:01.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:01.557729+0000) 2024-03-06T09:58:02.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:02.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:02.557929+0000) 2024-03-06T09:58:03.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:03.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:03.558153+0000) 2024-03-06T09:58:04.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:04.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:04.558399+0000) 2024-03-06T09:58:05.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:05.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:05.558658+0000) 2024-03-06T09:58:06.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:06.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:06.558885+0000) 2024-03-06T09:58:07.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:07.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:07.559112+0000) 2024-03-06T09:58:08.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:08.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:08.559316+0000) 2024-03-06T09:58:09.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:09.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:09.559605+0000) 2024-03-06T09:58:10.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:10.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:10.559859+0000) 2024-03-06T09:58:11.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:11.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:11.560086+0000) 2024-03-06T09:58:12.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:12.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:12.560287+0000) 2024-03-06T09:58:13.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:13.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:13.560471+0000) 2024-03-06T09:58:14.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:14.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:14.560738+0000) 2024-03-06T09:58:15.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:15.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:15.560931+0000) 2024-03-06T09:58:16.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:16.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:16.561157+0000) 2024-03-06T09:58:17.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:17.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:17.561377+0000) 2024-03-06T09:58:18.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:18.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:18.561646+0000) 2024-03-06T09:58:19.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:58:19.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:58:19.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:19.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:19.561849+0000) 2024-03-06T09:58:20.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:20.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:20.562149+0000) 2024-03-06T09:58:21.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:21.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:21.562366+0000) 2024-03-06T09:58:22.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:22.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:22.562573+0000) 2024-03-06T09:58:23.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:23.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:23.562822+0000) 2024-03-06T09:58:24.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:24.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:24.563083+0000) 2024-03-06T09:58:25.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:25.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:25.563285+0000) 2024-03-06T09:58:26.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:26.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:26.563514+0000) 2024-03-06T09:58:27.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:27.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:27.563779+0000) 2024-03-06T09:58:28.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:28.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:28.563970+0000) 2024-03-06T09:58:29.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:29.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:29.564221+0000) 2024-03-06T09:58:30.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:30.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:30.564453+0000) 2024-03-06T09:58:31.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:31.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:31.564663+0000) 2024-03-06T09:58:32.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:32.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:32.564880+0000) 2024-03-06T09:58:33.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:33.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:33.565084+0000) 2024-03-06T09:58:34.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:34.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:34.565338+0000) 2024-03-06T09:58:35.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:35.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:35.565516+0000) 2024-03-06T09:58:36.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:36.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:36.565789+0000) 2024-03-06T09:58:37.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:37.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:37.565976+0000) 2024-03-06T09:58:38.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:38.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:38.566196+0000) 2024-03-06T09:58:39.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:39.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:39.566431+0000) 2024-03-06T09:58:40.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:40.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:40.566661+0000) 2024-03-06T09:58:41.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:41.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:41.566852+0000) 2024-03-06T09:58:42.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:42.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:42.567062+0000) 2024-03-06T09:58:43.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:43.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:43.567299+0000) 2024-03-06T09:58:44.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:44.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:44.567509+0000) 2024-03-06T09:58:45.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:45.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:45.567738+0000) 2024-03-06T09:58:46.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:46.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:46.567964+0000) 2024-03-06T09:58:47.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:47.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:47.568140+0000) 2024-03-06T09:58:48.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:48.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:48.568370+0000) 2024-03-06T09:58:49.307 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:58:49.317 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:58:49.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:49.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:49.568635+0000) 2024-03-06T09:58:50.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:50.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:50.568897+0000) 2024-03-06T09:58:51.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:51.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:51.569143+0000) 2024-03-06T09:58:52.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:52.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:52.569360+0000) 2024-03-06T09:58:53.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:53.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:53.569640+0000) 2024-03-06T09:58:54.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:54.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:54.569903+0000) 2024-03-06T09:58:55.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:55.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:55.570158+0000) 2024-03-06T09:58:56.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:56.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:56.570401+0000) 2024-03-06T09:58:57.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:57.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:57.570656+0000) 2024-03-06T09:58:58.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:58.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:58.570879+0000) 2024-03-06T09:58:59.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:58:59.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:58:59.571100+0000) 2024-03-06T09:59:00.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:00.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:00.571313+0000) 2024-03-06T09:59:01.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:01.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:01.571568+0000) 2024-03-06T09:59:02.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:02.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:02.571810+0000) 2024-03-06T09:59:03.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:03.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:03.572045+0000) 2024-03-06T09:59:04.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:04.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:04.572273+0000) 2024-03-06T09:59:05.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:05.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:05.572512+0000) 2024-03-06T09:59:06.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:06.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:06.572797+0000) 2024-03-06T09:59:07.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:07.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:07.573018+0000) 2024-03-06T09:59:08.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:08.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:08.573264+0000) 2024-03-06T09:59:09.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:09.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:09.573449+0000) 2024-03-06T09:59:10.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:10.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:10.573671+0000) 2024-03-06T09:59:11.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:11.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:11.573895+0000) 2024-03-06T09:59:12.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:12.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:12.574124+0000) 2024-03-06T09:59:13.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:13.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:13.574344+0000) 2024-03-06T09:59:14.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:14.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:14.574673+0000) 2024-03-06T09:59:15.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:15.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:15.574902+0000) 2024-03-06T09:59:16.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:16.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:16.575092+0000) 2024-03-06T09:59:17.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:17.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:17.575359+0000) 2024-03-06T09:59:18.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:18.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:18.575629+0000) 2024-03-06T09:59:19.530 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:59:19.542 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:59:19.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:19.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:19.575795+0000) 2024-03-06T09:59:20.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:20.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:20.575980+0000) 2024-03-06T09:59:21.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:21.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:21.576214+0000) 2024-03-06T09:59:22.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:22.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:22.576405+0000) 2024-03-06T09:59:23.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:23.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:23.576672+0000) 2024-03-06T09:59:24.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:24.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:24.576921+0000) 2024-03-06T09:59:25.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:25.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:25.577112+0000) 2024-03-06T09:59:26.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:26.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:26.577349+0000) 2024-03-06T09:59:27.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:27.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:27.577636+0000) 2024-03-06T09:59:28.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:28.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:28.577826+0000) 2024-03-06T09:59:29.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:29.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:29.578064+0000) 2024-03-06T09:59:30.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:30.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:30.578271+0000) 2024-03-06T09:59:31.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:31.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:31.578507+0000) 2024-03-06T09:59:32.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:32.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:32.578781+0000) 2024-03-06T09:59:33.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:33.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:33.578980+0000) 2024-03-06T09:59:34.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:34.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:34.579216+0000) 2024-03-06T09:59:35.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:35.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:35.579467+0000) 2024-03-06T09:59:36.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:36.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:36.579692+0000) 2024-03-06T09:59:37.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:37.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:37.579901+0000) 2024-03-06T09:59:38.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:38.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:38.580146+0000) 2024-03-06T09:59:39.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:39.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:39.580387+0000) 2024-03-06T09:59:40.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:40.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:40.580652+0000) 2024-03-06T09:59:41.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:41.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:41.580872+0000) 2024-03-06T09:59:42.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:42.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:42.581062+0000) 2024-03-06T09:59:43.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:43.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:43.581283+0000) 2024-03-06T09:59:44.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:44.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:44.581566+0000) 2024-03-06T09:59:45.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:45.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:45.581812+0000) 2024-03-06T09:59:46.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:46.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:46.582039+0000) 2024-03-06T09:59:47.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:47.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:47.582262+0000) 2024-03-06T09:59:48.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:48.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:48.582491+0000) 2024-03-06T09:59:49.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:49.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:49.582722+0000) 2024-03-06T09:59:49.666 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:59:49.674 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T09:59:50.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:50.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:50.582916+0000) 2024-03-06T09:59:51.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:51.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:51.583137+0000) 2024-03-06T09:59:52.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:52.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:52.583361+0000) 2024-03-06T09:59:53.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:53.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:53.583616+0000) 2024-03-06T09:59:54.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:54.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:54.583880+0000) 2024-03-06T09:59:55.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:55.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:55.584119+0000) 2024-03-06T09:59:56.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:56.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:56.584319+0000) 2024-03-06T09:59:57.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:57.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:57.584589+0000) 2024-03-06T09:59:58.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:58.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:58.584786+0000) 2024-03-06T09:59:59.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T09:59:59.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T08:59:59.585048+0000) 2024-03-06T10:00:00.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:00.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:00.585294+0000) 2024-03-06T10:00:01.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:01.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:01.585529+0000) 2024-03-06T10:00:02.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:02.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:02.585878+0000) 2024-03-06T10:00:03.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:03.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:03.586112+0000) 2024-03-06T10:00:04.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:04.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:04.586365+0000) 2024-03-06T10:00:05.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:05.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:05.586622+0000) 2024-03-06T10:00:06.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:06.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:06.586860+0000) 2024-03-06T10:00:07.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:07.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:07.587097+0000) 2024-03-06T10:00:08.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:08.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:08.587350+0000) 2024-03-06T10:00:09.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:09.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:09.587642+0000) 2024-03-06T10:00:10.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:10.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:10.587888+0000) 2024-03-06T10:00:11.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:11.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:11.588110+0000) 2024-03-06T10:00:12.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:12.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:12.588304+0000) 2024-03-06T10:00:13.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:13.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:13.588575+0000) 2024-03-06T10:00:14.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:14.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:14.588818+0000) 2024-03-06T10:00:15.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:15.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:15.589054+0000) 2024-03-06T10:00:16.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:16.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:16.589305+0000) 2024-03-06T10:00:17.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:17.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:17.589575+0000) 2024-03-06T10:00:18.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:18.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:18.589827+0000) 2024-03-06T10:00:19.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:19.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:19.590085+0000) 2024-03-06T10:00:19.879 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:00:19.888 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:00:20.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:20.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:20.590322+0000) 2024-03-06T10:00:21.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:21.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:21.590501+0000) 2024-03-06T10:00:22.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:22.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:22.590781+0000) 2024-03-06T10:00:23.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:23.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:23.591004+0000) 2024-03-06T10:00:24.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:24.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:24.591208+0000) 2024-03-06T10:00:25.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:25.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:25.591432+0000) 2024-03-06T10:00:26.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:26.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:26.591661+0000) 2024-03-06T10:00:27.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:27.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:27.591874+0000) 2024-03-06T10:00:28.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:28.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:28.592126+0000) 2024-03-06T10:00:29.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:29.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:29.592379+0000) 2024-03-06T10:00:30.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:30.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:30.592670+0000) 2024-03-06T10:00:31.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:31.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:31.592855+0000) 2024-03-06T10:00:32.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:32.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:32.593092+0000) 2024-03-06T10:00:33.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:33.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:33.593319+0000) 2024-03-06T10:00:34.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:34.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:34.593610+0000) 2024-03-06T10:00:35.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:35.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:35.593839+0000) 2024-03-06T10:00:36.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:36.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:36.594050+0000) 2024-03-06T10:00:37.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:37.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:37.594283+0000) 2024-03-06T10:00:38.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:38.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:38.594517+0000) 2024-03-06T10:00:39.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:39.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:39.594802+0000) 2024-03-06T10:00:40.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:40.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:40.594997+0000) 2024-03-06T10:00:41.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:41.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:41.595182+0000) 2024-03-06T10:00:42.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:42.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:42.595408+0000) 2024-03-06T10:00:43.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:43.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:43.595667+0000) 2024-03-06T10:00:44.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:44.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:44.595958+0000) 2024-03-06T10:00:45.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:45.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:45.596223+0000) 2024-03-06T10:00:46.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:46.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:46.596451+0000) 2024-03-06T10:00:47.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:47.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:47.596673+0000) 2024-03-06T10:00:48.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:48.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:48.596898+0000) 2024-03-06T10:00:49.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:49.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:49.597129+0000) 2024-03-06T10:00:50.065 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:00:50.074 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:00:50.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:50.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:50.597372+0000) 2024-03-06T10:00:51.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:51.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:51.597632+0000) 2024-03-06T10:00:52.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:52.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:52.597883+0000) 2024-03-06T10:00:53.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:53.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:53.598074+0000) 2024-03-06T10:00:54.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:54.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:54.598320+0000) 2024-03-06T10:00:55.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:55.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:55.598648+0000) 2024-03-06T10:00:56.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:56.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:56.598840+0000) 2024-03-06T10:00:57.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:57.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:57.599067+0000) 2024-03-06T10:00:58.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:58.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:58.599299+0000) 2024-03-06T10:00:59.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:00:59.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:00:59.599570+0000) 2024-03-06T10:01:00.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:00.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:00.599815+0000) 2024-03-06T10:01:01.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:01.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:01.600043+0000) 2024-03-06T10:01:02.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:02.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:02.600283+0000) 2024-03-06T10:01:03.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:03.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:03.600471+0000) 2024-03-06T10:01:04.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:04.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:04.600701+0000) 2024-03-06T10:01:05.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:05.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:05.600910+0000) 2024-03-06T10:01:06.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:06.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:06.601158+0000) 2024-03-06T10:01:07.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:07.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:07.601385+0000) 2024-03-06T10:01:08.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:08.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:08.601661+0000) 2024-03-06T10:01:09.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:09.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:09.601896+0000) 2024-03-06T10:01:10.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:10.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:10.602042+0000) 2024-03-06T10:01:11.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:11.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:11.602267+0000) 2024-03-06T10:01:12.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:12.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:12.602497+0000) 2024-03-06T10:01:13.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:13.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:13.602701+0000) 2024-03-06T10:01:14.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:14.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:14.602915+0000) 2024-03-06T10:01:15.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:15.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:15.603109+0000) 2024-03-06T10:01:16.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:16.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:16.603305+0000) 2024-03-06T10:01:17.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:17.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:17.603510+0000) 2024-03-06T10:01:18.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:18.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:18.603804+0000) 2024-03-06T10:01:19.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:19.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:19.603988+0000) 2024-03-06T10:01:20.207 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:01:20.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:01:20.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:20.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:20.604223+0000) 2024-03-06T10:01:21.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:21.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:21.604454+0000) 2024-03-06T10:01:22.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:22.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:22.604717+0000) 2024-03-06T10:01:23.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:23.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:23.604944+0000) 2024-03-06T10:01:24.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:24.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:24.605145+0000) 2024-03-06T10:01:25.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:25.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:25.605385+0000) 2024-03-06T10:01:26.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:26.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:26.605665+0000) 2024-03-06T10:01:27.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:27.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:27.605886+0000) 2024-03-06T10:01:28.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:28.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:28.606140+0000) 2024-03-06T10:01:29.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:29.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:29.606329+0000) 2024-03-06T10:01:30.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:30.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:30.606619+0000) 2024-03-06T10:01:31.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:31.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:31.606854+0000) 2024-03-06T10:01:32.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:32.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:32.607066+0000) 2024-03-06T10:01:33.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:33.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:33.607257+0000) 2024-03-06T10:01:34.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:34.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:34.607479+0000) 2024-03-06T10:01:35.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:35.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:35.607681+0000) 2024-03-06T10:01:36.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:36.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:36.607908+0000) 2024-03-06T10:01:37.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:37.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:37.608127+0000) 2024-03-06T10:01:38.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:38.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:38.608357+0000) 2024-03-06T10:01:39.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:39.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:39.608564+0000) 2024-03-06T10:01:40.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:40.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:40.608798+0000) 2024-03-06T10:01:41.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:41.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:41.609024+0000) 2024-03-06T10:01:42.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:42.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:42.609246+0000) 2024-03-06T10:01:43.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:43.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:43.609517+0000) 2024-03-06T10:01:44.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:44.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:44.609757+0000) 2024-03-06T10:01:45.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:45.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:45.609965+0000) 2024-03-06T10:01:46.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:46.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:46.610109+0000) 2024-03-06T10:01:47.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:47.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:47.610289+0000) 2024-03-06T10:01:48.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:48.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:48.610521+0000) 2024-03-06T10:01:49.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:49.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:49.610786+0000) 2024-03-06T10:01:50.385 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:01:50.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:01:50.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:50.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:50.610955+0000) 2024-03-06T10:01:51.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:51.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:51.611176+0000) 2024-03-06T10:01:52.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:52.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:52.611361+0000) 2024-03-06T10:01:53.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:53.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:53.611572+0000) 2024-03-06T10:01:54.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:54.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:54.611818+0000) 2024-03-06T10:01:55.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:55.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:55.612046+0000) 2024-03-06T10:01:56.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:56.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:56.612255+0000) 2024-03-06T10:01:57.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:57.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:57.612506+0000) 2024-03-06T10:01:58.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:58.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:58.612782+0000) 2024-03-06T10:01:59.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:01:59.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:01:59.613058+0000) 2024-03-06T10:02:00.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:00.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:00.613326+0000) 2024-03-06T10:02:01.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:01.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:01.613613+0000) 2024-03-06T10:02:02.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:02.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:02.613894+0000) 2024-03-06T10:02:03.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:03.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:03.614173+0000) 2024-03-06T10:02:04.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:04.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:04.614467+0000) 2024-03-06T10:02:05.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:05.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:05.614689+0000) 2024-03-06T10:02:06.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:06.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:06.614926+0000) 2024-03-06T10:02:07.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:07.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:07.615145+0000) 2024-03-06T10:02:08.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:08.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:08.615372+0000) 2024-03-06T10:02:09.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:09.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:09.615641+0000) 2024-03-06T10:02:10.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:10.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:10.615879+0000) 2024-03-06T10:02:11.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:11.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:11.616099+0000) 2024-03-06T10:02:12.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:12.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:12.616328+0000) 2024-03-06T10:02:13.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:13.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:13.616591+0000) 2024-03-06T10:02:14.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:14.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:14.616834+0000) 2024-03-06T10:02:15.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:15.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:15.617038+0000) 2024-03-06T10:02:16.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:16.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:16.617264+0000) 2024-03-06T10:02:17.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:17.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:17.617575+0000) 2024-03-06T10:02:18.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:18.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:18.617853+0000) 2024-03-06T10:02:19.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:19.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:19.618130+0000) 2024-03-06T10:02:20.591 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:02:20.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:02:20.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:20.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:20.618316+0000) 2024-03-06T10:02:21.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:21.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:21.618623+0000) 2024-03-06T10:02:22.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:22.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:22.618903+0000) 2024-03-06T10:02:23.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:23.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:23.619182+0000) 2024-03-06T10:02:24.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:24.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:24.619407+0000) 2024-03-06T10:02:25.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:25.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:25.619652+0000) 2024-03-06T10:02:26.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:26.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:26.619883+0000) 2024-03-06T10:02:27.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:27.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:27.620112+0000) 2024-03-06T10:02:28.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:28.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:28.620341+0000) 2024-03-06T10:02:29.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:29.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:29.620604+0000) 2024-03-06T10:02:30.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:30.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:30.620821+0000) 2024-03-06T10:02:31.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:31.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:31.621077+0000) 2024-03-06T10:02:32.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:32.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:32.621281+0000) 2024-03-06T10:02:33.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:33.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:33.621508+0000) 2024-03-06T10:02:34.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:34.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:34.621806+0000) 2024-03-06T10:02:35.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:35.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:35.622041+0000) 2024-03-06T10:02:36.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:36.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:36.622359+0000) 2024-03-06T10:02:37.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:37.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:37.622677+0000) 2024-03-06T10:02:38.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:38.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:38.622880+0000) 2024-03-06T10:02:39.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:39.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:39.623116+0000) 2024-03-06T10:02:40.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:40.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:40.623381+0000) 2024-03-06T10:02:41.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:41.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:41.623680+0000) 2024-03-06T10:02:42.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:42.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:42.623952+0000) 2024-03-06T10:02:43.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:43.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:43.624239+0000) 2024-03-06T10:02:44.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:44.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:44.624463+0000) 2024-03-06T10:02:45.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:45.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:45.624712+0000) 2024-03-06T10:02:46.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:46.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:46.624955+0000) 2024-03-06T10:02:47.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:47.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:47.625205+0000) 2024-03-06T10:02:48.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:48.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:48.625455+0000) 2024-03-06T10:02:49.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:49.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:49.625639+0000) 2024-03-06T10:02:50.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:50.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:50.625874+0000) 2024-03-06T10:02:50.730 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:02:50.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:02:51.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:51.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:51.626115+0000) 2024-03-06T10:02:52.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:52.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:52.626346+0000) 2024-03-06T10:02:53.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:53.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:53.626655+0000) 2024-03-06T10:02:54.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:54.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:54.626870+0000) 2024-03-06T10:02:55.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:55.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:55.627129+0000) 2024-03-06T10:02:56.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:56.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:56.627371+0000) 2024-03-06T10:02:57.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:57.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:57.627616+0000) 2024-03-06T10:02:58.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:58.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:58.627882+0000) 2024-03-06T10:02:59.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:02:59.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:02:59.628132+0000) 2024-03-06T10:03:00.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:00.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:00.628362+0000) 2024-03-06T10:03:01.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:01.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:01.628592+0000) 2024-03-06T10:03:02.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:02.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:02.628800+0000) 2024-03-06T10:03:03.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:03.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:03.629085+0000) 2024-03-06T10:03:04.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:04.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:04.629329+0000) 2024-03-06T10:03:05.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:05.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:05.629633+0000) 2024-03-06T10:03:06.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:06.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:06.629868+0000) 2024-03-06T10:03:07.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:07.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:07.630099+0000) 2024-03-06T10:03:08.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:08.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:08.630341+0000) 2024-03-06T10:03:09.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:09.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:09.630604+0000) 2024-03-06T10:03:10.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:10.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:10.630837+0000) 2024-03-06T10:03:11.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:11.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:11.631041+0000) 2024-03-06T10:03:12.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:12.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:12.631246+0000) 2024-03-06T10:03:13.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:13.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:13.631446+0000) 2024-03-06T10:03:14.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:14.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:14.631678+0000) 2024-03-06T10:03:15.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:15.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:15.631887+0000) 2024-03-06T10:03:16.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:16.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:16.632083+0000) 2024-03-06T10:03:17.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:17.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:17.632263+0000) 2024-03-06T10:03:18.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:18.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:18.632505+0000) 2024-03-06T10:03:19.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:19.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:19.632790+0000) 2024-03-06T10:03:20.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:20.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:20.633045+0000) 2024-03-06T10:03:20.921 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:03:20.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:03:21.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:21.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:21.633256+0000) 2024-03-06T10:03:22.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:22.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:22.633503+0000) 2024-03-06T10:03:23.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:23.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:23.633753+0000) 2024-03-06T10:03:24.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:24.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:24.633969+0000) 2024-03-06T10:03:25.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:25.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:25.634229+0000) 2024-03-06T10:03:26.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:26.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:26.634434+0000) 2024-03-06T10:03:27.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:27.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:27.634746+0000) 2024-03-06T10:03:28.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:28.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:28.634985+0000) 2024-03-06T10:03:29.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:29.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:29.635168+0000) 2024-03-06T10:03:30.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:30.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:30.635469+0000) 2024-03-06T10:03:31.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:31.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:31.635733+0000) 2024-03-06T10:03:32.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:32.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:32.635945+0000) 2024-03-06T10:03:33.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:33.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:33.636132+0000) 2024-03-06T10:03:34.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:34.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:34.636359+0000) 2024-03-06T10:03:35.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:35.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:35.636637+0000) 2024-03-06T10:03:36.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:36.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:36.636878+0000) 2024-03-06T10:03:37.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:37.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:37.637056+0000) 2024-03-06T10:03:38.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:38.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:38.637283+0000) 2024-03-06T10:03:39.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:39.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:39.637458+0000) 2024-03-06T10:03:40.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:40.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:40.637709+0000) 2024-03-06T10:03:41.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:41.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:41.637927+0000) 2024-03-06T10:03:42.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:42.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:42.638152+0000) 2024-03-06T10:03:43.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:43.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:43.638368+0000) 2024-03-06T10:03:44.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:44.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:44.638659+0000) 2024-03-06T10:03:45.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:45.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:45.638891+0000) 2024-03-06T10:03:46.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:46.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:46.639134+0000) 2024-03-06T10:03:47.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:47.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:47.639385+0000) 2024-03-06T10:03:48.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:48.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:48.639694+0000) 2024-03-06T10:03:49.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:49.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:49.639923+0000) 2024-03-06T10:03:50.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:50.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:50.640151+0000) 2024-03-06T10:03:51.118 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:03:51.127 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:03:51.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:51.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:51.640401+0000) 2024-03-06T10:03:52.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:52.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:52.640670+0000) 2024-03-06T10:03:53.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:53.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:53.640856+0000) 2024-03-06T10:03:54.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:54.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:54.641074+0000) 2024-03-06T10:03:55.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:55.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:55.641310+0000) 2024-03-06T10:03:56.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:56.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:56.641501+0000) 2024-03-06T10:03:57.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:57.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:57.641729+0000) 2024-03-06T10:03:58.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:58.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:58.641940+0000) 2024-03-06T10:03:59.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:03:59.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:03:59.642138+0000) 2024-03-06T10:04:00.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:00.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:00.642383+0000) 2024-03-06T10:04:01.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:01.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:01.642627+0000) 2024-03-06T10:04:02.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:02.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:02.642875+0000) 2024-03-06T10:04:03.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:03.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:03.643070+0000) 2024-03-06T10:04:04.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:04.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:04.643316+0000) 2024-03-06T10:04:05.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:05.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:05.643599+0000) 2024-03-06T10:04:06.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:06.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:06.643795+0000) 2024-03-06T10:04:07.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:07.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:07.643997+0000) 2024-03-06T10:04:08.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:08.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:08.644194+0000) 2024-03-06T10:04:09.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:09.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:09.644398+0000) 2024-03-06T10:04:10.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:10.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:10.644648+0000) 2024-03-06T10:04:11.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:11.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:11.644841+0000) 2024-03-06T10:04:12.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:12.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:12.645055+0000) 2024-03-06T10:04:13.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:13.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:13.645289+0000) 2024-03-06T10:04:14.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:14.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:14.645564+0000) 2024-03-06T10:04:15.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:15.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:15.645801+0000) 2024-03-06T10:04:16.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:16.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:16.645996+0000) 2024-03-06T10:04:17.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:17.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:17.646213+0000) 2024-03-06T10:04:18.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:18.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:18.646413+0000) 2024-03-06T10:04:19.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:19.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:19.646662+0000) 2024-03-06T10:04:20.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:20.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:20.646913+0000) 2024-03-06T10:04:21.334 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:04:21.342 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:04:21.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:21.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:21.647113+0000) 2024-03-06T10:04:22.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:22.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:22.647334+0000) 2024-03-06T10:04:23.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:23.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:23.647526+0000) 2024-03-06T10:04:24.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:24.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:24.647805+0000) 2024-03-06T10:04:25.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:25.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:25.648055+0000) 2024-03-06T10:04:26.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:26.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:26.648263+0000) 2024-03-06T10:04:27.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:27.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:27.648504+0000) 2024-03-06T10:04:28.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:28.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:28.648737+0000) 2024-03-06T10:04:29.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:29.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:29.648956+0000) 2024-03-06T10:04:30.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:30.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:30.649195+0000) 2024-03-06T10:04:31.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:31.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:31.649418+0000) 2024-03-06T10:04:32.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:32.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:32.649660+0000) 2024-03-06T10:04:33.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:33.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:33.649846+0000) 2024-03-06T10:04:34.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:34.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:34.650045+0000) 2024-03-06T10:04:35.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:35.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:35.650278+0000) 2024-03-06T10:04:36.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:36.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:36.650507+0000) 2024-03-06T10:04:37.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:37.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:37.650746+0000) 2024-03-06T10:04:38.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:38.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:38.650952+0000) 2024-03-06T10:04:39.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:39.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:39.651177+0000) 2024-03-06T10:04:40.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:40.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:40.651378+0000) 2024-03-06T10:04:41.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:41.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:41.651641+0000) 2024-03-06T10:04:42.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:42.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:42.651826+0000) 2024-03-06T10:04:43.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:43.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:43.652026+0000) 2024-03-06T10:04:44.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:44.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:44.652233+0000) 2024-03-06T10:04:45.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:45.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:45.652471+0000) 2024-03-06T10:04:46.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:46.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:46.652682+0000) 2024-03-06T10:04:47.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:47.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:47.652893+0000) 2024-03-06T10:04:48.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:48.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:48.653128+0000) 2024-03-06T10:04:49.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:49.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:49.653350+0000) 2024-03-06T10:04:50.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:50.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:50.653569+0000) 2024-03-06T10:04:51.563 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:04:51.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:04:51.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:51.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:51.653746+0000) 2024-03-06T10:04:52.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:52.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:52.653967+0000) 2024-03-06T10:04:53.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:53.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:53.654188+0000) 2024-03-06T10:04:54.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:54.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:54.654413+0000) 2024-03-06T10:04:55.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:55.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:55.654654+0000) 2024-03-06T10:04:56.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:56.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:56.654837+0000) 2024-03-06T10:04:57.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:57.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:57.655060+0000) 2024-03-06T10:04:58.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:58.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:58.655256+0000) 2024-03-06T10:04:59.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:04:59.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:04:59.655435+0000) 2024-03-06T10:05:00.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:00.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:00.655686+0000) 2024-03-06T10:05:01.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:01.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:01.655887+0000) 2024-03-06T10:05:02.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:02.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:02.656123+0000) 2024-03-06T10:05:03.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:03.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:03.656353+0000) 2024-03-06T10:05:04.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:04.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:04.656605+0000) 2024-03-06T10:05:05.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:05.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:05.656810+0000) 2024-03-06T10:05:06.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:06.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:06.657099+0000) 2024-03-06T10:05:07.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:07.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:07.657318+0000) 2024-03-06T10:05:08.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:08.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:08.657561+0000) 2024-03-06T10:05:09.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:09.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:09.657752+0000) 2024-03-06T10:05:10.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:10.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:10.657990+0000) 2024-03-06T10:05:11.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:11.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:11.658221+0000) 2024-03-06T10:05:12.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:12.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:12.658408+0000) 2024-03-06T10:05:13.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:13.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:13.658638+0000) 2024-03-06T10:05:14.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:14.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:14.658838+0000) 2024-03-06T10:05:15.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:15.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:15.659039+0000) 2024-03-06T10:05:16.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:16.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:16.659277+0000) 2024-03-06T10:05:17.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:17.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:17.659553+0000) 2024-03-06T10:05:18.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:18.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:18.659777+0000) 2024-03-06T10:05:19.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:19.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:19.660000+0000) 2024-03-06T10:05:20.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:20.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:20.660259+0000) 2024-03-06T10:05:21.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:21.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:21.660507+0000) 2024-03-06T10:05:21.794 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:05:21.802 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:05:22.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:22.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:22.660757+0000) 2024-03-06T10:05:23.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:23.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:23.660957+0000) 2024-03-06T10:05:24.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:24.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:24.661161+0000) 2024-03-06T10:05:25.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:25.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:25.661409+0000) 2024-03-06T10:05:26.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:26.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:26.661678+0000) 2024-03-06T10:05:27.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:27.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:27.661919+0000) 2024-03-06T10:05:28.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:28.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:28.662127+0000) 2024-03-06T10:05:29.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:29.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:29.662357+0000) 2024-03-06T10:05:30.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:30.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:30.662639+0000) 2024-03-06T10:05:31.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:31.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:31.662850+0000) 2024-03-06T10:05:32.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:32.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:32.663067+0000) 2024-03-06T10:05:33.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:33.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:33.663298+0000) 2024-03-06T10:05:34.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:34.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:34.663486+0000) 2024-03-06T10:05:35.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:35.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:35.663683+0000) 2024-03-06T10:05:36.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:36.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:36.663918+0000) 2024-03-06T10:05:37.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:37.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:37.664114+0000) 2024-03-06T10:05:38.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:38.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:38.664321+0000) 2024-03-06T10:05:39.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:39.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:39.664510+0000) 2024-03-06T10:05:40.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:40.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:40.664757+0000) 2024-03-06T10:05:41.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:41.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:41.664940+0000) 2024-03-06T10:05:42.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:42.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:42.665135+0000) 2024-03-06T10:05:43.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:43.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:43.665338+0000) 2024-03-06T10:05:44.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:44.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:44.665613+0000) 2024-03-06T10:05:45.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:45.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:45.665822+0000) 2024-03-06T10:05:46.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:46.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:46.666014+0000) 2024-03-06T10:05:47.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:47.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:47.666235+0000) 2024-03-06T10:05:48.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:48.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:48.666467+0000) 2024-03-06T10:05:49.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:49.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:49.666685+0000) 2024-03-06T10:05:50.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:50.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:50.666890+0000) 2024-03-06T10:05:51.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:51.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:51.667070+0000) 2024-03-06T10:05:51.973 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:05:51.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:05:52.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:52.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:52.667301+0000) 2024-03-06T10:05:53.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:53.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:53.667520+0000) 2024-03-06T10:05:54.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:54.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:54.667789+0000) 2024-03-06T10:05:55.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:55.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:55.667997+0000) 2024-03-06T10:05:56.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:56.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:56.668237+0000) 2024-03-06T10:05:57.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:57.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:57.668495+0000) 2024-03-06T10:05:58.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:58.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:58.668722+0000) 2024-03-06T10:05:59.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:05:59.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:05:59.668899+0000) 2024-03-06T10:06:00.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:00.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:00.669127+0000) 2024-03-06T10:06:01.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:01.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:01.669340+0000) 2024-03-06T10:06:02.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:02.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:02.669596+0000) 2024-03-06T10:06:03.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:03.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:03.669815+0000) 2024-03-06T10:06:04.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:04.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:04.670015+0000) 2024-03-06T10:06:05.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:05.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:05.670212+0000) 2024-03-06T10:06:06.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:06.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:06.670434+0000) 2024-03-06T10:06:07.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:07.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:07.670663+0000) 2024-03-06T10:06:08.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:08.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:08.670828+0000) 2024-03-06T10:06:09.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:09.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:09.671030+0000) 2024-03-06T10:06:10.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:10.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:10.671282+0000) 2024-03-06T10:06:11.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:11.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:11.671506+0000) 2024-03-06T10:06:12.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:12.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:12.671737+0000) 2024-03-06T10:06:13.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:13.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:13.671932+0000) 2024-03-06T10:06:14.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:14.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:14.672130+0000) 2024-03-06T10:06:15.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:15.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:15.672348+0000) 2024-03-06T10:06:16.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:16.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:16.672618+0000) 2024-03-06T10:06:17.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:17.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:17.672806+0000) 2024-03-06T10:06:18.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:18.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:18.672999+0000) 2024-03-06T10:06:19.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:19.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:19.673180+0000) 2024-03-06T10:06:20.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:20.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:20.673416+0000) 2024-03-06T10:06:21.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:21.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:21.673649+0000) 2024-03-06T10:06:22.203 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:06:22.211 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:06:22.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:22.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:22.673838+0000) 2024-03-06T10:06:23.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:23.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:23.674076+0000) 2024-03-06T10:06:24.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:24.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:24.674323+0000) 2024-03-06T10:06:25.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:25.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:25.674526+0000) 2024-03-06T10:06:26.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:26.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:26.674805+0000) 2024-03-06T10:06:27.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:27.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:27.675002+0000) 2024-03-06T10:06:28.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:28.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:28.675278+0000) 2024-03-06T10:06:29.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:29.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:29.675567+0000) 2024-03-06T10:06:30.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:30.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:30.675778+0000) 2024-03-06T10:06:31.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:31.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:31.676004+0000) 2024-03-06T10:06:32.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:32.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:32.676248+0000) 2024-03-06T10:06:33.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:33.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:33.676460+0000) 2024-03-06T10:06:34.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:34.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:34.676680+0000) 2024-03-06T10:06:35.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:35.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:35.676919+0000) 2024-03-06T10:06:36.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:36.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:36.677158+0000) 2024-03-06T10:06:37.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:37.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:37.677352+0000) 2024-03-06T10:06:38.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:38.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:38.677596+0000) 2024-03-06T10:06:39.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:39.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:39.677793+0000) 2024-03-06T10:06:40.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:40.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:40.678016+0000) 2024-03-06T10:06:41.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:41.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:41.678220+0000) 2024-03-06T10:06:42.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:42.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:42.678465+0000) 2024-03-06T10:06:43.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:43.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:43.678709+0000) 2024-03-06T10:06:44.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:44.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:44.678927+0000) 2024-03-06T10:06:45.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:45.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:45.679186+0000) 2024-03-06T10:06:46.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:46.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:46.679435+0000) 2024-03-06T10:06:47.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:47.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:47.679743+0000) 2024-03-06T10:06:48.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:48.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:48.679962+0000) 2024-03-06T10:06:49.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:49.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:49.680149+0000) 2024-03-06T10:06:50.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:50.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:50.680376+0000) 2024-03-06T10:06:51.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:51.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:51.680678+0000) 2024-03-06T10:06:52.407 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:06:52.415 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:06:52.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:52.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:52.680931+0000) 2024-03-06T10:06:53.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:53.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:53.681190+0000) 2024-03-06T10:06:54.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:54.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:54.681476+0000) 2024-03-06T10:06:55.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:55.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:55.681728+0000) 2024-03-06T10:06:56.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:56.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:56.681981+0000) 2024-03-06T10:06:57.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:57.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:57.682225+0000) 2024-03-06T10:06:58.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:58.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:58.682510+0000) 2024-03-06T10:06:59.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:06:59.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:06:59.682835+0000) 2024-03-06T10:07:00.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:00.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:00.683122+0000) 2024-03-06T10:07:01.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:01.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:01.683403+0000) 2024-03-06T10:07:02.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:02.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:02.683683+0000) 2024-03-06T10:07:03.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:03.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:03.683897+0000) 2024-03-06T10:07:04.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:04.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:04.684112+0000) 2024-03-06T10:07:05.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:05.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:05.684311+0000) 2024-03-06T10:07:06.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:06.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:06.684598+0000) 2024-03-06T10:07:07.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:07.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:07.684839+0000) 2024-03-06T10:07:08.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:08.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:08.685096+0000) 2024-03-06T10:07:09.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:09.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:09.685299+0000) 2024-03-06T10:07:10.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:10.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:10.685610+0000) 2024-03-06T10:07:11.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:11.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:11.685840+0000) 2024-03-06T10:07:12.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:12.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:12.686054+0000) 2024-03-06T10:07:13.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:13.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:13.686254+0000) 2024-03-06T10:07:14.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:14.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:14.686489+0000) 2024-03-06T10:07:15.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:15.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:15.686718+0000) 2024-03-06T10:07:16.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:16.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:16.686948+0000) 2024-03-06T10:07:17.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:17.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:17.687157+0000) 2024-03-06T10:07:18.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:18.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:18.687437+0000) 2024-03-06T10:07:19.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:19.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:19.687716+0000) 2024-03-06T10:07:20.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:20.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:20.687964+0000) 2024-03-06T10:07:21.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:21.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:21.688155+0000) 2024-03-06T10:07:22.613 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:07:22.622 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:07:22.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:22.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:22.688344+0000) 2024-03-06T10:07:23.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:23.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:23.688511+0000) 2024-03-06T10:07:24.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:24.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:24.688787+0000) 2024-03-06T10:07:25.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:25.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:25.688973+0000) 2024-03-06T10:07:26.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:26.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:26.689197+0000) 2024-03-06T10:07:27.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:27.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:27.689371+0000) 2024-03-06T10:07:28.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:28.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:28.689632+0000) 2024-03-06T10:07:29.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:29.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:29.689813+0000) 2024-03-06T10:07:30.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:30.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:30.690011+0000) 2024-03-06T10:07:31.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:31.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:31.690201+0000) 2024-03-06T10:07:32.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:32.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:32.690428+0000) 2024-03-06T10:07:33.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:33.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:33.690665+0000) 2024-03-06T10:07:34.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:34.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:34.690870+0000) 2024-03-06T10:07:35.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:35.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:35.691106+0000) 2024-03-06T10:07:36.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:36.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:36.691328+0000) 2024-03-06T10:07:37.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:37.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:37.691514+0000) 2024-03-06T10:07:38.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:38.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:38.691756+0000) 2024-03-06T10:07:39.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:39.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:39.691945+0000) 2024-03-06T10:07:40.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:40.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:40.692132+0000) 2024-03-06T10:07:41.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:41.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:41.692325+0000) 2024-03-06T10:07:42.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:42.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:42.692593+0000) 2024-03-06T10:07:43.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:43.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:43.692809+0000) 2024-03-06T10:07:44.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:44.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:44.692999+0000) 2024-03-06T10:07:45.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:45.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:45.693227+0000) 2024-03-06T10:07:46.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:46.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:46.693462+0000) 2024-03-06T10:07:47.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:47.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:47.693680+0000) 2024-03-06T10:07:48.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:48.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:48.693880+0000) 2024-03-06T10:07:49.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:49.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:49.694075+0000) 2024-03-06T10:07:50.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:50.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:50.694272+0000) 2024-03-06T10:07:51.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:51.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:51.694513+0000) 2024-03-06T10:07:52.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:52.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:52.694748+0000) 2024-03-06T10:07:52.826 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:07:52.834 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:07:53.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:53.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:53.694969+0000) 2024-03-06T10:07:54.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:54.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:54.695215+0000) 2024-03-06T10:07:55.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:55.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:55.695440+0000) 2024-03-06T10:07:56.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:56.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:56.695689+0000) 2024-03-06T10:07:57.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:57.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:57.695871+0000) 2024-03-06T10:07:58.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:58.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:58.696063+0000) 2024-03-06T10:07:59.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:07:59.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:07:59.696289+0000) 2024-03-06T10:08:00.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:00.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:00.696520+0000) 2024-03-06T10:08:01.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:01.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:01.696782+0000) 2024-03-06T10:08:02.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:02.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:02.696975+0000) 2024-03-06T10:08:03.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:03.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:03.697155+0000) 2024-03-06T10:08:04.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:04.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:04.697345+0000) 2024-03-06T10:08:05.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:05.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:05.697526+0000) 2024-03-06T10:08:06.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:06.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:06.697779+0000) 2024-03-06T10:08:07.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:07.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:07.697999+0000) 2024-03-06T10:08:08.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:08.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:08.698200+0000) 2024-03-06T10:08:09.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:09.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:09.698438+0000) 2024-03-06T10:08:10.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:10.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:10.698678+0000) 2024-03-06T10:08:11.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:11.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:11.698929+0000) 2024-03-06T10:08:12.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:12.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:12.699159+0000) 2024-03-06T10:08:13.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:13.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:13.699390+0000) 2024-03-06T10:08:14.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:14.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:14.699617+0000) 2024-03-06T10:08:15.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:15.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:15.699776+0000) 2024-03-06T10:08:16.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:16.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:16.700018+0000) 2024-03-06T10:08:17.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:17.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:17.700245+0000) 2024-03-06T10:08:18.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:18.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:18.700470+0000) 2024-03-06T10:08:19.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:19.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:19.700646+0000) 2024-03-06T10:08:20.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:20.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:20.700872+0000) 2024-03-06T10:08:21.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:21.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:21.701106+0000) 2024-03-06T10:08:22.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:22.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:22.701373+0000) 2024-03-06T10:08:23.015 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:08:23.023 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:08:23.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:23.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:23.701692+0000) 2024-03-06T10:08:24.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:24.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:24.701946+0000) 2024-03-06T10:08:25.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:25.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:25.702173+0000) 2024-03-06T10:08:26.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:26.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:26.702402+0000) 2024-03-06T10:08:27.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:27.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:27.702656+0000) 2024-03-06T10:08:28.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:28.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:28.702939+0000) 2024-03-06T10:08:29.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:29.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:29.703158+0000) 2024-03-06T10:08:30.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:30.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:30.703389+0000) 2024-03-06T10:08:31.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:31.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:31.703623+0000) 2024-03-06T10:08:32.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:32.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:32.703841+0000) 2024-03-06T10:08:33.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:33.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:33.704014+0000) 2024-03-06T10:08:34.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:34.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:34.704207+0000) 2024-03-06T10:08:35.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:35.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:35.704439+0000) 2024-03-06T10:08:36.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:36.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:36.704665+0000) 2024-03-06T10:08:37.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:37.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:37.704889+0000) 2024-03-06T10:08:38.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:38.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:38.705079+0000) 2024-03-06T10:08:39.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:39.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:39.705309+0000) 2024-03-06T10:08:40.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:40.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:40.705626+0000) 2024-03-06T10:08:41.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:41.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:41.705874+0000) 2024-03-06T10:08:42.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:42.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:42.706107+0000) 2024-03-06T10:08:43.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:43.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:43.706376+0000) 2024-03-06T10:08:44.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:44.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:44.706635+0000) 2024-03-06T10:08:45.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:45.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:45.706817+0000) 2024-03-06T10:08:46.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:46.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:46.707013+0000) 2024-03-06T10:08:47.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:47.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:47.707209+0000) 2024-03-06T10:08:48.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:48.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:48.707451+0000) 2024-03-06T10:08:49.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:49.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:49.707693+0000) 2024-03-06T10:08:50.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:50.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:50.707888+0000) 2024-03-06T10:08:51.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:51.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:51.708080+0000) 2024-03-06T10:08:52.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:52.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:52.708353+0000) 2024-03-06T10:08:53.230 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:08:53.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:08:53.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:53.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:53.708586+0000) 2024-03-06T10:08:54.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:54.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:54.708798+0000) 2024-03-06T10:08:55.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:55.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:55.708984+0000) 2024-03-06T10:08:56.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:56.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:56.709216+0000) 2024-03-06T10:08:57.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:57.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:57.709439+0000) 2024-03-06T10:08:58.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:58.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:58.709700+0000) 2024-03-06T10:08:59.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:08:59.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:08:59.709886+0000) 2024-03-06T10:09:00.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:00.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:00.710086+0000) 2024-03-06T10:09:01.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:01.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:01.710315+0000) 2024-03-06T10:09:02.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:02.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:02.710512+0000) 2024-03-06T10:09:03.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:03.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:03.710735+0000) 2024-03-06T10:09:04.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:04.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:04.710925+0000) 2024-03-06T10:09:05.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:05.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:05.711110+0000) 2024-03-06T10:09:06.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:06.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:06.711343+0000) 2024-03-06T10:09:07.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:07.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:07.711524+0000) 2024-03-06T10:09:08.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:08.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:08.711749+0000) 2024-03-06T10:09:09.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:09.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:09.711932+0000) 2024-03-06T10:09:10.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:10.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:10.712133+0000) 2024-03-06T10:09:11.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:11.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:11.712422+0000) 2024-03-06T10:09:12.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:12.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:12.712701+0000) 2024-03-06T10:09:13.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:13.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:13.712926+0000) 2024-03-06T10:09:14.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:14.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:14.713161+0000) 2024-03-06T10:09:15.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:15.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:15.713391+0000) 2024-03-06T10:09:16.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:16.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:16.713663+0000) 2024-03-06T10:09:17.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:17.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:17.713884+0000) 2024-03-06T10:09:18.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:18.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:18.714116+0000) 2024-03-06T10:09:19.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:19.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:19.714294+0000) 2024-03-06T10:09:20.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:20.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:20.714553+0000) 2024-03-06T10:09:21.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:21.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:21.714774+0000) 2024-03-06T10:09:22.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:22.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:22.714965+0000) 2024-03-06T10:09:23.432 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:09:23.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:09:23.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:23.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:23.715146+0000) 2024-03-06T10:09:24.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:24.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:24.715381+0000) 2024-03-06T10:09:25.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:25.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:25.715645+0000) 2024-03-06T10:09:26.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:26.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:26.715841+0000) 2024-03-06T10:09:27.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:27.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:27.716030+0000) 2024-03-06T10:09:28.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:28.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:28.716259+0000) 2024-03-06T10:09:29.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:29.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:29.716478+0000) 2024-03-06T10:09:30.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:30.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:30.716704+0000) 2024-03-06T10:09:31.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:31.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:31.716910+0000) 2024-03-06T10:09:32.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:32.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:32.717107+0000) 2024-03-06T10:09:33.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:33.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:33.717327+0000) 2024-03-06T10:09:34.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:34.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:34.717573+0000) 2024-03-06T10:09:35.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:35.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:35.717751+0000) 2024-03-06T10:09:36.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:36.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:36.717990+0000) 2024-03-06T10:09:37.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:37.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:37.718211+0000) 2024-03-06T10:09:38.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:38.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:38.718439+0000) 2024-03-06T10:09:39.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:39.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:39.718656+0000) 2024-03-06T10:09:40.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:40.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:40.718846+0000) 2024-03-06T10:09:41.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:41.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:41.719086+0000) 2024-03-06T10:09:42.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:42.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:42.719318+0000) 2024-03-06T10:09:43.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:43.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:43.719560+0000) 2024-03-06T10:09:44.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:44.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:44.719720+0000) 2024-03-06T10:09:45.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:45.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:45.719914+0000) 2024-03-06T10:09:46.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:46.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:46.720151+0000) 2024-03-06T10:09:47.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:47.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:47.720328+0000) 2024-03-06T10:09:48.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:48.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:48.720635+0000) 2024-03-06T10:09:49.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:49.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:49.720801+0000) 2024-03-06T10:09:50.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:50.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:50.721029+0000) 2024-03-06T10:09:51.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:51.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:51.721267+0000) 2024-03-06T10:09:52.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:52.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:52.721466+0000) 2024-03-06T10:09:53.660 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:09:53.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:09:53.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:53.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:53.721717+0000) 2024-03-06T10:09:54.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:54.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:54.721912+0000) 2024-03-06T10:09:55.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:55.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:55.722136+0000) 2024-03-06T10:09:56.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:56.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:56.722389+0000) 2024-03-06T10:09:57.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:57.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:57.722670+0000) 2024-03-06T10:09:58.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:58.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:58.722902+0000) 2024-03-06T10:09:59.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:09:59.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:09:59.723145+0000) 2024-03-06T10:10:00.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:00.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:00.723426+0000) 2024-03-06T10:10:01.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:01.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:01.723716+0000) 2024-03-06T10:10:02.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:02.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:02.723963+0000) 2024-03-06T10:10:03.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:03.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:03.724199+0000) 2024-03-06T10:10:04.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:04.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:04.724565+0000) 2024-03-06T10:10:05.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:05.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:05.724776+0000) 2024-03-06T10:10:06.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:06.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:06.725024+0000) 2024-03-06T10:10:07.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:07.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:07.725247+0000) 2024-03-06T10:10:08.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:08.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:08.725471+0000) 2024-03-06T10:10:09.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:09.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:09.725659+0000) 2024-03-06T10:10:10.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:10.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:10.725904+0000) 2024-03-06T10:10:11.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:11.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:11.726107+0000) 2024-03-06T10:10:12.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:12.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:12.726339+0000) 2024-03-06T10:10:13.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:13.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:13.726643+0000) 2024-03-06T10:10:14.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:14.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:14.726807+0000) 2024-03-06T10:10:15.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:15.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:15.727002+0000) 2024-03-06T10:10:16.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:16.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:16.727239+0000) 2024-03-06T10:10:17.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:17.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:17.727470+0000) 2024-03-06T10:10:18.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:18.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:18.727694+0000) 2024-03-06T10:10:19.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:19.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:19.727880+0000) 2024-03-06T10:10:20.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:20.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:20.728078+0000) 2024-03-06T10:10:21.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:21.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:21.728296+0000) 2024-03-06T10:10:22.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:22.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:22.728565+0000) 2024-03-06T10:10:23.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:23.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:23.728754+0000) 2024-03-06T10:10:23.868 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:10:23.877 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:10:24.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:24.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:24.728947+0000) 2024-03-06T10:10:25.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:25.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:25.729106+0000) 2024-03-06T10:10:26.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:26.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:26.729309+0000) 2024-03-06T10:10:27.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:27.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:27.729502+0000) 2024-03-06T10:10:28.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:28.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:28.729698+0000) 2024-03-06T10:10:29.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:29.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:29.729923+0000) 2024-03-06T10:10:30.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:30.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:30.730199+0000) 2024-03-06T10:10:31.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:31.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:31.730490+0000) 2024-03-06T10:10:32.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:32.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:32.730710+0000) 2024-03-06T10:10:33.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:33.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:33.730897+0000) 2024-03-06T10:10:34.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:34.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:34.731125+0000) 2024-03-06T10:10:35.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:35.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:35.731347+0000) 2024-03-06T10:10:36.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:36.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:36.731613+0000) 2024-03-06T10:10:37.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:37.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:37.731799+0000) 2024-03-06T10:10:38.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:38.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:38.731988+0000) 2024-03-06T10:10:39.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:39.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:39.732204+0000) 2024-03-06T10:10:40.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:40.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:40.732403+0000) 2024-03-06T10:10:41.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:41.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:41.732706+0000) 2024-03-06T10:10:42.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:42.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:42.732947+0000) 2024-03-06T10:10:43.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:43.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:43.733219+0000) 2024-03-06T10:10:44.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:44.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:44.733425+0000) 2024-03-06T10:10:45.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:45.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:45.733660+0000) 2024-03-06T10:10:46.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:46.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:46.733908+0000) 2024-03-06T10:10:47.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:47.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:47.734098+0000) 2024-03-06T10:10:48.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:48.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:48.734347+0000) 2024-03-06T10:10:49.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:49.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:49.734647+0000) 2024-03-06T10:10:50.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:50.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:50.734896+0000) 2024-03-06T10:10:51.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:51.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:51.735179+0000) 2024-03-06T10:10:52.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:52.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:52.735402+0000) 2024-03-06T10:10:53.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:53.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:53.735671+0000) 2024-03-06T10:10:54.012 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:10:54.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:10:54.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:54.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:54.735928+0000) 2024-03-06T10:10:55.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:55.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:55.736180+0000) 2024-03-06T10:10:56.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:56.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:56.736468+0000) 2024-03-06T10:10:57.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:57.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:57.736680+0000) 2024-03-06T10:10:58.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:58.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:58.736926+0000) 2024-03-06T10:10:59.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:10:59.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:10:59.737114+0000) 2024-03-06T10:11:00.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:00.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:00.737437+0000) 2024-03-06T10:11:01.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:01.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:01.737668+0000) 2024-03-06T10:11:02.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:02.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:02.737905+0000) 2024-03-06T10:11:03.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:03.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:03.738133+0000) 2024-03-06T10:11:04.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:04.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:04.738379+0000) 2024-03-06T10:11:05.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:05.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:05.738610+0000) 2024-03-06T10:11:06.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:06.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:06.738811+0000) 2024-03-06T10:11:07.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:07.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:07.739036+0000) 2024-03-06T10:11:08.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:08.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:08.739302+0000) 2024-03-06T10:11:09.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:09.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:09.739591+0000) 2024-03-06T10:11:10.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:10.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:10.739870+0000) 2024-03-06T10:11:11.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:11.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:11.740137+0000) 2024-03-06T10:11:12.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:12.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:12.740362+0000) 2024-03-06T10:11:13.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:13.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:13.740640+0000) 2024-03-06T10:11:14.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:14.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:14.740873+0000) 2024-03-06T10:11:15.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:15.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:15.741127+0000) 2024-03-06T10:11:16.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:16.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:16.741416+0000) 2024-03-06T10:11:17.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:17.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:17.741699+0000) 2024-03-06T10:11:18.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:18.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:18.741917+0000) 2024-03-06T10:11:19.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:19.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:19.742142+0000) 2024-03-06T10:11:20.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:20.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:20.742423+0000) 2024-03-06T10:11:21.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:21.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:21.742728+0000) 2024-03-06T10:11:22.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:22.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:22.742974+0000) 2024-03-06T10:11:23.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:23.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:23.743182+0000) 2024-03-06T10:11:24.244 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:11:24.253 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:11:24.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:24.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:24.743418+0000) 2024-03-06T10:11:25.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:25.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:25.743683+0000) 2024-03-06T10:11:26.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:26.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:26.743945+0000) 2024-03-06T10:11:27.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:27.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:27.744155+0000) 2024-03-06T10:11:28.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:28.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:28.744344+0000) 2024-03-06T10:11:29.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:29.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:29.744576+0000) 2024-03-06T10:11:30.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:30.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:30.744830+0000) 2024-03-06T10:11:31.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:31.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:31.745092+0000) 2024-03-06T10:11:32.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:32.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:32.745371+0000) 2024-03-06T10:11:33.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:33.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:33.745672+0000) 2024-03-06T10:11:34.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:34.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:34.745890+0000) 2024-03-06T10:11:35.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:35.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:35.746137+0000) 2024-03-06T10:11:36.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:36.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:36.746383+0000) 2024-03-06T10:11:37.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:37.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:37.746653+0000) 2024-03-06T10:11:38.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:38.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:38.746898+0000) 2024-03-06T10:11:39.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:39.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:39.747177+0000) 2024-03-06T10:11:40.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:40.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:40.747388+0000) 2024-03-06T10:11:41.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:41.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:41.747635+0000) 2024-03-06T10:11:42.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:42.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:42.747862+0000) 2024-03-06T10:11:43.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:43.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:43.748087+0000) 2024-03-06T10:11:44.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:44.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:44.748315+0000) 2024-03-06T10:11:45.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:45.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:45.748525+0000) 2024-03-06T10:11:46.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:46.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:46.748854+0000) 2024-03-06T10:11:47.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:47.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:47.749086+0000) 2024-03-06T10:11:48.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:48.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:48.749314+0000) 2024-03-06T10:11:49.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:49.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:49.749529+0000) 2024-03-06T10:11:50.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:50.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:50.749793+0000) 2024-03-06T10:11:51.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:51.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:51.750022+0000) 2024-03-06T10:11:52.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:52.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:52.750258+0000) 2024-03-06T10:11:53.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:53.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:53.750500+0000) 2024-03-06T10:11:54.456 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:11:54.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:11:54.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:54.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:54.750805+0000) 2024-03-06T10:11:55.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:55.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:55.750983+0000) 2024-03-06T10:11:56.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:56.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:56.751220+0000) 2024-03-06T10:11:57.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:57.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:57.751447+0000) 2024-03-06T10:11:58.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:58.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:58.751683+0000) 2024-03-06T10:11:59.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:11:59.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:11:59.751904+0000) 2024-03-06T10:12:00.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:00.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:00.752126+0000) 2024-03-06T10:12:01.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:01.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:01.752313+0000) 2024-03-06T10:12:02.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:02.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:02.752567+0000) 2024-03-06T10:12:03.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:03.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:03.752760+0000) 2024-03-06T10:12:04.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:04.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:04.752960+0000) 2024-03-06T10:12:05.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:05.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:05.753197+0000) 2024-03-06T10:12:06.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:06.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:06.753425+0000) 2024-03-06T10:12:07.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:07.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:07.753677+0000) 2024-03-06T10:12:08.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:08.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:08.753899+0000) 2024-03-06T10:12:09.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:09.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:09.754090+0000) 2024-03-06T10:12:10.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:10.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:10.754316+0000) 2024-03-06T10:12:11.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:11.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:11.754500+0000) 2024-03-06T10:12:12.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:12.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:12.754735+0000) 2024-03-06T10:12:13.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:13.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:13.754908+0000) 2024-03-06T10:12:14.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:14.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:14.755108+0000) 2024-03-06T10:12:15.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:15.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:15.755341+0000) 2024-03-06T10:12:16.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:16.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:16.755608+0000) 2024-03-06T10:12:17.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:17.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:17.755843+0000) 2024-03-06T10:12:18.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:18.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:18.756071+0000) 2024-03-06T10:12:19.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:19.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:19.756301+0000) 2024-03-06T10:12:20.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:20.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:20.756527+0000) 2024-03-06T10:12:21.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:21.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:21.756794+0000) 2024-03-06T10:12:22.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:22.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:22.756979+0000) 2024-03-06T10:12:23.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:23.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:23.757170+0000) 2024-03-06T10:12:24.672 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:12:24.680 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:12:24.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:24.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:24.757349+0000) 2024-03-06T10:12:25.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:25.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:25.757582+0000) 2024-03-06T10:12:26.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:26.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:26.757778+0000) 2024-03-06T10:12:27.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:27.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:27.757969+0000) 2024-03-06T10:12:28.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:28.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:28.758161+0000) 2024-03-06T10:12:29.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:29.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:29.758347+0000) 2024-03-06T10:12:30.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:30.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:30.758612+0000) 2024-03-06T10:12:31.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:31.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:31.758858+0000) 2024-03-06T10:12:32.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:32.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:32.759051+0000) 2024-03-06T10:12:33.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:33.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:33.759272+0000) 2024-03-06T10:12:34.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:34.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:34.759515+0000) 2024-03-06T10:12:35.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:35.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:35.759736+0000) 2024-03-06T10:12:36.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:36.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:36.759980+0000) 2024-03-06T10:12:37.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:37.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:37.760225+0000) 2024-03-06T10:12:38.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:38.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:38.760467+0000) 2024-03-06T10:12:39.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:39.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:39.760754+0000) 2024-03-06T10:12:40.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:40.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:40.761004+0000) 2024-03-06T10:12:41.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:41.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:41.761261+0000) 2024-03-06T10:12:42.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:42.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:42.761490+0000) 2024-03-06T10:12:43.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:43.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:43.761823+0000) 2024-03-06T10:12:44.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:44.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:44.762083+0000) 2024-03-06T10:12:45.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:45.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:45.762306+0000) 2024-03-06T10:12:46.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:46.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:46.762568+0000) 2024-03-06T10:12:47.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:47.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:47.762803+0000) 2024-03-06T10:12:48.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:48.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:48.763028+0000) 2024-03-06T10:12:49.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:49.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:49.763250+0000) 2024-03-06T10:12:50.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:50.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:50.763467+0000) 2024-03-06T10:12:51.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:51.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:51.763697+0000) 2024-03-06T10:12:52.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:52.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:52.763932+0000) 2024-03-06T10:12:53.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:53.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:53.764156+0000) 2024-03-06T10:12:54.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:54.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:54.764380+0000) 2024-03-06T10:12:54.883 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:12:54.891 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:12:55.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:55.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:55.764660+0000) 2024-03-06T10:12:56.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:56.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:56.764888+0000) 2024-03-06T10:12:57.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:57.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:57.765062+0000) 2024-03-06T10:12:58.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:58.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:58.765287+0000) 2024-03-06T10:12:59.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:12:59.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:12:59.765574+0000) 2024-03-06T10:13:00.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:00.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:00.765796+0000) 2024-03-06T10:13:01.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:01.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:01.766025+0000) 2024-03-06T10:13:02.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:02.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:02.766302+0000) 2024-03-06T10:13:03.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:03.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:03.766606+0000) 2024-03-06T10:13:04.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:04.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:04.766865+0000) 2024-03-06T10:13:05.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:05.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:05.767095+0000) 2024-03-06T10:13:06.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:06.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:06.767365+0000) 2024-03-06T10:13:07.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:07.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:07.767634+0000) 2024-03-06T10:13:08.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:08.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:08.767841+0000) 2024-03-06T10:13:09.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:09.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:09.768057+0000) 2024-03-06T10:13:10.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:10.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:10.768281+0000) 2024-03-06T10:13:11.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:11.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:11.768474+0000) 2024-03-06T10:13:12.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:12.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:12.768711+0000) 2024-03-06T10:13:13.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:13.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:13.768939+0000) 2024-03-06T10:13:14.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:14.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:14.769151+0000) 2024-03-06T10:13:15.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:15.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:15.769395+0000) 2024-03-06T10:13:16.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:16.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:16.769663+0000) 2024-03-06T10:13:17.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:17.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:17.769898+0000) 2024-03-06T10:13:18.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:18.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:18.770132+0000) 2024-03-06T10:13:19.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:19.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:19.770355+0000) 2024-03-06T10:13:20.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:20.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:20.770571+0000) 2024-03-06T10:13:21.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:21.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:21.770787+0000) 2024-03-06T10:13:22.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:22.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:22.771069+0000) 2024-03-06T10:13:23.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:23.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:23.771243+0000) 2024-03-06T10:13:24.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:24.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:24.771491+0000) 2024-03-06T10:13:25.033 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:13:25.042 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:13:25.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:25.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:25.771730+0000) 2024-03-06T10:13:26.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:26.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:26.771923+0000) 2024-03-06T10:13:27.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:27.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:27.772128+0000) 2024-03-06T10:13:28.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:28.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:28.772364+0000) 2024-03-06T10:13:29.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:29.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:29.772554+0000) 2024-03-06T10:13:30.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:30.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:30.772754+0000) 2024-03-06T10:13:31.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:31.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:31.772947+0000) 2024-03-06T10:13:32.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:32.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:32.773143+0000) 2024-03-06T10:13:33.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:33.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:33.773359+0000) 2024-03-06T10:13:34.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:34.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:34.773625+0000) 2024-03-06T10:13:35.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:35.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:35.773817+0000) 2024-03-06T10:13:36.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:36.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:36.774007+0000) 2024-03-06T10:13:37.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:37.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:37.774236+0000) 2024-03-06T10:13:38.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:38.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:38.774469+0000) 2024-03-06T10:13:39.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:39.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:39.774723+0000) 2024-03-06T10:13:40.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:40.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:40.774923+0000) 2024-03-06T10:13:41.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:41.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:41.775135+0000) 2024-03-06T10:13:42.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:42.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:42.775365+0000) 2024-03-06T10:13:43.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:43.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:43.775630+0000) 2024-03-06T10:13:44.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:44.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:44.775870+0000) 2024-03-06T10:13:45.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:45.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:45.776091+0000) 2024-03-06T10:13:46.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:46.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:46.776311+0000) 2024-03-06T10:13:47.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:47.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:47.776574+0000) 2024-03-06T10:13:48.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:48.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:48.776782+0000) 2024-03-06T10:13:49.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:49.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:49.776984+0000) 2024-03-06T10:13:50.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:50.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:50.777205+0000) 2024-03-06T10:13:51.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:51.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:51.777427+0000) 2024-03-06T10:13:52.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:52.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:52.777668+0000) 2024-03-06T10:13:53.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:53.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:53.777898+0000) 2024-03-06T10:13:54.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:54.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:54.778084+0000) 2024-03-06T10:13:55.235 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:13:55.243 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:13:55.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:55.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:55.778357+0000) 2024-03-06T10:13:56.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:56.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:56.778688+0000) 2024-03-06T10:13:57.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:57.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:57.778934+0000) 2024-03-06T10:13:58.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:58.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:58.779184+0000) 2024-03-06T10:13:59.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:13:59.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:13:59.779441+0000) 2024-03-06T10:14:00.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:00.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:00.779738+0000) 2024-03-06T10:14:01.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:01.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:01.780001+0000) 2024-03-06T10:14:02.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:02.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:02.780268+0000) 2024-03-06T10:14:03.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:03.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:03.780511+0000) 2024-03-06T10:14:04.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:04.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:04.780754+0000) 2024-03-06T10:14:05.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:05.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:05.780940+0000) 2024-03-06T10:14:06.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:06.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:06.781202+0000) 2024-03-06T10:14:07.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:07.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:07.781481+0000) 2024-03-06T10:14:08.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:08.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:08.781765+0000) 2024-03-06T10:14:09.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:09.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:09.781992+0000) 2024-03-06T10:14:10.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:10.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:10.782264+0000) 2024-03-06T10:14:11.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:11.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:11.782463+0000) 2024-03-06T10:14:12.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:12.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:12.782699+0000) 2024-03-06T10:14:13.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:13.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:13.782898+0000) 2024-03-06T10:14:14.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:14.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:14.783135+0000) 2024-03-06T10:14:15.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:15.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:15.783409+0000) 2024-03-06T10:14:16.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:16.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:16.783688+0000) 2024-03-06T10:14:17.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:17.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:17.783900+0000) 2024-03-06T10:14:18.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:18.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:18.784143+0000) 2024-03-06T10:14:19.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:19.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:19.784381+0000) 2024-03-06T10:14:20.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:20.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:20.784656+0000) 2024-03-06T10:14:21.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:21.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:21.784855+0000) 2024-03-06T10:14:22.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:22.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:22.785083+0000) 2024-03-06T10:14:23.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:23.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:23.785259+0000) 2024-03-06T10:14:24.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:24.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:24.785459+0000) 2024-03-06T10:14:25.463 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:14:25.471 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:14:25.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:25.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:25.785695+0000) 2024-03-06T10:14:26.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:26.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:26.785857+0000) 2024-03-06T10:14:27.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:27.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:27.786047+0000) 2024-03-06T10:14:28.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:28.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:28.786233+0000) 2024-03-06T10:14:29.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:29.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:29.786423+0000) 2024-03-06T10:14:30.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:30.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:30.786653+0000) 2024-03-06T10:14:31.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:31.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:31.786843+0000) 2024-03-06T10:14:32.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:32.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:32.787036+0000) 2024-03-06T10:14:33.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:33.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:33.787254+0000) 2024-03-06T10:14:34.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:34.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:34.787495+0000) 2024-03-06T10:14:35.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:35.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:35.787734+0000) 2024-03-06T10:14:36.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:36.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:36.787924+0000) 2024-03-06T10:14:37.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:37.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:37.788170+0000) 2024-03-06T10:14:38.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:38.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:38.788397+0000) 2024-03-06T10:14:39.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:39.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:39.788646+0000) 2024-03-06T10:14:40.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:40.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:40.788879+0000) 2024-03-06T10:14:41.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:41.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:41.789094+0000) 2024-03-06T10:14:42.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:42.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:42.789304+0000) 2024-03-06T10:14:43.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:43.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:43.789582+0000) 2024-03-06T10:14:44.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:44.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:44.789818+0000) 2024-03-06T10:14:45.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:45.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:45.790051+0000) 2024-03-06T10:14:46.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:46.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:46.790293+0000) 2024-03-06T10:14:47.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:47.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:47.790511+0000) 2024-03-06T10:14:48.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:48.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:48.790765+0000) 2024-03-06T10:14:49.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:49.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:49.790975+0000) 2024-03-06T10:14:50.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:50.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:50.791220+0000) 2024-03-06T10:14:51.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:51.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:51.791459+0000) 2024-03-06T10:14:52.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:52.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:52.791738+0000) 2024-03-06T10:14:53.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:53.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:53.791964+0000) 2024-03-06T10:14:54.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:54.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:54.792256+0000) 2024-03-06T10:14:55.694 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:14:55.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:14:55.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:55.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:55.792431+0000) 2024-03-06T10:14:56.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:56.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:56.792691+0000) 2024-03-06T10:14:57.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:57.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:57.792974+0000) 2024-03-06T10:14:58.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:58.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:58.793216+0000) 2024-03-06T10:14:59.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:14:59.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:14:59.793478+0000) 2024-03-06T10:15:00.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:00.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:00.793742+0000) 2024-03-06T10:15:01.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:01.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:01.793964+0000) 2024-03-06T10:15:02.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:02.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:02.794214+0000) 2024-03-06T10:15:03.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:03.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:03.794431+0000) 2024-03-06T10:15:04.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:04.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:04.794711+0000) 2024-03-06T10:15:05.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:05.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:05.794912+0000) 2024-03-06T10:15:06.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:06.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:06.795150+0000) 2024-03-06T10:15:07.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:07.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:07.795415+0000) 2024-03-06T10:15:08.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:08.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:08.795653+0000) 2024-03-06T10:15:09.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:09.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:09.795860+0000) 2024-03-06T10:15:10.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:10.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:10.796067+0000) 2024-03-06T10:15:11.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:11.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:11.796264+0000) 2024-03-06T10:15:12.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:12.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:12.796506+0000) 2024-03-06T10:15:13.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:13.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:13.796772+0000) 2024-03-06T10:15:14.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:14.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:14.797012+0000) 2024-03-06T10:15:15.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:15.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:15.797237+0000) 2024-03-06T10:15:16.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:16.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:16.797476+0000) 2024-03-06T10:15:17.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:17.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:17.797752+0000) 2024-03-06T10:15:18.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:18.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:18.797997+0000) 2024-03-06T10:15:19.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:19.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:19.798275+0000) 2024-03-06T10:15:20.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:20.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:20.798481+0000) 2024-03-06T10:15:21.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:21.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:21.798702+0000) 2024-03-06T10:15:22.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:22.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:22.798951+0000) 2024-03-06T10:15:23.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:23.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:23.799155+0000) 2024-03-06T10:15:24.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:24.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:24.799391+0000) 2024-03-06T10:15:25.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:25.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:25.799648+0000) 2024-03-06T10:15:25.890 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:15:25.898 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:15:26.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:26.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:26.799848+0000) 2024-03-06T10:15:27.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:27.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:27.800084+0000) 2024-03-06T10:15:28.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:28.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:28.800308+0000) 2024-03-06T10:15:29.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:29.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:29.800566+0000) 2024-03-06T10:15:30.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:30.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:30.800813+0000) 2024-03-06T10:15:31.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:31.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:31.801035+0000) 2024-03-06T10:15:32.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:32.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:32.801269+0000) 2024-03-06T10:15:33.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:33.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:33.801506+0000) 2024-03-06T10:15:34.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:34.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:34.801750+0000) 2024-03-06T10:15:35.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:35.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:35.801943+0000) 2024-03-06T10:15:36.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:36.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:36.802142+0000) 2024-03-06T10:15:37.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:37.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:37.802391+0000) 2024-03-06T10:15:38.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:38.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:38.802631+0000) 2024-03-06T10:15:39.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:39.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:39.802857+0000) 2024-03-06T10:15:40.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:40.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:40.803050+0000) 2024-03-06T10:15:41.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:41.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:41.803284+0000) 2024-03-06T10:15:42.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:42.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:42.803571+0000) 2024-03-06T10:15:43.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:43.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:43.803805+0000) 2024-03-06T10:15:44.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:44.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:44.803998+0000) 2024-03-06T10:15:45.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:45.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:45.804217+0000) 2024-03-06T10:15:46.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:46.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:46.804479+0000) 2024-03-06T10:15:47.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:47.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:47.804724+0000) 2024-03-06T10:15:48.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:48.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:48.804968+0000) 2024-03-06T10:15:49.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:49.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:49.805208+0000) 2024-03-06T10:15:50.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:50.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:50.805451+0000) 2024-03-06T10:15:51.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:51.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:51.805702+0000) 2024-03-06T10:15:52.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:52.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:52.805960+0000) 2024-03-06T10:15:53.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:53.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:53.806172+0000) 2024-03-06T10:15:54.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:54.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:54.806419+0000) 2024-03-06T10:15:55.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:55.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:55.806653+0000) 2024-03-06T10:15:56.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:15:56.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:15:56.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:56.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:56.806939+0000) 2024-03-06T10:15:57.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:57.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:57.807201+0000) 2024-03-06T10:15:58.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:58.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:58.807444+0000) 2024-03-06T10:15:59.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:15:59.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:15:59.807686+0000) 2024-03-06T10:16:00.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:00.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:00.807890+0000) 2024-03-06T10:16:01.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:01.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:01.808118+0000) 2024-03-06T10:16:02.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:02.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:02.808362+0000) 2024-03-06T10:16:03.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:03.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:03.808595+0000) 2024-03-06T10:16:04.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:04.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:04.808847+0000) 2024-03-06T10:16:05.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:05.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:05.809050+0000) 2024-03-06T10:16:06.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:06.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:06.809233+0000) 2024-03-06T10:16:07.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:07.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:07.809484+0000) 2024-03-06T10:16:08.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:08.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:08.809734+0000) 2024-03-06T10:16:09.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:09.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:09.809953+0000) 2024-03-06T10:16:10.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:10.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:10.810175+0000) 2024-03-06T10:16:11.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:11.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:11.810436+0000) 2024-03-06T10:16:12.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:12.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:12.810704+0000) 2024-03-06T10:16:13.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:13.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:13.810899+0000) 2024-03-06T10:16:14.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:14.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:14.811148+0000) 2024-03-06T10:16:15.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:15.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:15.811383+0000) 2024-03-06T10:16:16.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:16.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:16.811659+0000) 2024-03-06T10:16:17.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:17.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:17.811935+0000) 2024-03-06T10:16:18.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:18.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:18.812187+0000) 2024-03-06T10:16:19.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:19.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:19.812458+0000) 2024-03-06T10:16:20.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:20.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:20.812730+0000) 2024-03-06T10:16:21.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:21.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:21.812949+0000) 2024-03-06T10:16:22.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:22.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:22.813161+0000) 2024-03-06T10:16:23.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:23.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:23.813422+0000) 2024-03-06T10:16:24.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:24.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:24.813613+0000) 2024-03-06T10:16:25.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:25.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:25.813839+0000) 2024-03-06T10:16:26.295 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:16:26.304 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:16:26.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:26.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:26.814085+0000) 2024-03-06T10:16:27.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:27.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:27.814368+0000) 2024-03-06T10:16:28.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:28.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:28.814642+0000) 2024-03-06T10:16:29.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:29.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:29.814845+0000) 2024-03-06T10:16:30.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:30.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:30.815038+0000) 2024-03-06T10:16:31.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:31.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:31.815274+0000) 2024-03-06T10:16:32.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:32.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:32.815594+0000) 2024-03-06T10:16:33.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:33.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:33.815819+0000) 2024-03-06T10:16:34.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:34.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:34.816070+0000) 2024-03-06T10:16:35.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:35.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:35.816348+0000) 2024-03-06T10:16:36.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:36.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:36.816646+0000) 2024-03-06T10:16:37.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:37.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:37.816934+0000) 2024-03-06T10:16:38.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:38.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:38.817220+0000) 2024-03-06T10:16:39.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:39.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:39.817494+0000) 2024-03-06T10:16:40.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:40.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:40.817793+0000) 2024-03-06T10:16:41.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:41.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:41.818046+0000) 2024-03-06T10:16:42.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:42.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:42.818325+0000) 2024-03-06T10:16:43.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:43.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:43.818639+0000) 2024-03-06T10:16:44.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:44.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:44.818909+0000) 2024-03-06T10:16:45.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:45.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:45.819122+0000) 2024-03-06T10:16:46.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:46.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:46.819401+0000) 2024-03-06T10:16:47.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:47.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:47.819655+0000) 2024-03-06T10:16:48.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:48.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:48.819883+0000) 2024-03-06T10:16:49.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:49.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:49.820115+0000) 2024-03-06T10:16:50.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:50.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:50.820441+0000) 2024-03-06T10:16:51.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:51.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:51.820699+0000) 2024-03-06T10:16:52.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:52.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:52.820918+0000) 2024-03-06T10:16:53.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:53.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:53.821123+0000) 2024-03-06T10:16:54.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:54.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:54.821394+0000) 2024-03-06T10:16:55.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:55.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:55.821620+0000) 2024-03-06T10:16:56.518 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:16:56.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:16:56.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:56.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:56.821913+0000) 2024-03-06T10:16:57.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:57.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:57.822205+0000) 2024-03-06T10:16:58.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:58.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:58.822447+0000) 2024-03-06T10:16:59.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:16:59.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:16:59.822665+0000) 2024-03-06T10:17:00.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:00.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:00.822904+0000) 2024-03-06T10:17:01.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:01.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:01.823127+0000) 2024-03-06T10:17:02.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:02.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:02.823336+0000) 2024-03-06T10:17:03.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:03.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:03.823606+0000) 2024-03-06T10:17:04.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:04.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:04.823855+0000) 2024-03-06T10:17:05.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:05.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:05.824097+0000) 2024-03-06T10:17:06.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:06.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:06.824364+0000) 2024-03-06T10:17:07.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:07.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:07.824684+0000) 2024-03-06T10:17:08.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:08.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:08.824931+0000) 2024-03-06T10:17:09.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:09.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:09.825156+0000) 2024-03-06T10:17:10.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:10.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:10.825363+0000) 2024-03-06T10:17:11.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:11.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:11.825671+0000) 2024-03-06T10:17:12.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:12.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:12.825919+0000) 2024-03-06T10:17:13.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:13.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:13.826151+0000) 2024-03-06T10:17:14.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:14.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:14.826439+0000) 2024-03-06T10:17:15.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:15.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:15.826659+0000) 2024-03-06T10:17:16.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:16.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:16.826868+0000) 2024-03-06T10:17:17.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:17.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:17.827077+0000) 2024-03-06T10:17:18.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:18.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:18.827309+0000) 2024-03-06T10:17:19.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:19.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:19.827611+0000) 2024-03-06T10:17:20.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:20.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:20.827855+0000) 2024-03-06T10:17:21.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:21.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:21.828076+0000) 2024-03-06T10:17:22.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:22.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:22.828368+0000) 2024-03-06T10:17:23.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:23.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:23.828687+0000) 2024-03-06T10:17:24.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:24.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:24.828932+0000) 2024-03-06T10:17:25.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:25.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:25.829166+0000) 2024-03-06T10:17:26.744 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:17:26.753 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:17:26.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:26.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:26.829361+0000) 2024-03-06T10:17:27.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:27.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:27.829633+0000) 2024-03-06T10:17:28.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:28.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:28.829879+0000) 2024-03-06T10:17:29.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:29.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:29.830108+0000) 2024-03-06T10:17:30.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:30.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:30.830387+0000) 2024-03-06T10:17:31.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:31.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:31.830675+0000) 2024-03-06T10:17:32.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:32.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:32.830972+0000) 2024-03-06T10:17:33.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:33.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:33.831213+0000) 2024-03-06T10:17:34.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:34.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:34.831486+0000) 2024-03-06T10:17:35.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:35.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:35.831722+0000) 2024-03-06T10:17:36.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:36.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:36.831968+0000) 2024-03-06T10:17:37.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:37.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:37.832254+0000) 2024-03-06T10:17:38.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:38.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:38.832578+0000) 2024-03-06T10:17:39.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:39.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:39.832818+0000) 2024-03-06T10:17:40.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:40.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:40.833068+0000) 2024-03-06T10:17:41.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:41.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:41.833278+0000) 2024-03-06T10:17:42.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:42.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:42.833521+0000) 2024-03-06T10:17:43.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:43.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:43.833773+0000) 2024-03-06T10:17:44.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:44.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:44.833978+0000) 2024-03-06T10:17:45.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:45.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:45.834187+0000) 2024-03-06T10:17:46.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:46.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:46.834470+0000) 2024-03-06T10:17:47.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:47.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:47.834754+0000) 2024-03-06T10:17:48.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:48.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:48.835006+0000) 2024-03-06T10:17:49.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:49.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:49.835215+0000) 2024-03-06T10:17:50.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:50.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:50.835429+0000) 2024-03-06T10:17:51.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:51.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:51.835689+0000) 2024-03-06T10:17:52.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:52.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:52.835937+0000) 2024-03-06T10:17:53.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:53.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:53.836180+0000) 2024-03-06T10:17:54.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:54.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:54.836468+0000) 2024-03-06T10:17:55.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:55.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:55.836755+0000) 2024-03-06T10:17:56.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:56.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:56.836966+0000) 2024-03-06T10:17:56.965 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:17:56.973 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:17:57.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:57.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:57.837199+0000) 2024-03-06T10:17:58.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:58.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:58.837416+0000) 2024-03-06T10:17:59.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:17:59.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:17:59.837665+0000) 2024-03-06T10:18:00.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:00.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:00.837909+0000) 2024-03-06T10:18:01.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:01.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:01.838139+0000) 2024-03-06T10:18:02.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:02.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:02.838420+0000) 2024-03-06T10:18:03.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:03.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:03.838713+0000) 2024-03-06T10:18:04.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:04.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:04.838971+0000) 2024-03-06T10:18:05.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:05.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:05.839212+0000) 2024-03-06T10:18:06.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:06.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:06.839488+0000) 2024-03-06T10:18:07.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:07.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:07.839719+0000) 2024-03-06T10:18:08.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:08.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:08.839965+0000) 2024-03-06T10:18:09.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:09.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:09.840189+0000) 2024-03-06T10:18:10.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:10.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:10.840456+0000) 2024-03-06T10:18:11.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:11.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:11.840732+0000) 2024-03-06T10:18:12.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:12.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:12.840982+0000) 2024-03-06T10:18:13.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:13.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:13.841259+0000) 2024-03-06T10:18:14.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:14.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:14.841563+0000) 2024-03-06T10:18:15.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:15.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:15.841849+0000) 2024-03-06T10:18:16.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:16.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:16.842095+0000) 2024-03-06T10:18:17.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:17.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:17.842362+0000) 2024-03-06T10:18:18.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:18.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:18.842664+0000) 2024-03-06T10:18:19.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:19.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:19.842893+0000) 2024-03-06T10:18:20.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:20.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:20.843151+0000) 2024-03-06T10:18:21.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:21.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:21.843420+0000) 2024-03-06T10:18:22.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:22.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:22.843693+0000) 2024-03-06T10:18:23.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:23.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:23.843968+0000) 2024-03-06T10:18:24.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:24.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:24.844242+0000) 2024-03-06T10:18:25.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:25.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:25.844517+0000) 2024-03-06T10:18:26.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:26.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:26.844852+0000) 2024-03-06T10:18:27.095 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:18:27.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:18:27.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:27.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:27.845054+0000) 2024-03-06T10:18:28.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:28.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:28.845297+0000) 2024-03-06T10:18:29.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:29.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:29.845520+0000) 2024-03-06T10:18:30.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:30.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:30.845796+0000) 2024-03-06T10:18:31.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:31.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:31.845985+0000) 2024-03-06T10:18:32.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:32.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:32.846167+0000) 2024-03-06T10:18:33.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:33.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:33.846398+0000) 2024-03-06T10:18:34.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:34.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:34.846638+0000) 2024-03-06T10:18:35.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:35.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:35.846862+0000) 2024-03-06T10:18:36.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:36.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:36.847091+0000) 2024-03-06T10:18:37.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:37.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:37.847311+0000) 2024-03-06T10:18:38.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:38.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:38.847566+0000) 2024-03-06T10:18:39.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:39.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:39.847793+0000) 2024-03-06T10:18:40.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:40.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:40.848031+0000) 2024-03-06T10:18:41.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:41.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:41.848226+0000) 2024-03-06T10:18:42.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:42.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:42.848440+0000) 2024-03-06T10:18:43.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:43.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:43.848631+0000) 2024-03-06T10:18:44.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:44.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:44.848828+0000) 2024-03-06T10:18:45.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:45.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:45.849055+0000) 2024-03-06T10:18:46.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:46.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:46.849332+0000) 2024-03-06T10:18:47.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:47.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:47.849641+0000) 2024-03-06T10:18:48.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:48.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:48.849853+0000) 2024-03-06T10:18:49.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:49.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:49.850051+0000) 2024-03-06T10:18:50.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:50.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:50.850334+0000) 2024-03-06T10:18:51.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:51.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:51.850655+0000) 2024-03-06T10:18:52.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:52.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:52.850905+0000) 2024-03-06T10:18:53.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:53.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:53.851130+0000) 2024-03-06T10:18:54.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:54.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:54.851373+0000) 2024-03-06T10:18:55.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:55.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:55.851674+0000) 2024-03-06T10:18:56.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:56.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:56.851924+0000) 2024-03-06T10:18:57.317 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:18:57.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:18:57.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:57.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:57.852172+0000) 2024-03-06T10:18:58.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:58.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:58.852439+0000) 2024-03-06T10:18:59.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:18:59.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:18:59.852707+0000) 2024-03-06T10:19:00.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:00.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:00.852957+0000) 2024-03-06T10:19:01.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:01.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:01.853186+0000) 2024-03-06T10:19:02.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:02.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:02.853450+0000) 2024-03-06T10:19:03.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:03.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:03.853727+0000) 2024-03-06T10:19:04.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:04.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:04.853957+0000) 2024-03-06T10:19:05.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:05.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:05.854225+0000) 2024-03-06T10:19:06.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:06.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:06.854488+0000) 2024-03-06T10:19:07.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:07.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:07.854682+0000) 2024-03-06T10:19:08.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:08.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:08.854917+0000) 2024-03-06T10:19:09.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:09.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:09.855174+0000) 2024-03-06T10:19:10.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:10.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:10.855411+0000) 2024-03-06T10:19:11.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:11.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:11.855696+0000) 2024-03-06T10:19:12.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:12.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:12.855947+0000) 2024-03-06T10:19:13.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:13.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:13.856169+0000) 2024-03-06T10:19:14.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:14.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:14.856460+0000) 2024-03-06T10:19:15.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:15.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:15.856706+0000) 2024-03-06T10:19:16.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:16.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:16.856901+0000) 2024-03-06T10:19:17.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:17.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:17.857096+0000) 2024-03-06T10:19:18.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:18.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:18.857323+0000) 2024-03-06T10:19:19.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:19.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:19.857571+0000) 2024-03-06T10:19:20.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:20.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:20.857810+0000) 2024-03-06T10:19:21.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:21.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:21.858030+0000) 2024-03-06T10:19:22.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:22.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:22.858260+0000) 2024-03-06T10:19:23.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:23.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:23.858504+0000) 2024-03-06T10:19:24.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:24.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:24.858756+0000) 2024-03-06T10:19:25.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:25.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:25.858986+0000) 2024-03-06T10:19:26.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:26.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:26.859227+0000) 2024-03-06T10:19:27.513 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:19:27.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:19:27.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:27.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:27.859476+0000) 2024-03-06T10:19:28.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:28.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:28.859716+0000) 2024-03-06T10:19:29.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:29.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:29.859949+0000) 2024-03-06T10:19:30.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:30.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:30.860191+0000) 2024-03-06T10:19:31.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:31.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:31.860437+0000) 2024-03-06T10:19:32.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:32.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:32.860678+0000) 2024-03-06T10:19:33.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:33.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:33.860857+0000) 2024-03-06T10:19:34.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:34.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:34.861090+0000) 2024-03-06T10:19:35.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:35.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:35.861295+0000) 2024-03-06T10:19:36.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:36.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:36.861513+0000) 2024-03-06T10:19:37.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:37.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:37.861778+0000) 2024-03-06T10:19:38.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:38.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:38.861986+0000) 2024-03-06T10:19:39.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:39.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:39.862179+0000) 2024-03-06T10:19:40.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:40.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:40.862391+0000) 2024-03-06T10:19:41.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:41.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:41.862653+0000) 2024-03-06T10:19:42.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:42.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:42.862844+0000) 2024-03-06T10:19:43.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:43.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:43.863039+0000) 2024-03-06T10:19:44.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:44.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:44.863239+0000) 2024-03-06T10:19:45.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:45.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:45.863429+0000) 2024-03-06T10:19:46.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:46.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:46.863659+0000) 2024-03-06T10:19:47.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:47.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:47.863898+0000) 2024-03-06T10:19:48.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:48.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:48.864151+0000) 2024-03-06T10:19:49.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:49.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:49.864383+0000) 2024-03-06T10:19:50.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:50.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:50.864644+0000) 2024-03-06T10:19:51.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:51.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:51.864864+0000) 2024-03-06T10:19:52.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:52.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:52.865062+0000) 2024-03-06T10:19:53.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:53.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:53.865286+0000) 2024-03-06T10:19:54.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:54.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:54.865508+0000) 2024-03-06T10:19:55.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:55.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:55.865758+0000) 2024-03-06T10:19:56.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:56.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:56.865977+0000) 2024-03-06T10:19:57.719 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:19:57.728 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:19:57.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:57.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:57.866204+0000) 2024-03-06T10:19:58.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:58.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:58.866417+0000) 2024-03-06T10:19:59.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:19:59.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:19:59.866676+0000) 2024-03-06T10:20:00.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:00.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:00.866883+0000) 2024-03-06T10:20:01.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:01.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:01.867084+0000) 2024-03-06T10:20:02.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:02.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:02.867320+0000) 2024-03-06T10:20:03.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:03.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:03.867589+0000) 2024-03-06T10:20:04.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:04.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:04.867805+0000) 2024-03-06T10:20:05.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:05.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:05.868015+0000) 2024-03-06T10:20:06.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:06.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:06.868215+0000) 2024-03-06T10:20:07.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:07.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:07.868476+0000) 2024-03-06T10:20:08.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:08.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:08.868712+0000) 2024-03-06T10:20:09.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:09.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:09.868864+0000) 2024-03-06T10:20:10.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:10.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:10.869034+0000) 2024-03-06T10:20:11.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:11.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:11.869252+0000) 2024-03-06T10:20:12.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:12.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:12.869442+0000) 2024-03-06T10:20:13.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:13.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:13.869671+0000) 2024-03-06T10:20:14.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:14.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:14.869905+0000) 2024-03-06T10:20:15.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:15.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:15.870127+0000) 2024-03-06T10:20:16.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:16.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:16.870359+0000) 2024-03-06T10:20:17.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:17.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:17.870612+0000) 2024-03-06T10:20:18.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:18.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:18.870840+0000) 2024-03-06T10:20:19.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:19.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:19.871058+0000) 2024-03-06T10:20:20.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:20.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:20.871260+0000) 2024-03-06T10:20:21.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:21.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:21.871486+0000) 2024-03-06T10:20:22.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:22.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:22.871710+0000) 2024-03-06T10:20:23.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:23.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:23.871936+0000) 2024-03-06T10:20:24.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:24.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:24.872164+0000) 2024-03-06T10:20:25.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:25.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:25.872392+0000) 2024-03-06T10:20:26.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:26.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:26.872650+0000) 2024-03-06T10:20:27.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:27.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:27.872793+0000) 2024-03-06T10:20:27.927 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:20:27.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:20:28.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:28.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:28.872985+0000) 2024-03-06T10:20:29.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:29.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:29.873204+0000) 2024-03-06T10:20:30.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:30.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:30.873428+0000) 2024-03-06T10:20:31.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:31.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:31.873604+0000) 2024-03-06T10:20:32.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:32.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:32.873821+0000) 2024-03-06T10:20:33.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:33.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:33.874050+0000) 2024-03-06T10:20:34.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:34.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:34.874281+0000) 2024-03-06T10:20:35.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:35.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:35.874505+0000) 2024-03-06T10:20:36.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:36.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:36.874777+0000) 2024-03-06T10:20:37.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:37.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:37.874962+0000) 2024-03-06T10:20:38.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:38.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:38.875200+0000) 2024-03-06T10:20:39.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:39.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:39.875465+0000) 2024-03-06T10:20:40.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:40.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:40.875730+0000) 2024-03-06T10:20:41.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:41.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:41.875951+0000) 2024-03-06T10:20:42.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:42.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:42.876146+0000) 2024-03-06T10:20:43.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:43.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:43.876377+0000) 2024-03-06T10:20:44.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:44.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:44.876666+0000) 2024-03-06T10:20:45.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:45.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:45.876950+0000) 2024-03-06T10:20:46.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:46.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:46.877228+0000) 2024-03-06T10:20:47.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:47.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:47.877495+0000) 2024-03-06T10:20:48.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:48.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:48.877718+0000) 2024-03-06T10:20:49.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:49.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:49.877946+0000) 2024-03-06T10:20:50.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:50.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:50.878140+0000) 2024-03-06T10:20:51.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:51.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:51.878322+0000) 2024-03-06T10:20:52.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:52.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:52.878499+0000) 2024-03-06T10:20:53.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:53.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:53.878702+0000) 2024-03-06T10:20:54.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:54.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:54.878902+0000) 2024-03-06T10:20:55.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:55.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:55.879175+0000) 2024-03-06T10:20:56.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:56.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:56.879453+0000) 2024-03-06T10:20:57.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:57.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:57.879686+0000) 2024-03-06T10:20:58.118 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:20:58.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:20:58.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:58.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:58.879877+0000) 2024-03-06T10:20:59.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:20:59.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:20:59.880067+0000) 2024-03-06T10:21:00.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:00.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:00.880311+0000) 2024-03-06T10:21:01.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:01.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:01.880506+0000) 2024-03-06T10:21:02.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:02.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:02.880722+0000) 2024-03-06T10:21:03.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:03.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:03.880915+0000) 2024-03-06T10:21:04.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:04.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:04.881138+0000) 2024-03-06T10:21:05.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:05.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:05.881364+0000) 2024-03-06T10:21:06.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:06.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:06.881646+0000) 2024-03-06T10:21:07.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:07.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:07.881872+0000) 2024-03-06T10:21:08.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:08.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:08.882104+0000) 2024-03-06T10:21:09.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:09.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:09.882325+0000) 2024-03-06T10:21:10.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:10.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:10.882582+0000) 2024-03-06T10:21:11.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:11.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:11.882781+0000) 2024-03-06T10:21:12.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:12.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:12.882999+0000) 2024-03-06T10:21:13.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:13.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:13.883234+0000) 2024-03-06T10:21:14.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:14.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:14.883438+0000) 2024-03-06T10:21:15.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:15.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:15.883673+0000) 2024-03-06T10:21:16.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:16.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:16.883900+0000) 2024-03-06T10:21:17.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:17.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:17.884124+0000) 2024-03-06T10:21:18.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:18.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:18.884359+0000) 2024-03-06T10:21:19.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:19.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:19.884565+0000) 2024-03-06T10:21:20.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:20.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:20.884791+0000) 2024-03-06T10:21:21.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:21.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:21.884984+0000) 2024-03-06T10:21:22.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:22.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:22.885222+0000) 2024-03-06T10:21:23.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:23.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:23.885458+0000) 2024-03-06T10:21:24.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:24.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:24.885707+0000) 2024-03-06T10:21:25.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:25.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:25.885896+0000) 2024-03-06T10:21:26.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:26.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:26.886083+0000) 2024-03-06T10:21:27.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:27.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:27.886291+0000) 2024-03-06T10:21:28.300 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:21:28.309 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:21:28.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:28.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:28.886527+0000) 2024-03-06T10:21:29.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:29.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:29.886788+0000) 2024-03-06T10:21:30.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:30.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:30.886995+0000) 2024-03-06T10:21:31.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:31.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:31.887196+0000) 2024-03-06T10:21:32.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:32.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:32.887428+0000) 2024-03-06T10:21:33.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:33.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:33.887682+0000) 2024-03-06T10:21:34.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:34.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:34.887906+0000) 2024-03-06T10:21:35.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:35.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:35.888111+0000) 2024-03-06T10:21:36.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:36.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:36.888330+0000) 2024-03-06T10:21:37.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:37.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:37.888577+0000) 2024-03-06T10:21:38.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:38.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:38.888815+0000) 2024-03-06T10:21:39.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:39.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:39.889001+0000) 2024-03-06T10:21:40.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:40.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:40.889206+0000) 2024-03-06T10:21:41.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:41.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:41.889409+0000) 2024-03-06T10:21:42.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:42.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:42.889649+0000) 2024-03-06T10:21:43.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:43.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:43.889813+0000) 2024-03-06T10:21:44.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:44.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:44.889997+0000) 2024-03-06T10:21:45.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:45.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:45.890186+0000) 2024-03-06T10:21:46.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:46.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:46.890406+0000) 2024-03-06T10:21:47.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:47.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:47.890649+0000) 2024-03-06T10:21:48.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:48.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:48.890914+0000) 2024-03-06T10:21:49.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:49.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:49.891112+0000) 2024-03-06T10:21:50.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:50.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:50.891351+0000) 2024-03-06T10:21:51.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:51.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:51.891625+0000) 2024-03-06T10:21:52.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:52.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:52.891842+0000) 2024-03-06T10:21:53.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:53.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:53.892058+0000) 2024-03-06T10:21:54.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:54.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:54.892289+0000) 2024-03-06T10:21:55.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:55.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:55.892512+0000) 2024-03-06T10:21:56.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:56.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:56.892727+0000) 2024-03-06T10:21:57.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:57.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:57.892917+0000) 2024-03-06T10:21:58.512 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:21:58.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:21:58.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:58.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:58.893141+0000) 2024-03-06T10:21:59.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:21:59.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:21:59.893380+0000) 2024-03-06T10:22:00.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:00.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:00.893654+0000) 2024-03-06T10:22:01.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:01.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:01.893887+0000) 2024-03-06T10:22:02.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:02.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:02.894070+0000) 2024-03-06T10:22:03.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:03.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:03.894259+0000) 2024-03-06T10:22:04.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:04.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:04.894469+0000) 2024-03-06T10:22:05.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:05.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:05.894678+0000) 2024-03-06T10:22:06.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:06.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:06.894872+0000) 2024-03-06T10:22:07.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:07.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:07.895053+0000) 2024-03-06T10:22:08.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:08.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:08.895242+0000) 2024-03-06T10:22:09.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:09.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:09.895460+0000) 2024-03-06T10:22:10.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:10.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:10.895660+0000) 2024-03-06T10:22:11.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:11.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:11.895883+0000) 2024-03-06T10:22:12.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:12.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:12.896110+0000) 2024-03-06T10:22:13.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:13.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:13.896305+0000) 2024-03-06T10:22:14.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:14.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:14.896566+0000) 2024-03-06T10:22:15.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:15.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:15.896786+0000) 2024-03-06T10:22:16.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:16.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:16.896995+0000) 2024-03-06T10:22:17.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:17.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:17.897210+0000) 2024-03-06T10:22:18.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:18.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:18.897448+0000) 2024-03-06T10:22:19.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:19.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:19.897683+0000) 2024-03-06T10:22:20.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:20.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:20.897890+0000) 2024-03-06T10:22:21.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:21.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:21.898102+0000) 2024-03-06T10:22:22.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:22.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:22.898322+0000) 2024-03-06T10:22:23.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:23.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:23.898527+0000) 2024-03-06T10:22:24.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:24.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:24.898770+0000) 2024-03-06T10:22:25.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:25.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:25.899006+0000) 2024-03-06T10:22:26.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:26.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:26.899230+0000) 2024-03-06T10:22:27.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:27.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:27.899512+0000) 2024-03-06T10:22:28.777 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:22:28.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:22:28.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:28.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:28.899779+0000) 2024-03-06T10:22:29.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:29.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:29.899971+0000) 2024-03-06T10:22:30.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:30.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:30.900177+0000) 2024-03-06T10:22:31.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:31.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:31.900388+0000) 2024-03-06T10:22:32.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:32.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:32.900644+0000) 2024-03-06T10:22:33.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:33.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:33.900883+0000) 2024-03-06T10:22:34.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:34.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:34.901095+0000) 2024-03-06T10:22:35.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:35.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:35.901331+0000) 2024-03-06T10:22:36.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:36.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:36.901499+0000) 2024-03-06T10:22:37.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:37.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:37.901681+0000) 2024-03-06T10:22:38.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:38.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:38.901923+0000) 2024-03-06T10:22:39.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:39.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:39.902139+0000) 2024-03-06T10:22:40.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:40.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:40.902329+0000) 2024-03-06T10:22:41.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:41.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:41.902578+0000) 2024-03-06T10:22:42.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:42.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:42.902777+0000) 2024-03-06T10:22:43.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:43.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:43.902998+0000) 2024-03-06T10:22:44.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:44.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:44.903235+0000) 2024-03-06T10:22:45.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:45.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:45.903457+0000) 2024-03-06T10:22:46.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:46.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:46.903679+0000) 2024-03-06T10:22:47.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:47.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:47.903873+0000) 2024-03-06T10:22:48.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:48.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:48.904094+0000) 2024-03-06T10:22:49.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:49.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:49.904273+0000) 2024-03-06T10:22:50.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:50.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:50.904516+0000) 2024-03-06T10:22:51.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:51.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:51.904780+0000) 2024-03-06T10:22:52.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:52.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:52.904990+0000) 2024-03-06T10:22:53.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:53.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:53.905193+0000) 2024-03-06T10:22:54.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:54.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:54.905441+0000) 2024-03-06T10:22:55.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:55.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:55.905684+0000) 2024-03-06T10:22:56.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:56.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:56.905889+0000) 2024-03-06T10:22:57.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:57.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:57.906090+0000) 2024-03-06T10:22:58.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:58.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:58.906314+0000) 2024-03-06T10:22:58.921 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:22:58.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:22:59.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:22:59.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:22:59.906577+0000) 2024-03-06T10:23:00.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:00.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:00.906776+0000) 2024-03-06T10:23:01.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:01.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:01.906999+0000) 2024-03-06T10:23:02.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:02.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:02.907237+0000) 2024-03-06T10:23:03.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:03.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:03.907579+0000) 2024-03-06T10:23:04.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:04.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:04.907862+0000) 2024-03-06T10:23:05.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:05.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:05.908049+0000) 2024-03-06T10:23:06.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:06.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:06.908271+0000) 2024-03-06T10:23:07.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:07.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:07.908495+0000) 2024-03-06T10:23:08.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:08.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:08.908748+0000) 2024-03-06T10:23:09.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:09.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:09.908974+0000) 2024-03-06T10:23:10.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:10.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:10.909163+0000) 2024-03-06T10:23:11.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:11.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:11.909343+0000) 2024-03-06T10:23:12.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:12.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:12.909596+0000) 2024-03-06T10:23:13.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:13.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:13.909792+0000) 2024-03-06T10:23:14.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:14.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:14.910024+0000) 2024-03-06T10:23:15.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:15.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:15.910199+0000) 2024-03-06T10:23:16.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:16.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:16.910446+0000) 2024-03-06T10:23:17.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:17.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:17.910690+0000) 2024-03-06T10:23:18.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:18.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:18.910960+0000) 2024-03-06T10:23:19.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:19.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:19.911183+0000) 2024-03-06T10:23:20.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:20.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:20.911357+0000) 2024-03-06T10:23:21.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:21.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:21.911522+0000) 2024-03-06T10:23:22.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:22.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:22.911741+0000) 2024-03-06T10:23:23.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:23.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:23.911940+0000) 2024-03-06T10:23:24.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:24.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:24.912220+0000) 2024-03-06T10:23:25.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:25.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:25.912454+0000) 2024-03-06T10:23:26.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:26.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:26.912687+0000) 2024-03-06T10:23:27.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:27.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:27.912871+0000) 2024-03-06T10:23:28.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:28.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:28.913082+0000) 2024-03-06T10:23:29.126 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:23:29.138 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:23:29.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:29.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:29.913297+0000) 2024-03-06T10:23:30.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:30.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:30.913563+0000) 2024-03-06T10:23:31.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:31.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:31.913787+0000) 2024-03-06T10:23:32.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:32.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:32.913985+0000) 2024-03-06T10:23:33.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:33.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:33.914193+0000) 2024-03-06T10:23:34.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:34.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:34.914445+0000) 2024-03-06T10:23:35.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:35.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:35.914700+0000) 2024-03-06T10:23:36.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:36.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:36.914902+0000) 2024-03-06T10:23:37.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:37.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:37.915104+0000) 2024-03-06T10:23:38.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:38.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:38.915353+0000) 2024-03-06T10:23:39.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:39.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:39.915613+0000) 2024-03-06T10:23:40.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:40.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:40.915823+0000) 2024-03-06T10:23:41.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:41.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:41.916019+0000) 2024-03-06T10:23:42.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:42.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:42.916241+0000) 2024-03-06T10:23:43.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:43.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:43.916472+0000) 2024-03-06T10:23:44.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:44.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:44.916657+0000) 2024-03-06T10:23:45.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:45.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:45.916861+0000) 2024-03-06T10:23:46.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:46.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:46.917082+0000) 2024-03-06T10:23:47.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:47.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:47.917270+0000) 2024-03-06T10:23:48.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:48.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:48.917455+0000) 2024-03-06T10:23:49.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:49.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:49.917677+0000) 2024-03-06T10:23:50.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:50.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:50.917863+0000) 2024-03-06T10:23:51.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:51.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:51.918054+0000) 2024-03-06T10:23:52.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:52.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:52.918263+0000) 2024-03-06T10:23:53.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:53.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:53.918507+0000) 2024-03-06T10:23:54.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:54.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:54.918759+0000) 2024-03-06T10:23:55.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:55.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:55.918950+0000) 2024-03-06T10:23:56.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:56.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:56.919143+0000) 2024-03-06T10:23:57.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:57.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:57.919367+0000) 2024-03-06T10:23:58.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:58.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:58.919622+0000) 2024-03-06T10:23:59.353 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:23:59.363 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:23:59.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:23:59.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:23:59.919806+0000) 2024-03-06T10:24:00.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:00.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:00.920031+0000) 2024-03-06T10:24:01.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:01.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:01.920236+0000) 2024-03-06T10:24:02.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:02.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:02.920464+0000) 2024-03-06T10:24:03.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:03.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:03.920666+0000) 2024-03-06T10:24:04.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:04.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:04.920908+0000) 2024-03-06T10:24:05.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:05.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:05.921131+0000) 2024-03-06T10:24:06.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:06.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:06.921328+0000) 2024-03-06T10:24:07.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:07.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:07.921574+0000) 2024-03-06T10:24:08.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:08.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:08.921797+0000) 2024-03-06T10:24:09.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:09.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:09.922018+0000) 2024-03-06T10:24:10.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:10.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:10.922208+0000) 2024-03-06T10:24:11.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:11.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:11.922429+0000) 2024-03-06T10:24:12.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:12.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:12.922619+0000) 2024-03-06T10:24:13.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:13.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:13.922810+0000) 2024-03-06T10:24:14.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:14.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:14.923059+0000) 2024-03-06T10:24:15.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:15.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:15.923254+0000) 2024-03-06T10:24:16.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:16.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:16.923475+0000) 2024-03-06T10:24:17.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:17.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:17.923710+0000) 2024-03-06T10:24:18.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:18.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:18.924004+0000) 2024-03-06T10:24:19.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:19.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:19.924222+0000) 2024-03-06T10:24:20.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:20.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:20.924469+0000) 2024-03-06T10:24:21.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:21.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:21.924676+0000) 2024-03-06T10:24:22.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:22.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:22.924853+0000) 2024-03-06T10:24:23.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:23.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:23.925074+0000) 2024-03-06T10:24:24.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:24.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:24.925308+0000) 2024-03-06T10:24:25.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:25.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:25.925515+0000) 2024-03-06T10:24:26.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:26.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:26.925734+0000) 2024-03-06T10:24:27.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:27.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:27.925928+0000) 2024-03-06T10:24:28.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:28.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:28.926110+0000) 2024-03-06T10:24:29.576 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:24:29.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:24:29.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:29.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:29.926336+0000) 2024-03-06T10:24:30.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:30.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:30.926599+0000) 2024-03-06T10:24:31.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:31.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:31.926784+0000) 2024-03-06T10:24:32.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:32.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:32.926974+0000) 2024-03-06T10:24:33.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:33.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:33.927160+0000) 2024-03-06T10:24:34.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:34.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:34.927428+0000) 2024-03-06T10:24:35.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:35.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:35.927677+0000) 2024-03-06T10:24:36.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:36.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:36.927840+0000) 2024-03-06T10:24:37.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:37.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:37.928024+0000) 2024-03-06T10:24:38.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:38.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:38.928245+0000) 2024-03-06T10:24:39.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:39.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:39.928437+0000) 2024-03-06T10:24:40.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:40.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:40.928699+0000) 2024-03-06T10:24:41.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:41.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:41.928890+0000) 2024-03-06T10:24:42.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:42.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:42.929114+0000) 2024-03-06T10:24:43.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:43.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:43.929330+0000) 2024-03-06T10:24:44.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:44.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:44.929599+0000) 2024-03-06T10:24:45.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:45.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:45.929787+0000) 2024-03-06T10:24:46.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:46.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:46.929979+0000) 2024-03-06T10:24:47.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:47.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:47.930186+0000) 2024-03-06T10:24:48.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:48.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:48.930386+0000) 2024-03-06T10:24:49.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:49.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:49.930661+0000) 2024-03-06T10:24:50.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:50.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:50.930905+0000) 2024-03-06T10:24:51.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:51.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:51.931116+0000) 2024-03-06T10:24:52.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:52.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:52.931298+0000) 2024-03-06T10:24:53.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:53.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:53.931519+0000) 2024-03-06T10:24:54.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:54.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:54.931815+0000) 2024-03-06T10:24:55.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:55.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:55.932026+0000) 2024-03-06T10:24:56.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:56.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:56.932270+0000) 2024-03-06T10:24:57.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:57.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:57.932496+0000) 2024-03-06T10:24:58.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:58.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:58.932658+0000) 2024-03-06T10:24:59.800 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:24:59.808 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:24:59.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:24:59.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:24:59.932893+0000) 2024-03-06T10:25:00.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:00.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:00.933132+0000) 2024-03-06T10:25:01.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:01.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:01.933362+0000) 2024-03-06T10:25:02.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:02.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:02.933649+0000) 2024-03-06T10:25:03.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:03.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:03.933876+0000) 2024-03-06T10:25:04.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:04.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:04.934125+0000) 2024-03-06T10:25:05.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:05.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:05.934360+0000) 2024-03-06T10:25:06.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:06.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:06.934674+0000) 2024-03-06T10:25:07.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:07.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:07.934909+0000) 2024-03-06T10:25:08.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:08.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:08.935144+0000) 2024-03-06T10:25:09.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:09.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:09.935323+0000) 2024-03-06T10:25:10.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:10.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:10.935563+0000) 2024-03-06T10:25:11.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:11.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:11.935764+0000) 2024-03-06T10:25:12.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:12.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:12.935973+0000) 2024-03-06T10:25:13.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:13.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:13.936202+0000) 2024-03-06T10:25:14.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:14.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:14.936455+0000) 2024-03-06T10:25:15.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:15.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:15.936728+0000) 2024-03-06T10:25:16.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:16.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:16.937012+0000) 2024-03-06T10:25:17.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:17.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:17.937237+0000) 2024-03-06T10:25:18.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:18.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:18.937475+0000) 2024-03-06T10:25:19.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:19.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:19.937696+0000) 2024-03-06T10:25:20.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:20.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:20.937903+0000) 2024-03-06T10:25:21.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:21.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:21.938103+0000) 2024-03-06T10:25:22.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:22.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:22.938333+0000) 2024-03-06T10:25:23.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:23.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:23.938576+0000) 2024-03-06T10:25:24.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:24.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:24.938784+0000) 2024-03-06T10:25:25.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:25.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:25.939015+0000) 2024-03-06T10:25:26.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:26.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:26.939232+0000) 2024-03-06T10:25:27.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:27.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:27.939454+0000) 2024-03-06T10:25:28.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:28.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:28.939714+0000) 2024-03-06T10:25:29.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:29.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:29.939924+0000) 2024-03-06T10:25:30.012 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:25:30.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:25:30.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:30.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:30.940187+0000) 2024-03-06T10:25:31.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:31.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:31.940422+0000) 2024-03-06T10:25:32.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:32.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:32.940674+0000) 2024-03-06T10:25:33.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:33.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:33.940860+0000) 2024-03-06T10:25:34.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:34.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:34.941071+0000) 2024-03-06T10:25:35.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:35.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:35.941308+0000) 2024-03-06T10:25:36.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:36.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:36.941574+0000) 2024-03-06T10:25:37.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:37.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:37.941784+0000) 2024-03-06T10:25:38.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:38.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:38.941988+0000) 2024-03-06T10:25:39.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:39.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:39.942223+0000) 2024-03-06T10:25:40.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:40.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:40.942463+0000) 2024-03-06T10:25:41.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:41.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:41.942698+0000) 2024-03-06T10:25:42.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:42.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:42.942930+0000) 2024-03-06T10:25:43.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:43.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:43.943132+0000) 2024-03-06T10:25:44.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:44.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:44.943375+0000) 2024-03-06T10:25:45.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:45.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:45.943646+0000) 2024-03-06T10:25:46.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:46.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:46.943854+0000) 2024-03-06T10:25:47.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:47.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:47.944059+0000) 2024-03-06T10:25:48.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:48.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:48.944264+0000) 2024-03-06T10:25:49.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:49.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:49.944475+0000) 2024-03-06T10:25:50.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:50.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:50.944685+0000) 2024-03-06T10:25:51.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:51.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:51.944920+0000) 2024-03-06T10:25:52.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:52.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:52.945112+0000) 2024-03-06T10:25:53.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:53.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:53.945345+0000) 2024-03-06T10:25:54.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:54.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:54.945609+0000) 2024-03-06T10:25:55.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:55.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:55.945859+0000) 2024-03-06T10:25:56.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:56.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:56.946062+0000) 2024-03-06T10:25:57.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:57.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:57.946298+0000) 2024-03-06T10:25:58.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:58.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:58.946561+0000) 2024-03-06T10:25:59.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:25:59.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:25:59.946824+0000) 2024-03-06T10:26:00.208 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:26:00.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:26:00.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:00.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:00.947045+0000) 2024-03-06T10:26:01.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:01.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:01.947279+0000) 2024-03-06T10:26:02.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:02.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:02.947511+0000) 2024-03-06T10:26:03.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:03.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:03.947782+0000) 2024-03-06T10:26:04.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:04.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:04.948003+0000) 2024-03-06T10:26:05.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:05.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:05.948239+0000) 2024-03-06T10:26:06.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:06.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:06.948474+0000) 2024-03-06T10:26:07.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:07.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:07.948667+0000) 2024-03-06T10:26:08.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:08.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:08.948868+0000) 2024-03-06T10:26:09.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:09.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:09.949072+0000) 2024-03-06T10:26:10.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:10.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:10.949294+0000) 2024-03-06T10:26:11.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:11.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:11.949527+0000) 2024-03-06T10:26:12.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:12.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:12.949772+0000) 2024-03-06T10:26:13.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:13.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:13.950010+0000) 2024-03-06T10:26:14.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:14.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:14.950259+0000) 2024-03-06T10:26:15.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:15.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:15.950499+0000) 2024-03-06T10:26:16.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:16.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:16.950688+0000) 2024-03-06T10:26:17.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:17.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:17.950881+0000) 2024-03-06T10:26:18.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:18.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:18.951096+0000) 2024-03-06T10:26:19.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:19.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:19.951309+0000) 2024-03-06T10:26:20.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:20.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:20.951519+0000) 2024-03-06T10:26:21.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:21.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:21.951802+0000) 2024-03-06T10:26:22.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:22.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:22.951989+0000) 2024-03-06T10:26:23.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:23.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:23.952232+0000) 2024-03-06T10:26:24.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:24.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:24.952485+0000) 2024-03-06T10:26:25.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:25.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:25.952731+0000) 2024-03-06T10:26:26.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:26.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:26.952988+0000) 2024-03-06T10:26:27.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:27.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:27.953212+0000) 2024-03-06T10:26:28.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:28.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:28.953436+0000) 2024-03-06T10:26:29.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:29.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:29.953657+0000) 2024-03-06T10:26:30.342 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:26:30.350 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:26:30.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:30.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:30.953845+0000) 2024-03-06T10:26:31.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:31.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:31.954039+0000) 2024-03-06T10:26:32.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:32.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:32.954274+0000) 2024-03-06T10:26:33.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:33.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:33.954475+0000) 2024-03-06T10:26:34.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:34.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:34.954644+0000) 2024-03-06T10:26:35.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:35.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:35.954833+0000) 2024-03-06T10:26:36.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:36.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:36.955062+0000) 2024-03-06T10:26:37.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:37.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:37.955265+0000) 2024-03-06T10:26:38.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:38.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:38.955454+0000) 2024-03-06T10:26:39.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:39.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:39.955633+0000) 2024-03-06T10:26:40.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:40.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:40.955835+0000) 2024-03-06T10:26:41.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:41.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:41.956057+0000) 2024-03-06T10:26:42.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:42.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:42.956332+0000) 2024-03-06T10:26:43.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:43.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:43.956619+0000) 2024-03-06T10:26:44.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:44.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:44.956869+0000) 2024-03-06T10:26:45.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:45.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:45.957108+0000) 2024-03-06T10:26:46.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:46.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:46.957331+0000) 2024-03-06T10:26:47.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:47.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:47.957567+0000) 2024-03-06T10:26:48.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:48.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:48.957771+0000) 2024-03-06T10:26:49.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:49.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:49.957967+0000) 2024-03-06T10:26:50.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:50.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:50.958216+0000) 2024-03-06T10:26:51.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:51.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:51.958465+0000) 2024-03-06T10:26:52.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:52.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:52.958642+0000) 2024-03-06T10:26:53.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:53.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:53.958847+0000) 2024-03-06T10:26:54.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:54.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:54.959050+0000) 2024-03-06T10:26:55.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:55.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:55.959270+0000) 2024-03-06T10:26:56.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:56.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:56.959493+0000) 2024-03-06T10:26:57.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:57.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:57.959737+0000) 2024-03-06T10:26:58.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:58.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:58.959941+0000) 2024-03-06T10:26:59.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:26:59.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:26:59.960150+0000) 2024-03-06T10:27:00.560 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:27:00.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:27:00.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:00.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:00.960401+0000) 2024-03-06T10:27:01.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:01.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:01.960642+0000) 2024-03-06T10:27:02.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:02.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:02.960874+0000) 2024-03-06T10:27:03.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:03.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:03.961080+0000) 2024-03-06T10:27:04.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:04.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:04.961330+0000) 2024-03-06T10:27:05.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:05.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:05.961589+0000) 2024-03-06T10:27:06.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:06.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:06.961789+0000) 2024-03-06T10:27:07.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:07.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:07.962009+0000) 2024-03-06T10:27:08.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:08.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:08.962240+0000) 2024-03-06T10:27:09.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:09.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:09.962484+0000) 2024-03-06T10:27:10.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:10.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:10.962694+0000) 2024-03-06T10:27:11.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:11.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:11.962874+0000) 2024-03-06T10:27:12.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:12.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:12.963055+0000) 2024-03-06T10:27:13.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:13.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:13.963259+0000) 2024-03-06T10:27:14.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:14.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:14.963516+0000) 2024-03-06T10:27:15.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:15.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:15.963750+0000) 2024-03-06T10:27:16.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:16.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:16.963937+0000) 2024-03-06T10:27:17.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:17.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:17.964153+0000) 2024-03-06T10:27:18.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:18.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:18.964371+0000) 2024-03-06T10:27:19.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:19.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:19.964645+0000) 2024-03-06T10:27:20.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:20.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:20.964834+0000) 2024-03-06T10:27:21.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:21.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:21.965054+0000) 2024-03-06T10:27:22.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:22.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:22.965236+0000) 2024-03-06T10:27:23.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:23.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:23.965468+0000) 2024-03-06T10:27:24.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:24.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:24.965655+0000) 2024-03-06T10:27:25.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:25.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:25.965852+0000) 2024-03-06T10:27:26.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:26.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:26.966020+0000) 2024-03-06T10:27:27.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:27.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:27.966240+0000) 2024-03-06T10:27:28.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:28.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:28.966437+0000) 2024-03-06T10:27:29.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:29.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:29.966714+0000) 2024-03-06T10:27:30.823 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:27:30.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:27:30.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:30.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:30.966902+0000) 2024-03-06T10:27:31.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:31.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:31.967087+0000) 2024-03-06T10:27:32.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:32.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:32.967272+0000) 2024-03-06T10:27:33.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:33.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:33.967492+0000) 2024-03-06T10:27:34.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:34.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:34.967693+0000) 2024-03-06T10:27:35.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:35.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:35.967918+0000) 2024-03-06T10:27:36.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:36.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:36.968142+0000) 2024-03-06T10:27:37.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:37.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:37.968380+0000) 2024-03-06T10:27:38.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:38.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:38.968610+0000) 2024-03-06T10:27:39.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:39.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:39.968805+0000) 2024-03-06T10:27:40.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:40.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:40.969000+0000) 2024-03-06T10:27:41.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:41.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:41.969222+0000) 2024-03-06T10:27:42.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:42.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:42.969435+0000) 2024-03-06T10:27:43.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:43.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:43.969666+0000) 2024-03-06T10:27:44.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:44.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:44.969859+0000) 2024-03-06T10:27:45.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:45.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:45.970086+0000) 2024-03-06T10:27:46.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:46.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:46.970318+0000) 2024-03-06T10:27:47.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:47.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:47.970510+0000) 2024-03-06T10:27:48.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:48.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:48.970733+0000) 2024-03-06T10:27:49.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:49.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:49.970932+0000) 2024-03-06T10:27:50.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:50.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:50.971154+0000) 2024-03-06T10:27:51.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:51.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:51.971374+0000) 2024-03-06T10:27:52.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:52.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:52.971593+0000) 2024-03-06T10:27:53.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:53.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:53.971811+0000) 2024-03-06T10:27:54.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:54.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:54.972021+0000) 2024-03-06T10:27:55.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:55.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:55.972260+0000) 2024-03-06T10:27:56.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:56.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:56.972468+0000) 2024-03-06T10:27:57.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:57.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:57.972692+0000) 2024-03-06T10:27:58.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:58.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:58.972881+0000) 2024-03-06T10:27:59.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:27:59.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:27:59.973089+0000) 2024-03-06T10:28:00.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:00.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:00.973363+0000) 2024-03-06T10:28:01.019 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:28:01.026 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:28:01.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:01.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:01.973613+0000) 2024-03-06T10:28:02.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:02.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:02.973813+0000) 2024-03-06T10:28:03.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:03.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:03.974002+0000) 2024-03-06T10:28:04.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:04.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:04.974211+0000) 2024-03-06T10:28:05.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:05.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:05.974433+0000) 2024-03-06T10:28:06.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:06.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:06.974658+0000) 2024-03-06T10:28:07.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:07.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:07.974859+0000) 2024-03-06T10:28:08.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:08.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:08.975081+0000) 2024-03-06T10:28:09.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:09.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:09.975315+0000) 2024-03-06T10:28:10.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:10.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:10.975511+0000) 2024-03-06T10:28:11.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:11.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:11.975740+0000) 2024-03-06T10:28:12.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:12.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:12.975931+0000) 2024-03-06T10:28:13.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:13.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:13.976120+0000) 2024-03-06T10:28:14.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:14.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:14.976363+0000) 2024-03-06T10:28:15.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:15.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:15.976623+0000) 2024-03-06T10:28:16.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:16.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:16.976844+0000) 2024-03-06T10:28:17.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:17.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:17.977066+0000) 2024-03-06T10:28:18.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:18.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:18.977298+0000) 2024-03-06T10:28:19.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:19.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:19.977528+0000) 2024-03-06T10:28:20.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:20.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:20.977745+0000) 2024-03-06T10:28:21.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:21.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:21.977928+0000) 2024-03-06T10:28:22.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:22.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:22.978116+0000) 2024-03-06T10:28:23.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:23.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:23.978387+0000) 2024-03-06T10:28:24.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:24.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:24.978657+0000) 2024-03-06T10:28:25.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:25.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:25.978883+0000) 2024-03-06T10:28:26.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:26.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:26.979074+0000) 2024-03-06T10:28:27.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:27.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:27.979312+0000) 2024-03-06T10:28:28.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:28.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:28.979567+0000) 2024-03-06T10:28:29.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:29.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:29.979773+0000) 2024-03-06T10:28:30.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:30.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:30.979970+0000) 2024-03-06T10:28:31.242 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:28:31.250 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:28:31.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:31.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:31.980174+0000) 2024-03-06T10:28:32.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:32.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:32.980402+0000) 2024-03-06T10:28:33.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:33.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:33.980639+0000) 2024-03-06T10:28:34.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:34.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:34.980872+0000) 2024-03-06T10:28:35.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:35.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:35.981081+0000) 2024-03-06T10:28:36.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:36.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:36.981286+0000) 2024-03-06T10:28:37.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:37.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:37.981483+0000) 2024-03-06T10:28:38.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:38.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:38.981677+0000) 2024-03-06T10:28:39.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:39.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:39.981870+0000) 2024-03-06T10:28:40.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:40.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:40.982064+0000) 2024-03-06T10:28:41.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:41.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:41.982237+0000) 2024-03-06T10:28:42.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:42.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:42.982378+0000) 2024-03-06T10:28:43.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:43.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:43.982632+0000) 2024-03-06T10:28:44.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:44.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:44.982899+0000) 2024-03-06T10:28:45.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:45.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:45.983126+0000) 2024-03-06T10:28:46.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:46.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:46.983347+0000) 2024-03-06T10:28:47.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:47.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:47.983608+0000) 2024-03-06T10:28:48.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:48.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:48.983829+0000) 2024-03-06T10:28:49.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:49.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:49.984075+0000) 2024-03-06T10:28:50.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:50.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:50.984324+0000) 2024-03-06T10:28:51.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:51.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:51.984583+0000) 2024-03-06T10:28:52.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:52.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:52.984758+0000) 2024-03-06T10:28:53.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:53.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:53.984942+0000) 2024-03-06T10:28:54.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:54.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:54.985164+0000) 2024-03-06T10:28:55.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:55.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:55.985404+0000) 2024-03-06T10:28:56.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:56.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:56.985656+0000) 2024-03-06T10:28:57.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:57.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:57.985874+0000) 2024-03-06T10:28:58.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:58.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:58.986073+0000) 2024-03-06T10:28:59.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:28:59.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:28:59.986327+0000) 2024-03-06T10:29:00.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:00.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:00.986599+0000) 2024-03-06T10:29:01.455 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:29:01.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:29:01.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:01.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:01.986805+0000) 2024-03-06T10:29:02.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:02.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:02.987005+0000) 2024-03-06T10:29:03.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:03.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:03.987241+0000) 2024-03-06T10:29:04.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:04.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:04.987499+0000) 2024-03-06T10:29:05.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:05.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:05.987744+0000) 2024-03-06T10:29:06.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:06.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:06.987947+0000) 2024-03-06T10:29:07.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:07.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:07.988152+0000) 2024-03-06T10:29:08.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:08.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:08.988391+0000) 2024-03-06T10:29:09.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:09.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:09.988663+0000) 2024-03-06T10:29:10.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:10.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:10.988847+0000) 2024-03-06T10:29:11.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:11.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:11.989070+0000) 2024-03-06T10:29:12.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:12.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:12.989296+0000) 2024-03-06T10:29:13.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:13.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:13.989486+0000) 2024-03-06T10:29:14.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:14.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:14.989831+0000) 2024-03-06T10:29:15.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:15.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:15.990111+0000) 2024-03-06T10:29:16.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:16.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:16.990435+0000) 2024-03-06T10:29:17.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:17.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:17.990716+0000) 2024-03-06T10:29:18.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:18.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:18.990993+0000) 2024-03-06T10:29:19.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:19.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:19.991211+0000) 2024-03-06T10:29:20.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:20.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:20.991453+0000) 2024-03-06T10:29:21.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:21.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:21.991687+0000) 2024-03-06T10:29:22.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:22.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:22.991889+0000) 2024-03-06T10:29:23.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:23.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:23.992093+0000) 2024-03-06T10:29:24.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:24.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:24.992349+0000) 2024-03-06T10:29:25.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:25.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:25.992619+0000) 2024-03-06T10:29:26.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:26.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:26.992831+0000) 2024-03-06T10:29:27.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:27.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:27.993071+0000) 2024-03-06T10:29:28.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:28.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:28.993288+0000) 2024-03-06T10:29:29.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:29.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:29.993483+0000) 2024-03-06T10:29:30.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:30.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:30.993727+0000) 2024-03-06T10:29:31.678 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:29:31.687 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:29:31.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:31.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:31.993934+0000) 2024-03-06T10:29:32.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:32.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:32.994132+0000) 2024-03-06T10:29:33.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:33.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:33.994358+0000) 2024-03-06T10:29:34.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:34.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:34.994622+0000) 2024-03-06T10:29:35.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:35.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:35.994813+0000) 2024-03-06T10:29:36.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:36.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:36.994998+0000) 2024-03-06T10:29:37.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:37.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:37.995188+0000) 2024-03-06T10:29:38.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:38.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:38.995412+0000) 2024-03-06T10:29:39.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:39.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:39.995650+0000) 2024-03-06T10:29:40.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:40.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:40.995835+0000) 2024-03-06T10:29:41.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:41.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:41.996031+0000) 2024-03-06T10:29:42.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:42.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:42.996240+0000) 2024-03-06T10:29:43.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:43.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:43.996513+0000) 2024-03-06T10:29:44.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:44.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:44.996829+0000) 2024-03-06T10:29:45.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:45.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:45.997060+0000) 2024-03-06T10:29:46.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:46.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:46.997247+0000) 2024-03-06T10:29:47.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:47.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:47.997500+0000) 2024-03-06T10:29:48.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:48.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:48.997761+0000) 2024-03-06T10:29:49.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:49.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:49.997968+0000) 2024-03-06T10:29:50.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:50.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:50.998166+0000) 2024-03-06T10:29:51.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:51.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:51.998390+0000) 2024-03-06T10:29:52.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:52.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:52.998637+0000) 2024-03-06T10:29:53.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:53.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:53.998855+0000) 2024-03-06T10:29:54.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:54.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:54.999059+0000) 2024-03-06T10:29:55.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:55.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:55.999286+0000) 2024-03-06T10:29:57.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:56.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:56.999462+0000) 2024-03-06T10:29:57.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:57.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:57.999685+0000) 2024-03-06T10:29:58.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:58.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:29:58.999872+0000) 2024-03-06T10:29:59.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:29:59.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:00.000065+0000) 2024-03-06T10:30:00.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:00.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:01.000298+0000) 2024-03-06T10:30:01.881 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:30:01.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:30:01.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:01.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:02.000492+0000) 2024-03-06T10:30:03.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:02.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:03.000700+0000) 2024-03-06T10:30:04.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:03.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:04.000892+0000) 2024-03-06T10:30:05.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:04.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:05.001091+0000) 2024-03-06T10:30:06.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:05.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:06.001321+0000) 2024-03-06T10:30:07.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:07.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:07.001560+0000) 2024-03-06T10:30:08.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:08.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:08.001749+0000) 2024-03-06T10:30:09.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:09.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:09.001973+0000) 2024-03-06T10:30:10.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:10.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:10.002159+0000) 2024-03-06T10:30:11.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:11.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:11.002443+0000) 2024-03-06T10:30:12.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:12.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:12.002645+0000) 2024-03-06T10:30:13.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:13.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:13.002861+0000) 2024-03-06T10:30:14.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:14.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:14.003053+0000) 2024-03-06T10:30:15.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:15.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:15.003290+0000) 2024-03-06T10:30:16.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:16.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:16.003517+0000) 2024-03-06T10:30:17.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:17.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:17.003776+0000) 2024-03-06T10:30:18.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:18.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:18.003962+0000) 2024-03-06T10:30:19.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:19.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:19.004142+0000) 2024-03-06T10:30:20.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:20.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:20.004367+0000) 2024-03-06T10:30:21.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:21.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:21.004619+0000) 2024-03-06T10:30:22.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:22.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:22.004818+0000) 2024-03-06T10:30:23.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:23.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:23.005009+0000) 2024-03-06T10:30:24.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:24.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:24.005230+0000) 2024-03-06T10:30:25.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:25.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:25.005472+0000) 2024-03-06T10:30:26.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:26.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:26.005704+0000) 2024-03-06T10:30:27.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:27.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:27.005906+0000) 2024-03-06T10:30:28.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:28.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:28.006099+0000) 2024-03-06T10:30:29.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:29.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:29.006324+0000) 2024-03-06T10:30:30.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:30.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:30.006578+0000) 2024-03-06T10:30:31.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:31.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:31.006774+0000) 2024-03-06T10:30:32.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:32.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:32.006965+0000) 2024-03-06T10:30:32.102 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:30:32.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:30:33.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:33.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:33.007200+0000) 2024-03-06T10:30:34.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:34.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:34.007382+0000) 2024-03-06T10:30:35.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:35.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:35.007677+0000) 2024-03-06T10:30:36.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:36.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:36.007921+0000) 2024-03-06T10:30:37.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:37.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:37.008122+0000) 2024-03-06T10:30:38.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:38.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:38.008342+0000) 2024-03-06T10:30:39.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:39.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:39.008622+0000) 2024-03-06T10:30:40.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:40.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:40.008855+0000) 2024-03-06T10:30:41.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:41.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:41.009068+0000) 2024-03-06T10:30:42.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:42.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:42.009298+0000) 2024-03-06T10:30:43.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:43.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:43.009526+0000) 2024-03-06T10:30:44.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:44.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:44.009792+0000) 2024-03-06T10:30:45.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:45.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:45.010032+0000) 2024-03-06T10:30:46.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:46.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:46.010270+0000) 2024-03-06T10:30:47.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:47.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:47.010469+0000) 2024-03-06T10:30:48.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:48.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:48.010689+0000) 2024-03-06T10:30:49.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:49.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:49.010925+0000) 2024-03-06T10:30:50.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:50.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:50.011161+0000) 2024-03-06T10:30:51.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:51.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:51.011442+0000) 2024-03-06T10:30:52.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:52.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:52.011723+0000) 2024-03-06T10:30:53.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:53.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:53.012014+0000) 2024-03-06T10:30:54.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:54.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:54.012246+0000) 2024-03-06T10:30:55.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:55.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:55.012502+0000) 2024-03-06T10:30:56.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:56.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:56.012706+0000) 2024-03-06T10:30:57.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:57.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:57.012980+0000) 2024-03-06T10:30:58.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:58.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:58.013259+0000) 2024-03-06T10:30:59.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:30:59.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:30:59.013571+0000) 2024-03-06T10:31:00.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:00.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:00.013816+0000) 2024-03-06T10:31:01.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:01.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:01.014066+0000) 2024-03-06T10:31:02.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:02.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:02.014330+0000) 2024-03-06T10:31:02.290 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:31:02.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:31:03.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:03.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:03.014652+0000) 2024-03-06T10:31:04.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:04.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:04.014864+0000) 2024-03-06T10:31:05.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:05.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:05.015109+0000) 2024-03-06T10:31:06.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:06.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:06.015347+0000) 2024-03-06T10:31:07.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:07.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:07.015612+0000) 2024-03-06T10:31:08.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:08.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:08.015793+0000) 2024-03-06T10:31:09.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:09.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:09.016028+0000) 2024-03-06T10:31:10.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:10.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:10.016254+0000) 2024-03-06T10:31:11.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:11.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:11.016451+0000) 2024-03-06T10:31:12.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:12.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:12.016715+0000) 2024-03-06T10:31:13.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:13.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:13.016963+0000) 2024-03-06T10:31:14.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:14.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:14.017196+0000) 2024-03-06T10:31:15.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:15.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:15.017476+0000) 2024-03-06T10:31:16.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:16.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:16.017759+0000) 2024-03-06T10:31:17.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:17.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:17.018014+0000) 2024-03-06T10:31:18.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:18.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:18.018284+0000) 2024-03-06T10:31:19.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:19.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:19.018601+0000) 2024-03-06T10:31:20.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:20.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:20.018852+0000) 2024-03-06T10:31:21.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:21.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:21.019105+0000) 2024-03-06T10:31:22.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:22.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:22.019411+0000) 2024-03-06T10:31:23.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:23.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:23.019689+0000) 2024-03-06T10:31:24.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:24.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:24.019889+0000) 2024-03-06T10:31:25.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:25.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:25.020132+0000) 2024-03-06T10:31:26.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:26.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:26.020367+0000) 2024-03-06T10:31:27.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:27.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:27.020643+0000) 2024-03-06T10:31:28.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:28.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:28.020858+0000) 2024-03-06T10:31:29.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:29.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:29.021104+0000) 2024-03-06T10:31:30.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:30.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:30.021371+0000) 2024-03-06T10:31:31.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:31.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:31.021668+0000) 2024-03-06T10:31:32.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:32.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:32.021918+0000) 2024-03-06T10:31:32.495 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:31:32.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:31:33.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:33.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:33.022132+0000) 2024-03-06T10:31:34.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:34.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:34.022355+0000) 2024-03-06T10:31:35.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:35.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:35.022650+0000) 2024-03-06T10:31:36.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:36.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:36.022864+0000) 2024-03-06T10:31:37.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:37.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:37.023105+0000) 2024-03-06T10:31:38.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:38.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:38.023338+0000) 2024-03-06T10:31:39.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:39.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:39.023571+0000) 2024-03-06T10:31:40.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:40.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:40.023756+0000) 2024-03-06T10:31:41.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:41.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:41.023948+0000) 2024-03-06T10:31:42.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:42.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:42.024193+0000) 2024-03-06T10:31:43.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:43.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:43.024434+0000) 2024-03-06T10:31:44.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:44.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:44.024721+0000) 2024-03-06T10:31:45.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:45.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:45.024951+0000) 2024-03-06T10:31:46.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:46.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:46.025188+0000) 2024-03-06T10:31:47.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:47.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:47.025467+0000) 2024-03-06T10:31:48.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:48.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:48.025681+0000) 2024-03-06T10:31:49.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:49.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:49.025885+0000) 2024-03-06T10:31:50.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:50.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:50.026119+0000) 2024-03-06T10:31:51.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:51.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:51.026399+0000) 2024-03-06T10:31:52.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:52.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:52.026670+0000) 2024-03-06T10:31:53.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:53.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:53.026915+0000) 2024-03-06T10:31:54.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:54.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:54.027164+0000) 2024-03-06T10:31:55.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:55.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:55.027373+0000) 2024-03-06T10:31:56.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:56.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:56.027640+0000) 2024-03-06T10:31:57.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:57.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:57.027871+0000) 2024-03-06T10:31:58.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:58.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:58.028104+0000) 2024-03-06T10:31:59.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:31:59.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:31:59.028312+0000) 2024-03-06T10:32:00.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:00.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:00.028575+0000) 2024-03-06T10:32:01.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:01.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:01.028726+0000) 2024-03-06T10:32:02.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:02.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:02.028931+0000) 2024-03-06T10:32:02.691 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:32:02.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:32:03.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:03.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:03.029151+0000) 2024-03-06T10:32:04.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:04.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:04.029375+0000) 2024-03-06T10:32:05.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:05.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:05.029657+0000) 2024-03-06T10:32:06.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:06.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:06.029855+0000) 2024-03-06T10:32:07.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:07.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:07.030091+0000) 2024-03-06T10:32:08.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:08.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:08.030326+0000) 2024-03-06T10:32:09.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:09.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:09.030600+0000) 2024-03-06T10:32:10.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:10.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:10.030848+0000) 2024-03-06T10:32:11.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:11.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:11.031061+0000) 2024-03-06T10:32:12.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:12.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:12.031263+0000) 2024-03-06T10:32:13.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:13.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:13.031569+0000) 2024-03-06T10:32:14.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:14.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:14.031847+0000) 2024-03-06T10:32:15.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:15.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:15.032070+0000) 2024-03-06T10:32:16.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:16.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:16.032300+0000) 2024-03-06T10:32:17.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:17.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:17.032560+0000) 2024-03-06T10:32:18.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:18.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:18.032764+0000) 2024-03-06T10:32:19.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:19.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:19.032975+0000) 2024-03-06T10:32:20.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:20.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:20.033187+0000) 2024-03-06T10:32:21.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:21.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:21.033426+0000) 2024-03-06T10:32:22.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:22.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:22.033657+0000) 2024-03-06T10:32:23.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:23.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:23.033890+0000) 2024-03-06T10:32:24.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:24.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:24.034121+0000) 2024-03-06T10:32:25.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:25.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:25.034415+0000) 2024-03-06T10:32:26.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:26.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:26.034675+0000) 2024-03-06T10:32:27.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:27.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:27.034958+0000) 2024-03-06T10:32:28.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:28.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:28.035236+0000) 2024-03-06T10:32:29.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:29.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:29.035515+0000) 2024-03-06T10:32:30.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:30.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:30.035792+0000) 2024-03-06T10:32:31.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:31.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:31.036040+0000) 2024-03-06T10:32:32.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:32.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:32.036303+0000) 2024-03-06T10:32:32.905 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:32:32.913 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:32:33.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:33.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:33.036555+0000) 2024-03-06T10:32:34.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:34.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:34.036782+0000) 2024-03-06T10:32:35.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:35.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:35.037099+0000) 2024-03-06T10:32:36.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:36.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:36.037364+0000) 2024-03-06T10:32:37.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:37.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:37.037682+0000) 2024-03-06T10:32:38.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:38.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:38.037913+0000) 2024-03-06T10:32:39.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:39.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:39.038117+0000) 2024-03-06T10:32:40.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:40.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:40.038340+0000) 2024-03-06T10:32:41.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:41.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:41.038641+0000) 2024-03-06T10:32:42.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:42.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:42.038863+0000) 2024-03-06T10:32:43.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:43.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:43.039087+0000) 2024-03-06T10:32:44.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:44.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:44.039278+0000) 2024-03-06T10:32:45.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:45.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:45.039521+0000) 2024-03-06T10:32:46.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:46.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:46.039772+0000) 2024-03-06T10:32:47.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:47.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:47.040007+0000) 2024-03-06T10:32:48.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:48.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:48.040232+0000) 2024-03-06T10:32:49.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:49.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:49.040463+0000) 2024-03-06T10:32:50.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:50.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:50.040681+0000) 2024-03-06T10:32:51.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:51.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:51.040896+0000) 2024-03-06T10:32:52.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:52.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:52.041138+0000) 2024-03-06T10:32:53.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:53.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:53.041418+0000) 2024-03-06T10:32:54.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:54.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:54.041714+0000) 2024-03-06T10:32:55.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:55.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:55.041974+0000) 2024-03-06T10:32:56.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:56.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:56.042187+0000) 2024-03-06T10:32:57.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:57.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:57.042389+0000) 2024-03-06T10:32:58.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:58.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:58.042652+0000) 2024-03-06T10:32:59.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:32:59.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:32:59.042876+0000) 2024-03-06T10:33:00.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:00.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:00.043102+0000) 2024-03-06T10:33:01.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:01.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:01.043339+0000) 2024-03-06T10:33:02.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:02.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:02.043604+0000) 2024-03-06T10:33:03.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:03.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:03.043815+0000) 2024-03-06T10:33:03.118 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:33:03.126 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:33:04.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:04.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:04.044057+0000) 2024-03-06T10:33:05.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:05.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:05.044277+0000) 2024-03-06T10:33:06.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:06.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:06.044526+0000) 2024-03-06T10:33:07.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:07.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:07.044801+0000) 2024-03-06T10:33:08.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:08.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:08.045029+0000) 2024-03-06T10:33:09.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:09.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:09.045261+0000) 2024-03-06T10:33:10.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:10.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:10.045460+0000) 2024-03-06T10:33:11.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:11.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:11.045675+0000) 2024-03-06T10:33:12.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:12.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:12.045887+0000) 2024-03-06T10:33:13.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:13.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:13.046128+0000) 2024-03-06T10:33:14.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:14.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:14.046353+0000) 2024-03-06T10:33:15.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:15.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:15.046635+0000) 2024-03-06T10:33:16.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:16.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:16.046840+0000) 2024-03-06T10:33:17.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:17.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:17.047068+0000) 2024-03-06T10:33:18.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:18.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:18.047287+0000) 2024-03-06T10:33:19.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:19.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:19.047512+0000) 2024-03-06T10:33:20.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:20.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:20.047772+0000) 2024-03-06T10:33:21.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:21.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:21.047964+0000) 2024-03-06T10:33:22.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:22.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:22.048188+0000) 2024-03-06T10:33:23.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:23.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:23.048416+0000) 2024-03-06T10:33:24.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:24.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:24.048649+0000) 2024-03-06T10:33:25.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:25.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:25.048845+0000) 2024-03-06T10:33:26.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:26.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:26.049051+0000) 2024-03-06T10:33:27.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:27.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:27.049277+0000) 2024-03-06T10:33:28.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:28.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:28.049586+0000) 2024-03-06T10:33:29.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:29.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:29.049815+0000) 2024-03-06T10:33:30.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:30.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:30.050078+0000) 2024-03-06T10:33:31.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:31.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:31.050332+0000) 2024-03-06T10:33:32.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:32.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:32.050525+0000) 2024-03-06T10:33:33.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:33.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:33.050792+0000) 2024-03-06T10:33:33.339 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:33:33.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:33:34.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:34.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:34.051022+0000) 2024-03-06T10:33:35.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:35.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:35.051251+0000) 2024-03-06T10:33:36.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:36.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:36.051570+0000) 2024-03-06T10:33:37.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:37.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:37.051777+0000) 2024-03-06T10:33:38.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:38.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:38.052004+0000) 2024-03-06T10:33:39.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:39.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:39.052234+0000) 2024-03-06T10:33:40.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:40.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:40.052456+0000) 2024-03-06T10:33:41.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:41.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:41.052699+0000) 2024-03-06T10:33:42.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:42.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:42.052933+0000) 2024-03-06T10:33:43.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:43.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:43.053158+0000) 2024-03-06T10:33:44.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:44.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:44.053390+0000) 2024-03-06T10:33:45.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:45.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:45.053674+0000) 2024-03-06T10:33:46.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:46.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:46.053932+0000) 2024-03-06T10:33:47.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:47.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:47.054171+0000) 2024-03-06T10:33:48.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:48.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:48.054397+0000) 2024-03-06T10:33:49.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:49.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:49.054735+0000) 2024-03-06T10:33:50.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:50.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:50.054953+0000) 2024-03-06T10:33:51.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:51.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:51.055184+0000) 2024-03-06T10:33:52.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:52.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:52.055388+0000) 2024-03-06T10:33:53.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:53.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:53.055650+0000) 2024-03-06T10:33:54.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:54.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:54.055838+0000) 2024-03-06T10:33:55.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:55.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:55.056046+0000) 2024-03-06T10:33:56.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:56.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:56.056245+0000) 2024-03-06T10:33:57.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:57.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:57.056450+0000) 2024-03-06T10:33:58.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:58.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:58.056661+0000) 2024-03-06T10:33:59.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:33:59.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:33:59.056907+0000) 2024-03-06T10:34:00.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:00.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:00.057128+0000) 2024-03-06T10:34:01.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:01.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:01.057421+0000) 2024-03-06T10:34:02.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:02.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:02.057691+0000) 2024-03-06T10:34:03.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:03.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:03.057969+0000) 2024-03-06T10:34:03.560 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:34:03.568 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:34:04.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:04.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:04.058257+0000) 2024-03-06T10:34:05.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:05.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:05.058586+0000) 2024-03-06T10:34:06.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:06.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:06.058867+0000) 2024-03-06T10:34:07.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:07.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:07.059139+0000) 2024-03-06T10:34:08.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:08.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:08.059407+0000) 2024-03-06T10:34:09.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:09.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:09.059703+0000) 2024-03-06T10:34:10.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:10.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:10.059986+0000) 2024-03-06T10:34:11.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:11.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:11.060277+0000) 2024-03-06T10:34:12.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:12.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:12.060604+0000) 2024-03-06T10:34:13.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:13.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:13.060823+0000) 2024-03-06T10:34:14.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:14.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:14.061032+0000) 2024-03-06T10:34:15.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:15.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:15.061287+0000) 2024-03-06T10:34:16.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:16.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:16.061524+0000) 2024-03-06T10:34:17.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:17.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:17.061797+0000) 2024-03-06T10:34:18.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:18.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:18.062013+0000) 2024-03-06T10:34:19.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:19.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:19.062291+0000) 2024-03-06T10:34:20.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:20.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:20.062510+0000) 2024-03-06T10:34:21.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:21.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:21.062780+0000) 2024-03-06T10:34:22.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:22.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:22.063023+0000) 2024-03-06T10:34:23.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:23.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:23.063211+0000) 2024-03-06T10:34:24.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:24.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:24.063453+0000) 2024-03-06T10:34:25.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:25.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:25.063685+0000) 2024-03-06T10:34:26.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:26.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:26.063933+0000) 2024-03-06T10:34:27.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:27.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:27.064173+0000) 2024-03-06T10:34:28.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:28.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:28.064419+0000) 2024-03-06T10:34:29.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:29.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:29.064668+0000) 2024-03-06T10:34:30.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:30.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:30.064882+0000) 2024-03-06T10:34:31.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:31.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:31.065127+0000) 2024-03-06T10:34:32.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:32.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:32.065341+0000) 2024-03-06T10:34:33.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:33.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:33.065605+0000) 2024-03-06T10:34:33.773 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:34:33.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:34:34.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:34.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:34.065885+0000) 2024-03-06T10:34:35.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:35.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:35.066149+0000) 2024-03-06T10:34:36.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:36.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:36.066428+0000) 2024-03-06T10:34:37.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:37.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:37.066715+0000) 2024-03-06T10:34:38.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:38.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:38.066953+0000) 2024-03-06T10:34:39.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:39.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:39.067179+0000) 2024-03-06T10:34:40.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:40.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:40.067395+0000) 2024-03-06T10:34:41.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:41.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:41.067687+0000) 2024-03-06T10:34:42.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:42.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:42.067930+0000) 2024-03-06T10:34:43.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:43.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:43.068162+0000) 2024-03-06T10:34:44.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:44.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:44.068374+0000) 2024-03-06T10:34:45.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:45.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:45.068651+0000) 2024-03-06T10:34:46.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:46.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:46.068855+0000) 2024-03-06T10:34:47.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:47.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:47.069090+0000) 2024-03-06T10:34:48.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:48.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:48.069327+0000) 2024-03-06T10:34:49.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:49.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:49.069576+0000) 2024-03-06T10:34:50.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:50.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:50.069786+0000) 2024-03-06T10:34:51.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:51.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:51.070039+0000) 2024-03-06T10:34:52.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:52.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:52.070320+0000) 2024-03-06T10:34:53.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:53.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:53.070613+0000) 2024-03-06T10:34:54.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:54.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:54.070823+0000) 2024-03-06T10:34:55.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:55.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:55.071048+0000) 2024-03-06T10:34:56.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:56.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:56.071247+0000) 2024-03-06T10:34:57.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:57.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:57.071482+0000) 2024-03-06T10:34:58.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:58.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:58.071706+0000) 2024-03-06T10:34:59.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:34:59.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:34:59.071939+0000) 2024-03-06T10:35:00.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:00.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:00.072166+0000) 2024-03-06T10:35:01.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:01.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:01.072405+0000) 2024-03-06T10:35:02.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:02.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:02.072639+0000) 2024-03-06T10:35:03.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:03.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:03.072851+0000) 2024-03-06T10:35:03.986 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:35:03.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:35:04.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:04.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:04.073037+0000) 2024-03-06T10:35:05.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:05.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:05.073270+0000) 2024-03-06T10:35:06.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:06.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:06.073490+0000) 2024-03-06T10:35:07.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:07.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:07.073750+0000) 2024-03-06T10:35:08.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:08.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:08.073945+0000) 2024-03-06T10:35:09.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:09.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:09.074147+0000) 2024-03-06T10:35:10.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:10.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:10.074327+0000) 2024-03-06T10:35:11.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:11.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:11.074573+0000) 2024-03-06T10:35:12.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:12.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:12.074823+0000) 2024-03-06T10:35:13.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:13.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:13.075005+0000) 2024-03-06T10:35:14.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:14.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:14.075227+0000) 2024-03-06T10:35:15.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:15.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:15.075466+0000) 2024-03-06T10:35:16.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:16.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:16.075701+0000) 2024-03-06T10:35:17.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:17.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:17.075914+0000) 2024-03-06T10:35:18.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:18.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:18.076148+0000) 2024-03-06T10:35:19.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:19.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:19.076379+0000) 2024-03-06T10:35:20.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:20.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:20.076646+0000) 2024-03-06T10:35:21.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:21.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:21.076857+0000) 2024-03-06T10:35:22.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:22.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:22.077108+0000) 2024-03-06T10:35:23.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:23.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:23.077292+0000) 2024-03-06T10:35:24.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:24.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:24.077567+0000) 2024-03-06T10:35:25.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:25.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:25.077801+0000) 2024-03-06T10:35:26.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:26.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:26.078043+0000) 2024-03-06T10:35:27.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:27.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:27.078298+0000) 2024-03-06T10:35:28.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:28.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:28.078487+0000) 2024-03-06T10:35:29.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:29.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:29.078660+0000) 2024-03-06T10:35:30.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:30.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:30.078848+0000) 2024-03-06T10:35:31.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:31.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:31.079058+0000) 2024-03-06T10:35:32.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:32.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:32.079250+0000) 2024-03-06T10:35:33.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:33.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:33.079429+0000) 2024-03-06T10:35:34.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:34.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:34.079665+0000) 2024-03-06T10:35:34.190 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:35:34.198 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:35:35.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:35.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:35.079880+0000) 2024-03-06T10:35:36.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:36.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:36.080071+0000) 2024-03-06T10:35:37.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:37.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:37.080297+0000) 2024-03-06T10:35:38.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:38.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:38.080618+0000) 2024-03-06T10:35:39.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:39.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:39.080907+0000) 2024-03-06T10:35:40.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:40.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:40.081193+0000) 2024-03-06T10:35:41.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:41.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:41.081465+0000) 2024-03-06T10:35:42.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:42.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:42.081740+0000) 2024-03-06T10:35:43.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:43.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:43.082021+0000) 2024-03-06T10:35:44.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:44.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:44.082298+0000) 2024-03-06T10:35:45.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:45.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:45.082637+0000) 2024-03-06T10:35:46.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:46.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:46.082914+0000) 2024-03-06T10:35:47.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:47.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:47.083190+0000) 2024-03-06T10:35:48.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:48.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:48.083472+0000) 2024-03-06T10:35:49.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:49.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:49.083755+0000) 2024-03-06T10:35:50.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:50.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:50.084034+0000) 2024-03-06T10:35:51.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:51.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:51.084314+0000) 2024-03-06T10:35:52.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:52.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:52.084506+0000) 2024-03-06T10:35:53.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:53.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:53.084697+0000) 2024-03-06T10:35:54.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:54.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:54.084966+0000) 2024-03-06T10:35:55.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:55.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:55.085260+0000) 2024-03-06T10:35:56.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:56.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:56.085505+0000) 2024-03-06T10:35:57.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:57.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:57.085780+0000) 2024-03-06T10:35:58.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:58.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:58.086016+0000) 2024-03-06T10:35:59.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:35:59.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:35:59.086252+0000) 2024-03-06T10:36:00.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:00.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:00.086423+0000) 2024-03-06T10:36:01.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:01.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:01.086697+0000) 2024-03-06T10:36:02.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:02.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:02.086907+0000) 2024-03-06T10:36:03.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:03.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:03.087120+0000) 2024-03-06T10:36:04.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:04.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:04.087366+0000) 2024-03-06T10:36:04.363 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:36:04.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:36:05.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:05.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:05.087651+0000) 2024-03-06T10:36:06.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:06.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:06.087856+0000) 2024-03-06T10:36:07.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:07.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:07.088098+0000) 2024-03-06T10:36:08.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:08.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:08.088339+0000) 2024-03-06T10:36:09.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:09.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:09.088623+0000) 2024-03-06T10:36:10.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:10.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:10.088800+0000) 2024-03-06T10:36:11.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:11.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:11.089029+0000) 2024-03-06T10:36:12.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:12.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:12.089267+0000) 2024-03-06T10:36:13.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:13.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:13.089448+0000) 2024-03-06T10:36:14.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:14.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:14.089691+0000) 2024-03-06T10:36:15.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:15.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:15.089888+0000) 2024-03-06T10:36:16.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:16.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:16.090109+0000) 2024-03-06T10:36:17.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:17.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:17.090350+0000) 2024-03-06T10:36:18.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:18.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:18.090609+0000) 2024-03-06T10:36:19.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:19.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:19.090820+0000) 2024-03-06T10:36:20.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:20.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:20.091056+0000) 2024-03-06T10:36:21.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:21.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:21.091254+0000) 2024-03-06T10:36:22.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:22.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:22.091501+0000) 2024-03-06T10:36:23.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:23.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:23.091768+0000) 2024-03-06T10:36:24.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:24.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:24.091970+0000) 2024-03-06T10:36:25.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:25.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:25.092204+0000) 2024-03-06T10:36:26.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:26.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:26.092390+0000) 2024-03-06T10:36:27.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:27.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:27.092662+0000) 2024-03-06T10:36:28.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:28.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:28.092855+0000) 2024-03-06T10:36:29.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:29.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:29.093094+0000) 2024-03-06T10:36:30.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:30.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:30.093314+0000) 2024-03-06T10:36:31.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:31.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:31.093522+0000) 2024-03-06T10:36:32.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:32.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:32.093803+0000) 2024-03-06T10:36:33.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:33.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:33.094040+0000) 2024-03-06T10:36:34.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:34.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:34.094261+0000) 2024-03-06T10:36:34.537 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:36:34.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:36:35.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:35.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:35.094461+0000) 2024-03-06T10:36:36.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:36.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:36.094704+0000) 2024-03-06T10:36:37.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:37.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:37.094927+0000) 2024-03-06T10:36:38.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:38.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:38.095130+0000) 2024-03-06T10:36:39.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:39.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:39.095363+0000) 2024-03-06T10:36:40.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:40.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:40.095586+0000) 2024-03-06T10:36:41.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:41.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:41.095828+0000) 2024-03-06T10:36:42.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:42.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:42.096060+0000) 2024-03-06T10:36:43.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:43.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:43.096304+0000) 2024-03-06T10:36:44.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:44.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:44.096504+0000) 2024-03-06T10:36:45.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:45.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:45.096825+0000) 2024-03-06T10:36:46.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:46.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:46.097014+0000) 2024-03-06T10:36:47.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:47.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:47.097249+0000) 2024-03-06T10:36:48.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:48.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:48.097470+0000) 2024-03-06T10:36:49.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:49.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:49.097700+0000) 2024-03-06T10:36:50.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:50.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:50.097892+0000) 2024-03-06T10:36:51.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:51.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:51.098094+0000) 2024-03-06T10:36:52.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:52.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:52.098322+0000) 2024-03-06T10:36:53.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:53.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:53.098572+0000) 2024-03-06T10:36:54.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:54.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:54.098770+0000) 2024-03-06T10:36:55.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:55.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:55.098992+0000) 2024-03-06T10:36:56.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:56.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:56.099234+0000) 2024-03-06T10:36:57.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:57.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:57.099495+0000) 2024-03-06T10:36:58.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:58.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:58.099695+0000) 2024-03-06T10:36:59.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:36:59.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:36:59.099885+0000) 2024-03-06T10:37:00.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:00.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:00.100076+0000) 2024-03-06T10:37:01.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:01.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:01.100309+0000) 2024-03-06T10:37:02.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:02.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:02.100565+0000) 2024-03-06T10:37:03.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:03.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:03.100806+0000) 2024-03-06T10:37:04.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:04.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:04.101003+0000) 2024-03-06T10:37:04.747 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:37:04.755 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:37:05.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:05.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:05.101259+0000) 2024-03-06T10:37:06.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:06.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:06.101486+0000) 2024-03-06T10:37:07.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:07.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:07.101701+0000) 2024-03-06T10:37:08.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:08.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:08.101896+0000) 2024-03-06T10:37:09.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:09.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:09.102124+0000) 2024-03-06T10:37:10.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:10.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:10.102306+0000) 2024-03-06T10:37:11.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:11.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:11.102631+0000) 2024-03-06T10:37:12.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:12.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:12.102810+0000) 2024-03-06T10:37:13.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:13.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:13.103003+0000) 2024-03-06T10:37:14.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:14.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:14.103185+0000) 2024-03-06T10:37:15.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:15.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:15.103437+0000) 2024-03-06T10:37:16.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:16.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:16.103682+0000) 2024-03-06T10:37:17.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:17.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:17.103862+0000) 2024-03-06T10:37:18.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:18.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:18.104052+0000) 2024-03-06T10:37:19.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:19.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:19.104271+0000) 2024-03-06T10:37:20.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:20.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:20.104493+0000) 2024-03-06T10:37:21.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:21.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:21.104689+0000) 2024-03-06T10:37:22.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:22.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:22.104889+0000) 2024-03-06T10:37:23.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:23.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:23.105084+0000) 2024-03-06T10:37:24.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:24.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:24.105307+0000) 2024-03-06T10:37:25.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:25.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:25.105501+0000) 2024-03-06T10:37:26.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:26.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:26.105742+0000) 2024-03-06T10:37:27.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:27.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:27.105971+0000) 2024-03-06T10:37:28.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:28.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:28.106195+0000) 2024-03-06T10:37:29.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:29.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:29.106396+0000) 2024-03-06T10:37:30.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:30.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:30.106662+0000) 2024-03-06T10:37:31.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:31.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:31.106889+0000) 2024-03-06T10:37:32.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:32.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:32.107110+0000) 2024-03-06T10:37:33.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:33.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:33.107335+0000) 2024-03-06T10:37:34.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:34.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:34.107596+0000) 2024-03-06T10:37:34.986 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:37:34.995 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:37:35.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:35.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:35.107834+0000) 2024-03-06T10:37:36.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:36.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:36.108020+0000) 2024-03-06T10:37:37.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:37.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:37.108258+0000) 2024-03-06T10:37:38.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:38.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:38.108443+0000) 2024-03-06T10:37:39.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:39.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:39.108724+0000) 2024-03-06T10:37:40.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:40.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:40.108920+0000) 2024-03-06T10:37:41.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:41.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:41.109149+0000) 2024-03-06T10:37:42.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:42.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:42.109386+0000) 2024-03-06T10:37:43.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:43.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:43.109645+0000) 2024-03-06T10:37:44.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:44.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:44.109868+0000) 2024-03-06T10:37:45.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:45.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:45.110070+0000) 2024-03-06T10:37:46.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:46.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:46.110290+0000) 2024-03-06T10:37:47.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:47.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:47.110492+0000) 2024-03-06T10:37:48.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:48.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:48.110736+0000) 2024-03-06T10:37:49.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:49.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:49.110976+0000) 2024-03-06T10:37:50.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:50.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:50.111196+0000) 2024-03-06T10:37:51.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:51.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:51.111410+0000) 2024-03-06T10:37:52.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:52.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:52.111656+0000) 2024-03-06T10:37:53.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:53.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:53.111898+0000) 2024-03-06T10:37:54.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:54.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:54.112098+0000) 2024-03-06T10:37:55.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:55.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:55.112359+0000) 2024-03-06T10:37:56.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:56.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:56.112630+0000) 2024-03-06T10:37:57.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:57.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:57.112851+0000) 2024-03-06T10:37:58.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:58.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:58.113077+0000) 2024-03-06T10:37:59.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:37:59.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:37:59.113299+0000) 2024-03-06T10:38:00.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:00.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:00.113485+0000) 2024-03-06T10:38:01.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:01.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:01.113738+0000) 2024-03-06T10:38:02.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:02.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:02.113980+0000) 2024-03-06T10:38:03.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:03.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:03.114219+0000) 2024-03-06T10:38:04.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:04.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:04.114457+0000) 2024-03-06T10:38:05.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:05.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:05.114696+0000) 2024-03-06T10:38:05.137 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:38:05.144 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:38:06.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:06.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:06.114883+0000) 2024-03-06T10:38:07.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:07.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:07.115112+0000) 2024-03-06T10:38:08.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:08.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:08.115336+0000) 2024-03-06T10:38:09.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:09.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:09.115599+0000) 2024-03-06T10:38:10.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:10.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:10.115774+0000) 2024-03-06T10:38:11.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:11.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:11.115983+0000) 2024-03-06T10:38:12.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:12.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:12.116226+0000) 2024-03-06T10:38:13.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:13.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:13.116467+0000) 2024-03-06T10:38:14.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:14.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:14.116684+0000) 2024-03-06T10:38:15.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:15.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:15.116905+0000) 2024-03-06T10:38:16.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:16.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:16.117099+0000) 2024-03-06T10:38:17.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:17.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:17.117335+0000) 2024-03-06T10:38:18.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:18.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:18.117598+0000) 2024-03-06T10:38:19.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:19.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:19.117825+0000) 2024-03-06T10:38:20.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:20.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:20.118062+0000) 2024-03-06T10:38:21.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:21.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:21.118238+0000) 2024-03-06T10:38:22.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:22.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:22.118422+0000) 2024-03-06T10:38:23.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:23.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:23.118660+0000) 2024-03-06T10:38:24.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:24.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:24.118863+0000) 2024-03-06T10:38:25.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:25.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:25.119079+0000) 2024-03-06T10:38:26.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:26.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:26.119322+0000) 2024-03-06T10:38:27.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:27.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:27.119648+0000) 2024-03-06T10:38:28.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:28.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:28.119861+0000) 2024-03-06T10:38:29.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:29.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:29.120070+0000) 2024-03-06T10:38:30.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:30.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:30.120300+0000) 2024-03-06T10:38:31.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:31.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:31.120553+0000) 2024-03-06T10:38:32.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:32.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:32.120788+0000) 2024-03-06T10:38:33.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:33.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:33.120984+0000) 2024-03-06T10:38:34.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:34.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:34.121171+0000) 2024-03-06T10:38:35.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:35.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:35.121414+0000) 2024-03-06T10:38:35.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:38:35.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:38:36.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:36.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:36.121667+0000) 2024-03-06T10:38:37.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:37.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:37.121883+0000) 2024-03-06T10:38:38.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:38.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:38.122074+0000) 2024-03-06T10:38:39.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:39.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:39.122298+0000) 2024-03-06T10:38:40.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:40.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:40.122492+0000) 2024-03-06T10:38:41.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:41.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:41.122719+0000) 2024-03-06T10:38:42.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:42.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:42.122958+0000) 2024-03-06T10:38:43.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:43.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:43.123195+0000) 2024-03-06T10:38:44.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:44.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:44.123420+0000) 2024-03-06T10:38:45.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:45.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:45.123664+0000) 2024-03-06T10:38:46.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:46.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:46.123897+0000) 2024-03-06T10:38:47.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:47.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:47.124099+0000) 2024-03-06T10:38:48.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:48.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:48.124341+0000) 2024-03-06T10:38:49.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:49.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:49.124606+0000) 2024-03-06T10:38:50.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:50.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:50.124842+0000) 2024-03-06T10:38:51.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:51.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:51.125074+0000) 2024-03-06T10:38:52.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:52.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:52.125286+0000) 2024-03-06T10:38:53.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:53.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:53.125552+0000) 2024-03-06T10:38:54.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:54.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:54.125786+0000) 2024-03-06T10:38:55.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:55.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:55.125976+0000) 2024-03-06T10:38:56.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:56.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:56.126166+0000) 2024-03-06T10:38:57.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:57.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:57.126402+0000) 2024-03-06T10:38:58.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:58.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:58.126639+0000) 2024-03-06T10:38:59.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:38:59.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:38:59.126855+0000) 2024-03-06T10:39:00.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:00.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:00.127064+0000) 2024-03-06T10:39:01.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:01.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:01.127274+0000) 2024-03-06T10:39:02.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:02.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:02.127513+0000) 2024-03-06T10:39:03.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:03.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:03.127739+0000) 2024-03-06T10:39:04.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:04.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:04.127969+0000) 2024-03-06T10:39:05.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:05.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:05.128225+0000) 2024-03-06T10:39:05.574 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:39:05.582 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:39:06.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:06.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:06.128474+0000) 2024-03-06T10:39:07.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:07.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:07.128674+0000) 2024-03-06T10:39:08.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:08.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:08.128882+0000) 2024-03-06T10:39:09.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:09.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:09.129089+0000) 2024-03-06T10:39:10.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:10.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:10.129293+0000) 2024-03-06T10:39:11.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:11.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:11.129526+0000) 2024-03-06T10:39:12.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:12.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:12.129806+0000) 2024-03-06T10:39:13.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:13.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:13.130029+0000) 2024-03-06T10:39:14.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:14.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:14.130258+0000) 2024-03-06T10:39:15.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:15.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:15.130474+0000) 2024-03-06T10:39:16.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:16.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:16.130723+0000) 2024-03-06T10:39:17.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:17.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:17.130930+0000) 2024-03-06T10:39:18.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:18.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:18.131154+0000) 2024-03-06T10:39:19.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:19.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:19.131384+0000) 2024-03-06T10:39:20.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:20.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:20.131708+0000) 2024-03-06T10:39:21.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:21.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:21.131859+0000) 2024-03-06T10:39:22.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:22.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:22.132075+0000) 2024-03-06T10:39:23.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:23.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:23.132279+0000) 2024-03-06T10:39:24.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:24.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:24.132512+0000) 2024-03-06T10:39:25.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:25.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:25.132837+0000) 2024-03-06T10:39:26.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:26.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:26.133075+0000) 2024-03-06T10:39:27.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:27.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:27.133328+0000) 2024-03-06T10:39:28.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:28.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:28.133612+0000) 2024-03-06T10:39:29.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:29.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:29.133858+0000) 2024-03-06T10:39:30.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:30.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:30.134070+0000) 2024-03-06T10:39:31.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:31.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:31.134323+0000) 2024-03-06T10:39:32.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:32.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:32.134615+0000) 2024-03-06T10:39:33.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:33.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:33.134825+0000) 2024-03-06T10:39:34.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:34.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:34.135068+0000) 2024-03-06T10:39:35.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:35.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:35.135427+0000) 2024-03-06T10:39:35.789 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:39:35.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:39:36.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:36.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:36.135690+0000) 2024-03-06T10:39:37.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:37.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:37.135949+0000) 2024-03-06T10:39:38.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:38.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:38.136170+0000) 2024-03-06T10:39:39.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:39.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:39.136413+0000) 2024-03-06T10:39:40.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:40.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:40.136672+0000) 2024-03-06T10:39:41.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:41.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:41.136918+0000) 2024-03-06T10:39:42.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:42.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:42.137158+0000) 2024-03-06T10:39:43.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:43.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:43.137387+0000) 2024-03-06T10:39:44.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:44.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:44.137608+0000) 2024-03-06T10:39:45.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:45.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:45.137831+0000) 2024-03-06T10:39:46.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:46.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:46.138052+0000) 2024-03-06T10:39:47.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:47.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:47.138324+0000) 2024-03-06T10:39:48.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:48.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:48.138458+0000) 2024-03-06T10:39:49.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:49.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:49.138700+0000) 2024-03-06T10:39:50.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:50.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:50.138931+0000) 2024-03-06T10:39:51.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:51.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:51.139169+0000) 2024-03-06T10:39:52.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:52.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:52.139392+0000) 2024-03-06T10:39:53.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:53.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:53.139655+0000) 2024-03-06T10:39:54.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:54.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:54.139841+0000) 2024-03-06T10:39:55.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:55.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:55.140057+0000) 2024-03-06T10:39:56.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:56.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:56.140258+0000) 2024-03-06T10:39:57.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:57.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:57.140494+0000) 2024-03-06T10:39:58.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:58.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:58.140761+0000) 2024-03-06T10:39:59.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:39:59.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:39:59.141002+0000) 2024-03-06T10:40:00.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:00.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:00.141243+0000) 2024-03-06T10:40:01.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:01.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:01.141482+0000) 2024-03-06T10:40:02.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:02.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:02.141747+0000) 2024-03-06T10:40:03.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:03.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:03.142042+0000) 2024-03-06T10:40:04.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:04.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:04.142311+0000) 2024-03-06T10:40:05.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:05.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:05.142624+0000) 2024-03-06T10:40:06.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:40:06.010 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:40:06.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:06.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:06.142852+0000) 2024-03-06T10:40:07.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:07.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:07.143104+0000) 2024-03-06T10:40:08.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:08.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:08.143309+0000) 2024-03-06T10:40:09.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:09.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:09.143523+0000) 2024-03-06T10:40:10.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:10.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:10.143787+0000) 2024-03-06T10:40:11.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:11.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:11.143982+0000) 2024-03-06T10:40:12.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:12.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:12.144246+0000) 2024-03-06T10:40:13.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:13.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:13.144502+0000) 2024-03-06T10:40:14.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:14.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:14.144778+0000) 2024-03-06T10:40:15.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:15.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:15.144981+0000) 2024-03-06T10:40:16.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:16.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:16.145201+0000) 2024-03-06T10:40:17.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:17.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:17.145448+0000) 2024-03-06T10:40:18.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:18.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:18.145657+0000) 2024-03-06T10:40:19.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:19.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:19.145907+0000) 2024-03-06T10:40:20.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:20.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:20.146125+0000) 2024-03-06T10:40:21.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:21.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:21.146368+0000) 2024-03-06T10:40:22.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:22.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:22.146615+0000) 2024-03-06T10:40:23.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:23.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:23.146870+0000) 2024-03-06T10:40:24.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:24.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:24.147086+0000) 2024-03-06T10:40:25.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:25.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:25.147338+0000) 2024-03-06T10:40:26.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:26.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:26.147566+0000) 2024-03-06T10:40:27.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:27.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:27.147813+0000) 2024-03-06T10:40:28.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:28.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:28.148040+0000) 2024-03-06T10:40:29.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:29.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:29.148275+0000) 2024-03-06T10:40:30.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:30.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:30.148500+0000) 2024-03-06T10:40:31.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:31.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:31.148739+0000) 2024-03-06T10:40:32.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:32.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:32.148976+0000) 2024-03-06T10:40:33.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:33.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:33.149215+0000) 2024-03-06T10:40:34.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:34.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:34.149455+0000) 2024-03-06T10:40:35.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:35.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:35.149744+0000) 2024-03-06T10:40:36.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:36.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:36.150002+0000) 2024-03-06T10:40:36.216 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:40:36.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:40:37.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:37.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:37.150189+0000) 2024-03-06T10:40:38.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:38.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:38.150410+0000) 2024-03-06T10:40:39.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:39.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:39.150625+0000) 2024-03-06T10:40:40.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:40.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:40.150878+0000) 2024-03-06T10:40:41.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:41.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:41.151089+0000) 2024-03-06T10:40:42.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:42.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:42.151337+0000) 2024-03-06T10:40:43.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:43.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:43.151627+0000) 2024-03-06T10:40:44.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:44.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:44.151835+0000) 2024-03-06T10:40:45.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:45.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:45.152072+0000) 2024-03-06T10:40:46.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:46.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:46.152297+0000) 2024-03-06T10:40:47.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:47.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:47.152528+0000) 2024-03-06T10:40:48.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:48.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:48.152763+0000) 2024-03-06T10:40:49.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:49.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:49.153003+0000) 2024-03-06T10:40:50.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:50.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:50.153246+0000) 2024-03-06T10:40:51.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:51.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:51.153490+0000) 2024-03-06T10:40:52.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:52.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:52.153710+0000) 2024-03-06T10:40:53.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:53.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:53.153961+0000) 2024-03-06T10:40:54.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:54.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:54.154193+0000) 2024-03-06T10:40:55.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:55.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:55.154458+0000) 2024-03-06T10:40:56.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:56.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:56.154759+0000) 2024-03-06T10:40:57.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:57.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:57.154995+0000) 2024-03-06T10:40:58.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:58.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:58.155218+0000) 2024-03-06T10:40:59.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:40:59.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:40:59.155446+0000) 2024-03-06T10:41:00.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:00.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:00.155638+0000) 2024-03-06T10:41:01.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:01.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:01.155829+0000) 2024-03-06T10:41:02.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:02.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:02.156062+0000) 2024-03-06T10:41:03.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:03.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:03.156291+0000) 2024-03-06T10:41:04.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:04.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:04.156519+0000) 2024-03-06T10:41:05.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:05.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:05.156792+0000) 2024-03-06T10:41:06.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:06.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:06.156939+0000) 2024-03-06T10:41:06.370 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:41:06.378 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:41:07.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:07.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:07.157171+0000) 2024-03-06T10:41:08.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:08.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:08.157358+0000) 2024-03-06T10:41:09.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:09.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:09.157564+0000) 2024-03-06T10:41:10.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:10.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:10.157756+0000) 2024-03-06T10:41:11.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:11.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:11.157982+0000) 2024-03-06T10:41:12.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:12.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:12.158162+0000) 2024-03-06T10:41:13.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:13.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:13.158370+0000) 2024-03-06T10:41:14.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:14.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:14.158653+0000) 2024-03-06T10:41:15.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:15.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:15.158887+0000) 2024-03-06T10:41:16.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:16.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:16.159094+0000) 2024-03-06T10:41:17.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:17.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:17.159339+0000) 2024-03-06T10:41:18.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:18.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:18.159619+0000) 2024-03-06T10:41:19.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:19.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:19.159848+0000) 2024-03-06T10:41:20.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:20.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:20.160096+0000) 2024-03-06T10:41:21.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:21.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:21.160287+0000) 2024-03-06T10:41:22.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:22.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:22.160513+0000) 2024-03-06T10:41:23.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:23.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:23.160778+0000) 2024-03-06T10:41:24.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:24.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:24.161011+0000) 2024-03-06T10:41:25.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:25.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:25.161248+0000) 2024-03-06T10:41:26.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:26.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:26.161472+0000) 2024-03-06T10:41:27.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:27.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:27.161684+0000) 2024-03-06T10:41:28.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:28.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:28.161878+0000) 2024-03-06T10:41:29.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:29.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:29.162074+0000) 2024-03-06T10:41:30.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:30.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:30.162303+0000) 2024-03-06T10:41:31.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:31.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:31.162529+0000) 2024-03-06T10:41:32.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:32.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:32.162784+0000) 2024-03-06T10:41:33.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:33.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:33.162974+0000) 2024-03-06T10:41:34.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:34.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:34.163168+0000) 2024-03-06T10:41:35.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:35.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:35.163450+0000) 2024-03-06T10:41:36.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:36.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:36.163690+0000) 2024-03-06T10:41:36.507 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:41:36.516 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:41:37.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:37.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:37.163882+0000) 2024-03-06T10:41:38.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:38.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:38.164081+0000) 2024-03-06T10:41:39.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:39.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:39.164327+0000) 2024-03-06T10:41:40.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:40.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:40.164460+0000) 2024-03-06T10:41:41.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:41.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:41.164696+0000) 2024-03-06T10:41:42.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:42.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:42.164920+0000) 2024-03-06T10:41:43.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:43.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:43.165150+0000) 2024-03-06T10:41:44.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:44.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:44.165379+0000) 2024-03-06T10:41:45.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:45.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:45.165615+0000) 2024-03-06T10:41:46.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:46.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:46.165827+0000) 2024-03-06T10:41:47.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:47.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:47.166051+0000) 2024-03-06T10:41:48.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:48.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:48.166294+0000) 2024-03-06T10:41:49.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:49.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:49.166476+0000) 2024-03-06T10:41:50.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:50.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:50.166672+0000) 2024-03-06T10:41:51.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:51.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:51.166910+0000) 2024-03-06T10:41:52.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:52.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:52.167142+0000) 2024-03-06T10:41:53.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:53.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:53.167369+0000) 2024-03-06T10:41:54.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:54.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:54.167641+0000) 2024-03-06T10:41:55.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:55.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:55.167882+0000) 2024-03-06T10:41:56.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:56.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:56.168124+0000) 2024-03-06T10:41:57.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:57.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:57.168354+0000) 2024-03-06T10:41:58.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:58.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:58.168635+0000) 2024-03-06T10:41:59.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:41:59.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:41:59.168865+0000) 2024-03-06T10:42:00.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:00.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:00.169091+0000) 2024-03-06T10:42:01.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:01.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:01.169282+0000) 2024-03-06T10:42:02.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:02.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:02.169462+0000) 2024-03-06T10:42:03.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:03.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:03.169663+0000) 2024-03-06T10:42:04.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:04.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:04.169844+0000) 2024-03-06T10:42:05.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:05.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:05.170088+0000) 2024-03-06T10:42:06.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:06.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:06.170336+0000) 2024-03-06T10:42:06.713 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:42:06.721 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:42:07.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:07.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:07.170600+0000) 2024-03-06T10:42:08.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:08.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:08.170797+0000) 2024-03-06T10:42:09.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:09.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:09.170998+0000) 2024-03-06T10:42:10.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:10.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:10.171234+0000) 2024-03-06T10:42:11.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:11.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:11.171424+0000) 2024-03-06T10:42:12.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:12.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:12.171670+0000) 2024-03-06T10:42:13.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:13.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:13.171917+0000) 2024-03-06T10:42:14.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:14.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:14.172164+0000) 2024-03-06T10:42:15.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:15.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:15.172373+0000) 2024-03-06T10:42:16.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:16.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:16.172643+0000) 2024-03-06T10:42:17.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:17.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:17.172870+0000) 2024-03-06T10:42:18.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:18.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:18.173052+0000) 2024-03-06T10:42:19.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:19.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:19.173284+0000) 2024-03-06T10:42:20.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:20.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:20.173512+0000) 2024-03-06T10:42:21.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:21.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:21.173776+0000) 2024-03-06T10:42:22.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:22.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:22.173961+0000) 2024-03-06T10:42:23.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:23.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:23.174196+0000) 2024-03-06T10:42:24.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:24.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:24.174420+0000) 2024-03-06T10:42:25.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:25.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:25.174663+0000) 2024-03-06T10:42:26.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:26.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:26.174871+0000) 2024-03-06T10:42:27.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:27.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:27.175069+0000) 2024-03-06T10:42:28.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:28.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:28.175313+0000) 2024-03-06T10:42:29.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:29.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:29.175592+0000) 2024-03-06T10:42:30.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:30.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:30.175829+0000) 2024-03-06T10:42:31.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:31.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:31.176030+0000) 2024-03-06T10:42:32.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:32.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:32.176269+0000) 2024-03-06T10:42:33.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:33.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:33.176478+0000) 2024-03-06T10:42:34.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:34.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:34.176735+0000) 2024-03-06T10:42:35.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:35.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:35.176977+0000) 2024-03-06T10:42:36.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:36.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:36.177203+0000) 2024-03-06T10:42:36.925 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:42:36.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:42:37.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:37.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:37.177474+0000) 2024-03-06T10:42:38.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:38.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:38.177712+0000) 2024-03-06T10:42:39.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:39.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:39.177944+0000) 2024-03-06T10:42:40.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:40.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:40.178189+0000) 2024-03-06T10:42:41.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:41.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:41.178431+0000) 2024-03-06T10:42:42.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:42.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:42.178678+0000) 2024-03-06T10:42:43.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:43.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:43.178932+0000) 2024-03-06T10:42:44.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:44.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:44.179194+0000) 2024-03-06T10:42:45.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:45.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:45.179478+0000) 2024-03-06T10:42:46.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:46.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:46.179763+0000) 2024-03-06T10:42:47.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:47.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:47.180040+0000) 2024-03-06T10:42:48.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:48.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:48.180325+0000) 2024-03-06T10:42:49.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:49.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:49.180645+0000) 2024-03-06T10:42:50.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:50.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:50.180932+0000) 2024-03-06T10:42:51.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:51.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:51.181220+0000) 2024-03-06T10:42:52.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:52.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:52.181500+0000) 2024-03-06T10:42:53.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:53.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:53.181754+0000) 2024-03-06T10:42:54.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:54.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:54.181995+0000) 2024-03-06T10:42:55.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:55.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:55.182249+0000) 2024-03-06T10:42:56.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:56.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:56.182484+0000) 2024-03-06T10:42:57.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:57.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:57.182702+0000) 2024-03-06T10:42:58.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:58.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:58.182940+0000) 2024-03-06T10:42:59.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:42:59.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:42:59.183173+0000) 2024-03-06T10:43:00.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:00.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:00.183377+0000) 2024-03-06T10:43:01.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:01.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:01.183639+0000) 2024-03-06T10:43:02.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:02.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:02.183811+0000) 2024-03-06T10:43:03.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:03.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:03.183999+0000) 2024-03-06T10:43:04.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:04.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:04.184215+0000) 2024-03-06T10:43:05.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:05.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:05.184471+0000) 2024-03-06T10:43:06.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:06.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:06.184666+0000) 2024-03-06T10:43:07.130 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:43:07.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:43:07.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:07.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:07.184828+0000) 2024-03-06T10:43:08.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:08.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:08.185046+0000) 2024-03-06T10:43:09.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:09.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:09.185233+0000) 2024-03-06T10:43:10.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:10.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:10.185503+0000) 2024-03-06T10:43:11.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:11.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:11.185807+0000) 2024-03-06T10:43:12.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:12.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:12.186094+0000) 2024-03-06T10:43:13.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:13.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:13.186382+0000) 2024-03-06T10:43:14.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:14.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:14.186693+0000) 2024-03-06T10:43:15.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:15.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:15.186981+0000) 2024-03-06T10:43:16.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:16.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:16.187189+0000) 2024-03-06T10:43:17.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:17.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:17.187401+0000) 2024-03-06T10:43:18.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:18.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:18.187648+0000) 2024-03-06T10:43:19.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:19.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:19.187844+0000) 2024-03-06T10:43:20.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:20.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:20.188074+0000) 2024-03-06T10:43:21.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:21.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:21.188308+0000) 2024-03-06T10:43:22.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:22.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:22.188569+0000) 2024-03-06T10:43:23.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:23.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:23.188763+0000) 2024-03-06T10:43:24.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:24.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:24.189030+0000) 2024-03-06T10:43:25.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:25.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:25.189319+0000) 2024-03-06T10:43:26.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:26.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:26.189633+0000) 2024-03-06T10:43:27.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:27.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:27.189919+0000) 2024-03-06T10:43:28.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:28.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:28.190189+0000) 2024-03-06T10:43:29.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:29.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:29.190467+0000) 2024-03-06T10:43:30.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:30.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:30.190699+0000) 2024-03-06T10:43:31.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:31.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:31.190985+0000) 2024-03-06T10:43:32.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:32.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:32.191182+0000) 2024-03-06T10:43:33.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:33.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:33.191369+0000) 2024-03-06T10:43:34.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:34.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:34.191585+0000) 2024-03-06T10:43:35.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:35.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:35.191836+0000) 2024-03-06T10:43:36.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:36.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:36.192066+0000) 2024-03-06T10:43:37.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:37.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:37.192315+0000) 2024-03-06T10:43:37.345 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:43:37.356 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:43:38.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:38.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:38.192507+0000) 2024-03-06T10:43:39.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:39.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:39.192749+0000) 2024-03-06T10:43:40.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:40.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:40.192989+0000) 2024-03-06T10:43:41.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:41.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:41.193188+0000) 2024-03-06T10:43:42.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:42.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:42.193472+0000) 2024-03-06T10:43:43.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:43.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:43.193748+0000) 2024-03-06T10:43:44.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:44.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:44.193989+0000) 2024-03-06T10:43:45.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:45.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:45.194189+0000) 2024-03-06T10:43:46.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:46.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:46.194432+0000) 2024-03-06T10:43:47.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:47.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:47.194681+0000) 2024-03-06T10:43:48.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:48.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:48.194885+0000) 2024-03-06T10:43:49.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:49.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:49.195085+0000) 2024-03-06T10:43:50.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:50.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:50.195311+0000) 2024-03-06T10:43:51.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:51.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:51.195569+0000) 2024-03-06T10:43:52.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:52.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:52.195789+0000) 2024-03-06T10:43:53.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:53.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:53.196017+0000) 2024-03-06T10:43:54.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:54.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:54.196242+0000) 2024-03-06T10:43:55.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:55.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:55.196489+0000) 2024-03-06T10:43:56.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:56.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:56.196719+0000) 2024-03-06T10:43:57.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:57.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:57.196905+0000) 2024-03-06T10:43:58.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:58.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:58.197133+0000) 2024-03-06T10:43:59.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:43:59.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:43:59.197366+0000) 2024-03-06T10:44:00.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:00.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:00.197501+0000) 2024-03-06T10:44:01.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:01.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:01.197670+0000) 2024-03-06T10:44:02.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:02.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:02.197926+0000) 2024-03-06T10:44:03.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:03.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:03.198182+0000) 2024-03-06T10:44:04.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:04.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:04.198421+0000) 2024-03-06T10:44:05.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:05.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:05.198703+0000) 2024-03-06T10:44:06.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:06.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:06.198881+0000) 2024-03-06T10:44:07.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:07.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:07.199097+0000) 2024-03-06T10:44:07.481 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:44:07.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:44:08.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:08.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:08.199330+0000) 2024-03-06T10:44:09.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:09.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:09.199508+0000) 2024-03-06T10:44:10.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:10.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:10.199770+0000) 2024-03-06T10:44:11.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:11.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:11.199995+0000) 2024-03-06T10:44:12.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:12.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:12.200218+0000) 2024-03-06T10:44:13.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:13.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:13.200509+0000) 2024-03-06T10:44:14.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:14.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:14.200788+0000) 2024-03-06T10:44:15.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:15.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:15.201021+0000) 2024-03-06T10:44:16.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:16.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:16.201242+0000) 2024-03-06T10:44:17.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:17.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:17.201423+0000) 2024-03-06T10:44:18.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:18.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:18.201699+0000) 2024-03-06T10:44:19.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:19.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:19.201962+0000) 2024-03-06T10:44:20.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:20.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:20.202200+0000) 2024-03-06T10:44:21.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:21.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:21.202442+0000) 2024-03-06T10:44:22.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:22.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:22.202657+0000) 2024-03-06T10:44:23.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:23.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:23.202861+0000) 2024-03-06T10:44:24.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:24.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:24.203057+0000) 2024-03-06T10:44:25.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:25.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:25.203292+0000) 2024-03-06T10:44:26.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:26.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:26.203520+0000) 2024-03-06T10:44:27.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:27.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:27.203779+0000) 2024-03-06T10:44:28.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:28.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:28.203990+0000) 2024-03-06T10:44:29.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:29.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:29.204225+0000) 2024-03-06T10:44:30.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:30.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:30.204404+0000) 2024-03-06T10:44:31.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:31.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:31.204661+0000) 2024-03-06T10:44:32.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:32.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:32.204890+0000) 2024-03-06T10:44:33.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:33.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:33.205121+0000) 2024-03-06T10:44:34.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:34.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:34.205340+0000) 2024-03-06T10:44:35.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:35.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:35.205612+0000) 2024-03-06T10:44:36.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:36.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:36.205815+0000) 2024-03-06T10:44:37.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:37.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:37.206058+0000) 2024-03-06T10:44:37.686 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:44:37.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:44:38.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:38.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:38.206278+0000) 2024-03-06T10:44:39.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:39.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:39.206473+0000) 2024-03-06T10:44:40.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:40.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:40.206698+0000) 2024-03-06T10:44:41.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:41.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:41.206919+0000) 2024-03-06T10:44:42.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:42.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:42.207141+0000) 2024-03-06T10:44:43.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:43.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:43.207376+0000) 2024-03-06T10:44:44.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:44.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:44.207636+0000) 2024-03-06T10:44:45.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:45.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:45.207876+0000) 2024-03-06T10:44:46.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:46.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:46.208099+0000) 2024-03-06T10:44:47.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:47.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:47.208324+0000) 2024-03-06T10:44:48.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:48.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:48.208598+0000) 2024-03-06T10:44:49.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:49.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:49.208823+0000) 2024-03-06T10:44:50.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:50.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:50.209054+0000) 2024-03-06T10:44:51.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:51.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:51.209292+0000) 2024-03-06T10:44:52.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:52.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:52.209514+0000) 2024-03-06T10:44:53.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:53.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:53.209754+0000) 2024-03-06T10:44:54.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:54.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:54.209984+0000) 2024-03-06T10:44:55.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:55.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:55.210219+0000) 2024-03-06T10:44:56.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:56.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:56.210444+0000) 2024-03-06T10:44:57.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:57.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:57.210680+0000) 2024-03-06T10:44:58.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:58.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:58.210916+0000) 2024-03-06T10:44:59.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:44:59.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:44:59.211144+0000) 2024-03-06T10:45:00.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:00.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:00.211369+0000) 2024-03-06T10:45:01.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:01.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:01.211595+0000) 2024-03-06T10:45:02.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:02.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:02.211814+0000) 2024-03-06T10:45:03.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:03.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:03.212047+0000) 2024-03-06T10:45:04.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:04.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:04.212283+0000) 2024-03-06T10:45:05.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:05.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:05.212513+0000) 2024-03-06T10:45:06.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:06.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:06.212746+0000) 2024-03-06T10:45:07.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:07.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:07.212991+0000) 2024-03-06T10:45:07.895 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:45:07.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:45:08.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:08.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:08.213251+0000) 2024-03-06T10:45:09.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:09.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:09.213494+0000) 2024-03-06T10:45:10.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:10.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:10.213724+0000) 2024-03-06T10:45:11.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:11.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:11.213956+0000) 2024-03-06T10:45:12.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:12.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:12.214184+0000) 2024-03-06T10:45:13.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:13.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:13.214420+0000) 2024-03-06T10:45:14.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:14.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:14.214659+0000) 2024-03-06T10:45:15.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:15.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:15.214914+0000) 2024-03-06T10:45:16.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:16.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:16.215152+0000) 2024-03-06T10:45:17.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:17.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:17.215400+0000) 2024-03-06T10:45:18.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:18.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:18.215656+0000) 2024-03-06T10:45:19.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:19.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:19.215881+0000) 2024-03-06T10:45:20.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:20.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:20.216093+0000) 2024-03-06T10:45:21.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:21.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:21.216318+0000) 2024-03-06T10:45:22.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:22.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:22.216568+0000) 2024-03-06T10:45:23.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:23.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:23.216811+0000) 2024-03-06T10:45:24.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:24.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:24.217039+0000) 2024-03-06T10:45:25.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:25.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:25.217288+0000) 2024-03-06T10:45:26.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:26.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:26.217465+0000) 2024-03-06T10:45:27.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:27.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:27.217688+0000) 2024-03-06T10:45:28.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:28.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:28.217925+0000) 2024-03-06T10:45:29.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:29.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:29.218148+0000) 2024-03-06T10:45:30.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:30.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:30.218416+0000) 2024-03-06T10:45:31.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:31.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:31.218696+0000) 2024-03-06T10:45:32.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:32.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:32.218909+0000) 2024-03-06T10:45:33.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:33.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:33.219119+0000) 2024-03-06T10:45:34.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:34.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:34.219351+0000) 2024-03-06T10:45:35.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:35.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:35.219599+0000) 2024-03-06T10:45:36.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:36.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:36.219829+0000) 2024-03-06T10:45:37.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:37.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:37.220061+0000) 2024-03-06T10:45:38.191 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:45:38.201 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:45:38.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:38.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:38.220300+0000) 2024-03-06T10:45:39.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:39.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:39.220519+0000) 2024-03-06T10:45:40.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:40.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:40.220840+0000) 2024-03-06T10:45:41.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:41.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:41.221059+0000) 2024-03-06T10:45:42.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:42.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:42.221257+0000) 2024-03-06T10:45:43.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:43.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:43.221492+0000) 2024-03-06T10:45:44.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:44.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:44.221779+0000) 2024-03-06T10:45:45.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:45.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:45.221997+0000) 2024-03-06T10:45:46.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:46.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:46.222181+0000) 2024-03-06T10:45:47.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:47.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:47.222403+0000) 2024-03-06T10:45:48.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:48.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:48.222645+0000) 2024-03-06T10:45:49.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:49.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:49.222891+0000) 2024-03-06T10:45:50.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:50.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:50.223134+0000) 2024-03-06T10:45:51.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:51.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:51.223369+0000) 2024-03-06T10:45:52.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:52.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:52.223651+0000) 2024-03-06T10:45:53.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:53.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:53.223909+0000) 2024-03-06T10:45:54.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:54.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:54.224155+0000) 2024-03-06T10:45:55.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:55.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:55.224459+0000) 2024-03-06T10:45:56.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:56.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:56.224709+0000) 2024-03-06T10:45:57.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:57.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:57.224907+0000) 2024-03-06T10:45:58.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:58.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:58.225125+0000) 2024-03-06T10:45:59.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:45:59.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:45:59.225301+0000) 2024-03-06T10:46:00.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:00.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:00.225525+0000) 2024-03-06T10:46:01.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:01.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:01.225880+0000) 2024-03-06T10:46:02.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:02.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:02.226109+0000) 2024-03-06T10:46:03.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:03.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:03.226350+0000) 2024-03-06T10:46:04.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:04.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:04.226643+0000) 2024-03-06T10:46:05.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:05.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:05.226849+0000) 2024-03-06T10:46:06.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:06.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:06.227058+0000) 2024-03-06T10:46:07.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:07.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:07.227241+0000) 2024-03-06T10:46:08.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:08.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:08.227472+0000) 2024-03-06T10:46:08.351 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:46:08.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:46:09.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:09.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:09.227658+0000) 2024-03-06T10:46:10.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:10.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:10.227894+0000) 2024-03-06T10:46:11.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:11.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:11.228122+0000) 2024-03-06T10:46:12.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:12.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:12.228363+0000) 2024-03-06T10:46:13.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:13.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:13.228653+0000) 2024-03-06T10:46:14.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:14.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:14.228894+0000) 2024-03-06T10:46:15.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:15.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:15.229145+0000) 2024-03-06T10:46:16.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:16.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:16.229391+0000) 2024-03-06T10:46:17.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:17.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:17.229676+0000) 2024-03-06T10:46:18.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:18.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:18.229887+0000) 2024-03-06T10:46:19.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:19.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:19.230128+0000) 2024-03-06T10:46:20.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:20.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:20.230351+0000) 2024-03-06T10:46:21.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:21.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:21.230619+0000) 2024-03-06T10:46:22.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:22.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:22.230834+0000) 2024-03-06T10:46:23.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:23.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:23.231057+0000) 2024-03-06T10:46:24.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:24.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:24.231231+0000) 2024-03-06T10:46:25.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:25.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:25.231465+0000) 2024-03-06T10:46:26.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:26.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:26.231684+0000) 2024-03-06T10:46:27.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:27.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:27.231930+0000) 2024-03-06T10:46:28.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:28.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:28.232140+0000) 2024-03-06T10:46:29.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:29.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:29.232336+0000) 2024-03-06T10:46:30.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:30.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:30.232604+0000) 2024-03-06T10:46:31.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:31.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:31.232786+0000) 2024-03-06T10:46:32.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:32.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:32.232998+0000) 2024-03-06T10:46:33.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:33.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:33.233186+0000) 2024-03-06T10:46:34.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:34.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:34.233416+0000) 2024-03-06T10:46:35.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:35.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:35.233648+0000) 2024-03-06T10:46:36.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:36.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:36.233874+0000) 2024-03-06T10:46:37.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:37.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:37.234073+0000) 2024-03-06T10:46:38.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:38.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:38.234320+0000) 2024-03-06T10:46:38.482 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:46:38.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:46:39.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:39.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:39.234519+0000) 2024-03-06T10:46:40.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:40.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:40.234794+0000) 2024-03-06T10:46:41.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:41.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:41.235032+0000) 2024-03-06T10:46:42.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:42.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:42.235249+0000) 2024-03-06T10:46:43.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:43.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:43.235442+0000) 2024-03-06T10:46:44.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:44.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:44.235687+0000) 2024-03-06T10:46:45.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:45.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:45.235903+0000) 2024-03-06T10:46:46.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:46.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:46.236148+0000) 2024-03-06T10:46:47.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:47.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:47.236326+0000) 2024-03-06T10:46:48.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:48.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:48.236642+0000) 2024-03-06T10:46:49.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:49.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:49.236849+0000) 2024-03-06T10:46:50.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:50.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:50.237085+0000) 2024-03-06T10:46:51.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:51.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:51.237356+0000) 2024-03-06T10:46:52.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:52.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:52.237672+0000) 2024-03-06T10:46:53.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:53.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:53.237920+0000) 2024-03-06T10:46:54.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:54.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:54.238145+0000) 2024-03-06T10:46:55.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:55.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:55.238376+0000) 2024-03-06T10:46:56.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:56.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:56.238670+0000) 2024-03-06T10:46:57.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:57.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:57.238940+0000) 2024-03-06T10:46:58.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:58.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:58.239222+0000) 2024-03-06T10:46:59.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:46:59.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:46:59.239451+0000) 2024-03-06T10:47:00.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:00.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:00.239707+0000) 2024-03-06T10:47:01.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:01.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:01.239938+0000) 2024-03-06T10:47:02.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:02.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:02.240156+0000) 2024-03-06T10:47:03.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:03.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:03.240404+0000) 2024-03-06T10:47:04.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:04.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:04.240648+0000) 2024-03-06T10:47:05.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:05.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:05.240905+0000) 2024-03-06T10:47:06.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:06.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:06.241146+0000) 2024-03-06T10:47:07.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:07.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:07.241385+0000) 2024-03-06T10:47:08.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:08.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:08.241582+0000) 2024-03-06T10:47:08.693 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:47:08.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:47:09.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:09.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:09.241777+0000) 2024-03-06T10:47:10.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:10.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:10.242023+0000) 2024-03-06T10:47:11.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:11.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:11.242225+0000) 2024-03-06T10:47:12.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:12.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:12.242459+0000) 2024-03-06T10:47:13.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:13.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:13.242700+0000) 2024-03-06T10:47:14.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:14.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:14.242939+0000) 2024-03-06T10:47:15.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:15.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:15.243202+0000) 2024-03-06T10:47:16.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:16.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:16.243445+0000) 2024-03-06T10:47:17.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:17.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:17.243704+0000) 2024-03-06T10:47:18.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:18.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:18.243950+0000) 2024-03-06T10:47:19.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:19.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:19.244197+0000) 2024-03-06T10:47:20.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:20.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:20.244443+0000) 2024-03-06T10:47:21.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:21.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:21.244705+0000) 2024-03-06T10:47:22.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:22.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:22.244930+0000) 2024-03-06T10:47:23.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:23.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:23.245134+0000) 2024-03-06T10:47:24.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:24.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:24.245386+0000) 2024-03-06T10:47:25.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:25.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:25.245677+0000) 2024-03-06T10:47:26.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:26.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:26.245886+0000) 2024-03-06T10:47:27.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:27.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:27.246086+0000) 2024-03-06T10:47:28.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:28.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:28.246323+0000) 2024-03-06T10:47:29.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:29.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:29.246513+0000) 2024-03-06T10:47:30.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:30.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:30.246768+0000) 2024-03-06T10:47:31.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:31.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:31.247018+0000) 2024-03-06T10:47:32.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:32.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:32.247203+0000) 2024-03-06T10:47:33.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:33.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:33.247431+0000) 2024-03-06T10:47:34.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:34.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:34.247669+0000) 2024-03-06T10:47:35.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:35.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:35.247915+0000) 2024-03-06T10:47:36.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:36.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:36.248194+0000) 2024-03-06T10:47:37.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:37.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:37.248420+0000) 2024-03-06T10:47:38.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:38.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:38.248652+0000) 2024-03-06T10:47:38.900 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:47:38.908 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:47:39.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:39.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:39.248890+0000) 2024-03-06T10:47:40.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:40.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:40.249123+0000) 2024-03-06T10:47:41.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:41.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:41.249369+0000) 2024-03-06T10:47:42.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:42.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:42.249649+0000) 2024-03-06T10:47:43.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:43.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:43.249860+0000) 2024-03-06T10:47:44.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:44.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:44.250102+0000) 2024-03-06T10:47:45.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:45.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:45.250295+0000) 2024-03-06T10:47:46.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:46.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:46.250519+0000) 2024-03-06T10:47:47.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:47.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:47.250783+0000) 2024-03-06T10:47:48.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:48.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:48.251010+0000) 2024-03-06T10:47:49.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:49.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:49.251241+0000) 2024-03-06T10:47:50.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:50.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:50.251465+0000) 2024-03-06T10:47:51.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:51.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:51.251693+0000) 2024-03-06T10:47:52.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:52.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:52.251873+0000) 2024-03-06T10:47:53.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:53.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:53.252055+0000) 2024-03-06T10:47:54.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:54.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:54.252247+0000) 2024-03-06T10:47:55.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:55.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:55.252447+0000) 2024-03-06T10:47:56.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:56.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:56.252681+0000) 2024-03-06T10:47:57.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:57.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:57.252862+0000) 2024-03-06T10:47:58.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:58.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:58.253036+0000) 2024-03-06T10:47:59.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:47:59.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:47:59.253311+0000) 2024-03-06T10:48:00.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:00.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:00.253567+0000) 2024-03-06T10:48:01.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:01.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:01.253807+0000) 2024-03-06T10:48:02.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:02.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:02.253997+0000) 2024-03-06T10:48:03.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:03.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:03.254215+0000) 2024-03-06T10:48:04.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:04.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:04.254443+0000) 2024-03-06T10:48:05.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:05.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:05.254677+0000) 2024-03-06T10:48:06.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:06.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:06.254861+0000) 2024-03-06T10:48:07.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:07.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:07.255089+0000) 2024-03-06T10:48:08.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:08.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:08.255336+0000) 2024-03-06T10:48:09.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:48:09.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:48:09.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:09.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:09.255619+0000) 2024-03-06T10:48:10.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:10.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:10.255846+0000) 2024-03-06T10:48:11.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:11.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:11.256045+0000) 2024-03-06T10:48:12.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:12.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:12.256239+0000) 2024-03-06T10:48:13.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:13.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:13.256396+0000) 2024-03-06T10:48:14.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:14.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:14.256650+0000) 2024-03-06T10:48:15.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:15.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:15.256883+0000) 2024-03-06T10:48:16.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:16.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:16.257162+0000) 2024-03-06T10:48:17.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:17.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:17.257443+0000) 2024-03-06T10:48:18.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:18.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:18.257723+0000) 2024-03-06T10:48:19.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:19.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:19.257953+0000) 2024-03-06T10:48:20.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:20.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:20.258231+0000) 2024-03-06T10:48:21.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:21.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:21.258468+0000) 2024-03-06T10:48:22.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:22.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:22.258704+0000) 2024-03-06T10:48:23.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:23.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:23.258904+0000) 2024-03-06T10:48:24.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:24.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:24.259200+0000) 2024-03-06T10:48:25.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:25.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:25.259453+0000) 2024-03-06T10:48:26.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:26.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:26.259692+0000) 2024-03-06T10:48:27.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:27.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:27.259893+0000) 2024-03-06T10:48:28.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:28.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:28.260133+0000) 2024-03-06T10:48:29.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:29.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:29.260335+0000) 2024-03-06T10:48:30.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:30.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:30.260571+0000) 2024-03-06T10:48:31.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:31.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:31.260800+0000) 2024-03-06T10:48:32.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:32.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:32.261015+0000) 2024-03-06T10:48:33.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:33.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:33.261256+0000) 2024-03-06T10:48:34.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:34.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:34.261444+0000) 2024-03-06T10:48:35.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:35.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:35.261666+0000) 2024-03-06T10:48:36.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:36.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:36.261876+0000) 2024-03-06T10:48:37.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:37.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:37.262093+0000) 2024-03-06T10:48:38.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:38.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:38.262279+0000) 2024-03-06T10:48:39.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:39.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:39.262497+0000) 2024-03-06T10:48:39.290 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:48:39.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:48:40.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:40.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:40.262681+0000) 2024-03-06T10:48:41.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:41.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:41.262891+0000) 2024-03-06T10:48:42.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:42.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:42.263106+0000) 2024-03-06T10:48:43.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:43.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:43.263326+0000) 2024-03-06T10:48:44.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:44.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:44.263573+0000) 2024-03-06T10:48:45.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:45.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:45.263788+0000) 2024-03-06T10:48:46.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:46.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:46.264014+0000) 2024-03-06T10:48:47.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:47.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:47.264202+0000) 2024-03-06T10:48:48.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:48.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:48.264390+0000) 2024-03-06T10:48:49.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:49.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:49.264663+0000) 2024-03-06T10:48:50.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:50.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:50.264881+0000) 2024-03-06T10:48:51.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:51.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:51.265143+0000) 2024-03-06T10:48:52.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:52.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:52.265342+0000) 2024-03-06T10:48:53.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:53.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:53.265596+0000) 2024-03-06T10:48:54.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:54.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:54.265840+0000) 2024-03-06T10:48:55.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:55.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:55.266072+0000) 2024-03-06T10:48:56.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:56.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:56.266304+0000) 2024-03-06T10:48:57.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:57.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:57.266578+0000) 2024-03-06T10:48:58.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:58.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:58.266805+0000) 2024-03-06T10:48:59.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:48:59.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:48:59.267072+0000) 2024-03-06T10:49:00.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:00.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:00.267350+0000) 2024-03-06T10:49:01.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:01.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:01.267615+0000) 2024-03-06T10:49:02.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:02.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:02.267837+0000) 2024-03-06T10:49:03.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:03.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:03.268062+0000) 2024-03-06T10:49:04.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:04.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:04.268253+0000) 2024-03-06T10:49:05.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:05.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:05.268460+0000) 2024-03-06T10:49:06.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:06.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:06.268719+0000) 2024-03-06T10:49:07.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:07.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:07.268946+0000) 2024-03-06T10:49:08.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:08.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:08.269169+0000) 2024-03-06T10:49:09.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:09.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:09.269394+0000) 2024-03-06T10:49:09.429 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:49:09.438 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:49:10.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:10.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:10.269618+0000) 2024-03-06T10:49:11.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:11.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:11.269845+0000) 2024-03-06T10:49:12.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:12.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:12.270067+0000) 2024-03-06T10:49:13.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:13.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:13.270231+0000) 2024-03-06T10:49:14.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:14.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:14.270452+0000) 2024-03-06T10:49:15.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:15.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:15.270631+0000) 2024-03-06T10:49:16.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:16.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:16.270868+0000) 2024-03-06T10:49:17.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:17.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:17.271106+0000) 2024-03-06T10:49:18.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:18.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:18.271335+0000) 2024-03-06T10:49:19.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:19.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:19.271518+0000) 2024-03-06T10:49:20.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:20.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:20.271765+0000) 2024-03-06T10:49:21.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:21.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:21.271981+0000) 2024-03-06T10:49:22.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:22.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:22.272182+0000) 2024-03-06T10:49:23.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:23.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:23.272414+0000) 2024-03-06T10:49:24.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:24.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:24.272652+0000) 2024-03-06T10:49:25.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:25.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:25.272849+0000) 2024-03-06T10:49:26.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:26.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:26.273072+0000) 2024-03-06T10:49:27.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:27.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:27.273258+0000) 2024-03-06T10:49:28.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:28.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:28.273480+0000) 2024-03-06T10:49:29.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:29.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:29.273699+0000) 2024-03-06T10:49:30.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:30.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:30.273922+0000) 2024-03-06T10:49:31.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:31.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:31.274142+0000) 2024-03-06T10:49:32.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:32.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:32.274367+0000) 2024-03-06T10:49:33.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:33.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:33.274610+0000) 2024-03-06T10:49:34.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:34.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:34.274795+0000) 2024-03-06T10:49:35.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:35.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:35.275025+0000) 2024-03-06T10:49:36.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:36.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:36.275248+0000) 2024-03-06T10:49:37.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:37.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:37.275435+0000) 2024-03-06T10:49:38.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:38.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:38.275659+0000) 2024-03-06T10:49:39.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:39.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:39.275862+0000) 2024-03-06T10:49:39.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:49:39.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:49:40.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:40.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:40.276094+0000) 2024-03-06T10:49:41.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:41.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:41.276314+0000) 2024-03-06T10:49:42.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:42.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:42.276581+0000) 2024-03-06T10:49:43.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:43.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:43.276809+0000) 2024-03-06T10:49:44.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:44.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:44.277022+0000) 2024-03-06T10:49:45.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:45.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:45.277249+0000) 2024-03-06T10:49:46.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:46.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:46.277470+0000) 2024-03-06T10:49:47.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:47.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:47.277669+0000) 2024-03-06T10:49:48.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:48.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:48.277871+0000) 2024-03-06T10:49:49.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:49.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:49.278133+0000) 2024-03-06T10:49:50.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:50.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:50.278373+0000) 2024-03-06T10:49:51.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:51.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:51.278628+0000) 2024-03-06T10:49:52.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:52.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:52.278820+0000) 2024-03-06T10:49:53.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:53.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:53.279038+0000) 2024-03-06T10:49:54.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:54.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:54.279252+0000) 2024-03-06T10:49:55.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:55.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:55.279488+0000) 2024-03-06T10:49:56.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:56.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:56.279716+0000) 2024-03-06T10:49:57.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:57.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:57.279938+0000) 2024-03-06T10:49:58.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:58.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:58.280168+0000) 2024-03-06T10:49:59.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:49:59.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:49:59.280390+0000) 2024-03-06T10:50:00.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:00.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:00.280676+0000) 2024-03-06T10:50:01.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:01.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:01.280854+0000) 2024-03-06T10:50:02.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:02.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:02.281044+0000) 2024-03-06T10:50:03.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:03.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:03.281264+0000) 2024-03-06T10:50:04.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:04.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:04.281503+0000) 2024-03-06T10:50:05.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:05.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:05.281848+0000) 2024-03-06T10:50:06.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:06.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:06.282030+0000) 2024-03-06T10:50:07.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:07.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:07.282247+0000) 2024-03-06T10:50:08.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:08.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:08.282470+0000) 2024-03-06T10:50:09.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:09.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:09.282655+0000) 2024-03-06T10:50:09.780 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:50:09.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:50:10.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:10.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:10.282851+0000) 2024-03-06T10:50:11.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:11.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:11.283081+0000) 2024-03-06T10:50:12.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:12.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:12.283271+0000) 2024-03-06T10:50:13.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:13.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:13.283507+0000) 2024-03-06T10:50:14.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:14.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:14.283752+0000) 2024-03-06T10:50:15.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:15.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:15.283968+0000) 2024-03-06T10:50:16.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:16.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:16.284190+0000) 2024-03-06T10:50:17.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:17.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:17.284394+0000) 2024-03-06T10:50:18.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:18.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:18.284616+0000) 2024-03-06T10:50:19.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:19.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:19.284855+0000) 2024-03-06T10:50:20.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:20.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:20.285093+0000) 2024-03-06T10:50:21.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:21.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:21.285332+0000) 2024-03-06T10:50:22.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:22.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:22.285612+0000) 2024-03-06T10:50:23.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:23.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:23.285861+0000) 2024-03-06T10:50:24.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:24.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:24.286072+0000) 2024-03-06T10:50:25.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:25.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:25.286274+0000) 2024-03-06T10:50:26.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:26.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:26.286507+0000) 2024-03-06T10:50:27.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:27.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:27.286771+0000) 2024-03-06T10:50:28.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:28.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:28.286987+0000) 2024-03-06T10:50:29.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:29.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:29.287209+0000) 2024-03-06T10:50:30.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:30.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:30.287418+0000) 2024-03-06T10:50:31.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:31.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:31.287656+0000) 2024-03-06T10:50:32.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:32.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:32.287887+0000) 2024-03-06T10:50:33.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:33.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:33.288067+0000) 2024-03-06T10:50:34.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:34.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:34.288316+0000) 2024-03-06T10:50:35.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:35.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:35.288578+0000) 2024-03-06T10:50:36.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:36.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:36.288811+0000) 2024-03-06T10:50:37.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:37.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:37.289052+0000) 2024-03-06T10:50:38.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:38.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:38.289295+0000) 2024-03-06T10:50:39.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:39.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:39.289528+0000) 2024-03-06T10:50:39.985 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:50:39.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:50:40.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:40.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:40.289791+0000) 2024-03-06T10:50:41.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:41.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:41.290015+0000) 2024-03-06T10:50:42.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:42.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:42.290198+0000) 2024-03-06T10:50:43.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:43.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:43.290418+0000) 2024-03-06T10:50:44.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:44.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:44.290665+0000) 2024-03-06T10:50:45.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:45.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:45.290924+0000) 2024-03-06T10:50:46.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:46.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:46.291164+0000) 2024-03-06T10:50:47.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:47.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:47.291398+0000) 2024-03-06T10:50:48.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:48.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:48.291643+0000) 2024-03-06T10:50:49.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:49.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:49.291849+0000) 2024-03-06T10:50:50.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:50.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:50.292057+0000) 2024-03-06T10:50:51.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:51.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:51.292300+0000) 2024-03-06T10:50:52.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:52.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:52.292570+0000) 2024-03-06T10:50:53.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:53.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:53.292809+0000) 2024-03-06T10:50:54.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:54.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:54.293062+0000) 2024-03-06T10:50:55.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:55.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:55.293308+0000) 2024-03-06T10:50:56.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:56.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:56.293561+0000) 2024-03-06T10:50:57.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:57.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:57.293742+0000) 2024-03-06T10:50:58.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:58.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:58.293973+0000) 2024-03-06T10:50:59.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:50:59.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:50:59.294253+0000) 2024-03-06T10:51:00.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:00.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:00.294473+0000) 2024-03-06T10:51:01.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:01.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:01.294713+0000) 2024-03-06T10:51:02.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:02.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:02.294936+0000) 2024-03-06T10:51:03.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:03.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:03.295153+0000) 2024-03-06T10:51:04.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:04.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:04.295387+0000) 2024-03-06T10:51:05.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:05.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:05.295657+0000) 2024-03-06T10:51:06.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:06.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:06.295877+0000) 2024-03-06T10:51:07.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:07.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:07.296061+0000) 2024-03-06T10:51:08.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:08.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:08.296284+0000) 2024-03-06T10:51:09.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:09.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:09.296513+0000) 2024-03-06T10:51:10.123 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:51:10.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:51:10.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:10.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:10.296746+0000) 2024-03-06T10:51:11.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:11.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:11.296962+0000) 2024-03-06T10:51:12.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:12.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:12.297192+0000) 2024-03-06T10:51:13.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:13.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:13.297432+0000) 2024-03-06T10:51:14.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:14.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:14.297688+0000) 2024-03-06T10:51:15.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:15.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:15.297944+0000) 2024-03-06T10:51:16.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:16.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:16.298190+0000) 2024-03-06T10:51:17.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:17.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:17.298427+0000) 2024-03-06T10:51:18.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:18.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:18.298666+0000) 2024-03-06T10:51:19.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:19.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:19.298903+0000) 2024-03-06T10:51:20.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:20.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:20.299145+0000) 2024-03-06T10:51:21.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:21.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:21.299381+0000) 2024-03-06T10:51:22.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:22.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:22.299660+0000) 2024-03-06T10:51:23.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:23.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:23.299899+0000) 2024-03-06T10:51:24.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:24.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:24.300115+0000) 2024-03-06T10:51:25.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:25.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:25.300332+0000) 2024-03-06T10:51:26.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:26.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:26.300566+0000) 2024-03-06T10:51:27.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:27.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:27.300789+0000) 2024-03-06T10:51:28.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:28.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:28.301029+0000) 2024-03-06T10:51:29.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:29.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:29.301249+0000) 2024-03-06T10:51:30.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:30.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:30.301480+0000) 2024-03-06T10:51:31.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:31.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:31.301687+0000) 2024-03-06T10:51:32.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:32.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:32.301892+0000) 2024-03-06T10:51:33.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:33.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:33.302133+0000) 2024-03-06T10:51:34.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:34.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:34.302388+0000) 2024-03-06T10:51:35.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:35.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:35.302638+0000) 2024-03-06T10:51:36.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:36.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:36.302835+0000) 2024-03-06T10:51:37.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:37.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:37.303022+0000) 2024-03-06T10:51:38.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:38.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:38.303264+0000) 2024-03-06T10:51:39.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:39.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:39.303504+0000) 2024-03-06T10:51:40.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:40.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:40.303785+0000) 2024-03-06T10:51:40.328 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:51:40.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:51:41.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:41.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:41.304030+0000) 2024-03-06T10:51:42.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:42.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:42.304281+0000) 2024-03-06T10:51:43.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:43.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:43.304513+0000) 2024-03-06T10:51:44.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:44.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:44.304921+0000) 2024-03-06T10:51:45.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:45.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:45.305183+0000) 2024-03-06T10:51:46.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:46.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:46.305426+0000) 2024-03-06T10:51:47.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:47.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:47.305714+0000) 2024-03-06T10:51:48.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:48.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:48.305957+0000) 2024-03-06T10:51:49.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:49.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:49.306199+0000) 2024-03-06T10:51:50.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:50.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:50.306455+0000) 2024-03-06T10:51:51.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:51.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:51.306710+0000) 2024-03-06T10:51:52.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:52.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:52.306960+0000) 2024-03-06T10:51:53.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:53.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:53.307162+0000) 2024-03-06T10:51:54.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:54.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:54.307383+0000) 2024-03-06T10:51:55.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:55.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:55.307682+0000) 2024-03-06T10:51:56.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:56.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:56.307853+0000) 2024-03-06T10:51:57.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:57.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:57.308062+0000) 2024-03-06T10:51:58.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:58.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:58.308299+0000) 2024-03-06T10:51:59.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:51:59.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:51:59.308493+0000) 2024-03-06T10:52:00.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:00.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:00.308726+0000) 2024-03-06T10:52:01.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:01.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:01.308961+0000) 2024-03-06T10:52:02.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:02.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:02.309190+0000) 2024-03-06T10:52:03.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:03.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:03.309421+0000) 2024-03-06T10:52:04.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:04.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:04.309673+0000) 2024-03-06T10:52:05.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:05.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:05.309917+0000) 2024-03-06T10:52:06.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:06.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:06.310143+0000) 2024-03-06T10:52:07.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:07.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:07.310357+0000) 2024-03-06T10:52:08.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:08.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:08.310587+0000) 2024-03-06T10:52:09.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:09.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:09.310838+0000) 2024-03-06T10:52:10.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:10.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:10.311062+0000) 2024-03-06T10:52:10.540 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:52:10.549 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:52:11.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:11.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:11.311282+0000) 2024-03-06T10:52:12.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:12.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:12.311500+0000) 2024-03-06T10:52:13.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:13.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:13.311726+0000) 2024-03-06T10:52:14.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:14.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:14.311948+0000) 2024-03-06T10:52:15.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:15.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:15.312178+0000) 2024-03-06T10:52:16.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:16.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:16.312418+0000) 2024-03-06T10:52:17.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:17.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:17.312638+0000) 2024-03-06T10:52:18.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:18.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:18.312859+0000) 2024-03-06T10:52:19.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:19.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:19.313046+0000) 2024-03-06T10:52:20.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:20.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:20.313267+0000) 2024-03-06T10:52:21.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:21.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:21.313499+0000) 2024-03-06T10:52:22.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:22.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:22.313760+0000) 2024-03-06T10:52:23.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:23.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:23.313930+0000) 2024-03-06T10:52:24.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:24.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:24.314130+0000) 2024-03-06T10:52:25.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:25.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:25.314364+0000) 2024-03-06T10:52:26.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:26.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:26.314639+0000) 2024-03-06T10:52:27.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:27.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:27.314864+0000) 2024-03-06T10:52:28.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:28.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:28.315080+0000) 2024-03-06T10:52:29.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:29.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:29.315329+0000) 2024-03-06T10:52:30.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:30.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:30.315592+0000) 2024-03-06T10:52:31.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:31.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:31.315774+0000) 2024-03-06T10:52:32.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:32.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:32.315991+0000) 2024-03-06T10:52:33.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:33.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:33.316200+0000) 2024-03-06T10:52:34.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:34.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:34.316438+0000) 2024-03-06T10:52:35.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:35.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:35.316687+0000) 2024-03-06T10:52:36.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:36.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:36.316934+0000) 2024-03-06T10:52:37.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:37.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:37.317170+0000) 2024-03-06T10:52:38.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:38.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:38.317365+0000) 2024-03-06T10:52:39.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:39.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:39.317605+0000) 2024-03-06T10:52:40.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:40.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:40.317831+0000) 2024-03-06T10:52:40.669 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:52:40.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:52:41.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:41.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:41.318029+0000) 2024-03-06T10:52:42.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:42.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:42.318222+0000) 2024-03-06T10:52:43.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:43.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:43.318438+0000) 2024-03-06T10:52:44.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:44.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:44.318671+0000) 2024-03-06T10:52:45.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:45.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:45.318904+0000) 2024-03-06T10:52:46.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:46.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:46.319133+0000) 2024-03-06T10:52:47.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:47.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:47.319349+0000) 2024-03-06T10:52:48.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:48.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:48.319610+0000) 2024-03-06T10:52:49.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:49.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:49.319832+0000) 2024-03-06T10:52:50.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:50.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:50.320062+0000) 2024-03-06T10:52:51.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:51.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:51.320290+0000) 2024-03-06T10:52:52.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:52.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:52.320503+0000) 2024-03-06T10:52:53.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:53.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:53.320772+0000) 2024-03-06T10:52:54.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:54.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:54.320990+0000) 2024-03-06T10:52:55.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:55.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:55.321223+0000) 2024-03-06T10:52:56.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:56.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:56.321453+0000) 2024-03-06T10:52:57.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:57.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:57.321680+0000) 2024-03-06T10:52:58.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:58.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:58.321911+0000) 2024-03-06T10:52:59.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:52:59.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:52:59.322133+0000) 2024-03-06T10:53:00.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:00.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:00.322365+0000) 2024-03-06T10:53:01.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:01.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:01.322631+0000) 2024-03-06T10:53:02.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:02.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:02.322855+0000) 2024-03-06T10:53:03.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:03.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:03.323074+0000) 2024-03-06T10:53:04.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:04.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:04.323306+0000) 2024-03-06T10:53:05.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:05.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:05.323567+0000) 2024-03-06T10:53:06.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:06.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:06.323803+0000) 2024-03-06T10:53:07.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:07.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:07.324024+0000) 2024-03-06T10:53:08.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:08.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:08.324253+0000) 2024-03-06T10:53:09.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:09.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:09.324477+0000) 2024-03-06T10:53:10.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:10.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:10.324699+0000) 2024-03-06T10:53:10.884 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:53:10.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:53:11.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:11.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:11.324900+0000) 2024-03-06T10:53:12.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:12.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:12.325134+0000) 2024-03-06T10:53:13.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:13.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:13.325316+0000) 2024-03-06T10:53:14.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:14.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:14.325573+0000) 2024-03-06T10:53:15.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:15.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:15.325807+0000) 2024-03-06T10:53:16.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:16.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:16.325999+0000) 2024-03-06T10:53:17.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:17.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:17.326193+0000) 2024-03-06T10:53:18.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:18.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:18.326419+0000) 2024-03-06T10:53:19.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:19.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:19.326628+0000) 2024-03-06T10:53:20.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:20.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:20.326855+0000) 2024-03-06T10:53:21.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:21.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:21.327076+0000) 2024-03-06T10:53:22.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:22.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:22.327269+0000) 2024-03-06T10:53:23.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:23.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:23.327579+0000) 2024-03-06T10:53:24.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:24.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:24.327807+0000) 2024-03-06T10:53:25.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:25.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:25.328039+0000) 2024-03-06T10:53:26.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:26.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:26.328277+0000) 2024-03-06T10:53:27.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:27.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:27.328501+0000) 2024-03-06T10:53:28.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:28.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:28.328732+0000) 2024-03-06T10:53:29.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:29.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:29.328954+0000) 2024-03-06T10:53:30.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:30.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:30.329184+0000) 2024-03-06T10:53:31.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:31.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:31.329402+0000) 2024-03-06T10:53:32.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:32.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:32.329667+0000) 2024-03-06T10:53:33.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:33.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:33.329890+0000) 2024-03-06T10:53:34.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:34.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:34.330118+0000) 2024-03-06T10:53:35.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:35.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:35.330306+0000) 2024-03-06T10:53:36.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:36.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:36.330499+0000) 2024-03-06T10:53:37.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:37.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:37.330757+0000) 2024-03-06T10:53:38.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:38.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:38.330997+0000) 2024-03-06T10:53:39.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:39.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:39.331232+0000) 2024-03-06T10:53:40.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:40.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:40.331462+0000) 2024-03-06T10:53:41.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:53:41.081 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:53:41.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:41.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:41.331700+0000) 2024-03-06T10:53:42.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:42.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:42.331905+0000) 2024-03-06T10:53:43.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:43.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:43.332090+0000) 2024-03-06T10:53:44.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:44.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:44.332281+0000) 2024-03-06T10:53:45.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:45.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:45.332520+0000) 2024-03-06T10:53:46.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:46.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:46.332801+0000) 2024-03-06T10:53:47.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:47.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:47.333040+0000) 2024-03-06T10:53:48.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:48.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:48.333248+0000) 2024-03-06T10:53:49.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:49.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:49.333444+0000) 2024-03-06T10:53:50.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:50.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:50.333669+0000) 2024-03-06T10:53:51.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:51.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:51.333887+0000) 2024-03-06T10:53:52.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:52.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:52.334116+0000) 2024-03-06T10:53:53.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:53.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:53.334335+0000) 2024-03-06T10:53:54.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:54.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:54.334597+0000) 2024-03-06T10:53:55.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:55.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:55.334789+0000) 2024-03-06T10:53:56.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:56.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:56.334990+0000) 2024-03-06T10:53:57.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:57.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:57.335228+0000) 2024-03-06T10:53:58.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:58.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:58.335455+0000) 2024-03-06T10:53:59.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:53:59.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:53:59.335694+0000) 2024-03-06T10:54:00.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:00.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:00.335912+0000) 2024-03-06T10:54:01.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:01.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:01.336144+0000) 2024-03-06T10:54:02.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:02.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:02.336340+0000) 2024-03-06T10:54:03.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:03.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:03.336598+0000) 2024-03-06T10:54:04.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:04.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:04.336842+0000) 2024-03-06T10:54:05.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:05.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:05.337089+0000) 2024-03-06T10:54:06.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:06.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:06.337281+0000) 2024-03-06T10:54:07.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:07.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:07.337503+0000) 2024-03-06T10:54:08.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:08.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:08.337734+0000) 2024-03-06T10:54:09.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:09.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:09.337930+0000) 2024-03-06T10:54:10.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:10.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:10.338176+0000) 2024-03-06T10:54:11.209 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:54:11.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:54:11.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:11.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:11.338356+0000) 2024-03-06T10:54:12.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:12.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:12.338625+0000) 2024-03-06T10:54:13.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:13.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:13.338849+0000) 2024-03-06T10:54:14.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:14.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:14.339073+0000) 2024-03-06T10:54:15.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:15.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:15.339252+0000) 2024-03-06T10:54:16.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:16.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:16.339527+0000) 2024-03-06T10:54:17.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:17.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:17.339792+0000) 2024-03-06T10:54:18.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:18.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:18.340036+0000) 2024-03-06T10:54:19.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:19.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:19.340270+0000) 2024-03-06T10:54:20.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:20.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:20.340513+0000) 2024-03-06T10:54:21.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:21.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:21.340809+0000) 2024-03-06T10:54:22.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:22.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:22.341054+0000) 2024-03-06T10:54:23.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:23.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:23.341289+0000) 2024-03-06T10:54:24.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:24.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:24.341514+0000) 2024-03-06T10:54:25.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:25.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:25.341786+0000) 2024-03-06T10:54:26.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:26.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:26.342029+0000) 2024-03-06T10:54:27.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:27.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:27.342269+0000) 2024-03-06T10:54:28.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:28.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:28.342508+0000) 2024-03-06T10:54:29.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:29.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:29.342752+0000) 2024-03-06T10:54:30.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:30.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:30.342977+0000) 2024-03-06T10:54:31.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:31.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:31.343190+0000) 2024-03-06T10:54:32.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:32.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:32.343429+0000) 2024-03-06T10:54:33.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:33.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:33.343693+0000) 2024-03-06T10:54:34.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:34.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:34.343912+0000) 2024-03-06T10:54:35.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:35.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:35.344146+0000) 2024-03-06T10:54:36.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:36.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:36.344365+0000) 2024-03-06T10:54:37.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:37.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:37.344627+0000) 2024-03-06T10:54:38.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:38.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:38.344857+0000) 2024-03-06T10:54:39.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:39.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:39.345074+0000) 2024-03-06T10:54:40.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:40.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:40.345267+0000) 2024-03-06T10:54:41.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:41.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:41.345482+0000) 2024-03-06T10:54:41.420 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:54:41.431 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:54:42.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:42.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:42.345718+0000) 2024-03-06T10:54:43.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:43.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:43.345899+0000) 2024-03-06T10:54:44.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:44.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:44.346125+0000) 2024-03-06T10:54:45.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:45.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:45.346356+0000) 2024-03-06T10:54:46.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:46.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:46.346625+0000) 2024-03-06T10:54:47.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:47.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:47.346843+0000) 2024-03-06T10:54:48.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:48.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:48.347077+0000) 2024-03-06T10:54:49.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:49.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:49.347266+0000) 2024-03-06T10:54:50.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:50.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:50.347504+0000) 2024-03-06T10:54:51.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:51.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:51.347732+0000) 2024-03-06T10:54:52.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:52.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:52.347965+0000) 2024-03-06T10:54:53.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:53.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:53.348157+0000) 2024-03-06T10:54:54.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:54.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:54.348381+0000) 2024-03-06T10:54:55.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:55.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:55.348652+0000) 2024-03-06T10:54:56.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:56.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:56.348923+0000) 2024-03-06T10:54:57.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:57.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:57.349190+0000) 2024-03-06T10:54:58.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:58.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:58.349440+0000) 2024-03-06T10:54:59.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:54:59.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:54:59.349665+0000) 2024-03-06T10:55:00.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:00.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:00.349919+0000) 2024-03-06T10:55:01.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:01.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:01.350196+0000) 2024-03-06T10:55:02.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:02.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:02.350467+0000) 2024-03-06T10:55:03.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:03.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:03.350747+0000) 2024-03-06T10:55:04.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:04.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:04.351069+0000) 2024-03-06T10:55:05.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:05.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:05.351362+0000) 2024-03-06T10:55:06.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:06.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:06.351666+0000) 2024-03-06T10:55:07.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:07.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:07.351943+0000) 2024-03-06T10:55:08.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:08.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:08.352165+0000) 2024-03-06T10:55:09.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:09.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:09.352441+0000) 2024-03-06T10:55:10.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:10.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:10.352734+0000) 2024-03-06T10:55:11.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:11.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:11.353010+0000) 2024-03-06T10:55:11.601 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:55:11.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:55:12.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:12.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:12.353256+0000) 2024-03-06T10:55:13.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:13.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:13.353518+0000) 2024-03-06T10:55:14.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:14.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:14.353833+0000) 2024-03-06T10:55:15.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:15.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:15.354111+0000) 2024-03-06T10:55:16.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:16.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:16.354386+0000) 2024-03-06T10:55:17.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:17.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:17.354649+0000) 2024-03-06T10:55:18.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:18.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:18.354933+0000) 2024-03-06T10:55:19.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:19.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:19.355209+0000) 2024-03-06T10:55:20.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:20.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:20.355483+0000) 2024-03-06T10:55:21.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:21.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:21.355762+0000) 2024-03-06T10:55:22.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:22.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:22.356039+0000) 2024-03-06T10:55:23.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:23.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:23.356324+0000) 2024-03-06T10:55:24.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:24.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:24.356527+0000) 2024-03-06T10:55:25.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:25.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:25.356807+0000) 2024-03-06T10:55:26.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:26.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:26.357031+0000) 2024-03-06T10:55:27.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:27.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:27.357218+0000) 2024-03-06T10:55:28.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:28.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:28.357439+0000) 2024-03-06T10:55:29.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:29.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:29.357698+0000) 2024-03-06T10:55:30.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:30.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:30.357899+0000) 2024-03-06T10:55:31.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:31.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:31.358105+0000) 2024-03-06T10:55:32.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:32.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:32.358342+0000) 2024-03-06T10:55:33.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:33.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:33.358603+0000) 2024-03-06T10:55:34.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:34.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:34.358829+0000) 2024-03-06T10:55:35.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:35.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:35.359024+0000) 2024-03-06T10:55:36.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:36.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:36.359312+0000) 2024-03-06T10:55:37.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:37.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:37.359637+0000) 2024-03-06T10:55:38.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:38.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:38.359917+0000) 2024-03-06T10:55:39.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:39.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:39.360195+0000) 2024-03-06T10:55:40.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:40.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:40.360485+0000) 2024-03-06T10:55:41.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:41.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:41.360769+0000) 2024-03-06T10:55:41.765 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:55:41.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:55:42.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:42.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:42.361048+0000) 2024-03-06T10:55:43.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:43.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:43.361328+0000) 2024-03-06T10:55:44.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:44.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:44.361524+0000) 2024-03-06T10:55:45.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:45.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:45.361808+0000) 2024-03-06T10:55:46.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:46.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:46.362033+0000) 2024-03-06T10:55:47.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:47.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:47.362257+0000) 2024-03-06T10:55:48.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:48.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:48.362486+0000) 2024-03-06T10:55:49.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:49.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:49.362714+0000) 2024-03-06T10:55:50.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:50.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:50.362986+0000) 2024-03-06T10:55:51.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:51.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:51.363224+0000) 2024-03-06T10:55:52.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:52.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:52.363419+0000) 2024-03-06T10:55:53.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:53.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:53.363684+0000) 2024-03-06T10:55:54.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:54.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:54.363914+0000) 2024-03-06T10:55:55.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:55.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:55.364166+0000) 2024-03-06T10:55:56.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:56.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:56.364395+0000) 2024-03-06T10:55:57.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:57.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:57.364679+0000) 2024-03-06T10:55:58.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:58.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:58.364918+0000) 2024-03-06T10:55:59.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:55:59.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:55:59.365156+0000) 2024-03-06T10:56:00.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:00.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:00.365379+0000) 2024-03-06T10:56:01.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:01.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:01.365645+0000) 2024-03-06T10:56:02.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:02.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:02.365886+0000) 2024-03-06T10:56:03.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:03.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:03.366121+0000) 2024-03-06T10:56:04.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:04.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:04.366356+0000) 2024-03-06T10:56:05.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:05.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:05.366676+0000) 2024-03-06T10:56:06.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:06.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:06.366971+0000) 2024-03-06T10:56:07.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:07.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:07.367238+0000) 2024-03-06T10:56:08.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:08.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:08.367511+0000) 2024-03-06T10:56:09.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:09.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:09.367794+0000) 2024-03-06T10:56:10.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:10.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:10.368028+0000) 2024-03-06T10:56:11.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:11.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:11.368257+0000) 2024-03-06T10:56:11.946 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:56:11.954 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:56:12.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:12.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:12.368504+0000) 2024-03-06T10:56:13.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:13.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:13.368796+0000) 2024-03-06T10:56:14.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:14.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:14.369036+0000) 2024-03-06T10:56:15.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:15.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:15.369286+0000) 2024-03-06T10:56:16.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:16.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:16.369529+0000) 2024-03-06T10:56:17.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:17.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:17.369853+0000) 2024-03-06T10:56:18.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:18.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:18.370131+0000) 2024-03-06T10:56:19.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:19.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:19.370405+0000) 2024-03-06T10:56:20.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:20.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:20.370689+0000) 2024-03-06T10:56:21.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:21.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:21.370930+0000) 2024-03-06T10:56:22.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:22.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:22.371160+0000) 2024-03-06T10:56:23.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:23.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:23.371346+0000) 2024-03-06T10:56:24.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:24.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:24.371613+0000) 2024-03-06T10:56:25.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:25.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:25.371843+0000) 2024-03-06T10:56:26.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:26.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:26.372064+0000) 2024-03-06T10:56:27.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:27.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:27.372284+0000) 2024-03-06T10:56:28.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:28.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:28.372515+0000) 2024-03-06T10:56:29.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:29.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:29.372781+0000) 2024-03-06T10:56:30.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:30.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:30.373018+0000) 2024-03-06T10:56:31.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:31.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:31.373240+0000) 2024-03-06T10:56:32.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:32.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:32.373468+0000) 2024-03-06T10:56:33.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:33.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:33.373687+0000) 2024-03-06T10:56:34.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:34.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:34.373911+0000) 2024-03-06T10:56:35.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:35.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:35.374150+0000) 2024-03-06T10:56:36.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:36.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:36.374380+0000) 2024-03-06T10:56:37.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:37.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:37.374627+0000) 2024-03-06T10:56:38.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:38.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:38.374857+0000) 2024-03-06T10:56:39.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:39.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:39.375077+0000) 2024-03-06T10:56:40.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:40.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:40.375312+0000) 2024-03-06T10:56:41.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:41.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:41.375564+0000) 2024-03-06T10:56:42.144 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:56:42.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:56:42.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:42.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:42.375781+0000) 2024-03-06T10:56:43.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:43.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:43.376000+0000) 2024-03-06T10:56:44.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:44.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:44.376178+0000) 2024-03-06T10:56:45.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:45.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:45.376413+0000) 2024-03-06T10:56:46.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:46.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:46.376659+0000) 2024-03-06T10:56:47.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:47.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:47.376844+0000) 2024-03-06T10:56:48.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:48.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:48.377032+0000) 2024-03-06T10:56:49.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:49.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:49.377266+0000) 2024-03-06T10:56:50.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:50.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:50.377515+0000) 2024-03-06T10:56:51.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:51.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:51.377854+0000) 2024-03-06T10:56:52.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:52.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:52.378129+0000) 2024-03-06T10:56:53.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:53.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:53.378373+0000) 2024-03-06T10:56:54.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:54.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:54.378636+0000) 2024-03-06T10:56:55.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:55.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:55.378909+0000) 2024-03-06T10:56:56.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:56.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:56.379184+0000) 2024-03-06T10:56:57.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:57.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:57.379462+0000) 2024-03-06T10:56:58.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:58.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:58.379708+0000) 2024-03-06T10:56:59.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:56:59.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:56:59.379969+0000) 2024-03-06T10:57:00.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:00.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:00.380174+0000) 2024-03-06T10:57:01.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:01.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:01.380399+0000) 2024-03-06T10:57:02.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:02.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:02.380696+0000) 2024-03-06T10:57:03.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:03.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:03.380965+0000) 2024-03-06T10:57:04.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:04.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:04.381202+0000) 2024-03-06T10:57:05.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:05.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:05.381481+0000) 2024-03-06T10:57:06.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:06.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:06.381701+0000) 2024-03-06T10:57:07.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:07.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:07.381888+0000) 2024-03-06T10:57:08.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:08.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:08.382120+0000) 2024-03-06T10:57:09.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:09.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:09.382333+0000) 2024-03-06T10:57:10.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:10.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:10.382665+0000) 2024-03-06T10:57:11.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:11.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:11.382871+0000) 2024-03-06T10:57:12.367 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:57:12.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:57:12.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:12.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:12.383040+0000) 2024-03-06T10:57:13.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:13.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:13.383250+0000) 2024-03-06T10:57:14.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:14.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:14.383476+0000) 2024-03-06T10:57:15.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:15.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:15.383727+0000) 2024-03-06T10:57:16.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:16.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:16.383963+0000) 2024-03-06T10:57:17.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:17.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:17.384183+0000) 2024-03-06T10:57:18.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:18.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:18.384415+0000) 2024-03-06T10:57:19.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:19.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:19.384637+0000) 2024-03-06T10:57:20.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:20.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:20.384873+0000) 2024-03-06T10:57:21.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:21.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:21.385138+0000) 2024-03-06T10:57:22.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:22.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:22.385421+0000) 2024-03-06T10:57:23.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:23.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:23.385700+0000) 2024-03-06T10:57:24.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:24.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:24.385984+0000) 2024-03-06T10:57:25.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:25.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:25.386311+0000) 2024-03-06T10:57:26.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:26.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:26.386638+0000) 2024-03-06T10:57:27.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:27.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:27.386895+0000) 2024-03-06T10:57:28.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:28.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:28.387172+0000) 2024-03-06T10:57:29.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:29.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:29.387447+0000) 2024-03-06T10:57:30.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:30.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:30.387743+0000) 2024-03-06T10:57:31.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:31.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:31.388023+0000) 2024-03-06T10:57:32.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:32.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:32.388301+0000) 2024-03-06T10:57:33.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:33.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:33.388661+0000) 2024-03-06T10:57:34.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:34.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:34.388868+0000) 2024-03-06T10:57:35.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:35.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:35.389085+0000) 2024-03-06T10:57:36.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:36.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:36.389300+0000) 2024-03-06T10:57:37.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:37.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:37.389569+0000) 2024-03-06T10:57:38.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:38.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:38.389848+0000) 2024-03-06T10:57:39.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:39.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:39.390127+0000) 2024-03-06T10:57:40.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:40.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:40.390344+0000) 2024-03-06T10:57:41.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:41.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:41.390603+0000) 2024-03-06T10:57:42.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:42.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:42.390894+0000) 2024-03-06T10:57:42.514 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:57:42.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:57:43.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:43.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:43.391135+0000) 2024-03-06T10:57:44.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:44.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:44.391400+0000) 2024-03-06T10:57:45.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:45.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:45.391710+0000) 2024-03-06T10:57:46.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:46.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:46.392001+0000) 2024-03-06T10:57:47.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:47.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:47.392238+0000) 2024-03-06T10:57:48.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:48.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:48.392466+0000) 2024-03-06T10:57:49.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:49.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:49.392661+0000) 2024-03-06T10:57:50.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:50.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:50.392876+0000) 2024-03-06T10:57:51.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:51.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:51.393078+0000) 2024-03-06T10:57:52.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:52.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:52.393317+0000) 2024-03-06T10:57:53.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:53.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:53.393556+0000) 2024-03-06T10:57:54.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:54.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:54.393782+0000) 2024-03-06T10:57:55.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:55.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:55.394023+0000) 2024-03-06T10:57:56.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:56.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:56.394269+0000) 2024-03-06T10:57:57.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:57.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:57.394581+0000) 2024-03-06T10:57:58.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:58.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:58.394838+0000) 2024-03-06T10:57:59.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:57:59.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:57:59.395082+0000) 2024-03-06T10:58:00.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:00.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:00.395347+0000) 2024-03-06T10:58:01.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:01.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:01.395581+0000) 2024-03-06T10:58:02.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:02.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:02.395804+0000) 2024-03-06T10:58:03.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:03.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:03.396024+0000) 2024-03-06T10:58:04.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:04.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:04.396249+0000) 2024-03-06T10:58:05.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:05.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:05.396485+0000) 2024-03-06T10:58:06.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:06.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:06.396710+0000) 2024-03-06T10:58:07.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:07.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:07.396928+0000) 2024-03-06T10:58:08.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:08.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:08.397156+0000) 2024-03-06T10:58:09.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:09.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:09.397374+0000) 2024-03-06T10:58:10.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:10.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:10.397578+0000) 2024-03-06T10:58:11.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:11.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:11.397795+0000) 2024-03-06T10:58:12.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:12.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:12.398018+0000) 2024-03-06T10:58:12.647 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:58:12.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:58:13.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:13.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:13.398242+0000) 2024-03-06T10:58:14.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:14.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:14.398465+0000) 2024-03-06T10:58:15.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:15.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:15.398699+0000) 2024-03-06T10:58:16.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:16.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:16.398891+0000) 2024-03-06T10:58:17.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:17.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:17.399120+0000) 2024-03-06T10:58:18.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:18.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:18.399362+0000) 2024-03-06T10:58:19.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:19.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:19.399623+0000) 2024-03-06T10:58:20.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:20.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:20.399801+0000) 2024-03-06T10:58:21.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:21.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:21.400060+0000) 2024-03-06T10:58:22.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:22.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:22.400260+0000) 2024-03-06T10:58:23.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:23.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:23.400487+0000) 2024-03-06T10:58:24.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:24.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:24.400716+0000) 2024-03-06T10:58:25.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:25.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:25.400945+0000) 2024-03-06T10:58:26.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:26.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:26.401176+0000) 2024-03-06T10:58:27.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:27.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:27.401448+0000) 2024-03-06T10:58:28.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:28.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:28.401696+0000) 2024-03-06T10:58:29.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:29.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:29.401923+0000) 2024-03-06T10:58:30.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:30.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:30.402147+0000) 2024-03-06T10:58:31.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:31.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:31.402370+0000) 2024-03-06T10:58:32.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:32.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:32.402638+0000) 2024-03-06T10:58:33.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:33.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:33.402859+0000) 2024-03-06T10:58:34.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:34.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:34.403113+0000) 2024-03-06T10:58:35.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:35.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:35.403359+0000) 2024-03-06T10:58:36.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:36.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:36.403639+0000) 2024-03-06T10:58:37.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:37.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:37.403826+0000) 2024-03-06T10:58:38.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:38.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:38.404052+0000) 2024-03-06T10:58:39.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:39.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:39.404333+0000) 2024-03-06T10:58:40.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:40.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:40.404637+0000) 2024-03-06T10:58:41.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:41.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:41.404793+0000) 2024-03-06T10:58:42.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:42.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:42.405021+0000) 2024-03-06T10:58:42.845 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:58:42.853 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:58:43.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:43.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:43.405259+0000) 2024-03-06T10:58:44.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:44.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:44.405524+0000) 2024-03-06T10:58:45.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:45.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:45.405844+0000) 2024-03-06T10:58:46.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:46.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:46.406056+0000) 2024-03-06T10:58:47.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:47.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:47.406265+0000) 2024-03-06T10:58:48.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:48.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:48.406501+0000) 2024-03-06T10:58:49.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:49.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:49.406721+0000) 2024-03-06T10:58:50.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:50.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:50.406966+0000) 2024-03-06T10:58:51.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:51.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:51.407165+0000) 2024-03-06T10:58:52.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:52.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:52.407359+0000) 2024-03-06T10:58:53.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:53.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:53.407562+0000) 2024-03-06T10:58:54.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:54.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:54.407791+0000) 2024-03-06T10:58:55.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:55.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:55.408208+0000) 2024-03-06T10:58:56.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:56.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:56.408400+0000) 2024-03-06T10:58:57.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:57.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:57.408647+0000) 2024-03-06T10:58:58.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:58.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:58.408875+0000) 2024-03-06T10:58:59.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:58:59.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:58:59.409065+0000) 2024-03-06T10:59:00.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:00.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:00.409304+0000) 2024-03-06T10:59:01.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:01.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:01.409526+0000) 2024-03-06T10:59:02.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:02.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:02.409753+0000) 2024-03-06T10:59:03.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:03.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:03.409971+0000) 2024-03-06T10:59:04.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:04.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:04.410198+0000) 2024-03-06T10:59:05.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:05.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:05.410423+0000) 2024-03-06T10:59:06.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:06.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:06.410659+0000) 2024-03-06T10:59:07.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:07.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:07.410880+0000) 2024-03-06T10:59:08.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:08.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:08.411084+0000) 2024-03-06T10:59:09.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:09.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:09.411267+0000) 2024-03-06T10:59:10.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:10.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:10.411445+0000) 2024-03-06T10:59:11.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:11.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:11.411725+0000) 2024-03-06T10:59:12.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:12.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:12.411971+0000) 2024-03-06T10:59:13.017 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:59:13.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:59:13.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:13.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:13.412164+0000) 2024-03-06T10:59:14.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:14.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:14.412404+0000) 2024-03-06T10:59:15.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:15.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:15.412742+0000) 2024-03-06T10:59:16.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:16.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:16.412938+0000) 2024-03-06T10:59:17.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:17.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:17.413157+0000) 2024-03-06T10:59:18.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:18.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:18.413395+0000) 2024-03-06T10:59:19.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:19.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:19.413646+0000) 2024-03-06T10:59:20.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:20.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:20.413844+0000) 2024-03-06T10:59:21.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:21.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:21.414034+0000) 2024-03-06T10:59:22.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:22.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:22.414256+0000) 2024-03-06T10:59:23.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:23.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:23.414498+0000) 2024-03-06T10:59:24.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:24.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:24.414702+0000) 2024-03-06T10:59:25.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:25.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:25.414870+0000) 2024-03-06T10:59:26.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:26.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:26.415075+0000) 2024-03-06T10:59:27.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:27.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:27.415305+0000) 2024-03-06T10:59:28.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:28.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:28.415527+0000) 2024-03-06T10:59:29.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:29.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:29.415736+0000) 2024-03-06T10:59:30.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:30.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:30.415927+0000) 2024-03-06T10:59:31.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:31.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:31.416140+0000) 2024-03-06T10:59:32.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:32.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:32.416347+0000) 2024-03-06T10:59:33.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:33.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:33.416599+0000) 2024-03-06T10:59:34.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:34.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:34.416883+0000) 2024-03-06T10:59:35.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:35.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:35.417197+0000) 2024-03-06T10:59:36.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:36.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:36.417404+0000) 2024-03-06T10:59:37.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:37.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:37.417640+0000) 2024-03-06T10:59:38.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:38.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:38.417840+0000) 2024-03-06T10:59:39.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:39.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:39.418044+0000) 2024-03-06T10:59:40.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:40.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:40.418244+0000) 2024-03-06T10:59:41.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:41.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:41.418458+0000) 2024-03-06T10:59:42.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:42.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:42.418700+0000) 2024-03-06T10:59:43.223 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:59:43.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T10:59:43.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:43.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:43.418924+0000) 2024-03-06T10:59:44.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:44.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:44.419204+0000) 2024-03-06T10:59:45.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:45.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:45.419435+0000) 2024-03-06T10:59:46.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:46.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:46.419677+0000) 2024-03-06T10:59:47.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:47.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:47.419935+0000) 2024-03-06T10:59:48.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:48.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:48.420147+0000) 2024-03-06T10:59:49.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:49.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:49.420339+0000) 2024-03-06T10:59:50.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:50.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:50.420594+0000) 2024-03-06T10:59:51.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:51.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:51.420787+0000) 2024-03-06T10:59:52.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:52.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:52.420991+0000) 2024-03-06T10:59:53.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:53.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:53.421205+0000) 2024-03-06T10:59:54.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:54.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:54.421484+0000) 2024-03-06T10:59:55.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:55.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:55.421824+0000) 2024-03-06T10:59:56.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:56.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:56.422059+0000) 2024-03-06T10:59:57.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:57.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:57.422333+0000) 2024-03-06T10:59:58.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:58.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:58.422575+0000) 2024-03-06T10:59:59.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T10:59:59.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T09:59:59.422798+0000) 2024-03-06T11:00:00.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:00.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:00.423012+0000) 2024-03-06T11:00:01.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:01.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:01.423218+0000) 2024-03-06T11:00:02.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:02.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:02.423404+0000) 2024-03-06T11:00:03.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:03.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:03.423640+0000) 2024-03-06T11:00:04.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:04.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:04.423831+0000) 2024-03-06T11:00:05.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:05.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:05.424023+0000) 2024-03-06T11:00:06.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:06.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:06.424217+0000) 2024-03-06T11:00:07.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:07.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:07.424457+0000) 2024-03-06T11:00:08.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:08.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:08.424686+0000) 2024-03-06T11:00:09.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:09.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:09.424905+0000) 2024-03-06T11:00:10.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:10.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:10.425094+0000) 2024-03-06T11:00:11.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:11.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:11.425311+0000) 2024-03-06T11:00:12.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:12.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:12.425555+0000) 2024-03-06T11:00:13.412 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:00:13.420 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:00:13.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:13.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:13.425754+0000) 2024-03-06T11:00:14.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:14.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:14.425954+0000) 2024-03-06T11:00:15.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:15.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:15.426178+0000) 2024-03-06T11:00:16.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:16.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:16.426403+0000) 2024-03-06T11:00:17.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:17.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:17.426651+0000) 2024-03-06T11:00:18.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:18.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:18.426837+0000) 2024-03-06T11:00:19.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:19.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:19.427025+0000) 2024-03-06T11:00:20.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:20.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:20.427251+0000) 2024-03-06T11:00:21.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:21.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:21.427468+0000) 2024-03-06T11:00:22.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:22.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:22.427690+0000) 2024-03-06T11:00:23.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:23.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:23.427908+0000) 2024-03-06T11:00:24.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:24.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:24.428114+0000) 2024-03-06T11:00:25.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:25.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:25.428371+0000) 2024-03-06T11:00:26.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:26.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:26.428622+0000) 2024-03-06T11:00:27.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:27.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:27.428850+0000) 2024-03-06T11:00:28.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:28.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:28.429082+0000) 2024-03-06T11:00:29.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:29.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:29.429311+0000) 2024-03-06T11:00:30.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:30.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:30.429487+0000) 2024-03-06T11:00:31.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:31.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:31.429679+0000) 2024-03-06T11:00:32.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:32.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:32.429911+0000) 2024-03-06T11:00:33.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:33.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:33.430142+0000) 2024-03-06T11:00:34.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:34.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:34.430366+0000) 2024-03-06T11:00:35.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:35.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:35.430630+0000) 2024-03-06T11:00:36.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:36.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:36.430855+0000) 2024-03-06T11:00:37.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:37.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:37.431037+0000) 2024-03-06T11:00:38.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:38.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:38.431226+0000) 2024-03-06T11:00:39.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:39.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:39.431416+0000) 2024-03-06T11:00:40.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:40.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:40.431656+0000) 2024-03-06T11:00:41.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:41.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:41.431854+0000) 2024-03-06T11:00:42.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:42.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:42.432042+0000) 2024-03-06T11:00:43.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:43.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:43.432258+0000) 2024-03-06T11:00:43.553 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:00:43.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:00:44.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:44.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:44.432483+0000) 2024-03-06T11:00:45.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:45.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:45.432676+0000) 2024-03-06T11:00:46.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:46.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:46.432867+0000) 2024-03-06T11:00:47.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:47.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:47.433087+0000) 2024-03-06T11:00:48.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:48.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:48.433321+0000) 2024-03-06T11:00:49.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:49.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:49.433492+0000) 2024-03-06T11:00:50.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:50.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:50.433694+0000) 2024-03-06T11:00:51.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:51.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:51.433886+0000) 2024-03-06T11:00:52.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:52.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:52.434078+0000) 2024-03-06T11:00:53.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:53.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:53.434305+0000) 2024-03-06T11:00:54.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:54.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:54.434483+0000) 2024-03-06T11:00:55.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:55.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:55.434714+0000) 2024-03-06T11:00:56.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:56.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:56.434918+0000) 2024-03-06T11:00:57.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:57.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:57.435117+0000) 2024-03-06T11:00:58.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:58.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:58.435347+0000) 2024-03-06T11:00:59.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:00:59.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:00:59.435572+0000) 2024-03-06T11:01:00.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:00.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:00.435781+0000) 2024-03-06T11:01:01.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:01.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:01.435973+0000) 2024-03-06T11:01:02.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:02.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:02.436202+0000) 2024-03-06T11:01:03.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:03.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:03.436393+0000) 2024-03-06T11:01:04.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:04.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:04.436670+0000) 2024-03-06T11:01:05.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:05.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:05.436879+0000) 2024-03-06T11:01:06.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:06.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:06.437089+0000) 2024-03-06T11:01:07.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:07.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:07.437310+0000) 2024-03-06T11:01:08.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:08.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:08.437559+0000) 2024-03-06T11:01:09.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:09.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:09.437750+0000) 2024-03-06T11:01:10.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:10.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:10.437942+0000) 2024-03-06T11:01:11.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:11.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:11.438122+0000) 2024-03-06T11:01:12.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:12.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:12.438350+0000) 2024-03-06T11:01:13.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:13.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:13.438621+0000) 2024-03-06T11:01:13.691 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:01:13.699 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:01:14.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:14.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:14.438869+0000) 2024-03-06T11:01:15.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:15.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:15.439118+0000) 2024-03-06T11:01:16.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:16.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:16.439359+0000) 2024-03-06T11:01:17.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:17.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:17.439638+0000) 2024-03-06T11:01:18.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:18.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:18.439875+0000) 2024-03-06T11:01:19.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:19.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:19.440117+0000) 2024-03-06T11:01:20.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:20.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:20.440354+0000) 2024-03-06T11:01:21.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:21.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:21.440627+0000) 2024-03-06T11:01:22.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:22.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:22.440871+0000) 2024-03-06T11:01:23.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:23.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:23.441111+0000) 2024-03-06T11:01:24.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:24.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:24.441347+0000) 2024-03-06T11:01:25.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:25.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:25.441622+0000) 2024-03-06T11:01:26.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:26.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:26.441873+0000) 2024-03-06T11:01:27.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:27.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:27.442115+0000) 2024-03-06T11:01:28.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:28.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:28.442368+0000) 2024-03-06T11:01:29.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:29.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:29.442640+0000) 2024-03-06T11:01:30.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:30.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:30.442874+0000) 2024-03-06T11:01:31.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:31.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:31.443095+0000) 2024-03-06T11:01:32.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:32.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:32.443292+0000) 2024-03-06T11:01:33.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:33.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:33.443556+0000) 2024-03-06T11:01:34.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:34.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:34.443751+0000) 2024-03-06T11:01:35.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:35.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:35.443950+0000) 2024-03-06T11:01:36.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:36.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:36.444167+0000) 2024-03-06T11:01:37.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:37.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:37.444405+0000) 2024-03-06T11:01:38.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:38.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:38.444631+0000) 2024-03-06T11:01:39.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:39.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:39.444817+0000) 2024-03-06T11:01:40.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:40.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:40.445003+0000) 2024-03-06T11:01:41.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:41.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:41.445219+0000) 2024-03-06T11:01:42.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:42.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:42.445446+0000) 2024-03-06T11:01:43.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:43.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:43.445682+0000) 2024-03-06T11:01:43.906 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:01:43.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:01:44.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:44.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:44.445934+0000) 2024-03-06T11:01:45.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:45.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:45.446168+0000) 2024-03-06T11:01:46.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:46.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:46.446392+0000) 2024-03-06T11:01:47.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:47.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:47.446653+0000) 2024-03-06T11:01:48.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:48.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:48.446844+0000) 2024-03-06T11:01:49.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:49.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:49.447036+0000) 2024-03-06T11:01:50.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:50.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:50.447212+0000) 2024-03-06T11:01:51.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:51.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:51.447450+0000) 2024-03-06T11:01:52.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:52.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:52.447689+0000) 2024-03-06T11:01:53.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:53.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:53.447931+0000) 2024-03-06T11:01:54.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:54.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:54.448187+0000) 2024-03-06T11:01:55.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:55.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:55.448439+0000) 2024-03-06T11:01:56.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:56.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:56.448701+0000) 2024-03-06T11:01:57.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:57.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:57.448940+0000) 2024-03-06T11:01:58.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:58.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:58.449182+0000) 2024-03-06T11:01:59.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:01:59.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:01:59.449418+0000) 2024-03-06T11:02:00.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:00.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:00.449674+0000) 2024-03-06T11:02:01.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:01.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:01.449914+0000) 2024-03-06T11:02:02.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:02.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:02.450117+0000) 2024-03-06T11:02:03.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:03.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:03.450334+0000) 2024-03-06T11:02:04.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:04.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:04.450585+0000) 2024-03-06T11:02:05.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:05.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:05.450775+0000) 2024-03-06T11:02:06.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:06.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:06.450974+0000) 2024-03-06T11:02:07.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:07.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:07.451172+0000) 2024-03-06T11:02:08.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:08.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:08.451408+0000) 2024-03-06T11:02:09.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:09.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:09.451636+0000) 2024-03-06T11:02:10.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:10.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:10.451872+0000) 2024-03-06T11:02:11.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:11.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:11.452072+0000) 2024-03-06T11:02:12.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:12.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:12.452302+0000) 2024-03-06T11:02:13.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:13.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:13.452570+0000) 2024-03-06T11:02:14.052 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:02:14.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:02:14.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:14.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:14.452799+0000) 2024-03-06T11:02:15.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:15.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:15.453043+0000) 2024-03-06T11:02:16.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:16.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:16.453283+0000) 2024-03-06T11:02:17.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:17.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:17.453467+0000) 2024-03-06T11:02:18.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:18.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:18.453685+0000) 2024-03-06T11:02:19.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:19.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:19.453872+0000) 2024-03-06T11:02:20.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:20.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:20.454104+0000) 2024-03-06T11:02:21.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:21.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:21.454334+0000) 2024-03-06T11:02:22.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:22.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:22.454582+0000) 2024-03-06T11:02:23.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:23.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:23.454764+0000) 2024-03-06T11:02:24.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:24.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:24.454975+0000) 2024-03-06T11:02:25.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:25.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:25.455221+0000) 2024-03-06T11:02:26.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:26.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:26.455460+0000) 2024-03-06T11:02:27.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:27.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:27.455681+0000) 2024-03-06T11:02:28.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:28.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:28.455871+0000) 2024-03-06T11:02:29.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:29.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:29.456110+0000) 2024-03-06T11:02:30.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:30.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:30.456293+0000) 2024-03-06T11:02:31.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:31.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:31.456516+0000) 2024-03-06T11:02:32.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:32.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:32.456863+0000) 2024-03-06T11:02:33.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:33.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:33.457071+0000) 2024-03-06T11:02:34.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:34.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:34.457274+0000) 2024-03-06T11:02:35.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:35.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:35.457506+0000) 2024-03-06T11:02:36.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:36.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:36.457748+0000) 2024-03-06T11:02:37.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:37.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:37.457949+0000) 2024-03-06T11:02:38.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:38.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:38.458138+0000) 2024-03-06T11:02:39.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:39.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:39.458351+0000) 2024-03-06T11:02:40.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:40.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:40.458572+0000) 2024-03-06T11:02:41.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:41.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:41.458811+0000) 2024-03-06T11:02:42.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:42.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:42.459017+0000) 2024-03-06T11:02:43.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:43.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:43.459222+0000) 2024-03-06T11:02:44.274 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:02:44.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:02:44.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:44.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:44.459431+0000) 2024-03-06T11:02:45.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:45.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:45.459687+0000) 2024-03-06T11:02:46.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:46.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:46.459888+0000) 2024-03-06T11:02:47.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:47.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:47.460082+0000) 2024-03-06T11:02:48.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:48.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:48.460309+0000) 2024-03-06T11:02:49.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:49.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:49.460520+0000) 2024-03-06T11:02:50.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:50.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:50.460805+0000) 2024-03-06T11:02:51.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:51.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:51.461016+0000) 2024-03-06T11:02:52.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:52.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:52.461251+0000) 2024-03-06T11:02:53.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:53.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:53.461497+0000) 2024-03-06T11:02:54.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:54.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:54.461736+0000) 2024-03-06T11:02:55.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:55.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:55.461926+0000) 2024-03-06T11:02:56.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:56.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:56.462136+0000) 2024-03-06T11:02:57.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:57.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:57.462320+0000) 2024-03-06T11:02:58.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:58.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:58.462500+0000) 2024-03-06T11:02:59.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:02:59.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:02:59.462739+0000) 2024-03-06T11:03:00.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:00.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:00.462949+0000) 2024-03-06T11:03:01.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:01.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:01.463171+0000) 2024-03-06T11:03:02.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:02.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:02.463342+0000) 2024-03-06T11:03:03.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:03.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:03.463604+0000) 2024-03-06T11:03:04.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:04.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:04.463810+0000) 2024-03-06T11:03:05.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:05.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:05.464019+0000) 2024-03-06T11:03:06.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:06.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:06.464230+0000) 2024-03-06T11:03:07.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:07.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:07.464409+0000) 2024-03-06T11:03:08.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:08.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:08.464659+0000) 2024-03-06T11:03:09.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:09.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:09.464846+0000) 2024-03-06T11:03:10.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:10.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:10.465036+0000) 2024-03-06T11:03:11.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:11.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:11.465223+0000) 2024-03-06T11:03:12.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:12.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:12.465466+0000) 2024-03-06T11:03:13.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:13.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:13.465685+0000) 2024-03-06T11:03:14.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:14.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:14.465881+0000) 2024-03-06T11:03:14.490 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:03:14.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:03:15.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:15.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:15.466137+0000) 2024-03-06T11:03:16.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:16.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:16.466327+0000) 2024-03-06T11:03:17.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:17.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:17.466577+0000) 2024-03-06T11:03:18.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:18.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:18.466768+0000) 2024-03-06T11:03:19.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:19.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:19.466947+0000) 2024-03-06T11:03:20.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:20.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:20.467143+0000) 2024-03-06T11:03:21.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:21.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:21.467304+0000) 2024-03-06T11:03:22.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:22.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:22.467584+0000) 2024-03-06T11:03:23.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:23.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:23.467794+0000) 2024-03-06T11:03:24.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:24.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:24.468033+0000) 2024-03-06T11:03:25.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:25.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:25.468272+0000) 2024-03-06T11:03:26.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:26.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:26.468490+0000) 2024-03-06T11:03:27.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:27.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:27.468741+0000) 2024-03-06T11:03:28.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:28.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:28.468940+0000) 2024-03-06T11:03:29.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:29.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:29.469104+0000) 2024-03-06T11:03:30.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:30.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:30.469342+0000) 2024-03-06T11:03:31.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:31.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:31.469599+0000) 2024-03-06T11:03:32.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:32.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:32.469795+0000) 2024-03-06T11:03:33.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:33.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:33.469985+0000) 2024-03-06T11:03:34.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:34.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:34.470205+0000) 2024-03-06T11:03:35.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:35.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:35.470428+0000) 2024-03-06T11:03:36.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:36.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:36.470684+0000) 2024-03-06T11:03:37.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:37.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:37.470879+0000) 2024-03-06T11:03:38.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:38.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:38.471115+0000) 2024-03-06T11:03:39.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:39.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:39.471329+0000) 2024-03-06T11:03:40.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:40.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:40.471487+0000) 2024-03-06T11:03:41.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:41.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:41.471709+0000) 2024-03-06T11:03:42.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:42.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:42.471929+0000) 2024-03-06T11:03:43.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:43.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:43.472077+0000) 2024-03-06T11:03:44.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:44.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:44.472279+0000) 2024-03-06T11:03:44.708 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:03:44.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:03:45.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:45.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:45.472476+0000) 2024-03-06T11:03:46.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:46.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:46.472689+0000) 2024-03-06T11:03:47.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:47.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:47.472886+0000) 2024-03-06T11:03:48.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:48.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:48.473077+0000) 2024-03-06T11:03:49.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:49.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:49.473304+0000) 2024-03-06T11:03:50.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:50.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:50.473527+0000) 2024-03-06T11:03:51.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:51.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:51.473699+0000) 2024-03-06T11:03:52.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:52.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:52.473944+0000) 2024-03-06T11:03:53.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:53.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:53.474151+0000) 2024-03-06T11:03:54.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:54.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:54.474411+0000) 2024-03-06T11:03:55.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:55.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:55.474644+0000) 2024-03-06T11:03:56.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:56.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:56.474872+0000) 2024-03-06T11:03:57.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:57.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:57.475026+0000) 2024-03-06T11:03:58.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:58.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:58.475233+0000) 2024-03-06T11:03:59.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:03:59.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:03:59.475402+0000) 2024-03-06T11:04:00.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:00.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:00.475646+0000) 2024-03-06T11:04:01.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:01.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:01.475859+0000) 2024-03-06T11:04:02.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:02.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:02.476093+0000) 2024-03-06T11:04:03.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:03.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:03.476256+0000) 2024-03-06T11:04:04.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:04.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:04.476493+0000) 2024-03-06T11:04:05.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:05.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:05.476689+0000) 2024-03-06T11:04:06.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:06.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:06.476885+0000) 2024-03-06T11:04:07.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:07.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:07.477086+0000) 2024-03-06T11:04:08.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:08.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:08.477251+0000) 2024-03-06T11:04:09.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:09.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:09.477529+0000) 2024-03-06T11:04:10.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:10.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:10.477819+0000) 2024-03-06T11:04:11.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:11.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:11.478053+0000) 2024-03-06T11:04:12.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:12.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:12.478280+0000) 2024-03-06T11:04:13.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:13.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:13.478480+0000) 2024-03-06T11:04:14.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:14.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:14.478694+0000) 2024-03-06T11:04:14.927 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:04:14.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:04:15.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:15.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:15.478911+0000) 2024-03-06T11:04:16.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:16.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:16.479075+0000) 2024-03-06T11:04:17.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:17.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:17.479287+0000) 2024-03-06T11:04:18.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:18.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:18.479498+0000) 2024-03-06T11:04:19.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:19.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:19.479783+0000) 2024-03-06T11:04:20.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:20.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:20.479977+0000) 2024-03-06T11:04:21.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:21.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:21.480168+0000) 2024-03-06T11:04:22.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:22.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:22.480363+0000) 2024-03-06T11:04:23.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:23.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:23.480524+0000) 2024-03-06T11:04:24.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:24.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:24.480734+0000) 2024-03-06T11:04:25.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:25.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:25.480949+0000) 2024-03-06T11:04:26.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:26.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:26.481135+0000) 2024-03-06T11:04:27.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:27.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:27.481358+0000) 2024-03-06T11:04:28.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:28.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:28.481608+0000) 2024-03-06T11:04:29.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:29.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:29.481799+0000) 2024-03-06T11:04:30.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:30.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:30.482052+0000) 2024-03-06T11:04:31.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:31.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:31.482278+0000) 2024-03-06T11:04:32.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:32.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:32.482516+0000) 2024-03-06T11:04:33.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:33.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:33.482793+0000) 2024-03-06T11:04:34.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:34.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:34.483012+0000) 2024-03-06T11:04:35.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:35.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:35.483189+0000) 2024-03-06T11:04:36.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:36.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:36.483364+0000) 2024-03-06T11:04:37.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:37.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:37.483637+0000) 2024-03-06T11:04:38.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:38.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:38.483847+0000) 2024-03-06T11:04:39.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:39.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:39.484044+0000) 2024-03-06T11:04:40.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:40.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:40.484245+0000) 2024-03-06T11:04:41.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:41.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:41.484464+0000) 2024-03-06T11:04:42.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:42.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:42.484672+0000) 2024-03-06T11:04:43.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:43.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:43.484912+0000) 2024-03-06T11:04:44.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:44.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:44.485143+0000) 2024-03-06T11:04:45.151 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:04:45.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:04:45.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:45.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:45.485389+0000) 2024-03-06T11:04:46.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:46.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:46.485653+0000) 2024-03-06T11:04:47.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:47.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:47.485884+0000) 2024-03-06T11:04:48.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:48.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:48.486110+0000) 2024-03-06T11:04:49.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:49.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:49.486299+0000) 2024-03-06T11:04:50.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:50.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:50.486487+0000) 2024-03-06T11:04:51.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:51.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:51.486676+0000) 2024-03-06T11:04:52.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:52.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:52.486879+0000) 2024-03-06T11:04:53.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:53.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:53.487067+0000) 2024-03-06T11:04:54.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:54.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:54.487289+0000) 2024-03-06T11:04:55.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:55.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:55.487659+0000) 2024-03-06T11:04:56.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:56.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:56.487882+0000) 2024-03-06T11:04:57.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:57.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:57.488123+0000) 2024-03-06T11:04:58.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:58.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:58.488321+0000) 2024-03-06T11:04:59.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:04:59.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:04:59.488588+0000) 2024-03-06T11:05:00.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:00.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:00.488833+0000) 2024-03-06T11:05:01.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:01.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:01.489072+0000) 2024-03-06T11:05:02.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:02.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:02.489317+0000) 2024-03-06T11:05:03.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:03.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:03.489587+0000) 2024-03-06T11:05:04.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:04.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:04.489800+0000) 2024-03-06T11:05:05.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:05.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:05.490043+0000) 2024-03-06T11:05:06.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:06.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:06.490264+0000) 2024-03-06T11:05:07.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:07.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:07.490591+0000) 2024-03-06T11:05:08.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:08.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:08.490833+0000) 2024-03-06T11:05:09.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:09.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:09.491068+0000) 2024-03-06T11:05:10.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:10.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:10.491308+0000) 2024-03-06T11:05:11.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:11.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:11.491513+0000) 2024-03-06T11:05:12.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:12.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:12.491795+0000) 2024-03-06T11:05:13.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:13.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:13.491936+0000) 2024-03-06T11:05:14.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:14.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:14.492153+0000) 2024-03-06T11:05:15.289 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:05:15.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:05:15.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:15.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:15.492397+0000) 2024-03-06T11:05:16.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:16.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:16.492662+0000) 2024-03-06T11:05:17.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:17.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:17.492882+0000) 2024-03-06T11:05:18.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:18.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:18.493071+0000) 2024-03-06T11:05:19.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:19.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:19.493308+0000) 2024-03-06T11:05:20.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:20.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:20.493576+0000) 2024-03-06T11:05:21.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:21.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:21.493809+0000) 2024-03-06T11:05:22.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:22.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:22.494043+0000) 2024-03-06T11:05:23.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:23.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:23.494268+0000) 2024-03-06T11:05:24.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:24.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:24.494498+0000) 2024-03-06T11:05:25.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:25.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:25.494764+0000) 2024-03-06T11:05:26.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:26.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:26.494987+0000) 2024-03-06T11:05:27.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:27.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:27.495195+0000) 2024-03-06T11:05:28.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:28.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:28.495439+0000) 2024-03-06T11:05:29.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:29.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:29.495654+0000) 2024-03-06T11:05:30.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:30.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:30.495895+0000) 2024-03-06T11:05:31.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:31.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:31.496116+0000) 2024-03-06T11:05:32.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:32.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:32.496396+0000) 2024-03-06T11:05:33.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:33.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:33.496666+0000) 2024-03-06T11:05:34.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:34.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:34.496945+0000) 2024-03-06T11:05:35.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:35.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:35.497239+0000) 2024-03-06T11:05:36.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:36.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:36.497444+0000) 2024-03-06T11:05:37.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:37.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:37.497665+0000) 2024-03-06T11:05:38.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:38.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:38.497889+0000) 2024-03-06T11:05:39.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:39.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:39.498080+0000) 2024-03-06T11:05:40.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:40.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:40.498288+0000) 2024-03-06T11:05:41.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:41.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:41.498522+0000) 2024-03-06T11:05:42.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:42.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:42.498780+0000) 2024-03-06T11:05:43.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:43.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:43.498981+0000) 2024-03-06T11:05:44.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:44.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:44.499201+0000) 2024-03-06T11:05:45.486 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:05:45.494 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:05:45.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:45.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:45.499382+0000) 2024-03-06T11:05:46.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:46.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:46.499647+0000) 2024-03-06T11:05:47.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:47.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:47.499870+0000) 2024-03-06T11:05:48.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:48.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:48.500067+0000) 2024-03-06T11:05:49.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:49.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:49.500290+0000) 2024-03-06T11:05:50.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:50.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:50.500519+0000) 2024-03-06T11:05:51.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:51.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:51.500805+0000) 2024-03-06T11:05:52.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:52.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:52.501081+0000) 2024-03-06T11:05:53.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:53.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:53.501360+0000) 2024-03-06T11:05:54.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:54.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:54.501636+0000) 2024-03-06T11:05:55.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:55.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:55.501924+0000) 2024-03-06T11:05:56.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:56.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:56.502206+0000) 2024-03-06T11:05:57.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:57.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:57.502489+0000) 2024-03-06T11:05:58.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:58.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:58.502770+0000) 2024-03-06T11:05:59.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:05:59.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:05:59.503052+0000) 2024-03-06T11:06:00.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:00.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:00.503329+0000) 2024-03-06T11:06:01.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:01.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:01.503635+0000) 2024-03-06T11:06:02.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:02.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:02.503907+0000) 2024-03-06T11:06:03.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:03.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:03.504187+0000) 2024-03-06T11:06:04.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:04.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:04.504466+0000) 2024-03-06T11:06:05.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:05.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:05.504748+0000) 2024-03-06T11:06:06.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:06.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:06.505030+0000) 2024-03-06T11:06:07.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:07.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:07.505267+0000) 2024-03-06T11:06:08.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:08.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:08.505514+0000) 2024-03-06T11:06:09.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:09.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:09.505755+0000) 2024-03-06T11:06:10.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:10.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:10.505987+0000) 2024-03-06T11:06:11.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:11.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:11.506240+0000) 2024-03-06T11:06:12.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:12.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:12.506468+0000) 2024-03-06T11:06:13.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:13.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:13.506714+0000) 2024-03-06T11:06:14.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:14.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:14.506938+0000) 2024-03-06T11:06:15.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:15.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:15.507192+0000) 2024-03-06T11:06:15.690 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:06:15.698 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:06:16.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:16.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:16.507418+0000) 2024-03-06T11:06:17.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:17.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:17.507643+0000) 2024-03-06T11:06:18.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:18.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:18.507906+0000) 2024-03-06T11:06:19.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:19.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:19.508174+0000) 2024-03-06T11:06:20.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:20.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:20.508453+0000) 2024-03-06T11:06:21.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:21.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:21.508739+0000) 2024-03-06T11:06:22.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:22.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:22.509027+0000) 2024-03-06T11:06:23.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:23.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:23.509294+0000) 2024-03-06T11:06:24.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:24.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:24.509612+0000) 2024-03-06T11:06:25.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:25.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:25.509860+0000) 2024-03-06T11:06:26.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:26.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:26.510081+0000) 2024-03-06T11:06:27.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:27.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:27.510304+0000) 2024-03-06T11:06:28.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:28.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:28.510516+0000) 2024-03-06T11:06:29.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:29.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:29.510797+0000) 2024-03-06T11:06:30.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:30.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:30.511023+0000) 2024-03-06T11:06:31.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:31.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:31.511245+0000) 2024-03-06T11:06:32.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:32.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:32.511460+0000) 2024-03-06T11:06:33.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:33.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:33.511726+0000) 2024-03-06T11:06:34.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:34.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:34.511944+0000) 2024-03-06T11:06:35.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:35.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:35.512236+0000) 2024-03-06T11:06:36.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:36.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:36.512439+0000) 2024-03-06T11:06:37.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:37.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:37.512710+0000) 2024-03-06T11:06:38.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:38.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:38.512988+0000) 2024-03-06T11:06:39.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:39.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:39.513268+0000) 2024-03-06T11:06:40.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:40.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:40.513582+0000) 2024-03-06T11:06:41.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:41.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:41.513879+0000) 2024-03-06T11:06:42.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:42.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:42.514151+0000) 2024-03-06T11:06:43.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:43.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:43.514470+0000) 2024-03-06T11:06:44.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:44.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:44.514744+0000) 2024-03-06T11:06:45.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:45.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:45.515055+0000) 2024-03-06T11:06:45.889 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:06:45.897 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:06:46.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:46.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:46.515329+0000) 2024-03-06T11:06:47.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:47.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:47.515634+0000) 2024-03-06T11:06:48.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:48.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:48.515916+0000) 2024-03-06T11:06:49.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:49.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:49.516199+0000) 2024-03-06T11:06:50.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:50.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:50.516478+0000) 2024-03-06T11:06:51.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:51.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:51.516758+0000) 2024-03-06T11:06:52.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:52.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:52.516977+0000) 2024-03-06T11:06:53.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:53.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:53.517181+0000) 2024-03-06T11:06:54.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:54.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:54.517399+0000) 2024-03-06T11:06:55.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:55.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:55.517648+0000) 2024-03-06T11:06:56.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:56.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:56.517879+0000) 2024-03-06T11:06:57.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:57.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:57.518105+0000) 2024-03-06T11:06:58.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:58.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:58.518327+0000) 2024-03-06T11:06:59.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:06:59.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:06:59.518592+0000) 2024-03-06T11:07:00.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:00.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:00.518828+0000) 2024-03-06T11:07:01.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:01.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:01.519043+0000) 2024-03-06T11:07:02.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:02.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:02.519266+0000) 2024-03-06T11:07:03.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:03.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:03.519573+0000) 2024-03-06T11:07:04.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:04.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:04.519852+0000) 2024-03-06T11:07:05.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:05.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:05.520096+0000) 2024-03-06T11:07:06.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:06.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:06.520322+0000) 2024-03-06T11:07:07.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:07.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:07.520638+0000) 2024-03-06T11:07:08.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:08.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:08.520923+0000) 2024-03-06T11:07:09.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:09.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:09.521205+0000) 2024-03-06T11:07:10.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:10.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:10.521489+0000) 2024-03-06T11:07:11.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:11.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:11.521700+0000) 2024-03-06T11:07:12.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:12.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:12.521941+0000) 2024-03-06T11:07:13.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:13.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:13.522156+0000) 2024-03-06T11:07:14.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:14.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:14.522380+0000) 2024-03-06T11:07:15.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:15.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:15.522689+0000) 2024-03-06T11:07:16.116 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:07:16.124 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:07:16.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:16.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:16.522974+0000) 2024-03-06T11:07:17.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:17.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:17.523219+0000) 2024-03-06T11:07:18.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:18.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:18.523452+0000) 2024-03-06T11:07:19.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:19.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:19.523722+0000) 2024-03-06T11:07:20.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:20.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:20.523990+0000) 2024-03-06T11:07:21.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:21.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:21.524217+0000) 2024-03-06T11:07:22.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:22.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:22.524441+0000) 2024-03-06T11:07:23.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:23.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:23.524662+0000) 2024-03-06T11:07:24.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:24.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:24.524891+0000) 2024-03-06T11:07:25.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:25.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:25.525126+0000) 2024-03-06T11:07:26.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:26.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:26.525348+0000) 2024-03-06T11:07:27.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:27.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:27.525620+0000) 2024-03-06T11:07:28.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:28.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:28.525897+0000) 2024-03-06T11:07:29.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:29.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:29.526181+0000) 2024-03-06T11:07:30.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:30.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:30.526379+0000) 2024-03-06T11:07:31.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:31.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:31.526639+0000) 2024-03-06T11:07:32.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:32.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:32.526884+0000) 2024-03-06T11:07:33.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:33.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:33.527107+0000) 2024-03-06T11:07:34.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:34.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:34.527303+0000) 2024-03-06T11:07:35.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:35.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:35.527580+0000) 2024-03-06T11:07:36.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:36.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:36.527867+0000) 2024-03-06T11:07:37.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:37.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:37.528099+0000) 2024-03-06T11:07:38.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:38.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:38.528329+0000) 2024-03-06T11:07:39.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:39.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:39.528585+0000) 2024-03-06T11:07:40.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:40.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:40.528782+0000) 2024-03-06T11:07:41.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:41.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:41.529001+0000) 2024-03-06T11:07:42.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:42.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:42.529288+0000) 2024-03-06T11:07:43.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:43.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:43.529585+0000) 2024-03-06T11:07:44.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:44.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:44.529866+0000) 2024-03-06T11:07:45.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:45.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:45.530158+0000) 2024-03-06T11:07:46.329 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:07:46.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:07:46.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:46.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:46.530377+0000) 2024-03-06T11:07:47.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:47.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:47.530637+0000) 2024-03-06T11:07:48.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:48.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:48.530854+0000) 2024-03-06T11:07:49.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:49.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:49.531074+0000) 2024-03-06T11:07:50.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:50.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:50.531295+0000) 2024-03-06T11:07:51.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:51.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:51.531482+0000) 2024-03-06T11:07:52.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:52.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:52.531703+0000) 2024-03-06T11:07:53.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:53.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:53.531936+0000) 2024-03-06T11:07:54.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:54.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:54.532166+0000) 2024-03-06T11:07:55.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:55.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:55.532402+0000) 2024-03-06T11:07:56.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:56.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:56.532658+0000) 2024-03-06T11:07:57.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:57.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:57.532843+0000) 2024-03-06T11:07:58.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:58.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:58.533078+0000) 2024-03-06T11:07:59.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:07:59.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:07:59.533297+0000) 2024-03-06T11:08:00.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:00.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:00.533498+0000) 2024-03-06T11:08:01.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:01.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:01.533742+0000) 2024-03-06T11:08:02.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:02.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:02.533950+0000) 2024-03-06T11:08:03.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:03.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:03.534137+0000) 2024-03-06T11:08:04.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:04.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:04.534345+0000) 2024-03-06T11:08:05.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:05.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:05.534678+0000) 2024-03-06T11:08:06.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:06.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:06.534926+0000) 2024-03-06T11:08:07.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:07.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:07.535161+0000) 2024-03-06T11:08:08.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:08.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:08.535346+0000) 2024-03-06T11:08:09.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:09.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:09.535648+0000) 2024-03-06T11:08:10.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:10.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:10.535936+0000) 2024-03-06T11:08:11.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:11.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:11.536149+0000) 2024-03-06T11:08:12.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:12.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:12.536361+0000) 2024-03-06T11:08:13.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:13.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:13.536628+0000) 2024-03-06T11:08:14.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:14.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:14.536851+0000) 2024-03-06T11:08:15.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:15.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:15.537081+0000) 2024-03-06T11:08:16.510 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:08:16.519 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:08:16.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:16.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:16.537308+0000) 2024-03-06T11:08:17.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:17.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:17.537580+0000) 2024-03-06T11:08:18.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:18.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:18.537824+0000) 2024-03-06T11:08:19.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:19.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:19.538051+0000) 2024-03-06T11:08:20.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:20.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:20.538229+0000) 2024-03-06T11:08:21.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:21.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:21.538453+0000) 2024-03-06T11:08:22.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:22.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:22.538719+0000) 2024-03-06T11:08:23.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:23.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:23.538943+0000) 2024-03-06T11:08:24.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:24.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:24.539163+0000) 2024-03-06T11:08:25.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:25.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:25.539362+0000) 2024-03-06T11:08:26.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:26.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:26.539630+0000) 2024-03-06T11:08:27.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:27.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:27.539869+0000) 2024-03-06T11:08:28.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:28.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:28.540097+0000) 2024-03-06T11:08:29.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:29.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:29.540276+0000) 2024-03-06T11:08:30.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:30.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:30.540501+0000) 2024-03-06T11:08:31.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:31.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:31.540727+0000) 2024-03-06T11:08:32.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:32.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:32.540971+0000) 2024-03-06T11:08:33.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:33.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:33.541157+0000) 2024-03-06T11:08:34.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:34.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:34.541406+0000) 2024-03-06T11:08:35.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:35.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:35.541648+0000) 2024-03-06T11:08:36.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:36.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:36.541841+0000) 2024-03-06T11:08:37.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:37.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:37.542063+0000) 2024-03-06T11:08:38.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:38.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:38.542290+0000) 2024-03-06T11:08:39.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:39.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:39.542525+0000) 2024-03-06T11:08:40.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:40.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:40.542782+0000) 2024-03-06T11:08:41.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:41.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:41.542991+0000) 2024-03-06T11:08:42.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:42.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:42.543248+0000) 2024-03-06T11:08:43.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:43.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:43.543425+0000) 2024-03-06T11:08:44.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:44.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:44.543658+0000) 2024-03-06T11:08:45.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:45.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:45.543881+0000) 2024-03-06T11:08:46.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:46.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:46.544068+0000) 2024-03-06T11:08:46.671 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:08:46.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:08:47.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:47.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:47.544356+0000) 2024-03-06T11:08:48.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:48.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:48.544674+0000) 2024-03-06T11:08:49.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:49.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:49.544872+0000) 2024-03-06T11:08:50.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:50.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:50.545119+0000) 2024-03-06T11:08:51.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:51.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:51.545362+0000) 2024-03-06T11:08:52.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:52.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:52.545664+0000) 2024-03-06T11:08:53.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:53.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:53.545905+0000) 2024-03-06T11:08:54.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:54.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:54.546149+0000) 2024-03-06T11:08:55.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:55.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:55.546353+0000) 2024-03-06T11:08:56.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:56.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:56.546584+0000) 2024-03-06T11:08:57.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:57.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:57.546819+0000) 2024-03-06T11:08:58.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:58.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:58.547045+0000) 2024-03-06T11:08:59.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:08:59.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:08:59.547270+0000) 2024-03-06T11:09:00.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:00.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:00.547459+0000) 2024-03-06T11:09:01.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:01.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:01.547706+0000) 2024-03-06T11:09:02.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:02.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:02.547908+0000) 2024-03-06T11:09:03.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:03.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:03.548132+0000) 2024-03-06T11:09:04.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:04.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:04.548355+0000) 2024-03-06T11:09:05.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:05.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:05.548627+0000) 2024-03-06T11:09:06.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:06.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:06.548820+0000) 2024-03-06T11:09:07.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:07.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:07.549051+0000) 2024-03-06T11:09:08.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:08.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:08.549277+0000) 2024-03-06T11:09:09.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:09.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:09.549504+0000) 2024-03-06T11:09:10.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:10.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:10.549770+0000) 2024-03-06T11:09:11.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:11.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:11.549996+0000) 2024-03-06T11:09:12.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:12.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:12.550244+0000) 2024-03-06T11:09:13.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:13.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:13.550429+0000) 2024-03-06T11:09:14.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:14.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:14.550654+0000) 2024-03-06T11:09:15.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:15.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:15.550892+0000) 2024-03-06T11:09:16.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:16.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:16.551134+0000) 2024-03-06T11:09:16.861 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:09:16.868 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:09:17.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:17.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:17.551433+0000) 2024-03-06T11:09:18.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:18.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:18.551679+0000) 2024-03-06T11:09:19.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:19.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:19.551873+0000) 2024-03-06T11:09:20.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:20.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:20.552050+0000) 2024-03-06T11:09:21.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:21.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:21.552242+0000) 2024-03-06T11:09:22.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:22.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:22.552466+0000) 2024-03-06T11:09:23.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:23.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:23.552696+0000) 2024-03-06T11:09:24.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:24.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:24.552895+0000) 2024-03-06T11:09:25.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:25.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:25.553101+0000) 2024-03-06T11:09:26.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:26.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:26.553300+0000) 2024-03-06T11:09:27.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:27.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:27.553482+0000) 2024-03-06T11:09:28.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:28.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:28.553665+0000) 2024-03-06T11:09:29.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:29.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:29.553903+0000) 2024-03-06T11:09:30.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:30.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:30.554125+0000) 2024-03-06T11:09:31.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:31.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:31.554363+0000) 2024-03-06T11:09:32.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:32.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:32.554629+0000) 2024-03-06T11:09:33.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:33.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:33.554911+0000) 2024-03-06T11:09:34.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:34.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:34.555190+0000) 2024-03-06T11:09:35.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:35.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:35.555469+0000) 2024-03-06T11:09:36.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:36.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:36.555696+0000) 2024-03-06T11:09:37.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:37.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:37.555876+0000) 2024-03-06T11:09:38.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:38.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:38.556120+0000) 2024-03-06T11:09:39.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:39.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:39.556362+0000) 2024-03-06T11:09:40.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:40.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:40.556641+0000) 2024-03-06T11:09:41.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:41.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:41.556878+0000) 2024-03-06T11:09:42.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:42.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:42.557126+0000) 2024-03-06T11:09:43.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:43.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:43.557370+0000) 2024-03-06T11:09:44.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:44.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:44.557669+0000) 2024-03-06T11:09:45.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:45.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:45.557878+0000) 2024-03-06T11:09:46.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:46.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:46.558070+0000) 2024-03-06T11:09:47.068 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:09:47.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:09:47.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:47.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:47.558294+0000) 2024-03-06T11:09:48.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:48.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:48.558567+0000) 2024-03-06T11:09:49.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:49.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:49.558807+0000) 2024-03-06T11:09:50.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:50.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:50.559029+0000) 2024-03-06T11:09:51.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:51.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:51.559222+0000) 2024-03-06T11:09:52.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:52.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:52.559430+0000) 2024-03-06T11:09:53.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:53.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:53.559658+0000) 2024-03-06T11:09:54.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:54.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:54.559850+0000) 2024-03-06T11:09:55.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:55.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:55.560073+0000) 2024-03-06T11:09:56.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:56.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:56.560305+0000) 2024-03-06T11:09:57.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:57.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:57.560526+0000) 2024-03-06T11:09:58.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:58.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:58.560754+0000) 2024-03-06T11:09:59.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:09:59.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:09:59.560986+0000) 2024-03-06T11:10:00.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:00.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:00.561186+0000) 2024-03-06T11:10:01.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:01.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:01.561401+0000) 2024-03-06T11:10:02.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:02.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:02.561647+0000) 2024-03-06T11:10:03.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:03.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:03.561874+0000) 2024-03-06T11:10:04.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:04.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:04.562082+0000) 2024-03-06T11:10:05.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:05.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:05.562321+0000) 2024-03-06T11:10:06.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:06.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:06.562585+0000) 2024-03-06T11:10:07.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:07.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:07.562854+0000) 2024-03-06T11:10:08.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:08.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:08.563138+0000) 2024-03-06T11:10:09.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:09.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:09.563411+0000) 2024-03-06T11:10:10.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:10.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:10.563689+0000) 2024-03-06T11:10:11.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:11.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:11.564010+0000) 2024-03-06T11:10:12.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:12.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:12.564177+0000) 2024-03-06T11:10:13.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:13.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:13.564421+0000) 2024-03-06T11:10:14.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:14.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:14.564678+0000) 2024-03-06T11:10:15.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:15.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:15.564931+0000) 2024-03-06T11:10:16.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:16.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:16.565177+0000) 2024-03-06T11:10:17.282 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:10:17.290 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:10:17.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:17.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:17.565400+0000) 2024-03-06T11:10:18.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:18.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:18.565651+0000) 2024-03-06T11:10:19.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:19.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:19.565894+0000) 2024-03-06T11:10:20.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:20.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:20.566139+0000) 2024-03-06T11:10:21.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:21.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:21.566338+0000) 2024-03-06T11:10:22.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:22.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:22.566615+0000) 2024-03-06T11:10:23.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:23.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:23.566915+0000) 2024-03-06T11:10:24.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:24.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:24.567188+0000) 2024-03-06T11:10:25.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:25.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:25.567476+0000) 2024-03-06T11:10:26.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:26.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:26.567691+0000) 2024-03-06T11:10:27.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:27.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:27.567970+0000) 2024-03-06T11:10:28.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:28.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:28.568181+0000) 2024-03-06T11:10:29.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:29.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:29.568447+0000) 2024-03-06T11:10:30.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:30.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:30.568736+0000) 2024-03-06T11:10:31.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:31.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:31.569002+0000) 2024-03-06T11:10:32.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:32.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:32.569282+0000) 2024-03-06T11:10:33.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:33.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:33.569616+0000) 2024-03-06T11:10:34.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:34.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:34.569822+0000) 2024-03-06T11:10:35.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:35.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:35.570053+0000) 2024-03-06T11:10:36.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:36.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:36.570331+0000) 2024-03-06T11:10:37.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:37.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:37.570658+0000) 2024-03-06T11:10:38.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:38.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:38.570910+0000) 2024-03-06T11:10:39.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:39.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:39.571152+0000) 2024-03-06T11:10:40.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:40.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:40.571355+0000) 2024-03-06T11:10:41.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:41.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:41.571577+0000) 2024-03-06T11:10:42.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:42.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:42.571800+0000) 2024-03-06T11:10:43.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:43.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:43.571994+0000) 2024-03-06T11:10:44.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:44.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:44.572228+0000) 2024-03-06T11:10:45.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:45.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:45.572467+0000) 2024-03-06T11:10:46.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:46.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:46.572694+0000) 2024-03-06T11:10:47.497 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:10:47.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:10:47.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:47.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:47.572881+0000) 2024-03-06T11:10:48.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:48.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:48.573158+0000) 2024-03-06T11:10:49.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:49.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:49.573420+0000) 2024-03-06T11:10:50.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:50.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:50.573702+0000) 2024-03-06T11:10:51.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:51.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:51.573919+0000) 2024-03-06T11:10:52.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:52.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:52.574116+0000) 2024-03-06T11:10:53.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:53.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:53.574391+0000) 2024-03-06T11:10:54.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:54.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:54.574682+0000) 2024-03-06T11:10:55.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:55.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:55.574890+0000) 2024-03-06T11:10:56.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:56.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:56.575124+0000) 2024-03-06T11:10:57.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:57.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:57.575350+0000) 2024-03-06T11:10:58.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:58.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:58.575580+0000) 2024-03-06T11:10:59.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:10:59.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:10:59.575777+0000) 2024-03-06T11:11:00.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:00.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:00.575975+0000) 2024-03-06T11:11:01.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:01.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:01.576169+0000) 2024-03-06T11:11:02.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:02.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:02.576401+0000) 2024-03-06T11:11:03.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:03.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:03.576636+0000) 2024-03-06T11:11:04.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:04.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:04.576796+0000) 2024-03-06T11:11:05.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:05.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:05.577024+0000) 2024-03-06T11:11:06.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:06.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:06.577202+0000) 2024-03-06T11:11:07.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:07.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:07.577397+0000) 2024-03-06T11:11:08.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:08.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:08.577638+0000) 2024-03-06T11:11:09.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:09.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:09.577784+0000) 2024-03-06T11:11:10.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:10.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:10.578008+0000) 2024-03-06T11:11:11.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:11.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:11.578234+0000) 2024-03-06T11:11:12.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:12.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:12.578464+0000) 2024-03-06T11:11:13.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:13.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:13.578688+0000) 2024-03-06T11:11:14.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:14.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:14.578874+0000) 2024-03-06T11:11:15.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:15.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:15.579102+0000) 2024-03-06T11:11:16.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:16.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:16.579333+0000) 2024-03-06T11:11:17.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:17.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:17.579616+0000) 2024-03-06T11:11:17.652 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:11:17.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:11:18.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:18.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:18.579831+0000) 2024-03-06T11:11:19.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:19.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:19.580011+0000) 2024-03-06T11:11:20.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:20.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:20.580244+0000) 2024-03-06T11:11:21.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:21.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:21.580479+0000) 2024-03-06T11:11:22.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:22.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:22.580695+0000) 2024-03-06T11:11:23.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:23.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:23.580944+0000) 2024-03-06T11:11:24.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:24.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:24.581173+0000) 2024-03-06T11:11:25.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:25.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:25.581472+0000) 2024-03-06T11:11:26.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:26.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:26.581748+0000) 2024-03-06T11:11:27.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:27.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:27.582009+0000) 2024-03-06T11:11:28.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:28.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:28.582292+0000) 2024-03-06T11:11:29.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:29.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:29.582497+0000) 2024-03-06T11:11:30.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:30.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:30.582731+0000) 2024-03-06T11:11:31.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:31.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:31.582928+0000) 2024-03-06T11:11:32.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:32.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:32.583108+0000) 2024-03-06T11:11:33.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:33.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:33.583367+0000) 2024-03-06T11:11:34.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:34.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:34.583669+0000) 2024-03-06T11:11:35.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:35.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:35.583885+0000) 2024-03-06T11:11:36.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:36.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:36.584126+0000) 2024-03-06T11:11:37.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:37.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:37.584388+0000) 2024-03-06T11:11:38.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:38.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:38.584694+0000) 2024-03-06T11:11:39.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:39.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:39.584970+0000) 2024-03-06T11:11:40.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:40.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:40.585242+0000) 2024-03-06T11:11:41.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:41.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:41.585513+0000) 2024-03-06T11:11:42.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:42.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:42.585804+0000) 2024-03-06T11:11:43.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:43.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:43.585988+0000) 2024-03-06T11:11:44.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:44.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:44.586204+0000) 2024-03-06T11:11:45.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:45.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:45.586466+0000) 2024-03-06T11:11:46.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:46.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:46.586743+0000) 2024-03-06T11:11:47.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:47.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:47.587076+0000) 2024-03-06T11:11:47.792 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:11:47.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:11:48.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:48.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:48.587292+0000) 2024-03-06T11:11:49.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:49.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:49.587500+0000) 2024-03-06T11:11:50.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:50.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:50.587756+0000) 2024-03-06T11:11:51.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:51.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:51.587992+0000) 2024-03-06T11:11:52.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:52.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:52.588206+0000) 2024-03-06T11:11:53.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:53.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:53.588463+0000) 2024-03-06T11:11:54.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:54.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:54.588736+0000) 2024-03-06T11:11:55.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:55.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:55.589024+0000) 2024-03-06T11:11:56.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:56.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:56.589261+0000) 2024-03-06T11:11:57.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:57.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:57.589467+0000) 2024-03-06T11:11:58.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:58.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:58.589692+0000) 2024-03-06T11:11:59.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:11:59.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:11:59.589896+0000) 2024-03-06T11:12:00.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:00.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:00.590173+0000) 2024-03-06T11:12:01.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:01.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:01.590452+0000) 2024-03-06T11:12:02.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:02.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:02.590743+0000) 2024-03-06T11:12:03.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:03.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:03.591019+0000) 2024-03-06T11:12:04.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:04.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:04.591305+0000) 2024-03-06T11:12:05.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:05.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:05.591708+0000) 2024-03-06T11:12:06.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:06.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:06.591917+0000) 2024-03-06T11:12:07.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:07.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:07.592108+0000) 2024-03-06T11:12:08.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:08.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:08.592360+0000) 2024-03-06T11:12:09.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:09.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:09.592626+0000) 2024-03-06T11:12:10.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:10.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:10.592837+0000) 2024-03-06T11:12:11.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:11.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:11.593039+0000) 2024-03-06T11:12:12.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:12.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:12.593251+0000) 2024-03-06T11:12:13.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:13.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:13.593453+0000) 2024-03-06T11:12:14.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:14.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:14.593689+0000) 2024-03-06T11:12:15.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:15.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:15.594148+0000) 2024-03-06T11:12:16.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:16.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:16.594374+0000) 2024-03-06T11:12:17.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:17.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:17.594630+0000) 2024-03-06T11:12:18.014 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:12:18.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:12:18.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:18.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:18.594839+0000) 2024-03-06T11:12:19.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:19.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:19.595117+0000) 2024-03-06T11:12:20.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:20.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:20.595395+0000) 2024-03-06T11:12:21.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:21.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:21.595690+0000) 2024-03-06T11:12:22.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:22.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:22.595927+0000) 2024-03-06T11:12:23.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:23.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:23.596215+0000) 2024-03-06T11:12:24.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:24.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:24.596481+0000) 2024-03-06T11:12:25.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:25.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:25.596812+0000) 2024-03-06T11:12:26.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:26.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:26.597044+0000) 2024-03-06T11:12:27.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:27.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:27.597321+0000) 2024-03-06T11:12:28.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:28.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:28.597609+0000) 2024-03-06T11:12:29.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:29.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:29.597844+0000) 2024-03-06T11:12:30.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:30.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:30.598076+0000) 2024-03-06T11:12:31.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:31.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:31.598301+0000) 2024-03-06T11:12:32.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:32.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:32.598521+0000) 2024-03-06T11:12:33.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:33.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:33.598789+0000) 2024-03-06T11:12:34.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:34.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:34.598995+0000) 2024-03-06T11:12:35.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:35.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:35.599293+0000) 2024-03-06T11:12:36.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:36.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:36.599505+0000) 2024-03-06T11:12:37.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:37.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:37.599809+0000) 2024-03-06T11:12:38.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:38.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:38.600096+0000) 2024-03-06T11:12:39.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:39.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:39.600315+0000) 2024-03-06T11:12:40.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:40.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:40.600568+0000) 2024-03-06T11:12:41.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:41.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:41.600802+0000) 2024-03-06T11:12:42.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:42.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:42.601035+0000) 2024-03-06T11:12:43.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:43.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:43.601309+0000) 2024-03-06T11:12:44.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:44.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:44.601511+0000) 2024-03-06T11:12:45.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:45.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:45.601766+0000) 2024-03-06T11:12:46.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:46.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:46.602022+0000) 2024-03-06T11:12:47.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:47.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:47.602265+0000) 2024-03-06T11:12:48.227 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:12:48.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:12:48.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:48.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:48.602501+0000) 2024-03-06T11:12:49.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:49.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:49.602733+0000) 2024-03-06T11:12:50.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:50.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:50.602971+0000) 2024-03-06T11:12:51.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:51.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:51.603178+0000) 2024-03-06T11:12:52.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:52.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:52.603453+0000) 2024-03-06T11:12:53.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:53.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:53.603733+0000) 2024-03-06T11:12:54.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:54.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:54.603943+0000) 2024-03-06T11:12:55.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:55.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:55.604191+0000) 2024-03-06T11:12:56.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:56.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:56.604404+0000) 2024-03-06T11:12:57.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:57.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:57.604658+0000) 2024-03-06T11:12:58.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:58.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:58.604926+0000) 2024-03-06T11:12:59.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:12:59.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:12:59.605141+0000) 2024-03-06T11:13:00.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:00.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:00.605415+0000) 2024-03-06T11:13:01.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:01.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:01.605691+0000) 2024-03-06T11:13:02.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:02.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:02.605967+0000) 2024-03-06T11:13:03.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:03.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:03.606242+0000) 2024-03-06T11:13:04.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:04.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:04.606451+0000) 2024-03-06T11:13:05.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:05.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:05.606673+0000) 2024-03-06T11:13:06.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:06.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:06.606954+0000) 2024-03-06T11:13:07.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:07.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:07.607231+0000) 2024-03-06T11:13:08.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:08.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:08.607499+0000) 2024-03-06T11:13:09.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:09.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:09.607824+0000) 2024-03-06T11:13:10.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:10.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:10.608036+0000) 2024-03-06T11:13:11.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:11.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:11.608269+0000) 2024-03-06T11:13:12.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:12.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:12.608501+0000) 2024-03-06T11:13:13.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:13.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:13.608692+0000) 2024-03-06T11:13:14.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:14.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:14.608906+0000) 2024-03-06T11:13:15.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:15.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:15.609148+0000) 2024-03-06T11:13:16.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:16.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:16.609372+0000) 2024-03-06T11:13:17.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:17.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:17.609647+0000) 2024-03-06T11:13:18.442 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:13:18.450 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:13:18.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:18.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:18.609847+0000) 2024-03-06T11:13:19.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:19.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:19.610088+0000) 2024-03-06T11:13:20.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:20.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:20.610365+0000) 2024-03-06T11:13:21.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:21.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:21.610656+0000) 2024-03-06T11:13:22.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:22.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:22.610898+0000) 2024-03-06T11:13:23.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:23.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:23.611132+0000) 2024-03-06T11:13:24.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:24.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:24.611407+0000) 2024-03-06T11:13:25.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:25.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:25.611711+0000) 2024-03-06T11:13:26.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:26.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:26.611919+0000) 2024-03-06T11:13:27.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:27.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:27.612114+0000) 2024-03-06T11:13:28.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:28.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:28.612412+0000) 2024-03-06T11:13:29.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:29.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:29.612640+0000) 2024-03-06T11:13:30.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:30.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:30.612838+0000) 2024-03-06T11:13:31.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:31.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:31.613058+0000) 2024-03-06T11:13:32.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:32.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:32.613284+0000) 2024-03-06T11:13:33.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:33.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:33.613577+0000) 2024-03-06T11:13:34.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:34.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:34.613826+0000) 2024-03-06T11:13:35.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:35.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:35.614041+0000) 2024-03-06T11:13:36.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:36.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:36.614304+0000) 2024-03-06T11:13:37.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:37.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:37.614511+0000) 2024-03-06T11:13:38.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:38.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:38.614783+0000) 2024-03-06T11:13:39.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:39.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:39.615005+0000) 2024-03-06T11:13:40.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:40.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:40.615220+0000) 2024-03-06T11:13:41.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:41.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:41.615479+0000) 2024-03-06T11:13:42.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:42.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:42.615766+0000) 2024-03-06T11:13:43.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:43.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:43.616023+0000) 2024-03-06T11:13:44.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:44.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:44.616304+0000) 2024-03-06T11:13:45.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:45.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:45.616496+0000) 2024-03-06T11:13:46.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:46.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:46.616741+0000) 2024-03-06T11:13:47.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:47.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:47.616968+0000) 2024-03-06T11:13:48.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:48.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:48.617248+0000) 2024-03-06T11:13:48.661 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:13:48.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:13:49.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:49.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:49.617526+0000) 2024-03-06T11:13:50.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:50.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:50.617818+0000) 2024-03-06T11:13:51.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:51.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:51.618049+0000) 2024-03-06T11:13:52.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:52.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:52.618297+0000) 2024-03-06T11:13:53.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:53.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:53.618576+0000) 2024-03-06T11:13:54.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:54.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:54.618826+0000) 2024-03-06T11:13:55.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:55.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:55.619083+0000) 2024-03-06T11:13:56.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:56.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:56.619288+0000) 2024-03-06T11:13:57.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:57.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:57.619508+0000) 2024-03-06T11:13:58.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:58.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:58.619754+0000) 2024-03-06T11:13:59.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:13:59.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:13:59.619945+0000) 2024-03-06T11:14:00.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:00.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:00.620133+0000) 2024-03-06T11:14:01.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:01.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:01.620360+0000) 2024-03-06T11:14:02.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:02.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:02.620619+0000) 2024-03-06T11:14:03.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:03.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:03.620826+0000) 2024-03-06T11:14:04.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:04.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:04.621046+0000) 2024-03-06T11:14:05.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:05.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:05.621253+0000) 2024-03-06T11:14:06.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:06.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:06.621488+0000) 2024-03-06T11:14:07.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:07.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:07.621720+0000) 2024-03-06T11:14:08.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:08.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:08.621920+0000) 2024-03-06T11:14:09.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:09.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:09.622147+0000) 2024-03-06T11:14:10.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:10.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:10.622341+0000) 2024-03-06T11:14:11.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:11.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:11.622599+0000) 2024-03-06T11:14:12.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:12.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:12.622835+0000) 2024-03-06T11:14:13.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:13.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:13.623037+0000) 2024-03-06T11:14:14.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:14.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:14.623280+0000) 2024-03-06T11:14:15.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:15.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:15.623572+0000) 2024-03-06T11:14:16.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:16.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:16.623866+0000) 2024-03-06T11:14:17.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:17.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:17.624048+0000) 2024-03-06T11:14:18.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:18.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:18.624308+0000) 2024-03-06T11:14:18.834 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:14:18.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:14:19.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:19.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:19.624484+0000) 2024-03-06T11:14:20.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:20.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:20.624707+0000) 2024-03-06T11:14:21.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:21.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:21.624906+0000) 2024-03-06T11:14:22.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:22.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:22.625100+0000) 2024-03-06T11:14:23.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:23.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:23.625355+0000) 2024-03-06T11:14:24.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:24.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:24.625613+0000) 2024-03-06T11:14:25.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:25.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:25.625814+0000) 2024-03-06T11:14:26.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:26.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:26.626000+0000) 2024-03-06T11:14:27.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:27.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:27.626229+0000) 2024-03-06T11:14:28.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:28.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:28.626481+0000) 2024-03-06T11:14:29.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:29.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:29.626688+0000) 2024-03-06T11:14:30.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:30.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:30.626945+0000) 2024-03-06T11:14:31.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:31.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:31.627137+0000) 2024-03-06T11:14:32.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:32.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:32.627385+0000) 2024-03-06T11:14:33.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:33.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:33.627670+0000) 2024-03-06T11:14:34.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:34.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:34.627845+0000) 2024-03-06T11:14:35.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:35.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:35.628075+0000) 2024-03-06T11:14:36.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:36.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:36.628316+0000) 2024-03-06T11:14:37.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:37.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:37.628570+0000) 2024-03-06T11:14:38.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:38.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:38.628796+0000) 2024-03-06T11:14:39.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:39.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:39.629028+0000) 2024-03-06T11:14:40.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:40.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:40.629260+0000) 2024-03-06T11:14:41.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:41.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:41.629477+0000) 2024-03-06T11:14:42.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:42.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:42.629707+0000) 2024-03-06T11:14:43.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:43.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:43.629936+0000) 2024-03-06T11:14:44.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:44.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:44.630161+0000) 2024-03-06T11:14:45.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:45.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:45.630388+0000) 2024-03-06T11:14:46.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:46.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:46.630658+0000) 2024-03-06T11:14:47.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:47.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:47.630886+0000) 2024-03-06T11:14:48.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:48.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:48.631125+0000) 2024-03-06T11:14:49.062 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:14:49.072 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:14:49.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:49.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:49.631354+0000) 2024-03-06T11:14:50.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:50.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:50.631638+0000) 2024-03-06T11:14:51.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:51.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:51.631857+0000) 2024-03-06T11:14:52.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:52.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:52.632057+0000) 2024-03-06T11:14:53.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:53.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:53.632296+0000) 2024-03-06T11:14:54.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:54.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:54.632522+0000) 2024-03-06T11:14:55.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:55.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:55.632761+0000) 2024-03-06T11:14:56.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:56.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:56.632967+0000) 2024-03-06T11:14:57.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:57.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:57.633205+0000) 2024-03-06T11:14:58.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:58.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:58.633440+0000) 2024-03-06T11:14:59.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:14:59.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:14:59.633672+0000) 2024-03-06T11:15:00.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:00.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:00.633861+0000) 2024-03-06T11:15:01.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:01.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:01.634044+0000) 2024-03-06T11:15:02.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:02.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:02.634236+0000) 2024-03-06T11:15:03.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:03.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:03.634482+0000) 2024-03-06T11:15:04.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:04.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:04.634742+0000) 2024-03-06T11:15:05.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:05.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:05.634974+0000) 2024-03-06T11:15:06.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:06.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:06.635162+0000) 2024-03-06T11:15:07.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:07.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:07.635377+0000) 2024-03-06T11:15:08.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:08.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:08.635643+0000) 2024-03-06T11:15:09.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:09.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:09.635878+0000) 2024-03-06T11:15:10.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:10.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:10.636127+0000) 2024-03-06T11:15:11.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:11.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:11.636407+0000) 2024-03-06T11:15:12.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:12.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:12.636708+0000) 2024-03-06T11:15:13.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:13.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:13.636946+0000) 2024-03-06T11:15:14.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:14.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:14.637227+0000) 2024-03-06T11:15:15.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:15.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:15.637517+0000) 2024-03-06T11:15:16.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:16.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:16.637822+0000) 2024-03-06T11:15:17.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:17.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:17.638055+0000) 2024-03-06T11:15:18.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:18.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:18.638337+0000) 2024-03-06T11:15:19.283 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:15:19.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:15:19.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:19.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:19.638653+0000) 2024-03-06T11:15:20.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:20.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:20.638825+0000) 2024-03-06T11:15:21.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:21.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:21.639012+0000) 2024-03-06T11:15:22.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:22.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:22.639205+0000) 2024-03-06T11:15:23.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:23.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:23.639445+0000) 2024-03-06T11:15:24.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:24.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:24.639676+0000) 2024-03-06T11:15:25.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:25.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:25.639912+0000) 2024-03-06T11:15:26.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:26.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:26.640093+0000) 2024-03-06T11:15:27.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:27.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:27.640307+0000) 2024-03-06T11:15:28.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:28.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:28.640559+0000) 2024-03-06T11:15:29.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:29.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:29.640747+0000) 2024-03-06T11:15:30.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:30.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:30.640930+0000) 2024-03-06T11:15:31.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:31.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:31.641123+0000) 2024-03-06T11:15:32.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:32.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:32.641328+0000) 2024-03-06T11:15:33.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:33.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:33.641560+0000) 2024-03-06T11:15:34.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:34.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:34.641760+0000) 2024-03-06T11:15:35.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:35.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:35.642050+0000) 2024-03-06T11:15:36.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:36.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:36.642256+0000) 2024-03-06T11:15:37.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:37.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:37.642491+0000) 2024-03-06T11:15:38.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:38.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:38.642717+0000) 2024-03-06T11:15:39.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:39.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:39.642915+0000) 2024-03-06T11:15:40.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:40.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:40.643138+0000) 2024-03-06T11:15:41.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:41.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:41.643375+0000) 2024-03-06T11:15:42.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:42.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:42.643625+0000) 2024-03-06T11:15:43.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:43.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:43.643813+0000) 2024-03-06T11:15:44.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:44.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:44.644008+0000) 2024-03-06T11:15:45.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:45.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:45.644210+0000) 2024-03-06T11:15:46.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:46.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:46.644443+0000) 2024-03-06T11:15:47.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:47.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:47.644645+0000) 2024-03-06T11:15:48.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:48.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:48.644873+0000) 2024-03-06T11:15:49.487 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:15:49.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:15:49.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:49.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:49.645095+0000) 2024-03-06T11:15:50.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:50.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:50.645333+0000) 2024-03-06T11:15:51.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:51.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:51.645610+0000) 2024-03-06T11:15:52.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:52.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:52.645811+0000) 2024-03-06T11:15:53.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:53.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:53.646010+0000) 2024-03-06T11:15:54.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:54.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:54.646215+0000) 2024-03-06T11:15:55.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:55.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:55.646472+0000) 2024-03-06T11:15:56.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:56.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:56.646684+0000) 2024-03-06T11:15:57.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:57.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:57.646842+0000) 2024-03-06T11:15:58.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:58.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:58.647022+0000) 2024-03-06T11:15:59.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:15:59.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:15:59.647260+0000) 2024-03-06T11:16:00.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:00.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:00.647454+0000) 2024-03-06T11:16:01.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:01.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:01.647677+0000) 2024-03-06T11:16:02.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:02.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:02.647867+0000) 2024-03-06T11:16:03.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:03.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:03.648122+0000) 2024-03-06T11:16:04.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:04.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:04.648382+0000) 2024-03-06T11:16:05.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:05.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:05.648659+0000) 2024-03-06T11:16:06.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:06.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:06.648858+0000) 2024-03-06T11:16:07.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:07.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:07.649057+0000) 2024-03-06T11:16:08.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:08.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:08.649279+0000) 2024-03-06T11:16:09.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:09.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:09.649503+0000) 2024-03-06T11:16:10.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:10.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:10.649755+0000) 2024-03-06T11:16:11.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:11.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:11.650074+0000) 2024-03-06T11:16:12.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:12.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:12.650399+0000) 2024-03-06T11:16:13.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:13.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:13.650697+0000) 2024-03-06T11:16:14.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:14.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:14.650908+0000) 2024-03-06T11:16:15.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:15.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:15.651117+0000) 2024-03-06T11:16:16.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:16.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:16.651341+0000) 2024-03-06T11:16:17.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:17.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:17.651636+0000) 2024-03-06T11:16:18.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:18.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:18.651845+0000) 2024-03-06T11:16:19.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:19.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:19.652152+0000) 2024-03-06T11:16:19.709 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:16:19.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:16:20.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:20.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:20.652481+0000) 2024-03-06T11:16:21.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:21.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:21.652785+0000) 2024-03-06T11:16:22.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:22.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:22.652975+0000) 2024-03-06T11:16:23.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:23.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:23.653166+0000) 2024-03-06T11:16:24.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:24.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:24.653367+0000) 2024-03-06T11:16:25.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:25.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:25.653642+0000) 2024-03-06T11:16:26.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:26.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:26.653836+0000) 2024-03-06T11:16:27.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:27.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:27.654031+0000) 2024-03-06T11:16:28.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:28.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:28.654275+0000) 2024-03-06T11:16:29.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:29.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:29.654508+0000) 2024-03-06T11:16:30.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:30.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:30.654681+0000) 2024-03-06T11:16:31.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:31.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:31.654860+0000) 2024-03-06T11:16:32.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:32.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:32.655086+0000) 2024-03-06T11:16:33.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:33.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:33.655311+0000) 2024-03-06T11:16:34.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:34.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:34.655567+0000) 2024-03-06T11:16:35.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:35.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:35.655801+0000) 2024-03-06T11:16:36.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:36.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:36.656041+0000) 2024-03-06T11:16:37.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:37.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:37.656279+0000) 2024-03-06T11:16:38.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:38.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:38.656598+0000) 2024-03-06T11:16:39.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:39.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:39.656842+0000) 2024-03-06T11:16:40.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:40.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:40.657091+0000) 2024-03-06T11:16:41.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:41.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:41.657357+0000) 2024-03-06T11:16:42.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:42.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:42.657589+0000) 2024-03-06T11:16:43.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:43.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:43.657813+0000) 2024-03-06T11:16:44.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:44.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:44.658047+0000) 2024-03-06T11:16:45.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:45.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:45.658352+0000) 2024-03-06T11:16:46.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:46.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:46.658630+0000) 2024-03-06T11:16:47.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:47.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:47.658823+0000) 2024-03-06T11:16:48.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:48.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:48.659059+0000) 2024-03-06T11:16:49.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:49.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:49.659244+0000) 2024-03-06T11:16:49.872 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:16:49.880 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:16:50.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:50.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:50.659484+0000) 2024-03-06T11:16:51.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:51.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:51.659707+0000) 2024-03-06T11:16:52.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:52.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:52.659916+0000) 2024-03-06T11:16:53.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:53.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:53.660108+0000) 2024-03-06T11:16:54.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:54.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:54.660350+0000) 2024-03-06T11:16:55.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:55.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:55.660615+0000) 2024-03-06T11:16:56.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:56.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:56.660811+0000) 2024-03-06T11:16:57.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:57.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:57.661018+0000) 2024-03-06T11:16:58.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:58.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:58.661257+0000) 2024-03-06T11:16:59.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:16:59.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:16:59.661418+0000) 2024-03-06T11:17:00.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:00.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:00.661781+0000) 2024-03-06T11:17:01.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:01.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:01.661980+0000) 2024-03-06T11:17:02.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:02.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:02.662213+0000) 2024-03-06T11:17:03.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:03.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:03.662388+0000) 2024-03-06T11:17:04.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:04.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:04.662650+0000) 2024-03-06T11:17:05.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:05.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:05.662874+0000) 2024-03-06T11:17:06.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:06.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:06.663106+0000) 2024-03-06T11:17:07.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:07.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:07.663354+0000) 2024-03-06T11:17:08.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:08.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:08.663613+0000) 2024-03-06T11:17:09.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:09.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:09.663852+0000) 2024-03-06T11:17:10.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:10.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:10.664076+0000) 2024-03-06T11:17:11.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:11.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:11.664304+0000) 2024-03-06T11:17:12.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:12.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:12.664566+0000) 2024-03-06T11:17:13.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:13.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:13.664791+0000) 2024-03-06T11:17:14.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:14.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:14.665018+0000) 2024-03-06T11:17:15.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:15.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:15.665236+0000) 2024-03-06T11:17:16.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:16.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:16.665452+0000) 2024-03-06T11:17:17.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:17.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:17.665658+0000) 2024-03-06T11:17:18.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:18.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:18.665856+0000) 2024-03-06T11:17:19.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:19.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:19.666080+0000) 2024-03-06T11:17:20.103 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:17:20.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:17:20.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:20.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:20.666312+0000) 2024-03-06T11:17:21.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:21.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:21.666583+0000) 2024-03-06T11:17:22.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:22.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:22.666834+0000) 2024-03-06T11:17:23.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:23.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:23.667036+0000) 2024-03-06T11:17:24.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:24.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:24.667250+0000) 2024-03-06T11:17:25.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:25.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:25.667526+0000) 2024-03-06T11:17:26.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:26.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:26.667828+0000) 2024-03-06T11:17:27.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:27.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:27.668066+0000) 2024-03-06T11:17:28.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:28.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:28.668317+0000) 2024-03-06T11:17:29.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:29.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:29.668608+0000) 2024-03-06T11:17:30.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:30.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:30.668857+0000) 2024-03-06T11:17:31.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:31.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:31.669091+0000) 2024-03-06T11:17:32.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:32.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:32.669290+0000) 2024-03-06T11:17:33.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:33.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:33.669471+0000) 2024-03-06T11:17:34.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:34.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:34.669721+0000) 2024-03-06T11:17:35.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:35.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:35.669937+0000) 2024-03-06T11:17:36.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:36.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:36.670169+0000) 2024-03-06T11:17:37.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:37.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:37.670412+0000) 2024-03-06T11:17:38.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:38.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:38.670680+0000) 2024-03-06T11:17:39.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:39.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:39.670978+0000) 2024-03-06T11:17:40.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:40.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:40.671193+0000) 2024-03-06T11:17:41.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:41.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:41.671395+0000) 2024-03-06T11:17:42.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:42.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:42.671659+0000) 2024-03-06T11:17:43.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:43.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:43.671879+0000) 2024-03-06T11:17:44.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:44.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:44.672087+0000) 2024-03-06T11:17:45.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:45.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:45.672304+0000) 2024-03-06T11:17:46.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:46.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:46.672521+0000) 2024-03-06T11:17:47.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:47.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:47.672754+0000) 2024-03-06T11:17:48.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:48.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:48.672990+0000) 2024-03-06T11:17:49.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:49.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:49.673222+0000) 2024-03-06T11:17:50.333 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:17:50.341 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:17:50.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:50.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:50.673419+0000) 2024-03-06T11:17:51.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:51.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:51.673674+0000) 2024-03-06T11:17:52.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:52.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:52.673910+0000) 2024-03-06T11:17:53.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:53.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:53.674134+0000) 2024-03-06T11:17:54.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:54.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:54.674365+0000) 2024-03-06T11:17:55.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:55.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:55.674690+0000) 2024-03-06T11:17:56.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:56.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:56.674900+0000) 2024-03-06T11:17:57.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:57.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:57.675126+0000) 2024-03-06T11:17:58.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:58.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:58.675353+0000) 2024-03-06T11:17:59.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:17:59.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:17:59.675620+0000) 2024-03-06T11:18:00.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:00.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:00.675866+0000) 2024-03-06T11:18:01.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:01.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:01.676089+0000) 2024-03-06T11:18:02.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:02.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:02.676328+0000) 2024-03-06T11:18:03.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:03.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:03.676593+0000) 2024-03-06T11:18:04.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:04.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:04.676812+0000) 2024-03-06T11:18:05.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:05.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:05.677020+0000) 2024-03-06T11:18:06.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:06.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:06.677248+0000) 2024-03-06T11:18:07.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:07.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:07.677472+0000) 2024-03-06T11:18:08.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:08.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:08.677693+0000) 2024-03-06T11:18:09.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:09.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:09.677888+0000) 2024-03-06T11:18:10.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:10.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:10.678115+0000) 2024-03-06T11:18:11.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:11.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:11.678312+0000) 2024-03-06T11:18:12.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:12.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:12.678512+0000) 2024-03-06T11:18:13.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:13.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:13.678733+0000) 2024-03-06T11:18:14.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:14.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:14.678958+0000) 2024-03-06T11:18:15.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:15.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:15.679188+0000) 2024-03-06T11:18:16.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:16.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:16.679395+0000) 2024-03-06T11:18:17.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:17.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:17.679638+0000) 2024-03-06T11:18:18.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:18.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:18.679858+0000) 2024-03-06T11:18:19.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:19.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:19.680082+0000) 2024-03-06T11:18:20.563 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:18:20.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:18:20.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:20.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:20.680249+0000) 2024-03-06T11:18:21.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:21.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:21.680471+0000) 2024-03-06T11:18:22.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:22.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:22.680708+0000) 2024-03-06T11:18:23.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:23.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:23.680941+0000) 2024-03-06T11:18:24.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:24.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:24.681187+0000) 2024-03-06T11:18:25.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:25.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:25.681386+0000) 2024-03-06T11:18:26.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:26.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:26.681644+0000) 2024-03-06T11:18:27.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:27.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:27.681919+0000) 2024-03-06T11:18:28.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:28.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:28.682116+0000) 2024-03-06T11:18:29.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:29.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:29.682354+0000) 2024-03-06T11:18:30.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:30.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:30.682612+0000) 2024-03-06T11:18:31.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:31.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:31.682802+0000) 2024-03-06T11:18:32.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:32.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:32.683023+0000) 2024-03-06T11:18:33.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:33.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:33.683242+0000) 2024-03-06T11:18:34.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:34.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:34.683473+0000) 2024-03-06T11:18:35.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:35.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:35.683699+0000) 2024-03-06T11:18:36.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:36.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:36.683916+0000) 2024-03-06T11:18:37.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:37.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:37.684134+0000) 2024-03-06T11:18:38.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:38.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:38.684363+0000) 2024-03-06T11:18:39.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:39.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:39.684620+0000) 2024-03-06T11:18:40.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:40.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:40.684847+0000) 2024-03-06T11:18:41.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:41.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:41.685066+0000) 2024-03-06T11:18:42.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:42.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:42.685245+0000) 2024-03-06T11:18:43.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:43.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:43.685435+0000) 2024-03-06T11:18:44.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:44.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:44.685667+0000) 2024-03-06T11:18:45.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:45.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:45.685903+0000) 2024-03-06T11:18:46.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:46.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:46.686127+0000) 2024-03-06T11:18:47.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:47.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:47.686358+0000) 2024-03-06T11:18:48.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:48.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:48.686638+0000) 2024-03-06T11:18:49.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:49.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:49.686876+0000) 2024-03-06T11:18:50.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:50.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:50.687119+0000) 2024-03-06T11:18:50.784 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:18:50.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:18:51.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:51.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:51.687340+0000) 2024-03-06T11:18:52.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:52.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:52.687623+0000) 2024-03-06T11:18:53.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:53.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:53.687871+0000) 2024-03-06T11:18:54.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:54.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:54.688119+0000) 2024-03-06T11:18:55.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:55.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:55.688307+0000) 2024-03-06T11:18:56.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:56.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:56.688569+0000) 2024-03-06T11:18:57.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:57.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:57.688799+0000) 2024-03-06T11:18:58.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:58.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:58.689023+0000) 2024-03-06T11:18:59.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:18:59.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:18:59.689200+0000) 2024-03-06T11:19:00.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:00.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:00.689391+0000) 2024-03-06T11:19:01.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:01.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:01.689667+0000) 2024-03-06T11:19:02.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:02.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:02.689892+0000) 2024-03-06T11:19:03.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:03.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:03.690115+0000) 2024-03-06T11:19:04.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:04.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:04.690356+0000) 2024-03-06T11:19:05.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:05.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:05.690636+0000) 2024-03-06T11:19:06.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:06.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:06.690862+0000) 2024-03-06T11:19:07.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:07.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:07.691049+0000) 2024-03-06T11:19:08.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:08.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:08.691296+0000) 2024-03-06T11:19:09.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:09.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:09.691509+0000) 2024-03-06T11:19:10.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:10.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:10.691784+0000) 2024-03-06T11:19:11.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:11.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:11.692017+0000) 2024-03-06T11:19:12.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:12.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:12.692243+0000) 2024-03-06T11:19:13.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:13.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:13.692416+0000) 2024-03-06T11:19:14.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:14.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:14.692641+0000) 2024-03-06T11:19:15.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:15.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:15.692888+0000) 2024-03-06T11:19:16.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:16.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:16.693120+0000) 2024-03-06T11:19:17.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:17.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:17.693343+0000) 2024-03-06T11:19:18.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:18.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:18.693613+0000) 2024-03-06T11:19:19.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:19.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:19.693863+0000) 2024-03-06T11:19:20.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:20.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:20.694093+0000) 2024-03-06T11:19:20.922 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:19:20.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:19:21.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:21.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:21.694324+0000) 2024-03-06T11:19:22.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:22.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:22.694639+0000) 2024-03-06T11:19:23.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:23.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:23.694873+0000) 2024-03-06T11:19:24.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:24.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:24.695151+0000) 2024-03-06T11:19:25.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:25.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:25.695456+0000) 2024-03-06T11:19:26.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:26.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:26.695740+0000) 2024-03-06T11:19:27.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:27.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:27.696020+0000) 2024-03-06T11:19:28.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:28.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:28.696224+0000) 2024-03-06T11:19:29.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:29.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:29.696457+0000) 2024-03-06T11:19:30.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:30.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:30.696670+0000) 2024-03-06T11:19:31.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:31.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:31.696942+0000) 2024-03-06T11:19:32.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:32.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:32.697154+0000) 2024-03-06T11:19:33.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:33.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:33.697325+0000) 2024-03-06T11:19:34.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:34.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:34.697569+0000) 2024-03-06T11:19:35.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:35.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:35.697844+0000) 2024-03-06T11:19:36.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:36.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:36.698117+0000) 2024-03-06T11:19:37.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:37.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:37.698355+0000) 2024-03-06T11:19:38.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:38.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:38.698619+0000) 2024-03-06T11:19:39.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:39.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:39.698879+0000) 2024-03-06T11:19:40.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:40.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:40.699127+0000) 2024-03-06T11:19:41.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:41.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:41.699321+0000) 2024-03-06T11:19:42.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:42.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:42.699525+0000) 2024-03-06T11:19:43.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:43.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:43.699768+0000) 2024-03-06T11:19:44.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:44.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:44.700009+0000) 2024-03-06T11:19:45.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:45.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:45.700244+0000) 2024-03-06T11:19:46.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:46.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:46.700486+0000) 2024-03-06T11:19:47.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:47.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:47.700718+0000) 2024-03-06T11:19:48.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:48.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:48.700968+0000) 2024-03-06T11:19:49.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:49.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:49.701194+0000) 2024-03-06T11:19:50.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:50.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:50.701445+0000) 2024-03-06T11:19:51.286 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:19:51.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:19:51.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:51.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:51.701658+0000) 2024-03-06T11:19:52.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:52.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:52.701853+0000) 2024-03-06T11:19:53.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:53.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:53.702074+0000) 2024-03-06T11:19:54.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:54.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:54.702275+0000) 2024-03-06T11:19:55.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:55.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:55.702429+0000) 2024-03-06T11:19:56.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:56.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:56.702676+0000) 2024-03-06T11:19:57.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:57.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:57.702895+0000) 2024-03-06T11:19:58.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:58.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:58.703126+0000) 2024-03-06T11:19:59.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:19:59.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:19:59.703372+0000) 2024-03-06T11:20:00.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:00.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:00.703614+0000) 2024-03-06T11:20:01.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:01.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:01.703821+0000) 2024-03-06T11:20:02.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:02.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:02.704055+0000) 2024-03-06T11:20:03.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:03.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:03.704333+0000) 2024-03-06T11:20:04.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:04.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:04.704638+0000) 2024-03-06T11:20:05.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:05.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:05.704870+0000) 2024-03-06T11:20:06.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:06.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:06.705099+0000) 2024-03-06T11:20:07.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:07.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:07.705320+0000) 2024-03-06T11:20:08.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:08.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:08.705577+0000) 2024-03-06T11:20:09.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:09.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:09.705813+0000) 2024-03-06T11:20:10.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:10.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:10.706062+0000) 2024-03-06T11:20:11.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:11.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:11.706290+0000) 2024-03-06T11:20:12.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:12.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:12.706525+0000) 2024-03-06T11:20:13.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:13.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:13.706793+0000) 2024-03-06T11:20:14.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:14.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:14.707022+0000) 2024-03-06T11:20:15.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:15.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:15.707260+0000) 2024-03-06T11:20:16.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:16.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:16.707509+0000) 2024-03-06T11:20:17.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:17.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:17.707731+0000) 2024-03-06T11:20:18.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:18.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:18.707955+0000) 2024-03-06T11:20:19.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:19.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:19.708205+0000) 2024-03-06T11:20:20.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:20.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:20.708401+0000) 2024-03-06T11:20:21.441 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:20:21.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:20:21.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:21.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:21.708704+0000) 2024-03-06T11:20:22.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:22.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:22.708871+0000) 2024-03-06T11:20:23.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:23.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:23.709052+0000) 2024-03-06T11:20:24.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:24.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:24.709252+0000) 2024-03-06T11:20:25.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:25.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:25.709491+0000) 2024-03-06T11:20:26.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:26.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:26.709724+0000) 2024-03-06T11:20:27.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:27.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:27.709934+0000) 2024-03-06T11:20:28.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:28.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:28.710108+0000) 2024-03-06T11:20:29.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:29.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:29.710334+0000) 2024-03-06T11:20:30.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:30.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:30.710604+0000) 2024-03-06T11:20:31.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:31.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:31.710831+0000) 2024-03-06T11:20:32.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:32.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:32.711053+0000) 2024-03-06T11:20:33.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:33.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:33.711274+0000) 2024-03-06T11:20:34.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:34.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:34.711499+0000) 2024-03-06T11:20:35.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:35.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:35.711787+0000) 2024-03-06T11:20:36.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:36.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:36.711986+0000) 2024-03-06T11:20:37.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:37.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:37.712224+0000) 2024-03-06T11:20:38.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:38.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:38.712444+0000) 2024-03-06T11:20:39.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:39.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:39.712676+0000) 2024-03-06T11:20:40.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:40.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:40.712906+0000) 2024-03-06T11:20:41.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:41.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:41.713116+0000) 2024-03-06T11:20:42.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:42.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:42.713350+0000) 2024-03-06T11:20:43.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:43.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:43.713626+0000) 2024-03-06T11:20:44.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:44.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:44.713838+0000) 2024-03-06T11:20:45.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:45.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:45.714045+0000) 2024-03-06T11:20:46.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:46.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:46.714282+0000) 2024-03-06T11:20:47.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:47.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:47.714478+0000) 2024-03-06T11:20:48.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:48.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:48.714702+0000) 2024-03-06T11:20:49.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:49.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:49.714910+0000) 2024-03-06T11:20:50.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:50.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:50.715149+0000) 2024-03-06T11:20:51.655 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:20:51.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:20:51.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:51.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:51.715305+0000) 2024-03-06T11:20:52.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:52.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:52.715527+0000) 2024-03-06T11:20:53.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:53.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:53.715808+0000) 2024-03-06T11:20:54.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:54.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:54.716013+0000) 2024-03-06T11:20:55.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:55.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:55.716258+0000) 2024-03-06T11:20:56.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:56.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:56.716493+0000) 2024-03-06T11:20:57.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:57.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:57.716778+0000) 2024-03-06T11:20:58.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:58.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:58.717004+0000) 2024-03-06T11:20:59.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:20:59.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:20:59.717237+0000) 2024-03-06T11:21:00.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:00.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:00.717436+0000) 2024-03-06T11:21:01.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:01.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:01.717674+0000) 2024-03-06T11:21:02.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:02.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:02.717898+0000) 2024-03-06T11:21:03.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:03.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:03.718118+0000) 2024-03-06T11:21:04.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:04.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:04.718320+0000) 2024-03-06T11:21:05.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:05.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:05.718602+0000) 2024-03-06T11:21:06.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:06.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:06.718823+0000) 2024-03-06T11:21:07.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:07.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:07.719049+0000) 2024-03-06T11:21:08.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:08.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:08.719251+0000) 2024-03-06T11:21:09.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:09.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:09.719476+0000) 2024-03-06T11:21:10.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:10.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:10.719720+0000) 2024-03-06T11:21:11.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:11.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:11.719943+0000) 2024-03-06T11:21:12.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:12.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:12.720161+0000) 2024-03-06T11:21:13.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:13.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:13.720381+0000) 2024-03-06T11:21:14.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:14.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:14.720641+0000) 2024-03-06T11:21:15.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:15.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:15.720848+0000) 2024-03-06T11:21:16.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:16.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:16.721063+0000) 2024-03-06T11:21:17.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:17.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:17.721290+0000) 2024-03-06T11:21:18.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:18.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:18.721511+0000) 2024-03-06T11:21:19.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:19.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:19.721773+0000) 2024-03-06T11:21:20.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:20.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:20.722000+0000) 2024-03-06T11:21:21.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:21.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:21.722213+0000) 2024-03-06T11:21:21.865 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:21:21.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:21:22.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:22.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:22.722449+0000) 2024-03-06T11:21:23.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:23.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:23.722675+0000) 2024-03-06T11:21:24.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:24.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:24.722904+0000) 2024-03-06T11:21:25.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:25.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:25.723134+0000) 2024-03-06T11:21:26.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:26.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:26.723315+0000) 2024-03-06T11:21:27.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:27.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:27.723561+0000) 2024-03-06T11:21:28.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:28.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:28.723796+0000) 2024-03-06T11:21:29.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:29.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:29.724016+0000) 2024-03-06T11:21:30.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:30.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:30.724248+0000) 2024-03-06T11:21:31.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:31.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:31.724471+0000) 2024-03-06T11:21:32.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:32.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:32.724715+0000) 2024-03-06T11:21:33.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:33.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:33.724938+0000) 2024-03-06T11:21:34.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:34.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:34.725161+0000) 2024-03-06T11:21:35.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:35.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:35.725387+0000) 2024-03-06T11:21:36.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:36.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:36.725704+0000) 2024-03-06T11:21:37.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:37.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:37.725930+0000) 2024-03-06T11:21:38.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:38.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:38.726154+0000) 2024-03-06T11:21:39.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:39.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:39.726381+0000) 2024-03-06T11:21:40.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:40.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:40.726647+0000) 2024-03-06T11:21:41.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:41.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:41.726829+0000) 2024-03-06T11:21:42.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:42.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:42.727066+0000) 2024-03-06T11:21:43.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:43.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:43.727283+0000) 2024-03-06T11:21:44.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:44.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:44.727590+0000) 2024-03-06T11:21:45.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:45.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:45.727823+0000) 2024-03-06T11:21:46.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:46.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:46.728047+0000) 2024-03-06T11:21:47.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:47.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:47.728278+0000) 2024-03-06T11:21:48.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:48.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:48.728512+0000) 2024-03-06T11:21:49.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:49.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:49.728787+0000) 2024-03-06T11:21:50.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:50.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:50.728987+0000) 2024-03-06T11:21:51.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:51.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:51.729189+0000) 2024-03-06T11:21:52.004 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:21:52.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:21:52.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:52.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:52.729428+0000) 2024-03-06T11:21:53.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:53.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:53.729654+0000) 2024-03-06T11:21:54.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:54.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:54.729876+0000) 2024-03-06T11:21:55.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:55.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:55.730117+0000) 2024-03-06T11:21:56.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:56.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:56.730319+0000) 2024-03-06T11:21:57.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:57.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:57.730580+0000) 2024-03-06T11:21:58.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:58.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:58.730820+0000) 2024-03-06T11:21:59.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:21:59.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:21:59.731003+0000) 2024-03-06T11:22:00.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:00.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:00.731253+0000) 2024-03-06T11:22:01.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:01.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:01.731452+0000) 2024-03-06T11:22:02.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:02.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:02.731679+0000) 2024-03-06T11:22:03.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:03.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:03.731909+0000) 2024-03-06T11:22:04.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:04.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:04.732138+0000) 2024-03-06T11:22:05.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:05.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:05.732382+0000) 2024-03-06T11:22:06.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:06.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:06.732652+0000) 2024-03-06T11:22:07.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:07.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:07.732858+0000) 2024-03-06T11:22:08.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:08.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:08.733090+0000) 2024-03-06T11:22:09.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:09.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:09.733308+0000) 2024-03-06T11:22:10.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:10.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:10.733573+0000) 2024-03-06T11:22:11.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:11.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:11.733765+0000) 2024-03-06T11:22:12.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:12.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:12.733998+0000) 2024-03-06T11:22:13.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:13.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:13.734193+0000) 2024-03-06T11:22:14.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:14.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:14.734417+0000) 2024-03-06T11:22:15.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:15.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:15.734673+0000) 2024-03-06T11:22:16.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:16.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:16.734900+0000) 2024-03-06T11:22:17.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:17.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:17.735124+0000) 2024-03-06T11:22:18.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:18.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:18.735361+0000) 2024-03-06T11:22:19.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:19.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:19.735623+0000) 2024-03-06T11:22:20.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:20.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:20.735837+0000) 2024-03-06T11:22:21.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:21.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:21.736066+0000) 2024-03-06T11:22:22.216 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:22:22.224 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:22:22.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:22.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:22.736295+0000) 2024-03-06T11:22:23.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:23.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:23.736517+0000) 2024-03-06T11:22:24.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:24.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:24.736787+0000) 2024-03-06T11:22:25.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:25.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:25.737014+0000) 2024-03-06T11:22:26.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:26.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:26.737241+0000) 2024-03-06T11:22:27.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:27.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:27.737468+0000) 2024-03-06T11:22:28.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:28.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:28.737713+0000) 2024-03-06T11:22:29.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:29.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:29.737937+0000) 2024-03-06T11:22:30.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:30.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:30.738164+0000) 2024-03-06T11:22:31.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:31.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:31.738391+0000) 2024-03-06T11:22:32.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:32.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:32.738654+0000) 2024-03-06T11:22:33.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:33.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:33.738877+0000) 2024-03-06T11:22:34.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:34.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:34.739100+0000) 2024-03-06T11:22:35.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:35.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:35.739367+0000) 2024-03-06T11:22:36.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:36.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:36.739596+0000) 2024-03-06T11:22:37.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:37.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:37.739884+0000) 2024-03-06T11:22:38.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:38.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:38.740134+0000) 2024-03-06T11:22:39.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:39.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:39.740350+0000) 2024-03-06T11:22:40.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:40.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:40.740656+0000) 2024-03-06T11:22:41.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:41.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:41.740892+0000) 2024-03-06T11:22:42.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:42.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:42.741130+0000) 2024-03-06T11:22:43.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:43.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:43.741380+0000) 2024-03-06T11:22:44.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:44.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:44.741699+0000) 2024-03-06T11:22:45.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:45.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:45.741920+0000) 2024-03-06T11:22:46.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:46.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:46.742141+0000) 2024-03-06T11:22:47.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:47.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:47.742325+0000) 2024-03-06T11:22:48.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:48.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:48.742597+0000) 2024-03-06T11:22:49.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:49.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:49.742801+0000) 2024-03-06T11:22:50.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:50.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:50.743033+0000) 2024-03-06T11:22:51.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:51.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:51.743281+0000) 2024-03-06T11:22:52.455 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:22:52.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:22:52.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:52.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:52.743577+0000) 2024-03-06T11:22:53.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:53.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:53.743805+0000) 2024-03-06T11:22:54.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:54.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:54.744050+0000) 2024-03-06T11:22:55.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:55.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:55.744275+0000) 2024-03-06T11:22:56.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:56.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:56.744577+0000) 2024-03-06T11:22:57.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:57.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:57.744811+0000) 2024-03-06T11:22:58.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:58.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:58.745039+0000) 2024-03-06T11:22:59.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:22:59.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:22:59.745275+0000) 2024-03-06T11:23:00.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:00.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:00.745514+0000) 2024-03-06T11:23:01.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:01.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:01.745798+0000) 2024-03-06T11:23:02.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:02.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:02.745986+0000) 2024-03-06T11:23:03.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:03.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:03.746185+0000) 2024-03-06T11:23:04.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:04.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:04.746427+0000) 2024-03-06T11:23:05.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:05.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:05.746668+0000) 2024-03-06T11:23:06.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:06.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:06.746895+0000) 2024-03-06T11:23:07.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:07.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:07.747122+0000) 2024-03-06T11:23:08.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:08.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:08.747353+0000) 2024-03-06T11:23:09.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:09.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:09.747603+0000) 2024-03-06T11:23:10.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:10.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:10.747858+0000) 2024-03-06T11:23:11.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:11.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:11.748127+0000) 2024-03-06T11:23:12.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:12.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:12.748402+0000) 2024-03-06T11:23:13.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:13.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:13.748678+0000) 2024-03-06T11:23:14.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:14.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:14.748957+0000) 2024-03-06T11:23:15.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:15.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:15.749194+0000) 2024-03-06T11:23:16.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:16.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:16.749433+0000) 2024-03-06T11:23:17.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:17.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:17.749623+0000) 2024-03-06T11:23:18.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:18.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:18.749844+0000) 2024-03-06T11:23:19.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:19.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:19.750037+0000) 2024-03-06T11:23:20.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:20.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:20.750271+0000) 2024-03-06T11:23:21.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:21.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:21.750571+0000) 2024-03-06T11:23:22.685 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:23:22.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:23:22.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:22.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:22.750808+0000) 2024-03-06T11:23:23.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:23.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:23.751046+0000) 2024-03-06T11:23:24.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:24.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:24.751275+0000) 2024-03-06T11:23:25.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:25.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:25.751583+0000) 2024-03-06T11:23:26.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:26.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:26.751866+0000) 2024-03-06T11:23:27.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:27.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:27.752092+0000) 2024-03-06T11:23:28.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:28.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:28.752327+0000) 2024-03-06T11:23:29.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:29.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:29.752529+0000) 2024-03-06T11:23:30.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:30.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:30.752806+0000) 2024-03-06T11:23:31.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:31.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:31.753130+0000) 2024-03-06T11:23:32.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:32.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:32.753363+0000) 2024-03-06T11:23:33.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:33.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:33.753676+0000) 2024-03-06T11:23:34.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:34.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:34.753949+0000) 2024-03-06T11:23:35.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:35.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:35.754186+0000) 2024-03-06T11:23:36.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:36.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:36.754431+0000) 2024-03-06T11:23:37.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:37.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:37.754659+0000) 2024-03-06T11:23:38.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:38.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:38.754875+0000) 2024-03-06T11:23:39.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:39.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:39.755088+0000) 2024-03-06T11:23:40.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:40.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:40.755285+0000) 2024-03-06T11:23:41.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:41.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:41.755499+0000) 2024-03-06T11:23:42.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:42.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:42.755756+0000) 2024-03-06T11:23:43.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:43.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:43.756022+0000) 2024-03-06T11:23:44.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:44.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:44.756290+0000) 2024-03-06T11:23:45.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:45.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:45.756619+0000) 2024-03-06T11:23:46.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:46.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:46.756901+0000) 2024-03-06T11:23:47.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:47.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:47.757169+0000) 2024-03-06T11:23:48.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:48.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:48.757451+0000) 2024-03-06T11:23:49.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:49.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:49.757697+0000) 2024-03-06T11:23:50.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:50.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:50.757941+0000) 2024-03-06T11:23:51.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:51.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:51.758173+0000) 2024-03-06T11:23:52.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:52.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:52.758433+0000) 2024-03-06T11:23:52.864 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:23:52.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:23:53.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:53.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:53.758652+0000) 2024-03-06T11:23:54.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:54.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:54.758924+0000) 2024-03-06T11:23:55.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:55.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:55.759219+0000) 2024-03-06T11:23:56.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:56.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:56.759426+0000) 2024-03-06T11:23:57.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:57.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:57.759651+0000) 2024-03-06T11:23:58.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:58.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:58.759878+0000) 2024-03-06T11:23:59.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:23:59.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:23:59.760093+0000) 2024-03-06T11:24:00.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:00.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:00.760323+0000) 2024-03-06T11:24:01.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:01.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:01.760600+0000) 2024-03-06T11:24:02.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:02.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:02.760812+0000) 2024-03-06T11:24:03.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:03.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:03.761080+0000) 2024-03-06T11:24:04.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:04.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:04.761357+0000) 2024-03-06T11:24:05.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:05.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:05.761697+0000) 2024-03-06T11:24:06.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:06.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:06.761867+0000) 2024-03-06T11:24:07.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:07.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:07.762043+0000) 2024-03-06T11:24:08.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:08.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:08.762248+0000) 2024-03-06T11:24:09.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:09.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:09.762447+0000) 2024-03-06T11:24:10.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:10.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:10.762717+0000) 2024-03-06T11:24:11.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:11.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:11.762995+0000) 2024-03-06T11:24:12.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:12.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:12.763261+0000) 2024-03-06T11:24:13.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:13.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:13.763590+0000) 2024-03-06T11:24:14.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:14.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:14.763794+0000) 2024-03-06T11:24:15.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:15.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:15.764006+0000) 2024-03-06T11:24:16.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:16.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:16.764257+0000) 2024-03-06T11:24:17.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:17.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:17.764566+0000) 2024-03-06T11:24:18.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:18.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:18.764823+0000) 2024-03-06T11:24:19.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:19.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:19.765047+0000) 2024-03-06T11:24:20.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:20.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:20.765297+0000) 2024-03-06T11:24:21.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:21.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:21.765502+0000) 2024-03-06T11:24:22.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:22.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:22.765740+0000) 2024-03-06T11:24:23.068 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:24:23.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:24:23.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:23.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:23.765963+0000) 2024-03-06T11:24:24.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:24.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:24.766191+0000) 2024-03-06T11:24:25.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:25.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:25.766400+0000) 2024-03-06T11:24:26.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:26.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:26.766674+0000) 2024-03-06T11:24:27.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:27.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:27.766890+0000) 2024-03-06T11:24:28.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:28.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:28.767097+0000) 2024-03-06T11:24:29.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:29.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:29.767334+0000) 2024-03-06T11:24:30.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:30.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:30.767610+0000) 2024-03-06T11:24:31.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:31.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:31.767826+0000) 2024-03-06T11:24:32.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:32.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:32.768021+0000) 2024-03-06T11:24:33.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:33.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:33.768254+0000) 2024-03-06T11:24:34.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:34.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:34.768465+0000) 2024-03-06T11:24:35.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:35.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:35.768704+0000) 2024-03-06T11:24:36.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:36.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:36.768907+0000) 2024-03-06T11:24:37.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:37.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:37.769133+0000) 2024-03-06T11:24:38.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:38.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:38.769350+0000) 2024-03-06T11:24:39.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:39.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:39.769625+0000) 2024-03-06T11:24:40.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:40.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:40.769893+0000) 2024-03-06T11:24:41.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:41.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:41.770172+0000) 2024-03-06T11:24:42.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:42.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:42.770450+0000) 2024-03-06T11:24:43.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:43.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:43.770688+0000) 2024-03-06T11:24:44.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:44.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:44.770905+0000) 2024-03-06T11:24:45.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:45.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:45.771197+0000) 2024-03-06T11:24:46.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:46.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:46.771476+0000) 2024-03-06T11:24:47.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:47.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:47.771752+0000) 2024-03-06T11:24:48.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:48.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:48.772024+0000) 2024-03-06T11:24:49.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:49.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:49.772298+0000) 2024-03-06T11:24:50.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:50.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:50.772619+0000) 2024-03-06T11:24:51.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:51.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:51.772894+0000) 2024-03-06T11:24:52.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:52.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:52.773137+0000) 2024-03-06T11:24:53.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:24:53.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:24:53.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:53.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:53.773410+0000) 2024-03-06T11:24:54.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:54.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:54.773713+0000) 2024-03-06T11:24:55.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:55.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:55.774019+0000) 2024-03-06T11:24:56.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:56.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:56.774257+0000) 2024-03-06T11:24:57.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:57.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:57.774571+0000) 2024-03-06T11:24:58.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:58.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:58.774841+0000) 2024-03-06T11:24:59.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:24:59.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:24:59.775137+0000) 2024-03-06T11:25:00.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:00.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:00.775386+0000) 2024-03-06T11:25:01.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:01.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:01.775689+0000) 2024-03-06T11:25:02.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:02.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:02.775900+0000) 2024-03-06T11:25:03.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:03.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:03.776124+0000) 2024-03-06T11:25:04.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:04.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:04.776355+0000) 2024-03-06T11:25:05.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:05.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:05.776681+0000) 2024-03-06T11:25:06.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:06.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:06.776880+0000) 2024-03-06T11:25:07.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:07.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:07.777142+0000) 2024-03-06T11:25:08.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:08.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:08.777415+0000) 2024-03-06T11:25:09.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:09.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:09.777668+0000) 2024-03-06T11:25:10.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:10.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:10.777917+0000) 2024-03-06T11:25:11.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:11.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:11.778133+0000) 2024-03-06T11:25:12.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:12.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:12.778342+0000) 2024-03-06T11:25:13.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:13.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:13.778575+0000) 2024-03-06T11:25:14.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:14.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:14.778806+0000) 2024-03-06T11:25:15.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:15.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:15.779027+0000) 2024-03-06T11:25:16.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:16.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:16.779270+0000) 2024-03-06T11:25:17.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:17.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:17.779490+0000) 2024-03-06T11:25:18.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:18.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:18.779717+0000) 2024-03-06T11:25:19.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:19.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:19.779918+0000) 2024-03-06T11:25:20.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:20.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:20.780198+0000) 2024-03-06T11:25:21.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:21.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:21.780481+0000) 2024-03-06T11:25:22.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:22.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:22.780763+0000) 2024-03-06T11:25:23.561 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:25:23.569 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:25:23.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:23.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:23.780982+0000) 2024-03-06T11:25:24.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:24.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:24.781260+0000) 2024-03-06T11:25:25.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:25.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:25.781600+0000) 2024-03-06T11:25:26.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:26.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:26.781813+0000) 2024-03-06T11:25:27.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:27.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:27.782038+0000) 2024-03-06T11:25:28.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:28.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:28.782235+0000) 2024-03-06T11:25:29.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:29.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:29.782433+0000) 2024-03-06T11:25:30.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:30.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:30.782664+0000) 2024-03-06T11:25:31.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:31.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:31.782908+0000) 2024-03-06T11:25:32.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:32.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:32.783099+0000) 2024-03-06T11:25:33.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:33.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:33.783336+0000) 2024-03-06T11:25:34.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:34.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:34.783601+0000) 2024-03-06T11:25:35.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:35.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:35.783870+0000) 2024-03-06T11:25:36.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:36.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:36.784103+0000) 2024-03-06T11:25:37.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:37.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:37.784356+0000) 2024-03-06T11:25:38.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:38.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:38.784628+0000) 2024-03-06T11:25:39.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:39.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:39.784873+0000) 2024-03-06T11:25:40.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:40.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:40.785093+0000) 2024-03-06T11:25:41.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:41.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:41.785309+0000) 2024-03-06T11:25:42.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:42.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:42.785554+0000) 2024-03-06T11:25:43.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:43.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:43.785813+0000) 2024-03-06T11:25:44.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:44.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:44.786032+0000) 2024-03-06T11:25:45.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:45.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:45.786266+0000) 2024-03-06T11:25:46.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:46.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:46.786493+0000) 2024-03-06T11:25:47.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:47.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:47.786689+0000) 2024-03-06T11:25:48.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:48.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:48.786918+0000) 2024-03-06T11:25:49.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:49.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:49.787100+0000) 2024-03-06T11:25:50.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:50.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:50.787337+0000) 2024-03-06T11:25:51.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:51.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:51.787527+0000) 2024-03-06T11:25:52.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:52.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:52.787802+0000) 2024-03-06T11:25:53.774 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:25:53.782 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:25:53.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:53.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:53.788020+0000) 2024-03-06T11:25:54.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:54.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:54.788219+0000) 2024-03-06T11:25:55.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:55.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:55.788577+0000) 2024-03-06T11:25:56.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:56.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:56.788766+0000) 2024-03-06T11:25:57.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:57.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:57.788993+0000) 2024-03-06T11:25:58.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:58.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:58.789226+0000) 2024-03-06T11:25:59.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:25:59.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:25:59.789407+0000) 2024-03-06T11:26:00.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:00.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:00.789678+0000) 2024-03-06T11:26:01.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:01.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:01.789906+0000) 2024-03-06T11:26:02.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:02.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:02.790136+0000) 2024-03-06T11:26:03.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:03.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:03.790325+0000) 2024-03-06T11:26:04.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:04.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:04.790575+0000) 2024-03-06T11:26:05.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:05.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:05.790842+0000) 2024-03-06T11:26:06.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:06.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:06.791070+0000) 2024-03-06T11:26:07.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:07.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:07.791285+0000) 2024-03-06T11:26:08.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:08.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:08.791511+0000) 2024-03-06T11:26:09.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:09.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:09.791780+0000) 2024-03-06T11:26:10.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:10.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:10.792004+0000) 2024-03-06T11:26:11.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:11.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:11.792268+0000) 2024-03-06T11:26:12.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:12.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:12.792501+0000) 2024-03-06T11:26:13.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:13.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:13.792729+0000) 2024-03-06T11:26:14.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:14.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:14.792941+0000) 2024-03-06T11:26:15.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:15.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:15.793173+0000) 2024-03-06T11:26:16.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:16.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:16.793403+0000) 2024-03-06T11:26:17.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:17.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:17.793653+0000) 2024-03-06T11:26:18.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:18.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:18.793928+0000) 2024-03-06T11:26:19.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:19.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:19.794167+0000) 2024-03-06T11:26:20.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:20.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:20.794397+0000) 2024-03-06T11:26:21.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:21.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:21.794663+0000) 2024-03-06T11:26:22.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:22.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:22.794878+0000) 2024-03-06T11:26:23.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:23.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:23.795091+0000) 2024-03-06T11:26:23.962 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:26:23.972 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:26:24.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:24.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:24.795345+0000) 2024-03-06T11:26:25.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:25.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:25.795677+0000) 2024-03-06T11:26:26.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:26.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:26.795958+0000) 2024-03-06T11:26:27.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:27.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:27.796168+0000) 2024-03-06T11:26:28.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:28.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:28.796399+0000) 2024-03-06T11:26:29.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:29.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:29.796653+0000) 2024-03-06T11:26:30.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:30.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:30.796926+0000) 2024-03-06T11:26:31.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:31.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:31.797199+0000) 2024-03-06T11:26:32.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:32.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:32.797414+0000) 2024-03-06T11:26:33.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:33.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:33.797644+0000) 2024-03-06T11:26:34.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:34.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:34.797872+0000) 2024-03-06T11:26:35.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:35.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:35.798127+0000) 2024-03-06T11:26:36.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:36.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:36.798353+0000) 2024-03-06T11:26:37.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:37.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:37.798603+0000) 2024-03-06T11:26:38.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:38.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:38.798807+0000) 2024-03-06T11:26:39.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:39.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:39.799059+0000) 2024-03-06T11:26:40.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:40.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:40.799262+0000) 2024-03-06T11:26:41.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:41.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:41.799499+0000) 2024-03-06T11:26:42.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:42.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:42.799730+0000) 2024-03-06T11:26:43.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:43.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:43.799925+0000) 2024-03-06T11:26:44.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:44.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:44.800156+0000) 2024-03-06T11:26:45.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:45.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:45.800351+0000) 2024-03-06T11:26:46.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:46.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:46.800620+0000) 2024-03-06T11:26:47.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:47.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:47.800838+0000) 2024-03-06T11:26:48.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:48.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:48.801105+0000) 2024-03-06T11:26:49.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:49.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:49.801323+0000) 2024-03-06T11:26:50.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:50.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:50.801568+0000) 2024-03-06T11:26:51.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:51.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:51.801782+0000) 2024-03-06T11:26:52.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:52.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:52.802033+0000) 2024-03-06T11:26:53.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:53.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:53.802259+0000) 2024-03-06T11:26:54.166 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:26:54.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:26:54.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:54.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:54.802489+0000) 2024-03-06T11:26:55.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:55.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:55.802732+0000) 2024-03-06T11:26:56.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:56.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:56.803006+0000) 2024-03-06T11:26:57.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:57.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:57.803276+0000) 2024-03-06T11:26:58.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:58.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:58.803591+0000) 2024-03-06T11:26:59.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:26:59.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:26:59.803810+0000) 2024-03-06T11:27:00.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:00.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:00.804085+0000) 2024-03-06T11:27:01.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:01.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:01.804369+0000) 2024-03-06T11:27:02.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:02.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:02.804638+0000) 2024-03-06T11:27:03.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:03.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:03.804878+0000) 2024-03-06T11:27:04.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:04.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:04.805155+0000) 2024-03-06T11:27:05.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:05.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:05.805408+0000) 2024-03-06T11:27:06.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:06.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:06.805661+0000) 2024-03-06T11:27:07.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:07.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:07.805945+0000) 2024-03-06T11:27:08.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:08.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:08.806227+0000) 2024-03-06T11:27:09.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:09.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:09.806503+0000) 2024-03-06T11:27:10.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:10.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:10.806808+0000) 2024-03-06T11:27:11.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:11.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:11.807076+0000) 2024-03-06T11:27:12.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:12.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:12.807353+0000) 2024-03-06T11:27:13.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:13.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:13.807585+0000) 2024-03-06T11:27:14.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:14.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:14.807808+0000) 2024-03-06T11:27:15.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:15.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:15.808044+0000) 2024-03-06T11:27:16.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:16.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:16.808281+0000) 2024-03-06T11:27:17.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:17.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:17.808503+0000) 2024-03-06T11:27:18.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:18.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:18.808763+0000) 2024-03-06T11:27:19.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:19.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:19.808992+0000) 2024-03-06T11:27:20.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:20.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:20.809227+0000) 2024-03-06T11:27:21.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:21.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:21.809452+0000) 2024-03-06T11:27:22.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:22.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:22.809777+0000) 2024-03-06T11:27:23.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:23.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:23.809968+0000) 2024-03-06T11:27:24.380 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:27:24.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:27:24.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:24.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:24.810212+0000) 2024-03-06T11:27:25.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:25.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:25.810416+0000) 2024-03-06T11:27:26.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:26.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:26.810654+0000) 2024-03-06T11:27:27.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:27.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:27.810881+0000) 2024-03-06T11:27:28.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:28.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:28.811103+0000) 2024-03-06T11:27:29.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:29.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:29.811286+0000) 2024-03-06T11:27:30.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:30.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:30.811471+0000) 2024-03-06T11:27:31.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:31.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:31.811704+0000) 2024-03-06T11:27:32.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:32.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:32.811924+0000) 2024-03-06T11:27:33.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:33.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:33.812152+0000) 2024-03-06T11:27:34.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:34.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:34.812378+0000) 2024-03-06T11:27:35.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:35.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:35.812657+0000) 2024-03-06T11:27:36.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:36.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:36.812889+0000) 2024-03-06T11:27:37.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:37.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:37.813074+0000) 2024-03-06T11:27:38.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:38.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:38.813304+0000) 2024-03-06T11:27:39.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:39.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:39.813574+0000) 2024-03-06T11:27:40.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:40.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:40.813821+0000) 2024-03-06T11:27:41.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:41.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:41.814065+0000) 2024-03-06T11:27:42.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:42.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:42.814279+0000) 2024-03-06T11:27:43.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:43.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:43.814525+0000) 2024-03-06T11:27:44.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:44.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:44.814782+0000) 2024-03-06T11:27:45.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:45.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:45.815034+0000) 2024-03-06T11:27:46.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:46.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:46.815273+0000) 2024-03-06T11:27:47.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:47.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:47.815512+0000) 2024-03-06T11:27:48.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:48.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:48.815736+0000) 2024-03-06T11:27:49.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:49.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:49.815930+0000) 2024-03-06T11:27:50.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:50.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:50.816161+0000) 2024-03-06T11:27:51.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:51.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:51.816359+0000) 2024-03-06T11:27:52.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:52.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:52.816662+0000) 2024-03-06T11:27:53.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:53.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:53.816889+0000) 2024-03-06T11:27:54.601 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:27:54.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:27:54.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:54.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:54.817134+0000) 2024-03-06T11:27:55.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:55.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:55.817375+0000) 2024-03-06T11:27:56.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:56.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:56.817646+0000) 2024-03-06T11:27:57.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:57.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:57.817853+0000) 2024-03-06T11:27:58.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:58.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:58.818040+0000) 2024-03-06T11:27:59.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:27:59.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:27:59.818245+0000) 2024-03-06T11:28:00.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:00.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:00.818480+0000) 2024-03-06T11:28:01.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:01.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:01.818723+0000) 2024-03-06T11:28:02.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:02.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:02.818942+0000) 2024-03-06T11:28:03.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:03.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:03.819187+0000) 2024-03-06T11:28:04.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:04.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:04.819502+0000) 2024-03-06T11:28:05.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:05.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:05.819819+0000) 2024-03-06T11:28:06.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:06.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:06.819995+0000) 2024-03-06T11:28:07.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:07.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:07.820221+0000) 2024-03-06T11:28:08.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:08.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:08.820454+0000) 2024-03-06T11:28:09.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:09.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:09.820700+0000) 2024-03-06T11:28:10.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:10.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:10.820892+0000) 2024-03-06T11:28:11.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:11.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:11.821089+0000) 2024-03-06T11:28:12.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:12.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:12.821332+0000) 2024-03-06T11:28:13.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:13.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:13.821595+0000) 2024-03-06T11:28:14.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:14.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:14.821748+0000) 2024-03-06T11:28:15.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:15.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:15.821992+0000) 2024-03-06T11:28:16.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:16.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:16.822236+0000) 2024-03-06T11:28:17.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:17.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:17.822474+0000) 2024-03-06T11:28:18.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:18.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:18.822702+0000) 2024-03-06T11:28:19.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:19.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:19.822947+0000) 2024-03-06T11:28:20.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:20.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:20.823158+0000) 2024-03-06T11:28:21.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:21.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:21.823392+0000) 2024-03-06T11:28:22.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:22.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:22.823673+0000) 2024-03-06T11:28:23.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:23.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:23.823911+0000) 2024-03-06T11:28:24.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:24.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:24.824162+0000) 2024-03-06T11:28:24.832 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:28:24.840 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:28:25.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:25.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:25.824421+0000) 2024-03-06T11:28:26.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:26.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:26.824658+0000) 2024-03-06T11:28:27.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:27.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:27.824904+0000) 2024-03-06T11:28:28.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:28.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:28.825135+0000) 2024-03-06T11:28:29.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:29.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:29.825295+0000) 2024-03-06T11:28:30.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:30.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:30.825491+0000) 2024-03-06T11:28:31.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:31.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:31.825723+0000) 2024-03-06T11:28:32.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:32.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:32.825934+0000) 2024-03-06T11:28:33.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:33.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:33.826125+0000) 2024-03-06T11:28:34.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:34.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:34.826358+0000) 2024-03-06T11:28:35.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:35.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:35.826648+0000) 2024-03-06T11:28:36.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:36.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:36.826893+0000) 2024-03-06T11:28:37.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:37.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:37.827124+0000) 2024-03-06T11:28:38.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:38.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:38.827312+0000) 2024-03-06T11:28:39.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:39.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:39.827570+0000) 2024-03-06T11:28:40.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:40.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:40.827812+0000) 2024-03-06T11:28:41.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:41.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:41.828038+0000) 2024-03-06T11:28:42.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:42.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:42.828264+0000) 2024-03-06T11:28:43.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:43.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:43.828490+0000) 2024-03-06T11:28:44.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:44.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:44.828736+0000) 2024-03-06T11:28:45.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:45.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:45.828936+0000) 2024-03-06T11:28:46.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:46.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:46.829171+0000) 2024-03-06T11:28:47.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:47.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:47.829398+0000) 2024-03-06T11:28:48.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:48.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:48.829660+0000) 2024-03-06T11:28:49.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:49.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:49.829881+0000) 2024-03-06T11:28:50.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:50.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:50.830107+0000) 2024-03-06T11:28:51.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:51.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:51.830319+0000) 2024-03-06T11:28:52.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:52.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:52.830525+0000) 2024-03-06T11:28:53.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:53.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:53.830802+0000) 2024-03-06T11:28:54.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:54.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:54.831027+0000) 2024-03-06T11:28:55.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:28:55.010 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:28:55.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:55.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:55.831256+0000) 2024-03-06T11:28:56.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:56.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:56.831489+0000) 2024-03-06T11:28:57.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:57.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:57.831714+0000) 2024-03-06T11:28:58.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:58.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:58.831874+0000) 2024-03-06T11:28:59.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:28:59.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:28:59.832116+0000) 2024-03-06T11:29:00.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:00.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:00.832346+0000) 2024-03-06T11:29:01.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:01.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:01.832626+0000) 2024-03-06T11:29:02.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:02.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:02.832893+0000) 2024-03-06T11:29:03.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:03.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:03.833140+0000) 2024-03-06T11:29:04.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:04.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:04.833353+0000) 2024-03-06T11:29:05.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:05.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:05.833645+0000) 2024-03-06T11:29:06.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:06.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:06.833843+0000) 2024-03-06T11:29:07.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:07.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:07.834116+0000) 2024-03-06T11:29:08.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:08.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:08.834317+0000) 2024-03-06T11:29:09.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:09.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:09.834503+0000) 2024-03-06T11:29:10.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:10.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:10.834764+0000) 2024-03-06T11:29:11.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:11.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:11.834974+0000) 2024-03-06T11:29:12.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:12.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:12.835234+0000) 2024-03-06T11:29:13.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:13.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:13.835503+0000) 2024-03-06T11:29:14.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:14.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:14.835722+0000) 2024-03-06T11:29:15.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:15.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:15.836002+0000) 2024-03-06T11:29:16.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:16.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:16.836249+0000) 2024-03-06T11:29:17.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:17.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:17.836516+0000) 2024-03-06T11:29:18.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:18.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:18.836771+0000) 2024-03-06T11:29:19.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:19.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:19.836978+0000) 2024-03-06T11:29:20.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:20.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:20.837215+0000) 2024-03-06T11:29:21.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:21.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:21.837460+0000) 2024-03-06T11:29:22.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:22.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:22.837695+0000) 2024-03-06T11:29:23.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:23.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:23.837926+0000) 2024-03-06T11:29:24.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:24.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:24.838126+0000) 2024-03-06T11:29:25.224 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:29:25.232 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:29:25.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:25.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:25.838353+0000) 2024-03-06T11:29:26.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:26.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:26.838594+0000) 2024-03-06T11:29:27.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:27.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:27.838861+0000) 2024-03-06T11:29:28.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:28.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:28.839130+0000) 2024-03-06T11:29:29.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:29.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:29.839367+0000) 2024-03-06T11:29:30.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:30.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:30.839635+0000) 2024-03-06T11:29:31.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:31.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:31.839874+0000) 2024-03-06T11:29:32.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:32.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:32.840122+0000) 2024-03-06T11:29:33.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:33.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:33.840344+0000) 2024-03-06T11:29:34.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:34.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:34.840652+0000) 2024-03-06T11:29:35.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:35.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:35.840902+0000) 2024-03-06T11:29:36.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:36.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:36.841150+0000) 2024-03-06T11:29:37.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:37.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:37.841390+0000) 2024-03-06T11:29:38.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:38.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:38.841628+0000) 2024-03-06T11:29:39.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:39.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:39.841881+0000) 2024-03-06T11:29:40.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:40.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:40.842114+0000) 2024-03-06T11:29:41.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:41.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:41.842345+0000) 2024-03-06T11:29:42.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:42.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:42.842572+0000) 2024-03-06T11:29:43.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:43.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:43.842794+0000) 2024-03-06T11:29:44.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:44.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:44.843029+0000) 2024-03-06T11:29:45.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:45.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:45.843310+0000) 2024-03-06T11:29:46.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:46.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:46.843627+0000) 2024-03-06T11:29:47.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:47.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:47.843908+0000) 2024-03-06T11:29:48.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:48.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:48.844194+0000) 2024-03-06T11:29:49.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:49.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:49.844455+0000) 2024-03-06T11:29:50.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:50.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:50.844678+0000) 2024-03-06T11:29:51.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:51.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:51.844916+0000) 2024-03-06T11:29:52.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:52.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:52.845131+0000) 2024-03-06T11:29:53.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:53.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:53.845372+0000) 2024-03-06T11:29:54.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:54.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:54.845658+0000) 2024-03-06T11:29:55.420 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:29:55.428 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:29:55.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:55.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:55.845921+0000) 2024-03-06T11:29:56.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:56.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:56.846169+0000) 2024-03-06T11:29:57.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:57.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:57.846397+0000) 2024-03-06T11:29:58.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:58.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:58.846680+0000) 2024-03-06T11:29:59.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:29:59.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:29:59.846870+0000) 2024-03-06T11:30:00.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:00.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:00.847078+0000) 2024-03-06T11:30:01.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:01.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:01.847269+0000) 2024-03-06T11:30:02.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:02.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:02.847498+0000) 2024-03-06T11:30:03.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:03.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:03.847729+0000) 2024-03-06T11:30:04.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:04.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:04.847887+0000) 2024-03-06T11:30:05.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:05.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:05.848131+0000) 2024-03-06T11:30:06.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:06.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:06.848360+0000) 2024-03-06T11:30:07.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:07.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:07.848615+0000) 2024-03-06T11:30:08.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:08.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:08.848845+0000) 2024-03-06T11:30:09.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:09.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:09.849070+0000) 2024-03-06T11:30:10.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:10.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:10.849263+0000) 2024-03-06T11:30:11.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:11.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:11.849508+0000) 2024-03-06T11:30:12.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:12.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:12.849832+0000) 2024-03-06T11:30:13.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:13.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:13.850067+0000) 2024-03-06T11:30:14.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:14.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:14.850305+0000) 2024-03-06T11:30:15.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:15.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:15.850577+0000) 2024-03-06T11:30:16.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:16.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:16.850844+0000) 2024-03-06T11:30:17.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:17.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:17.851055+0000) 2024-03-06T11:30:18.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:18.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:18.851297+0000) 2024-03-06T11:30:19.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:19.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:19.851634+0000) 2024-03-06T11:30:20.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:20.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:20.851884+0000) 2024-03-06T11:30:21.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:21.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:21.852141+0000) 2024-03-06T11:30:22.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:22.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:22.852376+0000) 2024-03-06T11:30:23.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:23.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:23.852692+0000) 2024-03-06T11:30:24.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:24.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:24.852915+0000) 2024-03-06T11:30:25.658 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:30:25.666 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:30:25.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:25.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:25.853127+0000) 2024-03-06T11:30:26.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:26.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:26.853326+0000) 2024-03-06T11:30:27.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:27.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:27.853582+0000) 2024-03-06T11:30:28.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:28.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:28.853864+0000) 2024-03-06T11:30:29.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:29.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:29.854087+0000) 2024-03-06T11:30:30.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:30.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:30.854275+0000) 2024-03-06T11:30:31.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:31.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:31.854467+0000) 2024-03-06T11:30:32.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:32.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:32.854669+0000) 2024-03-06T11:30:33.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:33.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:33.854853+0000) 2024-03-06T11:30:34.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:34.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:34.855102+0000) 2024-03-06T11:30:35.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:35.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:35.855322+0000) 2024-03-06T11:30:36.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:36.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:36.855576+0000) 2024-03-06T11:30:37.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:37.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:37.855776+0000) 2024-03-06T11:30:38.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:38.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:38.856003+0000) 2024-03-06T11:30:39.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:39.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:39.856239+0000) 2024-03-06T11:30:40.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:40.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:40.856452+0000) 2024-03-06T11:30:41.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:41.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:41.856696+0000) 2024-03-06T11:30:42.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:42.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:42.856883+0000) 2024-03-06T11:30:43.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:43.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:43.857063+0000) 2024-03-06T11:30:44.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:44.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:44.857275+0000) 2024-03-06T11:30:45.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:45.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:45.857483+0000) 2024-03-06T11:30:46.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:46.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:46.857698+0000) 2024-03-06T11:30:47.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:47.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:47.857869+0000) 2024-03-06T11:30:48.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:48.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:48.858093+0000) 2024-03-06T11:30:49.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:49.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:49.858287+0000) 2024-03-06T11:30:50.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:50.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:50.858488+0000) 2024-03-06T11:30:51.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:51.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:51.858690+0000) 2024-03-06T11:30:52.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:52.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:52.858892+0000) 2024-03-06T11:30:53.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:53.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:53.859074+0000) 2024-03-06T11:30:54.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:54.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:54.859267+0000) 2024-03-06T11:30:55.833 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:30:55.841 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:30:55.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:55.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:55.859564+0000) 2024-03-06T11:30:56.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:56.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:56.859787+0000) 2024-03-06T11:30:57.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:57.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:57.860047+0000) 2024-03-06T11:30:58.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:58.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:58.860306+0000) 2024-03-06T11:30:59.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:30:59.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:30:59.860592+0000) 2024-03-06T11:31:00.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:00.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:00.860827+0000) 2024-03-06T11:31:01.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:01.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:01.861035+0000) 2024-03-06T11:31:02.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:02.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:02.861211+0000) 2024-03-06T11:31:03.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:03.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:03.861395+0000) 2024-03-06T11:31:04.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:04.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:04.861651+0000) 2024-03-06T11:31:05.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:05.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:05.861827+0000) 2024-03-06T11:31:06.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:06.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:06.862031+0000) 2024-03-06T11:31:07.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:07.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:07.862225+0000) 2024-03-06T11:31:08.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:08.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:08.862430+0000) 2024-03-06T11:31:09.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:09.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:09.862706+0000) 2024-03-06T11:31:10.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:10.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:10.862884+0000) 2024-03-06T11:31:11.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:11.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:11.863070+0000) 2024-03-06T11:31:12.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:12.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:12.863256+0000) 2024-03-06T11:31:13.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:13.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:13.863434+0000) 2024-03-06T11:31:14.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:14.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:14.863670+0000) 2024-03-06T11:31:15.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:15.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:15.863910+0000) 2024-03-06T11:31:16.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:16.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:16.864103+0000) 2024-03-06T11:31:17.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:17.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:17.864329+0000) 2024-03-06T11:31:18.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:18.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:18.864520+0000) 2024-03-06T11:31:19.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:19.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:19.864762+0000) 2024-03-06T11:31:20.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:20.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:20.864946+0000) 2024-03-06T11:31:21.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:21.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:21.865173+0000) 2024-03-06T11:31:22.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:22.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:22.865392+0000) 2024-03-06T11:31:23.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:23.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:23.865617+0000) 2024-03-06T11:31:24.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:24.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:24.865801+0000) 2024-03-06T11:31:25.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:25.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:25.865991+0000) 2024-03-06T11:31:26.027 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:31:26.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:31:26.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:26.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:26.866194+0000) 2024-03-06T11:31:27.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:27.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:27.866378+0000) 2024-03-06T11:31:28.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:28.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:28.866681+0000) 2024-03-06T11:31:29.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:29.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:29.866877+0000) 2024-03-06T11:31:30.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:30.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:30.867033+0000) 2024-03-06T11:31:31.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:31.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:31.867242+0000) 2024-03-06T11:31:32.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:32.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:32.867428+0000) 2024-03-06T11:31:33.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:33.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:33.867670+0000) 2024-03-06T11:31:34.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:34.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:34.867802+0000) 2024-03-06T11:31:35.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:35.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:35.868059+0000) 2024-03-06T11:31:36.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:36.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:36.868274+0000) 2024-03-06T11:31:37.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:37.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:37.868517+0000) 2024-03-06T11:31:38.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:38.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:38.868728+0000) 2024-03-06T11:31:39.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:39.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:39.868919+0000) 2024-03-06T11:31:40.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:40.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:40.869109+0000) 2024-03-06T11:31:41.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:41.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:41.869314+0000) 2024-03-06T11:31:42.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:42.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:42.869504+0000) 2024-03-06T11:31:43.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:43.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:43.869656+0000) 2024-03-06T11:31:44.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:44.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:44.869874+0000) 2024-03-06T11:31:45.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:45.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:45.870089+0000) 2024-03-06T11:31:46.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:46.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:46.870295+0000) 2024-03-06T11:31:47.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:47.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:47.870609+0000) 2024-03-06T11:31:48.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:48.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:48.870798+0000) 2024-03-06T11:31:49.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:49.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:49.871024+0000) 2024-03-06T11:31:50.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:50.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:50.871217+0000) 2024-03-06T11:31:51.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:51.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:51.871447+0000) 2024-03-06T11:31:52.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:52.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:52.871670+0000) 2024-03-06T11:31:53.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:53.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:53.871835+0000) 2024-03-06T11:31:54.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:54.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:54.872026+0000) 2024-03-06T11:31:55.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:55.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:55.872229+0000) 2024-03-06T11:31:56.257 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:31:56.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:31:56.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:56.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:56.872439+0000) 2024-03-06T11:31:57.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:57.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:57.872662+0000) 2024-03-06T11:31:58.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:58.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:58.872857+0000) 2024-03-06T11:31:59.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:31:59.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:31:59.873057+0000) 2024-03-06T11:32:00.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:00.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:00.873243+0000) 2024-03-06T11:32:01.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:01.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:01.873455+0000) 2024-03-06T11:32:02.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:02.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:02.873731+0000) 2024-03-06T11:32:03.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:03.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:03.874363+0000) 2024-03-06T11:32:04.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:04.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:04.874605+0000) 2024-03-06T11:32:05.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:05.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:05.874815+0000) 2024-03-06T11:32:06.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:06.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:06.875009+0000) 2024-03-06T11:32:07.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:07.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:07.875235+0000) 2024-03-06T11:32:08.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:08.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:08.875479+0000) 2024-03-06T11:32:09.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:09.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:09.875721+0000) 2024-03-06T11:32:10.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:10.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:10.875931+0000) 2024-03-06T11:32:11.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:11.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:11.876167+0000) 2024-03-06T11:32:12.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:12.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:12.876394+0000) 2024-03-06T11:32:13.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:13.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:13.876657+0000) 2024-03-06T11:32:14.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:14.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:14.876850+0000) 2024-03-06T11:32:15.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:15.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:15.877087+0000) 2024-03-06T11:32:16.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:16.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:16.877277+0000) 2024-03-06T11:32:17.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:17.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:17.877582+0000) 2024-03-06T11:32:18.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:18.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:18.877802+0000) 2024-03-06T11:32:19.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:19.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:19.878047+0000) 2024-03-06T11:32:20.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:20.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:20.878260+0000) 2024-03-06T11:32:21.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:21.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:21.878578+0000) 2024-03-06T11:32:22.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:22.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:22.878793+0000) 2024-03-06T11:32:23.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:23.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:23.879012+0000) 2024-03-06T11:32:24.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:24.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:24.879207+0000) 2024-03-06T11:32:25.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:25.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:25.879443+0000) 2024-03-06T11:32:26.479 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:32:26.489 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:32:26.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:26.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:26.879684+0000) 2024-03-06T11:32:27.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:27.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:27.879873+0000) 2024-03-06T11:32:28.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:28.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:28.880085+0000) 2024-03-06T11:32:29.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:29.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:29.880310+0000) 2024-03-06T11:32:30.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:30.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:30.880580+0000) 2024-03-06T11:32:31.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:31.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:31.880778+0000) 2024-03-06T11:32:32.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:32.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:32.881013+0000) 2024-03-06T11:32:33.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:33.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:33.881198+0000) 2024-03-06T11:32:34.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:34.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:34.881428+0000) 2024-03-06T11:32:35.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:35.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:35.881715+0000) 2024-03-06T11:32:36.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:36.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:36.881932+0000) 2024-03-06T11:32:37.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:37.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:37.882151+0000) 2024-03-06T11:32:38.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:38.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:38.882376+0000) 2024-03-06T11:32:39.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:39.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:39.882642+0000) 2024-03-06T11:32:40.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:40.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:40.882844+0000) 2024-03-06T11:32:41.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:41.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:41.883080+0000) 2024-03-06T11:32:42.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:42.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:42.883261+0000) 2024-03-06T11:32:43.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:43.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:43.883485+0000) 2024-03-06T11:32:44.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:44.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:44.883673+0000) 2024-03-06T11:32:45.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:45.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:45.883895+0000) 2024-03-06T11:32:46.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:46.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:46.884098+0000) 2024-03-06T11:32:47.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:47.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:47.884281+0000) 2024-03-06T11:32:48.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:48.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:48.884500+0000) 2024-03-06T11:32:49.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:49.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:49.884762+0000) 2024-03-06T11:32:50.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:50.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:50.884967+0000) 2024-03-06T11:32:51.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:51.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:51.885157+0000) 2024-03-06T11:32:52.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:52.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:52.885398+0000) 2024-03-06T11:32:53.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:53.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:53.885644+0000) 2024-03-06T11:32:54.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:54.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:54.885881+0000) 2024-03-06T11:32:55.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:55.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:55.886125+0000) 2024-03-06T11:32:56.692 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:32:56.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:32:56.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:56.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:56.886342+0000) 2024-03-06T11:32:57.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:57.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:57.886590+0000) 2024-03-06T11:32:58.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:58.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:58.886790+0000) 2024-03-06T11:32:59.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:32:59.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:32:59.886975+0000) 2024-03-06T11:33:00.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:00.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:00.887203+0000) 2024-03-06T11:33:01.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:01.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:01.887426+0000) 2024-03-06T11:33:02.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:02.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:02.887664+0000) 2024-03-06T11:33:03.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:03.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:03.887861+0000) 2024-03-06T11:33:04.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:04.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:04.888087+0000) 2024-03-06T11:33:05.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:05.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:05.888330+0000) 2024-03-06T11:33:06.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:06.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:06.888511+0000) 2024-03-06T11:33:07.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:07.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:07.888800+0000) 2024-03-06T11:33:08.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:08.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:08.889011+0000) 2024-03-06T11:33:09.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:09.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:09.889238+0000) 2024-03-06T11:33:10.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:10.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:10.889466+0000) 2024-03-06T11:33:11.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:11.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:11.889665+0000) 2024-03-06T11:33:12.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:12.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:12.889892+0000) 2024-03-06T11:33:13.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:13.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:13.890098+0000) 2024-03-06T11:33:14.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:14.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:14.890338+0000) 2024-03-06T11:33:15.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:15.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:15.890629+0000) 2024-03-06T11:33:16.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:16.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:16.890879+0000) 2024-03-06T11:33:17.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:17.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:17.891084+0000) 2024-03-06T11:33:18.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:18.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:18.891321+0000) 2024-03-06T11:33:19.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:19.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:19.891601+0000) 2024-03-06T11:33:20.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:20.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:20.891808+0000) 2024-03-06T11:33:21.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:21.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:21.892020+0000) 2024-03-06T11:33:22.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:22.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:22.892239+0000) 2024-03-06T11:33:23.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:23.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:23.892483+0000) 2024-03-06T11:33:24.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:24.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:24.892662+0000) 2024-03-06T11:33:25.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:25.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:25.892872+0000) 2024-03-06T11:33:26.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:26.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:26.893113+0000) 2024-03-06T11:33:26.914 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:33:26.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:33:27.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:27.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:27.893353+0000) 2024-03-06T11:33:28.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:28.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:28.893656+0000) 2024-03-06T11:33:29.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:29.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:29.893866+0000) 2024-03-06T11:33:30.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:30.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:30.894110+0000) 2024-03-06T11:33:31.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:31.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:31.894352+0000) 2024-03-06T11:33:32.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:32.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:32.894644+0000) 2024-03-06T11:33:33.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:33.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:33.894852+0000) 2024-03-06T11:33:34.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:34.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:34.895067+0000) 2024-03-06T11:33:35.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:35.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:35.895315+0000) 2024-03-06T11:33:36.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:36.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:36.895601+0000) 2024-03-06T11:33:37.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:37.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:37.895849+0000) 2024-03-06T11:33:38.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:38.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:38.896053+0000) 2024-03-06T11:33:39.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:39.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:39.896237+0000) 2024-03-06T11:33:40.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:40.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:40.896481+0000) 2024-03-06T11:33:41.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:41.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:41.896716+0000) 2024-03-06T11:33:42.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:42.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:42.896932+0000) 2024-03-06T11:33:43.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:43.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:43.897135+0000) 2024-03-06T11:33:44.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:44.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:44.897326+0000) 2024-03-06T11:33:45.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:45.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:45.897613+0000) 2024-03-06T11:33:46.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:46.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:46.897863+0000) 2024-03-06T11:33:47.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:47.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:47.898108+0000) 2024-03-06T11:33:48.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:48.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:48.898302+0000) 2024-03-06T11:33:49.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:49.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:49.898527+0000) 2024-03-06T11:33:50.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:50.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:50.898787+0000) 2024-03-06T11:33:51.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:51.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:51.899011+0000) 2024-03-06T11:33:52.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:52.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:52.899239+0000) 2024-03-06T11:33:53.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:53.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:53.899466+0000) 2024-03-06T11:33:54.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:54.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:54.899675+0000) 2024-03-06T11:33:55.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:55.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:55.899906+0000) 2024-03-06T11:33:56.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:56.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:56.900133+0000) 2024-03-06T11:33:57.051 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:33:57.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:33:57.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:57.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:57.900351+0000) 2024-03-06T11:33:58.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:58.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:58.900608+0000) 2024-03-06T11:33:59.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:33:59.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:33:59.900833+0000) 2024-03-06T11:34:00.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:00.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:00.901076+0000) 2024-03-06T11:34:01.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:01.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:01.901317+0000) 2024-03-06T11:34:02.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:02.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:02.901588+0000) 2024-03-06T11:34:03.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:03.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:03.901807+0000) 2024-03-06T11:34:04.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:04.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:04.902055+0000) 2024-03-06T11:34:05.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:05.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:05.902289+0000) 2024-03-06T11:34:06.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:06.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:06.902519+0000) 2024-03-06T11:34:07.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:07.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:07.902781+0000) 2024-03-06T11:34:08.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:08.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:08.903009+0000) 2024-03-06T11:34:09.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:09.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:09.903225+0000) 2024-03-06T11:34:10.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:10.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:10.903414+0000) 2024-03-06T11:34:11.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:11.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:11.903671+0000) 2024-03-06T11:34:12.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:12.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:12.903917+0000) 2024-03-06T11:34:13.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:13.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:13.904105+0000) 2024-03-06T11:34:14.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:14.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:14.904332+0000) 2024-03-06T11:34:15.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:15.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:15.904621+0000) 2024-03-06T11:34:16.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:16.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:16.904901+0000) 2024-03-06T11:34:17.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:17.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:17.905132+0000) 2024-03-06T11:34:18.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:18.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:18.905313+0000) 2024-03-06T11:34:19.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:19.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:19.905595+0000) 2024-03-06T11:34:20.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:20.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:20.905811+0000) 2024-03-06T11:34:21.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:21.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:21.906038+0000) 2024-03-06T11:34:22.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:22.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:22.906274+0000) 2024-03-06T11:34:23.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:23.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:23.906469+0000) 2024-03-06T11:34:24.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:24.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:24.906654+0000) 2024-03-06T11:34:25.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:25.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:25.906887+0000) 2024-03-06T11:34:26.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:26.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:26.907097+0000) 2024-03-06T11:34:27.190 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:34:27.199 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:34:27.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:27.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:27.907321+0000) 2024-03-06T11:34:28.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:28.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:28.907589+0000) 2024-03-06T11:34:29.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:29.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:29.907766+0000) 2024-03-06T11:34:30.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:30.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:30.908005+0000) 2024-03-06T11:34:31.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:31.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:31.908198+0000) 2024-03-06T11:34:32.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:32.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:32.908445+0000) 2024-03-06T11:34:33.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:33.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:33.908685+0000) 2024-03-06T11:34:34.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:34.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:34.908922+0000) 2024-03-06T11:34:35.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:35.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:35.909173+0000) 2024-03-06T11:34:36.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:36.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:36.909404+0000) 2024-03-06T11:34:37.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:37.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:37.909640+0000) 2024-03-06T11:34:38.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:38.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:38.909879+0000) 2024-03-06T11:34:39.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:39.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:39.910039+0000) 2024-03-06T11:34:40.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:40.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:40.910265+0000) 2024-03-06T11:34:41.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:41.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:41.910483+0000) 2024-03-06T11:34:42.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:42.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:42.910713+0000) 2024-03-06T11:34:43.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:43.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:43.910911+0000) 2024-03-06T11:34:44.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:44.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:44.911101+0000) 2024-03-06T11:34:45.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:45.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:45.911331+0000) 2024-03-06T11:34:46.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:46.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:46.911524+0000) 2024-03-06T11:34:47.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:47.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:47.911767+0000) 2024-03-06T11:34:48.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:48.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:48.911969+0000) 2024-03-06T11:34:49.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:49.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:49.912190+0000) 2024-03-06T11:34:50.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:50.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:50.912387+0000) 2024-03-06T11:34:51.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:51.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:51.912684+0000) 2024-03-06T11:34:52.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:52.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:52.912857+0000) 2024-03-06T11:34:53.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:53.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:53.913102+0000) 2024-03-06T11:34:54.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:54.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:54.913340+0000) 2024-03-06T11:34:55.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:55.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:55.913638+0000) 2024-03-06T11:34:56.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:56.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:56.913881+0000) 2024-03-06T11:34:57.471 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:34:57.479 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:34:57.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:57.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:57.914129+0000) 2024-03-06T11:34:58.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:58.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:58.914380+0000) 2024-03-06T11:34:59.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:34:59.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:34:59.914646+0000) 2024-03-06T11:35:00.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:00.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:00.914844+0000) 2024-03-06T11:35:01.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:01.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:01.915066+0000) 2024-03-06T11:35:02.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:02.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:02.915304+0000) 2024-03-06T11:35:03.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:03.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:03.915574+0000) 2024-03-06T11:35:04.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:04.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:04.915779+0000) 2024-03-06T11:35:05.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:05.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:05.915965+0000) 2024-03-06T11:35:06.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:06.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:06.916224+0000) 2024-03-06T11:35:07.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:07.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:07.916483+0000) 2024-03-06T11:35:08.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:08.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:08.916687+0000) 2024-03-06T11:35:09.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:09.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:09.916908+0000) 2024-03-06T11:35:10.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:10.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:10.917123+0000) 2024-03-06T11:35:11.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:11.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:11.917341+0000) 2024-03-06T11:35:12.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:12.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:12.917616+0000) 2024-03-06T11:35:13.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:13.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:13.917835+0000) 2024-03-06T11:35:14.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:14.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:14.918024+0000) 2024-03-06T11:35:15.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:15.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:15.918212+0000) 2024-03-06T11:35:16.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:16.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:16.918421+0000) 2024-03-06T11:35:17.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:17.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:17.918666+0000) 2024-03-06T11:35:18.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:18.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:18.918895+0000) 2024-03-06T11:35:19.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:19.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:19.919119+0000) 2024-03-06T11:35:20.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:20.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:20.919357+0000) 2024-03-06T11:35:21.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:21.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:21.919610+0000) 2024-03-06T11:35:22.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:22.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:22.919835+0000) 2024-03-06T11:35:23.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:23.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:23.920015+0000) 2024-03-06T11:35:24.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:24.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:24.920246+0000) 2024-03-06T11:35:25.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:25.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:25.920481+0000) 2024-03-06T11:35:26.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:26.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:26.920676+0000) 2024-03-06T11:35:27.667 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:35:27.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:35:27.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:27.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:27.920871+0000) 2024-03-06T11:35:28.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:28.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:28.921076+0000) 2024-03-06T11:35:29.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:29.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:29.921294+0000) 2024-03-06T11:35:30.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:30.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:30.921515+0000) 2024-03-06T11:35:31.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:31.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:31.921761+0000) 2024-03-06T11:35:32.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:32.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:32.921956+0000) 2024-03-06T11:35:33.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:33.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:33.922177+0000) 2024-03-06T11:35:34.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:34.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:34.922374+0000) 2024-03-06T11:35:35.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:35.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:35.922660+0000) 2024-03-06T11:35:36.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:36.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:36.922867+0000) 2024-03-06T11:35:37.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:37.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:37.923080+0000) 2024-03-06T11:35:38.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:38.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:38.923319+0000) 2024-03-06T11:35:39.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:39.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:39.923595+0000) 2024-03-06T11:35:40.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:40.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:40.923791+0000) 2024-03-06T11:35:41.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:41.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:41.924014+0000) 2024-03-06T11:35:42.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:42.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:42.924196+0000) 2024-03-06T11:35:43.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:43.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:43.924424+0000) 2024-03-06T11:35:44.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:44.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:44.924672+0000) 2024-03-06T11:35:45.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:45.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:45.924874+0000) 2024-03-06T11:35:46.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:46.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:46.925107+0000) 2024-03-06T11:35:47.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:47.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:47.925307+0000) 2024-03-06T11:35:48.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:48.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:48.925575+0000) 2024-03-06T11:35:49.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:49.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:49.925759+0000) 2024-03-06T11:35:50.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:50.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:50.925935+0000) 2024-03-06T11:35:51.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:51.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:51.926166+0000) 2024-03-06T11:35:52.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:52.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:52.926420+0000) 2024-03-06T11:35:53.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:53.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:53.926663+0000) 2024-03-06T11:35:54.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:54.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:54.926901+0000) 2024-03-06T11:35:55.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:55.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:55.927114+0000) 2024-03-06T11:35:56.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:56.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:56.927294+0000) 2024-03-06T11:35:57.855 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:35:57.865 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:35:57.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:57.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:57.927485+0000) 2024-03-06T11:35:58.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:58.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:58.927692+0000) 2024-03-06T11:35:59.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:35:59.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:35:59.927884+0000) 2024-03-06T11:36:00.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:00.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:00.928113+0000) 2024-03-06T11:36:01.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:01.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:01.928340+0000) 2024-03-06T11:36:02.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:02.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:02.928624+0000) 2024-03-06T11:36:03.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:03.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:03.928825+0000) 2024-03-06T11:36:04.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:04.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:04.929031+0000) 2024-03-06T11:36:05.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:05.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:05.929258+0000) 2024-03-06T11:36:06.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:06.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:06.929488+0000) 2024-03-06T11:36:07.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:07.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:07.929664+0000) 2024-03-06T11:36:08.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:08.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:08.929866+0000) 2024-03-06T11:36:09.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:09.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:09.930087+0000) 2024-03-06T11:36:10.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:10.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:10.930313+0000) 2024-03-06T11:36:11.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:11.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:11.930565+0000) 2024-03-06T11:36:12.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:12.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:12.930706+0000) 2024-03-06T11:36:13.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:13.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:13.930880+0000) 2024-03-06T11:36:14.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:14.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:14.931071+0000) 2024-03-06T11:36:15.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:15.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:15.931264+0000) 2024-03-06T11:36:16.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:16.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:16.931470+0000) 2024-03-06T11:36:17.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:17.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:17.931696+0000) 2024-03-06T11:36:18.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:18.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:18.931902+0000) 2024-03-06T11:36:19.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:19.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:19.932135+0000) 2024-03-06T11:36:20.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:20.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:20.932365+0000) 2024-03-06T11:36:21.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:21.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:21.932661+0000) 2024-03-06T11:36:22.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:22.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:22.932868+0000) 2024-03-06T11:36:23.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:23.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:23.933094+0000) 2024-03-06T11:36:24.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:24.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:24.933322+0000) 2024-03-06T11:36:25.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:25.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:25.933508+0000) 2024-03-06T11:36:26.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:26.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:26.933779+0000) 2024-03-06T11:36:27.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:27.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:27.934021+0000) 2024-03-06T11:36:28.060 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:36:28.068 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:36:28.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:28.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:28.934239+0000) 2024-03-06T11:36:29.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:29.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:29.934476+0000) 2024-03-06T11:36:30.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:30.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:30.934707+0000) 2024-03-06T11:36:31.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:31.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:31.934932+0000) 2024-03-06T11:36:32.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:32.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:32.935172+0000) 2024-03-06T11:36:33.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:33.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:33.935403+0000) 2024-03-06T11:36:34.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:34.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:34.935675+0000) 2024-03-06T11:36:35.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:35.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:35.935913+0000) 2024-03-06T11:36:36.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:36.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:36.936106+0000) 2024-03-06T11:36:37.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:37.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:37.936335+0000) 2024-03-06T11:36:38.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:38.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:38.936599+0000) 2024-03-06T11:36:39.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:39.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:39.936821+0000) 2024-03-06T11:36:40.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:40.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:40.937052+0000) 2024-03-06T11:36:41.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:41.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:41.937279+0000) 2024-03-06T11:36:42.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:42.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:42.937494+0000) 2024-03-06T11:36:43.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:43.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:43.937697+0000) 2024-03-06T11:36:44.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:44.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:44.937894+0000) 2024-03-06T11:36:45.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:45.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:45.938170+0000) 2024-03-06T11:36:46.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:46.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:46.938414+0000) 2024-03-06T11:36:47.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:47.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:47.938706+0000) 2024-03-06T11:36:48.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:48.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:48.938955+0000) 2024-03-06T11:36:49.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:49.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:49.939166+0000) 2024-03-06T11:36:50.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:50.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:50.939364+0000) 2024-03-06T11:36:51.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:51.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:51.939628+0000) 2024-03-06T11:36:52.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:52.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:52.939815+0000) 2024-03-06T11:36:53.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:53.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:53.940009+0000) 2024-03-06T11:36:54.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:54.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:54.940221+0000) 2024-03-06T11:36:55.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:55.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:55.940467+0000) 2024-03-06T11:36:56.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:56.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:56.940694+0000) 2024-03-06T11:36:57.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:57.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:57.940881+0000) 2024-03-06T11:36:58.285 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:36:58.293 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:36:58.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:58.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:58.941112+0000) 2024-03-06T11:36:59.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:36:59.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:36:59.941390+0000) 2024-03-06T11:37:00.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:00.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:00.941699+0000) 2024-03-06T11:37:01.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:01.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:01.941982+0000) 2024-03-06T11:37:02.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:02.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:02.942275+0000) 2024-03-06T11:37:03.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:03.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:03.942501+0000) 2024-03-06T11:37:04.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:04.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:04.942782+0000) 2024-03-06T11:37:05.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:05.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:05.943055+0000) 2024-03-06T11:37:06.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:06.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:06.943307+0000) 2024-03-06T11:37:07.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:07.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:07.943523+0000) 2024-03-06T11:37:08.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:08.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:08.943749+0000) 2024-03-06T11:37:09.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:09.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:09.943939+0000) 2024-03-06T11:37:10.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:10.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:10.944144+0000) 2024-03-06T11:37:11.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:11.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:11.944379+0000) 2024-03-06T11:37:12.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:12.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:12.944643+0000) 2024-03-06T11:37:13.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:13.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:13.944840+0000) 2024-03-06T11:37:14.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:14.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:14.945022+0000) 2024-03-06T11:37:15.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:15.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:15.945259+0000) 2024-03-06T11:37:16.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:16.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:16.945451+0000) 2024-03-06T11:37:17.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:17.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:17.945726+0000) 2024-03-06T11:37:18.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:18.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:18.945977+0000) 2024-03-06T11:37:19.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:19.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:19.946203+0000) 2024-03-06T11:37:20.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:20.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:20.946397+0000) 2024-03-06T11:37:21.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:21.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:21.946673+0000) 2024-03-06T11:37:22.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:22.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:22.946931+0000) 2024-03-06T11:37:23.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:23.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:23.947179+0000) 2024-03-06T11:37:24.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:24.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:24.947414+0000) 2024-03-06T11:37:25.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:25.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:25.947653+0000) 2024-03-06T11:37:26.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:26.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:26.947839+0000) 2024-03-06T11:37:27.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:27.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:27.948076+0000) 2024-03-06T11:37:28.503 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:37:28.511 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:37:28.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:28.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:28.948269+0000) 2024-03-06T11:37:29.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:29.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:29.948501+0000) 2024-03-06T11:37:30.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:30.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:30.948683+0000) 2024-03-06T11:37:31.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:31.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:31.948901+0000) 2024-03-06T11:37:32.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:32.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:32.949207+0000) 2024-03-06T11:37:33.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:33.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:33.949523+0000) 2024-03-06T11:37:34.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:34.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:34.949786+0000) 2024-03-06T11:37:35.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:35.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:35.950038+0000) 2024-03-06T11:37:36.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:36.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:36.950307+0000) 2024-03-06T11:37:37.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:37.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:37.950562+0000) 2024-03-06T11:37:38.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:38.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:38.950752+0000) 2024-03-06T11:37:39.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:39.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:39.950947+0000) 2024-03-06T11:37:40.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:40.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:40.951224+0000) 2024-03-06T11:37:41.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:41.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:41.951500+0000) 2024-03-06T11:37:42.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:42.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:42.951737+0000) 2024-03-06T11:37:43.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:43.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:43.951963+0000) 2024-03-06T11:37:44.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:44.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:44.952192+0000) 2024-03-06T11:37:45.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:45.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:45.952434+0000) 2024-03-06T11:37:46.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:46.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:46.952682+0000) 2024-03-06T11:37:47.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:47.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:47.952921+0000) 2024-03-06T11:37:48.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:48.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:48.953148+0000) 2024-03-06T11:37:49.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:49.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:49.953374+0000) 2024-03-06T11:37:50.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:50.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:50.953607+0000) 2024-03-06T11:37:51.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:51.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:51.953815+0000) 2024-03-06T11:37:52.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:52.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:52.954079+0000) 2024-03-06T11:37:53.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:53.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:53.954334+0000) 2024-03-06T11:37:54.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:54.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:54.954618+0000) 2024-03-06T11:37:55.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:55.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:55.954877+0000) 2024-03-06T11:37:56.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:56.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:56.955107+0000) 2024-03-06T11:37:57.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:57.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:57.955329+0000) 2024-03-06T11:37:58.724 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:37:58.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:37:58.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:58.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:58.955567+0000) 2024-03-06T11:37:59.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:37:59.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:37:59.955807+0000) 2024-03-06T11:38:00.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:00.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:00.956052+0000) 2024-03-06T11:38:01.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:01.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:01.956299+0000) 2024-03-06T11:38:02.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:02.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:02.956576+0000) 2024-03-06T11:38:03.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:03.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:03.956833+0000) 2024-03-06T11:38:04.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:04.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:04.957075+0000) 2024-03-06T11:38:05.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:05.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:05.957266+0000) 2024-03-06T11:38:06.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:06.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:06.957511+0000) 2024-03-06T11:38:07.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:07.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:07.957796+0000) 2024-03-06T11:38:08.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:08.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:08.957998+0000) 2024-03-06T11:38:09.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:09.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:09.958201+0000) 2024-03-06T11:38:10.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:10.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:10.958439+0000) 2024-03-06T11:38:11.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:11.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:11.958684+0000) 2024-03-06T11:38:12.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:12.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:12.958887+0000) 2024-03-06T11:38:13.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:13.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:13.959084+0000) 2024-03-06T11:38:14.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:14.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:14.959330+0000) 2024-03-06T11:38:15.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:15.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:15.959634+0000) 2024-03-06T11:38:16.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:16.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:16.959848+0000) 2024-03-06T11:38:17.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:17.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:17.960029+0000) 2024-03-06T11:38:18.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:18.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:18.960255+0000) 2024-03-06T11:38:19.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:19.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:19.960518+0000) 2024-03-06T11:38:20.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:20.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:20.960799+0000) 2024-03-06T11:38:21.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:21.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:21.961039+0000) 2024-03-06T11:38:22.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:22.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:22.961233+0000) 2024-03-06T11:38:23.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:23.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:23.961457+0000) 2024-03-06T11:38:24.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:24.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:24.961698+0000) 2024-03-06T11:38:25.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:25.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:25.961932+0000) 2024-03-06T11:38:26.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:26.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:26.962153+0000) 2024-03-06T11:38:27.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:27.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:27.962442+0000) 2024-03-06T11:38:28.937 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:38:28.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:38:28.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:28.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:28.962633+0000) 2024-03-06T11:38:29.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:29.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:29.962865+0000) 2024-03-06T11:38:30.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:30.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:30.963046+0000) 2024-03-06T11:38:31.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:31.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:31.963272+0000) 2024-03-06T11:38:32.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:32.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:32.963508+0000) 2024-03-06T11:38:33.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:33.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:33.963801+0000) 2024-03-06T11:38:34.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:34.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:34.964041+0000) 2024-03-06T11:38:35.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:35.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:35.964245+0000) 2024-03-06T11:38:36.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:36.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:36.964473+0000) 2024-03-06T11:38:37.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:37.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:37.964695+0000) 2024-03-06T11:38:38.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:38.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:38.964921+0000) 2024-03-06T11:38:39.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:39.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:39.965147+0000) 2024-03-06T11:38:40.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:40.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:40.965339+0000) 2024-03-06T11:38:41.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:41.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:41.965525+0000) 2024-03-06T11:38:42.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:42.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:42.965780+0000) 2024-03-06T11:38:43.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:43.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:43.966035+0000) 2024-03-06T11:38:44.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:44.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:44.966264+0000) 2024-03-06T11:38:45.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:45.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:45.966523+0000) 2024-03-06T11:38:46.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:46.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:46.966797+0000) 2024-03-06T11:38:47.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:47.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:47.967061+0000) 2024-03-06T11:38:48.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:48.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:48.967280+0000) 2024-03-06T11:38:49.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:49.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:49.967481+0000) 2024-03-06T11:38:50.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:50.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:50.967705+0000) 2024-03-06T11:38:51.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:51.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:51.967934+0000) 2024-03-06T11:38:52.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:52.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:52.968146+0000) 2024-03-06T11:38:53.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:53.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:53.968383+0000) 2024-03-06T11:38:54.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:54.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:54.968648+0000) 2024-03-06T11:38:55.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:55.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:55.968876+0000) 2024-03-06T11:38:56.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:56.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:56.969107+0000) 2024-03-06T11:38:57.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:57.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:57.969372+0000) 2024-03-06T11:38:58.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:58.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:58.969679+0000) 2024-03-06T11:38:59.141 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:38:59.151 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:38:59.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:38:59.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:38:59.969886+0000) 2024-03-06T11:39:00.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:00.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:00.970114+0000) 2024-03-06T11:39:01.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:01.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:01.970342+0000) 2024-03-06T11:39:02.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:02.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:02.970678+0000) 2024-03-06T11:39:03.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:03.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:03.970928+0000) 2024-03-06T11:39:04.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:04.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:04.971124+0000) 2024-03-06T11:39:05.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:05.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:05.971314+0000) 2024-03-06T11:39:06.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:06.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:06.971575+0000) 2024-03-06T11:39:07.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:07.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:07.971805+0000) 2024-03-06T11:39:08.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:08.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:08.971984+0000) 2024-03-06T11:39:09.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:09.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:09.972213+0000) 2024-03-06T11:39:10.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:10.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:10.972442+0000) 2024-03-06T11:39:11.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:11.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:11.972632+0000) 2024-03-06T11:39:12.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:12.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:12.972854+0000) 2024-03-06T11:39:13.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:13.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:13.973080+0000) 2024-03-06T11:39:14.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:14.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:14.973363+0000) 2024-03-06T11:39:15.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:15.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:15.973708+0000) 2024-03-06T11:39:16.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:16.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:16.973939+0000) 2024-03-06T11:39:17.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:17.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:17.974191+0000) 2024-03-06T11:39:18.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:18.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:18.974468+0000) 2024-03-06T11:39:19.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:19.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:19.974752+0000) 2024-03-06T11:39:20.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:20.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:20.974988+0000) 2024-03-06T11:39:21.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:21.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:21.975190+0000) 2024-03-06T11:39:22.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:22.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:22.975424+0000) 2024-03-06T11:39:23.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:23.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:23.975661+0000) 2024-03-06T11:39:24.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:24.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:24.975866+0000) 2024-03-06T11:39:25.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:25.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:25.976146+0000) 2024-03-06T11:39:26.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:26.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:26.976424+0000) 2024-03-06T11:39:27.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:27.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:27.976689+0000) 2024-03-06T11:39:28.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:28.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:28.976934+0000) 2024-03-06T11:39:29.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:39:29.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:39:29.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:29.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:29.977165+0000) 2024-03-06T11:39:30.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:30.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:30.977392+0000) 2024-03-06T11:39:31.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:31.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:31.977630+0000) 2024-03-06T11:39:32.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:32.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:32.977827+0000) 2024-03-06T11:39:33.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:33.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:33.978015+0000) 2024-03-06T11:39:34.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:34.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:34.978219+0000) 2024-03-06T11:39:35.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:35.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:35.978465+0000) 2024-03-06T11:39:36.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:36.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:36.978703+0000) 2024-03-06T11:39:37.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:37.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:37.978922+0000) 2024-03-06T11:39:38.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:38.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:38.979154+0000) 2024-03-06T11:39:39.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:39.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:39.979384+0000) 2024-03-06T11:39:40.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:40.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:40.979652+0000) 2024-03-06T11:39:41.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:41.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:41.979876+0000) 2024-03-06T11:39:42.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:42.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:42.980096+0000) 2024-03-06T11:39:43.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:43.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:43.980323+0000) 2024-03-06T11:39:44.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:44.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:44.980584+0000) 2024-03-06T11:39:45.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:45.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:45.980790+0000) 2024-03-06T11:39:46.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:46.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:46.981018+0000) 2024-03-06T11:39:47.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:47.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:47.981217+0000) 2024-03-06T11:39:48.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:48.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:48.981457+0000) 2024-03-06T11:39:49.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:49.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:49.981690+0000) 2024-03-06T11:39:50.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:50.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:50.981881+0000) 2024-03-06T11:39:51.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:51.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:51.982137+0000) 2024-03-06T11:39:52.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:52.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:52.982357+0000) 2024-03-06T11:39:53.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:53.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:53.982623+0000) 2024-03-06T11:39:54.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:54.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:54.982843+0000) 2024-03-06T11:39:55.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:55.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:55.983141+0000) 2024-03-06T11:39:56.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:56.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:56.983424+0000) 2024-03-06T11:39:57.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:57.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:57.983702+0000) 2024-03-06T11:39:58.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:58.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:58.983931+0000) 2024-03-06T11:39:59.586 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:39:59.594 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:39:59.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:39:59.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:39:59.984174+0000) 2024-03-06T11:40:00.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:00.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:00.984396+0000) 2024-03-06T11:40:01.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:01.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:01.984638+0000) 2024-03-06T11:40:02.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:02.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:02.984876+0000) 2024-03-06T11:40:03.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:03.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:03.985107+0000) 2024-03-06T11:40:04.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:04.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:04.985294+0000) 2024-03-06T11:40:05.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:05.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:05.985526+0000) 2024-03-06T11:40:06.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:06.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:06.985757+0000) 2024-03-06T11:40:07.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:07.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:07.985998+0000) 2024-03-06T11:40:08.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:08.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:08.986289+0000) 2024-03-06T11:40:09.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:09.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:09.986470+0000) 2024-03-06T11:40:10.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:10.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:10.986653+0000) 2024-03-06T11:40:11.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:11.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:11.986874+0000) 2024-03-06T11:40:12.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:12.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:12.987099+0000) 2024-03-06T11:40:13.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:13.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:13.987331+0000) 2024-03-06T11:40:14.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:14.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:14.987586+0000) 2024-03-06T11:40:15.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:15.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:15.987826+0000) 2024-03-06T11:40:16.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:16.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:16.988054+0000) 2024-03-06T11:40:17.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:17.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:17.988248+0000) 2024-03-06T11:40:18.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:18.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:18.988459+0000) 2024-03-06T11:40:19.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:19.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:19.988686+0000) 2024-03-06T11:40:20.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:20.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:20.988925+0000) 2024-03-06T11:40:21.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:21.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:21.989148+0000) 2024-03-06T11:40:22.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:22.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:22.989365+0000) 2024-03-06T11:40:23.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:23.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:23.989626+0000) 2024-03-06T11:40:24.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:24.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:24.989855+0000) 2024-03-06T11:40:25.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:25.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:25.990053+0000) 2024-03-06T11:40:26.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:26.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:26.990286+0000) 2024-03-06T11:40:27.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:27.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:27.990517+0000) 2024-03-06T11:40:28.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:28.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:28.990792+0000) 2024-03-06T11:40:29.789 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:40:29.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:40:29.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:29.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:29.990973+0000) 2024-03-06T11:40:30.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:30.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:30.991194+0000) 2024-03-06T11:40:31.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:31.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:31.991420+0000) 2024-03-06T11:40:32.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:32.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:32.991644+0000) 2024-03-06T11:40:33.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:33.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:33.991882+0000) 2024-03-06T11:40:34.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:34.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:34.992107+0000) 2024-03-06T11:40:35.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:35.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:35.992348+0000) 2024-03-06T11:40:36.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:36.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:36.992566+0000) 2024-03-06T11:40:37.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:37.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:37.992832+0000) 2024-03-06T11:40:38.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:38.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:38.993038+0000) 2024-03-06T11:40:39.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:39.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:39.993260+0000) 2024-03-06T11:40:40.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:40.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:40.993446+0000) 2024-03-06T11:40:41.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:41.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:41.993674+0000) 2024-03-06T11:40:42.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:42.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:42.993879+0000) 2024-03-06T11:40:43.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:43.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:43.994044+0000) 2024-03-06T11:40:44.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:44.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:44.994295+0000) 2024-03-06T11:40:45.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:45.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:45.994498+0000) 2024-03-06T11:40:46.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:46.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:46.994706+0000) 2024-03-06T11:40:47.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:47.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:47.994924+0000) 2024-03-06T11:40:48.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:48.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:48.995150+0000) 2024-03-06T11:40:49.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:49.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:49.995372+0000) 2024-03-06T11:40:50.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:50.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:50.995621+0000) 2024-03-06T11:40:51.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:51.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:51.995878+0000) 2024-03-06T11:40:52.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:52.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:52.996103+0000) 2024-03-06T11:40:53.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:53.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:53.996314+0000) 2024-03-06T11:40:54.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:54.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:54.996510+0000) 2024-03-06T11:40:55.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:55.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:55.996753+0000) 2024-03-06T11:40:56.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:56.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:56.997042+0000) 2024-03-06T11:40:57.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:57.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:57.997303+0000) 2024-03-06T11:40:58.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:58.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:58.997607+0000) 2024-03-06T11:40:59.989 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:40:59.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:40:59.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:40:59.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:40:59.997810+0000) 2024-03-06T11:41:00.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:00.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:00.998021+0000) 2024-03-06T11:41:01.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:01.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:01.998301+0000) 2024-03-06T11:41:02.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:02.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:02.998620+0000) 2024-03-06T11:41:03.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:03.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:03.998826+0000) 2024-03-06T11:41:04.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:04.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:04.999051+0000) 2024-03-06T11:41:05.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:05.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:05.999251+0000) 2024-03-06T11:41:06.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:06.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:06.999438+0000) 2024-03-06T11:41:07.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:07.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:07.999688+0000) 2024-03-06T11:41:08.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:08.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:08.999885+0000) 2024-03-06T11:41:09.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:09.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:10.000112+0000) 2024-03-06T11:41:11.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:10.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:11.000337+0000) 2024-03-06T11:41:12.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:12.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:12.000602+0000) 2024-03-06T11:41:13.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:13.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:13.000882+0000) 2024-03-06T11:41:14.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:14.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:14.001155+0000) 2024-03-06T11:41:15.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:15.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:15.001432+0000) 2024-03-06T11:41:16.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:16.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:16.001722+0000) 2024-03-06T11:41:17.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:17.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:17.002008+0000) 2024-03-06T11:41:18.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:18.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:18.002285+0000) 2024-03-06T11:41:19.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:19.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:19.002609+0000) 2024-03-06T11:41:20.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:20.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:20.002891+0000) 2024-03-06T11:41:21.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:21.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:21.003168+0000) 2024-03-06T11:41:22.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:22.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:22.003445+0000) 2024-03-06T11:41:23.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:23.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:23.003722+0000) 2024-03-06T11:41:24.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:24.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:24.003926+0000) 2024-03-06T11:41:25.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:25.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:25.004124+0000) 2024-03-06T11:41:26.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:26.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:26.004374+0000) 2024-03-06T11:41:27.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:27.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:27.004618+0000) 2024-03-06T11:41:28.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:28.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:28.004776+0000) 2024-03-06T11:41:29.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:29.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:29.005007+0000) 2024-03-06T11:41:30.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:30.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:30.005248+0000) 2024-03-06T11:41:30.158 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:41:30.166 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:41:31.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:31.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:31.005393+0000) 2024-03-06T11:41:32.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:32.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:32.005625+0000) 2024-03-06T11:41:33.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:33.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:33.005859+0000) 2024-03-06T11:41:34.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:34.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:34.006044+0000) 2024-03-06T11:41:35.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:35.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:35.006273+0000) 2024-03-06T11:41:36.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:36.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:36.006510+0000) 2024-03-06T11:41:37.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:37.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:37.006785+0000) 2024-03-06T11:41:38.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:38.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:38.006974+0000) 2024-03-06T11:41:39.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:39.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:39.007164+0000) 2024-03-06T11:41:40.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:40.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:40.007397+0000) 2024-03-06T11:41:41.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:41.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:41.007652+0000) 2024-03-06T11:41:42.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:42.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:42.007920+0000) 2024-03-06T11:41:43.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:43.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:43.008197+0000) 2024-03-06T11:41:44.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:44.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:44.008440+0000) 2024-03-06T11:41:45.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:45.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:45.008767+0000) 2024-03-06T11:41:46.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:46.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:46.009019+0000) 2024-03-06T11:41:47.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:47.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:47.009299+0000) 2024-03-06T11:41:48.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:48.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:48.009619+0000) 2024-03-06T11:41:49.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:49.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:49.009864+0000) 2024-03-06T11:41:50.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:50.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:50.010118+0000) 2024-03-06T11:41:51.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:51.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:51.010322+0000) 2024-03-06T11:41:52.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:52.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:52.010602+0000) 2024-03-06T11:41:53.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:53.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:53.010826+0000) 2024-03-06T11:41:54.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:54.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:54.011059+0000) 2024-03-06T11:41:55.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:55.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:55.011253+0000) 2024-03-06T11:41:56.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:56.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:56.011460+0000) 2024-03-06T11:41:57.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:57.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:57.011686+0000) 2024-03-06T11:41:58.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:58.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:58.011912+0000) 2024-03-06T11:41:59.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:41:59.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:41:59.012146+0000) 2024-03-06T11:42:00.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:00.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:00.012371+0000) 2024-03-06T11:42:00.362 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:42:00.370 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:42:01.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:01.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:01.012599+0000) 2024-03-06T11:42:02.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:02.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:02.012825+0000) 2024-03-06T11:42:03.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:03.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:03.013047+0000) 2024-03-06T11:42:04.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:04.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:04.013257+0000) 2024-03-06T11:42:05.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:05.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:05.013463+0000) 2024-03-06T11:42:06.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:06.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:06.013714+0000) 2024-03-06T11:42:07.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:07.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:07.013935+0000) 2024-03-06T11:42:08.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:08.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:08.014159+0000) 2024-03-06T11:42:09.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:09.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:09.014364+0000) 2024-03-06T11:42:10.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:10.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:10.014648+0000) 2024-03-06T11:42:11.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:11.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:11.014872+0000) 2024-03-06T11:42:12.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:12.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:12.015067+0000) 2024-03-06T11:42:13.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:13.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:13.015263+0000) 2024-03-06T11:42:14.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:14.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:14.015494+0000) 2024-03-06T11:42:15.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:15.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:15.015685+0000) 2024-03-06T11:42:16.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:16.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:16.015902+0000) 2024-03-06T11:42:17.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:17.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:17.016114+0000) 2024-03-06T11:42:18.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:18.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:18.016311+0000) 2024-03-06T11:42:19.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:19.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:19.016574+0000) 2024-03-06T11:42:20.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:20.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:20.016778+0000) 2024-03-06T11:42:21.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:21.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:21.017023+0000) 2024-03-06T11:42:22.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:22.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:22.017254+0000) 2024-03-06T11:42:23.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:23.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:23.017473+0000) 2024-03-06T11:42:24.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:24.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:24.017686+0000) 2024-03-06T11:42:25.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:25.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:25.017932+0000) 2024-03-06T11:42:26.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:26.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:26.018164+0000) 2024-03-06T11:42:27.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:27.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:27.018363+0000) 2024-03-06T11:42:28.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:28.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:28.018642+0000) 2024-03-06T11:42:29.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:29.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:29.018823+0000) 2024-03-06T11:42:30.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:30.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:30.019044+0000) 2024-03-06T11:42:30.574 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:42:30.582 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:42:31.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:31.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:31.019277+0000) 2024-03-06T11:42:32.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:32.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:32.019520+0000) 2024-03-06T11:42:33.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:33.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:33.019791+0000) 2024-03-06T11:42:34.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:34.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:34.020031+0000) 2024-03-06T11:42:35.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:35.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:35.020266+0000) 2024-03-06T11:42:36.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:36.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:36.020504+0000) 2024-03-06T11:42:37.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:37.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:37.020741+0000) 2024-03-06T11:42:38.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:38.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:38.020957+0000) 2024-03-06T11:42:39.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:39.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:39.021197+0000) 2024-03-06T11:42:40.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:40.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:40.021427+0000) 2024-03-06T11:42:41.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:41.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:41.021645+0000) 2024-03-06T11:42:42.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:42.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:42.021865+0000) 2024-03-06T11:42:43.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:43.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:43.022088+0000) 2024-03-06T11:42:44.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:44.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:44.022323+0000) 2024-03-06T11:42:45.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:45.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:45.022589+0000) 2024-03-06T11:42:46.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:46.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:46.022825+0000) 2024-03-06T11:42:47.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:47.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:47.023041+0000) 2024-03-06T11:42:48.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:48.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:48.023261+0000) 2024-03-06T11:42:49.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:49.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:49.023502+0000) 2024-03-06T11:42:50.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:50.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:50.023692+0000) 2024-03-06T11:42:51.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:51.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:51.023912+0000) 2024-03-06T11:42:52.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:52.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:52.024142+0000) 2024-03-06T11:42:53.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:53.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:53.024374+0000) 2024-03-06T11:42:54.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:54.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:54.024656+0000) 2024-03-06T11:42:55.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:55.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:55.024883+0000) 2024-03-06T11:42:56.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:56.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:56.025112+0000) 2024-03-06T11:42:57.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:57.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:57.025337+0000) 2024-03-06T11:42:58.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:58.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:58.025590+0000) 2024-03-06T11:42:59.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:42:59.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:42:59.025829+0000) 2024-03-06T11:43:00.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:00.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:00.026032+0000) 2024-03-06T11:43:00.805 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:43:00.814 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:43:01.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:01.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:01.026239+0000) 2024-03-06T11:43:02.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:02.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:02.026480+0000) 2024-03-06T11:43:03.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:03.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:03.026707+0000) 2024-03-06T11:43:04.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:04.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:04.026943+0000) 2024-03-06T11:43:05.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:05.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:05.027134+0000) 2024-03-06T11:43:06.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:06.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:06.027398+0000) 2024-03-06T11:43:07.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:07.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:07.027620+0000) 2024-03-06T11:43:08.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:08.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:08.027849+0000) 2024-03-06T11:43:09.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:09.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:09.028091+0000) 2024-03-06T11:43:10.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:10.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:10.028317+0000) 2024-03-06T11:43:11.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:11.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:11.028561+0000) 2024-03-06T11:43:12.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:12.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:12.028802+0000) 2024-03-06T11:43:13.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:13.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:13.029039+0000) 2024-03-06T11:43:14.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:14.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:14.029234+0000) 2024-03-06T11:43:15.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:15.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:15.029479+0000) 2024-03-06T11:43:16.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:16.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:16.029722+0000) 2024-03-06T11:43:17.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:17.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:17.029975+0000) 2024-03-06T11:43:18.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:18.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:18.030253+0000) 2024-03-06T11:43:19.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:19.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:19.030521+0000) 2024-03-06T11:43:20.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:20.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:20.030851+0000) 2024-03-06T11:43:21.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:21.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:21.031093+0000) 2024-03-06T11:43:22.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:22.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:22.031297+0000) 2024-03-06T11:43:23.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:23.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:23.031498+0000) 2024-03-06T11:43:24.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:24.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:24.031787+0000) 2024-03-06T11:43:25.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:25.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:25.031977+0000) 2024-03-06T11:43:26.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:26.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:26.032272+0000) 2024-03-06T11:43:27.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:27.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:27.032479+0000) 2024-03-06T11:43:28.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:28.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:28.032652+0000) 2024-03-06T11:43:29.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:29.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:29.032878+0000) 2024-03-06T11:43:30.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:30.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:30.033062+0000) 2024-03-06T11:43:31.027 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:43:31.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:43:31.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:31.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:31.033256+0000) 2024-03-06T11:43:32.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:32.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:32.033482+0000) 2024-03-06T11:43:33.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:33.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:33.033737+0000) 2024-03-06T11:43:34.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:34.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:34.033960+0000) 2024-03-06T11:43:35.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:35.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:35.034179+0000) 2024-03-06T11:43:36.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:36.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:36.034411+0000) 2024-03-06T11:43:37.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:37.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:37.034673+0000) 2024-03-06T11:43:38.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:38.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:38.034867+0000) 2024-03-06T11:43:39.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:39.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:39.035054+0000) 2024-03-06T11:43:40.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:40.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:40.035300+0000) 2024-03-06T11:43:41.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:41.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:41.035510+0000) 2024-03-06T11:43:42.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:42.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:42.035766+0000) 2024-03-06T11:43:43.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:43.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:43.036001+0000) 2024-03-06T11:43:44.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:44.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:44.036232+0000) 2024-03-06T11:43:45.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:45.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:45.036460+0000) 2024-03-06T11:43:46.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:46.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:46.036701+0000) 2024-03-06T11:43:47.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:47.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:47.036927+0000) 2024-03-06T11:43:48.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:48.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:48.037118+0000) 2024-03-06T11:43:49.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:49.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:49.037352+0000) 2024-03-06T11:43:50.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:50.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:50.037563+0000) 2024-03-06T11:43:51.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:51.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:51.037820+0000) 2024-03-06T11:43:52.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:52.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:52.038057+0000) 2024-03-06T11:43:53.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:53.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:53.038288+0000) 2024-03-06T11:43:54.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:54.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:54.038529+0000) 2024-03-06T11:43:55.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:55.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:55.038806+0000) 2024-03-06T11:43:56.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:56.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:56.039042+0000) 2024-03-06T11:43:57.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:57.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:57.039276+0000) 2024-03-06T11:43:58.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:58.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:58.039496+0000) 2024-03-06T11:43:59.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:43:59.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:43:59.039732+0000) 2024-03-06T11:44:00.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:00.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:00.039964+0000) 2024-03-06T11:44:01.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:01.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:01.040193+0000) 2024-03-06T11:44:01.206 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:44:01.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:44:02.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:02.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:02.040419+0000) 2024-03-06T11:44:03.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:03.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:03.040662+0000) 2024-03-06T11:44:04.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:04.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:04.040893+0000) 2024-03-06T11:44:05.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:05.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:05.041089+0000) 2024-03-06T11:44:06.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:06.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:06.041322+0000) 2024-03-06T11:44:07.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:07.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:07.041505+0000) 2024-03-06T11:44:08.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:08.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:08.041774+0000) 2024-03-06T11:44:09.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:09.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:09.042003+0000) 2024-03-06T11:44:10.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:10.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:10.042229+0000) 2024-03-06T11:44:11.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:11.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:11.042463+0000) 2024-03-06T11:44:12.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:12.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:12.042693+0000) 2024-03-06T11:44:13.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:13.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:13.042919+0000) 2024-03-06T11:44:14.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:14.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:14.043151+0000) 2024-03-06T11:44:15.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:15.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:15.043380+0000) 2024-03-06T11:44:16.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:16.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:16.043662+0000) 2024-03-06T11:44:17.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:17.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:17.043889+0000) 2024-03-06T11:44:18.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:18.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:18.044108+0000) 2024-03-06T11:44:19.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:19.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:19.044303+0000) 2024-03-06T11:44:20.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:20.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:20.044629+0000) 2024-03-06T11:44:21.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:21.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:21.044859+0000) 2024-03-06T11:44:22.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:22.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:22.045060+0000) 2024-03-06T11:44:23.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:23.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:23.045257+0000) 2024-03-06T11:44:24.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:24.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:24.045495+0000) 2024-03-06T11:44:25.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:25.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:25.045698+0000) 2024-03-06T11:44:26.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:26.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:26.045952+0000) 2024-03-06T11:44:27.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:27.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:27.046161+0000) 2024-03-06T11:44:28.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:28.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:28.046412+0000) 2024-03-06T11:44:29.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:29.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:29.046661+0000) 2024-03-06T11:44:30.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:30.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:30.046891+0000) 2024-03-06T11:44:31.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:31.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:31.047069+0000) 2024-03-06T11:44:31.427 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:44:31.435 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:44:32.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:32.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:32.047301+0000) 2024-03-06T11:44:33.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:33.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:33.047525+0000) 2024-03-06T11:44:34.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:34.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:34.047782+0000) 2024-03-06T11:44:35.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:35.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:35.048015+0000) 2024-03-06T11:44:36.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:36.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:36.048257+0000) 2024-03-06T11:44:37.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:37.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:37.048488+0000) 2024-03-06T11:44:38.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:38.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:38.048718+0000) 2024-03-06T11:44:39.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:39.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:39.048956+0000) 2024-03-06T11:44:40.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:40.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:40.049195+0000) 2024-03-06T11:44:41.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:41.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:41.049441+0000) 2024-03-06T11:44:42.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:42.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:42.049670+0000) 2024-03-06T11:44:43.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:43.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:43.049908+0000) 2024-03-06T11:44:44.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:44.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:44.050113+0000) 2024-03-06T11:44:45.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:45.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:45.050360+0000) 2024-03-06T11:44:46.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:46.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:46.050605+0000) 2024-03-06T11:44:47.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:47.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:47.050846+0000) 2024-03-06T11:44:48.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:48.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:48.051077+0000) 2024-03-06T11:44:49.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:49.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:49.051317+0000) 2024-03-06T11:44:50.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:50.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:50.051508+0000) 2024-03-06T11:44:51.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:51.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:51.051793+0000) 2024-03-06T11:44:52.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:52.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:52.052008+0000) 2024-03-06T11:44:53.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:53.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:53.052236+0000) 2024-03-06T11:44:54.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:54.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:54.052481+0000) 2024-03-06T11:44:55.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:55.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:55.052715+0000) 2024-03-06T11:44:56.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:56.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:56.052953+0000) 2024-03-06T11:44:57.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:57.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:57.053182+0000) 2024-03-06T11:44:58.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:58.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:58.053401+0000) 2024-03-06T11:44:59.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:44:59.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:44:59.053663+0000) 2024-03-06T11:45:00.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:00.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:00.053893+0000) 2024-03-06T11:45:01.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:01.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:01.054129+0000) 2024-03-06T11:45:01.665 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:45:01.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:45:02.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:02.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:02.054374+0000) 2024-03-06T11:45:03.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:03.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:03.054647+0000) 2024-03-06T11:45:04.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:04.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:04.054892+0000) 2024-03-06T11:45:05.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:05.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:05.055140+0000) 2024-03-06T11:45:06.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:06.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:06.055384+0000) 2024-03-06T11:45:07.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:07.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:07.055652+0000) 2024-03-06T11:45:08.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:08.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:08.055952+0000) 2024-03-06T11:45:09.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:09.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:09.056193+0000) 2024-03-06T11:45:10.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:10.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:10.056398+0000) 2024-03-06T11:45:11.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:11.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:11.056658+0000) 2024-03-06T11:45:12.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:12.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:12.056852+0000) 2024-03-06T11:45:13.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:13.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:13.057088+0000) 2024-03-06T11:45:14.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:14.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:14.057279+0000) 2024-03-06T11:45:15.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:15.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:15.057506+0000) 2024-03-06T11:45:16.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:16.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:16.057759+0000) 2024-03-06T11:45:17.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:17.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:17.057976+0000) 2024-03-06T11:45:18.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:18.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:18.058166+0000) 2024-03-06T11:45:19.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:19.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:19.058360+0000) 2024-03-06T11:45:20.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:20.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:20.058637+0000) 2024-03-06T11:45:21.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:21.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:21.058860+0000) 2024-03-06T11:45:22.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:22.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:22.059082+0000) 2024-03-06T11:45:23.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:23.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:23.059312+0000) 2024-03-06T11:45:24.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:24.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:24.059512+0000) 2024-03-06T11:45:25.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:25.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:25.059801+0000) 2024-03-06T11:45:26.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:26.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:26.060051+0000) 2024-03-06T11:45:27.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:27.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:27.060294+0000) 2024-03-06T11:45:28.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:28.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:28.060489+0000) 2024-03-06T11:45:29.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:29.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:29.060738+0000) 2024-03-06T11:45:30.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:30.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:30.060922+0000) 2024-03-06T11:45:31.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:31.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:31.061106+0000) 2024-03-06T11:45:31.878 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:45:31.885 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:45:32.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:32.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:32.061321+0000) 2024-03-06T11:45:33.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:33.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:33.061524+0000) 2024-03-06T11:45:34.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:34.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:34.061797+0000) 2024-03-06T11:45:35.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:35.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:35.061983+0000) 2024-03-06T11:45:36.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:36.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:36.062217+0000) 2024-03-06T11:45:37.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:37.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:37.062398+0000) 2024-03-06T11:45:38.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:38.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:38.062660+0000) 2024-03-06T11:45:39.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:39.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:39.062880+0000) 2024-03-06T11:45:40.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:40.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:40.063114+0000) 2024-03-06T11:45:41.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:41.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:41.063351+0000) 2024-03-06T11:45:42.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:42.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:42.063560+0000) 2024-03-06T11:45:43.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:43.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:43.063736+0000) 2024-03-06T11:45:44.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:44.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:44.063967+0000) 2024-03-06T11:45:45.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:45.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:45.064171+0000) 2024-03-06T11:45:46.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:46.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:46.064419+0000) 2024-03-06T11:45:47.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:47.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:47.064662+0000) 2024-03-06T11:45:48.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:48.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:48.064901+0000) 2024-03-06T11:45:49.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:49.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:49.065089+0000) 2024-03-06T11:45:50.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:50.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:50.065319+0000) 2024-03-06T11:45:51.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:51.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:51.065579+0000) 2024-03-06T11:45:52.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:52.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:52.065811+0000) 2024-03-06T11:45:53.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:53.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:53.066031+0000) 2024-03-06T11:45:54.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:54.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:54.066262+0000) 2024-03-06T11:45:55.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:55.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:55.066491+0000) 2024-03-06T11:45:56.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:56.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:56.066787+0000) 2024-03-06T11:45:57.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:57.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:57.067022+0000) 2024-03-06T11:45:58.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:58.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:58.067198+0000) 2024-03-06T11:45:59.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:45:59.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:45:59.067436+0000) 2024-03-06T11:46:00.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:00.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:00.067655+0000) 2024-03-06T11:46:01.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:01.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:01.067897+0000) 2024-03-06T11:46:02.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:02.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:02.068127+0000) 2024-03-06T11:46:02.091 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:46:02.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:46:03.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:03.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:03.068376+0000) 2024-03-06T11:46:04.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:04.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:04.068647+0000) 2024-03-06T11:46:05.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:05.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:05.068869+0000) 2024-03-06T11:46:06.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:06.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:06.069121+0000) 2024-03-06T11:46:07.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:07.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:07.069325+0000) 2024-03-06T11:46:08.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:08.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:08.069573+0000) 2024-03-06T11:46:09.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:09.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:09.069791+0000) 2024-03-06T11:46:10.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:10.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:10.070047+0000) 2024-03-06T11:46:11.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:11.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:11.070280+0000) 2024-03-06T11:46:12.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:12.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:12.070479+0000) 2024-03-06T11:46:13.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:13.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:13.070678+0000) 2024-03-06T11:46:14.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:14.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:14.070894+0000) 2024-03-06T11:46:15.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:15.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:15.071126+0000) 2024-03-06T11:46:16.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:16.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:16.071441+0000) 2024-03-06T11:46:17.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:17.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:17.071668+0000) 2024-03-06T11:46:18.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:18.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:18.071903+0000) 2024-03-06T11:46:19.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:19.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:19.072141+0000) 2024-03-06T11:46:20.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:20.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:20.072366+0000) 2024-03-06T11:46:21.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:21.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:21.072645+0000) 2024-03-06T11:46:22.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:22.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:22.072839+0000) 2024-03-06T11:46:23.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:23.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:23.073076+0000) 2024-03-06T11:46:24.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:24.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:24.073314+0000) 2024-03-06T11:46:25.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:25.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:25.073580+0000) 2024-03-06T11:46:26.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:26.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:26.073840+0000) 2024-03-06T11:46:27.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:27.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:27.074079+0000) 2024-03-06T11:46:28.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:28.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:28.074256+0000) 2024-03-06T11:46:29.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:29.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:29.074471+0000) 2024-03-06T11:46:30.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:30.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:30.074720+0000) 2024-03-06T11:46:31.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:31.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:31.074919+0000) 2024-03-06T11:46:32.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:32.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:32.075109+0000) 2024-03-06T11:46:32.222 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:46:32.230 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:46:33.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:33.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:33.075373+0000) 2024-03-06T11:46:34.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:34.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:34.075666+0000) 2024-03-06T11:46:35.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:35.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:35.075892+0000) 2024-03-06T11:46:36.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:36.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:36.076099+0000) 2024-03-06T11:46:37.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:37.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:37.076310+0000) 2024-03-06T11:46:38.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:38.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:38.076568+0000) 2024-03-06T11:46:39.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:39.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:39.076764+0000) 2024-03-06T11:46:40.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:40.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:40.076962+0000) 2024-03-06T11:46:41.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:41.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:41.077151+0000) 2024-03-06T11:46:42.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:42.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:42.077375+0000) 2024-03-06T11:46:43.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:43.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:43.077620+0000) 2024-03-06T11:46:44.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:44.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:44.077901+0000) 2024-03-06T11:46:45.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:45.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:45.078099+0000) 2024-03-06T11:46:46.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:46.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:46.078332+0000) 2024-03-06T11:46:47.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:47.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:47.078520+0000) 2024-03-06T11:46:48.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:48.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:48.078747+0000) 2024-03-06T11:46:49.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:49.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:49.078929+0000) 2024-03-06T11:46:50.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:50.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:50.079116+0000) 2024-03-06T11:46:51.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:51.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:51.079349+0000) 2024-03-06T11:46:52.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:52.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:52.079558+0000) 2024-03-06T11:46:53.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:53.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:53.079752+0000) 2024-03-06T11:46:54.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:54.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:54.079946+0000) 2024-03-06T11:46:55.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:55.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:55.080128+0000) 2024-03-06T11:46:56.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:56.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:56.080361+0000) 2024-03-06T11:46:57.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:57.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:57.080618+0000) 2024-03-06T11:46:58.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:58.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:58.080801+0000) 2024-03-06T11:46:59.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:46:59.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:46:59.080977+0000) 2024-03-06T11:47:00.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:00.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:00.081157+0000) 2024-03-06T11:47:01.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:01.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:01.081392+0000) 2024-03-06T11:47:02.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:02.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:02.081649+0000) 2024-03-06T11:47:02.425 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:47:02.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:47:03.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:03.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:03.081853+0000) 2024-03-06T11:47:04.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:04.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:04.082052+0000) 2024-03-06T11:47:05.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:05.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:05.082296+0000) 2024-03-06T11:47:06.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:06.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:06.082576+0000) 2024-03-06T11:47:07.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:07.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:07.082792+0000) 2024-03-06T11:47:08.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:08.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:08.083002+0000) 2024-03-06T11:47:09.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:09.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:09.083239+0000) 2024-03-06T11:47:10.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:10.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:10.083512+0000) 2024-03-06T11:47:11.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:11.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:11.083753+0000) 2024-03-06T11:47:12.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:12.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:12.083963+0000) 2024-03-06T11:47:13.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:13.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:13.084160+0000) 2024-03-06T11:47:14.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:14.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:14.084396+0000) 2024-03-06T11:47:15.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:15.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:15.084663+0000) 2024-03-06T11:47:16.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:16.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:16.084886+0000) 2024-03-06T11:47:17.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:17.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:17.085096+0000) 2024-03-06T11:47:18.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:18.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:18.085335+0000) 2024-03-06T11:47:19.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:19.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:19.085625+0000) 2024-03-06T11:47:20.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:20.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:20.085876+0000) 2024-03-06T11:47:21.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:21.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:21.086124+0000) 2024-03-06T11:47:22.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:22.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:22.086325+0000) 2024-03-06T11:47:23.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:23.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:23.086565+0000) 2024-03-06T11:47:24.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:24.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:24.086759+0000) 2024-03-06T11:47:25.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:25.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:25.087006+0000) 2024-03-06T11:47:26.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:26.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:26.087311+0000) 2024-03-06T11:47:27.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:27.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:27.087613+0000) 2024-03-06T11:47:28.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:28.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:28.087817+0000) 2024-03-06T11:47:29.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:29.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:29.088044+0000) 2024-03-06T11:47:30.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:30.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:30.088273+0000) 2024-03-06T11:47:31.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:31.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:31.088492+0000) 2024-03-06T11:47:32.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:32.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:32.088775+0000) 2024-03-06T11:47:32.646 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:47:32.654 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:47:33.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:33.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:33.089058+0000) 2024-03-06T11:47:34.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:34.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:34.089329+0000) 2024-03-06T11:47:35.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:35.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:35.089639+0000) 2024-03-06T11:47:36.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:36.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:36.089902+0000) 2024-03-06T11:47:37.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:37.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:37.090158+0000) 2024-03-06T11:47:38.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:38.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:38.090382+0000) 2024-03-06T11:47:39.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:39.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:39.090640+0000) 2024-03-06T11:47:40.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:40.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:40.090843+0000) 2024-03-06T11:47:41.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:41.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:41.091037+0000) 2024-03-06T11:47:42.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:42.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:42.091240+0000) 2024-03-06T11:47:43.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:43.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:43.091441+0000) 2024-03-06T11:47:44.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:44.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:44.091676+0000) 2024-03-06T11:47:45.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:45.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:45.091901+0000) 2024-03-06T11:47:46.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:46.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:46.092137+0000) 2024-03-06T11:47:47.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:47.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:47.092362+0000) 2024-03-06T11:47:48.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:48.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:48.092620+0000) 2024-03-06T11:47:49.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:49.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:49.092800+0000) 2024-03-06T11:47:50.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:50.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:50.093045+0000) 2024-03-06T11:47:51.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:51.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:51.093313+0000) 2024-03-06T11:47:52.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:52.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:52.093612+0000) 2024-03-06T11:47:53.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:53.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:53.093854+0000) 2024-03-06T11:47:54.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:54.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:54.094081+0000) 2024-03-06T11:47:55.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:55.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:55.094297+0000) 2024-03-06T11:47:56.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:56.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:56.094619+0000) 2024-03-06T11:47:57.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:57.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:57.094824+0000) 2024-03-06T11:47:58.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:58.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:58.095049+0000) 2024-03-06T11:47:59.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:47:59.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:47:59.095278+0000) 2024-03-06T11:48:00.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:00.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:00.095612+0000) 2024-03-06T11:48:01.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:01.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:01.095865+0000) 2024-03-06T11:48:02.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:02.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:02.096101+0000) 2024-03-06T11:48:02.868 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:48:02.876 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:48:03.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:03.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:03.096349+0000) 2024-03-06T11:48:04.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:04.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:04.096607+0000) 2024-03-06T11:48:05.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:05.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:05.096801+0000) 2024-03-06T11:48:06.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:06.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:06.097003+0000) 2024-03-06T11:48:07.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:07.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:07.097222+0000) 2024-03-06T11:48:08.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:08.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:08.097454+0000) 2024-03-06T11:48:09.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:09.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:09.097693+0000) 2024-03-06T11:48:10.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:10.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:10.097894+0000) 2024-03-06T11:48:11.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:11.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:11.098105+0000) 2024-03-06T11:48:12.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:12.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:12.098332+0000) 2024-03-06T11:48:13.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:13.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:13.098591+0000) 2024-03-06T11:48:14.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:14.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:14.098818+0000) 2024-03-06T11:48:15.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:15.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:15.099090+0000) 2024-03-06T11:48:16.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:16.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:16.099384+0000) 2024-03-06T11:48:17.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:17.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:17.099693+0000) 2024-03-06T11:48:18.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:18.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:18.099973+0000) 2024-03-06T11:48:19.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:19.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:19.100216+0000) 2024-03-06T11:48:20.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:20.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:20.100451+0000) 2024-03-06T11:48:21.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:21.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:21.100690+0000) 2024-03-06T11:48:22.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:22.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:22.100915+0000) 2024-03-06T11:48:23.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:23.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:23.101159+0000) 2024-03-06T11:48:24.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:24.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:24.101434+0000) 2024-03-06T11:48:25.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:25.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:25.101775+0000) 2024-03-06T11:48:26.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:26.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:26.102070+0000) 2024-03-06T11:48:27.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:27.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:27.102348+0000) 2024-03-06T11:48:28.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:28.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:28.102645+0000) 2024-03-06T11:48:29.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:29.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:29.102844+0000) 2024-03-06T11:48:30.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:30.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:30.103096+0000) 2024-03-06T11:48:31.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:31.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:31.103271+0000) 2024-03-06T11:48:32.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:32.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:32.103515+0000) 2024-03-06T11:48:33.081 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:48:33.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:48:33.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:33.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:33.103693+0000) 2024-03-06T11:48:34.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:34.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:34.103902+0000) 2024-03-06T11:48:35.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:35.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:35.104157+0000) 2024-03-06T11:48:36.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:36.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:36.104387+0000) 2024-03-06T11:48:37.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:37.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:37.104639+0000) 2024-03-06T11:48:38.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:38.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:38.104881+0000) 2024-03-06T11:48:39.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:39.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:39.105075+0000) 2024-03-06T11:48:40.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:40.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:40.105316+0000) 2024-03-06T11:48:41.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:41.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:41.105562+0000) 2024-03-06T11:48:42.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:42.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:42.105757+0000) 2024-03-06T11:48:43.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:43.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:43.105942+0000) 2024-03-06T11:48:44.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:44.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:44.106149+0000) 2024-03-06T11:48:45.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:45.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:45.106400+0000) 2024-03-06T11:48:46.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:46.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:46.106653+0000) 2024-03-06T11:48:47.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:47.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:47.106848+0000) 2024-03-06T11:48:48.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:48.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:48.107047+0000) 2024-03-06T11:48:49.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:49.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:49.107263+0000) 2024-03-06T11:48:50.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:50.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:50.107466+0000) 2024-03-06T11:48:51.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:51.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:51.107665+0000) 2024-03-06T11:48:52.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:52.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:52.107937+0000) 2024-03-06T11:48:53.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:53.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:53.108216+0000) 2024-03-06T11:48:54.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:54.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:54.108495+0000) 2024-03-06T11:48:55.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:55.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:55.108744+0000) 2024-03-06T11:48:56.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:56.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:56.108929+0000) 2024-03-06T11:48:57.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:57.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:57.109114+0000) 2024-03-06T11:48:58.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:58.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:58.109298+0000) 2024-03-06T11:48:59.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:48:59.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:48:59.109522+0000) 2024-03-06T11:49:00.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:00.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:00.109752+0000) 2024-03-06T11:49:01.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:01.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:01.109963+0000) 2024-03-06T11:49:02.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:02.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:02.110191+0000) 2024-03-06T11:49:03.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:03.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:03.110370+0000) 2024-03-06T11:49:03.227 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:49:03.235 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:49:04.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:04.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:04.110622+0000) 2024-03-06T11:49:05.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:05.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:05.110813+0000) 2024-03-06T11:49:06.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:06.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:06.111016+0000) 2024-03-06T11:49:07.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:07.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:07.111210+0000) 2024-03-06T11:49:08.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:08.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:08.111453+0000) 2024-03-06T11:49:09.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:09.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:09.111684+0000) 2024-03-06T11:49:10.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:10.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:10.111878+0000) 2024-03-06T11:49:11.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:11.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:11.112106+0000) 2024-03-06T11:49:12.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:12.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:12.112288+0000) 2024-03-06T11:49:13.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:13.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:13.112466+0000) 2024-03-06T11:49:14.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:14.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:14.112683+0000) 2024-03-06T11:49:15.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:15.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:15.112875+0000) 2024-03-06T11:49:16.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:16.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:16.113076+0000) 2024-03-06T11:49:17.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:17.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:17.113275+0000) 2024-03-06T11:49:18.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:18.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:18.113572+0000) 2024-03-06T11:49:19.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:19.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:19.113758+0000) 2024-03-06T11:49:20.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:20.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:20.113954+0000) 2024-03-06T11:49:21.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:21.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:21.114147+0000) 2024-03-06T11:49:22.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:22.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:22.114388+0000) 2024-03-06T11:49:23.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:23.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:23.114643+0000) 2024-03-06T11:49:24.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:24.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:24.114889+0000) 2024-03-06T11:49:25.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:25.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:25.115137+0000) 2024-03-06T11:49:26.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:26.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:26.115386+0000) 2024-03-06T11:49:27.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:27.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:27.115643+0000) 2024-03-06T11:49:28.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:28.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:28.115882+0000) 2024-03-06T11:49:29.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:29.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:29.116129+0000) 2024-03-06T11:49:30.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:30.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:30.116377+0000) 2024-03-06T11:49:31.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:31.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:31.116595+0000) 2024-03-06T11:49:32.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:32.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:32.116804+0000) 2024-03-06T11:49:33.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:33.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:33.117011+0000) 2024-03-06T11:49:33.433 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:49:33.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:49:34.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:34.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:34.117248+0000) 2024-03-06T11:49:35.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:35.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:35.117486+0000) 2024-03-06T11:49:36.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:36.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:36.117730+0000) 2024-03-06T11:49:37.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:37.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:37.117915+0000) 2024-03-06T11:49:38.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:38.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:38.118103+0000) 2024-03-06T11:49:39.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:39.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:39.118329+0000) 2024-03-06T11:49:40.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:40.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:40.118586+0000) 2024-03-06T11:49:41.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:41.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:41.118803+0000) 2024-03-06T11:49:42.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:42.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:42.118996+0000) 2024-03-06T11:49:43.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:43.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:43.119220+0000) 2024-03-06T11:49:44.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:44.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:44.119451+0000) 2024-03-06T11:49:45.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:45.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:45.119691+0000) 2024-03-06T11:49:46.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:46.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:46.119931+0000) 2024-03-06T11:49:47.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:47.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:47.120109+0000) 2024-03-06T11:49:48.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:48.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:48.120350+0000) 2024-03-06T11:49:49.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:49.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:49.120624+0000) 2024-03-06T11:49:50.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:50.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:50.120867+0000) 2024-03-06T11:49:51.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:51.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:51.121076+0000) 2024-03-06T11:49:52.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:52.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:52.121318+0000) 2024-03-06T11:49:53.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:53.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:53.121587+0000) 2024-03-06T11:49:54.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:54.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:54.121795+0000) 2024-03-06T11:49:55.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:55.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:55.122025+0000) 2024-03-06T11:49:56.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:56.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:56.122227+0000) 2024-03-06T11:49:57.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:57.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:57.122465+0000) 2024-03-06T11:49:58.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:58.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:58.122707+0000) 2024-03-06T11:49:59.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:49:59.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:49:59.122918+0000) 2024-03-06T11:50:00.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:00.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:00.123122+0000) 2024-03-06T11:50:01.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:01.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:01.123367+0000) 2024-03-06T11:50:02.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:02.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:02.123645+0000) 2024-03-06T11:50:03.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:03.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:03.123846+0000) 2024-03-06T11:50:03.812 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:50:03.820 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:50:04.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:04.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:04.124086+0000) 2024-03-06T11:50:05.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:05.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:05.124281+0000) 2024-03-06T11:50:06.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:06.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:06.124512+0000) 2024-03-06T11:50:07.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:07.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:07.124734+0000) 2024-03-06T11:50:08.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:08.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:08.124927+0000) 2024-03-06T11:50:09.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:09.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:09.125121+0000) 2024-03-06T11:50:10.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:10.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:10.125360+0000) 2024-03-06T11:50:11.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:11.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:11.125627+0000) 2024-03-06T11:50:12.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:12.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:12.125835+0000) 2024-03-06T11:50:13.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:13.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:13.126082+0000) 2024-03-06T11:50:14.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:14.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:14.126363+0000) 2024-03-06T11:50:15.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:15.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:15.126678+0000) 2024-03-06T11:50:16.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:16.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:16.126949+0000) 2024-03-06T11:50:17.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:17.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:17.127231+0000) 2024-03-06T11:50:18.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:18.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:18.127500+0000) 2024-03-06T11:50:19.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:19.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:19.127739+0000) 2024-03-06T11:50:20.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:20.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:20.127960+0000) 2024-03-06T11:50:21.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:21.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:21.128169+0000) 2024-03-06T11:50:22.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:22.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:22.128368+0000) 2024-03-06T11:50:23.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:23.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:23.128636+0000) 2024-03-06T11:50:24.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:24.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:24.128827+0000) 2024-03-06T11:50:25.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:25.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:25.129016+0000) 2024-03-06T11:50:26.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:26.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:26.129216+0000) 2024-03-06T11:50:27.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:27.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:27.129419+0000) 2024-03-06T11:50:28.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:28.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:28.129671+0000) 2024-03-06T11:50:29.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:29.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:29.129864+0000) 2024-03-06T11:50:30.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:30.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:30.130054+0000) 2024-03-06T11:50:31.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:31.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:31.130250+0000) 2024-03-06T11:50:32.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:32.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:32.130492+0000) 2024-03-06T11:50:33.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:33.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:33.130735+0000) 2024-03-06T11:50:34.033 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:50:34.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:50:34.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:34.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:34.130952+0000) 2024-03-06T11:50:35.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:35.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:35.131163+0000) 2024-03-06T11:50:36.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:36.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:36.131368+0000) 2024-03-06T11:50:37.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:37.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:37.131632+0000) 2024-03-06T11:50:38.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:38.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:38.131854+0000) 2024-03-06T11:50:39.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:39.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:39.132098+0000) 2024-03-06T11:50:40.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:40.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:40.132303+0000) 2024-03-06T11:50:41.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:41.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:41.132522+0000) 2024-03-06T11:50:42.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:42.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:42.132742+0000) 2024-03-06T11:50:43.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:43.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:43.132927+0000) 2024-03-06T11:50:44.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:44.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:44.133124+0000) 2024-03-06T11:50:45.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:45.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:45.133363+0000) 2024-03-06T11:50:46.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:46.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:46.133624+0000) 2024-03-06T11:50:47.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:47.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:47.133833+0000) 2024-03-06T11:50:48.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:48.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:48.134022+0000) 2024-03-06T11:50:49.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:49.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:49.134251+0000) 2024-03-06T11:50:50.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:50.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:50.134441+0000) 2024-03-06T11:50:51.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:51.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:51.134740+0000) 2024-03-06T11:50:52.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:52.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:52.134985+0000) 2024-03-06T11:50:53.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:53.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:53.135213+0000) 2024-03-06T11:50:54.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:54.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:54.135484+0000) 2024-03-06T11:50:55.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:55.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:55.135719+0000) 2024-03-06T11:50:56.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:56.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:56.135897+0000) 2024-03-06T11:50:57.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:57.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:57.136108+0000) 2024-03-06T11:50:58.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:58.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:58.136339+0000) 2024-03-06T11:50:59.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:50:59.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:50:59.136603+0000) 2024-03-06T11:51:00.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:00.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:00.136801+0000) 2024-03-06T11:51:01.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:01.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:01.137076+0000) 2024-03-06T11:51:02.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:02.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:02.137341+0000) 2024-03-06T11:51:03.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:03.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:03.137585+0000) 2024-03-06T11:51:04.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:04.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:04.137818+0000) 2024-03-06T11:51:04.171 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:51:04.179 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:51:05.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:05.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:05.138075+0000) 2024-03-06T11:51:06.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:06.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:06.138386+0000) 2024-03-06T11:51:07.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:07.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:07.138695+0000) 2024-03-06T11:51:08.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:08.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:08.138966+0000) 2024-03-06T11:51:09.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:09.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:09.139247+0000) 2024-03-06T11:51:10.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:10.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:10.139526+0000) 2024-03-06T11:51:11.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:11.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:11.139821+0000) 2024-03-06T11:51:12.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:12.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:12.140051+0000) 2024-03-06T11:51:13.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:13.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:13.140276+0000) 2024-03-06T11:51:14.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:14.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:14.140482+0000) 2024-03-06T11:51:15.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:15.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:15.140721+0000) 2024-03-06T11:51:16.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:16.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:16.140966+0000) 2024-03-06T11:51:17.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:17.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:17.141194+0000) 2024-03-06T11:51:18.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:18.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:18.141398+0000) 2024-03-06T11:51:19.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:19.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:19.141638+0000) 2024-03-06T11:51:20.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:20.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:20.141837+0000) 2024-03-06T11:51:21.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:21.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:21.142068+0000) 2024-03-06T11:51:22.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:22.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:22.142332+0000) 2024-03-06T11:51:23.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:23.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:23.142594+0000) 2024-03-06T11:51:24.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:24.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:24.142784+0000) 2024-03-06T11:51:25.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:25.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:25.142971+0000) 2024-03-06T11:51:26.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:26.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:26.143174+0000) 2024-03-06T11:51:27.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:27.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:27.143395+0000) 2024-03-06T11:51:28.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:28.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:28.143688+0000) 2024-03-06T11:51:29.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:29.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:29.143919+0000) 2024-03-06T11:51:30.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:30.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:30.144151+0000) 2024-03-06T11:51:31.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:31.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:31.144359+0000) 2024-03-06T11:51:32.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:32.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:32.144627+0000) 2024-03-06T11:51:33.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:33.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:33.144833+0000) 2024-03-06T11:51:34.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:34.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:34.145066+0000) 2024-03-06T11:51:34.309 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:51:34.317 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:51:35.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:35.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:35.145296+0000) 2024-03-06T11:51:36.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:36.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:36.145568+0000) 2024-03-06T11:51:37.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:37.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:37.145784+0000) 2024-03-06T11:51:38.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:38.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:38.146010+0000) 2024-03-06T11:51:39.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:39.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:39.146276+0000) 2024-03-06T11:51:40.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:40.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:40.146587+0000) 2024-03-06T11:51:41.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:41.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:41.146869+0000) 2024-03-06T11:51:42.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:42.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:42.147101+0000) 2024-03-06T11:51:43.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:43.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:43.147299+0000) 2024-03-06T11:51:44.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:44.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:44.147522+0000) 2024-03-06T11:51:45.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:45.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:45.147785+0000) 2024-03-06T11:51:46.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:46.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:46.148029+0000) 2024-03-06T11:51:47.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:47.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:47.148274+0000) 2024-03-06T11:51:48.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:48.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:48.148597+0000) 2024-03-06T11:51:49.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:49.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:49.148857+0000) 2024-03-06T11:51:50.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:50.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:50.149079+0000) 2024-03-06T11:51:51.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:51.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:51.149361+0000) 2024-03-06T11:51:52.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:52.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:52.149615+0000) 2024-03-06T11:51:53.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:53.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:53.149833+0000) 2024-03-06T11:51:54.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:54.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:54.150105+0000) 2024-03-06T11:51:55.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:55.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:55.150371+0000) 2024-03-06T11:51:56.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:56.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:56.150684+0000) 2024-03-06T11:51:57.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:57.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:57.150920+0000) 2024-03-06T11:51:58.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:58.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:58.151147+0000) 2024-03-06T11:51:59.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:51:59.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:51:59.151374+0000) 2024-03-06T11:52:00.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:00.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:00.151642+0000) 2024-03-06T11:52:01.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:01.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:01.151899+0000) 2024-03-06T11:52:02.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:02.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:02.152092+0000) 2024-03-06T11:52:03.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:03.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:03.152316+0000) 2024-03-06T11:52:04.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:04.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:04.152565+0000) 2024-03-06T11:52:04.495 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:52:04.503 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:52:05.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:05.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:05.152787+0000) 2024-03-06T11:52:06.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:06.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:06.153100+0000) 2024-03-06T11:52:07.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:07.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:07.153383+0000) 2024-03-06T11:52:08.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:08.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:08.153674+0000) 2024-03-06T11:52:09.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:09.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:09.153952+0000) 2024-03-06T11:52:10.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:10.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:10.154232+0000) 2024-03-06T11:52:11.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:11.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:11.154511+0000) 2024-03-06T11:52:12.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:12.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:12.154815+0000) 2024-03-06T11:52:13.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:13.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:13.155098+0000) 2024-03-06T11:52:14.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:14.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:14.155384+0000) 2024-03-06T11:52:15.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:15.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:15.155680+0000) 2024-03-06T11:52:16.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:16.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:16.155923+0000) 2024-03-06T11:52:17.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:17.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:17.156127+0000) 2024-03-06T11:52:18.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:18.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:18.156347+0000) 2024-03-06T11:52:19.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:19.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:19.156568+0000) 2024-03-06T11:52:20.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:20.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:20.156822+0000) 2024-03-06T11:52:21.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:21.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:21.157091+0000) 2024-03-06T11:52:22.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:22.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:22.157370+0000) 2024-03-06T11:52:23.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:23.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:23.157604+0000) 2024-03-06T11:52:24.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:24.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:24.157795+0000) 2024-03-06T11:52:25.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:25.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:25.157998+0000) 2024-03-06T11:52:26.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:26.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:26.158236+0000) 2024-03-06T11:52:27.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:27.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:27.158468+0000) 2024-03-06T11:52:28.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:28.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:28.158708+0000) 2024-03-06T11:52:29.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:29.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:29.158885+0000) 2024-03-06T11:52:30.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:30.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:30.159075+0000) 2024-03-06T11:52:31.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:31.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:31.159298+0000) 2024-03-06T11:52:32.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:32.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:32.159501+0000) 2024-03-06T11:52:33.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:33.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:33.159743+0000) 2024-03-06T11:52:34.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:34.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:34.159935+0000) 2024-03-06T11:52:34.709 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:52:34.717 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:52:35.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:35.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:35.160124+0000) 2024-03-06T11:52:36.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:36.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:36.160390+0000) 2024-03-06T11:52:37.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:37.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:37.160645+0000) 2024-03-06T11:52:38.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:38.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:38.160842+0000) 2024-03-06T11:52:39.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:39.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:39.161034+0000) 2024-03-06T11:52:40.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:40.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:40.161253+0000) 2024-03-06T11:52:41.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:41.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:41.161448+0000) 2024-03-06T11:52:42.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:42.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:42.161677+0000) 2024-03-06T11:52:43.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:43.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:43.161862+0000) 2024-03-06T11:52:44.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:44.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:44.162057+0000) 2024-03-06T11:52:45.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:45.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:45.162285+0000) 2024-03-06T11:52:46.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:46.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:46.162611+0000) 2024-03-06T11:52:47.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:47.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:47.162891+0000) 2024-03-06T11:52:48.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:48.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:48.163169+0000) 2024-03-06T11:52:49.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:49.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:49.163451+0000) 2024-03-06T11:52:50.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:50.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:50.163682+0000) 2024-03-06T11:52:51.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:51.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:51.163868+0000) 2024-03-06T11:52:52.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:52.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:52.164061+0000) 2024-03-06T11:52:53.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:53.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:53.164286+0000) 2024-03-06T11:52:54.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:54.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:54.164517+0000) 2024-03-06T11:52:55.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:55.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:55.164754+0000) 2024-03-06T11:52:56.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:56.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:56.164970+0000) 2024-03-06T11:52:57.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:57.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:57.165176+0000) 2024-03-06T11:52:58.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:58.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:58.165396+0000) 2024-03-06T11:52:59.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:52:59.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:52:59.165638+0000) 2024-03-06T11:53:00.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:00.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:00.165828+0000) 2024-03-06T11:53:01.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:01.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:01.166019+0000) 2024-03-06T11:53:02.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:02.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:02.166248+0000) 2024-03-06T11:53:03.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:03.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:03.166447+0000) 2024-03-06T11:53:04.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:04.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:04.166695+0000) 2024-03-06T11:53:04.907 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:53:04.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:53:05.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:05.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:05.166923+0000) 2024-03-06T11:53:06.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:06.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:06.167161+0000) 2024-03-06T11:53:07.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:07.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:07.167407+0000) 2024-03-06T11:53:08.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:08.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:08.167624+0000) 2024-03-06T11:53:09.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:09.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:09.167815+0000) 2024-03-06T11:53:10.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:10.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:10.168001+0000) 2024-03-06T11:53:11.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:11.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:11.168192+0000) 2024-03-06T11:53:12.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:12.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:12.168387+0000) 2024-03-06T11:53:13.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:13.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:13.168641+0000) 2024-03-06T11:53:14.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:14.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:14.168830+0000) 2024-03-06T11:53:15.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:15.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:15.169012+0000) 2024-03-06T11:53:16.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:16.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:16.169227+0000) 2024-03-06T11:53:17.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:17.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:17.169426+0000) 2024-03-06T11:53:18.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:18.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:18.169682+0000) 2024-03-06T11:53:19.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:19.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:19.169868+0000) 2024-03-06T11:53:20.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:20.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:20.170045+0000) 2024-03-06T11:53:21.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:21.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:21.170268+0000) 2024-03-06T11:53:22.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:22.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:22.170446+0000) 2024-03-06T11:53:23.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:23.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:23.170684+0000) 2024-03-06T11:53:24.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:24.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:24.170856+0000) 2024-03-06T11:53:25.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:25.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:25.171078+0000) 2024-03-06T11:53:26.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:26.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:26.171310+0000) 2024-03-06T11:53:27.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:27.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:27.171499+0000) 2024-03-06T11:53:28.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:28.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:28.171659+0000) 2024-03-06T11:53:29.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:29.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:29.171938+0000) 2024-03-06T11:53:30.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:30.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:30.172166+0000) 2024-03-06T11:53:31.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:31.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:31.172409+0000) 2024-03-06T11:53:32.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:32.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:32.172681+0000) 2024-03-06T11:53:33.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:33.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:33.172890+0000) 2024-03-06T11:53:34.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:34.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:34.173085+0000) 2024-03-06T11:53:35.138 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:53:35.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:53:35.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:35.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:35.173270+0000) 2024-03-06T11:53:36.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:36.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:36.173507+0000) 2024-03-06T11:53:37.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:37.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:37.173798+0000) 2024-03-06T11:53:38.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:38.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:38.174082+0000) 2024-03-06T11:53:39.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:39.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:39.174296+0000) 2024-03-06T11:53:40.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:40.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:40.174567+0000) 2024-03-06T11:53:41.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:41.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:41.174797+0000) 2024-03-06T11:53:42.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:42.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:42.175015+0000) 2024-03-06T11:53:43.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:43.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:43.175268+0000) 2024-03-06T11:53:44.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:44.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:44.175587+0000) 2024-03-06T11:53:45.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:45.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:45.175820+0000) 2024-03-06T11:53:46.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:46.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:46.176080+0000) 2024-03-06T11:53:47.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:47.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:47.176359+0000) 2024-03-06T11:53:48.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:48.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:48.176671+0000) 2024-03-06T11:53:49.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:49.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:49.176954+0000) 2024-03-06T11:53:50.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:50.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:50.177233+0000) 2024-03-06T11:53:51.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:51.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:51.177456+0000) 2024-03-06T11:53:52.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:52.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:52.177685+0000) 2024-03-06T11:53:53.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:53.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:53.177884+0000) 2024-03-06T11:53:54.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:54.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:54.178104+0000) 2024-03-06T11:53:55.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:55.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:55.178339+0000) 2024-03-06T11:53:56.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:56.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:56.178654+0000) 2024-03-06T11:53:57.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:57.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:57.178937+0000) 2024-03-06T11:53:58.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:58.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:58.179209+0000) 2024-03-06T11:53:59.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:53:59.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:53:59.179491+0000) 2024-03-06T11:54:00.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:00.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:00.179810+0000) 2024-03-06T11:54:01.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:01.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:01.180018+0000) 2024-03-06T11:54:02.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:02.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:02.180288+0000) 2024-03-06T11:54:03.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:03.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:03.180502+0000) 2024-03-06T11:54:04.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:04.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:04.180728+0000) 2024-03-06T11:54:05.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:05.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:05.180918+0000) 2024-03-06T11:54:05.276 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:54:05.285 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:54:06.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:06.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:06.181166+0000) 2024-03-06T11:54:07.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:07.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:07.181392+0000) 2024-03-06T11:54:08.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:08.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:08.181645+0000) 2024-03-06T11:54:09.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:09.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:09.181840+0000) 2024-03-06T11:54:10.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:10.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:10.182026+0000) 2024-03-06T11:54:11.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:11.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:11.182260+0000) 2024-03-06T11:54:12.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:12.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:12.182488+0000) 2024-03-06T11:54:13.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:13.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:13.182691+0000) 2024-03-06T11:54:14.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:14.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:14.182903+0000) 2024-03-06T11:54:15.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:15.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:15.183102+0000) 2024-03-06T11:54:16.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:16.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:16.183353+0000) 2024-03-06T11:54:17.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:17.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:17.183629+0000) 2024-03-06T11:54:18.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:18.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:18.183837+0000) 2024-03-06T11:54:19.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:19.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:19.184072+0000) 2024-03-06T11:54:20.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:20.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:20.184336+0000) 2024-03-06T11:54:21.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:21.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:21.184589+0000) 2024-03-06T11:54:22.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:22.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:22.184869+0000) 2024-03-06T11:54:23.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:23.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:23.185152+0000) 2024-03-06T11:54:24.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:24.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:24.185349+0000) 2024-03-06T11:54:25.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:25.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:25.185589+0000) 2024-03-06T11:54:26.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:26.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:26.185798+0000) 2024-03-06T11:54:27.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:27.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:27.186026+0000) 2024-03-06T11:54:28.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:28.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:28.186271+0000) 2024-03-06T11:54:29.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:29.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:29.186457+0000) 2024-03-06T11:54:30.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:30.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:30.186679+0000) 2024-03-06T11:54:31.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:31.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:31.186922+0000) 2024-03-06T11:54:32.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:32.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:32.187155+0000) 2024-03-06T11:54:33.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:33.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:33.187377+0000) 2024-03-06T11:54:34.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:34.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:34.187661+0000) 2024-03-06T11:54:35.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:35.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:35.187880+0000) 2024-03-06T11:54:35.479 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:54:35.487 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:54:36.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:36.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:36.188098+0000) 2024-03-06T11:54:37.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:37.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:37.188298+0000) 2024-03-06T11:54:38.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:38.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:38.188529+0000) 2024-03-06T11:54:39.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:39.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:39.188765+0000) 2024-03-06T11:54:40.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:40.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:40.189005+0000) 2024-03-06T11:54:41.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:41.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:41.189192+0000) 2024-03-06T11:54:42.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:42.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:42.189414+0000) 2024-03-06T11:54:43.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:43.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:43.189661+0000) 2024-03-06T11:54:44.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:44.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:44.189863+0000) 2024-03-06T11:54:45.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:45.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:45.190049+0000) 2024-03-06T11:54:46.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:46.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:46.190246+0000) 2024-03-06T11:54:47.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:47.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:47.190483+0000) 2024-03-06T11:54:48.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:48.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:48.190675+0000) 2024-03-06T11:54:49.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:49.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:49.190871+0000) 2024-03-06T11:54:50.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:50.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:50.191059+0000) 2024-03-06T11:54:51.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:51.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:51.191283+0000) 2024-03-06T11:54:52.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:52.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:52.191517+0000) 2024-03-06T11:54:53.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:53.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:53.191771+0000) 2024-03-06T11:54:54.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:54.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:54.192015+0000) 2024-03-06T11:54:55.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:55.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:55.192250+0000) 2024-03-06T11:54:56.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:56.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:56.192490+0000) 2024-03-06T11:54:57.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:57.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:57.192729+0000) 2024-03-06T11:54:58.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:58.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:58.192957+0000) 2024-03-06T11:54:59.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:54:59.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:54:59.193168+0000) 2024-03-06T11:55:00.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:00.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:00.193418+0000) 2024-03-06T11:55:01.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:01.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:01.193657+0000) 2024-03-06T11:55:02.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:02.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:02.193896+0000) 2024-03-06T11:55:03.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:03.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:03.194083+0000) 2024-03-06T11:55:04.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:04.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:04.194301+0000) 2024-03-06T11:55:05.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:05.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:05.194605+0000) 2024-03-06T11:55:05.665 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:55:05.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:55:06.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:06.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:06.194820+0000) 2024-03-06T11:55:07.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:07.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:07.195021+0000) 2024-03-06T11:55:08.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:08.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:08.195260+0000) 2024-03-06T11:55:09.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:09.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:09.195574+0000) 2024-03-06T11:55:10.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:10.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:10.195848+0000) 2024-03-06T11:55:11.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:11.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:11.196136+0000) 2024-03-06T11:55:12.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:12.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:12.196416+0000) 2024-03-06T11:55:13.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:13.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:13.196704+0000) 2024-03-06T11:55:14.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:14.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:14.196971+0000) 2024-03-06T11:55:15.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:15.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:15.197204+0000) 2024-03-06T11:55:16.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:16.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:16.197474+0000) 2024-03-06T11:55:17.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:17.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:17.197738+0000) 2024-03-06T11:55:18.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:18.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:18.198016+0000) 2024-03-06T11:55:19.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:19.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:19.198291+0000) 2024-03-06T11:55:20.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:20.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:20.198527+0000) 2024-03-06T11:55:21.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:21.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:21.198823+0000) 2024-03-06T11:55:22.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:22.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:22.199102+0000) 2024-03-06T11:55:23.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:23.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:23.199295+0000) 2024-03-06T11:55:24.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:24.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:24.199527+0000) 2024-03-06T11:55:25.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:25.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:25.199754+0000) 2024-03-06T11:55:26.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:26.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:26.199952+0000) 2024-03-06T11:55:27.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:27.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:27.200181+0000) 2024-03-06T11:55:28.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:28.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:28.200368+0000) 2024-03-06T11:55:29.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:29.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:29.200618+0000) 2024-03-06T11:55:30.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:30.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:30.200826+0000) 2024-03-06T11:55:31.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:31.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:31.201033+0000) 2024-03-06T11:55:32.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:32.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:32.201259+0000) 2024-03-06T11:55:33.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:33.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:33.201491+0000) 2024-03-06T11:55:34.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:34.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:34.201742+0000) 2024-03-06T11:55:35.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:35.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:35.201933+0000) 2024-03-06T11:55:35.871 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:55:35.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:55:36.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:36.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:36.202180+0000) 2024-03-06T11:55:37.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:37.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:37.202420+0000) 2024-03-06T11:55:38.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:38.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:38.202644+0000) 2024-03-06T11:55:39.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:39.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:39.202799+0000) 2024-03-06T11:55:40.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:40.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:40.203035+0000) 2024-03-06T11:55:41.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:41.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:41.203251+0000) 2024-03-06T11:55:42.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:42.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:42.203488+0000) 2024-03-06T11:55:43.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:43.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:43.203678+0000) 2024-03-06T11:55:44.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:44.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:44.203895+0000) 2024-03-06T11:55:45.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:45.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:45.204103+0000) 2024-03-06T11:55:46.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:46.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:46.204318+0000) 2024-03-06T11:55:47.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:47.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:47.204583+0000) 2024-03-06T11:55:48.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:48.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:48.204831+0000) 2024-03-06T11:55:49.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:49.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:49.205040+0000) 2024-03-06T11:55:50.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:50.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:50.205237+0000) 2024-03-06T11:55:51.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:51.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:51.205437+0000) 2024-03-06T11:55:52.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:52.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:52.205678+0000) 2024-03-06T11:55:53.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:53.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:53.205975+0000) 2024-03-06T11:55:54.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:54.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:54.206218+0000) 2024-03-06T11:55:55.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:55.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:55.206438+0000) 2024-03-06T11:55:56.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:56.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:56.206648+0000) 2024-03-06T11:55:57.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:57.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:57.206880+0000) 2024-03-06T11:55:58.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:58.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:58.207105+0000) 2024-03-06T11:55:59.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:55:59.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:55:59.207338+0000) 2024-03-06T11:56:00.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:00.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:00.207585+0000) 2024-03-06T11:56:01.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:01.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:01.207811+0000) 2024-03-06T11:56:02.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:02.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:02.208057+0000) 2024-03-06T11:56:03.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:03.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:03.208276+0000) 2024-03-06T11:56:04.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:04.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:04.208505+0000) 2024-03-06T11:56:05.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:05.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:05.208759+0000) 2024-03-06T11:56:06.096 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:56:06.103 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:56:06.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:06.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:06.209005+0000) 2024-03-06T11:56:07.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:07.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:07.209236+0000) 2024-03-06T11:56:08.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:08.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:08.209457+0000) 2024-03-06T11:56:09.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:09.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:09.209695+0000) 2024-03-06T11:56:10.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:10.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:10.209874+0000) 2024-03-06T11:56:11.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:11.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:11.210055+0000) 2024-03-06T11:56:12.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:12.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:12.210253+0000) 2024-03-06T11:56:13.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:13.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:13.210499+0000) 2024-03-06T11:56:14.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:14.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:14.210703+0000) 2024-03-06T11:56:15.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:15.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:15.210888+0000) 2024-03-06T11:56:16.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:16.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:16.211114+0000) 2024-03-06T11:56:17.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:17.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:17.211296+0000) 2024-03-06T11:56:18.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:18.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:18.211518+0000) 2024-03-06T11:56:19.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:19.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:19.211748+0000) 2024-03-06T11:56:20.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:20.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:20.211993+0000) 2024-03-06T11:56:21.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:21.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:21.212184+0000) 2024-03-06T11:56:22.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:22.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:22.212417+0000) 2024-03-06T11:56:23.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:23.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:23.212652+0000) 2024-03-06T11:56:24.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:24.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:24.212851+0000) 2024-03-06T11:56:25.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:25.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:25.213058+0000) 2024-03-06T11:56:26.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:26.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:26.213288+0000) 2024-03-06T11:56:27.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:27.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:27.213517+0000) 2024-03-06T11:56:28.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:28.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:28.213759+0000) 2024-03-06T11:56:29.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:29.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:29.214052+0000) 2024-03-06T11:56:30.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:30.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:30.214315+0000) 2024-03-06T11:56:31.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:31.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:31.214640+0000) 2024-03-06T11:56:32.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:32.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:32.214867+0000) 2024-03-06T11:56:33.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:33.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:33.215083+0000) 2024-03-06T11:56:34.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:34.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:34.215319+0000) 2024-03-06T11:56:35.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:35.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:35.215512+0000) 2024-03-06T11:56:36.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:36.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:36.215795+0000) 2024-03-06T11:56:36.307 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:56:36.315 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:56:37.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:37.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:37.215989+0000) 2024-03-06T11:56:38.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:38.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:38.216173+0000) 2024-03-06T11:56:39.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:39.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:39.216366+0000) 2024-03-06T11:56:40.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:40.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:40.216594+0000) 2024-03-06T11:56:41.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:41.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:41.216804+0000) 2024-03-06T11:56:42.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:42.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:42.216996+0000) 2024-03-06T11:56:43.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:43.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:43.217186+0000) 2024-03-06T11:56:44.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:44.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:44.217418+0000) 2024-03-06T11:56:45.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:45.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:45.217623+0000) 2024-03-06T11:56:46.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:46.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:46.217887+0000) 2024-03-06T11:56:47.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:47.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:47.218075+0000) 2024-03-06T11:56:48.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:48.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:48.218310+0000) 2024-03-06T11:56:49.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:49.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:49.218560+0000) 2024-03-06T11:56:50.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:50.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:50.218792+0000) 2024-03-06T11:56:51.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:51.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:51.219017+0000) 2024-03-06T11:56:52.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:52.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:52.219203+0000) 2024-03-06T11:56:53.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:53.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:53.219469+0000) 2024-03-06T11:56:54.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:54.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:54.219700+0000) 2024-03-06T11:56:55.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:55.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:55.219935+0000) 2024-03-06T11:56:56.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:56.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:56.220124+0000) 2024-03-06T11:56:57.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:57.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:57.220350+0000) 2024-03-06T11:56:58.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:58.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:58.220607+0000) 2024-03-06T11:56:59.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:56:59.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:56:59.220803+0000) 2024-03-06T11:57:00.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:00.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:00.221008+0000) 2024-03-06T11:57:01.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:01.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:01.221193+0000) 2024-03-06T11:57:02.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:02.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:02.221382+0000) 2024-03-06T11:57:03.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:03.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:03.221643+0000) 2024-03-06T11:57:04.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:04.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:04.221864+0000) 2024-03-06T11:57:05.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:05.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:05.222043+0000) 2024-03-06T11:57:06.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:06.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:06.222274+0000) 2024-03-06T11:57:06.515 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:57:06.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:57:07.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:07.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:07.222498+0000) 2024-03-06T11:57:08.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:08.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:08.222686+0000) 2024-03-06T11:57:09.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:09.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:09.222874+0000) 2024-03-06T11:57:10.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:10.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:10.223060+0000) 2024-03-06T11:57:11.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:11.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:11.223286+0000) 2024-03-06T11:57:12.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:12.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:12.223463+0000) 2024-03-06T11:57:13.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:13.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:13.223651+0000) 2024-03-06T11:57:14.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:14.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:14.223871+0000) 2024-03-06T11:57:15.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:15.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:15.224088+0000) 2024-03-06T11:57:16.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:16.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:16.224335+0000) 2024-03-06T11:57:17.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:17.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:17.224577+0000) 2024-03-06T11:57:18.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:18.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:18.224769+0000) 2024-03-06T11:57:19.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:19.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:19.224994+0000) 2024-03-06T11:57:20.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:20.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:20.225224+0000) 2024-03-06T11:57:21.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:21.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:21.225487+0000) 2024-03-06T11:57:22.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:22.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:22.225729+0000) 2024-03-06T11:57:23.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:23.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:23.225964+0000) 2024-03-06T11:57:24.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:24.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:24.226205+0000) 2024-03-06T11:57:25.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:25.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:25.226491+0000) 2024-03-06T11:57:26.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:26.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:26.226751+0000) 2024-03-06T11:57:27.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:27.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:27.226980+0000) 2024-03-06T11:57:28.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:28.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:28.227240+0000) 2024-03-06T11:57:29.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:29.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:29.227445+0000) 2024-03-06T11:57:30.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:30.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:30.227650+0000) 2024-03-06T11:57:31.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:31.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:31.227885+0000) 2024-03-06T11:57:32.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:32.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:32.228087+0000) 2024-03-06T11:57:33.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:33.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:33.228316+0000) 2024-03-06T11:57:34.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:34.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:34.228511+0000) 2024-03-06T11:57:35.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:35.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:35.228782+0000) 2024-03-06T11:57:36.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:36.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:36.229016+0000) 2024-03-06T11:57:36.713 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:57:36.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:57:37.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:37.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:37.229246+0000) 2024-03-06T11:57:38.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:38.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:38.229468+0000) 2024-03-06T11:57:39.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:39.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:39.229709+0000) 2024-03-06T11:57:40.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:40.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:40.230005+0000) 2024-03-06T11:57:41.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:41.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:41.230221+0000) 2024-03-06T11:57:42.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:42.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:42.230423+0000) 2024-03-06T11:57:43.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:43.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:43.230647+0000) 2024-03-06T11:57:44.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:44.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:44.230870+0000) 2024-03-06T11:57:45.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:45.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:45.231091+0000) 2024-03-06T11:57:46.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:46.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:46.231380+0000) 2024-03-06T11:57:47.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:47.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:47.231646+0000) 2024-03-06T11:57:48.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:48.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:48.231873+0000) 2024-03-06T11:57:49.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:49.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:49.232063+0000) 2024-03-06T11:57:50.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:50.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:50.232322+0000) 2024-03-06T11:57:51.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:51.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:51.232563+0000) 2024-03-06T11:57:52.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:52.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:52.232805+0000) 2024-03-06T11:57:53.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:53.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:53.233037+0000) 2024-03-06T11:57:54.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:54.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:54.233303+0000) 2024-03-06T11:57:55.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:55.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:55.233578+0000) 2024-03-06T11:57:56.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:56.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:56.233861+0000) 2024-03-06T11:57:57.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:57.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:57.234111+0000) 2024-03-06T11:57:58.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:58.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:58.234368+0000) 2024-03-06T11:57:59.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:57:59.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:57:59.234670+0000) 2024-03-06T11:58:00.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:00.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:00.234950+0000) 2024-03-06T11:58:01.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:01.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:01.235193+0000) 2024-03-06T11:58:02.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:02.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:02.235461+0000) 2024-03-06T11:58:03.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:03.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:03.235724+0000) 2024-03-06T11:58:04.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:04.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:04.235966+0000) 2024-03-06T11:58:05.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:05.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:05.236207+0000) 2024-03-06T11:58:06.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:06.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:06.236478+0000) 2024-03-06T11:58:06.851 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:58:06.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:58:07.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:07.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:07.236692+0000) 2024-03-06T11:58:08.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:08.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:08.236886+0000) 2024-03-06T11:58:09.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:09.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:09.237073+0000) 2024-03-06T11:58:10.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:10.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:10.237261+0000) 2024-03-06T11:58:11.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:11.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:11.237492+0000) 2024-03-06T11:58:12.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:12.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:12.237735+0000) 2024-03-06T11:58:13.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:13.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:13.237980+0000) 2024-03-06T11:58:14.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:14.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:14.238204+0000) 2024-03-06T11:58:15.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:15.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:15.238399+0000) 2024-03-06T11:58:16.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:16.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:16.238659+0000) 2024-03-06T11:58:17.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:17.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:17.238908+0000) 2024-03-06T11:58:18.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:18.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:18.239149+0000) 2024-03-06T11:58:19.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:19.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:19.239377+0000) 2024-03-06T11:58:20.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:20.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:20.239602+0000) 2024-03-06T11:58:21.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:21.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:21.239829+0000) 2024-03-06T11:58:22.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:22.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:22.240079+0000) 2024-03-06T11:58:23.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:23.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:23.240272+0000) 2024-03-06T11:58:24.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:24.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:24.240514+0000) 2024-03-06T11:58:25.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:25.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:25.240777+0000) 2024-03-06T11:58:26.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:26.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:26.241005+0000) 2024-03-06T11:58:27.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:27.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:27.241239+0000) 2024-03-06T11:58:28.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:28.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:28.241504+0000) 2024-03-06T11:58:29.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:29.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:29.241752+0000) 2024-03-06T11:58:30.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:30.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:30.241947+0000) 2024-03-06T11:58:31.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:31.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:31.242173+0000) 2024-03-06T11:58:32.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:32.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:32.242442+0000) 2024-03-06T11:58:33.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:33.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:33.242732+0000) 2024-03-06T11:58:34.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:34.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:34.242972+0000) 2024-03-06T11:58:35.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:35.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:35.243236+0000) 2024-03-06T11:58:36.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:36.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:36.243567+0000) 2024-03-06T11:58:37.068 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:58:37.076 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:58:37.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:37.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:37.243821+0000) 2024-03-06T11:58:38.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:38.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:38.244065+0000) 2024-03-06T11:58:39.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:39.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:39.244296+0000) 2024-03-06T11:58:40.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:40.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:40.244599+0000) 2024-03-06T11:58:41.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:41.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:41.244823+0000) 2024-03-06T11:58:42.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:42.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:42.245067+0000) 2024-03-06T11:58:43.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:43.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:43.245327+0000) 2024-03-06T11:58:44.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:44.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:44.245639+0000) 2024-03-06T11:58:45.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:45.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:45.245881+0000) 2024-03-06T11:58:46.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:46.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:46.246115+0000) 2024-03-06T11:58:47.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:47.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:47.246361+0000) 2024-03-06T11:58:48.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:48.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:48.246583+0000) 2024-03-06T11:58:49.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:49.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:49.246839+0000) 2024-03-06T11:58:50.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:50.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:50.247062+0000) 2024-03-06T11:58:51.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:51.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:51.247290+0000) 2024-03-06T11:58:52.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:52.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:52.247525+0000) 2024-03-06T11:58:53.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:53.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:53.247759+0000) 2024-03-06T11:58:54.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:54.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:54.247958+0000) 2024-03-06T11:58:55.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:55.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:55.248161+0000) 2024-03-06T11:58:56.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:56.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:56.248402+0000) 2024-03-06T11:58:57.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:57.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:57.248664+0000) 2024-03-06T11:58:58.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:58.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:58.248858+0000) 2024-03-06T11:58:59.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:58:59.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:58:59.249097+0000) 2024-03-06T11:59:00.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:00.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:00.249338+0000) 2024-03-06T11:59:01.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:01.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:01.249613+0000) 2024-03-06T11:59:02.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:02.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:02.249819+0000) 2024-03-06T11:59:03.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:03.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:03.250019+0000) 2024-03-06T11:59:04.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:04.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:04.250258+0000) 2024-03-06T11:59:05.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:05.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:05.250504+0000) 2024-03-06T11:59:06.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:06.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:06.250784+0000) 2024-03-06T11:59:07.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:07.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:07.250993+0000) 2024-03-06T11:59:07.272 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:59:07.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:59:08.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:08.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:08.251183+0000) 2024-03-06T11:59:09.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:09.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:09.251415+0000) 2024-03-06T11:59:10.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:10.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:10.251655+0000) 2024-03-06T11:59:11.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:11.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:11.251850+0000) 2024-03-06T11:59:12.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:12.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:12.252087+0000) 2024-03-06T11:59:13.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:13.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:13.252313+0000) 2024-03-06T11:59:14.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:14.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:14.252572+0000) 2024-03-06T11:59:15.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:15.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:15.252791+0000) 2024-03-06T11:59:16.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:16.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:16.253095+0000) 2024-03-06T11:59:17.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:17.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:17.253332+0000) 2024-03-06T11:59:18.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:18.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:18.253582+0000) 2024-03-06T11:59:19.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:19.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:19.253804+0000) 2024-03-06T11:59:20.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:20.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:20.254028+0000) 2024-03-06T11:59:21.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:21.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:21.254250+0000) 2024-03-06T11:59:22.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:22.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:22.254475+0000) 2024-03-06T11:59:23.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:23.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:23.254688+0000) 2024-03-06T11:59:24.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:24.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:24.254884+0000) 2024-03-06T11:59:25.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:25.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:25.255114+0000) 2024-03-06T11:59:26.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:26.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:26.255346+0000) 2024-03-06T11:59:27.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:27.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:27.255620+0000) 2024-03-06T11:59:28.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:28.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:28.255782+0000) 2024-03-06T11:59:29.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:29.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:29.255952+0000) 2024-03-06T11:59:30.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:30.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:30.256150+0000) 2024-03-06T11:59:31.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:31.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:31.256380+0000) 2024-03-06T11:59:32.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:32.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:32.256641+0000) 2024-03-06T11:59:33.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:33.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:33.256848+0000) 2024-03-06T11:59:34.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:34.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:34.257054+0000) 2024-03-06T11:59:35.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:35.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:35.257287+0000) 2024-03-06T11:59:36.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:36.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:36.257571+0000) 2024-03-06T11:59:37.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:37.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:37.257800+0000) 2024-03-06T11:59:37.401 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:59:37.408 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T11:59:38.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:38.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:38.258008+0000) 2024-03-06T11:59:39.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:39.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:39.258253+0000) 2024-03-06T11:59:40.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:40.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:40.258425+0000) 2024-03-06T11:59:41.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:41.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:41.258620+0000) 2024-03-06T11:59:42.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:42.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:42.258818+0000) 2024-03-06T11:59:43.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:43.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:43.259047+0000) 2024-03-06T11:59:44.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:44.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:44.259271+0000) 2024-03-06T11:59:45.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:45.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:45.259469+0000) 2024-03-06T11:59:46.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:46.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:46.259695+0000) 2024-03-06T11:59:47.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:47.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:47.259931+0000) 2024-03-06T11:59:48.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:48.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:48.260128+0000) 2024-03-06T11:59:49.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:49.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:49.260367+0000) 2024-03-06T11:59:50.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:50.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:50.260612+0000) 2024-03-06T11:59:51.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:51.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:51.260778+0000) 2024-03-06T11:59:52.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:52.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:52.261029+0000) 2024-03-06T11:59:53.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:53.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:53.261239+0000) 2024-03-06T11:59:54.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:54.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:54.261425+0000) 2024-03-06T11:59:55.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:55.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:55.261660+0000) 2024-03-06T11:59:56.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:56.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:56.261856+0000) 2024-03-06T11:59:57.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:57.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:57.262028+0000) 2024-03-06T11:59:58.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:58.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:58.262247+0000) 2024-03-06T11:59:59.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T11:59:59.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T10:59:59.262526+0000) 2024-03-06T12:00:00.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:00.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:00.262828+0000) 2024-03-06T12:00:01.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:01.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:01.263049+0000) 2024-03-06T12:00:02.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:02.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:02.263290+0000) 2024-03-06T12:00:03.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:03.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:03.263508+0000) 2024-03-06T12:00:04.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:04.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:04.263781+0000) 2024-03-06T12:00:05.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:05.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:05.264001+0000) 2024-03-06T12:00:06.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:06.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:06.264255+0000) 2024-03-06T12:00:07.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:07.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:07.264464+0000) 2024-03-06T12:00:07.630 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:00:07.638 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:00:08.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:08.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:08.264691+0000) 2024-03-06T12:00:09.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:09.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:09.264884+0000) 2024-03-06T12:00:10.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:10.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:10.265068+0000) 2024-03-06T12:00:11.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:11.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:11.265254+0000) 2024-03-06T12:00:12.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:12.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:12.265417+0000) 2024-03-06T12:00:13.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:13.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:13.265714+0000) 2024-03-06T12:00:14.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:14.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:14.265942+0000) 2024-03-06T12:00:15.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:15.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:15.266115+0000) 2024-03-06T12:00:16.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:16.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:16.266353+0000) 2024-03-06T12:00:17.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:17.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:17.266615+0000) 2024-03-06T12:00:18.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:18.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:18.266807+0000) 2024-03-06T12:00:19.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:19.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:19.267000+0000) 2024-03-06T12:00:20.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:20.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:20.267197+0000) 2024-03-06T12:00:21.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:21.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:21.267434+0000) 2024-03-06T12:00:22.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:22.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:22.267692+0000) 2024-03-06T12:00:23.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:23.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:23.267929+0000) 2024-03-06T12:00:24.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:24.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:24.268203+0000) 2024-03-06T12:00:25.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:25.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:25.268436+0000) 2024-03-06T12:00:26.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:26.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:26.268722+0000) 2024-03-06T12:00:27.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:27.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:27.268952+0000) 2024-03-06T12:00:28.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:28.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:28.269175+0000) 2024-03-06T12:00:29.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:29.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:29.269405+0000) 2024-03-06T12:00:30.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:30.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:30.269701+0000) 2024-03-06T12:00:31.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:31.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:31.269919+0000) 2024-03-06T12:00:32.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:32.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:32.270156+0000) 2024-03-06T12:00:33.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:33.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:33.270403+0000) 2024-03-06T12:00:34.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:34.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:34.270661+0000) 2024-03-06T12:00:35.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:35.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:35.270872+0000) 2024-03-06T12:00:36.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:36.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:36.271122+0000) 2024-03-06T12:00:37.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:37.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:37.271458+0000) 2024-03-06T12:00:37.821 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:00:37.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:00:38.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:38.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:38.271703+0000) 2024-03-06T12:00:39.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:39.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:39.271932+0000) 2024-03-06T12:00:40.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:40.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:40.272134+0000) 2024-03-06T12:00:41.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:41.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:41.272378+0000) 2024-03-06T12:00:42.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:42.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:42.272622+0000) 2024-03-06T12:00:43.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:43.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:43.272789+0000) 2024-03-06T12:00:44.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:44.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:44.273019+0000) 2024-03-06T12:00:45.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:45.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:45.273210+0000) 2024-03-06T12:00:46.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:46.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:46.273460+0000) 2024-03-06T12:00:47.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:47.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:47.273694+0000) 2024-03-06T12:00:48.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:48.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:48.273889+0000) 2024-03-06T12:00:49.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:49.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:49.274079+0000) 2024-03-06T12:00:50.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:50.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:50.274285+0000) 2024-03-06T12:00:51.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:51.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:51.274502+0000) 2024-03-06T12:00:52.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:52.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:52.274761+0000) 2024-03-06T12:00:53.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:53.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:53.274997+0000) 2024-03-06T12:00:54.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:54.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:54.275224+0000) 2024-03-06T12:00:55.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:55.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:55.275446+0000) 2024-03-06T12:00:56.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:56.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:56.275687+0000) 2024-03-06T12:00:57.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:57.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:57.276004+0000) 2024-03-06T12:00:58.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:58.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:58.276249+0000) 2024-03-06T12:00:59.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:00:59.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:00:59.276473+0000) 2024-03-06T12:01:00.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:00.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:00.276677+0000) 2024-03-06T12:01:01.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:01.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:01.276922+0000) 2024-03-06T12:01:02.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:02.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:02.277128+0000) 2024-03-06T12:01:03.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:03.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:03.277326+0000) 2024-03-06T12:01:04.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:04.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:04.277601+0000) 2024-03-06T12:01:05.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:05.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:05.277798+0000) 2024-03-06T12:01:06.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:06.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:06.278053+0000) 2024-03-06T12:01:07.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:07.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:07.278286+0000) 2024-03-06T12:01:08.057 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:01:08.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:01:08.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:08.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:08.278527+0000) 2024-03-06T12:01:09.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:09.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:09.278776+0000) 2024-03-06T12:01:10.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:10.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:10.278977+0000) 2024-03-06T12:01:11.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:11.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:11.279223+0000) 2024-03-06T12:01:12.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:12.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:12.279461+0000) 2024-03-06T12:01:13.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:13.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:13.279681+0000) 2024-03-06T12:01:14.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:14.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:14.279921+0000) 2024-03-06T12:01:15.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:15.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:15.280103+0000) 2024-03-06T12:01:16.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:16.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:16.280379+0000) 2024-03-06T12:01:17.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:17.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:17.280617+0000) 2024-03-06T12:01:18.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:18.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:18.280813+0000) 2024-03-06T12:01:19.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:19.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:19.280987+0000) 2024-03-06T12:01:20.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:20.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:20.281201+0000) 2024-03-06T12:01:21.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:21.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:21.281484+0000) 2024-03-06T12:01:22.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:22.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:22.281728+0000) 2024-03-06T12:01:23.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:23.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:23.282060+0000) 2024-03-06T12:01:24.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:24.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:24.282308+0000) 2024-03-06T12:01:25.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:25.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:25.282569+0000) 2024-03-06T12:01:26.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:26.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:26.282826+0000) 2024-03-06T12:01:27.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:27.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:27.283014+0000) 2024-03-06T12:01:28.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:28.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:28.283233+0000) 2024-03-06T12:01:29.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:29.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:29.283440+0000) 2024-03-06T12:01:30.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:30.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:30.283672+0000) 2024-03-06T12:01:31.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:31.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:31.283874+0000) 2024-03-06T12:01:32.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:32.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:32.284064+0000) 2024-03-06T12:01:33.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:33.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:33.284302+0000) 2024-03-06T12:01:34.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:34.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:34.284529+0000) 2024-03-06T12:01:35.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:35.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:35.284739+0000) 2024-03-06T12:01:36.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:36.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:36.284931+0000) 2024-03-06T12:01:37.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:37.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:37.285149+0000) 2024-03-06T12:01:38.262 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:01:38.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:01:38.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:38.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:38.285307+0000) 2024-03-06T12:01:39.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:39.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:39.285481+0000) 2024-03-06T12:01:40.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:40.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:40.285696+0000) 2024-03-06T12:01:41.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:41.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:41.285887+0000) 2024-03-06T12:01:42.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:42.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:42.286084+0000) 2024-03-06T12:01:43.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:43.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:43.286318+0000) 2024-03-06T12:01:44.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:44.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:44.286512+0000) 2024-03-06T12:01:45.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:45.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:45.286728+0000) 2024-03-06T12:01:46.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:46.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:46.286961+0000) 2024-03-06T12:01:47.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:47.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:47.287186+0000) 2024-03-06T12:01:48.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:48.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:48.287414+0000) 2024-03-06T12:01:49.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:49.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:49.287641+0000) 2024-03-06T12:01:50.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:50.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:50.287818+0000) 2024-03-06T12:01:51.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:51.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:51.287996+0000) 2024-03-06T12:01:52.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:52.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:52.288169+0000) 2024-03-06T12:01:53.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:53.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:53.288366+0000) 2024-03-06T12:01:54.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:54.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:54.288591+0000) 2024-03-06T12:01:55.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:55.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:55.288862+0000) 2024-03-06T12:01:56.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:56.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:56.289078+0000) 2024-03-06T12:01:57.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:57.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:57.289307+0000) 2024-03-06T12:01:58.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:58.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:58.289616+0000) 2024-03-06T12:01:59.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:01:59.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:01:59.289860+0000) 2024-03-06T12:02:00.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:00.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:00.290089+0000) 2024-03-06T12:02:01.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:01.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:01.290371+0000) 2024-03-06T12:02:02.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:02.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:02.290681+0000) 2024-03-06T12:02:03.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:03.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:03.290931+0000) 2024-03-06T12:02:04.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:04.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:04.291173+0000) 2024-03-06T12:02:05.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:05.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:05.291403+0000) 2024-03-06T12:02:06.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:06.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:06.291641+0000) 2024-03-06T12:02:07.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:07.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:07.291827+0000) 2024-03-06T12:02:08.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:08.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:08.292029+0000) 2024-03-06T12:02:08.415 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:02:08.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:02:09.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:09.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:09.292309+0000) 2024-03-06T12:02:10.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:10.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:10.292610+0000) 2024-03-06T12:02:11.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:11.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:11.292840+0000) 2024-03-06T12:02:12.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:12.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:12.293069+0000) 2024-03-06T12:02:13.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:13.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:13.293362+0000) 2024-03-06T12:02:14.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:14.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:14.293670+0000) 2024-03-06T12:02:15.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:15.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:15.293899+0000) 2024-03-06T12:02:16.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:16.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:16.294205+0000) 2024-03-06T12:02:17.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:17.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:17.294479+0000) 2024-03-06T12:02:18.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:18.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:18.294721+0000) 2024-03-06T12:02:19.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:19.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:19.294963+0000) 2024-03-06T12:02:20.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:20.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:20.295158+0000) 2024-03-06T12:02:21.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:21.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:21.295388+0000) 2024-03-06T12:02:22.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:22.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:22.295597+0000) 2024-03-06T12:02:23.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:23.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:23.295861+0000) 2024-03-06T12:02:24.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:24.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:24.296137+0000) 2024-03-06T12:02:25.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:25.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:25.296409+0000) 2024-03-06T12:02:26.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:26.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:26.296693+0000) 2024-03-06T12:02:27.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:27.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:27.296886+0000) 2024-03-06T12:02:28.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:28.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:28.297136+0000) 2024-03-06T12:02:29.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:29.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:29.297365+0000) 2024-03-06T12:02:30.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:30.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:30.297630+0000) 2024-03-06T12:02:31.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:31.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:31.297859+0000) 2024-03-06T12:02:32.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:32.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:32.298087+0000) 2024-03-06T12:02:33.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:33.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:33.298282+0000) 2024-03-06T12:02:34.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:34.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:34.298504+0000) 2024-03-06T12:02:35.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:35.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:35.298727+0000) 2024-03-06T12:02:36.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:36.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:36.298912+0000) 2024-03-06T12:02:37.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:37.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:37.299132+0000) 2024-03-06T12:02:38.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:38.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:38.299402+0000) 2024-03-06T12:02:38.631 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:02:38.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:02:39.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:39.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:39.299683+0000) 2024-03-06T12:02:40.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:40.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:40.299960+0000) 2024-03-06T12:02:41.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:41.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:41.300170+0000) 2024-03-06T12:02:42.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:42.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:42.300441+0000) 2024-03-06T12:02:43.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:43.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:43.300712+0000) 2024-03-06T12:02:44.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:44.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:44.300976+0000) 2024-03-06T12:02:45.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:45.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:45.301181+0000) 2024-03-06T12:02:46.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:46.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:46.301400+0000) 2024-03-06T12:02:47.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:47.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:47.301675+0000) 2024-03-06T12:02:48.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:48.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:48.301936+0000) 2024-03-06T12:02:49.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:49.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:49.302211+0000) 2024-03-06T12:02:50.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:50.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:50.302492+0000) 2024-03-06T12:02:51.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:51.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:51.302730+0000) 2024-03-06T12:02:52.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:52.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:52.302946+0000) 2024-03-06T12:02:53.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:53.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:53.303151+0000) 2024-03-06T12:02:54.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:54.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:54.303437+0000) 2024-03-06T12:02:55.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:55.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:55.303645+0000) 2024-03-06T12:02:56.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:56.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:56.303874+0000) 2024-03-06T12:02:57.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:57.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:57.304094+0000) 2024-03-06T12:02:58.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:58.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:58.304296+0000) 2024-03-06T12:02:59.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:02:59.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:02:59.304494+0000) 2024-03-06T12:03:00.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:00.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:00.304745+0000) 2024-03-06T12:03:01.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:01.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:01.305023+0000) 2024-03-06T12:03:02.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:02.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:02.305205+0000) 2024-03-06T12:03:03.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:03.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:03.305416+0000) 2024-03-06T12:03:04.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:04.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:04.305692+0000) 2024-03-06T12:03:05.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:05.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:05.305956+0000) 2024-03-06T12:03:06.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:06.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:06.306246+0000) 2024-03-06T12:03:07.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:07.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:07.306522+0000) 2024-03-06T12:03:08.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:08.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:08.306836+0000) 2024-03-06T12:03:08.844 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:03:08.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:03:09.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:09.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:09.307106+0000) 2024-03-06T12:03:10.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:10.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:10.307342+0000) 2024-03-06T12:03:11.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:11.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:11.307564+0000) 2024-03-06T12:03:12.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:12.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:12.307761+0000) 2024-03-06T12:03:13.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:13.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:13.307999+0000) 2024-03-06T12:03:14.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:14.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:14.308205+0000) 2024-03-06T12:03:15.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:15.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:15.308430+0000) 2024-03-06T12:03:16.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:16.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:16.308661+0000) 2024-03-06T12:03:17.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:17.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:17.308883+0000) 2024-03-06T12:03:18.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:18.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:18.309104+0000) 2024-03-06T12:03:19.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:19.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:19.309324+0000) 2024-03-06T12:03:20.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:20.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:20.309573+0000) 2024-03-06T12:03:21.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:21.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:21.309806+0000) 2024-03-06T12:03:22.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:22.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:22.310019+0000) 2024-03-06T12:03:23.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:23.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:23.310252+0000) 2024-03-06T12:03:24.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:24.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:24.310519+0000) 2024-03-06T12:03:25.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:25.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:25.310816+0000) 2024-03-06T12:03:26.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:26.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:26.311144+0000) 2024-03-06T12:03:27.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:27.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:27.311365+0000) 2024-03-06T12:03:28.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:28.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:28.311643+0000) 2024-03-06T12:03:29.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:29.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:29.311854+0000) 2024-03-06T12:03:30.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:30.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:30.312041+0000) 2024-03-06T12:03:31.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:31.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:31.312262+0000) 2024-03-06T12:03:32.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:32.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:32.312466+0000) 2024-03-06T12:03:33.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:33.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:33.312719+0000) 2024-03-06T12:03:34.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:34.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:34.312942+0000) 2024-03-06T12:03:35.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:35.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:35.313135+0000) 2024-03-06T12:03:36.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:36.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:36.313380+0000) 2024-03-06T12:03:37.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:37.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:37.313585+0000) 2024-03-06T12:03:38.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:38.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:38.313788+0000) 2024-03-06T12:03:39.039 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:03:39.051 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:03:39.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:39.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:39.314042+0000) 2024-03-06T12:03:40.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:40.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:40.314214+0000) 2024-03-06T12:03:41.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:41.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:41.314432+0000) 2024-03-06T12:03:42.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:42.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:42.314698+0000) 2024-03-06T12:03:43.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:43.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:43.314995+0000) 2024-03-06T12:03:44.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:44.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:44.315222+0000) 2024-03-06T12:03:45.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:45.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:45.315444+0000) 2024-03-06T12:03:46.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:46.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:46.315719+0000) 2024-03-06T12:03:47.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:47.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:47.315916+0000) 2024-03-06T12:03:48.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:48.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:48.316146+0000) 2024-03-06T12:03:49.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:49.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:49.316350+0000) 2024-03-06T12:03:50.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:50.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:50.316636+0000) 2024-03-06T12:03:51.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:51.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:51.316905+0000) 2024-03-06T12:03:52.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:52.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:52.317172+0000) 2024-03-06T12:03:53.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:53.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:53.317440+0000) 2024-03-06T12:03:54.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:54.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:54.317725+0000) 2024-03-06T12:03:55.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:55.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:55.317942+0000) 2024-03-06T12:03:56.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:56.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:56.318151+0000) 2024-03-06T12:03:57.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:57.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:57.318411+0000) 2024-03-06T12:03:58.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:58.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:58.318683+0000) 2024-03-06T12:03:59.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:03:59.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:03:59.318964+0000) 2024-03-06T12:04:00.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:00.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:00.319165+0000) 2024-03-06T12:04:01.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:01.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:01.319423+0000) 2024-03-06T12:04:02.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:02.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:02.319701+0000) 2024-03-06T12:04:03.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:03.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:03.319981+0000) 2024-03-06T12:04:04.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:04.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:04.320253+0000) 2024-03-06T12:04:05.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:05.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:05.320457+0000) 2024-03-06T12:04:06.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:06.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:06.320740+0000) 2024-03-06T12:04:07.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:07.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:07.321014+0000) 2024-03-06T12:04:08.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:08.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:08.321226+0000) 2024-03-06T12:04:09.263 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:04:09.272 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:04:09.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:09.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:09.321418+0000) 2024-03-06T12:04:10.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:10.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:10.321638+0000) 2024-03-06T12:04:11.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:11.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:11.321860+0000) 2024-03-06T12:04:12.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:12.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:12.322038+0000) 2024-03-06T12:04:13.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:13.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:13.322269+0000) 2024-03-06T12:04:14.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:14.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:14.322573+0000) 2024-03-06T12:04:15.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:15.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:15.322805+0000) 2024-03-06T12:04:16.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:16.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:16.323027+0000) 2024-03-06T12:04:17.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:17.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:17.323208+0000) 2024-03-06T12:04:18.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:18.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:18.323415+0000) 2024-03-06T12:04:19.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:19.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:19.323666+0000) 2024-03-06T12:04:20.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:20.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:20.323908+0000) 2024-03-06T12:04:21.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:21.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:21.324155+0000) 2024-03-06T12:04:22.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:22.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:22.324347+0000) 2024-03-06T12:04:23.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:23.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:23.324618+0000) 2024-03-06T12:04:24.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:24.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:24.324801+0000) 2024-03-06T12:04:25.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:25.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:25.325037+0000) 2024-03-06T12:04:26.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:26.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:26.325268+0000) 2024-03-06T12:04:27.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:27.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:27.325565+0000) 2024-03-06T12:04:28.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:28.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:28.325767+0000) 2024-03-06T12:04:29.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:29.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:29.325972+0000) 2024-03-06T12:04:30.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:30.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:30.326148+0000) 2024-03-06T12:04:31.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:31.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:31.326359+0000) 2024-03-06T12:04:32.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:32.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:32.326599+0000) 2024-03-06T12:04:33.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:33.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:33.326858+0000) 2024-03-06T12:04:34.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:34.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:34.327040+0000) 2024-03-06T12:04:35.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:35.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:35.327263+0000) 2024-03-06T12:04:36.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:36.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:36.327456+0000) 2024-03-06T12:04:37.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:37.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:37.327697+0000) 2024-03-06T12:04:38.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:38.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:38.327924+0000) 2024-03-06T12:04:39.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:39.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:39.328149+0000) 2024-03-06T12:04:39.465 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:04:39.474 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:04:40.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:40.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:40.328374+0000) 2024-03-06T12:04:41.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:41.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:41.328639+0000) 2024-03-06T12:04:42.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:42.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:42.328862+0000) 2024-03-06T12:04:43.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:43.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:43.329045+0000) 2024-03-06T12:04:44.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:44.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:44.329274+0000) 2024-03-06T12:04:45.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:45.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:45.329495+0000) 2024-03-06T12:04:46.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:46.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:46.329696+0000) 2024-03-06T12:04:47.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:47.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:47.329931+0000) 2024-03-06T12:04:48.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:48.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:48.330162+0000) 2024-03-06T12:04:49.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:49.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:49.330387+0000) 2024-03-06T12:04:50.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:50.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:50.330595+0000) 2024-03-06T12:04:51.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:51.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:51.330781+0000) 2024-03-06T12:04:52.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:52.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:52.330968+0000) 2024-03-06T12:04:53.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:53.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:53.331177+0000) 2024-03-06T12:04:54.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:54.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:54.331422+0000) 2024-03-06T12:04:55.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:55.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:55.331646+0000) 2024-03-06T12:04:56.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:56.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:56.331873+0000) 2024-03-06T12:04:57.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:57.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:57.332091+0000) 2024-03-06T12:04:58.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:58.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:58.332321+0000) 2024-03-06T12:04:59.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:04:59.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:04:59.332590+0000) 2024-03-06T12:05:00.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:00.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:00.332821+0000) 2024-03-06T12:05:01.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:01.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:01.333047+0000) 2024-03-06T12:05:02.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:02.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:02.333244+0000) 2024-03-06T12:05:03.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:03.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:03.333458+0000) 2024-03-06T12:05:04.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:04.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:04.333694+0000) 2024-03-06T12:05:05.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:05.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:05.333870+0000) 2024-03-06T12:05:06.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:06.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:06.334105+0000) 2024-03-06T12:05:07.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:07.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:07.334327+0000) 2024-03-06T12:05:08.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:08.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:08.334649+0000) 2024-03-06T12:05:09.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:09.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:09.334878+0000) 2024-03-06T12:05:09.608 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:05:09.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:05:10.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:10.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:10.335106+0000) 2024-03-06T12:05:11.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:11.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:11.335325+0000) 2024-03-06T12:05:12.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:12.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:12.335503+0000) 2024-03-06T12:05:13.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:13.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:13.335731+0000) 2024-03-06T12:05:14.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:14.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:14.335906+0000) 2024-03-06T12:05:15.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:15.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:15.336095+0000) 2024-03-06T12:05:16.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:16.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:16.336334+0000) 2024-03-06T12:05:17.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:17.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:17.336609+0000) 2024-03-06T12:05:18.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:18.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:18.336811+0000) 2024-03-06T12:05:19.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:19.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:19.337011+0000) 2024-03-06T12:05:20.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:20.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:20.337212+0000) 2024-03-06T12:05:21.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:21.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:21.337439+0000) 2024-03-06T12:05:22.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:22.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:22.337688+0000) 2024-03-06T12:05:23.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:23.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:23.337917+0000) 2024-03-06T12:05:24.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:24.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:24.338147+0000) 2024-03-06T12:05:25.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:25.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:25.338365+0000) 2024-03-06T12:05:26.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:26.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:26.338635+0000) 2024-03-06T12:05:27.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:27.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:27.338863+0000) 2024-03-06T12:05:28.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:28.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:28.339103+0000) 2024-03-06T12:05:29.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:29.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:29.339318+0000) 2024-03-06T12:05:30.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:30.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:30.339585+0000) 2024-03-06T12:05:31.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:31.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:31.339812+0000) 2024-03-06T12:05:32.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:32.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:32.340011+0000) 2024-03-06T12:05:33.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:33.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:33.340260+0000) 2024-03-06T12:05:34.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:34.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:34.340484+0000) 2024-03-06T12:05:35.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:35.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:35.340716+0000) 2024-03-06T12:05:36.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:36.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:36.340972+0000) 2024-03-06T12:05:37.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:37.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:37.341210+0000) 2024-03-06T12:05:38.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:38.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:38.341461+0000) 2024-03-06T12:05:39.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:39.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:39.341702+0000) 2024-03-06T12:05:39.832 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:05:39.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:05:40.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:40.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:40.341890+0000) 2024-03-06T12:05:41.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:41.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:41.342131+0000) 2024-03-06T12:05:42.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:42.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:42.342365+0000) 2024-03-06T12:05:43.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:43.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:43.342621+0000) 2024-03-06T12:05:44.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:44.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:44.342835+0000) 2024-03-06T12:05:45.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:45.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:45.343083+0000) 2024-03-06T12:05:46.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:46.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:46.343316+0000) 2024-03-06T12:05:47.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:47.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:47.343569+0000) 2024-03-06T12:05:48.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:48.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:48.343816+0000) 2024-03-06T12:05:49.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:49.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:49.344008+0000) 2024-03-06T12:05:50.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:50.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:50.344232+0000) 2024-03-06T12:05:51.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:51.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:51.344417+0000) 2024-03-06T12:05:52.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:52.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:52.344642+0000) 2024-03-06T12:05:53.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:53.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:53.344876+0000) 2024-03-06T12:05:54.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:54.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:54.345108+0000) 2024-03-06T12:05:55.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:55.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:55.345334+0000) 2024-03-06T12:05:56.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:56.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:56.345664+0000) 2024-03-06T12:05:57.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:57.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:57.345891+0000) 2024-03-06T12:05:58.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:58.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:58.346099+0000) 2024-03-06T12:05:59.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:05:59.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:05:59.346338+0000) 2024-03-06T12:06:00.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:00.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:00.346617+0000) 2024-03-06T12:06:01.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:01.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:01.346852+0000) 2024-03-06T12:06:02.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:02.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:02.347431+0000) 2024-03-06T12:06:03.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:03.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:03.347647+0000) 2024-03-06T12:06:04.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:04.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:04.347857+0000) 2024-03-06T12:06:05.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:05.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:05.348016+0000) 2024-03-06T12:06:06.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:06.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:06.348261+0000) 2024-03-06T12:06:07.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:07.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:07.348486+0000) 2024-03-06T12:06:08.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:08.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:08.348712+0000) 2024-03-06T12:06:09.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:09.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:09.348937+0000) 2024-03-06T12:06:10.062 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:06:10.071 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:06:10.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:10.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:10.349152+0000) 2024-03-06T12:06:11.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:11.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:11.349378+0000) 2024-03-06T12:06:12.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:12.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:12.349661+0000) 2024-03-06T12:06:13.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:13.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:13.349896+0000) 2024-03-06T12:06:14.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:14.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:14.350124+0000) 2024-03-06T12:06:15.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:15.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:15.350311+0000) 2024-03-06T12:06:16.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:16.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:16.350579+0000) 2024-03-06T12:06:17.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:17.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:17.350808+0000) 2024-03-06T12:06:18.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:18.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:18.350973+0000) 2024-03-06T12:06:19.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:19.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:19.351157+0000) 2024-03-06T12:06:20.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:20.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:20.351370+0000) 2024-03-06T12:06:21.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:21.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:21.351619+0000) 2024-03-06T12:06:22.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:22.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:22.351869+0000) 2024-03-06T12:06:23.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:23.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:23.352134+0000) 2024-03-06T12:06:24.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:24.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:24.352397+0000) 2024-03-06T12:06:25.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:25.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:25.352631+0000) 2024-03-06T12:06:26.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:26.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:26.352866+0000) 2024-03-06T12:06:27.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:27.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:27.353113+0000) 2024-03-06T12:06:28.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:28.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:28.353350+0000) 2024-03-06T12:06:29.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:29.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:29.353615+0000) 2024-03-06T12:06:30.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:30.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:30.353830+0000) 2024-03-06T12:06:31.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:31.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:31.354053+0000) 2024-03-06T12:06:32.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:32.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:32.354274+0000) 2024-03-06T12:06:33.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:33.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:33.354511+0000) 2024-03-06T12:06:34.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:34.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:34.354711+0000) 2024-03-06T12:06:35.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:35.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:35.354920+0000) 2024-03-06T12:06:36.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:36.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:36.355153+0000) 2024-03-06T12:06:37.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:37.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:37.355394+0000) 2024-03-06T12:06:38.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:38.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:38.355667+0000) 2024-03-06T12:06:39.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:39.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:39.355896+0000) 2024-03-06T12:06:40.200 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:06:40.207 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:06:40.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:40.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:40.356125+0000) 2024-03-06T12:06:41.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:41.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:41.356353+0000) 2024-03-06T12:06:42.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:42.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:42.356627+0000) 2024-03-06T12:06:43.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:43.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:43.356848+0000) 2024-03-06T12:06:44.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:44.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:44.357079+0000) 2024-03-06T12:06:45.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:45.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:45.357305+0000) 2024-03-06T12:06:46.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:46.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:46.357564+0000) 2024-03-06T12:06:47.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:47.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:47.357784+0000) 2024-03-06T12:06:48.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:48.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:48.358003+0000) 2024-03-06T12:06:49.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:49.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:49.358237+0000) 2024-03-06T12:06:50.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:50.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:50.358462+0000) 2024-03-06T12:06:51.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:51.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:51.358690+0000) 2024-03-06T12:06:52.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:52.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:52.358851+0000) 2024-03-06T12:06:53.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:53.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:53.359129+0000) 2024-03-06T12:06:54.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:54.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:54.359399+0000) 2024-03-06T12:06:55.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:55.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:55.359634+0000) 2024-03-06T12:06:56.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:56.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:56.359862+0000) 2024-03-06T12:06:57.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:57.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:57.360088+0000) 2024-03-06T12:06:58.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:58.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:58.360302+0000) 2024-03-06T12:06:59.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:06:59.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:06:59.360501+0000) 2024-03-06T12:07:00.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:00.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:00.360736+0000) 2024-03-06T12:07:01.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:01.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:01.360960+0000) 2024-03-06T12:07:02.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:02.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:02.361185+0000) 2024-03-06T12:07:03.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:03.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:03.361413+0000) 2024-03-06T12:07:04.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:04.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:04.361655+0000) 2024-03-06T12:07:05.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:05.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:05.361895+0000) 2024-03-06T12:07:06.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:06.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:06.362146+0000) 2024-03-06T12:07:07.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:07.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:07.362383+0000) 2024-03-06T12:07:08.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:08.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:08.362645+0000) 2024-03-06T12:07:09.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:09.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:09.362835+0000) 2024-03-06T12:07:10.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:10.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:10.363040+0000) 2024-03-06T12:07:10.414 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:07:10.422 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:07:11.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:11.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:11.363239+0000) 2024-03-06T12:07:12.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:12.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:12.363459+0000) 2024-03-06T12:07:13.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:13.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:13.363657+0000) 2024-03-06T12:07:14.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:14.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:14.363889+0000) 2024-03-06T12:07:15.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:15.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:15.364054+0000) 2024-03-06T12:07:16.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:16.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:16.364302+0000) 2024-03-06T12:07:17.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:17.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:17.364527+0000) 2024-03-06T12:07:18.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:18.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:18.364777+0000) 2024-03-06T12:07:19.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:19.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:19.364960+0000) 2024-03-06T12:07:20.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:20.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:20.365180+0000) 2024-03-06T12:07:21.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:21.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:21.365419+0000) 2024-03-06T12:07:22.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:22.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:22.365654+0000) 2024-03-06T12:07:23.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:23.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:23.365854+0000) 2024-03-06T12:07:24.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:24.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:24.366021+0000) 2024-03-06T12:07:25.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:25.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:25.366268+0000) 2024-03-06T12:07:26.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:26.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:26.366495+0000) 2024-03-06T12:07:27.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:27.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:27.366766+0000) 2024-03-06T12:07:28.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:28.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:28.366922+0000) 2024-03-06T12:07:29.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:29.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:29.367108+0000) 2024-03-06T12:07:30.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:30.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:30.367273+0000) 2024-03-06T12:07:31.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:31.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:31.367513+0000) 2024-03-06T12:07:32.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:32.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:32.367785+0000) 2024-03-06T12:07:33.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:33.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:33.367979+0000) 2024-03-06T12:07:34.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:34.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:34.368216+0000) 2024-03-06T12:07:35.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:35.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:35.368363+0000) 2024-03-06T12:07:36.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:36.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:36.368630+0000) 2024-03-06T12:07:37.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:37.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:37.368947+0000) 2024-03-06T12:07:38.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:38.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:38.369153+0000) 2024-03-06T12:07:39.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:39.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:39.369432+0000) 2024-03-06T12:07:40.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:40.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:40.369660+0000) 2024-03-06T12:07:40.567 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:07:40.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:07:41.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:41.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:41.369804+0000) 2024-03-06T12:07:42.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:42.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:42.369958+0000) 2024-03-06T12:07:43.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:43.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:43.370194+0000) 2024-03-06T12:07:44.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:44.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:44.370414+0000) 2024-03-06T12:07:45.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:45.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:45.370636+0000) 2024-03-06T12:07:46.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:46.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:46.370865+0000) 2024-03-06T12:07:47.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:47.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:47.371080+0000) 2024-03-06T12:07:48.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:48.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:48.371277+0000) 2024-03-06T12:07:49.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:49.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:49.371520+0000) 2024-03-06T12:07:50.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:50.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:50.371820+0000) 2024-03-06T12:07:51.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:51.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:51.372093+0000) 2024-03-06T12:07:52.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:52.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:52.372322+0000) 2024-03-06T12:07:53.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:53.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:53.372616+0000) 2024-03-06T12:07:54.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:54.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:54.372786+0000) 2024-03-06T12:07:55.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:55.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:55.372945+0000) 2024-03-06T12:07:56.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:56.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:56.373191+0000) 2024-03-06T12:07:57.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:57.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:57.373414+0000) 2024-03-06T12:07:58.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:58.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:58.373641+0000) 2024-03-06T12:07:59.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:07:59.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:07:59.373868+0000) 2024-03-06T12:08:00.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:00.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:00.374086+0000) 2024-03-06T12:08:01.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:01.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:01.374311+0000) 2024-03-06T12:08:02.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:02.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:02.374566+0000) 2024-03-06T12:08:03.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:03.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:03.374794+0000) 2024-03-06T12:08:04.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:04.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:04.375025+0000) 2024-03-06T12:08:05.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:05.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:05.375251+0000) 2024-03-06T12:08:06.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:06.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:06.375397+0000) 2024-03-06T12:08:07.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:07.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:07.375653+0000) 2024-03-06T12:08:08.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:08.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:08.375811+0000) 2024-03-06T12:08:09.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:09.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:09.375985+0000) 2024-03-06T12:08:10.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:10.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:10.376219+0000) 2024-03-06T12:08:10.772 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:08:10.780 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:08:11.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:11.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:11.376443+0000) 2024-03-06T12:08:12.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:12.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:12.376669+0000) 2024-03-06T12:08:13.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:13.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:13.376897+0000) 2024-03-06T12:08:14.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:14.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:14.377082+0000) 2024-03-06T12:08:15.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:15.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:15.377306+0000) 2024-03-06T12:08:16.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:16.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:16.377462+0000) 2024-03-06T12:08:17.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:17.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:17.377700+0000) 2024-03-06T12:08:18.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:18.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:18.377921+0000) 2024-03-06T12:08:19.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:19.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:19.378152+0000) 2024-03-06T12:08:20.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:20.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:20.378375+0000) 2024-03-06T12:08:21.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:21.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:21.378635+0000) 2024-03-06T12:08:22.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:22.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:22.378824+0000) 2024-03-06T12:08:23.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:23.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:23.379058+0000) 2024-03-06T12:08:24.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:24.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:24.379286+0000) 2024-03-06T12:08:25.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:25.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:25.379604+0000) 2024-03-06T12:08:26.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:26.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:26.379837+0000) 2024-03-06T12:08:27.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:27.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:27.380059+0000) 2024-03-06T12:08:28.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:28.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:28.380281+0000) 2024-03-06T12:08:29.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:29.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:29.380511+0000) 2024-03-06T12:08:30.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:30.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:30.380767+0000) 2024-03-06T12:08:31.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:31.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:31.380911+0000) 2024-03-06T12:08:32.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:32.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:32.381115+0000) 2024-03-06T12:08:33.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:33.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:33.381389+0000) 2024-03-06T12:08:34.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:34.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:34.381691+0000) 2024-03-06T12:08:35.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:35.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:35.381968+0000) 2024-03-06T12:08:36.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:36.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:36.382243+0000) 2024-03-06T12:08:37.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:37.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:37.382485+0000) 2024-03-06T12:08:38.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:38.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:38.382699+0000) 2024-03-06T12:08:39.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:39.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:39.382936+0000) 2024-03-06T12:08:40.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:40.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:40.383155+0000) 2024-03-06T12:08:40.976 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:08:40.983 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:08:41.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:41.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:41.383350+0000) 2024-03-06T12:08:42.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:42.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:42.383596+0000) 2024-03-06T12:08:43.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:43.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:43.383818+0000) 2024-03-06T12:08:44.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:44.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:44.384051+0000) 2024-03-06T12:08:45.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:45.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:45.384267+0000) 2024-03-06T12:08:46.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:46.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:46.384435+0000) 2024-03-06T12:08:47.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:47.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:47.384715+0000) 2024-03-06T12:08:48.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:48.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:48.384888+0000) 2024-03-06T12:08:49.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:49.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:49.385125+0000) 2024-03-06T12:08:50.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:50.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:50.385342+0000) 2024-03-06T12:08:51.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:51.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:51.385576+0000) 2024-03-06T12:08:52.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:52.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:52.385836+0000) 2024-03-06T12:08:53.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:53.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:53.385987+0000) 2024-03-06T12:08:54.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:54.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:54.386225+0000) 2024-03-06T12:08:55.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:55.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:55.386452+0000) 2024-03-06T12:08:56.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:56.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:56.386659+0000) 2024-03-06T12:08:57.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:57.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:57.386880+0000) 2024-03-06T12:08:58.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:58.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:58.387101+0000) 2024-03-06T12:08:59.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:08:59.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:08:59.387347+0000) 2024-03-06T12:09:00.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:00.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:00.387620+0000) 2024-03-06T12:09:01.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:01.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:01.387851+0000) 2024-03-06T12:09:02.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:02.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:02.388080+0000) 2024-03-06T12:09:03.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:03.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:03.388303+0000) 2024-03-06T12:09:04.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:04.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:04.388579+0000) 2024-03-06T12:09:05.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:05.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:05.388806+0000) 2024-03-06T12:09:06.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:06.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:06.389038+0000) 2024-03-06T12:09:07.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:07.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:07.389263+0000) 2024-03-06T12:09:08.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:08.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:08.389488+0000) 2024-03-06T12:09:09.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:09.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:09.389701+0000) 2024-03-06T12:09:10.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:10.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:10.389883+0000) 2024-03-06T12:09:11.179 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:09:11.188 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:09:11.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:11.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:11.390083+0000) 2024-03-06T12:09:12.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:12.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:12.390274+0000) 2024-03-06T12:09:13.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:13.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:13.390506+0000) 2024-03-06T12:09:14.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:14.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:14.390693+0000) 2024-03-06T12:09:15.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:15.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:15.390882+0000) 2024-03-06T12:09:16.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:16.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:16.391100+0000) 2024-03-06T12:09:17.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:17.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:17.391332+0000) 2024-03-06T12:09:18.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:18.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:18.391612+0000) 2024-03-06T12:09:19.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:19.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:19.391806+0000) 2024-03-06T12:09:20.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:20.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:20.391996+0000) 2024-03-06T12:09:21.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:21.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:21.392185+0000) 2024-03-06T12:09:22.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:22.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:22.392370+0000) 2024-03-06T12:09:23.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:23.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:23.392621+0000) 2024-03-06T12:09:24.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:24.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:24.392819+0000) 2024-03-06T12:09:25.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:25.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:25.393058+0000) 2024-03-06T12:09:26.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:26.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:26.393289+0000) 2024-03-06T12:09:27.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:27.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:27.393490+0000) 2024-03-06T12:09:28.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:28.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:28.393716+0000) 2024-03-06T12:09:29.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:29.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:29.393950+0000) 2024-03-06T12:09:30.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:30.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:30.394177+0000) 2024-03-06T12:09:31.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:31.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:31.394403+0000) 2024-03-06T12:09:32.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:32.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:32.394627+0000) 2024-03-06T12:09:33.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:33.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:33.394816+0000) 2024-03-06T12:09:34.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:34.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:34.395011+0000) 2024-03-06T12:09:35.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:35.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:35.395177+0000) 2024-03-06T12:09:36.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:36.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:36.395410+0000) 2024-03-06T12:09:37.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:37.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:37.395643+0000) 2024-03-06T12:09:38.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:38.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:38.395825+0000) 2024-03-06T12:09:39.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:39.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:39.396020+0000) 2024-03-06T12:09:40.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:40.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:40.396213+0000) 2024-03-06T12:09:41.379 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:09:41.388 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:09:41.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:41.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:41.396410+0000) 2024-03-06T12:09:42.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:42.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:42.396692+0000) 2024-03-06T12:09:43.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:43.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:43.396933+0000) 2024-03-06T12:09:44.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:44.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:44.397173+0000) 2024-03-06T12:09:45.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:45.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:45.397368+0000) 2024-03-06T12:09:46.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:46.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:46.397623+0000) 2024-03-06T12:09:47.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:47.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:47.397829+0000) 2024-03-06T12:09:48.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:48.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:48.398034+0000) 2024-03-06T12:09:49.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:49.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:49.398278+0000) 2024-03-06T12:09:50.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:50.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:50.398425+0000) 2024-03-06T12:09:51.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:51.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:51.398645+0000) 2024-03-06T12:09:52.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:52.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:52.398867+0000) 2024-03-06T12:09:53.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:53.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:53.399091+0000) 2024-03-06T12:09:54.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:54.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:54.399287+0000) 2024-03-06T12:09:55.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:55.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:55.399503+0000) 2024-03-06T12:09:56.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:56.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:56.399794+0000) 2024-03-06T12:09:57.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:57.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:57.400034+0000) 2024-03-06T12:09:58.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:58.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:58.400220+0000) 2024-03-06T12:09:59.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:09:59.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:09:59.400463+0000) 2024-03-06T12:10:00.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:00.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:00.400701+0000) 2024-03-06T12:10:01.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:01.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:01.400927+0000) 2024-03-06T12:10:02.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:02.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:02.401155+0000) 2024-03-06T12:10:03.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:03.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:03.401376+0000) 2024-03-06T12:10:04.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:04.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:04.401637+0000) 2024-03-06T12:10:05.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:05.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:05.401914+0000) 2024-03-06T12:10:06.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:06.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:06.402209+0000) 2024-03-06T12:10:07.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:07.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:07.402446+0000) 2024-03-06T12:10:08.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:08.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:08.402698+0000) 2024-03-06T12:10:09.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:09.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:09.402895+0000) 2024-03-06T12:10:10.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:10.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:10.403083+0000) 2024-03-06T12:10:11.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:11.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:11.403256+0000) 2024-03-06T12:10:11.510 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:10:11.519 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:10:12.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:12.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:12.403486+0000) 2024-03-06T12:10:13.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:13.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:13.403715+0000) 2024-03-06T12:10:14.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:14.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:14.403941+0000) 2024-03-06T12:10:15.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:15.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:15.404089+0000) 2024-03-06T12:10:16.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:16.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:16.404330+0000) 2024-03-06T12:10:17.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:17.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:17.404628+0000) 2024-03-06T12:10:18.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:18.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:18.404901+0000) 2024-03-06T12:10:19.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:19.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:19.405140+0000) 2024-03-06T12:10:20.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:20.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:20.405419+0000) 2024-03-06T12:10:21.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:21.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:21.405677+0000) 2024-03-06T12:10:22.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:22.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:22.405890+0000) 2024-03-06T12:10:23.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:23.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:23.406139+0000) 2024-03-06T12:10:24.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:24.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:24.406409+0000) 2024-03-06T12:10:25.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:25.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:25.406711+0000) 2024-03-06T12:10:26.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:26.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:26.406911+0000) 2024-03-06T12:10:27.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:27.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:27.407128+0000) 2024-03-06T12:10:28.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:28.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:28.407397+0000) 2024-03-06T12:10:29.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:29.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:29.407685+0000) 2024-03-06T12:10:30.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:30.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:30.407919+0000) 2024-03-06T12:10:31.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:31.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:31.408132+0000) 2024-03-06T12:10:32.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:32.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:32.408413+0000) 2024-03-06T12:10:33.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:33.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:33.408681+0000) 2024-03-06T12:10:34.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:34.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:34.408915+0000) 2024-03-06T12:10:35.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:35.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:35.409155+0000) 2024-03-06T12:10:36.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:36.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:36.409449+0000) 2024-03-06T12:10:37.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:37.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:37.409727+0000) 2024-03-06T12:10:38.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:38.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:38.409968+0000) 2024-03-06T12:10:39.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:39.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:39.410246+0000) 2024-03-06T12:10:40.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:40.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:40.410520+0000) 2024-03-06T12:10:41.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:41.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:41.410814+0000) 2024-03-06T12:10:41.724 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:10:41.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:10:42.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:42.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:42.411091+0000) 2024-03-06T12:10:43.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:43.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:43.411361+0000) 2024-03-06T12:10:44.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:44.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:44.411672+0000) 2024-03-06T12:10:45.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:45.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:45.411918+0000) 2024-03-06T12:10:46.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:46.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:46.412131+0000) 2024-03-06T12:10:47.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:47.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:47.412346+0000) 2024-03-06T12:10:48.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:48.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:48.412571+0000) 2024-03-06T12:10:49.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:49.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:49.412810+0000) 2024-03-06T12:10:50.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:50.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:50.413019+0000) 2024-03-06T12:10:51.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:51.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:51.413263+0000) 2024-03-06T12:10:52.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:52.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:52.413487+0000) 2024-03-06T12:10:53.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:53.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:53.413682+0000) 2024-03-06T12:10:54.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:54.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:54.413875+0000) 2024-03-06T12:10:55.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:55.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:55.414063+0000) 2024-03-06T12:10:56.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:56.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:56.414291+0000) 2024-03-06T12:10:57.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:57.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:57.414512+0000) 2024-03-06T12:10:58.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:58.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:58.414736+0000) 2024-03-06T12:10:59.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:10:59.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:10:59.414940+0000) 2024-03-06T12:11:00.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:00.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:00.415175+0000) 2024-03-06T12:11:01.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:01.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:01.415315+0000) 2024-03-06T12:11:02.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:02.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:02.415575+0000) 2024-03-06T12:11:03.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:03.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:03.415806+0000) 2024-03-06T12:11:04.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:04.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:04.415984+0000) 2024-03-06T12:11:05.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:05.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:05.416213+0000) 2024-03-06T12:11:06.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:06.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:06.416439+0000) 2024-03-06T12:11:07.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:07.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:07.416657+0000) 2024-03-06T12:11:08.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:08.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:08.416887+0000) 2024-03-06T12:11:09.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:09.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:09.417069+0000) 2024-03-06T12:11:10.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:10.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:10.417293+0000) 2024-03-06T12:11:11.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:11.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:11.417485+0000) 2024-03-06T12:11:11.864 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:11:11.874 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:11:12.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:12.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:12.417698+0000) 2024-03-06T12:11:13.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:13.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:13.417895+0000) 2024-03-06T12:11:14.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:14.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:14.418123+0000) 2024-03-06T12:11:15.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:15.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:15.418343+0000) 2024-03-06T12:11:16.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:16.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:16.418623+0000) 2024-03-06T12:11:17.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:17.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:17.418844+0000) 2024-03-06T12:11:18.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:18.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:18.419064+0000) 2024-03-06T12:11:19.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:19.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:19.419274+0000) 2024-03-06T12:11:20.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:20.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:20.419524+0000) 2024-03-06T12:11:21.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:21.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:21.419788+0000) 2024-03-06T12:11:22.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:22.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:22.420011+0000) 2024-03-06T12:11:23.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:23.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:23.420196+0000) 2024-03-06T12:11:24.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:24.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:24.420392+0000) 2024-03-06T12:11:25.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:25.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:25.420647+0000) 2024-03-06T12:11:26.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:26.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:26.420877+0000) 2024-03-06T12:11:27.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:27.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:27.421113+0000) 2024-03-06T12:11:28.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:28.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:28.421339+0000) 2024-03-06T12:11:29.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:29.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:29.421607+0000) 2024-03-06T12:11:30.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:30.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:30.421789+0000) 2024-03-06T12:11:31.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:31.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:31.421990+0000) 2024-03-06T12:11:32.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:32.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:32.422199+0000) 2024-03-06T12:11:33.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:33.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:33.422444+0000) 2024-03-06T12:11:34.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:34.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:34.422700+0000) 2024-03-06T12:11:35.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:35.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:35.422889+0000) 2024-03-06T12:11:36.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:36.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:36.423124+0000) 2024-03-06T12:11:37.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:37.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:37.423309+0000) 2024-03-06T12:11:38.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:38.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:38.423568+0000) 2024-03-06T12:11:39.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:39.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:39.423789+0000) 2024-03-06T12:11:40.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:40.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:40.423992+0000) 2024-03-06T12:11:41.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:41.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:41.424194+0000) 2024-03-06T12:11:42.051 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:11:42.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:11:42.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:42.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:42.424443+0000) 2024-03-06T12:11:43.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:43.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:43.424656+0000) 2024-03-06T12:11:44.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:44.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:44.424876+0000) 2024-03-06T12:11:45.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:45.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:45.425065+0000) 2024-03-06T12:11:46.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:46.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:46.425255+0000) 2024-03-06T12:11:47.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:47.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:47.425491+0000) 2024-03-06T12:11:48.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:48.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:48.425787+0000) 2024-03-06T12:11:49.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:49.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:49.426008+0000) 2024-03-06T12:11:50.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:50.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:50.426256+0000) 2024-03-06T12:11:51.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:51.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:51.426487+0000) 2024-03-06T12:11:52.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:52.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:52.426687+0000) 2024-03-06T12:11:53.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:53.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:53.426910+0000) 2024-03-06T12:11:54.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:54.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:54.427138+0000) 2024-03-06T12:11:55.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:55.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:55.427363+0000) 2024-03-06T12:11:56.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:56.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:56.427594+0000) 2024-03-06T12:11:57.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:57.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:57.427820+0000) 2024-03-06T12:11:58.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:58.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:58.428049+0000) 2024-03-06T12:11:59.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:11:59.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:11:59.428247+0000) 2024-03-06T12:12:00.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:00.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:00.428449+0000) 2024-03-06T12:12:01.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:01.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:01.428681+0000) 2024-03-06T12:12:02.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:02.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:02.428904+0000) 2024-03-06T12:12:03.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:03.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:03.429122+0000) 2024-03-06T12:12:04.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:04.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:04.429350+0000) 2024-03-06T12:12:05.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:05.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:05.429562+0000) 2024-03-06T12:12:06.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:06.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:06.429819+0000) 2024-03-06T12:12:07.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:07.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:07.430034+0000) 2024-03-06T12:12:08.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:08.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:08.430259+0000) 2024-03-06T12:12:09.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:09.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:09.430489+0000) 2024-03-06T12:12:10.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:10.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:10.430693+0000) 2024-03-06T12:12:11.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:11.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:11.430902+0000) 2024-03-06T12:12:12.271 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:12:12.280 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:12:12.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:12.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:12.431109+0000) 2024-03-06T12:12:13.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:13.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:13.431353+0000) 2024-03-06T12:12:14.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:14.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:14.431568+0000) 2024-03-06T12:12:15.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:15.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:15.431777+0000) 2024-03-06T12:12:16.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:16.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:16.431982+0000) 2024-03-06T12:12:17.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:17.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:17.432200+0000) 2024-03-06T12:12:18.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:18.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:18.432420+0000) 2024-03-06T12:12:19.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:19.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:19.432651+0000) 2024-03-06T12:12:20.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:20.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:20.432799+0000) 2024-03-06T12:12:21.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:21.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:21.432988+0000) 2024-03-06T12:12:22.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:22.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:22.433216+0000) 2024-03-06T12:12:23.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:23.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:23.433457+0000) 2024-03-06T12:12:24.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:24.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:24.433702+0000) 2024-03-06T12:12:25.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:25.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:25.433908+0000) 2024-03-06T12:12:26.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:26.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:26.434162+0000) 2024-03-06T12:12:27.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:27.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:27.434400+0000) 2024-03-06T12:12:28.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:28.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:28.434642+0000) 2024-03-06T12:12:29.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:29.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:29.434853+0000) 2024-03-06T12:12:30.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:30.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:30.435057+0000) 2024-03-06T12:12:31.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:31.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:31.435267+0000) 2024-03-06T12:12:32.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:32.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:32.435486+0000) 2024-03-06T12:12:33.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:33.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:33.435679+0000) 2024-03-06T12:12:34.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:34.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:34.435901+0000) 2024-03-06T12:12:35.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:35.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:35.436088+0000) 2024-03-06T12:12:36.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:36.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:36.436377+0000) 2024-03-06T12:12:37.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:37.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:37.436654+0000) 2024-03-06T12:12:38.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:38.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:38.436869+0000) 2024-03-06T12:12:39.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:39.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:39.437071+0000) 2024-03-06T12:12:40.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:40.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:40.437309+0000) 2024-03-06T12:12:41.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:41.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:41.437525+0000) 2024-03-06T12:12:42.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:42.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:42.437763+0000) 2024-03-06T12:12:42.480 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:12:42.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:12:43.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:43.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:43.437956+0000) 2024-03-06T12:12:44.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:44.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:44.438134+0000) 2024-03-06T12:12:45.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:45.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:45.438354+0000) 2024-03-06T12:12:46.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:46.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:46.438562+0000) 2024-03-06T12:12:47.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:47.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:47.438750+0000) 2024-03-06T12:12:48.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:48.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:48.438935+0000) 2024-03-06T12:12:49.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:49.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:49.439127+0000) 2024-03-06T12:12:50.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:50.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:50.439343+0000) 2024-03-06T12:12:51.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:51.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:51.439584+0000) 2024-03-06T12:12:52.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:52.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:52.439775+0000) 2024-03-06T12:12:53.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:53.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:53.439967+0000) 2024-03-06T12:12:54.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:54.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:54.440152+0000) 2024-03-06T12:12:55.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:55.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:55.440372+0000) 2024-03-06T12:12:56.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:56.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:56.440605+0000) 2024-03-06T12:12:57.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:57.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:57.440802+0000) 2024-03-06T12:12:58.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:58.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:58.441024+0000) 2024-03-06T12:12:59.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:12:59.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:12:59.441260+0000) 2024-03-06T12:13:00.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:00.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:00.441491+0000) 2024-03-06T12:13:01.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:01.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:01.441684+0000) 2024-03-06T12:13:02.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:02.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:02.441876+0000) 2024-03-06T12:13:03.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:03.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:03.442096+0000) 2024-03-06T12:13:04.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:04.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:04.442319+0000) 2024-03-06T12:13:05.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:05.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:05.442564+0000) 2024-03-06T12:13:06.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:06.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:06.442822+0000) 2024-03-06T12:13:07.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:07.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:07.443011+0000) 2024-03-06T12:13:08.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:08.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:08.443236+0000) 2024-03-06T12:13:09.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:09.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:09.443422+0000) 2024-03-06T12:13:10.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:10.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:10.443646+0000) 2024-03-06T12:13:11.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:11.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:11.443833+0000) 2024-03-06T12:13:12.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:12.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:12.444020+0000) 2024-03-06T12:13:12.610 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:13:12.618 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:13:13.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:13.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:13.444257+0000) 2024-03-06T12:13:14.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:14.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:14.444496+0000) 2024-03-06T12:13:15.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:15.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:15.444734+0000) 2024-03-06T12:13:16.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:16.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:16.444930+0000) 2024-03-06T12:13:17.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:17.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:17.445130+0000) 2024-03-06T12:13:18.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:18.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:18.445354+0000) 2024-03-06T12:13:19.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:19.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:19.445636+0000) 2024-03-06T12:13:20.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:20.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:20.445826+0000) 2024-03-06T12:13:21.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:21.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:21.446009+0000) 2024-03-06T12:13:22.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:22.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:22.446236+0000) 2024-03-06T12:13:23.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:23.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:23.446472+0000) 2024-03-06T12:13:24.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:24.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:24.446741+0000) 2024-03-06T12:13:25.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:25.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:25.446943+0000) 2024-03-06T12:13:26.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:26.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:26.447171+0000) 2024-03-06T12:13:27.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:27.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:27.447422+0000) 2024-03-06T12:13:28.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:28.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:28.447698+0000) 2024-03-06T12:13:29.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:29.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:29.447980+0000) 2024-03-06T12:13:30.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:30.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:30.448247+0000) 2024-03-06T12:13:31.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:31.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:31.448463+0000) 2024-03-06T12:13:32.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:32.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:32.448735+0000) 2024-03-06T12:13:33.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:33.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:33.449008+0000) 2024-03-06T12:13:34.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:34.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:34.449207+0000) 2024-03-06T12:13:35.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:35.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:35.449444+0000) 2024-03-06T12:13:36.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:36.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:36.449679+0000) 2024-03-06T12:13:37.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:37.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:37.449885+0000) 2024-03-06T12:13:38.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:38.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:38.450086+0000) 2024-03-06T12:13:39.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:39.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:39.450299+0000) 2024-03-06T12:13:40.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:40.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:40.450498+0000) 2024-03-06T12:13:41.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:41.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:41.450732+0000) 2024-03-06T12:13:42.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:42.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:42.450981+0000) 2024-03-06T12:13:42.824 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:13:42.833 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:13:43.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:43.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:43.451203+0000) 2024-03-06T12:13:44.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:44.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:44.451446+0000) 2024-03-06T12:13:45.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:45.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:45.451701+0000) 2024-03-06T12:13:46.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:46.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:46.451933+0000) 2024-03-06T12:13:47.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:47.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:47.452126+0000) 2024-03-06T12:13:48.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:48.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:48.452402+0000) 2024-03-06T12:13:49.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:49.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:49.452621+0000) 2024-03-06T12:13:50.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:50.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:50.452812+0000) 2024-03-06T12:13:51.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:51.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:51.453009+0000) 2024-03-06T12:13:52.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:52.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:52.453241+0000) 2024-03-06T12:13:53.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:53.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:53.453489+0000) 2024-03-06T12:13:54.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:54.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:54.453722+0000) 2024-03-06T12:13:55.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:55.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:55.453901+0000) 2024-03-06T12:13:56.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:56.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:56.454139+0000) 2024-03-06T12:13:57.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:57.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:57.454316+0000) 2024-03-06T12:13:58.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:58.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:58.454570+0000) 2024-03-06T12:13:59.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:13:59.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:13:59.454822+0000) 2024-03-06T12:14:00.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:00.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:00.455061+0000) 2024-03-06T12:14:01.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:01.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:01.455284+0000) 2024-03-06T12:14:02.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:02.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:02.455578+0000) 2024-03-06T12:14:03.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:03.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:03.455857+0000) 2024-03-06T12:14:04.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:04.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:04.456097+0000) 2024-03-06T12:14:05.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:05.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:05.456335+0000) 2024-03-06T12:14:06.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:06.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:06.456645+0000) 2024-03-06T12:14:07.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:07.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:07.456917+0000) 2024-03-06T12:14:08.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:08.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:08.457193+0000) 2024-03-06T12:14:09.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:09.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:09.457475+0000) 2024-03-06T12:14:10.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:10.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:10.457709+0000) 2024-03-06T12:14:11.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:11.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:11.457939+0000) 2024-03-06T12:14:12.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:12.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:12.458202+0000) 2024-03-06T12:14:13.031 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:14:13.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:14:13.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:13.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:13.458403+0000) 2024-03-06T12:14:14.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:14.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:14.458640+0000) 2024-03-06T12:14:15.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:15.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:15.458883+0000) 2024-03-06T12:14:16.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:16.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:16.459157+0000) 2024-03-06T12:14:17.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:17.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:17.459479+0000) 2024-03-06T12:14:18.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:18.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:18.459736+0000) 2024-03-06T12:14:19.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:19.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:19.459970+0000) 2024-03-06T12:14:20.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:20.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:20.460168+0000) 2024-03-06T12:14:21.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:21.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:21.460396+0000) 2024-03-06T12:14:22.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:22.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:22.460689+0000) 2024-03-06T12:14:23.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:23.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:23.460963+0000) 2024-03-06T12:14:24.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:24.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:24.461246+0000) 2024-03-06T12:14:25.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:25.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:25.461482+0000) 2024-03-06T12:14:26.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:26.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:26.461758+0000) 2024-03-06T12:14:27.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:27.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:27.462034+0000) 2024-03-06T12:14:28.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:28.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:28.462302+0000) 2024-03-06T12:14:29.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:29.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:29.462608+0000) 2024-03-06T12:14:30.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:30.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:30.462841+0000) 2024-03-06T12:14:31.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:31.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:31.463066+0000) 2024-03-06T12:14:32.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:32.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:32.463326+0000) 2024-03-06T12:14:33.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:33.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:33.463630+0000) 2024-03-06T12:14:34.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:34.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:34.463832+0000) 2024-03-06T12:14:35.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:35.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:35.464026+0000) 2024-03-06T12:14:36.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:36.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:36.464306+0000) 2024-03-06T12:14:37.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:37.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:37.464503+0000) 2024-03-06T12:14:38.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:38.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:38.464730+0000) 2024-03-06T12:14:39.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:39.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:39.464919+0000) 2024-03-06T12:14:40.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:40.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:40.465130+0000) 2024-03-06T12:14:41.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:41.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:41.465403+0000) 2024-03-06T12:14:42.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:42.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:42.465672+0000) 2024-03-06T12:14:43.257 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:14:43.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:14:43.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:43.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:43.465922+0000) 2024-03-06T12:14:44.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:44.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:44.466167+0000) 2024-03-06T12:14:45.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:45.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:45.466368+0000) 2024-03-06T12:14:46.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:46.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:46.466637+0000) 2024-03-06T12:14:47.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:47.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:47.466834+0000) 2024-03-06T12:14:48.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:48.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:48.467035+0000) 2024-03-06T12:14:49.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:49.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:49.467268+0000) 2024-03-06T12:14:50.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:50.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:50.467465+0000) 2024-03-06T12:14:51.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:51.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:51.467680+0000) 2024-03-06T12:14:52.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:52.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:52.467871+0000) 2024-03-06T12:14:53.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:53.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:53.468064+0000) 2024-03-06T12:14:54.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:54.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:54.468295+0000) 2024-03-06T12:14:55.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:55.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:55.468512+0000) 2024-03-06T12:14:56.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:56.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:56.468807+0000) 2024-03-06T12:14:57.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:57.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:57.469077+0000) 2024-03-06T12:14:58.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:58.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:58.469351+0000) 2024-03-06T12:14:59.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:14:59.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:14:59.469667+0000) 2024-03-06T12:15:00.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:00.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:00.469881+0000) 2024-03-06T12:15:01.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:01.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:01.470093+0000) 2024-03-06T12:15:02.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:02.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:02.470304+0000) 2024-03-06T12:15:03.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:03.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:03.470513+0000) 2024-03-06T12:15:04.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:04.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:04.470742+0000) 2024-03-06T12:15:05.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:05.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:05.471019+0000) 2024-03-06T12:15:06.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:06.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:06.471237+0000) 2024-03-06T12:15:07.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:07.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:07.471458+0000) 2024-03-06T12:15:08.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:08.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:08.471714+0000) 2024-03-06T12:15:09.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:09.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:09.471918+0000) 2024-03-06T12:15:10.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:10.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:10.472124+0000) 2024-03-06T12:15:11.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:11.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:11.472362+0000) 2024-03-06T12:15:12.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:12.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:12.472636+0000) 2024-03-06T12:15:13.460 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:15:13.468 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:15:13.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:13.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:13.472844+0000) 2024-03-06T12:15:14.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:14.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:14.473037+0000) 2024-03-06T12:15:15.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:15.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:15.473261+0000) 2024-03-06T12:15:16.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:16.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:16.473503+0000) 2024-03-06T12:15:17.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:17.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:17.473736+0000) 2024-03-06T12:15:18.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:18.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:18.473964+0000) 2024-03-06T12:15:19.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:19.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:19.474188+0000) 2024-03-06T12:15:20.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:20.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:20.474323+0000) 2024-03-06T12:15:21.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:21.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:21.474568+0000) 2024-03-06T12:15:22.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:22.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:22.474764+0000) 2024-03-06T12:15:23.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:23.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:23.474953+0000) 2024-03-06T12:15:24.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:24.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:24.475148+0000) 2024-03-06T12:15:25.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:25.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:25.475338+0000) 2024-03-06T12:15:26.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:26.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:26.475616+0000) 2024-03-06T12:15:27.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:27.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:27.475808+0000) 2024-03-06T12:15:28.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:28.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:28.476010+0000) 2024-03-06T12:15:29.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:29.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:29.476231+0000) 2024-03-06T12:15:30.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:30.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:30.476418+0000) 2024-03-06T12:15:31.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:31.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:31.476655+0000) 2024-03-06T12:15:32.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:32.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:32.476907+0000) 2024-03-06T12:15:33.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:33.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:33.477102+0000) 2024-03-06T12:15:34.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:34.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:34.477301+0000) 2024-03-06T12:15:35.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:35.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:35.477570+0000) 2024-03-06T12:15:36.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:36.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:36.477772+0000) 2024-03-06T12:15:37.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:37.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:37.477963+0000) 2024-03-06T12:15:38.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:38.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:38.478161+0000) 2024-03-06T12:15:39.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:39.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:39.478383+0000) 2024-03-06T12:15:40.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:40.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:40.478651+0000) 2024-03-06T12:15:41.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:41.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:41.478841+0000) 2024-03-06T12:15:42.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:42.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:42.479031+0000) 2024-03-06T12:15:43.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:43.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:43.479256+0000) 2024-03-06T12:15:43.666 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:15:43.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:15:44.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:44.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:44.479493+0000) 2024-03-06T12:15:45.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:45.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:45.479682+0000) 2024-03-06T12:15:46.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:46.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:46.479873+0000) 2024-03-06T12:15:47.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:47.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:47.480055+0000) 2024-03-06T12:15:48.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:48.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:48.480285+0000) 2024-03-06T12:15:49.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:49.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:49.480512+0000) 2024-03-06T12:15:50.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:50.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:50.480733+0000) 2024-03-06T12:15:51.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:51.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:51.480938+0000) 2024-03-06T12:15:52.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:52.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:52.481129+0000) 2024-03-06T12:15:53.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:53.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:53.481355+0000) 2024-03-06T12:15:54.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:54.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:54.481636+0000) 2024-03-06T12:15:55.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:55.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:55.481830+0000) 2024-03-06T12:15:56.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:56.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:56.482019+0000) 2024-03-06T12:15:57.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:57.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:57.482247+0000) 2024-03-06T12:15:58.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:58.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:58.482515+0000) 2024-03-06T12:15:59.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:15:59.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:15:59.482734+0000) 2024-03-06T12:16:00.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:00.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:00.482930+0000) 2024-03-06T12:16:01.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:01.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:01.483118+0000) 2024-03-06T12:16:02.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:02.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:02.483341+0000) 2024-03-06T12:16:03.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:03.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:03.483644+0000) 2024-03-06T12:16:04.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:04.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:04.483900+0000) 2024-03-06T12:16:05.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:05.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:05.484140+0000) 2024-03-06T12:16:06.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:06.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:06.484378+0000) 2024-03-06T12:16:07.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:07.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:07.484649+0000) 2024-03-06T12:16:08.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:08.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:08.484894+0000) 2024-03-06T12:16:09.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:09.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:09.485135+0000) 2024-03-06T12:16:10.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:10.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:10.485372+0000) 2024-03-06T12:16:11.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:11.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:11.485636+0000) 2024-03-06T12:16:12.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:12.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:12.485884+0000) 2024-03-06T12:16:13.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:13.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:13.486100+0000) 2024-03-06T12:16:13.866 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:16:13.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:16:14.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:14.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:14.486350+0000) 2024-03-06T12:16:15.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:15.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:15.486645+0000) 2024-03-06T12:16:16.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:16.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:16.486884+0000) 2024-03-06T12:16:17.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:17.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:17.487109+0000) 2024-03-06T12:16:18.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:18.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:18.487304+0000) 2024-03-06T12:16:19.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:19.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:19.487557+0000) 2024-03-06T12:16:20.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:20.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:20.487772+0000) 2024-03-06T12:16:21.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:21.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:21.487997+0000) 2024-03-06T12:16:22.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:22.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:22.488230+0000) 2024-03-06T12:16:23.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:23.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:23.488477+0000) 2024-03-06T12:16:24.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:24.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:24.488721+0000) 2024-03-06T12:16:25.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:25.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:25.488973+0000) 2024-03-06T12:16:26.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:26.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:26.489221+0000) 2024-03-06T12:16:27.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:27.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:27.489398+0000) 2024-03-06T12:16:28.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:28.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:28.489650+0000) 2024-03-06T12:16:29.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:29.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:29.489897+0000) 2024-03-06T12:16:30.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:30.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:30.490115+0000) 2024-03-06T12:16:31.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:31.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:31.490350+0000) 2024-03-06T12:16:32.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:32.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:32.490641+0000) 2024-03-06T12:16:33.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:33.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:33.490853+0000) 2024-03-06T12:16:34.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:34.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:34.491097+0000) 2024-03-06T12:16:35.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:35.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:35.491348+0000) 2024-03-06T12:16:36.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:36.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:36.491631+0000) 2024-03-06T12:16:37.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:37.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:37.491839+0000) 2024-03-06T12:16:38.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:38.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:38.492055+0000) 2024-03-06T12:16:39.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:39.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:39.492248+0000) 2024-03-06T12:16:40.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:40.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:40.492492+0000) 2024-03-06T12:16:41.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:41.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:41.492685+0000) 2024-03-06T12:16:42.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:42.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:42.492878+0000) 2024-03-06T12:16:43.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:43.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:43.493078+0000) 2024-03-06T12:16:44.000 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:16:44.010 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:16:44.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:44.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:44.493308+0000) 2024-03-06T12:16:45.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:45.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:45.493503+0000) 2024-03-06T12:16:46.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:46.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:46.493747+0000) 2024-03-06T12:16:47.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:47.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:47.493981+0000) 2024-03-06T12:16:48.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:48.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:48.494208+0000) 2024-03-06T12:16:49.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:49.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:49.494413+0000) 2024-03-06T12:16:50.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:50.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:50.494681+0000) 2024-03-06T12:16:51.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:51.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:51.494907+0000) 2024-03-06T12:16:52.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:52.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:52.495140+0000) 2024-03-06T12:16:53.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:53.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:53.495331+0000) 2024-03-06T12:16:54.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:54.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:54.495602+0000) 2024-03-06T12:16:55.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:55.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:55.495824+0000) 2024-03-06T12:16:56.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:56.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:56.496046+0000) 2024-03-06T12:16:57.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:57.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:57.496290+0000) 2024-03-06T12:16:58.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:58.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:58.496529+0000) 2024-03-06T12:16:59.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:16:59.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:16:59.496775+0000) 2024-03-06T12:17:00.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:00.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:00.496992+0000) 2024-03-06T12:17:01.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:01.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:01.497235+0000) 2024-03-06T12:17:02.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:02.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:02.497433+0000) 2024-03-06T12:17:03.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:03.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:03.497657+0000) 2024-03-06T12:17:04.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:04.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:04.497854+0000) 2024-03-06T12:17:05.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:05.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:05.498073+0000) 2024-03-06T12:17:06.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:06.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:06.498319+0000) 2024-03-06T12:17:07.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:07.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:07.498612+0000) 2024-03-06T12:17:08.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:08.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:08.498851+0000) 2024-03-06T12:17:09.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:09.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:09.499100+0000) 2024-03-06T12:17:10.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:10.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:10.499369+0000) 2024-03-06T12:17:11.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:11.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:11.499670+0000) 2024-03-06T12:17:12.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:12.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:12.499901+0000) 2024-03-06T12:17:13.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:13.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:13.500166+0000) 2024-03-06T12:17:14.207 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:17:14.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:17:14.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:14.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:14.500409+0000) 2024-03-06T12:17:15.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:15.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:15.500720+0000) 2024-03-06T12:17:16.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:16.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:16.500983+0000) 2024-03-06T12:17:17.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:17.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:17.501226+0000) 2024-03-06T12:17:18.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:18.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:18.501452+0000) 2024-03-06T12:17:19.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:19.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:19.501696+0000) 2024-03-06T12:17:20.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:20.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:20.501910+0000) 2024-03-06T12:17:21.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:21.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:21.502116+0000) 2024-03-06T12:17:22.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:22.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:22.502321+0000) 2024-03-06T12:17:23.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:23.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:23.502567+0000) 2024-03-06T12:17:24.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:24.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:24.502817+0000) 2024-03-06T12:17:25.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:25.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:25.503045+0000) 2024-03-06T12:17:26.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:26.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:26.503296+0000) 2024-03-06T12:17:27.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:27.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:27.503491+0000) 2024-03-06T12:17:28.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:28.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:28.503726+0000) 2024-03-06T12:17:29.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:29.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:29.503960+0000) 2024-03-06T12:17:30.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:30.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:30.504205+0000) 2024-03-06T12:17:31.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:31.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:31.504485+0000) 2024-03-06T12:17:32.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:32.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:32.504735+0000) 2024-03-06T12:17:33.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:33.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:33.504979+0000) 2024-03-06T12:17:34.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:34.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:34.505297+0000) 2024-03-06T12:17:35.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:35.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:35.505603+0000) 2024-03-06T12:17:36.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:36.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:36.505890+0000) 2024-03-06T12:17:37.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:37.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:37.506081+0000) 2024-03-06T12:17:38.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:38.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:38.506306+0000) 2024-03-06T12:17:39.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:39.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:39.506522+0000) 2024-03-06T12:17:40.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:40.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:40.506711+0000) 2024-03-06T12:17:41.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:41.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:41.506901+0000) 2024-03-06T12:17:42.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:42.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:42.507124+0000) 2024-03-06T12:17:43.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:43.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:43.507319+0000) 2024-03-06T12:17:44.435 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:17:44.444 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:17:44.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:44.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:44.507581+0000) 2024-03-06T12:17:45.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:45.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:45.507819+0000) 2024-03-06T12:17:46.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:46.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:46.508057+0000) 2024-03-06T12:17:47.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:47.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:47.508277+0000) 2024-03-06T12:17:48.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:48.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:48.508467+0000) 2024-03-06T12:17:49.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:49.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:49.508703+0000) 2024-03-06T12:17:50.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:50.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:50.508929+0000) 2024-03-06T12:17:51.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:51.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:51.509153+0000) 2024-03-06T12:17:52.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:52.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:52.509304+0000) 2024-03-06T12:17:53.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:53.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:53.509529+0000) 2024-03-06T12:17:54.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:54.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:54.509758+0000) 2024-03-06T12:17:55.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:55.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:55.509950+0000) 2024-03-06T12:17:56.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:56.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:56.510153+0000) 2024-03-06T12:17:57.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:57.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:57.510378+0000) 2024-03-06T12:17:58.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:58.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:58.510625+0000) 2024-03-06T12:17:59.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:17:59.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:17:59.510817+0000) 2024-03-06T12:18:00.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:00.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:00.511022+0000) 2024-03-06T12:18:01.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:01.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:01.511265+0000) 2024-03-06T12:18:02.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:02.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:02.511489+0000) 2024-03-06T12:18:03.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:03.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:03.511728+0000) 2024-03-06T12:18:04.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:04.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:04.511920+0000) 2024-03-06T12:18:05.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:05.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:05.512153+0000) 2024-03-06T12:18:06.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:06.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:06.512386+0000) 2024-03-06T12:18:07.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:07.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:07.512628+0000) 2024-03-06T12:18:08.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:08.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:08.512835+0000) 2024-03-06T12:18:09.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:09.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:09.513053+0000) 2024-03-06T12:18:10.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:10.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:10.513282+0000) 2024-03-06T12:18:11.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:11.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:11.513465+0000) 2024-03-06T12:18:12.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:12.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:12.513697+0000) 2024-03-06T12:18:13.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:13.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:13.513928+0000) 2024-03-06T12:18:14.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:14.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:14.514116+0000) 2024-03-06T12:18:14.622 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:18:14.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:18:15.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:15.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:15.514335+0000) 2024-03-06T12:18:16.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:16.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:16.514572+0000) 2024-03-06T12:18:17.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:17.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:17.514813+0000) 2024-03-06T12:18:18.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:18.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:18.515055+0000) 2024-03-06T12:18:19.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:19.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:19.515291+0000) 2024-03-06T12:18:20.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:20.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:20.515529+0000) 2024-03-06T12:18:21.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:21.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:21.515818+0000) 2024-03-06T12:18:22.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:22.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:22.516058+0000) 2024-03-06T12:18:23.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:23.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:23.516293+0000) 2024-03-06T12:18:24.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:24.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:24.516507+0000) 2024-03-06T12:18:25.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:25.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:25.516767+0000) 2024-03-06T12:18:26.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:26.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:26.517011+0000) 2024-03-06T12:18:27.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:27.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:27.517240+0000) 2024-03-06T12:18:28.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:28.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:28.517576+0000) 2024-03-06T12:18:29.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:29.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:29.517819+0000) 2024-03-06T12:18:30.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:30.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:30.518056+0000) 2024-03-06T12:18:31.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:31.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:31.518296+0000) 2024-03-06T12:18:32.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:32.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:32.518599+0000) 2024-03-06T12:18:33.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:33.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:33.518847+0000) 2024-03-06T12:18:34.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:34.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:34.518999+0000) 2024-03-06T12:18:35.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:35.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:35.519191+0000) 2024-03-06T12:18:36.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:36.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:36.519445+0000) 2024-03-06T12:18:37.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:37.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:37.519653+0000) 2024-03-06T12:18:38.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:38.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:38.519893+0000) 2024-03-06T12:18:39.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:39.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:39.520086+0000) 2024-03-06T12:18:40.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:40.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:40.520307+0000) 2024-03-06T12:18:41.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:41.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:41.520560+0000) 2024-03-06T12:18:42.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:42.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:42.520783+0000) 2024-03-06T12:18:43.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:43.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:43.521059+0000) 2024-03-06T12:18:44.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:44.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:44.521299+0000) 2024-03-06T12:18:44.853 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:18:44.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:18:45.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:45.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:45.521481+0000) 2024-03-06T12:18:46.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:46.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:46.521643+0000) 2024-03-06T12:18:47.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:47.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:47.521844+0000) 2024-03-06T12:18:48.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:48.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:48.522040+0000) 2024-03-06T12:18:49.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:49.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:49.522248+0000) 2024-03-06T12:18:50.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:50.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:50.522443+0000) 2024-03-06T12:18:51.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:51.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:51.522675+0000) 2024-03-06T12:18:52.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:52.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:52.522944+0000) 2024-03-06T12:18:53.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:53.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:53.523218+0000) 2024-03-06T12:18:54.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:54.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:54.523500+0000) 2024-03-06T12:18:55.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:55.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:55.523826+0000) 2024-03-06T12:18:56.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:56.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:56.524084+0000) 2024-03-06T12:18:57.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:57.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:57.524314+0000) 2024-03-06T12:18:58.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:58.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:58.524564+0000) 2024-03-06T12:18:59.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:18:59.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:18:59.524839+0000) 2024-03-06T12:19:00.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:00.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:00.525114+0000) 2024-03-06T12:19:01.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:01.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:01.525314+0000) 2024-03-06T12:19:02.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:02.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:02.525568+0000) 2024-03-06T12:19:03.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:03.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:03.525780+0000) 2024-03-06T12:19:04.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:04.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:04.525986+0000) 2024-03-06T12:19:05.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:05.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:05.526228+0000) 2024-03-06T12:19:06.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:06.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:06.526461+0000) 2024-03-06T12:19:07.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:07.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:07.526683+0000) 2024-03-06T12:19:08.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:08.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:08.526890+0000) 2024-03-06T12:19:09.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:09.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:09.527100+0000) 2024-03-06T12:19:10.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:10.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:10.527340+0000) 2024-03-06T12:19:11.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:11.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:11.527608+0000) 2024-03-06T12:19:12.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:12.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:12.527821+0000) 2024-03-06T12:19:13.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:13.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:13.528060+0000) 2024-03-06T12:19:14.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:14.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:14.528256+0000) 2024-03-06T12:19:14.994 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:19:15.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:19:15.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:15.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:15.528521+0000) 2024-03-06T12:19:16.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:16.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:16.528821+0000) 2024-03-06T12:19:17.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:17.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:17.529003+0000) 2024-03-06T12:19:18.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:18.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:18.529230+0000) 2024-03-06T12:19:19.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:19.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:19.529501+0000) 2024-03-06T12:19:20.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:20.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:20.529830+0000) 2024-03-06T12:19:21.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:21.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:21.530079+0000) 2024-03-06T12:19:22.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:22.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:22.530275+0000) 2024-03-06T12:19:23.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:23.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:23.530501+0000) 2024-03-06T12:19:24.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:24.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:24.530802+0000) 2024-03-06T12:19:25.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:25.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:25.531032+0000) 2024-03-06T12:19:26.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:26.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:26.531324+0000) 2024-03-06T12:19:27.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:27.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:27.531562+0000) 2024-03-06T12:19:28.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:28.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:28.531807+0000) 2024-03-06T12:19:29.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:29.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:29.532021+0000) 2024-03-06T12:19:30.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:30.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:30.532212+0000) 2024-03-06T12:19:31.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:31.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:31.532440+0000) 2024-03-06T12:19:32.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:32.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:32.532672+0000) 2024-03-06T12:19:33.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:33.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:33.532873+0000) 2024-03-06T12:19:34.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:34.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:34.533065+0000) 2024-03-06T12:19:35.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:35.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:35.533328+0000) 2024-03-06T12:19:36.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:36.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:36.533640+0000) 2024-03-06T12:19:37.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:37.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:37.533885+0000) 2024-03-06T12:19:38.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:38.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:38.534167+0000) 2024-03-06T12:19:39.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:39.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:39.534433+0000) 2024-03-06T12:19:40.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:40.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:40.534713+0000) 2024-03-06T12:19:41.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:41.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:41.534938+0000) 2024-03-06T12:19:42.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:42.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:42.535168+0000) 2024-03-06T12:19:43.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:43.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:43.535392+0000) 2024-03-06T12:19:44.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:44.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:44.535656+0000) 2024-03-06T12:19:45.217 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:19:45.225 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:19:45.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:45.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:45.535889+0000) 2024-03-06T12:19:46.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:46.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:46.536129+0000) 2024-03-06T12:19:47.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:47.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:47.536357+0000) 2024-03-06T12:19:48.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:48.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:48.536626+0000) 2024-03-06T12:19:49.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:49.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:49.536855+0000) 2024-03-06T12:19:50.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:50.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:50.537076+0000) 2024-03-06T12:19:51.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:51.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:51.537267+0000) 2024-03-06T12:19:52.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:52.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:52.537505+0000) 2024-03-06T12:19:53.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:53.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:53.537794+0000) 2024-03-06T12:19:54.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:54.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:54.538031+0000) 2024-03-06T12:19:55.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:55.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:55.538316+0000) 2024-03-06T12:19:56.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:56.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:56.538528+0000) 2024-03-06T12:19:57.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:57.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:57.538743+0000) 2024-03-06T12:19:58.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:58.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:58.538970+0000) 2024-03-06T12:19:59.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:19:59.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:19:59.539225+0000) 2024-03-06T12:20:00.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:00.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:00.539452+0000) 2024-03-06T12:20:01.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:01.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:01.539671+0000) 2024-03-06T12:20:02.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:02.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:02.539949+0000) 2024-03-06T12:20:03.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:03.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:03.540163+0000) 2024-03-06T12:20:04.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:04.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:04.540416+0000) 2024-03-06T12:20:05.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:05.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:05.540629+0000) 2024-03-06T12:20:06.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:06.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:06.540908+0000) 2024-03-06T12:20:07.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:07.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:07.541136+0000) 2024-03-06T12:20:08.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:08.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:08.541403+0000) 2024-03-06T12:20:09.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:09.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:09.541725+0000) 2024-03-06T12:20:10.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:10.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:10.541931+0000) 2024-03-06T12:20:11.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:11.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:11.542173+0000) 2024-03-06T12:20:12.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:12.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:12.542437+0000) 2024-03-06T12:20:13.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:13.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:13.542713+0000) 2024-03-06T12:20:14.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:14.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:14.542993+0000) 2024-03-06T12:20:15.438 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:20:15.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:20:15.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:15.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:15.543248+0000) 2024-03-06T12:20:16.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:16.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:16.543494+0000) 2024-03-06T12:20:17.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:17.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:17.543697+0000) 2024-03-06T12:20:18.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:18.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:18.543886+0000) 2024-03-06T12:20:19.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:19.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:19.544115+0000) 2024-03-06T12:20:20.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:20.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:20.544338+0000) 2024-03-06T12:20:21.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:21.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:21.544523+0000) 2024-03-06T12:20:22.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:22.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:22.544768+0000) 2024-03-06T12:20:23.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:23.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:23.544993+0000) 2024-03-06T12:20:24.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:24.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:24.545258+0000) 2024-03-06T12:20:25.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:25.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:25.545568+0000) 2024-03-06T12:20:26.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:26.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:26.545808+0000) 2024-03-06T12:20:27.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:27.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:27.546045+0000) 2024-03-06T12:20:28.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:28.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:28.546293+0000) 2024-03-06T12:20:29.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:29.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:29.546501+0000) 2024-03-06T12:20:30.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:30.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:30.546773+0000) 2024-03-06T12:20:31.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:31.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:31.547002+0000) 2024-03-06T12:20:32.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:32.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:32.547227+0000) 2024-03-06T12:20:33.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:33.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:33.547425+0000) 2024-03-06T12:20:34.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:34.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:34.547651+0000) 2024-03-06T12:20:35.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:35.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:35.547892+0000) 2024-03-06T12:20:36.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:36.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:36.548136+0000) 2024-03-06T12:20:37.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:37.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:37.548375+0000) 2024-03-06T12:20:38.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:38.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:38.548659+0000) 2024-03-06T12:20:39.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:39.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:39.548902+0000) 2024-03-06T12:20:40.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:40.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:40.549102+0000) 2024-03-06T12:20:41.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:41.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:41.549346+0000) 2024-03-06T12:20:42.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:42.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:42.549591+0000) 2024-03-06T12:20:43.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:43.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:43.549788+0000) 2024-03-06T12:20:44.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:44.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:44.549977+0000) 2024-03-06T12:20:45.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:45.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:45.550213+0000) 2024-03-06T12:20:45.668 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:20:45.676 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:20:46.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:46.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:46.550426+0000) 2024-03-06T12:20:47.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:47.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:47.550659+0000) 2024-03-06T12:20:48.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:48.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:48.550890+0000) 2024-03-06T12:20:49.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:49.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:49.551085+0000) 2024-03-06T12:20:50.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:50.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:50.551312+0000) 2024-03-06T12:20:51.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:51.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:51.551565+0000) 2024-03-06T12:20:52.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:52.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:52.551766+0000) 2024-03-06T12:20:53.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:53.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:53.551959+0000) 2024-03-06T12:20:54.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:54.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:54.552155+0000) 2024-03-06T12:20:55.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:55.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:55.552396+0000) 2024-03-06T12:20:56.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:56.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:56.552665+0000) 2024-03-06T12:20:57.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:57.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:57.552905+0000) 2024-03-06T12:20:58.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:58.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:58.553104+0000) 2024-03-06T12:20:59.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:20:59.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:20:59.553311+0000) 2024-03-06T12:21:00.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:00.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:00.553575+0000) 2024-03-06T12:21:01.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:01.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:01.553827+0000) 2024-03-06T12:21:02.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:02.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:02.554058+0000) 2024-03-06T12:21:03.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:03.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:03.554281+0000) 2024-03-06T12:21:04.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:04.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:04.554509+0000) 2024-03-06T12:21:05.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:05.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:05.554775+0000) 2024-03-06T12:21:06.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:06.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:06.555017+0000) 2024-03-06T12:21:07.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:07.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:07.555245+0000) 2024-03-06T12:21:08.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:08.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:08.555451+0000) 2024-03-06T12:21:09.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:09.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:09.555711+0000) 2024-03-06T12:21:10.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:10.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:10.555943+0000) 2024-03-06T12:21:11.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:11.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:11.556197+0000) 2024-03-06T12:21:12.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:12.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:12.556434+0000) 2024-03-06T12:21:13.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:13.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:13.556672+0000) 2024-03-06T12:21:14.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:14.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:14.556915+0000) 2024-03-06T12:21:15.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:15.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:15.557152+0000) 2024-03-06T12:21:15.889 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:21:15.898 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:21:16.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:16.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:16.557360+0000) 2024-03-06T12:21:17.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:17.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:17.557642+0000) 2024-03-06T12:21:18.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:18.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:18.557856+0000) 2024-03-06T12:21:19.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:19.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:19.558065+0000) 2024-03-06T12:21:20.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:20.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:20.558302+0000) 2024-03-06T12:21:21.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:21.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:21.558589+0000) 2024-03-06T12:21:22.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:22.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:22.558806+0000) 2024-03-06T12:21:23.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:23.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:23.559046+0000) 2024-03-06T12:21:24.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:24.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:24.559263+0000) 2024-03-06T12:21:25.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:25.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:25.559499+0000) 2024-03-06T12:21:26.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:26.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:26.559737+0000) 2024-03-06T12:21:27.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:27.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:27.559973+0000) 2024-03-06T12:21:28.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:28.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:28.560150+0000) 2024-03-06T12:21:29.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:29.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:29.560377+0000) 2024-03-06T12:21:30.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:30.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:30.560633+0000) 2024-03-06T12:21:31.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:31.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:31.560873+0000) 2024-03-06T12:21:32.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:32.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:32.561103+0000) 2024-03-06T12:21:33.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:33.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:33.561334+0000) 2024-03-06T12:21:34.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:34.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:34.561600+0000) 2024-03-06T12:21:35.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:35.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:35.561794+0000) 2024-03-06T12:21:36.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:36.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:36.562032+0000) 2024-03-06T12:21:37.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:37.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:37.562231+0000) 2024-03-06T12:21:38.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:38.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:38.562455+0000) 2024-03-06T12:21:39.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:39.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:39.562683+0000) 2024-03-06T12:21:40.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:40.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:40.562914+0000) 2024-03-06T12:21:41.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:41.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:41.563153+0000) 2024-03-06T12:21:42.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:42.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:42.563383+0000) 2024-03-06T12:21:43.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:43.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:43.563625+0000) 2024-03-06T12:21:44.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:44.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:44.563864+0000) 2024-03-06T12:21:45.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:45.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:45.564091+0000) 2024-03-06T12:21:46.078 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:21:46.088 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:21:46.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:46.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:46.564329+0000) 2024-03-06T12:21:47.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:47.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:47.564516+0000) 2024-03-06T12:21:48.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:48.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:48.564716+0000) 2024-03-06T12:21:49.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:49.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:49.564905+0000) 2024-03-06T12:21:50.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:50.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:50.565097+0000) 2024-03-06T12:21:51.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:51.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:51.565300+0000) 2024-03-06T12:21:52.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:52.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:52.565508+0000) 2024-03-06T12:21:53.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:53.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:53.565781+0000) 2024-03-06T12:21:54.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:54.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:54.566005+0000) 2024-03-06T12:21:55.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:55.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:55.566272+0000) 2024-03-06T12:21:56.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:56.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:56.566563+0000) 2024-03-06T12:21:57.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:57.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:57.566797+0000) 2024-03-06T12:21:58.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:58.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:58.567040+0000) 2024-03-06T12:21:59.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:21:59.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:21:59.567281+0000) 2024-03-06T12:22:00.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:00.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:00.567499+0000) 2024-03-06T12:22:01.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:01.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:01.567774+0000) 2024-03-06T12:22:02.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:02.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:02.568035+0000) 2024-03-06T12:22:03.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:03.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:03.568278+0000) 2024-03-06T12:22:04.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:04.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:04.568529+0000) 2024-03-06T12:22:05.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:05.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:05.568794+0000) 2024-03-06T12:22:06.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:06.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:06.569059+0000) 2024-03-06T12:22:07.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:07.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:07.569338+0000) 2024-03-06T12:22:08.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:08.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:08.569646+0000) 2024-03-06T12:22:09.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:09.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:09.569891+0000) 2024-03-06T12:22:10.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:10.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:10.570122+0000) 2024-03-06T12:22:11.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:11.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:11.570379+0000) 2024-03-06T12:22:12.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:12.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:12.570638+0000) 2024-03-06T12:22:13.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:13.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:13.570830+0000) 2024-03-06T12:22:14.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:14.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:14.571021+0000) 2024-03-06T12:22:15.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:15.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:15.571285+0000) 2024-03-06T12:22:16.215 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:22:16.223 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:22:16.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:16.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:16.571621+0000) 2024-03-06T12:22:17.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:17.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:17.571869+0000) 2024-03-06T12:22:18.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:18.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:18.572148+0000) 2024-03-06T12:22:19.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:19.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:19.572418+0000) 2024-03-06T12:22:20.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:20.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:20.572717+0000) 2024-03-06T12:22:21.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:21.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:21.572974+0000) 2024-03-06T12:22:22.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:22.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:22.573171+0000) 2024-03-06T12:22:23.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:23.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:23.573355+0000) 2024-03-06T12:22:24.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:24.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:24.573586+0000) 2024-03-06T12:22:25.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:25.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:25.573789+0000) 2024-03-06T12:22:26.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:26.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:26.574017+0000) 2024-03-06T12:22:27.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:27.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:27.574254+0000) 2024-03-06T12:22:28.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:28.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:28.574523+0000) 2024-03-06T12:22:29.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:29.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:29.574813+0000) 2024-03-06T12:22:30.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:30.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:30.575014+0000) 2024-03-06T12:22:31.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:31.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:31.575254+0000) 2024-03-06T12:22:32.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:32.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:32.575481+0000) 2024-03-06T12:22:33.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:33.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:33.575764+0000) 2024-03-06T12:22:34.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:34.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:34.576006+0000) 2024-03-06T12:22:35.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:35.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:35.576279+0000) 2024-03-06T12:22:36.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:36.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:36.576568+0000) 2024-03-06T12:22:37.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:37.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:37.576767+0000) 2024-03-06T12:22:38.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:38.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:38.576970+0000) 2024-03-06T12:22:39.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:39.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:39.577192+0000) 2024-03-06T12:22:40.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:40.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:40.577372+0000) 2024-03-06T12:22:41.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:41.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:41.577646+0000) 2024-03-06T12:22:42.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:42.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:42.577839+0000) 2024-03-06T12:22:43.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:43.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:43.578060+0000) 2024-03-06T12:22:44.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:44.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:44.578290+0000) 2024-03-06T12:22:45.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:45.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:45.578529+0000) 2024-03-06T12:22:46.428 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:22:46.437 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:22:46.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:46.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:46.578758+0000) 2024-03-06T12:22:47.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:47.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:47.578997+0000) 2024-03-06T12:22:48.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:48.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:48.579241+0000) 2024-03-06T12:22:49.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:49.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:49.579433+0000) 2024-03-06T12:22:50.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:50.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:50.579692+0000) 2024-03-06T12:22:51.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:51.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:51.579901+0000) 2024-03-06T12:22:52.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:52.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:52.580101+0000) 2024-03-06T12:22:53.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:53.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:53.580326+0000) 2024-03-06T12:22:54.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:54.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:54.580591+0000) 2024-03-06T12:22:55.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:55.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:55.580798+0000) 2024-03-06T12:22:56.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:56.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:56.581035+0000) 2024-03-06T12:22:57.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:57.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:57.581279+0000) 2024-03-06T12:22:58.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:58.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:58.581507+0000) 2024-03-06T12:22:59.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:22:59.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:22:59.581744+0000) 2024-03-06T12:23:00.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:00.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:00.581973+0000) 2024-03-06T12:23:01.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:01.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:01.582218+0000) 2024-03-06T12:23:02.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:02.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:02.582404+0000) 2024-03-06T12:23:03.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:03.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:03.582671+0000) 2024-03-06T12:23:04.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:04.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:04.582894+0000) 2024-03-06T12:23:05.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:05.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:05.583096+0000) 2024-03-06T12:23:06.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:06.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:06.583367+0000) 2024-03-06T12:23:07.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:07.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:07.583667+0000) 2024-03-06T12:23:08.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:08.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:08.583805+0000) 2024-03-06T12:23:09.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:09.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:09.584006+0000) 2024-03-06T12:23:10.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:10.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:10.584237+0000) 2024-03-06T12:23:11.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:11.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:11.584472+0000) 2024-03-06T12:23:12.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:12.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:12.584689+0000) 2024-03-06T12:23:13.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:13.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:13.584921+0000) 2024-03-06T12:23:14.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:14.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:14.585099+0000) 2024-03-06T12:23:15.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:15.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:15.585333+0000) 2024-03-06T12:23:16.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:16.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:16.585603+0000) 2024-03-06T12:23:16.631 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:23:16.643 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:23:17.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:17.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:17.585808+0000) 2024-03-06T12:23:18.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:18.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:18.586033+0000) 2024-03-06T12:23:19.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:19.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:19.586254+0000) 2024-03-06T12:23:20.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:20.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:20.586475+0000) 2024-03-06T12:23:21.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:21.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:21.586691+0000) 2024-03-06T12:23:22.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:22.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:22.586881+0000) 2024-03-06T12:23:23.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:23.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:23.587068+0000) 2024-03-06T12:23:24.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:24.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:24.587294+0000) 2024-03-06T12:23:25.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:25.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:25.587521+0000) 2024-03-06T12:23:26.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:26.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:26.587849+0000) 2024-03-06T12:23:27.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:27.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:27.588073+0000) 2024-03-06T12:23:28.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:28.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:28.588301+0000) 2024-03-06T12:23:29.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:29.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:29.588561+0000) 2024-03-06T12:23:30.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:30.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:30.588789+0000) 2024-03-06T12:23:31.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:31.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:31.589004+0000) 2024-03-06T12:23:32.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:32.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:32.589232+0000) 2024-03-06T12:23:33.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:33.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:33.589460+0000) 2024-03-06T12:23:34.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:34.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:34.589701+0000) 2024-03-06T12:23:35.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:35.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:35.589921+0000) 2024-03-06T12:23:36.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:36.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:36.590104+0000) 2024-03-06T12:23:37.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:37.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:37.590331+0000) 2024-03-06T12:23:38.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:38.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:38.590596+0000) 2024-03-06T12:23:39.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:39.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:39.590787+0000) 2024-03-06T12:23:40.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:40.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:40.590982+0000) 2024-03-06T12:23:41.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:41.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:41.591225+0000) 2024-03-06T12:23:42.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:42.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:42.591451+0000) 2024-03-06T12:23:43.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:43.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:43.591676+0000) 2024-03-06T12:23:44.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:44.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:44.591872+0000) 2024-03-06T12:23:45.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:45.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:45.592072+0000) 2024-03-06T12:23:46.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:46.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:46.592304+0000) 2024-03-06T12:23:46.837 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:23:46.845 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:23:47.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:47.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:47.592511+0000) 2024-03-06T12:23:48.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:48.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:48.592748+0000) 2024-03-06T12:23:49.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:49.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:49.592937+0000) 2024-03-06T12:23:50.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:50.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:50.593130+0000) 2024-03-06T12:23:51.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:51.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:51.593356+0000) 2024-03-06T12:23:52.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:52.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:52.593626+0000) 2024-03-06T12:23:53.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:53.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:53.593822+0000) 2024-03-06T12:23:54.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:54.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:54.594029+0000) 2024-03-06T12:23:55.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:55.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:55.594222+0000) 2024-03-06T12:23:56.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:56.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:56.594461+0000) 2024-03-06T12:23:57.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:57.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:57.594704+0000) 2024-03-06T12:23:58.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:58.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:58.594947+0000) 2024-03-06T12:23:59.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:23:59.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:23:59.595151+0000) 2024-03-06T12:24:00.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:00.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:00.595379+0000) 2024-03-06T12:24:01.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:01.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:01.595683+0000) 2024-03-06T12:24:02.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:02.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:02.595909+0000) 2024-03-06T12:24:03.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:03.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:03.596115+0000) 2024-03-06T12:24:04.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:04.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:04.596372+0000) 2024-03-06T12:24:05.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:05.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:05.596673+0000) 2024-03-06T12:24:06.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:06.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:06.596880+0000) 2024-03-06T12:24:07.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:07.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:07.597105+0000) 2024-03-06T12:24:08.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:08.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:08.597318+0000) 2024-03-06T12:24:09.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:09.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:09.597580+0000) 2024-03-06T12:24:10.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:10.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:10.597824+0000) 2024-03-06T12:24:11.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:11.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:11.598067+0000) 2024-03-06T12:24:12.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:12.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:12.598264+0000) 2024-03-06T12:24:13.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:13.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:13.598508+0000) 2024-03-06T12:24:14.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:14.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:14.598803+0000) 2024-03-06T12:24:15.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:15.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:15.599041+0000) 2024-03-06T12:24:16.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:16.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:16.599293+0000) 2024-03-06T12:24:17.032 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:24:17.040 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:24:17.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:17.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:17.599576+0000) 2024-03-06T12:24:18.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:18.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:18.599827+0000) 2024-03-06T12:24:19.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:19.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:19.600054+0000) 2024-03-06T12:24:20.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:20.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:20.600302+0000) 2024-03-06T12:24:21.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:21.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:21.600507+0000) 2024-03-06T12:24:22.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:22.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:22.600785+0000) 2024-03-06T12:24:23.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:23.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:23.601012+0000) 2024-03-06T12:24:24.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:24.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:24.601302+0000) 2024-03-06T12:24:25.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:25.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:25.601625+0000) 2024-03-06T12:24:26.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:26.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:26.601912+0000) 2024-03-06T12:24:27.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:27.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:27.602157+0000) 2024-03-06T12:24:28.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:28.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:28.602437+0000) 2024-03-06T12:24:29.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:29.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:29.602654+0000) 2024-03-06T12:24:30.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:30.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:30.602885+0000) 2024-03-06T12:24:31.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:31.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:31.603113+0000) 2024-03-06T12:24:32.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:32.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:32.603399+0000) 2024-03-06T12:24:33.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:33.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:33.603678+0000) 2024-03-06T12:24:34.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:34.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:34.603909+0000) 2024-03-06T12:24:35.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:35.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:35.604176+0000) 2024-03-06T12:24:36.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:36.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:36.604456+0000) 2024-03-06T12:24:37.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:37.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:37.604722+0000) 2024-03-06T12:24:38.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:38.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:38.604939+0000) 2024-03-06T12:24:39.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:39.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:39.605167+0000) 2024-03-06T12:24:40.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:40.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:40.605386+0000) 2024-03-06T12:24:41.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:41.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:41.605642+0000) 2024-03-06T12:24:42.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:42.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:42.605877+0000) 2024-03-06T12:24:43.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:43.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:43.606104+0000) 2024-03-06T12:24:44.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:44.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:44.606287+0000) 2024-03-06T12:24:45.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:45.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:45.606512+0000) 2024-03-06T12:24:46.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:46.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:46.606786+0000) 2024-03-06T12:24:47.195 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:24:47.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:24:47.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:47.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:47.607036+0000) 2024-03-06T12:24:48.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:48.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:48.607258+0000) 2024-03-06T12:24:49.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:49.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:49.607525+0000) 2024-03-06T12:24:50.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:50.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:50.607759+0000) 2024-03-06T12:24:51.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:51.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:51.607946+0000) 2024-03-06T12:24:52.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:52.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:52.608142+0000) 2024-03-06T12:24:53.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:53.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:53.608334+0000) 2024-03-06T12:24:54.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:54.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:54.608562+0000) 2024-03-06T12:24:55.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:55.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:55.608776+0000) 2024-03-06T12:24:56.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:56.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:56.609005+0000) 2024-03-06T12:24:57.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:57.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:57.609214+0000) 2024-03-06T12:24:58.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:58.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:58.609438+0000) 2024-03-06T12:24:59.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:24:59.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:24:59.609708+0000) 2024-03-06T12:25:00.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:00.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:00.609903+0000) 2024-03-06T12:25:01.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:01.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:01.610135+0000) 2024-03-06T12:25:02.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:02.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:02.610363+0000) 2024-03-06T12:25:03.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:03.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:03.610639+0000) 2024-03-06T12:25:04.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:04.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:04.610867+0000) 2024-03-06T12:25:05.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:05.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:05.611095+0000) 2024-03-06T12:25:06.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:06.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:06.611324+0000) 2024-03-06T12:25:07.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:07.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:07.611583+0000) 2024-03-06T12:25:08.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:08.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:08.611834+0000) 2024-03-06T12:25:09.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:09.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:09.612030+0000) 2024-03-06T12:25:10.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:10.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:10.612249+0000) 2024-03-06T12:25:11.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:11.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:11.612476+0000) 2024-03-06T12:25:12.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:12.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:12.612660+0000) 2024-03-06T12:25:13.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:13.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:13.612856+0000) 2024-03-06T12:25:14.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:14.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:14.613164+0000) 2024-03-06T12:25:15.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:15.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:15.613395+0000) 2024-03-06T12:25:16.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:16.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:16.613650+0000) 2024-03-06T12:25:17.399 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:25:17.407 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:25:17.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:17.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:17.613807+0000) 2024-03-06T12:25:18.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:18.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:18.614019+0000) 2024-03-06T12:25:19.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:19.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:19.614235+0000) 2024-03-06T12:25:20.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:20.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:20.614473+0000) 2024-03-06T12:25:21.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:21.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:21.614682+0000) 2024-03-06T12:25:22.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:22.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:22.614881+0000) 2024-03-06T12:25:23.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:23.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:23.615101+0000) 2024-03-06T12:25:24.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:24.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:24.615333+0000) 2024-03-06T12:25:25.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:25.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:25.615634+0000) 2024-03-06T12:25:26.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:26.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:26.615892+0000) 2024-03-06T12:25:27.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:27.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:27.616172+0000) 2024-03-06T12:25:28.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:28.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:28.616386+0000) 2024-03-06T12:25:29.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:29.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:29.616666+0000) 2024-03-06T12:25:30.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:30.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:30.616910+0000) 2024-03-06T12:25:31.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:31.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:31.617091+0000) 2024-03-06T12:25:32.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:32.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:32.617323+0000) 2024-03-06T12:25:33.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:33.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:33.617596+0000) 2024-03-06T12:25:34.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:34.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:34.617794+0000) 2024-03-06T12:25:35.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:35.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:35.618014+0000) 2024-03-06T12:25:36.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:36.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:36.618272+0000) 2024-03-06T12:25:37.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:37.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:37.618503+0000) 2024-03-06T12:25:38.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:38.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:38.618739+0000) 2024-03-06T12:25:39.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:39.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:39.618972+0000) 2024-03-06T12:25:40.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:40.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:40.619201+0000) 2024-03-06T12:25:41.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:41.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:41.619427+0000) 2024-03-06T12:25:42.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:42.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:42.619716+0000) 2024-03-06T12:25:43.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:43.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:43.619995+0000) 2024-03-06T12:25:44.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:44.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:44.620195+0000) 2024-03-06T12:25:45.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:45.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:45.620469+0000) 2024-03-06T12:25:46.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:46.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:46.620736+0000) 2024-03-06T12:25:47.612 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:25:47.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:25:47.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:47.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:47.620993+0000) 2024-03-06T12:25:48.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:48.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:48.621245+0000) 2024-03-06T12:25:49.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:49.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:49.621475+0000) 2024-03-06T12:25:50.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:50.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:50.621708+0000) 2024-03-06T12:25:51.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:51.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:51.621948+0000) 2024-03-06T12:25:52.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:52.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:52.622145+0000) 2024-03-06T12:25:53.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:53.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:53.622382+0000) 2024-03-06T12:25:54.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:54.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:54.622632+0000) 2024-03-06T12:25:55.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:55.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:55.622832+0000) 2024-03-06T12:25:56.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:56.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:56.623100+0000) 2024-03-06T12:25:57.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:57.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:57.623296+0000) 2024-03-06T12:25:58.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:58.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:58.623523+0000) 2024-03-06T12:25:59.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:25:59.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:25:59.623748+0000) 2024-03-06T12:26:00.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:00.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:00.623943+0000) 2024-03-06T12:26:01.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:01.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:01.624138+0000) 2024-03-06T12:26:02.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:02.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:02.624386+0000) 2024-03-06T12:26:03.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:03.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:03.624609+0000) 2024-03-06T12:26:04.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:04.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:04.624843+0000) 2024-03-06T12:26:05.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:05.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:05.625033+0000) 2024-03-06T12:26:06.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:06.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:06.625240+0000) 2024-03-06T12:26:07.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:07.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:07.625454+0000) 2024-03-06T12:26:08.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:08.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:08.625696+0000) 2024-03-06T12:26:09.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:09.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:09.625891+0000) 2024-03-06T12:26:10.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:10.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:10.626130+0000) 2024-03-06T12:26:11.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:11.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:11.626322+0000) 2024-03-06T12:26:12.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:12.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:12.626614+0000) 2024-03-06T12:26:13.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:13.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:13.626847+0000) 2024-03-06T12:26:14.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:14.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:14.627033+0000) 2024-03-06T12:26:15.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:15.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:15.627253+0000) 2024-03-06T12:26:16.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:16.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:16.627442+0000) 2024-03-06T12:26:17.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:17.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:17.627683+0000) 2024-03-06T12:26:17.929 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:26:17.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:26:18.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:18.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:18.627885+0000) 2024-03-06T12:26:19.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:19.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:19.628093+0000) 2024-03-06T12:26:20.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:20.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:20.628325+0000) 2024-03-06T12:26:21.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:21.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:21.628582+0000) 2024-03-06T12:26:22.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:22.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:22.628778+0000) 2024-03-06T12:26:23.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:23.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:23.628962+0000) 2024-03-06T12:26:24.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:24.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:24.629199+0000) 2024-03-06T12:26:25.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:25.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:25.629440+0000) 2024-03-06T12:26:26.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:26.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:26.629647+0000) 2024-03-06T12:26:27.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:27.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:27.629899+0000) 2024-03-06T12:26:28.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:28.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:28.630136+0000) 2024-03-06T12:26:29.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:29.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:29.630362+0000) 2024-03-06T12:26:30.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:30.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:30.630627+0000) 2024-03-06T12:26:31.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:31.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:31.630778+0000) 2024-03-06T12:26:32.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:32.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:32.631015+0000) 2024-03-06T12:26:33.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:33.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:33.631247+0000) 2024-03-06T12:26:34.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:34.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:34.631481+0000) 2024-03-06T12:26:35.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:35.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:35.631678+0000) 2024-03-06T12:26:36.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:36.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:36.631887+0000) 2024-03-06T12:26:37.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:37.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:37.632089+0000) 2024-03-06T12:26:38.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:38.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:38.632298+0000) 2024-03-06T12:26:39.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:39.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:39.632574+0000) 2024-03-06T12:26:40.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:40.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:40.632772+0000) 2024-03-06T12:26:41.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:41.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:41.632993+0000) 2024-03-06T12:26:42.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:42.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:42.633227+0000) 2024-03-06T12:26:43.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:43.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:43.633447+0000) 2024-03-06T12:26:44.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:44.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:44.633736+0000) 2024-03-06T12:26:45.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:45.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:45.633935+0000) 2024-03-06T12:26:46.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:46.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:46.634131+0000) 2024-03-06T12:26:47.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:47.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:47.634367+0000) 2024-03-06T12:26:48.138 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:26:48.146 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:26:48.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:48.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:48.634650+0000) 2024-03-06T12:26:49.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:49.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:49.634885+0000) 2024-03-06T12:26:50.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:50.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:50.635081+0000) 2024-03-06T12:26:51.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:51.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:51.635302+0000) 2024-03-06T12:26:52.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:52.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:52.635559+0000) 2024-03-06T12:26:53.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:53.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:53.635788+0000) 2024-03-06T12:26:54.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:54.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:54.636025+0000) 2024-03-06T12:26:55.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:55.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:55.636257+0000) 2024-03-06T12:26:56.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:56.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:56.636499+0000) 2024-03-06T12:26:57.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:57.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:57.636667+0000) 2024-03-06T12:26:58.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:58.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:58.636857+0000) 2024-03-06T12:26:59.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:26:59.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:26:59.637049+0000) 2024-03-06T12:27:00.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:00.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:00.637273+0000) 2024-03-06T12:27:01.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:01.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:01.637459+0000) 2024-03-06T12:27:02.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:02.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:02.637719+0000) 2024-03-06T12:27:03.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:03.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:03.637919+0000) 2024-03-06T12:27:04.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:04.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:04.638130+0000) 2024-03-06T12:27:05.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:05.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:05.638370+0000) 2024-03-06T12:27:06.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:06.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:06.638646+0000) 2024-03-06T12:27:07.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:07.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:07.638883+0000) 2024-03-06T12:27:08.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:08.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:08.639083+0000) 2024-03-06T12:27:09.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:09.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:09.639306+0000) 2024-03-06T12:27:10.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:10.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:10.639513+0000) 2024-03-06T12:27:11.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:11.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:11.639836+0000) 2024-03-06T12:27:12.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:12.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:12.640091+0000) 2024-03-06T12:27:13.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:13.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:13.640295+0000) 2024-03-06T12:27:14.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:14.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:14.640519+0000) 2024-03-06T12:27:15.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:15.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:15.640749+0000) 2024-03-06T12:27:16.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:16.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:16.641010+0000) 2024-03-06T12:27:17.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:17.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:17.641302+0000) 2024-03-06T12:27:18.282 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:27:18.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:27:18.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:18.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:18.641625+0000) 2024-03-06T12:27:19.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:19.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:19.641903+0000) 2024-03-06T12:27:20.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:20.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:20.642146+0000) 2024-03-06T12:27:21.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:21.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:21.642408+0000) 2024-03-06T12:27:22.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:22.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:22.642698+0000) 2024-03-06T12:27:23.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:23.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:23.642927+0000) 2024-03-06T12:27:24.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:24.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:24.643180+0000) 2024-03-06T12:27:25.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:25.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:25.643455+0000) 2024-03-06T12:27:26.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:26.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:26.643722+0000) 2024-03-06T12:27:27.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:27.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:27.643967+0000) 2024-03-06T12:27:28.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:28.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:28.644239+0000) 2024-03-06T12:27:29.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:29.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:29.644514+0000) 2024-03-06T12:27:30.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:30.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:30.644790+0000) 2024-03-06T12:27:31.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:31.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:31.645027+0000) 2024-03-06T12:27:32.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:32.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:32.645256+0000) 2024-03-06T12:27:33.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:33.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:33.645471+0000) 2024-03-06T12:27:34.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:34.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:34.645706+0000) 2024-03-06T12:27:35.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:35.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:35.645933+0000) 2024-03-06T12:27:36.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:36.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:36.646178+0000) 2024-03-06T12:27:37.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:37.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:37.646452+0000) 2024-03-06T12:27:38.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:38.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:38.646715+0000) 2024-03-06T12:27:39.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:39.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:39.646958+0000) 2024-03-06T12:27:40.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:40.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:40.647204+0000) 2024-03-06T12:27:41.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:41.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:41.647440+0000) 2024-03-06T12:27:42.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:42.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:42.647720+0000) 2024-03-06T12:27:43.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:43.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:43.647998+0000) 2024-03-06T12:27:44.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:44.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:44.648226+0000) 2024-03-06T12:27:45.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:45.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:45.648489+0000) 2024-03-06T12:27:46.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:46.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:46.648773+0000) 2024-03-06T12:27:47.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:47.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:47.649025+0000) 2024-03-06T12:27:48.497 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:27:48.509 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:27:48.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:48.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:48.649314+0000) 2024-03-06T12:27:49.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:49.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:49.649635+0000) 2024-03-06T12:27:50.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:50.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:50.649799+0000) 2024-03-06T12:27:51.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:51.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:51.649987+0000) 2024-03-06T12:27:52.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:52.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:52.650181+0000) 2024-03-06T12:27:53.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:53.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:53.650451+0000) 2024-03-06T12:27:54.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:54.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:54.650691+0000) 2024-03-06T12:27:55.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:55.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:55.650913+0000) 2024-03-06T12:27:56.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:56.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:56.651110+0000) 2024-03-06T12:27:57.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:57.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:57.651343+0000) 2024-03-06T12:27:58.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:58.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:58.651609+0000) 2024-03-06T12:27:59.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:27:59.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:27:59.651763+0000) 2024-03-06T12:28:00.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:00.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:00.651956+0000) 2024-03-06T12:28:01.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:01.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:01.652179+0000) 2024-03-06T12:28:02.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:02.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:02.652410+0000) 2024-03-06T12:28:03.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:03.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:03.652637+0000) 2024-03-06T12:28:04.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:04.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:04.652836+0000) 2024-03-06T12:28:05.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:05.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:05.653059+0000) 2024-03-06T12:28:06.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:06.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:06.653289+0000) 2024-03-06T12:28:07.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:07.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:07.653558+0000) 2024-03-06T12:28:08.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:08.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:08.653750+0000) 2024-03-06T12:28:09.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:09.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:09.653946+0000) 2024-03-06T12:28:10.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:10.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:10.654152+0000) 2024-03-06T12:28:11.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:11.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:11.654391+0000) 2024-03-06T12:28:12.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:12.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:12.654669+0000) 2024-03-06T12:28:13.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:13.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:13.654911+0000) 2024-03-06T12:28:14.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:14.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:14.655149+0000) 2024-03-06T12:28:15.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:15.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:15.655409+0000) 2024-03-06T12:28:16.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:16.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:16.655665+0000) 2024-03-06T12:28:17.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:17.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:17.655885+0000) 2024-03-06T12:28:18.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:18.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:18.656108+0000) 2024-03-06T12:28:18.736 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:28:18.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:28:19.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:19.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:19.656304+0000) 2024-03-06T12:28:20.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:20.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:20.656563+0000) 2024-03-06T12:28:21.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:21.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:21.656754+0000) 2024-03-06T12:28:22.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:22.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:22.656965+0000) 2024-03-06T12:28:23.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:23.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:23.657146+0000) 2024-03-06T12:28:24.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:24.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:24.657370+0000) 2024-03-06T12:28:25.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:25.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:25.657637+0000) 2024-03-06T12:28:26.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:26.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:26.657796+0000) 2024-03-06T12:28:27.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:27.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:27.657999+0000) 2024-03-06T12:28:28.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:28.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:28.658229+0000) 2024-03-06T12:28:29.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:29.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:29.658435+0000) 2024-03-06T12:28:30.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:30.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:30.658659+0000) 2024-03-06T12:28:31.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:31.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:31.658844+0000) 2024-03-06T12:28:32.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:32.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:32.659049+0000) 2024-03-06T12:28:33.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:33.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:33.659272+0000) 2024-03-06T12:28:34.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:34.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:34.659493+0000) 2024-03-06T12:28:35.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:35.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:35.659691+0000) 2024-03-06T12:28:36.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:36.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:36.659908+0000) 2024-03-06T12:28:37.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:37.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:37.660130+0000) 2024-03-06T12:28:38.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:38.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:38.660356+0000) 2024-03-06T12:28:39.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:39.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:39.660612+0000) 2024-03-06T12:28:40.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:40.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:40.660817+0000) 2024-03-06T12:28:41.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:41.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:41.661023+0000) 2024-03-06T12:28:42.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:42.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:42.661274+0000) 2024-03-06T12:28:43.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:43.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:43.661515+0000) 2024-03-06T12:28:44.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:44.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:44.661760+0000) 2024-03-06T12:28:45.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:45.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:45.661967+0000) 2024-03-06T12:28:46.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:46.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:46.662217+0000) 2024-03-06T12:28:47.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:47.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:47.662468+0000) 2024-03-06T12:28:48.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:48.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:48.662688+0000) 2024-03-06T12:28:48.955 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:28:48.963 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:28:49.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:49.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:49.662853+0000) 2024-03-06T12:28:50.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:50.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:50.663077+0000) 2024-03-06T12:28:51.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:51.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:51.663307+0000) 2024-03-06T12:28:52.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:52.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:52.663579+0000) 2024-03-06T12:28:53.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:53.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:53.663829+0000) 2024-03-06T12:28:54.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:54.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:54.664094+0000) 2024-03-06T12:28:55.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:55.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:55.664331+0000) 2024-03-06T12:28:56.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:56.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:56.664606+0000) 2024-03-06T12:28:57.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:57.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:57.664829+0000) 2024-03-06T12:28:58.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:58.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:58.665040+0000) 2024-03-06T12:28:59.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:28:59.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:28:59.665258+0000) 2024-03-06T12:29:00.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:00.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:00.665496+0000) 2024-03-06T12:29:01.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:01.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:01.665736+0000) 2024-03-06T12:29:02.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:02.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:02.665949+0000) 2024-03-06T12:29:03.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:03.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:03.666164+0000) 2024-03-06T12:29:04.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:04.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:04.666399+0000) 2024-03-06T12:29:05.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:05.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:05.666679+0000) 2024-03-06T12:29:06.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:06.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:06.666970+0000) 2024-03-06T12:29:07.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:07.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:07.667178+0000) 2024-03-06T12:29:08.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:08.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:08.667436+0000) 2024-03-06T12:29:09.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:09.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:09.667726+0000) 2024-03-06T12:29:10.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:10.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:10.667928+0000) 2024-03-06T12:29:11.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:11.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:11.668149+0000) 2024-03-06T12:29:12.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:12.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:12.668348+0000) 2024-03-06T12:29:13.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:13.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:13.668621+0000) 2024-03-06T12:29:14.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:14.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:14.668852+0000) 2024-03-06T12:29:15.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:15.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:15.669076+0000) 2024-03-06T12:29:16.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:16.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:16.669367+0000) 2024-03-06T12:29:17.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:17.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:17.669681+0000) 2024-03-06T12:29:18.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:18.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:18.670003+0000) 2024-03-06T12:29:19.194 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:29:19.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:29:19.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:19.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:19.670290+0000) 2024-03-06T12:29:20.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:20.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:20.670616+0000) 2024-03-06T12:29:21.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:21.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:21.670772+0000) 2024-03-06T12:29:22.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:22.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:22.670978+0000) 2024-03-06T12:29:23.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:23.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:23.671172+0000) 2024-03-06T12:29:24.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:24.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:24.671390+0000) 2024-03-06T12:29:25.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:25.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:25.671704+0000) 2024-03-06T12:29:26.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:26.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:26.672020+0000) 2024-03-06T12:29:27.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:27.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:27.672309+0000) 2024-03-06T12:29:28.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:28.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:28.672572+0000) 2024-03-06T12:29:29.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:29.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:29.672814+0000) 2024-03-06T12:29:30.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:30.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:30.673093+0000) 2024-03-06T12:29:31.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:31.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:31.673355+0000) 2024-03-06T12:29:32.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:32.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:32.673676+0000) 2024-03-06T12:29:33.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:33.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:33.673960+0000) 2024-03-06T12:29:34.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:34.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:34.674248+0000) 2024-03-06T12:29:35.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:35.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:35.674583+0000) 2024-03-06T12:29:36.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:36.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:36.674889+0000) 2024-03-06T12:29:37.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:37.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:37.675109+0000) 2024-03-06T12:29:38.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:38.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:38.675354+0000) 2024-03-06T12:29:39.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:39.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:39.675629+0000) 2024-03-06T12:29:40.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:40.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:40.675834+0000) 2024-03-06T12:29:41.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:41.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:41.676026+0000) 2024-03-06T12:29:42.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:42.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:42.676257+0000) 2024-03-06T12:29:43.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:43.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:43.676438+0000) 2024-03-06T12:29:44.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:44.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:44.676667+0000) 2024-03-06T12:29:45.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:45.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:45.676869+0000) 2024-03-06T12:29:46.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:46.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:46.677069+0000) 2024-03-06T12:29:47.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:47.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:47.677292+0000) 2024-03-06T12:29:48.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:48.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:48.677523+0000) 2024-03-06T12:29:49.331 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:29:49.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:29:49.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:49.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:49.677753+0000) 2024-03-06T12:29:50.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:50.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:50.677950+0000) 2024-03-06T12:29:51.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:51.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:51.678141+0000) 2024-03-06T12:29:52.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:52.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:52.678368+0000) 2024-03-06T12:29:53.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:53.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:53.678637+0000) 2024-03-06T12:29:54.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:54.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:54.678826+0000) 2024-03-06T12:29:55.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:55.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:55.679014+0000) 2024-03-06T12:29:56.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:56.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:56.679254+0000) 2024-03-06T12:29:57.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:57.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:57.679446+0000) 2024-03-06T12:29:58.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:58.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:58.679661+0000) 2024-03-06T12:29:59.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:29:59.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:29:59.679873+0000) 2024-03-06T12:30:00.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:00.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:00.680050+0000) 2024-03-06T12:30:01.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:01.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:01.680326+0000) 2024-03-06T12:30:02.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:02.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:02.680578+0000) 2024-03-06T12:30:03.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:03.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:03.680778+0000) 2024-03-06T12:30:04.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:04.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:04.680973+0000) 2024-03-06T12:30:05.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:05.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:05.681166+0000) 2024-03-06T12:30:06.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:06.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:06.681405+0000) 2024-03-06T12:30:07.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:07.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:07.681632+0000) 2024-03-06T12:30:08.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:08.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:08.681864+0000) 2024-03-06T12:30:09.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:09.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:09.682094+0000) 2024-03-06T12:30:10.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:10.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:10.682328+0000) 2024-03-06T12:30:11.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:11.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:11.682564+0000) 2024-03-06T12:30:12.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:12.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:12.682774+0000) 2024-03-06T12:30:13.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:13.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:13.683016+0000) 2024-03-06T12:30:14.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:14.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:14.683217+0000) 2024-03-06T12:30:15.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:15.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:15.683458+0000) 2024-03-06T12:30:16.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:16.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:16.683681+0000) 2024-03-06T12:30:17.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:17.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:17.683861+0000) 2024-03-06T12:30:18.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:18.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:18.684058+0000) 2024-03-06T12:30:19.519 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:30:19.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:30:19.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:19.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:19.684247+0000) 2024-03-06T12:30:20.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:20.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:20.684422+0000) 2024-03-06T12:30:21.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:21.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:21.684671+0000) 2024-03-06T12:30:22.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:22.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:22.684866+0000) 2024-03-06T12:30:23.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:23.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:23.685066+0000) 2024-03-06T12:30:24.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:24.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:24.685257+0000) 2024-03-06T12:30:25.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:25.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:25.685479+0000) 2024-03-06T12:30:26.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:26.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:26.685692+0000) 2024-03-06T12:30:27.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:27.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:27.685887+0000) 2024-03-06T12:30:28.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:28.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:28.686099+0000) 2024-03-06T12:30:29.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:29.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:29.686280+0000) 2024-03-06T12:30:30.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:30.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:30.686479+0000) 2024-03-06T12:30:31.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:31.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:31.686684+0000) 2024-03-06T12:30:32.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:32.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:32.686889+0000) 2024-03-06T12:30:33.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:33.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:33.687092+0000) 2024-03-06T12:30:34.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:34.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:34.687321+0000) 2024-03-06T12:30:35.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:35.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:35.687568+0000) 2024-03-06T12:30:36.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:36.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:36.687779+0000) 2024-03-06T12:30:37.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:37.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:37.687996+0000) 2024-03-06T12:30:38.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:38.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:38.688247+0000) 2024-03-06T12:30:39.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:39.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:39.688470+0000) 2024-03-06T12:30:40.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:40.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:40.688668+0000) 2024-03-06T12:30:41.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:41.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:41.688899+0000) 2024-03-06T12:30:42.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:42.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:42.689090+0000) 2024-03-06T12:30:43.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:43.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:43.689316+0000) 2024-03-06T12:30:44.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:44.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:44.689518+0000) 2024-03-06T12:30:45.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:45.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:45.689735+0000) 2024-03-06T12:30:46.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:46.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:46.689925+0000) 2024-03-06T12:30:47.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:47.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:47.690114+0000) 2024-03-06T12:30:48.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:48.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:48.690341+0000) 2024-03-06T12:30:49.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:49.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:49.690625+0000) 2024-03-06T12:30:49.748 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:30:49.756 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:30:50.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:50.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:50.690818+0000) 2024-03-06T12:30:51.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:51.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:51.691028+0000) 2024-03-06T12:30:52.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:52.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:52.691288+0000) 2024-03-06T12:30:53.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:53.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:53.691572+0000) 2024-03-06T12:30:54.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:54.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:54.691823+0000) 2024-03-06T12:30:55.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:55.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:55.692008+0000) 2024-03-06T12:30:56.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:56.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:56.692254+0000) 2024-03-06T12:30:57.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:57.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:57.692487+0000) 2024-03-06T12:30:58.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:58.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:58.692660+0000) 2024-03-06T12:30:59.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:30:59.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:30:59.692900+0000) 2024-03-06T12:31:00.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:00.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:00.693169+0000) 2024-03-06T12:31:01.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:01.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:01.693366+0000) 2024-03-06T12:31:02.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:02.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:02.693596+0000) 2024-03-06T12:31:03.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:03.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:03.693804+0000) 2024-03-06T12:31:04.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:04.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:04.694009+0000) 2024-03-06T12:31:05.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:05.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:05.694282+0000) 2024-03-06T12:31:06.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:06.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:06.694604+0000) 2024-03-06T12:31:07.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:07.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:07.694770+0000) 2024-03-06T12:31:08.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:08.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:08.694964+0000) 2024-03-06T12:31:09.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:09.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:09.695187+0000) 2024-03-06T12:31:10.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:10.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:10.695394+0000) 2024-03-06T12:31:11.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:11.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:11.695646+0000) 2024-03-06T12:31:12.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:12.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:12.695862+0000) 2024-03-06T12:31:13.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:13.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:13.696093+0000) 2024-03-06T12:31:14.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:14.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:14.696313+0000) 2024-03-06T12:31:15.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:15.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:15.696511+0000) 2024-03-06T12:31:16.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:16.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:16.696769+0000) 2024-03-06T12:31:17.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:17.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:17.696971+0000) 2024-03-06T12:31:18.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:18.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:18.697174+0000) 2024-03-06T12:31:19.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:19.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:19.697410+0000) 2024-03-06T12:31:19.962 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:31:19.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:31:20.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:20.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:20.697652+0000) 2024-03-06T12:31:21.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:21.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:21.697896+0000) 2024-03-06T12:31:22.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:22.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:22.698173+0000) 2024-03-06T12:31:23.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:23.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:23.698456+0000) 2024-03-06T12:31:24.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:24.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:24.698684+0000) 2024-03-06T12:31:25.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:25.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:25.698869+0000) 2024-03-06T12:31:26.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:26.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:26.699067+0000) 2024-03-06T12:31:27.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:27.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:27.699264+0000) 2024-03-06T12:31:28.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:28.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:28.699502+0000) 2024-03-06T12:31:29.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:29.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:29.699690+0000) 2024-03-06T12:31:30.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:30.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:30.699875+0000) 2024-03-06T12:31:31.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:31.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:31.700104+0000) 2024-03-06T12:31:32.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:32.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:32.700326+0000) 2024-03-06T12:31:33.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:33.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:33.700578+0000) 2024-03-06T12:31:34.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:34.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:34.700776+0000) 2024-03-06T12:31:35.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:35.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:35.700976+0000) 2024-03-06T12:31:36.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:36.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:36.701236+0000) 2024-03-06T12:31:37.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:37.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:37.701514+0000) 2024-03-06T12:31:38.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:38.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:38.701811+0000) 2024-03-06T12:31:39.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:39.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:39.702010+0000) 2024-03-06T12:31:40.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:40.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:40.702252+0000) 2024-03-06T12:31:41.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:41.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:41.702489+0000) 2024-03-06T12:31:42.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:42.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:42.702721+0000) 2024-03-06T12:31:43.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:43.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:43.702970+0000) 2024-03-06T12:31:44.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:44.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:44.703248+0000) 2024-03-06T12:31:45.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:45.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:45.703525+0000) 2024-03-06T12:31:46.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:46.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:46.703851+0000) 2024-03-06T12:31:47.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:47.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:47.704123+0000) 2024-03-06T12:31:48.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:48.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:48.704426+0000) 2024-03-06T12:31:49.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:49.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:49.704649+0000) 2024-03-06T12:31:50.100 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:31:50.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:31:50.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:50.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:50.704851+0000) 2024-03-06T12:31:51.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:51.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:51.705124+0000) 2024-03-06T12:31:52.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:52.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:52.705397+0000) 2024-03-06T12:31:53.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:53.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:53.705703+0000) 2024-03-06T12:31:54.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:54.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:54.705932+0000) 2024-03-06T12:31:55.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:55.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:55.706158+0000) 2024-03-06T12:31:56.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:56.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:56.706368+0000) 2024-03-06T12:31:57.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:57.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:57.706652+0000) 2024-03-06T12:31:58.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:58.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:58.706860+0000) 2024-03-06T12:31:59.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:31:59.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:31:59.707116+0000) 2024-03-06T12:32:00.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:00.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:00.707359+0000) 2024-03-06T12:32:01.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:01.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:01.707623+0000) 2024-03-06T12:32:02.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:02.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:02.707816+0000) 2024-03-06T12:32:03.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:03.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:03.708054+0000) 2024-03-06T12:32:04.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:04.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:04.708253+0000) 2024-03-06T12:32:05.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:05.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:05.708452+0000) 2024-03-06T12:32:06.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:06.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:06.708676+0000) 2024-03-06T12:32:07.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:07.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:07.708839+0000) 2024-03-06T12:32:08.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:08.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:08.709053+0000) 2024-03-06T12:32:09.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:09.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:09.709294+0000) 2024-03-06T12:32:10.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:10.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:10.709564+0000) 2024-03-06T12:32:11.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:11.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:11.709761+0000) 2024-03-06T12:32:12.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:12.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:12.709953+0000) 2024-03-06T12:32:13.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:13.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:13.710156+0000) 2024-03-06T12:32:14.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:14.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:14.710395+0000) 2024-03-06T12:32:15.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:15.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:15.710623+0000) 2024-03-06T12:32:16.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:16.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:16.710871+0000) 2024-03-06T12:32:17.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:17.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:17.711066+0000) 2024-03-06T12:32:18.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:18.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:18.711314+0000) 2024-03-06T12:32:19.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:19.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:19.711521+0000) 2024-03-06T12:32:20.254 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:32:20.263 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:32:20.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:20.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:20.711813+0000) 2024-03-06T12:32:21.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:21.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:21.712092+0000) 2024-03-06T12:32:22.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:22.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:22.712369+0000) 2024-03-06T12:32:23.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:23.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:23.712626+0000) 2024-03-06T12:32:24.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:24.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:24.712850+0000) 2024-03-06T12:32:25.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:25.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:25.713039+0000) 2024-03-06T12:32:26.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:26.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:26.713265+0000) 2024-03-06T12:32:27.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:27.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:27.713489+0000) 2024-03-06T12:32:28.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:28.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:28.713706+0000) 2024-03-06T12:32:29.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:29.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:29.713952+0000) 2024-03-06T12:32:30.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:30.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:30.714154+0000) 2024-03-06T12:32:31.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:31.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:31.714372+0000) 2024-03-06T12:32:32.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:32.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:32.714614+0000) 2024-03-06T12:32:33.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:33.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:33.714855+0000) 2024-03-06T12:32:34.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:34.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:34.715084+0000) 2024-03-06T12:32:35.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:35.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:35.715307+0000) 2024-03-06T12:32:36.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:36.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:36.715630+0000) 2024-03-06T12:32:37.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:37.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:37.715860+0000) 2024-03-06T12:32:38.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:38.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:38.716057+0000) 2024-03-06T12:32:39.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:39.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:39.716285+0000) 2024-03-06T12:32:40.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:40.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:40.716501+0000) 2024-03-06T12:32:41.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:41.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:41.716680+0000) 2024-03-06T12:32:42.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:42.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:42.716867+0000) 2024-03-06T12:32:43.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:43.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:43.717068+0000) 2024-03-06T12:32:44.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:44.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:44.717296+0000) 2024-03-06T12:32:45.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:45.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:45.717516+0000) 2024-03-06T12:32:46.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:46.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:46.717752+0000) 2024-03-06T12:32:47.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:47.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:47.717990+0000) 2024-03-06T12:32:48.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:48.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:48.718247+0000) 2024-03-06T12:32:49.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:49.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:49.718490+0000) 2024-03-06T12:32:50.466 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:32:50.475 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:32:50.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:50.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:50.718761+0000) 2024-03-06T12:32:51.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:51.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:51.719005+0000) 2024-03-06T12:32:52.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:52.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:52.719245+0000) 2024-03-06T12:32:53.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:53.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:53.719457+0000) 2024-03-06T12:32:54.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:54.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:54.719685+0000) 2024-03-06T12:32:55.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:55.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:55.719926+0000) 2024-03-06T12:32:56.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:56.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:56.720140+0000) 2024-03-06T12:32:57.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:57.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:57.720383+0000) 2024-03-06T12:32:58.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:58.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:58.720664+0000) 2024-03-06T12:32:59.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:32:59.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:32:59.720871+0000) 2024-03-06T12:33:00.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:00.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:00.721085+0000) 2024-03-06T12:33:01.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:01.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:01.721330+0000) 2024-03-06T12:33:02.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:02.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:02.721590+0000) 2024-03-06T12:33:03.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:03.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:03.721813+0000) 2024-03-06T12:33:04.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:04.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:04.722020+0000) 2024-03-06T12:33:05.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:05.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:05.722258+0000) 2024-03-06T12:33:06.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:06.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:06.722503+0000) 2024-03-06T12:33:07.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:07.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:07.722683+0000) 2024-03-06T12:33:08.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:08.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:08.722843+0000) 2024-03-06T12:33:09.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:09.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:09.723094+0000) 2024-03-06T12:33:10.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:10.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:10.723339+0000) 2024-03-06T12:33:11.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:11.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:11.723626+0000) 2024-03-06T12:33:12.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:12.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:12.723849+0000) 2024-03-06T12:33:13.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:13.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:13.724106+0000) 2024-03-06T12:33:14.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:14.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:14.724306+0000) 2024-03-06T12:33:15.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:15.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:15.724520+0000) 2024-03-06T12:33:16.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:16.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:16.724815+0000) 2024-03-06T12:33:17.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:17.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:17.725044+0000) 2024-03-06T12:33:18.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:18.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:18.725294+0000) 2024-03-06T12:33:19.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:19.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:19.725508+0000) 2024-03-06T12:33:20.685 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:33:20.694 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:33:20.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:20.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:20.725726+0000) 2024-03-06T12:33:21.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:21.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:21.725954+0000) 2024-03-06T12:33:22.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:22.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:22.726194+0000) 2024-03-06T12:33:23.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:23.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:23.726443+0000) 2024-03-06T12:33:24.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:24.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:24.726690+0000) 2024-03-06T12:33:25.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:25.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:25.726927+0000) 2024-03-06T12:33:26.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:26.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:26.727159+0000) 2024-03-06T12:33:27.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:27.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:27.727370+0000) 2024-03-06T12:33:28.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:28.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:28.727587+0000) 2024-03-06T12:33:29.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:29.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:29.727899+0000) 2024-03-06T12:33:30.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:30.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:30.728109+0000) 2024-03-06T12:33:31.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:31.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:31.728350+0000) 2024-03-06T12:33:32.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:32.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:32.728619+0000) 2024-03-06T12:33:33.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:33.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:33.728859+0000) 2024-03-06T12:33:34.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:34.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:34.729073+0000) 2024-03-06T12:33:35.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:35.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:35.729282+0000) 2024-03-06T12:33:36.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:36.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:36.729561+0000) 2024-03-06T12:33:37.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:37.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:37.729714+0000) 2024-03-06T12:33:38.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:38.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:38.729941+0000) 2024-03-06T12:33:39.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:39.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:39.730188+0000) 2024-03-06T12:33:40.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:40.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:40.730459+0000) 2024-03-06T12:33:41.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:41.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:41.730735+0000) 2024-03-06T12:33:42.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:42.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:42.731017+0000) 2024-03-06T12:33:43.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:43.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:43.731308+0000) 2024-03-06T12:33:44.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:44.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:44.731617+0000) 2024-03-06T12:33:45.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:45.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:45.731819+0000) 2024-03-06T12:33:46.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:46.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:46.732033+0000) 2024-03-06T12:33:47.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:47.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:47.732268+0000) 2024-03-06T12:33:48.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:48.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:48.732516+0000) 2024-03-06T12:33:49.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:49.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:49.732758+0000) 2024-03-06T12:33:50.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:50.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:50.732970+0000) 2024-03-06T12:33:50.841 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:33:50.849 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:33:51.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:51.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:51.733185+0000) 2024-03-06T12:33:52.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:52.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:52.733421+0000) 2024-03-06T12:33:53.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:53.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:53.733662+0000) 2024-03-06T12:33:54.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:54.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:54.733855+0000) 2024-03-06T12:33:55.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:55.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:55.734061+0000) 2024-03-06T12:33:56.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:56.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:56.734303+0000) 2024-03-06T12:33:57.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:57.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:57.734559+0000) 2024-03-06T12:33:58.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:58.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:58.734767+0000) 2024-03-06T12:33:59.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:33:59.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:33:59.734963+0000) 2024-03-06T12:34:00.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:00.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:00.735151+0000) 2024-03-06T12:34:01.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:01.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:01.735378+0000) 2024-03-06T12:34:02.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:02.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:02.735628+0000) 2024-03-06T12:34:03.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:03.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:03.735875+0000) 2024-03-06T12:34:04.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:04.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:04.736070+0000) 2024-03-06T12:34:05.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:05.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:05.736300+0000) 2024-03-06T12:34:06.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:06.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:06.736562+0000) 2024-03-06T12:34:07.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:07.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:07.736709+0000) 2024-03-06T12:34:08.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:08.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:08.736908+0000) 2024-03-06T12:34:09.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:09.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:09.737136+0000) 2024-03-06T12:34:10.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:10.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:10.737395+0000) 2024-03-06T12:34:11.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:11.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:11.737668+0000) 2024-03-06T12:34:12.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:12.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:12.737879+0000) 2024-03-06T12:34:13.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:13.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:13.738134+0000) 2024-03-06T12:34:14.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:14.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:14.738323+0000) 2024-03-06T12:34:15.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:15.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:15.738606+0000) 2024-03-06T12:34:16.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:16.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:16.738839+0000) 2024-03-06T12:34:17.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:17.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:17.739056+0000) 2024-03-06T12:34:18.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:18.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:18.739250+0000) 2024-03-06T12:34:19.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:19.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:19.739482+0000) 2024-03-06T12:34:20.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:20.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:20.739674+0000) 2024-03-06T12:34:20.980 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:34:20.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:34:21.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:21.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:21.739903+0000) 2024-03-06T12:34:22.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:22.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:22.740131+0000) 2024-03-06T12:34:23.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:23.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:23.740388+0000) 2024-03-06T12:34:24.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:24.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:24.740653+0000) 2024-03-06T12:34:25.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:25.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:25.740866+0000) 2024-03-06T12:34:26.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:26.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:26.741117+0000) 2024-03-06T12:34:27.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:27.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:27.741356+0000) 2024-03-06T12:34:28.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:28.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:28.741651+0000) 2024-03-06T12:34:29.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:29.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:29.741865+0000) 2024-03-06T12:34:30.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:30.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:30.742104+0000) 2024-03-06T12:34:31.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:31.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:31.742339+0000) 2024-03-06T12:34:32.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:32.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:32.742596+0000) 2024-03-06T12:34:33.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:33.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:33.742850+0000) 2024-03-06T12:34:34.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:34.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:34.743081+0000) 2024-03-06T12:34:35.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:35.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:35.743311+0000) 2024-03-06T12:34:36.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:36.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:36.743513+0000) 2024-03-06T12:34:37.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:37.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:37.743815+0000) 2024-03-06T12:34:38.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:38.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:38.744089+0000) 2024-03-06T12:34:39.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:39.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:39.744365+0000) 2024-03-06T12:34:40.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:40.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:40.744596+0000) 2024-03-06T12:34:41.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:41.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:41.744817+0000) 2024-03-06T12:34:42.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:42.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:42.745022+0000) 2024-03-06T12:34:43.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:43.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:43.745264+0000) 2024-03-06T12:34:44.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:44.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:44.745481+0000) 2024-03-06T12:34:45.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:45.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:45.745761+0000) 2024-03-06T12:34:46.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:46.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:46.746004+0000) 2024-03-06T12:34:47.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:47.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:47.746244+0000) 2024-03-06T12:34:48.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:48.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:48.746487+0000) 2024-03-06T12:34:49.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:49.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:49.746813+0000) 2024-03-06T12:34:50.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:50.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:50.747052+0000) 2024-03-06T12:34:51.169 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:34:51.177 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:34:51.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:51.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:51.747297+0000) 2024-03-06T12:34:52.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:52.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:52.747529+0000) 2024-03-06T12:34:53.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:53.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:53.747829+0000) 2024-03-06T12:34:54.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:54.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:54.748110+0000) 2024-03-06T12:34:55.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:55.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:55.748393+0000) 2024-03-06T12:34:56.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:56.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:56.748655+0000) 2024-03-06T12:34:57.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:57.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:57.748894+0000) 2024-03-06T12:34:58.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:58.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:58.749128+0000) 2024-03-06T12:34:59.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:34:59.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:34:59.749357+0000) 2024-03-06T12:35:00.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:00.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:00.749578+0000) 2024-03-06T12:35:01.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:01.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:01.749822+0000) 2024-03-06T12:35:02.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:02.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:02.750007+0000) 2024-03-06T12:35:03.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:03.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:03.750250+0000) 2024-03-06T12:35:04.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:04.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:04.750447+0000) 2024-03-06T12:35:05.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:05.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:05.750725+0000) 2024-03-06T12:35:06.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:06.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:06.750936+0000) 2024-03-06T12:35:07.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:07.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:07.751125+0000) 2024-03-06T12:35:08.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:08.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:08.751360+0000) 2024-03-06T12:35:09.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:09.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:09.751649+0000) 2024-03-06T12:35:10.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:10.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:10.751792+0000) 2024-03-06T12:35:11.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:11.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:11.751987+0000) 2024-03-06T12:35:12.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:12.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:12.752181+0000) 2024-03-06T12:35:13.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:13.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:13.752421+0000) 2024-03-06T12:35:14.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:14.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:14.752662+0000) 2024-03-06T12:35:15.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:15.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:15.752852+0000) 2024-03-06T12:35:16.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:16.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:16.753058+0000) 2024-03-06T12:35:17.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:17.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:17.753278+0000) 2024-03-06T12:35:18.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:18.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:18.753512+0000) 2024-03-06T12:35:19.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:19.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:19.753731+0000) 2024-03-06T12:35:20.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:20.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:20.753897+0000) 2024-03-06T12:35:21.366 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:35:21.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:35:21.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:21.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:21.754137+0000) 2024-03-06T12:35:22.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:22.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:22.754358+0000) 2024-03-06T12:35:23.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:23.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:23.754608+0000) 2024-03-06T12:35:24.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:24.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:24.754838+0000) 2024-03-06T12:35:25.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:25.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:25.755062+0000) 2024-03-06T12:35:26.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:26.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:26.755293+0000) 2024-03-06T12:35:27.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:27.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:27.755524+0000) 2024-03-06T12:35:28.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:28.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:28.755791+0000) 2024-03-06T12:35:29.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:29.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:29.756025+0000) 2024-03-06T12:35:30.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:30.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:30.756244+0000) 2024-03-06T12:35:31.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:31.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:31.756465+0000) 2024-03-06T12:35:32.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:32.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:32.756673+0000) 2024-03-06T12:35:33.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:33.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:33.756879+0000) 2024-03-06T12:35:34.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:34.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:34.757063+0000) 2024-03-06T12:35:35.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:35.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:35.757288+0000) 2024-03-06T12:35:36.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:36.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:36.757513+0000) 2024-03-06T12:35:37.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:37.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:37.757730+0000) 2024-03-06T12:35:38.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:38.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:38.757918+0000) 2024-03-06T12:35:39.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:39.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:39.758154+0000) 2024-03-06T12:35:40.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:40.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:40.758424+0000) 2024-03-06T12:35:41.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:41.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:41.758710+0000) 2024-03-06T12:35:42.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:42.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:42.758988+0000) 2024-03-06T12:35:43.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:43.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:43.759275+0000) 2024-03-06T12:35:44.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:44.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:44.759607+0000) 2024-03-06T12:35:45.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:45.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:45.759845+0000) 2024-03-06T12:35:46.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:46.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:46.760115+0000) 2024-03-06T12:35:47.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:47.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:47.760384+0000) 2024-03-06T12:35:48.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:48.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:48.760651+0000) 2024-03-06T12:35:49.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:49.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:49.760895+0000) 2024-03-06T12:35:50.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:50.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:50.761170+0000) 2024-03-06T12:35:51.506 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:35:51.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:35:51.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:51.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:51.761460+0000) 2024-03-06T12:35:52.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:52.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:52.761682+0000) 2024-03-06T12:35:53.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:53.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:53.761874+0000) 2024-03-06T12:35:54.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:54.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:54.762091+0000) 2024-03-06T12:35:55.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:55.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:55.762261+0000) 2024-03-06T12:35:56.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:56.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:56.762570+0000) 2024-03-06T12:35:57.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:57.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:57.762792+0000) 2024-03-06T12:35:58.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:58.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:58.762986+0000) 2024-03-06T12:35:59.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:35:59.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:35:59.763199+0000) 2024-03-06T12:36:00.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:00.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:00.763394+0000) 2024-03-06T12:36:01.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:01.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:01.763679+0000) 2024-03-06T12:36:02.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:02.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:02.763899+0000) 2024-03-06T12:36:03.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:03.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:03.764133+0000) 2024-03-06T12:36:04.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:04.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:04.764370+0000) 2024-03-06T12:36:05.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:05.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:05.764659+0000) 2024-03-06T12:36:06.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:06.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:06.764895+0000) 2024-03-06T12:36:07.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:07.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:07.765087+0000) 2024-03-06T12:36:08.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:08.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:08.765311+0000) 2024-03-06T12:36:09.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:09.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:09.765569+0000) 2024-03-06T12:36:10.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:10.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:10.765797+0000) 2024-03-06T12:36:11.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:11.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:11.766009+0000) 2024-03-06T12:36:12.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:12.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:12.766286+0000) 2024-03-06T12:36:13.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:13.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:13.766503+0000) 2024-03-06T12:36:14.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:14.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:14.766747+0000) 2024-03-06T12:36:15.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:15.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:15.766956+0000) 2024-03-06T12:36:16.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:16.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:16.767165+0000) 2024-03-06T12:36:17.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:17.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:17.767384+0000) 2024-03-06T12:36:18.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:18.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:18.767643+0000) 2024-03-06T12:36:19.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:19.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:19.767917+0000) 2024-03-06T12:36:20.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:20.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:20.768136+0000) 2024-03-06T12:36:21.645 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:36:21.656 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:36:21.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:21.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:21.768342+0000) 2024-03-06T12:36:22.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:22.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:22.768581+0000) 2024-03-06T12:36:23.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:23.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:23.768845+0000) 2024-03-06T12:36:24.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:24.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:24.769091+0000) 2024-03-06T12:36:25.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:25.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:25.769317+0000) 2024-03-06T12:36:26.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:26.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:26.769615+0000) 2024-03-06T12:36:27.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:27.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:27.769794+0000) 2024-03-06T12:36:28.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:28.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:28.769998+0000) 2024-03-06T12:36:29.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:29.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:29.770200+0000) 2024-03-06T12:36:30.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:30.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:30.770426+0000) 2024-03-06T12:36:31.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:31.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:31.770697+0000) 2024-03-06T12:36:32.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:32.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:32.770865+0000) 2024-03-06T12:36:33.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:33.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:33.771090+0000) 2024-03-06T12:36:34.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:34.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:34.771322+0000) 2024-03-06T12:36:35.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:35.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:35.771577+0000) 2024-03-06T12:36:36.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:36.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:36.771821+0000) 2024-03-06T12:36:37.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:37.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:37.772046+0000) 2024-03-06T12:36:38.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:38.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:38.772235+0000) 2024-03-06T12:36:39.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:39.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:39.772464+0000) 2024-03-06T12:36:40.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:40.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:40.772702+0000) 2024-03-06T12:36:41.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:41.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:41.772933+0000) 2024-03-06T12:36:42.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:42.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:42.773114+0000) 2024-03-06T12:36:43.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:43.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:43.773299+0000) 2024-03-06T12:36:44.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:44.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:44.773499+0000) 2024-03-06T12:36:45.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:45.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:45.773770+0000) 2024-03-06T12:36:46.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:46.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:46.774055+0000) 2024-03-06T12:36:47.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:47.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:47.774282+0000) 2024-03-06T12:36:48.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:48.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:48.774505+0000) 2024-03-06T12:36:49.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:49.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:49.774735+0000) 2024-03-06T12:36:50.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:50.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:50.774957+0000) 2024-03-06T12:36:51.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:51.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:51.775180+0000) 2024-03-06T12:36:51.780 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:36:51.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:36:52.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:52.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:52.775411+0000) 2024-03-06T12:36:53.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:53.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:53.775639+0000) 2024-03-06T12:36:54.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:54.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:54.775837+0000) 2024-03-06T12:36:55.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:55.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:55.776031+0000) 2024-03-06T12:36:56.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:56.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:56.776270+0000) 2024-03-06T12:36:57.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:57.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:57.776503+0000) 2024-03-06T12:36:58.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:58.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:58.776698+0000) 2024-03-06T12:36:59.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:36:59.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:36:59.776912+0000) 2024-03-06T12:37:00.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:00.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:00.777162+0000) 2024-03-06T12:37:01.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:01.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:01.777360+0000) 2024-03-06T12:37:02.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:02.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:02.777639+0000) 2024-03-06T12:37:03.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:03.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:03.777833+0000) 2024-03-06T12:37:04.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:04.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:04.778065+0000) 2024-03-06T12:37:05.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:05.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:05.778294+0000) 2024-03-06T12:37:06.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:06.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:06.778508+0000) 2024-03-06T12:37:07.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:07.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:07.778792+0000) 2024-03-06T12:37:08.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:08.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:08.779021+0000) 2024-03-06T12:37:09.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:09.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:09.779252+0000) 2024-03-06T12:37:10.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:10.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:10.779485+0000) 2024-03-06T12:37:11.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:11.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:11.779705+0000) 2024-03-06T12:37:12.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:12.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:12.779933+0000) 2024-03-06T12:37:13.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:13.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:13.780158+0000) 2024-03-06T12:37:14.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:14.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:14.780383+0000) 2024-03-06T12:37:15.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:15.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:15.780645+0000) 2024-03-06T12:37:16.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:16.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:16.780886+0000) 2024-03-06T12:37:17.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:17.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:17.781115+0000) 2024-03-06T12:37:18.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:18.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:18.781341+0000) 2024-03-06T12:37:19.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:19.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:19.781654+0000) 2024-03-06T12:37:20.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:20.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:20.781891+0000) 2024-03-06T12:37:21.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:21.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:21.782072+0000) 2024-03-06T12:37:21.992 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:37:22.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:37:22.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:22.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:22.782298+0000) 2024-03-06T12:37:23.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:23.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:23.782522+0000) 2024-03-06T12:37:24.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:24.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:24.782757+0000) 2024-03-06T12:37:25.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:25.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:25.782962+0000) 2024-03-06T12:37:26.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:26.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:26.783167+0000) 2024-03-06T12:37:27.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:27.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:27.783397+0000) 2024-03-06T12:37:28.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:28.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:28.783639+0000) 2024-03-06T12:37:29.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:29.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:29.783842+0000) 2024-03-06T12:37:30.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:30.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:30.784023+0000) 2024-03-06T12:37:31.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:31.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:31.784243+0000) 2024-03-06T12:37:32.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:32.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:32.784454+0000) 2024-03-06T12:37:33.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:33.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:33.784689+0000) 2024-03-06T12:37:34.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:34.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:34.784890+0000) 2024-03-06T12:37:35.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:35.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:35.785086+0000) 2024-03-06T12:37:36.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:36.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:36.785318+0000) 2024-03-06T12:37:37.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:37.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:37.785564+0000) 2024-03-06T12:37:38.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:38.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:38.785712+0000) 2024-03-06T12:37:39.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:39.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:39.785901+0000) 2024-03-06T12:37:40.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:40.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:40.786139+0000) 2024-03-06T12:37:41.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:41.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:41.786360+0000) 2024-03-06T12:37:42.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:42.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:42.786616+0000) 2024-03-06T12:37:43.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:43.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:43.786848+0000) 2024-03-06T12:37:44.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:44.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:44.787077+0000) 2024-03-06T12:37:45.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:45.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:45.787356+0000) 2024-03-06T12:37:46.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:46.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:46.787602+0000) 2024-03-06T12:37:47.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:47.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:47.787833+0000) 2024-03-06T12:37:48.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:48.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:48.788060+0000) 2024-03-06T12:37:49.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:49.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:49.788287+0000) 2024-03-06T12:37:50.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:50.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:50.788516+0000) 2024-03-06T12:37:51.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:51.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:51.788780+0000) 2024-03-06T12:37:52.204 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:37:52.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:37:52.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:52.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:52.789005+0000) 2024-03-06T12:37:53.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:53.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:53.789236+0000) 2024-03-06T12:37:54.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:54.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:54.789484+0000) 2024-03-06T12:37:55.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:55.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:55.789688+0000) 2024-03-06T12:37:56.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:56.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:56.789845+0000) 2024-03-06T12:37:57.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:57.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:57.790073+0000) 2024-03-06T12:37:58.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:58.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:58.790268+0000) 2024-03-06T12:37:59.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:37:59.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:37:59.790470+0000) 2024-03-06T12:38:00.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:00.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:00.790736+0000) 2024-03-06T12:38:01.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:01.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:01.790988+0000) 2024-03-06T12:38:02.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:02.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:02.791185+0000) 2024-03-06T12:38:03.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:03.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:03.791407+0000) 2024-03-06T12:38:04.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:04.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:04.791655+0000) 2024-03-06T12:38:05.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:05.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:05.791844+0000) 2024-03-06T12:38:06.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:06.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:06.792048+0000) 2024-03-06T12:38:07.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:07.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:07.792330+0000) 2024-03-06T12:38:08.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:08.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:08.792645+0000) 2024-03-06T12:38:09.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:09.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:09.792899+0000) 2024-03-06T12:38:10.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:10.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:10.793150+0000) 2024-03-06T12:38:11.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:11.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:11.793412+0000) 2024-03-06T12:38:12.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:12.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:12.793671+0000) 2024-03-06T12:38:13.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:13.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:13.793884+0000) 2024-03-06T12:38:14.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:14.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:14.794137+0000) 2024-03-06T12:38:15.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:15.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:15.794332+0000) 2024-03-06T12:38:16.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:16.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:16.794598+0000) 2024-03-06T12:38:17.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:17.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:17.794843+0000) 2024-03-06T12:38:18.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:18.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:18.795066+0000) 2024-03-06T12:38:19.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:19.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:19.795293+0000) 2024-03-06T12:38:20.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:20.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:20.795604+0000) 2024-03-06T12:38:21.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:21.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:21.795849+0000) 2024-03-06T12:38:22.428 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:38:22.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:38:22.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:22.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:22.796093+0000) 2024-03-06T12:38:23.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:23.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:23.796337+0000) 2024-03-06T12:38:24.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:24.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:24.796652+0000) 2024-03-06T12:38:25.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:25.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:25.796932+0000) 2024-03-06T12:38:26.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:26.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:26.797171+0000) 2024-03-06T12:38:27.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:27.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:27.797396+0000) 2024-03-06T12:38:28.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:28.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:28.797710+0000) 2024-03-06T12:38:29.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:29.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:29.797942+0000) 2024-03-06T12:38:30.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:30.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:30.798167+0000) 2024-03-06T12:38:31.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:31.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:31.798382+0000) 2024-03-06T12:38:32.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:32.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:32.798683+0000) 2024-03-06T12:38:33.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:33.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:33.798863+0000) 2024-03-06T12:38:34.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:34.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:34.799096+0000) 2024-03-06T12:38:35.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:35.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:35.799325+0000) 2024-03-06T12:38:36.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:36.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:36.799597+0000) 2024-03-06T12:38:37.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:37.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:37.799828+0000) 2024-03-06T12:38:38.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:38.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:38.800019+0000) 2024-03-06T12:38:39.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:39.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:39.800265+0000) 2024-03-06T12:38:40.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:40.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:40.800486+0000) 2024-03-06T12:38:41.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:41.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:41.800710+0000) 2024-03-06T12:38:42.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:42.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:42.800940+0000) 2024-03-06T12:38:43.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:43.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:43.801144+0000) 2024-03-06T12:38:44.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:44.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:44.801347+0000) 2024-03-06T12:38:45.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:45.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:45.801651+0000) 2024-03-06T12:38:46.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:46.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:46.801948+0000) 2024-03-06T12:38:47.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:47.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:47.802231+0000) 2024-03-06T12:38:48.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:48.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:48.802509+0000) 2024-03-06T12:38:49.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:49.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:49.802845+0000) 2024-03-06T12:38:50.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:50.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:50.803124+0000) 2024-03-06T12:38:51.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:51.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:51.803333+0000) 2024-03-06T12:38:52.633 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:38:52.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:38:52.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:52.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:52.803645+0000) 2024-03-06T12:38:53.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:53.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:53.803933+0000) 2024-03-06T12:38:54.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:54.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:54.804153+0000) 2024-03-06T12:38:55.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:55.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:55.804427+0000) 2024-03-06T12:38:56.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:56.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:56.804654+0000) 2024-03-06T12:38:57.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:57.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:57.804844+0000) 2024-03-06T12:38:58.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:58.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:58.805047+0000) 2024-03-06T12:38:59.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:38:59.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:38:59.805297+0000) 2024-03-06T12:39:00.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:00.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:00.805564+0000) 2024-03-06T12:39:01.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:01.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:01.805777+0000) 2024-03-06T12:39:02.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:02.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:02.806014+0000) 2024-03-06T12:39:03.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:03.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:03.806242+0000) 2024-03-06T12:39:04.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:04.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:04.806497+0000) 2024-03-06T12:39:05.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:05.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:05.806744+0000) 2024-03-06T12:39:06.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:06.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:06.806974+0000) 2024-03-06T12:39:07.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:07.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:07.807185+0000) 2024-03-06T12:39:08.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:08.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:08.807430+0000) 2024-03-06T12:39:09.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:09.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:09.807730+0000) 2024-03-06T12:39:10.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:10.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:10.808028+0000) 2024-03-06T12:39:11.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:11.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:11.808243+0000) 2024-03-06T12:39:12.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:12.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:12.808504+0000) 2024-03-06T12:39:13.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:13.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:13.808751+0000) 2024-03-06T12:39:14.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:14.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:14.808954+0000) 2024-03-06T12:39:15.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:15.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:15.809152+0000) 2024-03-06T12:39:16.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:16.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:16.809455+0000) 2024-03-06T12:39:17.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:17.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:17.809689+0000) 2024-03-06T12:39:18.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:18.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:18.809879+0000) 2024-03-06T12:39:19.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:19.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:19.810072+0000) 2024-03-06T12:39:20.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:20.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:20.810313+0000) 2024-03-06T12:39:21.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:21.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:21.810499+0000) 2024-03-06T12:39:22.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:22.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:22.810645+0000) 2024-03-06T12:39:22.840 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:39:22.849 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:39:23.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:23.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:23.810884+0000) 2024-03-06T12:39:24.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:24.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:24.811094+0000) 2024-03-06T12:39:25.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:25.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:25.811331+0000) 2024-03-06T12:39:26.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:26.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:26.811526+0000) 2024-03-06T12:39:27.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:27.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:27.811773+0000) 2024-03-06T12:39:28.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:28.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:28.811980+0000) 2024-03-06T12:39:29.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:29.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:29.812196+0000) 2024-03-06T12:39:30.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:30.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:30.812470+0000) 2024-03-06T12:39:31.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:31.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:31.812710+0000) 2024-03-06T12:39:32.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:32.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:32.812952+0000) 2024-03-06T12:39:33.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:33.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:33.813198+0000) 2024-03-06T12:39:34.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:34.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:34.813484+0000) 2024-03-06T12:39:35.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:35.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:35.813744+0000) 2024-03-06T12:39:36.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:36.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:36.813993+0000) 2024-03-06T12:39:37.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:37.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:37.814237+0000) 2024-03-06T12:39:38.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:38.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:38.814455+0000) 2024-03-06T12:39:39.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:39.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:39.814696+0000) 2024-03-06T12:39:40.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:40.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:40.814904+0000) 2024-03-06T12:39:41.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:41.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:41.815159+0000) 2024-03-06T12:39:42.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:42.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:42.815375+0000) 2024-03-06T12:39:43.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:43.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:43.815607+0000) 2024-03-06T12:39:44.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:44.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:44.815820+0000) 2024-03-06T12:39:45.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:45.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:45.816027+0000) 2024-03-06T12:39:46.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:46.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:46.816274+0000) 2024-03-06T12:39:47.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:47.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:47.816483+0000) 2024-03-06T12:39:48.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:48.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:48.816736+0000) 2024-03-06T12:39:49.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:49.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:49.816942+0000) 2024-03-06T12:39:50.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:50.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:50.817183+0000) 2024-03-06T12:39:51.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:51.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:51.817372+0000) 2024-03-06T12:39:52.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:52.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:52.817640+0000) 2024-03-06T12:39:53.043 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:39:53.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:39:53.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:53.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:53.817836+0000) 2024-03-06T12:39:54.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:54.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:54.818009+0000) 2024-03-06T12:39:55.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:55.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:55.818248+0000) 2024-03-06T12:39:56.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:56.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:56.818582+0000) 2024-03-06T12:39:57.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:57.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:57.818755+0000) 2024-03-06T12:39:58.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:58.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:58.818988+0000) 2024-03-06T12:39:59.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:39:59.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:39:59.819228+0000) 2024-03-06T12:40:00.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:00.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:00.819450+0000) 2024-03-06T12:40:01.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:01.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:01.819693+0000) 2024-03-06T12:40:02.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:02.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:02.819878+0000) 2024-03-06T12:40:03.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:03.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:03.820075+0000) 2024-03-06T12:40:04.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:04.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:04.820396+0000) 2024-03-06T12:40:05.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:05.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:05.820647+0000) 2024-03-06T12:40:06.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:06.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:06.820875+0000) 2024-03-06T12:40:07.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:07.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:07.821071+0000) 2024-03-06T12:40:08.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:08.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:08.821306+0000) 2024-03-06T12:40:09.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:09.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:09.821569+0000) 2024-03-06T12:40:10.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:10.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:10.821788+0000) 2024-03-06T12:40:11.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:11.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:11.822009+0000) 2024-03-06T12:40:12.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:12.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:12.822206+0000) 2024-03-06T12:40:13.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:13.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:13.822452+0000) 2024-03-06T12:40:14.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:14.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:14.822696+0000) 2024-03-06T12:40:15.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:15.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:15.822888+0000) 2024-03-06T12:40:16.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:16.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:16.823089+0000) 2024-03-06T12:40:17.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:17.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:17.823295+0000) 2024-03-06T12:40:18.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:18.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:18.823528+0000) 2024-03-06T12:40:19.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:19.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:19.823756+0000) 2024-03-06T12:40:20.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:20.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:20.823956+0000) 2024-03-06T12:40:21.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:21.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:21.824145+0000) 2024-03-06T12:40:22.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:22.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:22.824337+0000) 2024-03-06T12:40:23.283 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:40:23.291 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:40:23.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:23.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:23.824524+0000) 2024-03-06T12:40:24.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:24.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:24.824709+0000) 2024-03-06T12:40:25.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:25.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:25.824942+0000) 2024-03-06T12:40:26.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:26.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:26.825172+0000) 2024-03-06T12:40:27.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:27.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:27.825401+0000) 2024-03-06T12:40:28.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:28.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:28.825641+0000) 2024-03-06T12:40:29.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:29.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:29.825859+0000) 2024-03-06T12:40:30.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:30.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:30.826086+0000) 2024-03-06T12:40:31.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:31.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:31.826304+0000) 2024-03-06T12:40:32.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:32.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:32.826528+0000) 2024-03-06T12:40:33.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:33.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:33.826799+0000) 2024-03-06T12:40:34.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:34.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:34.826994+0000) 2024-03-06T12:40:35.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:35.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:35.827240+0000) 2024-03-06T12:40:36.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:36.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:36.827483+0000) 2024-03-06T12:40:37.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:37.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:37.827709+0000) 2024-03-06T12:40:38.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:38.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:38.827937+0000) 2024-03-06T12:40:39.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:39.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:39.828162+0000) 2024-03-06T12:40:40.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:40.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:40.828369+0000) 2024-03-06T12:40:41.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:41.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:41.828636+0000) 2024-03-06T12:40:42.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:42.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:42.828866+0000) 2024-03-06T12:40:43.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:43.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:43.829097+0000) 2024-03-06T12:40:44.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:44.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:44.829334+0000) 2024-03-06T12:40:45.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:45.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:45.829601+0000) 2024-03-06T12:40:46.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:46.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:46.829765+0000) 2024-03-06T12:40:47.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:47.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:47.829965+0000) 2024-03-06T12:40:48.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:48.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:48.830165+0000) 2024-03-06T12:40:49.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:49.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:49.830373+0000) 2024-03-06T12:40:50.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:50.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:50.830608+0000) 2024-03-06T12:40:51.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:51.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:51.830845+0000) 2024-03-06T12:40:52.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:52.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:52.831037+0000) 2024-03-06T12:40:53.535 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:40:53.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:40:53.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:53.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:53.831266+0000) 2024-03-06T12:40:54.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:54.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:54.831458+0000) 2024-03-06T12:40:55.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:55.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:55.831700+0000) 2024-03-06T12:40:56.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:56.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:56.831927+0000) 2024-03-06T12:40:57.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:57.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:57.832269+0000) 2024-03-06T12:40:58.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:58.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:58.832489+0000) 2024-03-06T12:40:59.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:40:59.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:40:59.832673+0000) 2024-03-06T12:41:00.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:00.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:00.832921+0000) 2024-03-06T12:41:01.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:01.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:01.833166+0000) 2024-03-06T12:41:02.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:02.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:02.833451+0000) 2024-03-06T12:41:03.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:03.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:03.833688+0000) 2024-03-06T12:41:04.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:04.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:04.833912+0000) 2024-03-06T12:41:05.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:05.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:05.834112+0000) 2024-03-06T12:41:06.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:06.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:06.834346+0000) 2024-03-06T12:41:07.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:07.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:07.834657+0000) 2024-03-06T12:41:08.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:08.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:08.834937+0000) 2024-03-06T12:41:09.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:09.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:09.835185+0000) 2024-03-06T12:41:10.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:10.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:10.835460+0000) 2024-03-06T12:41:11.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:11.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:11.835723+0000) 2024-03-06T12:41:12.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:12.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:12.835929+0000) 2024-03-06T12:41:13.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:13.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:13.836119+0000) 2024-03-06T12:41:14.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:14.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:14.836341+0000) 2024-03-06T12:41:15.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:15.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:15.836596+0000) 2024-03-06T12:41:16.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:16.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:16.836799+0000) 2024-03-06T12:41:17.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:17.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:17.836994+0000) 2024-03-06T12:41:18.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:18.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:18.837213+0000) 2024-03-06T12:41:19.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:19.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:19.837447+0000) 2024-03-06T12:41:20.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:20.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:20.837647+0000) 2024-03-06T12:41:21.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:21.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:21.837852+0000) 2024-03-06T12:41:22.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:22.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:22.838041+0000) 2024-03-06T12:41:23.672 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:41:23.684 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:41:23.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:23.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:23.838281+0000) 2024-03-06T12:41:24.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:24.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:24.838473+0000) 2024-03-06T12:41:25.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:25.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:25.838710+0000) 2024-03-06T12:41:26.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:26.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:26.838916+0000) 2024-03-06T12:41:27.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:27.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:27.839121+0000) 2024-03-06T12:41:28.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:28.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:28.839340+0000) 2024-03-06T12:41:29.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:29.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:29.839592+0000) 2024-03-06T12:41:30.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:30.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:30.839796+0000) 2024-03-06T12:41:31.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:31.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:31.840005+0000) 2024-03-06T12:41:32.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:32.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:32.840216+0000) 2024-03-06T12:41:33.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:33.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:33.840465+0000) 2024-03-06T12:41:34.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:34.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:34.840668+0000) 2024-03-06T12:41:35.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:35.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:35.840872+0000) 2024-03-06T12:41:36.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:36.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:36.841106+0000) 2024-03-06T12:41:37.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:37.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:37.841330+0000) 2024-03-06T12:41:38.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:38.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:38.841524+0000) 2024-03-06T12:41:39.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:39.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:39.841765+0000) 2024-03-06T12:41:40.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:40.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:40.841965+0000) 2024-03-06T12:41:41.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:41.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:41.842155+0000) 2024-03-06T12:41:42.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:42.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:42.842339+0000) 2024-03-06T12:41:43.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:43.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:43.842562+0000) 2024-03-06T12:41:44.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:44.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:44.842766+0000) 2024-03-06T12:41:45.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:45.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:45.843055+0000) 2024-03-06T12:41:46.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:46.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:46.843297+0000) 2024-03-06T12:41:47.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:47.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:47.843511+0000) 2024-03-06T12:41:48.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:48.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:48.843759+0000) 2024-03-06T12:41:49.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:49.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:49.843959+0000) 2024-03-06T12:41:50.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:50.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:50.844178+0000) 2024-03-06T12:41:51.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:51.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:51.844391+0000) 2024-03-06T12:41:52.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:52.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:52.844630+0000) 2024-03-06T12:41:53.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:53.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:53.844829+0000) 2024-03-06T12:41:53.913 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:41:53.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:41:54.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:54.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:54.845013+0000) 2024-03-06T12:41:55.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:55.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:55.845191+0000) 2024-03-06T12:41:56.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:56.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:56.845420+0000) 2024-03-06T12:41:57.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:57.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:57.845676+0000) 2024-03-06T12:41:58.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:58.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:58.845865+0000) 2024-03-06T12:41:59.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:41:59.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:41:59.846094+0000) 2024-03-06T12:42:00.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:00.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:00.846362+0000) 2024-03-06T12:42:01.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:01.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:01.846660+0000) 2024-03-06T12:42:02.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:02.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:02.846898+0000) 2024-03-06T12:42:03.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:03.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:03.847101+0000) 2024-03-06T12:42:04.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:04.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:04.847321+0000) 2024-03-06T12:42:05.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:05.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:05.847626+0000) 2024-03-06T12:42:06.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:06.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:06.847874+0000) 2024-03-06T12:42:07.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:07.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:07.848111+0000) 2024-03-06T12:42:08.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:08.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:08.848320+0000) 2024-03-06T12:42:09.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:09.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:09.848620+0000) 2024-03-06T12:42:10.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:10.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:10.848830+0000) 2024-03-06T12:42:11.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:11.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:11.849054+0000) 2024-03-06T12:42:12.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:12.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:12.849281+0000) 2024-03-06T12:42:13.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:13.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:13.849465+0000) 2024-03-06T12:42:14.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:14.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:14.849706+0000) 2024-03-06T12:42:15.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:15.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:15.849933+0000) 2024-03-06T12:42:16.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:16.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:16.850175+0000) 2024-03-06T12:42:17.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:17.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:17.850379+0000) 2024-03-06T12:42:18.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:18.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:18.850640+0000) 2024-03-06T12:42:19.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:19.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:19.850854+0000) 2024-03-06T12:42:20.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:20.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:20.851058+0000) 2024-03-06T12:42:21.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:21.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:21.851293+0000) 2024-03-06T12:42:22.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:22.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:22.851509+0000) 2024-03-06T12:42:23.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:23.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:23.851775+0000) 2024-03-06T12:42:24.132 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:42:24.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:42:24.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:24.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:24.852005+0000) 2024-03-06T12:42:25.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:25.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:25.852230+0000) 2024-03-06T12:42:26.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:26.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:26.852461+0000) 2024-03-06T12:42:27.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:27.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:27.852708+0000) 2024-03-06T12:42:28.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:28.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:28.852911+0000) 2024-03-06T12:42:29.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:29.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:29.853188+0000) 2024-03-06T12:42:30.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:30.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:30.853406+0000) 2024-03-06T12:42:31.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:31.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:31.853656+0000) 2024-03-06T12:42:32.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:32.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:32.853873+0000) 2024-03-06T12:42:33.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:33.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:33.854118+0000) 2024-03-06T12:42:34.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:34.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:34.854354+0000) 2024-03-06T12:42:35.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:35.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:35.854649+0000) 2024-03-06T12:42:36.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:36.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:36.854939+0000) 2024-03-06T12:42:37.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:37.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:37.855222+0000) 2024-03-06T12:42:38.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:38.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:38.855574+0000) 2024-03-06T12:42:39.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:39.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:39.855791+0000) 2024-03-06T12:42:40.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:40.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:40.855993+0000) 2024-03-06T12:42:41.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:41.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:41.856189+0000) 2024-03-06T12:42:42.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:42.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:42.856419+0000) 2024-03-06T12:42:43.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:43.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:43.856645+0000) 2024-03-06T12:42:44.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:44.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:44.856834+0000) 2024-03-06T12:42:45.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:45.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:45.857023+0000) 2024-03-06T12:42:46.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:46.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:46.857202+0000) 2024-03-06T12:42:47.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:47.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:47.857380+0000) 2024-03-06T12:42:48.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:48.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:48.857597+0000) 2024-03-06T12:42:49.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:49.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:49.857792+0000) 2024-03-06T12:42:50.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:50.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:50.858020+0000) 2024-03-06T12:42:51.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:51.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:51.858206+0000) 2024-03-06T12:42:52.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:52.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:52.858385+0000) 2024-03-06T12:42:53.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:53.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:53.858622+0000) 2024-03-06T12:42:54.353 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:42:54.362 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:42:54.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:54.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:54.858879+0000) 2024-03-06T12:42:55.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:55.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:55.859129+0000) 2024-03-06T12:42:56.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:56.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:56.859364+0000) 2024-03-06T12:42:57.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:57.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:57.859640+0000) 2024-03-06T12:42:58.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:58.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:58.859875+0000) 2024-03-06T12:42:59.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:42:59.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:42:59.860121+0000) 2024-03-06T12:43:00.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:00.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:00.860363+0000) 2024-03-06T12:43:01.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:01.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:01.860631+0000) 2024-03-06T12:43:02.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:02.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:02.860851+0000) 2024-03-06T12:43:03.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:03.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:03.861027+0000) 2024-03-06T12:43:04.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:04.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:04.861207+0000) 2024-03-06T12:43:05.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:05.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:05.861398+0000) 2024-03-06T12:43:06.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:06.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:06.861651+0000) 2024-03-06T12:43:07.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:07.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:07.861889+0000) 2024-03-06T12:43:08.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:08.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:08.862088+0000) 2024-03-06T12:43:09.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:09.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:09.862307+0000) 2024-03-06T12:43:10.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:10.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:10.862566+0000) 2024-03-06T12:43:11.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:11.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:11.862793+0000) 2024-03-06T12:43:12.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:12.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:12.863008+0000) 2024-03-06T12:43:13.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:13.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:13.863236+0000) 2024-03-06T12:43:14.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:14.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:14.863440+0000) 2024-03-06T12:43:15.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:15.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:15.863701+0000) 2024-03-06T12:43:16.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:16.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:16.863967+0000) 2024-03-06T12:43:17.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:17.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:17.864243+0000) 2024-03-06T12:43:18.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:18.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:18.864516+0000) 2024-03-06T12:43:19.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:19.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:19.864847+0000) 2024-03-06T12:43:20.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:20.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:20.865136+0000) 2024-03-06T12:43:21.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:21.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:21.865413+0000) 2024-03-06T12:43:22.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:22.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:22.865694+0000) 2024-03-06T12:43:23.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:23.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:23.865975+0000) 2024-03-06T12:43:24.506 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:43:24.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:43:24.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:24.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:24.866183+0000) 2024-03-06T12:43:25.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:25.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:25.866412+0000) 2024-03-06T12:43:26.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:26.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:26.866643+0000) 2024-03-06T12:43:27.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:27.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:27.866876+0000) 2024-03-06T12:43:28.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:28.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:28.867099+0000) 2024-03-06T12:43:29.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:29.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:29.867325+0000) 2024-03-06T12:43:30.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:30.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:30.867589+0000) 2024-03-06T12:43:31.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:31.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:31.867818+0000) 2024-03-06T12:43:32.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:32.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:32.867988+0000) 2024-03-06T12:43:33.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:33.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:33.868174+0000) 2024-03-06T12:43:34.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:34.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:34.868360+0000) 2024-03-06T12:43:35.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:35.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:35.868631+0000) 2024-03-06T12:43:36.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:36.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:36.868867+0000) 2024-03-06T12:43:37.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:37.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:37.869065+0000) 2024-03-06T12:43:38.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:38.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:38.869253+0000) 2024-03-06T12:43:39.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:39.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:39.869453+0000) 2024-03-06T12:43:40.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:40.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:40.869696+0000) 2024-03-06T12:43:41.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:41.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:41.869924+0000) 2024-03-06T12:43:42.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:42.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:42.870110+0000) 2024-03-06T12:43:43.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:43.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:43.870309+0000) 2024-03-06T12:43:44.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:44.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:44.870575+0000) 2024-03-06T12:43:45.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:45.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:45.870813+0000) 2024-03-06T12:43:46.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:46.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:46.871050+0000) 2024-03-06T12:43:47.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:47.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:47.871275+0000) 2024-03-06T12:43:48.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:48.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:48.871489+0000) 2024-03-06T12:43:49.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:49.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:49.871720+0000) 2024-03-06T12:43:50.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:50.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:50.871957+0000) 2024-03-06T12:43:51.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:51.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:51.872159+0000) 2024-03-06T12:43:52.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:52.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:52.872390+0000) 2024-03-06T12:43:53.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:53.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:53.872609+0000) 2024-03-06T12:43:54.728 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:43:54.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:43:54.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:54.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:54.872812+0000) 2024-03-06T12:43:55.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:55.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:55.873006+0000) 2024-03-06T12:43:56.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:56.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:56.873208+0000) 2024-03-06T12:43:57.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:57.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:57.873430+0000) 2024-03-06T12:43:58.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:58.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:58.873663+0000) 2024-03-06T12:43:59.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:43:59.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:43:59.873867+0000) 2024-03-06T12:44:00.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:00.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:00.874095+0000) 2024-03-06T12:44:01.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:01.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:01.874318+0000) 2024-03-06T12:44:02.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:02.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:02.874497+0000) 2024-03-06T12:44:03.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:03.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:03.874765+0000) 2024-03-06T12:44:04.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:04.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:04.874991+0000) 2024-03-06T12:44:05.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:05.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:05.875218+0000) 2024-03-06T12:44:06.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:06.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:06.875450+0000) 2024-03-06T12:44:07.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:07.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:07.875683+0000) 2024-03-06T12:44:08.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:08.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:08.875892+0000) 2024-03-06T12:44:09.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:09.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:09.876142+0000) 2024-03-06T12:44:10.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:10.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:10.876385+0000) 2024-03-06T12:44:11.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:11.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:11.876660+0000) 2024-03-06T12:44:12.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:12.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:12.876889+0000) 2024-03-06T12:44:13.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:13.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:13.877129+0000) 2024-03-06T12:44:14.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:14.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:14.877412+0000) 2024-03-06T12:44:15.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:15.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:15.877704+0000) 2024-03-06T12:44:16.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:16.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:16.877954+0000) 2024-03-06T12:44:17.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:17.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:17.878132+0000) 2024-03-06T12:44:18.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:18.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:18.878376+0000) 2024-03-06T12:44:19.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:19.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:19.878658+0000) 2024-03-06T12:44:20.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:20.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:20.878903+0000) 2024-03-06T12:44:21.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:21.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:21.879147+0000) 2024-03-06T12:44:22.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:22.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:22.879409+0000) 2024-03-06T12:44:23.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:23.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:23.879656+0000) 2024-03-06T12:44:24.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:24.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:24.879898+0000) 2024-03-06T12:44:24.932 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:44:24.941 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:44:25.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:25.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:25.880154+0000) 2024-03-06T12:44:26.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:26.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:26.880365+0000) 2024-03-06T12:44:27.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:27.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:27.880646+0000) 2024-03-06T12:44:28.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:28.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:28.880847+0000) 2024-03-06T12:44:29.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:29.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:29.881103+0000) 2024-03-06T12:44:30.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:30.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:30.881349+0000) 2024-03-06T12:44:31.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:31.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:31.881529+0000) 2024-03-06T12:44:32.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:32.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:32.881784+0000) 2024-03-06T12:44:33.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:33.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:33.882007+0000) 2024-03-06T12:44:34.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:34.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:34.882213+0000) 2024-03-06T12:44:35.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:35.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:35.882447+0000) 2024-03-06T12:44:36.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:36.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:36.882727+0000) 2024-03-06T12:44:37.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:37.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:37.882936+0000) 2024-03-06T12:44:38.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:38.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:38.883163+0000) 2024-03-06T12:44:39.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:39.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:39.883399+0000) 2024-03-06T12:44:40.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:40.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:40.883668+0000) 2024-03-06T12:44:41.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:41.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:41.883863+0000) 2024-03-06T12:44:42.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:42.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:42.884045+0000) 2024-03-06T12:44:43.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:43.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:43.884281+0000) 2024-03-06T12:44:44.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:44.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:44.884523+0000) 2024-03-06T12:44:45.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:45.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:45.884831+0000) 2024-03-06T12:44:46.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:46.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:46.885078+0000) 2024-03-06T12:44:47.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:47.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:47.885322+0000) 2024-03-06T12:44:48.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:48.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:48.885579+0000) 2024-03-06T12:44:49.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:49.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:49.885810+0000) 2024-03-06T12:44:50.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:50.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:50.886054+0000) 2024-03-06T12:44:51.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:51.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:51.886273+0000) 2024-03-06T12:44:52.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:52.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:52.886491+0000) 2024-03-06T12:44:53.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:53.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:53.886718+0000) 2024-03-06T12:44:54.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:54.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:54.886945+0000) 2024-03-06T12:44:55.162 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:44:55.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:44:55.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:55.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:55.887175+0000) 2024-03-06T12:44:56.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:56.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:56.887403+0000) 2024-03-06T12:44:57.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:57.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:57.887681+0000) 2024-03-06T12:44:58.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:58.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:58.887923+0000) 2024-03-06T12:44:59.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:44:59.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:44:59.888170+0000) 2024-03-06T12:45:00.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:00.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:00.888420+0000) 2024-03-06T12:45:01.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:01.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:01.888686+0000) 2024-03-06T12:45:02.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:02.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:02.888921+0000) 2024-03-06T12:45:03.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:03.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:03.889163+0000) 2024-03-06T12:45:04.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:04.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:04.889357+0000) 2024-03-06T12:45:05.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:05.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:05.889639+0000) 2024-03-06T12:45:06.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:06.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:06.889865+0000) 2024-03-06T12:45:07.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:07.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:07.890085+0000) 2024-03-06T12:45:08.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:08.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:08.890355+0000) 2024-03-06T12:45:09.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:09.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:09.890671+0000) 2024-03-06T12:45:10.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:10.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:10.890905+0000) 2024-03-06T12:45:11.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:11.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:11.891120+0000) 2024-03-06T12:45:12.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:12.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:12.891377+0000) 2024-03-06T12:45:13.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:13.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:13.891690+0000) 2024-03-06T12:45:14.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:14.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:14.891892+0000) 2024-03-06T12:45:15.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:15.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:15.892128+0000) 2024-03-06T12:45:16.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:16.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:16.892363+0000) 2024-03-06T12:45:17.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:17.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:17.892634+0000) 2024-03-06T12:45:18.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:18.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:18.892861+0000) 2024-03-06T12:45:19.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:19.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:19.893092+0000) 2024-03-06T12:45:20.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:20.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:20.893333+0000) 2024-03-06T12:45:21.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:21.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:21.893593+0000) 2024-03-06T12:45:22.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:22.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:22.893776+0000) 2024-03-06T12:45:23.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:23.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:23.894002+0000) 2024-03-06T12:45:24.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:24.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:24.894240+0000) 2024-03-06T12:45:25.376 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:45:25.383 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:45:25.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:25.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:25.894467+0000) 2024-03-06T12:45:26.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:26.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:26.894699+0000) 2024-03-06T12:45:27.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:27.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:27.894897+0000) 2024-03-06T12:45:28.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:28.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:28.895093+0000) 2024-03-06T12:45:29.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:29.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:29.895325+0000) 2024-03-06T12:45:30.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:30.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:30.895596+0000) 2024-03-06T12:45:31.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:31.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:31.895874+0000) 2024-03-06T12:45:32.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:32.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:32.896157+0000) 2024-03-06T12:45:33.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:33.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:33.896440+0000) 2024-03-06T12:45:34.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:34.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:34.896689+0000) 2024-03-06T12:45:35.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:35.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:35.896979+0000) 2024-03-06T12:45:36.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:36.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:36.897213+0000) 2024-03-06T12:45:37.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:37.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:37.897476+0000) 2024-03-06T12:45:38.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:38.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:38.897759+0000) 2024-03-06T12:45:39.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:39.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:39.898042+0000) 2024-03-06T12:45:40.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:40.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:40.898312+0000) 2024-03-06T12:45:41.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:41.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:41.898496+0000) 2024-03-06T12:45:42.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:42.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:42.898700+0000) 2024-03-06T12:45:43.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:43.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:43.898927+0000) 2024-03-06T12:45:44.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:44.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:44.899153+0000) 2024-03-06T12:45:45.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:45.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:45.899393+0000) 2024-03-06T12:45:46.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:46.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:46.899647+0000) 2024-03-06T12:45:47.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:47.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:47.899901+0000) 2024-03-06T12:45:48.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:48.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:48.900187+0000) 2024-03-06T12:45:49.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:49.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:49.900421+0000) 2024-03-06T12:45:50.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:50.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:50.900721+0000) 2024-03-06T12:45:51.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:51.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:51.901006+0000) 2024-03-06T12:45:52.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:52.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:52.901183+0000) 2024-03-06T12:45:53.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:53.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:53.901368+0000) 2024-03-06T12:45:54.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:54.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:54.901528+0000) 2024-03-06T12:45:55.588 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:45:55.596 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:45:55.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:55.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:55.901748+0000) 2024-03-06T12:45:56.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:56.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:56.902018+0000) 2024-03-06T12:45:57.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:57.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:57.902244+0000) 2024-03-06T12:45:58.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:58.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:58.902480+0000) 2024-03-06T12:45:59.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:45:59.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:45:59.902713+0000) 2024-03-06T12:46:00.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:00.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:00.902942+0000) 2024-03-06T12:46:01.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:01.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:01.903145+0000) 2024-03-06T12:46:02.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:02.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:02.903420+0000) 2024-03-06T12:46:03.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:03.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:03.903686+0000) 2024-03-06T12:46:04.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:04.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:04.903922+0000) 2024-03-06T12:46:05.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:05.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:05.904160+0000) 2024-03-06T12:46:06.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:06.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:06.904467+0000) 2024-03-06T12:46:07.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:07.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:07.904742+0000) 2024-03-06T12:46:08.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:08.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:08.904965+0000) 2024-03-06T12:46:09.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:09.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:09.905187+0000) 2024-03-06T12:46:10.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:10.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:10.905462+0000) 2024-03-06T12:46:11.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:11.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:11.905701+0000) 2024-03-06T12:46:12.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:12.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:12.905942+0000) 2024-03-06T12:46:13.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:13.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:13.906141+0000) 2024-03-06T12:46:14.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:14.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:14.906387+0000) 2024-03-06T12:46:15.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:15.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:15.906645+0000) 2024-03-06T12:46:16.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:16.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:16.906898+0000) 2024-03-06T12:46:17.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:17.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:17.907101+0000) 2024-03-06T12:46:18.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:18.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:18.907377+0000) 2024-03-06T12:46:19.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:19.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:19.907685+0000) 2024-03-06T12:46:20.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:20.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:20.907969+0000) 2024-03-06T12:46:21.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:21.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:21.908238+0000) 2024-03-06T12:46:22.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:22.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:22.908457+0000) 2024-03-06T12:46:23.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:23.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:23.908735+0000) 2024-03-06T12:46:24.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:24.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:24.908981+0000) 2024-03-06T12:46:25.795 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:46:25.803 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:46:25.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:25.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:25.909230+0000) 2024-03-06T12:46:26.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:26.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:26.909581+0000) 2024-03-06T12:46:27.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:27.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:27.909868+0000) 2024-03-06T12:46:28.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:28.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:28.910140+0000) 2024-03-06T12:46:29.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:29.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:29.910423+0000) 2024-03-06T12:46:30.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:30.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:30.910698+0000) 2024-03-06T12:46:31.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:31.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:31.910894+0000) 2024-03-06T12:46:32.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:32.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:32.911115+0000) 2024-03-06T12:46:33.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:33.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:33.911353+0000) 2024-03-06T12:46:34.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:34.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:34.911634+0000) 2024-03-06T12:46:35.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:35.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:35.911882+0000) 2024-03-06T12:46:36.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:36.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:36.912101+0000) 2024-03-06T12:46:37.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:37.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:37.912309+0000) 2024-03-06T12:46:38.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:38.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:38.912497+0000) 2024-03-06T12:46:39.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:39.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:39.912782+0000) 2024-03-06T12:46:40.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:40.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:40.913027+0000) 2024-03-06T12:46:41.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:41.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:41.913277+0000) 2024-03-06T12:46:42.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:42.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:42.913521+0000) 2024-03-06T12:46:43.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:43.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:43.913772+0000) 2024-03-06T12:46:44.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:44.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:44.914004+0000) 2024-03-06T12:46:45.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:45.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:45.914254+0000) 2024-03-06T12:46:46.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:46.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:46.914442+0000) 2024-03-06T12:46:47.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:47.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:47.914666+0000) 2024-03-06T12:46:48.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:48.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:48.914889+0000) 2024-03-06T12:46:49.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:49.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:49.915169+0000) 2024-03-06T12:46:50.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:50.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:50.915393+0000) 2024-03-06T12:46:51.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:51.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:51.915681+0000) 2024-03-06T12:46:52.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:52.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:52.915906+0000) 2024-03-06T12:46:53.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:53.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:53.916143+0000) 2024-03-06T12:46:54.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:54.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:54.916352+0000) 2024-03-06T12:46:55.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:55.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:55.916639+0000) 2024-03-06T12:46:56.021 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:46:56.030 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:46:56.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:56.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:56.916874+0000) 2024-03-06T12:46:57.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:57.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:57.917065+0000) 2024-03-06T12:46:58.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:58.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:58.917286+0000) 2024-03-06T12:46:59.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:46:59.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:46:59.917512+0000) 2024-03-06T12:47:00.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:00.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:00.917779+0000) 2024-03-06T12:47:01.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:01.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:01.918001+0000) 2024-03-06T12:47:02.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:02.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:02.918211+0000) 2024-03-06T12:47:03.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:03.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:03.918416+0000) 2024-03-06T12:47:04.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:04.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:04.918644+0000) 2024-03-06T12:47:05.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:05.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:05.918798+0000) 2024-03-06T12:47:06.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:06.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:06.919006+0000) 2024-03-06T12:47:07.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:07.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:07.919247+0000) 2024-03-06T12:47:08.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:08.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:08.919564+0000) 2024-03-06T12:47:09.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:09.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:09.919781+0000) 2024-03-06T12:47:10.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:10.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:10.920006+0000) 2024-03-06T12:47:11.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:11.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:11.920228+0000) 2024-03-06T12:47:12.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:12.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:12.920423+0000) 2024-03-06T12:47:13.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:13.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:13.920651+0000) 2024-03-06T12:47:14.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:14.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:14.920875+0000) 2024-03-06T12:47:15.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:15.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:15.921038+0000) 2024-03-06T12:47:16.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:16.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:16.921270+0000) 2024-03-06T12:47:17.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:17.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:17.921506+0000) 2024-03-06T12:47:18.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:18.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:18.921787+0000) 2024-03-06T12:47:19.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:19.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:19.922027+0000) 2024-03-06T12:47:20.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:20.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:20.922216+0000) 2024-03-06T12:47:21.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:21.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:21.922461+0000) 2024-03-06T12:47:22.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:22.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:22.922668+0000) 2024-03-06T12:47:23.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:23.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:23.922888+0000) 2024-03-06T12:47:24.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:24.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:24.923131+0000) 2024-03-06T12:47:25.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:25.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:25.923434+0000) 2024-03-06T12:47:26.235 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:47:26.245 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:47:26.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:26.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:26.923713+0000) 2024-03-06T12:47:27.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:27.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:27.923956+0000) 2024-03-06T12:47:28.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:28.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:28.924192+0000) 2024-03-06T12:47:29.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:29.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:29.924417+0000) 2024-03-06T12:47:30.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:30.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:30.924660+0000) 2024-03-06T12:47:31.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:31.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:31.924849+0000) 2024-03-06T12:47:32.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:32.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:32.925112+0000) 2024-03-06T12:47:33.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:33.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:33.925309+0000) 2024-03-06T12:47:34.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:34.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:34.925570+0000) 2024-03-06T12:47:35.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:35.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:35.925815+0000) 2024-03-06T12:47:36.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:36.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:36.926051+0000) 2024-03-06T12:47:37.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:37.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:37.926289+0000) 2024-03-06T12:47:38.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:38.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:38.926483+0000) 2024-03-06T12:47:39.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:39.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:39.926693+0000) 2024-03-06T12:47:40.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:40.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:40.927044+0000) 2024-03-06T12:47:41.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:41.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:41.927233+0000) 2024-03-06T12:47:42.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:42.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:42.927423+0000) 2024-03-06T12:47:43.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:43.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:43.927672+0000) 2024-03-06T12:47:44.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:44.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:44.927896+0000) 2024-03-06T12:47:45.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:45.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:45.928087+0000) 2024-03-06T12:47:46.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:46.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:46.928297+0000) 2024-03-06T12:47:47.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:47.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:47.928595+0000) 2024-03-06T12:47:48.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:48.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:48.928738+0000) 2024-03-06T12:47:49.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:49.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:49.928933+0000) 2024-03-06T12:47:50.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:50.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:50.929124+0000) 2024-03-06T12:47:51.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:51.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:51.929352+0000) 2024-03-06T12:47:52.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:52.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:52.929595+0000) 2024-03-06T12:47:53.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:53.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:53.929787+0000) 2024-03-06T12:47:54.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:54.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:54.929976+0000) 2024-03-06T12:47:55.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:55.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:55.930166+0000) 2024-03-06T12:47:56.456 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:47:56.464 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:47:56.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:56.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:56.930355+0000) 2024-03-06T12:47:57.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:57.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:57.930615+0000) 2024-03-06T12:47:58.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:58.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:58.930829+0000) 2024-03-06T12:47:59.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:47:59.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:47:59.931021+0000) 2024-03-06T12:48:00.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:00.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:00.931241+0000) 2024-03-06T12:48:01.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:01.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:01.931462+0000) 2024-03-06T12:48:02.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:02.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:02.931679+0000) 2024-03-06T12:48:03.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:03.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:03.931903+0000) 2024-03-06T12:48:04.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:04.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:04.932073+0000) 2024-03-06T12:48:05.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:05.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:05.932310+0000) 2024-03-06T12:48:06.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:06.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:06.932586+0000) 2024-03-06T12:48:07.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:07.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:07.932795+0000) 2024-03-06T12:48:08.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:08.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:08.933035+0000) 2024-03-06T12:48:09.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:09.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:09.933277+0000) 2024-03-06T12:48:10.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:10.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:10.933507+0000) 2024-03-06T12:48:11.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:11.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:11.933741+0000) 2024-03-06T12:48:12.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:12.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:12.933967+0000) 2024-03-06T12:48:13.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:13.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:13.934158+0000) 2024-03-06T12:48:14.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:14.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:14.934391+0000) 2024-03-06T12:48:15.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:15.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:15.934665+0000) 2024-03-06T12:48:16.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:16.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:16.934906+0000) 2024-03-06T12:48:17.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:17.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:17.935133+0000) 2024-03-06T12:48:18.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:18.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:18.935355+0000) 2024-03-06T12:48:19.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:19.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:19.935625+0000) 2024-03-06T12:48:20.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:20.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:20.935869+0000) 2024-03-06T12:48:21.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:21.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:21.936068+0000) 2024-03-06T12:48:22.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:22.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:22.936310+0000) 2024-03-06T12:48:23.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:23.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:23.936575+0000) 2024-03-06T12:48:24.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:24.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:24.936805+0000) 2024-03-06T12:48:25.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:25.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:25.937086+0000) 2024-03-06T12:48:26.675 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:48:26.685 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:48:26.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:26.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:26.937331+0000) 2024-03-06T12:48:27.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:27.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:27.937636+0000) 2024-03-06T12:48:28.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:28.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:28.937866+0000) 2024-03-06T12:48:29.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:29.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:29.938119+0000) 2024-03-06T12:48:30.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:30.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:30.938363+0000) 2024-03-06T12:48:31.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:31.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:31.938665+0000) 2024-03-06T12:48:32.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:32.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:32.938921+0000) 2024-03-06T12:48:33.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:33.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:33.939167+0000) 2024-03-06T12:48:34.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:34.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:34.939409+0000) 2024-03-06T12:48:35.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:35.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:35.939713+0000) 2024-03-06T12:48:36.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:36.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:36.939954+0000) 2024-03-06T12:48:37.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:37.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:37.940184+0000) 2024-03-06T12:48:38.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:38.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:38.940412+0000) 2024-03-06T12:48:39.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:39.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:39.940650+0000) 2024-03-06T12:48:40.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:40.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:40.940851+0000) 2024-03-06T12:48:41.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:41.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:41.941043+0000) 2024-03-06T12:48:42.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:42.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:42.941271+0000) 2024-03-06T12:48:43.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:43.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:43.941447+0000) 2024-03-06T12:48:44.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:44.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:44.941688+0000) 2024-03-06T12:48:45.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:45.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:45.941917+0000) 2024-03-06T12:48:46.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:46.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:46.942158+0000) 2024-03-06T12:48:47.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:47.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:47.942388+0000) 2024-03-06T12:48:48.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:48.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:48.942648+0000) 2024-03-06T12:48:49.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:49.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:49.942853+0000) 2024-03-06T12:48:50.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:50.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:50.943073+0000) 2024-03-06T12:48:51.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:51.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:51.943330+0000) 2024-03-06T12:48:52.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:52.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:52.943602+0000) 2024-03-06T12:48:53.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:53.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:53.943806+0000) 2024-03-06T12:48:54.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:54.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:54.944075+0000) 2024-03-06T12:48:55.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:55.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:55.944317+0000) 2024-03-06T12:48:56.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:56.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:56.944636+0000) 2024-03-06T12:48:57.057 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:48:57.065 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:48:57.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:57.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:57.944872+0000) 2024-03-06T12:48:58.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:58.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:58.945109+0000) 2024-03-06T12:48:59.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:48:59.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:48:59.945294+0000) 2024-03-06T12:49:00.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:00.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:00.945529+0000) 2024-03-06T12:49:01.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:01.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:01.945774+0000) 2024-03-06T12:49:02.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:02.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:02.945998+0000) 2024-03-06T12:49:03.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:03.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:03.946265+0000) 2024-03-06T12:49:04.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:04.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:04.946592+0000) 2024-03-06T12:49:05.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:05.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:05.946873+0000) 2024-03-06T12:49:06.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:06.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:06.947121+0000) 2024-03-06T12:49:07.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:07.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:07.947322+0000) 2024-03-06T12:49:08.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:08.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:08.947630+0000) 2024-03-06T12:49:09.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:09.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:09.947881+0000) 2024-03-06T12:49:10.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:10.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:10.948094+0000) 2024-03-06T12:49:11.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:11.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:11.948346+0000) 2024-03-06T12:49:12.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:12.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:12.948608+0000) 2024-03-06T12:49:13.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:13.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:13.948890+0000) 2024-03-06T12:49:14.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:14.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:14.949111+0000) 2024-03-06T12:49:15.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:15.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:15.949373+0000) 2024-03-06T12:49:16.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:16.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:16.949642+0000) 2024-03-06T12:49:17.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:17.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:17.949873+0000) 2024-03-06T12:49:18.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:18.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:18.950109+0000) 2024-03-06T12:49:19.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:19.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:19.950334+0000) 2024-03-06T12:49:20.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:20.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:20.950599+0000) 2024-03-06T12:49:21.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:21.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:21.950842+0000) 2024-03-06T12:49:22.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:22.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:22.951077+0000) 2024-03-06T12:49:23.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:23.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:23.951275+0000) 2024-03-06T12:49:24.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:24.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:24.951520+0000) 2024-03-06T12:49:25.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:25.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:25.951793+0000) 2024-03-06T12:49:26.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:26.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:26.952051+0000) 2024-03-06T12:49:27.195 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:49:27.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:49:27.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:27.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:27.952296+0000) 2024-03-06T12:49:28.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:28.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:28.952570+0000) 2024-03-06T12:49:29.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:29.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:29.952773+0000) 2024-03-06T12:49:30.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:30.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:30.953012+0000) 2024-03-06T12:49:31.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:31.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:31.953247+0000) 2024-03-06T12:49:32.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:32.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:32.953468+0000) 2024-03-06T12:49:33.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:33.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:33.953692+0000) 2024-03-06T12:49:34.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:34.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:34.953913+0000) 2024-03-06T12:49:35.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:35.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:35.954138+0000) 2024-03-06T12:49:36.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:36.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:36.954412+0000) 2024-03-06T12:49:37.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:37.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:37.954673+0000) 2024-03-06T12:49:38.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:38.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:38.954918+0000) 2024-03-06T12:49:39.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:39.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:39.955163+0000) 2024-03-06T12:49:40.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:40.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:40.955392+0000) 2024-03-06T12:49:41.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:41.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:41.955694+0000) 2024-03-06T12:49:42.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:42.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:42.955909+0000) 2024-03-06T12:49:43.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:43.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:43.956149+0000) 2024-03-06T12:49:44.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:44.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:44.956394+0000) 2024-03-06T12:49:45.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:45.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:45.956671+0000) 2024-03-06T12:49:46.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:46.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:46.956903+0000) 2024-03-06T12:49:47.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:47.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:47.957145+0000) 2024-03-06T12:49:48.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:48.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:48.957393+0000) 2024-03-06T12:49:49.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:49.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:49.957671+0000) 2024-03-06T12:49:50.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:50.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:50.957896+0000) 2024-03-06T12:49:51.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:51.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:51.958145+0000) 2024-03-06T12:49:52.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:52.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:52.958385+0000) 2024-03-06T12:49:53.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:53.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:53.958669+0000) 2024-03-06T12:49:54.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:54.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:54.958877+0000) 2024-03-06T12:49:55.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:55.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:55.959119+0000) 2024-03-06T12:49:56.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:56.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:56.959341+0000) 2024-03-06T12:49:57.426 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:49:57.433 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:49:57.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:57.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:57.959642+0000) 2024-03-06T12:49:58.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:58.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:58.959867+0000) 2024-03-06T12:49:59.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:49:59.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:49:59.960101+0000) 2024-03-06T12:50:00.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:00.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:00.960370+0000) 2024-03-06T12:50:01.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:01.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:01.960651+0000) 2024-03-06T12:50:02.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:02.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:02.960871+0000) 2024-03-06T12:50:03.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:03.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:03.961064+0000) 2024-03-06T12:50:04.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:04.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:04.961298+0000) 2024-03-06T12:50:05.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:05.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:05.961529+0000) 2024-03-06T12:50:06.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:06.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:06.961871+0000) 2024-03-06T12:50:07.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:07.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:07.962111+0000) 2024-03-06T12:50:08.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:08.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:08.962309+0000) 2024-03-06T12:50:09.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:09.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:09.962514+0000) 2024-03-06T12:50:10.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:10.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:10.962788+0000) 2024-03-06T12:50:11.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:11.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:11.963016+0000) 2024-03-06T12:50:12.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:12.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:12.963260+0000) 2024-03-06T12:50:13.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:13.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:13.963447+0000) 2024-03-06T12:50:14.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:14.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:14.963691+0000) 2024-03-06T12:50:15.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:15.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:15.963843+0000) 2024-03-06T12:50:16.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:16.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:16.964099+0000) 2024-03-06T12:50:17.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:17.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:17.964279+0000) 2024-03-06T12:50:18.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:18.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:18.964468+0000) 2024-03-06T12:50:19.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:19.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:19.964668+0000) 2024-03-06T12:50:20.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:20.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:20.964876+0000) 2024-03-06T12:50:21.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:21.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:21.965118+0000) 2024-03-06T12:50:22.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:22.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:22.965299+0000) 2024-03-06T12:50:23.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:23.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:23.965571+0000) 2024-03-06T12:50:24.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:24.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:24.965781+0000) 2024-03-06T12:50:25.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:25.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:25.966063+0000) 2024-03-06T12:50:26.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:26.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:26.966363+0000) 2024-03-06T12:50:27.645 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:50:27.653 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:50:27.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:27.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:27.966668+0000) 2024-03-06T12:50:28.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:28.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:28.966845+0000) 2024-03-06T12:50:29.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:29.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:29.967072+0000) 2024-03-06T12:50:30.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:30.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:30.967292+0000) 2024-03-06T12:50:31.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:31.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:31.967528+0000) 2024-03-06T12:50:32.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:32.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:32.967808+0000) 2024-03-06T12:50:33.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:33.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:33.968042+0000) 2024-03-06T12:50:34.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:34.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:34.968283+0000) 2024-03-06T12:50:35.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:35.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:35.968566+0000) 2024-03-06T12:50:36.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:36.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:36.968838+0000) 2024-03-06T12:50:37.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:37.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:37.969070+0000) 2024-03-06T12:50:38.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:38.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:38.969374+0000) 2024-03-06T12:50:39.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:39.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:39.969679+0000) 2024-03-06T12:50:40.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:40.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:40.969902+0000) 2024-03-06T12:50:41.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:41.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:41.970135+0000) 2024-03-06T12:50:42.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:42.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:42.970303+0000) 2024-03-06T12:50:43.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:43.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:43.970575+0000) 2024-03-06T12:50:44.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:44.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:44.970789+0000) 2024-03-06T12:50:45.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:45.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:45.971015+0000) 2024-03-06T12:50:46.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:46.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:46.971259+0000) 2024-03-06T12:50:47.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:47.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:47.971458+0000) 2024-03-06T12:50:48.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:48.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:48.971702+0000) 2024-03-06T12:50:49.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:49.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:49.971932+0000) 2024-03-06T12:50:50.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:50.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:50.972156+0000) 2024-03-06T12:50:51.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:51.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:51.972401+0000) 2024-03-06T12:50:52.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:52.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:52.972668+0000) 2024-03-06T12:50:53.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:53.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:53.972907+0000) 2024-03-06T12:50:54.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:54.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:54.973066+0000) 2024-03-06T12:50:55.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:55.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:55.973269+0000) 2024-03-06T12:50:56.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:56.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:56.973519+0000) 2024-03-06T12:50:57.850 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:50:57.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:50:57.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:57.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:57.973782+0000) 2024-03-06T12:50:58.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:58.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:58.973975+0000) 2024-03-06T12:50:59.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:50:59.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:50:59.974218+0000) 2024-03-06T12:51:00.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:00.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:00.974472+0000) 2024-03-06T12:51:01.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:01.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:01.974721+0000) 2024-03-06T12:51:02.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:02.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:02.974956+0000) 2024-03-06T12:51:03.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:03.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:03.975189+0000) 2024-03-06T12:51:04.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:04.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:04.975411+0000) 2024-03-06T12:51:05.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:05.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:05.975640+0000) 2024-03-06T12:51:06.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:06.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:06.975893+0000) 2024-03-06T12:51:07.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:07.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:07.976130+0000) 2024-03-06T12:51:08.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:08.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:08.976361+0000) 2024-03-06T12:51:09.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:09.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:09.976640+0000) 2024-03-06T12:51:10.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:10.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:10.976855+0000) 2024-03-06T12:51:11.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:11.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:11.977054+0000) 2024-03-06T12:51:12.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:12.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:12.977281+0000) 2024-03-06T12:51:13.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:13.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:13.977462+0000) 2024-03-06T12:51:14.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:14.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:14.977693+0000) 2024-03-06T12:51:15.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:15.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:15.977853+0000) 2024-03-06T12:51:16.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:16.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:16.978047+0000) 2024-03-06T12:51:17.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:17.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:17.978263+0000) 2024-03-06T12:51:18.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:18.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:18.978500+0000) 2024-03-06T12:51:19.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:19.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:19.978733+0000) 2024-03-06T12:51:20.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:20.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:20.978934+0000) 2024-03-06T12:51:21.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:21.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:21.979208+0000) 2024-03-06T12:51:22.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:22.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:22.979372+0000) 2024-03-06T12:51:23.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:23.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:23.979566+0000) 2024-03-06T12:51:24.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:24.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:24.979842+0000) 2024-03-06T12:51:25.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:25.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:25.980063+0000) 2024-03-06T12:51:26.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:26.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:26.980311+0000) 2024-03-06T12:51:27.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:27.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:27.980567+0000) 2024-03-06T12:51:27.988 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:51:27.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:51:28.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:28.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:28.980801+0000) 2024-03-06T12:51:29.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:29.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:29.981019+0000) 2024-03-06T12:51:30.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:30.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:30.981244+0000) 2024-03-06T12:51:31.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:31.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:31.981484+0000) 2024-03-06T12:51:32.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:32.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:32.981728+0000) 2024-03-06T12:51:33.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:33.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:33.981962+0000) 2024-03-06T12:51:34.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:34.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:34.982158+0000) 2024-03-06T12:51:35.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:35.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:35.982400+0000) 2024-03-06T12:51:36.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:36.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:36.982668+0000) 2024-03-06T12:51:37.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:37.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:37.982911+0000) 2024-03-06T12:51:38.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:38.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:38.983154+0000) 2024-03-06T12:51:39.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:39.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:39.983369+0000) 2024-03-06T12:51:40.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:40.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:40.983688+0000) 2024-03-06T12:51:41.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:41.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:41.983925+0000) 2024-03-06T12:51:42.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:42.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:42.984160+0000) 2024-03-06T12:51:43.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:43.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:43.984365+0000) 2024-03-06T12:51:44.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:44.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:44.984618+0000) 2024-03-06T12:51:45.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:45.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:45.984848+0000) 2024-03-06T12:51:46.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:46.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:46.985103+0000) 2024-03-06T12:51:47.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:47.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:47.985318+0000) 2024-03-06T12:51:48.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:48.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:48.985655+0000) 2024-03-06T12:51:49.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:49.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:49.985881+0000) 2024-03-06T12:51:50.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:50.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:50.986107+0000) 2024-03-06T12:51:51.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:51.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:51.986329+0000) 2024-03-06T12:51:52.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:52.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:52.986576+0000) 2024-03-06T12:51:53.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:53.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:53.986801+0000) 2024-03-06T12:51:54.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:54.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:54.986993+0000) 2024-03-06T12:51:55.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:55.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:55.987185+0000) 2024-03-06T12:51:56.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:56.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:56.987471+0000) 2024-03-06T12:51:57.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:57.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:57.987751+0000) 2024-03-06T12:51:58.134 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:51:58.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:51:58.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:58.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:58.987988+0000) 2024-03-06T12:51:59.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:51:59.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:51:59.988214+0000) 2024-03-06T12:52:00.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:00.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:00.988482+0000) 2024-03-06T12:52:01.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:01.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:01.988770+0000) 2024-03-06T12:52:02.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:02.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:02.989051+0000) 2024-03-06T12:52:03.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:03.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:03.989341+0000) 2024-03-06T12:52:04.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:04.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:04.989654+0000) 2024-03-06T12:52:05.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:05.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:05.989852+0000) 2024-03-06T12:52:06.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:06.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:06.990094+0000) 2024-03-06T12:52:07.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:07.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:07.990300+0000) 2024-03-06T12:52:08.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:08.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:08.990513+0000) 2024-03-06T12:52:09.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:09.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:09.990820+0000) 2024-03-06T12:52:10.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:10.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:10.991045+0000) 2024-03-06T12:52:11.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:11.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:11.991229+0000) 2024-03-06T12:52:12.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:12.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:12.991458+0000) 2024-03-06T12:52:13.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:13.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:13.991685+0000) 2024-03-06T12:52:14.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:14.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:14.991912+0000) 2024-03-06T12:52:15.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:15.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:15.992135+0000) 2024-03-06T12:52:16.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:16.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:16.992383+0000) 2024-03-06T12:52:17.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:17.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:17.992647+0000) 2024-03-06T12:52:18.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:18.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:18.992858+0000) 2024-03-06T12:52:19.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:19.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:19.993069+0000) 2024-03-06T12:52:20.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:20.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:20.993335+0000) 2024-03-06T12:52:21.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:21.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:21.993638+0000) 2024-03-06T12:52:22.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:22.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:22.993864+0000) 2024-03-06T12:52:23.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:23.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:23.994088+0000) 2024-03-06T12:52:24.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:24.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:24.994309+0000) 2024-03-06T12:52:25.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:25.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:25.994524+0000) 2024-03-06T12:52:26.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:26.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:26.994808+0000) 2024-03-06T12:52:27.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:27.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:27.995056+0000) 2024-03-06T12:52:28.313 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:52:28.321 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:52:28.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:28.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:28.995272+0000) 2024-03-06T12:52:29.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:29.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:29.995479+0000) 2024-03-06T12:52:30.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:30.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:30.995713+0000) 2024-03-06T12:52:31.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:31.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:31.995924+0000) 2024-03-06T12:52:32.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:32.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:32.996160+0000) 2024-03-06T12:52:33.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:33.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:33.996401+0000) 2024-03-06T12:52:34.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:34.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:34.996650+0000) 2024-03-06T12:52:35.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:35.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:35.996856+0000) 2024-03-06T12:52:36.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:36.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:36.997161+0000) 2024-03-06T12:52:37.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:37.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:37.997382+0000) 2024-03-06T12:52:38.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:38.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:38.997662+0000) 2024-03-06T12:52:39.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:39.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:39.997847+0000) 2024-03-06T12:52:40.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:40.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:40.998093+0000) 2024-03-06T12:52:41.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:41.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:41.998318+0000) 2024-03-06T12:52:42.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:42.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:42.998653+0000) 2024-03-06T12:52:43.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:43.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:43.998920+0000) 2024-03-06T12:52:44.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:44.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:44.999190+0000) 2024-03-06T12:52:45.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:45.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:45.999426+0000) 2024-03-06T12:52:46.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:46.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:46.999659+0000) 2024-03-06T12:52:47.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:47.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:47.999885+0000) 2024-03-06T12:52:48.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:48.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:49.000164+0000) 2024-03-06T12:52:49.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:49.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:50.000447+0000) 2024-03-06T12:52:50.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:50.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:51.000740+0000) 2024-03-06T12:52:51.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:51.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:52.000962+0000) 2024-03-06T12:52:52.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:52.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:53.001245+0000) 2024-03-06T12:52:53.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:54.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:54.001584+0000) 2024-03-06T12:52:54.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:55.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:55.001860+0000) 2024-03-06T12:52:55.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:56.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:56.002075+0000) 2024-03-06T12:52:56.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:57.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:57.002344+0000) 2024-03-06T12:52:57.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:58.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:58.002591+0000) 2024-03-06T12:52:58.527 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:52:58.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:52:58.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:52:59.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:52:59.002867+0000) 2024-03-06T12:52:59.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:00.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:00.003060+0000) 2024-03-06T12:53:00.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:01.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:01.003298+0000) 2024-03-06T12:53:01.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:02.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:02.003566+0000) 2024-03-06T12:53:02.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:03.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:03.003757+0000) 2024-03-06T12:53:03.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:04.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:04.003999+0000) 2024-03-06T12:53:04.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:05.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:05.004203+0000) 2024-03-06T12:53:05.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:06.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:06.004420+0000) 2024-03-06T12:53:06.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:07.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:07.004690+0000) 2024-03-06T12:53:07.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:08.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:08.004898+0000) 2024-03-06T12:53:08.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:09.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:09.005149+0000) 2024-03-06T12:53:09.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:10.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:10.005350+0000) 2024-03-06T12:53:10.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:11.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:11.005580+0000) 2024-03-06T12:53:11.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:12.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:12.005817+0000) 2024-03-06T12:53:12.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:13.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:13.006046+0000) 2024-03-06T12:53:13.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:14.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:14.006280+0000) 2024-03-06T12:53:14.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:15.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:15.006494+0000) 2024-03-06T12:53:15.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:16.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:16.006771+0000) 2024-03-06T12:53:16.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:17.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:17.007023+0000) 2024-03-06T12:53:17.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:18.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:18.007251+0000) 2024-03-06T12:53:18.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:19.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:19.007482+0000) 2024-03-06T12:53:19.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:20.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:20.007711+0000) 2024-03-06T12:53:20.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:21.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:21.007940+0000) 2024-03-06T12:53:21.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:22.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:22.008164+0000) 2024-03-06T12:53:22.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:23.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:23.008390+0000) 2024-03-06T12:53:23.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:24.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:24.008661+0000) 2024-03-06T12:53:24.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:25.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:25.008950+0000) 2024-03-06T12:53:25.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:26.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:26.009175+0000) 2024-03-06T12:53:26.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:27.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:27.009405+0000) 2024-03-06T12:53:27.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:28.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:28.009665+0000) 2024-03-06T12:53:28.757 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:53:28.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:53:28.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:29.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:29.009877+0000) 2024-03-06T12:53:29.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:30.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:30.010103+0000) 2024-03-06T12:53:30.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:31.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:31.010304+0000) 2024-03-06T12:53:31.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:32.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:32.010518+0000) 2024-03-06T12:53:33.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:33.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:33.010760+0000) 2024-03-06T12:53:34.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:34.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:34.010990+0000) 2024-03-06T12:53:35.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:35.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:35.011217+0000) 2024-03-06T12:53:36.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:36.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:36.011392+0000) 2024-03-06T12:53:37.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:37.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:37.011621+0000) 2024-03-06T12:53:38.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:38.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:38.011862+0000) 2024-03-06T12:53:39.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:39.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:39.012052+0000) 2024-03-06T12:53:40.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:40.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:40.012282+0000) 2024-03-06T12:53:41.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:41.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:41.012472+0000) 2024-03-06T12:53:42.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:42.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:42.012701+0000) 2024-03-06T12:53:43.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:43.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:43.012919+0000) 2024-03-06T12:53:44.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:44.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:44.013150+0000) 2024-03-06T12:53:45.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:45.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:45.013367+0000) 2024-03-06T12:53:46.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:46.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:46.013638+0000) 2024-03-06T12:53:47.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:47.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:47.013851+0000) 2024-03-06T12:53:48.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:48.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:48.014080+0000) 2024-03-06T12:53:49.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:49.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:49.014286+0000) 2024-03-06T12:53:50.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:50.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:50.014488+0000) 2024-03-06T12:53:51.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:51.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:51.014720+0000) 2024-03-06T12:53:52.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:52.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:52.014951+0000) 2024-03-06T12:53:53.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:53.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:53.015166+0000) 2024-03-06T12:53:54.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:54.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:54.015394+0000) 2024-03-06T12:53:55.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:55.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:55.015651+0000) 2024-03-06T12:53:56.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:56.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:56.015861+0000) 2024-03-06T12:53:57.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:57.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:57.016091+0000) 2024-03-06T12:53:58.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:58.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:58.016338+0000) 2024-03-06T12:53:58.986 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:53:58.994 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:53:59.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:53:59.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:53:59.016513+0000) 2024-03-06T12:54:00.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:00.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:00.016778+0000) 2024-03-06T12:54:01.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:01.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:01.017018+0000) 2024-03-06T12:54:02.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:02.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:02.017225+0000) 2024-03-06T12:54:03.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:03.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:03.017483+0000) 2024-03-06T12:54:04.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:04.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:04.017665+0000) 2024-03-06T12:54:05.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:05.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:05.017892+0000) 2024-03-06T12:54:06.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:06.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:06.018087+0000) 2024-03-06T12:54:07.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:07.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:07.018326+0000) 2024-03-06T12:54:08.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:08.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:08.018603+0000) 2024-03-06T12:54:09.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:09.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:09.018830+0000) 2024-03-06T12:54:10.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:10.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:10.019021+0000) 2024-03-06T12:54:11.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:11.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:11.019246+0000) 2024-03-06T12:54:12.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:12.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:12.019437+0000) 2024-03-06T12:54:13.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:13.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:13.019676+0000) 2024-03-06T12:54:14.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:14.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:14.019862+0000) 2024-03-06T12:54:15.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:15.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:15.020056+0000) 2024-03-06T12:54:16.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:16.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:16.020251+0000) 2024-03-06T12:54:17.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:17.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:17.020483+0000) 2024-03-06T12:54:18.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:18.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:18.020765+0000) 2024-03-06T12:54:19.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:19.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:19.020998+0000) 2024-03-06T12:54:20.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:20.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:20.021222+0000) 2024-03-06T12:54:21.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:21.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:21.021457+0000) 2024-03-06T12:54:22.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:22.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:22.021664+0000) 2024-03-06T12:54:23.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:23.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:23.021900+0000) 2024-03-06T12:54:24.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:24.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:24.022122+0000) 2024-03-06T12:54:25.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:25.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:25.022303+0000) 2024-03-06T12:54:26.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:26.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:26.022517+0000) 2024-03-06T12:54:27.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:27.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:27.022750+0000) 2024-03-06T12:54:28.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:28.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:28.023007+0000) 2024-03-06T12:54:29.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:29.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:29.023240+0000) 2024-03-06T12:54:29.124 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:54:29.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:54:30.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:30.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:30.023461+0000) 2024-03-06T12:54:31.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:31.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:31.023702+0000) 2024-03-06T12:54:32.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:32.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:32.023889+0000) 2024-03-06T12:54:33.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:33.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:33.024080+0000) 2024-03-06T12:54:34.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:34.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:34.024300+0000) 2024-03-06T12:54:35.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:35.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:35.024525+0000) 2024-03-06T12:54:36.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:36.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:36.024787+0000) 2024-03-06T12:54:37.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:37.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:37.024963+0000) 2024-03-06T12:54:38.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:38.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:38.025145+0000) 2024-03-06T12:54:39.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:39.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:39.025362+0000) 2024-03-06T12:54:40.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:40.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:40.025613+0000) 2024-03-06T12:54:41.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:41.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:41.025796+0000) 2024-03-06T12:54:42.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:42.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:42.026021+0000) 2024-03-06T12:54:43.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:43.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:43.026253+0000) 2024-03-06T12:54:44.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:44.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:44.026474+0000) 2024-03-06T12:54:45.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:45.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:45.026702+0000) 2024-03-06T12:54:46.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:46.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:46.026892+0000) 2024-03-06T12:54:47.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:47.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:47.027123+0000) 2024-03-06T12:54:48.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:48.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:48.027331+0000) 2024-03-06T12:54:49.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:49.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:49.027570+0000) 2024-03-06T12:54:50.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:50.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:50.027777+0000) 2024-03-06T12:54:51.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:51.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:51.028010+0000) 2024-03-06T12:54:52.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:52.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:52.028187+0000) 2024-03-06T12:54:53.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:53.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:53.028379+0000) 2024-03-06T12:54:54.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:54.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:54.028667+0000) 2024-03-06T12:54:55.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:55.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:55.028934+0000) 2024-03-06T12:54:56.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:56.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:56.029169+0000) 2024-03-06T12:54:57.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:57.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:57.029424+0000) 2024-03-06T12:54:58.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:58.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:58.029701+0000) 2024-03-06T12:54:59.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:54:59.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:54:59.029917+0000) 2024-03-06T12:54:59.329 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:54:59.337 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:55:00.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:00.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:00.030138+0000) 2024-03-06T12:55:01.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:01.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:01.030400+0000) 2024-03-06T12:55:02.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:02.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:02.030680+0000) 2024-03-06T12:55:03.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:03.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:03.030919+0000) 2024-03-06T12:55:04.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:04.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:04.031201+0000) 2024-03-06T12:55:05.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:05.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:05.031420+0000) 2024-03-06T12:55:06.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:06.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:06.031663+0000) 2024-03-06T12:55:07.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:07.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:07.031847+0000) 2024-03-06T12:55:08.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:08.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:08.032075+0000) 2024-03-06T12:55:09.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:09.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:09.032275+0000) 2024-03-06T12:55:10.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:10.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:10.032510+0000) 2024-03-06T12:55:11.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:11.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:11.032724+0000) 2024-03-06T12:55:12.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:12.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:12.032944+0000) 2024-03-06T12:55:13.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:13.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:13.033174+0000) 2024-03-06T12:55:14.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:14.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:14.033401+0000) 2024-03-06T12:55:15.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:15.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:15.033642+0000) 2024-03-06T12:55:16.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:16.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:16.033878+0000) 2024-03-06T12:55:17.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:17.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:17.034130+0000) 2024-03-06T12:55:18.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:18.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:18.034352+0000) 2024-03-06T12:55:19.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:19.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:19.034593+0000) 2024-03-06T12:55:20.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:20.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:20.034785+0000) 2024-03-06T12:55:21.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:21.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:21.035032+0000) 2024-03-06T12:55:22.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:22.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:22.035255+0000) 2024-03-06T12:55:23.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:23.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:23.035481+0000) 2024-03-06T12:55:24.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:24.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:24.035716+0000) 2024-03-06T12:55:25.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:25.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:25.035960+0000) 2024-03-06T12:55:26.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:26.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:26.036180+0000) 2024-03-06T12:55:27.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:27.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:27.036414+0000) 2024-03-06T12:55:28.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:28.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:28.036640+0000) 2024-03-06T12:55:29.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:29.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:29.036877+0000) 2024-03-06T12:55:29.517 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:55:29.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:55:30.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:30.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:30.037100+0000) 2024-03-06T12:55:31.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:31.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:31.037326+0000) 2024-03-06T12:55:32.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:32.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:32.037584+0000) 2024-03-06T12:55:33.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:33.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:33.037782+0000) 2024-03-06T12:55:34.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:34.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:34.038004+0000) 2024-03-06T12:55:35.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:35.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:35.038230+0000) 2024-03-06T12:55:36.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:36.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:36.038453+0000) 2024-03-06T12:55:37.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:37.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:37.038650+0000) 2024-03-06T12:55:38.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:38.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:38.038849+0000) 2024-03-06T12:55:39.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:39.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:39.039073+0000) 2024-03-06T12:55:40.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:40.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:40.039294+0000) 2024-03-06T12:55:41.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:41.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:41.039520+0000) 2024-03-06T12:55:42.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:42.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:42.039727+0000) 2024-03-06T12:55:43.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:43.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:43.039932+0000) 2024-03-06T12:55:44.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:44.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:44.040146+0000) 2024-03-06T12:55:45.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:45.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:45.040372+0000) 2024-03-06T12:55:46.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:46.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:46.040673+0000) 2024-03-06T12:55:47.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:47.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:47.040957+0000) 2024-03-06T12:55:48.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:48.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:48.041212+0000) 2024-03-06T12:55:49.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:49.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:49.041424+0000) 2024-03-06T12:55:50.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:50.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:50.041661+0000) 2024-03-06T12:55:51.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:51.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:51.041903+0000) 2024-03-06T12:55:52.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:52.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:52.042134+0000) 2024-03-06T12:55:53.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:53.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:53.042384+0000) 2024-03-06T12:55:54.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:54.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:54.042660+0000) 2024-03-06T12:55:55.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:55.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:55.042901+0000) 2024-03-06T12:55:56.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:56.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:56.043147+0000) 2024-03-06T12:55:57.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:57.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:57.043369+0000) 2024-03-06T12:55:58.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:58.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:58.043589+0000) 2024-03-06T12:55:59.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:55:59.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:55:59.043877+0000) 2024-03-06T12:55:59.702 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:55:59.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:56:00.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:00.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:00.044152+0000) 2024-03-06T12:56:01.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:01.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:01.044418+0000) 2024-03-06T12:56:02.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:02.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:02.044694+0000) 2024-03-06T12:56:03.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:03.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:03.044903+0000) 2024-03-06T12:56:04.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:04.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:04.045126+0000) 2024-03-06T12:56:05.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:05.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:05.045359+0000) 2024-03-06T12:56:06.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:06.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:06.045614+0000) 2024-03-06T12:56:07.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:07.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:07.045819+0000) 2024-03-06T12:56:08.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:08.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:08.046008+0000) 2024-03-06T12:56:09.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:09.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:09.046247+0000) 2024-03-06T12:56:10.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:10.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:10.046472+0000) 2024-03-06T12:56:11.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:11.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:11.046697+0000) 2024-03-06T12:56:12.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:12.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:12.046918+0000) 2024-03-06T12:56:13.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:13.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:13.047108+0000) 2024-03-06T12:56:14.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:14.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:14.047324+0000) 2024-03-06T12:56:15.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:15.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:15.047517+0000) 2024-03-06T12:56:16.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:16.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:16.047749+0000) 2024-03-06T12:56:17.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:17.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:17.047950+0000) 2024-03-06T12:56:18.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:18.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:18.048193+0000) 2024-03-06T12:56:19.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:19.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:19.048394+0000) 2024-03-06T12:56:20.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:20.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:20.048657+0000) 2024-03-06T12:56:21.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:21.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:21.048883+0000) 2024-03-06T12:56:22.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:22.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:22.049100+0000) 2024-03-06T12:56:23.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:23.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:23.049338+0000) 2024-03-06T12:56:24.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:24.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:24.049595+0000) 2024-03-06T12:56:25.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:25.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:25.049790+0000) 2024-03-06T12:56:26.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:26.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:26.050014+0000) 2024-03-06T12:56:27.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:27.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:27.050218+0000) 2024-03-06T12:56:28.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:28.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:28.050465+0000) 2024-03-06T12:56:29.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:29.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:29.050698+0000) 2024-03-06T12:56:29.906 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:56:29.914 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:56:30.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:30.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:30.050890+0000) 2024-03-06T12:56:31.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:31.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:31.051135+0000) 2024-03-06T12:56:32.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:32.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:32.051319+0000) 2024-03-06T12:56:33.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:33.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:33.051600+0000) 2024-03-06T12:56:34.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:34.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:34.051896+0000) 2024-03-06T12:56:35.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:35.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:35.052091+0000) 2024-03-06T12:56:36.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:36.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:36.052285+0000) 2024-03-06T12:56:37.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:37.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:37.052572+0000) 2024-03-06T12:56:38.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:38.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:38.052795+0000) 2024-03-06T12:56:39.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:39.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:39.052957+0000) 2024-03-06T12:56:40.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:40.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:40.053176+0000) 2024-03-06T12:56:41.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:41.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:41.053363+0000) 2024-03-06T12:56:42.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:42.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:42.053672+0000) 2024-03-06T12:56:43.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:43.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:43.053918+0000) 2024-03-06T12:56:44.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:44.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:44.054127+0000) 2024-03-06T12:56:45.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:45.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:45.054318+0000) 2024-03-06T12:56:46.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:46.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:46.054564+0000) 2024-03-06T12:56:47.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:47.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:47.054782+0000) 2024-03-06T12:56:48.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:48.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:48.054963+0000) 2024-03-06T12:56:49.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:49.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:49.055172+0000) 2024-03-06T12:56:50.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:50.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:50.055412+0000) 2024-03-06T12:56:51.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:51.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:51.055652+0000) 2024-03-06T12:56:52.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:52.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:52.055848+0000) 2024-03-06T12:56:53.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:53.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:53.056090+0000) 2024-03-06T12:56:54.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:54.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:54.056317+0000) 2024-03-06T12:56:55.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:55.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:55.056576+0000) 2024-03-06T12:56:56.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:56.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:56.056806+0000) 2024-03-06T12:56:57.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:57.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:57.057117+0000) 2024-03-06T12:56:58.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:58.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:58.057386+0000) 2024-03-06T12:56:59.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:56:59.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:56:59.057699+0000) 2024-03-06T12:57:00.039 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:57:00.047 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:57:00.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:00.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:00.057859+0000) 2024-03-06T12:57:01.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:01.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:01.058057+0000) 2024-03-06T12:57:02.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:02.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:02.058283+0000) 2024-03-06T12:57:03.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:03.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:03.058520+0000) 2024-03-06T12:57:04.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:04.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:04.058755+0000) 2024-03-06T12:57:05.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:05.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:05.059003+0000) 2024-03-06T12:57:06.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:06.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:06.059235+0000) 2024-03-06T12:57:07.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:07.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:07.059489+0000) 2024-03-06T12:57:08.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:08.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:08.059745+0000) 2024-03-06T12:57:09.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:09.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:09.059997+0000) 2024-03-06T12:57:10.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:10.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:10.060204+0000) 2024-03-06T12:57:11.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:11.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:11.060441+0000) 2024-03-06T12:57:12.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:12.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:12.060688+0000) 2024-03-06T12:57:13.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:13.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:13.060933+0000) 2024-03-06T12:57:14.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:14.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:14.061161+0000) 2024-03-06T12:57:15.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:15.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:15.061402+0000) 2024-03-06T12:57:16.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:16.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:16.061654+0000) 2024-03-06T12:57:17.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:17.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:17.061909+0000) 2024-03-06T12:57:18.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:18.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:18.062150+0000) 2024-03-06T12:57:19.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:19.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:19.062375+0000) 2024-03-06T12:57:20.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:20.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:20.062610+0000) 2024-03-06T12:57:21.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:21.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:21.062812+0000) 2024-03-06T12:57:22.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:22.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:22.063035+0000) 2024-03-06T12:57:23.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:23.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:23.063265+0000) 2024-03-06T12:57:24.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:24.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:24.063509+0000) 2024-03-06T12:57:25.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:25.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:25.063795+0000) 2024-03-06T12:57:26.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:26.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:26.063968+0000) 2024-03-06T12:57:27.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:27.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:27.064174+0000) 2024-03-06T12:57:28.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:28.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:28.064406+0000) 2024-03-06T12:57:29.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:29.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:29.064672+0000) 2024-03-06T12:57:30.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:30.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:30.064862+0000) 2024-03-06T12:57:30.187 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:57:30.195 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:57:31.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:31.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:31.065113+0000) 2024-03-06T12:57:32.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:32.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:32.065345+0000) 2024-03-06T12:57:33.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:33.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:33.065634+0000) 2024-03-06T12:57:34.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:34.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:34.065829+0000) 2024-03-06T12:57:35.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:35.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:35.066124+0000) 2024-03-06T12:57:36.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:36.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:36.066410+0000) 2024-03-06T12:57:37.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:37.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:37.066688+0000) 2024-03-06T12:57:38.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:38.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:38.066969+0000) 2024-03-06T12:57:39.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:39.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:39.067247+0000) 2024-03-06T12:57:40.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:40.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:40.067520+0000) 2024-03-06T12:57:41.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:41.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:41.067784+0000) 2024-03-06T12:57:42.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:42.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:42.068008+0000) 2024-03-06T12:57:43.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:43.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:43.068197+0000) 2024-03-06T12:57:44.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:44.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:44.068419+0000) 2024-03-06T12:57:45.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:45.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:45.068652+0000) 2024-03-06T12:57:46.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:46.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:46.068891+0000) 2024-03-06T12:57:47.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:47.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:47.069121+0000) 2024-03-06T12:57:48.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:48.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:48.069360+0000) 2024-03-06T12:57:49.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:49.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:49.069574+0000) 2024-03-06T12:57:50.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:50.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:50.069805+0000) 2024-03-06T12:57:51.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:51.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:51.070034+0000) 2024-03-06T12:57:52.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:52.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:52.070254+0000) 2024-03-06T12:57:53.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:53.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:53.070483+0000) 2024-03-06T12:57:54.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:54.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:54.070723+0000) 2024-03-06T12:57:55.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:55.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:55.070953+0000) 2024-03-06T12:57:56.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:56.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:56.071150+0000) 2024-03-06T12:57:57.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:57.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:57.071406+0000) 2024-03-06T12:57:58.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:58.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:58.071639+0000) 2024-03-06T12:57:59.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:57:59.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:57:59.071867+0000) 2024-03-06T12:58:00.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:00.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:00.072095+0000) 2024-03-06T12:58:00.383 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:58:00.391 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:58:01.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:01.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:01.072371+0000) 2024-03-06T12:58:02.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:02.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:02.072678+0000) 2024-03-06T12:58:03.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:03.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:03.072897+0000) 2024-03-06T12:58:04.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:04.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:04.073172+0000) 2024-03-06T12:58:05.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:05.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:05.073386+0000) 2024-03-06T12:58:06.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:06.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:06.073661+0000) 2024-03-06T12:58:07.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:07.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:07.073870+0000) 2024-03-06T12:58:08.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:08.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:08.074097+0000) 2024-03-06T12:58:09.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:09.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:09.074326+0000) 2024-03-06T12:58:10.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:10.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:10.074565+0000) 2024-03-06T12:58:11.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:11.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:11.074787+0000) 2024-03-06T12:58:12.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:12.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:12.075019+0000) 2024-03-06T12:58:13.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:13.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:13.075257+0000) 2024-03-06T12:58:14.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:14.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:14.075472+0000) 2024-03-06T12:58:15.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:15.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:15.075737+0000) 2024-03-06T12:58:16.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:16.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:16.075941+0000) 2024-03-06T12:58:17.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:17.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:17.076186+0000) 2024-03-06T12:58:18.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:18.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:18.076407+0000) 2024-03-06T12:58:19.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:19.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:19.076677+0000) 2024-03-06T12:58:20.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:20.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:20.076896+0000) 2024-03-06T12:58:21.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:21.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:21.077126+0000) 2024-03-06T12:58:22.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:22.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:22.077346+0000) 2024-03-06T12:58:23.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:23.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:23.077613+0000) 2024-03-06T12:58:24.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:24.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:24.077839+0000) 2024-03-06T12:58:25.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:25.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:25.078059+0000) 2024-03-06T12:58:26.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:26.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:26.078270+0000) 2024-03-06T12:58:27.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:27.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:27.078509+0000) 2024-03-06T12:58:28.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:28.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:28.078783+0000) 2024-03-06T12:58:29.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:29.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:29.079067+0000) 2024-03-06T12:58:30.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:30.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:30.079291+0000) 2024-03-06T12:58:30.563 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:58:30.571 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:58:31.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:31.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:31.079488+0000) 2024-03-06T12:58:32.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:32.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:32.079708+0000) 2024-03-06T12:58:33.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:33.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:33.079906+0000) 2024-03-06T12:58:34.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:34.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:34.080136+0000) 2024-03-06T12:58:35.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:35.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:35.080369+0000) 2024-03-06T12:58:36.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:36.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:36.080632+0000) 2024-03-06T12:58:37.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:37.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:37.080831+0000) 2024-03-06T12:58:38.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:38.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:38.081098+0000) 2024-03-06T12:58:39.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:39.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:39.081305+0000) 2024-03-06T12:58:40.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:40.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:40.081518+0000) 2024-03-06T12:58:41.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:41.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:41.081788+0000) 2024-03-06T12:58:42.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:42.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:42.082009+0000) 2024-03-06T12:58:43.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:43.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:43.082213+0000) 2024-03-06T12:58:44.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:44.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:44.082436+0000) 2024-03-06T12:58:45.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:45.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:45.082668+0000) 2024-03-06T12:58:46.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:46.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:46.082880+0000) 2024-03-06T12:58:47.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:47.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:47.083114+0000) 2024-03-06T12:58:48.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:48.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:48.083333+0000) 2024-03-06T12:58:49.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:49.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:49.083595+0000) 2024-03-06T12:58:50.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:50.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:50.083794+0000) 2024-03-06T12:58:51.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:51.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:51.084025+0000) 2024-03-06T12:58:52.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:52.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:52.084206+0000) 2024-03-06T12:58:53.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:53.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:53.084448+0000) 2024-03-06T12:58:54.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:54.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:54.084648+0000) 2024-03-06T12:58:55.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:55.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:55.084864+0000) 2024-03-06T12:58:56.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:56.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:56.085094+0000) 2024-03-06T12:58:57.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:57.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:57.085326+0000) 2024-03-06T12:58:58.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:58.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:58.085571+0000) 2024-03-06T12:58:59.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:58:59.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:58:59.085846+0000) 2024-03-06T12:59:00.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:00.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:00.086125+0000) 2024-03-06T12:59:00.785 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:59:00.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:59:01.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:01.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:01.086373+0000) 2024-03-06T12:59:02.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:02.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:02.086675+0000) 2024-03-06T12:59:03.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:03.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:03.086884+0000) 2024-03-06T12:59:04.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:04.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:04.087108+0000) 2024-03-06T12:59:05.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:05.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:05.087356+0000) 2024-03-06T12:59:06.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:06.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:06.087628+0000) 2024-03-06T12:59:07.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:07.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:07.087886+0000) 2024-03-06T12:59:08.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:08.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:08.088088+0000) 2024-03-06T12:59:09.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:09.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:09.088324+0000) 2024-03-06T12:59:10.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:10.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:10.088574+0000) 2024-03-06T12:59:11.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:11.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:11.088812+0000) 2024-03-06T12:59:12.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:12.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:12.089037+0000) 2024-03-06T12:59:13.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:13.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:13.089261+0000) 2024-03-06T12:59:14.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:14.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:14.089569+0000) 2024-03-06T12:59:15.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:15.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:15.089849+0000) 2024-03-06T12:59:16.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:16.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:16.090127+0000) 2024-03-06T12:59:17.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:17.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:17.090477+0000) 2024-03-06T12:59:18.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:18.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:18.090760+0000) 2024-03-06T12:59:19.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:19.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:19.091029+0000) 2024-03-06T12:59:20.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:20.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:20.091296+0000) 2024-03-06T12:59:21.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:21.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:21.091620+0000) 2024-03-06T12:59:22.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:22.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:22.091813+0000) 2024-03-06T12:59:23.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:23.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:23.092037+0000) 2024-03-06T12:59:24.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:24.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:24.092238+0000) 2024-03-06T12:59:25.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:25.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:25.092491+0000) 2024-03-06T12:59:26.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:26.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:26.092712+0000) 2024-03-06T12:59:27.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:27.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:27.092925+0000) 2024-03-06T12:59:28.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:28.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:28.093142+0000) 2024-03-06T12:59:29.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:29.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:29.093368+0000) 2024-03-06T12:59:30.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:30.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:30.093646+0000) 2024-03-06T12:59:30.916 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:59:30.927 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T12:59:31.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:31.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:31.093885+0000) 2024-03-06T12:59:32.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:32.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:32.094099+0000) 2024-03-06T12:59:33.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:33.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:33.094329+0000) 2024-03-06T12:59:34.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:34.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:34.094487+0000) 2024-03-06T12:59:35.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:35.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:35.094719+0000) 2024-03-06T12:59:36.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:36.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:36.094939+0000) 2024-03-06T12:59:37.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:37.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:37.095173+0000) 2024-03-06T12:59:38.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:38.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:38.095382+0000) 2024-03-06T12:59:39.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:39.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:39.095644+0000) 2024-03-06T12:59:40.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:40.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:40.095869+0000) 2024-03-06T12:59:41.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:41.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:41.096097+0000) 2024-03-06T12:59:42.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:42.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:42.096318+0000) 2024-03-06T12:59:43.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:43.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:43.096583+0000) 2024-03-06T12:59:44.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:44.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:44.096863+0000) 2024-03-06T12:59:45.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:45.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:45.097144+0000) 2024-03-06T12:59:46.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:46.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:46.097423+0000) 2024-03-06T12:59:47.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:47.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:47.097708+0000) 2024-03-06T12:59:48.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:48.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:48.097905+0000) 2024-03-06T12:59:49.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:49.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:49.098119+0000) 2024-03-06T12:59:50.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:50.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:50.098397+0000) 2024-03-06T12:59:51.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:51.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:51.098676+0000) 2024-03-06T12:59:52.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:52.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:52.098951+0000) 2024-03-06T12:59:53.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:53.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:53.099217+0000) 2024-03-06T12:59:54.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:54.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:54.099459+0000) 2024-03-06T12:59:55.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:55.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:55.099700+0000) 2024-03-06T12:59:56.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:56.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:56.099922+0000) 2024-03-06T12:59:57.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:57.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:57.100193+0000) 2024-03-06T12:59:58.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:58.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:58.100457+0000) 2024-03-06T12:59:59.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T12:59:59.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T11:59:59.100694+0000) 2024-03-06T13:00:00.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:00.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:00.100907+0000) 2024-03-06T13:00:01.089 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:00:01.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:01.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:01.101104+0000) 2024-03-06T13:00:01.097 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:00:02.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:02.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:02.101326+0000) 2024-03-06T13:00:03.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:03.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:03.101578+0000) 2024-03-06T13:00:04.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:04.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:04.101805+0000) 2024-03-06T13:00:05.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:05.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:05.102046+0000) 2024-03-06T13:00:06.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:06.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:06.102266+0000) 2024-03-06T13:00:07.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:07.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:07.102499+0000) 2024-03-06T13:00:08.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:08.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:08.102724+0000) 2024-03-06T13:00:09.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:09.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:09.102994+0000) 2024-03-06T13:00:10.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:10.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:10.103322+0000) 2024-03-06T13:00:11.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:11.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:11.103639+0000) 2024-03-06T13:00:12.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:12.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:12.103917+0000) 2024-03-06T13:00:13.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:13.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:13.104182+0000) 2024-03-06T13:00:14.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:14.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:14.104460+0000) 2024-03-06T13:00:15.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:15.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:15.104737+0000) 2024-03-06T13:00:16.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:16.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:16.105010+0000) 2024-03-06T13:00:17.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:17.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:17.105304+0000) 2024-03-06T13:00:18.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:18.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:18.105616+0000) 2024-03-06T13:00:19.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:19.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:19.105829+0000) 2024-03-06T13:00:20.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:20.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:20.106054+0000) 2024-03-06T13:00:21.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:21.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:21.106284+0000) 2024-03-06T13:00:22.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:22.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:22.106518+0000) 2024-03-06T13:00:23.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:23.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:23.106781+0000) 2024-03-06T13:00:24.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:24.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:24.107012+0000) 2024-03-06T13:00:25.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:25.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:25.107254+0000) 2024-03-06T13:00:26.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:26.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:26.107493+0000) 2024-03-06T13:00:27.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:27.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:27.107786+0000) 2024-03-06T13:00:28.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:28.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:28.107977+0000) 2024-03-06T13:00:29.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:29.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:29.108167+0000) 2024-03-06T13:00:30.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:30.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:30.108388+0000) 2024-03-06T13:00:31.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:31.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:31.108652+0000) 2024-03-06T13:00:31.228 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:00:31.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:00:32.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:32.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:32.108866+0000) 2024-03-06T13:00:33.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:33.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:33.109052+0000) 2024-03-06T13:00:34.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:34.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:34.109275+0000) 2024-03-06T13:00:35.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:35.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:35.109501+0000) 2024-03-06T13:00:36.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:36.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:36.109735+0000) 2024-03-06T13:00:37.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:37.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:37.109977+0000) 2024-03-06T13:00:38.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:38.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:38.110198+0000) 2024-03-06T13:00:39.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:39.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:39.110407+0000) 2024-03-06T13:00:40.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:40.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:40.110795+0000) 2024-03-06T13:00:41.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:41.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:41.111022+0000) 2024-03-06T13:00:42.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:42.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:42.111292+0000) 2024-03-06T13:00:43.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:43.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:43.111509+0000) 2024-03-06T13:00:44.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:44.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:44.111770+0000) 2024-03-06T13:00:45.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:45.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:45.111974+0000) 2024-03-06T13:00:46.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:46.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:46.112195+0000) 2024-03-06T13:00:47.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:47.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:47.112390+0000) 2024-03-06T13:00:48.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:48.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:48.112641+0000) 2024-03-06T13:00:49.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:49.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:49.112869+0000) 2024-03-06T13:00:50.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:50.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:50.113078+0000) 2024-03-06T13:00:51.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:51.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:51.113308+0000) 2024-03-06T13:00:52.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:52.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:52.113527+0000) 2024-03-06T13:00:53.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:53.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:53.113835+0000) 2024-03-06T13:00:54.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:54.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:54.113988+0000) 2024-03-06T13:00:55.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:55.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:55.114225+0000) 2024-03-06T13:00:56.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:56.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:56.114432+0000) 2024-03-06T13:00:57.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:57.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:57.114700+0000) 2024-03-06T13:00:58.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:58.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:58.114946+0000) 2024-03-06T13:00:59.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:00:59.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:00:59.115136+0000) 2024-03-06T13:01:00.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:00.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:00.115374+0000) 2024-03-06T13:01:01.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:01.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:01.115636+0000) 2024-03-06T13:01:01.426 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:01:01.434 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:01:02.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:02.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:02.115823+0000) 2024-03-06T13:01:03.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:03.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:03.116030+0000) 2024-03-06T13:01:04.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:04.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:04.116288+0000) 2024-03-06T13:01:05.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:05.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:05.116458+0000) 2024-03-06T13:01:06.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:06.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:06.116687+0000) 2024-03-06T13:01:07.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:07.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:07.116862+0000) 2024-03-06T13:01:08.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:08.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:08.117067+0000) 2024-03-06T13:01:09.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:09.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:09.117265+0000) 2024-03-06T13:01:10.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:10.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:10.117487+0000) 2024-03-06T13:01:11.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:11.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:11.117713+0000) 2024-03-06T13:01:12.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:12.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:12.117898+0000) 2024-03-06T13:01:13.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:13.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:13.118086+0000) 2024-03-06T13:01:14.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:14.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:14.118321+0000) 2024-03-06T13:01:15.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:15.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:15.118581+0000) 2024-03-06T13:01:16.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:16.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:16.118818+0000) 2024-03-06T13:01:17.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:17.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:17.119080+0000) 2024-03-06T13:01:18.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:18.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:18.119246+0000) 2024-03-06T13:01:19.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:19.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:19.119454+0000) 2024-03-06T13:01:20.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:20.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:20.119681+0000) 2024-03-06T13:01:21.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:21.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:21.119903+0000) 2024-03-06T13:01:22.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:22.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:22.120123+0000) 2024-03-06T13:01:23.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:23.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:23.120346+0000) 2024-03-06T13:01:24.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:24.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:24.120612+0000) 2024-03-06T13:01:25.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:25.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:25.120797+0000) 2024-03-06T13:01:26.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:26.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:26.121033+0000) 2024-03-06T13:01:27.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:27.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:27.121271+0000) 2024-03-06T13:01:28.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:28.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:28.121493+0000) 2024-03-06T13:01:29.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:29.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:29.121703+0000) 2024-03-06T13:01:30.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:30.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:30.121925+0000) 2024-03-06T13:01:31.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:31.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:31.122145+0000) 2024-03-06T13:01:31.616 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:01:31.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:01:32.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:32.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:32.122318+0000) 2024-03-06T13:01:33.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:33.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:33.122466+0000) 2024-03-06T13:01:34.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:34.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:34.122699+0000) 2024-03-06T13:01:35.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:35.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:35.122930+0000) 2024-03-06T13:01:36.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:36.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:36.123106+0000) 2024-03-06T13:01:37.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:37.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:37.123334+0000) 2024-03-06T13:01:38.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:38.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:38.123584+0000) 2024-03-06T13:01:39.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:39.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:39.123792+0000) 2024-03-06T13:01:40.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:40.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:40.123982+0000) 2024-03-06T13:01:41.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:41.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:41.124224+0000) 2024-03-06T13:01:42.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:42.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:42.124447+0000) 2024-03-06T13:01:43.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:43.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:43.124668+0000) 2024-03-06T13:01:44.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:44.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:44.124892+0000) 2024-03-06T13:01:45.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:45.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:45.125124+0000) 2024-03-06T13:01:46.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:46.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:46.125393+0000) 2024-03-06T13:01:47.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:47.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:47.125664+0000) 2024-03-06T13:01:48.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:48.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:48.125890+0000) 2024-03-06T13:01:49.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:49.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:49.126082+0000) 2024-03-06T13:01:50.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:50.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:50.126313+0000) 2024-03-06T13:01:51.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:51.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:51.126576+0000) 2024-03-06T13:01:52.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:52.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:52.126799+0000) 2024-03-06T13:01:53.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:53.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:53.127023+0000) 2024-03-06T13:01:54.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:54.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:54.127242+0000) 2024-03-06T13:01:55.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:55.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:55.127468+0000) 2024-03-06T13:01:56.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:56.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:56.127651+0000) 2024-03-06T13:01:57.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:57.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:57.127912+0000) 2024-03-06T13:01:58.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:58.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:58.128108+0000) 2024-03-06T13:01:59.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:01:59.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:01:59.128287+0000) 2024-03-06T13:02:00.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:00.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:00.128526+0000) 2024-03-06T13:02:01.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:01.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:01.128790+0000) 2024-03-06T13:02:01.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:02:01.839 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:02:02.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:02.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:02.129027+0000) 2024-03-06T13:02:03.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:03.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:03.129255+0000) 2024-03-06T13:02:04.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:04.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:04.129486+0000) 2024-03-06T13:02:05.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:05.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:05.129660+0000) 2024-03-06T13:02:06.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:06.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:06.129859+0000) 2024-03-06T13:02:07.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:07.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:07.130108+0000) 2024-03-06T13:02:08.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:08.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:08.130253+0000) 2024-03-06T13:02:09.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:09.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:09.130498+0000) 2024-03-06T13:02:10.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:10.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:10.130722+0000) 2024-03-06T13:02:11.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:11.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:11.130941+0000) 2024-03-06T13:02:12.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:12.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:12.131164+0000) 2024-03-06T13:02:13.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:13.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:13.131367+0000) 2024-03-06T13:02:14.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:14.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:14.131601+0000) 2024-03-06T13:02:15.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:15.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:15.131833+0000) 2024-03-06T13:02:16.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:16.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:16.132028+0000) 2024-03-06T13:02:17.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:17.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:17.132276+0000) 2024-03-06T13:02:18.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:18.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:18.132504+0000) 2024-03-06T13:02:19.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:19.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:19.132754+0000) 2024-03-06T13:02:20.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:20.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:20.132980+0000) 2024-03-06T13:02:21.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:21.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:21.133162+0000) 2024-03-06T13:02:22.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:22.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:22.133353+0000) 2024-03-06T13:02:23.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:23.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:23.133647+0000) 2024-03-06T13:02:24.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:24.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:24.133848+0000) 2024-03-06T13:02:25.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:25.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:25.134011+0000) 2024-03-06T13:02:26.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:26.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:26.134224+0000) 2024-03-06T13:02:27.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:27.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:27.134450+0000) 2024-03-06T13:02:28.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:28.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:28.134684+0000) 2024-03-06T13:02:29.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:29.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:29.134896+0000) 2024-03-06T13:02:30.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:30.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:30.135126+0000) 2024-03-06T13:02:31.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:31.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:31.135350+0000) 2024-03-06T13:02:32.065 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:02:32.073 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:02:32.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:32.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:32.135574+0000) 2024-03-06T13:02:33.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:33.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:33.135802+0000) 2024-03-06T13:02:34.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:34.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:34.136038+0000) 2024-03-06T13:02:35.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:35.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:35.136270+0000) 2024-03-06T13:02:36.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:36.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:36.136493+0000) 2024-03-06T13:02:37.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:37.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:37.136740+0000) 2024-03-06T13:02:38.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:38.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:38.136911+0000) 2024-03-06T13:02:39.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:39.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:39.137162+0000) 2024-03-06T13:02:40.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:40.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:40.137392+0000) 2024-03-06T13:02:41.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:41.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:41.137579+0000) 2024-03-06T13:02:42.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:42.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:42.137818+0000) 2024-03-06T13:02:43.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:43.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:43.138011+0000) 2024-03-06T13:02:44.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:44.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:44.138269+0000) 2024-03-06T13:02:45.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:45.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:45.138527+0000) 2024-03-06T13:02:46.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:46.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:46.138785+0000) 2024-03-06T13:02:47.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:47.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:47.139012+0000) 2024-03-06T13:02:48.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:48.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:48.139236+0000) 2024-03-06T13:02:49.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:49.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:49.139472+0000) 2024-03-06T13:02:50.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:50.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:50.139709+0000) 2024-03-06T13:02:51.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:51.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:51.139931+0000) 2024-03-06T13:02:52.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:52.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:52.140155+0000) 2024-03-06T13:02:53.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:53.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:53.140324+0000) 2024-03-06T13:02:54.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:54.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:54.140606+0000) 2024-03-06T13:02:55.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:55.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:55.140811+0000) 2024-03-06T13:02:56.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:56.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:56.140963+0000) 2024-03-06T13:02:57.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:57.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:57.141215+0000) 2024-03-06T13:02:58.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:58.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:58.141440+0000) 2024-03-06T13:02:59.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:02:59.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:02:59.141681+0000) 2024-03-06T13:03:00.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:00.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:00.141870+0000) 2024-03-06T13:03:01.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:01.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:01.142129+0000) 2024-03-06T13:03:02.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:02.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:02.142407+0000) 2024-03-06T13:03:02.269 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:03:02.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:03:03.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:03.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:03.142681+0000) 2024-03-06T13:03:04.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:04.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:04.142923+0000) 2024-03-06T13:03:05.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:05.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:05.143161+0000) 2024-03-06T13:03:06.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:06.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:06.143425+0000) 2024-03-06T13:03:07.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:07.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:07.143726+0000) 2024-03-06T13:03:08.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:08.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:08.143998+0000) 2024-03-06T13:03:09.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:09.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:09.144242+0000) 2024-03-06T13:03:10.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:10.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:10.144464+0000) 2024-03-06T13:03:11.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:11.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:11.144742+0000) 2024-03-06T13:03:12.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:12.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:12.144947+0000) 2024-03-06T13:03:13.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:13.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:13.145211+0000) 2024-03-06T13:03:14.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:14.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:14.145386+0000) 2024-03-06T13:03:15.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:15.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:15.145662+0000) 2024-03-06T13:03:16.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:16.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:16.145841+0000) 2024-03-06T13:03:17.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:17.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:17.146121+0000) 2024-03-06T13:03:18.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:18.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:18.146395+0000) 2024-03-06T13:03:19.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:19.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:19.146703+0000) 2024-03-06T13:03:20.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:20.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:20.146857+0000) 2024-03-06T13:03:21.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:21.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:21.147052+0000) 2024-03-06T13:03:22.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:22.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:22.147275+0000) 2024-03-06T13:03:23.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:23.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:23.147496+0000) 2024-03-06T13:03:24.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:24.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:24.147723+0000) 2024-03-06T13:03:25.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:25.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:25.147916+0000) 2024-03-06T13:03:26.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:26.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:26.148145+0000) 2024-03-06T13:03:27.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:27.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:27.148432+0000) 2024-03-06T13:03:28.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:28.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:28.148666+0000) 2024-03-06T13:03:29.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:29.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:29.148899+0000) 2024-03-06T13:03:30.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:30.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:30.149057+0000) 2024-03-06T13:03:31.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:31.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:31.149264+0000) 2024-03-06T13:03:32.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:32.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:32.149497+0000) 2024-03-06T13:03:32.516 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:03:32.524 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:03:33.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:33.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:33.149737+0000) 2024-03-06T13:03:34.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:34.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:34.149887+0000) 2024-03-06T13:03:35.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:35.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:35.150136+0000) 2024-03-06T13:03:36.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:36.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:36.150359+0000) 2024-03-06T13:03:37.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:37.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:37.150637+0000) 2024-03-06T13:03:38.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:38.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:38.150842+0000) 2024-03-06T13:03:39.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:39.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:39.151017+0000) 2024-03-06T13:03:40.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:40.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:40.151255+0000) 2024-03-06T13:03:41.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:41.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:41.151409+0000) 2024-03-06T13:03:42.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:42.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:42.151599+0000) 2024-03-06T13:03:43.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:43.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:43.151761+0000) 2024-03-06T13:03:44.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:44.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:44.151960+0000) 2024-03-06T13:03:45.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:45.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:45.152198+0000) 2024-03-06T13:03:46.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:46.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:46.152350+0000) 2024-03-06T13:03:47.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:47.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:47.152630+0000) 2024-03-06T13:03:48.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:48.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:48.152816+0000) 2024-03-06T13:03:49.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:49.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:49.153060+0000) 2024-03-06T13:03:50.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:50.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:50.153283+0000) 2024-03-06T13:03:51.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:51.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:51.153500+0000) 2024-03-06T13:03:52.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:52.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:52.153704+0000) 2024-03-06T13:03:53.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:53.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:53.153937+0000) 2024-03-06T13:03:54.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:54.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:54.154129+0000) 2024-03-06T13:03:55.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:55.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:55.154370+0000) 2024-03-06T13:03:56.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:56.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:56.154625+0000) 2024-03-06T13:03:57.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:57.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:57.154862+0000) 2024-03-06T13:03:58.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:58.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:58.155085+0000) 2024-03-06T13:03:59.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:03:59.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:03:59.155312+0000) 2024-03-06T13:04:00.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:00.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:00.155572+0000) 2024-03-06T13:04:01.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:01.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:01.155802+0000) 2024-03-06T13:04:02.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:02.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:02.156022+0000) 2024-03-06T13:04:02.727 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:04:02.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:04:03.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:03.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:03.156253+0000) 2024-03-06T13:04:04.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:04.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:04.156476+0000) 2024-03-06T13:04:05.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:05.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:05.156733+0000) 2024-03-06T13:04:06.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:06.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:06.156951+0000) 2024-03-06T13:04:07.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:07.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:07.157191+0000) 2024-03-06T13:04:08.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:08.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:08.157414+0000) 2024-03-06T13:04:09.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:09.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:09.157644+0000) 2024-03-06T13:04:10.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:10.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:10.157869+0000) 2024-03-06T13:04:11.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:11.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:11.158051+0000) 2024-03-06T13:04:12.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:12.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:12.158265+0000) 2024-03-06T13:04:13.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:13.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:13.158491+0000) 2024-03-06T13:04:14.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:14.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:14.158722+0000) 2024-03-06T13:04:15.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:15.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:15.158916+0000) 2024-03-06T13:04:16.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:16.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:16.159137+0000) 2024-03-06T13:04:17.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:17.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:17.159371+0000) 2024-03-06T13:04:18.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:18.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:18.159612+0000) 2024-03-06T13:04:19.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:19.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:19.159812+0000) 2024-03-06T13:04:20.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:20.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:20.160049+0000) 2024-03-06T13:04:21.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:21.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:21.160336+0000) 2024-03-06T13:04:22.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:22.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:22.160578+0000) 2024-03-06T13:04:23.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:23.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:23.160803+0000) 2024-03-06T13:04:24.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:24.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:24.161027+0000) 2024-03-06T13:04:25.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:25.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:25.161256+0000) 2024-03-06T13:04:26.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:26.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:26.161479+0000) 2024-03-06T13:04:27.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:27.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:27.161707+0000) 2024-03-06T13:04:28.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:28.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:28.161882+0000) 2024-03-06T13:04:29.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:29.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:29.162114+0000) 2024-03-06T13:04:30.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:30.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:30.162337+0000) 2024-03-06T13:04:31.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:31.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:31.162599+0000) 2024-03-06T13:04:32.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:32.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:32.162823+0000) 2024-03-06T13:04:32.934 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:04:32.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:04:33.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:33.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:33.163000+0000) 2024-03-06T13:04:34.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:34.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:34.163190+0000) 2024-03-06T13:04:35.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:35.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:35.163356+0000) 2024-03-06T13:04:36.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:36.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:36.163517+0000) 2024-03-06T13:04:37.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:37.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:37.163794+0000) 2024-03-06T13:04:38.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:38.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:38.164018+0000) 2024-03-06T13:04:39.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:39.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:39.164238+0000) 2024-03-06T13:04:40.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:40.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:40.164463+0000) 2024-03-06T13:04:41.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:41.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:41.164680+0000) 2024-03-06T13:04:42.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:42.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:42.164870+0000) 2024-03-06T13:04:43.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:43.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:43.165096+0000) 2024-03-06T13:04:44.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:44.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:44.165320+0000) 2024-03-06T13:04:45.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:45.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:45.165524+0000) 2024-03-06T13:04:46.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:46.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:46.165790+0000) 2024-03-06T13:04:47.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:47.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:47.166022+0000) 2024-03-06T13:04:48.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:48.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:48.166242+0000) 2024-03-06T13:04:49.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:49.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:49.166461+0000) 2024-03-06T13:04:50.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:50.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:50.166695+0000) 2024-03-06T13:04:51.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:51.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:51.166913+0000) 2024-03-06T13:04:52.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:52.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:52.167071+0000) 2024-03-06T13:04:53.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:53.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:53.167273+0000) 2024-03-06T13:04:54.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:54.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:54.167494+0000) 2024-03-06T13:04:55.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:55.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:55.167699+0000) 2024-03-06T13:04:56.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:56.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:56.167898+0000) 2024-03-06T13:04:57.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:57.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:57.168145+0000) 2024-03-06T13:04:58.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:58.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:58.168369+0000) 2024-03-06T13:04:59.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:04:59.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:04:59.168636+0000) 2024-03-06T13:05:00.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:00.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:00.168861+0000) 2024-03-06T13:05:01.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:01.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:01.169022+0000) 2024-03-06T13:05:02.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:02.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:02.169252+0000) 2024-03-06T13:05:03.130 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:05:03.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:05:03.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:03.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:03.169437+0000) 2024-03-06T13:05:04.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:04.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:04.169627+0000) 2024-03-06T13:05:05.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:05.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:05.169807+0000) 2024-03-06T13:05:06.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:06.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:06.170042+0000) 2024-03-06T13:05:07.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:07.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:07.170274+0000) 2024-03-06T13:05:08.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:08.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:08.170492+0000) 2024-03-06T13:05:09.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:09.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:09.170773+0000) 2024-03-06T13:05:10.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:10.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:10.170997+0000) 2024-03-06T13:05:11.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:11.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:11.171225+0000) 2024-03-06T13:05:12.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:12.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:12.171421+0000) 2024-03-06T13:05:13.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:13.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:13.171656+0000) 2024-03-06T13:05:14.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:14.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:14.171879+0000) 2024-03-06T13:05:15.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:15.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:15.172104+0000) 2024-03-06T13:05:16.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:16.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:16.172331+0000) 2024-03-06T13:05:17.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:17.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:17.172621+0000) 2024-03-06T13:05:18.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:18.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:18.172820+0000) 2024-03-06T13:05:19.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:19.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:19.173044+0000) 2024-03-06T13:05:20.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:20.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:20.173237+0000) 2024-03-06T13:05:21.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:21.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:21.173413+0000) 2024-03-06T13:05:22.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:22.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:22.173623+0000) 2024-03-06T13:05:23.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:23.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:23.173837+0000) 2024-03-06T13:05:24.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:24.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:24.174060+0000) 2024-03-06T13:05:25.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:25.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:25.174297+0000) 2024-03-06T13:05:26.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:26.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:26.174518+0000) 2024-03-06T13:05:27.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:27.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:27.174779+0000) 2024-03-06T13:05:28.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:28.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:28.174998+0000) 2024-03-06T13:05:29.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:29.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:29.175227+0000) 2024-03-06T13:05:30.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:30.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:30.175388+0000) 2024-03-06T13:05:31.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:31.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:31.175651+0000) 2024-03-06T13:05:32.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:32.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:32.175809+0000) 2024-03-06T13:05:33.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:33.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:33.176040+0000) 2024-03-06T13:05:33.270 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:05:33.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:05:34.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:34.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:34.176184+0000) 2024-03-06T13:05:35.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:35.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:35.176430+0000) 2024-03-06T13:05:36.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:36.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:36.176653+0000) 2024-03-06T13:05:37.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:37.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:37.176883+0000) 2024-03-06T13:05:38.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:38.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:38.177105+0000) 2024-03-06T13:05:39.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:39.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:39.177311+0000) 2024-03-06T13:05:40.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:40.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:40.177575+0000) 2024-03-06T13:05:41.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:41.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:41.177845+0000) 2024-03-06T13:05:42.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:42.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:42.178066+0000) 2024-03-06T13:05:43.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:43.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:43.178283+0000) 2024-03-06T13:05:44.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:44.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:44.178436+0000) 2024-03-06T13:05:45.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:45.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:45.178702+0000) 2024-03-06T13:05:46.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:46.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:46.178858+0000) 2024-03-06T13:05:47.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:47.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:47.179107+0000) 2024-03-06T13:05:48.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:48.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:48.179335+0000) 2024-03-06T13:05:49.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:49.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:49.179613+0000) 2024-03-06T13:05:50.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:50.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:50.179784+0000) 2024-03-06T13:05:51.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:51.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:51.179996+0000) 2024-03-06T13:05:52.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:52.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:52.180193+0000) 2024-03-06T13:05:53.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:53.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:53.180441+0000) 2024-03-06T13:05:54.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:54.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:54.180641+0000) 2024-03-06T13:05:55.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:55.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:55.180883+0000) 2024-03-06T13:05:56.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:56.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:56.181072+0000) 2024-03-06T13:05:57.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:57.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:57.181365+0000) 2024-03-06T13:05:58.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:58.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:58.181635+0000) 2024-03-06T13:05:59.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:05:59.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:05:59.181856+0000) 2024-03-06T13:06:00.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:00.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:00.182083+0000) 2024-03-06T13:06:01.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:01.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:01.182263+0000) 2024-03-06T13:06:02.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:02.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:02.182487+0000) 2024-03-06T13:06:03.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:03.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:03.182713+0000) 2024-03-06T13:06:03.480 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:06:03.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:06:04.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:04.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:04.182934+0000) 2024-03-06T13:06:05.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:05.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:05.183136+0000) 2024-03-06T13:06:06.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:06.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:06.183367+0000) 2024-03-06T13:06:07.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:07.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:07.183647+0000) 2024-03-06T13:06:08.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:08.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:08.183866+0000) 2024-03-06T13:06:09.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:09.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:09.184102+0000) 2024-03-06T13:06:10.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:10.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:10.184329+0000) 2024-03-06T13:06:11.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:11.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:11.184590+0000) 2024-03-06T13:06:12.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:12.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:12.184815+0000) 2024-03-06T13:06:13.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:13.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:13.185044+0000) 2024-03-06T13:06:14.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:14.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:14.185269+0000) 2024-03-06T13:06:15.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:15.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:15.185504+0000) 2024-03-06T13:06:16.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:16.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:16.185739+0000) 2024-03-06T13:06:17.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:17.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:17.185961+0000) 2024-03-06T13:06:18.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:18.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:18.186189+0000) 2024-03-06T13:06:19.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:19.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:19.186416+0000) 2024-03-06T13:06:20.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:20.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:20.186687+0000) 2024-03-06T13:06:21.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:21.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:21.186933+0000) 2024-03-06T13:06:22.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:22.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:22.187150+0000) 2024-03-06T13:06:23.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:23.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:23.187375+0000) 2024-03-06T13:06:24.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:24.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:24.187648+0000) 2024-03-06T13:06:25.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:25.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:25.187891+0000) 2024-03-06T13:06:26.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:26.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:26.188116+0000) 2024-03-06T13:06:27.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:27.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:27.188356+0000) 2024-03-06T13:06:28.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:28.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:28.188628+0000) 2024-03-06T13:06:29.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:29.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:29.188827+0000) 2024-03-06T13:06:30.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:30.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:30.189047+0000) 2024-03-06T13:06:31.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:31.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:31.189273+0000) 2024-03-06T13:06:32.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:32.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:32.189491+0000) 2024-03-06T13:06:33.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:33.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:33.189718+0000) 2024-03-06T13:06:33.694 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:06:33.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:06:34.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:34.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:34.189944+0000) 2024-03-06T13:06:35.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:35.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:35.190176+0000) 2024-03-06T13:06:36.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:36.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:36.190402+0000) 2024-03-06T13:06:37.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:37.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:37.190599+0000) 2024-03-06T13:06:38.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:38.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:38.190827+0000) 2024-03-06T13:06:39.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:39.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:39.191025+0000) 2024-03-06T13:06:40.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:40.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:40.191262+0000) 2024-03-06T13:06:41.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:41.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:41.191497+0000) 2024-03-06T13:06:42.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:42.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:42.191723+0000) 2024-03-06T13:06:43.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:43.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:43.191962+0000) 2024-03-06T13:06:44.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:44.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:44.192201+0000) 2024-03-06T13:06:45.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:45.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:45.192507+0000) 2024-03-06T13:06:46.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:46.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:46.192785+0000) 2024-03-06T13:06:47.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:47.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:47.193034+0000) 2024-03-06T13:06:48.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:48.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:48.193262+0000) 2024-03-06T13:06:49.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:49.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:49.193486+0000) 2024-03-06T13:06:50.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:50.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:50.193695+0000) 2024-03-06T13:06:51.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:51.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:51.193935+0000) 2024-03-06T13:06:52.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:52.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:52.194159+0000) 2024-03-06T13:06:53.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:53.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:53.194385+0000) 2024-03-06T13:06:54.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:54.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:54.194562+0000) 2024-03-06T13:06:55.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:55.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:55.194751+0000) 2024-03-06T13:06:56.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:56.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:56.194990+0000) 2024-03-06T13:06:57.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:57.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:57.195189+0000) 2024-03-06T13:06:58.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:58.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:58.195410+0000) 2024-03-06T13:06:59.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:06:59.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:06:59.195678+0000) 2024-03-06T13:07:00.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:00.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:00.195900+0000) 2024-03-06T13:07:01.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:01.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:01.196092+0000) 2024-03-06T13:07:02.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:02.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:02.196320+0000) 2024-03-06T13:07:03.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:03.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:03.196482+0000) 2024-03-06T13:07:03.904 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:07:03.912 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:07:04.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:04.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:04.196712+0000) 2024-03-06T13:07:05.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:05.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:05.196951+0000) 2024-03-06T13:07:06.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:06.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:06.197177+0000) 2024-03-06T13:07:07.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:07.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:07.197418+0000) 2024-03-06T13:07:08.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:08.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:08.197639+0000) 2024-03-06T13:07:09.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:09.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:09.197851+0000) 2024-03-06T13:07:10.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:10.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:10.198085+0000) 2024-03-06T13:07:11.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:11.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:11.198307+0000) 2024-03-06T13:07:12.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:12.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:12.198522+0000) 2024-03-06T13:07:13.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:13.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:13.198800+0000) 2024-03-06T13:07:14.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:14.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:14.198950+0000) 2024-03-06T13:07:15.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:15.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:15.199133+0000) 2024-03-06T13:07:16.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:16.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:16.199365+0000) 2024-03-06T13:07:17.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:17.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:17.199663+0000) 2024-03-06T13:07:18.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:18.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:18.199884+0000) 2024-03-06T13:07:19.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:19.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:19.200120+0000) 2024-03-06T13:07:20.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:20.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:20.200295+0000) 2024-03-06T13:07:21.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:21.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:21.200461+0000) 2024-03-06T13:07:22.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:22.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:22.200678+0000) 2024-03-06T13:07:23.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:23.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:23.200914+0000) 2024-03-06T13:07:24.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:24.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:24.201102+0000) 2024-03-06T13:07:25.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:25.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:25.201333+0000) 2024-03-06T13:07:26.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:26.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:26.201577+0000) 2024-03-06T13:07:27.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:27.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:27.201806+0000) 2024-03-06T13:07:28.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:28.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:28.202028+0000) 2024-03-06T13:07:29.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:29.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:29.202253+0000) 2024-03-06T13:07:30.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:30.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:30.202479+0000) 2024-03-06T13:07:31.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:31.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:31.202702+0000) 2024-03-06T13:07:32.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:32.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:32.202926+0000) 2024-03-06T13:07:33.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:33.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:33.203157+0000) 2024-03-06T13:07:34.120 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:07:34.128 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:07:34.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:34.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:34.203365+0000) 2024-03-06T13:07:35.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:35.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:35.203651+0000) 2024-03-06T13:07:36.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:36.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:36.203845+0000) 2024-03-06T13:07:37.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:37.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:37.204055+0000) 2024-03-06T13:07:38.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:38.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:38.204323+0000) 2024-03-06T13:07:39.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:39.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:39.204581+0000) 2024-03-06T13:07:40.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:40.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:40.204810+0000) 2024-03-06T13:07:41.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:41.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:41.205039+0000) 2024-03-06T13:07:42.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:42.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:42.205259+0000) 2024-03-06T13:07:43.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:43.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:43.205481+0000) 2024-03-06T13:07:44.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:44.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:44.205704+0000) 2024-03-06T13:07:45.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:45.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:45.205916+0000) 2024-03-06T13:07:46.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:46.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:46.206115+0000) 2024-03-06T13:07:47.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:47.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:47.206312+0000) 2024-03-06T13:07:48.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:48.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:48.206467+0000) 2024-03-06T13:07:49.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:49.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:49.206724+0000) 2024-03-06T13:07:50.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:50.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:50.206949+0000) 2024-03-06T13:07:51.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:51.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:51.207179+0000) 2024-03-06T13:07:52.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:52.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:52.207399+0000) 2024-03-06T13:07:53.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:53.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:53.207641+0000) 2024-03-06T13:07:54.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:54.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:54.207861+0000) 2024-03-06T13:07:55.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:55.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:55.208101+0000) 2024-03-06T13:07:56.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:56.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:56.208324+0000) 2024-03-06T13:07:57.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:57.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:57.208591+0000) 2024-03-06T13:07:58.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:58.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:58.208890+0000) 2024-03-06T13:07:59.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:07:59.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:07:59.209177+0000) 2024-03-06T13:08:00.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:00.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:00.209454+0000) 2024-03-06T13:08:01.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:01.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:01.209682+0000) 2024-03-06T13:08:02.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:02.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:02.209915+0000) 2024-03-06T13:08:03.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:03.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:03.210105+0000) 2024-03-06T13:08:04.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:04.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:04.210335+0000) 2024-03-06T13:08:04.326 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:08:04.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:08:05.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:05.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:05.210611+0000) 2024-03-06T13:08:06.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:06.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:06.210875+0000) 2024-03-06T13:08:07.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:07.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:07.211129+0000) 2024-03-06T13:08:08.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:08.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:08.211349+0000) 2024-03-06T13:08:09.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:09.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:09.211595+0000) 2024-03-06T13:08:10.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:10.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:10.211850+0000) 2024-03-06T13:08:11.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:11.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:11.212075+0000) 2024-03-06T13:08:12.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:12.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:12.212341+0000) 2024-03-06T13:08:13.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:13.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:13.212659+0000) 2024-03-06T13:08:14.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:14.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:14.212940+0000) 2024-03-06T13:08:15.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:15.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:15.213132+0000) 2024-03-06T13:08:16.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:16.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:16.213375+0000) 2024-03-06T13:08:17.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:17.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:17.213612+0000) 2024-03-06T13:08:18.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:18.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:18.213797+0000) 2024-03-06T13:08:19.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:19.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:19.214023+0000) 2024-03-06T13:08:20.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:20.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:20.214280+0000) 2024-03-06T13:08:21.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:21.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:21.214519+0000) 2024-03-06T13:08:22.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:22.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:22.214745+0000) 2024-03-06T13:08:23.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:23.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:23.214958+0000) 2024-03-06T13:08:24.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:24.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:24.215198+0000) 2024-03-06T13:08:25.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:25.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:25.215454+0000) 2024-03-06T13:08:26.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:26.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:26.215757+0000) 2024-03-06T13:08:27.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:27.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:27.216005+0000) 2024-03-06T13:08:28.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:28.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:28.216266+0000) 2024-03-06T13:08:29.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:29.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:29.216571+0000) 2024-03-06T13:08:30.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:30.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:30.216861+0000) 2024-03-06T13:08:31.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:31.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:31.217147+0000) 2024-03-06T13:08:32.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:32.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:32.217421+0000) 2024-03-06T13:08:33.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:33.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:33.217700+0000) 2024-03-06T13:08:34.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:34.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:34.217927+0000) 2024-03-06T13:08:34.536 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:08:34.545 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:08:35.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:35.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:35.218115+0000) 2024-03-06T13:08:36.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:36.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:36.218377+0000) 2024-03-06T13:08:37.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:37.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:37.218707+0000) 2024-03-06T13:08:38.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:38.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:38.218986+0000) 2024-03-06T13:08:39.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:39.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:39.219265+0000) 2024-03-06T13:08:40.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:40.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:40.219572+0000) 2024-03-06T13:08:41.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:41.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:41.219747+0000) 2024-03-06T13:08:42.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:42.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:42.219959+0000) 2024-03-06T13:08:43.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:43.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:43.220207+0000) 2024-03-06T13:08:44.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:44.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:44.220420+0000) 2024-03-06T13:08:45.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:45.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:45.220694+0000) 2024-03-06T13:08:46.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:46.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:46.220960+0000) 2024-03-06T13:08:47.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:47.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:47.221247+0000) 2024-03-06T13:08:48.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:48.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:48.221522+0000) 2024-03-06T13:08:49.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:49.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:49.221786+0000) 2024-03-06T13:08:50.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:50.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:50.222011+0000) 2024-03-06T13:08:51.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:51.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:51.222304+0000) 2024-03-06T13:08:52.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:52.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:52.222638+0000) 2024-03-06T13:08:53.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:53.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:53.222824+0000) 2024-03-06T13:08:54.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:54.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:54.223029+0000) 2024-03-06T13:08:55.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:55.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:55.223283+0000) 2024-03-06T13:08:56.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:56.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:56.223521+0000) 2024-03-06T13:08:57.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:57.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:57.223760+0000) 2024-03-06T13:08:58.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:58.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:58.223971+0000) 2024-03-06T13:08:59.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:08:59.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:08:59.224315+0000) 2024-03-06T13:09:00.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:00.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:00.224604+0000) 2024-03-06T13:09:01.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:01.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:01.224853+0000) 2024-03-06T13:09:02.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:02.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:02.225037+0000) 2024-03-06T13:09:03.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:03.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:03.225251+0000) 2024-03-06T13:09:04.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:04.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:04.225450+0000) 2024-03-06T13:09:04.751 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:09:04.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:09:05.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:05.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:05.225711+0000) 2024-03-06T13:09:06.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:06.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:06.225934+0000) 2024-03-06T13:09:07.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:07.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:07.226163+0000) 2024-03-06T13:09:08.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:08.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:08.226385+0000) 2024-03-06T13:09:09.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:09.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:09.226664+0000) 2024-03-06T13:09:10.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:10.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:10.226887+0000) 2024-03-06T13:09:11.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:11.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:11.227121+0000) 2024-03-06T13:09:12.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:12.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:12.227340+0000) 2024-03-06T13:09:13.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:13.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:13.227593+0000) 2024-03-06T13:09:14.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:14.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:14.227802+0000) 2024-03-06T13:09:15.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:15.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:15.228058+0000) 2024-03-06T13:09:16.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:16.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:16.228256+0000) 2024-03-06T13:09:17.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:17.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:17.228507+0000) 2024-03-06T13:09:18.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:18.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:18.228762+0000) 2024-03-06T13:09:19.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:19.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:19.229031+0000) 2024-03-06T13:09:20.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:20.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:20.229261+0000) 2024-03-06T13:09:21.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:21.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:21.229490+0000) 2024-03-06T13:09:22.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:22.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:22.229712+0000) 2024-03-06T13:09:23.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:23.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:23.229937+0000) 2024-03-06T13:09:24.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:24.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:24.230157+0000) 2024-03-06T13:09:25.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:25.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:25.230396+0000) 2024-03-06T13:09:26.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:26.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:26.230657+0000) 2024-03-06T13:09:27.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:27.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:27.230814+0000) 2024-03-06T13:09:28.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:28.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:28.231003+0000) 2024-03-06T13:09:29.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:29.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:29.231234+0000) 2024-03-06T13:09:30.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:30.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:30.231441+0000) 2024-03-06T13:09:31.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:31.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:31.231681+0000) 2024-03-06T13:09:32.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:32.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:32.231901+0000) 2024-03-06T13:09:33.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:33.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:33.232138+0000) 2024-03-06T13:09:34.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:34.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:34.232359+0000) 2024-03-06T13:09:34.958 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:09:34.966 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:09:35.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:35.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:35.232633+0000) 2024-03-06T13:09:36.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:36.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:36.232835+0000) 2024-03-06T13:09:37.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:37.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:37.233064+0000) 2024-03-06T13:09:38.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:38.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:38.233285+0000) 2024-03-06T13:09:39.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:39.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:39.233524+0000) 2024-03-06T13:09:40.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:40.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:40.233776+0000) 2024-03-06T13:09:41.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:41.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:41.234015+0000) 2024-03-06T13:09:42.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:42.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:42.234242+0000) 2024-03-06T13:09:43.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:43.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:43.234468+0000) 2024-03-06T13:09:44.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:44.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:44.234681+0000) 2024-03-06T13:09:45.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:45.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:45.234887+0000) 2024-03-06T13:09:46.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:46.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:46.235091+0000) 2024-03-06T13:09:47.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:47.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:47.235344+0000) 2024-03-06T13:09:48.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:48.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:48.235576+0000) 2024-03-06T13:09:49.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:49.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:49.235776+0000) 2024-03-06T13:09:50.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:50.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:50.235936+0000) 2024-03-06T13:09:51.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:51.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:51.236138+0000) 2024-03-06T13:09:52.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:52.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:52.236364+0000) 2024-03-06T13:09:53.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:53.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:53.236635+0000) 2024-03-06T13:09:54.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:54.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:54.236843+0000) 2024-03-06T13:09:55.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:55.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:55.237050+0000) 2024-03-06T13:09:56.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:56.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:56.237261+0000) 2024-03-06T13:09:57.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:57.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:57.237461+0000) 2024-03-06T13:09:58.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:58.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:58.237644+0000) 2024-03-06T13:09:59.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:09:59.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:09:59.237855+0000) 2024-03-06T13:10:00.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:00.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:00.238050+0000) 2024-03-06T13:10:01.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:01.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:01.238280+0000) 2024-03-06T13:10:02.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:02.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:02.238514+0000) 2024-03-06T13:10:03.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:03.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:03.238807+0000) 2024-03-06T13:10:04.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:04.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:04.239011+0000) 2024-03-06T13:10:05.171 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:10:05.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:10:05.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:05.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:05.239203+0000) 2024-03-06T13:10:06.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:06.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:06.239428+0000) 2024-03-06T13:10:07.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:07.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:07.239690+0000) 2024-03-06T13:10:08.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:08.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:08.239910+0000) 2024-03-06T13:10:09.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:09.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:09.240132+0000) 2024-03-06T13:10:10.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:10.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:10.240353+0000) 2024-03-06T13:10:11.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:11.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:11.240620+0000) 2024-03-06T13:10:12.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:12.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:12.240856+0000) 2024-03-06T13:10:13.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:13.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:13.241072+0000) 2024-03-06T13:10:14.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:14.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:14.241319+0000) 2024-03-06T13:10:15.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:15.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:15.241517+0000) 2024-03-06T13:10:16.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:16.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:16.241755+0000) 2024-03-06T13:10:17.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:17.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:17.242000+0000) 2024-03-06T13:10:18.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:18.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:18.242237+0000) 2024-03-06T13:10:19.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:19.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:19.242455+0000) 2024-03-06T13:10:20.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:20.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:20.242669+0000) 2024-03-06T13:10:21.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:21.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:21.242912+0000) 2024-03-06T13:10:22.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:22.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:22.243096+0000) 2024-03-06T13:10:23.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:23.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:23.243320+0000) 2024-03-06T13:10:24.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:24.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:24.243578+0000) 2024-03-06T13:10:25.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:25.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:25.243771+0000) 2024-03-06T13:10:26.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:26.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:26.243996+0000) 2024-03-06T13:10:27.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:27.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:27.244182+0000) 2024-03-06T13:10:28.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:28.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:28.244402+0000) 2024-03-06T13:10:29.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:29.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:29.244653+0000) 2024-03-06T13:10:30.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:30.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:30.244863+0000) 2024-03-06T13:10:31.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:31.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:31.245083+0000) 2024-03-06T13:10:32.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:32.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:32.245278+0000) 2024-03-06T13:10:33.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:33.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:33.245515+0000) 2024-03-06T13:10:34.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:34.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:34.245780+0000) 2024-03-06T13:10:35.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:35.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:35.246010+0000) 2024-03-06T13:10:35.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:10:35.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:10:36.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:36.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:36.246193+0000) 2024-03-06T13:10:37.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:37.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:37.246404+0000) 2024-03-06T13:10:38.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:38.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:38.246634+0000) 2024-03-06T13:10:39.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:39.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:39.246847+0000) 2024-03-06T13:10:40.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:40.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:40.247054+0000) 2024-03-06T13:10:41.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:41.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:41.247296+0000) 2024-03-06T13:10:42.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:42.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:42.247507+0000) 2024-03-06T13:10:43.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:43.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:43.247743+0000) 2024-03-06T13:10:44.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:44.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:44.247888+0000) 2024-03-06T13:10:45.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:45.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:45.248125+0000) 2024-03-06T13:10:46.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:46.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:46.248348+0000) 2024-03-06T13:10:47.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:47.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:47.248610+0000) 2024-03-06T13:10:48.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:48.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:48.248829+0000) 2024-03-06T13:10:49.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:49.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:49.249051+0000) 2024-03-06T13:10:50.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:50.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:50.249274+0000) 2024-03-06T13:10:51.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:51.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:51.249503+0000) 2024-03-06T13:10:52.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:52.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:52.249730+0000) 2024-03-06T13:10:53.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:53.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:53.249923+0000) 2024-03-06T13:10:54.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:54.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:54.250126+0000) 2024-03-06T13:10:55.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:55.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:55.250382+0000) 2024-03-06T13:10:56.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:56.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:56.250649+0000) 2024-03-06T13:10:57.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:57.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:57.250873+0000) 2024-03-06T13:10:58.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:58.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:58.251081+0000) 2024-03-06T13:10:59.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:10:59.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:10:59.251317+0000) 2024-03-06T13:11:00.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:00.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:00.251581+0000) 2024-03-06T13:11:01.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:01.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:01.251808+0000) 2024-03-06T13:11:02.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:02.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:02.251983+0000) 2024-03-06T13:11:03.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:03.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:03.252168+0000) 2024-03-06T13:11:04.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:04.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:04.252390+0000) 2024-03-06T13:11:05.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:05.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:05.252596+0000) 2024-03-06T13:11:05.559 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:11:05.567 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:11:06.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:06.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:06.252793+0000) 2024-03-06T13:11:07.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:07.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:07.253006+0000) 2024-03-06T13:11:08.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:08.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:08.253210+0000) 2024-03-06T13:11:09.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:09.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:09.253453+0000) 2024-03-06T13:11:10.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:10.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:10.253680+0000) 2024-03-06T13:11:11.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:11.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:11.253888+0000) 2024-03-06T13:11:12.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:12.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:12.254122+0000) 2024-03-06T13:11:13.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:13.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:13.254341+0000) 2024-03-06T13:11:14.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:14.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:14.254625+0000) 2024-03-06T13:11:15.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:15.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:15.254835+0000) 2024-03-06T13:11:16.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:16.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:16.255042+0000) 2024-03-06T13:11:17.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:17.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:17.255299+0000) 2024-03-06T13:11:18.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:18.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:18.255458+0000) 2024-03-06T13:11:19.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:19.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:19.255689+0000) 2024-03-06T13:11:20.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:20.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:20.255912+0000) 2024-03-06T13:11:21.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:21.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:21.256145+0000) 2024-03-06T13:11:22.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:22.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:22.256345+0000) 2024-03-06T13:11:23.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:23.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:23.256574+0000) 2024-03-06T13:11:24.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:24.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:24.256778+0000) 2024-03-06T13:11:25.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:25.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:25.256974+0000) 2024-03-06T13:11:26.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:26.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:26.257166+0000) 2024-03-06T13:11:27.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:27.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:27.257372+0000) 2024-03-06T13:11:28.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:28.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:28.257594+0000) 2024-03-06T13:11:29.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:29.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:29.257841+0000) 2024-03-06T13:11:30.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:30.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:30.258062+0000) 2024-03-06T13:11:31.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:31.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:31.258250+0000) 2024-03-06T13:11:32.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:32.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:32.258457+0000) 2024-03-06T13:11:33.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:33.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:33.258692+0000) 2024-03-06T13:11:34.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:34.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:34.258911+0000) 2024-03-06T13:11:35.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:35.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:35.259171+0000) 2024-03-06T13:11:35.764 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:11:35.773 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:11:36.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:36.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:36.259398+0000) 2024-03-06T13:11:37.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:37.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:37.259666+0000) 2024-03-06T13:11:38.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:38.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:38.259853+0000) 2024-03-06T13:11:39.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:39.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:39.260044+0000) 2024-03-06T13:11:40.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:40.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:40.260239+0000) 2024-03-06T13:11:41.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:41.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:41.260444+0000) 2024-03-06T13:11:42.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:42.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:42.260680+0000) 2024-03-06T13:11:43.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:43.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:43.260872+0000) 2024-03-06T13:11:44.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:44.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:44.261093+0000) 2024-03-06T13:11:45.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:45.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:45.261322+0000) 2024-03-06T13:11:46.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:46.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:46.261578+0000) 2024-03-06T13:11:47.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:47.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:47.261827+0000) 2024-03-06T13:11:48.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:48.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:48.262061+0000) 2024-03-06T13:11:49.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:49.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:49.262358+0000) 2024-03-06T13:11:50.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:50.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:50.262645+0000) 2024-03-06T13:11:51.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:51.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:51.262811+0000) 2024-03-06T13:11:52.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:52.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:52.262995+0000) 2024-03-06T13:11:53.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:53.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:53.263223+0000) 2024-03-06T13:11:54.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:54.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:54.263414+0000) 2024-03-06T13:11:55.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:55.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:55.263668+0000) 2024-03-06T13:11:56.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:56.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:56.263866+0000) 2024-03-06T13:11:57.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:57.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:57.264081+0000) 2024-03-06T13:11:58.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:58.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:58.264346+0000) 2024-03-06T13:11:59.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:11:59.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:11:59.264653+0000) 2024-03-06T13:12:00.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:00.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:00.264924+0000) 2024-03-06T13:12:01.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:01.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:01.265204+0000) 2024-03-06T13:12:02.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:02.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:02.265486+0000) 2024-03-06T13:12:03.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:03.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:03.265727+0000) 2024-03-06T13:12:04.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:04.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:04.265918+0000) 2024-03-06T13:12:05.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:05.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:05.266157+0000) 2024-03-06T13:12:05.979 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:12:05.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:12:06.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:06.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:06.266381+0000) 2024-03-06T13:12:07.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:07.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:07.266641+0000) 2024-03-06T13:12:08.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:08.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:08.266874+0000) 2024-03-06T13:12:09.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:09.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:09.267096+0000) 2024-03-06T13:12:10.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:10.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:10.267320+0000) 2024-03-06T13:12:11.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:11.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:11.267568+0000) 2024-03-06T13:12:12.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:12.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:12.267766+0000) 2024-03-06T13:12:13.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:13.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:13.267993+0000) 2024-03-06T13:12:14.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:14.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:14.268172+0000) 2024-03-06T13:12:15.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:15.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:15.268374+0000) 2024-03-06T13:12:16.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:16.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:16.268677+0000) 2024-03-06T13:12:17.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:17.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:17.268920+0000) 2024-03-06T13:12:18.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:18.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:18.269112+0000) 2024-03-06T13:12:19.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:19.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:19.269413+0000) 2024-03-06T13:12:20.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:20.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:20.269697+0000) 2024-03-06T13:12:21.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:21.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:21.269954+0000) 2024-03-06T13:12:22.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:22.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:22.270185+0000) 2024-03-06T13:12:23.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:23.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:23.270445+0000) 2024-03-06T13:12:24.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:24.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:24.270710+0000) 2024-03-06T13:12:25.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:25.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:25.270958+0000) 2024-03-06T13:12:26.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:26.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:26.271240+0000) 2024-03-06T13:12:27.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:27.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:27.271587+0000) 2024-03-06T13:12:28.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:28.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:28.271818+0000) 2024-03-06T13:12:29.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:29.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:29.272029+0000) 2024-03-06T13:12:30.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:30.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:30.272235+0000) 2024-03-06T13:12:31.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:31.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:31.272440+0000) 2024-03-06T13:12:32.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:32.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:32.272677+0000) 2024-03-06T13:12:33.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:33.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:33.272865+0000) 2024-03-06T13:12:34.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:34.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:34.273054+0000) 2024-03-06T13:12:35.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:35.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:35.273277+0000) 2024-03-06T13:12:36.166 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:12:36.175 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:12:36.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:36.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:36.273440+0000) 2024-03-06T13:12:37.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:37.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:37.273732+0000) 2024-03-06T13:12:38.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:38.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:38.273921+0000) 2024-03-06T13:12:39.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:39.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:39.274146+0000) 2024-03-06T13:12:40.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:40.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:40.274369+0000) 2024-03-06T13:12:41.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:41.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:41.274644+0000) 2024-03-06T13:12:42.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:42.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:42.274834+0000) 2024-03-06T13:12:43.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:43.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:43.275027+0000) 2024-03-06T13:12:44.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:44.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:44.275228+0000) 2024-03-06T13:12:45.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:45.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:45.275475+0000) 2024-03-06T13:12:46.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:46.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:46.275696+0000) 2024-03-06T13:12:47.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:47.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:47.275887+0000) 2024-03-06T13:12:48.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:48.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:48.276069+0000) 2024-03-06T13:12:49.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:49.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:49.276288+0000) 2024-03-06T13:12:50.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:50.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:50.276518+0000) 2024-03-06T13:12:51.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:51.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:51.276760+0000) 2024-03-06T13:12:52.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:52.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:52.277042+0000) 2024-03-06T13:12:53.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:53.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:53.277249+0000) 2024-03-06T13:12:54.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:54.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:54.277473+0000) 2024-03-06T13:12:55.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:55.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:55.277697+0000) 2024-03-06T13:12:56.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:56.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:56.277890+0000) 2024-03-06T13:12:57.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:57.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:57.278093+0000) 2024-03-06T13:12:58.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:58.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:58.278322+0000) 2024-03-06T13:12:59.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:12:59.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:12:59.278594+0000) 2024-03-06T13:13:00.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:00.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:00.278787+0000) 2024-03-06T13:13:01.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:01.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:01.278984+0000) 2024-03-06T13:13:02.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:02.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:02.279181+0000) 2024-03-06T13:13:03.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:03.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:03.279411+0000) 2024-03-06T13:13:04.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:04.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:04.279650+0000) 2024-03-06T13:13:05.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:05.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:05.279847+0000) 2024-03-06T13:13:06.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:06.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:06.280061+0000) 2024-03-06T13:13:06.390 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:13:06.398 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:13:07.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:07.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:07.280326+0000) 2024-03-06T13:13:08.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:08.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:08.280563+0000) 2024-03-06T13:13:09.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:09.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:09.280794+0000) 2024-03-06T13:13:10.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:10.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:10.280998+0000) 2024-03-06T13:13:11.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:11.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:11.281233+0000) 2024-03-06T13:13:12.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:12.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:12.281452+0000) 2024-03-06T13:13:13.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:13.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:13.281674+0000) 2024-03-06T13:13:14.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:14.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:14.281871+0000) 2024-03-06T13:13:15.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:15.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:15.282029+0000) 2024-03-06T13:13:16.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:16.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:16.282220+0000) 2024-03-06T13:13:17.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:17.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:17.282501+0000) 2024-03-06T13:13:18.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:18.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:18.282748+0000) 2024-03-06T13:13:19.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:19.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:19.282958+0000) 2024-03-06T13:13:20.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:20.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:20.283202+0000) 2024-03-06T13:13:21.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:21.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:21.283439+0000) 2024-03-06T13:13:22.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:22.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:22.283658+0000) 2024-03-06T13:13:23.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:23.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:23.283874+0000) 2024-03-06T13:13:24.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:24.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:24.284101+0000) 2024-03-06T13:13:25.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:25.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:25.284335+0000) 2024-03-06T13:13:26.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:26.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:26.284610+0000) 2024-03-06T13:13:27.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:27.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:27.284853+0000) 2024-03-06T13:13:28.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:28.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:28.285080+0000) 2024-03-06T13:13:29.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:29.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:29.285261+0000) 2024-03-06T13:13:30.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:30.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:30.285559+0000) 2024-03-06T13:13:31.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:31.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:31.285757+0000) 2024-03-06T13:13:32.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:32.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:32.285997+0000) 2024-03-06T13:13:33.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:33.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:33.286195+0000) 2024-03-06T13:13:34.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:34.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:34.286438+0000) 2024-03-06T13:13:35.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:35.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:35.286665+0000) 2024-03-06T13:13:36.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:36.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:36.286904+0000) 2024-03-06T13:13:36.581 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:13:36.591 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:13:37.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:37.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:37.287115+0000) 2024-03-06T13:13:38.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:38.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:38.287328+0000) 2024-03-06T13:13:39.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:39.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:39.287587+0000) 2024-03-06T13:13:40.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:40.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:40.287777+0000) 2024-03-06T13:13:41.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:41.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:41.288019+0000) 2024-03-06T13:13:42.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:42.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:42.288202+0000) 2024-03-06T13:13:43.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:43.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:43.288436+0000) 2024-03-06T13:13:44.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:44.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:44.288677+0000) 2024-03-06T13:13:45.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:45.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:45.288870+0000) 2024-03-06T13:13:46.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:46.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:46.289112+0000) 2024-03-06T13:13:47.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:47.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:47.289346+0000) 2024-03-06T13:13:48.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:48.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:48.289620+0000) 2024-03-06T13:13:49.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:49.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:49.289854+0000) 2024-03-06T13:13:50.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:50.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:50.290039+0000) 2024-03-06T13:13:51.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:51.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:51.290278+0000) 2024-03-06T13:13:52.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:52.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:52.290500+0000) 2024-03-06T13:13:53.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:53.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:53.290694+0000) 2024-03-06T13:13:54.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:54.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:54.290868+0000) 2024-03-06T13:13:55.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:55.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:55.291096+0000) 2024-03-06T13:13:56.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:56.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:56.291317+0000) 2024-03-06T13:13:57.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:57.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:57.291576+0000) 2024-03-06T13:13:58.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:58.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:58.291778+0000) 2024-03-06T13:13:59.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:13:59.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:13:59.292011+0000) 2024-03-06T13:14:00.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:00.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:00.292249+0000) 2024-03-06T13:14:01.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:01.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:01.292449+0000) 2024-03-06T13:14:02.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:02.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:02.292671+0000) 2024-03-06T13:14:03.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:03.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:03.292919+0000) 2024-03-06T13:14:04.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:04.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:04.293139+0000) 2024-03-06T13:14:05.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:05.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:05.293376+0000) 2024-03-06T13:14:06.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:06.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:06.293656+0000) 2024-03-06T13:14:06.784 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:14:06.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:14:07.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:07.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:07.293902+0000) 2024-03-06T13:14:08.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:08.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:08.294126+0000) 2024-03-06T13:14:09.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:09.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:09.294353+0000) 2024-03-06T13:14:10.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:10.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:10.294572+0000) 2024-03-06T13:14:11.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:11.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:11.294816+0000) 2024-03-06T13:14:12.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:12.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:12.295050+0000) 2024-03-06T13:14:13.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:13.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:13.295307+0000) 2024-03-06T13:14:14.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:14.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:14.295575+0000) 2024-03-06T13:14:15.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:15.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:15.295734+0000) 2024-03-06T13:14:16.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:16.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:16.295963+0000) 2024-03-06T13:14:17.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:17.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:17.296205+0000) 2024-03-06T13:14:18.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:18.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:18.296430+0000) 2024-03-06T13:14:19.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:19.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:19.296668+0000) 2024-03-06T13:14:20.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:20.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:20.296856+0000) 2024-03-06T13:14:21.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:21.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:21.297095+0000) 2024-03-06T13:14:22.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:22.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:22.297322+0000) 2024-03-06T13:14:23.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:23.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:23.297572+0000) 2024-03-06T13:14:24.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:24.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:24.297799+0000) 2024-03-06T13:14:25.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:25.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:25.297996+0000) 2024-03-06T13:14:26.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:26.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:26.298190+0000) 2024-03-06T13:14:27.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:27.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:27.298434+0000) 2024-03-06T13:14:28.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:28.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:28.298656+0000) 2024-03-06T13:14:29.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:29.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:29.298854+0000) 2024-03-06T13:14:30.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:30.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:30.299077+0000) 2024-03-06T13:14:31.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:31.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:31.299274+0000) 2024-03-06T13:14:32.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:32.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:32.299448+0000) 2024-03-06T13:14:33.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:33.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:33.299647+0000) 2024-03-06T13:14:34.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:34.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:34.299890+0000) 2024-03-06T13:14:35.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:35.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:35.300122+0000) 2024-03-06T13:14:36.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:36.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:36.300316+0000) 2024-03-06T13:14:37.007 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:14:37.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:14:37.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:37.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:37.300585+0000) 2024-03-06T13:14:38.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:38.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:38.300786+0000) 2024-03-06T13:14:39.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:39.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:39.300979+0000) 2024-03-06T13:14:40.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:40.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:40.301207+0000) 2024-03-06T13:14:41.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:41.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:41.301436+0000) 2024-03-06T13:14:42.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:42.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:42.301666+0000) 2024-03-06T13:14:43.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:43.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:43.301893+0000) 2024-03-06T13:14:44.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:44.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:44.302120+0000) 2024-03-06T13:14:45.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:45.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:45.302379+0000) 2024-03-06T13:14:46.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:46.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:46.302690+0000) 2024-03-06T13:14:47.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:47.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:47.302976+0000) 2024-03-06T13:14:48.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:48.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:48.303216+0000) 2024-03-06T13:14:49.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:49.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:49.303448+0000) 2024-03-06T13:14:50.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:50.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:50.303730+0000) 2024-03-06T13:14:51.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:51.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:51.304016+0000) 2024-03-06T13:14:52.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:52.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:52.304293+0000) 2024-03-06T13:14:53.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:53.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:53.304614+0000) 2024-03-06T13:14:54.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:54.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:54.304855+0000) 2024-03-06T13:14:55.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:55.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:55.305084+0000) 2024-03-06T13:14:56.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:56.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:56.305313+0000) 2024-03-06T13:14:57.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:57.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:57.305589+0000) 2024-03-06T13:14:58.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:58.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:58.305817+0000) 2024-03-06T13:14:59.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:14:59.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:14:59.306049+0000) 2024-03-06T13:15:00.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:00.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:00.306308+0000) 2024-03-06T13:15:01.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:01.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:01.306528+0000) 2024-03-06T13:15:02.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:02.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:02.306793+0000) 2024-03-06T13:15:03.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:03.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:03.307010+0000) 2024-03-06T13:15:04.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:04.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:04.307227+0000) 2024-03-06T13:15:05.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:05.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:05.307457+0000) 2024-03-06T13:15:06.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:06.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:06.307748+0000) 2024-03-06T13:15:07.127 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:15:07.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:15:07.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:07.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:07.307960+0000) 2024-03-06T13:15:08.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:08.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:08.308220+0000) 2024-03-06T13:15:09.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:09.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:09.308438+0000) 2024-03-06T13:15:10.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:10.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:10.308657+0000) 2024-03-06T13:15:11.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:11.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:11.308939+0000) 2024-03-06T13:15:12.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:12.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:12.309176+0000) 2024-03-06T13:15:13.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:13.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:13.309441+0000) 2024-03-06T13:15:14.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:14.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:14.309686+0000) 2024-03-06T13:15:15.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:15.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:15.309912+0000) 2024-03-06T13:15:16.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:16.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:16.310137+0000) 2024-03-06T13:15:17.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:17.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:17.310379+0000) 2024-03-06T13:15:18.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:18.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:18.310695+0000) 2024-03-06T13:15:19.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:19.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:19.310922+0000) 2024-03-06T13:15:20.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:20.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:20.311102+0000) 2024-03-06T13:15:21.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:21.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:21.311372+0000) 2024-03-06T13:15:22.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:22.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:22.311681+0000) 2024-03-06T13:15:23.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:23.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:23.311918+0000) 2024-03-06T13:15:24.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:24.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:24.312159+0000) 2024-03-06T13:15:25.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:25.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:25.312393+0000) 2024-03-06T13:15:26.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:26.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:26.312643+0000) 2024-03-06T13:15:27.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:27.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:27.312840+0000) 2024-03-06T13:15:28.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:28.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:28.313060+0000) 2024-03-06T13:15:29.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:29.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:29.313290+0000) 2024-03-06T13:15:30.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:30.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:30.313490+0000) 2024-03-06T13:15:31.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:31.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:31.313768+0000) 2024-03-06T13:15:32.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:32.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:32.313943+0000) 2024-03-06T13:15:33.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:33.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:33.314175+0000) 2024-03-06T13:15:34.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:34.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:34.314366+0000) 2024-03-06T13:15:35.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:35.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:35.314618+0000) 2024-03-06T13:15:36.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:36.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:36.314862+0000) 2024-03-06T13:15:37.261 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:15:37.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:15:37.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:37.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:37.315078+0000) 2024-03-06T13:15:38.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:38.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:38.315310+0000) 2024-03-06T13:15:39.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:39.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:39.315564+0000) 2024-03-06T13:15:40.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:40.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:40.315787+0000) 2024-03-06T13:15:41.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:41.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:41.316015+0000) 2024-03-06T13:15:42.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:42.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:42.316212+0000) 2024-03-06T13:15:43.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:43.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:43.316436+0000) 2024-03-06T13:15:44.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:44.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:44.316677+0000) 2024-03-06T13:15:45.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:45.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:45.316883+0000) 2024-03-06T13:15:46.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:46.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:46.317091+0000) 2024-03-06T13:15:47.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:47.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:47.317335+0000) 2024-03-06T13:15:48.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:48.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:48.317595+0000) 2024-03-06T13:15:49.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:49.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:49.317805+0000) 2024-03-06T13:15:50.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:50.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:50.317998+0000) 2024-03-06T13:15:51.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:51.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:51.318232+0000) 2024-03-06T13:15:52.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:52.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:52.318455+0000) 2024-03-06T13:15:53.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:53.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:53.318684+0000) 2024-03-06T13:15:54.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:54.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:54.318952+0000) 2024-03-06T13:15:55.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:55.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:55.319180+0000) 2024-03-06T13:15:56.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:56.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:56.319405+0000) 2024-03-06T13:15:57.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:57.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:57.319652+0000) 2024-03-06T13:15:58.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:58.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:58.319883+0000) 2024-03-06T13:15:59.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:15:59.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:15:59.320096+0000) 2024-03-06T13:16:00.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:00.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:00.320338+0000) 2024-03-06T13:16:01.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:01.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:01.320596+0000) 2024-03-06T13:16:02.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:02.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:02.320819+0000) 2024-03-06T13:16:03.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:03.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:03.321008+0000) 2024-03-06T13:16:04.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:04.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:04.321247+0000) 2024-03-06T13:16:05.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:05.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:05.321452+0000) 2024-03-06T13:16:06.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:06.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:06.321684+0000) 2024-03-06T13:16:07.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:07.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:07.321912+0000) 2024-03-06T13:16:07.397 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:16:07.406 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:16:08.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:08.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:08.322135+0000) 2024-03-06T13:16:09.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:09.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:09.322367+0000) 2024-03-06T13:16:10.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:10.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:10.322637+0000) 2024-03-06T13:16:11.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:11.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:11.322823+0000) 2024-03-06T13:16:12.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:12.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:12.323043+0000) 2024-03-06T13:16:13.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:13.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:13.323276+0000) 2024-03-06T13:16:14.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:14.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:14.323500+0000) 2024-03-06T13:16:15.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:15.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:15.323742+0000) 2024-03-06T13:16:16.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:16.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:16.323967+0000) 2024-03-06T13:16:17.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:17.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:17.324220+0000) 2024-03-06T13:16:18.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:18.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:18.324441+0000) 2024-03-06T13:16:19.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:19.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:19.324682+0000) 2024-03-06T13:16:20.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:20.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:20.324879+0000) 2024-03-06T13:16:21.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:21.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:21.325083+0000) 2024-03-06T13:16:22.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:22.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:22.325286+0000) 2024-03-06T13:16:23.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:23.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:23.325494+0000) 2024-03-06T13:16:24.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:24.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:24.325805+0000) 2024-03-06T13:16:25.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:25.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:25.326052+0000) 2024-03-06T13:16:26.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:26.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:26.326277+0000) 2024-03-06T13:16:27.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:27.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:27.326485+0000) 2024-03-06T13:16:28.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:28.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:28.326773+0000) 2024-03-06T13:16:29.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:29.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:29.327055+0000) 2024-03-06T13:16:30.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:30.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:30.327336+0000) 2024-03-06T13:16:31.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:31.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:31.327660+0000) 2024-03-06T13:16:32.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:32.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:32.327932+0000) 2024-03-06T13:16:33.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:33.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:33.328182+0000) 2024-03-06T13:16:34.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:34.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:34.328449+0000) 2024-03-06T13:16:35.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:35.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:35.328731+0000) 2024-03-06T13:16:36.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:36.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:36.328969+0000) 2024-03-06T13:16:37.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:37.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:37.329192+0000) 2024-03-06T13:16:37.598 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:16:37.607 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:16:38.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:38.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:38.329449+0000) 2024-03-06T13:16:39.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:39.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:39.329677+0000) 2024-03-06T13:16:40.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:40.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:40.329901+0000) 2024-03-06T13:16:41.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:41.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:41.330089+0000) 2024-03-06T13:16:42.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:42.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:42.330324+0000) 2024-03-06T13:16:43.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:43.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:43.330578+0000) 2024-03-06T13:16:44.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:44.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:44.330811+0000) 2024-03-06T13:16:45.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:45.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:45.331040+0000) 2024-03-06T13:16:46.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:46.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:46.331262+0000) 2024-03-06T13:16:47.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:47.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:47.331576+0000) 2024-03-06T13:16:48.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:48.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:48.331754+0000) 2024-03-06T13:16:49.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:49.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:49.331980+0000) 2024-03-06T13:16:50.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:50.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:50.332206+0000) 2024-03-06T13:16:51.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:51.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:51.332430+0000) 2024-03-06T13:16:52.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:52.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:52.332652+0000) 2024-03-06T13:16:53.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:53.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:53.332884+0000) 2024-03-06T13:16:54.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:54.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:54.333078+0000) 2024-03-06T13:16:55.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:55.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:55.333293+0000) 2024-03-06T13:16:56.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:56.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:56.333528+0000) 2024-03-06T13:16:57.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:57.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:57.333790+0000) 2024-03-06T13:16:58.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:58.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:58.333978+0000) 2024-03-06T13:16:59.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:16:59.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:16:59.334168+0000) 2024-03-06T13:17:00.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:00.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:00.334390+0000) 2024-03-06T13:17:01.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:01.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:01.334660+0000) 2024-03-06T13:17:02.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:02.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:02.334883+0000) 2024-03-06T13:17:03.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:03.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:03.335104+0000) 2024-03-06T13:17:04.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:04.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:04.335326+0000) 2024-03-06T13:17:05.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:05.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:05.335587+0000) 2024-03-06T13:17:06.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:06.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:06.335812+0000) 2024-03-06T13:17:07.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:07.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:07.336053+0000) 2024-03-06T13:17:07.804 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:17:07.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:17:08.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:08.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:08.336279+0000) 2024-03-06T13:17:09.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:09.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:09.336506+0000) 2024-03-06T13:17:10.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:10.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:10.336756+0000) 2024-03-06T13:17:11.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:11.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:11.336980+0000) 2024-03-06T13:17:12.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:12.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:12.337198+0000) 2024-03-06T13:17:13.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:13.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:13.337421+0000) 2024-03-06T13:17:14.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:14.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:14.337658+0000) 2024-03-06T13:17:15.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:15.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:15.337859+0000) 2024-03-06T13:17:16.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:16.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:16.338084+0000) 2024-03-06T13:17:17.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:17.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:17.338254+0000) 2024-03-06T13:17:18.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:18.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:18.338473+0000) 2024-03-06T13:17:19.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:19.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:19.338673+0000) 2024-03-06T13:17:20.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:20.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:20.338875+0000) 2024-03-06T13:17:21.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:21.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:21.339077+0000) 2024-03-06T13:17:22.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:22.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:22.339286+0000) 2024-03-06T13:17:23.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:23.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:23.339523+0000) 2024-03-06T13:17:24.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:24.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:24.339761+0000) 2024-03-06T13:17:25.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:25.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:25.339968+0000) 2024-03-06T13:17:26.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:26.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:26.340201+0000) 2024-03-06T13:17:27.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:27.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:27.340436+0000) 2024-03-06T13:17:28.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:28.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:28.340661+0000) 2024-03-06T13:17:29.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:29.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:29.340846+0000) 2024-03-06T13:17:30.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:30.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:30.341077+0000) 2024-03-06T13:17:31.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:31.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:31.341300+0000) 2024-03-06T13:17:32.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:32.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:32.341522+0000) 2024-03-06T13:17:33.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:33.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:33.341789+0000) 2024-03-06T13:17:34.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:34.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:34.342012+0000) 2024-03-06T13:17:35.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:35.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:35.342193+0000) 2024-03-06T13:17:36.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:36.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:36.342421+0000) 2024-03-06T13:17:37.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:37.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:37.342660+0000) 2024-03-06T13:17:38.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:17:38.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:17:38.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:38.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:38.342864+0000) 2024-03-06T13:17:39.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:39.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:39.343103+0000) 2024-03-06T13:17:40.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:40.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:40.343330+0000) 2024-03-06T13:17:41.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:41.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:41.343655+0000) 2024-03-06T13:17:42.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:42.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:42.343843+0000) 2024-03-06T13:17:43.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:43.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:43.344063+0000) 2024-03-06T13:17:44.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:44.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:44.344287+0000) 2024-03-06T13:17:45.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:45.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:45.344517+0000) 2024-03-06T13:17:46.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:46.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:46.344782+0000) 2024-03-06T13:17:47.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:47.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:47.345018+0000) 2024-03-06T13:17:48.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:48.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:48.345223+0000) 2024-03-06T13:17:49.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:49.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:49.345472+0000) 2024-03-06T13:17:50.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:50.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:50.345695+0000) 2024-03-06T13:17:51.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:51.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:51.345892+0000) 2024-03-06T13:17:52.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:52.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:52.346070+0000) 2024-03-06T13:17:53.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:53.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:53.346297+0000) 2024-03-06T13:17:54.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:54.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:54.346516+0000) 2024-03-06T13:17:55.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:55.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:55.346781+0000) 2024-03-06T13:17:56.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:56.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:56.346957+0000) 2024-03-06T13:17:57.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:57.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:57.347201+0000) 2024-03-06T13:17:58.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:58.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:58.347425+0000) 2024-03-06T13:17:59.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:17:59.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:17:59.347659+0000) 2024-03-06T13:18:00.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:00.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:00.347851+0000) 2024-03-06T13:18:01.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:01.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:01.348124+0000) 2024-03-06T13:18:02.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:02.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:02.348364+0000) 2024-03-06T13:18:03.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:03.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:03.348630+0000) 2024-03-06T13:18:04.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:04.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:04.348858+0000) 2024-03-06T13:18:05.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:05.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:05.349095+0000) 2024-03-06T13:18:06.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:06.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:06.349315+0000) 2024-03-06T13:18:07.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:07.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:07.349635+0000) 2024-03-06T13:18:08.233 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:18:08.241 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:18:08.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:08.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:08.349921+0000) 2024-03-06T13:18:09.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:09.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:09.350124+0000) 2024-03-06T13:18:10.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:10.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:10.350347+0000) 2024-03-06T13:18:11.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:11.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:11.350562+0000) 2024-03-06T13:18:12.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:12.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:12.350829+0000) 2024-03-06T13:18:13.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:13.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:13.351079+0000) 2024-03-06T13:18:14.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:14.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:14.351344+0000) 2024-03-06T13:18:15.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:15.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:15.351633+0000) 2024-03-06T13:18:16.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:16.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:16.351868+0000) 2024-03-06T13:18:17.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:17.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:17.352126+0000) 2024-03-06T13:18:18.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:18.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:18.352423+0000) 2024-03-06T13:18:19.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:19.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:19.352704+0000) 2024-03-06T13:18:20.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:20.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:20.352984+0000) 2024-03-06T13:18:21.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:21.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:21.353266+0000) 2024-03-06T13:18:22.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:22.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:22.353501+0000) 2024-03-06T13:18:23.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:23.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:23.353718+0000) 2024-03-06T13:18:24.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:24.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:24.353946+0000) 2024-03-06T13:18:25.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:25.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:25.354180+0000) 2024-03-06T13:18:26.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:26.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:26.354408+0000) 2024-03-06T13:18:27.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:27.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:27.354661+0000) 2024-03-06T13:18:28.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:28.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:28.354896+0000) 2024-03-06T13:18:29.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:29.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:29.355128+0000) 2024-03-06T13:18:30.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:30.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:30.355362+0000) 2024-03-06T13:18:31.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:31.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:31.355617+0000) 2024-03-06T13:18:32.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:32.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:32.355845+0000) 2024-03-06T13:18:33.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:33.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:33.356065+0000) 2024-03-06T13:18:34.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:34.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:34.356265+0000) 2024-03-06T13:18:35.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:35.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:35.356492+0000) 2024-03-06T13:18:36.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:36.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:36.356715+0000) 2024-03-06T13:18:37.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:37.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:37.356946+0000) 2024-03-06T13:18:38.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:38.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:38.357165+0000) 2024-03-06T13:18:38.378 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:18:38.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:18:39.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:39.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:39.357393+0000) 2024-03-06T13:18:40.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:40.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:40.357640+0000) 2024-03-06T13:18:41.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:41.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:41.357865+0000) 2024-03-06T13:18:42.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:42.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:42.358083+0000) 2024-03-06T13:18:43.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:43.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:43.358363+0000) 2024-03-06T13:18:44.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:44.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:44.358665+0000) 2024-03-06T13:18:45.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:45.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:45.358942+0000) 2024-03-06T13:18:46.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:46.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:46.359141+0000) 2024-03-06T13:18:47.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:47.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:47.359393+0000) 2024-03-06T13:18:48.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:48.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:48.359666+0000) 2024-03-06T13:18:49.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:49.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:49.359890+0000) 2024-03-06T13:18:50.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:50.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:50.360093+0000) 2024-03-06T13:18:51.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:51.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:51.360337+0000) 2024-03-06T13:18:52.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:52.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:52.360605+0000) 2024-03-06T13:18:53.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:53.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:53.360835+0000) 2024-03-06T13:18:54.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:54.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:54.361061+0000) 2024-03-06T13:18:55.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:55.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:55.361294+0000) 2024-03-06T13:18:56.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:56.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:56.361513+0000) 2024-03-06T13:18:57.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:57.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:57.361785+0000) 2024-03-06T13:18:58.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:58.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:58.362021+0000) 2024-03-06T13:18:59.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:18:59.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:18:59.362247+0000) 2024-03-06T13:19:00.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:00.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:00.362513+0000) 2024-03-06T13:19:01.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:01.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:01.362801+0000) 2024-03-06T13:19:02.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:02.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:02.363095+0000) 2024-03-06T13:19:03.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:03.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:03.363379+0000) 2024-03-06T13:19:04.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:04.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:04.363689+0000) 2024-03-06T13:19:05.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:05.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:05.363971+0000) 2024-03-06T13:19:06.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:06.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:06.364251+0000) 2024-03-06T13:19:07.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:07.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:07.364529+0000) 2024-03-06T13:19:08.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:08.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:08.364868+0000) 2024-03-06T13:19:08.600 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:19:08.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:19:09.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:09.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:09.365151+0000) 2024-03-06T13:19:10.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:10.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:10.365368+0000) 2024-03-06T13:19:11.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:11.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:11.365680+0000) 2024-03-06T13:19:12.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:12.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:12.365964+0000) 2024-03-06T13:19:13.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:13.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:13.366205+0000) 2024-03-06T13:19:14.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:14.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:14.366460+0000) 2024-03-06T13:19:15.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:15.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:15.366682+0000) 2024-03-06T13:19:16.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:16.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:16.366931+0000) 2024-03-06T13:19:17.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:17.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:17.367255+0000) 2024-03-06T13:19:18.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:18.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:18.367440+0000) 2024-03-06T13:19:19.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:19.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:19.367659+0000) 2024-03-06T13:19:20.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:20.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:20.367881+0000) 2024-03-06T13:19:21.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:21.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:21.368146+0000) 2024-03-06T13:19:22.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:22.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:22.368358+0000) 2024-03-06T13:19:23.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:23.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:23.368589+0000) 2024-03-06T13:19:24.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:24.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:24.368828+0000) 2024-03-06T13:19:25.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:25.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:25.369013+0000) 2024-03-06T13:19:26.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:26.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:26.369274+0000) 2024-03-06T13:19:27.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:27.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:27.369617+0000) 2024-03-06T13:19:28.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:28.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:28.369896+0000) 2024-03-06T13:19:29.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:29.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:29.370120+0000) 2024-03-06T13:19:30.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:30.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:30.370399+0000) 2024-03-06T13:19:31.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:31.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:31.370643+0000) 2024-03-06T13:19:32.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:32.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:32.370884+0000) 2024-03-06T13:19:33.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:33.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:33.371119+0000) 2024-03-06T13:19:34.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:34.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:34.371341+0000) 2024-03-06T13:19:35.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:35.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:35.371483+0000) 2024-03-06T13:19:36.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:36.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:36.371703+0000) 2024-03-06T13:19:37.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:37.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:37.371961+0000) 2024-03-06T13:19:38.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:38.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:38.372188+0000) 2024-03-06T13:19:38.813 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:19:38.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:19:39.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:39.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:39.372419+0000) 2024-03-06T13:19:40.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:40.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:40.372648+0000) 2024-03-06T13:19:41.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:41.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:41.372830+0000) 2024-03-06T13:19:42.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:42.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:42.373052+0000) 2024-03-06T13:19:43.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:43.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:43.373334+0000) 2024-03-06T13:19:44.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:44.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:44.373631+0000) 2024-03-06T13:19:45.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:45.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:45.373821+0000) 2024-03-06T13:19:46.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:46.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:46.374043+0000) 2024-03-06T13:19:47.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:47.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:47.374297+0000) 2024-03-06T13:19:48.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:48.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:48.374527+0000) 2024-03-06T13:19:49.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:49.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:49.374804+0000) 2024-03-06T13:19:50.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:50.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:50.375037+0000) 2024-03-06T13:19:51.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:51.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:51.375264+0000) 2024-03-06T13:19:52.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:52.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:52.375513+0000) 2024-03-06T13:19:53.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:53.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:53.375781+0000) 2024-03-06T13:19:54.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:54.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:54.376033+0000) 2024-03-06T13:19:55.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:55.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:55.376234+0000) 2024-03-06T13:19:56.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:56.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:56.376467+0000) 2024-03-06T13:19:57.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:57.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:57.376740+0000) 2024-03-06T13:19:58.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:58.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:58.376945+0000) 2024-03-06T13:19:59.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:19:59.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:19:59.377181+0000) 2024-03-06T13:20:00.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:00.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:00.377417+0000) 2024-03-06T13:20:01.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:01.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:01.377640+0000) 2024-03-06T13:20:02.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:02.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:02.377865+0000) 2024-03-06T13:20:03.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:03.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:03.378093+0000) 2024-03-06T13:20:04.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:04.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:04.378318+0000) 2024-03-06T13:20:05.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:05.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:05.378648+0000) 2024-03-06T13:20:06.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:06.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:06.378861+0000) 2024-03-06T13:20:07.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:07.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:07.379113+0000) 2024-03-06T13:20:08.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:08.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:08.379354+0000) 2024-03-06T13:20:09.023 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:20:09.031 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:20:09.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:09.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:09.379624+0000) 2024-03-06T13:20:10.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:10.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:10.379818+0000) 2024-03-06T13:20:11.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:11.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:11.380037+0000) 2024-03-06T13:20:12.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:12.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:12.380257+0000) 2024-03-06T13:20:13.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:13.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:13.380452+0000) 2024-03-06T13:20:14.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:14.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:14.380683+0000) 2024-03-06T13:20:15.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:15.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:15.380913+0000) 2024-03-06T13:20:16.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:16.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:16.381145+0000) 2024-03-06T13:20:17.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:17.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:17.381383+0000) 2024-03-06T13:20:18.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:18.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:18.381643+0000) 2024-03-06T13:20:19.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:19.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:19.381874+0000) 2024-03-06T13:20:20.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:20.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:20.382068+0000) 2024-03-06T13:20:21.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:21.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:21.382256+0000) 2024-03-06T13:20:22.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:22.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:22.382461+0000) 2024-03-06T13:20:23.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:23.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:23.382677+0000) 2024-03-06T13:20:24.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:24.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:24.382889+0000) 2024-03-06T13:20:25.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:25.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:25.383073+0000) 2024-03-06T13:20:26.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:26.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:26.383281+0000) 2024-03-06T13:20:27.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:27.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:27.383512+0000) 2024-03-06T13:20:28.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:28.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:28.383747+0000) 2024-03-06T13:20:29.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:29.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:29.384003+0000) 2024-03-06T13:20:30.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:30.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:30.384229+0000) 2024-03-06T13:20:31.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:31.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:31.384477+0000) 2024-03-06T13:20:32.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:32.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:32.384745+0000) 2024-03-06T13:20:33.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:33.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:33.384962+0000) 2024-03-06T13:20:34.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:34.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:34.385162+0000) 2024-03-06T13:20:35.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:35.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:35.385411+0000) 2024-03-06T13:20:36.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:36.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:36.385673+0000) 2024-03-06T13:20:37.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:37.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:37.385904+0000) 2024-03-06T13:20:38.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:38.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:38.386173+0000) 2024-03-06T13:20:39.247 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:20:39.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:20:39.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:39.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:39.386404+0000) 2024-03-06T13:20:40.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:40.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:40.386663+0000) 2024-03-06T13:20:41.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:41.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:41.386911+0000) 2024-03-06T13:20:42.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:42.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:42.387144+0000) 2024-03-06T13:20:43.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:43.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:43.387395+0000) 2024-03-06T13:20:44.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:44.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:44.387704+0000) 2024-03-06T13:20:45.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:45.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:45.387985+0000) 2024-03-06T13:20:46.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:46.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:46.388233+0000) 2024-03-06T13:20:47.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:47.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:47.388444+0000) 2024-03-06T13:20:48.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:48.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:48.388699+0000) 2024-03-06T13:20:49.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:49.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:49.388937+0000) 2024-03-06T13:20:50.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:50.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:50.389129+0000) 2024-03-06T13:20:51.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:51.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:51.389356+0000) 2024-03-06T13:20:52.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:52.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:52.389574+0000) 2024-03-06T13:20:53.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:53.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:53.389777+0000) 2024-03-06T13:20:54.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:54.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:54.389999+0000) 2024-03-06T13:20:55.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:55.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:55.390225+0000) 2024-03-06T13:20:56.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:56.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:56.390450+0000) 2024-03-06T13:20:57.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:57.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:57.390694+0000) 2024-03-06T13:20:58.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:58.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:58.390950+0000) 2024-03-06T13:20:59.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:20:59.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:20:59.391191+0000) 2024-03-06T13:21:00.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:00.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:00.391440+0000) 2024-03-06T13:21:01.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:01.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:01.391671+0000) 2024-03-06T13:21:02.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:02.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:02.391875+0000) 2024-03-06T13:21:03.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:03.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:03.392091+0000) 2024-03-06T13:21:04.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:04.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:04.392371+0000) 2024-03-06T13:21:05.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:05.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:05.392691+0000) 2024-03-06T13:21:06.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:06.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:06.392972+0000) 2024-03-06T13:21:07.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:07.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:07.393275+0000) 2024-03-06T13:21:08.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:08.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:08.393493+0000) 2024-03-06T13:21:09.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:09.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:09.393719+0000) 2024-03-06T13:21:09.402 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:21:09.410 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:21:10.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:10.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:10.393991+0000) 2024-03-06T13:21:11.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:11.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:11.394240+0000) 2024-03-06T13:21:12.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:12.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:12.394462+0000) 2024-03-06T13:21:13.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:13.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:13.394697+0000) 2024-03-06T13:21:14.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:14.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:14.394853+0000) 2024-03-06T13:21:15.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:15.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:15.395095+0000) 2024-03-06T13:21:16.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:16.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:16.395333+0000) 2024-03-06T13:21:17.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:17.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:17.395633+0000) 2024-03-06T13:21:18.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:18.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:18.395874+0000) 2024-03-06T13:21:19.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:19.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:19.396116+0000) 2024-03-06T13:21:20.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:20.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:20.396332+0000) 2024-03-06T13:21:21.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:21.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:21.396609+0000) 2024-03-06T13:21:22.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:22.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:22.396817+0000) 2024-03-06T13:21:23.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:23.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:23.397028+0000) 2024-03-06T13:21:24.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:24.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:24.397281+0000) 2024-03-06T13:21:25.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:25.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:25.397594+0000) 2024-03-06T13:21:26.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:26.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:26.397882+0000) 2024-03-06T13:21:27.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:27.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:27.398108+0000) 2024-03-06T13:21:28.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:28.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:28.398349+0000) 2024-03-06T13:21:29.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:29.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:29.398619+0000) 2024-03-06T13:21:30.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:30.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:30.398831+0000) 2024-03-06T13:21:31.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:31.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:31.399054+0000) 2024-03-06T13:21:32.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:32.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:32.399297+0000) 2024-03-06T13:21:33.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:33.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:33.399586+0000) 2024-03-06T13:21:34.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:34.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:34.399808+0000) 2024-03-06T13:21:35.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:35.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:35.400057+0000) 2024-03-06T13:21:36.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:36.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:36.400310+0000) 2024-03-06T13:21:37.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:37.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:37.400644+0000) 2024-03-06T13:21:38.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:38.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:38.400931+0000) 2024-03-06T13:21:39.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:39.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:39.401180+0000) 2024-03-06T13:21:39.615 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:21:39.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:21:40.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:40.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:40.401426+0000) 2024-03-06T13:21:41.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:41.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:41.401667+0000) 2024-03-06T13:21:42.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:42.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:42.401892+0000) 2024-03-06T13:21:43.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:43.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:43.402123+0000) 2024-03-06T13:21:44.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:44.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:44.402302+0000) 2024-03-06T13:21:45.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:45.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:45.402586+0000) 2024-03-06T13:21:46.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:46.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:46.402804+0000) 2024-03-06T13:21:47.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:47.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:47.403036+0000) 2024-03-06T13:21:48.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:48.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:48.403263+0000) 2024-03-06T13:21:49.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:49.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:49.403486+0000) 2024-03-06T13:21:50.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:50.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:50.403714+0000) 2024-03-06T13:21:51.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:51.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:51.403909+0000) 2024-03-06T13:21:52.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:52.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:52.404088+0000) 2024-03-06T13:21:53.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:53.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:53.404294+0000) 2024-03-06T13:21:54.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:54.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:54.404525+0000) 2024-03-06T13:21:55.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:55.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:55.404791+0000) 2024-03-06T13:21:56.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:56.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:56.404976+0000) 2024-03-06T13:21:57.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:57.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:57.405233+0000) 2024-03-06T13:21:58.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:58.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:58.405471+0000) 2024-03-06T13:21:59.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:21:59.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:21:59.405687+0000) 2024-03-06T13:22:00.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:00.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:00.405889+0000) 2024-03-06T13:22:01.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:01.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:01.406111+0000) 2024-03-06T13:22:02.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:02.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:02.406298+0000) 2024-03-06T13:22:03.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:03.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:03.406575+0000) 2024-03-06T13:22:04.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:04.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:04.406779+0000) 2024-03-06T13:22:05.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:05.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:05.406997+0000) 2024-03-06T13:22:06.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:06.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:06.407209+0000) 2024-03-06T13:22:07.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:07.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:07.407449+0000) 2024-03-06T13:22:08.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:08.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:08.407689+0000) 2024-03-06T13:22:09.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:09.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:09.407903+0000) 2024-03-06T13:22:09.827 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:22:09.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:22:10.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:10.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:10.408106+0000) 2024-03-06T13:22:11.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:11.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:11.408297+0000) 2024-03-06T13:22:12.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:12.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:12.408564+0000) 2024-03-06T13:22:13.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:13.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:13.408798+0000) 2024-03-06T13:22:14.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:14.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:14.408997+0000) 2024-03-06T13:22:15.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:15.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:15.409233+0000) 2024-03-06T13:22:16.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:16.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:16.409457+0000) 2024-03-06T13:22:17.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:17.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:17.409703+0000) 2024-03-06T13:22:18.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:18.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:18.409928+0000) 2024-03-06T13:22:19.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:19.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:19.410159+0000) 2024-03-06T13:22:20.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:20.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:20.410393+0000) 2024-03-06T13:22:21.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:21.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:21.410658+0000) 2024-03-06T13:22:22.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:22.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:22.410844+0000) 2024-03-06T13:22:23.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:23.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:23.411079+0000) 2024-03-06T13:22:24.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:24.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:24.411315+0000) 2024-03-06T13:22:25.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:25.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:25.411557+0000) 2024-03-06T13:22:26.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:26.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:26.411776+0000) 2024-03-06T13:22:27.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:27.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:27.412022+0000) 2024-03-06T13:22:28.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:28.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:28.412262+0000) 2024-03-06T13:22:29.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:29.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:29.412497+0000) 2024-03-06T13:22:30.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:30.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:30.412735+0000) 2024-03-06T13:22:31.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:31.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:31.412935+0000) 2024-03-06T13:22:32.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:32.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:32.413173+0000) 2024-03-06T13:22:33.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:33.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:33.413353+0000) 2024-03-06T13:22:34.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:34.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:34.413585+0000) 2024-03-06T13:22:35.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:35.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:35.413827+0000) 2024-03-06T13:22:36.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:36.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:36.414051+0000) 2024-03-06T13:22:37.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:37.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:37.414308+0000) 2024-03-06T13:22:38.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:38.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:38.414489+0000) 2024-03-06T13:22:39.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:39.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:39.414739+0000) 2024-03-06T13:22:40.049 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:22:40.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:22:40.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:40.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:40.414976+0000) 2024-03-06T13:22:41.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:41.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:41.415160+0000) 2024-03-06T13:22:42.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:42.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:42.415353+0000) 2024-03-06T13:22:43.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:43.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:43.415637+0000) 2024-03-06T13:22:44.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:44.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:44.415862+0000) 2024-03-06T13:22:45.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:45.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:45.416096+0000) 2024-03-06T13:22:46.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:46.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:46.416316+0000) 2024-03-06T13:22:47.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:47.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:47.416589+0000) 2024-03-06T13:22:48.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:48.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:48.416749+0000) 2024-03-06T13:22:49.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:49.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:49.416968+0000) 2024-03-06T13:22:50.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:50.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:50.417227+0000) 2024-03-06T13:22:51.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:51.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:51.417461+0000) 2024-03-06T13:22:52.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:52.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:52.417688+0000) 2024-03-06T13:22:53.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:53.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:53.417874+0000) 2024-03-06T13:22:54.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:54.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:54.418103+0000) 2024-03-06T13:22:55.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:55.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:55.418335+0000) 2024-03-06T13:22:56.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:56.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:56.418609+0000) 2024-03-06T13:22:57.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:57.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:57.418867+0000) 2024-03-06T13:22:58.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:58.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:58.419096+0000) 2024-03-06T13:22:59.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:22:59.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:22:59.419280+0000) 2024-03-06T13:23:00.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:00.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:00.419512+0000) 2024-03-06T13:23:01.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:01.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:01.419746+0000) 2024-03-06T13:23:02.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:02.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:02.419934+0000) 2024-03-06T13:23:03.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:03.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:03.420170+0000) 2024-03-06T13:23:04.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:04.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:04.420464+0000) 2024-03-06T13:23:05.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:05.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:05.420725+0000) 2024-03-06T13:23:06.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:06.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:06.420956+0000) 2024-03-06T13:23:07.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:07.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:07.421212+0000) 2024-03-06T13:23:08.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:08.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:08.421460+0000) 2024-03-06T13:23:09.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:09.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:09.421698+0000) 2024-03-06T13:23:10.245 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:23:10.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:23:10.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:10.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:10.421912+0000) 2024-03-06T13:23:11.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:11.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:11.422111+0000) 2024-03-06T13:23:12.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:12.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:12.422334+0000) 2024-03-06T13:23:13.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:13.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:13.422620+0000) 2024-03-06T13:23:14.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:14.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:14.422823+0000) 2024-03-06T13:23:15.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:15.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:15.423061+0000) 2024-03-06T13:23:16.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:16.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:16.423291+0000) 2024-03-06T13:23:17.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:17.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:17.423491+0000) 2024-03-06T13:23:18.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:18.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:18.423698+0000) 2024-03-06T13:23:19.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:19.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:19.423899+0000) 2024-03-06T13:23:20.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:20.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:20.424134+0000) 2024-03-06T13:23:21.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:21.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:21.424368+0000) 2024-03-06T13:23:22.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:22.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:22.424690+0000) 2024-03-06T13:23:23.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:23.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:23.424935+0000) 2024-03-06T13:23:24.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:24.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:24.425186+0000) 2024-03-06T13:23:25.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:25.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:25.425414+0000) 2024-03-06T13:23:26.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:26.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:26.425656+0000) 2024-03-06T13:23:27.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:27.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:27.425929+0000) 2024-03-06T13:23:28.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:28.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:28.426167+0000) 2024-03-06T13:23:29.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:29.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:29.426394+0000) 2024-03-06T13:23:30.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:30.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:30.426671+0000) 2024-03-06T13:23:31.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:31.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:31.426852+0000) 2024-03-06T13:23:32.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:32.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:32.427077+0000) 2024-03-06T13:23:33.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:33.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:33.427313+0000) 2024-03-06T13:23:34.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:34.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:34.427577+0000) 2024-03-06T13:23:35.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:35.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:35.427823+0000) 2024-03-06T13:23:36.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:36.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:36.428054+0000) 2024-03-06T13:23:37.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:37.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:37.428305+0000) 2024-03-06T13:23:38.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:38.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:38.428514+0000) 2024-03-06T13:23:39.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:39.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:39.428779+0000) 2024-03-06T13:23:40.391 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:23:40.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:23:40.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:40.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:40.428967+0000) 2024-03-06T13:23:41.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:41.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:41.429199+0000) 2024-03-06T13:23:42.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:42.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:42.429438+0000) 2024-03-06T13:23:43.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:43.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:43.429696+0000) 2024-03-06T13:23:44.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:44.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:44.429920+0000) 2024-03-06T13:23:45.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:45.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:45.430149+0000) 2024-03-06T13:23:46.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:46.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:46.430407+0000) 2024-03-06T13:23:47.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:47.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:47.430640+0000) 2024-03-06T13:23:48.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:48.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:48.430796+0000) 2024-03-06T13:23:49.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:49.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:49.431050+0000) 2024-03-06T13:23:50.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:50.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:50.431242+0000) 2024-03-06T13:23:51.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:51.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:51.431461+0000) 2024-03-06T13:23:52.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:52.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:52.431699+0000) 2024-03-06T13:23:53.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:53.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:53.431932+0000) 2024-03-06T13:23:54.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:54.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:54.432130+0000) 2024-03-06T13:23:55.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:55.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:55.432362+0000) 2024-03-06T13:23:56.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:56.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:56.432641+0000) 2024-03-06T13:23:57.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:57.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:57.432899+0000) 2024-03-06T13:23:58.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:58.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:58.433133+0000) 2024-03-06T13:23:59.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:23:59.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:23:59.433342+0000) 2024-03-06T13:24:00.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:00.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:00.433566+0000) 2024-03-06T13:24:01.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:01.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:01.433817+0000) 2024-03-06T13:24:02.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:02.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:02.434046+0000) 2024-03-06T13:24:03.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:03.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:03.434273+0000) 2024-03-06T13:24:04.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:04.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:04.434515+0000) 2024-03-06T13:24:05.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:05.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:05.434732+0000) 2024-03-06T13:24:06.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:06.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:06.434960+0000) 2024-03-06T13:24:07.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:07.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:07.435207+0000) 2024-03-06T13:24:08.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:08.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:08.435389+0000) 2024-03-06T13:24:09.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:09.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:09.435673+0000) 2024-03-06T13:24:10.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:10.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:10.435875+0000) 2024-03-06T13:24:10.613 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:24:10.621 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:24:11.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:11.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:11.436103+0000) 2024-03-06T13:24:12.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:12.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:12.436323+0000) 2024-03-06T13:24:13.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:13.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:13.436600+0000) 2024-03-06T13:24:14.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:14.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:14.436818+0000) 2024-03-06T13:24:15.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:15.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:15.437116+0000) 2024-03-06T13:24:16.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:16.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:16.437368+0000) 2024-03-06T13:24:17.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:17.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:17.437657+0000) 2024-03-06T13:24:18.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:18.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:18.437888+0000) 2024-03-06T13:24:19.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:19.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:19.438117+0000) 2024-03-06T13:24:20.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:20.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:20.438341+0000) 2024-03-06T13:24:21.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:21.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:21.438626+0000) 2024-03-06T13:24:22.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:22.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:22.438847+0000) 2024-03-06T13:24:23.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:23.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:23.439078+0000) 2024-03-06T13:24:24.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:24.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:24.439271+0000) 2024-03-06T13:24:25.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:25.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:25.439511+0000) 2024-03-06T13:24:26.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:26.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:26.439772+0000) 2024-03-06T13:24:27.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:27.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:27.440001+0000) 2024-03-06T13:24:28.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:28.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:28.440233+0000) 2024-03-06T13:24:29.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:29.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:29.440466+0000) 2024-03-06T13:24:30.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:30.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:30.440703+0000) 2024-03-06T13:24:31.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:31.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:31.440936+0000) 2024-03-06T13:24:32.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:32.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:32.441161+0000) 2024-03-06T13:24:33.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:33.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:33.441396+0000) 2024-03-06T13:24:34.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:34.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:34.441655+0000) 2024-03-06T13:24:35.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:35.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:35.441854+0000) 2024-03-06T13:24:36.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:36.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:36.442037+0000) 2024-03-06T13:24:37.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:37.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:37.442295+0000) 2024-03-06T13:24:38.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:38.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:38.442574+0000) 2024-03-06T13:24:39.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:39.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:39.442783+0000) 2024-03-06T13:24:40.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:40.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:40.442959+0000) 2024-03-06T13:24:40.828 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:24:40.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:24:41.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:41.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:41.443190+0000) 2024-03-06T13:24:42.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:42.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:42.443405+0000) 2024-03-06T13:24:43.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:43.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:43.443690+0000) 2024-03-06T13:24:44.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:44.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:44.443872+0000) 2024-03-06T13:24:45.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:45.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:45.444121+0000) 2024-03-06T13:24:46.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:46.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:46.444364+0000) 2024-03-06T13:24:47.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:47.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:47.444637+0000) 2024-03-06T13:24:48.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:48.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:48.444841+0000) 2024-03-06T13:24:49.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:49.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:49.445073+0000) 2024-03-06T13:24:50.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:50.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:50.445304+0000) 2024-03-06T13:24:51.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:51.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:51.445572+0000) 2024-03-06T13:24:52.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:52.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:52.445781+0000) 2024-03-06T13:24:53.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:53.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:53.446020+0000) 2024-03-06T13:24:54.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:54.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:54.446248+0000) 2024-03-06T13:24:55.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:55.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:55.446476+0000) 2024-03-06T13:24:56.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:56.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:56.446687+0000) 2024-03-06T13:24:57.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:57.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:57.446893+0000) 2024-03-06T13:24:58.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:58.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:58.447132+0000) 2024-03-06T13:24:59.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:24:59.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:24:59.447361+0000) 2024-03-06T13:25:00.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:00.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:00.447597+0000) 2024-03-06T13:25:01.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:01.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:01.447851+0000) 2024-03-06T13:25:02.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:02.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:02.448065+0000) 2024-03-06T13:25:03.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:03.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:03.448303+0000) 2024-03-06T13:25:04.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:04.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:04.448492+0000) 2024-03-06T13:25:05.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:05.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:05.448666+0000) 2024-03-06T13:25:06.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:06.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:06.448883+0000) 2024-03-06T13:25:07.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:07.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:07.449115+0000) 2024-03-06T13:25:08.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:08.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:08.449356+0000) 2024-03-06T13:25:09.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:09.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:09.449674+0000) 2024-03-06T13:25:10.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:10.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:10.449904+0000) 2024-03-06T13:25:11.048 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:25:11.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:25:11.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:11.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:11.450133+0000) 2024-03-06T13:25:12.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:12.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:12.450352+0000) 2024-03-06T13:25:13.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:13.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:13.450631+0000) 2024-03-06T13:25:14.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:14.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:14.450864+0000) 2024-03-06T13:25:15.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:15.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:15.451072+0000) 2024-03-06T13:25:16.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:16.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:16.451248+0000) 2024-03-06T13:25:17.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:17.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:17.451489+0000) 2024-03-06T13:25:18.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:18.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:18.451703+0000) 2024-03-06T13:25:19.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:19.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:19.451905+0000) 2024-03-06T13:25:20.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:20.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:20.452096+0000) 2024-03-06T13:25:21.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:21.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:21.452327+0000) 2024-03-06T13:25:22.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:22.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:22.452579+0000) 2024-03-06T13:25:23.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:23.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:23.452825+0000) 2024-03-06T13:25:24.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:24.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:24.453014+0000) 2024-03-06T13:25:25.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:25.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:25.453250+0000) 2024-03-06T13:25:26.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:26.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:26.453474+0000) 2024-03-06T13:25:27.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:27.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:27.453707+0000) 2024-03-06T13:25:28.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:28.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:28.454001+0000) 2024-03-06T13:25:29.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:29.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:29.454217+0000) 2024-03-06T13:25:30.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:30.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:30.454433+0000) 2024-03-06T13:25:31.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:31.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:31.454717+0000) 2024-03-06T13:25:32.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:32.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:32.455001+0000) 2024-03-06T13:25:33.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:33.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:33.455298+0000) 2024-03-06T13:25:34.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:34.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:34.455592+0000) 2024-03-06T13:25:35.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:35.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:35.455842+0000) 2024-03-06T13:25:36.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:36.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:36.456060+0000) 2024-03-06T13:25:37.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:37.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:37.456313+0000) 2024-03-06T13:25:38.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:38.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:38.456612+0000) 2024-03-06T13:25:39.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:39.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:39.456858+0000) 2024-03-06T13:25:40.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:40.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:40.457113+0000) 2024-03-06T13:25:41.253 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:25:41.262 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:25:41.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:41.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:41.457371+0000) 2024-03-06T13:25:42.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:42.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:42.457672+0000) 2024-03-06T13:25:43.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:43.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:43.457892+0000) 2024-03-06T13:25:44.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:44.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:44.458120+0000) 2024-03-06T13:25:45.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:45.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:45.458326+0000) 2024-03-06T13:25:46.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:46.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:46.458523+0000) 2024-03-06T13:25:47.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:47.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:47.458771+0000) 2024-03-06T13:25:48.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:48.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:48.458981+0000) 2024-03-06T13:25:49.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:49.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:49.459177+0000) 2024-03-06T13:25:50.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:50.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:50.459420+0000) 2024-03-06T13:25:51.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:51.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:51.459661+0000) 2024-03-06T13:25:52.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:52.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:52.459849+0000) 2024-03-06T13:25:53.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:53.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:53.460080+0000) 2024-03-06T13:25:54.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:54.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:54.460300+0000) 2024-03-06T13:25:55.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:55.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:55.460565+0000) 2024-03-06T13:25:56.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:56.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:56.460797+0000) 2024-03-06T13:25:57.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:57.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:57.461087+0000) 2024-03-06T13:25:58.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:58.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:58.461317+0000) 2024-03-06T13:25:59.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:25:59.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:25:59.461575+0000) 2024-03-06T13:26:00.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:00.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:00.461770+0000) 2024-03-06T13:26:01.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:01.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:01.461956+0000) 2024-03-06T13:26:02.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:02.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:02.462136+0000) 2024-03-06T13:26:03.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:03.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:03.462368+0000) 2024-03-06T13:26:04.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:04.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:04.462604+0000) 2024-03-06T13:26:05.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:05.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:05.462822+0000) 2024-03-06T13:26:06.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:06.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:06.463042+0000) 2024-03-06T13:26:07.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:07.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:07.463229+0000) 2024-03-06T13:26:08.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:08.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:08.463463+0000) 2024-03-06T13:26:09.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:09.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:09.463716+0000) 2024-03-06T13:26:10.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:10.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:10.463890+0000) 2024-03-06T13:26:11.423 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:26:11.432 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:26:11.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:11.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:11.464083+0000) 2024-03-06T13:26:12.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:12.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:12.464301+0000) 2024-03-06T13:26:13.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:13.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:13.464571+0000) 2024-03-06T13:26:14.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:14.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:14.464804+0000) 2024-03-06T13:26:15.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:15.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:15.465033+0000) 2024-03-06T13:26:16.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:16.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:16.465232+0000) 2024-03-06T13:26:17.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:17.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:17.465449+0000) 2024-03-06T13:26:18.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:18.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:18.465715+0000) 2024-03-06T13:26:19.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:19.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:19.465893+0000) 2024-03-06T13:26:20.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:20.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:20.466127+0000) 2024-03-06T13:26:21.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:21.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:21.466367+0000) 2024-03-06T13:26:22.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:22.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:22.466658+0000) 2024-03-06T13:26:23.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:23.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:23.466939+0000) 2024-03-06T13:26:24.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:24.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:24.467217+0000) 2024-03-06T13:26:25.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:25.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:25.467488+0000) 2024-03-06T13:26:26.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:26.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:26.467788+0000) 2024-03-06T13:26:27.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:27.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:27.467965+0000) 2024-03-06T13:26:28.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:28.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:28.468169+0000) 2024-03-06T13:26:29.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:29.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:29.468416+0000) 2024-03-06T13:26:30.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:30.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:30.468679+0000) 2024-03-06T13:26:31.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:31.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:31.468896+0000) 2024-03-06T13:26:32.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:32.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:32.469118+0000) 2024-03-06T13:26:33.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:33.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:33.469348+0000) 2024-03-06T13:26:34.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:34.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:34.469578+0000) 2024-03-06T13:26:35.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:35.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:35.469790+0000) 2024-03-06T13:26:36.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:36.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:36.469975+0000) 2024-03-06T13:26:37.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:37.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:37.470192+0000) 2024-03-06T13:26:38.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:38.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:38.470457+0000) 2024-03-06T13:26:39.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:39.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:39.470741+0000) 2024-03-06T13:26:40.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:40.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:40.470975+0000) 2024-03-06T13:26:41.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:41.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:41.471215+0000) 2024-03-06T13:26:41.619 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:26:41.628 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:26:42.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:42.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:42.471446+0000) 2024-03-06T13:26:43.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:43.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:43.471694+0000) 2024-03-06T13:26:44.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:44.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:44.471900+0000) 2024-03-06T13:26:45.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:45.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:45.472114+0000) 2024-03-06T13:26:46.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:46.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:46.472345+0000) 2024-03-06T13:26:47.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:47.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:47.472572+0000) 2024-03-06T13:26:48.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:48.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:48.472740+0000) 2024-03-06T13:26:49.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:49.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:49.472983+0000) 2024-03-06T13:26:50.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:50.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:50.473261+0000) 2024-03-06T13:26:51.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:51.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:51.473486+0000) 2024-03-06T13:26:52.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:52.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:52.473747+0000) 2024-03-06T13:26:53.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:53.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:53.473995+0000) 2024-03-06T13:26:54.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:54.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:54.474236+0000) 2024-03-06T13:26:55.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:55.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:55.474469+0000) 2024-03-06T13:26:56.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:56.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:56.474696+0000) 2024-03-06T13:26:57.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:57.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:57.474932+0000) 2024-03-06T13:26:58.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:58.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:58.475156+0000) 2024-03-06T13:26:59.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:26:59.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:26:59.475342+0000) 2024-03-06T13:27:00.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:00.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:00.475520+0000) 2024-03-06T13:27:01.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:01.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:01.475831+0000) 2024-03-06T13:27:02.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:02.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:02.476028+0000) 2024-03-06T13:27:03.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:03.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:03.476255+0000) 2024-03-06T13:27:04.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:04.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:04.476478+0000) 2024-03-06T13:27:05.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:05.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:05.476681+0000) 2024-03-06T13:27:06.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:06.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:06.476888+0000) 2024-03-06T13:27:07.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:07.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:07.477126+0000) 2024-03-06T13:27:08.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:08.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:08.477330+0000) 2024-03-06T13:27:09.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:09.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:09.477615+0000) 2024-03-06T13:27:10.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:10.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:10.477819+0000) 2024-03-06T13:27:11.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:11.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:11.478026+0000) 2024-03-06T13:27:11.851 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:27:11.862 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:27:12.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:12.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:12.478270+0000) 2024-03-06T13:27:13.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:13.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:13.478514+0000) 2024-03-06T13:27:14.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:14.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:14.478786+0000) 2024-03-06T13:27:15.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:15.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:15.479023+0000) 2024-03-06T13:27:16.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:16.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:16.479264+0000) 2024-03-06T13:27:17.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:17.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:17.479476+0000) 2024-03-06T13:27:18.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:18.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:18.479679+0000) 2024-03-06T13:27:19.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:19.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:19.479885+0000) 2024-03-06T13:27:20.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:20.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:20.480071+0000) 2024-03-06T13:27:21.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:21.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:21.480296+0000) 2024-03-06T13:27:22.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:22.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:22.480510+0000) 2024-03-06T13:27:23.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:23.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:23.480724+0000) 2024-03-06T13:27:24.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:24.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:24.480948+0000) 2024-03-06T13:27:25.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:25.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:25.481150+0000) 2024-03-06T13:27:26.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:26.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:26.481377+0000) 2024-03-06T13:27:27.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:27.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:27.481648+0000) 2024-03-06T13:27:28.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:28.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:28.481833+0000) 2024-03-06T13:27:29.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:29.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:29.482027+0000) 2024-03-06T13:27:30.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:30.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:30.482259+0000) 2024-03-06T13:27:31.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:31.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:31.482462+0000) 2024-03-06T13:27:32.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:32.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:32.482665+0000) 2024-03-06T13:27:33.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:33.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:33.482924+0000) 2024-03-06T13:27:34.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:34.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:34.483153+0000) 2024-03-06T13:27:35.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:35.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:35.483332+0000) 2024-03-06T13:27:36.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:36.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:36.483519+0000) 2024-03-06T13:27:37.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:37.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:37.483763+0000) 2024-03-06T13:27:38.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:38.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:38.483966+0000) 2024-03-06T13:27:39.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:39.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:39.484232+0000) 2024-03-06T13:27:40.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:40.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:40.484512+0000) 2024-03-06T13:27:41.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:41.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:41.484822+0000) 2024-03-06T13:27:42.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:27:42.086 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:27:42.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:42.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:42.485109+0000) 2024-03-06T13:27:43.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:43.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:43.485355+0000) 2024-03-06T13:27:44.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:44.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:44.485577+0000) 2024-03-06T13:27:45.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:45.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:45.485819+0000) 2024-03-06T13:27:46.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:46.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:46.486002+0000) 2024-03-06T13:27:47.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:47.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:47.486238+0000) 2024-03-06T13:27:48.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:48.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:48.486450+0000) 2024-03-06T13:27:49.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:49.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:49.486703+0000) 2024-03-06T13:27:50.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:50.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:50.486885+0000) 2024-03-06T13:27:51.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:51.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:51.487081+0000) 2024-03-06T13:27:52.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:52.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:52.487298+0000) 2024-03-06T13:27:53.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:53.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:53.487514+0000) 2024-03-06T13:27:54.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:54.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:54.487734+0000) 2024-03-06T13:27:55.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:55.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:55.487941+0000) 2024-03-06T13:27:56.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:56.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:56.488150+0000) 2024-03-06T13:27:57.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:57.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:57.488401+0000) 2024-03-06T13:27:58.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:58.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:58.488662+0000) 2024-03-06T13:27:59.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:27:59.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:27:59.488858+0000) 2024-03-06T13:28:00.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:00.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:00.489050+0000) 2024-03-06T13:28:01.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:01.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:01.489225+0000) 2024-03-06T13:28:02.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:02.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:02.489449+0000) 2024-03-06T13:28:03.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:03.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:03.489660+0000) 2024-03-06T13:28:04.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:04.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:04.489849+0000) 2024-03-06T13:28:05.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:05.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:05.490092+0000) 2024-03-06T13:28:06.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:06.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:06.490310+0000) 2024-03-06T13:28:07.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:07.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:07.490623+0000) 2024-03-06T13:28:08.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:08.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:08.490890+0000) 2024-03-06T13:28:09.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:09.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:09.491175+0000) 2024-03-06T13:28:10.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:10.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:10.491451+0000) 2024-03-06T13:28:11.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:11.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:11.491694+0000) 2024-03-06T13:28:12.268 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:28:12.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:28:12.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:12.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:12.491901+0000) 2024-03-06T13:28:13.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:13.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:13.492136+0000) 2024-03-06T13:28:14.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:14.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:14.492363+0000) 2024-03-06T13:28:15.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:15.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:15.492612+0000) 2024-03-06T13:28:16.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:16.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:16.492831+0000) 2024-03-06T13:28:17.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:17.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:17.493034+0000) 2024-03-06T13:28:18.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:18.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:18.493245+0000) 2024-03-06T13:28:19.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:19.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:19.493477+0000) 2024-03-06T13:28:20.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:20.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:20.493661+0000) 2024-03-06T13:28:21.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:21.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:21.493872+0000) 2024-03-06T13:28:22.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:22.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:22.494056+0000) 2024-03-06T13:28:23.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:23.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:23.494280+0000) 2024-03-06T13:28:24.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:24.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:24.494500+0000) 2024-03-06T13:28:25.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:25.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:25.494701+0000) 2024-03-06T13:28:26.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:26.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:26.494969+0000) 2024-03-06T13:28:27.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:27.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:27.495187+0000) 2024-03-06T13:28:28.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:28.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:28.495396+0000) 2024-03-06T13:28:29.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:29.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:29.495614+0000) 2024-03-06T13:28:30.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:30.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:30.495796+0000) 2024-03-06T13:28:31.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:31.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:31.496025+0000) 2024-03-06T13:28:32.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:32.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:32.496194+0000) 2024-03-06T13:28:33.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:33.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:33.496425+0000) 2024-03-06T13:28:34.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:34.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:34.496660+0000) 2024-03-06T13:28:35.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:35.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:35.496897+0000) 2024-03-06T13:28:36.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:36.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:36.497082+0000) 2024-03-06T13:28:37.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:37.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:37.497318+0000) 2024-03-06T13:28:38.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:38.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:38.497566+0000) 2024-03-06T13:28:39.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:39.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:39.497763+0000) 2024-03-06T13:28:40.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:40.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:40.497946+0000) 2024-03-06T13:28:41.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:41.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:41.498133+0000) 2024-03-06T13:28:42.469 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:28:42.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:28:42.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:42.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:42.498412+0000) 2024-03-06T13:28:43.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:43.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:43.498667+0000) 2024-03-06T13:28:44.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:44.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:44.498861+0000) 2024-03-06T13:28:45.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:45.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:45.499109+0000) 2024-03-06T13:28:46.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:46.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:46.499300+0000) 2024-03-06T13:28:47.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:47.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:47.499555+0000) 2024-03-06T13:28:48.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:48.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:48.499736+0000) 2024-03-06T13:28:49.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:49.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:49.499968+0000) 2024-03-06T13:28:50.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:50.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:50.500193+0000) 2024-03-06T13:28:51.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:51.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:51.500476+0000) 2024-03-06T13:28:52.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:52.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:52.500710+0000) 2024-03-06T13:28:53.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:53.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:53.500912+0000) 2024-03-06T13:28:54.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:54.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:54.501145+0000) 2024-03-06T13:28:55.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:55.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:55.501376+0000) 2024-03-06T13:28:56.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:56.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:56.501668+0000) 2024-03-06T13:28:57.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:57.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:57.501961+0000) 2024-03-06T13:28:58.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:58.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:58.502177+0000) 2024-03-06T13:28:59.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:28:59.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:28:59.502414+0000) 2024-03-06T13:29:00.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:00.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:00.502657+0000) 2024-03-06T13:29:01.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:01.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:01.502872+0000) 2024-03-06T13:29:02.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:02.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:02.503056+0000) 2024-03-06T13:29:03.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:03.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:03.503255+0000) 2024-03-06T13:29:04.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:04.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:04.503485+0000) 2024-03-06T13:29:05.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:05.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:05.503709+0000) 2024-03-06T13:29:06.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:06.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:06.503894+0000) 2024-03-06T13:29:07.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:07.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:07.504092+0000) 2024-03-06T13:29:08.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:08.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:08.504282+0000) 2024-03-06T13:29:09.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:09.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:09.504491+0000) 2024-03-06T13:29:10.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:10.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:10.504770+0000) 2024-03-06T13:29:11.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:11.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:11.505012+0000) 2024-03-06T13:29:12.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:12.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:12.505252+0000) 2024-03-06T13:29:12.671 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:29:12.680 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:29:13.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:13.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:13.505579+0000) 2024-03-06T13:29:14.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:14.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:14.505859+0000) 2024-03-06T13:29:15.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:15.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:15.506133+0000) 2024-03-06T13:29:16.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:16.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:16.506367+0000) 2024-03-06T13:29:17.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:17.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:17.506623+0000) 2024-03-06T13:29:18.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:18.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:18.506837+0000) 2024-03-06T13:29:19.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:19.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:19.507072+0000) 2024-03-06T13:29:20.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:20.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:20.507269+0000) 2024-03-06T13:29:21.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:21.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:21.507514+0000) 2024-03-06T13:29:22.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:22.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:22.507720+0000) 2024-03-06T13:29:23.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:23.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:23.507932+0000) 2024-03-06T13:29:24.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:24.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:24.508178+0000) 2024-03-06T13:29:25.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:25.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:25.508380+0000) 2024-03-06T13:29:26.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:26.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:26.508649+0000) 2024-03-06T13:29:27.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:27.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:27.508846+0000) 2024-03-06T13:29:28.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:28.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:28.509059+0000) 2024-03-06T13:29:29.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:29.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:29.509290+0000) 2024-03-06T13:29:30.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:30.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:30.509516+0000) 2024-03-06T13:29:31.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:31.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:31.509769+0000) 2024-03-06T13:29:32.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:32.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:32.509989+0000) 2024-03-06T13:29:33.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:33.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:33.510184+0000) 2024-03-06T13:29:34.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:34.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:34.510405+0000) 2024-03-06T13:29:35.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:35.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:35.510648+0000) 2024-03-06T13:29:36.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:36.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:36.510870+0000) 2024-03-06T13:29:37.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:37.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:37.511104+0000) 2024-03-06T13:29:38.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:38.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:38.511320+0000) 2024-03-06T13:29:39.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:39.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:39.511578+0000) 2024-03-06T13:29:40.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:40.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:40.511825+0000) 2024-03-06T13:29:41.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:41.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:41.512057+0000) 2024-03-06T13:29:42.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:42.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:42.512277+0000) 2024-03-06T13:29:42.895 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:29:42.903 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:29:43.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:43.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:43.512465+0000) 2024-03-06T13:29:44.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:44.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:44.512696+0000) 2024-03-06T13:29:45.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:45.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:45.512891+0000) 2024-03-06T13:29:46.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:46.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:46.513114+0000) 2024-03-06T13:29:47.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:47.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:47.513349+0000) 2024-03-06T13:29:48.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:48.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:48.513594+0000) 2024-03-06T13:29:49.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:49.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:49.513843+0000) 2024-03-06T13:29:50.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:50.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:50.514026+0000) 2024-03-06T13:29:51.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:51.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:51.514256+0000) 2024-03-06T13:29:52.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:52.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:52.514478+0000) 2024-03-06T13:29:53.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:53.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:53.514700+0000) 2024-03-06T13:29:54.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:54.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:54.514898+0000) 2024-03-06T13:29:55.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:55.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:55.515088+0000) 2024-03-06T13:29:56.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:56.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:56.515298+0000) 2024-03-06T13:29:57.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:57.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:57.515518+0000) 2024-03-06T13:29:58.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:58.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:58.515770+0000) 2024-03-06T13:29:59.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:29:59.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:29:59.515968+0000) 2024-03-06T13:30:00.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:00.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:00.516207+0000) 2024-03-06T13:30:01.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:01.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:01.516449+0000) 2024-03-06T13:30:02.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:02.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:02.516684+0000) 2024-03-06T13:30:03.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:03.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:03.516901+0000) 2024-03-06T13:30:04.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:04.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:04.517113+0000) 2024-03-06T13:30:05.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:05.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:05.517362+0000) 2024-03-06T13:30:06.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:06.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:06.517628+0000) 2024-03-06T13:30:07.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:07.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:07.517901+0000) 2024-03-06T13:30:08.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:08.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:08.518087+0000) 2024-03-06T13:30:09.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:09.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:09.518293+0000) 2024-03-06T13:30:10.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:10.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:10.518511+0000) 2024-03-06T13:30:11.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:11.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:11.518756+0000) 2024-03-06T13:30:12.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:12.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:12.518981+0000) 2024-03-06T13:30:13.101 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:30:13.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:30:13.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:13.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:13.519213+0000) 2024-03-06T13:30:14.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:14.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:14.519446+0000) 2024-03-06T13:30:15.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:15.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:15.519740+0000) 2024-03-06T13:30:16.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:16.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:16.519995+0000) 2024-03-06T13:30:17.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:17.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:17.520286+0000) 2024-03-06T13:30:18.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:18.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:18.520477+0000) 2024-03-06T13:30:19.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:19.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:19.520694+0000) 2024-03-06T13:30:20.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:20.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:20.520883+0000) 2024-03-06T13:30:21.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:21.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:21.521095+0000) 2024-03-06T13:30:22.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:22.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:22.521287+0000) 2024-03-06T13:30:23.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:23.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:23.521523+0000) 2024-03-06T13:30:24.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:24.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:24.521754+0000) 2024-03-06T13:30:25.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:25.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:25.521966+0000) 2024-03-06T13:30:26.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:26.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:26.522158+0000) 2024-03-06T13:30:27.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:27.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:27.522405+0000) 2024-03-06T13:30:28.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:28.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:28.522658+0000) 2024-03-06T13:30:29.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:29.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:29.522866+0000) 2024-03-06T13:30:30.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:30.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:30.523064+0000) 2024-03-06T13:30:31.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:31.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:31.523299+0000) 2024-03-06T13:30:32.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:32.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:32.523511+0000) 2024-03-06T13:30:33.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:33.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:33.523747+0000) 2024-03-06T13:30:34.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:34.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:34.523950+0000) 2024-03-06T13:30:35.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:35.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:35.524147+0000) 2024-03-06T13:30:36.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:36.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:36.524334+0000) 2024-03-06T13:30:37.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:37.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:37.524604+0000) 2024-03-06T13:30:38.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:38.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:38.524781+0000) 2024-03-06T13:30:39.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:39.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:39.525070+0000) 2024-03-06T13:30:40.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:40.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:40.525330+0000) 2024-03-06T13:30:41.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:41.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:41.525640+0000) 2024-03-06T13:30:42.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:42.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:42.525868+0000) 2024-03-06T13:30:43.350 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:30:43.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:30:43.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:43.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:43.526105+0000) 2024-03-06T13:30:44.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:44.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:44.526385+0000) 2024-03-06T13:30:45.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:45.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:45.526704+0000) 2024-03-06T13:30:46.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:46.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:46.526958+0000) 2024-03-06T13:30:47.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:47.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:47.527263+0000) 2024-03-06T13:30:48.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:48.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:48.527515+0000) 2024-03-06T13:30:49.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:49.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:49.527810+0000) 2024-03-06T13:30:50.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:50.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:50.528089+0000) 2024-03-06T13:30:51.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:51.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:51.528372+0000) 2024-03-06T13:30:52.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:52.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:52.528667+0000) 2024-03-06T13:30:53.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:53.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:53.528938+0000) 2024-03-06T13:30:54.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:54.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:54.529219+0000) 2024-03-06T13:30:55.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:55.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:55.529423+0000) 2024-03-06T13:30:56.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:56.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:56.529648+0000) 2024-03-06T13:30:57.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:57.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:57.529883+0000) 2024-03-06T13:30:58.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:58.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:58.530050+0000) 2024-03-06T13:30:59.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:30:59.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:30:59.530302+0000) 2024-03-06T13:31:00.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:00.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:00.530586+0000) 2024-03-06T13:31:01.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:01.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:01.530862+0000) 2024-03-06T13:31:02.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:02.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:02.531088+0000) 2024-03-06T13:31:03.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:03.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:03.531319+0000) 2024-03-06T13:31:04.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:04.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:04.531589+0000) 2024-03-06T13:31:05.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:05.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:05.531822+0000) 2024-03-06T13:31:06.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:06.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:06.532007+0000) 2024-03-06T13:31:07.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:07.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:07.532204+0000) 2024-03-06T13:31:08.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:08.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:08.532420+0000) 2024-03-06T13:31:09.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:09.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:09.532666+0000) 2024-03-06T13:31:10.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:10.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:10.532890+0000) 2024-03-06T13:31:11.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:11.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:11.533090+0000) 2024-03-06T13:31:12.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:12.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:12.533287+0000) 2024-03-06T13:31:13.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:13.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:13.533521+0000) 2024-03-06T13:31:13.553 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:31:13.561 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:31:14.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:14.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:14.533744+0000) 2024-03-06T13:31:15.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:15.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:15.533947+0000) 2024-03-06T13:31:16.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:16.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:16.534129+0000) 2024-03-06T13:31:17.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:17.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:17.534417+0000) 2024-03-06T13:31:18.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:18.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:18.534637+0000) 2024-03-06T13:31:19.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:19.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:19.534854+0000) 2024-03-06T13:31:20.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:20.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:20.535044+0000) 2024-03-06T13:31:21.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:21.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:21.535271+0000) 2024-03-06T13:31:22.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:22.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:22.535481+0000) 2024-03-06T13:31:23.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:23.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:23.535685+0000) 2024-03-06T13:31:24.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:24.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:24.535904+0000) 2024-03-06T13:31:25.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:25.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:25.536132+0000) 2024-03-06T13:31:26.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:26.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:26.536362+0000) 2024-03-06T13:31:27.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:27.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:27.536622+0000) 2024-03-06T13:31:28.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:28.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:28.536803+0000) 2024-03-06T13:31:29.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:29.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:29.537002+0000) 2024-03-06T13:31:30.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:30.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:30.537222+0000) 2024-03-06T13:31:31.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:31.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:31.537444+0000) 2024-03-06T13:31:32.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:32.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:32.537696+0000) 2024-03-06T13:31:33.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:33.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:33.537852+0000) 2024-03-06T13:31:34.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:34.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:34.538054+0000) 2024-03-06T13:31:35.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:35.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:35.538280+0000) 2024-03-06T13:31:36.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:36.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:36.538561+0000) 2024-03-06T13:31:37.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:37.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:37.538800+0000) 2024-03-06T13:31:38.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:38.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:38.539017+0000) 2024-03-06T13:31:39.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:39.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:39.539255+0000) 2024-03-06T13:31:40.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:40.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:40.539478+0000) 2024-03-06T13:31:41.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:41.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:41.539761+0000) 2024-03-06T13:31:42.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:42.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:42.539990+0000) 2024-03-06T13:31:43.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:43.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:43.540276+0000) 2024-03-06T13:31:43.750 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:31:43.759 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:31:44.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:44.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:44.540514+0000) 2024-03-06T13:31:45.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:45.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:45.540730+0000) 2024-03-06T13:31:46.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:46.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:46.540918+0000) 2024-03-06T13:31:47.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:47.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:47.541140+0000) 2024-03-06T13:31:48.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:48.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:48.541364+0000) 2024-03-06T13:31:49.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:49.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:49.541623+0000) 2024-03-06T13:31:50.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:50.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:50.541821+0000) 2024-03-06T13:31:51.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:51.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:51.542098+0000) 2024-03-06T13:31:52.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:52.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:52.542356+0000) 2024-03-06T13:31:53.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:53.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:53.542663+0000) 2024-03-06T13:31:54.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:54.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:54.542938+0000) 2024-03-06T13:31:55.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:55.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:55.543188+0000) 2024-03-06T13:31:56.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:56.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:56.543446+0000) 2024-03-06T13:31:57.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:57.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:57.543728+0000) 2024-03-06T13:31:58.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:58.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:58.543930+0000) 2024-03-06T13:31:59.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:31:59.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:31:59.544178+0000) 2024-03-06T13:32:00.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:00.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:00.544398+0000) 2024-03-06T13:32:01.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:01.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:01.544651+0000) 2024-03-06T13:32:02.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:02.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:02.544868+0000) 2024-03-06T13:32:03.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:03.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:03.545097+0000) 2024-03-06T13:32:04.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:04.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:04.545316+0000) 2024-03-06T13:32:05.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:05.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:05.545521+0000) 2024-03-06T13:32:06.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:06.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:06.545729+0000) 2024-03-06T13:32:07.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:07.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:07.545944+0000) 2024-03-06T13:32:08.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:08.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:08.546172+0000) 2024-03-06T13:32:09.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:09.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:09.546405+0000) 2024-03-06T13:32:10.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:10.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:10.546636+0000) 2024-03-06T13:32:11.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:11.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:11.546850+0000) 2024-03-06T13:32:12.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:12.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:12.547034+0000) 2024-03-06T13:32:13.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:13.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:13.547260+0000) 2024-03-06T13:32:13.971 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:32:13.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:32:14.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:14.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:14.547485+0000) 2024-03-06T13:32:15.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:15.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:15.547690+0000) 2024-03-06T13:32:16.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:16.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:16.547901+0000) 2024-03-06T13:32:17.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:17.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:17.548192+0000) 2024-03-06T13:32:18.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:18.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:18.548439+0000) 2024-03-06T13:32:19.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:19.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:19.548725+0000) 2024-03-06T13:32:20.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:20.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:20.548965+0000) 2024-03-06T13:32:21.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:21.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:21.549244+0000) 2024-03-06T13:32:22.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:22.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:22.549506+0000) 2024-03-06T13:32:23.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:23.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:23.549820+0000) 2024-03-06T13:32:24.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:24.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:24.550068+0000) 2024-03-06T13:32:25.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:25.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:25.550352+0000) 2024-03-06T13:32:26.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:26.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:26.550651+0000) 2024-03-06T13:32:27.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:27.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:27.550847+0000) 2024-03-06T13:32:28.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:28.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:28.551063+0000) 2024-03-06T13:32:29.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:29.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:29.551297+0000) 2024-03-06T13:32:30.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:30.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:30.551519+0000) 2024-03-06T13:32:31.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:31.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:31.551786+0000) 2024-03-06T13:32:32.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:32.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:32.551966+0000) 2024-03-06T13:32:33.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:33.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:33.552187+0000) 2024-03-06T13:32:34.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:34.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:34.552369+0000) 2024-03-06T13:32:35.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:35.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:35.552642+0000) 2024-03-06T13:32:36.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:36.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:36.552840+0000) 2024-03-06T13:32:37.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:37.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:37.553107+0000) 2024-03-06T13:32:38.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:38.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:38.553305+0000) 2024-03-06T13:32:39.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:39.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:39.553580+0000) 2024-03-06T13:32:40.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:40.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:40.553863+0000) 2024-03-06T13:32:41.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:41.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:41.554095+0000) 2024-03-06T13:32:42.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:42.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:42.554310+0000) 2024-03-06T13:32:43.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:43.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:43.554565+0000) 2024-03-06T13:32:44.168 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:32:44.180 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:32:44.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:44.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:44.554830+0000) 2024-03-06T13:32:45.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:45.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:45.555115+0000) 2024-03-06T13:32:46.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:46.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:46.555313+0000) 2024-03-06T13:32:47.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:47.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:47.555601+0000) 2024-03-06T13:32:48.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:48.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:48.555788+0000) 2024-03-06T13:32:49.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:49.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:49.556001+0000) 2024-03-06T13:32:50.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:50.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:50.556240+0000) 2024-03-06T13:32:51.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:51.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:51.556479+0000) 2024-03-06T13:32:52.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:52.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:52.556700+0000) 2024-03-06T13:32:53.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:53.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:53.556946+0000) 2024-03-06T13:32:54.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:54.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:54.557166+0000) 2024-03-06T13:32:55.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:55.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:55.557353+0000) 2024-03-06T13:32:56.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:56.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:56.557628+0000) 2024-03-06T13:32:57.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:57.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:57.557874+0000) 2024-03-06T13:32:58.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:58.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:58.558104+0000) 2024-03-06T13:32:59.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:32:59.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:32:59.558373+0000) 2024-03-06T13:33:00.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:00.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:00.558654+0000) 2024-03-06T13:33:01.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:01.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:01.558910+0000) 2024-03-06T13:33:02.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:02.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:02.559161+0000) 2024-03-06T13:33:03.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:03.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:03.559443+0000) 2024-03-06T13:33:04.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:04.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:04.559721+0000) 2024-03-06T13:33:05.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:05.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:05.559969+0000) 2024-03-06T13:33:06.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:06.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:06.560163+0000) 2024-03-06T13:33:07.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:07.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:07.560390+0000) 2024-03-06T13:33:08.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:08.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:08.560663+0000) 2024-03-06T13:33:09.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:09.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:09.560895+0000) 2024-03-06T13:33:10.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:10.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:10.561101+0000) 2024-03-06T13:33:11.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:11.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:11.561326+0000) 2024-03-06T13:33:12.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:12.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:12.561568+0000) 2024-03-06T13:33:13.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:13.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:13.561795+0000) 2024-03-06T13:33:14.373 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:33:14.382 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:33:14.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:14.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:14.562036+0000) 2024-03-06T13:33:15.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:15.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:15.562267+0000) 2024-03-06T13:33:16.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:16.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:16.562505+0000) 2024-03-06T13:33:17.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:17.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:17.562829+0000) 2024-03-06T13:33:18.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:18.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:18.563084+0000) 2024-03-06T13:33:19.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:19.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:19.563354+0000) 2024-03-06T13:33:20.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:20.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:20.563595+0000) 2024-03-06T13:33:21.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:21.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:21.563834+0000) 2024-03-06T13:33:22.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:22.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:22.564075+0000) 2024-03-06T13:33:23.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:23.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:23.564271+0000) 2024-03-06T13:33:24.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:24.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:24.564506+0000) 2024-03-06T13:33:25.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:25.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:25.564827+0000) 2024-03-06T13:33:26.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:26.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:26.565015+0000) 2024-03-06T13:33:27.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:27.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:27.565270+0000) 2024-03-06T13:33:28.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:28.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:28.565498+0000) 2024-03-06T13:33:29.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:29.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:29.565794+0000) 2024-03-06T13:33:30.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:30.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:30.565994+0000) 2024-03-06T13:33:31.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:31.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:31.566205+0000) 2024-03-06T13:33:32.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:32.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:32.566436+0000) 2024-03-06T13:33:33.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:33.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:33.566683+0000) 2024-03-06T13:33:34.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:34.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:34.566893+0000) 2024-03-06T13:33:35.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:35.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:35.567113+0000) 2024-03-06T13:33:36.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:36.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:36.567339+0000) 2024-03-06T13:33:37.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:37.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:37.567602+0000) 2024-03-06T13:33:38.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:38.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:38.567825+0000) 2024-03-06T13:33:39.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:39.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:39.568022+0000) 2024-03-06T13:33:40.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:40.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:40.568206+0000) 2024-03-06T13:33:41.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:41.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:41.568435+0000) 2024-03-06T13:33:42.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:42.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:42.568659+0000) 2024-03-06T13:33:43.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:43.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:43.568876+0000) 2024-03-06T13:33:44.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:44.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:44.569085+0000) 2024-03-06T13:33:44.571 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:33:44.579 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:33:45.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:45.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:45.569330+0000) 2024-03-06T13:33:46.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:46.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:46.569574+0000) 2024-03-06T13:33:47.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:47.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:47.569816+0000) 2024-03-06T13:33:48.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:48.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:48.569996+0000) 2024-03-06T13:33:49.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:49.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:49.570223+0000) 2024-03-06T13:33:50.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:50.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:50.570441+0000) 2024-03-06T13:33:51.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:51.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:51.570688+0000) 2024-03-06T13:33:52.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:52.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:52.570938+0000) 2024-03-06T13:33:53.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:53.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:53.571217+0000) 2024-03-06T13:33:54.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:54.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:54.571479+0000) 2024-03-06T13:33:55.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:55.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:55.571765+0000) 2024-03-06T13:33:56.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:56.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:56.571998+0000) 2024-03-06T13:33:57.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:57.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:57.572241+0000) 2024-03-06T13:33:58.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:58.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:58.572487+0000) 2024-03-06T13:33:59.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:33:59.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:33:59.572708+0000) 2024-03-06T13:34:00.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:00.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:00.572927+0000) 2024-03-06T13:34:01.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:01.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:01.573204+0000) 2024-03-06T13:34:02.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:02.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:02.573377+0000) 2024-03-06T13:34:03.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:03.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:03.573670+0000) 2024-03-06T13:34:04.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:04.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:04.573950+0000) 2024-03-06T13:34:05.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:05.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:05.574186+0000) 2024-03-06T13:34:06.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:06.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:06.574389+0000) 2024-03-06T13:34:07.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:07.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:07.574647+0000) 2024-03-06T13:34:08.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:08.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:08.574901+0000) 2024-03-06T13:34:09.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:09.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:09.575129+0000) 2024-03-06T13:34:10.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:10.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:10.575326+0000) 2024-03-06T13:34:11.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:11.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:11.575589+0000) 2024-03-06T13:34:12.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:12.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:12.575835+0000) 2024-03-06T13:34:13.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:13.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:13.576114+0000) 2024-03-06T13:34:14.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:14.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:14.576401+0000) 2024-03-06T13:34:14.784 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:34:14.793 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:34:15.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:15.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:15.576690+0000) 2024-03-06T13:34:16.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:16.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:16.576955+0000) 2024-03-06T13:34:17.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:17.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:17.577248+0000) 2024-03-06T13:34:18.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:18.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:18.577603+0000) 2024-03-06T13:34:19.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:19.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:19.577887+0000) 2024-03-06T13:34:20.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:20.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:20.578165+0000) 2024-03-06T13:34:21.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:21.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:21.578371+0000) 2024-03-06T13:34:22.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:22.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:22.578676+0000) 2024-03-06T13:34:23.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:23.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:23.578954+0000) 2024-03-06T13:34:24.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:24.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:24.579233+0000) 2024-03-06T13:34:25.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:25.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:25.579507+0000) 2024-03-06T13:34:26.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:26.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:26.579824+0000) 2024-03-06T13:34:27.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:27.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:27.580049+0000) 2024-03-06T13:34:28.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:28.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:28.580333+0000) 2024-03-06T13:34:29.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:29.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:29.580654+0000) 2024-03-06T13:34:30.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:30.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:30.580933+0000) 2024-03-06T13:34:31.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:31.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:31.581178+0000) 2024-03-06T13:34:32.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:32.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:32.581429+0000) 2024-03-06T13:34:33.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:33.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:33.581717+0000) 2024-03-06T13:34:34.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:34.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:34.581997+0000) 2024-03-06T13:34:35.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:35.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:35.582270+0000) 2024-03-06T13:34:36.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:36.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:36.582502+0000) 2024-03-06T13:34:37.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:37.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:37.582769+0000) 2024-03-06T13:34:38.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:38.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:38.582986+0000) 2024-03-06T13:34:39.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:39.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:39.583211+0000) 2024-03-06T13:34:40.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:40.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:40.583439+0000) 2024-03-06T13:34:41.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:41.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:41.583673+0000) 2024-03-06T13:34:42.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:42.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:42.583898+0000) 2024-03-06T13:34:43.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:43.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:43.584124+0000) 2024-03-06T13:34:44.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:44.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:44.584316+0000) 2024-03-06T13:34:44.998 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:34:45.006 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:34:45.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:45.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:45.584562+0000) 2024-03-06T13:34:46.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:46.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:46.584803+0000) 2024-03-06T13:34:47.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:47.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:47.585038+0000) 2024-03-06T13:34:48.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:48.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:48.585261+0000) 2024-03-06T13:34:49.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:49.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:49.585486+0000) 2024-03-06T13:34:50.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:50.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:50.585720+0000) 2024-03-06T13:34:51.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:51.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:51.585922+0000) 2024-03-06T13:34:52.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:52.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:52.586132+0000) 2024-03-06T13:34:53.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:53.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:53.586321+0000) 2024-03-06T13:34:54.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:54.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:54.586505+0000) 2024-03-06T13:34:55.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:55.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:55.586775+0000) 2024-03-06T13:34:56.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:56.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:56.586992+0000) 2024-03-06T13:34:57.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:57.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:57.587234+0000) 2024-03-06T13:34:58.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:58.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:58.587405+0000) 2024-03-06T13:34:59.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:34:59.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:34:59.587656+0000) 2024-03-06T13:35:00.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:00.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:00.587842+0000) 2024-03-06T13:35:01.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:01.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:01.588037+0000) 2024-03-06T13:35:02.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:02.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:02.588260+0000) 2024-03-06T13:35:03.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:03.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:03.588482+0000) 2024-03-06T13:35:04.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:04.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:04.588688+0000) 2024-03-06T13:35:05.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:05.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:05.588888+0000) 2024-03-06T13:35:06.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:06.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:06.589194+0000) 2024-03-06T13:35:07.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:07.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:07.589408+0000) 2024-03-06T13:35:08.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:08.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:08.589657+0000) 2024-03-06T13:35:09.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:09.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:09.589879+0000) 2024-03-06T13:35:10.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:10.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:10.590100+0000) 2024-03-06T13:35:11.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:11.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:11.590278+0000) 2024-03-06T13:35:12.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:12.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:12.590501+0000) 2024-03-06T13:35:13.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:13.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:13.590695+0000) 2024-03-06T13:35:14.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:14.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:14.590879+0000) 2024-03-06T13:35:15.157 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:35:15.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:35:15.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:15.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:15.591070+0000) 2024-03-06T13:35:16.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:16.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:16.591254+0000) 2024-03-06T13:35:17.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:17.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:17.591489+0000) 2024-03-06T13:35:18.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:18.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:18.591676+0000) 2024-03-06T13:35:19.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:19.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:19.591868+0000) 2024-03-06T13:35:20.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:20.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:20.592133+0000) 2024-03-06T13:35:21.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:21.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:21.592410+0000) 2024-03-06T13:35:22.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:22.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:22.592656+0000) 2024-03-06T13:35:23.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:23.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:23.592899+0000) 2024-03-06T13:35:24.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:24.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:24.593182+0000) 2024-03-06T13:35:25.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:25.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:25.593423+0000) 2024-03-06T13:35:26.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:26.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:26.593633+0000) 2024-03-06T13:35:27.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:27.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:27.593861+0000) 2024-03-06T13:35:28.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:28.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:28.594109+0000) 2024-03-06T13:35:29.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:29.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:29.594335+0000) 2024-03-06T13:35:30.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:30.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:30.594597+0000) 2024-03-06T13:35:31.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:31.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:31.594820+0000) 2024-03-06T13:35:32.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:32.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:32.595019+0000) 2024-03-06T13:35:33.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:33.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:33.595244+0000) 2024-03-06T13:35:34.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:34.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:34.595467+0000) 2024-03-06T13:35:35.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:35.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:35.595700+0000) 2024-03-06T13:35:36.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:36.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:36.595884+0000) 2024-03-06T13:35:37.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:37.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:37.596082+0000) 2024-03-06T13:35:38.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:38.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:38.596341+0000) 2024-03-06T13:35:39.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:39.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:39.596655+0000) 2024-03-06T13:35:40.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:40.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:40.596934+0000) 2024-03-06T13:35:41.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:41.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:41.597175+0000) 2024-03-06T13:35:42.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:42.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:42.597359+0000) 2024-03-06T13:35:43.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:43.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:43.597604+0000) 2024-03-06T13:35:44.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:44.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:44.597853+0000) 2024-03-06T13:35:45.366 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:35:45.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:35:45.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:45.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:45.598017+0000) 2024-03-06T13:35:46.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:46.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:46.598236+0000) 2024-03-06T13:35:47.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:47.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:47.598465+0000) 2024-03-06T13:35:48.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:48.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:48.598684+0000) 2024-03-06T13:35:49.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:49.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:49.598897+0000) 2024-03-06T13:35:50.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:50.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:50.599114+0000) 2024-03-06T13:35:51.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:51.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:51.599354+0000) 2024-03-06T13:35:52.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:52.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:52.599628+0000) 2024-03-06T13:35:53.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:53.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:53.599893+0000) 2024-03-06T13:35:54.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:54.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:54.600100+0000) 2024-03-06T13:35:55.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:55.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:55.600353+0000) 2024-03-06T13:35:56.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:56.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:56.600567+0000) 2024-03-06T13:35:57.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:57.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:57.600784+0000) 2024-03-06T13:35:58.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:58.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:58.600978+0000) 2024-03-06T13:35:59.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:35:59.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:35:59.601263+0000) 2024-03-06T13:36:00.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:00.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:00.601495+0000) 2024-03-06T13:36:01.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:01.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:01.601725+0000) 2024-03-06T13:36:02.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:02.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:02.601916+0000) 2024-03-06T13:36:03.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:03.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:03.602152+0000) 2024-03-06T13:36:04.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:04.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:04.602321+0000) 2024-03-06T13:36:05.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:05.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:05.602479+0000) 2024-03-06T13:36:06.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:06.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:06.602696+0000) 2024-03-06T13:36:07.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:07.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:07.602899+0000) 2024-03-06T13:36:08.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:08.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:08.603098+0000) 2024-03-06T13:36:09.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:09.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:09.603272+0000) 2024-03-06T13:36:10.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:10.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:10.603498+0000) 2024-03-06T13:36:11.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:11.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:11.603780+0000) 2024-03-06T13:36:12.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:12.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:12.604003+0000) 2024-03-06T13:36:13.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:13.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:13.604232+0000) 2024-03-06T13:36:14.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:14.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:14.604417+0000) 2024-03-06T13:36:15.587 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:36:15.595 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:36:15.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:15.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:15.604653+0000) 2024-03-06T13:36:16.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:16.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:16.604847+0000) 2024-03-06T13:36:17.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:17.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:17.605097+0000) 2024-03-06T13:36:18.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:18.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:18.605278+0000) 2024-03-06T13:36:19.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:19.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:19.605521+0000) 2024-03-06T13:36:20.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:20.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:20.605823+0000) 2024-03-06T13:36:21.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:21.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:21.606069+0000) 2024-03-06T13:36:22.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:22.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:22.606262+0000) 2024-03-06T13:36:23.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:23.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:23.606452+0000) 2024-03-06T13:36:24.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:24.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:24.606704+0000) 2024-03-06T13:36:25.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:25.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:25.606917+0000) 2024-03-06T13:36:26.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:26.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:26.607134+0000) 2024-03-06T13:36:27.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:27.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:27.607359+0000) 2024-03-06T13:36:28.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:28.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:28.607638+0000) 2024-03-06T13:36:29.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:29.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:29.607817+0000) 2024-03-06T13:36:30.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:30.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:30.608053+0000) 2024-03-06T13:36:31.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:31.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:31.608231+0000) 2024-03-06T13:36:32.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:32.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:32.608452+0000) 2024-03-06T13:36:33.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:33.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:33.608672+0000) 2024-03-06T13:36:34.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:34.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:34.608844+0000) 2024-03-06T13:36:35.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:35.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:35.609076+0000) 2024-03-06T13:36:36.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:36.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:36.609315+0000) 2024-03-06T13:36:37.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:37.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:37.609604+0000) 2024-03-06T13:36:38.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:38.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:38.609835+0000) 2024-03-06T13:36:39.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:39.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:39.610075+0000) 2024-03-06T13:36:40.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:40.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:40.610307+0000) 2024-03-06T13:36:41.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:41.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:41.610506+0000) 2024-03-06T13:36:42.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:42.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:42.610696+0000) 2024-03-06T13:36:43.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:43.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:43.610911+0000) 2024-03-06T13:36:44.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:44.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:44.611151+0000) 2024-03-06T13:36:45.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:45.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:45.611406+0000) 2024-03-06T13:36:45.725 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:36:45.734 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:36:46.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:46.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:46.611628+0000) 2024-03-06T13:36:47.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:47.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:47.611849+0000) 2024-03-06T13:36:48.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:48.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:48.612037+0000) 2024-03-06T13:36:49.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:49.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:49.612217+0000) 2024-03-06T13:36:50.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:50.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:50.612465+0000) 2024-03-06T13:36:51.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:51.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:51.612684+0000) 2024-03-06T13:36:52.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:52.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:52.612924+0000) 2024-03-06T13:36:53.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:53.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:53.613129+0000) 2024-03-06T13:36:54.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:54.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:54.613356+0000) 2024-03-06T13:36:55.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:55.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:55.613622+0000) 2024-03-06T13:36:56.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:56.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:56.613807+0000) 2024-03-06T13:36:57.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:57.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:57.614004+0000) 2024-03-06T13:36:58.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:58.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:58.614222+0000) 2024-03-06T13:36:59.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:36:59.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:36:59.614416+0000) 2024-03-06T13:37:00.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:00.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:00.614671+0000) 2024-03-06T13:37:01.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:01.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:01.614884+0000) 2024-03-06T13:37:02.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:02.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:02.615090+0000) 2024-03-06T13:37:03.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:03.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:03.615311+0000) 2024-03-06T13:37:04.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:04.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:04.615529+0000) 2024-03-06T13:37:05.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:05.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:05.615755+0000) 2024-03-06T13:37:06.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:06.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:06.615940+0000) 2024-03-06T13:37:07.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:07.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:07.616139+0000) 2024-03-06T13:37:08.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:08.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:08.616372+0000) 2024-03-06T13:37:09.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:09.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:09.616602+0000) 2024-03-06T13:37:10.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:10.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:10.616832+0000) 2024-03-06T13:37:11.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:11.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:11.617065+0000) 2024-03-06T13:37:12.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:12.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:12.617243+0000) 2024-03-06T13:37:13.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:13.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:13.617479+0000) 2024-03-06T13:37:14.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:14.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:14.617700+0000) 2024-03-06T13:37:15.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:15.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:15.617947+0000) 2024-03-06T13:37:15.932 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:37:15.942 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:37:16.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:16.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:16.618155+0000) 2024-03-06T13:37:17.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:17.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:17.618407+0000) 2024-03-06T13:37:18.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:18.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:18.618675+0000) 2024-03-06T13:37:19.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:19.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:19.618922+0000) 2024-03-06T13:37:20.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:20.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:20.619202+0000) 2024-03-06T13:37:21.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:21.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:21.619485+0000) 2024-03-06T13:37:22.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:22.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:22.619768+0000) 2024-03-06T13:37:23.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:23.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:23.620029+0000) 2024-03-06T13:37:24.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:24.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:24.620271+0000) 2024-03-06T13:37:25.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:25.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:25.620524+0000) 2024-03-06T13:37:26.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:26.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:26.620772+0000) 2024-03-06T13:37:27.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:27.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:27.620993+0000) 2024-03-06T13:37:28.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:28.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:28.621252+0000) 2024-03-06T13:37:29.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:29.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:29.621579+0000) 2024-03-06T13:37:30.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:30.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:30.621804+0000) 2024-03-06T13:37:31.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:31.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:31.622042+0000) 2024-03-06T13:37:32.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:32.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:32.622260+0000) 2024-03-06T13:37:33.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:33.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:33.622491+0000) 2024-03-06T13:37:34.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:34.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:34.622716+0000) 2024-03-06T13:37:35.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:35.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:35.622964+0000) 2024-03-06T13:37:36.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:36.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:36.623204+0000) 2024-03-06T13:37:37.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:37.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:37.623434+0000) 2024-03-06T13:37:38.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:38.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:38.623653+0000) 2024-03-06T13:37:39.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:39.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:39.623847+0000) 2024-03-06T13:37:40.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:40.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:40.624038+0000) 2024-03-06T13:37:41.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:41.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:41.624222+0000) 2024-03-06T13:37:42.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:42.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:42.624445+0000) 2024-03-06T13:37:43.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:43.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:43.624665+0000) 2024-03-06T13:37:44.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:44.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:44.624817+0000) 2024-03-06T13:37:45.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:45.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:45.625061+0000) 2024-03-06T13:37:46.154 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:37:46.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:37:46.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:46.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:46.625307+0000) 2024-03-06T13:37:47.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:47.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:47.625592+0000) 2024-03-06T13:37:48.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:48.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:48.625796+0000) 2024-03-06T13:37:49.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:49.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:49.626004+0000) 2024-03-06T13:37:50.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:50.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:50.626246+0000) 2024-03-06T13:37:51.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:51.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:51.626485+0000) 2024-03-06T13:37:52.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:52.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:52.626685+0000) 2024-03-06T13:37:53.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:53.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:53.626926+0000) 2024-03-06T13:37:54.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:54.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:54.627159+0000) 2024-03-06T13:37:55.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:55.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:55.627445+0000) 2024-03-06T13:37:56.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:56.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:56.627704+0000) 2024-03-06T13:37:57.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:57.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:57.627902+0000) 2024-03-06T13:37:58.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:58.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:58.628121+0000) 2024-03-06T13:37:59.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:37:59.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:37:59.628307+0000) 2024-03-06T13:38:00.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:00.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:00.628512+0000) 2024-03-06T13:38:01.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:01.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:01.628752+0000) 2024-03-06T13:38:02.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:02.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:02.628927+0000) 2024-03-06T13:38:03.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:03.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:03.629155+0000) 2024-03-06T13:38:04.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:04.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:04.629374+0000) 2024-03-06T13:38:05.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:05.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:05.629698+0000) 2024-03-06T13:38:06.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:06.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:06.629984+0000) 2024-03-06T13:38:07.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:07.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:07.630232+0000) 2024-03-06T13:38:08.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:08.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:08.630510+0000) 2024-03-06T13:38:09.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:09.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:09.630840+0000) 2024-03-06T13:38:10.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:10.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:10.631090+0000) 2024-03-06T13:38:11.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:11.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:11.631326+0000) 2024-03-06T13:38:12.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:12.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:12.631642+0000) 2024-03-06T13:38:13.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:13.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:13.631922+0000) 2024-03-06T13:38:14.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:14.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:14.632130+0000) 2024-03-06T13:38:15.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:15.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:15.632408+0000) 2024-03-06T13:38:16.385 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:38:16.394 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:38:16.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:16.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:16.632665+0000) 2024-03-06T13:38:17.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:17.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:17.632935+0000) 2024-03-06T13:38:18.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:18.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:18.633179+0000) 2024-03-06T13:38:19.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:19.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:19.633404+0000) 2024-03-06T13:38:20.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:20.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:20.633625+0000) 2024-03-06T13:38:21.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:21.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:21.633807+0000) 2024-03-06T13:38:22.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:22.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:22.633989+0000) 2024-03-06T13:38:23.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:23.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:23.634181+0000) 2024-03-06T13:38:24.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:24.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:24.634412+0000) 2024-03-06T13:38:25.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:25.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:25.634647+0000) 2024-03-06T13:38:26.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:26.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:26.634871+0000) 2024-03-06T13:38:27.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:27.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:27.635105+0000) 2024-03-06T13:38:28.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:28.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:28.635341+0000) 2024-03-06T13:38:29.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:29.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:29.635702+0000) 2024-03-06T13:38:30.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:30.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:30.635930+0000) 2024-03-06T13:38:31.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:31.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:31.636122+0000) 2024-03-06T13:38:32.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:32.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:32.636302+0000) 2024-03-06T13:38:33.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:33.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:33.636489+0000) 2024-03-06T13:38:34.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:34.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:34.636691+0000) 2024-03-06T13:38:35.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:35.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:35.636896+0000) 2024-03-06T13:38:36.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:36.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:36.637107+0000) 2024-03-06T13:38:37.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:37.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:37.637325+0000) 2024-03-06T13:38:38.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:38.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:38.637575+0000) 2024-03-06T13:38:39.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:39.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:39.637729+0000) 2024-03-06T13:38:40.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:40.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:40.637930+0000) 2024-03-06T13:38:41.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:41.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:41.638145+0000) 2024-03-06T13:38:42.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:42.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:42.638384+0000) 2024-03-06T13:38:43.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:43.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:43.638599+0000) 2024-03-06T13:38:44.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:44.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:44.638823+0000) 2024-03-06T13:38:45.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:45.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:45.639081+0000) 2024-03-06T13:38:46.580 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:38:46.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:38:46.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:46.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:46.639273+0000) 2024-03-06T13:38:47.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:47.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:47.639477+0000) 2024-03-06T13:38:48.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:48.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:48.639720+0000) 2024-03-06T13:38:49.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:49.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:49.639951+0000) 2024-03-06T13:38:50.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:50.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:50.640143+0000) 2024-03-06T13:38:51.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:51.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:51.640374+0000) 2024-03-06T13:38:52.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:52.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:52.640623+0000) 2024-03-06T13:38:53.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:53.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:53.640820+0000) 2024-03-06T13:38:54.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:54.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:54.641003+0000) 2024-03-06T13:38:55.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:55.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:55.641230+0000) 2024-03-06T13:38:56.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:56.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:56.641412+0000) 2024-03-06T13:38:57.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:57.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:57.641631+0000) 2024-03-06T13:38:58.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:58.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:58.641830+0000) 2024-03-06T13:38:59.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:38:59.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:38:59.642025+0000) 2024-03-06T13:39:00.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:00.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:00.642246+0000) 2024-03-06T13:39:01.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:01.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:01.642458+0000) 2024-03-06T13:39:02.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:02.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:02.642693+0000) 2024-03-06T13:39:03.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:03.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:03.642976+0000) 2024-03-06T13:39:04.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:04.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:04.643245+0000) 2024-03-06T13:39:05.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:05.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:05.643524+0000) 2024-03-06T13:39:06.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:06.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:06.643828+0000) 2024-03-06T13:39:07.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:07.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:07.644118+0000) 2024-03-06T13:39:08.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:08.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:08.644318+0000) 2024-03-06T13:39:09.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:09.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:09.644524+0000) 2024-03-06T13:39:10.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:10.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:10.644791+0000) 2024-03-06T13:39:11.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:11.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:11.645017+0000) 2024-03-06T13:39:12.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:12.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:12.645210+0000) 2024-03-06T13:39:13.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:13.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:13.645444+0000) 2024-03-06T13:39:14.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:14.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:14.645663+0000) 2024-03-06T13:39:15.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:15.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:15.645851+0000) 2024-03-06T13:39:16.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:16.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:16.646045+0000) 2024-03-06T13:39:16.719 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:39:16.727 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:39:17.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:17.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:17.646345+0000) 2024-03-06T13:39:18.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:18.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:18.646608+0000) 2024-03-06T13:39:19.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:19.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:19.646798+0000) 2024-03-06T13:39:20.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:20.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:20.646987+0000) 2024-03-06T13:39:21.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:21.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:21.647191+0000) 2024-03-06T13:39:22.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:22.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:22.647410+0000) 2024-03-06T13:39:23.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:23.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:23.647650+0000) 2024-03-06T13:39:24.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:24.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:24.647834+0000) 2024-03-06T13:39:25.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:25.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:25.648033+0000) 2024-03-06T13:39:26.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:26.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:26.648249+0000) 2024-03-06T13:39:27.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:27.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:27.648440+0000) 2024-03-06T13:39:28.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:28.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:28.648662+0000) 2024-03-06T13:39:29.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:29.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:29.648943+0000) 2024-03-06T13:39:30.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:30.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:30.649226+0000) 2024-03-06T13:39:31.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:31.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:31.649473+0000) 2024-03-06T13:39:32.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:32.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:32.649647+0000) 2024-03-06T13:39:33.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:33.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:33.649846+0000) 2024-03-06T13:39:34.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:34.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:34.650066+0000) 2024-03-06T13:39:35.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:35.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:35.650262+0000) 2024-03-06T13:39:36.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:36.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:36.650454+0000) 2024-03-06T13:39:37.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:37.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:37.650684+0000) 2024-03-06T13:39:38.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:38.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:38.650882+0000) 2024-03-06T13:39:39.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:39.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:39.651088+0000) 2024-03-06T13:39:40.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:40.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:40.651328+0000) 2024-03-06T13:39:41.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:41.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:41.651619+0000) 2024-03-06T13:39:42.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:42.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:42.651822+0000) 2024-03-06T13:39:43.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:43.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:43.652015+0000) 2024-03-06T13:39:44.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:44.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:44.652219+0000) 2024-03-06T13:39:45.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:45.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:45.652447+0000) 2024-03-06T13:39:46.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:46.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:46.652686+0000) 2024-03-06T13:39:46.939 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:39:46.948 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:39:47.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:47.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:47.652912+0000) 2024-03-06T13:39:48.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:48.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:48.653096+0000) 2024-03-06T13:39:49.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:49.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:49.653279+0000) 2024-03-06T13:39:50.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:50.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:50.653458+0000) 2024-03-06T13:39:51.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:51.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:51.653657+0000) 2024-03-06T13:39:52.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:52.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:52.653854+0000) 2024-03-06T13:39:53.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:53.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:53.654078+0000) 2024-03-06T13:39:54.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:54.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:54.654300+0000) 2024-03-06T13:39:55.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:55.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:55.654556+0000) 2024-03-06T13:39:56.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:56.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:56.654748+0000) 2024-03-06T13:39:57.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:57.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:57.654916+0000) 2024-03-06T13:39:58.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:58.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:58.655116+0000) 2024-03-06T13:39:59.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:39:59.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:39:59.655363+0000) 2024-03-06T13:40:00.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:00.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:00.655629+0000) 2024-03-06T13:40:01.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:01.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:01.655825+0000) 2024-03-06T13:40:02.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:02.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:02.656057+0000) 2024-03-06T13:40:03.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:03.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:03.656337+0000) 2024-03-06T13:40:04.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:04.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:04.656647+0000) 2024-03-06T13:40:05.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:05.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:05.656920+0000) 2024-03-06T13:40:06.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:06.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:06.657198+0000) 2024-03-06T13:40:07.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:07.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:07.657484+0000) 2024-03-06T13:40:08.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:08.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:08.657744+0000) 2024-03-06T13:40:09.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:09.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:09.658025+0000) 2024-03-06T13:40:10.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:10.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:10.658344+0000) 2024-03-06T13:40:11.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:11.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:11.658652+0000) 2024-03-06T13:40:12.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:12.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:12.658917+0000) 2024-03-06T13:40:13.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:13.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:13.659118+0000) 2024-03-06T13:40:14.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:14.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:14.659305+0000) 2024-03-06T13:40:15.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:15.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:15.659527+0000) 2024-03-06T13:40:16.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:16.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:16.659742+0000) 2024-03-06T13:40:17.154 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:40:17.162 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:40:17.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:17.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:17.659982+0000) 2024-03-06T13:40:18.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:18.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:18.660214+0000) 2024-03-06T13:40:19.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:19.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:19.660466+0000) 2024-03-06T13:40:20.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:20.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:20.660682+0000) 2024-03-06T13:40:21.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:21.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:21.660895+0000) 2024-03-06T13:40:22.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:22.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:22.661112+0000) 2024-03-06T13:40:23.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:23.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:23.661331+0000) 2024-03-06T13:40:24.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:24.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:24.661571+0000) 2024-03-06T13:40:25.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:25.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:25.661825+0000) 2024-03-06T13:40:26.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:26.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:26.662067+0000) 2024-03-06T13:40:27.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:27.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:27.662319+0000) 2024-03-06T13:40:28.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:28.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:28.662529+0000) 2024-03-06T13:40:29.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:29.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:29.662722+0000) 2024-03-06T13:40:30.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:30.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:30.662907+0000) 2024-03-06T13:40:31.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:31.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:31.663093+0000) 2024-03-06T13:40:32.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:32.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:32.663315+0000) 2024-03-06T13:40:33.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:33.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:33.663563+0000) 2024-03-06T13:40:34.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:34.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:34.663821+0000) 2024-03-06T13:40:35.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:35.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:35.664058+0000) 2024-03-06T13:40:36.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:36.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:36.664243+0000) 2024-03-06T13:40:37.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:37.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:37.664468+0000) 2024-03-06T13:40:38.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:38.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:38.664686+0000) 2024-03-06T13:40:39.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:39.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:39.664911+0000) 2024-03-06T13:40:40.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:40.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:40.665130+0000) 2024-03-06T13:40:41.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:41.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:41.665355+0000) 2024-03-06T13:40:42.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:42.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:42.665622+0000) 2024-03-06T13:40:43.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:43.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:43.665849+0000) 2024-03-06T13:40:44.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:44.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:44.666018+0000) 2024-03-06T13:40:45.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:45.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:45.666210+0000) 2024-03-06T13:40:46.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:46.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:46.666409+0000) 2024-03-06T13:40:47.291 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:40:47.300 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:40:47.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:47.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:47.666701+0000) 2024-03-06T13:40:48.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:48.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:48.666884+0000) 2024-03-06T13:40:49.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:49.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:49.667060+0000) 2024-03-06T13:40:50.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:50.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:50.667236+0000) 2024-03-06T13:40:51.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:51.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:51.667433+0000) 2024-03-06T13:40:52.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:52.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:52.667664+0000) 2024-03-06T13:40:53.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:53.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:53.667912+0000) 2024-03-06T13:40:54.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:54.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:54.668099+0000) 2024-03-06T13:40:55.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:55.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:55.668294+0000) 2024-03-06T13:40:56.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:56.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:56.668488+0000) 2024-03-06T13:40:57.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:57.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:57.668731+0000) 2024-03-06T13:40:58.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:58.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:58.668960+0000) 2024-03-06T13:40:59.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:40:59.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:40:59.669216+0000) 2024-03-06T13:41:00.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:00.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:00.669455+0000) 2024-03-06T13:41:01.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:01.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:01.669725+0000) 2024-03-06T13:41:02.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:02.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:02.669998+0000) 2024-03-06T13:41:03.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:03.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:03.670240+0000) 2024-03-06T13:41:04.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:04.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:04.670465+0000) 2024-03-06T13:41:05.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:05.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:05.670694+0000) 2024-03-06T13:41:06.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:06.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:06.670923+0000) 2024-03-06T13:41:07.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:07.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:07.671157+0000) 2024-03-06T13:41:08.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:08.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:08.671375+0000) 2024-03-06T13:41:09.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:09.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:09.671650+0000) 2024-03-06T13:41:10.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:10.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:10.671873+0000) 2024-03-06T13:41:11.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:11.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:11.672105+0000) 2024-03-06T13:41:12.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:12.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:12.672259+0000) 2024-03-06T13:41:13.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:13.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:13.672523+0000) 2024-03-06T13:41:14.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:14.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:14.672777+0000) 2024-03-06T13:41:15.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:15.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:15.672966+0000) 2024-03-06T13:41:16.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:16.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:16.673174+0000) 2024-03-06T13:41:17.472 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:41:17.480 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:41:17.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:17.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:17.673440+0000) 2024-03-06T13:41:18.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:18.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:18.673716+0000) 2024-03-06T13:41:19.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:19.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:19.673984+0000) 2024-03-06T13:41:20.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:20.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:20.674218+0000) 2024-03-06T13:41:21.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:21.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:21.674461+0000) 2024-03-06T13:41:22.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:22.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:22.674685+0000) 2024-03-06T13:41:23.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:23.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:23.674879+0000) 2024-03-06T13:41:24.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:24.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:24.675076+0000) 2024-03-06T13:41:25.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:25.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:25.675277+0000) 2024-03-06T13:41:26.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:26.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:26.675475+0000) 2024-03-06T13:41:27.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:27.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:27.675662+0000) 2024-03-06T13:41:28.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:28.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:28.675851+0000) 2024-03-06T13:41:29.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:29.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:29.676051+0000) 2024-03-06T13:41:30.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:30.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:30.676244+0000) 2024-03-06T13:41:31.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:31.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:31.676448+0000) 2024-03-06T13:41:32.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:32.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:32.676724+0000) 2024-03-06T13:41:33.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:33.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:33.676999+0000) 2024-03-06T13:41:34.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:34.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:34.677285+0000) 2024-03-06T13:41:35.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:35.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:35.677521+0000) 2024-03-06T13:41:36.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:36.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:36.677771+0000) 2024-03-06T13:41:37.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:37.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:37.678024+0000) 2024-03-06T13:41:38.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:38.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:38.678262+0000) 2024-03-06T13:41:39.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:39.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:39.678508+0000) 2024-03-06T13:41:40.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:40.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:40.678789+0000) 2024-03-06T13:41:41.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:41.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:41.679030+0000) 2024-03-06T13:41:42.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:42.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:42.679224+0000) 2024-03-06T13:41:43.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:43.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:43.679456+0000) 2024-03-06T13:41:44.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:44.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:44.679676+0000) 2024-03-06T13:41:45.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:45.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:45.679871+0000) 2024-03-06T13:41:46.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:46.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:46.680145+0000) 2024-03-06T13:41:47.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:47.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:47.680381+0000) 2024-03-06T13:41:47.694 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:41:47.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:41:48.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:48.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:48.680629+0000) 2024-03-06T13:41:49.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:49.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:49.680821+0000) 2024-03-06T13:41:50.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:50.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:50.681008+0000) 2024-03-06T13:41:51.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:51.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:51.681208+0000) 2024-03-06T13:41:52.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:52.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:52.681410+0000) 2024-03-06T13:41:53.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:53.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:53.681670+0000) 2024-03-06T13:41:54.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:54.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:54.681861+0000) 2024-03-06T13:41:55.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:55.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:55.682045+0000) 2024-03-06T13:41:56.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:56.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:56.682320+0000) 2024-03-06T13:41:57.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:57.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:57.682651+0000) 2024-03-06T13:41:58.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:58.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:58.682919+0000) 2024-03-06T13:41:59.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:41:59.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:41:59.683169+0000) 2024-03-06T13:42:00.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:00.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:00.683370+0000) 2024-03-06T13:42:01.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:01.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:01.683627+0000) 2024-03-06T13:42:02.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:02.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:02.683809+0000) 2024-03-06T13:42:03.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:03.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:03.683999+0000) 2024-03-06T13:42:04.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:04.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:04.684221+0000) 2024-03-06T13:42:05.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:05.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:05.684443+0000) 2024-03-06T13:42:06.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:06.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:06.684681+0000) 2024-03-06T13:42:07.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:07.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:07.684877+0000) 2024-03-06T13:42:08.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:08.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:08.685065+0000) 2024-03-06T13:42:09.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:09.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:09.685263+0000) 2024-03-06T13:42:10.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:10.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:10.685471+0000) 2024-03-06T13:42:11.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:11.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:11.685719+0000) 2024-03-06T13:42:12.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:12.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:12.685908+0000) 2024-03-06T13:42:13.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:13.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:13.686140+0000) 2024-03-06T13:42:14.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:14.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:14.686329+0000) 2024-03-06T13:42:15.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:15.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:15.686567+0000) 2024-03-06T13:42:16.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:16.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:16.686774+0000) 2024-03-06T13:42:17.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:17.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:17.686987+0000) 2024-03-06T13:42:17.844 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:42:17.852 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:42:18.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:18.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:18.687203+0000) 2024-03-06T13:42:19.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:19.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:19.687386+0000) 2024-03-06T13:42:20.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:20.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:20.687629+0000) 2024-03-06T13:42:21.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:21.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:21.687881+0000) 2024-03-06T13:42:22.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:22.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:22.688079+0000) 2024-03-06T13:42:23.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:23.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:23.688274+0000) 2024-03-06T13:42:24.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:24.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:24.688445+0000) 2024-03-06T13:42:25.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:25.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:25.688646+0000) 2024-03-06T13:42:26.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:26.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:26.688861+0000) 2024-03-06T13:42:27.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:27.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:27.689118+0000) 2024-03-06T13:42:28.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:28.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:28.689345+0000) 2024-03-06T13:42:29.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:29.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:29.689562+0000) 2024-03-06T13:42:30.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:30.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:30.689767+0000) 2024-03-06T13:42:31.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:31.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:31.690028+0000) 2024-03-06T13:42:32.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:32.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:32.690232+0000) 2024-03-06T13:42:33.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:33.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:33.690440+0000) 2024-03-06T13:42:34.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:34.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:34.690740+0000) 2024-03-06T13:42:35.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:35.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:35.690940+0000) 2024-03-06T13:42:36.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:36.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:36.691140+0000) 2024-03-06T13:42:37.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:37.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:37.691297+0000) 2024-03-06T13:42:38.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:38.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:38.691466+0000) 2024-03-06T13:42:39.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:39.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:39.691643+0000) 2024-03-06T13:42:40.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:40.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:40.691821+0000) 2024-03-06T13:42:41.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:41.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:41.692012+0000) 2024-03-06T13:42:42.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:42.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:42.692186+0000) 2024-03-06T13:42:43.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:43.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:43.692367+0000) 2024-03-06T13:42:44.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:44.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:44.692629+0000) 2024-03-06T13:42:45.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:45.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:45.692832+0000) 2024-03-06T13:42:46.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:46.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:46.693036+0000) 2024-03-06T13:42:47.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:47.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:47.693270+0000) 2024-03-06T13:42:48.025 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:42:48.033 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:42:48.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:48.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:48.693472+0000) 2024-03-06T13:42:49.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:49.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:49.693704+0000) 2024-03-06T13:42:50.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:50.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:50.693900+0000) 2024-03-06T13:42:51.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:51.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:51.694097+0000) 2024-03-06T13:42:52.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:52.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:52.694303+0000) 2024-03-06T13:42:53.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:53.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:53.694503+0000) 2024-03-06T13:42:54.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:54.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:54.694681+0000) 2024-03-06T13:42:55.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:55.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:55.694877+0000) 2024-03-06T13:42:56.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:56.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:56.695061+0000) 2024-03-06T13:42:57.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:57.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:57.695292+0000) 2024-03-06T13:42:58.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:58.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:58.695513+0000) 2024-03-06T13:42:59.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:42:59.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:42:59.695739+0000) 2024-03-06T13:43:00.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:00.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:00.695917+0000) 2024-03-06T13:43:01.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:01.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:01.696107+0000) 2024-03-06T13:43:02.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:02.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:02.696309+0000) 2024-03-06T13:43:03.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:03.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:03.696572+0000) 2024-03-06T13:43:04.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:04.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:04.696800+0000) 2024-03-06T13:43:05.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:05.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:05.697022+0000) 2024-03-06T13:43:06.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:06.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:06.697214+0000) 2024-03-06T13:43:07.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:07.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:07.697435+0000) 2024-03-06T13:43:08.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:08.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:08.697646+0000) 2024-03-06T13:43:09.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:09.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:09.697871+0000) 2024-03-06T13:43:10.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:10.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:10.698053+0000) 2024-03-06T13:43:11.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:11.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:11.698284+0000) 2024-03-06T13:43:12.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:12.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:12.698505+0000) 2024-03-06T13:43:13.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:13.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:13.698741+0000) 2024-03-06T13:43:14.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:14.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:14.698946+0000) 2024-03-06T13:43:15.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:15.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:15.699131+0000) 2024-03-06T13:43:16.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:16.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:16.699356+0000) 2024-03-06T13:43:17.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:17.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:17.699591+0000) 2024-03-06T13:43:18.230 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:43:18.238 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:43:18.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:18.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:18.699835+0000) 2024-03-06T13:43:19.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:19.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:19.700068+0000) 2024-03-06T13:43:20.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:20.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:20.700291+0000) 2024-03-06T13:43:21.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:21.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:21.700474+0000) 2024-03-06T13:43:22.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:22.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:22.700694+0000) 2024-03-06T13:43:23.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:23.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:23.700924+0000) 2024-03-06T13:43:24.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:24.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:24.701113+0000) 2024-03-06T13:43:25.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:25.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:25.701342+0000) 2024-03-06T13:43:26.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:26.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:26.701594+0000) 2024-03-06T13:43:27.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:27.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:27.701895+0000) 2024-03-06T13:43:28.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:28.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:28.702117+0000) 2024-03-06T13:43:29.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:29.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:29.702318+0000) 2024-03-06T13:43:30.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:30.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:30.702501+0000) 2024-03-06T13:43:31.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:31.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:31.702695+0000) 2024-03-06T13:43:32.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:32.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:32.702880+0000) 2024-03-06T13:43:33.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:33.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:33.703112+0000) 2024-03-06T13:43:34.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:34.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:34.703290+0000) 2024-03-06T13:43:35.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:35.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:35.703488+0000) 2024-03-06T13:43:36.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:36.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:36.703733+0000) 2024-03-06T13:43:37.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:37.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:37.703983+0000) 2024-03-06T13:43:38.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:38.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:38.704206+0000) 2024-03-06T13:43:39.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:39.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:39.704445+0000) 2024-03-06T13:43:40.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:40.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:40.704681+0000) 2024-03-06T13:43:41.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:41.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:41.704913+0000) 2024-03-06T13:43:42.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:42.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:42.705138+0000) 2024-03-06T13:43:43.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:43.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:43.705366+0000) 2024-03-06T13:43:44.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:44.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:44.705634+0000) 2024-03-06T13:43:45.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:45.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:45.705830+0000) 2024-03-06T13:43:46.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:46.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:46.706052+0000) 2024-03-06T13:43:47.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:47.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:47.706288+0000) 2024-03-06T13:43:48.445 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:43:48.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:43:48.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:48.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:48.706510+0000) 2024-03-06T13:43:49.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:49.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:49.706743+0000) 2024-03-06T13:43:50.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:50.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:50.706925+0000) 2024-03-06T13:43:51.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:51.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:51.707117+0000) 2024-03-06T13:43:52.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:52.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:52.707337+0000) 2024-03-06T13:43:53.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:53.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:53.707600+0000) 2024-03-06T13:43:54.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:54.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:54.707820+0000) 2024-03-06T13:43:55.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:55.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:55.708040+0000) 2024-03-06T13:43:56.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:56.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:56.708259+0000) 2024-03-06T13:43:57.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:57.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:57.708460+0000) 2024-03-06T13:43:58.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:58.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:58.708686+0000) 2024-03-06T13:43:59.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:43:59.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:43:59.708872+0000) 2024-03-06T13:44:00.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:00.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:00.709095+0000) 2024-03-06T13:44:01.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:01.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:01.709325+0000) 2024-03-06T13:44:02.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:02.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:02.709575+0000) 2024-03-06T13:44:03.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:03.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:03.709807+0000) 2024-03-06T13:44:04.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:04.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:04.710032+0000) 2024-03-06T13:44:05.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:05.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:05.710259+0000) 2024-03-06T13:44:06.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:06.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:06.710457+0000) 2024-03-06T13:44:07.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:07.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:07.710713+0000) 2024-03-06T13:44:08.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:08.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:08.710941+0000) 2024-03-06T13:44:09.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:09.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:09.711171+0000) 2024-03-06T13:44:10.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:10.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:10.711398+0000) 2024-03-06T13:44:11.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:11.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:11.711651+0000) 2024-03-06T13:44:12.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:12.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:12.711892+0000) 2024-03-06T13:44:13.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:13.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:13.712120+0000) 2024-03-06T13:44:14.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:14.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:14.712343+0000) 2024-03-06T13:44:15.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:15.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:15.712622+0000) 2024-03-06T13:44:16.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:16.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:16.712856+0000) 2024-03-06T13:44:17.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:17.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:17.713110+0000) 2024-03-06T13:44:18.658 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:44:18.666 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:44:18.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:18.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:18.713299+0000) 2024-03-06T13:44:19.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:19.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:19.713517+0000) 2024-03-06T13:44:20.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:20.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:20.713803+0000) 2024-03-06T13:44:21.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:21.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:21.714097+0000) 2024-03-06T13:44:22.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:22.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:22.714340+0000) 2024-03-06T13:44:23.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:23.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:23.714624+0000) 2024-03-06T13:44:24.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:24.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:24.714843+0000) 2024-03-06T13:44:25.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:25.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:25.715081+0000) 2024-03-06T13:44:26.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:26.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:26.715346+0000) 2024-03-06T13:44:27.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:27.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:27.715674+0000) 2024-03-06T13:44:28.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:28.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:28.715950+0000) 2024-03-06T13:44:29.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:29.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:29.716224+0000) 2024-03-06T13:44:30.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:30.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:30.716517+0000) 2024-03-06T13:44:31.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:31.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:31.716799+0000) 2024-03-06T13:44:32.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:32.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:32.717004+0000) 2024-03-06T13:44:33.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:33.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:33.717278+0000) 2024-03-06T13:44:34.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:34.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:34.717572+0000) 2024-03-06T13:44:35.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:35.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:35.717849+0000) 2024-03-06T13:44:36.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:36.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:36.718127+0000) 2024-03-06T13:44:37.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:37.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:37.718415+0000) 2024-03-06T13:44:38.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:38.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:38.718693+0000) 2024-03-06T13:44:39.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:39.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:39.718969+0000) 2024-03-06T13:44:40.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:40.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:40.719232+0000) 2024-03-06T13:44:41.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:41.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:41.719446+0000) 2024-03-06T13:44:42.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:42.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:42.719692+0000) 2024-03-06T13:44:43.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:43.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:43.719953+0000) 2024-03-06T13:44:44.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:44.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:44.720152+0000) 2024-03-06T13:44:45.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:45.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:45.720372+0000) 2024-03-06T13:44:46.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:46.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:46.720576+0000) 2024-03-06T13:44:47.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:47.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:47.720830+0000) 2024-03-06T13:44:48.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:48.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:48.721039+0000) 2024-03-06T13:44:48.840 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:44:48.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:44:49.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:49.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:49.721294+0000) 2024-03-06T13:44:50.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:50.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:50.721528+0000) 2024-03-06T13:44:51.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:51.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:51.721787+0000) 2024-03-06T13:44:52.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:52.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:52.722018+0000) 2024-03-06T13:44:53.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:53.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:53.722233+0000) 2024-03-06T13:44:54.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:54.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:54.722477+0000) 2024-03-06T13:44:55.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:55.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:55.722685+0000) 2024-03-06T13:44:56.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:56.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:56.722961+0000) 2024-03-06T13:44:57.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:57.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:57.723189+0000) 2024-03-06T13:44:58.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:58.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:58.723433+0000) 2024-03-06T13:44:59.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:44:59.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:44:59.723699+0000) 2024-03-06T13:45:00.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:00.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:00.723977+0000) 2024-03-06T13:45:01.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:01.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:01.724246+0000) 2024-03-06T13:45:02.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:02.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:02.724580+0000) 2024-03-06T13:45:03.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:03.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:03.724901+0000) 2024-03-06T13:45:04.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:04.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:04.725165+0000) 2024-03-06T13:45:05.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:05.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:05.725431+0000) 2024-03-06T13:45:06.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:06.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:06.725702+0000) 2024-03-06T13:45:07.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:07.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:07.725915+0000) 2024-03-06T13:45:08.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:08.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:08.726127+0000) 2024-03-06T13:45:09.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:09.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:09.726337+0000) 2024-03-06T13:45:10.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:10.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:10.726608+0000) 2024-03-06T13:45:11.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:11.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:11.726870+0000) 2024-03-06T13:45:12.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:12.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:12.727146+0000) 2024-03-06T13:45:13.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:13.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:13.727348+0000) 2024-03-06T13:45:14.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:14.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:14.727635+0000) 2024-03-06T13:45:15.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:15.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:15.727842+0000) 2024-03-06T13:45:16.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:16.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:16.728055+0000) 2024-03-06T13:45:17.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:17.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:17.728311+0000) 2024-03-06T13:45:18.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:18.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:18.728625+0000) 2024-03-06T13:45:19.020 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:45:19.028 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:45:19.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:19.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:19.728902+0000) 2024-03-06T13:45:20.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:20.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:20.729168+0000) 2024-03-06T13:45:21.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:21.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:21.729378+0000) 2024-03-06T13:45:22.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:22.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:22.729637+0000) 2024-03-06T13:45:23.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:23.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:23.729822+0000) 2024-03-06T13:45:24.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:24.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:24.730093+0000) 2024-03-06T13:45:25.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:25.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:25.730289+0000) 2024-03-06T13:45:26.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:26.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:26.730507+0000) 2024-03-06T13:45:27.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:27.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:27.730778+0000) 2024-03-06T13:45:28.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:28.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:28.730997+0000) 2024-03-06T13:45:29.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:29.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:29.731222+0000) 2024-03-06T13:45:30.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:30.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:30.731449+0000) 2024-03-06T13:45:31.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:31.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:31.731689+0000) 2024-03-06T13:45:32.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:32.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:32.731881+0000) 2024-03-06T13:45:33.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:33.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:33.732114+0000) 2024-03-06T13:45:34.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:34.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:34.732293+0000) 2024-03-06T13:45:35.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:35.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:35.732479+0000) 2024-03-06T13:45:36.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:36.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:36.732685+0000) 2024-03-06T13:45:37.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:37.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:37.732894+0000) 2024-03-06T13:45:38.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:38.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:38.733087+0000) 2024-03-06T13:45:39.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:39.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:39.733346+0000) 2024-03-06T13:45:40.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:40.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:40.733621+0000) 2024-03-06T13:45:41.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:41.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:41.733857+0000) 2024-03-06T13:45:42.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:42.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:42.734133+0000) 2024-03-06T13:45:43.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:43.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:43.734346+0000) 2024-03-06T13:45:44.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:44.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:44.734611+0000) 2024-03-06T13:45:45.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:45.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:45.734812+0000) 2024-03-06T13:45:46.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:46.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:46.735033+0000) 2024-03-06T13:45:47.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:47.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:47.735271+0000) 2024-03-06T13:45:48.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:48.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:48.735461+0000) 2024-03-06T13:45:49.147 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:45:49.155 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:45:49.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:49.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:49.735670+0000) 2024-03-06T13:45:50.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:50.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:50.735892+0000) 2024-03-06T13:45:51.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:51.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:51.736072+0000) 2024-03-06T13:45:52.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:52.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:52.736288+0000) 2024-03-06T13:45:53.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:53.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:53.736515+0000) 2024-03-06T13:45:54.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:54.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:54.736819+0000) 2024-03-06T13:45:55.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:55.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:55.737096+0000) 2024-03-06T13:45:56.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:56.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:56.737323+0000) 2024-03-06T13:45:57.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:57.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:57.737713+0000) 2024-03-06T13:45:58.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:58.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:58.737910+0000) 2024-03-06T13:45:59.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:45:59.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:45:59.738134+0000) 2024-03-06T13:46:00.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:00.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:00.738348+0000) 2024-03-06T13:46:01.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:01.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:01.738627+0000) 2024-03-06T13:46:02.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:02.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:02.738826+0000) 2024-03-06T13:46:03.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:03.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:03.739040+0000) 2024-03-06T13:46:04.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:04.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:04.739271+0000) 2024-03-06T13:46:05.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:05.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:05.739493+0000) 2024-03-06T13:46:06.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:06.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:06.739710+0000) 2024-03-06T13:46:07.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:07.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:07.739955+0000) 2024-03-06T13:46:08.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:08.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:08.740192+0000) 2024-03-06T13:46:09.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:09.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:09.740433+0000) 2024-03-06T13:46:10.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:10.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:10.740676+0000) 2024-03-06T13:46:11.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:11.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:11.740865+0000) 2024-03-06T13:46:12.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:12.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:12.741108+0000) 2024-03-06T13:46:13.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:13.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:13.741350+0000) 2024-03-06T13:46:14.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:14.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:14.741627+0000) 2024-03-06T13:46:15.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:15.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:15.741867+0000) 2024-03-06T13:46:16.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:16.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:16.742100+0000) 2024-03-06T13:46:17.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:17.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:17.742327+0000) 2024-03-06T13:46:18.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:18.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:18.742598+0000) 2024-03-06T13:46:19.293 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:46:19.301 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:46:19.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:19.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:19.742850+0000) 2024-03-06T13:46:20.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:20.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:20.743084+0000) 2024-03-06T13:46:21.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:21.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:21.743322+0000) 2024-03-06T13:46:22.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:22.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:22.743523+0000) 2024-03-06T13:46:23.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:23.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:23.743816+0000) 2024-03-06T13:46:24.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:24.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:24.744049+0000) 2024-03-06T13:46:25.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:25.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:25.744278+0000) 2024-03-06T13:46:26.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:26.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:26.744497+0000) 2024-03-06T13:46:27.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:27.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:27.744756+0000) 2024-03-06T13:46:28.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:28.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:28.745023+0000) 2024-03-06T13:46:29.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:29.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:29.745301+0000) 2024-03-06T13:46:30.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:30.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:30.745497+0000) 2024-03-06T13:46:31.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:31.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:31.745704+0000) 2024-03-06T13:46:32.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:32.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:32.745929+0000) 2024-03-06T13:46:33.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:33.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:33.746208+0000) 2024-03-06T13:46:34.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:34.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:34.746482+0000) 2024-03-06T13:46:35.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:35.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:35.746759+0000) 2024-03-06T13:46:36.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:36.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:36.747037+0000) 2024-03-06T13:46:37.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:37.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:37.747294+0000) 2024-03-06T13:46:38.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:38.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:38.747511+0000) 2024-03-06T13:46:39.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:39.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:39.747781+0000) 2024-03-06T13:46:40.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:40.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:40.747998+0000) 2024-03-06T13:46:41.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:41.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:41.748222+0000) 2024-03-06T13:46:42.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:42.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:42.748455+0000) 2024-03-06T13:46:43.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:43.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:43.748694+0000) 2024-03-06T13:46:44.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:44.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:44.748920+0000) 2024-03-06T13:46:45.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:45.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:45.749193+0000) 2024-03-06T13:46:46.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:46.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:46.749369+0000) 2024-03-06T13:46:47.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:47.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:47.749645+0000) 2024-03-06T13:46:48.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:48.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:48.749870+0000) 2024-03-06T13:46:49.515 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:46:49.523 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:46:49.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:49.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:49.750109+0000) 2024-03-06T13:46:50.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:50.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:50.750332+0000) 2024-03-06T13:46:51.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:51.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:51.750592+0000) 2024-03-06T13:46:52.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:52.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:52.750804+0000) 2024-03-06T13:46:53.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:53.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:53.751042+0000) 2024-03-06T13:46:54.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:54.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:54.751262+0000) 2024-03-06T13:46:55.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:55.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:55.751571+0000) 2024-03-06T13:46:56.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:56.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:56.751776+0000) 2024-03-06T13:46:57.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:57.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:57.752029+0000) 2024-03-06T13:46:58.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:58.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:58.752254+0000) 2024-03-06T13:46:59.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:46:59.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:46:59.752483+0000) 2024-03-06T13:47:00.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:00.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:00.752713+0000) 2024-03-06T13:47:01.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:01.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:01.752909+0000) 2024-03-06T13:47:02.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:02.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:02.753103+0000) 2024-03-06T13:47:03.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:03.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:03.753289+0000) 2024-03-06T13:47:04.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:04.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:04.753473+0000) 2024-03-06T13:47:05.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:05.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:05.753677+0000) 2024-03-06T13:47:06.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:06.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:06.753917+0000) 2024-03-06T13:47:07.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:07.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:07.754153+0000) 2024-03-06T13:47:08.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:08.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:08.754370+0000) 2024-03-06T13:47:09.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:09.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:09.754632+0000) 2024-03-06T13:47:10.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:10.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:10.754823+0000) 2024-03-06T13:47:11.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:11.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:11.755015+0000) 2024-03-06T13:47:12.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:12.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:12.755240+0000) 2024-03-06T13:47:13.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:13.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:13.755435+0000) 2024-03-06T13:47:14.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:14.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:14.755674+0000) 2024-03-06T13:47:15.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:15.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:15.755885+0000) 2024-03-06T13:47:16.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:16.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:16.756076+0000) 2024-03-06T13:47:17.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:17.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:17.756332+0000) 2024-03-06T13:47:18.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:18.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:18.756590+0000) 2024-03-06T13:47:19.736 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:47:19.744 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:47:19.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:19.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:19.756819+0000) 2024-03-06T13:47:20.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:20.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:20.757074+0000) 2024-03-06T13:47:21.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:21.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:21.757354+0000) 2024-03-06T13:47:22.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:22.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:22.757681+0000) 2024-03-06T13:47:23.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:23.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:23.757961+0000) 2024-03-06T13:47:24.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:24.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:24.758245+0000) 2024-03-06T13:47:25.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:25.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:25.758521+0000) 2024-03-06T13:47:26.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:26.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:26.758820+0000) 2024-03-06T13:47:27.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:27.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:27.759035+0000) 2024-03-06T13:47:28.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:28.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:28.759265+0000) 2024-03-06T13:47:29.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:29.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:29.759500+0000) 2024-03-06T13:47:30.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:30.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:30.759740+0000) 2024-03-06T13:47:31.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:31.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:31.759979+0000) 2024-03-06T13:47:32.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:32.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:32.760235+0000) 2024-03-06T13:47:33.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:33.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:33.760431+0000) 2024-03-06T13:47:34.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:34.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:34.760654+0000) 2024-03-06T13:47:35.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:35.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:35.760938+0000) 2024-03-06T13:47:36.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:36.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:36.761196+0000) 2024-03-06T13:47:37.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:37.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:37.761495+0000) 2024-03-06T13:47:38.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:38.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:38.761812+0000) 2024-03-06T13:47:39.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:39.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:39.762009+0000) 2024-03-06T13:47:40.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:40.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:40.762239+0000) 2024-03-06T13:47:41.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:41.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:41.762479+0000) 2024-03-06T13:47:42.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:42.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:42.762713+0000) 2024-03-06T13:47:43.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:43.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:43.762920+0000) 2024-03-06T13:47:44.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:44.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:44.763158+0000) 2024-03-06T13:47:45.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:45.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:45.763390+0000) 2024-03-06T13:47:46.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:46.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:46.763677+0000) 2024-03-06T13:47:47.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:47.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:47.763882+0000) 2024-03-06T13:47:48.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:48.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:48.764108+0000) 2024-03-06T13:47:49.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:49.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:49.764355+0000) 2024-03-06T13:47:49.916 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:47:49.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:47:50.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:50.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:50.764585+0000) 2024-03-06T13:47:51.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:51.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:51.764796+0000) 2024-03-06T13:47:52.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:52.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:52.765037+0000) 2024-03-06T13:47:53.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:53.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:53.765264+0000) 2024-03-06T13:47:54.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:54.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:54.765491+0000) 2024-03-06T13:47:55.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:55.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:55.765747+0000) 2024-03-06T13:47:56.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:56.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:56.765952+0000) 2024-03-06T13:47:57.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:57.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:57.766191+0000) 2024-03-06T13:47:58.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:58.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:58.766410+0000) 2024-03-06T13:47:59.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:47:59.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:47:59.766639+0000) 2024-03-06T13:48:00.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:00.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:00.766859+0000) 2024-03-06T13:48:01.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:01.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:01.767078+0000) 2024-03-06T13:48:02.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:02.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:02.767324+0000) 2024-03-06T13:48:03.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:03.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:03.767621+0000) 2024-03-06T13:48:04.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:04.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:04.767855+0000) 2024-03-06T13:48:05.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:05.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:05.768121+0000) 2024-03-06T13:48:06.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:06.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:06.768385+0000) 2024-03-06T13:48:07.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:07.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:07.768675+0000) 2024-03-06T13:48:08.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:08.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:08.768942+0000) 2024-03-06T13:48:09.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:09.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:09.769171+0000) 2024-03-06T13:48:10.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:10.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:10.769447+0000) 2024-03-06T13:48:11.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:11.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:11.769735+0000) 2024-03-06T13:48:12.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:12.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:12.769974+0000) 2024-03-06T13:48:13.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:13.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:13.770213+0000) 2024-03-06T13:48:14.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:14.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:14.770398+0000) 2024-03-06T13:48:15.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:15.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:15.770646+0000) 2024-03-06T13:48:16.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:16.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:16.770881+0000) 2024-03-06T13:48:17.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:17.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:17.771146+0000) 2024-03-06T13:48:18.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:18.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:18.771383+0000) 2024-03-06T13:48:19.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:19.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:19.771658+0000) 2024-03-06T13:48:20.056 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:48:20.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:48:20.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:20.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:20.771870+0000) 2024-03-06T13:48:21.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:21.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:21.772125+0000) 2024-03-06T13:48:22.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:22.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:22.772367+0000) 2024-03-06T13:48:23.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:23.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:23.772652+0000) 2024-03-06T13:48:24.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:24.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:24.772859+0000) 2024-03-06T13:48:25.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:25.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:25.773117+0000) 2024-03-06T13:48:26.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:26.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:26.773319+0000) 2024-03-06T13:48:27.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:27.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:27.773591+0000) 2024-03-06T13:48:28.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:28.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:28.773866+0000) 2024-03-06T13:48:29.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:29.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:29.774066+0000) 2024-03-06T13:48:30.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:30.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:30.774289+0000) 2024-03-06T13:48:31.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:31.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:31.774512+0000) 2024-03-06T13:48:32.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:32.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:32.774764+0000) 2024-03-06T13:48:33.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:33.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:33.774990+0000) 2024-03-06T13:48:34.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:34.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:34.775269+0000) 2024-03-06T13:48:35.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:35.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:35.775581+0000) 2024-03-06T13:48:36.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:36.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:36.775849+0000) 2024-03-06T13:48:37.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:37.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:37.776103+0000) 2024-03-06T13:48:38.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:38.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:38.776318+0000) 2024-03-06T13:48:39.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:39.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:39.776572+0000) 2024-03-06T13:48:40.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:40.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:40.776798+0000) 2024-03-06T13:48:41.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:41.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:41.777005+0000) 2024-03-06T13:48:42.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:42.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:42.777200+0000) 2024-03-06T13:48:43.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:43.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:43.777443+0000) 2024-03-06T13:48:44.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:44.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:44.777711+0000) 2024-03-06T13:48:45.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:45.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:45.777910+0000) 2024-03-06T13:48:46.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:46.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:46.778143+0000) 2024-03-06T13:48:47.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:47.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:47.778388+0000) 2024-03-06T13:48:48.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:48.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:48.778643+0000) 2024-03-06T13:48:49.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:49.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:49.778883+0000) 2024-03-06T13:48:50.271 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:48:50.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:48:50.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:50.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:50.779093+0000) 2024-03-06T13:48:51.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:51.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:51.779314+0000) 2024-03-06T13:48:52.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:52.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:52.779569+0000) 2024-03-06T13:48:53.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:53.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:53.779831+0000) 2024-03-06T13:48:54.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:54.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:54.780043+0000) 2024-03-06T13:48:55.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:55.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:55.780274+0000) 2024-03-06T13:48:56.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:56.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:56.780494+0000) 2024-03-06T13:48:57.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:57.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:57.780749+0000) 2024-03-06T13:48:58.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:58.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:58.780983+0000) 2024-03-06T13:48:59.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:48:59.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:48:59.781188+0000) 2024-03-06T13:49:00.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:00.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:00.781429+0000) 2024-03-06T13:49:01.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:01.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:01.781655+0000) 2024-03-06T13:49:02.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:02.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:02.781879+0000) 2024-03-06T13:49:03.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:03.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:03.782081+0000) 2024-03-06T13:49:04.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:04.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:04.782348+0000) 2024-03-06T13:49:05.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:05.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:05.782583+0000) 2024-03-06T13:49:06.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:06.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:06.782804+0000) 2024-03-06T13:49:07.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:07.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:07.783040+0000) 2024-03-06T13:49:08.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:08.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:08.783264+0000) 2024-03-06T13:49:09.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:09.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:09.783500+0000) 2024-03-06T13:49:10.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:10.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:10.783781+0000) 2024-03-06T13:49:11.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:11.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:11.784027+0000) 2024-03-06T13:49:12.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:12.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:12.784221+0000) 2024-03-06T13:49:13.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:13.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:13.784457+0000) 2024-03-06T13:49:14.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:14.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:14.784698+0000) 2024-03-06T13:49:15.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:15.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:15.784959+0000) 2024-03-06T13:49:16.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:16.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:16.785198+0000) 2024-03-06T13:49:17.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:17.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:17.785458+0000) 2024-03-06T13:49:18.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:18.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:18.785699+0000) 2024-03-06T13:49:19.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:19.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:19.785946+0000) 2024-03-06T13:49:20.480 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:49:20.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:49:20.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:20.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:20.786183+0000) 2024-03-06T13:49:21.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:21.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:21.786421+0000) 2024-03-06T13:49:22.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:22.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:22.786682+0000) 2024-03-06T13:49:23.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:23.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:23.786873+0000) 2024-03-06T13:49:24.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:24.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:24.787096+0000) 2024-03-06T13:49:25.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:25.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:25.787323+0000) 2024-03-06T13:49:26.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:26.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:26.787575+0000) 2024-03-06T13:49:27.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:27.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:27.787807+0000) 2024-03-06T13:49:28.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:28.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:28.788015+0000) 2024-03-06T13:49:29.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:29.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:29.788255+0000) 2024-03-06T13:49:30.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:30.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:30.788464+0000) 2024-03-06T13:49:31.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:31.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:31.788712+0000) 2024-03-06T13:49:32.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:32.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:32.788904+0000) 2024-03-06T13:49:33.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:33.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:33.789142+0000) 2024-03-06T13:49:34.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:34.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:34.789364+0000) 2024-03-06T13:49:35.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:35.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:35.789621+0000) 2024-03-06T13:49:36.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:36.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:36.789816+0000) 2024-03-06T13:49:37.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:37.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:37.790051+0000) 2024-03-06T13:49:38.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:38.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:38.790235+0000) 2024-03-06T13:49:39.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:39.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:39.790473+0000) 2024-03-06T13:49:40.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:40.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:40.790684+0000) 2024-03-06T13:49:41.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:41.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:41.790889+0000) 2024-03-06T13:49:42.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:42.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:42.791079+0000) 2024-03-06T13:49:43.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:43.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:43.791305+0000) 2024-03-06T13:49:44.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:44.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:44.791525+0000) 2024-03-06T13:49:45.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:45.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:45.791778+0000) 2024-03-06T13:49:46.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:46.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:46.791959+0000) 2024-03-06T13:49:47.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:47.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:47.792190+0000) 2024-03-06T13:49:48.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:48.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:48.792396+0000) 2024-03-06T13:49:49.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:49.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:49.792662+0000) 2024-03-06T13:49:50.668 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:49:50.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:49:50.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:50.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:50.792816+0000) 2024-03-06T13:49:51.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:51.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:51.793042+0000) 2024-03-06T13:49:52.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:52.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:52.793263+0000) 2024-03-06T13:49:53.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:53.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:53.793483+0000) 2024-03-06T13:49:54.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:54.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:54.793718+0000) 2024-03-06T13:49:55.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:55.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:55.793939+0000) 2024-03-06T13:49:56.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:56.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:56.794168+0000) 2024-03-06T13:49:57.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:57.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:57.794422+0000) 2024-03-06T13:49:58.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:58.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:58.794660+0000) 2024-03-06T13:49:59.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:49:59.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:49:59.794897+0000) 2024-03-06T13:50:00.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:00.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:00.795082+0000) 2024-03-06T13:50:01.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:01.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:01.795277+0000) 2024-03-06T13:50:02.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:02.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:02.795515+0000) 2024-03-06T13:50:03.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:03.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:03.795748+0000) 2024-03-06T13:50:04.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:04.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:04.795984+0000) 2024-03-06T13:50:05.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:05.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:05.796230+0000) 2024-03-06T13:50:06.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:06.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:06.796472+0000) 2024-03-06T13:50:07.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:07.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:07.796726+0000) 2024-03-06T13:50:08.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:08.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:08.796952+0000) 2024-03-06T13:50:09.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:09.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:09.797192+0000) 2024-03-06T13:50:10.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:10.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:10.797425+0000) 2024-03-06T13:50:11.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:11.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:11.797672+0000) 2024-03-06T13:50:12.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:12.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:12.797884+0000) 2024-03-06T13:50:13.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:13.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:13.798130+0000) 2024-03-06T13:50:14.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:14.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:14.798371+0000) 2024-03-06T13:50:15.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:15.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:15.798640+0000) 2024-03-06T13:50:16.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:16.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:16.798820+0000) 2024-03-06T13:50:17.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:17.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:17.799051+0000) 2024-03-06T13:50:18.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:18.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:18.799258+0000) 2024-03-06T13:50:19.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:19.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:19.799484+0000) 2024-03-06T13:50:20.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:20.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:20.799682+0000) 2024-03-06T13:50:20.883 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:50:20.891 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:50:21.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:21.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:21.799939+0000) 2024-03-06T13:50:22.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:22.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:22.800158+0000) 2024-03-06T13:50:23.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:23.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:23.800394+0000) 2024-03-06T13:50:24.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:24.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:24.800643+0000) 2024-03-06T13:50:25.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:25.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:25.800877+0000) 2024-03-06T13:50:26.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:26.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:26.801079+0000) 2024-03-06T13:50:27.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:27.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:27.801335+0000) 2024-03-06T13:50:28.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:28.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:28.801516+0000) 2024-03-06T13:50:29.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:29.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:29.801781+0000) 2024-03-06T13:50:30.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:30.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:30.801957+0000) 2024-03-06T13:50:31.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:31.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:31.802138+0000) 2024-03-06T13:50:32.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:32.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:32.802357+0000) 2024-03-06T13:50:33.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:33.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:33.802621+0000) 2024-03-06T13:50:34.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:34.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:34.802838+0000) 2024-03-06T13:50:35.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:35.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:35.803065+0000) 2024-03-06T13:50:36.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:36.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:36.803236+0000) 2024-03-06T13:50:37.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:37.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:37.803430+0000) 2024-03-06T13:50:38.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:38.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:38.803658+0000) 2024-03-06T13:50:39.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:39.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:39.803859+0000) 2024-03-06T13:50:40.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:40.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:40.804038+0000) 2024-03-06T13:50:41.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:41.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:41.804260+0000) 2024-03-06T13:50:42.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:42.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:42.804479+0000) 2024-03-06T13:50:43.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:43.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:43.804666+0000) 2024-03-06T13:50:44.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:44.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:44.804871+0000) 2024-03-06T13:50:45.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:45.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:45.805112+0000) 2024-03-06T13:50:46.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:46.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:46.805329+0000) 2024-03-06T13:50:47.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:47.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:47.805598+0000) 2024-03-06T13:50:48.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:48.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:48.805784+0000) 2024-03-06T13:50:49.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:49.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:49.805985+0000) 2024-03-06T13:50:50.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:50.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:50.806207+0000) 2024-03-06T13:50:51.139 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:50:51.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:50:51.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:51.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:51.806399+0000) 2024-03-06T13:50:52.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:52.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:52.806657+0000) 2024-03-06T13:50:53.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:53.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:53.806858+0000) 2024-03-06T13:50:54.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:54.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:54.807082+0000) 2024-03-06T13:50:55.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:55.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:55.807403+0000) 2024-03-06T13:50:56.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:56.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:56.807659+0000) 2024-03-06T13:50:57.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:57.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:57.807890+0000) 2024-03-06T13:50:58.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:58.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:58.808130+0000) 2024-03-06T13:50:59.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:50:59.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:50:59.808357+0000) 2024-03-06T13:51:00.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:00.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:00.808619+0000) 2024-03-06T13:51:01.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:01.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:01.808899+0000) 2024-03-06T13:51:02.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:02.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:02.809092+0000) 2024-03-06T13:51:03.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:03.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:03.809287+0000) 2024-03-06T13:51:04.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:04.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:04.809465+0000) 2024-03-06T13:51:05.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:05.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:05.809694+0000) 2024-03-06T13:51:06.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:06.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:06.809844+0000) 2024-03-06T13:51:07.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:07.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:07.810073+0000) 2024-03-06T13:51:08.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:08.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:08.810293+0000) 2024-03-06T13:51:09.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:09.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:09.810521+0000) 2024-03-06T13:51:10.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:10.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:10.810773+0000) 2024-03-06T13:51:11.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:11.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:11.810962+0000) 2024-03-06T13:51:12.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:12.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:12.811139+0000) 2024-03-06T13:51:13.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:13.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:13.811369+0000) 2024-03-06T13:51:14.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:14.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:14.811633+0000) 2024-03-06T13:51:15.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:15.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:15.811860+0000) 2024-03-06T13:51:16.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:16.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:16.812084+0000) 2024-03-06T13:51:17.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:17.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:17.812318+0000) 2024-03-06T13:51:18.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:18.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:18.812562+0000) 2024-03-06T13:51:19.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:19.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:19.812798+0000) 2024-03-06T13:51:20.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:20.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:20.812989+0000) 2024-03-06T13:51:21.327 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:51:21.335 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:51:21.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:21.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:21.813181+0000) 2024-03-06T13:51:22.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:22.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:22.813412+0000) 2024-03-06T13:51:23.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:23.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:23.813642+0000) 2024-03-06T13:51:24.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:24.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:24.813823+0000) 2024-03-06T13:51:25.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:25.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:25.814045+0000) 2024-03-06T13:51:26.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:26.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:26.814232+0000) 2024-03-06T13:51:27.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:27.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:27.814484+0000) 2024-03-06T13:51:28.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:28.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:28.814699+0000) 2024-03-06T13:51:29.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:29.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:29.814898+0000) 2024-03-06T13:51:30.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:30.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:30.815130+0000) 2024-03-06T13:51:31.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:31.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:31.815355+0000) 2024-03-06T13:51:32.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:32.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:32.815628+0000) 2024-03-06T13:51:33.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:33.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:33.815851+0000) 2024-03-06T13:51:34.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:34.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:34.816052+0000) 2024-03-06T13:51:35.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:35.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:35.816278+0000) 2024-03-06T13:51:36.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:36.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:36.816503+0000) 2024-03-06T13:51:37.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:37.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:37.816787+0000) 2024-03-06T13:51:38.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:38.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:38.817022+0000) 2024-03-06T13:51:39.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:39.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:39.817252+0000) 2024-03-06T13:51:40.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:40.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:40.817498+0000) 2024-03-06T13:51:41.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:41.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:41.817793+0000) 2024-03-06T13:51:42.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:42.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:42.818006+0000) 2024-03-06T13:51:43.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:43.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:43.818206+0000) 2024-03-06T13:51:44.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:44.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:44.818420+0000) 2024-03-06T13:51:45.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:45.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:45.818667+0000) 2024-03-06T13:51:46.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:46.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:46.818900+0000) 2024-03-06T13:51:47.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:47.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:47.819136+0000) 2024-03-06T13:51:48.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:48.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:48.819397+0000) 2024-03-06T13:51:49.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:49.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:49.819706+0000) 2024-03-06T13:51:50.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:50.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:50.819897+0000) 2024-03-06T13:51:51.549 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:51:51.557 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:51:51.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:51.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:51.820114+0000) 2024-03-06T13:51:52.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:52.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:52.820324+0000) 2024-03-06T13:51:53.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:53.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:53.820621+0000) 2024-03-06T13:51:54.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:54.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:54.820869+0000) 2024-03-06T13:51:55.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:55.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:55.821109+0000) 2024-03-06T13:51:56.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:56.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:56.821297+0000) 2024-03-06T13:51:57.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:57.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:57.821571+0000) 2024-03-06T13:51:58.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:58.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:58.821829+0000) 2024-03-06T13:51:59.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:51:59.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:51:59.822047+0000) 2024-03-06T13:52:00.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:00.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:00.822257+0000) 2024-03-06T13:52:01.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:01.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:01.822493+0000) 2024-03-06T13:52:02.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:02.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:02.822746+0000) 2024-03-06T13:52:03.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:03.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:03.822995+0000) 2024-03-06T13:52:04.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:04.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:04.823237+0000) 2024-03-06T13:52:05.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:05.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:05.823479+0000) 2024-03-06T13:52:06.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:06.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:06.823698+0000) 2024-03-06T13:52:07.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:07.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:07.823878+0000) 2024-03-06T13:52:08.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:08.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:08.824070+0000) 2024-03-06T13:52:09.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:09.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:09.824303+0000) 2024-03-06T13:52:10.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:10.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:10.824559+0000) 2024-03-06T13:52:11.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:11.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:11.824780+0000) 2024-03-06T13:52:12.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:12.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:12.825007+0000) 2024-03-06T13:52:13.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:13.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:13.825238+0000) 2024-03-06T13:52:14.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:14.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:14.825458+0000) 2024-03-06T13:52:15.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:15.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:15.825692+0000) 2024-03-06T13:52:16.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:16.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:16.825924+0000) 2024-03-06T13:52:17.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:17.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:17.826175+0000) 2024-03-06T13:52:18.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:18.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:18.826413+0000) 2024-03-06T13:52:19.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:19.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:19.826649+0000) 2024-03-06T13:52:20.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:20.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:20.826881+0000) 2024-03-06T13:52:21.730 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:52:21.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:52:21.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:21.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:21.827046+0000) 2024-03-06T13:52:22.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:22.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:22.827266+0000) 2024-03-06T13:52:23.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:23.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:23.827512+0000) 2024-03-06T13:52:24.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:24.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:24.827801+0000) 2024-03-06T13:52:25.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:25.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:25.828009+0000) 2024-03-06T13:52:26.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:26.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:26.828234+0000) 2024-03-06T13:52:27.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:27.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:27.828433+0000) 2024-03-06T13:52:28.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:28.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:28.828658+0000) 2024-03-06T13:52:29.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:29.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:29.828878+0000) 2024-03-06T13:52:30.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:30.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:30.829084+0000) 2024-03-06T13:52:31.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:31.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:31.829313+0000) 2024-03-06T13:52:32.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:32.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:32.829521+0000) 2024-03-06T13:52:33.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:33.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:33.829824+0000) 2024-03-06T13:52:34.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:34.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:34.829987+0000) 2024-03-06T13:52:35.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:35.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:35.830159+0000) 2024-03-06T13:52:36.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:36.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:36.830374+0000) 2024-03-06T13:52:37.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:37.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:37.830697+0000) 2024-03-06T13:52:38.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:38.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:38.830900+0000) 2024-03-06T13:52:39.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:39.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:39.831092+0000) 2024-03-06T13:52:40.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:40.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:40.831263+0000) 2024-03-06T13:52:41.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:41.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:41.831486+0000) 2024-03-06T13:52:42.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:42.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:42.831662+0000) 2024-03-06T13:52:43.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:43.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:43.831846+0000) 2024-03-06T13:52:44.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:44.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:44.832020+0000) 2024-03-06T13:52:45.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:45.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:45.832187+0000) 2024-03-06T13:52:46.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:46.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:46.832367+0000) 2024-03-06T13:52:47.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:47.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:47.832643+0000) 2024-03-06T13:52:48.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:48.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:48.832839+0000) 2024-03-06T13:52:49.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:49.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:49.833032+0000) 2024-03-06T13:52:50.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:50.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:50.833217+0000) 2024-03-06T13:52:51.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:51.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:51.833433+0000) 2024-03-06T13:52:51.962 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:52:51.970 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:52:52.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:52.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:52.833647+0000) 2024-03-06T13:52:53.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:53.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:53.833851+0000) 2024-03-06T13:52:54.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:54.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:54.834042+0000) 2024-03-06T13:52:55.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:55.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:55.834230+0000) 2024-03-06T13:52:56.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:56.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:56.834420+0000) 2024-03-06T13:52:57.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:57.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:57.834671+0000) 2024-03-06T13:52:58.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:58.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:58.834881+0000) 2024-03-06T13:52:59.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:52:59.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:52:59.835112+0000) 2024-03-06T13:53:00.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:00.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:00.835345+0000) 2024-03-06T13:53:01.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:01.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:01.835568+0000) 2024-03-06T13:53:02.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:02.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:02.835751+0000) 2024-03-06T13:53:03.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:03.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:03.835957+0000) 2024-03-06T13:53:04.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:04.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:04.836153+0000) 2024-03-06T13:53:05.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:05.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:05.836378+0000) 2024-03-06T13:53:06.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:06.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:06.836606+0000) 2024-03-06T13:53:07.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:07.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:07.836833+0000) 2024-03-06T13:53:08.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:08.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:08.837081+0000) 2024-03-06T13:53:09.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:09.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:09.837327+0000) 2024-03-06T13:53:10.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:10.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:10.837611+0000) 2024-03-06T13:53:11.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:11.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:11.837808+0000) 2024-03-06T13:53:12.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:12.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:12.837998+0000) 2024-03-06T13:53:13.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:13.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:13.838247+0000) 2024-03-06T13:53:14.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:14.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:14.838427+0000) 2024-03-06T13:53:15.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:15.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:15.838668+0000) 2024-03-06T13:53:16.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:16.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:16.838813+0000) 2024-03-06T13:53:17.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:17.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:17.839060+0000) 2024-03-06T13:53:18.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:18.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:18.839264+0000) 2024-03-06T13:53:19.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:19.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:19.839472+0000) 2024-03-06T13:53:20.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:20.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:20.839676+0000) 2024-03-06T13:53:21.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:21.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:21.839900+0000) 2024-03-06T13:53:22.176 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:53:22.185 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:53:22.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:22.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:22.840088+0000) 2024-03-06T13:53:23.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:23.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:23.840325+0000) 2024-03-06T13:53:24.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:24.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:24.840577+0000) 2024-03-06T13:53:25.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:25.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:25.840810+0000) 2024-03-06T13:53:26.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:26.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:26.841013+0000) 2024-03-06T13:53:27.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:27.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:27.841257+0000) 2024-03-06T13:53:28.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:28.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:28.841495+0000) 2024-03-06T13:53:29.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:29.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:29.841690+0000) 2024-03-06T13:53:30.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:30.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:30.841962+0000) 2024-03-06T13:53:31.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:31.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:31.842188+0000) 2024-03-06T13:53:32.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:32.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:32.842401+0000) 2024-03-06T13:53:33.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:33.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:33.842657+0000) 2024-03-06T13:53:34.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:34.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:34.842837+0000) 2024-03-06T13:53:35.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:35.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:35.843023+0000) 2024-03-06T13:53:36.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:36.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:36.843262+0000) 2024-03-06T13:53:37.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:37.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:37.843484+0000) 2024-03-06T13:53:38.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:38.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:38.843695+0000) 2024-03-06T13:53:39.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:39.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:39.843884+0000) 2024-03-06T13:53:40.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:40.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:40.844068+0000) 2024-03-06T13:53:41.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:41.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:41.844292+0000) 2024-03-06T13:53:42.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:42.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:42.844573+0000) 2024-03-06T13:53:43.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:43.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:43.844779+0000) 2024-03-06T13:53:44.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:44.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:44.844978+0000) 2024-03-06T13:53:45.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:45.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:45.845201+0000) 2024-03-06T13:53:46.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:46.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:46.845424+0000) 2024-03-06T13:53:47.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:47.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:47.845655+0000) 2024-03-06T13:53:48.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:48.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:48.845856+0000) 2024-03-06T13:53:49.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:49.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:49.846044+0000) 2024-03-06T13:53:50.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:50.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:50.846270+0000) 2024-03-06T13:53:51.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:51.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:51.846479+0000) 2024-03-06T13:53:52.406 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:53:52.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:53:52.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:52.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:52.846697+0000) 2024-03-06T13:53:53.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:53.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:53.846909+0000) 2024-03-06T13:53:54.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:54.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:54.847107+0000) 2024-03-06T13:53:55.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:55.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:55.847322+0000) 2024-03-06T13:53:56.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:56.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:56.847557+0000) 2024-03-06T13:53:57.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:57.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:57.847718+0000) 2024-03-06T13:53:58.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:58.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:58.847970+0000) 2024-03-06T13:53:59.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:53:59.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:53:59.848194+0000) 2024-03-06T13:54:00.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:00.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:00.848420+0000) 2024-03-06T13:54:01.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:01.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:01.848638+0000) 2024-03-06T13:54:02.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:02.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:02.848860+0000) 2024-03-06T13:54:03.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:03.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:03.849103+0000) 2024-03-06T13:54:04.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:04.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:04.849285+0000) 2024-03-06T13:54:05.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:05.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:05.849459+0000) 2024-03-06T13:54:06.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:06.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:06.849701+0000) 2024-03-06T13:54:07.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:07.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:07.849950+0000) 2024-03-06T13:54:08.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:08.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:08.850178+0000) 2024-03-06T13:54:09.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:09.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:09.850406+0000) 2024-03-06T13:54:10.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:10.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:10.850655+0000) 2024-03-06T13:54:11.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:11.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:11.850829+0000) 2024-03-06T13:54:12.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:12.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:12.851015+0000) 2024-03-06T13:54:13.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:13.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:13.851269+0000) 2024-03-06T13:54:14.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:14.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:14.851464+0000) 2024-03-06T13:54:15.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:15.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:15.851697+0000) 2024-03-06T13:54:16.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:16.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:16.851920+0000) 2024-03-06T13:54:17.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:17.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:17.852156+0000) 2024-03-06T13:54:18.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:18.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:18.852391+0000) 2024-03-06T13:54:19.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:19.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:19.852663+0000) 2024-03-06T13:54:20.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:20.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:20.852888+0000) 2024-03-06T13:54:21.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:21.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:21.853114+0000) 2024-03-06T13:54:22.627 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:54:22.635 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:54:22.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:22.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:22.853292+0000) 2024-03-06T13:54:23.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:23.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:23.853518+0000) 2024-03-06T13:54:24.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:24.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:24.853736+0000) 2024-03-06T13:54:25.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:25.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:25.853919+0000) 2024-03-06T13:54:26.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:26.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:26.854103+0000) 2024-03-06T13:54:27.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:27.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:27.854331+0000) 2024-03-06T13:54:28.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:28.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:28.854582+0000) 2024-03-06T13:54:29.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:29.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:29.854782+0000) 2024-03-06T13:54:30.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:30.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:30.854966+0000) 2024-03-06T13:54:31.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:31.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:31.855175+0000) 2024-03-06T13:54:32.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:32.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:32.855388+0000) 2024-03-06T13:54:33.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:33.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:33.855641+0000) 2024-03-06T13:54:34.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:34.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:34.855829+0000) 2024-03-06T13:54:35.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:35.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:35.856017+0000) 2024-03-06T13:54:36.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:36.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:36.856241+0000) 2024-03-06T13:54:37.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:37.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:37.856483+0000) 2024-03-06T13:54:38.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:38.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:38.856691+0000) 2024-03-06T13:54:39.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:39.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:39.856846+0000) 2024-03-06T13:54:40.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:40.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:40.857041+0000) 2024-03-06T13:54:41.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:41.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:41.857244+0000) 2024-03-06T13:54:42.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:42.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:42.857458+0000) 2024-03-06T13:54:43.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:43.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:43.857688+0000) 2024-03-06T13:54:44.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:44.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:44.857879+0000) 2024-03-06T13:54:45.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:45.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:45.858069+0000) 2024-03-06T13:54:46.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:46.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:46.858294+0000) 2024-03-06T13:54:47.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:47.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:47.858525+0000) 2024-03-06T13:54:48.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:48.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:48.858747+0000) 2024-03-06T13:54:49.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:49.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:49.858939+0000) 2024-03-06T13:54:50.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:50.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:50.859118+0000) 2024-03-06T13:54:51.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:51.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:51.859337+0000) 2024-03-06T13:54:52.827 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:54:52.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:54:52.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:52.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:52.859528+0000) 2024-03-06T13:54:53.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:53.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:53.859778+0000) 2024-03-06T13:54:54.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:54.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:54.859984+0000) 2024-03-06T13:54:55.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:55.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:55.860186+0000) 2024-03-06T13:54:56.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:56.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:56.860387+0000) 2024-03-06T13:54:57.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:57.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:57.860636+0000) 2024-03-06T13:54:58.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:58.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:58.860861+0000) 2024-03-06T13:54:59.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:54:59.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:54:59.861094+0000) 2024-03-06T13:55:00.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:00.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:00.861271+0000) 2024-03-06T13:55:01.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:01.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:01.861497+0000) 2024-03-06T13:55:02.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:02.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:02.861693+0000) 2024-03-06T13:55:03.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:03.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:03.861942+0000) 2024-03-06T13:55:04.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:04.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:04.862182+0000) 2024-03-06T13:55:05.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:05.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:05.862420+0000) 2024-03-06T13:55:06.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:06.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:06.862746+0000) 2024-03-06T13:55:07.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:07.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:07.862995+0000) 2024-03-06T13:55:08.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:08.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:08.863222+0000) 2024-03-06T13:55:09.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:09.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:09.863450+0000) 2024-03-06T13:55:10.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:10.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:10.863656+0000) 2024-03-06T13:55:11.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:11.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:11.863847+0000) 2024-03-06T13:55:12.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:12.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:12.864042+0000) 2024-03-06T13:55:13.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:13.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:13.864274+0000) 2024-03-06T13:55:14.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:14.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:14.864498+0000) 2024-03-06T13:55:15.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:15.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:15.864688+0000) 2024-03-06T13:55:16.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:16.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:16.864868+0000) 2024-03-06T13:55:17.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:17.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:17.865074+0000) 2024-03-06T13:55:18.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:18.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:18.865306+0000) 2024-03-06T13:55:19.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:19.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:19.865508+0000) 2024-03-06T13:55:20.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:20.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:20.865737+0000) 2024-03-06T13:55:21.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:21.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:21.865930+0000) 2024-03-06T13:55:22.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:22.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:22.866116+0000) 2024-03-06T13:55:23.045 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:55:23.054 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:55:23.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:23.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:23.866352+0000) 2024-03-06T13:55:24.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:24.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:24.866565+0000) 2024-03-06T13:55:25.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:25.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:25.866759+0000) 2024-03-06T13:55:26.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:26.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:26.866942+0000) 2024-03-06T13:55:27.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:27.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:27.867175+0000) 2024-03-06T13:55:28.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:28.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:28.867394+0000) 2024-03-06T13:55:29.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:29.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:29.867629+0000) 2024-03-06T13:55:30.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:30.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:30.867849+0000) 2024-03-06T13:55:31.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:31.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:31.868072+0000) 2024-03-06T13:55:32.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:32.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:32.868286+0000) 2024-03-06T13:55:33.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:33.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:33.868575+0000) 2024-03-06T13:55:34.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:34.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:34.868798+0000) 2024-03-06T13:55:35.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:35.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:35.869023+0000) 2024-03-06T13:55:36.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:36.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:36.869241+0000) 2024-03-06T13:55:37.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:37.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:37.869430+0000) 2024-03-06T13:55:38.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:38.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:38.869658+0000) 2024-03-06T13:55:39.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:39.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:39.869890+0000) 2024-03-06T13:55:40.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:40.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:40.870117+0000) 2024-03-06T13:55:41.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:41.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:41.870360+0000) 2024-03-06T13:55:42.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:42.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:42.870632+0000) 2024-03-06T13:55:43.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:43.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:43.870873+0000) 2024-03-06T13:55:44.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:44.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:44.871057+0000) 2024-03-06T13:55:45.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:45.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:45.871278+0000) 2024-03-06T13:55:46.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:46.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:46.871502+0000) 2024-03-06T13:55:47.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:47.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:47.871714+0000) 2024-03-06T13:55:48.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:48.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:48.871925+0000) 2024-03-06T13:55:49.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:49.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:49.872171+0000) 2024-03-06T13:55:50.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:50.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:50.872406+0000) 2024-03-06T13:55:51.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:51.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:51.872648+0000) 2024-03-06T13:55:52.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:52.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:52.872885+0000) 2024-03-06T13:55:53.273 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:55:53.281 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:55:53.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:53.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:53.873131+0000) 2024-03-06T13:55:54.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:54.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:54.873367+0000) 2024-03-06T13:55:55.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:55.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:55.873579+0000) 2024-03-06T13:55:56.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:56.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:56.873726+0000) 2024-03-06T13:55:57.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:57.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:57.873963+0000) 2024-03-06T13:55:58.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:58.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:58.874193+0000) 2024-03-06T13:55:59.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:55:59.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:55:59.874417+0000) 2024-03-06T13:56:00.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:00.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:00.874661+0000) 2024-03-06T13:56:01.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:01.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:01.874929+0000) 2024-03-06T13:56:02.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:02.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:02.875149+0000) 2024-03-06T13:56:03.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:03.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:03.875386+0000) 2024-03-06T13:56:04.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:04.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:04.875654+0000) 2024-03-06T13:56:05.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:05.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:05.875889+0000) 2024-03-06T13:56:06.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:06.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:06.876107+0000) 2024-03-06T13:56:07.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:07.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:07.876348+0000) 2024-03-06T13:56:08.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:08.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:08.876635+0000) 2024-03-06T13:56:09.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:09.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:09.876812+0000) 2024-03-06T13:56:10.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:10.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:10.876988+0000) 2024-03-06T13:56:11.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:11.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:11.877187+0000) 2024-03-06T13:56:12.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:12.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:12.877423+0000) 2024-03-06T13:56:13.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:13.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:13.877664+0000) 2024-03-06T13:56:14.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:14.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:14.877902+0000) 2024-03-06T13:56:15.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:15.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:15.878142+0000) 2024-03-06T13:56:16.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:16.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:16.878378+0000) 2024-03-06T13:56:17.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:17.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:17.878655+0000) 2024-03-06T13:56:18.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:18.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:18.878866+0000) 2024-03-06T13:56:19.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:19.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:19.879086+0000) 2024-03-06T13:56:20.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:20.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:20.879339+0000) 2024-03-06T13:56:21.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:21.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:21.879529+0000) 2024-03-06T13:56:22.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:22.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:22.879746+0000) 2024-03-06T13:56:23.411 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:56:23.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:56:23.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:23.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:23.879914+0000) 2024-03-06T13:56:24.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:24.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:24.880156+0000) 2024-03-06T13:56:25.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:25.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:25.880393+0000) 2024-03-06T13:56:26.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:26.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:26.880709+0000) 2024-03-06T13:56:27.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:27.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:27.880957+0000) 2024-03-06T13:56:28.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:28.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:28.881183+0000) 2024-03-06T13:56:29.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:29.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:29.881410+0000) 2024-03-06T13:56:30.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:30.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:30.881630+0000) 2024-03-06T13:56:31.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:31.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:31.881777+0000) 2024-03-06T13:56:32.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:32.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:32.881926+0000) 2024-03-06T13:56:33.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:33.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:33.882119+0000) 2024-03-06T13:56:34.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:34.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:34.882346+0000) 2024-03-06T13:56:35.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:35.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:35.882620+0000) 2024-03-06T13:56:36.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:36.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:36.882808+0000) 2024-03-06T13:56:37.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:37.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:37.883048+0000) 2024-03-06T13:56:38.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:38.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:38.883231+0000) 2024-03-06T13:56:39.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:39.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:39.883456+0000) 2024-03-06T13:56:40.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:40.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:40.883683+0000) 2024-03-06T13:56:41.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:41.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:41.883914+0000) 2024-03-06T13:56:42.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:42.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:42.884102+0000) 2024-03-06T13:56:43.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:43.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:43.884323+0000) 2024-03-06T13:56:44.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:44.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:44.884522+0000) 2024-03-06T13:56:45.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:45.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:45.884748+0000) 2024-03-06T13:56:46.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:46.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:46.885023+0000) 2024-03-06T13:56:47.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:47.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:47.885374+0000) 2024-03-06T13:56:48.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:48.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:48.885684+0000) 2024-03-06T13:56:49.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:49.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:49.885957+0000) 2024-03-06T13:56:50.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:50.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:50.886218+0000) 2024-03-06T13:56:51.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:51.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:51.886505+0000) 2024-03-06T13:56:52.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:52.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:52.886811+0000) 2024-03-06T13:56:53.542 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:56:53.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:56:53.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:53.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:53.887078+0000) 2024-03-06T13:56:54.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:54.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:54.887315+0000) 2024-03-06T13:56:55.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:55.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:55.887565+0000) 2024-03-06T13:56:56.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:56.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:56.887816+0000) 2024-03-06T13:56:57.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:57.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:57.888122+0000) 2024-03-06T13:56:58.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:58.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:58.888401+0000) 2024-03-06T13:56:59.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:56:59.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:56:59.888672+0000) 2024-03-06T13:57:00.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:00.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:00.888866+0000) 2024-03-06T13:57:01.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:01.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:01.889075+0000) 2024-03-06T13:57:02.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:02.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:02.889253+0000) 2024-03-06T13:57:03.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:03.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:03.889479+0000) 2024-03-06T13:57:04.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:04.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:04.889692+0000) 2024-03-06T13:57:05.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:05.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:05.889876+0000) 2024-03-06T13:57:06.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:06.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:06.890067+0000) 2024-03-06T13:57:07.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:07.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:07.890299+0000) 2024-03-06T13:57:08.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:08.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:08.890513+0000) 2024-03-06T13:57:09.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:09.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:09.890759+0000) 2024-03-06T13:57:10.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:10.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:10.890942+0000) 2024-03-06T13:57:11.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:11.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:11.891128+0000) 2024-03-06T13:57:12.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:12.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:12.891350+0000) 2024-03-06T13:57:13.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:13.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:13.891607+0000) 2024-03-06T13:57:14.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:14.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:14.891859+0000) 2024-03-06T13:57:15.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:15.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:15.892058+0000) 2024-03-06T13:57:16.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:16.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:16.892275+0000) 2024-03-06T13:57:17.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:17.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:17.892583+0000) 2024-03-06T13:57:18.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:18.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:18.892808+0000) 2024-03-06T13:57:19.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:19.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:19.893025+0000) 2024-03-06T13:57:20.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:20.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:20.893228+0000) 2024-03-06T13:57:21.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:21.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:21.893455+0000) 2024-03-06T13:57:22.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:22.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:22.893693+0000) 2024-03-06T13:57:23.768 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:57:23.776 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:57:23.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:23.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:23.893900+0000) 2024-03-06T13:57:24.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:24.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:24.894124+0000) 2024-03-06T13:57:25.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:25.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:25.894373+0000) 2024-03-06T13:57:26.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:26.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:26.894620+0000) 2024-03-06T13:57:27.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:27.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:27.894776+0000) 2024-03-06T13:57:28.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:28.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:28.895010+0000) 2024-03-06T13:57:29.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:29.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:29.895238+0000) 2024-03-06T13:57:30.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:30.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:30.895465+0000) 2024-03-06T13:57:31.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:31.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:31.895687+0000) 2024-03-06T13:57:32.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:32.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:32.895954+0000) 2024-03-06T13:57:33.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:33.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:33.896164+0000) 2024-03-06T13:57:34.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:34.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:34.896370+0000) 2024-03-06T13:57:35.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:35.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:35.896637+0000) 2024-03-06T13:57:36.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:36.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:36.896795+0000) 2024-03-06T13:57:37.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:37.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:37.896982+0000) 2024-03-06T13:57:38.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:38.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:38.897170+0000) 2024-03-06T13:57:39.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:39.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:39.897400+0000) 2024-03-06T13:57:40.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:40.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:40.897652+0000) 2024-03-06T13:57:41.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:41.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:41.897916+0000) 2024-03-06T13:57:42.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:42.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:42.898105+0000) 2024-03-06T13:57:43.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:43.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:43.898325+0000) 2024-03-06T13:57:44.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:44.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:44.898567+0000) 2024-03-06T13:57:45.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:45.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:45.898746+0000) 2024-03-06T13:57:46.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:46.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:46.898983+0000) 2024-03-06T13:57:47.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:47.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:47.899263+0000) 2024-03-06T13:57:48.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:48.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:48.899476+0000) 2024-03-06T13:57:49.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:49.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:49.899753+0000) 2024-03-06T13:57:50.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:50.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:50.900028+0000) 2024-03-06T13:57:51.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:51.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:51.900304+0000) 2024-03-06T13:57:52.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:52.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:52.900627+0000) 2024-03-06T13:57:53.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:53.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:53.900898+0000) 2024-03-06T13:57:54.069 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:57:54.077 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:57:54.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:54.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:54.901169+0000) 2024-03-06T13:57:55.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:55.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:55.901452+0000) 2024-03-06T13:57:56.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:56.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:56.901731+0000) 2024-03-06T13:57:57.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:57.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:57.902011+0000) 2024-03-06T13:57:58.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:58.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:58.902242+0000) 2024-03-06T13:57:59.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:57:59.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:57:59.902460+0000) 2024-03-06T13:58:00.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:00.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:00.902655+0000) 2024-03-06T13:58:01.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:01.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:01.902918+0000) 2024-03-06T13:58:02.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:02.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:02.903133+0000) 2024-03-06T13:58:03.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:03.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:03.903356+0000) 2024-03-06T13:58:04.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:04.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:04.903644+0000) 2024-03-06T13:58:05.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:05.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:05.903871+0000) 2024-03-06T13:58:06.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:06.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:06.904056+0000) 2024-03-06T13:58:07.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:07.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:07.904309+0000) 2024-03-06T13:58:08.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:08.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:08.904567+0000) 2024-03-06T13:58:09.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:09.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:09.904779+0000) 2024-03-06T13:58:10.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:10.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:10.904977+0000) 2024-03-06T13:58:11.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:11.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:11.905181+0000) 2024-03-06T13:58:12.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:12.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:12.905417+0000) 2024-03-06T13:58:13.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:13.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:13.905676+0000) 2024-03-06T13:58:14.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:14.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:14.905839+0000) 2024-03-06T13:58:15.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:15.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:15.906009+0000) 2024-03-06T13:58:16.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:16.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:16.906242+0000) 2024-03-06T13:58:17.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:17.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:17.906487+0000) 2024-03-06T13:58:18.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:18.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:18.906727+0000) 2024-03-06T13:58:19.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:19.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:19.906929+0000) 2024-03-06T13:58:20.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:20.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:20.907133+0000) 2024-03-06T13:58:21.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:21.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:21.907382+0000) 2024-03-06T13:58:22.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:22.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:22.907646+0000) 2024-03-06T13:58:23.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:23.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:23.907907+0000) 2024-03-06T13:58:24.193 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:58:24.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:58:24.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:24.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:24.908122+0000) 2024-03-06T13:58:25.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:25.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:25.908354+0000) 2024-03-06T13:58:26.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:26.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:26.908612+0000) 2024-03-06T13:58:27.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:27.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:27.908835+0000) 2024-03-06T13:58:28.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:28.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:28.909086+0000) 2024-03-06T13:58:29.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:29.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:29.909339+0000) 2024-03-06T13:58:30.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:30.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:30.909615+0000) 2024-03-06T13:58:31.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:31.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:31.909816+0000) 2024-03-06T13:58:32.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:32.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:32.910016+0000) 2024-03-06T13:58:33.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:33.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:33.910251+0000) 2024-03-06T13:58:34.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:34.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:34.910498+0000) 2024-03-06T13:58:35.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:35.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:35.910722+0000) 2024-03-06T13:58:36.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:36.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:36.910920+0000) 2024-03-06T13:58:37.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:37.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:37.911136+0000) 2024-03-06T13:58:38.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:38.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:38.911330+0000) 2024-03-06T13:58:39.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:39.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:39.911613+0000) 2024-03-06T13:58:40.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:40.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:40.911797+0000) 2024-03-06T13:58:41.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:41.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:41.912014+0000) 2024-03-06T13:58:42.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:42.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:42.912236+0000) 2024-03-06T13:58:43.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:43.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:43.912469+0000) 2024-03-06T13:58:44.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:44.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:44.912696+0000) 2024-03-06T13:58:45.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:45.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:45.912935+0000) 2024-03-06T13:58:46.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:46.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:46.913145+0000) 2024-03-06T13:58:47.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:47.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:47.913390+0000) 2024-03-06T13:58:48.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:48.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:48.913635+0000) 2024-03-06T13:58:49.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:49.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:49.913854+0000) 2024-03-06T13:58:50.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:50.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:50.914062+0000) 2024-03-06T13:58:51.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:51.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:51.914303+0000) 2024-03-06T13:58:52.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:52.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:52.914507+0000) 2024-03-06T13:58:53.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:53.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:53.914742+0000) 2024-03-06T13:58:54.374 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:58:54.382 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:58:54.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:54.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:54.914941+0000) 2024-03-06T13:58:55.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:55.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:55.915121+0000) 2024-03-06T13:58:56.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:56.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:56.915342+0000) 2024-03-06T13:58:57.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:57.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:57.915614+0000) 2024-03-06T13:58:58.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:58.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:58.915831+0000) 2024-03-06T13:58:59.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:58:59.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:58:59.916026+0000) 2024-03-06T13:59:00.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:00.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:00.916243+0000) 2024-03-06T13:59:01.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:01.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:01.916470+0000) 2024-03-06T13:59:02.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:02.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:02.916693+0000) 2024-03-06T13:59:03.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:03.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:03.916846+0000) 2024-03-06T13:59:04.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:04.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:04.917089+0000) 2024-03-06T13:59:05.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:05.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:05.917325+0000) 2024-03-06T13:59:06.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:06.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:06.917568+0000) 2024-03-06T13:59:07.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:07.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:07.917803+0000) 2024-03-06T13:59:08.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:08.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:08.918019+0000) 2024-03-06T13:59:09.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:09.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:09.918274+0000) 2024-03-06T13:59:10.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:10.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:10.918447+0000) 2024-03-06T13:59:11.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:11.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:11.918672+0000) 2024-03-06T13:59:12.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:12.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:12.918860+0000) 2024-03-06T13:59:13.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:13.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:13.919050+0000) 2024-03-06T13:59:14.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:14.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:14.919264+0000) 2024-03-06T13:59:15.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:15.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:15.919502+0000) 2024-03-06T13:59:16.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:16.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:16.919699+0000) 2024-03-06T13:59:17.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:17.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:17.919925+0000) 2024-03-06T13:59:18.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:18.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:18.920149+0000) 2024-03-06T13:59:19.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:19.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:19.920387+0000) 2024-03-06T13:59:20.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:20.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:20.920649+0000) 2024-03-06T13:59:21.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:21.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:21.920867+0000) 2024-03-06T13:59:22.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:22.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:22.921109+0000) 2024-03-06T13:59:23.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:23.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:23.921338+0000) 2024-03-06T13:59:24.600 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:59:24.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:59:24.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:24.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:24.921611+0000) 2024-03-06T13:59:25.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:25.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:25.921830+0000) 2024-03-06T13:59:26.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:26.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:26.922052+0000) 2024-03-06T13:59:27.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:27.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:27.922249+0000) 2024-03-06T13:59:28.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:28.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:28.922470+0000) 2024-03-06T13:59:29.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:29.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:29.922669+0000) 2024-03-06T13:59:30.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:30.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:30.922942+0000) 2024-03-06T13:59:31.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:31.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:31.923202+0000) 2024-03-06T13:59:32.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:32.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:32.923440+0000) 2024-03-06T13:59:33.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:33.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:33.923692+0000) 2024-03-06T13:59:34.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:34.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:34.923952+0000) 2024-03-06T13:59:35.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:35.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:35.924174+0000) 2024-03-06T13:59:36.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:36.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:36.924385+0000) 2024-03-06T13:59:37.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:37.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:37.924680+0000) 2024-03-06T13:59:38.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:38.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:38.924974+0000) 2024-03-06T13:59:39.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:39.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:39.925263+0000) 2024-03-06T13:59:40.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:40.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:40.925473+0000) 2024-03-06T13:59:41.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:41.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:41.925717+0000) 2024-03-06T13:59:42.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:42.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:42.925938+0000) 2024-03-06T13:59:43.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:43.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:43.926126+0000) 2024-03-06T13:59:44.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:44.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:44.926370+0000) 2024-03-06T13:59:45.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:45.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:45.926646+0000) 2024-03-06T13:59:46.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:46.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:46.926848+0000) 2024-03-06T13:59:47.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:47.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:47.927054+0000) 2024-03-06T13:59:48.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:48.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:48.927270+0000) 2024-03-06T13:59:49.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:49.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:49.927478+0000) 2024-03-06T13:59:50.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:50.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:50.927701+0000) 2024-03-06T13:59:51.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:51.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:51.927905+0000) 2024-03-06T13:59:52.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:52.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:52.928092+0000) 2024-03-06T13:59:53.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:53.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:53.928322+0000) 2024-03-06T13:59:54.838 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:59:54.846 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T13:59:54.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:54.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:54.928516+0000) 2024-03-06T13:59:55.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:55.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:55.928734+0000) 2024-03-06T13:59:56.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:56.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:56.928933+0000) 2024-03-06T13:59:57.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:57.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:57.929126+0000) 2024-03-06T13:59:58.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:58.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:58.929318+0000) 2024-03-06T13:59:59.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T13:59:59.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T12:59:59.929595+0000) 2024-03-06T14:00:00.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:00.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:00.929834+0000) 2024-03-06T14:00:01.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:01.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:01.930036+0000) 2024-03-06T14:00:02.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:02.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:02.930271+0000) 2024-03-06T14:00:03.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:03.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:03.930509+0000) 2024-03-06T14:00:04.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:04.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:04.930751+0000) 2024-03-06T14:00:05.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:05.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:05.931043+0000) 2024-03-06T14:00:06.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:06.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:06.931274+0000) 2024-03-06T14:00:07.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:07.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:07.931497+0000) 2024-03-06T14:00:08.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:08.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:08.931724+0000) 2024-03-06T14:00:09.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:09.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:09.931931+0000) 2024-03-06T14:00:10.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:10.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:10.932144+0000) 2024-03-06T14:00:11.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:11.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:11.932378+0000) 2024-03-06T14:00:12.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:12.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:12.932628+0000) 2024-03-06T14:00:13.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:13.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:13.932819+0000) 2024-03-06T14:00:14.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:14.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:14.933008+0000) 2024-03-06T14:00:15.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:15.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:15.933226+0000) 2024-03-06T14:00:16.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:16.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:16.933491+0000) 2024-03-06T14:00:17.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:17.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:17.933773+0000) 2024-03-06T14:00:18.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:18.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:18.933983+0000) 2024-03-06T14:00:19.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:19.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:19.934231+0000) 2024-03-06T14:00:20.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:20.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:20.934451+0000) 2024-03-06T14:00:21.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:21.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:21.934730+0000) 2024-03-06T14:00:22.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:22.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:22.934957+0000) 2024-03-06T14:00:23.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:23.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:23.935192+0000) 2024-03-06T14:00:24.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:24.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:24.935413+0000) 2024-03-06T14:00:25.051 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:00:25.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:00:25.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:25.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:25.935700+0000) 2024-03-06T14:00:26.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:26.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:26.935942+0000) 2024-03-06T14:00:27.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:27.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:27.936240+0000) 2024-03-06T14:00:28.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:28.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:28.936513+0000) 2024-03-06T14:00:29.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:29.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:29.936807+0000) 2024-03-06T14:00:30.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:30.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:30.937035+0000) 2024-03-06T14:00:31.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:31.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:31.937311+0000) 2024-03-06T14:00:32.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:32.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:32.937612+0000) 2024-03-06T14:00:33.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:33.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:33.937856+0000) 2024-03-06T14:00:34.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:34.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:34.938137+0000) 2024-03-06T14:00:35.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:35.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:35.938417+0000) 2024-03-06T14:00:36.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:36.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:36.938696+0000) 2024-03-06T14:00:37.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:37.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:37.938942+0000) 2024-03-06T14:00:38.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:38.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:38.939171+0000) 2024-03-06T14:00:39.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:39.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:39.939452+0000) 2024-03-06T14:00:40.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:40.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:40.939720+0000) 2024-03-06T14:00:41.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:41.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:41.939932+0000) 2024-03-06T14:00:42.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:42.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:42.940205+0000) 2024-03-06T14:00:43.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:43.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:43.940481+0000) 2024-03-06T14:00:44.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:44.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:44.940733+0000) 2024-03-06T14:00:45.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:45.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:45.940940+0000) 2024-03-06T14:00:46.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:46.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:46.941182+0000) 2024-03-06T14:00:47.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:47.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:47.941409+0000) 2024-03-06T14:00:48.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:48.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:48.941656+0000) 2024-03-06T14:00:49.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:49.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:49.941851+0000) 2024-03-06T14:00:50.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:50.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:50.942039+0000) 2024-03-06T14:00:51.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:51.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:51.942205+0000) 2024-03-06T14:00:52.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:52.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:52.942426+0000) 2024-03-06T14:00:53.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:53.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:53.942672+0000) 2024-03-06T14:00:54.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:54.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:54.942831+0000) 2024-03-06T14:00:55.249 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:00:55.258 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:00:55.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:55.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:55.943003+0000) 2024-03-06T14:00:56.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:56.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:56.943217+0000) 2024-03-06T14:00:57.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:57.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:57.943465+0000) 2024-03-06T14:00:58.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:58.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:58.943722+0000) 2024-03-06T14:00:59.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:00:59.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:00:59.943904+0000) 2024-03-06T14:01:00.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:00.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:00.944118+0000) 2024-03-06T14:01:01.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:01.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:01.944367+0000) 2024-03-06T14:01:02.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:02.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:02.944632+0000) 2024-03-06T14:01:03.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:03.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:03.944778+0000) 2024-03-06T14:01:04.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:04.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:04.945024+0000) 2024-03-06T14:01:05.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:05.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:05.945299+0000) 2024-03-06T14:01:06.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:06.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:06.945600+0000) 2024-03-06T14:01:07.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:07.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:07.945842+0000) 2024-03-06T14:01:08.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:08.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:08.946076+0000) 2024-03-06T14:01:09.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:09.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:09.946345+0000) 2024-03-06T14:01:10.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:10.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:10.946591+0000) 2024-03-06T14:01:11.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:11.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:11.946798+0000) 2024-03-06T14:01:12.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:12.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:12.946987+0000) 2024-03-06T14:01:13.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:13.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:13.947222+0000) 2024-03-06T14:01:14.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:14.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:14.947448+0000) 2024-03-06T14:01:15.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:15.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:15.947645+0000) 2024-03-06T14:01:16.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:16.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:16.947843+0000) 2024-03-06T14:01:17.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:17.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:17.948080+0000) 2024-03-06T14:01:18.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:18.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:18.948314+0000) 2024-03-06T14:01:19.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:19.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:19.948525+0000) 2024-03-06T14:01:20.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:20.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:20.948782+0000) 2024-03-06T14:01:21.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:21.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:21.948998+0000) 2024-03-06T14:01:22.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:22.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:22.949216+0000) 2024-03-06T14:01:23.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:23.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:23.949438+0000) 2024-03-06T14:01:24.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:24.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:24.949648+0000) 2024-03-06T14:01:25.477 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:01:25.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:01:25.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:25.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:25.949887+0000) 2024-03-06T14:01:26.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:26.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:26.950107+0000) 2024-03-06T14:01:27.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:27.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:27.950346+0000) 2024-03-06T14:01:28.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:28.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:28.950611+0000) 2024-03-06T14:01:29.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:29.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:29.950839+0000) 2024-03-06T14:01:30.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:30.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:30.951019+0000) 2024-03-06T14:01:31.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:31.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:31.951247+0000) 2024-03-06T14:01:32.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:32.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:32.951468+0000) 2024-03-06T14:01:33.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:33.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:33.951728+0000) 2024-03-06T14:01:34.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:34.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:34.951996+0000) 2024-03-06T14:01:35.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:35.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:35.952212+0000) 2024-03-06T14:01:36.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:36.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:36.952485+0000) 2024-03-06T14:01:37.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:37.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:37.952768+0000) 2024-03-06T14:01:38.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:38.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:38.952995+0000) 2024-03-06T14:01:39.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:39.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:39.953233+0000) 2024-03-06T14:01:40.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:40.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:40.953429+0000) 2024-03-06T14:01:41.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:41.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:41.953668+0000) 2024-03-06T14:01:42.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:42.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:42.953848+0000) 2024-03-06T14:01:43.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:43.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:43.954115+0000) 2024-03-06T14:01:44.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:44.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:44.954379+0000) 2024-03-06T14:01:45.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:45.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:45.954638+0000) 2024-03-06T14:01:46.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:46.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:46.954846+0000) 2024-03-06T14:01:47.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:47.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:47.955046+0000) 2024-03-06T14:01:48.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:48.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:48.955261+0000) 2024-03-06T14:01:49.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:49.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:49.955570+0000) 2024-03-06T14:01:50.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:50.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:50.955773+0000) 2024-03-06T14:01:51.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:51.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:51.955962+0000) 2024-03-06T14:01:52.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:52.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:52.956162+0000) 2024-03-06T14:01:53.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:53.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:53.956370+0000) 2024-03-06T14:01:54.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:54.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:54.956637+0000) 2024-03-06T14:01:55.886 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:01:55.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:01:55.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:55.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:55.956793+0000) 2024-03-06T14:01:56.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:56.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:56.956998+0000) 2024-03-06T14:01:57.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:57.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:57.957243+0000) 2024-03-06T14:01:58.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:58.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:58.957468+0000) 2024-03-06T14:01:59.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:01:59.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:01:59.957655+0000) 2024-03-06T14:02:00.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:00.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:00.957840+0000) 2024-03-06T14:02:01.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:01.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:01.958064+0000) 2024-03-06T14:02:02.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:02.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:02.958286+0000) 2024-03-06T14:02:03.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:03.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:03.958505+0000) 2024-03-06T14:02:04.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:04.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:04.958689+0000) 2024-03-06T14:02:05.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:05.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:05.958889+0000) 2024-03-06T14:02:06.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:06.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:06.959074+0000) 2024-03-06T14:02:07.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:07.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:07.959307+0000) 2024-03-06T14:02:08.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:08.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:08.959529+0000) 2024-03-06T14:02:09.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:09.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:09.959752+0000) 2024-03-06T14:02:10.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:10.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:10.959942+0000) 2024-03-06T14:02:11.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:11.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:11.960150+0000) 2024-03-06T14:02:12.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:12.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:12.960384+0000) 2024-03-06T14:02:13.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:13.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:13.960643+0000) 2024-03-06T14:02:14.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:14.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:14.960893+0000) 2024-03-06T14:02:15.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:15.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:15.961094+0000) 2024-03-06T14:02:16.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:16.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:16.961342+0000) 2024-03-06T14:02:17.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:17.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:17.961639+0000) 2024-03-06T14:02:18.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:18.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:18.961825+0000) 2024-03-06T14:02:19.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:19.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:19.962021+0000) 2024-03-06T14:02:20.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:20.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:20.962241+0000) 2024-03-06T14:02:21.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:21.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:21.962431+0000) 2024-03-06T14:02:22.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:22.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:22.962670+0000) 2024-03-06T14:02:23.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:23.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:23.962853+0000) 2024-03-06T14:02:24.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:24.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:24.963069+0000) 2024-03-06T14:02:25.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:25.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:25.963288+0000) 2024-03-06T14:02:26.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:02:26.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:02:26.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:26.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:26.963564+0000) 2024-03-06T14:02:27.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:27.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:27.963808+0000) 2024-03-06T14:02:28.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:28.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:28.964005+0000) 2024-03-06T14:02:29.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:29.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:29.964239+0000) 2024-03-06T14:02:30.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:30.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:30.964420+0000) 2024-03-06T14:02:31.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:31.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:31.964652+0000) 2024-03-06T14:02:32.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:32.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:32.964806+0000) 2024-03-06T14:02:33.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:33.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:33.964991+0000) 2024-03-06T14:02:34.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:34.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:34.965229+0000) 2024-03-06T14:02:35.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:35.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:35.965453+0000) 2024-03-06T14:02:36.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:36.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:36.965683+0000) 2024-03-06T14:02:37.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:37.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:37.965925+0000) 2024-03-06T14:02:38.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:38.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:38.966143+0000) 2024-03-06T14:02:39.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:39.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:39.966348+0000) 2024-03-06T14:02:40.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:40.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:40.966569+0000) 2024-03-06T14:02:41.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:41.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:41.966758+0000) 2024-03-06T14:02:42.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:42.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:42.966982+0000) 2024-03-06T14:02:43.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:43.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:43.967264+0000) 2024-03-06T14:02:44.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:44.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:44.967573+0000) 2024-03-06T14:02:45.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:45.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:45.967782+0000) 2024-03-06T14:02:46.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:46.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:46.967974+0000) 2024-03-06T14:02:47.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:47.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:47.968218+0000) 2024-03-06T14:02:48.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:48.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:48.968436+0000) 2024-03-06T14:02:49.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:49.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:49.968668+0000) 2024-03-06T14:02:50.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:50.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:50.968862+0000) 2024-03-06T14:02:51.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:51.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:51.969050+0000) 2024-03-06T14:02:52.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:52.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:52.969268+0000) 2024-03-06T14:02:53.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:53.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:53.969468+0000) 2024-03-06T14:02:54.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:54.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:54.969688+0000) 2024-03-06T14:02:55.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:55.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:55.969918+0000) 2024-03-06T14:02:56.290 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:02:56.299 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:02:56.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:56.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:56.970117+0000) 2024-03-06T14:02:57.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:57.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:57.970347+0000) 2024-03-06T14:02:58.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:58.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:58.970602+0000) 2024-03-06T14:02:59.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:02:59.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:02:59.970793+0000) 2024-03-06T14:03:00.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:00.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:00.970982+0000) 2024-03-06T14:03:01.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:01.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:01.971178+0000) 2024-03-06T14:03:02.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:02.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:02.971396+0000) 2024-03-06T14:03:03.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:03.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:03.971660+0000) 2024-03-06T14:03:04.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:04.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:04.971884+0000) 2024-03-06T14:03:05.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:05.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:05.972116+0000) 2024-03-06T14:03:06.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:06.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:06.972328+0000) 2024-03-06T14:03:07.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:07.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:07.972633+0000) 2024-03-06T14:03:08.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:08.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:08.972859+0000) 2024-03-06T14:03:09.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:09.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:09.973054+0000) 2024-03-06T14:03:10.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:10.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:10.973293+0000) 2024-03-06T14:03:11.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:11.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:11.973473+0000) 2024-03-06T14:03:12.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:12.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:12.973681+0000) 2024-03-06T14:03:13.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:13.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:13.973868+0000) 2024-03-06T14:03:14.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:14.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:14.974050+0000) 2024-03-06T14:03:15.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:15.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:15.974286+0000) 2024-03-06T14:03:16.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:16.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:16.974509+0000) 2024-03-06T14:03:17.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:17.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:17.974762+0000) 2024-03-06T14:03:18.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:18.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:18.974972+0000) 2024-03-06T14:03:19.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:19.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:19.975199+0000) 2024-03-06T14:03:20.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:20.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:20.975421+0000) 2024-03-06T14:03:21.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:21.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:21.975662+0000) 2024-03-06T14:03:22.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:22.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:22.975881+0000) 2024-03-06T14:03:23.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:23.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:23.976072+0000) 2024-03-06T14:03:24.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:24.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:24.976300+0000) 2024-03-06T14:03:25.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:25.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:25.976590+0000) 2024-03-06T14:03:26.487 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:03:26.495 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:03:26.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:26.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:26.976771+0000) 2024-03-06T14:03:27.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:27.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:27.977011+0000) 2024-03-06T14:03:28.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:28.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:28.977221+0000) 2024-03-06T14:03:29.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:29.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:29.977497+0000) 2024-03-06T14:03:30.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:30.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:30.977751+0000) 2024-03-06T14:03:31.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:31.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:31.977978+0000) 2024-03-06T14:03:32.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:32.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:32.978200+0000) 2024-03-06T14:03:33.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:33.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:33.978440+0000) 2024-03-06T14:03:34.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:34.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:34.978637+0000) 2024-03-06T14:03:35.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:35.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:35.978831+0000) 2024-03-06T14:03:36.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:36.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:36.979046+0000) 2024-03-06T14:03:37.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:37.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:37.979275+0000) 2024-03-06T14:03:38.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:38.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:38.979501+0000) 2024-03-06T14:03:39.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:39.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:39.979734+0000) 2024-03-06T14:03:40.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:40.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:40.979926+0000) 2024-03-06T14:03:41.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:41.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:41.980127+0000) 2024-03-06T14:03:42.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:42.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:42.980373+0000) 2024-03-06T14:03:43.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:43.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:43.980625+0000) 2024-03-06T14:03:44.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:44.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:44.980813+0000) 2024-03-06T14:03:45.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:45.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:45.981003+0000) 2024-03-06T14:03:46.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:46.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:46.981223+0000) 2024-03-06T14:03:47.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:47.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:47.981418+0000) 2024-03-06T14:03:48.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:48.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:48.981632+0000) 2024-03-06T14:03:49.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:49.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:49.981783+0000) 2024-03-06T14:03:50.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:50.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:50.981973+0000) 2024-03-06T14:03:51.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:51.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:51.982156+0000) 2024-03-06T14:03:52.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:52.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:52.982376+0000) 2024-03-06T14:03:53.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:53.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:53.982655+0000) 2024-03-06T14:03:54.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:54.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:54.982841+0000) 2024-03-06T14:03:55.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:55.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:55.983030+0000) 2024-03-06T14:03:56.692 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:03:56.700 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:03:56.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:56.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:56.983223+0000) 2024-03-06T14:03:57.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:57.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:57.983457+0000) 2024-03-06T14:03:58.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:58.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:58.983693+0000) 2024-03-06T14:03:59.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:03:59.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:03:59.983884+0000) 2024-03-06T14:04:00.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:00.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:00.984074+0000) 2024-03-06T14:04:01.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:01.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:01.984297+0000) 2024-03-06T14:04:02.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:02.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:02.984513+0000) 2024-03-06T14:04:03.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:03.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:03.984742+0000) 2024-03-06T14:04:04.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:04.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:04.984934+0000) 2024-03-06T14:04:05.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:05.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:05.985132+0000) 2024-03-06T14:04:06.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:06.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:06.985359+0000) 2024-03-06T14:04:07.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:07.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:07.985612+0000) 2024-03-06T14:04:08.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:08.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:08.985767+0000) 2024-03-06T14:04:09.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:09.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:09.985991+0000) 2024-03-06T14:04:10.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:10.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:10.986184+0000) 2024-03-06T14:04:11.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:11.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:11.986406+0000) 2024-03-06T14:04:12.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:12.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:12.986650+0000) 2024-03-06T14:04:13.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:13.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:13.986850+0000) 2024-03-06T14:04:14.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:14.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:14.987077+0000) 2024-03-06T14:04:15.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:15.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:15.987365+0000) 2024-03-06T14:04:16.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:16.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:16.987640+0000) 2024-03-06T14:04:17.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:17.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:17.987880+0000) 2024-03-06T14:04:18.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:18.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:18.988068+0000) 2024-03-06T14:04:19.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:19.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:19.988289+0000) 2024-03-06T14:04:20.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:20.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:20.988512+0000) 2024-03-06T14:04:21.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:21.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:21.988727+0000) 2024-03-06T14:04:22.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:22.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:22.988914+0000) 2024-03-06T14:04:23.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:23.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:23.989100+0000) 2024-03-06T14:04:24.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:24.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:24.989321+0000) 2024-03-06T14:04:25.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:25.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:25.989588+0000) 2024-03-06T14:04:26.914 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:04:26.925 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:04:26.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:26.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:26.989777+0000) 2024-03-06T14:04:27.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:27.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:27.990015+0000) 2024-03-06T14:04:28.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:28.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:28.990236+0000) 2024-03-06T14:04:29.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:29.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:29.990472+0000) 2024-03-06T14:04:30.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:30.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:30.990707+0000) 2024-03-06T14:04:31.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:31.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:31.990892+0000) 2024-03-06T14:04:32.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:32.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:32.991076+0000) 2024-03-06T14:04:33.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:33.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:33.991260+0000) 2024-03-06T14:04:34.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:34.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:34.991466+0000) 2024-03-06T14:04:35.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:35.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:35.991684+0000) 2024-03-06T14:04:36.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:36.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:36.991906+0000) 2024-03-06T14:04:37.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:37.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:37.992148+0000) 2024-03-06T14:04:38.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:38.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:38.992318+0000) 2024-03-06T14:04:39.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:39.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:39.992511+0000) 2024-03-06T14:04:40.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:40.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:40.992705+0000) 2024-03-06T14:04:41.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:41.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:41.992930+0000) 2024-03-06T14:04:42.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:42.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:42.993153+0000) 2024-03-06T14:04:43.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:43.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:43.993338+0000) 2024-03-06T14:04:44.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:44.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:44.993604+0000) 2024-03-06T14:04:45.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:45.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:45.993793+0000) 2024-03-06T14:04:46.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:46.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:46.993978+0000) 2024-03-06T14:04:47.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:47.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:47.994188+0000) 2024-03-06T14:04:48.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:48.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:48.994424+0000) 2024-03-06T14:04:49.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:49.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:49.994654+0000) 2024-03-06T14:04:50.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:50.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:50.994901+0000) 2024-03-06T14:04:51.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:51.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:51.995131+0000) 2024-03-06T14:04:52.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:52.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:52.995324+0000) 2024-03-06T14:04:53.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:53.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:53.995591+0000) 2024-03-06T14:04:54.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:54.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:54.995788+0000) 2024-03-06T14:04:55.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:55.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:55.996015+0000) 2024-03-06T14:04:56.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:56.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:56.996236+0000) 2024-03-06T14:04:57.133 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:04:57.141 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:04:57.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:57.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:57.996474+0000) 2024-03-06T14:04:58.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:58.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:58.996709+0000) 2024-03-06T14:04:59.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:04:59.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:04:59.996938+0000) 2024-03-06T14:05:00.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:00.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:00.997175+0000) 2024-03-06T14:05:01.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:01.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:01.997364+0000) 2024-03-06T14:05:02.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:02.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:02.997583+0000) 2024-03-06T14:05:03.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:03.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:03.997771+0000) 2024-03-06T14:05:04.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:04.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:04.997956+0000) 2024-03-06T14:05:05.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:05.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:05.998197+0000) 2024-03-06T14:05:06.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:06.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:06.998416+0000) 2024-03-06T14:05:07.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:07.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:07.998665+0000) 2024-03-06T14:05:08.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:08.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:08.998848+0000) 2024-03-06T14:05:09.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:09.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:09.999098+0000) 2024-03-06T14:05:10.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:10.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:10.999333+0000) 2024-03-06T14:05:11.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:11.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:11.999579+0000) 2024-03-06T14:05:12.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:12.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:12.999796+0000) 2024-03-06T14:05:13.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:13.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:13.999976+0000) 2024-03-06T14:05:14.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:14.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:15.000178+0000) 2024-03-06T14:05:15.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:15.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:16.000429+0000) 2024-03-06T14:05:16.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:16.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:17.000673+0000) 2024-03-06T14:05:17.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:17.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:18.000872+0000) 2024-03-06T14:05:18.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:18.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:19.001058+0000) 2024-03-06T14:05:19.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:19.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:20.001292+0000) 2024-03-06T14:05:21.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:21.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:21.001558+0000) 2024-03-06T14:05:22.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:22.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:22.001753+0000) 2024-03-06T14:05:23.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:23.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:23.001938+0000) 2024-03-06T14:05:24.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:24.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:24.002128+0000) 2024-03-06T14:05:25.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:25.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:25.002356+0000) 2024-03-06T14:05:26.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:26.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:26.002647+0000) 2024-03-06T14:05:27.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:27.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:27.002835+0000) 2024-03-06T14:05:27.346 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:05:27.354 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:05:28.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:28.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:28.003074+0000) 2024-03-06T14:05:29.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:29.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:29.003292+0000) 2024-03-06T14:05:30.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:30.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:30.003514+0000) 2024-03-06T14:05:31.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:31.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:31.003744+0000) 2024-03-06T14:05:32.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:32.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:32.003938+0000) 2024-03-06T14:05:33.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:33.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:33.004159+0000) 2024-03-06T14:05:34.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:34.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:34.004390+0000) 2024-03-06T14:05:35.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:35.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:35.004605+0000) 2024-03-06T14:05:36.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:36.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:36.004840+0000) 2024-03-06T14:05:37.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:37.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:37.005066+0000) 2024-03-06T14:05:38.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:38.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:38.005306+0000) 2024-03-06T14:05:39.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:39.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:39.005521+0000) 2024-03-06T14:05:40.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:40.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:40.005748+0000) 2024-03-06T14:05:41.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:41.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:41.005974+0000) 2024-03-06T14:05:42.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:42.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:42.006209+0000) 2024-03-06T14:05:43.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:43.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:43.006427+0000) 2024-03-06T14:05:44.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:44.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:44.006657+0000) 2024-03-06T14:05:45.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:45.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:45.006881+0000) 2024-03-06T14:05:46.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:46.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:46.007104+0000) 2024-03-06T14:05:47.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:47.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:47.007328+0000) 2024-03-06T14:05:48.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:48.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:48.007644+0000) 2024-03-06T14:05:49.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:49.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:49.007873+0000) 2024-03-06T14:05:50.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:50.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:50.008064+0000) 2024-03-06T14:05:51.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:51.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:51.008283+0000) 2024-03-06T14:05:52.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:52.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:52.008515+0000) 2024-03-06T14:05:53.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:53.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:53.008733+0000) 2024-03-06T14:05:54.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:54.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:54.008922+0000) 2024-03-06T14:05:55.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:55.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:55.009109+0000) 2024-03-06T14:05:56.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:56.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:56.009295+0000) 2024-03-06T14:05:57.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:57.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:57.009512+0000) 2024-03-06T14:05:57.567 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:05:57.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:05:58.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:58.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:58.009825+0000) 2024-03-06T14:05:59.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:05:59.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:05:59.010008+0000) 2024-03-06T14:06:00.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:00.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:00.010243+0000) 2024-03-06T14:06:01.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:01.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:01.010484+0000) 2024-03-06T14:06:02.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:02.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:02.010707+0000) 2024-03-06T14:06:03.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:03.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:03.010893+0000) 2024-03-06T14:06:04.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:04.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:04.011078+0000) 2024-03-06T14:06:05.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:05.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:05.011296+0000) 2024-03-06T14:06:06.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:06.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:06.011521+0000) 2024-03-06T14:06:07.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:07.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:07.011755+0000) 2024-03-06T14:06:08.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:08.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:08.011970+0000) 2024-03-06T14:06:09.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:09.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:09.012207+0000) 2024-03-06T14:06:10.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:10.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:10.012390+0000) 2024-03-06T14:06:11.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:11.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:11.012625+0000) 2024-03-06T14:06:12.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:12.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:12.012870+0000) 2024-03-06T14:06:13.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:13.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:13.013056+0000) 2024-03-06T14:06:14.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:14.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:14.013286+0000) 2024-03-06T14:06:15.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:15.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:15.013503+0000) 2024-03-06T14:06:16.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:16.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:16.013745+0000) 2024-03-06T14:06:17.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:17.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:17.013979+0000) 2024-03-06T14:06:18.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:18.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:18.014212+0000) 2024-03-06T14:06:19.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:19.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:19.014411+0000) 2024-03-06T14:06:20.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:20.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:20.014644+0000) 2024-03-06T14:06:21.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:21.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:21.014833+0000) 2024-03-06T14:06:22.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:22.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:22.015061+0000) 2024-03-06T14:06:23.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:23.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:23.015280+0000) 2024-03-06T14:06:24.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:24.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:24.015498+0000) 2024-03-06T14:06:25.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:25.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:25.015704+0000) 2024-03-06T14:06:26.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:26.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:26.015945+0000) 2024-03-06T14:06:27.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:27.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:27.016165+0000) 2024-03-06T14:06:27.780 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:06:27.788 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:06:28.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:28.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:28.016309+0000) 2024-03-06T14:06:29.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:29.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:29.016520+0000) 2024-03-06T14:06:30.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:30.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:30.016747+0000) 2024-03-06T14:06:31.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:31.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:31.016930+0000) 2024-03-06T14:06:32.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:32.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:32.017271+0000) 2024-03-06T14:06:33.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:33.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:33.017503+0000) 2024-03-06T14:06:34.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:34.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:34.017734+0000) 2024-03-06T14:06:35.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:35.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:35.017962+0000) 2024-03-06T14:06:36.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:36.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:36.018193+0000) 2024-03-06T14:06:37.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:37.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:37.018386+0000) 2024-03-06T14:06:38.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:38.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:38.018641+0000) 2024-03-06T14:06:39.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:39.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:39.018861+0000) 2024-03-06T14:06:40.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:40.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:40.019088+0000) 2024-03-06T14:06:41.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:41.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:41.019286+0000) 2024-03-06T14:06:42.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:42.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:42.019566+0000) 2024-03-06T14:06:43.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:43.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:43.019789+0000) 2024-03-06T14:06:44.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:44.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:44.020022+0000) 2024-03-06T14:06:45.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:45.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:45.020248+0000) 2024-03-06T14:06:46.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:46.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:46.020484+0000) 2024-03-06T14:06:47.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:47.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:47.020714+0000) 2024-03-06T14:06:48.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:48.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:48.020876+0000) 2024-03-06T14:06:49.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:49.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:49.021046+0000) 2024-03-06T14:06:50.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:50.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:50.021271+0000) 2024-03-06T14:06:51.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:51.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:51.021563+0000) 2024-03-06T14:06:52.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:52.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:52.021759+0000) 2024-03-06T14:06:53.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:53.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:53.021988+0000) 2024-03-06T14:06:54.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:54.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:54.022210+0000) 2024-03-06T14:06:55.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:55.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:55.022428+0000) 2024-03-06T14:06:56.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:56.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:56.022685+0000) 2024-03-06T14:06:57.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:57.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:57.022938+0000) 2024-03-06T14:06:57.985 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:06:57.993 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:06:58.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:58.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:58.023124+0000) 2024-03-06T14:06:59.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:06:59.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:06:59.023344+0000) 2024-03-06T14:07:00.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:00.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:00.023584+0000) 2024-03-06T14:07:01.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:01.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:01.023771+0000) 2024-03-06T14:07:02.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:02.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:02.023959+0000) 2024-03-06T14:07:03.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:03.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:03.024181+0000) 2024-03-06T14:07:04.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:04.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:04.024415+0000) 2024-03-06T14:07:05.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:05.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:05.024641+0000) 2024-03-06T14:07:06.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:06.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:06.024881+0000) 2024-03-06T14:07:07.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:07.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:07.025104+0000) 2024-03-06T14:07:08.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:08.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:08.025370+0000) 2024-03-06T14:07:09.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:09.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:09.025582+0000) 2024-03-06T14:07:10.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:10.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:10.025783+0000) 2024-03-06T14:07:11.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:11.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:11.025987+0000) 2024-03-06T14:07:12.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:12.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:12.026230+0000) 2024-03-06T14:07:13.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:13.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:13.026496+0000) 2024-03-06T14:07:14.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:14.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:14.026809+0000) 2024-03-06T14:07:15.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:15.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:15.026999+0000) 2024-03-06T14:07:16.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:16.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:16.027225+0000) 2024-03-06T14:07:17.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:17.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:17.027452+0000) 2024-03-06T14:07:18.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:18.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:18.027650+0000) 2024-03-06T14:07:19.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:19.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:19.027871+0000) 2024-03-06T14:07:20.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:20.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:20.028101+0000) 2024-03-06T14:07:21.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:21.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:21.028322+0000) 2024-03-06T14:07:22.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:22.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:22.028590+0000) 2024-03-06T14:07:23.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:23.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:23.028779+0000) 2024-03-06T14:07:24.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:24.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:24.028987+0000) 2024-03-06T14:07:25.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:25.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:25.029257+0000) 2024-03-06T14:07:26.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:26.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:26.029450+0000) 2024-03-06T14:07:27.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:27.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:27.029679+0000) 2024-03-06T14:07:28.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:28.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:28.029923+0000) 2024-03-06T14:07:28.189 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:07:28.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:07:29.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:29.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:29.030148+0000) 2024-03-06T14:07:30.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:30.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:30.030371+0000) 2024-03-06T14:07:31.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:31.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:31.030636+0000) 2024-03-06T14:07:32.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:32.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:32.030835+0000) 2024-03-06T14:07:33.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:33.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:33.031019+0000) 2024-03-06T14:07:34.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:34.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:34.031293+0000) 2024-03-06T14:07:35.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:35.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:35.031483+0000) 2024-03-06T14:07:36.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:36.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:36.031697+0000) 2024-03-06T14:07:37.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:37.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:37.031926+0000) 2024-03-06T14:07:38.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:38.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:38.032169+0000) 2024-03-06T14:07:39.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:39.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:39.032429+0000) 2024-03-06T14:07:40.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:40.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:40.032724+0000) 2024-03-06T14:07:41.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:41.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:41.032925+0000) 2024-03-06T14:07:42.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:42.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:42.033108+0000) 2024-03-06T14:07:43.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:43.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:43.033301+0000) 2024-03-06T14:07:44.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:44.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:44.033560+0000) 2024-03-06T14:07:45.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:45.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:45.033748+0000) 2024-03-06T14:07:46.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:46.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:46.033939+0000) 2024-03-06T14:07:47.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:47.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:47.034162+0000) 2024-03-06T14:07:48.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:48.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:48.034394+0000) 2024-03-06T14:07:49.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:49.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:49.034644+0000) 2024-03-06T14:07:50.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:50.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:50.034860+0000) 2024-03-06T14:07:51.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:51.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:51.035061+0000) 2024-03-06T14:07:52.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:52.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:52.035287+0000) 2024-03-06T14:07:53.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:53.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:53.035503+0000) 2024-03-06T14:07:54.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:54.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:54.035656+0000) 2024-03-06T14:07:55.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:55.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:55.035902+0000) 2024-03-06T14:07:56.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:56.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:56.036093+0000) 2024-03-06T14:07:57.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:57.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:57.036357+0000) 2024-03-06T14:07:58.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:58.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:58.036680+0000) 2024-03-06T14:07:58.411 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:07:58.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:07:59.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:07:59.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:07:59.036920+0000) 2024-03-06T14:08:00.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:00.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:00.037115+0000) 2024-03-06T14:08:01.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:01.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:01.037384+0000) 2024-03-06T14:08:02.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:02.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:02.037670+0000) 2024-03-06T14:08:03.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:03.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:03.037891+0000) 2024-03-06T14:08:04.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:04.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:04.038120+0000) 2024-03-06T14:08:05.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:05.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:05.038410+0000) 2024-03-06T14:08:06.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:06.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:06.038636+0000) 2024-03-06T14:08:07.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:07.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:07.038817+0000) 2024-03-06T14:08:08.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:08.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:08.039063+0000) 2024-03-06T14:08:09.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:09.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:09.039324+0000) 2024-03-06T14:08:10.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:10.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:10.039633+0000) 2024-03-06T14:08:11.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:11.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:11.039873+0000) 2024-03-06T14:08:12.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:12.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:12.040061+0000) 2024-03-06T14:08:13.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:13.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:13.040311+0000) 2024-03-06T14:08:14.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:14.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:14.040566+0000) 2024-03-06T14:08:15.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:15.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:15.040781+0000) 2024-03-06T14:08:16.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:16.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:16.041019+0000) 2024-03-06T14:08:17.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:17.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:17.041298+0000) 2024-03-06T14:08:18.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:18.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:18.041613+0000) 2024-03-06T14:08:19.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:19.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:19.041847+0000) 2024-03-06T14:08:20.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:20.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:20.042091+0000) 2024-03-06T14:08:21.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:21.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:21.042323+0000) 2024-03-06T14:08:22.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:22.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:22.042593+0000) 2024-03-06T14:08:23.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:23.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:23.042810+0000) 2024-03-06T14:08:24.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:24.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:24.043018+0000) 2024-03-06T14:08:25.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:25.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:25.043245+0000) 2024-03-06T14:08:26.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:26.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:26.043462+0000) 2024-03-06T14:08:27.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:27.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:27.043742+0000) 2024-03-06T14:08:28.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:28.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:28.043941+0000) 2024-03-06T14:08:28.641 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:08:28.649 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:08:29.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:29.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:29.044170+0000) 2024-03-06T14:08:30.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:30.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:30.044448+0000) 2024-03-06T14:08:31.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:31.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:31.044686+0000) 2024-03-06T14:08:32.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:32.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:32.044879+0000) 2024-03-06T14:08:33.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:33.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:33.045141+0000) 2024-03-06T14:08:34.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:34.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:34.045417+0000) 2024-03-06T14:08:35.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:35.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:35.045696+0000) 2024-03-06T14:08:36.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:36.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:36.045903+0000) 2024-03-06T14:08:37.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:37.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:37.046098+0000) 2024-03-06T14:08:38.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:38.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:38.046332+0000) 2024-03-06T14:08:39.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:39.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:39.046587+0000) 2024-03-06T14:08:40.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:40.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:40.046767+0000) 2024-03-06T14:08:41.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:41.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:41.047007+0000) 2024-03-06T14:08:42.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:42.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:42.047252+0000) 2024-03-06T14:08:43.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:43.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:43.047494+0000) 2024-03-06T14:08:44.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:44.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:44.047715+0000) 2024-03-06T14:08:45.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:45.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:45.047955+0000) 2024-03-06T14:08:46.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:46.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:46.048190+0000) 2024-03-06T14:08:47.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:47.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:47.048411+0000) 2024-03-06T14:08:48.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:48.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:48.048623+0000) 2024-03-06T14:08:49.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:49.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:49.048817+0000) 2024-03-06T14:08:50.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:50.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:50.049005+0000) 2024-03-06T14:08:51.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:51.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:51.049223+0000) 2024-03-06T14:08:52.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:52.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:52.049446+0000) 2024-03-06T14:08:53.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:53.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:53.049659+0000) 2024-03-06T14:08:54.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:54.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:54.049922+0000) 2024-03-06T14:08:55.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:55.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:55.050194+0000) 2024-03-06T14:08:56.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:56.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:56.050480+0000) 2024-03-06T14:08:57.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:57.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:57.050693+0000) 2024-03-06T14:08:58.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:58.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:58.050891+0000) 2024-03-06T14:08:58.788 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:08:58.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:08:59.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:08:59.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:08:59.051087+0000) 2024-03-06T14:09:00.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:00.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:00.051314+0000) 2024-03-06T14:09:01.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:01.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:01.051528+0000) 2024-03-06T14:09:02.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:02.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:02.051762+0000) 2024-03-06T14:09:03.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:03.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:03.051968+0000) 2024-03-06T14:09:04.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:04.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:04.052211+0000) 2024-03-06T14:09:05.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:05.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:05.052473+0000) 2024-03-06T14:09:06.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:06.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:06.052685+0000) 2024-03-06T14:09:07.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:07.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:07.052883+0000) 2024-03-06T14:09:08.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:08.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:08.053082+0000) 2024-03-06T14:09:09.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:09.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:09.053270+0000) 2024-03-06T14:09:10.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:10.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:10.053490+0000) 2024-03-06T14:09:11.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:11.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:11.053685+0000) 2024-03-06T14:09:12.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:12.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:12.053896+0000) 2024-03-06T14:09:13.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:13.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:13.054113+0000) 2024-03-06T14:09:14.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:14.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:14.054324+0000) 2024-03-06T14:09:15.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:15.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:15.054658+0000) 2024-03-06T14:09:16.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:16.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:16.054849+0000) 2024-03-06T14:09:17.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:17.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:17.055040+0000) 2024-03-06T14:09:18.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:18.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:18.055276+0000) 2024-03-06T14:09:19.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:19.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:19.055493+0000) 2024-03-06T14:09:20.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:20.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:20.055714+0000) 2024-03-06T14:09:21.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:21.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:21.055874+0000) 2024-03-06T14:09:22.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:22.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:22.056104+0000) 2024-03-06T14:09:23.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:23.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:23.056295+0000) 2024-03-06T14:09:24.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:24.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:24.056526+0000) 2024-03-06T14:09:25.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:25.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:25.056740+0000) 2024-03-06T14:09:26.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:26.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:26.056929+0000) 2024-03-06T14:09:27.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:27.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:27.057133+0000) 2024-03-06T14:09:28.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:28.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:28.057361+0000) 2024-03-06T14:09:29.008 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:09:29.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:09:29.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:29.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:29.057519+0000) 2024-03-06T14:09:30.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:30.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:30.057684+0000) 2024-03-06T14:09:31.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:31.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:31.057874+0000) 2024-03-06T14:09:32.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:32.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:32.058062+0000) 2024-03-06T14:09:33.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:33.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:33.058281+0000) 2024-03-06T14:09:34.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:34.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:34.058506+0000) 2024-03-06T14:09:35.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:35.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:35.058677+0000) 2024-03-06T14:09:36.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:36.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:36.058866+0000) 2024-03-06T14:09:37.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:37.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:37.059055+0000) 2024-03-06T14:09:38.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:38.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:38.059295+0000) 2024-03-06T14:09:39.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:39.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:39.059613+0000) 2024-03-06T14:09:40.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:40.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:40.059898+0000) 2024-03-06T14:09:41.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:41.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:41.060176+0000) 2024-03-06T14:09:42.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:42.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:42.060383+0000) 2024-03-06T14:09:43.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:43.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:43.060628+0000) 2024-03-06T14:09:44.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:44.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:44.060814+0000) 2024-03-06T14:09:45.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:45.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:45.060999+0000) 2024-03-06T14:09:46.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:46.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:46.061217+0000) 2024-03-06T14:09:47.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:47.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:47.061487+0000) 2024-03-06T14:09:48.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:48.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:48.061812+0000) 2024-03-06T14:09:49.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:49.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:49.062054+0000) 2024-03-06T14:09:50.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:50.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:50.062297+0000) 2024-03-06T14:09:51.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:51.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:51.062478+0000) 2024-03-06T14:09:52.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:52.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:52.062703+0000) 2024-03-06T14:09:53.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:53.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:53.062896+0000) 2024-03-06T14:09:54.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:54.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:54.063096+0000) 2024-03-06T14:09:55.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:55.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:55.063278+0000) 2024-03-06T14:09:56.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:56.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:56.063500+0000) 2024-03-06T14:09:57.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:57.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:57.063689+0000) 2024-03-06T14:09:58.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:58.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:58.063870+0000) 2024-03-06T14:09:59.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:09:59.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:09:59.064060+0000) 2024-03-06T14:09:59.205 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:09:59.212 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:10:00.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:00.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:00.064248+0000) 2024-03-06T14:10:01.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:01.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:01.064477+0000) 2024-03-06T14:10:02.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:02.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:02.064691+0000) 2024-03-06T14:10:03.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:03.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:03.064883+0000) 2024-03-06T14:10:04.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:04.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:04.065076+0000) 2024-03-06T14:10:05.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:05.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:05.065315+0000) 2024-03-06T14:10:06.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:06.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:06.065557+0000) 2024-03-06T14:10:07.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:07.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:07.065770+0000) 2024-03-06T14:10:08.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:08.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:08.066048+0000) 2024-03-06T14:10:09.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:09.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:09.066268+0000) 2024-03-06T14:10:10.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:10.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:10.066448+0000) 2024-03-06T14:10:11.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:11.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:11.066710+0000) 2024-03-06T14:10:12.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:12.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:12.066972+0000) 2024-03-06T14:10:13.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:13.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:13.067168+0000) 2024-03-06T14:10:14.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:14.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:14.067394+0000) 2024-03-06T14:10:15.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:15.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:15.067650+0000) 2024-03-06T14:10:16.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:16.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:16.067834+0000) 2024-03-06T14:10:17.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:17.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:17.068026+0000) 2024-03-06T14:10:18.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:18.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:18.068254+0000) 2024-03-06T14:10:19.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:19.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:19.068465+0000) 2024-03-06T14:10:20.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:20.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:20.068740+0000) 2024-03-06T14:10:21.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:21.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:21.069011+0000) 2024-03-06T14:10:22.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:22.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:22.069287+0000) 2024-03-06T14:10:23.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:23.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:23.069613+0000) 2024-03-06T14:10:24.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:24.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:24.069856+0000) 2024-03-06T14:10:25.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:25.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:25.070140+0000) 2024-03-06T14:10:26.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:26.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:26.070417+0000) 2024-03-06T14:10:27.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:27.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:27.070679+0000) 2024-03-06T14:10:28.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:28.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:28.070935+0000) 2024-03-06T14:10:29.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:29.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:29.071200+0000) 2024-03-06T14:10:29.422 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:10:29.430 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:10:30.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:30.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:30.071473+0000) 2024-03-06T14:10:31.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:31.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:31.071749+0000) 2024-03-06T14:10:32.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:32.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:32.072022+0000) 2024-03-06T14:10:33.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:33.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:33.072291+0000) 2024-03-06T14:10:34.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:34.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:34.072518+0000) 2024-03-06T14:10:35.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:35.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:35.072772+0000) 2024-03-06T14:10:36.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:36.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:36.072982+0000) 2024-03-06T14:10:37.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:37.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:37.073183+0000) 2024-03-06T14:10:38.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:38.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:38.073418+0000) 2024-03-06T14:10:39.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:39.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:39.073651+0000) 2024-03-06T14:10:40.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:40.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:40.073848+0000) 2024-03-06T14:10:41.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:41.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:41.074025+0000) 2024-03-06T14:10:42.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:42.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:42.074255+0000) 2024-03-06T14:10:43.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:43.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:43.074476+0000) 2024-03-06T14:10:44.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:44.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:44.074681+0000) 2024-03-06T14:10:45.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:45.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:45.074880+0000) 2024-03-06T14:10:46.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:46.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:46.075097+0000) 2024-03-06T14:10:47.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:47.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:47.075334+0000) 2024-03-06T14:10:48.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:48.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:48.075619+0000) 2024-03-06T14:10:49.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:49.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:49.075803+0000) 2024-03-06T14:10:50.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:50.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:50.075992+0000) 2024-03-06T14:10:51.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:51.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:51.076210+0000) 2024-03-06T14:10:52.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:52.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:52.076447+0000) 2024-03-06T14:10:53.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:53.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:53.076694+0000) 2024-03-06T14:10:54.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:54.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:54.076845+0000) 2024-03-06T14:10:55.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:55.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:55.077022+0000) 2024-03-06T14:10:56.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:56.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:56.077193+0000) 2024-03-06T14:10:57.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:57.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:57.077438+0000) 2024-03-06T14:10:58.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:58.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:58.077669+0000) 2024-03-06T14:10:59.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:10:59.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:10:59.077870+0000) 2024-03-06T14:10:59.658 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:10:59.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:11:00.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:00.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:00.078104+0000) 2024-03-06T14:11:01.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:01.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:01.078338+0000) 2024-03-06T14:11:02.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:02.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:02.078679+0000) 2024-03-06T14:11:03.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:03.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:03.078952+0000) 2024-03-06T14:11:04.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:04.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:04.079231+0000) 2024-03-06T14:11:05.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:05.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:05.079513+0000) 2024-03-06T14:11:06.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:06.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:06.079833+0000) 2024-03-06T14:11:07.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:07.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:07.080120+0000) 2024-03-06T14:11:08.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:08.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:08.080401+0000) 2024-03-06T14:11:09.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:09.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:09.080690+0000) 2024-03-06T14:11:10.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:10.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:10.080936+0000) 2024-03-06T14:11:11.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:11.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:11.081216+0000) 2024-03-06T14:11:12.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:12.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:12.081493+0000) 2024-03-06T14:11:13.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:13.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:13.081781+0000) 2024-03-06T14:11:14.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:14.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:14.082017+0000) 2024-03-06T14:11:15.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:15.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:15.082207+0000) 2024-03-06T14:11:16.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:16.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:16.082427+0000) 2024-03-06T14:11:17.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:17.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:17.082661+0000) 2024-03-06T14:11:18.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:18.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:18.082925+0000) 2024-03-06T14:11:19.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:19.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:19.083189+0000) 2024-03-06T14:11:20.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:20.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:20.083467+0000) 2024-03-06T14:11:21.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:21.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:21.083749+0000) 2024-03-06T14:11:22.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:22.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:22.084032+0000) 2024-03-06T14:11:23.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:23.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:23.084301+0000) 2024-03-06T14:11:24.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:24.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:24.084614+0000) 2024-03-06T14:11:25.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:25.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:25.084862+0000) 2024-03-06T14:11:26.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:26.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:26.085100+0000) 2024-03-06T14:11:27.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:27.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:27.085364+0000) 2024-03-06T14:11:28.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:28.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:28.085680+0000) 2024-03-06T14:11:29.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:29.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:29.085842+0000) 2024-03-06T14:11:29.795 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:11:29.806 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:11:30.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:30.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:30.086040+0000) 2024-03-06T14:11:31.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:31.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:31.086261+0000) 2024-03-06T14:11:32.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:32.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:32.086487+0000) 2024-03-06T14:11:33.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:33.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:33.086682+0000) 2024-03-06T14:11:34.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:34.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:34.086870+0000) 2024-03-06T14:11:35.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:35.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:35.087095+0000) 2024-03-06T14:11:36.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:36.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:36.087306+0000) 2024-03-06T14:11:37.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:37.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:37.087527+0000) 2024-03-06T14:11:38.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:38.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:38.087755+0000) 2024-03-06T14:11:39.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:39.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:39.087952+0000) 2024-03-06T14:11:40.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:40.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:40.088132+0000) 2024-03-06T14:11:41.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:41.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:41.088353+0000) 2024-03-06T14:11:42.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:42.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:42.088605+0000) 2024-03-06T14:11:43.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:43.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:43.088826+0000) 2024-03-06T14:11:44.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:44.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:44.089064+0000) 2024-03-06T14:11:45.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:45.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:45.089236+0000) 2024-03-06T14:11:46.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:46.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:46.089457+0000) 2024-03-06T14:11:47.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:47.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:47.089682+0000) 2024-03-06T14:11:48.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:48.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:48.089880+0000) 2024-03-06T14:11:49.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:49.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:49.090102+0000) 2024-03-06T14:11:50.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:50.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:50.090330+0000) 2024-03-06T14:11:51.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:51.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:51.090523+0000) 2024-03-06T14:11:52.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:52.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:52.090746+0000) 2024-03-06T14:11:53.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:53.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:53.090935+0000) 2024-03-06T14:11:54.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:54.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:54.091160+0000) 2024-03-06T14:11:55.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:55.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:55.091381+0000) 2024-03-06T14:11:56.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:56.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:56.091629+0000) 2024-03-06T14:11:57.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:57.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:57.091855+0000) 2024-03-06T14:11:58.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:58.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:58.092087+0000) 2024-03-06T14:11:59.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:11:59.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:11:59.092296+0000) 2024-03-06T14:11:59.983 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:11:59.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:12:00.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:00.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:00.092501+0000) 2024-03-06T14:12:01.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:01.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:01.092696+0000) 2024-03-06T14:12:02.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:02.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:02.092919+0000) 2024-03-06T14:12:03.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:03.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:03.093106+0000) 2024-03-06T14:12:04.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:04.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:04.093328+0000) 2024-03-06T14:12:05.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:05.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:05.093512+0000) 2024-03-06T14:12:06.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:06.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:06.093787+0000) 2024-03-06T14:12:07.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:07.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:07.093981+0000) 2024-03-06T14:12:08.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:08.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:08.094261+0000) 2024-03-06T14:12:09.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:09.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:09.094525+0000) 2024-03-06T14:12:10.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:10.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:10.094788+0000) 2024-03-06T14:12:11.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:11.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:11.095015+0000) 2024-03-06T14:12:12.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:12.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:12.095244+0000) 2024-03-06T14:12:13.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:13.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:13.095439+0000) 2024-03-06T14:12:14.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:14.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:14.095676+0000) 2024-03-06T14:12:15.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:15.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:15.095895+0000) 2024-03-06T14:12:16.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:16.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:16.096124+0000) 2024-03-06T14:12:17.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:17.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:17.096366+0000) 2024-03-06T14:12:18.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:18.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:18.096647+0000) 2024-03-06T14:12:19.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:19.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:19.096870+0000) 2024-03-06T14:12:20.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:20.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:20.097098+0000) 2024-03-06T14:12:21.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:21.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:21.097315+0000) 2024-03-06T14:12:22.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:22.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:22.097559+0000) 2024-03-06T14:12:23.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:23.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:23.097793+0000) 2024-03-06T14:12:24.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:24.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:24.098007+0000) 2024-03-06T14:12:25.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:25.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:25.098382+0000) 2024-03-06T14:12:26.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:26.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:26.098690+0000) 2024-03-06T14:12:27.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:27.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:27.098910+0000) 2024-03-06T14:12:28.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:28.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:28.099198+0000) 2024-03-06T14:12:29.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:29.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:29.099475+0000) 2024-03-06T14:12:30.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:30.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:30.099707+0000) 2024-03-06T14:12:30.248 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:12:30.256 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:12:31.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:31.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:31.099987+0000) 2024-03-06T14:12:32.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:32.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:32.100273+0000) 2024-03-06T14:12:33.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:33.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:33.100584+0000) 2024-03-06T14:12:34.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:34.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:34.100797+0000) 2024-03-06T14:12:35.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:35.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:35.100999+0000) 2024-03-06T14:12:36.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:36.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:36.101203+0000) 2024-03-06T14:12:37.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:37.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:37.101415+0000) 2024-03-06T14:12:38.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:38.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:38.101738+0000) 2024-03-06T14:12:39.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:39.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:39.101924+0000) 2024-03-06T14:12:40.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:40.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:40.102120+0000) 2024-03-06T14:12:41.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:41.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:41.102351+0000) 2024-03-06T14:12:42.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:42.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:42.102659+0000) 2024-03-06T14:12:43.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:43.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:43.102906+0000) 2024-03-06T14:12:44.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:44.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:44.103103+0000) 2024-03-06T14:12:45.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:45.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:45.103346+0000) 2024-03-06T14:12:46.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:46.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:46.103644+0000) 2024-03-06T14:12:47.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:47.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:47.103922+0000) 2024-03-06T14:12:48.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:48.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:48.104158+0000) 2024-03-06T14:12:49.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:49.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:49.104366+0000) 2024-03-06T14:12:50.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:50.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:50.104642+0000) 2024-03-06T14:12:51.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:51.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:51.104865+0000) 2024-03-06T14:12:52.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:52.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:52.105100+0000) 2024-03-06T14:12:53.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:53.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:53.105327+0000) 2024-03-06T14:12:54.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:54.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:54.105616+0000) 2024-03-06T14:12:55.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:55.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:55.105834+0000) 2024-03-06T14:12:56.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:56.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:56.106112+0000) 2024-03-06T14:12:57.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:57.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:57.106383+0000) 2024-03-06T14:12:58.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:58.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:58.106681+0000) 2024-03-06T14:12:59.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:12:59.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:12:59.106963+0000) 2024-03-06T14:13:00.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:00.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:00.107243+0000) 2024-03-06T14:13:00.459 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:13:00.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:13:01.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:01.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:01.107476+0000) 2024-03-06T14:13:02.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:02.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:02.107681+0000) 2024-03-06T14:13:03.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:03.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:03.107907+0000) 2024-03-06T14:13:04.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:04.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:04.108115+0000) 2024-03-06T14:13:05.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:05.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:05.108335+0000) 2024-03-06T14:13:06.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:06.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:06.108652+0000) 2024-03-06T14:13:07.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:07.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:07.108921+0000) 2024-03-06T14:13:08.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:08.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:08.109210+0000) 2024-03-06T14:13:09.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:09.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:09.109486+0000) 2024-03-06T14:13:10.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:10.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:10.109742+0000) 2024-03-06T14:13:11.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:11.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:11.109984+0000) 2024-03-06T14:13:12.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:12.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:12.110211+0000) 2024-03-06T14:13:13.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:13.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:13.110437+0000) 2024-03-06T14:13:14.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:14.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:14.110647+0000) 2024-03-06T14:13:15.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:15.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:15.110851+0000) 2024-03-06T14:13:16.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:16.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:16.111007+0000) 2024-03-06T14:13:17.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:17.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:17.111246+0000) 2024-03-06T14:13:18.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:18.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:18.111485+0000) 2024-03-06T14:13:19.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:19.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:19.111713+0000) 2024-03-06T14:13:20.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:20.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:20.111942+0000) 2024-03-06T14:13:21.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:21.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:21.112163+0000) 2024-03-06T14:13:22.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:22.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:22.112368+0000) 2024-03-06T14:13:23.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:23.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:23.112631+0000) 2024-03-06T14:13:24.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:24.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:24.112842+0000) 2024-03-06T14:13:25.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:25.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:25.113063+0000) 2024-03-06T14:13:26.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:26.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:26.113343+0000) 2024-03-06T14:13:27.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:27.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:27.113586+0000) 2024-03-06T14:13:28.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:28.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:28.113794+0000) 2024-03-06T14:13:29.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:29.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:29.113973+0000) 2024-03-06T14:13:30.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:30.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:30.114197+0000) 2024-03-06T14:13:30.631 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:13:30.640 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:13:31.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:31.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:31.114428+0000) 2024-03-06T14:13:32.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:32.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:32.114658+0000) 2024-03-06T14:13:33.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:33.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:33.114886+0000) 2024-03-06T14:13:34.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:34.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:34.115112+0000) 2024-03-06T14:13:35.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:35.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:35.115280+0000) 2024-03-06T14:13:36.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:36.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:36.115519+0000) 2024-03-06T14:13:37.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:37.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:37.115775+0000) 2024-03-06T14:13:38.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:38.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:38.116008+0000) 2024-03-06T14:13:39.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:39.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:39.116238+0000) 2024-03-06T14:13:40.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:40.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:40.116463+0000) 2024-03-06T14:13:41.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:41.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:41.116691+0000) 2024-03-06T14:13:42.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:42.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:42.116915+0000) 2024-03-06T14:13:43.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:43.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:43.117107+0000) 2024-03-06T14:13:44.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:44.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:44.117334+0000) 2024-03-06T14:13:45.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:45.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:45.117513+0000) 2024-03-06T14:13:46.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:46.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:46.117726+0000) 2024-03-06T14:13:47.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:47.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:47.117966+0000) 2024-03-06T14:13:48.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:48.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:48.118202+0000) 2024-03-06T14:13:49.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:49.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:49.118385+0000) 2024-03-06T14:13:50.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:50.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:50.118629+0000) 2024-03-06T14:13:51.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:51.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:51.118820+0000) 2024-03-06T14:13:52.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:52.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:52.119031+0000) 2024-03-06T14:13:53.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:53.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:53.119257+0000) 2024-03-06T14:13:54.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:54.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:54.119485+0000) 2024-03-06T14:13:55.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:55.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:55.119669+0000) 2024-03-06T14:13:56.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:56.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:56.119935+0000) 2024-03-06T14:13:57.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:57.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:57.120108+0000) 2024-03-06T14:13:58.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:58.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:58.120327+0000) 2024-03-06T14:13:59.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:13:59.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:13:59.120527+0000) 2024-03-06T14:14:00.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:00.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:00.120758+0000) 2024-03-06T14:14:00.861 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:14:00.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:14:01.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:01.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:01.120953+0000) 2024-03-06T14:14:02.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:02.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:02.121144+0000) 2024-03-06T14:14:03.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:03.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:03.121379+0000) 2024-03-06T14:14:04.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:04.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:04.121641+0000) 2024-03-06T14:14:05.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:05.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:05.121852+0000) 2024-03-06T14:14:06.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:06.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:06.122017+0000) 2024-03-06T14:14:07.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:07.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:07.122192+0000) 2024-03-06T14:14:08.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:08.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:08.122442+0000) 2024-03-06T14:14:09.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:09.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:09.122668+0000) 2024-03-06T14:14:10.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:10.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:10.122894+0000) 2024-03-06T14:14:11.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:11.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:11.123094+0000) 2024-03-06T14:14:12.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:12.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:12.123331+0000) 2024-03-06T14:14:13.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:13.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:13.123589+0000) 2024-03-06T14:14:14.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:14.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:14.123824+0000) 2024-03-06T14:14:15.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:15.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:15.124045+0000) 2024-03-06T14:14:16.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:16.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:16.124273+0000) 2024-03-06T14:14:17.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:17.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:17.124469+0000) 2024-03-06T14:14:18.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:18.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:18.124678+0000) 2024-03-06T14:14:19.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:19.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:19.124912+0000) 2024-03-06T14:14:20.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:20.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:20.125153+0000) 2024-03-06T14:14:21.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:21.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:21.125340+0000) 2024-03-06T14:14:22.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:22.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:22.125562+0000) 2024-03-06T14:14:23.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:23.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:23.125800+0000) 2024-03-06T14:14:24.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:24.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:24.125985+0000) 2024-03-06T14:14:25.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:25.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:25.126148+0000) 2024-03-06T14:14:26.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:26.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:26.126378+0000) 2024-03-06T14:14:27.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:27.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:27.126631+0000) 2024-03-06T14:14:28.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:28.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:28.126872+0000) 2024-03-06T14:14:29.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:29.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:29.127093+0000) 2024-03-06T14:14:30.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:30.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:30.127329+0000) 2024-03-06T14:14:31.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:14:31.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:14:31.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:31.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:31.127462+0000) 2024-03-06T14:14:32.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:32.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:32.127709+0000) 2024-03-06T14:14:33.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:33.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:33.127909+0000) 2024-03-06T14:14:34.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:34.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:34.128142+0000) 2024-03-06T14:14:35.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:35.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:35.128367+0000) 2024-03-06T14:14:36.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:36.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:36.128525+0000) 2024-03-06T14:14:37.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:37.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:37.128810+0000) 2024-03-06T14:14:38.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:38.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:38.129021+0000) 2024-03-06T14:14:39.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:39.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:39.129224+0000) 2024-03-06T14:14:40.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:40.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:40.129464+0000) 2024-03-06T14:14:41.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:41.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:41.129698+0000) 2024-03-06T14:14:42.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:42.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:42.129927+0000) 2024-03-06T14:14:43.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:43.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:43.130153+0000) 2024-03-06T14:14:44.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:44.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:44.130384+0000) 2024-03-06T14:14:45.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:45.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:45.130669+0000) 2024-03-06T14:14:46.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:46.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:46.130820+0000) 2024-03-06T14:14:47.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:47.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:47.130982+0000) 2024-03-06T14:14:48.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:48.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:48.131232+0000) 2024-03-06T14:14:49.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:49.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:49.131424+0000) 2024-03-06T14:14:50.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:50.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:50.131650+0000) 2024-03-06T14:14:51.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:51.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:51.131891+0000) 2024-03-06T14:14:52.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:52.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:52.132121+0000) 2024-03-06T14:14:53.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:53.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:53.132350+0000) 2024-03-06T14:14:54.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:54.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:54.132624+0000) 2024-03-06T14:14:55.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:55.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:55.132852+0000) 2024-03-06T14:14:56.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:56.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:56.133096+0000) 2024-03-06T14:14:57.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:57.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:57.133334+0000) 2024-03-06T14:14:58.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:58.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:58.133516+0000) 2024-03-06T14:14:59.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:14:59.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:14:59.133779+0000) 2024-03-06T14:15:00.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:00.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:00.134001+0000) 2024-03-06T14:15:01.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:01.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:01.134248+0000) 2024-03-06T14:15:01.279 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:15:01.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:15:02.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:02.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:02.134481+0000) 2024-03-06T14:15:03.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:03.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:03.134654+0000) 2024-03-06T14:15:04.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:04.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:04.134889+0000) 2024-03-06T14:15:05.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:05.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:05.135114+0000) 2024-03-06T14:15:06.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:06.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:06.135291+0000) 2024-03-06T14:15:07.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:07.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:07.135523+0000) 2024-03-06T14:15:08.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:08.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:08.135770+0000) 2024-03-06T14:15:09.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:09.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:09.135937+0000) 2024-03-06T14:15:10.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:10.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:10.136110+0000) 2024-03-06T14:15:11.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:11.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:11.136343+0000) 2024-03-06T14:15:12.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:12.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:12.136579+0000) 2024-03-06T14:15:13.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:13.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:13.136755+0000) 2024-03-06T14:15:14.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:14.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:14.137000+0000) 2024-03-06T14:15:15.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:15.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:15.137235+0000) 2024-03-06T14:15:16.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:16.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:16.137461+0000) 2024-03-06T14:15:17.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:17.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:17.137646+0000) 2024-03-06T14:15:18.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:18.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:18.137887+0000) 2024-03-06T14:15:19.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:19.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:19.138118+0000) 2024-03-06T14:15:20.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:20.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:20.138348+0000) 2024-03-06T14:15:21.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:21.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:21.138608+0000) 2024-03-06T14:15:22.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:22.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:22.138839+0000) 2024-03-06T14:15:23.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:23.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:23.139069+0000) 2024-03-06T14:15:24.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:24.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:24.139291+0000) 2024-03-06T14:15:25.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:25.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:25.139517+0000) 2024-03-06T14:15:26.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:26.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:26.139784+0000) 2024-03-06T14:15:27.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:27.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:27.139977+0000) 2024-03-06T14:15:28.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:28.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:28.140235+0000) 2024-03-06T14:15:29.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:29.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:29.140458+0000) 2024-03-06T14:15:30.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:30.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:30.140662+0000) 2024-03-06T14:15:31.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:31.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:31.140897+0000) 2024-03-06T14:15:31.506 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:15:31.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:15:32.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:32.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:32.141047+0000) 2024-03-06T14:15:33.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:33.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:33.141319+0000) 2024-03-06T14:15:34.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:34.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:34.141630+0000) 2024-03-06T14:15:35.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:35.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:35.141855+0000) 2024-03-06T14:15:36.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:36.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:36.142053+0000) 2024-03-06T14:15:37.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:37.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:37.142275+0000) 2024-03-06T14:15:38.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:38.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:38.142499+0000) 2024-03-06T14:15:39.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:39.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:39.142776+0000) 2024-03-06T14:15:40.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:40.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:40.142965+0000) 2024-03-06T14:15:41.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:41.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:41.143195+0000) 2024-03-06T14:15:42.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:42.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:42.143377+0000) 2024-03-06T14:15:43.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:43.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:43.143634+0000) 2024-03-06T14:15:44.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:44.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:44.143866+0000) 2024-03-06T14:15:45.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:45.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:45.144055+0000) 2024-03-06T14:15:46.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:46.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:46.144293+0000) 2024-03-06T14:15:47.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:47.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:47.144517+0000) 2024-03-06T14:15:48.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:48.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:48.144786+0000) 2024-03-06T14:15:49.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:49.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:49.145011+0000) 2024-03-06T14:15:50.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:50.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:50.145243+0000) 2024-03-06T14:15:51.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:51.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:51.145473+0000) 2024-03-06T14:15:52.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:52.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:52.145697+0000) 2024-03-06T14:15:53.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:53.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:53.145907+0000) 2024-03-06T14:15:54.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:54.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:54.146144+0000) 2024-03-06T14:15:55.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:55.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:55.146426+0000) 2024-03-06T14:15:56.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:56.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:56.146683+0000) 2024-03-06T14:15:57.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:57.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:57.146862+0000) 2024-03-06T14:15:58.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:58.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:58.147102+0000) 2024-03-06T14:15:59.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:15:59.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:15:59.147322+0000) 2024-03-06T14:16:00.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:00.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:00.147482+0000) 2024-03-06T14:16:01.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:01.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:01.147713+0000) 2024-03-06T14:16:01.738 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:16:01.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:16:02.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:02.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:02.147963+0000) 2024-03-06T14:16:03.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:03.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:03.148146+0000) 2024-03-06T14:16:04.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:04.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:04.148371+0000) 2024-03-06T14:16:05.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:05.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:05.148638+0000) 2024-03-06T14:16:06.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:06.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:06.148871+0000) 2024-03-06T14:16:07.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:07.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:07.149064+0000) 2024-03-06T14:16:08.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:08.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:08.149316+0000) 2024-03-06T14:16:09.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:09.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:09.149573+0000) 2024-03-06T14:16:10.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:10.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:10.149763+0000) 2024-03-06T14:16:11.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:11.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:11.149942+0000) 2024-03-06T14:16:12.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:12.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:12.150148+0000) 2024-03-06T14:16:13.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:13.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:13.150338+0000) 2024-03-06T14:16:14.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:14.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:14.150644+0000) 2024-03-06T14:16:15.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:15.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:15.150864+0000) 2024-03-06T14:16:16.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:16.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:16.151080+0000) 2024-03-06T14:16:17.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:17.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:17.151292+0000) 2024-03-06T14:16:18.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:18.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:18.151529+0000) 2024-03-06T14:16:19.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:19.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:19.151744+0000) 2024-03-06T14:16:20.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:20.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:20.151997+0000) 2024-03-06T14:16:21.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:21.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:21.152239+0000) 2024-03-06T14:16:22.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:22.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:22.152390+0000) 2024-03-06T14:16:23.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:23.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:23.152664+0000) 2024-03-06T14:16:24.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:24.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:24.152896+0000) 2024-03-06T14:16:25.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:25.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:25.153148+0000) 2024-03-06T14:16:26.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:26.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:26.153385+0000) 2024-03-06T14:16:27.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:27.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:27.153606+0000) 2024-03-06T14:16:28.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:28.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:28.153833+0000) 2024-03-06T14:16:29.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:29.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:29.154010+0000) 2024-03-06T14:16:30.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:30.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:30.154194+0000) 2024-03-06T14:16:31.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:31.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:31.154406+0000) 2024-03-06T14:16:31.942 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:16:31.950 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:16:32.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:32.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:32.154646+0000) 2024-03-06T14:16:33.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:33.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:33.154879+0000) 2024-03-06T14:16:34.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:34.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:34.155102+0000) 2024-03-06T14:16:35.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:35.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:35.155265+0000) 2024-03-06T14:16:36.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:36.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:36.155500+0000) 2024-03-06T14:16:37.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:37.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:37.155734+0000) 2024-03-06T14:16:38.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:38.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:38.155982+0000) 2024-03-06T14:16:39.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:39.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:39.156175+0000) 2024-03-06T14:16:40.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:40.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:40.156432+0000) 2024-03-06T14:16:41.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:41.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:41.156684+0000) 2024-03-06T14:16:42.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:42.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:42.156888+0000) 2024-03-06T14:16:43.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:43.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:43.157188+0000) 2024-03-06T14:16:44.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:44.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:44.157427+0000) 2024-03-06T14:16:45.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:45.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:45.157653+0000) 2024-03-06T14:16:46.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:46.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:46.157896+0000) 2024-03-06T14:16:47.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:47.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:47.158122+0000) 2024-03-06T14:16:48.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:48.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:48.158370+0000) 2024-03-06T14:16:49.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:49.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:49.158639+0000) 2024-03-06T14:16:50.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:50.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:50.158818+0000) 2024-03-06T14:16:51.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:51.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:51.159039+0000) 2024-03-06T14:16:52.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:52.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:52.159201+0000) 2024-03-06T14:16:53.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:53.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:53.159449+0000) 2024-03-06T14:16:54.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:54.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:54.159697+0000) 2024-03-06T14:16:55.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:55.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:55.159889+0000) 2024-03-06T14:16:56.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:56.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:56.160121+0000) 2024-03-06T14:16:57.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:57.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:57.160344+0000) 2024-03-06T14:16:58.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:58.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:58.160655+0000) 2024-03-06T14:16:59.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:16:59.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:16:59.160925+0000) 2024-03-06T14:17:00.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:00.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:00.161180+0000) 2024-03-06T14:17:01.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:01.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:01.161424+0000) 2024-03-06T14:17:02.072 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:17:02.080 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:17:02.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:02.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:02.161629+0000) 2024-03-06T14:17:03.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:03.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:03.161832+0000) 2024-03-06T14:17:04.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:04.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:04.162071+0000) 2024-03-06T14:17:05.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:05.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:05.162241+0000) 2024-03-06T14:17:06.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:06.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:06.162461+0000) 2024-03-06T14:17:07.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:07.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:07.162688+0000) 2024-03-06T14:17:08.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:08.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:08.162924+0000) 2024-03-06T14:17:09.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:09.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:09.163159+0000) 2024-03-06T14:17:10.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:10.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:10.163336+0000) 2024-03-06T14:17:11.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:11.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:11.163521+0000) 2024-03-06T14:17:12.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:12.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:12.163790+0000) 2024-03-06T14:17:13.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:13.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:13.164045+0000) 2024-03-06T14:17:14.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:14.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:14.164321+0000) 2024-03-06T14:17:15.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:15.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:15.164620+0000) 2024-03-06T14:17:16.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:16.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:16.164895+0000) 2024-03-06T14:17:17.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:17.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:17.165179+0000) 2024-03-06T14:17:18.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:18.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:18.165439+0000) 2024-03-06T14:17:19.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:19.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:19.165704+0000) 2024-03-06T14:17:20.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:20.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:20.165952+0000) 2024-03-06T14:17:21.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:21.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:21.166217+0000) 2024-03-06T14:17:22.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:22.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:22.166488+0000) 2024-03-06T14:17:23.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:23.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:23.166772+0000) 2024-03-06T14:17:24.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:24.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:24.166996+0000) 2024-03-06T14:17:25.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:25.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:25.167229+0000) 2024-03-06T14:17:26.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:26.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:26.167499+0000) 2024-03-06T14:17:27.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:27.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:27.167708+0000) 2024-03-06T14:17:28.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:28.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:28.167915+0000) 2024-03-06T14:17:29.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:29.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:29.168141+0000) 2024-03-06T14:17:30.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:30.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:30.168384+0000) 2024-03-06T14:17:31.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:31.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:31.168661+0000) 2024-03-06T14:17:32.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:32.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:32.168899+0000) 2024-03-06T14:17:32.295 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:17:32.305 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:17:33.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:33.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:33.169157+0000) 2024-03-06T14:17:34.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:34.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:34.169388+0000) 2024-03-06T14:17:35.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:35.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:35.169689+0000) 2024-03-06T14:17:36.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:36.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:36.169959+0000) 2024-03-06T14:17:37.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:37.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:37.170193+0000) 2024-03-06T14:17:38.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:38.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:38.170445+0000) 2024-03-06T14:17:39.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:39.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:39.170685+0000) 2024-03-06T14:17:40.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:40.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:40.170923+0000) 2024-03-06T14:17:41.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:41.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:41.171153+0000) 2024-03-06T14:17:42.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:42.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:42.171371+0000) 2024-03-06T14:17:43.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:43.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:43.171651+0000) 2024-03-06T14:17:44.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:44.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:44.171856+0000) 2024-03-06T14:17:45.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:45.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:45.172062+0000) 2024-03-06T14:17:46.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:46.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:46.172297+0000) 2024-03-06T14:17:47.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:47.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:47.172516+0000) 2024-03-06T14:17:48.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:48.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:48.172876+0000) 2024-03-06T14:17:49.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:49.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:49.173115+0000) 2024-03-06T14:17:50.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:50.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:50.173306+0000) 2024-03-06T14:17:51.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:51.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:51.173529+0000) 2024-03-06T14:17:52.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:52.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:52.173769+0000) 2024-03-06T14:17:53.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:53.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:53.173992+0000) 2024-03-06T14:17:54.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:54.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:54.174264+0000) 2024-03-06T14:17:55.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:55.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:55.174579+0000) 2024-03-06T14:17:56.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:56.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:56.174870+0000) 2024-03-06T14:17:57.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:57.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:57.175137+0000) 2024-03-06T14:17:58.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:58.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:58.175421+0000) 2024-03-06T14:17:59.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:17:59.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:17:59.175699+0000) 2024-03-06T14:18:00.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:00.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:00.175984+0000) 2024-03-06T14:18:01.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:01.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:01.176262+0000) 2024-03-06T14:18:02.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:02.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:02.176573+0000) 2024-03-06T14:18:02.515 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:18:02.524 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:18:03.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:03.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:03.176783+0000) 2024-03-06T14:18:04.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:04.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:04.176978+0000) 2024-03-06T14:18:05.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:05.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:05.177215+0000) 2024-03-06T14:18:06.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:06.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:06.177484+0000) 2024-03-06T14:18:07.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:07.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:07.177684+0000) 2024-03-06T14:18:08.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:08.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:08.177874+0000) 2024-03-06T14:18:09.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:09.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:09.178126+0000) 2024-03-06T14:18:10.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:10.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:10.178413+0000) 2024-03-06T14:18:11.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:11.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:11.178641+0000) 2024-03-06T14:18:12.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:12.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:12.178847+0000) 2024-03-06T14:18:13.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:13.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:13.179094+0000) 2024-03-06T14:18:14.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:14.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:14.179325+0000) 2024-03-06T14:18:15.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:15.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:15.179590+0000) 2024-03-06T14:18:16.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:16.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:16.179818+0000) 2024-03-06T14:18:17.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:17.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:17.180001+0000) 2024-03-06T14:18:18.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:18.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:18.180239+0000) 2024-03-06T14:18:19.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:19.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:19.180476+0000) 2024-03-06T14:18:20.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:20.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:20.180692+0000) 2024-03-06T14:18:21.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:21.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:21.180917+0000) 2024-03-06T14:18:22.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:22.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:22.181186+0000) 2024-03-06T14:18:23.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:23.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:23.181429+0000) 2024-03-06T14:18:24.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:24.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:24.181663+0000) 2024-03-06T14:18:25.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:25.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:25.181872+0000) 2024-03-06T14:18:26.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:26.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:26.182144+0000) 2024-03-06T14:18:27.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:27.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:27.182357+0000) 2024-03-06T14:18:28.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:28.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:28.182646+0000) 2024-03-06T14:18:29.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:29.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:29.182833+0000) 2024-03-06T14:18:30.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:30.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:30.183116+0000) 2024-03-06T14:18:31.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:31.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:31.183342+0000) 2024-03-06T14:18:32.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:32.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:32.183601+0000) 2024-03-06T14:18:32.720 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:18:32.729 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:18:33.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:33.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:33.183831+0000) 2024-03-06T14:18:34.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:34.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:34.184057+0000) 2024-03-06T14:18:35.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:35.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:35.184269+0000) 2024-03-06T14:18:36.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:36.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:36.184582+0000) 2024-03-06T14:18:37.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:37.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:37.184822+0000) 2024-03-06T14:18:38.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:38.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:38.185073+0000) 2024-03-06T14:18:39.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:39.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:39.185301+0000) 2024-03-06T14:18:40.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:40.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:40.185557+0000) 2024-03-06T14:18:41.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:41.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:41.185768+0000) 2024-03-06T14:18:42.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:42.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:42.185991+0000) 2024-03-06T14:18:43.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:43.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:43.186269+0000) 2024-03-06T14:18:44.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:44.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:44.186570+0000) 2024-03-06T14:18:45.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:45.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:45.186808+0000) 2024-03-06T14:18:46.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:46.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:46.187010+0000) 2024-03-06T14:18:47.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:47.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:47.187234+0000) 2024-03-06T14:18:48.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:48.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:48.187464+0000) 2024-03-06T14:18:49.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:49.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:49.187723+0000) 2024-03-06T14:18:50.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:50.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:50.187927+0000) 2024-03-06T14:18:51.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:51.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:51.188137+0000) 2024-03-06T14:18:52.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:52.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:52.188415+0000) 2024-03-06T14:18:53.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:53.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:53.188656+0000) 2024-03-06T14:18:54.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:54.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:54.188891+0000) 2024-03-06T14:18:55.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:55.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:55.189106+0000) 2024-03-06T14:18:56.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:56.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:56.189371+0000) 2024-03-06T14:18:57.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:57.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:57.189651+0000) 2024-03-06T14:18:58.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:58.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:58.189858+0000) 2024-03-06T14:18:59.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:18:59.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:18:59.190118+0000) 2024-03-06T14:19:00.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:00.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:00.190394+0000) 2024-03-06T14:19:01.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:01.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:01.190645+0000) 2024-03-06T14:19:02.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:02.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:02.190863+0000) 2024-03-06T14:19:02.891 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:19:02.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:19:03.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:03.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:03.191093+0000) 2024-03-06T14:19:04.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:04.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:04.191357+0000) 2024-03-06T14:19:05.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:05.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:05.191663+0000) 2024-03-06T14:19:06.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:06.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:06.191942+0000) 2024-03-06T14:19:07.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:07.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:07.192211+0000) 2024-03-06T14:19:08.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:08.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:08.192483+0000) 2024-03-06T14:19:09.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:09.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:09.192769+0000) 2024-03-06T14:19:10.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:10.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:10.192985+0000) 2024-03-06T14:19:11.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:11.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:11.193252+0000) 2024-03-06T14:19:12.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:12.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:12.193526+0000) 2024-03-06T14:19:13.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:13.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:13.193766+0000) 2024-03-06T14:19:14.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:14.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:14.194039+0000) 2024-03-06T14:19:15.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:15.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:15.194283+0000) 2024-03-06T14:19:16.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:16.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:16.194503+0000) 2024-03-06T14:19:17.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:17.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:17.194692+0000) 2024-03-06T14:19:18.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:18.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:18.194970+0000) 2024-03-06T14:19:19.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:19.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:19.195230+0000) 2024-03-06T14:19:20.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:20.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:20.195450+0000) 2024-03-06T14:19:21.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:21.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:21.195673+0000) 2024-03-06T14:19:22.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:22.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:22.195895+0000) 2024-03-06T14:19:23.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:23.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:23.196120+0000) 2024-03-06T14:19:24.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:24.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:24.196326+0000) 2024-03-06T14:19:25.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:25.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:25.196601+0000) 2024-03-06T14:19:26.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:26.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:26.196796+0000) 2024-03-06T14:19:27.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:27.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:27.197023+0000) 2024-03-06T14:19:28.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:28.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:28.197209+0000) 2024-03-06T14:19:29.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:29.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:29.197434+0000) 2024-03-06T14:19:30.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:30.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:30.197659+0000) 2024-03-06T14:19:31.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:31.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:31.197878+0000) 2024-03-06T14:19:32.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:32.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:32.198103+0000) 2024-03-06T14:19:33.096 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:19:33.104 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:19:33.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:33.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:33.198300+0000) 2024-03-06T14:19:34.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:34.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:34.198520+0000) 2024-03-06T14:19:35.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:35.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:35.198749+0000) 2024-03-06T14:19:36.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:36.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:36.198934+0000) 2024-03-06T14:19:37.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:37.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:37.199158+0000) 2024-03-06T14:19:38.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:38.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:38.199385+0000) 2024-03-06T14:19:39.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:39.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:39.199651+0000) 2024-03-06T14:19:40.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:40.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:40.199856+0000) 2024-03-06T14:19:41.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:41.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:41.200101+0000) 2024-03-06T14:19:42.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:42.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:42.200303+0000) 2024-03-06T14:19:43.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:43.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:43.200519+0000) 2024-03-06T14:19:44.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:44.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:44.200757+0000) 2024-03-06T14:19:45.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:45.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:45.200952+0000) 2024-03-06T14:19:46.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:46.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:46.201177+0000) 2024-03-06T14:19:47.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:47.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:47.201401+0000) 2024-03-06T14:19:48.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:48.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:48.201635+0000) 2024-03-06T14:19:49.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:49.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:49.201842+0000) 2024-03-06T14:19:50.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:50.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:50.202039+0000) 2024-03-06T14:19:51.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:51.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:51.202221+0000) 2024-03-06T14:19:52.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:52.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:52.202441+0000) 2024-03-06T14:19:53.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:53.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:53.202661+0000) 2024-03-06T14:19:54.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:54.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:54.202868+0000) 2024-03-06T14:19:55.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:55.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:55.203143+0000) 2024-03-06T14:19:56.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:56.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:56.203379+0000) 2024-03-06T14:19:57.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:57.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:57.203617+0000) 2024-03-06T14:19:58.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:58.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:58.203826+0000) 2024-03-06T14:19:59.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:19:59.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:19:59.204028+0000) 2024-03-06T14:20:00.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:00.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:00.204265+0000) 2024-03-06T14:20:01.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:01.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:01.204484+0000) 2024-03-06T14:20:02.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:02.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:02.204686+0000) 2024-03-06T14:20:03.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:03.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:03.204862+0000) 2024-03-06T14:20:03.318 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:20:03.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:20:04.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:04.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:04.205045+0000) 2024-03-06T14:20:05.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:05.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:05.205251+0000) 2024-03-06T14:20:06.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:06.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:06.205460+0000) 2024-03-06T14:20:07.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:07.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:07.205682+0000) 2024-03-06T14:20:08.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:08.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:08.205844+0000) 2024-03-06T14:20:09.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:09.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:09.206037+0000) 2024-03-06T14:20:10.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:10.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:10.206237+0000) 2024-03-06T14:20:11.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:11.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:11.206511+0000) 2024-03-06T14:20:12.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:12.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:12.206758+0000) 2024-03-06T14:20:13.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:13.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:13.206973+0000) 2024-03-06T14:20:14.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:14.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:14.207205+0000) 2024-03-06T14:20:15.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:15.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:15.207391+0000) 2024-03-06T14:20:16.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:16.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:16.207647+0000) 2024-03-06T14:20:17.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:17.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:17.207800+0000) 2024-03-06T14:20:18.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:18.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:18.208031+0000) 2024-03-06T14:20:19.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:19.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:19.208255+0000) 2024-03-06T14:20:20.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:20.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:20.208470+0000) 2024-03-06T14:20:21.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:21.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:21.208755+0000) 2024-03-06T14:20:22.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:22.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:22.209005+0000) 2024-03-06T14:20:23.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:23.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:23.209228+0000) 2024-03-06T14:20:24.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:24.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:24.209456+0000) 2024-03-06T14:20:25.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:25.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:25.209689+0000) 2024-03-06T14:20:26.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:26.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:26.209911+0000) 2024-03-06T14:20:27.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:27.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:27.210137+0000) 2024-03-06T14:20:28.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:28.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:28.210367+0000) 2024-03-06T14:20:29.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:29.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:29.210553+0000) 2024-03-06T14:20:30.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:30.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:30.210746+0000) 2024-03-06T14:20:31.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:31.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:31.210970+0000) 2024-03-06T14:20:32.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:32.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:32.211192+0000) 2024-03-06T14:20:33.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:33.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:33.211413+0000) 2024-03-06T14:20:33.531 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:20:33.540 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:20:34.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:34.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:34.211650+0000) 2024-03-06T14:20:35.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:35.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:35.211887+0000) 2024-03-06T14:20:36.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:36.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:36.212109+0000) 2024-03-06T14:20:37.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:37.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:37.212344+0000) 2024-03-06T14:20:38.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:38.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:38.212618+0000) 2024-03-06T14:20:39.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:39.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:39.212844+0000) 2024-03-06T14:20:40.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:40.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:40.213073+0000) 2024-03-06T14:20:41.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:41.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:41.213299+0000) 2024-03-06T14:20:42.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:42.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:42.213523+0000) 2024-03-06T14:20:43.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:43.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:43.213781+0000) 2024-03-06T14:20:44.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:44.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:44.213995+0000) 2024-03-06T14:20:45.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:45.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:45.214236+0000) 2024-03-06T14:20:46.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:46.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:46.214457+0000) 2024-03-06T14:20:47.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:47.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:47.214699+0000) 2024-03-06T14:20:48.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:48.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:48.214920+0000) 2024-03-06T14:20:49.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:49.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:49.215167+0000) 2024-03-06T14:20:50.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:50.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:50.215351+0000) 2024-03-06T14:20:51.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:51.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:51.215609+0000) 2024-03-06T14:20:52.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:52.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:52.215777+0000) 2024-03-06T14:20:53.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:53.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:53.216003+0000) 2024-03-06T14:20:54.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:54.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:54.216286+0000) 2024-03-06T14:20:55.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:55.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:55.216518+0000) 2024-03-06T14:20:56.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:56.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:56.216775+0000) 2024-03-06T14:20:57.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:57.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:57.217001+0000) 2024-03-06T14:20:58.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:58.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:58.217208+0000) 2024-03-06T14:20:59.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:20:59.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:20:59.217426+0000) 2024-03-06T14:21:00.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:00.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:00.217674+0000) 2024-03-06T14:21:01.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:01.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:01.217883+0000) 2024-03-06T14:21:02.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:02.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:02.218121+0000) 2024-03-06T14:21:03.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:03.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:03.218313+0000) 2024-03-06T14:21:03.737 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:21:03.745 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:21:04.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:04.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:04.218515+0000) 2024-03-06T14:21:05.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:05.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:05.218773+0000) 2024-03-06T14:21:06.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:06.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:06.219010+0000) 2024-03-06T14:21:07.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:07.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:07.219246+0000) 2024-03-06T14:21:08.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:08.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:08.219477+0000) 2024-03-06T14:21:09.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:09.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:09.219727+0000) 2024-03-06T14:21:10.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:10.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:10.220001+0000) 2024-03-06T14:21:11.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:11.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:11.220244+0000) 2024-03-06T14:21:12.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:12.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:12.220475+0000) 2024-03-06T14:21:13.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:13.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:13.220690+0000) 2024-03-06T14:21:14.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:14.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:14.220884+0000) 2024-03-06T14:21:15.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:15.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:15.221127+0000) 2024-03-06T14:21:16.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:16.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:16.221366+0000) 2024-03-06T14:21:17.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:17.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:17.221613+0000) 2024-03-06T14:21:18.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:18.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:18.221832+0000) 2024-03-06T14:21:19.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:19.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:19.222078+0000) 2024-03-06T14:21:20.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:20.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:20.222327+0000) 2024-03-06T14:21:21.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:21.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:21.222571+0000) 2024-03-06T14:21:22.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:22.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:22.222788+0000) 2024-03-06T14:21:23.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:23.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:23.223004+0000) 2024-03-06T14:21:24.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:24.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:24.223227+0000) 2024-03-06T14:21:25.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:25.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:25.223467+0000) 2024-03-06T14:21:26.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:26.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:26.223698+0000) 2024-03-06T14:21:27.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:27.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:27.223920+0000) 2024-03-06T14:21:28.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:28.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:28.224115+0000) 2024-03-06T14:21:29.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:29.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:29.224341+0000) 2024-03-06T14:21:30.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:30.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:30.224577+0000) 2024-03-06T14:21:31.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:31.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:31.224851+0000) 2024-03-06T14:21:32.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:32.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:32.225114+0000) 2024-03-06T14:21:33.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:33.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:33.225395+0000) 2024-03-06T14:21:33.966 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:21:33.977 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:21:34.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:34.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:34.225676+0000) 2024-03-06T14:21:35.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:35.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:35.225919+0000) 2024-03-06T14:21:36.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:36.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:36.226196+0000) 2024-03-06T14:21:37.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:37.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:37.226406+0000) 2024-03-06T14:21:38.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:38.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:38.226658+0000) 2024-03-06T14:21:39.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:39.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:39.226865+0000) 2024-03-06T14:21:40.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:40.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:40.227109+0000) 2024-03-06T14:21:41.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:41.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:41.227318+0000) 2024-03-06T14:21:42.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:42.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:42.227581+0000) 2024-03-06T14:21:43.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:43.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:43.227786+0000) 2024-03-06T14:21:44.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:44.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:44.228036+0000) 2024-03-06T14:21:45.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:45.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:45.228281+0000) 2024-03-06T14:21:46.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:46.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:46.228494+0000) 2024-03-06T14:21:47.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:47.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:47.228901+0000) 2024-03-06T14:21:48.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:48.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:48.229161+0000) 2024-03-06T14:21:49.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:49.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:49.229412+0000) 2024-03-06T14:21:50.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:50.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:50.229645+0000) 2024-03-06T14:21:51.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:51.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:51.229839+0000) 2024-03-06T14:21:52.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:52.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:52.230037+0000) 2024-03-06T14:21:53.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:53.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:53.230273+0000) 2024-03-06T14:21:54.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:54.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:54.230494+0000) 2024-03-06T14:21:55.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:55.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:55.230738+0000) 2024-03-06T14:21:56.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:56.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:56.230968+0000) 2024-03-06T14:21:57.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:57.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:57.231218+0000) 2024-03-06T14:21:58.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:58.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:58.231524+0000) 2024-03-06T14:21:59.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:21:59.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:21:59.231806+0000) 2024-03-06T14:22:00.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:00.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:00.232057+0000) 2024-03-06T14:22:01.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:01.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:01.232334+0000) 2024-03-06T14:22:02.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:02.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:02.232628+0000) 2024-03-06T14:22:03.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:03.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:03.232903+0000) 2024-03-06T14:22:04.174 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:22:04.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:22:04.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:04.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:04.233103+0000) 2024-03-06T14:22:05.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:05.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:05.233344+0000) 2024-03-06T14:22:06.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:06.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:06.233639+0000) 2024-03-06T14:22:07.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:07.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:07.233880+0000) 2024-03-06T14:22:08.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:08.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:08.234151+0000) 2024-03-06T14:22:09.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:09.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:09.234428+0000) 2024-03-06T14:22:10.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:10.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:10.234715+0000) 2024-03-06T14:22:11.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:11.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:11.234950+0000) 2024-03-06T14:22:12.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:12.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:12.235129+0000) 2024-03-06T14:22:13.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:13.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:13.235347+0000) 2024-03-06T14:22:14.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:14.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:14.235625+0000) 2024-03-06T14:22:15.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:15.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:15.235824+0000) 2024-03-06T14:22:16.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:16.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:16.235998+0000) 2024-03-06T14:22:17.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:17.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:17.236221+0000) 2024-03-06T14:22:18.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:18.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:18.236456+0000) 2024-03-06T14:22:19.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:19.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:19.236697+0000) 2024-03-06T14:22:20.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:20.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:20.236858+0000) 2024-03-06T14:22:21.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:21.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:21.237060+0000) 2024-03-06T14:22:22.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:22.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:22.237309+0000) 2024-03-06T14:22:23.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:23.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:23.237561+0000) 2024-03-06T14:22:24.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:24.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:24.237768+0000) 2024-03-06T14:22:25.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:25.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:25.237975+0000) 2024-03-06T14:22:26.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:26.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:26.238175+0000) 2024-03-06T14:22:27.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:27.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:27.238347+0000) 2024-03-06T14:22:28.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:28.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:28.238617+0000) 2024-03-06T14:22:29.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:29.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:29.238819+0000) 2024-03-06T14:22:30.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:30.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:30.239045+0000) 2024-03-06T14:22:31.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:31.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:31.239215+0000) 2024-03-06T14:22:32.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:32.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:32.239397+0000) 2024-03-06T14:22:33.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:33.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:33.239666+0000) 2024-03-06T14:22:34.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:34.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:34.239860+0000) 2024-03-06T14:22:34.383 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:22:34.391 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:22:35.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:35.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:35.240076+0000) 2024-03-06T14:22:36.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:36.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:36.240252+0000) 2024-03-06T14:22:37.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:37.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:37.240461+0000) 2024-03-06T14:22:38.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:38.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:38.240652+0000) 2024-03-06T14:22:39.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:39.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:39.240884+0000) 2024-03-06T14:22:40.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:40.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:40.241090+0000) 2024-03-06T14:22:41.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:41.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:41.241328+0000) 2024-03-06T14:22:42.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:42.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:42.241578+0000) 2024-03-06T14:22:43.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:43.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:43.241782+0000) 2024-03-06T14:22:44.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:44.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:44.241975+0000) 2024-03-06T14:22:45.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:45.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:45.242162+0000) 2024-03-06T14:22:46.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:46.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:46.242357+0000) 2024-03-06T14:22:47.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:47.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:47.242640+0000) 2024-03-06T14:22:48.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:48.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:48.242843+0000) 2024-03-06T14:22:49.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:49.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:49.243090+0000) 2024-03-06T14:22:50.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:50.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:50.243312+0000) 2024-03-06T14:22:51.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:51.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:51.243573+0000) 2024-03-06T14:22:52.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:52.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:52.243775+0000) 2024-03-06T14:22:53.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:53.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:53.243967+0000) 2024-03-06T14:22:54.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:54.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:54.244205+0000) 2024-03-06T14:22:55.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:55.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:55.244405+0000) 2024-03-06T14:22:56.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:56.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:56.244635+0000) 2024-03-06T14:22:57.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:57.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:57.244857+0000) 2024-03-06T14:22:58.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:58.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:58.245089+0000) 2024-03-06T14:22:59.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:22:59.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:22:59.245283+0000) 2024-03-06T14:23:00.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:00.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:00.245499+0000) 2024-03-06T14:23:01.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:01.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:01.245687+0000) 2024-03-06T14:23:02.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:02.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:02.245860+0000) 2024-03-06T14:23:03.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:03.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:03.246036+0000) 2024-03-06T14:23:04.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:04.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:04.246260+0000) 2024-03-06T14:23:04.589 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:23:04.597 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:23:05.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:05.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:05.246486+0000) 2024-03-06T14:23:06.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:06.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:06.246676+0000) 2024-03-06T14:23:07.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:07.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:07.246846+0000) 2024-03-06T14:23:08.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:08.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:08.247072+0000) 2024-03-06T14:23:09.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:09.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:09.247233+0000) 2024-03-06T14:23:10.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:10.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:10.247439+0000) 2024-03-06T14:23:11.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:11.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:11.247685+0000) 2024-03-06T14:23:12.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:12.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:12.247874+0000) 2024-03-06T14:23:13.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:13.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:13.248062+0000) 2024-03-06T14:23:14.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:14.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:14.248289+0000) 2024-03-06T14:23:15.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:15.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:15.248517+0000) 2024-03-06T14:23:16.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:16.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:16.248746+0000) 2024-03-06T14:23:17.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:17.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:17.248985+0000) 2024-03-06T14:23:18.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:18.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:18.249245+0000) 2024-03-06T14:23:19.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:19.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:19.249468+0000) 2024-03-06T14:23:20.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:20.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:20.249667+0000) 2024-03-06T14:23:21.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:21.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:21.249863+0000) 2024-03-06T14:23:22.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:22.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:22.250084+0000) 2024-03-06T14:23:23.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:23.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:23.250308+0000) 2024-03-06T14:23:24.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:24.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:24.250467+0000) 2024-03-06T14:23:25.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:25.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:25.250710+0000) 2024-03-06T14:23:26.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:26.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:26.250907+0000) 2024-03-06T14:23:27.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:27.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:27.251130+0000) 2024-03-06T14:23:28.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:28.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:28.251415+0000) 2024-03-06T14:23:29.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:29.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:29.251641+0000) 2024-03-06T14:23:30.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:30.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:30.251843+0000) 2024-03-06T14:23:31.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:31.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:31.252062+0000) 2024-03-06T14:23:32.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:32.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:32.252272+0000) 2024-03-06T14:23:33.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:33.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:33.252515+0000) 2024-03-06T14:23:34.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:34.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:34.252756+0000) 2024-03-06T14:23:34.784 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:23:34.793 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:23:35.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:35.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:35.252951+0000) 2024-03-06T14:23:36.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:36.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:36.253166+0000) 2024-03-06T14:23:37.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:37.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:37.253387+0000) 2024-03-06T14:23:38.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:38.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:38.253657+0000) 2024-03-06T14:23:39.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:39.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:39.253880+0000) 2024-03-06T14:23:40.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:40.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:40.254114+0000) 2024-03-06T14:23:41.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:41.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:41.254343+0000) 2024-03-06T14:23:42.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:42.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:42.254523+0000) 2024-03-06T14:23:43.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:43.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:43.254792+0000) 2024-03-06T14:23:44.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:44.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:44.254985+0000) 2024-03-06T14:23:45.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:45.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:45.255208+0000) 2024-03-06T14:23:46.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:46.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:46.255426+0000) 2024-03-06T14:23:47.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:47.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:47.255653+0000) 2024-03-06T14:23:48.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:48.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:48.255848+0000) 2024-03-06T14:23:49.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:49.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:49.256077+0000) 2024-03-06T14:23:50.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:50.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:50.256265+0000) 2024-03-06T14:23:51.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:51.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:51.256491+0000) 2024-03-06T14:23:52.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:52.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:52.256713+0000) 2024-03-06T14:23:53.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:53.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:53.256906+0000) 2024-03-06T14:23:54.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:54.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:54.257102+0000) 2024-03-06T14:23:55.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:55.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:55.257320+0000) 2024-03-06T14:23:56.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:56.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:56.257590+0000) 2024-03-06T14:23:57.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:57.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:57.257792+0000) 2024-03-06T14:23:58.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:58.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:58.258036+0000) 2024-03-06T14:23:59.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:23:59.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:23:59.258261+0000) 2024-03-06T14:24:00.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:00.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:00.258472+0000) 2024-03-06T14:24:01.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:01.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:01.258722+0000) 2024-03-06T14:24:02.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:02.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:02.258946+0000) 2024-03-06T14:24:03.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:03.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:03.259198+0000) 2024-03-06T14:24:04.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:04.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:04.259418+0000) 2024-03-06T14:24:04.989 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:24:04.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:24:05.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:05.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:05.259657+0000) 2024-03-06T14:24:06.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:06.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:06.259855+0000) 2024-03-06T14:24:07.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:07.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:07.260072+0000) 2024-03-06T14:24:08.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:08.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:08.260271+0000) 2024-03-06T14:24:09.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:09.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:09.260513+0000) 2024-03-06T14:24:10.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:10.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:10.260753+0000) 2024-03-06T14:24:11.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:11.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:11.260951+0000) 2024-03-06T14:24:12.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:12.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:12.261179+0000) 2024-03-06T14:24:13.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:13.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:13.261374+0000) 2024-03-06T14:24:14.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:14.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:14.261650+0000) 2024-03-06T14:24:15.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:15.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:15.261872+0000) 2024-03-06T14:24:16.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:16.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:16.262092+0000) 2024-03-06T14:24:17.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:17.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:17.262315+0000) 2024-03-06T14:24:18.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:18.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:18.262495+0000) 2024-03-06T14:24:19.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:19.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:19.262689+0000) 2024-03-06T14:24:20.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:20.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:20.262931+0000) 2024-03-06T14:24:21.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:21.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:21.263130+0000) 2024-03-06T14:24:22.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:22.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:22.263420+0000) 2024-03-06T14:24:23.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:23.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:23.263666+0000) 2024-03-06T14:24:24.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:24.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:24.263905+0000) 2024-03-06T14:24:25.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:25.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:25.264112+0000) 2024-03-06T14:24:26.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:26.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:26.264352+0000) 2024-03-06T14:24:27.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:27.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:27.264654+0000) 2024-03-06T14:24:28.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:28.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:28.264874+0000) 2024-03-06T14:24:29.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:29.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:29.265116+0000) 2024-03-06T14:24:30.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:30.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:30.265359+0000) 2024-03-06T14:24:31.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:31.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:31.265637+0000) 2024-03-06T14:24:32.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:32.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:32.265848+0000) 2024-03-06T14:24:33.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:33.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:33.266083+0000) 2024-03-06T14:24:34.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:34.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:34.266237+0000) 2024-03-06T14:24:35.143 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:24:35.154 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:24:35.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:35.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:35.266468+0000) 2024-03-06T14:24:36.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:36.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:36.266696+0000) 2024-03-06T14:24:37.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:37.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:37.266943+0000) 2024-03-06T14:24:38.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:38.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:38.267174+0000) 2024-03-06T14:24:39.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:39.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:39.267370+0000) 2024-03-06T14:24:40.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:40.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:40.267608+0000) 2024-03-06T14:24:41.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:41.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:41.267815+0000) 2024-03-06T14:24:42.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:42.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:42.268037+0000) 2024-03-06T14:24:43.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:43.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:43.268248+0000) 2024-03-06T14:24:44.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:44.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:44.268492+0000) 2024-03-06T14:24:45.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:45.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:45.268699+0000) 2024-03-06T14:24:46.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:46.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:46.268924+0000) 2024-03-06T14:24:47.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:47.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:47.269103+0000) 2024-03-06T14:24:48.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:48.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:48.269351+0000) 2024-03-06T14:24:49.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:49.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:49.269577+0000) 2024-03-06T14:24:50.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:50.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:50.269803+0000) 2024-03-06T14:24:51.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:51.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:51.269996+0000) 2024-03-06T14:24:52.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:52.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:52.270190+0000) 2024-03-06T14:24:53.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:53.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:53.270384+0000) 2024-03-06T14:24:54.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:54.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:54.270655+0000) 2024-03-06T14:24:55.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:55.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:55.270883+0000) 2024-03-06T14:24:56.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:56.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:56.271071+0000) 2024-03-06T14:24:57.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:57.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:57.271284+0000) 2024-03-06T14:24:58.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:58.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:58.271519+0000) 2024-03-06T14:24:59.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:24:59.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:24:59.271807+0000) 2024-03-06T14:25:00.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:00.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:00.272001+0000) 2024-03-06T14:25:01.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:01.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:01.272190+0000) 2024-03-06T14:25:02.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:02.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:02.272411+0000) 2024-03-06T14:25:03.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:03.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:03.272650+0000) 2024-03-06T14:25:04.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:04.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:04.272892+0000) 2024-03-06T14:25:05.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:05.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:05.273105+0000) 2024-03-06T14:25:05.281 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:25:05.289 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:25:06.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:06.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:06.273365+0000) 2024-03-06T14:25:07.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:07.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:07.273690+0000) 2024-03-06T14:25:08.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:08.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:08.273912+0000) 2024-03-06T14:25:09.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:09.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:09.274132+0000) 2024-03-06T14:25:10.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:10.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:10.274363+0000) 2024-03-06T14:25:11.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:11.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:11.274650+0000) 2024-03-06T14:25:12.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:12.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:12.274855+0000) 2024-03-06T14:25:13.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:13.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:13.275090+0000) 2024-03-06T14:25:14.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:14.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:14.275282+0000) 2024-03-06T14:25:15.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:15.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:15.275514+0000) 2024-03-06T14:25:16.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:16.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:16.275762+0000) 2024-03-06T14:25:17.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:17.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:17.275996+0000) 2024-03-06T14:25:18.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:18.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:18.276231+0000) 2024-03-06T14:25:19.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:19.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:19.276456+0000) 2024-03-06T14:25:20.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:20.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:20.276660+0000) 2024-03-06T14:25:21.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:21.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:21.276847+0000) 2024-03-06T14:25:22.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:22.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:22.277087+0000) 2024-03-06T14:25:23.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:23.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:23.277323+0000) 2024-03-06T14:25:24.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:24.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:24.277615+0000) 2024-03-06T14:25:25.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:25.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:25.277861+0000) 2024-03-06T14:25:26.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:26.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:26.278111+0000) 2024-03-06T14:25:27.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:27.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:27.278343+0000) 2024-03-06T14:25:28.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:28.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:28.278637+0000) 2024-03-06T14:25:29.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:29.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:29.278844+0000) 2024-03-06T14:25:30.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:30.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:30.279082+0000) 2024-03-06T14:25:31.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:31.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:31.279311+0000) 2024-03-06T14:25:32.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:32.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:32.279527+0000) 2024-03-06T14:25:33.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:33.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:33.279799+0000) 2024-03-06T14:25:34.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:34.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:34.280028+0000) 2024-03-06T14:25:35.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:35.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:35.280261+0000) 2024-03-06T14:25:35.415 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:25:35.422 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:25:36.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:36.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:36.280490+0000) 2024-03-06T14:25:37.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:37.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:37.280716+0000) 2024-03-06T14:25:38.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:38.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:38.280945+0000) 2024-03-06T14:25:39.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:39.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:39.281135+0000) 2024-03-06T14:25:40.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:40.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:40.281384+0000) 2024-03-06T14:25:41.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:41.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:41.281621+0000) 2024-03-06T14:25:42.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:42.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:42.281812+0000) 2024-03-06T14:25:43.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:43.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:43.282036+0000) 2024-03-06T14:25:44.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:44.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:44.282262+0000) 2024-03-06T14:25:45.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:45.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:45.282487+0000) 2024-03-06T14:25:46.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:46.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:46.282694+0000) 2024-03-06T14:25:47.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:47.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:47.282890+0000) 2024-03-06T14:25:48.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:48.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:48.283115+0000) 2024-03-06T14:25:49.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:49.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:49.283388+0000) 2024-03-06T14:25:50.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:50.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:50.283682+0000) 2024-03-06T14:25:51.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:51.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:51.283923+0000) 2024-03-06T14:25:52.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:52.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:52.284206+0000) 2024-03-06T14:25:53.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:53.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:53.284481+0000) 2024-03-06T14:25:54.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:54.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:54.284699+0000) 2024-03-06T14:25:55.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:55.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:55.284897+0000) 2024-03-06T14:25:56.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:56.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:56.285139+0000) 2024-03-06T14:25:57.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:57.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:57.285377+0000) 2024-03-06T14:25:58.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:58.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:58.285701+0000) 2024-03-06T14:25:59.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:25:59.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:25:59.285893+0000) 2024-03-06T14:26:00.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:00.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:00.286082+0000) 2024-03-06T14:26:01.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:01.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:01.286342+0000) 2024-03-06T14:26:02.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:02.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:02.286650+0000) 2024-03-06T14:26:03.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:03.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:03.286889+0000) 2024-03-06T14:26:04.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:04.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:04.287113+0000) 2024-03-06T14:26:05.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:05.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:05.287370+0000) 2024-03-06T14:26:05.547 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:26:05.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:26:06.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:06.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:06.287568+0000) 2024-03-06T14:26:07.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:07.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:07.287806+0000) 2024-03-06T14:26:08.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:08.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:08.288046+0000) 2024-03-06T14:26:09.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:09.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:09.288245+0000) 2024-03-06T14:26:10.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:10.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:10.288487+0000) 2024-03-06T14:26:11.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:11.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:11.288692+0000) 2024-03-06T14:26:12.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:12.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:12.288873+0000) 2024-03-06T14:26:13.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:13.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:13.289095+0000) 2024-03-06T14:26:14.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:14.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:14.289320+0000) 2024-03-06T14:26:15.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:15.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:15.289581+0000) 2024-03-06T14:26:16.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:16.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:16.289807+0000) 2024-03-06T14:26:17.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:17.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:17.290024+0000) 2024-03-06T14:26:18.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:18.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:18.290252+0000) 2024-03-06T14:26:19.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:19.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:19.290475+0000) 2024-03-06T14:26:20.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:20.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:20.290676+0000) 2024-03-06T14:26:21.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:21.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:21.290887+0000) 2024-03-06T14:26:22.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:22.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:22.291113+0000) 2024-03-06T14:26:23.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:23.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:23.291336+0000) 2024-03-06T14:26:24.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:24.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:24.291624+0000) 2024-03-06T14:26:25.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:25.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:25.291845+0000) 2024-03-06T14:26:26.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:26.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:26.292054+0000) 2024-03-06T14:26:27.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:27.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:27.292232+0000) 2024-03-06T14:26:28.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:28.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:28.292468+0000) 2024-03-06T14:26:29.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:29.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:29.292693+0000) 2024-03-06T14:26:30.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:30.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:30.292891+0000) 2024-03-06T14:26:31.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:31.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:31.293116+0000) 2024-03-06T14:26:32.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:32.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:32.293336+0000) 2024-03-06T14:26:33.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:33.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:33.293517+0000) 2024-03-06T14:26:34.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:34.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:34.293739+0000) 2024-03-06T14:26:35.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:35.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:35.293967+0000) 2024-03-06T14:26:35.796 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:26:35.804 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:26:36.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:36.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:36.294165+0000) 2024-03-06T14:26:37.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:37.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:37.294324+0000) 2024-03-06T14:26:38.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:38.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:38.294640+0000) 2024-03-06T14:26:39.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:39.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:39.294868+0000) 2024-03-06T14:26:40.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:40.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:40.295088+0000) 2024-03-06T14:26:41.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:41.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:41.295291+0000) 2024-03-06T14:26:42.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:42.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:42.295527+0000) 2024-03-06T14:26:43.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:43.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:43.295745+0000) 2024-03-06T14:26:44.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:44.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:44.295937+0000) 2024-03-06T14:26:45.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:45.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:45.296155+0000) 2024-03-06T14:26:46.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:46.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:46.296485+0000) 2024-03-06T14:26:47.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:47.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:47.296743+0000) 2024-03-06T14:26:48.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:48.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:48.296960+0000) 2024-03-06T14:26:49.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:49.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:49.297190+0000) 2024-03-06T14:26:50.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:50.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:50.297413+0000) 2024-03-06T14:26:51.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:51.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:51.297694+0000) 2024-03-06T14:26:52.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:52.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:52.297894+0000) 2024-03-06T14:26:53.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:53.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:53.298096+0000) 2024-03-06T14:26:54.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:54.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:54.298287+0000) 2024-03-06T14:26:55.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:55.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:55.298583+0000) 2024-03-06T14:26:56.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:56.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:56.298824+0000) 2024-03-06T14:26:57.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:57.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:57.299103+0000) 2024-03-06T14:26:58.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:58.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:58.299389+0000) 2024-03-06T14:26:59.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:26:59.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:26:59.299682+0000) 2024-03-06T14:27:00.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:00.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:00.299928+0000) 2024-03-06T14:27:01.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:01.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:01.300137+0000) 2024-03-06T14:27:02.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:02.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:02.300303+0000) 2024-03-06T14:27:03.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:03.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:03.300575+0000) 2024-03-06T14:27:04.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:04.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:04.300778+0000) 2024-03-06T14:27:05.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:05.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:05.301004+0000) 2024-03-06T14:27:06.017 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:27:06.025 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:27:06.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:06.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:06.301229+0000) 2024-03-06T14:27:07.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:07.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:07.301456+0000) 2024-03-06T14:27:08.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:08.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:08.301672+0000) 2024-03-06T14:27:09.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:09.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:09.301859+0000) 2024-03-06T14:27:10.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:10.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:10.302089+0000) 2024-03-06T14:27:11.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:11.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:11.302312+0000) 2024-03-06T14:27:12.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:12.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:12.302506+0000) 2024-03-06T14:27:13.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:13.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:13.302752+0000) 2024-03-06T14:27:14.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:14.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:14.302942+0000) 2024-03-06T14:27:15.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:15.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:15.303166+0000) 2024-03-06T14:27:16.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:16.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:16.303395+0000) 2024-03-06T14:27:17.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:17.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:17.303634+0000) 2024-03-06T14:27:18.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:18.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:18.303836+0000) 2024-03-06T14:27:19.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:19.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:19.304064+0000) 2024-03-06T14:27:20.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:20.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:20.304288+0000) 2024-03-06T14:27:21.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:21.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:21.304513+0000) 2024-03-06T14:27:22.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:22.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:22.304763+0000) 2024-03-06T14:27:23.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:23.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:23.304954+0000) 2024-03-06T14:27:24.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:24.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:24.305201+0000) 2024-03-06T14:27:25.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:25.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:25.305444+0000) 2024-03-06T14:27:26.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:26.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:26.305652+0000) 2024-03-06T14:27:27.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:27.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:27.305855+0000) 2024-03-06T14:27:28.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:28.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:28.306063+0000) 2024-03-06T14:27:29.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:29.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:29.306315+0000) 2024-03-06T14:27:30.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:30.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:30.306574+0000) 2024-03-06T14:27:31.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:31.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:31.306820+0000) 2024-03-06T14:27:32.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:32.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:32.307058+0000) 2024-03-06T14:27:33.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:33.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:33.307305+0000) 2024-03-06T14:27:34.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:34.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:34.307604+0000) 2024-03-06T14:27:35.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:35.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:35.307823+0000) 2024-03-06T14:27:36.197 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:27:36.205 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:27:36.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:36.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:36.307983+0000) 2024-03-06T14:27:37.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:37.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:37.308208+0000) 2024-03-06T14:27:38.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:38.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:38.308449+0000) 2024-03-06T14:27:39.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:39.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:39.308753+0000) 2024-03-06T14:27:40.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:40.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:40.308968+0000) 2024-03-06T14:27:41.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:41.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:41.309171+0000) 2024-03-06T14:27:42.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:42.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:42.309401+0000) 2024-03-06T14:27:43.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:43.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:43.309651+0000) 2024-03-06T14:27:44.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:44.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:44.309867+0000) 2024-03-06T14:27:45.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:45.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:45.310048+0000) 2024-03-06T14:27:46.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:46.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:46.310272+0000) 2024-03-06T14:27:47.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:47.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:47.310511+0000) 2024-03-06T14:27:48.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:48.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:48.310786+0000) 2024-03-06T14:27:49.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:49.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:49.311017+0000) 2024-03-06T14:27:50.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:50.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:50.311252+0000) 2024-03-06T14:27:51.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:51.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:51.311472+0000) 2024-03-06T14:27:52.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:52.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:52.311689+0000) 2024-03-06T14:27:53.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:53.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:53.311884+0000) 2024-03-06T14:27:54.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:54.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:54.312073+0000) 2024-03-06T14:27:55.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:55.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:55.312307+0000) 2024-03-06T14:27:56.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:56.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:56.312476+0000) 2024-03-06T14:27:57.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:57.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:57.312711+0000) 2024-03-06T14:27:58.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:58.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:58.312911+0000) 2024-03-06T14:27:59.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:27:59.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:27:59.313134+0000) 2024-03-06T14:28:00.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:00.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:00.313361+0000) 2024-03-06T14:28:01.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:01.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:01.313633+0000) 2024-03-06T14:28:02.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:02.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:02.313797+0000) 2024-03-06T14:28:03.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:03.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:03.314036+0000) 2024-03-06T14:28:04.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:04.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:04.314192+0000) 2024-03-06T14:28:05.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:05.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:05.314437+0000) 2024-03-06T14:28:06.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:06.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:06.314611+0000) 2024-03-06T14:28:06.410 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:28:06.418 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:28:07.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:07.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:07.314849+0000) 2024-03-06T14:28:08.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:08.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:08.315051+0000) 2024-03-06T14:28:09.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:09.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:09.315286+0000) 2024-03-06T14:28:10.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:10.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:10.315467+0000) 2024-03-06T14:28:11.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:11.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:11.315679+0000) 2024-03-06T14:28:12.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:12.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:12.315839+0000) 2024-03-06T14:28:13.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:13.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:13.316038+0000) 2024-03-06T14:28:14.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:14.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:14.316238+0000) 2024-03-06T14:28:15.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:15.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:15.316419+0000) 2024-03-06T14:28:16.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:16.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:16.316672+0000) 2024-03-06T14:28:17.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:17.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:17.316847+0000) 2024-03-06T14:28:18.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:18.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:18.317076+0000) 2024-03-06T14:28:19.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:19.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:19.317301+0000) 2024-03-06T14:28:20.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:20.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:20.317559+0000) 2024-03-06T14:28:21.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:21.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:21.317754+0000) 2024-03-06T14:28:22.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:22.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:22.317948+0000) 2024-03-06T14:28:23.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:23.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:23.318174+0000) 2024-03-06T14:28:24.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:24.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:24.318364+0000) 2024-03-06T14:28:25.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:25.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:25.318618+0000) 2024-03-06T14:28:26.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:26.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:26.318822+0000) 2024-03-06T14:28:27.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:27.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:27.319054+0000) 2024-03-06T14:28:28.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:28.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:28.319226+0000) 2024-03-06T14:28:29.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:29.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:29.319466+0000) 2024-03-06T14:28:30.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:30.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:30.319699+0000) 2024-03-06T14:28:31.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:31.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:31.319858+0000) 2024-03-06T14:28:32.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:32.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:32.320056+0000) 2024-03-06T14:28:33.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:33.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:33.320279+0000) 2024-03-06T14:28:34.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:34.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:34.320497+0000) 2024-03-06T14:28:35.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:35.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:35.320716+0000) 2024-03-06T14:28:36.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:36.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:36.320959+0000) 2024-03-06T14:28:36.539 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:28:36.547 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:28:37.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:37.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:37.321229+0000) 2024-03-06T14:28:38.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:38.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:38.321562+0000) 2024-03-06T14:28:39.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:39.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:39.321765+0000) 2024-03-06T14:28:40.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:40.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:40.321949+0000) 2024-03-06T14:28:41.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:41.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:41.322173+0000) 2024-03-06T14:28:42.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:42.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:42.322393+0000) 2024-03-06T14:28:43.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:43.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:43.322645+0000) 2024-03-06T14:28:44.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:44.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:44.322836+0000) 2024-03-06T14:28:45.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:45.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:45.323029+0000) 2024-03-06T14:28:46.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:46.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:46.323256+0000) 2024-03-06T14:28:47.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:47.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:47.323484+0000) 2024-03-06T14:28:48.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:48.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:48.323676+0000) 2024-03-06T14:28:49.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:49.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:49.323870+0000) 2024-03-06T14:28:50.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:50.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:50.324096+0000) 2024-03-06T14:28:51.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:51.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:51.324321+0000) 2024-03-06T14:28:52.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:52.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:52.324585+0000) 2024-03-06T14:28:53.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:53.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:53.324740+0000) 2024-03-06T14:28:54.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:54.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:54.324942+0000) 2024-03-06T14:28:55.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:55.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:55.325133+0000) 2024-03-06T14:28:56.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:56.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:56.325351+0000) 2024-03-06T14:28:57.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:57.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:57.325611+0000) 2024-03-06T14:28:58.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:58.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:58.325775+0000) 2024-03-06T14:28:59.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:28:59.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:28:59.326010+0000) 2024-03-06T14:29:00.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:00.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:00.326208+0000) 2024-03-06T14:29:01.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:01.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:01.326351+0000) 2024-03-06T14:29:02.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:02.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:02.326595+0000) 2024-03-06T14:29:03.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:03.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:03.326782+0000) 2024-03-06T14:29:04.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:04.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:04.326972+0000) 2024-03-06T14:29:05.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:05.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:05.327197+0000) 2024-03-06T14:29:06.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:06.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:06.327417+0000) 2024-03-06T14:29:06.677 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:29:06.686 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:29:07.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:07.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:07.327577+0000) 2024-03-06T14:29:08.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:08.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:08.327738+0000) 2024-03-06T14:29:09.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:09.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:09.327935+0000) 2024-03-06T14:29:10.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:10.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:10.328139+0000) 2024-03-06T14:29:11.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:11.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:11.328372+0000) 2024-03-06T14:29:12.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:12.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:12.328626+0000) 2024-03-06T14:29:13.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:13.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:13.328809+0000) 2024-03-06T14:29:14.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:14.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:14.328987+0000) 2024-03-06T14:29:15.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:15.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:15.329265+0000) 2024-03-06T14:29:16.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:16.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:16.329494+0000) 2024-03-06T14:29:17.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:17.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:17.329738+0000) 2024-03-06T14:29:18.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:18.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:18.329909+0000) 2024-03-06T14:29:19.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:19.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:19.330133+0000) 2024-03-06T14:29:20.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:20.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:20.330366+0000) 2024-03-06T14:29:21.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:21.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:21.330612+0000) 2024-03-06T14:29:22.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:22.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:22.330797+0000) 2024-03-06T14:29:23.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:23.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:23.331021+0000) 2024-03-06T14:29:24.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:24.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:24.331210+0000) 2024-03-06T14:29:25.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:25.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:25.331437+0000) 2024-03-06T14:29:26.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:26.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:26.331672+0000) 2024-03-06T14:29:27.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:27.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:27.331876+0000) 2024-03-06T14:29:28.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:28.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:28.332085+0000) 2024-03-06T14:29:29.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:29.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:29.332309+0000) 2024-03-06T14:29:30.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:30.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:30.332496+0000) 2024-03-06T14:29:31.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:31.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:31.332725+0000) 2024-03-06T14:29:32.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:32.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:32.332878+0000) 2024-03-06T14:29:33.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:33.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:33.333066+0000) 2024-03-06T14:29:34.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:34.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:34.333295+0000) 2024-03-06T14:29:35.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:35.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:35.333498+0000) 2024-03-06T14:29:36.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:36.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:36.333753+0000) 2024-03-06T14:29:36.865 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:29:36.873 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:29:37.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:37.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:37.333957+0000) 2024-03-06T14:29:38.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:38.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:38.334199+0000) 2024-03-06T14:29:39.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:39.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:39.334414+0000) 2024-03-06T14:29:40.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:40.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:40.334648+0000) 2024-03-06T14:29:41.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:41.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:41.334852+0000) 2024-03-06T14:29:42.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:42.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:42.335052+0000) 2024-03-06T14:29:43.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:43.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:43.335239+0000) 2024-03-06T14:29:44.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:44.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:44.335458+0000) 2024-03-06T14:29:45.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:45.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:45.335684+0000) 2024-03-06T14:29:46.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:46.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:46.335870+0000) 2024-03-06T14:29:47.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:47.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:47.336067+0000) 2024-03-06T14:29:48.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:48.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:48.336300+0000) 2024-03-06T14:29:49.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:49.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:49.336573+0000) 2024-03-06T14:29:50.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:50.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:50.336738+0000) 2024-03-06T14:29:51.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:51.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:51.336942+0000) 2024-03-06T14:29:52.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:52.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:52.337148+0000) 2024-03-06T14:29:53.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:53.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:53.337337+0000) 2024-03-06T14:29:54.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:54.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:54.337589+0000) 2024-03-06T14:29:55.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:55.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:55.337821+0000) 2024-03-06T14:29:56.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:56.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:56.338006+0000) 2024-03-06T14:29:57.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:57.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:57.338249+0000) 2024-03-06T14:29:58.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:58.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:58.338502+0000) 2024-03-06T14:29:59.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:29:59.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:29:59.338782+0000) 2024-03-06T14:30:00.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:00.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:00.338988+0000) 2024-03-06T14:30:01.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:01.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:01.339230+0000) 2024-03-06T14:30:02.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:02.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:02.339445+0000) 2024-03-06T14:30:03.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:03.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:03.339684+0000) 2024-03-06T14:30:04.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:04.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:04.339914+0000) 2024-03-06T14:30:05.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:05.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:05.340129+0000) 2024-03-06T14:30:06.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:06.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:06.340374+0000) 2024-03-06T14:30:07.071 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:30:07.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:30:07.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:07.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:07.340681+0000) 2024-03-06T14:30:08.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:08.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:08.340933+0000) 2024-03-06T14:30:09.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:09.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:09.341152+0000) 2024-03-06T14:30:10.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:10.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:10.341386+0000) 2024-03-06T14:30:11.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:11.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:11.341658+0000) 2024-03-06T14:30:12.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:12.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:12.341884+0000) 2024-03-06T14:30:13.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:13.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:13.342117+0000) 2024-03-06T14:30:14.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:14.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:14.342353+0000) 2024-03-06T14:30:15.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:15.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:15.342650+0000) 2024-03-06T14:30:16.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:16.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:16.342890+0000) 2024-03-06T14:30:17.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:17.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:17.343120+0000) 2024-03-06T14:30:18.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:18.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:18.343349+0000) 2024-03-06T14:30:19.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:19.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:19.343615+0000) 2024-03-06T14:30:20.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:20.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:20.343856+0000) 2024-03-06T14:30:21.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:21.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:21.344031+0000) 2024-03-06T14:30:22.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:22.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:22.344232+0000) 2024-03-06T14:30:23.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:23.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:23.344407+0000) 2024-03-06T14:30:24.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:24.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:24.344659+0000) 2024-03-06T14:30:25.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:25.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:25.344904+0000) 2024-03-06T14:30:26.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:26.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:26.345128+0000) 2024-03-06T14:30:27.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:27.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:27.345350+0000) 2024-03-06T14:30:28.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:28.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:28.345638+0000) 2024-03-06T14:30:29.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:29.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:29.345833+0000) 2024-03-06T14:30:30.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:30.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:30.346078+0000) 2024-03-06T14:30:31.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:31.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:31.346268+0000) 2024-03-06T14:30:32.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:32.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:32.346452+0000) 2024-03-06T14:30:33.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:33.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:33.346687+0000) 2024-03-06T14:30:34.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:34.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:34.346917+0000) 2024-03-06T14:30:35.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:35.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:35.347160+0000) 2024-03-06T14:30:36.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:36.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:36.347398+0000) 2024-03-06T14:30:37.217 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:30:37.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:30:37.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:37.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:37.347641+0000) 2024-03-06T14:30:38.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:38.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:38.347862+0000) 2024-03-06T14:30:39.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:39.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:39.348075+0000) 2024-03-06T14:30:40.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:40.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:40.348310+0000) 2024-03-06T14:30:41.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:41.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:41.348559+0000) 2024-03-06T14:30:42.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:42.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:42.348748+0000) 2024-03-06T14:30:43.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:43.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:43.348950+0000) 2024-03-06T14:30:44.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:44.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:44.349231+0000) 2024-03-06T14:30:45.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:45.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:45.349507+0000) 2024-03-06T14:30:46.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:46.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:46.349789+0000) 2024-03-06T14:30:47.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:47.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:47.349986+0000) 2024-03-06T14:30:48.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:48.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:48.350200+0000) 2024-03-06T14:30:49.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:49.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:49.350432+0000) 2024-03-06T14:30:50.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:50.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:50.350633+0000) 2024-03-06T14:30:51.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:51.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:51.350819+0000) 2024-03-06T14:30:52.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:52.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:52.351048+0000) 2024-03-06T14:30:53.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:53.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:53.351276+0000) 2024-03-06T14:30:54.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:54.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:54.351473+0000) 2024-03-06T14:30:55.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:55.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:55.351693+0000) 2024-03-06T14:30:56.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:56.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:56.351969+0000) 2024-03-06T14:30:57.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:57.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:57.352174+0000) 2024-03-06T14:30:58.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:58.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:58.352394+0000) 2024-03-06T14:30:59.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:30:59.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:30:59.352642+0000) 2024-03-06T14:31:00.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:00.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:00.352872+0000) 2024-03-06T14:31:01.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:01.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:01.353084+0000) 2024-03-06T14:31:02.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:02.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:02.353344+0000) 2024-03-06T14:31:03.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:03.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:03.353622+0000) 2024-03-06T14:31:04.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:04.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:04.353854+0000) 2024-03-06T14:31:05.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:05.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:05.354090+0000) 2024-03-06T14:31:06.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:06.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:06.354277+0000) 2024-03-06T14:31:07.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:07.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:07.354517+0000) 2024-03-06T14:31:07.438 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:31:07.446 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:31:08.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:08.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:08.354817+0000) 2024-03-06T14:31:09.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:09.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:09.355016+0000) 2024-03-06T14:31:10.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:10.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:10.355263+0000) 2024-03-06T14:31:11.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:11.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:11.355505+0000) 2024-03-06T14:31:12.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:12.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:12.355779+0000) 2024-03-06T14:31:13.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:13.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:13.355972+0000) 2024-03-06T14:31:14.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:14.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:14.356203+0000) 2024-03-06T14:31:15.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:15.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:15.356412+0000) 2024-03-06T14:31:16.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:16.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:16.356669+0000) 2024-03-06T14:31:17.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:17.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:17.356916+0000) 2024-03-06T14:31:18.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:18.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:18.357148+0000) 2024-03-06T14:31:19.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:19.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:19.357350+0000) 2024-03-06T14:31:20.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:20.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:20.357636+0000) 2024-03-06T14:31:21.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:21.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:21.357878+0000) 2024-03-06T14:31:22.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:22.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:22.358100+0000) 2024-03-06T14:31:23.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:23.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:23.358269+0000) 2024-03-06T14:31:24.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:24.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:24.358498+0000) 2024-03-06T14:31:25.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:25.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:25.358728+0000) 2024-03-06T14:31:26.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:26.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:26.358973+0000) 2024-03-06T14:31:27.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:27.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:27.359210+0000) 2024-03-06T14:31:28.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:28.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:28.359427+0000) 2024-03-06T14:31:29.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:29.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:29.359682+0000) 2024-03-06T14:31:30.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:30.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:30.359920+0000) 2024-03-06T14:31:31.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:31.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:31.360144+0000) 2024-03-06T14:31:32.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:32.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:32.360367+0000) 2024-03-06T14:31:33.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:33.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:33.360591+0000) 2024-03-06T14:31:34.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:34.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:34.360829+0000) 2024-03-06T14:31:35.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:35.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:35.361056+0000) 2024-03-06T14:31:36.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:36.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:36.361281+0000) 2024-03-06T14:31:37.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:37.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:37.361488+0000) 2024-03-06T14:31:37.660 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:31:37.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:31:38.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:38.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:38.361743+0000) 2024-03-06T14:31:39.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:39.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:39.361977+0000) 2024-03-06T14:31:40.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:40.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:40.362205+0000) 2024-03-06T14:31:41.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:41.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:41.362426+0000) 2024-03-06T14:31:42.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:42.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:42.362667+0000) 2024-03-06T14:31:43.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:43.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:43.362862+0000) 2024-03-06T14:31:44.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:44.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:44.363085+0000) 2024-03-06T14:31:45.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:45.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:45.363346+0000) 2024-03-06T14:31:46.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:46.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:46.363662+0000) 2024-03-06T14:31:47.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:47.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:47.363940+0000) 2024-03-06T14:31:48.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:48.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:48.364240+0000) 2024-03-06T14:31:49.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:49.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:49.364506+0000) 2024-03-06T14:31:50.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:50.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:50.364873+0000) 2024-03-06T14:31:51.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:51.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:51.365139+0000) 2024-03-06T14:31:52.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:52.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:52.365419+0000) 2024-03-06T14:31:53.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:53.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:53.365681+0000) 2024-03-06T14:31:54.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:54.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:54.365912+0000) 2024-03-06T14:31:55.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:55.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:55.366147+0000) 2024-03-06T14:31:56.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:56.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:56.366359+0000) 2024-03-06T14:31:57.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:57.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:57.366681+0000) 2024-03-06T14:31:58.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:58.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:58.366909+0000) 2024-03-06T14:31:59.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:31:59.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:31:59.367144+0000) 2024-03-06T14:32:00.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:00.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:00.367377+0000) 2024-03-06T14:32:01.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:01.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:01.367639+0000) 2024-03-06T14:32:02.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:02.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:02.367813+0000) 2024-03-06T14:32:03.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:03.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:03.368014+0000) 2024-03-06T14:32:04.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:04.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:04.368240+0000) 2024-03-06T14:32:05.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:05.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:05.368461+0000) 2024-03-06T14:32:06.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:06.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:06.368665+0000) 2024-03-06T14:32:07.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:07.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:07.368903+0000) 2024-03-06T14:32:07.882 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:32:07.892 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:32:08.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:08.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:08.369060+0000) 2024-03-06T14:32:09.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:09.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:09.369278+0000) 2024-03-06T14:32:10.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:10.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:10.369492+0000) 2024-03-06T14:32:11.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:11.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:11.369813+0000) 2024-03-06T14:32:12.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:12.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:12.370013+0000) 2024-03-06T14:32:13.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:13.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:13.370189+0000) 2024-03-06T14:32:14.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:14.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:14.370372+0000) 2024-03-06T14:32:15.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:15.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:15.370638+0000) 2024-03-06T14:32:16.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:16.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:16.370826+0000) 2024-03-06T14:32:17.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:17.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:17.371055+0000) 2024-03-06T14:32:18.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:18.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:18.371255+0000) 2024-03-06T14:32:19.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:19.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:19.371501+0000) 2024-03-06T14:32:20.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:20.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:20.371797+0000) 2024-03-06T14:32:21.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:21.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:21.371998+0000) 2024-03-06T14:32:22.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:22.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:22.372235+0000) 2024-03-06T14:32:23.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:23.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:23.372458+0000) 2024-03-06T14:32:24.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:24.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:24.372740+0000) 2024-03-06T14:32:25.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:25.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:25.373018+0000) 2024-03-06T14:32:26.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:26.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:26.373213+0000) 2024-03-06T14:32:27.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:27.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:27.373433+0000) 2024-03-06T14:32:28.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:28.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:28.373657+0000) 2024-03-06T14:32:29.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:29.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:29.373838+0000) 2024-03-06T14:32:30.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:30.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:30.374068+0000) 2024-03-06T14:32:31.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:31.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:31.374260+0000) 2024-03-06T14:32:32.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:32.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:32.374497+0000) 2024-03-06T14:32:33.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:33.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:33.374724+0000) 2024-03-06T14:32:34.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:34.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:34.374947+0000) 2024-03-06T14:32:35.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:35.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:35.375159+0000) 2024-03-06T14:32:36.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:36.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:36.375398+0000) 2024-03-06T14:32:37.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:37.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:37.375592+0000) 2024-03-06T14:32:38.090 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:32:38.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:32:38.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:38.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:38.375881+0000) 2024-03-06T14:32:39.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:39.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:39.376100+0000) 2024-03-06T14:32:40.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:40.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:40.376308+0000) 2024-03-06T14:32:41.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:41.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:41.376524+0000) 2024-03-06T14:32:42.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:42.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:42.376805+0000) 2024-03-06T14:32:43.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:43.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:43.377027+0000) 2024-03-06T14:32:44.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:44.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:44.377248+0000) 2024-03-06T14:32:45.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:45.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:45.377464+0000) 2024-03-06T14:32:46.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:46.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:46.377701+0000) 2024-03-06T14:32:47.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:47.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:47.377926+0000) 2024-03-06T14:32:48.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:48.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:48.378161+0000) 2024-03-06T14:32:49.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:49.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:49.378391+0000) 2024-03-06T14:32:50.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:50.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:50.378647+0000) 2024-03-06T14:32:51.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:51.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:51.378849+0000) 2024-03-06T14:32:52.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:52.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:52.379088+0000) 2024-03-06T14:32:53.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:53.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:53.379314+0000) 2024-03-06T14:32:54.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:54.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:54.379567+0000) 2024-03-06T14:32:55.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:55.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:55.379792+0000) 2024-03-06T14:32:56.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:56.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:56.379997+0000) 2024-03-06T14:32:57.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:57.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:57.380206+0000) 2024-03-06T14:32:58.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:58.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:58.380418+0000) 2024-03-06T14:32:59.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:32:59.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:32:59.380670+0000) 2024-03-06T14:33:00.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:00.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:00.380924+0000) 2024-03-06T14:33:01.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:01.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:01.381183+0000) 2024-03-06T14:33:02.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:02.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:02.381524+0000) 2024-03-06T14:33:03.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:03.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:03.381825+0000) 2024-03-06T14:33:04.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:04.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:04.382061+0000) 2024-03-06T14:33:05.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:05.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:05.382302+0000) 2024-03-06T14:33:06.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:06.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:06.382575+0000) 2024-03-06T14:33:07.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:07.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:07.382807+0000) 2024-03-06T14:33:08.250 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:33:08.258 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:33:08.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:08.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:08.383051+0000) 2024-03-06T14:33:09.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:09.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:09.383236+0000) 2024-03-06T14:33:10.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:10.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:10.383432+0000) 2024-03-06T14:33:11.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:11.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:11.383688+0000) 2024-03-06T14:33:12.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:12.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:12.383890+0000) 2024-03-06T14:33:13.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:13.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:13.384126+0000) 2024-03-06T14:33:14.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:14.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:14.384349+0000) 2024-03-06T14:33:15.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:15.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:15.384564+0000) 2024-03-06T14:33:16.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:16.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:16.384729+0000) 2024-03-06T14:33:17.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:17.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:17.384929+0000) 2024-03-06T14:33:18.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:18.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:18.385164+0000) 2024-03-06T14:33:19.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:19.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:19.385387+0000) 2024-03-06T14:33:20.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:20.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:20.385667+0000) 2024-03-06T14:33:21.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:21.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:21.385906+0000) 2024-03-06T14:33:22.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:22.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:22.386085+0000) 2024-03-06T14:33:23.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:23.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:23.386304+0000) 2024-03-06T14:33:24.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:24.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:24.386487+0000) 2024-03-06T14:33:25.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:25.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:25.386677+0000) 2024-03-06T14:33:26.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:26.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:26.386881+0000) 2024-03-06T14:33:27.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:27.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:27.387111+0000) 2024-03-06T14:33:28.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:28.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:28.387339+0000) 2024-03-06T14:33:29.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:29.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:29.387590+0000) 2024-03-06T14:33:30.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:30.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:30.387815+0000) 2024-03-06T14:33:31.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:31.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:31.388070+0000) 2024-03-06T14:33:32.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:32.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:32.388264+0000) 2024-03-06T14:33:33.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:33.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:33.388500+0000) 2024-03-06T14:33:34.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:34.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:34.388702+0000) 2024-03-06T14:33:35.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:35.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:35.388934+0000) 2024-03-06T14:33:36.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:36.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:36.389224+0000) 2024-03-06T14:33:37.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:37.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:37.389494+0000) 2024-03-06T14:33:38.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:38.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:38.389810+0000) 2024-03-06T14:33:38.454 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:33:38.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:33:39.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:39.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:39.390089+0000) 2024-03-06T14:33:40.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:40.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:40.390381+0000) 2024-03-06T14:33:41.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:41.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:41.390647+0000) 2024-03-06T14:33:42.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:42.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:42.390814+0000) 2024-03-06T14:33:43.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:43.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:43.391019+0000) 2024-03-06T14:33:44.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:44.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:44.391257+0000) 2024-03-06T14:33:45.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:45.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:45.391505+0000) 2024-03-06T14:33:46.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:46.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:46.391784+0000) 2024-03-06T14:33:47.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:47.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:47.392032+0000) 2024-03-06T14:33:48.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:48.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:48.392284+0000) 2024-03-06T14:33:49.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:49.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:49.392622+0000) 2024-03-06T14:33:50.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:50.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:50.392913+0000) 2024-03-06T14:33:51.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:51.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:51.393187+0000) 2024-03-06T14:33:52.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:52.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:52.393391+0000) 2024-03-06T14:33:53.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:53.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:53.393664+0000) 2024-03-06T14:33:54.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:54.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:54.393899+0000) 2024-03-06T14:33:55.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:55.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:55.394151+0000) 2024-03-06T14:33:56.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:56.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:56.394409+0000) 2024-03-06T14:33:57.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:57.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:57.394673+0000) 2024-03-06T14:33:58.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:58.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:58.394890+0000) 2024-03-06T14:33:59.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:33:59.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:33:59.395136+0000) 2024-03-06T14:34:00.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:00.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:00.395378+0000) 2024-03-06T14:34:01.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:01.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:01.395684+0000) 2024-03-06T14:34:02.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:02.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:02.395930+0000) 2024-03-06T14:34:03.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:03.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:03.396137+0000) 2024-03-06T14:34:04.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:04.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:04.396386+0000) 2024-03-06T14:34:05.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:05.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:05.396688+0000) 2024-03-06T14:34:06.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:06.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:06.396937+0000) 2024-03-06T14:34:07.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:07.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:07.397174+0000) 2024-03-06T14:34:08.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:08.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:08.397429+0000) 2024-03-06T14:34:08.667 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:34:08.675 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:34:09.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:09.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:09.397672+0000) 2024-03-06T14:34:10.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:10.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:10.397923+0000) 2024-03-06T14:34:11.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:11.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:11.398167+0000) 2024-03-06T14:34:12.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:12.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:12.398386+0000) 2024-03-06T14:34:13.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:13.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:13.398666+0000) 2024-03-06T14:34:14.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:14.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:14.398886+0000) 2024-03-06T14:34:15.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:15.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:15.399060+0000) 2024-03-06T14:34:16.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:16.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:16.399287+0000) 2024-03-06T14:34:17.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:17.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:17.399512+0000) 2024-03-06T14:34:18.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:18.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:18.399792+0000) 2024-03-06T14:34:19.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:19.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:19.400021+0000) 2024-03-06T14:34:20.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:20.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:20.400263+0000) 2024-03-06T14:34:21.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:21.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:21.400497+0000) 2024-03-06T14:34:22.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:22.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:22.400700+0000) 2024-03-06T14:34:23.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:23.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:23.400889+0000) 2024-03-06T14:34:24.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:24.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:24.401116+0000) 2024-03-06T14:34:25.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:25.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:25.401300+0000) 2024-03-06T14:34:26.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:26.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:26.401595+0000) 2024-03-06T14:34:27.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:27.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:27.401824+0000) 2024-03-06T14:34:28.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:28.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:28.402060+0000) 2024-03-06T14:34:29.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:29.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:29.402299+0000) 2024-03-06T14:34:30.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:30.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:30.402520+0000) 2024-03-06T14:34:31.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:31.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:31.402883+0000) 2024-03-06T14:34:32.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:32.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:32.403170+0000) 2024-03-06T14:34:33.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:33.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:33.403385+0000) 2024-03-06T14:34:34.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:34.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:34.403686+0000) 2024-03-06T14:34:35.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:35.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:35.403960+0000) 2024-03-06T14:34:36.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:36.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:36.404251+0000) 2024-03-06T14:34:37.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:37.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:37.404581+0000) 2024-03-06T14:34:38.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:38.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:38.404805+0000) 2024-03-06T14:34:38.881 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:34:38.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:34:39.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:39.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:39.405039+0000) 2024-03-06T14:34:40.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:40.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:40.405252+0000) 2024-03-06T14:34:41.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:41.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:41.405498+0000) 2024-03-06T14:34:42.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:42.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:42.405730+0000) 2024-03-06T14:34:43.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:43.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:43.405960+0000) 2024-03-06T14:34:44.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:44.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:44.406183+0000) 2024-03-06T14:34:45.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:45.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:45.406379+0000) 2024-03-06T14:34:46.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:46.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:46.406648+0000) 2024-03-06T14:34:47.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:47.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:47.406882+0000) 2024-03-06T14:34:48.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:48.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:48.407114+0000) 2024-03-06T14:34:49.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:49.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:49.407315+0000) 2024-03-06T14:34:50.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:50.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:50.407590+0000) 2024-03-06T14:34:51.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:51.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:51.407838+0000) 2024-03-06T14:34:52.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:52.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:52.408073+0000) 2024-03-06T14:34:53.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:53.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:53.408306+0000) 2024-03-06T14:34:54.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:54.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:54.408568+0000) 2024-03-06T14:34:55.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:55.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:55.408799+0000) 2024-03-06T14:34:56.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:56.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:56.409003+0000) 2024-03-06T14:34:57.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:57.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:57.409243+0000) 2024-03-06T14:34:58.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:58.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:58.409479+0000) 2024-03-06T14:34:59.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:34:59.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:34:59.409714+0000) 2024-03-06T14:35:00.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:00.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:00.409913+0000) 2024-03-06T14:35:01.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:01.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:01.410163+0000) 2024-03-06T14:35:02.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:02.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:02.410385+0000) 2024-03-06T14:35:03.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:03.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:03.410658+0000) 2024-03-06T14:35:04.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:04.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:04.410867+0000) 2024-03-06T14:35:05.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:05.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:05.411084+0000) 2024-03-06T14:35:06.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:06.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:06.411333+0000) 2024-03-06T14:35:07.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:07.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:07.411644+0000) 2024-03-06T14:35:08.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:08.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:08.411884+0000) 2024-03-06T14:35:09.052 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:35:09.060 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:35:09.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:09.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:09.412062+0000) 2024-03-06T14:35:10.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:10.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:10.412300+0000) 2024-03-06T14:35:11.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:11.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:11.412507+0000) 2024-03-06T14:35:12.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:12.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:12.412765+0000) 2024-03-06T14:35:13.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:13.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:13.412972+0000) 2024-03-06T14:35:14.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:14.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:14.413173+0000) 2024-03-06T14:35:15.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:15.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:15.413389+0000) 2024-03-06T14:35:16.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:16.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:16.413664+0000) 2024-03-06T14:35:17.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:17.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:17.413915+0000) 2024-03-06T14:35:18.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:18.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:18.414118+0000) 2024-03-06T14:35:19.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:19.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:19.414349+0000) 2024-03-06T14:35:20.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:20.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:20.414628+0000) 2024-03-06T14:35:21.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:21.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:21.414816+0000) 2024-03-06T14:35:22.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:22.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:22.415053+0000) 2024-03-06T14:35:23.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:23.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:23.415295+0000) 2024-03-06T14:35:24.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:24.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:24.415580+0000) 2024-03-06T14:35:25.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:25.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:25.415755+0000) 2024-03-06T14:35:26.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:26.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:26.415960+0000) 2024-03-06T14:35:27.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:27.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:27.416147+0000) 2024-03-06T14:35:28.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:28.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:28.416401+0000) 2024-03-06T14:35:29.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:29.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:29.416656+0000) 2024-03-06T14:35:30.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:30.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:30.416888+0000) 2024-03-06T14:35:31.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:31.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:31.417108+0000) 2024-03-06T14:35:32.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:32.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:32.417361+0000) 2024-03-06T14:35:33.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:33.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:33.417606+0000) 2024-03-06T14:35:34.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:34.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:34.417807+0000) 2024-03-06T14:35:35.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:35.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:35.418040+0000) 2024-03-06T14:35:36.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:36.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:36.418262+0000) 2024-03-06T14:35:37.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:37.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:37.418489+0000) 2024-03-06T14:35:38.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:38.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:38.418731+0000) 2024-03-06T14:35:39.258 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:35:39.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:35:39.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:39.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:39.418945+0000) 2024-03-06T14:35:40.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:40.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:40.419181+0000) 2024-03-06T14:35:41.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:41.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:41.419410+0000) 2024-03-06T14:35:42.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:42.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:42.419664+0000) 2024-03-06T14:35:43.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:43.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:43.419862+0000) 2024-03-06T14:35:44.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:44.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:44.420091+0000) 2024-03-06T14:35:45.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:45.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:45.420322+0000) 2024-03-06T14:35:46.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:46.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:46.420584+0000) 2024-03-06T14:35:47.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:47.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:47.420768+0000) 2024-03-06T14:35:48.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:48.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:48.421004+0000) 2024-03-06T14:35:49.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:49.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:49.421248+0000) 2024-03-06T14:35:50.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:50.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:50.421485+0000) 2024-03-06T14:35:51.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:51.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:51.421707+0000) 2024-03-06T14:35:52.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:52.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:52.421955+0000) 2024-03-06T14:35:53.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:53.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:53.422140+0000) 2024-03-06T14:35:54.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:54.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:54.422366+0000) 2024-03-06T14:35:55.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:55.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:55.422716+0000) 2024-03-06T14:35:56.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:56.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:56.422911+0000) 2024-03-06T14:35:57.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:57.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:57.423120+0000) 2024-03-06T14:35:58.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:58.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:58.423316+0000) 2024-03-06T14:35:59.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:35:59.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:35:59.423592+0000) 2024-03-06T14:36:00.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:00.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:00.423815+0000) 2024-03-06T14:36:01.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:01.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:01.424049+0000) 2024-03-06T14:36:02.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:02.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:02.424242+0000) 2024-03-06T14:36:03.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:03.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:03.424460+0000) 2024-03-06T14:36:04.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:04.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:04.424681+0000) 2024-03-06T14:36:05.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:05.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:05.424931+0000) 2024-03-06T14:36:06.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:06.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:06.425178+0000) 2024-03-06T14:36:07.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:07.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:07.425364+0000) 2024-03-06T14:36:08.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:08.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:08.425654+0000) 2024-03-06T14:36:09.403 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:36:09.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:36:09.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:09.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:09.425834+0000) 2024-03-06T14:36:10.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:10.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:10.426055+0000) 2024-03-06T14:36:11.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:11.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:11.426262+0000) 2024-03-06T14:36:12.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:12.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:12.426464+0000) 2024-03-06T14:36:13.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:13.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:13.426711+0000) 2024-03-06T14:36:14.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:14.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:14.426983+0000) 2024-03-06T14:36:15.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:15.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:15.427264+0000) 2024-03-06T14:36:16.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:16.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:16.427488+0000) 2024-03-06T14:36:17.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:17.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:17.427750+0000) 2024-03-06T14:36:18.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:18.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:18.427971+0000) 2024-03-06T14:36:19.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:19.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:19.428197+0000) 2024-03-06T14:36:20.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:20.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:20.428429+0000) 2024-03-06T14:36:21.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:21.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:21.428664+0000) 2024-03-06T14:36:22.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:22.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:22.428869+0000) 2024-03-06T14:36:23.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:23.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:23.429093+0000) 2024-03-06T14:36:24.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:24.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:24.429291+0000) 2024-03-06T14:36:25.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:25.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:25.429569+0000) 2024-03-06T14:36:26.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:26.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:26.429762+0000) 2024-03-06T14:36:27.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:27.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:27.429973+0000) 2024-03-06T14:36:28.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:28.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:28.430230+0000) 2024-03-06T14:36:29.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:29.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:29.430461+0000) 2024-03-06T14:36:30.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:30.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:30.430665+0000) 2024-03-06T14:36:31.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:31.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:31.430843+0000) 2024-03-06T14:36:32.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:32.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:32.431078+0000) 2024-03-06T14:36:33.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:33.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:33.431260+0000) 2024-03-06T14:36:34.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:34.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:34.431438+0000) 2024-03-06T14:36:35.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:35.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:35.431667+0000) 2024-03-06T14:36:36.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:36.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:36.431878+0000) 2024-03-06T14:36:37.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:37.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:37.432130+0000) 2024-03-06T14:36:38.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:38.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:38.432402+0000) 2024-03-06T14:36:39.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:39.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:39.432680+0000) 2024-03-06T14:36:39.625 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:36:39.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:36:40.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:40.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:40.432962+0000) 2024-03-06T14:36:41.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:41.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:41.433241+0000) 2024-03-06T14:36:42.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:42.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:42.433528+0000) 2024-03-06T14:36:43.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:43.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:43.433859+0000) 2024-03-06T14:36:44.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:44.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:44.434138+0000) 2024-03-06T14:36:45.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:45.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:45.434422+0000) 2024-03-06T14:36:46.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:46.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:46.434702+0000) 2024-03-06T14:36:47.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:47.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:47.434983+0000) 2024-03-06T14:36:48.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:48.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:48.435333+0000) 2024-03-06T14:36:49.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:49.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:49.435635+0000) 2024-03-06T14:36:50.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:50.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:50.435921+0000) 2024-03-06T14:36:51.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:51.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:51.436198+0000) 2024-03-06T14:36:52.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:52.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:52.436398+0000) 2024-03-06T14:36:53.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:53.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:53.436665+0000) 2024-03-06T14:36:54.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:54.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:54.436890+0000) 2024-03-06T14:36:55.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:55.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:55.437118+0000) 2024-03-06T14:36:56.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:56.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:56.437348+0000) 2024-03-06T14:36:57.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:57.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:57.437633+0000) 2024-03-06T14:36:58.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:58.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:58.437839+0000) 2024-03-06T14:36:59.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:36:59.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:36:59.438070+0000) 2024-03-06T14:37:00.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:00.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:00.438301+0000) 2024-03-06T14:37:01.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:01.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:01.438571+0000) 2024-03-06T14:37:02.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:02.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:02.438806+0000) 2024-03-06T14:37:03.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:03.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:03.439040+0000) 2024-03-06T14:37:04.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:04.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:04.439236+0000) 2024-03-06T14:37:05.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:05.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:05.439484+0000) 2024-03-06T14:37:06.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:06.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:06.439725+0000) 2024-03-06T14:37:07.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:07.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:07.439922+0000) 2024-03-06T14:37:08.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:08.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:08.440179+0000) 2024-03-06T14:37:09.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:09.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:09.440472+0000) 2024-03-06T14:37:09.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:37:09.838 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:37:10.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:10.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:10.440766+0000) 2024-03-06T14:37:11.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:11.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:11.441052+0000) 2024-03-06T14:37:12.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:12.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:12.441339+0000) 2024-03-06T14:37:13.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:13.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:13.441650+0000) 2024-03-06T14:37:14.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:14.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:14.441923+0000) 2024-03-06T14:37:15.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:15.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:15.442120+0000) 2024-03-06T14:37:16.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:16.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:16.442323+0000) 2024-03-06T14:37:17.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:17.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:17.442529+0000) 2024-03-06T14:37:18.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:18.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:18.442813+0000) 2024-03-06T14:37:19.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:19.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:19.443059+0000) 2024-03-06T14:37:20.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:20.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:20.443351+0000) 2024-03-06T14:37:21.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:21.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:21.443671+0000) 2024-03-06T14:37:22.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:22.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:22.443902+0000) 2024-03-06T14:37:23.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:23.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:23.444141+0000) 2024-03-06T14:37:24.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:24.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:24.444368+0000) 2024-03-06T14:37:25.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:25.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:25.444593+0000) 2024-03-06T14:37:26.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:26.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:26.444810+0000) 2024-03-06T14:37:27.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:27.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:27.445020+0000) 2024-03-06T14:37:28.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:28.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:28.445232+0000) 2024-03-06T14:37:29.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:29.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:29.445436+0000) 2024-03-06T14:37:30.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:30.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:30.445667+0000) 2024-03-06T14:37:31.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:31.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:31.445909+0000) 2024-03-06T14:37:32.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:32.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:32.446127+0000) 2024-03-06T14:37:33.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:33.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:33.446376+0000) 2024-03-06T14:37:34.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:34.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:34.446662+0000) 2024-03-06T14:37:35.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:35.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:35.446915+0000) 2024-03-06T14:37:36.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:36.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:36.447161+0000) 2024-03-06T14:37:37.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:37.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:37.447422+0000) 2024-03-06T14:37:38.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:38.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:38.447679+0000) 2024-03-06T14:37:39.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:39.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:39.447924+0000) 2024-03-06T14:37:40.051 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:37:40.061 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:37:40.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:40.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:40.448164+0000) 2024-03-06T14:37:41.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:41.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:41.448359+0000) 2024-03-06T14:37:42.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:42.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:42.448693+0000) 2024-03-06T14:37:43.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:43.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:43.448887+0000) 2024-03-06T14:37:44.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:44.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:44.449086+0000) 2024-03-06T14:37:45.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:45.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:45.449291+0000) 2024-03-06T14:37:46.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:46.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:46.449513+0000) 2024-03-06T14:37:47.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:47.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:47.449712+0000) 2024-03-06T14:37:48.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:48.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:48.449917+0000) 2024-03-06T14:37:49.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:49.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:49.450126+0000) 2024-03-06T14:37:50.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:50.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:50.450356+0000) 2024-03-06T14:37:51.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:51.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:51.450623+0000) 2024-03-06T14:37:52.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:52.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:52.450868+0000) 2024-03-06T14:37:53.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:53.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:53.451100+0000) 2024-03-06T14:37:54.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:54.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:54.451282+0000) 2024-03-06T14:37:55.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:55.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:55.451479+0000) 2024-03-06T14:37:56.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:56.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:56.451700+0000) 2024-03-06T14:37:57.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:57.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:57.451903+0000) 2024-03-06T14:37:58.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:58.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:58.452171+0000) 2024-03-06T14:37:59.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:37:59.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:37:59.452420+0000) 2024-03-06T14:38:00.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:00.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:00.452681+0000) 2024-03-06T14:38:01.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:01.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:01.452874+0000) 2024-03-06T14:38:02.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:02.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:02.453108+0000) 2024-03-06T14:38:03.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:03.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:03.453292+0000) 2024-03-06T14:38:04.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:04.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:04.453518+0000) 2024-03-06T14:38:05.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:05.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:05.453767+0000) 2024-03-06T14:38:06.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:06.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:06.453991+0000) 2024-03-06T14:38:07.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:07.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:07.454198+0000) 2024-03-06T14:38:08.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:08.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:08.454443+0000) 2024-03-06T14:38:09.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:09.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:09.454669+0000) 2024-03-06T14:38:10.264 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:38:10.275 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:38:10.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:10.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:10.454885+0000) 2024-03-06T14:38:11.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:11.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:11.455117+0000) 2024-03-06T14:38:12.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:12.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:12.455356+0000) 2024-03-06T14:38:13.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:13.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:13.455647+0000) 2024-03-06T14:38:14.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:14.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:14.455852+0000) 2024-03-06T14:38:15.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:15.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:15.456100+0000) 2024-03-06T14:38:16.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:16.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:16.456342+0000) 2024-03-06T14:38:17.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:17.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:17.456631+0000) 2024-03-06T14:38:18.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:18.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:18.456884+0000) 2024-03-06T14:38:19.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:19.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:19.457081+0000) 2024-03-06T14:38:20.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:20.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:20.457309+0000) 2024-03-06T14:38:21.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:21.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:21.457569+0000) 2024-03-06T14:38:22.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:22.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:22.457777+0000) 2024-03-06T14:38:23.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:23.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:23.458015+0000) 2024-03-06T14:38:24.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:24.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:24.458297+0000) 2024-03-06T14:38:25.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:25.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:25.458568+0000) 2024-03-06T14:38:26.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:26.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:26.458765+0000) 2024-03-06T14:38:27.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:27.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:27.459001+0000) 2024-03-06T14:38:28.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:28.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:28.459209+0000) 2024-03-06T14:38:29.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:29.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:29.459454+0000) 2024-03-06T14:38:30.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:30.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:30.459687+0000) 2024-03-06T14:38:31.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:31.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:31.459885+0000) 2024-03-06T14:38:32.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:32.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:32.460079+0000) 2024-03-06T14:38:33.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:33.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:33.460312+0000) 2024-03-06T14:38:34.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:34.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:34.460614+0000) 2024-03-06T14:38:35.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:35.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:35.460848+0000) 2024-03-06T14:38:36.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:36.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:36.461051+0000) 2024-03-06T14:38:37.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:37.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:37.461286+0000) 2024-03-06T14:38:38.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:38.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:38.461527+0000) 2024-03-06T14:38:39.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:39.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:39.461897+0000) 2024-03-06T14:38:40.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:40.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:40.462119+0000) 2024-03-06T14:38:40.493 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:38:40.501 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:38:41.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:41.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:41.462359+0000) 2024-03-06T14:38:42.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:42.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:42.462663+0000) 2024-03-06T14:38:43.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:43.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:43.462864+0000) 2024-03-06T14:38:44.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:44.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:44.463040+0000) 2024-03-06T14:38:45.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:45.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:45.463274+0000) 2024-03-06T14:38:46.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:46.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:46.463475+0000) 2024-03-06T14:38:47.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:47.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:47.463729+0000) 2024-03-06T14:38:48.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:48.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:48.463928+0000) 2024-03-06T14:38:49.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:49.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:49.464106+0000) 2024-03-06T14:38:50.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:50.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:50.464333+0000) 2024-03-06T14:38:51.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:51.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:51.464598+0000) 2024-03-06T14:38:52.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:52.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:52.464804+0000) 2024-03-06T14:38:53.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:53.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:53.465042+0000) 2024-03-06T14:38:54.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:54.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:54.465223+0000) 2024-03-06T14:38:55.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:55.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:55.465419+0000) 2024-03-06T14:38:56.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:56.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:56.465649+0000) 2024-03-06T14:38:57.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:57.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:57.465885+0000) 2024-03-06T14:38:58.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:58.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:58.466082+0000) 2024-03-06T14:38:59.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:38:59.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:38:59.466309+0000) 2024-03-06T14:39:00.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:00.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:00.466568+0000) 2024-03-06T14:39:01.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:01.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:01.466806+0000) 2024-03-06T14:39:02.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:02.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:02.467047+0000) 2024-03-06T14:39:03.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:03.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:03.467290+0000) 2024-03-06T14:39:04.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:04.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:04.467518+0000) 2024-03-06T14:39:05.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:05.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:05.467708+0000) 2024-03-06T14:39:06.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:06.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:06.467900+0000) 2024-03-06T14:39:07.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:07.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:07.468104+0000) 2024-03-06T14:39:08.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:08.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:08.468332+0000) 2024-03-06T14:39:09.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:09.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:09.468599+0000) 2024-03-06T14:39:10.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:10.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:10.468796+0000) 2024-03-06T14:39:10.698 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:39:10.706 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:39:11.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:11.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:11.469035+0000) 2024-03-06T14:39:12.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:12.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:12.469274+0000) 2024-03-06T14:39:13.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:13.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:13.469460+0000) 2024-03-06T14:39:14.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:14.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:14.469704+0000) 2024-03-06T14:39:15.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:15.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:15.469901+0000) 2024-03-06T14:39:16.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:16.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:16.470130+0000) 2024-03-06T14:39:17.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:17.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:17.470289+0000) 2024-03-06T14:39:18.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:18.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:18.470621+0000) 2024-03-06T14:39:19.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:19.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:19.470805+0000) 2024-03-06T14:39:20.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:20.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:20.471040+0000) 2024-03-06T14:39:21.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:21.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:21.471264+0000) 2024-03-06T14:39:22.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:22.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:22.471468+0000) 2024-03-06T14:39:23.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:23.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:23.471680+0000) 2024-03-06T14:39:24.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:24.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:24.471858+0000) 2024-03-06T14:39:25.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:25.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:25.472101+0000) 2024-03-06T14:39:26.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:26.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:26.472252+0000) 2024-03-06T14:39:27.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:27.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:27.472495+0000) 2024-03-06T14:39:28.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:28.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:28.472698+0000) 2024-03-06T14:39:29.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:29.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:29.472891+0000) 2024-03-06T14:39:30.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:30.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:30.473124+0000) 2024-03-06T14:39:31.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:31.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:31.473347+0000) 2024-03-06T14:39:32.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:32.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:32.473609+0000) 2024-03-06T14:39:33.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:33.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:33.473828+0000) 2024-03-06T14:39:34.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:34.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:34.474093+0000) 2024-03-06T14:39:35.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:35.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:35.474300+0000) 2024-03-06T14:39:36.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:36.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:36.474614+0000) 2024-03-06T14:39:37.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:37.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:37.474828+0000) 2024-03-06T14:39:38.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:38.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:38.475063+0000) 2024-03-06T14:39:39.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:39.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:39.475262+0000) 2024-03-06T14:39:40.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:40.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:40.475508+0000) 2024-03-06T14:39:40.920 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:39:40.930 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:39:41.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:41.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:41.475778+0000) 2024-03-06T14:39:42.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:42.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:42.476031+0000) 2024-03-06T14:39:43.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:43.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:43.476261+0000) 2024-03-06T14:39:44.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:44.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:44.476489+0000) 2024-03-06T14:39:45.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:45.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:45.476734+0000) 2024-03-06T14:39:46.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:46.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:46.476912+0000) 2024-03-06T14:39:47.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:47.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:47.477156+0000) 2024-03-06T14:39:48.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:48.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:48.477398+0000) 2024-03-06T14:39:49.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:49.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:49.477629+0000) 2024-03-06T14:39:50.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:50.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:50.477864+0000) 2024-03-06T14:39:51.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:51.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:51.478086+0000) 2024-03-06T14:39:52.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:52.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:52.478359+0000) 2024-03-06T14:39:53.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:53.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:53.478642+0000) 2024-03-06T14:39:54.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:54.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:54.478838+0000) 2024-03-06T14:39:55.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:55.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:55.479082+0000) 2024-03-06T14:39:56.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:56.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:56.479350+0000) 2024-03-06T14:39:57.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:57.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:57.479679+0000) 2024-03-06T14:39:58.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:58.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:58.479960+0000) 2024-03-06T14:39:59.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:39:59.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:39:59.480240+0000) 2024-03-06T14:40:00.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:00.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:00.480523+0000) 2024-03-06T14:40:01.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:01.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:01.480857+0000) 2024-03-06T14:40:02.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:02.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:02.481135+0000) 2024-03-06T14:40:03.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:03.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:03.481418+0000) 2024-03-06T14:40:04.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:04.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:04.481707+0000) 2024-03-06T14:40:05.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:05.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:05.481984+0000) 2024-03-06T14:40:06.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:06.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:06.482304+0000) 2024-03-06T14:40:07.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:07.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:07.482620+0000) 2024-03-06T14:40:08.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:08.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:08.482875+0000) 2024-03-06T14:40:09.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:09.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:09.483156+0000) 2024-03-06T14:40:10.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:10.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:10.483435+0000) 2024-03-06T14:40:11.125 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:40:11.133 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:40:11.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:11.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:11.483675+0000) 2024-03-06T14:40:12.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:12.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:12.483932+0000) 2024-03-06T14:40:13.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:13.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:13.484145+0000) 2024-03-06T14:40:14.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:14.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:14.484353+0000) 2024-03-06T14:40:15.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:15.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:15.484632+0000) 2024-03-06T14:40:16.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:16.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:16.484861+0000) 2024-03-06T14:40:17.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:17.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:17.485113+0000) 2024-03-06T14:40:18.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:18.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:18.485376+0000) 2024-03-06T14:40:19.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:19.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:19.485640+0000) 2024-03-06T14:40:20.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:20.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:20.485880+0000) 2024-03-06T14:40:21.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:21.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:21.486124+0000) 2024-03-06T14:40:22.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:22.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:22.486345+0000) 2024-03-06T14:40:23.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:23.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:23.486629+0000) 2024-03-06T14:40:24.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:24.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:24.486864+0000) 2024-03-06T14:40:25.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:25.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:25.487056+0000) 2024-03-06T14:40:26.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:26.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:26.487281+0000) 2024-03-06T14:40:27.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:27.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:27.487514+0000) 2024-03-06T14:40:28.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:28.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:28.487787+0000) 2024-03-06T14:40:29.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:29.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:29.488022+0000) 2024-03-06T14:40:30.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:30.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:30.488219+0000) 2024-03-06T14:40:31.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:31.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:31.488441+0000) 2024-03-06T14:40:32.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:32.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:32.488679+0000) 2024-03-06T14:40:33.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:33.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:33.488908+0000) 2024-03-06T14:40:34.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:34.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:34.489136+0000) 2024-03-06T14:40:35.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:35.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:35.489330+0000) 2024-03-06T14:40:36.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:36.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:36.489618+0000) 2024-03-06T14:40:37.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:37.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:37.489828+0000) 2024-03-06T14:40:38.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:38.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:38.490071+0000) 2024-03-06T14:40:39.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:39.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:39.490303+0000) 2024-03-06T14:40:40.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:40.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:40.490565+0000) 2024-03-06T14:40:41.364 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:40:41.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:40:41.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:41.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:41.490699+0000) 2024-03-06T14:40:42.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:42.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:42.490880+0000) 2024-03-06T14:40:43.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:43.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:43.491067+0000) 2024-03-06T14:40:44.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:44.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:44.491255+0000) 2024-03-06T14:40:45.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:45.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:45.491491+0000) 2024-03-06T14:40:46.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:46.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:46.491735+0000) 2024-03-06T14:40:47.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:47.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:47.491978+0000) 2024-03-06T14:40:48.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:48.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:48.492222+0000) 2024-03-06T14:40:49.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:49.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:49.492399+0000) 2024-03-06T14:40:50.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:50.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:50.492667+0000) 2024-03-06T14:40:51.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:51.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:51.492857+0000) 2024-03-06T14:40:52.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:52.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:52.493097+0000) 2024-03-06T14:40:53.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:53.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:53.493327+0000) 2024-03-06T14:40:54.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:54.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:54.493512+0000) 2024-03-06T14:40:55.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:55.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:55.493836+0000) 2024-03-06T14:40:56.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:56.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:56.494123+0000) 2024-03-06T14:40:57.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:57.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:57.494349+0000) 2024-03-06T14:40:58.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:58.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:58.494648+0000) 2024-03-06T14:40:59.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:40:59.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:40:59.494958+0000) 2024-03-06T14:41:00.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:00.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:00.495212+0000) 2024-03-06T14:41:01.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:01.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:01.495414+0000) 2024-03-06T14:41:02.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:02.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:02.495651+0000) 2024-03-06T14:41:03.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:03.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:03.495889+0000) 2024-03-06T14:41:04.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:04.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:04.496116+0000) 2024-03-06T14:41:05.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:05.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:05.496350+0000) 2024-03-06T14:41:06.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:06.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:06.496625+0000) 2024-03-06T14:41:07.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:07.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:07.496855+0000) 2024-03-06T14:41:08.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:08.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:08.497068+0000) 2024-03-06T14:41:09.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:09.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:09.497312+0000) 2024-03-06T14:41:10.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:10.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:10.497575+0000) 2024-03-06T14:41:11.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:11.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:11.497726+0000) 2024-03-06T14:41:11.577 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:41:11.587 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:41:12.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:12.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:12.497928+0000) 2024-03-06T14:41:13.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:13.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:13.498112+0000) 2024-03-06T14:41:14.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:14.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:14.498338+0000) 2024-03-06T14:41:15.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:15.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:15.498631+0000) 2024-03-06T14:41:16.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:16.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:16.498865+0000) 2024-03-06T14:41:17.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:17.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:17.499103+0000) 2024-03-06T14:41:18.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:18.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:18.499308+0000) 2024-03-06T14:41:19.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:19.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:19.499566+0000) 2024-03-06T14:41:20.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:20.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:20.499734+0000) 2024-03-06T14:41:21.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:21.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:21.499917+0000) 2024-03-06T14:41:22.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:22.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:22.500145+0000) 2024-03-06T14:41:23.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:23.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:23.500413+0000) 2024-03-06T14:41:24.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:24.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:24.500706+0000) 2024-03-06T14:41:25.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:25.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:25.500882+0000) 2024-03-06T14:41:26.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:26.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:26.501106+0000) 2024-03-06T14:41:27.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:27.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:27.501290+0000) 2024-03-06T14:41:28.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:28.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:28.501497+0000) 2024-03-06T14:41:29.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:29.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:29.501720+0000) 2024-03-06T14:41:30.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:30.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:30.501918+0000) 2024-03-06T14:41:31.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:31.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:31.502147+0000) 2024-03-06T14:41:32.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:32.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:32.502350+0000) 2024-03-06T14:41:33.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:33.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:33.502568+0000) 2024-03-06T14:41:34.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:34.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:34.502770+0000) 2024-03-06T14:41:35.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:35.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:35.502968+0000) 2024-03-06T14:41:36.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:36.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:36.503174+0000) 2024-03-06T14:41:37.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:37.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:37.503455+0000) 2024-03-06T14:41:38.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:38.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:38.503748+0000) 2024-03-06T14:41:39.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:39.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:39.504010+0000) 2024-03-06T14:41:40.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:40.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:40.504297+0000) 2024-03-06T14:41:41.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:41.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:41.504635+0000) 2024-03-06T14:41:41.783 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:41:41.792 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:41:42.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:42.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:42.504852+0000) 2024-03-06T14:41:43.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:43.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:43.505075+0000) 2024-03-06T14:41:44.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:44.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:44.505263+0000) 2024-03-06T14:41:45.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:45.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:45.505478+0000) 2024-03-06T14:41:46.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:46.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:46.505714+0000) 2024-03-06T14:41:47.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:47.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:47.505874+0000) 2024-03-06T14:41:48.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:48.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:48.506074+0000) 2024-03-06T14:41:49.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:49.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:49.506302+0000) 2024-03-06T14:41:50.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:50.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:50.506513+0000) 2024-03-06T14:41:51.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:51.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:51.506777+0000) 2024-03-06T14:41:52.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:52.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:52.507021+0000) 2024-03-06T14:41:53.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:53.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:53.507268+0000) 2024-03-06T14:41:54.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:54.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:54.507515+0000) 2024-03-06T14:41:55.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:55.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:55.507771+0000) 2024-03-06T14:41:56.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:56.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:56.508013+0000) 2024-03-06T14:41:57.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:57.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:57.508222+0000) 2024-03-06T14:41:58.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:58.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:58.508487+0000) 2024-03-06T14:41:59.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:41:59.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:41:59.508719+0000) 2024-03-06T14:42:00.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:00.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:00.508948+0000) 2024-03-06T14:42:01.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:01.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:01.509194+0000) 2024-03-06T14:42:02.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:02.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:02.509434+0000) 2024-03-06T14:42:03.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:03.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:03.509663+0000) 2024-03-06T14:42:04.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:04.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:04.509907+0000) 2024-03-06T14:42:05.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:05.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:05.510193+0000) 2024-03-06T14:42:06.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:06.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:06.510405+0000) 2024-03-06T14:42:07.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:07.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:07.510647+0000) 2024-03-06T14:42:08.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:08.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:08.510859+0000) 2024-03-06T14:42:09.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:09.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:09.511063+0000) 2024-03-06T14:42:10.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:10.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:10.511310+0000) 2024-03-06T14:42:11.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:11.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:11.511493+0000) 2024-03-06T14:42:12.001 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:42:12.009 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:42:12.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:12.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:12.511782+0000) 2024-03-06T14:42:13.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:13.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:13.511982+0000) 2024-03-06T14:42:14.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:14.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:14.512168+0000) 2024-03-06T14:42:15.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:15.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:15.512404+0000) 2024-03-06T14:42:16.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:16.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:16.512655+0000) 2024-03-06T14:42:17.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:17.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:17.512903+0000) 2024-03-06T14:42:18.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:18.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:18.513141+0000) 2024-03-06T14:42:19.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:19.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:19.513426+0000) 2024-03-06T14:42:20.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:20.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:20.513718+0000) 2024-03-06T14:42:21.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:21.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:21.514004+0000) 2024-03-06T14:42:22.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:22.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:22.514278+0000) 2024-03-06T14:42:23.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:23.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:23.514527+0000) 2024-03-06T14:42:24.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:24.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:24.514793+0000) 2024-03-06T14:42:25.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:25.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:25.515028+0000) 2024-03-06T14:42:26.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:26.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:26.515285+0000) 2024-03-06T14:42:27.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:27.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:27.515525+0000) 2024-03-06T14:42:28.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:28.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:28.515809+0000) 2024-03-06T14:42:29.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:29.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:29.516084+0000) 2024-03-06T14:42:30.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:30.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:30.516291+0000) 2024-03-06T14:42:31.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:31.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:31.516518+0000) 2024-03-06T14:42:32.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:32.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:32.516780+0000) 2024-03-06T14:42:33.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:33.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:33.517043+0000) 2024-03-06T14:42:34.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:34.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:34.517321+0000) 2024-03-06T14:42:35.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:35.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:35.517714+0000) 2024-03-06T14:42:36.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:36.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:36.517962+0000) 2024-03-06T14:42:37.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:37.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:37.518175+0000) 2024-03-06T14:42:38.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:38.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:38.518417+0000) 2024-03-06T14:42:39.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:39.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:39.518667+0000) 2024-03-06T14:42:40.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:40.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:40.518936+0000) 2024-03-06T14:42:41.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:41.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:41.519170+0000) 2024-03-06T14:42:42.206 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:42:42.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:42:42.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:42.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:42.519376+0000) 2024-03-06T14:42:43.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:43.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:43.519659+0000) 2024-03-06T14:42:44.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:44.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:44.519951+0000) 2024-03-06T14:42:45.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:45.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:45.520200+0000) 2024-03-06T14:42:46.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:46.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:46.520395+0000) 2024-03-06T14:42:47.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:47.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:47.520664+0000) 2024-03-06T14:42:48.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:48.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:48.520914+0000) 2024-03-06T14:42:49.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:49.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:49.521138+0000) 2024-03-06T14:42:50.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:50.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:50.521332+0000) 2024-03-06T14:42:51.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:51.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:51.521598+0000) 2024-03-06T14:42:52.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:52.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:52.521831+0000) 2024-03-06T14:42:53.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:53.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:53.522063+0000) 2024-03-06T14:42:54.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:54.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:54.522283+0000) 2024-03-06T14:42:55.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:55.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:55.522486+0000) 2024-03-06T14:42:56.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:56.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:56.522709+0000) 2024-03-06T14:42:57.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:57.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:57.522971+0000) 2024-03-06T14:42:58.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:58.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:58.523270+0000) 2024-03-06T14:42:59.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:42:59.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:42:59.523585+0000) 2024-03-06T14:43:00.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:00.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:00.523869+0000) 2024-03-06T14:43:01.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:01.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:01.524117+0000) 2024-03-06T14:43:02.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:02.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:02.524346+0000) 2024-03-06T14:43:03.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:03.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:03.524624+0000) 2024-03-06T14:43:04.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:04.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:04.524879+0000) 2024-03-06T14:43:05.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:05.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:05.525108+0000) 2024-03-06T14:43:06.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:06.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:06.525337+0000) 2024-03-06T14:43:07.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:07.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:07.525602+0000) 2024-03-06T14:43:08.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:08.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:08.525797+0000) 2024-03-06T14:43:09.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:09.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:09.526065+0000) 2024-03-06T14:43:10.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:10.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:10.526340+0000) 2024-03-06T14:43:11.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:11.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:11.526643+0000) 2024-03-06T14:43:12.403 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:43:12.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:43:12.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:12.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:12.526877+0000) 2024-03-06T14:43:13.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:13.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:13.527132+0000) 2024-03-06T14:43:14.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:14.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:14.527370+0000) 2024-03-06T14:43:15.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:15.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:15.527649+0000) 2024-03-06T14:43:16.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:16.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:16.527912+0000) 2024-03-06T14:43:17.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:17.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:17.528156+0000) 2024-03-06T14:43:18.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:18.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:18.528448+0000) 2024-03-06T14:43:19.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:19.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:19.528712+0000) 2024-03-06T14:43:20.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:20.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:20.528993+0000) 2024-03-06T14:43:21.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:21.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:21.529264+0000) 2024-03-06T14:43:22.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:22.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:22.529572+0000) 2024-03-06T14:43:23.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:23.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:23.529900+0000) 2024-03-06T14:43:24.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:24.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:24.530111+0000) 2024-03-06T14:43:25.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:25.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:25.530359+0000) 2024-03-06T14:43:26.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:26.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:26.530641+0000) 2024-03-06T14:43:27.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:27.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:27.530846+0000) 2024-03-06T14:43:28.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:28.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:28.531097+0000) 2024-03-06T14:43:29.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:29.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:29.531340+0000) 2024-03-06T14:43:30.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:30.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:30.531620+0000) 2024-03-06T14:43:31.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:31.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:31.531824+0000) 2024-03-06T14:43:32.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:32.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:32.532027+0000) 2024-03-06T14:43:33.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:33.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:33.532276+0000) 2024-03-06T14:43:34.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:34.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:34.532513+0000) 2024-03-06T14:43:35.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:35.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:35.532822+0000) 2024-03-06T14:43:36.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:36.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:36.533059+0000) 2024-03-06T14:43:37.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:37.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:37.533294+0000) 2024-03-06T14:43:38.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:38.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:38.533632+0000) 2024-03-06T14:43:39.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:39.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:39.533883+0000) 2024-03-06T14:43:40.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:40.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:40.534167+0000) 2024-03-06T14:43:41.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:41.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:41.534452+0000) 2024-03-06T14:43:42.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:42.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:42.534658+0000) 2024-03-06T14:43:42.619 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:43:42.627 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:43:43.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:43.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:43.534873+0000) 2024-03-06T14:43:44.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:44.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:44.535107+0000) 2024-03-06T14:43:45.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:45.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:45.535340+0000) 2024-03-06T14:43:46.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:46.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:46.535608+0000) 2024-03-06T14:43:47.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:47.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:47.535832+0000) 2024-03-06T14:43:48.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:48.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:48.536115+0000) 2024-03-06T14:43:49.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:49.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:49.536392+0000) 2024-03-06T14:43:50.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:50.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:50.536701+0000) 2024-03-06T14:43:51.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:51.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:51.536939+0000) 2024-03-06T14:43:52.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:52.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:52.537144+0000) 2024-03-06T14:43:53.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:53.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:53.537393+0000) 2024-03-06T14:43:54.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:54.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:54.537654+0000) 2024-03-06T14:43:55.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:55.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:55.537862+0000) 2024-03-06T14:43:56.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:56.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:56.538086+0000) 2024-03-06T14:43:57.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:57.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:57.538309+0000) 2024-03-06T14:43:58.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:58.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:58.538664+0000) 2024-03-06T14:43:59.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:43:59.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:43:59.538951+0000) 2024-03-06T14:44:00.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:00.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:00.539239+0000) 2024-03-06T14:44:01.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:01.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:01.539574+0000) 2024-03-06T14:44:02.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:02.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:02.539853+0000) 2024-03-06T14:44:03.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:03.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:03.540140+0000) 2024-03-06T14:44:04.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:04.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:04.540412+0000) 2024-03-06T14:44:05.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:05.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:05.540699+0000) 2024-03-06T14:44:06.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:06.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:06.540977+0000) 2024-03-06T14:44:07.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:07.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:07.541189+0000) 2024-03-06T14:44:08.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:08.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:08.541486+0000) 2024-03-06T14:44:09.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:09.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:09.541776+0000) 2024-03-06T14:44:10.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:10.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:10.542062+0000) 2024-03-06T14:44:11.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:11.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:11.542282+0000) 2024-03-06T14:44:12.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:12.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:12.542502+0000) 2024-03-06T14:44:12.823 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:44:12.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:44:13.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:13.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:13.542750+0000) 2024-03-06T14:44:14.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:14.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:14.542951+0000) 2024-03-06T14:44:15.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:15.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:15.543194+0000) 2024-03-06T14:44:16.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:16.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:16.543466+0000) 2024-03-06T14:44:17.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:17.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:17.543697+0000) 2024-03-06T14:44:18.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:18.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:18.543952+0000) 2024-03-06T14:44:19.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:19.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:19.544183+0000) 2024-03-06T14:44:20.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:20.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:20.544398+0000) 2024-03-06T14:44:21.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:21.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:21.544681+0000) 2024-03-06T14:44:22.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:22.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:22.544964+0000) 2024-03-06T14:44:23.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:23.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:23.545245+0000) 2024-03-06T14:44:24.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:24.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:24.545524+0000) 2024-03-06T14:44:25.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:25.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:25.545852+0000) 2024-03-06T14:44:26.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:26.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:26.546110+0000) 2024-03-06T14:44:27.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:27.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:27.546380+0000) 2024-03-06T14:44:28.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:28.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:28.546683+0000) 2024-03-06T14:44:29.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:29.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:29.546925+0000) 2024-03-06T14:44:30.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:30.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:30.547135+0000) 2024-03-06T14:44:31.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:31.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:31.547338+0000) 2024-03-06T14:44:32.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:32.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:32.547627+0000) 2024-03-06T14:44:33.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:33.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:33.547843+0000) 2024-03-06T14:44:34.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:34.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:34.548051+0000) 2024-03-06T14:44:35.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:35.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:35.548293+0000) 2024-03-06T14:44:36.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:36.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:36.548521+0000) 2024-03-06T14:44:37.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:37.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:37.548751+0000) 2024-03-06T14:44:38.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:38.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:38.549004+0000) 2024-03-06T14:44:39.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:39.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:39.549226+0000) 2024-03-06T14:44:40.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:40.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:40.549506+0000) 2024-03-06T14:44:41.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:41.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:41.549822+0000) 2024-03-06T14:44:42.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:42.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:42.550019+0000) 2024-03-06T14:44:43.042 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:44:43.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:44:43.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:43.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:43.550294+0000) 2024-03-06T14:44:44.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:44.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:44.550610+0000) 2024-03-06T14:44:45.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:45.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:45.550898+0000) 2024-03-06T14:44:46.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:46.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:46.551096+0000) 2024-03-06T14:44:47.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:47.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:47.551327+0000) 2024-03-06T14:44:48.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:48.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:48.551622+0000) 2024-03-06T14:44:49.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:49.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:49.551859+0000) 2024-03-06T14:44:50.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:50.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:50.552089+0000) 2024-03-06T14:44:51.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:51.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:51.552314+0000) 2024-03-06T14:44:52.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:52.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:52.552574+0000) 2024-03-06T14:44:53.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:53.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:53.552778+0000) 2024-03-06T14:44:54.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:54.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:54.552973+0000) 2024-03-06T14:44:55.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:55.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:55.553215+0000) 2024-03-06T14:44:56.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:56.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:56.553396+0000) 2024-03-06T14:44:57.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:57.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:57.553632+0000) 2024-03-06T14:44:58.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:58.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:58.553884+0000) 2024-03-06T14:44:59.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:44:59.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:44:59.554079+0000) 2024-03-06T14:45:00.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:00.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:00.554321+0000) 2024-03-06T14:45:01.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:01.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:01.554521+0000) 2024-03-06T14:45:02.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:02.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:02.554762+0000) 2024-03-06T14:45:03.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:03.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:03.555014+0000) 2024-03-06T14:45:04.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:04.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:04.555241+0000) 2024-03-06T14:45:05.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:05.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:05.555470+0000) 2024-03-06T14:45:06.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:06.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:06.555655+0000) 2024-03-06T14:45:07.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:07.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:07.555884+0000) 2024-03-06T14:45:08.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:08.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:08.556113+0000) 2024-03-06T14:45:09.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:09.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:09.556354+0000) 2024-03-06T14:45:10.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:10.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:10.556723+0000) 2024-03-06T14:45:11.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:11.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:11.556964+0000) 2024-03-06T14:45:12.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:12.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:12.557167+0000) 2024-03-06T14:45:13.271 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:45:13.280 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:45:13.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:13.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:13.557411+0000) 2024-03-06T14:45:14.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:14.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:14.557649+0000) 2024-03-06T14:45:15.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:15.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:15.557840+0000) 2024-03-06T14:45:16.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:16.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:16.558061+0000) 2024-03-06T14:45:17.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:17.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:17.558268+0000) 2024-03-06T14:45:18.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:18.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:18.558620+0000) 2024-03-06T14:45:19.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:19.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:19.558869+0000) 2024-03-06T14:45:20.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:20.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:20.559146+0000) 2024-03-06T14:45:21.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:21.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:21.559429+0000) 2024-03-06T14:45:22.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:22.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:22.559659+0000) 2024-03-06T14:45:23.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:23.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:23.559908+0000) 2024-03-06T14:45:24.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:24.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:24.560137+0000) 2024-03-06T14:45:25.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:25.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:25.560332+0000) 2024-03-06T14:45:26.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:26.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:26.560597+0000) 2024-03-06T14:45:27.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:27.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:27.560826+0000) 2024-03-06T14:45:28.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:28.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:28.561045+0000) 2024-03-06T14:45:29.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:29.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:29.561259+0000) 2024-03-06T14:45:30.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:30.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:30.561468+0000) 2024-03-06T14:45:31.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:31.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:31.561706+0000) 2024-03-06T14:45:32.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:32.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:32.561933+0000) 2024-03-06T14:45:33.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:33.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:33.562169+0000) 2024-03-06T14:45:34.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:34.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:34.562376+0000) 2024-03-06T14:45:35.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:35.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:35.562654+0000) 2024-03-06T14:45:36.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:36.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:36.562873+0000) 2024-03-06T14:45:37.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:37.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:37.563110+0000) 2024-03-06T14:45:38.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:38.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:38.563366+0000) 2024-03-06T14:45:39.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:39.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:39.563650+0000) 2024-03-06T14:45:40.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:40.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:40.563847+0000) 2024-03-06T14:45:41.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:41.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:41.564034+0000) 2024-03-06T14:45:42.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:42.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:42.564234+0000) 2024-03-06T14:45:43.492 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:45:43.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:45:43.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:43.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:43.564415+0000) 2024-03-06T14:45:44.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:44.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:44.564669+0000) 2024-03-06T14:45:45.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:45.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:45.564897+0000) 2024-03-06T14:45:46.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:46.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:46.565087+0000) 2024-03-06T14:45:47.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:47.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:47.565323+0000) 2024-03-06T14:45:48.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:48.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:48.565528+0000) 2024-03-06T14:45:49.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:49.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:49.565835+0000) 2024-03-06T14:45:50.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:50.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:50.566087+0000) 2024-03-06T14:45:51.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:51.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:51.566266+0000) 2024-03-06T14:45:52.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:52.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:52.566481+0000) 2024-03-06T14:45:53.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:53.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:53.566708+0000) 2024-03-06T14:45:54.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:54.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:54.566946+0000) 2024-03-06T14:45:55.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:55.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:55.567200+0000) 2024-03-06T14:45:56.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:56.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:56.567438+0000) 2024-03-06T14:45:57.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:57.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:57.567662+0000) 2024-03-06T14:45:58.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:58.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:58.567941+0000) 2024-03-06T14:45:59.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:45:59.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:45:59.568165+0000) 2024-03-06T14:46:00.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:00.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:00.568404+0000) 2024-03-06T14:46:01.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:01.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:01.568672+0000) 2024-03-06T14:46:02.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:02.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:02.568898+0000) 2024-03-06T14:46:03.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:03.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:03.569130+0000) 2024-03-06T14:46:04.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:04.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:04.569385+0000) 2024-03-06T14:46:05.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:05.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:05.569674+0000) 2024-03-06T14:46:06.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:06.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:06.569867+0000) 2024-03-06T14:46:07.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:07.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:07.570059+0000) 2024-03-06T14:46:08.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:08.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:08.570295+0000) 2024-03-06T14:46:09.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:09.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:09.570523+0000) 2024-03-06T14:46:10.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:10.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:10.570778+0000) 2024-03-06T14:46:11.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:11.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:11.570997+0000) 2024-03-06T14:46:12.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:12.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:12.571178+0000) 2024-03-06T14:46:13.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:13.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:13.571372+0000) 2024-03-06T14:46:13.722 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:46:13.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:46:14.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:14.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:14.571651+0000) 2024-03-06T14:46:15.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:15.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:15.571902+0000) 2024-03-06T14:46:16.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:16.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:16.572143+0000) 2024-03-06T14:46:17.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:17.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:17.572353+0000) 2024-03-06T14:46:18.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:18.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:18.572667+0000) 2024-03-06T14:46:19.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:19.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:19.572942+0000) 2024-03-06T14:46:20.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:20.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:20.573157+0000) 2024-03-06T14:46:21.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:21.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:21.573393+0000) 2024-03-06T14:46:22.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:22.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:22.573625+0000) 2024-03-06T14:46:23.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:23.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:23.573858+0000) 2024-03-06T14:46:24.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:24.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:24.574142+0000) 2024-03-06T14:46:25.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:25.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:25.574408+0000) 2024-03-06T14:46:26.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:26.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:26.574670+0000) 2024-03-06T14:46:27.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:27.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:27.574906+0000) 2024-03-06T14:46:28.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:28.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:28.575185+0000) 2024-03-06T14:46:29.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:29.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:29.575422+0000) 2024-03-06T14:46:30.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:30.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:30.575695+0000) 2024-03-06T14:46:31.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:31.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:31.575938+0000) 2024-03-06T14:46:32.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:32.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:32.576148+0000) 2024-03-06T14:46:33.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:33.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:33.576353+0000) 2024-03-06T14:46:34.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:34.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:34.576624+0000) 2024-03-06T14:46:35.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:35.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:35.576855+0000) 2024-03-06T14:46:36.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:36.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:36.577090+0000) 2024-03-06T14:46:37.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:37.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:37.577325+0000) 2024-03-06T14:46:38.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:38.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:38.577607+0000) 2024-03-06T14:46:39.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:39.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:39.577810+0000) 2024-03-06T14:46:40.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:40.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:40.578013+0000) 2024-03-06T14:46:41.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:41.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:41.578214+0000) 2024-03-06T14:46:42.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:42.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:42.578438+0000) 2024-03-06T14:46:43.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:43.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:43.578704+0000) 2024-03-06T14:46:43.944 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:46:43.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:46:44.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:44.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:44.578990+0000) 2024-03-06T14:46:45.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:45.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:45.579246+0000) 2024-03-06T14:46:46.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:46.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:46.579497+0000) 2024-03-06T14:46:47.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:47.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:47.579765+0000) 2024-03-06T14:46:48.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:48.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:48.580021+0000) 2024-03-06T14:46:49.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:49.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:49.580269+0000) 2024-03-06T14:46:50.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:50.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:50.580506+0000) 2024-03-06T14:46:51.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:51.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:51.580761+0000) 2024-03-06T14:46:52.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:52.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:52.580951+0000) 2024-03-06T14:46:53.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:53.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:53.581225+0000) 2024-03-06T14:46:54.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:54.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:54.581436+0000) 2024-03-06T14:46:55.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:55.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:55.581656+0000) 2024-03-06T14:46:56.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:56.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:56.581900+0000) 2024-03-06T14:46:57.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:57.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:57.582148+0000) 2024-03-06T14:46:58.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:58.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:58.582413+0000) 2024-03-06T14:46:59.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:46:59.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:46:59.582709+0000) 2024-03-06T14:47:00.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:00.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:00.582948+0000) 2024-03-06T14:47:01.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:01.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:01.583180+0000) 2024-03-06T14:47:02.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:02.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:02.583405+0000) 2024-03-06T14:47:03.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:03.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:03.583642+0000) 2024-03-06T14:47:04.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:04.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:04.583854+0000) 2024-03-06T14:47:05.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:05.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:05.584059+0000) 2024-03-06T14:47:06.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:06.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:06.584303+0000) 2024-03-06T14:47:07.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:07.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:07.584575+0000) 2024-03-06T14:47:08.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:08.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:08.584822+0000) 2024-03-06T14:47:09.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:09.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:09.585062+0000) 2024-03-06T14:47:10.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:10.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:10.585303+0000) 2024-03-06T14:47:11.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:11.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:11.585578+0000) 2024-03-06T14:47:12.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:12.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:12.585782+0000) 2024-03-06T14:47:13.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:13.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:13.585977+0000) 2024-03-06T14:47:14.156 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:47:14.165 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:47:14.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:14.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:14.586233+0000) 2024-03-06T14:47:15.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:15.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:15.586477+0000) 2024-03-06T14:47:16.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:16.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:16.586728+0000) 2024-03-06T14:47:17.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:17.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:17.586975+0000) 2024-03-06T14:47:18.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:18.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:18.587230+0000) 2024-03-06T14:47:19.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:19.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:19.587427+0000) 2024-03-06T14:47:20.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:20.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:20.587679+0000) 2024-03-06T14:47:21.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:21.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:21.587923+0000) 2024-03-06T14:47:22.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:22.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:22.588168+0000) 2024-03-06T14:47:23.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:23.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:23.588399+0000) 2024-03-06T14:47:24.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:24.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:24.588668+0000) 2024-03-06T14:47:25.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:25.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:25.588864+0000) 2024-03-06T14:47:26.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:26.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:26.589114+0000) 2024-03-06T14:47:27.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:27.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:27.589359+0000) 2024-03-06T14:47:28.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:28.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:28.589644+0000) 2024-03-06T14:47:29.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:29.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:29.589890+0000) 2024-03-06T14:47:30.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:30.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:30.590132+0000) 2024-03-06T14:47:31.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:31.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:31.590377+0000) 2024-03-06T14:47:32.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:32.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:32.590660+0000) 2024-03-06T14:47:33.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:33.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:33.590909+0000) 2024-03-06T14:47:34.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:34.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:34.591228+0000) 2024-03-06T14:47:35.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:35.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:35.591478+0000) 2024-03-06T14:47:36.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:36.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:36.591662+0000) 2024-03-06T14:47:37.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:37.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:37.591864+0000) 2024-03-06T14:47:38.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:38.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:38.592105+0000) 2024-03-06T14:47:39.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:39.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:39.592381+0000) 2024-03-06T14:47:40.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:40.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:40.592698+0000) 2024-03-06T14:47:41.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:41.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:41.592904+0000) 2024-03-06T14:47:42.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:42.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:42.593131+0000) 2024-03-06T14:47:43.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:43.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:43.593357+0000) 2024-03-06T14:47:44.295 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:47:44.303 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:47:44.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:44.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:44.593641+0000) 2024-03-06T14:47:45.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:45.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:45.593852+0000) 2024-03-06T14:47:46.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:46.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:46.594065+0000) 2024-03-06T14:47:47.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:47.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:47.594298+0000) 2024-03-06T14:47:48.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:48.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:48.594571+0000) 2024-03-06T14:47:49.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:49.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:49.594816+0000) 2024-03-06T14:47:50.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:50.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:50.595042+0000) 2024-03-06T14:47:51.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:51.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:51.595260+0000) 2024-03-06T14:47:52.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:52.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:52.595491+0000) 2024-03-06T14:47:53.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:53.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:53.595722+0000) 2024-03-06T14:47:54.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:54.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:54.595924+0000) 2024-03-06T14:47:55.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:55.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:55.596128+0000) 2024-03-06T14:47:56.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:56.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:56.596343+0000) 2024-03-06T14:47:57.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:57.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:57.596564+0000) 2024-03-06T14:47:58.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:58.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:58.596831+0000) 2024-03-06T14:47:59.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:47:59.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:47:59.597061+0000) 2024-03-06T14:48:00.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:00.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:00.597296+0000) 2024-03-06T14:48:01.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:01.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:01.597567+0000) 2024-03-06T14:48:02.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:02.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:02.597801+0000) 2024-03-06T14:48:03.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:03.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:03.598031+0000) 2024-03-06T14:48:04.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:04.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:04.598264+0000) 2024-03-06T14:48:05.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:05.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:05.598453+0000) 2024-03-06T14:48:06.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:06.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:06.598701+0000) 2024-03-06T14:48:07.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:07.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:07.598930+0000) 2024-03-06T14:48:08.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:08.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:08.599141+0000) 2024-03-06T14:48:09.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:09.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:09.599352+0000) 2024-03-06T14:48:10.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:10.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:10.599597+0000) 2024-03-06T14:48:11.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:11.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:11.599829+0000) 2024-03-06T14:48:12.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:12.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:12.600081+0000) 2024-03-06T14:48:13.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:13.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:13.600334+0000) 2024-03-06T14:48:14.458 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:48:14.466 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:48:14.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:14.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:14.600653+0000) 2024-03-06T14:48:15.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:15.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:15.600894+0000) 2024-03-06T14:48:16.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:16.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:16.601114+0000) 2024-03-06T14:48:17.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:17.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:17.601316+0000) 2024-03-06T14:48:18.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:18.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:18.601574+0000) 2024-03-06T14:48:19.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:19.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:19.601781+0000) 2024-03-06T14:48:20.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:20.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:20.602016+0000) 2024-03-06T14:48:21.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:21.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:21.602248+0000) 2024-03-06T14:48:22.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:22.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:22.602434+0000) 2024-03-06T14:48:23.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:23.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:23.602685+0000) 2024-03-06T14:48:24.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:24.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:24.602932+0000) 2024-03-06T14:48:25.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:25.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:25.603157+0000) 2024-03-06T14:48:26.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:26.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:26.603399+0000) 2024-03-06T14:48:27.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:27.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:27.603701+0000) 2024-03-06T14:48:28.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:28.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:28.603957+0000) 2024-03-06T14:48:29.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:29.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:29.604154+0000) 2024-03-06T14:48:30.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:30.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:30.604340+0000) 2024-03-06T14:48:31.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:31.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:31.604629+0000) 2024-03-06T14:48:32.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:32.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:32.604874+0000) 2024-03-06T14:48:33.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:33.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:33.605120+0000) 2024-03-06T14:48:34.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:34.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:34.605410+0000) 2024-03-06T14:48:35.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:35.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:35.605649+0000) 2024-03-06T14:48:36.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:36.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:36.605832+0000) 2024-03-06T14:48:37.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:37.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:37.606071+0000) 2024-03-06T14:48:38.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:38.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:38.606299+0000) 2024-03-06T14:48:39.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:39.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:39.606472+0000) 2024-03-06T14:48:40.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:40.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:40.606716+0000) 2024-03-06T14:48:41.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:41.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:41.606939+0000) 2024-03-06T14:48:42.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:42.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:42.607167+0000) 2024-03-06T14:48:43.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:43.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:43.607397+0000) 2024-03-06T14:48:44.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:44.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:44.607673+0000) 2024-03-06T14:48:44.670 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:48:44.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:48:45.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:45.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:45.607854+0000) 2024-03-06T14:48:46.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:46.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:46.608075+0000) 2024-03-06T14:48:47.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:47.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:47.608301+0000) 2024-03-06T14:48:48.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:48.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:48.608493+0000) 2024-03-06T14:48:49.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:49.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:49.608727+0000) 2024-03-06T14:48:50.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:50.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:50.608912+0000) 2024-03-06T14:48:51.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:51.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:51.609136+0000) 2024-03-06T14:48:52.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:52.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:52.609367+0000) 2024-03-06T14:48:53.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:53.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:53.609624+0000) 2024-03-06T14:48:54.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:54.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:54.609858+0000) 2024-03-06T14:48:55.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:55.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:55.610041+0000) 2024-03-06T14:48:56.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:56.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:56.610234+0000) 2024-03-06T14:48:57.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:57.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:57.610424+0000) 2024-03-06T14:48:58.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:58.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:58.610668+0000) 2024-03-06T14:48:59.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:48:59.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:48:59.610917+0000) 2024-03-06T14:49:00.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:00.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:00.611138+0000) 2024-03-06T14:49:01.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:01.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:01.611382+0000) 2024-03-06T14:49:02.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:02.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:02.611632+0000) 2024-03-06T14:49:03.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:03.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:03.611828+0000) 2024-03-06T14:49:04.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:04.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:04.612009+0000) 2024-03-06T14:49:05.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:05.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:05.612238+0000) 2024-03-06T14:49:06.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:06.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:06.612462+0000) 2024-03-06T14:49:07.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:07.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:07.612694+0000) 2024-03-06T14:49:08.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:08.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:08.612886+0000) 2024-03-06T14:49:09.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:09.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:09.613134+0000) 2024-03-06T14:49:10.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:10.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:10.613388+0000) 2024-03-06T14:49:11.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:11.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:11.613662+0000) 2024-03-06T14:49:12.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:12.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:12.613901+0000) 2024-03-06T14:49:13.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:13.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:13.614142+0000) 2024-03-06T14:49:14.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:14.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:14.614399+0000) 2024-03-06T14:49:14.892 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:49:14.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:49:15.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:15.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:15.614649+0000) 2024-03-06T14:49:16.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:16.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:16.614877+0000) 2024-03-06T14:49:17.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:17.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:17.615066+0000) 2024-03-06T14:49:18.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:18.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:18.615278+0000) 2024-03-06T14:49:19.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:19.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:19.615506+0000) 2024-03-06T14:49:20.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:20.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:20.615782+0000) 2024-03-06T14:49:21.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:21.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:21.616006+0000) 2024-03-06T14:49:22.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:22.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:22.616192+0000) 2024-03-06T14:49:23.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:23.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:23.616413+0000) 2024-03-06T14:49:24.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:24.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:24.616660+0000) 2024-03-06T14:49:25.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:25.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:25.616872+0000) 2024-03-06T14:49:26.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:26.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:26.617096+0000) 2024-03-06T14:49:27.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:27.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:27.617320+0000) 2024-03-06T14:49:28.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:28.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:28.617597+0000) 2024-03-06T14:49:29.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:29.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:29.617813+0000) 2024-03-06T14:49:30.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:30.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:30.618044+0000) 2024-03-06T14:49:31.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:31.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:31.618289+0000) 2024-03-06T14:49:32.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:32.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:32.618521+0000) 2024-03-06T14:49:33.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:33.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:33.618759+0000) 2024-03-06T14:49:34.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:34.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:34.618995+0000) 2024-03-06T14:49:35.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:35.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:35.619234+0000) 2024-03-06T14:49:36.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:36.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:36.619463+0000) 2024-03-06T14:49:37.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:37.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:37.619718+0000) 2024-03-06T14:49:38.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:38.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:38.619920+0000) 2024-03-06T14:49:39.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:39.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:39.620181+0000) 2024-03-06T14:49:40.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:40.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:40.620393+0000) 2024-03-06T14:49:41.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:41.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:41.620648+0000) 2024-03-06T14:49:42.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:42.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:42.620857+0000) 2024-03-06T14:49:43.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:43.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:43.621094+0000) 2024-03-06T14:49:44.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:44.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:44.621326+0000) 2024-03-06T14:49:45.088 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:49:45.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:49:45.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:45.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:45.621600+0000) 2024-03-06T14:49:46.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:46.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:46.621835+0000) 2024-03-06T14:49:47.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:47.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:47.622081+0000) 2024-03-06T14:49:48.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:48.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:48.622335+0000) 2024-03-06T14:49:49.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:49.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:49.622613+0000) 2024-03-06T14:49:50.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:50.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:50.622846+0000) 2024-03-06T14:49:51.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:51.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:51.623089+0000) 2024-03-06T14:49:52.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:52.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:52.623332+0000) 2024-03-06T14:49:53.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:53.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:53.623607+0000) 2024-03-06T14:49:54.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:54.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:54.623847+0000) 2024-03-06T14:49:55.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:55.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:55.624075+0000) 2024-03-06T14:49:56.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:56.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:56.624272+0000) 2024-03-06T14:49:57.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:57.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:57.624528+0000) 2024-03-06T14:49:58.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:58.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:58.624752+0000) 2024-03-06T14:49:59.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:49:59.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:49:59.624954+0000) 2024-03-06T14:50:00.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:00.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:00.625154+0000) 2024-03-06T14:50:01.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:01.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:01.625396+0000) 2024-03-06T14:50:02.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:02.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:02.625659+0000) 2024-03-06T14:50:03.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:03.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:03.625859+0000) 2024-03-06T14:50:04.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:04.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:04.626096+0000) 2024-03-06T14:50:05.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:05.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:05.626285+0000) 2024-03-06T14:50:06.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:06.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:06.626512+0000) 2024-03-06T14:50:07.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:07.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:07.626773+0000) 2024-03-06T14:50:08.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:08.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:08.627054+0000) 2024-03-06T14:50:09.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:09.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:09.627260+0000) 2024-03-06T14:50:10.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:10.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:10.627491+0000) 2024-03-06T14:50:11.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:11.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:11.627696+0000) 2024-03-06T14:50:12.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:12.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:12.627923+0000) 2024-03-06T14:50:13.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:13.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:13.628151+0000) 2024-03-06T14:50:14.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:14.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:14.628395+0000) 2024-03-06T14:50:15.318 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:50:15.326 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:50:15.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:15.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:15.628669+0000) 2024-03-06T14:50:16.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:16.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:16.628851+0000) 2024-03-06T14:50:17.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:17.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:17.629094+0000) 2024-03-06T14:50:18.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:18.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:18.629340+0000) 2024-03-06T14:50:19.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:19.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:19.629619+0000) 2024-03-06T14:50:20.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:20.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:20.629849+0000) 2024-03-06T14:50:21.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:21.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:21.630050+0000) 2024-03-06T14:50:22.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:22.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:22.630269+0000) 2024-03-06T14:50:23.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:23.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:23.630490+0000) 2024-03-06T14:50:24.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:24.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:24.630707+0000) 2024-03-06T14:50:25.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:25.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:25.630904+0000) 2024-03-06T14:50:26.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:26.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:26.631134+0000) 2024-03-06T14:50:27.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:27.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:27.631363+0000) 2024-03-06T14:50:28.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:28.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:28.631636+0000) 2024-03-06T14:50:29.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:29.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:29.631875+0000) 2024-03-06T14:50:30.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:30.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:30.632109+0000) 2024-03-06T14:50:31.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:31.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:31.632340+0000) 2024-03-06T14:50:32.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:32.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:32.632614+0000) 2024-03-06T14:50:33.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:33.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:33.632840+0000) 2024-03-06T14:50:34.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:34.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:34.633092+0000) 2024-03-06T14:50:35.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:35.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:35.633327+0000) 2024-03-06T14:50:36.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:36.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:36.633610+0000) 2024-03-06T14:50:37.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:37.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:37.633809+0000) 2024-03-06T14:50:38.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:38.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:38.634041+0000) 2024-03-06T14:50:39.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:39.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:39.634244+0000) 2024-03-06T14:50:40.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:40.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:40.634476+0000) 2024-03-06T14:50:41.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:41.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:41.634703+0000) 2024-03-06T14:50:42.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:42.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:42.634944+0000) 2024-03-06T14:50:43.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:43.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:43.635126+0000) 2024-03-06T14:50:44.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:44.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:44.635373+0000) 2024-03-06T14:50:45.475 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:50:45.484 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:50:45.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:45.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:45.635614+0000) 2024-03-06T14:50:46.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:46.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:46.635807+0000) 2024-03-06T14:50:47.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:47.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:47.635994+0000) 2024-03-06T14:50:48.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:48.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:48.636185+0000) 2024-03-06T14:50:49.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:49.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:49.636429+0000) 2024-03-06T14:50:50.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:50.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:50.636667+0000) 2024-03-06T14:50:51.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:51.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:51.636866+0000) 2024-03-06T14:50:52.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:52.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:52.637090+0000) 2024-03-06T14:50:53.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:53.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:53.637325+0000) 2024-03-06T14:50:54.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:54.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:54.637627+0000) 2024-03-06T14:50:55.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:55.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:55.637825+0000) 2024-03-06T14:50:56.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:56.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:56.638047+0000) 2024-03-06T14:50:57.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:57.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:57.638248+0000) 2024-03-06T14:50:58.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:58.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:58.638442+0000) 2024-03-06T14:50:59.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:50:59.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:50:59.638691+0000) 2024-03-06T14:51:00.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:00.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:00.638896+0000) 2024-03-06T14:51:01.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:01.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:01.639129+0000) 2024-03-06T14:51:02.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:02.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:02.639414+0000) 2024-03-06T14:51:03.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:03.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:03.639641+0000) 2024-03-06T14:51:04.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:04.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:04.639888+0000) 2024-03-06T14:51:05.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:05.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:05.640073+0000) 2024-03-06T14:51:06.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:06.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:06.640290+0000) 2024-03-06T14:51:07.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:07.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:07.640517+0000) 2024-03-06T14:51:08.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:08.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:08.640756+0000) 2024-03-06T14:51:09.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:09.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:09.640994+0000) 2024-03-06T14:51:10.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:10.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:10.641221+0000) 2024-03-06T14:51:11.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:11.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:11.641443+0000) 2024-03-06T14:51:12.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:12.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:12.641644+0000) 2024-03-06T14:51:13.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:13.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:13.641872+0000) 2024-03-06T14:51:14.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:14.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:14.642073+0000) 2024-03-06T14:51:15.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:15.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:15.642302+0000) 2024-03-06T14:51:15.685 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:51:15.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:51:16.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:16.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:16.642486+0000) 2024-03-06T14:51:17.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:17.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:17.642689+0000) 2024-03-06T14:51:18.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:18.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:18.642856+0000) 2024-03-06T14:51:19.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:19.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:19.643061+0000) 2024-03-06T14:51:20.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:20.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:20.643295+0000) 2024-03-06T14:51:21.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:21.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:21.643568+0000) 2024-03-06T14:51:22.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:22.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:22.643809+0000) 2024-03-06T14:51:23.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:23.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:23.644048+0000) 2024-03-06T14:51:24.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:24.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:24.644304+0000) 2024-03-06T14:51:25.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:25.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:25.644585+0000) 2024-03-06T14:51:26.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:26.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:26.644783+0000) 2024-03-06T14:51:27.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:27.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:27.645008+0000) 2024-03-06T14:51:28.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:28.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:28.645202+0000) 2024-03-06T14:51:29.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:29.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:29.645448+0000) 2024-03-06T14:51:30.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:30.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:30.645696+0000) 2024-03-06T14:51:31.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:31.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:31.645935+0000) 2024-03-06T14:51:32.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:32.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:32.646180+0000) 2024-03-06T14:51:33.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:33.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:33.646429+0000) 2024-03-06T14:51:34.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:34.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:34.646657+0000) 2024-03-06T14:51:35.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:35.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:35.646891+0000) 2024-03-06T14:51:36.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:36.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:36.647103+0000) 2024-03-06T14:51:37.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:37.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:37.647356+0000) 2024-03-06T14:51:38.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:38.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:38.647608+0000) 2024-03-06T14:51:39.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:39.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:39.647847+0000) 2024-03-06T14:51:40.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:40.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:40.648082+0000) 2024-03-06T14:51:41.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:41.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:41.648317+0000) 2024-03-06T14:51:42.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:42.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:42.648577+0000) 2024-03-06T14:51:43.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:43.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:43.648822+0000) 2024-03-06T14:51:44.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:44.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:44.649127+0000) 2024-03-06T14:51:45.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:45.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:45.649364+0000) 2024-03-06T14:51:45.874 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:51:45.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:51:46.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:46.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:46.649604+0000) 2024-03-06T14:51:47.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:47.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:47.649832+0000) 2024-03-06T14:51:48.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:48.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:48.650048+0000) 2024-03-06T14:51:49.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:49.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:49.650293+0000) 2024-03-06T14:51:50.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:50.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:50.650511+0000) 2024-03-06T14:51:51.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:51.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:51.650772+0000) 2024-03-06T14:51:52.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:52.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:52.650960+0000) 2024-03-06T14:51:53.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:53.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:53.651196+0000) 2024-03-06T14:51:54.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:54.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:54.651421+0000) 2024-03-06T14:51:55.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:55.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:55.651673+0000) 2024-03-06T14:51:56.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:56.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:56.651859+0000) 2024-03-06T14:51:57.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:57.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:57.652089+0000) 2024-03-06T14:51:58.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:58.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:58.652291+0000) 2024-03-06T14:51:59.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:51:59.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:51:59.652578+0000) 2024-03-06T14:52:00.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:00.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:00.652815+0000) 2024-03-06T14:52:01.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:01.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:01.653045+0000) 2024-03-06T14:52:02.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:02.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:02.653234+0000) 2024-03-06T14:52:03.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:03.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:03.653461+0000) 2024-03-06T14:52:04.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:04.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:04.653695+0000) 2024-03-06T14:52:05.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:05.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:05.653930+0000) 2024-03-06T14:52:06.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:06.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:06.654158+0000) 2024-03-06T14:52:07.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:07.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:07.654379+0000) 2024-03-06T14:52:08.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:08.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:08.654643+0000) 2024-03-06T14:52:09.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:09.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:09.654878+0000) 2024-03-06T14:52:10.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:10.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:10.655108+0000) 2024-03-06T14:52:11.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:11.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:11.655337+0000) 2024-03-06T14:52:12.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:12.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:12.655601+0000) 2024-03-06T14:52:13.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:13.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:13.655802+0000) 2024-03-06T14:52:14.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:14.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:14.655987+0000) 2024-03-06T14:52:15.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:15.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:15.656213+0000) 2024-03-06T14:52:16.101 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:52:16.108 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:52:16.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:16.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:16.656414+0000) 2024-03-06T14:52:17.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:17.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:17.656643+0000) 2024-03-06T14:52:18.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:18.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:18.656879+0000) 2024-03-06T14:52:19.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:19.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:19.657111+0000) 2024-03-06T14:52:20.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:20.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:20.657346+0000) 2024-03-06T14:52:21.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:21.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:21.657618+0000) 2024-03-06T14:52:22.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:22.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:22.657843+0000) 2024-03-06T14:52:23.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:23.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:23.658074+0000) 2024-03-06T14:52:24.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:24.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:24.658256+0000) 2024-03-06T14:52:25.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:25.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:25.658491+0000) 2024-03-06T14:52:26.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:26.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:26.658697+0000) 2024-03-06T14:52:27.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:27.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:27.658923+0000) 2024-03-06T14:52:28.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:28.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:28.659165+0000) 2024-03-06T14:52:29.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:29.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:29.659402+0000) 2024-03-06T14:52:30.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:30.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:30.659680+0000) 2024-03-06T14:52:31.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:31.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:31.659904+0000) 2024-03-06T14:52:32.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:32.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:32.660139+0000) 2024-03-06T14:52:33.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:33.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:33.660364+0000) 2024-03-06T14:52:34.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:34.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:34.660638+0000) 2024-03-06T14:52:35.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:35.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:35.660841+0000) 2024-03-06T14:52:36.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:36.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:36.661065+0000) 2024-03-06T14:52:37.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:37.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:37.661310+0000) 2024-03-06T14:52:38.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:38.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:38.661655+0000) 2024-03-06T14:52:39.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:39.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:39.661896+0000) 2024-03-06T14:52:40.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:40.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:40.662128+0000) 2024-03-06T14:52:41.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:41.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:41.662350+0000) 2024-03-06T14:52:42.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:42.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:42.662573+0000) 2024-03-06T14:52:43.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:43.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:43.662837+0000) 2024-03-06T14:52:44.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:44.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:44.663111+0000) 2024-03-06T14:52:45.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:45.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:45.663384+0000) 2024-03-06T14:52:46.326 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:52:46.334 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:52:46.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:46.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:46.663658+0000) 2024-03-06T14:52:47.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:47.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:47.663832+0000) 2024-03-06T14:52:48.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:48.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:48.664080+0000) 2024-03-06T14:52:49.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:49.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:49.664347+0000) 2024-03-06T14:52:50.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:50.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:50.664621+0000) 2024-03-06T14:52:51.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:51.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:51.664904+0000) 2024-03-06T14:52:52.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:52.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:52.665118+0000) 2024-03-06T14:52:53.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:53.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:53.665364+0000) 2024-03-06T14:52:54.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:54.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:54.665646+0000) 2024-03-06T14:52:55.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:55.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:55.665918+0000) 2024-03-06T14:52:56.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:56.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:56.666164+0000) 2024-03-06T14:52:57.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:57.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:57.666386+0000) 2024-03-06T14:52:58.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:58.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:58.666653+0000) 2024-03-06T14:52:59.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:52:59.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:52:59.666840+0000) 2024-03-06T14:53:00.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:00.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:00.667043+0000) 2024-03-06T14:53:01.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:01.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:01.667276+0000) 2024-03-06T14:53:02.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:02.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:02.667572+0000) 2024-03-06T14:53:03.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:03.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:03.667822+0000) 2024-03-06T14:53:04.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:04.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:04.668045+0000) 2024-03-06T14:53:05.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:05.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:05.668280+0000) 2024-03-06T14:53:06.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:06.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:06.668509+0000) 2024-03-06T14:53:07.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:07.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:07.668744+0000) 2024-03-06T14:53:08.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:08.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:08.669003+0000) 2024-03-06T14:53:09.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:09.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:09.669241+0000) 2024-03-06T14:53:10.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:10.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:10.669484+0000) 2024-03-06T14:53:11.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:11.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:11.669767+0000) 2024-03-06T14:53:12.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:12.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:12.669964+0000) 2024-03-06T14:53:13.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:13.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:13.670161+0000) 2024-03-06T14:53:14.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:14.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:14.670398+0000) 2024-03-06T14:53:15.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:15.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:15.670646+0000) 2024-03-06T14:53:16.514 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:53:16.522 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:53:16.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:16.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:16.670820+0000) 2024-03-06T14:53:17.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:17.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:17.671020+0000) 2024-03-06T14:53:18.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:18.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:18.671254+0000) 2024-03-06T14:53:19.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:19.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:19.671429+0000) 2024-03-06T14:53:20.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:20.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:20.671676+0000) 2024-03-06T14:53:21.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:21.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:21.671898+0000) 2024-03-06T14:53:22.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:22.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:22.672047+0000) 2024-03-06T14:53:23.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:23.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:23.672272+0000) 2024-03-06T14:53:24.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:24.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:24.672498+0000) 2024-03-06T14:53:25.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:25.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:25.672767+0000) 2024-03-06T14:53:26.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:26.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:26.673034+0000) 2024-03-06T14:53:27.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:27.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:27.673213+0000) 2024-03-06T14:53:28.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:28.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:28.673408+0000) 2024-03-06T14:53:29.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:29.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:29.673645+0000) 2024-03-06T14:53:30.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:30.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:30.673876+0000) 2024-03-06T14:53:31.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:31.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:31.674102+0000) 2024-03-06T14:53:32.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:32.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:32.674281+0000) 2024-03-06T14:53:33.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:33.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:33.674465+0000) 2024-03-06T14:53:34.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:34.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:34.674688+0000) 2024-03-06T14:53:35.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:35.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:35.674845+0000) 2024-03-06T14:53:36.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:36.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:36.675067+0000) 2024-03-06T14:53:37.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:37.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:37.675299+0000) 2024-03-06T14:53:38.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:38.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:38.675575+0000) 2024-03-06T14:53:39.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:39.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:39.675808+0000) 2024-03-06T14:53:40.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:40.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:40.676006+0000) 2024-03-06T14:53:41.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:41.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:41.676240+0000) 2024-03-06T14:53:42.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:42.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:42.676472+0000) 2024-03-06T14:53:43.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:43.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:43.676662+0000) 2024-03-06T14:53:44.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:44.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:44.676855+0000) 2024-03-06T14:53:45.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:45.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:45.677088+0000) 2024-03-06T14:53:46.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:46.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:46.677311+0000) 2024-03-06T14:53:46.728 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:53:46.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:53:47.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:47.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:47.677512+0000) 2024-03-06T14:53:48.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:48.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:48.677769+0000) 2024-03-06T14:53:49.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:49.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:49.677967+0000) 2024-03-06T14:53:50.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:50.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:50.678206+0000) 2024-03-06T14:53:51.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:51.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:51.678450+0000) 2024-03-06T14:53:52.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:52.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:52.678687+0000) 2024-03-06T14:53:53.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:53.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:53.678919+0000) 2024-03-06T14:53:54.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:54.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:54.679147+0000) 2024-03-06T14:53:55.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:55.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:55.679379+0000) 2024-03-06T14:53:56.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:56.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:56.679645+0000) 2024-03-06T14:53:57.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:57.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:57.679874+0000) 2024-03-06T14:53:58.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:58.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:58.680107+0000) 2024-03-06T14:53:59.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:53:59.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:53:59.680332+0000) 2024-03-06T14:54:00.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:00.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:00.680610+0000) 2024-03-06T14:54:01.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:01.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:01.680843+0000) 2024-03-06T14:54:02.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:02.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:02.681056+0000) 2024-03-06T14:54:03.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:03.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:03.681257+0000) 2024-03-06T14:54:04.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:04.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:04.681464+0000) 2024-03-06T14:54:05.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:05.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:05.681657+0000) 2024-03-06T14:54:06.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:06.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:06.681888+0000) 2024-03-06T14:54:07.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:07.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:07.682127+0000) 2024-03-06T14:54:08.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:08.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:08.682343+0000) 2024-03-06T14:54:09.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:09.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:09.682577+0000) 2024-03-06T14:54:10.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:10.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:10.682831+0000) 2024-03-06T14:54:11.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:11.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:11.683053+0000) 2024-03-06T14:54:12.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:12.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:12.683246+0000) 2024-03-06T14:54:13.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:13.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:13.683477+0000) 2024-03-06T14:54:14.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:14.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:14.683687+0000) 2024-03-06T14:54:15.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:15.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:15.683921+0000) 2024-03-06T14:54:16.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:16.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:16.684198+0000) 2024-03-06T14:54:16.923 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:54:16.935 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:54:17.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:17.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:17.684481+0000) 2024-03-06T14:54:18.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:18.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:18.684775+0000) 2024-03-06T14:54:19.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:19.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:19.685095+0000) 2024-03-06T14:54:20.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:20.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:20.685387+0000) 2024-03-06T14:54:21.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:21.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:21.685705+0000) 2024-03-06T14:54:22.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:22.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:22.685924+0000) 2024-03-06T14:54:23.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:23.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:23.686204+0000) 2024-03-06T14:54:24.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:24.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:24.686445+0000) 2024-03-06T14:54:25.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:25.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:25.686731+0000) 2024-03-06T14:54:26.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:26.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:26.686949+0000) 2024-03-06T14:54:27.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:27.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:27.687189+0000) 2024-03-06T14:54:28.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:28.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:28.687427+0000) 2024-03-06T14:54:29.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:29.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:29.687659+0000) 2024-03-06T14:54:30.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:30.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:30.687861+0000) 2024-03-06T14:54:31.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:31.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:31.688094+0000) 2024-03-06T14:54:32.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:32.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:32.688309+0000) 2024-03-06T14:54:33.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:33.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:33.688566+0000) 2024-03-06T14:54:34.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:34.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:34.688795+0000) 2024-03-06T14:54:35.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:35.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:35.689027+0000) 2024-03-06T14:54:36.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:36.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:36.689258+0000) 2024-03-06T14:54:37.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:37.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:37.689457+0000) 2024-03-06T14:54:38.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:38.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:38.689687+0000) 2024-03-06T14:54:39.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:39.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:39.689842+0000) 2024-03-06T14:54:40.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:40.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:40.690084+0000) 2024-03-06T14:54:41.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:41.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:41.690291+0000) 2024-03-06T14:54:42.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:42.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:42.690578+0000) 2024-03-06T14:54:43.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:43.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:43.690829+0000) 2024-03-06T14:54:44.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:44.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:44.691029+0000) 2024-03-06T14:54:45.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:45.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:45.691260+0000) 2024-03-06T14:54:46.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:46.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:46.691454+0000) 2024-03-06T14:54:47.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:54:47.082 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:54:47.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:47.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:47.691686+0000) 2024-03-06T14:54:48.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:48.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:48.691908+0000) 2024-03-06T14:54:49.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:49.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:49.692138+0000) 2024-03-06T14:54:50.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:50.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:50.692337+0000) 2024-03-06T14:54:51.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:51.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:51.692596+0000) 2024-03-06T14:54:52.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:52.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:52.692756+0000) 2024-03-06T14:54:53.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:53.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:53.692965+0000) 2024-03-06T14:54:54.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:54.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:54.693250+0000) 2024-03-06T14:54:55.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:55.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:55.693511+0000) 2024-03-06T14:54:56.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:56.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:56.693785+0000) 2024-03-06T14:54:57.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:57.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:57.694027+0000) 2024-03-06T14:54:58.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:58.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:58.694281+0000) 2024-03-06T14:54:59.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:54:59.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:54:59.694570+0000) 2024-03-06T14:55:00.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:00.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:00.694798+0000) 2024-03-06T14:55:01.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:01.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:01.695005+0000) 2024-03-06T14:55:02.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:02.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:02.695215+0000) 2024-03-06T14:55:03.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:03.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:03.695450+0000) 2024-03-06T14:55:04.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:04.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:04.695689+0000) 2024-03-06T14:55:05.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:05.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:05.695894+0000) 2024-03-06T14:55:06.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:06.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:06.696118+0000) 2024-03-06T14:55:07.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:07.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:07.696350+0000) 2024-03-06T14:55:08.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:08.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:08.696635+0000) 2024-03-06T14:55:09.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:09.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:09.696839+0000) 2024-03-06T14:55:10.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:10.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:10.697097+0000) 2024-03-06T14:55:11.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:11.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:11.697339+0000) 2024-03-06T14:55:12.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:12.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:12.697624+0000) 2024-03-06T14:55:13.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:13.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:13.697833+0000) 2024-03-06T14:55:14.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:14.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:14.698061+0000) 2024-03-06T14:55:15.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:15.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:15.698290+0000) 2024-03-06T14:55:16.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:16.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:16.698483+0000) 2024-03-06T14:55:17.299 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:55:17.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:55:17.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:17.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:17.698661+0000) 2024-03-06T14:55:18.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:18.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:18.698867+0000) 2024-03-06T14:55:19.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:19.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:19.699057+0000) 2024-03-06T14:55:20.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:20.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:20.699244+0000) 2024-03-06T14:55:21.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:21.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:21.699471+0000) 2024-03-06T14:55:22.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:22.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:22.699704+0000) 2024-03-06T14:55:23.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:23.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:23.699935+0000) 2024-03-06T14:55:24.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:24.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:24.700160+0000) 2024-03-06T14:55:25.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:25.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:25.700353+0000) 2024-03-06T14:55:26.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:26.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:26.700612+0000) 2024-03-06T14:55:27.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:27.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:27.700829+0000) 2024-03-06T14:55:28.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:28.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:28.701084+0000) 2024-03-06T14:55:29.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:29.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:29.701314+0000) 2024-03-06T14:55:30.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:30.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:30.701579+0000) 2024-03-06T14:55:31.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:31.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:31.701778+0000) 2024-03-06T14:55:32.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:32.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:32.702008+0000) 2024-03-06T14:55:33.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:33.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:33.702196+0000) 2024-03-06T14:55:34.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:34.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:34.702428+0000) 2024-03-06T14:55:35.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:35.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:35.702664+0000) 2024-03-06T14:55:36.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:36.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:36.702885+0000) 2024-03-06T14:55:37.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:37.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:37.703131+0000) 2024-03-06T14:55:38.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:38.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:38.703416+0000) 2024-03-06T14:55:39.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:39.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:39.703647+0000) 2024-03-06T14:55:40.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:40.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:40.703886+0000) 2024-03-06T14:55:41.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:41.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:41.704042+0000) 2024-03-06T14:55:42.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:42.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:42.704266+0000) 2024-03-06T14:55:43.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:43.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:43.704446+0000) 2024-03-06T14:55:44.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:44.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:44.704693+0000) 2024-03-06T14:55:45.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:45.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:45.704926+0000) 2024-03-06T14:55:46.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:46.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:46.705149+0000) 2024-03-06T14:55:47.503 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:55:47.512 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:55:47.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:47.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:47.705382+0000) 2024-03-06T14:55:48.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:48.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:48.705672+0000) 2024-03-06T14:55:49.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:49.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:49.705921+0000) 2024-03-06T14:55:50.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:50.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:50.706183+0000) 2024-03-06T14:55:51.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:51.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:51.706428+0000) 2024-03-06T14:55:52.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:52.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:52.706669+0000) 2024-03-06T14:55:53.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:53.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:53.706877+0000) 2024-03-06T14:55:54.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:54.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:54.707133+0000) 2024-03-06T14:55:55.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:55.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:55.707438+0000) 2024-03-06T14:55:56.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:56.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:56.707706+0000) 2024-03-06T14:55:57.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:57.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:57.707946+0000) 2024-03-06T14:55:58.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:58.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:58.708195+0000) 2024-03-06T14:55:59.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:55:59.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:55:59.708421+0000) 2024-03-06T14:56:00.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:00.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:00.708658+0000) 2024-03-06T14:56:01.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:01.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:01.708852+0000) 2024-03-06T14:56:02.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:02.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:02.709094+0000) 2024-03-06T14:56:03.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:03.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:03.709338+0000) 2024-03-06T14:56:04.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:04.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:04.709568+0000) 2024-03-06T14:56:05.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:05.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:05.709824+0000) 2024-03-06T14:56:06.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:06.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:06.710060+0000) 2024-03-06T14:56:07.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:07.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:07.710287+0000) 2024-03-06T14:56:08.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:08.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:08.710572+0000) 2024-03-06T14:56:09.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:09.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:09.710803+0000) 2024-03-06T14:56:10.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:10.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:10.711047+0000) 2024-03-06T14:56:11.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:11.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:11.711275+0000) 2024-03-06T14:56:12.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:12.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:12.711505+0000) 2024-03-06T14:56:13.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:13.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:13.711731+0000) 2024-03-06T14:56:14.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:14.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:14.711956+0000) 2024-03-06T14:56:15.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:15.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:15.712164+0000) 2024-03-06T14:56:16.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:16.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:16.712389+0000) 2024-03-06T14:56:17.700 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:56:17.709 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:56:17.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:17.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:17.712629+0000) 2024-03-06T14:56:18.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:18.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:18.712896+0000) 2024-03-06T14:56:19.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:19.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:19.713125+0000) 2024-03-06T14:56:20.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:20.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:20.713320+0000) 2024-03-06T14:56:21.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:21.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:21.713579+0000) 2024-03-06T14:56:22.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:22.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:22.713805+0000) 2024-03-06T14:56:23.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:23.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:23.714038+0000) 2024-03-06T14:56:24.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:24.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:24.714265+0000) 2024-03-06T14:56:25.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:25.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:25.714505+0000) 2024-03-06T14:56:26.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:26.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:26.714776+0000) 2024-03-06T14:56:27.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:27.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:27.715005+0000) 2024-03-06T14:56:28.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:28.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:28.715246+0000) 2024-03-06T14:56:29.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:29.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:29.715428+0000) 2024-03-06T14:56:30.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:30.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:30.715674+0000) 2024-03-06T14:56:31.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:31.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:31.715906+0000) 2024-03-06T14:56:32.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:32.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:32.716134+0000) 2024-03-06T14:56:33.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:33.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:33.716362+0000) 2024-03-06T14:56:34.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:34.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:34.716572+0000) 2024-03-06T14:56:35.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:35.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:35.716781+0000) 2024-03-06T14:56:36.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:36.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:36.717006+0000) 2024-03-06T14:56:37.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:37.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:37.717239+0000) 2024-03-06T14:56:38.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:38.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:38.717495+0000) 2024-03-06T14:56:39.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:39.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:39.717761+0000) 2024-03-06T14:56:40.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:40.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:40.718012+0000) 2024-03-06T14:56:41.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:41.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:41.718261+0000) 2024-03-06T14:56:42.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:42.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:42.718507+0000) 2024-03-06T14:56:43.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:43.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:43.718758+0000) 2024-03-06T14:56:44.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:44.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:44.718989+0000) 2024-03-06T14:56:45.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:45.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:45.719225+0000) 2024-03-06T14:56:46.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:46.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:46.719451+0000) 2024-03-06T14:56:47.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:47.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:47.719702+0000) 2024-03-06T14:56:47.929 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:56:47.937 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:56:48.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:48.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:48.719925+0000) 2024-03-06T14:56:49.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:49.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:49.720119+0000) 2024-03-06T14:56:50.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:50.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:50.720333+0000) 2024-03-06T14:56:51.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:51.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:51.720599+0000) 2024-03-06T14:56:52.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:52.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:52.720832+0000) 2024-03-06T14:56:53.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:53.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:53.721024+0000) 2024-03-06T14:56:54.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:54.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:54.721266+0000) 2024-03-06T14:56:55.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:55.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:55.721520+0000) 2024-03-06T14:56:56.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:56.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:56.721805+0000) 2024-03-06T14:56:57.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:57.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:57.722038+0000) 2024-03-06T14:56:58.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:58.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:58.722300+0000) 2024-03-06T14:56:59.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:56:59.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:56:59.722579+0000) 2024-03-06T14:57:00.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:00.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:00.722830+0000) 2024-03-06T14:57:01.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:01.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:01.723078+0000) 2024-03-06T14:57:02.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:02.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:02.723359+0000) 2024-03-06T14:57:03.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:03.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:03.723677+0000) 2024-03-06T14:57:04.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:04.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:04.723890+0000) 2024-03-06T14:57:05.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:05.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:05.724111+0000) 2024-03-06T14:57:06.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:06.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:06.724355+0000) 2024-03-06T14:57:07.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:07.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:07.724652+0000) 2024-03-06T14:57:08.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:08.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:08.724932+0000) 2024-03-06T14:57:09.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:09.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:09.725157+0000) 2024-03-06T14:57:10.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:10.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:10.725372+0000) 2024-03-06T14:57:11.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:11.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:11.725637+0000) 2024-03-06T14:57:12.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:12.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:12.725950+0000) 2024-03-06T14:57:13.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:13.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:13.726193+0000) 2024-03-06T14:57:14.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:14.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:14.726381+0000) 2024-03-06T14:57:15.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:15.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:15.726649+0000) 2024-03-06T14:57:16.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:16.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:16.726878+0000) 2024-03-06T14:57:17.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:17.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:17.727104+0000) 2024-03-06T14:57:18.142 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:57:18.150 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:57:18.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:18.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:18.727356+0000) 2024-03-06T14:57:19.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:19.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:19.727631+0000) 2024-03-06T14:57:20.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:20.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:20.727863+0000) 2024-03-06T14:57:21.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:21.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:21.728092+0000) 2024-03-06T14:57:22.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:22.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:22.728308+0000) 2024-03-06T14:57:23.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:23.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:23.728573+0000) 2024-03-06T14:57:24.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:24.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:24.728818+0000) 2024-03-06T14:57:25.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:25.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:25.729064+0000) 2024-03-06T14:57:26.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:26.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:26.729282+0000) 2024-03-06T14:57:27.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:27.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:27.729561+0000) 2024-03-06T14:57:28.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:28.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:28.729797+0000) 2024-03-06T14:57:29.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:29.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:29.730030+0000) 2024-03-06T14:57:30.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:30.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:30.730273+0000) 2024-03-06T14:57:31.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:31.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:31.730529+0000) 2024-03-06T14:57:32.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:32.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:32.730826+0000) 2024-03-06T14:57:33.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:33.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:33.731075+0000) 2024-03-06T14:57:34.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:34.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:34.731310+0000) 2024-03-06T14:57:35.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:35.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:35.731571+0000) 2024-03-06T14:57:36.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:36.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:36.731885+0000) 2024-03-06T14:57:37.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:37.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:37.732113+0000) 2024-03-06T14:57:38.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:38.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:38.732399+0000) 2024-03-06T14:57:39.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:39.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:39.732686+0000) 2024-03-06T14:57:40.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:40.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:40.732934+0000) 2024-03-06T14:57:41.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:41.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:41.733219+0000) 2024-03-06T14:57:42.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:42.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:42.733464+0000) 2024-03-06T14:57:43.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:43.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:43.733674+0000) 2024-03-06T14:57:44.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:44.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:44.733902+0000) 2024-03-06T14:57:45.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:45.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:45.734138+0000) 2024-03-06T14:57:46.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:46.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:46.734366+0000) 2024-03-06T14:57:47.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:47.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:47.734598+0000) 2024-03-06T14:57:48.350 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:57:48.357 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:57:48.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:48.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:48.734842+0000) 2024-03-06T14:57:49.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:49.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:49.735091+0000) 2024-03-06T14:57:50.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:50.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:50.735346+0000) 2024-03-06T14:57:51.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:51.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:51.735644+0000) 2024-03-06T14:57:52.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:52.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:52.735914+0000) 2024-03-06T14:57:53.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:53.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:53.736197+0000) 2024-03-06T14:57:54.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:54.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:54.736477+0000) 2024-03-06T14:57:55.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:55.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:55.736721+0000) 2024-03-06T14:57:56.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:56.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:56.737012+0000) 2024-03-06T14:57:57.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:57.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:57.737261+0000) 2024-03-06T14:57:58.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:58.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:58.737585+0000) 2024-03-06T14:57:59.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:57:59.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:57:59.737840+0000) 2024-03-06T14:58:00.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:00.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:00.738092+0000) 2024-03-06T14:58:01.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:01.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:01.738297+0000) 2024-03-06T14:58:02.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:02.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:02.738524+0000) 2024-03-06T14:58:03.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:03.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:03.738781+0000) 2024-03-06T14:58:04.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:04.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:04.739050+0000) 2024-03-06T14:58:05.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:05.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:05.739299+0000) 2024-03-06T14:58:06.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:06.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:06.739573+0000) 2024-03-06T14:58:07.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:07.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:07.739804+0000) 2024-03-06T14:58:08.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:08.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:08.740034+0000) 2024-03-06T14:58:09.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:09.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:09.740267+0000) 2024-03-06T14:58:10.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:10.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:10.740455+0000) 2024-03-06T14:58:11.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:11.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:11.740661+0000) 2024-03-06T14:58:12.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:12.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:12.740902+0000) 2024-03-06T14:58:13.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:13.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:13.741132+0000) 2024-03-06T14:58:14.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:14.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:14.741331+0000) 2024-03-06T14:58:15.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:15.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:15.741625+0000) 2024-03-06T14:58:16.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:16.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:16.741857+0000) 2024-03-06T14:58:17.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:17.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:17.742099+0000) 2024-03-06T14:58:18.527 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:58:18.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:58:18.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:18.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:18.742308+0000) 2024-03-06T14:58:19.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:19.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:19.742581+0000) 2024-03-06T14:58:20.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:20.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:20.742831+0000) 2024-03-06T14:58:21.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:21.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:21.743064+0000) 2024-03-06T14:58:22.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:22.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:22.743239+0000) 2024-03-06T14:58:23.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:23.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:23.743417+0000) 2024-03-06T14:58:24.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:24.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:24.743658+0000) 2024-03-06T14:58:25.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:25.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:25.743859+0000) 2024-03-06T14:58:26.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:26.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:26.744089+0000) 2024-03-06T14:58:27.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:27.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:27.744317+0000) 2024-03-06T14:58:28.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:28.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:28.744515+0000) 2024-03-06T14:58:29.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:29.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:29.744781+0000) 2024-03-06T14:58:30.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:30.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:30.745026+0000) 2024-03-06T14:58:31.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:31.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:31.745260+0000) 2024-03-06T14:58:32.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:32.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:32.745485+0000) 2024-03-06T14:58:33.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:33.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:33.745714+0000) 2024-03-06T14:58:34.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:34.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:34.745948+0000) 2024-03-06T14:58:35.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:35.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:35.746206+0000) 2024-03-06T14:58:36.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:36.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:36.746447+0000) 2024-03-06T14:58:37.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:37.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:37.746656+0000) 2024-03-06T14:58:38.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:38.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:38.746892+0000) 2024-03-06T14:58:39.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:39.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:39.747086+0000) 2024-03-06T14:58:40.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:40.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:40.747309+0000) 2024-03-06T14:58:41.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:41.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:41.747513+0000) 2024-03-06T14:58:42.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:42.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:42.747771+0000) 2024-03-06T14:58:43.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:43.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:43.747964+0000) 2024-03-06T14:58:44.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:44.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:44.748243+0000) 2024-03-06T14:58:45.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:45.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:45.748499+0000) 2024-03-06T14:58:46.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:46.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:46.748742+0000) 2024-03-06T14:58:47.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:47.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:47.748934+0000) 2024-03-06T14:58:48.714 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:58:48.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:58:48.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:48.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:48.749148+0000) 2024-03-06T14:58:49.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:49.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:49.749373+0000) 2024-03-06T14:58:50.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:50.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:50.749652+0000) 2024-03-06T14:58:51.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:51.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:51.749891+0000) 2024-03-06T14:58:52.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:52.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:52.750117+0000) 2024-03-06T14:58:53.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:53.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:53.750334+0000) 2024-03-06T14:58:54.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:54.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:54.750472+0000) 2024-03-06T14:58:55.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:55.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:55.750716+0000) 2024-03-06T14:58:56.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:56.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:56.750948+0000) 2024-03-06T14:58:57.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:57.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:57.751177+0000) 2024-03-06T14:58:58.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:58.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:58.751414+0000) 2024-03-06T14:58:59.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:58:59.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:58:59.751647+0000) 2024-03-06T14:59:00.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:00.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:00.751878+0000) 2024-03-06T14:59:01.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:01.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:01.752119+0000) 2024-03-06T14:59:02.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:02.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:02.752310+0000) 2024-03-06T14:59:03.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:03.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:03.752496+0000) 2024-03-06T14:59:04.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:04.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:04.752725+0000) 2024-03-06T14:59:05.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:05.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:05.752922+0000) 2024-03-06T14:59:06.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:06.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:06.753160+0000) 2024-03-06T14:59:07.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:07.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:07.753395+0000) 2024-03-06T14:59:08.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:08.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:08.753656+0000) 2024-03-06T14:59:09.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:09.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:09.753848+0000) 2024-03-06T14:59:10.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:10.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:10.754087+0000) 2024-03-06T14:59:11.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:11.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:11.754359+0000) 2024-03-06T14:59:12.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:12.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:12.754637+0000) 2024-03-06T14:59:13.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:13.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:13.754844+0000) 2024-03-06T14:59:14.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:14.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:14.755038+0000) 2024-03-06T14:59:15.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:15.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:15.755225+0000) 2024-03-06T14:59:16.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:16.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:16.755438+0000) 2024-03-06T14:59:17.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:17.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:17.755657+0000) 2024-03-06T14:59:18.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:18.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:18.755941+0000) 2024-03-06T14:59:18.944 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:59:18.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:59:19.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:19.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:19.756171+0000) 2024-03-06T14:59:20.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:20.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:20.756402+0000) 2024-03-06T14:59:21.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:21.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:21.756668+0000) 2024-03-06T14:59:22.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:22.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:22.756893+0000) 2024-03-06T14:59:23.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:23.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:23.757085+0000) 2024-03-06T14:59:24.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:24.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:24.757269+0000) 2024-03-06T14:59:25.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:25.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:25.757573+0000) 2024-03-06T14:59:26.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:26.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:26.757807+0000) 2024-03-06T14:59:27.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:27.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:27.758034+0000) 2024-03-06T14:59:28.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:28.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:28.758256+0000) 2024-03-06T14:59:29.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:29.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:29.758485+0000) 2024-03-06T14:59:30.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:30.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:30.758720+0000) 2024-03-06T14:59:31.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:31.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:31.758938+0000) 2024-03-06T14:59:32.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:32.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:32.759199+0000) 2024-03-06T14:59:33.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:33.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:33.759513+0000) 2024-03-06T14:59:34.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:34.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:34.759792+0000) 2024-03-06T14:59:35.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:35.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:35.760016+0000) 2024-03-06T14:59:36.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:36.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:36.760247+0000) 2024-03-06T14:59:37.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:37.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:37.760475+0000) 2024-03-06T14:59:38.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:38.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:38.760701+0000) 2024-03-06T14:59:39.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:39.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:39.760924+0000) 2024-03-06T14:59:40.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:40.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:40.761177+0000) 2024-03-06T14:59:41.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:41.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:41.761411+0000) 2024-03-06T14:59:42.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:42.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:42.761639+0000) 2024-03-06T14:59:43.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:43.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:43.761870+0000) 2024-03-06T14:59:44.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:44.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:44.762099+0000) 2024-03-06T14:59:45.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:45.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:45.762329+0000) 2024-03-06T14:59:46.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:46.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:46.762591+0000) 2024-03-06T14:59:47.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:47.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:47.762822+0000) 2024-03-06T14:59:48.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:48.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:48.763057+0000) 2024-03-06T14:59:49.301 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:59:49.313 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T14:59:49.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:49.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:49.763308+0000) 2024-03-06T14:59:50.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:50.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:50.763565+0000) 2024-03-06T14:59:51.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:51.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:51.763798+0000) 2024-03-06T14:59:52.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:52.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:52.764024+0000) 2024-03-06T14:59:53.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:53.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:53.764253+0000) 2024-03-06T14:59:54.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:54.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:54.764471+0000) 2024-03-06T14:59:55.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:55.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:55.764709+0000) 2024-03-06T14:59:56.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:56.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:56.764941+0000) 2024-03-06T14:59:57.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:57.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:57.765126+0000) 2024-03-06T14:59:58.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:58.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:58.765360+0000) 2024-03-06T14:59:59.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T14:59:59.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T13:59:59.765668+0000) 2024-03-06T15:00:00.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:00.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:00.765939+0000) 2024-03-06T15:00:01.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:01.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:01.766157+0000) 2024-03-06T15:00:02.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:02.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:02.766374+0000) 2024-03-06T15:00:03.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:03.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:03.766644+0000) 2024-03-06T15:00:04.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:04.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:04.766837+0000) 2024-03-06T15:00:05.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:05.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:05.767089+0000) 2024-03-06T15:00:06.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:06.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:06.767392+0000) 2024-03-06T15:00:07.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:07.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:07.767647+0000) 2024-03-06T15:00:08.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:08.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:08.767909+0000) 2024-03-06T15:00:09.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:09.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:09.768184+0000) 2024-03-06T15:00:10.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:10.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:10.768464+0000) 2024-03-06T15:00:11.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:11.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:11.768725+0000) 2024-03-06T15:00:12.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:12.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:12.769005+0000) 2024-03-06T15:00:13.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:13.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:13.769206+0000) 2024-03-06T15:00:14.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:14.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:14.769459+0000) 2024-03-06T15:00:15.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:15.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:15.769731+0000) 2024-03-06T15:00:16.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:16.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:16.770004+0000) 2024-03-06T15:00:17.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:17.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:17.770286+0000) 2024-03-06T15:00:18.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:18.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:18.770619+0000) 2024-03-06T15:00:19.504 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:00:19.513 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:00:19.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:19.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:19.770904+0000) 2024-03-06T15:00:20.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:20.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:20.771102+0000) 2024-03-06T15:00:21.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:21.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:21.771328+0000) 2024-03-06T15:00:22.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:22.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:22.771612+0000) 2024-03-06T15:00:23.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:23.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:23.771882+0000) 2024-03-06T15:00:24.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:24.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:24.772168+0000) 2024-03-06T15:00:25.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:25.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:25.772455+0000) 2024-03-06T15:00:26.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:26.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:26.772740+0000) 2024-03-06T15:00:27.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:27.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:27.772969+0000) 2024-03-06T15:00:28.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:28.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:28.773271+0000) 2024-03-06T15:00:29.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:29.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:29.773477+0000) 2024-03-06T15:00:30.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:30.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:30.773710+0000) 2024-03-06T15:00:31.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:31.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:31.773943+0000) 2024-03-06T15:00:32.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:32.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:32.774144+0000) 2024-03-06T15:00:33.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:33.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:33.774380+0000) 2024-03-06T15:00:34.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:34.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:34.774666+0000) 2024-03-06T15:00:35.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:35.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:35.774946+0000) 2024-03-06T15:00:36.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:36.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:36.775194+0000) 2024-03-06T15:00:37.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:37.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:37.775478+0000) 2024-03-06T15:00:38.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:38.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:38.775780+0000) 2024-03-06T15:00:39.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:39.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:39.776063+0000) 2024-03-06T15:00:40.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:40.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:40.776267+0000) 2024-03-06T15:00:41.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:41.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:41.776515+0000) 2024-03-06T15:00:42.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:42.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:42.776779+0000) 2024-03-06T15:00:43.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:43.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:43.776997+0000) 2024-03-06T15:00:44.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:44.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:44.777226+0000) 2024-03-06T15:00:45.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:45.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:45.777466+0000) 2024-03-06T15:00:46.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:46.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:46.777736+0000) 2024-03-06T15:00:47.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:47.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:47.777982+0000) 2024-03-06T15:00:48.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:48.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:48.778249+0000) 2024-03-06T15:00:49.651 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:00:49.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:00:49.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:49.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:49.778594+0000) 2024-03-06T15:00:50.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:50.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:50.778791+0000) 2024-03-06T15:00:51.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:51.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:51.779018+0000) 2024-03-06T15:00:52.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:52.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:52.779239+0000) 2024-03-06T15:00:53.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:53.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:53.779446+0000) 2024-03-06T15:00:54.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:54.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:54.779660+0000) 2024-03-06T15:00:55.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:55.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:55.779904+0000) 2024-03-06T15:00:56.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:56.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:56.780142+0000) 2024-03-06T15:00:57.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:57.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:57.780385+0000) 2024-03-06T15:00:58.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:58.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:58.780637+0000) 2024-03-06T15:00:59.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:00:59.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:00:59.780865+0000) 2024-03-06T15:01:00.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:00.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:00.781082+0000) 2024-03-06T15:01:01.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:01.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:01.781314+0000) 2024-03-06T15:01:02.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:02.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:02.781509+0000) 2024-03-06T15:01:03.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:03.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:03.781771+0000) 2024-03-06T15:01:04.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:04.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:04.781988+0000) 2024-03-06T15:01:05.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:05.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:05.782218+0000) 2024-03-06T15:01:06.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:06.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:06.782405+0000) 2024-03-06T15:01:07.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:07.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:07.782657+0000) 2024-03-06T15:01:08.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:08.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:08.782900+0000) 2024-03-06T15:01:09.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:09.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:09.783154+0000) 2024-03-06T15:01:10.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:10.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:10.783414+0000) 2024-03-06T15:01:11.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:11.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:11.783705+0000) 2024-03-06T15:01:12.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:12.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:12.783958+0000) 2024-03-06T15:01:13.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:13.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:13.784241+0000) 2024-03-06T15:01:14.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:14.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:14.784513+0000) 2024-03-06T15:01:15.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:15.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:15.784814+0000) 2024-03-06T15:01:16.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:16.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:16.785051+0000) 2024-03-06T15:01:17.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:17.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:17.785255+0000) 2024-03-06T15:01:18.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:18.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:18.785457+0000) 2024-03-06T15:01:19.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:19.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:19.785706+0000) 2024-03-06T15:01:19.834 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:01:19.842 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:01:20.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:20.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:20.785949+0000) 2024-03-06T15:01:21.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:21.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:21.786200+0000) 2024-03-06T15:01:22.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:22.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:22.786419+0000) 2024-03-06T15:01:23.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:23.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:23.786696+0000) 2024-03-06T15:01:24.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:24.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:24.786965+0000) 2024-03-06T15:01:25.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:25.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:25.787178+0000) 2024-03-06T15:01:26.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:26.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:26.787411+0000) 2024-03-06T15:01:27.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:27.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:27.787645+0000) 2024-03-06T15:01:28.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:28.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:28.787840+0000) 2024-03-06T15:01:29.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:29.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:29.788039+0000) 2024-03-06T15:01:30.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:30.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:30.788268+0000) 2024-03-06T15:01:31.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:31.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:31.788504+0000) 2024-03-06T15:01:32.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:32.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:32.788759+0000) 2024-03-06T15:01:33.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:33.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:33.788979+0000) 2024-03-06T15:01:34.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:34.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:34.789188+0000) 2024-03-06T15:01:35.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:35.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:35.789417+0000) 2024-03-06T15:01:36.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:36.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:36.789646+0000) 2024-03-06T15:01:37.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:37.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:37.789846+0000) 2024-03-06T15:01:38.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:38.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:38.790048+0000) 2024-03-06T15:01:39.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:39.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:39.790285+0000) 2024-03-06T15:01:40.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:40.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:40.790589+0000) 2024-03-06T15:01:41.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:41.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:41.790847+0000) 2024-03-06T15:01:42.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:42.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:42.791067+0000) 2024-03-06T15:01:43.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:43.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:43.791347+0000) 2024-03-06T15:01:44.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:44.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:44.791653+0000) 2024-03-06T15:01:45.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:45.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:45.791942+0000) 2024-03-06T15:01:46.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:46.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:46.792186+0000) 2024-03-06T15:01:47.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:47.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:47.792473+0000) 2024-03-06T15:01:48.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:48.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:48.792745+0000) 2024-03-06T15:01:49.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:49.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:49.793027+0000) 2024-03-06T15:01:49.977 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:01:49.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:01:50.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:50.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:50.793212+0000) 2024-03-06T15:01:51.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:51.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:51.793454+0000) 2024-03-06T15:01:52.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:52.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:52.793674+0000) 2024-03-06T15:01:53.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:53.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:53.793909+0000) 2024-03-06T15:01:54.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:54.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:54.794153+0000) 2024-03-06T15:01:55.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:55.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:55.794436+0000) 2024-03-06T15:01:56.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:56.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:56.794725+0000) 2024-03-06T15:01:57.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:57.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:57.795011+0000) 2024-03-06T15:01:58.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:58.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:58.795282+0000) 2024-03-06T15:01:59.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:01:59.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:01:59.795616+0000) 2024-03-06T15:02:00.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:00.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:00.795817+0000) 2024-03-06T15:02:01.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:01.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:01.796059+0000) 2024-03-06T15:02:02.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:02.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:02.796274+0000) 2024-03-06T15:02:03.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:03.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:03.796514+0000) 2024-03-06T15:02:04.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:04.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:04.796779+0000) 2024-03-06T15:02:05.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:05.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:05.797006+0000) 2024-03-06T15:02:06.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:06.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:06.797242+0000) 2024-03-06T15:02:07.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:07.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:07.797488+0000) 2024-03-06T15:02:08.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:08.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:08.797701+0000) 2024-03-06T15:02:09.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:09.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:09.797935+0000) 2024-03-06T15:02:10.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:10.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:10.798162+0000) 2024-03-06T15:02:11.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:11.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:11.798412+0000) 2024-03-06T15:02:12.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:12.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:12.798638+0000) 2024-03-06T15:02:13.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:13.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:13.798824+0000) 2024-03-06T15:02:14.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:14.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:14.799006+0000) 2024-03-06T15:02:15.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:15.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:15.799239+0000) 2024-03-06T15:02:16.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:16.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:16.799461+0000) 2024-03-06T15:02:17.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:17.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:17.799688+0000) 2024-03-06T15:02:18.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:18.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:18.799905+0000) 2024-03-06T15:02:19.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:19.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:19.800155+0000) 2024-03-06T15:02:20.129 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:02:20.138 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:02:20.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:20.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:20.800424+0000) 2024-03-06T15:02:21.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:21.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:21.800719+0000) 2024-03-06T15:02:22.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:22.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:22.800954+0000) 2024-03-06T15:02:23.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:23.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:23.801185+0000) 2024-03-06T15:02:24.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:24.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:24.801387+0000) 2024-03-06T15:02:25.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:25.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:25.801655+0000) 2024-03-06T15:02:26.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:26.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:26.801850+0000) 2024-03-06T15:02:27.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:27.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:27.802083+0000) 2024-03-06T15:02:28.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:28.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:28.802314+0000) 2024-03-06T15:02:29.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:29.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:29.802518+0000) 2024-03-06T15:02:30.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:30.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:30.802763+0000) 2024-03-06T15:02:31.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:31.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:31.803022+0000) 2024-03-06T15:02:32.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:32.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:32.803243+0000) 2024-03-06T15:02:33.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:33.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:33.803461+0000) 2024-03-06T15:02:34.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:34.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:34.803723+0000) 2024-03-06T15:02:35.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:35.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:35.804078+0000) 2024-03-06T15:02:36.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:36.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:36.804299+0000) 2024-03-06T15:02:37.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:37.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:37.804584+0000) 2024-03-06T15:02:38.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:38.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:38.804817+0000) 2024-03-06T15:02:39.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:39.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:39.805049+0000) 2024-03-06T15:02:40.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:40.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:40.805269+0000) 2024-03-06T15:02:41.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:41.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:41.805585+0000) 2024-03-06T15:02:42.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:42.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:42.805862+0000) 2024-03-06T15:02:43.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:43.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:43.806094+0000) 2024-03-06T15:02:44.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:44.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:44.806355+0000) 2024-03-06T15:02:45.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:45.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:45.806667+0000) 2024-03-06T15:02:46.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:46.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:46.806899+0000) 2024-03-06T15:02:47.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:47.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:47.807143+0000) 2024-03-06T15:02:48.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:48.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:48.807334+0000) 2024-03-06T15:02:49.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:49.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:49.807603+0000) 2024-03-06T15:02:50.278 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:02:50.286 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:02:50.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:50.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:50.807827+0000) 2024-03-06T15:02:51.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:51.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:51.808060+0000) 2024-03-06T15:02:52.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:52.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:52.808283+0000) 2024-03-06T15:02:53.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:53.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:53.808527+0000) 2024-03-06T15:02:54.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:54.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:54.808782+0000) 2024-03-06T15:02:55.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:55.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:55.809061+0000) 2024-03-06T15:02:56.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:56.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:56.809337+0000) 2024-03-06T15:02:57.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:57.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:57.809658+0000) 2024-03-06T15:02:58.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:58.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:58.809926+0000) 2024-03-06T15:02:59.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:02:59.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:02:59.810212+0000) 2024-03-06T15:03:00.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:00.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:00.810489+0000) 2024-03-06T15:03:01.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:01.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:01.810777+0000) 2024-03-06T15:03:02.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:02.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:02.811045+0000) 2024-03-06T15:03:03.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:03.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:03.811330+0000) 2024-03-06T15:03:04.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:04.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:04.811626+0000) 2024-03-06T15:03:05.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:05.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:05.811877+0000) 2024-03-06T15:03:06.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:06.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:06.812110+0000) 2024-03-06T15:03:07.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:07.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:07.812265+0000) 2024-03-06T15:03:08.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:08.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:08.812494+0000) 2024-03-06T15:03:09.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:09.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:09.812731+0000) 2024-03-06T15:03:10.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:10.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:10.812957+0000) 2024-03-06T15:03:11.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:11.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:11.813211+0000) 2024-03-06T15:03:12.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:12.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:12.813457+0000) 2024-03-06T15:03:13.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:13.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:13.813705+0000) 2024-03-06T15:03:14.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:14.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:14.813898+0000) 2024-03-06T15:03:15.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:15.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:15.814100+0000) 2024-03-06T15:03:16.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:16.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:16.814321+0000) 2024-03-06T15:03:17.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:17.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:17.814643+0000) 2024-03-06T15:03:18.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:18.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:18.814894+0000) 2024-03-06T15:03:19.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:19.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:19.815082+0000) 2024-03-06T15:03:20.434 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:03:20.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:03:20.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:20.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:20.815317+0000) 2024-03-06T15:03:21.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:21.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:21.815517+0000) 2024-03-06T15:03:22.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:22.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:22.815770+0000) 2024-03-06T15:03:23.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:23.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:23.816001+0000) 2024-03-06T15:03:24.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:24.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:24.816186+0000) 2024-03-06T15:03:25.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:25.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:25.816416+0000) 2024-03-06T15:03:26.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:26.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:26.816650+0000) 2024-03-06T15:03:27.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:27.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:27.816866+0000) 2024-03-06T15:03:28.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:28.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:28.817210+0000) 2024-03-06T15:03:29.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:29.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:29.817423+0000) 2024-03-06T15:03:30.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:30.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:30.817665+0000) 2024-03-06T15:03:31.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:31.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:31.817896+0000) 2024-03-06T15:03:32.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:32.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:32.818075+0000) 2024-03-06T15:03:33.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:33.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:33.818310+0000) 2024-03-06T15:03:34.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:34.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:34.818568+0000) 2024-03-06T15:03:35.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:35.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:35.818826+0000) 2024-03-06T15:03:36.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:36.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:36.819066+0000) 2024-03-06T15:03:37.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:37.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:37.819316+0000) 2024-03-06T15:03:38.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:38.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:38.819603+0000) 2024-03-06T15:03:39.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:39.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:39.819833+0000) 2024-03-06T15:03:40.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:40.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:40.820026+0000) 2024-03-06T15:03:41.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:41.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:41.820275+0000) 2024-03-06T15:03:42.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:42.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:42.820516+0000) 2024-03-06T15:03:43.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:43.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:43.820807+0000) 2024-03-06T15:03:44.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:44.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:44.820965+0000) 2024-03-06T15:03:45.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:45.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:45.821173+0000) 2024-03-06T15:03:46.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:46.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:46.821352+0000) 2024-03-06T15:03:47.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:47.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:47.821587+0000) 2024-03-06T15:03:48.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:48.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:48.821773+0000) 2024-03-06T15:03:49.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:49.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:49.821966+0000) 2024-03-06T15:03:50.618 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:03:50.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:03:50.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:50.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:50.822105+0000) 2024-03-06T15:03:51.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:51.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:51.822324+0000) 2024-03-06T15:03:52.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:52.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:52.822600+0000) 2024-03-06T15:03:53.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:53.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:53.822853+0000) 2024-03-06T15:03:54.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:54.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:54.823095+0000) 2024-03-06T15:03:55.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:55.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:55.823353+0000) 2024-03-06T15:03:56.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:56.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:56.823622+0000) 2024-03-06T15:03:57.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:57.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:57.823835+0000) 2024-03-06T15:03:58.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:58.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:58.824087+0000) 2024-03-06T15:03:59.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:03:59.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:03:59.824319+0000) 2024-03-06T15:04:00.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:00.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:00.824569+0000) 2024-03-06T15:04:01.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:01.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:01.824801+0000) 2024-03-06T15:04:02.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:02.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:02.825035+0000) 2024-03-06T15:04:03.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:03.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:03.825250+0000) 2024-03-06T15:04:04.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:04.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:04.825475+0000) 2024-03-06T15:04:05.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:05.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:05.825691+0000) 2024-03-06T15:04:06.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:06.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:06.825912+0000) 2024-03-06T15:04:07.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:07.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:07.826142+0000) 2024-03-06T15:04:08.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:08.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:08.826370+0000) 2024-03-06T15:04:09.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:09.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:09.826642+0000) 2024-03-06T15:04:10.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:10.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:10.826824+0000) 2024-03-06T15:04:11.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:11.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:11.827021+0000) 2024-03-06T15:04:12.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:12.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:12.827200+0000) 2024-03-06T15:04:13.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:13.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:13.827420+0000) 2024-03-06T15:04:14.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:14.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:14.827679+0000) 2024-03-06T15:04:15.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:15.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:15.827965+0000) 2024-03-06T15:04:16.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:16.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:16.828178+0000) 2024-03-06T15:04:17.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:17.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:17.828417+0000) 2024-03-06T15:04:18.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:18.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:18.828657+0000) 2024-03-06T15:04:19.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:19.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:19.828900+0000) 2024-03-06T15:04:20.758 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:04:20.767 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:04:20.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:20.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:20.829081+0000) 2024-03-06T15:04:21.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:21.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:21.829268+0000) 2024-03-06T15:04:22.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:22.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:22.829500+0000) 2024-03-06T15:04:23.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:23.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:23.829728+0000) 2024-03-06T15:04:24.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:24.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:24.829940+0000) 2024-03-06T15:04:25.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:25.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:25.830180+0000) 2024-03-06T15:04:26.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:26.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:26.830389+0000) 2024-03-06T15:04:27.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:27.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:27.830644+0000) 2024-03-06T15:04:28.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:28.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:28.830892+0000) 2024-03-06T15:04:29.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:29.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:29.831143+0000) 2024-03-06T15:04:30.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:30.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:30.831369+0000) 2024-03-06T15:04:31.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:31.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:31.831638+0000) 2024-03-06T15:04:32.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:32.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:32.831840+0000) 2024-03-06T15:04:33.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:33.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:33.832066+0000) 2024-03-06T15:04:34.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:34.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:34.832250+0000) 2024-03-06T15:04:35.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:35.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:35.832484+0000) 2024-03-06T15:04:36.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:36.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:36.832703+0000) 2024-03-06T15:04:37.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:37.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:37.832887+0000) 2024-03-06T15:04:38.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:38.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:38.833069+0000) 2024-03-06T15:04:39.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:39.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:39.833296+0000) 2024-03-06T15:04:40.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:40.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:40.833520+0000) 2024-03-06T15:04:41.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:41.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:41.833780+0000) 2024-03-06T15:04:42.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:42.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:42.833957+0000) 2024-03-06T15:04:43.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:43.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:43.834152+0000) 2024-03-06T15:04:44.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:44.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:44.834383+0000) 2024-03-06T15:04:45.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:45.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:45.834657+0000) 2024-03-06T15:04:46.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:46.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:46.834884+0000) 2024-03-06T15:04:47.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:47.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:47.835116+0000) 2024-03-06T15:04:48.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:48.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:48.835339+0000) 2024-03-06T15:04:49.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:49.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:49.835528+0000) 2024-03-06T15:04:50.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:50.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:50.835793+0000) 2024-03-06T15:04:50.980 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:04:50.988 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:04:51.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:51.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:51.836042+0000) 2024-03-06T15:04:52.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:52.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:52.836282+0000) 2024-03-06T15:04:53.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:53.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:53.836574+0000) 2024-03-06T15:04:54.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:54.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:54.836774+0000) 2024-03-06T15:04:55.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:55.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:55.836967+0000) 2024-03-06T15:04:56.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:56.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:56.837193+0000) 2024-03-06T15:04:57.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:57.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:57.837421+0000) 2024-03-06T15:04:58.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:58.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:58.837645+0000) 2024-03-06T15:04:59.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:04:59.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:04:59.837875+0000) 2024-03-06T15:05:00.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:00.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:00.838101+0000) 2024-03-06T15:05:01.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:01.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:01.838333+0000) 2024-03-06T15:05:02.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:02.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:02.838597+0000) 2024-03-06T15:05:03.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:03.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:03.838846+0000) 2024-03-06T15:05:04.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:04.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:04.839130+0000) 2024-03-06T15:05:05.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:05.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:05.839361+0000) 2024-03-06T15:05:06.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:06.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:06.839613+0000) 2024-03-06T15:05:07.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:07.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:07.839862+0000) 2024-03-06T15:05:08.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:08.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:08.840142+0000) 2024-03-06T15:05:09.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:09.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:09.840424+0000) 2024-03-06T15:05:10.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:10.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:10.840669+0000) 2024-03-06T15:05:11.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:11.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:11.840892+0000) 2024-03-06T15:05:12.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:12.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:12.841111+0000) 2024-03-06T15:05:13.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:13.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:13.841375+0000) 2024-03-06T15:05:14.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:14.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:14.841659+0000) 2024-03-06T15:05:15.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:15.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:15.841938+0000) 2024-03-06T15:05:16.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:16.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:16.842152+0000) 2024-03-06T15:05:17.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:17.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:17.842406+0000) 2024-03-06T15:05:18.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:18.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:18.842670+0000) 2024-03-06T15:05:19.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:19.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:19.842917+0000) 2024-03-06T15:05:20.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:20.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:20.843190+0000) 2024-03-06T15:05:21.176 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:05:21.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:05:21.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:21.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:21.843478+0000) 2024-03-06T15:05:22.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:22.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:22.843692+0000) 2024-03-06T15:05:23.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:23.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:23.843927+0000) 2024-03-06T15:05:24.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:24.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:24.844168+0000) 2024-03-06T15:05:25.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:25.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:25.844406+0000) 2024-03-06T15:05:26.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:26.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:26.844661+0000) 2024-03-06T15:05:27.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:27.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:27.844905+0000) 2024-03-06T15:05:28.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:28.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:28.845146+0000) 2024-03-06T15:05:29.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:29.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:29.845360+0000) 2024-03-06T15:05:30.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:30.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:30.845591+0000) 2024-03-06T15:05:31.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:31.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:31.845785+0000) 2024-03-06T15:05:32.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:32.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:32.846020+0000) 2024-03-06T15:05:33.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:33.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:33.846271+0000) 2024-03-06T15:05:34.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:34.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:34.846490+0000) 2024-03-06T15:05:35.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:35.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:35.846779+0000) 2024-03-06T15:05:36.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:36.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:36.846956+0000) 2024-03-06T15:05:37.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:37.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:37.847248+0000) 2024-03-06T15:05:38.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:38.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:38.847509+0000) 2024-03-06T15:05:39.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:39.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:39.847764+0000) 2024-03-06T15:05:40.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:40.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:40.847996+0000) 2024-03-06T15:05:41.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:41.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:41.848232+0000) 2024-03-06T15:05:42.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:42.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:42.848450+0000) 2024-03-06T15:05:43.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:43.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:43.848667+0000) 2024-03-06T15:05:44.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:44.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:44.848875+0000) 2024-03-06T15:05:45.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:45.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:45.849105+0000) 2024-03-06T15:05:46.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:46.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:46.849356+0000) 2024-03-06T15:05:47.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:47.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:47.849680+0000) 2024-03-06T15:05:48.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:48.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:48.849912+0000) 2024-03-06T15:05:49.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:49.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:49.850196+0000) 2024-03-06T15:05:50.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:50.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:50.850476+0000) 2024-03-06T15:05:51.388 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:05:51.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:05:51.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:51.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:51.850697+0000) 2024-03-06T15:05:52.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:52.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:52.850917+0000) 2024-03-06T15:05:53.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:53.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:53.851144+0000) 2024-03-06T15:05:54.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:54.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:54.851363+0000) 2024-03-06T15:05:55.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:55.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:55.851638+0000) 2024-03-06T15:05:56.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:56.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:56.851838+0000) 2024-03-06T15:05:57.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:57.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:57.852042+0000) 2024-03-06T15:05:58.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:58.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:58.852339+0000) 2024-03-06T15:05:59.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:05:59.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:05:59.852601+0000) 2024-03-06T15:06:00.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:00.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:00.852830+0000) 2024-03-06T15:06:01.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:01.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:01.853058+0000) 2024-03-06T15:06:02.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:02.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:02.853290+0000) 2024-03-06T15:06:03.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:03.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:03.853486+0000) 2024-03-06T15:06:04.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:04.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:04.853718+0000) 2024-03-06T15:06:05.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:05.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:05.853943+0000) 2024-03-06T15:06:06.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:06.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:06.854176+0000) 2024-03-06T15:06:07.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:07.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:07.854374+0000) 2024-03-06T15:06:08.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:08.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:08.854643+0000) 2024-03-06T15:06:09.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:09.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:09.854868+0000) 2024-03-06T15:06:10.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:10.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:10.855089+0000) 2024-03-06T15:06:11.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:11.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:11.855311+0000) 2024-03-06T15:06:12.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:12.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:12.855570+0000) 2024-03-06T15:06:13.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:13.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:13.855760+0000) 2024-03-06T15:06:14.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:14.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:14.855962+0000) 2024-03-06T15:06:15.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:15.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:15.856162+0000) 2024-03-06T15:06:16.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:16.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:16.856351+0000) 2024-03-06T15:06:17.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:17.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:17.856586+0000) 2024-03-06T15:06:18.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:18.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:18.856781+0000) 2024-03-06T15:06:19.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:19.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:19.857016+0000) 2024-03-06T15:06:20.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:20.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:20.857239+0000) 2024-03-06T15:06:21.602 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:06:21.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:06:21.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:21.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:21.857494+0000) 2024-03-06T15:06:22.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:22.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:22.857773+0000) 2024-03-06T15:06:23.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:23.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:23.858018+0000) 2024-03-06T15:06:24.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:24.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:24.858255+0000) 2024-03-06T15:06:25.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:25.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:25.858503+0000) 2024-03-06T15:06:26.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:26.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:26.858790+0000) 2024-03-06T15:06:27.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:27.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:27.858990+0000) 2024-03-06T15:06:28.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:28.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:28.859225+0000) 2024-03-06T15:06:29.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:29.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:29.859456+0000) 2024-03-06T15:06:30.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:30.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:30.859694+0000) 2024-03-06T15:06:31.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:31.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:31.859937+0000) 2024-03-06T15:06:32.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:32.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:32.860173+0000) 2024-03-06T15:06:33.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:33.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:33.860418+0000) 2024-03-06T15:06:34.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:34.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:34.860636+0000) 2024-03-06T15:06:35.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:35.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:35.860850+0000) 2024-03-06T15:06:36.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:36.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:36.861090+0000) 2024-03-06T15:06:37.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:37.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:37.861330+0000) 2024-03-06T15:06:38.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:38.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:38.861590+0000) 2024-03-06T15:06:39.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:39.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:39.861827+0000) 2024-03-06T15:06:40.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:40.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:40.862051+0000) 2024-03-06T15:06:41.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:41.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:41.862256+0000) 2024-03-06T15:06:42.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:42.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:42.862489+0000) 2024-03-06T15:06:43.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:43.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:43.862754+0000) 2024-03-06T15:06:44.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:44.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:44.862971+0000) 2024-03-06T15:06:45.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:45.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:45.863182+0000) 2024-03-06T15:06:46.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:46.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:46.863447+0000) 2024-03-06T15:06:47.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:47.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:47.863677+0000) 2024-03-06T15:06:48.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:48.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:48.863877+0000) 2024-03-06T15:06:49.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:49.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:49.864105+0000) 2024-03-06T15:06:50.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:50.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:50.864327+0000) 2024-03-06T15:06:51.739 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:06:51.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:06:51.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:51.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:51.864605+0000) 2024-03-06T15:06:52.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:52.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:52.864793+0000) 2024-03-06T15:06:53.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:53.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:53.864998+0000) 2024-03-06T15:06:54.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:54.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:54.865218+0000) 2024-03-06T15:06:55.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:55.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:55.865451+0000) 2024-03-06T15:06:56.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:56.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:56.865650+0000) 2024-03-06T15:06:57.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:57.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:57.865899+0000) 2024-03-06T15:06:58.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:58.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:58.866145+0000) 2024-03-06T15:06:59.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:06:59.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:06:59.866389+0000) 2024-03-06T15:07:00.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:00.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:00.866665+0000) 2024-03-06T15:07:01.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:01.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:01.866915+0000) 2024-03-06T15:07:02.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:02.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:02.867121+0000) 2024-03-06T15:07:03.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:03.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:03.867318+0000) 2024-03-06T15:07:04.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:04.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:04.867564+0000) 2024-03-06T15:07:05.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:05.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:05.867766+0000) 2024-03-06T15:07:06.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:06.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:06.867996+0000) 2024-03-06T15:07:07.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:07.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:07.868246+0000) 2024-03-06T15:07:08.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:08.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:08.868498+0000) 2024-03-06T15:07:09.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:09.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:09.868739+0000) 2024-03-06T15:07:10.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:10.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:10.868978+0000) 2024-03-06T15:07:11.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:11.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:11.869174+0000) 2024-03-06T15:07:12.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:12.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:12.869416+0000) 2024-03-06T15:07:13.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:13.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:13.869670+0000) 2024-03-06T15:07:14.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:14.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:14.869907+0000) 2024-03-06T15:07:15.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:15.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:15.870153+0000) 2024-03-06T15:07:16.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:16.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:16.870390+0000) 2024-03-06T15:07:17.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:17.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:17.870671+0000) 2024-03-06T15:07:18.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:18.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:18.870918+0000) 2024-03-06T15:07:19.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:19.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:19.871164+0000) 2024-03-06T15:07:20.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:20.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:20.871399+0000) 2024-03-06T15:07:21.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:21.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:21.871671+0000) 2024-03-06T15:07:21.951 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:07:21.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:07:22.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:22.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:22.871916+0000) 2024-03-06T15:07:23.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:23.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:23.872150+0000) 2024-03-06T15:07:24.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:24.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:24.872351+0000) 2024-03-06T15:07:25.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:25.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:25.872649+0000) 2024-03-06T15:07:26.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:26.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:26.872870+0000) 2024-03-06T15:07:27.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:27.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:27.873069+0000) 2024-03-06T15:07:28.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:28.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:28.873312+0000) 2024-03-06T15:07:29.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:29.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:29.873584+0000) 2024-03-06T15:07:30.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:30.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:30.873816+0000) 2024-03-06T15:07:31.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:31.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:31.874058+0000) 2024-03-06T15:07:32.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:32.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:32.874291+0000) 2024-03-06T15:07:33.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:33.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:33.874578+0000) 2024-03-06T15:07:34.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:34.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:34.874834+0000) 2024-03-06T15:07:35.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:35.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:35.875116+0000) 2024-03-06T15:07:36.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:36.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:36.875380+0000) 2024-03-06T15:07:37.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:37.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:37.875660+0000) 2024-03-06T15:07:38.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:38.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:38.875885+0000) 2024-03-06T15:07:39.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:39.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:39.876172+0000) 2024-03-06T15:07:40.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:40.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:40.876403+0000) 2024-03-06T15:07:41.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:41.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:41.876701+0000) 2024-03-06T15:07:42.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:42.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:42.876921+0000) 2024-03-06T15:07:43.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:43.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:43.877124+0000) 2024-03-06T15:07:44.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:44.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:44.877353+0000) 2024-03-06T15:07:45.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:45.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:45.877659+0000) 2024-03-06T15:07:46.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:46.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:46.877865+0000) 2024-03-06T15:07:47.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:47.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:47.878116+0000) 2024-03-06T15:07:48.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:48.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:48.878341+0000) 2024-03-06T15:07:49.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:49.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:49.878566+0000) 2024-03-06T15:07:50.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:50.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:50.878809+0000) 2024-03-06T15:07:51.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:51.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:51.879049+0000) 2024-03-06T15:07:52.166 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:07:52.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:07:52.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:52.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:52.879290+0000) 2024-03-06T15:07:53.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:53.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:53.879574+0000) 2024-03-06T15:07:54.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:54.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:54.879759+0000) 2024-03-06T15:07:55.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:55.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:55.879929+0000) 2024-03-06T15:07:56.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:56.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:56.880122+0000) 2024-03-06T15:07:57.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:57.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:57.880370+0000) 2024-03-06T15:07:58.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:58.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:58.880675+0000) 2024-03-06T15:07:59.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:07:59.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:07:59.880943+0000) 2024-03-06T15:08:00.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:00.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:00.881171+0000) 2024-03-06T15:08:01.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:01.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:01.881413+0000) 2024-03-06T15:08:02.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:02.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:02.881642+0000) 2024-03-06T15:08:03.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:03.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:03.881836+0000) 2024-03-06T15:08:04.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:04.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:04.882048+0000) 2024-03-06T15:08:05.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:05.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:05.882297+0000) 2024-03-06T15:08:06.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:06.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:06.882525+0000) 2024-03-06T15:08:07.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:07.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:07.882767+0000) 2024-03-06T15:08:08.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:08.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:08.883000+0000) 2024-03-06T15:08:09.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:09.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:09.883227+0000) 2024-03-06T15:08:10.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:10.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:10.883487+0000) 2024-03-06T15:08:11.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:11.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:11.883703+0000) 2024-03-06T15:08:12.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:12.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:12.883938+0000) 2024-03-06T15:08:13.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:13.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:13.884162+0000) 2024-03-06T15:08:14.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:14.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:14.884375+0000) 2024-03-06T15:08:15.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:15.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:15.884636+0000) 2024-03-06T15:08:16.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:16.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:16.884820+0000) 2024-03-06T15:08:17.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:17.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:17.885065+0000) 2024-03-06T15:08:18.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:18.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:18.885291+0000) 2024-03-06T15:08:19.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:19.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:19.885517+0000) 2024-03-06T15:08:20.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:20.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:20.885772+0000) 2024-03-06T15:08:21.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:21.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:21.885957+0000) 2024-03-06T15:08:22.369 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:08:22.379 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:08:22.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:22.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:22.886259+0000) 2024-03-06T15:08:23.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:23.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:23.886496+0000) 2024-03-06T15:08:24.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:24.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:24.886726+0000) 2024-03-06T15:08:25.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:25.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:25.886929+0000) 2024-03-06T15:08:26.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:26.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:26.887146+0000) 2024-03-06T15:08:27.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:27.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:27.887382+0000) 2024-03-06T15:08:28.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:28.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:28.887588+0000) 2024-03-06T15:08:29.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:29.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:29.887795+0000) 2024-03-06T15:08:30.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:30.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:30.888010+0000) 2024-03-06T15:08:31.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:31.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:31.888234+0000) 2024-03-06T15:08:32.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:32.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:32.888432+0000) 2024-03-06T15:08:33.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:33.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:33.888673+0000) 2024-03-06T15:08:34.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:34.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:34.888865+0000) 2024-03-06T15:08:35.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:35.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:35.889096+0000) 2024-03-06T15:08:36.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:36.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:36.889319+0000) 2024-03-06T15:08:37.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:37.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:37.889584+0000) 2024-03-06T15:08:38.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:38.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:38.889838+0000) 2024-03-06T15:08:39.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:39.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:39.890124+0000) 2024-03-06T15:08:40.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:40.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:40.890402+0000) 2024-03-06T15:08:41.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:41.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:41.890689+0000) 2024-03-06T15:08:42.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:42.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:42.890965+0000) 2024-03-06T15:08:43.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:43.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:43.891241+0000) 2024-03-06T15:08:44.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:44.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:44.891484+0000) 2024-03-06T15:08:45.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:45.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:45.891753+0000) 2024-03-06T15:08:46.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:46.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:46.892036+0000) 2024-03-06T15:08:47.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:47.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:47.892319+0000) 2024-03-06T15:08:48.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:48.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:48.892666+0000) 2024-03-06T15:08:49.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:49.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:49.892885+0000) 2024-03-06T15:08:50.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:50.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:50.893147+0000) 2024-03-06T15:08:51.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:51.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:51.893400+0000) 2024-03-06T15:08:52.569 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:08:52.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:08:52.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:52.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:52.893637+0000) 2024-03-06T15:08:53.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:53.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:53.893884+0000) 2024-03-06T15:08:54.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:54.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:54.894111+0000) 2024-03-06T15:08:55.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:55.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:55.894335+0000) 2024-03-06T15:08:56.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:56.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:56.894585+0000) 2024-03-06T15:08:57.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:57.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:57.894826+0000) 2024-03-06T15:08:58.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:58.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:58.895059+0000) 2024-03-06T15:08:59.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:08:59.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:08:59.895293+0000) 2024-03-06T15:09:00.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:00.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:00.895479+0000) 2024-03-06T15:09:01.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:01.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:01.895678+0000) 2024-03-06T15:09:02.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:02.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:02.895923+0000) 2024-03-06T15:09:03.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:03.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:03.896134+0000) 2024-03-06T15:09:04.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:04.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:04.896329+0000) 2024-03-06T15:09:05.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:05.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:05.896637+0000) 2024-03-06T15:09:06.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:06.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:06.896857+0000) 2024-03-06T15:09:07.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:07.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:07.897104+0000) 2024-03-06T15:09:08.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:08.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:08.897339+0000) 2024-03-06T15:09:09.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:09.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:09.897636+0000) 2024-03-06T15:09:10.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:10.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:10.897871+0000) 2024-03-06T15:09:11.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:11.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:11.898114+0000) 2024-03-06T15:09:12.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:12.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:12.898353+0000) 2024-03-06T15:09:13.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:13.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:13.898640+0000) 2024-03-06T15:09:14.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:14.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:14.898882+0000) 2024-03-06T15:09:15.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:15.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:15.899176+0000) 2024-03-06T15:09:16.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:16.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:16.899450+0000) 2024-03-06T15:09:17.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:17.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:17.899680+0000) 2024-03-06T15:09:18.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:18.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:18.899929+0000) 2024-03-06T15:09:19.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:19.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:19.900158+0000) 2024-03-06T15:09:20.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:20.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:20.900382+0000) 2024-03-06T15:09:21.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:21.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:21.900663+0000) 2024-03-06T15:09:22.789 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:09:22.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:09:22.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:22.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:22.900897+0000) 2024-03-06T15:09:23.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:23.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:23.901097+0000) 2024-03-06T15:09:24.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:24.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:24.901324+0000) 2024-03-06T15:09:25.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:25.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:25.901573+0000) 2024-03-06T15:09:26.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:26.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:26.901801+0000) 2024-03-06T15:09:27.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:27.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:27.902038+0000) 2024-03-06T15:09:28.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:28.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:28.902279+0000) 2024-03-06T15:09:29.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:29.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:29.902514+0000) 2024-03-06T15:09:30.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:30.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:30.902777+0000) 2024-03-06T15:09:31.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:31.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:31.902969+0000) 2024-03-06T15:09:32.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:32.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:32.903191+0000) 2024-03-06T15:09:33.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:33.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:33.903386+0000) 2024-03-06T15:09:34.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:34.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:34.903616+0000) 2024-03-06T15:09:35.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:35.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:35.903873+0000) 2024-03-06T15:09:36.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:36.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:36.904144+0000) 2024-03-06T15:09:37.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:37.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:37.904435+0000) 2024-03-06T15:09:38.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:38.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:38.904749+0000) 2024-03-06T15:09:39.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:39.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:39.904975+0000) 2024-03-06T15:09:40.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:40.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:40.905194+0000) 2024-03-06T15:09:41.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:41.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:41.905393+0000) 2024-03-06T15:09:42.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:42.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:42.905688+0000) 2024-03-06T15:09:43.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:43.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:43.905977+0000) 2024-03-06T15:09:44.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:44.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:44.906260+0000) 2024-03-06T15:09:45.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:45.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:45.906470+0000) 2024-03-06T15:09:46.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:46.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:46.906696+0000) 2024-03-06T15:09:47.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:47.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:47.906938+0000) 2024-03-06T15:09:48.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:48.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:48.907182+0000) 2024-03-06T15:09:49.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:49.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:49.907405+0000) 2024-03-06T15:09:50.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:50.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:50.907675+0000) 2024-03-06T15:09:51.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:51.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:51.907886+0000) 2024-03-06T15:09:52.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:52.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:52.908097+0000) 2024-03-06T15:09:52.945 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:09:52.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:09:53.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:53.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:53.908341+0000) 2024-03-06T15:09:54.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:54.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:54.908606+0000) 2024-03-06T15:09:55.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:55.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:55.908838+0000) 2024-03-06T15:09:56.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:56.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:56.909026+0000) 2024-03-06T15:09:57.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:57.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:57.909303+0000) 2024-03-06T15:09:58.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:58.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:58.909523+0000) 2024-03-06T15:09:59.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:09:59.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:09:59.909779+0000) 2024-03-06T15:10:00.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:00.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:00.910006+0000) 2024-03-06T15:10:01.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:01.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:01.910233+0000) 2024-03-06T15:10:02.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:02.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:02.910429+0000) 2024-03-06T15:10:03.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:03.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:03.910750+0000) 2024-03-06T15:10:04.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:04.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:04.910934+0000) 2024-03-06T15:10:05.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:05.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:05.911167+0000) 2024-03-06T15:10:06.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:06.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:06.911390+0000) 2024-03-06T15:10:07.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:07.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:07.911677+0000) 2024-03-06T15:10:08.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:08.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:08.911922+0000) 2024-03-06T15:10:09.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:09.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:09.912151+0000) 2024-03-06T15:10:10.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:10.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:10.912373+0000) 2024-03-06T15:10:11.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:11.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:11.912632+0000) 2024-03-06T15:10:12.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:12.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:12.912864+0000) 2024-03-06T15:10:13.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:13.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:13.913071+0000) 2024-03-06T15:10:14.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:14.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:14.913289+0000) 2024-03-06T15:10:15.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:15.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:15.913516+0000) 2024-03-06T15:10:16.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:16.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:16.913785+0000) 2024-03-06T15:10:17.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:17.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:17.914012+0000) 2024-03-06T15:10:18.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:18.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:18.914254+0000) 2024-03-06T15:10:19.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:19.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:19.914480+0000) 2024-03-06T15:10:20.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:20.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:20.914712+0000) 2024-03-06T15:10:21.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:21.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:21.914941+0000) 2024-03-06T15:10:22.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:22.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:22.915166+0000) 2024-03-06T15:10:23.163 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:10:23.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:10:23.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:23.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:23.915405+0000) 2024-03-06T15:10:24.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:24.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:24.915646+0000) 2024-03-06T15:10:25.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:25.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:25.915884+0000) 2024-03-06T15:10:26.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:26.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:26.916112+0000) 2024-03-06T15:10:27.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:27.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:27.916343+0000) 2024-03-06T15:10:28.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:28.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:28.916609+0000) 2024-03-06T15:10:29.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:29.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:29.916826+0000) 2024-03-06T15:10:30.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:30.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:30.917050+0000) 2024-03-06T15:10:31.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:31.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:31.917275+0000) 2024-03-06T15:10:32.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:32.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:32.917500+0000) 2024-03-06T15:10:33.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:33.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:33.917735+0000) 2024-03-06T15:10:34.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:34.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:34.917963+0000) 2024-03-06T15:10:35.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:35.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:35.918204+0000) 2024-03-06T15:10:36.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:36.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:36.918426+0000) 2024-03-06T15:10:37.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:37.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:37.918668+0000) 2024-03-06T15:10:38.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:38.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:38.918908+0000) 2024-03-06T15:10:39.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:39.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:39.919132+0000) 2024-03-06T15:10:40.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:40.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:40.919350+0000) 2024-03-06T15:10:41.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:41.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:41.919621+0000) 2024-03-06T15:10:42.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:42.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:42.919763+0000) 2024-03-06T15:10:43.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:43.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:43.920021+0000) 2024-03-06T15:10:44.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:44.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:44.920267+0000) 2024-03-06T15:10:45.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:45.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:45.920514+0000) 2024-03-06T15:10:46.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:46.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:46.920803+0000) 2024-03-06T15:10:47.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:47.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:47.921045+0000) 2024-03-06T15:10:48.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:48.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:48.921294+0000) 2024-03-06T15:10:49.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:49.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:49.921502+0000) 2024-03-06T15:10:50.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:50.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:50.921787+0000) 2024-03-06T15:10:51.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:51.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:51.921998+0000) 2024-03-06T15:10:52.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:52.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:52.922218+0000) 2024-03-06T15:10:53.393 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:10:53.401 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:10:53.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:53.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:53.922432+0000) 2024-03-06T15:10:54.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:54.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:54.922656+0000) 2024-03-06T15:10:55.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:55.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:55.922831+0000) 2024-03-06T15:10:56.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:56.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:56.923116+0000) 2024-03-06T15:10:57.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:57.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:57.923358+0000) 2024-03-06T15:10:58.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:58.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:58.923663+0000) 2024-03-06T15:10:59.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:10:59.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:10:59.923872+0000) 2024-03-06T15:11:00.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:00.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:00.924114+0000) 2024-03-06T15:11:01.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:01.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:01.924360+0000) 2024-03-06T15:11:02.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:02.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:02.924649+0000) 2024-03-06T15:11:03.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:03.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:03.924850+0000) 2024-03-06T15:11:04.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:04.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:04.925079+0000) 2024-03-06T15:11:05.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:05.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:05.925310+0000) 2024-03-06T15:11:06.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:06.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:06.925563+0000) 2024-03-06T15:11:07.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:07.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:07.925801+0000) 2024-03-06T15:11:08.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:08.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:08.926033+0000) 2024-03-06T15:11:09.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:09.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:09.926256+0000) 2024-03-06T15:11:10.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:10.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:10.926520+0000) 2024-03-06T15:11:11.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:11.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:11.926814+0000) 2024-03-06T15:11:12.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:12.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:12.927020+0000) 2024-03-06T15:11:13.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:13.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:13.927228+0000) 2024-03-06T15:11:14.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:14.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:14.927466+0000) 2024-03-06T15:11:15.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:15.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:15.927728+0000) 2024-03-06T15:11:16.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:16.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:16.927981+0000) 2024-03-06T15:11:17.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:17.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:17.928197+0000) 2024-03-06T15:11:18.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:18.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:18.928473+0000) 2024-03-06T15:11:19.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:19.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:19.928753+0000) 2024-03-06T15:11:20.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:20.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:20.929015+0000) 2024-03-06T15:11:21.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:21.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:21.929300+0000) 2024-03-06T15:11:22.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:22.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:22.929615+0000) 2024-03-06T15:11:23.615 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:11:23.626 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:11:23.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:23.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:23.929842+0000) 2024-03-06T15:11:24.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:24.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:24.930038+0000) 2024-03-06T15:11:25.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:25.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:25.930268+0000) 2024-03-06T15:11:26.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:26.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:26.930455+0000) 2024-03-06T15:11:27.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:27.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:27.930708+0000) 2024-03-06T15:11:28.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:28.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:28.930902+0000) 2024-03-06T15:11:29.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:29.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:29.931135+0000) 2024-03-06T15:11:30.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:30.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:30.931355+0000) 2024-03-06T15:11:31.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:31.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:31.931592+0000) 2024-03-06T15:11:32.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:32.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:32.931884+0000) 2024-03-06T15:11:33.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:33.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:33.932134+0000) 2024-03-06T15:11:34.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:34.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:34.932360+0000) 2024-03-06T15:11:35.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:35.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:35.932631+0000) 2024-03-06T15:11:36.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:36.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:36.932830+0000) 2024-03-06T15:11:37.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:37.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:37.933116+0000) 2024-03-06T15:11:38.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:38.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:38.933373+0000) 2024-03-06T15:11:39.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:39.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:39.933637+0000) 2024-03-06T15:11:40.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:40.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:40.933841+0000) 2024-03-06T15:11:41.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:41.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:41.934095+0000) 2024-03-06T15:11:42.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:42.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:42.934315+0000) 2024-03-06T15:11:43.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:43.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:43.934604+0000) 2024-03-06T15:11:44.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:44.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:44.934830+0000) 2024-03-06T15:11:45.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:45.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:45.935066+0000) 2024-03-06T15:11:46.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:46.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:46.935288+0000) 2024-03-06T15:11:47.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:47.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:47.935511+0000) 2024-03-06T15:11:48.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:48.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:48.935810+0000) 2024-03-06T15:11:49.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:49.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:49.936002+0000) 2024-03-06T15:11:50.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:50.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:50.936233+0000) 2024-03-06T15:11:51.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:51.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:51.936458+0000) 2024-03-06T15:11:52.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:52.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:52.936653+0000) 2024-03-06T15:11:53.835 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:11:53.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:11:53.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:53.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:53.936830+0000) 2024-03-06T15:11:54.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:54.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:54.937048+0000) 2024-03-06T15:11:55.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:55.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:55.937284+0000) 2024-03-06T15:11:56.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:56.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:56.937512+0000) 2024-03-06T15:11:57.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:57.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:57.937782+0000) 2024-03-06T15:11:58.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:58.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:58.938022+0000) 2024-03-06T15:11:59.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:11:59.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:11:59.938269+0000) 2024-03-06T15:12:00.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:00.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:00.938499+0000) 2024-03-06T15:12:01.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:01.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:01.938717+0000) 2024-03-06T15:12:02.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:02.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:02.938949+0000) 2024-03-06T15:12:03.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:03.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:03.939137+0000) 2024-03-06T15:12:04.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:04.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:04.939358+0000) 2024-03-06T15:12:05.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:05.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:05.939642+0000) 2024-03-06T15:12:06.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:06.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:06.939869+0000) 2024-03-06T15:12:07.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:07.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:07.940096+0000) 2024-03-06T15:12:08.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:08.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:08.940338+0000) 2024-03-06T15:12:09.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:09.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:09.940623+0000) 2024-03-06T15:12:10.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:10.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:10.940838+0000) 2024-03-06T15:12:11.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:11.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:11.941085+0000) 2024-03-06T15:12:12.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:12.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:12.941310+0000) 2024-03-06T15:12:13.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:13.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:13.941570+0000) 2024-03-06T15:12:14.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:14.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:14.941810+0000) 2024-03-06T15:12:15.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:15.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:15.942013+0000) 2024-03-06T15:12:16.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:16.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:16.942225+0000) 2024-03-06T15:12:17.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:17.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:17.942464+0000) 2024-03-06T15:12:18.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:18.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:18.942751+0000) 2024-03-06T15:12:19.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:19.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:19.942985+0000) 2024-03-06T15:12:20.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:20.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:20.943222+0000) 2024-03-06T15:12:21.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:21.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:21.943456+0000) 2024-03-06T15:12:22.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:22.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:22.943688+0000) 2024-03-06T15:12:23.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:23.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:23.943899+0000) 2024-03-06T15:12:23.991 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:12:23.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:12:24.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:24.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:24.944144+0000) 2024-03-06T15:12:25.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:25.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:25.944394+0000) 2024-03-06T15:12:26.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:26.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:26.944648+0000) 2024-03-06T15:12:27.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:27.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:27.944866+0000) 2024-03-06T15:12:28.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:28.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:28.945129+0000) 2024-03-06T15:12:29.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:29.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:29.945385+0000) 2024-03-06T15:12:30.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:30.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:30.945656+0000) 2024-03-06T15:12:31.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:31.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:31.945868+0000) 2024-03-06T15:12:32.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:32.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:32.946129+0000) 2024-03-06T15:12:33.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:33.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:33.946373+0000) 2024-03-06T15:12:34.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:34.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:34.946640+0000) 2024-03-06T15:12:35.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:35.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:35.946863+0000) 2024-03-06T15:12:36.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:36.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:36.947084+0000) 2024-03-06T15:12:37.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:37.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:37.947312+0000) 2024-03-06T15:12:38.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:38.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:38.947559+0000) 2024-03-06T15:12:39.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:39.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:39.947843+0000) 2024-03-06T15:12:40.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:40.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:40.948105+0000) 2024-03-06T15:12:41.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:41.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:41.948391+0000) 2024-03-06T15:12:42.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:42.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:42.948635+0000) 2024-03-06T15:12:43.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:43.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:43.948884+0000) 2024-03-06T15:12:44.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:44.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:44.949123+0000) 2024-03-06T15:12:45.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:45.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:45.949359+0000) 2024-03-06T15:12:46.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:46.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:46.949654+0000) 2024-03-06T15:12:47.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:47.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:47.949817+0000) 2024-03-06T15:12:48.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:48.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:48.950094+0000) 2024-03-06T15:12:49.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:49.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:49.950381+0000) 2024-03-06T15:12:50.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:50.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:50.950680+0000) 2024-03-06T15:12:51.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:51.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:51.950925+0000) 2024-03-06T15:12:52.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:52.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:52.951199+0000) 2024-03-06T15:12:53.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:53.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:53.951446+0000) 2024-03-06T15:12:54.205 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:12:54.213 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:12:54.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:54.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:54.951723+0000) 2024-03-06T15:12:55.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:55.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:55.952007+0000) 2024-03-06T15:12:56.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:56.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:56.952286+0000) 2024-03-06T15:12:57.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:57.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:57.952610+0000) 2024-03-06T15:12:58.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:58.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:58.952906+0000) 2024-03-06T15:12:59.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:12:59.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:12:59.953156+0000) 2024-03-06T15:13:00.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:00.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:00.953383+0000) 2024-03-06T15:13:01.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:01.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:01.953701+0000) 2024-03-06T15:13:02.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:02.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:02.953998+0000) 2024-03-06T15:13:03.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:03.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:03.954245+0000) 2024-03-06T15:13:04.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:04.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:04.954458+0000) 2024-03-06T15:13:05.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:05.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:05.954723+0000) 2024-03-06T15:13:06.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:06.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:06.954957+0000) 2024-03-06T15:13:07.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:07.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:07.955198+0000) 2024-03-06T15:13:08.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:08.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:08.955433+0000) 2024-03-06T15:13:09.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:09.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:09.955666+0000) 2024-03-06T15:13:10.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:10.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:10.955912+0000) 2024-03-06T15:13:11.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:11.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:11.956145+0000) 2024-03-06T15:13:12.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:12.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:12.956400+0000) 2024-03-06T15:13:13.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:13.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:13.956696+0000) 2024-03-06T15:13:14.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:14.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:14.956935+0000) 2024-03-06T15:13:15.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:15.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:15.957185+0000) 2024-03-06T15:13:16.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:16.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:16.957470+0000) 2024-03-06T15:13:17.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:17.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:17.957750+0000) 2024-03-06T15:13:18.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:18.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:18.958024+0000) 2024-03-06T15:13:19.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:19.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:19.958276+0000) 2024-03-06T15:13:20.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:20.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:20.958514+0000) 2024-03-06T15:13:21.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:21.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:21.958772+0000) 2024-03-06T15:13:22.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:22.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:22.958963+0000) 2024-03-06T15:13:23.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:23.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:23.959221+0000) 2024-03-06T15:13:24.419 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:13:24.427 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:13:24.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:24.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:24.959406+0000) 2024-03-06T15:13:25.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:25.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:25.959704+0000) 2024-03-06T15:13:26.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:26.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:26.960021+0000) 2024-03-06T15:13:27.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:27.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:27.960297+0000) 2024-03-06T15:13:28.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:28.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:28.960529+0000) 2024-03-06T15:13:29.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:29.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:29.960811+0000) 2024-03-06T15:13:30.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:30.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:30.961012+0000) 2024-03-06T15:13:31.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:31.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:31.961226+0000) 2024-03-06T15:13:32.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:32.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:32.961428+0000) 2024-03-06T15:13:33.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:33.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:33.961678+0000) 2024-03-06T15:13:34.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:34.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:34.961875+0000) 2024-03-06T15:13:35.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:35.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:35.962086+0000) 2024-03-06T15:13:36.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:36.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:36.962328+0000) 2024-03-06T15:13:37.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:37.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:37.962567+0000) 2024-03-06T15:13:38.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:38.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:38.962802+0000) 2024-03-06T15:13:39.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:39.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:39.963030+0000) 2024-03-06T15:13:40.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:40.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:40.963267+0000) 2024-03-06T15:13:41.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:41.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:41.963511+0000) 2024-03-06T15:13:42.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:42.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:42.963745+0000) 2024-03-06T15:13:43.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:43.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:43.963977+0000) 2024-03-06T15:13:44.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:44.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:44.964168+0000) 2024-03-06T15:13:45.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:45.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:45.964381+0000) 2024-03-06T15:13:46.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:46.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:46.964637+0000) 2024-03-06T15:13:47.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:47.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:47.964876+0000) 2024-03-06T15:13:48.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:48.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:48.965123+0000) 2024-03-06T15:13:49.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:49.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:49.965373+0000) 2024-03-06T15:13:50.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:50.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:50.965664+0000) 2024-03-06T15:13:51.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:51.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:51.965941+0000) 2024-03-06T15:13:52.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:52.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:52.966218+0000) 2024-03-06T15:13:53.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:53.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:53.966472+0000) 2024-03-06T15:13:54.629 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:13:54.637 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:13:54.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:54.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:54.966757+0000) 2024-03-06T15:13:55.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:55.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:55.966989+0000) 2024-03-06T15:13:56.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:56.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:56.967176+0000) 2024-03-06T15:13:57.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:57.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:57.967413+0000) 2024-03-06T15:13:58.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:58.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:58.967669+0000) 2024-03-06T15:13:59.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:13:59.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:13:59.967868+0000) 2024-03-06T15:14:00.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:00.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:00.968091+0000) 2024-03-06T15:14:01.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:01.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:01.968312+0000) 2024-03-06T15:14:02.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:02.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:02.968587+0000) 2024-03-06T15:14:03.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:03.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:03.968826+0000) 2024-03-06T15:14:04.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:04.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:04.969037+0000) 2024-03-06T15:14:05.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:05.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:05.969298+0000) 2024-03-06T15:14:06.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:06.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:06.969518+0000) 2024-03-06T15:14:07.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:07.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:07.969774+0000) 2024-03-06T15:14:08.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:08.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:08.970051+0000) 2024-03-06T15:14:09.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:09.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:09.970330+0000) 2024-03-06T15:14:10.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:10.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:10.970648+0000) 2024-03-06T15:14:11.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:11.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:11.970888+0000) 2024-03-06T15:14:12.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:12.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:12.971106+0000) 2024-03-06T15:14:13.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:13.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:13.971344+0000) 2024-03-06T15:14:14.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:14.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:14.971715+0000) 2024-03-06T15:14:15.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:15.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:15.971910+0000) 2024-03-06T15:14:16.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:16.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:16.972147+0000) 2024-03-06T15:14:17.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:17.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:17.972381+0000) 2024-03-06T15:14:18.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:18.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:18.972674+0000) 2024-03-06T15:14:19.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:19.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:19.972901+0000) 2024-03-06T15:14:20.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:20.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:20.973095+0000) 2024-03-06T15:14:21.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:21.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:21.973323+0000) 2024-03-06T15:14:22.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:22.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:22.973582+0000) 2024-03-06T15:14:23.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:23.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:23.973817+0000) 2024-03-06T15:14:24.843 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:14:24.851 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:14:24.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:24.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:24.973980+0000) 2024-03-06T15:14:25.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:25.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:25.974209+0000) 2024-03-06T15:14:26.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:26.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:26.974435+0000) 2024-03-06T15:14:27.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:27.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:27.974668+0000) 2024-03-06T15:14:28.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:28.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:28.974901+0000) 2024-03-06T15:14:29.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:29.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:29.975157+0000) 2024-03-06T15:14:30.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:30.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:30.975528+0000) 2024-03-06T15:14:31.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:31.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:31.975825+0000) 2024-03-06T15:14:32.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:32.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:32.976101+0000) 2024-03-06T15:14:33.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:33.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:33.976392+0000) 2024-03-06T15:14:34.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:34.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:34.976664+0000) 2024-03-06T15:14:35.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:35.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:35.976861+0000) 2024-03-06T15:14:36.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:36.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:36.977049+0000) 2024-03-06T15:14:37.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:37.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:37.977284+0000) 2024-03-06T15:14:38.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:38.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:38.977518+0000) 2024-03-06T15:14:39.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:39.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:39.977768+0000) 2024-03-06T15:14:40.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:40.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:40.977958+0000) 2024-03-06T15:14:41.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:41.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:41.978140+0000) 2024-03-06T15:14:42.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:42.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:42.978362+0000) 2024-03-06T15:14:43.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:43.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:43.978742+0000) 2024-03-06T15:14:44.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:44.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:44.979019+0000) 2024-03-06T15:14:45.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:45.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:45.979263+0000) 2024-03-06T15:14:46.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:46.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:46.979459+0000) 2024-03-06T15:14:47.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:47.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:47.979693+0000) 2024-03-06T15:14:48.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:48.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:48.979929+0000) 2024-03-06T15:14:49.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:49.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:49.980139+0000) 2024-03-06T15:14:50.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:50.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:50.980373+0000) 2024-03-06T15:14:51.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:51.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:51.980606+0000) 2024-03-06T15:14:52.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:52.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:52.980889+0000) 2024-03-06T15:14:53.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:53.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:53.981152+0000) 2024-03-06T15:14:54.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:54.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:54.981385+0000) 2024-03-06T15:14:55.047 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:14:55.056 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:14:55.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:55.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:55.981656+0000) 2024-03-06T15:14:56.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:56.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:56.981898+0000) 2024-03-06T15:14:57.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:57.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:57.982141+0000) 2024-03-06T15:14:58.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:58.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:58.982361+0000) 2024-03-06T15:14:59.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:14:59.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:14:59.982588+0000) 2024-03-06T15:15:00.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:00.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:00.982824+0000) 2024-03-06T15:15:01.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:01.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:01.983046+0000) 2024-03-06T15:15:02.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:02.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:02.983271+0000) 2024-03-06T15:15:03.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:03.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:03.983520+0000) 2024-03-06T15:15:04.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:04.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:04.983801+0000) 2024-03-06T15:15:05.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:05.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:05.984053+0000) 2024-03-06T15:15:06.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:06.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:06.984309+0000) 2024-03-06T15:15:07.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:07.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:07.984636+0000) 2024-03-06T15:15:08.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:08.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:08.984869+0000) 2024-03-06T15:15:09.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:09.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:09.985100+0000) 2024-03-06T15:15:10.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:10.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:10.985342+0000) 2024-03-06T15:15:11.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:11.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:11.985621+0000) 2024-03-06T15:15:12.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:12.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:12.985820+0000) 2024-03-06T15:15:13.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:13.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:13.986054+0000) 2024-03-06T15:15:14.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:14.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:14.986277+0000) 2024-03-06T15:15:15.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:15.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:15.986515+0000) 2024-03-06T15:15:16.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:16.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:16.986775+0000) 2024-03-06T15:15:17.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:17.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:17.986957+0000) 2024-03-06T15:15:18.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:18.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:18.987154+0000) 2024-03-06T15:15:19.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:19.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:19.987351+0000) 2024-03-06T15:15:20.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:20.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:20.987626+0000) 2024-03-06T15:15:21.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:21.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:21.987851+0000) 2024-03-06T15:15:22.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:22.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:22.988067+0000) 2024-03-06T15:15:23.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:23.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:23.988260+0000) 2024-03-06T15:15:24.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:24.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:24.988487+0000) 2024-03-06T15:15:25.218 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:15:25.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:15:25.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:25.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:25.988731+0000) 2024-03-06T15:15:26.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:26.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:26.988923+0000) 2024-03-06T15:15:27.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:27.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:27.989171+0000) 2024-03-06T15:15:28.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:28.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:28.989407+0000) 2024-03-06T15:15:29.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:29.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:29.989653+0000) 2024-03-06T15:15:30.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:30.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:30.989873+0000) 2024-03-06T15:15:31.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:31.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:31.990101+0000) 2024-03-06T15:15:32.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:32.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:32.990330+0000) 2024-03-06T15:15:33.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:33.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:33.990614+0000) 2024-03-06T15:15:34.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:34.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:34.990850+0000) 2024-03-06T15:15:35.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:35.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:35.991089+0000) 2024-03-06T15:15:36.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:36.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:36.991285+0000) 2024-03-06T15:15:37.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:37.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:37.991520+0000) 2024-03-06T15:15:38.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:38.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:38.991795+0000) 2024-03-06T15:15:39.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:39.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:39.992041+0000) 2024-03-06T15:15:40.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:40.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:40.992264+0000) 2024-03-06T15:15:41.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:41.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:41.992471+0000) 2024-03-06T15:15:42.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:42.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:42.992689+0000) 2024-03-06T15:15:43.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:43.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:43.992904+0000) 2024-03-06T15:15:44.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:44.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:44.993142+0000) 2024-03-06T15:15:45.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:45.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:45.993371+0000) 2024-03-06T15:15:46.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:46.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:46.993622+0000) 2024-03-06T15:15:48.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:47.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:47.993801+0000) 2024-03-06T15:15:48.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:48.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:48.994033+0000) 2024-03-06T15:15:49.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:49.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:49.994232+0000) 2024-03-06T15:15:50.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:50.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:50.994519+0000) 2024-03-06T15:15:51.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:51.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:51.994735+0000) 2024-03-06T15:15:52.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:52.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:52.994961+0000) 2024-03-06T15:15:53.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:53.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:53.995155+0000) 2024-03-06T15:15:54.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:54.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:54.995341+0000) 2024-03-06T15:15:55.431 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:15:55.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:15:55.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:55.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:55.995561+0000) 2024-03-06T15:15:56.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:56.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:56.995798+0000) 2024-03-06T15:15:58.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:57.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:57.996024+0000) 2024-03-06T15:15:59.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:58.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:58.996261+0000) 2024-03-06T15:16:00.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:15:59.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:15:59.996526+0000) 2024-03-06T15:16:01.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:00.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:00.996802+0000) 2024-03-06T15:16:02.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:01.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:01.997023+0000) 2024-03-06T15:16:03.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:02.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:02.997240+0000) 2024-03-06T15:16:04.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:03.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:03.997435+0000) 2024-03-06T15:16:05.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:04.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:04.997679+0000) 2024-03-06T15:16:06.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:05.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:05.997907+0000) 2024-03-06T15:16:07.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:06.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:06.998131+0000) 2024-03-06T15:16:08.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:07.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:07.998335+0000) 2024-03-06T15:16:09.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:08.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:08.998640+0000) 2024-03-06T15:16:10.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:09.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:09.998848+0000) 2024-03-06T15:16:11.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:10.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:10.999083+0000) 2024-03-06T15:16:12.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:11.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:11.999302+0000) 2024-03-06T15:16:13.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:12.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:12.999520+0000) 2024-03-06T15:16:14.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:13.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:13.999789+0000) 2024-03-06T15:16:15.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:14.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:14.999971+0000) 2024-03-06T15:16:16.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:15.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:16.000204+0000) 2024-03-06T15:16:17.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:16.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:17.000451+0000) 2024-03-06T15:16:18.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:18.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:18.000709+0000) 2024-03-06T15:16:19.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:19.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:19.000942+0000) 2024-03-06T15:16:20.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:20.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:20.001127+0000) 2024-03-06T15:16:21.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:21.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:21.001356+0000) 2024-03-06T15:16:22.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:22.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:22.001591+0000) 2024-03-06T15:16:23.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:23.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:23.001820+0000) 2024-03-06T15:16:24.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:24.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:24.002032+0000) 2024-03-06T15:16:25.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:25.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:25.002242+0000) 2024-03-06T15:16:25.661 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:16:25.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:16:26.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:26.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:26.002439+0000) 2024-03-06T15:16:27.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:27.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:27.002661+0000) 2024-03-06T15:16:28.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:28.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:28.002904+0000) 2024-03-06T15:16:29.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:29.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:29.003151+0000) 2024-03-06T15:16:30.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:30.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:30.003398+0000) 2024-03-06T15:16:31.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:31.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:31.003663+0000) 2024-03-06T15:16:32.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:32.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:32.003896+0000) 2024-03-06T15:16:33.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:33.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:33.004135+0000) 2024-03-06T15:16:34.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:34.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:34.004372+0000) 2024-03-06T15:16:35.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:35.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:35.004635+0000) 2024-03-06T15:16:36.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:36.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:36.004880+0000) 2024-03-06T15:16:37.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:37.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:37.005119+0000) 2024-03-06T15:16:38.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:38.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:38.005342+0000) 2024-03-06T15:16:39.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:39.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:39.005640+0000) 2024-03-06T15:16:40.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:40.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:40.005872+0000) 2024-03-06T15:16:41.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:41.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:41.006092+0000) 2024-03-06T15:16:42.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:42.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:42.006320+0000) 2024-03-06T15:16:43.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:43.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:43.006592+0000) 2024-03-06T15:16:44.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:44.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:44.006848+0000) 2024-03-06T15:16:45.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:45.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:45.007006+0000) 2024-03-06T15:16:46.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:46.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:46.007198+0000) 2024-03-06T15:16:47.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:47.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:47.007426+0000) 2024-03-06T15:16:48.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:48.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:48.007655+0000) 2024-03-06T15:16:49.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:49.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:49.007936+0000) 2024-03-06T15:16:50.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:50.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:50.008165+0000) 2024-03-06T15:16:51.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:51.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:51.008391+0000) 2024-03-06T15:16:52.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:52.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:52.008607+0000) 2024-03-06T15:16:53.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:53.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:53.008793+0000) 2024-03-06T15:16:54.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:54.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:54.009014+0000) 2024-03-06T15:16:55.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:55.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:55.009279+0000) 2024-03-06T15:16:55.873 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:16:55.882 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:16:56.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:56.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:56.009500+0000) 2024-03-06T15:16:57.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:57.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:57.009722+0000) 2024-03-06T15:16:58.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:58.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:58.009948+0000) 2024-03-06T15:16:59.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:16:59.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:16:59.010196+0000) 2024-03-06T15:17:00.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:00.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:00.010427+0000) 2024-03-06T15:17:01.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:01.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:01.010662+0000) 2024-03-06T15:17:02.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:02.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:02.010932+0000) 2024-03-06T15:17:03.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:03.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:03.011166+0000) 2024-03-06T15:17:04.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:04.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:04.011361+0000) 2024-03-06T15:17:05.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:05.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:05.011633+0000) 2024-03-06T15:17:06.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:06.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:06.011880+0000) 2024-03-06T15:17:07.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:07.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:07.012149+0000) 2024-03-06T15:17:08.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:08.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:08.012366+0000) 2024-03-06T15:17:09.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:09.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:09.012655+0000) 2024-03-06T15:17:10.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:10.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:10.012863+0000) 2024-03-06T15:17:11.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:11.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:11.013087+0000) 2024-03-06T15:17:12.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:12.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:12.013269+0000) 2024-03-06T15:17:13.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:13.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:13.013487+0000) 2024-03-06T15:17:14.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:14.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:14.013706+0000) 2024-03-06T15:17:15.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:15.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:15.013924+0000) 2024-03-06T15:17:16.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:16.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:16.014157+0000) 2024-03-06T15:17:17.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:17.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:17.014381+0000) 2024-03-06T15:17:18.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:18.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:18.014648+0000) 2024-03-06T15:17:19.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:19.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:19.014857+0000) 2024-03-06T15:17:20.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:20.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:20.015102+0000) 2024-03-06T15:17:21.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:21.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:21.015329+0000) 2024-03-06T15:17:22.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:22.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:22.015592+0000) 2024-03-06T15:17:23.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:23.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:23.015779+0000) 2024-03-06T15:17:24.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:24.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:24.015965+0000) 2024-03-06T15:17:25.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:25.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:25.016174+0000) 2024-03-06T15:17:26.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:26.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:26.016367+0000) 2024-03-06T15:17:26.095 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:17:26.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:17:27.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:27.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:27.016630+0000) 2024-03-06T15:17:28.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:28.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:28.016859+0000) 2024-03-06T15:17:29.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:29.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:29.017084+0000) 2024-03-06T15:17:30.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:30.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:30.017309+0000) 2024-03-06T15:17:31.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:31.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:31.017492+0000) 2024-03-06T15:17:32.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:32.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:32.017772+0000) 2024-03-06T15:17:33.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:33.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:33.017988+0000) 2024-03-06T15:17:34.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:34.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:34.018229+0000) 2024-03-06T15:17:35.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:35.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:35.018460+0000) 2024-03-06T15:17:36.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:36.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:36.018687+0000) 2024-03-06T15:17:37.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:37.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:37.018963+0000) 2024-03-06T15:17:38.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:38.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:38.019185+0000) 2024-03-06T15:17:39.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:39.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:39.019408+0000) 2024-03-06T15:17:40.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:40.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:40.019665+0000) 2024-03-06T15:17:41.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:41.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:41.019877+0000) 2024-03-06T15:17:42.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:42.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:42.020111+0000) 2024-03-06T15:17:43.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:43.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:43.020335+0000) 2024-03-06T15:17:44.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:44.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:44.020596+0000) 2024-03-06T15:17:45.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:45.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:45.020842+0000) 2024-03-06T15:17:46.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:46.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:46.021065+0000) 2024-03-06T15:17:47.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:47.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:47.021257+0000) 2024-03-06T15:17:48.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:48.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:48.021503+0000) 2024-03-06T15:17:49.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:49.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:49.021768+0000) 2024-03-06T15:17:50.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:50.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:50.021962+0000) 2024-03-06T15:17:51.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:51.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:51.022195+0000) 2024-03-06T15:17:52.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:52.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:52.022416+0000) 2024-03-06T15:17:53.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:53.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:53.022646+0000) 2024-03-06T15:17:54.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:54.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:54.022881+0000) 2024-03-06T15:17:55.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:55.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:55.023137+0000) 2024-03-06T15:17:56.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:56.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:56.023379+0000) 2024-03-06T15:17:56.265 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:17:56.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:17:57.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:57.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:57.023591+0000) 2024-03-06T15:17:58.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:58.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:58.023785+0000) 2024-03-06T15:17:59.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:17:59.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:17:59.024030+0000) 2024-03-06T15:18:00.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:00.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:00.024227+0000) 2024-03-06T15:18:01.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:01.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:01.024422+0000) 2024-03-06T15:18:02.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:02.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:02.024656+0000) 2024-03-06T15:18:03.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:03.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:03.024832+0000) 2024-03-06T15:18:04.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:04.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:04.025043+0000) 2024-03-06T15:18:05.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:05.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:05.025270+0000) 2024-03-06T15:18:06.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:06.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:06.025500+0000) 2024-03-06T15:18:07.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:07.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:07.025703+0000) 2024-03-06T15:18:08.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:08.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:08.025908+0000) 2024-03-06T15:18:09.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:09.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:09.026190+0000) 2024-03-06T15:18:10.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:10.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:10.026468+0000) 2024-03-06T15:18:11.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:11.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:11.026747+0000) 2024-03-06T15:18:12.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:12.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:12.027027+0000) 2024-03-06T15:18:13.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:13.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:13.027302+0000) 2024-03-06T15:18:14.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:14.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:14.027513+0000) 2024-03-06T15:18:15.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:15.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:15.027827+0000) 2024-03-06T15:18:16.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:16.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:16.028113+0000) 2024-03-06T15:18:17.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:17.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:17.028360+0000) 2024-03-06T15:18:18.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:18.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:18.028622+0000) 2024-03-06T15:18:19.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:19.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:19.028832+0000) 2024-03-06T15:18:20.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:20.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:20.029043+0000) 2024-03-06T15:18:21.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:21.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:21.029266+0000) 2024-03-06T15:18:22.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:22.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:22.029456+0000) 2024-03-06T15:18:23.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:23.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:23.029689+0000) 2024-03-06T15:18:24.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:24.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:24.029915+0000) 2024-03-06T15:18:25.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:25.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:25.030199+0000) 2024-03-06T15:18:26.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:26.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:26.030424+0000) 2024-03-06T15:18:26.455 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:18:26.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:18:27.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:27.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:27.030650+0000) 2024-03-06T15:18:28.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:28.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:28.030875+0000) 2024-03-06T15:18:29.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:29.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:29.031061+0000) 2024-03-06T15:18:30.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:30.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:30.031306+0000) 2024-03-06T15:18:31.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:31.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:31.031637+0000) 2024-03-06T15:18:32.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:32.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:32.031919+0000) 2024-03-06T15:18:33.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:33.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:33.032185+0000) 2024-03-06T15:18:34.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:34.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:34.032461+0000) 2024-03-06T15:18:35.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:35.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:35.032690+0000) 2024-03-06T15:18:36.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:36.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:36.032918+0000) 2024-03-06T15:18:37.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:37.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:37.033198+0000) 2024-03-06T15:18:38.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:38.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:38.033441+0000) 2024-03-06T15:18:39.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:39.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:39.033711+0000) 2024-03-06T15:18:40.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:40.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:40.033945+0000) 2024-03-06T15:18:41.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:41.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:41.034223+0000) 2024-03-06T15:18:42.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:42.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:42.034459+0000) 2024-03-06T15:18:43.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:43.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:43.034679+0000) 2024-03-06T15:18:44.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:44.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:44.034911+0000) 2024-03-06T15:18:45.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:45.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:45.035131+0000) 2024-03-06T15:18:46.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:46.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:46.035381+0000) 2024-03-06T15:18:47.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:47.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:47.035648+0000) 2024-03-06T15:18:48.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:48.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:48.035857+0000) 2024-03-06T15:18:49.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:49.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:49.036070+0000) 2024-03-06T15:18:50.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:50.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:50.036342+0000) 2024-03-06T15:18:51.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:51.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:51.036636+0000) 2024-03-06T15:18:52.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:52.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:52.036883+0000) 2024-03-06T15:18:53.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:53.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:53.037117+0000) 2024-03-06T15:18:54.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:54.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:54.037344+0000) 2024-03-06T15:18:55.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:55.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:55.037608+0000) 2024-03-06T15:18:56.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:56.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:56.037865+0000) 2024-03-06T15:18:56.650 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:18:56.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:18:57.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:57.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:57.038057+0000) 2024-03-06T15:18:58.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:58.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:58.038244+0000) 2024-03-06T15:18:59.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:18:59.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:18:59.038495+0000) 2024-03-06T15:19:00.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:00.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:00.038764+0000) 2024-03-06T15:19:01.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:01.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:01.038999+0000) 2024-03-06T15:19:02.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:02.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:02.039241+0000) 2024-03-06T15:19:03.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:03.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:03.039439+0000) 2024-03-06T15:19:04.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:04.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:04.039678+0000) 2024-03-06T15:19:05.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:05.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:05.039883+0000) 2024-03-06T15:19:06.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:06.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:06.040110+0000) 2024-03-06T15:19:07.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:07.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:07.040332+0000) 2024-03-06T15:19:08.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:08.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:08.040512+0000) 2024-03-06T15:19:09.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:09.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:09.040784+0000) 2024-03-06T15:19:10.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:10.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:10.041020+0000) 2024-03-06T15:19:11.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:11.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:11.041261+0000) 2024-03-06T15:19:12.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:12.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:12.041498+0000) 2024-03-06T15:19:13.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:13.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:13.041776+0000) 2024-03-06T15:19:14.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:14.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:14.042017+0000) 2024-03-06T15:19:15.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:15.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:15.042262+0000) 2024-03-06T15:19:16.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:16.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:16.042493+0000) 2024-03-06T15:19:17.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:17.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:17.042692+0000) 2024-03-06T15:19:18.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:18.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:18.042904+0000) 2024-03-06T15:19:19.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:19.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:19.043198+0000) 2024-03-06T15:19:20.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:20.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:20.043376+0000) 2024-03-06T15:19:21.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:21.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:21.043679+0000) 2024-03-06T15:19:22.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:22.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:22.043885+0000) 2024-03-06T15:19:23.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:23.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:23.044106+0000) 2024-03-06T15:19:24.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:24.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:24.044293+0000) 2024-03-06T15:19:25.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:25.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:25.044571+0000) 2024-03-06T15:19:26.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:26.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:26.044772+0000) 2024-03-06T15:19:26.863 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:19:26.871 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:19:27.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:27.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:27.045031+0000) 2024-03-06T15:19:28.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:28.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:28.045249+0000) 2024-03-06T15:19:29.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:29.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:29.045486+0000) 2024-03-06T15:19:30.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:30.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:30.045715+0000) 2024-03-06T15:19:31.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:31.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:31.045967+0000) 2024-03-06T15:19:32.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:32.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:32.046172+0000) 2024-03-06T15:19:33.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:33.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:33.046364+0000) 2024-03-06T15:19:34.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:34.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:34.046608+0000) 2024-03-06T15:19:35.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:35.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:35.046841+0000) 2024-03-06T15:19:36.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:36.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:36.047102+0000) 2024-03-06T15:19:37.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:37.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:37.047312+0000) 2024-03-06T15:19:38.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:38.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:38.047594+0000) 2024-03-06T15:19:39.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:39.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:39.047799+0000) 2024-03-06T15:19:40.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:40.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:40.048004+0000) 2024-03-06T15:19:41.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:41.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:41.048211+0000) 2024-03-06T15:19:42.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:42.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:42.048428+0000) 2024-03-06T15:19:43.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:43.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:43.048673+0000) 2024-03-06T15:19:44.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:44.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:44.048896+0000) 2024-03-06T15:19:45.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:45.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:45.049137+0000) 2024-03-06T15:19:46.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:46.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:46.049368+0000) 2024-03-06T15:19:47.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:47.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:47.049623+0000) 2024-03-06T15:19:48.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:48.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:48.049855+0000) 2024-03-06T15:19:49.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:49.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:49.050091+0000) 2024-03-06T15:19:50.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:50.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:50.050314+0000) 2024-03-06T15:19:51.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:51.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:51.050576+0000) 2024-03-06T15:19:52.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:52.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:52.050809+0000) 2024-03-06T15:19:53.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:53.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:53.051011+0000) 2024-03-06T15:19:54.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:54.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:54.051254+0000) 2024-03-06T15:19:55.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:55.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:55.051464+0000) 2024-03-06T15:19:56.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:56.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:56.051719+0000) 2024-03-06T15:19:57.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:57.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:57.051954+0000) 2024-03-06T15:19:57.067 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:19:57.075 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:19:58.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:58.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:58.052176+0000) 2024-03-06T15:19:59.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:19:59.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:19:59.052410+0000) 2024-03-06T15:20:00.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:00.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:00.052650+0000) 2024-03-06T15:20:01.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:01.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:01.052834+0000) 2024-03-06T15:20:02.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:02.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:02.053055+0000) 2024-03-06T15:20:03.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:03.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:03.053276+0000) 2024-03-06T15:20:04.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:04.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:04.053472+0000) 2024-03-06T15:20:05.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:05.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:05.053698+0000) 2024-03-06T15:20:06.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:06.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:06.053943+0000) 2024-03-06T15:20:07.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:07.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:07.054187+0000) 2024-03-06T15:20:08.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:08.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:08.054375+0000) 2024-03-06T15:20:09.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:09.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:09.054642+0000) 2024-03-06T15:20:10.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:10.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:10.054877+0000) 2024-03-06T15:20:11.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:11.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:11.055099+0000) 2024-03-06T15:20:12.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:12.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:12.055285+0000) 2024-03-06T15:20:13.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:13.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:13.055522+0000) 2024-03-06T15:20:14.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:14.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:14.055779+0000) 2024-03-06T15:20:15.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:15.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:15.056016+0000) 2024-03-06T15:20:16.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:16.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:16.056268+0000) 2024-03-06T15:20:17.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:17.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:17.056509+0000) 2024-03-06T15:20:18.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:18.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:18.056798+0000) 2024-03-06T15:20:19.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:19.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:19.057046+0000) 2024-03-06T15:20:20.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:20.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:20.057295+0000) 2024-03-06T15:20:21.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:21.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:21.057575+0000) 2024-03-06T15:20:22.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:22.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:22.057823+0000) 2024-03-06T15:20:23.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:23.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:23.058026+0000) 2024-03-06T15:20:24.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:24.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:24.058226+0000) 2024-03-06T15:20:25.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:25.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:25.058470+0000) 2024-03-06T15:20:26.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:26.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:26.058714+0000) 2024-03-06T15:20:27.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:27.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:27.058915+0000) 2024-03-06T15:20:27.256 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:20:27.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:20:28.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:28.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:28.059163+0000) 2024-03-06T15:20:29.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:29.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:29.059365+0000) 2024-03-06T15:20:30.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:30.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:30.059612+0000) 2024-03-06T15:20:31.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:31.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:31.059883+0000) 2024-03-06T15:20:32.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:32.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:32.060163+0000) 2024-03-06T15:20:33.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:33.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:33.060381+0000) 2024-03-06T15:20:34.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:34.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:34.060647+0000) 2024-03-06T15:20:35.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:35.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:35.060936+0000) 2024-03-06T15:20:36.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:36.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:36.061165+0000) 2024-03-06T15:20:37.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:37.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:37.061443+0000) 2024-03-06T15:20:38.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:38.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:38.061710+0000) 2024-03-06T15:20:39.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:39.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:39.061947+0000) 2024-03-06T15:20:40.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:40.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:40.062179+0000) 2024-03-06T15:20:41.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:41.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:41.062462+0000) 2024-03-06T15:20:42.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:42.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:42.062728+0000) 2024-03-06T15:20:43.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:43.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:43.063000+0000) 2024-03-06T15:20:44.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:44.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:44.063280+0000) 2024-03-06T15:20:45.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:45.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:45.063586+0000) 2024-03-06T15:20:46.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:46.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:46.063844+0000) 2024-03-06T15:20:47.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:47.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:47.064099+0000) 2024-03-06T15:20:48.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:48.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:48.064376+0000) 2024-03-06T15:20:49.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:49.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:49.064691+0000) 2024-03-06T15:20:50.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:50.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:50.064924+0000) 2024-03-06T15:20:51.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:51.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:51.065198+0000) 2024-03-06T15:20:52.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:52.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:52.065464+0000) 2024-03-06T15:20:53.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:53.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:53.065739+0000) 2024-03-06T15:20:54.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:54.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:54.065943+0000) 2024-03-06T15:20:55.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:55.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:55.066150+0000) 2024-03-06T15:20:56.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:56.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:56.066398+0000) 2024-03-06T15:20:57.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:57.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:57.066658+0000) 2024-03-06T15:20:57.417 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:20:57.425 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:20:58.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:58.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:58.066911+0000) 2024-03-06T15:20:59.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:20:59.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:20:59.067209+0000) 2024-03-06T15:21:00.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:00.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:00.067469+0000) 2024-03-06T15:21:01.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:01.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:01.067701+0000) 2024-03-06T15:21:02.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:02.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:02.067933+0000) 2024-03-06T15:21:03.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:03.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:03.068156+0000) 2024-03-06T15:21:04.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:04.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:04.068381+0000) 2024-03-06T15:21:05.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:05.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:05.068640+0000) 2024-03-06T15:21:06.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:06.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:06.068828+0000) 2024-03-06T15:21:07.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:07.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:07.069048+0000) 2024-03-06T15:21:08.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:08.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:08.069251+0000) 2024-03-06T15:21:09.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:09.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:09.069459+0000) 2024-03-06T15:21:10.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:10.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:10.069701+0000) 2024-03-06T15:21:11.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:11.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:11.069926+0000) 2024-03-06T15:21:12.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:12.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:12.070146+0000) 2024-03-06T15:21:13.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:13.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:13.070373+0000) 2024-03-06T15:21:14.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:14.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:14.070634+0000) 2024-03-06T15:21:15.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:15.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:15.070871+0000) 2024-03-06T15:21:16.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:16.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:16.071087+0000) 2024-03-06T15:21:17.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:17.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:17.071281+0000) 2024-03-06T15:21:18.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:18.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:18.071511+0000) 2024-03-06T15:21:19.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:19.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:19.071784+0000) 2024-03-06T15:21:20.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:20.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:20.072060+0000) 2024-03-06T15:21:21.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:21.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:21.072284+0000) 2024-03-06T15:21:22.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:22.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:22.072468+0000) 2024-03-06T15:21:23.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:23.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:23.072694+0000) 2024-03-06T15:21:24.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:24.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:24.072848+0000) 2024-03-06T15:21:25.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:25.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:25.073039+0000) 2024-03-06T15:21:26.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:26.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:26.073264+0000) 2024-03-06T15:21:27.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:27.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:27.073484+0000) 2024-03-06T15:21:27.631 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:21:27.639 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:21:28.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:28.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:28.073696+0000) 2024-03-06T15:21:29.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:29.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:29.073899+0000) 2024-03-06T15:21:30.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:30.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:30.074134+0000) 2024-03-06T15:21:31.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:31.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:31.074312+0000) 2024-03-06T15:21:32.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:32.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:32.074564+0000) 2024-03-06T15:21:33.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:33.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:33.074758+0000) 2024-03-06T15:21:34.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:34.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:34.074961+0000) 2024-03-06T15:21:35.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:35.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:35.075188+0000) 2024-03-06T15:21:36.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:36.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:36.075416+0000) 2024-03-06T15:21:37.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:37.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:37.075642+0000) 2024-03-06T15:21:38.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:38.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:38.075869+0000) 2024-03-06T15:21:39.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:39.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:39.076099+0000) 2024-03-06T15:21:40.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:40.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:40.076338+0000) 2024-03-06T15:21:41.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:41.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:41.076605+0000) 2024-03-06T15:21:42.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:42.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:42.076833+0000) 2024-03-06T15:21:43.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:43.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:43.077053+0000) 2024-03-06T15:21:44.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:44.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:44.077279+0000) 2024-03-06T15:21:45.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:45.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:45.077507+0000) 2024-03-06T15:21:46.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:46.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:46.077752+0000) 2024-03-06T15:21:47.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:47.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:47.077958+0000) 2024-03-06T15:21:48.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:48.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:48.078206+0000) 2024-03-06T15:21:49.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:49.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:49.078396+0000) 2024-03-06T15:21:50.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:50.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:50.078656+0000) 2024-03-06T15:21:51.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:51.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:51.078876+0000) 2024-03-06T15:21:52.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:52.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:52.079100+0000) 2024-03-06T15:21:53.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:53.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:53.079322+0000) 2024-03-06T15:21:54.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:54.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:54.079587+0000) 2024-03-06T15:21:55.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:55.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:55.079811+0000) 2024-03-06T15:21:56.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:56.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:56.080054+0000) 2024-03-06T15:21:57.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:57.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:57.080242+0000) 2024-03-06T15:21:57.775 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:21:57.784 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:21:58.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:58.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:58.080486+0000) 2024-03-06T15:21:59.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:21:59.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:21:59.080718+0000) 2024-03-06T15:22:00.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:00.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:00.080918+0000) 2024-03-06T15:22:01.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:01.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:01.081112+0000) 2024-03-06T15:22:02.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:02.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:02.081334+0000) 2024-03-06T15:22:03.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:03.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:03.081582+0000) 2024-03-06T15:22:04.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:04.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:04.081775+0000) 2024-03-06T15:22:05.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:05.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:05.081996+0000) 2024-03-06T15:22:06.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:06.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:06.082232+0000) 2024-03-06T15:22:07.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:07.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:07.082451+0000) 2024-03-06T15:22:08.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:08.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:08.082655+0000) 2024-03-06T15:22:09.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:09.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:09.082882+0000) 2024-03-06T15:22:10.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:10.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:10.083109+0000) 2024-03-06T15:22:11.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:11.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:11.083303+0000) 2024-03-06T15:22:12.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:12.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:12.083574+0000) 2024-03-06T15:22:13.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:13.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:13.083789+0000) 2024-03-06T15:22:14.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:14.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:14.084020+0000) 2024-03-06T15:22:15.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:15.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:15.084222+0000) 2024-03-06T15:22:16.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:16.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:16.084439+0000) 2024-03-06T15:22:17.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:17.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:17.084670+0000) 2024-03-06T15:22:18.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:18.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:18.084892+0000) 2024-03-06T15:22:19.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:19.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:19.085122+0000) 2024-03-06T15:22:20.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:20.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:20.085352+0000) 2024-03-06T15:22:21.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:21.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:21.085629+0000) 2024-03-06T15:22:22.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:22.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:22.085856+0000) 2024-03-06T15:22:23.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:23.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:23.086075+0000) 2024-03-06T15:22:24.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:24.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:24.086300+0000) 2024-03-06T15:22:25.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:25.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:25.086524+0000) 2024-03-06T15:22:26.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:26.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:26.086778+0000) 2024-03-06T15:22:27.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:27.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:27.086960+0000) 2024-03-06T15:22:27.926 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:22:27.934 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:22:28.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:28.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:28.087212+0000) 2024-03-06T15:22:29.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:29.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:29.087457+0000) 2024-03-06T15:22:30.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:30.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:30.087705+0000) 2024-03-06T15:22:31.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:31.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:31.087918+0000) 2024-03-06T15:22:32.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:32.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:32.088113+0000) 2024-03-06T15:22:33.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:33.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:33.088333+0000) 2024-03-06T15:22:34.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:34.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:34.088603+0000) 2024-03-06T15:22:35.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:35.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:35.088793+0000) 2024-03-06T15:22:36.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:36.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:36.089072+0000) 2024-03-06T15:22:37.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:37.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:37.089289+0000) 2024-03-06T15:22:38.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:38.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:38.089518+0000) 2024-03-06T15:22:39.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:39.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:39.089794+0000) 2024-03-06T15:22:40.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:40.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:40.090037+0000) 2024-03-06T15:22:41.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:41.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:41.090263+0000) 2024-03-06T15:22:42.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:42.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:42.090463+0000) 2024-03-06T15:22:43.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:43.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:43.090675+0000) 2024-03-06T15:22:44.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:44.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:44.090878+0000) 2024-03-06T15:22:45.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:45.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:45.091105+0000) 2024-03-06T15:22:46.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:46.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:46.091298+0000) 2024-03-06T15:22:47.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:47.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:47.091572+0000) 2024-03-06T15:22:48.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:48.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:48.091772+0000) 2024-03-06T15:22:49.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:49.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:49.092008+0000) 2024-03-06T15:22:50.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:50.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:50.092260+0000) 2024-03-06T15:22:51.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:51.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:51.092508+0000) 2024-03-06T15:22:52.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:52.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:52.092734+0000) 2024-03-06T15:22:53.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:53.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:53.092913+0000) 2024-03-06T15:22:54.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:54.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:54.093150+0000) 2024-03-06T15:22:55.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:55.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:55.093402+0000) 2024-03-06T15:22:56.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:56.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:56.093675+0000) 2024-03-06T15:22:57.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:57.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:57.093877+0000) 2024-03-06T15:22:58.055 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:22:58.064 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:22:58.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:58.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:58.094082+0000) 2024-03-06T15:22:59.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:22:59.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:22:59.094309+0000) 2024-03-06T15:23:00.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:00.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:00.094597+0000) 2024-03-06T15:23:01.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:01.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:01.094806+0000) 2024-03-06T15:23:02.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:02.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:02.095034+0000) 2024-03-06T15:23:03.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:03.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:03.095214+0000) 2024-03-06T15:23:04.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:04.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:04.095437+0000) 2024-03-06T15:23:05.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:05.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:05.095668+0000) 2024-03-06T15:23:06.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:06.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:06.095903+0000) 2024-03-06T15:23:07.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:07.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:07.096112+0000) 2024-03-06T15:23:08.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:08.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:08.096355+0000) 2024-03-06T15:23:09.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:09.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:09.096654+0000) 2024-03-06T15:23:10.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:10.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:10.096855+0000) 2024-03-06T15:23:11.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:11.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:11.097064+0000) 2024-03-06T15:23:12.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:12.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:12.097294+0000) 2024-03-06T15:23:13.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:13.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:13.097490+0000) 2024-03-06T15:23:14.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:14.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:14.097698+0000) 2024-03-06T15:23:15.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:15.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:15.097877+0000) 2024-03-06T15:23:16.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:16.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:16.098108+0000) 2024-03-06T15:23:17.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:17.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:17.098313+0000) 2024-03-06T15:23:18.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:18.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:18.098566+0000) 2024-03-06T15:23:19.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:19.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:19.098811+0000) 2024-03-06T15:23:20.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:20.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:20.099014+0000) 2024-03-06T15:23:21.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:21.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:21.099205+0000) 2024-03-06T15:23:22.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:22.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:22.099404+0000) 2024-03-06T15:23:23.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:23.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:23.099652+0000) 2024-03-06T15:23:24.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:24.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:24.099866+0000) 2024-03-06T15:23:25.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:25.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:25.100128+0000) 2024-03-06T15:23:26.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:26.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:26.100362+0000) 2024-03-06T15:23:27.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:27.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:27.100580+0000) 2024-03-06T15:23:28.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:28.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:28.100803+0000) 2024-03-06T15:23:28.193 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:23:28.204 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:23:29.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:29.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:29.101047+0000) 2024-03-06T15:23:30.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:30.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:30.101280+0000) 2024-03-06T15:23:31.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:31.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:31.101505+0000) 2024-03-06T15:23:32.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:32.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:32.101772+0000) 2024-03-06T15:23:33.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:33.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:33.101991+0000) 2024-03-06T15:23:34.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:34.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:34.102221+0000) 2024-03-06T15:23:35.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:35.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:35.102455+0000) 2024-03-06T15:23:36.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:36.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:36.102663+0000) 2024-03-06T15:23:37.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:37.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:37.102896+0000) 2024-03-06T15:23:38.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:38.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:38.103120+0000) 2024-03-06T15:23:39.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:39.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:39.103348+0000) 2024-03-06T15:23:40.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:40.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:40.103620+0000) 2024-03-06T15:23:41.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:41.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:41.103785+0000) 2024-03-06T15:23:42.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:42.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:42.103992+0000) 2024-03-06T15:23:43.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:43.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:43.104188+0000) 2024-03-06T15:23:44.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:44.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:44.104394+0000) 2024-03-06T15:23:45.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:45.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:45.104685+0000) 2024-03-06T15:23:46.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:46.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:46.104918+0000) 2024-03-06T15:23:47.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:47.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:47.105140+0000) 2024-03-06T15:23:48.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:48.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:48.105357+0000) 2024-03-06T15:23:49.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:49.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:49.105606+0000) 2024-03-06T15:23:50.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:50.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:50.105818+0000) 2024-03-06T15:23:51.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:51.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:51.106053+0000) 2024-03-06T15:23:52.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:52.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:52.106279+0000) 2024-03-06T15:23:53.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:53.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:53.106497+0000) 2024-03-06T15:23:54.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:54.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:54.106733+0000) 2024-03-06T15:23:55.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:55.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:55.106932+0000) 2024-03-06T15:23:56.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:56.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:56.107127+0000) 2024-03-06T15:23:57.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:57.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:57.107357+0000) 2024-03-06T15:23:58.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:58.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:58.107644+0000) 2024-03-06T15:23:58.415 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:23:58.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:23:59.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:23:59.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:23:59.107855+0000) 2024-03-06T15:24:00.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:00.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:00.108086+0000) 2024-03-06T15:24:01.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:01.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:01.108276+0000) 2024-03-06T15:24:02.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:02.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:02.108457+0000) 2024-03-06T15:24:03.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:03.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:03.108658+0000) 2024-03-06T15:24:04.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:04.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:04.108888+0000) 2024-03-06T15:24:05.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:05.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:05.109108+0000) 2024-03-06T15:24:06.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:06.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:06.109389+0000) 2024-03-06T15:24:07.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:07.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:07.109668+0000) 2024-03-06T15:24:08.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:08.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:08.109843+0000) 2024-03-06T15:24:09.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:09.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:09.110088+0000) 2024-03-06T15:24:10.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:10.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:10.110318+0000) 2024-03-06T15:24:11.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:11.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:11.110581+0000) 2024-03-06T15:24:12.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:12.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:12.110832+0000) 2024-03-06T15:24:13.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:13.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:13.111072+0000) 2024-03-06T15:24:14.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:14.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:14.111286+0000) 2024-03-06T15:24:15.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:15.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:15.111497+0000) 2024-03-06T15:24:16.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:16.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:16.111779+0000) 2024-03-06T15:24:17.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:17.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:17.112014+0000) 2024-03-06T15:24:18.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:18.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:18.112215+0000) 2024-03-06T15:24:19.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:19.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:19.112458+0000) 2024-03-06T15:24:20.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:20.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:20.112682+0000) 2024-03-06T15:24:21.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:21.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:21.112925+0000) 2024-03-06T15:24:22.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:22.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:22.113127+0000) 2024-03-06T15:24:23.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:23.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:23.113319+0000) 2024-03-06T15:24:24.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:24.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:24.113568+0000) 2024-03-06T15:24:25.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:25.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:25.113792+0000) 2024-03-06T15:24:26.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:26.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:26.113990+0000) 2024-03-06T15:24:27.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:27.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:27.114209+0000) 2024-03-06T15:24:28.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:28.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:28.114399+0000) 2024-03-06T15:24:28.612 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:24:28.620 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:24:29.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:29.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:29.114640+0000) 2024-03-06T15:24:30.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:30.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:30.114871+0000) 2024-03-06T15:24:31.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:31.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:31.115098+0000) 2024-03-06T15:24:32.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:32.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:32.115317+0000) 2024-03-06T15:24:33.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:33.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:33.115506+0000) 2024-03-06T15:24:34.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:34.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:34.115764+0000) 2024-03-06T15:24:35.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:35.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:35.115991+0000) 2024-03-06T15:24:36.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:36.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:36.116211+0000) 2024-03-06T15:24:37.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:37.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:37.116489+0000) 2024-03-06T15:24:38.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:38.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:38.116661+0000) 2024-03-06T15:24:39.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:39.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:39.116856+0000) 2024-03-06T15:24:40.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:40.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:40.117096+0000) 2024-03-06T15:24:41.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:41.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:41.117335+0000) 2024-03-06T15:24:42.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:42.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:42.117575+0000) 2024-03-06T15:24:43.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:43.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:43.117830+0000) 2024-03-06T15:24:44.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:44.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:44.118075+0000) 2024-03-06T15:24:45.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:45.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:45.118304+0000) 2024-03-06T15:24:46.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:46.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:46.118505+0000) 2024-03-06T15:24:47.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:47.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:47.118759+0000) 2024-03-06T15:24:48.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:48.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:48.118969+0000) 2024-03-06T15:24:49.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:49.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:49.119237+0000) 2024-03-06T15:24:50.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:50.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:50.119479+0000) 2024-03-06T15:24:51.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:51.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:51.119722+0000) 2024-03-06T15:24:52.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:52.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:52.119921+0000) 2024-03-06T15:24:53.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:53.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:53.120159+0000) 2024-03-06T15:24:54.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:54.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:54.120370+0000) 2024-03-06T15:24:55.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:55.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:55.120646+0000) 2024-03-06T15:24:56.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:56.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:56.120851+0000) 2024-03-06T15:24:57.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:57.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:57.121077+0000) 2024-03-06T15:24:58.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:58.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:58.121269+0000) 2024-03-06T15:24:58.810 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:24:58.818 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:24:59.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:24:59.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:24:59.121524+0000) 2024-03-06T15:25:00.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:00.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:00.121797+0000) 2024-03-06T15:25:01.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:01.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:01.122048+0000) 2024-03-06T15:25:02.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:02.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:02.122276+0000) 2024-03-06T15:25:03.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:03.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:03.122468+0000) 2024-03-06T15:25:04.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:04.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:04.122705+0000) 2024-03-06T15:25:05.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:05.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:05.122911+0000) 2024-03-06T15:25:06.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:06.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:06.123163+0000) 2024-03-06T15:25:07.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:07.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:07.123356+0000) 2024-03-06T15:25:08.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:08.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:08.123582+0000) 2024-03-06T15:25:09.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:09.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:09.123786+0000) 2024-03-06T15:25:10.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:10.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:10.123994+0000) 2024-03-06T15:25:11.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:11.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:11.124304+0000) 2024-03-06T15:25:12.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:12.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:12.124566+0000) 2024-03-06T15:25:13.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:13.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:13.124721+0000) 2024-03-06T15:25:14.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:14.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:14.124932+0000) 2024-03-06T15:25:15.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:15.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:15.125136+0000) 2024-03-06T15:25:16.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:16.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:16.125395+0000) 2024-03-06T15:25:17.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:17.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:17.125636+0000) 2024-03-06T15:25:18.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:18.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:18.125893+0000) 2024-03-06T15:25:19.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:19.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:19.126187+0000) 2024-03-06T15:25:20.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:20.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:20.126422+0000) 2024-03-06T15:25:21.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:21.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:21.126706+0000) 2024-03-06T15:25:22.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:22.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:22.126988+0000) 2024-03-06T15:25:23.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:23.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:23.127257+0000) 2024-03-06T15:25:24.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:24.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:24.127578+0000) 2024-03-06T15:25:25.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:25.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:25.127827+0000) 2024-03-06T15:25:26.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:26.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:26.128118+0000) 2024-03-06T15:25:27.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:27.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:27.128337+0000) 2024-03-06T15:25:28.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:28.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:28.128561+0000) 2024-03-06T15:25:28.946 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:25:28.954 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:25:29.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:29.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:29.128804+0000) 2024-03-06T15:25:30.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:30.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:30.129027+0000) 2024-03-06T15:25:31.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:31.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:31.129259+0000) 2024-03-06T15:25:32.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:32.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:32.129481+0000) 2024-03-06T15:25:33.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:33.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:33.129704+0000) 2024-03-06T15:25:34.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:34.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:34.129883+0000) 2024-03-06T15:25:35.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:35.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:35.130128+0000) 2024-03-06T15:25:36.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:36.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:36.130359+0000) 2024-03-06T15:25:37.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:37.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:37.130626+0000) 2024-03-06T15:25:38.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:38.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:38.130816+0000) 2024-03-06T15:25:39.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:39.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:39.131044+0000) 2024-03-06T15:25:40.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:40.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:40.131273+0000) 2024-03-06T15:25:41.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:41.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:41.131499+0000) 2024-03-06T15:25:42.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:42.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:42.131710+0000) 2024-03-06T15:25:43.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:43.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:43.131937+0000) 2024-03-06T15:25:44.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:44.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:44.132180+0000) 2024-03-06T15:25:45.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:45.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:45.132416+0000) 2024-03-06T15:25:46.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:46.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:46.132663+0000) 2024-03-06T15:25:47.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:47.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:47.132848+0000) 2024-03-06T15:25:48.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:48.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:48.133092+0000) 2024-03-06T15:25:49.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:49.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:49.133347+0000) 2024-03-06T15:25:50.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:50.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:50.133625+0000) 2024-03-06T15:25:51.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:51.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:51.133815+0000) 2024-03-06T15:25:52.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:52.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:52.134050+0000) 2024-03-06T15:25:53.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:53.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:53.134276+0000) 2024-03-06T15:25:54.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:54.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:54.134502+0000) 2024-03-06T15:25:55.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:55.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:55.134709+0000) 2024-03-06T15:25:56.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:56.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:56.134915+0000) 2024-03-06T15:25:57.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:57.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:57.135102+0000) 2024-03-06T15:25:58.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:58.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:58.135326+0000) 2024-03-06T15:25:59.123 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:25:59.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:25:59.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:25:59.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:25:59.135596+0000) 2024-03-06T15:26:00.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:00.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:00.135805+0000) 2024-03-06T15:26:01.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:01.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:01.136026+0000) 2024-03-06T15:26:02.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:02.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:02.136255+0000) 2024-03-06T15:26:03.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:03.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:03.136529+0000) 2024-03-06T15:26:04.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:04.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:04.136852+0000) 2024-03-06T15:26:05.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:05.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:05.137132+0000) 2024-03-06T15:26:06.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:06.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:06.137419+0000) 2024-03-06T15:26:07.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:07.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:07.137661+0000) 2024-03-06T15:26:08.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:08.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:08.137912+0000) 2024-03-06T15:26:09.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:09.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:09.138191+0000) 2024-03-06T15:26:10.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:10.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:10.138476+0000) 2024-03-06T15:26:11.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:11.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:11.138756+0000) 2024-03-06T15:26:12.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:12.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:12.139042+0000) 2024-03-06T15:26:13.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:13.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:13.139330+0000) 2024-03-06T15:26:14.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:14.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:14.139643+0000) 2024-03-06T15:26:15.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:15.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:15.139858+0000) 2024-03-06T15:26:16.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:16.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:16.140058+0000) 2024-03-06T15:26:17.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:17.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:17.140300+0000) 2024-03-06T15:26:18.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:18.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:18.140568+0000) 2024-03-06T15:26:19.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:19.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:19.140805+0000) 2024-03-06T15:26:20.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:20.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:20.141023+0000) 2024-03-06T15:26:21.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:21.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:21.141218+0000) 2024-03-06T15:26:22.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:22.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:22.141464+0000) 2024-03-06T15:26:23.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:23.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:23.141688+0000) 2024-03-06T15:26:24.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:24.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:24.141880+0000) 2024-03-06T15:26:25.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:25.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:25.142104+0000) 2024-03-06T15:26:26.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:26.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:26.142329+0000) 2024-03-06T15:26:27.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:27.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:27.142602+0000) 2024-03-06T15:26:28.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:28.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:28.142817+0000) 2024-03-06T15:26:29.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:29.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:29.143058+0000) 2024-03-06T15:26:29.264 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:26:29.272 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:26:30.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:30.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:30.143255+0000) 2024-03-06T15:26:31.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:31.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:31.143497+0000) 2024-03-06T15:26:32.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:32.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:32.143757+0000) 2024-03-06T15:26:33.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:33.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:33.143965+0000) 2024-03-06T15:26:34.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:34.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:34.144216+0000) 2024-03-06T15:26:35.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:35.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:35.144414+0000) 2024-03-06T15:26:36.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:36.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:36.144666+0000) 2024-03-06T15:26:37.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:37.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:37.144906+0000) 2024-03-06T15:26:38.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:38.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:38.145166+0000) 2024-03-06T15:26:39.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:39.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:39.145400+0000) 2024-03-06T15:26:40.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:40.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:40.145657+0000) 2024-03-06T15:26:41.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:41.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:41.145872+0000) 2024-03-06T15:26:42.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:42.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:42.146107+0000) 2024-03-06T15:26:43.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:43.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:43.146326+0000) 2024-03-06T15:26:44.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:44.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:44.146507+0000) 2024-03-06T15:26:45.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:45.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:45.146794+0000) 2024-03-06T15:26:46.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:46.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:46.147027+0000) 2024-03-06T15:26:47.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:47.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:47.147330+0000) 2024-03-06T15:26:48.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:48.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:48.147624+0000) 2024-03-06T15:26:49.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:49.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:49.147850+0000) 2024-03-06T15:26:50.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:50.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:50.148036+0000) 2024-03-06T15:26:51.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:51.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:51.148261+0000) 2024-03-06T15:26:52.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:52.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:52.148493+0000) 2024-03-06T15:26:53.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:53.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:53.148715+0000) 2024-03-06T15:26:54.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:54.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:54.148907+0000) 2024-03-06T15:26:55.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:55.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:55.149133+0000) 2024-03-06T15:26:56.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:56.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:56.149360+0000) 2024-03-06T15:26:57.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:57.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:57.149642+0000) 2024-03-06T15:26:58.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:58.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:58.149877+0000) 2024-03-06T15:26:59.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:26:59.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:26:59.150125+0000) 2024-03-06T15:26:59.459 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:26:59.469 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:27:00.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:00.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:00.150373+0000) 2024-03-06T15:27:01.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:01.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:01.150651+0000) 2024-03-06T15:27:02.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:02.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:02.150834+0000) 2024-03-06T15:27:03.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:03.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:03.151062+0000) 2024-03-06T15:27:04.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:04.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:04.151251+0000) 2024-03-06T15:27:05.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:05.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:05.151461+0000) 2024-03-06T15:27:06.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:06.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:06.151696+0000) 2024-03-06T15:27:07.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:07.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:07.151888+0000) 2024-03-06T15:27:08.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:08.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:08.152132+0000) 2024-03-06T15:27:09.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:09.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:09.152328+0000) 2024-03-06T15:27:10.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:10.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:10.152584+0000) 2024-03-06T15:27:11.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:11.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:11.152771+0000) 2024-03-06T15:27:12.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:12.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:12.152972+0000) 2024-03-06T15:27:13.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:13.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:13.153207+0000) 2024-03-06T15:27:14.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:14.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:14.153432+0000) 2024-03-06T15:27:15.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:15.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:15.153670+0000) 2024-03-06T15:27:16.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:16.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:16.153893+0000) 2024-03-06T15:27:17.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:17.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:17.154116+0000) 2024-03-06T15:27:18.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:18.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:18.154341+0000) 2024-03-06T15:27:19.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:19.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:19.154614+0000) 2024-03-06T15:27:20.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:20.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:20.154808+0000) 2024-03-06T15:27:21.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:21.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:21.155029+0000) 2024-03-06T15:27:22.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:22.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:22.155256+0000) 2024-03-06T15:27:23.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:23.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:23.155485+0000) 2024-03-06T15:27:24.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:24.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:24.155723+0000) 2024-03-06T15:27:25.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:25.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:25.155888+0000) 2024-03-06T15:27:26.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:26.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:26.156083+0000) 2024-03-06T15:27:27.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:27.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:27.156284+0000) 2024-03-06T15:27:28.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:28.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:28.156466+0000) 2024-03-06T15:27:29.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:29.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:29.156711+0000) 2024-03-06T15:27:29.615 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:27:29.623 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:27:30.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:30.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:30.156904+0000) 2024-03-06T15:27:31.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:31.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:31.157131+0000) 2024-03-06T15:27:32.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:32.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:32.157334+0000) 2024-03-06T15:27:33.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:33.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:33.157586+0000) 2024-03-06T15:27:34.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:34.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:34.157798+0000) 2024-03-06T15:27:35.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:35.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:35.158045+0000) 2024-03-06T15:27:36.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:36.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:36.158224+0000) 2024-03-06T15:27:37.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:37.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:37.158449+0000) 2024-03-06T15:27:38.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:38.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:38.158672+0000) 2024-03-06T15:27:39.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:39.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:39.158900+0000) 2024-03-06T15:27:40.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:40.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:40.159087+0000) 2024-03-06T15:27:41.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:41.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:41.159319+0000) 2024-03-06T15:27:42.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:42.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:42.159513+0000) 2024-03-06T15:27:43.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:43.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:43.159787+0000) 2024-03-06T15:27:44.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:44.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:44.159989+0000) 2024-03-06T15:27:45.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:45.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:45.160205+0000) 2024-03-06T15:27:46.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:46.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:46.160391+0000) 2024-03-06T15:27:47.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:47.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:47.160667+0000) 2024-03-06T15:27:48.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:48.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:48.160864+0000) 2024-03-06T15:27:49.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:49.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:49.161117+0000) 2024-03-06T15:27:50.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:50.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:50.161361+0000) 2024-03-06T15:27:51.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:51.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:51.161643+0000) 2024-03-06T15:27:52.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:52.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:52.161889+0000) 2024-03-06T15:27:53.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:53.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:53.162131+0000) 2024-03-06T15:27:54.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:54.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:54.162371+0000) 2024-03-06T15:27:55.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:55.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:55.162672+0000) 2024-03-06T15:27:56.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:56.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:56.162879+0000) 2024-03-06T15:27:57.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:57.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:57.163107+0000) 2024-03-06T15:27:58.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:58.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:58.163331+0000) 2024-03-06T15:27:59.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:27:59.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:27:59.163584+0000) 2024-03-06T15:27:59.835 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:27:59.845 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:28:00.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:00.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:00.163779+0000) 2024-03-06T15:28:01.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:01.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:01.163993+0000) 2024-03-06T15:28:02.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:02.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:02.164220+0000) 2024-03-06T15:28:03.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:03.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:03.164444+0000) 2024-03-06T15:28:04.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:04.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:04.164666+0000) 2024-03-06T15:28:05.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:05.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:05.164892+0000) 2024-03-06T15:28:06.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:06.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:06.165074+0000) 2024-03-06T15:28:07.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:07.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:07.165312+0000) 2024-03-06T15:28:08.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:08.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:08.165565+0000) 2024-03-06T15:28:09.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:09.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:09.165803+0000) 2024-03-06T15:28:10.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:10.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:10.166025+0000) 2024-03-06T15:28:11.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:11.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:11.166248+0000) 2024-03-06T15:28:12.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:12.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:12.166430+0000) 2024-03-06T15:28:13.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:13.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:13.166654+0000) 2024-03-06T15:28:14.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:14.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:14.166882+0000) 2024-03-06T15:28:15.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:15.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:15.167058+0000) 2024-03-06T15:28:16.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:16.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:16.167289+0000) 2024-03-06T15:28:17.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:17.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:17.167520+0000) 2024-03-06T15:28:18.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:18.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:18.167798+0000) 2024-03-06T15:28:19.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:19.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:19.168026+0000) 2024-03-06T15:28:20.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:20.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:20.168222+0000) 2024-03-06T15:28:21.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:21.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:21.168458+0000) 2024-03-06T15:28:22.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:22.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:22.168691+0000) 2024-03-06T15:28:23.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:23.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:23.168942+0000) 2024-03-06T15:28:24.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:24.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:24.169183+0000) 2024-03-06T15:28:25.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:25.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:25.169401+0000) 2024-03-06T15:28:26.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:26.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:26.169664+0000) 2024-03-06T15:28:27.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:27.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:27.169855+0000) 2024-03-06T15:28:28.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:28.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:28.170142+0000) 2024-03-06T15:28:29.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:29.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:29.170426+0000) 2024-03-06T15:28:29.984 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:28:29.995 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:28:30.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:30.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:30.170697+0000) 2024-03-06T15:28:31.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:31.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:31.170977+0000) 2024-03-06T15:28:32.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:32.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:32.171167+0000) 2024-03-06T15:28:33.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:33.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:33.171354+0000) 2024-03-06T15:28:34.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:34.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:34.171639+0000) 2024-03-06T15:28:35.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:35.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:35.171844+0000) 2024-03-06T15:28:36.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:36.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:36.172067+0000) 2024-03-06T15:28:37.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:37.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:37.172289+0000) 2024-03-06T15:28:38.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:38.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:38.172516+0000) 2024-03-06T15:28:39.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:39.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:39.172812+0000) 2024-03-06T15:28:40.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:40.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:40.173038+0000) 2024-03-06T15:28:41.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:41.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:41.173258+0000) 2024-03-06T15:28:42.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:42.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:42.173485+0000) 2024-03-06T15:28:43.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:43.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:43.173708+0000) 2024-03-06T15:28:44.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:44.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:44.173902+0000) 2024-03-06T15:28:45.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:45.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:45.174128+0000) 2024-03-06T15:28:46.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:46.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:46.174309+0000) 2024-03-06T15:28:47.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:47.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:47.174571+0000) 2024-03-06T15:28:48.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:48.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:48.174812+0000) 2024-03-06T15:28:49.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:49.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:49.175050+0000) 2024-03-06T15:28:50.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:50.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:50.175278+0000) 2024-03-06T15:28:51.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:51.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:51.175449+0000) 2024-03-06T15:28:52.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:52.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:52.175671+0000) 2024-03-06T15:28:53.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:53.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:53.175861+0000) 2024-03-06T15:28:54.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:54.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:54.176093+0000) 2024-03-06T15:28:55.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:55.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:55.176281+0000) 2024-03-06T15:28:56.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:56.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:56.176507+0000) 2024-03-06T15:28:57.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:57.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:57.176760+0000) 2024-03-06T15:28:58.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:58.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:58.176958+0000) 2024-03-06T15:28:59.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:28:59.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:28:59.177179+0000) 2024-03-06T15:29:00.120 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:29:00.131 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:29:00.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:00.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:00.177377+0000) 2024-03-06T15:29:01.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:01.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:01.177633+0000) 2024-03-06T15:29:02.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:02.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:02.177859+0000) 2024-03-06T15:29:03.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:03.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:03.178085+0000) 2024-03-06T15:29:04.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:04.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:04.178311+0000) 2024-03-06T15:29:05.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:05.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:05.178592+0000) 2024-03-06T15:29:06.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:06.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:06.178787+0000) 2024-03-06T15:29:07.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:07.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:07.179009+0000) 2024-03-06T15:29:08.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:08.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:08.179236+0000) 2024-03-06T15:29:09.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:09.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:09.179467+0000) 2024-03-06T15:29:10.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:10.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:10.179701+0000) 2024-03-06T15:29:11.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:11.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:11.179896+0000) 2024-03-06T15:29:12.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:12.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:12.180122+0000) 2024-03-06T15:29:13.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:13.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:13.180344+0000) 2024-03-06T15:29:14.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:14.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:14.180527+0000) 2024-03-06T15:29:15.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:15.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:15.180787+0000) 2024-03-06T15:29:16.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:16.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:16.181055+0000) 2024-03-06T15:29:17.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:17.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:17.181279+0000) 2024-03-06T15:29:18.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:18.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:18.181464+0000) 2024-03-06T15:29:19.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:19.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:19.181676+0000) 2024-03-06T15:29:20.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:20.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:20.181910+0000) 2024-03-06T15:29:21.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:21.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:21.182137+0000) 2024-03-06T15:29:22.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:22.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:22.182319+0000) 2024-03-06T15:29:23.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:23.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:23.182573+0000) 2024-03-06T15:29:24.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:24.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:24.182798+0000) 2024-03-06T15:29:25.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:25.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:25.183013+0000) 2024-03-06T15:29:26.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:26.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:26.183220+0000) 2024-03-06T15:29:27.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:27.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:27.183462+0000) 2024-03-06T15:29:28.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:28.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:28.183681+0000) 2024-03-06T15:29:29.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:29.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:29.183879+0000) 2024-03-06T15:29:30.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:30.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:30.184108+0000) 2024-03-06T15:29:30.258 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:29:30.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:29:31.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:31.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:31.184282+0000) 2024-03-06T15:29:32.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:32.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:32.184516+0000) 2024-03-06T15:29:33.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:33.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:33.184744+0000) 2024-03-06T15:29:34.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:34.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:34.184946+0000) 2024-03-06T15:29:35.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:35.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:35.185168+0000) 2024-03-06T15:29:36.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:36.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:36.185389+0000) 2024-03-06T15:29:37.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:37.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:37.185656+0000) 2024-03-06T15:29:38.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:38.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:38.185855+0000) 2024-03-06T15:29:39.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:39.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:39.186057+0000) 2024-03-06T15:29:40.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:40.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:40.186294+0000) 2024-03-06T15:29:41.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:41.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:41.186518+0000) 2024-03-06T15:29:42.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:42.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:42.186768+0000) 2024-03-06T15:29:43.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:43.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:43.186968+0000) 2024-03-06T15:29:44.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:44.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:44.187191+0000) 2024-03-06T15:29:45.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:45.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:45.187416+0000) 2024-03-06T15:29:46.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:46.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:46.187650+0000) 2024-03-06T15:29:47.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:47.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:47.187855+0000) 2024-03-06T15:29:48.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:48.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:48.188080+0000) 2024-03-06T15:29:49.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:49.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:49.188310+0000) 2024-03-06T15:29:50.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:50.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:50.188578+0000) 2024-03-06T15:29:51.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:51.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:51.188775+0000) 2024-03-06T15:29:52.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:52.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:52.188989+0000) 2024-03-06T15:29:53.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:53.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:53.189212+0000) 2024-03-06T15:29:54.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:54.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:54.189441+0000) 2024-03-06T15:29:55.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:55.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:55.189690+0000) 2024-03-06T15:29:56.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:56.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:56.189911+0000) 2024-03-06T15:29:57.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:57.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:57.190099+0000) 2024-03-06T15:29:58.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:58.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:58.190274+0000) 2024-03-06T15:29:59.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:29:59.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:29:59.190506+0000) 2024-03-06T15:30:00.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:00.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:00.190775+0000) 2024-03-06T15:30:00.396 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:30:00.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:30:01.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:01.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:01.190996+0000) 2024-03-06T15:30:02.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:02.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:02.191234+0000) 2024-03-06T15:30:03.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:03.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:03.191458+0000) 2024-03-06T15:30:04.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:04.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:04.191682+0000) 2024-03-06T15:30:05.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:05.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:05.191858+0000) 2024-03-06T15:30:06.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:06.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:06.192088+0000) 2024-03-06T15:30:07.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:07.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:07.192335+0000) 2024-03-06T15:30:08.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:08.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:08.192617+0000) 2024-03-06T15:30:09.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:09.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:09.192924+0000) 2024-03-06T15:30:10.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:10.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:10.193148+0000) 2024-03-06T15:30:11.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:11.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:11.193390+0000) 2024-03-06T15:30:12.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:12.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:12.193667+0000) 2024-03-06T15:30:13.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:13.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:13.193916+0000) 2024-03-06T15:30:14.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:14.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:14.194158+0000) 2024-03-06T15:30:15.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:15.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:15.194398+0000) 2024-03-06T15:30:16.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:16.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:16.194631+0000) 2024-03-06T15:30:17.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:17.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:17.194892+0000) 2024-03-06T15:30:18.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:18.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:18.195142+0000) 2024-03-06T15:30:19.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:19.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:19.195352+0000) 2024-03-06T15:30:20.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:20.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:20.195634+0000) 2024-03-06T15:30:21.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:21.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:21.195874+0000) 2024-03-06T15:30:22.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:22.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:22.196095+0000) 2024-03-06T15:30:23.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:23.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:23.196345+0000) 2024-03-06T15:30:24.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:24.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:24.196623+0000) 2024-03-06T15:30:25.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:25.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:25.196848+0000) 2024-03-06T15:30:26.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:26.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:26.197075+0000) 2024-03-06T15:30:27.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:27.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:27.197338+0000) 2024-03-06T15:30:28.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:28.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:28.197574+0000) 2024-03-06T15:30:29.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:29.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:29.197849+0000) 2024-03-06T15:30:30.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:30.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:30.198096+0000) 2024-03-06T15:30:30.601 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:30:30.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:30:31.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:31.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:31.198343+0000) 2024-03-06T15:30:32.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:32.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:32.198653+0000) 2024-03-06T15:30:33.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:33.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:33.198893+0000) 2024-03-06T15:30:34.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:34.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:34.199120+0000) 2024-03-06T15:30:35.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:35.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:35.199306+0000) 2024-03-06T15:30:36.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:36.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:36.199577+0000) 2024-03-06T15:30:37.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:37.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:37.199815+0000) 2024-03-06T15:30:38.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:38.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:38.200055+0000) 2024-03-06T15:30:39.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:39.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:39.200287+0000) 2024-03-06T15:30:40.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:40.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:40.200520+0000) 2024-03-06T15:30:41.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:41.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:41.200745+0000) 2024-03-06T15:30:42.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:42.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:42.201000+0000) 2024-03-06T15:30:43.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:43.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:43.201205+0000) 2024-03-06T15:30:44.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:44.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:44.201404+0000) 2024-03-06T15:30:45.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:45.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:45.201684+0000) 2024-03-06T15:30:46.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:46.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:46.201881+0000) 2024-03-06T15:30:47.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:47.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:47.202075+0000) 2024-03-06T15:30:48.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:48.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:48.202298+0000) 2024-03-06T15:30:49.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:49.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:49.202566+0000) 2024-03-06T15:30:50.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:50.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:50.202768+0000) 2024-03-06T15:30:51.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:51.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:51.202959+0000) 2024-03-06T15:30:52.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:52.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:52.203274+0000) 2024-03-06T15:30:53.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:53.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:53.203514+0000) 2024-03-06T15:30:54.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:54.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:54.203809+0000) 2024-03-06T15:30:55.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:55.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:55.204037+0000) 2024-03-06T15:30:56.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:56.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:56.204241+0000) 2024-03-06T15:30:57.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:57.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:57.204496+0000) 2024-03-06T15:30:58.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:58.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:58.204723+0000) 2024-03-06T15:30:59.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:30:59.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:30:59.204973+0000) 2024-03-06T15:31:00.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:00.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:00.205169+0000) 2024-03-06T15:31:00.807 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:31:00.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:31:01.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:01.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:01.205392+0000) 2024-03-06T15:31:02.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:02.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:02.205642+0000) 2024-03-06T15:31:03.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:03.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:03.205885+0000) 2024-03-06T15:31:04.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:04.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:04.206123+0000) 2024-03-06T15:31:05.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:05.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:05.206368+0000) 2024-03-06T15:31:06.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:06.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:06.206618+0000) 2024-03-06T15:31:07.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:07.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:07.206826+0000) 2024-03-06T15:31:08.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:08.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:08.207051+0000) 2024-03-06T15:31:09.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:09.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:09.207280+0000) 2024-03-06T15:31:10.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:10.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:10.207509+0000) 2024-03-06T15:31:11.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:11.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:11.207741+0000) 2024-03-06T15:31:12.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:12.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:12.207945+0000) 2024-03-06T15:31:13.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:13.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:13.208175+0000) 2024-03-06T15:31:14.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:14.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:14.208461+0000) 2024-03-06T15:31:15.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:15.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:15.208747+0000) 2024-03-06T15:31:16.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:16.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:16.208981+0000) 2024-03-06T15:31:17.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:17.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:17.209245+0000) 2024-03-06T15:31:18.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:18.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:18.209460+0000) 2024-03-06T15:31:19.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:19.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:19.209676+0000) 2024-03-06T15:31:20.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:20.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:20.209917+0000) 2024-03-06T15:31:21.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:21.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:21.210139+0000) 2024-03-06T15:31:22.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:22.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:22.210319+0000) 2024-03-06T15:31:23.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:23.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:23.210573+0000) 2024-03-06T15:31:24.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:24.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:24.210788+0000) 2024-03-06T15:31:25.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:25.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:25.211016+0000) 2024-03-06T15:31:26.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:26.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:26.211240+0000) 2024-03-06T15:31:27.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:27.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:27.211461+0000) 2024-03-06T15:31:28.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:28.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:28.211682+0000) 2024-03-06T15:31:29.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:29.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:29.211897+0000) 2024-03-06T15:31:30.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:30.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:30.212153+0000) 2024-03-06T15:31:30.944 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:31:30.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:31:31.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:31.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:31.212372+0000) 2024-03-06T15:31:32.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:32.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:32.212644+0000) 2024-03-06T15:31:33.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:33.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:33.212839+0000) 2024-03-06T15:31:34.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:34.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:34.213066+0000) 2024-03-06T15:31:35.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:35.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:35.213294+0000) 2024-03-06T15:31:36.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:36.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:36.213514+0000) 2024-03-06T15:31:37.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:37.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:37.213745+0000) 2024-03-06T15:31:38.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:38.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:38.213943+0000) 2024-03-06T15:31:39.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:39.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:39.214190+0000) 2024-03-06T15:31:40.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:40.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:40.214389+0000) 2024-03-06T15:31:41.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:41.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:41.214664+0000) 2024-03-06T15:31:42.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:42.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:42.214868+0000) 2024-03-06T15:31:43.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:43.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:43.215040+0000) 2024-03-06T15:31:44.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:44.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:44.215217+0000) 2024-03-06T15:31:45.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:45.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:45.215492+0000) 2024-03-06T15:31:46.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:46.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:46.215696+0000) 2024-03-06T15:31:47.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:47.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:47.215908+0000) 2024-03-06T15:31:48.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:48.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:48.216082+0000) 2024-03-06T15:31:49.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:49.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:49.216343+0000) 2024-03-06T15:31:50.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:50.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:50.216581+0000) 2024-03-06T15:31:51.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:51.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:51.216777+0000) 2024-03-06T15:31:52.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:52.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:52.216962+0000) 2024-03-06T15:31:53.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:53.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:53.217141+0000) 2024-03-06T15:31:54.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:54.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:54.217326+0000) 2024-03-06T15:31:55.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:55.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:55.217508+0000) 2024-03-06T15:31:56.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:56.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:56.217754+0000) 2024-03-06T15:31:57.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:57.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:57.217975+0000) 2024-03-06T15:31:58.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:58.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:58.218202+0000) 2024-03-06T15:31:59.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:31:59.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:31:59.218438+0000) 2024-03-06T15:32:00.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:00.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:00.218675+0000) 2024-03-06T15:32:01.152 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:32:01.160 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:32:01.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:01.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:01.218884+0000) 2024-03-06T15:32:02.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:02.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:02.219106+0000) 2024-03-06T15:32:03.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:03.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:03.219331+0000) 2024-03-06T15:32:04.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:04.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:04.219588+0000) 2024-03-06T15:32:05.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:05.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:05.219789+0000) 2024-03-06T15:32:06.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:06.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:06.220022+0000) 2024-03-06T15:32:07.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:07.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:07.220246+0000) 2024-03-06T15:32:08.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:08.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:08.220476+0000) 2024-03-06T15:32:09.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:09.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:09.220680+0000) 2024-03-06T15:32:10.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:10.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:10.220888+0000) 2024-03-06T15:32:11.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:11.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:11.221108+0000) 2024-03-06T15:32:12.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:12.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:12.221293+0000) 2024-03-06T15:32:13.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:13.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:13.221513+0000) 2024-03-06T15:32:14.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:14.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:14.221762+0000) 2024-03-06T15:32:15.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:15.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:15.221991+0000) 2024-03-06T15:32:16.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:16.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:16.222204+0000) 2024-03-06T15:32:17.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:17.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:17.222437+0000) 2024-03-06T15:32:18.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:18.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:18.222662+0000) 2024-03-06T15:32:19.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:19.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:19.222865+0000) 2024-03-06T15:32:20.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:20.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:20.223091+0000) 2024-03-06T15:32:21.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:21.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:21.223313+0000) 2024-03-06T15:32:22.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:22.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:22.223584+0000) 2024-03-06T15:32:23.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:23.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:23.223786+0000) 2024-03-06T15:32:24.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:24.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:24.224009+0000) 2024-03-06T15:32:25.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:25.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:25.224213+0000) 2024-03-06T15:32:26.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:26.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:26.224437+0000) 2024-03-06T15:32:27.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:27.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:27.224661+0000) 2024-03-06T15:32:28.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:28.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:28.224895+0000) 2024-03-06T15:32:29.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:29.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:29.225128+0000) 2024-03-06T15:32:30.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:30.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:30.225356+0000) 2024-03-06T15:32:31.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:31.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:31.225622+0000) 2024-03-06T15:32:31.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:32:31.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:32:32.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:32.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:32.225851+0000) 2024-03-06T15:32:33.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:33.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:33.226077+0000) 2024-03-06T15:32:34.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:34.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:34.226262+0000) 2024-03-06T15:32:35.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:35.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:35.226490+0000) 2024-03-06T15:32:36.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:36.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:36.226715+0000) 2024-03-06T15:32:37.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:37.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:37.226946+0000) 2024-03-06T15:32:38.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:38.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:38.227169+0000) 2024-03-06T15:32:39.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:39.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:39.227447+0000) 2024-03-06T15:32:40.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:40.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:40.227650+0000) 2024-03-06T15:32:41.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:41.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:41.227876+0000) 2024-03-06T15:32:42.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:42.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:42.228103+0000) 2024-03-06T15:32:43.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:43.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:43.228320+0000) 2024-03-06T15:32:44.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:44.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:44.228574+0000) 2024-03-06T15:32:45.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:45.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:45.228731+0000) 2024-03-06T15:32:46.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:46.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:46.228920+0000) 2024-03-06T15:32:47.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:47.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:47.229100+0000) 2024-03-06T15:32:48.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:48.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:48.229332+0000) 2024-03-06T15:32:49.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:49.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:49.229514+0000) 2024-03-06T15:32:50.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:50.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:50.229740+0000) 2024-03-06T15:32:51.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:51.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:51.229960+0000) 2024-03-06T15:32:52.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:52.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:52.230143+0000) 2024-03-06T15:32:53.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:53.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:53.230358+0000) 2024-03-06T15:32:54.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:54.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:54.230661+0000) 2024-03-06T15:32:55.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:55.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:55.230886+0000) 2024-03-06T15:32:56.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:56.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:56.231108+0000) 2024-03-06T15:32:57.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:57.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:57.231341+0000) 2024-03-06T15:32:58.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:58.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:58.231623+0000) 2024-03-06T15:32:59.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:32:59.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:32:59.231830+0000) 2024-03-06T15:33:00.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:00.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:00.232056+0000) 2024-03-06T15:33:01.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:01.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:01.232243+0000) 2024-03-06T15:33:01.556 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:33:01.566 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:33:02.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:02.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:02.232491+0000) 2024-03-06T15:33:03.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:03.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:03.232740+0000) 2024-03-06T15:33:04.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:04.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:04.232992+0000) 2024-03-06T15:33:05.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:05.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:05.233233+0000) 2024-03-06T15:33:06.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:06.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:06.233465+0000) 2024-03-06T15:33:07.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:07.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:07.233673+0000) 2024-03-06T15:33:08.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:08.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:08.233917+0000) 2024-03-06T15:33:09.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:09.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:09.234171+0000) 2024-03-06T15:33:10.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:10.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:10.234403+0000) 2024-03-06T15:33:11.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:11.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:11.234649+0000) 2024-03-06T15:33:12.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:12.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:12.234890+0000) 2024-03-06T15:33:13.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:13.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:13.235109+0000) 2024-03-06T15:33:14.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:14.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:14.235334+0000) 2024-03-06T15:33:15.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:15.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:15.235603+0000) 2024-03-06T15:33:16.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:16.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:16.235832+0000) 2024-03-06T15:33:17.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:17.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:17.236065+0000) 2024-03-06T15:33:18.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:18.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:18.236304+0000) 2024-03-06T15:33:19.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:19.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:19.236507+0000) 2024-03-06T15:33:20.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:20.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:20.236740+0000) 2024-03-06T15:33:21.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:21.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:21.236962+0000) 2024-03-06T15:33:22.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:22.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:22.237188+0000) 2024-03-06T15:33:23.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:23.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:23.237370+0000) 2024-03-06T15:33:24.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:24.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:24.237592+0000) 2024-03-06T15:33:25.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:25.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:25.237821+0000) 2024-03-06T15:33:26.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:26.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:26.238036+0000) 2024-03-06T15:33:27.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:27.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:27.238306+0000) 2024-03-06T15:33:28.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:28.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:28.238524+0000) 2024-03-06T15:33:29.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:29.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:29.238803+0000) 2024-03-06T15:33:30.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:30.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:30.238996+0000) 2024-03-06T15:33:31.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:31.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:31.239167+0000) 2024-03-06T15:33:31.730 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:33:31.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:33:32.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:32.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:32.239390+0000) 2024-03-06T15:33:33.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:33.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:33.239604+0000) 2024-03-06T15:33:34.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:34.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:34.239831+0000) 2024-03-06T15:33:35.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:35.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:35.240012+0000) 2024-03-06T15:33:36.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:36.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:36.240194+0000) 2024-03-06T15:33:37.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:37.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:37.240424+0000) 2024-03-06T15:33:38.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:38.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:38.240651+0000) 2024-03-06T15:33:39.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:39.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:39.240855+0000) 2024-03-06T15:33:40.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:40.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:40.241048+0000) 2024-03-06T15:33:41.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:41.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:41.241281+0000) 2024-03-06T15:33:42.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:42.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:42.241521+0000) 2024-03-06T15:33:43.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:43.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:43.241764+0000) 2024-03-06T15:33:44.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:44.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:44.241953+0000) 2024-03-06T15:33:45.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:45.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:45.242205+0000) 2024-03-06T15:33:46.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:46.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:46.242442+0000) 2024-03-06T15:33:47.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:47.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:47.242656+0000) 2024-03-06T15:33:48.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:48.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:48.242905+0000) 2024-03-06T15:33:49.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:49.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:49.243158+0000) 2024-03-06T15:33:50.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:50.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:50.243402+0000) 2024-03-06T15:33:51.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:51.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:51.243641+0000) 2024-03-06T15:33:52.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:52.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:52.243878+0000) 2024-03-06T15:33:53.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:53.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:53.244124+0000) 2024-03-06T15:33:54.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:54.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:54.244364+0000) 2024-03-06T15:33:55.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:55.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:55.244638+0000) 2024-03-06T15:33:56.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:56.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:56.244815+0000) 2024-03-06T15:33:57.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:57.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:57.245041+0000) 2024-03-06T15:33:58.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:58.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:58.245268+0000) 2024-03-06T15:33:59.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:33:59.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:33:59.245505+0000) 2024-03-06T15:34:00.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:00.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:00.245710+0000) 2024-03-06T15:34:01.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:01.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:01.245927+0000) 2024-03-06T15:34:01.943 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:34:01.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:34:02.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:02.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:02.246156+0000) 2024-03-06T15:34:03.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:03.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:03.246368+0000) 2024-03-06T15:34:04.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:04.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:04.246638+0000) 2024-03-06T15:34:05.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:05.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:05.246858+0000) 2024-03-06T15:34:06.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:06.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:06.247077+0000) 2024-03-06T15:34:07.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:07.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:07.247304+0000) 2024-03-06T15:34:08.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:08.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:08.247557+0000) 2024-03-06T15:34:09.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:09.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:09.247769+0000) 2024-03-06T15:34:10.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:10.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:10.247957+0000) 2024-03-06T15:34:11.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:11.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:11.248171+0000) 2024-03-06T15:34:12.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:12.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:12.248366+0000) 2024-03-06T15:34:13.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:13.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:13.248647+0000) 2024-03-06T15:34:14.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:14.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:14.248815+0000) 2024-03-06T15:34:15.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:15.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:15.249043+0000) 2024-03-06T15:34:16.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:16.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:16.249233+0000) 2024-03-06T15:34:17.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:17.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:17.249435+0000) 2024-03-06T15:34:18.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:18.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:18.249672+0000) 2024-03-06T15:34:19.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:19.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:19.249869+0000) 2024-03-06T15:34:20.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:20.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:20.250119+0000) 2024-03-06T15:34:21.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:21.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:21.250334+0000) 2024-03-06T15:34:22.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:22.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:22.250523+0000) 2024-03-06T15:34:23.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:23.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:23.250791+0000) 2024-03-06T15:34:24.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:24.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:24.251012+0000) 2024-03-06T15:34:25.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:25.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:25.251229+0000) 2024-03-06T15:34:26.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:26.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:26.251451+0000) 2024-03-06T15:34:27.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:27.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:27.251683+0000) 2024-03-06T15:34:28.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:28.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:28.251888+0000) 2024-03-06T15:34:29.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:29.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:29.252134+0000) 2024-03-06T15:34:30.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:30.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:30.252361+0000) 2024-03-06T15:34:31.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:31.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:31.252635+0000) 2024-03-06T15:34:32.199 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:34:32.210 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:34:32.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:32.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:32.252792+0000) 2024-03-06T15:34:33.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:33.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:33.253002+0000) 2024-03-06T15:34:34.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:34.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:34.253228+0000) 2024-03-06T15:34:35.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:35.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:35.253452+0000) 2024-03-06T15:34:36.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:36.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:36.253670+0000) 2024-03-06T15:34:37.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:37.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:37.253861+0000) 2024-03-06T15:34:38.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:38.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:38.254079+0000) 2024-03-06T15:34:39.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:39.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:39.254313+0000) 2024-03-06T15:34:40.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:40.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:40.254584+0000) 2024-03-06T15:34:41.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:41.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:41.254795+0000) 2024-03-06T15:34:42.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:42.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:42.255023+0000) 2024-03-06T15:34:43.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:43.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:43.255247+0000) 2024-03-06T15:34:44.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:44.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:44.255476+0000) 2024-03-06T15:34:45.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:45.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:45.255691+0000) 2024-03-06T15:34:46.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:46.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:46.255909+0000) 2024-03-06T15:34:47.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:47.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:47.256134+0000) 2024-03-06T15:34:48.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:48.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:48.256368+0000) 2024-03-06T15:34:49.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:49.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:49.256647+0000) 2024-03-06T15:34:50.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:50.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:50.256848+0000) 2024-03-06T15:34:51.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:51.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:51.257066+0000) 2024-03-06T15:34:52.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:52.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:52.257288+0000) 2024-03-06T15:34:53.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:53.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:53.257518+0000) 2024-03-06T15:34:54.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:54.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:54.257814+0000) 2024-03-06T15:34:55.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:55.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:55.258047+0000) 2024-03-06T15:34:56.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:56.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:56.258271+0000) 2024-03-06T15:34:57.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:57.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:57.258496+0000) 2024-03-06T15:34:58.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:58.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:58.258718+0000) 2024-03-06T15:34:59.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:34:59.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:34:59.259000+0000) 2024-03-06T15:35:00.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:00.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:00.259282+0000) 2024-03-06T15:35:01.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:01.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:01.259495+0000) 2024-03-06T15:35:02.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:02.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:02.259813+0000) 2024-03-06T15:35:02.427 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:35:02.435 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:35:03.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:03.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:03.260095+0000) 2024-03-06T15:35:04.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:04.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:04.260312+0000) 2024-03-06T15:35:05.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:05.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:05.260569+0000) 2024-03-06T15:35:06.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:06.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:06.260777+0000) 2024-03-06T15:35:07.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:07.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:07.261007+0000) 2024-03-06T15:35:08.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:08.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:08.261204+0000) 2024-03-06T15:35:09.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:09.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:09.261473+0000) 2024-03-06T15:35:10.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:10.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:10.261725+0000) 2024-03-06T15:35:11.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:11.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:11.261928+0000) 2024-03-06T15:35:12.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:12.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:12.262159+0000) 2024-03-06T15:35:13.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:13.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:13.262423+0000) 2024-03-06T15:35:14.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:14.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:14.262701+0000) 2024-03-06T15:35:15.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:15.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:15.262972+0000) 2024-03-06T15:35:16.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:16.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:16.263172+0000) 2024-03-06T15:35:17.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:17.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:17.263443+0000) 2024-03-06T15:35:18.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:18.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:18.263677+0000) 2024-03-06T15:35:19.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:19.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:19.263895+0000) 2024-03-06T15:35:20.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:20.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:20.264094+0000) 2024-03-06T15:35:21.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:21.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:21.264327+0000) 2024-03-06T15:35:22.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:22.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:22.264638+0000) 2024-03-06T15:35:23.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:23.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:23.264924+0000) 2024-03-06T15:35:24.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:24.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:24.265188+0000) 2024-03-06T15:35:25.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:25.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:25.265423+0000) 2024-03-06T15:35:26.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:26.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:26.265705+0000) 2024-03-06T15:35:27.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:27.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:27.265988+0000) 2024-03-06T15:35:28.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:28.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:28.266270+0000) 2024-03-06T15:35:29.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:29.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:29.266520+0000) 2024-03-06T15:35:30.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:30.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:30.266782+0000) 2024-03-06T15:35:31.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:31.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:31.267043+0000) 2024-03-06T15:35:32.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:32.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:32.267305+0000) 2024-03-06T15:35:32.639 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:35:32.647 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:35:33.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:33.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:33.267565+0000) 2024-03-06T15:35:34.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:34.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:34.267818+0000) 2024-03-06T15:35:35.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:35.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:35.268049+0000) 2024-03-06T15:35:36.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:36.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:36.268311+0000) 2024-03-06T15:35:37.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:37.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:37.268632+0000) 2024-03-06T15:35:38.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:38.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:38.268883+0000) 2024-03-06T15:35:39.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:39.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:39.269080+0000) 2024-03-06T15:35:40.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:40.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:40.269305+0000) 2024-03-06T15:35:41.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:41.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:41.269570+0000) 2024-03-06T15:35:42.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:42.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:42.269798+0000) 2024-03-06T15:35:43.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:43.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:43.270097+0000) 2024-03-06T15:35:44.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:44.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:44.270380+0000) 2024-03-06T15:35:45.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:45.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:45.270640+0000) 2024-03-06T15:35:46.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:46.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:46.270846+0000) 2024-03-06T15:35:47.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:47.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:47.271088+0000) 2024-03-06T15:35:48.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:48.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:48.271366+0000) 2024-03-06T15:35:49.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:49.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:49.271699+0000) 2024-03-06T15:35:50.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:50.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:50.271975+0000) 2024-03-06T15:35:51.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:51.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:51.272218+0000) 2024-03-06T15:35:52.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:52.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:52.272483+0000) 2024-03-06T15:35:53.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:53.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:53.272783+0000) 2024-03-06T15:35:54.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:54.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:54.273015+0000) 2024-03-06T15:35:55.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:55.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:55.273239+0000) 2024-03-06T15:35:56.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:56.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:56.273620+0000) 2024-03-06T15:35:57.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:57.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:57.273858+0000) 2024-03-06T15:35:58.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:58.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:58.274138+0000) 2024-03-06T15:35:59.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:35:59.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:35:59.274428+0000) 2024-03-06T15:36:00.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:00.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:00.274710+0000) 2024-03-06T15:36:01.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:01.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:01.274922+0000) 2024-03-06T15:36:02.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:02.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:02.275100+0000) 2024-03-06T15:36:02.852 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:36:02.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:36:03.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:03.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:03.275370+0000) 2024-03-06T15:36:04.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:04.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:04.275685+0000) 2024-03-06T15:36:05.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:05.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:05.275927+0000) 2024-03-06T15:36:06.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:06.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:06.276144+0000) 2024-03-06T15:36:07.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:07.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:07.276340+0000) 2024-03-06T15:36:08.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:08.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:08.276569+0000) 2024-03-06T15:36:09.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:09.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:09.276818+0000) 2024-03-06T15:36:10.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:10.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:10.277051+0000) 2024-03-06T15:36:11.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:11.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:11.277250+0000) 2024-03-06T15:36:12.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:12.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:12.277472+0000) 2024-03-06T15:36:13.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:13.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:13.277725+0000) 2024-03-06T15:36:14.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:14.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:14.277943+0000) 2024-03-06T15:36:15.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:15.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:15.278133+0000) 2024-03-06T15:36:16.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:16.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:16.278362+0000) 2024-03-06T15:36:17.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:17.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:17.278594+0000) 2024-03-06T15:36:18.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:18.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:18.278818+0000) 2024-03-06T15:36:19.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:19.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:19.279017+0000) 2024-03-06T15:36:20.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:20.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:20.279242+0000) 2024-03-06T15:36:21.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:21.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:21.279458+0000) 2024-03-06T15:36:22.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:22.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:22.279685+0000) 2024-03-06T15:36:23.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:23.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:23.279901+0000) 2024-03-06T15:36:24.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:24.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:24.280115+0000) 2024-03-06T15:36:25.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:25.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:25.280355+0000) 2024-03-06T15:36:26.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:26.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:26.280580+0000) 2024-03-06T15:36:27.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:27.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:27.280796+0000) 2024-03-06T15:36:28.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:28.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:28.281077+0000) 2024-03-06T15:36:29.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:29.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:29.281368+0000) 2024-03-06T15:36:30.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:30.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:30.281609+0000) 2024-03-06T15:36:31.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:31.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:31.281833+0000) 2024-03-06T15:36:32.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:32.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:32.282056+0000) 2024-03-06T15:36:32.983 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:36:32.991 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:36:33.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:33.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:33.282305+0000) 2024-03-06T15:36:34.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:34.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:34.282495+0000) 2024-03-06T15:36:35.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:35.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:35.282694+0000) 2024-03-06T15:36:36.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:36.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:36.282927+0000) 2024-03-06T15:36:37.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:37.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:37.283141+0000) 2024-03-06T15:36:38.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:38.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:38.283374+0000) 2024-03-06T15:36:39.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:39.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:39.283606+0000) 2024-03-06T15:36:40.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:40.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:40.283840+0000) 2024-03-06T15:36:41.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:41.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:41.284058+0000) 2024-03-06T15:36:42.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:42.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:42.284254+0000) 2024-03-06T15:36:43.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:43.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:43.284505+0000) 2024-03-06T15:36:44.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:44.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:44.284837+0000) 2024-03-06T15:36:45.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:45.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:45.285078+0000) 2024-03-06T15:36:46.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:46.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:46.285301+0000) 2024-03-06T15:36:47.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:47.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:47.285509+0000) 2024-03-06T15:36:48.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:48.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:48.285761+0000) 2024-03-06T15:36:49.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:49.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:49.286007+0000) 2024-03-06T15:36:50.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:50.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:50.286233+0000) 2024-03-06T15:36:51.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:51.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:51.286474+0000) 2024-03-06T15:36:52.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:52.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:52.286704+0000) 2024-03-06T15:36:53.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:53.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:53.286912+0000) 2024-03-06T15:36:54.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:54.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:54.287136+0000) 2024-03-06T15:36:55.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:55.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:55.287400+0000) 2024-03-06T15:36:56.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:56.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:56.287668+0000) 2024-03-06T15:36:57.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:57.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:57.287861+0000) 2024-03-06T15:36:58.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:58.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:58.288086+0000) 2024-03-06T15:36:59.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:36:59.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:36:59.288372+0000) 2024-03-06T15:37:00.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:00.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:00.288689+0000) 2024-03-06T15:37:01.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:01.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:01.288967+0000) 2024-03-06T15:37:02.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:02.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:02.289249+0000) 2024-03-06T15:37:03.191 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:37:03.199 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:37:03.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:03.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:03.289469+0000) 2024-03-06T15:37:04.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:04.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:04.289689+0000) 2024-03-06T15:37:05.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:05.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:05.289916+0000) 2024-03-06T15:37:06.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:06.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:06.290173+0000) 2024-03-06T15:37:07.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:07.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:07.290442+0000) 2024-03-06T15:37:08.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:08.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:08.290722+0000) 2024-03-06T15:37:09.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:09.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:09.290959+0000) 2024-03-06T15:37:10.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:10.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:10.291189+0000) 2024-03-06T15:37:11.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:11.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:11.291413+0000) 2024-03-06T15:37:12.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:12.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:12.291647+0000) 2024-03-06T15:37:13.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:13.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:13.291888+0000) 2024-03-06T15:37:14.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:14.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:14.292111+0000) 2024-03-06T15:37:15.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:15.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:15.292331+0000) 2024-03-06T15:37:16.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:16.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:16.292592+0000) 2024-03-06T15:37:17.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:17.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:17.292818+0000) 2024-03-06T15:37:18.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:18.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:18.293045+0000) 2024-03-06T15:37:19.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:19.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:19.293277+0000) 2024-03-06T15:37:20.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:20.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:20.293511+0000) 2024-03-06T15:37:21.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:21.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:21.293764+0000) 2024-03-06T15:37:22.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:22.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:22.293982+0000) 2024-03-06T15:37:23.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:23.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:23.294173+0000) 2024-03-06T15:37:24.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:24.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:24.294419+0000) 2024-03-06T15:37:25.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:25.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:25.294659+0000) 2024-03-06T15:37:26.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:26.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:26.294858+0000) 2024-03-06T15:37:27.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:27.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:27.295094+0000) 2024-03-06T15:37:28.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:28.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:28.295326+0000) 2024-03-06T15:37:29.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:29.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:29.295493+0000) 2024-03-06T15:37:30.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:30.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:30.295695+0000) 2024-03-06T15:37:31.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:31.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:31.295881+0000) 2024-03-06T15:37:32.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:32.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:32.296066+0000) 2024-03-06T15:37:33.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:33.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:33.296274+0000) 2024-03-06T15:37:33.379 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:37:33.386 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:37:34.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:34.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:34.296475+0000) 2024-03-06T15:37:35.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:35.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:35.296690+0000) 2024-03-06T15:37:36.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:36.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:36.296874+0000) 2024-03-06T15:37:37.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:37.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:37.297158+0000) 2024-03-06T15:37:38.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:38.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:38.297379+0000) 2024-03-06T15:37:39.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:39.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:39.297594+0000) 2024-03-06T15:37:40.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:40.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:40.297787+0000) 2024-03-06T15:37:41.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:41.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:41.298024+0000) 2024-03-06T15:37:42.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:42.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:42.298264+0000) 2024-03-06T15:37:43.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:43.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:43.298457+0000) 2024-03-06T15:37:44.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:44.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:44.298699+0000) 2024-03-06T15:37:45.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:45.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:45.298924+0000) 2024-03-06T15:37:46.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:46.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:46.299112+0000) 2024-03-06T15:37:47.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:47.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:47.299327+0000) 2024-03-06T15:37:48.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:48.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:48.299524+0000) 2024-03-06T15:37:49.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:49.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:49.299807+0000) 2024-03-06T15:37:50.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:50.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:50.300048+0000) 2024-03-06T15:37:51.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:51.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:51.300278+0000) 2024-03-06T15:37:52.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:52.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:52.300504+0000) 2024-03-06T15:37:53.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:53.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:53.300788+0000) 2024-03-06T15:37:54.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:54.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:54.301014+0000) 2024-03-06T15:37:55.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:55.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:55.301239+0000) 2024-03-06T15:37:56.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:56.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:56.301414+0000) 2024-03-06T15:37:57.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:57.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:57.301644+0000) 2024-03-06T15:37:58.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:58.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:58.301832+0000) 2024-03-06T15:37:59.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:37:59.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:37:59.302068+0000) 2024-03-06T15:38:00.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:00.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:00.302294+0000) 2024-03-06T15:38:01.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:01.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:01.302519+0000) 2024-03-06T15:38:02.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:02.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:02.302819+0000) 2024-03-06T15:38:03.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:03.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:03.303052+0000) 2024-03-06T15:38:03.520 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:38:03.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:38:04.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:04.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:04.303268+0000) 2024-03-06T15:38:05.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:05.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:05.303514+0000) 2024-03-06T15:38:06.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:06.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:06.303735+0000) 2024-03-06T15:38:07.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:07.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:07.303927+0000) 2024-03-06T15:38:08.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:08.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:08.304158+0000) 2024-03-06T15:38:09.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:09.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:09.304392+0000) 2024-03-06T15:38:10.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:10.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:10.304623+0000) 2024-03-06T15:38:11.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:11.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:11.304863+0000) 2024-03-06T15:38:12.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:12.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:12.305076+0000) 2024-03-06T15:38:13.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:13.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:13.305316+0000) 2024-03-06T15:38:14.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:14.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:14.305574+0000) 2024-03-06T15:38:15.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:15.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:15.305797+0000) 2024-03-06T15:38:16.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:16.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:16.306029+0000) 2024-03-06T15:38:17.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:17.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:17.306227+0000) 2024-03-06T15:38:18.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:18.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:18.306428+0000) 2024-03-06T15:38:19.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:19.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:19.306695+0000) 2024-03-06T15:38:20.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:20.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:20.306917+0000) 2024-03-06T15:38:21.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:21.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:21.307164+0000) 2024-03-06T15:38:22.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:22.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:22.307366+0000) 2024-03-06T15:38:23.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:23.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:23.307649+0000) 2024-03-06T15:38:24.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:24.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:24.307877+0000) 2024-03-06T15:38:25.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:25.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:25.308099+0000) 2024-03-06T15:38:26.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:26.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:26.308324+0000) 2024-03-06T15:38:27.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:27.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:27.308497+0000) 2024-03-06T15:38:28.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:28.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:28.308779+0000) 2024-03-06T15:38:29.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:29.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:29.309025+0000) 2024-03-06T15:38:30.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:30.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:30.309236+0000) 2024-03-06T15:38:31.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:31.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:31.309520+0000) 2024-03-06T15:38:32.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:32.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:32.309748+0000) 2024-03-06T15:38:33.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:33.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:33.309983+0000) 2024-03-06T15:38:33.739 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:38:33.747 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:38:34.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:34.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:34.310168+0000) 2024-03-06T15:38:35.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:35.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:35.310409+0000) 2024-03-06T15:38:36.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:36.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:36.310662+0000) 2024-03-06T15:38:37.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:37.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:37.310905+0000) 2024-03-06T15:38:38.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:38.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:38.311117+0000) 2024-03-06T15:38:39.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:39.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:39.311364+0000) 2024-03-06T15:38:40.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:40.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:40.311653+0000) 2024-03-06T15:38:41.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:41.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:41.311866+0000) 2024-03-06T15:38:42.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:42.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:42.312081+0000) 2024-03-06T15:38:43.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:43.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:43.312296+0000) 2024-03-06T15:38:44.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:44.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:44.312497+0000) 2024-03-06T15:38:45.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:45.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:45.312774+0000) 2024-03-06T15:38:46.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:46.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:46.312957+0000) 2024-03-06T15:38:47.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:47.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:47.313183+0000) 2024-03-06T15:38:48.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:48.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:48.313398+0000) 2024-03-06T15:38:49.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:49.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:49.313639+0000) 2024-03-06T15:38:50.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:50.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:50.313879+0000) 2024-03-06T15:38:51.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:51.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:51.314064+0000) 2024-03-06T15:38:52.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:52.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:52.314250+0000) 2024-03-06T15:38:53.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:53.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:53.314425+0000) 2024-03-06T15:38:54.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:54.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:54.314660+0000) 2024-03-06T15:38:55.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:55.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:55.314860+0000) 2024-03-06T15:38:56.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:56.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:56.315052+0000) 2024-03-06T15:38:57.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:57.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:57.315257+0000) 2024-03-06T15:38:58.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:58.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:58.315495+0000) 2024-03-06T15:38:59.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:38:59.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:38:59.315778+0000) 2024-03-06T15:39:00.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:00.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:00.316018+0000) 2024-03-06T15:39:01.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:01.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:01.316260+0000) 2024-03-06T15:39:02.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:02.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:02.316460+0000) 2024-03-06T15:39:03.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:03.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:03.316651+0000) 2024-03-06T15:39:03.973 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:39:03.981 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:39:04.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:04.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:04.316863+0000) 2024-03-06T15:39:05.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:05.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:05.317087+0000) 2024-03-06T15:39:06.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:06.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:06.317317+0000) 2024-03-06T15:39:07.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:07.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:07.317599+0000) 2024-03-06T15:39:08.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:08.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:08.317836+0000) 2024-03-06T15:39:09.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:09.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:09.318043+0000) 2024-03-06T15:39:10.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:10.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:10.318233+0000) 2024-03-06T15:39:11.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:11.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:11.318432+0000) 2024-03-06T15:39:12.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:12.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:12.318672+0000) 2024-03-06T15:39:13.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:13.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:13.318908+0000) 2024-03-06T15:39:14.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:14.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:14.319136+0000) 2024-03-06T15:39:15.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:15.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:15.319365+0000) 2024-03-06T15:39:16.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:16.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:16.319612+0000) 2024-03-06T15:39:17.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:17.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:17.319820+0000) 2024-03-06T15:39:18.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:18.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:18.320086+0000) 2024-03-06T15:39:19.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:19.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:19.320342+0000) 2024-03-06T15:39:20.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:20.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:20.320612+0000) 2024-03-06T15:39:21.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:21.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:21.320834+0000) 2024-03-06T15:39:22.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:22.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:22.321028+0000) 2024-03-06T15:39:23.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:23.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:23.321258+0000) 2024-03-06T15:39:24.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:24.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:24.321485+0000) 2024-03-06T15:39:25.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:25.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:25.321684+0000) 2024-03-06T15:39:26.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:26.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:26.321913+0000) 2024-03-06T15:39:27.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:27.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:27.322163+0000) 2024-03-06T15:39:28.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:28.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:28.322397+0000) 2024-03-06T15:39:29.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:29.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:29.322670+0000) 2024-03-06T15:39:30.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:30.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:30.322873+0000) 2024-03-06T15:39:31.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:31.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:31.323096+0000) 2024-03-06T15:39:32.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:32.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:32.323325+0000) 2024-03-06T15:39:33.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:33.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:33.323572+0000) 2024-03-06T15:39:34.102 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:39:34.112 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:39:34.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:34.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:34.323754+0000) 2024-03-06T15:39:35.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:35.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:35.323990+0000) 2024-03-06T15:39:36.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:36.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:36.324211+0000) 2024-03-06T15:39:37.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:37.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:37.324430+0000) 2024-03-06T15:39:38.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:38.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:38.324638+0000) 2024-03-06T15:39:39.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:39.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:39.324873+0000) 2024-03-06T15:39:40.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:40.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:40.325061+0000) 2024-03-06T15:39:41.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:41.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:41.325239+0000) 2024-03-06T15:39:42.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:42.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:42.325461+0000) 2024-03-06T15:39:43.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:43.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:43.325683+0000) 2024-03-06T15:39:44.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:44.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:44.325910+0000) 2024-03-06T15:39:45.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:45.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:45.326131+0000) 2024-03-06T15:39:46.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:46.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:46.326352+0000) 2024-03-06T15:39:47.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:47.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:47.326611+0000) 2024-03-06T15:39:48.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:48.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:48.326839+0000) 2024-03-06T15:39:49.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:49.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:49.327034+0000) 2024-03-06T15:39:50.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:50.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:50.327241+0000) 2024-03-06T15:39:51.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:51.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:51.327471+0000) 2024-03-06T15:39:52.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:52.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:52.327700+0000) 2024-03-06T15:39:53.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:53.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:53.327926+0000) 2024-03-06T15:39:54.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:54.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:54.328124+0000) 2024-03-06T15:39:55.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:55.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:55.328365+0000) 2024-03-06T15:39:56.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:56.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:56.328592+0000) 2024-03-06T15:39:57.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:57.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:57.328826+0000) 2024-03-06T15:39:58.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:58.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:58.329017+0000) 2024-03-06T15:39:59.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:39:59.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:39:59.329269+0000) 2024-03-06T15:40:00.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:00.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:00.329512+0000) 2024-03-06T15:40:01.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:01.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:01.329723+0000) 2024-03-06T15:40:02.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:02.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:02.329949+0000) 2024-03-06T15:40:03.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:03.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:03.330169+0000) 2024-03-06T15:40:04.314 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:40:04.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:40:04.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:04.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:04.330355+0000) 2024-03-06T15:40:05.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:05.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:05.330558+0000) 2024-03-06T15:40:06.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:06.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:06.330805+0000) 2024-03-06T15:40:07.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:07.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:07.331075+0000) 2024-03-06T15:40:08.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:08.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:08.331305+0000) 2024-03-06T15:40:09.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:09.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:09.331521+0000) 2024-03-06T15:40:10.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:10.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:10.331815+0000) 2024-03-06T15:40:11.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:11.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:11.331993+0000) 2024-03-06T15:40:12.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:12.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:12.332171+0000) 2024-03-06T15:40:13.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:13.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:13.332391+0000) 2024-03-06T15:40:14.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:14.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:14.332654+0000) 2024-03-06T15:40:15.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:15.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:15.332840+0000) 2024-03-06T15:40:16.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:16.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:16.333055+0000) 2024-03-06T15:40:17.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:17.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:17.333280+0000) 2024-03-06T15:40:18.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:18.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:18.333505+0000) 2024-03-06T15:40:19.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:19.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:19.333776+0000) 2024-03-06T15:40:20.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:20.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:20.334024+0000) 2024-03-06T15:40:21.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:21.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:21.334280+0000) 2024-03-06T15:40:22.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:22.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:22.334519+0000) 2024-03-06T15:40:23.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:23.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:23.334819+0000) 2024-03-06T15:40:24.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:24.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:24.335065+0000) 2024-03-06T15:40:25.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:25.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:25.335339+0000) 2024-03-06T15:40:26.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:26.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:26.335527+0000) 2024-03-06T15:40:27.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:27.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:27.335797+0000) 2024-03-06T15:40:28.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:28.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:28.336017+0000) 2024-03-06T15:40:29.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:29.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:29.336252+0000) 2024-03-06T15:40:30.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:30.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:30.336483+0000) 2024-03-06T15:40:31.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:31.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:31.336703+0000) 2024-03-06T15:40:32.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:32.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:32.336928+0000) 2024-03-06T15:40:33.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:33.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:33.337149+0000) 2024-03-06T15:40:34.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:34.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:34.337353+0000) 2024-03-06T15:40:34.520 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:40:34.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:40:35.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:35.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:35.337620+0000) 2024-03-06T15:40:36.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:36.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:36.337849+0000) 2024-03-06T15:40:37.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:37.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:37.338071+0000) 2024-03-06T15:40:38.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:38.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:38.338216+0000) 2024-03-06T15:40:39.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:39.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:39.338425+0000) 2024-03-06T15:40:40.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:40.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:40.338657+0000) 2024-03-06T15:40:41.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:41.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:41.338861+0000) 2024-03-06T15:40:42.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:42.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:42.339090+0000) 2024-03-06T15:40:43.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:43.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:43.339315+0000) 2024-03-06T15:40:44.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:44.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:44.339511+0000) 2024-03-06T15:40:45.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:45.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:45.339773+0000) 2024-03-06T15:40:46.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:46.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:46.339997+0000) 2024-03-06T15:40:47.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:47.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:47.340223+0000) 2024-03-06T15:40:48.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:48.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:48.340450+0000) 2024-03-06T15:40:49.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:49.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:49.340704+0000) 2024-03-06T15:40:50.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:50.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:50.340903+0000) 2024-03-06T15:40:51.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:51.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:51.341112+0000) 2024-03-06T15:40:52.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:52.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:52.341288+0000) 2024-03-06T15:40:53.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:53.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:53.341517+0000) 2024-03-06T15:40:54.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:54.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:54.341753+0000) 2024-03-06T15:40:55.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:55.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:55.341984+0000) 2024-03-06T15:40:56.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:56.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:56.342195+0000) 2024-03-06T15:40:57.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:57.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:57.342416+0000) 2024-03-06T15:40:58.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:58.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:58.342638+0000) 2024-03-06T15:40:59.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:40:59.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:40:59.342873+0000) 2024-03-06T15:41:00.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:00.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:00.343112+0000) 2024-03-06T15:41:01.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:01.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:01.343336+0000) 2024-03-06T15:41:02.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:02.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:02.343610+0000) 2024-03-06T15:41:03.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:03.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:03.343858+0000) 2024-03-06T15:41:04.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:04.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:04.344047+0000) 2024-03-06T15:41:04.716 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:41:04.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:41:05.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:05.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:05.344295+0000) 2024-03-06T15:41:06.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:06.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:06.344602+0000) 2024-03-06T15:41:07.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:07.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:07.344787+0000) 2024-03-06T15:41:08.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:08.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:08.344976+0000) 2024-03-06T15:41:09.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:09.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:09.345234+0000) 2024-03-06T15:41:10.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:10.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:10.345478+0000) 2024-03-06T15:41:11.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:11.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:11.345714+0000) 2024-03-06T15:41:12.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:12.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:12.345897+0000) 2024-03-06T15:41:13.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:13.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:13.346084+0000) 2024-03-06T15:41:14.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:14.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:14.346331+0000) 2024-03-06T15:41:15.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:15.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:15.346621+0000) 2024-03-06T15:41:16.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:16.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:16.346822+0000) 2024-03-06T15:41:17.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:17.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:17.347021+0000) 2024-03-06T15:41:18.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:18.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:18.347258+0000) 2024-03-06T15:41:19.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:19.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:19.347496+0000) 2024-03-06T15:41:20.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:20.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:20.347684+0000) 2024-03-06T15:41:21.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:21.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:21.347870+0000) 2024-03-06T15:41:22.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:22.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:22.348090+0000) 2024-03-06T15:41:23.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:23.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:23.348319+0000) 2024-03-06T15:41:24.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:24.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:24.348621+0000) 2024-03-06T15:41:25.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:25.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:25.348827+0000) 2024-03-06T15:41:26.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:26.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:26.349025+0000) 2024-03-06T15:41:27.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:27.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:27.349223+0000) 2024-03-06T15:41:28.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:28.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:28.349422+0000) 2024-03-06T15:41:29.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:29.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:29.349652+0000) 2024-03-06T15:41:30.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:30.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:30.349860+0000) 2024-03-06T15:41:31.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:31.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:31.350044+0000) 2024-03-06T15:41:32.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:32.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:32.350227+0000) 2024-03-06T15:41:33.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:33.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:33.350445+0000) 2024-03-06T15:41:34.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:34.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:34.350679+0000) 2024-03-06T15:41:34.916 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:41:34.924 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:41:35.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:35.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:35.350911+0000) 2024-03-06T15:41:36.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:36.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:36.351142+0000) 2024-03-06T15:41:37.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:37.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:37.351363+0000) 2024-03-06T15:41:38.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:38.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:38.351609+0000) 2024-03-06T15:41:39.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:39.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:39.351826+0000) 2024-03-06T15:41:40.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:40.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:40.352058+0000) 2024-03-06T15:41:41.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:41.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:41.352310+0000) 2024-03-06T15:41:42.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:42.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:42.352588+0000) 2024-03-06T15:41:43.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:43.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:43.352816+0000) 2024-03-06T15:41:44.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:44.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:44.353039+0000) 2024-03-06T15:41:45.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:45.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:45.353258+0000) 2024-03-06T15:41:46.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:46.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:46.353472+0000) 2024-03-06T15:41:47.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:47.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:47.353689+0000) 2024-03-06T15:41:48.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:48.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:48.353960+0000) 2024-03-06T15:41:49.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:49.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:49.354246+0000) 2024-03-06T15:41:50.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:50.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:50.354517+0000) 2024-03-06T15:41:51.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:51.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:51.354794+0000) 2024-03-06T15:41:52.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:52.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:52.355070+0000) 2024-03-06T15:41:53.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:53.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:53.355287+0000) 2024-03-06T15:41:54.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:54.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:54.355579+0000) 2024-03-06T15:41:55.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:55.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:55.355865+0000) 2024-03-06T15:41:56.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:56.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:56.356138+0000) 2024-03-06T15:41:57.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:57.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:57.356355+0000) 2024-03-06T15:41:58.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:58.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:58.356635+0000) 2024-03-06T15:41:59.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:41:59.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:41:59.356890+0000) 2024-03-06T15:42:00.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:00.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:00.357121+0000) 2024-03-06T15:42:01.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:01.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:01.357315+0000) 2024-03-06T15:42:02.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:02.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:02.357523+0000) 2024-03-06T15:42:03.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:03.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:03.357793+0000) 2024-03-06T15:42:04.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:04.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:04.358043+0000) 2024-03-06T15:42:05.125 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:42:05.132 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:42:05.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:05.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:05.358265+0000) 2024-03-06T15:42:06.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:06.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:06.358498+0000) 2024-03-06T15:42:07.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:07.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:07.358730+0000) 2024-03-06T15:42:08.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:08.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:08.358904+0000) 2024-03-06T15:42:09.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:09.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:09.359154+0000) 2024-03-06T15:42:10.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:10.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:10.359381+0000) 2024-03-06T15:42:11.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:11.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:11.359641+0000) 2024-03-06T15:42:12.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:12.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:12.359863+0000) 2024-03-06T15:42:13.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:13.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:13.360081+0000) 2024-03-06T15:42:14.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:14.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:14.360332+0000) 2024-03-06T15:42:15.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:15.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:15.360601+0000) 2024-03-06T15:42:16.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:16.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:16.360821+0000) 2024-03-06T15:42:17.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:17.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:17.361035+0000) 2024-03-06T15:42:18.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:18.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:18.361257+0000) 2024-03-06T15:42:19.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:19.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:19.361509+0000) 2024-03-06T15:42:20.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:20.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:20.361775+0000) 2024-03-06T15:42:21.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:21.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:21.361985+0000) 2024-03-06T15:42:22.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:22.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:22.362208+0000) 2024-03-06T15:42:23.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:23.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:23.362398+0000) 2024-03-06T15:42:24.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:24.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:24.362668+0000) 2024-03-06T15:42:25.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:25.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:25.362893+0000) 2024-03-06T15:42:26.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:26.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:26.363171+0000) 2024-03-06T15:42:27.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:27.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:27.363364+0000) 2024-03-06T15:42:28.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:28.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:28.363569+0000) 2024-03-06T15:42:29.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:29.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:29.363806+0000) 2024-03-06T15:42:30.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:30.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:30.364000+0000) 2024-03-06T15:42:31.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:31.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:31.364257+0000) 2024-03-06T15:42:32.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:32.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:32.364491+0000) 2024-03-06T15:42:33.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:33.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:33.364682+0000) 2024-03-06T15:42:34.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:34.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:34.364929+0000) 2024-03-06T15:42:35.347 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:42:35.355 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:42:35.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:35.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:35.365202+0000) 2024-03-06T15:42:36.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:36.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:36.365486+0000) 2024-03-06T15:42:37.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:37.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:37.365689+0000) 2024-03-06T15:42:38.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:38.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:38.365910+0000) 2024-03-06T15:42:39.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:39.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:39.366164+0000) 2024-03-06T15:42:40.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:40.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:40.366369+0000) 2024-03-06T15:42:41.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:41.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:41.366595+0000) 2024-03-06T15:42:42.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:42.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:42.366821+0000) 2024-03-06T15:42:43.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:43.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:43.367078+0000) 2024-03-06T15:42:44.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:44.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:44.367288+0000) 2024-03-06T15:42:45.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:45.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:45.367528+0000) 2024-03-06T15:42:46.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:46.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:46.367787+0000) 2024-03-06T15:42:47.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:47.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:47.368014+0000) 2024-03-06T15:42:48.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:48.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:48.368207+0000) 2024-03-06T15:42:49.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:49.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:49.368520+0000) 2024-03-06T15:42:50.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:50.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:50.368811+0000) 2024-03-06T15:42:51.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:51.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:51.369030+0000) 2024-03-06T15:42:52.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:52.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:52.369300+0000) 2024-03-06T15:42:53.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:53.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:53.369610+0000) 2024-03-06T15:42:54.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:54.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:54.369896+0000) 2024-03-06T15:42:55.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:55.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:55.370112+0000) 2024-03-06T15:42:56.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:56.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:56.370338+0000) 2024-03-06T15:42:57.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:57.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:57.370613+0000) 2024-03-06T15:42:58.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:58.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:58.370830+0000) 2024-03-06T15:42:59.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:42:59.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:42:59.371063+0000) 2024-03-06T15:43:00.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:00.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:00.371322+0000) 2024-03-06T15:43:01.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:01.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:01.371525+0000) 2024-03-06T15:43:02.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:02.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:02.371797+0000) 2024-03-06T15:43:03.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:03.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:03.371988+0000) 2024-03-06T15:43:04.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:04.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:04.372222+0000) 2024-03-06T15:43:05.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:05.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:05.372401+0000) 2024-03-06T15:43:05.520 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:43:05.528 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:43:06.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:06.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:06.372619+0000) 2024-03-06T15:43:07.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:07.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:07.372836+0000) 2024-03-06T15:43:08.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:08.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:08.373057+0000) 2024-03-06T15:43:09.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:09.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:09.373280+0000) 2024-03-06T15:43:10.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:10.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:10.373517+0000) 2024-03-06T15:43:11.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:11.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:11.373796+0000) 2024-03-06T15:43:12.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:12.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:12.373998+0000) 2024-03-06T15:43:13.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:13.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:13.374244+0000) 2024-03-06T15:43:14.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:14.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:14.374465+0000) 2024-03-06T15:43:15.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:15.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:15.374689+0000) 2024-03-06T15:43:16.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:16.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:16.374902+0000) 2024-03-06T15:43:17.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:17.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:17.375123+0000) 2024-03-06T15:43:18.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:18.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:18.375335+0000) 2024-03-06T15:43:19.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:19.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:19.375661+0000) 2024-03-06T15:43:20.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:20.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:20.375942+0000) 2024-03-06T15:43:21.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:21.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:21.376157+0000) 2024-03-06T15:43:22.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:22.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:22.376376+0000) 2024-03-06T15:43:23.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:23.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:23.376692+0000) 2024-03-06T15:43:24.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:24.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:24.376954+0000) 2024-03-06T15:43:25.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:25.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:25.377170+0000) 2024-03-06T15:43:26.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:26.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:26.377383+0000) 2024-03-06T15:43:27.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:27.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:27.377640+0000) 2024-03-06T15:43:28.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:28.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:28.377866+0000) 2024-03-06T15:43:29.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:29.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:29.378152+0000) 2024-03-06T15:43:30.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:30.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:30.378395+0000) 2024-03-06T15:43:31.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:31.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:31.378707+0000) 2024-03-06T15:43:32.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:32.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:32.378916+0000) 2024-03-06T15:43:33.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:33.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:33.379099+0000) 2024-03-06T15:43:34.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:34.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:34.379329+0000) 2024-03-06T15:43:35.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:35.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:35.379591+0000) 2024-03-06T15:43:35.724 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:43:35.732 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:43:36.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:36.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:36.379823+0000) 2024-03-06T15:43:37.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:37.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:37.380047+0000) 2024-03-06T15:43:38.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:38.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:38.380329+0000) 2024-03-06T15:43:39.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:39.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:39.380612+0000) 2024-03-06T15:43:40.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:40.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:40.380891+0000) 2024-03-06T15:43:41.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:41.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:41.381168+0000) 2024-03-06T15:43:42.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:42.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:42.381442+0000) 2024-03-06T15:43:43.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:43.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:43.381698+0000) 2024-03-06T15:43:44.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:44.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:44.381942+0000) 2024-03-06T15:43:45.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:45.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:45.382175+0000) 2024-03-06T15:43:46.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:46.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:46.382376+0000) 2024-03-06T15:43:47.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:47.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:47.382643+0000) 2024-03-06T15:43:48.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:48.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:48.382882+0000) 2024-03-06T15:43:49.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:49.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:49.383123+0000) 2024-03-06T15:43:50.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:50.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:50.383392+0000) 2024-03-06T15:43:51.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:51.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:51.383650+0000) 2024-03-06T15:43:52.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:52.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:52.383884+0000) 2024-03-06T15:43:53.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:53.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:53.384092+0000) 2024-03-06T15:43:54.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:54.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:54.384322+0000) 2024-03-06T15:43:55.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:55.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:55.384589+0000) 2024-03-06T15:43:56.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:56.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:56.384820+0000) 2024-03-06T15:43:57.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:57.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:57.385044+0000) 2024-03-06T15:43:58.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:58.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:58.385263+0000) 2024-03-06T15:43:59.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:43:59.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:43:59.385496+0000) 2024-03-06T15:44:00.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:00.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:00.385699+0000) 2024-03-06T15:44:01.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:01.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:01.385946+0000) 2024-03-06T15:44:02.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:02.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:02.386195+0000) 2024-03-06T15:44:03.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:03.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:03.386382+0000) 2024-03-06T15:44:04.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:04.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:04.386662+0000) 2024-03-06T15:44:05.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:05.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:05.386895+0000) 2024-03-06T15:44:05.937 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:44:05.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:44:06.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:06.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:06.387147+0000) 2024-03-06T15:44:07.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:07.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:07.387395+0000) 2024-03-06T15:44:08.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:08.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:08.387653+0000) 2024-03-06T15:44:09.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:09.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:09.387892+0000) 2024-03-06T15:44:10.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:10.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:10.388090+0000) 2024-03-06T15:44:11.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:11.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:11.388400+0000) 2024-03-06T15:44:12.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:12.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:12.388711+0000) 2024-03-06T15:44:13.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:13.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:13.388945+0000) 2024-03-06T15:44:14.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:14.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:14.389191+0000) 2024-03-06T15:44:15.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:15.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:15.389428+0000) 2024-03-06T15:44:16.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:16.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:16.389666+0000) 2024-03-06T15:44:17.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:17.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:17.389896+0000) 2024-03-06T15:44:18.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:18.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:18.390114+0000) 2024-03-06T15:44:19.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:19.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:19.390426+0000) 2024-03-06T15:44:20.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:20.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:20.390707+0000) 2024-03-06T15:44:21.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:21.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:21.390985+0000) 2024-03-06T15:44:22.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:22.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:22.391264+0000) 2024-03-06T15:44:23.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:23.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:23.391528+0000) 2024-03-06T15:44:24.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:24.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:24.391841+0000) 2024-03-06T15:44:25.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:25.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:25.392093+0000) 2024-03-06T15:44:26.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:26.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:26.392335+0000) 2024-03-06T15:44:27.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:27.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:27.392616+0000) 2024-03-06T15:44:28.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:28.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:28.392898+0000) 2024-03-06T15:44:29.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:29.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:29.393188+0000) 2024-03-06T15:44:30.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:30.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:30.393436+0000) 2024-03-06T15:44:31.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:31.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:31.393662+0000) 2024-03-06T15:44:32.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:32.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:32.393885+0000) 2024-03-06T15:44:33.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:33.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:33.394059+0000) 2024-03-06T15:44:34.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:34.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:34.394246+0000) 2024-03-06T15:44:35.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:35.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:35.394486+0000) 2024-03-06T15:44:36.143 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:44:36.152 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:44:36.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:36.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:36.394748+0000) 2024-03-06T15:44:37.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:37.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:37.394991+0000) 2024-03-06T15:44:38.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:38.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:38.395234+0000) 2024-03-06T15:44:39.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:39.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:39.395488+0000) 2024-03-06T15:44:40.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:40.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:40.395663+0000) 2024-03-06T15:44:41.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:41.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:41.395889+0000) 2024-03-06T15:44:42.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:42.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:42.396129+0000) 2024-03-06T15:44:43.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:43.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:43.396375+0000) 2024-03-06T15:44:44.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:44.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:44.396645+0000) 2024-03-06T15:44:45.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:45.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:45.396876+0000) 2024-03-06T15:44:46.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:46.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:46.397119+0000) 2024-03-06T15:44:47.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:47.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:47.397309+0000) 2024-03-06T15:44:48.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:48.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:48.397565+0000) 2024-03-06T15:44:49.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:49.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:49.397770+0000) 2024-03-06T15:44:50.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:50.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:50.397999+0000) 2024-03-06T15:44:51.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:51.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:51.398220+0000) 2024-03-06T15:44:52.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:52.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:52.398446+0000) 2024-03-06T15:44:53.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:53.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:53.398666+0000) 2024-03-06T15:44:54.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:54.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:54.398846+0000) 2024-03-06T15:44:55.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:55.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:55.399080+0000) 2024-03-06T15:44:56.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:56.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:56.399299+0000) 2024-03-06T15:44:57.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:57.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:57.399522+0000) 2024-03-06T15:44:58.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:58.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:58.399810+0000) 2024-03-06T15:44:59.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:44:59.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:44:59.400038+0000) 2024-03-06T15:45:00.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:00.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:00.400243+0000) 2024-03-06T15:45:01.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:01.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:01.400482+0000) 2024-03-06T15:45:02.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:02.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:02.400710+0000) 2024-03-06T15:45:03.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:03.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:03.400930+0000) 2024-03-06T15:45:04.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:04.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:04.401156+0000) 2024-03-06T15:45:05.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:05.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:05.401432+0000) 2024-03-06T15:45:06.357 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:45:06.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:45:06.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:06.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:06.401602+0000) 2024-03-06T15:45:07.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:07.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:07.401815+0000) 2024-03-06T15:45:08.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:08.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:08.401994+0000) 2024-03-06T15:45:09.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:09.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:09.402224+0000) 2024-03-06T15:45:10.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:10.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:10.402416+0000) 2024-03-06T15:45:11.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:11.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:11.402667+0000) 2024-03-06T15:45:12.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:12.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:12.402871+0000) 2024-03-06T15:45:13.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:13.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:13.403116+0000) 2024-03-06T15:45:14.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:14.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:14.403315+0000) 2024-03-06T15:45:15.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:15.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:15.403567+0000) 2024-03-06T15:45:16.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:16.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:16.403805+0000) 2024-03-06T15:45:17.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:17.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:17.403999+0000) 2024-03-06T15:45:18.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:18.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:18.404249+0000) 2024-03-06T15:45:19.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:19.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:19.404492+0000) 2024-03-06T15:45:20.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:20.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:20.404684+0000) 2024-03-06T15:45:21.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:21.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:21.404900+0000) 2024-03-06T15:45:22.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:22.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:22.405126+0000) 2024-03-06T15:45:23.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:23.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:23.405307+0000) 2024-03-06T15:45:24.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:24.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:24.405500+0000) 2024-03-06T15:45:25.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:25.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:25.405783+0000) 2024-03-06T15:45:26.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:26.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:26.406006+0000) 2024-03-06T15:45:27.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:27.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:27.406181+0000) 2024-03-06T15:45:28.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:28.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:28.406401+0000) 2024-03-06T15:45:29.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:29.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:29.406640+0000) 2024-03-06T15:45:30.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:30.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:30.406836+0000) 2024-03-06T15:45:31.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:31.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:31.407055+0000) 2024-03-06T15:45:32.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:32.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:32.407277+0000) 2024-03-06T15:45:33.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:33.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:33.407493+0000) 2024-03-06T15:45:34.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:34.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:34.407710+0000) 2024-03-06T15:45:35.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:35.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:35.407936+0000) 2024-03-06T15:45:36.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:36.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:36.408155+0000) 2024-03-06T15:45:36.529 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:45:36.537 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:45:37.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:37.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:37.408388+0000) 2024-03-06T15:45:38.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:38.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:38.408647+0000) 2024-03-06T15:45:39.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:39.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:39.408875+0000) 2024-03-06T15:45:40.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:40.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:40.409065+0000) 2024-03-06T15:45:41.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:41.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:41.409269+0000) 2024-03-06T15:45:42.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:42.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:42.409496+0000) 2024-03-06T15:45:43.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:43.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:43.409718+0000) 2024-03-06T15:45:44.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:44.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:44.409896+0000) 2024-03-06T15:45:45.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:45.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:45.410119+0000) 2024-03-06T15:45:46.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:46.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:46.410343+0000) 2024-03-06T15:45:47.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:47.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:47.410517+0000) 2024-03-06T15:45:48.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:48.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:48.410776+0000) 2024-03-06T15:45:49.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:49.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:49.411007+0000) 2024-03-06T15:45:50.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:50.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:50.411241+0000) 2024-03-06T15:45:51.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:51.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:51.411463+0000) 2024-03-06T15:45:52.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:52.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:52.411688+0000) 2024-03-06T15:45:53.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:53.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:53.411920+0000) 2024-03-06T15:45:54.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:54.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:54.412144+0000) 2024-03-06T15:45:55.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:55.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:55.412325+0000) 2024-03-06T15:45:56.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:56.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:56.412573+0000) 2024-03-06T15:45:57.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:57.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:57.412800+0000) 2024-03-06T15:45:58.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:58.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:58.412996+0000) 2024-03-06T15:45:59.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:45:59.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:45:59.413278+0000) 2024-03-06T15:46:00.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:00.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:00.413485+0000) 2024-03-06T15:46:01.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:01.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:01.413735+0000) 2024-03-06T15:46:02.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:02.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:02.413960+0000) 2024-03-06T15:46:03.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:03.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:03.414136+0000) 2024-03-06T15:46:04.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:04.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:04.414368+0000) 2024-03-06T15:46:05.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:05.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:05.414639+0000) 2024-03-06T15:46:06.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:06.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:06.414833+0000) 2024-03-06T15:46:06.701 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:46:06.710 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:46:07.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:07.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:07.415010+0000) 2024-03-06T15:46:08.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:08.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:08.415224+0000) 2024-03-06T15:46:09.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:09.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:09.415454+0000) 2024-03-06T15:46:10.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:10.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:10.415705+0000) 2024-03-06T15:46:11.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:11.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:11.415896+0000) 2024-03-06T15:46:12.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:12.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:12.416135+0000) 2024-03-06T15:46:13.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:13.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:13.416318+0000) 2024-03-06T15:46:14.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:14.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:14.416580+0000) 2024-03-06T15:46:15.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:15.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:15.416797+0000) 2024-03-06T15:46:16.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:16.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:16.417009+0000) 2024-03-06T15:46:17.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:17.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:17.417242+0000) 2024-03-06T15:46:18.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:18.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:18.417456+0000) 2024-03-06T15:46:19.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:19.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:19.417680+0000) 2024-03-06T15:46:20.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:20.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:20.417900+0000) 2024-03-06T15:46:21.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:21.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:21.418125+0000) 2024-03-06T15:46:22.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:22.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:22.418310+0000) 2024-03-06T15:46:23.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:23.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:23.418573+0000) 2024-03-06T15:46:24.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:24.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:24.418818+0000) 2024-03-06T15:46:25.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:25.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:25.419059+0000) 2024-03-06T15:46:26.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:26.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:26.419317+0000) 2024-03-06T15:46:27.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:27.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:27.419598+0000) 2024-03-06T15:46:28.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:28.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:28.419828+0000) 2024-03-06T15:46:29.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:29.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:29.420074+0000) 2024-03-06T15:46:30.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:30.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:30.420284+0000) 2024-03-06T15:46:31.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:31.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:31.420486+0000) 2024-03-06T15:46:32.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:32.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:32.420717+0000) 2024-03-06T15:46:33.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:33.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:33.420940+0000) 2024-03-06T15:46:34.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:34.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:34.421122+0000) 2024-03-06T15:46:35.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:35.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:35.421318+0000) 2024-03-06T15:46:36.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:36.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:36.421626+0000) 2024-03-06T15:46:36.921 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:46:36.929 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:46:37.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:37.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:37.421823+0000) 2024-03-06T15:46:38.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:38.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:38.422054+0000) 2024-03-06T15:46:39.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:39.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:39.422253+0000) 2024-03-06T15:46:40.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:40.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:40.422498+0000) 2024-03-06T15:46:41.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:41.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:41.422705+0000) 2024-03-06T15:46:42.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:42.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:42.422907+0000) 2024-03-06T15:46:43.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:43.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:43.423125+0000) 2024-03-06T15:46:44.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:44.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:44.423348+0000) 2024-03-06T15:46:45.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:45.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:45.423632+0000) 2024-03-06T15:46:46.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:46.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:46.423855+0000) 2024-03-06T15:46:47.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:47.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:47.424078+0000) 2024-03-06T15:46:48.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:48.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:48.424274+0000) 2024-03-06T15:46:49.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:49.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:49.424498+0000) 2024-03-06T15:46:50.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:50.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:50.424779+0000) 2024-03-06T15:46:51.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:51.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:51.425022+0000) 2024-03-06T15:46:52.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:52.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:52.425244+0000) 2024-03-06T15:46:53.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:53.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:53.425416+0000) 2024-03-06T15:46:54.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:54.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:54.425648+0000) 2024-03-06T15:46:55.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:55.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:55.425861+0000) 2024-03-06T15:46:56.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:56.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:56.426053+0000) 2024-03-06T15:46:57.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:57.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:57.426278+0000) 2024-03-06T15:46:58.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:58.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:58.426490+0000) 2024-03-06T15:46:59.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:46:59.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:46:59.426731+0000) 2024-03-06T15:47:00.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:00.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:00.426972+0000) 2024-03-06T15:47:01.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:01.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:01.427150+0000) 2024-03-06T15:47:02.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:02.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:02.427374+0000) 2024-03-06T15:47:03.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:03.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:03.427608+0000) 2024-03-06T15:47:04.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:04.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:04.427840+0000) 2024-03-06T15:47:05.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:05.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:05.428090+0000) 2024-03-06T15:47:06.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:06.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:06.428318+0000) 2024-03-06T15:47:07.145 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:47:07.153 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:47:07.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:07.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:07.428583+0000) 2024-03-06T15:47:08.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:08.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:08.428797+0000) 2024-03-06T15:47:09.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:09.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:09.429073+0000) 2024-03-06T15:47:10.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:10.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:10.429292+0000) 2024-03-06T15:47:11.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:11.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:11.429575+0000) 2024-03-06T15:47:12.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:12.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:12.429830+0000) 2024-03-06T15:47:13.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:13.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:13.430066+0000) 2024-03-06T15:47:14.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:14.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:14.430298+0000) 2024-03-06T15:47:15.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:15.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:15.430524+0000) 2024-03-06T15:47:16.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:16.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:16.430788+0000) 2024-03-06T15:47:17.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:17.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:17.431008+0000) 2024-03-06T15:47:18.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:18.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:18.431201+0000) 2024-03-06T15:47:19.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:19.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:19.431449+0000) 2024-03-06T15:47:20.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:20.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:20.431683+0000) 2024-03-06T15:47:21.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:21.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:21.431874+0000) 2024-03-06T15:47:22.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:22.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:22.432097+0000) 2024-03-06T15:47:23.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:23.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:23.432325+0000) 2024-03-06T15:47:24.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:24.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:24.432523+0000) 2024-03-06T15:47:25.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:25.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:25.432839+0000) 2024-03-06T15:47:26.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:26.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:26.433028+0000) 2024-03-06T15:47:27.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:27.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:27.433198+0000) 2024-03-06T15:47:28.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:28.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:28.433419+0000) 2024-03-06T15:47:29.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:29.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:29.433651+0000) 2024-03-06T15:47:30.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:30.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:30.433895+0000) 2024-03-06T15:47:31.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:31.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:31.434075+0000) 2024-03-06T15:47:32.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:32.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:32.434296+0000) 2024-03-06T15:47:33.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:33.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:33.434527+0000) 2024-03-06T15:47:34.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:34.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:34.434733+0000) 2024-03-06T15:47:35.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:35.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:35.434976+0000) 2024-03-06T15:47:36.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:36.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:36.435164+0000) 2024-03-06T15:47:37.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:47:37.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:47:37.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:37.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:37.435333+0000) 2024-03-06T15:47:38.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:38.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:38.435587+0000) 2024-03-06T15:47:39.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:39.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:39.435826+0000) 2024-03-06T15:47:40.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:40.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:40.436153+0000) 2024-03-06T15:47:41.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:41.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:41.436390+0000) 2024-03-06T15:47:42.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:42.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:42.436611+0000) 2024-03-06T15:47:43.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:43.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:43.436838+0000) 2024-03-06T15:47:44.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:44.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:44.437040+0000) 2024-03-06T15:47:45.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:45.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:45.437305+0000) 2024-03-06T15:47:46.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:46.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:46.437572+0000) 2024-03-06T15:47:47.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:47.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:47.437802+0000) 2024-03-06T15:47:48.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:48.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:48.438002+0000) 2024-03-06T15:47:49.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:49.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:49.438209+0000) 2024-03-06T15:47:50.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:50.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:50.438435+0000) 2024-03-06T15:47:51.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:51.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:51.438661+0000) 2024-03-06T15:47:52.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:52.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:52.438858+0000) 2024-03-06T15:47:53.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:53.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:53.439039+0000) 2024-03-06T15:47:54.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:54.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:54.439266+0000) 2024-03-06T15:47:55.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:55.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:55.439508+0000) 2024-03-06T15:47:56.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:56.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:56.439746+0000) 2024-03-06T15:47:57.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:57.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:57.439970+0000) 2024-03-06T15:47:58.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:58.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:58.440194+0000) 2024-03-06T15:47:59.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:47:59.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:47:59.440430+0000) 2024-03-06T15:48:00.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:00.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:00.440678+0000) 2024-03-06T15:48:01.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:01.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:01.440907+0000) 2024-03-06T15:48:02.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:02.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:02.441123+0000) 2024-03-06T15:48:03.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:03.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:03.441321+0000) 2024-03-06T15:48:04.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:04.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:04.441632+0000) 2024-03-06T15:48:05.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:05.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:05.441880+0000) 2024-03-06T15:48:06.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:06.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:06.442115+0000) 2024-03-06T15:48:07.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:07.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:07.442350+0000) 2024-03-06T15:48:07.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:48:07.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:48:08.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:08.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:08.442616+0000) 2024-03-06T15:48:09.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:09.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:09.442882+0000) 2024-03-06T15:48:10.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:10.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:10.443098+0000) 2024-03-06T15:48:11.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:11.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:11.443322+0000) 2024-03-06T15:48:12.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:12.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:12.443528+0000) 2024-03-06T15:48:13.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:13.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:13.443824+0000) 2024-03-06T15:48:14.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:14.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:14.444051+0000) 2024-03-06T15:48:15.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:15.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:15.444298+0000) 2024-03-06T15:48:16.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:16.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:16.444609+0000) 2024-03-06T15:48:17.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:17.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:17.444875+0000) 2024-03-06T15:48:18.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:18.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:18.445083+0000) 2024-03-06T15:48:19.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:19.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:19.445321+0000) 2024-03-06T15:48:20.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:20.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:20.445528+0000) 2024-03-06T15:48:21.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:21.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:21.445812+0000) 2024-03-06T15:48:22.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:22.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:22.446035+0000) 2024-03-06T15:48:23.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:23.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:23.446261+0000) 2024-03-06T15:48:24.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:24.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:24.446484+0000) 2024-03-06T15:48:25.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:25.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:25.446713+0000) 2024-03-06T15:48:26.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:26.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:26.446989+0000) 2024-03-06T15:48:27.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:27.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:27.447208+0000) 2024-03-06T15:48:28.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:28.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:28.447482+0000) 2024-03-06T15:48:29.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:29.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:29.447721+0000) 2024-03-06T15:48:30.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:30.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:30.447959+0000) 2024-03-06T15:48:31.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:31.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:31.448184+0000) 2024-03-06T15:48:32.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:32.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:32.448383+0000) 2024-03-06T15:48:33.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:33.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:33.448647+0000) 2024-03-06T15:48:34.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:34.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:34.448844+0000) 2024-03-06T15:48:35.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:35.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:35.449072+0000) 2024-03-06T15:48:36.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:36.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:36.449296+0000) 2024-03-06T15:48:37.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:37.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:37.449514+0000) 2024-03-06T15:48:37.770 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:48:37.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:48:38.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:38.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:38.449783+0000) 2024-03-06T15:48:39.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:39.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:39.450019+0000) 2024-03-06T15:48:40.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:40.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:40.450247+0000) 2024-03-06T15:48:41.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:41.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:41.450453+0000) 2024-03-06T15:48:42.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:42.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:42.450704+0000) 2024-03-06T15:48:43.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:43.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:43.450922+0000) 2024-03-06T15:48:44.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:44.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:44.451115+0000) 2024-03-06T15:48:45.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:45.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:45.451346+0000) 2024-03-06T15:48:46.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:46.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:46.451571+0000) 2024-03-06T15:48:47.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:47.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:47.451795+0000) 2024-03-06T15:48:48.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:48.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:48.451983+0000) 2024-03-06T15:48:49.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:49.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:49.452194+0000) 2024-03-06T15:48:50.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:50.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:50.452424+0000) 2024-03-06T15:48:51.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:51.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:51.452632+0000) 2024-03-06T15:48:52.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:52.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:52.452846+0000) 2024-03-06T15:48:53.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:53.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:53.453059+0000) 2024-03-06T15:48:54.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:54.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:54.453276+0000) 2024-03-06T15:48:55.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:55.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:55.453503+0000) 2024-03-06T15:48:56.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:56.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:56.453729+0000) 2024-03-06T15:48:57.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:57.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:57.453947+0000) 2024-03-06T15:48:58.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:58.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:58.454167+0000) 2024-03-06T15:48:59.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:48:59.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:48:59.454395+0000) 2024-03-06T15:49:00.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:00.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:00.454667+0000) 2024-03-06T15:49:01.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:01.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:01.454898+0000) 2024-03-06T15:49:02.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:02.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:02.455108+0000) 2024-03-06T15:49:03.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:03.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:03.455388+0000) 2024-03-06T15:49:04.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:04.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:04.455691+0000) 2024-03-06T15:49:05.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:05.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:05.455938+0000) 2024-03-06T15:49:06.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:06.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:06.456133+0000) 2024-03-06T15:49:07.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:07.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:07.456368+0000) 2024-03-06T15:49:07.976 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:49:07.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:49:08.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:08.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:08.456652+0000) 2024-03-06T15:49:09.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:09.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:09.456853+0000) 2024-03-06T15:49:10.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:10.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:10.457078+0000) 2024-03-06T15:49:11.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:11.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:11.457343+0000) 2024-03-06T15:49:12.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:12.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:12.457654+0000) 2024-03-06T15:49:13.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:13.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:13.457923+0000) 2024-03-06T15:49:14.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:14.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:14.458159+0000) 2024-03-06T15:49:15.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:15.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:15.458345+0000) 2024-03-06T15:49:16.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:16.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:16.458665+0000) 2024-03-06T15:49:17.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:17.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:17.458908+0000) 2024-03-06T15:49:18.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:18.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:18.459141+0000) 2024-03-06T15:49:19.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:19.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:19.459336+0000) 2024-03-06T15:49:20.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:20.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:20.459615+0000) 2024-03-06T15:49:21.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:21.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:21.459842+0000) 2024-03-06T15:49:22.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:22.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:22.460027+0000) 2024-03-06T15:49:23.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:23.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:23.460255+0000) 2024-03-06T15:49:24.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:24.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:24.460516+0000) 2024-03-06T15:49:25.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:25.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:25.460831+0000) 2024-03-06T15:49:26.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:26.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:26.461071+0000) 2024-03-06T15:49:27.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:27.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:27.461275+0000) 2024-03-06T15:49:28.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:28.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:28.461507+0000) 2024-03-06T15:49:29.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:29.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:29.461743+0000) 2024-03-06T15:49:30.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:30.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:30.461935+0000) 2024-03-06T15:49:31.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:31.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:31.462160+0000) 2024-03-06T15:49:32.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:32.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:32.462441+0000) 2024-03-06T15:49:33.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:33.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:33.462728+0000) 2024-03-06T15:49:34.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:34.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:34.462978+0000) 2024-03-06T15:49:35.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:35.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:35.463227+0000) 2024-03-06T15:49:36.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:36.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:36.463507+0000) 2024-03-06T15:49:37.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:37.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:37.463791+0000) 2024-03-06T15:49:38.207 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:49:38.218 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:49:38.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:38.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:38.464038+0000) 2024-03-06T15:49:39.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:39.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:39.464326+0000) 2024-03-06T15:49:40.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:40.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:40.464651+0000) 2024-03-06T15:49:41.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:41.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:41.464874+0000) 2024-03-06T15:49:42.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:42.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:42.465078+0000) 2024-03-06T15:49:43.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:43.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:43.465272+0000) 2024-03-06T15:49:44.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:44.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:44.465495+0000) 2024-03-06T15:49:45.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:45.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:45.465693+0000) 2024-03-06T15:49:46.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:46.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:46.465877+0000) 2024-03-06T15:49:47.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:47.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:47.466065+0000) 2024-03-06T15:49:48.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:48.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:48.466291+0000) 2024-03-06T15:49:49.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:49.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:49.466587+0000) 2024-03-06T15:49:50.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:50.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:50.466835+0000) 2024-03-06T15:49:51.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:51.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:51.467080+0000) 2024-03-06T15:49:52.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:52.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:52.467319+0000) 2024-03-06T15:49:53.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:53.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:53.467577+0000) 2024-03-06T15:49:54.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:54.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:54.467769+0000) 2024-03-06T15:49:55.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:55.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:55.467967+0000) 2024-03-06T15:49:56.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:56.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:56.468150+0000) 2024-03-06T15:49:57.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:57.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:57.468391+0000) 2024-03-06T15:49:58.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:58.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:58.468627+0000) 2024-03-06T15:49:59.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:49:59.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:49:59.468897+0000) 2024-03-06T15:50:00.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:00.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:00.469096+0000) 2024-03-06T15:50:01.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:01.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:01.469318+0000) 2024-03-06T15:50:02.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:02.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:02.469572+0000) 2024-03-06T15:50:03.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:03.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:03.469763+0000) 2024-03-06T15:50:04.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:04.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:04.469951+0000) 2024-03-06T15:50:05.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:05.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:05.470163+0000) 2024-03-06T15:50:06.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:06.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:06.470395+0000) 2024-03-06T15:50:07.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:07.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:07.470639+0000) 2024-03-06T15:50:08.443 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:50:08.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:50:08.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:08.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:08.470836+0000) 2024-03-06T15:50:09.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:09.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:09.471083+0000) 2024-03-06T15:50:10.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:10.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:10.471265+0000) 2024-03-06T15:50:11.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:11.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:11.471444+0000) 2024-03-06T15:50:12.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:12.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:12.471664+0000) 2024-03-06T15:50:13.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:13.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:13.471885+0000) 2024-03-06T15:50:14.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:14.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:14.472161+0000) 2024-03-06T15:50:15.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:15.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:15.472444+0000) 2024-03-06T15:50:16.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:16.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:16.472618+0000) 2024-03-06T15:50:17.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:17.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:17.472848+0000) 2024-03-06T15:50:18.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:18.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:18.473084+0000) 2024-03-06T15:50:19.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:19.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:19.473333+0000) 2024-03-06T15:50:20.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:20.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:20.473600+0000) 2024-03-06T15:50:21.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:21.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:21.473812+0000) 2024-03-06T15:50:22.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:22.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:22.474009+0000) 2024-03-06T15:50:23.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:23.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:23.474237+0000) 2024-03-06T15:50:24.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:24.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:24.474464+0000) 2024-03-06T15:50:25.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:25.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:25.474702+0000) 2024-03-06T15:50:26.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:26.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:26.474917+0000) 2024-03-06T15:50:27.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:27.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:27.475158+0000) 2024-03-06T15:50:28.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:28.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:28.475390+0000) 2024-03-06T15:50:29.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:29.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:29.475767+0000) 2024-03-06T15:50:30.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:30.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:30.475983+0000) 2024-03-06T15:50:31.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:31.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:31.476237+0000) 2024-03-06T15:50:32.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:32.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:32.476439+0000) 2024-03-06T15:50:33.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:33.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:33.476689+0000) 2024-03-06T15:50:34.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:34.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:34.476932+0000) 2024-03-06T15:50:35.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:35.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:35.477148+0000) 2024-03-06T15:50:36.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:36.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:36.477393+0000) 2024-03-06T15:50:37.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:37.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:37.477628+0000) 2024-03-06T15:50:38.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:38.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:38.477865+0000) 2024-03-06T15:50:38.647 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:50:38.659 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:50:39.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:39.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:39.478063+0000) 2024-03-06T15:50:40.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:40.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:40.478292+0000) 2024-03-06T15:50:41.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:41.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:41.478487+0000) 2024-03-06T15:50:42.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:42.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:42.478740+0000) 2024-03-06T15:50:43.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:43.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:43.478927+0000) 2024-03-06T15:50:44.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:44.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:44.479123+0000) 2024-03-06T15:50:45.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:45.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:45.479300+0000) 2024-03-06T15:50:46.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:46.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:46.479524+0000) 2024-03-06T15:50:47.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:47.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:47.479765+0000) 2024-03-06T15:50:48.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:48.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:48.479938+0000) 2024-03-06T15:50:49.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:49.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:49.480127+0000) 2024-03-06T15:50:50.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:50.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:50.480364+0000) 2024-03-06T15:50:51.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:51.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:51.480636+0000) 2024-03-06T15:50:52.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:52.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:52.480838+0000) 2024-03-06T15:50:53.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:53.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:53.481030+0000) 2024-03-06T15:50:54.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:54.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:54.481254+0000) 2024-03-06T15:50:55.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:55.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:55.481480+0000) 2024-03-06T15:50:56.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:56.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:56.481709+0000) 2024-03-06T15:50:57.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:57.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:57.481949+0000) 2024-03-06T15:50:58.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:58.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:58.482139+0000) 2024-03-06T15:50:59.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:50:59.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:50:59.482349+0000) 2024-03-06T15:51:00.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:00.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:00.482525+0000) 2024-03-06T15:51:01.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:01.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:01.482753+0000) 2024-03-06T15:51:02.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:02.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:02.482981+0000) 2024-03-06T15:51:03.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:03.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:03.483221+0000) 2024-03-06T15:51:04.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:04.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:04.483414+0000) 2024-03-06T15:51:05.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:05.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:05.483651+0000) 2024-03-06T15:51:06.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:06.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:06.483883+0000) 2024-03-06T15:51:07.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:07.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:07.484116+0000) 2024-03-06T15:51:08.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:08.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:08.484331+0000) 2024-03-06T15:51:08.859 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:51:08.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:51:09.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:09.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:09.484578+0000) 2024-03-06T15:51:10.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:10.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:10.484814+0000) 2024-03-06T15:51:11.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:11.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:11.485007+0000) 2024-03-06T15:51:12.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:12.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:12.485247+0000) 2024-03-06T15:51:13.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:13.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:13.485488+0000) 2024-03-06T15:51:14.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:14.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:14.485730+0000) 2024-03-06T15:51:15.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:15.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:15.485966+0000) 2024-03-06T15:51:16.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:16.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:16.486162+0000) 2024-03-06T15:51:17.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:17.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:17.486394+0000) 2024-03-06T15:51:18.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:18.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:18.486644+0000) 2024-03-06T15:51:19.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:19.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:19.486875+0000) 2024-03-06T15:51:20.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:20.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:20.487101+0000) 2024-03-06T15:51:21.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:21.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:21.487286+0000) 2024-03-06T15:51:22.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:22.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:22.487480+0000) 2024-03-06T15:51:23.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:23.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:23.487685+0000) 2024-03-06T15:51:24.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:24.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:24.487928+0000) 2024-03-06T15:51:25.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:25.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:25.488179+0000) 2024-03-06T15:51:26.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:26.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:26.488420+0000) 2024-03-06T15:51:27.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:27.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:27.488673+0000) 2024-03-06T15:51:28.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:28.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:28.488915+0000) 2024-03-06T15:51:29.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:29.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:29.489150+0000) 2024-03-06T15:51:30.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:30.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:30.489379+0000) 2024-03-06T15:51:31.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:31.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:31.489645+0000) 2024-03-06T15:51:32.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:32.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:32.489866+0000) 2024-03-06T15:51:33.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:33.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:33.490087+0000) 2024-03-06T15:51:34.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:34.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:34.490308+0000) 2024-03-06T15:51:35.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:35.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:35.490528+0000) 2024-03-06T15:51:36.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:36.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:36.490749+0000) 2024-03-06T15:51:37.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:37.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:37.490983+0000) 2024-03-06T15:51:38.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:38.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:38.491189+0000) 2024-03-06T15:51:39.083 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:51:39.091 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:51:39.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:39.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:39.491387+0000) 2024-03-06T15:51:40.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:40.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:40.491662+0000) 2024-03-06T15:51:41.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:41.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:41.491895+0000) 2024-03-06T15:51:42.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:42.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:42.492107+0000) 2024-03-06T15:51:43.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:43.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:43.492334+0000) 2024-03-06T15:51:44.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:44.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:44.492592+0000) 2024-03-06T15:51:45.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:45.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:45.492809+0000) 2024-03-06T15:51:46.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:46.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:46.493028+0000) 2024-03-06T15:51:47.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:47.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:47.493248+0000) 2024-03-06T15:51:48.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:48.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:48.493446+0000) 2024-03-06T15:51:49.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:49.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:49.493712+0000) 2024-03-06T15:51:50.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:50.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:50.493936+0000) 2024-03-06T15:51:51.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:51.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:51.494216+0000) 2024-03-06T15:51:52.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:52.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:52.494441+0000) 2024-03-06T15:51:53.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:53.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:53.494696+0000) 2024-03-06T15:51:54.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:54.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:54.494911+0000) 2024-03-06T15:51:55.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:55.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:55.495136+0000) 2024-03-06T15:51:56.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:56.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:56.495368+0000) 2024-03-06T15:51:57.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:57.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:57.495609+0000) 2024-03-06T15:51:58.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:58.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:58.495846+0000) 2024-03-06T15:51:59.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:51:59.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:51:59.496046+0000) 2024-03-06T15:52:00.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:00.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:00.496251+0000) 2024-03-06T15:52:01.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:01.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:01.496492+0000) 2024-03-06T15:52:02.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:02.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:02.496746+0000) 2024-03-06T15:52:03.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:03.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:03.496950+0000) 2024-03-06T15:52:04.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:04.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:04.497168+0000) 2024-03-06T15:52:05.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:05.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:05.497361+0000) 2024-03-06T15:52:06.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:06.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:06.497638+0000) 2024-03-06T15:52:07.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:07.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:07.497841+0000) 2024-03-06T15:52:08.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:08.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:08.498078+0000) 2024-03-06T15:52:09.219 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:52:09.228 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:52:09.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:09.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:09.498321+0000) 2024-03-06T15:52:10.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:10.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:10.498605+0000) 2024-03-06T15:52:11.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:11.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:11.498816+0000) 2024-03-06T15:52:12.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:12.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:12.499011+0000) 2024-03-06T15:52:13.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:13.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:13.499240+0000) 2024-03-06T15:52:14.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:14.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:14.499468+0000) 2024-03-06T15:52:15.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:15.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:15.499687+0000) 2024-03-06T15:52:16.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:16.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:16.499901+0000) 2024-03-06T15:52:17.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:17.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:17.500110+0000) 2024-03-06T15:52:18.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:18.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:18.500330+0000) 2024-03-06T15:52:19.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:19.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:19.500610+0000) 2024-03-06T15:52:20.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:20.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:20.500842+0000) 2024-03-06T15:52:21.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:21.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:21.501029+0000) 2024-03-06T15:52:22.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:22.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:22.501249+0000) 2024-03-06T15:52:23.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:23.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:23.501487+0000) 2024-03-06T15:52:24.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:24.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:24.501730+0000) 2024-03-06T15:52:25.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:25.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:25.501964+0000) 2024-03-06T15:52:26.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:26.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:26.502202+0000) 2024-03-06T15:52:27.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:27.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:27.502448+0000) 2024-03-06T15:52:28.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:28.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:28.502705+0000) 2024-03-06T15:52:29.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:29.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:29.502926+0000) 2024-03-06T15:52:30.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:30.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:30.503152+0000) 2024-03-06T15:52:31.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:31.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:31.503374+0000) 2024-03-06T15:52:32.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:32.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:32.503597+0000) 2024-03-06T15:52:33.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:33.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:33.503802+0000) 2024-03-06T15:52:34.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:34.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:34.504041+0000) 2024-03-06T15:52:35.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:35.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:35.504267+0000) 2024-03-06T15:52:36.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:36.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:36.504498+0000) 2024-03-06T15:52:37.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:37.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:37.504739+0000) 2024-03-06T15:52:38.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:38.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:38.504966+0000) 2024-03-06T15:52:39.431 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:52:39.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:52:39.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:39.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:39.505182+0000) 2024-03-06T15:52:40.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:40.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:40.505356+0000) 2024-03-06T15:52:41.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:41.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:41.505640+0000) 2024-03-06T15:52:42.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:42.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:42.505864+0000) 2024-03-06T15:52:43.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:43.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:43.506076+0000) 2024-03-06T15:52:44.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:44.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:44.506310+0000) 2024-03-06T15:52:45.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:45.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:45.506504+0000) 2024-03-06T15:52:46.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:46.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:46.506750+0000) 2024-03-06T15:52:47.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:47.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:47.506967+0000) 2024-03-06T15:52:48.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:48.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:48.507183+0000) 2024-03-06T15:52:49.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:49.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:49.507420+0000) 2024-03-06T15:52:50.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:50.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:50.507654+0000) 2024-03-06T15:52:51.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:51.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:51.507889+0000) 2024-03-06T15:52:52.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:52.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:52.508130+0000) 2024-03-06T15:52:53.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:53.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:53.508344+0000) 2024-03-06T15:52:54.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:54.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:54.508572+0000) 2024-03-06T15:52:55.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:55.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:55.508779+0000) 2024-03-06T15:52:56.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:56.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:56.509015+0000) 2024-03-06T15:52:57.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:57.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:57.509241+0000) 2024-03-06T15:52:58.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:58.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:58.509417+0000) 2024-03-06T15:52:59.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:52:59.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:52:59.509638+0000) 2024-03-06T15:53:00.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:00.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:00.509878+0000) 2024-03-06T15:53:01.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:01.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:01.510129+0000) 2024-03-06T15:53:02.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:02.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:02.510376+0000) 2024-03-06T15:53:03.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:03.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:03.510655+0000) 2024-03-06T15:53:04.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:04.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:04.510843+0000) 2024-03-06T15:53:05.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:05.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:05.511070+0000) 2024-03-06T15:53:06.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:06.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:06.511288+0000) 2024-03-06T15:53:07.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:07.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:07.511502+0000) 2024-03-06T15:53:08.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:08.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:08.511709+0000) 2024-03-06T15:53:09.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:09.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:09.511967+0000) 2024-03-06T15:53:09.644 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:53:09.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:53:10.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:10.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:10.512250+0000) 2024-03-06T15:53:11.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:11.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:11.512581+0000) 2024-03-06T15:53:12.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:12.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:12.512796+0000) 2024-03-06T15:53:13.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:13.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:13.512972+0000) 2024-03-06T15:53:14.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:14.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:14.513193+0000) 2024-03-06T15:53:15.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:15.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:15.513400+0000) 2024-03-06T15:53:16.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:16.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:16.513686+0000) 2024-03-06T15:53:17.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:17.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:17.513929+0000) 2024-03-06T15:53:18.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:18.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:18.514117+0000) 2024-03-06T15:53:19.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:19.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:19.514326+0000) 2024-03-06T15:53:20.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:20.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:20.514555+0000) 2024-03-06T15:53:21.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:21.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:21.514762+0000) 2024-03-06T15:53:22.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:22.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:22.514934+0000) 2024-03-06T15:53:23.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:23.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:23.515145+0000) 2024-03-06T15:53:24.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:24.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:24.515392+0000) 2024-03-06T15:53:25.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:25.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:25.515655+0000) 2024-03-06T15:53:26.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:26.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:26.515901+0000) 2024-03-06T15:53:27.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:27.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:27.516170+0000) 2024-03-06T15:53:28.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:28.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:28.516370+0000) 2024-03-06T15:53:29.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:29.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:29.516650+0000) 2024-03-06T15:53:30.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:30.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:30.516852+0000) 2024-03-06T15:53:31.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:31.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:31.517043+0000) 2024-03-06T15:53:32.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:32.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:32.517288+0000) 2024-03-06T15:53:33.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:33.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:33.517571+0000) 2024-03-06T15:53:34.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:34.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:34.517811+0000) 2024-03-06T15:53:35.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:35.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:35.518040+0000) 2024-03-06T15:53:36.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:36.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:36.518280+0000) 2024-03-06T15:53:37.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:37.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:37.518496+0000) 2024-03-06T15:53:38.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:38.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:38.518690+0000) 2024-03-06T15:53:39.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:39.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:39.518930+0000) 2024-03-06T15:53:39.866 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:53:39.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:53:40.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:40.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:40.519122+0000) 2024-03-06T15:53:41.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:41.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:41.519394+0000) 2024-03-06T15:53:42.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:42.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:42.519686+0000) 2024-03-06T15:53:43.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:43.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:43.519920+0000) 2024-03-06T15:53:44.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:44.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:44.520180+0000) 2024-03-06T15:53:45.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:45.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:45.520464+0000) 2024-03-06T15:53:46.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:46.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:46.520719+0000) 2024-03-06T15:53:47.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:47.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:47.520984+0000) 2024-03-06T15:53:48.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:48.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:48.521249+0000) 2024-03-06T15:53:49.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:49.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:49.521484+0000) 2024-03-06T15:53:50.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:50.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:50.521771+0000) 2024-03-06T15:53:51.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:51.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:51.522066+0000) 2024-03-06T15:53:52.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:52.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:52.522337+0000) 2024-03-06T15:53:53.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:53.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:53.522646+0000) 2024-03-06T15:53:54.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:54.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:54.522922+0000) 2024-03-06T15:53:55.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:55.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:55.523202+0000) 2024-03-06T15:53:56.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:56.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:56.523441+0000) 2024-03-06T15:53:57.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:57.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:57.523724+0000) 2024-03-06T15:53:58.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:58.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:58.523908+0000) 2024-03-06T15:53:59.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:53:59.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:53:59.524140+0000) 2024-03-06T15:54:00.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:00.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:00.524365+0000) 2024-03-06T15:54:01.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:01.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:01.524637+0000) 2024-03-06T15:54:02.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:02.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:02.524850+0000) 2024-03-06T15:54:03.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:03.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:03.525090+0000) 2024-03-06T15:54:04.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:04.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:04.525318+0000) 2024-03-06T15:54:05.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:05.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:05.525512+0000) 2024-03-06T15:54:06.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:06.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:06.525785+0000) 2024-03-06T15:54:07.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:07.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:07.526044+0000) 2024-03-06T15:54:08.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:08.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:08.526323+0000) 2024-03-06T15:54:09.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:09.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:09.526564+0000) 2024-03-06T15:54:10.077 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:54:10.088 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:54:10.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:10.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:10.526791+0000) 2024-03-06T15:54:11.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:11.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:11.527035+0000) 2024-03-06T15:54:12.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:12.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:12.527277+0000) 2024-03-06T15:54:13.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:13.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:13.527505+0000) 2024-03-06T15:54:14.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:14.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:14.527793+0000) 2024-03-06T15:54:15.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:15.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:15.528013+0000) 2024-03-06T15:54:16.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:16.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:16.528238+0000) 2024-03-06T15:54:17.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:17.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:17.528464+0000) 2024-03-06T15:54:18.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:18.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:18.528680+0000) 2024-03-06T15:54:19.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:19.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:19.528968+0000) 2024-03-06T15:54:20.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:20.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:20.529237+0000) 2024-03-06T15:54:21.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:21.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:21.529462+0000) 2024-03-06T15:54:22.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:22.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:22.529748+0000) 2024-03-06T15:54:23.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:23.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:23.529971+0000) 2024-03-06T15:54:24.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:24.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:24.530166+0000) 2024-03-06T15:54:25.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:25.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:25.530374+0000) 2024-03-06T15:54:26.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:26.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:26.530634+0000) 2024-03-06T15:54:27.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:27.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:27.530811+0000) 2024-03-06T15:54:28.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:28.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:28.531035+0000) 2024-03-06T15:54:29.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:29.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:29.531243+0000) 2024-03-06T15:54:30.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:30.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:30.531456+0000) 2024-03-06T15:54:31.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:31.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:31.531710+0000) 2024-03-06T15:54:32.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:32.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:32.531939+0000) 2024-03-06T15:54:33.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:33.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:33.532165+0000) 2024-03-06T15:54:34.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:34.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:34.532351+0000) 2024-03-06T15:54:35.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:35.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:35.532589+0000) 2024-03-06T15:54:36.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:36.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:36.532859+0000) 2024-03-06T15:54:37.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:37.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:37.533055+0000) 2024-03-06T15:54:38.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:38.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:38.533235+0000) 2024-03-06T15:54:39.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:39.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:39.533456+0000) 2024-03-06T15:54:40.257 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:54:40.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:54:40.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:40.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:40.533726+0000) 2024-03-06T15:54:41.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:41.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:41.533960+0000) 2024-03-06T15:54:42.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:42.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:42.534180+0000) 2024-03-06T15:54:43.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:43.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:43.534376+0000) 2024-03-06T15:54:44.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:44.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:44.534653+0000) 2024-03-06T15:54:45.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:45.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:45.534882+0000) 2024-03-06T15:54:46.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:46.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:46.535098+0000) 2024-03-06T15:54:47.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:47.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:47.535321+0000) 2024-03-06T15:54:48.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:48.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:48.535590+0000) 2024-03-06T15:54:49.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:49.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:49.535841+0000) 2024-03-06T15:54:50.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:50.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:50.536085+0000) 2024-03-06T15:54:51.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:51.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:51.536330+0000) 2024-03-06T15:54:52.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:52.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:52.536519+0000) 2024-03-06T15:54:53.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:53.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:53.536797+0000) 2024-03-06T15:54:54.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:54.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:54.536986+0000) 2024-03-06T15:54:55.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:55.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:55.537221+0000) 2024-03-06T15:54:56.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:56.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:56.537463+0000) 2024-03-06T15:54:57.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:57.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:57.537671+0000) 2024-03-06T15:54:58.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:58.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:58.537876+0000) 2024-03-06T15:54:59.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:54:59.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:54:59.538094+0000) 2024-03-06T15:55:00.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:00.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:00.538334+0000) 2024-03-06T15:55:01.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:01.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:01.538605+0000) 2024-03-06T15:55:02.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:02.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:02.538800+0000) 2024-03-06T15:55:03.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:03.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:03.538981+0000) 2024-03-06T15:55:04.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:04.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:04.539165+0000) 2024-03-06T15:55:05.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:05.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:05.539403+0000) 2024-03-06T15:55:06.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:06.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:06.539661+0000) 2024-03-06T15:55:07.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:07.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:07.539888+0000) 2024-03-06T15:55:08.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:08.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:08.540158+0000) 2024-03-06T15:55:09.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:09.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:09.540344+0000) 2024-03-06T15:55:10.452 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:55:10.463 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:55:10.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:10.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:10.540511+0000) 2024-03-06T15:55:11.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:11.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:11.540761+0000) 2024-03-06T15:55:12.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:12.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:12.541045+0000) 2024-03-06T15:55:13.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:13.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:13.541274+0000) 2024-03-06T15:55:14.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:14.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:14.541460+0000) 2024-03-06T15:55:15.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:15.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:15.541687+0000) 2024-03-06T15:55:16.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:16.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:16.541889+0000) 2024-03-06T15:55:17.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:17.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:17.542116+0000) 2024-03-06T15:55:18.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:18.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:18.542383+0000) 2024-03-06T15:55:19.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:19.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:19.542712+0000) 2024-03-06T15:55:20.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:20.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:20.542952+0000) 2024-03-06T15:55:21.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:21.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:21.543187+0000) 2024-03-06T15:55:22.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:22.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:22.543425+0000) 2024-03-06T15:55:23.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:23.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:23.543657+0000) 2024-03-06T15:55:24.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:24.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:24.543899+0000) 2024-03-06T15:55:25.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:25.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:25.544145+0000) 2024-03-06T15:55:26.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:26.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:26.544346+0000) 2024-03-06T15:55:27.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:27.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:27.544572+0000) 2024-03-06T15:55:28.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:28.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:28.544812+0000) 2024-03-06T15:55:29.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:29.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:29.545022+0000) 2024-03-06T15:55:30.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:30.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:30.545247+0000) 2024-03-06T15:55:31.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:31.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:31.545471+0000) 2024-03-06T15:55:32.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:32.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:32.545703+0000) 2024-03-06T15:55:33.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:33.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:33.545935+0000) 2024-03-06T15:55:34.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:34.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:34.546169+0000) 2024-03-06T15:55:35.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:35.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:35.546362+0000) 2024-03-06T15:55:36.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:36.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:36.546620+0000) 2024-03-06T15:55:37.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:37.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:37.546813+0000) 2024-03-06T15:55:38.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:38.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:38.547051+0000) 2024-03-06T15:55:39.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:39.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:39.547297+0000) 2024-03-06T15:55:40.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:40.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:40.547502+0000) 2024-03-06T15:55:40.644 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:55:40.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:55:41.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:41.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:41.547754+0000) 2024-03-06T15:55:42.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:42.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:42.547948+0000) 2024-03-06T15:55:43.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:43.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:43.548137+0000) 2024-03-06T15:55:44.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:44.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:44.548374+0000) 2024-03-06T15:55:45.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:45.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:45.548622+0000) 2024-03-06T15:55:46.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:46.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:46.548869+0000) 2024-03-06T15:55:47.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:47.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:47.549066+0000) 2024-03-06T15:55:48.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:48.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:48.549296+0000) 2024-03-06T15:55:49.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:49.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:49.549569+0000) 2024-03-06T15:55:50.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:50.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:50.549775+0000) 2024-03-06T15:55:51.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:51.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:51.550016+0000) 2024-03-06T15:55:52.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:52.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:52.550218+0000) 2024-03-06T15:55:53.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:53.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:53.550454+0000) 2024-03-06T15:55:54.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:54.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:54.550701+0000) 2024-03-06T15:55:55.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:55.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:55.550922+0000) 2024-03-06T15:55:56.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:56.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:56.551201+0000) 2024-03-06T15:55:57.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:57.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:57.551389+0000) 2024-03-06T15:55:58.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:58.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:58.551609+0000) 2024-03-06T15:55:59.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:55:59.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:55:59.551814+0000) 2024-03-06T15:56:00.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:00.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:00.552060+0000) 2024-03-06T15:56:01.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:01.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:01.552297+0000) 2024-03-06T15:56:02.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:02.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:02.552572+0000) 2024-03-06T15:56:03.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:03.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:03.552817+0000) 2024-03-06T15:56:04.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:04.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:04.553006+0000) 2024-03-06T15:56:05.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:05.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:05.553203+0000) 2024-03-06T15:56:06.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:06.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:06.553427+0000) 2024-03-06T15:56:07.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:07.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:07.553656+0000) 2024-03-06T15:56:08.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:08.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:08.553881+0000) 2024-03-06T15:56:09.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:09.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:09.554114+0000) 2024-03-06T15:56:10.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:10.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:10.554344+0000) 2024-03-06T15:56:10.779 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:56:10.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:56:11.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:11.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:11.554612+0000) 2024-03-06T15:56:12.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:12.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:12.554838+0000) 2024-03-06T15:56:13.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:13.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:13.555024+0000) 2024-03-06T15:56:14.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:14.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:14.555220+0000) 2024-03-06T15:56:15.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:15.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:15.555460+0000) 2024-03-06T15:56:16.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:16.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:16.555717+0000) 2024-03-06T15:56:17.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:17.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:17.555923+0000) 2024-03-06T15:56:18.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:18.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:18.556112+0000) 2024-03-06T15:56:19.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:19.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:19.556366+0000) 2024-03-06T15:56:20.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:20.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:20.556640+0000) 2024-03-06T15:56:21.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:21.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:21.556831+0000) 2024-03-06T15:56:22.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:22.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:22.557071+0000) 2024-03-06T15:56:23.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:23.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:23.557301+0000) 2024-03-06T15:56:24.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:24.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:24.557524+0000) 2024-03-06T15:56:25.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:25.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:25.557800+0000) 2024-03-06T15:56:26.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:26.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:26.558021+0000) 2024-03-06T15:56:27.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:27.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:27.558239+0000) 2024-03-06T15:56:28.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:28.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:28.558464+0000) 2024-03-06T15:56:29.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:29.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:29.558696+0000) 2024-03-06T15:56:30.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:30.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:30.558901+0000) 2024-03-06T15:56:31.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:31.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:31.559135+0000) 2024-03-06T15:56:32.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:32.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:32.559361+0000) 2024-03-06T15:56:33.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:33.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:33.559628+0000) 2024-03-06T15:56:34.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:34.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:34.559858+0000) 2024-03-06T15:56:35.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:35.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:35.560084+0000) 2024-03-06T15:56:36.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:36.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:36.560309+0000) 2024-03-06T15:56:37.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:37.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:37.560515+0000) 2024-03-06T15:56:38.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:38.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:38.560770+0000) 2024-03-06T15:56:39.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:39.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:39.560961+0000) 2024-03-06T15:56:40.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:40.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:40.561187+0000) 2024-03-06T15:56:41.008 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:56:41.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:56:41.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:41.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:41.561321+0000) 2024-03-06T15:56:42.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:42.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:42.561568+0000) 2024-03-06T15:56:43.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:43.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:43.561767+0000) 2024-03-06T15:56:44.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:44.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:44.561962+0000) 2024-03-06T15:56:45.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:45.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:45.562186+0000) 2024-03-06T15:56:46.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:46.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:46.562411+0000) 2024-03-06T15:56:47.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:47.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:47.562656+0000) 2024-03-06T15:56:48.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:48.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:48.562894+0000) 2024-03-06T15:56:49.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:49.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:49.563183+0000) 2024-03-06T15:56:50.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:50.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:50.563405+0000) 2024-03-06T15:56:51.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:51.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:51.563658+0000) 2024-03-06T15:56:52.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:52.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:52.563900+0000) 2024-03-06T15:56:53.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:53.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:53.564095+0000) 2024-03-06T15:56:54.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:54.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:54.564284+0000) 2024-03-06T15:56:55.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:55.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:55.564579+0000) 2024-03-06T15:56:56.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:56.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:56.564826+0000) 2024-03-06T15:56:57.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:57.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:57.565068+0000) 2024-03-06T15:56:58.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:58.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:58.565279+0000) 2024-03-06T15:56:59.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:56:59.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:56:59.565514+0000) 2024-03-06T15:57:00.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:00.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:00.565751+0000) 2024-03-06T15:57:01.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:01.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:01.565983+0000) 2024-03-06T15:57:02.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:02.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:02.566234+0000) 2024-03-06T15:57:03.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:03.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:03.566479+0000) 2024-03-06T15:57:04.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:04.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:04.566729+0000) 2024-03-06T15:57:05.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:05.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:05.566974+0000) 2024-03-06T15:57:06.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:06.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:06.567211+0000) 2024-03-06T15:57:07.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:07.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:07.567414+0000) 2024-03-06T15:57:08.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:08.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:08.567705+0000) 2024-03-06T15:57:09.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:09.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:09.567966+0000) 2024-03-06T15:57:10.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:10.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:10.568203+0000) 2024-03-06T15:57:11.229 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:57:11.237 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:57:11.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:11.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:11.568410+0000) 2024-03-06T15:57:12.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:12.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:12.568664+0000) 2024-03-06T15:57:13.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:13.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:13.568905+0000) 2024-03-06T15:57:14.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:14.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:14.569145+0000) 2024-03-06T15:57:15.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:15.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:15.569358+0000) 2024-03-06T15:57:16.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:16.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:16.569693+0000) 2024-03-06T15:57:17.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:17.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:17.569969+0000) 2024-03-06T15:57:18.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:18.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:18.570247+0000) 2024-03-06T15:57:19.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:19.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:19.570572+0000) 2024-03-06T15:57:20.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:20.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:20.570847+0000) 2024-03-06T15:57:21.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:21.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:21.571123+0000) 2024-03-06T15:57:22.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:22.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:22.571396+0000) 2024-03-06T15:57:23.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:23.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:23.571703+0000) 2024-03-06T15:57:24.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:24.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:24.571942+0000) 2024-03-06T15:57:25.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:25.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:25.572169+0000) 2024-03-06T15:57:26.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:26.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:26.572406+0000) 2024-03-06T15:57:27.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:27.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:27.572705+0000) 2024-03-06T15:57:28.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:28.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:28.572943+0000) 2024-03-06T15:57:29.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:29.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:29.573174+0000) 2024-03-06T15:57:30.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:30.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:30.573383+0000) 2024-03-06T15:57:31.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:31.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:31.573683+0000) 2024-03-06T15:57:32.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:32.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:32.573917+0000) 2024-03-06T15:57:33.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:33.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:33.574127+0000) 2024-03-06T15:57:34.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:34.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:34.574362+0000) 2024-03-06T15:57:35.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:35.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:35.574575+0000) 2024-03-06T15:57:36.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:36.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:36.574802+0000) 2024-03-06T15:57:37.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:37.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:37.575012+0000) 2024-03-06T15:57:38.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:38.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:38.575234+0000) 2024-03-06T15:57:39.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:39.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:39.575459+0000) 2024-03-06T15:57:40.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:40.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:40.575705+0000) 2024-03-06T15:57:41.375 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:57:41.383 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:57:41.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:41.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:41.575901+0000) 2024-03-06T15:57:42.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:42.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:42.576118+0000) 2024-03-06T15:57:43.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:43.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:43.576338+0000) 2024-03-06T15:57:44.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:44.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:44.576602+0000) 2024-03-06T15:57:45.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:45.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:45.576786+0000) 2024-03-06T15:57:46.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:46.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:46.576960+0000) 2024-03-06T15:57:47.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:47.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:47.577143+0000) 2024-03-06T15:57:48.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:48.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:48.577331+0000) 2024-03-06T15:57:49.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:49.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:49.577610+0000) 2024-03-06T15:57:50.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:50.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:50.577808+0000) 2024-03-06T15:57:51.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:51.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:51.577999+0000) 2024-03-06T15:57:52.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:52.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:52.578244+0000) 2024-03-06T15:57:53.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:53.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:53.578468+0000) 2024-03-06T15:57:54.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:54.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:54.578698+0000) 2024-03-06T15:57:55.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:55.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:55.578924+0000) 2024-03-06T15:57:56.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:56.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:56.579123+0000) 2024-03-06T15:57:57.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:57.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:57.579364+0000) 2024-03-06T15:57:58.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:58.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:58.579636+0000) 2024-03-06T15:57:59.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:57:59.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:57:59.579875+0000) 2024-03-06T15:58:00.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:00.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:00.580055+0000) 2024-03-06T15:58:01.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:01.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:01.580279+0000) 2024-03-06T15:58:02.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:02.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:02.580511+0000) 2024-03-06T15:58:03.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:03.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:03.580791+0000) 2024-03-06T15:58:04.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:04.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:04.580974+0000) 2024-03-06T15:58:05.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:05.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:05.581149+0000) 2024-03-06T15:58:06.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:06.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:06.581335+0000) 2024-03-06T15:58:07.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:07.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:07.581605+0000) 2024-03-06T15:58:08.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:08.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:08.581853+0000) 2024-03-06T15:58:09.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:09.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:09.582131+0000) 2024-03-06T15:58:10.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:10.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:10.582405+0000) 2024-03-06T15:58:11.571 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:58:11.580 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:58:11.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:11.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:11.582642+0000) 2024-03-06T15:58:12.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:12.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:12.582927+0000) 2024-03-06T15:58:13.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:13.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:13.583130+0000) 2024-03-06T15:58:14.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:14.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:14.583352+0000) 2024-03-06T15:58:15.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:15.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:15.583641+0000) 2024-03-06T15:58:16.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:16.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:16.583859+0000) 2024-03-06T15:58:17.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:17.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:17.584090+0000) 2024-03-06T15:58:18.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:18.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:18.584337+0000) 2024-03-06T15:58:19.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:19.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:19.584665+0000) 2024-03-06T15:58:20.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:20.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:20.584934+0000) 2024-03-06T15:58:21.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:21.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:21.585208+0000) 2024-03-06T15:58:22.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:22.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:22.585491+0000) 2024-03-06T15:58:23.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:23.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:23.585817+0000) 2024-03-06T15:58:24.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:24.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:24.586062+0000) 2024-03-06T15:58:25.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:25.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:25.586386+0000) 2024-03-06T15:58:26.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:26.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:26.586693+0000) 2024-03-06T15:58:27.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:27.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:27.586900+0000) 2024-03-06T15:58:28.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:28.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:28.587131+0000) 2024-03-06T15:58:29.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:29.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:29.587358+0000) 2024-03-06T15:58:30.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:30.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:30.587596+0000) 2024-03-06T15:58:31.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:31.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:31.587830+0000) 2024-03-06T15:58:32.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:32.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:32.588080+0000) 2024-03-06T15:58:33.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:33.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:33.588328+0000) 2024-03-06T15:58:34.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:34.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:34.588586+0000) 2024-03-06T15:58:35.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:35.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:35.588822+0000) 2024-03-06T15:58:36.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:36.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:36.589062+0000) 2024-03-06T15:58:37.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:37.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:37.589313+0000) 2024-03-06T15:58:38.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:38.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:38.589598+0000) 2024-03-06T15:58:39.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:39.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:39.589850+0000) 2024-03-06T15:58:40.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:40.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:40.590095+0000) 2024-03-06T15:58:41.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:41.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:41.590331+0000) 2024-03-06T15:58:41.767 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:58:41.775 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:58:42.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:42.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:42.590630+0000) 2024-03-06T15:58:43.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:43.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:43.590869+0000) 2024-03-06T15:58:44.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:44.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:44.591113+0000) 2024-03-06T15:58:45.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:45.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:45.591358+0000) 2024-03-06T15:58:46.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:46.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:46.591636+0000) 2024-03-06T15:58:47.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:47.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:47.591902+0000) 2024-03-06T15:58:48.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:48.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:48.592126+0000) 2024-03-06T15:58:49.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:49.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:49.592362+0000) 2024-03-06T15:58:50.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:50.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:50.592606+0000) 2024-03-06T15:58:51.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:51.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:51.592828+0000) 2024-03-06T15:58:52.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:52.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:52.593098+0000) 2024-03-06T15:58:53.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:53.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:53.593293+0000) 2024-03-06T15:58:54.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:54.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:54.593577+0000) 2024-03-06T15:58:55.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:55.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:55.593806+0000) 2024-03-06T15:58:56.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:56.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:56.594051+0000) 2024-03-06T15:58:57.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:57.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:57.594288+0000) 2024-03-06T15:58:58.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:58.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:58.594508+0000) 2024-03-06T15:58:59.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:58:59.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:58:59.594746+0000) 2024-03-06T15:59:00.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:00.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:00.594955+0000) 2024-03-06T15:59:01.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:01.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:01.595202+0000) 2024-03-06T15:59:02.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:02.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:02.595443+0000) 2024-03-06T15:59:03.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:03.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:03.595679+0000) 2024-03-06T15:59:04.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:04.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:04.595906+0000) 2024-03-06T15:59:05.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:05.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:05.596098+0000) 2024-03-06T15:59:06.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:06.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:06.596319+0000) 2024-03-06T15:59:07.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:07.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:07.596608+0000) 2024-03-06T15:59:08.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:08.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:08.596819+0000) 2024-03-06T15:59:09.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:09.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:09.597080+0000) 2024-03-06T15:59:10.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:10.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:10.597305+0000) 2024-03-06T15:59:11.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:11.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:11.597581+0000) 2024-03-06T15:59:11.997 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:59:12.005 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:59:12.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:12.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:12.597847+0000) 2024-03-06T15:59:13.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:13.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:13.598061+0000) 2024-03-06T15:59:14.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:14.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:14.598297+0000) 2024-03-06T15:59:15.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:15.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:15.598481+0000) 2024-03-06T15:59:16.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:16.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:16.598707+0000) 2024-03-06T15:59:17.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:17.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:17.598907+0000) 2024-03-06T15:59:18.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:18.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:18.599078+0000) 2024-03-06T15:59:19.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:19.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:19.599359+0000) 2024-03-06T15:59:20.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:20.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:20.599635+0000) 2024-03-06T15:59:21.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:21.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:21.599833+0000) 2024-03-06T15:59:22.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:22.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:22.600093+0000) 2024-03-06T15:59:23.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:23.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:23.600308+0000) 2024-03-06T15:59:24.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:24.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:24.600507+0000) 2024-03-06T15:59:25.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:25.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:25.600811+0000) 2024-03-06T15:59:26.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:26.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:26.601065+0000) 2024-03-06T15:59:27.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:27.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:27.601315+0000) 2024-03-06T15:59:28.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:28.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:28.601585+0000) 2024-03-06T15:59:29.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:29.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:29.601796+0000) 2024-03-06T15:59:30.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:30.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:30.601986+0000) 2024-03-06T15:59:31.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:31.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:31.602176+0000) 2024-03-06T15:59:32.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:32.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:32.602424+0000) 2024-03-06T15:59:33.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:33.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:33.602655+0000) 2024-03-06T15:59:34.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:34.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:34.602843+0000) 2024-03-06T15:59:35.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:35.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:35.603080+0000) 2024-03-06T15:59:36.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:36.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:36.603306+0000) 2024-03-06T15:59:37.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:37.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:37.603576+0000) 2024-03-06T15:59:38.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:38.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:38.603813+0000) 2024-03-06T15:59:39.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:39.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:39.604046+0000) 2024-03-06T15:59:40.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:40.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:40.604238+0000) 2024-03-06T15:59:41.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:41.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:41.604517+0000) 2024-03-06T15:59:42.219 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:59:42.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T15:59:42.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:42.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:42.604804+0000) 2024-03-06T15:59:43.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:43.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:43.605005+0000) 2024-03-06T15:59:44.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:44.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:44.605208+0000) 2024-03-06T15:59:45.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:45.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:45.605437+0000) 2024-03-06T15:59:46.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:46.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:46.605672+0000) 2024-03-06T15:59:47.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:47.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:47.605894+0000) 2024-03-06T15:59:48.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:48.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:48.606093+0000) 2024-03-06T15:59:49.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:49.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:49.606345+0000) 2024-03-06T15:59:50.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:50.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:50.606576+0000) 2024-03-06T15:59:51.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:51.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:51.606797+0000) 2024-03-06T15:59:52.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:52.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:52.607044+0000) 2024-03-06T15:59:53.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:53.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:53.607263+0000) 2024-03-06T15:59:54.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:54.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:54.607482+0000) 2024-03-06T15:59:55.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:55.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:55.607744+0000) 2024-03-06T15:59:56.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:56.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:56.607961+0000) 2024-03-06T15:59:57.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:57.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:57.608216+0000) 2024-03-06T15:59:58.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:58.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:58.608440+0000) 2024-03-06T15:59:59.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T15:59:59.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T14:59:59.608679+0000) 2024-03-06T16:00:00.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:00.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:00.608910+0000) 2024-03-06T16:00:01.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:01.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:01.609159+0000) 2024-03-06T16:00:02.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:02.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:02.609350+0000) 2024-03-06T16:00:03.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:03.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:03.609602+0000) 2024-03-06T16:00:04.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:04.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:04.609842+0000) 2024-03-06T16:00:05.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:05.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:05.610108+0000) 2024-03-06T16:00:06.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:06.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:06.610385+0000) 2024-03-06T16:00:07.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:07.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:07.610745+0000) 2024-03-06T16:00:08.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:08.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:08.611029+0000) 2024-03-06T16:00:09.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:09.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:09.611312+0000) 2024-03-06T16:00:10.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:10.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:10.611616+0000) 2024-03-06T16:00:11.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:11.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:11.611902+0000) 2024-03-06T16:00:12.414 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:00:12.422 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:00:12.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:12.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:12.612121+0000) 2024-03-06T16:00:13.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:13.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:13.612387+0000) 2024-03-06T16:00:14.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:14.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:14.612639+0000) 2024-03-06T16:00:15.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:15.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:15.612865+0000) 2024-03-06T16:00:16.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:16.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:16.613088+0000) 2024-03-06T16:00:17.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:17.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:17.613282+0000) 2024-03-06T16:00:18.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:18.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:18.613588+0000) 2024-03-06T16:00:19.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:19.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:19.613843+0000) 2024-03-06T16:00:20.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:20.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:20.614126+0000) 2024-03-06T16:00:21.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:21.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:21.614375+0000) 2024-03-06T16:00:22.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:22.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:22.614580+0000) 2024-03-06T16:00:23.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:23.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:23.614779+0000) 2024-03-06T16:00:24.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:24.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:24.615004+0000) 2024-03-06T16:00:25.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:25.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:25.615228+0000) 2024-03-06T16:00:26.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:26.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:26.615452+0000) 2024-03-06T16:00:27.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:27.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:27.615672+0000) 2024-03-06T16:00:28.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:28.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:28.615903+0000) 2024-03-06T16:00:29.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:29.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:29.616100+0000) 2024-03-06T16:00:30.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:30.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:30.616337+0000) 2024-03-06T16:00:31.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:31.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:31.616595+0000) 2024-03-06T16:00:32.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:32.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:32.616779+0000) 2024-03-06T16:00:33.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:33.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:33.617003+0000) 2024-03-06T16:00:34.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:34.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:34.617232+0000) 2024-03-06T16:00:35.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:35.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:35.617453+0000) 2024-03-06T16:00:36.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:36.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:36.617685+0000) 2024-03-06T16:00:37.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:37.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:37.617881+0000) 2024-03-06T16:00:38.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:38.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:38.618092+0000) 2024-03-06T16:00:39.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:39.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:39.618342+0000) 2024-03-06T16:00:40.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:40.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:40.618611+0000) 2024-03-06T16:00:41.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:41.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:41.618843+0000) 2024-03-06T16:00:42.611 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:00:42.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:42.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:42.619078+0000) 2024-03-06T16:00:42.619 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:00:43.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:43.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:43.619265+0000) 2024-03-06T16:00:44.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:44.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:44.619489+0000) 2024-03-06T16:00:45.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:45.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:45.619716+0000) 2024-03-06T16:00:46.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:46.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:46.619901+0000) 2024-03-06T16:00:47.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:47.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:47.620076+0000) 2024-03-06T16:00:48.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:48.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:48.620305+0000) 2024-03-06T16:00:49.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:49.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:49.620565+0000) 2024-03-06T16:00:50.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:50.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:50.620803+0000) 2024-03-06T16:00:51.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:51.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:51.621026+0000) 2024-03-06T16:00:52.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:52.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:52.621257+0000) 2024-03-06T16:00:53.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:53.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:53.621486+0000) 2024-03-06T16:00:54.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:54.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:54.621694+0000) 2024-03-06T16:00:55.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:55.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:55.621888+0000) 2024-03-06T16:00:56.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:56.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:56.622089+0000) 2024-03-06T16:00:57.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:57.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:57.622334+0000) 2024-03-06T16:00:58.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:58.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:58.622626+0000) 2024-03-06T16:00:59.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:00:59.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:00:59.622860+0000) 2024-03-06T16:01:00.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:00.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:00.623134+0000) 2024-03-06T16:01:01.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:01.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:01.623312+0000) 2024-03-06T16:01:02.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:02.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:02.623565+0000) 2024-03-06T16:01:03.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:03.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:03.623814+0000) 2024-03-06T16:01:04.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:04.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:04.624003+0000) 2024-03-06T16:01:05.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:05.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:05.624213+0000) 2024-03-06T16:01:06.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:06.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:06.624446+0000) 2024-03-06T16:01:07.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:07.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:07.624683+0000) 2024-03-06T16:01:08.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:08.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:08.624910+0000) 2024-03-06T16:01:09.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:09.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:09.625184+0000) 2024-03-06T16:01:10.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:10.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:10.625409+0000) 2024-03-06T16:01:11.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:11.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:11.625644+0000) 2024-03-06T16:01:12.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:12.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:12.625878+0000) 2024-03-06T16:01:12.823 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:01:12.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:01:13.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:13.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:13.626077+0000) 2024-03-06T16:01:14.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:14.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:14.626274+0000) 2024-03-06T16:01:15.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:15.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:15.626495+0000) 2024-03-06T16:01:16.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:16.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:16.626723+0000) 2024-03-06T16:01:17.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:17.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:17.626954+0000) 2024-03-06T16:01:18.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:18.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:18.627186+0000) 2024-03-06T16:01:19.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:19.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:19.627423+0000) 2024-03-06T16:01:20.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:20.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:20.627647+0000) 2024-03-06T16:01:21.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:21.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:21.627886+0000) 2024-03-06T16:01:22.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:22.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:22.628116+0000) 2024-03-06T16:01:23.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:23.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:23.628340+0000) 2024-03-06T16:01:24.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:24.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:24.628613+0000) 2024-03-06T16:01:25.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:25.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:25.628850+0000) 2024-03-06T16:01:26.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:26.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:26.629090+0000) 2024-03-06T16:01:27.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:27.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:27.629335+0000) 2024-03-06T16:01:28.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:28.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:28.629616+0000) 2024-03-06T16:01:29.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:29.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:29.629862+0000) 2024-03-06T16:01:30.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:30.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:30.630138+0000) 2024-03-06T16:01:31.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:31.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:31.630414+0000) 2024-03-06T16:01:32.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:32.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:32.630699+0000) 2024-03-06T16:01:33.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:33.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:33.630983+0000) 2024-03-06T16:01:34.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:34.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:34.631264+0000) 2024-03-06T16:01:35.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:35.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:35.631574+0000) 2024-03-06T16:01:36.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:36.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:36.631776+0000) 2024-03-06T16:01:37.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:37.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:37.631999+0000) 2024-03-06T16:01:38.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:38.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:38.632277+0000) 2024-03-06T16:01:39.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:39.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:39.632576+0000) 2024-03-06T16:01:40.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:40.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:40.632823+0000) 2024-03-06T16:01:41.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:41.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:41.633081+0000) 2024-03-06T16:01:42.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:42.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:42.633302+0000) 2024-03-06T16:01:43.036 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:01:43.044 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:01:43.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:43.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:43.633526+0000) 2024-03-06T16:01:44.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:44.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:44.633796+0000) 2024-03-06T16:01:45.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:45.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:45.634018+0000) 2024-03-06T16:01:46.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:46.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:46.634240+0000) 2024-03-06T16:01:47.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:47.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:47.634467+0000) 2024-03-06T16:01:48.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:48.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:48.634702+0000) 2024-03-06T16:01:49.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:49.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:49.634899+0000) 2024-03-06T16:01:50.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:50.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:50.635144+0000) 2024-03-06T16:01:51.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:51.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:51.635383+0000) 2024-03-06T16:01:52.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:52.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:52.635619+0000) 2024-03-06T16:01:53.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:53.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:53.635870+0000) 2024-03-06T16:01:54.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:54.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:54.636107+0000) 2024-03-06T16:01:55.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:55.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:55.636344+0000) 2024-03-06T16:01:56.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:56.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:56.636609+0000) 2024-03-06T16:01:57.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:57.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:57.636817+0000) 2024-03-06T16:01:58.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:58.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:58.637025+0000) 2024-03-06T16:01:59.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:01:59.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:01:59.637242+0000) 2024-03-06T16:02:00.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:00.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:00.637480+0000) 2024-03-06T16:02:01.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:01.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:01.637710+0000) 2024-03-06T16:02:02.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:02.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:02.637934+0000) 2024-03-06T16:02:03.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:03.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:03.638162+0000) 2024-03-06T16:02:04.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:04.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:04.638410+0000) 2024-03-06T16:02:05.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:05.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:05.638650+0000) 2024-03-06T16:02:06.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:06.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:06.638876+0000) 2024-03-06T16:02:07.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:07.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:07.639134+0000) 2024-03-06T16:02:08.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:08.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:08.639384+0000) 2024-03-06T16:02:09.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:09.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:09.639645+0000) 2024-03-06T16:02:10.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:10.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:10.639888+0000) 2024-03-06T16:02:11.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:11.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:11.640124+0000) 2024-03-06T16:02:12.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:12.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:12.640350+0000) 2024-03-06T16:02:13.257 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:02:13.265 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:02:13.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:13.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:13.640616+0000) 2024-03-06T16:02:14.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:14.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:14.640848+0000) 2024-03-06T16:02:15.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:15.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:15.641067+0000) 2024-03-06T16:02:16.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:16.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:16.641288+0000) 2024-03-06T16:02:17.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:17.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:17.641513+0000) 2024-03-06T16:02:18.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:18.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:18.641794+0000) 2024-03-06T16:02:19.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:19.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:19.642069+0000) 2024-03-06T16:02:20.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:20.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:20.642288+0000) 2024-03-06T16:02:21.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:21.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:21.642570+0000) 2024-03-06T16:02:22.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:22.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:22.642842+0000) 2024-03-06T16:02:23.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:23.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:23.643091+0000) 2024-03-06T16:02:24.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:24.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:24.643315+0000) 2024-03-06T16:02:25.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:25.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:25.643574+0000) 2024-03-06T16:02:26.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:26.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:26.643802+0000) 2024-03-06T16:02:27.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:27.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:27.644024+0000) 2024-03-06T16:02:28.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:28.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:28.644281+0000) 2024-03-06T16:02:29.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:29.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:29.644488+0000) 2024-03-06T16:02:30.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:30.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:30.644711+0000) 2024-03-06T16:02:31.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:31.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:31.644920+0000) 2024-03-06T16:02:32.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:32.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:32.645178+0000) 2024-03-06T16:02:33.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:33.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:33.645375+0000) 2024-03-06T16:02:34.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:34.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:34.645641+0000) 2024-03-06T16:02:35.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:35.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:35.645870+0000) 2024-03-06T16:02:36.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:36.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:36.646140+0000) 2024-03-06T16:02:37.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:37.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:37.646377+0000) 2024-03-06T16:02:38.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:38.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:38.646691+0000) 2024-03-06T16:02:39.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:39.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:39.646914+0000) 2024-03-06T16:02:40.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:40.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:40.647141+0000) 2024-03-06T16:02:41.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:41.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:41.647371+0000) 2024-03-06T16:02:42.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:42.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:42.647653+0000) 2024-03-06T16:02:43.393 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:02:43.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:02:43.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:43.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:43.647900+0000) 2024-03-06T16:02:44.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:44.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:44.648145+0000) 2024-03-06T16:02:45.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:45.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:45.648358+0000) 2024-03-06T16:02:46.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:46.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:46.648588+0000) 2024-03-06T16:02:47.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:47.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:47.648815+0000) 2024-03-06T16:02:48.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:48.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:48.648995+0000) 2024-03-06T16:02:49.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:49.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:49.649215+0000) 2024-03-06T16:02:50.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:50.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:50.649470+0000) 2024-03-06T16:02:51.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:51.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:51.649719+0000) 2024-03-06T16:02:52.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:52.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:52.649983+0000) 2024-03-06T16:02:53.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:53.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:53.650182+0000) 2024-03-06T16:02:54.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:54.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:54.650392+0000) 2024-03-06T16:02:55.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:55.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:55.650646+0000) 2024-03-06T16:02:56.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:56.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:56.650870+0000) 2024-03-06T16:02:57.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:57.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:57.651084+0000) 2024-03-06T16:02:58.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:58.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:58.651329+0000) 2024-03-06T16:02:59.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:02:59.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:02:59.651591+0000) 2024-03-06T16:03:00.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:00.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:00.651803+0000) 2024-03-06T16:03:01.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:01.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:01.652065+0000) 2024-03-06T16:03:02.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:02.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:02.652267+0000) 2024-03-06T16:03:03.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:03.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:03.652499+0000) 2024-03-06T16:03:04.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:04.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:04.652781+0000) 2024-03-06T16:03:05.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:05.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:05.653003+0000) 2024-03-06T16:03:06.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:06.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:06.653187+0000) 2024-03-06T16:03:07.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:07.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:07.653421+0000) 2024-03-06T16:03:08.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:08.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:08.653659+0000) 2024-03-06T16:03:09.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:09.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:09.653873+0000) 2024-03-06T16:03:10.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:10.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:10.654111+0000) 2024-03-06T16:03:11.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:11.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:11.654313+0000) 2024-03-06T16:03:12.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:12.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:12.654574+0000) 2024-03-06T16:03:13.599 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:03:13.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:03:13.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:13.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:13.654727+0000) 2024-03-06T16:03:14.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:14.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:14.654911+0000) 2024-03-06T16:03:15.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:15.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:15.655132+0000) 2024-03-06T16:03:16.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:16.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:16.655319+0000) 2024-03-06T16:03:17.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:17.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:17.655500+0000) 2024-03-06T16:03:18.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:18.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:18.655782+0000) 2024-03-06T16:03:19.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:19.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:19.655994+0000) 2024-03-06T16:03:20.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:20.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:20.656191+0000) 2024-03-06T16:03:21.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:21.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:21.656392+0000) 2024-03-06T16:03:22.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:22.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:22.656664+0000) 2024-03-06T16:03:23.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:23.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:23.656861+0000) 2024-03-06T16:03:24.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:24.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:24.657144+0000) 2024-03-06T16:03:25.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:25.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:25.657371+0000) 2024-03-06T16:03:26.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:26.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:26.657638+0000) 2024-03-06T16:03:27.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:27.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:27.657860+0000) 2024-03-06T16:03:28.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:28.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:28.658094+0000) 2024-03-06T16:03:29.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:29.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:29.658326+0000) 2024-03-06T16:03:30.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:30.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:30.658592+0000) 2024-03-06T16:03:31.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:31.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:31.658809+0000) 2024-03-06T16:03:32.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:32.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:32.659028+0000) 2024-03-06T16:03:33.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:33.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:33.659259+0000) 2024-03-06T16:03:34.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:34.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:34.659425+0000) 2024-03-06T16:03:35.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:35.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:35.659665+0000) 2024-03-06T16:03:36.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:36.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:36.659843+0000) 2024-03-06T16:03:37.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:37.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:37.660062+0000) 2024-03-06T16:03:38.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:38.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:38.660275+0000) 2024-03-06T16:03:39.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:39.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:39.660512+0000) 2024-03-06T16:03:40.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:40.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:40.660773+0000) 2024-03-06T16:03:41.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:41.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:41.660991+0000) 2024-03-06T16:03:42.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:42.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:42.661220+0000) 2024-03-06T16:03:43.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:43.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:43.661445+0000) 2024-03-06T16:03:43.786 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:03:43.794 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:03:44.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:44.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:44.661665+0000) 2024-03-06T16:03:45.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:45.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:45.661874+0000) 2024-03-06T16:03:46.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:46.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:46.662119+0000) 2024-03-06T16:03:47.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:47.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:47.662336+0000) 2024-03-06T16:03:48.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:48.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:48.662612+0000) 2024-03-06T16:03:49.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:49.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:49.662806+0000) 2024-03-06T16:03:50.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:50.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:50.662990+0000) 2024-03-06T16:03:51.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:51.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:51.663208+0000) 2024-03-06T16:03:52.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:52.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:52.663425+0000) 2024-03-06T16:03:53.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:53.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:53.663686+0000) 2024-03-06T16:03:54.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:54.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:54.663893+0000) 2024-03-06T16:03:55.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:55.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:55.664120+0000) 2024-03-06T16:03:56.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:56.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:56.664348+0000) 2024-03-06T16:03:57.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:57.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:57.664627+0000) 2024-03-06T16:03:58.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:58.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:58.664868+0000) 2024-03-06T16:03:59.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:03:59.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:03:59.665104+0000) 2024-03-06T16:04:00.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:00.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:00.665331+0000) 2024-03-06T16:04:01.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:01.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:01.665585+0000) 2024-03-06T16:04:02.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:02.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:02.665813+0000) 2024-03-06T16:04:03.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:03.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:03.666035+0000) 2024-03-06T16:04:04.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:04.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:04.666265+0000) 2024-03-06T16:04:05.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:05.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:05.666490+0000) 2024-03-06T16:04:06.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:06.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:06.666682+0000) 2024-03-06T16:04:07.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:07.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:07.666928+0000) 2024-03-06T16:04:08.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:08.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:08.667140+0000) 2024-03-06T16:04:09.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:09.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:09.667373+0000) 2024-03-06T16:04:10.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:10.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:10.667641+0000) 2024-03-06T16:04:11.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:11.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:11.667880+0000) 2024-03-06T16:04:12.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:12.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:12.668134+0000) 2024-03-06T16:04:13.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:13.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:13.668394+0000) 2024-03-06T16:04:13.992 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:04:14.000 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:04:14.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:14.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:14.668648+0000) 2024-03-06T16:04:15.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:15.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:15.668848+0000) 2024-03-06T16:04:16.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:16.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:16.669056+0000) 2024-03-06T16:04:17.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:17.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:17.669356+0000) 2024-03-06T16:04:18.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:18.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:18.669590+0000) 2024-03-06T16:04:19.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:19.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:19.669845+0000) 2024-03-06T16:04:20.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:20.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:20.670047+0000) 2024-03-06T16:04:21.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:21.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:21.670282+0000) 2024-03-06T16:04:22.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:22.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:22.670482+0000) 2024-03-06T16:04:23.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:23.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:23.670747+0000) 2024-03-06T16:04:24.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:24.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:24.670970+0000) 2024-03-06T16:04:25.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:25.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:25.671144+0000) 2024-03-06T16:04:26.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:26.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:26.671368+0000) 2024-03-06T16:04:27.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:27.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:27.671598+0000) 2024-03-06T16:04:28.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:28.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:28.671809+0000) 2024-03-06T16:04:29.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:29.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:29.672061+0000) 2024-03-06T16:04:30.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:30.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:30.672282+0000) 2024-03-06T16:04:31.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:31.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:31.672508+0000) 2024-03-06T16:04:32.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:32.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:32.672746+0000) 2024-03-06T16:04:33.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:33.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:33.672969+0000) 2024-03-06T16:04:34.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:34.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:34.673206+0000) 2024-03-06T16:04:35.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:35.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:35.673386+0000) 2024-03-06T16:04:36.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:36.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:36.673675+0000) 2024-03-06T16:04:37.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:37.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:37.673935+0000) 2024-03-06T16:04:38.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:38.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:38.674178+0000) 2024-03-06T16:04:39.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:39.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:39.674371+0000) 2024-03-06T16:04:40.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:40.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:40.674634+0000) 2024-03-06T16:04:41.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:41.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:41.674847+0000) 2024-03-06T16:04:42.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:42.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:42.675071+0000) 2024-03-06T16:04:43.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:43.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:43.675308+0000) 2024-03-06T16:04:44.247 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:04:44.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:04:44.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:44.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:44.675496+0000) 2024-03-06T16:04:45.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:45.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:45.675737+0000) 2024-03-06T16:04:46.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:46.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:46.675958+0000) 2024-03-06T16:04:47.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:47.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:47.676182+0000) 2024-03-06T16:04:48.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:48.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:48.676386+0000) 2024-03-06T16:04:49.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:49.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:49.676660+0000) 2024-03-06T16:04:50.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:50.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:50.676885+0000) 2024-03-06T16:04:51.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:51.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:51.677140+0000) 2024-03-06T16:04:52.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:52.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:52.677418+0000) 2024-03-06T16:04:53.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:53.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:53.677703+0000) 2024-03-06T16:04:54.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:54.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:54.677979+0000) 2024-03-06T16:04:55.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:55.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:55.678183+0000) 2024-03-06T16:04:56.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:56.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:56.678413+0000) 2024-03-06T16:04:57.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:57.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:57.678650+0000) 2024-03-06T16:04:58.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:58.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:58.678880+0000) 2024-03-06T16:04:59.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:04:59.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:04:59.679166+0000) 2024-03-06T16:05:00.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:00.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:00.679430+0000) 2024-03-06T16:05:01.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:01.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:01.679669+0000) 2024-03-06T16:05:02.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:02.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:02.679885+0000) 2024-03-06T16:05:03.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:03.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:03.680116+0000) 2024-03-06T16:05:04.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:04.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:04.680392+0000) 2024-03-06T16:05:05.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:05.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:05.680686+0000) 2024-03-06T16:05:06.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:06.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:06.680965+0000) 2024-03-06T16:05:07.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:07.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:07.681240+0000) 2024-03-06T16:05:08.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:08.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:08.681582+0000) 2024-03-06T16:05:09.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:09.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:09.681890+0000) 2024-03-06T16:05:10.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:10.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:10.682156+0000) 2024-03-06T16:05:11.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:11.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:11.682476+0000) 2024-03-06T16:05:12.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:12.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:12.682754+0000) 2024-03-06T16:05:13.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:13.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:13.683049+0000) 2024-03-06T16:05:14.401 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:05:14.409 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:05:14.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:14.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:14.683330+0000) 2024-03-06T16:05:15.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:15.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:15.683560+0000) 2024-03-06T16:05:16.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:16.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:16.683796+0000) 2024-03-06T16:05:17.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:17.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:17.683997+0000) 2024-03-06T16:05:18.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:18.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:18.684243+0000) 2024-03-06T16:05:19.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:19.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:19.684442+0000) 2024-03-06T16:05:20.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:20.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:20.684681+0000) 2024-03-06T16:05:21.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:21.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:21.684934+0000) 2024-03-06T16:05:22.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:22.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:22.685145+0000) 2024-03-06T16:05:23.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:23.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:23.685395+0000) 2024-03-06T16:05:24.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:24.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:24.685655+0000) 2024-03-06T16:05:25.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:25.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:25.685851+0000) 2024-03-06T16:05:26.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:26.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:26.686081+0000) 2024-03-06T16:05:27.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:27.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:27.686266+0000) 2024-03-06T16:05:28.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:28.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:28.686514+0000) 2024-03-06T16:05:29.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:29.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:29.686748+0000) 2024-03-06T16:05:30.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:30.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:30.686975+0000) 2024-03-06T16:05:31.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:31.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:31.687202+0000) 2024-03-06T16:05:32.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:32.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:32.687446+0000) 2024-03-06T16:05:33.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:33.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:33.687745+0000) 2024-03-06T16:05:34.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:34.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:34.687957+0000) 2024-03-06T16:05:35.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:35.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:35.688156+0000) 2024-03-06T16:05:36.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:36.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:36.688401+0000) 2024-03-06T16:05:37.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:37.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:37.688653+0000) 2024-03-06T16:05:38.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:38.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:38.688898+0000) 2024-03-06T16:05:39.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:39.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:39.689130+0000) 2024-03-06T16:05:40.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:40.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:40.689361+0000) 2024-03-06T16:05:41.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:41.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:41.689625+0000) 2024-03-06T16:05:42.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:42.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:42.689848+0000) 2024-03-06T16:05:43.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:43.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:43.690078+0000) 2024-03-06T16:05:44.606 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:05:44.614 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:05:44.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:44.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:44.690327+0000) 2024-03-06T16:05:45.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:45.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:45.690511+0000) 2024-03-06T16:05:46.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:46.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:46.690799+0000) 2024-03-06T16:05:47.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:47.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:47.691043+0000) 2024-03-06T16:05:48.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:48.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:48.691330+0000) 2024-03-06T16:05:49.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:49.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:49.691646+0000) 2024-03-06T16:05:50.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:50.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:50.691915+0000) 2024-03-06T16:05:51.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:51.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:51.692153+0000) 2024-03-06T16:05:52.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:52.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:52.692428+0000) 2024-03-06T16:05:53.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:53.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:53.692675+0000) 2024-03-06T16:05:54.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:54.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:54.692902+0000) 2024-03-06T16:05:55.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:55.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:55.693106+0000) 2024-03-06T16:05:56.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:56.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:56.693348+0000) 2024-03-06T16:05:57.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:57.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:57.693636+0000) 2024-03-06T16:05:58.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:58.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:58.693859+0000) 2024-03-06T16:05:59.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:05:59.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:05:59.694141+0000) 2024-03-06T16:06:00.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:00.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:00.694346+0000) 2024-03-06T16:06:01.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:01.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:01.694569+0000) 2024-03-06T16:06:02.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:02.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:02.694809+0000) 2024-03-06T16:06:03.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:03.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:03.694995+0000) 2024-03-06T16:06:04.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:04.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:04.695224+0000) 2024-03-06T16:06:05.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:05.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:05.695453+0000) 2024-03-06T16:06:06.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:06.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:06.695679+0000) 2024-03-06T16:06:07.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:07.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:07.695897+0000) 2024-03-06T16:06:08.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:08.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:08.696125+0000) 2024-03-06T16:06:09.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:09.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:09.696327+0000) 2024-03-06T16:06:10.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:10.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:10.696595+0000) 2024-03-06T16:06:11.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:11.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:11.696789+0000) 2024-03-06T16:06:12.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:12.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:12.696975+0000) 2024-03-06T16:06:13.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:13.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:13.697225+0000) 2024-03-06T16:06:14.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:14.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:14.697457+0000) 2024-03-06T16:06:14.793 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:06:14.803 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:06:15.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:15.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:15.697700+0000) 2024-03-06T16:06:16.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:16.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:16.697927+0000) 2024-03-06T16:06:17.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:17.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:17.698118+0000) 2024-03-06T16:06:18.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:18.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:18.698360+0000) 2024-03-06T16:06:19.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:19.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:19.698654+0000) 2024-03-06T16:06:20.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:20.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:20.698902+0000) 2024-03-06T16:06:21.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:21.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:21.699108+0000) 2024-03-06T16:06:22.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:22.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:22.699339+0000) 2024-03-06T16:06:23.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:23.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:23.699592+0000) 2024-03-06T16:06:24.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:24.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:24.699870+0000) 2024-03-06T16:06:25.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:25.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:25.700137+0000) 2024-03-06T16:06:26.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:26.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:26.700411+0000) 2024-03-06T16:06:27.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:27.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:27.700692+0000) 2024-03-06T16:06:28.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:28.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:28.700938+0000) 2024-03-06T16:06:29.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:29.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:29.701181+0000) 2024-03-06T16:06:30.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:30.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:30.701372+0000) 2024-03-06T16:06:31.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:31.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:31.701616+0000) 2024-03-06T16:06:32.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:32.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:32.701854+0000) 2024-03-06T16:06:33.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:33.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:33.702056+0000) 2024-03-06T16:06:34.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:34.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:34.702272+0000) 2024-03-06T16:06:35.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:35.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:35.702581+0000) 2024-03-06T16:06:36.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:36.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:36.702847+0000) 2024-03-06T16:06:37.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:37.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:37.703123+0000) 2024-03-06T16:06:38.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:38.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:38.703337+0000) 2024-03-06T16:06:39.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:39.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:39.703635+0000) 2024-03-06T16:06:40.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:40.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:40.703843+0000) 2024-03-06T16:06:41.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:41.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:41.704076+0000) 2024-03-06T16:06:42.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:42.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:42.704347+0000) 2024-03-06T16:06:43.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:43.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:43.704589+0000) 2024-03-06T16:06:44.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:44.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:44.704824+0000) 2024-03-06T16:06:45.006 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:06:45.015 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:06:45.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:45.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:45.705055+0000) 2024-03-06T16:06:46.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:46.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:46.705276+0000) 2024-03-06T16:06:47.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:47.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:47.705586+0000) 2024-03-06T16:06:48.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:48.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:48.705876+0000) 2024-03-06T16:06:49.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:49.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:49.706170+0000) 2024-03-06T16:06:50.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:50.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:50.706453+0000) 2024-03-06T16:06:51.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:51.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:51.706736+0000) 2024-03-06T16:06:52.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:52.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:52.706942+0000) 2024-03-06T16:06:53.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:53.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:53.707173+0000) 2024-03-06T16:06:54.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:54.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:54.707407+0000) 2024-03-06T16:06:55.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:55.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:55.707642+0000) 2024-03-06T16:06:56.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:56.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:56.707875+0000) 2024-03-06T16:06:57.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:57.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:57.708104+0000) 2024-03-06T16:06:58.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:58.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:58.708328+0000) 2024-03-06T16:06:59.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:06:59.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:06:59.708588+0000) 2024-03-06T16:07:00.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:00.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:00.708815+0000) 2024-03-06T16:07:01.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:01.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:01.709023+0000) 2024-03-06T16:07:02.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:02.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:02.709285+0000) 2024-03-06T16:07:03.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:03.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:03.709497+0000) 2024-03-06T16:07:04.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:04.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:04.709741+0000) 2024-03-06T16:07:05.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:05.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:05.710015+0000) 2024-03-06T16:07:06.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:06.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:06.710292+0000) 2024-03-06T16:07:07.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:07.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:07.710509+0000) 2024-03-06T16:07:08.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:08.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:08.710794+0000) 2024-03-06T16:07:09.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:09.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:09.711040+0000) 2024-03-06T16:07:10.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:10.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:10.711269+0000) 2024-03-06T16:07:11.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:11.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:11.711491+0000) 2024-03-06T16:07:12.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:12.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:12.711719+0000) 2024-03-06T16:07:13.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:13.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:13.711905+0000) 2024-03-06T16:07:14.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:14.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:14.712138+0000) 2024-03-06T16:07:15.194 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:07:15.202 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:07:15.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:15.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:15.712382+0000) 2024-03-06T16:07:16.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:16.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:16.712673+0000) 2024-03-06T16:07:17.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:17.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:17.712913+0000) 2024-03-06T16:07:18.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:18.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:18.713152+0000) 2024-03-06T16:07:19.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:19.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:19.713417+0000) 2024-03-06T16:07:20.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:20.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:20.713642+0000) 2024-03-06T16:07:21.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:21.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:21.713871+0000) 2024-03-06T16:07:22.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:22.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:22.714062+0000) 2024-03-06T16:07:23.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:23.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:23.714301+0000) 2024-03-06T16:07:24.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:24.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:24.714578+0000) 2024-03-06T16:07:25.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:25.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:25.714771+0000) 2024-03-06T16:07:26.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:26.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:26.715007+0000) 2024-03-06T16:07:27.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:27.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:27.715245+0000) 2024-03-06T16:07:28.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:28.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:28.715486+0000) 2024-03-06T16:07:29.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:29.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:29.715779+0000) 2024-03-06T16:07:30.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:30.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:30.715995+0000) 2024-03-06T16:07:31.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:31.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:31.716201+0000) 2024-03-06T16:07:32.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:32.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:32.716398+0000) 2024-03-06T16:07:33.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:33.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:33.716633+0000) 2024-03-06T16:07:34.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:34.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:34.716838+0000) 2024-03-06T16:07:35.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:35.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:35.717141+0000) 2024-03-06T16:07:36.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:36.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:36.717360+0000) 2024-03-06T16:07:37.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:37.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:37.717594+0000) 2024-03-06T16:07:38.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:38.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:38.717801+0000) 2024-03-06T16:07:39.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:39.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:39.718044+0000) 2024-03-06T16:07:40.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:40.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:40.718277+0000) 2024-03-06T16:07:41.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:41.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:41.718480+0000) 2024-03-06T16:07:42.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:42.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:42.718673+0000) 2024-03-06T16:07:43.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:43.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:43.718898+0000) 2024-03-06T16:07:44.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:44.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:44.719099+0000) 2024-03-06T16:07:45.407 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:07:45.416 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:07:45.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:45.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:45.719351+0000) 2024-03-06T16:07:46.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:46.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:46.719623+0000) 2024-03-06T16:07:47.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:47.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:47.719850+0000) 2024-03-06T16:07:48.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:48.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:48.720038+0000) 2024-03-06T16:07:49.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:49.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:49.720296+0000) 2024-03-06T16:07:50.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:50.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:50.720500+0000) 2024-03-06T16:07:51.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:51.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:51.720771+0000) 2024-03-06T16:07:52.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:52.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:52.720986+0000) 2024-03-06T16:07:53.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:53.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:53.721212+0000) 2024-03-06T16:07:54.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:54.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:54.721437+0000) 2024-03-06T16:07:55.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:55.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:55.721680+0000) 2024-03-06T16:07:56.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:56.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:56.721906+0000) 2024-03-06T16:07:57.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:57.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:57.722342+0000) 2024-03-06T16:07:58.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:58.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:58.722624+0000) 2024-03-06T16:07:59.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:07:59.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:07:59.722803+0000) 2024-03-06T16:08:00.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:00.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:00.723051+0000) 2024-03-06T16:08:01.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:01.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:01.723275+0000) 2024-03-06T16:08:02.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:02.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:02.723498+0000) 2024-03-06T16:08:03.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:03.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:03.723726+0000) 2024-03-06T16:08:04.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:04.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:04.723966+0000) 2024-03-06T16:08:05.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:05.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:05.724188+0000) 2024-03-06T16:08:06.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:06.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:06.724409+0000) 2024-03-06T16:08:07.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:07.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:07.724680+0000) 2024-03-06T16:08:08.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:08.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:08.724959+0000) 2024-03-06T16:08:09.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:09.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:09.725247+0000) 2024-03-06T16:08:10.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:10.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:10.725528+0000) 2024-03-06T16:08:11.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:11.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:11.725786+0000) 2024-03-06T16:08:12.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:12.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:12.726016+0000) 2024-03-06T16:08:13.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:13.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:13.726275+0000) 2024-03-06T16:08:14.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:14.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:14.726500+0000) 2024-03-06T16:08:15.612 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:08:15.620 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:08:15.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:15.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:15.726718+0000) 2024-03-06T16:08:16.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:16.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:16.726911+0000) 2024-03-06T16:08:17.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:17.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:17.727148+0000) 2024-03-06T16:08:18.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:18.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:18.727326+0000) 2024-03-06T16:08:19.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:19.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:19.727522+0000) 2024-03-06T16:08:20.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:20.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:20.727806+0000) 2024-03-06T16:08:21.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:21.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:21.728045+0000) 2024-03-06T16:08:22.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:22.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:22.728252+0000) 2024-03-06T16:08:23.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:23.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:23.728448+0000) 2024-03-06T16:08:24.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:24.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:24.728674+0000) 2024-03-06T16:08:25.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:25.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:25.728879+0000) 2024-03-06T16:08:26.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:26.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:26.729116+0000) 2024-03-06T16:08:27.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:27.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:27.729353+0000) 2024-03-06T16:08:28.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:28.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:28.729681+0000) 2024-03-06T16:08:29.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:29.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:29.729950+0000) 2024-03-06T16:08:30.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:30.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:30.730194+0000) 2024-03-06T16:08:31.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:31.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:31.730404+0000) 2024-03-06T16:08:32.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:32.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:32.730679+0000) 2024-03-06T16:08:33.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:33.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:33.730888+0000) 2024-03-06T16:08:34.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:34.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:34.731163+0000) 2024-03-06T16:08:35.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:35.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:35.731445+0000) 2024-03-06T16:08:36.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:36.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:36.731715+0000) 2024-03-06T16:08:37.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:37.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:37.731989+0000) 2024-03-06T16:08:38.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:38.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:38.732271+0000) 2024-03-06T16:08:39.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:39.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:39.732593+0000) 2024-03-06T16:08:40.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:40.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:40.732819+0000) 2024-03-06T16:08:41.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:41.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:41.733078+0000) 2024-03-06T16:08:42.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:42.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:42.733342+0000) 2024-03-06T16:08:43.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:43.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:43.733648+0000) 2024-03-06T16:08:44.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:44.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:44.733863+0000) 2024-03-06T16:08:45.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:45.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:45.734047+0000) 2024-03-06T16:08:45.819 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:08:45.826 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:08:46.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:46.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:46.734248+0000) 2024-03-06T16:08:47.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:47.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:47.734481+0000) 2024-03-06T16:08:48.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:48.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:48.734716+0000) 2024-03-06T16:08:49.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:49.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:49.734947+0000) 2024-03-06T16:08:50.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:50.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:50.735161+0000) 2024-03-06T16:08:51.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:51.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:51.735335+0000) 2024-03-06T16:08:52.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:52.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:52.735613+0000) 2024-03-06T16:08:53.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:53.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:53.735854+0000) 2024-03-06T16:08:54.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:54.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:54.736088+0000) 2024-03-06T16:08:55.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:55.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:55.736316+0000) 2024-03-06T16:08:56.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:56.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:56.736616+0000) 2024-03-06T16:08:57.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:57.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:57.736818+0000) 2024-03-06T16:08:58.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:58.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:58.737022+0000) 2024-03-06T16:08:59.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:08:59.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:08:59.737277+0000) 2024-03-06T16:09:00.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:00.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:00.737595+0000) 2024-03-06T16:09:01.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:01.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:01.737856+0000) 2024-03-06T16:09:02.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:02.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:02.738089+0000) 2024-03-06T16:09:03.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:03.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:03.738320+0000) 2024-03-06T16:09:04.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:04.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:04.738601+0000) 2024-03-06T16:09:05.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:05.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:05.738836+0000) 2024-03-06T16:09:06.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:06.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:06.739074+0000) 2024-03-06T16:09:07.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:07.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:07.739306+0000) 2024-03-06T16:09:08.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:08.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:08.739582+0000) 2024-03-06T16:09:09.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:09.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:09.739837+0000) 2024-03-06T16:09:10.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:10.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:10.740081+0000) 2024-03-06T16:09:11.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:11.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:11.740372+0000) 2024-03-06T16:09:12.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:12.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:12.740646+0000) 2024-03-06T16:09:13.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:13.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:13.740819+0000) 2024-03-06T16:09:14.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:14.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:14.741013+0000) 2024-03-06T16:09:15.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:15.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:15.741238+0000) 2024-03-06T16:09:16.054 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:09:16.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:09:16.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:16.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:16.741464+0000) 2024-03-06T16:09:17.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:17.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:17.741700+0000) 2024-03-06T16:09:18.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:18.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:18.741921+0000) 2024-03-06T16:09:19.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:19.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:19.742156+0000) 2024-03-06T16:09:20.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:20.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:20.742381+0000) 2024-03-06T16:09:21.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:21.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:21.742644+0000) 2024-03-06T16:09:22.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:22.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:22.742870+0000) 2024-03-06T16:09:23.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:23.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:23.743100+0000) 2024-03-06T16:09:24.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:24.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:24.743332+0000) 2024-03-06T16:09:25.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:25.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:25.743592+0000) 2024-03-06T16:09:26.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:26.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:26.743777+0000) 2024-03-06T16:09:27.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:27.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:27.744017+0000) 2024-03-06T16:09:28.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:28.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:28.744215+0000) 2024-03-06T16:09:29.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:29.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:29.744476+0000) 2024-03-06T16:09:30.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:30.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:30.744669+0000) 2024-03-06T16:09:31.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:31.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:31.744906+0000) 2024-03-06T16:09:32.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:32.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:32.745115+0000) 2024-03-06T16:09:33.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:33.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:33.745368+0000) 2024-03-06T16:09:34.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:34.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:34.745610+0000) 2024-03-06T16:09:35.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:35.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:35.745815+0000) 2024-03-06T16:09:36.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:36.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:36.746046+0000) 2024-03-06T16:09:37.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:37.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:37.746268+0000) 2024-03-06T16:09:38.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:38.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:38.746491+0000) 2024-03-06T16:09:39.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:39.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:39.746717+0000) 2024-03-06T16:09:40.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:40.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:40.746931+0000) 2024-03-06T16:09:41.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:41.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:41.747127+0000) 2024-03-06T16:09:42.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:42.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:42.747312+0000) 2024-03-06T16:09:43.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:43.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:43.747567+0000) 2024-03-06T16:09:44.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:44.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:44.747848+0000) 2024-03-06T16:09:45.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:45.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:45.748059+0000) 2024-03-06T16:09:46.269 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:09:46.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:09:46.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:46.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:46.748327+0000) 2024-03-06T16:09:47.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:47.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:47.748626+0000) 2024-03-06T16:09:48.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:48.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:48.748875+0000) 2024-03-06T16:09:49.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:49.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:49.749120+0000) 2024-03-06T16:09:50.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:50.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:50.749300+0000) 2024-03-06T16:09:51.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:51.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:51.749525+0000) 2024-03-06T16:09:52.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:52.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:52.749765+0000) 2024-03-06T16:09:53.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:53.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:53.750032+0000) 2024-03-06T16:09:54.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:54.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:54.750272+0000) 2024-03-06T16:09:55.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:55.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:55.750517+0000) 2024-03-06T16:09:56.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:56.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:56.750809+0000) 2024-03-06T16:09:57.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:57.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:57.751048+0000) 2024-03-06T16:09:58.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:58.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:58.751289+0000) 2024-03-06T16:09:59.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:09:59.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:09:59.751573+0000) 2024-03-06T16:10:00.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:00.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:00.751811+0000) 2024-03-06T16:10:01.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:01.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:01.752057+0000) 2024-03-06T16:10:02.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:02.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:02.752263+0000) 2024-03-06T16:10:03.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:03.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:03.752451+0000) 2024-03-06T16:10:04.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:04.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:04.752677+0000) 2024-03-06T16:10:05.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:05.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:05.752926+0000) 2024-03-06T16:10:06.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:06.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:06.753128+0000) 2024-03-06T16:10:07.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:07.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:07.753312+0000) 2024-03-06T16:10:08.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:08.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:08.753564+0000) 2024-03-06T16:10:09.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:09.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:09.753875+0000) 2024-03-06T16:10:10.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:10.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:10.754060+0000) 2024-03-06T16:10:11.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:11.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:11.754255+0000) 2024-03-06T16:10:12.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:12.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:12.754504+0000) 2024-03-06T16:10:13.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:13.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:13.754788+0000) 2024-03-06T16:10:14.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:14.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:14.755039+0000) 2024-03-06T16:10:15.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:15.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:15.755268+0000) 2024-03-06T16:10:16.404 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:10:16.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:10:16.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:16.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:16.755436+0000) 2024-03-06T16:10:17.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:17.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:17.755657+0000) 2024-03-06T16:10:18.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:18.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:18.755894+0000) 2024-03-06T16:10:19.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:19.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:19.756165+0000) 2024-03-06T16:10:20.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:20.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:20.756415+0000) 2024-03-06T16:10:21.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:21.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:21.756660+0000) 2024-03-06T16:10:22.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:22.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:22.756902+0000) 2024-03-06T16:10:23.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:23.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:23.757119+0000) 2024-03-06T16:10:24.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:24.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:24.757350+0000) 2024-03-06T16:10:25.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:25.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:25.757639+0000) 2024-03-06T16:10:26.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:26.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:26.757866+0000) 2024-03-06T16:10:27.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:27.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:27.758058+0000) 2024-03-06T16:10:28.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:28.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:28.758299+0000) 2024-03-06T16:10:29.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:29.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:29.758574+0000) 2024-03-06T16:10:30.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:30.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:30.758811+0000) 2024-03-06T16:10:31.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:31.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:31.759035+0000) 2024-03-06T16:10:32.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:32.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:32.759257+0000) 2024-03-06T16:10:33.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:33.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:33.759482+0000) 2024-03-06T16:10:34.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:34.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:34.759727+0000) 2024-03-06T16:10:35.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:35.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:35.759953+0000) 2024-03-06T16:10:36.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:36.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:36.760206+0000) 2024-03-06T16:10:37.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:37.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:37.760476+0000) 2024-03-06T16:10:38.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:38.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:38.760757+0000) 2024-03-06T16:10:39.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:39.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:39.761057+0000) 2024-03-06T16:10:40.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:40.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:40.761275+0000) 2024-03-06T16:10:41.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:41.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:41.761483+0000) 2024-03-06T16:10:42.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:42.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:42.761687+0000) 2024-03-06T16:10:43.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:43.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:43.761898+0000) 2024-03-06T16:10:44.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:44.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:44.762196+0000) 2024-03-06T16:10:45.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:45.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:45.762465+0000) 2024-03-06T16:10:46.634 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:10:46.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:10:46.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:46.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:46.762707+0000) 2024-03-06T16:10:47.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:47.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:47.762984+0000) 2024-03-06T16:10:48.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:48.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:48.763262+0000) 2024-03-06T16:10:49.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:49.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:49.763605+0000) 2024-03-06T16:10:50.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:50.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:50.763874+0000) 2024-03-06T16:10:51.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:51.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:51.764090+0000) 2024-03-06T16:10:52.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:52.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:52.764338+0000) 2024-03-06T16:10:53.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:53.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:53.764612+0000) 2024-03-06T16:10:54.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:54.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:54.764913+0000) 2024-03-06T16:10:55.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:55.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:55.765192+0000) 2024-03-06T16:10:56.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:56.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:56.765418+0000) 2024-03-06T16:10:57.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:57.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:57.765658+0000) 2024-03-06T16:10:58.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:58.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:58.765873+0000) 2024-03-06T16:10:59.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:10:59.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:10:59.766126+0000) 2024-03-06T16:11:00.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:00.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:00.766355+0000) 2024-03-06T16:11:01.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:01.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:01.766623+0000) 2024-03-06T16:11:02.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:02.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:02.766857+0000) 2024-03-06T16:11:03.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:03.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:03.767079+0000) 2024-03-06T16:11:04.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:04.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:04.767330+0000) 2024-03-06T16:11:05.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:05.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:05.767516+0000) 2024-03-06T16:11:06.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:06.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:06.767767+0000) 2024-03-06T16:11:07.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:07.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:07.767955+0000) 2024-03-06T16:11:08.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:08.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:08.768155+0000) 2024-03-06T16:11:09.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:09.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:09.768420+0000) 2024-03-06T16:11:10.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:10.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:10.768699+0000) 2024-03-06T16:11:11.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:11.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:11.768974+0000) 2024-03-06T16:11:12.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:12.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:12.769250+0000) 2024-03-06T16:11:13.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:13.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:13.769525+0000) 2024-03-06T16:11:14.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:14.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:14.769854+0000) 2024-03-06T16:11:15.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:15.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:15.770130+0000) 2024-03-06T16:11:16.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:16.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:16.770403+0000) 2024-03-06T16:11:16.839 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:11:16.847 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:11:17.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:17.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:17.770668+0000) 2024-03-06T16:11:18.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:18.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:18.770887+0000) 2024-03-06T16:11:19.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:19.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:19.771174+0000) 2024-03-06T16:11:20.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:20.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:20.771391+0000) 2024-03-06T16:11:21.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:21.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:21.771695+0000) 2024-03-06T16:11:22.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:22.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:22.771970+0000) 2024-03-06T16:11:23.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:23.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:23.772249+0000) 2024-03-06T16:11:24.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:24.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:24.772579+0000) 2024-03-06T16:11:25.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:25.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:25.772856+0000) 2024-03-06T16:11:26.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:26.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:26.773093+0000) 2024-03-06T16:11:27.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:27.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:27.773310+0000) 2024-03-06T16:11:28.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:28.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:28.773566+0000) 2024-03-06T16:11:29.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:29.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:29.773831+0000) 2024-03-06T16:11:30.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:30.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:30.774024+0000) 2024-03-06T16:11:31.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:31.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:31.774248+0000) 2024-03-06T16:11:32.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:32.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:32.774586+0000) 2024-03-06T16:11:33.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:33.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:33.774807+0000) 2024-03-06T16:11:34.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:34.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:34.774998+0000) 2024-03-06T16:11:35.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:35.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:35.775232+0000) 2024-03-06T16:11:36.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:36.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:36.775452+0000) 2024-03-06T16:11:37.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:37.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:37.775671+0000) 2024-03-06T16:11:38.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:38.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:38.775865+0000) 2024-03-06T16:11:39.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:39.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:39.776189+0000) 2024-03-06T16:11:40.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:40.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:40.776437+0000) 2024-03-06T16:11:41.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:41.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:41.776669+0000) 2024-03-06T16:11:42.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:42.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:42.776897+0000) 2024-03-06T16:11:43.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:43.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:43.777088+0000) 2024-03-06T16:11:44.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:44.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:44.777324+0000) 2024-03-06T16:11:45.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:45.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:45.777649+0000) 2024-03-06T16:11:46.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:46.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:46.777870+0000) 2024-03-06T16:11:47.035 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:11:47.043 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:11:47.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:47.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:47.778050+0000) 2024-03-06T16:11:48.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:48.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:48.778288+0000) 2024-03-06T16:11:49.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:49.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:49.778525+0000) 2024-03-06T16:11:50.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:50.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:50.778744+0000) 2024-03-06T16:11:51.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:51.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:51.778934+0000) 2024-03-06T16:11:52.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:52.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:52.779154+0000) 2024-03-06T16:11:53.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:53.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:53.779384+0000) 2024-03-06T16:11:54.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:54.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:54.779651+0000) 2024-03-06T16:11:55.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:55.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:55.779833+0000) 2024-03-06T16:11:56.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:56.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:56.780064+0000) 2024-03-06T16:11:57.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:57.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:57.780290+0000) 2024-03-06T16:11:58.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:58.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:58.780515+0000) 2024-03-06T16:11:59.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:11:59.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:11:59.780786+0000) 2024-03-06T16:12:00.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:00.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:00.781014+0000) 2024-03-06T16:12:01.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:01.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:01.781236+0000) 2024-03-06T16:12:02.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:02.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:02.781476+0000) 2024-03-06T16:12:03.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:03.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:03.781684+0000) 2024-03-06T16:12:04.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:04.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:04.781915+0000) 2024-03-06T16:12:05.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:05.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:05.782104+0000) 2024-03-06T16:12:06.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:06.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:06.782289+0000) 2024-03-06T16:12:07.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:07.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:07.782518+0000) 2024-03-06T16:12:08.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:08.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:08.782765+0000) 2024-03-06T16:12:09.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:09.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:09.783016+0000) 2024-03-06T16:12:10.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:10.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:10.783240+0000) 2024-03-06T16:12:11.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:11.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:11.783443+0000) 2024-03-06T16:12:12.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:12.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:12.783707+0000) 2024-03-06T16:12:13.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:13.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:13.783904+0000) 2024-03-06T16:12:14.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:14.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:14.784150+0000) 2024-03-06T16:12:15.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:15.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:15.784376+0000) 2024-03-06T16:12:16.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:16.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:16.784637+0000) 2024-03-06T16:12:17.273 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:12:17.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:12:17.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:17.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:17.784822+0000) 2024-03-06T16:12:18.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:18.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:18.785058+0000) 2024-03-06T16:12:19.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:19.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:19.785307+0000) 2024-03-06T16:12:20.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:20.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:20.785578+0000) 2024-03-06T16:12:21.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:21.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:21.785830+0000) 2024-03-06T16:12:22.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:22.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:22.786073+0000) 2024-03-06T16:12:23.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:23.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:23.786272+0000) 2024-03-06T16:12:24.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:24.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:24.786488+0000) 2024-03-06T16:12:25.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:25.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:25.786724+0000) 2024-03-06T16:12:26.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:26.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:26.786943+0000) 2024-03-06T16:12:27.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:27.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:27.787167+0000) 2024-03-06T16:12:28.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:28.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:28.787353+0000) 2024-03-06T16:12:29.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:29.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:29.787655+0000) 2024-03-06T16:12:30.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:30.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:30.787897+0000) 2024-03-06T16:12:31.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:31.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:31.788096+0000) 2024-03-06T16:12:32.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:32.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:32.788334+0000) 2024-03-06T16:12:33.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:33.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:33.788613+0000) 2024-03-06T16:12:34.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:34.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:34.788855+0000) 2024-03-06T16:12:35.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:35.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:35.789092+0000) 2024-03-06T16:12:36.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:36.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:36.789287+0000) 2024-03-06T16:12:37.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:37.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:37.789515+0000) 2024-03-06T16:12:38.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:38.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:38.789770+0000) 2024-03-06T16:12:39.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:39.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:39.790064+0000) 2024-03-06T16:12:40.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:40.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:40.790299+0000) 2024-03-06T16:12:41.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:41.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:41.790502+0000) 2024-03-06T16:12:42.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:42.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:42.790728+0000) 2024-03-06T16:12:43.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:43.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:43.790947+0000) 2024-03-06T16:12:44.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:44.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:44.791181+0000) 2024-03-06T16:12:45.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:45.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:45.791381+0000) 2024-03-06T16:12:46.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:46.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:46.791664+0000) 2024-03-06T16:12:47.489 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:12:47.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:12:47.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:47.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:47.791891+0000) 2024-03-06T16:12:48.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:48.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:48.792066+0000) 2024-03-06T16:12:49.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:49.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:49.792286+0000) 2024-03-06T16:12:50.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:50.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:50.792528+0000) 2024-03-06T16:12:51.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:51.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:51.792760+0000) 2024-03-06T16:12:52.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:52.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:52.792997+0000) 2024-03-06T16:12:53.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:53.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:53.793218+0000) 2024-03-06T16:12:54.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:54.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:54.793398+0000) 2024-03-06T16:12:55.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:55.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:55.793677+0000) 2024-03-06T16:12:56.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:56.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:56.793867+0000) 2024-03-06T16:12:57.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:57.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:57.794086+0000) 2024-03-06T16:12:58.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:58.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:58.794318+0000) 2024-03-06T16:12:59.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:12:59.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:12:59.794511+0000) 2024-03-06T16:13:00.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:00.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:00.794760+0000) 2024-03-06T16:13:01.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:01.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:01.794955+0000) 2024-03-06T16:13:02.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:02.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:02.795178+0000) 2024-03-06T16:13:03.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:03.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:03.795396+0000) 2024-03-06T16:13:04.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:04.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:04.795660+0000) 2024-03-06T16:13:05.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:05.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:05.795854+0000) 2024-03-06T16:13:06.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:06.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:06.796035+0000) 2024-03-06T16:13:07.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:07.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:07.796261+0000) 2024-03-06T16:13:08.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:08.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:08.796488+0000) 2024-03-06T16:13:09.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:09.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:09.796751+0000) 2024-03-06T16:13:10.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:10.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:10.796997+0000) 2024-03-06T16:13:11.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:11.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:11.797204+0000) 2024-03-06T16:13:12.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:12.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:12.797447+0000) 2024-03-06T16:13:13.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:13.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:13.797662+0000) 2024-03-06T16:13:14.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:14.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:14.797859+0000) 2024-03-06T16:13:15.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:15.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:15.798040+0000) 2024-03-06T16:13:16.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:16.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:16.798236+0000) 2024-03-06T16:13:17.732 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:13:17.740 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:13:17.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:17.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:17.798434+0000) 2024-03-06T16:13:18.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:18.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:18.798669+0000) 2024-03-06T16:13:19.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:19.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:19.798901+0000) 2024-03-06T16:13:20.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:20.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:20.799158+0000) 2024-03-06T16:13:21.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:21.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:21.799395+0000) 2024-03-06T16:13:22.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:22.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:22.799635+0000) 2024-03-06T16:13:23.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:23.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:23.799859+0000) 2024-03-06T16:13:24.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:24.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:24.800088+0000) 2024-03-06T16:13:25.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:25.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:25.800315+0000) 2024-03-06T16:13:26.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:26.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:26.800565+0000) 2024-03-06T16:13:27.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:27.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:27.800791+0000) 2024-03-06T16:13:28.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:28.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:28.801004+0000) 2024-03-06T16:13:29.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:29.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:29.801236+0000) 2024-03-06T16:13:30.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:30.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:30.801425+0000) 2024-03-06T16:13:31.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:31.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:31.801651+0000) 2024-03-06T16:13:32.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:32.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:32.801853+0000) 2024-03-06T16:13:33.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:33.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:33.802051+0000) 2024-03-06T16:13:34.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:34.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:34.802263+0000) 2024-03-06T16:13:35.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:35.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:35.802487+0000) 2024-03-06T16:13:36.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:36.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:36.802674+0000) 2024-03-06T16:13:37.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:37.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:37.802904+0000) 2024-03-06T16:13:38.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:38.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:38.803131+0000) 2024-03-06T16:13:39.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:39.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:39.803369+0000) 2024-03-06T16:13:40.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:40.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:40.803631+0000) 2024-03-06T16:13:41.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:41.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:41.803782+0000) 2024-03-06T16:13:42.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:42.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:42.804007+0000) 2024-03-06T16:13:43.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:43.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:43.804230+0000) 2024-03-06T16:13:44.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:44.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:44.804461+0000) 2024-03-06T16:13:45.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:45.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:45.804695+0000) 2024-03-06T16:13:46.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:46.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:46.804897+0000) 2024-03-06T16:13:47.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:47.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:47.805128+0000) 2024-03-06T16:13:47.953 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:13:47.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:13:48.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:48.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:48.805357+0000) 2024-03-06T16:13:49.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:49.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:49.805520+0000) 2024-03-06T16:13:50.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:50.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:50.805775+0000) 2024-03-06T16:13:51.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:51.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:51.806015+0000) 2024-03-06T16:13:52.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:52.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:52.806256+0000) 2024-03-06T16:13:53.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:53.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:53.806502+0000) 2024-03-06T16:13:54.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:54.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:54.806764+0000) 2024-03-06T16:13:55.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:55.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:55.806975+0000) 2024-03-06T16:13:56.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:56.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:56.807188+0000) 2024-03-06T16:13:57.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:57.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:57.807362+0000) 2024-03-06T16:13:58.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:58.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:58.807642+0000) 2024-03-06T16:13:59.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:13:59.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:13:59.807878+0000) 2024-03-06T16:14:00.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:00.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:00.808070+0000) 2024-03-06T16:14:01.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:01.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:01.808308+0000) 2024-03-06T16:14:02.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:02.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:02.808590+0000) 2024-03-06T16:14:03.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:03.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:03.808808+0000) 2024-03-06T16:14:04.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:04.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:04.809051+0000) 2024-03-06T16:14:05.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:05.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:05.809287+0000) 2024-03-06T16:14:06.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:06.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:06.809514+0000) 2024-03-06T16:14:07.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:07.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:07.809765+0000) 2024-03-06T16:14:08.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:08.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:08.810002+0000) 2024-03-06T16:14:09.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:09.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:09.810256+0000) 2024-03-06T16:14:10.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:10.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:10.810500+0000) 2024-03-06T16:14:11.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:11.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:11.810775+0000) 2024-03-06T16:14:12.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:12.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:12.811014+0000) 2024-03-06T16:14:13.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:13.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:13.811255+0000) 2024-03-06T16:14:14.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:14.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:14.811504+0000) 2024-03-06T16:14:15.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:15.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:15.811751+0000) 2024-03-06T16:14:16.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:16.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:16.811988+0000) 2024-03-06T16:14:17.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:17.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:17.812226+0000) 2024-03-06T16:14:18.174 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:14:18.184 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:14:18.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:18.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:18.812452+0000) 2024-03-06T16:14:19.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:19.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:19.812689+0000) 2024-03-06T16:14:20.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:20.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:20.812959+0000) 2024-03-06T16:14:21.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:21.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:21.813165+0000) 2024-03-06T16:14:22.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:22.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:22.813398+0000) 2024-03-06T16:14:23.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:23.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:23.813673+0000) 2024-03-06T16:14:24.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:24.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:24.813877+0000) 2024-03-06T16:14:25.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:25.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:25.814120+0000) 2024-03-06T16:14:26.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:26.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:26.814342+0000) 2024-03-06T16:14:27.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:27.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:27.814626+0000) 2024-03-06T16:14:28.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:28.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:28.814810+0000) 2024-03-06T16:14:29.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:29.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:29.815044+0000) 2024-03-06T16:14:30.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:30.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:30.815281+0000) 2024-03-06T16:14:31.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:31.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:31.815509+0000) 2024-03-06T16:14:32.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:32.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:32.815696+0000) 2024-03-06T16:14:33.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:33.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:33.815891+0000) 2024-03-06T16:14:34.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:34.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:34.816116+0000) 2024-03-06T16:14:35.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:35.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:35.816353+0000) 2024-03-06T16:14:36.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:36.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:36.816639+0000) 2024-03-06T16:14:37.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:37.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:37.816888+0000) 2024-03-06T16:14:38.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:38.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:38.817127+0000) 2024-03-06T16:14:39.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:39.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:39.817376+0000) 2024-03-06T16:14:40.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:40.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:40.817613+0000) 2024-03-06T16:14:41.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:41.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:41.817846+0000) 2024-03-06T16:14:42.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:42.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:42.818068+0000) 2024-03-06T16:14:43.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:43.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:43.818263+0000) 2024-03-06T16:14:44.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:44.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:44.818492+0000) 2024-03-06T16:14:45.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:45.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:45.818736+0000) 2024-03-06T16:14:46.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:46.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:46.818939+0000) 2024-03-06T16:14:47.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:47.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:47.819162+0000) 2024-03-06T16:14:48.488 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:14:48.496 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:14:48.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:48.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:48.819396+0000) 2024-03-06T16:14:49.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:49.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:49.819649+0000) 2024-03-06T16:14:50.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:50.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:50.819840+0000) 2024-03-06T16:14:51.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:51.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:51.820061+0000) 2024-03-06T16:14:52.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:52.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:52.820301+0000) 2024-03-06T16:14:53.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:53.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:53.820495+0000) 2024-03-06T16:14:54.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:54.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:54.820768+0000) 2024-03-06T16:14:55.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:55.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:55.820999+0000) 2024-03-06T16:14:56.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:56.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:56.821225+0000) 2024-03-06T16:14:57.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:57.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:57.821446+0000) 2024-03-06T16:14:58.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:58.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:58.821662+0000) 2024-03-06T16:14:59.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:14:59.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:14:59.821885+0000) 2024-03-06T16:15:00.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:00.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:00.822112+0000) 2024-03-06T16:15:01.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:01.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:01.822334+0000) 2024-03-06T16:15:02.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:02.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:02.822528+0000) 2024-03-06T16:15:03.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:03.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:03.822856+0000) 2024-03-06T16:15:04.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:04.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:04.823080+0000) 2024-03-06T16:15:05.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:05.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:05.823306+0000) 2024-03-06T16:15:06.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:06.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:06.823498+0000) 2024-03-06T16:15:07.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:07.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:07.823764+0000) 2024-03-06T16:15:08.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:08.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:08.823994+0000) 2024-03-06T16:15:09.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:09.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:09.824236+0000) 2024-03-06T16:15:10.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:10.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:10.824469+0000) 2024-03-06T16:15:11.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:11.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:11.824689+0000) 2024-03-06T16:15:12.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:12.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:12.824912+0000) 2024-03-06T16:15:13.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:13.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:13.825134+0000) 2024-03-06T16:15:14.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:14.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:14.825316+0000) 2024-03-06T16:15:15.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:15.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:15.825592+0000) 2024-03-06T16:15:16.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:16.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:16.825807+0000) 2024-03-06T16:15:17.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:17.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:17.826043+0000) 2024-03-06T16:15:18.639 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:15:18.650 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:15:18.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:18.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:18.826297+0000) 2024-03-06T16:15:19.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:19.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:19.826523+0000) 2024-03-06T16:15:20.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:20.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:20.826765+0000) 2024-03-06T16:15:21.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:21.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:21.826995+0000) 2024-03-06T16:15:22.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:22.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:22.827185+0000) 2024-03-06T16:15:23.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:23.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:23.827412+0000) 2024-03-06T16:15:24.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:24.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:24.827657+0000) 2024-03-06T16:15:25.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:25.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:25.827835+0000) 2024-03-06T16:15:26.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:26.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:26.828071+0000) 2024-03-06T16:15:27.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:27.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:27.828334+0000) 2024-03-06T16:15:28.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:28.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:28.828625+0000) 2024-03-06T16:15:29.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:29.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:29.828918+0000) 2024-03-06T16:15:30.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:30.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:30.829204+0000) 2024-03-06T16:15:31.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:31.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:31.829481+0000) 2024-03-06T16:15:32.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:32.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:32.829772+0000) 2024-03-06T16:15:33.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:33.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:33.829968+0000) 2024-03-06T16:15:34.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:34.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:34.830198+0000) 2024-03-06T16:15:35.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:35.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:35.830438+0000) 2024-03-06T16:15:36.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:36.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:36.830653+0000) 2024-03-06T16:15:37.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:37.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:37.830852+0000) 2024-03-06T16:15:38.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:38.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:38.831074+0000) 2024-03-06T16:15:39.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:39.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:39.831308+0000) 2024-03-06T16:15:40.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:40.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:40.831491+0000) 2024-03-06T16:15:41.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:41.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:41.831718+0000) 2024-03-06T16:15:42.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:42.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:42.831981+0000) 2024-03-06T16:15:43.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:43.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:43.832249+0000) 2024-03-06T16:15:44.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:44.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:44.832463+0000) 2024-03-06T16:15:45.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:45.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:45.832734+0000) 2024-03-06T16:15:46.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:46.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:46.833016+0000) 2024-03-06T16:15:47.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:47.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:47.833208+0000) 2024-03-06T16:15:48.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:48.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:48.833432+0000) 2024-03-06T16:15:48.859 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:15:48.867 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:15:49.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:49.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:49.833666+0000) 2024-03-06T16:15:50.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:50.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:50.833906+0000) 2024-03-06T16:15:51.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:51.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:51.834121+0000) 2024-03-06T16:15:52.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:52.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:52.834306+0000) 2024-03-06T16:15:53.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:53.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:53.834607+0000) 2024-03-06T16:15:54.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:54.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:54.834846+0000) 2024-03-06T16:15:55.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:55.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:55.835064+0000) 2024-03-06T16:15:56.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:56.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:56.835307+0000) 2024-03-06T16:15:57.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:57.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:57.835495+0000) 2024-03-06T16:15:58.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:58.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:58.835690+0000) 2024-03-06T16:15:59.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:15:59.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:15:59.835932+0000) 2024-03-06T16:16:00.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:00.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:00.836165+0000) 2024-03-06T16:16:01.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:01.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:01.836375+0000) 2024-03-06T16:16:02.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:02.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:02.836681+0000) 2024-03-06T16:16:03.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:03.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:03.836950+0000) 2024-03-06T16:16:04.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:04.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:04.837147+0000) 2024-03-06T16:16:05.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:05.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:05.837381+0000) 2024-03-06T16:16:06.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:06.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:06.837649+0000) 2024-03-06T16:16:07.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:07.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:07.837846+0000) 2024-03-06T16:16:08.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:08.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:08.838064+0000) 2024-03-06T16:16:09.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:09.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:09.838290+0000) 2024-03-06T16:16:10.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:10.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:10.838624+0000) 2024-03-06T16:16:11.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:11.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:11.838905+0000) 2024-03-06T16:16:12.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:12.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:12.839150+0000) 2024-03-06T16:16:13.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:13.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:13.839419+0000) 2024-03-06T16:16:14.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:14.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:14.839700+0000) 2024-03-06T16:16:15.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:15.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:15.839945+0000) 2024-03-06T16:16:16.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:16.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:16.840182+0000) 2024-03-06T16:16:17.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:17.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:17.840404+0000) 2024-03-06T16:16:18.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:18.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:18.840679+0000) 2024-03-06T16:16:18.989 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:16:18.997 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:16:19.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:19.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:19.840868+0000) 2024-03-06T16:16:20.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:20.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:20.841160+0000) 2024-03-06T16:16:21.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:21.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:21.841397+0000) 2024-03-06T16:16:22.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:22.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:22.841670+0000) 2024-03-06T16:16:23.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:23.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:23.841885+0000) 2024-03-06T16:16:24.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:24.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:24.842109+0000) 2024-03-06T16:16:25.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:25.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:25.842394+0000) 2024-03-06T16:16:26.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:26.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:26.842695+0000) 2024-03-06T16:16:27.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:27.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:27.842941+0000) 2024-03-06T16:16:28.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:28.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:28.843203+0000) 2024-03-06T16:16:29.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:29.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:29.843442+0000) 2024-03-06T16:16:30.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:30.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:30.843729+0000) 2024-03-06T16:16:31.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:31.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:31.843970+0000) 2024-03-06T16:16:32.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:32.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:32.844229+0000) 2024-03-06T16:16:33.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:33.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:33.844503+0000) 2024-03-06T16:16:34.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:34.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:34.844756+0000) 2024-03-06T16:16:35.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:35.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:35.844993+0000) 2024-03-06T16:16:36.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:36.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:36.845229+0000) 2024-03-06T16:16:37.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:37.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:37.845451+0000) 2024-03-06T16:16:38.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:38.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:38.845686+0000) 2024-03-06T16:16:39.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:39.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:39.845956+0000) 2024-03-06T16:16:40.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:40.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:40.846186+0000) 2024-03-06T16:16:41.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:41.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:41.846403+0000) 2024-03-06T16:16:42.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:42.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:42.846648+0000) 2024-03-06T16:16:43.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:43.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:43.846881+0000) 2024-03-06T16:16:44.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:44.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:44.847133+0000) 2024-03-06T16:16:45.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:45.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:45.847357+0000) 2024-03-06T16:16:46.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:46.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:46.847640+0000) 2024-03-06T16:16:47.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:47.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:47.847844+0000) 2024-03-06T16:16:48.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:48.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:48.848052+0000) 2024-03-06T16:16:49.211 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:16:49.219 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:16:49.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:49.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:49.848304+0000) 2024-03-06T16:16:50.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:50.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:50.848602+0000) 2024-03-06T16:16:51.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:51.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:51.848795+0000) 2024-03-06T16:16:52.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:52.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:52.849035+0000) 2024-03-06T16:16:53.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:53.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:53.849232+0000) 2024-03-06T16:16:54.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:54.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:54.849459+0000) 2024-03-06T16:16:55.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:55.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:55.849675+0000) 2024-03-06T16:16:56.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:56.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:56.849871+0000) 2024-03-06T16:16:57.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:57.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:57.850101+0000) 2024-03-06T16:16:58.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:58.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:58.850354+0000) 2024-03-06T16:16:59.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:16:59.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:16:59.850648+0000) 2024-03-06T16:17:00.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:00.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:00.850903+0000) 2024-03-06T16:17:01.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:01.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:01.851148+0000) 2024-03-06T16:17:02.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:02.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:02.851394+0000) 2024-03-06T16:17:03.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:03.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:03.851645+0000) 2024-03-06T16:17:04.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:04.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:04.851893+0000) 2024-03-06T16:17:05.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:05.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:05.852179+0000) 2024-03-06T16:17:06.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:06.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:06.852425+0000) 2024-03-06T16:17:07.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:07.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:07.852679+0000) 2024-03-06T16:17:08.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:08.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:08.852931+0000) 2024-03-06T16:17:09.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:09.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:09.853162+0000) 2024-03-06T16:17:10.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:10.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:10.853442+0000) 2024-03-06T16:17:11.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:11.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:11.853709+0000) 2024-03-06T16:17:12.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:12.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:12.853948+0000) 2024-03-06T16:17:13.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:13.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:13.854153+0000) 2024-03-06T16:17:14.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:14.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:14.854379+0000) 2024-03-06T16:17:15.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:15.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:15.854694+0000) 2024-03-06T16:17:16.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:16.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:16.854936+0000) 2024-03-06T16:17:17.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:17.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:17.855214+0000) 2024-03-06T16:17:18.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:18.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:18.855477+0000) 2024-03-06T16:17:19.433 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:17:19.441 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:17:19.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:19.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:19.855706+0000) 2024-03-06T16:17:20.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:20.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:20.855918+0000) 2024-03-06T16:17:21.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:21.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:21.856109+0000) 2024-03-06T16:17:22.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:22.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:22.856348+0000) 2024-03-06T16:17:23.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:23.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:23.856644+0000) 2024-03-06T16:17:24.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:24.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:24.856893+0000) 2024-03-06T16:17:25.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:25.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:25.857121+0000) 2024-03-06T16:17:26.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:26.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:26.857314+0000) 2024-03-06T16:17:27.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:27.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:27.857569+0000) 2024-03-06T16:17:28.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:28.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:28.857789+0000) 2024-03-06T16:17:29.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:29.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:29.858024+0000) 2024-03-06T16:17:30.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:30.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:30.858253+0000) 2024-03-06T16:17:31.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:31.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:31.858474+0000) 2024-03-06T16:17:32.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:32.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:32.858742+0000) 2024-03-06T16:17:33.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:33.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:33.859010+0000) 2024-03-06T16:17:34.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:34.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:34.859293+0000) 2024-03-06T16:17:35.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:35.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:35.859610+0000) 2024-03-06T16:17:36.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:36.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:36.859816+0000) 2024-03-06T16:17:37.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:37.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:37.860086+0000) 2024-03-06T16:17:38.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:38.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:38.860282+0000) 2024-03-06T16:17:39.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:39.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:39.860485+0000) 2024-03-06T16:17:40.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:40.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:40.860688+0000) 2024-03-06T16:17:41.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:41.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:41.860877+0000) 2024-03-06T16:17:42.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:42.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:42.861059+0000) 2024-03-06T16:17:43.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:43.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:43.861287+0000) 2024-03-06T16:17:44.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:44.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:44.861509+0000) 2024-03-06T16:17:45.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:45.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:45.861731+0000) 2024-03-06T16:17:46.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:46.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:46.861965+0000) 2024-03-06T16:17:47.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:47.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:47.862184+0000) 2024-03-06T16:17:48.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:48.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:48.862406+0000) 2024-03-06T16:17:49.673 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:17:49.681 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:17:49.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:49.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:49.862649+0000) 2024-03-06T16:17:50.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:50.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:50.862848+0000) 2024-03-06T16:17:51.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:51.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:51.863068+0000) 2024-03-06T16:17:52.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:52.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:52.863252+0000) 2024-03-06T16:17:53.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:53.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:53.863489+0000) 2024-03-06T16:17:54.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:54.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:54.863734+0000) 2024-03-06T16:17:55.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:55.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:55.863966+0000) 2024-03-06T16:17:56.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:56.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:56.864170+0000) 2024-03-06T16:17:57.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:57.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:57.864402+0000) 2024-03-06T16:17:58.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:58.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:58.864644+0000) 2024-03-06T16:17:59.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:17:59.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:17:59.864894+0000) 2024-03-06T16:18:00.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:00.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:00.865138+0000) 2024-03-06T16:18:01.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:01.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:01.865373+0000) 2024-03-06T16:18:02.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:02.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:02.865644+0000) 2024-03-06T16:18:03.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:03.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:03.865875+0000) 2024-03-06T16:18:04.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:04.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:04.866115+0000) 2024-03-06T16:18:05.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:05.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:05.866312+0000) 2024-03-06T16:18:06.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:06.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:06.866569+0000) 2024-03-06T16:18:07.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:07.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:07.866777+0000) 2024-03-06T16:18:08.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:08.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:08.867003+0000) 2024-03-06T16:18:09.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:09.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:09.867226+0000) 2024-03-06T16:18:10.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:10.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:10.867501+0000) 2024-03-06T16:18:11.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:11.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:11.867779+0000) 2024-03-06T16:18:12.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:12.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:12.868047+0000) 2024-03-06T16:18:13.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:13.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:13.868324+0000) 2024-03-06T16:18:14.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:14.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:14.868590+0000) 2024-03-06T16:18:15.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:15.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:15.868807+0000) 2024-03-06T16:18:16.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:16.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:16.869027+0000) 2024-03-06T16:18:17.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:17.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:17.869252+0000) 2024-03-06T16:18:18.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:18.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:18.869468+0000) 2024-03-06T16:18:19.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:19.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:19.869694+0000) 2024-03-06T16:18:19.886 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:18:19.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:18:20.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:20.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:20.869905+0000) 2024-03-06T16:18:21.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:21.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:21.870140+0000) 2024-03-06T16:18:22.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:22.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:22.870357+0000) 2024-03-06T16:18:23.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:23.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:23.870671+0000) 2024-03-06T16:18:24.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:24.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:24.870874+0000) 2024-03-06T16:18:25.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:25.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:25.871130+0000) 2024-03-06T16:18:26.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:26.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:26.871331+0000) 2024-03-06T16:18:27.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:27.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:27.871597+0000) 2024-03-06T16:18:28.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:28.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:28.871803+0000) 2024-03-06T16:18:29.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:29.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:29.872004+0000) 2024-03-06T16:18:30.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:30.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:30.872250+0000) 2024-03-06T16:18:31.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:31.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:31.872565+0000) 2024-03-06T16:18:32.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:32.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:32.872818+0000) 2024-03-06T16:18:33.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:33.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:33.873039+0000) 2024-03-06T16:18:34.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:34.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:34.873320+0000) 2024-03-06T16:18:35.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:35.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:35.873639+0000) 2024-03-06T16:18:36.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:36.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:36.873834+0000) 2024-03-06T16:18:37.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:37.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:37.874014+0000) 2024-03-06T16:18:38.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:38.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:38.874232+0000) 2024-03-06T16:18:39.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:39.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:39.874467+0000) 2024-03-06T16:18:40.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:40.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:40.874691+0000) 2024-03-06T16:18:41.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:41.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:41.874919+0000) 2024-03-06T16:18:42.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:42.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:42.875108+0000) 2024-03-06T16:18:43.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:43.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:43.875324+0000) 2024-03-06T16:18:44.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:44.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:44.875523+0000) 2024-03-06T16:18:45.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:45.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:45.875770+0000) 2024-03-06T16:18:46.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:46.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:46.876003+0000) 2024-03-06T16:18:47.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:47.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:47.876230+0000) 2024-03-06T16:18:48.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:48.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:48.876457+0000) 2024-03-06T16:18:49.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:49.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:49.876705+0000) 2024-03-06T16:18:50.106 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:18:50.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:18:50.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:50.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:50.876935+0000) 2024-03-06T16:18:51.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:51.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:51.877162+0000) 2024-03-06T16:18:52.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:52.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:52.877444+0000) 2024-03-06T16:18:53.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:53.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:53.877711+0000) 2024-03-06T16:18:54.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:54.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:54.877952+0000) 2024-03-06T16:18:55.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:55.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:55.878226+0000) 2024-03-06T16:18:56.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:56.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:56.878463+0000) 2024-03-06T16:18:57.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:57.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:57.878696+0000) 2024-03-06T16:18:58.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:58.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:58.878924+0000) 2024-03-06T16:18:59.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:18:59.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:18:59.879130+0000) 2024-03-06T16:19:00.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:00.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:00.879360+0000) 2024-03-06T16:19:01.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:01.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:01.879636+0000) 2024-03-06T16:19:02.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:02.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:02.879849+0000) 2024-03-06T16:19:03.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:03.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:03.880074+0000) 2024-03-06T16:19:04.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:04.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:04.880299+0000) 2024-03-06T16:19:05.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:05.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:05.880505+0000) 2024-03-06T16:19:06.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:06.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:06.880728+0000) 2024-03-06T16:19:07.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:07.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:07.880924+0000) 2024-03-06T16:19:08.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:08.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:08.881150+0000) 2024-03-06T16:19:09.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:09.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:09.881390+0000) 2024-03-06T16:19:10.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:10.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:10.881654+0000) 2024-03-06T16:19:11.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:11.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:11.881879+0000) 2024-03-06T16:19:12.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:12.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:12.882111+0000) 2024-03-06T16:19:13.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:13.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:13.882334+0000) 2024-03-06T16:19:14.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:14.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:14.882613+0000) 2024-03-06T16:19:15.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:15.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:15.882865+0000) 2024-03-06T16:19:16.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:16.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:16.883087+0000) 2024-03-06T16:19:17.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:17.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:17.883285+0000) 2024-03-06T16:19:18.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:18.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:18.883509+0000) 2024-03-06T16:19:19.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:19.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:19.883846+0000) 2024-03-06T16:19:20.243 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:19:20.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:19:20.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:20.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:20.884108+0000) 2024-03-06T16:19:21.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:21.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:21.884388+0000) 2024-03-06T16:19:22.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:22.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:22.884682+0000) 2024-03-06T16:19:23.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:23.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:23.884910+0000) 2024-03-06T16:19:24.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:24.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:24.885114+0000) 2024-03-06T16:19:25.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:25.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:25.885382+0000) 2024-03-06T16:19:26.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:26.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:26.885658+0000) 2024-03-06T16:19:27.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:27.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:27.885923+0000) 2024-03-06T16:19:28.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:28.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:28.886195+0000) 2024-03-06T16:19:29.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:29.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:29.886440+0000) 2024-03-06T16:19:30.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:30.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:30.886669+0000) 2024-03-06T16:19:31.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:31.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:31.886866+0000) 2024-03-06T16:19:32.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:32.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:32.887092+0000) 2024-03-06T16:19:33.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:33.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:33.887281+0000) 2024-03-06T16:19:34.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:34.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:34.887511+0000) 2024-03-06T16:19:35.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:35.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:35.887781+0000) 2024-03-06T16:19:36.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:36.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:36.887961+0000) 2024-03-06T16:19:37.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:37.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:37.888213+0000) 2024-03-06T16:19:38.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:38.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:38.888455+0000) 2024-03-06T16:19:39.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:39.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:39.888719+0000) 2024-03-06T16:19:40.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:40.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:40.888994+0000) 2024-03-06T16:19:41.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:41.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:41.889266+0000) 2024-03-06T16:19:42.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:42.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:42.889580+0000) 2024-03-06T16:19:43.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:43.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:43.889833+0000) 2024-03-06T16:19:44.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:44.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:44.890024+0000) 2024-03-06T16:19:45.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:45.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:45.890258+0000) 2024-03-06T16:19:46.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:46.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:46.890488+0000) 2024-03-06T16:19:47.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:47.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:47.890751+0000) 2024-03-06T16:19:48.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:48.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:48.890966+0000) 2024-03-06T16:19:49.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:49.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:49.891174+0000) 2024-03-06T16:19:50.457 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:19:50.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:19:50.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:50.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:50.891407+0000) 2024-03-06T16:19:51.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:51.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:51.891679+0000) 2024-03-06T16:19:52.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:52.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:52.891907+0000) 2024-03-06T16:19:53.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:53.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:53.892142+0000) 2024-03-06T16:19:54.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:54.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:54.892367+0000) 2024-03-06T16:19:55.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:55.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:55.892630+0000) 2024-03-06T16:19:56.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:56.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:56.892873+0000) 2024-03-06T16:19:57.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:57.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:57.893075+0000) 2024-03-06T16:19:58.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:58.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:58.893297+0000) 2024-03-06T16:19:59.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:19:59.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:19:59.893591+0000) 2024-03-06T16:20:00.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:00.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:00.893817+0000) 2024-03-06T16:20:01.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:01.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:01.894025+0000) 2024-03-06T16:20:02.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:02.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:02.894257+0000) 2024-03-06T16:20:03.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:03.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:03.894480+0000) 2024-03-06T16:20:04.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:04.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:04.894682+0000) 2024-03-06T16:20:05.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:05.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:05.894883+0000) 2024-03-06T16:20:06.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:06.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:06.895117+0000) 2024-03-06T16:20:07.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:07.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:07.895405+0000) 2024-03-06T16:20:08.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:08.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:08.895646+0000) 2024-03-06T16:20:09.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:09.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:09.895882+0000) 2024-03-06T16:20:10.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:10.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:10.896124+0000) 2024-03-06T16:20:11.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:11.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:11.896347+0000) 2024-03-06T16:20:12.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:12.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:12.896642+0000) 2024-03-06T16:20:13.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:13.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:13.896885+0000) 2024-03-06T16:20:14.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:14.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:14.897122+0000) 2024-03-06T16:20:15.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:15.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:15.897332+0000) 2024-03-06T16:20:16.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:16.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:16.897614+0000) 2024-03-06T16:20:17.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:17.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:17.897824+0000) 2024-03-06T16:20:18.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:18.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:18.898050+0000) 2024-03-06T16:20:19.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:19.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:19.898274+0000) 2024-03-06T16:20:20.591 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:20:20.599 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:20:20.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:20.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:20.898455+0000) 2024-03-06T16:20:21.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:21.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:21.898703+0000) 2024-03-06T16:20:22.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:22.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:22.898935+0000) 2024-03-06T16:20:23.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:23.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:23.899172+0000) 2024-03-06T16:20:24.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:24.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:24.899408+0000) 2024-03-06T16:20:25.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:25.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:25.899662+0000) 2024-03-06T16:20:26.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:26.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:26.899913+0000) 2024-03-06T16:20:27.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:27.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:27.900156+0000) 2024-03-06T16:20:28.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:28.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:28.900398+0000) 2024-03-06T16:20:29.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:29.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:29.900666+0000) 2024-03-06T16:20:30.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:30.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:30.900876+0000) 2024-03-06T16:20:31.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:31.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:31.901128+0000) 2024-03-06T16:20:32.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:32.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:32.901369+0000) 2024-03-06T16:20:33.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:33.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:33.901604+0000) 2024-03-06T16:20:34.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:34.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:34.901826+0000) 2024-03-06T16:20:35.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:35.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:35.902050+0000) 2024-03-06T16:20:36.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:36.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:36.902286+0000) 2024-03-06T16:20:37.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:37.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:37.902529+0000) 2024-03-06T16:20:38.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:38.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:38.902753+0000) 2024-03-06T16:20:39.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:39.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:39.902963+0000) 2024-03-06T16:20:40.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:40.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:40.903207+0000) 2024-03-06T16:20:41.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:41.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:41.903439+0000) 2024-03-06T16:20:42.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:42.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:42.903694+0000) 2024-03-06T16:20:43.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:43.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:43.903910+0000) 2024-03-06T16:20:44.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:44.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:44.904139+0000) 2024-03-06T16:20:45.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:45.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:45.904382+0000) 2024-03-06T16:20:46.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:46.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:46.904659+0000) 2024-03-06T16:20:47.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:47.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:47.904849+0000) 2024-03-06T16:20:48.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:48.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:48.905072+0000) 2024-03-06T16:20:49.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:49.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:49.905263+0000) 2024-03-06T16:20:50.809 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:20:50.817 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:20:50.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:50.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:50.905444+0000) 2024-03-06T16:20:51.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:51.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:51.905691+0000) 2024-03-06T16:20:52.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:52.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:52.905922+0000) 2024-03-06T16:20:53.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:53.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:53.906129+0000) 2024-03-06T16:20:54.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:54.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:54.906367+0000) 2024-03-06T16:20:55.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:55.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:55.906681+0000) 2024-03-06T16:20:56.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:56.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:56.906937+0000) 2024-03-06T16:20:57.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:57.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:57.907149+0000) 2024-03-06T16:20:58.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:58.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:58.907405+0000) 2024-03-06T16:20:59.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:20:59.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:20:59.907689+0000) 2024-03-06T16:21:00.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:00.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:00.907933+0000) 2024-03-06T16:21:01.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:01.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:01.908182+0000) 2024-03-06T16:21:02.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:02.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:02.908443+0000) 2024-03-06T16:21:03.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:03.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:03.908661+0000) 2024-03-06T16:21:04.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:04.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:04.908864+0000) 2024-03-06T16:21:05.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:05.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:05.909090+0000) 2024-03-06T16:21:06.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:06.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:06.909331+0000) 2024-03-06T16:21:07.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:07.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:07.909608+0000) 2024-03-06T16:21:08.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:08.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:08.909863+0000) 2024-03-06T16:21:09.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:09.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:09.910072+0000) 2024-03-06T16:21:10.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:10.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:10.910258+0000) 2024-03-06T16:21:11.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:11.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:11.910489+0000) 2024-03-06T16:21:12.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:12.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:12.910718+0000) 2024-03-06T16:21:13.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:13.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:13.910967+0000) 2024-03-06T16:21:14.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:14.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:14.911175+0000) 2024-03-06T16:21:15.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:15.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:15.911406+0000) 2024-03-06T16:21:16.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:16.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:16.911679+0000) 2024-03-06T16:21:17.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:17.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:17.911870+0000) 2024-03-06T16:21:18.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:18.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:18.912108+0000) 2024-03-06T16:21:19.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:19.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:19.912356+0000) 2024-03-06T16:21:20.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:20.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:20.912666+0000) 2024-03-06T16:21:21.015 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:21:21.022 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:21:21.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:21.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:21.912915+0000) 2024-03-06T16:21:22.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:22.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:22.913121+0000) 2024-03-06T16:21:23.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:23.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:23.913348+0000) 2024-03-06T16:21:24.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:24.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:24.913620+0000) 2024-03-06T16:21:25.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:25.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:25.913832+0000) 2024-03-06T16:21:26.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:26.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:26.914063+0000) 2024-03-06T16:21:27.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:27.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:27.914286+0000) 2024-03-06T16:21:28.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:28.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:28.914521+0000) 2024-03-06T16:21:29.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:29.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:29.914746+0000) 2024-03-06T16:21:30.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:30.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:30.914981+0000) 2024-03-06T16:21:31.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:31.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:31.915224+0000) 2024-03-06T16:21:32.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:32.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:32.915470+0000) 2024-03-06T16:21:33.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:33.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:33.915653+0000) 2024-03-06T16:21:34.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:34.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:34.915855+0000) 2024-03-06T16:21:35.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:35.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:35.916095+0000) 2024-03-06T16:21:36.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:36.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:36.916333+0000) 2024-03-06T16:21:37.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:37.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:37.916527+0000) 2024-03-06T16:21:38.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:38.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:38.916811+0000) 2024-03-06T16:21:39.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:39.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:39.917042+0000) 2024-03-06T16:21:40.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:40.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:40.917266+0000) 2024-03-06T16:21:41.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:41.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:41.917521+0000) 2024-03-06T16:21:42.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:42.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:42.917733+0000) 2024-03-06T16:21:43.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:43.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:43.917929+0000) 2024-03-06T16:21:44.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:44.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:44.918146+0000) 2024-03-06T16:21:45.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:45.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:45.918380+0000) 2024-03-06T16:21:46.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:46.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:46.918652+0000) 2024-03-06T16:21:47.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:47.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:47.918878+0000) 2024-03-06T16:21:48.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:48.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:48.919155+0000) 2024-03-06T16:21:49.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:49.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:49.919391+0000) 2024-03-06T16:21:50.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:50.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:50.919659+0000) 2024-03-06T16:21:51.219 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:21:51.229 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:21:51.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:51.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:51.919900+0000) 2024-03-06T16:21:52.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:52.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:52.920132+0000) 2024-03-06T16:21:53.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:53.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:53.920364+0000) 2024-03-06T16:21:54.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:54.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:54.920628+0000) 2024-03-06T16:21:55.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:55.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:55.920852+0000) 2024-03-06T16:21:56.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:56.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:56.921086+0000) 2024-03-06T16:21:57.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:57.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:57.921309+0000) 2024-03-06T16:21:58.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:58.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:58.921503+0000) 2024-03-06T16:21:59.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:21:59.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:21:59.921778+0000) 2024-03-06T16:22:00.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:00.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:00.921985+0000) 2024-03-06T16:22:01.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:01.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:01.922230+0000) 2024-03-06T16:22:02.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:02.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:02.922453+0000) 2024-03-06T16:22:03.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:03.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:03.922658+0000) 2024-03-06T16:22:04.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:04.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:04.922881+0000) 2024-03-06T16:22:05.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:05.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:05.923068+0000) 2024-03-06T16:22:06.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:06.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:06.923290+0000) 2024-03-06T16:22:07.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:07.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:07.923471+0000) 2024-03-06T16:22:08.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:08.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:08.923703+0000) 2024-03-06T16:22:09.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:09.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:09.923935+0000) 2024-03-06T16:22:10.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:10.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:10.924169+0000) 2024-03-06T16:22:11.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:11.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:11.924415+0000) 2024-03-06T16:22:12.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:12.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:12.924641+0000) 2024-03-06T16:22:13.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:13.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:13.924827+0000) 2024-03-06T16:22:14.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:14.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:14.925005+0000) 2024-03-06T16:22:15.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:15.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:15.925237+0000) 2024-03-06T16:22:16.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:16.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:16.925499+0000) 2024-03-06T16:22:17.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:17.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:17.925750+0000) 2024-03-06T16:22:18.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:18.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:18.925960+0000) 2024-03-06T16:22:19.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:19.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:19.926204+0000) 2024-03-06T16:22:20.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:20.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:20.926437+0000) 2024-03-06T16:22:21.457 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:22:21.465 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:22:21.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:21.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:21.926635+0000) 2024-03-06T16:22:22.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:22.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:22.926818+0000) 2024-03-06T16:22:23.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:23.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:23.927031+0000) 2024-03-06T16:22:24.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:24.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:24.927205+0000) 2024-03-06T16:22:25.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:25.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:25.927432+0000) 2024-03-06T16:22:26.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:26.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:26.927681+0000) 2024-03-06T16:22:27.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:27.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:27.927909+0000) 2024-03-06T16:22:28.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:28.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:28.928108+0000) 2024-03-06T16:22:29.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:29.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:29.928353+0000) 2024-03-06T16:22:30.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:30.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:30.928615+0000) 2024-03-06T16:22:31.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:31.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:31.928832+0000) 2024-03-06T16:22:32.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:32.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:32.929037+0000) 2024-03-06T16:22:33.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:33.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:33.929205+0000) 2024-03-06T16:22:34.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:34.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:34.929430+0000) 2024-03-06T16:22:35.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:35.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:35.929707+0000) 2024-03-06T16:22:36.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:36.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:36.929991+0000) 2024-03-06T16:22:37.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:37.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:37.930178+0000) 2024-03-06T16:22:38.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:38.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:38.930407+0000) 2024-03-06T16:22:39.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:39.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:39.930635+0000) 2024-03-06T16:22:40.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:40.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:40.930859+0000) 2024-03-06T16:22:41.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:41.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:41.931092+0000) 2024-03-06T16:22:42.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:42.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:42.931316+0000) 2024-03-06T16:22:43.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:43.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:43.931573+0000) 2024-03-06T16:22:44.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:44.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:44.931786+0000) 2024-03-06T16:22:45.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:45.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:45.932022+0000) 2024-03-06T16:22:46.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:46.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:46.932267+0000) 2024-03-06T16:22:47.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:47.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:47.932502+0000) 2024-03-06T16:22:48.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:48.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:48.932747+0000) 2024-03-06T16:22:49.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:49.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:49.932986+0000) 2024-03-06T16:22:50.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:50.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:50.933206+0000) 2024-03-06T16:22:51.620 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:22:51.629 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:22:51.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:51.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:51.933460+0000) 2024-03-06T16:22:52.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:52.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:52.933662+0000) 2024-03-06T16:22:53.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:53.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:53.933810+0000) 2024-03-06T16:22:54.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:54.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:54.934030+0000) 2024-03-06T16:22:55.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:55.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:55.934260+0000) 2024-03-06T16:22:56.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:56.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:56.934508+0000) 2024-03-06T16:22:57.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:57.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:57.934779+0000) 2024-03-06T16:22:58.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:58.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:58.934974+0000) 2024-03-06T16:22:59.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:22:59.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:22:59.935216+0000) 2024-03-06T16:23:00.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:00.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:00.935400+0000) 2024-03-06T16:23:01.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:01.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:01.935637+0000) 2024-03-06T16:23:02.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:02.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:02.935824+0000) 2024-03-06T16:23:03.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:03.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:03.935989+0000) 2024-03-06T16:23:04.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:04.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:04.936206+0000) 2024-03-06T16:23:05.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:05.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:05.936427+0000) 2024-03-06T16:23:06.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:06.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:06.936648+0000) 2024-03-06T16:23:07.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:07.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:07.936877+0000) 2024-03-06T16:23:08.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:08.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:08.937112+0000) 2024-03-06T16:23:09.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:09.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:09.937345+0000) 2024-03-06T16:23:10.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:10.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:10.937632+0000) 2024-03-06T16:23:11.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:11.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:11.937878+0000) 2024-03-06T16:23:12.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:12.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:12.938104+0000) 2024-03-06T16:23:13.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:13.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:13.938297+0000) 2024-03-06T16:23:14.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:14.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:14.938528+0000) 2024-03-06T16:23:15.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:15.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:15.938775+0000) 2024-03-06T16:23:16.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:16.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:16.938963+0000) 2024-03-06T16:23:17.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:17.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:17.939134+0000) 2024-03-06T16:23:18.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:18.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:18.939356+0000) 2024-03-06T16:23:19.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:19.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:19.939634+0000) 2024-03-06T16:23:20.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:20.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:20.939859+0000) 2024-03-06T16:23:21.835 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:23:21.843 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:23:21.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:21.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:21.940040+0000) 2024-03-06T16:23:22.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:22.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:22.940214+0000) 2024-03-06T16:23:23.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:23.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:23.940393+0000) 2024-03-06T16:23:24.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:24.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:24.940636+0000) 2024-03-06T16:23:25.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:25.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:25.940841+0000) 2024-03-06T16:23:26.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:26.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:26.941077+0000) 2024-03-06T16:23:27.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:27.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:27.941299+0000) 2024-03-06T16:23:28.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:28.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:28.941519+0000) 2024-03-06T16:23:29.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:29.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:29.941861+0000) 2024-03-06T16:23:30.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:30.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:30.942137+0000) 2024-03-06T16:23:31.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:31.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:31.942377+0000) 2024-03-06T16:23:32.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:32.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:32.942649+0000) 2024-03-06T16:23:33.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:33.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:33.942918+0000) 2024-03-06T16:23:34.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:34.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:34.943198+0000) 2024-03-06T16:23:35.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:35.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:35.943468+0000) 2024-03-06T16:23:36.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:36.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:36.943705+0000) 2024-03-06T16:23:37.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:37.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:37.943929+0000) 2024-03-06T16:23:38.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:38.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:38.944170+0000) 2024-03-06T16:23:39.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:39.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:39.944372+0000) 2024-03-06T16:23:40.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:40.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:40.944660+0000) 2024-03-06T16:23:41.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:41.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:41.944893+0000) 2024-03-06T16:23:42.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:42.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:42.945126+0000) 2024-03-06T16:23:43.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:43.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:43.945337+0000) 2024-03-06T16:23:44.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:44.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:44.945620+0000) 2024-03-06T16:23:45.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:45.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:45.945833+0000) 2024-03-06T16:23:46.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:46.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:46.946077+0000) 2024-03-06T16:23:47.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:47.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:47.946287+0000) 2024-03-06T16:23:48.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:48.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:48.946508+0000) 2024-03-06T16:23:49.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:49.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:49.946779+0000) 2024-03-06T16:23:50.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:50.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:50.946967+0000) 2024-03-06T16:23:51.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:51.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:51.947199+0000) 2024-03-06T16:23:52.047 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:23:52.057 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:23:52.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:52.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:52.947395+0000) 2024-03-06T16:23:53.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:53.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:53.947640+0000) 2024-03-06T16:23:54.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:54.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:54.947924+0000) 2024-03-06T16:23:55.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:55.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:55.948129+0000) 2024-03-06T16:23:56.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:56.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:56.948363+0000) 2024-03-06T16:23:57.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:57.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:57.948582+0000) 2024-03-06T16:23:58.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:58.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:58.948762+0000) 2024-03-06T16:23:59.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:23:59.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:23:59.948986+0000) 2024-03-06T16:24:00.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:00.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:00.949209+0000) 2024-03-06T16:24:01.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:01.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:01.949451+0000) 2024-03-06T16:24:02.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:02.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:02.949683+0000) 2024-03-06T16:24:03.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:03.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:03.949903+0000) 2024-03-06T16:24:04.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:04.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:04.950129+0000) 2024-03-06T16:24:05.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:05.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:05.950392+0000) 2024-03-06T16:24:06.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:06.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:06.950605+0000) 2024-03-06T16:24:07.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:07.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:07.950882+0000) 2024-03-06T16:24:08.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:08.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:08.951119+0000) 2024-03-06T16:24:09.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:09.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:09.951379+0000) 2024-03-06T16:24:10.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:10.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:10.951639+0000) 2024-03-06T16:24:11.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:11.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:11.951885+0000) 2024-03-06T16:24:12.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:12.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:12.952132+0000) 2024-03-06T16:24:13.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:13.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:13.952363+0000) 2024-03-06T16:24:14.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:14.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:14.952576+0000) 2024-03-06T16:24:15.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:15.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:15.952816+0000) 2024-03-06T16:24:16.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:16.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:16.953077+0000) 2024-03-06T16:24:17.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:17.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:17.953329+0000) 2024-03-06T16:24:18.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:18.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:18.953606+0000) 2024-03-06T16:24:19.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:19.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:19.953816+0000) 2024-03-06T16:24:20.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:20.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:20.954063+0000) 2024-03-06T16:24:21.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:21.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:21.954282+0000) 2024-03-06T16:24:22.260 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:24:22.268 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:24:22.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:22.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:22.954465+0000) 2024-03-06T16:24:23.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:23.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:23.954717+0000) 2024-03-06T16:24:24.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:24.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:24.954965+0000) 2024-03-06T16:24:25.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:25.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:25.955212+0000) 2024-03-06T16:24:26.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:26.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:26.955454+0000) 2024-03-06T16:24:27.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:27.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:27.955690+0000) 2024-03-06T16:24:28.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:28.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:28.955918+0000) 2024-03-06T16:24:29.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:29.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:29.956133+0000) 2024-03-06T16:24:30.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:30.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:30.956410+0000) 2024-03-06T16:24:31.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:31.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:31.956684+0000) 2024-03-06T16:24:32.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:32.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:32.956961+0000) 2024-03-06T16:24:33.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:33.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:33.957225+0000) 2024-03-06T16:24:34.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:34.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:34.957469+0000) 2024-03-06T16:24:35.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:35.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:35.957696+0000) 2024-03-06T16:24:36.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:36.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:36.957930+0000) 2024-03-06T16:24:37.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:37.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:37.958142+0000) 2024-03-06T16:24:38.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:38.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:38.958350+0000) 2024-03-06T16:24:39.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:39.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:39.958641+0000) 2024-03-06T16:24:40.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:40.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:40.958889+0000) 2024-03-06T16:24:41.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:41.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:41.959067+0000) 2024-03-06T16:24:42.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:42.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:42.959290+0000) 2024-03-06T16:24:43.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:43.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:43.959573+0000) 2024-03-06T16:24:44.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:44.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:44.959826+0000) 2024-03-06T16:24:45.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:45.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:45.960025+0000) 2024-03-06T16:24:46.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:46.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:46.960246+0000) 2024-03-06T16:24:47.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:47.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:47.960436+0000) 2024-03-06T16:24:48.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:48.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:48.960659+0000) 2024-03-06T16:24:49.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:49.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:49.960867+0000) 2024-03-06T16:24:50.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:50.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:50.961058+0000) 2024-03-06T16:24:51.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:51.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:51.961264+0000) 2024-03-06T16:24:52.474 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:24:52.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:24:52.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:52.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:52.961490+0000) 2024-03-06T16:24:53.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:53.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:53.961710+0000) 2024-03-06T16:24:54.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:54.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:54.961908+0000) 2024-03-06T16:24:55.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:55.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:55.962136+0000) 2024-03-06T16:24:56.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:56.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:56.962361+0000) 2024-03-06T16:24:57.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:57.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:57.962575+0000) 2024-03-06T16:24:58.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:58.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:58.962781+0000) 2024-03-06T16:24:59.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:24:59.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:24:59.963006+0000) 2024-03-06T16:25:00.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:00.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:00.963253+0000) 2024-03-06T16:25:01.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:01.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:01.963434+0000) 2024-03-06T16:25:02.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:02.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:02.963667+0000) 2024-03-06T16:25:03.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:03.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:03.963864+0000) 2024-03-06T16:25:04.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:04.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:04.964051+0000) 2024-03-06T16:25:05.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:05.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:05.964339+0000) 2024-03-06T16:25:06.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:06.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:06.964602+0000) 2024-03-06T16:25:07.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:07.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:07.964778+0000) 2024-03-06T16:25:08.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:08.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:08.965000+0000) 2024-03-06T16:25:09.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:09.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:09.965206+0000) 2024-03-06T16:25:10.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:10.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:10.965430+0000) 2024-03-06T16:25:11.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:11.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:11.965653+0000) 2024-03-06T16:25:12.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:12.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:12.965876+0000) 2024-03-06T16:25:13.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:13.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:13.966095+0000) 2024-03-06T16:25:14.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:14.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:14.966327+0000) 2024-03-06T16:25:15.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:15.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:15.966567+0000) 2024-03-06T16:25:16.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:16.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:16.966776+0000) 2024-03-06T16:25:17.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:17.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:17.966984+0000) 2024-03-06T16:25:18.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:18.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:18.967226+0000) 2024-03-06T16:25:19.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:19.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:19.967461+0000) 2024-03-06T16:25:20.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:20.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:20.967659+0000) 2024-03-06T16:25:21.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:21.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:21.967861+0000) 2024-03-06T16:25:22.670 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:25:22.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:25:22.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:22.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:22.968103+0000) 2024-03-06T16:25:23.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:23.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:23.968338+0000) 2024-03-06T16:25:24.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:24.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:24.968629+0000) 2024-03-06T16:25:25.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:25.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:25.968836+0000) 2024-03-06T16:25:26.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:26.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:26.969057+0000) 2024-03-06T16:25:27.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:27.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:27.969244+0000) 2024-03-06T16:25:28.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:28.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:28.969487+0000) 2024-03-06T16:25:29.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:29.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:29.969788+0000) 2024-03-06T16:25:30.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:30.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:30.970027+0000) 2024-03-06T16:25:31.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:31.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:31.970240+0000) 2024-03-06T16:25:32.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:32.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:32.970494+0000) 2024-03-06T16:25:33.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:33.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:33.970723+0000) 2024-03-06T16:25:34.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:34.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:34.970928+0000) 2024-03-06T16:25:35.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:35.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:35.971136+0000) 2024-03-06T16:25:36.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:36.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:36.971344+0000) 2024-03-06T16:25:37.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:37.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:37.971627+0000) 2024-03-06T16:25:38.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:38.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:38.971840+0000) 2024-03-06T16:25:39.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:39.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:39.972086+0000) 2024-03-06T16:25:40.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:40.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:40.972273+0000) 2024-03-06T16:25:41.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:41.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:41.972500+0000) 2024-03-06T16:25:42.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:42.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:42.972708+0000) 2024-03-06T16:25:43.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:43.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:43.972939+0000) 2024-03-06T16:25:44.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:44.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:44.973167+0000) 2024-03-06T16:25:45.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:45.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:45.973386+0000) 2024-03-06T16:25:46.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:46.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:46.973655+0000) 2024-03-06T16:25:47.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:47.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:47.973856+0000) 2024-03-06T16:25:48.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:48.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:48.974094+0000) 2024-03-06T16:25:49.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:49.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:49.974342+0000) 2024-03-06T16:25:50.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:50.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:50.974635+0000) 2024-03-06T16:25:51.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:51.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:51.974863+0000) 2024-03-06T16:25:52.866 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:25:52.874 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:25:52.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:52.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:52.975021+0000) 2024-03-06T16:25:53.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:53.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:53.975238+0000) 2024-03-06T16:25:54.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:54.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:54.975420+0000) 2024-03-06T16:25:55.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:55.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:55.975649+0000) 2024-03-06T16:25:56.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:56.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:56.975806+0000) 2024-03-06T16:25:57.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:57.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:57.976027+0000) 2024-03-06T16:25:58.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:58.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:58.976256+0000) 2024-03-06T16:25:59.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:25:59.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:25:59.976593+0000) 2024-03-06T16:26:00.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:00.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:00.976791+0000) 2024-03-06T16:26:01.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:01.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:01.977009+0000) 2024-03-06T16:26:02.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:02.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:02.977237+0000) 2024-03-06T16:26:03.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:03.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:03.977458+0000) 2024-03-06T16:26:04.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:04.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:04.977691+0000) 2024-03-06T16:26:05.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:05.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:05.977872+0000) 2024-03-06T16:26:06.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:06.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:06.978051+0000) 2024-03-06T16:26:07.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:07.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:07.978235+0000) 2024-03-06T16:26:08.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:08.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:08.978417+0000) 2024-03-06T16:26:09.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:09.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:09.978677+0000) 2024-03-06T16:26:10.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:10.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:10.978873+0000) 2024-03-06T16:26:11.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:11.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:11.979091+0000) 2024-03-06T16:26:12.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:12.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:12.979303+0000) 2024-03-06T16:26:13.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:13.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:13.979485+0000) 2024-03-06T16:26:14.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:14.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:14.979686+0000) 2024-03-06T16:26:15.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:15.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:15.979930+0000) 2024-03-06T16:26:16.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:16.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:16.980102+0000) 2024-03-06T16:26:17.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:17.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:17.980347+0000) 2024-03-06T16:26:18.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:18.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:18.980623+0000) 2024-03-06T16:26:19.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:19.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:19.980860+0000) 2024-03-06T16:26:20.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:20.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:20.981114+0000) 2024-03-06T16:26:21.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:21.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:21.981346+0000) 2024-03-06T16:26:22.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:22.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:22.981641+0000) 2024-03-06T16:26:23.087 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:26:23.096 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:26:23.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:23.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:23.981859+0000) 2024-03-06T16:26:24.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:24.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:24.982095+0000) 2024-03-06T16:26:25.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:25.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:25.982323+0000) 2024-03-06T16:26:26.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:26.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:26.982577+0000) 2024-03-06T16:26:27.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:27.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:27.982842+0000) 2024-03-06T16:26:28.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:28.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:28.983127+0000) 2024-03-06T16:26:29.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:29.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:29.983380+0000) 2024-03-06T16:26:30.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:30.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:30.983649+0000) 2024-03-06T16:26:31.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:31.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:31.983916+0000) 2024-03-06T16:26:32.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:32.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:32.984155+0000) 2024-03-06T16:26:33.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:33.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:33.984385+0000) 2024-03-06T16:26:34.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:34.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:34.984645+0000) 2024-03-06T16:26:35.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:35.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:35.984833+0000) 2024-03-06T16:26:36.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:36.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:36.985061+0000) 2024-03-06T16:26:37.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:37.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:37.985255+0000) 2024-03-06T16:26:38.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:38.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:38.985482+0000) 2024-03-06T16:26:39.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:39.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:39.985714+0000) 2024-03-06T16:26:40.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:40.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:40.985936+0000) 2024-03-06T16:26:41.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:41.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:41.986169+0000) 2024-03-06T16:26:42.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:42.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:42.986401+0000) 2024-03-06T16:26:43.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:43.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:43.986664+0000) 2024-03-06T16:26:44.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:44.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:44.986862+0000) 2024-03-06T16:26:45.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:45.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:45.987095+0000) 2024-03-06T16:26:46.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:46.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:46.987281+0000) 2024-03-06T16:26:47.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:47.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:47.987637+0000) 2024-03-06T16:26:48.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:48.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:48.987885+0000) 2024-03-06T16:26:49.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:49.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:49.988199+0000) 2024-03-06T16:26:50.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:50.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:50.988451+0000) 2024-03-06T16:26:51.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:51.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:51.988690+0000) 2024-03-06T16:26:52.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:52.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:52.988902+0000) 2024-03-06T16:26:53.303 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:26:53.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:26:53.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:53.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:53.989139+0000) 2024-03-06T16:26:54.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:54.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:54.989357+0000) 2024-03-06T16:26:55.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:55.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:55.989628+0000) 2024-03-06T16:26:56.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:56.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:56.989850+0000) 2024-03-06T16:26:57.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:57.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:57.990041+0000) 2024-03-06T16:26:58.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:58.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:58.990227+0000) 2024-03-06T16:26:59.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:26:59.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:26:59.990478+0000) 2024-03-06T16:27:00.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:00.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:00.990698+0000) 2024-03-06T16:27:01.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:01.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:01.990897+0000) 2024-03-06T16:27:02.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:02.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:02.991130+0000) 2024-03-06T16:27:03.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:03.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:03.991366+0000) 2024-03-06T16:27:04.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:04.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:04.991631+0000) 2024-03-06T16:27:05.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:05.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:05.991826+0000) 2024-03-06T16:27:06.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:06.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:06.992054+0000) 2024-03-06T16:27:07.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:07.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:07.992287+0000) 2024-03-06T16:27:08.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:08.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:08.992495+0000) 2024-03-06T16:27:09.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:09.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:09.992752+0000) 2024-03-06T16:27:10.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:10.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:10.992940+0000) 2024-03-06T16:27:11.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:11.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:11.993157+0000) 2024-03-06T16:27:12.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:12.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:12.993359+0000) 2024-03-06T16:27:13.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:13.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:13.993650+0000) 2024-03-06T16:27:14.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:14.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:14.993847+0000) 2024-03-06T16:27:15.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:15.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:15.994052+0000) 2024-03-06T16:27:16.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:16.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:16.994272+0000) 2024-03-06T16:27:17.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:17.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:17.994469+0000) 2024-03-06T16:27:18.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:18.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:18.994698+0000) 2024-03-06T16:27:19.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:19.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:19.994920+0000) 2024-03-06T16:27:20.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:20.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:20.995119+0000) 2024-03-06T16:27:21.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:21.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:21.995336+0000) 2024-03-06T16:27:22.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:22.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:22.995614+0000) 2024-03-06T16:27:23.523 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:27:23.531 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:27:23.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:23.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:23.995798+0000) 2024-03-06T16:27:24.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:24.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:24.996040+0000) 2024-03-06T16:27:25.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:25.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:25.996284+0000) 2024-03-06T16:27:26.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:26.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:26.996501+0000) 2024-03-06T16:27:27.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:27.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:27.996736+0000) 2024-03-06T16:27:28.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:28.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:28.996957+0000) 2024-03-06T16:27:29.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:29.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:29.997150+0000) 2024-03-06T16:27:30.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:30.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:30.997384+0000) 2024-03-06T16:27:31.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:31.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:31.997642+0000) 2024-03-06T16:27:32.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:32.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:32.997840+0000) 2024-03-06T16:27:33.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:33.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:33.998020+0000) 2024-03-06T16:27:34.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:34.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:34.998207+0000) 2024-03-06T16:27:35.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:35.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:35.998443+0000) 2024-03-06T16:27:36.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:36.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:36.998671+0000) 2024-03-06T16:27:37.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:37.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:37.998871+0000) 2024-03-06T16:27:38.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:38.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:38.999097+0000) 2024-03-06T16:27:39.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:39.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:39.999295+0000) 2024-03-06T16:27:40.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:40.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:40.999618+0000) 2024-03-06T16:27:41.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:41.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:41.999818+0000) 2024-03-06T16:27:42.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:42.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:43.000029+0000) 2024-03-06T16:27:43.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:43.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:44.000266+0000) 2024-03-06T16:27:44.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:44.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:45.000494+0000) 2024-03-06T16:27:45.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:45.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:46.000697+0000) 2024-03-06T16:27:46.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:46.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:47.000886+0000) 2024-03-06T16:27:47.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:47.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:48.001123+0000) 2024-03-06T16:27:48.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:48.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:49.001314+0000) 2024-03-06T16:27:49.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:50.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:50.001588+0000) 2024-03-06T16:27:51.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:51.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:51.001791+0000) 2024-03-06T16:27:52.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:52.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:52.002033+0000) 2024-03-06T16:27:53.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:53.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:53.002277+0000) 2024-03-06T16:27:53.670 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:27:53.677 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:27:54.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:54.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:54.002512+0000) 2024-03-06T16:27:55.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:55.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:55.002769+0000) 2024-03-06T16:27:56.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:56.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:56.002957+0000) 2024-03-06T16:27:57.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:57.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:57.003186+0000) 2024-03-06T16:27:58.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:58.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:58.003386+0000) 2024-03-06T16:27:59.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:27:59.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:27:59.003653+0000) 2024-03-06T16:28:00.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:00.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:00.003874+0000) 2024-03-06T16:28:01.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:01.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:01.004116+0000) 2024-03-06T16:28:02.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:02.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:02.004354+0000) 2024-03-06T16:28:03.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:03.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:03.004626+0000) 2024-03-06T16:28:04.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:04.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:04.004868+0000) 2024-03-06T16:28:05.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:05.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:05.005139+0000) 2024-03-06T16:28:06.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:06.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:06.005346+0000) 2024-03-06T16:28:07.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:07.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:07.005614+0000) 2024-03-06T16:28:08.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:08.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:08.005846+0000) 2024-03-06T16:28:09.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:09.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:09.006115+0000) 2024-03-06T16:28:10.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:10.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:10.006415+0000) 2024-03-06T16:28:11.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:11.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:11.006649+0000) 2024-03-06T16:28:12.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:12.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:12.006868+0000) 2024-03-06T16:28:13.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:13.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:13.007101+0000) 2024-03-06T16:28:14.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:14.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:14.007306+0000) 2024-03-06T16:28:15.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:15.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:15.007580+0000) 2024-03-06T16:28:16.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:16.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:16.007808+0000) 2024-03-06T16:28:17.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:17.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:17.007999+0000) 2024-03-06T16:28:18.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:18.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:18.008246+0000) 2024-03-06T16:28:19.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:19.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:19.008467+0000) 2024-03-06T16:28:20.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:20.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:20.008690+0000) 2024-03-06T16:28:21.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:21.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:21.008855+0000) 2024-03-06T16:28:22.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:22.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:22.009079+0000) 2024-03-06T16:28:23.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:23.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:23.009314+0000) 2024-03-06T16:28:23.866 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:28:23.875 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:28:24.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:24.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:24.009589+0000) 2024-03-06T16:28:25.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:25.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:25.009788+0000) 2024-03-06T16:28:26.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:26.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:26.009969+0000) 2024-03-06T16:28:27.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:27.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:27.010177+0000) 2024-03-06T16:28:28.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:28.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:28.010429+0000) 2024-03-06T16:28:29.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:29.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:29.010729+0000) 2024-03-06T16:28:30.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:30.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:30.011012+0000) 2024-03-06T16:28:31.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:31.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:31.011293+0000) 2024-03-06T16:28:32.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:32.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:32.011614+0000) 2024-03-06T16:28:33.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:33.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:33.011861+0000) 2024-03-06T16:28:34.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:34.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:34.012093+0000) 2024-03-06T16:28:35.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:35.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:35.012371+0000) 2024-03-06T16:28:36.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:36.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:36.012678+0000) 2024-03-06T16:28:37.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:37.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:37.012909+0000) 2024-03-06T16:28:38.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:38.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:38.013119+0000) 2024-03-06T16:28:39.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:39.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:39.013361+0000) 2024-03-06T16:28:40.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:40.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:40.013677+0000) 2024-03-06T16:28:41.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:41.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:41.013916+0000) 2024-03-06T16:28:42.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:42.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:42.014135+0000) 2024-03-06T16:28:43.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:43.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:43.014415+0000) 2024-03-06T16:28:44.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:44.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:44.014713+0000) 2024-03-06T16:28:45.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:45.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:45.014958+0000) 2024-03-06T16:28:46.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:46.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:46.015168+0000) 2024-03-06T16:28:47.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:47.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:47.015401+0000) 2024-03-06T16:28:48.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:48.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:48.015646+0000) 2024-03-06T16:28:49.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:49.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:49.015842+0000) 2024-03-06T16:28:50.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:50.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:50.016048+0000) 2024-03-06T16:28:51.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:51.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:51.016314+0000) 2024-03-06T16:28:52.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:52.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:52.016616+0000) 2024-03-06T16:28:53.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:53.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:53.016861+0000) 2024-03-06T16:28:54.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:54.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:54.017061+0000) 2024-03-06T16:28:54.044 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:28:54.052 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:28:55.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:55.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:55.017261+0000) 2024-03-06T16:28:56.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:56.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:56.017505+0000) 2024-03-06T16:28:57.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:57.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:57.017729+0000) 2024-03-06T16:28:58.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:58.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:58.017978+0000) 2024-03-06T16:28:59.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:28:59.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:28:59.018205+0000) 2024-03-06T16:29:00.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:00.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:00.018487+0000) 2024-03-06T16:29:01.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:01.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:01.018735+0000) 2024-03-06T16:29:02.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:02.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:02.018975+0000) 2024-03-06T16:29:03.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:03.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:03.019254+0000) 2024-03-06T16:29:04.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:04.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:04.019563+0000) 2024-03-06T16:29:05.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:05.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:05.019743+0000) 2024-03-06T16:29:06.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:06.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:06.019970+0000) 2024-03-06T16:29:07.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:07.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:07.020198+0000) 2024-03-06T16:29:08.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:08.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:08.020424+0000) 2024-03-06T16:29:09.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:09.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:09.020659+0000) 2024-03-06T16:29:10.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:10.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:10.020862+0000) 2024-03-06T16:29:11.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:11.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:11.021075+0000) 2024-03-06T16:29:12.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:12.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:12.021308+0000) 2024-03-06T16:29:13.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:13.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:13.021562+0000) 2024-03-06T16:29:14.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:14.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:14.021708+0000) 2024-03-06T16:29:15.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:15.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:15.021909+0000) 2024-03-06T16:29:16.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:16.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:16.022109+0000) 2024-03-06T16:29:17.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:17.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:17.022391+0000) 2024-03-06T16:29:18.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:18.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:18.022634+0000) 2024-03-06T16:29:19.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:19.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:19.022878+0000) 2024-03-06T16:29:20.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:20.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:20.023114+0000) 2024-03-06T16:29:21.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:21.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:21.023353+0000) 2024-03-06T16:29:22.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:22.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:22.023631+0000) 2024-03-06T16:29:23.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:23.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:23.023873+0000) 2024-03-06T16:29:24.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:24.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:24.024122+0000) 2024-03-06T16:29:24.184 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:29:24.192 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:29:25.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:25.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:25.024366+0000) 2024-03-06T16:29:26.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:26.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:26.024628+0000) 2024-03-06T16:29:27.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:27.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:27.024847+0000) 2024-03-06T16:29:28.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:28.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:28.025051+0000) 2024-03-06T16:29:29.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:29.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:29.025321+0000) 2024-03-06T16:29:30.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:30.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:30.025623+0000) 2024-03-06T16:29:31.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:31.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:31.025822+0000) 2024-03-06T16:29:32.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:32.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:32.026011+0000) 2024-03-06T16:29:33.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:33.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:33.026239+0000) 2024-03-06T16:29:34.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:34.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:34.026437+0000) 2024-03-06T16:29:35.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:35.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:35.026643+0000) 2024-03-06T16:29:36.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:36.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:36.026832+0000) 2024-03-06T16:29:37.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:37.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:37.027051+0000) 2024-03-06T16:29:38.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:38.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:38.027285+0000) 2024-03-06T16:29:39.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:39.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:39.027484+0000) 2024-03-06T16:29:40.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:40.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:40.027730+0000) 2024-03-06T16:29:41.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:41.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:41.027931+0000) 2024-03-06T16:29:42.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:42.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:42.028120+0000) 2024-03-06T16:29:43.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:43.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:43.028325+0000) 2024-03-06T16:29:44.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:44.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:44.028565+0000) 2024-03-06T16:29:45.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:45.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:45.028809+0000) 2024-03-06T16:29:46.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:46.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:46.029018+0000) 2024-03-06T16:29:47.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:47.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:47.029263+0000) 2024-03-06T16:29:48.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:48.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:48.029502+0000) 2024-03-06T16:29:49.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:49.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:49.029703+0000) 2024-03-06T16:29:50.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:50.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:50.029924+0000) 2024-03-06T16:29:51.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:51.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:51.030117+0000) 2024-03-06T16:29:52.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:52.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:52.030394+0000) 2024-03-06T16:29:53.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:53.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:53.030657+0000) 2024-03-06T16:29:54.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:54.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:54.030917+0000) 2024-03-06T16:29:54.374 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:29:54.383 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:29:55.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:55.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:55.031103+0000) 2024-03-06T16:29:56.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:56.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:56.031290+0000) 2024-03-06T16:29:57.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:57.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:57.031518+0000) 2024-03-06T16:29:58.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:58.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:58.031709+0000) 2024-03-06T16:29:59.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:29:59.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:29:59.031900+0000) 2024-03-06T16:30:00.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:00.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:00.032142+0000) 2024-03-06T16:30:01.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:01.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:01.032401+0000) 2024-03-06T16:30:02.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:02.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:02.032696+0000) 2024-03-06T16:30:03.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:03.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:03.032937+0000) 2024-03-06T16:30:04.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:04.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:04.033173+0000) 2024-03-06T16:30:05.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:05.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:05.033484+0000) 2024-03-06T16:30:06.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:06.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:06.033722+0000) 2024-03-06T16:30:07.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:07.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:07.033946+0000) 2024-03-06T16:30:08.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:08.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:08.034160+0000) 2024-03-06T16:30:09.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:09.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:09.034384+0000) 2024-03-06T16:30:10.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:10.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:10.034636+0000) 2024-03-06T16:30:11.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:11.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:11.034825+0000) 2024-03-06T16:30:12.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:12.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:12.035014+0000) 2024-03-06T16:30:13.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:13.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:13.035276+0000) 2024-03-06T16:30:14.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:14.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:14.035575+0000) 2024-03-06T16:30:15.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:15.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:15.035818+0000) 2024-03-06T16:30:16.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:16.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:16.036052+0000) 2024-03-06T16:30:17.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:17.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:17.036307+0000) 2024-03-06T16:30:18.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:18.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:18.036626+0000) 2024-03-06T16:30:19.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:19.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:19.036830+0000) 2024-03-06T16:30:20.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:20.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:20.037063+0000) 2024-03-06T16:30:21.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:21.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:21.037272+0000) 2024-03-06T16:30:22.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:22.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:22.037513+0000) 2024-03-06T16:30:23.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:23.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:23.037764+0000) 2024-03-06T16:30:24.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:24.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:24.037993+0000) 2024-03-06T16:30:24.580 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:30:24.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:30:25.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:25.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:25.038218+0000) 2024-03-06T16:30:26.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:26.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:26.038448+0000) 2024-03-06T16:30:27.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:27.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:27.038661+0000) 2024-03-06T16:30:28.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:28.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:28.038890+0000) 2024-03-06T16:30:29.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:29.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:29.039070+0000) 2024-03-06T16:30:30.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:30.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:30.039265+0000) 2024-03-06T16:30:31.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:31.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:31.039463+0000) 2024-03-06T16:30:32.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:32.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:32.039677+0000) 2024-03-06T16:30:33.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:33.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:33.039872+0000) 2024-03-06T16:30:34.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:34.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:34.040083+0000) 2024-03-06T16:30:35.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:35.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:35.040299+0000) 2024-03-06T16:30:36.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:36.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:36.040523+0000) 2024-03-06T16:30:37.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:37.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:37.040710+0000) 2024-03-06T16:30:38.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:38.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:38.040909+0000) 2024-03-06T16:30:39.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:39.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:39.041102+0000) 2024-03-06T16:30:40.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:40.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:40.041348+0000) 2024-03-06T16:30:41.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:41.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:41.041647+0000) 2024-03-06T16:30:42.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:42.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:42.041886+0000) 2024-03-06T16:30:43.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:43.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:43.042151+0000) 2024-03-06T16:30:44.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:44.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:44.042425+0000) 2024-03-06T16:30:45.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:45.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:45.042708+0000) 2024-03-06T16:30:46.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:46.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:46.042953+0000) 2024-03-06T16:30:47.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:47.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:47.043213+0000) 2024-03-06T16:30:48.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:48.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:48.043482+0000) 2024-03-06T16:30:49.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:49.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:49.043729+0000) 2024-03-06T16:30:50.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:50.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:50.044008+0000) 2024-03-06T16:30:51.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:51.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:51.044272+0000) 2024-03-06T16:30:52.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:52.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:52.044493+0000) 2024-03-06T16:30:53.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:53.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:53.044759+0000) 2024-03-06T16:30:54.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:54.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:54.044992+0000) 2024-03-06T16:30:54.749 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:30:54.757 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:30:55.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:55.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:55.045232+0000) 2024-03-06T16:30:56.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:56.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:56.045447+0000) 2024-03-06T16:30:57.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:57.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:57.045722+0000) 2024-03-06T16:30:58.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:58.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:58.046008+0000) 2024-03-06T16:30:59.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:30:59.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:30:59.046284+0000) 2024-03-06T16:31:00.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:00.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:00.046644+0000) 2024-03-06T16:31:01.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:01.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:01.046924+0000) 2024-03-06T16:31:02.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:02.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:02.047151+0000) 2024-03-06T16:31:03.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:03.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:03.047381+0000) 2024-03-06T16:31:04.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:04.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:04.047668+0000) 2024-03-06T16:31:05.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:05.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:05.047870+0000) 2024-03-06T16:31:06.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:06.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:06.048058+0000) 2024-03-06T16:31:07.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:07.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:07.048258+0000) 2024-03-06T16:31:08.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:08.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:08.048580+0000) 2024-03-06T16:31:09.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:09.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:09.048821+0000) 2024-03-06T16:31:10.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:10.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:10.049066+0000) 2024-03-06T16:31:11.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:11.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:11.049268+0000) 2024-03-06T16:31:12.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:12.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:12.049471+0000) 2024-03-06T16:31:13.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:13.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:13.049701+0000) 2024-03-06T16:31:14.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:14.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:14.049885+0000) 2024-03-06T16:31:15.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:15.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:15.050118+0000) 2024-03-06T16:31:16.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:16.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:16.050343+0000) 2024-03-06T16:31:17.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:17.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:17.050601+0000) 2024-03-06T16:31:18.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:18.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:18.050778+0000) 2024-03-06T16:31:19.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:19.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:19.051023+0000) 2024-03-06T16:31:20.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:20.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:20.051258+0000) 2024-03-06T16:31:21.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:21.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:21.051496+0000) 2024-03-06T16:31:22.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:22.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:22.051698+0000) 2024-03-06T16:31:23.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:23.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:23.051914+0000) 2024-03-06T16:31:24.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:24.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:24.052107+0000) 2024-03-06T16:31:24.878 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:31:24.886 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:31:25.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:25.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:25.052338+0000) 2024-03-06T16:31:26.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:26.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:26.052661+0000) 2024-03-06T16:31:27.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:27.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:27.052914+0000) 2024-03-06T16:31:28.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:28.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:28.053152+0000) 2024-03-06T16:31:29.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:29.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:29.053385+0000) 2024-03-06T16:31:30.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:30.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:30.053617+0000) 2024-03-06T16:31:31.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:31.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:31.053820+0000) 2024-03-06T16:31:32.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:32.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:32.054060+0000) 2024-03-06T16:31:33.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:33.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:33.054295+0000) 2024-03-06T16:31:34.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:34.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:34.054562+0000) 2024-03-06T16:31:35.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:35.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:35.054750+0000) 2024-03-06T16:31:36.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:36.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:36.054941+0000) 2024-03-06T16:31:37.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:37.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:37.055122+0000) 2024-03-06T16:31:38.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:38.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:38.055361+0000) 2024-03-06T16:31:39.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:39.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:39.055646+0000) 2024-03-06T16:31:40.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:40.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:40.055863+0000) 2024-03-06T16:31:41.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:41.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:41.056086+0000) 2024-03-06T16:31:42.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:42.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:42.056338+0000) 2024-03-06T16:31:43.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:43.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:43.056624+0000) 2024-03-06T16:31:44.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:44.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:44.056844+0000) 2024-03-06T16:31:45.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:45.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:45.057046+0000) 2024-03-06T16:31:46.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:46.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:46.057267+0000) 2024-03-06T16:31:47.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:47.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:47.057441+0000) 2024-03-06T16:31:48.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:48.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:48.057660+0000) 2024-03-06T16:31:49.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:49.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:49.057827+0000) 2024-03-06T16:31:50.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:50.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:50.058025+0000) 2024-03-06T16:31:51.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:51.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:51.058284+0000) 2024-03-06T16:31:52.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:52.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:52.058651+0000) 2024-03-06T16:31:53.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:53.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:53.058911+0000) 2024-03-06T16:31:54.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:54.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:54.059144+0000) 2024-03-06T16:31:55.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:55.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:55.059421+0000) 2024-03-06T16:31:55.099 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:31:55.107 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:31:56.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:56.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:56.059699+0000) 2024-03-06T16:31:57.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:57.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:57.059948+0000) 2024-03-06T16:31:58.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:58.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:58.060174+0000) 2024-03-06T16:31:59.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:31:59.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:31:59.060411+0000) 2024-03-06T16:32:00.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:00.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:00.060686+0000) 2024-03-06T16:32:01.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:01.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:01.060930+0000) 2024-03-06T16:32:02.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:02.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:02.061158+0000) 2024-03-06T16:32:03.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:03.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:03.061424+0000) 2024-03-06T16:32:04.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:04.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:04.061723+0000) 2024-03-06T16:32:05.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:05.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:05.061933+0000) 2024-03-06T16:32:06.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:06.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:06.062164+0000) 2024-03-06T16:32:07.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:07.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:07.062379+0000) 2024-03-06T16:32:08.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:08.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:08.062668+0000) 2024-03-06T16:32:09.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:09.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:09.062912+0000) 2024-03-06T16:32:10.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:10.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:10.063169+0000) 2024-03-06T16:32:11.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:11.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:11.063431+0000) 2024-03-06T16:32:12.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:12.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:12.063706+0000) 2024-03-06T16:32:13.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:13.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:13.063948+0000) 2024-03-06T16:32:14.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:14.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:14.064183+0000) 2024-03-06T16:32:15.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:15.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:15.064446+0000) 2024-03-06T16:32:16.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:16.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:16.064721+0000) 2024-03-06T16:32:17.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:17.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:17.064957+0000) 2024-03-06T16:32:18.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:18.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:18.065187+0000) 2024-03-06T16:32:19.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:19.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:19.065398+0000) 2024-03-06T16:32:20.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:20.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:20.065644+0000) 2024-03-06T16:32:21.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:21.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:21.065871+0000) 2024-03-06T16:32:22.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:22.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:22.066101+0000) 2024-03-06T16:32:23.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:23.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:23.066311+0000) 2024-03-06T16:32:24.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:24.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:24.066504+0000) 2024-03-06T16:32:25.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:25.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:25.066773+0000) 2024-03-06T16:32:25.229 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:32:25.239 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:32:26.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:26.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:26.066994+0000) 2024-03-06T16:32:27.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:27.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:27.067216+0000) 2024-03-06T16:32:28.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:28.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:28.067437+0000) 2024-03-06T16:32:29.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:29.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:29.067671+0000) 2024-03-06T16:32:30.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:30.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:30.067910+0000) 2024-03-06T16:32:31.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:31.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:31.068131+0000) 2024-03-06T16:32:32.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:32.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:32.068355+0000) 2024-03-06T16:32:33.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:33.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:33.068613+0000) 2024-03-06T16:32:34.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:34.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:34.068855+0000) 2024-03-06T16:32:35.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:35.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:35.069090+0000) 2024-03-06T16:32:36.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:36.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:36.069283+0000) 2024-03-06T16:32:37.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:37.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:37.069512+0000) 2024-03-06T16:32:38.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:38.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:38.069753+0000) 2024-03-06T16:32:39.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:39.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:39.069996+0000) 2024-03-06T16:32:40.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:40.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:40.070240+0000) 2024-03-06T16:32:41.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:41.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:41.070465+0000) 2024-03-06T16:32:42.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:42.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:42.070700+0000) 2024-03-06T16:32:43.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:43.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:43.070917+0000) 2024-03-06T16:32:44.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:44.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:44.071151+0000) 2024-03-06T16:32:45.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:45.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:45.071375+0000) 2024-03-06T16:32:46.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:46.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:46.071643+0000) 2024-03-06T16:32:47.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:47.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:47.071859+0000) 2024-03-06T16:32:48.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:48.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:48.072095+0000) 2024-03-06T16:32:49.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:49.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:49.072319+0000) 2024-03-06T16:32:50.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:50.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:50.072593+0000) 2024-03-06T16:32:51.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:51.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:51.072778+0000) 2024-03-06T16:32:52.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:52.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:52.072991+0000) 2024-03-06T16:32:53.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:53.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:53.073228+0000) 2024-03-06T16:32:54.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:54.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:54.073435+0000) 2024-03-06T16:32:55.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:55.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:55.073650+0000) 2024-03-06T16:32:55.367 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:32:55.375 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:32:56.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:56.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:56.073849+0000) 2024-03-06T16:32:57.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:57.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:57.074049+0000) 2024-03-06T16:32:58.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:58.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:58.074282+0000) 2024-03-06T16:32:59.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:32:59.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:32:59.074513+0000) 2024-03-06T16:33:00.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:00.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:00.074756+0000) 2024-03-06T16:33:01.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:01.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:01.074955+0000) 2024-03-06T16:33:02.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:02.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:02.075230+0000) 2024-03-06T16:33:03.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:03.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:03.075494+0000) 2024-03-06T16:33:04.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:04.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:04.075799+0000) 2024-03-06T16:33:05.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:05.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:05.075994+0000) 2024-03-06T16:33:06.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:06.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:06.076222+0000) 2024-03-06T16:33:07.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:07.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:07.076440+0000) 2024-03-06T16:33:08.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:08.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:08.076666+0000) 2024-03-06T16:33:09.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:09.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:09.076881+0000) 2024-03-06T16:33:10.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:10.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:10.077100+0000) 2024-03-06T16:33:11.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:11.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:11.077322+0000) 2024-03-06T16:33:12.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:12.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:12.077575+0000) 2024-03-06T16:33:13.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:13.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:13.077838+0000) 2024-03-06T16:33:14.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:14.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:14.078051+0000) 2024-03-06T16:33:15.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:15.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:15.078296+0000) 2024-03-06T16:33:16.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:16.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:16.078519+0000) 2024-03-06T16:33:17.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:17.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:17.078715+0000) 2024-03-06T16:33:18.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:18.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:18.078958+0000) 2024-03-06T16:33:19.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:19.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:19.079228+0000) 2024-03-06T16:33:20.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:20.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:20.079509+0000) 2024-03-06T16:33:21.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:21.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:21.079812+0000) 2024-03-06T16:33:22.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:22.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:22.080042+0000) 2024-03-06T16:33:23.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:23.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:23.080294+0000) 2024-03-06T16:33:24.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:24.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:24.080612+0000) 2024-03-06T16:33:25.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:25.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:25.080839+0000) 2024-03-06T16:33:25.506 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:33:25.514 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:33:26.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:26.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:26.081108+0000) 2024-03-06T16:33:27.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:27.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:27.081370+0000) 2024-03-06T16:33:28.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:28.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:28.081708+0000) 2024-03-06T16:33:29.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:29.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:29.081985+0000) 2024-03-06T16:33:30.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:30.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:30.082184+0000) 2024-03-06T16:33:31.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:31.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:31.082399+0000) 2024-03-06T16:33:32.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:32.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:32.082623+0000) 2024-03-06T16:33:33.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:33.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:33.082815+0000) 2024-03-06T16:33:34.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:34.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:34.083012+0000) 2024-03-06T16:33:35.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:35.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:35.083226+0000) 2024-03-06T16:33:36.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:36.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:36.083503+0000) 2024-03-06T16:33:37.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:37.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:37.083782+0000) 2024-03-06T16:33:38.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:38.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:38.083970+0000) 2024-03-06T16:33:39.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:39.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:39.084173+0000) 2024-03-06T16:33:40.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:40.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:40.084410+0000) 2024-03-06T16:33:41.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:41.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:41.084687+0000) 2024-03-06T16:33:42.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:42.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:42.084931+0000) 2024-03-06T16:33:43.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:43.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:43.085145+0000) 2024-03-06T16:33:44.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:44.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:44.085431+0000) 2024-03-06T16:33:45.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:45.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:45.085715+0000) 2024-03-06T16:33:46.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:46.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:46.085997+0000) 2024-03-06T16:33:47.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:47.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:47.086198+0000) 2024-03-06T16:33:48.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:48.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:48.086375+0000) 2024-03-06T16:33:49.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:49.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:49.086649+0000) 2024-03-06T16:33:50.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:50.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:50.086901+0000) 2024-03-06T16:33:51.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:51.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:51.087097+0000) 2024-03-06T16:33:52.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:52.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:52.087289+0000) 2024-03-06T16:33:53.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:53.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:53.087515+0000) 2024-03-06T16:33:54.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:54.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:54.087778+0000) 2024-03-06T16:33:55.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:55.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:55.087987+0000) 2024-03-06T16:33:55.660 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:33:55.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:33:56.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:56.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:56.088195+0000) 2024-03-06T16:33:57.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:57.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:57.088422+0000) 2024-03-06T16:33:58.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:58.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:58.088644+0000) 2024-03-06T16:33:59.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:33:59.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:33:59.088884+0000) 2024-03-06T16:34:00.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:00.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:00.089077+0000) 2024-03-06T16:34:01.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:01.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:01.089309+0000) 2024-03-06T16:34:02.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:02.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:02.089574+0000) 2024-03-06T16:34:03.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:03.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:03.089820+0000) 2024-03-06T16:34:04.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:04.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:04.090079+0000) 2024-03-06T16:34:05.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:05.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:05.090323+0000) 2024-03-06T16:34:06.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:06.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:06.090605+0000) 2024-03-06T16:34:07.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:07.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:07.090839+0000) 2024-03-06T16:34:08.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:08.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:08.091082+0000) 2024-03-06T16:34:09.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:09.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:09.091333+0000) 2024-03-06T16:34:10.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:10.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:10.091624+0000) 2024-03-06T16:34:11.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:11.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:11.091820+0000) 2024-03-06T16:34:12.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:12.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:12.092050+0000) 2024-03-06T16:34:13.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:13.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:13.092237+0000) 2024-03-06T16:34:14.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:14.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:14.092474+0000) 2024-03-06T16:34:15.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:15.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:15.092700+0000) 2024-03-06T16:34:16.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:16.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:16.092977+0000) 2024-03-06T16:34:17.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:17.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:17.093183+0000) 2024-03-06T16:34:18.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:18.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:18.093460+0000) 2024-03-06T16:34:19.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:19.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:19.093681+0000) 2024-03-06T16:34:20.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:20.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:20.093950+0000) 2024-03-06T16:34:21.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:21.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:21.094216+0000) 2024-03-06T16:34:22.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:22.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:22.094417+0000) 2024-03-06T16:34:23.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:23.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:23.094646+0000) 2024-03-06T16:34:24.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:24.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:24.094880+0000) 2024-03-06T16:34:25.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:25.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:25.095125+0000) 2024-03-06T16:34:25.798 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:34:25.806 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:34:26.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:26.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:26.095393+0000) 2024-03-06T16:34:27.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:27.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:27.095674+0000) 2024-03-06T16:34:28.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:28.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:28.095918+0000) 2024-03-06T16:34:29.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:29.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:29.096125+0000) 2024-03-06T16:34:30.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:30.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:30.096378+0000) 2024-03-06T16:34:31.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:31.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:31.096666+0000) 2024-03-06T16:34:32.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:32.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:32.096910+0000) 2024-03-06T16:34:33.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:33.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:33.097143+0000) 2024-03-06T16:34:34.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:34.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:34.097368+0000) 2024-03-06T16:34:35.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:35.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:35.097639+0000) 2024-03-06T16:34:36.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:36.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:36.097862+0000) 2024-03-06T16:34:37.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:37.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:37.098056+0000) 2024-03-06T16:34:38.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:38.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:38.098299+0000) 2024-03-06T16:34:39.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:39.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:39.098571+0000) 2024-03-06T16:34:40.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:40.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:40.098839+0000) 2024-03-06T16:34:41.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:41.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:41.099077+0000) 2024-03-06T16:34:42.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:42.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:42.099305+0000) 2024-03-06T16:34:43.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:43.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:43.099498+0000) 2024-03-06T16:34:44.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:44.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:44.099732+0000) 2024-03-06T16:34:45.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:45.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:45.099976+0000) 2024-03-06T16:34:46.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:46.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:46.100200+0000) 2024-03-06T16:34:47.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:47.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:47.100420+0000) 2024-03-06T16:34:48.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:48.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:48.100648+0000) 2024-03-06T16:34:49.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:49.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:49.100888+0000) 2024-03-06T16:34:50.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:50.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:50.101139+0000) 2024-03-06T16:34:51.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:51.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:51.101381+0000) 2024-03-06T16:34:52.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:52.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:52.101656+0000) 2024-03-06T16:34:53.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:53.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:53.101868+0000) 2024-03-06T16:34:54.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:54.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:54.102109+0000) 2024-03-06T16:34:55.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:55.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:55.102340+0000) 2024-03-06T16:34:55.936 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:34:55.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:34:56.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:56.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:56.102618+0000) 2024-03-06T16:34:57.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:57.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:57.102762+0000) 2024-03-06T16:34:58.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:58.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:58.102951+0000) 2024-03-06T16:34:59.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:34:59.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:34:59.103141+0000) 2024-03-06T16:35:00.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:00.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:00.103374+0000) 2024-03-06T16:35:01.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:01.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:01.103618+0000) 2024-03-06T16:35:02.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:02.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:02.103840+0000) 2024-03-06T16:35:03.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:03.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:03.104025+0000) 2024-03-06T16:35:04.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:04.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:04.104256+0000) 2024-03-06T16:35:05.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:05.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:05.104456+0000) 2024-03-06T16:35:06.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:06.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:06.104691+0000) 2024-03-06T16:35:07.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:07.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:07.104922+0000) 2024-03-06T16:35:08.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:08.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:08.105149+0000) 2024-03-06T16:35:09.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:09.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:09.105428+0000) 2024-03-06T16:35:10.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:10.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:10.105675+0000) 2024-03-06T16:35:11.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:11.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:11.105905+0000) 2024-03-06T16:35:12.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:12.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:12.106134+0000) 2024-03-06T16:35:13.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:13.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:13.106372+0000) 2024-03-06T16:35:14.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:14.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:14.106643+0000) 2024-03-06T16:35:15.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:15.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:15.106891+0000) 2024-03-06T16:35:16.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:16.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:16.107099+0000) 2024-03-06T16:35:17.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:17.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:17.107337+0000) 2024-03-06T16:35:18.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:18.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:18.107603+0000) 2024-03-06T16:35:19.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:19.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:19.107847+0000) 2024-03-06T16:35:20.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:20.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:20.108106+0000) 2024-03-06T16:35:21.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:21.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:21.108332+0000) 2024-03-06T16:35:22.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:22.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:22.108566+0000) 2024-03-06T16:35:23.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:23.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:23.108772+0000) 2024-03-06T16:35:24.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:24.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:24.108983+0000) 2024-03-06T16:35:25.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:25.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:25.109214+0000) 2024-03-06T16:35:26.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:35:26.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:35:26.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:26.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:26.109420+0000) 2024-03-06T16:35:27.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:27.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:27.109678+0000) 2024-03-06T16:35:28.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:28.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:28.109872+0000) 2024-03-06T16:35:29.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:29.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:29.110118+0000) 2024-03-06T16:35:30.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:30.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:30.110373+0000) 2024-03-06T16:35:31.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:31.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:31.110653+0000) 2024-03-06T16:35:32.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:32.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:32.110815+0000) 2024-03-06T16:35:33.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:33.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:33.111012+0000) 2024-03-06T16:35:34.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:34.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:34.111244+0000) 2024-03-06T16:35:35.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:35.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:35.111481+0000) 2024-03-06T16:35:36.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:36.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:36.111702+0000) 2024-03-06T16:35:37.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:37.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:37.111892+0000) 2024-03-06T16:35:38.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:38.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:38.112084+0000) 2024-03-06T16:35:39.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:39.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:39.112315+0000) 2024-03-06T16:35:40.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:40.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:40.112575+0000) 2024-03-06T16:35:41.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:41.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:41.112779+0000) 2024-03-06T16:35:42.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:42.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:42.112971+0000) 2024-03-06T16:35:43.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:43.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:43.113196+0000) 2024-03-06T16:35:44.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:44.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:44.113421+0000) 2024-03-06T16:35:45.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:45.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:45.113667+0000) 2024-03-06T16:35:46.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:46.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:46.113874+0000) 2024-03-06T16:35:47.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:47.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:47.114093+0000) 2024-03-06T16:35:48.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:48.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:48.114330+0000) 2024-03-06T16:35:49.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:49.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:49.114566+0000) 2024-03-06T16:35:50.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:50.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:50.114784+0000) 2024-03-06T16:35:51.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:51.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:51.114982+0000) 2024-03-06T16:35:52.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:52.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:52.115239+0000) 2024-03-06T16:35:53.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:53.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:53.115439+0000) 2024-03-06T16:35:54.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:54.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:54.115698+0000) 2024-03-06T16:35:55.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:55.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:55.115891+0000) 2024-03-06T16:35:56.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:56.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:56.116107+0000) 2024-03-06T16:35:56.211 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:35:56.220 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:35:57.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:57.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:57.116350+0000) 2024-03-06T16:35:58.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:58.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:58.116589+0000) 2024-03-06T16:35:59.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:35:59.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:35:59.116788+0000) 2024-03-06T16:36:00.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:00.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:00.116999+0000) 2024-03-06T16:36:01.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:01.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:01.117226+0000) 2024-03-06T16:36:02.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:02.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:02.117430+0000) 2024-03-06T16:36:03.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:03.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:03.117651+0000) 2024-03-06T16:36:04.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:04.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:04.117853+0000) 2024-03-06T16:36:05.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:05.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:05.118068+0000) 2024-03-06T16:36:06.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:06.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:06.118301+0000) 2024-03-06T16:36:07.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:07.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:07.118526+0000) 2024-03-06T16:36:08.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:08.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:08.118750+0000) 2024-03-06T16:36:09.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:09.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:09.118943+0000) 2024-03-06T16:36:10.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:10.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:10.119146+0000) 2024-03-06T16:36:11.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:11.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:11.119338+0000) 2024-03-06T16:36:12.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:12.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:12.119571+0000) 2024-03-06T16:36:13.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:13.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:13.119811+0000) 2024-03-06T16:36:14.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:14.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:14.119997+0000) 2024-03-06T16:36:15.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:15.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:15.120226+0000) 2024-03-06T16:36:16.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:16.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:16.120471+0000) 2024-03-06T16:36:17.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:17.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:17.120669+0000) 2024-03-06T16:36:18.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:18.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:18.120873+0000) 2024-03-06T16:36:19.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:19.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:19.121070+0000) 2024-03-06T16:36:20.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:20.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:20.121306+0000) 2024-03-06T16:36:21.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:21.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:21.121562+0000) 2024-03-06T16:36:22.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:22.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:22.121781+0000) 2024-03-06T16:36:23.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:23.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:23.122003+0000) 2024-03-06T16:36:24.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:24.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:24.122231+0000) 2024-03-06T16:36:25.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:25.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:25.122457+0000) 2024-03-06T16:36:26.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:26.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:26.122696+0000) 2024-03-06T16:36:26.340 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:36:26.348 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:36:27.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:27.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:27.122838+0000) 2024-03-06T16:36:28.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:28.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:28.123045+0000) 2024-03-06T16:36:29.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:29.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:29.123234+0000) 2024-03-06T16:36:30.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:30.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:30.123474+0000) 2024-03-06T16:36:31.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:31.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:31.123676+0000) 2024-03-06T16:36:32.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:32.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:32.123894+0000) 2024-03-06T16:36:33.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:33.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:33.124079+0000) 2024-03-06T16:36:34.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:34.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:34.124261+0000) 2024-03-06T16:36:35.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:35.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:35.124497+0000) 2024-03-06T16:36:36.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:36.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:36.124721+0000) 2024-03-06T16:36:37.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:37.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:37.124949+0000) 2024-03-06T16:36:38.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:38.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:38.125177+0000) 2024-03-06T16:36:39.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:39.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:39.125398+0000) 2024-03-06T16:36:40.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:40.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:40.125669+0000) 2024-03-06T16:36:41.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:41.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:41.125875+0000) 2024-03-06T16:36:42.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:42.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:42.126116+0000) 2024-03-06T16:36:43.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:43.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:43.126333+0000) 2024-03-06T16:36:44.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:44.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:44.126596+0000) 2024-03-06T16:36:45.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:45.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:45.126833+0000) 2024-03-06T16:36:46.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:46.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:46.127061+0000) 2024-03-06T16:36:47.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:47.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:47.127271+0000) 2024-03-06T16:36:48.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:48.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:48.127490+0000) 2024-03-06T16:36:49.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:49.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:49.127713+0000) 2024-03-06T16:36:50.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:50.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:50.128001+0000) 2024-03-06T16:36:51.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:51.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:51.128225+0000) 2024-03-06T16:36:52.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:52.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:52.128404+0000) 2024-03-06T16:36:53.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:53.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:53.128655+0000) 2024-03-06T16:36:54.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:54.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:54.128855+0000) 2024-03-06T16:36:55.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:55.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:55.129088+0000) 2024-03-06T16:36:56.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:56.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:56.129324+0000) 2024-03-06T16:36:56.479 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:36:56.488 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:36:57.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:57.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:57.129575+0000) 2024-03-06T16:36:58.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:58.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:58.129751+0000) 2024-03-06T16:36:59.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:36:59.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:36:59.129965+0000) 2024-03-06T16:37:00.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:00.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:00.130230+0000) 2024-03-06T16:37:01.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:01.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:01.130453+0000) 2024-03-06T16:37:02.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:02.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:02.130656+0000) 2024-03-06T16:37:03.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:03.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:03.130811+0000) 2024-03-06T16:37:04.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:04.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:04.130996+0000) 2024-03-06T16:37:05.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:05.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:05.131221+0000) 2024-03-06T16:37:06.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:06.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:06.131443+0000) 2024-03-06T16:37:07.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:07.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:07.131672+0000) 2024-03-06T16:37:08.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:08.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:08.131876+0000) 2024-03-06T16:37:09.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:09.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:09.132086+0000) 2024-03-06T16:37:10.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:10.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:10.132275+0000) 2024-03-06T16:37:11.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:11.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:11.132502+0000) 2024-03-06T16:37:12.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:12.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:12.132684+0000) 2024-03-06T16:37:13.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:13.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:13.132874+0000) 2024-03-06T16:37:14.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:14.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:14.133086+0000) 2024-03-06T16:37:15.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:15.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:15.133360+0000) 2024-03-06T16:37:16.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:16.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:16.133636+0000) 2024-03-06T16:37:17.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:17.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:17.133839+0000) 2024-03-06T16:37:18.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:18.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:18.134039+0000) 2024-03-06T16:37:19.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:19.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:19.134274+0000) 2024-03-06T16:37:20.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:20.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:20.134569+0000) 2024-03-06T16:37:21.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:21.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:21.134797+0000) 2024-03-06T16:37:22.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:22.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:22.134988+0000) 2024-03-06T16:37:23.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:23.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:23.135212+0000) 2024-03-06T16:37:24.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:24.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:24.135435+0000) 2024-03-06T16:37:25.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:25.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:25.135645+0000) 2024-03-06T16:37:26.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:26.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:26.135848+0000) 2024-03-06T16:37:26.618 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:37:26.625 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:37:27.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:27.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:27.136048+0000) 2024-03-06T16:37:28.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:28.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:28.136286+0000) 2024-03-06T16:37:29.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:29.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:29.136427+0000) 2024-03-06T16:37:30.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:30.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:30.136660+0000) 2024-03-06T16:37:31.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:31.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:31.136865+0000) 2024-03-06T16:37:32.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:32.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:32.137065+0000) 2024-03-06T16:37:33.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:33.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:33.137356+0000) 2024-03-06T16:37:34.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:34.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:34.137584+0000) 2024-03-06T16:37:35.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:35.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:35.137787+0000) 2024-03-06T16:37:36.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:36.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:36.137993+0000) 2024-03-06T16:37:37.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:37.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:37.138242+0000) 2024-03-06T16:37:38.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:38.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:38.138432+0000) 2024-03-06T16:37:39.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:39.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:39.138653+0000) 2024-03-06T16:37:40.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:40.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:40.138892+0000) 2024-03-06T16:37:41.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:41.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:41.139123+0000) 2024-03-06T16:37:42.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:42.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:42.139364+0000) 2024-03-06T16:37:43.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:43.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:43.139624+0000) 2024-03-06T16:37:44.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:44.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:44.139853+0000) 2024-03-06T16:37:45.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:45.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:45.140038+0000) 2024-03-06T16:37:46.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:46.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:46.140289+0000) 2024-03-06T16:37:47.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:47.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:47.140509+0000) 2024-03-06T16:37:48.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:48.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:48.140744+0000) 2024-03-06T16:37:49.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:49.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:49.140940+0000) 2024-03-06T16:37:50.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:50.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:50.141220+0000) 2024-03-06T16:37:51.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:51.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:51.141438+0000) 2024-03-06T16:37:52.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:52.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:52.141642+0000) 2024-03-06T16:37:53.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:53.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:53.141840+0000) 2024-03-06T16:37:54.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:54.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:54.142028+0000) 2024-03-06T16:37:55.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:55.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:55.142252+0000) 2024-03-06T16:37:56.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:56.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:56.142485+0000) 2024-03-06T16:37:56.815 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:37:56.823 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:37:57.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:57.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:57.142701+0000) 2024-03-06T16:37:58.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:58.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:58.142912+0000) 2024-03-06T16:37:59.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:37:59.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:37:59.143100+0000) 2024-03-06T16:38:00.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:00.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:00.143342+0000) 2024-03-06T16:38:01.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:01.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:01.143600+0000) 2024-03-06T16:38:02.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:02.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:02.143785+0000) 2024-03-06T16:38:03.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:03.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:03.143965+0000) 2024-03-06T16:38:04.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:04.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:04.144153+0000) 2024-03-06T16:38:05.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:05.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:05.144381+0000) 2024-03-06T16:38:06.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:06.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:06.144610+0000) 2024-03-06T16:38:07.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:07.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:07.144814+0000) 2024-03-06T16:38:08.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:08.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:08.145008+0000) 2024-03-06T16:38:09.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:09.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:09.145175+0000) 2024-03-06T16:38:10.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:10.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:10.145414+0000) 2024-03-06T16:38:11.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:11.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:11.145642+0000) 2024-03-06T16:38:12.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:12.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:12.145865+0000) 2024-03-06T16:38:13.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:13.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:13.146065+0000) 2024-03-06T16:38:14.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:14.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:14.146300+0000) 2024-03-06T16:38:15.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:15.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:15.146529+0000) 2024-03-06T16:38:16.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:16.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:16.146795+0000) 2024-03-06T16:38:17.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:17.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:17.146990+0000) 2024-03-06T16:38:18.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:18.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:18.147222+0000) 2024-03-06T16:38:19.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:19.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:19.147441+0000) 2024-03-06T16:38:20.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:20.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:20.147692+0000) 2024-03-06T16:38:21.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:21.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:21.147877+0000) 2024-03-06T16:38:22.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:22.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:22.148122+0000) 2024-03-06T16:38:23.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:23.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:23.148347+0000) 2024-03-06T16:38:24.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:24.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:24.148611+0000) 2024-03-06T16:38:25.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:25.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:25.148838+0000) 2024-03-06T16:38:26.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:26.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:26.149122+0000) 2024-03-06T16:38:27.027 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:38:27.035 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:38:27.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:27.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:27.149328+0000) 2024-03-06T16:38:28.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:28.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:28.149516+0000) 2024-03-06T16:38:29.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:29.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:29.149767+0000) 2024-03-06T16:38:30.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:30.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:30.149987+0000) 2024-03-06T16:38:31.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:31.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:31.150224+0000) 2024-03-06T16:38:32.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:32.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:32.150462+0000) 2024-03-06T16:38:33.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:33.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:33.150698+0000) 2024-03-06T16:38:34.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:34.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:34.150905+0000) 2024-03-06T16:38:35.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:35.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:35.151138+0000) 2024-03-06T16:38:36.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:36.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:36.151370+0000) 2024-03-06T16:38:37.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:37.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:37.151626+0000) 2024-03-06T16:38:38.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:38.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:38.151819+0000) 2024-03-06T16:38:39.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:39.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:39.152030+0000) 2024-03-06T16:38:40.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:40.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:40.152254+0000) 2024-03-06T16:38:41.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:41.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:41.152471+0000) 2024-03-06T16:38:42.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:42.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:42.152684+0000) 2024-03-06T16:38:43.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:43.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:43.152874+0000) 2024-03-06T16:38:44.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:44.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:44.153063+0000) 2024-03-06T16:38:45.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:45.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:45.153307+0000) 2024-03-06T16:38:46.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:46.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:46.153575+0000) 2024-03-06T16:38:47.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:47.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:47.153788+0000) 2024-03-06T16:38:48.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:48.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:48.153989+0000) 2024-03-06T16:38:49.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:49.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:49.154226+0000) 2024-03-06T16:38:50.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:50.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:50.154464+0000) 2024-03-06T16:38:51.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:51.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:51.154690+0000) 2024-03-06T16:38:52.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:52.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:52.154873+0000) 2024-03-06T16:38:53.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:53.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:53.155050+0000) 2024-03-06T16:38:54.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:54.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:54.155287+0000) 2024-03-06T16:38:55.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:55.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:55.155498+0000) 2024-03-06T16:38:56.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:56.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:56.155828+0000) 2024-03-06T16:38:57.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:57.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:57.156035+0000) 2024-03-06T16:38:57.257 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:38:57.264 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:38:58.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:58.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:58.156274+0000) 2024-03-06T16:38:59.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:38:59.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:38:59.156507+0000) 2024-03-06T16:39:00.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:00.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:00.156735+0000) 2024-03-06T16:39:01.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:01.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:01.156962+0000) 2024-03-06T16:39:02.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:02.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:02.157144+0000) 2024-03-06T16:39:03.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:03.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:03.157372+0000) 2024-03-06T16:39:04.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:04.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:04.157613+0000) 2024-03-06T16:39:05.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:05.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:05.157809+0000) 2024-03-06T16:39:06.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:06.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:06.158000+0000) 2024-03-06T16:39:07.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:07.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:07.158204+0000) 2024-03-06T16:39:08.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:08.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:08.158436+0000) 2024-03-06T16:39:09.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:09.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:09.158663+0000) 2024-03-06T16:39:10.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:10.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:10.158902+0000) 2024-03-06T16:39:11.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:11.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:11.159132+0000) 2024-03-06T16:39:12.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:12.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:12.159312+0000) 2024-03-06T16:39:13.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:13.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:13.159564+0000) 2024-03-06T16:39:14.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:14.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:14.159773+0000) 2024-03-06T16:39:15.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:15.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:15.160036+0000) 2024-03-06T16:39:16.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:16.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:16.160238+0000) 2024-03-06T16:39:17.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:17.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:17.160471+0000) 2024-03-06T16:39:18.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:18.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:18.160680+0000) 2024-03-06T16:39:19.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:19.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:19.160871+0000) 2024-03-06T16:39:20.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:20.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:20.161130+0000) 2024-03-06T16:39:21.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:21.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:21.161357+0000) 2024-03-06T16:39:22.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:22.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:22.161563+0000) 2024-03-06T16:39:23.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:23.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:23.161772+0000) 2024-03-06T16:39:24.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:24.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:24.161989+0000) 2024-03-06T16:39:25.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:25.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:25.162192+0000) 2024-03-06T16:39:26.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:26.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:26.162436+0000) 2024-03-06T16:39:27.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:27.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:27.162690+0000) 2024-03-06T16:39:27.479 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:39:27.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:39:28.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:28.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:28.162930+0000) 2024-03-06T16:39:29.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:29.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:29.163133+0000) 2024-03-06T16:39:30.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:30.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:30.163398+0000) 2024-03-06T16:39:31.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:31.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:31.163648+0000) 2024-03-06T16:39:32.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:32.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:32.163850+0000) 2024-03-06T16:39:33.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:33.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:33.164065+0000) 2024-03-06T16:39:34.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:34.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:34.164303+0000) 2024-03-06T16:39:35.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:35.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:35.164559+0000) 2024-03-06T16:39:36.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:36.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:36.164764+0000) 2024-03-06T16:39:37.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:37.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:37.164968+0000) 2024-03-06T16:39:38.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:38.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:38.165189+0000) 2024-03-06T16:39:39.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:39.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:39.165412+0000) 2024-03-06T16:39:40.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:40.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:40.165667+0000) 2024-03-06T16:39:41.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:41.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:41.165882+0000) 2024-03-06T16:39:42.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:42.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:42.166080+0000) 2024-03-06T16:39:43.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:43.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:43.166304+0000) 2024-03-06T16:39:44.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:44.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:44.166554+0000) 2024-03-06T16:39:45.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:45.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:45.166710+0000) 2024-03-06T16:39:46.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:46.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:46.166918+0000) 2024-03-06T16:39:47.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:47.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:47.167108+0000) 2024-03-06T16:39:48.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:48.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:48.167342+0000) 2024-03-06T16:39:49.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:49.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:49.167606+0000) 2024-03-06T16:39:50.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:50.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:50.167812+0000) 2024-03-06T16:39:51.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:51.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:51.168001+0000) 2024-03-06T16:39:52.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:52.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:52.168229+0000) 2024-03-06T16:39:53.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:53.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:53.168465+0000) 2024-03-06T16:39:54.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:54.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:54.168690+0000) 2024-03-06T16:39:55.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:55.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:55.168923+0000) 2024-03-06T16:39:56.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:56.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:56.169131+0000) 2024-03-06T16:39:57.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:57.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:57.169394+0000) 2024-03-06T16:39:57.678 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:39:57.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:39:58.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:58.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:58.169668+0000) 2024-03-06T16:39:59.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:39:59.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:39:59.169899+0000) 2024-03-06T16:40:00.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:00.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:00.170144+0000) 2024-03-06T16:40:01.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:01.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:01.170348+0000) 2024-03-06T16:40:02.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:02.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:02.170642+0000) 2024-03-06T16:40:03.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:03.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:03.170870+0000) 2024-03-06T16:40:04.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:04.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:04.171107+0000) 2024-03-06T16:40:05.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:05.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:05.171339+0000) 2024-03-06T16:40:06.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:06.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:06.171601+0000) 2024-03-06T16:40:07.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:07.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:07.171838+0000) 2024-03-06T16:40:08.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:08.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:08.172060+0000) 2024-03-06T16:40:09.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:09.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:09.172247+0000) 2024-03-06T16:40:10.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:10.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:10.172463+0000) 2024-03-06T16:40:11.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:11.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:11.172711+0000) 2024-03-06T16:40:12.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:12.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:12.172910+0000) 2024-03-06T16:40:13.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:13.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:13.173153+0000) 2024-03-06T16:40:14.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:14.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:14.173382+0000) 2024-03-06T16:40:15.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:15.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:15.173649+0000) 2024-03-06T16:40:16.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:16.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:16.173875+0000) 2024-03-06T16:40:17.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:17.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:17.174085+0000) 2024-03-06T16:40:18.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:18.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:18.174335+0000) 2024-03-06T16:40:19.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:19.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:19.174587+0000) 2024-03-06T16:40:20.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:20.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:20.174828+0000) 2024-03-06T16:40:21.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:21.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:21.175060+0000) 2024-03-06T16:40:22.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:22.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:22.175254+0000) 2024-03-06T16:40:23.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:23.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:23.175471+0000) 2024-03-06T16:40:24.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:24.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:24.175678+0000) 2024-03-06T16:40:25.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:25.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:25.175911+0000) 2024-03-06T16:40:26.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:26.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:26.176154+0000) 2024-03-06T16:40:27.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:27.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:27.176371+0000) 2024-03-06T16:40:27.888 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:40:27.896 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:40:28.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:28.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:28.176659+0000) 2024-03-06T16:40:29.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:29.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:29.176847+0000) 2024-03-06T16:40:30.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:30.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:30.177069+0000) 2024-03-06T16:40:31.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:31.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:31.177310+0000) 2024-03-06T16:40:32.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:32.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:32.177505+0000) 2024-03-06T16:40:33.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:33.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:33.177704+0000) 2024-03-06T16:40:34.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:34.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:34.177895+0000) 2024-03-06T16:40:35.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:35.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:35.178091+0000) 2024-03-06T16:40:36.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:36.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:36.178319+0000) 2024-03-06T16:40:37.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:37.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:37.178575+0000) 2024-03-06T16:40:38.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:38.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:38.178753+0000) 2024-03-06T16:40:39.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:39.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:39.178939+0000) 2024-03-06T16:40:40.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:40.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:40.179147+0000) 2024-03-06T16:40:41.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:41.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:41.179376+0000) 2024-03-06T16:40:42.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:42.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:42.179618+0000) 2024-03-06T16:40:43.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:43.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:43.179807+0000) 2024-03-06T16:40:44.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:44.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:44.179999+0000) 2024-03-06T16:40:45.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:45.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:45.180244+0000) 2024-03-06T16:40:46.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:46.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:46.180475+0000) 2024-03-06T16:40:47.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:47.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:47.180692+0000) 2024-03-06T16:40:48.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:48.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:48.180893+0000) 2024-03-06T16:40:49.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:49.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:49.181078+0000) 2024-03-06T16:40:50.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:50.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:50.181315+0000) 2024-03-06T16:40:51.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:51.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:51.181572+0000) 2024-03-06T16:40:52.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:52.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:52.181795+0000) 2024-03-06T16:40:53.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:53.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:53.181985+0000) 2024-03-06T16:40:54.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:54.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:54.182204+0000) 2024-03-06T16:40:55.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:55.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:55.182449+0000) 2024-03-06T16:40:56.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:56.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:56.182687+0000) 2024-03-06T16:40:57.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:57.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:57.182883+0000) 2024-03-06T16:40:58.110 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:40:58.118 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:40:58.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:58.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:58.183126+0000) 2024-03-06T16:40:59.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:40:59.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:40:59.183372+0000) 2024-03-06T16:41:00.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:00.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:00.183634+0000) 2024-03-06T16:41:01.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:01.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:01.183846+0000) 2024-03-06T16:41:02.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:02.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:02.184053+0000) 2024-03-06T16:41:03.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:03.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:03.184274+0000) 2024-03-06T16:41:04.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:04.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:04.184495+0000) 2024-03-06T16:41:05.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:05.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:05.184726+0000) 2024-03-06T16:41:06.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:06.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:06.184996+0000) 2024-03-06T16:41:07.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:07.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:07.185275+0000) 2024-03-06T16:41:08.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:08.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:08.185570+0000) 2024-03-06T16:41:09.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:09.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:09.185802+0000) 2024-03-06T16:41:10.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:10.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:10.186023+0000) 2024-03-06T16:41:11.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:11.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:11.186260+0000) 2024-03-06T16:41:12.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:12.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:12.186440+0000) 2024-03-06T16:41:13.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:13.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:13.186677+0000) 2024-03-06T16:41:14.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:14.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:14.186869+0000) 2024-03-06T16:41:15.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:15.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:15.187069+0000) 2024-03-06T16:41:16.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:16.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:16.187298+0000) 2024-03-06T16:41:17.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:17.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:17.187522+0000) 2024-03-06T16:41:18.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:18.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:18.187737+0000) 2024-03-06T16:41:19.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:19.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:19.187925+0000) 2024-03-06T16:41:20.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:20.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:20.188126+0000) 2024-03-06T16:41:21.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:21.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:21.188351+0000) 2024-03-06T16:41:22.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:22.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:22.188611+0000) 2024-03-06T16:41:23.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:23.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:23.188805+0000) 2024-03-06T16:41:24.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:24.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:24.189057+0000) 2024-03-06T16:41:25.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:25.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:25.189264+0000) 2024-03-06T16:41:26.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:26.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:26.189494+0000) 2024-03-06T16:41:27.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:27.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:27.189687+0000) 2024-03-06T16:41:28.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:28.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:28.189872+0000) 2024-03-06T16:41:28.322 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:41:28.330 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:41:29.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:29.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:29.190068+0000) 2024-03-06T16:41:30.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:30.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:30.190280+0000) 2024-03-06T16:41:31.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:31.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:31.190528+0000) 2024-03-06T16:41:32.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:32.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:32.190735+0000) 2024-03-06T16:41:33.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:33.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:33.190928+0000) 2024-03-06T16:41:34.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:34.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:34.191108+0000) 2024-03-06T16:41:35.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:35.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:35.191333+0000) 2024-03-06T16:41:36.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:36.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:36.191585+0000) 2024-03-06T16:41:37.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:37.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:37.191774+0000) 2024-03-06T16:41:38.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:38.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:38.191961+0000) 2024-03-06T16:41:39.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:39.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:39.192149+0000) 2024-03-06T16:41:40.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:40.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:40.192406+0000) 2024-03-06T16:41:41.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:41.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:41.192693+0000) 2024-03-06T16:41:42.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:42.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:42.192932+0000) 2024-03-06T16:41:43.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:43.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:43.193155+0000) 2024-03-06T16:41:44.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:44.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:44.193444+0000) 2024-03-06T16:41:45.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:45.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:45.193745+0000) 2024-03-06T16:41:46.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:46.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:46.193987+0000) 2024-03-06T16:41:47.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:47.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:47.194212+0000) 2024-03-06T16:41:48.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:48.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:48.194400+0000) 2024-03-06T16:41:49.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:49.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:49.194627+0000) 2024-03-06T16:41:50.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:50.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:50.194826+0000) 2024-03-06T16:41:51.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:51.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:51.195010+0000) 2024-03-06T16:41:52.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:52.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:52.195229+0000) 2024-03-06T16:41:53.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:53.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:53.195450+0000) 2024-03-06T16:41:54.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:54.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:54.195672+0000) 2024-03-06T16:41:55.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:55.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:55.195896+0000) 2024-03-06T16:41:56.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:56.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:56.196120+0000) 2024-03-06T16:41:57.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:57.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:57.196340+0000) 2024-03-06T16:41:58.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:58.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:58.196566+0000) 2024-03-06T16:41:58.553 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:41:58.563 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:41:59.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:41:59.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:41:59.196763+0000) 2024-03-06T16:42:00.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:00.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:00.196976+0000) 2024-03-06T16:42:01.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:01.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:01.197206+0000) 2024-03-06T16:42:02.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:02.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:02.197377+0000) 2024-03-06T16:42:03.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:03.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:03.197660+0000) 2024-03-06T16:42:04.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:04.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:04.197868+0000) 2024-03-06T16:42:05.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:05.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:05.198076+0000) 2024-03-06T16:42:06.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:06.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:06.198321+0000) 2024-03-06T16:42:07.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:07.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:07.198591+0000) 2024-03-06T16:42:08.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:08.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:08.198793+0000) 2024-03-06T16:42:09.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:09.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:09.199023+0000) 2024-03-06T16:42:10.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:10.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:10.199272+0000) 2024-03-06T16:42:11.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:11.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:11.199477+0000) 2024-03-06T16:42:12.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:12.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:12.199698+0000) 2024-03-06T16:42:13.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:13.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:13.199919+0000) 2024-03-06T16:42:14.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:14.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:14.200140+0000) 2024-03-06T16:42:15.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:15.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:15.200363+0000) 2024-03-06T16:42:16.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:16.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:16.200640+0000) 2024-03-06T16:42:17.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:17.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:17.200862+0000) 2024-03-06T16:42:18.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:18.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:18.201050+0000) 2024-03-06T16:42:19.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:19.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:19.201277+0000) 2024-03-06T16:42:20.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:20.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:20.201482+0000) 2024-03-06T16:42:21.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:21.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:21.201715+0000) 2024-03-06T16:42:22.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:22.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:22.201886+0000) 2024-03-06T16:42:23.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:23.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:23.202088+0000) 2024-03-06T16:42:24.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:24.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:24.202293+0000) 2024-03-06T16:42:25.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:25.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:25.202501+0000) 2024-03-06T16:42:26.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:26.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:26.202753+0000) 2024-03-06T16:42:27.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:27.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:27.202962+0000) 2024-03-06T16:42:28.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:28.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:28.203151+0000) 2024-03-06T16:42:28.782 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:42:28.790 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:42:29.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:29.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:29.203383+0000) 2024-03-06T16:42:30.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:30.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:30.203645+0000) 2024-03-06T16:42:31.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:31.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:31.203879+0000) 2024-03-06T16:42:32.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:32.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:32.204104+0000) 2024-03-06T16:42:33.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:33.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:33.204329+0000) 2024-03-06T16:42:34.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:34.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:34.204576+0000) 2024-03-06T16:42:35.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:35.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:35.204812+0000) 2024-03-06T16:42:36.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:36.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:36.205045+0000) 2024-03-06T16:42:37.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:37.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:37.205290+0000) 2024-03-06T16:42:38.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:38.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:38.205586+0000) 2024-03-06T16:42:39.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:39.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:39.205870+0000) 2024-03-06T16:42:40.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:40.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:40.206122+0000) 2024-03-06T16:42:41.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:41.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:41.206401+0000) 2024-03-06T16:42:42.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:42.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:42.206684+0000) 2024-03-06T16:42:43.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:43.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:43.206918+0000) 2024-03-06T16:42:44.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:44.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:44.207193+0000) 2024-03-06T16:42:45.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:45.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:45.207418+0000) 2024-03-06T16:42:46.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:46.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:46.207694+0000) 2024-03-06T16:42:47.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:47.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:47.207871+0000) 2024-03-06T16:42:48.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:48.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:48.208090+0000) 2024-03-06T16:42:49.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:49.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:49.208283+0000) 2024-03-06T16:42:50.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:50.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:50.208520+0000) 2024-03-06T16:42:51.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:51.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:51.208793+0000) 2024-03-06T16:42:52.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:52.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:52.208992+0000) 2024-03-06T16:42:53.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:53.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:53.209250+0000) 2024-03-06T16:42:54.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:54.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:54.209479+0000) 2024-03-06T16:42:55.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:55.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:55.209677+0000) 2024-03-06T16:42:56.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:56.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:56.209940+0000) 2024-03-06T16:42:57.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:57.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:57.210276+0000) 2024-03-06T16:42:58.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:58.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:58.210504+0000) 2024-03-06T16:42:58.920 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:42:58.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:42:59.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:42:59.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:42:59.210828+0000) 2024-03-06T16:43:00.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:00.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:00.211122+0000) 2024-03-06T16:43:01.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:01.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:01.211400+0000) 2024-03-06T16:43:02.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:02.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:02.211618+0000) 2024-03-06T16:43:03.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:03.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:03.211826+0000) 2024-03-06T16:43:04.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:04.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:04.212036+0000) 2024-03-06T16:43:05.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:05.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:05.212280+0000) 2024-03-06T16:43:06.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:06.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:06.212591+0000) 2024-03-06T16:43:07.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:07.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:07.212869+0000) 2024-03-06T16:43:08.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:08.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:08.213150+0000) 2024-03-06T16:43:09.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:09.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:09.213431+0000) 2024-03-06T16:43:10.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:10.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:10.213741+0000) 2024-03-06T16:43:11.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:11.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:11.213952+0000) 2024-03-06T16:43:12.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:12.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:12.214186+0000) 2024-03-06T16:43:13.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:13.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:13.214404+0000) 2024-03-06T16:43:14.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:14.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:14.214661+0000) 2024-03-06T16:43:15.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:15.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:15.214881+0000) 2024-03-06T16:43:16.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:16.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:16.215144+0000) 2024-03-06T16:43:17.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:17.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:17.215423+0000) 2024-03-06T16:43:18.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:18.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:18.215673+0000) 2024-03-06T16:43:19.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:19.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:19.215883+0000) 2024-03-06T16:43:20.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:20.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:20.216181+0000) 2024-03-06T16:43:21.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:21.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:21.216428+0000) 2024-03-06T16:43:22.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:22.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:22.216656+0000) 2024-03-06T16:43:23.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:23.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:23.216877+0000) 2024-03-06T16:43:24.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:24.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:24.217098+0000) 2024-03-06T16:43:25.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:25.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:25.217330+0000) 2024-03-06T16:43:26.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:26.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:26.217602+0000) 2024-03-06T16:43:27.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:27.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:27.217795+0000) 2024-03-06T16:43:28.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:28.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:28.218004+0000) 2024-03-06T16:43:29.058 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:43:29.066 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:43:29.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:29.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:29.218285+0000) 2024-03-06T16:43:30.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:30.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:30.218616+0000) 2024-03-06T16:43:31.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:31.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:31.218888+0000) 2024-03-06T16:43:32.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:32.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:32.219171+0000) 2024-03-06T16:43:33.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:33.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:33.219456+0000) 2024-03-06T16:43:34.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:34.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:34.219731+0000) 2024-03-06T16:43:35.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:35.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:35.219987+0000) 2024-03-06T16:43:36.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:36.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:36.220218+0000) 2024-03-06T16:43:37.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:37.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:37.220446+0000) 2024-03-06T16:43:38.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:38.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:38.220656+0000) 2024-03-06T16:43:39.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:39.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:39.220858+0000) 2024-03-06T16:43:40.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:40.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:40.221090+0000) 2024-03-06T16:43:41.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:41.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:41.221339+0000) 2024-03-06T16:43:42.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:42.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:42.221566+0000) 2024-03-06T16:43:43.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:43.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:43.221764+0000) 2024-03-06T16:43:44.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:44.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:44.221968+0000) 2024-03-06T16:43:45.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:45.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:45.222210+0000) 2024-03-06T16:43:46.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:46.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:46.222463+0000) 2024-03-06T16:43:47.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:47.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:47.222693+0000) 2024-03-06T16:43:48.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:48.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:48.222891+0000) 2024-03-06T16:43:49.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:49.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:49.223099+0000) 2024-03-06T16:43:50.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:50.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:50.223350+0000) 2024-03-06T16:43:51.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:51.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:51.223607+0000) 2024-03-06T16:43:52.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:52.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:52.223812+0000) 2024-03-06T16:43:53.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:53.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:53.224048+0000) 2024-03-06T16:43:54.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:54.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:54.224284+0000) 2024-03-06T16:43:55.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:55.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:55.224483+0000) 2024-03-06T16:43:56.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:56.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:56.224711+0000) 2024-03-06T16:43:57.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:57.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:57.224935+0000) 2024-03-06T16:43:58.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:58.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:58.225161+0000) 2024-03-06T16:43:59.186 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:43:59.194 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:43:59.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:43:59.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:43:59.225359+0000) 2024-03-06T16:44:00.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:00.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:00.225662+0000) 2024-03-06T16:44:01.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:01.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:01.225888+0000) 2024-03-06T16:44:02.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:02.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:02.226112+0000) 2024-03-06T16:44:03.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:03.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:03.226334+0000) 2024-03-06T16:44:04.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:04.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:04.226594+0000) 2024-03-06T16:44:05.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:05.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:05.226828+0000) 2024-03-06T16:44:06.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:06.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:06.227056+0000) 2024-03-06T16:44:07.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:07.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:07.227238+0000) 2024-03-06T16:44:08.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:08.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:08.227452+0000) 2024-03-06T16:44:09.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:09.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:09.227664+0000) 2024-03-06T16:44:10.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:10.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:10.227874+0000) 2024-03-06T16:44:11.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:11.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:11.228119+0000) 2024-03-06T16:44:12.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:12.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:12.228319+0000) 2024-03-06T16:44:13.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:13.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:13.228581+0000) 2024-03-06T16:44:14.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:14.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:14.228766+0000) 2024-03-06T16:44:15.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:15.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:15.228955+0000) 2024-03-06T16:44:16.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:16.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:16.229157+0000) 2024-03-06T16:44:17.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:17.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:17.229402+0000) 2024-03-06T16:44:18.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:18.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:18.229701+0000) 2024-03-06T16:44:19.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:19.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:19.229898+0000) 2024-03-06T16:44:20.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:20.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:20.230136+0000) 2024-03-06T16:44:21.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:21.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:21.230339+0000) 2024-03-06T16:44:22.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:22.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:22.230597+0000) 2024-03-06T16:44:23.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:23.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:23.230800+0000) 2024-03-06T16:44:24.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:24.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:24.231043+0000) 2024-03-06T16:44:25.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:25.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:25.231270+0000) 2024-03-06T16:44:26.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:26.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:26.231503+0000) 2024-03-06T16:44:27.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:27.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:27.231684+0000) 2024-03-06T16:44:28.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:28.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:28.231919+0000) 2024-03-06T16:44:29.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:29.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:29.232123+0000) 2024-03-06T16:44:29.316 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:44:29.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:44:30.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:30.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:30.232382+0000) 2024-03-06T16:44:31.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:31.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:31.232668+0000) 2024-03-06T16:44:32.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:32.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:32.232876+0000) 2024-03-06T16:44:33.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:33.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:33.233106+0000) 2024-03-06T16:44:34.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:34.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:34.233314+0000) 2024-03-06T16:44:35.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:35.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:35.233511+0000) 2024-03-06T16:44:36.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:36.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:36.233750+0000) 2024-03-06T16:44:37.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:37.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:37.233963+0000) 2024-03-06T16:44:38.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:38.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:38.234157+0000) 2024-03-06T16:44:39.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:39.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:39.234379+0000) 2024-03-06T16:44:40.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:40.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:40.234650+0000) 2024-03-06T16:44:41.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:41.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:41.234882+0000) 2024-03-06T16:44:42.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:42.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:42.235125+0000) 2024-03-06T16:44:43.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:43.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:43.235362+0000) 2024-03-06T16:44:44.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:44.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:44.235637+0000) 2024-03-06T16:44:45.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:45.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:45.235880+0000) 2024-03-06T16:44:46.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:46.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:46.236125+0000) 2024-03-06T16:44:47.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:47.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:47.236351+0000) 2024-03-06T16:44:48.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:48.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:48.236607+0000) 2024-03-06T16:44:49.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:49.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:49.236863+0000) 2024-03-06T16:44:50.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:50.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:50.237126+0000) 2024-03-06T16:44:51.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:51.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:51.237368+0000) 2024-03-06T16:44:52.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:52.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:52.237621+0000) 2024-03-06T16:44:53.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:53.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:53.237872+0000) 2024-03-06T16:44:54.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:54.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:54.238115+0000) 2024-03-06T16:44:55.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:55.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:55.238302+0000) 2024-03-06T16:44:56.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:56.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:56.238575+0000) 2024-03-06T16:44:57.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:57.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:57.238772+0000) 2024-03-06T16:44:58.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:58.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:58.238962+0000) 2024-03-06T16:44:59.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:44:59.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:44:59.239157+0000) 2024-03-06T16:44:59.455 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:44:59.462 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:45:00.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:00.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:00.239394+0000) 2024-03-06T16:45:01.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:01.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:01.239655+0000) 2024-03-06T16:45:02.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:02.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:02.239845+0000) 2024-03-06T16:45:03.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:03.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:03.240040+0000) 2024-03-06T16:45:04.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:04.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:04.240245+0000) 2024-03-06T16:45:05.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:05.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:05.240484+0000) 2024-03-06T16:45:06.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:06.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:06.240714+0000) 2024-03-06T16:45:07.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:07.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:07.240898+0000) 2024-03-06T16:45:08.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:08.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:08.241134+0000) 2024-03-06T16:45:09.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:09.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:09.241352+0000) 2024-03-06T16:45:10.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:10.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:10.241618+0000) 2024-03-06T16:45:11.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:11.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:11.241862+0000) 2024-03-06T16:45:12.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:12.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:12.242152+0000) 2024-03-06T16:45:13.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:13.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:13.242432+0000) 2024-03-06T16:45:14.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:14.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:14.242720+0000) 2024-03-06T16:45:15.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:15.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:15.242978+0000) 2024-03-06T16:45:16.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:16.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:16.243192+0000) 2024-03-06T16:45:17.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:17.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:17.243406+0000) 2024-03-06T16:45:18.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:18.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:18.243666+0000) 2024-03-06T16:45:19.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:19.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:19.243892+0000) 2024-03-06T16:45:20.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:20.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:20.244134+0000) 2024-03-06T16:45:21.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:21.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:21.244370+0000) 2024-03-06T16:45:22.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:22.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:22.244649+0000) 2024-03-06T16:45:23.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:23.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:23.244939+0000) 2024-03-06T16:45:24.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:24.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:24.245165+0000) 2024-03-06T16:45:25.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:25.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:25.245437+0000) 2024-03-06T16:45:26.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:26.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:26.245654+0000) 2024-03-06T16:45:27.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:27.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:27.245918+0000) 2024-03-06T16:45:28.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:28.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:28.246161+0000) 2024-03-06T16:45:29.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:29.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:29.246389+0000) 2024-03-06T16:45:29.680 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:45:29.688 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:45:30.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:30.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:30.246674+0000) 2024-03-06T16:45:31.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:31.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:31.246918+0000) 2024-03-06T16:45:32.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:32.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:32.247162+0000) 2024-03-06T16:45:33.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:33.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:33.247391+0000) 2024-03-06T16:45:34.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:34.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:34.247608+0000) 2024-03-06T16:45:35.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:35.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:35.247798+0000) 2024-03-06T16:45:36.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:36.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:36.247992+0000) 2024-03-06T16:45:37.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:37.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:37.248213+0000) 2024-03-06T16:45:38.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:38.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:38.248435+0000) 2024-03-06T16:45:39.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:39.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:39.248676+0000) 2024-03-06T16:45:40.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:40.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:40.248876+0000) 2024-03-06T16:45:41.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:41.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:41.249097+0000) 2024-03-06T16:45:42.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:42.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:42.249299+0000) 2024-03-06T16:45:43.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:43.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:43.249529+0000) 2024-03-06T16:45:44.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:44.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:44.249740+0000) 2024-03-06T16:45:45.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:45.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:45.249930+0000) 2024-03-06T16:45:46.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:46.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:46.250133+0000) 2024-03-06T16:45:47.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:47.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:47.250364+0000) 2024-03-06T16:45:48.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:48.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:48.250610+0000) 2024-03-06T16:45:49.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:49.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:49.250797+0000) 2024-03-06T16:45:50.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:50.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:50.250998+0000) 2024-03-06T16:45:51.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:51.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:51.251191+0000) 2024-03-06T16:45:52.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:52.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:52.251414+0000) 2024-03-06T16:45:53.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:53.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:53.251646+0000) 2024-03-06T16:45:54.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:54.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:54.251884+0000) 2024-03-06T16:45:55.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:55.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:55.252077+0000) 2024-03-06T16:45:56.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:56.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:56.252305+0000) 2024-03-06T16:45:57.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:57.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:57.252577+0000) 2024-03-06T16:45:58.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:58.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:58.252792+0000) 2024-03-06T16:45:59.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:45:59.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:45:59.252993+0000) 2024-03-06T16:45:59.897 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:45:59.905 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:46:00.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:00.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:00.253216+0000) 2024-03-06T16:46:01.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:01.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:01.253464+0000) 2024-03-06T16:46:02.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:02.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:02.253645+0000) 2024-03-06T16:46:03.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:03.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:03.253867+0000) 2024-03-06T16:46:04.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:04.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:04.254064+0000) 2024-03-06T16:46:05.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:05.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:05.254293+0000) 2024-03-06T16:46:06.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:06.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:06.254565+0000) 2024-03-06T16:46:07.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:07.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:07.254776+0000) 2024-03-06T16:46:08.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:08.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:08.254975+0000) 2024-03-06T16:46:09.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:09.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:09.255207+0000) 2024-03-06T16:46:10.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:10.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:10.255462+0000) 2024-03-06T16:46:11.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:11.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:11.255700+0000) 2024-03-06T16:46:12.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:12.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:12.255905+0000) 2024-03-06T16:46:13.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:13.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:13.256146+0000) 2024-03-06T16:46:14.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:14.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:14.256391+0000) 2024-03-06T16:46:15.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:15.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:15.256655+0000) 2024-03-06T16:46:16.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:16.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:16.256902+0000) 2024-03-06T16:46:17.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:17.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:17.257106+0000) 2024-03-06T16:46:18.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:18.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:18.257355+0000) 2024-03-06T16:46:19.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:19.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:19.257612+0000) 2024-03-06T16:46:20.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:20.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:20.257827+0000) 2024-03-06T16:46:21.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:21.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:21.258041+0000) 2024-03-06T16:46:22.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:22.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:22.258285+0000) 2024-03-06T16:46:23.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:23.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:23.258525+0000) 2024-03-06T16:46:24.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:24.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:24.258766+0000) 2024-03-06T16:46:25.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:25.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:25.258970+0000) 2024-03-06T16:46:26.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:26.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:26.259212+0000) 2024-03-06T16:46:27.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:27.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:27.259422+0000) 2024-03-06T16:46:28.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:28.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:28.259695+0000) 2024-03-06T16:46:29.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:29.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:29.259902+0000) 2024-03-06T16:46:30.098 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:46:30.106 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:46:30.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:30.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:30.260119+0000) 2024-03-06T16:46:31.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:31.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:31.260358+0000) 2024-03-06T16:46:32.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:32.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:32.260606+0000) 2024-03-06T16:46:33.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:33.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:33.260791+0000) 2024-03-06T16:46:34.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:34.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:34.260996+0000) 2024-03-06T16:46:35.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:35.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:35.261203+0000) 2024-03-06T16:46:36.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:36.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:36.261444+0000) 2024-03-06T16:46:37.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:37.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:37.261695+0000) 2024-03-06T16:46:38.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:38.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:38.261925+0000) 2024-03-06T16:46:39.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:39.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:39.262151+0000) 2024-03-06T16:46:40.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:40.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:40.262390+0000) 2024-03-06T16:46:41.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:41.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:41.262596+0000) 2024-03-06T16:46:42.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:42.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:42.262800+0000) 2024-03-06T16:46:43.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:43.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:43.263001+0000) 2024-03-06T16:46:44.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:44.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:44.263192+0000) 2024-03-06T16:46:45.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:45.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:45.263418+0000) 2024-03-06T16:46:46.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:46.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:46.263645+0000) 2024-03-06T16:46:47.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:47.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:47.263846+0000) 2024-03-06T16:46:48.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:48.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:48.264094+0000) 2024-03-06T16:46:49.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:49.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:49.264331+0000) 2024-03-06T16:46:50.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:50.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:50.264623+0000) 2024-03-06T16:46:51.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:51.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:51.264871+0000) 2024-03-06T16:46:52.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:52.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:52.265096+0000) 2024-03-06T16:46:53.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:53.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:53.265323+0000) 2024-03-06T16:46:54.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:54.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:54.265522+0000) 2024-03-06T16:46:55.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:55.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:55.265720+0000) 2024-03-06T16:46:56.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:56.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:56.265922+0000) 2024-03-06T16:46:57.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:57.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:57.266116+0000) 2024-03-06T16:46:58.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:58.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:58.266353+0000) 2024-03-06T16:46:59.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:46:59.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:46:59.266594+0000) 2024-03-06T16:47:00.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:00.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:00.266828+0000) 2024-03-06T16:47:00.299 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:47:00.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:47:01.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:01.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:01.267038+0000) 2024-03-06T16:47:02.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:02.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:02.267252+0000) 2024-03-06T16:47:03.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:03.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:03.267473+0000) 2024-03-06T16:47:04.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:04.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:04.267685+0000) 2024-03-06T16:47:05.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:05.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:05.267909+0000) 2024-03-06T16:47:06.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:06.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:06.268104+0000) 2024-03-06T16:47:07.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:07.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:07.268327+0000) 2024-03-06T16:47:08.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:08.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:08.268526+0000) 2024-03-06T16:47:09.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:09.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:09.268810+0000) 2024-03-06T16:47:10.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:10.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:10.269045+0000) 2024-03-06T16:47:11.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:11.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:11.269272+0000) 2024-03-06T16:47:12.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:12.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:12.269498+0000) 2024-03-06T16:47:13.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:13.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:13.269727+0000) 2024-03-06T16:47:14.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:14.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:14.269950+0000) 2024-03-06T16:47:15.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:15.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:15.270134+0000) 2024-03-06T16:47:16.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:16.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:16.270358+0000) 2024-03-06T16:47:17.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:17.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:17.270573+0000) 2024-03-06T16:47:18.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:18.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:18.270805+0000) 2024-03-06T16:47:19.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:19.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:19.271029+0000) 2024-03-06T16:47:20.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:20.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:20.271228+0000) 2024-03-06T16:47:21.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:21.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:21.271423+0000) 2024-03-06T16:47:22.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:22.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:22.271657+0000) 2024-03-06T16:47:23.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:23.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:23.271843+0000) 2024-03-06T16:47:24.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:24.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:24.272040+0000) 2024-03-06T16:47:25.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:25.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:25.272272+0000) 2024-03-06T16:47:26.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:26.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:26.272504+0000) 2024-03-06T16:47:27.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:27.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:27.272704+0000) 2024-03-06T16:47:28.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:28.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:28.272891+0000) 2024-03-06T16:47:29.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:29.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:29.273079+0000) 2024-03-06T16:47:30.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:30.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:30.273278+0000) 2024-03-06T16:47:30.544 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:47:30.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:47:31.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:31.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:31.273507+0000) 2024-03-06T16:47:32.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:32.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:32.273738+0000) 2024-03-06T16:47:33.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:33.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:33.273962+0000) 2024-03-06T16:47:34.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:34.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:34.274139+0000) 2024-03-06T16:47:35.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:35.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:35.274364+0000) 2024-03-06T16:47:36.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:36.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:36.274628+0000) 2024-03-06T16:47:37.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:37.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:37.274856+0000) 2024-03-06T16:47:38.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:38.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:38.275080+0000) 2024-03-06T16:47:39.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:39.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:39.275300+0000) 2024-03-06T16:47:40.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:40.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:40.275566+0000) 2024-03-06T16:47:41.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:41.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:41.275793+0000) 2024-03-06T16:47:42.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:42.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:42.275987+0000) 2024-03-06T16:47:43.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:43.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:43.276215+0000) 2024-03-06T16:47:44.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:44.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:44.276444+0000) 2024-03-06T16:47:45.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:45.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:45.276675+0000) 2024-03-06T16:47:46.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:46.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:46.276898+0000) 2024-03-06T16:47:47.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:47.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:47.277128+0000) 2024-03-06T16:47:48.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:48.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:48.277355+0000) 2024-03-06T16:47:49.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:49.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:49.277614+0000) 2024-03-06T16:47:50.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:50.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:50.277804+0000) 2024-03-06T16:47:51.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:51.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:51.277983+0000) 2024-03-06T16:47:52.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:52.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:52.278213+0000) 2024-03-06T16:47:53.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:53.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:53.278429+0000) 2024-03-06T16:47:54.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:54.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:54.278650+0000) 2024-03-06T16:47:55.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:55.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:55.278855+0000) 2024-03-06T16:47:56.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:56.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:56.279058+0000) 2024-03-06T16:47:57.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:57.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:57.279300+0000) 2024-03-06T16:47:58.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:58.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:58.279562+0000) 2024-03-06T16:47:59.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:47:59.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:47:59.279773+0000) 2024-03-06T16:48:00.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:00.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:00.279996+0000) 2024-03-06T16:48:00.715 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:48:00.724 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:48:01.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:01.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:01.280237+0000) 2024-03-06T16:48:02.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:02.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:02.280482+0000) 2024-03-06T16:48:03.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:03.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:03.280727+0000) 2024-03-06T16:48:04.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:04.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:04.280892+0000) 2024-03-06T16:48:05.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:05.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:05.281123+0000) 2024-03-06T16:48:06.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:06.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:06.281307+0000) 2024-03-06T16:48:07.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:07.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:07.281505+0000) 2024-03-06T16:48:08.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:08.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:08.281790+0000) 2024-03-06T16:48:09.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:09.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:09.281979+0000) 2024-03-06T16:48:10.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:10.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:10.282208+0000) 2024-03-06T16:48:11.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:11.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:11.282394+0000) 2024-03-06T16:48:12.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:12.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:12.282656+0000) 2024-03-06T16:48:13.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:13.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:13.282834+0000) 2024-03-06T16:48:14.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:14.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:14.283059+0000) 2024-03-06T16:48:15.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:15.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:15.283241+0000) 2024-03-06T16:48:16.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:16.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:16.283460+0000) 2024-03-06T16:48:17.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:17.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:17.283692+0000) 2024-03-06T16:48:18.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:18.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:18.283881+0000) 2024-03-06T16:48:19.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:19.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:19.284117+0000) 2024-03-06T16:48:20.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:20.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:20.284362+0000) 2024-03-06T16:48:21.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:21.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:21.284644+0000) 2024-03-06T16:48:22.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:22.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:22.284852+0000) 2024-03-06T16:48:23.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:23.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:23.285060+0000) 2024-03-06T16:48:24.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:24.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:24.285302+0000) 2024-03-06T16:48:25.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:25.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:25.285565+0000) 2024-03-06T16:48:26.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:26.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:26.285769+0000) 2024-03-06T16:48:27.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:27.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:27.286010+0000) 2024-03-06T16:48:28.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:28.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:28.286241+0000) 2024-03-06T16:48:29.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:29.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:29.286563+0000) 2024-03-06T16:48:30.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:30.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:30.286788+0000) 2024-03-06T16:48:30.853 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:48:30.861 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:48:31.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:31.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:31.286993+0000) 2024-03-06T16:48:32.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:32.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:32.287195+0000) 2024-03-06T16:48:33.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:33.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:33.287434+0000) 2024-03-06T16:48:34.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:34.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:34.287667+0000) 2024-03-06T16:48:35.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:35.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:35.287830+0000) 2024-03-06T16:48:36.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:36.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:36.288023+0000) 2024-03-06T16:48:37.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:37.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:37.288268+0000) 2024-03-06T16:48:38.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:38.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:38.288513+0000) 2024-03-06T16:48:39.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:39.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:39.288745+0000) 2024-03-06T16:48:40.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:40.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:40.288976+0000) 2024-03-06T16:48:41.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:41.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:41.289207+0000) 2024-03-06T16:48:42.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:42.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:42.289433+0000) 2024-03-06T16:48:43.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:43.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:43.289675+0000) 2024-03-06T16:48:44.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:44.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:44.289848+0000) 2024-03-06T16:48:45.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:45.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:45.290097+0000) 2024-03-06T16:48:46.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:46.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:46.290341+0000) 2024-03-06T16:48:47.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:47.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:47.290610+0000) 2024-03-06T16:48:48.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:48.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:48.290810+0000) 2024-03-06T16:48:49.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:49.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:49.291043+0000) 2024-03-06T16:48:50.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:50.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:50.291292+0000) 2024-03-06T16:48:51.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:51.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:51.291524+0000) 2024-03-06T16:48:52.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:52.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:52.291778+0000) 2024-03-06T16:48:53.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:53.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:53.292038+0000) 2024-03-06T16:48:54.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:54.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:54.292277+0000) 2024-03-06T16:48:55.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:55.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:55.292523+0000) 2024-03-06T16:48:56.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:56.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:56.292768+0000) 2024-03-06T16:48:57.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:57.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:57.292973+0000) 2024-03-06T16:48:58.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:58.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:58.293206+0000) 2024-03-06T16:48:59.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:48:59.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:48:59.293442+0000) 2024-03-06T16:49:00.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:00.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:00.293705+0000) 2024-03-06T16:49:00.999 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:49:01.008 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:49:01.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:01.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:01.293913+0000) 2024-03-06T16:49:02.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:02.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:02.294122+0000) 2024-03-06T16:49:03.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:03.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:03.294358+0000) 2024-03-06T16:49:04.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:04.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:04.294626+0000) 2024-03-06T16:49:05.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:05.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:05.294828+0000) 2024-03-06T16:49:06.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:06.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:06.295032+0000) 2024-03-06T16:49:07.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:07.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:07.295271+0000) 2024-03-06T16:49:08.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:08.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:08.295505+0000) 2024-03-06T16:49:09.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:09.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:09.295701+0000) 2024-03-06T16:49:10.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:10.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:10.295950+0000) 2024-03-06T16:49:11.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:11.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:11.296132+0000) 2024-03-06T16:49:12.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:12.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:12.296360+0000) 2024-03-06T16:49:13.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:13.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:13.296630+0000) 2024-03-06T16:49:14.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:14.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:14.296821+0000) 2024-03-06T16:49:15.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:15.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:15.297031+0000) 2024-03-06T16:49:16.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:16.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:16.297277+0000) 2024-03-06T16:49:17.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:17.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:17.297599+0000) 2024-03-06T16:49:18.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:18.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:18.297804+0000) 2024-03-06T16:49:19.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:19.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:19.298006+0000) 2024-03-06T16:49:20.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:20.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:20.298256+0000) 2024-03-06T16:49:21.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:21.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:21.298492+0000) 2024-03-06T16:49:22.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:22.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:22.298665+0000) 2024-03-06T16:49:23.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:23.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:23.298868+0000) 2024-03-06T16:49:24.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:24.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:24.299069+0000) 2024-03-06T16:49:25.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:25.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:25.299267+0000) 2024-03-06T16:49:26.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:26.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:26.299448+0000) 2024-03-06T16:49:27.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:27.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:27.299645+0000) 2024-03-06T16:49:28.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:28.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:28.299856+0000) 2024-03-06T16:49:29.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:29.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:29.300044+0000) 2024-03-06T16:49:30.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:30.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:30.300277+0000) 2024-03-06T16:49:31.212 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:49:31.221 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:49:31.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:31.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:31.300527+0000) 2024-03-06T16:49:32.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:32.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:32.300733+0000) 2024-03-06T16:49:33.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:33.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:33.300925+0000) 2024-03-06T16:49:34.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:34.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:34.301113+0000) 2024-03-06T16:49:35.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:35.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:35.301319+0000) 2024-03-06T16:49:36.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:36.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:36.301579+0000) 2024-03-06T16:49:37.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:37.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:37.301791+0000) 2024-03-06T16:49:38.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:38.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:38.302034+0000) 2024-03-06T16:49:39.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:39.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:39.302248+0000) 2024-03-06T16:49:40.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:40.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:40.302445+0000) 2024-03-06T16:49:41.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:41.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:41.302682+0000) 2024-03-06T16:49:42.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:42.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:42.302908+0000) 2024-03-06T16:49:43.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:43.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:43.303130+0000) 2024-03-06T16:49:44.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:44.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:44.303359+0000) 2024-03-06T16:49:45.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:45.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:45.303626+0000) 2024-03-06T16:49:46.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:46.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:46.303853+0000) 2024-03-06T16:49:47.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:47.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:47.304083+0000) 2024-03-06T16:49:48.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:48.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:48.304308+0000) 2024-03-06T16:49:49.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:49.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:49.304562+0000) 2024-03-06T16:49:50.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:50.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:50.304804+0000) 2024-03-06T16:49:51.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:51.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:51.304995+0000) 2024-03-06T16:49:52.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:52.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:52.305200+0000) 2024-03-06T16:49:53.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:53.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:53.305432+0000) 2024-03-06T16:49:54.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:54.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:54.305670+0000) 2024-03-06T16:49:55.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:55.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:55.305863+0000) 2024-03-06T16:49:56.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:56.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:56.306083+0000) 2024-03-06T16:49:57.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:57.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:57.306257+0000) 2024-03-06T16:49:58.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:58.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:58.306479+0000) 2024-03-06T16:49:59.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:49:59.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:49:59.306699+0000) 2024-03-06T16:50:00.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:00.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:00.306900+0000) 2024-03-06T16:50:01.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:01.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:01.307124+0000) 2024-03-06T16:50:01.434 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:50:01.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:50:02.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:02.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:02.307348+0000) 2024-03-06T16:50:03.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:03.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:03.307590+0000) 2024-03-06T16:50:04.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:04.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:04.307774+0000) 2024-03-06T16:50:05.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:05.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:05.308003+0000) 2024-03-06T16:50:06.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:06.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:06.308194+0000) 2024-03-06T16:50:07.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:07.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:07.308342+0000) 2024-03-06T16:50:08.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:08.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:08.308517+0000) 2024-03-06T16:50:09.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:09.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:09.308727+0000) 2024-03-06T16:50:10.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:10.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:10.309003+0000) 2024-03-06T16:50:11.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:11.307+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:11.309245+0000) 2024-03-06T16:50:12.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:12.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:12.309523+0000) 2024-03-06T16:50:13.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:13.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:13.309729+0000) 2024-03-06T16:50:14.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:14.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:14.309968+0000) 2024-03-06T16:50:15.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:15.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:15.310196+0000) 2024-03-06T16:50:16.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:16.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:16.310422+0000) 2024-03-06T16:50:17.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:17.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:17.310648+0000) 2024-03-06T16:50:18.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:18.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:18.310871+0000) 2024-03-06T16:50:19.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:19.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:19.311130+0000) 2024-03-06T16:50:20.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:20.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:20.311424+0000) 2024-03-06T16:50:21.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:21.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:21.311701+0000) 2024-03-06T16:50:22.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:22.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:22.311936+0000) 2024-03-06T16:50:23.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:23.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:23.312180+0000) 2024-03-06T16:50:24.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:24.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:24.312459+0000) 2024-03-06T16:50:25.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:25.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:25.312690+0000) 2024-03-06T16:50:26.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:26.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:26.312911+0000) 2024-03-06T16:50:27.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:27.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:27.313145+0000) 2024-03-06T16:50:28.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:28.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:28.313367+0000) 2024-03-06T16:50:29.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:29.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:29.313644+0000) 2024-03-06T16:50:30.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:30.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:30.313838+0000) 2024-03-06T16:50:31.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:31.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:31.314066+0000) 2024-03-06T16:50:31.656 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:50:31.664 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:50:32.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:32.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:32.314295+0000) 2024-03-06T16:50:33.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:33.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:33.314523+0000) 2024-03-06T16:50:34.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:34.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:34.314776+0000) 2024-03-06T16:50:35.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:35.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:35.315025+0000) 2024-03-06T16:50:36.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:36.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:36.315261+0000) 2024-03-06T16:50:37.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:37.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:37.315513+0000) 2024-03-06T16:50:38.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:38.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:38.315811+0000) 2024-03-06T16:50:39.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:39.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:39.316021+0000) 2024-03-06T16:50:40.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:40.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:40.316283+0000) 2024-03-06T16:50:41.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:41.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:41.316480+0000) 2024-03-06T16:50:42.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:42.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:42.316785+0000) 2024-03-06T16:50:43.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:43.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:43.316995+0000) 2024-03-06T16:50:44.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:44.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:44.317240+0000) 2024-03-06T16:50:45.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:45.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:45.317463+0000) 2024-03-06T16:50:46.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:46.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:46.317653+0000) 2024-03-06T16:50:47.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:47.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:47.317882+0000) 2024-03-06T16:50:48.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:48.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:48.318157+0000) 2024-03-06T16:50:49.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:49.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:49.318380+0000) 2024-03-06T16:50:50.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:50.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:50.318665+0000) 2024-03-06T16:50:51.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:51.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:51.318879+0000) 2024-03-06T16:50:52.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:52.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:52.319108+0000) 2024-03-06T16:50:53.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:53.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:53.319305+0000) 2024-03-06T16:50:54.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:54.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:54.319577+0000) 2024-03-06T16:50:55.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:55.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:55.319816+0000) 2024-03-06T16:50:56.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:56.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:56.320072+0000) 2024-03-06T16:50:57.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:57.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:57.320313+0000) 2024-03-06T16:50:58.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:58.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:58.320565+0000) 2024-03-06T16:50:59.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:50:59.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:50:59.320775+0000) 2024-03-06T16:51:00.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:00.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:00.320978+0000) 2024-03-06T16:51:01.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:01.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:01.321221+0000) 2024-03-06T16:51:01.885 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:51:01.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:51:02.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:02.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:02.321452+0000) 2024-03-06T16:51:03.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:03.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:03.321663+0000) 2024-03-06T16:51:04.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:04.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:04.321904+0000) 2024-03-06T16:51:05.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:05.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:05.322129+0000) 2024-03-06T16:51:06.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:06.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:06.322353+0000) 2024-03-06T16:51:07.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:07.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:07.322616+0000) 2024-03-06T16:51:08.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:08.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:08.322801+0000) 2024-03-06T16:51:09.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:09.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:09.323062+0000) 2024-03-06T16:51:10.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:10.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:10.323304+0000) 2024-03-06T16:51:11.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:11.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:11.323515+0000) 2024-03-06T16:51:12.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:12.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:12.323768+0000) 2024-03-06T16:51:13.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:13.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:13.323976+0000) 2024-03-06T16:51:14.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:14.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:14.324165+0000) 2024-03-06T16:51:15.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:15.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:15.324378+0000) 2024-03-06T16:51:16.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:16.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:16.324605+0000) 2024-03-06T16:51:17.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:17.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:17.324804+0000) 2024-03-06T16:51:18.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:18.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:18.325038+0000) 2024-03-06T16:51:19.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:19.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:19.325234+0000) 2024-03-06T16:51:20.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:20.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:20.325494+0000) 2024-03-06T16:51:21.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:21.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:21.325692+0000) 2024-03-06T16:51:22.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:22.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:22.325932+0000) 2024-03-06T16:51:23.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:23.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:23.326159+0000) 2024-03-06T16:51:24.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:24.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:24.326373+0000) 2024-03-06T16:51:25.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:25.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:25.326659+0000) 2024-03-06T16:51:26.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:26.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:26.326845+0000) 2024-03-06T16:51:27.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:27.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:27.327033+0000) 2024-03-06T16:51:28.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:28.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:28.327254+0000) 2024-03-06T16:51:29.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:29.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:29.327478+0000) 2024-03-06T16:51:30.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:30.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:30.327710+0000) 2024-03-06T16:51:31.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:31.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:31.327935+0000) 2024-03-06T16:51:32.048 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:51:32.059 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:51:32.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:32.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:32.328166+0000) 2024-03-06T16:51:33.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:33.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:33.328349+0000) 2024-03-06T16:51:34.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:34.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:34.328609+0000) 2024-03-06T16:51:35.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:35.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:35.328821+0000) 2024-03-06T16:51:36.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:36.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:36.329064+0000) 2024-03-06T16:51:37.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:37.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:37.329295+0000) 2024-03-06T16:51:38.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:38.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:38.329498+0000) 2024-03-06T16:51:39.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:39.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:39.329777+0000) 2024-03-06T16:51:40.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:40.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:40.329965+0000) 2024-03-06T16:51:41.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:41.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:41.330173+0000) 2024-03-06T16:51:42.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:42.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:42.330415+0000) 2024-03-06T16:51:43.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:43.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:43.330647+0000) 2024-03-06T16:51:44.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:44.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:44.330841+0000) 2024-03-06T16:51:45.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:45.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:45.331079+0000) 2024-03-06T16:51:46.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:46.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:46.331349+0000) 2024-03-06T16:51:47.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:47.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:47.331585+0000) 2024-03-06T16:51:48.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:48.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:48.331829+0000) 2024-03-06T16:51:49.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:49.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:49.332006+0000) 2024-03-06T16:51:50.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:50.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:50.332239+0000) 2024-03-06T16:51:51.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:51.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:51.332465+0000) 2024-03-06T16:51:52.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:52.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:52.332722+0000) 2024-03-06T16:51:53.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:53.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:53.332993+0000) 2024-03-06T16:51:54.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:54.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:54.333276+0000) 2024-03-06T16:51:55.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:55.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:55.333588+0000) 2024-03-06T16:51:56.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:56.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:56.333869+0000) 2024-03-06T16:51:57.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:57.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:57.334072+0000) 2024-03-06T16:51:58.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:58.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:58.334254+0000) 2024-03-06T16:51:59.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:51:59.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:51:59.334490+0000) 2024-03-06T16:52:00.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:00.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:00.334725+0000) 2024-03-06T16:52:01.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:01.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:01.334952+0000) 2024-03-06T16:52:02.244 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:52:02.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:52:02.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:02.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:02.335138+0000) 2024-03-06T16:52:03.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:03.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:03.335397+0000) 2024-03-06T16:52:04.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:04.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:04.335668+0000) 2024-03-06T16:52:05.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:05.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:05.335908+0000) 2024-03-06T16:52:06.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:06.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:06.336111+0000) 2024-03-06T16:52:07.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:07.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:07.336335+0000) 2024-03-06T16:52:08.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:08.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:08.336570+0000) 2024-03-06T16:52:09.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:09.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:09.336794+0000) 2024-03-06T16:52:10.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:10.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:10.337014+0000) 2024-03-06T16:52:11.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:11.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:11.337247+0000) 2024-03-06T16:52:12.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:12.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:12.337443+0000) 2024-03-06T16:52:13.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:13.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:13.337687+0000) 2024-03-06T16:52:14.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:14.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:14.337918+0000) 2024-03-06T16:52:15.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:15.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:15.338149+0000) 2024-03-06T16:52:16.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:16.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:16.338352+0000) 2024-03-06T16:52:17.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:17.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:17.338627+0000) 2024-03-06T16:52:18.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:18.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:18.338819+0000) 2024-03-06T16:52:19.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:19.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:19.339039+0000) 2024-03-06T16:52:20.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:20.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:20.339281+0000) 2024-03-06T16:52:21.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:21.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:21.339503+0000) 2024-03-06T16:52:22.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:22.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:22.339701+0000) 2024-03-06T16:52:23.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:23.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:23.339945+0000) 2024-03-06T16:52:24.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:24.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:24.340170+0000) 2024-03-06T16:52:25.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:25.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:25.340396+0000) 2024-03-06T16:52:26.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:26.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:26.340622+0000) 2024-03-06T16:52:27.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:27.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:27.340859+0000) 2024-03-06T16:52:28.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:28.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:28.341088+0000) 2024-03-06T16:52:29.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:29.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:29.341276+0000) 2024-03-06T16:52:30.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:30.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:30.341511+0000) 2024-03-06T16:52:31.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:31.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:31.341769+0000) 2024-03-06T16:52:32.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:32.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:32.341994+0000) 2024-03-06T16:52:32.382 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:52:32.390 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:52:33.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:33.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:33.342222+0000) 2024-03-06T16:52:34.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:34.340+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:34.342484+0000) 2024-03-06T16:52:35.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:35.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:35.342718+0000) 2024-03-06T16:52:36.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:36.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:36.342958+0000) 2024-03-06T16:52:37.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:37.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:37.343213+0000) 2024-03-06T16:52:38.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:38.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:38.343476+0000) 2024-03-06T16:52:39.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:39.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:39.343689+0000) 2024-03-06T16:52:40.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:40.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:40.343990+0000) 2024-03-06T16:52:41.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:41.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:41.344202+0000) 2024-03-06T16:52:42.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:42.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:42.344444+0000) 2024-03-06T16:52:43.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:43.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:43.344678+0000) 2024-03-06T16:52:44.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:44.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:44.344898+0000) 2024-03-06T16:52:45.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:45.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:45.345100+0000) 2024-03-06T16:52:46.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:46.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:46.345347+0000) 2024-03-06T16:52:47.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:47.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:47.345581+0000) 2024-03-06T16:52:48.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:48.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:48.345789+0000) 2024-03-06T16:52:49.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:49.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:49.346021+0000) 2024-03-06T16:52:50.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:50.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:50.346302+0000) 2024-03-06T16:52:51.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:51.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:51.346507+0000) 2024-03-06T16:52:52.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:52.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:52.346754+0000) 2024-03-06T16:52:53.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:53.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:53.346977+0000) 2024-03-06T16:52:54.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:54.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:54.347204+0000) 2024-03-06T16:52:55.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:55.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:55.347427+0000) 2024-03-06T16:52:56.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:56.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:56.347666+0000) 2024-03-06T16:52:57.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:57.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:57.347894+0000) 2024-03-06T16:52:58.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:58.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:58.348101+0000) 2024-03-06T16:52:59.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:52:59.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:52:59.348326+0000) 2024-03-06T16:53:00.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:00.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:00.348632+0000) 2024-03-06T16:53:01.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:01.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:01.348831+0000) 2024-03-06T16:53:02.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:02.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:02.349059+0000) 2024-03-06T16:53:02.517 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:53:02.525 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:53:03.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:03.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:03.349274+0000) 2024-03-06T16:53:04.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:04.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:04.349573+0000) 2024-03-06T16:53:05.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:05.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:05.349849+0000) 2024-03-06T16:53:06.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:06.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:06.350052+0000) 2024-03-06T16:53:07.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:07.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:07.350286+0000) 2024-03-06T16:53:08.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:08.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:08.350501+0000) 2024-03-06T16:53:09.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:09.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:09.350744+0000) 2024-03-06T16:53:10.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:10.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:10.350960+0000) 2024-03-06T16:53:11.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:11.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:11.351192+0000) 2024-03-06T16:53:12.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:12.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:12.351389+0000) 2024-03-06T16:53:13.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:13.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:13.351643+0000) 2024-03-06T16:53:14.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:14.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:14.351841+0000) 2024-03-06T16:53:15.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:15.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:15.352081+0000) 2024-03-06T16:53:16.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:16.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:16.352302+0000) 2024-03-06T16:53:17.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:17.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:17.352503+0000) 2024-03-06T16:53:18.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:18.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:18.352786+0000) 2024-03-06T16:53:19.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:19.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:19.353015+0000) 2024-03-06T16:53:20.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:20.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:20.353250+0000) 2024-03-06T16:53:21.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:21.351+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:21.353442+0000) 2024-03-06T16:53:22.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:22.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:22.353701+0000) 2024-03-06T16:53:23.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:23.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:23.353981+0000) 2024-03-06T16:53:24.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:24.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:24.354262+0000) 2024-03-06T16:53:25.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:25.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:25.354484+0000) 2024-03-06T16:53:26.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:26.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:26.354704+0000) 2024-03-06T16:53:27.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:27.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:27.354927+0000) 2024-03-06T16:53:28.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:28.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:28.355236+0000) 2024-03-06T16:53:29.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:29.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:29.355513+0000) 2024-03-06T16:53:30.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:30.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:30.355865+0000) 2024-03-06T16:53:31.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:31.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:31.356141+0000) 2024-03-06T16:53:32.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:32.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:32.356418+0000) 2024-03-06T16:53:32.665 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:53:32.673 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:53:33.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:33.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:33.356691+0000) 2024-03-06T16:53:34.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:34.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:34.356964+0000) 2024-03-06T16:53:35.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:35.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:35.357243+0000) 2024-03-06T16:53:36.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:36.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:36.357520+0000) 2024-03-06T16:53:37.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:37.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:37.357791+0000) 2024-03-06T16:53:38.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:38.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:38.358039+0000) 2024-03-06T16:53:39.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:39.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:39.358290+0000) 2024-03-06T16:53:40.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:40.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:40.358620+0000) 2024-03-06T16:53:41.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:41.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:41.358900+0000) 2024-03-06T16:53:42.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:42.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:42.359128+0000) 2024-03-06T16:53:43.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:43.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:43.359298+0000) 2024-03-06T16:53:44.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:44.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:44.359474+0000) 2024-03-06T16:53:45.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:45.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:45.359694+0000) 2024-03-06T16:53:46.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:46.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:46.359914+0000) 2024-03-06T16:53:47.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:47.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:47.360140+0000) 2024-03-06T16:53:48.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:48.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:48.360367+0000) 2024-03-06T16:53:49.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:49.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:49.360636+0000) 2024-03-06T16:53:50.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:50.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:50.360869+0000) 2024-03-06T16:53:51.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:51.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:51.361094+0000) 2024-03-06T16:53:52.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:52.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:52.361310+0000) 2024-03-06T16:53:53.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:53.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:53.361573+0000) 2024-03-06T16:53:54.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:54.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:54.361764+0000) 2024-03-06T16:53:55.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:55.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:55.361991+0000) 2024-03-06T16:53:56.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:56.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:56.362216+0000) 2024-03-06T16:53:57.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:57.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:57.362436+0000) 2024-03-06T16:53:58.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:58.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:58.362685+0000) 2024-03-06T16:53:59.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:53:59.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:53:59.362966+0000) 2024-03-06T16:54:00.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:00.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:00.363186+0000) 2024-03-06T16:54:01.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:01.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:01.363421+0000) 2024-03-06T16:54:02.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:02.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:02.363644+0000) 2024-03-06T16:54:02.846 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:54:02.854 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:54:03.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:03.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:03.363833+0000) 2024-03-06T16:54:04.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:04.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:04.364058+0000) 2024-03-06T16:54:05.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:05.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:05.364285+0000) 2024-03-06T16:54:06.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:06.362+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:06.364511+0000) 2024-03-06T16:54:07.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:07.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:07.364766+0000) 2024-03-06T16:54:08.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:08.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:08.364995+0000) 2024-03-06T16:54:09.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:09.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:09.365220+0000) 2024-03-06T16:54:10.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:10.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:10.365508+0000) 2024-03-06T16:54:11.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:11.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:11.365834+0000) 2024-03-06T16:54:12.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:12.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:12.366065+0000) 2024-03-06T16:54:13.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:13.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:13.366265+0000) 2024-03-06T16:54:14.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:14.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:14.366487+0000) 2024-03-06T16:54:15.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:15.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:15.366727+0000) 2024-03-06T16:54:16.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:16.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:16.366910+0000) 2024-03-06T16:54:17.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:17.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:17.367143+0000) 2024-03-06T16:54:18.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:18.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:18.367367+0000) 2024-03-06T16:54:19.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:19.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:19.367637+0000) 2024-03-06T16:54:20.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:20.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:20.367873+0000) 2024-03-06T16:54:21.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:21.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:21.368106+0000) 2024-03-06T16:54:22.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:22.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:22.368331+0000) 2024-03-06T16:54:23.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:23.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:23.368628+0000) 2024-03-06T16:54:24.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:24.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:24.368911+0000) 2024-03-06T16:54:25.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:25.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:25.369182+0000) 2024-03-06T16:54:26.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:26.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:26.369449+0000) 2024-03-06T16:54:27.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:27.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:27.369671+0000) 2024-03-06T16:54:28.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:28.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:28.369886+0000) 2024-03-06T16:54:29.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:29.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:29.370102+0000) 2024-03-06T16:54:30.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:30.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:30.370321+0000) 2024-03-06T16:54:31.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:31.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:31.370584+0000) 2024-03-06T16:54:32.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:32.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:32.370836+0000) 2024-03-06T16:54:33.059 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:54:33.067 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:54:33.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:33.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:33.371116+0000) 2024-03-06T16:54:34.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:34.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:34.371313+0000) 2024-03-06T16:54:35.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:35.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:35.371517+0000) 2024-03-06T16:54:36.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:36.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:36.371844+0000) 2024-03-06T16:54:37.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:37.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:37.372121+0000) 2024-03-06T16:54:38.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:38.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:38.372386+0000) 2024-03-06T16:54:39.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:39.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:39.372614+0000) 2024-03-06T16:54:40.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:40.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:40.372864+0000) 2024-03-06T16:54:41.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:41.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:41.373057+0000) 2024-03-06T16:54:42.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:42.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:42.373280+0000) 2024-03-06T16:54:43.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:43.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:43.373581+0000) 2024-03-06T16:54:44.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:44.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:44.373857+0000) 2024-03-06T16:54:45.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:45.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:45.374099+0000) 2024-03-06T16:54:46.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:46.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:46.374289+0000) 2024-03-06T16:54:47.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:47.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:47.374472+0000) 2024-03-06T16:54:48.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:48.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:48.374720+0000) 2024-03-06T16:54:49.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:49.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:49.374945+0000) 2024-03-06T16:54:50.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:50.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:50.375152+0000) 2024-03-06T16:54:51.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:51.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:51.375356+0000) 2024-03-06T16:54:52.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:52.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:52.375579+0000) 2024-03-06T16:54:53.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:53.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:53.375824+0000) 2024-03-06T16:54:54.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:54.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:54.376005+0000) 2024-03-06T16:54:55.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:55.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:55.376231+0000) 2024-03-06T16:54:56.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:56.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:56.376456+0000) 2024-03-06T16:54:57.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:57.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:57.376687+0000) 2024-03-06T16:54:58.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:58.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:58.376917+0000) 2024-03-06T16:54:59.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:54:59.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:54:59.377108+0000) 2024-03-06T16:55:00.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:00.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:00.377367+0000) 2024-03-06T16:55:01.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:01.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:01.377637+0000) 2024-03-06T16:55:02.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:02.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:02.377884+0000) 2024-03-06T16:55:03.266 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:55:03.273 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:55:03.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:03.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:03.378123+0000) 2024-03-06T16:55:04.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:04.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:04.378370+0000) 2024-03-06T16:55:05.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:05.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:05.378614+0000) 2024-03-06T16:55:06.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:06.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:06.378841+0000) 2024-03-06T16:55:07.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:07.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:07.379101+0000) 2024-03-06T16:55:08.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:08.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:08.379327+0000) 2024-03-06T16:55:09.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:09.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:09.379585+0000) 2024-03-06T16:55:10.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:10.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:10.379838+0000) 2024-03-06T16:55:11.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:11.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:11.380025+0000) 2024-03-06T16:55:12.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:12.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:12.380249+0000) 2024-03-06T16:55:13.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:13.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:13.380458+0000) 2024-03-06T16:55:14.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:14.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:14.380695+0000) 2024-03-06T16:55:15.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:15.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:15.381003+0000) 2024-03-06T16:55:16.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:16.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:16.381200+0000) 2024-03-06T16:55:17.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:17.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:17.381440+0000) 2024-03-06T16:55:18.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:18.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:18.381695+0000) 2024-03-06T16:55:19.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:19.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:19.381904+0000) 2024-03-06T16:55:20.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:20.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:20.382106+0000) 2024-03-06T16:55:21.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:21.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:21.382340+0000) 2024-03-06T16:55:22.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:22.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:22.382576+0000) 2024-03-06T16:55:23.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:23.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:23.382858+0000) 2024-03-06T16:55:24.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:24.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:24.383072+0000) 2024-03-06T16:55:25.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:25.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:25.383281+0000) 2024-03-06T16:55:26.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:26.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:26.383519+0000) 2024-03-06T16:55:27.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:27.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:27.383810+0000) 2024-03-06T16:55:28.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:28.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:28.384019+0000) 2024-03-06T16:55:29.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:29.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:29.384254+0000) 2024-03-06T16:55:30.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:30.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:30.384476+0000) 2024-03-06T16:55:31.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:31.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:31.384723+0000) 2024-03-06T16:55:32.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:32.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:32.384968+0000) 2024-03-06T16:55:33.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:33.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:33.385168+0000) 2024-03-06T16:55:33.412 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:55:33.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:55:34.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:34.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:34.385410+0000) 2024-03-06T16:55:35.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:35.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:35.385641+0000) 2024-03-06T16:55:36.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:36.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:36.385835+0000) 2024-03-06T16:55:37.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:37.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:37.386016+0000) 2024-03-06T16:55:38.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:38.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:38.386271+0000) 2024-03-06T16:55:39.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:39.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:39.386584+0000) 2024-03-06T16:55:40.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:40.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:40.386797+0000) 2024-03-06T16:55:41.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:41.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:41.387025+0000) 2024-03-06T16:55:42.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:42.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:42.387306+0000) 2024-03-06T16:55:43.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:43.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:43.387629+0000) 2024-03-06T16:55:44.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:44.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:44.387892+0000) 2024-03-06T16:55:45.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:45.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:45.388091+0000) 2024-03-06T16:55:46.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:46.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:46.388327+0000) 2024-03-06T16:55:47.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:47.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:47.388580+0000) 2024-03-06T16:55:48.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:48.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:48.388848+0000) 2024-03-06T16:55:49.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:49.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:49.389127+0000) 2024-03-06T16:55:50.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:50.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:50.389427+0000) 2024-03-06T16:55:51.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:51.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:51.389706+0000) 2024-03-06T16:55:52.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:52.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:52.389972+0000) 2024-03-06T16:55:53.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:53.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:53.390249+0000) 2024-03-06T16:55:54.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:54.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:54.390493+0000) 2024-03-06T16:55:55.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:55.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:55.390726+0000) 2024-03-06T16:55:56.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:56.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:56.390950+0000) 2024-03-06T16:55:57.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:57.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:57.391249+0000) 2024-03-06T16:55:58.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:58.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:58.391445+0000) 2024-03-06T16:55:59.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:55:59.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:55:59.391690+0000) 2024-03-06T16:56:00.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:00.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:00.391900+0000) 2024-03-06T16:56:01.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:01.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:01.392117+0000) 2024-03-06T16:56:02.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:02.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:02.392379+0000) 2024-03-06T16:56:03.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:03.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:03.392628+0000) 2024-03-06T16:56:03.633 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:56:03.641 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:56:04.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:04.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:04.392871+0000) 2024-03-06T16:56:05.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:05.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:05.393097+0000) 2024-03-06T16:56:06.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:06.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:06.393323+0000) 2024-03-06T16:56:07.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:07.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:07.393567+0000) 2024-03-06T16:56:08.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:08.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:08.393800+0000) 2024-03-06T16:56:09.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:09.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:09.394014+0000) 2024-03-06T16:56:10.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:10.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:10.394274+0000) 2024-03-06T16:56:11.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:11.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:11.394506+0000) 2024-03-06T16:56:12.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:12.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:12.394784+0000) 2024-03-06T16:56:13.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:13.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:13.394977+0000) 2024-03-06T16:56:14.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:14.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:14.395223+0000) 2024-03-06T16:56:15.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:15.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:15.395449+0000) 2024-03-06T16:56:16.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:16.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:16.395662+0000) 2024-03-06T16:56:17.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:17.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:17.395870+0000) 2024-03-06T16:56:18.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:18.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:18.396107+0000) 2024-03-06T16:56:19.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:19.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:19.396322+0000) 2024-03-06T16:56:20.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:20.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:20.396572+0000) 2024-03-06T16:56:21.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:21.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:21.396814+0000) 2024-03-06T16:56:22.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:22.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:22.397063+0000) 2024-03-06T16:56:23.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:23.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:23.397317+0000) 2024-03-06T16:56:24.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:24.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:24.397597+0000) 2024-03-06T16:56:25.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:25.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:25.397839+0000) 2024-03-06T16:56:26.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:26.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:26.398076+0000) 2024-03-06T16:56:27.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:27.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:27.398310+0000) 2024-03-06T16:56:28.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:28.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:28.398607+0000) 2024-03-06T16:56:29.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:29.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:29.398825+0000) 2024-03-06T16:56:30.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:30.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:30.399086+0000) 2024-03-06T16:56:31.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:31.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:31.399269+0000) 2024-03-06T16:56:32.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:32.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:32.399493+0000) 2024-03-06T16:56:33.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:33.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:33.399732+0000) 2024-03-06T16:56:33.829 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:56:33.837 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:56:34.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:34.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:34.399928+0000) 2024-03-06T16:56:35.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:35.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:35.400152+0000) 2024-03-06T16:56:36.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:36.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:36.400372+0000) 2024-03-06T16:56:37.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:37.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:37.400617+0000) 2024-03-06T16:56:38.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:38.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:38.400834+0000) 2024-03-06T16:56:39.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:39.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:39.401061+0000) 2024-03-06T16:56:40.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:40.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:40.401305+0000) 2024-03-06T16:56:41.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:41.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:41.401497+0000) 2024-03-06T16:56:42.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:42.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:42.401765+0000) 2024-03-06T16:56:43.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:43.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:43.402049+0000) 2024-03-06T16:56:44.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:44.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:44.402258+0000) 2024-03-06T16:56:45.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:45.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:45.402562+0000) 2024-03-06T16:56:46.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:46.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:46.402829+0000) 2024-03-06T16:56:47.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:47.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:47.403065+0000) 2024-03-06T16:56:48.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:48.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:48.403345+0000) 2024-03-06T16:56:49.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:49.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:49.403581+0000) 2024-03-06T16:56:50.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:50.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:50.403801+0000) 2024-03-06T16:56:51.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:51.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:51.403975+0000) 2024-03-06T16:56:52.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:52.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:52.404204+0000) 2024-03-06T16:56:53.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:53.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:53.404468+0000) 2024-03-06T16:56:54.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:54.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:54.404745+0000) 2024-03-06T16:56:55.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:55.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:55.404953+0000) 2024-03-06T16:56:56.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:56.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:56.405178+0000) 2024-03-06T16:56:57.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:57.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:57.405396+0000) 2024-03-06T16:56:58.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:58.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:58.405641+0000) 2024-03-06T16:56:59.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:56:59.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:56:59.405833+0000) 2024-03-06T16:57:00.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:00.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:00.406046+0000) 2024-03-06T16:57:01.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:01.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:01.406255+0000) 2024-03-06T16:57:02.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:02.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:02.406526+0000) 2024-03-06T16:57:03.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:03.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:03.406846+0000) 2024-03-06T16:57:04.041 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:57:04.050 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:57:04.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:04.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:04.407121+0000) 2024-03-06T16:57:05.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:05.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:05.407399+0000) 2024-03-06T16:57:06.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:06.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:06.407665+0000) 2024-03-06T16:57:07.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:07.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:07.407948+0000) 2024-03-06T16:57:08.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:08.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:08.408226+0000) 2024-03-06T16:57:09.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:09.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:09.408420+0000) 2024-03-06T16:57:10.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:10.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:10.408669+0000) 2024-03-06T16:57:11.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:11.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:11.408896+0000) 2024-03-06T16:57:12.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:12.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:12.409122+0000) 2024-03-06T16:57:13.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:13.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:13.409360+0000) 2024-03-06T16:57:14.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:14.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:14.409628+0000) 2024-03-06T16:57:15.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:15.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:15.409823+0000) 2024-03-06T16:57:16.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:16.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:16.410004+0000) 2024-03-06T16:57:17.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:17.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:17.410227+0000) 2024-03-06T16:57:18.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:18.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:18.410456+0000) 2024-03-06T16:57:19.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:19.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:19.410704+0000) 2024-03-06T16:57:20.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:20.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:20.410906+0000) 2024-03-06T16:57:21.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:21.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:21.411129+0000) 2024-03-06T16:57:22.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:22.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:22.411357+0000) 2024-03-06T16:57:23.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:23.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:23.411619+0000) 2024-03-06T16:57:24.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:24.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:24.411802+0000) 2024-03-06T16:57:25.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:25.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:25.412028+0000) 2024-03-06T16:57:26.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:26.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:26.412251+0000) 2024-03-06T16:57:27.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:27.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:27.412469+0000) 2024-03-06T16:57:28.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:28.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:28.412737+0000) 2024-03-06T16:57:29.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:29.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:29.413018+0000) 2024-03-06T16:57:30.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:30.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:30.413293+0000) 2024-03-06T16:57:31.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:31.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:31.413615+0000) 2024-03-06T16:57:32.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:32.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:32.413847+0000) 2024-03-06T16:57:33.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:33.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:33.414128+0000) 2024-03-06T16:57:34.180 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:57:34.189 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:57:34.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:34.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:34.414410+0000) 2024-03-06T16:57:35.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:35.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:35.414676+0000) 2024-03-06T16:57:36.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:36.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:36.414952+0000) 2024-03-06T16:57:37.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:37.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:37.415154+0000) 2024-03-06T16:57:38.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:38.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:38.415441+0000) 2024-03-06T16:57:39.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:39.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:39.415674+0000) 2024-03-06T16:57:40.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:40.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:40.415908+0000) 2024-03-06T16:57:41.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:41.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:41.416131+0000) 2024-03-06T16:57:42.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:42.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:42.416382+0000) 2024-03-06T16:57:43.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:43.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:43.416682+0000) 2024-03-06T16:57:44.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:44.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:44.416883+0000) 2024-03-06T16:57:45.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:45.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:45.417067+0000) 2024-03-06T16:57:46.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:46.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:46.417261+0000) 2024-03-06T16:57:47.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:47.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:47.417495+0000) 2024-03-06T16:57:48.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:48.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:48.417720+0000) 2024-03-06T16:57:49.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:49.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:49.417948+0000) 2024-03-06T16:57:50.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:50.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:50.418171+0000) 2024-03-06T16:57:51.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:51.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:51.418446+0000) 2024-03-06T16:57:52.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:52.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:52.418730+0000) 2024-03-06T16:57:53.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:53.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:53.419008+0000) 2024-03-06T16:57:54.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:54.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:54.419275+0000) 2024-03-06T16:57:55.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:55.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:55.419473+0000) 2024-03-06T16:57:56.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:56.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:56.419670+0000) 2024-03-06T16:57:57.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:57.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:57.419912+0000) 2024-03-06T16:57:58.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:58.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:58.420136+0000) 2024-03-06T16:57:59.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:57:59.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:57:59.420336+0000) 2024-03-06T16:58:00.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:00.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:00.420631+0000) 2024-03-06T16:58:01.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:01.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:01.420830+0000) 2024-03-06T16:58:02.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:02.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:02.421051+0000) 2024-03-06T16:58:03.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:03.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:03.421312+0000) 2024-03-06T16:58:04.384 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:58:04.397 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:58:04.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:04.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:04.421471+0000) 2024-03-06T16:58:05.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:05.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:05.421715+0000) 2024-03-06T16:58:06.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:06.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:06.421937+0000) 2024-03-06T16:58:07.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:07.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:07.422160+0000) 2024-03-06T16:58:08.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:08.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:08.422353+0000) 2024-03-06T16:58:09.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:09.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:09.422633+0000) 2024-03-06T16:58:10.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:10.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:10.422869+0000) 2024-03-06T16:58:11.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:11.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:11.423088+0000) 2024-03-06T16:58:12.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:12.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:12.423311+0000) 2024-03-06T16:58:13.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:13.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:13.423559+0000) 2024-03-06T16:58:14.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:14.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:14.423787+0000) 2024-03-06T16:58:15.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:15.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:15.424018+0000) 2024-03-06T16:58:16.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:16.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:16.424244+0000) 2024-03-06T16:58:17.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:17.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:17.424432+0000) 2024-03-06T16:58:18.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:18.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:18.424670+0000) 2024-03-06T16:58:19.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:19.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:19.424896+0000) 2024-03-06T16:58:20.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:20.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:20.425130+0000) 2024-03-06T16:58:21.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:21.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:21.425353+0000) 2024-03-06T16:58:22.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:22.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:22.425609+0000) 2024-03-06T16:58:23.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:23.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:23.425805+0000) 2024-03-06T16:58:24.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:24.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:24.426044+0000) 2024-03-06T16:58:25.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:25.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:25.426245+0000) 2024-03-06T16:58:26.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:26.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:26.426485+0000) 2024-03-06T16:58:27.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:27.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:27.426711+0000) 2024-03-06T16:58:28.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:28.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:28.426926+0000) 2024-03-06T16:58:29.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:29.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:29.427177+0000) 2024-03-06T16:58:30.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:30.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:30.427398+0000) 2024-03-06T16:58:31.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:31.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:31.427636+0000) 2024-03-06T16:58:32.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:32.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:32.427890+0000) 2024-03-06T16:58:33.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:33.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:33.428116+0000) 2024-03-06T16:58:34.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:34.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:34.428307+0000) 2024-03-06T16:58:34.597 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:58:34.608 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:58:35.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:35.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:35.428591+0000) 2024-03-06T16:58:36.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:36.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:36.428835+0000) 2024-03-06T16:58:37.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:37.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:37.429037+0000) 2024-03-06T16:58:38.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:38.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:38.429268+0000) 2024-03-06T16:58:39.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:39.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:39.429582+0000) 2024-03-06T16:58:40.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:40.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:40.429902+0000) 2024-03-06T16:58:41.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:41.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:41.430108+0000) 2024-03-06T16:58:42.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:42.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:42.430314+0000) 2024-03-06T16:58:43.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:43.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:43.430584+0000) 2024-03-06T16:58:44.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:44.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:44.430820+0000) 2024-03-06T16:58:45.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:45.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:45.431099+0000) 2024-03-06T16:58:46.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:46.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:46.431380+0000) 2024-03-06T16:58:47.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:47.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:47.431610+0000) 2024-03-06T16:58:48.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:48.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:48.431846+0000) 2024-03-06T16:58:49.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:49.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:49.432066+0000) 2024-03-06T16:58:50.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:50.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:50.432357+0000) 2024-03-06T16:58:51.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:51.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:51.432674+0000) 2024-03-06T16:58:52.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:52.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:52.432955+0000) 2024-03-06T16:58:53.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:53.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:53.433233+0000) 2024-03-06T16:58:54.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:54.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:54.433501+0000) 2024-03-06T16:58:55.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:55.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:55.433758+0000) 2024-03-06T16:58:56.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:56.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:56.433998+0000) 2024-03-06T16:58:57.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:57.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:57.434202+0000) 2024-03-06T16:58:58.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:58.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:58.434446+0000) 2024-03-06T16:58:59.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:58:59.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:58:59.434693+0000) 2024-03-06T16:59:00.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:00.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:00.434922+0000) 2024-03-06T16:59:01.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:01.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:01.435148+0000) 2024-03-06T16:59:02.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:02.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:02.435373+0000) 2024-03-06T16:59:03.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:03.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:03.435637+0000) 2024-03-06T16:59:04.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:04.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:04.435880+0000) 2024-03-06T16:59:04.803 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:59:04.811 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:59:05.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:05.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:05.436120+0000) 2024-03-06T16:59:06.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:06.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:06.436344+0000) 2024-03-06T16:59:07.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:07.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:07.436625+0000) 2024-03-06T16:59:08.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:08.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:08.436868+0000) 2024-03-06T16:59:09.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:09.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:09.437106+0000) 2024-03-06T16:59:10.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:10.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:10.437420+0000) 2024-03-06T16:59:11.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:11.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:11.437667+0000) 2024-03-06T16:59:12.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:12.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:12.437903+0000) 2024-03-06T16:59:13.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:13.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:13.438144+0000) 2024-03-06T16:59:14.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:14.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:14.438437+0000) 2024-03-06T16:59:15.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:15.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:15.438714+0000) 2024-03-06T16:59:16.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:16.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:16.438942+0000) 2024-03-06T16:59:17.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:17.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:17.439169+0000) 2024-03-06T16:59:18.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:18.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:18.439411+0000) 2024-03-06T16:59:19.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:19.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:19.439647+0000) 2024-03-06T16:59:20.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:20.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:20.439883+0000) 2024-03-06T16:59:21.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:21.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:21.440070+0000) 2024-03-06T16:59:22.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:22.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:22.440270+0000) 2024-03-06T16:59:23.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:23.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:23.440465+0000) 2024-03-06T16:59:24.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:24.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:24.440721+0000) 2024-03-06T16:59:25.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:25.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:25.440989+0000) 2024-03-06T16:59:26.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:26.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:26.441211+0000) 2024-03-06T16:59:27.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:27.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:27.441486+0000) 2024-03-06T16:59:28.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:28.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:28.441772+0000) 2024-03-06T16:59:29.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:29.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:29.442039+0000) 2024-03-06T16:59:30.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:30.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:30.442343+0000) 2024-03-06T16:59:31.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:31.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:31.442647+0000) 2024-03-06T16:59:32.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:32.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:32.442928+0000) 2024-03-06T16:59:33.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:33.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:33.443142+0000) 2024-03-06T16:59:34.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:34.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:34.443341+0000) 2024-03-06T16:59:35.015 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:59:35.024 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T16:59:35.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:35.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:35.443607+0000) 2024-03-06T16:59:36.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:36.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:36.443837+0000) 2024-03-06T16:59:37.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:37.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:37.444062+0000) 2024-03-06T16:59:38.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:38.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:38.444285+0000) 2024-03-06T16:59:39.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:39.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:39.444510+0000) 2024-03-06T16:59:40.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:40.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:40.444782+0000) 2024-03-06T16:59:41.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:41.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:41.444981+0000) 2024-03-06T16:59:42.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:42.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:42.445230+0000) 2024-03-06T16:59:43.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:43.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:43.445452+0000) 2024-03-06T16:59:44.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:44.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:44.445689+0000) 2024-03-06T16:59:45.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:45.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:45.445969+0000) 2024-03-06T16:59:46.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:46.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:46.446235+0000) 2024-03-06T16:59:47.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:47.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:47.446512+0000) 2024-03-06T16:59:48.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:48.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:48.446840+0000) 2024-03-06T16:59:49.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:49.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:49.447085+0000) 2024-03-06T16:59:50.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:50.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:50.447325+0000) 2024-03-06T16:59:51.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:51.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:51.447601+0000) 2024-03-06T16:59:52.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:52.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:52.447845+0000) 2024-03-06T16:59:53.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:53.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:53.448031+0000) 2024-03-06T16:59:54.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:54.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:54.448263+0000) 2024-03-06T16:59:55.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:55.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:55.448464+0000) 2024-03-06T16:59:56.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:56.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:56.448726+0000) 2024-03-06T16:59:57.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:57.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:57.448969+0000) 2024-03-06T16:59:58.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:58.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:58.449198+0000) 2024-03-06T16:59:59.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T16:59:59.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T15:59:59.449406+0000) 2024-03-06T17:00:00.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:00.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:00.449702+0000) 2024-03-06T17:00:01.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:01.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:01.449979+0000) 2024-03-06T17:00:02.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:02.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:02.450258+0000) 2024-03-06T17:00:03.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:03.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:03.450565+0000) 2024-03-06T17:00:04.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:04.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:04.450848+0000) 2024-03-06T17:00:05.140 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:00:05.148 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:00:05.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:05.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:05.451094+0000) 2024-03-06T17:00:06.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:06.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:06.451366+0000) 2024-03-06T17:00:07.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:07.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:07.451654+0000) 2024-03-06T17:00:08.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:08.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:08.451953+0000) 2024-03-06T17:00:09.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:09.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:09.452198+0000) 2024-03-06T17:00:10.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:10.451+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:10.452478+0000) 2024-03-06T17:00:11.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:11.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:11.452726+0000) 2024-03-06T17:00:12.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:12.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:12.453003+0000) 2024-03-06T17:00:13.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:13.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:13.453266+0000) 2024-03-06T17:00:14.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:14.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:14.453471+0000) 2024-03-06T17:00:15.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:15.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:15.453733+0000) 2024-03-06T17:00:16.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:16.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:16.453987+0000) 2024-03-06T17:00:17.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:17.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:17.454190+0000) 2024-03-06T17:00:18.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:18.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:18.454415+0000) 2024-03-06T17:00:19.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:19.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:19.454644+0000) 2024-03-06T17:00:20.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:20.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:20.454880+0000) 2024-03-06T17:00:21.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:21.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:21.455101+0000) 2024-03-06T17:00:22.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:22.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:22.455364+0000) 2024-03-06T17:00:23.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:23.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:23.455655+0000) 2024-03-06T17:00:24.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:24.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:24.455877+0000) 2024-03-06T17:00:25.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:25.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:25.456089+0000) 2024-03-06T17:00:26.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:26.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:26.456324+0000) 2024-03-06T17:00:27.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:27.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:27.456500+0000) 2024-03-06T17:00:28.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:28.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:28.456727+0000) 2024-03-06T17:00:29.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:29.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:29.456954+0000) 2024-03-06T17:00:30.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:30.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:30.457199+0000) 2024-03-06T17:00:31.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:31.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:31.457433+0000) 2024-03-06T17:00:32.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:32.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:32.457661+0000) 2024-03-06T17:00:33.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:33.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:33.457871+0000) 2024-03-06T17:00:34.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:34.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:34.458053+0000) 2024-03-06T17:00:35.341 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:00:35.352 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:00:35.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:35.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:35.458243+0000) 2024-03-06T17:00:36.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:36.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:36.458470+0000) 2024-03-06T17:00:37.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:37.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:37.458713+0000) 2024-03-06T17:00:38.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:38.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:38.458936+0000) 2024-03-06T17:00:39.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:39.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:39.459136+0000) 2024-03-06T17:00:40.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:40.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:40.459354+0000) 2024-03-06T17:00:41.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:41.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:41.459614+0000) 2024-03-06T17:00:42.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:42.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:42.459824+0000) 2024-03-06T17:00:43.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:43.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:43.459982+0000) 2024-03-06T17:00:44.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:44.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:44.460174+0000) 2024-03-06T17:00:45.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:45.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:45.460367+0000) 2024-03-06T17:00:46.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:46.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:46.460602+0000) 2024-03-06T17:00:47.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:47.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:47.460840+0000) 2024-03-06T17:00:48.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:48.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:48.461068+0000) 2024-03-06T17:00:49.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:49.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:49.461295+0000) 2024-03-06T17:00:50.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:50.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:50.461566+0000) 2024-03-06T17:00:51.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:51.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:51.461785+0000) 2024-03-06T17:00:52.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:52.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:52.462010+0000) 2024-03-06T17:00:53.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:53.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:53.462228+0000) 2024-03-06T17:00:54.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:54.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:54.462458+0000) 2024-03-06T17:00:55.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:55.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:55.462688+0000) 2024-03-06T17:00:56.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:56.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:56.462929+0000) 2024-03-06T17:00:57.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:57.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:57.463128+0000) 2024-03-06T17:00:58.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:58.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:58.463344+0000) 2024-03-06T17:00:59.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:00:59.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:00:59.463636+0000) 2024-03-06T17:01:00.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:00.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:00.463869+0000) 2024-03-06T17:01:01.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:01.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:01.464032+0000) 2024-03-06T17:01:02.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:02.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:02.464261+0000) 2024-03-06T17:01:03.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:03.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:03.464483+0000) 2024-03-06T17:01:04.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:04.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:04.464709+0000) 2024-03-06T17:01:05.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:05.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:05.464903+0000) 2024-03-06T17:01:05.545 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:01:05.553 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:01:06.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:06.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:06.465148+0000) 2024-03-06T17:01:07.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:07.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:07.465373+0000) 2024-03-06T17:01:08.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:08.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:08.465602+0000) 2024-03-06T17:01:09.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:09.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:09.465850+0000) 2024-03-06T17:01:10.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:10.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:10.466107+0000) 2024-03-06T17:01:11.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:11.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:11.466313+0000) 2024-03-06T17:01:12.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:12.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:12.466586+0000) 2024-03-06T17:01:13.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:13.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:13.466815+0000) 2024-03-06T17:01:14.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:14.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:14.467022+0000) 2024-03-06T17:01:15.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:15.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:15.467265+0000) 2024-03-06T17:01:16.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:16.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:16.467507+0000) 2024-03-06T17:01:17.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:17.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:17.467789+0000) 2024-03-06T17:01:18.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:18.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:18.467995+0000) 2024-03-06T17:01:19.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:19.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:19.468205+0000) 2024-03-06T17:01:20.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:20.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:20.468412+0000) 2024-03-06T17:01:21.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:21.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:21.468637+0000) 2024-03-06T17:01:22.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:22.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:22.468826+0000) 2024-03-06T17:01:23.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:23.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:23.468997+0000) 2024-03-06T17:01:24.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:24.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:24.469176+0000) 2024-03-06T17:01:25.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:25.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:25.469358+0000) 2024-03-06T17:01:26.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:26.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:26.469567+0000) 2024-03-06T17:01:27.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:27.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:27.469754+0000) 2024-03-06T17:01:28.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:28.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:28.469938+0000) 2024-03-06T17:01:29.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:29.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:29.470184+0000) 2024-03-06T17:01:30.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:30.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:30.470396+0000) 2024-03-06T17:01:31.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:31.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:31.470640+0000) 2024-03-06T17:01:32.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:32.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:32.470844+0000) 2024-03-06T17:01:33.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:33.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:33.471062+0000) 2024-03-06T17:01:34.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:34.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:34.471292+0000) 2024-03-06T17:01:35.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:35.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:35.471519+0000) 2024-03-06T17:01:35.733 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:01:35.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:01:36.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:36.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:36.471784+0000) 2024-03-06T17:01:37.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:37.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:37.471983+0000) 2024-03-06T17:01:38.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:38.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:38.472201+0000) 2024-03-06T17:01:39.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:39.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:39.472440+0000) 2024-03-06T17:01:40.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:40.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:40.472695+0000) 2024-03-06T17:01:41.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:41.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:41.472914+0000) 2024-03-06T17:01:42.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:42.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:42.473120+0000) 2024-03-06T17:01:43.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:43.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:43.473361+0000) 2024-03-06T17:01:44.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:44.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:44.473603+0000) 2024-03-06T17:01:45.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:45.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:45.473831+0000) 2024-03-06T17:01:46.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:46.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:46.474025+0000) 2024-03-06T17:01:47.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:47.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:47.474247+0000) 2024-03-06T17:01:48.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:48.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:48.474502+0000) 2024-03-06T17:01:49.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:49.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:49.474796+0000) 2024-03-06T17:01:50.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:50.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:50.474991+0000) 2024-03-06T17:01:51.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:51.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:51.475209+0000) 2024-03-06T17:01:52.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:52.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:52.475436+0000) 2024-03-06T17:01:53.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:53.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:53.475684+0000) 2024-03-06T17:01:54.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:54.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:54.475939+0000) 2024-03-06T17:01:55.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:55.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:55.476171+0000) 2024-03-06T17:01:56.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:56.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:56.476396+0000) 2024-03-06T17:01:57.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:57.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:57.476656+0000) 2024-03-06T17:01:58.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:58.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:58.476878+0000) 2024-03-06T17:01:59.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:01:59.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:01:59.477113+0000) 2024-03-06T17:02:00.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:00.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:00.477352+0000) 2024-03-06T17:02:01.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:01.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:01.477621+0000) 2024-03-06T17:02:02.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:02.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:02.477832+0000) 2024-03-06T17:02:03.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:03.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:03.478067+0000) 2024-03-06T17:02:04.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:04.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:04.478295+0000) 2024-03-06T17:02:05.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:05.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:05.478603+0000) 2024-03-06T17:02:05.946 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:02:05.957 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:02:06.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:06.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:06.478843+0000) 2024-03-06T17:02:07.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:07.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:07.479088+0000) 2024-03-06T17:02:08.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:08.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:08.479313+0000) 2024-03-06T17:02:09.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:09.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:09.479590+0000) 2024-03-06T17:02:10.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:10.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:10.479896+0000) 2024-03-06T17:02:11.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:11.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:11.480133+0000) 2024-03-06T17:02:12.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:12.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:12.480415+0000) 2024-03-06T17:02:13.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:13.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:13.480697+0000) 2024-03-06T17:02:14.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:14.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:14.480979+0000) 2024-03-06T17:02:15.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:15.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:15.481260+0000) 2024-03-06T17:02:16.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:16.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:16.481523+0000) 2024-03-06T17:02:17.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:17.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:17.481857+0000) 2024-03-06T17:02:18.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:18.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:18.482055+0000) 2024-03-06T17:02:19.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:19.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:19.482297+0000) 2024-03-06T17:02:20.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:20.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:20.482516+0000) 2024-03-06T17:02:21.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:21.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:21.482799+0000) 2024-03-06T17:02:22.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:22.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:22.483032+0000) 2024-03-06T17:02:23.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:23.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:23.483250+0000) 2024-03-06T17:02:24.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:24.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:24.483494+0000) 2024-03-06T17:02:25.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:25.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:25.483748+0000) 2024-03-06T17:02:26.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:26.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:26.483951+0000) 2024-03-06T17:02:27.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:27.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:27.484127+0000) 2024-03-06T17:02:28.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:28.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:28.484346+0000) 2024-03-06T17:02:29.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:29.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:29.484628+0000) 2024-03-06T17:02:30.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:30.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:30.484873+0000) 2024-03-06T17:02:31.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:31.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:31.485107+0000) 2024-03-06T17:02:32.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:32.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:32.485387+0000) 2024-03-06T17:02:33.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:33.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:33.485630+0000) 2024-03-06T17:02:34.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:34.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:34.485878+0000) 2024-03-06T17:02:35.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:35.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:35.486109+0000) 2024-03-06T17:02:36.150 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:02:36.159 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:02:36.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:36.484+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:36.486337+0000) 2024-03-06T17:02:37.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:37.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:37.486616+0000) 2024-03-06T17:02:38.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:38.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:38.486834+0000) 2024-03-06T17:02:39.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:39.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:39.487120+0000) 2024-03-06T17:02:40.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:40.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:40.487416+0000) 2024-03-06T17:02:41.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:41.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:41.487710+0000) 2024-03-06T17:02:42.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:42.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:42.487989+0000) 2024-03-06T17:02:43.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:43.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:43.488228+0000) 2024-03-06T17:02:44.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:44.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:44.488465+0000) 2024-03-06T17:02:45.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:45.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:45.488665+0000) 2024-03-06T17:02:46.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:46.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:46.488876+0000) 2024-03-06T17:02:47.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:47.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:47.489120+0000) 2024-03-06T17:02:48.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:48.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:48.489347+0000) 2024-03-06T17:02:49.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:49.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:49.489675+0000) 2024-03-06T17:02:50.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:50.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:50.489994+0000) 2024-03-06T17:02:51.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:51.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:51.490231+0000) 2024-03-06T17:02:52.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:52.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:52.490459+0000) 2024-03-06T17:02:53.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:53.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:53.490739+0000) 2024-03-06T17:02:54.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:54.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:54.491032+0000) 2024-03-06T17:02:55.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:55.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:55.491320+0000) 2024-03-06T17:02:56.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:56.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:56.491569+0000) 2024-03-06T17:02:57.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:57.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:57.491798+0000) 2024-03-06T17:02:58.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:58.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:58.492025+0000) 2024-03-06T17:02:59.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:02:59.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:02:59.492269+0000) 2024-03-06T17:03:00.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:00.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:00.492511+0000) 2024-03-06T17:03:01.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:01.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:01.492778+0000) 2024-03-06T17:03:02.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:02.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:02.492982+0000) 2024-03-06T17:03:03.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:03.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:03.493174+0000) 2024-03-06T17:03:04.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:04.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:04.493444+0000) 2024-03-06T17:03:05.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:05.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:05.493733+0000) 2024-03-06T17:03:06.372 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:03:06.380 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:03:06.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:06.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:06.493972+0000) 2024-03-06T17:03:07.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:07.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:07.494252+0000) 2024-03-06T17:03:08.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:08.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:08.494453+0000) 2024-03-06T17:03:09.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:09.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:09.494696+0000) 2024-03-06T17:03:10.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:10.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:10.494921+0000) 2024-03-06T17:03:11.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:11.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:11.495184+0000) 2024-03-06T17:03:12.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:12.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:12.495467+0000) 2024-03-06T17:03:13.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:13.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:13.495746+0000) 2024-03-06T17:03:14.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:14.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:14.496018+0000) 2024-03-06T17:03:15.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:15.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:15.496304+0000) 2024-03-06T17:03:16.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:16.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:16.496629+0000) 2024-03-06T17:03:17.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:17.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:17.496874+0000) 2024-03-06T17:03:18.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:18.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:18.497090+0000) 2024-03-06T17:03:19.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:19.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:19.497340+0000) 2024-03-06T17:03:20.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:20.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:20.497679+0000) 2024-03-06T17:03:21.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:21.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:21.497944+0000) 2024-03-06T17:03:22.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:22.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:22.498225+0000) 2024-03-06T17:03:23.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:23.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:23.498491+0000) 2024-03-06T17:03:24.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:24.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:24.498808+0000) 2024-03-06T17:03:25.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:25.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:25.499093+0000) 2024-03-06T17:03:26.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:26.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:26.499300+0000) 2024-03-06T17:03:27.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:27.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:27.499515+0000) 2024-03-06T17:03:28.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:28.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:28.499790+0000) 2024-03-06T17:03:29.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:29.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:29.500015+0000) 2024-03-06T17:03:30.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:30.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:30.500335+0000) 2024-03-06T17:03:31.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:31.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:31.500605+0000) 2024-03-06T17:03:32.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:32.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:32.500832+0000) 2024-03-06T17:03:33.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:33.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:33.501064+0000) 2024-03-06T17:03:34.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:34.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:34.501299+0000) 2024-03-06T17:03:35.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:35.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:35.501567+0000) 2024-03-06T17:03:36.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:36.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:36.501793+0000) 2024-03-06T17:03:36.544 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:03:36.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:03:37.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:37.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:37.502022+0000) 2024-03-06T17:03:38.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:38.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:38.502252+0000) 2024-03-06T17:03:39.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:39.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:39.502518+0000) 2024-03-06T17:03:40.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:40.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:40.502876+0000) 2024-03-06T17:03:41.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:41.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:41.503155+0000) 2024-03-06T17:03:42.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:42.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:42.503399+0000) 2024-03-06T17:03:43.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:43.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:43.503651+0000) 2024-03-06T17:03:44.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:44.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:44.503846+0000) 2024-03-06T17:03:45.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:45.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:45.504075+0000) 2024-03-06T17:03:46.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:46.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:46.504302+0000) 2024-03-06T17:03:47.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:47.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:47.504607+0000) 2024-03-06T17:03:48.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:48.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:48.504902+0000) 2024-03-06T17:03:49.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:49.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:49.505187+0000) 2024-03-06T17:03:50.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:50.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:50.505411+0000) 2024-03-06T17:03:51.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:51.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:51.505650+0000) 2024-03-06T17:03:52.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:52.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:52.505830+0000) 2024-03-06T17:03:53.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:53.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:53.506051+0000) 2024-03-06T17:03:54.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:54.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:54.506263+0000) 2024-03-06T17:03:55.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:55.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:55.506524+0000) 2024-03-06T17:03:56.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:56.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:56.506781+0000) 2024-03-06T17:03:57.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:57.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:57.507026+0000) 2024-03-06T17:03:58.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:58.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:58.507261+0000) 2024-03-06T17:03:59.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:03:59.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:03:59.507501+0000) 2024-03-06T17:04:00.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:00.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:00.507760+0000) 2024-03-06T17:04:01.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:01.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:01.508004+0000) 2024-03-06T17:04:02.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:02.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:02.508249+0000) 2024-03-06T17:04:03.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:03.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:03.508456+0000) 2024-03-06T17:04:04.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:04.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:04.508692+0000) 2024-03-06T17:04:05.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:05.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:05.508885+0000) 2024-03-06T17:04:06.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:06.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:06.509068+0000) 2024-03-06T17:04:06.889 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:04:06.898 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:04:07.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:07.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:07.509257+0000) 2024-03-06T17:04:08.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:08.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:08.509516+0000) 2024-03-06T17:04:09.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:09.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:09.509806+0000) 2024-03-06T17:04:10.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:10.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:10.510067+0000) 2024-03-06T17:04:11.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:11.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:11.510278+0000) 2024-03-06T17:04:12.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:12.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:12.510517+0000) 2024-03-06T17:04:13.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:13.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:13.510815+0000) 2024-03-06T17:04:14.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:14.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:14.511009+0000) 2024-03-06T17:04:15.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:15.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:15.511200+0000) 2024-03-06T17:04:16.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:16.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:16.511420+0000) 2024-03-06T17:04:17.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:17.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:17.511637+0000) 2024-03-06T17:04:18.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:18.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:18.511859+0000) 2024-03-06T17:04:19.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:19.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:19.512059+0000) 2024-03-06T17:04:20.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:20.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:20.512320+0000) 2024-03-06T17:04:21.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:21.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:21.512560+0000) 2024-03-06T17:04:22.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:22.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:22.512791+0000) 2024-03-06T17:04:23.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:23.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:23.513010+0000) 2024-03-06T17:04:24.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:24.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:24.513244+0000) 2024-03-06T17:04:25.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:25.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:25.513475+0000) 2024-03-06T17:04:26.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:26.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:26.513707+0000) 2024-03-06T17:04:27.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:27.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:27.513918+0000) 2024-03-06T17:04:28.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:28.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:28.514141+0000) 2024-03-06T17:04:29.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:29.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:29.514331+0000) 2024-03-06T17:04:30.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:30.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:30.514629+0000) 2024-03-06T17:04:31.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:31.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:31.514858+0000) 2024-03-06T17:04:32.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:32.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:32.515011+0000) 2024-03-06T17:04:33.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:33.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:33.515195+0000) 2024-03-06T17:04:34.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:34.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:34.515416+0000) 2024-03-06T17:04:35.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:35.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:35.515686+0000) 2024-03-06T17:04:36.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:36.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:36.515934+0000) 2024-03-06T17:04:37.085 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:04:37.093 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:04:37.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:37.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:37.516166+0000) 2024-03-06T17:04:38.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:38.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:38.516393+0000) 2024-03-06T17:04:39.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:39.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:39.516659+0000) 2024-03-06T17:04:40.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:40.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:40.516863+0000) 2024-03-06T17:04:41.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:41.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:41.517085+0000) 2024-03-06T17:04:42.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:42.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:42.517307+0000) 2024-03-06T17:04:43.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:43.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:43.517487+0000) 2024-03-06T17:04:44.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:44.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:44.517722+0000) 2024-03-06T17:04:45.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:45.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:45.517955+0000) 2024-03-06T17:04:46.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:46.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:46.518192+0000) 2024-03-06T17:04:47.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:47.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:47.518385+0000) 2024-03-06T17:04:48.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:48.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:48.518671+0000) 2024-03-06T17:04:49.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:49.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:49.518898+0000) 2024-03-06T17:04:50.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:50.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:50.519128+0000) 2024-03-06T17:04:51.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:51.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:51.519347+0000) 2024-03-06T17:04:52.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:52.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:52.519616+0000) 2024-03-06T17:04:53.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:53.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:53.519853+0000) 2024-03-06T17:04:54.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:54.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:54.520078+0000) 2024-03-06T17:04:55.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:55.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:55.520273+0000) 2024-03-06T17:04:56.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:56.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:56.520500+0000) 2024-03-06T17:04:57.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:57.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:57.520743+0000) 2024-03-06T17:04:58.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:58.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:58.520967+0000) 2024-03-06T17:04:59.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:04:59.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:04:59.521196+0000) 2024-03-06T17:05:00.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:00.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:00.521438+0000) 2024-03-06T17:05:01.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:01.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:01.521673+0000) 2024-03-06T17:05:02.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:02.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:02.521875+0000) 2024-03-06T17:05:03.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:03.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:03.522097+0000) 2024-03-06T17:05:04.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:04.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:04.522324+0000) 2024-03-06T17:05:05.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:05.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:05.522574+0000) 2024-03-06T17:05:06.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:06.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:06.522852+0000) 2024-03-06T17:05:07.302 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:05:07.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:05:07.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:07.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:07.523107+0000) 2024-03-06T17:05:08.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:08.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:08.523350+0000) 2024-03-06T17:05:09.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:09.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:09.523634+0000) 2024-03-06T17:05:10.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:10.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:10.523897+0000) 2024-03-06T17:05:11.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:11.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:11.524139+0000) 2024-03-06T17:05:12.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:12.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:12.524392+0000) 2024-03-06T17:05:13.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:13.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:13.524654+0000) 2024-03-06T17:05:14.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:14.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:14.524895+0000) 2024-03-06T17:05:15.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:15.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:15.525147+0000) 2024-03-06T17:05:16.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:16.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:16.525390+0000) 2024-03-06T17:05:17.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:17.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:17.525664+0000) 2024-03-06T17:05:18.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:18.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:18.525890+0000) 2024-03-06T17:05:19.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:19.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:19.526116+0000) 2024-03-06T17:05:20.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:20.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:20.526333+0000) 2024-03-06T17:05:21.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:21.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:21.526592+0000) 2024-03-06T17:05:22.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:22.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:22.526794+0000) 2024-03-06T17:05:23.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:23.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:23.526985+0000) 2024-03-06T17:05:24.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:24.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:24.527209+0000) 2024-03-06T17:05:25.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:25.526+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:25.527437+0000) 2024-03-06T17:05:26.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:26.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:26.527667+0000) 2024-03-06T17:05:27.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:27.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:27.527893+0000) 2024-03-06T17:05:28.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:28.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:28.528116+0000) 2024-03-06T17:05:29.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:29.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:29.528338+0000) 2024-03-06T17:05:30.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:30.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:30.528627+0000) 2024-03-06T17:05:31.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:31.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:31.528806+0000) 2024-03-06T17:05:32.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:32.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:32.529028+0000) 2024-03-06T17:05:33.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:33.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:33.529212+0000) 2024-03-06T17:05:34.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:34.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:34.529440+0000) 2024-03-06T17:05:35.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:35.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:35.529684+0000) 2024-03-06T17:05:36.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:36.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:36.529908+0000) 2024-03-06T17:05:37.511 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:05:37.518 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:05:37.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:37.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:37.530173+0000) 2024-03-06T17:05:38.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:38.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:38.530401+0000) 2024-03-06T17:05:39.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:39.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:39.530672+0000) 2024-03-06T17:05:40.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:40.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:40.530907+0000) 2024-03-06T17:05:41.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:41.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:41.531130+0000) 2024-03-06T17:05:42.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:42.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:42.531374+0000) 2024-03-06T17:05:43.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:43.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:43.531610+0000) 2024-03-06T17:05:44.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:44.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:44.531840+0000) 2024-03-06T17:05:45.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:45.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:45.532075+0000) 2024-03-06T17:05:46.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:46.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:46.532264+0000) 2024-03-06T17:05:47.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:47.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:47.532478+0000) 2024-03-06T17:05:48.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:48.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:48.532725+0000) 2024-03-06T17:05:49.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:49.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:49.532953+0000) 2024-03-06T17:05:50.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:50.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:50.533204+0000) 2024-03-06T17:05:51.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:51.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:51.533426+0000) 2024-03-06T17:05:52.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:52.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:52.533644+0000) 2024-03-06T17:05:53.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:53.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:53.533866+0000) 2024-03-06T17:05:54.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:54.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:54.534085+0000) 2024-03-06T17:05:55.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:55.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:55.534311+0000) 2024-03-06T17:05:56.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:56.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:56.534564+0000) 2024-03-06T17:05:57.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:57.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:57.534752+0000) 2024-03-06T17:05:58.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:58.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:58.534970+0000) 2024-03-06T17:05:59.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:05:59.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:05:59.535189+0000) 2024-03-06T17:06:00.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:00.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:00.535473+0000) 2024-03-06T17:06:01.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:01.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:01.535656+0000) 2024-03-06T17:06:02.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:02.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:02.535845+0000) 2024-03-06T17:06:03.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:03.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:03.536076+0000) 2024-03-06T17:06:04.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:04.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:04.536267+0000) 2024-03-06T17:06:05.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:05.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:05.536489+0000) 2024-03-06T17:06:06.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:06.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:06.536721+0000) 2024-03-06T17:06:07.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:07.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:07.536945+0000) 2024-03-06T17:06:07.716 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:06:07.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:06:08.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:08.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:08.537139+0000) 2024-03-06T17:06:09.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:09.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:09.537376+0000) 2024-03-06T17:06:10.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:10.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:10.537585+0000) 2024-03-06T17:06:11.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:11.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:11.537776+0000) 2024-03-06T17:06:12.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:12.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:12.538017+0000) 2024-03-06T17:06:13.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:13.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:13.538194+0000) 2024-03-06T17:06:14.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:14.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:14.538417+0000) 2024-03-06T17:06:15.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:15.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:15.538639+0000) 2024-03-06T17:06:16.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:16.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:16.538867+0000) 2024-03-06T17:06:17.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:17.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:17.539092+0000) 2024-03-06T17:06:18.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:18.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:18.539317+0000) 2024-03-06T17:06:19.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:19.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:19.539568+0000) 2024-03-06T17:06:20.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:20.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:20.539816+0000) 2024-03-06T17:06:21.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:21.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:21.540048+0000) 2024-03-06T17:06:22.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:22.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:22.540293+0000) 2024-03-06T17:06:23.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:23.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:23.540472+0000) 2024-03-06T17:06:24.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:24.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:24.540690+0000) 2024-03-06T17:06:25.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:25.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:25.540930+0000) 2024-03-06T17:06:26.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:26.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:26.541182+0000) 2024-03-06T17:06:27.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:27.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:27.541422+0000) 2024-03-06T17:06:28.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:28.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:28.541668+0000) 2024-03-06T17:06:29.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:29.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:29.541872+0000) 2024-03-06T17:06:30.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:30.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:30.542129+0000) 2024-03-06T17:06:31.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:31.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:31.542353+0000) 2024-03-06T17:06:32.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:32.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:32.542652+0000) 2024-03-06T17:06:33.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:33.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:33.542891+0000) 2024-03-06T17:06:34.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:34.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:34.543085+0000) 2024-03-06T17:06:35.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:35.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:35.543273+0000) 2024-03-06T17:06:36.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:36.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:36.543515+0000) 2024-03-06T17:06:37.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:37.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:37.543810+0000) 2024-03-06T17:06:37.928 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:06:37.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:06:38.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:38.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:38.544014+0000) 2024-03-06T17:06:39.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:39.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:39.544203+0000) 2024-03-06T17:06:40.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:40.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:40.544461+0000) 2024-03-06T17:06:41.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:41.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:41.544692+0000) 2024-03-06T17:06:42.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:42.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:42.544918+0000) 2024-03-06T17:06:43.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:43.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:43.545198+0000) 2024-03-06T17:06:44.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:44.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:44.545410+0000) 2024-03-06T17:06:45.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:45.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:45.545696+0000) 2024-03-06T17:06:46.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:46.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:46.545975+0000) 2024-03-06T17:06:47.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:47.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:47.546169+0000) 2024-03-06T17:06:48.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:48.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:48.546403+0000) 2024-03-06T17:06:49.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:49.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:49.546623+0000) 2024-03-06T17:06:50.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:50.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:50.546822+0000) 2024-03-06T17:06:51.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:51.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:51.547011+0000) 2024-03-06T17:06:52.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:52.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:52.547233+0000) 2024-03-06T17:06:53.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:53.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:53.547422+0000) 2024-03-06T17:06:54.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:54.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:54.547651+0000) 2024-03-06T17:06:55.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:55.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:55.547883+0000) 2024-03-06T17:06:56.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:56.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:56.548080+0000) 2024-03-06T17:06:57.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:57.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:57.548298+0000) 2024-03-06T17:06:58.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:58.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:58.548571+0000) 2024-03-06T17:06:59.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:06:59.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:06:59.548823+0000) 2024-03-06T17:07:00.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:00.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:00.549082+0000) 2024-03-06T17:07:01.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:01.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:01.549318+0000) 2024-03-06T17:07:02.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:02.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:02.549637+0000) 2024-03-06T17:07:03.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:03.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:03.549852+0000) 2024-03-06T17:07:04.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:04.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:04.550080+0000) 2024-03-06T17:07:05.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:05.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:05.550280+0000) 2024-03-06T17:07:06.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:06.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:06.550524+0000) 2024-03-06T17:07:07.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:07.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:07.550778+0000) 2024-03-06T17:07:08.149 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:07:08.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:07:08.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:08.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:08.551008+0000) 2024-03-06T17:07:09.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:09.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:09.551228+0000) 2024-03-06T17:07:10.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:10.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:10.551464+0000) 2024-03-06T17:07:11.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:11.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:11.551686+0000) 2024-03-06T17:07:12.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:12.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:12.551887+0000) 2024-03-06T17:07:13.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:13.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:13.552103+0000) 2024-03-06T17:07:14.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:14.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:14.552328+0000) 2024-03-06T17:07:15.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:15.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:15.552589+0000) 2024-03-06T17:07:16.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:16.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:16.552788+0000) 2024-03-06T17:07:17.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:17.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:17.553006+0000) 2024-03-06T17:07:18.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:18.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:18.553213+0000) 2024-03-06T17:07:19.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:19.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:19.553401+0000) 2024-03-06T17:07:20.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:20.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:20.553672+0000) 2024-03-06T17:07:21.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:21.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:21.553909+0000) 2024-03-06T17:07:22.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:22.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:22.554119+0000) 2024-03-06T17:07:23.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:23.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:23.554347+0000) 2024-03-06T17:07:24.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:24.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:24.554623+0000) 2024-03-06T17:07:25.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:25.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:25.554866+0000) 2024-03-06T17:07:26.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:26.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:26.555108+0000) 2024-03-06T17:07:27.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:27.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:27.555340+0000) 2024-03-06T17:07:28.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:28.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:28.555628+0000) 2024-03-06T17:07:29.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:29.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:29.555863+0000) 2024-03-06T17:07:30.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:30.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:30.556103+0000) 2024-03-06T17:07:31.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:31.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:31.556361+0000) 2024-03-06T17:07:32.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:32.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:32.556635+0000) 2024-03-06T17:07:33.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:33.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:33.556871+0000) 2024-03-06T17:07:34.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:34.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:34.557103+0000) 2024-03-06T17:07:35.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:35.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:35.557343+0000) 2024-03-06T17:07:36.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:36.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:36.557634+0000) 2024-03-06T17:07:37.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:37.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:37.557869+0000) 2024-03-06T17:07:38.355 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:07:38.365 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:07:38.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:38.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:38.558080+0000) 2024-03-06T17:07:39.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:39.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:39.558304+0000) 2024-03-06T17:07:40.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:40.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:40.558610+0000) 2024-03-06T17:07:41.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:41.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:41.558901+0000) 2024-03-06T17:07:42.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:42.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:42.559095+0000) 2024-03-06T17:07:43.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:43.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:43.559268+0000) 2024-03-06T17:07:44.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:44.558+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:44.559490+0000) 2024-03-06T17:07:45.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:45.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:45.559700+0000) 2024-03-06T17:07:46.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:46.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:46.559901+0000) 2024-03-06T17:07:47.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:47.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:47.560261+0000) 2024-03-06T17:07:48.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:48.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:48.560510+0000) 2024-03-06T17:07:49.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:49.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:49.560756+0000) 2024-03-06T17:07:50.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:50.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:50.561022+0000) 2024-03-06T17:07:51.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:51.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:51.561259+0000) 2024-03-06T17:07:52.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:52.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:52.561504+0000) 2024-03-06T17:07:53.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:53.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:53.561735+0000) 2024-03-06T17:07:54.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:54.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:54.561977+0000) 2024-03-06T17:07:55.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:55.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:55.562205+0000) 2024-03-06T17:07:56.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:56.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:56.562444+0000) 2024-03-06T17:07:57.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:57.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:57.562688+0000) 2024-03-06T17:07:58.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:58.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:58.562900+0000) 2024-03-06T17:07:59.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:07:59.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:07:59.563137+0000) 2024-03-06T17:08:00.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:00.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:00.563361+0000) 2024-03-06T17:08:01.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:01.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:01.563602+0000) 2024-03-06T17:08:02.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:02.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:02.563821+0000) 2024-03-06T17:08:03.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:03.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:03.564052+0000) 2024-03-06T17:08:04.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:04.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:04.564292+0000) 2024-03-06T17:08:05.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:05.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:05.564569+0000) 2024-03-06T17:08:06.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:06.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:06.564819+0000) 2024-03-06T17:08:07.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:07.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:07.565054+0000) 2024-03-06T17:08:08.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:08.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:08.565302+0000) 2024-03-06T17:08:08.583 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:08:08.592 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:08:09.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:09.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:09.565560+0000) 2024-03-06T17:08:10.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:10.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:10.565807+0000) 2024-03-06T17:08:11.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:11.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:11.566042+0000) 2024-03-06T17:08:12.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:12.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:12.566286+0000) 2024-03-06T17:08:13.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:13.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:13.566524+0000) 2024-03-06T17:08:14.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:14.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:14.566810+0000) 2024-03-06T17:08:15.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:15.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:15.567087+0000) 2024-03-06T17:08:16.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:16.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:16.567359+0000) 2024-03-06T17:08:17.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:17.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:17.567633+0000) 2024-03-06T17:08:18.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:18.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:18.567830+0000) 2024-03-06T17:08:19.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:19.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:19.568031+0000) 2024-03-06T17:08:20.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:20.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:20.568295+0000) 2024-03-06T17:08:21.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:21.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:21.568622+0000) 2024-03-06T17:08:22.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:22.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:22.568790+0000) 2024-03-06T17:08:23.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:23.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:23.569006+0000) 2024-03-06T17:08:24.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:24.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:24.569288+0000) 2024-03-06T17:08:25.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:25.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:25.569507+0000) 2024-03-06T17:08:26.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:26.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:26.569750+0000) 2024-03-06T17:08:27.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:27.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:27.569947+0000) 2024-03-06T17:08:28.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:28.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:28.570185+0000) 2024-03-06T17:08:29.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:29.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:29.570408+0000) 2024-03-06T17:08:30.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:30.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:30.570675+0000) 2024-03-06T17:08:31.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:31.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:31.570917+0000) 2024-03-06T17:08:32.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:32.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:32.571098+0000) 2024-03-06T17:08:33.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:33.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:33.571335+0000) 2024-03-06T17:08:34.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:34.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:34.571613+0000) 2024-03-06T17:08:35.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:35.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:35.571859+0000) 2024-03-06T17:08:36.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:36.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:36.572104+0000) 2024-03-06T17:08:37.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:37.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:37.572343+0000) 2024-03-06T17:08:38.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:38.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:38.572567+0000) 2024-03-06T17:08:38.789 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:08:38.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:08:39.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:39.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:39.572801+0000) 2024-03-06T17:08:40.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:40.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:40.573074+0000) 2024-03-06T17:08:41.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:41.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:41.573310+0000) 2024-03-06T17:08:42.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:42.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:42.573622+0000) 2024-03-06T17:08:43.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:43.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:43.573901+0000) 2024-03-06T17:08:44.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:44.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:44.574181+0000) 2024-03-06T17:08:45.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:45.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:45.574449+0000) 2024-03-06T17:08:46.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:46.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:46.574715+0000) 2024-03-06T17:08:47.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:47.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:47.574955+0000) 2024-03-06T17:08:48.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:48.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:48.575183+0000) 2024-03-06T17:08:49.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:49.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:49.575405+0000) 2024-03-06T17:08:50.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:50.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:50.575652+0000) 2024-03-06T17:08:51.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:51.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:51.575874+0000) 2024-03-06T17:08:52.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:52.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:52.576099+0000) 2024-03-06T17:08:53.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:53.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:53.576300+0000) 2024-03-06T17:08:54.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:54.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:54.576518+0000) 2024-03-06T17:08:55.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:55.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:55.576860+0000) 2024-03-06T17:08:56.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:56.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:56.577103+0000) 2024-03-06T17:08:57.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:57.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:57.577320+0000) 2024-03-06T17:08:58.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:58.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:58.577585+0000) 2024-03-06T17:08:59.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:08:59.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:08:59.577770+0000) 2024-03-06T17:09:00.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:00.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:00.578015+0000) 2024-03-06T17:09:01.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:01.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:01.578219+0000) 2024-03-06T17:09:02.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:02.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:02.578453+0000) 2024-03-06T17:09:03.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:03.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:03.578653+0000) 2024-03-06T17:09:04.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:04.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:04.578834+0000) 2024-03-06T17:09:05.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:05.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:05.579053+0000) 2024-03-06T17:09:06.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:06.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:06.579282+0000) 2024-03-06T17:09:07.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:07.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:07.579433+0000) 2024-03-06T17:09:08.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:08.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:08.579637+0000) 2024-03-06T17:09:08.944 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:09:08.952 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:09:09.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:09.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:09.579858+0000) 2024-03-06T17:09:10.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:10.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:10.580080+0000) 2024-03-06T17:09:11.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:11.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:11.580266+0000) 2024-03-06T17:09:12.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:12.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:12.580512+0000) 2024-03-06T17:09:13.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:13.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:13.580798+0000) 2024-03-06T17:09:14.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:14.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:14.581009+0000) 2024-03-06T17:09:15.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:15.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:15.581248+0000) 2024-03-06T17:09:16.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:16.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:16.581441+0000) 2024-03-06T17:09:17.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:17.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:17.581734+0000) 2024-03-06T17:09:18.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:18.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:18.581975+0000) 2024-03-06T17:09:19.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:19.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:19.582206+0000) 2024-03-06T17:09:20.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:20.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:20.582465+0000) 2024-03-06T17:09:21.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:21.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:21.582671+0000) 2024-03-06T17:09:22.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:22.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:22.582887+0000) 2024-03-06T17:09:23.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:23.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:23.583120+0000) 2024-03-06T17:09:24.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:24.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:24.583357+0000) 2024-03-06T17:09:25.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:25.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:25.583678+0000) 2024-03-06T17:09:26.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:26.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:26.583892+0000) 2024-03-06T17:09:27.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:27.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:27.584122+0000) 2024-03-06T17:09:28.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:28.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:28.584334+0000) 2024-03-06T17:09:29.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:29.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:29.584577+0000) 2024-03-06T17:09:30.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:30.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:30.584818+0000) 2024-03-06T17:09:31.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:31.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:31.585037+0000) 2024-03-06T17:09:32.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:32.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:32.585231+0000) 2024-03-06T17:09:33.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:33.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:33.585413+0000) 2024-03-06T17:09:34.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:34.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:34.585640+0000) 2024-03-06T17:09:35.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:35.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:35.585826+0000) 2024-03-06T17:09:36.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:36.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:36.586017+0000) 2024-03-06T17:09:37.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:37.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:37.586205+0000) 2024-03-06T17:09:38.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:38.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:38.586447+0000) 2024-03-06T17:09:39.164 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:09:39.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:09:39.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:39.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:39.586682+0000) 2024-03-06T17:09:40.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:40.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:40.586882+0000) 2024-03-06T17:09:41.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:41.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:41.587069+0000) 2024-03-06T17:09:42.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:42.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:42.587293+0000) 2024-03-06T17:09:43.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:43.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:43.587514+0000) 2024-03-06T17:09:44.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:44.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:44.587694+0000) 2024-03-06T17:09:45.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:45.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:45.587885+0000) 2024-03-06T17:09:46.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:46.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:46.588130+0000) 2024-03-06T17:09:47.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:47.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:47.588355+0000) 2024-03-06T17:09:48.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:48.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:48.588617+0000) 2024-03-06T17:09:49.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:49.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:49.588843+0000) 2024-03-06T17:09:50.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:50.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:50.589085+0000) 2024-03-06T17:09:51.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:51.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:51.589309+0000) 2024-03-06T17:09:52.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:52.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:52.589565+0000) 2024-03-06T17:09:53.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:53.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:53.589764+0000) 2024-03-06T17:09:54.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:54.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:54.590020+0000) 2024-03-06T17:09:55.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:55.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:55.590260+0000) 2024-03-06T17:09:56.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:56.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:56.590450+0000) 2024-03-06T17:09:57.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:57.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:57.590681+0000) 2024-03-06T17:09:58.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:58.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:58.590909+0000) 2024-03-06T17:09:59.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:09:59.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:09:59.591133+0000) 2024-03-06T17:10:00.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:00.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:00.591411+0000) 2024-03-06T17:10:01.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:01.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:01.591625+0000) 2024-03-06T17:10:02.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:02.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:02.591842+0000) 2024-03-06T17:10:03.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:03.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:03.592013+0000) 2024-03-06T17:10:04.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:04.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:04.592241+0000) 2024-03-06T17:10:05.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:05.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:05.592463+0000) 2024-03-06T17:10:06.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:06.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:06.592658+0000) 2024-03-06T17:10:07.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:07.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:07.592860+0000) 2024-03-06T17:10:08.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:08.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:08.593049+0000) 2024-03-06T17:10:09.302 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:10:09.310 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:10:09.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:09.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:09.593275+0000) 2024-03-06T17:10:10.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:10.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:10.593524+0000) 2024-03-06T17:10:11.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:11.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:11.593721+0000) 2024-03-06T17:10:12.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:12.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:12.593940+0000) 2024-03-06T17:10:13.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:13.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:13.594139+0000) 2024-03-06T17:10:14.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:14.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:14.594332+0000) 2024-03-06T17:10:15.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:15.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:15.594624+0000) 2024-03-06T17:10:16.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:16.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:16.594825+0000) 2024-03-06T17:10:17.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:17.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:17.595048+0000) 2024-03-06T17:10:18.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:18.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:18.595271+0000) 2024-03-06T17:10:19.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:19.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:19.595495+0000) 2024-03-06T17:10:20.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:20.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:20.595708+0000) 2024-03-06T17:10:21.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:21.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:21.595878+0000) 2024-03-06T17:10:22.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:22.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:22.596071+0000) 2024-03-06T17:10:23.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:23.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:23.596289+0000) 2024-03-06T17:10:24.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:24.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:24.596515+0000) 2024-03-06T17:10:25.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:25.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:25.596745+0000) 2024-03-06T17:10:26.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:26.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:26.596974+0000) 2024-03-06T17:10:27.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:27.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:27.597193+0000) 2024-03-06T17:10:28.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:28.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:28.597417+0000) 2024-03-06T17:10:29.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:29.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:29.597639+0000) 2024-03-06T17:10:30.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:30.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:30.597850+0000) 2024-03-06T17:10:31.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:31.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:31.598079+0000) 2024-03-06T17:10:32.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:32.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:32.598267+0000) 2024-03-06T17:10:33.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:33.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:33.598466+0000) 2024-03-06T17:10:34.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:34.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:34.598725+0000) 2024-03-06T17:10:35.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:35.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:35.599005+0000) 2024-03-06T17:10:36.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:36.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:36.599286+0000) 2024-03-06T17:10:37.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:37.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:37.599498+0000) 2024-03-06T17:10:38.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:38.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:38.599767+0000) 2024-03-06T17:10:39.490 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:10:39.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:10:39.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:39.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:39.599924+0000) 2024-03-06T17:10:40.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:40.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:40.600152+0000) 2024-03-06T17:10:41.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:41.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:41.600381+0000) 2024-03-06T17:10:42.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:42.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:42.600672+0000) 2024-03-06T17:10:43.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:43.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:43.600937+0000) 2024-03-06T17:10:44.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:44.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:44.601221+0000) 2024-03-06T17:10:45.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:45.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:45.601501+0000) 2024-03-06T17:10:46.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:46.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:46.601793+0000) 2024-03-06T17:10:47.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:47.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:47.602016+0000) 2024-03-06T17:10:48.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:48.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:48.602251+0000) 2024-03-06T17:10:49.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:49.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:49.602435+0000) 2024-03-06T17:10:50.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:50.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:50.602679+0000) 2024-03-06T17:10:51.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:51.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:51.602871+0000) 2024-03-06T17:10:52.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:52.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:52.603098+0000) 2024-03-06T17:10:53.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:53.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:53.603320+0000) 2024-03-06T17:10:54.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:54.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:54.603514+0000) 2024-03-06T17:10:55.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:55.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:55.603847+0000) 2024-03-06T17:10:56.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:56.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:56.604123+0000) 2024-03-06T17:10:57.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:57.603+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:57.604344+0000) 2024-03-06T17:10:58.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:58.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:58.604681+0000) 2024-03-06T17:10:59.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:10:59.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:10:59.604893+0000) 2024-03-06T17:11:00.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:00.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:00.605176+0000) 2024-03-06T17:11:01.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:01.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:01.605379+0000) 2024-03-06T17:11:02.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:02.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:02.605651+0000) 2024-03-06T17:11:03.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:03.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:03.605840+0000) 2024-03-06T17:11:04.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:04.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:04.606081+0000) 2024-03-06T17:11:05.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:05.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:05.606305+0000) 2024-03-06T17:11:06.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:06.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:06.606526+0000) 2024-03-06T17:11:07.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:07.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:07.606778+0000) 2024-03-06T17:11:08.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:08.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:08.607002+0000) 2024-03-06T17:11:09.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:09.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:09.607237+0000) 2024-03-06T17:11:09.704 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:11:09.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:11:10.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:10.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:10.607527+0000) 2024-03-06T17:11:11.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:11.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:11.607791+0000) 2024-03-06T17:11:12.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:12.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:12.608020+0000) 2024-03-06T17:11:13.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:13.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:13.608237+0000) 2024-03-06T17:11:14.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:14.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:14.608454+0000) 2024-03-06T17:11:15.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:15.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:15.608648+0000) 2024-03-06T17:11:16.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:16.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:16.608886+0000) 2024-03-06T17:11:17.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:17.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:17.609105+0000) 2024-03-06T17:11:18.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:18.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:18.609395+0000) 2024-03-06T17:11:19.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:19.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:19.609639+0000) 2024-03-06T17:11:20.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:20.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:20.609803+0000) 2024-03-06T17:11:21.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:21.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:21.610024+0000) 2024-03-06T17:11:22.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:22.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:22.610250+0000) 2024-03-06T17:11:23.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:23.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:23.610466+0000) 2024-03-06T17:11:24.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:24.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:24.610693+0000) 2024-03-06T17:11:25.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:25.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:25.610916+0000) 2024-03-06T17:11:26.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:26.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:26.611150+0000) 2024-03-06T17:11:27.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:27.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:27.611346+0000) 2024-03-06T17:11:28.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:28.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:28.611669+0000) 2024-03-06T17:11:29.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:29.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:29.611940+0000) 2024-03-06T17:11:30.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:30.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:30.612195+0000) 2024-03-06T17:11:31.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:31.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:31.612483+0000) 2024-03-06T17:11:32.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:32.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:32.612759+0000) 2024-03-06T17:11:33.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:33.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:33.613038+0000) 2024-03-06T17:11:34.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:34.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:34.613319+0000) 2024-03-06T17:11:35.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:35.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:35.613629+0000) 2024-03-06T17:11:36.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:36.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:36.613877+0000) 2024-03-06T17:11:37.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:37.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:37.614095+0000) 2024-03-06T17:11:38.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:38.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:38.614323+0000) 2024-03-06T17:11:39.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:39.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:39.614569+0000) 2024-03-06T17:11:39.907 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:11:39.915 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:11:40.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:40.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:40.614743+0000) 2024-03-06T17:11:41.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:41.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:41.614945+0000) 2024-03-06T17:11:42.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:42.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:42.615130+0000) 2024-03-06T17:11:43.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:43.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:43.615359+0000) 2024-03-06T17:11:44.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:44.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:44.615649+0000) 2024-03-06T17:11:45.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:45.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:45.615862+0000) 2024-03-06T17:11:46.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:46.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:46.616070+0000) 2024-03-06T17:11:47.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:47.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:47.616288+0000) 2024-03-06T17:11:48.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:48.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:48.616515+0000) 2024-03-06T17:11:49.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:49.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:49.616791+0000) 2024-03-06T17:11:50.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:50.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:50.616992+0000) 2024-03-06T17:11:51.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:51.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:51.617219+0000) 2024-03-06T17:11:52.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:52.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:52.617484+0000) 2024-03-06T17:11:53.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:53.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:53.617689+0000) 2024-03-06T17:11:54.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:54.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:54.617888+0000) 2024-03-06T17:11:55.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:55.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:55.618112+0000) 2024-03-06T17:11:56.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:56.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:56.618338+0000) 2024-03-06T17:11:57.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:57.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:57.618606+0000) 2024-03-06T17:11:58.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:58.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:58.618830+0000) 2024-03-06T17:11:59.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:11:59.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:11:59.619049+0000) 2024-03-06T17:12:00.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:00.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:00.619243+0000) 2024-03-06T17:12:01.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:01.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:01.619445+0000) 2024-03-06T17:12:02.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:02.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:02.619659+0000) 2024-03-06T17:12:03.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:03.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:03.619920+0000) 2024-03-06T17:12:04.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:04.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:04.620198+0000) 2024-03-06T17:12:05.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:05.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:05.620473+0000) 2024-03-06T17:12:06.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:06.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:06.620754+0000) 2024-03-06T17:12:07.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:07.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:07.620957+0000) 2024-03-06T17:12:08.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:08.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:08.621194+0000) 2024-03-06T17:12:09.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:09.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:09.621389+0000) 2024-03-06T17:12:10.112 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:12:10.120 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:12:10.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:10.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:10.621640+0000) 2024-03-06T17:12:11.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:11.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:11.621865+0000) 2024-03-06T17:12:12.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:12.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:12.622098+0000) 2024-03-06T17:12:13.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:13.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:13.622320+0000) 2024-03-06T17:12:14.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:14.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:14.622579+0000) 2024-03-06T17:12:15.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:15.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:15.622797+0000) 2024-03-06T17:12:16.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:16.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:16.623040+0000) 2024-03-06T17:12:17.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:17.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:17.623301+0000) 2024-03-06T17:12:18.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:18.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:18.623589+0000) 2024-03-06T17:12:19.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:19.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:19.623862+0000) 2024-03-06T17:12:20.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:20.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:20.624125+0000) 2024-03-06T17:12:21.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:21.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:21.624399+0000) 2024-03-06T17:12:22.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:22.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:22.624682+0000) 2024-03-06T17:12:23.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:23.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:23.624948+0000) 2024-03-06T17:12:24.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:24.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:24.625219+0000) 2024-03-06T17:12:25.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:25.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:25.625495+0000) 2024-03-06T17:12:26.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:26.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:26.625822+0000) 2024-03-06T17:12:27.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:27.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:27.626086+0000) 2024-03-06T17:12:28.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:28.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:28.626363+0000) 2024-03-06T17:12:29.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:29.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:29.626651+0000) 2024-03-06T17:12:30.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:30.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:30.626855+0000) 2024-03-06T17:12:31.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:31.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:31.627076+0000) 2024-03-06T17:12:32.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:32.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:32.627301+0000) 2024-03-06T17:12:33.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:33.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:33.627517+0000) 2024-03-06T17:12:34.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:34.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:34.627848+0000) 2024-03-06T17:12:35.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:35.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:35.628081+0000) 2024-03-06T17:12:36.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:36.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:36.628318+0000) 2024-03-06T17:12:37.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:37.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:37.628611+0000) 2024-03-06T17:12:38.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:38.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:38.628838+0000) 2024-03-06T17:12:39.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:39.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:39.629071+0000) 2024-03-06T17:12:40.317 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:12:40.325 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:12:40.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:40.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:40.629336+0000) 2024-03-06T17:12:41.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:41.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:41.629574+0000) 2024-03-06T17:12:42.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:42.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:42.629797+0000) 2024-03-06T17:12:43.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:43.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:43.629997+0000) 2024-03-06T17:12:44.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:44.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:44.630221+0000) 2024-03-06T17:12:45.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:45.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:45.630428+0000) 2024-03-06T17:12:46.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:46.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:46.630684+0000) 2024-03-06T17:12:47.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:47.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:47.630920+0000) 2024-03-06T17:12:48.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:48.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:48.631157+0000) 2024-03-06T17:12:49.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:49.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:49.631394+0000) 2024-03-06T17:12:50.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:50.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:50.631652+0000) 2024-03-06T17:12:51.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:51.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:51.631886+0000) 2024-03-06T17:12:52.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:52.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:52.632125+0000) 2024-03-06T17:12:53.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:53.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:53.632315+0000) 2024-03-06T17:12:54.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:54.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:54.632569+0000) 2024-03-06T17:12:55.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:55.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:55.632799+0000) 2024-03-06T17:12:56.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:56.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:56.633036+0000) 2024-03-06T17:12:57.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:57.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:57.633262+0000) 2024-03-06T17:12:58.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:58.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:58.633488+0000) 2024-03-06T17:12:59.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:12:59.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:12:59.633706+0000) 2024-03-06T17:13:00.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:00.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:00.633947+0000) 2024-03-06T17:13:01.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:01.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:01.634151+0000) 2024-03-06T17:13:02.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:02.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:02.634399+0000) 2024-03-06T17:13:03.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:03.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:03.634644+0000) 2024-03-06T17:13:04.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:04.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:04.634874+0000) 2024-03-06T17:13:05.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:05.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:05.635065+0000) 2024-03-06T17:13:06.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:06.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:06.635289+0000) 2024-03-06T17:13:07.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:07.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:07.635509+0000) 2024-03-06T17:13:08.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:08.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:08.635768+0000) 2024-03-06T17:13:09.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:09.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:09.635958+0000) 2024-03-06T17:13:10.521 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:13:10.529 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:13:10.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:10.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:10.636198+0000) 2024-03-06T17:13:11.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:11.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:11.636452+0000) 2024-03-06T17:13:12.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:12.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:12.636691+0000) 2024-03-06T17:13:13.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:13.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:13.636875+0000) 2024-03-06T17:13:14.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:14.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:14.637118+0000) 2024-03-06T17:13:15.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:15.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:15.637310+0000) 2024-03-06T17:13:16.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:16.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:16.637575+0000) 2024-03-06T17:13:17.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:17.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:17.637801+0000) 2024-03-06T17:13:18.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:18.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:18.638034+0000) 2024-03-06T17:13:19.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:19.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:19.638254+0000) 2024-03-06T17:13:20.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:20.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:20.638496+0000) 2024-03-06T17:13:21.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:21.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:21.638729+0000) 2024-03-06T17:13:22.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:22.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:22.638937+0000) 2024-03-06T17:13:23.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:23.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:23.639155+0000) 2024-03-06T17:13:24.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:24.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:24.639374+0000) 2024-03-06T17:13:25.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:25.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:25.639635+0000) 2024-03-06T17:13:26.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:26.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:26.639801+0000) 2024-03-06T17:13:27.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:27.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:27.639982+0000) 2024-03-06T17:13:28.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:28.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:28.640241+0000) 2024-03-06T17:13:29.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:29.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:29.640424+0000) 2024-03-06T17:13:30.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:30.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:30.640672+0000) 2024-03-06T17:13:31.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:31.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:31.640900+0000) 2024-03-06T17:13:32.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:32.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:32.641129+0000) 2024-03-06T17:13:33.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:33.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:33.641325+0000) 2024-03-06T17:13:34.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:34.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:34.641597+0000) 2024-03-06T17:13:35.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:35.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:35.641805+0000) 2024-03-06T17:13:36.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:36.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:36.642048+0000) 2024-03-06T17:13:37.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:37.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:37.642267+0000) 2024-03-06T17:13:38.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:38.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:38.642492+0000) 2024-03-06T17:13:39.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:39.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:39.642688+0000) 2024-03-06T17:13:40.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:40.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:40.642901+0000) 2024-03-06T17:13:40.718 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:13:40.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:13:41.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:41.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:41.643098+0000) 2024-03-06T17:13:42.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:42.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:42.643339+0000) 2024-03-06T17:13:43.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:43.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:43.643595+0000) 2024-03-06T17:13:44.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:44.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:44.643819+0000) 2024-03-06T17:13:45.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:45.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:45.644010+0000) 2024-03-06T17:13:46.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:46.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:46.644249+0000) 2024-03-06T17:13:47.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:47.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:47.644468+0000) 2024-03-06T17:13:48.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:48.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:48.644693+0000) 2024-03-06T17:13:49.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:49.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:49.644893+0000) 2024-03-06T17:13:50.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:50.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:50.645140+0000) 2024-03-06T17:13:51.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:51.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:51.645366+0000) 2024-03-06T17:13:52.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:52.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:52.645647+0000) 2024-03-06T17:13:53.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:53.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:53.645881+0000) 2024-03-06T17:13:54.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:54.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:54.646072+0000) 2024-03-06T17:13:55.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:55.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:55.646276+0000) 2024-03-06T17:13:56.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:56.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:56.646511+0000) 2024-03-06T17:13:57.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:57.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:57.646800+0000) 2024-03-06T17:13:58.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:58.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:58.647048+0000) 2024-03-06T17:13:59.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:13:59.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:13:59.647289+0000) 2024-03-06T17:14:00.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:00.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:00.647501+0000) 2024-03-06T17:14:01.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:01.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:01.647719+0000) 2024-03-06T17:14:02.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:02.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:02.647918+0000) 2024-03-06T17:14:03.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:03.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:03.648108+0000) 2024-03-06T17:14:04.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:04.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:04.648336+0000) 2024-03-06T17:14:05.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:05.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:05.648565+0000) 2024-03-06T17:14:06.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:06.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:06.648783+0000) 2024-03-06T17:14:07.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:07.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:07.648980+0000) 2024-03-06T17:14:08.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:08.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:08.649212+0000) 2024-03-06T17:14:09.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:09.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:09.649446+0000) 2024-03-06T17:14:10.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:10.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:10.649732+0000) 2024-03-06T17:14:10.864 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:14:10.872 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:14:11.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:11.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:11.649970+0000) 2024-03-06T17:14:12.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:12.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:12.650242+0000) 2024-03-06T17:14:13.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:13.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:13.650478+0000) 2024-03-06T17:14:14.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:14.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:14.650747+0000) 2024-03-06T17:14:15.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:15.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:15.650981+0000) 2024-03-06T17:14:16.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:16.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:16.651170+0000) 2024-03-06T17:14:17.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:17.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:17.651360+0000) 2024-03-06T17:14:18.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:18.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:18.651663+0000) 2024-03-06T17:14:19.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:19.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:19.651856+0000) 2024-03-06T17:14:20.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:20.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:20.652043+0000) 2024-03-06T17:14:21.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:21.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:21.652243+0000) 2024-03-06T17:14:22.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:22.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:22.652438+0000) 2024-03-06T17:14:23.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:23.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:23.652669+0000) 2024-03-06T17:14:24.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:24.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:24.652874+0000) 2024-03-06T17:14:25.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:25.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:25.653094+0000) 2024-03-06T17:14:26.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:26.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:26.653343+0000) 2024-03-06T17:14:27.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:27.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:27.653617+0000) 2024-03-06T17:14:28.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:28.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:28.653816+0000) 2024-03-06T17:14:29.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:29.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:29.654037+0000) 2024-03-06T17:14:30.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:30.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:30.654227+0000) 2024-03-06T17:14:31.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:31.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:31.654465+0000) 2024-03-06T17:14:32.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:32.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:32.654676+0000) 2024-03-06T17:14:33.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:33.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:33.654881+0000) 2024-03-06T17:14:34.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:34.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:34.655078+0000) 2024-03-06T17:14:35.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:35.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:35.655298+0000) 2024-03-06T17:14:36.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:36.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:36.655523+0000) 2024-03-06T17:14:37.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:37.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:37.655791+0000) 2024-03-06T17:14:38.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:38.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:38.655980+0000) 2024-03-06T17:14:39.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:39.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:39.656209+0000) 2024-03-06T17:14:40.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:40.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:40.656406+0000) 2024-03-06T17:14:41.053 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:14:41.061 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:14:41.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:41.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:41.656634+0000) 2024-03-06T17:14:42.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:42.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:42.656866+0000) 2024-03-06T17:14:43.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:43.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:43.657047+0000) 2024-03-06T17:14:44.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:44.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:44.657279+0000) 2024-03-06T17:14:45.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:45.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:45.657498+0000) 2024-03-06T17:14:46.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:46.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:46.657695+0000) 2024-03-06T17:14:47.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:47.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:47.657845+0000) 2024-03-06T17:14:48.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:48.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:48.658037+0000) 2024-03-06T17:14:49.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:49.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:49.658219+0000) 2024-03-06T17:14:50.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:50.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:50.658423+0000) 2024-03-06T17:14:51.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:51.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:51.658641+0000) 2024-03-06T17:14:52.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:52.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:52.658831+0000) 2024-03-06T17:14:53.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:53.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:53.659052+0000) 2024-03-06T17:14:54.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:54.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:54.659280+0000) 2024-03-06T17:14:55.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:55.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:55.659499+0000) 2024-03-06T17:14:56.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:56.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:56.659721+0000) 2024-03-06T17:14:57.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:57.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:57.659944+0000) 2024-03-06T17:14:58.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:58.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:58.660171+0000) 2024-03-06T17:14:59.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:14:59.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:14:59.660387+0000) 2024-03-06T17:15:00.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:00.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:00.660659+0000) 2024-03-06T17:15:01.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:01.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:01.660864+0000) 2024-03-06T17:15:02.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:02.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:02.661076+0000) 2024-03-06T17:15:03.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:03.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:03.661305+0000) 2024-03-06T17:15:04.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:04.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:04.661520+0000) 2024-03-06T17:15:05.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:05.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:05.661773+0000) 2024-03-06T17:15:06.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:06.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:06.661991+0000) 2024-03-06T17:15:07.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:07.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:07.662215+0000) 2024-03-06T17:15:08.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:08.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:08.662437+0000) 2024-03-06T17:15:09.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:09.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:09.662709+0000) 2024-03-06T17:15:10.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:10.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:10.662950+0000) 2024-03-06T17:15:11.248 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:15:11.257 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:15:11.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:11.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:11.663168+0000) 2024-03-06T17:15:12.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:12.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:12.663391+0000) 2024-03-06T17:15:13.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:13.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:13.663608+0000) 2024-03-06T17:15:14.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:14.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:14.663802+0000) 2024-03-06T17:15:15.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:15.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:15.664014+0000) 2024-03-06T17:15:16.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:16.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:16.664254+0000) 2024-03-06T17:15:17.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:17.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:17.664494+0000) 2024-03-06T17:15:18.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:18.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:18.664727+0000) 2024-03-06T17:15:19.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:19.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:19.664963+0000) 2024-03-06T17:15:20.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:20.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:20.665217+0000) 2024-03-06T17:15:21.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:21.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:21.665454+0000) 2024-03-06T17:15:22.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:22.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:22.665694+0000) 2024-03-06T17:15:23.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:23.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:23.665898+0000) 2024-03-06T17:15:24.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:24.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:24.666101+0000) 2024-03-06T17:15:25.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:25.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:25.666318+0000) 2024-03-06T17:15:26.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:26.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:26.666568+0000) 2024-03-06T17:15:27.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:27.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:27.666787+0000) 2024-03-06T17:15:28.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:28.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:28.667014+0000) 2024-03-06T17:15:29.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:29.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:29.667194+0000) 2024-03-06T17:15:30.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:30.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:30.667399+0000) 2024-03-06T17:15:31.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:31.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:31.667643+0000) 2024-03-06T17:15:32.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:32.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:32.667880+0000) 2024-03-06T17:15:33.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:33.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:33.668109+0000) 2024-03-06T17:15:34.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:34.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:34.668289+0000) 2024-03-06T17:15:35.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:35.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:35.668511+0000) 2024-03-06T17:15:36.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:36.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:36.668766+0000) 2024-03-06T17:15:37.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:37.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:37.668987+0000) 2024-03-06T17:15:38.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:38.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:38.669195+0000) 2024-03-06T17:15:39.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:39.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:39.669404+0000) 2024-03-06T17:15:40.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:40.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:40.669652+0000) 2024-03-06T17:15:41.386 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:15:41.396 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:15:41.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:41.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:41.669847+0000) 2024-03-06T17:15:42.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:42.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:42.670077+0000) 2024-03-06T17:15:43.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:43.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:43.670296+0000) 2024-03-06T17:15:44.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:44.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:44.670494+0000) 2024-03-06T17:15:45.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:45.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:45.670724+0000) 2024-03-06T17:15:46.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:46.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:46.670923+0000) 2024-03-06T17:15:47.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:47.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:47.671127+0000) 2024-03-06T17:15:48.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:48.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:48.671341+0000) 2024-03-06T17:15:49.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:49.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:49.671647+0000) 2024-03-06T17:15:50.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:50.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:50.671843+0000) 2024-03-06T17:15:51.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:51.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:51.672068+0000) 2024-03-06T17:15:52.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:52.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:52.672254+0000) 2024-03-06T17:15:53.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:53.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:53.672441+0000) 2024-03-06T17:15:54.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:54.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:54.672667+0000) 2024-03-06T17:15:55.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:55.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:55.672910+0000) 2024-03-06T17:15:56.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:56.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:56.673131+0000) 2024-03-06T17:15:57.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:57.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:57.673405+0000) 2024-03-06T17:15:58.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:58.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:58.673659+0000) 2024-03-06T17:15:59.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:15:59.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:15:59.673880+0000) 2024-03-06T17:16:00.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:00.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:00.674119+0000) 2024-03-06T17:16:01.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:01.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:01.674337+0000) 2024-03-06T17:16:02.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:02.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:02.674606+0000) 2024-03-06T17:16:03.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:03.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:03.674826+0000) 2024-03-06T17:16:04.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:04.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:04.675054+0000) 2024-03-06T17:16:05.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:05.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:05.675271+0000) 2024-03-06T17:16:06.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:06.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:06.675493+0000) 2024-03-06T17:16:07.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:07.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:07.675682+0000) 2024-03-06T17:16:08.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:08.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:08.675910+0000) 2024-03-06T17:16:09.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:09.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:09.676083+0000) 2024-03-06T17:16:10.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:10.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:10.676318+0000) 2024-03-06T17:16:11.600 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:16:11.609 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:16:11.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:11.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:11.676594+0000) 2024-03-06T17:16:12.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:12.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:12.676803+0000) 2024-03-06T17:16:13.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:13.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:13.677042+0000) 2024-03-06T17:16:14.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:14.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:14.677290+0000) 2024-03-06T17:16:15.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:15.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:15.677569+0000) 2024-03-06T17:16:16.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:16.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:16.677777+0000) 2024-03-06T17:16:17.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:17.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:17.678004+0000) 2024-03-06T17:16:18.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:18.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:18.678207+0000) 2024-03-06T17:16:19.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:19.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:19.678432+0000) 2024-03-06T17:16:20.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:20.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:20.678671+0000) 2024-03-06T17:16:21.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:21.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:21.678890+0000) 2024-03-06T17:16:22.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:22.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:22.679135+0000) 2024-03-06T17:16:23.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:23.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:23.679324+0000) 2024-03-06T17:16:24.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:24.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:24.679574+0000) 2024-03-06T17:16:25.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:25.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:25.679790+0000) 2024-03-06T17:16:26.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:26.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:26.679986+0000) 2024-03-06T17:16:27.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:27.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:27.680220+0000) 2024-03-06T17:16:28.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:28.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:28.680460+0000) 2024-03-06T17:16:29.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:29.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:29.680664+0000) 2024-03-06T17:16:30.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:30.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:30.680838+0000) 2024-03-06T17:16:31.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:31.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:31.681037+0000) 2024-03-06T17:16:32.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:32.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:32.681284+0000) 2024-03-06T17:16:33.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:33.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:33.681516+0000) 2024-03-06T17:16:34.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:34.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:34.681774+0000) 2024-03-06T17:16:35.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:35.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:35.682015+0000) 2024-03-06T17:16:36.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:36.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:36.682251+0000) 2024-03-06T17:16:37.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:37.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:37.682488+0000) 2024-03-06T17:16:38.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:38.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:38.682702+0000) 2024-03-06T17:16:39.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:39.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:39.682902+0000) 2024-03-06T17:16:40.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:40.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:40.683196+0000) 2024-03-06T17:16:41.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:41.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:41.683392+0000) 2024-03-06T17:16:41.819 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:16:41.830 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:16:42.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:42.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:42.683640+0000) 2024-03-06T17:16:43.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:43.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:43.683835+0000) 2024-03-06T17:16:44.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:44.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:44.684014+0000) 2024-03-06T17:16:45.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:45.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:45.684285+0000) 2024-03-06T17:16:46.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:46.682+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:46.684471+0000) 2024-03-06T17:16:47.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:47.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:47.684719+0000) 2024-03-06T17:16:48.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:48.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:48.684957+0000) 2024-03-06T17:16:49.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:49.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:49.685214+0000) 2024-03-06T17:16:50.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:50.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:50.685491+0000) 2024-03-06T17:16:51.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:51.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:51.685711+0000) 2024-03-06T17:16:52.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:52.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:52.685929+0000) 2024-03-06T17:16:53.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:53.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:53.686190+0000) 2024-03-06T17:16:54.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:54.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:54.686433+0000) 2024-03-06T17:16:55.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:55.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:55.686668+0000) 2024-03-06T17:16:56.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:56.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:56.686891+0000) 2024-03-06T17:16:57.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:57.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:57.687168+0000) 2024-03-06T17:16:58.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:58.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:58.687434+0000) 2024-03-06T17:16:59.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:16:59.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:16:59.687715+0000) 2024-03-06T17:17:00.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:00.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:00.687979+0000) 2024-03-06T17:17:01.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:01.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:01.688263+0000) 2024-03-06T17:17:02.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:02.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:02.688585+0000) 2024-03-06T17:17:03.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:03.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:03.688821+0000) 2024-03-06T17:17:04.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:04.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:04.689088+0000) 2024-03-06T17:17:05.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:05.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:05.689352+0000) 2024-03-06T17:17:06.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:06.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:06.689618+0000) 2024-03-06T17:17:07.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:07.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:07.689857+0000) 2024-03-06T17:17:08.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:08.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:08.690133+0000) 2024-03-06T17:17:09.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:09.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:09.690328+0000) 2024-03-06T17:17:10.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:10.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:10.690600+0000) 2024-03-06T17:17:11.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:11.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:11.690822+0000) 2024-03-06T17:17:12.034 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:17:12.042 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:17:12.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:12.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:12.691057+0000) 2024-03-06T17:17:13.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:13.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:13.691271+0000) 2024-03-06T17:17:14.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:14.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:14.691586+0000) 2024-03-06T17:17:15.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:15.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:15.691826+0000) 2024-03-06T17:17:16.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:16.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:16.692102+0000) 2024-03-06T17:17:17.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:17.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:17.692377+0000) 2024-03-06T17:17:18.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:18.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:18.692702+0000) 2024-03-06T17:17:19.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:19.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:19.692939+0000) 2024-03-06T17:17:20.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:20.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:20.693230+0000) 2024-03-06T17:17:21.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:21.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:21.693509+0000) 2024-03-06T17:17:22.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:22.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:22.693759+0000) 2024-03-06T17:17:23.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:23.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:23.693965+0000) 2024-03-06T17:17:24.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:24.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:24.694198+0000) 2024-03-06T17:17:25.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:25.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:25.694418+0000) 2024-03-06T17:17:26.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:26.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:26.694637+0000) 2024-03-06T17:17:27.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:27.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:27.694867+0000) 2024-03-06T17:17:28.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:28.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:28.695144+0000) 2024-03-06T17:17:29.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:29.693+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:29.695365+0000) 2024-03-06T17:17:30.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:30.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:30.695638+0000) 2024-03-06T17:17:31.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:31.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:31.695856+0000) 2024-03-06T17:17:32.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:32.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:32.696093+0000) 2024-03-06T17:17:33.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:33.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:33.696318+0000) 2024-03-06T17:17:34.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:34.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:34.696577+0000) 2024-03-06T17:17:35.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:35.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:35.696808+0000) 2024-03-06T17:17:36.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:36.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:36.697023+0000) 2024-03-06T17:17:37.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:37.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:37.697223+0000) 2024-03-06T17:17:38.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:38.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:38.697441+0000) 2024-03-06T17:17:39.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:39.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:39.697718+0000) 2024-03-06T17:17:40.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:40.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:40.697972+0000) 2024-03-06T17:17:41.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:41.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:41.698257+0000) 2024-03-06T17:17:42.238 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:17:42.246 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:17:42.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:42.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:42.698573+0000) 2024-03-06T17:17:43.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:43.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:43.698783+0000) 2024-03-06T17:17:44.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:44.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:44.699015+0000) 2024-03-06T17:17:45.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:45.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:45.699198+0000) 2024-03-06T17:17:46.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:46.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:46.699387+0000) 2024-03-06T17:17:47.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:47.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:47.699670+0000) 2024-03-06T17:17:48.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:48.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:48.699870+0000) 2024-03-06T17:17:49.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:49.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:49.700019+0000) 2024-03-06T17:17:50.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:50.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:50.700245+0000) 2024-03-06T17:17:51.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:51.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:51.700466+0000) 2024-03-06T17:17:52.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:52.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:52.700698+0000) 2024-03-06T17:17:53.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:53.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:53.700890+0000) 2024-03-06T17:17:54.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:54.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:54.701125+0000) 2024-03-06T17:17:55.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:55.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:55.701351+0000) 2024-03-06T17:17:56.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:56.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:56.701614+0000) 2024-03-06T17:17:57.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:57.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:57.701804+0000) 2024-03-06T17:17:58.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:58.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:58.702033+0000) 2024-03-06T17:17:59.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:17:59.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:17:59.702255+0000) 2024-03-06T17:18:00.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:00.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:00.702500+0000) 2024-03-06T17:18:01.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:01.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:01.702701+0000) 2024-03-06T17:18:02.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:02.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:02.702905+0000) 2024-03-06T17:18:03.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:03.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:03.703129+0000) 2024-03-06T17:18:04.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:04.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:04.703353+0000) 2024-03-06T17:18:05.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:05.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:05.703637+0000) 2024-03-06T17:18:06.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:06.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:06.703856+0000) 2024-03-06T17:18:07.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:07.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:07.704080+0000) 2024-03-06T17:18:08.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:08.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:08.704309+0000) 2024-03-06T17:18:09.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:09.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:09.704564+0000) 2024-03-06T17:18:10.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:10.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:10.704780+0000) 2024-03-06T17:18:11.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:11.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:11.705002+0000) 2024-03-06T17:18:12.449 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:18:12.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:18:12.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:12.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:12.705201+0000) 2024-03-06T17:18:13.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:13.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:13.705429+0000) 2024-03-06T17:18:14.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:14.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:14.705656+0000) 2024-03-06T17:18:15.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:15.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:15.705846+0000) 2024-03-06T17:18:16.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:16.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:16.706070+0000) 2024-03-06T17:18:17.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:17.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:17.706288+0000) 2024-03-06T17:18:18.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:18.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:18.706476+0000) 2024-03-06T17:18:19.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:19.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:19.706685+0000) 2024-03-06T17:18:20.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:20.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:20.706892+0000) 2024-03-06T17:18:21.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:21.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:21.707119+0000) 2024-03-06T17:18:22.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:22.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:22.707305+0000) 2024-03-06T17:18:23.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:23.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:23.707576+0000) 2024-03-06T17:18:24.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:24.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:24.707820+0000) 2024-03-06T17:18:25.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:25.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:25.708021+0000) 2024-03-06T17:18:26.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:26.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:26.708263+0000) 2024-03-06T17:18:27.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:27.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:27.708501+0000) 2024-03-06T17:18:28.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:28.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:28.708767+0000) 2024-03-06T17:18:29.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:29.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:29.709008+0000) 2024-03-06T17:18:30.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:30.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:30.709256+0000) 2024-03-06T17:18:31.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:31.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:31.709447+0000) 2024-03-06T17:18:32.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:32.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:32.709657+0000) 2024-03-06T17:18:33.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:33.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:33.709885+0000) 2024-03-06T17:18:34.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:34.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:34.710114+0000) 2024-03-06T17:18:35.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:35.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:35.710333+0000) 2024-03-06T17:18:36.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:36.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:36.710588+0000) 2024-03-06T17:18:37.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:37.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:37.710782+0000) 2024-03-06T17:18:38.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:38.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:38.711013+0000) 2024-03-06T17:18:39.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:39.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:39.711231+0000) 2024-03-06T17:18:40.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:40.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:40.711467+0000) 2024-03-06T17:18:41.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:41.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:41.711679+0000) 2024-03-06T17:18:42.647 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:18:42.655 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:18:42.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:42.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:42.711868+0000) 2024-03-06T17:18:43.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:43.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:43.712037+0000) 2024-03-06T17:18:44.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:44.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:44.712271+0000) 2024-03-06T17:18:45.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:45.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:45.712491+0000) 2024-03-06T17:18:46.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:46.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:46.712713+0000) 2024-03-06T17:18:47.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:47.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:47.712915+0000) 2024-03-06T17:18:48.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:48.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:48.713152+0000) 2024-03-06T17:18:49.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:49.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:49.713373+0000) 2024-03-06T17:18:50.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:50.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:50.713664+0000) 2024-03-06T17:18:51.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:51.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:51.713853+0000) 2024-03-06T17:18:52.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:52.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:52.714090+0000) 2024-03-06T17:18:53.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:53.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:53.714317+0000) 2024-03-06T17:18:54.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:54.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:54.714591+0000) 2024-03-06T17:18:55.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:55.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:55.714820+0000) 2024-03-06T17:18:56.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:56.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:56.715038+0000) 2024-03-06T17:18:57.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:57.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:57.715264+0000) 2024-03-06T17:18:58.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:58.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:58.715493+0000) 2024-03-06T17:18:59.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:18:59.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:18:59.715684+0000) 2024-03-06T17:19:00.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:00.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:00.715883+0000) 2024-03-06T17:19:01.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:01.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:01.716115+0000) 2024-03-06T17:19:02.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:02.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:02.716350+0000) 2024-03-06T17:19:03.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:03.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:03.716640+0000) 2024-03-06T17:19:04.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:04.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:04.716839+0000) 2024-03-06T17:19:05.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:05.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:05.717062+0000) 2024-03-06T17:19:06.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:06.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:06.717247+0000) 2024-03-06T17:19:07.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:07.715+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:07.717490+0000) 2024-03-06T17:19:08.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:08.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:08.717697+0000) 2024-03-06T17:19:09.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:09.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:09.717877+0000) 2024-03-06T17:19:10.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:10.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:10.718074+0000) 2024-03-06T17:19:11.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:11.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:11.718291+0000) 2024-03-06T17:19:12.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:12.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:12.718510+0000) 2024-03-06T17:19:12.885 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:19:12.894 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:19:13.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:13.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:13.718748+0000) 2024-03-06T17:19:14.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:14.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:14.718942+0000) 2024-03-06T17:19:15.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:15.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:15.719150+0000) 2024-03-06T17:19:16.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:16.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:16.719392+0000) 2024-03-06T17:19:17.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:17.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:17.719629+0000) 2024-03-06T17:19:18.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:18.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:18.719854+0000) 2024-03-06T17:19:19.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:19.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:19.720096+0000) 2024-03-06T17:19:20.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:20.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:20.720395+0000) 2024-03-06T17:19:21.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:21.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:21.720648+0000) 2024-03-06T17:19:22.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:22.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:22.720887+0000) 2024-03-06T17:19:23.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:23.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:23.721089+0000) 2024-03-06T17:19:24.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:24.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:24.721361+0000) 2024-03-06T17:19:25.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:25.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:25.721652+0000) 2024-03-06T17:19:26.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:26.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:26.721879+0000) 2024-03-06T17:19:27.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:27.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:27.722105+0000) 2024-03-06T17:19:28.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:28.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:28.722336+0000) 2024-03-06T17:19:29.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:29.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:29.722602+0000) 2024-03-06T17:19:30.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:30.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:30.722846+0000) 2024-03-06T17:19:31.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:31.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:31.723064+0000) 2024-03-06T17:19:32.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:32.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:32.723299+0000) 2024-03-06T17:19:33.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:33.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:33.723524+0000) 2024-03-06T17:19:34.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:34.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:34.723773+0000) 2024-03-06T17:19:35.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:35.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:35.723995+0000) 2024-03-06T17:19:36.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:36.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:36.724172+0000) 2024-03-06T17:19:37.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:37.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:37.724402+0000) 2024-03-06T17:19:38.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:38.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:38.724647+0000) 2024-03-06T17:19:39.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:39.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:39.724870+0000) 2024-03-06T17:19:40.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:40.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:40.725096+0000) 2024-03-06T17:19:41.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:41.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:41.725319+0000) 2024-03-06T17:19:42.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:42.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:42.725588+0000) 2024-03-06T17:19:43.109 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:19:43.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:19:43.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:43.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:43.725782+0000) 2024-03-06T17:19:44.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:44.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:44.726014+0000) 2024-03-06T17:19:45.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:45.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:45.726236+0000) 2024-03-06T17:19:46.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:46.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:46.726462+0000) 2024-03-06T17:19:47.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:47.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:47.726698+0000) 2024-03-06T17:19:48.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:48.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:48.726926+0000) 2024-03-06T17:19:49.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:49.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:49.727105+0000) 2024-03-06T17:19:50.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:50.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:50.727334+0000) 2024-03-06T17:19:51.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:51.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:51.727584+0000) 2024-03-06T17:19:52.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:52.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:52.727826+0000) 2024-03-06T17:19:53.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:53.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:53.728045+0000) 2024-03-06T17:19:54.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:54.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:54.728274+0000) 2024-03-06T17:19:55.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:55.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:55.728499+0000) 2024-03-06T17:19:56.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:56.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:56.728728+0000) 2024-03-06T17:19:57.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:57.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:57.728962+0000) 2024-03-06T17:19:58.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:58.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:58.729240+0000) 2024-03-06T17:19:59.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:19:59.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:19:59.729433+0000) 2024-03-06T17:20:00.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:00.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:00.729674+0000) 2024-03-06T17:20:01.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:01.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:01.729867+0000) 2024-03-06T17:20:02.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:02.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:02.730106+0000) 2024-03-06T17:20:03.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:03.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:03.730360+0000) 2024-03-06T17:20:04.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:04.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:04.730635+0000) 2024-03-06T17:20:05.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:05.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:05.730938+0000) 2024-03-06T17:20:06.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:06.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:06.731130+0000) 2024-03-06T17:20:07.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:07.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:07.731404+0000) 2024-03-06T17:20:08.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:08.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:08.731848+0000) 2024-03-06T17:20:09.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:09.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:09.732114+0000) 2024-03-06T17:20:10.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:10.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:10.732413+0000) 2024-03-06T17:20:11.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:11.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:11.732679+0000) 2024-03-06T17:20:12.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:12.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:12.732898+0000) 2024-03-06T17:20:13.311 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:20:13.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:20:13.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:13.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:13.733148+0000) 2024-03-06T17:20:14.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:14.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:14.733385+0000) 2024-03-06T17:20:15.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:15.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:15.733657+0000) 2024-03-06T17:20:16.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:16.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:16.733879+0000) 2024-03-06T17:20:17.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:17.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:17.734111+0000) 2024-03-06T17:20:18.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:18.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:18.734337+0000) 2024-03-06T17:20:19.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:19.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:19.734526+0000) 2024-03-06T17:20:20.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:20.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:20.734791+0000) 2024-03-06T17:20:21.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:21.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:21.735035+0000) 2024-03-06T17:20:22.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:22.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:22.735299+0000) 2024-03-06T17:20:23.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:23.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:23.735490+0000) 2024-03-06T17:20:24.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:24.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:24.735742+0000) 2024-03-06T17:20:25.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:25.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:25.735922+0000) 2024-03-06T17:20:26.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:26.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:26.736145+0000) 2024-03-06T17:20:27.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:27.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:27.736370+0000) 2024-03-06T17:20:28.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:28.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:28.736636+0000) 2024-03-06T17:20:29.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:29.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:29.737128+0000) 2024-03-06T17:20:30.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:30.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:30.737390+0000) 2024-03-06T17:20:31.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:31.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:31.737640+0000) 2024-03-06T17:20:32.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:32.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:32.737832+0000) 2024-03-06T17:20:33.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:33.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:33.738033+0000) 2024-03-06T17:20:34.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:34.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:34.738247+0000) 2024-03-06T17:20:35.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:35.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:35.738448+0000) 2024-03-06T17:20:36.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:36.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:36.738702+0000) 2024-03-06T17:20:37.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:37.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:37.738954+0000) 2024-03-06T17:20:38.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:38.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:38.739202+0000) 2024-03-06T17:20:39.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:39.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:39.739390+0000) 2024-03-06T17:20:40.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:40.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:40.739683+0000) 2024-03-06T17:20:41.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:41.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:41.739901+0000) 2024-03-06T17:20:42.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:42.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:42.740136+0000) 2024-03-06T17:20:43.536 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:20:43.546 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:20:43.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:43.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:43.740329+0000) 2024-03-06T17:20:44.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:44.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:44.740621+0000) 2024-03-06T17:20:45.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:45.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:45.740828+0000) 2024-03-06T17:20:46.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:46.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:46.741058+0000) 2024-03-06T17:20:47.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:47.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:47.741273+0000) 2024-03-06T17:20:48.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:48.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:48.741568+0000) 2024-03-06T17:20:49.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:49.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:49.741808+0000) 2024-03-06T17:20:50.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:50.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:50.742043+0000) 2024-03-06T17:20:51.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:51.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:51.742314+0000) 2024-03-06T17:20:52.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:52.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:52.742633+0000) 2024-03-06T17:20:53.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:53.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:53.742850+0000) 2024-03-06T17:20:54.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:54.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:54.743048+0000) 2024-03-06T17:20:55.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:55.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:55.743254+0000) 2024-03-06T17:20:56.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:56.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:56.743443+0000) 2024-03-06T17:20:57.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:57.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:57.743689+0000) 2024-03-06T17:20:58.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:58.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:58.743894+0000) 2024-03-06T17:20:59.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:20:59.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:20:59.744157+0000) 2024-03-06T17:21:00.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:00.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:00.744449+0000) 2024-03-06T17:21:01.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:01.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:01.744724+0000) 2024-03-06T17:21:02.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:02.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:02.744954+0000) 2024-03-06T17:21:03.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:03.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:03.745178+0000) 2024-03-06T17:21:04.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:04.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:04.745372+0000) 2024-03-06T17:21:05.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:05.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:05.745640+0000) 2024-03-06T17:21:06.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:06.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:06.745864+0000) 2024-03-06T17:21:07.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:07.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:07.746081+0000) 2024-03-06T17:21:08.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:08.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:08.746307+0000) 2024-03-06T17:21:09.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:09.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:09.746573+0000) 2024-03-06T17:21:10.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:10.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:10.746804+0000) 2024-03-06T17:21:11.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:11.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:11.747007+0000) 2024-03-06T17:21:12.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:12.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:12.747239+0000) 2024-03-06T17:21:13.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:13.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:13.747463+0000) 2024-03-06T17:21:13.771 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:21:13.778 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:21:14.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:14.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:14.747692+0000) 2024-03-06T17:21:15.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:15.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:15.747920+0000) 2024-03-06T17:21:16.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:16.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:16.748139+0000) 2024-03-06T17:21:17.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:17.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:17.748306+0000) 2024-03-06T17:21:18.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:18.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:18.748517+0000) 2024-03-06T17:21:19.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:19.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:19.748855+0000) 2024-03-06T17:21:20.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:20.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:20.749138+0000) 2024-03-06T17:21:21.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:21.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:21.749387+0000) 2024-03-06T17:21:22.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:22.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:22.749659+0000) 2024-03-06T17:21:23.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:23.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:23.749882+0000) 2024-03-06T17:21:24.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:24.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:24.750100+0000) 2024-03-06T17:21:25.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:25.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:25.750281+0000) 2024-03-06T17:21:26.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:26.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:26.750509+0000) 2024-03-06T17:21:27.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:27.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:27.750775+0000) 2024-03-06T17:21:28.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:28.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:28.751008+0000) 2024-03-06T17:21:29.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:29.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:29.751218+0000) 2024-03-06T17:21:30.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:30.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:30.751457+0000) 2024-03-06T17:21:31.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:31.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:31.751696+0000) 2024-03-06T17:21:32.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:32.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:32.751926+0000) 2024-03-06T17:21:33.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:33.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:33.752106+0000) 2024-03-06T17:21:34.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:34.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:34.752305+0000) 2024-03-06T17:21:35.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:35.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:35.752510+0000) 2024-03-06T17:21:36.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:36.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:36.752749+0000) 2024-03-06T17:21:37.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:37.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:37.752962+0000) 2024-03-06T17:21:38.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:38.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:38.753212+0000) 2024-03-06T17:21:39.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:39.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:39.753431+0000) 2024-03-06T17:21:40.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:40.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:40.753690+0000) 2024-03-06T17:21:41.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:41.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:41.753860+0000) 2024-03-06T17:21:42.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:42.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:42.754056+0000) 2024-03-06T17:21:43.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:43.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:43.754336+0000) 2024-03-06T17:21:43.966 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:21:43.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:21:44.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:44.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:44.754522+0000) 2024-03-06T17:21:45.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:45.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:45.754735+0000) 2024-03-06T17:21:46.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:46.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:46.754917+0000) 2024-03-06T17:21:47.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:47.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:47.755095+0000) 2024-03-06T17:21:48.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:48.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:48.755306+0000) 2024-03-06T17:21:49.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:49.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:49.755582+0000) 2024-03-06T17:21:50.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:50.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:50.755797+0000) 2024-03-06T17:21:51.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:51.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:51.756004+0000) 2024-03-06T17:21:52.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:52.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:52.756265+0000) 2024-03-06T17:21:53.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:53.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:53.756489+0000) 2024-03-06T17:21:54.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:54.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:54.756712+0000) 2024-03-06T17:21:55.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:55.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:55.756936+0000) 2024-03-06T17:21:56.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:56.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:56.757160+0000) 2024-03-06T17:21:57.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:57.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:57.757409+0000) 2024-03-06T17:21:58.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:58.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:58.757646+0000) 2024-03-06T17:21:59.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:21:59.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:21:59.757840+0000) 2024-03-06T17:22:00.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:00.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:00.758072+0000) 2024-03-06T17:22:01.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:01.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:01.758292+0000) 2024-03-06T17:22:02.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:02.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:02.758518+0000) 2024-03-06T17:22:03.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:03.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:03.758785+0000) 2024-03-06T17:22:04.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:04.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:04.759016+0000) 2024-03-06T17:22:05.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:05.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:05.759261+0000) 2024-03-06T17:22:06.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:06.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:06.759504+0000) 2024-03-06T17:22:07.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:07.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:07.759801+0000) 2024-03-06T17:22:08.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:08.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:08.760051+0000) 2024-03-06T17:22:09.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:09.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:09.760258+0000) 2024-03-06T17:22:10.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:10.758+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:10.760511+0000) 2024-03-06T17:22:11.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:11.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:11.760780+0000) 2024-03-06T17:22:12.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:12.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:12.761014+0000) 2024-03-06T17:22:13.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:13.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:13.761229+0000) 2024-03-06T17:22:14.163 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:22:14.171 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:22:14.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:14.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:14.761472+0000) 2024-03-06T17:22:15.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:15.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:15.761725+0000) 2024-03-06T17:22:16.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:16.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:16.761977+0000) 2024-03-06T17:22:17.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:17.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:17.762214+0000) 2024-03-06T17:22:18.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:18.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:18.762461+0000) 2024-03-06T17:22:19.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:19.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:19.762697+0000) 2024-03-06T17:22:20.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:20.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:20.762871+0000) 2024-03-06T17:22:21.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:21.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:21.763081+0000) 2024-03-06T17:22:22.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:22.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:22.763313+0000) 2024-03-06T17:22:23.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:23.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:23.763556+0000) 2024-03-06T17:22:24.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:24.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:24.763785+0000) 2024-03-06T17:22:25.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:25.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:25.763972+0000) 2024-03-06T17:22:26.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:26.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:26.764202+0000) 2024-03-06T17:22:27.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:27.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:27.764396+0000) 2024-03-06T17:22:28.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:28.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:28.764621+0000) 2024-03-06T17:22:29.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:29.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:29.764802+0000) 2024-03-06T17:22:30.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:30.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:30.765006+0000) 2024-03-06T17:22:31.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:31.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:31.765201+0000) 2024-03-06T17:22:32.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:32.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:32.765432+0000) 2024-03-06T17:22:33.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:33.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:33.765654+0000) 2024-03-06T17:22:34.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:34.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:34.765882+0000) 2024-03-06T17:22:35.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:35.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:35.766107+0000) 2024-03-06T17:22:36.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:36.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:36.766294+0000) 2024-03-06T17:22:37.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:37.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:37.766567+0000) 2024-03-06T17:22:38.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:38.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:38.766758+0000) 2024-03-06T17:22:39.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:39.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:39.766955+0000) 2024-03-06T17:22:40.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:40.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:40.767170+0000) 2024-03-06T17:22:41.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:41.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:41.767403+0000) 2024-03-06T17:22:42.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:42.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:42.767665+0000) 2024-03-06T17:22:43.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:43.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:43.767905+0000) 2024-03-06T17:22:44.311 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:22:44.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:22:44.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:44.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:44.768138+0000) 2024-03-06T17:22:45.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:45.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:45.768382+0000) 2024-03-06T17:22:46.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:46.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:46.768649+0000) 2024-03-06T17:22:47.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:47.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:47.768866+0000) 2024-03-06T17:22:48.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:48.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:48.769099+0000) 2024-03-06T17:22:49.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:49.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:49.769320+0000) 2024-03-06T17:22:50.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:50.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:50.769565+0000) 2024-03-06T17:22:51.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:51.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:51.769810+0000) 2024-03-06T17:22:52.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:52.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:52.770006+0000) 2024-03-06T17:22:53.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:53.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:53.770247+0000) 2024-03-06T17:22:54.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:54.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:54.770477+0000) 2024-03-06T17:22:55.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:55.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:55.770701+0000) 2024-03-06T17:22:56.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:56.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:56.770924+0000) 2024-03-06T17:22:57.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:57.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:57.771144+0000) 2024-03-06T17:22:58.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:58.769+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:58.771358+0000) 2024-03-06T17:22:59.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:22:59.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:22:59.771641+0000) 2024-03-06T17:23:00.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:00.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:00.771836+0000) 2024-03-06T17:23:01.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:01.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:01.772064+0000) 2024-03-06T17:23:02.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:02.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:02.772290+0000) 2024-03-06T17:23:03.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:03.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:03.772513+0000) 2024-03-06T17:23:04.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:04.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:04.772786+0000) 2024-03-06T17:23:05.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:05.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:05.772997+0000) 2024-03-06T17:23:06.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:06.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:06.773220+0000) 2024-03-06T17:23:07.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:07.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:07.773438+0000) 2024-03-06T17:23:08.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:08.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:08.773667+0000) 2024-03-06T17:23:09.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:09.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:09.773862+0000) 2024-03-06T17:23:10.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:10.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:10.774107+0000) 2024-03-06T17:23:11.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:11.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:11.774334+0000) 2024-03-06T17:23:12.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:12.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:12.774516+0000) 2024-03-06T17:23:13.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:13.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:13.774789+0000) 2024-03-06T17:23:14.447 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:23:14.456 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:23:14.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:14.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:14.775026+0000) 2024-03-06T17:23:15.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:15.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:15.775251+0000) 2024-03-06T17:23:16.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:16.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:16.775446+0000) 2024-03-06T17:23:17.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:17.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:17.775683+0000) 2024-03-06T17:23:18.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:18.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:18.775913+0000) 2024-03-06T17:23:19.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:19.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:19.776138+0000) 2024-03-06T17:23:20.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:20.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:20.776388+0000) 2024-03-06T17:23:21.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:21.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:21.776644+0000) 2024-03-06T17:23:22.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:22.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:22.776867+0000) 2024-03-06T17:23:23.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:23.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:23.777088+0000) 2024-03-06T17:23:24.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:24.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:24.777314+0000) 2024-03-06T17:23:25.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:25.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:25.777623+0000) 2024-03-06T17:23:26.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:26.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:26.777842+0000) 2024-03-06T17:23:27.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:27.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:27.778058+0000) 2024-03-06T17:23:28.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:28.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:28.778249+0000) 2024-03-06T17:23:29.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:29.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:29.778468+0000) 2024-03-06T17:23:30.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:30.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:30.778693+0000) 2024-03-06T17:23:31.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:31.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:31.778912+0000) 2024-03-06T17:23:32.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:32.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:32.779137+0000) 2024-03-06T17:23:33.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:33.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:33.779321+0000) 2024-03-06T17:23:34.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:34.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:34.779506+0000) 2024-03-06T17:23:35.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:35.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:35.779766+0000) 2024-03-06T17:23:36.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:36.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:36.779992+0000) 2024-03-06T17:23:37.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:37.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:37.780209+0000) 2024-03-06T17:23:38.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:38.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:38.780439+0000) 2024-03-06T17:23:39.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:39.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:39.780665+0000) 2024-03-06T17:23:40.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:40.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:40.780898+0000) 2024-03-06T17:23:41.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:41.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:41.781106+0000) 2024-03-06T17:23:42.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:42.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:42.781296+0000) 2024-03-06T17:23:43.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:43.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:43.781478+0000) 2024-03-06T17:23:44.594 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:23:44.602 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:23:44.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:44.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:44.781726+0000) 2024-03-06T17:23:45.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:45.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:45.781941+0000) 2024-03-06T17:23:46.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:46.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:46.782164+0000) 2024-03-06T17:23:47.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:47.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:47.782344+0000) 2024-03-06T17:23:48.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:48.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:48.782583+0000) 2024-03-06T17:23:49.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:49.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:49.782859+0000) 2024-03-06T17:23:50.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:50.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:50.783119+0000) 2024-03-06T17:23:51.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:51.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:51.783394+0000) 2024-03-06T17:23:52.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:52.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:52.783673+0000) 2024-03-06T17:23:53.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:53.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:53.783911+0000) 2024-03-06T17:23:54.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:54.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:54.784147+0000) 2024-03-06T17:23:55.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:55.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:55.784425+0000) 2024-03-06T17:23:56.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:56.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:56.784689+0000) 2024-03-06T17:23:57.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:57.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:57.784973+0000) 2024-03-06T17:23:58.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:58.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:58.785251+0000) 2024-03-06T17:23:59.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:23:59.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:23:59.785518+0000) 2024-03-06T17:24:00.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:00.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:00.785849+0000) 2024-03-06T17:24:01.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:01.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:01.786068+0000) 2024-03-06T17:24:02.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:02.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:02.786270+0000) 2024-03-06T17:24:03.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:03.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:03.786509+0000) 2024-03-06T17:24:04.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:04.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:04.786779+0000) 2024-03-06T17:24:05.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:05.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:05.787006+0000) 2024-03-06T17:24:06.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:06.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:06.787227+0000) 2024-03-06T17:24:07.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:07.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:07.787446+0000) 2024-03-06T17:24:08.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:08.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:08.787703+0000) 2024-03-06T17:24:09.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:09.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:09.787922+0000) 2024-03-06T17:24:10.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:10.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:10.788157+0000) 2024-03-06T17:24:11.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:11.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:11.788364+0000) 2024-03-06T17:24:12.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:12.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:12.788602+0000) 2024-03-06T17:24:13.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:13.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:13.788831+0000) 2024-03-06T17:24:14.723 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:24:14.731 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:24:14.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:14.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:14.789028+0000) 2024-03-06T17:24:15.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:15.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:15.789291+0000) 2024-03-06T17:24:16.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:16.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:16.789485+0000) 2024-03-06T17:24:17.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:17.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:17.789723+0000) 2024-03-06T17:24:18.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:18.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:18.789977+0000) 2024-03-06T17:24:19.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:19.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:19.790199+0000) 2024-03-06T17:24:20.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:20.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:20.790431+0000) 2024-03-06T17:24:21.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:21.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:21.790739+0000) 2024-03-06T17:24:22.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:22.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:22.790936+0000) 2024-03-06T17:24:23.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:23.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:23.791149+0000) 2024-03-06T17:24:24.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:24.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:24.791429+0000) 2024-03-06T17:24:25.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:25.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:25.791683+0000) 2024-03-06T17:24:26.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:26.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:26.791903+0000) 2024-03-06T17:24:27.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:27.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:27.792121+0000) 2024-03-06T17:24:28.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:28.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:28.792348+0000) 2024-03-06T17:24:29.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:29.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:29.792584+0000) 2024-03-06T17:24:30.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:30.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:30.792804+0000) 2024-03-06T17:24:31.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:31.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:31.793020+0000) 2024-03-06T17:24:32.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:32.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:32.793218+0000) 2024-03-06T17:24:33.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:33.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:33.793458+0000) 2024-03-06T17:24:34.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:34.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:34.793729+0000) 2024-03-06T17:24:35.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:35.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:35.794005+0000) 2024-03-06T17:24:36.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:36.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:36.794280+0000) 2024-03-06T17:24:37.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:37.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:37.794521+0000) 2024-03-06T17:24:38.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:38.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:38.794801+0000) 2024-03-06T17:24:39.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:39.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:39.795023+0000) 2024-03-06T17:24:40.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:40.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:40.795261+0000) 2024-03-06T17:24:41.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:41.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:41.795481+0000) 2024-03-06T17:24:42.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:42.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:42.795690+0000) 2024-03-06T17:24:43.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:43.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:43.795923+0000) 2024-03-06T17:24:44.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:44.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:44.796144+0000) 2024-03-06T17:24:44.911 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:24:44.919 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:24:45.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:45.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:45.796350+0000) 2024-03-06T17:24:46.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:46.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:46.796612+0000) 2024-03-06T17:24:47.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:47.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:47.796837+0000) 2024-03-06T17:24:48.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:48.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:48.797043+0000) 2024-03-06T17:24:49.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:49.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:49.797266+0000) 2024-03-06T17:24:50.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:50.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:50.797508+0000) 2024-03-06T17:24:51.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:51.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:51.797776+0000) 2024-03-06T17:24:52.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:52.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:52.798011+0000) 2024-03-06T17:24:53.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:53.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:53.798515+0000) 2024-03-06T17:24:54.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:54.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:54.798797+0000) 2024-03-06T17:24:55.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:55.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:55.799019+0000) 2024-03-06T17:24:56.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:56.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:56.799211+0000) 2024-03-06T17:24:57.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:57.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:57.799446+0000) 2024-03-06T17:24:58.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:58.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:58.799715+0000) 2024-03-06T17:24:59.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:24:59.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:24:59.800003+0000) 2024-03-06T17:25:00.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:00.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:00.800287+0000) 2024-03-06T17:25:01.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:01.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:01.800597+0000) 2024-03-06T17:25:02.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:02.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:02.800873+0000) 2024-03-06T17:25:03.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:03.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:03.801074+0000) 2024-03-06T17:25:04.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:04.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:04.801258+0000) 2024-03-06T17:25:05.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:05.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:05.801478+0000) 2024-03-06T17:25:06.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:06.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:06.801684+0000) 2024-03-06T17:25:07.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:07.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:07.801883+0000) 2024-03-06T17:25:08.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:08.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:08.802099+0000) 2024-03-06T17:25:09.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:09.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:09.802302+0000) 2024-03-06T17:25:10.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:10.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:10.802504+0000) 2024-03-06T17:25:11.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:11.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:11.802678+0000) 2024-03-06T17:25:12.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:12.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:12.802921+0000) 2024-03-06T17:25:13.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:13.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:13.803118+0000) 2024-03-06T17:25:14.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:14.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:14.803342+0000) 2024-03-06T17:25:15.134 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:25:15.142 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:25:15.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:15.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:15.803585+0000) 2024-03-06T17:25:16.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:16.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:16.803757+0000) 2024-03-06T17:25:17.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:17.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:17.803945+0000) 2024-03-06T17:25:18.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:18.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:18.804149+0000) 2024-03-06T17:25:19.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:19.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:19.804370+0000) 2024-03-06T17:25:20.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:20.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:20.804635+0000) 2024-03-06T17:25:21.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:21.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:21.804860+0000) 2024-03-06T17:25:22.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:22.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:22.805051+0000) 2024-03-06T17:25:23.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:23.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:23.805285+0000) 2024-03-06T17:25:24.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:24.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:24.805470+0000) 2024-03-06T17:25:25.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:25.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:25.805665+0000) 2024-03-06T17:25:26.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:26.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:26.805870+0000) 2024-03-06T17:25:27.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:27.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:27.806073+0000) 2024-03-06T17:25:28.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:28.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:28.806325+0000) 2024-03-06T17:25:29.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:29.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:29.806589+0000) 2024-03-06T17:25:30.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:30.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:30.806797+0000) 2024-03-06T17:25:31.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:31.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:31.807028+0000) 2024-03-06T17:25:32.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:32.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:32.807224+0000) 2024-03-06T17:25:33.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:33.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:33.807443+0000) 2024-03-06T17:25:34.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:34.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:34.807660+0000) 2024-03-06T17:25:35.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:35.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:35.807894+0000) 2024-03-06T17:25:36.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:36.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:36.808120+0000) 2024-03-06T17:25:37.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:37.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:37.808349+0000) 2024-03-06T17:25:38.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:38.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:38.808630+0000) 2024-03-06T17:25:39.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:39.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:39.808854+0000) 2024-03-06T17:25:40.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:40.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:40.809093+0000) 2024-03-06T17:25:41.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:41.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:41.809314+0000) 2024-03-06T17:25:42.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:42.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:42.809496+0000) 2024-03-06T17:25:43.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:43.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:43.809707+0000) 2024-03-06T17:25:44.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:44.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:44.809902+0000) 2024-03-06T17:25:45.270 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:25:45.278 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:25:45.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:45.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:45.810079+0000) 2024-03-06T17:25:46.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:46.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:46.810299+0000) 2024-03-06T17:25:47.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:47.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:47.810465+0000) 2024-03-06T17:25:48.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:48.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:48.810699+0000) 2024-03-06T17:25:49.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:49.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:49.810899+0000) 2024-03-06T17:25:50.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:50.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:50.811104+0000) 2024-03-06T17:25:51.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:51.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:51.811299+0000) 2024-03-06T17:25:52.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:52.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:52.811525+0000) 2024-03-06T17:25:53.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:53.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:53.811775+0000) 2024-03-06T17:25:54.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:54.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:54.812064+0000) 2024-03-06T17:25:55.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:55.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:55.812256+0000) 2024-03-06T17:25:56.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:56.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:56.812440+0000) 2024-03-06T17:25:57.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:57.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:57.812676+0000) 2024-03-06T17:25:58.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:58.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:58.812880+0000) 2024-03-06T17:25:59.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:25:59.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:25:59.813104+0000) 2024-03-06T17:26:00.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:00.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:00.813315+0000) 2024-03-06T17:26:01.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:01.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:01.813556+0000) 2024-03-06T17:26:02.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:02.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:02.813799+0000) 2024-03-06T17:26:03.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:03.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:03.814022+0000) 2024-03-06T17:26:04.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:04.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:04.814256+0000) 2024-03-06T17:26:05.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:05.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:05.814495+0000) 2024-03-06T17:26:06.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:06.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:06.814729+0000) 2024-03-06T17:26:07.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:07.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:07.814959+0000) 2024-03-06T17:26:08.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:08.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:08.815192+0000) 2024-03-06T17:26:09.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:09.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:09.815415+0000) 2024-03-06T17:26:10.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:10.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:10.815653+0000) 2024-03-06T17:26:11.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:11.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:11.815872+0000) 2024-03-06T17:26:12.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:12.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:12.816083+0000) 2024-03-06T17:26:13.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:13.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:13.816279+0000) 2024-03-06T17:26:14.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:14.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:14.816513+0000) 2024-03-06T17:26:15.409 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:26:15.417 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:26:15.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:15.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:15.816735+0000) 2024-03-06T17:26:16.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:16.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:16.816928+0000) 2024-03-06T17:26:17.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:17.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:17.817112+0000) 2024-03-06T17:26:18.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:18.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:18.817391+0000) 2024-03-06T17:26:19.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:19.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:19.817645+0000) 2024-03-06T17:26:20.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:20.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:20.817852+0000) 2024-03-06T17:26:21.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:21.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:21.818055+0000) 2024-03-06T17:26:22.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:22.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:22.818280+0000) 2024-03-06T17:26:23.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:23.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:23.818511+0000) 2024-03-06T17:26:24.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:24.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:24.818743+0000) 2024-03-06T17:26:25.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:25.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:25.818924+0000) 2024-03-06T17:26:26.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:26.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:26.819117+0000) 2024-03-06T17:26:27.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:27.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:27.819343+0000) 2024-03-06T17:26:28.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:28.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:28.819603+0000) 2024-03-06T17:26:29.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:29.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:29.819798+0000) 2024-03-06T17:26:30.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:30.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:30.819999+0000) 2024-03-06T17:26:31.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:31.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:31.820186+0000) 2024-03-06T17:26:32.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:32.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:32.820416+0000) 2024-03-06T17:26:33.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:33.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:33.820658+0000) 2024-03-06T17:26:34.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:34.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:34.820890+0000) 2024-03-06T17:26:35.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:35.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:35.821102+0000) 2024-03-06T17:26:36.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:36.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:36.821326+0000) 2024-03-06T17:26:37.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:37.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:37.821596+0000) 2024-03-06T17:26:38.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:38.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:38.821829+0000) 2024-03-06T17:26:39.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:39.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:39.822059+0000) 2024-03-06T17:26:40.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:40.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:40.822289+0000) 2024-03-06T17:26:41.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:41.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:41.822527+0000) 2024-03-06T17:26:42.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:42.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:42.822785+0000) 2024-03-06T17:26:43.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:43.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:43.823002+0000) 2024-03-06T17:26:44.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:44.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:44.823209+0000) 2024-03-06T17:26:45.555 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:26:45.564 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:26:45.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:45.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:45.823460+0000) 2024-03-06T17:26:46.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:46.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:46.823659+0000) 2024-03-06T17:26:47.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:47.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:47.823873+0000) 2024-03-06T17:26:48.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:48.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:48.824162+0000) 2024-03-06T17:26:49.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:49.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:49.824334+0000) 2024-03-06T17:26:50.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:50.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:50.824588+0000) 2024-03-06T17:26:51.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:51.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:51.824824+0000) 2024-03-06T17:26:52.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:52.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:52.825036+0000) 2024-03-06T17:26:53.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:53.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:53.825280+0000) 2024-03-06T17:26:54.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:54.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:54.825469+0000) 2024-03-06T17:26:55.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:55.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:55.825697+0000) 2024-03-06T17:26:56.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:56.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:56.825898+0000) 2024-03-06T17:26:57.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:57.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:57.826097+0000) 2024-03-06T17:26:58.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:58.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:58.826324+0000) 2024-03-06T17:26:59.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:26:59.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:26:59.826558+0000) 2024-03-06T17:27:00.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:00.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:00.826760+0000) 2024-03-06T17:27:01.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:01.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:01.826949+0000) 2024-03-06T17:27:02.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:02.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:02.827159+0000) 2024-03-06T17:27:03.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:03.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:03.827400+0000) 2024-03-06T17:27:04.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:04.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:04.827627+0000) 2024-03-06T17:27:05.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:05.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:05.827852+0000) 2024-03-06T17:27:06.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:06.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:06.828083+0000) 2024-03-06T17:27:07.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:07.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:07.828316+0000) 2024-03-06T17:27:08.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:08.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:08.828508+0000) 2024-03-06T17:27:09.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:09.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:09.828751+0000) 2024-03-06T17:27:10.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:10.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:10.828994+0000) 2024-03-06T17:27:11.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:11.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:11.829220+0000) 2024-03-06T17:27:12.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:12.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:12.829454+0000) 2024-03-06T17:27:13.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:13.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:13.829682+0000) 2024-03-06T17:27:14.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:14.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:14.829913+0000) 2024-03-06T17:27:15.775 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:27:15.783 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:27:15.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:15.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:15.830072+0000) 2024-03-06T17:27:16.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:16.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:16.830290+0000) 2024-03-06T17:27:17.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:17.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:17.830483+0000) 2024-03-06T17:27:18.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:18.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:18.830682+0000) 2024-03-06T17:27:19.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:19.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:19.830888+0000) 2024-03-06T17:27:20.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:20.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:20.831109+0000) 2024-03-06T17:27:21.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:21.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:21.831336+0000) 2024-03-06T17:27:22.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:22.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:22.831598+0000) 2024-03-06T17:27:23.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:23.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:23.831801+0000) 2024-03-06T17:27:24.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:24.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:24.831999+0000) 2024-03-06T17:27:25.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:25.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:25.832210+0000) 2024-03-06T17:27:26.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:26.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:26.832446+0000) 2024-03-06T17:27:27.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:27.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:27.832674+0000) 2024-03-06T17:27:28.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:28.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:28.832869+0000) 2024-03-06T17:27:29.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:29.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:29.833065+0000) 2024-03-06T17:27:30.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:30.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:30.833331+0000) 2024-03-06T17:27:31.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:31.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:31.833603+0000) 2024-03-06T17:27:32.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:32.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:32.833813+0000) 2024-03-06T17:27:33.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:33.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:33.834056+0000) 2024-03-06T17:27:34.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:34.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:34.834296+0000) 2024-03-06T17:27:35.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:35.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:35.834567+0000) 2024-03-06T17:27:36.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:36.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:36.834869+0000) 2024-03-06T17:27:37.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:37.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:37.835087+0000) 2024-03-06T17:27:38.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:38.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:38.835319+0000) 2024-03-06T17:27:39.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:39.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:39.835529+0000) 2024-03-06T17:27:40.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:40.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:40.835800+0000) 2024-03-06T17:27:41.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:41.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:41.836006+0000) 2024-03-06T17:27:42.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:42.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:42.836240+0000) 2024-03-06T17:27:43.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:43.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:43.836422+0000) 2024-03-06T17:27:44.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:44.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:44.836702+0000) 2024-03-06T17:27:45.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:45.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:45.836928+0000) 2024-03-06T17:27:45.953 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:27:45.961 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:27:46.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:46.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:46.837111+0000) 2024-03-06T17:27:47.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:47.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:47.837344+0000) 2024-03-06T17:27:48.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:48.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:48.837614+0000) 2024-03-06T17:27:49.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:49.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:49.837836+0000) 2024-03-06T17:27:50.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:50.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:50.838037+0000) 2024-03-06T17:27:51.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:51.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:51.838210+0000) 2024-03-06T17:27:52.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:52.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:52.838441+0000) 2024-03-06T17:27:53.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:53.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:53.838665+0000) 2024-03-06T17:27:54.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:54.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:54.838858+0000) 2024-03-06T17:27:55.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:55.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:55.839031+0000) 2024-03-06T17:27:56.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:56.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:56.839244+0000) 2024-03-06T17:27:57.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:57.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:57.839487+0000) 2024-03-06T17:27:58.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:58.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:58.839705+0000) 2024-03-06T17:27:59.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:27:59.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:27:59.839910+0000) 2024-03-06T17:28:00.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:00.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:00.840148+0000) 2024-03-06T17:28:01.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:01.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:01.840355+0000) 2024-03-06T17:28:02.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:02.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:02.840590+0000) 2024-03-06T17:28:03.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:03.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:03.840779+0000) 2024-03-06T17:28:04.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:04.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:04.841036+0000) 2024-03-06T17:28:05.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:05.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:05.841263+0000) 2024-03-06T17:28:06.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:06.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:06.841581+0000) 2024-03-06T17:28:07.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:07.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:07.841832+0000) 2024-03-06T17:28:08.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:08.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:08.842064+0000) 2024-03-06T17:28:09.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:09.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:09.842294+0000) 2024-03-06T17:28:10.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:10.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:10.842575+0000) 2024-03-06T17:28:11.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:11.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:11.842815+0000) 2024-03-06T17:28:12.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:12.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:12.843067+0000) 2024-03-06T17:28:13.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:13.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:13.843310+0000) 2024-03-06T17:28:14.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:14.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:14.843618+0000) 2024-03-06T17:28:15.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:15.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:15.843897+0000) 2024-03-06T17:28:16.143 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:28:16.151 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:28:16.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:16.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:16.844115+0000) 2024-03-06T17:28:17.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:17.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:17.844391+0000) 2024-03-06T17:28:18.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:18.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:18.844705+0000) 2024-03-06T17:28:19.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:19.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:19.844903+0000) 2024-03-06T17:28:20.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:20.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:20.845118+0000) 2024-03-06T17:28:21.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:21.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:21.845389+0000) 2024-03-06T17:28:22.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:22.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:22.845711+0000) 2024-03-06T17:28:23.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:23.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:23.845905+0000) 2024-03-06T17:28:24.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:24.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:24.846173+0000) 2024-03-06T17:28:25.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:25.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:25.846454+0000) 2024-03-06T17:28:26.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:26.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:26.846744+0000) 2024-03-06T17:28:27.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:27.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:27.847008+0000) 2024-03-06T17:28:28.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:28.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:28.847266+0000) 2024-03-06T17:28:29.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:29.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:29.847569+0000) 2024-03-06T17:28:30.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:30.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:30.847815+0000) 2024-03-06T17:28:31.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:31.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:31.848038+0000) 2024-03-06T17:28:32.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:32.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:32.848310+0000) 2024-03-06T17:28:33.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:33.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:33.848625+0000) 2024-03-06T17:28:34.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:34.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:34.848898+0000) 2024-03-06T17:28:35.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:35.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:35.849137+0000) 2024-03-06T17:28:36.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:36.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:36.849328+0000) 2024-03-06T17:28:37.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:37.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:37.849586+0000) 2024-03-06T17:28:38.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:38.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:38.849808+0000) 2024-03-06T17:28:39.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:39.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:39.850004+0000) 2024-03-06T17:28:40.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:40.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:40.850220+0000) 2024-03-06T17:28:41.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:41.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:41.850455+0000) 2024-03-06T17:28:42.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:42.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:42.850735+0000) 2024-03-06T17:28:43.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:43.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:43.850973+0000) 2024-03-06T17:28:44.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:44.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:44.851163+0000) 2024-03-06T17:28:45.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:45.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:45.851428+0000) 2024-03-06T17:28:46.365 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:28:46.374 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:28:46.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:46.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:46.851671+0000) 2024-03-06T17:28:47.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:47.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:47.851865+0000) 2024-03-06T17:28:48.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:48.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:48.852095+0000) 2024-03-06T17:28:49.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:49.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:49.852323+0000) 2024-03-06T17:28:50.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:50.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:50.852567+0000) 2024-03-06T17:28:51.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:51.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:51.852756+0000) 2024-03-06T17:28:52.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:52.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:52.852954+0000) 2024-03-06T17:28:53.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:53.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:53.853183+0000) 2024-03-06T17:28:54.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:54.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:54.853413+0000) 2024-03-06T17:28:55.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:55.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:55.853645+0000) 2024-03-06T17:28:56.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:56.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:56.853870+0000) 2024-03-06T17:28:57.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:57.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:57.854091+0000) 2024-03-06T17:28:58.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:58.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:58.854317+0000) 2024-03-06T17:28:59.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:28:59.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:28:59.854568+0000) 2024-03-06T17:29:00.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:00.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:00.854792+0000) 2024-03-06T17:29:01.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:01.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:01.855041+0000) 2024-03-06T17:29:02.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:02.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:02.855268+0000) 2024-03-06T17:29:03.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:03.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:03.855489+0000) 2024-03-06T17:29:04.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:04.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:04.855729+0000) 2024-03-06T17:29:05.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:05.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:05.855957+0000) 2024-03-06T17:29:06.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:06.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:06.856098+0000) 2024-03-06T17:29:07.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:07.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:07.856278+0000) 2024-03-06T17:29:08.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:08.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:08.856507+0000) 2024-03-06T17:29:09.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:09.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:09.856735+0000) 2024-03-06T17:29:10.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:10.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:10.856931+0000) 2024-03-06T17:29:11.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:11.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:11.857148+0000) 2024-03-06T17:29:12.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:12.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:12.857430+0000) 2024-03-06T17:29:13.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:13.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:13.857656+0000) 2024-03-06T17:29:14.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:14.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:14.857885+0000) 2024-03-06T17:29:15.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:15.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:15.858090+0000) 2024-03-06T17:29:16.528 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:29:16.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:29:16.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:16.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:16.858267+0000) 2024-03-06T17:29:17.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:17.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:17.858448+0000) 2024-03-06T17:29:18.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:18.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:18.858691+0000) 2024-03-06T17:29:19.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:19.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:19.858919+0000) 2024-03-06T17:29:20.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:20.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:20.859179+0000) 2024-03-06T17:29:21.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:21.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:21.859416+0000) 2024-03-06T17:29:22.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:22.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:22.859667+0000) 2024-03-06T17:29:23.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:23.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:23.859911+0000) 2024-03-06T17:29:24.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:24.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:24.860162+0000) 2024-03-06T17:29:25.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:25.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:25.860406+0000) 2024-03-06T17:29:26.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:26.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:26.860641+0000) 2024-03-06T17:29:27.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:27.859+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:27.860845+0000) 2024-03-06T17:29:28.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:28.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:28.861082+0000) 2024-03-06T17:29:29.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:29.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:29.861307+0000) 2024-03-06T17:29:30.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:30.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:30.861593+0000) 2024-03-06T17:29:31.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:31.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:31.861823+0000) 2024-03-06T17:29:32.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:32.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:32.862041+0000) 2024-03-06T17:29:33.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:33.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:33.862317+0000) 2024-03-06T17:29:34.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:34.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:34.862645+0000) 2024-03-06T17:29:35.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:35.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:35.862907+0000) 2024-03-06T17:29:36.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:36.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:36.863142+0000) 2024-03-06T17:29:37.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:37.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:37.863377+0000) 2024-03-06T17:29:38.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:38.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:38.863688+0000) 2024-03-06T17:29:39.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:39.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:39.863906+0000) 2024-03-06T17:29:40.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:40.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:40.864163+0000) 2024-03-06T17:29:41.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:41.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:41.864396+0000) 2024-03-06T17:29:42.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:42.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:42.864654+0000) 2024-03-06T17:29:43.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:43.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:43.864897+0000) 2024-03-06T17:29:44.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:44.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:44.865123+0000) 2024-03-06T17:29:45.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:45.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:45.865359+0000) 2024-03-06T17:29:46.734 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:29:46.742 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:29:46.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:46.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:46.865645+0000) 2024-03-06T17:29:47.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:47.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:47.865835+0000) 2024-03-06T17:29:48.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:48.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:48.866072+0000) 2024-03-06T17:29:49.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:49.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:49.866298+0000) 2024-03-06T17:29:50.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:50.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:50.866589+0000) 2024-03-06T17:29:51.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:51.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:51.866834+0000) 2024-03-06T17:29:52.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:52.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:52.867079+0000) 2024-03-06T17:29:53.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:53.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:53.867285+0000) 2024-03-06T17:29:54.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:54.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:54.867567+0000) 2024-03-06T17:29:55.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:55.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:55.867811+0000) 2024-03-06T17:29:56.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:56.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:56.868049+0000) 2024-03-06T17:29:57.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:57.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:57.868294+0000) 2024-03-06T17:29:58.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:58.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:58.868583+0000) 2024-03-06T17:29:59.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:29:59.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:29:59.868790+0000) 2024-03-06T17:30:00.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:00.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:00.869050+0000) 2024-03-06T17:30:01.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:01.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:01.869288+0000) 2024-03-06T17:30:02.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:02.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:02.869519+0000) 2024-03-06T17:30:03.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:03.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:03.869799+0000) 2024-03-06T17:30:04.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:04.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:04.870053+0000) 2024-03-06T17:30:05.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:05.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:05.870346+0000) 2024-03-06T17:30:06.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:06.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:06.870567+0000) 2024-03-06T17:30:07.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:07.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:07.870764+0000) 2024-03-06T17:30:08.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:08.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:08.870999+0000) 2024-03-06T17:30:09.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:09.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:09.871207+0000) 2024-03-06T17:30:10.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:10.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:10.871471+0000) 2024-03-06T17:30:11.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:11.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:11.871706+0000) 2024-03-06T17:30:12.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:12.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:12.871993+0000) 2024-03-06T17:30:13.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:13.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:13.872271+0000) 2024-03-06T17:30:14.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:14.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:14.872597+0000) 2024-03-06T17:30:15.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:15.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:15.872834+0000) 2024-03-06T17:30:16.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:16.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:16.873040+0000) 2024-03-06T17:30:16.938 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:30:16.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:30:17.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:17.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:17.873275+0000) 2024-03-06T17:30:18.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:18.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:18.873503+0000) 2024-03-06T17:30:19.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:19.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:19.873730+0000) 2024-03-06T17:30:20.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:20.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:20.873926+0000) 2024-03-06T17:30:21.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:21.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:21.874153+0000) 2024-03-06T17:30:22.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:22.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:22.874388+0000) 2024-03-06T17:30:23.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:23.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:23.874631+0000) 2024-03-06T17:30:24.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:24.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:24.874862+0000) 2024-03-06T17:30:25.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:25.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:25.875064+0000) 2024-03-06T17:30:26.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:26.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:26.875255+0000) 2024-03-06T17:30:27.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:27.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:27.875480+0000) 2024-03-06T17:30:28.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:28.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:28.875657+0000) 2024-03-06T17:30:29.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:29.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:29.875897+0000) 2024-03-06T17:30:30.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:30.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:30.876131+0000) 2024-03-06T17:30:31.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:31.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:31.876361+0000) 2024-03-06T17:30:32.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:32.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:32.876586+0000) 2024-03-06T17:30:33.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:33.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:33.876805+0000) 2024-03-06T17:30:34.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:34.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:34.877034+0000) 2024-03-06T17:30:35.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:35.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:35.877258+0000) 2024-03-06T17:30:36.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:36.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:36.877489+0000) 2024-03-06T17:30:37.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:37.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:37.877691+0000) 2024-03-06T17:30:38.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:38.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:38.877866+0000) 2024-03-06T17:30:39.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:39.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:39.878096+0000) 2024-03-06T17:30:40.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:40.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:40.878339+0000) 2024-03-06T17:30:41.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:41.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:41.878589+0000) 2024-03-06T17:30:42.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:42.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:42.878827+0000) 2024-03-06T17:30:43.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:43.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:43.879046+0000) 2024-03-06T17:30:44.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:44.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:44.879283+0000) 2024-03-06T17:30:45.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:45.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:45.879468+0000) 2024-03-06T17:30:46.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:46.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:46.879690+0000) 2024-03-06T17:30:47.160 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:30:47.168 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:30:47.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:47.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:47.879866+0000) 2024-03-06T17:30:48.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:48.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:48.880062+0000) 2024-03-06T17:30:49.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:49.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:49.880296+0000) 2024-03-06T17:30:50.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:50.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:50.880579+0000) 2024-03-06T17:30:51.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:51.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:51.880751+0000) 2024-03-06T17:30:52.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:52.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:52.880978+0000) 2024-03-06T17:30:53.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:53.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:53.881213+0000) 2024-03-06T17:30:54.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:54.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:54.881462+0000) 2024-03-06T17:30:55.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:55.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:55.881680+0000) 2024-03-06T17:30:56.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:56.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:56.881905+0000) 2024-03-06T17:30:57.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:57.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:57.882132+0000) 2024-03-06T17:30:58.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:58.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:58.882357+0000) 2024-03-06T17:30:59.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:30:59.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:30:59.882592+0000) 2024-03-06T17:31:00.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:00.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:00.882809+0000) 2024-03-06T17:31:01.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:01.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:01.883042+0000) 2024-03-06T17:31:02.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:02.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:02.883240+0000) 2024-03-06T17:31:03.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:03.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:03.883463+0000) 2024-03-06T17:31:04.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:04.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:04.883708+0000) 2024-03-06T17:31:05.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:05.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:05.883930+0000) 2024-03-06T17:31:06.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:06.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:06.884162+0000) 2024-03-06T17:31:07.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:07.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:07.884363+0000) 2024-03-06T17:31:08.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:08.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:08.884638+0000) 2024-03-06T17:31:09.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:09.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:09.884884+0000) 2024-03-06T17:31:10.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:10.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:10.885126+0000) 2024-03-06T17:31:11.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:11.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:11.885355+0000) 2024-03-06T17:31:12.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:12.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:12.885617+0000) 2024-03-06T17:31:13.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:13.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:13.885838+0000) 2024-03-06T17:31:14.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:14.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:14.886075+0000) 2024-03-06T17:31:15.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:15.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:15.886267+0000) 2024-03-06T17:31:16.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:16.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:16.886460+0000) 2024-03-06T17:31:17.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:31:17.364 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:31:17.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:17.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:17.886651+0000) 2024-03-06T17:31:18.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:18.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:18.886838+0000) 2024-03-06T17:31:19.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:19.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:19.887052+0000) 2024-03-06T17:31:20.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:20.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:20.887295+0000) 2024-03-06T17:31:21.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:21.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:21.887525+0000) 2024-03-06T17:31:22.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:22.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:22.887764+0000) 2024-03-06T17:31:23.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:23.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:23.887984+0000) 2024-03-06T17:31:24.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:24.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:24.888219+0000) 2024-03-06T17:31:25.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:25.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:25.888402+0000) 2024-03-06T17:31:26.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:26.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:26.888662+0000) 2024-03-06T17:31:27.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:27.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:27.888884+0000) 2024-03-06T17:31:28.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:28.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:28.889113+0000) 2024-03-06T17:31:29.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:29.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:29.889298+0000) 2024-03-06T17:31:30.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:30.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:30.889568+0000) 2024-03-06T17:31:31.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:31.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:31.889804+0000) 2024-03-06T17:31:32.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:32.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:32.890022+0000) 2024-03-06T17:31:33.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:33.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:33.890231+0000) 2024-03-06T17:31:34.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:34.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:34.890486+0000) 2024-03-06T17:31:35.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:35.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:35.890720+0000) 2024-03-06T17:31:36.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:36.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:36.890965+0000) 2024-03-06T17:31:37.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:37.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:37.891158+0000) 2024-03-06T17:31:38.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:38.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:38.891384+0000) 2024-03-06T17:31:39.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:39.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:39.891659+0000) 2024-03-06T17:31:40.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:40.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:40.891868+0000) 2024-03-06T17:31:41.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:41.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:41.892093+0000) 2024-03-06T17:31:42.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:42.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:42.892335+0000) 2024-03-06T17:31:43.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:43.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:43.892582+0000) 2024-03-06T17:31:44.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:44.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:44.892804+0000) 2024-03-06T17:31:45.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:45.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:45.893001+0000) 2024-03-06T17:31:46.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:46.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:46.893277+0000) 2024-03-06T17:31:47.578 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:31:47.586 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:31:47.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:47.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:47.893507+0000) 2024-03-06T17:31:48.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:48.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:48.893758+0000) 2024-03-06T17:31:49.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:49.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:49.893978+0000) 2024-03-06T17:31:50.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:50.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:50.894200+0000) 2024-03-06T17:31:51.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:51.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:51.894424+0000) 2024-03-06T17:31:52.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:52.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:52.894661+0000) 2024-03-06T17:31:53.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:53.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:53.894853+0000) 2024-03-06T17:31:54.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:54.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:54.895067+0000) 2024-03-06T17:31:55.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:55.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:55.895304+0000) 2024-03-06T17:31:56.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:56.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:56.895494+0000) 2024-03-06T17:31:57.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:57.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:57.895732+0000) 2024-03-06T17:31:58.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:58.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:58.895943+0000) 2024-03-06T17:31:59.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:31:59.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:31:59.896132+0000) 2024-03-06T17:32:00.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:00.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:00.896389+0000) 2024-03-06T17:32:01.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:01.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:01.896621+0000) 2024-03-06T17:32:02.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:02.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:02.896849+0000) 2024-03-06T17:32:03.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:03.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:03.897070+0000) 2024-03-06T17:32:04.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:04.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:04.897259+0000) 2024-03-06T17:32:05.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:05.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:05.897511+0000) 2024-03-06T17:32:06.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:06.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:06.897778+0000) 2024-03-06T17:32:07.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:07.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:07.897969+0000) 2024-03-06T17:32:08.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:08.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:08.898173+0000) 2024-03-06T17:32:09.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:09.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:09.898354+0000) 2024-03-06T17:32:10.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:10.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:10.898655+0000) 2024-03-06T17:32:11.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:11.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:11.898871+0000) 2024-03-06T17:32:12.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:12.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:12.899117+0000) 2024-03-06T17:32:13.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:13.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:13.899356+0000) 2024-03-06T17:32:14.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:14.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:14.899639+0000) 2024-03-06T17:32:15.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:15.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:15.899875+0000) 2024-03-06T17:32:16.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:16.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:16.900085+0000) 2024-03-06T17:32:17.791 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:32:17.799 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:32:17.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:17.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:17.900272+0000) 2024-03-06T17:32:18.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:18.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:18.900504+0000) 2024-03-06T17:32:19.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:19.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:19.900711+0000) 2024-03-06T17:32:20.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:20.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:20.900965+0000) 2024-03-06T17:32:21.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:21.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:21.901156+0000) 2024-03-06T17:32:22.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:22.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:22.901383+0000) 2024-03-06T17:32:23.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:23.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:23.901653+0000) 2024-03-06T17:32:24.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:24.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:24.901859+0000) 2024-03-06T17:32:25.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:25.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:25.902046+0000) 2024-03-06T17:32:26.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:26.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:26.902231+0000) 2024-03-06T17:32:27.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:27.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:27.902465+0000) 2024-03-06T17:32:28.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:28.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:28.902724+0000) 2024-03-06T17:32:29.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:29.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:29.902931+0000) 2024-03-06T17:32:30.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:30.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:30.903130+0000) 2024-03-06T17:32:31.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:31.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:31.903367+0000) 2024-03-06T17:32:32.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:32.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:32.903665+0000) 2024-03-06T17:32:33.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:33.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:33.903864+0000) 2024-03-06T17:32:34.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:34.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:34.904094+0000) 2024-03-06T17:32:35.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:35.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:35.904278+0000) 2024-03-06T17:32:36.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:36.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:36.904483+0000) 2024-03-06T17:32:37.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:37.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:37.904720+0000) 2024-03-06T17:32:38.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:38.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:38.904948+0000) 2024-03-06T17:32:39.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:39.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:39.905139+0000) 2024-03-06T17:32:40.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:40.904+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:40.905432+0000) 2024-03-06T17:32:41.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:41.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:41.905657+0000) 2024-03-06T17:32:42.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:42.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:42.905881+0000) 2024-03-06T17:32:43.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:43.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:43.906057+0000) 2024-03-06T17:32:44.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:44.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:44.906290+0000) 2024-03-06T17:32:45.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:45.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:45.906574+0000) 2024-03-06T17:32:46.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:46.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:46.906749+0000) 2024-03-06T17:32:47.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:47.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:47.906967+0000) 2024-03-06T17:32:48.013 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:32:48.023 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:32:48.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:48.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:48.907189+0000) 2024-03-06T17:32:49.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:49.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:49.907397+0000) 2024-03-06T17:32:50.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:50.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:50.907656+0000) 2024-03-06T17:32:51.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:51.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:51.907863+0000) 2024-03-06T17:32:52.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:52.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:52.908095+0000) 2024-03-06T17:32:53.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:53.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:53.908278+0000) 2024-03-06T17:32:54.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:54.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:54.908512+0000) 2024-03-06T17:32:55.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:55.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:55.908781+0000) 2024-03-06T17:32:56.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:56.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:56.908981+0000) 2024-03-06T17:32:57.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:57.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:57.909178+0000) 2024-03-06T17:32:58.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:58.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:58.909410+0000) 2024-03-06T17:32:59.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:32:59.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:32:59.909698+0000) 2024-03-06T17:33:00.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:00.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:00.910002+0000) 2024-03-06T17:33:01.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:01.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:01.910228+0000) 2024-03-06T17:33:02.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:02.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:02.910456+0000) 2024-03-06T17:33:03.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:03.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:03.910692+0000) 2024-03-06T17:33:04.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:04.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:04.910928+0000) 2024-03-06T17:33:05.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:05.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:05.911165+0000) 2024-03-06T17:33:06.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:06.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:06.911394+0000) 2024-03-06T17:33:07.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:07.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:07.911659+0000) 2024-03-06T17:33:08.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:08.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:08.911901+0000) 2024-03-06T17:33:09.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:09.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:09.912126+0000) 2024-03-06T17:33:10.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:10.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:10.912418+0000) 2024-03-06T17:33:11.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:11.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:11.912687+0000) 2024-03-06T17:33:12.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:12.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:12.912890+0000) 2024-03-06T17:33:13.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:13.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:13.913110+0000) 2024-03-06T17:33:14.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:14.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:14.913351+0000) 2024-03-06T17:33:15.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:15.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:15.913608+0000) 2024-03-06T17:33:16.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:16.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:16.913846+0000) 2024-03-06T17:33:17.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:17.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:17.914074+0000) 2024-03-06T17:33:18.225 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:33:18.236 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:33:18.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:18.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:18.914276+0000) 2024-03-06T17:33:19.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:19.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:19.914518+0000) 2024-03-06T17:33:20.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:20.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:20.914794+0000) 2024-03-06T17:33:21.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:21.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:21.915011+0000) 2024-03-06T17:33:22.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:22.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:22.915203+0000) 2024-03-06T17:33:23.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:23.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:23.915422+0000) 2024-03-06T17:33:24.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:24.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:24.915657+0000) 2024-03-06T17:33:25.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:25.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:25.915885+0000) 2024-03-06T17:33:26.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:26.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:26.916065+0000) 2024-03-06T17:33:27.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:27.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:27.916298+0000) 2024-03-06T17:33:28.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:28.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:28.916500+0000) 2024-03-06T17:33:29.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:29.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:29.916689+0000) 2024-03-06T17:33:30.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:30.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:30.916926+0000) 2024-03-06T17:33:31.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:31.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:31.917099+0000) 2024-03-06T17:33:32.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:32.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:32.917368+0000) 2024-03-06T17:33:33.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:33.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:33.917612+0000) 2024-03-06T17:33:34.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:34.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:34.917806+0000) 2024-03-06T17:33:35.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:35.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:35.918028+0000) 2024-03-06T17:33:36.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:36.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:36.918239+0000) 2024-03-06T17:33:37.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:37.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:37.918419+0000) 2024-03-06T17:33:38.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:38.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:38.918668+0000) 2024-03-06T17:33:39.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:39.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:39.918859+0000) 2024-03-06T17:33:40.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:40.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:40.919096+0000) 2024-03-06T17:33:41.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:41.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:41.919270+0000) 2024-03-06T17:33:42.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:42.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:42.919503+0000) 2024-03-06T17:33:43.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:43.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:43.919759+0000) 2024-03-06T17:33:44.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:44.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:44.920028+0000) 2024-03-06T17:33:45.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:45.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:45.920306+0000) 2024-03-06T17:33:46.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:46.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:46.920510+0000) 2024-03-06T17:33:47.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:47.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:47.920752+0000) 2024-03-06T17:33:48.454 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:33:48.462 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:33:48.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:48.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:48.920958+0000) 2024-03-06T17:33:49.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:49.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:49.921201+0000) 2024-03-06T17:33:50.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:50.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:50.921416+0000) 2024-03-06T17:33:51.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:51.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:51.921671+0000) 2024-03-06T17:33:52.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:52.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:52.921910+0000) 2024-03-06T17:33:53.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:53.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:53.922129+0000) 2024-03-06T17:33:54.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:54.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:54.922355+0000) 2024-03-06T17:33:55.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:55.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:55.922587+0000) 2024-03-06T17:33:56.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:56.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:56.922800+0000) 2024-03-06T17:33:57.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:57.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:57.923025+0000) 2024-03-06T17:33:58.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:58.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:58.923252+0000) 2024-03-06T17:33:59.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:33:59.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:33:59.923460+0000) 2024-03-06T17:34:00.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:00.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:00.923715+0000) 2024-03-06T17:34:01.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:01.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:01.923934+0000) 2024-03-06T17:34:02.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:02.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:02.924198+0000) 2024-03-06T17:34:03.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:03.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:03.924426+0000) 2024-03-06T17:34:04.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:04.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:04.924654+0000) 2024-03-06T17:34:05.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:05.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:05.924886+0000) 2024-03-06T17:34:06.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:06.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:06.925113+0000) 2024-03-06T17:34:07.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:07.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:07.925332+0000) 2024-03-06T17:34:08.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:08.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:08.925589+0000) 2024-03-06T17:34:09.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:09.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:09.925809+0000) 2024-03-06T17:34:10.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:10.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:10.926041+0000) 2024-03-06T17:34:11.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:11.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:11.926258+0000) 2024-03-06T17:34:12.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:12.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:12.926498+0000) 2024-03-06T17:34:13.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:13.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:13.926735+0000) 2024-03-06T17:34:14.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:14.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:14.926960+0000) 2024-03-06T17:34:15.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:15.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:15.927183+0000) 2024-03-06T17:34:16.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:16.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:16.927448+0000) 2024-03-06T17:34:17.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:17.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:17.927669+0000) 2024-03-06T17:34:18.667 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:34:18.678 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:34:18.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:18.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:18.927911+0000) 2024-03-06T17:34:19.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:19.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:19.928130+0000) 2024-03-06T17:34:20.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:20.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:20.928365+0000) 2024-03-06T17:34:21.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:21.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:21.928632+0000) 2024-03-06T17:34:22.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:22.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:22.928851+0000) 2024-03-06T17:34:23.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:23.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:23.929070+0000) 2024-03-06T17:34:24.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:24.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:24.929298+0000) 2024-03-06T17:34:25.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:25.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:25.929484+0000) 2024-03-06T17:34:26.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:26.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:26.929652+0000) 2024-03-06T17:34:27.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:27.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:27.929872+0000) 2024-03-06T17:34:28.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:28.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:28.930054+0000) 2024-03-06T17:34:29.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:29.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:29.930244+0000) 2024-03-06T17:34:30.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:30.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:30.930488+0000) 2024-03-06T17:34:31.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:31.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:31.930706+0000) 2024-03-06T17:34:32.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:32.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:32.930940+0000) 2024-03-06T17:34:33.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:33.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:33.931158+0000) 2024-03-06T17:34:34.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:34.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:34.931347+0000) 2024-03-06T17:34:35.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:35.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:35.931624+0000) 2024-03-06T17:34:36.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:36.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:36.931806+0000) 2024-03-06T17:34:37.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:37.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:37.932031+0000) 2024-03-06T17:34:38.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:38.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:38.932225+0000) 2024-03-06T17:34:39.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:39.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:39.932408+0000) 2024-03-06T17:34:40.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:40.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:40.932647+0000) 2024-03-06T17:34:41.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:41.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:41.932820+0000) 2024-03-06T17:34:42.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:42.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:42.933044+0000) 2024-03-06T17:34:43.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:43.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:43.933259+0000) 2024-03-06T17:34:44.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:44.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:44.933485+0000) 2024-03-06T17:34:45.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:45.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:45.933697+0000) 2024-03-06T17:34:46.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:46.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:46.933923+0000) 2024-03-06T17:34:47.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:47.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:47.934144+0000) 2024-03-06T17:34:48.888 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:34:48.899 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:34:48.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:48.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:48.934332+0000) 2024-03-06T17:34:49.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:49.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:49.934482+0000) 2024-03-06T17:34:50.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:50.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:50.934691+0000) 2024-03-06T17:34:51.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:51.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:51.934913+0000) 2024-03-06T17:34:52.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:52.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:52.935185+0000) 2024-03-06T17:34:53.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:53.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:53.935424+0000) 2024-03-06T17:34:54.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:54.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:54.935659+0000) 2024-03-06T17:34:55.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:55.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:55.935859+0000) 2024-03-06T17:34:56.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:56.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:56.936079+0000) 2024-03-06T17:34:57.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:57.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:57.936281+0000) 2024-03-06T17:34:58.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:58.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:58.936510+0000) 2024-03-06T17:34:59.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:34:59.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:34:59.936743+0000) 2024-03-06T17:35:00.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:00.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:00.936998+0000) 2024-03-06T17:35:01.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:01.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:01.937231+0000) 2024-03-06T17:35:02.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:02.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:02.937477+0000) 2024-03-06T17:35:03.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:03.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:03.937667+0000) 2024-03-06T17:35:04.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:04.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:04.937911+0000) 2024-03-06T17:35:05.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:05.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:05.938124+0000) 2024-03-06T17:35:06.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:06.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:06.938347+0000) 2024-03-06T17:35:07.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:07.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:07.938579+0000) 2024-03-06T17:35:08.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:08.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:08.938818+0000) 2024-03-06T17:35:09.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:09.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:09.939046+0000) 2024-03-06T17:35:10.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:10.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:10.939259+0000) 2024-03-06T17:35:11.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:11.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:11.939483+0000) 2024-03-06T17:35:12.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:12.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:12.939676+0000) 2024-03-06T17:35:13.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:13.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:13.939910+0000) 2024-03-06T17:35:14.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:14.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:14.940137+0000) 2024-03-06T17:35:15.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:15.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:15.940362+0000) 2024-03-06T17:35:16.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:16.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:16.940658+0000) 2024-03-06T17:35:17.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:17.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:17.940886+0000) 2024-03-06T17:35:18.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:18.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:18.941116+0000) 2024-03-06T17:35:19.101 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:35:19.110 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:35:19.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:19.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:19.941307+0000) 2024-03-06T17:35:20.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:20.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:20.941579+0000) 2024-03-06T17:35:21.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:21.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:21.941802+0000) 2024-03-06T17:35:22.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:22.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:22.942028+0000) 2024-03-06T17:35:23.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:23.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:23.942248+0000) 2024-03-06T17:35:24.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:24.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:24.942491+0000) 2024-03-06T17:35:25.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:25.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:25.942745+0000) 2024-03-06T17:35:26.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:26.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:26.942922+0000) 2024-03-06T17:35:27.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:27.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:27.943177+0000) 2024-03-06T17:35:28.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:28.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:28.943420+0000) 2024-03-06T17:35:29.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:29.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:29.943704+0000) 2024-03-06T17:35:30.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:30.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:30.944009+0000) 2024-03-06T17:35:31.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:31.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:31.944232+0000) 2024-03-06T17:35:32.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:32.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:32.944475+0000) 2024-03-06T17:35:33.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:33.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:33.944706+0000) 2024-03-06T17:35:34.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:34.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:34.944932+0000) 2024-03-06T17:35:35.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:35.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:35.945203+0000) 2024-03-06T17:35:36.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:36.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:36.945444+0000) 2024-03-06T17:35:37.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:37.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:37.945676+0000) 2024-03-06T17:35:38.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:38.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:38.945904+0000) 2024-03-06T17:35:39.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:39.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:39.946079+0000) 2024-03-06T17:35:40.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:40.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:40.946337+0000) 2024-03-06T17:35:41.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:41.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:41.946622+0000) 2024-03-06T17:35:42.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:42.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:42.946872+0000) 2024-03-06T17:35:43.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:43.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:43.947084+0000) 2024-03-06T17:35:44.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:44.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:44.947283+0000) 2024-03-06T17:35:45.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:45.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:45.947521+0000) 2024-03-06T17:35:46.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:46.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:46.947800+0000) 2024-03-06T17:35:47.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:47.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:47.948036+0000) 2024-03-06T17:35:48.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:48.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:48.948278+0000) 2024-03-06T17:35:49.305 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:35:49.314 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:35:49.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:49.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:49.948521+0000) 2024-03-06T17:35:50.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:50.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:50.948851+0000) 2024-03-06T17:35:51.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:51.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:51.949087+0000) 2024-03-06T17:35:52.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:52.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:52.949321+0000) 2024-03-06T17:35:53.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:53.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:53.949526+0000) 2024-03-06T17:35:54.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:54.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:54.949800+0000) 2024-03-06T17:35:55.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:55.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:55.949998+0000) 2024-03-06T17:35:56.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:56.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:56.950199+0000) 2024-03-06T17:35:57.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:57.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:57.950458+0000) 2024-03-06T17:35:58.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:58.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:58.950699+0000) 2024-03-06T17:35:59.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:35:59.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:35:59.950923+0000) 2024-03-06T17:36:00.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:00.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:00.951169+0000) 2024-03-06T17:36:01.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:01.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:01.951345+0000) 2024-03-06T17:36:02.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:02.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:02.951632+0000) 2024-03-06T17:36:03.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:03.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:03.951825+0000) 2024-03-06T17:36:04.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:04.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:04.952058+0000) 2024-03-06T17:36:05.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:05.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:05.952297+0000) 2024-03-06T17:36:06.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:06.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:06.952526+0000) 2024-03-06T17:36:07.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:07.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:07.952791+0000) 2024-03-06T17:36:08.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:08.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:08.953008+0000) 2024-03-06T17:36:09.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:09.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:09.953226+0000) 2024-03-06T17:36:10.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:10.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:10.953509+0000) 2024-03-06T17:36:11.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:11.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:11.953773+0000) 2024-03-06T17:36:12.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:12.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:12.953981+0000) 2024-03-06T17:36:13.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:13.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:13.954216+0000) 2024-03-06T17:36:14.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:14.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:14.954444+0000) 2024-03-06T17:36:15.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:15.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:15.954679+0000) 2024-03-06T17:36:16.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:16.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:16.954898+0000) 2024-03-06T17:36:17.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:17.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:17.955145+0000) 2024-03-06T17:36:18.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:18.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:18.955375+0000) 2024-03-06T17:36:19.519 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:36:19.527 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:36:19.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:19.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:19.955635+0000) 2024-03-06T17:36:20.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:20.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:20.955843+0000) 2024-03-06T17:36:21.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:21.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:21.956084+0000) 2024-03-06T17:36:22.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:22.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:22.956312+0000) 2024-03-06T17:36:23.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:23.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:23.956508+0000) 2024-03-06T17:36:24.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:24.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:24.956793+0000) 2024-03-06T17:36:25.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:25.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:25.957032+0000) 2024-03-06T17:36:26.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:26.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:26.957256+0000) 2024-03-06T17:36:27.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:27.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:27.957441+0000) 2024-03-06T17:36:28.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:28.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:28.957677+0000) 2024-03-06T17:36:29.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:29.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:29.957877+0000) 2024-03-06T17:36:30.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:30.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:30.958126+0000) 2024-03-06T17:36:31.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:31.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:31.958349+0000) 2024-03-06T17:36:32.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:32.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:32.958616+0000) 2024-03-06T17:36:33.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:33.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:33.958823+0000) 2024-03-06T17:36:34.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:34.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:34.959082+0000) 2024-03-06T17:36:35.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:35.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:35.959278+0000) 2024-03-06T17:36:36.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:36.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:36.959503+0000) 2024-03-06T17:36:37.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:37.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:37.959737+0000) 2024-03-06T17:36:38.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:38.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:38.959942+0000) 2024-03-06T17:36:39.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:39.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:39.960169+0000) 2024-03-06T17:36:40.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:40.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:40.960475+0000) 2024-03-06T17:36:41.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:41.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:41.960699+0000) 2024-03-06T17:36:42.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:42.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:42.960884+0000) 2024-03-06T17:36:43.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:43.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:43.961069+0000) 2024-03-06T17:36:44.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:44.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:44.961327+0000) 2024-03-06T17:36:45.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:45.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:45.961609+0000) 2024-03-06T17:36:46.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:46.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:46.961770+0000) 2024-03-06T17:36:47.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:47.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:47.961986+0000) 2024-03-06T17:36:48.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:48.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:48.962213+0000) 2024-03-06T17:36:49.730 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:36:49.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:36:49.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:49.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:49.962453+0000) 2024-03-06T17:36:50.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:50.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:50.962701+0000) 2024-03-06T17:36:51.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:51.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:51.962930+0000) 2024-03-06T17:36:52.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:52.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:52.963129+0000) 2024-03-06T17:36:53.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:53.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:53.963362+0000) 2024-03-06T17:36:54.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:54.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:54.963639+0000) 2024-03-06T17:36:55.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:55.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:55.963890+0000) 2024-03-06T17:36:56.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:56.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:56.964078+0000) 2024-03-06T17:36:57.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:57.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:57.964260+0000) 2024-03-06T17:36:58.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:58.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:58.964484+0000) 2024-03-06T17:36:59.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:36:59.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:36:59.964704+0000) 2024-03-06T17:37:00.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:00.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:00.964966+0000) 2024-03-06T17:37:01.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:01.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:01.965175+0000) 2024-03-06T17:37:02.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:02.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:02.965422+0000) 2024-03-06T17:37:03.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:03.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:03.965676+0000) 2024-03-06T17:37:04.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:04.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:04.965924+0000) 2024-03-06T17:37:05.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:05.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:05.966179+0000) 2024-03-06T17:37:06.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:06.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:06.966433+0000) 2024-03-06T17:37:07.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:07.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:07.966676+0000) 2024-03-06T17:37:08.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:08.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:08.966918+0000) 2024-03-06T17:37:09.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:09.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:09.967125+0000) 2024-03-06T17:37:10.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:10.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:10.967363+0000) 2024-03-06T17:37:11.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:11.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:11.967633+0000) 2024-03-06T17:37:12.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:12.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:12.967861+0000) 2024-03-06T17:37:13.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:13.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:13.968080+0000) 2024-03-06T17:37:14.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:14.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:14.968282+0000) 2024-03-06T17:37:15.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:15.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:15.968562+0000) 2024-03-06T17:37:16.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:16.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:16.968800+0000) 2024-03-06T17:37:17.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:17.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:17.969008+0000) 2024-03-06T17:37:18.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:18.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:18.969210+0000) 2024-03-06T17:37:19.936 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:37:19.944 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:37:19.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:19.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:19.969370+0000) 2024-03-06T17:37:20.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:20.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:20.969637+0000) 2024-03-06T17:37:21.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:21.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:21.969860+0000) 2024-03-06T17:37:22.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:22.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:22.970066+0000) 2024-03-06T17:37:23.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:23.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:23.970245+0000) 2024-03-06T17:37:24.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:24.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:24.970487+0000) 2024-03-06T17:37:25.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:25.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:25.970765+0000) 2024-03-06T17:37:26.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:26.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:26.971004+0000) 2024-03-06T17:37:27.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:27.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:27.971163+0000) 2024-03-06T17:37:28.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:28.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:28.971361+0000) 2024-03-06T17:37:29.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:29.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:29.971580+0000) 2024-03-06T17:37:30.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:30.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:30.971839+0000) 2024-03-06T17:37:31.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:31.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:31.972059+0000) 2024-03-06T17:37:32.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:32.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:32.972290+0000) 2024-03-06T17:37:33.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:33.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:33.972518+0000) 2024-03-06T17:37:34.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:34.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:34.972752+0000) 2024-03-06T17:37:35.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:35.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:35.972993+0000) 2024-03-06T17:37:36.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:36.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:36.973216+0000) 2024-03-06T17:37:37.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:37.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:37.973440+0000) 2024-03-06T17:37:38.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:38.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:38.973722+0000) 2024-03-06T17:37:39.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:39.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:39.973919+0000) 2024-03-06T17:37:40.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:40.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:40.974146+0000) 2024-03-06T17:37:41.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:41.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:41.974336+0000) 2024-03-06T17:37:42.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:42.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:42.974621+0000) 2024-03-06T17:37:43.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:43.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:43.974803+0000) 2024-03-06T17:37:44.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:44.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:44.975035+0000) 2024-03-06T17:37:45.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:45.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:45.975270+0000) 2024-03-06T17:37:46.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:46.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:46.975498+0000) 2024-03-06T17:37:47.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:47.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:47.975705+0000) 2024-03-06T17:37:48.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:48.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:48.975947+0000) 2024-03-06T17:37:49.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:49.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:49.976162+0000) 2024-03-06T17:37:50.111 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:37:50.119 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:37:50.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:50.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:50.976410+0000) 2024-03-06T17:37:51.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:51.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:51.976622+0000) 2024-03-06T17:37:52.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:52.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:52.976845+0000) 2024-03-06T17:37:53.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:53.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:53.977065+0000) 2024-03-06T17:37:54.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:54.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:54.977292+0000) 2024-03-06T17:37:55.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:55.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:55.977523+0000) 2024-03-06T17:37:56.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:56.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:56.977774+0000) 2024-03-06T17:37:57.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:57.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:57.978018+0000) 2024-03-06T17:37:58.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:58.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:58.978203+0000) 2024-03-06T17:37:59.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:37:59.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:37:59.978414+0000) 2024-03-06T17:38:00.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:00.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:00.978688+0000) 2024-03-06T17:38:01.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:01.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:01.978913+0000) 2024-03-06T17:38:02.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:02.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:02.979151+0000) 2024-03-06T17:38:03.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:03.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:03.979323+0000) 2024-03-06T17:38:04.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:04.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:04.979611+0000) 2024-03-06T17:38:05.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:05.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:05.979868+0000) 2024-03-06T17:38:06.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:06.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:06.980084+0000) 2024-03-06T17:38:07.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:07.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:07.980308+0000) 2024-03-06T17:38:08.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:08.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:08.980517+0000) 2024-03-06T17:38:09.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:09.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:09.980781+0000) 2024-03-06T17:38:10.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:10.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:10.981021+0000) 2024-03-06T17:38:11.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:11.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:11.981240+0000) 2024-03-06T17:38:12.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:12.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:12.981483+0000) 2024-03-06T17:38:13.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:13.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:13.981701+0000) 2024-03-06T17:38:14.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:14.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:14.981930+0000) 2024-03-06T17:38:15.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:15.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:15.982161+0000) 2024-03-06T17:38:16.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:16.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:16.982346+0000) 2024-03-06T17:38:17.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:17.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:17.982616+0000) 2024-03-06T17:38:18.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:18.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:18.982795+0000) 2024-03-06T17:38:19.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:19.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:19.983033+0000) 2024-03-06T17:38:20.246 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:38:20.255 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:38:20.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:20.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:20.983261+0000) 2024-03-06T17:38:21.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:21.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:21.983445+0000) 2024-03-06T17:38:22.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:22.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:22.983682+0000) 2024-03-06T17:38:23.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:23.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:23.983882+0000) 2024-03-06T17:38:24.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:24.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:24.984065+0000) 2024-03-06T17:38:25.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:25.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:25.984271+0000) 2024-03-06T17:38:26.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:26.983+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:26.984508+0000) 2024-03-06T17:38:27.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:27.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:27.984760+0000) 2024-03-06T17:38:28.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:28.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:28.984979+0000) 2024-03-06T17:38:29.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:29.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:29.985211+0000) 2024-03-06T17:38:30.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:30.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:30.985417+0000) 2024-03-06T17:38:31.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:31.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:31.985638+0000) 2024-03-06T17:38:32.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:32.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:32.985923+0000) 2024-03-06T17:38:33.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:33.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:33.986142+0000) 2024-03-06T17:38:34.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:34.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:34.986324+0000) 2024-03-06T17:38:35.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:35.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:35.986586+0000) 2024-03-06T17:38:36.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:36.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:36.986780+0000) 2024-03-06T17:38:37.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:37.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:37.986971+0000) 2024-03-06T17:38:38.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:38.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:38.987150+0000) 2024-03-06T17:38:39.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:39.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:39.987371+0000) 2024-03-06T17:38:40.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:40.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:40.987659+0000) 2024-03-06T17:38:41.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:41.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:41.987844+0000) 2024-03-06T17:38:42.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:42.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:42.988079+0000) 2024-03-06T17:38:43.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:43.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:43.988296+0000) 2024-03-06T17:38:44.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:44.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:44.988529+0000) 2024-03-06T17:38:45.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:45.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:45.988800+0000) 2024-03-06T17:38:46.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:46.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:46.989020+0000) 2024-03-06T17:38:47.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:47.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:47.989241+0000) 2024-03-06T17:38:48.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:48.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:48.989484+0000) 2024-03-06T17:38:49.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:49.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:49.989703+0000) 2024-03-06T17:38:50.385 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:38:50.399 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:38:50.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:50.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:50.989904+0000) 2024-03-06T17:38:51.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:51.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:51.990097+0000) 2024-03-06T17:38:52.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:52.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:52.990328+0000) 2024-03-06T17:38:53.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:53.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:53.990505+0000) 2024-03-06T17:38:54.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:54.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:54.990780+0000) 2024-03-06T17:38:55.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:55.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:55.991003+0000) 2024-03-06T17:38:56.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:56.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:56.991225+0000) 2024-03-06T17:38:57.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:57.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:57.991446+0000) 2024-03-06T17:38:58.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:58.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:58.991675+0000) 2024-03-06T17:38:59.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:38:59.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:38:59.991900+0000) 2024-03-06T17:39:00.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:00.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:00.992118+0000) 2024-03-06T17:39:01.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:01.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:01.992312+0000) 2024-03-06T17:39:02.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:02.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:02.992568+0000) 2024-03-06T17:39:03.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:03.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:03.992788+0000) 2024-03-06T17:39:04.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:04.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:04.992984+0000) 2024-03-06T17:39:05.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:05.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:05.993173+0000) 2024-03-06T17:39:06.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:06.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:06.993417+0000) 2024-03-06T17:39:07.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:07.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:07.993656+0000) 2024-03-06T17:39:08.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:08.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:08.993852+0000) 2024-03-06T17:39:09.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:09.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:09.994092+0000) 2024-03-06T17:39:10.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:10.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:10.994313+0000) 2024-03-06T17:39:11.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:11.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:11.994577+0000) 2024-03-06T17:39:12.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:12.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:12.994826+0000) 2024-03-06T17:39:13.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:13.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:13.995062+0000) 2024-03-06T17:39:14.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:14.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:14.995359+0000) 2024-03-06T17:39:15.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:15.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:15.995621+0000) 2024-03-06T17:39:16.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:16.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:16.995838+0000) 2024-03-06T17:39:17.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:17.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:17.996071+0000) 2024-03-06T17:39:18.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:18.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:18.996300+0000) 2024-03-06T17:39:19.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:19.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:19.996483+0000) 2024-03-06T17:39:20.541 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:39:20.558 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:39:20.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:20.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:20.996806+0000) 2024-03-06T17:39:21.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:21.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:21.997006+0000) 2024-03-06T17:39:22.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:22.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:22.997193+0000) 2024-03-06T17:39:23.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:23.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:23.997418+0000) 2024-03-06T17:39:24.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:24.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:24.997694+0000) 2024-03-06T17:39:25.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:25.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:25.997901+0000) 2024-03-06T17:39:26.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:26.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:26.998107+0000) 2024-03-06T17:39:27.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:27.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:27.998303+0000) 2024-03-06T17:39:28.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:28.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:28.998529+0000) 2024-03-06T17:39:30.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:29.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:29.998786+0000) 2024-03-06T17:39:31.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:30.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:30.999023+0000) 2024-03-06T17:39:32.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:31.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:31.999301+0000) 2024-03-06T17:39:33.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:32.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:32.999511+0000) 2024-03-06T17:39:34.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:33.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:33.999743+0000) 2024-03-06T17:39:35.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:34.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:34.999978+0000) 2024-03-06T17:39:36.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:35.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:36.000190+0000) 2024-03-06T17:39:37.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:36.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:37.000436+0000) 2024-03-06T17:39:38.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:37.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:38.000673+0000) 2024-03-06T17:39:39.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:38.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:39.000888+0000) 2024-03-06T17:39:40.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:39.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:40.001090+0000) 2024-03-06T17:39:41.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:41.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:41.001280+0000) 2024-03-06T17:39:42.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:42.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:42.001515+0000) 2024-03-06T17:39:43.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:43.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:43.001783+0000) 2024-03-06T17:39:44.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:44.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:44.002010+0000) 2024-03-06T17:39:45.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:45.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:45.002199+0000) 2024-03-06T17:39:46.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:46.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:46.002420+0000) 2024-03-06T17:39:47.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:47.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:47.002646+0000) 2024-03-06T17:39:48.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:48.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:48.002865+0000) 2024-03-06T17:39:49.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:49.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:49.003058+0000) 2024-03-06T17:39:50.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:50.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:50.003286+0000) 2024-03-06T17:39:50.769 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:39:50.777 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:39:51.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:51.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:51.003529+0000) 2024-03-06T17:39:52.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:52.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:52.003743+0000) 2024-03-06T17:39:53.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:53.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:53.003977+0000) 2024-03-06T17:39:54.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:54.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:54.004201+0000) 2024-03-06T17:39:55.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:55.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:55.004422+0000) 2024-03-06T17:39:56.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:56.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:56.004672+0000) 2024-03-06T17:39:57.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:57.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:57.004823+0000) 2024-03-06T17:39:58.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:58.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:58.005011+0000) 2024-03-06T17:39:59.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:39:59.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:39:59.005237+0000) 2024-03-06T17:40:00.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:00.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:00.005458+0000) 2024-03-06T17:40:01.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:01.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:01.005711+0000) 2024-03-06T17:40:02.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:02.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:02.005935+0000) 2024-03-06T17:40:03.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:03.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:03.006116+0000) 2024-03-06T17:40:04.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:04.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:04.006334+0000) 2024-03-06T17:40:05.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:05.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:05.006563+0000) 2024-03-06T17:40:06.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:06.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:06.006766+0000) 2024-03-06T17:40:07.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:07.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:07.006960+0000) 2024-03-06T17:40:08.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:08.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:08.007149+0000) 2024-03-06T17:40:09.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:09.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:09.007430+0000) 2024-03-06T17:40:10.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:10.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:10.007703+0000) 2024-03-06T17:40:11.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:11.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:11.007960+0000) 2024-03-06T17:40:12.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:12.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:12.008180+0000) 2024-03-06T17:40:13.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:13.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:13.008404+0000) 2024-03-06T17:40:14.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:14.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:14.008703+0000) 2024-03-06T17:40:15.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:15.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:15.008981+0000) 2024-03-06T17:40:16.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:16.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:16.009222+0000) 2024-03-06T17:40:17.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:17.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:17.009483+0000) 2024-03-06T17:40:18.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:18.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:18.009677+0000) 2024-03-06T17:40:19.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:19.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:19.009918+0000) 2024-03-06T17:40:20.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:20.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:20.010145+0000) 2024-03-06T17:40:21.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:21.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:21.010448+0000) 2024-03-06T17:40:21.038 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:40:21.046 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:40:22.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:22.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:22.010657+0000) 2024-03-06T17:40:23.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:23.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:23.010857+0000) 2024-03-06T17:40:24.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:24.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:24.011115+0000) 2024-03-06T17:40:25.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:25.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:25.011388+0000) 2024-03-06T17:40:26.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:26.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:26.011643+0000) 2024-03-06T17:40:27.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:27.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:27.011873+0000) 2024-03-06T17:40:28.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:28.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:28.012111+0000) 2024-03-06T17:40:29.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:29.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:29.012378+0000) 2024-03-06T17:40:30.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:30.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:30.012649+0000) 2024-03-06T17:40:31.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:31.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:31.012886+0000) 2024-03-06T17:40:32.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:32.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:32.013072+0000) 2024-03-06T17:40:33.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:33.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:33.013307+0000) 2024-03-06T17:40:34.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:34.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:34.013589+0000) 2024-03-06T17:40:35.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:35.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:35.013807+0000) 2024-03-06T17:40:36.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:36.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:36.014010+0000) 2024-03-06T17:40:37.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:37.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:37.014235+0000) 2024-03-06T17:40:38.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:38.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:38.014458+0000) 2024-03-06T17:40:39.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:39.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:39.014667+0000) 2024-03-06T17:40:40.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:40.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:40.014865+0000) 2024-03-06T17:40:41.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:41.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:41.015111+0000) 2024-03-06T17:40:42.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:42.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:42.015338+0000) 2024-03-06T17:40:43.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:43.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:43.015663+0000) 2024-03-06T17:40:44.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:44.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:44.015833+0000) 2024-03-06T17:40:45.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:45.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:45.016065+0000) 2024-03-06T17:40:46.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:46.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:46.016284+0000) 2024-03-06T17:40:47.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:47.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:47.016576+0000) 2024-03-06T17:40:48.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:48.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:48.016819+0000) 2024-03-06T17:40:49.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:49.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:49.017028+0000) 2024-03-06T17:40:50.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:50.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:50.017261+0000) 2024-03-06T17:40:51.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:51.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:51.017519+0000) 2024-03-06T17:40:51.246 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:40:51.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:40:52.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:52.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:52.017793+0000) 2024-03-06T17:40:53.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:53.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:53.017996+0000) 2024-03-06T17:40:54.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:54.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:54.018173+0000) 2024-03-06T17:40:55.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:55.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:55.018411+0000) 2024-03-06T17:40:56.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:56.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:56.018667+0000) 2024-03-06T17:40:57.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:57.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:57.018881+0000) 2024-03-06T17:40:58.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:58.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:58.019080+0000) 2024-03-06T17:40:59.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:40:59.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:40:59.019319+0000) 2024-03-06T17:41:00.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:00.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:00.019579+0000) 2024-03-06T17:41:01.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:01.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:01.019793+0000) 2024-03-06T17:41:02.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:02.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:02.019992+0000) 2024-03-06T17:41:03.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:03.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:03.020253+0000) 2024-03-06T17:41:04.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:04.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:04.020485+0000) 2024-03-06T17:41:05.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:05.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:05.020694+0000) 2024-03-06T17:41:06.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:06.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:06.020930+0000) 2024-03-06T17:41:07.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:07.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:07.021179+0000) 2024-03-06T17:41:08.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:08.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:08.021415+0000) 2024-03-06T17:41:09.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:09.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:09.021675+0000) 2024-03-06T17:41:10.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:10.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:10.021880+0000) 2024-03-06T17:41:11.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:11.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:11.022104+0000) 2024-03-06T17:41:12.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:12.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:12.022293+0000) 2024-03-06T17:41:13.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:13.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:13.022568+0000) 2024-03-06T17:41:14.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:14.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:14.022757+0000) 2024-03-06T17:41:15.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:15.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:15.022953+0000) 2024-03-06T17:41:16.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:16.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:16.023169+0000) 2024-03-06T17:41:17.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:17.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:17.023430+0000) 2024-03-06T17:41:18.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:18.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:18.023666+0000) 2024-03-06T17:41:19.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:19.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:19.023894+0000) 2024-03-06T17:41:20.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:20.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:20.024024+0000) 2024-03-06T17:41:21.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:21.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:21.024261+0000) 2024-03-06T17:41:21.458 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:41:21.467 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:41:22.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:22.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:22.024493+0000) 2024-03-06T17:41:23.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:23.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:23.024679+0000) 2024-03-06T17:41:24.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:24.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:24.024898+0000) 2024-03-06T17:41:25.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:25.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:25.025089+0000) 2024-03-06T17:41:26.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:26.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:26.025284+0000) 2024-03-06T17:41:27.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:27.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:27.025523+0000) 2024-03-06T17:41:28.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:28.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:28.025734+0000) 2024-03-06T17:41:29.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:29.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:29.025924+0000) 2024-03-06T17:41:30.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:30.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:30.026106+0000) 2024-03-06T17:41:31.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:31.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:31.026345+0000) 2024-03-06T17:41:32.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:32.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:32.026527+0000) 2024-03-06T17:41:33.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:33.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:33.026796+0000) 2024-03-06T17:41:34.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:34.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:34.026988+0000) 2024-03-06T17:41:35.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:35.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:35.027234+0000) 2024-03-06T17:41:36.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:36.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:36.027459+0000) 2024-03-06T17:41:37.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:37.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:37.027686+0000) 2024-03-06T17:41:38.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:38.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:38.027903+0000) 2024-03-06T17:41:39.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:39.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:39.028090+0000) 2024-03-06T17:41:40.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:40.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:40.028308+0000) 2024-03-06T17:41:41.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:41.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:41.028574+0000) 2024-03-06T17:41:42.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:42.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:42.028773+0000) 2024-03-06T17:41:43.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:43.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:43.029010+0000) 2024-03-06T17:41:44.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:44.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:44.029207+0000) 2024-03-06T17:41:45.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:45.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:45.029452+0000) 2024-03-06T17:41:46.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:46.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:46.029690+0000) 2024-03-06T17:41:47.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:47.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:47.029919+0000) 2024-03-06T17:41:48.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:48.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:48.030144+0000) 2024-03-06T17:41:49.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:49.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:49.030332+0000) 2024-03-06T17:41:50.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:50.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:50.030632+0000) 2024-03-06T17:41:51.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:51.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:51.030859+0000) 2024-03-06T17:41:51.655 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:41:51.665 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:41:52.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:52.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:52.031087+0000) 2024-03-06T17:41:53.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:53.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:53.031306+0000) 2024-03-06T17:41:54.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:54.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:54.031566+0000) 2024-03-06T17:41:55.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:55.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:55.031753+0000) 2024-03-06T17:41:56.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:56.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:56.031943+0000) 2024-03-06T17:41:57.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:57.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:57.032140+0000) 2024-03-06T17:41:58.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:58.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:58.032312+0000) 2024-03-06T17:41:59.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:41:59.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:41:59.032563+0000) 2024-03-06T17:42:00.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:00.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:00.032769+0000) 2024-03-06T17:42:01.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:01.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:01.033029+0000) 2024-03-06T17:42:02.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:02.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:02.033238+0000) 2024-03-06T17:42:03.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:03.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:03.033445+0000) 2024-03-06T17:42:04.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:04.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:04.033638+0000) 2024-03-06T17:42:05.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:05.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:05.033834+0000) 2024-03-06T17:42:06.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:06.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:06.034060+0000) 2024-03-06T17:42:07.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:07.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:07.034322+0000) 2024-03-06T17:42:08.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:08.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:08.034568+0000) 2024-03-06T17:42:09.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:09.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:09.034793+0000) 2024-03-06T17:42:10.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:10.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:10.034982+0000) 2024-03-06T17:42:11.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:11.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:11.035180+0000) 2024-03-06T17:42:12.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:12.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:12.035420+0000) 2024-03-06T17:42:13.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:13.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:13.035666+0000) 2024-03-06T17:42:14.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:14.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:14.035852+0000) 2024-03-06T17:42:15.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:15.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:15.036079+0000) 2024-03-06T17:42:16.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:16.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:16.036302+0000) 2024-03-06T17:42:17.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:17.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:17.036565+0000) 2024-03-06T17:42:18.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:18.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:18.036754+0000) 2024-03-06T17:42:19.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:19.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:19.036942+0000) 2024-03-06T17:42:20.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:20.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:20.037282+0000) 2024-03-06T17:42:21.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:21.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:21.037568+0000) 2024-03-06T17:42:21.862 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:42:21.870 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:42:22.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:22.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:22.037810+0000) 2024-03-06T17:42:23.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:23.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:23.038057+0000) 2024-03-06T17:42:24.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:24.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:24.038244+0000) 2024-03-06T17:42:25.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:25.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:25.038478+0000) 2024-03-06T17:42:26.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:26.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:26.038698+0000) 2024-03-06T17:42:27.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:27.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:27.038938+0000) 2024-03-06T17:42:28.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:28.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:28.039176+0000) 2024-03-06T17:42:29.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:29.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:29.039405+0000) 2024-03-06T17:42:30.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:30.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:30.039638+0000) 2024-03-06T17:42:31.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:31.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:31.039875+0000) 2024-03-06T17:42:32.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:32.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:32.040115+0000) 2024-03-06T17:42:33.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:33.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:33.040321+0000) 2024-03-06T17:42:34.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:34.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:34.040508+0000) 2024-03-06T17:42:35.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:35.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:35.040784+0000) 2024-03-06T17:42:36.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:36.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:36.040982+0000) 2024-03-06T17:42:37.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:37.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:37.041203+0000) 2024-03-06T17:42:38.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:38.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:38.041529+0000) 2024-03-06T17:42:39.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:39.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:39.041750+0000) 2024-03-06T17:42:40.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:40.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:40.041988+0000) 2024-03-06T17:42:41.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:41.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:41.042238+0000) 2024-03-06T17:42:42.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:42.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:42.042526+0000) 2024-03-06T17:42:43.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:43.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:43.042815+0000) 2024-03-06T17:42:44.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:44.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:44.042989+0000) 2024-03-06T17:42:45.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:45.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:45.043214+0000) 2024-03-06T17:42:46.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:46.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:46.043456+0000) 2024-03-06T17:42:47.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:47.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:47.043704+0000) 2024-03-06T17:42:48.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:48.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:48.043905+0000) 2024-03-06T17:42:49.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:49.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:49.044153+0000) 2024-03-06T17:42:50.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:50.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:50.044353+0000) 2024-03-06T17:42:51.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:51.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:51.044628+0000) 2024-03-06T17:42:52.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:52.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:52.044796+0000) 2024-03-06T17:42:52.075 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:42:52.083 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:42:53.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:53.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:53.044987+0000) 2024-03-06T17:42:54.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:54.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:54.045173+0000) 2024-03-06T17:42:55.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:55.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:55.045406+0000) 2024-03-06T17:42:56.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:56.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:56.045639+0000) 2024-03-06T17:42:57.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:57.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:57.045866+0000) 2024-03-06T17:42:58.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:58.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:58.046106+0000) 2024-03-06T17:42:59.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:42:59.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:42:59.046302+0000) 2024-03-06T17:43:00.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:00.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:00.046559+0000) 2024-03-06T17:43:01.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:01.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:01.046762+0000) 2024-03-06T17:43:02.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:02.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:02.046970+0000) 2024-03-06T17:43:03.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:03.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:03.047196+0000) 2024-03-06T17:43:04.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:04.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:04.047413+0000) 2024-03-06T17:43:05.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:05.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:05.047650+0000) 2024-03-06T17:43:06.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:06.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:06.047835+0000) 2024-03-06T17:43:07.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:07.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:07.048040+0000) 2024-03-06T17:43:08.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:08.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:08.048260+0000) 2024-03-06T17:43:09.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:09.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:09.048447+0000) 2024-03-06T17:43:10.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:10.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:10.048673+0000) 2024-03-06T17:43:11.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:11.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:11.048881+0000) 2024-03-06T17:43:12.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:12.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:12.049066+0000) 2024-03-06T17:43:13.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:13.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:13.049293+0000) 2024-03-06T17:43:14.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:14.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:14.049487+0000) 2024-03-06T17:43:15.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:15.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:15.049713+0000) 2024-03-06T17:43:16.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:16.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:16.049901+0000) 2024-03-06T17:43:17.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:17.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:17.050112+0000) 2024-03-06T17:43:18.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:18.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:18.050354+0000) 2024-03-06T17:43:19.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:19.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:19.050563+0000) 2024-03-06T17:43:20.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:20.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:20.050765+0000) 2024-03-06T17:43:21.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:21.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:21.050985+0000) 2024-03-06T17:43:22.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:22.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:22.051178+0000) 2024-03-06T17:43:22.286 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:43:22.294 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:43:23.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:23.049+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:23.051405+0000) 2024-03-06T17:43:24.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:24.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:24.051659+0000) 2024-03-06T17:43:25.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:25.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:25.051892+0000) 2024-03-06T17:43:26.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:26.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:26.052140+0000) 2024-03-06T17:43:27.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:27.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:27.052375+0000) 2024-03-06T17:43:28.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:28.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:28.052624+0000) 2024-03-06T17:43:29.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:29.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:29.052813+0000) 2024-03-06T17:43:30.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:30.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:30.053002+0000) 2024-03-06T17:43:31.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:31.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:31.053302+0000) 2024-03-06T17:43:32.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:32.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:32.053560+0000) 2024-03-06T17:43:33.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:33.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:33.053787+0000) 2024-03-06T17:43:34.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:34.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:34.054054+0000) 2024-03-06T17:43:35.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:35.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:35.054336+0000) 2024-03-06T17:43:36.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:36.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:36.054583+0000) 2024-03-06T17:43:37.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:37.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:37.054842+0000) 2024-03-06T17:43:38.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:38.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:38.055069+0000) 2024-03-06T17:43:39.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:39.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:39.055249+0000) 2024-03-06T17:43:40.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:40.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:40.055492+0000) 2024-03-06T17:43:41.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:41.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:41.055689+0000) 2024-03-06T17:43:42.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:42.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:42.055911+0000) 2024-03-06T17:43:43.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:43.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:43.056156+0000) 2024-03-06T17:43:44.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:44.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:44.056399+0000) 2024-03-06T17:43:45.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:45.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:45.056677+0000) 2024-03-06T17:43:46.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:46.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:46.056896+0000) 2024-03-06T17:43:47.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:47.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:47.057087+0000) 2024-03-06T17:43:48.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:48.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:48.057334+0000) 2024-03-06T17:43:49.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:49.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:49.057604+0000) 2024-03-06T17:43:50.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:50.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:50.057803+0000) 2024-03-06T17:43:51.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:51.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:51.058006+0000) 2024-03-06T17:43:52.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:52.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:52.058227+0000) 2024-03-06T17:43:52.507 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:43:52.515 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:43:53.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:53.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:53.058458+0000) 2024-03-06T17:43:54.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:54.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:54.058683+0000) 2024-03-06T17:43:55.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:55.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:55.058908+0000) 2024-03-06T17:43:56.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:56.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:56.059136+0000) 2024-03-06T17:43:57.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:57.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:57.059373+0000) 2024-03-06T17:43:58.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:58.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:58.059643+0000) 2024-03-06T17:43:59.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:43:59.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:43:59.059876+0000) 2024-03-06T17:44:00.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:00.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:00.060098+0000) 2024-03-06T17:44:01.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:01.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:01.060307+0000) 2024-03-06T17:44:02.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:02.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:02.060623+0000) 2024-03-06T17:44:03.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:03.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:03.060897+0000) 2024-03-06T17:44:04.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:04.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:04.061166+0000) 2024-03-06T17:44:05.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:05.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:05.061453+0000) 2024-03-06T17:44:06.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:06.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:06.061699+0000) 2024-03-06T17:44:07.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:07.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:07.061965+0000) 2024-03-06T17:44:08.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:08.060+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:08.062246+0000) 2024-03-06T17:44:09.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:09.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:09.062625+0000) 2024-03-06T17:44:10.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:10.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:10.062851+0000) 2024-03-06T17:44:11.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:11.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:11.063138+0000) 2024-03-06T17:44:12.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:12.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:12.063413+0000) 2024-03-06T17:44:13.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:13.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:13.063704+0000) 2024-03-06T17:44:14.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:14.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:14.063993+0000) 2024-03-06T17:44:15.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:15.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:15.064221+0000) 2024-03-06T17:44:16.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:16.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:16.064406+0000) 2024-03-06T17:44:17.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:17.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:17.064648+0000) 2024-03-06T17:44:18.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:18.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:18.064899+0000) 2024-03-06T17:44:19.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:19.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:19.065146+0000) 2024-03-06T17:44:20.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:20.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:20.065383+0000) 2024-03-06T17:44:21.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:21.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:21.065663+0000) 2024-03-06T17:44:22.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:22.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:22.065914+0000) 2024-03-06T17:44:22.727 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:44:22.735 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:44:23.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:23.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:23.066154+0000) 2024-03-06T17:44:24.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:24.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:24.066378+0000) 2024-03-06T17:44:25.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:25.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:25.066645+0000) 2024-03-06T17:44:26.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:26.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:26.066866+0000) 2024-03-06T17:44:27.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:27.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:27.067066+0000) 2024-03-06T17:44:28.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:28.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:28.067303+0000) 2024-03-06T17:44:29.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:29.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:29.067566+0000) 2024-03-06T17:44:30.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:30.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:30.067793+0000) 2024-03-06T17:44:31.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:31.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:31.067994+0000) 2024-03-06T17:44:32.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:32.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:32.068222+0000) 2024-03-06T17:44:33.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:33.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:33.068450+0000) 2024-03-06T17:44:34.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:34.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:34.068682+0000) 2024-03-06T17:44:35.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:35.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:35.068911+0000) 2024-03-06T17:44:36.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:36.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:36.069095+0000) 2024-03-06T17:44:37.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:37.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:37.069295+0000) 2024-03-06T17:44:38.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:38.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:38.069491+0000) 2024-03-06T17:44:39.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:39.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:39.069715+0000) 2024-03-06T17:44:40.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:40.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:40.069940+0000) 2024-03-06T17:44:41.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:41.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:41.070182+0000) 2024-03-06T17:44:42.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:42.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:42.070368+0000) 2024-03-06T17:44:43.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:43.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:43.070638+0000) 2024-03-06T17:44:44.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:44.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:44.070860+0000) 2024-03-06T17:44:45.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:45.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:45.071114+0000) 2024-03-06T17:44:46.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:46.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:46.071343+0000) 2024-03-06T17:44:47.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:47.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:47.071619+0000) 2024-03-06T17:44:48.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:48.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:48.071850+0000) 2024-03-06T17:44:49.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:49.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:49.072083+0000) 2024-03-06T17:44:50.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:50.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:50.072307+0000) 2024-03-06T17:44:51.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:51.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:51.072567+0000) 2024-03-06T17:44:52.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:52.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:52.072802+0000) 2024-03-06T17:44:52.942 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:44:52.950 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:44:53.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:53.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:53.073003+0000) 2024-03-06T17:44:54.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:54.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:54.073224+0000) 2024-03-06T17:44:55.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:55.071+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:55.073443+0000) 2024-03-06T17:44:56.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:56.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:56.073663+0000) 2024-03-06T17:44:57.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:57.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:57.073900+0000) 2024-03-06T17:44:58.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:58.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:58.074126+0000) 2024-03-06T17:44:59.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:44:59.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:44:59.074356+0000) 2024-03-06T17:45:00.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:00.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:00.074623+0000) 2024-03-06T17:45:01.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:01.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:01.074858+0000) 2024-03-06T17:45:02.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:02.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:02.075090+0000) 2024-03-06T17:45:03.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:03.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:03.075326+0000) 2024-03-06T17:45:04.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:04.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:04.075571+0000) 2024-03-06T17:45:05.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:05.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:05.075800+0000) 2024-03-06T17:45:06.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:06.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:06.076041+0000) 2024-03-06T17:45:07.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:07.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:07.076354+0000) 2024-03-06T17:45:08.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:08.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:08.076652+0000) 2024-03-06T17:45:09.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:09.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:09.076900+0000) 2024-03-06T17:45:10.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:10.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:10.077139+0000) 2024-03-06T17:45:11.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:11.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:11.077433+0000) 2024-03-06T17:45:12.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:12.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:12.077710+0000) 2024-03-06T17:45:13.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:13.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:13.077947+0000) 2024-03-06T17:45:14.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:14.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:14.078222+0000) 2024-03-06T17:45:15.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:15.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:15.078490+0000) 2024-03-06T17:45:16.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:16.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:16.078658+0000) 2024-03-06T17:45:17.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:17.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:17.078868+0000) 2024-03-06T17:45:18.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:18.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:18.079063+0000) 2024-03-06T17:45:19.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:19.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:19.079289+0000) 2024-03-06T17:45:20.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:20.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:20.079506+0000) 2024-03-06T17:45:21.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:21.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:21.079740+0000) 2024-03-06T17:45:22.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:22.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:22.079978+0000) 2024-03-06T17:45:23.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:23.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:23.080206+0000) 2024-03-06T17:45:23.161 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:45:23.170 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:45:24.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:24.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:24.080426+0000) 2024-03-06T17:45:25.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:25.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:25.080661+0000) 2024-03-06T17:45:26.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:26.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:26.080896+0000) 2024-03-06T17:45:27.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:27.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:27.081100+0000) 2024-03-06T17:45:28.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:28.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:28.081273+0000) 2024-03-06T17:45:29.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:29.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:29.081505+0000) 2024-03-06T17:45:30.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:30.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:30.081756+0000) 2024-03-06T17:45:31.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:31.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:31.081953+0000) 2024-03-06T17:45:32.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:32.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:32.082146+0000) 2024-03-06T17:45:33.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:33.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:33.082392+0000) 2024-03-06T17:45:34.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:34.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:34.082656+0000) 2024-03-06T17:45:35.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:35.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:35.082839+0000) 2024-03-06T17:45:36.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:36.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:36.083066+0000) 2024-03-06T17:45:37.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:37.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:37.083329+0000) 2024-03-06T17:45:38.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:38.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:38.083642+0000) 2024-03-06T17:45:39.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:39.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:39.083911+0000) 2024-03-06T17:45:40.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:40.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:40.084142+0000) 2024-03-06T17:45:41.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:41.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:41.084437+0000) 2024-03-06T17:45:42.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:42.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:42.084720+0000) 2024-03-06T17:45:43.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:43.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:43.084954+0000) 2024-03-06T17:45:44.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:44.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:44.085195+0000) 2024-03-06T17:45:45.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:45.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:45.085482+0000) 2024-03-06T17:45:46.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:46.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:46.085705+0000) 2024-03-06T17:45:47.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:47.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:47.085911+0000) 2024-03-06T17:45:48.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:48.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:48.086093+0000) 2024-03-06T17:45:49.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:49.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:49.086331+0000) 2024-03-06T17:45:50.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:50.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:50.086605+0000) 2024-03-06T17:45:51.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:51.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:51.086811+0000) 2024-03-06T17:45:52.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:52.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:52.087017+0000) 2024-03-06T17:45:53.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:53.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:53.087261+0000) 2024-03-06T17:45:53.298 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:45:53.306 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:45:54.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:54.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:54.087499+0000) 2024-03-06T17:45:55.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:55.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:55.087745+0000) 2024-03-06T17:45:56.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:56.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:56.087995+0000) 2024-03-06T17:45:57.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:57.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:57.088210+0000) 2024-03-06T17:45:58.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:58.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:58.088472+0000) 2024-03-06T17:45:59.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:45:59.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:45:59.088734+0000) 2024-03-06T17:46:00.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:00.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:00.089015+0000) 2024-03-06T17:46:01.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:01.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:01.089339+0000) 2024-03-06T17:46:02.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:02.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:02.089652+0000) 2024-03-06T17:46:03.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:03.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:03.089924+0000) 2024-03-06T17:46:04.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:04.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:04.090130+0000) 2024-03-06T17:46:05.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:05.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:05.090359+0000) 2024-03-06T17:46:06.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:06.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:06.090588+0000) 2024-03-06T17:46:07.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:07.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:07.090819+0000) 2024-03-06T17:46:08.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:08.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:08.091044+0000) 2024-03-06T17:46:09.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:09.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:09.091271+0000) 2024-03-06T17:46:10.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:10.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:10.091494+0000) 2024-03-06T17:46:11.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:11.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:11.091738+0000) 2024-03-06T17:46:12.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:12.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:12.091961+0000) 2024-03-06T17:46:13.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:13.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:13.092199+0000) 2024-03-06T17:46:14.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:14.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:14.092423+0000) 2024-03-06T17:46:15.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:15.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:15.092685+0000) 2024-03-06T17:46:16.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:16.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:16.092949+0000) 2024-03-06T17:46:17.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:17.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:17.093191+0000) 2024-03-06T17:46:18.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:18.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:18.093414+0000) 2024-03-06T17:46:19.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:19.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:19.093672+0000) 2024-03-06T17:46:20.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:20.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:20.093895+0000) 2024-03-06T17:46:21.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:21.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:21.094116+0000) 2024-03-06T17:46:22.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:22.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:22.094358+0000) 2024-03-06T17:46:23.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:23.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:23.094613+0000) 2024-03-06T17:46:23.497 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:46:23.505 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:46:24.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:24.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:24.094809+0000) 2024-03-06T17:46:25.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:25.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:25.095039+0000) 2024-03-06T17:46:26.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:26.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:26.095238+0000) 2024-03-06T17:46:27.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:27.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:27.095483+0000) 2024-03-06T17:46:28.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:28.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:28.095696+0000) 2024-03-06T17:46:29.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:29.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:29.095943+0000) 2024-03-06T17:46:30.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:30.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:30.096180+0000) 2024-03-06T17:46:31.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:31.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:31.096438+0000) 2024-03-06T17:46:32.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:32.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:32.096671+0000) 2024-03-06T17:46:33.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:33.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:33.096915+0000) 2024-03-06T17:46:34.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:34.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:34.097153+0000) 2024-03-06T17:46:35.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:35.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:35.097372+0000) 2024-03-06T17:46:36.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:36.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:36.097652+0000) 2024-03-06T17:46:37.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:37.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:37.097894+0000) 2024-03-06T17:46:38.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:38.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:38.098146+0000) 2024-03-06T17:46:39.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:39.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:39.098322+0000) 2024-03-06T17:46:40.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:40.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:40.098509+0000) 2024-03-06T17:46:41.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:41.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:41.098766+0000) 2024-03-06T17:46:42.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:42.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:42.098973+0000) 2024-03-06T17:46:43.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:43.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:43.099180+0000) 2024-03-06T17:46:44.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:44.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:44.099422+0000) 2024-03-06T17:46:45.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:45.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:45.099668+0000) 2024-03-06T17:46:46.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:46.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:46.099903+0000) 2024-03-06T17:46:47.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:47.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:47.100134+0000) 2024-03-06T17:46:48.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:48.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:48.100340+0000) 2024-03-06T17:46:49.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:49.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:49.100652+0000) 2024-03-06T17:46:50.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:50.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:50.100888+0000) 2024-03-06T17:46:51.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:51.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:51.101121+0000) 2024-03-06T17:46:52.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:52.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:52.101337+0000) 2024-03-06T17:46:53.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:53.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:53.101605+0000) 2024-03-06T17:46:53.634 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:46:53.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:46:54.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:54.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:54.101859+0000) 2024-03-06T17:46:55.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:55.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:55.102104+0000) 2024-03-06T17:46:56.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:56.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:56.102297+0000) 2024-03-06T17:46:57.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:57.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:57.102495+0000) 2024-03-06T17:46:58.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:58.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:58.102738+0000) 2024-03-06T17:46:59.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:46:59.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:46:59.102955+0000) 2024-03-06T17:47:00.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:00.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:00.103146+0000) 2024-03-06T17:47:01.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:01.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:01.103383+0000) 2024-03-06T17:47:02.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:02.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:02.103601+0000) 2024-03-06T17:47:03.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:03.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:03.103818+0000) 2024-03-06T17:47:04.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:04.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:04.104005+0000) 2024-03-06T17:47:05.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:05.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:05.104209+0000) 2024-03-06T17:47:06.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:06.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:06.104430+0000) 2024-03-06T17:47:07.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:07.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:07.104669+0000) 2024-03-06T17:47:08.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:08.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:08.104868+0000) 2024-03-06T17:47:09.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:09.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:09.105065+0000) 2024-03-06T17:47:10.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:10.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:10.105299+0000) 2024-03-06T17:47:11.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:11.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:11.105575+0000) 2024-03-06T17:47:12.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:12.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:12.105768+0000) 2024-03-06T17:47:13.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:13.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:13.105967+0000) 2024-03-06T17:47:14.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:14.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:14.106146+0000) 2024-03-06T17:47:15.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:15.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:15.106375+0000) 2024-03-06T17:47:16.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:16.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:16.106623+0000) 2024-03-06T17:47:17.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:17.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:17.106805+0000) 2024-03-06T17:47:18.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:18.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:18.107040+0000) 2024-03-06T17:47:19.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:19.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:19.107261+0000) 2024-03-06T17:47:20.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:20.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:20.107496+0000) 2024-03-06T17:47:21.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:21.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:21.107697+0000) 2024-03-06T17:47:22.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:22.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:22.107886+0000) 2024-03-06T17:47:23.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:23.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:23.108102+0000) 2024-03-06T17:47:23.848 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:47:23.856 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:47:24.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:24.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:24.108341+0000) 2024-03-06T17:47:25.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:25.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:25.108602+0000) 2024-03-06T17:47:26.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:26.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:26.108790+0000) 2024-03-06T17:47:27.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:27.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:27.108982+0000) 2024-03-06T17:47:28.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:28.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:28.109213+0000) 2024-03-06T17:47:29.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:29.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:29.109447+0000) 2024-03-06T17:47:30.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:30.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:30.109658+0000) 2024-03-06T17:47:31.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:31.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:31.109875+0000) 2024-03-06T17:47:32.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:32.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:32.110060+0000) 2024-03-06T17:47:33.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:33.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:33.110262+0000) 2024-03-06T17:47:34.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:34.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:34.110464+0000) 2024-03-06T17:47:35.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:35.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:35.110717+0000) 2024-03-06T17:47:36.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:36.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:36.110910+0000) 2024-03-06T17:47:37.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:37.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:37.111143+0000) 2024-03-06T17:47:38.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:38.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:38.111430+0000) 2024-03-06T17:47:39.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:39.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:39.111719+0000) 2024-03-06T17:47:40.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:40.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:40.111982+0000) 2024-03-06T17:47:41.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:41.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:41.112234+0000) 2024-03-06T17:47:42.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:42.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:42.112502+0000) 2024-03-06T17:47:43.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:43.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:43.112790+0000) 2024-03-06T17:47:44.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:44.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:44.113022+0000) 2024-03-06T17:47:45.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:45.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:45.113254+0000) 2024-03-06T17:47:46.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:46.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:46.113559+0000) 2024-03-06T17:47:47.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:47.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:47.113778+0000) 2024-03-06T17:47:48.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:48.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:48.114004+0000) 2024-03-06T17:47:49.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:49.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:49.114254+0000) 2024-03-06T17:47:50.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:50.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:50.114569+0000) 2024-03-06T17:47:51.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:51.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:51.114801+0000) 2024-03-06T17:47:52.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:52.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:52.115007+0000) 2024-03-06T17:47:53.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:53.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:53.115285+0000) 2024-03-06T17:47:54.060 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:47:54.069 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:47:54.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:54.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:54.115478+0000) 2024-03-06T17:47:55.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:55.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:55.115687+0000) 2024-03-06T17:47:56.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:56.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:56.115860+0000) 2024-03-06T17:47:57.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:57.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:57.116047+0000) 2024-03-06T17:47:58.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:58.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:58.116284+0000) 2024-03-06T17:47:59.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:47:59.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:47:59.116514+0000) 2024-03-06T17:48:00.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:00.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:00.116752+0000) 2024-03-06T17:48:01.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:01.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:01.116993+0000) 2024-03-06T17:48:02.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:02.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:02.117227+0000) 2024-03-06T17:48:03.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:03.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:03.117461+0000) 2024-03-06T17:48:04.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:04.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:04.117691+0000) 2024-03-06T17:48:05.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:05.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:05.117892+0000) 2024-03-06T17:48:06.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:06.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:06.118120+0000) 2024-03-06T17:48:07.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:07.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:07.118368+0000) 2024-03-06T17:48:08.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:08.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:08.118667+0000) 2024-03-06T17:48:09.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:09.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:09.118898+0000) 2024-03-06T17:48:10.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:10.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:10.119101+0000) 2024-03-06T17:48:11.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:11.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:11.119294+0000) 2024-03-06T17:48:12.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:12.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:12.119489+0000) 2024-03-06T17:48:13.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:13.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:13.119695+0000) 2024-03-06T17:48:14.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:14.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:14.119891+0000) 2024-03-06T17:48:15.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:15.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:15.120084+0000) 2024-03-06T17:48:16.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:16.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:16.120304+0000) 2024-03-06T17:48:17.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:17.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:17.120588+0000) 2024-03-06T17:48:18.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:18.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:18.120789+0000) 2024-03-06T17:48:19.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:19.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:19.121025+0000) 2024-03-06T17:48:20.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:20.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:20.121200+0000) 2024-03-06T17:48:21.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:21.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:21.121432+0000) 2024-03-06T17:48:22.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:22.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:22.121655+0000) 2024-03-06T17:48:23.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:23.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:23.121858+0000) 2024-03-06T17:48:24.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:24.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:24.122047+0000) 2024-03-06T17:48:24.274 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:48:24.282 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:48:25.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:25.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:25.122251+0000) 2024-03-06T17:48:26.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:26.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:26.122489+0000) 2024-03-06T17:48:27.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:27.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:27.122680+0000) 2024-03-06T17:48:28.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:28.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:28.122895+0000) 2024-03-06T17:48:29.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:29.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:29.123125+0000) 2024-03-06T17:48:30.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:30.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:30.123311+0000) 2024-03-06T17:48:31.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:31.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:31.123522+0000) 2024-03-06T17:48:32.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:32.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:32.123748+0000) 2024-03-06T17:48:33.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:33.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:33.123902+0000) 2024-03-06T17:48:34.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:34.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:34.124087+0000) 2024-03-06T17:48:35.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:35.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:35.124278+0000) 2024-03-06T17:48:36.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:36.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:36.124504+0000) 2024-03-06T17:48:37.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:37.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:37.124805+0000) 2024-03-06T17:48:38.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:38.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:38.125048+0000) 2024-03-06T17:48:39.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:39.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:39.125223+0000) 2024-03-06T17:48:40.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:40.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:40.125446+0000) 2024-03-06T17:48:41.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:41.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:41.125706+0000) 2024-03-06T17:48:42.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:42.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:42.125919+0000) 2024-03-06T17:48:43.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:43.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:43.126171+0000) 2024-03-06T17:48:44.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:44.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:44.126438+0000) 2024-03-06T17:48:45.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:45.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:45.126736+0000) 2024-03-06T17:48:46.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:46.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:46.126969+0000) 2024-03-06T17:48:47.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:47.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:47.127244+0000) 2024-03-06T17:48:48.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:48.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:48.127583+0000) 2024-03-06T17:48:49.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:49.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:49.127803+0000) 2024-03-06T17:48:50.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:50.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:50.128005+0000) 2024-03-06T17:48:51.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:51.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:51.128245+0000) 2024-03-06T17:48:52.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:52.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:52.128495+0000) 2024-03-06T17:48:53.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:53.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:53.128804+0000) 2024-03-06T17:48:54.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:54.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:54.129052+0000) 2024-03-06T17:48:54.478 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:48:54.486 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:48:55.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:55.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:55.129338+0000) 2024-03-06T17:48:56.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:56.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:56.129634+0000) 2024-03-06T17:48:57.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:57.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:57.129881+0000) 2024-03-06T17:48:58.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:58.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:58.130118+0000) 2024-03-06T17:48:59.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:48:59.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:48:59.130397+0000) 2024-03-06T17:49:00.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:00.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:00.130672+0000) 2024-03-06T17:49:01.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:01.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:01.130887+0000) 2024-03-06T17:49:02.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:02.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:02.131078+0000) 2024-03-06T17:49:03.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:03.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:03.131310+0000) 2024-03-06T17:49:04.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:04.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:04.131572+0000) 2024-03-06T17:49:05.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:05.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:05.131807+0000) 2024-03-06T17:49:06.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:06.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:06.132037+0000) 2024-03-06T17:49:07.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:07.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:07.132297+0000) 2024-03-06T17:49:08.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:08.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:08.132521+0000) 2024-03-06T17:49:09.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:09.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:09.132760+0000) 2024-03-06T17:49:10.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:10.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:10.132959+0000) 2024-03-06T17:49:11.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:11.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:11.133197+0000) 2024-03-06T17:49:12.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:12.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:12.133432+0000) 2024-03-06T17:49:13.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:13.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:13.133703+0000) 2024-03-06T17:49:14.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:14.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:14.133939+0000) 2024-03-06T17:49:15.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:15.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:15.134162+0000) 2024-03-06T17:49:16.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:16.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:16.134403+0000) 2024-03-06T17:49:17.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:17.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:17.134627+0000) 2024-03-06T17:49:18.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:18.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:18.134905+0000) 2024-03-06T17:49:19.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:19.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:19.135092+0000) 2024-03-06T17:49:20.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:20.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:20.135311+0000) 2024-03-06T17:49:21.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:21.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:21.135580+0000) 2024-03-06T17:49:22.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:22.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:22.135737+0000) 2024-03-06T17:49:23.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:23.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:23.135984+0000) 2024-03-06T17:49:24.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:24.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:24.136188+0000) 2024-03-06T17:49:24.654 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:49:24.661 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:49:25.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:25.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:25.136415+0000) 2024-03-06T17:49:26.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:26.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:26.136642+0000) 2024-03-06T17:49:27.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:27.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:27.136865+0000) 2024-03-06T17:49:28.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:28.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:28.137093+0000) 2024-03-06T17:49:29.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:29.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:29.137362+0000) 2024-03-06T17:49:30.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:30.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:30.137596+0000) 2024-03-06T17:49:31.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:31.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:31.137844+0000) 2024-03-06T17:49:32.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:32.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:32.138075+0000) 2024-03-06T17:49:33.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:33.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:33.138366+0000) 2024-03-06T17:49:34.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:34.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:34.138602+0000) 2024-03-06T17:49:35.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:35.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:35.138802+0000) 2024-03-06T17:49:36.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:36.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:36.139039+0000) 2024-03-06T17:49:37.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:37.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:37.139226+0000) 2024-03-06T17:49:38.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:38.137+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:38.139463+0000) 2024-03-06T17:49:39.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:39.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:39.139701+0000) 2024-03-06T17:49:40.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:40.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:40.139928+0000) 2024-03-06T17:49:41.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:41.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:41.140219+0000) 2024-03-06T17:49:42.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:42.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:42.140460+0000) 2024-03-06T17:49:43.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:43.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:43.140707+0000) 2024-03-06T17:49:44.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:44.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:44.140911+0000) 2024-03-06T17:49:45.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:45.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:45.141143+0000) 2024-03-06T17:49:46.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:46.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:46.141376+0000) 2024-03-06T17:49:47.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:47.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:47.141596+0000) 2024-03-06T17:49:48.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:48.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:48.141825+0000) 2024-03-06T17:49:49.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:49.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:49.142057+0000) 2024-03-06T17:49:50.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:50.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:50.142249+0000) 2024-03-06T17:49:51.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:51.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:51.142477+0000) 2024-03-06T17:49:52.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:52.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:52.142715+0000) 2024-03-06T17:49:53.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:53.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:53.142972+0000) 2024-03-06T17:49:54.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:54.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:54.143162+0000) 2024-03-06T17:49:54.881 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:49:54.889 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:49:55.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:55.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:55.143366+0000) 2024-03-06T17:49:56.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:56.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:56.143602+0000) 2024-03-06T17:49:57.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:57.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:57.143812+0000) 2024-03-06T17:49:58.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:58.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:58.144056+0000) 2024-03-06T17:49:59.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:49:59.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:49:59.144287+0000) 2024-03-06T17:50:00.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:00.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:00.144511+0000) 2024-03-06T17:50:01.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:01.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:01.144745+0000) 2024-03-06T17:50:02.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:02.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:02.144978+0000) 2024-03-06T17:50:03.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:03.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:03.145255+0000) 2024-03-06T17:50:04.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:04.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:04.145567+0000) 2024-03-06T17:50:05.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:05.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:05.145748+0000) 2024-03-06T17:50:06.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:06.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:06.145985+0000) 2024-03-06T17:50:07.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:07.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:07.146215+0000) 2024-03-06T17:50:08.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:08.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:08.146445+0000) 2024-03-06T17:50:09.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:09.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:09.146671+0000) 2024-03-06T17:50:10.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:10.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:10.146889+0000) 2024-03-06T17:50:11.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:11.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:11.147140+0000) 2024-03-06T17:50:12.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:12.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:12.147383+0000) 2024-03-06T17:50:13.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:13.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:13.147659+0000) 2024-03-06T17:50:14.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:14.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:14.147860+0000) 2024-03-06T17:50:15.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:15.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:15.148069+0000) 2024-03-06T17:50:16.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:16.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:16.148258+0000) 2024-03-06T17:50:17.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:17.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:17.148492+0000) 2024-03-06T17:50:18.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:18.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:18.148684+0000) 2024-03-06T17:50:19.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:19.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:19.148893+0000) 2024-03-06T17:50:20.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:20.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:20.149079+0000) 2024-03-06T17:50:21.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:21.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:21.149269+0000) 2024-03-06T17:50:22.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:22.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:22.149579+0000) 2024-03-06T17:50:23.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:23.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:23.149835+0000) 2024-03-06T17:50:24.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:24.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:24.150084+0000) 2024-03-06T17:50:25.084 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:50:25.092 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:50:25.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:25.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:25.150368+0000) 2024-03-06T17:50:26.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:26.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:26.150649+0000) 2024-03-06T17:50:27.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:27.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:27.150841+0000) 2024-03-06T17:50:28.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:28.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:28.151033+0000) 2024-03-06T17:50:29.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:29.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:29.151278+0000) 2024-03-06T17:50:30.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:30.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:30.151503+0000) 2024-03-06T17:50:31.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:31.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:31.151743+0000) 2024-03-06T17:50:32.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:32.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:32.151975+0000) 2024-03-06T17:50:33.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:33.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:33.152221+0000) 2024-03-06T17:50:34.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:34.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:34.152459+0000) 2024-03-06T17:50:35.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:35.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:35.152696+0000) 2024-03-06T17:50:36.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:36.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:36.152902+0000) 2024-03-06T17:50:37.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:37.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:37.153140+0000) 2024-03-06T17:50:38.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:38.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:38.153346+0000) 2024-03-06T17:50:39.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:39.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:39.153620+0000) 2024-03-06T17:50:40.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:40.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:40.153826+0000) 2024-03-06T17:50:41.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:41.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:41.154043+0000) 2024-03-06T17:50:42.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:42.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:42.154265+0000) 2024-03-06T17:50:43.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:43.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:43.154501+0000) 2024-03-06T17:50:44.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:44.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:44.154781+0000) 2024-03-06T17:50:45.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:45.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:45.155009+0000) 2024-03-06T17:50:46.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:46.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:46.155239+0000) 2024-03-06T17:50:47.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:47.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:47.155464+0000) 2024-03-06T17:50:48.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:48.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:48.155672+0000) 2024-03-06T17:50:49.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:49.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:49.155933+0000) 2024-03-06T17:50:50.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:50.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:50.156214+0000) 2024-03-06T17:50:51.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:51.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:51.156500+0000) 2024-03-06T17:50:52.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:52.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:52.156751+0000) 2024-03-06T17:50:53.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:53.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:53.156991+0000) 2024-03-06T17:50:54.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:54.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:54.157261+0000) 2024-03-06T17:50:55.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:55.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:55.157582+0000) 2024-03-06T17:50:55.314 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:50:55.322 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:50:56.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:56.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:56.157757+0000) 2024-03-06T17:50:57.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:57.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:57.157961+0000) 2024-03-06T17:50:58.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:58.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:58.158202+0000) 2024-03-06T17:50:59.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:50:59.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:50:59.158494+0000) 2024-03-06T17:51:00.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:00.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:00.158731+0000) 2024-03-06T17:51:01.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:01.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:01.158955+0000) 2024-03-06T17:51:02.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:02.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:02.159159+0000) 2024-03-06T17:51:03.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:03.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:03.159354+0000) 2024-03-06T17:51:04.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:04.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:04.159637+0000) 2024-03-06T17:51:05.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:05.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:05.159838+0000) 2024-03-06T17:51:06.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:06.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:06.160054+0000) 2024-03-06T17:51:07.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:07.158+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:07.160293+0000) 2024-03-06T17:51:08.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:08.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:08.160561+0000) 2024-03-06T17:51:09.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:09.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:09.160732+0000) 2024-03-06T17:51:10.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:10.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:10.160965+0000) 2024-03-06T17:51:11.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:11.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:11.161192+0000) 2024-03-06T17:51:12.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:12.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:12.161388+0000) 2024-03-06T17:51:13.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:13.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:13.161631+0000) 2024-03-06T17:51:14.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:14.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:14.161877+0000) 2024-03-06T17:51:15.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:15.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:15.162054+0000) 2024-03-06T17:51:16.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:16.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:16.162277+0000) 2024-03-06T17:51:17.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:17.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:17.162499+0000) 2024-03-06T17:51:18.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:18.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:18.162732+0000) 2024-03-06T17:51:19.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:19.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:19.162965+0000) 2024-03-06T17:51:20.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:20.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:20.163153+0000) 2024-03-06T17:51:21.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:21.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:21.163393+0000) 2024-03-06T17:51:22.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:22.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:22.163638+0000) 2024-03-06T17:51:23.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:23.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:23.163811+0000) 2024-03-06T17:51:24.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:24.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:24.164052+0000) 2024-03-06T17:51:25.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:25.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:25.164294+0000) 2024-03-06T17:51:25.527 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:51:25.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:51:26.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:26.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:26.164505+0000) 2024-03-06T17:51:27.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:27.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:27.164775+0000) 2024-03-06T17:51:28.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:28.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:28.165005+0000) 2024-03-06T17:51:29.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:29.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:29.165282+0000) 2024-03-06T17:51:30.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:30.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:30.165596+0000) 2024-03-06T17:51:31.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:31.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:31.165872+0000) 2024-03-06T17:51:32.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:32.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:32.166136+0000) 2024-03-06T17:51:33.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:33.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:33.166414+0000) 2024-03-06T17:51:34.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:34.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:34.166670+0000) 2024-03-06T17:51:35.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:35.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:35.166879+0000) 2024-03-06T17:51:36.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:36.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:36.167084+0000) 2024-03-06T17:51:37.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:37.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:37.167321+0000) 2024-03-06T17:51:38.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:38.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:38.167586+0000) 2024-03-06T17:51:39.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:39.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:39.167828+0000) 2024-03-06T17:51:40.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:40.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:40.168022+0000) 2024-03-06T17:51:41.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:41.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:41.168244+0000) 2024-03-06T17:51:42.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:42.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:42.168511+0000) 2024-03-06T17:51:43.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:43.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:43.168806+0000) 2024-03-06T17:51:44.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:44.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:44.169081+0000) 2024-03-06T17:51:45.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:45.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:45.169363+0000) 2024-03-06T17:51:46.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:46.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:46.169657+0000) 2024-03-06T17:51:47.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:47.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:47.169908+0000) 2024-03-06T17:51:48.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:48.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:48.170094+0000) 2024-03-06T17:51:49.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:49.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:49.170326+0000) 2024-03-06T17:51:50.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:50.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:50.170525+0000) 2024-03-06T17:51:51.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:51.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:51.170791+0000) 2024-03-06T17:51:52.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:52.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:52.171010+0000) 2024-03-06T17:51:53.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:53.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:53.171286+0000) 2024-03-06T17:51:54.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:54.169+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:54.171479+0000) 2024-03-06T17:51:55.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:55.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:55.171685+0000) 2024-03-06T17:51:55.757 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:51:55.765 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:51:56.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:56.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:56.171843+0000) 2024-03-06T17:51:57.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:57.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:57.172064+0000) 2024-03-06T17:51:58.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:58.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:58.172293+0000) 2024-03-06T17:51:59.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:51:59.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:51:59.172517+0000) 2024-03-06T17:52:00.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:00.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:00.172747+0000) 2024-03-06T17:52:01.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:01.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:01.172983+0000) 2024-03-06T17:52:02.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:02.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:02.173204+0000) 2024-03-06T17:52:03.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:03.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:03.173409+0000) 2024-03-06T17:52:04.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:04.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:04.173663+0000) 2024-03-06T17:52:05.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:05.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:05.173868+0000) 2024-03-06T17:52:06.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:06.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:06.174069+0000) 2024-03-06T17:52:07.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:07.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:07.174289+0000) 2024-03-06T17:52:08.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:08.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:08.174568+0000) 2024-03-06T17:52:09.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:09.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:09.174806+0000) 2024-03-06T17:52:10.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:10.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:10.175028+0000) 2024-03-06T17:52:11.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:11.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:11.175257+0000) 2024-03-06T17:52:12.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:12.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:12.175475+0000) 2024-03-06T17:52:13.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:13.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:13.175681+0000) 2024-03-06T17:52:14.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:14.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:14.175926+0000) 2024-03-06T17:52:15.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:15.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:15.176149+0000) 2024-03-06T17:52:16.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:16.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:16.176372+0000) 2024-03-06T17:52:17.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:17.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:17.176572+0000) 2024-03-06T17:52:18.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:18.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:18.176755+0000) 2024-03-06T17:52:19.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:19.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:19.176989+0000) 2024-03-06T17:52:20.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:20.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:20.177178+0000) 2024-03-06T17:52:21.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:21.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:21.177449+0000) 2024-03-06T17:52:22.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:22.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:22.177725+0000) 2024-03-06T17:52:23.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:23.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:23.177967+0000) 2024-03-06T17:52:24.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:24.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:24.178207+0000) 2024-03-06T17:52:25.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:25.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:25.178472+0000) 2024-03-06T17:52:25.932 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:52:25.940 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:52:26.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:26.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:26.178745+0000) 2024-03-06T17:52:27.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:27.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:27.178987+0000) 2024-03-06T17:52:28.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:28.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:28.179216+0000) 2024-03-06T17:52:29.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:29.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:29.179439+0000) 2024-03-06T17:52:30.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:30.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:30.179715+0000) 2024-03-06T17:52:31.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:31.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:31.179982+0000) 2024-03-06T17:52:32.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:32.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:32.180237+0000) 2024-03-06T17:52:33.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:33.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:33.180518+0000) 2024-03-06T17:52:34.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:34.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:34.180801+0000) 2024-03-06T17:52:35.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:35.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:35.181053+0000) 2024-03-06T17:52:36.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:36.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:36.181273+0000) 2024-03-06T17:52:37.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:37.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:37.181490+0000) 2024-03-06T17:52:38.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:38.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:38.181738+0000) 2024-03-06T17:52:39.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:39.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:39.181929+0000) 2024-03-06T17:52:40.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:40.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:40.182153+0000) 2024-03-06T17:52:41.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:41.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:41.182373+0000) 2024-03-06T17:52:42.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:42.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:42.182606+0000) 2024-03-06T17:52:43.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:43.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:43.182848+0000) 2024-03-06T17:52:44.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:44.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:44.183077+0000) 2024-03-06T17:52:45.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:45.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:45.183313+0000) 2024-03-06T17:52:46.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:46.181+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:46.183486+0000) 2024-03-06T17:52:47.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:47.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:47.183677+0000) 2024-03-06T17:52:48.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:48.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:48.183897+0000) 2024-03-06T17:52:49.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:49.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:49.184148+0000) 2024-03-06T17:52:50.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:50.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:50.184412+0000) 2024-03-06T17:52:51.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:51.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:51.184683+0000) 2024-03-06T17:52:52.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:52.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:52.184907+0000) 2024-03-06T17:52:53.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:53.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:53.185111+0000) 2024-03-06T17:52:54.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:54.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:54.185370+0000) 2024-03-06T17:52:55.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:55.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:55.185666+0000) 2024-03-06T17:52:56.141 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:52:56.149 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:52:56.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:56.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:56.185856+0000) 2024-03-06T17:52:57.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:57.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:57.186071+0000) 2024-03-06T17:52:58.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:58.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:58.186256+0000) 2024-03-06T17:52:59.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:52:59.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:52:59.186497+0000) 2024-03-06T17:53:00.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:00.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:00.186729+0000) 2024-03-06T17:53:01.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:01.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:01.186968+0000) 2024-03-06T17:53:02.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:02.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:02.187193+0000) 2024-03-06T17:53:03.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:03.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:03.187403+0000) 2024-03-06T17:53:04.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:04.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:04.187627+0000) 2024-03-06T17:53:05.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:05.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:05.187864+0000) 2024-03-06T17:53:06.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:06.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:06.188057+0000) 2024-03-06T17:53:07.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:07.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:07.188295+0000) 2024-03-06T17:53:08.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:08.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:08.188512+0000) 2024-03-06T17:53:09.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:09.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:09.188741+0000) 2024-03-06T17:53:10.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:10.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:10.188968+0000) 2024-03-06T17:53:11.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:11.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:11.189199+0000) 2024-03-06T17:53:12.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:12.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:12.189421+0000) 2024-03-06T17:53:13.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:13.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:13.189665+0000) 2024-03-06T17:53:14.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:14.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:14.189880+0000) 2024-03-06T17:53:15.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:15.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:15.190112+0000) 2024-03-06T17:53:16.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:16.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:16.190335+0000) 2024-03-06T17:53:17.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:17.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:17.190513+0000) 2024-03-06T17:53:18.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:18.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:18.190787+0000) 2024-03-06T17:53:19.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:19.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:19.191022+0000) 2024-03-06T17:53:20.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:20.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:20.191214+0000) 2024-03-06T17:53:21.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:21.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:21.191444+0000) 2024-03-06T17:53:22.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:22.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:22.191672+0000) 2024-03-06T17:53:23.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:23.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:23.191824+0000) 2024-03-06T17:53:24.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:24.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:24.192037+0000) 2024-03-06T17:53:25.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:25.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:25.192240+0000) 2024-03-06T17:53:26.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:26.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:26.192452+0000) 2024-03-06T17:53:26.353 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:53:26.361 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:53:27.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:27.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:27.192675+0000) 2024-03-06T17:53:28.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:28.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:28.192890+0000) 2024-03-06T17:53:29.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:29.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:29.193183+0000) 2024-03-06T17:53:30.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:30.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:30.193407+0000) 2024-03-06T17:53:31.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:31.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:31.193646+0000) 2024-03-06T17:53:32.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:32.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:32.193852+0000) 2024-03-06T17:53:33.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:33.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:33.194042+0000) 2024-03-06T17:53:34.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:34.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:34.194248+0000) 2024-03-06T17:53:35.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:35.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:35.194460+0000) 2024-03-06T17:53:36.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:36.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:36.194698+0000) 2024-03-06T17:53:37.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:37.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:37.194922+0000) 2024-03-06T17:53:38.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:38.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:38.195130+0000) 2024-03-06T17:53:39.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:39.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:39.195371+0000) 2024-03-06T17:53:40.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:40.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:40.195637+0000) 2024-03-06T17:53:41.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:41.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:41.195871+0000) 2024-03-06T17:53:42.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:42.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:42.196101+0000) 2024-03-06T17:53:43.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:43.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:43.196322+0000) 2024-03-06T17:53:44.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:44.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:44.196597+0000) 2024-03-06T17:53:45.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:45.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:45.196830+0000) 2024-03-06T17:53:46.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:46.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:46.197019+0000) 2024-03-06T17:53:47.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:47.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:47.197242+0000) 2024-03-06T17:53:48.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:48.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:48.197465+0000) 2024-03-06T17:53:49.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:49.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:49.197678+0000) 2024-03-06T17:53:50.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:50.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:50.197922+0000) 2024-03-06T17:53:51.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:51.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:51.198122+0000) 2024-03-06T17:53:52.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:52.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:52.198344+0000) 2024-03-06T17:53:53.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:53.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:53.198610+0000) 2024-03-06T17:53:54.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:54.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:54.198852+0000) 2024-03-06T17:53:55.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:55.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:55.199083+0000) 2024-03-06T17:53:56.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:56.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:56.199305+0000) 2024-03-06T17:53:56.550 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:53:56.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:53:57.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:57.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:57.199565+0000) 2024-03-06T17:53:58.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:58.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:58.199763+0000) 2024-03-06T17:53:59.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:53:59.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:53:59.199970+0000) 2024-03-06T17:54:00.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:00.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:00.200195+0000) 2024-03-06T17:54:01.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:01.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:01.200433+0000) 2024-03-06T17:54:02.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:02.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:02.200650+0000) 2024-03-06T17:54:03.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:03.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:03.200834+0000) 2024-03-06T17:54:04.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:04.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:04.201057+0000) 2024-03-06T17:54:05.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:05.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:05.201303+0000) 2024-03-06T17:54:06.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:06.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:06.201524+0000) 2024-03-06T17:54:07.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:07.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:07.201752+0000) 2024-03-06T17:54:08.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:08.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:08.201953+0000) 2024-03-06T17:54:09.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:09.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:09.202179+0000) 2024-03-06T17:54:10.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:10.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:10.202389+0000) 2024-03-06T17:54:11.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:11.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:11.202645+0000) 2024-03-06T17:54:12.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:12.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:12.202831+0000) 2024-03-06T17:54:13.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:13.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:13.203075+0000) 2024-03-06T17:54:14.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:14.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:14.203275+0000) 2024-03-06T17:54:15.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:15.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:15.203505+0000) 2024-03-06T17:54:16.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:16.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:16.203744+0000) 2024-03-06T17:54:17.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:17.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:17.204026+0000) 2024-03-06T17:54:18.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:18.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:18.204211+0000) 2024-03-06T17:54:19.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:19.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:19.204431+0000) 2024-03-06T17:54:20.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:20.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:20.204671+0000) 2024-03-06T17:54:21.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:21.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:21.204868+0000) 2024-03-06T17:54:22.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:22.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:22.205075+0000) 2024-03-06T17:54:23.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:23.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:23.205313+0000) 2024-03-06T17:54:24.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:24.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:24.205594+0000) 2024-03-06T17:54:25.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:25.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:25.205814+0000) 2024-03-06T17:54:26.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:26.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:26.206014+0000) 2024-03-06T17:54:26.754 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:54:26.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:54:27.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:27.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:27.206261+0000) 2024-03-06T17:54:28.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:28.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:28.206494+0000) 2024-03-06T17:54:29.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:29.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:29.206750+0000) 2024-03-06T17:54:30.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:30.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:30.207012+0000) 2024-03-06T17:54:31.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:31.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:31.207277+0000) 2024-03-06T17:54:32.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:32.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:32.207500+0000) 2024-03-06T17:54:33.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:33.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:33.207769+0000) 2024-03-06T17:54:34.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:34.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:34.208010+0000) 2024-03-06T17:54:35.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:35.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:35.208241+0000) 2024-03-06T17:54:36.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:36.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:36.208466+0000) 2024-03-06T17:54:37.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:37.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:37.209154+0000) 2024-03-06T17:54:38.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:38.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:38.209342+0000) 2024-03-06T17:54:39.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:39.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:39.209616+0000) 2024-03-06T17:54:40.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:40.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:40.209854+0000) 2024-03-06T17:54:41.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:41.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:41.210092+0000) 2024-03-06T17:54:42.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:42.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:42.210315+0000) 2024-03-06T17:54:43.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:43.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:43.210529+0000) 2024-03-06T17:54:44.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:44.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:44.210798+0000) 2024-03-06T17:54:45.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:45.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:45.211035+0000) 2024-03-06T17:54:46.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:46.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:46.211272+0000) 2024-03-06T17:54:47.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:47.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:47.211513+0000) 2024-03-06T17:54:48.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:48.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:48.211788+0000) 2024-03-06T17:54:49.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:49.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:49.212027+0000) 2024-03-06T17:54:50.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:50.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:50.212231+0000) 2024-03-06T17:54:51.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:51.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:51.212485+0000) 2024-03-06T17:54:52.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:52.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:52.212729+0000) 2024-03-06T17:54:53.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:53.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:53.212950+0000) 2024-03-06T17:54:54.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:54.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:54.213161+0000) 2024-03-06T17:54:55.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:55.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:55.213409+0000) 2024-03-06T17:54:56.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:56.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:56.213677+0000) 2024-03-06T17:54:56.884 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:54:56.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:54:57.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:57.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:57.213896+0000) 2024-03-06T17:54:58.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:58.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:58.214093+0000) 2024-03-06T17:54:59.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:54:59.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:54:59.214311+0000) 2024-03-06T17:55:00.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:00.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:00.214579+0000) 2024-03-06T17:55:01.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:01.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:01.214791+0000) 2024-03-06T17:55:02.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:02.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:02.214970+0000) 2024-03-06T17:55:03.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:03.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:03.215203+0000) 2024-03-06T17:55:04.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:04.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:04.215450+0000) 2024-03-06T17:55:05.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:05.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:05.215716+0000) 2024-03-06T17:55:06.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:06.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:06.215957+0000) 2024-03-06T17:55:07.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:07.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:07.216158+0000) 2024-03-06T17:55:08.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:08.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:08.216368+0000) 2024-03-06T17:55:09.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:09.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:09.216615+0000) 2024-03-06T17:55:10.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:10.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:10.216922+0000) 2024-03-06T17:55:11.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:11.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:11.217214+0000) 2024-03-06T17:55:12.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:12.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:12.217479+0000) 2024-03-06T17:55:13.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:13.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:13.217709+0000) 2024-03-06T17:55:14.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:14.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:14.217991+0000) 2024-03-06T17:55:15.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:15.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:15.218273+0000) 2024-03-06T17:55:16.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:16.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:16.218587+0000) 2024-03-06T17:55:17.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:17.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:17.218807+0000) 2024-03-06T17:55:18.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:18.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:18.219089+0000) 2024-03-06T17:55:19.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:19.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:19.219295+0000) 2024-03-06T17:55:20.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:20.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:20.219521+0000) 2024-03-06T17:55:21.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:21.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:21.219798+0000) 2024-03-06T17:55:22.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:22.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:22.220244+0000) 2024-03-06T17:55:23.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:23.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:23.220481+0000) 2024-03-06T17:55:24.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:24.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:24.220765+0000) 2024-03-06T17:55:25.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:25.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:25.221052+0000) 2024-03-06T17:55:26.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:26.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:26.221318+0000) 2024-03-06T17:55:27.097 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:55:27.105 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:55:27.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:27.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:27.221554+0000) 2024-03-06T17:55:28.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:28.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:28.221739+0000) 2024-03-06T17:55:29.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:29.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:29.221971+0000) 2024-03-06T17:55:30.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:30.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:30.222199+0000) 2024-03-06T17:55:31.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:31.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:31.222437+0000) 2024-03-06T17:55:32.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:32.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:32.222691+0000) 2024-03-06T17:55:33.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:33.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:33.222938+0000) 2024-03-06T17:55:34.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:34.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:34.223125+0000) 2024-03-06T17:55:35.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:35.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:35.223351+0000) 2024-03-06T17:55:36.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:36.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:36.223526+0000) 2024-03-06T17:55:37.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:37.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:37.223767+0000) 2024-03-06T17:55:38.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:38.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:38.223990+0000) 2024-03-06T17:55:39.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:39.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:39.224185+0000) 2024-03-06T17:55:40.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:40.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:40.224412+0000) 2024-03-06T17:55:41.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:41.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:41.224643+0000) 2024-03-06T17:55:42.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:42.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:42.224885+0000) 2024-03-06T17:55:43.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:43.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:43.225120+0000) 2024-03-06T17:55:44.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:44.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:44.225355+0000) 2024-03-06T17:55:45.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:45.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:45.225592+0000) 2024-03-06T17:55:46.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:46.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:46.225781+0000) 2024-03-06T17:55:47.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:47.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:47.225982+0000) 2024-03-06T17:55:48.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:48.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:48.226221+0000) 2024-03-06T17:55:49.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:49.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:49.226461+0000) 2024-03-06T17:55:50.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:50.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:50.226700+0000) 2024-03-06T17:55:51.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:51.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:51.226888+0000) 2024-03-06T17:55:52.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:52.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:52.227078+0000) 2024-03-06T17:55:53.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:53.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:53.227354+0000) 2024-03-06T17:55:54.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:54.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:54.227640+0000) 2024-03-06T17:55:55.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:55.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:55.227881+0000) 2024-03-06T17:55:56.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:56.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:56.228105+0000) 2024-03-06T17:55:57.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:57.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:57.228345+0000) 2024-03-06T17:55:57.304 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:55:57.312 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:55:58.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:58.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:58.228644+0000) 2024-03-06T17:55:59.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:55:59.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:55:59.228851+0000) 2024-03-06T17:56:00.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:00.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:00.229038+0000) 2024-03-06T17:56:01.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:01.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:01.229280+0000) 2024-03-06T17:56:02.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:02.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:02.229476+0000) 2024-03-06T17:56:03.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:03.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:03.229697+0000) 2024-03-06T17:56:04.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:04.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:04.229923+0000) 2024-03-06T17:56:05.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:05.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:05.230109+0000) 2024-03-06T17:56:06.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:06.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:06.230347+0000) 2024-03-06T17:56:07.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:07.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:07.230522+0000) 2024-03-06T17:56:08.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:08.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:08.230749+0000) 2024-03-06T17:56:09.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:09.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:09.230999+0000) 2024-03-06T17:56:10.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:10.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:10.231243+0000) 2024-03-06T17:56:11.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:11.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:11.231503+0000) 2024-03-06T17:56:12.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:12.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:12.231777+0000) 2024-03-06T17:56:13.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:13.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:13.231990+0000) 2024-03-06T17:56:14.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:14.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:14.232234+0000) 2024-03-06T17:56:15.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:15.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:15.232480+0000) 2024-03-06T17:56:16.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:16.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:16.232715+0000) 2024-03-06T17:56:17.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:17.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:17.232941+0000) 2024-03-06T17:56:18.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:18.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:18.233179+0000) 2024-03-06T17:56:19.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:19.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:19.233411+0000) 2024-03-06T17:56:20.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:20.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:20.233664+0000) 2024-03-06T17:56:21.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:21.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:21.233874+0000) 2024-03-06T17:56:22.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:22.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:22.234117+0000) 2024-03-06T17:56:23.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:23.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:23.234350+0000) 2024-03-06T17:56:24.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:24.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:24.234567+0000) 2024-03-06T17:56:25.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:25.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:25.234805+0000) 2024-03-06T17:56:26.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:26.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:26.235034+0000) 2024-03-06T17:56:27.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:27.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:27.235258+0000) 2024-03-06T17:56:27.437 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:56:27.447 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:56:28.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:28.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:28.235475+0000) 2024-03-06T17:56:29.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:29.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:29.235714+0000) 2024-03-06T17:56:30.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:30.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:30.235942+0000) 2024-03-06T17:56:31.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:31.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:31.236171+0000) 2024-03-06T17:56:32.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:32.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:32.236388+0000) 2024-03-06T17:56:33.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:33.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:33.236643+0000) 2024-03-06T17:56:34.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:34.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:34.236867+0000) 2024-03-06T17:56:35.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:35.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:35.237103+0000) 2024-03-06T17:56:36.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:36.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:36.237312+0000) 2024-03-06T17:56:37.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:37.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:37.237563+0000) 2024-03-06T17:56:38.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:38.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:38.237756+0000) 2024-03-06T17:56:39.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:39.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:39.237956+0000) 2024-03-06T17:56:40.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:40.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:40.238141+0000) 2024-03-06T17:56:41.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:41.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:41.238380+0000) 2024-03-06T17:56:42.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:42.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:42.238639+0000) 2024-03-06T17:56:43.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:43.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:43.238863+0000) 2024-03-06T17:56:44.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:44.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:44.239083+0000) 2024-03-06T17:56:45.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:45.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:45.239318+0000) 2024-03-06T17:56:46.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:46.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:46.239633+0000) 2024-03-06T17:56:47.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:47.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:47.239868+0000) 2024-03-06T17:56:48.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:48.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:48.240068+0000) 2024-03-06T17:56:49.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:49.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:49.240291+0000) 2024-03-06T17:56:50.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:50.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:50.240470+0000) 2024-03-06T17:56:51.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:51.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:51.240710+0000) 2024-03-06T17:56:52.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:52.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:52.240883+0000) 2024-03-06T17:56:53.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:53.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:53.241084+0000) 2024-03-06T17:56:54.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:54.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:54.241275+0000) 2024-03-06T17:56:55.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:55.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:55.241564+0000) 2024-03-06T17:56:56.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:56.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:56.241786+0000) 2024-03-06T17:56:57.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:57.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:57.241981+0000) 2024-03-06T17:56:57.579 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:56:57.587 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:56:58.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:58.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:58.242201+0000) 2024-03-06T17:56:59.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:56:59.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:56:59.242425+0000) 2024-03-06T17:57:00.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:00.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:00.242666+0000) 2024-03-06T17:57:01.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:01.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:01.242899+0000) 2024-03-06T17:57:02.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:02.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:02.243141+0000) 2024-03-06T17:57:03.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:03.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:03.243385+0000) 2024-03-06T17:57:04.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:04.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:04.243660+0000) 2024-03-06T17:57:05.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:05.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:05.243908+0000) 2024-03-06T17:57:06.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:06.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:06.244143+0000) 2024-03-06T17:57:07.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:07.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:07.244381+0000) 2024-03-06T17:57:08.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:08.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:08.244625+0000) 2024-03-06T17:57:09.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:09.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:09.244853+0000) 2024-03-06T17:57:10.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:10.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:10.245043+0000) 2024-03-06T17:57:11.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:11.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:11.245275+0000) 2024-03-06T17:57:12.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:12.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:12.245487+0000) 2024-03-06T17:57:13.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:13.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:13.245692+0000) 2024-03-06T17:57:14.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:14.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:14.245920+0000) 2024-03-06T17:57:15.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:15.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:15.246150+0000) 2024-03-06T17:57:16.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:16.245+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:16.246347+0000) 2024-03-06T17:57:17.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:17.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:17.246598+0000) 2024-03-06T17:57:18.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:18.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:18.246850+0000) 2024-03-06T17:57:19.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:19.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:19.247087+0000) 2024-03-06T17:57:20.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:20.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:20.247273+0000) 2024-03-06T17:57:21.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:21.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:21.247510+0000) 2024-03-06T17:57:22.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:22.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:22.247730+0000) 2024-03-06T17:57:23.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:23.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:23.247945+0000) 2024-03-06T17:57:24.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:24.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:24.248145+0000) 2024-03-06T17:57:25.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:25.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:25.248389+0000) 2024-03-06T17:57:26.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:26.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:26.248677+0000) 2024-03-06T17:57:27.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:27.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:27.248912+0000) 2024-03-06T17:57:27.707 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:57:27.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:57:28.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:28.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:28.249154+0000) 2024-03-06T17:57:29.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:29.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:29.249365+0000) 2024-03-06T17:57:30.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:30.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:30.249639+0000) 2024-03-06T17:57:31.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:31.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:31.249828+0000) 2024-03-06T17:57:32.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:32.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:32.250056+0000) 2024-03-06T17:57:33.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:33.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:33.250340+0000) 2024-03-06T17:57:34.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:34.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:34.250689+0000) 2024-03-06T17:57:35.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:35.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:35.250964+0000) 2024-03-06T17:57:36.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:36.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:36.251225+0000) 2024-03-06T17:57:37.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:37.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:37.251452+0000) 2024-03-06T17:57:38.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:38.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:38.251654+0000) 2024-03-06T17:57:39.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:39.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:39.251847+0000) 2024-03-06T17:57:40.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:40.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:40.252043+0000) 2024-03-06T17:57:41.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:41.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:41.252295+0000) 2024-03-06T17:57:42.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:42.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:42.252516+0000) 2024-03-06T17:57:43.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:43.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:43.252778+0000) 2024-03-06T17:57:44.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:44.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:44.253004+0000) 2024-03-06T17:57:45.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:45.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:45.253192+0000) 2024-03-06T17:57:46.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:46.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:46.253432+0000) 2024-03-06T17:57:47.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:47.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:47.253660+0000) 2024-03-06T17:57:48.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:48.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:48.253852+0000) 2024-03-06T17:57:49.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:49.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:49.254087+0000) 2024-03-06T17:57:50.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:50.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:50.254274+0000) 2024-03-06T17:57:51.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:51.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:51.254484+0000) 2024-03-06T17:57:52.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:52.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:52.254713+0000) 2024-03-06T17:57:53.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:53.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:53.254940+0000) 2024-03-06T17:57:54.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:54.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:54.255177+0000) 2024-03-06T17:57:55.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:55.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:55.255429+0000) 2024-03-06T17:57:56.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:56.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:56.255694+0000) 2024-03-06T17:57:57.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:57.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:57.255901+0000) 2024-03-06T17:57:57.930 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:57:57.939 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:57:58.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:58.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:58.256090+0000) 2024-03-06T17:57:59.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:57:59.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:57:59.256313+0000) 2024-03-06T17:58:00.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:00.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:00.256567+0000) 2024-03-06T17:58:01.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:01.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:01.256809+0000) 2024-03-06T17:58:02.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:02.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:02.257028+0000) 2024-03-06T17:58:03.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:03.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:03.257250+0000) 2024-03-06T17:58:04.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:04.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:04.257431+0000) 2024-03-06T17:58:05.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:05.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:05.257687+0000) 2024-03-06T17:58:06.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:06.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:06.257882+0000) 2024-03-06T17:58:07.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:07.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:07.258082+0000) 2024-03-06T17:58:08.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:08.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:08.258303+0000) 2024-03-06T17:58:09.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:09.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:09.258507+0000) 2024-03-06T17:58:10.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:10.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:10.258791+0000) 2024-03-06T17:58:11.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:11.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:11.259027+0000) 2024-03-06T17:58:12.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:12.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:12.259247+0000) 2024-03-06T17:58:13.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:13.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:13.259479+0000) 2024-03-06T17:58:14.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:14.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:14.259679+0000) 2024-03-06T17:58:15.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:15.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:15.259922+0000) 2024-03-06T17:58:16.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:16.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:16.260132+0000) 2024-03-06T17:58:17.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:17.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:17.260319+0000) 2024-03-06T17:58:18.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:18.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:18.260568+0000) 2024-03-06T17:58:19.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:19.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:19.260783+0000) 2024-03-06T17:58:20.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:20.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:20.261055+0000) 2024-03-06T17:58:21.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:21.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:21.261353+0000) 2024-03-06T17:58:22.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:22.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:22.261667+0000) 2024-03-06T17:58:23.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:23.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:23.261890+0000) 2024-03-06T17:58:24.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:24.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:24.262130+0000) 2024-03-06T17:58:25.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:25.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:25.262380+0000) 2024-03-06T17:58:26.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:26.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:26.262670+0000) 2024-03-06T17:58:27.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:27.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:27.262957+0000) 2024-03-06T17:58:28.127 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:58:28.135 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:58:28.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:28.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:28.263203+0000) 2024-03-06T17:58:29.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:29.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:29.263448+0000) 2024-03-06T17:58:30.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:30.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:30.263736+0000) 2024-03-06T17:58:31.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:31.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:31.264041+0000) 2024-03-06T17:58:32.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:32.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:32.264304+0000) 2024-03-06T17:58:33.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:33.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:33.264609+0000) 2024-03-06T17:58:34.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:34.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:34.264890+0000) 2024-03-06T17:58:35.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:35.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:35.265182+0000) 2024-03-06T17:58:36.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:36.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:36.265460+0000) 2024-03-06T17:58:37.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:37.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:37.265696+0000) 2024-03-06T17:58:38.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:38.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:38.265883+0000) 2024-03-06T17:58:39.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:39.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:39.266109+0000) 2024-03-06T17:58:40.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:40.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:40.266339+0000) 2024-03-06T17:58:41.267 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:41.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:41.266619+0000) 2024-03-06T17:58:42.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:42.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:42.266795+0000) 2024-03-06T17:58:43.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:43.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:43.267005+0000) 2024-03-06T17:58:44.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:44.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:44.267186+0000) 2024-03-06T17:58:45.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:45.266+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:45.267427+0000) 2024-03-06T17:58:46.268 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:46.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:46.267651+0000) 2024-03-06T17:58:47.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:47.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:47.267878+0000) 2024-03-06T17:58:48.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:48.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:48.268108+0000) 2024-03-06T17:58:49.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:49.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:49.268341+0000) 2024-03-06T17:58:50.269 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:50.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:50.268599+0000) 2024-03-06T17:58:51.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:51.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:51.268856+0000) 2024-03-06T17:58:52.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:52.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:52.269074+0000) 2024-03-06T17:58:53.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:53.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:53.269267+0000) 2024-03-06T17:58:54.270 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:54.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:54.269458+0000) 2024-03-06T17:58:55.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:55.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:55.269709+0000) 2024-03-06T17:58:56.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:56.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:56.269949+0000) 2024-03-06T17:58:57.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:57.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:57.270182+0000) 2024-03-06T17:58:58.271 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:58.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:58.270364+0000) 2024-03-06T17:58:58.293 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:58:58.300 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:58:59.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:58:59.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:58:59.270635+0000) 2024-03-06T17:59:00.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:00.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:00.270853+0000) 2024-03-06T17:59:01.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:01.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:01.271094+0000) 2024-03-06T17:59:02.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:02.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:02.271319+0000) 2024-03-06T17:59:03.272 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:03.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:03.271513+0000) 2024-03-06T17:59:04.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:04.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:04.271772+0000) 2024-03-06T17:59:05.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:05.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:05.272005+0000) 2024-03-06T17:59:06.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:06.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:06.272221+0000) 2024-03-06T17:59:07.273 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:07.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:07.272435+0000) 2024-03-06T17:59:08.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:08.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:08.272654+0000) 2024-03-06T17:59:09.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:09.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:09.272882+0000) 2024-03-06T17:59:10.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:10.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:10.273087+0000) 2024-03-06T17:59:11.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:11.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:11.273332+0000) 2024-03-06T17:59:12.274 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:12.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:12.273600+0000) 2024-03-06T17:59:13.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:13.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:13.273842+0000) 2024-03-06T17:59:14.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:14.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:14.274062+0000) 2024-03-06T17:59:15.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:15.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:15.274316+0000) 2024-03-06T17:59:16.275 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:16.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:16.274494+0000) 2024-03-06T17:59:17.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:17.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:17.274719+0000) 2024-03-06T17:59:18.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:18.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:18.274944+0000) 2024-03-06T17:59:19.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:19.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:19.275173+0000) 2024-03-06T17:59:20.276 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:20.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:20.275408+0000) 2024-03-06T17:59:21.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:21.274+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:21.275690+0000) 2024-03-06T17:59:22.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:22.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:22.275910+0000) 2024-03-06T17:59:23.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:23.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:23.276126+0000) 2024-03-06T17:59:24.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:24.275+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:24.276347+0000) 2024-03-06T17:59:25.277 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:25.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:25.276616+0000) 2024-03-06T17:59:26.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:26.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:26.276854+0000) 2024-03-06T17:59:27.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:27.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:27.277041+0000) 2024-03-06T17:59:28.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:28.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:28.277256+0000) 2024-03-06T17:59:28.522 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:59:28.530 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:59:29.278 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:29.276+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:29.277440+0000) 2024-03-06T17:59:30.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:30.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:30.277682+0000) 2024-03-06T17:59:31.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:31.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:31.277928+0000) 2024-03-06T17:59:32.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:32.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:32.278146+0000) 2024-03-06T17:59:33.279 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:33.277+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:33.278327+0000) 2024-03-06T17:59:34.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:34.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:34.278574+0000) 2024-03-06T17:59:35.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:35.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:35.278780+0000) 2024-03-06T17:59:36.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:36.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:36.278980+0000) 2024-03-06T17:59:37.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:37.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:37.279170+0000) 2024-03-06T17:59:38.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:38.278+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:38.279352+0000) 2024-03-06T17:59:39.280 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:39.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:39.279624+0000) 2024-03-06T17:59:40.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:40.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:40.279865+0000) 2024-03-06T17:59:41.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:41.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:41.280058+0000) 2024-03-06T17:59:42.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:42.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:42.280286+0000) 2024-03-06T17:59:43.281 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:43.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:43.280516+0000) 2024-03-06T17:59:44.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:44.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:44.280746+0000) 2024-03-06T17:59:45.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:45.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:45.280995+0000) 2024-03-06T17:59:46.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:46.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:46.281217+0000) 2024-03-06T17:59:47.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:47.279+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:47.281437+0000) 2024-03-06T17:59:48.282 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:48.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:48.281664+0000) 2024-03-06T17:59:49.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:49.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:49.281887+0000) 2024-03-06T17:59:50.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:50.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:50.282084+0000) 2024-03-06T17:59:51.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:51.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:51.282295+0000) 2024-03-06T17:59:52.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:52.280+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:52.282495+0000) 2024-03-06T17:59:53.283 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:53.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:53.282772+0000) 2024-03-06T17:59:54.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:54.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:54.282990+0000) 2024-03-06T17:59:55.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:55.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:55.283232+0000) 2024-03-06T17:59:56.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:56.281+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:56.283455+0000) 2024-03-06T17:59:57.284 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:57.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:57.283685+0000) 2024-03-06T17:59:58.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:58.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:58.283911+0000) 2024-03-06T17:59:58.726 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:59:58.736 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T17:59:59.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T17:59:59.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T16:59:59.284131+0000) 2024-03-06T18:00:00.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:00.282+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:00.284368+0000) 2024-03-06T18:00:01.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:01.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:01.284643+0000) 2024-03-06T18:00:02.285 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:02.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:02.284833+0000) 2024-03-06T18:00:03.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:03.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:03.285058+0000) 2024-03-06T18:00:04.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:04.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:04.285238+0000) 2024-03-06T18:00:05.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:05.283+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:05.285492+0000) 2024-03-06T18:00:06.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:06.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:06.285757+0000) 2024-03-06T18:00:07.286 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:07.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:07.285953+0000) 2024-03-06T18:00:08.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:08.284+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:08.286183+0000) 2024-03-06T18:00:09.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:09.285+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:09.286415+0000) 2024-03-06T18:00:10.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:10.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:10.286676+0000) 2024-03-06T18:00:11.287 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:11.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:11.286897+0000) 2024-03-06T18:00:12.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:12.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:12.287146+0000) 2024-03-06T18:00:13.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:13.286+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:13.287419+0000) 2024-03-06T18:00:14.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:14.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:14.287656+0000) 2024-03-06T18:00:15.288 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:15.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:15.287932+0000) 2024-03-06T18:00:16.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:16.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:16.288210+0000) 2024-03-06T18:00:17.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:17.287+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:17.288483+0000) 2024-03-06T18:00:18.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:18.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:18.288729+0000) 2024-03-06T18:00:19.289 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:19.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:19.288959+0000) 2024-03-06T18:00:20.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:20.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:20.289180+0000) 2024-03-06T18:00:21.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:21.288+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:21.289433+0000) 2024-03-06T18:00:22.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:22.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:22.289688+0000) 2024-03-06T18:00:23.290 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:23.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:23.289886+0000) 2024-03-06T18:00:24.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:24.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:24.290082+0000) 2024-03-06T18:00:25.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:25.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:25.290291+0000) 2024-03-06T18:00:26.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:26.289+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:26.290468+0000) 2024-03-06T18:00:27.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:27.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:27.290699+0000) 2024-03-06T18:00:28.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:28.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:28.290962+0000) 2024-03-06T18:00:28.939 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:00:28.946 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:00:29.291 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:29.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:29.291171+0000) 2024-03-06T18:00:30.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:30.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:30.291410+0000) 2024-03-06T18:00:31.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:31.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:31.291642+0000) 2024-03-06T18:00:32.292 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:32.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:32.291802+0000) 2024-03-06T18:00:33.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:33.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:33.292015+0000) 2024-03-06T18:00:34.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:34.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:34.292217+0000) 2024-03-06T18:00:35.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:35.290+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:35.292411+0000) 2024-03-06T18:00:36.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:36.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:36.292643+0000) 2024-03-06T18:00:37.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:37.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:37.292838+0000) 2024-03-06T18:00:38.293 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:38.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:38.293022+0000) 2024-03-06T18:00:39.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:39.291+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:39.293289+0000) 2024-03-06T18:00:40.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:40.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:40.293570+0000) 2024-03-06T18:00:41.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:41.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:41.293819+0000) 2024-03-06T18:00:42.294 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:42.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:42.294031+0000) 2024-03-06T18:00:43.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:43.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:43.294277+0000) 2024-03-06T18:00:44.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:44.292+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:44.294436+0000) 2024-03-06T18:00:45.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:45.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:45.294675+0000) 2024-03-06T18:00:46.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:46.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:46.294869+0000) 2024-03-06T18:00:47.295 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:47.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:47.295124+0000) 2024-03-06T18:00:48.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:48.293+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:48.295357+0000) 2024-03-06T18:00:49.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:49.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:49.295625+0000) 2024-03-06T18:00:50.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:50.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:50.295825+0000) 2024-03-06T18:00:51.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:51.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:51.296064+0000) 2024-03-06T18:00:52.296 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:52.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:52.296248+0000) 2024-03-06T18:00:53.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:53.294+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:53.296484+0000) 2024-03-06T18:00:54.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:54.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:54.296726+0000) 2024-03-06T18:00:55.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:55.295+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:55.296945+0000) 2024-03-06T18:00:56.297 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:56.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:56.297158+0000) 2024-03-06T18:00:57.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:57.296+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:57.297470+0000) 2024-03-06T18:00:58.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:58.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:58.297750+0000) 2024-03-06T18:00:59.085 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:00:59.093 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:00:59.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:00:59.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:00:59.297967+0000) 2024-03-06T18:01:00.298 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:00.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:00.298173+0000) 2024-03-06T18:01:01.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:01.297+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:01.298376+0000) 2024-03-06T18:01:02.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:02.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:02.298665+0000) 2024-03-06T18:01:03.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:03.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:03.298878+0000) 2024-03-06T18:01:04.299 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:04.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:04.299164+0000) 2024-03-06T18:01:05.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:05.298+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:05.299447+0000) 2024-03-06T18:01:06.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:06.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:06.299736+0000) 2024-03-06T18:01:07.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:07.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:07.299952+0000) 2024-03-06T18:01:08.300 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:08.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:08.300226+0000) 2024-03-06T18:01:09.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:09.299+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:09.300508+0000) 2024-03-06T18:01:10.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:10.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:10.300848+0000) 2024-03-06T18:01:11.301 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:11.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:11.301137+0000) 2024-03-06T18:01:12.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:12.300+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:12.301403+0000) 2024-03-06T18:01:13.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:13.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:13.301678+0000) 2024-03-06T18:01:14.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:14.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:14.301911+0000) 2024-03-06T18:01:15.302 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:15.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:15.302194+0000) 2024-03-06T18:01:16.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:16.301+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:16.302473+0000) 2024-03-06T18:01:17.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:17.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:17.302681+0000) 2024-03-06T18:01:18.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:18.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:18.302891+0000) 2024-03-06T18:01:19.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:19.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:19.303129+0000) 2024-03-06T18:01:20.303 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:20.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:20.303372+0000) 2024-03-06T18:01:21.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:21.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:21.303665+0000) 2024-03-06T18:01:22.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:22.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:22.303875+0000) 2024-03-06T18:01:23.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:23.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:23.304107+0000) 2024-03-06T18:01:24.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:24.302+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:24.304347+0000) 2024-03-06T18:01:25.304 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:25.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:25.304650+0000) 2024-03-06T18:01:26.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:26.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:26.304889+0000) 2024-03-06T18:01:27.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:27.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:27.305119+0000) 2024-03-06T18:01:28.305 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:28.303+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:28.305301+0000) 2024-03-06T18:01:29.299 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:01:29.307 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:01:29.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:29.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:29.305521+0000) 2024-03-06T18:01:30.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:30.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:30.305781+0000) 2024-03-06T18:01:31.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:31.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:31.305983+0000) 2024-03-06T18:01:32.306 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:32.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:32.306194+0000) 2024-03-06T18:01:33.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:33.304+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:33.306370+0000) 2024-03-06T18:01:34.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:34.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:34.306643+0000) 2024-03-06T18:01:35.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:35.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:35.306840+0000) 2024-03-06T18:01:36.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:36.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:36.307066+0000) 2024-03-06T18:01:37.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:37.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:37.307283+0000) 2024-03-06T18:01:38.307 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:38.305+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:38.307488+0000) 2024-03-06T18:01:39.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:39.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:39.307723+0000) 2024-03-06T18:01:40.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:40.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:40.308009+0000) 2024-03-06T18:01:41.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:41.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:41.308311+0000) 2024-03-06T18:01:42.308 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:42.306+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:42.308503+0000) 2024-03-06T18:01:43.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:43.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:43.308733+0000) 2024-03-06T18:01:44.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:44.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:44.308957+0000) 2024-03-06T18:01:45.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:45.308+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:45.309284+0000) 2024-03-06T18:01:46.309 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:46.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:46.309608+0000) 2024-03-06T18:01:47.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:47.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:47.309861+0000) 2024-03-06T18:01:48.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:48.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:48.310133+0000) 2024-03-06T18:01:49.310 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:49.309+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:49.310330+0000) 2024-03-06T18:01:50.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:50.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:50.310589+0000) 2024-03-06T18:01:51.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:51.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:51.310891+0000) 2024-03-06T18:01:52.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:52.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:52.311165+0000) 2024-03-06T18:01:53.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:53.310+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:53.311371+0000) 2024-03-06T18:01:54.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:54.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:54.311635+0000) 2024-03-06T18:01:55.311 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:55.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:55.311863+0000) 2024-03-06T18:01:56.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:56.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:56.312086+0000) 2024-03-06T18:01:57.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:57.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:57.312293+0000) 2024-03-06T18:01:58.312 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:58.311+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:58.312519+0000) 2024-03-06T18:01:59.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:01:59.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:01:59.312775+0000) 2024-03-06T18:01:59.494 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:01:59.504 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:02:00.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:00.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:00.312971+0000) 2024-03-06T18:02:01.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:01.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:01.313169+0000) 2024-03-06T18:02:02.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:02.312+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:02.313397+0000) 2024-03-06T18:02:03.313 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:03.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:03.313646+0000) 2024-03-06T18:02:04.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:04.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:04.313871+0000) 2024-03-06T18:02:05.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:05.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:05.314137+0000) 2024-03-06T18:02:06.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:06.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:06.314426+0000) 2024-03-06T18:02:07.314 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:07.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:07.314692+0000) 2024-03-06T18:02:08.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:08.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:08.314977+0000) 2024-03-06T18:02:09.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:09.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:09.315261+0000) 2024-03-06T18:02:10.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:10.313+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:10.315483+0000) 2024-03-06T18:02:11.315 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:11.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:11.315735+0000) 2024-03-06T18:02:12.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:12.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:12.315963+0000) 2024-03-06T18:02:13.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:13.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:13.316153+0000) 2024-03-06T18:02:14.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:14.314+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:14.316365+0000) 2024-03-06T18:02:15.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:15.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:15.316621+0000) 2024-03-06T18:02:16.316 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:16.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:16.316849+0000) 2024-03-06T18:02:17.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:17.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:17.317036+0000) 2024-03-06T18:02:18.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:18.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:18.317256+0000) 2024-03-06T18:02:19.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:19.315+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:19.317464+0000) 2024-03-06T18:02:20.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:20.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:20.317702+0000) 2024-03-06T18:02:21.317 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:21.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:21.317939+0000) 2024-03-06T18:02:22.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:22.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:22.318164+0000) 2024-03-06T18:02:23.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:23.316+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:23.318375+0000) 2024-03-06T18:02:24.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:24.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:24.318663+0000) 2024-03-06T18:02:25.318 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:25.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:25.318900+0000) 2024-03-06T18:02:26.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:26.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:26.319128+0000) 2024-03-06T18:02:27.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:27.317+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:27.319339+0000) 2024-03-06T18:02:28.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:28.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:28.319563+0000) 2024-03-06T18:02:29.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:29.318+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:29.319801+0000) 2024-03-06T18:02:29.715 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:02:29.723 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:02:30.319 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:30.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:30.320028+0000) 2024-03-06T18:02:31.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:31.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:31.320268+0000) 2024-03-06T18:02:32.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:32.319+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:32.320493+0000) 2024-03-06T18:02:33.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:33.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:33.320696+0000) 2024-03-06T18:02:34.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:34.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:34.320922+0000) 2024-03-06T18:02:35.320 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:35.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:35.321142+0000) 2024-03-06T18:02:36.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:36.320+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:36.321369+0000) 2024-03-06T18:02:37.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:37.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:37.321595+0000) 2024-03-06T18:02:38.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:38.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:38.321871+0000) 2024-03-06T18:02:39.321 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:39.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:39.322092+0000) 2024-03-06T18:02:40.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:40.321+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:40.322324+0000) 2024-03-06T18:02:41.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:41.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:41.322608+0000) 2024-03-06T18:02:42.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:42.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:42.322800+0000) 2024-03-06T18:02:43.322 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:43.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:43.323023+0000) 2024-03-06T18:02:44.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:44.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:44.323247+0000) 2024-03-06T18:02:45.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:45.322+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:45.323470+0000) 2024-03-06T18:02:46.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:46.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:46.323702+0000) 2024-03-06T18:02:47.323 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:47.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:47.323920+0000) 2024-03-06T18:02:48.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:48.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:48.324143+0000) 2024-03-06T18:02:49.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:49.323+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:49.324367+0000) 2024-03-06T18:02:50.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:50.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:50.324636+0000) 2024-03-06T18:02:51.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:51.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:51.324878+0000) 2024-03-06T18:02:52.324 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:52.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:52.325074+0000) 2024-03-06T18:02:53.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:53.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:53.325308+0000) 2024-03-06T18:02:54.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:54.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:54.325495+0000) 2024-03-06T18:02:55.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:55.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:55.325758+0000) 2024-03-06T18:02:56.325 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:56.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:56.325983+0000) 2024-03-06T18:02:57.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:57.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:57.326195+0000) 2024-03-06T18:02:58.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:58.324+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:58.326437+0000) 2024-03-06T18:02:59.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:02:59.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:02:59.326682+0000) 2024-03-06T18:02:59.945 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:02:59.953 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:03:00.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:00.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:00.326882+0000) 2024-03-06T18:03:01.326 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:01.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:01.327127+0000) 2024-03-06T18:03:02.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:02.325+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:02.327348+0000) 2024-03-06T18:03:03.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:03.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:03.327603+0000) 2024-03-06T18:03:04.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:04.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:04.327872+0000) 2024-03-06T18:03:05.327 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:05.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:05.328119+0000) 2024-03-06T18:03:06.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:06.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:06.328328+0000) 2024-03-06T18:03:07.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:07.326+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:07.328509+0000) 2024-03-06T18:03:08.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:08.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:08.328772+0000) 2024-03-06T18:03:09.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:09.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:09.328990+0000) 2024-03-06T18:03:10.328 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:10.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:10.329218+0000) 2024-03-06T18:03:11.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:11.327+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:11.329425+0000) 2024-03-06T18:03:12.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:12.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:12.329660+0000) 2024-03-06T18:03:13.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:13.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:13.329856+0000) 2024-03-06T18:03:14.329 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:14.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:14.330087+0000) 2024-03-06T18:03:15.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:15.328+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:15.330312+0000) 2024-03-06T18:03:16.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:16.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:16.330527+0000) 2024-03-06T18:03:17.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:17.329+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:17.330767+0000) 2024-03-06T18:03:18.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:18.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:18.330993+0000) 2024-03-06T18:03:19.330 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:19.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:19.331211+0000) 2024-03-06T18:03:20.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:20.330+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:20.331436+0000) 2024-03-06T18:03:21.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:21.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:21.331763+0000) 2024-03-06T18:03:22.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:22.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:22.331961+0000) 2024-03-06T18:03:23.331 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:23.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:23.332177+0000) 2024-03-06T18:03:24.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:24.331+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:24.332408+0000) 2024-03-06T18:03:25.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:25.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:25.332693+0000) 2024-03-06T18:03:26.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:26.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:26.332904+0000) 2024-03-06T18:03:27.332 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:27.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:27.333117+0000) 2024-03-06T18:03:28.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:28.332+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:28.333339+0000) 2024-03-06T18:03:29.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:29.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:29.333597+0000) 2024-03-06T18:03:30.081 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:03:30.089 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:03:30.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:30.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:30.333823+0000) 2024-03-06T18:03:31.333 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:31.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:31.334039+0000) 2024-03-06T18:03:32.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:32.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:32.334236+0000) 2024-03-06T18:03:33.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:33.333+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:33.334435+0000) 2024-03-06T18:03:34.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:34.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:34.334668+0000) 2024-03-06T18:03:35.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:35.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:35.334850+0000) 2024-03-06T18:03:36.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:36.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:36.335077+0000) 2024-03-06T18:03:37.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:37.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:37.335296+0000) 2024-03-06T18:03:38.334 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:38.334+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:38.335516+0000) 2024-03-06T18:03:39.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:39.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:39.335740+0000) 2024-03-06T18:03:40.335 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:40.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:40.335966+0000) 2024-03-06T18:03:41.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:41.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:41.336251+0000) 2024-03-06T18:03:42.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:42.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:42.336495+0000) 2024-03-06T18:03:43.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:43.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:43.336710+0000) 2024-03-06T18:03:44.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:44.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:44.336941+0000) 2024-03-06T18:03:45.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:45.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:45.337161+0000) 2024-03-06T18:03:46.336 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:46.335+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:46.337393+0000) 2024-03-06T18:03:47.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:47.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:47.337617+0000) 2024-03-06T18:03:48.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:48.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:48.337813+0000) 2024-03-06T18:03:49.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:49.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:49.338008+0000) 2024-03-06T18:03:50.337 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:50.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:50.338183+0000) 2024-03-06T18:03:51.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:51.336+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:51.338423+0000) 2024-03-06T18:03:52.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:52.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:52.338657+0000) 2024-03-06T18:03:53.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:53.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:53.338878+0000) 2024-03-06T18:03:54.338 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:54.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:54.339104+0000) 2024-03-06T18:03:55.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:55.337+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:55.339334+0000) 2024-03-06T18:03:56.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:56.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:56.339575+0000) 2024-03-06T18:03:57.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:57.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:57.339766+0000) 2024-03-06T18:03:58.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:58.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:58.339999+0000) 2024-03-06T18:03:59.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:03:59.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:03:59.340182+0000) 2024-03-06T18:04:00.328 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:04:00.338 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:04:00.339 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:00.338+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:00.340384+0000) 2024-03-06T18:04:01.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:01.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:01.340648+0000) 2024-03-06T18:04:02.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:02.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:02.340874+0000) 2024-03-06T18:04:03.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:03.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:03.341092+0000) 2024-03-06T18:04:04.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:04.339+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:04.341313+0000) 2024-03-06T18:04:05.340 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:05.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:05.341570+0000) 2024-03-06T18:04:06.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:06.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:06.341771+0000) 2024-03-06T18:04:07.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:07.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:07.341971+0000) 2024-03-06T18:04:08.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:08.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:08.342201+0000) 2024-03-06T18:04:09.341 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:09.341+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:09.342418+0000) 2024-03-06T18:04:10.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:10.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:10.342641+0000) 2024-03-06T18:04:11.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:11.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:11.342838+0000) 2024-03-06T18:04:12.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:12.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:12.343051+0000) 2024-03-06T18:04:13.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:13.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:13.343234+0000) 2024-03-06T18:04:14.342 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:14.342+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:14.343451+0000) 2024-03-06T18:04:15.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:15.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:15.343688+0000) 2024-03-06T18:04:16.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:16.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:16.343929+0000) 2024-03-06T18:04:17.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:17.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:17.344102+0000) 2024-03-06T18:04:18.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:18.343+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:18.344334+0000) 2024-03-06T18:04:19.343 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:19.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:19.344607+0000) 2024-03-06T18:04:20.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:20.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:20.344831+0000) 2024-03-06T18:04:21.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:21.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:21.345061+0000) 2024-03-06T18:04:22.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:22.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:22.345258+0000) 2024-03-06T18:04:23.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:23.344+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:23.345433+0000) 2024-03-06T18:04:24.344 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:24.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:24.345662+0000) 2024-03-06T18:04:25.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:25.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:25.345893+0000) 2024-03-06T18:04:26.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:26.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:26.346139+0000) 2024-03-06T18:04:27.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:27.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:27.346312+0000) 2024-03-06T18:04:28.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:28.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:28.346589+0000) 2024-03-06T18:04:29.345 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:29.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:29.346836+0000) 2024-03-06T18:04:30.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:30.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:30.347002+0000) 2024-03-06T18:04:30.524 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:04:30.532 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:04:31.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:31.345+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:31.347251+0000) 2024-03-06T18:04:32.346 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:32.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:32.347523+0000) 2024-03-06T18:04:33.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:33.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:33.347831+0000) 2024-03-06T18:04:34.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:34.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:34.348072+0000) 2024-03-06T18:04:35.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:35.346+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:35.348293+0000) 2024-03-06T18:04:36.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:36.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:36.348568+0000) 2024-03-06T18:04:37.347 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:37.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:37.348787+0000) 2024-03-06T18:04:38.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:38.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:38.349011+0000) 2024-03-06T18:04:39.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:39.347+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:39.349249+0000) 2024-03-06T18:04:40.348 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:40.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:40.349577+0000) 2024-03-06T18:04:41.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:41.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:41.349853+0000) 2024-03-06T18:04:42.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:42.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:42.350118+0000) 2024-03-06T18:04:43.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:43.348+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:43.350381+0000) 2024-03-06T18:04:44.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:44.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:44.350611+0000) 2024-03-06T18:04:45.349 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:45.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:45.350853+0000) 2024-03-06T18:04:46.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:46.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:46.351118+0000) 2024-03-06T18:04:47.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:47.349+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:47.351351+0000) 2024-03-06T18:04:48.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:48.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:48.351631+0000) 2024-03-06T18:04:49.350 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:49.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:49.351868+0000) 2024-03-06T18:04:50.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:50.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:50.352153+0000) 2024-03-06T18:04:51.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:51.350+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:51.352374+0000) 2024-03-06T18:04:52.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:52.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:52.352640+0000) 2024-03-06T18:04:53.351 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:53.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:53.352877+0000) 2024-03-06T18:04:54.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:54.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:54.353110+0000) 2024-03-06T18:04:55.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:55.352+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:55.353317+0000) 2024-03-06T18:04:56.352 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:56.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:56.353618+0000) 2024-03-06T18:04:57.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:57.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:57.353891+0000) 2024-03-06T18:04:58.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:58.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:58.354172+0000) 2024-03-06T18:04:59.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:04:59.353+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:04:59.354407+0000) 2024-03-06T18:05:00.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:00.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:00.354683+0000) 2024-03-06T18:05:00.729 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:05:00.737 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:05:01.353 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:01.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:01.354980+0000) 2024-03-06T18:05:02.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:02.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:02.355189+0000) 2024-03-06T18:05:03.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:03.354+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:03.355410+0000) 2024-03-06T18:05:04.354 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:04.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:04.355634+0000) 2024-03-06T18:05:05.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:05.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:05.355857+0000) 2024-03-06T18:05:06.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:06.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:06.356099+0000) 2024-03-06T18:05:07.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:07.355+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:07.356291+0000) 2024-03-06T18:05:08.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:08.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:08.356526+0000) 2024-03-06T18:05:09.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:09.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:09.356791+0000) 2024-03-06T18:05:10.355 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:10.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:10.357021+0000) 2024-03-06T18:05:11.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:11.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:11.357267+0000) 2024-03-06T18:05:12.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:12.356+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:12.357485+0000) 2024-03-06T18:05:13.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:13.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:13.357749+0000) 2024-03-06T18:05:14.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:14.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:14.357966+0000) 2024-03-06T18:05:15.356 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:15.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:15.358186+0000) 2024-03-06T18:05:16.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:16.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:16.358425+0000) 2024-03-06T18:05:17.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:17.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:17.358639+0000) 2024-03-06T18:05:18.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:18.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:18.358828+0000) 2024-03-06T18:05:19.357 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:19.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:19.359046+0000) 2024-03-06T18:05:20.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:20.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:20.359282+0000) 2024-03-06T18:05:21.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:21.357+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:21.359493+0000) 2024-03-06T18:05:22.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:22.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:22.359689+0000) 2024-03-06T18:05:23.358 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:23.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:23.359903+0000) 2024-03-06T18:05:24.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:24.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:24.360098+0000) 2024-03-06T18:05:25.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:25.358+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:25.360334+0000) 2024-03-06T18:05:26.359 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:26.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:26.360619+0000) 2024-03-06T18:05:27.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:27.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:27.360837+0000) 2024-03-06T18:05:28.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:28.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:28.361042+0000) 2024-03-06T18:05:29.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:29.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:29.361261+0000) 2024-03-06T18:05:30.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:30.359+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:30.361483+0000) 2024-03-06T18:05:30.950 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:05:30.958 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:05:31.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:31.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:31.361719+0000) 2024-03-06T18:05:32.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:32.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:32.361963+0000) 2024-03-06T18:05:33.360 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:33.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:33.362184+0000) 2024-03-06T18:05:34.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:34.360+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:34.362412+0000) 2024-03-06T18:05:35.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:35.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:35.362658+0000) 2024-03-06T18:05:36.361 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:36.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:36.362901+0000) 2024-03-06T18:05:37.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:37.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:37.363152+0000) 2024-03-06T18:05:38.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:38.361+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:38.363386+0000) 2024-03-06T18:05:39.362 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:39.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:39.363637+0000) 2024-03-06T18:05:40.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:40.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:40.363869+0000) 2024-03-06T18:05:41.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:41.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:41.364103+0000) 2024-03-06T18:05:42.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:42.363+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:42.364310+0000) 2024-03-06T18:05:43.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:43.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:43.364560+0000) 2024-03-06T18:05:44.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:44.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:44.364775+0000) 2024-03-06T18:05:45.363 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:45.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:45.365052+0000) 2024-03-06T18:05:46.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:46.364+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:46.365344+0000) 2024-03-06T18:05:47.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:47.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:47.365658+0000) 2024-03-06T18:05:48.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:48.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:48.365939+0000) 2024-03-06T18:05:49.364 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:49.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:49.366176+0000) 2024-03-06T18:05:50.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:50.365+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:50.366430+0000) 2024-03-06T18:05:51.365 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:51.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:51.366734+0000) 2024-03-06T18:05:52.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:52.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:52.367006+0000) 2024-03-06T18:05:53.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:53.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:53.367218+0000) 2024-03-06T18:05:54.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:54.366+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:54.367433+0000) 2024-03-06T18:05:55.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:55.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:55.367727+0000) 2024-03-06T18:05:56.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:56.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:56.367965+0000) 2024-03-06T18:05:57.366 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:57.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:57.368200+0000) 2024-03-06T18:05:58.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:58.367+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:58.368374+0000) 2024-03-06T18:05:59.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:05:59.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:05:59.368619+0000) 2024-03-06T18:06:00.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:00.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:00.368825+0000) 2024-03-06T18:06:01.078 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:06:01.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:06:01.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:01.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:01.369018+0000) 2024-03-06T18:06:02.367 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:02.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:02.369246+0000) 2024-03-06T18:06:03.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:03.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:03.369466+0000) 2024-03-06T18:06:04.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:04.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:04.369704+0000) 2024-03-06T18:06:05.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:05.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:05.369900+0000) 2024-03-06T18:06:06.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:06.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:06.370158+0000) 2024-03-06T18:06:07.368 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:07.368+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:07.370352+0000) 2024-03-06T18:06:08.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:08.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:08.370607+0000) 2024-03-06T18:06:09.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:09.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:09.370843+0000) 2024-03-06T18:06:10.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:10.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:10.371068+0000) 2024-03-06T18:06:11.369 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:11.369+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:11.371313+0000) 2024-03-06T18:06:12.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:12.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:12.371569+0000) 2024-03-06T18:06:13.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:13.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:13.371811+0000) 2024-03-06T18:06:14.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:14.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:14.371993+0000) 2024-03-06T18:06:15.370 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:15.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:15.372230+0000) 2024-03-06T18:06:16.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:16.370+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:16.372469+0000) 2024-03-06T18:06:17.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:17.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:17.372753+0000) 2024-03-06T18:06:18.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:18.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:18.373000+0000) 2024-03-06T18:06:19.371 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:19.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:19.373218+0000) 2024-03-06T18:06:20.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:20.371+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:20.373421+0000) 2024-03-06T18:06:21.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:21.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:21.373691+0000) 2024-03-06T18:06:22.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:22.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:22.373933+0000) 2024-03-06T18:06:23.372 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:23.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:23.374151+0000) 2024-03-06T18:06:24.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:24.372+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:24.374379+0000) 2024-03-06T18:06:25.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:25.373+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:25.374603+0000) 2024-03-06T18:06:26.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:26.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:26.374837+0000) 2024-03-06T18:06:27.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:27.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:27.375019+0000) 2024-03-06T18:06:28.373 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:28.374+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:28.375290+0000) 2024-03-06T18:06:29.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:29.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:29.375568+0000) 2024-03-06T18:06:30.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:30.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:30.375812+0000) 2024-03-06T18:06:31.275 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:06:31.283 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:06:31.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:31.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:31.376044+0000) 2024-03-06T18:06:32.374 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:32.375+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:32.376320+0000) 2024-03-06T18:06:33.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:33.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:33.376640+0000) 2024-03-06T18:06:34.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:34.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:34.376918+0000) 2024-03-06T18:06:35.375 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:35.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:35.377201+0000) 2024-03-06T18:06:36.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:36.376+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:36.377489+0000) 2024-03-06T18:06:37.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:37.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:37.377694+0000) 2024-03-06T18:06:38.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:38.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:38.377938+0000) 2024-03-06T18:06:39.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:39.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:39.378145+0000) 2024-03-06T18:06:40.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:40.377+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:40.378381+0000) 2024-03-06T18:06:41.376 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:41.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:41.378634+0000) 2024-03-06T18:06:42.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:42.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:42.378860+0000) 2024-03-06T18:06:43.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:43.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:43.379083+0000) 2024-03-06T18:06:44.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:44.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:44.379283+0000) 2024-03-06T18:06:45.377 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:45.378+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:45.379499+0000) 2024-03-06T18:06:46.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:46.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:46.379777+0000) 2024-03-06T18:06:47.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:47.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:47.379951+0000) 2024-03-06T18:06:48.378 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:48.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:48.380134+0000) 2024-03-06T18:06:49.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:49.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:49.380396+0000) 2024-03-06T18:06:50.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:50.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:50.380646+0000) 2024-03-06T18:06:51.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:51.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:51.380853+0000) 2024-03-06T18:06:52.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:52.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:52.381083+0000) 2024-03-06T18:06:53.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:53.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:53.381272+0000) 2024-03-06T18:06:54.379 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:54.379+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:54.381517+0000) 2024-03-06T18:06:55.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:55.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:55.381777+0000) 2024-03-06T18:06:56.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:56.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:56.381993+0000) 2024-03-06T18:06:57.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:57.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:57.382232+0000) 2024-03-06T18:06:58.380 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:58.380+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:58.382456+0000) 2024-03-06T18:06:59.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:06:59.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:06:59.382692+0000) 2024-03-06T18:07:00.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:00.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:00.382885+0000) 2024-03-06T18:07:01.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:01.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:01.383107+0000) 2024-03-06T18:07:01.479 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:07:01.487 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:07:02.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:02.381+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:02.383334+0000) 2024-03-06T18:07:03.381 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:03.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:03.383559+0000) 2024-03-06T18:07:04.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:04.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:04.383808+0000) 2024-03-06T18:07:05.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:05.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:05.384050+0000) 2024-03-06T18:07:06.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:06.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:06.384284+0000) 2024-03-06T18:07:07.382 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:07.382+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:07.384510+0000) 2024-03-06T18:07:08.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:08.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:08.384770+0000) 2024-03-06T18:07:09.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:09.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:09.385058+0000) 2024-03-06T18:07:10.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:10.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:10.385268+0000) 2024-03-06T18:07:11.383 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:11.383+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:11.385519+0000) 2024-03-06T18:07:12.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:12.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:12.385756+0000) 2024-03-06T18:07:13.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:13.384+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:13.385965+0000) 2024-03-06T18:07:14.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:14.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:14.386202+0000) 2024-03-06T18:07:15.384 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:15.385+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:15.386461+0000) 2024-03-06T18:07:16.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:16.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:16.386698+0000) 2024-03-06T18:07:17.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:17.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:17.386925+0000) 2024-03-06T18:07:18.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:18.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:18.387175+0000) 2024-03-06T18:07:19.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:19.386+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:19.387389+0000) 2024-03-06T18:07:20.385 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:20.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:20.387639+0000) 2024-03-06T18:07:21.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:21.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:21.387870+0000) 2024-03-06T18:07:22.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:22.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:22.388114+0000) 2024-03-06T18:07:23.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:23.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:23.388306+0000) 2024-03-06T18:07:24.386 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:24.387+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:24.388513+0000) 2024-03-06T18:07:25.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:25.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:25.388780+0000) 2024-03-06T18:07:26.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:26.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:26.389001+0000) 2024-03-06T18:07:27.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:27.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:27.389228+0000) 2024-03-06T18:07:28.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:28.388+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:28.389446+0000) 2024-03-06T18:07:29.387 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:29.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:29.389667+0000) 2024-03-06T18:07:30.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:30.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:30.389894+0000) 2024-03-06T18:07:31.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:31.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:31.390099+0000) 2024-03-06T18:07:31.752 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:07:31.760 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:07:32.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:32.389+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:32.390333+0000) 2024-03-06T18:07:33.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:33.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:33.390595+0000) 2024-03-06T18:07:34.388 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:34.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:34.390826+0000) 2024-03-06T18:07:35.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:35.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:35.391051+0000) 2024-03-06T18:07:36.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:36.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:36.391226+0000) 2024-03-06T18:07:37.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:37.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:37.391422+0000) 2024-03-06T18:07:38.389 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:38.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:38.391665+0000) 2024-03-06T18:07:39.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:39.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:39.391902+0000) 2024-03-06T18:07:40.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:40.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:40.392140+0000) 2024-03-06T18:07:41.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:41.390+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:41.392394+0000) 2024-03-06T18:07:42.390 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:42.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:42.392630+0000) 2024-03-06T18:07:43.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:43.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:43.392857+0000) 2024-03-06T18:07:44.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:44.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:44.393076+0000) 2024-03-06T18:07:45.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:45.391+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:45.393309+0000) 2024-03-06T18:07:46.391 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:46.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:46.393566+0000) 2024-03-06T18:07:47.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:47.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:47.393795+0000) 2024-03-06T18:07:48.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:48.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:48.394015+0000) 2024-03-06T18:07:49.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:49.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:49.394238+0000) 2024-03-06T18:07:50.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:50.392+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:50.394423+0000) 2024-03-06T18:07:51.392 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:51.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:51.394684+0000) 2024-03-06T18:07:52.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:52.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:52.394900+0000) 2024-03-06T18:07:53.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:53.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:53.395113+0000) 2024-03-06T18:07:54.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:54.393+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:54.395353+0000) 2024-03-06T18:07:55.393 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:55.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:55.395657+0000) 2024-03-06T18:07:56.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:56.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:56.395922+0000) 2024-03-06T18:07:57.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:57.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:57.396162+0000) 2024-03-06T18:07:58.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:58.394+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:58.396450+0000) 2024-03-06T18:07:59.394 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:07:59.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:07:59.396699+0000) 2024-03-06T18:08:00.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:00.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:00.396869+0000) 2024-03-06T18:08:01.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:01.395+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:01.397119+0000) 2024-03-06T18:08:01.880 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:08:01.890 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:08:02.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:02.396+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:02.397333+0000) 2024-03-06T18:08:03.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:03.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:03.397634+0000) 2024-03-06T18:08:04.395 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:04.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:04.397868+0000) 2024-03-06T18:08:05.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:05.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:05.398116+0000) 2024-03-06T18:08:06.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:06.397+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:06.398313+0000) 2024-03-06T18:08:07.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:07.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:07.398572+0000) 2024-03-06T18:08:08.396 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:08.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:08.398776+0000) 2024-03-06T18:08:09.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:09.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:09.398998+0000) 2024-03-06T18:08:10.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:10.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:10.399231+0000) 2024-03-06T18:08:11.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:11.398+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:11.399517+0000) 2024-03-06T18:08:12.397 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:12.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:12.399843+0000) 2024-03-06T18:08:13.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:13.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:13.400083+0000) 2024-03-06T18:08:14.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:14.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:14.400312+0000) 2024-03-06T18:08:15.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:15.399+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:15.400510+0000) 2024-03-06T18:08:16.398 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:16.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:16.400781+0000) 2024-03-06T18:08:17.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:17.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:17.400973+0000) 2024-03-06T18:08:18.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:18.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:18.401195+0000) 2024-03-06T18:08:19.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:19.400+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:19.401423+0000) 2024-03-06T18:08:20.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:20.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:20.401647+0000) 2024-03-06T18:08:21.399 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:21.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:21.401888+0000) 2024-03-06T18:08:22.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:22.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:22.402115+0000) 2024-03-06T18:08:23.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:23.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:23.402341+0000) 2024-03-06T18:08:24.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:24.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:24.402568+0000) 2024-03-06T18:08:25.400 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:25.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:25.402798+0000) 2024-03-06T18:08:26.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:26.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:26.403037+0000) 2024-03-06T18:08:27.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:27.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:27.403219+0000) 2024-03-06T18:08:28.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:28.401+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:28.403439+0000) 2024-03-06T18:08:29.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:29.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:29.403686+0000) 2024-03-06T18:08:30.401 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:30.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:30.403953+0000) 2024-03-06T18:08:31.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:31.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:31.404191+0000) 2024-03-06T18:08:32.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:08:32.102 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:08:32.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:32.402+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:32.404418+0000) 2024-03-06T18:08:33.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:33.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:33.404675+0000) 2024-03-06T18:08:34.402 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:34.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:34.404866+0000) 2024-03-06T18:08:35.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:35.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:35.405048+0000) 2024-03-06T18:08:36.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:36.403+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:36.405266+0000) 2024-03-06T18:08:37.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:37.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:37.405529+0000) 2024-03-06T18:08:38.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:38.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:38.405851+0000) 2024-03-06T18:08:39.403 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:39.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:39.406058+0000) 2024-03-06T18:08:40.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:40.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:40.406274+0000) 2024-03-06T18:08:41.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:41.404+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:41.406470+0000) 2024-03-06T18:08:42.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:42.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:42.406660+0000) 2024-03-06T18:08:43.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:43.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:43.406892+0000) 2024-03-06T18:08:44.404 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:44.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:44.407074+0000) 2024-03-06T18:08:45.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:45.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:45.407263+0000) 2024-03-06T18:08:46.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:46.405+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:46.407500+0000) 2024-03-06T18:08:47.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:47.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:47.407745+0000) 2024-03-06T18:08:48.405 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:48.406+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:48.408007+0000) 2024-03-06T18:08:49.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:49.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:49.408231+0000) 2024-03-06T18:08:50.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:50.407+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:50.408453+0000) 2024-03-06T18:08:51.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:51.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:51.408704+0000) 2024-03-06T18:08:52.406 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:52.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:52.408935+0000) 2024-03-06T18:08:53.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:53.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:53.409173+0000) 2024-03-06T18:08:54.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:54.408+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:54.409397+0000) 2024-03-06T18:08:55.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:55.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:55.409653+0000) 2024-03-06T18:08:56.407 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:56.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:56.409839+0000) 2024-03-06T18:08:57.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:57.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:57.410126+0000) 2024-03-06T18:08:58.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:58.409+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:58.410365+0000) 2024-03-06T18:08:59.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:08:59.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:08:59.410596+0000) 2024-03-06T18:09:00.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:00.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:00.410833+0000) 2024-03-06T18:09:01.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:01.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:01.411109+0000) 2024-03-06T18:09:02.315 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:09:02.324 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:09:02.408 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:02.410+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:02.411297+0000) 2024-03-06T18:09:03.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:03.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:03.411611+0000) 2024-03-06T18:09:04.409 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:04.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:04.411894+0000) 2024-03-06T18:09:05.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:05.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:05.412174+0000) 2024-03-06T18:09:06.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:06.411+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:06.412463+0000) 2024-03-06T18:09:07.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:07.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:07.412736+0000) 2024-03-06T18:09:08.410 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:08.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:08.412966+0000) 2024-03-06T18:09:09.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:09.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:09.413200+0000) 2024-03-06T18:09:10.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:10.412+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:10.413480+0000) 2024-03-06T18:09:11.411 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:11.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:11.413807+0000) 2024-03-06T18:09:12.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:12.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:12.414089+0000) 2024-03-06T18:09:13.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:13.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:13.414358+0000) 2024-03-06T18:09:14.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:14.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:14.414677+0000) 2024-03-06T18:09:15.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:15.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:15.414953+0000) 2024-03-06T18:09:16.412 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:16.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:16.415164+0000) 2024-03-06T18:09:17.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:17.413+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:17.415385+0000) 2024-03-06T18:09:18.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:18.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:18.415710+0000) 2024-03-06T18:09:19.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:19.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:19.415930+0000) 2024-03-06T18:09:20.413 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:20.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:20.416115+0000) 2024-03-06T18:09:21.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:21.414+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:21.416340+0000) 2024-03-06T18:09:22.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:22.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:22.416595+0000) 2024-03-06T18:09:23.414 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:23.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:23.416820+0000) 2024-03-06T18:09:24.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:24.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:24.417035+0000) 2024-03-06T18:09:25.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:25.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:25.417278+0000) 2024-03-06T18:09:26.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:26.415+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:26.417502+0000) 2024-03-06T18:09:27.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:27.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:27.417744+0000) 2024-03-06T18:09:28.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:28.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:28.417927+0000) 2024-03-06T18:09:29.415 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:29.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:29.418160+0000) 2024-03-06T18:09:30.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:30.416+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:30.418340+0000) 2024-03-06T18:09:31.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:31.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:31.418562+0000) 2024-03-06T18:09:32.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:32.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:32.418765+0000) 2024-03-06T18:09:32.528 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:09:32.535 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:09:33.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:33.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:33.418974+0000) 2024-03-06T18:09:34.416 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:34.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:34.419158+0000) 2024-03-06T18:09:35.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:35.417+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:35.419366+0000) 2024-03-06T18:09:36.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:36.418+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:36.419614+0000) 2024-03-06T18:09:37.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:37.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:37.419839+0000) 2024-03-06T18:09:38.417 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:38.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:38.420083+0000) 2024-03-06T18:09:39.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:39.419+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:39.420308+0000) 2024-03-06T18:09:40.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:40.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:40.420563+0000) 2024-03-06T18:09:41.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:41.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:41.420799+0000) 2024-03-06T18:09:42.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:42.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:42.421029+0000) 2024-03-06T18:09:43.418 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:43.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:43.421259+0000) 2024-03-06T18:09:44.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:44.420+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:44.421481+0000) 2024-03-06T18:09:45.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:45.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:45.421696+0000) 2024-03-06T18:09:46.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:46.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:46.421883+0000) 2024-03-06T18:09:47.419 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:47.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:47.422114+0000) 2024-03-06T18:09:48.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:48.421+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:48.422335+0000) 2024-03-06T18:09:49.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:49.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:49.422522+0000) 2024-03-06T18:09:50.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:50.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:50.422766+0000) 2024-03-06T18:09:51.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:51.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:51.422961+0000) 2024-03-06T18:09:52.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:52.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:52.423150+0000) 2024-03-06T18:09:53.420 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:53.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:53.423344+0000) 2024-03-06T18:09:54.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:54.422+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:54.423506+0000) 2024-03-06T18:09:55.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:55.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:55.423735+0000) 2024-03-06T18:09:56.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:56.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:56.423908+0000) 2024-03-06T18:09:57.421 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:57.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:57.424136+0000) 2024-03-06T18:09:58.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:58.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:58.424361+0000) 2024-03-06T18:09:59.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:09:59.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:09:59.424634+0000) 2024-03-06T18:10:00.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:00.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:00.424829+0000) 2024-03-06T18:10:01.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:01.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:01.425044+0000) 2024-03-06T18:10:02.422 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:02.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:02.425278+0000) 2024-03-06T18:10:02.758 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:10:02.766 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:10:03.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:03.423+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:03.425504+0000) 2024-03-06T18:10:04.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:04.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:04.425705+0000) 2024-03-06T18:10:05.423 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:05.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:05.425904+0000) 2024-03-06T18:10:06.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:06.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:06.426186+0000) 2024-03-06T18:10:07.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:07.424+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:07.426391+0000) 2024-03-06T18:10:08.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:08.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:08.426674+0000) 2024-03-06T18:10:09.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:09.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:09.426871+0000) 2024-03-06T18:10:10.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:10.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:10.427047+0000) 2024-03-06T18:10:11.424 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:11.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:11.427238+0000) 2024-03-06T18:10:12.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:12.425+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:12.427427+0000) 2024-03-06T18:10:13.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:13.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:13.427657+0000) 2024-03-06T18:10:14.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:14.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:14.427826+0000) 2024-03-06T18:10:15.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:15.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:15.428049+0000) 2024-03-06T18:10:16.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:16.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:16.428231+0000) 2024-03-06T18:10:17.425 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:17.426+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:17.428438+0000) 2024-03-06T18:10:18.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:18.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:18.428674+0000) 2024-03-06T18:10:19.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:19.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:19.428856+0000) 2024-03-06T18:10:20.426 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:20.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:20.429093+0000) 2024-03-06T18:10:21.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:21.427+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:21.429341+0000) 2024-03-06T18:10:22.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:22.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:22.429643+0000) 2024-03-06T18:10:23.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:23.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:23.429891+0000) 2024-03-06T18:10:24.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:24.428+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:24.430133+0000) 2024-03-06T18:10:25.427 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:25.429+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:25.430374+0000) 2024-03-06T18:10:26.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:26.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:26.430644+0000) 2024-03-06T18:10:27.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:27.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:27.430895+0000) 2024-03-06T18:10:28.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:28.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:28.431118+0000) 2024-03-06T18:10:29.428 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:29.430+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:29.431341+0000) 2024-03-06T18:10:30.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:30.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:30.431571+0000) 2024-03-06T18:10:31.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:31.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:31.431740+0000) 2024-03-06T18:10:32.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:32.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:32.431942+0000) 2024-03-06T18:10:32.970 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:10:32.979 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:10:33.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:33.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:33.432190+0000) 2024-03-06T18:10:34.429 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:34.431+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:34.432383+0000) 2024-03-06T18:10:35.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:35.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:35.432621+0000) 2024-03-06T18:10:36.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:36.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:36.432821+0000) 2024-03-06T18:10:37.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:37.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:37.433023+0000) 2024-03-06T18:10:38.430 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:38.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:38.433247+0000) 2024-03-06T18:10:39.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:39.432+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:39.433469+0000) 2024-03-06T18:10:40.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:40.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:40.433681+0000) 2024-03-06T18:10:41.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:41.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:41.433890+0000) 2024-03-06T18:10:42.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:42.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:42.434154+0000) 2024-03-06T18:10:43.431 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:43.433+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:43.434448+0000) 2024-03-06T18:10:44.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:44.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:44.434667+0000) 2024-03-06T18:10:45.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:45.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:45.434861+0000) 2024-03-06T18:10:46.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:46.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:46.435084+0000) 2024-03-06T18:10:47.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:47.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:47.435324+0000) 2024-03-06T18:10:48.432 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:48.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:48.435584+0000) 2024-03-06T18:10:49.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:49.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:49.435877+0000) 2024-03-06T18:10:50.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:50.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:50.436102+0000) 2024-03-06T18:10:51.433 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:51.434+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:51.436335+0000) 2024-03-06T18:10:52.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:52.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:52.436631+0000) 2024-03-06T18:10:53.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:53.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:53.436843+0000) 2024-03-06T18:10:54.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:54.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:54.437080+0000) 2024-03-06T18:10:55.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:55.435+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:55.437309+0000) 2024-03-06T18:10:56.434 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:56.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:56.437571+0000) 2024-03-06T18:10:57.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:57.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:57.437786+0000) 2024-03-06T18:10:58.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:58.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:58.437984+0000) 2024-03-06T18:10:59.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:10:59.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:10:59.438165+0000) 2024-03-06T18:11:00.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:00.436+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:00.438406+0000) 2024-03-06T18:11:01.435 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:01.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:01.438655+0000) 2024-03-06T18:11:02.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:02.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:02.438862+0000) 2024-03-06T18:11:03.108 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:11:03.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:11:03.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:03.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:03.439095+0000) 2024-03-06T18:11:04.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:04.437+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:04.439301+0000) 2024-03-06T18:11:05.436 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:05.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:05.439579+0000) 2024-03-06T18:11:06.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:06.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:06.439757+0000) 2024-03-06T18:11:07.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:07.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:07.440008+0000) 2024-03-06T18:11:08.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:08.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:08.440225+0000) 2024-03-06T18:11:09.437 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:09.438+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:09.440457+0000) 2024-03-06T18:11:10.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:10.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:10.440705+0000) 2024-03-06T18:11:11.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:11.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:11.440935+0000) 2024-03-06T18:11:12.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:12.439+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:12.441139+0000) 2024-03-06T18:11:13.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:13.440+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:13.441366+0000) 2024-03-06T18:11:14.438 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:14.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:14.441636+0000) 2024-03-06T18:11:15.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:15.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:15.441836+0000) 2024-03-06T18:11:16.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:16.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:16.442051+0000) 2024-03-06T18:11:17.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:17.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:17.442271+0000) 2024-03-06T18:11:18.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:18.441+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:18.442470+0000) 2024-03-06T18:11:19.439 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:19.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:19.442702+0000) 2024-03-06T18:11:20.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:20.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:20.442927+0000) 2024-03-06T18:11:21.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:21.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:21.443179+0000) 2024-03-06T18:11:22.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:22.442+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:22.443435+0000) 2024-03-06T18:11:23.440 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:23.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:23.443686+0000) 2024-03-06T18:11:24.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:24.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:24.443919+0000) 2024-03-06T18:11:25.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:25.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:25.444161+0000) 2024-03-06T18:11:26.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:26.443+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:26.444401+0000) 2024-03-06T18:11:27.441 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:27.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:27.444661+0000) 2024-03-06T18:11:28.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:28.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:28.444898+0000) 2024-03-06T18:11:29.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:29.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:29.445146+0000) 2024-03-06T18:11:30.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:30.444+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:30.445388+0000) 2024-03-06T18:11:31.442 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:31.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:31.445639+0000) 2024-03-06T18:11:32.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:32.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:32.445877+0000) 2024-03-06T18:11:33.315 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:11:33.323 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:11:33.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:33.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:33.446142+0000) 2024-03-06T18:11:34.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:34.445+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:34.446379+0000) 2024-03-06T18:11:35.443 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:35.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:35.446676+0000) 2024-03-06T18:11:36.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:36.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:36.446880+0000) 2024-03-06T18:11:37.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:37.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:37.447105+0000) 2024-03-06T18:11:38.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:38.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:38.447363+0000) 2024-03-06T18:11:39.444 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:39.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:39.447639+0000) 2024-03-06T18:11:40.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:40.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:40.447886+0000) 2024-03-06T18:11:41.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:41.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:41.448142+0000) 2024-03-06T18:11:42.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:42.446+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:42.448353+0000) 2024-03-06T18:11:43.445 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:43.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:43.448649+0000) 2024-03-06T18:11:44.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:44.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:44.448844+0000) 2024-03-06T18:11:45.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:45.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:45.449036+0000) 2024-03-06T18:11:46.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:46.447+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:46.449266+0000) 2024-03-06T18:11:47.446 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:47.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:47.449587+0000) 2024-03-06T18:11:48.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:48.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:48.449787+0000) 2024-03-06T18:11:49.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:49.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:49.450013+0000) 2024-03-06T18:11:50.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:50.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:50.450236+0000) 2024-03-06T18:11:51.447 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:51.448+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:51.450468+0000) 2024-03-06T18:11:52.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:52.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:52.450655+0000) 2024-03-06T18:11:53.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:53.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:53.450864+0000) 2024-03-06T18:11:54.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:54.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:54.451092+0000) 2024-03-06T18:11:55.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:55.449+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:55.451321+0000) 2024-03-06T18:11:56.448 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:56.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:56.451573+0000) 2024-03-06T18:11:57.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:57.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:57.451815+0000) 2024-03-06T18:11:58.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:58.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:58.452037+0000) 2024-03-06T18:11:59.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:11:59.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:11:59.452284+0000) 2024-03-06T18:12:00.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:00.450+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:00.452513+0000) 2024-03-06T18:12:01.449 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:01.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:01.452772+0000) 2024-03-06T18:12:02.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:02.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:02.453010+0000) 2024-03-06T18:12:03.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:03.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:03.453213+0000) 2024-03-06T18:12:03.530 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:12:03.538 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:12:04.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:04.452+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:04.453407+0000) 2024-03-06T18:12:05.450 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:05.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:05.453648+0000) 2024-03-06T18:12:06.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:06.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:06.453893+0000) 2024-03-06T18:12:07.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:07.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:07.454155+0000) 2024-03-06T18:12:08.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:08.453+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:08.454400+0000) 2024-03-06T18:12:09.451 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:09.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:09.454661+0000) 2024-03-06T18:12:10.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:10.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:10.454903+0000) 2024-03-06T18:12:11.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:11.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:11.455154+0000) 2024-03-06T18:12:12.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:12.454+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:12.455364+0000) 2024-03-06T18:12:13.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:13.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:13.455643+0000) 2024-03-06T18:12:14.452 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:14.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:14.455799+0000) 2024-03-06T18:12:15.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:15.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:15.456026+0000) 2024-03-06T18:12:16.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:16.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:16.456246+0000) 2024-03-06T18:12:17.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:17.455+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:17.456479+0000) 2024-03-06T18:12:18.453 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:18.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:18.456684+0000) 2024-03-06T18:12:19.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:19.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:19.456896+0000) 2024-03-06T18:12:20.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:20.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:20.457131+0000) 2024-03-06T18:12:21.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:21.456+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:21.457325+0000) 2024-03-06T18:12:22.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:22.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:22.457618+0000) 2024-03-06T18:12:23.454 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:23.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:23.457865+0000) 2024-03-06T18:12:24.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:24.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:24.458106+0000) 2024-03-06T18:12:25.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:25.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:25.458357+0000) 2024-03-06T18:12:26.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:26.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:26.458598+0000) 2024-03-06T18:12:27.455 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:27.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:27.458869+0000) 2024-03-06T18:12:28.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:28.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:28.459114+0000) 2024-03-06T18:12:29.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:29.457+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:29.459351+0000) 2024-03-06T18:12:30.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:30.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:30.459642+0000) 2024-03-06T18:12:31.456 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:31.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:31.459832+0000) 2024-03-06T18:12:32.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:32.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:32.460056+0000) 2024-03-06T18:12:33.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:33.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:33.460285+0000) 2024-03-06T18:12:33.743 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:12:33.751 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:12:34.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:34.458+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:34.460512+0000) 2024-03-06T18:12:35.457 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:35.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:35.460773+0000) 2024-03-06T18:12:36.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:36.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:36.460964+0000) 2024-03-06T18:12:37.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:37.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:37.461203+0000) 2024-03-06T18:12:38.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:38.459+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:38.461384+0000) 2024-03-06T18:12:39.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:39.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:39.461605+0000) 2024-03-06T18:12:40.458 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:40.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:40.461804+0000) 2024-03-06T18:12:41.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:41.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:41.462091+0000) 2024-03-06T18:12:42.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:42.460+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:42.462326+0000) 2024-03-06T18:12:43.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:43.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:43.462588+0000) 2024-03-06T18:12:44.459 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:44.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:44.462789+0000) 2024-03-06T18:12:45.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:45.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:45.462973+0000) 2024-03-06T18:12:46.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:46.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:46.463195+0000) 2024-03-06T18:12:47.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:47.461+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:47.463426+0000) 2024-03-06T18:12:48.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:48.462+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:48.463642+0000) 2024-03-06T18:12:49.460 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:49.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:49.463863+0000) 2024-03-06T18:12:50.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:50.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:50.464087+0000) 2024-03-06T18:12:51.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:51.463+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:51.464301+0000) 2024-03-06T18:12:52.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:52.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:52.464568+0000) 2024-03-06T18:12:53.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:53.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:53.464723+0000) 2024-03-06T18:12:54.461 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:54.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:54.464918+0000) 2024-03-06T18:12:55.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:55.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:55.465098+0000) 2024-03-06T18:12:56.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:56.464+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:56.465327+0000) 2024-03-06T18:12:57.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:57.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:57.465592+0000) 2024-03-06T18:12:58.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:58.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:58.465852+0000) 2024-03-06T18:12:59.462 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:12:59.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:12:59.466058+0000) 2024-03-06T18:13:00.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:00.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:00.466302+0000) 2024-03-06T18:13:01.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:01.465+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:01.466512+0000) 2024-03-06T18:13:02.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:02.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:02.466803+0000) 2024-03-06T18:13:03.463 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:03.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:03.467038+0000) 2024-03-06T18:13:03.974 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:13:03.982 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:13:04.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:04.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:04.467274+0000) 2024-03-06T18:13:05.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:05.466+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:05.467503+0000) 2024-03-06T18:13:06.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:06.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:06.467686+0000) 2024-03-06T18:13:07.464 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:07.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:07.467888+0000) 2024-03-06T18:13:08.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:08.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:08.468099+0000) 2024-03-06T18:13:09.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:09.467+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:09.468356+0000) 2024-03-06T18:13:10.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:10.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:10.468623+0000) 2024-03-06T18:13:11.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:11.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:11.468863+0000) 2024-03-06T18:13:12.465 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:12.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:12.469056+0000) 2024-03-06T18:13:13.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:13.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:13.469294+0000) 2024-03-06T18:13:14.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:14.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:14.469526+0000) 2024-03-06T18:13:15.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:15.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:15.469791+0000) 2024-03-06T18:13:16.466 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:16.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:16.470016+0000) 2024-03-06T18:13:17.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:17.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:17.470247+0000) 2024-03-06T18:13:18.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:18.468+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:18.470480+0000) 2024-03-06T18:13:19.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:19.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:19.470697+0000) 2024-03-06T18:13:20.467 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:20.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:20.470898+0000) 2024-03-06T18:13:21.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:21.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:21.471198+0000) 2024-03-06T18:13:22.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:22.469+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:22.471445+0000) 2024-03-06T18:13:23.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:23.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:23.471669+0000) 2024-03-06T18:13:24.468 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:24.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:24.471881+0000) 2024-03-06T18:13:25.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:25.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:25.472102+0000) 2024-03-06T18:13:26.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:26.470+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:26.472279+0000) 2024-03-06T18:13:27.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:27.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:27.472529+0000) 2024-03-06T18:13:28.469 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:28.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:28.472805+0000) 2024-03-06T18:13:29.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:29.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:29.473104+0000) 2024-03-06T18:13:30.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:30.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:30.473271+0000) 2024-03-06T18:13:31.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:31.471+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:31.473518+0000) 2024-03-06T18:13:32.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:32.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:32.473772+0000) 2024-03-06T18:13:33.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:33.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:33.474009+0000) 2024-03-06T18:13:34.163 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:13:34.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:13:34.470 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:34.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:34.474251+0000) 2024-03-06T18:13:35.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:35.472+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:35.474438+0000) 2024-03-06T18:13:36.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:36.473+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:36.474682+0000) 2024-03-06T18:13:37.471 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:37.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:37.474831+0000) 2024-03-06T18:13:38.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:38.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:38.475064+0000) 2024-03-06T18:13:39.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:39.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:39.475297+0000) 2024-03-06T18:13:40.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:40.474+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:40.475492+0000) 2024-03-06T18:13:41.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:41.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:41.475784+0000) 2024-03-06T18:13:42.472 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:42.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:42.476015+0000) 2024-03-06T18:13:43.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:43.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:43.476250+0000) 2024-03-06T18:13:44.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:44.475+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:44.476474+0000) 2024-03-06T18:13:45.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:45.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:45.476692+0000) 2024-03-06T18:13:46.473 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:46.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:46.476958+0000) 2024-03-06T18:13:47.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:47.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:47.477236+0000) 2024-03-06T18:13:48.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:48.476+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:48.477477+0000) 2024-03-06T18:13:49.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:49.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:49.477697+0000) 2024-03-06T18:13:50.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:50.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:50.477930+0000) 2024-03-06T18:13:51.474 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:51.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:51.478229+0000) 2024-03-06T18:13:52.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:52.477+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:52.478493+0000) 2024-03-06T18:13:53.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:53.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:53.478713+0000) 2024-03-06T18:13:54.475 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:54.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:54.478918+0000) 2024-03-06T18:13:55.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:55.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:55.479149+0000) 2024-03-06T18:13:56.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:56.478+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:56.479389+0000) 2024-03-06T18:13:57.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:57.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:57.479655+0000) 2024-03-06T18:13:58.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:58.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:58.479817+0000) 2024-03-06T18:13:59.476 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:13:59.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:13:59.480007+0000) 2024-03-06T18:14:00.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:00.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:00.480235+0000) 2024-03-06T18:14:01.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:01.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:01.480406+0000) 2024-03-06T18:14:02.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:02.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:02.480654+0000) 2024-03-06T18:14:03.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:03.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:03.480883+0000) 2024-03-06T18:14:04.310 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:14:04.319 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:14:04.477 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:04.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:04.481099+0000) 2024-03-06T18:14:05.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:05.479+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:05.481316+0000) 2024-03-06T18:14:06.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:06.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:06.481566+0000) 2024-03-06T18:14:07.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:07.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:07.481752+0000) 2024-03-06T18:14:08.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:08.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:08.481989+0000) 2024-03-06T18:14:09.478 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:09.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:09.482213+0000) 2024-03-06T18:14:10.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:10.480+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:10.482437+0000) 2024-03-06T18:14:11.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:11.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:11.482672+0000) 2024-03-06T18:14:12.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:12.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:12.482892+0000) 2024-03-06T18:14:13.479 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:13.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:13.483078+0000) 2024-03-06T18:14:14.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:14.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:14.483276+0000) 2024-03-06T18:14:15.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:15.481+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:15.483515+0000) 2024-03-06T18:14:16.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:16.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:16.483733+0000) 2024-03-06T18:14:17.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:17.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:17.483971+0000) 2024-03-06T18:14:18.480 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:18.482+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:18.484212+0000) 2024-03-06T18:14:19.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:19.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:19.484589+0000) 2024-03-06T18:14:20.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:20.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:20.484802+0000) 2024-03-06T18:14:21.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:21.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:21.485009+0000) 2024-03-06T18:14:22.481 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:22.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:22.485232+0000) 2024-03-06T18:14:23.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:23.483+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:23.485461+0000) 2024-03-06T18:14:24.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:24.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:24.485686+0000) 2024-03-06T18:14:25.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:25.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:25.485877+0000) 2024-03-06T18:14:26.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:26.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:26.486105+0000) 2024-03-06T18:14:27.482 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:27.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:27.486285+0000) 2024-03-06T18:14:28.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:28.485+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:28.486463+0000) 2024-03-06T18:14:29.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:29.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:29.486680+0000) 2024-03-06T18:14:30.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:30.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:30.486913+0000) 2024-03-06T18:14:31.483 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:31.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:31.487141+0000) 2024-03-06T18:14:32.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:32.486+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:32.487369+0000) 2024-03-06T18:14:33.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:33.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:33.487596+0000) 2024-03-06T18:14:34.449 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:14:34.458 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:14:34.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:34.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:34.487753+0000) 2024-03-06T18:14:35.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:35.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:35.487984+0000) 2024-03-06T18:14:36.484 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:36.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:36.488210+0000) 2024-03-06T18:14:37.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:37.487+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:37.488428+0000) 2024-03-06T18:14:38.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:38.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:38.488669+0000) 2024-03-06T18:14:39.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:39.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:39.488870+0000) 2024-03-06T18:14:40.485 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:40.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:40.489062+0000) 2024-03-06T18:14:41.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:41.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:41.489282+0000) 2024-03-06T18:14:42.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:42.488+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:42.489515+0000) 2024-03-06T18:14:43.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:43.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:43.489785+0000) 2024-03-06T18:14:44.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:44.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:44.490034+0000) 2024-03-06T18:14:45.486 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:45.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:45.490252+0000) 2024-03-06T18:14:46.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:46.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:46.490491+0000) 2024-03-06T18:14:47.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:47.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:47.490755+0000) 2024-03-06T18:14:48.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:48.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:48.490956+0000) 2024-03-06T18:14:49.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:49.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:49.491164+0000) 2024-03-06T18:14:50.487 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:50.489+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:50.491403+0000) 2024-03-06T18:14:51.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:51.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:51.491668+0000) 2024-03-06T18:14:52.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:52.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:52.491871+0000) 2024-03-06T18:14:53.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:53.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:53.492089+0000) 2024-03-06T18:14:54.488 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:54.490+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:54.492337+0000) 2024-03-06T18:14:55.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:55.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:55.492563+0000) 2024-03-06T18:14:56.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:56.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:56.492806+0000) 2024-03-06T18:14:57.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:57.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:57.493088+0000) 2024-03-06T18:14:58.489 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:58.491+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:58.493373+0000) 2024-03-06T18:14:59.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:14:59.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:14:59.493661+0000) 2024-03-06T18:15:00.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:00.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:00.493898+0000) 2024-03-06T18:15:01.490 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:01.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:01.494170+0000) 2024-03-06T18:15:02.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:02.492+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:02.494411+0000) 2024-03-06T18:15:03.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:03.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:03.494650+0000) 2024-03-06T18:15:04.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:04.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:04.494835+0000) 2024-03-06T18:15:04.689 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:15:04.696 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:15:05.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:05.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:05.495031+0000) 2024-03-06T18:15:06.491 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:06.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:06.495231+0000) 2024-03-06T18:15:07.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:07.493+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:07.495469+0000) 2024-03-06T18:15:08.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:08.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:08.495686+0000) 2024-03-06T18:15:09.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:09.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:09.495949+0000) 2024-03-06T18:15:10.492 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:10.494+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:10.496221+0000) 2024-03-06T18:15:11.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:11.495+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:11.496514+0000) 2024-03-06T18:15:12.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:12.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:12.496716+0000) 2024-03-06T18:15:13.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:13.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:13.496969+0000) 2024-03-06T18:15:14.493 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:14.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:14.497158+0000) 2024-03-06T18:15:15.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:15.496+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:15.497378+0000) 2024-03-06T18:15:16.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:16.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:16.497583+0000) 2024-03-06T18:15:17.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:17.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:17.497783+0000) 2024-03-06T18:15:18.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:18.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:18.497964+0000) 2024-03-06T18:15:19.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:19.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:19.498148+0000) 2024-03-06T18:15:20.494 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:20.497+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:20.498408+0000) 2024-03-06T18:15:21.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:21.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:21.498690+0000) 2024-03-06T18:15:22.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:22.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:22.498924+0000) 2024-03-06T18:15:23.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:23.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:23.499154+0000) 2024-03-06T18:15:24.495 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:24.498+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:24.499411+0000) 2024-03-06T18:15:25.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:25.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:25.499664+0000) 2024-03-06T18:15:26.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:26.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:26.499900+0000) 2024-03-06T18:15:27.496 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:27.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:27.500084+0000) 2024-03-06T18:15:28.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:28.499+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:28.500313+0000) 2024-03-06T18:15:29.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:29.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:29.500568+0000) 2024-03-06T18:15:30.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:30.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:30.500769+0000) 2024-03-06T18:15:31.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:31.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:31.501012+0000) 2024-03-06T18:15:32.497 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:32.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:32.501166+0000) 2024-03-06T18:15:33.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:33.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:33.501401+0000) 2024-03-06T18:15:34.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:34.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:34.501628+0000) 2024-03-06T18:15:34.896 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:15:34.904 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:15:35.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:35.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:35.501847+0000) 2024-03-06T18:15:36.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:36.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:36.502049+0000) 2024-03-06T18:15:37.498 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:37.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:37.502296+0000) 2024-03-06T18:15:38.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:38.500+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:38.502488+0000) 2024-03-06T18:15:39.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:39.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:39.502723+0000) 2024-03-06T18:15:40.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:40.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:40.502947+0000) 2024-03-06T18:15:41.499 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:41.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:41.503188+0000) 2024-03-06T18:15:42.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:42.501+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:42.503420+0000) 2024-03-06T18:15:43.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:43.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:43.503661+0000) 2024-03-06T18:15:44.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:44.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:44.503885+0000) 2024-03-06T18:15:45.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:45.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:45.504117+0000) 2024-03-06T18:15:46.500 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:46.502+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:46.504315+0000) 2024-03-06T18:15:47.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:47.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:47.504576+0000) 2024-03-06T18:15:48.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:48.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:48.504810+0000) 2024-03-06T18:15:49.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:49.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:49.504999+0000) 2024-03-06T18:15:50.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:50.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:50.505225+0000) 2024-03-06T18:15:51.501 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:51.503+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:51.505455+0000) 2024-03-06T18:15:52.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:52.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:52.505703+0000) 2024-03-06T18:15:53.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:53.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:53.505944+0000) 2024-03-06T18:15:54.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:54.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:54.506125+0000) 2024-03-06T18:15:55.502 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:55.504+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:55.506362+0000) 2024-03-06T18:15:56.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:56.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:56.506625+0000) 2024-03-06T18:15:57.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:57.505+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:57.506824+0000) 2024-03-06T18:15:58.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:58.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:58.507085+0000) 2024-03-06T18:15:59.503 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:15:59.506+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:15:59.507304+0000) 2024-03-06T18:16:00.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:00.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:00.507525+0000) 2024-03-06T18:16:01.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:01.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:01.507747+0000) 2024-03-06T18:16:02.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:02.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:02.507935+0000) 2024-03-06T18:16:03.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:03.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:03.508136+0000) 2024-03-06T18:16:04.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:04.507+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:04.508350+0000) 2024-03-06T18:16:05.108 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:16:05.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:16:05.504 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:05.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:05.508626+0000) 2024-03-06T18:16:06.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:06.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:06.508811+0000) 2024-03-06T18:16:07.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:07.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:07.509008+0000) 2024-03-06T18:16:08.505 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:08.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:08.509204+0000) 2024-03-06T18:16:09.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:09.508+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:09.509412+0000) 2024-03-06T18:16:10.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:10.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:10.509657+0000) 2024-03-06T18:16:11.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:11.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:11.509849+0000) 2024-03-06T18:16:12.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:12.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:12.510053+0000) 2024-03-06T18:16:13.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:13.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:13.510314+0000) 2024-03-06T18:16:14.506 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:14.509+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:14.510508+0000) 2024-03-06T18:16:15.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:15.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:15.510745+0000) 2024-03-06T18:16:16.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:16.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:16.510981+0000) 2024-03-06T18:16:17.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:17.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:17.511204+0000) 2024-03-06T18:16:18.507 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:18.510+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:18.511436+0000) 2024-03-06T18:16:19.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:19.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:19.511667+0000) 2024-03-06T18:16:20.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:20.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:20.511889+0000) 2024-03-06T18:16:21.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:21.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:21.512086+0000) 2024-03-06T18:16:22.508 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:22.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:22.512316+0000) 2024-03-06T18:16:23.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:23.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:23.512587+0000) 2024-03-06T18:16:24.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:24.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:24.512816+0000) 2024-03-06T18:16:25.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:25.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:25.513019+0000) 2024-03-06T18:16:26.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:26.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:26.513244+0000) 2024-03-06T18:16:27.509 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:27.511+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:27.513474+0000) 2024-03-06T18:16:28.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:28.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:28.513715+0000) 2024-03-06T18:16:29.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:29.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:29.513865+0000) 2024-03-06T18:16:30.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:30.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:30.514069+0000) 2024-03-06T18:16:31.510 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:31.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:31.514295+0000) 2024-03-06T18:16:32.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:32.512+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:32.514511+0000) 2024-03-06T18:16:33.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:33.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:33.514782+0000) 2024-03-06T18:16:34.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:34.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:34.514964+0000) 2024-03-06T18:16:35.258 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:16:35.267 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:16:35.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:35.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:35.515161+0000) 2024-03-06T18:16:36.511 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:36.513+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:36.515382+0000) 2024-03-06T18:16:37.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:37.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:37.515627+0000) 2024-03-06T18:16:38.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:38.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:38.515827+0000) 2024-03-06T18:16:39.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:39.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:39.516020+0000) 2024-03-06T18:16:40.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:40.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:40.516226+0000) 2024-03-06T18:16:41.512 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:41.514+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:41.516483+0000) 2024-03-06T18:16:42.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:42.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:42.516727+0000) 2024-03-06T18:16:43.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:43.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:43.516982+0000) 2024-03-06T18:16:44.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:44.515+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:44.517182+0000) 2024-03-06T18:16:45.513 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:45.516+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:45.517375+0000) 2024-03-06T18:16:46.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:46.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:46.517670+0000) 2024-03-06T18:16:47.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:47.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:47.517871+0000) 2024-03-06T18:16:48.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:48.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:48.518096+0000) 2024-03-06T18:16:49.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:49.517+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:49.518335+0000) 2024-03-06T18:16:50.514 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:50.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:50.518599+0000) 2024-03-06T18:16:51.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:51.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:51.518831+0000) 2024-03-06T18:16:52.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:52.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:52.519007+0000) 2024-03-06T18:16:53.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:53.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:53.519270+0000) 2024-03-06T18:16:54.515 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:54.518+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:54.519510+0000) 2024-03-06T18:16:55.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:55.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:55.519803+0000) 2024-03-06T18:16:56.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:56.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:56.520061+0000) 2024-03-06T18:16:57.516 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:57.519+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:57.520285+0000) 2024-03-06T18:16:58.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:58.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:58.520521+0000) 2024-03-06T18:16:59.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:16:59.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:16:59.520780+0000) 2024-03-06T18:17:00.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:00.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:00.520966+0000) 2024-03-06T18:17:01.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:01.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:01.521197+0000) 2024-03-06T18:17:02.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:02.520+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:02.521418+0000) 2024-03-06T18:17:03.517 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:03.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:03.521656+0000) 2024-03-06T18:17:04.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:04.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:04.521860+0000) 2024-03-06T18:17:05.428 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:17:05.436 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:17:05.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:05.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:05.522021+0000) 2024-03-06T18:17:06.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:06.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:06.522237+0000) 2024-03-06T18:17:07.518 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:07.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:07.522462+0000) 2024-03-06T18:17:08.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:08.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:08.522696+0000) 2024-03-06T18:17:09.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:09.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:09.522913+0000) 2024-03-06T18:17:10.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:10.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:10.523122+0000) 2024-03-06T18:17:11.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:11.521+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:11.523356+0000) 2024-03-06T18:17:12.519 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:12.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:12.523604+0000) 2024-03-06T18:17:13.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:13.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:13.523765+0000) 2024-03-06T18:17:14.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:14.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:14.523997+0000) 2024-03-06T18:17:15.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:15.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:15.524243+0000) 2024-03-06T18:17:16.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:16.522+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:16.524448+0000) 2024-03-06T18:17:17.520 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:17.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:17.524657+0000) 2024-03-06T18:17:18.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:18.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:18.524844+0000) 2024-03-06T18:17:19.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:19.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:19.525074+0000) 2024-03-06T18:17:20.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:20.523+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:20.525298+0000) 2024-03-06T18:17:21.521 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:21.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:21.525564+0000) 2024-03-06T18:17:22.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:22.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:22.525699+0000) 2024-03-06T18:17:23.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:23.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:23.525895+0000) 2024-03-06T18:17:24.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:24.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:24.526088+0000) 2024-03-06T18:17:25.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:25.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:25.526317+0000) 2024-03-06T18:17:26.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:26.524+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:26.526499+0000) 2024-03-06T18:17:27.522 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:27.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:27.526732+0000) 2024-03-06T18:17:28.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:28.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:28.526922+0000) 2024-03-06T18:17:29.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:29.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:29.527157+0000) 2024-03-06T18:17:30.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:30.525+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:30.527379+0000) 2024-03-06T18:17:31.523 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:31.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:31.527653+0000) 2024-03-06T18:17:32.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:32.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:32.527873+0000) 2024-03-06T18:17:33.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:33.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:33.528113+0000) 2024-03-06T18:17:34.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:34.527+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:34.528347+0000) 2024-03-06T18:17:35.524 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:35.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:35.528601+0000) 2024-03-06T18:17:35.609 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:17:35.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:17:36.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:36.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:36.528853+0000) 2024-03-06T18:17:37.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:37.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:37.529080+0000) 2024-03-06T18:17:38.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:38.528+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:38.529309+0000) 2024-03-06T18:17:39.525 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:39.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:39.529622+0000) 2024-03-06T18:17:40.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:40.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:40.529841+0000) 2024-03-06T18:17:41.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:41.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:41.530076+0000) 2024-03-06T18:17:42.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:42.529+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:42.530301+0000) 2024-03-06T18:17:43.526 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:43.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:43.530576+0000) 2024-03-06T18:17:44.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:44.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:44.530807+0000) 2024-03-06T18:17:45.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:45.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:45.530984+0000) 2024-03-06T18:17:46.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:46.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:46.531204+0000) 2024-03-06T18:17:47.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:47.530+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:47.531396+0000) 2024-03-06T18:17:48.527 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:48.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:48.531652+0000) 2024-03-06T18:17:49.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:49.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:49.531842+0000) 2024-03-06T18:17:50.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:50.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:50.532036+0000) 2024-03-06T18:17:51.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:51.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:51.532267+0000) 2024-03-06T18:17:52.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:52.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:52.532442+0000) 2024-03-06T18:17:53.528 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:53.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:53.532694+0000) 2024-03-06T18:17:54.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:54.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:54.532884+0000) 2024-03-06T18:17:55.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:55.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:55.533077+0000) 2024-03-06T18:17:56.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:56.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:56.533298+0000) 2024-03-06T18:17:57.529 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:57.531+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:57.533499+0000) 2024-03-06T18:17:58.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:58.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:58.533755+0000) 2024-03-06T18:17:59.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:17:59.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:17:59.533999+0000) 2024-03-06T18:18:00.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:00.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:00.534224+0000) 2024-03-06T18:18:01.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:01.532+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:01.534462+0000) 2024-03-06T18:18:02.530 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:02.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:02.534682+0000) 2024-03-06T18:18:03.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:03.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:03.534893+0000) 2024-03-06T18:18:04.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:04.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:04.535105+0000) 2024-03-06T18:18:05.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:05.533+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:05.535345+0000) 2024-03-06T18:18:05.814 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:18:05.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:18:06.531 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:06.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:06.535619+0000) 2024-03-06T18:18:07.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:07.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:07.535839+0000) 2024-03-06T18:18:08.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:08.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:08.536051+0000) 2024-03-06T18:18:09.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:09.534+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:09.536304+0000) 2024-03-06T18:18:10.532 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:10.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:10.536585+0000) 2024-03-06T18:18:11.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:11.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:11.536784+0000) 2024-03-06T18:18:12.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:12.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:12.536968+0000) 2024-03-06T18:18:13.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:13.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:13.537197+0000) 2024-03-06T18:18:14.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:14.535+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:14.537392+0000) 2024-03-06T18:18:15.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:15.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:15.537642+0000) 2024-03-06T18:18:16.533 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:16.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:16.537850+0000) 2024-03-06T18:18:17.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:17.536+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:17.538103+0000) 2024-03-06T18:18:18.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:18.537+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:18.538340+0000) 2024-03-06T18:18:19.534 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:19.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:19.538617+0000) 2024-03-06T18:18:20.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:20.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:20.538776+0000) 2024-03-06T18:18:21.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:21.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:21.538998+0000) 2024-03-06T18:18:22.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:22.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:22.539245+0000) 2024-03-06T18:18:23.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:23.538+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:23.539494+0000) 2024-03-06T18:18:24.535 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:24.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:24.539746+0000) 2024-03-06T18:18:25.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:25.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:25.539991+0000) 2024-03-06T18:18:26.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:26.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:26.540228+0000) 2024-03-06T18:18:27.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:27.539+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:27.540495+0000) 2024-03-06T18:18:28.536 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:28.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:28.540777+0000) 2024-03-06T18:18:29.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:29.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:29.540995+0000) 2024-03-06T18:18:30.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:30.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:30.541222+0000) 2024-03-06T18:18:31.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:31.540+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:31.541425+0000) 2024-03-06T18:18:32.537 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:32.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:32.541665+0000) 2024-03-06T18:18:33.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:33.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:33.541908+0000) 2024-03-06T18:18:34.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:34.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:34.542144+0000) 2024-03-06T18:18:35.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:35.541+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:35.542381+0000) 2024-03-06T18:18:36.012 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:18:36.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:18:36.538 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:36.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:36.542651+0000) 2024-03-06T18:18:37.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:37.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:37.542900+0000) 2024-03-06T18:18:38.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:38.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:38.543112+0000) 2024-03-06T18:18:39.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:39.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:39.543282+0000) 2024-03-06T18:18:40.539 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:40.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:40.543483+0000) 2024-03-06T18:18:41.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:41.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:41.543760+0000) 2024-03-06T18:18:42.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:42.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:42.543916+0000) 2024-03-06T18:18:43.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:43.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:43.544111+0000) 2024-03-06T18:18:44.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:44.542+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:44.544334+0000) 2024-03-06T18:18:45.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:45.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:45.544528+0000) 2024-03-06T18:18:46.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:46.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:46.544758+0000) 2024-03-06T18:18:47.540 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:47.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:47.544929+0000) 2024-03-06T18:18:48.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:48.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:48.545121+0000) 2024-03-06T18:18:49.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:49.543+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:49.545318+0000) 2024-03-06T18:18:50.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:50.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:50.545523+0000) 2024-03-06T18:18:51.541 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:51.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:51.545777+0000) 2024-03-06T18:18:52.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:52.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:52.546008+0000) 2024-03-06T18:18:53.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:53.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:53.546213+0000) 2024-03-06T18:18:54.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:54.544+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:54.546453+0000) 2024-03-06T18:18:55.542 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:55.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:55.546677+0000) 2024-03-06T18:18:56.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:56.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:56.546898+0000) 2024-03-06T18:18:57.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:57.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:57.547099+0000) 2024-03-06T18:18:58.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:58.545+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:58.547356+0000) 2024-03-06T18:18:59.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:18:59.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:18:59.547621+0000) 2024-03-06T18:19:00.543 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:00.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:00.547850+0000) 2024-03-06T18:19:01.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:01.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:01.548084+0000) 2024-03-06T18:19:02.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:02.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:02.548274+0000) 2024-03-06T18:19:03.544 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:03.546+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:03.548510+0000) 2024-03-06T18:19:04.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:04.547+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:04.548765+0000) 2024-03-06T18:19:05.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:05.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:05.549022+0000) 2024-03-06T18:19:06.200 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:19:06.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:19:06.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:06.548+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:06.549259+0000) 2024-03-06T18:19:07.545 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:07.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:07.549593+0000) 2024-03-06T18:19:08.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:08.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:08.549832+0000) 2024-03-06T18:19:09.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:09.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:09.550060+0000) 2024-03-06T18:19:10.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:10.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:10.550283+0000) 2024-03-06T18:19:11.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:11.549+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:11.550513+0000) 2024-03-06T18:19:12.546 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:12.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:12.550737+0000) 2024-03-06T18:19:13.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:13.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:13.550976+0000) 2024-03-06T18:19:14.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:14.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:14.551202+0000) 2024-03-06T18:19:15.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:15.550+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:15.551443+0000) 2024-03-06T18:19:16.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:16.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:16.551671+0000) 2024-03-06T18:19:17.547 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:17.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:17.551893+0000) 2024-03-06T18:19:18.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:18.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:18.552119+0000) 2024-03-06T18:19:19.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:19.551+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:19.552343+0000) 2024-03-06T18:19:20.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:20.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:20.552602+0000) 2024-03-06T18:19:21.548 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:21.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:21.552836+0000) 2024-03-06T18:19:22.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:22.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:22.553034+0000) 2024-03-06T18:19:23.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:23.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:23.553270+0000) 2024-03-06T18:19:24.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:24.552+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:24.553495+0000) 2024-03-06T18:19:25.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:25.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:25.553723+0000) 2024-03-06T18:19:26.549 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:26.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:26.553947+0000) 2024-03-06T18:19:27.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:27.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:27.554172+0000) 2024-03-06T18:19:28.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:28.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:28.554398+0000) 2024-03-06T18:19:29.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:29.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:29.554656+0000) 2024-03-06T18:19:30.550 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:30.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:30.554884+0000) 2024-03-06T18:19:31.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:31.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:31.555116+0000) 2024-03-06T18:19:32.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:32.553+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:32.555307+0000) 2024-03-06T18:19:33.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:33.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:33.555571+0000) 2024-03-06T18:19:34.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:34.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:34.555765+0000) 2024-03-06T18:19:35.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:35.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:35.555999+0000) 2024-03-06T18:19:36.415 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:19:36.423 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:19:36.551 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:36.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:36.556229+0000) 2024-03-06T18:19:37.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:37.554+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:37.556463+0000) 2024-03-06T18:19:38.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:38.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:38.556699+0000) 2024-03-06T18:19:39.552 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:39.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:39.556921+0000) 2024-03-06T18:19:40.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:40.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:40.557101+0000) 2024-03-06T18:19:41.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:41.555+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:41.557355+0000) 2024-03-06T18:19:42.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:42.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:42.557563+0000) 2024-03-06T18:19:43.553 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:43.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:43.557770+0000) 2024-03-06T18:19:44.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:44.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:44.557980+0000) 2024-03-06T18:19:45.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:45.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:45.558181+0000) 2024-03-06T18:19:46.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:46.556+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:46.558418+0000) 2024-03-06T18:19:47.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:47.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:47.558662+0000) 2024-03-06T18:19:48.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:48.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:48.558863+0000) 2024-03-06T18:19:49.554 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:49.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:49.559047+0000) 2024-03-06T18:19:50.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:50.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:50.559277+0000) 2024-03-06T18:19:51.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:51.557+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:51.559508+0000) 2024-03-06T18:19:52.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:52.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:52.559759+0000) 2024-03-06T18:19:53.555 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:53.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:53.559963+0000) 2024-03-06T18:19:54.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:54.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:54.560184+0000) 2024-03-06T18:19:55.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:55.559+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:55.560383+0000) 2024-03-06T18:19:56.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:56.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:56.560640+0000) 2024-03-06T18:19:57.556 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:57.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:57.560866+0000) 2024-03-06T18:19:58.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:58.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:58.561102+0000) 2024-03-06T18:19:59.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:19:59.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:19:59.561287+0000) 2024-03-06T18:20:00.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:00.560+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:00.561518+0000) 2024-03-06T18:20:01.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:01.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:01.561795+0000) 2024-03-06T18:20:02.557 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:02.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:02.562034+0000) 2024-03-06T18:20:03.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:03.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:03.562263+0000) 2024-03-06T18:20:04.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:04.561+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:04.562504+0000) 2024-03-06T18:20:05.558 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:05.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:05.562802+0000) 2024-03-06T18:20:06.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:06.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:06.563027+0000) 2024-03-06T18:20:06.602 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:20:06.610 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:20:07.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:07.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:07.563222+0000) 2024-03-06T18:20:08.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:08.562+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:08.563451+0000) 2024-03-06T18:20:09.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:09.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:09.563741+0000) 2024-03-06T18:20:10.559 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:10.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:10.563934+0000) 2024-03-06T18:20:11.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:11.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:11.564129+0000) 2024-03-06T18:20:12.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:12.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:12.564316+0000) 2024-03-06T18:20:13.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:13.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:13.564493+0000) 2024-03-06T18:20:14.560 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:14.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:14.564686+0000) 2024-03-06T18:20:15.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:15.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:15.564881+0000) 2024-03-06T18:20:16.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:16.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:16.565111+0000) 2024-03-06T18:20:17.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:17.563+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:17.565304+0000) 2024-03-06T18:20:18.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:18.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:18.565592+0000) 2024-03-06T18:20:19.561 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:19.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:19.565802+0000) 2024-03-06T18:20:20.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:20.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:20.566032+0000) 2024-03-06T18:20:21.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:21.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:21.566222+0000) 2024-03-06T18:20:22.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:22.564+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:22.566450+0000) 2024-03-06T18:20:23.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:23.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:23.566684+0000) 2024-03-06T18:20:24.562 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:24.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:24.566911+0000) 2024-03-06T18:20:25.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:25.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:25.567142+0000) 2024-03-06T18:20:26.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:26.565+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:26.567368+0000) 2024-03-06T18:20:27.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:27.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:27.567634+0000) 2024-03-06T18:20:28.563 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:28.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:28.567844+0000) 2024-03-06T18:20:29.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:29.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:29.568081+0000) 2024-03-06T18:20:30.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:30.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:30.568272+0000) 2024-03-06T18:20:31.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:31.566+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:31.568510+0000) 2024-03-06T18:20:32.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:32.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:32.568741+0000) 2024-03-06T18:20:33.564 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:33.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:33.568980+0000) 2024-03-06T18:20:34.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:34.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:34.569210+0000) 2024-03-06T18:20:35.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:35.567+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:35.569401+0000) 2024-03-06T18:20:36.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:36.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:36.569687+0000) 2024-03-06T18:20:36.733 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:20:36.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:20:37.565 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:37.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:37.569871+0000) 2024-03-06T18:20:38.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:38.568+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:38.570059+0000) 2024-03-06T18:20:39.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:39.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:39.570289+0000) 2024-03-06T18:20:40.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:40.569+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:40.570513+0000) 2024-03-06T18:20:41.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:41.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:41.570795+0000) 2024-03-06T18:20:42.566 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:42.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:42.571012+0000) 2024-03-06T18:20:43.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:43.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:43.571239+0000) 2024-03-06T18:20:44.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:44.570+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:44.571497+0000) 2024-03-06T18:20:45.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:45.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:45.571814+0000) 2024-03-06T18:20:46.567 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:46.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:46.572088+0000) 2024-03-06T18:20:47.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:47.571+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:47.572363+0000) 2024-03-06T18:20:48.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:48.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:48.572632+0000) 2024-03-06T18:20:49.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:49.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:49.572803+0000) 2024-03-06T18:20:50.568 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:50.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:50.573013+0000) 2024-03-06T18:20:51.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:51.572+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:51.573292+0000) 2024-03-06T18:20:52.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:52.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:52.573630+0000) 2024-03-06T18:20:53.569 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:53.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:53.573874+0000) 2024-03-06T18:20:54.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:54.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:54.574155+0000) 2024-03-06T18:20:55.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:55.573+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:55.574433+0000) 2024-03-06T18:20:56.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:56.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:56.574726+0000) 2024-03-06T18:20:57.570 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:57.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:57.574989+0000) 2024-03-06T18:20:58.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:58.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:58.575178+0000) 2024-03-06T18:20:59.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:20:59.574+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:20:59.575373+0000) 2024-03-06T18:21:00.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:00.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:00.575657+0000) 2024-03-06T18:21:01.571 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:01.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:01.575887+0000) 2024-03-06T18:21:02.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:02.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:02.576156+0000) 2024-03-06T18:21:03.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:03.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:03.576435+0000) 2024-03-06T18:21:04.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:04.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:04.576711+0000) 2024-03-06T18:21:05.572 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:05.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:05.576960+0000) 2024-03-06T18:21:06.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:06.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:06.577198+0000) 2024-03-06T18:21:06.956 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:21:06.964 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:21:07.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:07.575+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:07.577458+0000) 2024-03-06T18:21:08.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:08.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:08.577689+0000) 2024-03-06T18:21:09.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:09.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:09.577919+0000) 2024-03-06T18:21:10.573 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:10.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:10.578152+0000) 2024-03-06T18:21:11.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:11.576+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:11.578395+0000) 2024-03-06T18:21:12.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:12.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:12.578714+0000) 2024-03-06T18:21:13.574 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:13.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:13.578964+0000) 2024-03-06T18:21:14.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:14.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:14.579238+0000) 2024-03-06T18:21:15.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:15.577+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:15.579514+0000) 2024-03-06T18:21:16.575 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:16.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:16.579794+0000) 2024-03-06T18:21:17.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:17.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:17.580037+0000) 2024-03-06T18:21:18.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:18.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:18.580271+0000) 2024-03-06T18:21:19.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:19.578+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:19.580501+0000) 2024-03-06T18:21:20.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:20.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:20.580740+0000) 2024-03-06T18:21:21.576 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:21.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:21.580950+0000) 2024-03-06T18:21:22.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:22.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:22.581176+0000) 2024-03-06T18:21:23.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:23.579+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:23.581403+0000) 2024-03-06T18:21:24.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:24.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:24.581667+0000) 2024-03-06T18:21:25.577 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:25.580+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:25.581871+0000) 2024-03-06T18:21:26.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:26.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:26.582054+0000) 2024-03-06T18:21:27.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:27.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:27.582280+0000) 2024-03-06T18:21:28.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:28.581+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:28.582506+0000) 2024-03-06T18:21:29.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:29.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:29.582755+0000) 2024-03-06T18:21:30.578 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:30.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:30.583003+0000) 2024-03-06T18:21:31.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:31.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:31.583222+0000) 2024-03-06T18:21:32.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:32.582+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:32.583450+0000) 2024-03-06T18:21:33.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:33.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:33.583673+0000) 2024-03-06T18:21:34.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:34.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:34.583910+0000) 2024-03-06T18:21:35.579 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:35.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:35.584138+0000) 2024-03-06T18:21:36.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:36.583+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:36.584365+0000) 2024-03-06T18:21:37.137 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:21:37.147 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:21:37.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:37.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:37.584645+0000) 2024-03-06T18:21:38.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:38.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:38.584837+0000) 2024-03-06T18:21:39.580 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:39.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:39.585068+0000) 2024-03-06T18:21:40.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:40.584+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:40.585299+0000) 2024-03-06T18:21:41.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:41.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:41.585578+0000) 2024-03-06T18:21:42.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:42.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:42.585774+0000) 2024-03-06T18:21:43.581 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:43.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:43.585967+0000) 2024-03-06T18:21:44.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:44.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:44.586215+0000) 2024-03-06T18:21:45.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:45.585+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:45.586473+0000) 2024-03-06T18:21:46.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:46.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:46.586692+0000) 2024-03-06T18:21:47.582 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:47.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:47.586931+0000) 2024-03-06T18:21:48.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:48.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:48.587157+0000) 2024-03-06T18:21:49.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:49.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:49.587401+0000) 2024-03-06T18:21:50.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:50.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:50.587719+0000) 2024-03-06T18:21:51.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:51.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:51.587969+0000) 2024-03-06T18:21:52.583 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:52.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:52.588190+0000) 2024-03-06T18:21:53.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:53.586+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:53.588417+0000) 2024-03-06T18:21:54.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:54.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:54.588661+0000) 2024-03-06T18:21:55.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:55.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:55.588890+0000) 2024-03-06T18:21:56.584 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:56.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:56.589064+0000) 2024-03-06T18:21:57.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:57.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:57.589245+0000) 2024-03-06T18:21:58.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:58.587+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:58.589436+0000) 2024-03-06T18:21:59.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:21:59.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:21:59.589709+0000) 2024-03-06T18:22:00.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:00.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:00.589971+0000) 2024-03-06T18:22:01.585 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:01.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:01.590211+0000) 2024-03-06T18:22:02.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:02.588+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:02.590437+0000) 2024-03-06T18:22:03.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:03.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:03.590694+0000) 2024-03-06T18:22:04.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:04.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:04.590940+0000) 2024-03-06T18:22:05.586 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:05.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:05.591166+0000) 2024-03-06T18:22:06.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:06.589+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:06.591403+0000) 2024-03-06T18:22:07.358 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:22:07.366 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:22:07.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:07.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:07.591711+0000) 2024-03-06T18:22:08.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:08.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:08.591938+0000) 2024-03-06T18:22:09.587 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:09.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:09.592178+0000) 2024-03-06T18:22:10.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:10.590+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:10.592399+0000) 2024-03-06T18:22:11.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:11.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:11.592628+0000) 2024-03-06T18:22:12.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:12.591+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:12.592815+0000) 2024-03-06T18:22:13.588 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:13.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:13.593039+0000) 2024-03-06T18:22:14.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:14.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:14.593277+0000) 2024-03-06T18:22:15.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:15.592+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:15.593507+0000) 2024-03-06T18:22:16.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:16.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:16.593757+0000) 2024-03-06T18:22:17.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:17.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:17.593985+0000) 2024-03-06T18:22:18.589 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:18.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:18.594211+0000) 2024-03-06T18:22:19.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:19.593+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:19.594454+0000) 2024-03-06T18:22:20.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:20.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:20.594694+0000) 2024-03-06T18:22:21.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:21.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:21.594887+0000) 2024-03-06T18:22:22.590 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:22.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:22.595116+0000) 2024-03-06T18:22:23.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:23.594+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:23.595345+0000) 2024-03-06T18:22:24.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:24.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:24.595644+0000) 2024-03-06T18:22:25.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:25.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:25.595873+0000) 2024-03-06T18:22:26.591 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:26.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:26.596097+0000) 2024-03-06T18:22:27.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:27.595+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:27.596317+0000) 2024-03-06T18:22:28.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:28.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:28.596571+0000) 2024-03-06T18:22:29.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:29.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:29.596810+0000) 2024-03-06T18:22:30.592 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:30.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:30.597036+0000) 2024-03-06T18:22:31.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:31.596+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:31.597314+0000) 2024-03-06T18:22:32.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:32.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:32.597621+0000) 2024-03-06T18:22:33.593 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:33.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:33.597960+0000) 2024-03-06T18:22:34.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:34.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:34.598241+0000) 2024-03-06T18:22:35.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:35.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:35.598482+0000) 2024-03-06T18:22:36.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:36.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:36.598688+0000) 2024-03-06T18:22:37.580 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:22:37.588 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:22:37.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:37.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:37.598855+0000) 2024-03-06T18:22:38.594 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:38.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:38.599114+0000) 2024-03-06T18:22:39.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:39.597+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:39.599387+0000) 2024-03-06T18:22:40.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:40.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:40.599702+0000) 2024-03-06T18:22:41.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:41.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:41.599946+0000) 2024-03-06T18:22:42.595 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:42.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:42.600181+0000) 2024-03-06T18:22:43.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:43.598+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:43.600424+0000) 2024-03-06T18:22:44.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:44.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:44.600712+0000) 2024-03-06T18:22:45.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:45.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:45.600981+0000) 2024-03-06T18:22:46.596 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:46.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:46.601256+0000) 2024-03-06T18:22:47.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:47.599+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:47.601519+0000) 2024-03-06T18:22:48.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:48.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:48.601818+0000) 2024-03-06T18:22:49.597 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:49.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:49.602083+0000) 2024-03-06T18:22:50.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:50.600+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:50.602329+0000) 2024-03-06T18:22:51.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:51.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:51.602564+0000) 2024-03-06T18:22:52.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:52.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:52.602785+0000) 2024-03-06T18:22:53.598 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:53.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:53.603035+0000) 2024-03-06T18:22:54.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:54.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:54.603318+0000) 2024-03-06T18:22:55.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:55.601+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:55.603520+0000) 2024-03-06T18:22:56.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:56.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:56.603778+0000) 2024-03-06T18:22:57.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:57.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:57.603993+0000) 2024-03-06T18:22:58.599 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:58.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:58.604193+0000) 2024-03-06T18:22:59.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:22:59.602+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:22:59.604436+0000) 2024-03-06T18:23:00.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:00.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:00.604659+0000) 2024-03-06T18:23:01.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:01.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:01.604890+0000) 2024-03-06T18:23:02.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:02.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:02.605114+0000) 2024-03-06T18:23:03.600 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:03.604+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:03.605337+0000) 2024-03-06T18:23:04.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:04.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:04.605618+0000) 2024-03-06T18:23:05.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:05.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:05.605846+0000) 2024-03-06T18:23:06.601 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:06.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:06.606067+0000) 2024-03-06T18:23:07.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:07.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:07.606285+0000) 2024-03-06T18:23:07.718 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:23:07.726 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:23:08.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:08.605+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:08.606508+0000) 2024-03-06T18:23:09.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:09.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:09.606750+0000) 2024-03-06T18:23:10.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:10.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:10.606986+0000) 2024-03-06T18:23:11.602 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:11.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:11.607170+0000) 2024-03-06T18:23:12.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:12.606+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:12.607391+0000) 2024-03-06T18:23:13.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:13.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:13.607649+0000) 2024-03-06T18:23:14.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:14.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:14.607876+0000) 2024-03-06T18:23:15.603 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:15.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:15.608102+0000) 2024-03-06T18:23:16.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:16.607+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:16.608330+0000) 2024-03-06T18:23:17.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:17.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:17.608568+0000) 2024-03-06T18:23:18.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:18.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:18.608776+0000) 2024-03-06T18:23:19.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:19.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:19.609019+0000) 2024-03-06T18:23:20.604 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:20.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:20.609214+0000) 2024-03-06T18:23:21.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:21.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:21.609471+0000) 2024-03-06T18:23:22.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:22.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:22.609661+0000) 2024-03-06T18:23:23.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:23.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:23.609908+0000) 2024-03-06T18:23:24.605 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:24.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:24.610100+0000) 2024-03-06T18:23:25.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:25.608+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:25.610283+0000) 2024-03-06T18:23:26.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:26.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:26.610673+0000) 2024-03-06T18:23:27.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:27.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:27.610908+0000) 2024-03-06T18:23:28.606 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:28.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:28.611129+0000) 2024-03-06T18:23:29.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:29.609+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:29.611336+0000) 2024-03-06T18:23:30.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:30.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:30.611527+0000) 2024-03-06T18:23:31.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:31.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:31.611767+0000) 2024-03-06T18:23:32.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:32.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:32.611957+0000) 2024-03-06T18:23:33.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:33.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:33.612154+0000) 2024-03-06T18:23:34.607 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:34.610+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:34.612380+0000) 2024-03-06T18:23:35.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:35.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:35.612670+0000) 2024-03-06T18:23:36.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:36.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:36.612893+0000) 2024-03-06T18:23:37.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:37.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:37.613097+0000) 2024-03-06T18:23:37.851 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:23:37.859 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:23:38.608 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:38.611+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:38.613332+0000) 2024-03-06T18:23:39.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:39.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:39.613518+0000) 2024-03-06T18:23:40.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:40.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:40.613787+0000) 2024-03-06T18:23:41.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:41.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:41.613990+0000) 2024-03-06T18:23:42.609 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:42.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:42.614224+0000) 2024-03-06T18:23:43.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:43.612+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:43.614448+0000) 2024-03-06T18:23:44.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:44.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:44.614675+0000) 2024-03-06T18:23:45.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:45.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:45.614893+0000) 2024-03-06T18:23:46.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:46.613+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:46.615120+0000) 2024-03-06T18:23:47.610 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:47.614+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:47.615340+0000) 2024-03-06T18:23:48.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:48.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:48.615600+0000) 2024-03-06T18:23:49.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:49.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:49.615834+0000) 2024-03-06T18:23:50.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:50.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:50.616076+0000) 2024-03-06T18:23:51.611 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:51.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:51.616282+0000) 2024-03-06T18:23:52.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:52.615+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:52.616504+0000) 2024-03-06T18:23:53.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:53.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:53.616733+0000) 2024-03-06T18:23:54.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:54.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:54.616962+0000) 2024-03-06T18:23:55.612 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:55.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:55.617178+0000) 2024-03-06T18:23:56.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:56.616+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:56.617437+0000) 2024-03-06T18:23:57.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:57.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:57.617722+0000) 2024-03-06T18:23:58.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:58.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:58.617997+0000) 2024-03-06T18:23:59.613 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:23:59.617+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:23:59.618273+0000) 2024-03-06T18:24:00.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:00.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:00.618589+0000) 2024-03-06T18:24:01.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:01.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:01.618884+0000) 2024-03-06T18:24:02.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:02.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:02.619114+0000) 2024-03-06T18:24:03.614 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:03.618+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:03.619357+0000) 2024-03-06T18:24:04.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:04.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:04.619644+0000) 2024-03-06T18:24:05.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:05.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:05.619853+0000) 2024-03-06T18:24:06.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:06.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:06.620092+0000) 2024-03-06T18:24:07.615 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:07.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:07.620275+0000) 2024-03-06T18:24:07.976 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:24:07.984 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:24:08.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:08.619+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:08.620508+0000) 2024-03-06T18:24:09.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:09.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:09.620777+0000) 2024-03-06T18:24:10.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:10.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:10.620991+0000) 2024-03-06T18:24:11.616 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:11.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:11.621203+0000) 2024-03-06T18:24:12.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:12.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:12.621438+0000) 2024-03-06T18:24:13.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:13.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:13.621684+0000) 2024-03-06T18:24:14.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:14.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:14.621974+0000) 2024-03-06T18:24:15.617 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:15.620+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:15.622246+0000) 2024-03-06T18:24:16.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:16.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:16.622522+0000) 2024-03-06T18:24:17.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:17.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:17.622774+0000) 2024-03-06T18:24:18.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:18.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:18.622999+0000) 2024-03-06T18:24:19.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:19.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:19.623226+0000) 2024-03-06T18:24:20.618 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:20.621+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:20.623453+0000) 2024-03-06T18:24:21.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:21.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:21.623719+0000) 2024-03-06T18:24:22.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:22.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:22.623994+0000) 2024-03-06T18:24:23.619 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:23.622+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:23.624280+0000) 2024-03-06T18:24:24.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:24.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:24.624590+0000) 2024-03-06T18:24:25.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:25.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:25.624872+0000) 2024-03-06T18:24:26.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:26.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:26.625150+0000) 2024-03-06T18:24:27.620 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:27.623+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:27.625371+0000) 2024-03-06T18:24:28.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:28.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:28.625645+0000) 2024-03-06T18:24:29.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:29.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:29.625850+0000) 2024-03-06T18:24:30.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:30.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:30.626054+0000) 2024-03-06T18:24:31.621 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:31.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:31.626262+0000) 2024-03-06T18:24:32.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:32.624+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:32.626501+0000) 2024-03-06T18:24:33.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:33.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:33.626719+0000) 2024-03-06T18:24:34.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:34.625+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:34.626918+0000) 2024-03-06T18:24:35.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:35.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:35.627153+0000) 2024-03-06T18:24:36.622 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:36.626+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:36.627377+0000) 2024-03-06T18:24:37.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:37.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:37.627642+0000) 2024-03-06T18:24:38.106 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:24:38.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:24:38.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:38.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:38.627831+0000) 2024-03-06T18:24:39.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:39.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:39.628017+0000) 2024-03-06T18:24:40.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:40.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:40.628221+0000) 2024-03-06T18:24:41.623 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:41.627+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:41.628463+0000) 2024-03-06T18:24:42.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:42.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:42.628730+0000) 2024-03-06T18:24:43.624 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:43.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:43.628970+0000) 2024-03-06T18:24:44.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:44.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:44.629200+0000) 2024-03-06T18:24:45.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:45.628+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:45.629439+0000) 2024-03-06T18:24:46.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:46.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:46.629666+0000) 2024-03-06T18:24:47.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:47.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:47.629889+0000) 2024-03-06T18:24:48.625 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:48.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:48.630110+0000) 2024-03-06T18:24:49.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:49.629+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:49.630323+0000) 2024-03-06T18:24:50.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:50.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:50.630585+0000) 2024-03-06T18:24:51.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:51.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:51.630869+0000) 2024-03-06T18:24:52.626 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:52.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:52.631145+0000) 2024-03-06T18:24:53.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:53.630+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:53.631414+0000) 2024-03-06T18:24:54.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:54.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:54.631673+0000) 2024-03-06T18:24:55.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:55.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:55.631878+0000) 2024-03-06T18:24:56.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:56.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:56.632109+0000) 2024-03-06T18:24:57.627 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:57.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:57.632328+0000) 2024-03-06T18:24:58.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:58.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:58.632584+0000) 2024-03-06T18:24:59.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:24:59.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:24:59.632767+0000) 2024-03-06T18:25:00.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:00.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:00.633016+0000) 2024-03-06T18:25:01.628 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:01.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:01.633252+0000) 2024-03-06T18:25:02.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:02.631+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:02.633482+0000) 2024-03-06T18:25:03.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:03.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:03.633686+0000) 2024-03-06T18:25:04.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:04.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:04.633899+0000) 2024-03-06T18:25:05.629 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:05.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:05.634144+0000) 2024-03-06T18:25:06.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:06.632+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:06.634365+0000) 2024-03-06T18:25:07.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:07.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:07.634666+0000) 2024-03-06T18:25:08.245 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:25:08.254 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:25:08.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:08.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:08.634908+0000) 2024-03-06T18:25:09.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:09.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:09.635183+0000) 2024-03-06T18:25:10.630 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:10.633+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:10.635439+0000) 2024-03-06T18:25:11.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:11.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:11.635657+0000) 2024-03-06T18:25:12.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:12.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:12.635895+0000) 2024-03-06T18:25:13.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:13.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:13.636137+0000) 2024-03-06T18:25:14.631 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:14.634+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:14.636378+0000) 2024-03-06T18:25:15.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:15.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:15.636654+0000) 2024-03-06T18:25:16.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:16.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:16.636896+0000) 2024-03-06T18:25:17.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:17.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:17.637139+0000) 2024-03-06T18:25:18.632 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:18.635+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:18.637395+0000) 2024-03-06T18:25:19.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:19.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:19.637652+0000) 2024-03-06T18:25:20.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:20.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:20.637884+0000) 2024-03-06T18:25:21.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:21.636+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:21.638112+0000) 2024-03-06T18:25:22.633 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:22.637+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:22.638327+0000) 2024-03-06T18:25:23.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:23.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:23.638586+0000) 2024-03-06T18:25:24.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:24.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:24.638819+0000) 2024-03-06T18:25:25.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:25.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:25.639055+0000) 2024-03-06T18:25:26.634 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:26.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:26.639290+0000) 2024-03-06T18:25:27.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:27.638+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:27.639512+0000) 2024-03-06T18:25:28.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:28.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:28.639818+0000) 2024-03-06T18:25:29.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:29.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:29.640083+0000) 2024-03-06T18:25:30.635 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:30.639+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:30.640337+0000) 2024-03-06T18:25:31.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:31.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:31.640606+0000) 2024-03-06T18:25:32.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:32.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:32.640831+0000) 2024-03-06T18:25:33.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:33.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:33.641011+0000) 2024-03-06T18:25:34.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:34.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:34.641239+0000) 2024-03-06T18:25:35.636 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:35.640+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:35.641470+0000) 2024-03-06T18:25:36.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:36.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:36.641716+0000) 2024-03-06T18:25:37.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:37.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:37.641935+0000) 2024-03-06T18:25:38.383 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:25:38.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:25:38.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:38.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:38.642172+0000) 2024-03-06T18:25:39.637 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:39.641+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:39.642396+0000) 2024-03-06T18:25:40.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:40.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:40.642662+0000) 2024-03-06T18:25:41.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:41.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:41.642899+0000) 2024-03-06T18:25:42.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:42.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:42.643122+0000) 2024-03-06T18:25:43.638 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:43.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:43.643340+0000) 2024-03-06T18:25:44.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:44.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:44.643599+0000) 2024-03-06T18:25:45.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:45.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:45.643851+0000) 2024-03-06T18:25:46.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:46.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:46.644088+0000) 2024-03-06T18:25:47.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:47.642+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:47.644315+0000) 2024-03-06T18:25:48.639 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:48.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:48.644576+0000) 2024-03-06T18:25:49.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:49.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:49.644801+0000) 2024-03-06T18:25:50.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:50.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:50.644991+0000) 2024-03-06T18:25:51.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:51.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:51.645200+0000) 2024-03-06T18:25:52.640 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:52.643+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:52.645388+0000) 2024-03-06T18:25:53.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:53.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:53.645662+0000) 2024-03-06T18:25:54.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:54.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:54.645842+0000) 2024-03-06T18:25:55.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:55.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:55.646071+0000) 2024-03-06T18:25:56.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:56.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:56.646267+0000) 2024-03-06T18:25:57.641 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:57.644+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:57.646504+0000) 2024-03-06T18:25:58.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:58.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:58.646786+0000) 2024-03-06T18:25:59.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:25:59.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:25:59.646977+0000) 2024-03-06T18:26:00.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:00.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:00.647214+0000) 2024-03-06T18:26:01.642 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:01.645+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:01.647483+0000) 2024-03-06T18:26:02.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:02.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:02.647734+0000) 2024-03-06T18:26:03.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:03.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:03.647931+0000) 2024-03-06T18:26:04.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:04.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:04.648155+0000) 2024-03-06T18:26:05.643 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:05.646+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:05.648382+0000) 2024-03-06T18:26:06.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:06.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:06.648641+0000) 2024-03-06T18:26:07.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:07.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:07.648824+0000) 2024-03-06T18:26:08.522 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:26:08.530 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:26:08.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:08.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:08.649006+0000) 2024-03-06T18:26:09.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:09.647+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:09.649220+0000) 2024-03-06T18:26:10.644 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:10.648+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:10.649457+0000) 2024-03-06T18:26:11.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:11.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:11.649706+0000) 2024-03-06T18:26:12.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:12.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:12.649925+0000) 2024-03-06T18:26:13.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:13.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:13.650107+0000) 2024-03-06T18:26:14.645 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:14.649+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:14.650336+0000) 2024-03-06T18:26:15.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:15.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:15.650604+0000) 2024-03-06T18:26:16.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:16.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:16.650838+0000) 2024-03-06T18:26:17.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:17.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:17.651082+0000) 2024-03-06T18:26:18.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:18.650+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:18.651305+0000) 2024-03-06T18:26:19.646 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:19.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:19.651525+0000) 2024-03-06T18:26:20.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:20.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:20.651805+0000) 2024-03-06T18:26:21.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:21.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:21.652038+0000) 2024-03-06T18:26:22.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:22.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:22.652232+0000) 2024-03-06T18:26:23.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:23.651+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:23.652438+0000) 2024-03-06T18:26:24.647 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:24.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:24.652668+0000) 2024-03-06T18:26:25.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:25.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:25.652886+0000) 2024-03-06T18:26:26.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:26.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:26.653117+0000) 2024-03-06T18:26:27.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:27.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:27.653347+0000) 2024-03-06T18:26:28.648 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:28.652+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:28.653498+0000) 2024-03-06T18:26:29.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:29.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:29.653716+0000) 2024-03-06T18:26:30.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:30.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:30.653943+0000) 2024-03-06T18:26:31.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:31.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:31.654147+0000) 2024-03-06T18:26:32.649 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:32.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:32.654360+0000) 2024-03-06T18:26:33.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:33.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:33.654566+0000) 2024-03-06T18:26:34.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:34.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:34.654791+0000) 2024-03-06T18:26:35.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:35.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:35.655024+0000) 2024-03-06T18:26:36.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:36.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:36.655228+0000) 2024-03-06T18:26:37.650 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:37.653+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:37.655449+0000) 2024-03-06T18:26:38.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:38.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:38.655649+0000) 2024-03-06T18:26:38.660 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:26:38.668 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:26:39.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:39.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:39.655879+0000) 2024-03-06T18:26:40.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:40.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:40.656055+0000) 2024-03-06T18:26:41.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:41.654+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:41.656330+0000) 2024-03-06T18:26:42.651 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:42.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:42.656611+0000) 2024-03-06T18:26:43.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:43.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:43.656856+0000) 2024-03-06T18:26:44.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:44.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:44.657089+0000) 2024-03-06T18:26:45.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:45.655+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:45.657340+0000) 2024-03-06T18:26:46.652 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:46.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:46.657623+0000) 2024-03-06T18:26:47.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:47.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:47.657859+0000) 2024-03-06T18:26:48.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:48.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:48.658079+0000) 2024-03-06T18:26:49.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:49.656+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:49.658319+0000) 2024-03-06T18:26:50.653 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:50.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:50.658586+0000) 2024-03-06T18:26:51.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:51.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:51.658824+0000) 2024-03-06T18:26:52.654 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:52.657+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:52.659034+0000) 2024-03-06T18:26:53.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:53.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:53.659890+0000) 2024-03-06T18:26:54.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:54.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:54.660160+0000) 2024-03-06T18:26:55.655 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:55.658+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:55.660443+0000) 2024-03-06T18:26:56.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:56.659+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:56.660727+0000) 2024-03-06T18:26:57.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:57.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:57.660937+0000) 2024-03-06T18:26:58.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:58.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:58.661132+0000) 2024-03-06T18:26:59.656 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:26:59.660+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:26:59.661373+0000) 2024-03-06T18:27:00.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:00.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:00.661664+0000) 2024-03-06T18:27:01.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:01.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:01.661904+0000) 2024-03-06T18:27:02.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:02.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:02.662122+0000) 2024-03-06T18:27:03.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:03.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:03.662306+0000) 2024-03-06T18:27:04.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:04.661+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:04.662512+0000) 2024-03-06T18:27:05.657 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:05.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:05.662741+0000) 2024-03-06T18:27:06.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:06.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:06.662943+0000) 2024-03-06T18:27:07.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:07.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:07.663172+0000) 2024-03-06T18:27:08.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:08.662+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:08.663389+0000) 2024-03-06T18:27:08.789 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:27:08.797 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:27:09.658 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:09.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:09.663624+0000) 2024-03-06T18:27:10.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:10.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:10.663844+0000) 2024-03-06T18:27:11.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:11.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:11.664076+0000) 2024-03-06T18:27:12.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:12.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:12.664249+0000) 2024-03-06T18:27:13.659 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:13.663+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:13.664477+0000) 2024-03-06T18:27:14.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:14.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:14.664709+0000) 2024-03-06T18:27:15.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:15.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:15.664977+0000) 2024-03-06T18:27:16.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:16.664+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:16.665254+0000) 2024-03-06T18:27:17.660 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:17.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:17.665529+0000) 2024-03-06T18:27:18.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:18.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:18.665775+0000) 2024-03-06T18:27:19.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:19.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:19.665989+0000) 2024-03-06T18:27:20.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:20.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:20.666201+0000) 2024-03-06T18:27:21.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:21.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:21.666422+0000) 2024-03-06T18:27:22.661 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:22.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:22.666645+0000) 2024-03-06T18:27:23.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:23.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:23.666866+0000) 2024-03-06T18:27:24.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:24.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:24.667045+0000) 2024-03-06T18:27:25.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:25.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:25.667280+0000) 2024-03-06T18:27:26.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:26.665+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:26.667512+0000) 2024-03-06T18:27:27.662 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:27.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:27.667741+0000) 2024-03-06T18:27:28.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:28.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:28.667977+0000) 2024-03-06T18:27:29.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:29.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:29.668182+0000) 2024-03-06T18:27:30.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:30.666+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:30.668461+0000) 2024-03-06T18:27:31.663 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:31.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:31.668717+0000) 2024-03-06T18:27:32.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:32.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:32.668908+0000) 2024-03-06T18:27:33.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:33.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:33.669140+0000) 2024-03-06T18:27:34.664 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:34.667+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:34.669392+0000) 2024-03-06T18:27:35.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:35.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:35.669728+0000) 2024-03-06T18:27:36.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:36.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:36.669930+0000) 2024-03-06T18:27:37.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:37.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:37.670219+0000) 2024-03-06T18:27:38.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:38.668+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:38.670440+0000) 2024-03-06T18:27:38.927 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:27:38.936 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:27:39.665 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:39.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:39.670641+0000) 2024-03-06T18:27:40.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:40.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:40.670887+0000) 2024-03-06T18:27:41.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:41.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:41.671121+0000) 2024-03-06T18:27:42.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:42.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:42.671307+0000) 2024-03-06T18:27:43.666 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:43.669+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:43.671507+0000) 2024-03-06T18:27:44.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:44.670+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:44.671768+0000) 2024-03-06T18:27:45.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:45.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:45.672019+0000) 2024-03-06T18:27:46.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:46.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:46.672231+0000) 2024-03-06T18:27:47.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:47.671+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:47.672466+0000) 2024-03-06T18:27:48.667 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:48.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:48.672686+0000) 2024-03-06T18:27:49.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:49.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:49.672875+0000) 2024-03-06T18:27:50.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:50.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:50.673091+0000) 2024-03-06T18:27:51.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:51.672+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:51.673326+0000) 2024-03-06T18:27:52.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:52.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:52.673526+0000) 2024-03-06T18:27:53.668 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:53.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:53.673794+0000) 2024-03-06T18:27:54.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:54.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:54.674000+0000) 2024-03-06T18:27:55.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:55.673+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:55.674282+0000) 2024-03-06T18:27:56.669 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:56.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:56.674601+0000) 2024-03-06T18:27:57.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:57.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:57.674838+0000) 2024-03-06T18:27:58.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:58.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:58.675083+0000) 2024-03-06T18:27:59.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:27:59.674+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:27:59.675340+0000) 2024-03-06T18:28:00.670 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:00.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:00.675639+0000) 2024-03-06T18:28:01.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:01.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:01.675859+0000) 2024-03-06T18:28:02.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:02.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:02.676078+0000) 2024-03-06T18:28:03.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:03.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:03.676303+0000) 2024-03-06T18:28:04.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:04.675+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:04.676496+0000) 2024-03-06T18:28:05.671 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:05.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:05.676755+0000) 2024-03-06T18:28:06.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:06.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:06.677007+0000) 2024-03-06T18:28:07.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:07.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:07.677221+0000) 2024-03-06T18:28:08.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:08.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:08.677442+0000) 2024-03-06T18:28:09.129 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:28:09.139 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:28:09.672 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:09.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:09.677699+0000) 2024-03-06T18:28:10.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:10.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:10.677990+0000) 2024-03-06T18:28:11.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:11.676+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:11.678281+0000) 2024-03-06T18:28:12.673 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:12.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:12.678613+0000) 2024-03-06T18:28:13.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:13.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:13.678900+0000) 2024-03-06T18:28:14.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:14.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:14.679172+0000) 2024-03-06T18:28:15.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:15.677+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:15.679460+0000) 2024-03-06T18:28:16.674 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:16.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:16.679742+0000) 2024-03-06T18:28:17.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:17.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:17.679960+0000) 2024-03-06T18:28:18.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:18.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:18.680201+0000) 2024-03-06T18:28:19.675 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:19.678+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:19.680429+0000) 2024-03-06T18:28:20.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:20.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:20.680721+0000) 2024-03-06T18:28:21.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:21.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:21.680979+0000) 2024-03-06T18:28:22.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:22.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:22.681198+0000) 2024-03-06T18:28:23.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:23.679+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:23.681390+0000) 2024-03-06T18:28:24.676 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:24.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:24.681643+0000) 2024-03-06T18:28:25.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:25.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:25.681956+0000) 2024-03-06T18:28:26.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:26.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:26.682184+0000) 2024-03-06T18:28:27.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:27.680+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:27.682448+0000) 2024-03-06T18:28:28.677 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:28.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:28.682677+0000) 2024-03-06T18:28:29.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:29.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:29.682865+0000) 2024-03-06T18:28:30.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:30.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:30.683147+0000) 2024-03-06T18:28:31.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:31.681+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:31.683429+0000) 2024-03-06T18:28:32.678 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:32.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:32.683672+0000) 2024-03-06T18:28:33.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:33.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:33.683898+0000) 2024-03-06T18:28:34.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:34.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:34.684128+0000) 2024-03-06T18:28:35.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:35.683+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:35.684388+0000) 2024-03-06T18:28:36.679 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:36.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:36.684627+0000) 2024-03-06T18:28:37.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:37.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:37.684824+0000) 2024-03-06T18:28:38.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:38.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:38.685069+0000) 2024-03-06T18:28:39.364 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:28:39.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:28:39.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:39.684+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:39.685325+0000) 2024-03-06T18:28:40.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:40.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:40.685620+0000) 2024-03-06T18:28:41.680 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:41.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:41.685863+0000) 2024-03-06T18:28:42.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:42.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:42.686041+0000) 2024-03-06T18:28:43.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:43.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:43.686229+0000) 2024-03-06T18:28:44.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:44.685+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:44.686417+0000) 2024-03-06T18:28:45.681 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:45.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:45.686682+0000) 2024-03-06T18:28:46.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:46.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:46.686895+0000) 2024-03-06T18:28:47.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:47.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:47.687124+0000) 2024-03-06T18:28:48.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:48.686+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:48.687347+0000) 2024-03-06T18:28:49.682 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:49.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:49.687626+0000) 2024-03-06T18:28:50.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:50.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:50.687881+0000) 2024-03-06T18:28:51.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:51.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:51.688102+0000) 2024-03-06T18:28:52.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:52.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:52.688325+0000) 2024-03-06T18:28:53.683 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:53.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:53.688515+0000) 2024-03-06T18:28:54.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:54.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:54.688708+0000) 2024-03-06T18:28:55.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:55.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:55.688919+0000) 2024-03-06T18:28:56.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:56.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:56.689143+0000) 2024-03-06T18:28:57.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:57.687+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:57.689379+0000) 2024-03-06T18:28:58.684 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:58.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:58.689672+0000) 2024-03-06T18:28:59.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:28:59.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:28:59.689835+0000) 2024-03-06T18:29:00.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:00.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:00.690075+0000) 2024-03-06T18:29:01.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:01.688+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:01.690308+0000) 2024-03-06T18:29:02.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:02.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:02.690572+0000) 2024-03-06T18:29:03.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:03.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:03.690777+0000) 2024-03-06T18:29:04.685 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:04.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:04.690978+0000) 2024-03-06T18:29:05.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:05.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:05.691214+0000) 2024-03-06T18:29:06.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:06.689+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:06.691445+0000) 2024-03-06T18:29:07.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:07.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:07.691675+0000) 2024-03-06T18:29:08.686 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:08.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:08.691868+0000) 2024-03-06T18:29:09.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:29:09.576 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:29:09.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:09.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:09.692149+0000) 2024-03-06T18:29:10.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:10.690+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:10.692408+0000) 2024-03-06T18:29:11.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:11.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:11.692677+0000) 2024-03-06T18:29:12.687 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:12.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:12.692892+0000) 2024-03-06T18:29:13.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:13.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:13.693137+0000) 2024-03-06T18:29:14.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:14.691+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:14.693388+0000) 2024-03-06T18:29:15.688 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:15.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:15.693701+0000) 2024-03-06T18:29:16.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:16.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:16.693900+0000) 2024-03-06T18:29:17.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:17.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:17.694130+0000) 2024-03-06T18:29:18.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:18.692+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:18.694429+0000) 2024-03-06T18:29:19.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:19.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:19.694658+0000) 2024-03-06T18:29:20.689 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:20.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:20.694855+0000) 2024-03-06T18:29:21.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:21.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:21.695095+0000) 2024-03-06T18:29:22.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:22.694+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:22.695320+0000) 2024-03-06T18:29:23.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:23.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:23.695571+0000) 2024-03-06T18:29:24.690 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:24.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:24.695779+0000) 2024-03-06T18:29:25.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:25.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:25.696013+0000) 2024-03-06T18:29:26.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:26.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:26.696210+0000) 2024-03-06T18:29:27.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:27.695+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:27.696398+0000) 2024-03-06T18:29:28.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:28.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:28.696647+0000) 2024-03-06T18:29:29.691 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:29.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:29.696862+0000) 2024-03-06T18:29:30.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:30.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:30.697081+0000) 2024-03-06T18:29:31.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:31.696+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:31.697326+0000) 2024-03-06T18:29:32.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:32.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:32.697572+0000) 2024-03-06T18:29:33.692 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:33.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:33.697802+0000) 2024-03-06T18:29:34.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:34.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:34.698033+0000) 2024-03-06T18:29:35.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:35.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:35.698271+0000) 2024-03-06T18:29:36.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:36.697+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:36.698504+0000) 2024-03-06T18:29:37.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:37.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:37.698760+0000) 2024-03-06T18:29:38.693 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:38.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:38.698963+0000) 2024-03-06T18:29:39.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:39.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:39.699180+0000) 2024-03-06T18:29:39.706 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:29:39.716 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:29:40.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:40.698+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:40.699394+0000) 2024-03-06T18:29:41.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:41.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:41.699656+0000) 2024-03-06T18:29:42.694 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:42.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:42.699841+0000) 2024-03-06T18:29:43.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:43.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:43.700104+0000) 2024-03-06T18:29:44.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:44.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:44.700382+0000) 2024-03-06T18:29:45.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:45.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:45.700707+0000) 2024-03-06T18:29:46.695 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:46.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:46.700960+0000) 2024-03-06T18:29:47.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:47.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:47.701155+0000) 2024-03-06T18:29:48.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:48.699+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:48.701390+0000) 2024-03-06T18:29:49.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:49.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:49.701595+0000) 2024-03-06T18:29:50.696 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:50.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:50.701837+0000) 2024-03-06T18:29:51.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:51.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:51.702072+0000) 2024-03-06T18:29:52.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:52.700+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:52.702299+0000) 2024-03-06T18:29:53.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:53.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:53.702521+0000) 2024-03-06T18:29:54.697 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:54.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:54.702779+0000) 2024-03-06T18:29:55.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:55.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:55.703013+0000) 2024-03-06T18:29:56.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:56.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:56.703196+0000) 2024-03-06T18:29:57.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:57.701+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:57.703381+0000) 2024-03-06T18:29:58.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:58.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:58.703709+0000) 2024-03-06T18:29:59.698 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:29:59.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:29:59.703944+0000) 2024-03-06T18:30:00.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:00.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:00.704158+0000) 2024-03-06T18:30:01.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:01.702+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:01.704500+0000) 2024-03-06T18:30:02.699 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:02.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:02.704812+0000) 2024-03-06T18:30:03.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:03.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:03.705085+0000) 2024-03-06T18:30:04.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:04.703+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:04.705356+0000) 2024-03-06T18:30:05.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:05.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:05.705625+0000) 2024-03-06T18:30:06.700 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:06.704+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:06.705904+0000) 2024-03-06T18:30:07.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:07.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:07.706063+0000) 2024-03-06T18:30:08.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:08.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:08.706267+0000) 2024-03-06T18:30:09.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:09.705+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:09.706502+0000) 2024-03-06T18:30:09.920 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:30:09.928 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:30:10.701 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:10.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:10.706766+0000) 2024-03-06T18:30:11.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:11.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:11.707026+0000) 2024-03-06T18:30:12.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:12.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:12.707221+0000) 2024-03-06T18:30:13.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:13.706+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:13.707443+0000) 2024-03-06T18:30:14.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:14.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:14.707667+0000) 2024-03-06T18:30:15.702 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:15.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:15.707874+0000) 2024-03-06T18:30:16.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:16.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:16.708116+0000) 2024-03-06T18:30:17.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:17.707+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:17.708359+0000) 2024-03-06T18:30:18.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:18.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:18.708643+0000) 2024-03-06T18:30:19.703 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:19.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:19.708852+0000) 2024-03-06T18:30:20.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:20.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:20.709073+0000) 2024-03-06T18:30:21.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:21.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:21.709308+0000) 2024-03-06T18:30:22.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:22.708+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:22.709514+0000) 2024-03-06T18:30:23.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:23.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:23.709747+0000) 2024-03-06T18:30:24.704 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:24.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:24.709936+0000) 2024-03-06T18:30:25.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:25.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:25.710116+0000) 2024-03-06T18:30:26.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:26.709+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:26.710344+0000) 2024-03-06T18:30:27.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:27.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:27.710604+0000) 2024-03-06T18:30:28.705 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:28.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:28.710828+0000) 2024-03-06T18:30:29.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:29.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:29.711059+0000) 2024-03-06T18:30:30.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:30.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:30.711296+0000) 2024-03-06T18:30:31.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:31.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:31.711563+0000) 2024-03-06T18:30:32.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:32.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:32.711716+0000) 2024-03-06T18:30:33.706 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:33.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:33.711905+0000) 2024-03-06T18:30:34.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:34.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:34.712078+0000) 2024-03-06T18:30:35.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:35.710+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:35.712305+0000) 2024-03-06T18:30:36.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:36.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:36.712558+0000) 2024-03-06T18:30:37.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:37.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:37.712756+0000) 2024-03-06T18:30:38.707 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:38.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:38.712951+0000) 2024-03-06T18:30:39.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:39.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:39.713180+0000) 2024-03-06T18:30:40.150 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:30:40.158 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:30:40.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:40.711+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:40.713407+0000) 2024-03-06T18:30:41.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:41.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:41.713643+0000) 2024-03-06T18:30:42.708 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:42.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:42.713862+0000) 2024-03-06T18:30:43.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:43.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:43.714084+0000) 2024-03-06T18:30:44.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:44.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:44.714307+0000) 2024-03-06T18:30:45.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:45.712+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:45.714477+0000) 2024-03-06T18:30:46.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:46.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:46.714671+0000) 2024-03-06T18:30:47.709 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:47.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:47.714907+0000) 2024-03-06T18:30:48.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:48.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:48.715156+0000) 2024-03-06T18:30:49.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:49.713+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:49.715417+0000) 2024-03-06T18:30:50.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:50.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:50.715703+0000) 2024-03-06T18:30:51.710 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:51.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:51.715930+0000) 2024-03-06T18:30:52.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:52.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:52.716153+0000) 2024-03-06T18:30:53.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:53.714+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:53.716365+0000) 2024-03-06T18:30:54.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:54.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:54.716639+0000) 2024-03-06T18:30:55.711 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:55.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:55.716870+0000) 2024-03-06T18:30:56.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:56.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:56.717094+0000) 2024-03-06T18:30:57.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:57.716+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:57.717314+0000) 2024-03-06T18:30:58.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:58.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:58.717602+0000) 2024-03-06T18:30:59.712 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:30:59.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:30:59.717899+0000) 2024-03-06T18:31:00.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:00.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:00.718177+0000) 2024-03-06T18:31:01.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:01.717+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:01.718467+0000) 2024-03-06T18:31:02.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:02.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:02.718742+0000) 2024-03-06T18:31:03.713 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:03.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:03.719015+0000) 2024-03-06T18:31:04.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:04.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:04.719213+0000) 2024-03-06T18:31:05.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:05.718+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:05.719403+0000) 2024-03-06T18:31:06.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:06.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:06.719651+0000) 2024-03-06T18:31:07.714 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:07.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:07.719842+0000) 2024-03-06T18:31:08.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:08.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:08.720066+0000) 2024-03-06T18:31:09.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:09.719+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:09.720288+0000) 2024-03-06T18:31:10.337 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:31:10.346 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:31:10.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:10.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:10.720569+0000) 2024-03-06T18:31:11.715 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:11.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:11.720842+0000) 2024-03-06T18:31:12.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:12.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:12.721118+0000) 2024-03-06T18:31:13.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:13.720+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:13.721393+0000) 2024-03-06T18:31:14.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:14.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:14.721616+0000) 2024-03-06T18:31:15.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:15.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:15.721843+0000) 2024-03-06T18:31:16.716 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:16.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:16.722058+0000) 2024-03-06T18:31:17.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:17.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:17.722259+0000) 2024-03-06T18:31:18.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:18.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:18.722476+0000) 2024-03-06T18:31:19.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:19.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:19.722682+0000) 2024-03-06T18:31:20.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:20.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:20.722872+0000) 2024-03-06T18:31:21.717 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:21.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:21.723118+0000) 2024-03-06T18:31:22.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:22.721+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:22.723337+0000) 2024-03-06T18:31:23.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:23.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:23.723607+0000) 2024-03-06T18:31:24.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:24.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:24.723802+0000) 2024-03-06T18:31:25.718 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:25.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:25.724021+0000) 2024-03-06T18:31:26.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:26.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:26.724252+0000) 2024-03-06T18:31:27.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:27.722+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:27.724482+0000) 2024-03-06T18:31:28.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:28.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:28.724720+0000) 2024-03-06T18:31:29.719 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:29.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:29.724915+0000) 2024-03-06T18:31:30.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:30.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:30.725098+0000) 2024-03-06T18:31:31.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:31.723+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:31.725307+0000) 2024-03-06T18:31:32.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:32.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:32.725523+0000) 2024-03-06T18:31:33.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:33.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:33.725787+0000) 2024-03-06T18:31:34.720 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:34.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:34.726013+0000) 2024-03-06T18:31:35.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:35.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:35.726202+0000) 2024-03-06T18:31:36.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:36.724+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:36.726478+0000) 2024-03-06T18:31:37.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:37.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:37.726722+0000) 2024-03-06T18:31:38.721 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:38.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:38.726911+0000) 2024-03-06T18:31:39.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:39.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:39.727178+0000) 2024-03-06T18:31:40.562 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:31:40.570 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:31:40.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:40.725+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:40.727462+0000) 2024-03-06T18:31:41.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:41.726+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:41.727687+0000) 2024-03-06T18:31:42.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:42.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:42.727879+0000) 2024-03-06T18:31:43.722 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:43.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:43.728111+0000) 2024-03-06T18:31:44.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:44.727+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:44.728335+0000) 2024-03-06T18:31:45.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:45.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:45.728626+0000) 2024-03-06T18:31:46.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:46.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:46.728911+0000) 2024-03-06T18:31:47.723 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:47.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:47.729171+0000) 2024-03-06T18:31:48.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:48.728+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:48.729366+0000) 2024-03-06T18:31:49.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:49.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:49.729586+0000) 2024-03-06T18:31:50.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:50.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:50.729787+0000) 2024-03-06T18:31:51.724 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:51.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:51.730006+0000) 2024-03-06T18:31:52.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:52.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:52.730226+0000) 2024-03-06T18:31:53.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:53.729+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:53.730422+0000) 2024-03-06T18:31:54.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:54.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:54.730649+0000) 2024-03-06T18:31:55.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:55.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:55.730829+0000) 2024-03-06T18:31:56.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:56.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:56.731016+0000) 2024-03-06T18:31:57.725 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:57.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:57.731203+0000) 2024-03-06T18:31:58.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:58.730+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:58.731391+0000) 2024-03-06T18:31:59.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:31:59.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:31:59.731613+0000) 2024-03-06T18:32:00.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:00.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:00.731794+0000) 2024-03-06T18:32:01.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:01.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:01.731939+0000) 2024-03-06T18:32:02.726 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:02.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:02.732127+0000) 2024-03-06T18:32:03.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:03.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:03.732299+0000) 2024-03-06T18:32:04.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:04.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:04.732514+0000) 2024-03-06T18:32:05.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:05.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:05.732781+0000) 2024-03-06T18:32:06.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:06.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:06.733007+0000) 2024-03-06T18:32:07.727 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:07.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:07.733181+0000) 2024-03-06T18:32:08.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:08.731+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:08.733370+0000) 2024-03-06T18:32:09.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:09.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:09.733646+0000) 2024-03-06T18:32:10.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:10.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:10.733862+0000) 2024-03-06T18:32:10.777 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:32:10.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:32:11.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:11.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:11.734025+0000) 2024-03-06T18:32:12.728 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:12.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:12.734229+0000) 2024-03-06T18:32:13.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:13.732+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:13.734430+0000) 2024-03-06T18:32:14.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:14.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:14.734665+0000) 2024-03-06T18:32:15.729 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:15.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:15.734857+0000) 2024-03-06T18:32:16.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:16.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:16.735085+0000) 2024-03-06T18:32:17.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:17.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:17.735307+0000) 2024-03-06T18:32:18.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:18.733+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:18.735507+0000) 2024-03-06T18:32:19.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:19.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:19.735737+0000) 2024-03-06T18:32:20.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:20.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:20.735927+0000) 2024-03-06T18:32:21.730 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:21.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:21.736131+0000) 2024-03-06T18:32:22.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:22.734+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:22.736342+0000) 2024-03-06T18:32:23.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:23.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:23.736602+0000) 2024-03-06T18:32:24.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:24.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:24.736809+0000) 2024-03-06T18:32:25.731 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:25.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:25.737018+0000) 2024-03-06T18:32:26.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:26.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:26.737226+0000) 2024-03-06T18:32:27.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:27.735+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:27.737428+0000) 2024-03-06T18:32:28.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:28.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:28.737678+0000) 2024-03-06T18:32:29.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:29.736+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:29.737892+0000) 2024-03-06T18:32:30.732 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:30.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:30.738087+0000) 2024-03-06T18:32:31.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:31.737+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:31.738303+0000) 2024-03-06T18:32:32.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:32.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:32.738565+0000) 2024-03-06T18:32:33.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:33.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:33.738785+0000) 2024-03-06T18:32:34.733 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:34.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:34.739017+0000) 2024-03-06T18:32:35.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:35.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:35.739258+0000) 2024-03-06T18:32:36.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:36.738+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:36.739442+0000) 2024-03-06T18:32:37.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:37.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:37.739706+0000) 2024-03-06T18:32:38.734 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:38.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:38.739917+0000) 2024-03-06T18:32:39.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:39.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:39.740122+0000) 2024-03-06T18:32:40.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:40.739+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:40.740399+0000) 2024-03-06T18:32:41.001 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:32:41.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:32:41.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:41.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:41.740680+0000) 2024-03-06T18:32:42.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:42.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:42.740868+0000) 2024-03-06T18:32:43.735 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:43.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:43.741101+0000) 2024-03-06T18:32:44.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:44.740+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:44.741325+0000) 2024-03-06T18:32:45.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:45.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:45.741599+0000) 2024-03-06T18:32:46.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:46.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:46.741794+0000) 2024-03-06T18:32:47.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:47.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:47.742018+0000) 2024-03-06T18:32:48.736 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:48.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:48.742246+0000) 2024-03-06T18:32:49.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:49.741+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:49.742447+0000) 2024-03-06T18:32:50.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:50.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:50.742682+0000) 2024-03-06T18:32:51.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:51.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:51.742938+0000) 2024-03-06T18:32:52.737 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:52.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:52.743161+0000) 2024-03-06T18:32:53.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:53.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:53.743390+0000) 2024-03-06T18:32:54.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:54.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:54.743643+0000) 2024-03-06T18:32:55.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:55.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:55.743845+0000) 2024-03-06T18:32:56.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:56.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:56.744098+0000) 2024-03-06T18:32:57.738 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:57.742+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:57.744298+0000) 2024-03-06T18:32:58.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:58.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:58.744575+0000) 2024-03-06T18:32:59.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:32:59.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:32:59.744806+0000) 2024-03-06T18:33:00.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:00.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:00.745034+0000) 2024-03-06T18:33:01.739 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:01.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:01.745250+0000) 2024-03-06T18:33:02.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:02.743+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:02.745452+0000) 2024-03-06T18:33:03.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:03.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:03.745687+0000) 2024-03-06T18:33:04.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:04.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:04.745876+0000) 2024-03-06T18:33:05.740 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:05.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:05.746111+0000) 2024-03-06T18:33:06.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:06.744+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:06.746342+0000) 2024-03-06T18:33:07.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:07.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:07.746581+0000) 2024-03-06T18:33:08.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:08.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:08.746785+0000) 2024-03-06T18:33:09.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:09.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:09.746973+0000) 2024-03-06T18:33:10.741 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:10.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:10.747207+0000) 2024-03-06T18:33:11.337 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:33:11.344 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:33:11.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:11.745+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:11.747449+0000) 2024-03-06T18:33:12.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:12.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:12.747681+0000) 2024-03-06T18:33:13.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:13.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:13.747836+0000) 2024-03-06T18:33:14.742 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:14.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:14.748062+0000) 2024-03-06T18:33:15.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:15.746+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:15.748307+0000) 2024-03-06T18:33:16.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:16.747+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:16.748576+0000) 2024-03-06T18:33:17.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:17.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:17.748777+0000) 2024-03-06T18:33:18.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:18.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:18.748965+0000) 2024-03-06T18:33:19.743 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:19.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:19.749186+0000) 2024-03-06T18:33:20.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:20.748+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:20.749400+0000) 2024-03-06T18:33:21.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:21.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:21.749662+0000) 2024-03-06T18:33:22.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:22.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:22.749862+0000) 2024-03-06T18:33:23.744 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:23.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:23.750089+0000) 2024-03-06T18:33:24.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:24.749+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:24.750314+0000) 2024-03-06T18:33:25.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:25.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:25.750591+0000) 2024-03-06T18:33:26.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:26.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:26.750807+0000) 2024-03-06T18:33:27.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:27.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:27.751026+0000) 2024-03-06T18:33:28.745 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:28.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:28.751252+0000) 2024-03-06T18:33:29.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:29.750+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:29.751513+0000) 2024-03-06T18:33:30.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:30.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:30.751735+0000) 2024-03-06T18:33:31.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:31.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:31.751947+0000) 2024-03-06T18:33:32.746 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:32.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:32.752173+0000) 2024-03-06T18:33:33.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:33.751+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:33.752402+0000) 2024-03-06T18:33:34.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:34.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:34.752684+0000) 2024-03-06T18:33:35.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:35.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:35.752882+0000) 2024-03-06T18:33:36.747 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:36.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:36.753114+0000) 2024-03-06T18:33:37.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:37.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:37.753289+0000) 2024-03-06T18:33:38.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:38.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:38.753464+0000) 2024-03-06T18:33:39.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:39.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:39.753652+0000) 2024-03-06T18:33:40.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:40.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:40.753843+0000) 2024-03-06T18:33:41.552 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:33:41.560 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:33:41.748 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:41.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:41.754089+0000) 2024-03-06T18:33:42.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:42.752+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:42.754318+0000) 2024-03-06T18:33:43.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:43.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:43.754576+0000) 2024-03-06T18:33:44.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:44.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:44.754806+0000) 2024-03-06T18:33:45.749 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:45.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:45.755037+0000) 2024-03-06T18:33:46.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:46.753+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:46.755296+0000) 2024-03-06T18:33:47.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:47.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:47.755570+0000) 2024-03-06T18:33:48.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:48.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:48.755822+0000) 2024-03-06T18:33:49.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:49.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:49.756065+0000) 2024-03-06T18:33:50.750 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:50.754+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:50.756269+0000) 2024-03-06T18:33:51.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:51.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:51.756525+0000) 2024-03-06T18:33:52.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:52.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:52.756796+0000) 2024-03-06T18:33:53.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:53.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:53.757026+0000) 2024-03-06T18:33:54.751 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:54.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:54.757262+0000) 2024-03-06T18:33:55.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:55.755+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:55.757509+0000) 2024-03-06T18:33:56.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:56.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:56.757797+0000) 2024-03-06T18:33:57.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:57.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:57.758040+0000) 2024-03-06T18:33:58.752 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:58.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:58.758281+0000) 2024-03-06T18:33:59.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:33:59.756+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:33:59.758488+0000) 2024-03-06T18:34:00.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:00.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:00.758739+0000) 2024-03-06T18:34:01.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:01.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:01.759012+0000) 2024-03-06T18:34:02.753 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:02.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:02.759211+0000) 2024-03-06T18:34:03.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:03.757+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:03.759376+0000) 2024-03-06T18:34:04.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:04.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:04.759692+0000) 2024-03-06T18:34:05.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:05.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:05.759889+0000) 2024-03-06T18:34:06.754 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:06.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:06.760132+0000) 2024-03-06T18:34:07.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:07.759+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:07.760363+0000) 2024-03-06T18:34:08.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:08.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:08.760635+0000) 2024-03-06T18:34:09.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:09.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:09.760873+0000) 2024-03-06T18:34:10.755 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:10.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:10.761114+0000) 2024-03-06T18:34:11.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:11.760+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:11.761357+0000) 2024-03-06T18:34:11.775 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:34:11.785 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:34:12.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:12.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:12.761596+0000) 2024-03-06T18:34:13.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:13.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:13.761836+0000) 2024-03-06T18:34:14.756 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:14.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:14.762059+0000) 2024-03-06T18:34:15.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:15.761+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:15.762288+0000) 2024-03-06T18:34:16.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:16.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:16.762525+0000) 2024-03-06T18:34:17.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:17.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:17.762785+0000) 2024-03-06T18:34:18.757 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:18.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:18.762980+0000) 2024-03-06T18:34:19.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:19.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:19.763224+0000) 2024-03-06T18:34:20.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:20.762+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:20.763465+0000) 2024-03-06T18:34:21.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:21.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:21.763750+0000) 2024-03-06T18:34:22.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:22.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:22.764019+0000) 2024-03-06T18:34:23.758 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:23.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:23.764262+0000) 2024-03-06T18:34:24.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:24.763+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:24.764483+0000) 2024-03-06T18:34:25.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:25.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:25.764709+0000) 2024-03-06T18:34:26.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:26.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:26.764939+0000) 2024-03-06T18:34:27.759 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:27.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:27.765158+0000) 2024-03-06T18:34:28.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:28.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:28.765382+0000) 2024-03-06T18:34:29.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:29.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:29.765656+0000) 2024-03-06T18:34:30.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:30.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:30.765833+0000) 2024-03-06T18:34:31.760 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:31.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:31.766079+0000) 2024-03-06T18:34:32.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:32.764+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:32.766300+0000) 2024-03-06T18:34:33.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:33.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:33.766527+0000) 2024-03-06T18:34:34.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:34.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:34.766788+0000) 2024-03-06T18:34:35.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:35.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:35.766975+0000) 2024-03-06T18:34:36.761 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:36.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:36.767227+0000) 2024-03-06T18:34:37.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:37.765+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:37.767444+0000) 2024-03-06T18:34:38.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:38.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:38.767650+0000) 2024-03-06T18:34:39.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:39.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:39.767818+0000) 2024-03-06T18:34:40.762 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:40.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:40.768055+0000) 2024-03-06T18:34:41.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:41.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:41.768295+0000) 2024-03-06T18:34:41.969 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:34:41.978 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:34:42.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:42.766+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:42.768491+0000) 2024-03-06T18:34:43.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:43.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:43.768711+0000) 2024-03-06T18:34:44.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:44.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:44.768923+0000) 2024-03-06T18:34:45.763 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:45.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:45.769126+0000) 2024-03-06T18:34:46.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:46.767+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:46.769358+0000) 2024-03-06T18:34:47.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:47.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:47.769623+0000) 2024-03-06T18:34:48.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:48.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:48.769847+0000) 2024-03-06T18:34:49.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:49.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:49.770072+0000) 2024-03-06T18:34:50.764 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:50.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:50.770266+0000) 2024-03-06T18:34:51.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:51.768+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:51.770501+0000) 2024-03-06T18:34:52.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:52.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:52.770752+0000) 2024-03-06T18:34:53.765 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:53.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:53.770990+0000) 2024-03-06T18:34:54.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:54.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:54.771165+0000) 2024-03-06T18:34:55.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:55.770+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:55.771388+0000) 2024-03-06T18:34:56.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:56.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:56.771657+0000) 2024-03-06T18:34:57.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:57.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:57.771849+0000) 2024-03-06T18:34:58.766 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:58.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:58.772070+0000) 2024-03-06T18:34:59.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:34:59.771+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:34:59.772310+0000) 2024-03-06T18:35:00.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:00.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:00.772588+0000) 2024-03-06T18:35:01.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:01.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:01.772831+0000) 2024-03-06T18:35:02.767 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:02.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:02.773070+0000) 2024-03-06T18:35:03.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:03.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:03.773305+0000) 2024-03-06T18:35:04.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:04.772+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:04.773500+0000) 2024-03-06T18:35:05.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:05.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:05.773763+0000) 2024-03-06T18:35:06.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:06.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:06.773997+0000) 2024-03-06T18:35:07.768 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:07.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:07.774217+0000) 2024-03-06T18:35:08.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:08.773+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:08.774452+0000) 2024-03-06T18:35:09.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:09.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:09.774686+0000) 2024-03-06T18:35:10.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:10.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:10.774944+0000) 2024-03-06T18:35:11.769 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:11.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:11.775193+0000) 2024-03-06T18:35:12.190 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:35:12.201 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:35:12.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:12.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:12.775375+0000) 2024-03-06T18:35:13.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:13.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:13.775637+0000) 2024-03-06T18:35:14.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:14.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:14.775848+0000) 2024-03-06T18:35:15.770 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:15.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:15.776087+0000) 2024-03-06T18:35:16.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:16.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:16.776293+0000) 2024-03-06T18:35:17.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:17.774+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:17.776511+0000) 2024-03-06T18:35:18.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:18.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:18.776759+0000) 2024-03-06T18:35:19.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:19.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:19.776940+0000) 2024-03-06T18:35:20.771 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:20.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:20.777175+0000) 2024-03-06T18:35:21.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:21.775+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:21.777416+0000) 2024-03-06T18:35:22.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:22.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:22.777661+0000) 2024-03-06T18:35:23.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:23.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:23.777858+0000) 2024-03-06T18:35:24.772 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:24.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:24.778080+0000) 2024-03-06T18:35:25.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:25.776+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:25.778311+0000) 2024-03-06T18:35:26.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:26.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:26.778586+0000) 2024-03-06T18:35:27.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:27.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:27.778792+0000) 2024-03-06T18:35:28.773 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:28.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:28.779011+0000) 2024-03-06T18:35:29.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:29.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:29.779286+0000) 2024-03-06T18:35:30.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:30.777+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:30.779516+0000) 2024-03-06T18:35:31.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:31.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:31.779773+0000) 2024-03-06T18:35:32.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:32.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:32.779984+0000) 2024-03-06T18:35:33.774 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:33.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:33.780227+0000) 2024-03-06T18:35:34.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:34.778+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:34.780460+0000) 2024-03-06T18:35:35.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:35.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:35.780684+0000) 2024-03-06T18:35:36.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:36.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:36.780881+0000) 2024-03-06T18:35:37.775 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:37.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:37.781110+0000) 2024-03-06T18:35:38.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:38.779+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:38.781290+0000) 2024-03-06T18:35:39.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:39.780+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:39.781462+0000) 2024-03-06T18:35:40.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:40.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:40.781694+0000) 2024-03-06T18:35:41.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:41.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:41.781902+0000) 2024-03-06T18:35:42.396 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:35:42.404 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:35:42.776 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:42.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:42.782131+0000) 2024-03-06T18:35:43.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:43.781+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:43.782358+0000) 2024-03-06T18:35:44.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:44.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:44.782627+0000) 2024-03-06T18:35:45.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:45.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:45.782817+0000) 2024-03-06T18:35:46.777 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:46.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:46.783032+0000) 2024-03-06T18:35:47.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:47.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:47.783255+0000) 2024-03-06T18:35:48.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:48.782+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:48.783491+0000) 2024-03-06T18:35:49.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:49.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:49.783717+0000) 2024-03-06T18:35:50.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:50.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:50.783945+0000) 2024-03-06T18:35:51.778 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:51.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:51.784184+0000) 2024-03-06T18:35:52.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:52.783+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:52.784415+0000) 2024-03-06T18:35:53.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:53.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:53.784647+0000) 2024-03-06T18:35:54.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:54.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:54.784876+0000) 2024-03-06T18:35:55.779 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:55.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:55.785102+0000) 2024-03-06T18:35:56.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:56.784+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:56.785329+0000) 2024-03-06T18:35:57.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:57.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:57.785577+0000) 2024-03-06T18:35:58.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:58.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:58.785873+0000) 2024-03-06T18:35:59.780 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:35:59.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:35:59.786141+0000) 2024-03-06T18:36:00.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:00.785+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:00.786412+0000) 2024-03-06T18:36:01.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:01.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:01.786734+0000) 2024-03-06T18:36:02.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:02.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:02.786979+0000) 2024-03-06T18:36:03.781 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:03.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:03.787174+0000) 2024-03-06T18:36:04.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:04.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:04.787349+0000) 2024-03-06T18:36:05.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:05.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:05.787589+0000) 2024-03-06T18:36:06.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:06.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:06.787801+0000) 2024-03-06T18:36:07.782 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:07.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:07.788026+0000) 2024-03-06T18:36:08.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:08.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:08.788248+0000) 2024-03-06T18:36:09.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:09.786+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:09.788475+0000) 2024-03-06T18:36:10.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:10.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:10.788689+0000) 2024-03-06T18:36:11.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:11.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:11.788925+0000) 2024-03-06T18:36:12.625 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:36:12.633 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:36:12.783 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:12.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:12.789082+0000) 2024-03-06T18:36:13.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:13.787+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:13.789295+0000) 2024-03-06T18:36:14.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:14.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:14.789527+0000) 2024-03-06T18:36:15.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:15.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:15.789784+0000) 2024-03-06T18:36:16.784 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:16.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:16.790018+0000) 2024-03-06T18:36:17.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:17.788+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:17.790287+0000) 2024-03-06T18:36:18.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:18.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:18.790610+0000) 2024-03-06T18:36:19.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:19.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:19.790883+0000) 2024-03-06T18:36:20.785 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:20.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:20.791168+0000) 2024-03-06T18:36:21.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:21.789+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:21.791469+0000) 2024-03-06T18:36:22.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:22.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:22.791756+0000) 2024-03-06T18:36:23.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:23.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:23.791996+0000) 2024-03-06T18:36:24.786 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:24.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:24.792183+0000) 2024-03-06T18:36:25.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:25.790+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:25.792405+0000) 2024-03-06T18:36:26.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:26.791+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:26.792678+0000) 2024-03-06T18:36:27.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:27.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:27.792878+0000) 2024-03-06T18:36:28.787 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:28.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:28.793079+0000) 2024-03-06T18:36:29.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:29.792+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:29.793316+0000) 2024-03-06T18:36:30.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:30.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:30.793593+0000) 2024-03-06T18:36:31.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:31.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:31.793800+0000) 2024-03-06T18:36:32.788 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:32.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:32.794029+0000) 2024-03-06T18:36:33.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:33.793+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:33.794309+0000) 2024-03-06T18:36:34.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:34.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:34.794565+0000) 2024-03-06T18:36:35.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:35.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:35.794728+0000) 2024-03-06T18:36:36.789 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:36.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:36.794951+0000) 2024-03-06T18:36:37.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:37.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:37.795186+0000) 2024-03-06T18:36:38.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:38.794+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:38.795423+0000) 2024-03-06T18:36:39.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:39.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:39.795662+0000) 2024-03-06T18:36:40.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:40.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:40.795858+0000) 2024-03-06T18:36:41.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:41.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:41.796094+0000) 2024-03-06T18:36:42.760 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:36:42.768 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:36:42.790 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:42.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:42.796264+0000) 2024-03-06T18:36:43.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:43.795+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:43.796481+0000) 2024-03-06T18:36:44.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:44.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:44.796691+0000) 2024-03-06T18:36:45.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:45.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:45.796891+0000) 2024-03-06T18:36:46.791 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:46.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:46.797170+0000) 2024-03-06T18:36:47.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:47.796+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:47.797397+0000) 2024-03-06T18:36:48.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:48.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:48.797660+0000) 2024-03-06T18:36:49.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:49.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:49.797855+0000) 2024-03-06T18:36:50.792 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:50.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:50.798066+0000) 2024-03-06T18:36:51.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:51.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:51.798314+0000) 2024-03-06T18:36:52.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:52.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:52.798581+0000) 2024-03-06T18:36:53.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:53.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:53.798795+0000) 2024-03-06T18:36:54.793 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:54.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:54.799021+0000) 2024-03-06T18:36:55.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:55.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:55.799252+0000) 2024-03-06T18:36:56.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:56.797+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:56.799479+0000) 2024-03-06T18:36:57.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:57.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:57.799704+0000) 2024-03-06T18:36:58.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:58.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:58.799904+0000) 2024-03-06T18:36:59.794 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:36:59.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:36:59.800127+0000) 2024-03-06T18:37:00.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:00.798+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:00.800320+0000) 2024-03-06T18:37:01.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:01.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:01.800620+0000) 2024-03-06T18:37:02.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:02.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:02.800854+0000) 2024-03-06T18:37:03.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:03.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:03.801063+0000) 2024-03-06T18:37:04.795 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:04.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:04.801273+0000) 2024-03-06T18:37:05.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:05.799+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:05.801514+0000) 2024-03-06T18:37:06.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:06.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:06.801769+0000) 2024-03-06T18:37:07.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:07.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:07.801975+0000) 2024-03-06T18:37:08.796 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:08.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:08.802197+0000) 2024-03-06T18:37:09.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:09.800+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:09.802426+0000) 2024-03-06T18:37:10.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:10.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:10.802654+0000) 2024-03-06T18:37:11.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:11.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:11.802900+0000) 2024-03-06T18:37:12.797 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:12.801+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:12.803128+0000) 2024-03-06T18:37:12.952 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:37:12.959 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:37:13.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:13.802+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:13.803366+0000) 2024-03-06T18:37:14.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:14.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:14.803653+0000) 2024-03-06T18:37:15.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:15.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:15.803887+0000) 2024-03-06T18:37:16.798 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:16.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:16.804125+0000) 2024-03-06T18:37:17.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:17.803+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:17.804331+0000) 2024-03-06T18:37:18.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:18.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:18.804632+0000) 2024-03-06T18:37:19.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:19.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:19.804845+0000) 2024-03-06T18:37:20.799 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:20.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:20.805068+0000) 2024-03-06T18:37:21.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:21.804+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:21.805302+0000) 2024-03-06T18:37:22.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:22.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:22.805529+0000) 2024-03-06T18:37:23.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:23.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:23.805794+0000) 2024-03-06T18:37:24.800 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:24.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:24.805975+0000) 2024-03-06T18:37:25.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:25.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:25.806214+0000) 2024-03-06T18:37:26.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:26.805+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:26.806457+0000) 2024-03-06T18:37:27.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:27.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:27.806697+0000) 2024-03-06T18:37:28.801 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:28.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:28.806903+0000) 2024-03-06T18:37:29.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:29.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:29.807196+0000) 2024-03-06T18:37:30.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:30.806+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:30.807439+0000) 2024-03-06T18:37:31.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:31.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:31.807686+0000) 2024-03-06T18:37:32.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:32.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:32.807882+0000) 2024-03-06T18:37:33.802 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:33.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:33.808091+0000) 2024-03-06T18:37:34.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:34.807+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:34.808330+0000) 2024-03-06T18:37:35.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:35.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:35.808621+0000) 2024-03-06T18:37:36.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:36.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:36.808849+0000) 2024-03-06T18:37:37.803 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:37.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:37.809082+0000) 2024-03-06T18:37:38.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:38.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:38.809281+0000) 2024-03-06T18:37:39.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:39.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:39.809496+0000) 2024-03-06T18:37:40.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:40.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:40.809729+0000) 2024-03-06T18:37:41.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:41.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:41.809937+0000) 2024-03-06T18:37:42.804 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:42.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:42.810135+0000) 2024-03-06T18:37:43.173 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:37:43.183 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:37:43.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:43.808+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:43.810378+0000) 2024-03-06T18:37:44.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:44.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:44.810659+0000) 2024-03-06T18:37:45.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:45.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:45.810854+0000) 2024-03-06T18:37:46.805 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:46.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:46.811039+0000) 2024-03-06T18:37:47.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:47.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:47.811281+0000) 2024-03-06T18:37:48.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:48.809+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:48.811504+0000) 2024-03-06T18:37:49.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:49.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:49.811678+0000) 2024-03-06T18:37:50.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:50.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:50.811870+0000) 2024-03-06T18:37:51.806 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:51.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:51.812100+0000) 2024-03-06T18:37:52.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:52.810+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:52.812326+0000) 2024-03-06T18:37:53.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:53.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:53.812593+0000) 2024-03-06T18:37:54.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:54.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:54.812781+0000) 2024-03-06T18:37:55.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:55.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:55.813010+0000) 2024-03-06T18:37:56.807 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:56.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:56.813198+0000) 2024-03-06T18:37:57.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:57.811+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:57.813473+0000) 2024-03-06T18:37:58.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:58.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:58.813750+0000) 2024-03-06T18:37:59.808 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:37:59.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:37:59.814018+0000) 2024-03-06T18:38:00.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:00.812+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:00.814293+0000) 2024-03-06T18:38:01.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:01.813+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:01.814629+0000) 2024-03-06T18:38:02.809 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:02.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:02.814911+0000) 2024-03-06T18:38:03.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:03.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:03.815187+0000) 2024-03-06T18:38:04.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:04.814+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:04.815465+0000) 2024-03-06T18:38:05.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:05.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:05.815746+0000) 2024-03-06T18:38:06.810 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:06.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:06.816011+0000) 2024-03-06T18:38:07.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:07.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:07.816286+0000) 2024-03-06T18:38:08.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:08.815+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:08.816493+0000) 2024-03-06T18:38:09.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:09.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:09.816733+0000) 2024-03-06T18:38:10.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:10.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:10.816958+0000) 2024-03-06T18:38:11.811 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:11.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:11.817194+0000) 2024-03-06T18:38:12.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:12.816+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:12.817378+0000) 2024-03-06T18:38:13.394 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:38:13.403 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:38:13.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:13.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:13.817652+0000) 2024-03-06T18:38:14.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:14.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:14.817831+0000) 2024-03-06T18:38:15.812 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:15.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:15.818009+0000) 2024-03-06T18:38:16.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:16.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:16.818236+0000) 2024-03-06T18:38:17.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:17.817+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:17.818452+0000) 2024-03-06T18:38:18.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:18.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:18.818687+0000) 2024-03-06T18:38:19.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:19.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:19.818875+0000) 2024-03-06T18:38:20.813 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:20.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:20.819117+0000) 2024-03-06T18:38:21.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:21.818+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:21.819313+0000) 2024-03-06T18:38:22.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:22.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:22.819575+0000) 2024-03-06T18:38:23.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:23.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:23.819801+0000) 2024-03-06T18:38:24.814 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:24.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:24.820005+0000) 2024-03-06T18:38:25.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:25.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:25.820233+0000) 2024-03-06T18:38:26.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:26.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:26.820440+0000) 2024-03-06T18:38:27.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:27.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:27.820745+0000) 2024-03-06T18:38:28.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:28.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:28.820978+0000) 2024-03-06T18:38:29.815 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:29.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:29.821201+0000) 2024-03-06T18:38:30.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:30.819+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:30.821384+0000) 2024-03-06T18:38:31.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:31.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:31.821625+0000) 2024-03-06T18:38:32.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:32.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:32.821788+0000) 2024-03-06T18:38:33.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:33.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:33.822014+0000) 2024-03-06T18:38:34.816 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:34.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:34.822214+0000) 2024-03-06T18:38:35.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:35.820+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:35.822451+0000) 2024-03-06T18:38:36.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:36.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:36.822673+0000) 2024-03-06T18:38:37.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:37.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:37.822870+0000) 2024-03-06T18:38:38.817 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:38.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:38.823092+0000) 2024-03-06T18:38:39.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:39.821+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:39.823285+0000) 2024-03-06T18:38:40.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:40.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:40.823524+0000) 2024-03-06T18:38:41.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:41.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:41.823719+0000) 2024-03-06T18:38:42.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:42.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:42.823915+0000) 2024-03-06T18:38:43.592 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:38:43.600 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:38:43.818 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:43.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:43.824158+0000) 2024-03-06T18:38:44.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:44.822+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:44.824385+0000) 2024-03-06T18:38:45.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:45.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:45.824649+0000) 2024-03-06T18:38:46.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:46.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:46.824842+0000) 2024-03-06T18:38:47.819 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:47.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:47.825081+0000) 2024-03-06T18:38:48.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:48.823+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:48.825309+0000) 2024-03-06T18:38:49.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:49.824+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:49.825517+0000) 2024-03-06T18:38:50.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:50.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:50.825739+0000) 2024-03-06T18:38:51.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:51.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:51.825903+0000) 2024-03-06T18:38:52.820 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:52.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:52.826142+0000) 2024-03-06T18:38:53.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:53.825+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:53.826405+0000) 2024-03-06T18:38:54.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:54.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:54.826695+0000) 2024-03-06T18:38:55.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:55.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:55.826891+0000) 2024-03-06T18:38:56.821 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:56.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:56.827125+0000) 2024-03-06T18:38:57.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:57.826+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:57.827330+0000) 2024-03-06T18:38:58.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:58.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:58.827625+0000) 2024-03-06T18:38:59.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:38:59.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:38:59.827821+0000) 2024-03-06T18:39:00.822 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:00.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:00.828049+0000) 2024-03-06T18:39:01.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:01.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:01.828280+0000) 2024-03-06T18:39:02.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:02.827+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:02.828510+0000) 2024-03-06T18:39:03.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:03.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:03.828752+0000) 2024-03-06T18:39:04.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:04.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:04.828930+0000) 2024-03-06T18:39:05.823 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:05.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:05.829157+0000) 2024-03-06T18:39:06.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:06.828+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:06.829381+0000) 2024-03-06T18:39:07.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:07.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:07.829636+0000) 2024-03-06T18:39:08.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:08.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:08.829851+0000) 2024-03-06T18:39:09.824 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:09.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:09.830088+0000) 2024-03-06T18:39:10.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:10.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:10.830284+0000) 2024-03-06T18:39:11.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:11.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:11.830577+0000) 2024-03-06T18:39:12.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:12.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:12.830750+0000) 2024-03-06T18:39:13.804 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:39:13.812 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:39:13.825 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:13.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:13.830906+0000) 2024-03-06T18:39:14.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:14.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:14.831133+0000) 2024-03-06T18:39:15.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:15.829+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:15.831358+0000) 2024-03-06T18:39:16.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:16.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:16.831590+0000) 2024-03-06T18:39:17.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:17.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:17.831857+0000) 2024-03-06T18:39:18.826 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:18.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:18.832084+0000) 2024-03-06T18:39:19.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:19.830+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:19.832281+0000) 2024-03-06T18:39:20.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:20.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:20.832524+0000) 2024-03-06T18:39:21.827 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:21.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:21.832855+0000) 2024-03-06T18:39:22.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:22.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:22.833059+0000) 2024-03-06T18:39:23.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:23.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:23.833260+0000) 2024-03-06T18:39:24.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:24.831+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:24.833471+0000) 2024-03-06T18:39:25.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:25.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:25.833710+0000) 2024-03-06T18:39:26.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:26.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:26.833895+0000) 2024-03-06T18:39:27.828 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:27.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:27.834138+0000) 2024-03-06T18:39:28.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:28.832+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:28.834357+0000) 2024-03-06T18:39:29.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:29.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:29.834628+0000) 2024-03-06T18:39:30.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:30.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:30.834816+0000) 2024-03-06T18:39:31.829 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:31.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:31.835014+0000) 2024-03-06T18:39:32.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:32.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:32.835208+0000) 2024-03-06T18:39:33.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:33.833+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:33.835373+0000) 2024-03-06T18:39:34.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:34.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:34.835525+0000) 2024-03-06T18:39:35.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:35.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:35.835775+0000) 2024-03-06T18:39:36.830 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:36.834+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:36.835976+0000) 2024-03-06T18:39:37.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:37.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:37.836227+0000) 2024-03-06T18:39:38.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:38.835+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:38.836463+0000) 2024-03-06T18:39:39.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:39.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:39.836686+0000) 2024-03-06T18:39:40.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:40.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:40.836906+0000) 2024-03-06T18:39:41.831 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:41.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:41.837173+0000) 2024-03-06T18:39:42.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:42.836+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:42.837370+0000) 2024-03-06T18:39:43.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:43.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:43.837644+0000) 2024-03-06T18:39:44.008 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:39:44.017 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:39:44.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:44.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:44.837847+0000) 2024-03-06T18:39:45.832 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:45.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:45.838056+0000) 2024-03-06T18:39:46.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:46.837+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:46.838297+0000) 2024-03-06T18:39:47.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:47.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:47.838578+0000) 2024-03-06T18:39:48.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:48.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:48.838801+0000) 2024-03-06T18:39:49.833 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:49.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:49.839000+0000) 2024-03-06T18:39:50.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:50.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:50.839185+0000) 2024-03-06T18:39:51.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:51.838+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:51.839437+0000) 2024-03-06T18:39:52.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:52.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:52.839709+0000) 2024-03-06T18:39:53.834 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:53.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:53.839979+0000) 2024-03-06T18:39:54.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:54.839+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:54.840262+0000) 2024-03-06T18:39:55.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:55.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:55.840575+0000) 2024-03-06T18:39:56.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:56.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:56.840842+0000) 2024-03-06T18:39:57.835 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:57.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:57.841064+0000) 2024-03-06T18:39:58.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:58.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:58.841297+0000) 2024-03-06T18:39:59.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:39:59.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:39:59.841525+0000) 2024-03-06T18:40:00.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:00.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:00.841798+0000) 2024-03-06T18:40:01.836 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:01.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:01.842031+0000) 2024-03-06T18:40:02.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:02.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:02.842224+0000) 2024-03-06T18:40:03.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:03.840+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:03.842455+0000) 2024-03-06T18:40:04.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:04.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:04.842731+0000) 2024-03-06T18:40:05.837 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:05.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:05.843019+0000) 2024-03-06T18:40:06.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:06.841+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:06.843316+0000) 2024-03-06T18:40:07.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:07.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:07.843646+0000) 2024-03-06T18:40:08.838 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:08.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:08.843930+0000) 2024-03-06T18:40:09.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:09.842+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:09.844255+0000) 2024-03-06T18:40:10.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:10.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:10.844571+0000) 2024-03-06T18:40:11.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:11.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:11.844842+0000) 2024-03-06T18:40:12.839 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:12.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:12.845077+0000) 2024-03-06T18:40:13.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:13.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:13.845261+0000) 2024-03-06T18:40:14.222 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:40:14.231 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:40:14.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:14.843+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:14.845503+0000) 2024-03-06T18:40:15.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:15.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:15.845779+0000) 2024-03-06T18:40:16.840 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:16.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:16.846018+0000) 2024-03-06T18:40:17.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:17.844+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:17.846294+0000) 2024-03-06T18:40:18.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:18.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:18.846523+0000) 2024-03-06T18:40:19.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:19.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:19.846841+0000) 2024-03-06T18:40:20.841 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:20.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:20.847076+0000) 2024-03-06T18:40:21.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:21.845+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:21.847293+0000) 2024-03-06T18:40:22.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:22.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:22.847621+0000) 2024-03-06T18:40:23.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:23.846+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:23.847891+0000) 2024-03-06T18:40:24.842 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:24.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:24.848140+0000) 2024-03-06T18:40:25.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:25.847+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:25.848418+0000) 2024-03-06T18:40:26.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:26.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:26.848672+0000) 2024-03-06T18:40:27.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:27.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:27.848961+0000) 2024-03-06T18:40:28.843 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:28.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:28.849227+0000) 2024-03-06T18:40:29.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:29.848+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:29.849426+0000) 2024-03-06T18:40:30.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:30.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:30.849676+0000) 2024-03-06T18:40:31.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:31.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:31.849897+0000) 2024-03-06T18:40:32.844 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:32.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:32.850111+0000) 2024-03-06T18:40:33.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:33.849+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:33.850353+0000) 2024-03-06T18:40:34.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:34.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:34.850595+0000) 2024-03-06T18:40:35.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:35.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:35.850857+0000) 2024-03-06T18:40:36.845 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:36.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:36.851121+0000) 2024-03-06T18:40:37.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:37.850+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:37.851405+0000) 2024-03-06T18:40:38.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:38.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:38.851693+0000) 2024-03-06T18:40:39.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:39.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:39.851922+0000) 2024-03-06T18:40:40.846 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:40.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:40.852193+0000) 2024-03-06T18:40:41.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:41.851+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:41.852411+0000) 2024-03-06T18:40:42.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:42.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:42.852696+0000) 2024-03-06T18:40:43.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:43.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:43.852932+0000) 2024-03-06T18:40:44.350 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:40:44.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:40:44.847 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:44.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:44.853137+0000) 2024-03-06T18:40:45.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:45.852+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:45.853323+0000) 2024-03-06T18:40:46.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:46.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:46.853584+0000) 2024-03-06T18:40:47.848 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:47.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:47.853856+0000) 2024-03-06T18:40:48.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:48.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:48.854140+0000) 2024-03-06T18:40:49.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:49.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:49.854420+0000) 2024-03-06T18:40:50.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:50.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:50.854690+0000) 2024-03-06T18:40:51.849 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:51.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:51.854975+0000) 2024-03-06T18:40:52.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:52.853+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:52.855255+0000) 2024-03-06T18:40:53.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:53.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:53.855579+0000) 2024-03-06T18:40:54.850 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:54.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:54.855836+0000) 2024-03-06T18:40:55.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:55.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:55.856098+0000) 2024-03-06T18:40:56.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:56.854+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:56.856373+0000) 2024-03-06T18:40:57.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:57.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:57.856729+0000) 2024-03-06T18:40:58.851 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:58.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:58.857005+0000) 2024-03-06T18:40:59.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:40:59.855+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:40:59.857272+0000) 2024-03-06T18:41:00.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:00.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:00.857589+0000) 2024-03-06T18:41:01.852 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:01.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:01.857871+0000) 2024-03-06T18:41:02.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:02.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:02.858142+0000) 2024-03-06T18:41:03.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:03.856+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:03.858344+0000) 2024-03-06T18:41:04.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:04.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:04.858641+0000) 2024-03-06T18:41:05.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:05.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:05.858882+0000) 2024-03-06T18:41:06.853 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:06.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:06.859125+0000) 2024-03-06T18:41:07.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:07.857+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:07.859376+0000) 2024-03-06T18:41:08.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:08.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:08.859650+0000) 2024-03-06T18:41:09.854 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:09.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:09.859853+0000) 2024-03-06T18:41:10.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:10.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:10.860119+0000) 2024-03-06T18:41:11.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:11.858+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:11.860360+0000) 2024-03-06T18:41:12.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:12.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:12.860614+0000) 2024-03-06T18:41:13.855 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:13.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:13.860856+0000) 2024-03-06T18:41:14.499 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:41:14.507 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:41:14.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:14.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:14.861119+0000) 2024-03-06T18:41:15.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:15.860+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:15.861317+0000) 2024-03-06T18:41:16.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:16.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:16.861588+0000) 2024-03-06T18:41:17.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:17.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:17.861801+0000) 2024-03-06T18:41:18.856 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:18.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:18.862022+0000) 2024-03-06T18:41:19.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:19.861+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:19.862298+0000) 2024-03-06T18:41:20.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:20.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:20.862615+0000) 2024-03-06T18:41:21.857 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:21.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:21.862905+0000) 2024-03-06T18:41:22.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:22.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:22.863191+0000) 2024-03-06T18:41:23.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:23.862+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:23.863418+0000) 2024-03-06T18:41:24.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:24.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:24.863714+0000) 2024-03-06T18:41:25.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:25.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:25.863979+0000) 2024-03-06T18:41:26.858 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:26.863+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:26.864257+0000) 2024-03-06T18:41:27.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:27.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:27.864576+0000) 2024-03-06T18:41:28.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:28.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:28.864832+0000) 2024-03-06T18:41:29.859 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:29.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:29.865085+0000) 2024-03-06T18:41:30.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:30.864+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:30.865324+0000) 2024-03-06T18:41:31.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:31.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:31.865568+0000) 2024-03-06T18:41:32.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:32.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:32.865870+0000) 2024-03-06T18:41:33.860 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:33.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:33.866107+0000) 2024-03-06T18:41:34.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:34.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:34.866337+0000) 2024-03-06T18:41:35.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:35.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:35.866595+0000) 2024-03-06T18:41:36.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:36.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:36.866815+0000) 2024-03-06T18:41:37.861 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:37.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:37.867053+0000) 2024-03-06T18:41:38.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:38.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:38.867247+0000) 2024-03-06T18:41:39.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:39.865+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:39.867492+0000) 2024-03-06T18:41:40.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:40.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:40.867717+0000) 2024-03-06T18:41:41.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:41.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:41.867961+0000) 2024-03-06T18:41:42.862 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:42.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:42.868177+0000) 2024-03-06T18:41:43.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:43.866+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:43.868453+0000) 2024-03-06T18:41:44.712 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:41:44.722 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:41:44.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:44.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:44.868751+0000) 2024-03-06T18:41:45.863 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:45.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:45.868993+0000) 2024-03-06T18:41:46.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:46.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:46.869273+0000) 2024-03-06T18:41:47.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:47.867+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:47.869481+0000) 2024-03-06T18:41:48.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:48.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:48.869722+0000) 2024-03-06T18:41:49.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:49.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:49.869916+0000) 2024-03-06T18:41:50.864 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:50.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:50.870142+0000) 2024-03-06T18:41:51.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:51.868+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:51.870376+0000) 2024-03-06T18:41:52.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:52.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:52.870616+0000) 2024-03-06T18:41:53.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:53.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:53.870860+0000) 2024-03-06T18:41:54.865 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:54.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:54.871093+0000) 2024-03-06T18:41:55.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:55.869+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:55.871321+0000) 2024-03-06T18:41:56.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:56.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:56.871519+0000) 2024-03-06T18:41:57.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:57.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:57.871795+0000) 2024-03-06T18:41:58.866 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:58.870+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:58.872030+0000) 2024-03-06T18:41:59.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:41:59.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:41:59.872238+0000) 2024-03-06T18:42:00.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:00.871+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:00.872453+0000) 2024-03-06T18:42:01.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:01.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:01.872653+0000) 2024-03-06T18:42:02.867 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:02.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:02.872887+0000) 2024-03-06T18:42:03.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:03.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:03.873088+0000) 2024-03-06T18:42:04.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:04.872+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:04.873333+0000) 2024-03-06T18:42:05.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:05.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:05.873597+0000) 2024-03-06T18:42:06.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:06.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:06.873828+0000) 2024-03-06T18:42:07.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:07.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:07.874050+0000) 2024-03-06T18:42:08.868 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:08.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:08.874233+0000) 2024-03-06T18:42:09.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:09.873+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:09.874464+0000) 2024-03-06T18:42:10.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:10.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:10.874672+0000) 2024-03-06T18:42:11.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:11.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:11.874892+0000) 2024-03-06T18:42:12.869 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:12.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:12.875116+0000) 2024-03-06T18:42:13.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:13.874+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:13.875327+0000) 2024-03-06T18:42:14.857 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:42:14.866 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:42:14.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:14.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:14.875675+0000) 2024-03-06T18:42:15.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:15.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:15.875915+0000) 2024-03-06T18:42:16.870 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:16.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:16.876145+0000) 2024-03-06T18:42:17.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:17.875+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:17.876337+0000) 2024-03-06T18:42:18.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:18.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:18.876622+0000) 2024-03-06T18:42:19.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:19.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:19.876864+0000) 2024-03-06T18:42:20.871 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:20.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:20.877057+0000) 2024-03-06T18:42:21.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:21.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:21.877270+0000) 2024-03-06T18:42:22.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:22.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:22.877517+0000) 2024-03-06T18:42:23.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:23.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:23.877738+0000) 2024-03-06T18:42:24.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:24.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:24.877966+0000) 2024-03-06T18:42:25.872 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:25.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:25.878148+0000) 2024-03-06T18:42:26.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:26.876+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:26.878389+0000) 2024-03-06T18:42:27.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:27.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:27.878637+0000) 2024-03-06T18:42:28.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:28.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:28.878835+0000) 2024-03-06T18:42:29.873 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:29.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:29.879058+0000) 2024-03-06T18:42:30.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:30.877+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:30.879284+0000) 2024-03-06T18:42:31.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:31.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:31.879525+0000) 2024-03-06T18:42:32.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:32.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:32.879755+0000) 2024-03-06T18:42:33.874 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:33.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:33.880021+0000) 2024-03-06T18:42:34.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:34.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:34.880238+0000) 2024-03-06T18:42:35.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:35.878+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:35.880512+0000) 2024-03-06T18:42:36.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:36.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:36.880718+0000) 2024-03-06T18:42:37.875 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:37.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:37.880958+0000) 2024-03-06T18:42:38.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:38.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:38.881244+0000) 2024-03-06T18:42:39.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:39.879+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:39.881491+0000) 2024-03-06T18:42:40.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:40.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:40.881672+0000) 2024-03-06T18:42:41.876 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:41.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:41.881912+0000) 2024-03-06T18:42:42.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:42.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:42.882154+0000) 2024-03-06T18:42:43.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:43.880+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:43.882398+0000) 2024-03-06T18:42:44.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:44.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:44.882684+0000) 2024-03-06T18:42:44.979 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:42:44.987 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:42:45.877 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:45.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:45.882880+0000) 2024-03-06T18:42:46.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:46.881+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:46.883109+0000) 2024-03-06T18:42:47.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:47.882+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:47.883378+0000) 2024-03-06T18:42:48.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:48.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:48.883628+0000) 2024-03-06T18:42:49.878 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:49.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:49.883872+0000) 2024-03-06T18:42:50.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:50.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:50.884099+0000) 2024-03-06T18:42:51.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:51.883+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:51.884348+0000) 2024-03-06T18:42:52.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:52.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:52.884622+0000) 2024-03-06T18:42:53.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:53.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:53.884857+0000) 2024-03-06T18:42:54.879 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:54.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:54.885086+0000) 2024-03-06T18:42:55.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:55.884+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:55.885307+0000) 2024-03-06T18:42:56.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:56.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:56.885565+0000) 2024-03-06T18:42:57.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:57.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:57.885793+0000) 2024-03-06T18:42:58.880 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:58.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:58.886019+0000) 2024-03-06T18:42:59.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:42:59.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:42:59.886253+0000) 2024-03-06T18:43:00.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:00.885+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:00.886427+0000) 2024-03-06T18:43:01.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:01.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:01.886681+0000) 2024-03-06T18:43:02.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:02.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:02.886924+0000) 2024-03-06T18:43:03.881 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:03.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:03.887167+0000) 2024-03-06T18:43:04.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:04.886+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:04.887445+0000) 2024-03-06T18:43:05.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:05.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:05.887727+0000) 2024-03-06T18:43:06.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:06.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:06.887967+0000) 2024-03-06T18:43:07.882 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:07.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:07.888243+0000) 2024-03-06T18:43:08.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:08.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:08.888487+0000) 2024-03-06T18:43:09.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:09.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:09.888692+0000) 2024-03-06T18:43:10.883 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:10.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:10.888940+0000) 2024-03-06T18:43:11.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:11.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:11.889217+0000) 2024-03-06T18:43:12.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:12.887+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:12.889410+0000) 2024-03-06T18:43:13.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:13.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:13.889664+0000) 2024-03-06T18:43:14.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:14.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:14.889904+0000) 2024-03-06T18:43:15.108 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:43:15.116 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:43:15.884 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:15.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:15.890158+0000) 2024-03-06T18:43:16.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:16.888+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:16.890399+0000) 2024-03-06T18:43:17.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:17.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:17.890613+0000) 2024-03-06T18:43:18.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:18.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:18.890794+0000) 2024-03-06T18:43:19.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:19.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:19.891003+0000) 2024-03-06T18:43:20.885 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:20.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:20.891213+0000) 2024-03-06T18:43:21.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:21.889+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:21.891414+0000) 2024-03-06T18:43:22.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:22.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:22.891656+0000) 2024-03-06T18:43:23.886 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:23.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:23.891902+0000) 2024-03-06T18:43:24.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:24.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:24.892129+0000) 2024-03-06T18:43:25.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:25.890+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:25.892387+0000) 2024-03-06T18:43:26.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:26.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:26.892674+0000) 2024-03-06T18:43:27.887 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:27.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:27.892911+0000) 2024-03-06T18:43:28.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:28.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:28.893158+0000) 2024-03-06T18:43:29.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:29.891+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:29.893401+0000) 2024-03-06T18:43:30.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:30.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:30.893643+0000) 2024-03-06T18:43:31.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:31.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:31.893814+0000) 2024-03-06T18:43:32.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:32.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:32.894007+0000) 2024-03-06T18:43:33.888 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:33.892+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:33.894237+0000) 2024-03-06T18:43:34.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:34.893+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:34.894455+0000) 2024-03-06T18:43:35.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:35.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:35.894685+0000) 2024-03-06T18:43:36.889 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:36.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:36.894896+0000) 2024-03-06T18:43:37.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:37.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:37.895223+0000) 2024-03-06T18:43:38.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:38.894+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:38.895470+0000) 2024-03-06T18:43:39.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:39.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:39.895709+0000) 2024-03-06T18:43:40.890 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:40.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:40.895900+0000) 2024-03-06T18:43:41.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:41.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:41.896120+0000) 2024-03-06T18:43:42.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:42.895+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:42.896333+0000) 2024-03-06T18:43:43.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:43.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:43.896611+0000) 2024-03-06T18:43:44.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:44.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:44.896803+0000) 2024-03-06T18:43:45.238 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:43:45.246 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:43:45.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:45.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:45.896988+0000) 2024-03-06T18:43:46.891 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:46.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:46.897163+0000) 2024-03-06T18:43:47.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:47.896+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:47.897442+0000) 2024-03-06T18:43:48.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:48.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:48.897685+0000) 2024-03-06T18:43:49.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:49.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:49.897899+0000) 2024-03-06T18:43:50.892 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:50.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:50.898128+0000) 2024-03-06T18:43:51.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:51.897+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:51.898333+0000) 2024-03-06T18:43:52.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:52.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:52.898591+0000) 2024-03-06T18:43:53.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:53.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:53.898839+0000) 2024-03-06T18:43:54.893 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:54.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:54.899070+0000) 2024-03-06T18:43:55.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:55.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:55.899300+0000) 2024-03-06T18:43:56.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:56.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:56.899529+0000) 2024-03-06T18:43:57.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:57.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:57.899754+0000) 2024-03-06T18:43:58.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:58.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:58.899984+0000) 2024-03-06T18:43:59.894 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:43:59.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:43:59.900185+0000) 2024-03-06T18:44:00.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:00.898+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:00.900378+0000) 2024-03-06T18:44:01.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:01.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:01.900648+0000) 2024-03-06T18:44:02.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:02.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:02.900874+0000) 2024-03-06T18:44:03.895 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:03.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:03.901109+0000) 2024-03-06T18:44:04.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:04.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:04.901272+0000) 2024-03-06T18:44:05.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:05.899+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:05.901516+0000) 2024-03-06T18:44:06.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:06.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:06.901808+0000) 2024-03-06T18:44:07.896 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:07.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:07.902056+0000) 2024-03-06T18:44:08.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:08.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:08.902302+0000) 2024-03-06T18:44:09.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:09.900+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:09.902509+0000) 2024-03-06T18:44:10.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:10.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:10.902792+0000) 2024-03-06T18:44:11.897 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:11.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:11.903041+0000) 2024-03-06T18:44:12.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:12.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:12.903270+0000) 2024-03-06T18:44:13.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:13.901+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:13.903506+0000) 2024-03-06T18:44:14.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:14.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:14.903786+0000) 2024-03-06T18:44:15.376 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:44:15.385 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:44:15.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:15.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:15.903980+0000) 2024-03-06T18:44:16.898 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:16.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:16.904190+0000) 2024-03-06T18:44:17.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:17.902+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:17.904431+0000) 2024-03-06T18:44:18.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:18.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:18.904660+0000) 2024-03-06T18:44:19.899 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:19.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:19.904952+0000) 2024-03-06T18:44:20.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:20.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:20.905210+0000) 2024-03-06T18:44:21.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:21.903+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:21.905491+0000) 2024-03-06T18:44:22.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:22.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:22.905798+0000) 2024-03-06T18:44:23.900 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:23.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:23.906075+0000) 2024-03-06T18:44:24.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:24.905+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:24.906322+0000) 2024-03-06T18:44:25.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:25.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:25.906631+0000) 2024-03-06T18:44:26.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:26.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:26.906874+0000) 2024-03-06T18:44:27.901 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:27.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:27.907136+0000) 2024-03-06T18:44:28.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:28.906+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:28.907408+0000) 2024-03-06T18:44:29.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:29.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:29.907649+0000) 2024-03-06T18:44:30.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:30.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:30.907878+0000) 2024-03-06T18:44:31.902 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:31.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:31.908107+0000) 2024-03-06T18:44:32.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:32.907+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:32.908328+0000) 2024-03-06T18:44:33.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:33.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:33.908577+0000) 2024-03-06T18:44:34.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:34.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:34.908801+0000) 2024-03-06T18:44:35.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:35.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:35.909019+0000) 2024-03-06T18:44:36.903 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:36.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:36.909239+0000) 2024-03-06T18:44:37.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:37.908+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:37.909467+0000) 2024-03-06T18:44:38.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:38.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:38.909709+0000) 2024-03-06T18:44:39.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:39.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:39.909929+0000) 2024-03-06T18:44:40.904 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:40.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:40.910110+0000) 2024-03-06T18:44:41.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:41.909+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:41.910353+0000) 2024-03-06T18:44:42.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:42.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:42.910626+0000) 2024-03-06T18:44:43.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:43.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:43.910854+0000) 2024-03-06T18:44:44.905 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:44.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:44.911090+0000) 2024-03-06T18:44:45.513 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:44:45.521 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:44:45.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:45.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:45.911340+0000) 2024-03-06T18:44:46.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:46.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:46.911584+0000) 2024-03-06T18:44:47.906 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:47.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:47.911878+0000) 2024-03-06T18:44:48.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:48.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:48.912163+0000) 2024-03-06T18:44:49.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:49.910+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:49.912367+0000) 2024-03-06T18:44:50.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:50.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:50.912639+0000) 2024-03-06T18:44:51.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:51.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:51.912850+0000) 2024-03-06T18:44:52.907 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:52.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:52.913074+0000) 2024-03-06T18:44:53.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:53.911+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:53.913298+0000) 2024-03-06T18:44:54.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:54.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:54.913529+0000) 2024-03-06T18:44:55.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:55.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:55.913812+0000) 2024-03-06T18:44:56.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:56.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:56.913993+0000) 2024-03-06T18:44:57.908 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:57.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:57.914258+0000) 2024-03-06T18:44:58.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:58.912+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:58.914486+0000) 2024-03-06T18:44:59.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:44:59.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:44:59.914778+0000) 2024-03-06T18:45:00.909 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:00.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:00.914978+0000) 2024-03-06T18:45:01.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:01.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:01.915233+0000) 2024-03-06T18:45:02.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:02.913+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:02.915515+0000) 2024-03-06T18:45:03.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:03.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:03.915805+0000) 2024-03-06T18:45:04.910 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:04.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:04.916051+0000) 2024-03-06T18:45:05.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:05.914+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:05.916294+0000) 2024-03-06T18:45:06.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:06.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:06.916528+0000) 2024-03-06T18:45:07.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:07.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:07.916791+0000) 2024-03-06T18:45:08.911 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:08.915+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:08.917075+0000) 2024-03-06T18:45:09.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:09.916+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:09.917355+0000) 2024-03-06T18:45:10.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:10.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:10.917675+0000) 2024-03-06T18:45:11.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:11.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:11.917967+0000) 2024-03-06T18:45:12.912 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:12.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:12.918205+0000) 2024-03-06T18:45:13.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:13.917+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:13.918443+0000) 2024-03-06T18:45:14.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:14.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:14.918677+0000) 2024-03-06T18:45:15.652 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:45:15.660 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:45:15.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:15.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:15.918857+0000) 2024-03-06T18:45:16.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:16.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:16.919048+0000) 2024-03-06T18:45:17.913 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:17.918+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:17.919250+0000) 2024-03-06T18:45:18.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:18.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:18.919572+0000) 2024-03-06T18:45:19.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:19.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:19.919801+0000) 2024-03-06T18:45:20.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:20.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:20.919988+0000) 2024-03-06T18:45:21.914 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:21.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:21.920250+0000) 2024-03-06T18:45:22.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:22.919+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:22.920480+0000) 2024-03-06T18:45:23.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:23.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:23.920762+0000) 2024-03-06T18:45:24.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:24.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:24.920980+0000) 2024-03-06T18:45:25.915 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:25.920+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:25.921247+0000) 2024-03-06T18:45:26.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:26.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:26.921522+0000) 2024-03-06T18:45:27.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:27.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:27.921832+0000) 2024-03-06T18:45:28.916 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:28.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:28.922066+0000) 2024-03-06T18:45:29.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:29.921+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:29.922297+0000) 2024-03-06T18:45:30.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:30.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:30.922588+0000) 2024-03-06T18:45:31.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:31.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:31.922797+0000) 2024-03-06T18:45:32.917 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:32.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:32.923018+0000) 2024-03-06T18:45:33.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:33.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:33.923258+0000) 2024-03-06T18:45:34.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:34.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:34.923461+0000) 2024-03-06T18:45:35.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:35.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:35.923726+0000) 2024-03-06T18:45:36.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:36.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:36.923945+0000) 2024-03-06T18:45:37.918 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:37.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:37.924164+0000) 2024-03-06T18:45:38.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:38.922+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:38.924406+0000) 2024-03-06T18:45:39.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:39.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:39.924662+0000) 2024-03-06T18:45:40.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:40.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:40.924886+0000) 2024-03-06T18:45:41.919 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:41.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:41.925123+0000) 2024-03-06T18:45:42.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:42.923+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:42.925359+0000) 2024-03-06T18:45:43.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:43.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:43.925622+0000) 2024-03-06T18:45:44.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:44.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:44.925906+0000) 2024-03-06T18:45:45.790 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:45:45.801 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:45:45.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:45.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:45.926170+0000) 2024-03-06T18:45:46.920 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:46.924+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:46.926363+0000) 2024-03-06T18:45:47.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:47.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:47.926626+0000) 2024-03-06T18:45:48.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:48.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:48.926856+0000) 2024-03-06T18:45:49.921 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:49.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:49.927079+0000) 2024-03-06T18:45:50.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:50.925+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:50.927325+0000) 2024-03-06T18:45:51.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:51.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:51.927614+0000) 2024-03-06T18:45:52.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:52.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:52.927848+0000) 2024-03-06T18:45:53.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:53.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:53.928082+0000) 2024-03-06T18:45:54.922 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:54.926+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:54.928294+0000) 2024-03-06T18:45:55.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:55.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:55.928629+0000) 2024-03-06T18:45:56.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:56.927+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:56.928856+0000) 2024-03-06T18:45:57.923 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:57.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:57.929086+0000) 2024-03-06T18:45:58.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:58.928+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:58.929299+0000) 2024-03-06T18:45:59.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:45:59.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:45:59.929625+0000) 2024-03-06T18:46:00.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:00.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:00.929855+0000) 2024-03-06T18:46:01.924 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:01.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:01.930067+0000) 2024-03-06T18:46:02.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:02.929+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:02.930324+0000) 2024-03-06T18:46:03.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:03.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:03.930641+0000) 2024-03-06T18:46:04.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:04.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:04.930915+0000) 2024-03-06T18:46:05.925 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:05.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:05.931186+0000) 2024-03-06T18:46:06.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:06.930+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:06.931461+0000) 2024-03-06T18:46:07.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:07.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:07.931720+0000) 2024-03-06T18:46:08.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:08.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:08.932002+0000) 2024-03-06T18:46:09.926 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:09.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:09.932232+0000) 2024-03-06T18:46:10.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:10.931+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:10.932425+0000) 2024-03-06T18:46:11.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:11.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:11.932674+0000) 2024-03-06T18:46:12.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:12.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:12.932894+0000) 2024-03-06T18:46:13.927 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:13.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:13.933134+0000) 2024-03-06T18:46:14.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:14.932+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:14.933364+0000) 2024-03-06T18:46:15.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:15.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:15.933628+0000) 2024-03-06T18:46:15.938 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:46:15.945 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:46:16.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:16.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:16.933851+0000) 2024-03-06T18:46:17.928 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:17.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:17.934034+0000) 2024-03-06T18:46:18.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:18.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:18.934267+0000) 2024-03-06T18:46:19.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:19.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:19.934480+0000) 2024-03-06T18:46:20.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:20.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:20.934735+0000) 2024-03-06T18:46:21.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:21.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:21.934931+0000) 2024-03-06T18:46:22.929 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:22.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:22.935158+0000) 2024-03-06T18:46:23.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:23.933+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:23.935392+0000) 2024-03-06T18:46:24.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:24.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:24.935655+0000) 2024-03-06T18:46:25.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:25.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:25.935882+0000) 2024-03-06T18:46:26.930 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:26.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:26.936112+0000) 2024-03-06T18:46:27.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:27.934+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:27.936339+0000) 2024-03-06T18:46:28.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:28.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:28.936573+0000) 2024-03-06T18:46:29.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:29.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:29.936787+0000) 2024-03-06T18:46:30.931 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:30.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:30.937027+0000) 2024-03-06T18:46:31.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:31.935+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:31.937260+0000) 2024-03-06T18:46:32.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:32.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:32.937525+0000) 2024-03-06T18:46:33.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:33.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:33.937852+0000) 2024-03-06T18:46:34.932 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:34.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:34.938046+0000) 2024-03-06T18:46:35.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:35.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:35.938276+0000) 2024-03-06T18:46:36.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:36.936+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:36.938508+0000) 2024-03-06T18:46:37.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:37.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:37.938783+0000) 2024-03-06T18:46:38.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:38.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:38.938992+0000) 2024-03-06T18:46:39.933 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:39.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:39.939217+0000) 2024-03-06T18:46:40.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:40.937+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:40.939492+0000) 2024-03-06T18:46:41.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:41.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:41.939815+0000) 2024-03-06T18:46:42.934 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:42.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:42.940092+0000) 2024-03-06T18:46:43.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:43.938+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:43.940369+0000) 2024-03-06T18:46:44.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:44.939+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:44.940631+0000) 2024-03-06T18:46:45.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:45.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:45.940867+0000) 2024-03-06T18:46:46.075 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:46:46.085 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:46:46.935 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:46.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:46.941064+0000) 2024-03-06T18:46:47.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:47.940+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:47.941324+0000) 2024-03-06T18:46:48.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:48.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:48.941640+0000) 2024-03-06T18:46:49.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:49.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:49.941886+0000) 2024-03-06T18:46:50.936 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:50.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:50.942064+0000) 2024-03-06T18:46:51.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:51.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:51.942295+0000) 2024-03-06T18:46:52.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:52.941+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:52.942518+0000) 2024-03-06T18:46:53.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:53.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:53.942757+0000) 2024-03-06T18:46:54.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:54.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:54.942988+0000) 2024-03-06T18:46:55.937 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:55.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:55.943198+0000) 2024-03-06T18:46:56.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:56.942+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:56.943435+0000) 2024-03-06T18:46:57.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:57.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:57.943688+0000) 2024-03-06T18:46:58.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:58.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:58.943915+0000) 2024-03-06T18:46:59.938 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:46:59.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:46:59.944143+0000) 2024-03-06T18:47:00.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:00.943+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:00.944370+0000) 2024-03-06T18:47:01.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:01.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:01.944647+0000) 2024-03-06T18:47:02.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:02.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:02.944858+0000) 2024-03-06T18:47:03.939 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:03.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:03.945073+0000) 2024-03-06T18:47:04.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:04.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:04.945273+0000) 2024-03-06T18:47:05.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:05.944+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:05.945469+0000) 2024-03-06T18:47:06.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:06.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:06.945714+0000) 2024-03-06T18:47:07.940 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:07.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:07.945949+0000) 2024-03-06T18:47:08.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:08.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:08.946182+0000) 2024-03-06T18:47:09.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:09.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:09.946472+0000) 2024-03-06T18:47:10.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:10.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:10.946738+0000) 2024-03-06T18:47:11.941 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:11.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:11.947045+0000) 2024-03-06T18:47:12.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:12.945+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:12.947330+0000) 2024-03-06T18:47:13.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:13.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:13.947572+0000) 2024-03-06T18:47:14.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:14.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:14.947820+0000) 2024-03-06T18:47:15.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:15.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:15.948046+0000) 2024-03-06T18:47:16.213 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:47:16.222 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:47:16.942 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:16.946+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:16.948275+0000) 2024-03-06T18:47:17.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:17.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:17.948520+0000) 2024-03-06T18:47:18.943 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:18.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:18.948799+0000) 2024-03-06T18:47:19.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:19.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:19.949021+0000) 2024-03-06T18:47:20.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:20.947+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:20.949302+0000) 2024-03-06T18:47:21.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:21.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:21.949597+0000) 2024-03-06T18:47:22.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:22.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:22.949824+0000) 2024-03-06T18:47:23.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:23.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:23.950029+0000) 2024-03-06T18:47:24.944 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:24.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:24.950250+0000) 2024-03-06T18:47:25.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:25.948+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:25.950468+0000) 2024-03-06T18:47:26.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:26.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:26.950684+0000) 2024-03-06T18:47:27.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:27.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:27.950927+0000) 2024-03-06T18:47:28.945 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:28.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:28.951167+0000) 2024-03-06T18:47:29.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:29.949+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:29.951402+0000) 2024-03-06T18:47:30.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:30.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:30.951634+0000) 2024-03-06T18:47:31.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:31.950+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:31.951859+0000) 2024-03-06T18:47:32.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:32.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:32.952085+0000) 2024-03-06T18:47:33.946 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:33.951+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:33.952322+0000) 2024-03-06T18:47:34.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:34.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:34.952595+0000) 2024-03-06T18:47:35.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:35.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:35.952827+0000) 2024-03-06T18:47:36.947 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:36.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:36.953053+0000) 2024-03-06T18:47:37.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:37.952+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:37.953347+0000) 2024-03-06T18:47:38.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:38.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:38.953629+0000) 2024-03-06T18:47:39.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:39.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:39.953842+0000) 2024-03-06T18:47:40.948 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:40.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:40.954086+0000) 2024-03-06T18:47:41.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:41.953+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:41.954337+0000) 2024-03-06T18:47:42.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:42.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:42.954621+0000) 2024-03-06T18:47:43.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:43.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:43.954865+0000) 2024-03-06T18:47:44.949 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:44.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:44.955097+0000) 2024-03-06T18:47:45.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:45.954+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:45.955324+0000) 2024-03-06T18:47:46.351 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:47:46.359 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:47:46.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:46.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:46.955587+0000) 2024-03-06T18:47:47.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:47.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:47.955810+0000) 2024-03-06T18:47:48.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:48.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:48.956013+0000) 2024-03-06T18:47:49.950 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:49.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:49.956234+0000) 2024-03-06T18:47:50.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:50.955+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:50.956460+0000) 2024-03-06T18:47:51.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:51.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:51.956720+0000) 2024-03-06T18:47:52.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:52.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:52.956942+0000) 2024-03-06T18:47:53.951 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:53.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:53.957179+0000) 2024-03-06T18:47:54.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:54.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:54.957411+0000) 2024-03-06T18:47:55.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:55.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:55.957649+0000) 2024-03-06T18:47:56.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:56.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:56.957878+0000) 2024-03-06T18:47:57.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:57.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:57.958104+0000) 2024-03-06T18:47:58.952 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:58.956+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:58.958287+0000) 2024-03-06T18:47:59.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:47:59.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:47:59.958521+0000) 2024-03-06T18:48:00.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:00.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:00.958796+0000) 2024-03-06T18:48:01.953 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:01.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:01.959033+0000) 2024-03-06T18:48:02.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:02.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:02.959256+0000) 2024-03-06T18:48:03.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:03.957+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:03.959441+0000) 2024-03-06T18:48:04.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:04.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:04.959672+0000) 2024-03-06T18:48:05.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:05.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:05.959847+0000) 2024-03-06T18:48:06.954 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:06.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:06.960084+0000) 2024-03-06T18:48:07.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:07.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:07.960307+0000) 2024-03-06T18:48:08.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:08.958+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:08.960454+0000) 2024-03-06T18:48:09.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:09.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:09.960658+0000) 2024-03-06T18:48:10.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:10.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:10.960888+0000) 2024-03-06T18:48:11.955 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:11.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:11.961047+0000) 2024-03-06T18:48:12.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:12.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:12.961282+0000) 2024-03-06T18:48:13.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:13.959+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:13.961508+0000) 2024-03-06T18:48:14.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:14.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:14.961755+0000) 2024-03-06T18:48:15.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:15.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:15.961989+0000) 2024-03-06T18:48:16.490 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:48:16.498 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:48:16.956 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:16.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:16.962141+0000) 2024-03-06T18:48:17.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:17.960+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:17.962300+0000) 2024-03-06T18:48:18.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:18.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:18.962582+0000) 2024-03-06T18:48:19.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:19.961+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:19.962840+0000) 2024-03-06T18:48:20.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:20.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:20.963063+0000) 2024-03-06T18:48:21.957 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:21.962+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:21.963297+0000) 2024-03-06T18:48:22.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:22.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:22.963587+0000) 2024-03-06T18:48:23.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:23.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:23.963793+0000) 2024-03-06T18:48:24.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:24.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:24.964030+0000) 2024-03-06T18:48:25.958 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:25.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:25.964261+0000) 2024-03-06T18:48:26.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:26.963+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:26.964483+0000) 2024-03-06T18:48:27.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:27.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:27.964688+0000) 2024-03-06T18:48:28.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:28.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:28.964910+0000) 2024-03-06T18:48:29.959 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:29.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:29.965139+0000) 2024-03-06T18:48:30.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:30.964+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:30.965420+0000) 2024-03-06T18:48:31.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:31.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:31.965658+0000) 2024-03-06T18:48:32.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:32.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:32.965852+0000) 2024-03-06T18:48:33.960 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:33.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:33.966099+0000) 2024-03-06T18:48:34.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:34.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:34.966288+0000) 2024-03-06T18:48:35.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:35.965+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:35.966491+0000) 2024-03-06T18:48:36.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:36.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:36.966692+0000) 2024-03-06T18:48:37.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:37.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:37.966948+0000) 2024-03-06T18:48:38.961 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:38.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:38.967200+0000) 2024-03-06T18:48:39.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:39.966+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:39.967432+0000) 2024-03-06T18:48:40.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:40.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:40.967662+0000) 2024-03-06T18:48:41.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:41.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:41.967869+0000) 2024-03-06T18:48:42.962 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:42.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:42.968111+0000) 2024-03-06T18:48:43.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:43.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:43.968345+0000) 2024-03-06T18:48:44.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:44.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:44.968635+0000) 2024-03-06T18:48:45.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:45.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:45.968849+0000) 2024-03-06T18:48:46.628 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:48:46.636 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:48:46.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:46.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:46.969092+0000) 2024-03-06T18:48:47.963 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:47.967+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:47.969315+0000) 2024-03-06T18:48:48.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:48.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:48.969574+0000) 2024-03-06T18:48:49.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:49.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:49.969775+0000) 2024-03-06T18:48:50.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:50.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:50.969994+0000) 2024-03-06T18:48:51.964 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:51.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:51.970226+0000) 2024-03-06T18:48:52.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:52.968+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:52.970452+0000) 2024-03-06T18:48:53.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:53.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:53.970678+0000) 2024-03-06T18:48:54.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:54.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:54.970903+0000) 2024-03-06T18:48:55.965 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:55.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:55.971079+0000) 2024-03-06T18:48:56.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:56.969+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:56.971301+0000) 2024-03-06T18:48:57.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:57.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:57.971581+0000) 2024-03-06T18:48:58.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:58.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:58.971795+0000) 2024-03-06T18:48:59.966 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:48:59.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:48:59.972000+0000) 2024-03-06T18:49:00.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:00.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:00.972219+0000) 2024-03-06T18:49:01.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:01.970+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:01.972474+0000) 2024-03-06T18:49:02.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:02.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:02.972712+0000) 2024-03-06T18:49:03.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:03.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:03.972934+0000) 2024-03-06T18:49:04.967 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:04.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:04.973135+0000) 2024-03-06T18:49:05.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:05.971+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:05.973366+0000) 2024-03-06T18:49:06.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:06.972+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:06.973613+0000) 2024-03-06T18:49:07.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:07.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:07.973834+0000) 2024-03-06T18:49:08.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:08.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:08.974054+0000) 2024-03-06T18:49:09.968 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:09.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:09.974285+0000) 2024-03-06T18:49:10.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:10.973+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:10.974490+0000) 2024-03-06T18:49:11.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:11.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:11.974766+0000) 2024-03-06T18:49:12.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:12.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:12.974958+0000) 2024-03-06T18:49:13.969 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:13.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:13.975237+0000) 2024-03-06T18:49:14.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:14.974+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:14.975483+0000) 2024-03-06T18:49:15.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:15.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:15.975713+0000) 2024-03-06T18:49:16.771 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:49:16.779 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:49:16.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:16.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:16.975937+0000) 2024-03-06T18:49:17.970 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:17.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:17.976157+0000) 2024-03-06T18:49:18.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:18.975+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:18.976392+0000) 2024-03-06T18:49:19.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:19.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:19.976656+0000) 2024-03-06T18:49:20.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:20.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:20.976882+0000) 2024-03-06T18:49:21.971 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:21.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:21.977112+0000) 2024-03-06T18:49:22.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:22.976+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:22.977334+0000) 2024-03-06T18:49:23.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:23.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:23.977593+0000) 2024-03-06T18:49:24.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:24.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:24.977819+0000) 2024-03-06T18:49:25.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:25.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:25.978003+0000) 2024-03-06T18:49:26.972 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:26.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:26.978187+0000) 2024-03-06T18:49:27.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:27.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:27.978363+0000) 2024-03-06T18:49:28.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:28.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:28.978621+0000) 2024-03-06T18:49:29.973 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:29.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:29.978837+0000) 2024-03-06T18:49:30.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:30.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:30.979028+0000) 2024-03-06T18:49:31.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:31.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:31.979263+0000) 2024-03-06T18:49:32.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:32.977+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:32.979487+0000) 2024-03-06T18:49:33.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:33.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:33.979712+0000) 2024-03-06T18:49:34.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:34.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:34.979946+0000) 2024-03-06T18:49:35.974 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:35.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:35.980162+0000) 2024-03-06T18:49:36.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:36.978+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:36.980381+0000) 2024-03-06T18:49:37.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:37.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:37.980641+0000) 2024-03-06T18:49:38.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:38.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:38.980870+0000) 2024-03-06T18:49:39.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:39.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:39.981104+0000) 2024-03-06T18:49:40.975 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:40.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:40.981287+0000) 2024-03-06T18:49:41.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:41.979+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:41.981481+0000) 2024-03-06T18:49:42.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:42.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:42.981702+0000) 2024-03-06T18:49:43.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:43.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:43.981890+0000) 2024-03-06T18:49:44.976 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:44.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:44.982113+0000) 2024-03-06T18:49:45.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:45.980+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:45.982342+0000) 2024-03-06T18:49:46.903 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:49:46.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:49:46.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:46.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:46.982627+0000) 2024-03-06T18:49:47.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:47.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:47.982832+0000) 2024-03-06T18:49:48.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:48.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:48.983056+0000) 2024-03-06T18:49:49.977 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:49.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:49.983284+0000) 2024-03-06T18:49:50.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:50.981+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:50.983517+0000) 2024-03-06T18:49:51.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:51.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:51.983750+0000) 2024-03-06T18:49:52.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:52.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:52.983944+0000) 2024-03-06T18:49:53.978 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:53.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:53.984140+0000) 2024-03-06T18:49:54.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:54.982+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:54.984389+0000) 2024-03-06T18:49:55.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:55.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:55.984659+0000) 2024-03-06T18:49:56.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:56.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:56.984900+0000) 2024-03-06T18:49:57.979 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:57.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:57.985134+0000) 2024-03-06T18:49:58.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:58.984+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:58.985333+0000) 2024-03-06T18:49:59.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:49:59.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:49:59.985605+0000) 2024-03-06T18:50:00.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:00.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:00.985867+0000) 2024-03-06T18:50:01.980 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:01.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:01.986115+0000) 2024-03-06T18:50:02.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:02.985+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:02.986341+0000) 2024-03-06T18:50:03.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:03.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:03.986594+0000) 2024-03-06T18:50:04.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:04.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:04.986797+0000) 2024-03-06T18:50:05.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:05.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:05.986991+0000) 2024-03-06T18:50:06.981 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:06.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:06.987284+0000) 2024-03-06T18:50:07.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:07.986+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:07.987472+0000) 2024-03-06T18:50:08.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:08.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:08.987712+0000) 2024-03-06T18:50:09.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:09.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:09.987949+0000) 2024-03-06T18:50:10.982 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:10.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:10.988176+0000) 2024-03-06T18:50:11.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:11.987+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:11.988385+0000) 2024-03-06T18:50:12.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:12.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:12.988659+0000) 2024-03-06T18:50:13.983 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:13.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:13.988863+0000) 2024-03-06T18:50:14.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:14.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:14.989070+0000) 2024-03-06T18:50:15.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:15.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:15.989289+0000) 2024-03-06T18:50:16.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:16.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:16.989509+0000) 2024-03-06T18:50:17.043 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:50:17.053 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:50:17.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:17.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:17.989783+0000) 2024-03-06T18:50:18.984 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:18.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:18.989964+0000) 2024-03-06T18:50:19.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:19.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:19.990189+0000) 2024-03-06T18:50:20.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:20.988+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:20.990380+0000) 2024-03-06T18:50:21.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:21.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:21.990664+0000) 2024-03-06T18:50:22.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:22.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:22.990865+0000) 2024-03-06T18:50:23.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:23.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:23.991065+0000) 2024-03-06T18:50:24.985 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:24.989+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:24.991303+0000) 2024-03-06T18:50:25.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:25.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:25.991562+0000) 2024-03-06T18:50:26.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:26.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:26.991789+0000) 2024-03-06T18:50:27.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:27.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:27.992014+0000) 2024-03-06T18:50:28.986 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:28.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:28.992234+0000) 2024-03-06T18:50:29.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:29.990+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:29.992416+0000) 2024-03-06T18:50:30.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:30.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:30.992641+0000) 2024-03-06T18:50:31.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:31.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:31.992851+0000) 2024-03-06T18:50:32.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:32.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:32.993073+0000) 2024-03-06T18:50:33.987 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:33.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:33.993257+0000) 2024-03-06T18:50:34.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:34.991+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:34.993494+0000) 2024-03-06T18:50:35.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:35.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:35.993702+0000) 2024-03-06T18:50:36.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:36.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:36.993884+0000) 2024-03-06T18:50:37.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:37.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:37.994105+0000) 2024-03-06T18:50:38.988 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:38.992+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:38.994334+0000) 2024-03-06T18:50:39.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:39.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:39.994598+0000) 2024-03-06T18:50:40.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:40.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:40.994836+0000) 2024-03-06T18:50:41.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:41.993+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:41.995068+0000) 2024-03-06T18:50:42.989 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:42.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:42.995259+0000) 2024-03-06T18:50:43.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:43.994+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:43.995501+0000) 2024-03-06T18:50:44.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:44.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:44.995745+0000) 2024-03-06T18:50:45.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:45.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:45.995968+0000) 2024-03-06T18:50:46.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:46.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:46.996191+0000) 2024-03-06T18:50:47.258 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:50:47.266 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:50:47.990 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:47.995+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:47.996376+0000) 2024-03-06T18:50:48.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:48.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:48.996643+0000) 2024-03-06T18:50:49.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:49.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:49.996874+0000) 2024-03-06T18:50:50.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:50.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:50.997049+0000) 2024-03-06T18:50:51.991 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:51.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:51.997276+0000) 2024-03-06T18:50:52.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:52.996+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:52.997453+0000) 2024-03-06T18:50:53.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:53.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:53.997680+0000) 2024-03-06T18:50:54.992 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:54.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:54.997969+0000) 2024-03-06T18:50:55.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:55.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:55.998172+0000) 2024-03-06T18:50:56.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:56.997+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:56.998402+0000) 2024-03-06T18:50:57.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:57.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:57.998669+0000) 2024-03-06T18:50:58.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:58.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:58.998862+0000) 2024-03-06T18:50:59.993 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:50:59.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:50:59.999056+0000) 2024-03-06T18:51:00.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:00.998+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:00.999284+0000) 2024-03-06T18:51:01.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:01.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:01.999525+0000) 2024-03-06T18:51:02.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:02.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:02.999794+0000) 2024-03-06T18:51:03.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:03.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:04.000021+0000) 2024-03-06T18:51:04.994 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:04.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:05.000253+0000) 2024-03-06T18:51:05.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:05.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:06.000487+0000) 2024-03-06T18:51:06.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:07.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:07.000707+0000) 2024-03-06T18:51:07.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:07.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:08.000900+0000) 2024-03-06T18:51:08.995 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:08.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:09.001127+0000) 2024-03-06T18:51:09.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:09.999+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:10.001392+0000) 2024-03-06T18:51:10.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:11.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:11.001653+0000) 2024-03-06T18:51:11.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:12.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:12.001851+0000) 2024-03-06T18:51:12.996 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:13.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:13.002036+0000) 2024-03-06T18:51:13.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:14.000+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:14.002313+0000) 2024-03-06T18:51:14.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:15.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:15.002628+0000) 2024-03-06T18:51:15.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:16.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:16.002807+0000) 2024-03-06T18:51:16.997 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:17.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:17.003043+0000) 2024-03-06T18:51:17.451 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:51:17.459 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:51:17.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:18.001+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:18.003289+0000) 2024-03-06T18:51:18.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:19.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:19.003580+0000) 2024-03-06T18:51:19.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:20.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:20.003829+0000) 2024-03-06T18:51:20.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:21.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:21.004066+0000) 2024-03-06T18:51:21.998 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:22.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:22.004274+0000) 2024-03-06T18:51:22.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:23.002+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:23.004466+0000) 2024-03-06T18:51:23.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:24.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:24.004685+0000) 2024-03-06T18:51:24.999 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:25.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:25.004913+0000) 2024-03-06T18:51:26.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:26.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:26.005223+0000) 2024-03-06T18:51:27.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:27.003+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:27.005432+0000) 2024-03-06T18:51:28.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:28.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:28.005661+0000) 2024-03-06T18:51:29.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:29.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:29.005818+0000) 2024-03-06T18:51:30.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:30.004+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:30.006061+0000) 2024-03-06T18:51:31.000 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:31.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:31.006288+0000) 2024-03-06T18:51:32.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:32.005+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:32.006506+0000) 2024-03-06T18:51:33.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:33.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:33.006748+0000) 2024-03-06T18:51:34.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:34.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:34.006960+0000) 2024-03-06T18:51:35.001 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:35.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:35.007211+0000) 2024-03-06T18:51:36.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:36.006+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:36.007445+0000) 2024-03-06T18:51:37.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:37.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:37.007690+0000) 2024-03-06T18:51:38.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:38.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:38.007935+0000) 2024-03-06T18:51:39.002 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:39.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:39.008139+0000) 2024-03-06T18:51:40.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:40.007+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:40.008397+0000) 2024-03-06T18:51:41.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:41.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:41.008643+0000) 2024-03-06T18:51:42.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:42.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:42.008889+0000) 2024-03-06T18:51:43.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:43.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:43.009089+0000) 2024-03-06T18:51:44.003 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:44.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:44.009275+0000) 2024-03-06T18:51:45.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:45.008+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:45.009489+0000) 2024-03-06T18:51:46.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:46.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:46.009732+0000) 2024-03-06T18:51:47.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:47.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:47.009964+0000) 2024-03-06T18:51:47.660 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:51:47.669 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:51:48.004 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:48.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:48.010253+0000) 2024-03-06T18:51:49.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:49.009+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:49.010500+0000) 2024-03-06T18:51:50.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:50.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:50.010759+0000) 2024-03-06T18:51:51.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:51.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:51.010963+0000) 2024-03-06T18:51:52.005 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:52.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:52.011191+0000) 2024-03-06T18:51:53.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:53.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:53.011415+0000) 2024-03-06T18:51:54.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:54.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:54.011647+0000) 2024-03-06T18:51:55.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:55.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:55.011844+0000) 2024-03-06T18:51:56.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:56.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:56.012068+0000) 2024-03-06T18:51:57.006 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:57.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:57.012288+0000) 2024-03-06T18:51:58.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:58.010+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:58.012482+0000) 2024-03-06T18:51:59.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:51:59.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:51:59.012734+0000) 2024-03-06T18:52:00.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:00.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:00.012970+0000) 2024-03-06T18:52:01.007 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:01.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:01.013242+0000) 2024-03-06T18:52:02.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:02.011+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:02.013517+0000) 2024-03-06T18:52:03.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:03.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:03.013767+0000) 2024-03-06T18:52:04.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:04.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:04.013993+0000) 2024-03-06T18:52:05.008 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:05.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:05.014264+0000) 2024-03-06T18:52:06.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:06.012+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:06.014506+0000) 2024-03-06T18:52:07.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:07.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:07.014789+0000) 2024-03-06T18:52:08.009 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:08.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:08.015063+0000) 2024-03-06T18:52:09.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:09.013+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:09.015344+0000) 2024-03-06T18:52:10.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:10.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:10.015662+0000) 2024-03-06T18:52:11.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:11.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:11.015940+0000) 2024-03-06T18:52:12.010 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:12.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:12.016234+0000) 2024-03-06T18:52:13.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:13.014+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:13.016512+0000) 2024-03-06T18:52:14.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:14.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:14.016773+0000) 2024-03-06T18:52:15.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:15.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:15.016995+0000) 2024-03-06T18:52:16.011 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:16.015+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:16.017281+0000) 2024-03-06T18:52:17.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:17.016+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:17.017607+0000) 2024-03-06T18:52:17.887 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:52:17.895 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:52:18.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:18.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:18.017903+0000) 2024-03-06T18:52:19.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:19.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:19.018189+0000) 2024-03-06T18:52:20.012 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:20.017+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:20.018404+0000) 2024-03-06T18:52:21.013 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:21.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:21.018675+0000) 2024-03-06T18:52:22.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:22.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:22.018901+0000) 2024-03-06T18:52:23.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:23.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:23.019143+0000) 2024-03-06T18:52:24.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:24.018+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:24.019343+0000) 2024-03-06T18:52:25.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:25.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:25.019588+0000) 2024-03-06T18:52:26.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:26.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:26.019838+0000) 2024-03-06T18:52:27.014 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:27.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:27.020031+0000) 2024-03-06T18:52:28.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:28.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:28.020260+0000) 2024-03-06T18:52:29.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:29.019+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:29.020499+0000) 2024-03-06T18:52:30.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:30.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:30.020683+0000) 2024-03-06T18:52:31.015 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:31.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:31.020828+0000) 2024-03-06T18:52:32.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:32.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:32.021031+0000) 2024-03-06T18:52:33.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:33.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:33.021218+0000) 2024-03-06T18:52:34.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:34.020+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:34.021401+0000) 2024-03-06T18:52:35.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:35.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:35.021630+0000) 2024-03-06T18:52:36.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:36.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:36.021888+0000) 2024-03-06T18:52:37.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:37.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:37.022133+0000) 2024-03-06T18:52:38.016 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:38.021+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:38.022295+0000) 2024-03-06T18:52:39.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:39.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:39.022565+0000) 2024-03-06T18:52:40.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:40.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:40.022797+0000) 2024-03-06T18:52:41.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:41.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:41.023025+0000) 2024-03-06T18:52:42.017 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:42.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:42.023265+0000) 2024-03-06T18:52:43.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:43.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:43.023486+0000) 2024-03-06T18:52:44.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:44.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:44.023688+0000) 2024-03-06T18:52:45.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:45.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:45.023854+0000) 2024-03-06T18:52:46.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:46.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:46.024027+0000) 2024-03-06T18:52:47.018 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:47.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:47.024204+0000) 2024-03-06T18:52:48.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:48.022+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:48.024426+0000) 2024-03-06T18:52:48.071 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:52:48.088 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:52:49.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:49.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:49.024692+0000) 2024-03-06T18:52:50.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:50.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:50.024925+0000) 2024-03-06T18:52:51.019 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:51.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:51.025191+0000) 2024-03-06T18:52:52.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:52.023+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:52.025401+0000) 2024-03-06T18:52:53.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:53.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:53.025644+0000) 2024-03-06T18:52:54.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:54.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:54.025829+0000) 2024-03-06T18:52:55.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:55.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:55.026028+0000) 2024-03-06T18:52:56.020 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:56.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:56.026227+0000) 2024-03-06T18:52:57.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:57.024+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:57.026446+0000) 2024-03-06T18:52:58.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:58.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:58.026681+0000) 2024-03-06T18:52:59.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:52:59.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:52:59.026872+0000) 2024-03-06T18:53:00.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:00.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:00.027101+0000) 2024-03-06T18:53:01.021 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:01.025+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:01.027327+0000) 2024-03-06T18:53:02.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:02.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:02.027590+0000) 2024-03-06T18:53:03.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:03.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:03.027780+0000) 2024-03-06T18:53:04.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:04.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:04.027967+0000) 2024-03-06T18:53:05.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:05.026+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:05.028153+0000) 2024-03-06T18:53:06.022 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:06.027+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:06.028378+0000) 2024-03-06T18:53:07.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:07.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:07.028618+0000) 2024-03-06T18:53:08.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:08.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:08.028800+0000) 2024-03-06T18:53:09.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:09.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:09.028986+0000) 2024-03-06T18:53:10.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:10.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:10.029182+0000) 2024-03-06T18:53:11.023 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:11.028+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:11.029404+0000) 2024-03-06T18:53:12.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:12.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:12.029647+0000) 2024-03-06T18:53:13.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:13.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:13.029849+0000) 2024-03-06T18:53:14.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:14.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:14.030056+0000) 2024-03-06T18:53:15.024 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:15.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:15.030240+0000) 2024-03-06T18:53:16.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:16.029+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:16.030470+0000) 2024-03-06T18:53:17.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:17.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:17.030700+0000) 2024-03-06T18:53:18.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:18.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:18.030900+0000) 2024-03-06T18:53:18.270 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:53:18.279 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:53:19.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:19.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:19.031084+0000) 2024-03-06T18:53:20.025 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:20.030+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:20.031319+0000) 2024-03-06T18:53:21.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:21.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:21.031629+0000) 2024-03-06T18:53:22.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:22.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:22.031838+0000) 2024-03-06T18:53:23.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:23.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:23.032009+0000) 2024-03-06T18:53:24.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:24.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:24.032251+0000) 2024-03-06T18:53:25.026 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:25.031+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:25.032494+0000) 2024-03-06T18:53:26.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:26.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:26.032783+0000) 2024-03-06T18:53:27.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:27.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:27.033000+0000) 2024-03-06T18:53:28.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:28.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:28.033228+0000) 2024-03-06T18:53:29.027 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:29.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:29.033451+0000) 2024-03-06T18:53:30.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:30.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:30.033689+0000) 2024-03-06T18:53:31.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:31.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:31.033899+0000) 2024-03-06T18:53:32.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:32.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:32.034137+0000) 2024-03-06T18:53:33.028 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:33.032+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:33.034400+0000) 2024-03-06T18:53:34.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:34.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:34.034663+0000) 2024-03-06T18:53:35.029 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:35.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:35.034875+0000) 2024-03-06T18:53:36.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:36.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:36.035120+0000) 2024-03-06T18:53:37.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:37.033+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:37.035340+0000) 2024-03-06T18:53:38.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:38.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:38.035522+0000) 2024-03-06T18:53:39.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:39.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:39.035743+0000) 2024-03-06T18:53:40.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:40.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:40.035947+0000) 2024-03-06T18:53:41.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:41.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:41.036159+0000) 2024-03-06T18:53:42.030 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:42.034+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:42.036358+0000) 2024-03-06T18:53:43.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:43.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:43.036591+0000) 2024-03-06T18:53:44.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:44.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:44.036777+0000) 2024-03-06T18:53:45.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:45.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:45.036965+0000) 2024-03-06T18:53:46.031 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:46.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:46.037195+0000) 2024-03-06T18:53:47.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:47.035+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:47.037415+0000) 2024-03-06T18:53:48.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:48.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:48.037648+0000) 2024-03-06T18:53:48.509 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:53:48.520 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:53:49.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:49.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:49.037876+0000) 2024-03-06T18:53:50.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:50.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:50.038075+0000) 2024-03-06T18:53:51.032 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:51.036+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:51.038315+0000) 2024-03-06T18:53:52.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:52.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:52.038579+0000) 2024-03-06T18:53:53.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:53.037+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:53.038770+0000) 2024-03-06T18:53:54.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:54.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:54.038964+0000) 2024-03-06T18:53:55.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:55.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:55.039186+0000) 2024-03-06T18:53:56.033 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:56.038+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:56.039420+0000) 2024-03-06T18:53:57.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:57.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:57.039666+0000) 2024-03-06T18:53:58.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:58.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:58.039844+0000) 2024-03-06T18:53:59.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:53:59.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:53:59.040041+0000) 2024-03-06T18:54:00.034 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:00.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:00.040278+0000) 2024-03-06T18:54:01.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:01.039+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:01.040474+0000) 2024-03-06T18:54:02.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:02.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:02.040758+0000) 2024-03-06T18:54:03.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:03.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:03.041039+0000) 2024-03-06T18:54:04.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:04.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:04.041232+0000) 2024-03-06T18:54:05.035 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:05.040+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:05.041439+0000) 2024-03-06T18:54:06.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:06.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:06.041725+0000) 2024-03-06T18:54:07.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:07.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:07.041940+0000) 2024-03-06T18:54:08.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:08.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:08.042166+0000) 2024-03-06T18:54:09.036 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:09.041+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:09.042387+0000) 2024-03-06T18:54:10.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:10.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:10.042684+0000) 2024-03-06T18:54:11.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:11.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:11.042967+0000) 2024-03-06T18:54:12.037 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:12.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:12.043255+0000) 2024-03-06T18:54:13.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:13.042+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:13.043486+0000) 2024-03-06T18:54:14.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:14.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:14.043736+0000) 2024-03-06T18:54:15.038 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:15.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:15.044003+0000) 2024-03-06T18:54:16.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:16.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:16.044280+0000) 2024-03-06T18:54:17.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:17.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:17.044518+0000) 2024-03-06T18:54:18.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:18.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:18.044754+0000) 2024-03-06T18:54:18.740 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:54:18.749 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:54:19.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:19.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:19.044945+0000) 2024-03-06T18:54:20.039 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:20.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:20.045166+0000) 2024-03-06T18:54:21.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:21.043+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:21.045441+0000) 2024-03-06T18:54:22.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:22.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:22.045707+0000) 2024-03-06T18:54:23.040 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:23.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:23.045930+0000) 2024-03-06T18:54:24.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:24.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:24.046152+0000) 2024-03-06T18:54:25.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:25.044+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:25.046380+0000) 2024-03-06T18:54:26.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:26.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:26.046618+0000) 2024-03-06T18:54:27.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:27.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:27.046894+0000) 2024-03-06T18:54:28.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:28.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:28.047153+0000) 2024-03-06T18:54:29.041 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:29.045+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:29.047378+0000) 2024-03-06T18:54:30.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:30.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:30.047614+0000) 2024-03-06T18:54:31.042 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:31.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:31.047822+0000) 2024-03-06T18:54:32.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:32.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:32.048036+0000) 2024-03-06T18:54:33.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:33.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:33.048292+0000) 2024-03-06T18:54:34.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:34.046+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:34.048501+0000) 2024-03-06T18:54:35.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:35.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:35.048763+0000) 2024-03-06T18:54:36.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:36.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:36.049005+0000) 2024-03-06T18:54:37.043 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:37.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:37.049226+0000) 2024-03-06T18:54:38.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:38.047+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:38.049509+0000) 2024-03-06T18:54:39.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:39.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:39.049844+0000) 2024-03-06T18:54:40.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:40.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:40.050046+0000) 2024-03-06T18:54:41.044 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:41.048+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:41.050322+0000) 2024-03-06T18:54:42.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:42.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:42.050645+0000) 2024-03-06T18:54:43.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:43.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:43.050926+0000) 2024-03-06T18:54:44.045 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:44.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:44.051206+0000) 2024-03-06T18:54:45.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:45.050+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:45.051487+0000) 2024-03-06T18:54:46.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:46.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:46.051759+0000) 2024-03-06T18:54:47.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:47.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:47.052044+0000) 2024-03-06T18:54:48.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:48.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:48.052324+0000) 2024-03-06T18:54:48.967 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:54:48.976 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:54:49.046 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:49.051+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:49.052474+0000) 2024-03-06T18:54:50.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:50.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:50.052651+0000) 2024-03-06T18:54:51.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:51.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:51.052847+0000) 2024-03-06T18:54:52.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:52.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:52.053042+0000) 2024-03-06T18:54:53.047 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:53.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:53.053266+0000) 2024-03-06T18:54:54.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:54.052+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:54.053474+0000) 2024-03-06T18:54:55.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:55.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:55.053686+0000) 2024-03-06T18:54:56.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:56.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:56.053888+0000) 2024-03-06T18:54:57.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:57.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:57.054087+0000) 2024-03-06T18:54:58.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:58.053+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:58.054318+0000) 2024-03-06T18:54:59.048 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:54:59.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:54:59.054569+0000) 2024-03-06T18:55:00.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:00.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:00.054791+0000) 2024-03-06T18:55:01.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:01.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:01.055030+0000) 2024-03-06T18:55:02.049 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:02.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:02.055250+0000) 2024-03-06T18:55:03.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:03.054+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:03.055493+0000) 2024-03-06T18:55:04.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:04.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:04.055734+0000) 2024-03-06T18:55:05.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:05.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:05.056001+0000) 2024-03-06T18:55:06.050 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:06.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:06.056311+0000) 2024-03-06T18:55:07.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:07.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:07.056574+0000) 2024-03-06T18:55:08.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:08.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:08.056801+0000) 2024-03-06T18:55:09.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:09.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:09.056996+0000) 2024-03-06T18:55:10.051 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:10.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:10.057204+0000) 2024-03-06T18:55:11.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:11.055+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:11.057460+0000) 2024-03-06T18:55:12.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:12.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:12.057690+0000) 2024-03-06T18:55:13.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:13.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:13.057914+0000) 2024-03-06T18:55:14.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:14.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:14.058106+0000) 2024-03-06T18:55:15.052 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:15.056+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:15.058317+0000) 2024-03-06T18:55:16.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:16.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:16.058597+0000) 2024-03-06T18:55:17.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:17.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:17.058789+0000) 2024-03-06T18:55:18.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:18.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:18.058986+0000) 2024-03-06T18:55:19.053 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:19.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:19.059205+0000) 2024-03-06T18:55:19.108 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:55:19.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:55:20.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:20.057+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:20.059432+0000) 2024-03-06T18:55:21.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:21.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:21.059687+0000) 2024-03-06T18:55:22.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:22.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:22.059920+0000) 2024-03-06T18:55:23.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:23.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:23.060116+0000) 2024-03-06T18:55:24.054 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:24.058+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:24.060337+0000) 2024-03-06T18:55:25.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:25.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:25.060595+0000) 2024-03-06T18:55:26.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:26.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:26.060777+0000) 2024-03-06T18:55:27.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:27.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:27.060970+0000) 2024-03-06T18:55:28.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:28.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:28.061174+0000) 2024-03-06T18:55:29.055 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:29.059+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:29.061413+0000) 2024-03-06T18:55:30.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:30.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:30.061673+0000) 2024-03-06T18:55:31.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:31.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:31.061938+0000) 2024-03-06T18:55:32.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:32.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:32.062212+0000) 2024-03-06T18:55:33.056 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:33.061+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:33.062486+0000) 2024-03-06T18:55:34.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:34.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:34.062745+0000) 2024-03-06T18:55:35.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:35.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:35.062980+0000) 2024-03-06T18:55:36.057 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:36.062+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:36.063257+0000) 2024-03-06T18:55:37.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:37.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:37.063577+0000) 2024-03-06T18:55:38.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:38.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:38.063843+0000) 2024-03-06T18:55:39.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:39.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:39.064056+0000) 2024-03-06T18:55:40.058 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:40.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:40.064243+0000) 2024-03-06T18:55:41.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:41.063+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:41.064469+0000) 2024-03-06T18:55:42.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:42.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:42.064712+0000) 2024-03-06T18:55:43.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:43.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:43.064936+0000) 2024-03-06T18:55:44.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:44.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:44.065156+0000) 2024-03-06T18:55:45.059 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:45.064+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:45.065360+0000) 2024-03-06T18:55:46.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:46.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:46.065585+0000) 2024-03-06T18:55:47.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:47.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:47.065772+0000) 2024-03-06T18:55:48.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:48.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:48.066003+0000) 2024-03-06T18:55:49.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:49.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:49.066225+0000) 2024-03-06T18:55:49.243 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:55:49.252 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:55:50.060 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:50.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:50.066416+0000) 2024-03-06T18:55:51.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:51.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:51.066661+0000) 2024-03-06T18:55:52.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:52.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:52.066865+0000) 2024-03-06T18:55:53.061 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:53.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:53.067051+0000) 2024-03-06T18:55:54.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:54.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:54.067281+0000) 2024-03-06T18:55:55.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:55.065+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:55.067479+0000) 2024-03-06T18:55:56.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:56.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:56.067690+0000) 2024-03-06T18:55:57.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:57.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:57.067871+0000) 2024-03-06T18:55:58.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:58.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:58.068069+0000) 2024-03-06T18:55:59.062 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:55:59.066+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:55:59.068301+0000) 2024-03-06T18:56:00.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:00.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:00.068526+0000) 2024-03-06T18:56:01.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:01.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:01.068793+0000) 2024-03-06T18:56:02.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:02.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:02.068990+0000) 2024-03-06T18:56:03.063 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:03.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:03.069217+0000) 2024-03-06T18:56:04.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:04.067+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:04.069438+0000) 2024-03-06T18:56:05.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:05.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:05.069659+0000) 2024-03-06T18:56:06.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:06.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:06.069866+0000) 2024-03-06T18:56:07.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:07.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:07.070055+0000) 2024-03-06T18:56:08.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:08.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:08.070246+0000) 2024-03-06T18:56:09.064 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:09.068+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:09.070469+0000) 2024-03-06T18:56:10.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:10.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:10.070689+0000) 2024-03-06T18:56:11.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:11.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:11.070883+0000) 2024-03-06T18:56:12.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:12.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:12.071084+0000) 2024-03-06T18:56:13.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:13.069+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:13.071318+0000) 2024-03-06T18:56:14.065 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:14.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:14.071575+0000) 2024-03-06T18:56:15.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:15.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:15.071825+0000) 2024-03-06T18:56:16.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:16.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:16.072057+0000) 2024-03-06T18:56:17.066 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:17.070+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:17.072329+0000) 2024-03-06T18:56:18.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:18.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:18.072637+0000) 2024-03-06T18:56:19.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:19.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:19.072863+0000) 2024-03-06T18:56:19.446 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:56:19.455 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:56:20.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:20.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:20.073070+0000) 2024-03-06T18:56:21.067 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:21.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:21.073293+0000) 2024-03-06T18:56:22.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:22.072+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:22.073494+0000) 2024-03-06T18:56:23.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:23.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:23.073771+0000) 2024-03-06T18:56:24.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:24.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:24.074010+0000) 2024-03-06T18:56:25.068 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:25.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:25.074254+0000) 2024-03-06T18:56:26.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:26.073+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:26.074505+0000) 2024-03-06T18:56:27.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:27.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:27.074773+0000) 2024-03-06T18:56:28.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:28.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:28.075011+0000) 2024-03-06T18:56:29.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:29.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:29.075205+0000) 2024-03-06T18:56:30.069 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:30.074+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:30.075436+0000) 2024-03-06T18:56:31.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:31.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:31.075679+0000) 2024-03-06T18:56:32.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:32.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:32.075889+0000) 2024-03-06T18:56:33.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:33.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:33.076091+0000) 2024-03-06T18:56:34.070 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:34.075+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:34.076329+0000) 2024-03-06T18:56:35.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:35.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:35.076598+0000) 2024-03-06T18:56:36.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:36.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:36.076816+0000) 2024-03-06T18:56:37.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:37.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:37.077020+0000) 2024-03-06T18:56:38.071 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:38.076+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:38.077296+0000) 2024-03-06T18:56:39.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:39.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:39.077637+0000) 2024-03-06T18:56:40.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:40.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:40.077884+0000) 2024-03-06T18:56:41.072 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:41.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:41.078090+0000) 2024-03-06T18:56:42.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:42.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:42.078418+0000) 2024-03-06T18:56:43.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:43.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:43.078675+0000) 2024-03-06T18:56:44.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:44.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:44.078865+0000) 2024-03-06T18:56:45.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:45.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:45.079053+0000) 2024-03-06T18:56:46.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:46.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:46.079244+0000) 2024-03-06T18:56:47.073 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:47.077+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:47.079470+0000) 2024-03-06T18:56:48.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:48.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:48.079665+0000) 2024-03-06T18:56:49.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:49.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:49.079866+0000) 2024-03-06T18:56:49.594 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:56:49.603 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:56:50.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:50.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:50.080110+0000) 2024-03-06T18:56:51.074 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:51.078+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:51.080356+0000) 2024-03-06T18:56:52.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:52.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:52.080639+0000) 2024-03-06T18:56:53.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:53.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:53.080881+0000) 2024-03-06T18:56:54.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:54.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:54.081128+0000) 2024-03-06T18:56:55.075 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:55.079+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:55.081406+0000) 2024-03-06T18:56:56.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:56.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:56.081697+0000) 2024-03-06T18:56:57.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:57.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:57.081934+0000) 2024-03-06T18:56:58.076 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:58.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:58.082164+0000) 2024-03-06T18:56:59.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:56:59.080+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:56:59.082424+0000) 2024-03-06T18:57:00.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:00.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:00.082704+0000) 2024-03-06T18:57:01.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:01.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:01.082949+0000) 2024-03-06T18:57:02.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:02.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:02.083189+0000) 2024-03-06T18:57:03.077 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:03.081+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:03.083459+0000) 2024-03-06T18:57:04.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:04.082+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:04.083726+0000) 2024-03-06T18:57:05.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:05.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:05.083941+0000) 2024-03-06T18:57:06.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:06.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:06.084208+0000) 2024-03-06T18:57:07.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:07.083+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:07.084434+0000) 2024-03-06T18:57:08.078 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:08.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:08.084658+0000) 2024-03-06T18:57:09.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:09.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:09.084852+0000) 2024-03-06T18:57:10.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:10.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:10.085084+0000) 2024-03-06T18:57:11.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:11.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:11.085279+0000) 2024-03-06T18:57:12.079 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:12.084+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:12.085517+0000) 2024-03-06T18:57:13.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:13.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:13.085775+0000) 2024-03-06T18:57:14.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:14.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:14.086006+0000) 2024-03-06T18:57:15.080 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:15.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:15.086239+0000) 2024-03-06T18:57:16.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:16.085+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:16.086482+0000) 2024-03-06T18:57:17.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:17.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:17.086722+0000) 2024-03-06T18:57:18.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:18.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:18.086978+0000) 2024-03-06T18:57:19.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:19.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:19.087205+0000) 2024-03-06T18:57:19.732 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:57:19.741 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:57:20.081 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:20.086+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:20.087438+0000) 2024-03-06T18:57:21.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:21.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:21.087691+0000) 2024-03-06T18:57:22.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:22.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:22.087884+0000) 2024-03-06T18:57:23.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:23.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:23.088108+0000) 2024-03-06T18:57:24.082 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:24.087+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:24.088336+0000) 2024-03-06T18:57:25.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:25.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:25.088645+0000) 2024-03-06T18:57:26.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:26.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:26.088907+0000) 2024-03-06T18:57:27.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:27.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:27.089129+0000) 2024-03-06T18:57:28.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:28.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:28.089344+0000) 2024-03-06T18:57:29.083 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:29.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:29.089607+0000) 2024-03-06T18:57:30.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:30.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:30.089813+0000) 2024-03-06T18:57:31.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:31.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:31.090057+0000) 2024-03-06T18:57:32.084 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:32.088+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:32.090295+0000) 2024-03-06T18:57:33.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:33.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:33.090559+0000) 2024-03-06T18:57:34.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:34.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:34.090806+0000) 2024-03-06T18:57:35.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:35.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:35.091032+0000) 2024-03-06T18:57:36.085 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:36.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:36.091270+0000) 2024-03-06T18:57:37.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:37.089+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:37.091506+0000) 2024-03-06T18:57:38.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:38.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:38.091767+0000) 2024-03-06T18:57:39.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:39.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:39.091992+0000) 2024-03-06T18:57:40.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:40.090+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:40.092271+0000) 2024-03-06T18:57:41.086 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:41.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:41.092527+0000) 2024-03-06T18:57:42.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:42.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:42.092782+0000) 2024-03-06T18:57:43.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:43.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:43.093021+0000) 2024-03-06T18:57:44.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:44.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:44.093198+0000) 2024-03-06T18:57:45.087 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:45.091+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:45.093384+0000) 2024-03-06T18:57:46.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:46.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:46.093655+0000) 2024-03-06T18:57:47.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:47.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:47.093885+0000) 2024-03-06T18:57:48.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:48.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:48.094110+0000) 2024-03-06T18:57:49.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:49.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:49.094288+0000) 2024-03-06T18:57:49.870 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:57:49.878 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:57:50.088 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:50.092+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:50.094389+0000) 2024-03-06T18:57:51.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:51.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:51.094656+0000) 2024-03-06T18:57:52.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:52.093+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:52.094865+0000) 2024-03-06T18:57:53.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:53.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:53.095016+0000) 2024-03-06T18:57:54.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:54.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:54.095256+0000) 2024-03-06T18:57:55.089 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:55.094+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:55.095483+0000) 2024-03-06T18:57:56.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:56.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:56.095689+0000) 2024-03-06T18:57:57.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:57.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:57.095893+0000) 2024-03-06T18:57:58.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:58.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:58.096121+0000) 2024-03-06T18:57:59.090 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:57:59.095+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:57:59.096351+0000) 2024-03-06T18:58:00.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:00.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:00.096623+0000) 2024-03-06T18:58:01.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:01.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:01.096850+0000) 2024-03-06T18:58:02.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:02.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:02.097024+0000) 2024-03-06T18:58:03.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:03.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:03.097220+0000) 2024-03-06T18:58:04.091 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:04.096+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:04.097462+0000) 2024-03-06T18:58:05.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:05.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:05.097704+0000) 2024-03-06T18:58:06.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:06.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:06.097943+0000) 2024-03-06T18:58:07.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:07.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:07.098179+0000) 2024-03-06T18:58:08.092 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:08.097+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:08.098399+0000) 2024-03-06T18:58:09.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:09.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:09.098644+0000) 2024-03-06T18:58:10.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:10.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:10.098870+0000) 2024-03-06T18:58:11.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:11.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:11.099073+0000) 2024-03-06T18:58:12.093 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:12.098+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:12.099301+0000) 2024-03-06T18:58:13.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:13.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:13.099566+0000) 2024-03-06T18:58:14.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:14.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:14.099745+0000) 2024-03-06T18:58:15.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:15.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:15.099984+0000) 2024-03-06T18:58:16.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:16.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:16.100208+0000) 2024-03-06T18:58:17.094 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:17.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:17.100422+0000) 2024-03-06T18:58:18.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:18.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:18.100645+0000) 2024-03-06T18:58:19.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:19.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:19.100886+0000) 2024-03-06T18:58:20.012 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:58:20.020 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:58:20.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:20.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:20.101051+0000) 2024-03-06T18:58:21.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:21.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:21.101275+0000) 2024-03-06T18:58:22.095 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:22.099+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:22.101503+0000) 2024-03-06T18:58:23.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:23.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:23.101931+0000) 2024-03-06T18:58:24.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:24.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:24.102171+0000) 2024-03-06T18:58:25.096 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:25.100+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:25.102411+0000) 2024-03-06T18:58:26.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:26.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:26.102636+0000) 2024-03-06T18:58:27.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:27.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:27.102836+0000) 2024-03-06T18:58:28.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:28.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:28.103061+0000) 2024-03-06T18:58:29.097 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:29.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:29.103269+0000) 2024-03-06T18:58:30.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:30.101+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:30.103511+0000) 2024-03-06T18:58:31.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:31.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:31.103815+0000) 2024-03-06T18:58:32.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:32.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:32.104085+0000) 2024-03-06T18:58:33.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:33.102+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:33.104310+0000) 2024-03-06T18:58:34.098 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:34.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:34.104577+0000) 2024-03-06T18:58:35.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:35.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:35.104838+0000) 2024-03-06T18:58:36.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:36.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:36.105063+0000) 2024-03-06T18:58:37.099 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:37.103+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:37.105287+0000) 2024-03-06T18:58:38.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:38.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:38.105578+0000) 2024-03-06T18:58:39.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:39.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:39.105755+0000) 2024-03-06T18:58:40.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:40.104+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:40.105979+0000) 2024-03-06T18:58:41.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:41.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:41.106204+0000) 2024-03-06T18:58:42.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:42.105+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:42.106450+0000) 2024-03-06T18:58:43.100 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:43.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:43.106637+0000) 2024-03-06T18:58:44.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:44.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:44.106869+0000) 2024-03-06T18:58:45.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:45.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:45.107097+0000) 2024-03-06T18:58:46.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:46.106+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:46.107320+0000) 2024-03-06T18:58:47.101 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:47.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:47.107604+0000) 2024-03-06T18:58:48.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:48.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:48.107846+0000) 2024-03-06T18:58:49.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:49.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:49.108054+0000) 2024-03-06T18:58:50.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:50.107+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:50.108302+0000) 2024-03-06T18:58:50.154 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:58:50.162 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:58:51.102 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:51.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:51.108585+0000) 2024-03-06T18:58:52.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:52.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:52.108852+0000) 2024-03-06T18:58:53.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:53.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:53.109091+0000) 2024-03-06T18:58:54.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:54.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:54.109319+0000) 2024-03-06T18:58:55.103 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:55.108+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:55.109516+0000) 2024-03-06T18:58:56.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:56.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:56.109778+0000) 2024-03-06T18:58:57.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:57.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:57.110026+0000) 2024-03-06T18:58:58.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:58.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:58.110266+0000) 2024-03-06T18:58:59.104 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:58:59.109+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:58:59.110482+0000) 2024-03-06T18:59:00.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:00.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:00.110730+0000) 2024-03-06T18:59:01.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:01.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:01.110910+0000) 2024-03-06T18:59:02.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:02.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:02.111125+0000) 2024-03-06T18:59:03.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:03.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:03.111299+0000) 2024-03-06T18:59:04.105 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:04.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:04.111510+0000) 2024-03-06T18:59:05.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:05.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:05.111743+0000) 2024-03-06T18:59:06.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:06.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:06.111943+0000) 2024-03-06T18:59:07.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:07.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:07.112155+0000) 2024-03-06T18:59:08.106 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:08.110+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:08.112365+0000) 2024-03-06T18:59:09.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:09.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:09.112568+0000) 2024-03-06T18:59:10.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:10.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:10.112775+0000) 2024-03-06T18:59:11.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:11.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:11.113013+0000) 2024-03-06T18:59:12.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:12.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:12.113244+0000) 2024-03-06T18:59:13.107 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:13.111+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:13.113467+0000) 2024-03-06T18:59:14.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:14.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:14.113703+0000) 2024-03-06T18:59:15.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:15.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:15.113919+0000) 2024-03-06T18:59:16.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:16.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:16.114196+0000) 2024-03-06T18:59:17.108 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:17.112+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:17.114435+0000) 2024-03-06T18:59:18.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:18.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:18.114664+0000) 2024-03-06T18:59:19.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:19.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:19.114885+0000) 2024-03-06T18:59:20.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:20.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:20.115086+0000) 2024-03-06T18:59:20.289 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:59:20.297 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:59:21.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:21.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:21.115261+0000) 2024-03-06T18:59:22.109 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:22.113+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:22.115519+0000) 2024-03-06T18:59:23.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:23.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:23.115778+0000) 2024-03-06T18:59:24.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:24.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:24.116004+0000) 2024-03-06T18:59:25.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:25.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:25.116159+0000) 2024-03-06T18:59:26.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:26.114+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:26.116326+0000) 2024-03-06T18:59:27.110 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:27.115+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:27.116569+0000) 2024-03-06T18:59:28.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:28.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:28.116819+0000) 2024-03-06T18:59:29.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:29.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:29.117048+0000) 2024-03-06T18:59:30.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:30.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:30.117276+0000) 2024-03-06T18:59:31.111 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:31.116+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:31.117466+0000) 2024-03-06T18:59:32.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:32.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:32.117708+0000) 2024-03-06T18:59:33.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:33.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:33.117893+0000) 2024-03-06T18:59:34.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:34.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:34.118133+0000) 2024-03-06T18:59:35.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:35.117+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:35.118340+0000) 2024-03-06T18:59:36.112 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:36.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:36.118612+0000) 2024-03-06T18:59:37.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:37.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:37.118845+0000) 2024-03-06T18:59:38.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:38.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:38.119005+0000) 2024-03-06T18:59:39.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:39.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:39.119240+0000) 2024-03-06T18:59:40.113 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:40.118+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:40.119465+0000) 2024-03-06T18:59:41.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:41.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:41.119690+0000) 2024-03-06T18:59:42.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:42.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:42.119924+0000) 2024-03-06T18:59:43.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:43.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:43.120150+0000) 2024-03-06T18:59:44.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:44.119+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:44.120375+0000) 2024-03-06T18:59:45.114 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:45.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:45.120647+0000) 2024-03-06T18:59:46.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:46.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:46.120800+0000) 2024-03-06T18:59:47.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:47.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:47.121038+0000) 2024-03-06T18:59:48.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:48.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:48.121260+0000) 2024-03-06T18:59:49.115 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:49.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:49.121488+0000) 2024-03-06T18:59:50.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:50.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:50.121713+0000) 2024-03-06T18:59:50.430 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:59:50.440 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T18:59:51.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:51.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:51.121853+0000) 2024-03-06T18:59:52.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:52.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:52.122021+0000) 2024-03-06T18:59:53.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:53.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:53.122219+0000) 2024-03-06T18:59:54.116 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:54.120+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:54.122458+0000) 2024-03-06T18:59:55.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:55.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:55.122697+0000) 2024-03-06T18:59:56.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:56.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:56.122887+0000) 2024-03-06T18:59:57.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:57.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:57.123113+0000) 2024-03-06T18:59:58.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:58.121+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:58.123363+0000) 2024-03-06T18:59:59.117 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T18:59:59.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T17:59:59.123526+0000) 2024-03-06T19:00:00.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:00.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:00.123798+0000) 2024-03-06T19:00:01.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:01.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:01.124026+0000) 2024-03-06T19:00:02.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:02.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:02.124268+0000) 2024-03-06T19:00:03.118 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:03.122+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:03.124441+0000) 2024-03-06T19:00:04.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:04.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:04.124688+0000) 2024-03-06T19:00:05.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:05.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:05.124913+0000) 2024-03-06T19:00:06.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:06.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:06.125143+0000) 2024-03-06T19:00:07.119 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:07.123+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:07.125419+0000) 2024-03-06T19:00:08.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:08.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:08.125696+0000) 2024-03-06T19:00:09.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:09.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:09.125960+0000) 2024-03-06T19:00:10.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:10.124+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:10.126281+0000) 2024-03-06T19:00:11.120 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:11.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:11.126590+0000) 2024-03-06T19:00:12.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:12.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:12.126904+0000) 2024-03-06T19:00:13.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:13.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:13.127129+0000) 2024-03-06T19:00:14.121 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:14.125+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:14.127387+0000) 2024-03-06T19:00:15.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:15.126+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:15.127664+0000) 2024-03-06T19:00:16.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:16.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:16.127889+0000) 2024-03-06T19:00:17.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:17.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:17.128084+0000) 2024-03-06T19:00:18.122 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:18.127+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:18.128323+0000) 2024-03-06T19:00:19.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:19.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:19.128632+0000) 2024-03-06T19:00:20.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:20.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:20.128901+0000) 2024-03-06T19:00:20.569 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:00:20.577 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:00:21.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:21.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:21.129187+0000) 2024-03-06T19:00:22.123 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:22.128+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:22.129488+0000) 2024-03-06T19:00:23.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:23.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:23.129649+0000) 2024-03-06T19:00:24.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:24.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:24.129882+0000) 2024-03-06T19:00:25.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:25.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:25.130119+0000) 2024-03-06T19:00:26.124 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:26.129+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:26.130352+0000) 2024-03-06T19:00:27.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:27.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:27.130613+0000) 2024-03-06T19:00:28.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:28.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:28.130836+0000) 2024-03-06T19:00:29.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:29.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:29.131044+0000) 2024-03-06T19:00:30.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:30.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:30.131298+0000) 2024-03-06T19:00:31.125 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:31.130+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:31.131514+0000) 2024-03-06T19:00:32.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:32.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:32.131794+0000) 2024-03-06T19:00:33.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:33.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:33.132023+0000) 2024-03-06T19:00:34.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:34.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:34.132174+0000) 2024-03-06T19:00:35.126 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:35.131+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:35.132410+0000) 2024-03-06T19:00:36.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:36.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:36.132657+0000) 2024-03-06T19:00:37.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:37.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:37.132823+0000) 2024-03-06T19:00:38.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:38.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:38.133054+0000) 2024-03-06T19:00:39.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:39.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:39.133202+0000) 2024-03-06T19:00:40.127 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:40.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:40.133373+0000) 2024-03-06T19:00:41.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:41.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:41.133685+0000) 2024-03-06T19:00:42.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:42.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:42.133898+0000) 2024-03-06T19:00:43.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:43.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:43.134137+0000) 2024-03-06T19:00:44.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:44.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:44.134252+0000) 2024-03-06T19:00:45.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:45.132+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:45.134478+0000) 2024-03-06T19:00:46.128 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:46.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:46.134693+0000) 2024-03-06T19:00:47.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:47.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:47.134825+0000) 2024-03-06T19:00:48.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:48.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:48.135025+0000) 2024-03-06T19:00:49.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:49.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:49.135252+0000) 2024-03-06T19:00:50.129 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:50.133+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:50.135514+0000) 2024-03-06T19:00:50.697 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:00:50.705 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:00:51.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:51.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:51.135841+0000) 2024-03-06T19:00:52.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:52.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:52.136120+0000) 2024-03-06T19:00:53.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:53.134+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:53.136390+0000) 2024-03-06T19:00:54.130 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:54.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:54.136638+0000) 2024-03-06T19:00:55.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:55.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:55.136853+0000) 2024-03-06T19:00:56.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:56.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:56.137024+0000) 2024-03-06T19:00:57.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:57.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:57.137261+0000) 2024-03-06T19:00:58.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:58.135+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:58.137493+0000) 2024-03-06T19:00:59.131 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:00:59.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:00:59.137663+0000) 2024-03-06T19:01:00.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:00.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:00.137904+0000) 2024-03-06T19:01:01.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:01.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:01.138097+0000) 2024-03-06T19:01:02.132 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:02.136+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:02.138322+0000) 2024-03-06T19:01:03.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:03.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:03.138596+0000) 2024-03-06T19:01:04.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:04.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:04.138803+0000) 2024-03-06T19:01:05.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:05.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:05.139025+0000) 2024-03-06T19:01:06.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:06.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:06.139252+0000) 2024-03-06T19:01:07.133 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:07.138+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:07.139475+0000) 2024-03-06T19:01:08.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:08.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:08.139697+0000) 2024-03-06T19:01:09.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:09.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:09.139924+0000) 2024-03-06T19:01:10.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:10.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:10.140153+0000) 2024-03-06T19:01:11.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:11.139+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:11.140315+0000) 2024-03-06T19:01:12.134 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:12.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:12.140632+0000) 2024-03-06T19:01:13.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:13.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:13.140914+0000) 2024-03-06T19:01:14.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:14.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:14.141180+0000) 2024-03-06T19:01:15.135 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:15.140+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:15.141461+0000) 2024-03-06T19:01:16.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:16.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:16.141752+0000) 2024-03-06T19:01:17.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:17.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:17.141931+0000) 2024-03-06T19:01:18.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:18.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:18.142191+0000) 2024-03-06T19:01:19.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:19.141+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:19.142438+0000) 2024-03-06T19:01:20.136 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:20.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:20.142679+0000) 2024-03-06T19:01:20.827 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:01:20.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:01:21.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:21.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:21.142940+0000) 2024-03-06T19:01:22.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:22.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:22.143223+0000) 2024-03-06T19:01:23.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:23.142+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:23.143439+0000) 2024-03-06T19:01:24.137 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:24.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:24.143694+0000) 2024-03-06T19:01:25.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:25.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:25.143931+0000) 2024-03-06T19:01:26.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:26.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:26.144134+0000) 2024-03-06T19:01:27.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:27.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:27.144337+0000) 2024-03-06T19:01:28.138 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:28.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:28.144569+0000) 2024-03-06T19:01:29.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:29.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:29.144900+0000) 2024-03-06T19:01:30.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:30.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:30.145134+0000) 2024-03-06T19:01:31.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:31.143+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:31.145362+0000) 2024-03-06T19:01:32.139 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:32.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:32.145627+0000) 2024-03-06T19:01:33.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:33.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:33.145816+0000) 2024-03-06T19:01:34.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:34.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:34.146042+0000) 2024-03-06T19:01:35.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:35.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:35.146267+0000) 2024-03-06T19:01:36.140 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:36.144+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:36.146493+0000) 2024-03-06T19:01:37.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:37.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:37.146764+0000) 2024-03-06T19:01:38.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:38.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:38.147003+0000) 2024-03-06T19:01:39.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:39.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:39.147246+0000) 2024-03-06T19:01:40.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:40.145+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:40.147478+0000) 2024-03-06T19:01:41.141 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:41.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:41.147687+0000) 2024-03-06T19:01:42.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:42.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:42.147855+0000) 2024-03-06T19:01:43.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:43.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:43.148020+0000) 2024-03-06T19:01:44.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:44.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:44.148229+0000) 2024-03-06T19:01:45.142 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:45.146+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:45.148461+0000) 2024-03-06T19:01:46.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:46.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:46.148693+0000) 2024-03-06T19:01:47.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:47.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:47.148898+0000) 2024-03-06T19:01:48.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:48.147+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:48.149103+0000) 2024-03-06T19:01:49.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:49.148+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:49.149308+0000) 2024-03-06T19:01:50.143 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:50.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:50.149576+0000) 2024-03-06T19:01:51.055 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:01:51.062 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:01:51.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:51.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:51.149764+0000) 2024-03-06T19:01:52.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:52.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:52.149968+0000) 2024-03-06T19:01:53.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:53.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:53.150207+0000) 2024-03-06T19:01:54.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:54.149+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:54.150364+0000) 2024-03-06T19:01:55.144 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:55.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:55.150643+0000) 2024-03-06T19:01:56.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:56.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:56.150874+0000) 2024-03-06T19:01:57.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:57.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:57.151096+0000) 2024-03-06T19:01:58.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:58.150+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:58.151321+0000) 2024-03-06T19:01:59.145 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:01:59.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:01:59.151561+0000) 2024-03-06T19:02:00.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:00.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:00.151777+0000) 2024-03-06T19:02:01.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:01.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:01.151929+0000) 2024-03-06T19:02:02.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:02.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:02.152184+0000) 2024-03-06T19:02:03.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:03.151+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:03.152411+0000) 2024-03-06T19:02:04.146 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:04.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:04.152616+0000) 2024-03-06T19:02:05.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:05.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:05.152834+0000) 2024-03-06T19:02:06.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:06.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:06.153026+0000) 2024-03-06T19:02:07.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:07.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:07.153258+0000) 2024-03-06T19:02:08.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:08.152+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:08.153476+0000) 2024-03-06T19:02:09.147 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:09.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:09.153692+0000) 2024-03-06T19:02:10.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:10.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:10.153911+0000) 2024-03-06T19:02:11.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:11.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:11.154135+0000) 2024-03-06T19:02:12.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:12.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:12.154364+0000) 2024-03-06T19:02:13.148 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:13.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:13.154630+0000) 2024-03-06T19:02:14.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:14.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:14.154852+0000) 2024-03-06T19:02:15.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:15.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:15.154997+0000) 2024-03-06T19:02:16.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:16.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:16.155159+0000) 2024-03-06T19:02:17.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:17.153+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:17.155363+0000) 2024-03-06T19:02:18.149 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:18.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:18.155621+0000) 2024-03-06T19:02:19.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:19.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:19.155847+0000) 2024-03-06T19:02:20.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:20.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:20.156072+0000) 2024-03-06T19:02:21.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:21.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:21.156298+0000) 2024-03-06T19:02:21.192 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:02:21.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:02:22.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:22.154+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:22.156451+0000) 2024-03-06T19:02:23.150 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:23.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:23.156644+0000) 2024-03-06T19:02:24.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:24.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:24.156875+0000) 2024-03-06T19:02:25.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:25.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:25.157096+0000) 2024-03-06T19:02:26.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:26.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:26.157255+0000) 2024-03-06T19:02:27.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:27.155+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:27.157504+0000) 2024-03-06T19:02:28.151 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:28.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:28.157734+0000) 2024-03-06T19:02:29.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:29.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:29.157924+0000) 2024-03-06T19:02:30.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:30.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:30.158115+0000) 2024-03-06T19:02:31.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:31.156+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:31.158313+0000) 2024-03-06T19:02:32.152 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:32.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:32.158575+0000) 2024-03-06T19:02:33.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:33.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:33.158800+0000) 2024-03-06T19:02:34.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:34.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:34.158992+0000) 2024-03-06T19:02:35.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:35.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:35.159197+0000) 2024-03-06T19:02:36.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:36.157+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:36.159401+0000) 2024-03-06T19:02:37.153 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:37.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:37.159655+0000) 2024-03-06T19:02:38.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:38.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:38.159875+0000) 2024-03-06T19:02:39.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:39.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:39.160162+0000) 2024-03-06T19:02:40.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:40.159+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:40.160357+0000) 2024-03-06T19:02:41.154 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:41.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:41.160613+0000) 2024-03-06T19:02:42.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:42.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:42.160826+0000) 2024-03-06T19:02:43.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:43.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:43.161048+0000) 2024-03-06T19:02:44.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:44.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:44.161270+0000) 2024-03-06T19:02:45.155 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:45.160+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:45.161501+0000) 2024-03-06T19:02:46.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:46.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:46.161735+0000) 2024-03-06T19:02:47.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:47.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:47.161965+0000) 2024-03-06T19:02:48.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:48.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:48.162152+0000) 2024-03-06T19:02:49.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:49.161+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:49.162390+0000) 2024-03-06T19:02:50.156 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:50.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:50.162654+0000) 2024-03-06T19:02:51.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:51.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:51.162870+0000) 2024-03-06T19:02:51.406 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:02:51.414 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:02:52.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:52.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:52.163101+0000) 2024-03-06T19:02:53.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:53.162+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:53.163333+0000) 2024-03-06T19:02:54.157 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:54.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:54.163589+0000) 2024-03-06T19:02:55.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:55.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:55.163823+0000) 2024-03-06T19:02:56.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:56.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:56.164052+0000) 2024-03-06T19:02:57.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:57.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:57.164275+0000) 2024-03-06T19:02:58.158 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:58.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:58.164496+0000) 2024-03-06T19:02:59.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:02:59.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:02:59.164723+0000) 2024-03-06T19:03:00.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:00.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:00.164951+0000) 2024-03-06T19:03:01.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:01.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:01.165114+0000) 2024-03-06T19:03:02.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:02.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:02.165323+0000) 2024-03-06T19:03:03.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:03.163+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:03.165492+0000) 2024-03-06T19:03:04.159 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:04.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:04.165757+0000) 2024-03-06T19:03:05.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:05.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:05.165985+0000) 2024-03-06T19:03:06.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:06.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:06.166216+0000) 2024-03-06T19:03:07.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:07.164+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:07.166446+0000) 2024-03-06T19:03:08.160 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:08.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:08.166664+0000) 2024-03-06T19:03:09.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:09.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:09.166876+0000) 2024-03-06T19:03:10.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:10.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:10.167095+0000) 2024-03-06T19:03:11.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:11.165+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:11.167319+0000) 2024-03-06T19:03:12.161 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:12.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:12.167592+0000) 2024-03-06T19:03:13.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:13.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:13.167834+0000) 2024-03-06T19:03:14.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:14.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:14.168065+0000) 2024-03-06T19:03:15.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:15.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:15.168262+0000) 2024-03-06T19:03:16.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:16.166+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:16.168507+0000) 2024-03-06T19:03:17.162 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:17.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:17.168702+0000) 2024-03-06T19:03:18.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:18.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:18.168882+0000) 2024-03-06T19:03:19.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:19.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:19.169110+0000) 2024-03-06T19:03:20.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:20.167+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:20.169333+0000) 2024-03-06T19:03:21.163 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:21.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:21.169590+0000) 2024-03-06T19:03:21.546 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:03:21.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:03:22.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:22.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:22.169802+0000) 2024-03-06T19:03:23.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:23.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:23.170009+0000) 2024-03-06T19:03:24.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:24.168+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:24.170275+0000) 2024-03-06T19:03:25.164 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:25.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:25.170590+0000) 2024-03-06T19:03:26.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:26.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:26.170851+0000) 2024-03-06T19:03:27.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:27.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:27.171169+0000) 2024-03-06T19:03:28.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:28.170+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:28.171439+0000) 2024-03-06T19:03:29.165 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:29.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:29.171731+0000) 2024-03-06T19:03:30.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:30.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:30.171967+0000) 2024-03-06T19:03:31.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:31.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:31.172202+0000) 2024-03-06T19:03:32.166 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:32.171+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:32.172487+0000) 2024-03-06T19:03:33.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:33.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:33.172769+0000) 2024-03-06T19:03:34.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:34.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:34.173014+0000) 2024-03-06T19:03:35.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:35.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:35.173286+0000) 2024-03-06T19:03:36.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:36.172+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:36.173495+0000) 2024-03-06T19:03:37.167 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:37.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:37.173760+0000) 2024-03-06T19:03:38.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:38.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:38.173987+0000) 2024-03-06T19:03:39.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:39.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:39.174240+0000) 2024-03-06T19:03:40.168 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:40.173+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:40.174514+0000) 2024-03-06T19:03:41.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:41.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:41.174836+0000) 2024-03-06T19:03:42.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:42.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:42.175131+0000) 2024-03-06T19:03:43.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:43.174+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:43.175414+0000) 2024-03-06T19:03:44.169 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:44.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:44.175678+0000) 2024-03-06T19:03:45.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:45.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:45.175953+0000) 2024-03-06T19:03:46.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:46.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:46.176168+0000) 2024-03-06T19:03:47.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:47.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:47.176445+0000) 2024-03-06T19:03:48.170 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:48.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:48.176702+0000) 2024-03-06T19:03:49.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:49.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:49.176986+0000) 2024-03-06T19:03:50.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:50.175+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:50.177265+0000) 2024-03-06T19:03:51.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:51.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:51.177574+0000) 2024-03-06T19:03:51.961 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:03:51.969 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:03:52.171 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:52.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:52.177866+0000) 2024-03-06T19:03:53.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:53.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:53.178040+0000) 2024-03-06T19:03:54.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:54.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:54.178282+0000) 2024-03-06T19:03:55.172 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:55.176+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:55.178515+0000) 2024-03-06T19:03:56.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:56.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:56.178794+0000) 2024-03-06T19:03:57.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:57.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:57.179032+0000) 2024-03-06T19:03:58.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:58.177+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:58.179278+0000) 2024-03-06T19:03:59.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:03:59.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:03:59.179519+0000) 2024-03-06T19:04:00.173 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:00.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:00.179719+0000) 2024-03-06T19:04:01.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:01.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:01.179961+0000) 2024-03-06T19:04:02.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:02.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:02.180162+0000) 2024-03-06T19:04:03.174 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:03.178+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:03.180402+0000) 2024-03-06T19:04:04.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:04.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:04.180658+0000) 2024-03-06T19:04:05.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:05.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:05.180890+0000) 2024-03-06T19:04:06.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:06.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:06.181116+0000) 2024-03-06T19:04:07.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:07.179+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:07.181337+0000) 2024-03-06T19:04:08.175 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:08.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:08.181598+0000) 2024-03-06T19:04:09.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:09.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:09.181829+0000) 2024-03-06T19:04:10.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:10.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:10.182108+0000) 2024-03-06T19:04:11.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:11.180+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:11.182314+0000) 2024-03-06T19:04:12.176 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:12.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:12.182615+0000) 2024-03-06T19:04:13.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:13.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:13.182828+0000) 2024-03-06T19:04:14.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:14.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:14.183056+0000) 2024-03-06T19:04:15.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:15.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:15.183250+0000) 2024-03-06T19:04:16.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:16.182+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:16.183482+0000) 2024-03-06T19:04:17.177 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:17.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:17.183700+0000) 2024-03-06T19:04:18.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:18.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:18.183896+0000) 2024-03-06T19:04:19.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:19.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:19.184123+0000) 2024-03-06T19:04:20.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:20.183+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:20.184395+0000) 2024-03-06T19:04:21.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:21.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:21.184640+0000) 2024-03-06T19:04:22.164 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:04:22.172 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:04:22.178 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:22.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:22.184875+0000) 2024-03-06T19:04:23.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:23.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:23.185082+0000) 2024-03-06T19:04:24.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:24.184+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:24.185285+0000) 2024-03-06T19:04:25.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:25.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:25.185599+0000) 2024-03-06T19:04:26.179 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:26.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:26.185870+0000) 2024-03-06T19:04:27.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:27.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:27.186132+0000) 2024-03-06T19:04:28.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:28.185+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:28.186394+0000) 2024-03-06T19:04:29.180 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:29.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:29.186674+0000) 2024-03-06T19:04:30.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:30.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:30.186918+0000) 2024-03-06T19:04:31.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:31.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:31.187185+0000) 2024-03-06T19:04:32.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:32.186+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:32.187468+0000) 2024-03-06T19:04:33.181 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:33.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:33.187733+0000) 2024-03-06T19:04:34.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:34.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:34.188008+0000) 2024-03-06T19:04:35.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:35.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:35.188282+0000) 2024-03-06T19:04:36.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:36.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:36.188591+0000) 2024-03-06T19:04:37.182 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:37.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:37.188851+0000) 2024-03-06T19:04:38.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:38.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:38.189056+0000) 2024-03-06T19:04:39.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:39.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:39.189292+0000) 2024-03-06T19:04:40.183 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:40.187+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:40.189518+0000) 2024-03-06T19:04:41.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:41.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:41.189756+0000) 2024-03-06T19:04:42.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:42.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:42.189941+0000) 2024-03-06T19:04:43.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:43.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:43.190175+0000) 2024-03-06T19:04:44.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:44.188+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:44.190401+0000) 2024-03-06T19:04:45.184 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:45.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:45.190649+0000) 2024-03-06T19:04:46.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:46.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:46.190849+0000) 2024-03-06T19:04:47.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:47.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:47.191083+0000) 2024-03-06T19:04:48.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:48.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:48.191310+0000) 2024-03-06T19:04:49.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:49.189+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:49.191511+0000) 2024-03-06T19:04:50.185 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:50.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:50.191792+0000) 2024-03-06T19:04:51.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:51.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:51.192021+0000) 2024-03-06T19:04:52.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:52.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:52.192259+0000) 2024-03-06T19:04:52.385 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:04:52.393 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:04:53.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:53.190+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:53.192490+0000) 2024-03-06T19:04:54.186 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:54.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:54.192717+0000) 2024-03-06T19:04:55.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:55.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:55.192942+0000) 2024-03-06T19:04:56.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:56.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:56.193123+0000) 2024-03-06T19:04:57.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:57.191+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:57.193349+0000) 2024-03-06T19:04:58.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:58.192+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:58.193620+0000) 2024-03-06T19:04:59.187 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:04:59.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:04:59.193851+0000) 2024-03-06T19:05:00.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:00.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:00.194075+0000) 2024-03-06T19:05:01.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:01.193+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:01.194295+0000) 2024-03-06T19:05:02.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:02.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:02.194567+0000) 2024-03-06T19:05:03.188 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:03.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:03.194772+0000) 2024-03-06T19:05:04.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:04.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:04.194958+0000) 2024-03-06T19:05:05.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:05.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:05.195189+0000) 2024-03-06T19:05:06.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:06.194+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:06.195409+0000) 2024-03-06T19:05:07.189 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:07.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:07.195645+0000) 2024-03-06T19:05:08.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:08.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:08.195949+0000) 2024-03-06T19:05:09.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:09.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:09.196177+0000) 2024-03-06T19:05:10.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:10.195+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:10.196323+0000) 2024-03-06T19:05:11.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:11.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:11.196615+0000) 2024-03-06T19:05:12.190 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:12.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:12.196827+0000) 2024-03-06T19:05:13.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:13.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:13.197067+0000) 2024-03-06T19:05:14.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:14.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:14.197295+0000) 2024-03-06T19:05:15.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:15.196+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:15.197474+0000) 2024-03-06T19:05:16.191 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:16.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:16.197669+0000) 2024-03-06T19:05:17.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:17.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:17.197925+0000) 2024-03-06T19:05:18.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:18.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:18.198150+0000) 2024-03-06T19:05:19.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:19.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:19.198304+0000) 2024-03-06T19:05:20.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:20.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:20.198573+0000) 2024-03-06T19:05:21.192 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:21.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:21.198767+0000) 2024-03-06T19:05:22.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:22.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:22.199006+0000) 2024-03-06T19:05:22.547 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:05:22.555 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:05:23.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:23.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:23.199194+0000) 2024-03-06T19:05:24.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:24.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:24.199354+0000) 2024-03-06T19:05:25.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:25.197+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:25.199516+0000) 2024-03-06T19:05:26.193 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:26.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:26.199703+0000) 2024-03-06T19:05:27.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:27.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:27.199933+0000) 2024-03-06T19:05:28.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:28.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:28.200160+0000) 2024-03-06T19:05:29.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:29.198+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:29.200385+0000) 2024-03-06T19:05:30.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:30.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:30.200658+0000) 2024-03-06T19:05:31.194 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:31.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:31.200861+0000) 2024-03-06T19:05:32.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:32.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:32.201091+0000) 2024-03-06T19:05:33.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:33.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:33.201284+0000) 2024-03-06T19:05:34.195 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:34.199+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:34.201509+0000) 2024-03-06T19:05:35.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:35.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:35.201770+0000) 2024-03-06T19:05:36.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:36.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:36.201993+0000) 2024-03-06T19:05:37.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:37.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:37.202217+0000) 2024-03-06T19:05:38.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:38.200+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:38.202442+0000) 2024-03-06T19:05:39.196 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:39.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:39.202661+0000) 2024-03-06T19:05:40.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:40.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:40.202904+0000) 2024-03-06T19:05:41.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:41.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:41.203128+0000) 2024-03-06T19:05:42.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:42.201+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:42.203355+0000) 2024-03-06T19:05:43.197 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:43.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:43.203621+0000) 2024-03-06T19:05:44.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:44.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:44.203882+0000) 2024-03-06T19:05:45.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:45.202+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:45.204116+0000) 2024-03-06T19:05:46.198 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:46.203+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:46.204370+0000) 2024-03-06T19:05:47.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:47.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:47.204667+0000) 2024-03-06T19:05:48.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:48.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:48.204936+0000) 2024-03-06T19:05:49.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:49.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:49.205105+0000) 2024-03-06T19:05:50.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:50.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:50.205294+0000) 2024-03-06T19:05:51.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:51.204+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:51.205443+0000) 2024-03-06T19:05:52.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:52.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:52.205687+0000) 2024-03-06T19:05:52.753 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:05:52.763 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:05:53.199 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:53.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:53.205915+0000) 2024-03-06T19:05:54.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:54.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:54.206141+0000) 2024-03-06T19:05:55.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:55.205+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:55.206342+0000) 2024-03-06T19:05:56.200 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:56.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:56.206623+0000) 2024-03-06T19:05:57.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:57.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:57.206873+0000) 2024-03-06T19:05:58.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:58.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:58.207145+0000) 2024-03-06T19:05:59.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:05:59.206+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:05:59.207422+0000) 2024-03-06T19:06:00.201 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:00.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:00.207696+0000) 2024-03-06T19:06:01.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:01.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:01.207976+0000) 2024-03-06T19:06:02.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:02.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:02.208241+0000) 2024-03-06T19:06:03.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:03.207+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:03.208479+0000) 2024-03-06T19:06:04.202 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:04.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:04.208684+0000) 2024-03-06T19:06:05.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:05.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:05.208914+0000) 2024-03-06T19:06:06.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:06.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:06.209103+0000) 2024-03-06T19:06:07.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:07.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:07.209342+0000) 2024-03-06T19:06:08.203 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:08.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:08.209605+0000) 2024-03-06T19:06:09.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:09.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:09.209832+0000) 2024-03-06T19:06:10.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:10.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:10.210098+0000) 2024-03-06T19:06:11.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:11.208+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:11.210343+0000) 2024-03-06T19:06:12.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:12.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:12.210618+0000) 2024-03-06T19:06:13.204 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:13.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:13.210790+0000) 2024-03-06T19:06:14.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:14.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:14.211009+0000) 2024-03-06T19:06:15.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:15.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:15.211236+0000) 2024-03-06T19:06:16.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:16.209+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:16.211458+0000) 2024-03-06T19:06:17.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:17.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:17.211690+0000) 2024-03-06T19:06:18.205 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:18.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:18.211922+0000) 2024-03-06T19:06:19.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:19.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:19.212119+0000) 2024-03-06T19:06:20.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:20.210+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:20.212312+0000) 2024-03-06T19:06:21.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:21.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:21.212575+0000) 2024-03-06T19:06:22.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:22.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:22.212738+0000) 2024-03-06T19:06:22.991 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:06:22.999 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:06:23.206 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:23.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:23.212917+0000) 2024-03-06T19:06:24.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:24.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:24.213164+0000) 2024-03-06T19:06:25.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:25.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:25.213312+0000) 2024-03-06T19:06:26.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:26.211+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:26.213480+0000) 2024-03-06T19:06:27.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:27.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:27.213698+0000) 2024-03-06T19:06:28.207 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:28.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:28.213930+0000) 2024-03-06T19:06:29.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:29.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:29.214161+0000) 2024-03-06T19:06:30.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:30.212+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:30.214356+0000) 2024-03-06T19:06:31.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:31.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:31.214600+0000) 2024-03-06T19:06:32.208 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:32.213+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:32.214828+0000) 2024-03-06T19:06:33.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:33.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:33.215054+0000) 2024-03-06T19:06:34.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:34.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:34.215222+0000) 2024-03-06T19:06:35.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:35.214+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:35.215430+0000) 2024-03-06T19:06:36.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:36.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:36.215618+0000) 2024-03-06T19:06:37.209 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:37.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:37.215845+0000) 2024-03-06T19:06:38.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:38.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:38.216079+0000) 2024-03-06T19:06:39.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:39.215+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:39.216309+0000) 2024-03-06T19:06:40.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:40.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:40.216567+0000) 2024-03-06T19:06:41.210 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:41.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:41.216756+0000) 2024-03-06T19:06:42.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:42.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:42.216963+0000) 2024-03-06T19:06:43.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:43.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:43.217193+0000) 2024-03-06T19:06:44.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:44.216+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:44.217453+0000) 2024-03-06T19:06:45.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:45.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:45.217689+0000) 2024-03-06T19:06:46.211 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:46.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:46.217826+0000) 2024-03-06T19:06:47.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:47.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:47.218041+0000) 2024-03-06T19:06:48.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:48.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:48.218229+0000) 2024-03-06T19:06:49.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:49.217+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:49.218451+0000) 2024-03-06T19:06:50.212 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:50.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:50.218728+0000) 2024-03-06T19:06:51.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:51.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:51.218966+0000) 2024-03-06T19:06:52.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:52.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:52.219198+0000) 2024-03-06T19:06:53.195 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:06:53.203 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:06:53.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:53.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:53.219348+0000) 2024-03-06T19:06:54.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:54.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:54.219602+0000) 2024-03-06T19:06:55.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:55.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:55.219744+0000) 2024-03-06T19:06:56.213 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:56.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:56.219980+0000) 2024-03-06T19:06:57.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:57.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:57.220200+0000) 2024-03-06T19:06:58.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:58.218+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:58.220427+0000) 2024-03-06T19:06:59.214 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:06:59.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:06:59.220653+0000) 2024-03-06T19:07:00.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:00.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:00.220878+0000) 2024-03-06T19:07:01.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:01.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:01.221108+0000) 2024-03-06T19:07:02.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:02.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:02.221260+0000) 2024-03-06T19:07:03.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:03.219+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:03.221420+0000) 2024-03-06T19:07:04.215 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:04.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:04.221662+0000) 2024-03-06T19:07:05.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:05.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:05.221861+0000) 2024-03-06T19:07:06.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:06.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:06.222075+0000) 2024-03-06T19:07:07.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:07.220+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:07.222316+0000) 2024-03-06T19:07:08.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:08.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:08.222579+0000) 2024-03-06T19:07:09.216 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:09.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:09.222786+0000) 2024-03-06T19:07:10.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:10.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:10.223031+0000) 2024-03-06T19:07:11.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:11.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:11.223248+0000) 2024-03-06T19:07:12.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:12.221+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:12.223501+0000) 2024-03-06T19:07:13.217 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:13.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:13.223730+0000) 2024-03-06T19:07:14.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:14.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:14.223934+0000) 2024-03-06T19:07:15.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:15.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:15.224143+0000) 2024-03-06T19:07:16.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:16.222+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:16.224382+0000) 2024-03-06T19:07:17.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:17.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:17.224604+0000) 2024-03-06T19:07:18.218 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:18.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:18.224795+0000) 2024-03-06T19:07:19.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:19.223+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:19.225018+0000) 2024-03-06T19:07:20.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:20.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:20.225241+0000) 2024-03-06T19:07:21.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:21.224+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:21.225464+0000) 2024-03-06T19:07:22.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:22.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:22.225705+0000) 2024-03-06T19:07:23.219 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:23.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:23.225874+0000) 2024-03-06T19:07:23.417 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:07:23.425 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:07:24.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:24.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:24.226056+0000) 2024-03-06T19:07:25.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:25.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:25.226332+0000) 2024-03-06T19:07:26.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:26.225+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:26.226497+0000) 2024-03-06T19:07:27.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:27.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:27.226773+0000) 2024-03-06T19:07:28.220 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:28.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:28.227012+0000) 2024-03-06T19:07:29.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:29.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:29.227234+0000) 2024-03-06T19:07:30.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:30.226+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:30.227375+0000) 2024-03-06T19:07:31.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:31.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:31.227655+0000) 2024-03-06T19:07:32.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:32.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:32.227850+0000) 2024-03-06T19:07:33.221 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:33.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:33.228012+0000) 2024-03-06T19:07:34.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:34.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:34.228256+0000) 2024-03-06T19:07:35.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:35.227+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:35.228481+0000) 2024-03-06T19:07:36.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:36.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:36.228682+0000) 2024-03-06T19:07:37.222 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:37.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:37.228885+0000) 2024-03-06T19:07:38.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:38.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:38.229175+0000) 2024-03-06T19:07:39.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:39.228+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:39.229375+0000) 2024-03-06T19:07:40.223 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:40.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:40.229679+0000) 2024-03-06T19:07:41.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:41.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:41.229957+0000) 2024-03-06T19:07:42.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:42.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:42.230163+0000) 2024-03-06T19:07:43.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:43.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:43.230373+0000) 2024-03-06T19:07:44.224 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:44.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:44.230677+0000) 2024-03-06T19:07:45.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:45.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:45.230964+0000) 2024-03-06T19:07:46.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:46.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:46.231246+0000) 2024-03-06T19:07:47.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:47.229+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:47.231514+0000) 2024-03-06T19:07:48.225 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:48.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:48.231829+0000) 2024-03-06T19:07:49.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:49.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:49.232107+0000) 2024-03-06T19:07:50.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:50.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:50.232333+0000) 2024-03-06T19:07:51.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:51.230+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:51.232506+0000) 2024-03-06T19:07:52.226 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:52.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:52.232769+0000) 2024-03-06T19:07:53.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:53.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:53.232939+0000) 2024-03-06T19:07:53.609 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:07:53.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:07:54.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:54.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:54.233184+0000) 2024-03-06T19:07:55.227 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:55.231+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:55.233424+0000) 2024-03-06T19:07:56.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:56.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:56.233653+0000) 2024-03-06T19:07:57.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:57.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:57.233877+0000) 2024-03-06T19:07:58.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:58.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:58.234107+0000) 2024-03-06T19:07:59.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:07:59.232+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:07:59.234341+0000) 2024-03-06T19:08:00.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:00.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:00.234606+0000) 2024-03-06T19:08:01.228 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:01.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:01.234828+0000) 2024-03-06T19:08:02.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:02.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:02.235060+0000) 2024-03-06T19:08:03.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:03.233+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:03.235296+0000) 2024-03-06T19:08:04.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:04.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:04.235526+0000) 2024-03-06T19:08:05.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:05.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:05.235817+0000) 2024-03-06T19:08:06.229 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:06.234+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:06.236002+0000) 2024-03-06T19:08:07.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:07.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:07.236183+0000) 2024-03-06T19:08:08.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:08.235+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:08.236418+0000) 2024-03-06T19:08:09.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:09.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:09.236649+0000) 2024-03-06T19:08:10.230 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:10.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:10.236879+0000) 2024-03-06T19:08:11.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:11.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:11.237103+0000) 2024-03-06T19:08:12.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:12.236+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:12.237331+0000) 2024-03-06T19:08:13.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:13.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:13.237610+0000) 2024-03-06T19:08:14.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:14.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:14.237853+0000) 2024-03-06T19:08:15.231 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:15.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:15.238052+0000) 2024-03-06T19:08:16.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:16.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:16.238289+0000) 2024-03-06T19:08:17.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:17.237+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:17.238517+0000) 2024-03-06T19:08:18.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:18.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:18.238814+0000) 2024-03-06T19:08:19.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:19.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:19.239011+0000) 2024-03-06T19:08:20.232 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:20.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:20.239234+0000) 2024-03-06T19:08:21.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:21.238+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:21.239385+0000) 2024-03-06T19:08:22.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:22.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:22.239649+0000) 2024-03-06T19:08:23.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:23.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:23.239840+0000) 2024-03-06T19:08:23.790 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:08:23.798 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:08:24.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:24.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:24.240068+0000) 2024-03-06T19:08:25.233 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:25.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:25.240289+0000) 2024-03-06T19:08:26.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:26.239+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:26.240477+0000) 2024-03-06T19:08:27.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:27.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:27.240668+0000) 2024-03-06T19:08:28.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:28.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:28.240901+0000) 2024-03-06T19:08:29.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:29.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:29.241170+0000) 2024-03-06T19:08:30.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:30.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:30.241323+0000) 2024-03-06T19:08:31.234 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:31.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:31.241489+0000) 2024-03-06T19:08:32.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:32.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:32.241772+0000) 2024-03-06T19:08:33.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:33.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:33.241936+0000) 2024-03-06T19:08:34.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:34.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:34.242178+0000) 2024-03-06T19:08:35.235 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:35.240+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:35.242411+0000) 2024-03-06T19:08:36.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:36.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:36.242638+0000) 2024-03-06T19:08:37.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:37.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:37.242863+0000) 2024-03-06T19:08:38.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:38.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:38.243066+0000) 2024-03-06T19:08:39.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:39.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:39.243266+0000) 2024-03-06T19:08:40.236 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:40.241+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:40.243440+0000) 2024-03-06T19:08:41.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:41.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:41.243678+0000) 2024-03-06T19:08:42.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:42.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:42.243853+0000) 2024-03-06T19:08:43.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:43.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:43.244064+0000) 2024-03-06T19:08:44.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:44.242+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:44.244290+0000) 2024-03-06T19:08:45.237 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:45.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:45.244523+0000) 2024-03-06T19:08:46.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:46.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:46.244784+0000) 2024-03-06T19:08:47.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:47.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:47.245006+0000) 2024-03-06T19:08:48.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:48.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:48.245245+0000) 2024-03-06T19:08:49.238 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:49.243+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:49.245441+0000) 2024-03-06T19:08:50.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:50.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:50.245701+0000) 2024-03-06T19:08:51.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:51.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:51.245926+0000) 2024-03-06T19:08:52.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:52.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:52.246088+0000) 2024-03-06T19:08:53.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:53.244+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:53.246332+0000) 2024-03-06T19:08:54.003 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:08:54.012 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:08:54.239 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:54.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:54.246615+0000) 2024-03-06T19:08:55.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:55.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:55.246871+0000) 2024-03-06T19:08:56.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:56.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:56.247069+0000) 2024-03-06T19:08:57.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:57.246+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:57.247299+0000) 2024-03-06T19:08:58.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:58.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:58.247576+0000) 2024-03-06T19:08:59.240 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:08:59.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:08:59.247810+0000) 2024-03-06T19:09:00.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:00.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:00.248031+0000) 2024-03-06T19:09:01.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:01.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:01.248259+0000) 2024-03-06T19:09:02.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:02.247+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:02.248485+0000) 2024-03-06T19:09:03.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:03.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:03.248711+0000) 2024-03-06T19:09:04.241 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:04.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:04.248940+0000) 2024-03-06T19:09:05.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:05.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:05.249163+0000) 2024-03-06T19:09:06.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:06.248+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:06.249389+0000) 2024-03-06T19:09:07.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:07.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:07.249604+0000) 2024-03-06T19:09:08.242 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:08.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:08.249811+0000) 2024-03-06T19:09:09.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:09.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:09.249976+0000) 2024-03-06T19:09:10.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:10.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:10.250216+0000) 2024-03-06T19:09:11.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:11.249+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:11.250364+0000) 2024-03-06T19:09:12.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:12.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:12.250608+0000) 2024-03-06T19:09:13.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:13.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:13.250816+0000) 2024-03-06T19:09:14.243 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:14.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:14.250982+0000) 2024-03-06T19:09:15.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:15.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:15.251212+0000) 2024-03-06T19:09:16.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:16.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:16.251370+0000) 2024-03-06T19:09:17.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:17.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:17.251636+0000) 2024-03-06T19:09:18.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:18.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:18.251874+0000) 2024-03-06T19:09:19.244 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:19.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:19.252159+0000) 2024-03-06T19:09:20.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:20.250+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:20.252404+0000) 2024-03-06T19:09:21.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:21.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:21.252637+0000) 2024-03-06T19:09:22.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:22.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:22.252878+0000) 2024-03-06T19:09:23.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:23.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:23.253110+0000) 2024-03-06T19:09:24.166 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:09:24.174 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:09:24.245 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:24.251+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:24.253313+0000) 2024-03-06T19:09:25.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:25.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:25.253597+0000) 2024-03-06T19:09:26.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:26.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:26.253779+0000) 2024-03-06T19:09:27.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:27.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:27.254008+0000) 2024-03-06T19:09:28.246 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:28.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:28.254202+0000) 2024-03-06T19:09:29.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:29.252+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:29.254443+0000) 2024-03-06T19:09:30.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:30.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:30.254710+0000) 2024-03-06T19:09:31.247 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:31.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:31.254935+0000) 2024-03-06T19:09:32.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:32.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:32.255128+0000) 2024-03-06T19:09:33.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:33.253+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:33.255352+0000) 2024-03-06T19:09:34.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:34.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:34.255619+0000) 2024-03-06T19:09:35.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:35.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:35.255837+0000) 2024-03-06T19:09:36.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:36.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:36.256033+0000) 2024-03-06T19:09:37.248 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:37.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:37.256266+0000) 2024-03-06T19:09:38.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:38.254+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:38.256493+0000) 2024-03-06T19:09:39.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:39.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:39.256724+0000) 2024-03-06T19:09:40.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:40.255+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:40.256949+0000) 2024-03-06T19:09:41.249 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:41.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:41.257174+0000) 2024-03-06T19:09:42.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:42.256+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:42.257408+0000) 2024-03-06T19:09:43.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:43.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:43.257641+0000) 2024-03-06T19:09:44.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:44.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:44.257864+0000) 2024-03-06T19:09:45.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:45.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:45.258091+0000) 2024-03-06T19:09:46.250 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:46.257+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:46.258275+0000) 2024-03-06T19:09:47.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:47.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:47.258529+0000) 2024-03-06T19:09:48.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:48.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:48.258753+0000) 2024-03-06T19:09:49.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:49.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:49.258977+0000) 2024-03-06T19:09:50.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:50.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:50.259231+0000) 2024-03-06T19:09:51.251 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:51.258+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:51.259475+0000) 2024-03-06T19:09:52.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:52.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:52.259734+0000) 2024-03-06T19:09:53.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:53.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:53.259954+0000) 2024-03-06T19:09:54.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:54.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:54.260184+0000) 2024-03-06T19:09:54.363 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:09:54.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:09:55.252 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:55.259+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:55.260431+0000) 2024-03-06T19:09:56.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:56.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:56.260669+0000) 2024-03-06T19:09:57.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:57.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:57.260866+0000) 2024-03-06T19:09:58.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:58.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:58.261099+0000) 2024-03-06T19:09:59.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:09:59.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:09:59.261281+0000) 2024-03-06T19:10:00.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:00.260+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:00.261453+0000) 2024-03-06T19:10:01.253 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:01.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:01.261695+0000) 2024-03-06T19:10:02.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:02.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:02.261892+0000) 2024-03-06T19:10:03.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:03.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:03.262069+0000) 2024-03-06T19:10:04.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:04.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:04.262275+0000) 2024-03-06T19:10:05.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:05.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:05.262491+0000) 2024-03-06T19:10:06.254 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:06.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:06.262708+0000) 2024-03-06T19:10:07.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:07.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:07.262979+0000) 2024-03-06T19:10:08.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:08.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:08.263168+0000) 2024-03-06T19:10:09.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:09.261+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:09.263376+0000) 2024-03-06T19:10:10.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:10.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:10.263608+0000) 2024-03-06T19:10:11.255 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:11.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:11.263848+0000) 2024-03-06T19:10:12.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:12.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:12.264058+0000) 2024-03-06T19:10:13.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:13.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:13.264258+0000) 2024-03-06T19:10:14.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:14.262+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:14.264486+0000) 2024-03-06T19:10:15.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:15.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:15.264649+0000) 2024-03-06T19:10:16.256 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:16.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:16.264874+0000) 2024-03-06T19:10:17.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:17.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:17.265057+0000) 2024-03-06T19:10:18.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:18.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:18.265301+0000) 2024-03-06T19:10:19.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:19.263+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:19.265500+0000) 2024-03-06T19:10:20.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:20.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:20.265695+0000) 2024-03-06T19:10:21.257 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:21.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:21.265875+0000) 2024-03-06T19:10:22.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:22.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:22.266101+0000) 2024-03-06T19:10:23.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:23.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:23.266335+0000) 2024-03-06T19:10:24.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:24.264+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:24.266481+0000) 2024-03-06T19:10:24.492 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:10:24.500 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:10:25.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:25.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:25.266710+0000) 2024-03-06T19:10:26.258 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:26.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:26.266957+0000) 2024-03-06T19:10:27.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:27.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:27.267187+0000) 2024-03-06T19:10:28.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:28.265+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:28.267430+0000) 2024-03-06T19:10:29.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:29.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:29.267679+0000) 2024-03-06T19:10:30.259 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:30.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:30.267885+0000) 2024-03-06T19:10:31.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:31.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:31.268124+0000) 2024-03-06T19:10:32.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:32.267+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:32.268327+0000) 2024-03-06T19:10:33.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:33.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:33.268589+0000) 2024-03-06T19:10:34.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:34.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:34.268821+0000) 2024-03-06T19:10:35.260 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:35.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:35.269053+0000) 2024-03-06T19:10:36.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:36.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:36.269234+0000) 2024-03-06T19:10:37.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:37.268+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:37.269423+0000) 2024-03-06T19:10:38.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:38.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:38.269672+0000) 2024-03-06T19:10:39.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:39.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:39.269902+0000) 2024-03-06T19:10:40.261 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:40.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:40.270109+0000) 2024-03-06T19:10:41.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:41.269+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:41.270353+0000) 2024-03-06T19:10:42.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:42.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:42.270562+0000) 2024-03-06T19:10:43.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:43.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:43.270783+0000) 2024-03-06T19:10:44.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:44.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:44.271012+0000) 2024-03-06T19:10:45.262 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:45.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:45.271212+0000) 2024-03-06T19:10:46.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:46.270+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:46.271453+0000) 2024-03-06T19:10:47.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:47.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:47.271686+0000) 2024-03-06T19:10:48.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:48.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:48.271915+0000) 2024-03-06T19:10:49.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:49.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:49.272142+0000) 2024-03-06T19:10:50.263 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:50.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:50.272372+0000) 2024-03-06T19:10:51.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:51.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:51.272637+0000) 2024-03-06T19:10:52.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:52.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:52.272833+0000) 2024-03-06T19:10:53.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:53.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:53.273057+0000) 2024-03-06T19:10:54.264 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:54.271+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:54.273283+0000) 2024-03-06T19:10:54.717 DEBUG:teuthology.orchestra.run.smithi086:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:10:54.725 DEBUG:teuthology.orchestra.run.smithi194:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-06T19:10:55.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:55.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:55.273572+0000) 2024-03-06T19:10:56.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:56.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:56.273802+0000) 2024-03-06T19:10:57.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:57.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:57.273988+0000) 2024-03-06T19:10:58.265 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:58.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:58.274185+0000) 2024-03-06T19:10:59.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:10:59.272+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:10:59.274433+0000) 2024-03-06T19:11:00.266 INFO:tasks.ceph.mds.a.smithi086.stderr:2024-03-06T19:11:00.273+0000 7f3114c00640 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2024-03-06T18:11:00.274708+0000) 2024-03-06T19:11:00.772 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2024-03-06T19:11:00.772 DEBUG:teuthology.task.console_log:Killing console logger for smithi086 2024-03-06T19:11:00.773 DEBUG:teuthology.task.console_log:Killing console logger for smithi194 2024-03-06T19:11:00.773 DEBUG:teuthology.exit:Finished running handlers